def train(file_io, data_spec, api_key, project, **kwargs):
"""
file_io:
The train method must accept a file_io argument. This is a function
that works the same way as the builtin open(), except it reads from
and writes to the root of a special storage location in the cloud
that belongs to the current model version.
data_spec:
An argument we pass in ourself when we initiate the training.
api_key, project:
Optional arguments that are passed in automatically from Model
Hosting for your convenience. The API key is the one that were
used to initiate this training routine through the Model Hosting
HTTP API.
"""
dts = DataTransferService(data_spec, api_key=api_key, project=project)
X = pd.read_csv(dts.get_file("data"))
y = pd.read_csv(dts.get_file("target"))
# Add a feature of constant value 1
X.insert(0, "f0", 1)
# Least squares
coefficients = pd.DataFrame(np.linalg.inv(X.T.dot(X)).dot(X.T).dot(y),
columns=["beta_hat"])
# Persist our result
with file_io("coefficients.csv", "w") as f:
coefficients.to_csv(f, index=False)
def train(file_io, data_spec, api_key, project, **kwargs):
"""
file_io:
The train method must accept a file_io argument. This is a function
that works the same way as the builtin open(), except it reads from
and writes to the root of a special storage location in the cloud
that belongs to the current model version.
data_spec:
An argument we pass in ourself when we initiate the training.
api_key, project:
Optional arguments that are passed in automatically from Model
Hosting for your convenience. The API key is the one that were
used to initiate this training routine through the Model Hosting
HTTP API.
"""
dts = DataTransferService(data_spec, api_key=api_key, project=project)
df = dts.get_dataframe().dropna()
X = df[["temp", "pressure", "rpm"]].values
y = df["production_rate"].values
regressor = RandomForestRegressor(
n_estimators=10,
min_samples_split=100) # We'll mostly use default settings
regressor.fit(X, y)
# Persist our regressor model
with file_io("regressor.pickle", "wb") as f:
pickle.dump(regressor, f)
def test_get_dataframes(self, ts_data_spec_dtos):
data_spec = DataSpec(time_series_data_specs=ts_data_spec_dtos)
service = DataTransferService(data_spec)
dataframes = service.get_dataframes()
assert isinstance(dataframes.get("ds1"), pd.DataFrame)
assert isinstance(dataframes.get("ds2"), pd.DataFrame)
def test_json_dumps_after_used_by_dts(self, ts_data_spec_dtos, files_data_spec_dto):
data_spec = DataSpec(time_series_data_specs=ts_data_spec_dtos, files_data_spec=files_data_spec_dto)
json_repr = data_spec.to_JSON()
dts = DataTransferService(data_spec)
dts.get_dataframes()
json_repr_after_dts = data_spec.to_JSON()
assert json_repr == json_repr_after_dts
def test_get_files(self):
data_spec = DataSpec(files_data_spec=FilesDataSpec(file_ids={"test": 7725800487412823}))
dts = DataTransferService(data_spec)
data = dts.get_file("test")
assert isinstance(data, BytesIO)
assert (
data.getvalue()
== b'import os\n\nfrom cognite.config import configure_session\nfrom cognite.v05 import files\n\nconfigure_session(os.getenv("COGNITE_TEST_API_KEY"), "mltest")\n\n\nres = files.upload_file("test.py", "./test.py")\n\nprint(res)\n'
)
def test_get_dataframes_w_column_mapping(self, time_series_in_cdp):
ts1 = TimeSeries(id=time_series_in_cdp[0], aggregates=["avg"], label="cavg")
ts2 = TimeSeries(id=time_series_in_cdp[0], aggregates=["cv"], label="ccv")
ts3 = TimeSeries(id=time_series_in_cdp[1], aggregates=["avg"], label="sinavg")
tsds = TimeSeriesDataSpec(time_series=[ts1, ts2, ts3], aggregates=["avg"], granularity="1h", start="300d-ago")
dts = DataTransferService(DataSpec([tsds]))
dfs = dts.get_dataframes()
expected = ["timestamp", "cavg", "ccv", "sinavg"]
assert expected == list(dfs["default"].columns.values)
def test_get_dataframes_column_mapping_no_drop_agg_suffix(self, data_spec):
dts = DataTransferService(data_spec, num_of_workers=3)
dfs = dts.get_dataframes(drop_agg_suffix=False)
assert list(dfs["default"].columns.values) == [
"timestamp",
"ts1|average",
"ts1|min",
"ts2|continuousvariance",
"ts3|max",
"ts3|count",
"ts4|stepinterpolation",
]
def test_get_dataframes_column_mapping_drop_agg_suffixes(self, data_spec):
dts = DataTransferService(data_spec, num_of_workers=3)
dfs = dts.get_dataframes(drop_agg_suffix=True)
assert list(dfs["default"].columns.values) == [
"timestamp",
"ts1|average",
"ts1|min",
"ts2",
"ts3|max",
"ts3|count",
"ts4",
]
def predict(self, instance, api_key, project, **kwargs):
"""
instance:
Since we're doing scheduled prediction, this will be a data spec
describing the data we should do prediction on.
Note that it's also possible to take api_key and project in as
optional arguments here the same way as in train().
"""
dts = DataTransferService(instance, api_key=api_key, project=project)
df = dts.get_dataframe().dropna()
X = df[["temp", "pressure", "rpm"]].values
df["production_rate"] = self.regressor.predict(X)
# For scheduled prediction we need to return output on the format:
# {
# "timestamp": [t0, t1, t2, ...],
# "production_rate": [p0, p1, p2, ...]
# }
# And we can call to_dict(orient="list") on our pandas DataFrame to get
# our prediction on that format.
return df[["timestamp", "production_rate"]].to_dict(orient="list")
def test_dict_dto_equal(self, ts_data_spec_dicts, ts_data_spec_dtos):
data_spec_dtos = DataSpec(time_series_data_specs=ts_data_spec_dtos)
data_spec_dicts = DataSpec(time_series_data_specs=ts_data_spec_dicts)
service = DataTransferService(data_spec_dicts)
service2 = DataTransferService(data_spec_dtos)
dataframes_by_dicts = service.get_dataframes()
dataframes_by_dtos = service2.get_dataframes()
for df1, df2 in zip(dataframes_by_dtos.values(),
dataframes_by_dicts.values()):
pd.testing.assert_frame_equal(df1, df2)
def main():
configure_session(api_key=os.getenv("COGNITE_API_KEY"),
project="akerbp",
debug=True)
tags_d03 = []
tags_d02 = []
for root, subdirs, files in os.walk("../tags"):
for file in files:
if file in ("well_tags.csv", "routing.csv", "output.csv",
"riser_tags.csv", "template_tags.csv"):
with open(os.path.join(root, file)) as f:
df = pd.read_csv(f)
placements = ["T3 WGM", "Template", "Riser"]
placements_d03 = ["WellD03"] + placements
placements_d02 = ["WellD02"] + placements
df = df[~df["tag"].isin(EXCLUDE_TAGS)]
tags_d03.append(df[df["placement"].isin(placements_d03)])
tags_d02.append(df[df["placement"].isin(placements_d02)])
tags_d02_concat = pd.concat(tags_d02, ignore_index=True)
tags_d03_concat = pd.concat(tags_d03, ignore_index=True)
tags_d02_concat = tags_d02_concat.drop_duplicates(subset="tag")
tags_d03_concat = tags_d03_concat.drop_duplicates(subset="tag")
d02_input_time_series = []
d03_input_time_series = []
for tag in tags_d02_concat["tag"]:
aggregate = "step" if ("ESV" in tag or "18HV" in tag) else "avg"
missing_data_strategy = "ffill" if (
"ESV" in tag or "18HV" in tag) else "linearInterpolation"
ts = TimeSeries(name=tag,
missing_data_strategy=missing_data_strategy,
aggregates=[aggregate])
d02_input_time_series.append(ts)
for tag in tags_d03_concat["tag"]:
aggregate = "step" if ("ESV" in tag or "18HV" in tag) else "avg"
missing_data_strategy = "ffill" if (
"ESV" in tag or "18HV" in tag) else "linearInterpolation"
ts = TimeSeries(name=tag,
missing_data_strategy=missing_data_strategy,
aggregates=[aggregate])
d03_input_time_series.append(ts)
d02_tsds = TimeSeriesDataSpec(
time_series=d02_input_time_series,
aggregates=["avg"],
granularity="10s",
start=int(datetime(2017, 3, 1).timestamp() * 1e3),
label="d2",
)
d03_tsds = TimeSeriesDataSpec(
time_series=d03_input_time_series,
aggregates=["avg"],
granularity="10s",
start=int(datetime(2017, 3, 1).timestamp() * 1e3),
label="d3",
)
data_spec = DataSpec(time_series_data_specs=[d02_tsds, d03_tsds])
dts = DataTransferService(data_spec, num_of_processes=10)
print(data_spec.to_JSON())
df_dict = dts.get_dataframes()
for label, df in df_dict.items():
df.to_csv(f"../data/{label}.csv")
print(df.shape)
def main():
output_columns = [
"SKAP_18FI381-VFlLGas/Y/10sSAMP|average",
"SKAP_18FI381-VFlLH2O/Y/10sSAMP|average",
"SKAP_18FI381-VFlLOil/Y/10sSAMP|average",
]
router = "SKAP_18HV3806/BCH/10sSAMP|stepinterpolation"
one_hour_ago = datetime.now() - timedelta(0, 3600)
last_processed_timestamp = int(one_hour_ago.timestamp() * 1e3)
is_first = True
while True:
d2_inputs = pd.DataFrame([[np.nan, np.nan, np.nan, np.nan, np.nan, np.nan]])
d2_inputs.columns = ["hoho", "blaa", "hgi"] + output_columns
input_has_nans = True
while input_has_nans:
ds = generate_data_spec(last_processed_timestamp)
dts = DataTransferService(data_spec=ds)
while True:
try:
d2_inputs = dts.get_dataframes()["d2"]
break
except:
time.sleep(2)
any_nans_per_column = d2_inputs.drop(output_columns, axis=1).isna().any()
all_nans_per_column = d2_inputs.drop(output_columns, axis=1).isna().all()
print(any_nans_per_column)
print(all_nans_per_column)
if any_nans_per_column.any() and not all_nans_per_column.any():
last_processed_timestamp -= 10000
print(datetime.fromtimestamp(last_processed_timestamp * 1e-3))
time.sleep(2)
input_has_nans = d2_inputs.drop(output_columns, axis=1).isna().any().any()
last_ts = d2_inputs["timestamp"].iloc[-1]
print(d2_inputs[output_columns[0]].values.tolist())
d2_inputs_formatted = (
d2_inputs.drop("timestamp", axis=1).drop(router, axis=1).drop(output_columns, axis=1).values.tolist()
)
timestamps = d2_inputs["timestamp"]
res = models.online_predict(
model_id=3885574571413770, version_id=4299054386152423, instances=[d2_inputs_formatted]
)
predictions = res["predictions"][0]
formatted_predictions = [int(pred[0]) for pred in predictions]
last_processed_timestamp = int(last_ts)
dps = [Datapoint(ts, value) for ts, value in zip(timestamps.values.tolist(), formatted_predictions)]
print([dp.value for dp in dps])
if is_first:
post_datapoints(name="SKAP_18FI381-VFlLGas/Y/10sSAMP_calc_D02_2", datapoints=dps)
is_first = False
else:
for dp in dps:
post_datapoints(name="SKAP_18FI381-VFlLGas/Y/10sSAMP_calc_D02_2", datapoints=[dp])
time.sleep(5)
def test_get_timeseries_name(self, data_spec):
dts = DataTransferService(data_spec, num_of_workers=3)
for ts_label in ["ts1", "ts2", "ts3", "ts4"]:
assert dts.get_time_series_name(ts_label) == TEST_TS_1_NAME