def test_parquet_target_vector_overwrite(self):
df1 = pd.DataFrame({"name": ["ABC", "DEF", "GHI"], "value": [1, 2, 3]})
fset = fs.FeatureSet(name="fvec-parquet-fset", entities=[fs.Entity("name")])
fs.ingest(fset, df1)
features = ["fvec-parquet-fset.*"]
fvec = fs.FeatureVector("fvec-parquet", features=features)
target = ParquetTarget()
off1 = fs.get_offline_features(fvec, target=target)
dfout1 = pd.read_parquet(target._target_path)
assert (
df1.set_index(keys="name")
.sort_index()
.equals(off1.to_dataframe().sort_index())
)
assert df1.set_index(keys="name").sort_index().equals(dfout1.sort_index())
df2 = pd.DataFrame({"name": ["JKL", "MNO", "PQR"], "value": [4, 5, 6]})
fs.ingest(fset, df2)
off2 = fs.get_offline_features(fvec, target=target)
dfout2 = pd.read_parquet(target._target_path)
assert (
df2.set_index(keys="name")
.sort_index()
.equals(off2.to_dataframe().sort_index())
)
assert df2.set_index(keys="name").sort_index().equals(dfout2.sort_index())
def test_unaggregated_columns(self):
test_base_time = datetime(2020, 12, 1, 17, 33, 15)
data = pd.DataFrame({
"time": [test_base_time, test_base_time - pd.Timedelta(minutes=1)],
"first_name": ["moshe", "yosi"],
"last_name": ["cohen", "levi"],
"bid": [2000, 10],
})
name = f"measurements_{uuid.uuid4()}"
# write to kv
data_set = fs.FeatureSet(name, entities=[Entity("first_name")])
data_set.add_aggregation(
name="bids",
column="bid",
operations=["sum", "max"],
windows="1h",
period="10m",
)
fs.ingest(data_set, data, return_df=True)
features = [f"{name}.bids_sum_1h", f"{name}.last_name"]
vector = fs.FeatureVector("my-vec", features)
svc = fs.get_online_feature_service(vector)
resp = svc.get([{"first_name": "moshe"}])
expected = {"bids_sum_1h": 2000.0, "last_name": "cohen"}
assert resp[0] == expected
svc.close()
def test_serverless_ingest(self):
key = "patient_id"
measurements = fs.FeatureSet("measurements",
entities=[Entity(key)],
timestamp_key="timestamp")
target_path = os.path.relpath(str(self.results_path / "mycsv.csv"))
source = CSVSource("mycsv",
path=os.path.relpath(
str(self.assets_path / "testdata.csv")))
targets = [CSVTarget("mycsv", path=target_path)]
if os.path.exists(target_path):
os.remove(target_path)
fs.ingest(
measurements,
source,
targets,
infer_options=fs.InferOptions.schema() + fs.InferOptions.Stats,
run_config=fs.RunConfig(local=True),
)
assert os.path.exists(target_path), "result file was not generated"
features = sorted(measurements.spec.features.keys())
stats = sorted(measurements.status.stats.keys())
print(features)
print(stats)
stats.remove("timestamp")
assert features == stats, "didnt infer stats for all features"
def test_override_false(self):
df1 = pd.DataFrame({"name": ["ABC", "DEF", "GHI"], "value": [1, 2, 3]})
df2 = pd.DataFrame({"name": ["JKL", "MNO", "PQR"], "value": [4, 5, 6]})
df3 = pd.concat([df1, df2])
fset = fs.FeatureSet(name="override-false", entities=[fs.Entity("name")])
fs.ingest(fset, df1)
features = ["override-false.*"]
fvec = fs.FeatureVector("override-false-vec", features=features)
off1 = fs.get_offline_features(fvec).to_dataframe()
assert df1.set_index(keys="name").sort_index().equals(off1.sort_index())
fs.ingest(fset, df2, overwrite=False)
off2 = fs.get_offline_features(fvec).to_dataframe()
assert df3.set_index(keys="name").sort_index().equals(off2.sort_index())
fs.ingest(fset, df1, targets=[ParquetTarget()])
off1 = fs.get_offline_features(fvec).to_dataframe()
assert df1.set_index(keys="name").sort_index().equals(off1.sort_index())
svc = fs.get_online_feature_service(fvec)
resp = svc.get(entity_rows=[{"name": "PQR"}])
assert resp[0]["value"] == 6
svc.close()
with pytest.raises(mlrun.errors.MLRunInvalidArgumentError):
fs.ingest(fset, df1, targets=[CSVTarget()], overwrite=False)
fset.set_targets(targets=[CSVTarget()])
with pytest.raises(mlrun.errors.MLRunInvalidArgumentError):
fs.ingest(fset, df1, overwrite=False)
def test_read_csv(self):
from storey import ReadCSV, ReduceToDataFrame, build_flow
csv_path = str(self.results_path / _generate_random_name() / ".csv")
targets = [CSVTarget("mycsv", path=csv_path)]
stocks_set = fs.FeatureSet(
"tests", entities=[Entity("ticker", ValueType.STRING)])
fs.ingest(stocks_set,
stocks,
infer_options=fs.InferOptions.default(),
targets=targets)
# reading csv file
controller = build_flow([ReadCSV(csv_path), ReduceToDataFrame()]).run()
termination_result = controller.await_termination()
expected = pd.DataFrame({
0: ["ticker", "MSFT", "GOOG", "AAPL"],
1: ["name", "Microsoft Corporation", "Alphabet Inc", "Apple Inc"],
2: ["exchange", "NASDAQ", "NASDAQ", "NASDAQ"],
})
assert termination_result.equals(
expected), f"{termination_result}\n!=\n{expected}"
os.remove(csv_path)
def test_feature_set_db(self):
name = "stocks_test"
stocks_set = fs.FeatureSet(
name, entities=[Entity("ticker", ValueType.STRING)])
fs.preview(
stocks_set,
stocks,
)
stocks_set.save()
db = mlrun.get_run_db()
sets = db.list_feature_sets(self.project_name, name)
assert len(sets) == 1, "bad number of results"
feature_set = fs.get_feature_set(name, self.project_name)
assert feature_set.metadata.name == name, "bad feature set response"
fs.ingest(stocks_set, stocks)
with pytest.raises(mlrun.errors.MLRunPreconditionFailedError):
fs.delete_feature_set(name, self.project_name)
stocks_set.purge_targets()
fs.delete_feature_set(name, self.project_name)
sets = db.list_feature_sets(self.project_name, name)
assert not sets, "Feature set should be deleted"
def test_purge(self):
key = "patient_id"
fset = fs.FeatureSet("purge",
entities=[Entity(key)],
timestamp_key="timestamp")
path = os.path.relpath(str(self.assets_path / "testdata.csv"))
source = CSVSource(
"mycsv",
path=path,
time_field="timestamp",
)
targets = [
CSVTarget(),
CSVTarget(name="specified-path",
path="v3io:///bigdata/csv-purge-test.csv"),
ParquetTarget(partitioned=True, partition_cols=["timestamp"]),
NoSqlTarget(),
]
fset.set_targets(
targets=targets,
with_defaults=False,
)
fs.ingest(fset, source)
verify_purge(fset, targets)
fs.ingest(fset, source)
targets_to_purge = targets[:-1]
verify_purge(fset, targets_to_purge)
def test_ingest_dataframe_index(self):
orig_df = pd.DataFrame([{"x", "y"}])
orig_df.index.name = "idx"
fset = fs.FeatureSet("myfset", entities=[Entity("idx")])
fs.ingest(
fset, orig_df, [ParquetTarget()], infer_options=fs.InferOptions.default()
)
def test_check_permissions():
data = pd.DataFrame({
"time_stamp": [
pd.Timestamp("2021-06-09 09:30:06.008"),
pd.Timestamp("2021-06-09 10:29:07.009"),
pd.Timestamp("2021-06-09 09:29:08.010"),
],
"data": [10, 20, 30],
"string": ["ab", "cd", "ef"],
})
data_set1 = fs.FeatureSet("fs1", entities=[Entity("string")])
mlrun.db.FileRunDB.verify_authorization = unittest.mock.Mock(
side_effect=mlrun.errors.MLRunAccessDeniedError(""))
try:
fs.preview(
data_set1,
data,
entity_columns=[Entity("string")],
timestamp_key="time_stamp",
)
assert False
except mlrun.errors.MLRunAccessDeniedError:
pass
try:
fs.ingest(data_set1, data, infer_options=fs.InferOptions.default())
assert False
except mlrun.errors.MLRunAccessDeniedError:
pass
features = ["fs1.*"]
feature_vector = fs.FeatureVector("test", features)
try:
fs.get_offline_features(feature_vector,
entity_timestamp_column="time_stamp")
assert False
except mlrun.errors.MLRunAccessDeniedError:
pass
try:
fs.get_online_feature_service(feature_vector)
assert False
except mlrun.errors.MLRunAccessDeniedError:
pass
try:
fs.deploy_ingestion_service(featureset=data_set1)
assert False
except mlrun.errors.MLRunAccessDeniedError:
pass
try:
data_set1.purge_targets()
assert False
except mlrun.errors.MLRunAccessDeniedError:
pass
def verify_target_list_fail(targets, with_defaults=None):
feature_set = fs.FeatureSet(name="target-list-fail", entities=[fs.Entity("ticker")])
with pytest.raises(mlrun.errors.MLRunInvalidArgumentError):
if with_defaults:
feature_set.set_targets(targets=targets, with_defaults=with_defaults)
else:
feature_set.set_targets(targets=targets)
with pytest.raises(mlrun.errors.MLRunInvalidArgumentError):
fs.ingest(feature_set, quotes, targets=targets)
def test_multiple_entities(self):
name = f"measurements_{uuid.uuid4()}"
current_time = pd.Timestamp.now()
data = pd.DataFrame(
{
"time": [
current_time,
current_time - pd.Timedelta(minutes=1),
current_time - pd.Timedelta(minutes=2),
current_time - pd.Timedelta(minutes=3),
current_time - pd.Timedelta(minutes=4),
current_time - pd.Timedelta(minutes=5),
],
"first_name": ["moshe", "yosi", "yosi", "yosi", "moshe", "yosi"],
"last_name": ["cohen", "levi", "levi", "levi", "cohen", "levi"],
"bid": [2000, 10, 11, 12, 2500, 14],
}
)
# write to kv
data_set = fs.FeatureSet(
name, entities=[Entity("first_name"), Entity("last_name")]
)
data_set.add_aggregation(
name="bids",
column="bid",
operations=["sum", "max"],
windows="1h",
period="10m",
emit_policy=EmitAfterMaxEvent(1),
)
fs.infer_metadata(
data_set,
data, # source
entity_columns=["first_name", "last_name"],
timestamp_key="time",
options=fs.InferOptions.default(),
)
data_set.plot(
str(self.results_path / "pipe.png"), rankdir="LR", with_targets=True
)
fs.ingest(data_set, data, return_df=True)
features = [
f"{name}.bids_sum_1h",
]
vector = fs.FeatureVector("my-vec", features)
svc = fs.get_online_feature_service(vector)
resp = svc.get([{"first_name": "yosi", "last_name": "levi"}])
assert resp[0]["bids_sum_1h"] == 47.0
svc.close()
def test_ingest_twice_with_nulls(self):
name = f"test_ingest_twice_with_nulls_{uuid.uuid4()}"
key = "key"
measurements = fs.FeatureSet(
name, entities=[Entity(key)], timestamp_key="my_time"
)
columns = [key, "my_string", "my_time"]
df = pd.DataFrame(
[["mykey1", "hello", pd.Timestamp("2019-01-26 14:52:37")]], columns=columns
)
df.set_index("my_string")
source = DataFrameSource(df)
measurements.set_targets(
targets=[ParquetTarget(partitioned=True)], with_defaults=False,
)
resp1 = fs.ingest(measurements, source)
assert resp1.to_dict() == {
"my_string": {"mykey1": "hello"},
"my_time": {"mykey1": pd.Timestamp("2019-01-26 14:52:37")},
}
features = [
f"{name}.*",
]
vector = fs.FeatureVector("myvector", features)
resp2 = fs.get_offline_features(vector)
resp2 = resp2.to_dataframe()
assert resp2.to_dict() == {"my_string": {"mykey1": "hello"}}
measurements = fs.FeatureSet(
name, entities=[Entity(key)], timestamp_key="my_time"
)
columns = [key, "my_string", "my_time"]
df = pd.DataFrame(
[["mykey2", None, pd.Timestamp("2019-01-26 14:52:37")]], columns=columns
)
df.set_index("my_string")
source = DataFrameSource(df)
measurements.set_targets(
targets=[ParquetTarget(partitioned=True)], with_defaults=False,
)
resp1 = fs.ingest(measurements, source, overwrite=False)
assert resp1.to_dict() == {
"my_string": {"mykey2": None},
"my_time": {"mykey2": pd.Timestamp("2019-01-26 14:52:37")},
}
features = [
f"{name}.*",
]
vector = fs.FeatureVector("myvector", features)
resp2 = fs.get_offline_features(vector)
resp2 = resp2.to_dataframe()
assert resp2.to_dict() == {"my_string": {"mykey1": "hello", "mykey2": None}}
def test_overwrite_single_parquet_file(self):
df1 = pd.DataFrame({"name": ["ABC", "DEF", "GHI"], "value": [1, 2, 3]})
df2 = pd.DataFrame({"name": ["JKL", "MNO", "PQR"], "value": [4, 5, 6]})
targets = [
ParquetTarget(path="v3io:///bigdata/overwrite-pq-spec/my.parquet")
]
fset = fs.FeatureSet(name="overwrite-pq-spec-path",
entities=[fs.Entity("name")])
fs.ingest(fset, df1, targets=targets)
with pytest.raises(mlrun.errors.MLRunInvalidArgumentError):
fs.ingest(fset, df2, targets=targets, overwrite=False)
def test_non_partitioned_target_in_dir(self):
source = CSVSource(
"mycsv", path=os.path.relpath(str(self.assets_path / "testdata.csv"))
)
path = str(self.results_path / _generate_random_name())
target = ParquetTarget(path=path)
fset = fs.FeatureSet(
name="test", entities=[Entity("patient_id")], timestamp_key="timestamp"
)
fs.ingest(fset, source, targets=[target])
list_files = os.listdir(path)
assert len(list_files) == 1 and not os.path.isdir(path + "/" + list_files[0])
os.remove(path + "/" + list_files[0])
def test_overwrite(self):
df1 = pd.DataFrame({"name": ["ABC", "DEF", "GHI"], "value": [1, 2, 3]})
df2 = pd.DataFrame({"name": ["JKL", "MNO", "PQR"], "value": [4, 5, 6]})
fset = fs.FeatureSet(name="overwrite-fs", entities=[fs.Entity("name")])
fs.ingest(fset, df1, targets=[CSVTarget(), ParquetTarget(), NoSqlTarget()])
features = ["overwrite-fs.*"]
fvec = fs.FeatureVector("overwrite-vec", features=features)
csv_path = fset.get_target_path(name="csv")
csv_df = pd.read_csv(csv_path)
assert (
df1.set_index(keys="name")
.sort_index()
.equals(csv_df.set_index(keys="name").sort_index())
)
parquet_path = fset.get_target_path(name="parquet")
parquet_df = pd.read_parquet(parquet_path)
assert df1.set_index(keys="name").sort_index().equals(parquet_df.sort_index())
svc = fs.get_online_feature_service(fvec)
resp = svc.get(entity_rows=[{"name": "GHI"}])
assert resp[0]["value"] == 3
svc.close()
fs.ingest(fset, df2)
csv_path = fset.get_target_path(name="csv")
csv_df = pd.read_csv(csv_path)
assert (
df1.set_index(keys="name")
.sort_index()
.equals(csv_df.set_index(keys="name").sort_index())
)
parquet_path = fset.get_target_path(name="parquet")
parquet_df = pd.read_parquet(parquet_path)
assert df2.set_index(keys="name").sort_index().equals(parquet_df.sort_index())
svc = fs.get_online_feature_service(fvec)
resp = svc.get(entity_rows=[{"name": "GHI"}])
assert resp[0] is None
resp = svc.get(entity_rows=[{"name": "PQR"}])
assert resp[0]["value"] == 6
svc.close()
def test_csv_time_columns(self):
df = pd.DataFrame(
{
"key": ["key1", "key2"],
"time_stamp": [
datetime(2020, 11, 1, 17, 33, 15),
datetime(2020, 10, 1, 17, 33, 15),
],
"another_time_column": [
datetime(2020, 9, 1, 17, 33, 15),
datetime(2020, 8, 1, 17, 33, 15),
],
}
)
csv_path = "/tmp/multiple_time_columns.csv"
df.to_csv(path_or_buf=csv_path, index=False)
source = CSVSource(
path=csv_path, time_field="time_stamp", parse_dates=["another_time_column"]
)
measurements = fs.FeatureSet(
"fs", entities=[Entity("key")], timestamp_key="time_stamp"
)
try:
resp = fs.ingest(measurements, source)
df.set_index("key", inplace=True)
assert_frame_equal(df, resp)
finally:
os.remove(csv_path)
def test_none_value(self):
data = pd.DataFrame(
{"first_name": ["moshe", "yossi"], "bid": [2000, 10], "bool": [True, None]}
)
# write to kv
data_set = fs.FeatureSet("tests2", entities=[Entity("first_name")])
fs.ingest(data_set, data, return_df=True)
features = ["tests2.*"]
vector = fs.FeatureVector("my-vec", features)
svc = fs.get_online_feature_service(vector)
resp = svc.get([{"first_name": "yossi"}])
assert resp[0] == {"bid": 10, "bool": None}
svc.close()
def _ingest_quotes_featureset(self):
quotes_set = FeatureSet("stock-quotes", entities=[Entity("ticker")])
flow = quotes_set.graph
flow.to("MyMap", multiplier=3).to("storey.Extend",
_fn="({'z': event['bid'] * 77})").to(
"storey.Filter",
"filter",
_fn="(event['bid'] > 51.92)").to(
FeaturesetValidator())
quotes_set.add_aggregation("asks", "ask", ["sum", "max"], ["1h", "5h"],
"10m")
quotes_set.add_aggregation("bids", "bid", ["min", "max"], ["1h"],
"10m")
df = fs.infer_metadata(
quotes_set,
quotes,
entity_columns=["ticker"],
timestamp_key="time",
options=fs.InferOptions.default(),
)
self._logger.info(f"quotes spec: {quotes_set.spec.to_yaml()}")
assert df["zz"].mean() == 9, "map didnt set the zz column properly"
quotes_set["bid"].validator = MinMaxValidator(min=52, severity="info")
quotes_set.plot(str(self.results_path / "pipe.png"),
rankdir="LR",
with_targets=True)
df = fs.ingest(quotes_set, quotes, return_df=True)
self._logger.info(f"output df:\n{df}")
assert quotes_set.status.stats.get("asks_sum_1h"), "stats not created"
def test_basic_remote_spark_ingest(self):
key = "patient_id"
measurements = fs.FeatureSet(
"measurements",
entities=[fs.Entity(key)],
timestamp_key="timestamp",
engine="spark",
)
source = ParquetSource("myparquet",
path=self.get_remote_pq_source_path())
fs.ingest(
measurements,
source,
return_df=True,
spark_context=self.spark_service,
run_config=fs.RunConfig(local=False),
)
def prepare_feature_set(name: str, entity: str, data: pd.DataFrame, timestamp_key=None):
df_source = mlrun.datastore.sources.DataFrameSource(data, entity, timestamp_key)
feature_set = fs.FeatureSet(
name, entities=[fs.Entity(entity)], timestamp_key=timestamp_key
)
feature_set.set_targets()
df = fs.ingest(feature_set, df_source, infer_options=fs.InferOptions.default())
return feature_set, df
def test_error_flow(self):
df = pd.DataFrame({
"name": ["Jean", "Jacques", "Pierre"],
"last_name": ["Dubois", "Dupont", "Lavigne"],
})
measurements = fs.FeatureSet(
"measurements",
entities=[fs.Entity("name")],
engine="spark",
)
with pytest.raises(mlrun.errors.MLRunInvalidArgumentError):
fs.ingest(
measurements,
df,
return_df=True,
spark_context=self.spark_service,
run_config=fs.RunConfig(local=False),
)
def test_overwrite_specified_nosql_path(self):
df1 = pd.DataFrame({"name": ["ABC", "DEF", "GHI"], "value": [1, 2, 3]})
df2 = pd.DataFrame({"name": ["JKL", "MNO", "PQR"], "value": [4, 5, 6]})
targets = [NoSqlTarget(path="v3io:///bigdata/overwrite-spec")]
fset = fs.FeatureSet(name="overwrite-spec-path",
entities=[fs.Entity("name")])
features = ["overwrite-spec-path.*"]
fvec = fs.FeatureVector("overwrite-spec-path-fvec", features=features)
fs.ingest(fset, df1, targets=targets)
fs.ingest(fset, df2, targets=targets)
svc = fs.get_online_feature_service(fvec)
resp = svc.get(entity_rows=[{"name": "PQR"}])
assert resp[0]["value"] == 6
resp = svc.get(entity_rows=[{"name": "ABC"}])
assert resp[0] is None
svc.close()
def test_ingest_with_timestamp(self):
key = "patient_id"
measurements = fs.FeatureSet("measurements",
entities=[Entity(key)],
timestamp_key="timestamp")
source = CSVSource(
"mycsv",
path=os.path.relpath(str(self.assets_path / "testdata.csv")),
time_field="timestamp",
)
resp = fs.ingest(measurements, source)
assert resp["timestamp"].head(
n=1)[0] == datetime.fromisoformat("2020-12-01 17:24:15.906352")
def test_offline_features_filter_non_partitioned(self):
data = pd.DataFrame({
"time_stamp": [
pd.Timestamp("2021-06-09 09:30:06.008"),
pd.Timestamp("2021-06-09 10:29:07.009"),
pd.Timestamp("2021-06-09 09:29:08.010"),
],
"data": [10, 20, 30],
"string": ["ab", "cd", "ef"],
})
data_set1 = fs.FeatureSet("fs1", entities=[Entity("string")])
fs.ingest(data_set1, data, infer_options=fs.InferOptions.default())
features = ["fs1.*"]
vector = fs.FeatureVector("vector", features)
resp = fs.get_offline_features(
vector,
entity_timestamp_column="time_stamp",
start_time=datetime(2021, 6, 9, 9, 30),
end_time=datetime(2021, 6, 9, 10, 30),
)
assert len(resp.to_dataframe()) == 2
def handler(context, event):
context.logger.debug(event.body)
event_body = json.loads(event.body)
options = fs.InferOptions.Null
if context.need_to_infer:
options = fs.InferOptions.default()
context.need_to_infer = False
events = []
if "headers" in event_body and "values" in event_body:
for values in event_body["values"]:
events.append(
{k: v
for k, v in zip(event_body["headers"], values)})
else:
events.append(event_body)
for enriched in map(enrich_even_details, events):
if enriched is not None:
enriched[TIMESTAMP] = datetime.strptime(enriched["when"],
ISO_8061_UTC)
if enriched.get("class"):
# class is illegal column name in pandas df
enriched[MODEL_CLASS] = enriched["class"]
del enriched["class"]
fs.ingest(
context.fset,
pd.DataFrame({k: [v]
for k, v in enriched.items()}),
infer_options=options,
return_df=False,
overwrite=False,
)
else:
pass
def test_split_graph(self):
quotes_set = fs.FeatureSet("stock-quotes",
entities=[fs.Entity("ticker")])
quotes_set.graph.to("MyMap", "somemap1", field="multi1",
multiplier=3).to(
"storey.Extend",
_fn="({'extra': event['bid'] * 77})").to(
"storey.Filter",
"filter",
_fn="(event['bid'] > 70)").to(
FeaturesetValidator())
side_step_name = "side-step"
quotes_set.graph.to("storey.Extend",
name=side_step_name,
_fn="({'extra2': event['bid'] * 17})")
with pytest.raises(mlrun.errors.MLRunPreconditionFailedError):
fs.infer_metadata(quotes_set, quotes)
non_default_target_name = "side-target"
quotes_set.set_targets(
targets=[
CSVTarget(name=non_default_target_name,
after_state=side_step_name)
],
default_final_state="FeaturesetValidator",
)
quotes_set.plot(with_targets=True)
inf_out = fs.infer_metadata(quotes_set, quotes)
ing_out = fs.ingest(quotes_set, quotes, return_df=True)
default_file_path = quotes_set.get_target_path(TargetTypes.parquet)
side_file_path = quotes_set.get_target_path(non_default_target_name)
side_file_out = pd.read_csv(side_file_path)
default_file_out = pd.read_parquet(default_file_path)
self._split_graph_expected_default.set_index("ticker", inplace=True)
assert all(
self._split_graph_expected_default == default_file_out.round(2))
assert all(self._split_graph_expected_default == ing_out.round(2))
assert all(self._split_graph_expected_default == inf_out.round(2))
assert all(
self._split_graph_expected_side.sort_index(
axis=1) == side_file_out.sort_index(axis=1).round(2))
def _ingest_stocks_featureset(self):
stocks_set = fs.FeatureSet(
"stocks", entities=[Entity("ticker", ValueType.STRING)])
df = fs.ingest(stocks_set,
stocks,
infer_options=fs.InferOptions.default())
self._logger.info(f"output df:\n{df}")
stocks_set["name"].description = "some name"
self._logger.info(f"stocks spec: {stocks_set.to_yaml()}")
assert (stocks_set.spec.features["name"].description == "some name"
), "description was not set"
assert len(df) == len(stocks), "dataframe size doesnt match"
assert stocks_set.status.stats["exchange"], "stats not created"
def test_time_with_timezone(self):
data = pd.DataFrame({
"time": [
datetime(2021, 6, 30, 15, 9, 35, tzinfo=timezone.utc),
datetime(2021, 6, 30, 15, 9, 35, tzinfo=timezone.utc),
],
"first_name": ["katya", "dina"],
"bid": [2000, 10],
})
data_set = fs.FeatureSet("fs4", entities=[Entity("first_name")])
df = fs.ingest(data_set, data, return_df=True)
data.set_index("first_name", inplace=True)
assert_frame_equal(df, data)
def test_sync_pipeline(self):
stocks_set = fs.FeatureSet(
"stocks-sync",
entities=[Entity("ticker", ValueType.STRING)],
engine="pandas",
)
stocks_set.graph.to(name="s1", handler="myfunc1")
df = fs.ingest(stocks_set, stocks)
self._logger.info(f"output df:\n{df}")
features = list(stocks_set.spec.features.keys())
assert len(features) == 1, "wrong num of features"
assert "exchange" not in features, "field was not dropped"
assert len(df) == len(stocks), "dataframe size doesnt match"
def test_filtering_parquet_by_time(self):
key = "patient_id"
measurements = fs.FeatureSet(
"measurements", entities=[Entity(key)], timestamp_key="timestamp"
)
source = ParquetSource(
"myparquet",
path=os.path.relpath(str(self.assets_path / "testdata.parquet")),
time_field="timestamp",
start_time=datetime(2020, 12, 1, 17, 33, 15),
end_time="2020-12-01 17:33:16",
)
resp = fs.ingest(measurements, source, return_df=True,)
assert len(resp) == 10