def test_check_permissions(): data = pd.DataFrame({ "time_stamp": [ pd.Timestamp("2021-06-09 09:30:06.008"), pd.Timestamp("2021-06-09 10:29:07.009"), pd.Timestamp("2021-06-09 09:29:08.010"), ], "data": [10, 20, 30], "string": ["ab", "cd", "ef"], }) data_set1 = fs.FeatureSet("fs1", entities=[Entity("string")]) mlrun.db.FileRunDB.verify_authorization = unittest.mock.Mock( side_effect=mlrun.errors.MLRunAccessDeniedError("")) try: fs.preview( data_set1, data, entity_columns=[Entity("string")], timestamp_key="time_stamp", ) assert False except mlrun.errors.MLRunAccessDeniedError: pass try: fs.ingest(data_set1, data, infer_options=fs.InferOptions.default()) assert False except mlrun.errors.MLRunAccessDeniedError: pass features = ["fs1.*"] feature_vector = fs.FeatureVector("test", features) try: fs.get_offline_features(feature_vector, entity_timestamp_column="time_stamp") assert False except mlrun.errors.MLRunAccessDeniedError: pass try: fs.get_online_feature_service(feature_vector) assert False except mlrun.errors.MLRunAccessDeniedError: pass try: fs.deploy_ingestion_service(featureset=data_set1) assert False except mlrun.errors.MLRunAccessDeniedError: pass try: data_set1.purge_targets() assert False except mlrun.errors.MLRunAccessDeniedError: pass
def test_multiple_entities(self): name = f"measurements_{uuid.uuid4()}" current_time = pd.Timestamp.now() data = pd.DataFrame( { "time": [ current_time, current_time - pd.Timedelta(minutes=1), current_time - pd.Timedelta(minutes=2), current_time - pd.Timedelta(minutes=3), current_time - pd.Timedelta(minutes=4), current_time - pd.Timedelta(minutes=5), ], "first_name": ["moshe", "yosi", "yosi", "yosi", "moshe", "yosi"], "last_name": ["cohen", "levi", "levi", "levi", "cohen", "levi"], "bid": [2000, 10, 11, 12, 2500, 14], } ) # write to kv data_set = fs.FeatureSet( name, entities=[Entity("first_name"), Entity("last_name")] ) data_set.add_aggregation( name="bids", column="bid", operations=["sum", "max"], windows="1h", period="10m", emit_policy=EmitAfterMaxEvent(1), ) fs.infer_metadata( data_set, data, # source entity_columns=["first_name", "last_name"], timestamp_key="time", options=fs.InferOptions.default(), ) data_set.plot( str(self.results_path / "pipe.png"), rankdir="LR", with_targets=True ) fs.ingest(data_set, data, return_df=True) features = [ f"{name}.bids_sum_1h", ] vector = fs.FeatureVector("my-vec", features) svc = fs.get_online_feature_service(vector) resp = svc.get([{"first_name": "yosi", "last_name": "levi"}]) assert resp[0]["bids_sum_1h"] == 47.0 svc.close()
def test_ingest_twice_with_nulls(self): name = f"test_ingest_twice_with_nulls_{uuid.uuid4()}" key = "key" measurements = fs.FeatureSet( name, entities=[Entity(key)], timestamp_key="my_time" ) columns = [key, "my_string", "my_time"] df = pd.DataFrame( [["mykey1", "hello", pd.Timestamp("2019-01-26 14:52:37")]], columns=columns ) df.set_index("my_string") source = DataFrameSource(df) measurements.set_targets( targets=[ParquetTarget(partitioned=True)], with_defaults=False, ) resp1 = fs.ingest(measurements, source) assert resp1.to_dict() == { "my_string": {"mykey1": "hello"}, "my_time": {"mykey1": pd.Timestamp("2019-01-26 14:52:37")}, } features = [ f"{name}.*", ] vector = fs.FeatureVector("myvector", features) resp2 = fs.get_offline_features(vector) resp2 = resp2.to_dataframe() assert resp2.to_dict() == {"my_string": {"mykey1": "hello"}} measurements = fs.FeatureSet( name, entities=[Entity(key)], timestamp_key="my_time" ) columns = [key, "my_string", "my_time"] df = pd.DataFrame( [["mykey2", None, pd.Timestamp("2019-01-26 14:52:37")]], columns=columns ) df.set_index("my_string") source = DataFrameSource(df) measurements.set_targets( targets=[ParquetTarget(partitioned=True)], with_defaults=False, ) resp1 = fs.ingest(measurements, source, overwrite=False) assert resp1.to_dict() == { "my_string": {"mykey2": None}, "my_time": {"mykey2": pd.Timestamp("2019-01-26 14:52:37")}, } features = [ f"{name}.*", ] vector = fs.FeatureVector("myvector", features) resp2 = fs.get_offline_features(vector) resp2 = resp2.to_dataframe() assert resp2.to_dict() == {"my_string": {"mykey1": "hello", "mykey2": None}}
def test_feature_set_db(self): name = "stocks_test" stocks_set = fs.FeatureSet( name, entities=[Entity("ticker", ValueType.STRING)]) fs.preview( stocks_set, stocks, ) stocks_set.save() db = mlrun.get_run_db() sets = db.list_feature_sets(self.project_name, name) assert len(sets) == 1, "bad number of results" feature_set = fs.get_feature_set(name, self.project_name) assert feature_set.metadata.name == name, "bad feature set response" fs.ingest(stocks_set, stocks) with pytest.raises(mlrun.errors.MLRunPreconditionFailedError): fs.delete_feature_set(name, self.project_name) stocks_set.purge_targets() fs.delete_feature_set(name, self.project_name) sets = db.list_feature_sets(self.project_name, name) assert not sets, "Feature set should be deleted"
def _ingest_quotes_featureset(self): quotes_set = FeatureSet("stock-quotes", entities=[Entity("ticker")]) flow = quotes_set.graph flow.to("MyMap", multiplier=3).to("storey.Extend", _fn="({'z': event['bid'] * 77})").to( "storey.Filter", "filter", _fn="(event['bid'] > 51.92)").to( FeaturesetValidator()) quotes_set.add_aggregation("asks", "ask", ["sum", "max"], ["1h", "5h"], "10m") quotes_set.add_aggregation("bids", "bid", ["min", "max"], ["1h"], "10m") df = fs.infer_metadata( quotes_set, quotes, entity_columns=["ticker"], timestamp_key="time", options=fs.InferOptions.default(), ) self._logger.info(f"quotes spec: {quotes_set.spec.to_yaml()}") assert df["zz"].mean() == 9, "map didnt set the zz column properly" quotes_set["bid"].validator = MinMaxValidator(min=52, severity="info") quotes_set.plot(str(self.results_path / "pipe.png"), rankdir="LR", with_targets=True) df = fs.ingest(quotes_set, quotes, return_df=True) self._logger.info(f"output df:\n{df}") assert quotes_set.status.stats.get("asks_sum_1h"), "stats not created"
def test_purge(self): key = "patient_id" fset = fs.FeatureSet("purge", entities=[Entity(key)], timestamp_key="timestamp") path = os.path.relpath(str(self.assets_path / "testdata.csv")) source = CSVSource( "mycsv", path=path, time_field="timestamp", ) targets = [ CSVTarget(), CSVTarget(name="specified-path", path="v3io:///bigdata/csv-purge-test.csv"), ParquetTarget(partitioned=True, partition_cols=["timestamp"]), NoSqlTarget(), ] fset.set_targets( targets=targets, with_defaults=False, ) fs.ingest(fset, source) verify_purge(fset, targets) fs.ingest(fset, source) targets_to_purge = targets[:-1] verify_purge(fset, targets_to_purge)
def test_serverless_ingest(self): key = "patient_id" measurements = fs.FeatureSet("measurements", entities=[Entity(key)], timestamp_key="timestamp") target_path = os.path.relpath(str(self.results_path / "mycsv.csv")) source = CSVSource("mycsv", path=os.path.relpath( str(self.assets_path / "testdata.csv"))) targets = [CSVTarget("mycsv", path=target_path)] if os.path.exists(target_path): os.remove(target_path) fs.ingest( measurements, source, targets, infer_options=fs.InferOptions.schema() + fs.InferOptions.Stats, run_config=fs.RunConfig(local=True), ) assert os.path.exists(target_path), "result file was not generated" features = sorted(measurements.spec.features.keys()) stats = sorted(measurements.status.stats.keys()) print(features) print(stats) stats.remove("timestamp") assert features == stats, "didnt infer stats for all features"
def test_csv_time_columns(self): df = pd.DataFrame( { "key": ["key1", "key2"], "time_stamp": [ datetime(2020, 11, 1, 17, 33, 15), datetime(2020, 10, 1, 17, 33, 15), ], "another_time_column": [ datetime(2020, 9, 1, 17, 33, 15), datetime(2020, 8, 1, 17, 33, 15), ], } ) csv_path = "/tmp/multiple_time_columns.csv" df.to_csv(path_or_buf=csv_path, index=False) source = CSVSource( path=csv_path, time_field="time_stamp", parse_dates=["another_time_column"] ) measurements = fs.FeatureSet( "fs", entities=[Entity("key")], timestamp_key="time_stamp" ) try: resp = fs.ingest(measurements, source) df.set_index("key", inplace=True) assert_frame_equal(df, resp) finally: os.remove(csv_path)
def test_read_csv(self): from storey import ReadCSV, ReduceToDataFrame, build_flow csv_path = str(self.results_path / _generate_random_name() / ".csv") targets = [CSVTarget("mycsv", path=csv_path)] stocks_set = fs.FeatureSet( "tests", entities=[Entity("ticker", ValueType.STRING)]) fs.ingest(stocks_set, stocks, infer_options=fs.InferOptions.default(), targets=targets) # reading csv file controller = build_flow([ReadCSV(csv_path), ReduceToDataFrame()]).run() termination_result = controller.await_termination() expected = pd.DataFrame({ 0: ["ticker", "MSFT", "GOOG", "AAPL"], 1: ["name", "Microsoft Corporation", "Alphabet Inc", "Apple Inc"], 2: ["exchange", "NASDAQ", "NASDAQ", "NASDAQ"], }) assert termination_result.equals( expected), f"{termination_result}\n!=\n{expected}" os.remove(csv_path)
def test_serverless_ingest(): init_store() measurements = fs.FeatureSet("measurements", entities=[Entity("patient_id")]) src_df = pd.read_csv(local_dir + "testdata.csv") df = fs.infer_metadata( measurements, src_df, timestamp_key="timestamp", options=fs.InferOptions.default(), ) print(df.head(5)) target_path = os.path.relpath(results_dir + "mycsv.csv") source = CSVSource("mycsv", path=os.path.relpath(local_dir + "testdata.csv")) targets = [CSVTarget("mycsv", path=target_path)] if os.path.exists(target_path): os.remove(target_path) run_ingestion_task( measurements, source, targets, name="test_ingest", infer_options=fs.InferOptions.Null, parameters={}, function=None, local=True, ) assert os.path.exists(target_path), "result file was not generated"
def test_unaggregated_columns(self): test_base_time = datetime(2020, 12, 1, 17, 33, 15) data = pd.DataFrame({ "time": [test_base_time, test_base_time - pd.Timedelta(minutes=1)], "first_name": ["moshe", "yosi"], "last_name": ["cohen", "levi"], "bid": [2000, 10], }) name = f"measurements_{uuid.uuid4()}" # write to kv data_set = fs.FeatureSet(name, entities=[Entity("first_name")]) data_set.add_aggregation( name="bids", column="bid", operations=["sum", "max"], windows="1h", period="10m", ) fs.ingest(data_set, data, return_df=True) features = [f"{name}.bids_sum_1h", f"{name}.last_name"] vector = fs.FeatureVector("my-vec", features) svc = fs.get_online_feature_service(vector) resp = svc.get([{"first_name": "moshe"}]) expected = {"bids_sum_1h": 2000.0, "last_name": "cohen"} assert resp[0] == expected svc.close()
def test_serverless_ingest(): init_store() key = "patient_id" measurements = fs.FeatureSet("measurements", entities=[Entity(key)], timestamp_key="timestamp") target_path = os.path.relpath(results_dir + "mycsv.csv") source = CSVSource("mycsv", path=os.path.relpath(local_dir + "testdata.csv")) targets = [CSVTarget("mycsv", path=target_path)] if os.path.exists(target_path): os.remove(target_path) run_ingestion_job( measurements, source, targets, name="test_ingest", infer_options=fs.InferOptions.schema() + fs.InferOptions.Stats, parameters={}, function=None, local=True, ) assert os.path.exists(target_path), "result file was not generated" features = sorted(measurements.spec.features.keys()) stats = sorted(measurements.status.stats.keys()) print(features) print(stats) stats.remove("timestamp") assert features == stats, "didnt infer stats for all features" print(measurements.to_yaml())
def test_ingest_dataframe_index(self): orig_df = pd.DataFrame([{"x", "y"}]) orig_df.index.name = "idx" fset = fs.FeatureSet("myfset", entities=[Entity("idx")]) fs.ingest( fset, orig_df, [ParquetTarget()], infer_options=fs.InferOptions.default() )
def test_ingest_with_timestamp(self): key = "patient_id" measurements = fs.FeatureSet("measurements", entities=[Entity(key)], timestamp_key="timestamp") source = CSVSource( "mycsv", path=os.path.relpath(str(self.assets_path / "testdata.csv")), time_field="timestamp", ) resp = fs.ingest(measurements, source) assert resp["timestamp"].head( n=1)[0] == datetime.fromisoformat("2020-12-01 17:24:15.906352")
def test_filtering_parquet_by_time(self): key = "patient_id" measurements = fs.FeatureSet( "measurements", entities=[Entity(key)], timestamp_key="timestamp" ) source = ParquetSource( "myparquet", path=os.path.relpath(str(self.assets_path / "testdata.parquet")), time_field="timestamp", start_time=datetime(2020, 12, 1, 17, 33, 15), end_time="2020-12-01 17:33:16", ) resp = fs.ingest(measurements, source, return_df=True,) assert len(resp) == 10
def test_sync_pipeline(self): stocks_set = fs.FeatureSet( "stocks-sync", entities=[Entity("ticker", ValueType.STRING)], engine="pandas", ) stocks_set.graph.to(name="s1", handler="myfunc1") df = fs.ingest(stocks_set, stocks) self._logger.info(f"output df:\n{df}") features = list(stocks_set.spec.features.keys()) assert len(features) == 1, "wrong num of features" assert "exchange" not in features, "field was not dropped" assert len(df) == len(stocks), "dataframe size doesnt match"
def test_time_with_timezone(self): data = pd.DataFrame({ "time": [ datetime(2021, 6, 30, 15, 9, 35, tzinfo=timezone.utc), datetime(2021, 6, 30, 15, 9, 35, tzinfo=timezone.utc), ], "first_name": ["katya", "dina"], "bid": [2000, 10], }) data_set = fs.FeatureSet("fs4", entities=[Entity("first_name")]) df = fs.ingest(data_set, data, return_df=True) data.set_index("first_name", inplace=True) assert_frame_equal(df, data)
def test_non_partitioned_target_in_dir(self): source = CSVSource( "mycsv", path=os.path.relpath(str(self.assets_path / "testdata.csv")) ) path = str(self.results_path / _generate_random_name()) target = ParquetTarget(path=path) fset = fs.FeatureSet( name="test", entities=[Entity("patient_id")], timestamp_key="timestamp" ) fs.ingest(fset, source, targets=[target]) list_files = os.listdir(path) assert len(list_files) == 1 and not os.path.isdir(path + "/" + list_files[0]) os.remove(path + "/" + list_files[0])
def _ingest_stocks_featureset(self): stocks_set = fs.FeatureSet( "stocks", entities=[Entity("ticker", ValueType.STRING)]) df = fs.ingest(stocks_set, stocks, infer_options=fs.InferOptions.default()) self._logger.info(f"output df:\n{df}") stocks_set["name"].description = "some name" self._logger.info(f"stocks spec: {stocks_set.to_yaml()}") assert (stocks_set.spec.features["name"].description == "some name" ), "description was not set" assert len(df) == len(stocks), "dataframe size doesnt match" assert stocks_set.status.stats["exchange"], "stats not created"
def test_none_value(self): data = pd.DataFrame( {"first_name": ["moshe", "yossi"], "bid": [2000, 10], "bool": [True, None]} ) # write to kv data_set = fs.FeatureSet("tests2", entities=[Entity("first_name")]) fs.ingest(data_set, data, return_df=True) features = ["tests2.*"] vector = fs.FeatureVector("my-vec", features) svc = fs.get_online_feature_service(vector) resp = svc.get([{"first_name": "yossi"}]) assert resp[0] == {"bid": 10, "bool": None} svc.close()
def test_basic_featureset(): init_store() # add feature set without time column (stock ticker metadata) stocks_set = fs.FeatureSet("stocks", entities=[Entity("ticker", ValueType.STRING)]) df = fs.ingest(stocks_set, stocks, infer_options=fs.InferOptions.default()) logger.info(f"output df:\n{df}") stocks_set["name"].description = "some name" logger.info(f"stocks spec: {stocks_set.to_yaml()}") assert (stocks_set.spec.features["name"].description == "some name" ), "description was not set" assert len(df) == len(stocks), "datafreame size doesnt match" assert stocks_set.status.stats["exchange"], "stats not created"
def test_feature_set_db(self): name = "stocks_test" stocks_set = fs.FeatureSet(name, entities=[Entity("ticker", ValueType.STRING)]) fs.infer_metadata( stocks_set, stocks, ) stocks_set.save() db = mlrun.get_run_db() sets = db.list_feature_sets(self.project_name, name) assert len(sets) == 1, "bad number of results" feature_set = fs.get_feature_set(name, self.project_name) assert feature_set.metadata.name == name, "bad feature set response" fs.delete_feature_set(name, self.project_name) sets = db.list_feature_sets(self.project_name, name) assert not sets, "Feature set should be deleted"
def test_feature_set_db(): init_store() name = "stocks_test" stocks_set = fs.FeatureSet(name, entities=[Entity("ticker", ValueType.STRING)]) fs.infer_metadata( stocks_set, stocks, ) stocks_set.save() db = mlrun.get_run_db() sets = db.list_feature_sets("", name) assert len(sets) == 1, "bad number of results" feature_set = db.get_feature_set(name) assert feature_set.metadata.name == name, "bad feature set response"
def test_offline_features_filter_non_partitioned(self): data = pd.DataFrame({ "time_stamp": [ pd.Timestamp("2021-06-09 09:30:06.008"), pd.Timestamp("2021-06-09 10:29:07.009"), pd.Timestamp("2021-06-09 09:29:08.010"), ], "data": [10, 20, 30], "string": ["ab", "cd", "ef"], }) data_set1 = fs.FeatureSet("fs1", entities=[Entity("string")]) fs.ingest(data_set1, data, infer_options=fs.InferOptions.default()) features = ["fs1.*"] vector = fs.FeatureVector("vector", features) resp = fs.get_offline_features( vector, entity_timestamp_column="time_stamp", start_time=datetime(2021, 6, 9, 9, 30), end_time=datetime(2021, 6, 9, 10, 30), ) assert len(resp.to_dataframe()) == 2
def test_ingest_partitioned_by_key_and_time( self, key_bucketing_number, partition_cols, time_partitioning_granularity ): key = "patient_id" name = f"measurements_{uuid.uuid4()}" measurements = fs.FeatureSet(name, entities=[Entity(key)]) source = CSVSource( "mycsv", path=os.path.relpath(str(self.assets_path / "testdata.csv")), time_field="timestamp", ) measurements.set_targets( targets=[ ParquetTarget( partitioned=True, key_bucketing_number=key_bucketing_number, partition_cols=partition_cols, time_partitioning_granularity=time_partitioning_granularity, ) ], with_defaults=False, ) resp1 = fs.ingest(measurements, source) features = [ f"{name}.*", ] vector = fs.FeatureVector("myvector", features) resp = fs.get_offline_features(vector) resp2 = resp.to_dataframe() assert resp1.to_dict() == resp2.to_dict() file_system = fsspec.filesystem("v3io") kind = TargetTypes.parquet path = f"{get_default_prefix_for_target(kind)}/sets/{name}-latest" path = path.format(name=name, kind=kind, project="system-test-project") dataset = pq.ParquetDataset(path, filesystem=file_system,) partitions = [key for key, _ in dataset.pieces[0].partition_keys] if key_bucketing_number is None: expected_partitions = [] elif key_bucketing_number == 0: expected_partitions = ["igzpart_key"] else: expected_partitions = [f"igzpart_hash{key_bucketing_number}_key"] expected_partitions += partition_cols or [] if all( value is None for value in [ key_bucketing_number, partition_cols, time_partitioning_granularity, ] ): time_partitioning_granularity = "hour" if time_partitioning_granularity: for unit in ["year", "month", "day", "hour"]: expected_partitions.append(f"igzpart_{unit}") if unit == time_partitioning_granularity: break assert partitions == expected_partitions resp = fs.get_offline_features( vector, start_time=datetime(2020, 12, 1, 17, 33, 15), end_time=datetime(2020, 12, 1, 17, 33, 16), entity_timestamp_column="timestamp", ) resp2 = resp.to_dataframe() assert len(resp2) == 10
def test_ingest_with_column_conversion(self): orig_df = source = pd.DataFrame( { "time_stamp": [ pd.Timestamp("2002-04-01 04:32:34.000"), pd.Timestamp("2002-04-01 15:05:37.000"), pd.Timestamp("2002-03-31 23:46:07.000"), ], "ssrxbtok": [488441267876, 438975336749, 298802679370], "nkxuonfx": [0.241233, 0.160264, 0.045345], "xzvipbmo": [True, False, None], "bikyseca": ["ONE", "TWO", "THREE"], "napxsuhp": [True, False, True], "oegndrxe": [ pd.Timestamp("2002-04-01 04:32:34.000"), pd.Timestamp("2002-04-01 05:06:34.000"), pd.Timestamp("2002-04-01 05:38:34.000"), ], "aatxnkgx": [-227504700006, -470002151801, -33193685176], "quupyoxi": ["FOUR", "FIVE", "SIX"], "temdojgz": [0.570031, 0.677182, 0.276053], }, index=None, ) fset = fs.FeatureSet( "rWQTKqbhje", timestamp_key="time_stamp", entities=[ Entity("{}".format(k["name"])) for k in [ { "dtype": "float", "null_values": False, "name": "temdojgz", "df_dtype": "float64", }, { "dtype": "str", "null_values": False, "name": "bikyseca", "df_dtype": "object", }, { "dtype": "float", "null_values": False, "name": "nkxuonfx", "df_dtype": "float64", }, ] ], ) fset.graph.to(name="s1", handler="my_func") ikjqkfcz = ParquetTarget(path="v3io:///bigdata/ifrlsjvxgv", partitioned=False) fs.ingest(fset, source, targets=[ikjqkfcz]) features = ["rWQTKqbhje.*"] vector = fs.FeatureVector("WPAyrYux", features) vector.spec.with_indexes = False resp = fs.get_offline_features(vector) off_df = resp.to_dataframe() del orig_df["time_stamp"] if None in list(orig_df.index.names): orig_df.set_index(["temdojgz", "bikyseca", "nkxuonfx"], inplace=True) orig_df = orig_df.sort_values( by=["temdojgz", "bikyseca", "nkxuonfx"]).sort_index(axis=1) off_df = off_df.sort_values( by=["temdojgz", "bikyseca", "nkxuonfx"]).sort_index(axis=1) pd.testing.assert_frame_equal( off_df, orig_df, check_dtype=True, check_index_type=True, check_column_type=True, check_like=True, check_names=True, )
def test_feature_set(): myset = FeatureSet("set1", entities=[Entity("key")]) myset["f1"] = Feature(ValueType.INT64, description="my f1") assert list(myset.spec.entities.keys()) == ["key"], "index wasnt set" assert list(myset.spec.features.keys()) == ["f1"], "feature wasnt set"