def test_register_feature_set(self, sqlite_store):
fs = FeatureSet("my-feature-set")
fs.add(Feature(name="my-feature-1", dtype=ValueType.INT64))
fs.add(Feature(name="my-feature-2", dtype=ValueType.INT64))
fs.add(Entity(name="my-entity-1", dtype=ValueType.INT64))
fs._version = 1
feature_set_spec_proto = fs.to_proto().spec
sqlite_store.register_feature_set(feature_set_spec_proto)
feature_row = FeatureRowProto.FeatureRow(
feature_set="feature_set_1",
event_timestamp=Timestamp(),
fields=[
FieldProto.Field(
name="feature_1", value=ValueProto.Value(float_val=1.2)
),
FieldProto.Field(
name="feature_2", value=ValueProto.Value(float_val=1.2)
),
FieldProto.Field(
name="feature_3", value=ValueProto.Value(float_val=1.2)
),
],
)
# sqlite_store.upsert_feature_row(feature_set_proto, feature_row)
assert True
def test_feature_class_contains_labels():
fs = FeatureSet("my-feature-set", labels={"key1": "val1", "key2": "val2"})
fs.add(
Feature(
name="my-feature-1",
dtype=ValueType.INT64,
labels={"feature_key1": "feature_val1"},
))
assert "feature_key1" in fs.features[0].labels.keys()
assert fs.features[0].labels["feature_key1"] == "feature_val1"
def test_feature_set_ingest_fail_if_pending(self, dataframe, exception,
test_client, mocker):
with pytest.raises(exception):
test_client.set_project("project1")
driver_fs = FeatureSet(
"driver-feature-set",
source=KafkaSource(brokers="kafka:9092", topic="test"),
)
driver_fs.add(Feature(name="feature_1", dtype=ValueType.FLOAT))
driver_fs.add(Feature(name="feature_2", dtype=ValueType.STRING))
driver_fs.add(Feature(name="feature_3", dtype=ValueType.INT64))
driver_fs.add(Entity(name="entity_id", dtype=ValueType.INT64))
# Register with Feast core
test_client.apply(driver_fs)
driver_fs = driver_fs.to_proto()
driver_fs.meta.status = FeatureSetStatusProto.STATUS_PENDING
mocker.patch.object(
test_client._core_service_stub,
"GetFeatureSet",
return_value=GetFeatureSetResponse(feature_set=driver_fs),
)
# Need to create a mock producer
with patch("feast.client.get_producer"):
# Ingest data into Feast
test_client.ingest("driver-feature-set", dataframe, timeout=1)
def test_feature_set_ingest_throws_exception_if_kafka_down(
self, dataframe, test_client, exception, mocker):
test_client.set_project("project1")
driver_fs = FeatureSet(
"driver-feature-set",
source=KafkaSource(brokers="localhost:4412", topic="test"),
)
driver_fs.add(Feature(name="feature_1", dtype=ValueType.FLOAT))
driver_fs.add(Feature(name="feature_2", dtype=ValueType.STRING))
driver_fs.add(Feature(name="feature_3", dtype=ValueType.INT64))
driver_fs.add(Entity(name="entity_id", dtype=ValueType.INT64))
# Register with Feast core
test_client.apply(driver_fs)
driver_fs = driver_fs.to_proto()
driver_fs.meta.status = FeatureSetStatusProto.STATUS_READY
mocker.patch.object(
test_client._core_service_stub,
"GetFeatureSet",
return_value=GetFeatureSetResponse(feature_set=driver_fs),
)
with pytest.raises(exception):
test_client.ingest("driver-feature-set", dataframe)
def test_feature_set_ingest_success(self, dataframe, client, mocker):
client.set_project("project1")
driver_fs = FeatureSet("driver-feature-set",
source=KafkaSource(brokers="kafka:9092",
topic="test"))
driver_fs.add(Feature(name="feature_1", dtype=ValueType.FLOAT))
driver_fs.add(Feature(name="feature_2", dtype=ValueType.STRING))
driver_fs.add(Feature(name="feature_3", dtype=ValueType.INT64))
driver_fs.add(Entity(name="entity_id", dtype=ValueType.INT64))
# Register with Feast core
client.apply(driver_fs)
driver_fs = driver_fs.to_proto()
driver_fs.meta.status = FeatureSetStatusProto.STATUS_READY
mocker.patch.object(
client._core_service_stub,
"GetFeatureSet",
return_value=GetFeatureSetResponse(feature_set=driver_fs),
)
# Need to create a mock producer
with patch("feast.client.get_producer") as mocked_queue:
# Ingest data into Feast
client.ingest("driver-feature-set", dataframe)
def test_feature_set_ingest_success(self, dataframe, client, mocker):
driver_fs = FeatureSet("driver-feature-set")
driver_fs.add(Feature(name="feature_1", dtype=ValueType.FLOAT))
driver_fs.add(Feature(name="feature_2", dtype=ValueType.STRING))
driver_fs.add(Feature(name="feature_3", dtype=ValueType.INT64))
driver_fs.add(Entity(name="entity_id", dtype=ValueType.INT64))
driver_fs.source = KafkaSource(topic="feature-topic",
brokers="127.0.0.1")
client._message_producer = MagicMock()
client._message_producer.produce = MagicMock()
# Register with Feast core
client.apply(driver_fs)
mocker.patch.object(
client._core_service_stub,
"GetFeatureSet",
return_value=GetFeatureSetResponse(
feature_set=driver_fs.to_proto()),
)
# Ingest data into Feast
client.ingest("driver-feature-set", dataframe=dataframe)
def test_feature_set_ingest_success(self, dataframe, client, mocker):
driver_fs = FeatureSet("driver-feature-set")
driver_fs.add(Feature(name="feature_1", dtype=ValueType.FLOAT))
driver_fs.add(Feature(name="feature_2", dtype=ValueType.STRING))
driver_fs.add(Feature(name="feature_3", dtype=ValueType.INT64))
driver_fs.add(Entity(name="entity_id", dtype=ValueType.INT64))
# Register with Feast core
client.apply(driver_fs)
mocker.patch.object(
client._core_service_stub,
"GetFeatureSet",
return_value=GetFeatureSetResponse(feature_set=driver_fs.to_proto()),
)
# Ingest data into Feast
client.ingest("driver-feature-set", dataframe=dataframe)
def test_feature_set_ingest_success(self, dataframe, client):
driver_fs = FeatureSet("driver-feature-set")
driver_fs.add(Feature(name="feature_1", dtype=ValueType.FLOAT))
driver_fs.add(Feature(name="feature_2", dtype=ValueType.STRING))
driver_fs.add(Feature(name="feature_3", dtype=ValueType.INT64))
driver_fs.add(Entity(name="entity_id", dtype=ValueType.INT64))
driver_fs.source = KafkaSource(topic="feature-topic",
brokers="127.0.0.1")
driver_fs._message_producer = MagicMock()
driver_fs._message_producer.send = MagicMock()
# Register with Feast core
client.apply(driver_fs)
# Ingest data into Feast
driver_fs.ingest(dataframe=dataframe)
# Make sure message producer is called
driver_fs._message_producer.send.assert_called()
def test_apply_feature_set(self, client):
# Create Feature Sets
fs1 = FeatureSet("my-feature-set-1")
fs1.add(Feature(name="fs1-my-feature-1", dtype=ValueType.INT64))
fs1.add(Feature(name="fs1-my-feature-2", dtype=ValueType.STRING))
fs2 = FeatureSet("my-feature-set-2")
fs2.add(Feature(name="fs2-my-feature-1", dtype=ValueType.STRING_LIST))
fs2.add(Feature(name="fs2-my-feature-2", dtype=ValueType.BYTES_LIST))
# Register Feature Set with Core
client.apply(fs1)
client.apply(fs2)
feature_sets = client.list_feature_sets()
# List Feature Sets
assert (len(feature_sets) == 2
and feature_sets[0].name == "my-feature-set-1"
and feature_sets[0].features[0].name == "fs1-my-feature-1"
and feature_sets[0].features[0].dtype == ValueType.INT64
and feature_sets[1].features[1].dtype == ValueType.BYTES_LIST)
def test_add_remove_features_success(self):
fs = FeatureSet("my-feature-set")
fs.add(Feature(name="my-feature-1", dtype=ValueType.INT64))
fs.add(Feature(name="my-feature-2", dtype=ValueType.INT64))
fs.drop(name="my-feature-1")
assert len(fs.features) == 1 and fs.features[0].name == "my-feature-2"
def test_apply_feature_set_success(self, test_client):
test_client.set_project("project1")
# Create Feature Sets
fs1 = FeatureSet("my-feature-set-1")
fs1.add(Feature(name="fs1-my-feature-1", dtype=ValueType.INT64))
fs1.add(Feature(name="fs1-my-feature-2", dtype=ValueType.STRING))
fs1.add(Entity(name="fs1-my-entity-1", dtype=ValueType.INT64))
fs2 = FeatureSet("my-feature-set-2")
fs2.add(Feature(name="fs2-my-feature-1", dtype=ValueType.STRING_LIST))
fs2.add(Feature(name="fs2-my-feature-2", dtype=ValueType.BYTES_LIST))
fs2.add(Entity(name="fs2-my-entity-1", dtype=ValueType.INT64))
# Register Feature Set with Core
test_client.apply(fs1)
test_client.apply(fs2)
feature_sets = test_client.list_feature_sets()
# List Feature Sets
assert (len(feature_sets) == 2
and feature_sets[0].name == "my-feature-set-1"
and feature_sets[0].features[0].name == "fs1-my-feature-1"
and feature_sets[0].features[0].dtype == ValueType.INT64
and feature_sets[0].features[1].name == "fs1-my-feature-2"
and feature_sets[0].features[1].dtype == ValueType.STRING
and feature_sets[0].entities[0].name == "fs1-my-entity-1"
and feature_sets[0].entities[0].dtype == ValueType.INT64
and feature_sets[1].features[0].name == "fs2-my-feature-1"
and feature_sets[1].features[0].dtype == ValueType.STRING_LIST
and feature_sets[1].features[1].name == "fs2-my-feature-2"
and feature_sets[1].features[1].dtype == ValueType.BYTES_LIST
and feature_sets[1].entities[0].name == "fs2-my-entity-1"
and feature_sets[1].entities[0].dtype == ValueType.INT64)
