def test_create_stream_feature_view():
stream_source = KafkaSource(
name="kafka",
event_timestamp_column="",
bootstrap_servers="",
message_format=AvroFormat(""),
topic="topic",
batch_source=FileSource(path="some path"),
)
StreamFeatureView(
name="test batch feature view",
entities=[],
ttl=timedelta(days=30),
source=stream_source,
)
with pytest.raises(ValueError):
StreamFeatureView(name="test batch feature view",
entities=[],
ttl=timedelta(days=30))
with pytest.raises(ValueError):
StreamFeatureView(
name="test batch feature view",
entities=[],
ttl=timedelta(days=30),
source=FileSource(path="some path"),
)
def test_infer_odfv_list_features(environment, infer_features, tmp_path):
fake_embedding = [1.0, 1.0]
items_df = pd.DataFrame(
data={
"item_id": [0],
"embedding_float": [fake_embedding],
"embedding_double": [fake_embedding],
"event_timestamp": [pd.Timestamp(datetime.utcnow())],
"created": [pd.Timestamp(datetime.utcnow())],
}
)
output_path = f"{tmp_path}/items.parquet"
items_df.to_parquet(output_path)
fake_items_src = FileSource(
path=output_path,
event_timestamp_column="event_timestamp",
created_timestamp_column="created",
)
items = create_item_embeddings_feature_view(fake_items_src)
sim_odfv = similarity_feature_view(
{"items": items, "input_request": create_similarity_request_data_source()},
infer_features=infer_features,
)
store = environment.feature_store
store.apply([item(), items, sim_odfv])
odfv = store.get_on_demand_feature_view("similarity")
assert len(odfv.features) == 2
def test_feature_view_kw_args_normal():
file_source = FileSource(name="my-file-source", path="test.parquet")
feature_view = FeatureView(
name="my-feature-view",
entities=[],
schema=[
Field(name="feature1", dtype=Float32),
Field(name="feature2", dtype=Float32),
],
source=file_source,
)
_ = FeatureService(
name="my-feature-service", features=[feature_view[["feature1", "feature2"]]]
)
def test_hash():
file_source = FileSource(name="my-file-source", path="test.parquet")
feature_view_1 = FeatureView(
name="my-feature-view",
entities=[],
schema=[
Field(name="feature1", dtype=Float32),
Field(name="feature2", dtype=Float32),
],
source=file_source,
)
feature_view_2 = FeatureView(
name="my-feature-view",
entities=[],
schema=[
Field(name="feature1", dtype=Float32),
Field(name="feature2", dtype=Float32),
],
source=file_source,
)
feature_view_3 = FeatureView(
name="my-feature-view",
entities=[],
schema=[Field(name="feature1", dtype=Float32)],
source=file_source,
)
feature_view_4 = FeatureView(
name="my-feature-view",
entities=[],
schema=[Field(name="feature1", dtype=Float32)],
source=file_source,
description="test",
)
s1 = {feature_view_1, feature_view_2}
assert len(s1) == 1
s2 = {feature_view_1, feature_view_3}
assert len(s2) == 2
s3 = {feature_view_3, feature_view_4}
assert len(s3) == 2
s4 = {feature_view_1, feature_view_2, feature_view_3, feature_view_4}
assert len(s4) == 3
def _localize_feature_view(self, feature_view: FeatureView):
"""
This function ensures that the `FeatureView` object points to files in the local disk
"""
if not isinstance(feature_view.batch_source, FileSource):
return
# Copy parquet file to a local file
file_source: FileSource = feature_view.batch_source
random_local_path = (
FlyteContext.current_context().file_access.get_random_local_path(
file_source.path))
FlyteContext.current_context().file_access.get_data(
file_source.path,
random_local_path,
is_multipart=True,
)
feature_view.batch_source = FileSource(
path=random_local_path,
event_timestamp_column=file_source.event_timestamp_column,
)
def test_hash():
file_source = FileSource(name="my-file-source", path="test.parquet")
feature_view = FeatureView(
name="my-feature-view",
entities=[],
schema=[
Field(name="feature1", dtype=Float32),
Field(name="feature2", dtype=Float32),
],
source=file_source,
)
sources = [feature_view]
on_demand_feature_view_1 = OnDemandFeatureView(
name="my-on-demand-feature-view",
sources=sources,
schema=[
Field(name="output1", dtype=Float32),
Field(name="output2", dtype=Float32),
],
udf=udf1,
)
on_demand_feature_view_2 = OnDemandFeatureView(
name="my-on-demand-feature-view",
sources=sources,
schema=[
Field(name="output1", dtype=Float32),
Field(name="output2", dtype=Float32),
],
udf=udf1,
)
on_demand_feature_view_3 = OnDemandFeatureView(
name="my-on-demand-feature-view",
sources=sources,
schema=[
Field(name="output1", dtype=Float32),
Field(name="output2", dtype=Float32),
],
udf=udf2,
)
on_demand_feature_view_4 = OnDemandFeatureView(
name="my-on-demand-feature-view",
sources=sources,
schema=[
Field(name="output1", dtype=Float32),
Field(name="output2", dtype=Float32),
],
udf=udf2,
description="test",
)
s1 = {on_demand_feature_view_1, on_demand_feature_view_2}
assert len(s1) == 1
s2 = {on_demand_feature_view_1, on_demand_feature_view_3}
assert len(s2) == 2
s3 = {on_demand_feature_view_3, on_demand_feature_view_4}
assert len(s3) == 2
s4 = {
on_demand_feature_view_1,
on_demand_feature_view_2,
on_demand_feature_view_3,
on_demand_feature_view_4,
}
assert len(s4) == 3