def test_apply_entity_integration(test_feature_store):
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
labels={"team": "matchmaking"},
)
# Register Entity
test_feature_store.apply([entity])
entities = test_feature_store.list_entities()
entity = entities[0]
assert (len(entities) == 1 and entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
entity = test_feature_store.get_entity("driver_car_id")
assert (entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
test_feature_store.teardown()
def test_apply_entity_integration(self, test_client):
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
labels={"team": "matchmaking"},
)
# Register Entity with Core
test_client.apply(entity)
entities = test_client.list_entities()
entity = entities[0]
assert (len(entities) == 1 and entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
entity = test_client.get_entity("driver_car_id")
assert (entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
def test_apply_entity_success(test_registry):
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
labels={"team": "matchmaking"},
)
project = "project"
# Register Entity
test_registry.apply_entity(entity, project)
entities = test_registry.list_entities(project)
entity = entities[0]
assert (len(entities) == 1 and entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
entity = test_registry.get_entity("driver_car_id", project)
assert (entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
test_registry.teardown()
# Will try to reload registry, which will fail because the file has been deleted
with pytest.raises(FileNotFoundError):
test_registry._get_registry_proto()
def __eq__(self, other):
if not isinstance(other, EntityV2):
raise TypeError(
"Comparisons should only involve EntityV2 class objects.")
if isinstance(self.value_type, int):
self.value_type = ValueType(self.value_type).name
if isinstance(other.value_type, int):
other.value_type = ValueType(other.value_type).name
if (self.labels != other.labels or self.name != other.name
or self.description != other.description
or self.value_type != other.value_type):
return False
return True
def from_proto(data_source: DataSourceProto):
deprecated_schema = data_source.request_data_options.deprecated_schema
schema_pb = data_source.request_data_options.schema
if deprecated_schema and not schema_pb:
warnings.warn(
"Schema in RequestSource is changing type. The schema data type Dict[str, ValueType] is being deprecated in Feast 0.23. "
"Please use List[Field] instead for the schema",
DeprecationWarning,
)
dict_schema = {}
for key, val in deprecated_schema.items():
dict_schema[key] = ValueType(val)
return RequestSource(
name=data_source.name,
schema=dict_schema,
description=data_source.description,
tags=dict(data_source.tags),
owner=data_source.owner,
)
else:
list_schema = []
for field_proto in schema_pb:
list_schema.append(Field.from_proto(field_proto))
return RequestSource(
name=data_source.name,
schema=list_schema,
description=data_source.description,
tags=dict(data_source.tags),
owner=data_source.owner,
)
def _feature_table_to_argument(
client: "Client", project: str, feature_table: FeatureTable, use_gc_threshold=True,
):
max_age = feature_table.max_age.ToSeconds() if feature_table.max_age else None
if use_gc_threshold:
try:
gc_threshold = int(feature_table.labels["gcThresholdSec"])
except (KeyError, ValueError, TypeError):
pass
else:
max_age = max(max_age or 0, gc_threshold)
return {
"features": [
{"name": f.name, "type": ValueType(f.dtype).name}
for f in feature_table.features
],
"project": project,
"name": feature_table.name,
"entities": [
{
"name": n,
"type": client.feature_store.get_entity(n, project=project).value_type,
}
for n in feature_table.entities
],
"max_age": max_age,
"labels": dict(feature_table.labels),
}
def from_proto(cls, entity_proto: EntityV2Proto):
"""
Creates an entity from a protobuf representation of an entity.
Args:
entity_proto: A protobuf representation of an entity.
Returns:
An EntityV2 object based on the entity protobuf.
"""
entity = cls(
name=entity_proto.spec.name,
description=entity_proto.spec.description,
value_type=ValueType(entity_proto.spec.value_type),
labels=entity_proto.spec.labels,
join_key=entity_proto.spec.join_key,
)
if entity_proto.meta.HasField("created_timestamp"):
entity._created_timestamp = entity_proto.meta.created_timestamp.ToDatetime()
if entity_proto.meta.HasField("last_updated_timestamp"):
entity._last_updated_timestamp = (
entity_proto.meta.last_updated_timestamp.ToDatetime()
)
return entity
def from_proto(data_source: DataSourceProto):
schema_pb = data_source.request_data_options.schema
schema = {}
for key in schema_pb.keys():
schema[key] = ValueType(schema_pb.get(key))
return RequestDataSource(name=data_source.request_data_options.name,
schema=schema)
def from_proto(cls, feature_view_proto: FeatureViewProto):
"""
Creates a feature view from a protobuf representation of a feature view
Args:
feature_view_proto: A protobuf representation of a feature view
Returns:
Returns a FeatureViewProto object based on the feature view protobuf
"""
feature_view = cls(
name=feature_view_proto.spec.name,
entities=[entity for entity in feature_view_proto.spec.entities],
features=[
Feature(
name=feature.name,
dtype=ValueType(feature.value_type),
labels=feature.labels,
) for feature in feature_view_proto.spec.features
],
tags=dict(feature_view_proto.spec.tags),
online=feature_view_proto.spec.online,
ttl=(None if feature_view_proto.spec.ttl.seconds == 0
and feature_view_proto.spec.ttl.nanos == 0 else
feature_view_proto.spec.ttl),
input=DataSource.from_proto(feature_view_proto.spec.input),
)
feature_view.created_timestamp = feature_view_proto.meta.created_timestamp
return feature_view
def from_proto(cls, feature_view_proto: FeatureViewProto):
"""
Creates a feature view from a protobuf representation of a feature view.
Args:
feature_view_proto: A protobuf representation of a feature view.
Returns:
A FeatureViewProto object based on the feature view protobuf.
"""
batch_source = DataSource.from_proto(feature_view_proto.spec.batch_source)
stream_source = (
DataSource.from_proto(feature_view_proto.spec.stream_source)
if feature_view_proto.spec.HasField("stream_source")
else None
)
feature_view = cls(
name=feature_view_proto.spec.name,
entities=[entity for entity in feature_view_proto.spec.entities],
features=[
Feature(
name=feature.name,
dtype=ValueType(feature.value_type),
labels=dict(feature.labels),
)
for feature in feature_view_proto.spec.features
],
tags=dict(feature_view_proto.spec.tags),
online=feature_view_proto.spec.online,
ttl=(
None
if feature_view_proto.spec.ttl.seconds == 0
and feature_view_proto.spec.ttl.nanos == 0
else feature_view_proto.spec.ttl
),
batch_source=batch_source,
stream_source=stream_source,
)
if feature_view_proto.meta.HasField("created_timestamp"):
feature_view.created_timestamp = (
feature_view_proto.meta.created_timestamp.ToDatetime()
)
if feature_view_proto.meta.HasField("last_updated_timestamp"):
feature_view.last_updated_timestamp = (
feature_view_proto.meta.last_updated_timestamp.ToDatetime()
)
for interval in feature_view_proto.meta.materialization_intervals:
feature_view.materialization_intervals.append(
(
utils.make_tzaware(interval.start_time.ToDatetime()),
utils.make_tzaware(interval.end_time.ToDatetime()),
)
)
return feature_view
def from_proto(cls, feature_proto: FeatureSpecProto):
"""
Args:
feature_proto: FeatureSpecV2 protobuf object
Returns:
Feature object
"""
data_type = ValueType(feature_proto.value_type)
if data_type == ValueType.SQL:
return SQLFeature.from_proto(feature_proto)
else:
return cls(
name=feature_proto.name,
dtype=ValueType(feature_proto.value_type),
labels=feature_proto.labels,
)
def from_proto(cls, field_proto: FieldProto):
"""
Creates a Field object from a protobuf representation.
Args:
field_proto: FieldProto protobuf object
"""
value_type = ValueType(field_proto.value_type)
return cls(name=field_proto.name,
dtype=from_value_type(value_type=value_type))
def from_proto(cls, entity_proto: EntityProto):
"""
Creates a Feast Entity object from its Protocol Buffer representation
Args:
entity_proto: EntitySpec protobuf object
Returns:
Entity object
"""
entity = cls(name=entity_proto.name, dtype=ValueType(entity_proto.value_type))
return entity
def from_proto(cls,
on_demand_feature_view_proto: OnDemandFeatureViewProto):
"""
Creates an on demand feature view from a protobuf representation.
Args:
on_demand_feature_view_proto: A protobuf representation of an on-demand feature view.
Returns:
A OnDemandFeatureView object based on the on-demand feature view protobuf.
"""
inputs = {}
for (
input_name,
on_demand_input,
) in on_demand_feature_view_proto.spec.inputs.items():
if on_demand_input.WhichOneof("input") == "feature_view":
inputs[input_name] = FeatureView.from_proto(
on_demand_input.feature_view).projection
elif on_demand_input.WhichOneof(
"input") == "feature_view_projection":
inputs[input_name] = FeatureViewProjection.from_proto(
on_demand_input.feature_view_projection)
else:
inputs[input_name] = RequestDataSource.from_proto(
on_demand_input.request_data_source)
on_demand_feature_view_obj = cls(
name=on_demand_feature_view_proto.spec.name,
features=[
Feature(
name=feature.name,
dtype=ValueType(feature.value_type),
labels=dict(feature.labels),
) for feature in on_demand_feature_view_proto.spec.features
],
inputs=inputs,
udf=dill.loads(
on_demand_feature_view_proto.spec.user_defined_function.body),
)
# FeatureViewProjections are not saved in the OnDemandFeatureView proto.
# Create the default projection.
on_demand_feature_view_obj.projection = FeatureViewProjection.from_definition(
on_demand_feature_view_obj)
if on_demand_feature_view_proto.meta.HasField("created_timestamp"):
on_demand_feature_view_obj.created_timestamp = (
on_demand_feature_view_proto.meta.created_timestamp.ToDatetime(
))
return on_demand_feature_view_obj
def _feature_table_to_argument(client: "Client", feature_table: FeatureTable):
return {
"features": [
{"name": f.name, "type": ValueType(f.dtype).name}
for f in feature_table.features
],
"project": "default",
"name": feature_table.name,
"entities": [
{"name": n, "type": client.get_entity(n).value_type}
for n in feature_table.entities
],
"max_age": feature_table.max_age.ToSeconds() if feature_table.max_age else None,
}
def from_proto(cls, feature_proto: FeatureSpecProto):
"""
Args:
feature_proto: FeatureSpecV2 protobuf object
Returns:
Feature object
"""
feature = cls(
name=feature_proto.name,
dtype=ValueType(feature_proto.value_type),
labels=dict(feature_proto.labels),
)
return feature
def from_proto(cls, feature_proto: FeatureProto):
"""
Args:
feature_proto: FeatureSpec protobuf object
Returns:
Feature object
"""
feature = cls(name=feature_proto.name,
dtype=ValueType(feature_proto.value_type))
feature.update_presence_constraints(feature_proto)
feature.update_shape_type(feature_proto)
feature.update_domain_info(feature_proto)
return feature
def _feature_table_to_json(client: Client, feature_table):
return {
"features": [{
"name": f.name,
"type": ValueType(f.dtype).name
} for f in feature_table.features],
"project":
"default",
"name":
feature_table.name,
"entities": [{
"name": n,
"type": client.get_entity(n).value_type
} for n in feature_table.entities],
}
def from_proto(cls, entity_proto: EntityProto):
"""
Creates a Feast Entity object from its Protocol Buffer representation
Args:
entity_proto: EntitySpec protobuf object
Returns:
Entity object
"""
entity = cls(name=entity_proto.name,
dtype=ValueType(entity_proto.value_type))
entity.update_presence_constraints(entity_proto)
entity.update_shape_type(entity_proto)
entity.update_domain_info(entity_proto)
return entity
def _feature_table_to_argument(
client: "Client", project: str, feature_table: FeatureTable
):
return {
"features": [
{"name": f.name, "type": ValueType(f.dtype).name}
for f in feature_table.features
],
"project": project,
"name": feature_table.name,
"entities": [
{"name": n, "type": client.get_entity(n, project=project).value_type}
for n in feature_table.entities
],
"max_age": feature_table.max_age.ToSeconds() if feature_table.max_age else None,
"labels": dict(feature_table.labels),
}
def test_apply_entity_success(test_feature_store):
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
tags={"team": "matchmaking"},
)
# Register Entity
test_feature_store.apply(entity)
entities = test_feature_store.list_entities()
entity = entities[0]
assert (len(entities) == 1 and entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id" and "team" in entity.tags
and entity.tags["team"] == "matchmaking")
test_feature_store.teardown()
def from_proto(cls, feature_table_proto: FeatureTableProto):
"""
Creates a feature table from a protobuf representation of a feature table
Args:
feature_table_proto: A protobuf representation of a feature table
Returns:
Returns a FeatureTableProto object based on the feature table protobuf
"""
feature_table = cls(
name=feature_table_proto.spec.name,
entities=[entity for entity in feature_table_proto.spec.entities],
features=[
Feature(
name=feature.name,
dtype=ValueType(feature.value_type),
labels=dict(feature.labels),
)
for feature in feature_table_proto.spec.features
],
labels=feature_table_proto.spec.labels,
max_age=(
None
if feature_table_proto.spec.max_age.seconds == 0
and feature_table_proto.spec.max_age.nanos == 0
else feature_table_proto.spec.max_age
),
batch_source=DataSource.from_proto(feature_table_proto.spec.batch_source),
stream_source=(
None
if not feature_table_proto.spec.stream_source.ByteSize()
else DataSource.from_proto(feature_table_proto.spec.stream_source)
),
)
feature_table._created_timestamp = feature_table_proto.meta.created_timestamp
return feature_table
def from_proto(cls, entity_proto: EntityV2Proto):
"""
Creates an entity from a protobuf representation of an entity
Args:
entity_proto: A protobuf representation of an entity
Returns:
Returns a EntityV2 object based on the entity protobuf
"""
entity = cls(
name=entity_proto.spec.name,
description=entity_proto.spec.description,
value_type=ValueType(entity_proto.spec.value_type),
labels=entity_proto.spec.labels,
)
entity._created_timestamp = entity_proto.meta.created_timestamp
entity._last_updated_timestamp = entity_proto.meta.last_updated_timestamp
return entity
def from_proto(cls,
on_demand_feature_view_proto: OnDemandFeatureViewProto):
"""
Creates an on demand feature view from a protobuf representation.
Args:
on_demand_feature_view_proto: A protobuf representation of an on-demand feature view.
Returns:
A OnDemandFeatureView object based on the on-demand feature view protobuf.
"""
inputs = {}
for (
input_name,
on_demand_input,
) in on_demand_feature_view_proto.spec.inputs.items():
if on_demand_input.WhichOneof("input") == "feature_view":
inputs[input_name] = FeatureView.from_proto(
on_demand_input.feature_view)
else:
inputs[input_name] = RequestDataSource.from_proto(
on_demand_input.request_data_source)
on_demand_feature_view_obj = cls(
name=on_demand_feature_view_proto.spec.name,
features=[
Feature(
name=feature.name,
dtype=ValueType(feature.value_type),
labels=dict(feature.labels),
) for feature in on_demand_feature_view_proto.spec.features
],
inputs=inputs,
udf=dill.loads(
on_demand_feature_view_proto.spec.user_defined_function.body),
)
return on_demand_feature_view_obj
def test_commit():
fd, registry_path = mkstemp()
registry_config = RegistryConfig(path=registry_path, cache_ttl_seconds=600)
test_registry = Registry(registry_config, None)
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
labels={"team": "matchmaking"},
)
project = "project"
# Register Entity without commiting
test_registry.apply_entity(entity, project, commit=False)
# Retrieving the entity should still succeed
entities = test_registry.list_entities(project, allow_cache=True)
entity = entities[0]
assert (len(entities) == 1 and entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
entity = test_registry.get_entity("driver_car_id",
project,
allow_cache=True)
assert (entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
# Create new registry that points to the same store
registry_with_same_store = Registry(registry_config, None)
# Retrieving the entity should fail since the store is empty
entities = registry_with_same_store.list_entities(project)
assert len(entities) == 0
# commit from the original registry
test_registry.commit()
# Reconstruct the new registry in order to read the newly written store
registry_with_same_store = Registry(registry_config, None)
# Retrieving the entity should now succeed
entities = registry_with_same_store.list_entities(project)
entity = entities[0]
assert (len(entities) == 1 and entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
entity = test_registry.get_entity("driver_car_id", project)
assert (entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
test_registry.teardown()
# Will try to reload registry, which will fail because the file has been deleted
with pytest.raises(FileNotFoundError):
test_registry._get_registry_proto()
def from_proto(cls, entity_proto: EntityProto):
return cls(name=entity_proto.name,
dtype=ValueType(entity_proto.value_type))
def from_proto(cls, feature_proto: FeatureProto):
"""Converts Protobuf Feature to its SDK equivalent"""
return cls(name=feature_proto.name,
dtype=ValueType(feature_proto.value_type))
def from_proto(cls, feature_proto: FeatureProto):
return cls(name=feature_proto.name, dtype=ValueType(feature_proto.value_type))
def from_proto(cls,
on_demand_feature_view_proto: OnDemandFeatureViewProto):
"""
Creates an on demand feature view from a protobuf representation.
Args:
on_demand_feature_view_proto: A protobuf representation of an on-demand feature view.
Returns:
A OnDemandFeatureView object based on the on-demand feature view protobuf.
"""
sources = []
for (
_,
on_demand_source,
) in on_demand_feature_view_proto.spec.sources.items():
if on_demand_source.WhichOneof("source") == "feature_view":
sources.append(
FeatureView.from_proto(
on_demand_source.feature_view).projection)
elif on_demand_source.WhichOneof(
"source") == "feature_view_projection":
sources.append(
FeatureViewProjection.from_proto(
on_demand_source.feature_view_projection))
else:
sources.append(
RequestSource.from_proto(
on_demand_source.request_data_source))
on_demand_feature_view_obj = cls(
name=on_demand_feature_view_proto.spec.name,
schema=[
Field(
name=feature.name,
dtype=from_value_type(ValueType(feature.value_type)),
) for feature in on_demand_feature_view_proto.spec.features
],
sources=sources,
udf=dill.loads(
on_demand_feature_view_proto.spec.user_defined_function.body),
description=on_demand_feature_view_proto.spec.description,
tags=dict(on_demand_feature_view_proto.spec.tags),
owner=on_demand_feature_view_proto.spec.owner,
)
# FeatureViewProjections are not saved in the OnDemandFeatureView proto.
# Create the default projection.
on_demand_feature_view_obj.projection = FeatureViewProjection.from_definition(
on_demand_feature_view_obj)
if on_demand_feature_view_proto.meta.HasField("created_timestamp"):
on_demand_feature_view_obj.created_timestamp = (
on_demand_feature_view_proto.meta.created_timestamp.ToDatetime(
))
if on_demand_feature_view_proto.meta.HasField(
"last_updated_timestamp"):
on_demand_feature_view_obj.last_updated_timestamp = (
on_demand_feature_view_proto.meta.last_updated_timestamp.
ToDatetime())
return on_demand_feature_view_obj