def _get_join_keys(project: str, feature_views: List[FeatureView],
registry: Registry) -> Set[str]:
join_keys = set()
for feature_view in feature_views:
entities = feature_view.entities
for entity_name in entities:
entity = registry.get_entity(entity_name, project)
join_keys.add(entity.join_key)
return join_keys
def get_feature_view_query_context(
feature_refs: List[str],
feature_views: List[FeatureView],
registry: Registry,
project: str,
) -> List[FeatureViewQueryContext]:
"""Build a query context containing all information required to template a BigQuery point-in-time SQL query"""
feature_views_to_feature_map = _get_requested_feature_views_to_features_dict(
feature_refs, feature_views
)
query_context = []
for feature_view, features in feature_views_to_feature_map.items():
join_keys = []
entity_selections = []
reverse_field_mapping = {
v: k for k, v in feature_view.input.field_mapping.items()
}
for entity_name in feature_view.entities:
entity = registry.get_entity(entity_name, project)
join_keys.append(entity.join_key)
join_key_column = reverse_field_mapping.get(
entity.join_key, entity.join_key
)
entity_selections.append(f"{join_key_column} AS {entity.join_key}")
if isinstance(feature_view.ttl, timedelta):
ttl_seconds = int(feature_view.ttl.total_seconds())
else:
ttl_seconds = 0
assert isinstance(feature_view.input, BigQuerySource)
event_timestamp_column = feature_view.input.event_timestamp_column
created_timestamp_column = feature_view.input.created_timestamp_column
context = FeatureViewQueryContext(
name=feature_view.name,
ttl=ttl_seconds,
entities=join_keys,
features=features,
table_ref=feature_view.input.table_ref,
event_timestamp_column=reverse_field_mapping.get(
event_timestamp_column, event_timestamp_column
),
created_timestamp_column=reverse_field_mapping.get(
created_timestamp_column, created_timestamp_column
),
# TODO: Make created column optional and not hardcoded
query=feature_view.input.query,
table_subquery=feature_view.input.get_table_query_string(),
entity_selections=entity_selections,
)
query_context.append(context)
return query_context
def get_expected_join_keys(project: str,
feature_views: List["feast.FeatureView"],
registry: Registry) -> Set[str]:
join_keys = set()
for feature_view in feature_views:
entities = feature_view.entities
for entity_name in entities:
entity = registry.get_entity(entity_name, project)
join_key = feature_view.projection.join_key_map.get(
entity.join_key, entity.join_key)
join_keys.add(join_key)
return join_keys
def materialize_single_feature_view(
self,
config: RepoConfig,
feature_view: FeatureView,
start_date: datetime,
end_date: datetime,
registry: Registry,
project: str,
tqdm_builder: Callable[[int], tqdm],
) -> None:
set_usage_attribute("provider", self.__class__.__name__)
entities = []
for entity_name in feature_view.entities:
entities.append(registry.get_entity(entity_name, project))
(
join_key_columns,
feature_name_columns,
event_timestamp_column,
created_timestamp_column,
) = _get_column_names(feature_view, entities)
offline_job = self.offline_store.pull_latest_from_table_or_query(
config=config,
data_source=feature_view.batch_source,
join_key_columns=join_key_columns,
feature_name_columns=feature_name_columns,
event_timestamp_column=event_timestamp_column,
created_timestamp_column=created_timestamp_column,
start_date=start_date,
end_date=end_date,
)
table = offline_job.to_arrow()
if feature_view.batch_source.field_mapping is not None:
table = _run_field_mapping(table,
feature_view.batch_source.field_mapping)
join_keys = {entity.join_key: entity.value_type for entity in entities}
with tqdm_builder(table.num_rows) as pbar:
for batch in table.to_batches(DEFAULT_BATCH_SIZE):
rows_to_write = _convert_arrow_to_proto(
batch, feature_view, join_keys)
self.online_write_batch(
self.repo_config,
feature_view,
rows_to_write,
lambda x: pbar.update(x),
)
def materialize_single_feature_view(
self,
feature_view: FeatureView,
start_date: datetime,
end_date: datetime,
registry: Registry,
project: str,
tqdm_builder: Callable[[int], tqdm],
) -> None:
entities = []
for entity_name in feature_view.entities:
entities.append(registry.get_entity(entity_name, project))
(
join_key_columns,
feature_name_columns,
event_timestamp_column,
created_timestamp_column,
) = _get_column_names(feature_view, entities)
start_date = utils.make_tzaware(start_date)
end_date = utils.make_tzaware(end_date)
table = self.offline_store.pull_latest_from_table_or_query(
data_source=feature_view.input,
join_key_columns=join_key_columns,
feature_name_columns=feature_name_columns,
event_timestamp_column=event_timestamp_column,
created_timestamp_column=created_timestamp_column,
start_date=start_date,
end_date=end_date,
)
if feature_view.input.field_mapping is not None:
table = _run_field_mapping(table, feature_view.input.field_mapping)
join_keys = [entity.join_key for entity in entities]
rows_to_write = _convert_arrow_to_proto(table, feature_view, join_keys)
with tqdm_builder(len(rows_to_write)) as pbar:
self.online_write_batch(
project, feature_view, rows_to_write, lambda x: pbar.update(x)
)
feature_view.materialization_intervals.append((start_date, end_date))
registry.apply_feature_view(feature_view, project)
def materialize_single_feature_view(
self,
feature_view: FeatureView,
start_date: datetime,
end_date: datetime,
registry: Registry,
project: str,
) -> None:
assert isinstance(feature_view.input, BigQuerySource)
entities = []
for entity_name in feature_view.entities:
entities.append(registry.get_entity(entity_name, project))
(
join_key_columns,
feature_name_columns,
event_timestamp_column,
created_timestamp_column,
) = _get_column_names(feature_view, entities)
start_date = utils.make_tzaware(start_date)
end_date = utils.make_tzaware(end_date)
offline_store = get_offline_store_from_sources([feature_view.input])
table = offline_store.pull_latest_from_table_or_query(
data_source=feature_view.input,
join_key_columns=join_key_columns,
feature_name_columns=feature_name_columns,
event_timestamp_column=event_timestamp_column,
created_timestamp_column=created_timestamp_column,
start_date=start_date,
end_date=end_date,
)
if feature_view.input.field_mapping is not None:
table = _run_field_mapping(table, feature_view.input.field_mapping)
join_keys = [entity.join_key for entity in entities]
rows_to_write = _convert_arrow_to_proto(table, feature_view, join_keys)
self.online_write_batch(project, feature_view, rows_to_write, None)
feature_view.materialization_intervals.append((start_date, end_date))
registry.apply_feature_view(feature_view, project)
def materialize_single_feature_view(
self,
config: RepoConfig,
feature_view: FeatureView,
start_date: datetime,
end_date: datetime,
registry: Registry,
project: str,
tqdm_builder: Callable[[int], tqdm],
) -> None:
entities = []
for entity_name in feature_view.entities:
entities.append(registry.get_entity(entity_name, project))
(
join_key_columns,
feature_name_columns,
event_timestamp_column,
created_timestamp_column,
) = _get_column_names(feature_view, entities)
offline_job = self.offline_store.pull_latest_from_table_or_query(
config=config,
data_source=feature_view.batch_source,
join_key_columns=join_key_columns,
feature_name_columns=feature_name_columns,
event_timestamp_column=event_timestamp_column,
created_timestamp_column=created_timestamp_column,
start_date=start_date,
end_date=end_date,
)
table = offline_job.to_arrow()
if feature_view.batch_source.field_mapping is not None:
table = _run_field_mapping(table,
feature_view.batch_source.field_mapping)
join_keys = [entity.join_key for entity in entities]
rows_to_write = _convert_arrow_to_proto(table, feature_view, join_keys)
with tqdm_builder(len(rows_to_write)) as pbar:
self.online_write_batch(self.repo_config, feature_view,
rows_to_write, lambda x: pbar.update(x))
class FeatureStore:
"""
A FeatureStore object is used to define, create, and retrieve features.
Args:
repo_path: Path to a `feature_store.yaml` used to configure the feature store
config (RepoConfig): Configuration object used to configure the feature store
"""
config: RepoConfig
repo_path: Path
_registry: Registry
@log_exceptions
def __init__(
self,
repo_path: Optional[str] = None,
config: Optional[RepoConfig] = None,
):
if repo_path is not None and config is not None:
raise ValueError("You cannot specify both repo_path and config")
if config is not None:
self.repo_path = Path(os.getcwd())
self.config = config
elif repo_path is not None:
self.repo_path = Path(repo_path)
self.config = load_repo_config(Path(repo_path))
else:
raise ValueError("Please specify one of repo_path or config")
registry_config = self.config.get_registry_config()
self._registry = Registry(
registry_path=registry_config.path,
repo_path=self.repo_path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
@log_exceptions
def version(self) -> str:
"""Returns the version of the current Feast SDK/CLI"""
return get_version()
@property
def project(self) -> str:
return self.config.project
def _get_provider(self) -> Provider:
# TODO: Bake self.repo_path into self.config so that we dont only have one interface to paths
return get_provider(self.config, self.repo_path)
@log_exceptions_and_usage
def refresh_registry(self):
"""Fetches and caches a copy of the feature registry in memory.
Explicitly calling this method allows for direct control of the state of the registry cache. Every time this
method is called the complete registry state will be retrieved from the remote registry store backend
(e.g., GCS, S3), and the cache timer will be reset. If refresh_registry() is run before get_online_features()
is called, then get_online_feature() will use the cached registry instead of retrieving (and caching) the
registry itself.
Additionally, the TTL for the registry cache can be set to infinity (by setting it to 0), which means that
refresh_registry() will become the only way to update the cached registry. If the TTL is set to a value
greater than 0, then once the cache becomes stale (more time than the TTL has passed), a new cache will be
downloaded synchronously, which may increase latencies if the triggering method is get_online_features()
"""
registry_config = self.config.get_registry_config()
self._registry = Registry(
registry_path=registry_config.path,
repo_path=self.repo_path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
self._registry.refresh()
@log_exceptions_and_usage
def list_entities(self, allow_cache: bool = False) -> List[Entity]:
"""
Retrieve a list of entities from the registry
Args:
allow_cache (bool): Whether to allow returning entities from a cached registry
Returns:
List of entities
"""
return self._registry.list_entities(self.project,
allow_cache=allow_cache)
@log_exceptions_and_usage
def list_feature_views(self) -> List[FeatureView]:
"""
Retrieve a list of feature views from the registry
Returns:
List of feature views
"""
return self._registry.list_feature_views(self.project)
@log_exceptions_and_usage
def get_entity(self, name: str) -> Entity:
"""
Retrieves an entity.
Args:
name: Name of entity
Returns:
Returns either the specified entity, or raises an exception if
none is found
"""
return self._registry.get_entity(name, self.project)
@log_exceptions_and_usage
def get_feature_view(self, name: str) -> FeatureView:
"""
Retrieves a feature view.
Args:
name: Name of feature view
Returns:
Returns either the specified feature view, or raises an exception if
none is found
"""
return self._registry.get_feature_view(name, self.project)
@log_exceptions_and_usage
def delete_feature_view(self, name: str):
"""
Deletes a feature view or raises an exception if not found.
Args:
name: Name of feature view
"""
return self._registry.delete_feature_view(name, self.project)
@log_exceptions_and_usage
def apply(self, objects: Union[Entity, FeatureView, List[Union[FeatureView,
Entity]]]):
"""Register objects to metadata store and update related infrastructure.
The apply method registers one or more definitions (e.g., Entity, FeatureView) and registers or updates these
objects in the Feast registry. Once the registry has been updated, the apply method will update related
infrastructure (e.g., create tables in an online store) in order to reflect these new definitions. All
operations are idempotent, meaning they can safely be rerun.
Args:
objects (List[Union[FeatureView, Entity]]): A list of FeatureView or Entity objects that should be
registered
Examples:
Register a single Entity and FeatureView.
>>> from feast.feature_store import FeatureStore
>>> from feast import Entity, FeatureView, Feature, ValueType, FileSource
>>> from datetime import timedelta
>>>
>>> fs = FeatureStore()
>>> customer_entity = Entity(name="customer", value_type=ValueType.INT64, description="customer entity")
>>> customer_feature_view = FeatureView(
>>> name="customer_fv",
>>> entities=["customer"],
>>> features=[Feature(name="age", dtype=ValueType.INT64)],
>>> input=FileSource(path="file.parquet", event_timestamp_column="timestamp"),
>>> ttl=timedelta(days=1)
>>> )
>>> fs.apply([customer_entity, customer_feature_view])
"""
# TODO: Add locking
# TODO: Optimize by only making a single call (read/write)
if isinstance(objects, Entity) or isinstance(objects, FeatureView):
objects = [objects]
assert isinstance(objects, list)
views_to_update = []
entities_to_update = []
for ob in objects:
if isinstance(ob, FeatureView):
self._registry.apply_feature_view(ob, project=self.project)
views_to_update.append(ob)
elif isinstance(ob, Entity):
self._registry.apply_entity(ob, project=self.project)
entities_to_update.append(ob)
else:
raise ValueError(
f"Unknown object type ({type(ob)}) provided as part of apply() call"
)
self._get_provider().update_infra(
project=self.project,
tables_to_delete=[],
tables_to_keep=views_to_update,
entities_to_delete=[],
entities_to_keep=entities_to_update,
partial=True,
)
@log_exceptions_and_usage
def get_historical_features(
self,
entity_df: Union[pd.DataFrame, str],
feature_refs: List[str],
) -> RetrievalJob:
"""Enrich an entity dataframe with historical feature values for either training or batch scoring.
This method joins historical feature data from one or more feature views to an entity dataframe by using a time
travel join.
Each feature view is joined to the entity dataframe using all entities configured for the respective feature
view. All configured entities must be available in the entity dataframe. Therefore, the entity dataframe must
contain all entities found in all feature views, but the individual feature views can have different entities.
Time travel is based on the configured TTL for each feature view. A shorter TTL will limit the
amount of scanning that will be done in order to find feature data for a specific entity key. Setting a short
TTL may result in null values being returned.
Args:
entity_df (Union[pd.DataFrame, str]): An entity dataframe is a collection of rows containing all entity
columns (e.g., customer_id, driver_id) on which features need to be joined, as well as a event_timestamp
column used to ensure point-in-time correctness. Either a Pandas DataFrame can be provided or a string
SQL query. The query must be of a format supported by the configured offline store (e.g., BigQuery)
feature_refs: A list of features that should be retrieved from the offline store. Feature references are of
the format "feature_view:feature", e.g., "customer_fv:daily_transactions".
Returns:
RetrievalJob which can be used to materialize the results.
Examples:
Retrieve historical features using a BigQuery SQL entity dataframe
>>> from feast.feature_store import FeatureStore
>>>
>>> fs = FeatureStore(config=RepoConfig(provider="gcp"))
>>> retrieval_job = fs.get_historical_features(
>>> entity_df="SELECT event_timestamp, order_id, customer_id from gcp_project.my_ds.customer_orders",
>>> feature_refs=["customer:age", "customer:avg_orders_1d", "customer:avg_orders_7d"]
>>> )
>>> feature_data = job.to_df()
>>> model.fit(feature_data) # insert your modeling framework here.
"""
all_feature_views = self._registry.list_feature_views(
project=self.project)
try:
feature_views = _get_requested_feature_views(
feature_refs, all_feature_views)
except FeatureViewNotFoundException as e:
sys.exit(e)
provider = self._get_provider()
try:
job = provider.get_historical_features(
self.config,
feature_views,
feature_refs,
entity_df,
self._registry,
self.project,
)
except FeastProviderLoginError as e:
sys.exit(e)
return job
@log_exceptions_and_usage
def materialize_incremental(
self,
end_date: datetime,
feature_views: Optional[List[str]] = None,
) -> None:
"""
Materialize incremental new data from the offline store into the online store.
This method loads incremental new feature data up to the specified end time from either
the specified feature views, or all feature views if none are specified,
into the online store where it is available for online serving. The start time of
the interval materialized is either the most recent end time of a prior materialization or
(now - ttl) if no such prior materialization exists.
Args:
end_date (datetime): End date for time range of data to materialize into the online store
feature_views (List[str]): Optional list of feature view names. If selected, will only run
materialization for the specified feature views.
Examples:
Materialize all features into the online store up to 5 minutes ago.
>>> from datetime import datetime, timedelta
>>> from feast.feature_store import FeatureStore
>>>
>>> fs = FeatureStore(config=RepoConfig(provider="gcp", registry="gs://my-fs/", project="my_fs_proj"))
>>> fs.materialize_incremental(end_date=datetime.utcnow() - timedelta(minutes=5))
"""
feature_views_to_materialize = []
if feature_views is None:
feature_views_to_materialize = self._registry.list_feature_views(
self.project)
else:
for name in feature_views:
feature_view = self._registry.get_feature_view(
name, self.project)
feature_views_to_materialize.append(feature_view)
_print_materialization_log(
None,
end_date,
len(feature_views_to_materialize),
self.config.online_store.type,
)
# TODO paging large loads
for feature_view in feature_views_to_materialize:
start_date = feature_view.most_recent_end_time
if start_date is None:
if feature_view.ttl is None:
raise Exception(
f"No start time found for feature view {feature_view.name}. materialize_incremental() requires"
f" either a ttl to be set or for materialize() to have been run at least once."
)
start_date = datetime.utcnow() - feature_view.ttl
provider = self._get_provider()
print(
f"{Style.BRIGHT + Fore.GREEN}{feature_view.name}{Style.RESET_ALL}"
f" from {Style.BRIGHT + Fore.GREEN}{start_date.replace(microsecond=0).astimezone()}{Style.RESET_ALL}"
f" to {Style.BRIGHT + Fore.GREEN}{end_date.replace(microsecond=0).astimezone()}{Style.RESET_ALL}:"
)
def tqdm_builder(length):
return tqdm(total=length, ncols=100)
provider.materialize_single_feature_view(
feature_view,
start_date,
end_date,
self._registry,
self.project,
tqdm_builder,
)
@log_exceptions_and_usage
def materialize(
self,
start_date: datetime,
end_date: datetime,
feature_views: Optional[List[str]] = None,
) -> None:
"""
Materialize data from the offline store into the online store.
This method loads feature data in the specified interval from either
the specified feature views, or all feature views if none are specified,
into the online store where it is available for online serving.
Args:
start_date (datetime): Start date for time range of data to materialize into the online store
end_date (datetime): End date for time range of data to materialize into the online store
feature_views (List[str]): Optional list of feature view names. If selected, will only run
materialization for the specified feature views.
Examples:
Materialize all features into the online store over the interval
from 3 hours ago to 10 minutes ago.
>>> from datetime import datetime, timedelta
>>> from feast.feature_store import FeatureStore
>>>
>>> fs = FeatureStore(config=RepoConfig(provider="gcp"))
>>> fs.materialize(
>>> start_date=datetime.utcnow() - timedelta(hours=3), end_date=datetime.utcnow() - timedelta(minutes=10)
>>> )
"""
if utils.make_tzaware(start_date) > utils.make_tzaware(end_date):
raise ValueError(
f"The given start_date {start_date} is greater than the given end_date {end_date}."
)
feature_views_to_materialize = []
if feature_views is None:
feature_views_to_materialize = self._registry.list_feature_views(
self.project)
else:
for name in feature_views:
feature_view = self._registry.get_feature_view(
name, self.project)
feature_views_to_materialize.append(feature_view)
_print_materialization_log(
start_date,
end_date,
len(feature_views_to_materialize),
self.config.online_store.type,
)
# TODO paging large loads
for feature_view in feature_views_to_materialize:
provider = self._get_provider()
print(
f"{Style.BRIGHT + Fore.GREEN}{feature_view.name}{Style.RESET_ALL}:"
)
def tqdm_builder(length):
return tqdm(total=length, ncols=100)
provider.materialize_single_feature_view(
feature_view,
start_date,
end_date,
self._registry,
self.project,
tqdm_builder,
)
@log_exceptions_and_usage
def get_online_features(
self,
feature_refs: List[str],
entity_rows: List[Dict[str, Any]],
) -> OnlineResponse:
"""
Retrieves the latest online feature data.
Note: This method will download the full feature registry the first time it is run. If you are using a
remote registry like GCS or S3 then that may take a few seconds. The registry remains cached up to a TTL
duration (which can be set to infinitey). If the cached registry is stale (more time than the TTL has
passed), then a new registry will be downloaded synchronously by this method. This download may
introduce latency to online feature retrieval. In order to avoid synchronous downloads, please call
refresh_registry() prior to the TTL being reached. Remember it is possible to set the cache TTL to
infinity (cache forever).
Args:
feature_refs: List of feature references that will be returned for each entity.
Each feature reference should have the following format:
"feature_table:feature" where "feature_table" & "feature" refer to
the feature and feature table names respectively.
Only the feature name is required.
entity_rows: A list of dictionaries where each key-value is an entity-name, entity-value pair.
Returns:
OnlineResponse containing the feature data in records.
Examples:
>>> from feast import FeatureStore
>>>
>>> store = FeatureStore(repo_path="...")
>>> feature_refs = ["sales:daily_transactions"]
>>> entity_rows = [{"customer_id": 0},{"customer_id": 1}]
>>>
>>> online_response = store.get_online_features(
>>> feature_refs, entity_rows)
>>> online_response_dict = online_response.to_dict()
>>> print(online_response_dict)
{'sales:daily_transactions': [1.1,1.2], 'sales:customer_id': [0,1]}
"""
provider = self._get_provider()
entities = self.list_entities(allow_cache=True)
entity_name_to_join_key_map = {}
for entity in entities:
entity_name_to_join_key_map[entity.name] = entity.join_key
join_key_rows = []
for row in entity_rows:
join_key_row = {}
for entity_name, entity_value in row.items():
try:
join_key = entity_name_to_join_key_map[entity_name]
except KeyError:
raise Exception(
f"Entity {entity_name} does not exist in project {self.project}"
)
join_key_row[join_key] = entity_value
join_key_rows.append(join_key_row)
entity_row_proto_list = _infer_online_entity_rows(join_key_rows)
union_of_entity_keys = []
result_rows: List[GetOnlineFeaturesResponse.FieldValues] = []
for entity_row_proto in entity_row_proto_list:
union_of_entity_keys.append(_entity_row_to_key(entity_row_proto))
result_rows.append(_entity_row_to_field_values(entity_row_proto))
all_feature_views = self._registry.list_feature_views(
project=self.project, allow_cache=True)
grouped_refs = _group_refs(feature_refs, all_feature_views)
for table, requested_features in grouped_refs:
entity_keys = _get_table_entity_keys(table, union_of_entity_keys,
entity_name_to_join_key_map)
read_rows = provider.online_read(
project=self.project,
table=table,
entity_keys=entity_keys,
)
for row_idx, read_row in enumerate(read_rows):
row_ts, feature_data = read_row
result_row = result_rows[row_idx]
if feature_data is None:
for feature_name in requested_features:
feature_ref = f"{table.name}__{feature_name}"
result_row.statuses[
feature_ref] = GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND
else:
for feature_name in feature_data:
feature_ref = f"{table.name}__{feature_name}"
if feature_name in requested_features:
result_row.fields[feature_ref].CopyFrom(
feature_data[feature_name])
result_row.statuses[
feature_ref] = GetOnlineFeaturesResponse.FieldStatus.PRESENT
return OnlineResponse(
GetOnlineFeaturesResponse(field_values=result_rows))
def test_commit():
fd, registry_path = mkstemp()
registry_config = RegistryConfig(path=registry_path, cache_ttl_seconds=600)
test_registry = Registry(registry_config, None)
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
labels={"team": "matchmaking"},
)
project = "project"
# Register Entity without commiting
test_registry.apply_entity(entity, project, commit=False)
# Retrieving the entity should still succeed
entities = test_registry.list_entities(project, allow_cache=True)
entity = entities[0]
assert (len(entities) == 1 and entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
entity = test_registry.get_entity("driver_car_id",
project,
allow_cache=True)
assert (entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
# Create new registry that points to the same store
registry_with_same_store = Registry(registry_config, None)
# Retrieving the entity should fail since the store is empty
entities = registry_with_same_store.list_entities(project)
assert len(entities) == 0
# commit from the original registry
test_registry.commit()
# Reconstruct the new registry in order to read the newly written store
registry_with_same_store = Registry(registry_config, None)
# Retrieving the entity should now succeed
entities = registry_with_same_store.list_entities(project)
entity = entities[0]
assert (len(entities) == 1 and entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
entity = test_registry.get_entity("driver_car_id", project)
assert (entity.name == "driver_car_id"
and entity.value_type == ValueType(ValueProto.ValueType.STRING)
and entity.description == "Car driver id"
and "team" in entity.labels
and entity.labels["team"] == "matchmaking")
test_registry.teardown()
# Will try to reload registry, which will fail because the file has been deleted
with pytest.raises(FileNotFoundError):
test_registry._get_registry_proto()
def get_feature_view_query_context(
feature_refs: List[str],
feature_views: List[FeatureView],
registry: Registry,
project: str,
entity_df_timestamp_range: Tuple[datetime, datetime],
) -> List[FeatureViewQueryContext]:
"""Build a query context containing all information required to template a BigQuery and Redshift point-in-time SQL query"""
(
feature_views_to_feature_map,
on_demand_feature_views_to_features,
) = _get_requested_feature_views_to_features_dict(
feature_refs, feature_views,
registry.list_on_demand_feature_views(project))
query_context = []
for feature_view, features in feature_views_to_feature_map.items():
join_keys = []
entity_selections = []
reverse_field_mapping = {
v: k
for k, v in feature_view.input.field_mapping.items()
}
for entity_name in feature_view.entities:
entity = registry.get_entity(entity_name, project)
join_key = feature_view.projection.join_key_map.get(
entity.join_key, entity.join_key)
join_keys.append(join_key)
entity_selections.append(f"{entity.join_key} AS {join_key}")
if isinstance(feature_view.ttl, timedelta):
ttl_seconds = int(feature_view.ttl.total_seconds())
else:
ttl_seconds = 0
event_timestamp_column = feature_view.input.event_timestamp_column
created_timestamp_column = feature_view.input.created_timestamp_column
min_event_timestamp = None
if feature_view.ttl:
min_event_timestamp = to_naive_utc(entity_df_timestamp_range[0] -
feature_view.ttl).isoformat()
max_event_timestamp = to_naive_utc(
entity_df_timestamp_range[1]).isoformat()
context = FeatureViewQueryContext(
name=feature_view.projection.name_to_use(),
ttl=ttl_seconds,
entities=join_keys,
features=features,
event_timestamp_column=reverse_field_mapping.get(
event_timestamp_column, event_timestamp_column),
created_timestamp_column=reverse_field_mapping.get(
created_timestamp_column, created_timestamp_column),
# TODO: Make created column optional and not hardcoded
table_subquery=feature_view.input.get_table_query_string(),
entity_selections=entity_selections,
min_event_timestamp=min_event_timestamp,
max_event_timestamp=max_event_timestamp,
)
query_context.append(context)
return query_context
class FeatureStore:
"""
A FeatureStore object is used to define, create, and retrieve features.
Args:
repo_path (optional): Path to a `feature_store.yaml` used to configure the
feature store.
config (optional): Configuration object used to configure the feature store.
"""
config: RepoConfig
repo_path: Path
_registry: Registry
@log_exceptions
def __init__(
self,
repo_path: Optional[str] = None,
config: Optional[RepoConfig] = None,
):
"""
Creates a FeatureStore object.
Raises:
ValueError: If both or neither of repo_path and config are specified.
"""
if repo_path is not None and config is not None:
raise ValueError("You cannot specify both repo_path and config.")
if config is not None:
self.repo_path = Path(os.getcwd())
self.config = config
elif repo_path is not None:
self.repo_path = Path(repo_path)
self.config = load_repo_config(Path(repo_path))
else:
raise ValueError("Please specify one of repo_path or config.")
registry_config = self.config.get_registry_config()
self._registry = Registry(
registry_path=registry_config.path,
repo_path=self.repo_path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
@log_exceptions
def version(self) -> str:
"""Returns the version of the current Feast SDK/CLI."""
return get_version()
@property
def registry(self) -> Registry:
"""Gets the registry of this feature store."""
return self._registry
@property
def project(self) -> str:
"""Gets the project of this feature store."""
return self.config.project
def _get_provider(self) -> Provider:
# TODO: Bake self.repo_path into self.config so that we dont only have one interface to paths
return get_provider(self.config, self.repo_path)
@log_exceptions_and_usage
def refresh_registry(self):
"""Fetches and caches a copy of the feature registry in memory.
Explicitly calling this method allows for direct control of the state of the registry cache. Every time this
method is called the complete registry state will be retrieved from the remote registry store backend
(e.g., GCS, S3), and the cache timer will be reset. If refresh_registry() is run before get_online_features()
is called, then get_online_feature() will use the cached registry instead of retrieving (and caching) the
registry itself.
Additionally, the TTL for the registry cache can be set to infinity (by setting it to 0), which means that
refresh_registry() will become the only way to update the cached registry. If the TTL is set to a value
greater than 0, then once the cache becomes stale (more time than the TTL has passed), a new cache will be
downloaded synchronously, which may increase latencies if the triggering method is get_online_features()
"""
registry_config = self.config.get_registry_config()
self._registry = Registry(
registry_path=registry_config.path,
repo_path=self.repo_path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
self._registry.refresh()
@log_exceptions_and_usage
def list_entities(self, allow_cache: bool = False) -> List[Entity]:
"""
Retrieves the list of entities from the registry.
Args:
allow_cache: Whether to allow returning entities from a cached registry.
Returns:
A list of entities.
"""
return self._registry.list_entities(self.project,
allow_cache=allow_cache)
@log_exceptions_and_usage
def list_feature_services(self) -> List[FeatureService]:
"""
Retrieves the list of feature services from the registry.
Returns:
A list of feature services.
"""
return self._registry.list_feature_services(self.project)
@log_exceptions_and_usage
def list_feature_views(self) -> List[FeatureView]:
"""
Retrieves the list of feature views from the registry.
Returns:
A list of feature views.
"""
return self._registry.list_feature_views(self.project)
@log_exceptions_and_usage
def get_entity(self, name: str) -> Entity:
"""
Retrieves an entity.
Args:
name: Name of entity.
Returns:
The specified entity.
Raises:
EntityNotFoundException: The entity could not be found.
"""
return self._registry.get_entity(name, self.project)
@log_exceptions_and_usage
def get_feature_service(self, name: str) -> FeatureService:
"""
Retrieves a feature service.
Args:
name: Name of feature service.
Returns:
The specified feature service.
Raises:
FeatureServiceNotFoundException: The feature service could not be found.
"""
return self._registry.get_feature_service(name, self.project)
@log_exceptions_and_usage
def get_feature_view(self, name: str) -> FeatureView:
"""
Retrieves a feature view.
Args:
name: Name of feature view.
Returns:
The specified feature view.
Raises:
FeatureViewNotFoundException: The feature view could not be found.
"""
return self._registry.get_feature_view(name, self.project)
@log_exceptions_and_usage
def delete_feature_view(self, name: str):
"""
Deletes a feature view.
Args:
name: Name of feature view.
Raises:
FeatureViewNotFoundException: The feature view could not be found.
"""
return self._registry.delete_feature_view(name, self.project)
@log_exceptions_and_usage
def delete_feature_service(self, name: str):
"""
Deletes a feature service.
Args:
name: Name of feature service.
Raises:
FeatureServiceNotFoundException: The feature view could not be found.
"""
return self._registry.delete_feature_service(name, self.project)
def _get_features(
self,
features: Optional[Union[List[str], FeatureService]],
feature_refs: Optional[List[str]],
) -> List[str]:
_features = features or feature_refs
if not _features:
raise ValueError("No features specified for retrieval")
_feature_refs: List[str]
if isinstance(_features, FeatureService):
# Get the latest value of the feature service, in case the object passed in has been updated underneath us.
_feature_refs = _get_feature_refs_from_feature_services(
self.get_feature_service(_features.name))
else:
_feature_refs = _features
return _feature_refs
@log_exceptions_and_usage
def apply(
self,
objects: Union[Entity, FeatureView, FeatureService,
List[Union[FeatureView, Entity, FeatureService]], ],
commit: bool = True,
):
"""Register objects to metadata store and update related infrastructure.
The apply method registers one or more definitions (e.g., Entity, FeatureView) and registers or updates these
objects in the Feast registry. Once the registry has been updated, the apply method will update related
infrastructure (e.g., create tables in an online store) in order to reflect these new definitions. All
operations are idempotent, meaning they can safely be rerun.
Args:
objects: A single object, or a list of objects that should be registered with the Feature Store.
commit: whether to commit changes to the registry
Raises:
ValueError: The 'objects' parameter could not be parsed properly.
Examples:
Register an Entity and a FeatureView.
>>> from feast import FeatureStore, Entity, FeatureView, Feature, ValueType, FileSource, RepoConfig
>>> from datetime import timedelta
>>> fs = FeatureStore(repo_path="feature_repo")
>>> driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id")
>>> driver_hourly_stats = FileSource(
... path="feature_repo/data/driver_stats.parquet",
... event_timestamp_column="event_timestamp",
... created_timestamp_column="created",
... )
>>> driver_hourly_stats_view = FeatureView(
... name="driver_hourly_stats",
... entities=["driver_id"],
... ttl=timedelta(seconds=86400 * 1),
... batch_source=driver_hourly_stats,
... )
>>> fs.apply([driver_hourly_stats_view, driver]) # register entity and feature view
"""
# TODO: Add locking
if not isinstance(objects, Iterable):
objects = [objects]
assert isinstance(objects, list)
views_to_update = [ob for ob in objects if isinstance(ob, FeatureView)]
_validate_feature_views(views_to_update)
entities_to_update = [ob for ob in objects if isinstance(ob, Entity)]
services_to_update = [
ob for ob in objects if isinstance(ob, FeatureService)
]
# Make inferences
update_entities_with_inferred_types_from_feature_views(
entities_to_update, views_to_update, self.config)
update_data_sources_with_inferred_event_timestamp_col(
[view.batch_source for view in views_to_update], self.config)
for view in views_to_update:
view.infer_features_from_batch_source(self.config)
if len(views_to_update) + len(entities_to_update) + len(
services_to_update) != len(objects):
raise ValueError(
"Unknown object type provided as part of apply() call")
for view in views_to_update:
self._registry.apply_feature_view(view,
project=self.project,
commit=False)
for ent in entities_to_update:
self._registry.apply_entity(ent,
project=self.project,
commit=False)
for feature_service in services_to_update:
self._registry.apply_feature_service(feature_service,
project=self.project)
self._get_provider().update_infra(
project=self.project,
tables_to_delete=[],
tables_to_keep=views_to_update,
entities_to_delete=[],
entities_to_keep=entities_to_update,
partial=True,
)
if commit:
self._registry.commit()
@log_exceptions_and_usage
def teardown(self):
"""Tears down all local and cloud resources for the feature store."""
tables: List[Union[FeatureView, FeatureTable]] = []
feature_views = self.list_feature_views()
feature_tables = self._registry.list_feature_tables(self.project)
tables.extend(feature_views)
tables.extend(feature_tables)
entities = self.list_entities()
self._get_provider().teardown_infra(self.project, tables, entities)
self._registry.teardown()
@log_exceptions_and_usage
def get_historical_features(
self,
entity_df: Union[pd.DataFrame, str],
features: Optional[Union[List[str], FeatureService]] = None,
feature_refs: Optional[List[str]] = None,
full_feature_names: bool = False,
) -> RetrievalJob:
"""Enrich an entity dataframe with historical feature values for either training or batch scoring.
This method joins historical feature data from one or more feature views to an entity dataframe by using a time
travel join.
Each feature view is joined to the entity dataframe using all entities configured for the respective feature
view. All configured entities must be available in the entity dataframe. Therefore, the entity dataframe must
contain all entities found in all feature views, but the individual feature views can have different entities.
Time travel is based on the configured TTL for each feature view. A shorter TTL will limit the
amount of scanning that will be done in order to find feature data for a specific entity key. Setting a short
TTL may result in null values being returned.
Args:
entity_df (Union[pd.DataFrame, str]): An entity dataframe is a collection of rows containing all entity
columns (e.g., customer_id, driver_id) on which features need to be joined, as well as a event_timestamp
column used to ensure point-in-time correctness. Either a Pandas DataFrame can be provided or a string
SQL query. The query must be of a format supported by the configured offline store (e.g., BigQuery)
features: A list of features, that should be retrieved from the offline store.
Either a list of string feature references can be provided or a FeatureService object.
Feature references are of the format "feature_view:feature", e.g., "customer_fv:daily_transactions".
full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names,
changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to
"customer_fv__daily_transactions"). By default, this value is set to False.
Returns:
RetrievalJob which can be used to materialize the results.
Raises:
ValueError: Both or neither of features and feature_refs are specified.
Examples:
Retrieve historical features from a local offline store.
>>> from feast import FeatureStore, RepoConfig
>>> import pandas as pd
>>> fs = FeatureStore(repo_path="feature_repo")
>>> entity_df = pd.DataFrame.from_dict(
... {
... "driver_id": [1001, 1002],
... "event_timestamp": [
... datetime(2021, 4, 12, 10, 59, 42),
... datetime(2021, 4, 12, 8, 12, 10),
... ],
... }
... )
>>> retrieval_job = fs.get_historical_features(
... entity_df=entity_df,
... features=[
... "driver_hourly_stats:conv_rate",
... "driver_hourly_stats:acc_rate",
... "driver_hourly_stats:avg_daily_trips",
... ],
... )
>>> feature_data = retrieval_job.to_df()
"""
if (features is not None
and feature_refs is not None) or (features is None
and feature_refs is None):
raise ValueError(
"You must specify exactly one of features and feature_refs.")
if feature_refs:
warnings.warn(
("The argument 'feature_refs' is being deprecated. Please use 'features' "
"instead. Feast 0.13 and onwards will not support the argument 'feature_refs'."
),
DeprecationWarning,
)
_feature_refs = self._get_features(features, feature_refs)
all_feature_views = self._registry.list_feature_views(
project=self.project)
feature_views = list(view for view, _ in _group_feature_refs(
_feature_refs, all_feature_views))
_validate_feature_refs(_feature_refs, full_feature_names)
provider = self._get_provider()
job = provider.get_historical_features(
self.config,
feature_views,
_feature_refs,
entity_df,
self._registry,
self.project,
full_feature_names,
)
return job
@log_exceptions_and_usage
def materialize_incremental(
self,
end_date: datetime,
feature_views: Optional[List[str]] = None,
) -> None:
"""
Materialize incremental new data from the offline store into the online store.
This method loads incremental new feature data up to the specified end time from either
the specified feature views, or all feature views if none are specified,
into the online store where it is available for online serving. The start time of
the interval materialized is either the most recent end time of a prior materialization or
(now - ttl) if no such prior materialization exists.
Args:
end_date (datetime): End date for time range of data to materialize into the online store
feature_views (List[str]): Optional list of feature view names. If selected, will only run
materialization for the specified feature views.
Raises:
Exception: A feature view being materialized does not have a TTL set.
Examples:
Materialize all features into the online store up to 5 minutes ago.
>>> from feast import FeatureStore, RepoConfig
>>> from datetime import datetime, timedelta
>>> fs = FeatureStore(repo_path="feature_repo")
>>> fs.materialize_incremental(end_date=datetime.utcnow() - timedelta(minutes=5))
Materializing...
<BLANKLINE>
...
"""
feature_views_to_materialize = []
if feature_views is None:
feature_views_to_materialize = self._registry.list_feature_views(
self.project)
else:
for name in feature_views:
feature_view = self._registry.get_feature_view(
name, self.project)
feature_views_to_materialize.append(feature_view)
_print_materialization_log(
None,
end_date,
len(feature_views_to_materialize),
self.config.online_store.type,
)
# TODO paging large loads
for feature_view in feature_views_to_materialize:
start_date = feature_view.most_recent_end_time
if start_date is None:
if feature_view.ttl is None:
raise Exception(
f"No start time found for feature view {feature_view.name}. materialize_incremental() requires"
f" either a ttl to be set or for materialize() to have been run at least once."
)
start_date = datetime.utcnow() - feature_view.ttl
provider = self._get_provider()
print(
f"{Style.BRIGHT + Fore.GREEN}{feature_view.name}{Style.RESET_ALL}"
f" from {Style.BRIGHT + Fore.GREEN}{start_date.replace(microsecond=0).astimezone()}{Style.RESET_ALL}"
f" to {Style.BRIGHT + Fore.GREEN}{end_date.replace(microsecond=0).astimezone()}{Style.RESET_ALL}:"
)
def tqdm_builder(length):
return tqdm(total=length, ncols=100)
start_date = utils.make_tzaware(start_date)
end_date = utils.make_tzaware(end_date)
provider.materialize_single_feature_view(
config=self.config,
feature_view=feature_view,
start_date=start_date,
end_date=end_date,
registry=self._registry,
project=self.project,
tqdm_builder=tqdm_builder,
)
self._registry.apply_materialization(feature_view, self.project,
start_date, end_date)
@log_exceptions_and_usage
def materialize(
self,
start_date: datetime,
end_date: datetime,
feature_views: Optional[List[str]] = None,
) -> None:
"""
Materialize data from the offline store into the online store.
This method loads feature data in the specified interval from either
the specified feature views, or all feature views if none are specified,
into the online store where it is available for online serving.
Args:
start_date (datetime): Start date for time range of data to materialize into the online store
end_date (datetime): End date for time range of data to materialize into the online store
feature_views (List[str]): Optional list of feature view names. If selected, will only run
materialization for the specified feature views.
Examples:
Materialize all features into the online store over the interval
from 3 hours ago to 10 minutes ago.
>>> from feast import FeatureStore, RepoConfig
>>> from datetime import datetime, timedelta
>>> fs = FeatureStore(repo_path="feature_repo")
>>> fs.materialize(
... start_date=datetime.utcnow() - timedelta(hours=3), end_date=datetime.utcnow() - timedelta(minutes=10)
... )
Materializing...
<BLANKLINE>
...
"""
if utils.make_tzaware(start_date) > utils.make_tzaware(end_date):
raise ValueError(
f"The given start_date {start_date} is greater than the given end_date {end_date}."
)
feature_views_to_materialize = []
if feature_views is None:
feature_views_to_materialize = self._registry.list_feature_views(
self.project)
else:
for name in feature_views:
feature_view = self._registry.get_feature_view(
name, self.project)
feature_views_to_materialize.append(feature_view)
_print_materialization_log(
start_date,
end_date,
len(feature_views_to_materialize),
self.config.online_store.type,
)
# TODO paging large loads
for feature_view in feature_views_to_materialize:
provider = self._get_provider()
print(
f"{Style.BRIGHT + Fore.GREEN}{feature_view.name}{Style.RESET_ALL}:"
)
def tqdm_builder(length):
return tqdm(total=length, ncols=100)
start_date = utils.make_tzaware(start_date)
end_date = utils.make_tzaware(end_date)
provider.materialize_single_feature_view(
config=self.config,
feature_view=feature_view,
start_date=start_date,
end_date=end_date,
registry=self._registry,
project=self.project,
tqdm_builder=tqdm_builder,
)
self._registry.apply_materialization(feature_view, self.project,
start_date, end_date)
@log_exceptions_and_usage
def get_online_features(
self,
features: Union[List[str], FeatureService],
entity_rows: List[Dict[str, Any]],
feature_refs: Optional[List[str]] = None,
full_feature_names: bool = False,
) -> OnlineResponse:
"""
Retrieves the latest online feature data.
Note: This method will download the full feature registry the first time it is run. If you are using a
remote registry like GCS or S3 then that may take a few seconds. The registry remains cached up to a TTL
duration (which can be set to infinity). If the cached registry is stale (more time than the TTL has
passed), then a new registry will be downloaded synchronously by this method. This download may
introduce latency to online feature retrieval. In order to avoid synchronous downloads, please call
refresh_registry() prior to the TTL being reached. Remember it is possible to set the cache TTL to
infinity (cache forever).
Args:
features: List of feature references that will be returned for each entity.
Each feature reference should have the following format:
"feature_table:feature" where "feature_table" & "feature" refer to
the feature and feature table names respectively.
Only the feature name is required.
entity_rows: A list of dictionaries where each key-value is an entity-name, entity-value pair.
Returns:
OnlineResponse containing the feature data in records.
Raises:
Exception: No entity with the specified name exists.
Examples:
Materialize all features into the online store over the interval
from 3 hours ago to 10 minutes ago, and then retrieve these online features.
>>> from feast import FeatureStore, RepoConfig
>>> fs = FeatureStore(repo_path="feature_repo")
>>> online_response = fs.get_online_features(
... features=[
... "driver_hourly_stats:conv_rate",
... "driver_hourly_stats:acc_rate",
... "driver_hourly_stats:avg_daily_trips",
... ],
... entity_rows=[{"driver_id": 1001}, {"driver_id": 1002}, {"driver_id": 1003}, {"driver_id": 1004}],
... )
>>> online_response_dict = online_response.to_dict()
"""
_feature_refs = self._get_features(features, feature_refs)
provider = self._get_provider()
entities = self.list_entities(allow_cache=True)
entity_name_to_join_key_map = {}
for entity in entities:
entity_name_to_join_key_map[entity.name] = entity.join_key
join_key_rows = []
for row in entity_rows:
join_key_row = {}
for entity_name, entity_value in row.items():
try:
join_key = entity_name_to_join_key_map[entity_name]
except KeyError:
raise EntityNotFoundException(entity_name, self.project)
join_key_row[join_key] = entity_value
join_key_rows.append(join_key_row)
entity_row_proto_list = _infer_online_entity_rows(join_key_rows)
union_of_entity_keys = []
result_rows: List[GetOnlineFeaturesResponse.FieldValues] = []
for entity_row_proto in entity_row_proto_list:
union_of_entity_keys.append(_entity_row_to_key(entity_row_proto))
result_rows.append(_entity_row_to_field_values(entity_row_proto))
all_feature_views = self._registry.list_feature_views(
project=self.project, allow_cache=True)
_validate_feature_refs(_feature_refs, full_feature_names)
grouped_refs = _group_feature_refs(_feature_refs, all_feature_views)
for table, requested_features in grouped_refs:
entity_keys = _get_table_entity_keys(table, union_of_entity_keys,
entity_name_to_join_key_map)
read_rows = provider.online_read(
config=self.config,
table=table,
entity_keys=entity_keys,
requested_features=requested_features,
)
for row_idx, read_row in enumerate(read_rows):
row_ts, feature_data = read_row
result_row = result_rows[row_idx]
if feature_data is None:
for feature_name in requested_features:
feature_ref = (f"{table.name}__{feature_name}"
if full_feature_names else feature_name)
result_row.statuses[
feature_ref] = GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND
else:
for feature_name in feature_data:
feature_ref = (f"{table.name}__{feature_name}"
if full_feature_names else feature_name)
if feature_name in requested_features:
result_row.fields[feature_ref].CopyFrom(
feature_data[feature_name])
result_row.statuses[
feature_ref] = GetOnlineFeaturesResponse.FieldStatus.PRESENT
return OnlineResponse(
GetOnlineFeaturesResponse(field_values=result_rows))
@log_exceptions_and_usage
def serve(self, port: int) -> None:
"""Start the feature consumption server locally on a given port."""
feature_server.start_server(self, port)
