def test_update_data_sources_with_inferred_event_timestamp_col(
universal_data_sources):
(_, _, data_sources) = universal_data_sources
data_sources_copy = deepcopy(data_sources)
# remove defined timestamp_field to allow for inference
for data_source in data_sources_copy.values():
data_source.timestamp_field = None
data_source.event_timestamp_column = None
update_data_sources_with_inferred_event_timestamp_col(
data_sources_copy.values(),
RepoConfig(provider="local", project="test"),
)
actual_event_timestamp_cols = [
source.timestamp_field for source in data_sources_copy.values()
]
assert actual_event_timestamp_cols == ["event_timestamp"] * len(
data_sources_copy.values())
def test_update_data_sources_with_inferred_event_timestamp_col(
simple_dataset_1):
df_with_two_viable_timestamp_cols = simple_dataset_1.copy(deep=True)
df_with_two_viable_timestamp_cols["ts_2"] = simple_dataset_1["ts_1"]
with prep_file_source(df=simple_dataset_1) as file_source:
data_sources = [
file_source,
simple_bq_source_using_table_ref_arg(simple_dataset_1),
simple_bq_source_using_query_arg(simple_dataset_1),
]
update_data_sources_with_inferred_event_timestamp_col(data_sources)
actual_event_timestamp_cols = [
source.event_timestamp_column for source in data_sources
]
assert actual_event_timestamp_cols == ["ts_1", "ts_1", "ts_1"]
with prep_file_source(df=df_with_two_viable_timestamp_cols) as file_source:
with pytest.raises(RegistryInferenceFailure):
# two viable event_timestamp_columns
update_data_sources_with_inferred_event_timestamp_col(
[file_source])
def test_update_file_data_source_with_inferred_event_timestamp_col(
simple_dataset_1):
df_with_two_viable_timestamp_cols = simple_dataset_1.copy(deep=True)
df_with_two_viable_timestamp_cols["ts_2"] = simple_dataset_1["ts_1"]
with prep_file_source(df=simple_dataset_1) as file_source:
data_sources = [
file_source,
simple_bq_source_using_table_arg(simple_dataset_1),
simple_bq_source_using_query_arg(simple_dataset_1),
]
update_data_sources_with_inferred_event_timestamp_col(
data_sources, RepoConfig(provider="local", project="test"))
actual_event_timestamp_cols = [
source.timestamp_field for source in data_sources
]
assert actual_event_timestamp_cols == ["ts_1", "ts_1", "ts_1"]
with prep_file_source(df=df_with_two_viable_timestamp_cols) as file_source:
with pytest.raises(RegistryInferenceFailure):
# two viable timestamp_fields
update_data_sources_with_inferred_event_timestamp_col(
[file_source], RepoConfig(provider="local", project="test"))
def apply_total(repo_config: RepoConfig, repo_path: Path,
skip_source_validation: bool):
from colorama import Fore, Style
os.chdir(repo_path)
registry_config = repo_config.get_registry_config()
project = repo_config.project
if not is_valid_name(project):
print(
f"{project} is not valid. Project name should only have "
f"alphanumerical values and underscores but not start with an underscore."
)
sys.exit(1)
registry = Registry(
registry_path=registry_config.path,
repo_path=repo_path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
registry._initialize_registry()
sys.dont_write_bytecode = True
repo = parse_repo(repo_path)
_validate_feature_views(repo.feature_views)
data_sources = [t.batch_source for t in repo.feature_views]
if not skip_source_validation:
# Make sure the data source used by this feature view is supported by Feast
for data_source in data_sources:
data_source.validate(repo_config)
# Make inferences
update_entities_with_inferred_types_from_feature_views(
repo.entities, repo.feature_views, repo_config)
update_data_sources_with_inferred_event_timestamp_col(
data_sources, repo_config)
for view in repo.feature_views:
view.infer_features_from_batch_source(repo_config)
repo_table_names = set(t.name for t in repo.feature_tables)
for t in repo.feature_views:
repo_table_names.add(t.name)
tables_to_delete = []
for registry_table in registry.list_feature_tables(project=project):
if registry_table.name not in repo_table_names:
tables_to_delete.append(registry_table)
views_to_delete = []
for registry_view in registry.list_feature_views(project=project):
if registry_view.name not in repo_table_names:
views_to_delete.append(registry_view)
sys.dont_write_bytecode = False
for entity in repo.entities:
registry.apply_entity(entity, project=project, commit=False)
click.echo(
f"Registered entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL}"
)
# Delete tables that should not exist
for registry_table in tables_to_delete:
registry.delete_feature_table(registry_table.name,
project=project,
commit=False)
click.echo(
f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL} from registry"
)
# Create tables that should
for table in repo.feature_tables:
registry.apply_feature_table(table, project, commit=False)
click.echo(
f"Registered feature table {Style.BRIGHT + Fore.GREEN}{table.name}{Style.RESET_ALL}"
)
# Delete views that should not exist
for registry_view in views_to_delete:
registry.delete_feature_view(registry_view.name,
project=project,
commit=False)
click.echo(
f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{registry_view.name}{Style.RESET_ALL} from registry"
)
# Create views that should exist
for view in repo.feature_views:
registry.apply_feature_view(view, project, commit=False)
click.echo(
f"Registered feature view {Style.BRIGHT + Fore.GREEN}{view.name}{Style.RESET_ALL}"
)
registry.commit()
apply_feature_services(registry, project, repo)
infra_provider = get_provider(repo_config, repo_path)
all_to_delete: List[Union[FeatureTable, FeatureView]] = []
all_to_delete.extend(tables_to_delete)
all_to_delete.extend(views_to_delete)
all_to_keep: List[Union[FeatureTable, FeatureView]] = []
all_to_keep.extend(repo.feature_tables)
all_to_keep.extend(repo.feature_views)
entities_to_delete: List[Entity] = []
repo_entities_names = set([e.name for e in repo.entities])
for registry_entity in registry.list_entities(project=project):
if registry_entity.name not in repo_entities_names:
entities_to_delete.append(registry_entity)
entities_to_keep: List[Entity] = repo.entities
for name in [view.name for view in repo.feature_tables
] + [table.name for table in repo.feature_views]:
click.echo(
f"Deploying infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
for name in [view.name for view in views_to_delete
] + [table.name for table in tables_to_delete]:
click.echo(
f"Removing infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
infra_provider.update_infra(
project,
tables_to_delete=all_to_delete,
tables_to_keep=all_to_keep,
entities_to_delete=entities_to_delete,
entities_to_keep=entities_to_keep,
partial=False,
)
def apply(
self,
objects: Union[Entity, FeatureView, FeatureService,
List[Union[FeatureView, Entity, FeatureService]], ],
commit: bool = True,
):
"""Register objects to metadata store and update related infrastructure.
The apply method registers one or more definitions (e.g., Entity, FeatureView) and registers or updates these
objects in the Feast registry. Once the registry has been updated, the apply method will update related
infrastructure (e.g., create tables in an online store) in order to reflect these new definitions. All
operations are idempotent, meaning they can safely be rerun.
Args:
objects: A single object, or a list of objects that should be registered with the Feature Store.
commit: whether to commit changes to the registry
Raises:
ValueError: The 'objects' parameter could not be parsed properly.
Examples:
Register an Entity and a FeatureView.
>>> from feast import FeatureStore, Entity, FeatureView, Feature, ValueType, FileSource, RepoConfig
>>> from datetime import timedelta
>>> fs = FeatureStore(repo_path="feature_repo")
>>> driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id")
>>> driver_hourly_stats = FileSource(
... path="feature_repo/data/driver_stats.parquet",
... event_timestamp_column="event_timestamp",
... created_timestamp_column="created",
... )
>>> driver_hourly_stats_view = FeatureView(
... name="driver_hourly_stats",
... entities=["driver_id"],
... ttl=timedelta(seconds=86400 * 1),
... batch_source=driver_hourly_stats,
... )
>>> fs.apply([driver_hourly_stats_view, driver]) # register entity and feature view
"""
# TODO: Add locking
if not isinstance(objects, Iterable):
objects = [objects]
assert isinstance(objects, list)
views_to_update = [ob for ob in objects if isinstance(ob, FeatureView)]
_validate_feature_views(views_to_update)
entities_to_update = [ob for ob in objects if isinstance(ob, Entity)]
services_to_update = [
ob for ob in objects if isinstance(ob, FeatureService)
]
# Make inferences
update_entities_with_inferred_types_from_feature_views(
entities_to_update, views_to_update, self.config)
update_data_sources_with_inferred_event_timestamp_col(
[view.batch_source for view in views_to_update], self.config)
for view in views_to_update:
view.infer_features_from_batch_source(self.config)
if len(views_to_update) + len(entities_to_update) + len(
services_to_update) != len(objects):
raise ValueError(
"Unknown object type provided as part of apply() call")
for view in views_to_update:
self._registry.apply_feature_view(view,
project=self.project,
commit=False)
for ent in entities_to_update:
self._registry.apply_entity(ent,
project=self.project,
commit=False)
for feature_service in services_to_update:
self._registry.apply_feature_service(feature_service,
project=self.project)
self._get_provider().update_infra(
project=self.project,
tables_to_delete=[],
tables_to_keep=views_to_update,
entities_to_delete=[],
entities_to_keep=entities_to_update,
partial=True,
)
if commit:
self._registry.commit()
def apply(
self, objects: Union[Entity, FeatureView, List[Union[FeatureView, Entity]]]
):
"""Register objects to metadata store and update related infrastructure.
The apply method registers one or more definitions (e.g., Entity, FeatureView) and registers or updates these
objects in the Feast registry. Once the registry has been updated, the apply method will update related
infrastructure (e.g., create tables in an online store) in order to reflect these new definitions. All
operations are idempotent, meaning they can safely be rerun.
Args:
objects (List[Union[FeatureView, Entity]]): A list of FeatureView or Entity objects that should be
registered
Examples:
Register a single Entity and FeatureView.
>>> from feast.feature_store import FeatureStore
>>> from feast import Entity, FeatureView, Feature, ValueType, FileSource
>>> from datetime import timedelta
>>>
>>> fs = FeatureStore()
>>> customer_entity = Entity(name="customer", value_type=ValueType.INT64, description="customer entity")
>>> customer_feature_view = FeatureView(
>>> name="customer_fv",
>>> entities=["customer"],
>>> features=[Feature(name="age", dtype=ValueType.INT64)],
>>> input=FileSource(path="file.parquet", event_timestamp_column="timestamp"),
>>> ttl=timedelta(days=1)
>>> )
>>> fs.apply([customer_entity, customer_feature_view])
"""
# TODO: Add locking
# TODO: Optimize by only making a single call (read/write)
if isinstance(objects, Entity) or isinstance(objects, FeatureView):
objects = [objects]
assert isinstance(objects, list)
views_to_update = [ob for ob in objects if isinstance(ob, FeatureView)]
entities_to_update = [ob for ob in objects if isinstance(ob, Entity)]
# Make inferences
update_entities_with_inferred_types_from_feature_views(
entities_to_update, views_to_update
)
update_data_sources_with_inferred_event_timestamp_col(
[view.input for view in views_to_update]
)
for view in views_to_update:
view.infer_features_from_input_source()
if len(views_to_update) + len(entities_to_update) != len(objects):
raise ValueError("Unknown object type provided as part of apply() call")
for view in views_to_update:
self._registry.apply_feature_view(view, project=self.project)
for ent in entities_to_update:
self._registry.apply_entity(ent, project=self.project)
self._get_provider().update_infra(
project=self.project,
tables_to_delete=[],
tables_to_keep=views_to_update,
entities_to_delete=[],
entities_to_keep=entities_to_update,
partial=True,
)
def apply(
self,
objects: Union[Entity, FeatureView, FeatureService,
List[Union[FeatureView, Entity, FeatureService]], ],
):
"""Register objects to metadata store and update related infrastructure.
The apply method registers one or more definitions (e.g., Entity, FeatureView) and registers or updates these
objects in the Feast registry. Once the registry has been updated, the apply method will update related
infrastructure (e.g., create tables in an online store) in order to reflect these new definitions. All
operations are idempotent, meaning they can safely be rerun.
Args:
objects: A single object, or a list of objects that should be registered with the Feature Store.
Raises:
ValueError: The 'objects' parameter could not be parsed properly.
Examples:
Register a single Entity and FeatureView.
>>> from feast.feature_store import FeatureStore
>>> from feast import Entity, FeatureView, Feature, ValueType, FileSource
>>> from datetime import timedelta
>>>
>>> fs = FeatureStore()
>>> customer_entity = Entity(name="customer", value_type=ValueType.INT64, description="customer entity")
>>> customer_feature_view = FeatureView(
>>> name="customer_fv",
>>> entities=["customer"],
>>> features=[Feature(name="age", dtype=ValueType.INT64)],
>>> batch_source=FileSource(path="file.parquet", event_timestamp_column="timestamp"),
>>> ttl=timedelta(days=1)
>>> )
>>> fs.apply([customer_entity, customer_feature_view])
"""
# TODO: Add locking
if not isinstance(objects, Iterable):
objects = [objects]
assert isinstance(objects, list)
views_to_update = [ob for ob in objects if isinstance(ob, FeatureView)]
_validate_feature_views(views_to_update)
entities_to_update = [ob for ob in objects if isinstance(ob, Entity)]
services_to_update = [
ob for ob in objects if isinstance(ob, FeatureService)
]
# Make inferences
update_entities_with_inferred_types_from_feature_views(
entities_to_update, views_to_update, self.config)
update_data_sources_with_inferred_event_timestamp_col(
[view.batch_source for view in views_to_update], self.config)
for view in views_to_update:
view.infer_features_from_batch_source(self.config)
if len(views_to_update) + len(entities_to_update) + len(
services_to_update) != len(objects):
raise ValueError(
"Unknown object type provided as part of apply() call")
for view in views_to_update:
self._registry.apply_feature_view(view,
project=self.project,
commit=False)
for ent in entities_to_update:
self._registry.apply_entity(ent,
project=self.project,
commit=False)
for feature_service in services_to_update:
self._registry.apply_feature_service(feature_service,
project=self.project)
self._registry.commit()
self._get_provider().update_infra(
project=self.project,
tables_to_delete=[],
tables_to_keep=views_to_update,
entities_to_delete=[],
entities_to_keep=entities_to_update,
partial=True,
)