def apply_total(repo_config: RepoConfig, repo_path: Path):
os.chdir(repo_path)
sys.path.append("")
project = repo_config.project
registry = Registry(repo_config.metadata_store)
repo = parse_repo(repo_path)
for entity in repo.entities:
registry.apply_entity(entity, project=project)
repo_table_names = set(t.name for t in repo.feature_tables)
for t in repo.feature_views:
repo_table_names.add(t.name)
tables_to_delete = []
for registry_table in registry.list_feature_tables(project=project):
if registry_table.name not in repo_table_names:
tables_to_delete.append(registry_table)
views_to_delete = []
for registry_view in registry.list_feature_views(project=project):
if registry_view.name not in repo_table_names:
views_to_delete.append(registry_view)
# Delete tables that should not exist
for registry_table in tables_to_delete:
registry.delete_feature_table(registry_table.name, project=project)
# Create tables that should
for table in repo.feature_tables:
registry.apply_feature_table(table, project)
# Delete views that should not exist
for registry_view in views_to_delete:
registry.delete_feature_view(registry_view.name, project=project)
# Create views that should
for view in repo.feature_views:
registry.apply_feature_view(view, project)
infra_provider = get_provider(repo_config)
all_to_delete: List[Union[FeatureTable, FeatureView]] = []
all_to_delete.extend(tables_to_delete)
all_to_delete.extend(views_to_delete)
all_to_keep: List[Union[FeatureTable, FeatureView]] = []
all_to_keep.extend(repo.feature_tables)
all_to_keep.extend(repo.feature_views)
infra_provider.update_infra(
project,
tables_to_delete=all_to_delete,
tables_to_keep=all_to_keep,
partial=False,
)
print("Done!")
def apply_total(repo_config: RepoConfig, repo_path: Path):
os.chdir(repo_path)
sys.path.append("")
project = repo_config.project
registry = Registry(repo_config.metadata_store)
repo = parse_repo(repo_path)
for entity in repo.entities:
registry.apply_entity(entity, project=project)
repo_table_names = set(t.name for t in repo.feature_tables)
tables_to_delete = []
for registry_table in registry.list_feature_tables(project=project):
if registry_table.name not in repo_table_names:
tables_to_delete.append(registry_table)
# Delete tables that should not exist
for registry_table in tables_to_delete:
registry.delete_feature_table(registry_table.name, project=project)
for table in repo.feature_tables:
registry.apply_feature_table(table, project)
infra_provider = get_provider(repo_config)
infra_provider.update_infra(project,
tables_to_delete=tables_to_delete,
tables_to_keep=repo.feature_tables)
print("Done!")
def teardown(repo_config: RepoConfig, repo_path: Path):
registry = Registry(repo_config.metadata_store)
project = repo_config.project
registry_tables: List[Union[FeatureTable, FeatureView]] = []
registry_tables.extend(registry.list_feature_tables(project=project))
registry_tables.extend(registry.list_feature_views(project=project))
infra_provider = get_provider(repo_config)
infra_provider.teardown_infra(project, tables=registry_tables)
def teardown(repo_config: RepoConfig, repo_path: Path):
registry_config = repo_config.get_registry_config()
registry = Registry(
registry_path=registry_config.path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
project = repo_config.project
registry_tables: List[Union[FeatureTable, FeatureView]] = []
registry_tables.extend(registry.list_feature_tables(project=project))
registry_tables.extend(registry.list_feature_views(project=project))
infra_provider = get_provider(repo_config)
infra_provider.teardown_infra(project, tables=registry_tables)
def apply_total(repo_config: RepoConfig, repo_path: Path):
from colorama import Fore, Style
os.chdir(repo_path)
sys.path.append("")
registry_config = repo_config.get_registry_config()
project = repo_config.project
registry = Registry(
registry_path=registry_config.path,
repo_path=repo_path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
registry._initialize_registry()
sys.dont_write_bytecode = True
repo = parse_repo(repo_path)
sys.dont_write_bytecode = False
for entity in repo.entities:
registry.apply_entity(entity, project=project)
click.echo(
f"Registered entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL}"
)
repo_table_names = set(t.name for t in repo.feature_tables)
for t in repo.feature_views:
repo_table_names.add(t.name)
tables_to_delete = []
for registry_table in registry.list_feature_tables(project=project):
if registry_table.name not in repo_table_names:
tables_to_delete.append(registry_table)
views_to_delete = []
for registry_view in registry.list_feature_views(project=project):
if registry_view.name not in repo_table_names:
views_to_delete.append(registry_view)
# Delete tables that should not exist
for registry_table in tables_to_delete:
registry.delete_feature_table(registry_table.name, project=project)
click.echo(
f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL} from registry"
)
# Create tables that should
for table in repo.feature_tables:
registry.apply_feature_table(table, project)
click.echo(
f"Registered feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL}"
)
# Delete views that should not exist
for registry_view in views_to_delete:
registry.delete_feature_view(registry_view.name, project=project)
click.echo(
f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{registry_view.name}{Style.RESET_ALL} from registry"
)
# Create views that should
for view in repo.feature_views:
registry.apply_feature_view(view, project)
click.echo(
f"Registered feature view {Style.BRIGHT + Fore.GREEN}{view.name}{Style.RESET_ALL}"
)
infra_provider = get_provider(repo_config, repo_path)
all_to_delete: List[Union[FeatureTable, FeatureView]] = []
all_to_delete.extend(tables_to_delete)
all_to_delete.extend(views_to_delete)
all_to_keep: List[Union[FeatureTable, FeatureView]] = []
all_to_keep.extend(repo.feature_tables)
all_to_keep.extend(repo.feature_views)
for name in [view.name for view in repo.feature_tables
] + [table.name for table in repo.feature_views]:
click.echo(
f"Deploying infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
for name in [view.name for view in views_to_delete
] + [table.name for table in tables_to_delete]:
click.echo(
f"Removing infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
infra_provider.update_infra(
project,
tables_to_delete=all_to_delete,
tables_to_keep=all_to_keep,
partial=False,
)
def _get_provider(self) -> Provider:
# TODO: Bake self.repo_path into self.config so that we dont only have one interface to paths
return get_provider(self.config, self.repo_path)
def apply_total(repo_config: RepoConfig, repo_path: Path,
skip_source_validation: bool):
from colorama import Fore, Style
os.chdir(repo_path)
registry_config = repo_config.get_registry_config()
project = repo_config.project
if not is_valid_name(project):
print(
f"{project} is not valid. Project name should only have "
f"alphanumerical values and underscores but not start with an underscore."
)
sys.exit(1)
registry = Registry(
registry_path=registry_config.path,
repo_path=repo_path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
registry._initialize_registry()
sys.dont_write_bytecode = True
repo = parse_repo(repo_path)
_validate_feature_views(repo.feature_views)
data_sources = [t.batch_source for t in repo.feature_views]
if not skip_source_validation:
# Make sure the data source used by this feature view is supported by Feast
for data_source in data_sources:
data_source.validate(repo_config)
# Make inferences
update_entities_with_inferred_types_from_feature_views(
repo.entities, repo.feature_views, repo_config)
update_data_sources_with_inferred_event_timestamp_col(
data_sources, repo_config)
for view in repo.feature_views:
view.infer_features_from_batch_source(repo_config)
repo_table_names = set(t.name for t in repo.feature_tables)
for t in repo.feature_views:
repo_table_names.add(t.name)
tables_to_delete = []
for registry_table in registry.list_feature_tables(project=project):
if registry_table.name not in repo_table_names:
tables_to_delete.append(registry_table)
views_to_delete = []
for registry_view in registry.list_feature_views(project=project):
if registry_view.name not in repo_table_names:
views_to_delete.append(registry_view)
sys.dont_write_bytecode = False
for entity in repo.entities:
registry.apply_entity(entity, project=project, commit=False)
click.echo(
f"Registered entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL}"
)
# Delete tables that should not exist
for registry_table in tables_to_delete:
registry.delete_feature_table(registry_table.name,
project=project,
commit=False)
click.echo(
f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL} from registry"
)
# Create tables that should
for table in repo.feature_tables:
registry.apply_feature_table(table, project, commit=False)
click.echo(
f"Registered feature table {Style.BRIGHT + Fore.GREEN}{table.name}{Style.RESET_ALL}"
)
# Delete views that should not exist
for registry_view in views_to_delete:
registry.delete_feature_view(registry_view.name,
project=project,
commit=False)
click.echo(
f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{registry_view.name}{Style.RESET_ALL} from registry"
)
# Create views that should exist
for view in repo.feature_views:
registry.apply_feature_view(view, project, commit=False)
click.echo(
f"Registered feature view {Style.BRIGHT + Fore.GREEN}{view.name}{Style.RESET_ALL}"
)
registry.commit()
apply_feature_services(registry, project, repo)
infra_provider = get_provider(repo_config, repo_path)
all_to_delete: List[Union[FeatureTable, FeatureView]] = []
all_to_delete.extend(tables_to_delete)
all_to_delete.extend(views_to_delete)
all_to_keep: List[Union[FeatureTable, FeatureView]] = []
all_to_keep.extend(repo.feature_tables)
all_to_keep.extend(repo.feature_views)
entities_to_delete: List[Entity] = []
repo_entities_names = set([e.name for e in repo.entities])
for registry_entity in registry.list_entities(project=project):
if registry_entity.name not in repo_entities_names:
entities_to_delete.append(registry_entity)
entities_to_keep: List[Entity] = repo.entities
for name in [view.name for view in repo.feature_tables
] + [table.name for table in repo.feature_views]:
click.echo(
f"Deploying infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
for name in [view.name for view in views_to_delete
] + [table.name for table in tables_to_delete]:
click.echo(
f"Removing infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
infra_provider.update_infra(
project,
tables_to_delete=all_to_delete,
tables_to_keep=all_to_keep,
entities_to_delete=entities_to_delete,
entities_to_keep=entities_to_keep,
partial=False,
)
def _get_provider(self) -> Provider:
return get_provider(self.config)
def apply_total(repo_config: RepoConfig, repo_path: Path, skip_source_validation: bool):
from colorama import Fore, Style
os.chdir(repo_path)
store = FeatureStore(repo_path=str(repo_path))
project = store.project
if not is_valid_name(project):
print(
f"{project} is not valid. Project name should only have "
f"alphanumerical values and underscores but not start with an underscore."
)
sys.exit(1)
registry = store.registry
registry._initialize_registry()
sys.dont_write_bytecode = True
repo = parse_repo(repo_path)
_validate_feature_views(repo.feature_views)
if not skip_source_validation:
data_sources = [t.batch_source for t in repo.feature_views]
# Make sure the data source used by this feature view is supported by Feast
for data_source in data_sources:
data_source.validate(store.config)
entities_to_keep, entities_to_delete = _tag_registry_entities_for_keep_delete(
project, registry, repo
)
views_to_keep, views_to_delete = _tag_registry_views_for_keep_delete(
project, registry, repo
)
(
odfvs_to_keep,
odfvs_to_delete,
) = _tag_registry_on_demand_feature_views_for_keep_delete(project, registry, repo)
tables_to_keep, tables_to_delete = _tag_registry_tables_for_keep_delete(
project, registry, repo
)
(services_to_keep, services_to_delete,) = _tag_registry_services_for_keep_delete(
project, registry, repo
)
sys.dont_write_bytecode = False
# Delete views that should not exist
for registry_view in views_to_delete:
registry.delete_feature_view(registry_view.name, project=project, commit=False)
click.echo(
f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{registry_view.name}{Style.RESET_ALL} from registry"
)
# Delete feature services that should not exist
for feature_service_to_delete in services_to_delete:
registry.delete_feature_service(
feature_service_to_delete.name, project=project, commit=False
)
click.echo(
f"Deleted feature service {Style.BRIGHT + Fore.GREEN}{feature_service_to_delete.name}{Style.RESET_ALL} "
f"from registry"
)
# Delete tables that should not exist
for registry_table in tables_to_delete:
registry.delete_feature_table(
registry_table.name, project=project, commit=False
)
click.echo(
f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL} from registry"
)
# TODO: delete entities from the registry too
# Add / update views + entities + services
all_to_apply: List[
Union[Entity, FeatureView, FeatureService, OnDemandFeatureView]
] = []
all_to_apply.extend(entities_to_keep)
all_to_apply.extend(views_to_keep)
all_to_apply.extend(services_to_keep)
all_to_apply.extend(odfvs_to_keep)
# TODO: delete odfvs
store.apply(all_to_apply, commit=False)
for entity in entities_to_keep:
click.echo(
f"Registered entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL}"
)
for view in views_to_keep:
click.echo(
f"Registered feature view {Style.BRIGHT + Fore.GREEN}{view.name}{Style.RESET_ALL}"
)
for odfv in odfvs_to_keep:
click.echo(
f"Registered on demand feature view {Style.BRIGHT + Fore.GREEN}{odfv.name}{Style.RESET_ALL}"
)
for feature_service in services_to_keep:
click.echo(
f"Registered feature service {Style.BRIGHT + Fore.GREEN}{feature_service.name}{Style.RESET_ALL}"
)
# Create tables that should exist
for table in tables_to_keep:
registry.apply_feature_table(table, project, commit=False)
click.echo(
f"Registered feature table {Style.BRIGHT + Fore.GREEN}{table.name}{Style.RESET_ALL}"
)
infra_provider = get_provider(repo_config, repo_path)
for name in [view.name for view in repo.feature_tables] + [
table.name for table in repo.feature_views
]:
click.echo(
f"Deploying infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
for name in [view.name for view in views_to_delete] + [
table.name for table in tables_to_delete
]:
click.echo(
f"Removing infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
# TODO: consider echoing also entities being deployed/removed
all_to_delete: List[Union[FeatureTable, FeatureView]] = []
all_to_delete.extend(tables_to_delete)
all_to_delete.extend(views_to_delete)
all_to_keep: List[Union[FeatureTable, FeatureView]] = []
all_to_keep.extend(tables_to_keep)
all_to_keep.extend(views_to_delete)
infra_provider.update_infra(
project,
tables_to_delete=all_to_delete,
tables_to_keep=all_to_keep,
entities_to_delete=entities_to_delete,
entities_to_keep=entities_to_keep,
partial=False,
)
# Commit the update to the registry only after successful infra update
registry.commit()
def teardown(repo_config: RepoConfig, repo_path: Path):
registry = Registry(repo_config.metadata_store)
project = repo_config.project
registry_tables = registry.list_feature_tables(project=project)
infra_provider = get_provider(repo_config)
infra_provider.teardown_infra(project, tables=registry_tables)
