def apply_total(repo_config: RepoConfig, repo_path: Path): os.chdir(repo_path) sys.path.append("") project = repo_config.project registry = Registry(repo_config.metadata_store) repo = parse_repo(repo_path) for entity in repo.entities: registry.apply_entity(entity, project=project) repo_table_names = set(t.name for t in repo.feature_tables) for t in repo.feature_views: repo_table_names.add(t.name) tables_to_delete = [] for registry_table in registry.list_feature_tables(project=project): if registry_table.name not in repo_table_names: tables_to_delete.append(registry_table) views_to_delete = [] for registry_view in registry.list_feature_views(project=project): if registry_view.name not in repo_table_names: views_to_delete.append(registry_view) # Delete tables that should not exist for registry_table in tables_to_delete: registry.delete_feature_table(registry_table.name, project=project) # Create tables that should for table in repo.feature_tables: registry.apply_feature_table(table, project) # Delete views that should not exist for registry_view in views_to_delete: registry.delete_feature_view(registry_view.name, project=project) # Create views that should for view in repo.feature_views: registry.apply_feature_view(view, project) infra_provider = get_provider(repo_config) all_to_delete: List[Union[FeatureTable, FeatureView]] = [] all_to_delete.extend(tables_to_delete) all_to_delete.extend(views_to_delete) all_to_keep: List[Union[FeatureTable, FeatureView]] = [] all_to_keep.extend(repo.feature_tables) all_to_keep.extend(repo.feature_views) infra_provider.update_infra( project, tables_to_delete=all_to_delete, tables_to_keep=all_to_keep, partial=False, ) print("Done!")
def apply_total(repo_config: RepoConfig, repo_path: Path): os.chdir(repo_path) sys.path.append("") project = repo_config.project registry = Registry(repo_config.metadata_store) repo = parse_repo(repo_path) for entity in repo.entities: registry.apply_entity(entity, project=project) repo_table_names = set(t.name for t in repo.feature_tables) tables_to_delete = [] for registry_table in registry.list_feature_tables(project=project): if registry_table.name not in repo_table_names: tables_to_delete.append(registry_table) # Delete tables that should not exist for registry_table in tables_to_delete: registry.delete_feature_table(registry_table.name, project=project) for table in repo.feature_tables: registry.apply_feature_table(table, project) infra_provider = get_provider(repo_config) infra_provider.update_infra(project, tables_to_delete=tables_to_delete, tables_to_keep=repo.feature_tables) print("Done!")
def teardown(repo_config: RepoConfig, repo_path: Path): registry = Registry(repo_config.metadata_store) project = repo_config.project registry_tables: List[Union[FeatureTable, FeatureView]] = [] registry_tables.extend(registry.list_feature_tables(project=project)) registry_tables.extend(registry.list_feature_views(project=project)) infra_provider = get_provider(repo_config) infra_provider.teardown_infra(project, tables=registry_tables)
def teardown(repo_config: RepoConfig, repo_path: Path): registry_config = repo_config.get_registry_config() registry = Registry( registry_path=registry_config.path, cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds), ) project = repo_config.project registry_tables: List[Union[FeatureTable, FeatureView]] = [] registry_tables.extend(registry.list_feature_tables(project=project)) registry_tables.extend(registry.list_feature_views(project=project)) infra_provider = get_provider(repo_config) infra_provider.teardown_infra(project, tables=registry_tables)
def apply_total(repo_config: RepoConfig, repo_path: Path): from colorama import Fore, Style os.chdir(repo_path) sys.path.append("") registry_config = repo_config.get_registry_config() project = repo_config.project registry = Registry( registry_path=registry_config.path, repo_path=repo_path, cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds), ) registry._initialize_registry() sys.dont_write_bytecode = True repo = parse_repo(repo_path) sys.dont_write_bytecode = False for entity in repo.entities: registry.apply_entity(entity, project=project) click.echo( f"Registered entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL}" ) repo_table_names = set(t.name for t in repo.feature_tables) for t in repo.feature_views: repo_table_names.add(t.name) tables_to_delete = [] for registry_table in registry.list_feature_tables(project=project): if registry_table.name not in repo_table_names: tables_to_delete.append(registry_table) views_to_delete = [] for registry_view in registry.list_feature_views(project=project): if registry_view.name not in repo_table_names: views_to_delete.append(registry_view) # Delete tables that should not exist for registry_table in tables_to_delete: registry.delete_feature_table(registry_table.name, project=project) click.echo( f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL} from registry" ) # Create tables that should for table in repo.feature_tables: registry.apply_feature_table(table, project) click.echo( f"Registered feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL}" ) # Delete views that should not exist for registry_view in views_to_delete: registry.delete_feature_view(registry_view.name, project=project) click.echo( f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{registry_view.name}{Style.RESET_ALL} from registry" ) # Create views that should for view in repo.feature_views: registry.apply_feature_view(view, project) click.echo( f"Registered feature view {Style.BRIGHT + Fore.GREEN}{view.name}{Style.RESET_ALL}" ) infra_provider = get_provider(repo_config, repo_path) all_to_delete: List[Union[FeatureTable, FeatureView]] = [] all_to_delete.extend(tables_to_delete) all_to_delete.extend(views_to_delete) all_to_keep: List[Union[FeatureTable, FeatureView]] = [] all_to_keep.extend(repo.feature_tables) all_to_keep.extend(repo.feature_views) for name in [view.name for view in repo.feature_tables ] + [table.name for table in repo.feature_views]: click.echo( f"Deploying infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}" ) for name in [view.name for view in views_to_delete ] + [table.name for table in tables_to_delete]: click.echo( f"Removing infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}" ) infra_provider.update_infra( project, tables_to_delete=all_to_delete, tables_to_keep=all_to_keep, partial=False, )
def _get_provider(self) -> Provider: # TODO: Bake self.repo_path into self.config so that we dont only have one interface to paths return get_provider(self.config, self.repo_path)
def apply_total(repo_config: RepoConfig, repo_path: Path, skip_source_validation: bool): from colorama import Fore, Style os.chdir(repo_path) registry_config = repo_config.get_registry_config() project = repo_config.project if not is_valid_name(project): print( f"{project} is not valid. Project name should only have " f"alphanumerical values and underscores but not start with an underscore." ) sys.exit(1) registry = Registry( registry_path=registry_config.path, repo_path=repo_path, cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds), ) registry._initialize_registry() sys.dont_write_bytecode = True repo = parse_repo(repo_path) _validate_feature_views(repo.feature_views) data_sources = [t.batch_source for t in repo.feature_views] if not skip_source_validation: # Make sure the data source used by this feature view is supported by Feast for data_source in data_sources: data_source.validate(repo_config) # Make inferences update_entities_with_inferred_types_from_feature_views( repo.entities, repo.feature_views, repo_config) update_data_sources_with_inferred_event_timestamp_col( data_sources, repo_config) for view in repo.feature_views: view.infer_features_from_batch_source(repo_config) repo_table_names = set(t.name for t in repo.feature_tables) for t in repo.feature_views: repo_table_names.add(t.name) tables_to_delete = [] for registry_table in registry.list_feature_tables(project=project): if registry_table.name not in repo_table_names: tables_to_delete.append(registry_table) views_to_delete = [] for registry_view in registry.list_feature_views(project=project): if registry_view.name not in repo_table_names: views_to_delete.append(registry_view) sys.dont_write_bytecode = False for entity in repo.entities: registry.apply_entity(entity, project=project, commit=False) click.echo( f"Registered entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL}" ) # Delete tables that should not exist for registry_table in tables_to_delete: registry.delete_feature_table(registry_table.name, project=project, commit=False) click.echo( f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL} from registry" ) # Create tables that should for table in repo.feature_tables: registry.apply_feature_table(table, project, commit=False) click.echo( f"Registered feature table {Style.BRIGHT + Fore.GREEN}{table.name}{Style.RESET_ALL}" ) # Delete views that should not exist for registry_view in views_to_delete: registry.delete_feature_view(registry_view.name, project=project, commit=False) click.echo( f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{registry_view.name}{Style.RESET_ALL} from registry" ) # Create views that should exist for view in repo.feature_views: registry.apply_feature_view(view, project, commit=False) click.echo( f"Registered feature view {Style.BRIGHT + Fore.GREEN}{view.name}{Style.RESET_ALL}" ) registry.commit() apply_feature_services(registry, project, repo) infra_provider = get_provider(repo_config, repo_path) all_to_delete: List[Union[FeatureTable, FeatureView]] = [] all_to_delete.extend(tables_to_delete) all_to_delete.extend(views_to_delete) all_to_keep: List[Union[FeatureTable, FeatureView]] = [] all_to_keep.extend(repo.feature_tables) all_to_keep.extend(repo.feature_views) entities_to_delete: List[Entity] = [] repo_entities_names = set([e.name for e in repo.entities]) for registry_entity in registry.list_entities(project=project): if registry_entity.name not in repo_entities_names: entities_to_delete.append(registry_entity) entities_to_keep: List[Entity] = repo.entities for name in [view.name for view in repo.feature_tables ] + [table.name for table in repo.feature_views]: click.echo( f"Deploying infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}" ) for name in [view.name for view in views_to_delete ] + [table.name for table in tables_to_delete]: click.echo( f"Removing infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}" ) infra_provider.update_infra( project, tables_to_delete=all_to_delete, tables_to_keep=all_to_keep, entities_to_delete=entities_to_delete, entities_to_keep=entities_to_keep, partial=False, )
def _get_provider(self) -> Provider: return get_provider(self.config)
def apply_total(repo_config: RepoConfig, repo_path: Path, skip_source_validation: bool): from colorama import Fore, Style os.chdir(repo_path) store = FeatureStore(repo_path=str(repo_path)) project = store.project if not is_valid_name(project): print( f"{project} is not valid. Project name should only have " f"alphanumerical values and underscores but not start with an underscore." ) sys.exit(1) registry = store.registry registry._initialize_registry() sys.dont_write_bytecode = True repo = parse_repo(repo_path) _validate_feature_views(repo.feature_views) if not skip_source_validation: data_sources = [t.batch_source for t in repo.feature_views] # Make sure the data source used by this feature view is supported by Feast for data_source in data_sources: data_source.validate(store.config) entities_to_keep, entities_to_delete = _tag_registry_entities_for_keep_delete( project, registry, repo ) views_to_keep, views_to_delete = _tag_registry_views_for_keep_delete( project, registry, repo ) ( odfvs_to_keep, odfvs_to_delete, ) = _tag_registry_on_demand_feature_views_for_keep_delete(project, registry, repo) tables_to_keep, tables_to_delete = _tag_registry_tables_for_keep_delete( project, registry, repo ) (services_to_keep, services_to_delete,) = _tag_registry_services_for_keep_delete( project, registry, repo ) sys.dont_write_bytecode = False # Delete views that should not exist for registry_view in views_to_delete: registry.delete_feature_view(registry_view.name, project=project, commit=False) click.echo( f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{registry_view.name}{Style.RESET_ALL} from registry" ) # Delete feature services that should not exist for feature_service_to_delete in services_to_delete: registry.delete_feature_service( feature_service_to_delete.name, project=project, commit=False ) click.echo( f"Deleted feature service {Style.BRIGHT + Fore.GREEN}{feature_service_to_delete.name}{Style.RESET_ALL} " f"from registry" ) # Delete tables that should not exist for registry_table in tables_to_delete: registry.delete_feature_table( registry_table.name, project=project, commit=False ) click.echo( f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL} from registry" ) # TODO: delete entities from the registry too # Add / update views + entities + services all_to_apply: List[ Union[Entity, FeatureView, FeatureService, OnDemandFeatureView] ] = [] all_to_apply.extend(entities_to_keep) all_to_apply.extend(views_to_keep) all_to_apply.extend(services_to_keep) all_to_apply.extend(odfvs_to_keep) # TODO: delete odfvs store.apply(all_to_apply, commit=False) for entity in entities_to_keep: click.echo( f"Registered entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL}" ) for view in views_to_keep: click.echo( f"Registered feature view {Style.BRIGHT + Fore.GREEN}{view.name}{Style.RESET_ALL}" ) for odfv in odfvs_to_keep: click.echo( f"Registered on demand feature view {Style.BRIGHT + Fore.GREEN}{odfv.name}{Style.RESET_ALL}" ) for feature_service in services_to_keep: click.echo( f"Registered feature service {Style.BRIGHT + Fore.GREEN}{feature_service.name}{Style.RESET_ALL}" ) # Create tables that should exist for table in tables_to_keep: registry.apply_feature_table(table, project, commit=False) click.echo( f"Registered feature table {Style.BRIGHT + Fore.GREEN}{table.name}{Style.RESET_ALL}" ) infra_provider = get_provider(repo_config, repo_path) for name in [view.name for view in repo.feature_tables] + [ table.name for table in repo.feature_views ]: click.echo( f"Deploying infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}" ) for name in [view.name for view in views_to_delete] + [ table.name for table in tables_to_delete ]: click.echo( f"Removing infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}" ) # TODO: consider echoing also entities being deployed/removed all_to_delete: List[Union[FeatureTable, FeatureView]] = [] all_to_delete.extend(tables_to_delete) all_to_delete.extend(views_to_delete) all_to_keep: List[Union[FeatureTable, FeatureView]] = [] all_to_keep.extend(tables_to_keep) all_to_keep.extend(views_to_delete) infra_provider.update_infra( project, tables_to_delete=all_to_delete, tables_to_keep=all_to_keep, entities_to_delete=entities_to_delete, entities_to_keep=entities_to_keep, partial=False, ) # Commit the update to the registry only after successful infra update registry.commit()
def teardown(repo_config: RepoConfig, repo_path: Path): registry = Registry(repo_config.metadata_store) project = repo_config.project registry_tables = registry.list_feature_tables(project=project) infra_provider = get_provider(repo_config) infra_provider.teardown_infra(project, tables=registry_tables)