def apply_total(repo_config: RepoConfig, repo_path: Path): from colorama import Fore, Style os.chdir(repo_path) sys.path.append("") registry_config = repo_config.get_registry_config() project = repo_config.project if not_valid_name(project): print(f"{project} is not valid. Project name should only have " f"alphanumerical values and underscores.") sys.exit(1) registry = Registry( registry_path=registry_config.path, repo_path=repo_path, cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds), ) registry._initialize_registry() sys.dont_write_bytecode = True repo = parse_repo(repo_path) sys.dont_write_bytecode = False for entity in repo.entities: registry.apply_entity(entity, project=project) click.echo( f"Registered entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL}" ) repo_table_names = set(t.name for t in repo.feature_tables) for t in repo.feature_views: repo_table_names.add(t.name) data_sources = [t.input for t in repo.feature_views] # Make sure the data source used by this feature view is supported by for data_source in data_sources: assert_offline_store_supports_data_source(repo_config.offline_store, data_source) tables_to_delete = [] for registry_table in registry.list_feature_tables(project=project): if registry_table.name not in repo_table_names: tables_to_delete.append(registry_table) views_to_delete = [] for registry_view in registry.list_feature_views(project=project): if registry_view.name not in repo_table_names: views_to_delete.append(registry_view) # Delete tables that should not exist for registry_table in tables_to_delete: registry.delete_feature_table(registry_table.name, project=project) click.echo( f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL} from registry" ) # Create tables that should for table in repo.feature_tables: registry.apply_feature_table(table, project) click.echo( f"Registered feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL}" ) # Delete views that should not exist for registry_view in views_to_delete: registry.delete_feature_view(registry_view.name, project=project) click.echo( f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{registry_view.name}{Style.RESET_ALL} from registry" ) # Create views that should for view in repo.feature_views: registry.apply_feature_view(view, project) click.echo( f"Registered feature view {Style.BRIGHT + Fore.GREEN}{view.name}{Style.RESET_ALL}" ) infra_provider = get_provider(repo_config, repo_path) all_to_delete: List[Union[FeatureTable, FeatureView]] = [] all_to_delete.extend(tables_to_delete) all_to_delete.extend(views_to_delete) all_to_keep: List[Union[FeatureTable, FeatureView]] = [] all_to_keep.extend(repo.feature_tables) all_to_keep.extend(repo.feature_views) entities_to_delete: List[Entity] = [] repo_entities_names = set([e.name for e in repo.entities]) for registry_entity in registry.list_entities(project=project): if registry_entity.name not in repo_entities_names: entities_to_delete.append(registry_entity) entities_to_keep: List[Entity] = repo.entities for name in [view.name for view in repo.feature_tables ] + [table.name for table in repo.feature_views]: click.echo( f"Deploying infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}" ) for name in [view.name for view in views_to_delete ] + [table.name for table in tables_to_delete]: click.echo( f"Removing infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}" ) infra_provider.update_infra( project, tables_to_delete=all_to_delete, tables_to_keep=all_to_keep, entities_to_delete=entities_to_delete, entities_to_keep=entities_to_keep, partial=False, )
def apply_total(repo_config: RepoConfig, repo_path: Path): os.chdir(repo_path) sys.path.append("") registry_config = repo_config.get_registry_config() project = repo_config.project registry = Registry( registry_path=registry_config.path, cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds), ) repo = parse_repo(repo_path) for entity in repo.entities: registry.apply_entity(entity, project=project) repo_table_names = set(t.name for t in repo.feature_tables) for t in repo.feature_views: repo_table_names.add(t.name) tables_to_delete = [] for registry_table in registry.list_feature_tables(project=project): if registry_table.name not in repo_table_names: tables_to_delete.append(registry_table) views_to_delete = [] for registry_view in registry.list_feature_views(project=project): if registry_view.name not in repo_table_names: views_to_delete.append(registry_view) # Delete tables that should not exist for registry_table in tables_to_delete: registry.delete_feature_table(registry_table.name, project=project) # Create tables that should for table in repo.feature_tables: registry.apply_feature_table(table, project) # Delete views that should not exist for registry_view in views_to_delete: registry.delete_feature_view(registry_view.name, project=project) # Create views that should for view in repo.feature_views: registry.apply_feature_view(view, project) infra_provider = get_provider(repo_config) all_to_delete: List[Union[FeatureTable, FeatureView]] = [] all_to_delete.extend(tables_to_delete) all_to_delete.extend(views_to_delete) all_to_keep: List[Union[FeatureTable, FeatureView]] = [] all_to_keep.extend(repo.feature_tables) all_to_keep.extend(repo.feature_views) infra_provider.update_infra( project, tables_to_delete=all_to_delete, tables_to_keep=all_to_keep, partial=False, ) print("Done!")