def apply_total(repo_config: RepoConfig, repo_path: Path):
os.chdir(repo_path)
sys.path.append("")
project = repo_config.project
registry = Registry(repo_config.metadata_store)
repo = parse_repo(repo_path)
for entity in repo.entities:
registry.apply_entity(entity, project=project)
repo_table_names = set(t.name for t in repo.feature_tables)
for t in repo.feature_views:
repo_table_names.add(t.name)
tables_to_delete = []
for registry_table in registry.list_feature_tables(project=project):
if registry_table.name not in repo_table_names:
tables_to_delete.append(registry_table)
views_to_delete = []
for registry_view in registry.list_feature_views(project=project):
if registry_view.name not in repo_table_names:
views_to_delete.append(registry_view)
# Delete tables that should not exist
for registry_table in tables_to_delete:
registry.delete_feature_table(registry_table.name, project=project)
# Create tables that should
for table in repo.feature_tables:
registry.apply_feature_table(table, project)
# Delete views that should not exist
for registry_view in views_to_delete:
registry.delete_feature_view(registry_view.name, project=project)
# Create views that should
for view in repo.feature_views:
registry.apply_feature_view(view, project)
infra_provider = get_provider(repo_config)
all_to_delete: List[Union[FeatureTable, FeatureView]] = []
all_to_delete.extend(tables_to_delete)
all_to_delete.extend(views_to_delete)
all_to_keep: List[Union[FeatureTable, FeatureView]] = []
all_to_keep.extend(repo.feature_tables)
all_to_keep.extend(repo.feature_views)
infra_provider.update_infra(
project,
tables_to_delete=all_to_delete,
tables_to_keep=all_to_keep,
partial=False,
)
print("Done!")
def apply_total(repo_config: RepoConfig, repo_path: Path):
os.chdir(repo_path)
sys.path.append("")
project = repo_config.project
registry = Registry(repo_config.metadata_store)
repo = parse_repo(repo_path)
for entity in repo.entities:
registry.apply_entity(entity, project=project)
repo_table_names = set(t.name for t in repo.feature_tables)
tables_to_delete = []
for registry_table in registry.list_feature_tables(project=project):
if registry_table.name not in repo_table_names:
tables_to_delete.append(registry_table)
# Delete tables that should not exist
for registry_table in tables_to_delete:
registry.delete_feature_table(registry_table.name, project=project)
for table in repo.feature_tables:
registry.apply_feature_table(table, project)
infra_provider = get_provider(repo_config)
infra_provider.update_infra(project,
tables_to_delete=tables_to_delete,
tables_to_keep=repo.feature_tables)
print("Done!")
def apply_total(repo_config: RepoConfig, repo_path: Path):
from colorama import Fore, Style
os.chdir(repo_path)
sys.path.append("")
registry_config = repo_config.get_registry_config()
project = repo_config.project
registry = Registry(
registry_path=registry_config.path,
repo_path=repo_path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
registry._initialize_registry()
sys.dont_write_bytecode = True
repo = parse_repo(repo_path)
sys.dont_write_bytecode = False
for entity in repo.entities:
registry.apply_entity(entity, project=project)
click.echo(
f"Registered entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL}"
)
repo_table_names = set(t.name for t in repo.feature_tables)
for t in repo.feature_views:
repo_table_names.add(t.name)
tables_to_delete = []
for registry_table in registry.list_feature_tables(project=project):
if registry_table.name not in repo_table_names:
tables_to_delete.append(registry_table)
views_to_delete = []
for registry_view in registry.list_feature_views(project=project):
if registry_view.name not in repo_table_names:
views_to_delete.append(registry_view)
# Delete tables that should not exist
for registry_table in tables_to_delete:
registry.delete_feature_table(registry_table.name, project=project)
click.echo(
f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL} from registry"
)
# Create tables that should
for table in repo.feature_tables:
registry.apply_feature_table(table, project)
click.echo(
f"Registered feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL}"
)
# Delete views that should not exist
for registry_view in views_to_delete:
registry.delete_feature_view(registry_view.name, project=project)
click.echo(
f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{registry_view.name}{Style.RESET_ALL} from registry"
)
# Create views that should
for view in repo.feature_views:
registry.apply_feature_view(view, project)
click.echo(
f"Registered feature view {Style.BRIGHT + Fore.GREEN}{view.name}{Style.RESET_ALL}"
)
infra_provider = get_provider(repo_config, repo_path)
all_to_delete: List[Union[FeatureTable, FeatureView]] = []
all_to_delete.extend(tables_to_delete)
all_to_delete.extend(views_to_delete)
all_to_keep: List[Union[FeatureTable, FeatureView]] = []
all_to_keep.extend(repo.feature_tables)
all_to_keep.extend(repo.feature_views)
for name in [view.name for view in repo.feature_tables
] + [table.name for table in repo.feature_views]:
click.echo(
f"Deploying infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
for name in [view.name for view in views_to_delete
] + [table.name for table in tables_to_delete]:
click.echo(
f"Removing infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
infra_provider.update_infra(
project,
tables_to_delete=all_to_delete,
tables_to_keep=all_to_keep,
partial=False,
)
def apply_total(repo_config: RepoConfig, repo_path: Path,
skip_source_validation: bool):
from colorama import Fore, Style
os.chdir(repo_path)
registry_config = repo_config.get_registry_config()
project = repo_config.project
if not is_valid_name(project):
print(
f"{project} is not valid. Project name should only have "
f"alphanumerical values and underscores but not start with an underscore."
)
sys.exit(1)
registry = Registry(
registry_path=registry_config.path,
repo_path=repo_path,
cache_ttl=timedelta(seconds=registry_config.cache_ttl_seconds),
)
registry._initialize_registry()
sys.dont_write_bytecode = True
repo = parse_repo(repo_path)
_validate_feature_views(repo.feature_views)
data_sources = [t.batch_source for t in repo.feature_views]
if not skip_source_validation:
# Make sure the data source used by this feature view is supported by Feast
for data_source in data_sources:
data_source.validate(repo_config)
# Make inferences
update_entities_with_inferred_types_from_feature_views(
repo.entities, repo.feature_views, repo_config)
update_data_sources_with_inferred_event_timestamp_col(
data_sources, repo_config)
for view in repo.feature_views:
view.infer_features_from_batch_source(repo_config)
repo_table_names = set(t.name for t in repo.feature_tables)
for t in repo.feature_views:
repo_table_names.add(t.name)
tables_to_delete = []
for registry_table in registry.list_feature_tables(project=project):
if registry_table.name not in repo_table_names:
tables_to_delete.append(registry_table)
views_to_delete = []
for registry_view in registry.list_feature_views(project=project):
if registry_view.name not in repo_table_names:
views_to_delete.append(registry_view)
sys.dont_write_bytecode = False
for entity in repo.entities:
registry.apply_entity(entity, project=project, commit=False)
click.echo(
f"Registered entity {Style.BRIGHT + Fore.GREEN}{entity.name}{Style.RESET_ALL}"
)
# Delete tables that should not exist
for registry_table in tables_to_delete:
registry.delete_feature_table(registry_table.name,
project=project,
commit=False)
click.echo(
f"Deleted feature table {Style.BRIGHT + Fore.GREEN}{registry_table.name}{Style.RESET_ALL} from registry"
)
# Create tables that should
for table in repo.feature_tables:
registry.apply_feature_table(table, project, commit=False)
click.echo(
f"Registered feature table {Style.BRIGHT + Fore.GREEN}{table.name}{Style.RESET_ALL}"
)
# Delete views that should not exist
for registry_view in views_to_delete:
registry.delete_feature_view(registry_view.name,
project=project,
commit=False)
click.echo(
f"Deleted feature view {Style.BRIGHT + Fore.GREEN}{registry_view.name}{Style.RESET_ALL} from registry"
)
# Create views that should exist
for view in repo.feature_views:
registry.apply_feature_view(view, project, commit=False)
click.echo(
f"Registered feature view {Style.BRIGHT + Fore.GREEN}{view.name}{Style.RESET_ALL}"
)
registry.commit()
apply_feature_services(registry, project, repo)
infra_provider = get_provider(repo_config, repo_path)
all_to_delete: List[Union[FeatureTable, FeatureView]] = []
all_to_delete.extend(tables_to_delete)
all_to_delete.extend(views_to_delete)
all_to_keep: List[Union[FeatureTable, FeatureView]] = []
all_to_keep.extend(repo.feature_tables)
all_to_keep.extend(repo.feature_views)
entities_to_delete: List[Entity] = []
repo_entities_names = set([e.name for e in repo.entities])
for registry_entity in registry.list_entities(project=project):
if registry_entity.name not in repo_entities_names:
entities_to_delete.append(registry_entity)
entities_to_keep: List[Entity] = repo.entities
for name in [view.name for view in repo.feature_tables
] + [table.name for table in repo.feature_views]:
click.echo(
f"Deploying infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
for name in [view.name for view in views_to_delete
] + [table.name for table in tables_to_delete]:
click.echo(
f"Removing infrastructure for {Style.BRIGHT + Fore.GREEN}{name}{Style.RESET_ALL}"
)
infra_provider.update_infra(
project,
tables_to_delete=all_to_delete,
tables_to_keep=all_to_keep,
entities_to_delete=entities_to_delete,
entities_to_keep=entities_to_keep,
partial=False,
)
