def add_table_to_database_container( self, dataset_urn: str, db_name: str) -> Iterable[MetadataWorkUnit]: database_container_key = self.gen_database_key(db_name) container_workunits = add_dataset_to_container( container_key=database_container_key, dataset_urn=dataset_urn, ) for wu in container_workunits: self.report.report_workunit(wu) yield wu
def add_table_to_schema_container( self, dataset_urn: str, db_name: str, schema: str) -> Iterable[Union[MetadataWorkUnit, SqlWorkUnit]]: schema_container_key = self.gen_schema_key(db_name, schema) container_workunits = add_dataset_to_container( container_key=schema_container_key, dataset_urn=dataset_urn, ) for wu in container_workunits: self.report.report_workunit(wu) yield wu
def create_container_hierarchy( self, table_data: TableData, dataset_urn: str) -> Iterable[MetadataWorkUnit]: logger.debug(f"Creating containers for {dataset_urn}") base_full_path = table_data.table_path parent_key = None if table_data.is_s3: bucket_name = get_bucket_name(table_data.table_path) bucket_key = self.gen_bucket_key(bucket_name) yield from self.create_emit_containers( container_key=bucket_key, name=bucket_name, sub_types=["S3 bucket"], parent_container_key=None, ) parent_key = bucket_key base_full_path = get_bucket_relative_path(table_data.table_path) parent_folder_path = (base_full_path[:base_full_path.rfind("/")] if base_full_path.rfind("/") != -1 else "") for folder in parent_folder_path.split("/"): abs_path = folder if parent_key: prefix: str = "" if isinstance(parent_key, S3BucketKey): prefix = parent_key.bucket_name elif isinstance(parent_key, FolderKey): prefix = parent_key.folder_abs_path abs_path = prefix + "/" + folder folder_key = self.gen_folder_key(abs_path) yield from self.create_emit_containers( container_key=folder_key, name=folder, sub_types=["Folder"], parent_container_key=parent_key, ) parent_key = folder_key if parent_key is None: logger.warning( f"Failed to associate Dataset ({dataset_urn}) with container") return yield from add_dataset_to_container(parent_key, dataset_urn)