def test_enrich_columns_with_tagging(tagger: tg.Tagger): columns = [ ds.DataSourceColumn(name="col_foo", dataType="", remoteType="", nullable=False), ds.DataSourceColumn(name="col_bar", dataType="", remoteType="", nullable=False), ds.DataSourceColumn(name="bad_col", dataType="", remoteType="", nullable=False), ] enriched_cols = tagger.enrich_columns_with_tagging(columns) assert len(enriched_cols) == len(columns) for col in enriched_cols: assert len(col.tags) == len(TAG_MAP.get(col.name, []))
def main(config_file: str, search_text: str, dry_run: bool, debug: bool): logging.basicConfig( format="[%(name)s][%(levelname)s][%(asctime)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=(logging.DEBUG if debug else logging.INFO), ) config = parse_config(config_file=config_file) client = get_client(**config) tagger = Tagger(config_root=config["config_root"]) logging.info("Making tags") tagger.make_tags(client) logging.info("Gathering data sources to tag") data_sources_to_tag = [] with Paginator(client.get_data_source_list, search_text=search_text) as paginator: for data_source in paginator: data_sources_to_tag.append({ "id": data_source["id"], "name": data_source["name"] }) progress_iterator = tqdm(data_sources_to_tag) for data_source in progress_iterator: progress_iterator.set_description( desc= f"Tagging ID: {data_source['id']}, Name: {data_source['name']} :") data_source_tags = tagger.get_tags_for_data_source( name=data_source["name"]) if data_source_tags: logging.debug(f"Adding data source tags to {data_source['name']}.") if not dry_run: client.tag_data_source(id=data_source["id"], tag_data=data_source_tags) dictionary = client.get_data_source_dictionary(id=data_source["id"]) enriched_columns = tagger.enrich_columns_with_tagging( dictionary.metadata) if enriched_columns == dictionary.metadata: logging.debug( f"No change to column tags for data source: {data_source['name']}. Skipping." ) continue logging.debug(f"Enriched columns for {data_source['name']}:" f" {dictionary.dict()['metadata']}") logging.info( f"Change detected to column tags. Updating data source {data_source['name']}'s data dictionary." ) dictionary.metadata = enriched_columns if not dry_run: client.update_data_source_dictionary(id=data_source["id"], dictionary=dictionary) logging.info("FIN.")