def create_or_update_policies(client: "ImmutaClient", config_root: str, dry_run: bool, debug: bool, type: str) -> bool: logging.info("Gathering existing policies") existing_policies = {} for policy in client.get_global_policies(): existing_policies[policy.name] = policy logging.debug(f"Existing policies: {existing_policies.keys()}") tagger = Tagger(config_root=config_root) policy_config = PolicyConfig(config_root=config_root) # if it is not subscription-only, do the data-policies work if type != "subscription": create_or_update_data_policies( client=client, dry_run=dry_run, tagger=tagger, existing_policies=existing_policies, policy_config=policy_config, ) # if it is not data-only, do the subscription-policies work if type != "data": create_or_update_subscription_policies( client=client, dry_run=dry_run, tagger=tagger, existing_policies=existing_policies, policy_config=policy_config, ) logging.info("Fin.") return True
def main(config_file: str, search_text: str, dry_run: bool, debug: bool): logging.basicConfig( format="[%(name)s][%(levelname)s][%(asctime)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=(logging.DEBUG if debug else logging.INFO), ) config = parse_config(config_file=config_file) client = get_client(**config) tagger = Tagger(config_root=config["config_root"]) logging.info("Making tags") tagger.make_tags(client) logging.info("Gathering data sources to tag") data_sources_to_tag = [] with Paginator(client.get_data_source_list, search_text=search_text) as paginator: for data_source in paginator: data_sources_to_tag.append({ "id": data_source["id"], "name": data_source["name"] }) progress_iterator = tqdm(data_sources_to_tag) for data_source in progress_iterator: progress_iterator.set_description( desc= f"Tagging ID: {data_source['id']}, Name: {data_source['name']} :") dictionary = client.get_data_source_dictionary(id=data_source["id"]) enriched_columns = tagger.enrich_columns_with_tagging( dictionary.metadata) if enriched_columns == dictionary.metadata: logging.debug( f"No change for data source: {data_source['name']}. Skipping.") continue logging.debug( f"Enriched columns for {data_source['name']}: {dictionary.dict()['metadata']}" ) logging.info( f"Change detected. Updating data source {data_source['name']}.") dictionary.metadata = enriched_columns if not dry_run: client.update_data_source_dictionary(id=data_source["id"], dictionary=dictionary) logging.info("FIN.")
def test_create_message_body_for_tag_creation( tagger: tg.Tagger, root_tag: str, children: List[str], expected: Dict[str, Any] ): assert ( tagger.create_message_body_for_tag_creation( root_tag=root_tag, children=children ) == expected )
def test_enrich_columns_with_tagging(tagger: tg.Tagger): columns = [ ds.DataSourceColumn(name="col_foo", dataType="", remoteType="", nullable=False), ds.DataSourceColumn(name="col_bar", dataType="", remoteType="", nullable=False), ds.DataSourceColumn(name="bad_col", dataType="", remoteType="", nullable=False), ] enriched_cols = tagger.enrich_columns_with_tagging(columns) assert len(enriched_cols) == len(columns) for col in enriched_cols: assert len(col.tags) == len(TAG_MAP.get(col.name, []))
def create_or_update_policies( client: "ImmutaClient", config_root: str, dry_run: bool, debug: bool ) -> bool: logging.info("Gathering existing policies") existing_policies = {} for policy in client.get_global_policies(): existing_policies[policy.name] = policy logging.debug(f"Existing policies: {existing_policies.keys()}") tagger = Tagger(config_root=config_root) logging.debug(f"Tag groups: {tagger.tag_groups.keys()}") progress_iterator = tqdm(tagger.tag_groups.keys()) for tag_name in progress_iterator: progress_iterator.set_description(desc=f"Tag: {tag_name}") iam_groups = tagger.tag_groups[tag_name] policy_name = f"{tag_name}_access_policy" policy = make_global_subscription_policy( policy_name=policy_name, tags=[tag_name], iam_groups=iam_groups, tagger=tagger, ) logging.debug(f"Policy to create/update: {policy.json()}") if policy_name in existing_policies.keys(): policy.id = existing_policies[policy_name].id logging.debug(f"Existing policy: {existing_policies[policy_name].json()}") if existing_policies[policy_name] == policy: logging.info(f"No change for policy {policy_name}. Skipping.") continue logging.info(f"Updating existing policy with name {policy_name}.") if not dry_run: client.update_global_policy( policy=policy, id=existing_policies[policy_name].id ) else: logging.info(f"Creating new policy with name {policy_name}.") if not dry_run: client.create_global_policy(policy=policy) logging.info("Fin.") return True
def test_get_tags_for_column(tagger: tg.Tagger, col: str, expected: List[str]): assert tagger.get_tags_for_column(column_name=col) == expected
def test_is_root_tag(tagger: tg.Tagger, tag: str, is_root: bool): assert tagger.is_root_tag(tag) == is_root