def create_or_update_policies(client: "ImmutaClient", config_root: str,
dry_run: bool, debug: bool, type: str) -> bool:
logging.info("Gathering existing policies")
existing_policies = {}
for policy in client.get_global_policies():
existing_policies[policy.name] = policy
logging.debug(f"Existing policies: {existing_policies.keys()}")
tagger = Tagger(config_root=config_root)
policy_config = PolicyConfig(config_root=config_root)
# if it is not subscription-only, do the data-policies work
if type != "subscription":
create_or_update_data_policies(
client=client,
dry_run=dry_run,
tagger=tagger,
existing_policies=existing_policies,
policy_config=policy_config,
)
# if it is not data-only, do the subscription-policies work
if type != "data":
create_or_update_subscription_policies(
client=client,
dry_run=dry_run,
tagger=tagger,
existing_policies=existing_policies,
policy_config=policy_config,
)
logging.info("Fin.")
return True
def main(config_file: str, search_text: str, dry_run: bool, debug: bool):
logging.basicConfig(
format="[%(name)s][%(levelname)s][%(asctime)s] %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=(logging.DEBUG if debug else logging.INFO),
)
config = parse_config(config_file=config_file)
client = get_client(**config)
tagger = Tagger(config_root=config["config_root"])
logging.info("Making tags")
tagger.make_tags(client)
logging.info("Gathering data sources to tag")
data_sources_to_tag = []
with Paginator(client.get_data_source_list,
search_text=search_text) as paginator:
for data_source in paginator:
data_sources_to_tag.append({
"id": data_source["id"],
"name": data_source["name"]
})
progress_iterator = tqdm(data_sources_to_tag)
for data_source in progress_iterator:
progress_iterator.set_description(
desc=
f"Tagging ID: {data_source['id']}, Name: {data_source['name']} :")
dictionary = client.get_data_source_dictionary(id=data_source["id"])
enriched_columns = tagger.enrich_columns_with_tagging(
dictionary.metadata)
if enriched_columns == dictionary.metadata:
logging.debug(
f"No change for data source: {data_source['name']}. Skipping.")
continue
logging.debug(
f"Enriched columns for {data_source['name']}: {dictionary.dict()['metadata']}"
)
logging.info(
f"Change detected. Updating data source {data_source['name']}.")
dictionary.metadata = enriched_columns
if not dry_run:
client.update_data_source_dictionary(id=data_source["id"],
dictionary=dictionary)
logging.info("FIN.")
def test_create_message_body_for_tag_creation(
tagger: tg.Tagger, root_tag: str, children: List[str], expected: Dict[str, Any]
):
assert (
tagger.create_message_body_for_tag_creation(
root_tag=root_tag, children=children
)
== expected
)
def test_enrich_columns_with_tagging(tagger: tg.Tagger):
columns = [
ds.DataSourceColumn(name="col_foo", dataType="", remoteType="", nullable=False),
ds.DataSourceColumn(name="col_bar", dataType="", remoteType="", nullable=False),
ds.DataSourceColumn(name="bad_col", dataType="", remoteType="", nullable=False),
]
enriched_cols = tagger.enrich_columns_with_tagging(columns)
assert len(enriched_cols) == len(columns)
for col in enriched_cols:
assert len(col.tags) == len(TAG_MAP.get(col.name, []))
def create_or_update_policies(
client: "ImmutaClient", config_root: str, dry_run: bool, debug: bool
) -> bool:
logging.info("Gathering existing policies")
existing_policies = {}
for policy in client.get_global_policies():
existing_policies[policy.name] = policy
logging.debug(f"Existing policies: {existing_policies.keys()}")
tagger = Tagger(config_root=config_root)
logging.debug(f"Tag groups: {tagger.tag_groups.keys()}")
progress_iterator = tqdm(tagger.tag_groups.keys())
for tag_name in progress_iterator:
progress_iterator.set_description(desc=f"Tag: {tag_name}")
iam_groups = tagger.tag_groups[tag_name]
policy_name = f"{tag_name}_access_policy"
policy = make_global_subscription_policy(
policy_name=policy_name,
tags=[tag_name],
iam_groups=iam_groups,
tagger=tagger,
)
logging.debug(f"Policy to create/update: {policy.json()}")
if policy_name in existing_policies.keys():
policy.id = existing_policies[policy_name].id
logging.debug(f"Existing policy: {existing_policies[policy_name].json()}")
if existing_policies[policy_name] == policy:
logging.info(f"No change for policy {policy_name}. Skipping.")
continue
logging.info(f"Updating existing policy with name {policy_name}.")
if not dry_run:
client.update_global_policy(
policy=policy, id=existing_policies[policy_name].id
)
else:
logging.info(f"Creating new policy with name {policy_name}.")
if not dry_run:
client.create_global_policy(policy=policy)
logging.info("Fin.")
return True
def test_get_tags_for_column(tagger: tg.Tagger, col: str, expected: List[str]):
assert tagger.get_tags_for_column(column_name=col) == expected
def test_is_root_tag(tagger: tg.Tagger, tag: str, is_root: bool):
assert tagger.is_root_tag(tag) == is_root