def read(self):
"""Processes a single Bible."""
output_text_file = None
if FLAGS.output_text_file:
logging.info("Saving original text to \"%s\".",
FLAGS.output_text_file)
output_text_file = open(FLAGS.output_text_file,
"w",
encoding=_ENCODING)
output_data_file = None
if FLAGS.output_data_file:
logging.info("Saving the training/test data to \"%s\" ...",
FLAGS.output_data_file)
output_data_file = open(FLAGS.output_data_file,
"w",
encoding=_ENCODING)
test_ids = self._read_test_ids()
logging.info("Read %d test set verse IDs.", len(test_ids))
logging.info("Reading Bible from \"%s\" ...", FLAGS.input_xml_file)
with open(FLAGS.input_xml_file, "r", encoding=_ENCODING) as f:
root = et.fromstring(f.read())
num_sentences = 0
for n in root.iter("seg"):
num_sentences += 1
progress_bar = bar.IncrementalBar("Processing", max=num_sentences)
for n in root.iter("seg"):
if not n.text:
continue
sentence = n.text.strip()
sent_id = n.attrib["id"]
if FLAGS.verbose:
logging.info("%s: %s", sent_id, sentence)
# Simply save the original text.
if output_text_file:
output_text_file.write(n.text.strip() + "\n")
# Process and save the training/test data.
sentence = re.sub(_CLEANUP_RE, "", sentence)
word_prons, bad_prons = self._process_sentence(sentence)
if sent_id in test_ids:
sent_id = "test_" + sent_id
else:
sent_id = "train_" + sent_id
if bad_prons:
sent_id += "_NULLPRON"
if output_data_file:
output_data_file.write("%s\t%s\n" %
(sent_id, " ".join(word_prons)))
progress_bar.next()
# Cleanup.
progress_bar.finish()
if output_text_file:
output_text_file.close()
def make_migrations(role_arn: str, root_id: str) -> None:
with betterboto_client.CrossAccountClientContextManager(
"organizations",
role_arn,
f"organizations",
) as orgs_client:
progress = bar.IncrementalBar("Making migrations", max=2)
progress.next()
make_migrations_for_organizational_units(orgs_client, root_id)
progress.next()
make_migrations_for_accounts(orgs_client, root_id)
progress.finish()
def import_organization_policies(role_arn, root_id) -> None:
with betterboto_client.CrossAccountClientContextManager(
"organizations", role_arn, f"organizations"
) as organizations:
progress = bar.IncrementalBar("Importing SCPs", max=4)
progress.next()
remove_any_existing_policy_records(root_id)
progress.next()
save_all_policies_from_org(root_id, organizations)
progress.next()
save_targets_for_policy(root_id, organizations)
progress.next()
progress.finish()
def import_organization(role_arn, root_id) -> None:
remove_any_existing_records(root_id)
with betterboto_client.CrossAccountClientContextManager(
"organizations", role_arn, f"organizations"
) as organizations:
delegated_administrators = (
organizations.list_delegated_administrators_single_page().get(
"DelegatedAdministrators", []
)
)
progress = bar.IncrementalBar(
"Importing Delegated Administrators", max=len(delegated_administrators)
)
for delegated_administrator in delegated_administrators:
progress.next()
account_id = delegated_administrator.get("Id")
account_name = delegated_administrator.get("Name")
delegated_services = (
organizations.list_delegated_services_for_account_single_page(
AccountId=account_id
).get("DelegatedServices")
)
account_file = glob.glob(
f"environment/{root_id}/**/{account_name}/_meta.yaml",
recursive=True,
)
assert (
len(account_file) == 1
), "found more or less than 1 account_meta file when searching by name"
delegated_administrators_file = account_file[0].replace(
"_meta", "_delegated_administrators"
)
open(delegated_administrators_file, "w").write(
yaml.safe_dump(delegated_services)
)
def update_state(role_arn) -> None:
with betterboto_client.CrossAccountClientContextManager(
"organizations",
role_arn,
f"organizations",
) as organizations:
all_accounts = dict()
result = dict(accounts=all_accounts)
list_roots_response = organizations.list_roots_single_page()
tree = dict()
by_name = dict()
by_id = dict()
organizational_units = dict(tree=tree, by_name=by_name, by_id=by_id)
result["organizational_units"] = organizational_units
progress = bar.IncrementalBar("Adding roots",
max=len(
list_roots_response.get("Roots",
[])))
for root in list_roots_response.get("Roots", []):
progress.next()
root_id = str(root.get("Id"))
details = dict(
Type="Root",
Id=root_id,
Name="Root",
)
by_name["/"] = root_id
by_id[root_id] = dict(path="/", details=details)
tree[root_id] = dict(
details=details,
organizational_units=
get_children_details_filter_by_organizational_unit(
organizations, root_id, by_name, "", by_id),
policies=dict(service_control_policies=organizations.
list_policies_for_target(
TargetId=root_id,
Filter=SERVICE_CONTROL_POLICY,
).get("Policies", []), ),
)
progress.finish()
with open(STATE_FILE, "w") as f:
f.write(yaml.safe_dump(result))
accounts = organizations.list_accounts_single_page().get(
"Accounts", [])
progress = bar.IncrementalBar("Adding accounts", max=len(accounts))
counter = 1
for account in accounts:
progress.next()
account_id = account.get("Id")
all_accounts[account_id] = dict(
details=account,
parents=organizations.list_parents_single_page(
ChildId=account_id).get("Parents"),
policies=dict(service_control_policies=organizations.
list_policies_for_target(
TargetId=account_id,
Filter=SERVICE_CONTROL_POLICY,
).get("Policies", []), ),
)
counter += 1
progress.finish()
with open(STATE_FILE, "w") as f:
f.write(yaml.safe_dump(result))
return result
def migrate(root_id: str, role_arn: str, ssm_parameter_prefix: str) -> None:
with betterboto_client.CrossAccountClientContextManager(
"ssm",
role_arn,
f"ssm",
) as ssm:
progress = bar.IncrementalBar(
"Migrating",
max=len(os.listdir(f"environment/{root_id}/_migrations")))
for migration_file in sorted(
os.listdir(f"environment/{root_id}/_migrations")):
progress.next()
migration_id = migration_file.split(SEP)[-1].replace(".yaml", "")
try:
ssm.get_parameter(
Name=f"{ssm_parameter_prefix}/migrations/{migration_id}")
click.echo(f" Migration: {migration_id} already run")
except ssm.exceptions.ParameterNotFound:
# click.echo(
# f" Record of migration: {migration_id} being run not found, running now"
# )
migration = yaml.safe_load(
open(f"environment/{root_id}/_migrations/{migration_file}",
"r").read())
migration_extension = migration.get("extension")
migration_type = migration.get("migration_type")
migration_params = migration.get("migration_params")
if migration_extension == EXTENSION:
migration_function = migrations.get_function(
migration_type)
elif migration_extension == service_control_policies.EXTENSION:
migration_function = (service_control_policies.migrations.
get_function(migration_type))
try:
with betterboto_client.CrossAccountClientContextManager(
"organizations",
role_arn=role_arn,
role_session_name="ou_create",
) as client:
result, message = migration_function(
root_id, client, **migration_params)
except Exception as ex:
result = False
message = "Unhandled error: {0}".format(ex)
status = "Ok" if result else "FAILED"
click.echo(f"{migration_id}: {status} - {message}")
ssm.put_parameter(
Name=f"{ssm_parameter_prefix}/migrations/{migration_id}",
Description=f"Migration run: {datetime.utcnow()}",
Value=status if result else f"{status}: {message}",
Type="String",
Tags=[
{
"Key": "AWS-Organized:Actor",
"Value": "Framework"
},
],
)
progress.finish()
def save_targets_for_policy(root_id, organizations) -> None:
policies = glob.glob(
f"environment/{root_id}/_policies/service_control_policies/*/*.yaml"
)
state = yaml.safe_load(open("state.yaml", "r").read())
progress = bar.IncrementalBar("Importing policies", max=len(policies))
for policy_file in policies:
progress.next()
policy = yaml.safe_load(open(policy_file, "r").read())
policy_id = policy.get("Id")
targets = organizations.list_targets_for_policy_single_page(
PolicyId=policy_id
).get("Targets", [])
for target in targets:
inherited = list()
if target.get("Type") == ACCOUNT:
account = (
state.get("accounts").get(target.get("TargetId")).get("details")
)
attached = glob.glob(
f"environment/{root_id}/**/{account.get('Name')}/_meta.yaml",
recursive=True,
)
elif target.get("Type") == ORGANIZATIONAL_UNIT:
ou = (
state.get("organizational_units")
.get("by_id")
.get(target.get("TargetId"))
)
path_to_ou = get_path_for_ou(root_id, ou)
attached = glob.glob(f"{path_to_ou}/_meta.yaml")
inherited += glob.glob(
f"{path_to_ou}/_accounts/**/_meta.yaml", recursive=True
)
inherited += glob.glob(
f"{path_to_ou}/_organizational_units/**/_meta.yaml", recursive=True
)
elif target.get("Type") == "ROOT":
attached = glob.glob(
f"environment/{root_id}/_meta.yaml", recursive=True
)
inherited += glob.glob(
f"environment/{root_id}/_meta.yaml", recursive=True
)
else:
raise Exception(f"Not handled type: {target.get('Type')}")
if attached:
assert (
len(attached) == 1
), f"mapping to attached entity found {len(attached)} entities for {target}"
attached = attached[0]
output_path = attached.replace(
"_meta.yaml", "_service_control_policies.yaml"
)
output = dict(Attached=list(), Inherited=list())
if os.path.exists(output_path):
output = yaml.safe_load(open(output_path, "r").read())
output["Attached"].append(policy)
open(output_path, "w").write(yaml.safe_dump(output))
for thing in inherited:
output_path = thing.replace(
"_meta.yaml", "_service_control_policies.yaml"
)
output = dict(Attached=list(), Inherited=list())
if os.path.exists(output_path):
output = yaml.safe_load(open(output_path, "r").read())
i = dict(Source=target.get("Name"))
i.update(policy)
output["Inherited"].append(i)
open(output_path, "w").write(yaml.safe_dump(output))
progress.finish()