def read(self): """Processes a single Bible.""" output_text_file = None if FLAGS.output_text_file: logging.info("Saving original text to \"%s\".", FLAGS.output_text_file) output_text_file = open(FLAGS.output_text_file, "w", encoding=_ENCODING) output_data_file = None if FLAGS.output_data_file: logging.info("Saving the training/test data to \"%s\" ...", FLAGS.output_data_file) output_data_file = open(FLAGS.output_data_file, "w", encoding=_ENCODING) test_ids = self._read_test_ids() logging.info("Read %d test set verse IDs.", len(test_ids)) logging.info("Reading Bible from \"%s\" ...", FLAGS.input_xml_file) with open(FLAGS.input_xml_file, "r", encoding=_ENCODING) as f: root = et.fromstring(f.read()) num_sentences = 0 for n in root.iter("seg"): num_sentences += 1 progress_bar = bar.IncrementalBar("Processing", max=num_sentences) for n in root.iter("seg"): if not n.text: continue sentence = n.text.strip() sent_id = n.attrib["id"] if FLAGS.verbose: logging.info("%s: %s", sent_id, sentence) # Simply save the original text. if output_text_file: output_text_file.write(n.text.strip() + "\n") # Process and save the training/test data. sentence = re.sub(_CLEANUP_RE, "", sentence) word_prons, bad_prons = self._process_sentence(sentence) if sent_id in test_ids: sent_id = "test_" + sent_id else: sent_id = "train_" + sent_id if bad_prons: sent_id += "_NULLPRON" if output_data_file: output_data_file.write("%s\t%s\n" % (sent_id, " ".join(word_prons))) progress_bar.next() # Cleanup. progress_bar.finish() if output_text_file: output_text_file.close()
def make_migrations(role_arn: str, root_id: str) -> None: with betterboto_client.CrossAccountClientContextManager( "organizations", role_arn, f"organizations", ) as orgs_client: progress = bar.IncrementalBar("Making migrations", max=2) progress.next() make_migrations_for_organizational_units(orgs_client, root_id) progress.next() make_migrations_for_accounts(orgs_client, root_id) progress.finish()
def import_organization_policies(role_arn, root_id) -> None: with betterboto_client.CrossAccountClientContextManager( "organizations", role_arn, f"organizations" ) as organizations: progress = bar.IncrementalBar("Importing SCPs", max=4) progress.next() remove_any_existing_policy_records(root_id) progress.next() save_all_policies_from_org(root_id, organizations) progress.next() save_targets_for_policy(root_id, organizations) progress.next() progress.finish()
def import_organization(role_arn, root_id) -> None: remove_any_existing_records(root_id) with betterboto_client.CrossAccountClientContextManager( "organizations", role_arn, f"organizations" ) as organizations: delegated_administrators = ( organizations.list_delegated_administrators_single_page().get( "DelegatedAdministrators", [] ) ) progress = bar.IncrementalBar( "Importing Delegated Administrators", max=len(delegated_administrators) ) for delegated_administrator in delegated_administrators: progress.next() account_id = delegated_administrator.get("Id") account_name = delegated_administrator.get("Name") delegated_services = ( organizations.list_delegated_services_for_account_single_page( AccountId=account_id ).get("DelegatedServices") ) account_file = glob.glob( f"environment/{root_id}/**/{account_name}/_meta.yaml", recursive=True, ) assert ( len(account_file) == 1 ), "found more or less than 1 account_meta file when searching by name" delegated_administrators_file = account_file[0].replace( "_meta", "_delegated_administrators" ) open(delegated_administrators_file, "w").write( yaml.safe_dump(delegated_services) )
def update_state(role_arn) -> None: with betterboto_client.CrossAccountClientContextManager( "organizations", role_arn, f"organizations", ) as organizations: all_accounts = dict() result = dict(accounts=all_accounts) list_roots_response = organizations.list_roots_single_page() tree = dict() by_name = dict() by_id = dict() organizational_units = dict(tree=tree, by_name=by_name, by_id=by_id) result["organizational_units"] = organizational_units progress = bar.IncrementalBar("Adding roots", max=len( list_roots_response.get("Roots", []))) for root in list_roots_response.get("Roots", []): progress.next() root_id = str(root.get("Id")) details = dict( Type="Root", Id=root_id, Name="Root", ) by_name["/"] = root_id by_id[root_id] = dict(path="/", details=details) tree[root_id] = dict( details=details, organizational_units= get_children_details_filter_by_organizational_unit( organizations, root_id, by_name, "", by_id), policies=dict(service_control_policies=organizations. list_policies_for_target( TargetId=root_id, Filter=SERVICE_CONTROL_POLICY, ).get("Policies", []), ), ) progress.finish() with open(STATE_FILE, "w") as f: f.write(yaml.safe_dump(result)) accounts = organizations.list_accounts_single_page().get( "Accounts", []) progress = bar.IncrementalBar("Adding accounts", max=len(accounts)) counter = 1 for account in accounts: progress.next() account_id = account.get("Id") all_accounts[account_id] = dict( details=account, parents=organizations.list_parents_single_page( ChildId=account_id).get("Parents"), policies=dict(service_control_policies=organizations. list_policies_for_target( TargetId=account_id, Filter=SERVICE_CONTROL_POLICY, ).get("Policies", []), ), ) counter += 1 progress.finish() with open(STATE_FILE, "w") as f: f.write(yaml.safe_dump(result)) return result
def migrate(root_id: str, role_arn: str, ssm_parameter_prefix: str) -> None: with betterboto_client.CrossAccountClientContextManager( "ssm", role_arn, f"ssm", ) as ssm: progress = bar.IncrementalBar( "Migrating", max=len(os.listdir(f"environment/{root_id}/_migrations"))) for migration_file in sorted( os.listdir(f"environment/{root_id}/_migrations")): progress.next() migration_id = migration_file.split(SEP)[-1].replace(".yaml", "") try: ssm.get_parameter( Name=f"{ssm_parameter_prefix}/migrations/{migration_id}") click.echo(f" Migration: {migration_id} already run") except ssm.exceptions.ParameterNotFound: # click.echo( # f" Record of migration: {migration_id} being run not found, running now" # ) migration = yaml.safe_load( open(f"environment/{root_id}/_migrations/{migration_file}", "r").read()) migration_extension = migration.get("extension") migration_type = migration.get("migration_type") migration_params = migration.get("migration_params") if migration_extension == EXTENSION: migration_function = migrations.get_function( migration_type) elif migration_extension == service_control_policies.EXTENSION: migration_function = (service_control_policies.migrations. get_function(migration_type)) try: with betterboto_client.CrossAccountClientContextManager( "organizations", role_arn=role_arn, role_session_name="ou_create", ) as client: result, message = migration_function( root_id, client, **migration_params) except Exception as ex: result = False message = "Unhandled error: {0}".format(ex) status = "Ok" if result else "FAILED" click.echo(f"{migration_id}: {status} - {message}") ssm.put_parameter( Name=f"{ssm_parameter_prefix}/migrations/{migration_id}", Description=f"Migration run: {datetime.utcnow()}", Value=status if result else f"{status}: {message}", Type="String", Tags=[ { "Key": "AWS-Organized:Actor", "Value": "Framework" }, ], ) progress.finish()
def save_targets_for_policy(root_id, organizations) -> None: policies = glob.glob( f"environment/{root_id}/_policies/service_control_policies/*/*.yaml" ) state = yaml.safe_load(open("state.yaml", "r").read()) progress = bar.IncrementalBar("Importing policies", max=len(policies)) for policy_file in policies: progress.next() policy = yaml.safe_load(open(policy_file, "r").read()) policy_id = policy.get("Id") targets = organizations.list_targets_for_policy_single_page( PolicyId=policy_id ).get("Targets", []) for target in targets: inherited = list() if target.get("Type") == ACCOUNT: account = ( state.get("accounts").get(target.get("TargetId")).get("details") ) attached = glob.glob( f"environment/{root_id}/**/{account.get('Name')}/_meta.yaml", recursive=True, ) elif target.get("Type") == ORGANIZATIONAL_UNIT: ou = ( state.get("organizational_units") .get("by_id") .get(target.get("TargetId")) ) path_to_ou = get_path_for_ou(root_id, ou) attached = glob.glob(f"{path_to_ou}/_meta.yaml") inherited += glob.glob( f"{path_to_ou}/_accounts/**/_meta.yaml", recursive=True ) inherited += glob.glob( f"{path_to_ou}/_organizational_units/**/_meta.yaml", recursive=True ) elif target.get("Type") == "ROOT": attached = glob.glob( f"environment/{root_id}/_meta.yaml", recursive=True ) inherited += glob.glob( f"environment/{root_id}/_meta.yaml", recursive=True ) else: raise Exception(f"Not handled type: {target.get('Type')}") if attached: assert ( len(attached) == 1 ), f"mapping to attached entity found {len(attached)} entities for {target}" attached = attached[0] output_path = attached.replace( "_meta.yaml", "_service_control_policies.yaml" ) output = dict(Attached=list(), Inherited=list()) if os.path.exists(output_path): output = yaml.safe_load(open(output_path, "r").read()) output["Attached"].append(policy) open(output_path, "w").write(yaml.safe_dump(output)) for thing in inherited: output_path = thing.replace( "_meta.yaml", "_service_control_policies.yaml" ) output = dict(Attached=list(), Inherited=list()) if os.path.exists(output_path): output = yaml.safe_load(open(output_path, "r").read()) i = dict(Source=target.get("Name")) i.update(policy) output["Inherited"].append(i) open(output_path, "w").write(yaml.safe_dump(output)) progress.finish()