def generate_terraform_entrypoint(environment, key_name, run_dir, apply_immediately): context = environment.terraform_config.to_generated_json() if key_name not in environment.users_config.dev_users.present: raise UnauthorizedUser(key_name) context.update({ 'SITE_HOST': environment.proxy_config.SITE_HOST, 'NO_WWW_SITE_HOST': environment.proxy_config.NO_WWW_SITE_HOST or '', 'ALTERNATE_HOSTS': environment.public_vars.get('ALTERNATE_HOSTS', []), 'account_id': environment.aws_config.sso_config.sso_account_id, 'users': [{ 'username': username, 'public_key': environment.get_authorized_key(username) } for username in environment.users_config.dev_users.present], 'key_name': key_name, 'postgresql_params': get_postgresql_params_by_rds_instance(environment), 'commcarehq_xml_post_urls_regex': compact_waf_regexes(COMMCAREHQ_XML_POST_URLS_REGEX), 'commcarehq_xml_querystring_urls_regex': compact_waf_regexes(COMMCAREHQ_XML_QUERYSTRING_URLS_REGEX), }) context.update({ 'apply_immediately': apply_immediately }) template_root = os.path.join(os.path.dirname(__file__), 'templates') for template_file, output_file in ( ('terraform.tf.j2', 'terraform.tf'), ('commcarehq.tf.j2', 'commcarehq.tf'), ('postgresql.tf.j2', 'postgresql.tf'), ('variables.tf.j2', 'variables.tf'), ('terraform.tfvars.j2', 'terraform.tfvars'), ): with open(os.path.join(run_dir, output_file), 'w', encoding='utf-8') as f: f.write(render_template(template_file, context, template_root))
def generate_shard_prune_playbook(migration): """Create a playbook for deleting unused files. :returns: List of nodes that have files to remove """ # get shard allocation from DB directly instead of using plan in case they are different full_plan = get_db_allocations(migration.target_couch_config) shard_suffix_by_db = { db_name: shard_allocation_doc.usable_shard_suffix for db_name, shard_allocation_doc in full_plan.items() } _, deletable_files_by_node = figure_out_what_you_can_and_cannot_delete( full_plan, shard_suffix_by_db) if not any(deletable_files_by_node.values()): return None deletable_files_by_node = { node.split('@')[1]: files for node, files in deletable_files_by_node.items() if files } prune_playbook = render_template( 'prune.yml.j2', { 'deletable_files_by_node': deletable_files_by_node, 'couch_data_dir': migration.couchdb2_data_dir }, TEMPLATE_DIR) with open(migration.prune_playbook_path, 'w', encoding='utf-8') as f: f.write(prune_playbook) return list(deletable_files_by_node)
def generate_terraform_entrypoint(environment, key_name, run_dir): context = environment.terraform_config.to_json() if key_name not in environment.users_config.dev_users.present: raise UnauthorizedUser(key_name) context.update({ 'users': [{ 'username': username, 'public_key': environment.get_authorized_key(username) } for username in environment.users_config.dev_users.present], 'key_name': key_name, 'postgresql_params': get_postgresql_params(environment) }) template_root = os.path.join(os.path.dirname(__file__), 'templates') for template_file, output_file in ( ('terraform.tf.j2', 'terraform.tf'), ('commcarehq.tf.j2', 'commcarehq.tf'), ('postgresql.tf.j2', 'postgresql.tf'), ('variables.tf.j2', 'variables.tf'), ('terraform.tfvars.j2', 'terraform.tfvars'), ): with open(os.path.join(run_dir, output_file), 'w') as f: f.write( render_template(template_file, context, template_root).encode('utf-8'))
def generate_terraform_entrypoint(environment, key_name, run_dir, apply_immediately): context = environment.terraform_config.to_json() if key_name not in environment.users_config.dev_users.present: raise UnauthorizedUser(key_name) context.update({ 'users': [{ 'username': username, 'public_key': environment.get_authorized_key(username) } for username in environment.users_config.dev_users.present], 'key_name': key_name, 'postgresql_params': get_postgresql_params_by_rds_instance(environment) }) context.update({ 'apply_immediately': apply_immediately }) template_root = os.path.join(os.path.dirname(__file__), 'templates') for template_file, output_file in ( ('terraform.tf.j2', 'terraform.tf'), ('commcarehq.tf.j2', 'commcarehq.tf'), ('postgresql.tf.j2', 'postgresql.tf'), ('variables.tf.j2', 'variables.tf'), ('terraform.tfvars.j2', 'terraform.tfvars'), ): with open(os.path.join(run_dir, output_file), 'w') as f: f.write(render_template(template_file, context, template_root).encode('utf-8'))
def prepare_file_copy_scripts(target_host, source_file_configs, script_root): target_script_root = os.path.join(script_root, target_host) if not os.path.exists(target_script_root): os.makedirs(target_script_root) files_for_node = [] for config in source_file_configs: files = sorted(config.files) if not files: files_for_node.append((config, None)) else: filename = get_file_list_filename(config) path = os.path.join(target_script_root, filename) with open(path, 'w') as f: f.write('{}\n'.format('\n'.join(files))) files_for_node.append((config, filename)) if files_for_node: # create rsync script rsync_script_contents = render_template('file_migration_rsync.sh.j2', { 'rsync_file_list': files_for_node, 'rsync_file_root': os.path.join('/tmp', REMOTE_MIGRATION_ROOT) }, TEMPLATE_DIR) rsync_script_path = os.path.join(target_script_root, FILE_MIGRATION_RSYNC_SCRIPT) with open(rsync_script_path, 'w') as f: f.write(rsync_script_contents)
def prepare_file_copy_scripts(target_host, source_file_configs, script_root): target_script_root = os.path.join(script_root, target_host) if not os.path.exists(target_script_root): os.makedirs(target_script_root) files_for_node = [] for config in source_file_configs: files = sorted(config.files) if not files: files_for_node.append((config, None)) else: filename = get_file_list_filename(config) path = os.path.join(target_script_root, filename) with open(path, 'w') as f: f.write('{}\n'.format('\n'.join(files))) files_for_node.append((config, filename)) if files_for_node: # create rsync script rsync_script_contents = render_template('file_migration_rsync.sh.j2', { 'rsync_file_list': files_for_node, 'rsync_file_root': os.path.join('/tmp', REMOTE_MIGRATION_ROOT) }, TEMPLATE_DIR) rsync_script_path = os.path.join(target_script_root, FILE_MIGRATION_RSYNC_SCRIPT) with open(rsync_script_path, 'w') as f: f.write(rsync_script_contents)
def generate_shard_prune_playbook(migration): """Create a playbook for deleting unused files. :returns: List of nodes that have files to remove """ # get shard allocation from DB directly instead of using plan in case they are different full_plan = get_db_allocations(migration.target_couch_config) shard_suffix_by_db = { db_name: shard_allocation_doc.usable_shard_suffix for db_name, shard_allocation_doc in full_plan.items() } _, deletable_files_by_node = figure_out_what_you_can_and_cannot_delete(full_plan, shard_suffix_by_db) if not any(deletable_files_by_node.values()): return None deletable_files_by_node = { node.split('@')[1]: files for node, files in deletable_files_by_node.items() if files } prune_playbook = render_template('prune.yml.j2', { 'deletable_files_by_node': deletable_files_by_node, 'couch_data_dir': migration.couchdb2_data_dir }, TEMPLATE_DIR) with open(migration.prune_playbook_path, 'w') as f: f.write(prune_playbook) return list(deletable_files_by_node)