def generate_terraform_entrypoint(environment, key_name, run_dir, apply_immediately):
context = environment.terraform_config.to_generated_json()
if key_name not in environment.users_config.dev_users.present:
raise UnauthorizedUser(key_name)
context.update({
'SITE_HOST': environment.proxy_config.SITE_HOST,
'NO_WWW_SITE_HOST': environment.proxy_config.NO_WWW_SITE_HOST or '',
'ALTERNATE_HOSTS': environment.public_vars.get('ALTERNATE_HOSTS', []),
'account_id': environment.aws_config.sso_config.sso_account_id,
'users': [{
'username': username,
'public_key': environment.get_authorized_key(username)
} for username in environment.users_config.dev_users.present],
'key_name': key_name,
'postgresql_params': get_postgresql_params_by_rds_instance(environment),
'commcarehq_xml_post_urls_regex': compact_waf_regexes(COMMCAREHQ_XML_POST_URLS_REGEX),
'commcarehq_xml_querystring_urls_regex': compact_waf_regexes(COMMCAREHQ_XML_QUERYSTRING_URLS_REGEX),
})
context.update({
'apply_immediately': apply_immediately
})
template_root = os.path.join(os.path.dirname(__file__), 'templates')
for template_file, output_file in (
('terraform.tf.j2', 'terraform.tf'),
('commcarehq.tf.j2', 'commcarehq.tf'),
('postgresql.tf.j2', 'postgresql.tf'),
('variables.tf.j2', 'variables.tf'),
('terraform.tfvars.j2', 'terraform.tfvars'),
):
with open(os.path.join(run_dir, output_file), 'w', encoding='utf-8') as f:
f.write(render_template(template_file, context, template_root))
def generate_shard_prune_playbook(migration):
"""Create a playbook for deleting unused files.
:returns: List of nodes that have files to remove
"""
# get shard allocation from DB directly instead of using plan in case they are different
full_plan = get_db_allocations(migration.target_couch_config)
shard_suffix_by_db = {
db_name: shard_allocation_doc.usable_shard_suffix
for db_name, shard_allocation_doc in full_plan.items()
}
_, deletable_files_by_node = figure_out_what_you_can_and_cannot_delete(
full_plan, shard_suffix_by_db)
if not any(deletable_files_by_node.values()):
return None
deletable_files_by_node = {
node.split('@')[1]: files
for node, files in deletable_files_by_node.items() if files
}
prune_playbook = render_template(
'prune.yml.j2', {
'deletable_files_by_node': deletable_files_by_node,
'couch_data_dir': migration.couchdb2_data_dir
}, TEMPLATE_DIR)
with open(migration.prune_playbook_path, 'w', encoding='utf-8') as f:
f.write(prune_playbook)
return list(deletable_files_by_node)
def generate_terraform_entrypoint(environment, key_name, run_dir):
context = environment.terraform_config.to_json()
if key_name not in environment.users_config.dev_users.present:
raise UnauthorizedUser(key_name)
context.update({
'users': [{
'username': username,
'public_key': environment.get_authorized_key(username)
} for username in environment.users_config.dev_users.present],
'key_name':
key_name,
'postgresql_params':
get_postgresql_params(environment)
})
template_root = os.path.join(os.path.dirname(__file__), 'templates')
for template_file, output_file in (
('terraform.tf.j2', 'terraform.tf'),
('commcarehq.tf.j2', 'commcarehq.tf'),
('postgresql.tf.j2', 'postgresql.tf'),
('variables.tf.j2', 'variables.tf'),
('terraform.tfvars.j2', 'terraform.tfvars'),
):
with open(os.path.join(run_dir, output_file), 'w') as f:
f.write(
render_template(template_file, context,
template_root).encode('utf-8'))
def generate_terraform_entrypoint(environment, key_name, run_dir, apply_immediately):
context = environment.terraform_config.to_json()
if key_name not in environment.users_config.dev_users.present:
raise UnauthorizedUser(key_name)
context.update({
'users': [{
'username': username,
'public_key': environment.get_authorized_key(username)
} for username in environment.users_config.dev_users.present],
'key_name': key_name,
'postgresql_params': get_postgresql_params_by_rds_instance(environment)
})
context.update({
'apply_immediately': apply_immediately
})
template_root = os.path.join(os.path.dirname(__file__), 'templates')
for template_file, output_file in (
('terraform.tf.j2', 'terraform.tf'),
('commcarehq.tf.j2', 'commcarehq.tf'),
('postgresql.tf.j2', 'postgresql.tf'),
('variables.tf.j2', 'variables.tf'),
('terraform.tfvars.j2', 'terraform.tfvars'),
):
with open(os.path.join(run_dir, output_file), 'w') as f:
f.write(render_template(template_file, context, template_root).encode('utf-8'))
def prepare_file_copy_scripts(target_host, source_file_configs, script_root):
target_script_root = os.path.join(script_root, target_host)
if not os.path.exists(target_script_root):
os.makedirs(target_script_root)
files_for_node = []
for config in source_file_configs:
files = sorted(config.files)
if not files:
files_for_node.append((config, None))
else:
filename = get_file_list_filename(config)
path = os.path.join(target_script_root, filename)
with open(path, 'w') as f:
f.write('{}\n'.format('\n'.join(files)))
files_for_node.append((config, filename))
if files_for_node:
# create rsync script
rsync_script_contents = render_template('file_migration_rsync.sh.j2', {
'rsync_file_list': files_for_node,
'rsync_file_root': os.path.join('/tmp', REMOTE_MIGRATION_ROOT)
}, TEMPLATE_DIR)
rsync_script_path = os.path.join(target_script_root, FILE_MIGRATION_RSYNC_SCRIPT)
with open(rsync_script_path, 'w') as f:
f.write(rsync_script_contents)
def prepare_file_copy_scripts(target_host, source_file_configs, script_root):
target_script_root = os.path.join(script_root, target_host)
if not os.path.exists(target_script_root):
os.makedirs(target_script_root)
files_for_node = []
for config in source_file_configs:
files = sorted(config.files)
if not files:
files_for_node.append((config, None))
else:
filename = get_file_list_filename(config)
path = os.path.join(target_script_root, filename)
with open(path, 'w') as f:
f.write('{}\n'.format('\n'.join(files)))
files_for_node.append((config, filename))
if files_for_node:
# create rsync script
rsync_script_contents = render_template('file_migration_rsync.sh.j2', {
'rsync_file_list': files_for_node,
'rsync_file_root': os.path.join('/tmp', REMOTE_MIGRATION_ROOT)
}, TEMPLATE_DIR)
rsync_script_path = os.path.join(target_script_root, FILE_MIGRATION_RSYNC_SCRIPT)
with open(rsync_script_path, 'w') as f:
f.write(rsync_script_contents)
def generate_shard_prune_playbook(migration):
"""Create a playbook for deleting unused files.
:returns: List of nodes that have files to remove
"""
# get shard allocation from DB directly instead of using plan in case they are different
full_plan = get_db_allocations(migration.target_couch_config)
shard_suffix_by_db = {
db_name: shard_allocation_doc.usable_shard_suffix
for db_name, shard_allocation_doc in full_plan.items()
}
_, deletable_files_by_node = figure_out_what_you_can_and_cannot_delete(full_plan, shard_suffix_by_db)
if not any(deletable_files_by_node.values()):
return None
deletable_files_by_node = {
node.split('@')[1]: files
for node, files in deletable_files_by_node.items()
if files
}
prune_playbook = render_template('prune.yml.j2', {
'deletable_files_by_node': deletable_files_by_node,
'couch_data_dir': migration.couchdb2_data_dir
}, TEMPLATE_DIR)
with open(migration.prune_playbook_path, 'w') as f:
f.write(prune_playbook)
return list(deletable_files_by_node)
