def run_vm_script(workflow_dict, context_dict, script):
try:
instances_detail = workflow_dict['instances_detail']
final_context_dict = dict(context_dict.items() + workflow_dict['initial_context_dict'].items())
for instance_detail in instances_detail:
instance = instance_detail['instance']
host = instance.hostname
host_csattr = HostAttr.objects.get(host=host)
final_context_dict['HOSTADDRESS'] = instance.address
final_context_dict['PORT'] = instance.port
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(server = host.address,
username = host_csattr.vm_user,
password = host_csattr.vm_password,
command = command,
output = output)
if return_code:
raise Exception, "Could not run script. Output: {}".format(output)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def run_vm_script(workflow_dict, context_dict, script):
try:
instances_detail = workflow_dict["instances_detail"]
final_context_dict = dict(context_dict.items() + workflow_dict["initial_context_dict"].items())
for instance_detail in instances_detail:
host = instance_detail["instance"].hostname
host_csattr = HostAttr.objects.get(host=host)
final_context_dict["IS_MASTER"] = instance_detail["is_master"]
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=command,
output=output,
)
if return_code:
raise Exception, "Could not run script. Output: {}".format(output)
return True
except Exception:
traceback = full_stack()
workflow_dict["exceptions"]["error_codes"].append(DBAAS_0015)
workflow_dict["exceptions"]["traceback"].append(traceback)
return False
def run_vm_script(workflow_dict, context_dict, script):
try:
final_context_dict = dict(context_dict.items())
instance = workflow_dict['instance']
host = workflow_dict['host']
host_csattr = HostAttr.objects.get(host=host)
final_context_dict['HOSTADDRESS'] = instance.address
final_context_dict['PORT'] = instance.port
final_context_dict['DBPASSWORD'] = workflow_dict['databaseinfra'].password
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=command,
output=output)
if return_code:
raise Exception("Could not run script. Output: {}".format(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def change_instance_binaries(self, instance, connect_string, run_authschemaupgrade):
script = test_bash_script_error()
script += util.build_cp_mongodb_binary_file()
script += util.build_stop_database_script(clean_data=False)
script += util.build_change_release_alias_script()
script += util.build_start_database_script(wait_time=30)
script += util.build_change_limits_script()
script += util.build_remove_reprecated_index_counter_metrics()
if run_authschemaupgrade:
script += util.build_authschemaupgrade_script()
context_dict = {
'SOURCE_PATH': '/mnt/software/db/mongodb',
'TARGET_PATH': '/usr/local/',
'MONGODB_RELEASE_FILE': 'mongodb-linux-x86_64-3.0.8.tgz',
'MONGODB_RELEASE_FOLDER': 'mongodb-linux-x86_64-3.0.8',
'CONNECT_STRING': connect_string,
}
script = build_context_script(context_dict, script)
output = {}
host = instance.hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def do(self, workflow_dict):
try:
host = workflow_dict['source_hosts'][0]
databaseinfra = workflow_dict['databaseinfra']
cs_host_attr = CS_HostAttr.objects.get(host=host)
context_dict = {
'MASTERPAIRNAME': databaseinfra.name,
}
script = test_bash_script_error()
script += build_turn_flipper_ip_down_script()
script = build_context_script(context_dict, script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
script = build_clean_database_dir_script()
script = build_context_script({}, script)
for source_host in workflow_dict['source_hosts']:
target_host = source_host.future_host
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
output = {}
exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for source_host in workflow_dict['source_hosts']:
host = source_host.future_host
LOG.info("Starting td_agent on host {}".format(host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
context_dict = {}
script = test_bash_script_error()
script += build_start_td_agent_script()
script = build_context_script(context_dict, script)
LOG.info(script)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
LOG.error("Error starting td_agent")
LOG.error(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def run_vm_script(workflow_dict, context_dict, script, reverse=False, wait=0):
try:
instances_detail = workflow_dict['instances_detail']
final_context_dict = dict(context_dict.items() + workflow_dict['initial_context_dict'].items())
if reverse:
instances_detail_final = instances_detail[::-1]
else:
instances_detail_final = instances_detail
for instance_detail in instances_detail_final:
host = instance_detail['instance'].hostname
host_csattr = HostAttr.objects.get(host=host)
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(server = host.address,
username = host_csattr.vm_user,
password = host_csattr.vm_password,
command = command,
output = output)
if return_code:
raise Exception, "Could not run script. Output: {}".format(output)
sleep(wait)
return True
except Exception, e:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def switch_master(self, instance=None):
sentinel_instance = self.instances_filtered.first()
host = sentinel_instance.hostname
script = """
#!/bin/bash
die_if_error()
{
local err=$?
if [ "$err" != "0" ];
then
echo "$*"
exit $err
fi
}"""
script += """
/usr/local/redis/src/redis-cli -h {} -p {} <<EOF_DBAAS
SENTINEL failover {}
exit
\nEOF_DBAAS
die_if_error "Error reseting sentinel"
""".format(
sentinel_instance.address, sentinel_instance.port,
self.databaseinfra.name
)
script = build_context_script({}, script)
output = {}
return_code = exec_remote_command_host(host, script, output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def switch_master(self):
master = self.get_master_instance()
slave = self.get_slave_instances()[0]
host = master.hostname
host_attr = HostAttr.objects.get(host=host)
script = """
sudo -u flipper /usr/bin/flipper {{MASTERPAIRNAME}} set write {{HOST01.address}}
sudo -u flipper /usr/bin/flipper {{MASTERPAIRNAME}} set read {{HOST02.address}}
"""
context_dict = {
'MASTERPAIRNAME': self.databaseinfra.name,
'HOST01': slave.hostname,
'HOST02': master.hostname,
}
script = build_context_script(context_dict, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_attr.vm_user,
password=host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
script = build_stop_database_script()
script = build_context_script({}, script)
for target_instance in workflow_dict['target_instances']:
target_host = target_instance.hostname
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
output = {}
exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
try:
if 'region_migration_dir_infra_name' in workflow_dict:
shutil.rmtree(workflow_dict['region_migration_dir_infra_name'])
except Exception:
pass
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
connect_string = build_mongodb_connect_string(instances=workflow_dict['source_instances'],
databaseinfra=databaseinfra)
client = databaseinfra.get_driver().get_client(None)
rsconf = client.local.system.replset.find_one()
member_ids = []
for member in rsconf['members']:
member_ids.append(member['_id'])
max_member_id = max(member_ids)
secundary_one_member_id = max_member_id + 1
secundary_two_member_id = max_member_id + 2
arbiter_member_id = max_member_id + 3
context_dict = {
'CONNECT_STRING': connect_string,
'SECUNDARY_ONE': "{}:{}".format(workflow_dict['target_instances'][0].address, workflow_dict['target_instances'][0].port),
'SECUNDARY_TWO': "{}:{}".format(workflow_dict['target_instances'][1].address, workflow_dict['target_instances'][1].port),
'ARBITER': "{}:{}".format(workflow_dict['target_instances'][2].address, workflow_dict['target_instances'][2].port),
'SECUNDARY_ONE_MEMBER_ID': secundary_one_member_id,
'SECUNDARY_TWO_MEMBER_ID': secundary_two_member_id,
'ARBITER_MEMBER_ID': arbiter_member_id,
}
script = test_bash_script_error()
script += build_add_replica_set_members_script()
script = build_context_script(context_dict, script)
output = {}
host = workflow_dict['source_instances'][0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def change_slave_priority_file(host, original_value, final_value):
script = test_bash_script_error()
script += """
sed -i 's/slave-priority {}/slave-priority {}/g' /data/redis.conf
""".format(original_value, final_value)
script = build_context_script({}, script)
output = {}
return_code = exec_remote_command_host(host, script, output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
connect_string = build_mongodb_connect_string(instances=workflow_dict['source_instances'],
databaseinfra=databaseinfra)
context_dict = {
'CONNECT_STRING': connect_string,
}
script = test_bash_script_error()
script += build_switch_primary_to_new_instances_script()
script = build_context_script(context_dict, script)
output = {}
host = workflow_dict['source_instances'][0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
sleep(30)
driver = databaseinfra.get_driver()
client = driver.get_client(instance=None)
rsconf = client['local'].system.replset.find_one()
priority_0 = rsconf['members'][0].get('priority', 1)
priority_1 = rsconf['members'][1].get('priority', 1)
if priority_0 != 0 or priority_1 != 0:
errormsg = "The priority of the old mongodb instances should be zero."
LOG.error(errormsg)
raise Exception(errormsg)
replSetGetStatus = client.admin.command('replSetGetStatus')
if 'PRIMARY' not in (replSetGetStatus['members'][3]['stateStr'],
replSetGetStatus['members'][4]['stateStr']):
errormsg = "One of the new instances should be PRIMARY."
LOG.error(errormsg)
raise Exception(errormsg)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for index, instance in enumerate(workflow_dict['target_instances']):
if instance.instance_type == instance.MONGODB_ARBITER:
continue
if instance.instance_type == instance.REDIS_SENTINEL:
continue
host = instance.hostname
LOG.info("Mounting disks on host {}".format(host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
nfs_host_attr = NFS_HostAttr.objects.get(host=host)
LOG.info("Cheking host ssh...")
host_ready = check_ssh(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
wait=5, interval=10)
if not host_ready:
raise Exception(str("Host %s is not ready..." % host))
context_dict = {
'EXPORTPATH': nfs_host_attr.nfsaas_path,
}
script = test_bash_script_error()
script += build_mount_disk_script()
script = build_context_script(context_dict, script)
LOG.info(script)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
instances = workflow_dict['instances']
databaseinfra = workflow_dict['databaseinfra']
connect_string = util.build_mongodb_connect_string(instances=instances,
databaseinfra=databaseinfra)
script = test_bash_script_error()
script += util.build_cp_mongodb_binary_file()
script += util.build_stop_database_script(clean_data=False)
script += util.build_change_release_alias_script()
script += util.build_start_database_script()
script += util.build_authschemaupgrade_script()
script += util.build_change_limits_script()
script += util.build_change_in_serverstatus_file_script()
script += util.build_reinstal_mongo_gen_script()
script += util.build_remove_reprecated_index_counter_metrics()
context_dict = {
'SOURCE_PATH': '/mnt/software/db/mongodb',
'TARGET_PATH': '/usr/local/',
'MONGODB_RELEASE_FILE': 'mongodb-linux-x86_64-3.0.8.tgz',
'MONGODB_RELEASE_FOLDER': 'mongodb-linux-x86_64-3.0.8',
'CONNECT_STRING': connect_string,
}
script = build_context_script(context_dict, script)
output = {}
host = instances[0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0023)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def _execute_script(self, script_variables, script):
final_script = build_context_script(
script_variables, script
)
output = {}
return_code = exec_remote_command_host(self.host, final_script, output)
if return_code != 0:
raise EnvironmentError(
'Could not execute replica script {}: {}'.format(
return_code, output
)
)
def do(self, workflow_dict):
try:
flipper_crdentials = get_credentials_for(workflow_dict['source_environment'],
CredentialType.FLIPPER)
flipper_vip = flipper_crdentials.get_parameter_by_name('vip')
for host in workflow_dict['target_hosts']:
cs_host_attr = CS_HostAttr.objects.get(host=host)
source_host = workflow_dict['source_hosts'][0]
nf_host_attr = NF_HostAttr.objects.get(host=source_host)
script = test_bash_script_error()
script += build_mount_snapshot_volume_script()
script += build_remove_deprecated_files_script()
script += build_permission_script()
script += build_start_database_script()
script += build_flipper_script()
context_dict = {
'EXPORT_PATH': nf_host_attr.nfsaas_path,
'SNAPSHOPT_NAME': workflow_dict['snapshot_name'],
'VIP_FLIPPER': flipper_vip,
'IPWRITE': workflow_dict['target_secondary_ips'][0].ip,
'HOST01': workflow_dict['target_hosts'][0],
'HOST02': workflow_dict['target_hosts'][1]
}
script = build_context_script(context_dict, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
driver = databaseinfra.get_driver()
sentinel = driver.get_sentinel_client()
master = sentinel.discover_master(databaseinfra.name)
master_host = master[0]
master_port = master[1]
for index, source_host in enumerate(workflow_dict['source_hosts']):
target_host = source_host.future_host
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
script = test_bash_script_error()
script += build_start_stop_scripts()
if index < 2:
script += build_start_database_script()
script += build_start_sentinel_script()
script += build_start_http_script()
script = build_context_script({}, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
for target_instance in workflow_dict['target_instances']:
if target_instance.instance_type == target_instance.REDIS:
LOG.info('Changing master of host: {}'.format(target_instance.address))
client = driver.get_client(target_instance)
client.slaveof(master_host, master_port)
LOG.info('New master: {}:{}'.format(master_host, master_port))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict["databaseinfra"]
for index, source_host in enumerate(workflow_dict["source_hosts"]):
source_cs_host_attr = CS_HostAttr.objects.get(host=source_host)
script = test_bash_script_error()
script += build_start_stop_scripts()
if index < 2:
script += build_stop_database_script()
script += build_stop_sentinel_script()
script += build_start_http_script()
script = build_context_script({}, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(
server=source_host.address,
username=source_cs_host_attr.vm_user,
password=source_cs_host_attr.vm_password,
command=script,
output=output,
)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
for source_instance in workflow_dict["source_instances"]:
if source_instance.instance_type == source_instance.REDIS_SENTINEL:
target_instance = source_instance.future_instance
reset_sentinel(
host=target_instance.hostname,
sentinel_host=target_instance.address,
sentinel_port=target_instance.port,
service_name=databaseinfra.name,
)
sleep(30)
return True
except Exception:
traceback = full_stack()
workflow_dict["exceptions"]["error_codes"].append(DBAAS_0020)
workflow_dict["exceptions"]["traceback"].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
target_instances = []
for source_instance in workflow_dict['source_instances']:
target_instances.append(source_instance.future_instance)
connect_string = build_mongodb_connect_string(instances=target_instances,
databaseinfra=databaseinfra)
context_dict = {
'CONNECT_STRING': connect_string,
'SECUNDARY_ONE': "{}:{}".format(workflow_dict['source_instances'][0].address, workflow_dict['source_instances'][0].port),
'SECUNDARY_TWO': "{}:{}".format(workflow_dict['source_instances'][1].address, workflow_dict['source_instances'][1].port),
'ARBITER': "{}:{}".format(workflow_dict['source_instances'][2].address, workflow_dict['source_instances'][2].port),
}
script = test_bash_script_error()
script += build_remove_replica_set_members_script()
script = build_context_script(context_dict, script)
output = {}
host = workflow_dict['source_instances'][0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
for source_instance in workflow_dict['source_instances']:
source_instance.delete()
LOG.info("Source instance deleted")
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def change_slave_priority_file(host, original_value, final_value):
host_attr = HostAttr.objects.get(host=host)
script = test_bash_script_error()
script += """
sed -i 's/slave-priority {}/slave-priority {}/g' /data/redis.conf
""".format(original_value, final_value)
script = build_context_script({}, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_attr.vm_user,
password=host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def failover_sentinel(host, sentinel_host, sentinel_port, service_name):
LOG.info('Failover of Sentinel {}:{}'.format(sentinel_host, sentinel_port))
script = test_bash_script_error()
script += """
/usr/local/redis/src/redis-cli -h {} -p {} <<EOF_DBAAS
SENTINEL failover {}
exit
\nEOF_DBAAS
die_if_error "Error reseting sentinel"
""".format(sentinel_host, sentinel_port, service_name)
script = build_context_script({}, script)
output = {}
return_code = exec_remote_command_host(host, script, output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def do(self, workflow_dict):
try:
backup_log_dict = get_backup_log_configuration_dict(environment=workflow_dict['environment'],
databaseinfra=workflow_dict['databaseinfra'])
if backup_log_dict is None:
LOG.info("There is not any backup log configuration for this database...")
return True
for index, instance in enumerate(workflow_dict['instances']):
if instance.instance_type == instance.MONGODB_ARBITER:
continue
if instance.instance_type == instance.REDIS_SENTINEL:
continue
host = instance.hostname
LOG.info("Getting vm credentials...")
host_csattr = CsHostAttr.objects.get(host=host)
script = test_bash_script_error()
script += build_backup_log_script()
contextdict = backup_log_dict
script = build_context_script(contextdict, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=script,
output=output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0014)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
initial_script = '#!/bin/bash\n\ndie_if_error()\n{\n local err=$?\n if [ "$err" != "0" ]; then\n echo "$*"\n exit $err\n fi\n}'
for index, instance in enumerate(workflow_dict['target_instances']):
if instance.instance_type == instance.MONGODB_ARBITER:
continue
host = instance.hostname
LOG.info("Mounting disks on host {}".format(host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
nfs_host_attr = NFS_HostAttr.objects.get(host=host)
context_dict = {
'EXPORTPATH': nfs_host_attr.nfsaas_path,
}
script = initial_script
script += '\necho ""; echo $(date "+%Y-%m-%d %T") "- Mounting data disk"'
script += '\necho "{{EXPORTPATH}} /data nfs defaults,bg,intr,nolock 0 0" >> /etc/fstab'
script += '\ndie_if_error "Error setting fstab"'
script += '\nmount /data'
script += '\ndie_if_error "Error setting fstab"'
script = build_context_script(context_dict, script)
LOG.info(script)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception, output['stderr']
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0019)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
target_host_zero = workflow_dict['source_hosts'][0].future_host
target_host_one = workflow_dict['source_hosts'][1].future_host
databaseinfra = workflow_dict['databaseinfra']
cs_host_attr = CS_HostAttr.objects.get(host=target_host_zero)
context_dict = {
'MASTERPAIRNAME': databaseinfra.name,
'HOST01': target_host_zero,
'HOST02': target_host_one,
}
script = test_bash_script_error()
script += build_set_flipper_ips_script()
script = build_context_script(context_dict, script)
output = {}
return_code = exec_remote_command(
server=target_host_zero.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
databaseinfraattr = workflow_dict[
'source_secondary_ips'][0].equivalent_dbinfraattr
databaseinfra = workflow_dict['databaseinfra']
databaseinfra.endpoint = databaseinfraattr.ip + ":3306"
databaseinfra.save()
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
instances = workflow_dict['instances']
script = test_bash_script_error()
script += build_change_mongodb_conf_file_script()
script += build_stop_database_script(clean_data=False)
script += build_clean_data_data_script()
script += build_start_database_script()
script += build_mongorestore_database_script()
script += build_enable_authentication_single_instance_script()
script += build_restart_database_script()
script += td_agent_script(option='restart')
context_dict = {}
script = build_context_script(context_dict, script)
output = {}
host = instances[0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0023)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for host in workflow_dict['target_hosts']:
cs_host_attr = CS_HostAttr.objects.get(host=host)
source_host = workflow_dict['source_hosts'][0]
nf_host_attr = NF_HostAttr.objects.get(host=source_host)
script = test_bash_script_error()
script += build_mount_snapshot_volume_script()
script += build_remove_deprecated_files_script()
script += build_permission_script()
script += build_start_database_script()
context_dict = {
'EXPORT_PATH': nf_host_attr.nfsaas_path,
'SNAPSHOPT_NAME': workflow_dict['snapshot_name'],
}
script = build_context_script(context_dict, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def failover_sentinel(host, sentinel_host, sentinel_port, service_name):
LOG.info('Failover of Sentinel {}:{}'.format(sentinel_host, sentinel_port))
host_attr = HostAttr.objects.get(host=host)
script = test_bash_script_error()
script += """
/usr/local/redis/src/redis-cli -h {} -p {} <<EOF_DBAAS
SENTINEL failover {}
exit
\nEOF_DBAAS
die_if_error "Error reseting sentinel"
""".format(sentinel_host, sentinel_port, service_name)
script = build_context_script({}, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_attr.vm_user,
password=host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def switch_master(self):
sentinel_instance = self.databaseinfra.instances.filter(
instance_type=Instance.REDIS_SENTINEL, is_active=True).all()[0]
host = sentinel_instance.hostname
host_attr = HostAttr.objects.get(host=host)
script = """
#!/bin/bash
die_if_error()
{
local err=$?
if [ "$err" != "0" ];
then
echo "$*"
exit $err
fi
}"""
script += """
/usr/local/redis/src/redis-cli -h {} -p {} <<EOF_DBAAS
SENTINEL failover {}
exit
\nEOF_DBAAS
die_if_error "Error reseting sentinel"
""".format(sentinel_instance.address, sentinel_instance.port,
self.databaseinfra.name)
script = build_context_script({}, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_attr.vm_user,
password=host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def switch_master(self, instance=None):
if instance is None:
raise Exception('Cannot switch master in a redis cluster without instance.')
slave_instance = self.get_slave_for(instance)
if not slave_instance:
raise Exception('There is no slave for {}'.format(instance))
host = slave_instance.hostname
script = """
#!/bin/bash
die_if_error()
{
local err=$?
if [ "$err" != "0" ];
then
echo "$*"
exit $err
fi
}"""
script += """
/usr/local/redis/src/redis-cli -h {} -p {} -a {} -c<<EOF_DBAAS
CLUSTER FAILOVER
exit
\nEOF_DBAAS
die_if_error "Error executing cluster failover"
""".format(
slave_instance.address, slave_instance.port,
self.databaseinfra.password
)
script = build_context_script({}, script)
output = {}
return_code = exec_remote_command_host(host, script, output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
databaseinfra = workflow_dict['databaseinfra']
for index, source_host in enumerate(workflow_dict['source_hosts']):
target_host = source_host.future_host
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
script = test_bash_script_error()
script += build_start_stop_scripts()
script += build_stop_sentinel_script()
if index < 2:
script += build_stop_database_script()
script = build_context_script({}, script)
output = {}
exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
for source_instance in workflow_dict['source_instances']:
if source_instance.instance_type == source_instance.REDIS_SENTINEL:
reset_sentinel(host=source_instance.hostname,
sentinel_host=source_instance.address,
sentinel_port=source_instance.port,
service_name=databaseinfra.name)
sleep(30)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def switch_master(self, instance=None):
if instance is None:
raise Exception('Cannot switch master in a redis cluster without instance.')
slave_instance = self.get_slave_for(instance)
if not slave_instance:
raise Exception('There is no slave for {}'.format(instance))
host = slave_instance.hostname
script = """
#!/bin/bash
die_if_error()
{
local err=$?
if [ "$err" != "0" ];
then
echo "$*"
exit $err
fi
}"""
script += """
/usr/local/redis/src/redis-cli -h {} -p {} -a {} -c<<EOF_DBAAS
CLUSTER FAILOVER
exit
\nEOF_DBAAS
die_if_error "Error executing cluster failover"
""".format(
slave_instance.address, slave_instance.port,
self.databaseinfra.password
)
script = build_context_script({}, script)
output = {}
return_code = exec_remote_command_host(host, script, output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
source_host_zero = workflow_dict['source_hosts'][0]
source_host_one = workflow_dict['source_hosts'][1]
databaseinfra = workflow_dict['databaseinfra']
cs_host_attr = CS_HostAttr.objects.get(host=source_host_zero)
context_dict = {
'MASTERPAIRNAME': databaseinfra.name,
'HOST01': source_host_zero,
'HOST02': source_host_one,
}
script = test_bash_script_error()
script += build_set_flipper_ips_script()
script = build_context_script(context_dict, script)
output = {}
return_code = exec_remote_command(
server=source_host_zero.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def switch_master(self):
sentinel_instance = self.databaseinfra.instances.filter(
instance_type=Instance.REDIS_SENTINEL, is_active=True).all()[0]
host = sentinel_instance.hostname
host_attr = HostAttr.objects.get(host=host)
script = """
#!/bin/bash
die_if_error()
{
local err=$?
if [ "$err" != "0" ];
then
echo "$*"
exit $err
fi
}"""
script += """
/usr/local/redis/src/redis-cli -h {} -p {} <<EOF_DBAAS
SENTINEL failover {}
exit
\nEOF_DBAAS
die_if_error "Error reseting sentinel"
""".format(sentinel_instance.address, sentinel_instance.port, self.databaseinfra.name)
script = build_context_script({}, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_attr.vm_user,
password=host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def run_vm_script(workflow_dict, context_dict, script, reverse=False, wait=0):
try:
instances_detail = workflow_dict['instances_detail']
final_context_dict = dict(
context_dict.items() +
workflow_dict['initial_context_dict'].items())
if reverse:
instances_detail_final = instances_detail[::-1]
else:
instances_detail_final = instances_detail
for instance_detail in instances_detail_final:
host = instance_detail['instance'].hostname
host_csattr = HostAttr.objects.get(host=host)
final_context_dict['IS_MASTER'] = instance_detail['is_master']
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=command,
output=output)
if return_code:
raise Exception(
"Could not run script. Output: {}".format(output))
sleep(wait)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
databaseinfra = workflow_dict['databaseinfra']
connect_string = build_mongodb_connect_string(
instances=workflow_dict['source_instances'],
databaseinfra=databaseinfra)
context_dict = {
'CONNECT_STRING': connect_string,
}
script = test_bash_script_error()
script += build_switch_primary_to_old_instances_script()
script = build_context_script(context_dict, script)
output = {}
host = workflow_dict['source_instances'][0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def run_vm_script(workflow_dict, context_dict, script):
try:
host = workflow_dict['host']
host_csattr = HostAttr.objects.get(host=host)
command = build_context_script(context_dict, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=command,
output=output)
if return_code:
raise Exception("Could not run script. Output: {}".format(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
for source_host in workflow_dict['source_hosts']:
future_host = source_host.future_host
hosts_option = [(future_host, 'stop'), (source_host, 'start')]
for host, option in hosts_option:
LOG.info("Starting td_agent on host {}".format(host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
context_dict = {}
script = test_bash_script_error()
script += build_mk_heartbeat_daemon_script(option)
script = build_context_script(context_dict, script)
LOG.info(script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
LOG.error("Error starting mysql_statsd")
LOG.error(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def make_script(self, plan_script, script_variables=None):
return build_context_script(
script_variables or self.script_variables,
plan_script
)
def do(self, workflow_dict):
try:
region_migration_dir = Configuration.get_by_name('region_migration_dir')
if not region_migration_dir:
region_migration_dir = '/tmp'
workflow_dict['region_migration_dir_infra_name'] = "{}/{}".format(region_migration_dir, workflow_dict['databaseinfra'].name)
for index, source_instance in enumerate(workflow_dict['source_instances']):
source_host = source_instance.hostname
source_cs_host_attr = CS_HostAttr.objects.get(host=source_host)
hostname = source_host.hostname.split('.')[0]
localpath = "{}/{}".format(workflow_dict['region_migration_dir_infra_name'], hostname)
os.makedirs(localpath)
LOG.info('Get source host files to {}'.format(localpath))
if not scp_get_file(server=source_host.address,
username=source_cs_host_attr.vm_user,
password=source_cs_host_attr.vm_password,
localpath="{}/mongodb.key".format(localpath),
remotepath="/data/mongodb.key"):
raise Exception("FTP Error")
if not scp_get_file(server=source_host.address,
username=source_cs_host_attr.vm_user,
password=source_cs_host_attr.vm_password,
localpath="{}/mongodb.conf".format(localpath),
remotepath="/data/mongodb.conf"):
raise Exception("FTP Error")
if not scp_get_file(server=source_host.address,
username=source_cs_host_attr.vm_user,
password=source_cs_host_attr.vm_password,
localpath="{}/td-agent.conf".format(localpath),
remotepath="/etc/td-agent/td-agent.conf"):
raise Exception("FTP Error")
target_host = source_host.future_host
LOG.info(target_host)
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
target_instance = source_instance.future_instance
if target_instance.instance_type == target_instance.MONGODB_ARBITER:
LOG.info("Cheking host ssh...")
host_ready = check_ssh(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
wait=5, interval=10)
if not host_ready:
raise Exception(str("Host %s is not ready..." % target_host))
if not scp_put_file(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
localpath="{}/mongodb.key".format(localpath),
remotepath="/data/mongodb.key"):
raise Exception("FTP Error")
if not scp_put_file(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
localpath="{}/mongodb.conf".format(localpath),
remotepath="/data/mongodb.conf"):
raise Exception("FTP Error")
if not scp_put_file(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
localpath="{}/td-agent.conf".format(localpath),
remotepath="/etc/td-agent/td-agent.conf"):
raise Exception("FTP Error")
script = test_bash_script_error()
script += build_permission_script()
script += build_start_database_script()
script = build_context_script({}, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
shutil.rmtree(workflow_dict['region_migration_dir_infra_name'])
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def execute_scheduled_maintenance(self, maintenance_id):
LOG.debug("Maintenance id: {}".format(maintenance_id))
maintenance = models.Maintenance.objects.get(id=maintenance_id)
models.Maintenance.objects.filter(id=maintenance_id, ).update(
status=maintenance.RUNNING, started_at=datetime.now())
LOG.info("Maintenance {} is RUNNING".format(maintenance, ))
worker_name = get_worker_name()
task_history = TaskHistory.register(request=self.request,
worker_name=worker_name)
LOG.info("id: %s | task: %s | kwargs: %s | args: %s" %
(self.request.id, self.request.task, self.request.kwargs,
str(self.request.args)))
task_history.update_details(
persist=True, details="Executing Maintenance: {}".format(maintenance))
for hm in models.HostMaintenance.objects.filter(maintenance=maintenance):
main_output = {}
hm.status = hm.RUNNING
hm.started_at = datetime.now()
hm.save()
if hm.host is None:
hm.status = hm.UNAVAILABLEHOST
hm.finished_at = datetime.now()
hm.save()
continue
host = hm.host
update_task = "\nRunning Maintenance on {}".format(host)
param_dict = {}
for param in models.MaintenanceParameters.objects.filter(
maintenance=maintenance):
param_function = _get_function(param.function_name)
param_dict[param.parameter_name] = param_function(host.id)
main_script = build_context_script(param_dict, maintenance.main_script)
exit_status = exec_remote_command_host(host, main_script, main_output)
if exit_status == 0:
hm.status = hm.SUCCESS
else:
if maintenance.rollback_script:
rollback_output = {}
hm.status = hm.ROLLBACK
hm.save()
rollback_script = build_context_script(
param_dict, maintenance.rollback_script)
exit_status = exec_remote_command_host(host, rollback_script,
rollback_output)
if exit_status == 0:
hm.status = hm.ROLLBACK_SUCCESS
else:
hm.status = hm.ROLLBACK_ERROR
hm.rollback_log = get_dict_lines(rollback_output)
else:
hm.status = hm.ERROR
update_task += "...status: {}".format(hm.status)
task_history.update_details(persist=True, details=update_task)
hm.main_log = get_dict_lines(main_output)
hm.finished_at = datetime.now()
hm.save()
models.Maintenance.objects.filter(id=maintenance_id, ).update(
status=maintenance.FINISHED, finished_at=datetime.now())
task_history.update_status_for(TaskHistory.STATUS_SUCCESS,
details='Maintenance executed succesfully')
LOG.info("Maintenance: {} has FINISHED".format(maintenance, ))
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
connect_string = build_mongodb_connect_string(
instances=workflow_dict['source_instances'],
databaseinfra=databaseinfra)
client = databaseinfra.get_driver().get_client(None)
rsconf = client.local.system.replset.find_one()
member_ids = []
for member in rsconf['members']:
member_ids.append(member['_id'])
max_member_id = max(member_ids)
secundary_one_member_id = max_member_id + 1
secundary_two_member_id = max_member_id + 2
arbiter_member_id = max_member_id + 3
context_dict = {
'CONNECT_STRING':
connect_string,
'SECUNDARY_ONE':
"{}:{}".format(workflow_dict['target_instances'][0].address,
workflow_dict['target_instances'][0].port),
'SECUNDARY_TWO':
"{}:{}".format(workflow_dict['target_instances'][1].address,
workflow_dict['target_instances'][1].port),
'ARBITER':
"{}:{}".format(workflow_dict['target_instances'][2].address,
workflow_dict['target_instances'][2].port),
'SECUNDARY_ONE_MEMBER_ID':
secundary_one_member_id,
'SECUNDARY_TWO_MEMBER_ID':
secundary_two_member_id,
'ARBITER_MEMBER_ID':
arbiter_member_id,
}
script = test_bash_script_error()
script += build_add_replica_set_members_script()
script = build_context_script(context_dict, script)
output = {}
host = workflow_dict['source_instances'][0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def execute_scheduled_maintenance(self, maintenance_id):
LOG.debug("Maintenance id: {}".format(maintenance_id))
maintenance = models.Maintenance.objects.get(id=maintenance_id)
models.Maintenance.objects.filter(id=maintenance_id).update(
status=maintenance.RUNNING, started_at=datetime.now())
LOG.info("Maintenance {} is RUNNING".format(maintenance, ))
worker_name = get_worker_name()
task_history = TaskHistory.register(request=self.request,
worker_name=worker_name)
task_history.relevance = TaskHistory.RELEVANCE_CRITICAL
LOG.info("id: {} | task: {} | kwargs: {} | args: {}".format(
self.request.id, self.request.task, self.request.kwargs,
str(self.request.args)))
task_history.update_details(
persist=True, details="Executing Maintenance: {}".format(maintenance))
for hm in models.HostMaintenance.objects.filter(maintenance=maintenance):
# main_output = {}
hm.status = hm.RUNNING
hm.started_at = datetime.now()
hm.save()
if hm.host is None:
hm.status = hm.UNAVAILABLEHOST
hm.finished_at = datetime.now()
hm.save()
continue
host = hm.host
update_task = "\nRunning Maintenance on {}".format(host)
if maintenance.disable_alarms:
disable_alarms(hm.host)
param_dict = {}
params = models.MaintenanceParameters.objects.filter(
maintenance=maintenance)
for param in params:
param_function = get_function(param.function_name)
param_dict[param.parameter_name] = param_function(host.id)
main_script = build_context_script(param_dict, maintenance.main_script)
main_output = host.ssh.run_script(script=main_script,
raise_if_error=False)
if main_output['exit_code'] == 0:
hm.status = hm.SUCCESS
else:
if maintenance.rollback_script:
hm.status = hm.ROLLBACK
hm.save()
rollback_script = build_context_script(
param_dict, maintenance.rollback_script)
rollback_output = host.ssh.run_script(script=rollback_script,
raise_if_error=False)
if rollback_output['exit_code'] == 0:
hm.status = hm.ROLLBACK_SUCCESS
else:
hm.status = hm.ROLLBACK_ERROR
hm.rollback_log = get_dict_lines(rollback_output)
else:
hm.status = hm.ERROR
if maintenance.disable_alarms:
enable_alarms(hm.host)
update_task += "...status: {}".format(hm.status)
task_history.update_details(persist=True, details=update_task)
hm.main_log = get_dict_lines(main_output)
hm.finished_at = datetime.now()
hm.save()
models.Maintenance.objects.filter(id=maintenance_id).update(
status=maintenance.FINISHED, finished_at=datetime.now())
task_history.update_status_for(TaskHistory.STATUS_SUCCESS,
details='Maintenance executed succesfully')
LOG.info("Maintenance: {} has FINISHED".format(maintenance))
def do(self):
if not self.is_valid:
return
template_script = self.plan.script.metric_collector_template
script = build_context_script(self.script_variables, template_script)
return self.exec_script(script)
def do(self, workflow_dict):
try:
LOG.info("Getting cloudstack credentials...")
statsd_credentials = get_credentials_for(
environment=workflow_dict['target_environment'],
credential_type=CredentialType.STATSD)
statsd_host, statsd_port = statsd_credentials.endpoint.split(':')
databaseinfra = workflow_dict['databaseinfra']
sentinel = databaseinfra.get_driver().get_sentinel_client()
master = sentinel.discover_master(databaseinfra.name)
master_host = master[0]
master_port = master[1]
for index, source_host in enumerate(workflow_dict['source_hosts']):
target_host = source_host.future_host
LOG.info(target_host)
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
if index == 2:
LOG.info("Cheking host ssh...")
host_ready = check_ssh(
server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
wait=5,
interval=10)
if not host_ready:
raise Exception(
str("Host %s is not ready..." % target_host))
script = test_bash_script_error()
script += build_permission_script()
script = build_context_script({}, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(
server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
instances_redis = Instance.objects.filter(
hostname=target_host, instance_type=Instance.REDIS)
instances_sentinel = Instance.objects.filter(
hostname=target_host,
instance_type=Instance.REDIS_SENTINEL)
if instances_redis:
only_sentinel = False
instance_redis_address = instances_redis[0].address
instance_redis_port = instances_redis[0].port
else:
only_sentinel = True
instance_redis_address = ''
instance_redis_port = ''
if instances_sentinel:
instance_sentinel_address = instances_sentinel[0].address
instance_sentinel_port = instances_sentinel[0].port
else:
instance_sentinel_address = ''
instance_sentinel_port = ''
contextdict = {
'DATABASENAME': workflow_dict['database'].name,
'DBPASSWORD': databaseinfra.password,
'HOSTADDRESS': instance_redis_address,
'PORT': instance_redis_port,
'ENGINE': 'redis',
'HOST': source_host.hostname.split('.')[0],
'STATSD_HOST': statsd_host,
'STATSD_PORT': statsd_port,
'IS_HA': databaseinfra.plan.is_ha,
'SENTINELMASTER': master_host,
'SENTINELMASTERPORT': master_port,
'SENTINELADDRESS': instance_sentinel_address,
'SENTINELPORT': instance_sentinel_port,
'MASTERNAME': databaseinfra.name,
'ONLY_SENTINEL': only_sentinel,
}
planattr = PlanAttr.objects.get(
plan=workflow_dict['source_plan'])
script = build_context_script(contextdict,
planattr.configuration_script)
output = {}
LOG.info(script)
return_code = exec_remote_command(
server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
if index < 2:
change_slave_priority_file(host=target_host,
original_value=100,
final_value=0)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
flipper_fox_migration_dir = Configuration.get_by_name(
'flipper_fox_migration_dir')
if not flipper_fox_migration_dir:
flipper_fox_migration_dir = '/tmp'
workflow_dict['flipper_fox_migration_dir_infra_name'] = "{}/{}".format(
flipper_fox_migration_dir, workflow_dict['databaseinfra'].name)
for index, source_instance in enumerate(workflow_dict['source_instances']):
source_host = source_instance.hostname
source_cs_host_attr = CS_HostAttr.objects.get(host=source_host)
hostname = source_host.hostname.split('.')[0]
localpath = "{}/{}".format(
workflow_dict['flipper_fox_migration_dir_infra_name'], hostname)
os.makedirs(localpath)
LOG.info('Get source host files to {}'.format(localpath))
if not scp_get_file(server=source_host.address,
username=source_cs_host_attr.vm_user,
password=source_cs_host_attr.vm_password,
localpath="{}/my.cnf".format(localpath),
remotepath="/etc/my.cnf"):
raise Exception("FTP Error")
target_host = source_host.future_host
LOG.info(target_host)
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
if not scp_put_file(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
localpath="{}/my.cnf".format(localpath),
remotepath="/etc/my.cnf"):
raise Exception("FTP Error")
script = test_bash_script_error()
script += build_server_id_conf_script()
context_dict = {
'SERVERID': index + 5,
}
script = build_context_script(context_dict, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
shutil.rmtree(workflow_dict['flipper_fox_migration_dir_infra_name'])
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
statsd_credentials = get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.STATSD)
statsd_host, statsd_port = statsd_credentials.endpoint.split(':')
for index, hosts in enumerate(permutations(
workflow_dict['hosts'])):
LOG.info("Getting vm credentials...")
host_csattr = CsHostAttr.objects.get(host=hosts[0])
LOG.info("Cheking host ssh...")
host_ready = check_ssh(server=hosts[0].address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
wait=5,
interval=10)
if not host_ready:
LOG.warn("Host %s is not ready..." % hosts[0])
return False
host_nfsattr = HostAttr.objects.get(host=hosts[0])
planattr = PlanAttr.objects.get(plan=workflow_dict['plan'])
contextdict = {
'EXPORTPATH':
host_nfsattr.nfsaas_path,
'DATABASENAME':
workflow_dict['name'],
'DBPASSWORD':
get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.MYSQL).password,
'HOST':
workflow_dict['hosts'][index].hostname.split('.')[0],
'ENGINE':
'mysql',
'STATSD_HOST':
statsd_host,
'STATSD_PORT':
statsd_port,
}
if len(workflow_dict['hosts']) > 1:
LOG.info("Updating contexdict for %s" % hosts[0])
contextdict.update({
'SERVERID':
index + 1,
'IPMASTER':
hosts[1].address,
'IPWRITE':
workflow_dict['databaseinfraattr'][0].ip,
'IPREAD':
workflow_dict['databaseinfraattr'][1].ip,
'MASTERPAIRNAME':
workflow_dict['databaseinfra'].name,
'HOST01':
workflow_dict['hosts'][0],
'HOST02':
workflow_dict['hosts'][1],
'INSTANCE01':
workflow_dict['instances'][0],
'INSTANCE02':
workflow_dict['instances'][1],
'SECOND_SCRIPT_FILE':
'/opt/dbaas/scripts/dbaas_second_script.sh'
})
scripts = (planattr.initialization_script,
planattr.configuration_script,
planattr.start_database_script)
host = hosts[0]
for script in scripts:
LOG.info("Executing script on %s" % host)
script = build_context_script(contextdict, script)
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=script)
if return_code != 0:
return False
if len(workflow_dict['hosts']) > 1:
for hosts in permutations(workflow_dict['hosts']):
script = planattr.start_replication_script
host = hosts[0]
contextdict.update({'IPMASTER': hosts[1].address})
script = build_context_script(contextdict, script)
host_csattr = CsHostAttr.objects.get(host=host)
LOG.info("Executing script on %s" % host)
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=script)
if return_code != 0:
return False
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0013)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
LOG.info("Getting cloudstack credentials...")
cs_credentials = get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.CLOUDSTACK)
cs_provider = CloudStackProvider(credentials=cs_credentials)
statsd_credentials = get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.STATSD)
statsd_host, statsd_port = statsd_credentials.endpoint.split(':')
for index, instance in enumerate(workflow_dict['instances']):
host = instance.hostname
LOG.info("Getting vm credentials...")
host_csattr = CsHostAttr.objects.get(host=host)
LOG.info("Cheking host ssh...")
host_ready = check_ssh(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
wait=5,
interval=10)
if not host_ready:
LOG.warn("Host %s is not ready..." % host)
return False
host_nfsattr = HostAttr.objects.get(host=host)
planattr = PlanAttr.objects.get(plan=workflow_dict['plan'])
contextdict = {
'EXPORTPATH': host_nfsattr.nfsaas_path,
'DATABASENAME': workflow_dict['name'],
'DBPASSWORD': workflow_dict['databaseinfra'].password,
'HOSTADDRESS': instance.address,
'PORT': instance.port,
'ENGINE': 'redis',
'DATABASENAME': workflow_dict['name'],
'HOST':
workflow_dict['hosts'][index].hostname.split('.')[0],
'STATSD_HOST': statsd_host,
'STATSD_PORT': statsd_port,
}
LOG.info(contextdict)
LOG.info("Updating userdata for %s" % host)
planattr = PlanAttr.objects.get(plan=workflow_dict['plan'])
script = build_context_script(contextdict, planattr.userdata)
#cs_provider.update_userdata(
# vm_id=host_csattr.vm_id, contextdict=contextdict, userdata=planattr.userdata)
LOG.info("Executing script on %s" % host)
LOG.info(script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
return False
return True
except Exception, e:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0016)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
LOG.info("Getting cloudstack credentials...")
statsd_credentials = get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.STATSD)
statsd_host, statsd_port = statsd_credentials.endpoint.split(':')
for index, host in enumerate(workflow_dict['hosts']):
LOG.info("Getting vm credentials...")
host_csattr = CsHostAttr.objects.get(host=host)
LOG.info("Cheking host ssh...")
host_ready = check_ssh(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
wait=5,
interval=10)
if not host_ready:
LOG.warn("Host %s is not ready..." % host)
return False
instances_redis = Instance.objects.filter(
hostname=host, instance_type=Instance.REDIS)
instances_sentinel = Instance.objects.filter(
hostname=host, instance_type=Instance.REDIS_SENTINEL)
if instances_redis:
host_nfsattr = HostAttr.objects.get(host=host)
nfsaas_path = host_nfsattr.nfsaas_path
only_sentinel = False
instance_redis_address = instances_redis[0].address
instance_redis_port = instances_redis[0].port
else:
nfsaas_path = ""
only_sentinel = True
instance_redis_address = ''
instance_redis_port = ''
if instances_sentinel:
instance_sentinel_address = instances_sentinel[0].address
instance_sentinel_port = instances_sentinel[0].port
else:
instance_sentinel_address = ''
instance_sentinel_port = ''
if index == 0:
master_host = instance_redis_address
master_port = instance_redis_port
contextdict = {
'EXPORTPATH': nfsaas_path,
'DATABASENAME': workflow_dict['name'],
'DBPASSWORD': workflow_dict['databaseinfra'].password,
'HOSTADDRESS': instance_redis_address,
'PORT': instance_redis_port,
'ENGINE': 'redis',
'DATABASENAME': workflow_dict['name'],
'HOST': host.hostname.split('.')[0],
'STATSD_HOST': statsd_host,
'STATSD_PORT': statsd_port,
'IS_HA': workflow_dict['databaseinfra'].plan.is_ha,
'SENTINELMASTER': master_host,
'SENTINELMASTERPORT': master_port,
'SENTINELADDRESS': instance_sentinel_address,
'SENTINELPORT': instance_sentinel_port,
'MASTERNAME': workflow_dict['databaseinfra'].name,
'ONLY_SENTINEL': only_sentinel,
}
LOG.info(contextdict)
LOG.info("Updating userdata for %s" % host)
planattr = PlanAttr.objects.get(plan=workflow_dict['plan'])
script = build_context_script(contextdict, planattr.userdata)
LOG.info("Executing script on %s" % host)
LOG.info(script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
return False
if index > 0 and instances_redis:
client = instances_redis[0].databaseinfra.get_driver(
).get_client(instances_redis[0])
client.slaveof(master_host, master_port)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0016)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
mongodbkey = ''.join(
random.choice(string.hexdigits) for i in range(50))
workflow_dict['replicasetname'] = 'ReplicaSet_' + \
workflow_dict['databaseinfra'].name
mongodb_password = get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.MONGODB).password
for index, instance in enumerate(workflow_dict['instances']):
host = instance.hostname
LOG.info("Getting vm credentials...")
host_csattr = CsHostAttr.objects.get(host=host)
LOG.info("Cheking host ssh...")
host_ready = check_ssh(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
wait=5,
interval=10)
if not host_ready:
LOG.warn("Host %s is not ready..." % host)
return False
host.update_os_description()
if instance.is_arbiter:
contextdict = {
'HOST':
workflow_dict['hosts'][index].hostname.split('.')[0],
'DATABASENAME':
workflow_dict['name'],
'ENGINE':
'mongodb',
'IS_HA':
workflow_dict['databaseinfra'].plan.is_ha
}
databaserule = 'ARBITER'
else:
host_nfsattr = HostAttr.objects.get(host=host)
contextdict = {
'EXPORTPATH':
host_nfsattr.nfsaas_path,
'HOST':
workflow_dict['hosts'][index].hostname.split('.')[0],
'DATABASENAME':
workflow_dict['name'],
'ENGINE':
'mongodb',
'DBPASSWORD':
mongodb_password,
'IS_HA':
workflow_dict['databaseinfra'].plan.is_ha
}
if index == 0:
databaserule = 'PRIMARY'
else:
databaserule = 'SECONDARY'
if len(workflow_dict['hosts']) > 1:
LOG.info("Updating contexdict for %s" % host)
contextdict.update({
'REPLICASETNAME':
workflow_dict['replicasetname'],
'HOST01':
workflow_dict['hosts'][0],
'HOST02':
workflow_dict['hosts'][1],
'HOST03':
workflow_dict['hosts'][2],
'MONGODBKEY':
mongodbkey,
'DATABASERULE':
databaserule,
'HOST':
workflow_dict['hosts'][index].hostname.split('.')[0],
})
else:
contextdict.update({'DATABASERULE': databaserule})
planattr = PlanAttr.objects.get(plan=workflow_dict['plan'])
scripts = (planattr.initialization_script,
planattr.configuration_script,
planattr.start_database_script)
for script in scripts:
LOG.info("Executing script on %s" % host)
script = build_context_script(contextdict, script)
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=script)
if return_code != 0:
return False
if len(workflow_dict['hosts']) > 1:
scripts_to_run = planattr.start_replication_script
contextdict.update({
'DBPASSWORD': mongodb_password,
'DATABASERULE': 'PRIMARY'
})
scripts_to_run = build_context_script(contextdict,
scripts_to_run)
host = workflow_dict['hosts'][0]
host_csattr = CsHostAttr.objects.get(host=host)
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=scripts_to_run)
if return_code != 0:
return False
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0014)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
offering = workflow_dict['plan'].stronger_offering
configuration = configuration_factory(
workflow_dict['databaseinfra'], offering.memory_size_mb)
graylog_credential = get_credentials_for(
environment=workflow_dict['databaseinfra'].environment,
credential_type=CredentialType.GRAYLOG)
graylog_endpoint = graylog_credential.get_parameter_by_name(
'endpoint_log')
replica_credential = get_credentials_for(
environment=workflow_dict['databaseinfra'].environment,
credential_type=CredentialType.MYSQL_REPLICA)
plan = workflow_dict['plan']
for index, hosts in enumerate(permutations(
workflow_dict['hosts'])):
LOG.info("Getting vm credentials...")
host = hosts[0]
LOG.info("Cheking host ssh...")
host_ready = check_ssh(host, retries=60, wait=30, interval=10)
if not host_ready:
LOG.warn("Host %s is not ready..." % host)
return False
host_nfsattr = HostAttr.objects.get(host=host)
contextdict = {
'EXPORTPATH':
host_nfsattr.nfsaas_path,
'DATABASENAME':
workflow_dict['name'],
'DBPASSWORD':
get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.MYSQL).password,
'HOST':
workflow_dict['hosts'][index].hostname.split('.')[0],
'ENGINE':
'mysql',
'ENVIRONMENT':
workflow_dict['databaseinfra'].environment,
'configuration':
configuration,
'GRAYLOG_ENDPOINT':
graylog_endpoint,
}
if len(workflow_dict['hosts']) > 1:
LOG.info("Updating contexdict for %s" % host)
contextdict.update({
'SERVERID':
index + 1,
'IPMASTER':
hosts[1].address,
'REPLICA_USER':
replica_credential.user,
'REPLICA_PASSWORD':
replica_credential.password,
})
scripts = (plan.script.initialization_template,
plan.script.configuration_template,
plan.script.start_database_template)
host.update_os_description()
for script in scripts:
LOG.info("Executing script on %s" % host)
script = build_context_script(contextdict, script)
output = {}
return_code = exec_remote_command_host(
host, script, output)
if return_code != 0:
error_msg = "Error executing script. Stdout: {} - " \
"stderr: {}".format(output['stdout'],
output['stderr'])
raise Exception(error_msg)
if len(workflow_dict['hosts']) > 1:
for hosts in permutations(workflow_dict['hosts']):
script = plan.script.start_replication_template
host = hosts[0]
contextdict.update({'IPMASTER': hosts[1].address})
script = build_context_script(contextdict, script)
LOG.info("Executing script on %s" % host)
output = {}
return_code = exec_remote_command_host(
host, script, output)
if return_code != 0:
error_msg = "Error executing script. output: {}".format(
output)
raise Exception(error_msg)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0013)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
cloud_stack = workflow_dict['plan'].cs_plan_attributes.first()
offering = cloud_stack.get_stronger_offering()
configuration = configuration_factory(
workflow_dict['databaseinfra'], offering.memory_size_mb)
mongodbkey = ''.join(
random.choice(string.hexdigits) for i in range(50))
infra = workflow_dict['databaseinfra']
if infra.plan.is_ha:
infra.database_key = mongodbkey
infra.save()
workflow_dict['replicasetname'] = infra.get_driver(
).replica_set_name
mongodb_password = get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.MONGODB).password
disk_offering = workflow_dict['plan'].disk_offering
graylog_credential = get_credentials_for(
environment=workflow_dict['databaseinfra'].environment,
credential_type=CredentialType.GRAYLOG)
graylog_endpoint = graylog_credential.get_parameter_by_name(
'endpoint_log')
plan = workflow_dict['plan']
for index, instance in enumerate(workflow_dict['instances']):
host = instance.hostname
LOG.info("Getting vm credentials...")
LOG.info("Cheking host ssh...")
host_ready = check_ssh(host, wait=5, interval=10)
if not host_ready:
LOG.warn("Host %s is not ready..." % host)
return False
host.update_os_description()
if instance.instance_type == instance.MONGODB_ARBITER:
contextdict = {
'HOST':
workflow_dict['hosts'][index].hostname.split('.')[0],
'DATABASENAME':
workflow_dict['name'],
'ENGINE':
'mongodb',
'DRIVER_NAME':
infra.get_driver().topology_name(),
'configuration':
configuration,
}
databaserule = 'ARBITER'
else:
host_nfsattr = HostAttr.objects.get(host=host)
contextdict = {
'EXPORTPATH':
host_nfsattr.nfsaas_path,
'HOST':
workflow_dict['hosts'][index].hostname.split('.')[0],
'DATABASENAME':
workflow_dict['name'],
'ENGINE':
'mongodb',
'DBPASSWORD':
mongodb_password,
'DRIVER_NAME':
infra.get_driver().topology_name(),
'configuration':
configuration,
}
if index == 0:
databaserule = 'PRIMARY'
else:
databaserule = 'SECONDARY'
if len(workflow_dict['hosts']) > 1:
LOG.info("Updating contexdict for %s" % host)
contextdict.update({
'REPLICASETNAME':
workflow_dict['replicasetname'],
'HOST01':
workflow_dict['hosts'][0],
'HOST02':
workflow_dict['hosts'][1],
'HOST03':
workflow_dict['hosts'][2],
'MONGODBKEY':
mongodbkey,
'DATABASERULE':
databaserule,
'HOST':
workflow_dict['hosts'][index].hostname.split('.')[0],
})
else:
contextdict.update({'DATABASERULE': databaserule})
contextdict.update({
'ENVIRONMENT':
workflow_dict['databaseinfra'].environment,
'DISK_SIZE_IN_GB':
disk_offering.size_gb(),
'GRAYLOG_ENDPOINT':
graylog_endpoint
})
scripts = (plan.script.initialization_template,
plan.script.configuration_template,
plan.script.start_database_template)
for script in scripts:
LOG.info("Executing script on %s" % host)
script = build_context_script(contextdict, script)
output = {}
return_code = exec_remote_command_host(
host, script, output)
if return_code != 0:
error_msg = "Error executing script. Stdout: {} - " \
"stderr: {}".format(output['stdout'],
output['stderr'])
raise Exception(error_msg)
if len(workflow_dict['hosts']) > 1:
scripts_to_run = plan.script.start_replication_template
contextdict.update({
'DBPASSWORD': mongodb_password,
'DATABASERULE': 'PRIMARY'
})
scripts_to_run = build_context_script(contextdict,
scripts_to_run)
host = workflow_dict['hosts'][0]
output = {}
return_code = exec_remote_command_host(host, scripts_to_run,
output)
if return_code != 0:
error_msg = "Error executing script. Stdout: {} - " \
"stderr: {}".format(output['stdout'],
output['stderr'])
raise Exception(error_msg)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0014)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def _execute_script(self, script_variables, script):
final_script = build_context_script(script_variables, script)
self.host.ssh.run_script(final_script)
def do(self, workflow_dict):
try:
initial_script = '#!/bin/bash\n\ndie_if_error()\n{\n local err=$?\n if [ "$err" != "0" ]; then\n echo "$*"\n exit $err\n fi\n}'
region_migration_dir = Configuration.get_by_name(
'region_migration_dir')
if not region_migration_dir:
region_migration_dir = '/tmp'
workflow_dict['region_migration_dir_infra_name'] = "{}/{}".format(
region_migration_dir, workflow_dict['databaseinfra'].name)
for index, source_instance in enumerate(
workflow_dict['source_instances']):
source_host = source_instance.hostname
source_cs_host_attr = CS_HostAttr.objects.get(host=source_host)
hostname = source_host.hostname.split('.')[0]
localpath = "{}/{}".format(
workflow_dict['region_migration_dir_infra_name'], hostname)
os.makedirs(localpath)
LOG.info('Get source host files to {}'.format(localpath))
if not scp_get_file(
server=source_host.address,
username=source_cs_host_attr.vm_user,
password=source_cs_host_attr.vm_password,
localpath="{}/mongodb.key".format(localpath),
remotepath="/data/mongodb.key"):
raise Exception("FTP Error")
if not scp_get_file(
server=source_host.address,
username=source_cs_host_attr.vm_user,
password=source_cs_host_attr.vm_password,
localpath="{}/mongodb.conf".format(localpath),
remotepath="/data/mongodb.conf"):
raise Exception("FTP Error")
if not scp_get_file(
server=source_host.address,
username=source_cs_host_attr.vm_user,
password=source_cs_host_attr.vm_password,
localpath="{}/td-agent.conf".format(localpath),
remotepath="/etc/td-agent/td-agent.conf"):
raise Exception("FTP Error")
target_host = source_host.future_host
LOG.info(target_host)
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
if not scp_put_file(
server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
localpath="{}/mongodb.key".format(localpath),
remotepath="/data/mongodb.key"):
raise Exception("FTP Error")
if not scp_put_file(
server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
localpath="{}/mongodb.conf".format(localpath),
remotepath="/data/mongodb.conf"):
raise Exception("FTP Error")
if not scp_put_file(
server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
localpath="{}/td-agent.conf".format(localpath),
remotepath="/etc/td-agent/td-agent.conf"):
raise Exception("FTP Error")
script = initial_script
script += '\nmkdir /data/data'
script += '\ndie_if_error "Error creating data dir"'
script += '\nchown mongodb:mongodb /data'
script += '\ndie_if_error "Error changing datadir permission"'
script += '\nchown -R mongodb:mongodb /data/*'
script += '\ndie_if_error "Error changing datadir permission"'
script += '\nchmod 600 /data/mongodb.key'
script += '\ndie_if_error "Error changing mongodb key file permission"'
script += '\necho ""; echo $(date "+%Y-%m-%d %T") "- Starting the database"'
script += '\n/etc/init.d/mongodb start > /dev/null'
script += '\ndie_if_error "Error starting database"'
script = build_context_script({}, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(
server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception, str(output)
shutil.rmtree(workflow_dict['region_migration_dir_infra_name'])
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0019)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def execute_scheduled_maintenance(self, maintenance_id):
LOG.debug("Maintenance id: {}".format(maintenance_id))
maintenance = models.Maintenance.objects.get(id=maintenance_id)
models.Maintenance.objects.filter(id=maintenance_id, ).update(
status=maintenance.RUNNING, started_at=datetime.now())
LOG.info("Maintenance {} is RUNNING".format(maintenance, ))
worker_name = get_worker_name()
task_history = TaskHistory.register(request=self.request,
worker_name=worker_name)
LOG.info("id: %s | task: %s | kwargs: %s | args: %s" %
(self.request.id, self.request.task, self.request.kwargs,
str(self.request.args)))
task_history.update_details(
persist=True, details="Executing Maintenance: {}".format(maintenance))
for hm in models.HostMaintenance.objects.filter(maintenance=maintenance):
main_output = {}
hm.status = hm.RUNNING
hm.started_at = datetime.now()
hm.save()
if hm.host is None:
hm.status = hm.UNAVAILABLEHOST
hm.finished_at = datetime.now()
hm.save()
continue
host = hm.host
update_task = "\nRunning Maintenance on {}".format(host)
try:
cloudstack_host_attributes = host.cs_host_attributes.get()
except ObjectDoesNotExist, e:
LOG.warn("Host {} does not have cloudstack attrs...{}".format(
hm.host, e))
hm.status = hm.UNAVAILABLECSHOSTATTR
hm.finished_at = datetime.now()
hm.save()
continue
param_dict = {}
for param in models.MaintenanceParameters.objects.filter(
maintenance=maintenance):
param_function = _get_function(param.function_name)
param_dict[param.parameter_name] = param_function(host.id)
main_script = build_context_script(param_dict, maintenance.main_script)
exit_status = exec_remote_command(
server=host.address,
username=cloudstack_host_attributes.vm_user,
password=cloudstack_host_attributes.vm_password,
command=main_script,
output=main_output)
if exit_status == 0:
hm.status = hm.SUCCESS
else:
if maintenance.rollback_script:
rollback_output = {}
hm.status = hm.ROLLBACK
hm.save()
rollback_script = build_context_script(
param_dict, maintenance.rollback_script)
exit_status = exec_remote_command(
server=host.address,
username=cloudstack_host_attributes.vm_user,
password=cloudstack_host_attributes.vm_password,
command=rollback_script,
output=rollback_output)
if exit_status == 0:
hm.status = hm.ROLLBACK_SUCCESS
else:
hm.status = hm.ROLLBACK_ERROR
hm.rollback_log = get_dict_lines(rollback_output)
else:
hm.status = hm.ERROR
update_task += "...status: {}".format(hm.status)
task_history.update_details(persist=True, details=update_task)
hm.main_log = get_dict_lines(main_output)
hm.finished_at = datetime.now()
hm.save()
def do(self, workflow_dict):
try:
cloud_stack = workflow_dict['plan'].cs_plan_attributes.first()
offering = cloud_stack.get_stronger_offering()
configuration = configuration_factory(
workflow_dict['databaseinfra'], offering.memory_size_mb)
graylog_credential = get_credentials_for(
environment=workflow_dict['databaseinfra'].environment,
credential_type=CredentialType.GRAYLOG)
graylog_endpoint = graylog_credential.get_parameter_by_name(
'endpoint_log')
plan = workflow_dict['plan']
for index, hosts in enumerate(permutations(
workflow_dict['hosts'])):
LOG.info("Getting vm credentials...")
host_csattr = CsHostAttr.objects.get(host=hosts[0])
LOG.info("Cheking host ssh...")
host_ready = check_ssh(server=hosts[0].address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
wait=5,
interval=10)
if not host_ready:
LOG.warn("Host %s is not ready..." % hosts[0])
return False
host_nfsattr = HostAttr.objects.get(host=hosts[0])
contextdict = {
'EXPORTPATH':
host_nfsattr.nfsaas_path,
'DATABASENAME':
workflow_dict['name'],
'DBPASSWORD':
get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.MYSQL).password,
'HOST':
workflow_dict['hosts'][index].hostname.split('.')[0],
'ENGINE':
'mysql',
'configuration':
configuration,
'ENVIRONMENT':
workflow_dict['databaseinfra'].environment,
'GRAYLOG_ENDPOINT':
graylog_endpoint
}
if len(workflow_dict['hosts']) > 1:
LOG.info("Updating contexdict for %s" % hosts[0])
contextdict.update({
'SERVERID':
index + 1,
'IPMASTER':
hosts[1].address,
'IPWRITE':
workflow_dict['databaseinfraattr'][0].ip,
'IPREAD':
workflow_dict['databaseinfraattr'][1].ip,
'MASTERPAIRNAME':
workflow_dict['databaseinfra'].name,
'HOST01':
workflow_dict['hosts'][0],
'HOST02':
workflow_dict['hosts'][1],
'INSTANCE01':
workflow_dict['instances'][0],
'INSTANCE02':
workflow_dict['instances'][1],
'SECOND_SCRIPT_FILE':
'/opt/dbaas/scripts/dbaas_second_script.sh',
})
scripts = (plan.script.initialization_template,
plan.script.configuration_template,
plan.script.start_database_template)
host = hosts[0]
host.update_os_description()
for script in scripts:
LOG.info("Executing script on %s" % host)
script = build_context_script(contextdict, script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=script,
output=output)
if return_code != 0:
error_msg = "Error executing script. Stdout: {} - " \
"stderr: {}".format(output['stdout'],
output['stderr'])
raise Exception(error_msg)
if len(workflow_dict['hosts']) > 1:
for hosts in permutations(workflow_dict['hosts']):
script = plan.script.start_replication_template
host = hosts[0]
contextdict.update({'IPMASTER': hosts[1].address})
script = build_context_script(contextdict, script)
host_csattr = CsHostAttr.objects.get(host=host)
LOG.info("Executing script on %s" % host)
output = {}
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=script,
output=output)
if return_code != 0:
error_msg = "Error executing script. Stdout: {} - " \
"stderr: {}".format(output['stdout'],
output['stderr'])
raise Exception(error_msg)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0013)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
offering = workflow_dict['plan'].stronger_offering
configuration = configuration_factory(
workflow_dict['databaseinfra'], offering.memory_size_mb)
graylog_credential = get_credentials_for(
environment=workflow_dict['databaseinfra'].environment,
credential_type=CredentialType.GRAYLOG)
graylog_endpoint = graylog_credential.get_parameter_by_name(
'endpoint_log')
plan = workflow_dict['plan']
for index, host in enumerate(workflow_dict['hosts']):
LOG.info("Getting vm credentials...")
LOG.info("Cheking host ssh...")
host_ready = check_ssh(host, wait=5, interval=10)
if not host_ready:
LOG.warn("Host %s is not ready..." % host)
return False
host.update_os_description()
instances_redis = Instance.objects.filter(
hostname=host, instance_type=Instance.REDIS)
instances_sentinel = Instance.objects.filter(
hostname=host, instance_type=Instance.REDIS_SENTINEL)
if instances_redis:
host_nfsattr = HostAttr.objects.get(host=host)
nfsaas_path = host_nfsattr.nfsaas_path
only_sentinel = False
instance_redis_address = instances_redis[0].address
instance_redis_port = instances_redis[0].port
else:
nfsaas_path = ""
only_sentinel = True
instance_redis_address = ''
instance_redis_port = ''
if instances_sentinel:
instance_sentinel_address = instances_sentinel[0].address
instance_sentinel_port = instances_sentinel[0].port
else:
instance_sentinel_address = ''
instance_sentinel_port = ''
if index == 0:
master_host = instance_redis_address
master_port = instance_redis_port
contextdict = {
'EXPORTPATH':
nfsaas_path,
'DATABASENAME':
workflow_dict['name'],
'DBPASSWORD':
workflow_dict['databaseinfra'].password,
'HOSTADDRESS':
instance_redis_address,
'PORT':
instance_redis_port,
'ENGINE':
'redis',
'HOST':
host.hostname.split('.')[0],
'DRIVER_NAME':
workflow_dict['databaseinfra'].get_driver().topology_name(
),
'SENTINELMASTER':
master_host,
'SENTINELMASTERPORT':
master_port,
'SENTINELADDRESS':
instance_sentinel_address,
'SENTINELPORT':
instance_sentinel_port,
'MASTERNAME':
workflow_dict['databaseinfra'].name,
'ONLY_SENTINEL':
only_sentinel,
'HAS_PERSISTENCE':
workflow_dict['plan'].has_persistence,
'ENVIRONMENT':
workflow_dict['databaseinfra'].environment,
'configuration':
configuration,
'GRAYLOG_ENDPOINT':
graylog_endpoint,
}
LOG.info(contextdict)
scripts = (plan.script.initialization_template,
plan.script.configuration_template,
plan.script.start_database_template,
plan.script.start_replication_template)
for script in scripts:
LOG.info("Executing script on %s" % host)
script = build_context_script(contextdict, script)
output = {}
return_code = exec_remote_command_host(
host, script, output)
if return_code != 0:
error_msg = "Error executing script. Stdout: {} - " \
"stderr: {}".format(output['stdout'],
output['stderr'])
raise Exception(error_msg)
if index > 0 and instances_redis:
client = instances_redis[0].databaseinfra.get_driver(
).get_client(instances_redis[0])
client.slaveof(master_host, master_port)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0016)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
