def undo(self, workflow_dict):
try:
LOG.info("Remove all backup log files")
backup_log_dict = get_backup_log_configuration_dict(environment=workflow_dict['environment'],
databaseinfra=workflow_dict['databaseinfra'])
if backup_log_dict is None:
return True
instance = workflow_dict['instances'][0]
host = instance.hostname
host_csattr = CsHostAttr.objects.get(host=host)
exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=backup_log_dict['CLEAN_BACKUP_LOG_SCRIPT'])
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0014)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
script = build_clean_database_dir_script()
script = build_context_script({}, script)
for source_host in workflow_dict['source_hosts']:
target_host = source_host.future_host
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
output = {}
exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def mysql_binlog_save(client, instance, cloudstack_hostattr):
try:
client.query('show master status')
r = client.store_result()
row = r.fetch_row(maxrows=0, how=1)
binlog_file = row[0]['File']
binlog_pos = row[0]['Position']
client.query("show variables like 'datadir'")
r = client.store_result()
row = r.fetch_row(maxrows=0, how=1)
datadir = row[0]['Value']
output = {}
command = 'echo "master=%s;position=%s" > %smysql_binlog_master_file_pos' % (
binlog_file, binlog_pos, datadir)
exec_remote_command(server=instance.hostname.address,
username=cloudstack_hostattr.vm_user,
password=cloudstack_hostattr.vm_password,
command=command,
output=output)
except Exception as e:
LOG.error(
"Error saving mysql master binlog file and position: %s" % (e))
def do(self, workflow_dict):
try:
for host in workflow_dict['source_hosts']:
LOG.info("Removing database files on host %s" % host)
host_csattr = CsHostAttr.objects.get(host=host)
output = {}
exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command="/opt/dbaas/scripts/dbaas_deletedatabasefiles.sh",
output=output)
LOG.info(output)
LOG.info("Removing disks on host %s" % host)
NfsaasProvider().destroy_disk(
environment=workflow_dict['source_environment'], host=host)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
script = build_clean_database_dir_script()
script = build_context_script({}, script)
for source_host in workflow_dict['source_hosts']:
target_host = source_host.future_host
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
output = {}
exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for host in workflow_dict['source_hosts']:
LOG.info("Removing database files on host %s" % host)
host_csattr = CsHostAttr.objects.get(host=host)
output = {}
exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command="/opt/dbaas/scripts/dbaas_deletedatabasefiles.sh",
output=output)
LOG.info(output)
LOG.info("Removing disks on host %s" % host)
NfsaasProvider().destroy_disk(environment=workflow_dict['source_environment'],
host=host)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
script = build_stop_database_script()
script = build_context_script({}, script)
for target_instance in workflow_dict['target_instances']:
target_host = target_instance.hostname
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
output = {}
exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
try:
if 'region_migration_dir_infra_name' in workflow_dict:
shutil.rmtree(workflow_dict['region_migration_dir_infra_name'])
except Exception:
pass
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
try:
LOG.info("Remove all backup log files")
backup_log_dict = get_backup_log_configuration_dict(
environment=workflow_dict['target_environment'],
databaseinfra=workflow_dict['databaseinfra'])
if backup_log_dict is None:
return True
instance = workflow_dict['target_instances'][0]
host = instance.hostname
host_csattr = CsHostAttr.objects.get(host=host)
exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=backup_log_dict['CLEAN_BACKUP_LOG_SCRIPT'])
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0014)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def clean_unused_data(export_id, export_path, host, databaseinfra):
provider = NfsaasProvider()
provider.grant_access(environment=databaseinfra.environment,
host=host,
export_id=export_id)
mount_path = "/mnt_{}_{}".format(databaseinfra.name, export_id)
command = "mkdir -p {}".format(mount_path)
command += "\nmount -t nfs -o bg,intr {} {}".format(
export_path, mount_path)
command += "\nrm -rf {}/*".format(mount_path)
command += "\numount {}".format(mount_path)
command += "\nrm -rf {}".format(mount_path)
LOG.info(command)
cs_host_attr = CsHostAttr.objects.get(host=host)
output = {}
exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
LOG.info(output)
def mysql_binlog_save(client, instance, cloudstack_hostattr):
try:
client.query('show master status')
r = client.store_result()
row = r.fetch_row(maxrows=0, how=1)
binlog_file = row[0]['File']
binlog_pos = row[0]['Position']
client.query("show variables like 'datadir'")
r = client.store_result()
row = r.fetch_row(maxrows=0, how=1)
datadir = row[0]['Value']
output = {}
command = 'echo "master=%s;position=%s" > %smysql_binlog_master_file_pos' % (
binlog_file, binlog_pos, datadir)
exec_remote_command(server=instance.hostname.address,
username=cloudstack_hostattr.vm_user,
password=cloudstack_hostattr.vm_password,
command=command,
output=output)
except Exception as e:
LOG.error(
"Error saving mysql master binlog file and position: %s" % (e))
def get_puppet_code_status(self, host, cloudstack):
output = {}
LOG.info("Puppet-setup LOG info:")
exec_remote_command(server=host.address,
username=cloudstack.vm_user,
password=cloudstack.vm_password,
command="tail -7 /var/log/ks-post.log",
output=output)
for line in output["stdout"]:
if "puppet-setup" in line and "return code:" in line:
return int(line.split("return code: ")[1]), output
return 0, output
def run_vm_script(workflow_dict, context_dict, script):
try:
final_context_dict = dict(context_dict.items())
instance = workflow_dict['instance']
host = workflow_dict['host']
host_csattr = HostAttr.objects.get(host=host)
final_context_dict['HOSTADDRESS'] = instance.address
final_context_dict['PORT'] = instance.port
final_context_dict['DBPASSWORD'] = workflow_dict['databaseinfra'].password
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=command,
output=output)
if return_code:
raise Exception("Could not run script. Output: {}".format(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def run_vm_script(workflow_dict, context_dict, script):
try:
final_context_dict = dict(context_dict.items())
instance = workflow_dict['instance']
host = workflow_dict['host']
host_csattr = HostAttr.objects.get(host=host)
final_context_dict['HOSTADDRESS'] = instance.address
final_context_dict['PORT'] = instance.port
final_context_dict['DBPASSWORD'] = workflow_dict[
'databaseinfra'].password
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=command,
output=output)
if return_code:
raise Exception("Could not run script. Output: {}".format(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
sleep(60)
databaseinfra = workflow_dict['databaseinfra']
driver = databaseinfra.get_driver()
files_to_remove = driver.remove_deprectaed_files()
command = files_to_remove + " && cp -rp /data/* /data2"
host = workflow_dict['host']
cs_host_attr = CsHostAttr.objects.get(host=host)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0022)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
command = "mkdir /data2 && mount -t nfs -o bg,intr {} /data2"
host = workflow_dict['host']
volume = workflow_dict['volume']
command = command.format(volume.nfsaas_path)
cs_host_attr = CsHostAttr.objects.get(host=host)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
if return_code != 0:
raise Exception(str(output))
workflow_dict['mount'] = command
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0022)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def run_vm_script(workflow_dict, context_dict, script):
try:
instances_detail = workflow_dict['instances_detail']
final_context_dict = dict(
context_dict.items() +
workflow_dict['initial_context_dict'].items())
for instance_detail in instances_detail:
host = instance_detail['instance'].hostname
host_csattr = HostAttr.objects.get(host=host)
final_context_dict['IS_MASTER'] = instance_detail['is_master']
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=command,
output=output)
if return_code:
raise Exception, "Could not run script. Output: {}".format(
output)
return True
except Exception, e:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def run_vm_script(workflow_dict, context_dict, script):
try:
instances_detail = workflow_dict['instances_detail']
final_context_dict = dict(context_dict.items() + workflow_dict['initial_context_dict'].items())
for instance_detail in instances_detail:
instance = instance_detail['instance']
host = instance.hostname
host_csattr = HostAttr.objects.get(host=host)
final_context_dict['HOSTADDRESS'] = instance.address
final_context_dict['PORT'] = instance.port
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(server = host.address,
username = host_csattr.vm_user,
password = host_csattr.vm_password,
command = command,
output = output)
if return_code:
raise Exception, "Could not run script. Output: {}".format(output)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
host = workflow_dict['source_hosts'][0]
databaseinfra = workflow_dict['databaseinfra']
cs_host_attr = CS_HostAttr.objects.get(host=host)
context_dict = {
'MASTERPAIRNAME': databaseinfra.name,
}
script = test_bash_script_error()
script += build_turn_flipper_ip_down_script()
script = build_context_script(context_dict, script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
command = 'mount /data'
for host_and_export in workflow_dict['hosts_and_exports']:
host = host_and_export['host']
LOG.info('mount data volume on host {}'.format(host))
cs_host_attr = CsHostAttr.objects.get(host=host)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
if return_code != 0:
LOG.info(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0021)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def change_instance_binaries(self, instance, connect_string, run_authschemaupgrade):
script = test_bash_script_error()
script += util.build_cp_mongodb_binary_file()
script += util.build_stop_database_script(clean_data=False)
script += util.build_change_release_alias_script()
script += util.build_start_database_script(wait_time=30)
script += util.build_change_limits_script()
script += util.build_remove_reprecated_index_counter_metrics()
if run_authschemaupgrade:
script += util.build_authschemaupgrade_script()
context_dict = {
'SOURCE_PATH': '/mnt/software/db/mongodb',
'TARGET_PATH': '/usr/local/',
'MONGODB_RELEASE_FILE': 'mongodb-linux-x86_64-3.0.8.tgz',
'MONGODB_RELEASE_FOLDER': 'mongodb-linux-x86_64-3.0.8',
'CONNECT_STRING': connect_string,
}
script = build_context_script(context_dict, script)
output = {}
host = instance.hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def run_vm_script(workflow_dict, context_dict, script):
try:
instances_detail = workflow_dict["instances_detail"]
final_context_dict = dict(context_dict.items() + workflow_dict["initial_context_dict"].items())
for instance_detail in instances_detail:
host = instance_detail["instance"].hostname
host_csattr = HostAttr.objects.get(host=host)
final_context_dict["IS_MASTER"] = instance_detail["is_master"]
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=command,
output=output,
)
if return_code:
raise Exception, "Could not run script. Output: {}".format(output)
return True
except Exception:
traceback = full_stack()
workflow_dict["exceptions"]["error_codes"].append(DBAAS_0015)
workflow_dict["exceptions"]["traceback"].append(traceback)
return False
def do(self, workflow_dict):
try:
sleep(60)
databaseinfra = workflow_dict['databaseinfra']
driver = databaseinfra.get_driver()
files_to_remove = driver.remove_deprectaed_files()
command = files_to_remove + " && cp -rp /data/* /data2"
host = workflow_dict['host']
cs_host_attr = CsHostAttr.objects.get(host=host)
output = {}
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0022)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
databaseinfra = workflow_dict['databaseinfra']
host = workflow_dict['host']
return_code, output = use_database_initialization_script(
databaseinfra=databaseinfra, host=host, option='stop')
if return_code != 0:
LOG.info(str(output))
cs_host_attr = CsHostAttr.objects.get(host=host)
output = {}
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command='umount /data',
output=output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0022)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
databaseinfra = workflow_dict['databaseinfra']
host = workflow_dict['host']
return_code, output = use_database_initialization_script(databaseinfra=databaseinfra,
host=host,
option='stop')
if return_code != 0:
LOG.info(str(output))
cs_host_attr = CsHostAttr.objects.get(host=host)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command='umount /data',
output=output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0022)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def host_mount_data_percentage(address, task):
host = Host.objects.filter(address=address).first()
vm = host.cs_host_attributes.first()
output_message = {}
command_status = exec_remote_command(server=host.address,
username=vm.vm_user,
password=vm.vm_password,
command='df -hk | grep /data',
output=output_message)
if command_status != 0:
task.add_detail(
message='Could not load mount size: {}'.format(output_message),
level=4)
return None, None, None
values = output_message['stdout'][0].strip().split()
values = {
'total': int(values[0]),
'used': int(values[1]),
'free': int(values[2]),
'percentage': int(values[3].replace('%', ''))
}
task.add_detail(message='Mount /data: {}% ({}kb/{}kb)'.format(
values['percentage'], values['used'], values['total']),
level=3)
return values['percentage'], values['used'], values['total']
def change_instance_binaries(self, instance):
script = test_bash_script_error()
script += build_change_mongodb_conf_file_script()
script += build_stop_database_script(clean_data=False)
script += build_clean_data_data_script()
script += build_start_database_script()
script += build_wait_admin_be_created_script()
script += td_agent_script(option='restart')
context_dict = {}
script = build_context_script(context_dict, script)
output = {}
host = instance.hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def do(self, workflow_dict):
try:
option = 'start'
for host in workflow_dict['hosts']:
LOG.info("{} monit on host {}".format(option, host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
script = test_bash_script_error()
script += monit_script(option)
LOG.info(script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
LOG.error("Error monit")
LOG.error(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0002)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for source_host in workflow_dict['source_hosts']:
future_host = source_host.future_host
hosts_option = [(future_host, 'stop')]
for host, option in hosts_option:
LOG.info("{} td_agent on host {}".format(option, host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
script = test_bash_script_error()
script += monit_script(option)
script += td_agent_script(option)
LOG.info(script)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
LOG.error("Error stopping td_agent")
LOG.error(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
command = 'rm -rf /data2 && {} && mount /data'
host = workflow_dict['host']
cs_host_attr = CsHostAttr.objects.get(host=host)
mount = workflow_dict['mount']
command = command.format(mount)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
if return_code != 0:
LOG.info(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0021)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
for source_host in workflow_dict['source_hosts']:
future_host = source_host.future_host
cs_host_attr = CS_HostAttr.objects.get(host=future_host)
LOG.info("Removing rsyslog config in {}".format(future_host))
script = self.rsyslog_remove_config()
LOG.info(script)
output = {}
return_code = exec_remote_command(
server=future_host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
error_msg = "Error removing configuring rsyslog: {}".format(
str(output))
LOG.error(error_msg)
raise EnvironmentError(error_msg)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def switch_master(self):
master = self.get_master_instance()
slave = self.get_slave_instances()[0]
host = master.hostname
host_attr = HostAttr.objects.get(host=host)
script = """
sudo -u flipper /usr/bin/flipper {{MASTERPAIRNAME}} set write {{HOST01.address}}
sudo -u flipper /usr/bin/flipper {{MASTERPAIRNAME}} set read {{HOST02.address}}
"""
context_dict = {
'MASTERPAIRNAME': self.databaseinfra.name,
'HOST01': slave.hostname,
'HOST02': master.hostname,
}
script = build_context_script(context_dict, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_attr.vm_user,
password=host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def do(self, workflow_dict):
try:
for source_host in workflow_dict['source_hosts']:
host = source_host.future_host
LOG.info("Starting td_agent on host {}".format(host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
context_dict = {}
script = test_bash_script_error()
script += build_start_td_agent_script()
script = build_context_script(context_dict, script)
LOG.info(script)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
LOG.error("Error starting td_agent")
LOG.error(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
option = 'restart'
for host in workflow_dict['hosts']:
LOG.info("{} monit on host {}".format(option, host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
script = test_bash_script_error()
script += td_agent_script(option)
LOG.info(script)
output = {}
sleep(30)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
LOG.error("Error monit")
LOG.error(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0002)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
driver = databaseinfra.get_driver()
host = workflow_dict['host']
host_attr = HostAttr.objects.get(host=host)
sleep(30)
for agent in driver.get_database_agents():
script = '/etc/init.d/{} start'.format(agent)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_attr.vm_user,
password=host_attr.vm_password,
command=script,
output=output)
LOG.info('Running {} - Return Code: {}. Output scrit: {}'.format(
script, return_code, output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0022)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for host in workflow_dict['target_hosts']:
cs_host_attr = CS_HostAttr.objects.get(host=host)
script = test_bash_script_error()
script += build_uncomment_skip_slave_script()
script = build_context_script({}, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
driver = databaseinfra.get_driver()
host = workflow_dict['host']
host_attr = HostAttr.objects.get(host=host)
sleep(30)
for agent in driver.get_database_agents():
script = '/etc/init.d/{} start'.format(agent)
output = {}
return_code = exec_remote_command(
server=host.address,
username=host_attr.vm_user,
password=host_attr.vm_password,
command=script,
output=output)
LOG.info(
'Running {} - Return Code: {}. Output scrit: {}'.format(
script, return_code, output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0022)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
script = "puppet-setup"
for host in workflow_dict['target_hosts']:
LOG.info("Getting vm credentials...")
host_csattr = CsHostAttr.objects.get(host=host)
host.update_os_description()
LOG.info("Run puppet-setup on host {}".format(host))
output = {}
return_code = exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=script,
output=output)
#if return_code != 0:
# raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0013)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def switch_master(self, driver):
master = driver.get_master_instance()
slave = driver.get_slave_instances()[0]
host = master.hostname
host_attr = HostAttr.objects.get(host=host)
script = """
sudo -u flipper /usr/bin/flipper {{MASTERPAIRNAME}} set write {{HOST01.address}}
sudo -u flipper /usr/bin/flipper {{MASTERPAIRNAME}} set read {{HOST02.address}}
"""
context_dict = {
'MASTERPAIRNAME': driver.databaseinfra.name,
'HOST01': slave.hostname,
'HOST02': master.hostname,
}
script = build_context_script(context_dict, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_attr.vm_user,
password=host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
def run_vm_script(workflow_dict, context_dict, script, reverse=False, wait=0):
try:
instances_detail = workflow_dict['instances_detail']
final_context_dict = dict(context_dict.items() + workflow_dict['initial_context_dict'].items())
if reverse:
instances_detail_final = instances_detail[::-1]
else:
instances_detail_final = instances_detail
for instance_detail in instances_detail_final:
host = instance_detail['instance'].hostname
host_csattr = HostAttr.objects.get(host=host)
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(server = host.address,
username = host_csattr.vm_user,
password = host_csattr.vm_password,
command = command,
output = output)
if return_code:
raise Exception, "Could not run script. Output: {}".format(output)
sleep(wait)
return True
except Exception, e:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
command = 'umount /data2 && rmdir /data2'
host = workflow_dict['host']
cs_host_attr = CsHostAttr.objects.get(host=host)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
if return_code != 0:
LOG.info(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0022)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for source_host in workflow_dict['source_hosts']:
future_host = source_host.future_host
hosts_option = [(source_host, 'stop'), (future_host, 'start')]
for host, option in hosts_option:
LOG.info("Starting td_agent on host {}".format(host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
context_dict = {}
script = test_bash_script_error()
script += build_mk_heartbeat_daemon_script(option)
script = build_context_script(context_dict, script)
LOG.info(script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
LOG.error("Error starting mysql_statsd")
LOG.error(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
script = "ps -ef | grep bootstrap-puppet3-loop.sh | grep -v grep | wc -l"
for host in workflow_dict['target_hosts']:
LOG.info("Getting vm credentials...")
host_csattr = CsHostAttr.objects.get(host=host)
attempt = 1
retries = 60
interval = 20
sleep(interval)
while True:
LOG.info(
"Check if puppet-setup is runnig on {} - attempt {} of {}"
.format(host, attempt, retries))
output = {}
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=script,
output=output)
if return_code != 0:
raise Exception(str(output))
ret_value = int(output['stdout'][0])
if ret_value == 0:
LOG.info(
"Puppet-setup is not runnig on {}".format(host))
break
LOG.info("Puppet-setup is runnig on {}".format(host))
attempt += 1
if attempt == retries:
error = "Maximum number of attempts check is puppet is running on {}.".format(
host)
LOG.error(error)
raise Exception(error)
sleep(interval)
puppet_code_status, output = self.get_puppet_code_status(
host, host_csattr)
if puppet_code_status != 0:
message = "Puppet-setup returned an error on {}. Output: {}".format(
host, output)
raise EnvironmentError(message)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0013)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for index, instance in enumerate(
workflow_dict['target_instances']):
if instance.instance_type == instance.MONGODB_ARBITER:
continue
if instance.instance_type == instance.REDIS_SENTINEL:
continue
host = instance.hostname
LOG.info("Mounting disks on host {}".format(host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
nfs_host_attr = NFS_HostAttr.objects.get(host=host)
LOG.info("Cheking host ssh...")
host_ready = check_ssh(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
wait=5,
interval=10)
if not host_ready:
raise Exception(str("Host %s is not ready..." % host))
context_dict = {
'EXPORTPATH': nfs_host_attr.nfsaas_path,
}
script = test_bash_script_error()
script += build_mount_disk_script()
script = build_context_script(context_dict, script)
LOG.info(script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
connect_string = build_mongodb_connect_string(instances=workflow_dict['source_instances'],
databaseinfra=databaseinfra)
client = databaseinfra.get_driver().get_client(None)
rsconf = client.local.system.replset.find_one()
member_ids = []
for member in rsconf['members']:
member_ids.append(member['_id'])
max_member_id = max(member_ids)
secundary_one_member_id = max_member_id + 1
secundary_two_member_id = max_member_id + 2
arbiter_member_id = max_member_id + 3
context_dict = {
'CONNECT_STRING': connect_string,
'SECUNDARY_ONE': "{}:{}".format(workflow_dict['target_instances'][0].address, workflow_dict['target_instances'][0].port),
'SECUNDARY_TWO': "{}:{}".format(workflow_dict['target_instances'][1].address, workflow_dict['target_instances'][1].port),
'ARBITER': "{}:{}".format(workflow_dict['target_instances'][2].address, workflow_dict['target_instances'][2].port),
'SECUNDARY_ONE_MEMBER_ID': secundary_one_member_id,
'SECUNDARY_TWO_MEMBER_ID': secundary_two_member_id,
'ARBITER_MEMBER_ID': arbiter_member_id,
}
script = test_bash_script_error()
script += build_add_replica_set_members_script()
script = build_context_script(context_dict, script)
output = {}
host = workflow_dict['source_instances'][0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
sleep(10)
host = workflow_dict['host']
cs_host_attr = CsHostAttr.objects.get(host=host)
command = 'umount /data'
output = {}
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
if return_code != 0:
raise Exception(str(output))
if len(workflow_dict['not_primary_hosts']) >= 1:
for host in workflow_dict['not_primary_hosts']:
cs_host_attr = CsHostAttr.objects.get(host=host)
command = 'rm -rf /data/data/*'
output = {}
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0021)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
connect_string = build_mongodb_connect_string(instances=workflow_dict['source_instances'],
databaseinfra=databaseinfra)
context_dict = {
'CONNECT_STRING': connect_string,
}
script = test_bash_script_error()
script += build_switch_primary_to_new_instances_script()
script = build_context_script(context_dict, script)
output = {}
host = workflow_dict['source_instances'][0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
sleep(30)
driver = databaseinfra.get_driver()
client = driver.get_client(instance=None)
rsconf = client['local'].system.replset.find_one()
priority_0 = rsconf['members'][0].get('priority', 1)
priority_1 = rsconf['members'][1].get('priority', 1)
if priority_0 != 0 or priority_1 != 0:
errormsg = "The priority of the old mongodb instances should be zero."
LOG.error(errormsg)
raise Exception(errormsg)
replSetGetStatus = client.admin.command('replSetGetStatus')
if 'PRIMARY' not in (replSetGetStatus['members'][3]['stateStr'],
replSetGetStatus['members'][4]['stateStr']):
errormsg = "One of the new instances should be PRIMARY."
LOG.error(errormsg)
raise Exception(errormsg)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def update_fstab(host, source_export_path, target_export_path):
cs_host_attr = CsHostAttr.objects.get(host=host)
command = """sed -i s/"{}"/"{}"/g /etc/fstab""".format(
source_export_path, target_export_path)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
return return_code, output
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
databaseinfra = workflow_dict['databaseinfra']
for index, source_host in enumerate(workflow_dict['source_hosts']):
target_host = source_host.future_host
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
script = test_bash_script_error()
script += build_start_stop_scripts()
script += build_stop_sentinel_script()
if index < 2:
script += build_stop_database_script()
script = build_context_script({}, script)
output = {}
exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
for source_instance in workflow_dict['source_instances']:
if source_instance.instance_type == source_instance.REDIS_SENTINEL:
reset_sentinel(host=source_instance.hostname,
sentinel_host=source_instance.address,
sentinel_port=source_instance.port,
service_name=databaseinfra.name)
sleep(30)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
instances = workflow_dict['instances']
databaseinfra = workflow_dict['databaseinfra']
connect_string = util.build_mongodb_connect_string(instances=instances,
databaseinfra=databaseinfra)
script = test_bash_script_error()
script += util.build_cp_mongodb_binary_file()
script += util.build_stop_database_script(clean_data=False)
script += util.build_change_release_alias_script()
script += util.build_start_database_script()
script += util.build_authschemaupgrade_script()
script += util.build_change_limits_script()
script += util.build_change_in_serverstatus_file_script()
script += util.build_reinstal_mongo_gen_script()
script += util.build_remove_reprecated_index_counter_metrics()
context_dict = {
'SOURCE_PATH': '/mnt/software/db/mongodb',
'TARGET_PATH': '/usr/local/',
'MONGODB_RELEASE_FILE': 'mongodb-linux-x86_64-3.0.8.tgz',
'MONGODB_RELEASE_FOLDER': 'mongodb-linux-x86_64-3.0.8',
'CONNECT_STRING': connect_string,
}
script = build_context_script(context_dict, script)
output = {}
host = instances[0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0023)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for index, instance in enumerate(workflow_dict['target_instances']):
if instance.instance_type == instance.MONGODB_ARBITER:
continue
if instance.instance_type == instance.REDIS_SENTINEL:
continue
host = instance.hostname
LOG.info("Mounting disks on host {}".format(host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
nfs_host_attr = NFS_HostAttr.objects.get(host=host)
LOG.info("Cheking host ssh...")
host_ready = check_ssh(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
wait=5, interval=10)
if not host_ready:
raise Exception(str("Host %s is not ready..." % host))
context_dict = {
'EXPORTPATH': nfs_host_attr.nfsaas_path,
}
script = test_bash_script_error()
script += build_mount_disk_script()
script = build_context_script(context_dict, script)
LOG.info(script)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
try:
LOG.info("Remove all database files")
for host in workflow_dict['hosts']:
LOG.info("Removing database files on host %s" % host)
host_csattr = CsHostAttr.objects.get(host=host)
exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command="/opt/dbaas/scripts/dbaas_deletedatabasefiles.sh")
return True
except Exception, e:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0013)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def update_fstab(host, source_export_path, target_export_path):
cs_host_attr = CsHostAttr.objects.get(host=host)
command = """sed -i s/"{}"/"{}"/g /etc/fstab""".format(source_export_path,
target_export_path)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
return return_code, output
def do(self, workflow_dict):
try:
flipper_crdentials = get_credentials_for(workflow_dict['source_environment'],
CredentialType.FLIPPER)
flipper_vip = flipper_crdentials.get_parameter_by_name('vip')
for host in workflow_dict['target_hosts']:
cs_host_attr = CS_HostAttr.objects.get(host=host)
source_host = workflow_dict['source_hosts'][0]
nf_host_attr = NF_HostAttr.objects.get(host=source_host)
script = test_bash_script_error()
script += build_mount_snapshot_volume_script()
script += build_remove_deprecated_files_script()
script += build_permission_script()
script += build_start_database_script()
script += build_flipper_script()
context_dict = {
'EXPORT_PATH': nf_host_attr.nfsaas_path,
'SNAPSHOPT_NAME': workflow_dict['snapshot_name'],
'VIP_FLIPPER': flipper_vip,
'IPWRITE': workflow_dict['target_secondary_ips'][0].ip,
'HOST01': workflow_dict['target_hosts'][0],
'HOST02': workflow_dict['target_hosts'][1]
}
script = build_context_script(context_dict, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
driver = databaseinfra.get_driver()
sentinel = driver.get_sentinel_client()
master = sentinel.discover_master(databaseinfra.name)
master_host = master[0]
master_port = master[1]
for index, source_host in enumerate(workflow_dict['source_hosts']):
target_host = source_host.future_host
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
script = test_bash_script_error()
script += build_start_stop_scripts()
if index < 2:
script += build_start_database_script()
script += build_start_sentinel_script()
script += build_start_http_script()
script = build_context_script({}, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
for target_instance in workflow_dict['target_instances']:
if target_instance.instance_type == target_instance.REDIS:
LOG.info('Changing master of host: {}'.format(target_instance.address))
client = driver.get_client(target_instance)
client.slaveof(master_host, master_port)
LOG.info('New master: {}:{}'.format(master_host, master_port))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
