def undo(self, workflow_dict): try: LOG.info("Remove all backup log files") backup_log_dict = get_backup_log_configuration_dict(environment=workflow_dict['environment'], databaseinfra=workflow_dict['databaseinfra']) if backup_log_dict is None: return True instance = workflow_dict['instances'][0] host = instance.hostname host_csattr = CsHostAttr.objects.get(host=host) exec_remote_command(server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, command=backup_log_dict['CLEAN_BACKUP_LOG_SCRIPT']) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0014) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): LOG.info("Running undo...") try: script = build_clean_database_dir_script() script = build_context_script({}, script) for source_host in workflow_dict['source_hosts']: target_host = source_host.future_host target_cs_host_attr = CS_HostAttr.objects.get(host=target_host) output = {} exec_remote_command(server=target_host.address, username=target_cs_host_attr.vm_user, password=target_cs_host_attr.vm_password, command=script, output=output) LOG.info(output) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def mysql_binlog_save(client, instance, cloudstack_hostattr): try: client.query('show master status') r = client.store_result() row = r.fetch_row(maxrows=0, how=1) binlog_file = row[0]['File'] binlog_pos = row[0]['Position'] client.query("show variables like 'datadir'") r = client.store_result() row = r.fetch_row(maxrows=0, how=1) datadir = row[0]['Value'] output = {} command = 'echo "master=%s;position=%s" > %smysql_binlog_master_file_pos' % ( binlog_file, binlog_pos, datadir) exec_remote_command(server=instance.hostname.address, username=cloudstack_hostattr.vm_user, password=cloudstack_hostattr.vm_password, command=command, output=output) except Exception as e: LOG.error( "Error saving mysql master binlog file and position: %s" % (e))
def do(self, workflow_dict): try: for host in workflow_dict['source_hosts']: LOG.info("Removing database files on host %s" % host) host_csattr = CsHostAttr.objects.get(host=host) output = {} exec_remote_command( server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, command="/opt/dbaas/scripts/dbaas_deletedatabasefiles.sh", output=output) LOG.info(output) LOG.info("Removing disks on host %s" % host) NfsaasProvider().destroy_disk( environment=workflow_dict['source_environment'], host=host) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: for host in workflow_dict['source_hosts']: LOG.info("Removing database files on host %s" % host) host_csattr = CsHostAttr.objects.get(host=host) output = {} exec_remote_command(server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, command="/opt/dbaas/scripts/dbaas_deletedatabasefiles.sh", output=output) LOG.info(output) LOG.info("Removing disks on host %s" % host) NfsaasProvider().destroy_disk(environment=workflow_dict['source_environment'], host=host) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): LOG.info("Running undo...") try: script = build_stop_database_script() script = build_context_script({}, script) for target_instance in workflow_dict['target_instances']: target_host = target_instance.hostname target_cs_host_attr = CS_HostAttr.objects.get(host=target_host) output = {} exec_remote_command(server=target_host.address, username=target_cs_host_attr.vm_user, password=target_cs_host_attr.vm_password, command=script, output=output) LOG.info(output) try: if 'region_migration_dir_infra_name' in workflow_dict: shutil.rmtree(workflow_dict['region_migration_dir_infra_name']) except Exception: pass return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): try: LOG.info("Remove all backup log files") backup_log_dict = get_backup_log_configuration_dict( environment=workflow_dict['target_environment'], databaseinfra=workflow_dict['databaseinfra']) if backup_log_dict is None: return True instance = workflow_dict['target_instances'][0] host = instance.hostname host_csattr = CsHostAttr.objects.get(host=host) exec_remote_command( server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, command=backup_log_dict['CLEAN_BACKUP_LOG_SCRIPT']) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0014) workflow_dict['exceptions']['traceback'].append(traceback) return False
def clean_unused_data(export_id, export_path, host, databaseinfra): provider = NfsaasProvider() provider.grant_access(environment=databaseinfra.environment, host=host, export_id=export_id) mount_path = "/mnt_{}_{}".format(databaseinfra.name, export_id) command = "mkdir -p {}".format(mount_path) command += "\nmount -t nfs -o bg,intr {} {}".format( export_path, mount_path) command += "\nrm -rf {}/*".format(mount_path) command += "\numount {}".format(mount_path) command += "\nrm -rf {}".format(mount_path) LOG.info(command) cs_host_attr = CsHostAttr.objects.get(host=host) output = {} exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=command, output=output) LOG.info(output)
def get_puppet_code_status(self, host, cloudstack): output = {} LOG.info("Puppet-setup LOG info:") exec_remote_command(server=host.address, username=cloudstack.vm_user, password=cloudstack.vm_password, command="tail -7 /var/log/ks-post.log", output=output) for line in output["stdout"]: if "puppet-setup" in line and "return code:" in line: return int(line.split("return code: ")[1]), output return 0, output
def run_vm_script(workflow_dict, context_dict, script): try: final_context_dict = dict(context_dict.items()) instance = workflow_dict['instance'] host = workflow_dict['host'] host_csattr = HostAttr.objects.get(host=host) final_context_dict['HOSTADDRESS'] = instance.address final_context_dict['PORT'] = instance.port final_context_dict['DBPASSWORD'] = workflow_dict['databaseinfra'].password command = build_context_script(final_context_dict, script) output = {} return_code = exec_remote_command(server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, command=command, output=output) if return_code: raise Exception("Could not run script. Output: {}".format(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0015) workflow_dict['exceptions']['traceback'].append(traceback) return False
def run_vm_script(workflow_dict, context_dict, script): try: final_context_dict = dict(context_dict.items()) instance = workflow_dict['instance'] host = workflow_dict['host'] host_csattr = HostAttr.objects.get(host=host) final_context_dict['HOSTADDRESS'] = instance.address final_context_dict['PORT'] = instance.port final_context_dict['DBPASSWORD'] = workflow_dict[ 'databaseinfra'].password command = build_context_script(final_context_dict, script) output = {} return_code = exec_remote_command(server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, command=command, output=output) if return_code: raise Exception("Could not run script. Output: {}".format(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0015) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: sleep(60) databaseinfra = workflow_dict['databaseinfra'] driver = databaseinfra.get_driver() files_to_remove = driver.remove_deprectaed_files() command = files_to_remove + " && cp -rp /data/* /data2" host = workflow_dict['host'] cs_host_attr = CsHostAttr.objects.get(host=host) output = {} return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=command, output=output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0022) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: command = "mkdir /data2 && mount -t nfs -o bg,intr {} /data2" host = workflow_dict['host'] volume = workflow_dict['volume'] command = command.format(volume.nfsaas_path) cs_host_attr = CsHostAttr.objects.get(host=host) output = {} return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=command, output=output) if return_code != 0: raise Exception(str(output)) workflow_dict['mount'] = command return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0022) workflow_dict['exceptions']['traceback'].append(traceback) return False
def run_vm_script(workflow_dict, context_dict, script): try: instances_detail = workflow_dict['instances_detail'] final_context_dict = dict( context_dict.items() + workflow_dict['initial_context_dict'].items()) for instance_detail in instances_detail: host = instance_detail['instance'].hostname host_csattr = HostAttr.objects.get(host=host) final_context_dict['IS_MASTER'] = instance_detail['is_master'] command = build_context_script(final_context_dict, script) output = {} return_code = exec_remote_command(server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, command=command, output=output) if return_code: raise Exception, "Could not run script. Output: {}".format( output) return True except Exception, e: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0015) workflow_dict['exceptions']['traceback'].append(traceback) return False
def run_vm_script(workflow_dict, context_dict, script): try: instances_detail = workflow_dict['instances_detail'] final_context_dict = dict(context_dict.items() + workflow_dict['initial_context_dict'].items()) for instance_detail in instances_detail: instance = instance_detail['instance'] host = instance.hostname host_csattr = HostAttr.objects.get(host=host) final_context_dict['HOSTADDRESS'] = instance.address final_context_dict['PORT'] = instance.port command = build_context_script(final_context_dict, script) output = {} return_code = exec_remote_command(server = host.address, username = host_csattr.vm_user, password = host_csattr.vm_password, command = command, output = output) if return_code: raise Exception, "Could not run script. Output: {}".format(output) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0015) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: host = workflow_dict['source_hosts'][0] databaseinfra = workflow_dict['databaseinfra'] cs_host_attr = CS_HostAttr.objects.get(host=host) context_dict = { 'MASTERPAIRNAME': databaseinfra.name, } script = test_bash_script_error() script += build_turn_flipper_ip_down_script() script = build_context_script(context_dict, script) output = {} return_code = exec_remote_command( server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): LOG.info("Running undo...") try: command = 'mount /data' for host_and_export in workflow_dict['hosts_and_exports']: host = host_and_export['host'] LOG.info('mount data volume on host {}'.format(host)) cs_host_attr = CsHostAttr.objects.get(host=host) output = {} return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=command, output=output) if return_code != 0: LOG.info(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0021) workflow_dict['exceptions']['traceback'].append(traceback) return False
def change_instance_binaries(self, instance, connect_string, run_authschemaupgrade): script = test_bash_script_error() script += util.build_cp_mongodb_binary_file() script += util.build_stop_database_script(clean_data=False) script += util.build_change_release_alias_script() script += util.build_start_database_script(wait_time=30) script += util.build_change_limits_script() script += util.build_remove_reprecated_index_counter_metrics() if run_authschemaupgrade: script += util.build_authschemaupgrade_script() context_dict = { 'SOURCE_PATH': '/mnt/software/db/mongodb', 'TARGET_PATH': '/usr/local/', 'MONGODB_RELEASE_FILE': 'mongodb-linux-x86_64-3.0.8.tgz', 'MONGODB_RELEASE_FOLDER': 'mongodb-linux-x86_64-3.0.8', 'CONNECT_STRING': connect_string, } script = build_context_script(context_dict, script) output = {} host = instance.hostname cs_host_attr = CS_HostAttr.objects.get(host=host) return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output))
def run_vm_script(workflow_dict, context_dict, script): try: instances_detail = workflow_dict["instances_detail"] final_context_dict = dict(context_dict.items() + workflow_dict["initial_context_dict"].items()) for instance_detail in instances_detail: host = instance_detail["instance"].hostname host_csattr = HostAttr.objects.get(host=host) final_context_dict["IS_MASTER"] = instance_detail["is_master"] command = build_context_script(final_context_dict, script) output = {} return_code = exec_remote_command( server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, command=command, output=output, ) if return_code: raise Exception, "Could not run script. Output: {}".format(output) return True except Exception: traceback = full_stack() workflow_dict["exceptions"]["error_codes"].append(DBAAS_0015) workflow_dict["exceptions"]["traceback"].append(traceback) return False
def do(self, workflow_dict): try: sleep(60) databaseinfra = workflow_dict['databaseinfra'] driver = databaseinfra.get_driver() files_to_remove = driver.remove_deprectaed_files() command = files_to_remove + " && cp -rp /data/* /data2" host = workflow_dict['host'] cs_host_attr = CsHostAttr.objects.get(host=host) output = {} return_code = exec_remote_command( server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=command, output=output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0022) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): LOG.info("Running undo...") try: databaseinfra = workflow_dict['databaseinfra'] host = workflow_dict['host'] return_code, output = use_database_initialization_script( databaseinfra=databaseinfra, host=host, option='stop') if return_code != 0: LOG.info(str(output)) cs_host_attr = CsHostAttr.objects.get(host=host) output = {} return_code = exec_remote_command( server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command='umount /data', output=output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0022) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): LOG.info("Running undo...") try: databaseinfra = workflow_dict['databaseinfra'] host = workflow_dict['host'] return_code, output = use_database_initialization_script(databaseinfra=databaseinfra, host=host, option='stop') if return_code != 0: LOG.info(str(output)) cs_host_attr = CsHostAttr.objects.get(host=host) output = {} return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command='umount /data', output=output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0022) workflow_dict['exceptions']['traceback'].append(traceback) return False
def host_mount_data_percentage(address, task): host = Host.objects.filter(address=address).first() vm = host.cs_host_attributes.first() output_message = {} command_status = exec_remote_command(server=host.address, username=vm.vm_user, password=vm.vm_password, command='df -hk | grep /data', output=output_message) if command_status != 0: task.add_detail( message='Could not load mount size: {}'.format(output_message), level=4) return None, None, None values = output_message['stdout'][0].strip().split() values = { 'total': int(values[0]), 'used': int(values[1]), 'free': int(values[2]), 'percentage': int(values[3].replace('%', '')) } task.add_detail(message='Mount /data: {}% ({}kb/{}kb)'.format( values['percentage'], values['used'], values['total']), level=3) return values['percentage'], values['used'], values['total']
def change_instance_binaries(self, instance): script = test_bash_script_error() script += build_change_mongodb_conf_file_script() script += build_stop_database_script(clean_data=False) script += build_clean_data_data_script() script += build_start_database_script() script += build_wait_admin_be_created_script() script += td_agent_script(option='restart') context_dict = {} script = build_context_script(context_dict, script) output = {} host = instance.hostname cs_host_attr = CS_HostAttr.objects.get(host=host) return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output))
def do(self, workflow_dict): try: option = 'start' for host in workflow_dict['hosts']: LOG.info("{} monit on host {}".format(option, host)) cs_host_attr = CS_HostAttr.objects.get(host=host) script = test_bash_script_error() script += monit_script(option) LOG.info(script) output = {} return_code = exec_remote_command( server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: LOG.error("Error monit") LOG.error(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0002) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: for source_host in workflow_dict['source_hosts']: future_host = source_host.future_host hosts_option = [(future_host, 'stop')] for host, option in hosts_option: LOG.info("{} td_agent on host {}".format(option, host)) cs_host_attr = CS_HostAttr.objects.get(host=host) script = test_bash_script_error() script += monit_script(option) script += td_agent_script(option) LOG.info(script) output = {} return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: LOG.error("Error stopping td_agent") LOG.error(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): LOG.info("Running undo...") try: command = 'rm -rf /data2 && {} && mount /data' host = workflow_dict['host'] cs_host_attr = CsHostAttr.objects.get(host=host) mount = workflow_dict['mount'] command = command.format(mount) output = {} return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=command, output=output) if return_code != 0: LOG.info(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0021) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): LOG.info("Running undo...") try: for source_host in workflow_dict['source_hosts']: future_host = source_host.future_host cs_host_attr = CS_HostAttr.objects.get(host=future_host) LOG.info("Removing rsyslog config in {}".format(future_host)) script = self.rsyslog_remove_config() LOG.info(script) output = {} return_code = exec_remote_command( server=future_host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: error_msg = "Error removing configuring rsyslog: {}".format( str(output)) LOG.error(error_msg) raise EnvironmentError(error_msg) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def switch_master(self): master = self.get_master_instance() slave = self.get_slave_instances()[0] host = master.hostname host_attr = HostAttr.objects.get(host=host) script = """ sudo -u flipper /usr/bin/flipper {{MASTERPAIRNAME}} set write {{HOST01.address}} sudo -u flipper /usr/bin/flipper {{MASTERPAIRNAME}} set read {{HOST02.address}} """ context_dict = { 'MASTERPAIRNAME': self.databaseinfra.name, 'HOST01': slave.hostname, 'HOST02': master.hostname, } script = build_context_script(context_dict, script) output = {} return_code = exec_remote_command(server=host.address, username=host_attr.vm_user, password=host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output))
def do(self, workflow_dict): try: for source_host in workflow_dict['source_hosts']: host = source_host.future_host LOG.info("Starting td_agent on host {}".format(host)) cs_host_attr = CS_HostAttr.objects.get(host=host) context_dict = {} script = test_bash_script_error() script += build_start_td_agent_script() script = build_context_script(context_dict, script) LOG.info(script) output = {} return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: LOG.error("Error starting td_agent") LOG.error(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: option = 'restart' for host in workflow_dict['hosts']: LOG.info("{} monit on host {}".format(option, host)) cs_host_attr = CS_HostAttr.objects.get(host=host) script = test_bash_script_error() script += td_agent_script(option) LOG.info(script) output = {} sleep(30) return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: LOG.error("Error monit") LOG.error(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0002) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: databaseinfra = workflow_dict['databaseinfra'] driver = databaseinfra.get_driver() host = workflow_dict['host'] host_attr = HostAttr.objects.get(host=host) sleep(30) for agent in driver.get_database_agents(): script = '/etc/init.d/{} start'.format(agent) output = {} return_code = exec_remote_command(server=host.address, username=host_attr.vm_user, password=host_attr.vm_password, command=script, output=output) LOG.info('Running {} - Return Code: {}. Output scrit: {}'.format( script, return_code, output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0022) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: for host in workflow_dict['target_hosts']: cs_host_attr = CS_HostAttr.objects.get(host=host) script = test_bash_script_error() script += build_uncomment_skip_slave_script() script = build_context_script({}, script) output = {} LOG.info(script) return_code = exec_remote_command( server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: databaseinfra = workflow_dict['databaseinfra'] driver = databaseinfra.get_driver() host = workflow_dict['host'] host_attr = HostAttr.objects.get(host=host) sleep(30) for agent in driver.get_database_agents(): script = '/etc/init.d/{} start'.format(agent) output = {} return_code = exec_remote_command( server=host.address, username=host_attr.vm_user, password=host_attr.vm_password, command=script, output=output) LOG.info( 'Running {} - Return Code: {}. Output scrit: {}'.format( script, return_code, output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0022) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: script = "puppet-setup" for host in workflow_dict['target_hosts']: LOG.info("Getting vm credentials...") host_csattr = CsHostAttr.objects.get(host=host) host.update_os_description() LOG.info("Run puppet-setup on host {}".format(host)) output = {} return_code = exec_remote_command(server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, command=script, output=output) #if return_code != 0: # raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0013) workflow_dict['exceptions']['traceback'].append(traceback) return False
def switch_master(self, driver): master = driver.get_master_instance() slave = driver.get_slave_instances()[0] host = master.hostname host_attr = HostAttr.objects.get(host=host) script = """ sudo -u flipper /usr/bin/flipper {{MASTERPAIRNAME}} set write {{HOST01.address}} sudo -u flipper /usr/bin/flipper {{MASTERPAIRNAME}} set read {{HOST02.address}} """ context_dict = { 'MASTERPAIRNAME': driver.databaseinfra.name, 'HOST01': slave.hostname, 'HOST02': master.hostname, } script = build_context_script(context_dict, script) output = {} return_code = exec_remote_command(server=host.address, username=host_attr.vm_user, password=host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output))
def run_vm_script(workflow_dict, context_dict, script, reverse=False, wait=0): try: instances_detail = workflow_dict['instances_detail'] final_context_dict = dict(context_dict.items() + workflow_dict['initial_context_dict'].items()) if reverse: instances_detail_final = instances_detail[::-1] else: instances_detail_final = instances_detail for instance_detail in instances_detail_final: host = instance_detail['instance'].hostname host_csattr = HostAttr.objects.get(host=host) command = build_context_script(final_context_dict, script) output = {} return_code = exec_remote_command(server = host.address, username = host_csattr.vm_user, password = host_csattr.vm_password, command = command, output = output) if return_code: raise Exception, "Could not run script. Output: {}".format(output) sleep(wait) return True except Exception, e: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0015) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): LOG.info("Running undo...") try: command = 'umount /data2 && rmdir /data2' host = workflow_dict['host'] cs_host_attr = CsHostAttr.objects.get(host=host) output = {} return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=command, output=output) if return_code != 0: LOG.info(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0022) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: for source_host in workflow_dict['source_hosts']: future_host = source_host.future_host hosts_option = [(source_host, 'stop'), (future_host, 'start')] for host, option in hosts_option: LOG.info("Starting td_agent on host {}".format(host)) cs_host_attr = CS_HostAttr.objects.get(host=host) context_dict = {} script = test_bash_script_error() script += build_mk_heartbeat_daemon_script(option) script = build_context_script(context_dict, script) LOG.info(script) output = {} return_code = exec_remote_command( server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: LOG.error("Error starting mysql_statsd") LOG.error(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: script = "ps -ef | grep bootstrap-puppet3-loop.sh | grep -v grep | wc -l" for host in workflow_dict['target_hosts']: LOG.info("Getting vm credentials...") host_csattr = CsHostAttr.objects.get(host=host) attempt = 1 retries = 60 interval = 20 sleep(interval) while True: LOG.info( "Check if puppet-setup is runnig on {} - attempt {} of {}" .format(host, attempt, retries)) output = {} return_code = exec_remote_command( server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, command=script, output=output) if return_code != 0: raise Exception(str(output)) ret_value = int(output['stdout'][0]) if ret_value == 0: LOG.info( "Puppet-setup is not runnig on {}".format(host)) break LOG.info("Puppet-setup is runnig on {}".format(host)) attempt += 1 if attempt == retries: error = "Maximum number of attempts check is puppet is running on {}.".format( host) LOG.error(error) raise Exception(error) sleep(interval) puppet_code_status, output = self.get_puppet_code_status( host, host_csattr) if puppet_code_status != 0: message = "Puppet-setup returned an error on {}. Output: {}".format( host, output) raise EnvironmentError(message) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0013) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: for index, instance in enumerate( workflow_dict['target_instances']): if instance.instance_type == instance.MONGODB_ARBITER: continue if instance.instance_type == instance.REDIS_SENTINEL: continue host = instance.hostname LOG.info("Mounting disks on host {}".format(host)) cs_host_attr = CS_HostAttr.objects.get(host=host) nfs_host_attr = NFS_HostAttr.objects.get(host=host) LOG.info("Cheking host ssh...") host_ready = check_ssh(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, wait=5, interval=10) if not host_ready: raise Exception(str("Host %s is not ready..." % host)) context_dict = { 'EXPORTPATH': nfs_host_attr.nfsaas_path, } script = test_bash_script_error() script += build_mount_disk_script() script = build_context_script(context_dict, script) LOG.info(script) output = {} return_code = exec_remote_command( server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: databaseinfra = workflow_dict['databaseinfra'] connect_string = build_mongodb_connect_string(instances=workflow_dict['source_instances'], databaseinfra=databaseinfra) client = databaseinfra.get_driver().get_client(None) rsconf = client.local.system.replset.find_one() member_ids = [] for member in rsconf['members']: member_ids.append(member['_id']) max_member_id = max(member_ids) secundary_one_member_id = max_member_id + 1 secundary_two_member_id = max_member_id + 2 arbiter_member_id = max_member_id + 3 context_dict = { 'CONNECT_STRING': connect_string, 'SECUNDARY_ONE': "{}:{}".format(workflow_dict['target_instances'][0].address, workflow_dict['target_instances'][0].port), 'SECUNDARY_TWO': "{}:{}".format(workflow_dict['target_instances'][1].address, workflow_dict['target_instances'][1].port), 'ARBITER': "{}:{}".format(workflow_dict['target_instances'][2].address, workflow_dict['target_instances'][2].port), 'SECUNDARY_ONE_MEMBER_ID': secundary_one_member_id, 'SECUNDARY_TWO_MEMBER_ID': secundary_two_member_id, 'ARBITER_MEMBER_ID': arbiter_member_id, } script = test_bash_script_error() script += build_add_replica_set_members_script() script = build_context_script(context_dict, script) output = {} host = workflow_dict['source_instances'][0].hostname cs_host_attr = CS_HostAttr.objects.get(host=host) return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: sleep(10) host = workflow_dict['host'] cs_host_attr = CsHostAttr.objects.get(host=host) command = 'umount /data' output = {} return_code = exec_remote_command( server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=command, output=output) if return_code != 0: raise Exception(str(output)) if len(workflow_dict['not_primary_hosts']) >= 1: for host in workflow_dict['not_primary_hosts']: cs_host_attr = CsHostAttr.objects.get(host=host) command = 'rm -rf /data/data/*' output = {} return_code = exec_remote_command( server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=command, output=output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0021) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: databaseinfra = workflow_dict['databaseinfra'] connect_string = build_mongodb_connect_string(instances=workflow_dict['source_instances'], databaseinfra=databaseinfra) context_dict = { 'CONNECT_STRING': connect_string, } script = test_bash_script_error() script += build_switch_primary_to_new_instances_script() script = build_context_script(context_dict, script) output = {} host = workflow_dict['source_instances'][0].hostname cs_host_attr = CS_HostAttr.objects.get(host=host) return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) sleep(30) driver = databaseinfra.get_driver() client = driver.get_client(instance=None) rsconf = client['local'].system.replset.find_one() priority_0 = rsconf['members'][0].get('priority', 1) priority_1 = rsconf['members'][1].get('priority', 1) if priority_0 != 0 or priority_1 != 0: errormsg = "The priority of the old mongodb instances should be zero." LOG.error(errormsg) raise Exception(errormsg) replSetGetStatus = client.admin.command('replSetGetStatus') if 'PRIMARY' not in (replSetGetStatus['members'][3]['stateStr'], replSetGetStatus['members'][4]['stateStr']): errormsg = "One of the new instances should be PRIMARY." LOG.error(errormsg) raise Exception(errormsg) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def update_fstab(host, source_export_path, target_export_path): cs_host_attr = CsHostAttr.objects.get(host=host) command = """sed -i s/"{}"/"{}"/g /etc/fstab""".format( source_export_path, target_export_path) output = {} return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=command, output=output) return return_code, output
def undo(self, workflow_dict): LOG.info("Running undo...") try: databaseinfra = workflow_dict['databaseinfra'] for index, source_host in enumerate(workflow_dict['source_hosts']): target_host = source_host.future_host target_cs_host_attr = CS_HostAttr.objects.get(host=target_host) script = test_bash_script_error() script += build_start_stop_scripts() script += build_stop_sentinel_script() if index < 2: script += build_stop_database_script() script = build_context_script({}, script) output = {} exec_remote_command(server=target_host.address, username=target_cs_host_attr.vm_user, password=target_cs_host_attr.vm_password, command=script, output=output) LOG.info(output) for source_instance in workflow_dict['source_instances']: if source_instance.instance_type == source_instance.REDIS_SENTINEL: reset_sentinel(host=source_instance.hostname, sentinel_host=source_instance.address, sentinel_port=source_instance.port, service_name=databaseinfra.name) sleep(30) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: instances = workflow_dict['instances'] databaseinfra = workflow_dict['databaseinfra'] connect_string = util.build_mongodb_connect_string(instances=instances, databaseinfra=databaseinfra) script = test_bash_script_error() script += util.build_cp_mongodb_binary_file() script += util.build_stop_database_script(clean_data=False) script += util.build_change_release_alias_script() script += util.build_start_database_script() script += util.build_authschemaupgrade_script() script += util.build_change_limits_script() script += util.build_change_in_serverstatus_file_script() script += util.build_reinstal_mongo_gen_script() script += util.build_remove_reprecated_index_counter_metrics() context_dict = { 'SOURCE_PATH': '/mnt/software/db/mongodb', 'TARGET_PATH': '/usr/local/', 'MONGODB_RELEASE_FILE': 'mongodb-linux-x86_64-3.0.8.tgz', 'MONGODB_RELEASE_FOLDER': 'mongodb-linux-x86_64-3.0.8', 'CONNECT_STRING': connect_string, } script = build_context_script(context_dict, script) output = {} host = instances[0].hostname cs_host_attr = CS_HostAttr.objects.get(host=host) return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0023) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: for index, instance in enumerate(workflow_dict['target_instances']): if instance.instance_type == instance.MONGODB_ARBITER: continue if instance.instance_type == instance.REDIS_SENTINEL: continue host = instance.hostname LOG.info("Mounting disks on host {}".format(host)) cs_host_attr = CS_HostAttr.objects.get(host=host) nfs_host_attr = NFS_HostAttr.objects.get(host=host) LOG.info("Cheking host ssh...") host_ready = check_ssh(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, wait=5, interval=10) if not host_ready: raise Exception(str("Host %s is not ready..." % host)) context_dict = { 'EXPORTPATH': nfs_host_attr.nfsaas_path, } script = test_bash_script_error() script += build_mount_disk_script() script = build_context_script(context_dict, script) LOG.info(script) output = {} return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): try: LOG.info("Remove all database files") for host in workflow_dict['hosts']: LOG.info("Removing database files on host %s" % host) host_csattr = CsHostAttr.objects.get(host=host) exec_remote_command(server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, command="/opt/dbaas/scripts/dbaas_deletedatabasefiles.sh") return True except Exception, e: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0013) workflow_dict['exceptions']['traceback'].append(traceback) return False
def update_fstab(host, source_export_path, target_export_path): cs_host_attr = CsHostAttr.objects.get(host=host) command = """sed -i s/"{}"/"{}"/g /etc/fstab""".format(source_export_path, target_export_path) output = {} return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=command, output=output) return return_code, output
def do(self, workflow_dict): try: flipper_crdentials = get_credentials_for(workflow_dict['source_environment'], CredentialType.FLIPPER) flipper_vip = flipper_crdentials.get_parameter_by_name('vip') for host in workflow_dict['target_hosts']: cs_host_attr = CS_HostAttr.objects.get(host=host) source_host = workflow_dict['source_hosts'][0] nf_host_attr = NF_HostAttr.objects.get(host=source_host) script = test_bash_script_error() script += build_mount_snapshot_volume_script() script += build_remove_deprecated_files_script() script += build_permission_script() script += build_start_database_script() script += build_flipper_script() context_dict = { 'EXPORT_PATH': nf_host_attr.nfsaas_path, 'SNAPSHOPT_NAME': workflow_dict['snapshot_name'], 'VIP_FLIPPER': flipper_vip, 'IPWRITE': workflow_dict['target_secondary_ips'][0].ip, 'HOST01': workflow_dict['target_hosts'][0], 'HOST02': workflow_dict['target_hosts'][1] } script = build_context_script(context_dict, script) output = {} LOG.info(script) return_code = exec_remote_command(server=host.address, username=cs_host_attr.vm_user, password=cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: databaseinfra = workflow_dict['databaseinfra'] driver = databaseinfra.get_driver() sentinel = driver.get_sentinel_client() master = sentinel.discover_master(databaseinfra.name) master_host = master[0] master_port = master[1] for index, source_host in enumerate(workflow_dict['source_hosts']): target_host = source_host.future_host target_cs_host_attr = CS_HostAttr.objects.get(host=target_host) script = test_bash_script_error() script += build_start_stop_scripts() if index < 2: script += build_start_database_script() script += build_start_sentinel_script() script += build_start_http_script() script = build_context_script({}, script) output = {} LOG.info(script) return_code = exec_remote_command(server=target_host.address, username=target_cs_host_attr.vm_user, password=target_cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) for target_instance in workflow_dict['target_instances']: if target_instance.instance_type == target_instance.REDIS: LOG.info('Changing master of host: {}'.format(target_instance.address)) client = driver.get_client(target_instance) client.slaveof(master_host, master_port) LOG.info('New master: {}:{}'.format(master_host, master_port)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False