def undo(self, workflow_dict): LOG.info("Running undo...") try: for source_instance in workflow_dict['source_instances']: if source_instance.instance_type == source_instance.REDIS: source_host = source_instance.hostname change_slave_priority_file( host=source_host, original_value=0, final_value=100) change_slave_priority_instance( instance=source_instance, final_value=100) target_instance = source_instance.future_instance target_host = target_instance.hostname change_slave_priority_file( host=target_host, original_value=100, final_value=0) change_slave_priority_instance( instance=target_instance, final_value=0) for source_instance in workflow_dict['source_instances']: if source_instance.instance_type == source_instance.REDIS_SENTINEL: failover_sentinel(host=source_instance.hostname, sentinel_host=source_instance.address, sentinel_port=source_instance.port, service_name=workflow_dict['databaseinfra'].name) break return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): LOG.info("Running undo...") try: for source_instance in workflow_dict['source_instances']: if source_instance.instance_type == source_instance.REDIS: source_host = source_instance.hostname change_slave_priority_file(host=source_host, original_value=0, final_value=100) change_slave_priority_instance(instance=source_instance, final_value=100) target_instance = source_instance.future_instance target_host = target_instance.hostname change_slave_priority_file(host=target_host, original_value=100, final_value=0) change_slave_priority_instance(instance=target_instance, final_value=0) for source_instance in workflow_dict['source_instances']: if source_instance.instance_type == source_instance.REDIS_SENTINEL: failover_sentinel(host=source_instance.hostname, sentinel_host=source_instance.address, sentinel_port=source_instance.port, service_name=workflow_dict['databaseinfra'].name) break return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self): if not self.is_valid: return change_slave_priority_instance(self.target_instance, self.original_value) change_slave_priority_file(self.target_instance.hostname, self.final_value, self.original_value)
def do(self, workflow_dict): try: LOG.info("Getting cloudstack credentials...") statsd_credentials = get_credentials_for( environment=workflow_dict['target_environment'], credential_type=CredentialType.STATSD) statsd_host, statsd_port = statsd_credentials.endpoint.split(':') databaseinfra = workflow_dict['databaseinfra'] sentinel = databaseinfra.get_driver().get_sentinel_client() master = sentinel.discover_master(databaseinfra.name) master_host = master[0] master_port = master[1] for index, source_host in enumerate(workflow_dict['source_hosts']): target_host = source_host.future_host LOG.info(target_host) target_cs_host_attr = CS_HostAttr.objects.get(host=target_host) if index == 2: LOG.info("Cheking host ssh...") host_ready = check_ssh(server=target_host.address, username=target_cs_host_attr.vm_user, password=target_cs_host_attr.vm_password, wait=5, interval=10) if not host_ready: raise Exception( str("Host %s is not ready..." % target_host)) script = test_bash_script_error() script += build_permission_script() script = build_context_script({}, script) output = {} LOG.info(script) return_code = exec_remote_command(server=target_host.address, username=target_cs_host_attr.vm_user, password=target_cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) instances_redis = Instance.objects.filter(hostname=target_host, instance_type=Instance.REDIS) instances_sentinel = Instance.objects.filter(hostname=target_host, instance_type=Instance.REDIS_SENTINEL) if instances_redis: only_sentinel = False instance_redis_address = instances_redis[0].address instance_redis_port = instances_redis[0].port else: only_sentinel = True instance_redis_address = '' instance_redis_port = '' if instances_sentinel: instance_sentinel_address = instances_sentinel[0].address instance_sentinel_port = instances_sentinel[0].port else: instance_sentinel_address = '' instance_sentinel_port = '' contextdict = { 'DATABASENAME': workflow_dict['database'].name, 'DBPASSWORD': databaseinfra.password, 'HOSTADDRESS': instance_redis_address, 'PORT': instance_redis_port, 'ENGINE': 'redis', 'HOST': source_host.hostname.split('.')[0], 'STATSD_HOST': statsd_host, 'STATSD_PORT': statsd_port, 'IS_HA': databaseinfra.plan.is_ha, 'SENTINELMASTER': master_host, 'SENTINELMASTERPORT': master_port, 'SENTINELADDRESS': instance_sentinel_address, 'SENTINELPORT': instance_sentinel_port, 'MASTERNAME': databaseinfra.name, 'ONLY_SENTINEL': only_sentinel, } planattr = PlanAttr.objects.get( plan=workflow_dict['source_plan']) script = build_context_script( contextdict, planattr.configuration_script) output = {} LOG.info(script) return_code = exec_remote_command(server=target_host.address, username=target_cs_host_attr.vm_user, password=target_cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) if index < 2: change_slave_priority_file(host=target_host, original_value=100, final_value=0) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def do(self, workflow_dict): try: LOG.info("Getting cloudstack credentials...") statsd_credentials = get_credentials_for( environment=workflow_dict['target_environment'], credential_type=CredentialType.STATSD) statsd_host, statsd_port = statsd_credentials.endpoint.split(':') databaseinfra = workflow_dict['databaseinfra'] sentinel = databaseinfra.get_driver().get_sentinel_client() master = sentinel.discover_master(databaseinfra.name) master_host = master[0] master_port = master[1] for index, source_host in enumerate(workflow_dict['source_hosts']): target_host = source_host.future_host LOG.info(target_host) target_cs_host_attr = CS_HostAttr.objects.get(host=target_host) if index == 2: LOG.info("Cheking host ssh...") host_ready = check_ssh( server=target_host.address, username=target_cs_host_attr.vm_user, password=target_cs_host_attr.vm_password, wait=5, interval=10) if not host_ready: raise Exception( str("Host %s is not ready..." % target_host)) script = test_bash_script_error() script += build_permission_script() script = build_context_script({}, script) output = {} LOG.info(script) return_code = exec_remote_command( server=target_host.address, username=target_cs_host_attr.vm_user, password=target_cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) instances_redis = Instance.objects.filter( hostname=target_host, instance_type=Instance.REDIS) instances_sentinel = Instance.objects.filter( hostname=target_host, instance_type=Instance.REDIS_SENTINEL) if instances_redis: only_sentinel = False instance_redis_address = instances_redis[0].address instance_redis_port = instances_redis[0].port else: only_sentinel = True instance_redis_address = '' instance_redis_port = '' if instances_sentinel: instance_sentinel_address = instances_sentinel[0].address instance_sentinel_port = instances_sentinel[0].port else: instance_sentinel_address = '' instance_sentinel_port = '' contextdict = { 'DATABASENAME': workflow_dict['database'].name, 'DBPASSWORD': databaseinfra.password, 'HOSTADDRESS': instance_redis_address, 'PORT': instance_redis_port, 'ENGINE': 'redis', 'HOST': source_host.hostname.split('.')[0], 'STATSD_HOST': statsd_host, 'STATSD_PORT': statsd_port, 'IS_HA': databaseinfra.plan.is_ha, 'SENTINELMASTER': master_host, 'SENTINELMASTERPORT': master_port, 'SENTINELADDRESS': instance_sentinel_address, 'SENTINELPORT': instance_sentinel_port, 'MASTERNAME': databaseinfra.name, 'ONLY_SENTINEL': only_sentinel, } planattr = PlanAttr.objects.get( plan=workflow_dict['source_plan']) script = build_context_script(contextdict, planattr.configuration_script) output = {} LOG.info(script) return_code = exec_remote_command( server=target_host.address, username=target_cs_host_attr.vm_user, password=target_cs_host_attr.vm_password, command=script, output=output) LOG.info(output) if return_code != 0: raise Exception(str(output)) if index < 2: change_slave_priority_file(host=target_host, original_value=100, final_value=0) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): LOG.info("Running undo...") try: databaseinfra = workflow_dict['databaseinfra'] driver = databaseinfra.get_driver() for source_instance in workflow_dict['source_instances']: if source_instance.instance_type == source_instance.REDIS: source_host = source_instance.hostname change_slave_priority_file( host=source_host, original_value=0, final_value=100) change_slave_priority_instance( instance=source_instance, final_value=100) target_instance = source_instance.future_instance target_host = target_instance.hostname change_slave_priority_file( host=target_host, original_value=100, final_value=0) change_slave_priority_instance( instance=target_instance, final_value=0) attempts = 1 max_attempts = 10 while True: sentinel_instance = driver.get_non_database_instances()[0] failover_sentinel(host=sentinel_instance.hostname, sentinel_host=sentinel_instance.address, sentinel_port=sentinel_instance.port, service_name=databaseinfra.name) sleep(30) sentinel_client = driver.get_sentinel_client(sentinel_instance) master_address = sentinel_client.discover_master(databaseinfra.name)[0] source_is_master = False for source_instance in workflow_dict['source_instances']: if source_instance.address == master_address: source_is_master = True LOG.info("{} is master".format(source_instance)) break if source_is_master: break if attempts >= max_attempts: raise Exception('It could not switch the master.') attempts += 1 LOG.info("There was something wrong in sentinel failover.") LOG.info("Trying again ( {} of {} )".format(attempts, max_attempts)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False
def undo(self, workflow_dict): LOG.info("Running undo...") try: databaseinfra = workflow_dict['databaseinfra'] driver = databaseinfra.get_driver() for source_instance in workflow_dict['source_instances']: if source_instance.instance_type == source_instance.REDIS: source_host = source_instance.hostname change_slave_priority_file(host=source_host, original_value=0, final_value=100) change_slave_priority_instance(instance=source_instance, final_value=100) target_instance = source_instance.future_instance target_host = target_instance.hostname change_slave_priority_file(host=target_host, original_value=100, final_value=0) change_slave_priority_instance(instance=target_instance, final_value=0) attempts = 1 max_attempts = 10 while True: sentinel_instance = driver.get_non_database_instances()[0] failover_sentinel(host=sentinel_instance.hostname, sentinel_host=sentinel_instance.address, sentinel_port=sentinel_instance.port, service_name=databaseinfra.name) sleep(30) sentinel_client = driver.get_sentinel_client(sentinel_instance) master_address = sentinel_client.discover_master( databaseinfra.name)[0] source_is_master = False for source_instance in workflow_dict['source_instances']: if source_instance.address == master_address: source_is_master = True LOG.info("{} is master".format(source_instance)) break if source_is_master: break if attempts >= max_attempts: raise Exception('It could not switch the master.') attempts += 1 LOG.info("There was something wrong in sentinel failover.") LOG.info("Trying again ( {} of {} )".format( attempts, max_attempts)) return True except Exception: traceback = full_stack() workflow_dict['exceptions']['error_codes'].append(DBAAS_0020) workflow_dict['exceptions']['traceback'].append(traceback) return False