def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
for source_instance in workflow_dict['source_instances']:
if source_instance.instance_type == source_instance.REDIS:
source_host = source_instance.hostname
change_slave_priority_file(
host=source_host, original_value=0, final_value=100)
change_slave_priority_instance(
instance=source_instance, final_value=100)
target_instance = source_instance.future_instance
target_host = target_instance.hostname
change_slave_priority_file(
host=target_host, original_value=100, final_value=0)
change_slave_priority_instance(
instance=target_instance, final_value=0)
for source_instance in workflow_dict['source_instances']:
if source_instance.instance_type == source_instance.REDIS_SENTINEL:
failover_sentinel(host=source_instance.hostname,
sentinel_host=source_instance.address,
sentinel_port=source_instance.port,
service_name=workflow_dict['databaseinfra'].name)
break
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
for source_instance in workflow_dict['source_instances']:
if source_instance.instance_type == source_instance.REDIS:
source_host = source_instance.hostname
change_slave_priority_file(host=source_host, original_value=0, final_value=100)
change_slave_priority_instance(instance=source_instance, final_value=100)
target_instance = source_instance.future_instance
target_host = target_instance.hostname
change_slave_priority_file(host=target_host, original_value=100, final_value=0)
change_slave_priority_instance(instance=target_instance, final_value=0)
for source_instance in workflow_dict['source_instances']:
if source_instance.instance_type == source_instance.REDIS_SENTINEL:
failover_sentinel(host=source_instance.hostname,
sentinel_host=source_instance.address,
sentinel_port=source_instance.port,
service_name=workflow_dict['databaseinfra'].name)
break
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self):
if not self.is_valid:
return
change_slave_priority_instance(self.target_instance,
self.original_value)
change_slave_priority_file(self.target_instance.hostname,
self.final_value, self.original_value)
def do(self, workflow_dict):
try:
LOG.info("Getting cloudstack credentials...")
statsd_credentials = get_credentials_for(
environment=workflow_dict['target_environment'],
credential_type=CredentialType.STATSD)
statsd_host, statsd_port = statsd_credentials.endpoint.split(':')
databaseinfra = workflow_dict['databaseinfra']
sentinel = databaseinfra.get_driver().get_sentinel_client()
master = sentinel.discover_master(databaseinfra.name)
master_host = master[0]
master_port = master[1]
for index, source_host in enumerate(workflow_dict['source_hosts']):
target_host = source_host.future_host
LOG.info(target_host)
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
if index == 2:
LOG.info("Cheking host ssh...")
host_ready = check_ssh(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
wait=5, interval=10)
if not host_ready:
raise Exception(
str("Host %s is not ready..." % target_host))
script = test_bash_script_error()
script += build_permission_script()
script = build_context_script({}, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
instances_redis = Instance.objects.filter(hostname=target_host,
instance_type=Instance.REDIS)
instances_sentinel = Instance.objects.filter(hostname=target_host,
instance_type=Instance.REDIS_SENTINEL)
if instances_redis:
only_sentinel = False
instance_redis_address = instances_redis[0].address
instance_redis_port = instances_redis[0].port
else:
only_sentinel = True
instance_redis_address = ''
instance_redis_port = ''
if instances_sentinel:
instance_sentinel_address = instances_sentinel[0].address
instance_sentinel_port = instances_sentinel[0].port
else:
instance_sentinel_address = ''
instance_sentinel_port = ''
contextdict = {
'DATABASENAME': workflow_dict['database'].name,
'DBPASSWORD': databaseinfra.password,
'HOSTADDRESS': instance_redis_address,
'PORT': instance_redis_port,
'ENGINE': 'redis',
'HOST': source_host.hostname.split('.')[0],
'STATSD_HOST': statsd_host,
'STATSD_PORT': statsd_port,
'IS_HA': databaseinfra.plan.is_ha,
'SENTINELMASTER': master_host,
'SENTINELMASTERPORT': master_port,
'SENTINELADDRESS': instance_sentinel_address,
'SENTINELPORT': instance_sentinel_port,
'MASTERNAME': databaseinfra.name,
'ONLY_SENTINEL': only_sentinel,
}
planattr = PlanAttr.objects.get(
plan=workflow_dict['source_plan'])
script = build_context_script(
contextdict, planattr.configuration_script)
output = {}
LOG.info(script)
return_code = exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
if index < 2:
change_slave_priority_file(host=target_host,
original_value=100,
final_value=0)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
LOG.info("Getting cloudstack credentials...")
statsd_credentials = get_credentials_for(
environment=workflow_dict['target_environment'],
credential_type=CredentialType.STATSD)
statsd_host, statsd_port = statsd_credentials.endpoint.split(':')
databaseinfra = workflow_dict['databaseinfra']
sentinel = databaseinfra.get_driver().get_sentinel_client()
master = sentinel.discover_master(databaseinfra.name)
master_host = master[0]
master_port = master[1]
for index, source_host in enumerate(workflow_dict['source_hosts']):
target_host = source_host.future_host
LOG.info(target_host)
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
if index == 2:
LOG.info("Cheking host ssh...")
host_ready = check_ssh(
server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
wait=5,
interval=10)
if not host_ready:
raise Exception(
str("Host %s is not ready..." % target_host))
script = test_bash_script_error()
script += build_permission_script()
script = build_context_script({}, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(
server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
instances_redis = Instance.objects.filter(
hostname=target_host, instance_type=Instance.REDIS)
instances_sentinel = Instance.objects.filter(
hostname=target_host,
instance_type=Instance.REDIS_SENTINEL)
if instances_redis:
only_sentinel = False
instance_redis_address = instances_redis[0].address
instance_redis_port = instances_redis[0].port
else:
only_sentinel = True
instance_redis_address = ''
instance_redis_port = ''
if instances_sentinel:
instance_sentinel_address = instances_sentinel[0].address
instance_sentinel_port = instances_sentinel[0].port
else:
instance_sentinel_address = ''
instance_sentinel_port = ''
contextdict = {
'DATABASENAME': workflow_dict['database'].name,
'DBPASSWORD': databaseinfra.password,
'HOSTADDRESS': instance_redis_address,
'PORT': instance_redis_port,
'ENGINE': 'redis',
'HOST': source_host.hostname.split('.')[0],
'STATSD_HOST': statsd_host,
'STATSD_PORT': statsd_port,
'IS_HA': databaseinfra.plan.is_ha,
'SENTINELMASTER': master_host,
'SENTINELMASTERPORT': master_port,
'SENTINELADDRESS': instance_sentinel_address,
'SENTINELPORT': instance_sentinel_port,
'MASTERNAME': databaseinfra.name,
'ONLY_SENTINEL': only_sentinel,
}
planattr = PlanAttr.objects.get(
plan=workflow_dict['source_plan'])
script = build_context_script(contextdict,
planattr.configuration_script)
output = {}
LOG.info(script)
return_code = exec_remote_command(
server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
if index < 2:
change_slave_priority_file(host=target_host,
original_value=100,
final_value=0)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
databaseinfra = workflow_dict['databaseinfra']
driver = databaseinfra.get_driver()
for source_instance in workflow_dict['source_instances']:
if source_instance.instance_type == source_instance.REDIS:
source_host = source_instance.hostname
change_slave_priority_file(
host=source_host, original_value=0, final_value=100)
change_slave_priority_instance(
instance=source_instance, final_value=100)
target_instance = source_instance.future_instance
target_host = target_instance.hostname
change_slave_priority_file(
host=target_host, original_value=100, final_value=0)
change_slave_priority_instance(
instance=target_instance, final_value=0)
attempts = 1
max_attempts = 10
while True:
sentinel_instance = driver.get_non_database_instances()[0]
failover_sentinel(host=sentinel_instance.hostname,
sentinel_host=sentinel_instance.address,
sentinel_port=sentinel_instance.port,
service_name=databaseinfra.name)
sleep(30)
sentinel_client = driver.get_sentinel_client(sentinel_instance)
master_address = sentinel_client.discover_master(databaseinfra.name)[0]
source_is_master = False
for source_instance in workflow_dict['source_instances']:
if source_instance.address == master_address:
source_is_master = True
LOG.info("{} is master".format(source_instance))
break
if source_is_master:
break
if attempts >= max_attempts:
raise Exception('It could not switch the master.')
attempts += 1
LOG.info("There was something wrong in sentinel failover.")
LOG.info("Trying again ( {} of {} )".format(attempts, max_attempts))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
databaseinfra = workflow_dict['databaseinfra']
driver = databaseinfra.get_driver()
for source_instance in workflow_dict['source_instances']:
if source_instance.instance_type == source_instance.REDIS:
source_host = source_instance.hostname
change_slave_priority_file(host=source_host,
original_value=0,
final_value=100)
change_slave_priority_instance(instance=source_instance,
final_value=100)
target_instance = source_instance.future_instance
target_host = target_instance.hostname
change_slave_priority_file(host=target_host,
original_value=100,
final_value=0)
change_slave_priority_instance(instance=target_instance,
final_value=0)
attempts = 1
max_attempts = 10
while True:
sentinel_instance = driver.get_non_database_instances()[0]
failover_sentinel(host=sentinel_instance.hostname,
sentinel_host=sentinel_instance.address,
sentinel_port=sentinel_instance.port,
service_name=databaseinfra.name)
sleep(30)
sentinel_client = driver.get_sentinel_client(sentinel_instance)
master_address = sentinel_client.discover_master(
databaseinfra.name)[0]
source_is_master = False
for source_instance in workflow_dict['source_instances']:
if source_instance.address == master_address:
source_is_master = True
LOG.info("{} is master".format(source_instance))
break
if source_is_master:
break
if attempts >= max_attempts:
raise Exception('It could not switch the master.')
attempts += 1
LOG.info("There was something wrong in sentinel failover.")
LOG.info("Trying again ( {} of {} )".format(
attempts, max_attempts))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
