def handle(self, *args, **kwargs):
self.task.arguments = {'args': args, 'kwargs': kwargs}
if not kwargs['celery_hosts']:
raise CommandError("Please specified the --celery_hosts count")
try:
tasks_with_problem = self.check_tasks(kwargs['celery_hosts'])
except CeleryActivesNodeError as celery_error:
self.task.update_status_for(
TaskHistory.STATUS_WARNING,
'Could not check celery tasks.\n{}{}'.format(
full_stack(), celery_error
)
)
return
except Exception as e:
self.task.update_status_for(
TaskHistory.STATUS_ERROR,
'Could not execute task.\n{}{}'.format(full_stack(), e)
)
return
problems = len(tasks_with_problem)
status = TaskHistory.STATUS_SUCCESS
if problems > 0:
status = TaskHistory.STATUS_WARNING
self.task.update_status_for(status, 'Problems: {}'.format(problems))
self.check_unique_keys()
def do(self, workflow_dict):
try:
if workflow_dict['qt'] == 1:
return True
flipper = FlipperProvider()
LOG.info("Creating Flipper...")
flipper.create_flipper_dependencies(
masterpairname=workflow_dict['names']['infra'],
hostname1=workflow_dict[
'hosts'][0].address,
writeip=workflow_dict[
'databaseinfraattr'][0].ip,
readip=workflow_dict[
'databaseinfraattr'][1].ip,
hostname2=workflow_dict[
'hosts'][1].address,
environment=workflow_dict['environment'])
return True
except Exception, e:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0008)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
workflow_dict['objects_changed'] = []
switch_dns_forward(databaseinfra=databaseinfra,
source_object_list=workflow_dict['source_hosts'],
ip_attribute_name='address',
dns_attribute_name='hostname',
equivalent_atribute_name='future_host',
workflow_dict=workflow_dict)
switch_dns_forward(databaseinfra=databaseinfra,
source_object_list=workflow_dict['source_instances'],
ip_attribute_name='address',
dns_attribute_name='dns',
equivalent_atribute_name='future_instance',
workflow_dict=workflow_dict)
switch_dns_forward(databaseinfra=databaseinfra,
source_object_list=workflow_dict['source_secondary_ips'],
ip_attribute_name='ip',
dns_attribute_name='dns',
equivalent_atribute_name='equivalent_dbinfraattr',
workflow_dict=workflow_dict)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
workflow_dict['names'] = gen_infra_names(
name=workflow_dict['name'], qt=workflow_dict['qt'])
databaseinfra = DatabaseInfra()
databaseinfra.name = workflow_dict['names']['infra']
databaseinfra.user = ''
databaseinfra.password = make_db_random_password()
databaseinfra.engine = workflow_dict[
'plan'].engine
databaseinfra.plan = workflow_dict['plan']
databaseinfra.environment = workflow_dict['environment']
databaseinfra.capacity = 1
databaseinfra.per_database_size_mbytes = workflow_dict[
'plan'].max_db_size
databaseinfra.save()
LOG.info("DatabaseInfra created!")
workflow_dict['databaseinfra'] = databaseinfra
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0002)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
sleep(60)
databaseinfra = workflow_dict['databaseinfra']
driver = databaseinfra.get_driver()
files_to_remove = driver.remove_deprectaed_files()
command = files_to_remove + " && cp -rp /data/* /data2"
host = workflow_dict['host']
cs_host_attr = CsHostAttr.objects.get(host=host)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0022)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
database = workflow_dict['database']
instances_detail = []
for instance in database.databaseinfra.instances.filter(instance_type=Instance.REDIS):
instances_detail.append({
'instance': instance,
#'is_master': is_master,
'offering_changed': False,
})
workflow_dict['instances_detail'] = instances_detail
context_dict = {
'IS_HA': False,
'DBPASSWORD': database.databaseinfra.password,
'DATABASENAME': database.name,
}
workflow_dict['initial_context_dict'] = context_dict
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
workflow_dict['disks'] = []
for instance in workflow_dict['instances']:
if instance.instance_type == Instance.REDIS_SENTINEL:
continue
host = instance.hostname
LOG.info("Creating nfsaas disk...")
disk = NfsaasProvider(
).create_disk(environment=workflow_dict['environment'],
plan=workflow_dict[
'plan'],
host=host)
if not disk:
return False
workflow_dict['disks'].append(disk)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0009)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
databaseinfra = workflow_dict['databaseinfra']
databaseinfra.environment = workflow_dict['source_environment']
databaseinfra.plan = workflow_dict['source_plan']
databaseinfra.save()
database = workflow_dict['database']
database.environment = workflow_dict['source_environment']
database.save()
dbinfraoffering = DatabaseInfraOffering.objects.get(
databaseinfra=databaseinfra)
dbinfraoffering.offering = workflow_dict['source_offering']
dbinfraoffering.save()
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
master_source_instance = workflow_dict['source_instances'][0]
master_target_instance = workflow_dict['source_instances'][0].future_instance
slave_target_instance = workflow_dict['source_instances'][1].future_instance
master_log_file, master_log_pos = get_replication_info(master_target_instance)
change_master_to(instance=master_target_instance,
master_host=master_source_instance.address,
bin_log_file=workflow_dict['binlog_file'],
bin_log_position=workflow_dict['binlog_pos'])
change_master_to(instance=slave_target_instance,
master_host=master_target_instance.address,
bin_log_file=master_log_file,
bin_log_position=master_log_pos)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
try:
original_cloudstackpack = workflow_dict['original_cloudstackpack']
environment = workflow_dict['environment']
cs_credentials = get_credentials_for(
environment=environment, credential_type=CredentialType.CLOUDSTACK)
cs_provider = CloudStackProvider(credentials=cs_credentials)
original_serviceofferingid = original_cloudstackpack.offering.serviceofferingid
if workflow_dict['offering_changed']:
host = workflow_dict['host']
host_csattr = HostAttr.objects.get(host=host)
offering_changed = cs_provider.change_service_for_vm(
vm_id=host_csattr.vm_id, serviceofferingid=original_serviceofferingid)
if not offering_changed:
raise Exception("Could not change offering for Host {}".format(host))
else:
LOG.info('No resize to instance {}'.format(workflow_dict['instance']))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def run_vm_script(workflow_dict, context_dict, script):
try:
instances_detail = workflow_dict["instances_detail"]
final_context_dict = dict(context_dict.items() + workflow_dict["initial_context_dict"].items())
for instance_detail in instances_detail:
host = instance_detail["instance"].hostname
host_csattr = HostAttr.objects.get(host=host)
final_context_dict["IS_MASTER"] = instance_detail["is_master"]
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=command,
output=output,
)
if return_code:
raise Exception, "Could not run script. Output: {}".format(output)
return True
except Exception:
traceback = full_stack()
workflow_dict["exceptions"]["error_codes"].append(DBAAS_0015)
workflow_dict["exceptions"]["traceback"].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
databaseinfra = workflow_dict['databaseinfra']
host = workflow_dict['host']
hosts = [host, ]
if workflow_dict['not_primary_hosts'] >= 1:
hosts.extend(workflow_dict['not_primary_hosts'])
for host in hosts:
return_code, output = use_database_initialization_script(databaseinfra=databaseinfra,
host=host,
option='stop')
if return_code != 0:
LOG.info(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0021)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
script = build_stop_database_script()
script = build_context_script({}, script)
for target_instance in workflow_dict['target_instances']:
target_host = target_instance.hostname
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
output = {}
exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
try:
if 'region_migration_dir_infra_name' in workflow_dict:
shutil.rmtree(workflow_dict['region_migration_dir_infra_name'])
except Exception:
pass
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
if 'databaseinfra' not in workflow_dict \
or 'clone' not in workflow_dict:
return False
args = get_clone_args(
workflow_dict['clone'], workflow_dict['database'])
script_name = factory_for(
workflow_dict['clone'].databaseinfra).clone()
return_code, output = call_script(
script_name, working_dir=settings.SCRIPTS_PATH, args=args, split_lines=False,)
LOG.info("Script Output: {}".format(output))
LOG.info("Return code: {}".format(return_code))
if return_code != 0:
workflow_dict['exceptions']['traceback'].append(output)
return False
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0017)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
command = 'rm -rf /data2 && {} && mount /data'
host = workflow_dict['host']
cs_host_attr = CsHostAttr.objects.get(host=host)
mount = workflow_dict['mount']
command = command.format(mount)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
if return_code != 0:
LOG.info(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0021)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
command = "mkdir /data2 && mount -t nfs -o bg,intr {} /data2"
host = workflow_dict['host']
volume = workflow_dict['volume']
command = command.format(volume.nfsaas_path)
cs_host_attr = CsHostAttr.objects.get(host=host)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=command,
output=output)
if return_code != 0:
raise Exception(str(output))
workflow_dict['mount'] = command
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0022)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
target_host_one = workflow_dict[
'source_hosts'][0].future_host.address
target_host_two = workflow_dict[
'source_hosts'][1].future_host.address
target_secondary_ip_one = workflow_dict[
'source_secondary_ips'][0].equivalent_dbinfraattr.ip
target_secondary_ip_two = workflow_dict[
'source_secondary_ips'][1].equivalent_dbinfraattr.ip
flipper = FlipperProvider()
LOG.info("Creating Flipper...")
flipper.create_flipper_dependencies(
masterpairname=databaseinfra.name,
hostname1=target_host_one,
writeip=target_secondary_ip_one,
readip=target_secondary_ip_two,
hostname2=target_host_two,
environment=workflow_dict['target_environment'])
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for source_host in workflow_dict['source_hosts']:
host = source_host.future_host
LOG.info("Starting td_agent on host {}".format(host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
context_dict = {}
script = test_bash_script_error()
script += build_start_td_agent_script()
script = build_context_script(context_dict, script)
LOG.info(script)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
LOG.error("Error starting td_agent")
LOG.error(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
try:
if not 'database' in workflow_dict:
return False
database = workflow_dict['database']
if not database.is_in_quarantine:
LOG.info("Putting Database in quarentine...")
database.is_in_quarantine = True
database.quarantine_dt = datetime.datetime.now().date()
database.save()
database.delete()
LOG.info("Database destroyed....")
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0003)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
host = workflow_dict['host']
hosts = [host, ]
if workflow_dict['not_primary_hosts'] >= 1:
hosts.extend(workflow_dict['not_primary_hosts'])
for host in hosts:
return_code, output = use_database_initialization_script(databaseinfra=databaseinfra,
host=host,
option='start')
if return_code != 0:
raise Exception(str(output))
LOG.info('Wait 1 minute before start other instance')
sleep(60)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0021)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
try:
LOG.info("Remove all backup log files")
backup_log_dict = get_backup_log_configuration_dict(environment=workflow_dict['environment'],
databaseinfra=workflow_dict['databaseinfra'])
if backup_log_dict is None:
return True
instance = workflow_dict['instances'][0]
host = instance.hostname
host_csattr = CsHostAttr.objects.get(host=host)
exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=backup_log_dict['CLEAN_BACKUP_LOG_SCRIPT'])
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0014)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def start_vm(workflow_dict):
try:
environment = workflow_dict['environment']
cs_credentials = get_credentials_for(environment = environment, credential_type = CredentialType.CLOUDSTACK)
cs_provider = CloudStackProvider(credentials = cs_credentials)
instances_detail = workflow_dict['instances_detail']
for instance_detail in instances_detail:
instance = instance_detail['instance']
host = instance.hostname
host_csattr = HostAttr.objects.get(host=host)
started = cs_provider.start_virtual_machine(vm_id = host_csattr.vm_id)
if not started:
raise Exception, "Could not start host {}".format(host)
for instance_detail in instances_detail:
instance = instance_detail['instance']
host = instance.hostname
host_csattr = HostAttr.objects.get(host=host)
host_ready = check_ssh(server=host.address, username=host_csattr.vm_user, password=host_csattr.vm_password, wait=5, interval=10)
if not host_ready:
error = "Host %s is not ready..." % host
LOG.warn(error)
raise Exception, error
return True
except Exception, e:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def stop_vm(workflow_dict):
try:
environment = workflow_dict['environment']
cs_credentials = get_credentials_for(
environment=environment, credential_type=CredentialType.CLOUDSTACK)
cs_provider = CloudStackProvider(credentials=cs_credentials)
instances_detail = workflow_dict['instances_detail']
for instance_detail in instances_detail:
instance = instance_detail['instance']
host = instance.hostname
host_csattr = HostAttr.objects.get(host=host)
stoped = cs_provider.stop_virtual_machine(vm_id=host_csattr.vm_id)
if not stoped:
raise Exception("Could not stop host {}".format(host))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
workflow_dict['disks'] = []
for instance in workflow_dict['instances']:
host = instance.hostname
if instance.is_arbiter:
LOG.info("Do not creat nfsaas disk for Arbiter...")
continue
LOG.info("Creating nfsaas disk...")
disk = NfsaasProvider(
).create_disk(environment=workflow_dict['environment'],
plan=workflow_dict[
'plan'],
host=host)
if not disk:
return False
workflow_dict['disks'].append(disk)
return True
except Exception, e:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0009)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
script = build_clean_database_dir_script()
script = build_context_script({}, script)
for source_host in workflow_dict['source_hosts']:
target_host = source_host.future_host
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
output = {}
exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def run_vm_script(workflow_dict, context_dict, script):
try:
final_context_dict = dict(context_dict.items())
instance = workflow_dict['instance']
host = workflow_dict['host']
host_csattr = HostAttr.objects.get(host=host)
final_context_dict['HOSTADDRESS'] = instance.address
final_context_dict['PORT'] = instance.port
final_context_dict['DBPASSWORD'] = workflow_dict['databaseinfra'].password
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=command,
output=output)
if return_code:
raise Exception("Could not run script. Output: {}".format(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
host = workflow_dict['host']
hosts = [host, ]
workflow_dict['stoped_hosts'] = []
if len(workflow_dict['not_primary_hosts']) >= 1:
LOG.info("SECONDARY HOSTS: {}".format(workflow_dict['not_primary_hosts']))
hosts.extend(workflow_dict['not_primary_hosts'])
LOG.debug("HOSTS: {}".format(hosts))
for host in hosts:
return_code, output = use_database_initialization_script(databaseinfra=databaseinfra,
host=host,
option='stop')
if return_code != 0:
raise Exception(str(output))
workflow_dict['stoped_hosts'].append(host)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0021)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for source_host in workflow_dict['source_hosts']:
future_host = source_host.future_host
hosts_option = [(future_host, 'stop')]
for host, option in hosts_option:
LOG.info("{} td_agent on host {}".format(option, host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
script = test_bash_script_error()
script += monit_script(option)
script += td_agent_script(option)
LOG.info(script)
output = {}
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
LOG.error("Error stopping td_agent")
LOG.error(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def run_vm_script(workflow_dict, context_dict, script, reverse=False, wait=0):
try:
instances_detail = workflow_dict['instances_detail']
final_context_dict = dict(context_dict.items() + workflow_dict['initial_context_dict'].items())
if reverse:
instances_detail_final = instances_detail[::-1]
else:
instances_detail_final = instances_detail
for instance_detail in instances_detail_final:
host = instance_detail['instance'].hostname
host_csattr = HostAttr.objects.get(host=host)
command = build_context_script(final_context_dict, script)
output = {}
return_code = exec_remote_command(server = host.address,
username = host_csattr.vm_user,
password = host_csattr.vm_password,
command = command,
output = output)
if return_code:
raise Exception, "Could not run script. Output: {}".format(output)
sleep(wait)
return True
except Exception, e:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
for source_instance in workflow_dict['source_instances']:
if source_instance.instance_type == source_instance.REDIS:
source_host = source_instance.hostname
change_slave_priority_file(
host=source_host, original_value=0, final_value=100)
change_slave_priority_instance(
instance=source_instance, final_value=100)
target_instance = source_instance.future_instance
target_host = target_instance.hostname
change_slave_priority_file(
host=target_host, original_value=100, final_value=0)
change_slave_priority_instance(
instance=target_instance, final_value=0)
for source_instance in workflow_dict['source_instances']:
if source_instance.instance_type == source_instance.REDIS_SENTINEL:
failover_sentinel(host=source_instance.hostname,
sentinel_host=source_instance.address,
sentinel_port=source_instance.port,
service_name=workflow_dict['databaseinfra'].name)
break
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
script = "ps -ef | grep bootstrap-puppet3-loop.sh | grep -v grep | wc -l"
for host in workflow_dict['target_hosts']:
LOG.info("Getting vm credentials...")
host_csattr = CsHostAttr.objects.get(host=host)
attempt = 1
retries = 60
interval = 20
sleep(interval)
while True:
LOG.info(
"Check if puppet-setup is runnig on {} - attempt {} of {}"
.format(host, attempt, retries))
output = {}
return_code = exec_remote_command(
server=host.address,
username=host_csattr.vm_user,
password=host_csattr.vm_password,
command=script,
output=output)
if return_code != 0:
raise Exception(str(output))
ret_value = int(output['stdout'][0])
if ret_value == 0:
LOG.info(
"Puppet-setup is not runnig on {}".format(host))
break
LOG.info("Puppet-setup is runnig on {}".format(host))
attempt += 1
if attempt == retries:
error = "Maximum number of attempts check is puppet is running on {}.".format(
host)
LOG.error(error)
raise Exception(error)
sleep(interval)
puppet_code_status, output = self.get_puppet_code_status(
host, host_csattr)
if puppet_code_status != 0:
message = "Puppet-setup returned an error on {}. Output: {}".format(
host, output)
raise EnvironmentError(message)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0013)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
workflow_dict['hosts_and_exports'] = []
databaseinfra = workflow_dict['databaseinfra']
snapshot_id = workflow_dict['snapshot_id']
nfsaas_export_id = workflow_dict['export_id_snapshot']
restore_job = restore_snapshot(
environment=databaseinfra.environment,
export_id=nfsaas_export_id,
snapshot_id=snapshot_id)
job_result = restore_wait_for_finished(
environment=databaseinfra.environment,
job_id=restore_job['job'])
if 'id' in job_result:
new_export_id = job_result['id']
new_export_path_host = job_result['path']
new_export_path = job_result['full_path']
else:
raise Exception('Error while restoring nfs snapshot')
host = workflow_dict['host']
disk = HostAttr.objects.get(host=host, is_active=True)
workflow_dict['hosts_and_exports'].append({
'host':
host,
'old_export_id':
disk.nfsaas_export_id,
'old_export_path':
disk.nfsaas_path,
'old_export_path_host':
disk.nfsaas_path_host,
'new_export_id':
new_export_id,
'new_export_path':
new_export_path,
'new_export_path_host':
new_export_path_host,
})
old_disk = HostAttr.objects.get(nfsaas_export_id=nfsaas_export_id)
new_disk = HostAttr()
new_disk.host = old_disk.host
new_disk.nfsaas_export_id = new_export_id
new_disk.nfsaas_path = new_export_path
new_disk.nfsaas_path_host = new_export_path_host
new_disk.is_active = False
new_disk.nfsaas_size_kb = old_disk.nfsaas_size_kb
new_disk.save()
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0021)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
cs_credentials = get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.CLOUDSTACK)
cs_provider = CloudStackProvider(credentials=cs_credentials)
instances = workflow_dict['databaseinfra'].instances.all()
if not instances:
for vm_id in workflow_dict['vms_id']:
cs_provider.destroy_virtual_machine(
project_id=cs_credentials.project,
environment=workflow_dict['environment'],
vm_id=vm_id)
for host in workflow_dict['hosts']:
host_attr = HostAttr.objects.filter(host=host)
host.delete()
LOG.info("Host deleted!")
if host_attr:
host_attr[0].delete()
LOG.info("HostAttr deleted!")
for instance in instances:
host = instance.hostname
host_attr = HostAttr.objects.get(host=host)
LOG.info("Destroying virtualmachine %s" % host_attr.vm_id)
cs_provider.destroy_virtual_machine(
project_id=cs_credentials.project,
environment=workflow_dict['environment'],
vm_id=host_attr.vm_id)
host_attr.delete()
LOG.info("HostAttr deleted!")
instance.delete()
LOG.info("Instance deleted")
host.delete()
LOG.info("Host deleted!")
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0011)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def create_database(self, name, plan, environment, team, project, description, task_history=None, user=None):
AuditRequest.new_request("create_database", user, "localhost")
try:
worker_name = get_worker_name()
task_history = TaskHistory.register(request=self.request, task_history=task_history,
user=user, worker_name=worker_name)
LOG.info("id: %s | task: %s | kwargs: %s | args: %s" % (
self.request.id, self.request.task, self.request.kwargs, str(self.request.args)))
task_history.update_details(persist=True, details="Loading Process...")
result = make_infra(plan=plan,
environment=environment,
name=name,
team=team,
project=project,
description=description,
task=task_history,
)
if result['created'] == False:
if 'exceptions' in result:
error = "\n".join(": ".join(err)
for err in result['exceptions']['error_codes'])
traceback = "\nException Traceback\n".join(
result['exceptions']['traceback'])
error = "{}\n{}\n{}".format(error, traceback, error)
else:
error = "There is not any infra-structure to allocate this database."
task_history.update_status_for(
TaskHistory.STATUS_ERROR, details=error)
return
task_history.update_dbid(db=result['database'])
task_history.update_status_for(
TaskHistory.STATUS_SUCCESS, details='Database created successfully')
return
except Exception as e:
traceback = full_stack()
LOG.error("Ops... something went wrong: %s" % e)
LOG.error(traceback)
if 'result' in locals() and result['created']:
destroy_infra(
databaseinfra=result['databaseinfra'], task=task_history)
task_history.update_status_for(
TaskHistory.STATUS_ERROR, details=traceback)
return
finally:
AuditRequest.cleanup_request()
def stop_workflow(workflow_dict, task=None):
if 'database_pinned' not in workflow_dict:
if not _lock_databases(workflow_dict, task):
return False
if 'steps' not in workflow_dict:
return False
if 'exceptions' not in workflow_dict:
workflow_dict['exceptions'] = {}
workflow_dict['exceptions']['traceback'] = []
workflow_dict['exceptions']['error_codes'] = []
workflow_dict['total_steps'] = len(workflow_dict['steps'])
if 'step_counter' not in workflow_dict:
workflow_dict['step_counter'] = len(workflow_dict['steps'])
workflow_dict['msgs'] = []
workflow_dict['created'] = False
try:
for step in workflow_dict['steps'][::-1]:
my_class = import_by_path(step)
my_instance = my_class()
time_now = str(time.strftime("%m/%d/%Y %H:%M:%S"))
msg = "\n%s - Rollback Step %i of %i - %s" % (
time_now, workflow_dict['step_counter'], workflow_dict['total_steps'], str(my_instance))
LOG.info(msg)
workflow_dict['step_counter'] -= 1
if task:
workflow_dict['msgs'].append(msg)
task.update_details(persist=True, details=msg)
my_instance.undo(workflow_dict)
if task:
task.update_details(persist=True, details="DONE!")
_unlock_databases(workflow_dict, task)
return True
except Exception as e:
LOG.info("Exception: {}".format(e))
if not workflow_dict['exceptions']['error_codes'] or not workflow_dict['exceptions']['traceback']:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0001)
workflow_dict['exceptions']['traceback'].append(traceback)
LOG.warn("\n".join(": ".join(error)
for error in workflow_dict['exceptions']['error_codes']))
LOG.warn("\nException Traceback\n".join(
workflow_dict['exceptions']['traceback']))
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
databaseinfra = workflow_dict['databaseinfra']
vip_ip = get_vip_ip_from_databaseinfra(databaseinfra=databaseinfra)
databaseinfraattr = workflow_dict['source_secondary_ips'][0]
dnslist = DatabaseInfraDNSList.objects.filter(
dns__startswith="{}.".format(databaseinfra.name), type=FOXHA)
if dnslist:
infradns = dnslist[0]
infradns.type = FLIPPER
infradns.save()
DNSAPIProvider.update_database_dns_content(
databaseinfra=databaseinfra,
dns=infradns.dns,
old_ip=vip_ip,
new_ip=databaseinfraattr.ip)
if 'objects_changed' in workflow_dict:
for object_changed in workflow_dict['objects_changed']:
switch_dns_backward(
databaseinfra=databaseinfra,
source_object_list=[
object_changed['source_object'],
],
ip_attribute_name=object_changed['ip_attribute_name'],
dns_attribute_name=object_changed[
'dns_attribute_name'],
equivalent_atribute_name=object_changed[
'equivalent_atribute_name'])
return True
switch_dns_backward(
databaseinfra=databaseinfra,
source_object_list=workflow_dict['source_hosts'],
ip_attribute_name='address',
dns_attribute_name='hostname',
equivalent_atribute_name='future_host')
switch_dns_backward(
databaseinfra=databaseinfra,
source_object_list=workflow_dict['source_instances'],
ip_attribute_name='address',
dns_attribute_name='dns',
equivalent_atribute_name='future_instance')
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
workflow_dict['hosts_and_exports'] = []
databaseinfra = workflow_dict['databaseinfra']
snapshot_id = workflow_dict['snapshot_id']
nfsaas_export_id = workflow_dict['export_id_snapshot']
provider = NfsaasProvider()
restore_result = provider.restore_snapshot(
environment=databaseinfra.environment,
export_id=nfsaas_export_id,
snapshot_id=snapshot_id)
job_result = provider.check_restore_nfsaas_job(
environment=databaseinfra.environment,
job_id=restore_result['job'])
if 'id' in job_result['result']:
new_export_id = job_result['result']['id']
new_export_path = job_result['result']['path']
else:
raise Exception('Error while restoring nfs snapshot')
host = workflow_dict['host']
workflow_dict['hosts_and_exports'].append({
'host':
host,
'old_export_id':
workflow_dict['export_id'],
'old_export_path':
workflow_dict['export_path'],
'new_export_id':
new_export_id,
'new_export_path':
new_export_path,
})
old_host_attr = HostAttr.objects.get(
nfsaas_export_id=nfsaas_export_id)
new_host_attr = HostAttr()
new_host_attr.host = old_host_attr.host
new_host_attr.nfsaas_export_id = new_export_id
new_host_attr.nfsaas_path = new_export_path
new_host_attr.is_active = False
new_host_attr.nfsaas_team_id = old_host_attr.nfsaas_team_id
new_host_attr.nfsaas_project_id = old_host_attr.nfsaas_project_id
new_host_attr.nfsaas_environment_id = old_host_attr.nfsaas_environment_id
new_host_attr.nfsaas_size_id = old_host_attr.nfsaas_size_id
new_host_attr.save()
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0021)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
if 'databaseinfra' not in workflow_dict and 'hosts' not in workflow_dict:
LOG.info("We could not find a databaseinfra inside the workflow_dict")
return False
if len(workflow_dict['hosts']) == 1:
return True
databaseinfraattr = DatabaseInfraAttr.objects.filter(
databaseinfra=workflow_dict['databaseinfra'])
cs_credentials = get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.CLOUDSTACK)
networkapi_credentials = get_credentials_for(
environment=workflow_dict['environment'],
credential_type=CredentialType.NETWORKAPI)
cs_provider = CloudStackProvider(credentials=cs_credentials,
networkapi_credentials=networkapi_credentials)
networkapi_equipment_id = workflow_dict.get('networkapi_equipment_id')
for infra_attr in databaseinfraattr:
networkapi_equipment_id = infra_attr.networkapi_equipment_id
networkapi_ip_id = infra_attr.networkapi_ip_id
if networkapi_ip_id:
LOG.info("Removing network api IP for %s" % networkapi_ip_id)
if not cs_provider.remove_networkapi_ip(equipment_id=networkapi_equipment_id,
ip_id=networkapi_ip_id):
return False
LOG.info("Removing secondary_ip for %s" % infra_attr.cs_ip_id)
if not cs_provider.remove_secondary_ips(infra_attr.cs_ip_id):
return False
LOG.info("Secondary ip deleted!")
infra_attr.delete()
LOG.info("Databaseinfraattr deleted!")
if networkapi_equipment_id:
cs_provider.remove_networkapi_equipment(equipment_id=networkapi_equipment_id)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0010)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
master_host = workflow_dict['host']
master_instance = Instance.objects.get(hostname=master_host)
secondary_host = workflow_dict['not_primary_hosts'][0]
secondary_instance = Instance.objects.get(hostname=secondary_host)
master_log_file, master_log_pos = get_replication_information_from_file(
host=master_host)
return_code, output = use_database_initialization_script(databaseinfra=databaseinfra,
host=master_host,
option='start')
if return_code != 0:
raise Exception(str(output))
return_code, output = use_database_initialization_script(databaseinfra=databaseinfra,
host=secondary_host,
option='start')
if return_code != 0:
raise Exception(str(output))
LOG.info("Waiting 1 minute to continue")
sleep(60)
change_master_to(instance=master_instance,
master_host=secondary_host.address,
bin_log_file=master_log_file,
bin_log_position=master_log_pos)
change_master_to(instance=secondary_instance,
master_host=master_host.address,
bin_log_file=master_log_file,
bin_log_position=master_log_pos)
start_slave(instance=master_instance)
start_slave(instance=secondary_instance)
LOG.info("Waiting 30 seconds to continue")
sleep(30)
driver = databaseinfra.get_driver()
driver.set_read_ip(instance=master_instance)
driver.set_master(instance=secondary_instance)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
try:
if not 'databaseinfra' in workflow_dict:
return False
action = 'deny'
database = workflow_dict['databaseinfra'].databases.get()
for database_bind in database.acl_binds.all():
acl_environment, acl_vlan = database_bind.bind_address.split('/')
data = {"kind":"object#acl", "rules":[]}
default_options = {"protocol": "tcp",
"source": "",
"destination": "",
"description": "{} access for database {} in {}".\
format(database_bind.bind_address, database.name,
database.environment.name),
"action": action,
"l4-options":{ "dest-port-start":"",
"dest-port-op":"eq"
}
}
LOG.info("Default options: {}".format(default_options))
databaseinfra = database.infra
infra_instances_binds = DatabaseInfraInstanceBind.objects.\
filter(databaseinfra= databaseinfra,bind_address= database_bind.bind_address)
for infra_instance_bind in infra_instances_binds:
custom_options = copy.deepcopy(default_options)
custom_options['source'] = database_bind.bind_address
custom_options['destination'] = infra_instance_bind.instance + '/32'
custom_options['l4-options']['dest-port-start'] = infra_instance_bind.instance_port
data['rules'].append(custom_options)
acl_credential = get_credentials_for(environment= database.environment,
credential_type=CredentialType.ACLAPI)
acl_client = AclClient(acl_credential.endpoint, acl_credential.user,
acl_credential.password)
LOG.info("Data used on payload: {}".format(data))
acl_client.revoke_acl_for(environment= acl_environment,
vlan= acl_vlan, payload=data)
infra_instances_binds.delete()
database_bind.delete()
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0019)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def steps_for_instances(list_of_groups_of_steps,
instances,
task,
step_counter_method=None,
since_step=0):
steps_total = 0
for group_of_steps in list_of_groups_of_steps:
steps_total += len(group_of_steps.items()[0][1])
steps_total = steps_total * len(instances)
step_current = 0
task.add_detail('Instances: {}'.format(len(instances)))
for instance in instances:
task.add_detail('{}'.format(instance), level=2)
task.add_detail('')
if since_step:
task.add_detail('Skipping until step {}\n'.format(since_step))
for count, group_of_steps in enumerate(list_of_groups_of_steps, start=1):
task.add_detail('Starting group of steps {} of {} - {}'.format(
count, len(list_of_groups_of_steps),
group_of_steps.keys()[0]))
steps = group_of_steps.items()[0][1]
for instance in instances:
task.add_detail('Instance: {}'.format(instance))
for step in steps:
step_current += 1
if step_counter_method:
step_counter_method(step_current)
try:
step_class = import_by_path(step)
step_instance = step_class(instance)
task.add_step(step_current, steps_total,
str(step_instance))
if step_current < since_step:
task.update_details("SKIPPED!", persist=True)
else:
step_instance.do()
task.update_details("SUCCESS!", persist=True)
except Exception as e:
task.update_details("FAILED!", persist=True)
task.add_detail(str(e))
task.add_detail(full_stack())
return False
task.add_detail('Ending group of steps: {} of {}\n'.format(
count, len(list_of_groups_of_steps)))
return True
def undo(self, workflow_dict):
try:
return stop_vm_func(workflow_dict)
except Exception as e:
LOG.error(e.message)
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0015)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for index, instance in enumerate(
workflow_dict['target_instances']):
if instance.instance_type == instance.MONGODB_ARBITER:
continue
if instance.instance_type == instance.REDIS_SENTINEL:
continue
host = instance.hostname
LOG.info("Mounting disks on host {}".format(host))
cs_host_attr = CS_HostAttr.objects.get(host=host)
nfs_host_attr = NFS_HostAttr.objects.get(host=host)
LOG.info("Cheking host ssh...")
host_ready = check_ssh(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
wait=5,
interval=10)
if not host_ready:
raise Exception(str("Host %s is not ready..." % host))
context_dict = {
'EXPORTPATH': nfs_host_attr.nfsaas_path,
}
script = test_bash_script_error()
script += build_mount_disk_script()
script = build_context_script(context_dict, script)
LOG.info(script)
output = {}
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
vip = workflow_dict['vip']
vip_ip = vip.vip_ip
mysql_fox_credentials = get_credentials_for(
environment=databaseinfra.environment,
credential_type=CredentialType.MYSQL_FOXHA)
mysql_repl_credentials = get_credentials_for(
environment=databaseinfra.environment,
credential_type=CredentialType.MYSQL_REPLICA)
foxha_credentials = get_credentials_for(
environment=databaseinfra.environment,
credential_type=CredentialType.FOXHA)
dbaas_api = DatabaseAsAServiceApi(databaseinfra=databaseinfra,
credentials=foxha_credentials)
foxprovider = FoxHAProvider(dbaas_api=dbaas_api)
LOG.info('Adding foxah group {}'.format(databaseinfra.name))
foxprovider.add_group(
group_name=databaseinfra.name,
description=databaseinfra.name,
vip_address=vip_ip,
mysql_user=mysql_fox_credentials.user,
mysql_password=str(mysql_fox_credentials.password),
repl_user=mysql_repl_credentials.user,
repl_password=str(mysql_repl_credentials.password))
for index, instance in enumerate(
workflow_dict['target_instances']):
if index == 0:
mode = 'read_write'
else:
mode = 'read_only'
LOG.info('Adding foxah node {}'.format(instance.dns))
foxprovider.add_node(group_name=databaseinfra.name,
name=instance.dns,
node_ip=instance.address,
port=instance.port,
mode=mode,
status='enabled')
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0013)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0021)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0019)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
if 'databaseinfra' not in workflow_dict:
LOG.info("We could not find a databaseinfra inside the workflow_dict")
return False
source_secondary_ip_ids = [secondary_ip.id for secondary_ip in workflow_dict['source_secondary_ips']]
databaseinfraattr = DatabaseInfraAttr.objects.filter(
databaseinfra=workflow_dict['databaseinfra'],
equivalent_dbinfraattr=None).exclude(id__in=source_secondary_ip_ids)
LOG.info("databaseinfraattr: {}".format(databaseinfraattr))
LOG.info("old infra ip: {}".format(workflow_dict['source_secondary_ips']))
cs_credentials = get_credentials_for(
environment=workflow_dict['target_environment'],
credential_type=CredentialType.CLOUDSTACK)
networkapi_credentials = get_credentials_for(
environment=workflow_dict['target_environment'],
credential_type=CredentialType.NETWORKAPI)
cs_provider = CloudStackProvider(credentials=cs_credentials,
networkapi_credentials=networkapi_credentials)
for infra_attr in databaseinfraattr:
networkapi_equipment_id = infra_attr.networkapi_equipment_id
networkapi_ip_id = infra_attr.networkapi_ip_id
if networkapi_ip_id:
LOG.info("Removing network api IP for %s" % networkapi_ip_id)
if not cs_provider.remove_networkapi_ip(equipment_id=networkapi_equipment_id,
ip_id=networkapi_ip_id):
return False
LOG.info("Removing secondary_ip for %s" % infra_attr.cs_ip_id)
if not cs_provider.remove_secondary_ips(infra_attr.cs_ip_id):
return False
LOG.info("Secondary ip deleted!")
infra_attr.delete()
LOG.info("Databaseinfraattr deleted!")
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0010)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
LOG.info("Creating dns for VIP...")
vip_ip = get_vip_ip_from_databaseinfra(databaseinfra=databaseinfra)
vip_dns = add_dns_record(databaseinfra=databaseinfra,
name=databaseinfra.name,
ip=vip_ip,
type=FOXHA)
LOG.info("Updating databaseinfra endpoint...")
databaseinfra.endpoint_dns = "{}:{}".format(vip_dns, 3306)
databaseinfra.save()
workflow_dict['databaseinfra'] = databaseinfra
LOG.info("Creating dns for hosts...")
for host_name in zip(workflow_dict['hosts'],
workflow_dict['names']['vms']):
host = host_name[0]
host.hostname = add_dns_record(
databaseinfra=workflow_dict['databaseinfra'],
name=host_name[1],
ip=host.address,
type=HOST)
host.save()
LOG.info("Creating dns for instances...")
for instance_name in zip(workflow_dict['instances'],
workflow_dict['names']['vms']):
instance = instance_name[0]
instance.dns = add_dns_record(
databaseinfra=workflow_dict['databaseinfra'],
name=instance_name[1],
ip=instance.address,
type=INSTANCE)
instance.save()
LOG.info("Calling dnsapi provider...")
DNSAPIProvider.create_database_dns(
databaseinfra=workflow_dict['databaseinfra'])
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0007)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def start_workflow_ha(workflow_dict, task=None):
if 'steps' not in workflow_dict:
return False
init_workflow_vars(workflow_dict)
try:
for instance in workflow_dict['instances']:
workflow_dict['instance_step_counter'] = 0
task.update_details(
persist=True,
details='\n>> Starting steps for VM {}:'.format(
instance.hostname))
workflow_dict['instance'] = instance
workflow_dict['host'] = instance.hostname
if workflow_dict['databaseinfra'].plan.is_ha and workflow_dict[
'driver'].check_instance_is_master(instance):
LOG.info("Waiting 60s to check continue...")
time.sleep(60)
workflow_dict['driver'].check_replication_and_switch(instance)
LOG.info("Waiting 60s to check continue...")
time.sleep(60)
for step in workflow_dict['steps']:
workflow_dict['global_step_counter'] += 1
workflow_dict['instance_step_counter'] += 1
execute(step, workflow_dict, False, task)
workflow_dict['completed_instances'].append(instance)
workflow_dict['created'] = True
return True
except Exception:
if not workflow_dict['exceptions']['error_codes'] or not workflow_dict[
'exceptions']['traceback']:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0001)
workflow_dict['exceptions']['traceback'].append(traceback)
LOG.warn("\n".join(
": ".join(error)
for error in workflow_dict['exceptions']['error_codes']))
LOG.warn("\nException Traceback\n".join(
workflow_dict['exceptions']['traceback']))
workflow_dict['steps_until_stopped'] = workflow_dict[
'steps'][:workflow_dict['instance_step_counter']]
stop_workflow_ha(workflow_dict, task)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
connect_string = build_mongodb_connect_string(instances=workflow_dict['source_instances'],
databaseinfra=databaseinfra)
context_dict = {
'CONNECT_STRING': connect_string,
}
script = test_bash_script_error()
script += build_switch_primary_to_new_instances_script()
script = build_context_script(context_dict, script)
output = {}
host = workflow_dict['source_instances'][0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
sleep(30)
driver = databaseinfra.get_driver()
client = driver.get_client(instance=None)
rsconf = client['local'].system.replset.find_one()
priority_0 = rsconf['members'][0].get('priority', 1)
priority_1 = rsconf['members'][1].get('priority', 1)
if priority_0 != 0 or priority_1 != 0:
errormsg = "The priority of the old mongodb instances should be zero."
LOG.error(errormsg)
raise Exception(errormsg)
replSetGetStatus = client.admin.command('replSetGetStatus')
if 'PRIMARY' not in (replSetGetStatus['members'][3]['stateStr'],
replSetGetStatus['members'][4]['stateStr']):
errormsg = "One of the new instances should be PRIMARY."
LOG.error(errormsg)
raise Exception(errormsg)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
driver = databaseinfra.get_driver()
sentinel = driver.get_sentinel_client()
master = sentinel.discover_master(databaseinfra.name)
master_host = master[0]
master_port = master[1]
for index, source_host in enumerate(workflow_dict['source_hosts']):
target_host = source_host.future_host
target_cs_host_attr = CS_HostAttr.objects.get(host=target_host)
script = test_bash_script_error()
script += build_start_stop_scripts()
if index < 2:
script += build_start_database_script()
script += build_start_sentinel_script()
script += build_start_http_script()
script = build_context_script({}, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(server=target_host.address,
username=target_cs_host_attr.vm_user,
password=target_cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
for target_instance in workflow_dict['target_instances']:
if target_instance.instance_type == target_instance.REDIS:
LOG.info(
'Changing master of host: {}'.format(target_instance.address))
client = driver.get_client(target_instance)
client.slaveof(master_host, master_port)
LOG.info(
'New master: {}:{}'.format(master_host, master_port))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
driver = databaseinfra.get_driver()
for instance in workflow_dict['instances']:
attempt = 1
retries = 20
interval = 30
while True:
LOG.info(
"Check if replication is ok on {} - attempt {} of {}".
format(instance, attempt, retries))
if driver.is_replication_ok(instance):
if driver.is_heartbeat_replication_ok(instance):
LOG.info(
"Replication is ok on {}".format(instance))
break
else:
LOG.info(
"Heartbeat replication is not ok on {}".format(
instance))
LOG.info("Restarting slave on {}".format(instance))
driver.stop_slave(instance)
sleep(1)
driver.start_slave(instance)
else:
LOG.info(
"Replication is not ok on {}".format(instance))
attempt += 1
if attempt == retries:
error = "Maximum number of attempts check replication on {}.".format(
instance)
LOG.error(error)
raise Exception(error)
sleep(interval)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0013)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
LOG.info("Creating dns for hosts...")
for host_name in zip(workflow_dict['hosts'],
workflow_dict['names']['vms']):
host = host_name[0]
LOG.info("Updating host dns - host: {}".format(host))
host.hostname = add_dns_record(
databaseinfra=workflow_dict['databaseinfra'],
name=host_name[1],
ip=host.address,
type=HOST)
host.save()
LOG.info("DNS host updated - dns: {}".format(host.hostname))
LOG.info("Creating dns for instances...")
for instance_name in zip(workflow_dict['instances'],
workflow_dict['names']['vms']):
instance = instance_name[0]
LOG.info(
"Updating instance dns - instance: {}".format(instance))
instance.dns = add_dns_record(
databaseinfra=workflow_dict['databaseinfra'],
name=instance_name[1],
ip=instance.address,
type=INSTANCE)
instance.save()
LOG.info("DNS instance updated - dns: {}".format(instance.dns))
if workflow_dict['qt'] == 1:
LOG.info("Updating databaseinfra dns endpoint")
databaseinfra = workflow_dict['databaseinfra']
databaseinfra.endpoint_dns = instance.dns + \
':%i' % instance.port
databaseinfra.save()
workflow_dict['databaseinfra'] = databaseinfra
LOG.info("Calling dnsapi provider...")
DNSAPIProvider.create_database_dns(
databaseinfra=workflow_dict['databaseinfra'])
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0007)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
databaseinfra = workflow_dict['databaseinfra']
target_instances = []
for source_instance in workflow_dict['source_instances']:
target_instances.append(source_instance.future_instance)
connect_string = build_mongodb_connect_string(
instances=target_instances, databaseinfra=databaseinfra)
context_dict = {
'CONNECT_STRING':
connect_string,
'SECUNDARY_ONE':
"{}:{}".format(workflow_dict['source_instances'][0].address,
workflow_dict['source_instances'][0].port),
'SECUNDARY_TWO':
"{}:{}".format(workflow_dict['source_instances'][1].address,
workflow_dict['source_instances'][1].port),
'ARBITER':
"{}:{}".format(workflow_dict['source_instances'][2].address,
workflow_dict['source_instances'][2].port),
}
script = test_bash_script_error()
script += build_remove_replica_set_members_script()
script = build_context_script(context_dict, script)
output = {}
host = workflow_dict['source_instances'][0].hostname
cs_host_attr = CS_HostAttr.objects.get(host=host)
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
database = workflow_dict['database']
databaseinfra = workflow_dict['databaseinfra']
acl_credential = get_credentials_for(
environment=database.environment,
credential_type=CredentialType.ACLAPI)
acl_client = AclClient(acl_credential.endpoint,
acl_credential.user,
acl_credential.password,
database.environment)
instances = databaseinfra.instances.filter(
future_instance__isnull=False)
databaseinfraattr_instances = DatabaseInfraAttr.objects.filter(
databaseinfra=databaseinfra,
equivalent_dbinfraattr__isnull=False)
instance_address_list = []
for instance in instances:
instance_address_list.append(instance.address)
for instance in databaseinfraattr_instances:
instance_address_list.append(instance.ip)
for database_bind in database.acl_binds.all():
if helpers.bind_address(
database_bind,
acl_client,
instances=instances,
infra_attr_instances=databaseinfraattr_instances):
continue
else:
LOG.error("The AclApi is not working properly.")
database_bind.bind_status = ERROR
database_bind.save()
DatabaseInfraInstanceBind.objects.filter(
databaseinfra=databaseinfra,
bind_address=database_bind.bind_address,
instance__in=instance_address_list).update(
bind_status=ERROR)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def collect_tweets_by_search_terms(search_configs_filepath, output_folder,
config):
apikeys = list(config['apikeys'].values()).pop()
search_configs = {}
with open(os.path.abspath(search_configs_filepath),
'r') as search_configs_rf:
search_configs = json.load(search_configs_rf)
for search_config_id in itertools.cycle(search_configs):
search_config = search_configs[search_config_id]
search_terms = [term.lower() for term in search_config['terms']]
querystring = '%s' % (' OR '.join('(' + term + ')'
for term in search_terms))
since_id = search_config[
'since_id'] if 'since_id' in search_config else 0
geocode = tuple(search_config['geocode']) if (
'geocode' in search_config and search_config['geocode']) else None
logger.info(
'REQUEST -> (md5(querystring): [%s]; since_id: [%d]; geocode: [%s])'
% (util.md5(querystring.encode('utf-8')), since_id, geocode))
try:
twitterCralwer = TwitterCrawler(apikeys=apikeys,
client_args=CLIENT_ARGS,
output_folder=output_folder)
since_id = twitterCralwer.search_by_query(querystring,
geocode=geocode,
since_id=since_id)
except Exception as exc:
logger.error(exc)
logger.error(util.full_stack())
pass
search_config['since_id'] = since_id
search_config['querystring'] = querystring
search_config['geocode'] = geocode
search_configs[search_config_id] = search_config
flash_cmd_config(search_configs, search_configs_filepath,
output_folder)
logger.info(
'COMPLETED -> (md5(querystring): [%s]; since_id: [%d]; geocode: [%s])'
% (util.md5(querystring.encode('utf-8')), since_id, geocode))
logger.info('PAUSE %ds to CONTINUE...' % WAIT_TIME)
time.sleep(WAIT_TIME)
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
DNSAPIProvider.update_database_dns_ttl(databaseinfra=workflow_dict['databaseinfra'], ttl=None)
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
for host_and_export in workflow_dict['hosts_and_exports']:
clean_unused_data(export_id=host_and_export['old_export_id'])
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0021)
workflow_dict['exceptions']['traceback'].append(traceback)
return True
def undo(self, workflow_dict):
LOG.info("Running undo...")
try:
LOG.info('Rollback mounting disks - nothing to do')
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False
def do(self, workflow_dict):
try:
flipper_crdentials = get_credentials_for(
workflow_dict['source_environment'], CredentialType.FLIPPER)
flipper_vip = flipper_crdentials.get_parameter_by_name('vip')
for host in workflow_dict['target_hosts']:
cs_host_attr = CS_HostAttr.objects.get(host=host)
source_host = workflow_dict['source_hosts'][0]
nf_host_attr = NF_HostAttr.objects.get(host=source_host)
script = test_bash_script_error()
script += build_mount_snapshot_volume_script()
script += build_remove_deprecated_files_script()
script += build_permission_script()
script += build_start_database_script()
script += build_flipper_script()
context_dict = {
'EXPORT_PATH': nf_host_attr.nfsaas_path,
'SNAPSHOPT_NAME': workflow_dict['snapshot_name'],
'VIP_FLIPPER': flipper_vip,
'IPWRITE': workflow_dict['target_secondary_ips'][0].ip,
'HOST01': workflow_dict['target_hosts'][0],
'HOST02': workflow_dict['target_hosts'][1]
}
script = build_context_script(context_dict, script)
output = {}
LOG.info(script)
return_code = exec_remote_command(
server=host.address,
username=cs_host_attr.vm_user,
password=cs_host_attr.vm_password,
command=script,
output=output)
LOG.info(output)
if return_code != 0:
raise Exception(str(output))
return True
except Exception:
traceback = full_stack()
workflow_dict['exceptions']['error_codes'].append(DBAAS_0020)
workflow_dict['exceptions']['traceback'].append(traceback)
return False