def launch_action_disk(action, hostname, user, port, from_scratch=False):
disk_path = action['disk_path']
id_domain = action['domain']
index_disk = action['index_disk']
array_out_err = execute_commands(hostname,
ssh_commands=action['ssh_commands'],
user=user,
port=port)
if action['type'] in ['create_disk', 'create_disk_from_scratch']:
if len([k['err'] for k in array_out_err if len(k['err']) == 0]):
##TODO: TEST WITH MORE THAN ONE DISK, 2 list_backing_chain must be created
log.debug('all operations creating disk {} for new domain {} runned ok'.format(disk_path, id_domain))
if from_scratch is False:
out_cmd_backing_chain = array_out_err[-1]['out']
list_backing_chain = extract_list_backing_chain(out_cmd_backing_chain)
if id_domain is not False:
update_domain_parents(id_domain)
update_disk_backing_chain(id_domain, index_disk, disk_path, list_backing_chain)
##INFO TO DEVELOPER
# ahora ya se puede llamar a starting paused
if id_domain is not False:
#update parents if have
#update_domain_parents(id_domain)
update_domain_status('CreatingDomain', id_domain, None,
detail='new disk created, now go to creating desktop and testing if desktop start')
else:
log.error('operations creating disk {} for new domain {} failed.'.format(disk_path, id_domain))
log.error('\n'.join(['cmd: {} / out: {} / err: {}'.format(action['ssh_commands'][i],
array_out_err[i]['out'],
array_out_err[i]['err']) for i in
range(len(action['ssh_commands']))]))
if id_domain is not False:
update_domain_status('Failed', id_domain, detail='new disk create operation failed, details in logs')
elif action['type'] == 'delete_disk':
if len(array_out_err[0]['err']) > 0:
log.error(
'disk from domain {} not found, or permission denied or access to data problems'.format(id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['err']))
update_domain_status('DiskDeleted', id_domain, detail='delete disk operation failed, disk not found: {}'.format(
array_out_err[0]['err']))
elif len(array_out_err[1]['err']) > 0:
log.error('disk from domain {} found, but erase command fail'.format(id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['err']))
update_domain_status('DiskDeleted', id_domain, detail='delete disk command failed')
elif len(array_out_err[2]['out']) > 0:
log.error(
'disk from domain {} found, erase command not failed with error message, but disk is in directory.'.format(
id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['out']))
update_domain_status('DiskDeleted', id_domain,
detail='delete disk operation failed, disk in directory can not erase')
else:
log.info('disk {} from domain {} erased'.format(disk_path, id_domain))
update_disk_backing_chain(id_domain, index_disk, 'DISK_ERASED', [])
update_domain_status('DiskDeleted', id_domain, detail='delete disk operation run ok')
def test_hypers_disk_operations(hyps_disk_operations):
list_hyps_ok = list()
str_random = ''.join(choices(string.ascii_uppercase + string.digits, k=8))
for hyp_id in hyps_disk_operations:
d_hyp = get_hyp_hostname_user_port_from_id(hyp_id)
cmds1 = list()
for pool_id in get_pools_from_hyp(hyp_id):
# test write permissions in root dir of all paths defined in pool
paths = {
k: [l['path'] for l in d]
for k, d in get_pool(pool_id)['paths'].items()
}
for k, p in paths.items():
for path in p:
cmds1.append({
'title':
f'try create dir if not exists - pool:{pool_id}, hypervisor: {hyp_id}, path_kind: {k}',
'cmd': f'mkdir -p {path}'
})
cmds1.append({
'title':
f'touch random file - pool:{pool_id}, hypervisor: {hyp_id}, path_kind: {k}',
'cmd':
f'touch {path}/test_random_{str_random}'
})
cmds1.append({
'title':
'delete random file - pool:{pool_id}, hypervisor: {hyp_id}, path_kind: {k}',
'cmd':
f'rm -f {path}/test_random_{str_random}'
})
try:
array_out_err = execute_commands(d_hyp['hostname'],
ssh_commands=cmds1,
dict_mode=True,
user=d_hyp['user'],
port=d_hyp['port'])
#if error in some path hypervisor is not valid
if len([d['err'] for d in array_out_err if len(d['err']) > 0]) > 0:
logs.main.error(
f'Hypervisor {hyp_id} can not be disk_operations, some errors when testing if can create files in all paths_'
)
for d_cmd_err in [
d for d in array_out_err if len(d['err']) > 0
]:
cmd = d_cmd_err['cmd']
err = d_cmd_err['err']
logs.main.error(f'Command: {cmd} -- Error: {err}')
else:
list_hyps_ok.append(hyp_id)
except Exception as e:
if __name__ == '__main__':
logs.main.err(
f'Error when launch commands to test hypervisor {hyp_id} disk_operations: {e}'
)
return list_hyps_ok
def launch_delete_disk_action(action, hostname, user, port):
disk_path = action['disk_path']
id_domain = action['domain']
array_out_err = execute_commands(hostname,
ssh_commands=action['ssh_commands'],
user=user,
port=port)
# ALBERTO FALTA ACABAR
pass
def launch_action_delete_disk(action, hostname, user, port):
disk_path = action['disk_path']
id_domain = action['domain']
array_out_err = execute_commands(hostname,
ssh_commands=action['ssh_commands'],
user=user,
port=port)
# last ls must fail
if len([k['err'] for k in array_out_err if len(k['err']) == 1]):
log.debug('all operations deleting disk {} for domain {} runned ok'.format(disk_path, id_domain))
def launch_delete_disk_action(action, hostname, user, port):
disk_path = action['disk_path']
id_domain = action['domain']
array_out_err = execute_commands(hostname,
ssh_commands=action['ssh_commands'],
user=user,
port=port)
# ALBERTO FALTA ACABAR
pass
def launch_action_delete_disk(action, hostname, user, port):
disk_path = action['disk_path']
id_domain = action['domain']
array_out_err = execute_commands(hostname,
ssh_commands=action['ssh_commands'],
user=user,
port=port)
# last ls must fail
if len([k['err'] for k in array_out_err if len(k['err']) == 1]):
log.debug(
'all operations deleting disk {} for domain {} runned ok'.format(
disk_path, id_domain))
def launch_killall_curl(hostname,user,port):
ssh_commands = ['killall curl']
try:
array_out_err = execute_commands(hostname,
ssh_commands=ssh_commands,
user=user,
port=port)
out = array_out_err[0]['out']
err = array_out_err[0]['err']
logs.downloads.info(f'kill al curl process in hypervisor {hostname}: {out} {err}')
return True
except Exception as e:
logs.downloads.error(f'Kill all curl process in hypervisor {hostname} fail: {e}')
def launch_killall_curl(hostname, user, port):
ssh_commands = ['killall curl']
try:
array_out_err = execute_commands(hostname,
ssh_commands=ssh_commands,
user=user,
port=port)
out = array_out_err[0]['out']
err = array_out_err[0]['err']
logs.downloads.info(
f'kill al curl process in hypervisor {hostname}: {out} {err}')
return True
except Exception as e:
logs.downloads.error(
f'Kill all curl process in hypervisor {hostname} fail: {e}')
def launch_delete_media(action, hostname, user, port):
array_out_err = execute_commands(hostname,
ssh_commands=action['ssh_commands'],
user=user,
port=port)
path = action['path']
id_media = action['id_media']
if len([k['err'] for k in array_out_err if len(k['err']) == 0]) != 2:
log.error('failed deleting media {}'.format(id_media))
update_status_media_from_path(path, 'FailedDeleted')
return False
# ls of the file after deleted failed, has deleted ok
elif len(array_out_err[2]['err']) > 0:
update_status_media_from_path(path, 'Deleted')
return True
else:
log.error('failed deleting media {}'.format(id_media))
update_status_media_from_path(path, 'FailedDeleted')
return False
def launch_delete_media(action,hostname,user,port,final_status='Deleted'):
array_out_err = execute_commands(hostname,
ssh_commands=action['ssh_commands'],
user=user,
port=port)
path = action['path']
id_media = action['id_media']
if len([k['err'] for k in array_out_err if len(k['err']) == 0]) != 2:
log.error('failed deleting media {}'.format(id_media))
update_status_media_from_path(path, 'FailedDeleted')
return False
# ls of the file after deleted failed, has deleted ok
elif len(array_out_err[2]['err']) > 0:
if final_status == 'DownloadFailed':
update_status_media_from_path(path, final_status)
else:
update_status_media_from_path(path, 'Deleted')
return True
else:
log.error('failed deleting media {}'.format(id_media))
update_status_media_from_path(path, 'FailedDeleted')
return False
def test_hypers_disk_operations(hyps_disk_operations):
list_hyps_ok = list()
str_random = ''.join(choices(string.ascii_uppercase + string.digits, k=8))
for hyp_id in hyps_disk_operations:
d_hyp = get_hyp_hostname_user_port_from_id(hyp_id)
cmds1 = list()
for pool_id in get_pools_from_hyp(hyp_id):
# test write permissions in root dir of all paths defined in pool
paths = {k: [l['path'] for l in d] for k, d in get_pool(pool_id)['paths'].items()}
for k, p in paths.items():
for path in p:
cmds1.append({'title': f'try create dir if not exists - pool:{pool_id}, hypervisor: {hyp_id}, path_kind: {k}',
'cmd': f'mkdir -p {path}'})
cmds1.append({'title': f'touch random file - pool:{pool_id}, hypervisor: {hyp_id}, path_kind: {k}',
'cmd': f'touch {path}/test_random_{str_random}'})
cmds1.append({'title': 'delete random file - pool:{pool_id}, hypervisor: {hyp_id}, path_kind: {k}',
'cmd': f'rm -f {path}/test_random_{str_random}'})
try:
array_out_err = execute_commands(d_hyp['hostname'],
ssh_commands=cmds1,
dict_mode=True,
user=d_hyp['user'],
port=d_hyp['port'])
#if error in some path hypervisor is not valid
if len([d['err'] for d in array_out_err if len(d['err']) > 0]) > 0:
logs.main.error(f'Hypervisor {hyp_id} can not be disk_operations, some errors when testing if can create files in all paths_')
for d_cmd_err in [d for d in array_out_err if len(d['err']) > 0]:
cmd = d_cmd_err['cmd']
err = d_cmd_err['err']
logs.main.error(f'Command: {cmd} -- Error: {err}')
else:
list_hyps_ok.append(hyp_id)
except Exception as e:
if __name__ == '__main__':
logs.main.err(f'Error when launch commands to test hypervisor {hyp_id} disk_operations: {e}')
return list_hyps_ok
def launch_action_create_template_disk(action, hostname, user, port):
path_template_disk = action['path_template_disk']
path_domain_disk = action['path_domain_disk']
id_domain = action['id_domain']
disk_index = action['disk_index']
cmds1, cmds2, cmds3 = create_cmds_disk_template_from_domain(
path_template_disk, path_domain_disk)
# cmds1: Firsts commands: test if perms, df, files are ok
cmds_done = execute_commands(hostname,
ssh_commands=cmds1,
dict_mode=True,
user=user,
port=port)
error_severity, move_tool, cmd_to_move = verify_output_cmds1_template_from_domain(
cmds_done, path_domain_disk, path_template_disk, id_domain)
if error_severity == None:
# move file
log.debug('commnad to move disk template: {}'.format(cmd_to_move))
if move_tool == 'mv':
cmds_done = execute_commands(hostname,
ssh_commands=[cmd_to_move],
dict_mode=False,
user=user,
port=port)
if move_tool == 'rsync':
execute_command_with_progress(hostname=hostname,
ssh_command=cmd_to_move,
id_domain=id_domain,
user=user,
port=port)
# cmds2: Seconds commands: test if perms, df, files are ok
cmds_done = execute_commands(hostname,
ssh_commands=cmds2,
dict_mode=True,
user=user,
port=port)
error = verify_output_cmds2(cmds_done, path_domain_disk,
path_template_disk, id_domain)
if error is None:
cmds_done = execute_commands(hostname,
ssh_commands=cmds3,
dict_mode=True,
user=user,
port=port)
error, backing_chain_domain, backing_chain_template = verify_output_cmds3(
cmds_done, path_domain_disk, path_template_disk, id_domain)
if error is None:
# update_domain to status: TemplateDiskCreated
##### CREATED OK ######
update_disk_template_created(id_domain, disk_index)
update_disk_backing_chain(
id_domain,
disk_index,
path_template_disk,
backing_chain_template,
new_template=True,
list_backing_chain_template=backing_chain_template)
# disk created, update parents and status
#update_domain_parents(id_domain)
update_domain_status(
status='TemplateDiskCreated',
id_domain=id_domain,
hyp_id=False,
detail=
'new template disk {} for template created from domain {}'.
format(path_template_disk, id_domain))
else:
update_domain_status(
'Crashed',
id_domain,
detail=
'new template disk from domain {} ok, but domain and disk is unknown, details in logs'
.format(id_domain))
else:
if error == 'Crashed':
update_domain_status(
'Crashed',
id_domain,
detail=
'new template from domain {} failed and disk is unknown, details in logs'
.format(id_domain))
else:
update_domain_status(
'Stopped',
id_domain,
detail=
'new template from domain {} failed, disk domain remain in place, details in logs'
.format(id_domain))
else:
update_domain_status(
'Stopped',
id_domain,
detail='new template from domain {} failed, details in logs'.
format(id_domain))
def launch_action_disk(action, hostname, user, port, from_scratch=False):
disk_path = action['disk_path']
id_domain = action['domain']
index_disk = action['index_disk']
array_out_err = execute_commands(hostname,
ssh_commands=action['ssh_commands'],
user=user,
port=port)
if action['type'] in ['create_disk', 'create_disk_from_scratch']:
if len([k['err'] for k in array_out_err if len(k['err']) == 0]):
##TODO: TEST WITH MORE THAN ONE DISK, 2 list_backing_chain must be created
log.debug(
'all operations creating disk {} for new domain {} runned ok'.
format(disk_path, id_domain))
if from_scratch is False:
out_cmd_backing_chain = array_out_err[-1]['out']
list_backing_chain = extract_list_backing_chain(
out_cmd_backing_chain)
if id_domain is not False:
update_domain_parents(id_domain)
update_disk_backing_chain(id_domain, index_disk, disk_path,
list_backing_chain)
##INFO TO DEVELOPER
# ahora ya se puede llamar a starting paused
if id_domain is not False:
#update parents if have
#update_domain_parents(id_domain)
update_domain_status(
'CreatingDomain',
id_domain,
None,
detail=
'new disk created, now go to creating desktop and testing if desktop start'
)
else:
log.error(
'operations creating disk {} for new domain {} failed.'.format(
disk_path, id_domain))
log.error('\n'.join([
'cmd: {} / out: {} / err: {}'.format(action['ssh_commands'][i],
array_out_err[i]['out'],
array_out_err[i]['err'])
for i in range(len(action['ssh_commands']))
]))
if id_domain is not False:
update_domain_status(
'Failed',
id_domain,
detail='new disk create operation failed, details in logs')
elif action['type'] == 'delete_disk':
if len(array_out_err[0]['err']) > 0:
log.error(
'disk from domain {} not found, or permission denied or access to data problems'
.format(id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['err']))
update_domain_status(
'Failed',
id_domain,
detail='delete disk operation failed, disk not found: {}'.
format(array_out_err[0]['err']))
elif len(array_out_err[1]['err']) > 0:
log.error(
'disk from domain {} found, but erase command fail'.format(
id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['err']))
update_domain_status('Failed',
id_domain,
detail='delete disk command failed')
elif len(array_out_err[2]['out']) > 0:
log.error(
'disk from domain {} found, erase command not failed with error message, but disk is in directory.'
.format(id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['out']))
update_domain_status(
'Failed',
id_domain,
detail=
'delete disk operation failed, disk in directory can not erase'
)
else:
log.info('disk {} from domain {} erased'.format(
disk_path, id_domain))
update_disk_backing_chain(id_domain, index_disk, 'DISK_ERASED', [])
update_domain_status('DiskDeleted',
id_domain,
detail='delete disk operation run ok')
def long_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
id_domain = action['domain']
if action['type'] in ['create_disk_virt_builder']:
cmds_done = execute_commands(host=self.hostname,
ssh_commands=action['ssh_commands'],
dict_mode=True,
user=self.user,
port=self.port
)
if len([d for d in cmds_done if len(d['err']) > 0]) > 1:
log.error('some error in virt builder operations')
log.error('Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'.format(
action['disk_path'], action['domain'], self.hyp_id))
log.debug('print cmds_done:')
log.debug(pprint.pprint(cmds_done))
log.debug('print ssh_commands:')
log.debug(pprint.pprint(action['ssh_commands']))
update_domain_status('Failed', id_domain,
detail='Virt Builder Failed creating disk file')
else:
log.info('Disk created from virt-builder. Domain: {} , disk: {}'.format(action['domain'],
action['disk_path']))
xml_virt_install = cmds_done[-1]['out']
update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install)
update_domain_status('CreatingDomainFromBuilder', id_domain,
detail='disk created from virt-builder')
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format(
disk_path, id_domain))
log.error('\n'.join(
['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))]))
update_domain_status('Failed', id_domain,
detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def launch_action_create_template_disk(action, hostname, user, port):
path_template_disk = action['path_template_disk']
path_domain_disk = action['path_domain_disk']
id_domain = action['id_domain']
disk_index = action['disk_index']
cmds1, cmds2, cmds3 = create_cmds_disk_template_from_domain(path_template_disk, path_domain_disk)
# cmds1: Firsts commands: test if perms, df, files are ok
cmds_done = execute_commands(hostname, ssh_commands=cmds1, dict_mode=True, user=user, port=port)
error_severity, move_tool, cmd_to_move = verify_output_cmds1_template_from_domain(cmds_done,
path_domain_disk,
path_template_disk,
id_domain)
if error_severity == None:
# move file
log.debug('commnad to move disk template: {}'.format(cmd_to_move))
if move_tool == 'mv':
cmds_done = execute_commands(hostname, ssh_commands=[cmd_to_move], dict_mode=False, user=user, port=port)
if move_tool == 'rsync':
execute_command_with_progress(hostname=hostname,
ssh_command=cmd_to_move,
id_domain=id_domain,
user=user,
port=port)
# cmds2: Seconds commands: test if perms, df, files are ok
cmds_done = execute_commands(hostname, ssh_commands=cmds2, dict_mode=True, user=user, port=port)
error = verify_output_cmds2(cmds_done, path_domain_disk, path_template_disk, id_domain)
if error is None:
cmds_done = execute_commands(hostname, ssh_commands=cmds3, dict_mode=True, user=user, port=port)
error, backing_chain_domain, backing_chain_template = verify_output_cmds3(cmds_done, path_domain_disk,
path_template_disk, id_domain)
if error is None:
# update_domain to status: TemplateDiskCreated
##### CREATED OK ######
update_disk_template_created(id_domain, disk_index)
update_disk_backing_chain(id_domain,
disk_index,
path_template_disk,
backing_chain_template,
new_template=True,
list_backing_chain_template=backing_chain_template)
# disk created, update parents and status
#update_domain_parents(id_domain)
update_domain_status(status='TemplateDiskCreated',
id_domain=id_domain,
hyp_id=False,
detail='new template disk {} for template created from domain {}'.format(
path_template_disk, id_domain))
else:
update_domain_status('Crashed', id_domain,
detail='new template disk from domain {} ok, but domain and disk is unknown, details in logs'.format(
id_domain))
else:
if error == 'Crashed':
update_domain_status('Crashed', id_domain,
detail='new template from domain {} failed and disk is unknown, details in logs'.format(
id_domain))
else:
update_domain_status('Stopped', id_domain,
detail='new template from domain {} failed, disk domain remain in place, details in logs'.format(
id_domain))
else:
update_domain_status('Stopped', id_domain,
detail='new template from domain {} failed, details in logs'.format(id_domain))
def long_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug(
'Thread to launchdisks operations in host {} with TID: {}...'.
format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
id_domain = action['domain']
if action['type'] in ['create_disk_virt_builder']:
cmds_done = execute_commands(
host=self.hostname,
ssh_commands=action['ssh_commands'],
dict_mode=True,
user=self.user,
port=self.port)
if len([d for d in cmds_done if len(d['err']) > 0]) > 1:
log.error('some error in virt builder operations')
log.error(
'Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'
.format(action['disk_path'], action['domain'],
self.hyp_id))
log.debug('print cmds_done:')
log.debug(pprint.pprint(cmds_done))
log.debug('print ssh_commands:')
log.debug(pprint.pprint(action['ssh_commands']))
update_domain_status(
'Failed',
id_domain,
detail='Virt Builder Failed creating disk file')
else:
log.info(
'Disk created from virt-builder. Domain: {} , disk: {}'
.format(action['domain'], action['disk_path']))
xml_virt_install = cmds_done[-1]['out']
update_table_field('domains', id_domain,
'xml_virt_install',
xml_virt_install)
update_domain_status(
'CreatingDomainFromBuilder',
id_domain,
detail='disk created from virt-builder')
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(
action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'
.format(disk_path, id_domain))
log.error('\n'.join([
'cmd: {}'.format(action['ssh_commands'][i])
for i in range(len(action['ssh_commands']))
]))
update_domain_status(
'Failed',
id_domain,
detail=
'new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs'
)
