def set_domains_coherence(dict_hyps_ready):
for hyp_id, hostname in dict_hyps_ready.items():
hyp_obj = hyp_from_hyp_id(hyp_id)
try:
hyp_obj.get_domains()
except:
log.error('hypervisor {} can not get domains'.format(hyp_id))
update_hyp_status(hyp_id, 'Error')
break
# update domain_status
update_all_domains_status(reset_status='Stopped', from_status=['Starting'])
update_all_domains_status(reset_status='Started', from_status=['Stopping'])
domains_started_in_rethink = get_domains_started_in_hyp(hyp_id)
domains_are_started = []
for domain_name, domain_obj in hyp_obj.domains.items():
domain_state_libvirt = domain_obj.state()
state, reason = state_and_cause_to_str(domain_state_libvirt[0], domain_state_libvirt[1])
status_isard = dict_domain_libvirt_state_to_isard_state[state]
update_domain_status(status=status_isard, id_domain=domain_name, hyp_id=hyp_id, detail=reason)
domains_are_started.append(domain_name)
if len(domains_started_in_rethink) > 0:
domains_are_shutdown = list(set(domains_started_in_rethink).difference(set(domains_are_started)))
for domain_stopped in domains_are_shutdown:
update_domain_status(status='Stopped', id_domain=domain_stopped, hyp_id='')
# TODO INFO TO DEVELOPER: faltaría revisar que ningún dominio está duplicado en y started en dos hypervisores
# a nivel de libvirt, porque a nivel de rethink es imposible, y si pasa poner un erroraco gigante
# a parte de dejarlo en unknown
update_hyp_status(hyp_id, 'ReadyToStart')
def bulk_action_with_domains(action,
groupby,
filterby=None,
quantity=False,
prefix=False,
randomize=False,
template=None):
domain_list = get_domain_list_groupby(groupby, filterby, quantity, randomize)
if action == 'start':
starting_domain_list = []
for domain_id in domain_list:
status = get_domain_status(domain_id)
if status in ['Stopped', 'Failed']:
update_domain_status('Starting')
starting_domain_list.append(domain_id)
return starting_domain_list
def bulk_action_with_domains(action,
groupby,
filterby=None,
quantity=False,
prefix=False,
randomize=False,
template=None):
domain_list = get_domain_list_groupby(groupby, filterby, quantity,
randomize)
if action == 'start':
starting_domain_list = []
for domain_id in domain_list:
status = get_domain_status(domain_id)
if status in ['Stopped', 'Failed']:
update_domain_status('Starting')
starting_domain_list.append(domain_id)
return starting_domain_list
def set_domains_coherence(dict_hyps_ready):
for hyp_id, hostname in dict_hyps_ready.items():
hyp_obj = hyp_from_hyp_id(hyp_id)
try:
hyp_obj.get_domains()
except:
log.error('hypervisor {} can not get domains'.format(hyp_id))
update_hyp_status(hyp_id, 'Error')
break
# update domain_status
update_all_domains_status(reset_status='Stopped',
from_status=['Starting'])
update_all_domains_status(reset_status='Started',
from_status=['Stopping'])
domains_started_in_rethink = get_domains_started_in_hyp(hyp_id)
domains_are_started = []
for domain_name, domain_obj in hyp_obj.domains.items():
domain_state_libvirt = domain_obj.state()
state, reason = state_and_cause_to_str(domain_state_libvirt[0],
domain_state_libvirt[1])
status_isard = dict_domain_libvirt_state_to_isard_state[state]
update_domain_status(status=status_isard,
id_domain=domain_name,
hyp_id=hyp_id,
detail=reason)
domains_are_started.append(domain_name)
if len(domains_started_in_rethink) > 0:
domains_are_shutdown = list(
set(domains_started_in_rethink).difference(
set(domains_are_started)))
for domain_stopped in domains_are_shutdown:
update_domain_status(status='Stopped',
id_domain=domain_stopped,
hyp_id='')
# TODO INFO TO DEVELOPER: faltaría revisar que ningún dominio está duplicado en y started en dos hypervisores
# a nivel de libvirt, porque a nivel de rethink es imposible, y si pasa poner un erroraco gigante
# a parte de dejarlo en unknown
update_hyp_status(hyp_id, 'ReadyToStart')
def update_domains_started_and_stopped(self,domains_with_stats):
if self.id_hyp_rethink is None:
try:
self.id_hyp_rethink = get_id_hyp_from_uri(hostname_to_uri(self.hostname, user=self.user, port=self.port))
except Exception as e:
log.error('error when hypervisor have not rethink id. {}'.format(e))
return False
l_all_domains = get_domains_with_status_in_list(list_status=['Started', 'Stopped', 'Failed'])
for d in l_all_domains:
if d['id'] in domains_with_stats:
if d['status'] == 'Started':
#if status started check if has the same hypervisor
if d['hyp_started'] != self.id_hyp_rethink:
log.error(f"Domain {d['id']} started in hypervisor ({self.id_hyp_rethink}) but database says that is started in {d['hyp_started']} !! ")
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Started in other hypervisor!! {self.id_hyp_rethink}. Updated by status thread',
hyp_id=self.id_hyp_rethink)
else:
#if status is Stopped or Failed update, the domain is started
log.info('Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread')
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread',
hyp_id=self.id_hyp_rethink)
elif d['hyp_started'] == self.id_hyp_rethink:
#Domain is started in this hypervisor in database, but is stopped
if d['status'] == 'Started':
update_domain_status(status='Stopped',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is stopped in {self.id_hyp_rethink} but in database was Started, updated by status thread',
)
def update_domains_started_and_stopped(self,domains_with_stats):
if self.id_hyp_rethink is None:
try:
self.id_hyp_rethink = get_id_hyp_from_uri(hostname_to_uri(self.hostname, user=self.user, port=self.port))
except Exception as e:
log.error('error when hypervisor have not rethink id. {}'.format(e))
return False
l_all_domains = get_domains_with_status_in_list(list_status=['Started', 'Stopped', 'Failed'])
for d in l_all_domains:
if d['id'] in domains_with_stats:
if d['status'] == 'Started':
#if status started check if has the same hypervisor
if d['hyp_started'] != self.id_hyp_rethink:
log.error(f"Domain {d['id']} started in hypervisor ({self.id_hyp_rethink}) but database says that is started in {d['hyp_started']} !! ")
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Started in other hypervisor!! {self.id_hyp_rethink}. Updated by status thread',
hyp_id=self.id_hyp_rethink)
else:
#if status is Stopped or Failed update, the domain is started
log.info('Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread')
update_domain_status(status='Started',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is started in {self.id_hyp_rethink} but in database was Stopped or Failed, updated by status thread',
hyp_id=self.id_hyp_rethink)
elif d['hyp_started'] == self.id_hyp_rethink:
#Domain is started in this hypervisor in database, but is stopped
if d['status'] == 'Started':
update_domain_status(status='Stopped',
id_domain='_admin_downloaded_tetros',
detail=f'Domain is stopped in {self.id_hyp_rethink} but in database was Started, updated by status thread',
)
def analize_backing_chains_outputs(array_out_err=[],
path_to_write_json=None,
path_to_read_json=None):
if path_to_write_json is not None:
f = open(path_to_write_json, 'w')
json.dump(array_out_err, f)
f.close()
if path_to_read_json is not None:
f = open(path_to_read_json, 'r')
array_out_err = json.load(f)
log.debug(len(array_out_err))
f.close()
domains_ok = 0
domains_err = 0
for d in array_out_err:
id = d['title']
if len(d['err']) > 0:
domains_err += 1
log.info(d['err'])
update_domain_status('Failed', id, detail=d['err'])
else:
log.debug(id)
domains_ok += 1
if type(d['out']) is not str:
out = out.decode('utf-8')
else:
out = d['out']
l = json.loads(out)
from pprint import pprint
pprint(l)
update_disk_backing_chain(id, 0, l[0]['filename'], l)
return ({'ok': domains_ok, 'err': domains_err})
def launch_action_disk(action, hostname, user, port, from_scratch=False):
disk_path = action['disk_path']
id_domain = action['domain']
index_disk = action['index_disk']
array_out_err = execute_commands(hostname,
ssh_commands=action['ssh_commands'],
user=user,
port=port)
if action['type'] in ['create_disk', 'create_disk_from_scratch']:
if len([k['err'] for k in array_out_err if len(k['err']) == 0]):
##TODO: TEST WITH MORE THAN ONE DISK, 2 list_backing_chain must be created
log.debug(
'all operations creating disk {} for new domain {} runned ok'.
format(disk_path, id_domain))
if from_scratch is False:
out_cmd_backing_chain = array_out_err[-1]['out']
list_backing_chain = extract_list_backing_chain(
out_cmd_backing_chain)
if id_domain is not False:
update_domain_parents(id_domain)
update_disk_backing_chain(id_domain, index_disk, disk_path,
list_backing_chain)
##INFO TO DEVELOPER
# ahora ya se puede llamar a starting paused
if id_domain is not False:
#update parents if have
#update_domain_parents(id_domain)
update_domain_status(
'CreatingDomain',
id_domain,
None,
detail=
'new disk created, now go to creating desktop and testing if desktop start'
)
else:
log.error(
'operations creating disk {} for new domain {} failed.'.format(
disk_path, id_domain))
log.error('\n'.join([
'cmd: {} / out: {} / err: {}'.format(action['ssh_commands'][i],
array_out_err[i]['out'],
array_out_err[i]['err'])
for i in range(len(action['ssh_commands']))
]))
if id_domain is not False:
update_domain_status(
'Failed',
id_domain,
detail='new disk create operation failed, details in logs')
elif action['type'] == 'delete_disk':
if len(array_out_err[0]['err']) > 0:
log.error(
'disk from domain {} not found, or permission denied or access to data problems'
.format(id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['err']))
update_domain_status(
'Failed',
id_domain,
detail='delete disk operation failed, disk not found: {}'.
format(array_out_err[0]['err']))
elif len(array_out_err[1]['err']) > 0:
log.error(
'disk from domain {} found, but erase command fail'.format(
id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['err']))
update_domain_status('Failed',
id_domain,
detail='delete disk command failed')
elif len(array_out_err[2]['out']) > 0:
log.error(
'disk from domain {} found, erase command not failed with error message, but disk is in directory.'
.format(id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['out']))
update_domain_status(
'Failed',
id_domain,
detail=
'delete disk operation failed, disk in directory can not erase'
)
else:
log.info('disk {} from domain {} erased'.format(
disk_path, id_domain))
update_disk_backing_chain(id_domain, index_disk, 'DISK_ERASED', [])
update_domain_status('DiskDeleted',
id_domain,
detail='delete disk operation run ok')
def run(self):
# if self.table == 'domains':
# type_path_selected = 'groups'
# elif self.table in ['isos']:
# type_path_selected = 'isos'
# else:
# type_path_selected = 'media'
#
# new_file, path_selected = get_path_to_disk(self.path, pool=self.pool, type_path=type_path_selected)
# logs.downloads.debug("PATHS ___________________________________________________________________")
# logs.downloads.debug(new_file)
# logs.downloads.debug(path_selected)
# logs.downloads.debug(pprint.pformat(self.__dict__))
#
# hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=self.pool,
# type_path=type_path_selected)
# hypervisor to launch download command
# wait to threads disk_operations are alive
time_elapsed = 0
path_selected = self.path_selected
while True:
if len(self.manager.t_disk_operations) > 0:
hyp_to_disk_create = get_host_disk_operations_from_path(
path_selected,
pool=self.pool_id,
type_path=self.type_path_selected)
logs.downloads.debug(
f'Thread download started to in hypervisor: {hyp_to_disk_create}'
)
if self.manager.t_disk_operations.get(hyp_to_disk_create,
False) is not False:
if self.manager.t_disk_operations[
hyp_to_disk_create].is_alive():
d = get_hyp_hostname_user_port_from_id(
hyp_to_disk_create)
self.hostname = d['hostname']
self.user = d['user']
self.port = d['port']
break
sleep(0.2)
time_elapsed += 0.2
if time_elapsed > TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD:
logs.downloads.info(
f'Timeout ({TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD} sec) waiting hypervisor online to download {url_base}'
)
if self.table == 'domains':
update_domain_status('DownloadFailed',
self.id,
detail="downloaded disk")
else:
update_status_table(self.table, 'DownloadFailed', self.id)
self.finalished_threads.append(self.path)
return False
header_template = "--header '{header_key}: {header_value}' "
headers = ''
if URL_DOWNLOAD_INSECURE_SSL == True:
insecure_option = '--insecure'
else:
insecure_option = ''
dict_header = {}
for k, v in self.dict_header.items():
headers += header_template.format(header_key=k, header_value=v)
dict_header[k] = v
# TEST IF url return an stream of data
ok, error_msg = test_url_for_download(
self.url,
url_download_insecure_ssl=URL_DOWNLOAD_INSECURE_SSL,
timeout_time_limit=TIMEOUT_WAITING_HYPERVISOR_TO_DOWNLOAD,
dict_header=dict_header)
if ok is False:
logs.downloads.error(f'URL check failed for url: {self.url}')
logs.downloads.error(f'Failed url check reason: {error_msg}')
update_status_table(self.table,
'DownloadFailed',
self.id,
detail=error_msg)
return False
curl_template = "curl {insecure_option} -L -o '{path}' {headers} '{url}'"
ssh_template = """ssh -oBatchMode=yes -p {port} {user}@{hostname} """ \
""" "mkdir -p '{path_dir}'; """ + curl_template + '"'
logs.downloads.debug(ssh_template)
ssh_command = ssh_template.format(port=self.port,
user=self.user,
hostname=self.hostname,
path=self.path,
path_dir=dirname(self.path),
headers=headers,
url=self.url,
insecure_option=insecure_option)
logs.downloads.debug("SSH COMMAND: {}".format(ssh_command))
p = subprocess.Popen(ssh_command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid)
rc = p.poll()
update_status_table(
self.table, 'Downloading', self.id,
"downloading in hypervisor: {}".format(self.hostname))
while rc != 0:
header = p.stderr.readline().decode('utf8')
header2 = p.stderr.readline().decode('utf8')
keys = [
'total_percent', 'total', 'received_percent', 'received',
'xferd_percent', 'xferd', 'speed_download_average',
'speed_upload_average', 'time_total', 'time_spent',
'time_left', 'speed_current'
]
line = ""
while True:
c = p.stderr.read(1).decode('utf8')
if self.stop is True:
curl_cmd = curl_template.format(
path=self.path,
headers=headers,
url=self.url,
insecure_option=insecure_option)
# for pkill curl order is cleaned
curl_cmd = curl_cmd.replace("'", "")
curl_cmd = curl_cmd.replace(" ", " ")
ssh_cmd_kill_curl = """ssh -p {port} {user}@{hostname} "pkill -f \\"^{curl_cmd}\\" " """.format(
port=self.port,
user=self.user,
hostname=self.hostname,
curl_cmd=curl_cmd)
logs.downloads.info(
'download {} aborted, ready to send ssh kill to curl in hypervisor {}'
.format(self.path, self.hostname))
# destroy curl in hypervisor
p_kill_curl = subprocess.Popen(ssh_cmd_kill_curl,
shell=True)
p_kill_curl.wait(timeout=5)
# destroy ssh command
try:
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
except Exception as e:
logs.downloads.debug(
'ssh process not killed, has finalished')
if self.table == 'media':
remove_media(self.id)
if self.table == 'domains':
delete_domain(self.id)
# update_status_table(self.table, 'DownloadFailed', self.id, detail="download aborted")
return False
if not c:
break
if c == '\r':
if len(line) > 60:
values = line.split()
logs.downloads.debug(self.url)
logs.downloads.debug(line)
d_progress = dict(zip(keys, values))
try:
d_progress['total_percent'] = int(
float(d_progress['total_percent']))
d_progress['received_percent'] = int(
float(d_progress['received_percent']))
if d_progress['received_percent'] > 1:
pass
except:
d_progress['total_percent'] = 0
d_progress['received_percent'] = 0
update_download_percent(d_progress, self.table,
self.id)
line = p.stderr.read(60).decode('utf8')
else:
line = line + c
rc = p.poll()
if self.stop is True:
return False
else:
logs.downloads.info('File downloaded: {}'.format(self.path))
assert rc == 0
if self.table == 'domains':
# update_table_field(self.table, self.id, 'path_downloaded', self.path)
d_update_domain = get_domain(self.id)['create_dict']
# d_update_domain = {'hardware': {'disks': [{}]}}
d_update_domain['hardware']['disks'][0]['file'] = self.path
update_domain_dict_create_dict(self.id, d_update_domain)
self.finalished_threads.append(self.path)
update_domain_status('Downloaded',
self.id,
detail="downloaded disk")
update_domain_status('Updating',
self.id,
detail="downloaded disk")
else:
self.finalished_threads.append(self.path)
update_table_field(self.table, self.id, 'path_downloaded',
self.path)
update_status_table(self.table, 'Downloaded', self.id)
def run(self):
self.tid = get_tid()
logs.downloads.debug(
'RUN-DOWNLOAD-THREAD-------------------------------------')
pool_id = 'default'
first_loop = True
if self.stop is False:
if first_loop is True:
# if domains or media have status Downloading when engine restart
# we need to resetdownloading deleting file and
first_loop = False
# wait a hyp to downloads
next_hyp = False
while next_hyp is False:
logs.downloads.info(
'waiting an hypervisor online to launch downloading actions'
)
if pool_id in self.manager.pools.keys():
next_hyp = self.manager.pools[pool_id].get_next()
sleep(1)
for hyp_id in get_hypers_in_pool():
self.killall_curl(hyp_id)
domains_status_downloading = get_domains_with_status(
'Downloading')
medias_status_downloading = get_media_with_status(
'Downloading')
for id_domain in domains_status_downloading:
create_dict = get_domain(id_domain)['create_dict']
dict_changes = {
'id': id_domain,
'table': 'domains',
'create_dict': create_dict
}
update_domain_status('ResetDownloading', id_domain)
self.abort_download(dict_changes,
final_status='DownloadFailed')
for id_media in medias_status_downloading:
dict_media = get_media(id_media)
dict_changes = {
'id': id_media,
'table': 'media',
'path': dict_media['path'],
'hypervisors_pools': dict_media['hypervisors_pools']
}
update_status_table('media', 'ResetDownloading', id_media)
self.abort_download(dict_changes,
final_status='DownloadFailed')
self.r_conn = new_rethink_connection()
update_table_field('hypervisors_pools', pool_id,
'download_changes', 'Started')
for c in r.table('media').get_all(r.args(
['Deleting', 'Deleted', 'Downloaded', 'DownloadFailed', 'DownloadStarting', 'Downloading', 'Download',
'DownloadAborting','ResetDownloading']), index='status'). \
pluck('id',
'path',
'url-isard',
'url-web',
'status'
).merge(
{'table': 'media'}).changes(include_initial=True).union(
r.table('domains').get_all(
r.args(['Downloaded', 'DownloadFailed','DownloadStarting', 'Downloading', 'DownloadAborting','ResetDownloading']), index='status'). \
pluck('id',
'create_dict',
'url-isard',
'url-web',
'status').merge(
{"table": "domains"}).changes(include_initial=True)).union(
r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).run(
self.r_conn):
if self.stop:
break
if c.get('new_val', None) is not None:
if c['new_val'].get('table', False) == 'engine':
if c['new_val']['status_all_threads'] == 'Stopping':
break
else:
continue
logs.downloads.debug('DOWNLOAD CHANGES DETECTED:')
logs.downloads.debug(pprint.pformat(c))
if c.get('old_val', None) is None:
if c['new_val']['status'] == 'DownloadStarting':
self.start_download(c['new_val'])
elif c.get('new_val', None) is None:
if c['old_val']['status'] in ['DownloadAborting']:
self.remove_download_thread(c['old_val'])
elif 'old_val' in c and 'new_val' in c:
if c['old_val']['status'] == 'DownloadFailed' and c[
'new_val']['status'] == 'DownloadStarting':
self.start_download(c['new_val'])
elif c['old_val']['status'] == 'Downloaded' and c[
'new_val']['status'] == 'Deleting':
if c['new_val']['table'] == 'media':
self.delete_media(c['new_val'])
elif c['old_val']['status'] == 'Deleting' and c['new_val'][
'status'] == 'Deleted':
if c['new_val']['table'] == 'media':
remove_media(c['new_val']['id'])
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'DownloadFailed':
pass
elif c['old_val']['status'] == 'DownloadStarting' and c[
'new_val']['status'] == 'Downloading':
pass
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'Downloaded':
pass
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'DownloadAborting':
self.abort_download(c['new_val'])
elif c['old_val']['status'] == 'Downloading' and c[
'new_val']['status'] == 'ResetDownloading':
self.abort_download(c['new_val'],
final_status='DownloadFailed')
def launch_action_create_template_disk(action, hostname, user, port):
path_template_disk = action['path_template_disk']
path_domain_disk = action['path_domain_disk']
id_domain = action['id_domain']
disk_index = action['disk_index']
cmds1, cmds2, cmds3 = create_cmds_disk_template_from_domain(path_template_disk, path_domain_disk)
# cmds1: Firsts commands: test if perms, df, files are ok
cmds_done = execute_commands(hostname, ssh_commands=cmds1, dict_mode=True, user=user, port=port)
error_severity, move_tool, cmd_to_move = verify_output_cmds1_template_from_domain(cmds_done,
path_domain_disk,
path_template_disk,
id_domain)
if error_severity == None:
# move file
log.debug('commnad to move disk template: {}'.format(cmd_to_move))
if move_tool == 'mv':
cmds_done = execute_commands(hostname, ssh_commands=[cmd_to_move], dict_mode=False, user=user, port=port)
if move_tool == 'rsync':
execute_command_with_progress(hostname=hostname,
ssh_command=cmd_to_move,
id_domain=id_domain,
user=user,
port=port)
# cmds2: Seconds commands: test if perms, df, files are ok
cmds_done = execute_commands(hostname, ssh_commands=cmds2, dict_mode=True, user=user, port=port)
error = verify_output_cmds2(cmds_done, path_domain_disk, path_template_disk, id_domain)
if error is None:
cmds_done = execute_commands(hostname, ssh_commands=cmds3, dict_mode=True, user=user, port=port)
error, backing_chain_domain, backing_chain_template = verify_output_cmds3(cmds_done, path_domain_disk,
path_template_disk, id_domain)
if error is None:
# update_domain to status: TemplateDiskCreated
##### CREATED OK ######
update_disk_template_created(id_domain, disk_index)
update_disk_backing_chain(id_domain,
disk_index,
path_template_disk,
backing_chain_template,
new_template=True,
list_backing_chain_template=backing_chain_template)
# disk created, update parents and status
#update_domain_parents(id_domain)
update_domain_status(status='TemplateDiskCreated',
id_domain=id_domain,
hyp_id=False,
detail='new template disk {} for template created from domain {}'.format(
path_template_disk, id_domain))
else:
update_domain_status('Crashed', id_domain,
detail='new template disk from domain {} ok, but domain and disk is unknown, details in logs'.format(
id_domain))
else:
if error == 'Crashed':
update_domain_status('Crashed', id_domain,
detail='new template from domain {} failed and disk is unknown, details in logs'.format(
id_domain))
else:
update_domain_status('Stopped', id_domain,
detail='new template from domain {} failed, disk domain remain in place, details in logs'.format(
id_domain))
else:
update_domain_status('Stopped', id_domain,
detail='new template from domain {} failed, details in logs'.format(id_domain))
def launch_action_disk(action, hostname, user, port, from_scratch=False):
disk_path = action['disk_path']
id_domain = action['domain']
index_disk = action['index_disk']
array_out_err = execute_commands(hostname,
ssh_commands=action['ssh_commands'],
user=user,
port=port)
if action['type'] in ['create_disk', 'create_disk_from_scratch']:
if len([k['err'] for k in array_out_err if len(k['err']) == 0]):
##TODO: TEST WITH MORE THAN ONE DISK, 2 list_backing_chain must be created
log.debug('all operations creating disk {} for new domain {} runned ok'.format(disk_path, id_domain))
if from_scratch is False:
out_cmd_backing_chain = array_out_err[-1]['out']
list_backing_chain = extract_list_backing_chain(out_cmd_backing_chain)
if id_domain is not False:
update_domain_parents(id_domain)
update_disk_backing_chain(id_domain, index_disk, disk_path, list_backing_chain)
##INFO TO DEVELOPER
# ahora ya se puede llamar a starting paused
if id_domain is not False:
#update parents if have
#update_domain_parents(id_domain)
update_domain_status('CreatingDomain', id_domain, None,
detail='new disk created, now go to creating desktop and testing if desktop start')
else:
log.error('operations creating disk {} for new domain {} failed.'.format(disk_path, id_domain))
log.error('\n'.join(['cmd: {} / out: {} / err: {}'.format(action['ssh_commands'][i],
array_out_err[i]['out'],
array_out_err[i]['err']) for i in
range(len(action['ssh_commands']))]))
if id_domain is not False:
update_domain_status('Failed', id_domain, detail='new disk create operation failed, details in logs')
elif action['type'] == 'delete_disk':
if len(array_out_err[0]['err']) > 0:
log.error(
'disk from domain {} not found, or permission denied or access to data problems'.format(id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['err']))
update_domain_status('DiskDeleted', id_domain, detail='delete disk operation failed, disk not found: {}'.format(
array_out_err[0]['err']))
elif len(array_out_err[1]['err']) > 0:
log.error('disk from domain {} found, but erase command fail'.format(id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['err']))
update_domain_status('DiskDeleted', id_domain, detail='delete disk command failed')
elif len(array_out_err[2]['out']) > 0:
log.error(
'disk from domain {} found, erase command not failed with error message, but disk is in directory.'.format(
id_domain))
log.error('ERROR: {}'.format(array_out_err[0]['out']))
update_domain_status('DiskDeleted', id_domain,
detail='delete disk operation failed, disk in directory can not erase')
else:
log.info('disk {} from domain {} erased'.format(disk_path, id_domain))
update_disk_backing_chain(id_domain, index_disk, 'DISK_ERASED', [])
update_domain_status('DiskDeleted', id_domain, detail='delete disk operation run ok')
def run(self):
# if self.table == 'domains':
# type_path_selected = 'groups'
# elif self.table in ['isos']:
# type_path_selected = 'isos'
# else:
# type_path_selected = 'media'
#
# new_file, path_selected = get_path_to_disk(self.path, pool=self.pool, type_path=type_path_selected)
# logs.downloads.debug("PATHS ___________________________________________________________________")
# logs.downloads.debug(new_file)
# logs.downloads.debug(path_selected)
# logs.downloads.debug(pprint.pformat(self.__dict__))
#
# hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=self.pool,
# type_path=type_path_selected)
header_template = "--header '{header_key}: {header_value}' "
headers = ''
if URL_DOWNLOAD_INSECURE_SSL == True:
insecure_option = '--insecure'
else:
insecure_option = ''
for k, v in self.dict_header.items():
headers += header_template.format(header_key=k, header_value=v)
curl_template = "curl {insecure_option} -L -o '{path}' {headers} '{url}'"
ssh_template = """ssh -oBatchMode=yes -p {port} {user}@{hostname} """ \
""" "mkdir -p '{path_dir}'; """ + curl_template + '"'
logs.downloads.debug(ssh_template)
ssh_command = ssh_template.format(port=self.port,
user=self.user,
hostname=self.hostname,
path=self.path,
path_dir=dirname(self.path),
headers=headers,
url=self.url,
insecure_option=insecure_option)
logs.downloads.debug("SSH COMMAND: {}".format(ssh_command))
p = subprocess.Popen(ssh_command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid)
rc = p.poll()
update_status_table(
self.table, 'Downloading', self.id,
"downloading in hypervisor: {}".format(self.hostname))
while rc != 0:
header = p.stderr.readline().decode('utf8')
header2 = p.stderr.readline().decode('utf8')
keys = [
'total_percent', 'total', 'received_percent', 'received',
'xferd_percent', 'xferd', 'speed_download_average',
'speed_upload_average', 'time_total', 'time_spent',
'time_left', 'speed_current'
]
line = ""
while True:
c = p.stderr.read(1).decode('utf8')
if self.stop is True:
curl_cmd = curl_template.format(
path=self.path,
headers=headers,
url=self.url,
insecure_option=insecure_option)
#for pkill curl order is cleaned
curl_cmd = curl_cmd.replace("'", "")
curl_cmd = curl_cmd.replace(" ", " ")
ssh_cmd_kill_curl = """ssh -p {port} {user}@{hostname} "pkill -f \\"^{curl_cmd}\\" " """.format(
port=self.port,
user=self.user,
hostname=self.hostname,
curl_cmd=curl_cmd)
logs.downloads.info(
'download {} aborted, ready to send ssh kill to curl in hypervisor {}'
.format(self.path, self.hostname))
#destroy curl in hypervisor
p_kill_curl = subprocess.Popen(ssh_cmd_kill_curl,
shell=True)
p_kill_curl.wait(timeout=5)
#destroy ssh command
try:
os.killpg(os.getpgid(p.pid), signal.SIGTERM)
except Exception as e:
logs.downloads.debug(
'ssh process not killed, has finalished')
if self.table == 'media':
remove_media(self.id)
if self.table == 'domains':
delete_domain(self.id)
#update_status_table(self.table, 'FailedDownload', self.id, detail="download aborted")
return False
if not c:
break
if c == '\r':
if len(line) > 60:
logs.downloads.debug(line)
values = line.split()
logs.downloads.debug(self.url)
logs.downloads.debug(line)
d_progress = dict(zip(keys, values))
d_progress['total_percent'] = int(
float(d_progress['total_percent']))
d_progress['received_percent'] = int(
float(d_progress['received_percent']))
update_download_percent(d_progress, self.table,
self.id)
line = p.stderr.read(60).decode('utf8')
else:
line = line + c
rc = p.poll()
if self.stop is True:
return False
else:
logs.downloads.info('File downloaded: {}'.format(self.path))
assert rc == 0
if self.table == 'domains':
#update_table_field(self.table, self.id, 'path_downloaded', self.path)
d_update_domain = get_domain(self.id)['create_dict']
#d_update_domain = {'hardware': {'disks': [{}]}}
d_update_domain['hardware']['disks'][0]['file'] = self.path
update_domain_dict_create_dict(self.id, d_update_domain)
self.finalished_threads.append(self.path)
update_domain_status('Downloaded',
self.id,
detail="downloaded disk")
update_domain_status('Updating',
self.id,
detail="downloaded disk")
else:
self.finalished_threads.append(self.path)
update_table_field(self.table, self.id, 'path_downloaded',
self.path)
update_status_table(self.table, 'Downloaded', self.id)
def run(self):
self.tid = get_tid()
logs.changes.info('starting thread: {} (TID {})'.format(self.name, self.tid))
logs.changes.debug('^^^^^^^^^^^^^^^^^^^ DOMAIN CHANGES THREAD ^^^^^^^^^^^^^^^^^')
ui = UiActions(self.manager)
self.r_conn = new_rethink_connection()
cursor = r.table('domains').pluck('id', 'kind', 'status', 'detail').merge({'table': 'domains'}).changes().\
union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).\
run(self.r_conn)
for c in cursor:
if self.stop is True:
break
if c.get('new_val', None) is not None:
if c['new_val']['table'] == 'engine':
if c['new_val']['status_all_threads'] == 'Stopping':
break
else:
continue
logs.changes.debug('domain changes detected in main thread')
detail_msg_if_no_hyps_online = 'No hypervisors Online in pool'
if self.manager.check_actions_domains_enabled() is False:
if c.get('new_val', None) is not None:
if c.get('old_val', None) is not None:
if c['new_val']['status'][-3:] == 'ing':
update_domain_status(c['old_val']['status'], c['old_val']['id'], detail=detail_msg_if_no_hyps_online)
#if no hypervisor availables no check status changes
continue
new_domain = False
new_status = False
old_status = False
logs.changes.debug(pprint.pformat(c))
# action deleted
if c.get('new_val', None) is None:
pass
# action created
if c.get('old_val', None) is None:
new_domain = True
new_status = c['new_val']['status']
domain_id = c['new_val']['id']
logs.changes.debug('domain_id: {}'.format(new_domain))
# if engine is stopped/restarting or not hypervisors online
if self.manager.check_actions_domains_enabled() is False:
continue
pass
if c.get('new_val', None) is not None and c.get('old_val', None) is not None:
old_status = c['old_val']['status']
new_status = c['new_val']['status']
new_detail = c['new_val']['detail']
domain_id = c['new_val']['id']
logs.changes.debug('domain_id: {}'.format(domain_id))
# if engine is stopped/restarting or not hypervisors online
if old_status != new_status:
# print('&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'.format(domain_id,old_status,new_status, new_detail))
# if new_status[-3:] == 'ing':
if 1 > 0:
date_now = datetime.now()
update_domain_history_from_id_domain(domain_id, new_status, new_detail, date_now)
else:
# print('&&&&&&&ESTADOS IGUALES OJO &&&&&&&\n&&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'.
# format(domain_id,old_status,new_status,new_detail))
pass
if (new_domain is True and new_status == "CreatingDiskFromScratch") or \
(old_status == 'FailedCreatingDomain' and new_status == "CreatingDiskFromScratch"):
ui.creating_disk_from_scratch(domain_id)
if (new_domain is True and new_status == "Creating") or \
(old_status == 'FailedCreatingDomain' and new_status == "Creating"):
ui.creating_disks_from_template(domain_id)
if (new_domain is True and new_status == "CreatingAndStarting"):
update_domain_start_after_created(domain_id)
ui.creating_disks_from_template(domain_id)
# INFO TO DEVELOPER
# recoger template de la que hay que derivar
# verificar que realmente es una template
# hay que recoger ram?? cpu?? o si no hay nada copiamos de la template??
if (new_domain is True and new_status == "CreatingFromBuilder") or \
(old_status == 'FailedCreatingDomain' and new_status == "CreatingFromBuilder"):
ui.creating_disk_from_virtbuilder(domain_id)
if (old_status in ['CreatingDisk','CreatingDiskFromScratch'] and new_status == "CreatingDomain") or \
(old_status == 'RunningVirtBuilder' and new_status == "CreatingDomainFromBuilder"):
logs.changes.debug('llamo a creating_and_test_xml con domain id {}'.format(domain_id))
if new_status == "CreatingDomainFromBuilder":
ui.creating_and_test_xml_start(domain_id,
creating_from_create_dict=True,
xml_from_virt_install=True,ssl=True)
if new_status == "CreatingDomain":
ui.creating_and_test_xml_start(domain_id,
creating_from_create_dict=True,ssl=True)
if old_status == 'Stopped' and new_status == "CreatingTemplate":
ui.create_template_disks_from_domain(domain_id)
if old_status != 'Started' and new_status == "Deleting":
# or \
# old_status == 'Failed' and new_status == "Deleting" or \
# old_status == 'Downloaded' and new_status == "Deleting":
ui.deleting_disks_from_domain(domain_id)
if (old_status == 'Stopped' and new_status == "Updating") or \
(old_status == 'Downloaded' and new_status == "Updating"):
ui.updating_from_create_dict(domain_id,ssl=True)
if old_status == 'DeletingDomainDisk' and new_status == "DiskDeleted":
logs.changes.debug('disk deleted, mow remove domain form database')
remove_domain(domain_id)
if get_domain(domain_id) is None:
logs.changes.info('domain {} deleted from database'.format(domain_id))
else:
update_domain_status('Failed', domain_id,
detail='domain {} can not be deleted from database'.format(domain_id))
if old_status == 'CreatingTemplateDisk' and new_status == "TemplateDiskCreated":
# create_template_from_dict(dict_new_template)
if get_if_all_disk_template_created(domain_id):
ui.create_template_in_db(domain_id)
else:
# INFO TO DEVELOPER, este else no se si tiene mucho sentido, hay que hacer pruebas con la
# creación de una template con dos discos y ver si pasa por aquí
# waiting to create other disks
update_domain_status(status='CreatingTemplateDisk',
id_domain=domain_id,
hyp_id=False,
detail='Waiting to create more disks for template')
if (old_status == 'Stopped' and new_status == "Starting") or \
(old_status == 'Failed' and new_status == "Starting"):
ui.start_domain_from_id(id=domain_id, ssl=True)
if (old_status == 'Started' and new_status == "Stopping" ) or \
(old_status == 'Suspended' and new_status == "Stopping" ):
# INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started
# ui.stop_domain_from_id(id=domain_id)
hyp_started = get_domain_hyp_started(domain_id)
if hyp_started:
ui.stop_domain(id_domain=domain_id, hyp_id=hyp_started)
else:
logs.main.error('domain without hyp_started can not be stopped: {}. '.format(domain_id))
if (old_status == 'Started' and new_status == "StoppingAndDeleting" ) or \
(old_status == 'Suspended' and new_status == "StoppingAndDeleting" ):
# INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started
# ui.stop_domain_from_id(id=domain_id)
hyp_started = get_domain_hyp_started(domain_id)
print(hyp_started)
ui.stop_domain(id_domain=domain_id, hyp_id=hyp_started, delete_after_stopped=True)
logs.main.info('finalished thread domain changes')
def launch_action_create_template_disk(action, hostname, user, port):
path_template_disk = action['path_template_disk']
path_domain_disk = action['path_domain_disk']
id_domain = action['id_domain']
disk_index = action['disk_index']
cmds1, cmds2, cmds3 = create_cmds_disk_template_from_domain(
path_template_disk, path_domain_disk)
# cmds1: Firsts commands: test if perms, df, files are ok
cmds_done = execute_commands(hostname,
ssh_commands=cmds1,
dict_mode=True,
user=user,
port=port)
error_severity, move_tool, cmd_to_move = verify_output_cmds1_template_from_domain(
cmds_done, path_domain_disk, path_template_disk, id_domain)
if error_severity == None:
# move file
log.debug('commnad to move disk template: {}'.format(cmd_to_move))
if move_tool == 'mv':
cmds_done = execute_commands(hostname,
ssh_commands=[cmd_to_move],
dict_mode=False,
user=user,
port=port)
if move_tool == 'rsync':
execute_command_with_progress(hostname=hostname,
ssh_command=cmd_to_move,
id_domain=id_domain,
user=user,
port=port)
# cmds2: Seconds commands: test if perms, df, files are ok
cmds_done = execute_commands(hostname,
ssh_commands=cmds2,
dict_mode=True,
user=user,
port=port)
error = verify_output_cmds2(cmds_done, path_domain_disk,
path_template_disk, id_domain)
if error is None:
cmds_done = execute_commands(hostname,
ssh_commands=cmds3,
dict_mode=True,
user=user,
port=port)
error, backing_chain_domain, backing_chain_template = verify_output_cmds3(
cmds_done, path_domain_disk, path_template_disk, id_domain)
if error is None:
# update_domain to status: TemplateDiskCreated
##### CREATED OK ######
update_disk_template_created(id_domain, disk_index)
update_disk_backing_chain(
id_domain,
disk_index,
path_template_disk,
backing_chain_template,
new_template=True,
list_backing_chain_template=backing_chain_template)
# disk created, update parents and status
#update_domain_parents(id_domain)
update_domain_status(
status='TemplateDiskCreated',
id_domain=id_domain,
hyp_id=False,
detail=
'new template disk {} for template created from domain {}'.
format(path_template_disk, id_domain))
else:
update_domain_status(
'Crashed',
id_domain,
detail=
'new template disk from domain {} ok, but domain and disk is unknown, details in logs'
.format(id_domain))
else:
if error == 'Crashed':
update_domain_status(
'Crashed',
id_domain,
detail=
'new template from domain {} failed and disk is unknown, details in logs'
.format(id_domain))
else:
update_domain_status(
'Stopped',
id_domain,
detail=
'new template from domain {} failed, disk domain remain in place, details in logs'
.format(id_domain))
else:
update_domain_status(
'Stopped',
id_domain,
detail='new template from domain {} failed, details in logs'.
format(id_domain))
def run(self):
self.tid = get_tid()
logs.changes.info('starting thread: {} (TID {})'.format(
self.name, self.tid))
logs.changes.debug(
'^^^^^^^^^^^^^^^^^^^ DOMAIN CHANGES THREAD ^^^^^^^^^^^^^^^^^')
ui = UiActions(self.manager)
self.r_conn = new_rethink_connection()
cursor = r.table('domains').pluck('id', 'kind', 'status', 'detail').merge({'table': 'domains'}).changes().\
union(r.table('engine').pluck('threads', 'status_all_threads').merge({'table': 'engine'}).changes()).\
run(self.r_conn)
for c in cursor:
if self.stop is True:
break
if c.get('new_val', None) is not None:
if c['new_val']['table'] == 'engine':
if c['new_val']['status_all_threads'] == 'Stopping':
break
else:
continue
logs.changes.debug('domain changes detected in main thread')
detail_msg_if_no_hyps_online = 'No hypervisors Online in pool'
if self.manager.check_actions_domains_enabled() is False:
if c.get('new_val', None) is not None:
if c.get('old_val', None) is not None:
if c['new_val']['status'][-3:] == 'ing':
update_domain_status(
c['old_val']['status'],
c['old_val']['id'],
detail=detail_msg_if_no_hyps_online)
#if no hypervisor availables no check status changes
continue
new_domain = False
new_status = False
old_status = False
import pprint
logs.changes.debug(pprint.pformat(c))
# action deleted
if c.get('new_val', None) is None:
pass
# action created
if c.get('old_val', None) is None:
new_domain = True
new_status = c['new_val']['status']
domain_id = c['new_val']['id']
logs.changes.debug('domain_id: {}'.format(new_domain))
# if engine is stopped/restarting or not hypervisors online
if self.manager.check_actions_domains_enabled() is False:
continue
pass
if c.get('new_val', None) is not None and c.get(
'old_val', None) is not None:
old_status = c['old_val']['status']
new_status = c['new_val']['status']
new_detail = c['new_val']['detail']
domain_id = c['new_val']['id']
logs.changes.debug('domain_id: {}'.format(domain_id))
# if engine is stopped/restarting or not hypervisors online
if old_status != new_status:
# print('&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'.format(domain_id,old_status,new_status, new_detail))
# if new_status[-3:] == 'ing':
if 1 > 0:
date_now = datetime.now()
update_domain_history_from_id_domain(
domain_id, new_status, new_detail, date_now)
else:
# print('&&&&&&&ESTADOS IGUALES OJO &&&&&&&\n&&&&&&&& ID DOMAIN {} - old_status: {} , new_status: {}, detail: {}'.
# format(domain_id,old_status,new_status,new_detail))
pass
if (new_domain is True and new_status == "CreatingDiskFromScratch") or \
(old_status == 'FailedCreatingDomain' and new_status == "CreatingDiskFromScratch"):
ui.creating_disk_from_scratch(domain_id)
if (new_domain is True and new_status == "Creating") or \
(old_status == 'FailedCreatingDomain' and new_status == "Creating"):
ui.creating_disks_from_template(domain_id)
if (new_domain is True
and new_status == "CreatingAndStarting"):
update_domain_start_after_created(domain_id)
ui.creating_disks_from_template(domain_id)
# INFO TO DEVELOPER
# recoger template de la que hay que derivar
# verificar que realmente es una template
# hay que recoger ram?? cpu?? o si no hay nada copiamos de la template??
if (new_domain is True and new_status == "CreatingFromBuilder") or \
(old_status == 'FailedCreatingDomain' and new_status == "CreatingFromBuilder"):
ui.creating_disk_from_virtbuilder(domain_id)
if (old_status in ['CreatingDisk','CreatingDiskFromScratch'] and new_status == "CreatingDomain") or \
(old_status == 'RunningVirtBuilder' and new_status == "CreatingDomainFromBuilder"):
logs.changes.debug(
'llamo a creating_and_test_xml con domain id {}'.
format(domain_id))
if new_status == "CreatingDomainFromBuilder":
ui.creating_and_test_xml_start(
domain_id,
creating_from_create_dict=True,
xml_from_virt_install=True)
if new_status == "CreatingDomain":
ui.creating_and_test_xml_start(
domain_id, creating_from_create_dict=True)
if old_status == 'Stopped' and new_status == "CreatingTemplate":
ui.create_template_disks_from_domain(domain_id)
if old_status == 'Stopped' and new_status == "Deleting":
ui.deleting_disks_from_domain(domain_id)
if (old_status == 'Stopped' and new_status == "Updating") or \
(old_status == 'Downloaded' and new_status == "Updating"):
ui.updating_from_create_dict(domain_id)
if old_status == 'DeletingDomainDisk' and new_status == "DiskDeleted":
logs.changes.debug(
'disk deleted, mow remove domain form database')
remove_domain(domain_id)
if get_domain(domain_id) is None:
logs.changes.info(
'domain {} deleted from database'.format(
domain_id))
else:
update_domain_status(
'Failed',
id_domain,
detail='domain {} can not be deleted from database'
.format(domain_id))
if old_status == 'CreatingTemplateDisk' and new_status == "TemplateDiskCreated":
# create_template_from_dict(dict_new_template)
if get_if_all_disk_template_created(domain_id):
ui.create_template_in_db(domain_id)
else:
# INFO TO DEVELOPER, este else no se si tiene mucho sentido, hay que hacer pruebas con la
# creación de una template con dos discos y ver si pasa por aquí
# waiting to create other disks
update_domain_status(
status='CreatingTemplateDisk',
id_domain=domain_id,
hyp_id=False,
detail='Waiting to create more disks for template')
if (old_status == 'Stopped' and new_status == "Starting") or \
(old_status == 'Failed' and new_status == "Starting"):
ui.start_domain_from_id(id=domain_id, ssl=True)
if (old_status == 'Started' and new_status == "Stopping" ) or \
(old_status == 'Suspended' and new_status == "Stopping" ):
# INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started
# ui.stop_domain_from_id(id=domain_id)
hyp_started = get_domain_hyp_started(domain_id)
if hyp_started:
ui.stop_domain(id_domain=domain_id, hyp_id=hyp_started)
else:
logs.main.error(
'domain without hyp_started can not be stopped: {}. '
.format(domain_id))
if (old_status == 'Started' and new_status == "StoppingAndDeleting" ) or \
(old_status == 'Suspended' and new_status == "StoppingAndDeleting" ):
# INFO TO DEVELOPER Esto es lo que debería ser, pero hay líos con el hyp_started
# ui.stop_domain_from_id(id=domain_id)
hyp_started = get_domain_hyp_started(domain_id)
print(hyp_started)
ui.stop_domain(id_domain=domain_id,
hyp_id=hyp_started,
delete_after_stopped=True)
logs.main.info('finalished thread domain changes')
def run(self):
# if self.table == 'domains':
# type_path_selected = 'groups'
# elif self.table in ['isos']:
# type_path_selected = 'isos'
# else:
# type_path_selected = 'media'
#
# new_file, path_selected = get_path_to_disk(self.path, pool=self.pool, type_path=type_path_selected)
# logs.downloads.debug("PATHS ___________________________________________________________________")
# logs.downloads.debug(new_file)
# logs.downloads.debug(path_selected)
# logs.downloads.debug(pprint.pformat(self.__dict__))
#
# hyp_to_disk_create = get_host_disk_operations_from_path(path_selected, pool=self.pool,
# type_path=type_path_selected)
header_template = "--header '{header_key}: {header_value}' "
headers = ''
for k, v in self.dict_header.items():
headers += header_template.format(header_key=k, header_value=v)
ssh_template = """ssh -oBatchMode=yes -p {port} {user}@{hostname} """ \
""" "mkdir -p '{path_dir}'; curl -L -o '{path}' {headers} '{url}' " """
print(ssh_template)
ssh_command = ssh_template.format(port=self.port,
user=self.user,
hostname=self.hostname,
path=self.path,
path_dir=dirname(self.path),
headers=headers,
url=self.url)
print(ssh_command)
logs.downloads.debug("SSH COMMAND: {}".format(ssh_command))
p = subprocess.Popen(ssh_command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
rc = p.poll()
update_status_table(
self.table, 'Downloading', self.id,
"downloading in hypervisor: {}".format(self.hostname))
while rc != 0:
header = p.stderr.readline().decode('utf8')
header2 = p.stderr.readline().decode('utf8')
keys = [
'total_percent', 'total', 'received_percent', 'received',
'xferd_percent', 'xferd', 'speed_download_average',
'speed_upload_average', 'time_total', 'time_spent',
'time_left', 'speed_current'
]
line = ""
while True:
c = p.stderr.read(1).decode('utf8')
if self.stop is True:
update_domain_status(self.table,
'FailedDownload',
id,
detail="download aborted")
break
if not c:
break
if c == '\r':
if len(line) > 60:
logs.downloads.debug(line)
values = line.split()
print(self.url)
print(line)
d_progress = dict(zip(keys, values))
d_progress['total_percent'] = int(
float(d_progress['total_percent']))
d_progress['received_percent'] = int(
float(d_progress['received_percent']))
update_download_percent(d_progress, self.table,
self.id)
line = p.stderr.read(60).decode('utf8')
else:
line = line + c
rc = p.poll()
logs.downloads.info('File downloaded: {}'.format(self.path))
assert rc == 0
if self.table == 'domains':
#update_table_field(self.table, self.id, 'path_downloaded', self.path)
d_update_domain = get_domain(self.id)['create_dict']
#d_update_domain = {'hardware': {'disks': [{}]}}
d_update_domain['hardware']['disks'][0]['file'] = self.path
update_domain_dict_create_dict(self.id, d_update_domain)
self.finalished_threads.append(self.path)
update_domain_status('Downloaded',
self.id,
detail="downloaded disk")
update_domain_status('Updating', self.id, detail="downloaded disk")
else:
self.finalished_threads.append(self.path)
update_table_field(self.table, self.id, 'path_downloaded',
self.path)
update_status_table(self.table, 'Downloaded', self.id)
