def update_output(self, out):
self.out_process = out
# log.debug(out)
tmp = out[:str(out).find('%')]
percent = tmp[tmp.rfind(' ') + 1:]
self.percent = percent
log.debug('percent: ' + percent + '% \r')
def analize_backing_chains_outputs(array_out_err=[], path_to_write_json=None, path_to_read_json=None):
if path_to_write_json is not None:
f = open(path_to_write_json, 'w')
json.dump(array_out_err, f)
f.close()
if path_to_read_json is not None:
f = open(path_to_read_json, 'r')
array_out_err = json.load(f)
log.debug(len(array_out_err))
f.close()
domains_ok = 0
domains_err = 0
for d in array_out_err:
id = d['title']
if len(d['err']) > 0:
domains_err += 1
log.info(d['err'])
update_domain_status('Failed', id, detail=d['err'])
else:
log.debug(id)
domains_ok += 1
if type(d['out']) is not str:
out = out.decode('utf-8')
else:
out = d['out']
l = json.loads(out)
from pprint import pprint
pprint(l)
update_disk_backing_chain(id, 0, l[0]['filename'], l)
return ({'ok': domains_ok, 'err': domains_err})
def exec_remote_updating_progress(command, hostname, progress=[], username='******', port=22, sudo=False, id_domain=None):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostname, port=port, username=username)
stdin, stdout, stderr = client.exec_command(command)
progress.append(0)
# INFO TO DEVELOPER, AQUÍ FALTA ACTUALIZAR EL PROGRESO A LA BASE DE DATOS
# A PARTIR DE LA LISTA PROGRESS, COGIENDO EL ÚLTIMO PARÁMETRO
# SE PODRÍA IR MIRANDO, AUNQUE LO SUYO ES QUE SE ACTUALIZASE AQUÍ??
log.debug('jujuju {} '.format(type(stdout)))
while True:
# out = stdout.readline(64).decode('utf-8')
out = stdout.readline(64)
if out.find('%') >= 0:
tmp = out[:str(out).find('%')]
percent = tmp[tmp.rfind(' ') + 1:]
if len(percent) > 0:
if percent.isdigit():
percent = int(percent)
if progress[-1] < percent:
progress.append(percent)
if id_domain is not None:
update_domain_progress(id_domain, percent)
log.debug(out)
if out == '':
break
out = stdout.read()
err = stderr.read()
client.close()
return {'out': out, 'err': err}
def update_output(self, out):
self.out_process = out
# log.debug(out)
tmp = out[:str(out).find('%')]
percent = tmp[tmp.rfind(' ') + 1:]
self.percent = percent
log.debug('percent: ' + percent + '% \r')
def exec_remote_list_of_cmds_dict(hostname,
list_dict_commands,
username='******',
port=22,
ssh_key_str='',
sudo=False):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if len(ssh_key_str) > 0:
# TODO: make ssh_key login
pass
else:
client.connect(hostname, port=port, username=username)
returned_array = list_dict_commands.copy()
i = 0
for command in list_dict_commands:
log.debug('command to launch in ssh in {}: {}'.format(
hostname, command['cmd']))
stdin, stdout, stderr = client.exec_command(command['cmd'])
returned_array[i]['out'] = stdout.read().decode('utf-8')
returned_array[i]['err'] = stderr.read().decode('utf-8')
log.debug('commnad launched / out: {} / error: {}'.format(
returned_array[i]['out'], returned_array[i]['err']))
i = i + 1
client.close()
return returned_array
def insert_event_in_db(self, dict_event):
log.debug(pformat(dict_event))
r_conn = new_rethink_connection()
try:
r.table('hypervisors_events').insert(dict_event).run(r_conn)
close_rethink_connection(r_conn)
except Exception as e:
log.error('rethink insert hyp event fail: {}'.format(e))
def insert_event_in_db(self, dict_event):
log.debug(pformat(dict_event))
r_conn = new_rethink_connection()
try:
r.table('hypervisors_events').insert(dict_event).run(r_conn)
close_rethink_connection(r_conn)
except Exception as e:
log.error('rethink insert hyp event fail: {}'.format(e))
def disk_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
if action['type'] in ['create_disk']:
launch_action_disk(action,
self.hostname,
self.user,
self.port)
elif action['type'] in ['create_disk_from_scratch']:
launch_action_disk(action,
self.hostname,
self.user,
self.port,
from_scratch=True)
elif action['type'] in ['delete_disk']:
launch_delete_disk_action(action,
self.hostname,
self.user,
self.port)
elif action['type'] in ['create_template_disk_from_domain']:
launch_action_create_template_disk(action,
self.hostname,
self.user,
self.port)
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format(
disk_path, id_domain))
log.error('\n'.join(
['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))]))
update_domain_status('Failed', id_domain,
detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def run(self):
self.tid = get_tid()
log.info('starting thread: {} (TID {})'.format(self.name, self.tid))
self.hyp_obj, self.ok = try_hyp_connection(self.hyp_id, self.hostname,
self.port, self.user)
log.debug('Exiting from thread {} try_hyp {}'.format(
self.name, self.hostname))
return self.ok
def get_hyp_hostnames_online():
r_conn = new_rethink_connection()
rtable = r.table('hypervisors')
l = list(rtable. \
filter({'enabled': True, 'status': 'Online'}). \
pluck('id', 'hostname'). \
run(r_conn))
close_rethink_connection(r_conn)
log.debug(l)
if len(l) > 0:
hyps_hostnames = {d['id']: d['hostname'] for d in l}
return hyps_hostnames
else:
return dict()
def exec_remote_list_of_cmds(hostname, commands, username='******', port=22, sudo=False):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostname, port=port, username=username)
returned_array = []
for command in commands:
log.debug('command to launch in ssh in {}: {}'.format(hostname, command))
stdin, stdout, stderr = client.exec_command(command)
out = stdout.read().decode('utf-8')
err = stderr.read().decode('utf-8')
returned_array.append({'out': out, 'err': err})
log.debug('commnad launched / out: {} / error: {}'.format(out, err))
client.close()
return returned_array
def __init__(self, src, dst, verbose=True, show_progress=True, bwlimit=''):
cmd = 'rsync -a'
if verbose:
cmd += 'v'
if len(bwlimit) > 0:
cmd += ' --bwlimit={}'.format(bwlimit)
if show_progress:
cmd += ' --progress'
cmd += ' ' + src + ' '
cmd += '' + dst + ''
self.dst = dst
self.src = src
self.cmd = cmd
log.debug('rsync command: ' + self.cmd)
self.proc = False
self.status = 0
def __init__(self, src, dst, verbose=True, show_progress=True, bwlimit=''):
cmd = 'rsync -a'
if verbose:
cmd += 'v'
if len(bwlimit) > 0:
cmd += ' --bwlimit={}'.format(bwlimit)
if show_progress:
cmd += ' --progress'
cmd += ' ' + src + ' '
cmd += '' + dst + ''
self.dst = dst
self.src = src
self.cmd = cmd
log.debug('rsync command: ' + self.cmd)
self.proc = False
self.status = 0
def exec_remote_updating_progress(command,
hostname,
progress=[],
username='******',
port=22,
sudo=False,
id_domain=None):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostname, port=port, username=username)
stdin, stdout, stderr = client.exec_command(command)
progress.append(0)
# INFO TO DEVELOPER, AQUÍ FALTA ACTUALIZAR EL PROGRESO A LA BASE DE DATOS
# A PARTIR DE LA LISTA PROGRESS, COGIENDO EL ÚLTIMO PARÁMETRO
# SE PODRÍA IR MIRANDO, AUNQUE LO SUYO ES QUE SE ACTUALIZASE AQUÍ??
log.debug('jujuju {} '.format(type(stdout)))
while True:
# out = stdout.readline(64).decode('utf-8')
out = stdout.readline(64)
if out.find('%') >= 0:
tmp = out[:str(out).find('%')]
percent = tmp[tmp.rfind(' ') + 1:]
if len(percent) > 0:
if percent.isdigit():
percent = int(percent)
if progress[-1] < percent:
progress.append(percent)
if id_domain is not None:
update_domain_progress(id_domain, percent)
log.debug(out)
if out == '':
break
out = stdout.read()
err = stderr.read()
client.close()
return {'out': out, 'err': err}
def exec_remote_list_of_cmds(hostname,
commands,
username='******',
port=22,
sudo=False):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostname, port=port, username=username)
returned_array = []
for command in commands:
log.debug('command to launch in ssh in {}: {}'.format(
hostname, command))
stdin, stdout, stderr = client.exec_command(command)
out = stdout.read().decode('utf-8')
err = stderr.read().decode('utf-8')
returned_array.append({'out': out, 'err': err})
log.debug('commnad launched / out: {} / error: {}'.format(out, err))
client.close()
return returned_array
def exec_remote_list_of_cmds_dict(hostname, list_dict_commands, username='******', port=22, ssh_key_str='', sudo=False):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if len(ssh_key_str) > 0:
# TODO: make ssh_key login
pass
else:
client.connect(hostname, port=port, username=username)
returned_array = list_dict_commands.copy()
i = 0
for command in list_dict_commands:
log.debug('command to launch in ssh in {}: {}'.format(hostname, command['cmd']))
stdin, stdout, stderr = client.exec_command(command['cmd'])
returned_array[i]['out'] = stdout.read().decode('utf-8')
returned_array[i]['err'] = stderr.read().decode('utf-8')
log.debug('commnad launched / out: {} / error: {}'.format(returned_array[i]['out'], returned_array[i]['err']))
i = i + 1
client.close()
return returned_array
def analize_backing_chains_outputs(array_out_err=[],
path_to_write_json=None,
path_to_read_json=None):
if path_to_write_json is not None:
f = open(path_to_write_json, 'w')
json.dump(array_out_err, f)
f.close()
if path_to_read_json is not None:
f = open(path_to_read_json, 'r')
array_out_err = json.load(f)
log.debug(len(array_out_err))
f.close()
domains_ok = 0
domains_err = 0
for d in array_out_err:
id = d['title']
if len(d['err']) > 0:
domains_err += 1
log.info(d['err'])
update_domain_status('Failed', id, detail=d['err'])
else:
log.debug(id)
domains_ok += 1
if type(d['out']) is not str:
out = out.decode('utf-8')
else:
out = d['out']
l = json.loads(out)
from pprint import pprint
pprint(l)
update_disk_backing_chain(id, 0, l[0]['filename'], l)
return ({'ok': domains_ok, 'err': domains_err})
def get_process_stdout(self):
self.stdout = ''
while True:
out = self.proc.stdout.readline(64).decode('utf-8')
if out == '' and self.proc.poll() != None:
break
self.update_output(out)
log.debug(out)
log.debug(type(out))
self.stdout += out
self.stdout = self.stdout.replace('\r', '\n')
self.status = 1
log.debug('rsync finished: ' + self.stdout)
def get_process_stdout(self):
self.stdout = ''
while True:
out = self.proc.stdout.readline(64).decode('utf-8')
if out == '' and self.proc.poll() != None:
break
self.update_output(out)
log.debug(out)
log.debug(type(out))
self.stdout += out
self.stdout = self.stdout.replace('\r', '\n')
self.status = 1
log.debug('rsync finished: ' + self.stdout)
def remove_dst(self):
if os.path.isdir(self.dst):
if self.dst.endswith('/'):
path = self.dst + self.src[self.src.rfind('/') + 1:]
else:
path = self.dst + self.src[self.src.rfind('/'):]
else:
path = self.dst
if os.path.isfile(path):
try:
os.remove(path)
log.debug('file removed: ' + path)
except:
log.debug('remove fail')
else:
log.debug('destination don\'t exist')
def remove_dst(self):
if os.path.isdir(self.dst):
if self.dst.endswith('/'):
path = self.dst + self.src[self.src.rfind('/') + 1:]
else:
path = self.dst + self.src[self.src.rfind('/'):]
else:
path = self.dst
if os.path.isfile(path):
try:
os.remove(path)
log.debug('file removed: ' + path)
except:
log.debug('remove fail')
else:
log.debug('destination don\'t exist')
def start_and_wait_silence(self):
try:
self.start_process()
self.wait_communicate()
except:
log.debug('call to subprocess rsync failed: ' + self.cmd)
def wait_communicate(self):
self.stdout, self.stderr = self.proc.communicate()
self.status = 1
log.debug('rsync finished: ' + self.stdout)
def long_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
id_domain = action['domain']
if action['type'] in ['create_disk_virt_builder']:
cmds_done = execute_commands(host=self.hostname,
ssh_commands=action['ssh_commands'],
dict_mode=True,
user=self.user,
port=self.port
)
if len([d for d in cmds_done if len(d['err']) > 0]) > 1:
log.error('some error in virt builder operations')
log.error('Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'.format(
action['disk_path'], action['domain'], self.hyp_id))
log.debug('print cmds_done:')
log.debug(pprint.pprint(cmds_done))
log.debug('print ssh_commands:')
log.debug(pprint.pprint(action['ssh_commands']))
update_domain_status('Failed', id_domain,
detail='Virt Builder Failed creating disk file')
else:
log.info('Disk created from virt-builder. Domain: {} , disk: {}'.format(action['domain'],
action['disk_path']))
xml_virt_install = cmds_done[-1]['out']
update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install)
update_domain_status('CreatingDomainFromBuilder', id_domain,
detail='disk created from virt-builder')
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format(
disk_path, id_domain))
log.error('\n'.join(
['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))]))
update_domain_status('Failed', id_domain,
detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def wait_communicate(self):
self.stdout, self.stderr = self.proc.communicate()
self.status = 1
log.debug('rsync finished: ' + self.stdout)
def start_and_wait_verbose(self):
try:
self.start_process()
self.get_process_stdo
except:
log.debug('call to subprocess rsync failed: ' + self.cmd)
def try_ssh(hostname, port, user, timeout):
if try_socket(hostname, port, timeout) is True:
ip = socket.gethostbyname(hostname)
ssh = paramiko.SSHClient()
# ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
##INFO TO DEVELOPER TERMINAR DE ENTENDER POR QUE A VECES VA Y A VECES NO
ssh.load_system_host_keys()
# ssh.load_host_keys('/home/vimet/.ssh/known_hosts')
time.sleep(1)
try:
# timelimit(3,test_hypervisor_conn,hostname,
# username=user,
# port= port,
# timeout=CONFIG_DICT['TIMEOUTS']['ssh_paramiko_hyp_test_connection'])
log.debug('@@@@@@@@@@@@@@@@@@@@')
log.debug('@@@@@@@@@@@@@@@@@@@@')
log.debug('@@@@@@@@@@@@@@@@@@@@')
ssh.connect(hostname,
username=user,
port=port,
timeout=timeout, banner_timeout=timeout)
log.debug("host {} with ip {} can connect with ssh without password with paramiko".format(hostname, ip))
log.debug('############################################')
log.debug('############################################')
log.debug('############################################')
log.debug('############################################')
ssh.close()
#
# cmd = 'timeout 20 ssh -p {} {}@{} pwd'.format(port,user,hostname)
# # proc = subprocess.Popen(["bash", "-c", cmd],stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
# proc = subprocess.Popen(shlex.split(cmd),stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
# proc = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=True)
# stdout,stderr=proc.communicate()
#
# if len(stdout) > 0
# log.debug(stdout)
# log.debug(stderr)
# if
return True
except Exception as e:
try:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname,
username=user,
port=port,
timeout=timeout, banner_timeout=timeout)
log.debug("The authenticity of host '{} ({})' can't be established".format(hostname, ip))
log.debug(
"host {} with ip {} can connect with ssh without password but the key fingerprint must be incorporated in ~/.ssh/known_hosts".format(
hostname, ip))
ssh.close()
return False
except:
log.error(
"host {} with ip {} can't connect with ssh without password. Reasons? timeout, ssh authentication with keys is needed, port is correct?".format(
hostname, ip))
log.error('reason: {}'.format(e))
return False
else:
log.error('socket error, try if ssh is listen in hostname {} and port {}'.format(hostname, port))
return False
def start_and_wait_silence(self):
try:
self.start_process()
self.wait_communicate()
except:
log.debug('call to subprocess rsync failed: ' + self.cmd)
def start_and_wait_verbose(self):
try:
self.start_process()
self.get_process_stdo
except:
log.debug('call to subprocess rsync failed: ' + self.cmd)
def start_and_continue(self):
try:
self.start_process()
self.thread()
except:
log.debug('call to subprocess rsync failed')
def start_and_continue(self):
try:
self.start_process()
self.thread()
except:
log.debug('call to subprocess rsync failed')
def try_ssh(hostname, port, user, timeout):
if try_socket(hostname, port, timeout) is True:
ip = socket.gethostbyname(hostname)
ssh = paramiko.SSHClient()
# ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
##INFO TO DEVELOPER TERMINAR DE ENTENDER POR QUE A VECES VA Y A VECES NO
ssh.load_system_host_keys()
# ssh.load_host_keys('/home/vimet/.ssh/known_hosts')
time.sleep(1)
try:
# timelimit(3,test_hypervisor_conn,hostname,
# username=user,
# port= port,
# timeout=CONFIG_DICT['TIMEOUTS']['ssh_paramiko_hyp_test_connection'])
log.debug('@@@@@@@@@@@@@@@@@@@@')
log.debug('@@@@@@@@@@@@@@@@@@@@')
log.debug('@@@@@@@@@@@@@@@@@@@@')
ssh.connect(hostname,
username=user,
port=port,
timeout=timeout,
banner_timeout=timeout)
log.debug(
"host {} with ip {} can connect with ssh without password with paramiko"
.format(hostname, ip))
log.debug('############################################')
log.debug('############################################')
log.debug('############################################')
log.debug('############################################')
ssh.close()
#
# cmd = 'timeout 20 ssh -p {} {}@{} pwd'.format(port,user,hostname)
# # proc = subprocess.Popen(["bash", "-c", cmd],stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
# proc = subprocess.Popen(shlex.split(cmd),stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
# proc = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=True)
# stdout,stderr=proc.communicate()
#
# if len(stdout) > 0
# log.debug(stdout)
# log.debug(stderr)
# if
return True
except Exception as e:
try:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname,
username=user,
port=port,
timeout=timeout,
banner_timeout=timeout)
log.debug(
"The authenticity of host '{} ({})' can't be established".
format(hostname, ip))
log.debug(
"host {} with ip {} can connect with ssh without password but the key fingerprint must be incorporated in ~/.ssh/known_hosts"
.format(hostname, ip))
ssh.close()
return False
except:
log.error(
"host {} with ip {} can't connect with ssh without password. Reasons? timeout, ssh authentication with keys is needed, port is correct?"
.format(hostname, ip))
log.error('reason: {}'.format(e))
return False
else:
log.error(
'socket error, try if ssh is listen in hostname {} and port {}'.
format(hostname, port))
return False
def long_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug(
'Thread to launchdisks operations in host {} with TID: {}...'.
format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
id_domain = action['domain']
if action['type'] in ['create_disk_virt_builder']:
cmds_done = execute_commands(
host=self.hostname,
ssh_commands=action['ssh_commands'],
dict_mode=True,
user=self.user,
port=self.port)
if len([d for d in cmds_done if len(d['err']) > 0]) > 1:
log.error('some error in virt builder operations')
log.error(
'Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'
.format(action['disk_path'], action['domain'],
self.hyp_id))
log.debug('print cmds_done:')
log.debug(pprint.pprint(cmds_done))
log.debug('print ssh_commands:')
log.debug(pprint.pprint(action['ssh_commands']))
update_domain_status(
'Failed',
id_domain,
detail='Virt Builder Failed creating disk file')
else:
log.info(
'Disk created from virt-builder. Domain: {} , disk: {}'
.format(action['domain'], action['disk_path']))
xml_virt_install = cmds_done[-1]['out']
update_table_field('domains', id_domain,
'xml_virt_install',
xml_virt_install)
update_domain_status(
'CreatingDomainFromBuilder',
id_domain,
detail='disk created from virt-builder')
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(
action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'
.format(disk_path, id_domain))
log.error('\n'.join([
'cmd: {}'.format(action['ssh_commands'][i])
for i in range(len(action['ssh_commands']))
]))
update_domain_status(
'Failed',
id_domain,
detail=
'new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs'
)
