def test_hypervisor_conn(uri):
""" test hypervisor connecton, if fail an error message in log """
try:
handle = libvirt.open(uri)
return handle
except:
log.error(sys.exc_info()[1])
return False
def test_hypervisor_conn(uri):
""" test hypervisor connecton, if fail an error message in log """
try:
handle = libvirt.open(uri)
return handle
except:
log.error(sys.exc_info()[1])
return False
def insert_event_in_db(self, dict_event):
log.debug(pformat(dict_event))
r_conn = new_rethink_connection()
try:
r.table('hypervisors_events').insert(dict_event).run(r_conn)
close_rethink_connection(r_conn)
except Exception as e:
log.error('rethink insert hyp event fail: {}'.format(e))
def insert_event_in_db(self, dict_event):
log.debug(pformat(dict_event))
r_conn = new_rethink_connection()
try:
r.table('hypervisors_events').insert(dict_event).run(r_conn)
close_rethink_connection(r_conn)
except Exception as e:
log.error('rethink insert hyp event fail: {}'.format(e))
def get_hyp_hostname_from_id(id):
r_conn = new_rethink_connection()
l = r.table('hypervisors').get(id).pluck('hostname', 'port', 'user').run(r_conn)
close_rethink_connection(r_conn)
if len(l) > 0:
if l.__contains__('user') and l.__contains__('port'):
return l['hostname'], l['port'], l['user']
else:
log.error('hypervisor {} does not contain user or port in database'.format(id))
return False
else:
return False
def get_id_hyp_from_uri(uri):
r_conn = new_rethink_connection()
rtable = r.table('hypervisors')
l = list(rtable. \
filter({'uri': uri}). \
pluck('id'). \
run(r_conn))
close_rethink_connection(r_conn)
if len(l) > 0:
return l[0]['id']
else:
log.error('function: {} uri {} not defined in hypervisors table'.format(str(__name__), uri))
def new_template_from_domain(name,
description,
original_domain_id,
user,
group,
category,
kind='public_template'):
id = '_' + user + '_' + name
dir_disk = category + '/' + group + '/' + name
disk_filename = name + '.qcow2'
relative_dir = dir_disk + '/' + disk_filename
old_domain = get_domain(original_domain_id)
new_backing_chain = [relative_dir]
new_backing_chain.append(old_domain['hardware']['disks'][0]['file'])
new_hardware_dict = old_domain['hardware']
new_hardware_dict['disks'][0]['file'] = relative_dir
new_hardware_dict['disks'][0]['backing_chain'] = new_backing_chain
new_hardware_dict['name'] = id
new_hardware_dict['uuid'] = None
template_field = {
'id': id,
'kind': kind,
'user': user,
'status': 'Creating',
'detail': None,
'category': category,
'group': group,
'server': False,
'hardware': new_hardware_dict,
'hypervisors_pools': old_domain['hypervisors_pools'],
'xml': None,
'name': name,
'description': description,
'icon': old_domain['icon'],
'os': old_domain['icon']
}
result = update_domain_createing_template(original_domain_id,
template_field)
if result['errors'] > 0:
log.error(
'error when inserting new domain {} in db, duplicate id??'.format(
id))
return False
else:
return id
def new_domain_from_template(name, description, template_origin, user, group,
category):
id = '_' + user + '_' + name
dir_disk = category + '/' + group + '/' + name
disk_filename = name + '.qcow2'
relative_path = dir_disk + '/' + disk_filename
template = get_domain(template_origin)
# new_backing_chain = [relative_dir]
# new_backing_chain.append(template['hardware']['disks'][0]['file'])
base_disk_path = template['hardware']['disks'][0]['file']
new_hardware_dict = template['hardware'].copy()
new_hardware_dict['disks'][0]['file'] = relative_path
new_hardware_dict['disks'][0]['parent'] = base_disk_path
new_hardware_dict['name'] = id
new_hardware_dict['uuid'] = None
domain = {
'id': id,
'kind': 'desktop',
'user': user,
'status': 'Creating',
'detail': None,
'category': category,
'group': group,
'server': False,
'hardware': new_hardware_dict,
'hypervisors_pools': template['hypervisors_pools'],
'xml': template['xml'],
'name': name,
'description': description,
'icon': template['icon'],
'os': template['icon']
}
result = insert_domain(domain)
if result['errors'] > 0:
log.error(
'error when inserting new domain {} in db, duplicate id??'.format(
id))
return False
else:
return id
def new_template_from_domain(name, description, original_domain_id,
user, group, category, kind='public_template'):
id = '_' + user + '_' + name
dir_disk = category + '/' + group + '/' + name
disk_filename = name + '.qcow2'
relative_dir = dir_disk + '/' + disk_filename
old_domain = get_domain(original_domain_id)
new_backing_chain = [relative_dir]
new_backing_chain.append(old_domain['hardware']['disks'][0]['file'])
new_hardware_dict = old_domain['hardware']
new_hardware_dict['disks'][0]['file'] = relative_dir
new_hardware_dict['disks'][0]['backing_chain'] = new_backing_chain
new_hardware_dict['name'] = id
new_hardware_dict['uuid'] = None
template_field = {'id': id,
'kind': kind,
'user': user,
'status': 'Creating',
'detail': None,
'category': category,
'group': group,
'server': False,
'hardware': new_hardware_dict,
'hypervisors_pools': old_domain['hypervisors_pools'],
'xml': None,
'name': name,
'description': description,
'icon': old_domain['icon'],
'os': old_domain['icon']}
result = update_domain_createing_template(original_domain_id, template_field)
if result['errors'] > 0:
log.error('error when inserting new domain {} in db, duplicate id??'.format(id))
return False
else:
return id
def new_domain_from_template(name, description, template_origin, user, group, category):
id = '_' + user + '_' + name
dir_disk = category + '/' + group + '/' + name
disk_filename = name + '.qcow2'
relative_path = dir_disk + '/' + disk_filename
template = get_domain(template_origin)
# new_backing_chain = [relative_dir]
# new_backing_chain.append(template['hardware']['disks'][0]['file'])
base_disk_path = template['hardware']['disks'][0]['file']
new_hardware_dict = template['hardware'].copy()
new_hardware_dict['disks'][0]['file'] = relative_path
new_hardware_dict['disks'][0]['parent'] = base_disk_path
new_hardware_dict['name'] = id
new_hardware_dict['uuid'] = None
domain = {'id': id,
'kind': 'desktop',
'user': user,
'status': 'Creating',
'detail': None,
'category': category,
'group': group,
'server': False,
'hardware': new_hardware_dict,
'hypervisors_pools': template['hypervisors_pools'],
'xml': template['xml'],
'name': name,
'description': description,
'icon': template['icon'],
'os': template['icon']}
result = insert_domain(domain)
if result['errors'] > 0:
log.error('error when inserting new domain {} in db, duplicate id??'.format(id))
return False
else:
return id
def try_socket(hostname, port, timeout):
try:
ip = socket.gethostbyname(hostname)
addr = (hostname, port)
sock = socket.socket(2, socket.SOCK_STREAM)
sock.settimeout(timeout)
try:
sock.connect(addr)
sock.close()
return True
except socket.error as e:
log.error('trying socket has error: {}'.format(e))
return False
except Exception as e:
log.error(e)
return False
except socket.error as e:
log.error(e)
log.error('not resolves ip from hostname: {}'.format(hostname))
return False
def try_socket(hostname, port, timeout):
try:
ip = socket.gethostbyname(hostname)
addr = (hostname, port)
sock = socket.socket(2, socket.SOCK_STREAM)
sock.settimeout(timeout)
try:
sock.connect(addr)
sock.close()
return True
except socket.error as e:
log.error('trying socket has error: {}'.format(e))
return False
except Exception as e:
log.error(e)
return False
except socket.error as e:
log.error(e)
log.error('not resolves ip from hostname: {}'.format(hostname))
return False
def disk_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
if action['type'] in ['create_disk']:
launch_action_disk(action,
self.hostname,
self.user,
self.port)
elif action['type'] in ['create_disk_from_scratch']:
launch_action_disk(action,
self.hostname,
self.user,
self.port,
from_scratch=True)
elif action['type'] in ['delete_disk']:
launch_delete_disk_action(action,
self.hostname,
self.user,
self.port)
elif action['type'] in ['create_template_disk_from_domain']:
launch_action_create_template_disk(action,
self.hostname,
self.user,
self.port)
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format(
disk_path, id_domain))
log.error('\n'.join(
['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))]))
update_domain_status('Failed', id_domain,
detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def try_ssh(hostname, port, user, timeout):
if try_socket(hostname, port, timeout) is True:
ip = socket.gethostbyname(hostname)
ssh = paramiko.SSHClient()
# ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
##INFO TO DEVELOPER TERMINAR DE ENTENDER POR QUE A VECES VA Y A VECES NO
ssh.load_system_host_keys()
# ssh.load_host_keys('/home/vimet/.ssh/known_hosts')
time.sleep(1)
try:
# timelimit(3,test_hypervisor_conn,hostname,
# username=user,
# port= port,
# timeout=CONFIG_DICT['TIMEOUTS']['ssh_paramiko_hyp_test_connection'])
log.debug('@@@@@@@@@@@@@@@@@@@@')
log.debug('@@@@@@@@@@@@@@@@@@@@')
log.debug('@@@@@@@@@@@@@@@@@@@@')
ssh.connect(hostname,
username=user,
port=port,
timeout=timeout, banner_timeout=timeout)
log.debug("host {} with ip {} can connect with ssh without password with paramiko".format(hostname, ip))
log.debug('############################################')
log.debug('############################################')
log.debug('############################################')
log.debug('############################################')
ssh.close()
#
# cmd = 'timeout 20 ssh -p {} {}@{} pwd'.format(port,user,hostname)
# # proc = subprocess.Popen(["bash", "-c", cmd],stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
# proc = subprocess.Popen(shlex.split(cmd),stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
# proc = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=True)
# stdout,stderr=proc.communicate()
#
# if len(stdout) > 0
# log.debug(stdout)
# log.debug(stderr)
# if
return True
except Exception as e:
try:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname,
username=user,
port=port,
timeout=timeout, banner_timeout=timeout)
log.debug("The authenticity of host '{} ({})' can't be established".format(hostname, ip))
log.debug(
"host {} with ip {} can connect with ssh without password but the key fingerprint must be incorporated in ~/.ssh/known_hosts".format(
hostname, ip))
ssh.close()
return False
except:
log.error(
"host {} with ip {} can't connect with ssh without password. Reasons? timeout, ssh authentication with keys is needed, port is correct?".format(
hostname, ip))
log.error('reason: {}'.format(e))
return False
else:
log.error('socket error, try if ssh is listen in hostname {} and port {}'.format(hostname, port))
return False
def long_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug('Thread to launchdisks operations in host {} with TID: {}...'.format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
id_domain = action['domain']
if action['type'] in ['create_disk_virt_builder']:
cmds_done = execute_commands(host=self.hostname,
ssh_commands=action['ssh_commands'],
dict_mode=True,
user=self.user,
port=self.port
)
if len([d for d in cmds_done if len(d['err']) > 0]) > 1:
log.error('some error in virt builder operations')
log.error('Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'.format(
action['disk_path'], action['domain'], self.hyp_id))
log.debug('print cmds_done:')
log.debug(pprint.pprint(cmds_done))
log.debug('print ssh_commands:')
log.debug(pprint.pprint(action['ssh_commands']))
update_domain_status('Failed', id_domain,
detail='Virt Builder Failed creating disk file')
else:
log.info('Disk created from virt-builder. Domain: {} , disk: {}'.format(action['domain'],
action['disk_path']))
xml_virt_install = cmds_done[-1]['out']
update_table_field('domains', id_domain, 'xml_virt_install', xml_virt_install)
update_domain_status('CreatingDomainFromBuilder', id_domain,
detail='disk created from virt-builder')
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'.format(
disk_path, id_domain))
log.error('\n'.join(
['cmd: {}'.format(action['ssh_commands'][i]) for i in range(len(action['ssh_commands']))]))
update_domain_status('Failed', id_domain,
detail='new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs')
def update_hyp_status(id, status, detail='', uri=''):
# INFO TO DEVELOPER: TODO debería pillar el estado anterior del hypervisor y ponerlo en un campo,
# o mejor aún, guardar un histórico con los tiempos de cambios en un diccionario que
# en python puede ser internamente una cola de X elementos (número de elementos de configuración)
# como una especie de log de cuando cambio de estado
# INFO TO DEVELOPER: pasarlo a una función en functions
defined_status = ['Offline',
'TryConnection',
'ReadyToStart',
'StartingThreads',
'Error',
'Online',
'Blocked',
'DestroyingDomains',
'StoppingThreads']
if status in defined_status:
r_conn = new_rethink_connection()
rtable = r.table('hypervisors')
if len(uri) > 0:
dict_update = {'status': status, 'uri': uri}
else:
dict_update = {'status': status}
d = rtable.get(id).pluck('status',
'status_time',
'prev_status',
'detail').run(r_conn)
if 'status' in d.keys():
if 'prev_status' not in d.keys():
dict_update['prev_status'] = []
else:
if type(d['prev_status']) is list:
dict_update['prev_status'] = d['prev_status']
else:
dict_update['prev_status'] = []
d_old_status = {}
d_old_status['status'] = d['status']
if 'detail' in d.keys():
d_old_status['detail'] = d['detail']
else:
d_old_status['detail'] = ''
if 'status_time' in d.keys():
d_old_status['status_time'] = d['status_time']
dict_update['prev_status'].insert(0, d_old_status)
dict_update['prev_status'] = dict_update['prev_status'][:MAX_LEN_PREV_STATUS_HYP]
now = time.time()
dict_update['status_time'] = now
# if len(detail) == 0:
# rtable.filter({'id':id}).\
# update(dict_update).\
# run(r_conn)
# # rtable.filter({'id':id}).\
# # replace(r.row.without('detail')).\
# # run(r_conn)
# close_rethink_connection(r_conn)
#
# else:
dict_update['detail'] = str(detail)
rtable.filter({'id': id}). \
update(dict_update). \
run(r_conn)
close_rethink_connection(r_conn)
else:
log.error('hypervisor status {} is not defined'.format(status))
return False
def try_ssh(hostname, port, user, timeout):
if try_socket(hostname, port, timeout) is True:
ip = socket.gethostbyname(hostname)
ssh = paramiko.SSHClient()
# ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
##INFO TO DEVELOPER TERMINAR DE ENTENDER POR QUE A VECES VA Y A VECES NO
ssh.load_system_host_keys()
# ssh.load_host_keys('/home/vimet/.ssh/known_hosts')
time.sleep(1)
try:
# timelimit(3,test_hypervisor_conn,hostname,
# username=user,
# port= port,
# timeout=CONFIG_DICT['TIMEOUTS']['ssh_paramiko_hyp_test_connection'])
log.debug('@@@@@@@@@@@@@@@@@@@@')
log.debug('@@@@@@@@@@@@@@@@@@@@')
log.debug('@@@@@@@@@@@@@@@@@@@@')
ssh.connect(hostname,
username=user,
port=port,
timeout=timeout,
banner_timeout=timeout)
log.debug(
"host {} with ip {} can connect with ssh without password with paramiko"
.format(hostname, ip))
log.debug('############################################')
log.debug('############################################')
log.debug('############################################')
log.debug('############################################')
ssh.close()
#
# cmd = 'timeout 20 ssh -p {} {}@{} pwd'.format(port,user,hostname)
# # proc = subprocess.Popen(["bash", "-c", cmd],stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
# proc = subprocess.Popen(shlex.split(cmd),stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
# proc = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,shell=True)
# stdout,stderr=proc.communicate()
#
# if len(stdout) > 0
# log.debug(stdout)
# log.debug(stderr)
# if
return True
except Exception as e:
try:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname,
username=user,
port=port,
timeout=timeout,
banner_timeout=timeout)
log.debug(
"The authenticity of host '{} ({})' can't be established".
format(hostname, ip))
log.debug(
"host {} with ip {} can connect with ssh without password but the key fingerprint must be incorporated in ~/.ssh/known_hosts"
.format(hostname, ip))
ssh.close()
return False
except:
log.error(
"host {} with ip {} can't connect with ssh without password. Reasons? timeout, ssh authentication with keys is needed, port is correct?"
.format(hostname, ip))
log.error('reason: {}'.format(e))
return False
else:
log.error(
'socket error, try if ssh is listen in hostname {} and port {}'.
format(hostname, port))
return False
def long_operations_thread(self):
host = self.hostname
self.tid = get_tid()
log.debug(
'Thread to launchdisks operations in host {} with TID: {}...'.
format(host, self.tid))
while self.stop is not True:
try:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
# for ssh commands
id_domain = action['domain']
if action['type'] in ['create_disk_virt_builder']:
cmds_done = execute_commands(
host=self.hostname,
ssh_commands=action['ssh_commands'],
dict_mode=True,
user=self.user,
port=self.port)
if len([d for d in cmds_done if len(d['err']) > 0]) > 1:
log.error('some error in virt builder operations')
log.error(
'Virt Builder Failed creating disk file {} in domain {} in hypervisor {}'
.format(action['disk_path'], action['domain'],
self.hyp_id))
log.debug('print cmds_done:')
log.debug(pprint.pprint(cmds_done))
log.debug('print ssh_commands:')
log.debug(pprint.pprint(action['ssh_commands']))
update_domain_status(
'Failed',
id_domain,
detail='Virt Builder Failed creating disk file')
else:
log.info(
'Disk created from virt-builder. Domain: {} , disk: {}'
.format(action['domain'], action['disk_path']))
xml_virt_install = cmds_done[-1]['out']
update_table_field('domains', id_domain,
'xml_virt_install',
xml_virt_install)
update_domain_status(
'CreatingDomainFromBuilder',
id_domain,
detail='disk created from virt-builder')
elif action['type'] == 'stop_thread':
self.stop = True
else:
log.error('type action {} not supported'.format(
action['type']))
except queue.Empty:
pass
except Exception as e:
log.error('Exception when creating disk: {}'.format(e))
log.error('Action: {}'.format(pprint.pformat(action)))
log.error('Traceback: {}'.format(traceback.format_exc()))
return False
if self.stop is True:
while self.queue_actions.empty() is not True:
action = self.queue_actions.get(timeout=TIMEOUT_QUEUES)
if action['type'] == 'create_disk':
disk_path = action['disk_path']
id_domain = action['domain']
log.error(
'operations creating disk {} for new domain {} failed. Commands, outs and errors: {}'
.format(disk_path, id_domain))
log.error('\n'.join([
'cmd: {}'.format(action['ssh_commands'][i])
for i in range(len(action['ssh_commands']))
]))
update_domain_status(
'Failed',
id_domain,
detail=
'new disk create operation failed, thread disk operations is stopping, detail of operations cancelled in logs'
)
