def pytest_sessionfinish(self, exitstatus):
# Now master/standalone needs to move all the reports to an appliance for the source report
if store.parallelizer_role != 'master':
manager().collect()
# for slaves, everything is done at this point
if store.parallelizer_role == 'slave':
return
# on master/standalone, merge all the collected reports and bring them back
manager().merge()
try:
global ui_coverage_percent
last_run = json.load(log_path.join('coverage', 'merged', '.last_run.json').open())
ui_coverage_percent = last_run['result']['covered_percent']
style = {'bold': True}
if ui_coverage_percent > 40:
style['green'] = True
else:
style['red'] = True
store.write_line('UI Coverage Result: {}%'.format(ui_coverage_percent),
**style)
except Exception as ex:
logger.error('Error printing coverage report to terminal')
logger.exception(ex)
def deploy_template(provider_key, vm_name, template_name=None, timeout=900, **deploy_args):
"""
Args:
provider_key: Provider key on which the VM is to be created
vm_name: Name of the VM to be deployed
template_name: Name of the template that the VM is deployed from
timeout: the timeout for template deploy
"""
allow_skip = deploy_args.pop("allow_skip", ())
if isinstance(allow_skip, dict):
skip_exceptions = allow_skip.keys()
callable_mapping = allow_skip
elif isinstance(allow_skip, six.string_types) and allow_skip.lower() == "default":
skip_exceptions = DEFAULT_SKIP
callable_mapping = {}
else:
skip_exceptions = allow_skip
callable_mapping = {}
provider_crud = get_crud(provider_key)
deploy_args.update(vm_name=vm_name)
if template_name is None:
try:
deploy_args.update(template=provider_crud.data['templates']['small_template']['name'])
except KeyError:
raise KeyError('small_template not defined for Provider {} in cfme_data.yaml'
.format(provider_key))
else:
deploy_args.update(template=template_name)
deploy_args.update(provider_crud.deployment_helper(deploy_args))
logger.info("Getting ready to deploy VM/instance %s from template %s on provider %s",
vm_name, deploy_args['template'], provider_crud.data['name'])
try:
try:
logger.debug("Deploy args: %s", deploy_args)
vm_name = provider_crud.mgmt.deploy_template(timeout=timeout, **deploy_args)
logger.info("Provisioned VM/instance %s", vm_name) # instance ID in case of EC2
except Exception as e:
logger.exception('Could not provisioning VM/instance %s (%s: %s)',
vm_name, type(e).__name__, str(e))
try:
provider_crud.mgmt.delete_vm(vm_name)
except Exception:
logger.exception("Unable to clean up vm:", vm_name)
raise
except skip_exceptions as e:
e_c = type(e)
if e_c in callable_mapping and not callable_mapping[e_c](e):
raise
# Make it visible also in the log.
store.write_line(
"Skipping due to a provider error: {}: {}\n".format(e_c.__name__, str(e)), purple=True)
logger.exception(e)
pytest.skip("{}: {}".format(e_c.__name__, str(e)))
return vm_name
def appliance_police(appliance):
if not store.slave_manager:
return
try:
available_ports = {
'ssh': (appliance.hostname, appliance.ssh_port),
'https': (appliance.hostname, appliance.ui_port),
'postgres': (appliance.db_host or appliance.hostname, appliance.db_port)}
port_results = {pn: net_check(addr=p_addr, port=p_port, force=True)
for pn, (p_addr, p_port) in available_ports.items()}
for port, result in port_results.items():
if port == 'ssh' and appliance.is_pod:
# ssh is not available for podified appliance
continue
if not result:
raise AppliancePoliceException('Unable to connect', available_ports[port][1])
try:
status_code = requests.get(appliance.url, verify=False,
timeout=120).status_code
except Exception:
raise AppliancePoliceException('Getting status code failed',
available_ports['https'][1])
if status_code != 200:
raise AppliancePoliceException('Status code was {}, should be 200'.format(
status_code), available_ports['https'][1])
return
except AppliancePoliceException as e:
# special handling for known failure conditions
if e.port == 443:
# Lots of rdbs lately where evm seems to have entirely crashed
# and (sadly) the only fix is a rude restart
appliance.restart_evm_service(rude=True)
try:
appliance.wait_for_web_ui(900)
store.write_line('EVM was frozen and had to be restarted.', purple=True)
return
except TimedOutError:
pass
e_message = str(e)
except Exception as e:
e_message = str(e)
# Regardles of the exception raised, we didn't return anywhere above
# time to call a human
msg = 'Help! My appliance {} crashed with: {}'.format(appliance.url, e_message)
store.slave_manager.message(msg)
if 'appliance_police_recipients' in rdb:
rdb_kwargs = {
'subject': 'RDB Breakpoint: Appliance failure',
'recipients': rdb.appliance_police_recipients,
}
else:
rdb_kwargs = {}
Rdb(msg).set_trace(**rdb_kwargs)
store.slave_manager.message('Resuming testing following remote debugging')
def appliance_police():
if not store.slave_manager:
return
try:
port_numbers = {
'ssh': ports.SSH,
'https': store.current_appliance.ui_port,
'postgres': ports.DB}
port_results = {pn: net_check(pp, force=True) for pn, pp in port_numbers.items()}
for port, result in port_results.items():
if not result:
raise _AppliancePoliceException('Port {} was not contactable'.format(port),
port_numbers[port])
try:
status_code = requests.get(store.current_appliance.url, verify=False,
timeout=120).status_code
except Exception:
raise _AppliancePoliceException('Getting status code failed', port_numbers['https'])
if status_code != 200:
raise _AppliancePoliceException('Status code was {}, should be 200'.format(
status_code), port_numbers['https'])
return
except _AppliancePoliceException as e:
# special handling for known failure conditions
if e.port == 443:
# Lots of rdbs lately where evm seems to have entirely crashed
# and (sadly) the only fix is a rude restart
store.current_appliance.restart_evm_service(rude=True)
try:
store.current_appliance.wait_for_web_ui(900)
store.write_line('EVM was frozen and had to be restarted.', purple=True)
return
except TimedOutError:
pass
e_message = e.message
except Exception as e:
e_message = e.args[0]
# Regardles of the exception raised, we didn't return anywhere above
# time to call a human
msg = 'Help! My appliance {} crashed with: {}'.format(store.current_appliance.url, e_message)
store.slave_manager.message(msg)
if 'appliance_police_recipients' in rdb:
rdb_kwargs = {
'subject': 'RDB Breakpoint: Appliance failure',
'recipients': rdb.appliance_police_recipients,
}
else:
rdb_kwargs = {}
Rdb(msg).set_trace(**rdb_kwargs)
store.slave_manager.message('Resuming testing following remote debugging')
def appliance_police():
if not store.slave_manager:
return
try:
port_numbers = {
'ssh': ports.SSH,
'https': store.current_appliance.ui_port,
'postgres': ports.DB}
port_results = {pn: net_check(pp, force=True) for pn, pp in port_numbers.items()}
for port, result in port_results.items():
if not result:
raise _AppliancePoliceException('Port {} was not contactable'.format(port),
port_numbers[port])
try:
status_code = requests.get(store.current_appliance.url, verify=False,
timeout=120).status_code
except Exception:
raise _AppliancePoliceException('Getting status code failed', port_numbers['https'])
if status_code != 200:
raise _AppliancePoliceException('Status code was {}, should be 200'.format(
status_code), port_numbers['https'])
return
except _AppliancePoliceException as e:
# special handling for known failure conditions
if e.port == 443:
# If we had an error on 443, about 101% of the time it means the UI worker is frozen
store.current_appliance.ssh_client().run_rails_command('MiqUiWorker.first.kill')
try:
store.current_appliance.wait_for_web_ui(900)
store.write_line('UI worker was frozen and had to be restarted.', purple=True)
return
except TimedOutError:
pass
e_message = e.message
except Exception as e:
e_message = e.args[0]
# Regardles of the exception raised, we didn't return anywhere above
# time to call a human
msg = 'Help! My appliance {} crashed with: {}'.format(store.current_appliance.url, e_message)
store.slave_manager.message(msg)
Rdb(msg).set_trace(**{
'subject': 'RDB Breakpoint: Appliance failure',
'recipients': ['*****@*****.**', '*****@*****.**'],
})
store.slave_manager.message('Resuming testing following remote debugging')
def deploy_template(provider_key,
vm_name,
template_name=None,
timeout=900,
**deploy_args):
"""
Args:
provider_key: Provider key on which the VM is to be created
vm_name: Name of the VM to be deployed
template_name: Name of the template that the VM is deployed from
timeout: the timeout for template deploy
"""
allow_skip = deploy_args.pop("allow_skip", ())
if isinstance(allow_skip, dict):
skip_exceptions = allow_skip.keys()
callable_mapping = allow_skip
elif isinstance(allow_skip,
basestring) and allow_skip.lower() == "default":
skip_exceptions = (OSOverLimit, RHEVRequestError,
exceptions.VMInstanceNotCloned, SSLError)
callable_mapping = {}
else:
skip_exceptions = allow_skip
callable_mapping = {}
provider_crud = get_crud(provider_key)
deploy_args.update(vm_name=vm_name)
if template_name is None:
try:
deploy_args.update(template=provider_crud.data['templates']
['small_template']['name'])
except KeyError:
raise KeyError(
'small_template not defined for Provider {} in cfme_data.yaml'.
format(provider_key))
else:
deploy_args.update(template=template_name)
deploy_args.update(provider_crud.deployment_helper(deploy_args))
logger.info(
"Getting ready to deploy VM/instance %s from template %s on provider %s",
vm_name, deploy_args['template'], provider_crud.data['name'])
try:
try:
logger.debug("Deploy args: %s", deploy_args)
vm_name = provider_crud.mgmt.deploy_template(timeout=timeout,
**deploy_args)
logger.info("Provisioned VM/instance %s",
vm_name) # instance ID in case of EC2
except Exception as e:
logger.exception('Could not provisioning VM/instance %s (%s: %s)',
vm_name,
type(e).__name__, str(e))
try:
provider_crud.mgmt.delete_vm(vm_name)
except Exception:
logger.exception("Unable to clean up vm:", vm_name)
raise
except skip_exceptions as e:
e_c = type(e)
if e_c in callable_mapping and not callable_mapping[e_c](e):
raise
# Make it visible also in the log.
store.write_line("Skipping due to a provider error: {}: {}\n".format(
e_c.__name__, str(e)),
purple=True)
logger.exception(e)
pytest.skip("{}: {}".format(e_c.__name__, str(e)))
return vm_name
def deploy_template(provider_key,
vm_name,
template_name=None,
timeout=900,
**deploy_args):
"""
Args:
provider_key: Provider key on which the VM is to be created
vm_name: Name of the VM to be deployed
template_name: Name of the template that the VM is deployed from
"""
allow_skip = deploy_args.pop("allow_skip", ())
if isinstance(allow_skip, dict):
skip_exceptions = allow_skip.keys()
callable_mapping = allow_skip
elif isinstance(allow_skip,
basestring) and allow_skip.lower() == "default":
skip_exceptions = (OSOverLimit, RHEVRequestError,
exceptions.VMInstanceNotCloned, SSLError)
callable_mapping = {}
else:
skip_exceptions = allow_skip
callable_mapping = {}
provider_crud = get_crud(provider_key)
mgmt = provider_crud.get_mgmt_system()
data = provider_crud.get_yaml_data()
deploy_args.update(vm_name=vm_name)
if template_name is None:
try:
deploy_args.update(template=data['small_template'])
except KeyError:
raise ValueError(
'small_template not defined for Provider {} in cfme_data.yaml'
).format(provider_key)
else:
deploy_args.update(template=template_name)
if isinstance(mgmt, RHEVMSystem):
if 'default_cluster' not in deploy_args:
deploy_args.update(cluster=data['default_cluster'])
elif isinstance(mgmt, VMWareSystem):
if "allowed_datastores" not in deploy_args and "allowed_datastores" in data:
deploy_args.update(allowed_datastores=data['allowed_datastores'])
elif isinstance(mgmt, SCVMMSystem):
if 'host_group' not in deploy_args:
deploy_args.update(host_group=data.get("host_group", "All Hosts"))
elif isinstance(mgmt, EC2System):
pass
elif isinstance(mgmt, OpenstackSystem):
if ('network_name' not in deploy_args) and data.get('network'):
deploy_args.update(network_name=data['network'])
else:
raise Exception("Unsupported provider type: %s" %
mgmt.__class__.__name__)
logger.info(
"Getting ready to deploy VM/instance %s from template %s on provider %s"
% (vm_name, deploy_args['template'], data['name']))
try:
try:
logger.debug("Deploy args: {}".format(deploy_args))
vm_name = mgmt.deploy_template(timeout=timeout, **deploy_args)
logger.info("Provisioned VM/instance %s" %
vm_name) # instance ID in case of EC2
except Exception as e:
logger.error(
'Could not provisioning VM/instance {} ({}: {})'.format(
vm_name,
type(e).__name__, str(e)))
_vm_cleanup(mgmt, vm_name)
raise
except skip_exceptions as e:
e_c = type(e)
if e_c in callable_mapping and not callable_mapping[e_c](e):
raise
# Make it visible also in the log.
store.write_line("Skipping due to a provider error: {}: {}\n".format(
e_c.__name__, str(e)),
purple=True)
pytest.skip("{}: {}".format(e_c.__name__, str(e)))
return vm_name
def deploy_template(provider_key,
vm_name,
template_name=None,
timeout=900,
**deploy_args):
allow_skip = deploy_args.pop("allow_skip", ())
if isinstance(allow_skip, dict):
skip_exceptions = allow_skip.keys()
callable_mapping = allow_skip
elif isinstance(allow_skip,
basestring) and allow_skip.lower() == "default":
skip_exceptions = (OSOverLimit, RHEVRequestError, VMInstanceNotCloned,
SSLError)
callable_mapping = {}
else:
skip_exceptions = allow_skip
callable_mapping = {}
provider_crud = get_crud(provider_key)
mgmt = provider_crud.get_mgmt_system()
data = provider_crud.get_yaml_data()
deploy_args.update(vm_name=vm_name)
if isinstance(mgmt, RHEVMSystem):
if 'default_cluster' not in deploy_args:
deploy_args.update(cluster=data['default_cluster'])
elif isinstance(mgmt, VMWareSystem):
if "allowed_datastores" not in deploy_args and "allowed_datastores" in data:
deploy_args.update(allowed_datastores=data['allowed_datastores'])
elif isinstance(mgmt, SCVMMSystem):
if 'host_group' not in deploy_args:
deploy_args.update(host_group=data.get("host_group", "All Hosts"))
elif isinstance(mgmt, EC2System):
pass
elif isinstance(mgmt, OpenstackSystem):
if ('network_name' not in deploy_args) and data.get('network'):
deploy_args.update(network_name=data['network'])
else:
raise Exception("Unsupported provider type: %s" %
mgmt.__class__.__name__)
if template_name is None:
template_name = data['small_template']
logger.info(
"Getting ready to deploy VM/instance %s from template %s on provider %s"
% (vm_name, template_name, data['name']))
try:
try:
logger.debug("Deploy args: " + str(deploy_args))
vm_name = mgmt.deploy_template(template_name,
timeout=timeout,
**deploy_args)
logger.info("Provisioned VM/instance %s" %
vm_name) # instance ID in case of EC2
except Exception as e:
logger.error('Could not provisioning VM/instance %s (%s)', vm_name,
e)
try:
logger.info("VM/Instance status: {}".format(
mgmt.vm_status(vm_name)))
except Exception as f:
logger.error(
"Could not retrieve VM/Instance status: {}: {}".format(
type(f).__name__, str(f)))
logger.info('Attempting cleanup on VM/instance %s', vm_name)
try:
if mgmt.does_vm_exist(vm_name):
# Stop the vm first
logger.warning('Destroying VM/instance %s', vm_name)
if mgmt.delete_vm(vm_name):
logger.info('VM/instance %s destroyed', vm_name)
else:
logger.error('Error destroying VM/instance %s',
vm_name)
except Exception as f:
logger.error(
'Could not destroy VM/instance {} ({}: {})'.format(
vm_name,
type(f).__name__, str(f)))
finally:
raise e
except skip_exceptions as e:
e_c = type(e)
if e_c in callable_mapping and not callable_mapping[e_c](e):
raise
# Make it visible also in the log.
store.write_line("Skipping due to a provider error: {}: {}\n".format(
e_c.__name__, str(e)),
purple=True)
pytest.skip("{}: {}".format(e_c.__name__, str(e)))
return vm_name
def appliance_police(appliance):
if not store.slave_manager:
return
try:
available_ports = {
'ssh': (appliance.hostname, appliance.ssh_port),
'https': (appliance.hostname, appliance.ui_port),
'postgres': (appliance.db_host
or appliance.hostname, appliance.db_port)
}
port_results = {
pn: net_check(addr=p_addr, port=p_port, force=True)
for pn, (p_addr, p_port) in available_ports.items()
}
for port, result in port_results.items():
if port == 'ssh' and appliance.is_pod:
# ssh is not available for podified appliance
continue
if not result:
raise AppliancePoliceException('Unable to connect',
available_ports[port][1])
try:
status_code = requests.get(appliance.url,
verify=False,
timeout=120).status_code
except Exception:
raise AppliancePoliceException('Getting status code failed',
available_ports['https'][1])
if status_code != 200:
raise AppliancePoliceException(
'Status code was {}, should be 200'.format(status_code),
available_ports['https'][1])
return
except AppliancePoliceException as e:
# special handling for known failure conditions
if e.port == 443:
# Lots of rdbs lately where evm seems to have entirely crashed
# and (sadly) the only fix is a rude restart
appliance.restart_evm_service(rude=True)
try:
appliance.wait_for_web_ui(900)
store.write_line('EVM was frozen and had to be restarted.',
purple=True)
return
except TimedOutError:
pass
e_message = str(e)
except Exception as e:
e_message = str(e)
# Regardles of the exception raised, we didn't return anywhere above
# time to call a human
msg = 'Help! My appliance {} crashed with: {}'.format(
appliance.url, e_message)
store.slave_manager.message(msg)
if 'appliance_police_recipients' in rdb:
rdb_kwargs = {
'subject': 'RDB Breakpoint: Appliance failure',
'recipients': rdb.appliance_police_recipients,
}
else:
rdb_kwargs = {}
Rdb(msg).set_trace(**rdb_kwargs)
store.slave_manager.message('Resuming testing following remote debugging')
def deploy_template(provider_key, vm_name, template_name=None, timeout=900, **deploy_args):
allow_skip = deploy_args.pop("allow_skip", ())
if isinstance(allow_skip, dict):
skip_exceptions = allow_skip.keys()
callable_mapping = allow_skip
elif isinstance(allow_skip, basestring) and allow_skip.lower() == "default":
skip_exceptions = (OSOverLimit, RHEVRequestError, VMInstanceNotCloned)
callable_mapping = {}
else:
skip_exceptions = allow_skip
callable_mapping = {}
provider_type = conf.cfme_data.get('management_systems', {})[provider_key]['type']
if provider_type in infra_provider_type_map:
provider_crud = get_infra_from_config(provider_key)
else:
provider_crud = get_cloud_from_config(provider_key)
mgmt = provider_crud.get_mgmt_system()
data = provider_crud.get_yaml_data()
deploy_args.update(vm_name=vm_name)
if isinstance(mgmt, RHEVMSystem):
if 'default_cluster' not in deploy_args:
deploy_args.update(cluster=data['default_cluster'])
elif isinstance(mgmt, VMWareSystem):
if "allowed_datastores" not in deploy_args and "allowed_datastores" in data:
deploy_args.update(allowed_datastores=data['allowed_datastores'])
elif isinstance(mgmt, SCVMMSystem):
if 'host_group' not in deploy_args:
deploy_args.update(host_group=data.get("host_group", "All Hosts"))
elif isinstance(mgmt, EC2System):
pass
elif isinstance(mgmt, OpenstackSystem):
if ('network_name' not in deploy_args) and data.get('network'):
deploy_args.update(network_name=data['network'])
else:
raise Exception("Unsupported provider type: %s" % mgmt.__class__.__name__)
if template_name is None:
template_name = data['small_template']
logger.info("Getting ready to deploy VM/instance %s from template %s on provider %s" %
(vm_name, template_name, data['name']))
try:
try:
logger.debug("Deploy args: " + str(deploy_args))
vm_name = mgmt.deploy_template(template_name, timeout=timeout, **deploy_args)
logger.info("Provisioned VM/instance %s" % vm_name) # instance ID in case of EC2
except Exception as e:
logger.error('Could not provisioning VM/instance %s (%s)', vm_name, e)
try:
logger.info("VM/Instance status: {}".format(mgmt.vm_status(vm_name)))
except Exception as f:
logger.error(
"Could not retrieve VM/Instance status: {}: {}".format(
type(f).__name__, str(f)))
logger.info('Attempting cleanup on VM/instance %s', vm_name)
try:
if mgmt.does_vm_exist(vm_name):
# Stop the vm first
logger.warning('Destroying VM/instance %s', vm_name)
if mgmt.delete_vm(vm_name):
logger.info('VM/instance %s destroyed', vm_name)
else:
logger.error('Error destroying VM/instance %s', vm_name)
except Exception as f:
logger.error(
'Could not destroy VM/instance {} ({}: {})'.format(
vm_name, type(f).__name__, str(f)))
finally:
raise e
except skip_exceptions as e:
e_c = type(e)
if e_c in callable_mapping and not callable_mapping[e_c](e):
raise
# Make it visible also in the log.
store.write_line(
"Skipping due to a provider error: {}: {}\n".format(e_c.__name__, str(e)), purple=True)
pytest.skip("{}: {}".format(e_c.__name__, str(e)))
return vm_name
def deploy_template(provider_key, vm_name, template_name=None, timeout=900,
**deploy_args):
"""
Args:
provider_key: Provider key on which the VM is to be created
vm_name: Name of the VM to be deployed
template_name: Name of the template that the VM is deployed from
"""
allow_skip = deploy_args.pop("allow_skip", ())
if isinstance(allow_skip, dict):
skip_exceptions = allow_skip.keys()
callable_mapping = allow_skip
elif isinstance(allow_skip, basestring) and allow_skip.lower() == "default":
skip_exceptions = (OSOverLimit, RHEVRequestError, exceptions.VMInstanceNotCloned, SSLError)
callable_mapping = {}
else:
skip_exceptions = allow_skip
callable_mapping = {}
provider_crud = get_crud(provider_key)
mgmt = provider_crud.get_mgmt_system()
data = provider_crud.get_yaml_data()
deploy_args.update(vm_name=vm_name)
if template_name is None:
try:
deploy_args.update(template=data['small_template'])
except KeyError:
raise ValueError('small_template not defined for Provider {} in cfme_data.yaml').format(
provider_key)
else:
deploy_args.update(template=template_name)
if isinstance(mgmt, RHEVMSystem):
if 'default_cluster' not in deploy_args:
deploy_args.update(cluster=data['default_cluster'])
elif isinstance(mgmt, VMWareSystem):
if "allowed_datastores" not in deploy_args and "allowed_datastores" in data:
deploy_args.update(allowed_datastores=data['allowed_datastores'])
elif isinstance(mgmt, SCVMMSystem):
if 'host_group' not in deploy_args:
deploy_args.update(host_group=data.get("host_group", "All Hosts"))
elif isinstance(mgmt, EC2System):
pass
elif isinstance(mgmt, OpenstackSystem):
if ('network_name' not in deploy_args) and data.get('network'):
deploy_args.update(network_name=data['network'])
else:
raise Exception("Unsupported provider type: %s" % mgmt.__class__.__name__)
logger.info("Getting ready to deploy VM/instance %s from template %s on provider %s" %
(vm_name, deploy_args['template'], data['name']))
try:
try:
logger.debug("Deploy args: {}".format(deploy_args))
vm_name = mgmt.deploy_template(timeout=timeout, **deploy_args)
logger.info("Provisioned VM/instance %s" % vm_name) # instance ID in case of EC2
except Exception as e:
logger.error('Could not provisioning VM/instance {} ({}: {})'.format(
vm_name, type(e).__name__, str(e)))
_vm_cleanup(mgmt, vm_name)
raise
except skip_exceptions as e:
e_c = type(e)
if e_c in callable_mapping and not callable_mapping[e_c](e):
raise
# Make it visible also in the log.
store.write_line(
"Skipping due to a provider error: {}: {}\n".format(e_c.__name__, str(e)), purple=True)
pytest.skip("{}: {}".format(e_c.__name__, str(e)))
return vm_name
