def _pick_responding_ip(vm, port):
for ip in _trying_ips(vm):
if net_check(port, ip):
return ip
else:
raise TimedOutError(
f"Coudln't find an IP of vm {vm} with port {port} responding")
def appliance_police(appliance):
if not store.slave_manager:
return
try:
available_ports = {
'ssh': (appliance.hostname, appliance.ssh_port),
'https': (appliance.hostname, appliance.ui_port),
'postgres': (appliance.db_host or appliance.hostname, appliance.db_port)}
port_results = {pn: net_check(addr=p_addr, port=p_port, force=True)
for pn, (p_addr, p_port) in available_ports.items()}
for port, result in port_results.items():
if port == 'ssh' and appliance.is_pod:
# ssh is not available for podified appliance
continue
if not result:
raise AppliancePoliceException('Unable to connect', available_ports[port][1])
try:
status_code = requests.get(appliance.url, verify=False,
timeout=120).status_code
except Exception:
raise AppliancePoliceException('Getting status code failed',
available_ports['https'][1])
if status_code != 200:
raise AppliancePoliceException('Status code was {}, should be 200'.format(
status_code), available_ports['https'][1])
return
except AppliancePoliceException as e:
# special handling for known failure conditions
if e.port == 443:
# Lots of rdbs lately where evm seems to have entirely crashed
# and (sadly) the only fix is a rude restart
appliance.restart_evm_service(rude=True)
try:
appliance.wait_for_web_ui(900)
store.write_line('EVM was frozen and had to be restarted.', purple=True)
return
except TimedOutError:
pass
e_message = str(e)
except Exception as e:
e_message = str(e)
# Regardles of the exception raised, we didn't return anywhere above
# time to call a human
msg = 'Help! My appliance {} crashed with: {}'.format(appliance.url, e_message)
store.slave_manager.message(msg)
if 'appliance_police_recipients' in rdb:
rdb_kwargs = {
'subject': 'RDB Breakpoint: Appliance failure',
'recipients': rdb.appliance_police_recipients,
}
else:
rdb_kwargs = {}
Rdb(msg).set_trace(**rdb_kwargs)
store.slave_manager.message('Resuming testing following remote debugging')
def run(**kwargs):
thread_queue = []
providers = list_provider_keys("openstack")
if kwargs['provider_data']:
provider_data = kwargs['provider_data']
mgmt_sys = providers = provider_data['management_systems']
else:
mgmt_sys = cfme_data.management_systems
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload')
and mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info(
'Skipping upload on {} due to block_upload'.format(provider))
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'openstack':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
else:
mgmt_sys = cfme_data['management_systems']
rhos_credentials = credentials[mgmt_sys[provider]['credentials']]
default_host_creds = credentials['host_default']
username = rhos_credentials['username']
password = rhos_credentials['password']
sshname = default_host_creds['username']
sshpass = default_host_creds['password']
rhosip = mgmt_sys[provider]['ipaddress']
auth_url = mgmt_sys[provider]['auth_url']
if not net.is_pingable(rhosip):
continue
if not net.net_check(ports.SSH, rhosip):
logger.error("SSH connection to %r:%r failed, port unavailable",
provider, ports.SSH)
continue
thread = Thread(target=upload_template,
args=(rhosip, sshname, sshpass, username, password,
auth_url, provider, kwargs.get('image_url'),
kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def run(**kwargs):
thread_queue = []
providers = list_provider_keys("openstack")
if kwargs['provider_data']:
provider_data = kwargs['provider_data']
mgmt_sys = providers = provider_data['management_systems']
else:
mgmt_sys = cfme_data.management_systems
for provider in providers:
# skip provider if block_upload is set
if (mgmt_sys[provider].get('template_upload') and
mgmt_sys[provider]['template_upload'].get('block_upload')):
logger.info('Skipping upload on {} due to block_upload'.format(provider))
continue
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'openstack':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
else:
mgmt_sys = cfme_data['management_systems']
rhos_credentials = credentials[mgmt_sys[provider]['credentials']]
default_host_creds = credentials['host_default']
username = rhos_credentials['username']
password = rhos_credentials['password']
sshname = default_host_creds['username']
sshpass = default_host_creds['password']
rhosip = mgmt_sys[provider]['ipaddress']
auth_url = mgmt_sys[provider]['auth_url']
if not net.is_pingable(rhosip):
continue
if not net.net_check(ports.SSH, rhosip):
logger.error("SSH connection to %r:%r failed, port unavailable", provider, ports.SSH)
continue
thread = Thread(target=upload_template,
args=(rhosip, sshname, sshpass, username, password, auth_url, provider,
kwargs.get('image_url'), kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def _check_port(self):
hostname = self._connect_kwargs['hostname']
if not net_check(ports.SSH, hostname, force=True):
raise Exception(
"SSH connection to {}:{} failed, port unavailable".format(
hostname, ports.SSH))
def _check_port(self):
hostname = self._connect_kwargs['hostname']
if not net_check(ports.SSH, hostname, force=True):
raise Exception("SSH connection to {}:{} failed, port unavailable".format(
hostname, ports.SSH))
