def connect(self, hostname, *args, **kwargs):
"""See paramiko.SSHClient.connect"""
port = int(kwargs.get('port', 22))
if not net_check(port, hostname):
raise Exception("Connection to %s is not available as port %d is unavailable"
% (hostname, port))
super(SSHClient, self).connect(hostname, *args, **kwargs)
def run(**kwargs):
thread_queue = []
for provider in list_providers("openstack"):
mgmt_sys = cfme_data['management_systems'][provider]
rhos_credentials = credentials[mgmt_sys['credentials']]
default_host_creds = credentials['host_default']
username = rhos_credentials['username']
password = rhos_credentials['password']
auth_url = mgmt_sys['auth_url']
rhosip = mgmt_sys['ipaddress']
sshname = default_host_creds['username']
sshpass = default_host_creds['password']
if not net.is_pingable(rhosip):
continue
if not net.net_check(ports.SSH, rhosip):
print("SSH connection to {}:{} failed, port unavailable".format(
provider, ports.SSH))
continue
thread = Thread(target=upload_template,
args=(rhosip, sshname, sshpass, username, password, auth_url, provider,
kwargs.get('image_url'), kwargs.get('template_name')))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def connect(self, hostname, *args, **kwargs):
"""See paramiko.SSHClient.connect"""
port = int(kwargs.get('port', 22))
if not net_check(port, hostname):
raise Exception("Connection to %s is not available as port %d is unavailable"
% (hostname, port))
super(SSHClient, self).connect(hostname, *args, **kwargs)
def appliance_police():
if not store.slave_manager:
return
try:
port_numbers = {
'ssh': ports.SSH,
'https': store.current_appliance.ui_port,
'postgres': ports.DB}
port_results = {pn: net_check(pp, force=True) for pn, pp in port_numbers.items()}
for port, result in port_results.items():
if not result:
raise _AppliancePoliceException('Port {} was not contactable'.format(port),
port_numbers[port])
try:
status_code = requests.get(store.current_appliance.url, verify=False,
timeout=120).status_code
except Exception:
raise _AppliancePoliceException('Getting status code failed', port_numbers['https'])
if status_code != 200:
raise _AppliancePoliceException('Status code was {}, should be 200'.format(
status_code), port_numbers['https'])
return
except _AppliancePoliceException as e:
# special handling for known failure conditions
if e.port == 443:
# Lots of rdbs lately where evm seems to have entirely crashed
# and (sadly) the only fix is a rude restart
store.current_appliance.restart_evm_service(rude=True)
try:
store.current_appliance.wait_for_web_ui(900)
store.write_line('EVM was frozen and had to be restarted.', purple=True)
return
except TimedOutError:
pass
e_message = e.message
except Exception as e:
e_message = e.args[0]
# Regardles of the exception raised, we didn't return anywhere above
# time to call a human
msg = 'Help! My appliance {} crashed with: {}'.format(store.current_appliance.url, e_message)
store.slave_manager.message(msg)
if 'appliance_police_recipients' in rdb:
rdb_kwargs = {
'subject': 'RDB Breakpoint: Appliance failure',
'recipients': rdb.appliance_police_recipients,
}
else:
rdb_kwargs = {}
Rdb(msg).set_trace(**rdb_kwargs)
store.slave_manager.message('Resuming testing following remote debugging')
def appliance_police():
if not store.slave_manager:
return
try:
ports = {'ssh': 22, 'https': 443, 'postgres': 5432}
port_results = {pn: net_check(pp, force=True) for pn, pp in ports.items()}
for port, result in port_results.items():
if not result:
raise _AppliancePoliceException('Port {} was not contactable'.format(port), port)
status_code = requests.get(store.current_appliance.url, verify=False,
timeout=60).status_code
if status_code != 200:
raise _AppliancePoliceException('Status code was {}, should be 200'.format(
status_code), port)
return
except _AppliancePoliceException as e:
# special handling for known failure conditions
if e.port == 443:
# if the web ui worker merely crashed, give it 15 minutes
# to come back up
try:
store.current_appliance.wait_for_web_ui(900)
return
except TimedOutError:
# the UI didn't come back up after 15 minutes, and is
# probably frozen; kill it and restart
# fortunately we already check SSH is working...
store.current_appliance.restart_evm_service(900, rude=True)
# take another shot at letting the web UI come up
try:
store.current_appliance.wait_for_web_ui(900)
return
except TimedOutError:
# so much for that
pass
e_message = e.message
except Exception as e:
e_message = e.args[0]
# Regardles of the exception raised, we didn't return anywhere above
# time to call a human
msg = 'Help! My appliance {} crashed with: {}'.format(store.current_appliance.url, e_message)
store.slave_manager.message(msg)
Rdb(msg).set_trace(**{
'subject': 'RDB Breakpoint: Appliance failure',
'recipients': ['*****@*****.**', '*****@*****.**'],
})
store.slave_manager.message('Resuming testing following remote debugging')
def appliance_police():
if not store.slave_manager:
return
try:
port_numbers = {
'ssh': ports.SSH,
'https': store.current_appliance.ui_port,
'postgres': ports.DB}
port_results = {pn: net_check(pp, force=True) for pn, pp in port_numbers.items()}
for port, result in port_results.items():
if not result:
raise _AppliancePoliceException('Port {} was not contactable'.format(port),
port_numbers[port])
try:
status_code = requests.get(store.current_appliance.url, verify=False,
timeout=120).status_code
except Exception:
raise _AppliancePoliceException('Getting status code failed', port_numbers['https'])
if status_code != 200:
raise _AppliancePoliceException('Status code was {}, should be 200'.format(
status_code), port_numbers['https'])
return
except _AppliancePoliceException as e:
# special handling for known failure conditions
if e.port == 443:
# If we had an error on 443, about 101% of the time it means the UI worker is frozen
store.current_appliance.ssh_client().run_rails_command('MiqUiWorker.first.kill')
try:
store.current_appliance.wait_for_web_ui(900)
store.write_line('UI worker was frozen and had to be restarted.', purple=True)
return
except TimedOutError:
pass
e_message = e.message
except Exception as e:
e_message = e.args[0]
# Regardles of the exception raised, we didn't return anywhere above
# time to call a human
msg = 'Help! My appliance {} crashed with: {}'.format(store.current_appliance.url, e_message)
store.slave_manager.message(msg)
Rdb(msg).set_trace(**{
'subject': 'RDB Breakpoint: Appliance failure',
'recipients': ['*****@*****.**', '*****@*****.**'],
})
store.slave_manager.message('Resuming testing following remote debugging')
def run(**kwargs):
thread_queue = []
providers = list_provider_keys("openstack")
if kwargs['provider_data']:
provider_data = kwargs['provider_data']
mgmt_sys = providers = provider_data['management_systems']
for provider in providers:
if kwargs['provider_data']:
if mgmt_sys[provider]['type'] != 'openstack':
continue
username = mgmt_sys[provider]['username']
password = mgmt_sys[provider]['password']
sshname = mgmt_sys[provider]['sshname']
sshpass = mgmt_sys[provider]['sshpass']
else:
mgmt_sys = cfme_data['management_systems']
rhos_credentials = credentials[mgmt_sys[provider]['credentials']]
default_host_creds = credentials['host_default']
username = rhos_credentials['username']
password = rhos_credentials['password']
sshname = default_host_creds['username']
sshpass = default_host_creds['password']
rhosip = mgmt_sys[provider]['ipaddress']
auth_url = mgmt_sys[provider]['auth_url']
if not net.is_pingable(rhosip):
continue
if not net.net_check(ports.SSH, rhosip):
print("SSH connection to {}:{} failed, port unavailable".format(
provider, ports.SSH))
continue
thread = Thread(target=upload_template,
args=(rhosip, sshname, sshpass, username, password,
auth_url, provider, kwargs.get('image_url'),
kwargs.get('template_name'),
kwargs['provider_data'], kwargs['stream']))
thread.daemon = True
thread_queue.append(thread)
thread.start()
for thread in thread_queue:
thread.join()
def _check_port(self):
hostname = self._connect_kwargs['hostname']
if not net_check(ports.SSH, hostname, force=True):
raise Exception("SSH connection to {}:{} failed, port unavailable".format(
hostname, ports.SSH))
def _check_port(self):
hostname = self._connect_kwargs['hostname']
if not net_check(ports.SSH, hostname, force=True):
raise Exception("SSH connection to {}:{} failed, port unavailable".format(
hostname, ports.SSH))
interaction.add_argument('--image', help='Choose WebDriver port',
default=docker_conf.get('selff', 'cfme/sel_ff_chrome'))
args = parser.parse_args()
print("Starting container...")
dkb = SeleniumDocker(bindings={'VNC_PORT': (5999, args.vnc),
'WEBDRIVER': (4444, args.webdriver)},
image=args.image)
dkb.run()
if args.watch:
print
print(" Waiting for VNC port to open...")
wait_for(lambda: net_check(args.vnc, '127.0.0.1'), num_sec=60)
time.sleep(10)
print net_check(args.vnc, '127.0.0.1')
print(" Initiating VNC watching...")
ipport = "vnc://127.0.0.1:" + str(args.vnc)
cmd = ['xdg-open', ipport]
subprocess.Popen(cmd)
print(" Hit Ctrl+C to end container")
try:
dkb.wait()
except KeyboardInterrupt:
print(" Killing Container.....please wait...")
pass
dkb.kill()
def _check_port(self):
hostname = self._connect_kwargs['hostname']
port = int(self._connect_kwargs.get('port', 22))
if not net_check(port, hostname):
raise Exception("SSH connection to %s:%d failed, port unavailable".format(
hostname, port))