def allocate_packet_capturer(lock_manager, hostname, prefix):
"""Allocates a machine to capture packets.
Locks the allocated machine if the machine was discovered via AFE
to prevent tests stomping on each other.
@param lock_manager HostLockManager object.
@param hostname string optional hostname of a packet capture machine.
@param prefix string chamber location (ex. chromeos3, chromeos5, chromeos7)
@return: An SSHHost object representing a locked packet_capture machine.
"""
if hostname is not None:
return hosts.SSHHost(hostname)
afe = frontend.AFE(debug=True, server=site_utils.get_global_afe_hostname())
available_pcaps = afe.get_hosts(label='packet_capture')
for pcap in available_pcaps:
pcap_prefix = pcap.hostname.split('-')[0]
# Ensure the pcap and dut are in the same subnet
if pcap_prefix == prefix:
if lock_manager.lock([pcap.hostname]):
return hosts.SSHHost(pcap.hostname + '.cros')
else:
logging.info('Unable to lock %s', pcap.hostname)
continue
raise error.TestError(
'Unable to lock any pcaps - check in cautotest if '
'pcaps in %s are locked.', prefix)
def setup(self):
"""Construct the state used in a WiFi test."""
self._router = site_linux_router.build_router_proxy(
test_name=self._test_name,
client_hostname=self.client.host.hostname,
router_addr=self._cmdline_args.get(self.CMDLINE_ROUTER_ADDR, None))
# The attenuator host gives us the ability to attenuate particular
# antennas on the router. Most setups don't have this capability
# and most tests do not require it. We use this for RvR
# (network_WiFi_AttenuatedPerf) and some roaming tests.
attenuator_addr = dnsname_mangler.get_attenuator_addr(
self.client.host.hostname,
cmdline_override=self._cmdline_args.get(self.CMDLINE_ATTEN_ADDR,
None),
allow_failure=True)
ping_helper = ping_runner.PingRunner()
if attenuator_addr and ping_helper.simple_ping(attenuator_addr):
self._attenuator = attenuator_controller.AttenuatorController(
hosts.SSHHost(attenuator_addr, port=22))
# Set up a clean context to conduct WiFi tests in.
self.client.shill.init_test_network_state()
if self.CMDLINE_CLIENT_PACKET_CAPTURES in self._cmdline_args:
self._enable_client_packet_captures = True
if self.CMDLINE_ROUTER_PACKET_CAPTURES in self._cmdline_args:
self._enable_router_packet_captures = True
if self.CMDLINE_PACKET_CAPTURE_SNAPLEN in self._cmdline_args:
self._packet_capture_snaplen = int(
self._cmdline_args[self.CMDLINE_PACKET_CAPTURE_SNAPLEN])
self.client.conductive = self._get_bool_cmdline_value(
self.CMDLINE_CONDUCTIVE_RIG, None)
for system in (self.client, self.router):
system.sync_host_times()
def __init__(self,
omaha_host,
update_payload_staged_url,
max_updates=1,
critical_update=True):
"""Starts a private devserver instance, operating at Omaha capacity.
@param omaha_host: host address where the devserver is spawned.
@param update_payload_staged_url: URL to provision for update requests.
@param max_updates: int number of updates this devserver will handle.
This is passed to src/platform/dev/devserver.py.
@param critical_update: Whether to set a deadline in responses.
"""
self._devserver_dir = '/home/chromeos-test/chromiumos/src/platform/dev'
if not update_payload_staged_url:
raise error.TestError('Missing update payload url')
self._critical_update = critical_update
self._max_updates = max_updates
self._omaha_host = omaha_host
self._devserver_pid = 0
self._devserver_port = 0 # Determined later from devserver portfile.
self._update_payload_staged_url = update_payload_staged_url
self._devserver_ssh = hosts.SSHHost(self._omaha_host,
user='******')
# Temporary files for various devserver outputs.
self._devserver_logfile = None
self._devserver_stdoutfile = None
self._devserver_portfile = None
self._devserver_pidfile = None
self._devserver_static_dir = None
def create_host(hostname):
# TODO(crbug.com/739466) Delay import to avoid a ~0.7 second penalty
# drone_utility calls that don't actually interact with DUTs.
from autotest_lib.server import hosts
username = global_config.global_config.get_config_value(
'SCHEDULER', hostname + '_username', default=getpass.getuser())
return hosts.SSHHost(hostname, user=username)
def allocate_packet_capturer(lock_manager, hostname):
"""Allocates a machine to capture packets.
Locks the allocated machine if the machine was discovered via AFE
to prevent tests stomping on each other.
@param lock_manager HostLockManager object.
@param hostname string optional hostname of a packet capture machine.
@return: An SSHHost object representing a locked packet_capture machine.
"""
if hostname is not None:
return hosts.SSHHost(hostname)
afe = frontend.AFE(debug=True,
server=site_utils.get_global_afe_hostname())
return hosts.SSHHost(site_utils.lock_host_with_labels(
afe, lock_manager, labels=['packet_capture']) + '.cros')
def main(argv):
common_util.setup_logging()
args = common_util.parse_args(argv)
requested_server_set = set(args.servers)
devservers = devserver_list()
master, drones = autotest_scheduler_drones()
if args.operation == common_util.SYNC:
if common_util.DEVS in requested_server_set:
for server in devservers:
devserver_sync(hosts.SSHHost(server))
if common_util.DRONES in requested_server_set:
for server in drones:
autotest_sync(hosts.SSHHost(server))
if common_util.SCHEDULER in requested_server_set:
autotest_sync(master)
elif args.operation == common_util.RESTART:
if common_util.DEVS in requested_server_set:
for server in devservers:
devserver_restart(hosts.SSHHost(server))
if common_util.DRONES in requested_server_set:
for server in drones:
autotest_restart(hosts.SSHHost(server))
if common_util.SCHEDULER in requested_server_set:
autotest_restart(master)
elif args.operation == common_util.PRINT:
if common_util.DEVS in requested_server_set:
for server in devservers:
print server
if common_util.DRONES in requested_server_set:
for server in drones:
print server
if common_util.SCHEDULER in requested_server_set:
print master
return 0
def allocate_webdriver_instance(lock_manager):
"""Allocates a machine to capture webdriver instance.
Locks the allocated machine if the machine was discovered via AFE
to prevent tests stomping on each other.
@param lock_manager HostLockManager object.
@return An SSHHost object representing a locked webdriver instance.
"""
afe = frontend.AFE(debug=True, server=site_utils.get_global_afe_hostname())
hostname = '%s.cros' % site_utils.lock_host_with_labels(
afe, lock_manager, labels=['webdriver'])
webdriver_host = hosts.SSHHost(hostname)
if webdriver_host is not None:
return webdriver_host
logging.error("Unable to allocate VM instance")
return None
def run(self, job, batch_size=10, tries=10, capturer_hostname=None,
conn_worker=None, work_client_hostname=None,
disabled_sysinfo=False):
"""Executes Chaos test.
@param job: an Autotest job object.
@param batch_size: an integer, max number of APs to lock in one batch.
@param tries: an integer, number of iterations to run per AP.
@param capturer_hostname: a string or None, hostname or IP of capturer.
@param conn_worker: ConnectionWorkerAbstract or None, to run extra
work after successful connection.
@param work_client_hostname: a string or None, hostname of work client
@param disabled_sysinfo: a bool, disable collection of logs from DUT.
@raises TestError: Issues locking VM webdriver instance
"""
lock_manager = host_lock_manager.HostLockManager()
webdriver_master = hosts.SSHHost(MASTERNAME, user='******')
host_prefix = self._host.hostname.split('-')[0]
with host_lock_manager.HostsLockedBy(lock_manager):
capture_host = utils.allocate_packet_capturer(
lock_manager, hostname=capturer_hostname,
prefix=host_prefix)
# Cleanup and reboot packet capturer before the test.
utils.sanitize_client(capture_host)
capturer = site_linux_system.LinuxSystem(capture_host, {},
'packet_capturer')
# Run iw scan and abort if more than allowed number of APs are up.
iw_command = iw_runner.IwRunner(capture_host)
start_time = time.time()
logging.info('Performing a scan with a max timeout of 30 seconds.')
capture_interface = 'wlan0'
capturer_info = capture_host.run('cat /etc/lsb-release',
ignore_status=True, timeout=5).stdout
if 'whirlwind' in capturer_info:
# Use the dual band aux radio for scanning networks.
capture_interface = 'wlan2'
while time.time() - start_time <= ap_constants.MAX_SCAN_TIMEOUT:
networks = iw_command.scan(capture_interface)
if networks is None:
if (time.time() - start_time ==
ap_constants.MAX_SCAN_TIMEOUT):
raise error.TestError(
'Packet capturer is not responding to scans. Check'
'device and re-run test')
continue
elif len(networks) < ap_constants.MAX_SSID_COUNT:
break
elif len(networks) >= ap_constants.MAX_SSID_COUNT:
raise error.TestError(
'Probably someone is already running a '
'chaos test?!')
if conn_worker is not None:
work_client_machine = utils.allocate_packet_capturer(
lock_manager, hostname=work_client_hostname)
conn_worker.prepare_work_client(work_client_machine)
# Lock VM. If on, power off; always power on. Then create a tunnel.
webdriver_instance = utils.allocate_webdriver_instance(lock_manager)
if utils.is_VM_running(webdriver_master, webdriver_instance):
logging.info('VM %s was on; powering off for a clean instance',
webdriver_instance)
utils.power_off_VM(webdriver_master, webdriver_instance)
logging.info('Allow VM time to gracefully shut down')
time.sleep(5)
logging.info('Starting up VM %s', webdriver_instance)
utils.power_on_VM(webdriver_master, webdriver_instance)
logging.info('Allow VM time to power on before creating a tunnel.')
time.sleep(30)
if not client_utils.host_is_in_lab_zone(webdriver_instance.hostname):
self._ap_spec._webdriver_hostname = webdriver_instance.hostname
else:
# If in the lab then port forwarding must be done so webdriver
# connection will be over localhost.
self._ap_spec._webdriver_hostname = 'localhost'
webdriver_tunnel = webdriver_instance.create_ssh_tunnel(
WEBDRIVER_PORT, WEBDRIVER_PORT)
logging.info('Wait for tunnel to be created.')
for i in range(3):
time.sleep(10)
results = client_utils.run('lsof -i:%s' % WEBDRIVER_PORT,
ignore_status=True)
if results:
break
if not results:
raise error.TestError(
'Unable to listen to WEBDRIVER_PORT: %s', results)
batch_locker = ap_batch_locker.ApBatchLocker(
lock_manager, self._ap_spec,
ap_test_type=ap_constants.AP_TEST_TYPE_CHAOS)
while batch_locker.has_more_aps():
# Work around for CrOS devices only:crbug.com/358716
utils.sanitize_client(self._host)
healthy_dut = True
with contextlib.closing(wifi_client.WiFiClient(
hosts.create_host(
{
'hostname' : self._host.hostname,
'afe_host' : self._host._afe_host,
'host_info_store':
self._host.host_info_store,
},
host_class=self._host.__class__,
),
'./debug',
False,
)) as client:
aps = batch_locker.get_ap_batch(batch_size=batch_size)
if not aps:
logging.info('No more APs to test.')
break
# Power down all of the APs because some can get grumpy
# if they are configured several times and remain on.
# User the cartridge to down group power downs and
# configurations.
utils.power_down_aps(aps, self._broken_pdus)
utils.configure_aps(aps, self._ap_spec, self._broken_pdus)
aps = utils.filter_quarantined_and_config_failed_aps(aps,
batch_locker, job, self._broken_pdus)
for ap in aps:
# http://crbug.com/306687
if ap.ssid == None:
logging.error('The SSID was not set for the AP:%s',
ap)
healthy_dut = utils.is_dut_healthy(client, ap)
if not healthy_dut:
logging.error('DUT is not healthy, rebooting.')
batch_locker.unlock_and_reclaim_aps()
break
networks = utils.return_available_networks(
ap, capturer, job, self._ap_spec)
if networks is None:
# If scan returned no networks, iw scan failed.
# Reboot the packet capturer device and
# reconfigure the capturer.
batch_locker.unlock_and_reclaim_ap(ap.host_name)
logging.error('Packet capture is not healthy, '
'rebooting.')
capturer.host.reboot()
capturer = site_linux_system.LinuxSystem(
capture_host, {},'packet_capturer')
continue
if networks == list():
# Packet capturer did not find the SSID in scan or
# there was a security mismatch.
utils.release_ap(ap, batch_locker, self._broken_pdus)
continue
assoc_params = ap.get_association_parameters()
if not utils.is_conn_worker_healthy(
conn_worker, ap, assoc_params, job):
utils.release_ap(
ap, batch_locker, self._broken_pdus)
continue
name = ap.name
kernel_ver = self._host.get_kernel_ver()
firmware_ver = utils.get_firmware_ver(self._host)
if not firmware_ver:
firmware_ver = "Unknown"
debug_dict = {'+++PARSE DATA+++': '+++PARSE DATA+++',
'SSID': ap._ssid,
'DUT': client.wifi_mac,
'AP Info': ap.name,
'kernel_version': kernel_ver,
'wifi_firmware_version': firmware_ver}
debug_string = pprint.pformat(debug_dict)
logging.info('Waiting %d seconds for the AP dhcp '
'server', ap.dhcp_delay)
time.sleep(ap.dhcp_delay)
result = job.run_test(self._test,
capturer=capturer,
capturer_frequency=networks[0].frequency,
capturer_ht_type=networks[0].ht,
host=self._host,
assoc_params=assoc_params,
client=client,
tries=tries,
debug_info=debug_string,
# Copy all logs from the system
disabled_sysinfo=disabled_sysinfo,
conn_worker=conn_worker,
tag=ap.ssid if conn_worker is None else
'%s.%s' % (conn_worker.name, ap.ssid))
utils.release_ap(ap, batch_locker, self._broken_pdus)
if conn_worker is not None:
conn_worker.cleanup()
if not healthy_dut:
continue
batch_locker.unlock_aps()
if webdriver_tunnel:
webdriver_instance.disconnect_ssh_tunnel(webdriver_tunnel,
WEBDRIVER_PORT)
webdriver_instance.close()
capturer.close()
logging.info('Powering off VM %s', webdriver_instance)
utils.power_off_VM(webdriver_master, webdriver_instance)
lock_manager.unlock(webdriver_instance.hostname)
if self._broken_pdus:
logging.info('PDU is down!!!\nThe following PDUs are down:\n')
pprint.pprint(self._broken_pdus)
factory = ap_configurator_factory.APConfiguratorFactory(
ap_constants.AP_TEST_TYPE_CHAOS)
factory.turn_off_all_routers(self._broken_pdus)
def create_host(hostname):
username = global_config.global_config.get_config_value(
'SCHEDULER', hostname + '_username', default=getpass.getuser())
return hosts.SSHHost(hostname, user=username)
def run(self,
job,
batch_size=7,
tries=10,
capturer_hostname=None,
conn_worker=None,
work_client_hostname=None,
disabled_sysinfo=False):
"""Executes Chaos test.
@param job: an Autotest job object.
@param batch_size: an integer, max number of APs to lock in one batch.
@param tries: an integer, number of iterations to run per AP.
@param capturer_hostname: a string or None, hostname or IP of capturer.
@param conn_worker: ConnectionWorkerAbstract or None, to run extra
work after successful connection.
@param work_client_hostname: a string or None, hostname of work client
@param disabled_sysinfo: a bool, disable collection of logs from DUT.
@raises TestError: Issues locking VM webdriver instance
"""
lock_manager = host_lock_manager.HostLockManager()
webdriver_master = hosts.SSHHost(MASTERNAME, user='******')
with host_lock_manager.HostsLockedBy(lock_manager):
capture_host = utils.allocate_packet_capturer(
lock_manager, hostname=capturer_hostname)
# Cleanup and reboot packet capturer before the test.
utils.sanitize_client(capture_host)
capturer = site_linux_system.LinuxSystem(capture_host, {},
'packet_capturer')
# Run iw scan and abort if more than allowed number of APs are up.
iw_command = iw_runner.IwRunner(capture_host)
start_time = time.time()
logging.info('Performing a scan with a max timeout of 30 seconds.')
while time.time() - start_time <= ap_constants.MAX_SCAN_TIMEOUT:
networks = iw_command.scan('wlan0')
if networks is None:
if (time.time() -
start_time == ap_constants.MAX_SCAN_TIMEOUT):
raise error.TestError(
'Packet capturer is not responding to scans. Check'
'device and re-run test')
continue
elif len(networks) < ap_constants.MAX_SSID_COUNT:
break
elif len(networks) >= ap_constants.MAX_SSID_COUNT:
raise error.TestError(
'Probably someone is already running a'
'chaos test?!')
if conn_worker is not None:
work_client_machine = utils.allocate_packet_capturer(
lock_manager, hostname=work_client_hostname)
conn_worker.prepare_work_client(work_client_machine)
webdriver_instance = utils.allocate_webdriver_instance(
lock_manager)
self._ap_spec._webdriver_hostname = webdriver_instance
# If a test is cancelled or aborted the VM may be left on. Always
# turn of the VM to return it to a clean state.
try:
logging.info('Always power off VM %s', webdriver_instance)
utils.power_off_VM(webdriver_master, webdriver_instance)
except:
logging.debug('VM was already off, ignoring.')
logging.info('Starting up VM %s', webdriver_instance)
utils.power_on_VM(webdriver_master, webdriver_instance)
batch_locker = ap_batch_locker.ApBatchLocker(
lock_manager,
self._ap_spec,
ap_test_type=ap_constants.AP_TEST_TYPE_CHAOS)
while batch_locker.has_more_aps():
# Work around crbug.com/358716
utils.sanitize_client(self._host)
healthy_dut = True
with contextlib.closing(
wifi_client.WiFiClient(
hosts.create_host(self._host.hostname), './debug',
False)) as client:
aps = batch_locker.get_ap_batch(batch_size=batch_size)
if not aps:
logging.info('No more APs to test.')
break
# Power down all of the APs because some can get grumpy
# if they are configured several times and remain on.
# User the cartridge to down group power downs and
# configurations.
utils.power_down_aps(aps, self._broken_pdus)
utils.configure_aps(aps, self._ap_spec, self._broken_pdus)
aps = utils.filter_quarantined_and_config_failed_aps(
aps, batch_locker, job, self._broken_pdus)
for ap in aps:
# http://crbug.com/306687
if ap.ssid == None:
logging.error('The SSID was not set for the AP:%s',
ap)
healthy_dut = utils.is_dut_healthy(client, ap)
if not healthy_dut:
logging.error('DUT is not healthy, rebooting.')
batch_locker.unlock_and_reclaim_aps()
break
networks = utils.return_available_networks(
ap, capturer, job, self._ap_spec)
if networks is None:
# If scan returned no networks, iw scan failed.
# Reboot the packet capturer device and
# reconfigure the capturer.
batch_locker.unlock_and_reclaim_ap(ap.host_name)
logging.error('Packet capture is not healthy, '
'rebooting.')
capturer.host.reboot()
capturer = site_linux_system.LinuxSystem(
capture_host, {}, 'packet_capturer')
continue
if networks == list():
# Packet capturer did not find the SSID in scan or
# there was a security mismatch.
utils.release_ap(ap, batch_locker,
self._broken_pdus)
continue
assoc_params = ap.get_association_parameters()
if not utils.is_conn_worker_healthy(
conn_worker, ap, assoc_params, job):
utils.release_ap(ap, batch_locker,
self._broken_pdus)
continue
name = ap.name
kernel_ver = self._host.get_kernel_ver()
firmware_ver = utils.get_firmware_ver(self._host)
if not firmware_ver:
firmware_ver = "Unknown"
debug_dict = {
'+++PARSE DATA+++': '+++PARSE DATA+++',
'SSID': ap._ssid,
'DUT': client.wifi_mac,
'AP Info': ap.name,
'kernel_version': kernel_ver,
'wifi_firmware_version': firmware_ver
}
debug_string = pprint.pformat(debug_dict)
logging.info(
'Waiting %d seconds for the AP dhcp '
'server', ap.dhcp_delay)
time.sleep(ap.dhcp_delay)
result = job.run_test(
self._test,
capturer=capturer,
capturer_frequency=networks[0].frequency,
capturer_ht_type=networks[0].ht,
host=self._host,
assoc_params=assoc_params,
client=client,
tries=tries,
debug_info=debug_string,
# Copy all logs from the system
disabled_sysinfo=disabled_sysinfo,
conn_worker=conn_worker,
tag=ap.ssid if conn_worker is None else '%s.%s' %
(conn_worker.name, ap.ssid))
utils.release_ap(ap, batch_locker, self._broken_pdus)
if conn_worker is not None:
conn_worker.cleanup()
if not healthy_dut:
continue
batch_locker.unlock_aps()
capturer.close()
logging.info('Powering off VM %s', webdriver_instance)
utils.power_off_VM(webdriver_master, webdriver_instance)
lock_manager.unlock(webdriver_instance)
if self._broken_pdus:
logging.info('PDU is down!!!\nThe following PDUs are down:\n')
pprint.pprint(self._broken_pdus)
