def check_all(args):
"""
Checks that configurations are valid for all the devices either fast or
accurately.
The difference between these is whether tests are run parallel or serial.
Parallel testing may cause false positives (everything appears to be Ok)
when device configurations are mixed. For example, if two devices have
their power cutter settings mixed (powering on device 1 actually powers
device 2 and vice versa), everything could appear to be ok during parallel
testing as both devices would be powered on roughly at the same time.
Args
args (configuration object): Program command line arguments
Returns:
None
"""
if not args.topology:
raise errors.AFTConfigurationError("Topology file must be specified")
manager = DevicesManager(args)
configs = manager.get_configs()
if args.checkall == "fast":
return check_all_parallel(args, configs)
elif args.checkall == "accurate":
return check_all_serial(args, configs)
else:
raise errors.AFTConfigurationError("Invalid option " + args.checkall)
def check_poweron(self):
"""
Checks if device powers on sucessfully by checking if it enters DFU mode
correctly
Returns:
None
Raises:
aft.errors.AFTDeviceError on failure to connect to the device after
running out of retries
aft.errors.AFTConfigurationError if for some reason all retries fail
and no other exception is raised
"""
attempts = 3
exception = None
for i in range(attempts):
logger.info("Attempt " + str(i + 1) + " of " + str(attempts) +
" to power on the device " +
self._configuration["name"])
try:
self._power_cycle()
self._wait_for_device()
except errors.AFTDeviceError as error:
exception = error
pass
else:
return
if exception:
raise exception
raise errors.AFTConfigurationError("Failed to power on the device")
def check_poweron(self):
"""
Check that PEM can be connected into. The device powers PEM, so this
is a good sign that the device is powered on
Returns:
None
Raises:
aft.errors.AFTConfigurationError if device fails to power on
"""
self._power_cycle()
attempts = 2
attempt_timeout = 60
try:
self._send_PEM_keystrokes(self._config_check_keystrokes,
attempts=attempts,
timeout=attempt_timeout)
except errors.AFTDeviceError:
raise errors.AFTConfigurationError(
"Could not connect to PEM - check power and pem settings and "
+ "connections")
def check_poweroff(self):
"""
Check that PEM is offline by checking that the PEM process is still
alive after a timeout (failed to connect and send the keystrokes)
Returns:
None
Raises:
aft.errors.AFTConfigurationError if PEM process has terminated
before the timeout
"""
# run Device class power of tests as well
super(PCDevice, self).check_poweroff()
try:
self._send_PEM_keystrokes(self._config_check_keystrokes,
timeout=20)
except errors.AFTDeviceError as err:
return
raise errors.AFTConfigurationError(
"Device seems to have failed to shut down - " +
"PEM is still accessible")
def reserve_specific(self, machine_name, timeout=3600, model=None):
"""
Reserve and lock a specific device. If model is given, check if
the device is the given model
"""
# Basically very similar to a reserve-method
# we just populate they devices array with a single device
devices = []
for device_config in self.device_configs:
if device_config["name"].lower() == machine_name.lower():
cutter = devicefactory.build_cutter(device_config["settings"])
kb_emulator = devicefactory.build_kb_emulator(
device_config["settings"])
device = devicefactory.build_device(device_config["settings"],
cutter, kb_emulator)
devices.append(device)
break
#Check if device is a given model
if model and len(devices):
if not devices[0].model.lower() == model.lower():
raise errors.AFTConfigurationError(
"Device and machine doesn't match")
return self._do_reserve(devices, machine_name, timeout)
def __init__(self, device):
self._device = device
self.test_cases = []
self._results = []
self._start_time = None
self._end_time = None
test_plan_name = device.test_plan
test_plan_file = os.path.join("/etc/aft/test_plan/",
device.test_plan + ".cfg")
test_plan_config = ConfigParser.SafeConfigParser()
test_plan_config.read(test_plan_file)
if len(test_plan_config.sections()) == 0:
raise errors.AFTConfigurationError(
"Test plan " + str(test_plan_name) + " (" +
str(test_plan_file) + ") doesn't " +
"have any test cases. Does the file exist?")
for test_case_name in test_plan_config.sections():
test_case_config = dict(test_plan_config.items(test_case_name))
test_case_config["name"] = test_case_name
test_case = aft.testcasefactory.build_test_case(test_case_config)
self.test_cases.append(test_case)
logger.info("Built test plan with " + str(len(self.test_cases)) +
" test cases.")
def check(args):
"""
Checks that the specified device is configured correctly
Args:
args (configuration object): Program command line arguments
Returns:
Tuple (Bool, String): Test status code and result message string. True
indicates tests passed succesfully, false indicates that there were
failures
"""
if not args.device:
raise errors.AFTConfigurationError(
"You must specify the device that will be checked")
if args.verbose:
print("Running configuration check on " + args.device)
if args.checkall:
logger.init_thread(args.device + "_")
logger.info("Running configuration check on " + args.device)
manager = DevicesManager(args)
device = manager.reserve_specific(args.device)
if args.verbose:
print("Device " + args.device + " acquired, running checks")
try:
sanity_results = _run_sanity_tests(args, device)
image_test_results = (True, "Image Test result: Not run")
# only run image test if sanity test passed
if sanity_results[0] == True:
image_test_results = _run_tests_on_know_good_image(args, device)
finally:
if args.verbose:
print("Releasing device " + args.device)
if not args.nopoweroff:
device.detach()
manager.release(device)
results = (sanity_results[0] and image_test_results[0],
sanity_results[1] + "\n" + image_test_results[1])
if not results[0]:
common.blacklist_device(device.dev_id, args.device,
"Failed device health check")
msg = "Device " + args.device + " failed health test - blacklisting"
logger.info(msg)
if args.verbose:
print msg
return results
def check(args):
"""
Checks that the specified device is configured correctly
Args:
args (configuration object): Program command line arguments
Returns:
Tuple (Bool, String): Test status code and result message string. True
indicates tests passed succesfully, false indicates that there were
failures
"""
if not args.device:
raise errors.AFTConfigurationError(
"You must specify the device that will be checked")
if args.verbose:
print "Running configuration check on " + args.device
logging.info("Running configuration check on " + args.device)
manager = DevicesManager(args)
device = manager.reserve_specific(args.device)
if args.verbose:
print "Device " + args.device + " acquired, running checks"
test_results = _run_tests(args, device)
if args.verbose:
print "Releasing device " + args.device
manager.release(device)
results = _handle_test_results(args, device, test_results)
if not results[0]:
common.blacklist_device(
device.dev_id,
args.device,
"Failed device health check")
msg = "Device " + args.device + " failed health test - blacklisting"
logging.info(msg)
if args.verbose:
print msg
return results
def record_serial(self):
"""
Start a serialrecorder.py subprocess and add its killer
atexit handles
"""
if not ("serial_port" in self.parameters
and "serial_bauds" in self.parameters):
raise errors.AFTConfigurationError("Configuration for device " +
self.name + " doesn't include " +
"serial_port and/or serial_bauds.")
recorder = subprocess32.Popen(["python",
os.path.join(os.path.dirname(__file__), "tools",
"serialrecorder.py"),
self.parameters["serial_port"],
self.parameters["serial_log_name"],
"--rate",
self.parameters["serial_bauds"]])
atexit.register(misc.subprocess_killer, recorder)
def record_serial(self):
"""
Start a serialrecorder.py subprocess and add its killer
atexit handles
"""
if not ("serial_port" in self.parameters
and "serial_bauds" in self.parameters):
raise errors.AFTConfigurationError("Configuration for device " +
self.name + " doesn't include " +
"serial_port and/or serial_bauds.")
recorder = threading.Thread(target=serialrecorder.main,
args=(self.parameters["serial_port"],
self.parameters["serial_bauds"],
self.parameters["serial_log_name"]),
name=(str(os.getpid()) + "recorder"))
recorder.start()
thread_handler.add_thread(recorder)
def check_poweroff(self):
"""
Checks that device was powered down by checking that attempt to enter
into DFU mode fails.
Returns:
None
Raises:
aft.errors.AFTConfigurationError if the device succesfully entered
DFU mode
"""
self.detach()
try:
self._wait_for_device()
except errors.AFTDeviceError as error:
pass
else:
raise errors.AFTConfigurationError("The device seems to be on")
def _do_reserve(self, devices, name, timeout):
devices = self._remove_blacklisted_devices(devices)
if len(devices) == 0:
raise errors.AFTConfigurationError(
"No device configurations when reserving " + name +
" - check that given machine type or name is correct")
start = time.time()
while time.time() - start < timeout:
for device in devices:
logging.info("Attempting to acquire " + device.name)
try:
# This is a non-atomic operation which may cause trouble
# Using a locking database system could be a viable fix.
lockfile = os.fdopen(
os.open(
os.path.join(config.LOCK_FILE,
"aft_" + device.dev_id),
os.O_WRONLY | os.O_CREAT, 0660), "w")
fcntl.flock(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
logging.info("Device acquired.")
self._lockfiles.append((device.dev_id, lockfile))
atexit.register(self.release, device)
return device
except IOError as err:
if err.errno in {errno.EACCES, errno.EAGAIN}:
logging.info("Device busy.")
else:
logging.critical("Cannot obtain lock file.")
sys.exit(-1)
logging.info(
"All devices busy ... waiting 10 seconds and trying again.")
time.sleep(10)
raise errors.AFTTimeoutError("Could not reserve " + name + " in " +
str(timeout) + " seconds.")
def check_poweroff(self):
"""
Checks that device has been powered off by checking that ip no longer
can be acquired
"""
# Note that this test assumes that the device ip is not available when
# the device is powered off. This is not valid assumption, for example,
# for Edison.
sleep_delay = 30
logger.info("Powering down the device and waiting for " +
str(sleep_delay) + " seconds")
self.detach()
sleep(30)
logger.info("Attempting to acquire ip")
ip = self.get_ip()
if ip:
raise errors.AFTConfigurationError("Failed to power off device")
logger.info("No ip could be acquired - device seems to be powered off")
def _construct_configs(self):
"""
Find and merge the device configurations into single data structure.
Returns:
List of dictionaries, where dictionaries have the following format:
{
"name": "device_name",
"model": "device_model",
"settings": {
"device_specific_setting1": "value1",
"device_specific_setting2": "value2",
"platform_setting1": "value3",
"catalog_setting1": "value4",
}
}
Note:
catalog\platform entries are not device specific, but these are
duplicated for each device for ease of access.
"""
platform_config_file = self.__PLATFORM_FILE_NAME
catalog_config_file = self._args.catalog
topology_config_file = self._args.topology
platform_config = ConfigParser.SafeConfigParser()
platform_config.read(platform_config_file)
catalog_config = ConfigParser.SafeConfigParser()
catalog_config.read(catalog_config_file)
topology_config = ConfigParser.SafeConfigParser()
topology_config.read(topology_config_file)
configs = []
for device_title in topology_config.sections():
device_entry = dict(topology_config.items(device_title))
model = device_entry["model"]
catalog_entry = dict(catalog_config.items(model))
platform = catalog_entry["platform"]
platform_entry = dict(platform_config.items(platform))
settings = {}
# note the order: more specific file overrides changes from
# more generic. This should be maintained
settings.update(platform_entry)
settings.update(catalog_entry)
settings.update(device_entry)
settings["name"] = device_title.lower()
settings["serial_log_name"] = config.SERIAL_LOG_NAME
device_param = {}
device_param["name"] = device_title.lower()
device_param["model"] = model.lower()
device_param["settings"] = settings
configs.append(device_param)
if len(configs) == 0:
raise errors.AFTConfigurationError(
"Zero device configurations built - is this really correct? " +
"Check that paths for topology, catalog and platform files "
"are correct and that the files have some settings inside")
logger.info("Built configuration sets for " + str(len(configs)) +
" devices")
return configs
None
"""
try:
self._enter_mode(self._service_mode)
except KeyboardInterrupt:
pass
except Exception, error:
exception_queue.put(error)
process = Process(target=invoker, args=(exception_queue, ))
process.start()
process.join(1.5 * self._RETRY_ATTEMPTS * self._BOOT_TIMEOUT)
if process.is_alive():
process.terminate()
raise errors.AFTConfigurationError(
"Timeout - PEM likely failed to connect")
if not exception_queue.empty():
raise exception_queue.get()
logging.info("Succesfully booted device into service mode")
def check_poweroff(self):
"""
Check that PEM is offline by checking that the PEM process is still
alive after a timeout (failed to connect and send the keystrokes)
Returns:
None
Raises:
def check_connection(self):
"""
Boot into service mode, and check if ssh connection can be established
Returns:
None
Raises:
aft.errors.AFTConfigurationError on timeout
"""
# set the retry count to lower value, as otherwise on failing device
# this stage can take up to 2*retry_count*boot timeout seconds
# (with values 8 and 240 that would be 3840 seconds or 64 minutes!)
# retry count should be > 1 so that the occasional failed boot won't
# fail the test
self._RETRY_ATTEMPTS = 3
# run in a process, as pem itself has no timeout and if there is a
# connection or configuration issue, it will get stuck.
# Queue is used to pass any exceptions from the subprocess back to main
# process
exception_queue = Queue()
def invoker(exception_queue):
"""
Helper function for process invocation.
Attempts to enter service mode, and catches any exceptions and
puts them in a Queue
Args:
exception_queue (multiprocessing.Queue):
The queue used to return the exceptions back to the main
process
Returns:
None
"""
try:
self._enter_mode(self._service_mode)
except KeyboardInterrupt:
pass
except Exception as error:
exception_queue.put(error)
process = Process(target=invoker, args=(exception_queue, ))
process.start()
process.join(1.5 * self._RETRY_ATTEMPTS * self._BOOT_TIMEOUT)
if process.is_alive():
process.terminate()
raise errors.AFTConfigurationError(
"Timeout - PEM likely failed to connect")
if not exception_queue.empty():
raise exception_queue.get()
logger.info("Succesfully booted device into service mode")
logging.info("Attempt " + str(i + 1) + " of " + str(attempts) +
" to power on the device " + self._configuration["name"])
try:
self._power_cycle()
self._wait_for_device()
except errors.AFTDeviceError, error:
exception = error
pass
else:
return
if exception:
raise exception
raise errors.AFTConfigurationError("Failed to power on the device")
def check_connection(self):
"""
Checks the connectivity by checking if an interface could be opened
Returns:
None
Raises:
aft.errors.AFTConfigurationError if interface could not be opened
"""
attempts = 3
for i in range(attempts):
