def test_job_create_singlenode(self): """ Test the creation of a LavaTestJob within the dispatcher to identify issues before the job has started to run. """ factory = DispatcherFactory() target = factory.create_qemu_target(self.name, {}) self.assertEqual(target.config.hostname, self.name) json_str = factory.singlenode_jobdata() self.assertNotEqual(json_str, None) jobdata = json.loads(json_str) self.assertEqual(jobdata['health_check'], False) validate_job_data(jobdata) # single node self.assertNotIn('target_group', jobdata) self.assertNotIn('is_vmhost', jobdata) job = LavaTestJob(json_str, sys.stderr, get_config(), None) self.assertEqual(job.target, target.config.hostname) self.assertIsNotNone(get_all_cmds()) # FIXME: would be useful to not have the metadata population only accessible via LavaTestJob.run() job.run() self.assertEqual(job.context.test_data.metadata['target'], self.name) self.assertEqual(job.context.test_data.metadata['target.hostname'], self.name) self.assertNotIn('is_vmhost', job.context.test_data.metadata) self.assertNotIn('host_ip', job.context.test_data.metadata) self.assertNotIn('target_group', job.context.test_data.metadata) self.assertNotIn('vm_group', job.context.test_data.metadata)
def validate_job_data(job_data): schema = Schema(job_schema) Validator.validate(schema, job_data) lava_commands = get_all_cmds() for action in job_data['actions']: command_name = action['command'] command = lava_commands.get(command_name) if command is None: raise ValueError("action %r not known" % command_name) command.validate_parameters(action.get('parameters'))
def run(self, transport=None, group_data=None, vm_host_ip=None): self.context.assign_transport(transport) self.context.assign_group_data(group_data) validate_job_data(self.job_data) self._set_logging_level() lava_commands = get_all_cmds() if self.job_data['actions'][-1]['command'].startswith( "submit_results"): submit_results = self.job_data['actions'].pop(-1) else: submit_results = None metadata = { 'target.hostname': self.target, } if 'device_type' in self.job_data: metadata['target.device_type'] = self.job_data['device_type'] self.context.test_data.add_metadata(metadata) self.context.test_data.add_tags(self.tags) if 'target' in self.job_data: metadata['target'] = self.job_data['target'] self.context.test_data.add_metadata(metadata) if 'logging_level' in self.job_data: metadata['logging_level'] = self.job_data['logging_level'] self.context.test_data.add_metadata(metadata) if 'is_vmhost' in self.job_data: metadata['is_vmhost'] = "true" if self.job_data['is_vmhost'] else "false" metadata['host_ip'] = str(vm_host_ip) logging.debug("[ACTION-B] VM group test!") if not self.job_data['is_vmhost']: logging.debug("[ACTION-B] VM host IP is (%s)." % metadata['host_ip']) if 'auto_start_vms' in self.job_data: metadata['auto_start_vms'] = str(self.job_data['auto_start_vms']).lower() else: metadata['auto_start_vms'] = 'true' self.context.test_data.add_metadata(metadata) if 'target_group' in self.job_data: metadata['target_group'] = self.job_data['target_group'] if 'is_slave' in self.job_data: metadata['is_slave'] = 'true' if self.job_data.get('is_slave') else 'false' self.context.test_data.add_metadata(metadata) if 'role' in self.job_data: metadata['role'] = self.job_data['role'] self.context.test_data.add_metadata(metadata) if 'group_size' in self.job_data: metadata['group_size'] = self.job_data['group_size'] self.context.test_data.add_metadata(metadata) logging.debug("[ACTION-B] Multi Node test!") logging.debug("[ACTION-B] target_group is (%s)." % self.context.test_data.metadata['target_group']) else: logging.debug("[ACTION-B] Single node test!") if 'lmp_module' in self.job_data: metadata['lmp_module'] = json.dumps(self.job_data['lmp_module']) self.context.test_data.add_metadata(metadata) # init LMP module lmp_init_boards.init(self.job_data['lmp_module'], self.context.device_config) def term_handler(signum, frame): self.context.finish() sys.exit(1) signal.signal(signal.SIGTERM, term_handler) try: job_length = len(self.job_data['actions']) job_num = 0 for cmd in self.job_data['actions']: job_num += 1 params = cmd.get('parameters', {}) if cmd.get('command').startswith('lava_android_test'): if not params.get('timeout') and \ self.job_data.get('timeout'): params['timeout'] = self.job_data['timeout'] logging.info("[ACTION-B] %s is started with %s" % (cmd['command'], params)) metadata = cmd.get('metadata', {}) self.context.test_data.add_metadata(metadata) action = lava_commands[cmd['command']](self.context) err = None try: status = 'fail' action.run(**params) except ADBConnectError as err: logging.info("ADBConnectError") if cmd.get('command') == 'boot_linaro_android_image': logging.warning(('[ACTION-E] %s failed to create the' ' adb connection') % (cmd['command'])) # Sometimes the adb problem is caused by the adb # command, and as workround we need to kill the adb # process to make it work logging.warning( 'Now will try to kill the adb process') rc = commands.getstatusoutput('adb devices')[0] if rc != 0: kill_process_with_option(process="adb", key_option="fork-server") # clear the session on the serial and wait a while # and not put the following 3 sentences into the # boot_linaro_android_image method just for # avoiding effects when the method being called # in other places logging.warning( 'Now will reboot the image to try again') self.context.client.proc.sendcontrol("c") self.context.client.proc.sendline("") time.sleep(5) self.context.client.boot_linaro_android_image( adb_check=True) # mark it as pass if the second boot works status = 'pass' except TimeoutError as err: logging.info("TimeoutError") if cmd.get('command').startswith('lava_android_test'): logging.warning("[ACTION-E] %s times out." % (cmd['command'])) if job_num == job_length: # not reboot the android image for # the last test action pass else: # clear the session on the serial and wait a while # and not put the following 3 sentences into the # boot_linaro_android_image method just for # avoiding effects when the method being called # in other places logging.warning( "Now the android image will be rebooted") self.context.client.proc.sendcontrol("c") self.context.client.proc.sendline("") time.sleep(5) self.context.client.boot_linaro_android_image() else: logging.warn("Unhandled timeout condition") continue except CriticalError as err: logging.info("CriticalError") raise except (pexpect.TIMEOUT, GeneralError) as err: logging.warn("pexpect timed out with status %s" % status) pass except KeyboardInterrupt: logging.info("Cancel operation") err = "Cancel" pass except subprocess.CalledProcessError as err: if err.output is not None: logging.info("Command error code: %d, with stdout/stderr:" % (err.returncode)) for line in err.output.rstrip('\n').split('\n'): logging.info("| > %s" % (line)) else: logging.info("Command error code: %d, without stdout/stderr" % (err.returncode)) raise except Exception as err: logging.info("General Exception: %s" % unicode(str(err))) raise else: logging.debug("setting status pass") status = 'pass' finally: logging.debug("finally status %s" % status) err_msg = "" if status == 'fail': # XXX mwhudson, 2013-01-17: I have no idea what this # code is doing. logging.warning( "[ACTION-E] %s is finished with error (%s)." % (cmd['command'], err)) err_msg = ("Lava failed at action %s with error:" "%s\n") % (cmd['command'], unicode(str(err), 'ascii', 'replace')) if cmd['command'] == 'lava_test_run': err_msg += "Lava failed on test: %s" % \ params.get('test_name', "Unknown") if err and err.message != "Cancel" and err.message != 'Timeout': err_msg = err_msg + traceback.format_exc() self.context.log("ErrorMessage: %s" % unicode(str(err))) self.context.log(err_msg) else: logging.info( "[ACTION-E] %s is finished successfully." % (cmd['command'])) err_msg = "" self.context.test_data.add_result( action.test_name(**params), status, err_msg) except: # Capture all user-defined and non-user-defined critical errors self.context.test_data.job_status = 'fail' raise finally: self.context.finish() device_version = self.context.get_device_version() or 'error' self.context.test_data.add_metadata({ 'target.device_version': device_version }) if 'target_group' in self.job_data: # all nodes call aggregate, even if there is no submit_results command self._aggregate_bundle(transport, lava_commands, submit_results) elif submit_results: params = submit_results.get('parameters', {}) action = lava_commands[submit_results['command']]( self.context) params_for_display = params.copy() if 'token' in params_for_display: params_for_display['token'] = '<HIDDEN>' try: logging.info("Submitting the test result with parameters = %s", params_for_display) action.run(**params) except Exception as err: logging.error("Failed to submit the test result. Error = %s", err) raise
def run(self, transport=None, group_data=None, vm_host_ip=None): if group_data: logging.debug("Group initialisation: %s" % json.dumps(group_data)) self.context.assign_transport(transport) self.context.assign_group_data(group_data) validate_job_data(self.job_data) self._set_logging_level() lava_commands = get_all_cmds() lmp_init_data = [] if self.job_data['actions'][-1]['command'].startswith( "submit_results"): submit_results = self.job_data['actions'].pop(-1) else: submit_results = None metadata = { 'target.hostname': self.target, } if 'device_type' in self.job_data: metadata['target.device_type'] = self.job_data['device_type'] self.context.test_data.add_metadata(metadata) self.context.test_data.add_tags(self.tags) if 'target' in self.job_data: metadata['target'] = self.job_data['target'] self.context.test_data.add_metadata(metadata) if 'logging_level' in self.job_data: metadata['logging_level'] = self.job_data['logging_level'] self.context.test_data.add_metadata(metadata) if 'is_vmhost' in self.job_data: metadata['is_vmhost'] = "true" if self.job_data[ 'is_vmhost'] else "false" metadata['host_ip'] = str(vm_host_ip) logging.debug("[ACTION-B] VM group test!") if not self.job_data['is_vmhost']: logging.debug("[ACTION-B] VM host IP is (%s)." % metadata['host_ip']) if 'auto_start_vms' in self.job_data: metadata['auto_start_vms'] = str( self.job_data['auto_start_vms']).lower() else: metadata['auto_start_vms'] = 'true' self.context.test_data.add_metadata(metadata) if 'target_group' in self.job_data: metadata['target_group'] = self.job_data['target_group'] if 'is_slave' in self.job_data: metadata['is_slave'] = 'true' if self.job_data.get( 'is_slave') else 'false' self.context.test_data.add_metadata(metadata) if 'role' in self.job_data: metadata['role'] = self.job_data['role'] self.context.test_data.add_metadata(metadata) if 'group_size' in self.job_data: metadata['group_size'] = self.job_data['group_size'] self.context.test_data.add_metadata(metadata) if 'shared_config' in self.job_data: metadata['shared_config'] = self.job_data['shared_config'] self.context.test_data.add_metadata(metadata) logging.debug("[ACTION-B] Multi Node test!") logging.debug("[ACTION-B] target_group is (%s)." % self.context.test_data.metadata['target_group']) else: logging.debug("[ACTION-B] Single node test!") # get LMP init data, if it exists. if 'lmp_module' in self.job_data: lmp_init_data = self.job_data['lmp_module'] metadata['lmp_module'] = json.dumps(lmp_init_data) self.context.test_data.add_metadata(metadata) # integrate LMP init data to LMP default config lmp_init_data = lmp_init_boards.data_integrate( lmp_init_data, self.context.device_config) # init LMP module, if necessary if lmp_init_data is not []: lmp_init_boards.init(lmp_init_data, self.context.device_config) def term_handler(signum, frame): self.context.finish() sys.exit(1) signal.signal(signal.SIGTERM, term_handler) try: job_length = len(self.job_data['actions']) job_num = 0 for cmd in self.job_data['actions']: job_num += 1 params = cmd.get('parameters', {}) if cmd.get('command').startswith('lava_android_test'): if not params.get('timeout') and \ self.job_data.get('timeout'): params['timeout'] = self.job_data['timeout'] logging.info("[ACTION-B] %s is started with %s" % (cmd['command'], params)) metadata = cmd.get('metadata', {}) self.context.test_data.add_metadata(metadata) action = lava_commands[cmd['command']](self.context) err = None try: status = 'fail' action.run(**params) except ADBConnectError as err: logging.info("ADBConnectError") if cmd.get('command') == 'boot_linaro_android_image': logging.warning(('[ACTION-E] %s failed to create the' ' adb connection') % (cmd['command'])) # Sometimes the adb problem is caused by the adb # command, and as workround we need to kill the adb # process to make it work logging.warning('Now will try to kill the adb process') rc = commands.getstatusoutput('adb devices')[0] if rc != 0: kill_process_with_option(process="adb", key_option="fork-server") # clear the session on the serial and wait a while # and not put the following 3 sentences into the # boot_linaro_android_image method just for # avoiding effects when the method being called # in other places logging.warning( 'Now will reboot the image to try again') self.context.client.proc.sendcontrol("c") self.context.client.proc.sendline("") time.sleep(5) self.context.client.boot_linaro_android_image( adb_check=True) # mark it as pass if the second boot works status = 'pass' except TimeoutError as err: logging.info("TimeoutError") if cmd.get('command').startswith('lava_android_test'): logging.warning("[ACTION-E] %s times out." % (cmd['command'])) if job_num == job_length: # not reboot the android image for # the last test action pass else: # clear the session on the serial and wait a while # and not put the following 3 sentences into the # boot_linaro_android_image method just for # avoiding effects when the method being called # in other places logging.warning( "Now the android image will be rebooted") self.context.client.proc.sendcontrol("c") self.context.client.proc.sendline("") time.sleep(5) self.context.client.boot_linaro_android_image() else: logging.warning("Unhandled timeout condition") continue except CriticalError as err: logging.info("CriticalError") raise except (pexpect.TIMEOUT, GeneralError) as err: logging.warning("pexpect timed out with status %s" % status) pass except KeyboardInterrupt: logging.info("Cancel operation") err = "Cancel" pass except subprocess.CalledProcessError as err: if err.output is not None: logging.info( "Command error code: %d, with stdout/stderr:" % (err.returncode)) for line in err.output.rstrip('\n').split('\n'): logging.info("| > %s" % (line)) else: logging.info( "Command error code: %d, without stdout/stderr" % (err.returncode)) raise except Exception as err: logging.info("General Exception: %s" % unicode(str(err))) raise else: logging.debug("setting status pass") status = 'pass' finally: logging.debug("finally status %s" % status) err_msg = "" if status == 'fail': # XXX mwhudson, 2013-01-17: I have no idea what this # code is doing. logging.warning( "[ACTION-E] %s is finished with error (%s)." % (cmd['command'], err)) err_msg = ("Lava failed at action %s with error:" "%s\n") % (cmd['command'], unicode(str(err), 'ascii', 'replace')) if cmd['command'] == 'lava_test_run': err_msg += "Lava failed on test: %s" % \ params.get('test_name', "Unknown") if err and err.message != "Cancel" and err.message != 'Timeout': err_msg = err_msg + traceback.format_exc() self.context.log("ErrorMessage: %s" % unicode(str(err))) self.context.log(err_msg) else: logging.info( "[ACTION-E] %s is finished successfully." % (cmd['command'])) err_msg = "" self.context.test_data.add_result( action.test_name(**params), status, message=err_msg) except: # Capture all user-defined and non-user-defined critical errors self.context.test_data.job_status = 'fail' raise finally: self.context.finish() device_version = self.context.get_device_version() or 'error' self.context.test_data.add_metadata( {'target.device_version': device_version}) if 'target_group' in self.job_data: # all nodes call aggregate, even if there is no submit_results command self._aggregate_bundle(transport, lava_commands, submit_results) elif submit_results: params = submit_results.get('parameters', {}) action = lava_commands[submit_results['command']](self.context) params_for_display = params.copy() if 'token' in params_for_display: params_for_display['token'] = '<HIDDEN>' try: logging.info( "Submitting the test result with parameters = %s", params_for_display) action.run(**params) except Exception as err: logging.error( "Failed to submit the test result. Error = %s", err) raise