Beispiel #1
0
 def test_job_create_singlenode(self):
     """
     Test the creation of a LavaTestJob within the
     dispatcher to identify issues before the job has
     started to run.
     """
     factory = DispatcherFactory()
     target = factory.create_qemu_target(self.name, {})
     self.assertEqual(target.config.hostname, self.name)
     json_str = factory.singlenode_jobdata()
     self.assertNotEqual(json_str, None)
     jobdata = json.loads(json_str)
     self.assertEqual(jobdata['health_check'], False)
     validate_job_data(jobdata)
     # single node
     self.assertNotIn('target_group', jobdata)
     self.assertNotIn('is_vmhost', jobdata)
     job = LavaTestJob(json_str, sys.stderr, get_config(), None)
     self.assertEqual(job.target, target.config.hostname)
     self.assertIsNotNone(get_all_cmds())
     # FIXME: would be useful to not have the metadata population only accessible via LavaTestJob.run()
     job.run()
     self.assertEqual(job.context.test_data.metadata['target'], self.name)
     self.assertEqual(job.context.test_data.metadata['target.hostname'], self.name)
     self.assertNotIn('is_vmhost', job.context.test_data.metadata)
     self.assertNotIn('host_ip', job.context.test_data.metadata)
     self.assertNotIn('target_group', job.context.test_data.metadata)
     self.assertNotIn('vm_group', job.context.test_data.metadata)
Beispiel #2
0
def validate_job_data(job_data):
    schema = Schema(job_schema)
    Validator.validate(schema, job_data)
    lava_commands = get_all_cmds()
    for action in job_data['actions']:
        command_name = action['command']
        command = lava_commands.get(command_name)
        if command is None:
            raise ValueError("action %r not known" % command_name)
        command.validate_parameters(action.get('parameters'))
def validate_job_data(job_data):
    schema = Schema(job_schema)
    Validator.validate(schema, job_data)
    lava_commands = get_all_cmds()
    for action in job_data['actions']:
        command_name = action['command']
        command = lava_commands.get(command_name)
        if command is None:
            raise ValueError("action %r not known" % command_name)
        command.validate_parameters(action.get('parameters'))
Beispiel #4
0
    def run(self, transport=None, group_data=None, vm_host_ip=None):
        self.context.assign_transport(transport)
        self.context.assign_group_data(group_data)
        validate_job_data(self.job_data)
        self._set_logging_level()
        lava_commands = get_all_cmds()

        if self.job_data['actions'][-1]['command'].startswith(
                "submit_results"):
            submit_results = self.job_data['actions'].pop(-1)
        else:
            submit_results = None

        metadata = {
            'target.hostname': self.target,
        }

        if 'device_type' in self.job_data:
            metadata['target.device_type'] = self.job_data['device_type']
        self.context.test_data.add_metadata(metadata)

        self.context.test_data.add_tags(self.tags)

        if 'target' in self.job_data:
            metadata['target'] = self.job_data['target']
            self.context.test_data.add_metadata(metadata)

        if 'logging_level' in self.job_data:
            metadata['logging_level'] = self.job_data['logging_level']
            self.context.test_data.add_metadata(metadata)

        if 'is_vmhost' in self.job_data:
            metadata['is_vmhost'] = "true" if self.job_data['is_vmhost'] else "false"
            metadata['host_ip'] = str(vm_host_ip)
            logging.debug("[ACTION-B] VM group test!")
            if not self.job_data['is_vmhost']:
                logging.debug("[ACTION-B] VM host IP is (%s)." % metadata['host_ip'])

            if 'auto_start_vms' in self.job_data:
                metadata['auto_start_vms'] = str(self.job_data['auto_start_vms']).lower()
            else:
                metadata['auto_start_vms'] = 'true'

            self.context.test_data.add_metadata(metadata)

        if 'target_group' in self.job_data:
            metadata['target_group'] = self.job_data['target_group']
            if 'is_slave' in self.job_data:
                metadata['is_slave'] = 'true' if self.job_data.get('is_slave') else 'false'
            self.context.test_data.add_metadata(metadata)

            if 'role' in self.job_data:
                metadata['role'] = self.job_data['role']
                self.context.test_data.add_metadata(metadata)

            if 'group_size' in self.job_data:
                metadata['group_size'] = self.job_data['group_size']
                self.context.test_data.add_metadata(metadata)

            logging.debug("[ACTION-B] Multi Node test!")
            logging.debug("[ACTION-B] target_group is (%s)." % self.context.test_data.metadata['target_group'])
        else:
            logging.debug("[ACTION-B] Single node test!")

        if 'lmp_module' in self.job_data:
            metadata['lmp_module'] = json.dumps(self.job_data['lmp_module'])
            self.context.test_data.add_metadata(metadata)
            # init LMP module
            lmp_init_boards.init(self.job_data['lmp_module'],
                                 self.context.device_config)

        def term_handler(signum, frame):
            self.context.finish()
            sys.exit(1)

        signal.signal(signal.SIGTERM, term_handler)

        try:
            job_length = len(self.job_data['actions'])
            job_num = 0
            for cmd in self.job_data['actions']:
                job_num += 1
                params = cmd.get('parameters', {})
                if cmd.get('command').startswith('lava_android_test'):
                    if not params.get('timeout') and \
                       self.job_data.get('timeout'):
                        params['timeout'] = self.job_data['timeout']
                logging.info("[ACTION-B] %s is started with %s" %
                             (cmd['command'], params))
                metadata = cmd.get('metadata', {})
                self.context.test_data.add_metadata(metadata)
                action = lava_commands[cmd['command']](self.context)
                err = None
                try:
                    status = 'fail'
                    action.run(**params)
                except ADBConnectError as err:
                    logging.info("ADBConnectError")
                    if cmd.get('command') == 'boot_linaro_android_image':
                        logging.warning(('[ACTION-E] %s failed to create the'
                                         ' adb connection') % (cmd['command']))

                        # Sometimes the adb problem is caused by the adb
                        # command, and as workround we need to kill the adb
                        # process to make it work
                        logging.warning(
                            'Now will try to kill the adb process')
                        rc = commands.getstatusoutput('adb devices')[0]
                        if rc != 0:
                            kill_process_with_option(process="adb",
                                                     key_option="fork-server")

                        # clear the session on the serial and wait a while
                        # and not put the following 3 sentences into the
                        # boot_linaro_android_image method just for
                        # avoiding effects when the method being called
                        # in other places
                        logging.warning(
                            'Now will reboot the image to try again')
                        self.context.client.proc.sendcontrol("c")
                        self.context.client.proc.sendline("")
                        time.sleep(5)
                        self.context.client.boot_linaro_android_image(
                            adb_check=True)
                        # mark it as pass if the second boot works
                        status = 'pass'
                except TimeoutError as err:
                    logging.info("TimeoutError")
                    if cmd.get('command').startswith('lava_android_test'):
                        logging.warning("[ACTION-E] %s times out." %
                                        (cmd['command']))
                        if job_num == job_length:
                            # not reboot the android image for
                            # the last test action
                            pass
                        else:
                            # clear the session on the serial and wait a while
                            # and not put the following 3 sentences into the
                            # boot_linaro_android_image method just for
                            # avoiding effects when the method being called
                            # in other places
                            logging.warning(
                                "Now the android image will be rebooted")
                            self.context.client.proc.sendcontrol("c")
                            self.context.client.proc.sendline("")
                            time.sleep(5)
                            self.context.client.boot_linaro_android_image()
                    else:
                        logging.warn("Unhandled timeout condition")
                        continue
                except CriticalError as err:
                    logging.info("CriticalError")
                    raise
                except (pexpect.TIMEOUT, GeneralError) as err:
                    logging.warn("pexpect timed out with status %s" % status)
                    pass
                except KeyboardInterrupt:
                    logging.info("Cancel operation")
                    err = "Cancel"
                    pass
                except subprocess.CalledProcessError as err:
                    if err.output is not None:
                        logging.info("Command error code: %d, with stdout/stderr:" % (err.returncode))
                        for line in err.output.rstrip('\n').split('\n'):
                            logging.info("| > %s" % (line))
                    else:
                        logging.info("Command error code: %d, without stdout/stderr" % (err.returncode))
                    raise

                except Exception as err:
                    logging.info("General Exception: %s" % unicode(str(err)))
                    raise
                else:
                    logging.debug("setting status pass")
                    status = 'pass'
                finally:
                    logging.debug("finally status %s" % status)
                    err_msg = ""
                    if status == 'fail':
                        # XXX mwhudson, 2013-01-17: I have no idea what this
                        # code is doing.
                        logging.warning(
                            "[ACTION-E] %s is finished with error (%s)." %
                            (cmd['command'], err))
                        err_msg = ("Lava failed at action %s with error:"
                                   "%s\n") % (cmd['command'],
                                              unicode(str(err),
                                                      'ascii', 'replace'))
                        if cmd['command'] == 'lava_test_run':
                            err_msg += "Lava failed on test: %s" % \
                                       params.get('test_name', "Unknown")
                        if err and err.message != "Cancel" and err.message != 'Timeout':
                            err_msg = err_msg + traceback.format_exc()
                            self.context.log("ErrorMessage: %s" % unicode(str(err)))
                        self.context.log(err_msg)
                    else:
                        logging.info(
                            "[ACTION-E] %s is finished successfully." %
                            (cmd['command']))
                        err_msg = ""
                    self.context.test_data.add_result(
                        action.test_name(**params), status, err_msg)
        except:
            # Capture all user-defined and non-user-defined critical errors
            self.context.test_data.job_status = 'fail'
            raise
        finally:
            self.context.finish()
            device_version = self.context.get_device_version() or 'error'
            self.context.test_data.add_metadata({
                'target.device_version': device_version
            })
            if 'target_group' in self.job_data:
                # all nodes call aggregate, even if there is no submit_results command
                self._aggregate_bundle(transport, lava_commands, submit_results)
            elif submit_results:
                params = submit_results.get('parameters', {})
                action = lava_commands[submit_results['command']](
                    self.context)
                params_for_display = params.copy()
                if 'token' in params_for_display:
                    params_for_display['token'] = '<HIDDEN>'
                try:
                    logging.info("Submitting the test result with parameters = %s", params_for_display)
                    action.run(**params)
                except Exception as err:
                    logging.error("Failed to submit the test result. Error = %s", err)
                    raise
    def run(self, transport=None, group_data=None, vm_host_ip=None):
        if group_data:
            logging.debug("Group initialisation: %s" % json.dumps(group_data))
        self.context.assign_transport(transport)
        self.context.assign_group_data(group_data)
        validate_job_data(self.job_data)
        self._set_logging_level()
        lava_commands = get_all_cmds()
        lmp_init_data = []

        if self.job_data['actions'][-1]['command'].startswith(
                "submit_results"):
            submit_results = self.job_data['actions'].pop(-1)
        else:
            submit_results = None

        metadata = {
            'target.hostname': self.target,
        }

        if 'device_type' in self.job_data:
            metadata['target.device_type'] = self.job_data['device_type']
        self.context.test_data.add_metadata(metadata)

        self.context.test_data.add_tags(self.tags)

        if 'target' in self.job_data:
            metadata['target'] = self.job_data['target']
            self.context.test_data.add_metadata(metadata)

        if 'logging_level' in self.job_data:
            metadata['logging_level'] = self.job_data['logging_level']
            self.context.test_data.add_metadata(metadata)

        if 'is_vmhost' in self.job_data:
            metadata['is_vmhost'] = "true" if self.job_data[
                'is_vmhost'] else "false"
            metadata['host_ip'] = str(vm_host_ip)
            logging.debug("[ACTION-B] VM group test!")
            if not self.job_data['is_vmhost']:
                logging.debug("[ACTION-B] VM host IP is (%s)." %
                              metadata['host_ip'])

            if 'auto_start_vms' in self.job_data:
                metadata['auto_start_vms'] = str(
                    self.job_data['auto_start_vms']).lower()
            else:
                metadata['auto_start_vms'] = 'true'

            self.context.test_data.add_metadata(metadata)

        if 'target_group' in self.job_data:
            metadata['target_group'] = self.job_data['target_group']
            if 'is_slave' in self.job_data:
                metadata['is_slave'] = 'true' if self.job_data.get(
                    'is_slave') else 'false'
            self.context.test_data.add_metadata(metadata)

            if 'role' in self.job_data:
                metadata['role'] = self.job_data['role']
                self.context.test_data.add_metadata(metadata)

            if 'group_size' in self.job_data:
                metadata['group_size'] = self.job_data['group_size']
                self.context.test_data.add_metadata(metadata)

            if 'shared_config' in self.job_data:
                metadata['shared_config'] = self.job_data['shared_config']
                self.context.test_data.add_metadata(metadata)

            logging.debug("[ACTION-B] Multi Node test!")
            logging.debug("[ACTION-B] target_group is (%s)." %
                          self.context.test_data.metadata['target_group'])
        else:
            logging.debug("[ACTION-B] Single node test!")

        # get LMP init data, if it exists.
        if 'lmp_module' in self.job_data:
            lmp_init_data = self.job_data['lmp_module']
            metadata['lmp_module'] = json.dumps(lmp_init_data)
            self.context.test_data.add_metadata(metadata)
        # integrate LMP init data to LMP default config
        lmp_init_data = lmp_init_boards.data_integrate(
            lmp_init_data, self.context.device_config)
        # init LMP module, if necessary
        if lmp_init_data is not []:
            lmp_init_boards.init(lmp_init_data, self.context.device_config)

        def term_handler(signum, frame):
            self.context.finish()
            sys.exit(1)

        signal.signal(signal.SIGTERM, term_handler)

        try:
            job_length = len(self.job_data['actions'])
            job_num = 0
            for cmd in self.job_data['actions']:
                job_num += 1
                params = cmd.get('parameters', {})
                if cmd.get('command').startswith('lava_android_test'):
                    if not params.get('timeout') and \
                       self.job_data.get('timeout'):
                        params['timeout'] = self.job_data['timeout']
                logging.info("[ACTION-B] %s is started with %s" %
                             (cmd['command'], params))
                metadata = cmd.get('metadata', {})
                self.context.test_data.add_metadata(metadata)
                action = lava_commands[cmd['command']](self.context)
                err = None
                try:
                    status = 'fail'
                    action.run(**params)
                except ADBConnectError as err:
                    logging.info("ADBConnectError")
                    if cmd.get('command') == 'boot_linaro_android_image':
                        logging.warning(('[ACTION-E] %s failed to create the'
                                         ' adb connection') % (cmd['command']))

                        # Sometimes the adb problem is caused by the adb
                        # command, and as workround we need to kill the adb
                        # process to make it work
                        logging.warning('Now will try to kill the adb process')
                        rc = commands.getstatusoutput('adb devices')[0]
                        if rc != 0:
                            kill_process_with_option(process="adb",
                                                     key_option="fork-server")

                        # clear the session on the serial and wait a while
                        # and not put the following 3 sentences into the
                        # boot_linaro_android_image method just for
                        # avoiding effects when the method being called
                        # in other places
                        logging.warning(
                            'Now will reboot the image to try again')
                        self.context.client.proc.sendcontrol("c")
                        self.context.client.proc.sendline("")
                        time.sleep(5)
                        self.context.client.boot_linaro_android_image(
                            adb_check=True)
                        # mark it as pass if the second boot works
                        status = 'pass'
                except TimeoutError as err:
                    logging.info("TimeoutError")
                    if cmd.get('command').startswith('lava_android_test'):
                        logging.warning("[ACTION-E] %s times out." %
                                        (cmd['command']))
                        if job_num == job_length:
                            # not reboot the android image for
                            # the last test action
                            pass
                        else:
                            # clear the session on the serial and wait a while
                            # and not put the following 3 sentences into the
                            # boot_linaro_android_image method just for
                            # avoiding effects when the method being called
                            # in other places
                            logging.warning(
                                "Now the android image will be rebooted")
                            self.context.client.proc.sendcontrol("c")
                            self.context.client.proc.sendline("")
                            time.sleep(5)
                            self.context.client.boot_linaro_android_image()
                    else:
                        logging.warning("Unhandled timeout condition")
                        continue
                except CriticalError as err:
                    logging.info("CriticalError")
                    raise
                except (pexpect.TIMEOUT, GeneralError) as err:
                    logging.warning("pexpect timed out with status %s" %
                                    status)
                    pass
                except KeyboardInterrupt:
                    logging.info("Cancel operation")
                    err = "Cancel"
                    pass
                except subprocess.CalledProcessError as err:
                    if err.output is not None:
                        logging.info(
                            "Command error code: %d, with stdout/stderr:" %
                            (err.returncode))
                        for line in err.output.rstrip('\n').split('\n'):
                            logging.info("| > %s" % (line))
                    else:
                        logging.info(
                            "Command error code: %d, without stdout/stderr" %
                            (err.returncode))
                    raise

                except Exception as err:
                    logging.info("General Exception: %s" % unicode(str(err)))
                    raise
                else:
                    logging.debug("setting status pass")
                    status = 'pass'
                finally:
                    logging.debug("finally status %s" % status)
                    err_msg = ""
                    if status == 'fail':
                        # XXX mwhudson, 2013-01-17: I have no idea what this
                        # code is doing.
                        logging.warning(
                            "[ACTION-E] %s is finished with error (%s)." %
                            (cmd['command'], err))
                        err_msg = ("Lava failed at action %s with error:"
                                   "%s\n") % (cmd['command'],
                                              unicode(str(err), 'ascii',
                                                      'replace'))
                        if cmd['command'] == 'lava_test_run':
                            err_msg += "Lava failed on test: %s" % \
                                       params.get('test_name', "Unknown")
                        if err and err.message != "Cancel" and err.message != 'Timeout':
                            err_msg = err_msg + traceback.format_exc()
                            self.context.log("ErrorMessage: %s" %
                                             unicode(str(err)))
                        self.context.log(err_msg)
                    else:
                        logging.info(
                            "[ACTION-E] %s is finished successfully." %
                            (cmd['command']))
                        err_msg = ""
                    self.context.test_data.add_result(
                        action.test_name(**params), status, message=err_msg)
        except:
            # Capture all user-defined and non-user-defined critical errors
            self.context.test_data.job_status = 'fail'
            raise
        finally:
            self.context.finish()
            device_version = self.context.get_device_version() or 'error'
            self.context.test_data.add_metadata(
                {'target.device_version': device_version})
            if 'target_group' in self.job_data:
                # all nodes call aggregate, even if there is no submit_results command
                self._aggregate_bundle(transport, lava_commands,
                                       submit_results)
            elif submit_results:
                params = submit_results.get('parameters', {})
                action = lava_commands[submit_results['command']](self.context)
                params_for_display = params.copy()
                if 'token' in params_for_display:
                    params_for_display['token'] = '<HIDDEN>'
                try:
                    logging.info(
                        "Submitting the test result with parameters = %s",
                        params_for_display)
                    action.run(**params)
                except Exception as err:
                    logging.error(
                        "Failed to submit the test result. Error = %s", err)
                    raise