def _tradefed_run_command(self, template): """Build tradefed 'run' command from template.""" cmd = template[:] # If we are running outside of the lab we can collect more data. if not utils.is_in_container(): logging.info('Running outside of lab, adding extra debug options.') cmd.append('--log-level-display=DEBUG') return cmd
def select_32bit_java(): """Switches to 32 bit java if installed (like in lab lxc images) to save about 30-40% server/shard memory during the run.""" if utils.is_in_container() and not client_utils.is_moblab(): java = '/usr/lib/jvm/java-8-openjdk-i386' if os.path.exists(java): logging.info('Found 32 bit java, switching to use it.') os.environ['JAVA_HOME'] = java os.environ['PATH'] = (os.path.join(java, 'bin') + os.pathsep + os.environ['PATH'])
def initialize(self, host=None, adb_dir=_ADB_DIR_M, sdk_tools_dir=_SDK_TOOLS_DIR_M): """Sets up the tools and binary bundles for the test.""" logging.info('Hostname: %s', host.hostname) self._host = host self._install_paths = [] # Tests in the lab run within individual lxc container instances. if utils.is_in_container(): cache_root = _TRADEFED_CACHE_CONTAINER else: cache_root = _TRADEFED_CACHE_LOCAL # TODO(ihf): reevaluate this again when we run out of memory. We could # for example use 32 bit java on the first run but not during retries. # b/62895114. If select_32bit_java gets deleted for good also remove it # from the base image. # Try to save server memory (crbug.com/717413). # select_32bit_java() # Quick sanity check and spew of java version installed on the server. utils.run('java', args=('-version', ), ignore_status=False, verbose=True, stdout_tee=utils.TEE_TO_LOGS, stderr_tee=utils.TEE_TO_LOGS) # The content of the cache survives across jobs. self._safe_makedirs(cache_root) self._tradefed_cache = os.path.join(cache_root, 'cache') self._tradefed_cache_lock = os.path.join(cache_root, 'lock') # The content of the install location does not survive across jobs and # is isolated (by using a unique path)_against other autotest instances. # This is not needed for the lab, but if somebody wants to run multiple # TradedefTest instance. self._tradefed_install = tempfile.mkdtemp(prefix=_TRADEFED_PREFIX) # Under lxc the cache is shared between multiple autotest/tradefed # instances. We need to synchronize access to it. All binaries are # installed through the (shared) cache into the local (unshared) # lxc/autotest instance storage. # If clearing the cache it must happen before all downloads. self._clear_download_cache_if_needed() # Set permissions (rwxr-xr-x) to the executable binaries. permission = (stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) self._install_files(adb_dir, _ADB_FILES, permission) self._install_files(sdk_tools_dir, _SDK_TOOLS_FILES, permission)
def schedule(self, job, timeout_mins, machine): """ Sequence a job on the running AFE. Will schedule a given test on the job machine(s). Support a subset of tests: - server job - no hostless. - no cleanup around tests. @param job: server_job object that will server as parent. @param timeout_mins: timeout to set up: if the test last more than timeout_mins, the test will fail. @param machine: machine to run the test on. @returns a maximal time in minutes that the sequence can take. """ afe = frontend_wrappers.RetryingAFE(timeout_min=30, delay_sec=10, user=job.user, debug=False) # job_directores.get_job_id_or_task_id() will return a non-int opaque id # for Chrome OS Skylab tasks. But sequences will break in that case # anyway, because they try to create AFE jobs internally. current_job_id = int( job_directories.get_job_id_or_task_id(job.resultdir)) logging.debug('Current job id: %s', current_job_id) runtime_mins = self.child_job_timeout() hostname = utils.get_hostname_from_machine(machine) for i in xrange(0, self._iteration): child_job_name = self.child_job_name(hostname, i) logging.debug('Creating job: %s', child_job_name) afe.create_job( self.child_control_file(), name=child_job_name, priority=priorities.Priority.DEFAULT, control_type=control_data.CONTROL_TYPE.SERVER, hosts=[hostname], meta_hosts=(), one_time_hosts=(), synch_count=None, is_template=False, timeout_mins=timeout_mins + (i + 1) * runtime_mins, max_runtime_mins=runtime_mins, run_verify=False, email_list='', dependencies=(), reboot_before=None, reboot_after=None, parse_failed_repair=None, hostless=False, keyvals=None, drone_set=None, image=None, parent_job_id=current_job_id, test_retry=0, run_reset=False, require_ssp=utils.is_in_container()) return runtime_mins * self._iteration
def _tradefed_run_command(self, template): """Build tradefed 'run' command from template.""" cmd = [] for arg in template: if (arg == '--skip-device-info' and self._get_board_name() in self._NEED_DEVICE_INFO_BOARDS): continue cmd.append(arg) # If we are running outside of the lab we can collect more data. if not utils.is_in_container(): logging.info('Running outside of lab, adding extra debug options.') cmd.append('--log-level-display=DEBUG') cmd.append('--screenshot-on-failure') # TODO(ihf): Add log collection once b/28333587 fixed. # cmd.append('--collect-deqp-logs') return cmd
def main(): start_time = datetime.datetime.now() parser = autoserv_parser.autoserv_parser parser.parse_args() if len(sys.argv) == 1: parser.parser.print_help() sys.exit(1) if parser.options.no_logging: results = None else: results = parser.options.results if not results: results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S') results = os.path.abspath(results) resultdir_exists = False for filename in ('control.srv', 'status.log', '.autoserv_execute'): if os.path.exists(os.path.join(results, filename)): resultdir_exists = True if not parser.options.use_existing_results and resultdir_exists: error = "Error: results directory already exists: %s\n" % results sys.stderr.write(error) sys.exit(1) # Now that we certified that there's no leftover results dir from # previous jobs, lets create the result dir since the logging system # needs to create the log file in there. if not os.path.isdir(results): os.makedirs(results) if parser.options.require_ssp: # This is currently only used for skylab (i.e., when --control-name is # used). use_ssp = _require_ssp_from_control(parser.options.control_name) else: use_ssp = False if use_ssp: log_dir = os.path.join(results, 'ssp_logs') if results else None if log_dir and not os.path.exists(log_dir): os.makedirs(log_dir) else: log_dir = results logging_manager.configure_logging( server_logging_config.ServerLoggingConfig(), results_dir=log_dir, use_console=not parser.options.no_tee, verbose=parser.options.verbose, no_console_prefix=parser.options.no_console_prefix) logging.debug('autoserv is running in drone %s.', socket.gethostname()) logging.debug('autoserv command was: %s', ' '.join(sys.argv)) logging.debug('autoserv parsed options: %s', parser.options) if use_ssp: ssp_url = _stage_ssp(parser, results) else: ssp_url = None if results: logging.info("Results placed in %s" % results) # wait until now to perform this check, so it get properly logged if (parser.options.use_existing_results and not resultdir_exists and not utils.is_in_container()): logging.error("No existing results directory found: %s", results) sys.exit(1) if parser.options.write_pidfile and results: pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label, results) pid_file_manager.open_file() else: pid_file_manager = None autotest.Autotest.set_install_in_tmpdir(parser.options.install_in_tmpdir) exit_code = 0 # TODO(beeps): Extend this to cover different failure modes. # Testing exceptions are matched against labels sent to autoserv. Eg, # to allow only the hostless job to run, specify # testing_exceptions: test_suite in the shadow_config. To allow both # the hostless job and dummy_Pass to run, specify # testing_exceptions: test_suite,dummy_Pass. You can figure out # what label autoserv is invoked with by looking through the logs of a test # for the autoserv command's -l option. testing_exceptions = _CONFIG.get_config_value('AUTOSERV', 'testing_exceptions', type=list, default=[]) test_mode = _CONFIG.get_config_value('AUTOSERV', 'testing_mode', type=bool, default=False) test_mode = ( results_mocker and test_mode and not any([ex in parser.options.label for ex in testing_exceptions])) is_task = (parser.options.verify or parser.options.repair or parser.options.provision or parser.options.reset or parser.options.cleanup or parser.options.collect_crashinfo) trace_labels = { 'job_id': job_directories.get_job_id_or_task_id(parser.options.results) } trace = cloud_trace.SpanStack( labels=trace_labels, global_context=parser.options.cloud_trace_context) trace.enabled = parser.options.cloud_trace_context_enabled == 'True' try: try: if test_mode: # The parser doesn't run on tasks anyway, so we can just return # happy signals without faking results. if not is_task: machine = parser.options.results.split('/')[-1] # TODO(beeps): The proper way to do this would be to # refactor job creation so we can invoke job.record # directly. To do that one needs to pipe the test_name # through run_autoserv and bail just before invoking # the server job. See the comment in # puppylab/results_mocker for more context. results_mocker.ResultsMocker('unknown-test', parser.options.results, machine).mock_results() return else: with trace.Span(get_job_status(parser.options)): run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp) except SystemExit as e: exit_code = e.code if exit_code: logging.exception('Uncaught SystemExit with code %s', exit_code) except Exception: # If we don't know what happened, we'll classify it as # an 'abort' and return 1. logging.exception('Uncaught Exception, exit_code = 1.') exit_code = 1 finally: if pid_file_manager: pid_file_manager.close_file(exit_code) sys.exit(exit_code)
def run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp): """Run server job with given options. @param pid_file_manager: PidFileManager used to monitor the autoserv process @param results: Folder to store results. @param parser: Parser for the command line arguments. @param ssp_url: Url to server-side package. @param use_ssp: Set to True to run with server-side packaging. """ # send stdin to /dev/null dev_null = os.open(os.devnull, os.O_RDONLY) os.dup2(dev_null, sys.stdin.fileno()) os.close(dev_null) # Create separate process group if the process is not a process group # leader. This allows autoserv process to keep running after the caller # process (drone manager call) exits. if os.getpid() != os.getpgid(0): os.setsid() # Container name is predefined so the container can be destroyed in # handle_sigterm. job_or_task_id = job_directories.get_job_id_or_task_id( parser.options.results) container_id = lxc.ContainerId(job_or_task_id, time.time(), os.getpid()) # Implement SIGTERM handler def handle_sigterm(signum, frame): logging.debug('Received SIGTERM') if pid_file_manager: pid_file_manager.close_file(1, signal.SIGTERM) logging.debug('Finished writing to pid_file. Killing process.') # Update results folder's file permission. This needs to be done ASAP # before the parsing process tries to access the log. if use_ssp and results: correct_results_folder_permission(results) # TODO (sbasi) - remove the time.sleep when crbug.com/302815 is solved. # This sleep allows the pending output to be logged before the kill # signal is sent. time.sleep(.1) if use_ssp: logging.debug( 'Destroy container %s before aborting the autoserv ' 'process.', container_id) try: bucket = lxc.ContainerBucket() container = bucket.get_container(container_id) if container: container.destroy() else: logging.debug('Container %s is not found.', container_id) except: # Handle any exception so the autoserv process can be aborted. logging.exception('Failed to destroy container %s.', container_id) # Try to correct the result file permission again after the # container is destroyed, as the container might have created some # new files in the result folder. if results: correct_results_folder_permission(results) os.killpg(os.getpgrp(), signal.SIGKILL) # Set signal handler signal.signal(signal.SIGTERM, handle_sigterm) # faulthandler is only needed to debug in the Lab and is not avaliable to # be imported in the chroot as part of VMTest, so Try-Except it. try: import faulthandler faulthandler.register(signal.SIGTERM, all_threads=True, chain=True) logging.debug('faulthandler registered on SIGTERM.') except ImportError: sys.exc_clear() # Ignore SIGTTOU's generated by output from forked children. signal.signal(signal.SIGTTOU, signal.SIG_IGN) # If we received a SIGALARM, let's be loud about it. signal.signal(signal.SIGALRM, log_alarm) # Server side tests that call shell scripts often depend on $USER being set # but depending on how you launch your autotest scheduler it may not be set. os.environ['USER'] = getpass.getuser() label = parser.options.label group_name = parser.options.group_name user = parser.options.user client = parser.options.client server = parser.options.server verify = parser.options.verify repair = parser.options.repair cleanup = parser.options.cleanup provision = parser.options.provision reset = parser.options.reset job_labels = parser.options.job_labels no_tee = parser.options.no_tee execution_tag = parser.options.execution_tag ssh_user = parser.options.ssh_user ssh_port = parser.options.ssh_port ssh_pass = parser.options.ssh_pass collect_crashinfo = parser.options.collect_crashinfo control_filename = parser.options.control_filename verify_job_repo_url = parser.options.verify_job_repo_url skip_crash_collection = parser.options.skip_crash_collection ssh_verbosity = int(parser.options.ssh_verbosity) ssh_options = parser.options.ssh_options no_use_packaging = parser.options.no_use_packaging in_lab = bool(parser.options.lab) # can't be both a client and a server side test if client and server: parser.parser.error( "Can not specify a test as both server and client!") if provision and client: parser.parser.error("Cannot specify provisioning and client!") is_special_task = (verify or repair or cleanup or collect_crashinfo or provision or reset) use_client_trampoline = False if parser.options.control_name: if use_ssp: # When use_ssp is True, autoserv will be re-executed inside a # container preserving the --control-name argument. Control file # will be staged inside the rexecuted autoserv. control = None else: try: control = _stage_control_file(parser.options.control_name, results) except error.AutoservError as e: logging.info("Using client trampoline because of: %s", e) control = parser.options.control_name use_client_trampoline = True elif parser.args: control = parser.args[0] else: if not is_special_task: parser.parser.error("Missing argument: control file") control = None if ssh_verbosity > 0: # ssh_verbosity is an integer between 0 and 3, inclusive ssh_verbosity_flag = '-' + 'v' * ssh_verbosity else: ssh_verbosity_flag = '' machines = _get_machines(parser) if group_name and len(machines) < 2: parser.parser.error('-G %r may only be supplied with more than one ' 'machine.' % group_name) job_kwargs = { 'control': control, 'args': parser.args[1:], 'resultdir': results, 'label': label, 'user': user, 'machines': machines, 'machine_dict_list': server_job.get_machine_dicts( machine_names=machines, store_dir=os.path.join(results, parser.options.host_info_subdir), in_lab=in_lab, use_shadow_store=not parser.options.local_only_host_info, host_attributes=parser.options.host_attributes, ), 'client': client, 'ssh_user': ssh_user, 'ssh_port': ssh_port, 'ssh_pass': ssh_pass, 'ssh_verbosity_flag': ssh_verbosity_flag, 'ssh_options': ssh_options, 'group_name': group_name, 'tag': execution_tag, 'disable_sysinfo': parser.options.disable_sysinfo, 'in_lab': in_lab, 'use_client_trampoline': use_client_trampoline, } if parser.options.parent_job_id: job_kwargs['parent_job_id'] = int(parser.options.parent_job_id) if control_filename: job_kwargs['control_filename'] = control_filename job = server_job.server_job(**job_kwargs) job.logging.start_logging() # perform checks job.precheck() # run the job exit_code = 0 auto_start_servod = _CONFIG.get_config_value('AUTOSERV', 'auto_start_servod', type=bool, default=False) site_utils.SetupTsMonGlobalState('autoserv', indirect=False, short_lived=True) try: try: if repair: if auto_start_servod and len(machines) == 1: _start_servod(machines[0]) job.repair(job_labels) elif verify: job.verify(job_labels) elif provision: job.provision(job_labels) elif reset: job.reset(job_labels) elif cleanup: job.cleanup(job_labels) else: if auto_start_servod and len(machines) == 1: _start_servod(machines[0]) if use_ssp: try: _run_with_ssp(job, container_id, job_or_task_id, results, parser, ssp_url, machines) finally: # Update the ownership of files in result folder. correct_results_folder_permission(results) else: if collect_crashinfo: # Update the ownership of files in result folder. If the # job to collect crashinfo was running inside container # (SSP) and crashed before correcting folder permission, # the result folder might have wrong permission setting. try: correct_results_folder_permission(results) except: # Ignore any error as the user may not have root # permission to run sudo command. pass metric_name = ('chromeos/autotest/experimental/' 'autoserv_job_run_duration') f = { 'in_container': utils.is_in_container(), 'success': False } with metrics.SecondsTimer(metric_name, fields=f) as c: job.run(verify_job_repo_url=verify_job_repo_url, only_collect_crashinfo=collect_crashinfo, skip_crash_collection=skip_crash_collection, job_labels=job_labels, use_packaging=(not no_use_packaging)) c['success'] = True finally: job.close() # Special task doesn't run parse, so result summary needs to be # built here. if results and (repair or verify or reset or cleanup or provision): # Throttle the result on the server side. try: result_utils.execute( results, control_data.DEFAULT_MAX_RESULT_SIZE_KB) except: logging.exception( 'Non-critical failure: Failed to throttle results ' 'in directory %s.', results) # Build result view and report metrics for result sizes. site_utils.collect_result_sizes(results) except: exit_code = 1 traceback.print_exc() finally: metrics.Flush() sys.exit(exit_code)
def _tradefed_run_command(self, package=None, plan=None, session_id=None, test_class=None, test_method=None): """Builds the CTS tradefed 'run' command line. There are five different usages: 1. Test a package: assign the package name via |package|. 2. Test a plan: assign the plan name via |plan|. 3. Continue a session: assign the session ID via |session_id|. 4. Run all test cases of a class: assign the class name via |test_class|. 5. Run a specific test case: assign the class name and method name in |test_class| and |test_method|. @param package: the name of test package to be run. @param plan: name of the plan to be run. @param session_id: tradefed session id to continue. @param test_class: the name of the class of which all test cases will be run. @param test_name: the name of the method of |test_class| to be run. Must be used with |test_class|. @return: list of command tokens for the 'run' command. """ if package is not None: cmd = ['run', 'commandAndExit', 'cts', '--package', package] elif plan is not None: cmd = ['run', 'commandAndExit', 'cts', '--plan', plan] elif session_id is not None: cmd = [ 'run', 'commandAndExit', 'cts', '--continue-session', '%d' % session_id ] elif test_class is not None: cmd = ['run', 'commandAndExit', 'cts', '-c', test_class] if test_method is not None: cmd += ['-m', test_method] else: logging.warning('Running all tests. This can take several days.') cmd = ['run', 'commandAndExit', 'cts', '--plan', 'CTS'] # Automated media download is broken, so disable it. Instead we handle # this explicitly via _push_media(). This has the benefit of being # cached on the dev server. b/27245577 cmd.append('--skip-media-download') # If we are running outside of the lab we can collect more data. if not utils.is_in_container(): logging.info('Running outside of lab, adding extra debug options.') cmd.append('--log-level-display=DEBUG') cmd.append('--screenshot-on-failure') cmd.append('--collect-deqp-logs') # At early stage, cts-tradefed tries to reboot the device by # "adb reboot" command. In a real Android device case, when the # rebooting is completed, adb connection is re-established # automatically, and cts-tradefed expects that behavior. # However, in ARC, it doesn't work, so the whole test process # is just stuck. Here, disable the feature. cmd.append('--disable-reboot') # Create a logcat file for each individual failure. cmd.append('--logcat-on-failure') return cmd
def _tradefed_run_command(self, module=None, plan=None, session_id=None, test_class=None, test_method=None): """Builds the CTS tradefed 'run' command line. There are five different usages: 1. Test a module: assign the module name via |module|. 2. Test a plan: assign the plan name via |plan|. 3. Continue a session: assign the session ID via |session_id|. 4. Run all test cases of a class: assign the class name via |test_class|. 5. Run a specific test case: assign the class name and method name in |test_class| and |test_method|. @param module: the name of test module to be run. @param plan: name of the plan to be run. @param session_id: tradefed session id to continue. @param test_class: the name of the class of which all test cases will be run. @param test_name: the name of the method of |test_class| to be run. Must be used with |test_class|. @return: list of command tokens for the 'run' command. """ if module is not None: # Run a particular module (used to be called package in M). cmd = ['run', 'commandAndExit', 'cts', '--module', module] if test_class is not None: if test_method is not None: cmd += ['-t', test_class + '#' + test_method] else: cmd += ['-t', test_class] elif plan is not None and session_id is not None: # In 7.1 R2 we can only retry session_id with the original plan. cmd = ['run', 'commandAndExit', 'cts', '--plan', plan, '--retry', '%d' % session_id] elif plan is not None: # Subplan for any customized CTS test plan in form of xml. cmd = ['run', 'commandAndExit', 'cts', '--subplan', plan] else: logging.warning('Running all tests. This can take several days.') cmd = ['run', 'commandAndExit', 'cts'] # We handle media download ourselves in the lab, as lazy as possible. cmd.append('--precondition-arg') cmd.append('skip-media-download') # If we are running outside of the lab we can collect more data. if not utils.is_in_container(): logging.info('Running outside of lab, adding extra debug options.') cmd.append('--log-level-display=DEBUG') cmd.append('--screenshot-on-failure') # TODO(ihf): Add log collection once b/28333587 fixed. #cmd.append('--collect-deqp-logs') # TODO(ihf): Add tradefed_test.adb_keepalive() and remove # --disable-reboot. This might be more efficient. # At early stage, cts-tradefed tries to reboot the device by # "adb reboot" command. In a real Android device case, when the # rebooting is completed, adb connection is re-established # automatically, and cts-tradefed expects that behavior. # However, in ARC, it doesn't work, so the whole test process # is just stuck. Here, disable the feature. cmd.append('--disable-reboot') # Create a logcat file for each individual failure. cmd.append('--logcat-on-failure') return cmd
def main(): start_time = datetime.datetime.now() # grab the parser parser = autoserv_parser.autoserv_parser parser.parse_args() if len(sys.argv) == 1: parser.parser.print_help() sys.exit(1) if parser.options.no_logging: results = None else: results = parser.options.results if not results: results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S') results = os.path.abspath(results) resultdir_exists = False for filename in ('control.srv', 'status.log', '.autoserv_execute'): if os.path.exists(os.path.join(results, filename)): resultdir_exists = True if not parser.options.use_existing_results and resultdir_exists: error = "Error: results directory already exists: %s\n" % results sys.stderr.write(error) sys.exit(1) # Now that we certified that there's no leftover results dir from # previous jobs, lets create the result dir since the logging system # needs to create the log file in there. if not os.path.isdir(results): os.makedirs(results) # If the job requires to run with server-side package, try to stage server- # side package first. If that fails with error that autotest server package # does not exist, fall back to run the job without using server-side # packaging. If option warn_no_ssp is specified, that means autoserv is # running in a drone does not support SSP, thus no need to stage server-side # package. ssp_url = None ssp_url_warning = False if (not parser.options.warn_no_ssp and parser.options.require_ssp): ssp_url, ssp_error_msg = _stage_ssp(parser, results) # The build does not have autotest server package. Fall back to not # to use server-side package. Logging is postponed until logging being # set up. ssp_url_warning = not ssp_url # Server-side packaging will only be used if it's required and the package # is available. If warn_no_ssp is specified, it means that autoserv is # running in a drone does not have SSP supported and a warning will be logs. # Therefore, it should not run with SSP. use_ssp = (not parser.options.warn_no_ssp and parser.options.require_ssp and ssp_url) if use_ssp: log_dir = os.path.join(results, 'ssp_logs') if results else None if log_dir and not os.path.exists(log_dir): os.makedirs(log_dir) else: log_dir = results logging_manager.configure_logging( server_logging_config.ServerLoggingConfig(), results_dir=log_dir, use_console=not parser.options.no_tee, verbose=parser.options.verbose, no_console_prefix=parser.options.no_console_prefix) if ssp_url_warning: logging.warn( 'Autoserv is required to run with server-side packaging. ' 'However, no server-side package can be staged based on ' '`--image`, host attribute job_repo_url or host OS version ' 'label. It could be that the build to test is older than the ' 'minimum version that supports server-side packaging, or no ' 'devserver can be found to stage server-side package. The test ' 'will be executed without using erver-side packaging. ' 'Following is the detailed error:\n%s', ssp_error_msg) if results: logging.info("Results placed in %s" % results) # wait until now to perform this check, so it get properly logged if (parser.options.use_existing_results and not resultdir_exists and not utils.is_in_container()): logging.error("No existing results directory found: %s", results) sys.exit(1) logging.debug('autoserv is running in drone %s.', socket.gethostname()) logging.debug('autoserv command was: %s', ' '.join(sys.argv)) if parser.options.write_pidfile and results: pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label, results) pid_file_manager.open_file() else: pid_file_manager = None autotest.Autotest.set_install_in_tmpdir(parser.options.install_in_tmpdir) exit_code = 0 # TODO(beeps): Extend this to cover different failure modes. # Testing exceptions are matched against labels sent to autoserv. Eg, # to allow only the hostless job to run, specify # testing_exceptions: test_suite in the shadow_config. To allow both # the hostless job and dummy_Pass to run, specify # testing_exceptions: test_suite,dummy_Pass. You can figure out # what label autoserv is invoked with by looking through the logs of a test # for the autoserv command's -l option. testing_exceptions = _CONFIG.get_config_value('AUTOSERV', 'testing_exceptions', type=list, default=[]) test_mode = _CONFIG.get_config_value('AUTOSERV', 'testing_mode', type=bool, default=False) test_mode = ( results_mocker and test_mode and not any([ex in parser.options.label for ex in testing_exceptions])) is_task = (parser.options.verify or parser.options.repair or parser.options.provision or parser.options.reset or parser.options.cleanup or parser.options.collect_crashinfo) trace_labels = { 'job_id': job_directories.get_job_id_or_task_id(parser.options.results) } trace = cloud_trace.SpanStack( labels=trace_labels, global_context=parser.options.cloud_trace_context) trace.enabled = parser.options.cloud_trace_context_enabled == 'True' try: try: if test_mode: # The parser doesn't run on tasks anyway, so we can just return # happy signals without faking results. if not is_task: machine = parser.options.results.split('/')[-1] # TODO(beeps): The proper way to do this would be to # refactor job creation so we can invoke job.record # directly. To do that one needs to pipe the test_name # through run_autoserv and bail just before invoking # the server job. See the comment in # puppylab/results_mocker for more context. results_mocker.ResultsMocker('unknown-test', parser.options.results, machine).mock_results() return else: with trace.Span(get_job_status(parser.options)): run_autoserv(pid_file_manager, results, parser, ssp_url, use_ssp) except SystemExit as e: exit_code = e.code if exit_code: logging.exception('Uncaught SystemExit with code %s', exit_code) except Exception: # If we don't know what happened, we'll classify it as # an 'abort' and return 1. logging.exception('Uncaught Exception, exit_code = 1.') exit_code = 1 finally: if pid_file_manager: pid_file_manager.close_file(exit_code) # Record the autoserv duration time. Must be called # just before the system exits to ensure accuracy. record_autoserv(parser.options, start_time) sys.exit(exit_code)
def initialize(self, bundle=None, uri=None, host=None, max_retry=None, retry_manual_tests=False, warn_on_test_retry=True): """Sets up the tools and binary bundles for the test.""" logging.info('Hostname: %s', host.hostname) self._host = host self._max_retry = self._get_max_retry(max_retry, self._host) self._install_paths = [] self._warn_on_test_retry = warn_on_test_retry # Tests in the lab run within individual lxc container instances. if utils.is_in_container(): cache_root = constants.TRADEFED_CACHE_CONTAINER else: cache_root = constants.TRADEFED_CACHE_LOCAL # TODO(ihf): reevaluate this again when we run out of memory. We could # for example use 32 bit java on the first run but not during retries. # b/62895114. If select_32bit_java gets deleted for good also remove it # from the base image. # Try to save server memory (crbug.com/717413). # select_32bit_java() # The content of the cache survives across jobs. self._safe_makedirs(cache_root) self._tradefed_cache = os.path.join(cache_root, 'cache') self._tradefed_cache_lock = os.path.join(cache_root, 'lock') self._tradefed_cache_dirty = os.path.join(cache_root, 'dirty') # The content of the install location does not survive across jobs and # is isolated (by using a unique path)_against other autotest instances. # This is not needed for the lab, but if somebody wants to run multiple # TradedefTest instance. self._tradefed_install = tempfile.mkdtemp( prefix=constants.TRADEFED_PREFIX) # Under lxc the cache is shared between multiple autotest/tradefed # instances. We need to synchronize access to it. All binaries are # installed through the (shared) cache into the local (unshared) # lxc/autotest instance storage. # If clearing the cache it must happen before all downloads. self._clean_download_cache_if_needed() # Set permissions (rwxr-xr-x) to the executable binaries. permission = (stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) self._install_files(constants.ADB_DIR, constants.ADB_FILES, permission) self._install_files(constants.SDK_TOOLS_DIR, constants.SDK_TOOLS_FILES, permission) # Install the tradefed bundle. bundle_install_path = self._install_bundle( uri or self._get_default_bundle_url(bundle)) self._repository = os.path.join(bundle_install_path, self._get_tradefed_base_dir()) # Load expected test failures to exclude them from re-runs. self._waivers = self._get_expected_failures('expectations') if not retry_manual_tests: self._waivers.update(self._get_expected_failures('manual_tests')) # Load modules with no tests. self._notest_modules = self._get_expected_failures('notest_modules')