def perform_bootstrap_into_autotest_root(arguments, autotest_path, argv):
"""
Perfoms a bootstrap to run test_that from the |autotest_path|.
This function is to be called from test_that's main() script, when
test_that is executed from the source tree location. It runs
autotest_quickmerge to update the sysroot unless arguments.no_quickmerge
is set. It then executes and waits on the version of test_that.py
in |autotest_path|.
@param arguments: A parsed arguments object, as returned from
test_that.parse_arguments(...).
@param autotest_path: Full absolute path to the autotest root directory.
@param argv: The arguments list, as passed to main(...)
@returns: The return code of the test_that script that was executed in
|autotest_path|.
"""
logging_manager.configure_logging(
server_logging_config.ServerLoggingConfig(),
use_console=True,
verbose=arguments.debug)
if arguments.no_quickmerge:
logging.info('Skipping quickmerge step.')
else:
logging.info('Running autotest_quickmerge step.')
command = [_QUICKMERGE_SCRIPTNAME, '--board='+arguments.board]
s = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
for message in iter(s.stdout.readline, b''):
logging.info('quickmerge| %s', message.strip())
return_code = s.wait()
if return_code:
raise test_runner_utils.TestThatRunError(
'autotest_quickmerge failed with error code %s.' %
return_code)
logging.info('Re-running test_that script in %s copy of autotest.',
autotest_path)
script_command = os.path.join(autotest_path, 'site_utils',
'test_that.py')
if not os.path.exists(script_command):
raise test_runner_utils.TestThatRunError(
'Unable to bootstrap to autotest root, %s not found.' %
script_command)
proc = None
def resend_sig(signum, stack_frame):
#pylint: disable-msg=C0111
if proc:
proc.send_signal(signum)
signal.signal(signal.SIGINT, resend_sig)
signal.signal(signal.SIGTERM, resend_sig)
proc = subprocess.Popen([script_command] + argv)
return proc.wait()
def _start(args):
"""Starts up the container pool service.
This function instantiates and starts up the pool service on the current
thread (i.e. the function will block, and not return until the service is
shut down).
"""
# TODO(dshi): crbug.com/459344 Set remove this enforcement when test
# container can be unprivileged container.
if utils.sudo_require_password():
logging.warning('SSP requires root privilege to run commands, please '
'grant root access to this process.')
utils.run('sudo true')
# Configure logging.
config = server_logging_config.ServerLoggingConfig()
config.configure_logging(verbose=args.verbose)
config.add_debug_file_handlers(log_dir=_LOG_LOCATION, log_name=_LOG_NAME)
# Pool code is heavily multi-threaded. This will help debugging.
logging_config.add_threadname_in_log()
host_dir = lxc.SharedHostDir()
service = container_pool.Service(host_dir)
# Catch signals, and send the appropriate stop request to the service
# instead of killing the main thread.
# - SIGINT is generated by Ctrl-C
# - SIGTERM is generated by an upstart stopping event.
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, lambda s, f: service.stop())
with ts_mon_config.SetupTsMonGlobalState(service_name='lxc_pool_service',
indirect=True,
short_lived=False):
# Start the service. This blocks and does not return till the service
# shuts down.
service.start(pool_size=args.size)
def perform_run_from_autotest_root(autotest_path,
argv,
tests,
remote,
build=NO_BUILD,
board=NO_BOARD,
args=None,
pretend=False,
no_experimental=False,
ignore_deps=True,
results_directory=None,
ssh_verbosity=0,
ssh_options=None,
iterations=1,
fast_mode=False,
debug=False,
whitelist_chrome_crashes=False,
host_attributes={}):
"""
Perform a test_that run, from the |autotest_path|.
This function is to be called from test_that/test_droid's main() script,
when tests are executed from the |autotest_path|. It handles all stages
of a test run that come after the bootstrap into |autotest_path|.
@param autotest_path: Full absolute path to the autotest root directory.
@param argv: The arguments list, as passed to main(...)
@param tests: List of strings naming tests and suites to run. Suite strings
should be formed like "suite:smoke".
@param remote: Remote hostname.
@param build: String specifying build for local run.
@param board: String specifying board for local run.
@param args: String that should be passed as args parameter to autoserv,
and then ultimitely to test itself.
@param pretend: If True, will print out autoserv commands rather than
running them.
@param no_experimental: Skip experimental tests when scheduling a suite.
@param ignore_deps: If True, test dependencies will be ignored.
@param results_directory: Directory to store results in. Defaults to None,
in which case results will be stored in a new
subdirectory of /tmp
@param ssh_verbosity: SSH verbosity level, passed through to
autoserv_utils.
@param ssh_options: Additional ssh options to be passed to autoserv_utils
@param autoserv_verbose: If true, pass the --verbose flag to autoserv.
@param iterations: int number of times to schedule tests.
@param fast_mode: bool to use fast mode (disables slow autotest features).
@param debug: Logging and autoserv verbosity.
@param whitelist_chrome_crashes: If True, whitelist chrome crashes.
@param host_attributes: Dict of host attributes to pass into autoserv.
@returns: A return code that test_that should exit with.
"""
if results_directory is None or not os.path.exists(results_directory):
raise ValueError('Expected valid results directory, got %s' %
results_directory)
logging_manager.configure_logging(
server_logging_config.ServerLoggingConfig(),
results_dir=results_directory,
use_console=True,
verbose=debug,
debug_log_name='test_that')
logging.info('Began logging to %s', results_directory)
logging.debug('test_that command line was: %s', argv)
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
afe = setup_local_afe()
codes = perform_local_run(afe,
autotest_path,
tests,
remote,
fast_mode,
build,
board,
args=args,
pretend=pretend,
no_experimental=no_experimental,
ignore_deps=ignore_deps,
results_directory=results_directory,
ssh_verbosity=ssh_verbosity,
ssh_options=ssh_options,
autoserv_verbose=debug,
iterations=iterations,
host_attributes=host_attributes)
if pretend:
logging.info('Finished pretend run. Exiting.')
return 0
final_result = generate_report(
results_directory,
whitelist_chrome_crashes=whitelist_chrome_crashes,
html_report=True)
try:
os.unlink(_LATEST_RESULTS_DIRECTORY)
except OSError:
pass
link_target = os.path.relpath(results_directory,
os.path.dirname(_LATEST_RESULTS_DIRECTORY))
if any(codes):
logging.error('Autoserv encountered unexpected errors '
'when executing jobs.')
final_result = final_result or 1
os.symlink(link_target, _LATEST_RESULTS_DIRECTORY)
logging.info('Finished running tests. Results can be found in %s or %s',
results_directory, _LATEST_RESULTS_DIRECTORY)
return final_result
def main():
start_time = datetime.datetime.now()
parser = autoserv_parser.autoserv_parser
parser.parse_args()
if len(sys.argv) == 1:
parser.parser.print_help()
sys.exit(1)
if parser.options.no_logging:
results = None
else:
results = parser.options.results
if not results:
results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S')
results = os.path.abspath(results)
resultdir_exists = False
for filename in ('control.srv', 'status.log', '.autoserv_execute'):
if os.path.exists(os.path.join(results, filename)):
resultdir_exists = True
if not parser.options.use_existing_results and resultdir_exists:
error = "Error: results directory already exists: %s\n" % results
sys.stderr.write(error)
sys.exit(1)
# Now that we certified that there's no leftover results dir from
# previous jobs, lets create the result dir since the logging system
# needs to create the log file in there.
if not os.path.isdir(results):
os.makedirs(results)
if parser.options.require_ssp:
# This is currently only used for skylab (i.e., when --control-name is
# used).
use_ssp = _require_ssp_from_control(parser.options.control_name)
else:
use_ssp = False
if use_ssp:
log_dir = os.path.join(results, 'ssp_logs') if results else None
if log_dir and not os.path.exists(log_dir):
os.makedirs(log_dir)
else:
log_dir = results
logging_manager.configure_logging(
server_logging_config.ServerLoggingConfig(),
results_dir=log_dir,
use_console=not parser.options.no_tee,
verbose=parser.options.verbose,
no_console_prefix=parser.options.no_console_prefix)
logging.debug('autoserv is running in drone %s.', socket.gethostname())
logging.debug('autoserv command was: %s', ' '.join(sys.argv))
logging.debug('autoserv parsed options: %s', parser.options)
if use_ssp:
ssp_url = _stage_ssp(parser, results)
else:
ssp_url = None
if results:
logging.info("Results placed in %s" % results)
# wait until now to perform this check, so it get properly logged
if (parser.options.use_existing_results and not resultdir_exists
and not utils.is_in_container()):
logging.error("No existing results directory found: %s", results)
sys.exit(1)
if parser.options.write_pidfile and results:
pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label,
results)
pid_file_manager.open_file()
else:
pid_file_manager = None
autotest.Autotest.set_install_in_tmpdir(parser.options.install_in_tmpdir)
exit_code = 0
# TODO(beeps): Extend this to cover different failure modes.
# Testing exceptions are matched against labels sent to autoserv. Eg,
# to allow only the hostless job to run, specify
# testing_exceptions: test_suite in the shadow_config. To allow both
# the hostless job and dummy_Pass to run, specify
# testing_exceptions: test_suite,dummy_Pass. You can figure out
# what label autoserv is invoked with by looking through the logs of a test
# for the autoserv command's -l option.
testing_exceptions = _CONFIG.get_config_value('AUTOSERV',
'testing_exceptions',
type=list,
default=[])
test_mode = _CONFIG.get_config_value('AUTOSERV',
'testing_mode',
type=bool,
default=False)
test_mode = (
results_mocker and test_mode
and not any([ex in parser.options.label for ex in testing_exceptions]))
is_task = (parser.options.verify or parser.options.repair
or parser.options.provision or parser.options.reset
or parser.options.cleanup or parser.options.collect_crashinfo)
trace_labels = {
'job_id': job_directories.get_job_id_or_task_id(parser.options.results)
}
trace = cloud_trace.SpanStack(
labels=trace_labels, global_context=parser.options.cloud_trace_context)
trace.enabled = parser.options.cloud_trace_context_enabled == 'True'
try:
try:
if test_mode:
# The parser doesn't run on tasks anyway, so we can just return
# happy signals without faking results.
if not is_task:
machine = parser.options.results.split('/')[-1]
# TODO(beeps): The proper way to do this would be to
# refactor job creation so we can invoke job.record
# directly. To do that one needs to pipe the test_name
# through run_autoserv and bail just before invoking
# the server job. See the comment in
# puppylab/results_mocker for more context.
results_mocker.ResultsMocker('unknown-test',
parser.options.results,
machine).mock_results()
return
else:
with trace.Span(get_job_status(parser.options)):
run_autoserv(pid_file_manager, results, parser, ssp_url,
use_ssp)
except SystemExit as e:
exit_code = e.code
if exit_code:
logging.exception('Uncaught SystemExit with code %s',
exit_code)
except Exception:
# If we don't know what happened, we'll classify it as
# an 'abort' and return 1.
logging.exception('Uncaught Exception, exit_code = 1.')
exit_code = 1
finally:
if pid_file_manager:
pid_file_manager.close_file(exit_code)
sys.exit(exit_code)
def main():
# grab the parser
parser = autoserv_parser.autoserv_parser
parser.parse_args()
if len(sys.argv) == 1:
parser.parser.print_help()
sys.exit(1)
if parser.options.no_logging:
results = None
else:
results = parser.options.results
if not results:
results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S')
results = os.path.abspath(results)
resultdir_exists = False
for filename in ('control.srv', 'status.log', '.autoserv_execute'):
if os.path.exists(os.path.join(results, filename)):
resultdir_exists = True
if not parser.options.use_existing_results and resultdir_exists:
error = "Error: results directory already exists: %s\n" % results
sys.stderr.write(error)
sys.exit(1)
# Now that we certified that there's no leftover results dir from
# previous jobs, lets create the result dir since the logging system
# needs to create the log file in there.
if not os.path.isdir(results):
os.makedirs(results)
logging_manager.configure_logging(
server_logging_config.ServerLoggingConfig(),
results_dir=results,
use_console=not parser.options.no_tee,
verbose=parser.options.verbose,
no_console_prefix=parser.options.no_console_prefix)
if results:
logging.info("Results placed in %s" % results)
# wait until now to perform this check, so it get properly logged
if parser.options.use_existing_results and not resultdir_exists:
logging.error("No existing results directory found: %s", results)
sys.exit(1)
if parser.options.write_pidfile:
pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label,
results)
pid_file_manager.open_file()
else:
pid_file_manager = None
autotest_remote.BaseAutotest.set_install_in_tmpdir(
parser.options.install_in_tmpdir)
exit_code = 0
try:
try:
run_autoserv(pid_file_manager, results, parser)
except SystemExit, e:
exit_code = e.code
except:
traceback.print_exc()
# If we don't know what happened, we'll classify it as
# an 'abort' and return 1.
exit_code = 1
def main():
start_time = datetime.datetime.now()
# grab the parser
parser = autoserv_parser.autoserv_parser
parser.parse_args()
if len(sys.argv) == 1:
parser.parser.print_help()
sys.exit(1)
# If the job requires to run with server-side package, try to stage server-
# side package first. If that fails with error that autotest server package
# does not exist, fall back to run the job without using server-side
# packaging. If option warn_no_ssp is specified, that means autoserv is
# running in a drone does not support SSP, thus no need to stage server-side
# package.
ssp_url = None
ssp_url_warning = False
if (not parser.options.warn_no_ssp and parser.options.require_ssp):
ssp_url, ssp_error_msg = _stage_ssp(parser)
# The build does not have autotest server package. Fall back to not
# to use server-side package. Logging is postponed until logging being
# set up.
ssp_url_warning = not ssp_url
if parser.options.no_logging:
results = None
else:
results = parser.options.results
if not results:
results = 'results.' + time.strftime('%Y-%m-%d-%H.%M.%S')
results = os.path.abspath(results)
resultdir_exists = False
for filename in ('control.srv', 'status.log', '.autoserv_execute'):
if os.path.exists(os.path.join(results, filename)):
resultdir_exists = True
if not parser.options.use_existing_results and resultdir_exists:
error = "Error: results directory already exists: %s\n" % results
sys.stderr.write(error)
sys.exit(1)
# Now that we certified that there's no leftover results dir from
# previous jobs, lets create the result dir since the logging system
# needs to create the log file in there.
if not os.path.isdir(results):
os.makedirs(results)
# Server-side packaging will only be used if it's required and the package
# is available. If warn_no_ssp is specified, it means that autoserv is
# running in a drone does not have SSP supported and a warning will be logs.
# Therefore, it should not run with SSP.
use_ssp = (not parser.options.warn_no_ssp and parser.options.require_ssp
and ssp_url)
if use_ssp:
log_dir = os.path.join(results, 'ssp_logs') if results else None
if log_dir and not os.path.exists(log_dir):
os.makedirs(log_dir)
else:
log_dir = results
logging_manager.configure_logging(
server_logging_config.ServerLoggingConfig(),
results_dir=log_dir,
use_console=not parser.options.no_tee,
verbose=parser.options.verbose,
no_console_prefix=parser.options.no_console_prefix)
if ssp_url_warning:
logging.warn(
'Autoserv is required to run with server-side packaging. '
'However, no server-side package can be found based on '
'`--image`, host attribute job_repo_url or host OS version '
'label. It could be that the build to test is older than the '
'minimum version that supports server-side packaging. The test '
'will be executed without using erver-side packaging. '
'Following is the detailed error:\n%s', ssp_error_msg)
if results:
logging.info("Results placed in %s" % results)
# wait until now to perform this check, so it get properly logged
if (parser.options.use_existing_results and not resultdir_exists
and not utils.is_in_container()):
logging.error("No existing results directory found: %s", results)
sys.exit(1)
logging.debug('autoserv is running in drone %s.', socket.gethostname())
logging.debug('autoserv command was: %s', ' '.join(sys.argv))
if parser.options.write_pidfile and results:
pid_file_manager = pidfile.PidFileManager(parser.options.pidfile_label,
results)
pid_file_manager.open_file()
else:
pid_file_manager = None
autotest.BaseAutotest.set_install_in_tmpdir(
parser.options.install_in_tmpdir)
try:
# Take the first argument as control file name, get the test name from
# the control file.
if (len(parser.args) > 0 and parser.args[0] != ''
and parser.options.machines):
try:
test_name = control_data.parse_control(
parser.args[0], raise_warnings=True).name
except control_data.ControlVariableException:
logging.debug(
'Failed to retrieve test name from control file.')
test_name = None
except control_data.ControlVariableException as e:
logging.error(str(e))
exit_code = 0
# TODO(beeps): Extend this to cover different failure modes.
# Testing exceptions are matched against labels sent to autoserv. Eg,
# to allow only the hostless job to run, specify
# testing_exceptions: test_suite in the shadow_config. To allow both
# the hostless job and dummy_Pass to run, specify
# testing_exceptions: test_suite,dummy_Pass. You can figure out
# what label autoserv is invoked with by looking through the logs of a test
# for the autoserv command's -l option.
testing_exceptions = _CONFIG.get_config_value('AUTOSERV',
'testing_exceptions',
type=list,
default=[])
test_mode = _CONFIG.get_config_value('AUTOSERV',
'testing_mode',
type=bool,
default=False)
test_mode = (
results_mocker and test_mode
and not any([ex in parser.options.label for ex in testing_exceptions]))
is_task = (parser.options.verify or parser.options.repair
or parser.options.provision or parser.options.reset
or parser.options.cleanup or parser.options.collect_crashinfo)
try:
try:
if test_mode:
# The parser doesn't run on tasks anyway, so we can just return
# happy signals without faking results.
if not is_task:
machine = parser.options.results.split('/')[-1]
# TODO(beeps): The proper way to do this would be to
# refactor job creation so we can invoke job.record
# directly. To do that one needs to pipe the test_name
# through run_autoserv and bail just before invoking
# the server job. See the comment in
# puppylab/results_mocker for more context.
results_mocker.ResultsMocker(
test_name if test_name else 'unknown-test',
parser.options.results, machine).mock_results()
return
else:
run_autoserv(pid_file_manager, results, parser, ssp_url,
use_ssp)
except SystemExit as e:
exit_code = e.code
if exit_code:
logging.exception(e)
except Exception as e:
# If we don't know what happened, we'll classify it as
# an 'abort' and return 1.
logging.exception(e)
exit_code = 1
finally:
if pid_file_manager:
pid_file_manager.close_file(exit_code)
# Record the autoserv duration time. Must be called
# just before the system exits to ensure accuracy.
duration_secs = (datetime.datetime.now() - start_time).total_seconds()
record_autoserv(parser.options, duration_secs)
sys.exit(exit_code)
