def test_chromite_load(name, expected):
"""Test load()."""
with mock.patch('importlib.import_module', autospec=True) \
as import_module, \
mock.patch.object(autotest, '_setup_done', True):
autotest.chromite_load(name)
import_module.assert_called_once_with(expected)
def _main(args):
"""Main program body, running under a lease file.
@param args: Namespace object containing parsed arguments
"""
ts_mon_config = autotest.chromite_load('ts_mon_config')
metrics = autotest.chromite_load('metrics')
with ts_mon_config.SetupTsMonGlobalState('job_reporter', short_lived=True):
atexit.register(metrics.Flush)
return _run_autotest_job(args)
def log_wait_task(suite_name, task_id):
"""Print create task of suite."""
annotations = autotest.chromite_load('buildbot_annotations')
print(annotations.StepLink(
'Link to the suite wait task: %s' % suite_name,
swarming_lib.get_task_link(
get_task_id_for_task_summaries(task_id))))
def _create_test_task(test_spec, suite_id=None,
is_provision=False, dry_run=False):
"""Create a test task for a given test spec.
@param test_spec: A cros_suite.TestSpec object.
@param suite_id: the suite task id of the test.
@param dry_run: If true, don't actually create task.
@return the swarming task id of this task.
"""
logging.info('Creating task for test %s', test_spec.test.name)
skylab_tool_path = os.environ.get('SKYLAB_TOOL', SKYLAB_TOOL)
cmd = [
skylab_tool_path, 'create-test',
'-board', test_spec.board,
'-image', test_spec.build,
'-service-account-json', os.environ['SWARMING_CREDS'],
]
if _is_dev():
cmd += ['-dev']
if test_spec.pool:
# TODO(akeshet): Clean up this hack around pool name translation.
autotest_pool_label = 'pool:%s' % test_spec.pool
pool_dependency_value = swarming_lib.task_dependencies_from_labels(
[autotest_pool_label])['label-pool']
cmd += ['-pool', pool_dependency_value]
if test_spec.model:
cmd += ['-model', test_spec.model]
if test_spec.quota_account:
cmd += ['-qs-account', test_spec.quota_account]
if test_spec.test.test_type.lower() == 'client':
cmd += ['-client-test']
tags = _compute_tags(test_spec.build, suite_id)
dimensions = _compute_dimensions(
test_spec.bot_id, test_spec.test.dependencies)
keyvals_flat = _compute_job_keyvals_flat(test_spec.keyvals, suite_id)
for tag in tags:
cmd += ['-tag', tag]
for keyval in keyvals_flat:
cmd += ['-keyval', keyval]
cmd += [test_spec.test.name]
cmd += dimensions
if dry_run:
logging.info('Would have created task with command %s', cmd)
return
# TODO(akeshet): Avoid this late chromite import.
cros_build_lib = autotest.chromite_load('cros_build_lib')
result = cros_build_lib.RunCommand(cmd, capture_output=True)
# TODO(akeshet): Use -json flag and json-parse output of the command instead
# of regex matching to determine task_id.
m = re.match('.*id=(.*)$', result.output)
task_id = m.group(1)
logging.info('Created task with id %s', task_id)
return task_id
def _run(self, board, pool, build, suite_name, timeout_s, raw_cmd):
timeout_s = str(int(timeout_s))
task_name = '%s-%s' % (build, suite_name)
# This is a subset of the tags used by builders when creating suites.
# These tags are used by the result reporting pipeline in various ways.
tags = {
'board': board,
'build': build,
# Required for proper rendering of MILO UI.
'luci_project': 'chromeos',
'skylab': 'run_suite',
'skylab': 'staging',
'suite': suite_name,
'task_name': task_name,
}
osutils = autotest.chromite_load('osutils')
with osutils.TempDir() as tempdir:
summary_file = os.path.join(tempdir, 'summary.json')
cmd = self._base_cmd('run') + [
'--dimension',
'pool',
_SWARMING_POOL_SKYLAB_SUITE_BOTS,
'--expiration',
timeout_s,
'--io-timeout',
timeout_s,
'--hard-timeout',
timeout_s,
'--print-status-update',
'--priority',
str(_TEST_PUSH_SUITE_PRIORITY),
'--raw-cmd',
'--task-name',
task_name,
'--task-summary-json',
summary_file,
'--timeout',
timeout_s,
]
for key, val in tags.iteritems():
cmd += ['--tags', '%s:%s' % (key, val)]
cmd += ['--'] + raw_cmd
cros_build_lib = autotest.chromite_load('cros_build_lib')
cros_build_lib.RunCommand(cmd, error_code_ok=True)
return _extract_run_id(summary_file)
def __init__(self):
# Metrics
metrics = autotest.chromite_load('metrics')
self._hqe_completion_metric = metrics.Counter(
'chromeos/autotest/scheduler/hqe_completion_count')
self._reset_after_failure_metric = metrics.Counter(
'chromeos/autotest/scheduler/postjob_tasks/'
'reset_after_failure')
def main():
"""Entry point of test_push."""
autotest.monkeypatch()
metrics = autotest.chromite_load('metrics')
ts_mon_config = autotest.chromite_load('ts_mon_config')
parser = _get_parser()
loglib.add_logging_options(parser)
args = parser.parse_args()
loglib.configure_logging_with_args(parser, args)
with ts_mon_config.SetupTsMonGlobalState(service_name='skylab_test_push',
indirect=True):
success = False
try:
with metrics.SecondsTimer(_METRICS_PREFIX + '/durations/total',
add_exception_field=True):
_run_test_push(args)
success = True
finally:
metrics.Counter(_METRICS_PREFIX +
'/tick').increment(fields={'success': success})
def _wait_for_results(suite_handler, dry_run=False):
"""Wait for child tasks to finish and return their results.
@param suite_handler: a cros_suite.SuiteHandler object.
"""
timeout_util = autotest.chromite_load('timeout_util')
try:
with timeout_util.Timeout(suite_handler.timeout_mins * 60 -
suite_handler.passed_mins * 60):
_loop_and_wait_forever(suite_handler, dry_run)
except timeout_util.TimeoutError:
logging.error('Timeout in waiting for child tasks.')
return
logging.info('Finished to wait for child tasks.')
def query(self, path, qargs):
"""Run a Swarming 'query' call.
@param path: Path of the query RPC call.
@qargs: Arguments for the RPC call.
@returns: json response from the Swarming call.
"""
cros_build_lib = autotest.chromite_load('cros_build_lib')
cmdarg = path
if qargs:
cmdarg += "?%s" % urllib.urlencode(qargs)
cmd = self._base_cmd('query') + [cmdarg]
result = cros_build_lib.RunCommand(cmd, capture_output=True)
return json.loads(result.output)
def send_hqe_duration(self, hqe):
"""Send CloudTrace metrics for HQE duration."""
if not (hqe.started_on and hqe.finished_on):
return
scheduler_models = autotest.load('scheduler.scheduler_models')
cloud_trace = autotest.chromite_load('cloud_trace')
types = autotest.deps_load('google.protobuf.internal.well_known_types')
hqe_trace_id = scheduler_models.hqe_trace_id
span = cloud_trace.Span(
'HQE', spanId='0', traceId=hqe_trace_id(hqe.id))
span.startTime = types.Timestamp()
span.startTime.FromDatetime(hqe.started_on)
span.endTime = types.Timestamp()
span.endTime.FromDatetime(hqe.finished_on)
cloud_trace.LogSpan(span)
def main(args):
"""Main function
@param args: list of command line args
"""
parser = argparse.ArgumentParser(prog='job_aborter', description=__doc__)
parser.add_argument('--jobdir', required=True)
loglib.add_logging_options(parser)
args = parser.parse_args(args)
loglib.configure_logging_with_args(parser, args)
logger.info('Starting with args: %r', args)
autotest.monkeypatch()
ts_mon_config = autotest.chromite_load('ts_mon_config')
with ts_mon_config.SetupTsMonGlobalState('job_aborter'):
_main_loop(jobdir=args.jobdir)
assert False # cannot exit normally
def _print_task_result_link_annotation(task_id, text):
"""Print the link of task logs.
Given text: 'dummy_Pass-chromeos4-row7-rack6-host19'
task_id: '3ee300e77a576e10'
The printed output will be:
[Test-logs]: dummy_Pass-chromeos4-row7-rack6-host19
Clicking it will direct you to
https://chrome-swarming.appspot.com/task?id=3ee300e77a576e10
@param anchor_test: a string to show on link.
@param task_id: a string task_id to form the swarming url.
"""
annotations = autotest.chromite_load('buildbot_annotations')
print(annotations.StepLink('[Test-logs]: %s' % text,
swarming_lib.get_stainless_logs_link(task_id)))
def __init__(self):
metrics = autotest.chromite_load('metrics')
prefix = 'chromeos/lucifer/job_aborter'
self._starting_m = metrics.Counter(prefix + '/start')
self._tick_m = metrics.Counter(prefix + '/tick')
self._expired_m = metrics.Counter(prefix + '/expired_jobs')
def _log_buildbot_links_for_test_history(task_id, test_name):
annotations = autotest.chromite_load('buildbot_annotations')
reporting_utils = autotest.load('server.cros.dynamic_suite.reporting_utils')
print(annotations.StepLink(
'[Test-History]: %s' % test_name,
reporting_utils.link_test_history(test_name)))
def _run_test_push(args):
"""Meat of the test_push flow."""
metrics = autotest.chromite_load('metrics')
deadline = time.time() + (args.timeout_mins * 60)
swclient = swarming.Client(args.swarming_cli, args.swarming_url,
args.service_account_json)
if args.num_min_duts:
_ensure_duts_ready(
swclient,
args.dut_board,
args.dut_pool,
args.num_min_duts,
min(deadline - time.time(), _WAIT_FOR_DUTS_TIMEOUT_S),
)
# Just like the builders, first run a provision suite to provision required
# DUTs, then run the actual suite.
with metrics.SecondsTimer(_METRICS_PREFIX + '/durations/provision_suite',
add_exception_field=True):
task_id = swclient.trigger_suite(
args.dut_board,
args.dut_pool,
args.build,
'provision',
deadline - time.time(),
)
_logger.info('Triggered provision suite. Task id: %s', task_id)
swclient.wait_for_suite(
task_id,
args.dut_board,
args.dut_pool,
args.build,
'provision',
deadline - time.time(),
)
_logger.info('Finished provision suite.')
with metrics.SecondsTimer(_METRICS_PREFIX +
'/durations/push_to_prod_suite',
add_exception_field=True):
task_id = swclient.trigger_suite(
args.dut_board,
args.dut_pool,
args.build,
'skylab_staging_test',
deadline - time.time(),
)
_logger.info('Triggered skylab_staging_test suite. Task id: %s',
task_id)
_verify_suite_creation(swclient, task_id)
_logger.info('Check push_to_prod suite on: \n %s',
swclient.task_url(task_id))
swclient.wait_for_suite(
task_id,
args.dut_board,
args.dut_pool,
args.build,
'skylab_staging_test',
deadline - time.time(),
)
_logger.info('Finished skylab_staging_test suite.')
_verify_test_results(task_id, _EXPECTED_TEST_RESULTS)