def test_load(name, expected):
"""Test load()."""
with mock.patch('importlib.import_module', autospec=True) \
as import_module, \
mock.patch.object(autotest, '_setup_done', True):
autotest.load(name)
import_module.assert_called_once_with(expected)
def _find_tests(self, available_bots_num=0):
"""Fetch the child tests."""
control_file_getter = autotest.load(
'server.cros.dynamic_suite.control_file_getter')
suite_common = autotest.load('server.cros.dynamic_suite.suite_common')
cf_getter = control_file_getter.DevServerGetter(
self.test_source_build, self.ds)
tests = suite_common.retrieve_for_suite(cf_getter, self.suite_name)
return suite_common.filter_tests(
tests, suite_common.name_in_tag_predicate(self.suite_name))
def _handle_host_needs_reset(self, msg):
models = autotest.load('frontend.afe.models')
host = models.Host.objects.get(hostname=msg)
models.SpecialTask.objects.create(
host_id=host.id,
task=models.SpecialTask.Task.RESET,
requested_by=models.User.objects.get(login=self._job.owner))
def _add_starting_args(command_args, args, job):
"""Add STARTING level arguments for lucifer test.
command_args is modified in place.
"""
RebootAfter = autotest.load('frontend.afe.model_attributes').RebootAfter
command_args.extend([
'-x-control-file', jobx.control_file_path(args.results_dir),
])
if args.execution_tag is not None:
command_args.extend(['-x-execution-tag', args.execution_tag])
command_args.extend(['-x-job-owner', job.owner])
command_args.extend(['-x-job-name', job.name])
command_args.extend(
['-x-reboot-after',
RebootAfter.get_string(job.reboot_after).lower()])
if args.parsing_only:
command_args.append('-x-parse-only')
if job.run_reset:
command_args.append('-x-run-reset')
if jobx.is_client_job(job):
command_args.append('-x-client-test')
if jobx.needs_ssp(job):
command_args.append('-x-require-ssp')
test_source_build = job.keyval_dict().get('test_source_build', None)
if test_source_build:
command_args.extend(['-x-test-source-build', test_source_build])
if job.parent_job_id:
command_args.extend(['-x-parent-job-id', str(job.parent_job_id)])
def _clean_up_hqes(hqes):
models = autotest.load('frontend.afe.models')
logger.debug('Cleaning up HQEs: %r', hqes.values_list('id', flat=True))
hqes.update(complete=True,
active=False,
status=models.HostQueueEntry.Status.FAILED)
(hqes.exclude(started_on=None).update(finished_on=datetime.datetime.now()))
def _handle_host_running(self, msg):
models = autotest.load('frontend.afe.models')
host = models.Host.objects.get(hostname=msg)
host.status = models.Host.Status.RUNNING
host.dirty = 1
host.save(update_fields=['status', 'dirty'])
self._metrics.send_host_status(host)
def _final_status(self):
models = autotest.load('frontend.afe.models')
Status = models.HostQueueEntry.Status
if jobx.is_aborted(self._job):
return Status.ABORTED
if self._autoserv_exit == 0:
return Status.COMPLETED
return Status.FAILED
def _aborting_jobs_queryset():
"""Return a QuerySet of aborting Jobs.
@returns: Django QuerySet
"""
models = autotest.load('frontend.afe.models')
return (models.Job.objects.filter(hostqueueentry__aborted=True).filter(
hostqueueentry__complete=False).distinct())
def _get_prejob_hqes(job, include_active=True):
"""Return a queryset of not run HQEs for the job (for synch_count)."""
models = autotest.load('frontend.afe.models')
if include_active:
statuses = list(models.HostQueueEntry.PRE_JOB_STATUSES)
else:
statuses = list(models.HostQueueEntry.IDLE_PRE_JOB_STATUSES)
return models.HostQueueEntry.objects.filter(job=job, status__in=statuses)
def _stage_suite_artifacts(self):
"""Stage suite control files and suite-to-tests mapping file.
@param build: The build to stage artifacts.
"""
suite_common = autotest.load('server.cros.dynamic_suite.suite_common')
ds, _ = suite_common.stage_build_artifacts(self.test_source_build)
self._ds = ds
def mark_complete(job_ids):
"""Mark the corresponding JobHandoffs as completed."""
if not job_ids:
return
models = autotest.load('frontend.afe.models')
logger.info('Marking job handoffs complete: %r', job_ids)
(models.JobHandoff.objects.filter(job_id__in=job_ids).update(
completed=True))
def _timed_out_jobs_queryset():
"""Return a QuerySet of timed out Jobs.
@returns: Django QuerySet
"""
models = autotest.load('frontend.afe.models')
return (models.Job.objects.filter(hostqueueentry__complete=False).extra(
where=['created_on + INTERVAL timeout_mins MINUTE < NOW()'
]).distinct())
def _host_labels(host):
"""Return an iterable of labels for a host.
@param host: frontend.afe.models.Host instance
"""
if autotest.load('scheduler.scheduler_models').RESPECT_STATIC_LABELS:
return _host_labels_with_static(host)
else:
return host.labels.all()
def _stop_job_if_necessary(self, final_status):
"""Equivalent to scheduler.modes.Job.stop_if_necessary().
The name isn't informative, but this will stop pre-job tasks as
necessary.
"""
models = autotest.load('frontend.afe.models')
if final_status is not models.HostQueueEntry.Status.ABORTED:
_stop_prejob_hqes(self._job)
def _find_tests(self, available_bots_num=0):
"""Fetch the child tests for provision suite."""
control_file_getter = autotest.load(
'server.cros.dynamic_suite.control_file_getter')
suite_common = autotest.load('server.cros.dynamic_suite.suite_common')
cf_getter = control_file_getter.DevServerGetter(
self.test_source_build, self.ds)
dummy_test = suite_common.retrieve_control_data_for_test(
cf_getter, 'dummy_Pass')
logging.info('Get %d available DUTs for provision.',
available_bots_num)
if available_bots_num < self._num_required:
logging.warning('Not enough available DUTs for provision.')
raise errors.NoAvailableDUTsError(self.board, self.pool,
available_bots_num,
self._num_required)
return [dummy_test] * max(self._num_required, available_bots_num)
def _compute_job_keyvals_flat(keyvals, suite_id):
# Job keyvals calculation.
job_keyvals = keyvals.copy()
if suite_id is not None:
# TODO(akeshet): Avoid this late autotest constants import.
constants = autotest.load('server.cros.dynamic_suite.constants')
job_keyvals[constants.PARENT_JOB_ID] = suite_id
keyvals_flat = sorted(
['%s:%s' % (k, v) for k, v in job_keyvals.items()])
return keyvals_flat
def _parse_suite_args(self):
"""Get the suite args.
The suite args includes:
a. suite args in suite control file.
b. passed-in suite args by user.
"""
suite_common = autotest.load('server.cros.dynamic_suite.suite_common')
self.control_file = suite_common.get_control_file_by_build(
self.test_source_build, self.ds, self.suite_file_name)
def _make_handler(args):
"""Make event handler for lucifer_run_job."""
models = autotest.load('frontend.afe.models')
if args.autoserv_exit is None:
# TODO(crbug.com/748234): autoserv not implemented yet.
raise NotImplementedError('not implemented yet (crbug.com/748234)')
job = models.Job.objects.get(id=args.job_id)
return handlers.EventHandler(
metrics=handlers.Metrics(),
job=job,
autoserv_exit=args.autoserv_exit,
)
def _handle_completed(self, _msg):
models = autotest.load('frontend.afe.models')
final_status = self._final_status()
for hqe in self._job.hostqueueentry_set.all():
self._set_completed_status(hqe, final_status)
if final_status is not models.HostQueueEntry.Status.ABORTED:
_stop_prejob_hqes(self._job)
if self._job.shard_id is not None:
# If shard_id is None, the job will be synced back to the master
self._job.shard_id = None
self._job.save()
self.completed = True
def is_client_job(job):
"""Return whether the job is a client job.
If the job is not a client job, it is a server job.
(In theory a job can be neither. I have no idea what you should do
in that case.)
@param job: frontend.afe.models.Job instance
"""
CONTROL_TYPE = autotest.load('client.common_lib.control_data').CONTROL_TYPE
return CONTROL_TYPE.get_value(job.control_type) == CONTROL_TYPE.CLIENT
def _clean_up_hosts(host_ids):
models = autotest.load('frontend.afe.models')
transaction = autotest.deps_load('django.db.transaction')
with transaction.commit_on_success():
active_hosts = {
id
for id in (models.HostQueueEntry.objects.filter(
active=True, complete=False).values_list('host_id', flat=True))
if id is not None
}
logger.debug('Found active Hosts: %r', active_hosts)
(models.Host.objects.filter(id__in=host_ids).exclude(
id__in=active_hosts).update(status=models.Host.Status.READY))
def send_hqe_completion(self, hqe):
"""Send ts_mon metrics for HQE completion."""
fields = {
'status': hqe.status.lower(),
'board': 'NO_HOST',
'pool': 'NO_HOST',
}
if hqe.host:
labellib = autotest.load('utils.labellib')
labels = labellib.LabelsMapping.from_host(hqe.host)
fields['board'] = labels.get('board', '')
fields['pool'] = labels.get('pool', '')
self._hqe_completion_metric.increment(fields=fields)
def _run_autotest_job(args):
"""Run a job as seen from Autotest.
This include some Autotest setup and cleanup around lucifer starting
proper.
"""
models = autotest.load('frontend.afe.models')
job = models.Job.objects.get(id=args.job_id)
_prepare_autotest_job_files(args, job)
handler = _make_handler(args, job)
ret = _run_lucifer_job(handler, args, job)
if handler.completed:
_mark_handoff_completed(args.job_id)
return ret
def _create_suite_keyvals(self):
constants = autotest.load('server.cros.dynamic_suite.constants')
provision = autotest.load('server.cros.provision')
cros_build = self._get_cros_build()
keyvals = {
constants.JOB_BUILD_KEY: cros_build,
constants.JOB_SUITE_KEY: self.suite_name,
constants.JOB_BUILDS_KEY: self.builds
}
if (cros_build != self.test_source_build or len(self.builds) > 1):
keyvals[constants.JOB_TEST_SOURCE_BUILD_KEY] = (
self.test_source_build)
for prefix, build in self.builds.iteritems():
if prefix == provision.FW_RW_VERSION_PREFIX:
keyvals[constants.FWRW_BUILD] = build
elif prefix == provision.FW_RO_VERSION_PREFIX:
keyvals[constants.FWRO_BUILD] = build
for key in self.job_keyvals:
if key in constants.INHERITED_KEYVALS:
keyvals[key] = self.job_keyvals[key]
return keyvals
def _stop_prejob_hqes(job):
"""Stop pending HQEs for a job (for synch_count)."""
models = autotest.load('frontend.afe.models')
HQEStatus = models.HostQueueEntry.Status
HostStatus = models.Host.Status
not_yet_run = _get_prejob_hqes(job)
if not_yet_run.count() == job.synch_count:
return
entries_to_stop = _get_prejob_hqes(job, include_active=False)
for hqe in entries_to_stop:
if hqe.status == HQEStatus.PENDING:
hqe.host.status = HostStatus.READY
hqe.host.save()
hqe.status = HQEStatus.STOPPED
hqe.save()
def _should_reboot_duts(self, autoserv_exit, failures, reset_after_failure):
models = autotest.load('frontend.afe.models')
reboot_after = self._job.reboot_after
if self._final_status() == models.HostQueueEntry.Status.ABORTED:
logger.debug('Should reboot because reboot_after=ABORTED')
return True
elif reboot_after == models.Job.RebootAfter.ALWAYS:
logger.debug('Should reboot because reboot_after=ALWAYS')
return True
elif (reboot_after == models.Job.RebootAfter.IF_ALL_TESTS_PASSED
and autoserv_exit == 0 and failures == 0):
logger.debug('Should reboot because'
' reboot_after=IF_ALL_TESTS_PASSED')
return True
else:
return failures > 0 and not reset_after_failure
def send_hqe_duration(self, hqe):
"""Send CloudTrace metrics for HQE duration."""
if not (hqe.started_on and hqe.finished_on):
return
scheduler_models = autotest.load('scheduler.scheduler_models')
cloud_trace = autotest.chromite_load('cloud_trace')
types = autotest.deps_load('google.protobuf.internal.well_known_types')
hqe_trace_id = scheduler_models.hqe_trace_id
span = cloud_trace.Span(
'HQE', spanId='0', traceId=hqe_trace_id(hqe.id))
span.startTime = types.Timestamp()
span.startTime.FromDatetime(hqe.started_on)
span.endTime = types.Timestamp()
span.endTime.FromDatetime(hqe.finished_on)
cloud_trace.LogSpan(span)
def clean_up(job_ids):
"""Clean up failed jobs failed in database.
This brings the database into a clean state, which includes marking
the job, HQEs, and hosts.
"""
if not job_ids:
return
models = autotest.load('frontend.afe.models')
logger.info('Cleaning up failed jobs: %r', job_ids)
hqes = models.HostQueueEntry.objects.filter(job_id__in=job_ids)
logger.debug('Cleaning up HQEs: %r', hqes.values_list('id', flat=True))
_clean_up_hqes(hqes)
host_ids = {id for id in hqes.values_list('host_id', flat=True)
if id is not None}
logger.debug('Found Hosts associated with jobs: %r', host_ids)
_clean_up_hosts(host_ids)
def _host_labels_with_static(host):
"""Return a generator of labels for a host, respecting static labels.
@param host: frontend.afe.models.Host instance
"""
models = autotest.load('frontend.afe.models')
replaced_label_ids = frozenset(models.ReplacedLabel.objects.all()
.values_list('label_id', flat=True))
shadowed_labels = set()
for label in host.labels.all():
if label.id in replaced_label_ids:
shadowed_labels.add(label.name)
else:
yield label
for label in host.static_labels.all():
if label.name in shadowed_labels:
yield label
def _get_final_suite_states():
run_suite_common = autotest.load('site_utils.run_suite_common')
return {
swarming_lib.TASK_COMPLETED_FAILURE:
(
swarming_lib.TASK_COMPLETED_FAILURE,
run_suite_common.RETURN_CODES.ERROR,
),
# Task No_Resource means no available bots to accept the task.
# Deputy should check whether it's infra failure.
swarming_lib.TASK_NO_RESOURCE:
(
swarming_lib.TASK_NO_RESOURCE,
run_suite_common.RETURN_CODES.INFRA_FAILURE,
),
# Task expired means a task is not triggered, could be caused by
# 1. No healthy DUTs/bots to run it.
# 2. Expiration seconds are too low.
# 3. Suite run is too slow to finish.
# Deputy should check whether it's infra failure.
swarming_lib.TASK_EXPIRED:
(
swarming_lib.TASK_EXPIRED,
run_suite_common.RETURN_CODES.INFRA_FAILURE,
),
# Task canceled means a task is canceled intentionally. Deputy
# should check whether it's infra failure.
swarming_lib.TASK_CANCELED:
(
swarming_lib.TASK_CANCELED,
run_suite_common.RETURN_CODES.INFRA_FAILURE,
),
swarming_lib.TASK_TIMEDOUT:
(
swarming_lib.TASK_TIMEDOUT,
run_suite_common.RETURN_CODES.SUITE_TIMEOUT,
),
# Task pending means a task is still waiting for picking up, but
# the suite already hits deadline. So report it as suite TIMEOUT.
# It could also be an INFRA_FAILURE due to DUTs/bots shortage.
swarming_lib.TASK_PENDING:
(
swarming_lib.TASK_TIMEDOUT,
run_suite_common.RETURN_CODES.SUITE_TIMEOUT,
),
}