def testSuiteMaxRetries(self):
"""Test suite max retries."""
name_list = [
'name-data_one', 'name-data_two', 'name-data_three',
'name-data_four', 'name-data_five', 'name-data_six',
'name-data_seven'
]
keyval_dict = {
constants.SCHEDULED_TEST_COUNT_KEY: 7,
constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)
}
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder)
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
afe=self.afe,
tko=self.tko,
job_retry=True,
max_retries=1)
suite.schedule(recorder.record_entry)
self.assertEqual(suite._retry_handler._max_retries, 1)
# Find the job_id of the test that allows retry
job_id = suite._retry_handler._retry_map.iterkeys().next()
suite._retry_handler.add_retry(old_job_id=job_id, new_job_id=10)
self.assertEqual(suite._retry_handler._max_retries, 0)
def testSuiteDependencies(self):
"""Should add suite dependencies to tests scheduled."""
name_list = [
'name-data_one', 'name-data_two', 'name-data_three',
'name-data_four', 'name-data_five', 'name-data_six',
'name-data_seven'
]
keyval_dict = {
constants.SCHEDULED_TEST_COUNT_KEY: len(name_list),
constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)
}
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, suite_deps=['extra'])
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
extra_deps=['extra'],
afe=self.afe,
tko=self.tko)
suite.schedule(recorder.record_entry)
def testRetryMapAfterScheduling(self):
"""Test job-test and test-job mapping are correctly updated."""
name_list = ['name-data_one', 'name-data_two', 'name-data_three',
'name-data_four', 'name-data_five', 'name-data_six',
'name-data_seven']
keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 7,
constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)}
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder)
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(None, keyval_dict)
all_files = self.files.items()
# Sort tests in self.files so that they are in the same
# order as they are scheduled.
expected_retry_map = {}
for n in range(len(all_files)):
test = all_files[n][1]
job_id = n + 1
job_retries = 1 if test.job_retries is None else test.job_retries
if job_retries > 0:
expected_retry_map[job_id] = {
'state': RetryHandler.States.NOT_ATTEMPTED,
'retry_max': job_retries}
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,
self.devserver,
afe=self.afe, tko=self.tko,
job_retry=True)
suite.schedule(recorder.record_entry)
self.assertEqual(expected_retry_map, suite._retry_handler._retry_map)
def testScheduleTestsAndRecord(self):
"""Should schedule stable and experimental tests with the AFE."""
name_list = [
'name-data_one', 'name-data_two', 'name-data_three',
'name-data_four', 'name-data_five', 'name-data_six',
'name-data_seven'
]
keyval_dict = {
constants.SCHEDULED_TEST_COUNT_KEY: 7,
constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)
}
self.mock_control_file_parsing()
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG,
self._BUILDS,
self._BOARD,
self.devserver,
afe=self.afe,
tko=self.tko,
results_dir=self.tmpdir)
self.mox.ResetAll()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, suite=suite)
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(self.tmpdir, keyval_dict)
self.mox.ReplayAll()
suite.schedule(recorder.record_entry)
for job in suite._jobs:
self.assertTrue(hasattr(job, 'test_name'))
def wrapper(machine):
self.push_execution_context(machine)
os.chdir(self.resultdir)
machine_data = {"hostname": machine, "status_version": str(self._STATUS_VERSION)}
utils.write_keyval(self.resultdir, machine_data)
result = function(machine)
return result
def testInheritedKeyvals(self):
"""Tests should inherit some whitelisted job keyvals."""
# Only keyvals in constants.INHERITED_KEYVALS are inherited to tests.
job_keyvals = {
constants.KEYVAL_CIDB_BUILD_ID: '111',
constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222',
'your': 'name',
}
test_keyvals = {
constants.KEYVAL_CIDB_BUILD_ID: '111',
constants.KEYVAL_CIDB_BUILD_STAGE_ID: '222',
}
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(
recorder,
extra_keyvals=test_keyvals)
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(None, job_keyvals)
utils.write_keyval(None, mox.IgnoreArg())
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,
self.devserver,
afe=self.afe, tko=self.tko,
job_keyvals=job_keyvals)
suite.schedule(recorder.record_entry)
def __init__(self, server_job, machine, work_dir, test_queue, queue_lock,
continuous_parsing=False):
"""Creates an instance of machine_worker to run tests on a remote host.
Retrieves that host attributes for this machine and creates the set of
True attributes to validate against test include/exclude attributes.
Creates a directory to hold the log files for tests run and writes the
hostname and tko parser version into keyvals file.
Args:
server_job: run tests for this server_job.
machine: name of remote host.
work_dir: directory server job is using.
test_queue: queue of tests.
queue_lock: lock protecting test_queue.
continuous_parsing: bool, enable continuous parsing.
"""
self._server_job = server_job
self._test_queue = test_queue
self._test_queue_lock = queue_lock
self._continuous_parsing = continuous_parsing
self._tests_run = 0
self._machine = machine
self._host = hosts.create_host(self._machine)
self._client_at = autotest.Autotest(self._host)
client_attributes = host_attributes.host_attributes(machine)
self.attribute_set = set(client_attributes.get_attributes())
self._results_dir = work_dir
if not os.path.exists(self._results_dir):
os.makedirs(self._results_dir)
machine_data = {'hostname': self._machine,
'status_version': str(1)}
utils.write_keyval(self._results_dir, machine_data)
def wrapper(machine):
self.resultdir = os.path.join(self.resultdir, machine)
os.chdir(self.resultdir)
machine_data = {'hostname' : machine,
'status_version' : str(self.STATUS_VERSION)}
utils.write_keyval(self.resultdir, machine_data)
result = function(machine)
return result
def wrapper(machine):
self.push_execution_context(machine)
os.chdir(self.resultdir)
machine_data = {'hostname' : machine,
'status_version' : str(self._STATUS_VERSION)}
utils.write_keyval(self.resultdir, machine_data)
result = function(machine)
return result
def wrapper(machine):
self.parse_job += "/" + machine
self.using_parser = True
self.machines = [machine]
self.resultdir = os.path.join(self.resultdir, machine)
os.chdir(self.resultdir)
utils.write_keyval(self.resultdir, {"hostname": machine})
self.init_parser(self.resultdir)
result = function(machine)
self.cleanup_parser()
return result
def wrapper(machine):
self._parse_job += "/" + machine
self._using_parser = True
self.machines = [machine]
self.push_execution_context(machine)
os.chdir(self.resultdir)
utils.write_keyval(self.resultdir, {"hostname": machine})
self.init_parser()
result = function(machine)
self.cleanup_parser()
return result
def wrapper(machine):
self._parse_job += "/" + machine
self._using_parser = True
self.machines = [machine]
self.push_execution_context(machine)
os.chdir(self.resultdir)
utils.write_keyval(self.resultdir, {"hostname": machine})
self.init_parser()
result = function(machine)
self.cleanup_parser()
return result
def write_keyval(self, filename, dictionary, expected_filename=None, type_tag=None):
if expected_filename is None:
expected_filename = filename
test_file = StringIO.StringIO()
self.god.stub_function(test_file, "close")
utils.open.expect_call(expected_filename, "a").and_return(test_file)
test_file.close.expect_call()
if type_tag is None:
utils.write_keyval(filename, dictionary)
else:
utils.write_keyval(filename, dictionary, type_tag)
return test_file.getvalue()
def record_summary(self,
status_code,
test_name,
reason='',
attributes=None,
distinguishing_attributes=(),
child_test_ids=None):
"""Record a summary test result.
@param status_code: status code string, see
common_lib.log.is_valid_status()
@param test_name: name of the test
@param reason: (optional) string providing detailed reason for test
outcome
@param attributes: (optional) dict of string keyvals to associate with
this result
@param distinguishing_attributes: (optional) list of attribute names
that should be used to distinguish identically-named test
results. These attributes should be present in the attributes
parameter. This is used to generate user-friendly subdirectory
names.
@param child_test_ids: (optional) list of test indices for test results
used in generating this result.
"""
subdirectory_name_parts = [test_name]
for attribute in distinguishing_attributes:
assert attributes
assert attribute in attributes, '%s not in %s' % (attribute,
attributes)
subdirectory_name_parts.append(attributes[attribute])
base_subdirectory_name = '.'.join(subdirectory_name_parts)
subdirectory = self._unique_subdirectory(base_subdirectory_name)
subdirectory_path = os.path.join(self.resultdir, subdirectory)
os.mkdir(subdirectory_path)
self.record(status_code,
subdirectory,
test_name,
status=reason,
optional_fields={'is_summary': True})
if attributes:
utils.write_keyval(subdirectory_path, attributes)
if child_test_ids:
ids_string = ','.join(str(test_id) for test_id in child_test_ids)
summary_data = {'child_test_ids': ids_string}
utils.write_keyval(os.path.join(subdirectory_path, 'summary_data'),
summary_data)
def write_keyval(self,
filename,
dictionary,
expected_filename=None,
type_tag=None):
if expected_filename is None:
expected_filename = filename
test_file = StringIO.StringIO()
self.god.stub_function(test_file, "close")
utils.open.expect_call(expected_filename, "a").and_return(test_file)
test_file.close.expect_call()
if type_tag is None:
utils.write_keyval(filename, dictionary)
else:
utils.write_keyval(filename, dictionary, type_tag)
return test_file.getvalue()
def _run_suite(suite, job, run_prod_code, devserver, build, delay_minutes,
bug_template):
"""
Run a suite.
@param suite: _BaseSuite instance.
@param job: an instance of client.common_lib.base_job representing the
currently running suite job.
@param run_prod_code: whether to use prod test code.
@param devserver: devserver for staging artifacts.
@param build: the build to install e.g. 'x86-alex-release/R18-1655.0.0'
@param delay_minutes: Delay the creation of test jobs for a given number
of minutes.
@param bug_template: A template dictionary specifying the default bug
filing options for failures in this suite.
"""
# We can't do anything else until the devserver has finished downloading
# control_files and test_suites packages so that we can get the control
# files we should schedule.
if not run_prod_code:
_stage_artifacts_for_build(devserver, build)
timestamp = datetime.datetime.now().strftime(time_utils.TIME_FMT)
utils.write_keyval(job.resultdir,
{constants.ARTIFACT_FINISHED_TIME: timestamp})
if delay_minutes:
logging.debug(
'delay_minutes is set. Sleeping %d minutes before '
'creating test jobs.', delay_minutes)
time.sleep(delay_minutes * 60)
logging.debug(
'Finished waiting for %d minutes before creating test '
'jobs.', delay_minutes)
# Now we get to asychronously schedule tests.
suite.schedule(job.record_entry)
if suite.wait_for_results:
logging.debug('Waiting on suite.')
reporter = suite.get_result_reporter(bug_template)
suite.wait(job.record_entry, reporter=reporter)
logging.debug('Finished waiting on suite. '
'Returning from _perform_reimage_and_run.')
else:
logging.info('wait_for_results is set to False, suite job will exit '
'without waiting for test jobs to finish.')
def testScheduleUnrunnableTestsTESTNA(self):
"""Tests which fail to schedule should be TEST_NA."""
# Since all tests will be fail to schedule, the num of scheduled tests
# will be zero.
name_list = []
keyval_dict = {constants.SCHEDULED_TEST_COUNT_KEY: 0,
constants.SCHEDULED_TEST_NAMES_KEY: repr(name_list)}
self.mock_control_file_parsing()
recorder = self.mox.CreateMock(base_job.base_job)
self.expect_job_scheduling(recorder, raises=True)
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(None, keyval_dict)
self.mox.ReplayAll()
suite = Suite.create_from_name(self._TAG, self._BUILDS, self._BOARD,
self.devserver,
afe=self.afe, tko=self.tko)
suite.schedule(recorder.record_entry)
def mock_bug_filing(self, test_results):
"""A helper function that mocks bug filing.
@param test_results: A named tuple (predicates, fallout) representing
a bad test report.
"""
def check_result(result):
"""
Checks to see if the status passed to the bug reporter contains all
the arguments required to file bugs.
@param result: The result we get when a test fails.
"""
test_predicates = test_results[0]
test_fallout = test_results[1]
expected_result = job_status.Status(
test_predicates.status,
test_predicates.testname,
reason=test_predicates.reason,
job_id=test_fallout.job_id,
owner=test_fallout.username,
hostname=test_fallout.hostname,
begin_time_str=test_fallout.time_start)
return all(
getattr(result, k, None) == v
for k, v in expected_result.__dict__.iteritems()
if 'timestamp' not in str(k))
self.mox.StubOutWithMock(reporting, 'TestBug')
reporting.TestBug(self._BUILDS[provision.CROS_VERSION_PREFIX],
mox.IgnoreArg(), mox.IgnoreArg(),
mox.Func(check_result))
self.mox.StubOutClassWithMocks(phapi_lib, 'ProjectHostingApiClient')
mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
self.mox.StubOutWithMock(reporting.Reporter, 'report')
reporting.Reporter.report(mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(
(0, 0))
self.mox.StubOutWithMock(utils, 'write_keyval')
utils.write_keyval(mox.IgnoreArg(), mox.IgnoreArg())
def record_summary(self, status_code, test_name, reason='', attributes=None,
distinguishing_attributes=(), child_test_ids=None):
"""Record a summary test result.
@param status_code: status code string, see
common_lib.log.is_valid_status()
@param test_name: name of the test
@param reason: (optional) string providing detailed reason for test
outcome
@param attributes: (optional) dict of string keyvals to associate with
this result
@param distinguishing_attributes: (optional) list of attribute names
that should be used to distinguish identically-named test
results. These attributes should be present in the attributes
parameter. This is used to generate user-friendly subdirectory
names.
@param child_test_ids: (optional) list of test indices for test results
used in generating this result.
"""
subdirectory_name_parts = [test_name]
for attribute in distinguishing_attributes:
assert attributes
assert attribute in attributes, '%s not in %s' % (attribute,
attributes)
subdirectory_name_parts.append(attributes[attribute])
base_subdirectory_name = '.'.join(subdirectory_name_parts)
subdirectory = self._unique_subdirectory(base_subdirectory_name)
subdirectory_path = os.path.join(self.resultdir, subdirectory)
os.mkdir(subdirectory_path)
self.record(status_code, subdirectory, test_name,
status=reason, optional_fields={'is_summary': True})
if attributes:
utils.write_keyval(subdirectory_path, attributes)
if child_test_ids:
ids_string = ','.join(str(test_id) for test_id in child_test_ids)
summary_data = {'child_test_ids': ids_string}
utils.write_keyval(os.path.join(subdirectory_path, 'summary_data'),
summary_data)
def _run_suite(
suite,
job,
delay_minutes,
bug_template):
"""
Run a suite.
@param suite: _BaseSuite instance.
@param job: an instance of client.common_lib.base_job representing the
currently running suite job.
@param delay_minutes: Delay the creation of test jobs for a given number
of minutes.
@param bug_template: A template dictionary specifying the default bug
filing options for failures in this suite.
"""
timestamp = datetime.datetime.now().strftime(time_utils.TIME_FMT)
utils.write_keyval(
job.resultdir,
{constants.ARTIFACT_FINISHED_TIME: timestamp})
if delay_minutes:
logging.debug('delay_minutes is set. Sleeping %d minutes before '
'creating test jobs.', delay_minutes)
time.sleep(delay_minutes*60)
logging.debug('Finished waiting for %d minutes before creating test '
'jobs.', delay_minutes)
# Now we get to asychronously schedule tests.
suite.schedule(job.record_entry)
if suite.wait_for_results:
logging.debug('Waiting on suite.')
suite.wait(job.record_entry)
logging.debug('Finished waiting on suite. '
'Returning from _perform_reimage_and_run.')
else:
logging.info('wait_for_results is set to False, suite job will exit '
'without waiting for test jobs to finish.')
def _perform_reimage_and_run(spec, afe, tko, suite_job_id=None):
"""
Do the work of reimaging hosts and running tests.
@param spec: a populated SuiteSpec object.
@param afe: an instance of AFE as defined in server/frontend.py.
@param tko: an instance of TKO as defined in server/frontend.py.
@param suite_job_id: Job id that will act as parent id to all sub jobs.
Default: None
"""
# We can't do anything else until the devserver has finished downloading
# control_files and test_suites packages so that we can get the control
# files we should schedule.
if not spec.run_prod_code:
_stage_artifacts(spec)
timestamp = datetime.datetime.now().strftime(time_utils.TIME_FMT)
utils.write_keyval(
spec.job.resultdir,
{constants.ARTIFACT_FINISHED_TIME: timestamp})
suite = Suite.create_from_predicates(
predicates=[spec.predicate],
name=spec.name,
builds=spec.builds,
board=spec.board,
devserver=spec.devserver,
afe=afe,
tko=tko,
pool=spec.pool,
results_dir=spec.job.resultdir,
max_runtime_mins=spec.max_runtime_mins,
timeout_mins=spec.timeout_mins,
file_bugs=spec.file_bugs,
file_experimental_bugs=spec.file_experimental_bugs,
suite_job_id=suite_job_id,
extra_deps=spec.suite_dependencies,
priority=spec.priority,
wait_for_results=spec.wait_for_results,
job_retry=spec.job_retry,
max_retries=spec.max_retries,
offload_failures_only=spec.offload_failures_only,
test_source_build=spec.test_source_build,
run_prod_code=spec.run_prod_code,
job_keyvals=spec.job_keyvals,
test_args=spec.test_args)
if spec.delay_minutes:
logging.debug('delay_minutes is set. Sleeping %d minutes before '
'creating test jobs.', spec.delay_minutes)
time.sleep(spec.delay_minutes*60)
logging.debug('Finished waiting for %d minutes before creating test '
'jobs.', spec.delay_minutes)
# Now we get to asychronously schedule tests.
suite.schedule(spec.job.record_entry, spec.add_experimental)
if suite.wait_for_results:
logging.debug('Waiting on suite.')
suite.wait(spec.job.record_entry, spec.bug_template)
logging.debug('Finished waiting on suite. '
'Returning from _perform_reimage_and_run.')
else:
logging.info('wait_for_results is set to False, suite job will exit '
'without waiting for test jobs to finish.')
def __init__(self,
control,
args,
resultdir,
label,
user,
machines,
client=False,
parse_job='',
ssh_user='******',
ssh_port=22,
ssh_pass='',
group_name='',
tag='',
control_filename=SERVER_CONTROL_FILENAME):
"""
Create a server side job object.
@param control: The pathname of the control file.
@param args: Passed to the control file.
@param resultdir: Where to throw the results.
@param label: Description of the job.
@param user: Username for the job (email address).
@param client: True if this is a client-side control file.
@param parse_job: string, if supplied it is the job execution tag that
the results will be passed through to the TKO parser with.
@param ssh_user: The SSH username. [root]
@param ssh_port: The SSH port number. [22]
@param ssh_pass: The SSH passphrase, if needed.
@param group_name: If supplied, this will be written out as
host_group_name in the keyvals file for the parser.
@param tag: The job execution tag from the scheduler. [optional]
@param control_filename: The filename where the server control file
should be written in the results directory.
"""
super(base_server_job, self).__init__(resultdir=resultdir)
path = os.path.dirname(__file__)
self.control = control
self._uncollected_log_file = os.path.join(self.resultdir,
'uncollected_logs')
debugdir = os.path.join(self.resultdir, 'debug')
if not os.path.exists(debugdir):
os.mkdir(debugdir)
if user:
self.user = user
else:
self.user = getpass.getuser()
self.args = args
self.machines = machines
self._client = client
self.warning_loggers = set()
self.warning_manager = warning_manager()
self._ssh_user = ssh_user
self._ssh_port = ssh_port
self._ssh_pass = ssh_pass
self.tag = tag
self.last_boot_tag = None
self.hosts = set()
self.drop_caches = False
self.drop_caches_between_iterations = False
self._control_filename = control_filename
self.logging = logging_manager.get_logging_manager(
manage_stdout_and_stderr=True, redirect_fds=True)
subcommand.logging_manager_object = self.logging
self.sysinfo = sysinfo.sysinfo(self.resultdir)
self.profilers = profilers.profilers(self)
job_data = {
'label': label,
'user': user,
'hostname': ','.join(machines),
'drone': platform.node(),
'status_version': str(self._STATUS_VERSION),
'job_started': str(int(time.time()))
}
if group_name:
job_data['host_group_name'] = group_name
# only write these keyvals out on the first job in a resultdir
if 'job_started' not in utils.read_keyval(self.resultdir):
job_data.update(get_site_job_data(self))
utils.write_keyval(self.resultdir, job_data)
self._parse_job = parse_job
self._using_parser = (self._parse_job and len(machines) <= 1)
self.pkgmgr = packages.PackageManager(
self.autodir, run_function_dargs={'timeout': 600})
self.num_tests_run = 0
self.num_tests_failed = 0
self._register_subcommand_hooks()
# these components aren't usable on the server
self.bootloader = None
self.harness = None
# set up the status logger
self._indenter = status_indenter()
self._logger = base_job.status_logger(
self,
self._indenter,
'status.log',
'status.log',
record_hook=server_job_record_hook(self))
def postprocess_iteration(self):
"""Postprocess: write keyvals / log and send data to power dashboard."""
def _log_stats(prefix, stats):
if not len(stats):
return
np = numpy.array(stats)
logging.debug("%s samples: %d", prefix, len(np))
logging.debug("%s mean: %.2f", prefix, np.mean())
logging.debug("%s stdev: %.2f", prefix, np.std())
logging.debug("%s max: %.2f", prefix, np.max())
logging.debug("%s min: %.2f", prefix, np.min())
def _log_per_loop_stats():
samples_per_loop = self._loop_time / self._wait_time + 1
for kname in self._stats:
start_idx = 0
loop = 1
for end_idx in xrange(samples_per_loop, len(self._stats[kname]),
samples_per_loop):
_log_stats("%s loop %d" % (kname, loop),
self._stats[kname][start_idx:end_idx])
loop += 1
start_idx = end_idx
def _log_all_stats():
for kname in self._stats:
_log_stats(kname, self._stats[kname])
for task, tstart, tend in self._task_tracker:
self._checkpoint_logger.checkpoint('_' + task, tstart, tend)
keyvals = {}
for log in self._meas_logs:
keyvals.update(log.calc())
keyvals.update(self._statomatic.publish())
if self._log_mem_bandwidth:
self._mlog.stop()
self._mlog.join()
_log_all_stats()
_log_per_loop_stats()
# record battery stats
if self._power_status.battery:
keyvals['a_current_now'] = self._power_status.battery[0].current_now
keyvals['ah_charge_full'] = \
self._power_status.battery[0].charge_full
keyvals['ah_charge_full_design'] = \
self._power_status.battery[0].charge_full_design
keyvals['ah_charge_start'] = self._ah_charge_start
keyvals['ah_charge_now'] = self._power_status.battery[0].charge_now
keyvals['ah_charge_used'] = keyvals['ah_charge_start'] - \
keyvals['ah_charge_now']
keyvals['wh_energy_start'] = self._wh_energy_start
keyvals['wh_energy_now'] = self._power_status.battery[0].energy
keyvals['wh_energy_used'] = keyvals['wh_energy_start'] - \
keyvals['wh_energy_now']
keyvals['v_voltage_min_design'] = \
self._power_status.battery[0].voltage_min_design
keyvals['wh_energy_full_design'] = \
self._power_status.battery[0].energy_full_design
keyvals['v_voltage_now'] = self._power_status.battery[0].voltage_now
keyvals.update(self._tmp_keyvals)
keyvals['percent_sys_low_battery'] = self._sys_low_batt_p
keyvals['seconds_sys_low_battery'] = self._sys_low_batt_s
voltage_np = numpy.array(self._stats['v_voltage_now'])
voltage_mean = voltage_np.mean()
keyvals['v_voltage_mean'] = voltage_mean
keyvals['wh_energy_powerlogger'] = \
self._energy_use_from_powerlogger(keyvals)
if not self._power_status.on_ac() and keyvals['ah_charge_used'] > 0:
# For full runs, we should use charge to scale for battery life,
# since the voltage swing is accounted for.
# For short runs, energy will be a better estimate.
if self._loop_count > 1:
estimated_reps = (keyvals['ah_charge_full_design'] /
keyvals['ah_charge_used'])
else:
estimated_reps = (keyvals['wh_energy_full_design'] /
keyvals['wh_energy_powerlogger'])
bat_life_scale = estimated_reps * \
((100 - keyvals['percent_sys_low_battery']) / 100)
keyvals['minutes_battery_life'] = bat_life_scale * \
keyvals['minutes_battery_life_tested']
# In the case where sys_low_batt_s is non-zero subtract those
# minutes from the final extrapolation.
if self._sys_low_batt_s:
keyvals['minutes_battery_life'] -= self._sys_low_batt_s / 60
keyvals['a_current_rate'] = keyvals['ah_charge_used'] * 60 / \
keyvals['minutes_battery_life_tested']
keyvals['w_energy_rate'] = keyvals['wh_energy_used'] * 60 / \
keyvals['minutes_battery_life_tested']
if self._gaia_login:
self.output_perf_value(description='minutes_battery_life',
value=keyvals['minutes_battery_life'],
units='minutes',
higher_is_better=True)
minutes_battery_life_tested = keyvals['minutes_battery_life_tested']
# TODO(coconutruben): overwrite write_perf_keyvals for all power
# tests and replace this once power_LoadTest inherits from power_Test.
# Dump all keyvals into debug keyvals.
_utils.write_keyval(os.path.join(self.resultsdir, 'debug_keyval'),
keyvals)
# Avoid polluting the keyvals with non-core domains.
core_keyvals = power_utils.get_core_keyvals(keyvals)
if not self._gaia_login:
core_keyvals = {'INVALID_%s' % str(k): v for k, v in
core_keyvals.iteritems()}
else:
for key, value in core_keyvals.iteritems():
if key.startswith('percent_cpuidle') and \
key.endswith('C0_time'):
self.output_perf_value(description=key,
value=value,
units='percent',
higher_is_better=False)
self.write_perf_keyval(core_keyvals)
for log in self._meas_logs:
log.save_results(self.resultsdir)
self._checkpoint_logger.save_checkpoint_data(self.resultsdir)
if minutes_battery_life_tested * 60 < self._loop_time :
logging.info('Data is less than 1 loop, skip sending to dashboard.')
return
pdash = power_dashboard.PowerLoggerDashboard( \
self._plog, self.tagged_testname, self.resultsdir,
note=self._pdash_note)
pdash.upload()
cdash = power_dashboard.CPUStatsLoggerDashboard( \
self._clog, self.tagged_testname, self.resultsdir,
note=self._pdash_note)
cdash.upload()
tdash = power_dashboard.TempLoggerDashboard( \
self._tlog, self.tagged_testname, self.resultsdir,
note=self._pdash_note)
tdash.upload()
def _perform_reimage_and_run(spec, afe, tko, predicate, suite_job_id=None):
"""
Do the work of reimaging hosts and running tests.
@param spec: a populated SuiteSpec object.
@param afe: an instance of AFE as defined in server/frontend.py.
@param tko: an instance of TKO as defined in server/frontend.py.
@param predicate: A function mapping ControlData objects to True if they
should be included in the suite.
@param suite_job_id: Job id that will act as parent id to all sub jobs.
Default: None
"""
# We can't do anything else until the devserver has finished downloading
# control_files and test_suites packages so that we can get the control
# files we should schedule.
try:
if not spec.run_prod_code:
spec.devserver.stage_artifacts(spec.test_source_build,
['control_files', 'test_suites'])
except dev_server.DevServerException as e:
# If we can't get the control files, there's nothing to run.
raise error.AsynchronousBuildFailure(e)
timestamp = datetime.datetime.now().strftime(time_utils.TIME_FMT)
utils.write_keyval(spec.job.resultdir,
{constants.ARTIFACT_FINISHED_TIME: timestamp})
suite = Suite.create_from_predicates(
predicates=[predicate],
name=spec.name,
builds=spec.builds,
board=spec.board,
devserver=spec.devserver,
afe=afe,
tko=tko,
pool=spec.pool,
results_dir=spec.job.resultdir,
max_runtime_mins=spec.max_runtime_mins,
timeout_mins=spec.timeout_mins,
file_bugs=spec.file_bugs,
file_experimental_bugs=spec.file_experimental_bugs,
suite_job_id=suite_job_id,
extra_deps=spec.suite_dependencies,
priority=spec.priority,
wait_for_results=spec.wait_for_results,
job_retry=spec.job_retry,
max_retries=spec.max_retries,
offload_failures_only=spec.offload_failures_only,
test_source_build=spec.test_source_build,
run_prod_code=spec.run_prod_code)
# Now we get to asychronously schedule tests.
suite.schedule(spec.job.record_entry, spec.add_experimental)
if suite.wait_for_results:
logging.debug('Waiting on suite.')
suite.wait(spec.job.record_entry, spec.bug_template)
logging.debug('Finished waiting on suite. '
'Returning from _perform_reimage_and_run.')
else:
logging.info('wait_for_results is set to False, suite job will exit '
'without waiting for test jobs to finish.')
# If the problem was something other than the
# directory already existing, this chmod should throw as well
# exception.
os.chmod(self.tmpdir, stat.S_IRWXU)
job_data = {'label' : label, 'user' : user,
'hostname' : ','.join(machines),
'status_version' : str(self.STATUS_VERSION),
'job_started' : str(int(time.time()))}
if group_name:
job_data['host_group_name'] = group_name
if self.resultdir:
# only write these keyvals out on the first job in a resultdir
if 'job_started' not in utils.read_keyval(self.resultdir):
job_data.update(get_site_job_data(self))
utils.write_keyval(self.resultdir, job_data)
self.parse_job = parse_job
if self.parse_job and len(machines) == 1:
self.using_parser = True
self.init_parser(resultdir)
else:
self.using_parser = False
self.pkgmgr = packages.PackageManager(self.autodir,
run_function_dargs={'timeout':600})
self.pkgdir = os.path.join(self.autodir, 'packages')
self.num_tests_run = 0
self.num_tests_failed = 0
self._register_subcommand_hooks()
def __init__(
self,
control,
args,
resultdir,
label,
user,
machines,
client=False,
parse_job="",
ssh_user="******",
ssh_port=22,
ssh_pass="",
group_name="",
tag="",
control_filename=SERVER_CONTROL_FILENAME,
):
"""
Create a server side job object.
@param control: The pathname of the control file.
@param args: Passed to the control file.
@param resultdir: Where to throw the results.
@param label: Description of the job.
@param user: Username for the job (email address).
@param client: True if this is a client-side control file.
@param parse_job: string, if supplied it is the job execution tag that
the results will be passed through to the TKO parser with.
@param ssh_user: The SSH username. [root]
@param ssh_port: The SSH port number. [22]
@param ssh_pass: The SSH passphrase, if needed.
@param group_name: If supplied, this will be written out as
host_group_name in the keyvals file for the parser.
@param tag: The job execution tag from the scheduler. [optional]
@param control_filename: The filename where the server control file
should be written in the results directory.
"""
super(base_server_job, self).__init__(resultdir=resultdir)
path = os.path.dirname(__file__)
self.control = control
self._uncollected_log_file = os.path.join(self.resultdir, "uncollected_logs")
debugdir = os.path.join(self.resultdir, "debug")
if not os.path.exists(debugdir):
os.mkdir(debugdir)
if user:
self.user = user
else:
self.user = getpass.getuser()
self.args = args
self.machines = machines
self._client = client
self.warning_loggers = set()
self.warning_manager = warning_manager()
self._ssh_user = ssh_user
self._ssh_port = ssh_port
self._ssh_pass = ssh_pass
self.tag = tag
self.last_boot_tag = None
self.hosts = set()
self.drop_caches = False
self.drop_caches_between_iterations = False
self._control_filename = control_filename
self.logging = logging_manager.get_logging_manager(manage_stdout_and_stderr=True, redirect_fds=True)
subcommand.logging_manager_object = self.logging
self.sysinfo = sysinfo.sysinfo(self.resultdir)
self.profilers = profilers.profilers(self)
job_data = {
"label": label,
"user": user,
"hostname": ",".join(machines),
"drone": platform.node(),
"status_version": str(self._STATUS_VERSION),
"job_started": str(int(time.time())),
}
if group_name:
job_data["host_group_name"] = group_name
# only write these keyvals out on the first job in a resultdir
if "job_started" not in utils.read_keyval(self.resultdir):
job_data.update(get_site_job_data(self))
utils.write_keyval(self.resultdir, job_data)
self._parse_job = parse_job
self._using_parser = self._parse_job and len(machines) <= 1
self.pkgmgr = packages.PackageManager(self.autodir, run_function_dargs={"timeout": 600})
self.num_tests_run = 0
self.num_tests_failed = 0
self._register_subcommand_hooks()
# these components aren't usable on the server
self.bootloader = None
self.harness = None
# set up the status logger
self._indenter = status_indenter()
self._logger = base_job.status_logger(
self, self._indenter, "status.log", "status.log", record_hook=server_job_record_hook(self)
)
