def __init__(self):
self.reporter = reporting.Reporter()
self.phapi_client = self.reporter.get_bug_tracker_client()
self.issues = None
self.all_autofiled_query = 'ANCHOR TestFailure'
self.all_autofiled_label = 'autofiled'
self.prompted = False
def check_bug_filed_and_deduped(old_issue_ids):
"""Confirm bug related to dummy_Fail was filed and deduped.
@param old_issue_ids: A list of issue ids that was closed earlier. id of the
new issue must be not in this list.
@raise TestPushException: If auto bug file failed to create a new issue or
dedupe multiple failures.
"""
reporter = reporting.Reporter()
issue = reporter.find_issue_by_marker(BUG_ANCHOR)
if not issue:
raise TestPushException('Auto bug file failed. Unable to locate bug '
'with marker %s' % BUG_ANCHOR)
if old_issue_ids and issue.id in old_issue_ids:
raise TestPushException('Auto bug file failed to create a new issue. '
'id of the old issue found is %d.' % issue.id)
if not ('%s2' % reporter.AUTOFILED_COUNT) in issue.labels:
raise TestPushException(('Auto bug file failed to dedupe for issue %d '
'with labels of %s.') %
(issue.id, issue.labels))
# Close the bug, and do the search again, which should return None.
reporter.modify_bug_report(issue.id,
comment='Issue closed by test_push script.',
label_update='',
status='WontFix')
second_issue = reporter.find_issue_by_marker(BUG_ANCHOR)
if second_issue:
ids = '%d, %d' % (issue.id, second_issue.id)
raise TestPushException(('Auto bug file failed. Multiple issues (%s) '
'filed with marker %s') % (ids, BUG_ANCHOR))
print 'Issue %d was filed and deduped successfully.' % issue.id
def testDuplicateIssue(self):
"""Dedupe to an existing issue when one is found.
Confirms that we call AppendTrackerIssueById with the same issue
returned by the issue search.
"""
self.mox.StubOutWithMock(reporting.Reporter, 'find_issue_by_marker')
self.mox.StubOutWithMock(reporting.TestBug, 'summary')
issue = self.mox.CreateMock(phapi_lib.Issue)
issue.id = self._FAKE_ISSUE_ID
issue.labels = []
issue.state = constants.ISSUE_OPEN
client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
client.update_issue(self._FAKE_ISSUE_ID, mox.IgnoreArg())
reporting.Reporter.find_issue_by_marker(
mox.IgnoreArg()).AndReturn(issue)
reporting.TestBug.summary().AndReturn('')
self.mox.ReplayAll()
bug_id, bug_count = reporting.Reporter().report(self._get_failure())
self.assertEqual(bug_id, self._FAKE_ISSUE_ID)
self.assertEqual(bug_count, 2)
def testProjectLabelExtraction(self):
"""Test that the project label is correctly extracted from the title."""
TITLE_EMPTY = ''
TITLE_NO_PROJ = '[stress] platformDevice Failure on release/47-75.0.0'
TITLE_PROJ = '[stress] p_Device Failure on rikku-release/R44-7075.0.0'
TITLE_PROJ2 = '[stress] p_Device Failure on ' \
'rikku-freon-release/R44-7075.0.0'
TITLE_PROJ_SUBBOARD = '[stress] p_Device Failure on ' \
'veyron_rikku-release/R44-7075.0.0'
client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
self.mox.ReplayAll()
reporter = reporting.Reporter()
self.assertEqual(reporter._get_project_label_from_title(TITLE_EMPTY),
'')
self.assertEqual(reporter._get_project_label_from_title(TITLE_NO_PROJ),
'')
self.assertEqual(reporter._get_project_label_from_title(TITLE_PROJ),
'Proj-rikku')
self.assertEqual(reporter._get_project_label_from_title(TITLE_PROJ2),
'Proj-rikku')
self.assertEqual(
reporter._get_project_label_from_title(TITLE_PROJ_SUBBOARD),
'Proj-rikku')
def testReturnNoneIfMarkerIsNone(self):
"""Test that we do not look up an issue if the search marker is None."""
mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
self.mox.ReplayAll()
result = reporting.Reporter().find_issue_by_marker(None)
self.assertTrue(result is None)
def test(self):
"""Does a basic test of as much of the system as possible."""
afe_models.Test(name='test1', test_type=0, path='test1',
experimental=True).save()
afe_models.Test(name='test2', test_type=0, path='test2',
experimental=True).save()
tko_models.Status(status_idx=6, word='GOOD').save()
job = tko_models.Job(job_idx=1)
kernel = tko_models.Kernel(kernel_idx=1)
machine = tko_models.Machine(machine_idx=1)
success_status = tko_models.Status(status_idx=GOOD_STATUS_IDX)
fail_status = tko_models.Status(status_idx=FAIL_STATUS_IDX)
tko_test1 = tko_models.Test(job=job, status=success_status,
kernel=kernel, machine=machine,
test='test1',
started_time=self.datetime(2012, 1, 20))
tko_test1.save()
tko_test2 = tko_models.Test(job=job, status=success_status,
kernel=kernel, machine=machine,
test='test2',
started_time=self.datetime(2012, 1, 20))
tko_test2.save()
passing_experimental._MAX_DAYS_SINCE_LAST_PASS = 10
passing_experimental._MIN_DAYS_SINCE_FAILURE = 10
MockDatetime.today().AndReturn(self.datetime(2012, 1, 21))
MockDatetime.today().AndReturn(self.datetime(2012, 1, 21))
reporter1 = reporting.Reporter()
bug1 = reporting.Bug(
title=u'test1 should be promoted to non-experimental.',
summary=mox.IgnoreArg(),
search_marker=u'PassingExperimental(test1)')
reporter1.report(bug1).AndReturn((11, 1))
reporter2 = reporting.Reporter()
bug2 = reporting.Bug(
title=u'test2 should be promoted to non-experimental.',
summary=mox.IgnoreArg(),
search_marker=u'PassingExperimental(test2)')
reporter2.report(bug2).AndReturn((11, 1))
self.mox.ReplayAll()
passing_experimental.main()
def check_dut_availability(self, board, pool, minimum_duts=0):
"""Check if DUT availability for a given board and pool is less than
minimum.
@param board: The board to check DUT availability.
@param pool: The pool to check DUT availability.
@param minimum_duts: Minimum Number of available machines required to
run the suite. Default is set to 0, which means do
not force the check of available machines before
running the suite.
@raise: NotEnoughDutsError if DUT availability is lower than minimum.
@raise: BoardNotAvailableError if no host found for requested
board/pool.
"""
if minimum_duts == 0:
return
hosts = self.rpc_interface.get_hosts(
invalid=False,
multiple_labels=('pool:%s' % pool, 'board:%s' % board))
if not hosts:
raise BoardNotAvailableError(
'No hosts found for board:%s in pool:%s. The test lab '
'currently does not cover test for this board and pool.' %
(board, pool))
if len(hosts) < minimum_duts:
logging.debug(
'The total number of DUTs for %s in pool:%s is %d, '
'which is no more than the required minimum number of'
' available DUTS of %d. Minimum available DUT rule is'
' not enforced.', board, pool, len(hosts), minimum_duts)
return
# TODO(dshi): Replace the hard coded string with enum value,
# models.Host.Status.REPAIRING and REPAIR_FAILED
# setup_django_environment can't be imported now as paygen server does
# not have django package.
bad_statuses = ('Repair Failed', 'Repairing', 'Verifying')
unusable_hosts = []
available_hosts = []
for host in hosts:
if host.status in bad_statuses or host.locked:
unusable_hosts.append(host.hostname)
else:
available_hosts.append(host)
logging.debug('%d of %d DUTs are available for board %s pool %s.',
len(available_hosts), len(hosts), board, pool)
if len(available_hosts) < minimum_duts:
if unusable_hosts:
pool_health_bug = reporting.PoolHealthBug(
pool, board, unusable_hosts)
self.bug = reporting.Reporter().report(pool_health_bug)[0]
raise NotEnoughDutsError(
'Number of available DUTs for board %s pool %s is %d, which'
' is less than the minimum value %d. '
'Filed https://crbug.com/%s' %
(board, pool, len(available_hosts), minimum_duts, self.bug))
def test_summary_returned_untouched_if_no_search_maker(self):
"""Test that we just return the summary if we have no search marker."""
mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
bug = reporting.Bug('title', 'summary', None)
self.mox.ReplayAll()
result = reporting.Reporter()._anchor_summary(bug)
self.assertEqual(result, 'summary')
def __init__(self, autocommit=False):
"""Initialize update manager.
@param autocommit: If False just print out the update instead
of committing it.
"""
self.history = {}
self.present = {}
self.reporter = reporting.Reporter()
self.phapi_lib = self.reporter.get_bug_tracker_client()
self.autocommit = autocommit
def test_accepts_key_word_arguments(self):
"""
Test that the functions accepts the key_word arguments.
This basically tests that no exceptions are thrown.
"""
reporter = reporting.Reporter()
reporter.report(mox.IgnoreArg()).AndReturn((11, 1))
self.mox.ReplayAll()
reporting.submit_generic_bug_report('test', 'summary', labels=[])
def test_append_anchor_to_summary_if_search_marker(self):
"""Test that we add an anchor to the search marker."""
mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
bug = reporting.Bug('title', 'summary', 'marker')
self.mox.ReplayAll()
result = reporting.Reporter()._anchor_summary(bug)
self.assertEqual(
result,
'summary\n\n%smarker\n' % reporting.Reporter._SEARCH_MARKER)
def testWithSearchMarkerSetToNoneIsNotDeduped(self):
"""Test that we do not dedupe bugs that have no search marker."""
bug = reporting.Bug('title', 'summary', search_marker=None)
mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
mock_host.create_issue(mox.IgnoreArg()).AndReturn(
{'id': self._FAKE_ISSUE_ID})
self.mox.ReplayAll()
bug_id, bug_count = reporting.Reporter().report(bug)
self.assertEqual(bug_id, self._FAKE_ISSUE_ID)
self.assertEqual(bug_count, 1)
def close_bug():
"""Close all existing bugs filed for dummy_Fail.
@return: A list of issue ids to be used in check_bug_filed_and_deduped.
"""
old_issue_ids = []
reporter = reporting.Reporter()
while True:
issue = reporter.find_issue_by_marker(BUG_ANCHOR)
if not issue:
return old_issue_ids
if issue.id in old_issue_ids:
raise TestPushException('Failed to close issue %d' % issue.id)
old_issue_ids.append(issue.id)
reporter.modify_bug_report(issue.id,
comment='Issue closed by test_push script.',
label_update='',
status='WontFix')
def testGenericBugCanBeFiled(self):
"""Test that we can use a Bug object to file a bug report."""
self.mox.StubOutWithMock(reporting.Reporter, 'find_issue_by_marker')
bug = reporting.Bug('title', 'summary', 'marker')
reporting.Reporter.find_issue_by_marker(
mox.IgnoreArg()).AndReturn(None)
mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
mock_host.create_issue(mox.IgnoreArg()).AndReturn(
{'id': self._FAKE_ISSUE_ID})
self.mox.ReplayAll()
bug_id, bug_count = reporting.Reporter().report(bug)
self.assertEqual(bug_id, self._FAKE_ISSUE_ID)
self.assertEqual(bug_count, 1)
def _test_count_label_update(self, labels, remove, expected_count):
"""Utility to test _create_autofiled_count_update().
@param labels Input list of labels.
@param remove List of labels expected to be removed
in the result.
@param expected_count Count value expected to be returned
from the call.
"""
client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
self.mox.ReplayAll()
issue = self.mox.CreateMock(gdata_lib.Issue)
issue.labels = labels
reporter = reporting.Reporter()
new_labels, count = reporter._create_autofiled_count_update(issue)
expected = map(lambda l: '-' + l, remove)
expected.append(self._create_count_label(expected_count))
self.assertEqual(new_labels, expected)
self.assertEqual(count, expected_count)
def flag_problem_test(machine):
"""
Notify people about the last job that ran on a machine.
This method is invoked everytime a machine fails to repair, and attempts
to identify the last test that ran on the machine. If successfull, it files
a bug, or sends out an email, or just logs the fact.
@param machine: The hostname (e.g. IP address) of the machine to find the
last job ran on it.
"""
rpc = frontend.AFE()
logger = MachineDeathLogger()
try:
problem_test = _find_problem_test(machine, rpc)
except (urllib2.URLError, xmlrpclib.ProtocolError):
logger.logger.error('%s | ERROR: Could not contact RPC server'
% machine)
return
if problem_test:
job_id = problem_test['job']['id']
job_name = problem_test['job']['name']
bug = reporting.MachineKillerBug(job_id=job_id,
job_name=job_name,
machine=machine)
reporter = reporting.Reporter()
bug_id = reporter.report(bug)[0]
if bug_id is None:
try:
email_prefix = ('The following test is killing a machine, '
'could not file a bug to report this:\n\n')
mail.send(_SENDER_ADDRESS, _NOTIFY_ADDRESS, '',
bug.title(), email_prefix + bug.summary())
except smtplib.SMTPDataError:
logger.logger.error('%s | %d | %s'
% (machine, job_id, job_name))
def testNewIssue(self):
"""Add a new issue to the tracker when a matching issue isn't found.
Confirms that we call CreateTrackerIssue when an Issue search
returns None.
"""
self.mox.StubOutWithMock(reporting.Reporter, 'find_issue_by_marker')
self.mox.StubOutWithMock(reporting.TestBug, 'summary')
client = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg())
client.create_issue(mox.IgnoreArg()).AndReturn(
{'id': self._FAKE_ISSUE_ID})
reporting.Reporter.find_issue_by_marker(
mox.IgnoreArg()).AndReturn(None)
reporting.TestBug.summary().AndReturn('')
self.mox.ReplayAll()
bug_id, bug_count = reporting.Reporter().report(self._get_failure())
self.assertEqual(bug_id, self._FAKE_ISSUE_ID)
self.assertEqual(bug_count, 1)
def testSuiteIssueConfig(self):
"""Test that the suite bug template values are not overridden."""
def check_suite_options(issue):
"""
Checks to see if the options specified in bug_template reflect in
the issue we're about to file, and that the autofiled label was not
lost in the process.
@param issue: issue to check labels on.
"""
assert ('autofiled' in issue.labels)
for k, v in self.bug_template.iteritems():
if (isinstance(v, list)
and all(item in getattr(issue, k) for item in v)):
continue
if v and getattr(issue, k) is not v:
return False
return True
self.mox.StubOutWithMock(reporting.Reporter, '_find_issue_by_marker')
self.mox.StubOutWithMock(reporting.TestBug, 'summary')
reporting.Reporter._find_issue_by_marker(
mox.IgnoreArg()).AndReturn(None)
reporting.TestBug.summary().AndReturn('Summary')
mock_host = phapi_lib.ProjectHostingApiClient(mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg())
mock_host.create_issue(mox.IgnoreArg()).AndReturn(
{'id': self._FAKE_ISSUE_ID})
self.mox.ReplayAll()
bug_id, bug_count = reporting.Reporter().report(
self._get_failure(), self.bug_template)
self.assertEqual(bug_id, self._FAKE_ISSUE_ID)
self.assertEqual(bug_count, 1)
def _Schedule(self, suite, board, build, pool, num, priority, timeout,
file_bugs=False, firmware_rw_build=None,
firmware_ro_build=None, test_source_build=None,
job_retry=False, launch_control_build=None,
run_prod_code=False, testbed_dut_count=None, no_delay=False):
"""Schedule |suite|, if it hasn't already been run.
@param suite: the name of the suite to run, e.g. 'bvt'
@param board: the board to run the suite on, e.g. x86-alex
@param build: the ChromeOS build to install e.g.
x86-alex-release/R18-1655.0.0-a1-b1584.
@param pool: the pool of machines to use for scheduling purposes.
Default: None
@param num: the number of devices across which to shard the test suite.
Type: integer or None
Default: None (uses sharding factor in global_config.ini).
@param priority: One of the values from
client.common_lib.priorities.Priority.
@param timeout: The max lifetime of the suite in hours.
@param file_bugs: True if bug filing is desired for this suite.
@param firmware_rw_build: Firmware build to update RW firmware. Default
to None.
@param firmware_ro_build: Firmware build to update RO firmware. Default
to None.
@param test_source_build: Build that contains the server-side test code.
Default to None to use the ChromeOS build
(defined by `build`).
@param job_retry: Set to True to enable job-level retry. Default is
False.
@param launch_control_build: Name of a Launch Control build, e.g.,
'git_mnc_release/shamu-eng/123'
@param run_prod_code: If True, the suite will run the test code that
lives in prod aka the test code currently on the
lab servers. If False, the control files and test
code for this suite run will be retrieved from the
build artifacts. Default is False.
@param testbed_dut_count: Number of duts to test when using a testbed.
@param no_delay: Set to True to allow suite to be created without
configuring delay_minutes. Default is False.
@return True if the suite got scheduled
@raise ScheduleException if an error occurs while scheduling.
"""
try:
if build:
builds = {provision.CROS_VERSION_PREFIX: build}
if firmware_rw_build:
builds[provision.FW_RW_VERSION_PREFIX] = firmware_rw_build
if firmware_ro_build:
builds[provision.FW_RO_VERSION_PREFIX] = firmware_ro_build
if launch_control_build:
if testbed_dut_count is None:
builds = {provision.ANDROID_BUILD_VERSION_PREFIX:
launch_control_build}
else:
builds = {provision.TESTBED_BUILD_VERSION_PREFIX:
launch_control_build}
# Suite scheduler handles all boards in parallel, to guarantee each
# call of `create_suite_job` use different value of delay_minutes,
# we need a lock around get/set attempts of self.delay_minutes.
# To prevent suite jobs from running too long, the value for
# self.delay_minutes is limited between 0 and MAX_DELAY_MINUTES (4
# hours). The value starts at 0 and is increased by
# DELAY_MINUTES_INTERVAL, when it reaches MAX_DELAY_MINUTES, the
# logic here allows its value to step back by DELAY_MINUTES_INTERVAL
# at each call of this method. When the value drops back to 0, it
# will increase again in the next call of this method.
# Such logic allows the values of delay_minutes for all calls
# of `create_suite_job` running in parallel to be evenly distributed
# between 0 and MAX_DELAY_MINUTES.
delay_minutes = 0
if not no_delay:
with self._lock:
delay_minutes = self.delay_minutes
if ((self.delay_minutes < MAX_DELAY_MINUTES and
self.delay_minutes_interval > 0) or
(self.delay_minutes >= DELAY_MINUTES_INTERVAL and
self.delay_minutes_interval < 0)):
self.delay_minutes += self.delay_minutes_interval
else:
limit = ('Maximum' if self.delay_minutes_interval > 0
else 'Minimum')
logging.info(
'%s delay minutes reached when scheduling '
'%s on %s against %s (pool: %s)',
limit, suite, builds, board, pool)
self.delay_minutes_interval = (
-self.delay_minutes_interval)
# Update timeout settings for the suite job with delay_minutes.
# `timeout` is in hours.
if not timeout:
timeout = JOB_MAX_RUNTIME_MINS_DEFAULT / 60.0
timeout += delay_minutes / 60.0
max_runtime_mins = JOB_MAX_RUNTIME_MINS_DEFAULT + delay_minutes
timeout_mins = JOB_MAX_RUNTIME_MINS_DEFAULT + delay_minutes
logging.info('Scheduling %s on %s against %s (pool: %s)...',
suite, builds, board, pool)
job_id = self._afe.run('create_suite_job', name=suite, board=board,
builds=builds, check_hosts=False, num=num,
pool=pool, priority=priority,
timeout=timeout,
max_runtime_mins=max_runtime_mins,
timeout_mins=timeout_mins,
file_bugs=file_bugs,
wait_for_results=file_bugs,
test_source_build=test_source_build,
job_retry=job_retry,
delay_minutes=delay_minutes,
run_prod_code=run_prod_code,
min_rpc_timeout=_MIN_RPC_TIMEOUT)
if job_id is not None:
logging.info('... created as suite job id %s', job_id)
# Report data to metrics.
fields = {'suite': suite,
'board': board,
'pool': pool,
'priority': str(priority)}
self._SUITE_SCHEDULER_SUITE_COUNT.increment(fields=fields)
return True
else:
raise ScheduleException(
"Can't schedule %s for %s." % (suite, builds))
except (error.ControlFileNotFound, error.ControlFileEmpty,
error.ControlFileMalformed, error.NoControlFileList) as e:
if self._file_bug:
# File bug on test_source_build if it's specified.
b = reporting.SuiteSchedulerBug(
suite, test_source_build or build, board, e)
# If a bug has filed with the same <suite, build, error type>
# will not file again, but simply gets the existing bug id.
bid, _ = reporting.Reporter().report(
b, ignore_duplicate=True)
if bid is not None:
return False
# Raise the exception if not filing a bug or failed to file bug.
raise ScheduleException(e)
except Exception as e:
raise ScheduleException(e)
def _Schedule(self, suite, board, build, pool, num, priority, timeout,
file_bugs=False, firmware_rw_build=None,
test_source_build=None, job_retry=False):
"""Schedule |suite|, if it hasn't already been run.
@param suite: the name of the suite to run, e.g. 'bvt'
@param board: the board to run the suite on, e.g. x86-alex
@param build: the build to install e.g.
x86-alex-release/R18-1655.0.0-a1-b1584.
@param pool: the pool of machines to use for scheduling purposes.
Default: None
@param num: the number of devices across which to shard the test suite.
Type: integer or None
Default: None (uses sharding factor in global_config.ini).
@param priority: One of the values from
client.common_lib.priorities.Priority.
@param timeout: The max lifetime of the suite in hours.
@param file_bugs: True if bug filing is desired for this suite.
@param firmware_rw_build: Firmware build to update RW firmware. Default
to None.
@param test_source_build: Build that contains the server-side test code.
Default to None to use the ChromeOS build
(defined by `build`).
@param job_retry: Set to True to enable job-level retry. Default is
False.
@return True if the suite got scheduled
@raise ScheduleException if an error occurs while scheduling.
"""
try:
builds = {provision.CROS_VERSION_PREFIX: build}
if firmware_rw_build:
builds[provision.FW_RW_VERSION_PREFIX] = firmware_rw_build
logging.info('Scheduling %s on %s against %s (pool: %s)',
suite, builds, board, pool)
if self._afe.run(
'create_suite_job', name=suite, board=board,
builds=builds, check_hosts=False, num=num, pool=pool,
priority=priority, timeout=timeout, file_bugs=file_bugs,
wait_for_results=file_bugs,
test_source_build=test_source_build,
job_retry=job_retry) is not None:
return True
else:
raise ScheduleException(
"Can't schedule %s for %s." % (suite, builds))
except (error.ControlFileNotFound, error.ControlFileEmpty,
error.ControlFileMalformed, error.NoControlFileList) as e:
if self._file_bug:
# File bug on test_source_build if it's specified.
b = reporting.SuiteSchedulerBug(
suite, test_source_build or build, board, e)
# If a bug has filed with the same <suite, build, error type>
# will not file again, but simply gets the existing bug id.
bid, _ = reporting.Reporter().report(
b, ignore_duplicate=True)
if bid is not None:
return False
# Raise the exception if not filing a bug or failed to file bug.
raise ScheduleException(e)
except Exception as e:
raise ScheduleException(e)
