def Poll(self):
"""Update the Execution status, with error handling.
Pinpoint has two kinds of errors: Job-level errors, which are fatal and
cause the Job to fail; and Execution-level errors, which only cause that
one execution to fail. If something might pass at one commit and fail at
another, it should be an Execution-level error. The type of Exception
indicates the error level. StandardErrors are Job-level and all other
errors are Execution-level.
"""
assert not self.completed
try:
self._Poll()
except client.AccessTokenRefreshError:
raise errors.RecoverableError()
except StandardError:
# StandardError most likely indicates a bug in the code.
# We should fail fast to aid debugging.
raise
except Exception: # pylint: disable=broad-except
# We allow broad exception handling here, because we log the exception and
# display it in the UI.
self._completed = True
self._exception = traceback.format_exc()
def Poll(self):
"""Update the Execution status, with error handling.
Pinpoint has two kinds of errors: Job-level errors, which are fatal and
cause the Job to fail; and Execution-level errors, which only cause that
one execution to fail. If something might pass at one commit and fail at
another, it should be an Execution-level error. The type of Exception
indicates the error level. StandardErrors are Job-level and all other
errors are Execution-level.
"""
assert not self.completed
try:
self._Poll()
except client.AccessTokenRefreshError:
raise errors.RecoverableError()
except (errors.FatalError, RuntimeError):
# Some built-in exceptions are derived from RuntimeError which we'd like
# to treat as errors.
raise
except Exception: # pylint: disable=broad-except
# We allow broad exception handling here, because we log the exception and
# display it in the UI.
self._completed = True
self._exception = traceback.format_exc()
except:
# All other exceptions must be propagated.
raise
def Explore(self):
"""Compare Changes and bisect by adding additional Changes as needed.
For every pair of adjacent Changes, compare their results as probability
distributions. If the results are different, find surrounding Changes and
add it to the Job. If the results are the same, do nothing. If the results
are inconclusive, add more Attempts to the Change with fewer Attempts until
we decide they are the same or different.
Intermediate points can only be added if the end Change represents a
commit that comes after the start Change. Otherwise, this method won't
explore further. For example, if Change A is repo@abc, and Change B is
repo@abc + patch, there's no way to pick additional Changes to try.
"""
if not self._changes:
return
def DetectChange(change_a, change_b):
comparison = self._Compare(change_a, change_b)
# We return None if the comparison determines that the result is
# inconclusive.
if comparison == compare.UNKNOWN:
return None
return comparison == compare.DIFFERENT
changes_to_refine = []
def CollectChangesToRefine(change_a, change_b):
changes_to_refine.append(
change_a if len(self._attempts[change_a]) <= len(
self._attempts[change_b]) else change_b)
def FindMidpoint(change_a, change_b):
try:
return change_module.Change.Midpoint(change_a, change_b)
except change_module.NonLinearError:
return None
try:
additional_changes = exploration.Speculate(
self._changes,
change_detected=DetectChange,
on_unknown=CollectChangesToRefine,
midpoint=FindMidpoint,
levels=_DEFAULT_SPECULATION_LEVELS)
logging.debug('Refinement list: %s', changes_to_refine)
for change in changes_to_refine:
self.AddAttempts(change)
logging.debug('Edit list: %s', additional_changes)
for index, change in additional_changes:
self.AddChange(change, index)
except (httplib.HTTPException,
urlfetch_errors.DeadlineExceededError) as e:
logging.debug('Encountered error: %s', e)
raise errors.RecoverableError(e)
def _Schedule(self, countdown=_TASK_INTERVAL):
# Set a task name to deduplicate retries. This adds some latency, but we're
# not latency-sensitive. If Job.Run() works asynchronously in the future,
# we don't need to worry about duplicate tasks.
# https://github.com/catapult-project/catapult/issues/3900
task_name = str(uuid.uuid4())
try:
task = taskqueue.add(
queue_name='job-queue', url='/api/run/' + self.job_id,
name=task_name, countdown=countdown)
except (apiproxy_errors.DeadlineExceededError, taskqueue.TransientError):
raise errors.RecoverableError()
self.task = task.name
def Explore(self):
"""Compare Changes and bisect by adding additional Changes as needed.
For every pair of adjacent Changes, compare their results as probability
distributions. If the results are different, find the midpoint of the
Changes and add it to the Job. If the results are the same, do nothing.
If the results are inconclusive, add more Attempts to the Change with fewer
Attempts until we decide they are the same or different.
The midpoint can only be added if the second Change represents a commit that
comes after the first Change. Otherwise, this method won't explore further.
For example, if Change A is repo@abc, and Change B is repo@abc + patch,
there's no way to pick additional Changes to try.
"""
# This loop adds Changes to the _changes list while looping through it.
# The Change insertion simultaneously uses and modifies the list indices.
# However, the loop index goes in reverse order and Changes are only added
# after the loop index, so the loop never encounters the modified items.
# TODO: consider using `reduce(...)` here to implement a fold?
for index in range(len(self._changes) - 1, 0, -1):
change_a = self._changes[index - 1]
change_b = self._changes[index]
comparison = self._Compare(change_a, change_b)
if comparison == compare.DIFFERENT:
try:
midpoint = change_module.Change.Midpoint(
change_a, change_b)
except change_module.NonLinearError:
continue
except (httplib.HTTPException,
urlfetch_errors.DeadlineExceededError):
raise errors.RecoverableError()
logging.info('Adding Change %s.', midpoint)
self.AddChange(midpoint, index)
elif comparison == compare.UNKNOWN:
if len(self._attempts[change_a]) <= len(
self._attempts[change_b]):
self.AddAttempts(change_a)
else:
self.AddAttempts(change_b)
def testStarted_RecoverableError_BacksOff(self):
j = job.Job.New((), (), comparison_mode='performance')
j.Start()
j.state.Explore = mock.MagicMock(side_effect=errors.RecoverableError(None))
j._Schedule = mock.MagicMock()
j.put = mock.MagicMock()
j.Fail = mock.MagicMock()
j.Run()
j.Run()
j.Run()
self.assertEqual(j._Schedule.call_args_list[0],
mock.call(countdown=job._TASK_INTERVAL * 2))
self.assertEqual(j._Schedule.call_args_list[1],
mock.call(countdown=job._TASK_INTERVAL * 4))
self.assertEqual(j._Schedule.call_args_list[2],
mock.call(countdown=job._TASK_INTERVAL * 8))
self.assertFalse(j.Fail.called)
j.Run()
self.assertTrue(j.Fail.called)
def _Poll(self):
try:
self._PollSwarming()
except client.AccessTokenRefreshError:
raise errors.RecoverableError()
def _Poll(self): raise errors.RecoverableError()
