def testSpeculateEmpty(self):
results = exploration.Speculate(
[],
change_detected=lambda *_: False,
on_unknown=lambda *args: self.fail("on_unknown called with %r" %
(args, )),
midpoint=lambda *_: None,
levels=100)
self.assertEqual(results, [])
def testSpeculateHandleChangeNeverDetected(self):
results = exploration.Speculate(
[0, 1000],
change_detected=lambda *_: False,
on_unknown=lambda *args: self.fail("on_unknown called with %r" %
(args, )),
midpoint=FindMidpoint,
levels=2)
self.assertEqual(list(results), [])
def Explore(self):
"""Compare Changes and bisect by adding additional Changes as needed.
For every pair of adjacent Changes, compare their results as probability
distributions. If the results are different, find surrounding Changes and
add it to the Job. If the results are the same, do nothing. If the results
are inconclusive, add more Attempts to the Change with fewer Attempts until
we decide they are the same or different.
Intermediate points can only be added if the end Change represents a
commit that comes after the start Change. Otherwise, this method won't
explore further. For example, if Change A is repo@abc, and Change B is
repo@abc + patch, there's no way to pick additional Changes to try.
"""
if not self._changes:
return
def DetectChange(change_a, change_b):
comparison = self._Compare(change_a, change_b)
# We return None if the comparison determines that the result is
# inconclusive.
if comparison == compare.UNKNOWN:
return None
return comparison == compare.DIFFERENT
changes_to_refine = []
def CollectChangesToRefine(change_a, change_b):
changes_to_refine.append(
change_a if len(self._attempts[change_a]) <= len(
self._attempts[change_b]) else change_b)
def FindMidpoint(change_a, change_b):
try:
return change_module.Change.Midpoint(change_a, change_b)
except change_module.NonLinearError:
return None
try:
additional_changes = exploration.Speculate(
self._changes,
change_detected=DetectChange,
on_unknown=CollectChangesToRefine,
midpoint=FindMidpoint,
levels=_DEFAULT_SPECULATION_LEVELS)
logging.debug('Refinement list: %s', changes_to_refine)
for change in changes_to_refine:
self.AddAttempts(change)
logging.debug('Edit list: %s', additional_changes)
for index, change in additional_changes:
self.AddChange(change, index)
except (httplib.HTTPException,
urlfetch_errors.DeadlineExceededError) as e:
logging.debug('Encountered error: %s', e)
raise errors.RecoverableError(e)
def testSpeculateUnbalanced(self):
changes = [0, 9, 100]
results = exploration.Speculate(
changes,
change_detected=ChangeAlwaysDetected,
on_unknown=lambda *args: self.fail("on_unknown called with %r" %
(args, )),
midpoint=FindMidpoint,
levels=2)
for index, change in results:
changes.insert(index, change)
self.assertEqual(changes, [0, 2, 4, 6, 9, 31, 54, 77, 100])
def testSpeculateOdd(self):
changes = [1, 6]
results = exploration.Speculate(
changes,
change_detected=ChangeAlwaysDetected,
on_unknown=lambda *args: self.fail("on_unknown called with %r" %
(args, )),
midpoint=FindMidpoint,
levels=2)
for index, change in results:
changes.insert(index, change)
self.assertEqual(changes, [1, 2, 3, 4, 6])
def ExplorationEvaluator(task, event, accumulator):
logging.debug('Evaluating: %s, %s, %s', task, event, accumulator)
if task.task_type == 'revision':
accumulator[task.id] = task.payload
return
if task.task_type == 'bisection':
rev_positions = list(
sorted(
accumulator.get(dep).get('position')
for dep in task.dependencies))
results = list(rev_positions)
insertion_list = exploration.Speculate(
rev_positions,
# Assume we always find a difference between two positions.
lambda *_: True,
# Do nothing when we encounter an unknown error.
lambda _: None,
# Provide the function that will find the midpoint between two
# revisions.
FindMidpoint,
# Speculate two levels deep in the bisection space.
levels=2)
for index, change in insertion_list:
results.insert(index, change)
new_positions = set(results) - set(rev_positions)
if new_positions:
def GraphExtender(_):
logging.debug('New revisions: %s', new_positions)
task_module.ExtendTaskGraph(job, [
task_module.TaskVertex(id='rev_%s' % (rev, ),
vertex_type='revision',
payload={
'revision': '%s' %
(rev, ),
'position': rev
}) for rev in new_positions
], [
task_module.Dependency(from_='bisection',
to='rev_%s' % (rev, ))
for rev in new_positions
])
return [GraphExtender]
def testSpeculateIterations(self):
on_unknown_mock = mock.MagicMock()
changes = [0, 10]
results = exploration.Speculate(changes,
change_detected=ChangeAlwaysDetected,
on_unknown=on_unknown_mock,
midpoint=FindMidpoint,
levels=2)
for index, change in results:
changes.insert(index, change)
self.assertEqual(changes, [0, 2, 5, 7, 10])
# Run the bisection again and get the full range.
results = exploration.Speculate(
changes,
change_detected=ChangeAlwaysDetected,
on_unknown=lambda *args: self.fail("on_unknown called with %r" %
(args, )),
midpoint=FindMidpoint,
levels=2)
for index, change in results:
changes.insert(index, change)
self.assertEqual(changes, range(11))
def testSpeculateHandleUnknown(self):
on_unknown_mock = mock.MagicMock()
changes = [0, 5, 10]
def ChangeUnknownDetected(a, _):
if a >= 5:
return None
return True
results = exploration.Speculate(changes,
change_detected=ChangeUnknownDetected,
on_unknown=on_unknown_mock,
midpoint=FindMidpoint,
levels=2)
for index, change in results:
changes.insert(index, change)
self.assertTrue(on_unknown_mock.called)
self.assertEqual(changes, [0, 1, 2, 3, 5, 10])
def __call__(self, task, _, accumulator):
# Outline:
# - If the task is still pending, this means this is the first time we're
# encountering the task in an evaluation. Set up the payload data to
# include the full range of commits, so that we load it once and have it
# ready, and emit an action to mark the task ongoing.
#
# - If the task is ongoing, gather all the dependency data (both results
# and status) and see whether we have enough data to determine the next
# action. We have three main cases:
#
# 1. We cannot detect a significant difference between the results from
# two different CLs. We call this the NoReproduction case.
#
# 2. We do not have enough confidence that there's a difference. We call
# this the Indeterminate case.
#
# 3. We have enough confidence that there's a difference between any two
# ordered changes. We call this the SignificantChange case.
#
# - Delegate the implementation to handle the independent cases for each
# change point we find in the CL continuum.
if task.status == 'pending':
return [PrepareCommits(self.job, task)]
all_changes = None
actions = []
if 'changes' not in task.payload:
all_changes = [
change_module.Change(
commits=[
change_module.Commit(
repository=commit.get('repository'),
git_hash=commit.get('git_hash'))
],
patch=task.payload.get('pinned_change'))
for commit in task.payload.get('commits', [])
]
task.payload.update({
'changes': [change.AsDict() for change in all_changes],
})
actions.append(UpdateTaskPayloadAction(self.job, task))
else:
# We need to reconstitute the Change instances from the dicts we've stored
# in the payload.
all_changes = [
change_module.ReconstituteChange(change)
for change in task.payload.get('changes')
]
if task.status == 'ongoing':
# TODO(dberris): Validate and fail gracefully instead of asserting?
assert 'commits' in task.payload, ('Programming error, need commits to '
'proceed!')
# Collect all the dependency task data and analyse the results.
# Group them by change.
# Order them by appearance in the CL range.
# Also count the status per CL (failed, ongoing, etc.)
deps = set(task.dependencies)
results_by_change = collections.defaultdict(list)
status_by_change = collections.defaultdict(dict)
changes_with_data = set()
changes_by_status = collections.defaultdict(set)
associated_results = [(change_module.ReconstituteChange(t.get('change')),
t.get('status'), t.get('result_values'))
for dep, t in accumulator.items()
if dep in deps]
for change, status, result_values in associated_results:
if result_values:
filtered_results = [r for r in result_values if r is not None]
if filtered_results:
results_by_change[change].append(filtered_results)
status_by_change[change].update({
status: status_by_change[change].get(status, 0) + 1,
})
changes_by_status[status].add(change)
changes_with_data.add(change)
# If the dependencies have converged into a single status, we can make
# decisions on the terminal state of the bisection.
if len(changes_by_status) == 1 and changes_with_data:
# Check whether all dependencies are completed and if we do
# not have data in any of the dependencies.
if changes_by_status.get('completed') == changes_with_data:
changes_with_empty_results = [
change for change in changes_with_data
if not results_by_change.get(change)
]
if changes_with_empty_results:
task.payload.update({
'errors':
task.payload.get('errors', []) + [{
'reason':
'BisectionFailed',
'message': ('We did not find any results from '
'successful test runs.')
}]
})
return [CompleteExplorationAction(self.job, task, 'failed')]
# Check whether all the dependencies had the tests fail consistently.
elif changes_by_status.get('failed') == changes_with_data:
task.payload.update({
'errors':
task.payload.get('errors', []) + [{
'reason': 'BisectionFailed',
'message': 'All attempts in all dependencies failed.'
}]
})
return [CompleteExplorationAction(self.job, task, 'failed')]
# If they're all pending or ongoing, then we don't do anything yet.
else:
return actions
# We want to reduce the list of ordered changes to only the ones that have
# data available.
change_index = {change: index for index, change in enumerate(all_changes)}
ordered_changes = [c for c in all_changes if c in changes_with_data]
# From here we can then do the analysis on a pairwise basis, as we're
# going through the list of Change instances we have data for.
# NOTE: A lot of this algorithm is already in pinpoint/models/job_state.py
# which we're adapting.
def Compare(a, b):
# This is the comparison function which determines whether the samples
# we have from the two changes (a and b) are statistically significant.
if a is None or b is None:
return None
if 'pending' in status_by_change[a] or 'pending' in status_by_change[b]:
return compare.PENDING
# NOTE: Here we're attempting to scale the provided comparison magnitude
# threshold by the larger inter-quartile range (a measure of dispersion,
# simply computed as the 75th percentile minus the 25th percentile). The
# reason we're doing this is so that we can scale the tolerance
# according to the noise inherent in the measurements -- i.e. more noisy
# measurements will require a larger difference for us to consider
# statistically significant.
values_for_a = tuple(itertools.chain(*results_by_change[a]))
values_for_b = tuple(itertools.chain(*results_by_change[b]))
if not values_for_a:
return None
if not values_for_b:
return None
max_iqr = max(
math_utils.Iqr(values_for_a), math_utils.Iqr(values_for_b), 0.001)
comparison_magnitude = task.payload.get('comparison_magnitude',
1.0) / max_iqr
attempts = (len(values_for_a) + len(values_for_b)) // 2
result = compare.Compare(values_for_a, values_for_b, attempts,
'performance', comparison_magnitude)
return result.result
def DetectChange(change_a, change_b):
# We return None if the comparison determines that the result is
# inconclusive. This is required by the exploration.Speculate contract.
comparison = Compare(change_a, change_b)
if comparison == compare.UNKNOWN:
return None
return comparison == compare.DIFFERENT
changes_to_refine = []
def CollectChangesToRefine(a, b):
# Here we're collecting changes that need refinement, which happens when
# two changes when compared yield the "unknown" result.
attempts_for_a = sum(status_by_change[a].values())
attempts_for_b = sum(status_by_change[b].values())
# Grow the attempts of both changes by 50% every time when increasing
# attempt counts. This number is arbitrary, and we should probably use
# something like a Fibonacci sequence when scaling attempt counts.
new_attempts_size_a = min(
attempts_for_a + (attempts_for_a // 2),
task.payload.get('analysis_options', {}).get('max_attempts', 100))
new_attempts_size_b = min(
attempts_for_b + (attempts_for_b // 2),
task.payload.get('analysis_options', {}).get('max_attempts', 100))
# Only refine if the new attempt sizes are not large enough.
if new_attempts_size_a > attempts_for_a:
changes_to_refine.append((a, new_attempts_size_a))
if new_attempts_size_b > attempts_for_b:
changes_to_refine.append((b, new_attempts_size_b))
def FindMidpoint(a, b):
# Here we use the (very simple) midpoint finding algorithm given that we
# already have the full range of commits to bisect through.
a_index = change_index[a]
b_index = change_index[b]
subrange = all_changes[a_index:b_index + 1]
return None if len(subrange) <= 2 else subrange[len(subrange) // 2]
# We have a striding iterable, which will give us the before, current, and
# after for a given index in the iterable.
def SlidingTriple(iterable):
"""s -> (None, s0, s1), (s0, s1, s2), (s1, s2, s3), ..."""
p, c, n = itertools.tee(iterable, 3)
p = itertools.chain([None], p)
n = itertools.chain(itertools.islice(n, 1, None), [None])
return itertools.izip(p, c, n)
# This is a comparison between values at a change and the values at
# the previous change and the next change.
comparisons = [{
'prev': Compare(p, c),
'next': Compare(c, n),
} for (p, c, n) in SlidingTriple(ordered_changes)]
# Collect the result values for each change with values.
result_values = [
list(itertools.chain(*results_by_change.get(change, [])))
for change in ordered_changes
]
if task.payload.get('comparisons') != comparisons or task.payload.get(
'result_values') != result_values:
task.payload.update({
'comparisons': comparisons,
'result_values': result_values,
})
actions.append(UpdateTaskPayloadAction(self.job, task))
if len(ordered_changes) < 2:
# We do not have enough data yet to determine whether we should do
# anything.
return actions
additional_changes = exploration.Speculate(
ordered_changes,
change_detected=DetectChange,
on_unknown=CollectChangesToRefine,
midpoint=FindMidpoint,
levels=_DEFAULT_SPECULATION_LEVELS)
# At this point we can collect the actions to extend the task graph based
# on the results of the speculation, only if the changes don't have any
# more associated pending/ongoing work.
min_attempts = task.payload.get('analysis_options',
{}).get('min_attempts', 10)
actions += [
RefineExplorationAction(self.job, task, change, new_size)
for change, new_size in itertools.chain(
[(c, min_attempts) for _, c in additional_changes],
[(c, a) for c, a in changes_to_refine],
)
if not bool({'pending', 'ongoing'} & set(status_by_change[change]))
]
# Here we collect the points where we've found the changes.
def Pairwise(iterable):
"""s -> (s0, s1), (s1, s2), (s2, s3), ..."""
a, b = itertools.tee(iterable)
next(b, None)
return itertools.izip(a, b)
task.payload.update({
'culprits': [(a.AsDict(), b.AsDict())
for a, b in Pairwise(ordered_changes)
if DetectChange(a, b)],
})
can_complete = not bool(set(changes_by_status) - {'failed', 'completed'})
if not actions and can_complete:
# Mark this operation complete, storing the differences we can compute.
actions = [CompleteExplorationAction(self.job, task, 'completed')]
return actions
