def testOnTimeout(self, mock_fn):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
isolate_sha = 'sha1'
iterations = 50
timeout_seconds = 1200
task_id = 'task_id'
run_flake_swarming_task_input = RunFlakeSwarmingTaskInput(
builder_name=builder_name,
commit_position=commit_position,
isolate_sha=isolate_sha,
iterations=iterations,
master_name=master_name,
reference_build_number=build_number,
step_name=step_name,
test_name=test_name,
timeout_seconds=timeout_seconds)
pipeline_job = RunFlakeSwarmingTaskPipeline(run_flake_swarming_task_input)
pipeline_job.OnTimeout(run_flake_swarming_task_input, {'task_id': task_id})
mock_fn.assert_called_once_with(run_flake_swarming_task_input, task_id)
def testRunImplNotTriggerSameTaskTwice(self, mock_trigger, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
isolate_sha = 'sha1'
iterations = 50
timeout_seconds = 1200
run_flake_swarming_task_input = RunFlakeSwarmingTaskInput(
builder_name=builder_name,
commit_position=commit_position,
isolate_sha=isolate_sha,
iterations=iterations,
master_name=master_name,
reference_build_number=build_number,
step_name=step_name,
test_name=test_name,
timeout_seconds=timeout_seconds)
pipeline_job = RunFlakeSwarmingTaskPipeline(run_flake_swarming_task_input)
pipeline_job.RunImpl(run_flake_swarming_task_input)
self.assertFalse(mock_trigger.called)
def testCallbackImplFailedRun(self, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
isolate_sha = 'sha1'
iterations = 50
timeout_seconds = 1200
run_flake_swarming_task_input = RunFlakeSwarmingTaskInput(
builder_name=builder_name,
commit_position=commit_position,
isolate_sha=isolate_sha,
iterations=iterations,
master_name=master_name,
reference_build_number=build_number,
step_name=step_name,
test_name=test_name,
timeout_seconds=timeout_seconds)
pipeline_job = RunFlakeSwarmingTaskPipeline(run_flake_swarming_task_input)
result = pipeline_job.CallbackImpl(run_flake_swarming_task_input,
{'task_id': 'task_id'})
self.assertEqual(('Error getting swarming task result: m', None), result)
def testCallbackImplNoTaskID(self, mocked_pipeline_id, mock_fn):
mocked_pipeline_id.__get__ = mock.Mock(return_value='pipeline-id')
pipeline_input = RunFlakeSwarmingTaskInput.FromSerializable({})
p = RunFlakeSwarmingTaskPipeline(pipeline_input)
result = p.CallbackImpl(pipeline_input, {})
self.assertEqual(('Task_id not found for pipeline pipeline-id', None),
result)
self.assertFalse(mock_fn.called)
def testRunImplTriggerTaskFailed(self, mocked_pipeline_id, mock_trigger, _):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
isolate_sha = 'sha1'
iterations = 50
timeout_seconds = 1200
pipeline_id = 'pipeline-id'
run_flake_swarming_task_input = RunFlakeSwarmingTaskInput(
builder_name=builder_name,
commit_position=commit_position,
isolate_sha=isolate_sha,
iterations=iterations,
master_name=master_name,
reference_build_number=build_number,
step_name=step_name,
test_name=test_name,
timeout_seconds=timeout_seconds)
mock_trigger.return_value = None
mocked_pipeline_id.__get__ = mock.Mock(return_value=pipeline_id)
pipeline_job = RunFlakeSwarmingTaskPipeline(run_flake_swarming_task_input)
with self.assertRaises(pipeline.Retry):
pipeline_job.RunImpl(run_flake_swarming_task_input)
mock_trigger.assert_called_once_with(
master_name=master_name,
builder_name=builder_name,
reference_build_number=build_number,
step_name=step_name,
test_name=test_name,
isolate_sha=isolate_sha,
iterations=iterations,
timeout_seconds=timeout_seconds,
runner_id=pipeline_id)
def testTimeoutSeconds(self):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
isolate_sha = 'sha1'
iterations = 50
timeout_seconds = 1200
run_flake_swarming_task_input = RunFlakeSwarmingTaskInput(
builder_name=builder_name,
commit_position=commit_position,
isolate_sha=isolate_sha,
iterations=iterations,
master_name=master_name,
reference_build_number=build_number,
step_name=step_name,
test_name=test_name,
timeout_seconds=timeout_seconds)
pipeline_job = RunFlakeSwarmingTaskPipeline(run_flake_swarming_task_input)
self.assertEqual(24 * 60 * 60, pipeline_job.TimeoutSeconds())
def testCallbackImplCompleted(self, mocked_output):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
isolate_sha = 'sha1'
iterations = 50
timeout_seconds = 1200
run_flake_swarming_task_input = RunFlakeSwarmingTaskInput(
builder_name=builder_name,
commit_position=commit_position,
isolate_sha=isolate_sha,
iterations=iterations,
master_name=master_name,
reference_build_number=build_number,
step_name=step_name,
test_name=test_name,
timeout_seconds=timeout_seconds)
flake_swarming_task_output = FlakeSwarmingTaskOutput(
completed_time=datetime(2018, 4, 1, 0, 1, 0),
error=None,
iterations=iterations,
pass_count=iterations,
started_time=datetime(2018, 4, 1, 0, 0, 0),
task_id='task_id')
mocked_output.return_value = flake_swarming_task_output
pipeline_job = RunFlakeSwarmingTaskPipeline(run_flake_swarming_task_input)
result = pipeline_job.CallbackImpl(run_flake_swarming_task_input,
{'task_id': 'task_id'})
self.assertEqual((None, flake_swarming_task_output), result)
def RunImpl(self, parameters):
"""Pipeline to find the true pass rate of a test at a commit position."""
master_name = parameters.master_name
builder_name = parameters.builder_name
reference_build_number = parameters.reference_build_number
step_name = parameters.step_name
test_name = parameters.test_name
commit_position = parameters.commit_position
get_isolate_sha_output = parameters.get_isolate_sha_output
build_url = get_isolate_sha_output.build_url
try_job_url = get_isolate_sha_output.try_job_url
flakiness_thus_far = parameters.flakiness_thus_far
previous_swarming_task_output = parameters.previous_swarming_task_output
# Extract pass rate and iterations already-completed up to this point.
if previous_swarming_task_output:
assert flakiness_thus_far, (
'Previous swarming task output not captured properly')
error = previous_swarming_task_output.error
pass_rate_at_commit_position = flakiness_thus_far.pass_rate
previous_pass_count = previous_swarming_task_output.pass_count
previous_iterations = previous_swarming_task_output.iterations
previous_pass_rate = (float(previous_pass_count /
previous_iterations)
if previous_iterations else None)
else:
error = None
pass_rate_at_commit_position = None
previous_iterations = 0
previous_pass_count = 0
previous_pass_rate = None
# Create a fresh Flakiness instance to aggregate swarming rerun data.
flakiness_thus_far = Flakiness(
build_number=get_isolate_sha_output.build_number,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=0,
error=None,
failed_swarming_task_attempts=0,
iterations=0,
pass_rate=None,
revision=parameters.revision,
task_ids=ListOfBasestring.FromSerializable([]),
try_job_url=try_job_url)
# Bail out if there were too many errors.
if (error and flakiness_util.MaximumSwarmingTaskRetriesReached(
flakiness_thus_far)):
logging.error(
'Swarming task ended in error after %d failed attempts. Giving '
'up' % flakiness_thus_far.failed_swarming_task_attempts)
flakiness_thus_far.error = error
yield AggregateFlakinessPipeline(
self.CreateInputObjectInstance(
AggregateFlakinessInput,
flakiness_thus_far=flakiness_thus_far,
incoming_swarming_task_output=None))
return
# Move on if the maximum number of iterations has been reached or exceeded.
if flakiness_util.MaximumIterationsReached(flakiness_thus_far):
logging.info('Max iterations reached for commit_position %d' %
commit_position)
yield AggregateFlakinessPipeline(
self.CreateInputObjectInstance(
AggregateFlakinessInput,
flakiness_thus_far=flakiness_thus_far,
incoming_swarming_task_output=None))
return
# Move on if the test doesn't exist.
if pass_rate_util.TestDoesNotExist(pass_rate_at_commit_position):
logging.info('No test found at commit position %d' %
commit_position)
yield AggregateFlakinessPipeline(
self.CreateInputObjectInstance(
AggregateFlakinessInput,
flakiness_thus_far=flakiness_thus_far,
incoming_swarming_task_output=None))
return
# Move on if there is sufficient information about the pass rate.
if pass_rate_util.HasSufficientInformation(
pass_rate_at_commit_position, flakiness_thus_far.iterations,
previous_pass_rate, previous_iterations):
logging.info(
'There is sufficient information for commit position %d with pass '
'rate %s after %d iterations' %
(commit_position, pass_rate_at_commit_position,
flakiness_thus_far.iterations))
yield AggregateFlakinessPipeline(
self.CreateInputObjectInstance(
AggregateFlakinessInput,
flakiness_thus_far=flakiness_thus_far,
incoming_swarming_task_output=None))
return
# Another swarming task is needed. Determine parameters for it to run.
iterations_for_task, time_for_task_seconds = (
run_swarming_util.CalculateRunParametersForSwarmingTask(
flakiness_thus_far, error))
# Run swarming task, update data points with results, and recurse.
with pipeline.InOrder():
swarming_task_output = yield RunFlakeSwarmingTaskPipeline(
self.CreateInputObjectInstance(
RunFlakeSwarmingTaskInput,
master_name=master_name,
builder_name=builder_name,
reference_build_number=reference_build_number,
step_name=step_name,
test_name=test_name,
commit_position=commit_position,
isolate_sha=get_isolate_sha_output.isolate_sha,
iterations=iterations_for_task,
timeout_seconds=time_for_task_seconds))
aggregated_flakiness = yield AggregateFlakinessPipeline(
self.CreateInputObjectInstance(
AggregateFlakinessInput,
flakiness_thus_far=flakiness_thus_far,
incoming_swarming_task_output=swarming_task_output))
yield DetermineApproximatePassRatePipelineWrapper(
self.CreateInputObjectInstance(
DetermineApproximatePassRateInput,
builder_name=parameters.builder_name,
commit_position=commit_position,
flakiness_thus_far=aggregated_flakiness,
get_isolate_sha_output=get_isolate_sha_output,
master_name=parameters.master_name,
previous_swarming_task_output=swarming_task_output,
reference_build_number=parameters.reference_build_number,
revision=parameters.revision,
step_name=parameters.step_name,
test_name=parameters.test_name))