def testDetermineApproximatePassRateConverged(self, *_):
master_name = 'm'
builder_name = 'b'
reference_build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
incoming_pass_count = 15
iterations = 30
incoming_pass_rate = 0.5
isolate_sha = 'sha1'
revision = 'r1000'
started_time = datetime(2018, 1, 1, 0, 0, 0)
completed_time = datetime(2018, 1, 1, 1, 0, 0)
build_url = 'url'
try_job_url = None
isolate_sha_output = GetIsolateShaOutput(
build_number=None,
build_url=None,
isolate_sha=isolate_sha,
try_job_url='url')
flake_swarming_task_output = FlakeSwarmingTaskOutput(
error=None,
pass_count=incoming_pass_count,
iterations=iterations,
started_time=started_time,
completed_time=completed_time,
task_id='task_id')
flakiness_thus_far = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=3600,
error=None,
failed_swarming_task_attempts=0,
iterations=iterations,
pass_rate=incoming_pass_rate,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable(['task_id_1']))
determine_approximate_pass_rate_input = DetermineApproximatePassRateInput(
builder_name=builder_name,
commit_position=commit_position,
get_isolate_sha_output=isolate_sha_output,
flakiness_thus_far=flakiness_thus_far,
previous_swarming_task_output=flake_swarming_task_output,
master_name=master_name,
reference_build_number=reference_build_number,
revision=revision,
step_name=step_name,
test_name=test_name)
pipeline_job = DetermineApproximatePassRatePipeline(
determine_approximate_pass_rate_input)
pipeline_job.start()
self.execute_queued_tasks()
def testDetermineApproximatePassRatePipelineWrapper(self):
master_name = 'm'
builder_name = 'b'
reference_build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
incoming_pass_rate = 0.5
isolate_sha = 'sha1'
revision = 'r1000'
build_url = None
try_job_url = 'url'
isolate_sha_output = GetIsolateShaOutput(
build_number=None,
build_url=build_url,
isolate_sha=isolate_sha,
try_job_url=try_job_url)
flakiness_thus_far = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=60,
error=None,
failed_swarming_task_attempts=0,
iterations=10,
pass_rate=incoming_pass_rate,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable(['task_id_1']))
determine_approximate_pass_rate_input = DetermineApproximatePassRateInput(
builder_name=builder_name,
commit_position=commit_position,
get_isolate_sha_output=isolate_sha_output,
flakiness_thus_far=flakiness_thus_far,
master_name=master_name,
previous_swarming_task_output=None,
reference_build_number=reference_build_number,
revision=revision,
step_name=step_name,
test_name=test_name)
self.MockGeneratorPipeline(DetermineApproximatePassRatePipeline,
determine_approximate_pass_rate_input, None)
pipeline_job = DetermineApproximatePassRatePipelineWrapper(
determine_approximate_pass_rate_input)
pipeline_job.start()
self.execute_queued_tasks()
def testGetIsolateShaForTryJobPipeline(self):
step_name = 'interactive_ui_tests'
expected_sha = 'ddd3e494ed97366f99453e5a8321b77449f26a58'
url = (
'https://ci.chromium.org/p/chromium/builders/luci.chromium.findit/'
'findit_variable/1391')
expected_output = GetIsolateShaOutput(
isolate_sha=expected_sha,
build_number=None,
build_url=None,
try_job_url=url).ToSerializable()
try_job_result = FlakeTryJobResult.FromSerializable({
'report': {
'previously_checked_out_revision':
'f5dc74a384d48f1a0929dc056cadae2a0019f8b5',
'previously_cached_revision':
'17b7ff2ff8e107b0e9cebcd9d6894072acc98639',
'isolated_tests': {
'interactive_ui_tests':
'ddd3e494ed97366f99453e5a8321b77449f26a58'
},
'last_checked_out_revision': None,
'metadata': {}
},
'url':
url,
'try_job_id':
'8951342990533358272'
})
get_try_job_sha_parameters = GetIsolateShaForTryJobParameters(
try_job_result=try_job_result, step_name=step_name)
pipeline_job = GetIsolateShaForTryJobPipeline(
get_try_job_sha_parameters)
pipeline_job.start()
self.execute_queued_tasks()
pipeline_job = pipelines.pipeline.Pipeline.from_id(
pipeline_job.pipeline_id)
pipeline_output = pipeline_job.outputs.default.value
self.assertEqual(expected_output, pipeline_output)
def testAnalyzeRecentFlakinessPipeline(self, mocked_commit_position,
mocked_step_metadata):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
analysis = MasterFlakeAnalysis.Create(master_name, builder_name,
build_number, step_name,
test_name)
analysis.data_points = [DataPoint.Create(commit_position=999)]
analysis.Save()
isolate_sha = 'sha'
latest_revision = 'r'
latest_commit_position = 1000
mocked_commit_position.return_value = (latest_commit_position,
latest_revision)
pass_rate = 0.5
get_sha_output = GetIsolateShaOutput(isolate_sha=isolate_sha,
build_url='url',
try_job_url=None)
step_metadata = StepMetadata(canonical_step_name=step_name,
dimensions=None,
full_step_name='s',
patched=False,
swarm_task_ids=None,
waterfall_buildername=builder_name,
waterfall_mastername=master_name,
isolate_target_name=step_name)
mocked_step_metadata.return_value = step_metadata.ToSerializable()
expected_flakiness = Flakiness(build_url='url',
commit_position=latest_commit_position,
revision=latest_revision,
pass_rate=pass_rate)
analyze_recent_flakiness_input = AnalyzeRecentFlakinessInput(
analysis_urlsafe_key=analysis.key.urlsafe())
expected_isolate_sha_input = GetIsolateShaForCommitPositionParameters(
analysis_urlsafe_key=analysis.key.urlsafe(),
commit_position=latest_commit_position,
dimensions=None,
revision=latest_revision,
step_metadata=step_metadata,
upper_bound_build_number=analysis.build_number)
expected_pass_rate_input = DetermineApproximatePassRateInput(
builder_name=analysis.builder_name,
commit_position=latest_commit_position,
flakiness_thus_far=None,
get_isolate_sha_output=get_sha_output,
master_name=analysis.master_name,
previous_swarming_task_output=None,
reference_build_number=analysis.build_number,
revision=latest_revision,
step_name=analysis.step_name,
test_name=analysis.test_name)
expected_save_flakiness_verification_input = SaveFlakinessVerificationInput(
analysis_urlsafe_key=analysis.key.urlsafe(),
flakiness=expected_flakiness)
self.MockGeneratorPipeline(GetIsolateShaForCommitPositionPipeline,
expected_isolate_sha_input, get_sha_output)
self.MockGeneratorPipeline(DetermineApproximatePassRatePipeline,
expected_pass_rate_input,
expected_flakiness)
self.MockGeneratorPipeline(SaveFlakinessVerificationPipeline,
expected_save_flakiness_verification_input,
None)
pipeline_job = AnalyzeRecentFlakinessPipeline(
analyze_recent_flakiness_input)
pipeline_job.start()
self.execute_queued_tasks()
mocked_step_metadata.assert_called_with(master_name, builder_name,
build_number, step_name)
mocked_commit_position.assert_called_with(master_name, builder_name,
step_name)
def testDetermineApproximatePassRateNotYetConverged(self, *_):
master_name = 'm'
builder_name = 'b'
reference_build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
incoming_pass_count = 15
iterations_completed = 30
expected_iterations = 15
incoming_pass_rate = 0.5
isolate_sha = 'sha1'
revision = 'r1000'
timeout_seconds = 3600
started_time = datetime(2018, 1, 1, 0, 0, 0)
completed_time = datetime(2018, 1, 1, 1, 0, 0)
build_url = None
try_job_url = 'url'
isolate_sha_output = GetIsolateShaOutput(
build_number=None,
build_url=build_url,
isolate_sha=isolate_sha,
try_job_url=try_job_url)
flakiness_thus_far = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=timeout_seconds,
error=None,
failed_swarming_task_attempts=0,
iterations=iterations_completed,
pass_rate=incoming_pass_rate,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable(['task_id_1']))
incoming_flake_swarming_task_output = FlakeSwarmingTaskOutput(
error=None,
pass_count=incoming_pass_count,
iterations=iterations_completed,
started_time=started_time,
completed_time=completed_time,
task_id='task_id_2')
expected_aggregate_flakiness_input = AggregateFlakinessInput(
flakiness_thus_far=flakiness_thus_far,
incoming_swarming_task_output=incoming_flake_swarming_task_output)
expected_aggregate_flakiness_output = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=timeout_seconds,
error=None,
failed_swarming_task_attempts=0,
iterations=45,
pass_rate=0.5,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable(['task_id_1']))
determine_approximate_pass_rate_input = DetermineApproximatePassRateInput(
builder_name=builder_name,
commit_position=commit_position,
flakiness_thus_far=flakiness_thus_far,
get_isolate_sha_output=isolate_sha_output,
master_name=master_name,
previous_swarming_task_output=incoming_flake_swarming_task_output,
reference_build_number=reference_build_number,
revision=revision,
step_name=step_name,
test_name=test_name)
flake_swarming_task_input = RunFlakeSwarmingTaskInput(
builder_name=builder_name,
commit_position=commit_position,
isolate_sha=isolate_sha,
iterations=expected_iterations,
master_name=master_name,
reference_build_number=reference_build_number,
step_name=step_name,
test_name=test_name,
timeout_seconds=timeout_seconds)
recursive_input = DetermineApproximatePassRateInput(
builder_name=builder_name,
commit_position=commit_position,
flakiness_thus_far=expected_aggregate_flakiness_output,
get_isolate_sha_output=isolate_sha_output,
master_name=master_name,
previous_swarming_task_output=incoming_flake_swarming_task_output,
reference_build_number=reference_build_number,
revision=revision,
step_name=step_name,
test_name=test_name)
self.MockAsynchronousPipeline(RunFlakeSwarmingTaskPipeline,
flake_swarming_task_input,
incoming_flake_swarming_task_output)
self.MockSynchronousPipeline(AggregateFlakinessPipeline,
expected_aggregate_flakiness_input,
expected_aggregate_flakiness_output)
self.MockGeneratorPipeline(DetermineApproximatePassRatePipelineWrapper,
recursive_input, None)
pipeline_job = DetermineApproximatePassRatePipeline(
determine_approximate_pass_rate_input)
pipeline_job.start()
self.execute_queued_tasks()
def testDetermineApproximatePassRateTestDoesNotExist(self, *_):
master_name = 'm'
builder_name = 'b'
reference_build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
incoming_pass_count = 0
iterations = 10
incoming_pass_rate = flake_constants.PASS_RATE_TEST_NOT_FOUND
isolate_sha = 'sha1'
revision = 'r1000'
task_id = 'task_id'
started_time = datetime(2018, 1, 1, 0, 0, 0)
completed_time = datetime(2018, 1, 1, 1, 0, 0)
build_url = 'url'
try_job_url = None
flakiness_thus_far = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=3600,
error=None,
failed_swarming_task_attempts=0,
iterations=iterations,
pass_rate=incoming_pass_rate,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable(['task_id_1']))
isolate_sha_output = GetIsolateShaOutput(
build_number=None,
build_url=build_url,
isolate_sha=isolate_sha,
try_job_url=try_job_url)
flake_swarming_task_output = FlakeSwarmingTaskOutput(
error=None,
pass_count=incoming_pass_count,
iterations=iterations,
task_id=task_id,
started_time=started_time,
completed_time=completed_time)
determine_approximate_pass_rate_input = DetermineApproximatePassRateInput(
builder_name=builder_name,
commit_position=commit_position,
flakiness_thus_far=flakiness_thus_far,
get_isolate_sha_output=isolate_sha_output,
master_name=master_name,
previous_swarming_task_output=flake_swarming_task_output,
reference_build_number=reference_build_number,
revision=revision,
step_name=step_name,
test_name=test_name)
pipeline_job = DetermineApproximatePassRatePipeline(
determine_approximate_pass_rate_input)
pipeline_job.start()
self.execute_queued_tasks()
pipeline_job = pipelines.pipeline.Pipeline.from_id(pipeline_job.pipeline_id)
self.assertEqual(flakiness_thus_far.ToSerializable(),
pipeline_job.outputs.default.value)
def testDetermineApproximatePassRateMaximumRetriesPerSwarmingTaskReached(
self, _):
master_name = 'm'
builder_name = 'b'
reference_build_number = 123
step_name = 's'
test_name = 't'
commit_position = 1000
incoming_pass_count = 15
iterations = 30
incoming_pass_rate = float(incoming_pass_count / iterations)
isolate_sha = 'sha1'
revision = 'r1000'
task_id = 'task_id_2'
started_time = datetime(2018, 1, 1, 0, 0, 0)
completed_time = datetime(2018, 1, 1, 1, 0, 0)
build_url = 'url'
try_job_url = None
swarming_task_error = SwarmingTaskError(code=1, message='error')
isolate_sha_output = GetIsolateShaOutput(
build_number=None,
build_url=build_url,
isolate_sha=isolate_sha,
try_job_url=try_job_url)
flakiness_thus_far = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=3600,
error=None,
failed_swarming_task_attempts=0,
iterations=iterations,
pass_rate=incoming_pass_rate,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable(['task_id_1']))
expected_flakiness_thus_far = Flakiness(
build_number=None,
build_url=build_url,
commit_position=commit_position,
total_test_run_seconds=3600,
error=swarming_task_error,
failed_swarming_task_attempts=0,
iterations=iterations,
pass_rate=incoming_pass_rate,
revision=revision,
try_job_url=try_job_url,
task_ids=ListOfBasestring.FromSerializable(['task_id_1']))
incoming_flake_swarming_task_output = FlakeSwarmingTaskOutput(
completed_time=completed_time,
error=swarming_task_error,
pass_count=incoming_pass_count,
iterations=iterations,
started_time=started_time,
task_id=task_id)
determine_approximate_pass_rate_input = DetermineApproximatePassRateInput(
builder_name=builder_name,
commit_position=commit_position,
flakiness_thus_far=flakiness_thus_far,
get_isolate_sha_output=isolate_sha_output,
master_name=master_name,
previous_swarming_task_output=incoming_flake_swarming_task_output,
reference_build_number=reference_build_number,
revision=revision,
step_name=step_name,
test_name=test_name)
pipeline_job = DetermineApproximatePassRatePipeline(
determine_approximate_pass_rate_input)
pipeline_job.start()
self.execute_queued_tasks()
pipeline_job = pipelines.pipeline.Pipeline.from_id(pipeline_job.pipeline_id)
self.assertEqual(expected_flakiness_thus_far.ToSerializable(),
pipeline_job.outputs.default.value)
def testGetIsolateShaForCommitPositionPipelineFallbackBuildLevel(
self, _, mocked_reference_build, mocked_url, mocked_build_info):
master_name = 'm'
builder_name = 'b'
build_number = 100
step_name = 's'
test_name = 't'
requested_commit_position = 1000
requested_revision = 'r1000'
expected_sha = 'sha1'
build_url = 'url'
isolate_target_name = 'browser_tests'
step_metadata = StepMetadata(canonical_step_name=None,
dimensions=None,
full_step_name=None,
isolate_target_name=isolate_target_name,
patched=True,
swarm_task_ids=None,
waterfall_buildername=None,
waterfall_mastername=None)
mocked_url.return_value = build_url
expected_output = GetIsolateShaOutput(isolate_sha=expected_sha,
build_number=build_number,
build_url=build_url,
try_job_url=None)
build = BuildInfo(master_name, builder_name, build_number)
build.commit_position = requested_commit_position
mocked_build_info.return_value = (None, build)
mocked_reference_build.return_value = build
analysis = MasterFlakeAnalysis.Create(master_name, builder_name,
build_number, step_name,
test_name)
analysis.Save()
get_build_sha_parameters = GetIsolateShaForBuildParameters(
master_name=master_name,
builder_name=builder_name,
build_number=build_number,
url=build_url,
step_name=step_name)
get_sha_input = GetIsolateShaForCommitPositionParameters(
analysis_urlsafe_key=unicode(analysis.key.urlsafe()),
commit_position=requested_commit_position,
dimensions=ListOfBasestring.FromSerializable([]),
revision=requested_revision,
step_metadata=step_metadata,
upper_bound_build_number=analysis.build_number)
self.MockSynchronousPipeline(GetIsolateShaForBuildPipeline,
get_build_sha_parameters, expected_output)
pipeline_job = GetIsolateShaForCommitPositionPipeline(get_sha_input)
pipeline_job.start()
self.execute_queued_tasks()
pipeline_job = pipelines.pipeline.Pipeline.from_id(
pipeline_job.pipeline_id)
pipeline_output = pipeline_job.outputs.default.value
self.assertEqual(expected_output.ToSerializable(), pipeline_output)
def testGetIsolateShaForCommitPositionPipelineMatchingTarget(
self, mocked_reference_build):
master_name = 'm'
builder_name = 'b'
parent_mastername = 'p_m'
parent_buildername = 'p_b'
build_number = 100
build_id = 123
test_name = 't'
requested_commit_position = 1000
requested_revision = 'r1000'
expected_sha = 'sha1'
build_url = 'url'
luci_name = 'chromium'
bucket_name = 'ci'
gitiles_host = 'chromium.googlesource.com'
gitiles_project = 'chromium/src'
gitiles_ref = 'refs/heads/master'
gerrit_patch = ''
isolate_target_name = 'browser_tests'
step_name = 's'
isolated_hash = 'isolated_hash'
expected_output = GetIsolateShaOutput(isolate_sha=expected_sha,
build_number=None,
build_url=build_url,
try_job_url=None)
analysis = MasterFlakeAnalysis.Create(master_name, builder_name,
build_number, step_name,
test_name)
analysis.Save()
build = BuildInfo(master_name, builder_name, build_number)
build.commit_position = requested_commit_position
build.parent_mastername = parent_mastername
build.parent_buildername = parent_buildername
mocked_reference_build.return_value = build
isolated_target = IsolatedTarget.Create(
build_id, luci_name, bucket_name, parent_mastername,
parent_buildername, gitiles_host, gitiles_project, gitiles_ref,
gerrit_patch, isolate_target_name, isolated_hash,
requested_commit_position, requested_revision)
isolated_target.put()
step_metadata = StepMetadata(canonical_step_name=None,
dimensions=None,
full_step_name=None,
isolate_target_name=isolate_target_name,
patched=True,
swarm_task_ids=None,
waterfall_buildername=None,
waterfall_mastername=None)
get_sha_input = GetIsolateShaForCommitPositionParameters(
analysis_urlsafe_key=unicode(analysis.key.urlsafe()),
commit_position=requested_commit_position,
dimensions=ListOfBasestring.FromSerializable([]),
revision=requested_revision,
step_metadata=step_metadata,
upper_bound_build_number=analysis.build_number)
get_sha_for_target_input = GetIsolateShaForTargetInput(
isolated_target_urlsafe_key=isolated_target.key.urlsafe())
self.MockSynchronousPipeline(GetIsolateShaForTargetPipeline,
get_sha_for_target_input, expected_output)
pipeline_job = GetIsolateShaForCommitPositionPipeline(get_sha_input)
pipeline_job.start()
self.execute_queued_tasks()
pipeline_job = pipelines.pipeline.Pipeline.from_id(
pipeline_job.pipeline_id)
pipeline_output = pipeline_job.outputs.default.value
self.assertEqual(expected_output.ToSerializable(), pipeline_output)
def testAnalyzeFlakePipelineCanStartAnalysisImmediately(self, _):
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.Save()
start_commit_position = 1000
start_revision = 'r1000'
isolate_sha = 'sha1'
next_commit_position = 999
pass_rate = 0.5
build_url = 'url'
try_job_url = None
get_sha_output = GetIsolateShaOutput(
isolate_sha=isolate_sha, build_url=build_url, try_job_url=try_job_url)
step_metadata = StepMetadata(
canonical_step_name='s',
dimensions=None,
full_step_name='s',
patched=False,
swarm_task_ids=None,
waterfall_buildername='b',
waterfall_mastername='w',
isolate_target_name='s')
expected_flakiness = Flakiness(
build_url=build_url,
commit_position=start_commit_position,
revision=start_revision,
pass_rate=pass_rate)
analyze_flake_input = AnalyzeFlakeInput(
analysis_urlsafe_key=analysis.key.urlsafe(),
analyze_commit_position_parameters=NextCommitPositionOutput(
next_commit_id=CommitID(
commit_position=start_commit_position, revision=start_revision),
culprit_commit_id=None),
commit_position_range=IntRange(lower=None, upper=None),
dimensions=ListOfBasestring.FromSerializable(['os:testOS']),
manually_triggered=False,
rerun=False,
retries=0,
step_metadata=step_metadata)
expected_isolate_sha_input = GetIsolateShaForCommitPositionParameters(
analysis_urlsafe_key=analysis.key.urlsafe(),
commit_position=start_commit_position,
dimensions=ListOfBasestring.FromSerializable(['os:testOS']),
revision=start_revision,
step_metadata=step_metadata,
upper_bound_build_number=analysis.build_number)
expected_pass_rate_input = DetermineApproximatePassRateInput(
builder_name=analysis.builder_name,
commit_position=start_commit_position,
flakiness_thus_far=None,
get_isolate_sha_output=get_sha_output,
master_name=analysis.master_name,
previous_swarming_task_output=None,
reference_build_number=analysis.build_number,
revision=start_revision,
step_name=analysis.step_name,
test_name=analysis.test_name)
expected_update_data_points_input = UpdateFlakeAnalysisDataPointsInput(
analysis_urlsafe_key=analysis.key.urlsafe(),
flakiness=expected_flakiness)
expected_next_commit_position_input = NextCommitPositionInput(
analysis_urlsafe_key=analysis.key.urlsafe(),
commit_position_range=IntRange(lower=None, upper=None),
step_metadata=step_metadata)
next_commit_id = CommitID(
commit_position=next_commit_position, revision='r999')
expected_next_commit_position_output = NextCommitPositionOutput(
next_commit_id=next_commit_id, culprit_commit_id=None)
expected_recursive_analyze_flake_input = AnalyzeFlakeInput(
analysis_urlsafe_key=analysis.key.urlsafe(),
analyze_commit_position_parameters=expected_next_commit_position_output,
commit_position_range=IntRange(lower=None, upper=None),
dimensions=ListOfBasestring.FromSerializable(['os:testOS']),
manually_triggered=False,
rerun=False,
retries=0,
step_metadata=step_metadata)
self.MockGeneratorPipeline(GetIsolateShaForCommitPositionPipeline,
expected_isolate_sha_input, get_sha_output)
self.MockGeneratorPipeline(DetermineApproximatePassRatePipeline,
expected_pass_rate_input, expected_flakiness)
self.MockSynchronousPipeline(UpdateFlakeAnalysisDataPointsPipeline,
expected_update_data_points_input, None)
self.MockSynchronousPipeline(NextCommitPositionPipeline,
expected_next_commit_position_input,
expected_next_commit_position_output)
self.MockGeneratorPipeline(RecursiveAnalyzeFlakePipeline,
expected_recursive_analyze_flake_input, None)
pipeline_job = AnalyzeFlakePipeline(analyze_flake_input)
pipeline_job.start()
self.execute_queued_tasks()