def expand_jobs(self, step, phase_config):
"""
Creates and runs JobSteps for a set of tests, based on a phase config.
This phase config comes from a tests.json file that the collection
jobstep should generate. This method is then called by the TestsJsonHandler.
"""
assert phase_config['cmd']
assert '{test_names}' in phase_config['cmd']
assert 'tests' in phase_config
num_tests = len(phase_config['tests'])
test_stats, avg_test_time = TestsExpander.get_test_stats(self.get_test_stats_from() or step.project.slug)
phase, _ = get_or_create(JobPhase, where={
'job': step.job,
'project': step.project,
'label': phase_config.get('phase') or 'Test',
}, defaults={
'status': Status.queued
})
db.session.commit()
# If there are no tests to run, the phase is done.
if num_tests == 0:
phase.status = Status.finished
phase.result = Result.passed
db.session.add(phase)
db.session.commit()
return
# Check for whether a previous run of this task has already
# created JobSteps for us, since doing it again would create a
# double-sharded build.
steps = JobStep.query.filter_by(phase_id=phase.id, replacement_id=None).all()
if steps:
step_shard_counts = [s.data.get('shard_count', 1) for s in steps]
assert len(set(step_shard_counts)) == 1, "Mixed shard counts in phase!"
assert len(steps) == step_shard_counts[0]
else:
# Create all of the job steps and commit them together.
groups = TestsExpander.shard_tests(phase_config['tests'], self.max_shards,
test_stats, avg_test_time)
steps = [
self._create_jobstep(phase, phase_config['cmd'], phase_config.get('path', ''),
weight, test_list, len(groups))
for weight, test_list in groups
]
assert len(steps) == len(groups)
db.session.commit()
# Now that that database transaction is done, we'll do the slow work of
# creating jenkins builds.
for step in steps:
self._create_jenkins_build(step)
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def test_sharding(self):
tests = [
'foo/bar.py',
'foo/baz.py',
'foo.bar.test_biz',
'foo.bar.test_buz',
]
test_weights = {
('foo', 'bar'): 50,
('foo', 'baz'): 15,
('foo', 'bar', 'test_biz'): 10,
('foo', 'bar', 'test_buz'): 200,
}
avg_test_time = sum(test_weights.values()) / len(test_weights)
groups = TestsExpander.shard_tests(tests, 2, test_weights, avg_test_time)
assert len(groups) == 2
groups.sort()
assert groups[0] == (78, ['foo/bar.py', 'foo/baz.py', 'foo.bar.test_biz'])
assert groups[1] == (201, ['foo.bar.test_buz'])
groups = TestsExpander.shard_tests(tests, 3, test_weights, avg_test_time)
assert len(groups) == 3
groups.sort()
assert groups[0] == (27, ['foo/baz.py', 'foo.bar.test_biz'])
assert groups[1] == (51, ['foo/bar.py'])
assert groups[2] == (201, ['foo.bar.test_buz'])
# more shards than tests
groups = TestsExpander.shard_tests(tests, len(tests) * 2, test_weights, avg_test_time)
assert len(groups) == len(tests)
def expand_jobs(self, step, phase_config):
"""
Creates and runs JobSteps for a set of tests, based on a phase config.
This phase config comes from a tests.json file that the collection
jobstep should generate. This method is then called by the TestsJsonHandler.
"""
if not phase_config.get("cmd"):
raise ArtifactParseError("No cmd attribute")
if "{test_names}" not in phase_config["cmd"]:
raise ArtifactParseError("No {test_names} in cmd")
if "tests" not in phase_config:
raise ArtifactParseError("No tests attribute")
num_tests = len(phase_config["tests"])
test_stats, avg_test_time = TestsExpander.get_test_stats(self.get_test_stats_from() or step.project.slug)
phase, _ = get_or_create(
JobPhase,
where={"job": step.job, "project": step.project, "label": phase_config.get("phase") or "Test"},
defaults={"status": Status.queued},
)
db.session.commit()
# If there are no tests to run, the phase is done.
if num_tests == 0:
phase.status = Status.finished
phase.result = Result.passed
db.session.add(phase)
db.session.commit()
return
# Check for whether a previous run of this task has already
# created JobSteps for us, since doing it again would create a
# double-sharded build.
steps = JobStep.query.filter_by(phase_id=phase.id, replacement_id=None).all()
if steps:
step_shard_counts = [s.data.get("shard_count", 1) for s in steps]
if len(set(step_shard_counts)) != 1:
raise Exception("Mixed shard counts in phase!")
elif len(steps) != step_shard_counts[0]:
raise Exception("Shard count incorrect")
else:
# Create all of the job steps and commit them together.
groups = TestsExpander.shard_tests(phase_config["tests"], self.max_shards, test_stats, avg_test_time)
steps = [
self._create_jobstep(
phase, phase_config["cmd"], phase_config.get("path", ""), weight, test_list, len(groups)
)
for weight, test_list in groups
]
if len(steps) != len(groups):
raise Exception("Didn't create correct number of shards")
db.session.commit()
# Now that that database transaction is done, we'll do the slow work of
# creating jenkins builds.
for step in steps:
self._create_jenkins_build(step)
sync_job_step.delay_if_needed(step_id=step.id.hex, task_id=step.id.hex, parent_task_id=phase.job.id.hex)
def expand_jobs(self, step, phase_config):
"""
Creates and runs JobSteps for a set of tests, based on a phase config.
This phase config comes from a tests.json file that the collection
jobstep should generate. This method is then called by the TestsJsonHandler.
"""
assert phase_config['cmd']
assert '{test_names}' in phase_config['cmd']
assert 'tests' in phase_config
num_tests = len(phase_config['tests'])
test_stats, avg_test_time = TestsExpander.get_test_stats(self.get_test_stats_from() or step.project.slug)
phase, _ = get_or_create(JobPhase, where={
'job': step.job,
'project': step.project,
'label': phase_config.get('phase') or 'Test',
}, defaults={
'status': Status.queued
})
db.session.commit()
# If there are no tests to run, the phase is done.
if num_tests == 0:
phase.status = Status.finished
phase.result = Result.passed
db.session.add(phase)
db.session.commit()
return
# Check for whether a previous run of this task has already
# created JobSteps for us, since doing it again would create a
# double-sharded build.
steps = JobStep.query.filter_by(phase_id=phase.id, replacement_id=None).all()
if steps:
step_shard_counts = [s.data.get('shard_count', 1) for s in steps]
assert len(set(step_shard_counts)) == 1, "Mixed shard counts in phase!"
assert len(steps) == step_shard_counts[0]
else:
# Create all of the job steps and commit them together.
groups = TestsExpander.shard_tests(phase_config['tests'], self.max_shards,
test_stats, avg_test_time)
steps = [
self._create_jobstep(phase, phase_config['cmd'], phase_config.get('path', ''),
weight, test_list, len(groups))
for weight, test_list in groups
]
assert len(steps) == len(groups)
db.session.commit()
# Now that that database transaction is done, we'll do the slow work of
# creating jenkins builds.
for step in steps:
self._create_jenkins_build(step)
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def test_sharding(self):
tests = [
'foo/bar.py',
'foo/baz.py',
'foo.bar.test_biz',
'foo.bar.test_buz',
]
test_weights = {
('foo', 'bar'): 50,
('foo', 'baz'): 15,
('foo', 'bar', 'test_biz'): 10,
('foo', 'bar', 'test_buz'): 200,
}
avg_test_time = sum(test_weights.values()) / len(test_weights)
groups = TestsExpander.shard_tests(tests, 2, test_weights,
avg_test_time)
assert len(groups) == 2
groups.sort()
assert groups[0] == (78,
['foo/bar.py', 'foo/baz.py', 'foo.bar.test_biz'])
assert groups[1] == (201, ['foo.bar.test_buz'])
groups = TestsExpander.shard_tests(tests, 3, test_weights,
avg_test_time)
assert len(groups) == 3
groups.sort()
assert groups[0] == (27, ['foo/baz.py', 'foo.bar.test_biz'])
assert groups[1] == (51, ['foo/bar.py'])
assert groups[2] == (201, ['foo.bar.test_buz'])
# more shards than tests
groups = TestsExpander.shard_tests(tests,
len(tests) * 2, test_weights,
avg_test_time)
assert len(groups) == len(tests)
def get_expander(self, data):
return TestsExpander(self.project, data)
def expand_jobs(self, step, phase_config):
"""
Creates and runs JobSteps for a set of tests, based on a phase config.
This phase config comes from a tests.json file that the collection
jobstep should generate. This method is then called by the TestsJsonHandler.
"""
if not phase_config.get('cmd'):
raise ArtifactParseError('No cmd attribute')
if '{test_names}' not in phase_config['cmd']:
raise ArtifactParseError('No {test_names} in cmd')
if 'tests' not in phase_config:
raise ArtifactParseError('No tests attribute')
num_tests = len(phase_config['tests'])
test_stats, avg_test_time = TestsExpander.get_test_stats(
self.get_test_stats_from() or step.project.slug)
phase, _ = get_or_create(JobPhase,
where={
'job': step.job,
'project': step.project,
'label': phase_config.get('phase')
or 'Test',
},
defaults={'status': Status.queued})
db.session.commit()
# If there are no tests to run, the phase is done.
if num_tests == 0:
phase.status = Status.finished
phase.result = Result.passed
db.session.add(phase)
db.session.commit()
return
# Check for whether a previous run of this task has already
# created JobSteps for us, since doing it again would create a
# double-sharded build.
steps = JobStep.query.filter_by(phase_id=phase.id,
replacement_id=None).all()
if steps:
step_shard_counts = [s.data.get('shard_count', 1) for s in steps]
if len(set(step_shard_counts)) != 1:
raise Exception("Mixed shard counts in phase!")
elif len(steps) != step_shard_counts[0]:
raise Exception("Shard count incorrect")
else:
# Create all of the job steps and commit them together.
groups = shard(
phase_config['tests'],
self.max_shards,
test_stats,
avg_test_time,
normalize_object_name=TestsExpander._normalize_test_segments)
steps = [
self._create_jobstep(phase,
phase_config['cmd'],
phase_config.get('path', ''),
weight,
test_list,
len(groups),
cluster=step.cluster)
for weight, test_list in groups
]
if len(steps) != len(groups):
raise Exception("Didn't create correct number of shards")
db.session.commit()
# Now that the database transaction is done, we'll do the slow work of
# creating jenkins builds.
for step in steps:
self._create_jenkins_build(step)
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def test_normalize_test_segments(test_name, normalized):
assert TestsExpander._normalize_test_segments(test_name) == normalized
def test_normalize_test_segments(test_name, normalized):
assert TestsExpander._normalize_test_segments(test_name) == normalized
