def expand_jobs(self, step, phase_config):
"""
Creates and runs JobSteps for a set of tests, based on a phase config.
This phase config comes from a tests.json file that the collection
jobstep should generate. This method is then called by the TestsJsonHandler.
"""
if not phase_config.get("cmd"):
raise ArtifactParseError("No cmd attribute")
if "{test_names}" not in phase_config["cmd"]:
raise ArtifactParseError("No {test_names} in cmd")
if "tests" not in phase_config:
raise ArtifactParseError("No tests attribute")
num_tests = len(phase_config["tests"])
test_stats, avg_test_time = TestsExpander.get_test_stats(self.get_test_stats_from() or step.project.slug)
phase, _ = get_or_create(
JobPhase,
where={"job": step.job, "project": step.project, "label": phase_config.get("phase") or "Test"},
defaults={"status": Status.queued},
)
db.session.commit()
# If there are no tests to run, the phase is done.
if num_tests == 0:
phase.status = Status.finished
phase.result = Result.passed
db.session.add(phase)
db.session.commit()
return
# Check for whether a previous run of this task has already
# created JobSteps for us, since doing it again would create a
# double-sharded build.
steps = JobStep.query.filter_by(phase_id=phase.id, replacement_id=None).all()
if steps:
step_shard_counts = [s.data.get("shard_count", 1) for s in steps]
if len(set(step_shard_counts)) != 1:
raise Exception("Mixed shard counts in phase!")
elif len(steps) != step_shard_counts[0]:
raise Exception("Shard count incorrect")
else:
# Create all of the job steps and commit them together.
groups = TestsExpander.shard_tests(phase_config["tests"], self.max_shards, test_stats, avg_test_time)
steps = [
self._create_jobstep(
phase, phase_config["cmd"], phase_config.get("path", ""), weight, test_list, len(groups)
)
for weight, test_list in groups
]
if len(steps) != len(groups):
raise Exception("Didn't create correct number of shards")
db.session.commit()
# Now that that database transaction is done, we'll do the slow work of
# creating jenkins builds.
for step in steps:
self._create_jenkins_build(step)
sync_job_step.delay_if_needed(step_id=step.id.hex, task_id=step.id.hex, parent_task_id=phase.job.id.hex)
def _expand_job(self, phase, job_config):
label = job_config.get('name') or md5(job_config['cmd']).hexdigest()
step, created = get_or_create(JobStep,
where={
'job': phase.job,
'project': phase.project,
'phase': phase,
'label': label,
},
defaults={
'data': {
'cmd': job_config['cmd'],
'job_name': self.job_name,
'build_no': None,
'expanded': True,
},
'status': Status.queued,
})
# TODO(dcramer): due to no unique constraints this section of code
# presents a race condition when run concurrently
if not step.data.get('build_no'):
builder = self.get_builder()
params = builder.get_job_parameters(step.job,
changes_bid=step.id.hex,
script=step.data['cmd'])
success = False
exn = None
for _ in range(0, 3):
try:
job_data = builder.create_job_from_params(
changes_bid=step.id.hex,
params=params,
job_name=step.data['job_name'],
)
step.data.update(job_data)
db.session.add(step)
db.session.commit()
success = True
break
except Exception as ex:
logging.exception("Failed to create jobstep")
exn = ex
if not success:
step.status = Status.finished
step.result = Result.infra_failed
db.session.add(step)
db.session.commit()
if exn:
raise exn
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def sync_step(self, step):
if step.data.get('job_name') != self.job_name:
return super(JenkinsFactoryBuilder, self).sync_step(step)
# for any downstream jobs, pull their results using xpath magic
for downstream_job_name in self.downstream_job_names:
downstream_build_nos = self._get_downstream_jobs(step, downstream_job_name)
if not downstream_build_nos:
continue
phase, created = get_or_create(JobPhase, where={
'job': step.job,
'label': downstream_job_name,
}, defaults={
'project_id': step.job.project_id,
})
db.session.commit()
for build_no in downstream_build_nos:
# XXX(dcramer): ideally we would grab this with the first query
# but because we dont want to rely on an XML parser, we're doing
# a second http request for build details
downstream_step = self._create_job_step(
phase, downstream_job_name, build_no)
db.session.commit()
sync_job_step.delay_if_needed(
step_id=downstream_step.id.hex,
task_id=downstream_step.id.hex,
parent_task_id=step.job.id.hex,
)
return super(JenkinsFactoryBuilder, self).sync_step(step)
def _expand_job(self, phase, label, cmd, replaces=None):
where = {
'job': phase.job,
'project': phase.project,
'phase': phase,
'label': label,
}
if replaces:
# uuid is unique which forces jobstep to be created
where['id'] = uuid.uuid4()
step, created = get_or_create(JobStep, where=where, defaults={
'data': {
'cmd': cmd,
'job_name': self.job_name,
'build_no': None,
'expanded': True,
},
'status': Status.queued,
})
assert created or not replaces
BuildStep.handle_debug_infra_failures(step, self.debug_config, 'expanded')
if replaces:
replaces.replacement_id = step.id
db.session.add(replaces)
builder = self.get_builder()
builder.create_jenkins_build(step, job_name=step.data['job_name'], script=step.data['cmd'])
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
return step
def expand_jobs(self, step, phase_config):
"""
Creates and runs JobSteps for a set of tests, based on a phase config.
This phase config comes from a tests.json file that the collection
jobstep should generate. This method is then called by the TestsJsonHandler.
"""
assert phase_config['cmd']
assert '{test_names}' in phase_config['cmd']
assert 'tests' in phase_config
num_tests = len(phase_config['tests'])
test_stats, avg_test_time = TestsExpander.get_test_stats(self.get_test_stats_from() or step.project.slug)
phase, _ = get_or_create(JobPhase, where={
'job': step.job,
'project': step.project,
'label': phase_config.get('phase') or 'Test',
}, defaults={
'status': Status.queued
})
db.session.commit()
# If there are no tests to run, the phase is done.
if num_tests == 0:
phase.status = Status.finished
phase.result = Result.passed
db.session.add(phase)
db.session.commit()
return
# Check for whether a previous run of this task has already
# created JobSteps for us, since doing it again would create a
# double-sharded build.
steps = JobStep.query.filter_by(phase_id=phase.id, replacement_id=None).all()
if steps:
step_shard_counts = [s.data.get('shard_count', 1) for s in steps]
assert len(set(step_shard_counts)) == 1, "Mixed shard counts in phase!"
assert len(steps) == step_shard_counts[0]
else:
# Create all of the job steps and commit them together.
groups = TestsExpander.shard_tests(phase_config['tests'], self.max_shards,
test_stats, avg_test_time)
steps = [
self._create_jobstep(phase, phase_config['cmd'], phase_config.get('path', ''),
weight, test_list, len(groups))
for weight, test_list in groups
]
assert len(steps) == len(groups)
db.session.commit()
# Now that that database transaction is done, we'll do the slow work of
# creating jenkins builds.
for step in steps:
self._create_jenkins_build(step)
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def expand_jobs(self, step, phase_config):
"""
Creates and runs JobSteps for a set of tests, based on a phase config.
This phase config comes from a tests.json file that the collection
jobstep should generate. This method is then called by the TestsJsonHandler.
"""
assert phase_config['cmd']
assert '{test_names}' in phase_config['cmd']
assert 'tests' in phase_config
num_tests = len(phase_config['tests'])
test_stats, avg_test_time = TestsExpander.get_test_stats(self.get_test_stats_from() or step.project.slug)
phase, _ = get_or_create(JobPhase, where={
'job': step.job,
'project': step.project,
'label': phase_config.get('phase') or 'Test',
}, defaults={
'status': Status.queued
})
db.session.commit()
# If there are no tests to run, the phase is done.
if num_tests == 0:
phase.status = Status.finished
phase.result = Result.passed
db.session.add(phase)
db.session.commit()
return
# Check for whether a previous run of this task has already
# created JobSteps for us, since doing it again would create a
# double-sharded build.
steps = JobStep.query.filter_by(phase_id=phase.id, replacement_id=None).all()
if steps:
step_shard_counts = [s.data.get('shard_count', 1) for s in steps]
assert len(set(step_shard_counts)) == 1, "Mixed shard counts in phase!"
assert len(steps) == step_shard_counts[0]
else:
# Create all of the job steps and commit them together.
groups = TestsExpander.shard_tests(phase_config['tests'], self.max_shards,
test_stats, avg_test_time)
steps = [
self._create_jobstep(phase, phase_config['cmd'], phase_config.get('path', ''),
weight, test_list, len(groups))
for weight, test_list in groups
]
assert len(steps) == len(groups)
db.session.commit()
# Now that that database transaction is done, we'll do the slow work of
# creating jenkins builds.
for step in steps:
self._create_jenkins_build(step)
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def _expand_jobs(self, step, artifact):
builder = self.get_builder()
artifact_data = builder.fetch_artifact(step, artifact.data)
phase_config = artifact_data.json()
assert phase_config['cmd']
assert '{test_names}' in phase_config['cmd']
assert 'tests' in phase_config
num_tests = len(phase_config['tests'])
test_stats, avg_test_time = self.get_test_stats(step.project)
phase, created = get_or_create(JobPhase, where={
'job': step.job,
'project': step.project,
'label': phase_config.get('phase') or 'Test',
}, defaults={
'status': Status.queued
})
db.session.commit()
# If there are no tests to run, the phase is done.
if num_tests == 0:
phase.status = Status.finished
phase.result = Result.passed
db.session.add(phase)
db.session.commit()
return
# Check for whether a previous run of this task has already
# created JobSteps for us, since doing it again would create a
# double-sharded build.
steps = JobStep.query.filter_by(phase_id=phase.id).all()
if steps:
step_shard_counts = [s.data.get('shard_count', 1) for s in steps]
assert len(set(step_shard_counts)) == 1, "Mixed shard counts in phase!"
assert len(steps) == step_shard_counts[0]
else:
# Create all of the job steps and commit them together.
groups = self._shard_tests(phase_config['tests'], self.max_shards,
test_stats, avg_test_time)
steps = [
self._create_jobstep(phase, phase_config, weight, test_list, len(groups))
for weight, test_list in groups
]
assert len(steps) == len(groups)
db.session.commit()
# Now that that database transaction is done, we'll do the slow work of
# creating jenkins builds.
for step in steps:
self._create_jenkins_build(phase, step)
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def _expand_job(self, phase, job_config):
label = job_config.get('name') or md5(job_config['cmd']).hexdigest()
step, created = get_or_create(JobStep, where={
'job': phase.job,
'project': phase.project,
'phase': phase,
'label': label,
}, defaults={
'data': {
'cmd': job_config['cmd'],
'job_name': self.job_name,
'build_no': None,
'expanded': True,
},
'status': Status.queued,
})
# TODO(dcramer): due to no unique constraints this section of code
# presents a race condition when run concurrently
if not step.data.get('build_no'):
builder = self.get_builder()
params = builder.get_job_parameters(
step.job, changes_bid=step.id.hex, script=step.data['cmd'])
success = False
exn = None
for _ in range(0, 3):
try:
job_data = builder.create_job_from_params(
changes_bid=step.id.hex,
params=params,
job_name=step.data['job_name'],
)
step.data.update(job_data)
db.session.add(step)
db.session.commit()
success = True
break
except Exception as ex:
logging.exception("Failed to create jobstep")
exn = ex
if not success:
step.status = Status.finished
step.result = Result.infra_failed
db.session.add(step)
db.session.commit()
if exn:
raise exn
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def _expand_jobs(self, step, artifact):
builder = self.get_builder()
artifact_data = builder.fetch_artifact(step, artifact.data)
phase_config = artifact_data.json()
assert phase_config['cmd']
assert '{test_names}' in phase_config['cmd']
assert phase_config['tests']
test_stats, avg_test_time = self.get_test_stats(step.project)
phase, created = get_or_create(JobPhase,
where={
'job':
step.job,
'project':
step.project,
'label':
phase_config.get('phase') or 'Test',
},
defaults={
'status': Status.queued,
})
db.session.commit()
# Check for whether a previous run of this task has already
# created JobSteps for us, since doing it again would create a
# double-sharded build.
steps = JobStep.query.filter_by(phase_id=phase.id).all()
if steps:
step_shard_counts = [s.data.get('shard_count', 1) for s in steps]
assert len(
set(step_shard_counts)) == 1, "Mixed shard counts in phase!"
assert len(steps) == step_shard_counts[0]
else:
# Create all of the job steps and commit them together.
groups = self._shard_tests(phase_config['tests'], self.max_shards,
test_stats, avg_test_time)
steps = [
self._create_jobstep(phase, phase_config, weight, test_list,
len(groups))
for weight, test_list in groups
]
assert len(steps) == len(groups)
db.session.commit()
# Now that that database transaction is done, we'll do the slow work of
# creating jenkins builds.
for step in steps:
self._create_jenkins_build(phase, step)
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def _expand_job(self, phase, job_config):
assert job_config['tests']
test_names = ' '.join(job_config['tests'])
label = md5(test_names).hexdigest()
step, created = get_or_create(JobStep,
where={
'job': phase.job,
'project': phase.project,
'phase': phase,
'label': label,
},
defaults={
'data': {
'cmd': job_config['cmd'],
'path': job_config['path'],
'tests': job_config['tests'],
'expanded': True,
'job_name': self.job_name,
'build_no': None,
'weight': job_config['weight']
},
'status': Status.queued,
})
# TODO(dcramer): due to no unique constraints this section of code
# presents a race condition when run concurrently
if not step.data.get('build_no'):
builder = self.get_builder()
params = builder.get_job_parameters(
step.job,
script=step.data['cmd'].format(test_names=test_names, ),
target_id=step.id.hex,
path=step.data['path'],
)
is_diff = not step.job.source.is_commit()
job_data = builder.create_job_from_params(
target_id=step.id.hex,
params=params,
job_name=step.data['job_name'],
is_diff=is_diff)
step.data.update(job_data)
db.session.add(step)
db.session.commit()
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def _expand_job(self, phase, job_config):
assert job_config['tests']
test_names = ' '.join(job_config['tests'])
label = md5(test_names).hexdigest()
step, created = get_or_create(JobStep, where={
'job': phase.job,
'project': phase.project,
'phase': phase,
'label': label,
}, defaults={
'data': {
'cmd': job_config['cmd'],
'path': job_config['path'],
'tests': job_config['tests'],
'expanded': True,
'job_name': self.job_name,
'build_no': None,
'weight': job_config['weight']
},
'status': Status.queued,
})
# TODO(dcramer): due to no unique constraints this section of code
# presents a race condition when run concurrently
if not step.data.get('build_no'):
builder = self.get_builder()
params = builder.get_job_parameters(
step.job,
script=step.data['cmd'].format(
test_names=test_names,
),
target_id=step.id.hex,
path=step.data['path'],
)
job_data = builder.create_job_from_params(
target_id=step.id.hex,
params=params,
job_name=step.data['job_name'],
)
step.data.update(job_data)
db.session.add(step)
db.session.commit()
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def create_replacement_jobstep(self, step):
if not step.data.get("expanded", False):
return self._setup_jobstep(step.phase, step.job, replaces=step)
future_commands = map(FutureCommand.from_command, step.commands)
future_jobstep = FutureJobStep(step.label, commands=future_commands)
# we skip adding setup and teardown commands because these will already
# be present in the old, failed JobStep.
new_jobstep = self.create_expanded_jobstep(step, step.phase, future_jobstep, skip_setup_teardown=True)
db.session.flush()
step.replacement_id = new_jobstep.id
db.session.add(step)
db.session.commit()
sync_job_step.delay_if_needed(
step_id=new_jobstep.id.hex, task_id=new_jobstep.id.hex, parent_task_id=new_jobstep.job.id.hex
)
return new_jobstep
def create_replacement_jobstep(self, step):
if not step.data.get('expanded'):
return super(JenkinsTestCollectorBuildStep, self).create_replacement_jobstep(step)
newstep = self._create_jobstep(step.phase, step.data['cmd'], step.data['path'],
step.data['weight'], step.data['tests'],
step.data['shard_count'], force_create=True)
step.replacement_id = newstep.id
db.session.add(step)
db.session.commit()
self._create_jenkins_build(newstep)
sync_job_step.delay_if_needed(
step_id=newstep.id.hex,
task_id=newstep.id.hex,
parent_task_id=newstep.phase.job.id.hex,
)
return newstep
def sync_step(self, step):
if step.data.get('job_name') != self.job_name:
return super(JenkinsFactoryBuilder, self).sync_step(step)
job = step.job
# for any downstream jobs, pull their results using xpath magic
for downstream_job_name in self.downstream_job_names:
downstream_build_nos = self._get_downstream_jobs(
step, downstream_job_name)
if not downstream_build_nos:
continue
phase, created = get_or_create(JobPhase,
where={
'job': job,
'label': downstream_job_name,
},
defaults={
'project_id': job.project_id,
})
db.session.commit()
for build_no in downstream_build_nos:
# XXX(dcramer): ideally we would grab this with the first query
# but because we dont want to rely on an XML parser, we're doing
# a second http request for build details
downstream_step = self._create_job_step(
phase,
data={
'job_name': downstream_job_name,
'build_no': build_no,
'queued': False,
'master': step.data['master']
})
db.session.commit()
sync_job_step.delay_if_needed(
step_id=downstream_step.id.hex,
task_id=downstream_step.id.hex,
parent_task_id=job.id.hex,
)
return super(JenkinsFactoryBuilder, self).sync_step(step)
def create_replacement_jobstep(self, step):
if not step.data.get('expanded'):
return super(JenkinsTestCollectorBuildStep, self).create_replacement_jobstep(step)
newstep = self._create_jobstep(step.phase, step.data['cmd'], step.data['path'],
step.data['weight'], step.data['tests'],
step.data['shard_count'], force_create=True)
step.replacement_id = newstep.id
db.session.add(step)
db.session.commit()
self._create_jenkins_build(newstep)
sync_job_step.delay_if_needed(
step_id=newstep.id.hex,
task_id=newstep.id.hex,
parent_task_id=newstep.phase.job.id.hex,
)
return newstep
def _expand_job(self, phase, job_config):
label = job_config.get("name") or md5(job_config["cmd"]).hexdigest()
step, created = get_or_create(
JobStep,
where={"job": phase.job, "project": phase.project, "phase": phase, "label": label},
defaults={
"data": {"cmd": job_config["cmd"], "job_name": self.job_name, "build_no": None, "expanded": True},
"status": Status.queued,
},
)
# TODO(dcramer): due to no unique constraints this section of code
# presents a race condition when run concurrently
if not step.data.get("build_no"):
builder = self.get_builder()
params = builder.get_job_parameters(step.job, changes_bid=step.id.hex, script=step.data["cmd"])
success = False
exn = None
for _ in range(0, 3):
try:
job_data = builder.create_job_from_params(
changes_bid=step.id.hex, params=params, job_name=step.data["job_name"]
)
step.data.update(job_data)
db.session.add(step)
db.session.commit()
success = True
break
except Exception as ex:
logging.exception("Failed to create jobstep")
exn = ex
if not success:
step.status = Status.finished
step.result = Result.infra_failed
db.session.add(step)
db.session.commit()
if exn:
raise exn
sync_job_step.delay_if_needed(step_id=step.id.hex, task_id=step.id.hex, parent_task_id=phase.job.id.hex)
def create_replacement_jobstep(self, step):
if not step.data.get('expanded', False):
return self._setup_jobstep(step.phase, step.job, replaces=step)
future_commands = map(FutureCommand.from_command, step.commands)
future_jobstep = FutureJobStep(step.label, commands=future_commands)
# we skip adding setup and teardown commands because these will already
# be present in the old, failed JobStep.
new_jobstep = self.create_expanded_jobstep(step, step.phase, future_jobstep,
skip_setup_teardown=True)
db.session.flush()
step.replacement_id = new_jobstep.id
db.session.add(step)
db.session.commit()
sync_job_step.delay_if_needed(
step_id=new_jobstep.id.hex,
task_id=new_jobstep.id.hex,
parent_task_id=new_jobstep.job.id.hex,
)
return new_jobstep
def expand_command(self, command, expander, data):
jobstep = command.jobstep
phase_name = data.get('phase')
if not phase_name:
phase_count = db.session.query(
func.count(),
).filter(
JobPhase.job_id == jobstep.job_id,
).scalar()
phase_name = 'Phase #{}'.format(phase_count)
new_jobphase = JobPhase(
job_id=jobstep.job_id,
project_id=jobstep.project_id,
label=phase_name,
status=Status.queued,
)
db.session.add(new_jobphase)
_, buildstep = JobPlan.get_build_step_for_job(jobstep.job_id)
results = []
for future_jobstep in expander.expand(max_executors=jobstep.data['max_executors'],
test_stats_from=buildstep.get_test_stats_from()):
new_jobstep = buildstep.create_expanded_jobstep(jobstep, new_jobphase, future_jobstep)
results.append(new_jobstep)
# If there are no tests to run, the phase is done.
if len(results) == 0:
new_jobphase.status = Status.finished
new_jobphase.result = Result.passed
db.session.add(new_jobphase)
db.session.flush()
for new_jobstep in results:
sync_job_step.delay_if_needed(
step_id=new_jobstep.id.hex,
task_id=new_jobstep.id.hex,
parent_task_id=new_jobphase.job.id.hex,
)
return results
def expand_command(self, command, expander, data):
jobstep = command.jobstep
phase_name = data.get('phase')
if not phase_name:
phase_count = db.session.query(func.count(), ).filter(
JobPhase.job_id == jobstep.job_id, ).scalar()
phase_name = 'Phase #{}'.format(phase_count)
new_jobphase = JobPhase(
job_id=jobstep.job_id,
project_id=jobstep.project_id,
label=phase_name,
status=Status.queued,
)
db.session.add(new_jobphase)
_, buildstep = JobPlan.get_build_step_for_job(jobstep.job_id)
results = []
for future_jobstep in expander.expand(
max_executors=jobstep.data['max_executors'],
test_stats_from=buildstep.get_test_stats_from()):
new_jobstep = buildstep.create_expanded_jobstep(
jobstep, new_jobphase, future_jobstep)
results.append(new_jobstep)
# If there are no tests to run, the phase is done.
if len(results) == 0:
new_jobphase.status = Status.finished
new_jobphase.result = Result.passed
db.session.add(new_jobphase)
db.session.flush()
for new_jobstep in results:
sync_job_step.delay_if_needed(
step_id=new_jobstep.id.hex,
task_id=new_jobstep.id.hex,
parent_task_id=new_jobphase.job.id.hex,
)
return results
def _expand_job(self, phase, job_config):
label = job_config.get('name') or md5(job_config['cmd']).hexdigest()
step, created = get_or_create(JobStep,
where={
'job': phase.job,
'project': phase.project,
'phase': phase,
'label': label,
},
defaults={
'data': {
'cmd': job_config['cmd'],
'job_name': self.job_name,
'build_no': None,
},
'status': Status.queued,
})
# TODO(dcramer): due to no unique constraints this section of code
# presents a race condition when run concurrently
if not step.data.get('build_no'):
builder = self.get_builder()
params = builder.get_job_parameters(step.job,
script=step.data['cmd'])
job_data = builder.create_job_from_params(
job_id=step.id.hex,
params=params,
job_name=step.data['job_name'],
)
step.data.update(job_data)
db.session.add(step)
db.session.commit()
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def expand_command(self, command, expander, data):
jobstep = command.jobstep
phase_name = data.get('phase')
if not phase_name:
phase_count = db.session.query(
func.count(),
).filter(
JobPhase.job_id == jobstep.job_id,
).scalar()
phase_name = 'Phase #{}'.format(phase_count)
jobstep.data['expanded'] = True
db.session.add(jobstep)
new_jobphase = JobPhase(
job_id=jobstep.job_id,
project_id=jobstep.project_id,
label=phase_name,
status=Status.queued,
)
db.session.add(new_jobphase)
_, buildstep = JobPlan.get_build_step_for_job(jobstep.job_id)
results = []
for future_jobstep in expander.expand(max_executors=jobstep.data['max_executors']):
new_jobstep = buildstep.expand_jobstep(jobstep, new_jobphase, future_jobstep)
results.append(new_jobstep)
db.session.flush()
for new_jobstep in results:
sync_job_step.delay_if_needed(
step_id=new_jobstep.id.hex,
task_id=new_jobstep.id.hex,
parent_task_id=new_jobphase.job.id.hex,
)
return results
def expand_command(self, command, expander, data):
jobstep = command.jobstep
phase_name = data.get('phase')
if not phase_name:
phase_count = db.session.query(func.count(), ).filter(
JobPhase.job_id == jobstep.job_id, ).scalar()
phase_name = 'Phase #{}'.format(phase_count)
jobstep.data['expanded'] = True
db.session.add(jobstep)
new_jobphase = JobPhase(
job_id=jobstep.job_id,
project_id=jobstep.project_id,
label=phase_name,
status=Status.queued,
)
db.session.add(new_jobphase)
_, buildstep = JobPlan.get_build_step_for_job(jobstep.job_id)
results = []
for future_jobstep in expander.expand(
max_executors=jobstep.data['max_executors']):
new_jobstep = buildstep.expand_jobstep(jobstep, new_jobphase,
future_jobstep)
results.append(new_jobstep)
db.session.flush()
for new_jobstep in results:
sync_job_step.delay_if_needed(
step_id=new_jobstep.id.hex,
task_id=new_jobstep.id.hex,
parent_task_id=new_jobphase.job.id.hex,
)
return results
def _expand_jobs(self, step, artifact):
builder = self.get_builder()
artifact_data = builder.fetch_artifact(step, artifact.data)
phase_config = artifact_data.json()
assert phase_config['cmd']
assert '{test_names}' in phase_config['cmd']
assert phase_config['tests']
test_stats, avg_test_time = self.get_test_stats(step.project)
groups = self._shard_tests(phase_config['tests'], self.num_shards, test_stats, avg_test_time)
phase, created = get_or_create(JobPhase, where={
'job': step.job,
'project': step.project,
'label': phase_config.get('phase') or 'Test',
}, defaults={
'status': Status.queued,
})
db.session.commit()
assert len(groups) == self.num_shards
# Create all of the job steps and commit them together.
steps = [self._create_jobstep(phase, phase_config, weight, test_list)
for weight, test_list in groups]
db.session.commit()
# Now that that database transaction is done, we'll do the slow work of
# creating jenkins builds.
for step in steps:
self._create_jenkins_build(phase, step)
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)
def _sync_phased_results(self, step, artifacts):
# due to the limitations of Jenkins and our requirement to have more
# insight into the actual steps a build process takes and unfortunately
# the best way to do this is to rewrite history within Changes
job = step.job
is_diff = not job.source.is_commit()
project = step.project
artifacts_by_name = dict((a["fileName"], a) for a in artifacts)
pending_artifacts = set(artifacts_by_name.keys())
phase_steps = set()
phase_step_data = {
"job_name": step.data["job_name"],
"build_no": step.data["build_no"],
"generated": True,
# TODO: _pick_master here seems very suspicious.
"master": self._pick_master(step.data["job_name"], is_diff),
}
phases = set()
# fetch each phase and create it immediately (as opposed to async)
for artifact_data in artifacts:
artifact_filename = artifact_data["fileName"]
if not artifact_filename.endswith("phase.json"):
continue
pending_artifacts.remove(artifact_filename)
resp = self.fetch_artifact(step, artifact_data)
phase_data = resp.json()
if phase_data["retcode"]:
result = Result.failed
else:
result = Result.passed
date_started = datetime.utcfromtimestamp(phase_data["startTime"])
date_finished = datetime.utcfromtimestamp(phase_data["endTime"])
jobphase, created = get_or_create(
JobPhase,
where={"job": job, "label": phase_data["name"]},
defaults={
"project": project,
"result": result,
"status": Status.finished,
"date_started": date_started,
"date_finished": date_finished,
},
)
phases.add(jobphase)
jobstep, created = get_or_create(
JobStep,
where={"phase": jobphase, "label": step.label},
defaults={
"job": job,
"node": step.node,
"project": project,
"result": result,
"status": Status.finished,
"date_started": date_started,
"date_finished": date_finished,
"data": phase_step_data,
},
)
sync_job_step.delay_if_needed(task_id=jobstep.id.hex, parent_task_id=job.id.hex, step_id=jobstep.id.hex)
phase_steps.add(jobstep)
# capture the log if available
try:
log_artifact = artifacts_by_name[phase_data["log"]]
except KeyError:
self.logger.warning("Unable to find logfile for phase: %s", phase_data)
else:
pending_artifacts.remove(log_artifact["fileName"])
self._handle_generic_artifact(jobstep=jobstep, artifact=log_artifact, skip_checks=True)
# ideally we don't mark the base step as a failure if any of the phases
# report more correct results
if phases and step.result == Result.failed and any(p.result == Result.failed for p in phases):
step.result = Result.passed
db.session.add(step)
if not pending_artifacts:
return
# Alias to clarify that this is the JobStep that actually ran on a slave.
original_step = step
# all remaining artifacts get bound to the final phase
final_step = sorted(phase_steps, key=lambda x: x.date_finished, reverse=True)[0]
for artifact_name in pending_artifacts:
# Manifest files are associated with the original step so we can validate that the ID is correct.
responsible_step = original_step if ManifestJsonHandler.can_process(artifact_name) else final_step
self._handle_generic_artifact(jobstep=responsible_step, artifact=artifacts_by_name[artifact_name])
def _sync_phased_results(self, step, artifacts):
# due to the limitations of Jenkins and our requirement to have more
# insight into the actual steps a build process takes and unfortunately
# the best way to do this is to rewrite history within Changes
job = step.job
project = step.project
artifacts_by_name = dict((a['fileName'], a) for a in artifacts)
pending_artifacts = set(artifacts_by_name.keys())
phase_steps = set()
phase_step_data = {
'job_name': step.data['job_name'],
'build_no': step.data['build_no'],
'generated': True,
}
phases = set()
# fetch each phase and create it immediately (as opposed to async)
for artifact_data in artifacts:
artifact_filename = artifact_data['fileName']
if not artifact_filename.endswith('phase.json'):
continue
pending_artifacts.remove(artifact_filename)
resp = self.fetch_artifact(step, artifact_data)
phase_data = resp.json()
if phase_data['retcode']:
result = Result.failed
else:
result = Result.passed
date_started = datetime.utcfromtimestamp(phase_data['startTime'])
date_finished = datetime.utcfromtimestamp(phase_data['endTime'])
jobphase, created = get_or_create(JobPhase,
where={
'job': job,
'label': phase_data['name'],
},
defaults={
'project': project,
'result': result,
'status': Status.finished,
'date_started': date_started,
'date_finished':
date_finished,
})
phases.add(jobphase)
jobstep, created = get_or_create(JobStep,
where={
'phase': jobphase,
'label': step.label,
},
defaults={
'job': job,
'node': step.node,
'project': project,
'result': result,
'status': Status.finished,
'date_started': date_started,
'date_finished':
date_finished,
'data': phase_step_data,
})
sync_job_step.delay_if_needed(
task_id=jobstep.id.hex,
parent_task_id=job.id.hex,
step_id=jobstep.id.hex,
)
phase_steps.add(jobstep)
# capture the log if available
try:
log_artifact = artifacts_by_name[phase_data['log']]
except KeyError:
self.logger.warning('Unable to find logfile for phase: %s',
phase_data)
else:
pending_artifacts.remove(log_artifact['fileName'])
self._handle_generic_artifact(
jobstep=jobstep,
artifact=log_artifact,
skip_checks=True,
)
# ideally we don't mark the base step as a failure if any of the phases
# report more correct results
if phases and step.result == Result.failed and any(
p.result == Result.failed for p in phases):
step.result = Result.passed
db.session.add(step)
if not pending_artifacts:
return
# all remaining artifacts get bound to the final phase
final_step = sorted(phase_steps,
key=lambda x: x.date_finished,
reverse=True)[0]
for artifact_name in pending_artifacts:
self._handle_generic_artifact(
jobstep=final_step,
artifact=artifacts_by_name[artifact_name],
)
def _sync_phased_results(self, step, artifacts):
# due to the limitations of Jenkins and our requirement to have more
# insight into the actual steps a build process takes and unfortunately
# the best way to do this is to rewrite history within Changes
job = step.job
project = step.project
artifacts_by_name = dict(
(a['fileName'], a)
for a in artifacts
)
pending_artifacts = set(artifacts_by_name.keys())
phase_steps = set()
phase_step_data = {
'job_name': step.data['job_name'],
'build_no': step.data['build_no'],
'generated': True,
'master': self._pick_master(step.data['job_name']),
}
phases = set()
# fetch each phase and create it immediately (as opposed to async)
for artifact_data in artifacts:
artifact_filename = artifact_data['fileName']
if not artifact_filename.endswith('phase.json'):
continue
pending_artifacts.remove(artifact_filename)
resp = self.fetch_artifact(step, artifact_data)
phase_data = resp.json()
if phase_data['retcode']:
result = Result.failed
else:
result = Result.passed
date_started = datetime.utcfromtimestamp(phase_data['startTime'])
date_finished = datetime.utcfromtimestamp(phase_data['endTime'])
jobphase, created = get_or_create(JobPhase, where={
'job': job,
'label': phase_data['name'],
}, defaults={
'project': project,
'result': result,
'status': Status.finished,
'date_started': date_started,
'date_finished': date_finished,
})
phases.add(jobphase)
jobstep, created = get_or_create(JobStep, where={
'phase': jobphase,
'label': step.label,
}, defaults={
'job': job,
'node': step.node,
'project': project,
'result': result,
'status': Status.finished,
'date_started': date_started,
'date_finished': date_finished,
'data': phase_step_data,
})
sync_job_step.delay_if_needed(
task_id=jobstep.id.hex,
parent_task_id=job.id.hex,
step_id=jobstep.id.hex,
)
phase_steps.add(jobstep)
# capture the log if available
try:
log_artifact = artifacts_by_name[phase_data['log']]
except KeyError:
self.logger.warning('Unable to find logfile for phase: %s', phase_data)
else:
pending_artifacts.remove(log_artifact['fileName'])
self._handle_generic_artifact(
jobstep=jobstep,
artifact=log_artifact,
skip_checks=True,
)
# ideally we don't mark the base step as a failure if any of the phases
# report more correct results
if phases and step.result == Result.failed and any(p.result == Result.failed for p in phases):
step.result = Result.passed
db.session.add(step)
if not pending_artifacts:
return
# all remaining artifacts get bound to the final phase
final_step = sorted(phase_steps, key=lambda x: x.date_finished, reverse=True)[0]
for artifact_name in pending_artifacts:
self._handle_generic_artifact(
jobstep=final_step,
artifact=artifacts_by_name[artifact_name],
)
def expand_jobs(self, step, phase_config):
"""
Creates and runs JobSteps for a set of tests, based on a phase config.
This phase config comes from a tests.json file that the collection
jobstep should generate. This method is then called by the TestsJsonHandler.
"""
if not phase_config.get('cmd'):
raise ArtifactParseError('No cmd attribute')
if '{test_names}' not in phase_config['cmd']:
raise ArtifactParseError('No {test_names} in cmd')
if 'tests' not in phase_config:
raise ArtifactParseError('No tests attribute')
num_tests = len(phase_config['tests'])
test_stats, avg_test_time = TestsExpander.get_test_stats(
self.get_test_stats_from() or step.project.slug)
phase, _ = get_or_create(JobPhase,
where={
'job': step.job,
'project': step.project,
'label': phase_config.get('phase')
or 'Test',
},
defaults={'status': Status.queued})
db.session.commit()
# If there are no tests to run, the phase is done.
if num_tests == 0:
phase.status = Status.finished
phase.result = Result.passed
db.session.add(phase)
db.session.commit()
return
# Check for whether a previous run of this task has already
# created JobSteps for us, since doing it again would create a
# double-sharded build.
steps = JobStep.query.filter_by(phase_id=phase.id,
replacement_id=None).all()
if steps:
step_shard_counts = [s.data.get('shard_count', 1) for s in steps]
if len(set(step_shard_counts)) != 1:
raise Exception("Mixed shard counts in phase!")
elif len(steps) != step_shard_counts[0]:
raise Exception("Shard count incorrect")
else:
# Create all of the job steps and commit them together.
groups = shard(
phase_config['tests'],
self.max_shards,
test_stats,
avg_test_time,
normalize_object_name=TestsExpander._normalize_test_segments)
steps = [
self._create_jobstep(phase,
phase_config['cmd'],
phase_config.get('path', ''),
weight,
test_list,
len(groups),
cluster=step.cluster)
for weight, test_list in groups
]
if len(steps) != len(groups):
raise Exception("Didn't create correct number of shards")
db.session.commit()
# Now that the database transaction is done, we'll do the slow work of
# creating jenkins builds.
for step in steps:
self._create_jenkins_build(step)
sync_job_step.delay_if_needed(
step_id=step.id.hex,
task_id=step.id.hex,
parent_task_id=phase.job.id.hex,
)