def create_job(job_id):
job = Job.query.get(job_id)
if not job:
return
job_plan = JobPlan.query.options(subqueryload_all('plan.steps')).filter(
JobPlan.job_id == job.id, ).join(Plan).first()
try:
if not job_plan:
raise UnrecoverableException(
'Got create_job task without job plan: %s' % (job_id, ))
try:
step = job_plan.plan.steps[0]
except IndexError:
raise UnrecoverableException('Missing steps for plan')
implementation = step.get_implementation()
implementation.execute(job=job)
except UnrecoverableException:
job.status = Status.finished
job.result = Result.aborted
current_app.logger.exception('Unrecoverable exception creating %s',
job_id)
return
sync_job.delay(
job_id=job.id.hex,
task_id=job.id.hex,
parent_task_id=job.build_id.hex,
)
def _get_jenkins_job(self, step):
try:
job_name = step.data['job_name']
build_no = step.data['build_no']
except KeyError:
raise UnrecoverableException('Missing Jenkins job information')
try:
return self._get_json_response(
step.data['master'],
'/job/{}/{}'.format(job_name, build_no),
)
except NotFound:
raise UnrecoverableException('Unable to find job in Jenkins')
def get_build_step(job_id):
job_plan = JobPlan.query.options(subqueryload_all('plan.steps')).filter(
JobPlan.job_id == job_id, ).join(Plan).first()
if not job_plan:
raise UnrecoverableException('Missing job plan for job: %s' %
(job_id, ))
try:
step = job_plan.plan.steps[0]
except IndexError:
raise UnrecoverableException('Missing steps for plan: %s' %
(job_plan.plan.id))
implementation = step.get_implementation()
return implementation
def create_job_from_params(self, target_id, params, job_name=None):
if job_name is None:
job_name = self.job_name
if not job_name:
raise UnrecoverableException(
'Missing Jenkins project configuration')
json_data = {'parameter': params}
# TODO: Jenkins will return a 302 if it cannot queue the job which I
# believe implies that there is already a job with the same parameters
# queued.
self._get_response('/job/{}/build'.format(job_name),
method='POST',
data={
'json': json.dumps(json_data),
})
# we retry for a period of time as Jenkins doesn't have strong consistency
# guarantees and the job may not show up right away
t = time.time() + 5
job_data = None
while time.time() < t:
job_data = self._find_job(job_name, target_id)
if job_data:
break
time.sleep(0.3)
if job_data is None:
raise Exception('Unable to find matching job after creation. GLHF')
return job_data
def create_job(self, job):
project_id = self.project_id
if not project_id:
raise UnrecoverableException(
'Missing Koality project configuration')
req_kwargs = {}
if job.build.source.patch:
req_kwargs['files'] = {
'patch': job.build.source.patch.diff,
}
response = self._get_response(
'POST',
'{base_uri}/api/v/0/repositories/{project_id}/changes'.format(
base_uri=self.base_url,
project_id=project_id,
),
data={
# XXX: passing an empty value for email causes Koality to not
# send out an email notification
'emailTo': '',
'sha': job.build.source.revision_sha,
},
**req_kwargs)
job.data = {
'project_id': project_id,
'change_id': response['changeId'],
}
db.session.add(job)
def _sync_step_from_active(self, step):
try:
job_name = step.data['job_name']
build_no = step.data['build_no']
except KeyError:
raise UnrecoverableException('Missing Jenkins job information')
try:
item = self._get_response(
step.data['master'],
'/job/{}/{}'.format(job_name, build_no),
)
except NotFound:
raise UnrecoverableException('Unable to find job in Jenkins')
if not step.data.get('uri'):
step.data['uri'] = item['url']
# TODO(dcramer): we're doing a lot of work here when we might
# not need to due to it being sync'd previously
node = self._get_node(step.data['master'], item['builtOn'])
step.node = node
step.date_started = datetime.utcfromtimestamp(
item['timestamp'] / 1000)
if item['building']:
step.status = Status.in_progress
else:
step.status = Status.finished
step.result = RESULT_MAP[item['result']]
step.date_finished = datetime.utcfromtimestamp(
(item['timestamp'] + item['duration']) / 1000)
if step.status == Status.finished:
self._sync_results(step, item)
if db.session.is_modified(step):
db.session.add(step)
db.session.commit()
def cancel_step(self, step):
if step.data.get('build_no'):
url = '/job/{}/{}/stop/'.format(
step.data['job_name'], step.data['build_no'])
else:
url = '/queue/cancelItem?id={}'.format(step.data['item_id'])
try:
self._get_raw_response(url)
except NotFound:
raise UnrecoverableException('Unable to find job in Jenkins')
step.status = Status.finished
step.result = Result.aborted
db.session.add(step)
def sync_job(job_id):
job = Job.query.get(job_id)
if not job:
return
if job.status == Status.finished:
return
# TODO(dcramer): we make an assumption that there is a single step
job_plan = JobPlan.query.options(subqueryload_all('plan.steps')).filter(
JobPlan.job_id == job.id, ).join(Plan).first()
try:
if not job_plan:
raise UnrecoverableException(
'Got sync_job task without job plan: %s' % (job.id, ))
try:
step = job_plan.plan.steps[0]
except IndexError:
raise UnrecoverableException('Missing steps for plan')
implementation = step.get_implementation()
implementation.update(job=job)
except UnrecoverableException:
job.status = Status.finished
job.result = Result.aborted
current_app.logger.exception('Unrecoverable exception syncing %s',
job.id)
is_finished = sync_job.verify_all_children() == Status.finished
if is_finished:
job.status = Status.finished
all_phases = list(job.phases)
job.date_started = safe_agg(min, (j.date_started
for j in all_phases if j.date_started))
if is_finished:
job.date_finished = safe_agg(max,
(j.date_finished
for j in all_phases if j.date_finished))
else:
job.date_finished = None
if job.date_started and job.date_finished:
job.duration = int(
(job.date_finished - job.date_started).total_seconds() * 1000)
else:
job.duration = None
# if any phases are marked as failing, fail the build
if any(j.result is Result.failed for j in all_phases):
job.result = Result.failed
# if any test cases were marked as failing, fail the build
elif TestCase.query.filter(TestCase.result == Result.failed,
TestCase.job_id == job.id).first():
job.result = Result.failed
# if we've finished all phases, use the best result available
elif is_finished:
job.result = safe_agg(max, (j.result for j in all_phases),
Result.unknown)
else:
job.result = Result.unknown
if is_finished:
job.status = Status.finished
elif any(j.status is Status.in_progress for j in all_phases):
job.status = Status.in_progress
else:
job.status = Status.queued
if db.session.is_modified(job):
job.date_modified = datetime.utcnow()
db.session.add(job)
db.session.commit()
publish_job_update(job)
if not is_finished:
raise sync_job.NotFinished
_record_tests_missing(job)
queue.delay('notify_job_finished', kwargs={
'job_id': job.id.hex,
})
if job_plan:
queue.delay('update_project_plan_stats',
kwargs={
'project_id': job.project_id.hex,
'plan_id': job_plan.plan_id.hex,
},
countdown=1)
def _sync_step_from_active(self, step):
try:
job_name = step.data['job_name']
build_no = step.data['build_no']
except KeyError:
raise UnrecoverableException('Missing Jenkins job information')
try:
item = self._get_response('/job/{}/{}'.format(
job_name, build_no))
except NotFound:
raise UnrecoverableException('Unable to find job in Jenkins')
# TODO(dcramer): we're doing a lot of work here when we might
# not need to due to it being sync'd previously
node, _ = get_or_create(Node, where={
'label': item['builtOn'],
})
step.node = node
step.label = item['fullDisplayName']
step.date_started = datetime.utcfromtimestamp(
item['timestamp'] / 1000)
if item['building']:
step.status = Status.in_progress
else:
step.status = Status.finished
step.result = RESULT_MAP[item['result']]
# values['duration'] = item['duration'] or None
step.date_finished = datetime.utcfromtimestamp(
(item['timestamp'] + item['duration']) / 1000)
# step.data.update({
# 'backend': {
# 'uri': item['url'],
# 'label': item['fullDisplayName'],
# }
# })
db.session.add(step)
db.session.commit()
# TODO(dcramer): we shoudl abstract this into a sync_phase
phase = step.phase
if not phase.date_started:
phase.date_started = safe_agg(
min, (s.date_started for s in phase.steps), step.date_started)
db.session.add(phase)
if phase.status != step.status:
phase.status = step.status
db.session.add(phase)
if step.status == Status.finished:
phase.status = Status.finished
phase.date_finished = safe_agg(
max, (s.date_finished for s in phase.steps), step.date_finished)
if any(s.result is Result.failed for s in phase.steps):
phase.result = Result.failed
else:
phase.result = safe_agg(
max, (s.result for s in phase.steps), Result.unknown)
db.session.add(phase)
db.session.commit()
if step.status != Status.finished:
return
# sync artifacts
for artifact in item.get('artifacts', ()):
artifact, created = get_or_create(Artifact, where={
'step': step,
'name': artifact['fileName'],
}, defaults={
'project': step.project,
'job': step.job,
'data': artifact,
})
db.session.commit()
sync_artifact.delay_if_needed(
artifact_id=artifact.id.hex,
task_id=artifact.id.hex,
parent_task_id=step.id.hex,
)
# sync test results
try:
with db.session.begin_nested():
self._sync_test_results(
step=step,
job_name=job_name,
build_no=build_no,
)
except Exception:
self.logger.exception(
'Failed to sync test results for %s #%s', job_name, build_no)
else:
db.session.commit()
# sync console log
try:
result = True
while result:
result = self._sync_log(
jobstep=step,
name=step.label,
job_name=job_name,
build_no=build_no,
)
except Exception:
db.session.rollback()
current_app.logger.exception(
'Unable to sync console log for job step %r',
step.id.hex)