def handle_add(queue_name):
"""Adds a task to a queue."""
source = request.form.get('source', request.remote_addr, type=str)
try:
task_id = work_queue.add(queue_name,
payload=request.form.get('payload', type=str),
content_type=request.form.get('content_type',
type=str),
source=source,
task_id=request.form.get('task_id', type=str))
except work_queue.Error, e:
return utils.jsonify_error(e)
def report_run(build):
"""Reports data for a run for a release candidate."""
release, run = _get_or_create_run(build)
current_url = request.form.get('url', type=str)
current_image = request.form.get('image', type=str)
current_log = request.form.get('log', type=str)
current_config = request.form.get('config', type=str)
ref_url = request.form.get('ref_url', type=str)
ref_image = request.form.get('ref_image', type=str)
ref_log = request.form.get('ref_log', type=str)
ref_config = request.form.get('ref_config', type=str)
diff_success = request.form.get('diff_success', type=str)
diff_image = request.form.get('diff_image', type=str)
diff_log = request.form.get('diff_log', type=str)
if current_url:
run.url = current_url
if current_image:
run.image = current_image
if current_log:
run.log = current_log
if current_config:
run.config = current_config
if current_image or current_log or current_config:
logging.info('Saved run data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, url=%r, '
'image=%r, log=%r, config=%r',
build.id, release.name, release.number, run.name,
run.url, run.image, run.log, run.config)
if ref_url:
run.ref_url = ref_url
if ref_image:
run.ref_image = ref_image
if ref_log:
run.ref_log = ref_log
if ref_config:
run.ref_config = ref_config
if ref_image or ref_log or ref_config:
logging.info('Saved reference data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, ref_url=%r, '
'ref_image=%r, ref_log=%r, ref_config=%r',
build.id, release.name, release.number, run.name,
run.ref_url, run.ref_image, run.ref_log, run.ref_config)
if diff_image:
run.diff_image = diff_image
if diff_log:
run.diff_log = diff_log
if diff_image or diff_log:
logging.info('Saved pdiff: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, '
'diff_image=%r, diff_log=%r',
build.id, release.name, release.number, run.name,
run.diff_image, run.diff_log)
if run.image and run.diff_image:
run.status = models.Run.DIFF_FOUND
elif run.image and run.ref_image and not run.diff_log:
run.status = models.Run.NEEDS_DIFF
elif run.image and run.ref_image and diff_success:
run.status = models.Run.DIFF_NOT_FOUND
elif run.image and not run.ref_config:
run.status = models.Run.NO_DIFF_NEEDED
# TODO: Consider adding another status of "diff failed" or capture
# failed for situations where it couldn't finish.
# TODO: Verify the build has access to both the current_image and
# the reference_sha1sum so they can't make a diff from a black image
# and still see private data in the diff image.
if run.status == models.Run.NEEDS_DIFF:
task_id = '%s:%s:%s' % (run.id, run.image, run.ref_image)
logging.info('Enqueuing pdiff task=%r', task_id)
work_queue.add(
constants.PDIFF_QUEUE_NAME,
payload=dict(
build_id=build.id,
release_name=release.name,
release_number=release.number,
run_name=run.name,
run_sha1sum=run.image,
reference_sha1sum=run.ref_image,
),
source='report_run',
task_id=task_id)
# Flush the run so querying for Runs in _check_release_done_processing
# will be find the new run too.
db.session.add(run)
_check_release_done_processing(release)
db.session.commit()
logging.info('Updated run: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, status=%r',
build.id, release.name, release.number, run.name, run.status)
return flask.jsonify(success=True)
db.session.flush()
suffix = ''
if baseline:
suffix = ':baseline'
task_id = '%s:%s%s' % (run.id, hashlib.sha1(url).hexdigest(), suffix)
logging.info('Enqueueing capture task=%r, baseline=%r', task_id, baseline)
work_queue.add(
constants.CAPTURE_QUEUE_NAME,
payload=dict(
build_id=build.id,
release_name=release.name,
release_number=release.number,
run_name=run.name,
url=url,
config_sha1sum=config_artifact.id,
baseline=baseline,
),
source='request_run',
task_id=task_id)
# Set the URL and config early to indicate to report_run that there is
# still data pending even if 'image' and 'ref_image' are unset.
if baseline:
run.ref_url = url
run.ref_config = config_artifact.id
else:
run.url = url
run.config = config_artifact.id
def report_run():
"""Reports data for a run for a release candidate."""
build = g.build
release, run = _get_or_create_run(build)
db.session.refresh(run, lockmode='update')
current_url = request.form.get('url', type=str)
current_image = request.form.get('image', type=str)
current_log = request.form.get('log', type=str)
current_config = request.form.get('config', type=str)
ref_url = request.form.get('ref_url', type=str)
ref_image = request.form.get('ref_image', type=str)
ref_log = request.form.get('ref_log', type=str)
ref_config = request.form.get('ref_config', type=str)
diff_failed = request.form.get('diff_failed', type=str)
diff_image = request.form.get('diff_image', type=str)
diff_log = request.form.get('diff_log', type=str)
distortion = request.form.get('distortion', default=None, type=float)
run_failed = request.form.get('run_failed', type=str)
if current_url:
run.url = current_url
if current_image:
run.image = current_image
if current_log:
run.log = current_log
if current_config:
run.config = current_config
if current_image or current_log or current_config:
logging.info('Saving run data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, url=%r, '
'image=%r, log=%r, config=%r, run_failed=%r',
build.id, release.name, release.number, run.name,
run.url, run.image, run.log, run.config, run_failed)
if ref_url:
run.ref_url = ref_url
if ref_image:
run.ref_image = ref_image
if ref_log:
run.ref_log = ref_log
if ref_config:
run.ref_config = ref_config
if ref_image or ref_log or ref_config:
logging.info('Saved reference data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, ref_url=%r, '
'ref_image=%r, ref_log=%r, ref_config=%r',
build.id, release.name, release.number, run.name,
run.ref_url, run.ref_image, run.ref_log, run.ref_config)
if diff_image:
run.diff_image = diff_image
if diff_log:
run.diff_log = diff_log
if distortion:
run.distortion = distortion
if diff_image or diff_log:
logging.info('Saved pdiff: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, diff_image=%r, '
'diff_log=%r, diff_failed=%r, distortion=%r',
build.id, release.name, release.number, run.name,
run.diff_image, run.diff_log, diff_failed, distortion)
if run.image and run.diff_image:
run.status = models.Run.DIFF_FOUND
elif run.image and run.ref_image and not run.diff_log:
run.status = models.Run.NEEDS_DIFF
elif run.image and run.ref_image and not diff_failed:
run.status = models.Run.DIFF_NOT_FOUND
elif run.image and not run.ref_config:
run.status = models.Run.NO_DIFF_NEEDED
elif run_failed or diff_failed:
run.status = models.Run.FAILED
else:
# NOTE: Intentionally do not transition state here in the default case.
# We allow multiple background workers to be writing to the same Run in
# parallel updating its various properties.
pass
# TODO: Verify the build has access to both the current_image and
# the reference_sha1sum so they can't make a diff from a black image
# and still see private data in the diff image.
if run.status == models.Run.NEEDS_DIFF:
task_id = '%s:%s:%s' % (run.id, run.image, run.ref_image)
logging.info('Enqueuing pdiff task=%r', task_id)
work_queue.add(
constants.PDIFF_QUEUE_NAME,
payload=dict(
build_id=build.id,
release_name=release.name,
release_number=release.number,
run_name=run.name,
run_sha1sum=run.image,
reference_sha1sum=run.ref_image,
),
build_id=build.id,
release_id=release.id,
run_id=run.id,
source='report_run',
task_id=task_id)
# Flush the run so querying for Runs in _check_release_done_processing
# will be find the new run too and we won't deadlock.
db.session.add(run)
db.session.flush()
_check_release_done_processing(release)
db.session.commit()
signals.run_updated_via_api.send(
app, build=build, release=release, run=run)
logging.info('Updated run: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, status=%r',
build.id, release.name, release.number, run.name, run.status)
return flask.jsonify(success=True)
suffix = ''
if baseline:
suffix = ':baseline'
task_id = '%s:%s%s' % (run.id, hashlib.sha1(url).hexdigest(), suffix)
logging.info('Enqueueing capture task=%r, baseline=%r', task_id, baseline)
work_queue.add(
constants.CAPTURE_QUEUE_NAME,
payload=dict(
build_id=build.id,
release_name=release.name,
release_number=release.number,
run_name=run.name,
url=url,
config_sha1sum=config_artifact.id,
baseline=baseline,
),
build_id=build.id,
release_id=release.id,
run_id=run.id,
source='request_run',
task_id=task_id)
# Set the URL and config early to indicate to report_run that there is
# still data pending even if 'image' and 'ref_image' are unset.
if baseline:
run.ref_url = url
run.ref_config = config_artifact.id
else:
run.url = url
def report_run():
"""Reports a new run for a release candidate."""
build_id, release_name, release_number = _get_release_params()
run_name = request.form.get('run_name', type=str)
utils.jsonify_assert(run_name, 'run_name required')
release = (
models.Release.query
.filter_by(build_id=build_id, name=release_name, number=release_number)
.first())
utils.jsonify_assert(release, 'release does not exist')
# TODO: Make sure requesting user is owner of the build_id
current_image = request.form.get('image', type=str)
utils.jsonify_assert(current_image, 'image must be supplied')
current_log = request.form.get('log', type=str)
current_config = request.form.get('config', type=str)
no_diff = request.form.get('no_diff')
diff_image = request.form.get('diff_image', type=str)
diff_log = request.form.get('diff_log', type=str)
needs_diff = not (no_diff or diff_image or diff_log)
# Find the previous corresponding run and automatically connect it.
last_good_release = (
models.Release.query
.filter_by(
build_id=build_id,
status=models.Release.GOOD)
.order_by(models.Release.created.desc())
.first())
previous_id = None
last_image = None
if last_good_release:
logging.debug('Found last good release for: build_id=%r, '
'release_name=%r, release_number=%d, '
'last_good_release_id=%d',
build_id, release_name, release_number,
last_good_release.id)
last_good_run = (
models.Run.query
.filter_by(release_id=last_good_release.id, name=run_name)
.first())
if last_good_run:
logging.debug('Found last good run for: build_id=%r, '
'release_name=%r, release_number=%d, '
'last_good_release_id=%d, last_good_run_id=%r, '
'last_good_image=%r',
build_id, release_name, release_number,
last_good_release.id, last_good_run.id,
last_good_run.image)
previous_id = last_good_run.id
last_image = last_good_run.image
run = models.Run(
name=run_name,
release_id=release.id,
image=current_image,
log=current_log,
config=current_config,
previous_id=previous_id,
needs_diff=bool(needs_diff and last_image),
diff_image=diff_image,
diff_log=diff_log)
db.session.add(run)
db.session.flush()
# Schedule pdiff if there isn't already an image.
if needs_diff and last_image:
# TODO: Move this queue name to a flag.
work_queue.add('run-pdiff', dict(
build_id=build_id,
release_name=release_name,
release_number=release_number,
run_name=run_name,
reference_sha1sum=current_image,
run_sha1sum=last_image,
))
db.session.commit()
logging.info('Created run: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r',
build_id, release_name, release_number, run_name)
return flask.jsonify(success=True)
# Rewrite the config JSON to include the URL specified in this request.
# Blindly overwrite anything that was there.
config_dict['targetUrl'] = current_url
config_data = json.dumps(config_dict)
config_artifact = _save_artifact(build, config_data, 'application/json')
db.session.add(config_artifact)
db.session.flush()
current_run.url = current_url
current_run.config = config_artifact.id
work_queue.add(constants.CAPTURE_QUEUE_NAME, dict(
build_id=build.id,
release_name=current_release.name,
release_number=current_release.number,
run_name=current_run.name,
url=current_run.url,
config_sha1sum=current_run.config,
))
db.session.add(current_run)
db.session.commit()
return flask.jsonify(
success=True,
build_id=build.id,
release_name=current_release.name,
release_number=current_release.number,
run_name=current_run.name,
url=current_run.url,
config=current_run.config)
