def handle_add(queue_name): """Adds a task to a queue.""" source = request.form.get('source', request.remote_addr, type=str) try: task_id = work_queue.add(queue_name, payload=request.form.get('payload', type=str), content_type=request.form.get('content_type', type=str), source=source, task_id=request.form.get('task_id', type=str)) except work_queue.Error, e: return utils.jsonify_error(e)
def report_run(build): """Reports data for a run for a release candidate.""" release, run = _get_or_create_run(build) current_url = request.form.get('url', type=str) current_image = request.form.get('image', type=str) current_log = request.form.get('log', type=str) current_config = request.form.get('config', type=str) ref_url = request.form.get('ref_url', type=str) ref_image = request.form.get('ref_image', type=str) ref_log = request.form.get('ref_log', type=str) ref_config = request.form.get('ref_config', type=str) diff_success = request.form.get('diff_success', type=str) diff_image = request.form.get('diff_image', type=str) diff_log = request.form.get('diff_log', type=str) if current_url: run.url = current_url if current_image: run.image = current_image if current_log: run.log = current_log if current_config: run.config = current_config if current_image or current_log or current_config: logging.info('Saved run data: build_id=%r, release_name=%r, ' 'release_number=%d, run_name=%r, url=%r, ' 'image=%r, log=%r, config=%r', build.id, release.name, release.number, run.name, run.url, run.image, run.log, run.config) if ref_url: run.ref_url = ref_url if ref_image: run.ref_image = ref_image if ref_log: run.ref_log = ref_log if ref_config: run.ref_config = ref_config if ref_image or ref_log or ref_config: logging.info('Saved reference data: build_id=%r, release_name=%r, ' 'release_number=%d, run_name=%r, ref_url=%r, ' 'ref_image=%r, ref_log=%r, ref_config=%r', build.id, release.name, release.number, run.name, run.ref_url, run.ref_image, run.ref_log, run.ref_config) if diff_image: run.diff_image = diff_image if diff_log: run.diff_log = diff_log if diff_image or diff_log: logging.info('Saved pdiff: build_id=%r, release_name=%r, ' 'release_number=%d, run_name=%r, ' 'diff_image=%r, diff_log=%r', build.id, release.name, release.number, run.name, run.diff_image, run.diff_log) if run.image and run.diff_image: run.status = models.Run.DIFF_FOUND elif run.image and run.ref_image and not run.diff_log: run.status = models.Run.NEEDS_DIFF elif run.image and run.ref_image and diff_success: run.status = models.Run.DIFF_NOT_FOUND elif run.image and not run.ref_config: run.status = models.Run.NO_DIFF_NEEDED # TODO: Consider adding another status of "diff failed" or capture # failed for situations where it couldn't finish. # TODO: Verify the build has access to both the current_image and # the reference_sha1sum so they can't make a diff from a black image # and still see private data in the diff image. if run.status == models.Run.NEEDS_DIFF: task_id = '%s:%s:%s' % (run.id, run.image, run.ref_image) logging.info('Enqueuing pdiff task=%r', task_id) work_queue.add( constants.PDIFF_QUEUE_NAME, payload=dict( build_id=build.id, release_name=release.name, release_number=release.number, run_name=run.name, run_sha1sum=run.image, reference_sha1sum=run.ref_image, ), source='report_run', task_id=task_id) # Flush the run so querying for Runs in _check_release_done_processing # will be find the new run too. db.session.add(run) _check_release_done_processing(release) db.session.commit() logging.info('Updated run: build_id=%r, release_name=%r, ' 'release_number=%d, run_name=%r, status=%r', build.id, release.name, release.number, run.name, run.status) return flask.jsonify(success=True)
db.session.flush() suffix = '' if baseline: suffix = ':baseline' task_id = '%s:%s%s' % (run.id, hashlib.sha1(url).hexdigest(), suffix) logging.info('Enqueueing capture task=%r, baseline=%r', task_id, baseline) work_queue.add( constants.CAPTURE_QUEUE_NAME, payload=dict( build_id=build.id, release_name=release.name, release_number=release.number, run_name=run.name, url=url, config_sha1sum=config_artifact.id, baseline=baseline, ), source='request_run', task_id=task_id) # Set the URL and config early to indicate to report_run that there is # still data pending even if 'image' and 'ref_image' are unset. if baseline: run.ref_url = url run.ref_config = config_artifact.id else: run.url = url run.config = config_artifact.id
def report_run(): """Reports data for a run for a release candidate.""" build = g.build release, run = _get_or_create_run(build) db.session.refresh(run, lockmode='update') current_url = request.form.get('url', type=str) current_image = request.form.get('image', type=str) current_log = request.form.get('log', type=str) current_config = request.form.get('config', type=str) ref_url = request.form.get('ref_url', type=str) ref_image = request.form.get('ref_image', type=str) ref_log = request.form.get('ref_log', type=str) ref_config = request.form.get('ref_config', type=str) diff_failed = request.form.get('diff_failed', type=str) diff_image = request.form.get('diff_image', type=str) diff_log = request.form.get('diff_log', type=str) distortion = request.form.get('distortion', default=None, type=float) run_failed = request.form.get('run_failed', type=str) if current_url: run.url = current_url if current_image: run.image = current_image if current_log: run.log = current_log if current_config: run.config = current_config if current_image or current_log or current_config: logging.info('Saving run data: build_id=%r, release_name=%r, ' 'release_number=%d, run_name=%r, url=%r, ' 'image=%r, log=%r, config=%r, run_failed=%r', build.id, release.name, release.number, run.name, run.url, run.image, run.log, run.config, run_failed) if ref_url: run.ref_url = ref_url if ref_image: run.ref_image = ref_image if ref_log: run.ref_log = ref_log if ref_config: run.ref_config = ref_config if ref_image or ref_log or ref_config: logging.info('Saved reference data: build_id=%r, release_name=%r, ' 'release_number=%d, run_name=%r, ref_url=%r, ' 'ref_image=%r, ref_log=%r, ref_config=%r', build.id, release.name, release.number, run.name, run.ref_url, run.ref_image, run.ref_log, run.ref_config) if diff_image: run.diff_image = diff_image if diff_log: run.diff_log = diff_log if distortion: run.distortion = distortion if diff_image or diff_log: logging.info('Saved pdiff: build_id=%r, release_name=%r, ' 'release_number=%d, run_name=%r, diff_image=%r, ' 'diff_log=%r, diff_failed=%r, distortion=%r', build.id, release.name, release.number, run.name, run.diff_image, run.diff_log, diff_failed, distortion) if run.image and run.diff_image: run.status = models.Run.DIFF_FOUND elif run.image and run.ref_image and not run.diff_log: run.status = models.Run.NEEDS_DIFF elif run.image and run.ref_image and not diff_failed: run.status = models.Run.DIFF_NOT_FOUND elif run.image and not run.ref_config: run.status = models.Run.NO_DIFF_NEEDED elif run_failed or diff_failed: run.status = models.Run.FAILED else: # NOTE: Intentionally do not transition state here in the default case. # We allow multiple background workers to be writing to the same Run in # parallel updating its various properties. pass # TODO: Verify the build has access to both the current_image and # the reference_sha1sum so they can't make a diff from a black image # and still see private data in the diff image. if run.status == models.Run.NEEDS_DIFF: task_id = '%s:%s:%s' % (run.id, run.image, run.ref_image) logging.info('Enqueuing pdiff task=%r', task_id) work_queue.add( constants.PDIFF_QUEUE_NAME, payload=dict( build_id=build.id, release_name=release.name, release_number=release.number, run_name=run.name, run_sha1sum=run.image, reference_sha1sum=run.ref_image, ), build_id=build.id, release_id=release.id, run_id=run.id, source='report_run', task_id=task_id) # Flush the run so querying for Runs in _check_release_done_processing # will be find the new run too and we won't deadlock. db.session.add(run) db.session.flush() _check_release_done_processing(release) db.session.commit() signals.run_updated_via_api.send( app, build=build, release=release, run=run) logging.info('Updated run: build_id=%r, release_name=%r, ' 'release_number=%d, run_name=%r, status=%r', build.id, release.name, release.number, run.name, run.status) return flask.jsonify(success=True)
suffix = '' if baseline: suffix = ':baseline' task_id = '%s:%s%s' % (run.id, hashlib.sha1(url).hexdigest(), suffix) logging.info('Enqueueing capture task=%r, baseline=%r', task_id, baseline) work_queue.add( constants.CAPTURE_QUEUE_NAME, payload=dict( build_id=build.id, release_name=release.name, release_number=release.number, run_name=run.name, url=url, config_sha1sum=config_artifact.id, baseline=baseline, ), build_id=build.id, release_id=release.id, run_id=run.id, source='request_run', task_id=task_id) # Set the URL and config early to indicate to report_run that there is # still data pending even if 'image' and 'ref_image' are unset. if baseline: run.ref_url = url run.ref_config = config_artifact.id else: run.url = url
def report_run(): """Reports a new run for a release candidate.""" build_id, release_name, release_number = _get_release_params() run_name = request.form.get('run_name', type=str) utils.jsonify_assert(run_name, 'run_name required') release = ( models.Release.query .filter_by(build_id=build_id, name=release_name, number=release_number) .first()) utils.jsonify_assert(release, 'release does not exist') # TODO: Make sure requesting user is owner of the build_id current_image = request.form.get('image', type=str) utils.jsonify_assert(current_image, 'image must be supplied') current_log = request.form.get('log', type=str) current_config = request.form.get('config', type=str) no_diff = request.form.get('no_diff') diff_image = request.form.get('diff_image', type=str) diff_log = request.form.get('diff_log', type=str) needs_diff = not (no_diff or diff_image or diff_log) # Find the previous corresponding run and automatically connect it. last_good_release = ( models.Release.query .filter_by( build_id=build_id, status=models.Release.GOOD) .order_by(models.Release.created.desc()) .first()) previous_id = None last_image = None if last_good_release: logging.debug('Found last good release for: build_id=%r, ' 'release_name=%r, release_number=%d, ' 'last_good_release_id=%d', build_id, release_name, release_number, last_good_release.id) last_good_run = ( models.Run.query .filter_by(release_id=last_good_release.id, name=run_name) .first()) if last_good_run: logging.debug('Found last good run for: build_id=%r, ' 'release_name=%r, release_number=%d, ' 'last_good_release_id=%d, last_good_run_id=%r, ' 'last_good_image=%r', build_id, release_name, release_number, last_good_release.id, last_good_run.id, last_good_run.image) previous_id = last_good_run.id last_image = last_good_run.image run = models.Run( name=run_name, release_id=release.id, image=current_image, log=current_log, config=current_config, previous_id=previous_id, needs_diff=bool(needs_diff and last_image), diff_image=diff_image, diff_log=diff_log) db.session.add(run) db.session.flush() # Schedule pdiff if there isn't already an image. if needs_diff and last_image: # TODO: Move this queue name to a flag. work_queue.add('run-pdiff', dict( build_id=build_id, release_name=release_name, release_number=release_number, run_name=run_name, reference_sha1sum=current_image, run_sha1sum=last_image, )) db.session.commit() logging.info('Created run: build_id=%r, release_name=%r, ' 'release_number=%d, run_name=%r', build_id, release_name, release_number, run_name) return flask.jsonify(success=True)
# Rewrite the config JSON to include the URL specified in this request. # Blindly overwrite anything that was there. config_dict['targetUrl'] = current_url config_data = json.dumps(config_dict) config_artifact = _save_artifact(build, config_data, 'application/json') db.session.add(config_artifact) db.session.flush() current_run.url = current_url current_run.config = config_artifact.id work_queue.add(constants.CAPTURE_QUEUE_NAME, dict( build_id=build.id, release_name=current_release.name, release_number=current_release.number, run_name=current_run.name, url=current_run.url, config_sha1sum=current_run.config, )) db.session.add(current_run) db.session.commit() return flask.jsonify( success=True, build_id=build.id, release_name=current_release.name, release_number=current_release.number, run_name=current_run.name, url=current_run.url, config=current_run.config)