def testLeaseAndFinish(self):
"""Tests leasing tasks and finishing them."""
task_ids = []
for i in xrange(10):
next_id = work_queue.add(TEST_QUEUE,
payload={
'foo': i,
'bar': 5.5,
'baz': 'banana'
})
task_ids.append(next_id)
db.session.commit()
item = queue_worker.RemoteQueueWorkflow(TEST_QUEUE,
TestQueueWorkflow,
max_tasks=1,
wait_seconds=0.01)
item.root = True
self.coordinator.input_queue.put(item)
time.sleep(2)
item.stop()
self.coordinator.wait_one()
db.session.flush()
for task_id in task_ids:
found = work_queue.WorkQueue.query.get((task_id, TEST_QUEUE))
self.assertEquals(work_queue.WorkQueue.DONE, found.status)
def testLeaseAndFinish(self):
"""Tests leasing tasks and finishing them."""
task_ids = []
for i in xrange(10):
next_id = work_queue.add(
TEST_QUEUE,
payload={'foo': i, 'bar': 5.5, 'baz': 'banana'})
task_ids.append(next_id)
db.session.commit()
item = queue_worker.RemoteQueueWorkflow(
TEST_QUEUE,
TestQueueWorkflow,
max_tasks=1,
wait_seconds=0.01)
item.root = True
self.coordinator.input_queue.put(item)
time.sleep(2)
item.stop()
self.coordinator.wait_one()
db.session.flush()
for task_id in task_ids:
found = work_queue.WorkQueue.query.get((task_id, TEST_QUEUE))
self.assertEquals(work_queue.WorkQueue.DONE, found.status)
def handle_add(queue_name):
"""Adds a task to a queue."""
source = request.form.get('source', request.remote_addr, type=str)
try:
task_id = work_queue.add(
queue_name,
payload=request.form.get('payload', type=str),
content_type=request.form.get('content_type', type=str),
source=source,
task_id=request.form.get('task_id', type=str))
except work_queue.Error, e:
return utils.jsonify_error(e)
def handle_add(queue_name):
"""Adds a task to a queue."""
source = request.form.get('source', request.remote_addr, type=str)
try:
task_id = work_queue.add(
queue_name,
payload=request.form.get('payload', type=str),
content_type=request.form.get('content_type', type=str),
source=source,
task_id=request.form.get('task_id', type=str))
except work_queue.Error, e:
return utils.jsonify_error(e)
def report_run():
"""Reports data for a run for a release candidate."""
build = g.build
release, run = _get_or_create_run(build)
db.session.refresh(run, lockmode='update')
current_url = request.form.get('url', type=str)
current_image = request.form.get('image', type=str)
current_log = request.form.get('log', type=str)
current_config = request.form.get('config', type=str)
ref_url = request.form.get('ref_url', type=str)
ref_image = request.form.get('ref_image', type=str)
ref_log = request.form.get('ref_log', type=str)
ref_config = request.form.get('ref_config', type=str)
diff_success = request.form.get('diff_success', type=str)
diff_image = request.form.get('diff_image', type=str)
diff_log = request.form.get('diff_log', type=str)
if current_url:
run.url = current_url
if current_image:
run.image = current_image
if current_log:
run.log = current_log
if current_config:
run.config = current_config
if current_image or current_log or current_config:
logging.info(
'Saved run data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, url=%r, '
'image=%r, log=%r, config=%r', build.id, release.name,
release.number, run.name, run.url, run.image, run.log, run.config)
if ref_url:
run.ref_url = ref_url
if ref_image:
run.ref_image = ref_image
if ref_log:
run.ref_log = ref_log
if ref_config:
run.ref_config = ref_config
if ref_image or ref_log or ref_config:
logging.info(
'Saved reference data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, ref_url=%r, '
'ref_image=%r, ref_log=%r, ref_config=%r', build.id, release.name,
release.number, run.name, run.ref_url, run.ref_image, run.ref_log,
run.ref_config)
if diff_image:
run.diff_image = diff_image
if diff_log:
run.diff_log = diff_log
if diff_image or diff_log:
logging.info(
'Saved pdiff: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, '
'diff_image=%r, diff_log=%r', build.id, release.name,
release.number, run.name, run.diff_image, run.diff_log)
if run.image and run.diff_image:
run.status = models.Run.DIFF_FOUND
elif run.image and run.ref_image and not run.diff_log:
run.status = models.Run.NEEDS_DIFF
elif run.image and run.ref_image and diff_success:
run.status = models.Run.DIFF_NOT_FOUND
elif run.image and not run.ref_config:
run.status = models.Run.NO_DIFF_NEEDED
# TODO: Consider adding another status of "diff failed" or capture
# failed for situations where it couldn't finish.
# TODO: Verify the build has access to both the current_image and
# the reference_sha1sum so they can't make a diff from a black image
# and still see private data in the diff image.
if run.status == models.Run.NEEDS_DIFF:
task_id = '%s:%s:%s' % (run.id, run.image, run.ref_image)
logging.info('Enqueuing pdiff task=%r', task_id)
work_queue.add(constants.PDIFF_QUEUE_NAME,
payload=dict(
build_id=build.id,
release_name=release.name,
release_number=release.number,
run_name=run.name,
run_sha1sum=run.image,
reference_sha1sum=run.ref_image,
),
build_id=build.id,
release_id=release.id,
run_id=run.id,
source='report_run',
task_id=task_id)
# Flush the run so querying for Runs in _check_release_done_processing
# will be find the new run too and we won't deadlock.
db.session.add(run)
db.session.flush()
_check_release_done_processing(release)
db.session.commit()
signals.run_updated_via_api.send(app,
build=build,
release=release,
run=run)
logging.info(
'Updated run: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, status=%r', build.id, release.name,
release.number, run.name, run.status)
return flask.jsonify(success=True)
suffix = ''
if baseline:
suffix = ':baseline'
task_id = '%s:%s%s' % (run.id, hashlib.sha1(url).hexdigest(), suffix)
logging.info('Enqueueing capture task=%r, baseline=%r', task_id, baseline)
work_queue.add(constants.CAPTURE_QUEUE_NAME,
payload=dict(
build_id=build.id,
release_name=release.name,
release_number=release.number,
run_name=run.name,
url=url,
config_sha1sum=config_artifact.id,
baseline=baseline,
),
build_id=build.id,
release_id=release.id,
run_id=run.id,
source='request_run',
task_id=task_id)
# Set the URL and config early to indicate to report_run that there is
# still data pending even if 'image' and 'ref_image' are unset.
if baseline:
run.ref_url = url
run.ref_config = config_artifact.id
else:
run.url = url
def report_run():
"""Reports data for a run for a release candidate."""
build = g.build
release, run = _get_or_create_run(build)
current_url = request.form.get('url', type=str)
current_image = request.form.get('image', type=str)
current_log = request.form.get('log', type=str)
current_config = request.form.get('config', type=str)
ref_url = request.form.get('ref_url', type=str)
ref_image = request.form.get('ref_image', type=str)
ref_log = request.form.get('ref_log', type=str)
ref_config = request.form.get('ref_config', type=str)
diff_success = request.form.get('diff_success', type=str)
diff_image = request.form.get('diff_image', type=str)
diff_log = request.form.get('diff_log', type=str)
if current_url:
run.url = current_url
if current_image:
run.image = current_image
if current_log:
run.log = current_log
if current_config:
run.config = current_config
if current_image or current_log or current_config:
logging.info('Saved run data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, url=%r, '
'image=%r, log=%r, config=%r',
build.id, release.name, release.number, run.name,
run.url, run.image, run.log, run.config)
if ref_url:
run.ref_url = ref_url
if ref_image:
run.ref_image = ref_image
if ref_log:
run.ref_log = ref_log
if ref_config:
run.ref_config = ref_config
if ref_image or ref_log or ref_config:
logging.info('Saved reference data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, ref_url=%r, '
'ref_image=%r, ref_log=%r, ref_config=%r',
build.id, release.name, release.number, run.name,
run.ref_url, run.ref_image, run.ref_log, run.ref_config)
if diff_image:
run.diff_image = diff_image
if diff_log:
run.diff_log = diff_log
if diff_image or diff_log:
logging.info('Saved pdiff: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, '
'diff_image=%r, diff_log=%r',
build.id, release.name, release.number, run.name,
run.diff_image, run.diff_log)
if run.image and run.diff_image:
run.status = models.Run.DIFF_FOUND
elif run.image and run.ref_image and not run.diff_log:
run.status = models.Run.NEEDS_DIFF
elif run.image and run.ref_image and diff_success:
run.status = models.Run.DIFF_NOT_FOUND
elif run.image and not run.ref_config:
run.status = models.Run.NO_DIFF_NEEDED
# TODO: Consider adding another status of "diff failed" or capture
# failed for situations where it couldn't finish.
# TODO: Verify the build has access to both the current_image and
# the reference_sha1sum so they can't make a diff from a black image
# and still see private data in the diff image.
if run.status == models.Run.NEEDS_DIFF:
task_id = '%s:%s:%s' % (run.id, run.image, run.ref_image)
logging.info('Enqueuing pdiff task=%r', task_id)
work_queue.add(
constants.PDIFF_QUEUE_NAME,
payload=dict(
build_id=build.id,
release_name=release.name,
release_number=release.number,
run_name=run.name,
run_sha1sum=run.image,
reference_sha1sum=run.ref_image,
),
source='report_run',
task_id=task_id)
# Flush the run so querying for Runs in _check_release_done_processing
# will be find the new run too.
db.session.add(run)
_check_release_done_processing(release)
db.session.commit()
logging.info('Updated run: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, status=%r',
build.id, release.name, release.number, run.name, run.status)
return flask.jsonify(success=True)
db.session.flush()
suffix = ''
if baseline:
suffix = ':baseline'
task_id = '%s:%s%s' % (run.id, hashlib.sha1(url).hexdigest(), suffix)
logging.info('Enqueueing capture task=%r, baseline=%r', task_id, baseline)
work_queue.add(
constants.CAPTURE_QUEUE_NAME,
payload=dict(
build_id=build.id,
release_name=release.name,
release_number=release.number,
run_name=run.name,
url=url,
config_sha1sum=config_artifact.id,
baseline=baseline,
),
source='request_run',
task_id=task_id)
# Set the URL and config early to indicate to report_run that there is
# still data pending even if 'image' and 'ref_image' are unset.
if baseline:
run.ref_url = url
run.ref_config = config_artifact.id
else:
run.url = url
run.config = config_artifact.id
def report_run():
"""Reports data for a run for a release candidate."""
build = g.build
release, run = _get_or_create_run(build)
db.session.refresh(run, lockmode='update')
current_url = request.form.get('url', type=str)
current_image = request.form.get('image', type=str)
current_log = request.form.get('log', type=str)
current_config = request.form.get('config', type=str)
ref_url = request.form.get('ref_url', type=str)
ref_image = request.form.get('ref_image', type=str)
ref_log = request.form.get('ref_log', type=str)
ref_config = request.form.get('ref_config', type=str)
diff_failed = request.form.get('diff_failed', type=str)
diff_image = request.form.get('diff_image', type=str)
diff_log = request.form.get('diff_log', type=str)
distortion = request.form.get('distortion', default=None, type=float)
run_failed = request.form.get('run_failed', type=str)
if current_url:
run.url = current_url
if current_image:
run.image = current_image
if current_log:
run.log = current_log
if current_config:
run.config = current_config
if current_image or current_log or current_config:
logging.info(
'Saving run data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, url=%r, '
'image=%r, log=%r, config=%r, run_failed=%r', build.id,
release.name, release.number, run.name, run.url, run.image,
run.log, run.config, run_failed)
if ref_url:
run.ref_url = ref_url
if ref_image:
run.ref_image = ref_image
if ref_log:
run.ref_log = ref_log
if ref_config:
run.ref_config = ref_config
if ref_image or ref_log or ref_config:
logging.info(
'Saved reference data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, ref_url=%r, '
'ref_image=%r, ref_log=%r, ref_config=%r', build.id, release.name,
release.number, run.name, run.ref_url, run.ref_image, run.ref_log,
run.ref_config)
if diff_image:
run.diff_image = diff_image
if diff_log:
run.diff_log = diff_log
if distortion:
run.distortion = distortion
if diff_image or diff_log:
logging.info(
'Saved pdiff: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, diff_image=%r, '
'diff_log=%r, diff_failed=%r, distortion=%r', build.id,
release.name, release.number, run.name, run.diff_image,
run.diff_log, diff_failed, distortion)
if run.image and run.diff_image:
run.status = models.Run.DIFF_FOUND
elif run.image and run.ref_image and not run.diff_log:
run.status = models.Run.NEEDS_DIFF
elif run.image and run.ref_image and not diff_failed:
run.status = models.Run.DIFF_NOT_FOUND
elif run.image and not run.ref_config:
run.status = models.Run.NO_DIFF_NEEDED
elif run_failed or diff_failed:
run.status = models.Run.FAILED
else:
# NOTE: Intentionally do not transition state here in the default case.
# We allow multiple background workers to be writing to the same Run in
# parallel updating its various properties.
pass
# TODO: Verify the build has access to both the current_image and
# the reference_sha1sum so they can't make a diff from a black image
# and still see private data in the diff image.
if run.status == models.Run.NEEDS_DIFF:
task_id = '%s:%s:%s' % (run.id, run.image, run.ref_image)
logging.info('Enqueuing pdiff task=%r', task_id)
work_queue.add(constants.PDIFF_QUEUE_NAME,
payload=dict(
build_id=build.id,
release_name=release.name,
release_number=release.number,
run_name=run.name,
run_sha1sum=run.image,
reference_sha1sum=run.ref_image,
),
build_id=build.id,
release_id=release.id,
run_id=run.id,
source='report_run',
task_id=task_id)
# Flush the run so querying for Runs in _check_release_done_processing
# will be find the new run too and we won't deadlock.
db.session.add(run)
db.session.flush()
_check_release_done_processing(release)
db.session.commit()
signals.run_updated_via_api.send(app,
build=build,
release=release,
run=run)
logging.info(
'Updated run: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, status=%r', build.id, release.name,
release.number, run.name, run.status)
return flask.jsonify(success=True)
def report_run():
"""Reports data for a run for a release candidate."""
build = g.build
release, run = _get_or_create_run(build)
db.session.refresh(run, lockmode='update')
current_url = request.form.get('url', type=str)
current_image = request.form.get('image', type=str)
current_log = request.form.get('log', type=str)
current_config = request.form.get('config', type=str)
ref_url = request.form.get('ref_url', type=str)
ref_image = request.form.get('ref_image', type=str)
ref_log = request.form.get('ref_log', type=str)
ref_config = request.form.get('ref_config', type=str)
diff_failed = request.form.get('diff_failed', type=str)
diff_image = request.form.get('diff_image', type=str)
diff_log = request.form.get('diff_log', type=str)
distortion = request.form.get('distortion', default=None, type=float)
run_failed = request.form.get('run_failed', type=str)
if current_url:
run.url = current_url
if current_image:
run.image = current_image
if current_log:
run.log = current_log
if current_config:
run.config = current_config
if current_image or current_log or current_config:
logging.info('Saving run data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, url=%r, '
'image=%r, log=%r, config=%r, run_failed=%r',
build.id, release.name, release.number, run.name,
run.url, run.image, run.log, run.config, run_failed)
if ref_url:
run.ref_url = ref_url
if ref_image:
run.ref_image = ref_image
if ref_log:
run.ref_log = ref_log
if ref_config:
run.ref_config = ref_config
if ref_image or ref_log or ref_config:
logging.info('Saved reference data: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, ref_url=%r, '
'ref_image=%r, ref_log=%r, ref_config=%r',
build.id, release.name, release.number, run.name,
run.ref_url, run.ref_image, run.ref_log, run.ref_config)
if diff_image:
run.diff_image = diff_image
if diff_log:
run.diff_log = diff_log
if distortion:
run.distortion = distortion
if diff_image or diff_log:
logging.info('Saved pdiff: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, diff_image=%r, '
'diff_log=%r, diff_failed=%r, distortion=%r',
build.id, release.name, release.number, run.name,
run.diff_image, run.diff_log, diff_failed, distortion)
if run.image and run.diff_image:
run.status = models.Run.DIFF_FOUND
elif run.image and run.ref_image and not run.diff_log:
run.status = models.Run.NEEDS_DIFF
elif run.image and run.ref_image and not diff_failed:
run.status = models.Run.DIFF_NOT_FOUND
elif run.image and not run.ref_config:
run.status = models.Run.NO_DIFF_NEEDED
elif run_failed or diff_failed:
run.status = models.Run.FAILED
else:
# NOTE: Intentionally do not transition state here in the default case.
# We allow multiple background workers to be writing to the same Run in
# parallel updating its various properties.
pass
# TODO: Verify the build has access to both the current_image and
# the reference_sha1sum so they can't make a diff from a black image
# and still see private data in the diff image.
if run.status == models.Run.NEEDS_DIFF:
task_id = '%s:%s:%s' % (run.id, run.image, run.ref_image)
logging.info('Enqueuing pdiff task=%r', task_id)
work_queue.add(
constants.PDIFF_QUEUE_NAME,
payload=dict(
build_id=build.id,
release_name=release.name,
release_number=release.number,
run_name=run.name,
run_sha1sum=run.image,
reference_sha1sum=run.ref_image,
),
build_id=build.id,
release_id=release.id,
run_id=run.id,
source='report_run',
task_id=task_id)
# Flush the run so querying for Runs in _check_release_done_processing
# will be find the new run too and we won't deadlock.
db.session.add(run)
db.session.flush()
_check_release_done_processing(release)
db.session.commit()
signals.run_updated_via_api.send(
app, build=build, release=release, run=run)
logging.info('Updated run: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, status=%r',
build.id, release.name, release.number, run.name, run.status)
return flask.jsonify(success=True)
def report_run():
"""Reports data for a run for a release candidate."""
build = g.build
release, run = _get_or_create_run(build)
db.session.refresh(run, lockmode="update")
current_url = request.form.get("url", type=str)
current_image = request.form.get("image", type=str)
current_log = request.form.get("log", type=str)
current_config = request.form.get("config", type=str)
ref_url = request.form.get("ref_url", type=str)
ref_image = request.form.get("ref_image", type=str)
ref_log = request.form.get("ref_log", type=str)
ref_config = request.form.get("ref_config", type=str)
diff_success = request.form.get("diff_success", type=str)
diff_image = request.form.get("diff_image", type=str)
diff_log = request.form.get("diff_log", type=str)
if current_url:
run.url = current_url
if current_image:
run.image = current_image
if current_log:
run.log = current_log
if current_config:
run.config = current_config
if current_image or current_log or current_config:
logging.info(
"Saved run data: build_id=%r, release_name=%r, "
"release_number=%d, run_name=%r, url=%r, "
"image=%r, log=%r, config=%r",
build.id,
release.name,
release.number,
run.name,
run.url,
run.image,
run.log,
run.config,
)
if ref_url:
run.ref_url = ref_url
if ref_image:
run.ref_image = ref_image
if ref_log:
run.ref_log = ref_log
if ref_config:
run.ref_config = ref_config
if ref_image or ref_log or ref_config:
logging.info(
"Saved reference data: build_id=%r, release_name=%r, "
"release_number=%d, run_name=%r, ref_url=%r, "
"ref_image=%r, ref_log=%r, ref_config=%r",
build.id,
release.name,
release.number,
run.name,
run.ref_url,
run.ref_image,
run.ref_log,
run.ref_config,
)
if diff_image:
run.diff_image = diff_image
if diff_log:
run.diff_log = diff_log
if diff_image or diff_log:
logging.info(
"Saved pdiff: build_id=%r, release_name=%r, "
"release_number=%d, run_name=%r, "
"diff_image=%r, diff_log=%r",
build.id,
release.name,
release.number,
run.name,
run.diff_image,
run.diff_log,
)
if run.image and run.diff_image:
run.status = models.Run.DIFF_FOUND
elif run.image and run.ref_image and not run.diff_log:
run.status = models.Run.NEEDS_DIFF
elif run.image and run.ref_image and diff_success:
run.status = models.Run.DIFF_NOT_FOUND
elif run.image and not run.ref_config:
run.status = models.Run.NO_DIFF_NEEDED
# TODO: Consider adding another status of "diff failed" or capture
# failed for situations where it couldn't finish.
# TODO: Verify the build has access to both the current_image and
# the reference_sha1sum so they can't make a diff from a black image
# and still see private data in the diff image.
if run.status == models.Run.NEEDS_DIFF:
task_id = "%s:%s:%s" % (run.id, run.image, run.ref_image)
logging.info("Enqueuing pdiff task=%r", task_id)
work_queue.add(
constants.PDIFF_QUEUE_NAME,
payload=dict(
build_id=build.id,
release_name=release.name,
release_number=release.number,
run_name=run.name,
run_sha1sum=run.image,
reference_sha1sum=run.ref_image,
),
source="report_run",
task_id=task_id,
)
# Flush the run so querying for Runs in _check_release_done_processing
# will be find the new run too and we won't deadlock.
db.session.add(run)
db.session.flush()
_check_release_done_processing(release)
db.session.commit()
signals.run_updated_via_api.send(app, build=build, release=release, run=run)
logging.info(
"Updated run: build_id=%r, release_name=%r, " "release_number=%d, run_name=%r, status=%r",
build.id,
release.name,
release.number,
run.name,
run.status,
)
return flask.jsonify(success=True)
