def create_release(build):
"""Creates a new release candidate for a build."""
release_name = request.form.get('release_name')
utils.jsonify_assert(release_name, 'release_name required')
url = request.form.get('url')
utils.jsonify_assert(release_name, 'url required')
release = models.Release(
name=release_name,
url=url,
number=1,
build_id=build.id)
last_candidate = (
models.Release.query
.filter_by(build_id=build.id, name=release_name)
.order_by(models.Release.number.desc())
.first())
if last_candidate:
release.number += last_candidate.number
db.session.add(release)
db.session.commit()
logging.info('Created release: build_id=%r, release_name=%r, url=%r, '
'release_number=%d', build.id, release.name,
url, release.number)
return flask.jsonify(
success=True,
build_id=build.id,
release_name=release.name,
release_number=release.number,
url=url)
def _find_last_good_run(build):
"""Finds the last good release and run for a build."""
run_name = request.form.get('run_name', type=str)
utils.jsonify_assert(run_name, 'run_name required')
last_good_release = (
models.Release.query
.filter_by(
build_id=build.id,
status=models.Release.GOOD)
.order_by(models.Release.created.desc())
.first())
last_good_run = None
if last_good_release:
logging.debug('Found last good release for: build_id=%r, '
'release_name=%r, release_number=%d',
build.id, last_good_release.name,
last_good_release.number)
last_good_run = (
models.Run.query
.filter_by(release_id=last_good_release.id, name=run_name)
.first())
if last_good_run:
logging.debug('Found last good run for: build_id=%r, '
'release_name=%r, release_number=%d, '
'run_name=%r',
build.id, last_good_release.name,
last_good_release.number, last_good_run.name)
return last_good_release, last_good_run
def can_api_key_access_build(param_name):
"""Determines if the current API key can access the build in the request.
Args:
param_name: Parameter name to use for getting the build ID from the
request. Will fetch from GET or POST requests.
Returns:
(api_key, build) The API Key and the Build it has access to.
"""
build_id = (request.args.get(param_name, type=int)
or request.form.get(param_name, type=int))
utils.jsonify_assert(build_id, 'build_id required')
if app.config.get('IGNORE_AUTH'):
api_key = models.ApiKey(id='anonymous_superuser',
secret='',
superuser=True)
build = models.Build.query.get(build_id)
utils.jsonify_assert(build is not None, 'build must exist', 404)
else:
ops = _get_api_key_ops()
api_key, build = ops.can_access_build(build_id)
return api_key, build
def _get_release_params():
"""Gets the release params from the current request."""
release_name = request.form.get('release_name')
utils.jsonify_assert(release_name, 'release_name required')
release_number = request.form.get('release_number', type=int)
utils.jsonify_assert(release_number is not None, 'release_number required')
return release_name, release_number
def _get_or_create_run(build):
"""Gets a run for a build or creates it if it does not exist."""
release_name, release_number = _get_release_params()
run_name = request.form.get('run_name', type=str)
utils.jsonify_assert(run_name, 'run_name required')
release = (
models.Release.query
.filter_by(build_id=build.id, name=release_name, number=release_number)
.first())
utils.jsonify_assert(release, 'release does not exist')
run = (
models.Run.query
.filter_by(release_id=release.id, name=run_name)
.first())
if not run:
# Ignore re-reports of the same run name for this release.
logging.info('Created run: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r',
build.id, release.name, release.number, run_name)
run = models.Run(
release_id=release.id,
name=run_name,
status=models.Run.DATA_PENDING)
db.session.add(run)
db.session.flush()
return release, run
def runs_done():
"""Marks a release candidate as having all runs reported."""
build = g.build
release_name, release_number = _get_release_params()
release = (
models.Release.query
.filter_by(build_id=build.id, name=release_name, number=release_number)
.with_lockmode('update')
.first())
utils.jsonify_assert(release, 'Release does not exist')
release.status = models.Release.PROCESSING
db.session.add(release)
_check_release_done_processing(release)
db.session.commit()
signals.release_updated_via_api.send(app, build=build, release=release)
logging.info('Runs done for release: build_id=%r, release_name=%r, '
'release_number=%d', build.id, release.name, release.number)
results_url = url_for(
'view_release',
id=build.id,
name=release.name,
number=release.number,
_external=True)
return flask.jsonify(
success=True,
results_url=results_url)
def _find_last_good_run(build):
"""Finds the last good release and run for a build."""
run_name = request.form.get('run_name', type=str)
utils.jsonify_assert(run_name, 'run_name required')
last_good_release = (
models.Release.query
.filter_by(
build_id=build.id,
status=models.Release.GOOD)
.order_by(models.Release.created.desc())
.first())
last_good_run = None
if last_good_release:
logging.debug('Found last good release for: build_id=%r, '
'release_name=%r, release_number=%d',
build.id, last_good_release.name,
last_good_release.number)
last_good_run = (
models.Run.query
.filter_by(release_id=last_good_release.id, name=run_name)
.first())
if last_good_run:
logging.debug('Found last good run for: build_id=%r, '
'release_name=%r, release_number=%d, '
'run_name=%r',
build.id, last_good_release.name,
last_good_release.number, last_good_run.name)
return last_good_release, last_good_run
def create_release():
"""Creates a new release candidate for a build."""
build_id = request.form.get('build_id', type=int)
utils.jsonify_assert(build_id is not None, 'build_id required')
release_name = request.form.get('release_name')
utils.jsonify_assert(release_name, 'release_name required')
# TODO: Make sure build_id exists
# TODO: Make sure requesting user is owner of the build_id
release = models.Release(
name=release_name,
number=1,
build_id=build_id)
last_candidate = (
models.Release.query
.filter_by(build_id=build_id, name=release_name)
.order_by(models.Release.number.desc())
.first())
if last_candidate:
release.number += last_candidate.number
db.session.add(release)
db.session.commit()
logging.info('Created release: build_id=%r, release_name=%r, '
'release_number=%d', build_id, release_name, release.number)
return flask.jsonify(
build_id=build_id,
release_name=release_name,
release_number=release.number)
def _get_or_create_run(build):
"""Gets a run for a build or creates it if it does not exist."""
release_name, release_number = _get_release_params()
run_name = request.form.get('run_name', type=str)
utils.jsonify_assert(run_name, 'run_name required')
release = (
models.Release.query
.filter_by(build_id=build.id, name=release_name, number=release_number)
.first())
utils.jsonify_assert(release, 'release does not exist')
run = (
models.Run.query
.filter_by(release_id=release.id, name=run_name)
.first())
if not run:
# Ignore re-reports of the same run name for this release.
logging.info('Created run: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r',
build.id, release.name, release.number, run_name)
run = models.Run(
release_id=release.id,
name=run_name,
status=models.Run.DATA_PENDING)
db.session.add(run)
db.session.flush()
return release, run
def _get_release_params():
"""Gets the release params from the current request."""
release_name = request.form.get('release_name')
utils.jsonify_assert(release_name, 'release_name required')
release_number = request.form.get('release_number', type=int)
utils.jsonify_assert(release_number is not None, 'release_number required')
return release_name, release_number
def upload():
"""Uploads an artifact referenced by a run."""
# TODO: Require an API key on the basic auth header
utils.jsonify_assert(len(request.files) == 1,
'Need exactly one uploaded file')
file_storage = request.files.values()[0]
data = file_storage.read()
sha1sum = hashlib.sha1(data).hexdigest()
exists = models.Artifact.query.filter_by(id=sha1sum).first()
if exists:
logging.info('Upload already exists: artifact_id=%r', sha1sum)
return flask.jsonify(sha1sum=sha1sum)
# TODO: Mark that this owner/build has access to this sha1sum, to prevent
# users from pointing at sha1sums of images they don't own? Maybe too
# paranoid.
content_type, _ = mimetypes.guess_type(file_storage.filename)
artifact = models.Artifact(
id=sha1sum,
content_type=content_type,
data=data)
db.session.add(artifact)
db.session.commit()
logging.info('Upload received: artifact_id=%r, content_type=%r',
sha1sum, content_type)
return flask.jsonify(sha1sum=sha1sum)
def runs_done(build):
"""Marks a release candidate as having all runs reported."""
release_name, release_number = _get_release_params()
release = (
models.Release.query
.filter_by(build_id=build.id, name=release_name, number=release_number)
.first())
utils.jsonify_assert(release, 'Release does not exist')
release.status = models.Release.PROCESSING
db.session.add(release)
_check_release_done_processing(release)
db.session.commit()
logging.info('Runs done for release: build_id=%r, release_name=%r, '
'release_number=%d', build.id, release.name, release.number)
results_url = url_for(
'view_release',
id=build.id,
name=release.name,
number=release.number,
_external=True)
return flask.jsonify(
success=True,
results_url=results_url)
def wrapped(*args, **kwargs):
api_key = current_api_key()
g.api_key = api_key
utils.jsonify_assert(api_key.superuser,
'API key=%r must be a super user' % api_key.id,
403)
return f(*args, **kwargs)
def wrapped(*args, **kwargs):
api_key = current_api_key()
utils.jsonify_assert(
api_key.superuser,
'API key=%r must be a super user' % api_key.id,
403)
return f(*args, **kwargs)
def create_build():
"""Creates a new build for a user."""
# TODO: Make sure the requesting user is logged in
name = request.form.get('name')
utils.jsonify_assert(name, 'name required')
build = models.Build(name=name)
db.session.add(build)
db.session.commit()
logging.info('Created build: build_id=%r, name=%r', build.id, name)
return flask.jsonify(build_id=build.id, name=name)
def create_release():
"""Creates a new release candidate for a build."""
build = g.build
release_name = request.form.get('release_name')
utils.jsonify_assert(release_name, 'release_name required')
url = request.form.get('url')
utils.jsonify_assert(release_name, 'url required')
release = models.Release(
name=release_name,
url=url,
number=1,
build_id=build.id)
last_candidate = (
models.Release.query
.filter_by(build_id=build.id, name=release_name)
.order_by(models.Release.number.desc())
.first())
if last_candidate:
release.number += last_candidate.number
if last_candidate.status == models.Release.PROCESSING:
canceled_task_count = work_queue.cancel(
release_id=last_candidate.id)
logging.info('Canceling %d tasks for previous attempt '
'build_id=%r, release_name=%r, release_number=%d',
canceled_task_count, build.id, last_candidate.name,
last_candidate.number)
last_candidate.status = models.Release.BAD
db.session.add(last_candidate)
db.session.add(release)
db.session.commit()
signals.release_updated_via_api.send(app, build=build, release=release)
logging.info('Created release: build_id=%r, release_name=%r, url=%r, '
'release_number=%d', build.id, release.name,
url, release.number)
return flask.jsonify(
success=True,
build_id=build.id,
release_name=release.name,
release_number=release.number,
url=url)
def upload(build):
"""Uploads an artifact referenced by a run."""
utils.jsonify_assert(len(request.files) == 1,
'Need exactly one uploaded file')
file_storage = request.files.values()[0]
data = file_storage.read()
content_type, _ = mimetypes.guess_type(file_storage.filename)
artifact = _save_artifact(build, data, content_type)
db.session.add(artifact)
db.session.commit()
return flask.jsonify(
success=True,
build_id=build.id,
sha1sum=artifact.id,
content_type=content_type)
def upload():
"""Uploads an artifact referenced by a run."""
build = g.build
utils.jsonify_assert(len(request.files) == 1,
'Need exactly one uploaded file')
file_storage = request.files.values()[0]
data = file_storage.read()
content_type, _ = mimetypes.guess_type(file_storage.filename)
artifact = _save_artifact(build, data, content_type)
db.session.add(artifact)
db.session.commit()
return flask.jsonify(
success=True,
build_id=build.id,
sha1sum=artifact.id,
content_type=content_type)
def request_run(build):
"""Requests a new run for a release candidate."""
current_release, current_run = _get_or_create_run(build)
last_good_release, last_good_run = _find_last_good_run(build)
if last_good_run:
current_run.ref_url = last_good_run.url
current_run.ref_image = last_good_run.image
current_run.ref_log = last_good_run.log
current_run.ref_config = last_good_run.config
current_url = request.form.get('url', type=str)
config_data = request.form.get('config', default='{}', type=str)
utils.jsonify_assert(current_url, 'url to capture required')
utils.jsonify_assert(config_data, 'config document required')
# Validate the JSON config parses.
try:
config_dict = json.loads(config_data)
except Exception, e:
return jsonify_error(e)
def current_api_key():
"""Determines the API key for the current request.
Returns:
The API key.
"""
if app.config.get('IGNORE_AUTH'):
return models.ApiKey(
id='anonymous_superuser',
secret='',
superuser=True)
auth_header = request.authorization
if not auth_header:
logging.debug('API request lacks authorization header')
abort(flask.Response(
'API key required', 401,
{'WWW-Authenticate': 'Basic realm="API key required"'}))
api_key = models.ApiKey.query.get(auth_header.username)
utils.jsonify_assert(api_key, 'API key must exist', 403)
utils.jsonify_assert(api_key.active, 'API key must be active', 403)
utils.jsonify_assert(api_key.secret == auth_header.password,
'Must have good credentials', 403)
logging.debug('Authenticated as API key=%r', api_key.id)
return api_key
def release_done():
"""Marks a release candidate as good or bad."""
build_id, release_name, release_number = _get_release_params()
status = request.form.get('status')
valid_statuses = (models.Release.GOOD, models.Release.BAD)
utils.jsonify_assert(status in valid_statuses,
'status must be in %r' % valid_statuses)
release = (
models.Release.query
.filter_by(build_id=build_id, name=release_name, number=release_number)
.first())
utils.jsonify_assert(release, 'Release does not exist')
release.status = status
db.session.add(release)
db.session.commit()
logging.info('Release marked as %s: build_id=%r, release_name=%r, '
'number=%d', status, build_id, release_name, release_number)
return flask.jsonify(success=True)
def request_run():
"""Requests a new run for a release candidate."""
build = g.build
current_release, current_run = _get_or_create_run(build)
current_url = request.form.get('url', type=str)
config_data = request.form.get('config', default='{}', type=str)
utils.jsonify_assert(current_url, 'url to capture required')
utils.jsonify_assert(config_data, 'config document required')
config_artifact = _enqueue_capture(
build, current_release, current_run, current_url, config_data)
ref_url = request.form.get('ref_url', type=str)
ref_config_data = request.form.get('ref_config', type=str)
utils.jsonify_assert(
bool(ref_url) == bool(ref_config_data),
'ref_url and ref_config must both be specified or not specified')
if ref_url and ref_config_data:
ref_config_artifact = _enqueue_capture(
build, current_release, current_run, ref_url, ref_config_data,
baseline=True)
else:
_, last_good_run = _find_last_good_run(build)
if last_good_run:
current_run.ref_url = last_good_run.url
current_run.ref_image = last_good_run.image
current_run.ref_log = last_good_run.log
current_run.ref_config = last_good_run.config
db.session.add(current_run)
db.session.commit()
signals.run_updated_via_api.send(
app, build=build, release=current_release, run=current_run)
return flask.jsonify(
success=True,
build_id=build.id,
release_name=current_release.name,
release_number=current_release.number,
run_name=current_run.name,
url=current_run.url,
config=current_run.config,
ref_url=current_run.ref_url,
ref_config=current_run.ref_config)
def request_run():
"""Requests a new run for a release candidate."""
build = g.build
current_release, current_run = _get_or_create_run(build)
current_url = request.form.get('url', type=str)
config_data = request.form.get('config', default='{}', type=str)
utils.jsonify_assert(current_url, 'url to capture required')
utils.jsonify_assert(config_data, 'config document required')
config_artifact = _enqueue_capture(
build, current_release, current_run, current_url, config_data)
ref_url = request.form.get('ref_url', type=str)
ref_config_data = request.form.get('ref_config', type=str)
utils.jsonify_assert(
bool(ref_url) == bool(ref_config_data),
'ref_url and ref_config must both be specified or not specified')
if ref_url and ref_config_data:
ref_config_artifact = _enqueue_capture(
build, current_release, current_run, ref_url, ref_config_data,
baseline=True)
else:
_, last_good_run = _find_last_good_run(build)
if last_good_run:
current_run.ref_url = last_good_run.url
current_run.ref_image = last_good_run.image
current_run.ref_log = last_good_run.log
current_run.ref_config = last_good_run.config
db.session.add(current_run)
db.session.commit()
return flask.jsonify(
success=True,
build_id=build.id,
release_name=current_release.name,
release_number=current_release.number,
run_name=current_run.name,
url=current_run.url,
config=current_run.config,
ref_url=current_run.ref_url,
ref_config=current_run.ref_config)
def can_api_key_access_build(param_name):
"""Determines if the current API key can access the build in the request.
Args:
param_name: Parameter name to use for getting the build ID from the
request. Will fetch from GET or POST requests.
Returns:
The Build the API key has access to.
"""
api_key = current_api_key()
build_id = request.args.get(param_name, type=int) or request.form.get(param_name, type=int)
utils.jsonify_assert(build_id, "build_id required")
build = models.Build.query.get(build_id)
utils.jsonify_assert(build is not None, "build must exist", 404)
if not api_key.superuser:
utils.jsonify_assert(api_key.build_id == build_id, "API key must have access", 404)
return build
def report_pdiff():
"""Reports a pdiff for a run.
When there is no diff to report, supply the "no_diff" parameter.
"""
build_id, release_name, release_number = _get_release_params()
run_name = request.form.get('run_name', type=str)
utils.jsonify_assert(run_name, 'run_name required')
release = (
models.Release.query
.filter_by(
build_id=build_id,
name=release_name)
.first())
utils.jsonify_assert(release, 'Release does not exist')
run = (
models.Run.query
.filter_by(release_id=release.id, name=run_name)
.first())
utils.jsonify_assert(release, 'Run does not exist')
no_diff = request.form.get('no_diff')
run.needs_diff = not (no_diff or run.diff_image or run.diff_log)
run.diff_image = request.form.get('diff_image', type=str)
run.diff_log = request.form.get('diff_log', type=str)
db.session.add(run)
logging.info('Saved pdiff: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r, '
'no_diff=%r, diff_image=%r, diff_log=%r',
build_id, release_name, release_number, run_name,
no_diff, run.diff_image, run.diff_log)
_check_release_done_processing(run.release_id)
db.session.commit()
return flask.jsonify(success=True)
def report_run():
"""Reports a new run for a release candidate."""
build_id, release_name, release_number = _get_release_params()
run_name = request.form.get('run_name', type=str)
utils.jsonify_assert(run_name, 'run_name required')
release = (
models.Release.query
.filter_by(build_id=build_id, name=release_name, number=release_number)
.first())
utils.jsonify_assert(release, 'release does not exist')
# TODO: Make sure requesting user is owner of the build_id
current_image = request.form.get('image', type=str)
utils.jsonify_assert(current_image, 'image must be supplied')
current_log = request.form.get('log', type=str)
current_config = request.form.get('config', type=str)
no_diff = request.form.get('no_diff')
diff_image = request.form.get('diff_image', type=str)
diff_log = request.form.get('diff_log', type=str)
needs_diff = not (no_diff or diff_image or diff_log)
# Find the previous corresponding run and automatically connect it.
last_good_release = (
models.Release.query
.filter_by(
build_id=build_id,
status=models.Release.GOOD)
.order_by(models.Release.created.desc())
.first())
previous_id = None
last_image = None
if last_good_release:
logging.debug('Found last good release for: build_id=%r, '
'release_name=%r, release_number=%d, '
'last_good_release_id=%d',
build_id, release_name, release_number,
last_good_release.id)
last_good_run = (
models.Run.query
.filter_by(release_id=last_good_release.id, name=run_name)
.first())
if last_good_run:
logging.debug('Found last good run for: build_id=%r, '
'release_name=%r, release_number=%d, '
'last_good_release_id=%d, last_good_run_id=%r, '
'last_good_image=%r',
build_id, release_name, release_number,
last_good_release.id, last_good_run.id,
last_good_run.image)
previous_id = last_good_run.id
last_image = last_good_run.image
run = models.Run(
name=run_name,
release_id=release.id,
image=current_image,
log=current_log,
config=current_config,
previous_id=previous_id,
needs_diff=bool(needs_diff and last_image),
diff_image=diff_image,
diff_log=diff_log)
db.session.add(run)
db.session.flush()
# Schedule pdiff if there isn't already an image.
if needs_diff and last_image:
# TODO: Move this queue name to a flag.
work_queue.add('run-pdiff', dict(
build_id=build_id,
release_name=release_name,
release_number=release_number,
run_name=run_name,
reference_sha1sum=current_image,
run_sha1sum=last_image,
))
db.session.commit()
logging.info('Created run: build_id=%r, release_name=%r, '
'release_number=%d, run_name=%r',
build_id, release_name, release_number, run_name)
return flask.jsonify(success=True)
