def promoted_build_get(proj, name):
b = get_or_404(
Build.query.join(Project).filter(
Project.name == proj,
Build.status == BuildStatus.PROMOTED,
Build.name == name,
))
return jsendify({'build': _promoted_as_json(Storage(), b)})
def run_rerun(proj, build_id, run):
r = _get_run(proj, build_id, run)
permissions.assert_internal_user()
for t in r.tests:
db.session.delete(t)
r.set_status(BuildStatus.QUEUED)
db.session.commit()
return jsendify({})
def test_incomplete_list():
permissions.assert_internal_user()
tests = []
complete = (BuildStatus.PASSED, BuildStatus.FAILED)
for t in Test.query.filter(~Test.status.in_(complete)):
tests.append(t.as_json(detailed=True))
tests[-1]['metadata'] = t.run.meta
tests[-1]['api_key'] = t.run.api_key
return jsendify({'tests': tests})
def project_trigger_list():
permissions.assert_internal_user()
t = request.args.get("type")
if t:
t = TriggerTypes[t].value
query = ProjectTrigger.query.filter(ProjectTrigger.type == t)
else:
query = ProjectTrigger.query.all()
return jsendify([x.as_json() for x in query])
def data_error(e):
data = {
'message': 'An unexpected error occurred inserting data',
'error_msg': str(e),
'stack_trace': traceback.format_exc(),
}
current_app.logger.exception(
'Unexpected DB error caught in BP error handler')
return jsendify(data, 400)
def data_error(e):
data = {
"message": "An unexpected error occurred inserting data",
"error_msg": str(e),
"stack_trace": traceback.format_exc(),
}
current_app.logger.exception(
"Unexpected DB error caught in BP error handler")
return jsendify(data, 400)
def simulator_validate():
data = request.get_json()
if not data:
raise ApiError(400, "run-definition must be posted as json data")
try:
ProjectDefinition.validate_data(data)
except Exception as e:
raise ApiError(400, str(e))
return jsendify({})
def unexpected_error(e):
data = {
'message': 'An unexpected error occurred',
'error_msg': str(e),
'stack_trace': traceback.format_exc(),
}
print(dir(bp))
current_app.logger.exception(
'Unexpected error caught in BP error handler')
return jsendify(data, 500)
def unexpected_error(e):
data = {
"message": "An unexpected error occurred",
"error_msg": str(e),
"stack_trace": traceback.format_exc(),
}
print(dir(bp))
current_app.logger.exception(
"Unexpected error caught in BP error handler")
return jsendify(data, 500)
def run_get(proj, build_id, run):
r = _get_run(proj, build_id, run)
data = r.as_json(detailed=True)
artifacts = []
for a in Storage().list_artifacts(r):
u = url_for('api_run.run_get_artifact', proj=proj, build_id=build_id,
run=run, path=a, _external=True)
artifacts.append(u)
data['artifacts'] = artifacts
return jsendify({'run': data})
def worker_update(name):
w = get_or_404(Worker.query.filter_by(name=name, deleted=False))
data = request.get_json() or {}
attrs = ('distro', 'mem_total', 'cpu_total', 'cpu_type',
'concurrent_runs', 'host_tags')
for attr in attrs:
val = data.get(attr)
if val is not None:
setattr(w, attr, val)
db.session.commit()
return jsendify({}, 200)
def build_create(proj):
u = permissions.assert_can_build(proj)
p = Project.query.filter(Project.name == proj).first_or_404()
d = request.get_json() or {}
secrets = {}
# Check if the caller wants to inherit secrets from something like the
# "git-poller" trigger for the project.
trigger_type = d.get('trigger-type')
if trigger_type:
optional = trigger_type.endswith('-optional')
if optional:
trigger_type = trigger_type[:-9] # strip off the "-optional"
for t in p.triggers:
if TriggerTypes(t.type).name == trigger_type:
secrets.update(t.secret_data)
break
else:
if not optional:
raise ApiError(400, 'No such trigger-type: %s' % trigger_type)
# Check if the caller wants to inherit secrets from a specific trigger
# definied for the project.
trigger_id = d.get('trigger-id')
if trigger_id:
for t in p.triggers:
if t.id == trigger_id:
secrets.update(t.secret_data)
break
else:
raise ApiError(400, 'Unknown trigger-id: %s' % trigger_id)
secrets.update(d.get('secrets') or {})
if 'triggered-by' in secrets:
# Let's not allow triggered-by to be set externally.
del secrets['triggered-by']
if u:
secrets['triggered-by'] = str(u)
b = trigger_build(p, d.get('reason'), d.get('trigger-name'),
d.get('params'), secrets, d.get('project-definition'),
d.get('queue-priority', 0))
url = url_for('api_build.build_get',
proj=p.name,
build_id=b.build_id,
_external=True)
weburl = None
if BUILD_URL_FMT:
weburl = BUILD_URL_FMT.format(project=b.project.name, build=b.build_id)
return jsendify({
'url': url,
'build_id': b.build_id,
'web_url': weburl
}, 201)
def build_get_latest(proj):
'''Return the most recent successful build'''
qs = Build.query.join(Build.project).filter(
Project.name == proj,
Build.status == BuildStatus.PASSED,
)
trigger = request.args.get('trigger_name')
if trigger:
qs = qs.filter(Build.trigger_name == trigger)
b = get_or_404(qs.order_by(Build.id.desc()))
return jsendify({'build': b.as_json(detailed=True)})
def run_upload(proj, build_id, run):
r = _get_run(proj, build_id, run)
_authenticate_runner(r)
data = request.get_json()
urls = {}
if data:
# determine url expiration, default 1800 = 30 minues
expiration = request.headers.get('X-URL-EXPIRATION', 1800)
urls = Storage().generate_signed(r, data, expiration)
return jsendify({'urls': urls})
def test_find():
permissions.assert_internal_user()
context = request.args.get('context')
if not context:
raise ApiError(401, {'message': 'Missing "context" query argument'})
tests = []
for t in Test.query.filter_by(context=context):
tests.append(t.as_json(detailed=True))
tests[-1]['metadata'] = t.run.meta
tests[-1]['api_key'] = t.run.api_key
return jsendify({'tests': tests})
def build_get_latest(proj):
'''Return the most recent successful build'''
b = get_or_404(
Build.query.join(
Build.project
).filter(
Project.name == proj,
Build.status == BuildStatus.PASSED,
).order_by(
Build.id.desc()
)
)
return jsendify({'build': b.as_json(detailed=True)})
def project_create():
d = request.get_json() or {}
proj = d.get('name')
if not proj:
raise ApiError(401, 'Missing required parameter: "name"')
sync = d.get('synchronous-builds', False)
permissions.assert_internal_user()
db.session.add(Project(proj, sync))
db.session.commit()
url = url_for('api_project.project_get', proj=proj, _external=True)
return jsendify({'url': url}, 201)
def build_get_latest(proj):
"""Return the most recent successful build"""
status = BuildStatus.PASSED
promoted = request.args.get("promoted")
if promoted:
status = BuildStatus.PROMOTED
qs = Build.query.join(Build.project).filter(
Project.name == proj,
Build.status == status,
)
trigger = request.args.get("trigger_name")
if trigger:
qs = qs.filter(Build.trigger_name == trigger)
b = get_or_404(qs.order_by(Build.id.desc()))
return jsendify({"build": b.as_json(detailed=True)})
def on_webhook(proj):
trigger = _find_trigger(proj)
event = request.headers.get("X-Github-Event")
_filter_events(event)
data = request.get_json()
if event == "issue_comment":
if "ci-retest" not in request.json["comment"]["body"]:
return "Ingoring comment"
pr_num = data["issue"]["number"]
repo = data["repository"]["full_name"]
elif event == "pull_request":
if data["action"] not in ("opened", "synchronize"):
return "Ignoring action: " + request.json["action"]
pr_num = data["pull_request"]["number"]
repo = data["pull_request"]["base"]["repo"]["full_name"]
reason = "GitHub PR(%s): %s, https://github.com/%s/pull/%d" % (
pr_num,
event,
repo,
pr_num,
)
secrets = trigger.secret_data
token = secrets["githubtok"]
owner, repo = repo.split("/")
params = _get_params(owner, repo, pr_num, token)
try:
trig, proj = _get_proj_def(trigger, owner, repo, params["GIT_SHA"],
token)
b = trigger_build(trigger.project, reason, trig, params, secrets, proj,
trigger.queue_priority)
_update_pr(b, params["GH_STATUS_URL"], token)
url = url_for(
"api_build.build_get",
proj=trigger.project.name,
build_id=b.build_id,
_external=True,
)
return jsendify({"url": url}, 201)
except ApiError as e:
url = e.resp.headers.get("Location")
_fail_pr(repo, pr_num, params["GIT_SHA"], url, token)
raise
except Exception:
_fail_pr(repo, pr_num, params["GIT_SHA"], None, token)
tb = traceback.format_exc()
return "FAILED: %s: %s\n%s" % (repo, pr_num, tb), 500
def project_trigger_list(proj):
permissions.assert_can_build(proj)
p = get_or_404(Project.query.filter_by(name=proj))
triggers = p.triggers
t = request.args.get("type")
if t:
triggers = [x for x in triggers if x.type == TriggerTypes[t].value]
# Remove the secret values, no need to ever expose them
redacted = []
for t in triggers:
data = t.as_json()
data["secrets"] = [{"name": x} for x in (data.get("secrets") or {}).keys()]
redacted.append(data)
return jsendify(redacted)
def build_promote(proj, build_id):
permissions.assert_can_promote(proj, build_id)
p = get_or_404(Project.query.filter_by(name=proj))
b = get_or_404(Build.query.filter_by(project=p, build_id=build_id))
if not b.complete:
raise ApiError(400, "Build is not yet complete")
data = request.get_json()
if not data:
raise ApiError(400, "Input data must be JSON")
b.status = BuildStatus.PROMOTED
b.name = data.get("name")
b.annotation = data.get("annotation")
db.session.commit()
return jsendify({}, 201)
def worker_create(name):
worker = request.get_json() or {}
required = ('api_key', 'distro', 'mem_total', 'cpu_total', 'cpu_type',
'concurrent_runs', 'host_tags')
missing = []
for x in required:
if x not in worker:
missing.append(x)
if missing:
raise ApiError(400, 'Missing required field(s): ' + ', '.join(missing))
w = Worker(name, worker['distro'], worker['mem_total'],
worker['cpu_total'], worker['cpu_type'], worker['api_key'],
worker['concurrent_runs'], worker['host_tags'])
w.surges_only = worker.get('surges_only', False)
db.session.add(w)
db.session.commit()
return jsendify({}, 201)
def run_health():
health = {}
# get an overall count for each run state
vals = db.session.query(Run.status,
func.count(Run.status)).group_by(Run.status)
health["statuses"] = {
BuildStatus(status).name: count
for status, count in vals
}
# now give some details about what's queued and what's running
health["RUNNING"] = {}
health["QUEUED"] = []
active = (
BuildStatus.QUEUED,
BuildStatus.RUNNING,
BuildStatus.UPLOADING,
BuildStatus.CANCELLING,
)
runs = Run.query.filter(Run.status.in_(active)).order_by(
Run.queue_priority.asc(), Run.build_id.asc(), Run.id.asc())
for run in runs:
url = url_for(
"api_run.run_get",
proj=run.build.project.name,
build_id=run.build.build_id,
run=run.name,
_external=True,
)
item = {
"project": run.build.project.name,
"build": run.build.build_id,
"run": run.name,
"url": url,
"created": run.build.status_events[0].time,
}
if run.status == BuildStatus.QUEUED:
health["QUEUED"].append(item)
else:
worker = run.worker_name or "?"
health["RUNNING"].setdefault(worker, []).append(item)
return jsendify({"health": health})
def on_webhook(proj):
trigger = _find_trigger(proj)
event = request.headers["X-Gitlab-Event"]
_filter_events(event)
data = request.get_json()
mr_actions = ("open", "reopen", "update")
if event == "Note Hook":
if "ci-retest" not in data["object_attributes"]["note"]:
return "Ingoring comment"
elif data["object_attributes"]["action"] not in mr_actions:
return "Ingoring Merge Request action"
params = _get_params(data)
reason = "GitLab MR: " + params["GL_MR"]
secrets = trigger.secret_data
if "gitlabtok" not in secrets or "gitlabuser" not in secrets:
raise ApiError(
400, 'Trigger secrets is missing "gitlabtok" or "gitlabuser"')
token = secrets["gitlabtok"]
try:
_set_base_sha(params, token)
trig, proj = _get_proj_def(trigger, token, params)
b = trigger_build(trigger.project, reason, trig, params, secrets, proj,
trigger.queue_priority)
_update_pr(b, params["GL_STATUS_URL"], token)
url = url_for(
"api_build.build_get",
proj=trigger.project.name,
build_id=b.build_id,
_external=True,
)
return jsendify({"url": url}, 201)
except ApiError as e:
url = e.resp.headers.get("Location")
_fail_pr(params, token, url)
raise
except Exception:
_fail_pr(params, token, None)
tb = traceback.format_exc()
return "FAILED:\n" + tb, 500
def project_delete(proj):
permissions.assert_can_delete(proj)
p = get_or_404(Project.query.filter_by(name=proj))
if request.get_json().get('I_REALLY_MEAN_TO_DO_THIS') != 'YES':
raise ApiError(401, 'Missing required parameter: "name"')
for t in p.triggers:
db.session.delete(t)
db.session.commit()
for b in p.builds:
db.session.delete(b)
db.session.commit()
db.session.delete(p)
db.session.commit()
return jsendify({'TODO': 'Delete storage artifacts'})
def on_webhook(proj):
trigger = _find_trigger(proj)
event = request.headers.get('X-Github-Event')
_filter_events(event)
data = request.get_json()
if event == 'issue_comment':
if 'ci-retest' not in request.json['comment']['body']:
return 'Ingoring comment'
pr_num = data['issue']['number']
repo = data['repository']['full_name']
elif event == 'pull_request':
if data['action'] not in ('opened', 'synchronize'):
return 'Ignoring action: ' + request.json['action']
pr_num = data['pull_request']['number']
repo = data['pull_request']['base']['repo']['full_name']
reason = 'GitHub PR(%s): %s, https://github.com/%s/pull/%d' % (
pr_num, event, repo, pr_num)
secrets = trigger.secret_data
token = secrets['githubtok']
owner, repo = repo.split('/')
params = _get_params(owner, repo, pr_num, token)
try:
trig, proj = _get_proj_def(trigger, owner, repo, params['GIT_SHA'],
token)
b = trigger_build(trigger.project, reason, trig, params, secrets, proj,
trigger.queue_priority)
_update_pr(b, params['GH_STATUS_URL'], token)
url = url_for('api_build.build_get',
proj=trigger.project.name,
build_id=b.build_id,
_external=True)
return jsendify({'url': url}, 201)
except ApiError as e:
url = e.resp.headers.get('Location')
_fail_pr(repo, pr_num, params['GIT_SHA'], url, token)
raise
except Exception:
_fail_pr(repo, pr_num, params['GIT_SHA'], None, token)
tb = traceback.format_exc()
return 'FAILED: %s: %s\n%s' % (repo, pr_num, tb), 500
def on_webhook(proj):
trigger = _find_trigger(proj)
event = request.headers['X-Gitlab-Event']
_filter_events(event)
data = request.get_json()
mr_actions = ('open', 'reopen', 'update')
if event == 'Note Hook':
if 'ci-retest' not in data['object_attributes']['note']:
return 'Ingoring comment'
elif data['object_attributes']['action'] not in mr_actions:
return 'Ingoring Merge Request action'
params = _get_params(data)
reason = 'GitLab MR: ' + params['GL_MR']
secrets = trigger.secret_data
if 'gitlabtok' not in secrets or 'gitlabuser' not in secrets:
raise ApiError(
400, 'Trigger secrets is missing "gitlabtok" or "gitlabuser"')
token = secrets['gitlabtok']
try:
_set_base_sha(params, token)
trig, proj = _get_proj_def(trigger, token, params)
b = trigger_build(trigger.project, reason, trig, params, secrets, proj,
trigger.queue_priority)
_update_pr(b, params['GL_STATUS_URL'], token)
url = url_for('api_build.build_get',
proj=trigger.project.name,
build_id=b.build_id,
_external=True)
return jsendify({'url': url}, 201)
except ApiError as e:
url = e.resp.headers.get('Location')
_fail_pr(params, token, url)
raise
except Exception:
_fail_pr(params, token, None)
tb = traceback.format_exc()
return 'FAILED:\n' + tb, 500
def test_create(proj, build_id, run, test):
r = _get_run(proj, build_id, run)
_authenticate_runner(r)
context = ""
status = results = None
json = request.get_json()
if json:
context = json.get("context")
status = json.get("status")
results = json.get("results")
t = Test(r, test, context)
db.session.add(t)
if status:
t.status = status
if results:
db.session.flush()
for tr in results:
create_test_result(t, tr)
db.session.commit()
return jsendify({})
def worker_update(name):
data = request.get_json() or {}
attrs = (
"distro",
"mem_total",
"cpu_total",
"cpu_type",
"concurrent_runs",
"host_tags",
)
for attr in attrs:
val = data.get(attr)
if val is not None:
if attr == "host_tags" and request.worker.allowed_tags:
# make sure the worker isn't try to access things it shouldn't
rejects = set(val.split(",")) - set(
request.worker.allowed_tags)
if rejects:
raise ApiError(
403,
f"Worker not allowed access to host_tags: {rejects}")
setattr(request.worker, attr, val)
db.session.commit()
return jsendify({}, 200)
def wrapper(*args, **kwargs):
key = request.headers.get("Authorization", None)
if not key:
return jsendify("No Authorization header provided", 401)
parts = key.split(" ")
if len(parts) != 2 or parts[0] not in ("Token", "Bearer"):
return jsendify("Invalid Authorization header", 401)
if parts[0] == "Bearer":
try:
w = worker_from_jwt(parts[1])
except PyJWTError as e:
return jsendify(str(e), 401)
if w.name != kwargs["name"]:
# worker can only access its self
return jsendify("Not found", 404)
worker = Worker.query.filter(Worker.name == w.name).first()
if worker is None:
# This looks a little nutty - constructing this object with
# basically "I have no idea" data. But the worker will call
# us with `worker_update` on its first connection which will
# fill these handy but not mission-cricital fields out.
worker = Worker(w.name, "?", 1, 1, "?", "", 1, w.allowed_tags)
worker.enlisted = True
db.session.add(worker)
db.session.commit()
elif worker.deleted:
return jsendify("Not found", 404)
worker.allowed_tags = w.allowed_tags
else:
worker = get_or_404(
Worker.query.filter_by(name=kwargs["name"], deleted=False))
if not worker.validate_api_key(parts[1]):
return jsendify("Incorrect API key for host", 401)
worker.allowed_tags = []
request.worker = worker
return f(*args, **kwargs)