def _validate_payload(trigger):
key = trigger.secret_data.get("webhook-key")
if not key:
raise ApiError(403, "Trigger has no webhook-key secret defined")
if not hmac.compare_digest(key, request.headers["X-Gitlab-Token"]):
raise ApiError(403, "Invalid X-Gitlab-Token")
def build_create(proj):
u = permissions.assert_can_build(proj)
p = Project.query.filter(Project.name == proj).first_or_404()
d = request.get_json() or {}
secrets = {}
# Check if the caller wants to inherit secrets from something like the
# "git-poller" trigger for the project.
trigger_type = d.get("trigger-type")
if trigger_type:
optional = trigger_type.endswith("-optional")
if optional:
trigger_type = trigger_type[:-9] # strip off the "-optional"
for t in p.triggers:
if TriggerTypes(t.type).name == trigger_type:
secrets.update(t.secret_data)
break
else:
if not optional:
raise ApiError(400, "No such trigger-type: %s" % trigger_type)
# Check if the caller wants to inherit secrets from a specific trigger
# definied for the project.
trigger_id = d.get("trigger-id")
if trigger_id:
for t in p.triggers:
if t.id == trigger_id:
secrets.update(t.secret_data)
break
else:
raise ApiError(400, "Unknown trigger-id: %s" % trigger_id)
secrets.update(d.get("secrets") or {})
if "triggered-by" in secrets:
# Let's not allow triggered-by to be set externally.
del secrets["triggered-by"]
if u:
secrets["triggered-by"] = str(u)
b = trigger_build(
p,
d.get("reason"),
d.get("trigger-name"),
d.get("params"),
secrets,
d.get("project-definition"),
d.get("queue-priority", 0),
)
url = url_for("api_build.build_get",
proj=p.name,
build_id=b.build_id,
_external=True)
weburl = None
if BUILD_URL_FMT:
weburl = BUILD_URL_FMT.format(project=b.project.name, build=b.build_id)
return jsendify({
"url": url,
"build_id": b.build_id,
"web_url": weburl
}, 201)
def _validate_payload(trigger):
key = trigger.secret_data.get('webhook-key')
if not key:
raise ApiError(403, 'Trigger has no webhook-key secret defined')
if not hmac.compare_digest(key, request.headers['X-Gitlab-Token']):
raise ApiError(403, 'Invalid X-Gitlab-Token')
def _filter_events(event):
ignores = ('fork', 'ping', 'push', 'status', 'pull_request_review',
'pull_request_review_comment')
events = ignores + ('issue_comment', 'pull_request')
if event not in events:
raise ApiError(400, 'Invalid action: ' + event)
if event in ignores:
raise ApiError(200, 'OK, ignoring')
def simulator_validate():
data = request.get_json()
if not data:
raise ApiError(400, "run-definition must be posted as json data")
try:
ProjectDefinition.validate_data(data)
except Exception as e:
raise ApiError(400, str(e))
return jsendify({})
def _validate_payload(trigger):
key = trigger.secret_data.get("webhook-key")
if not key:
raise ApiError(403, "Trigger has no webhook-key secret defined")
computed = hmac.new(key.encode(), request.data, "sha1").hexdigest()
delivered = request.headers.get("X_HUB_SIGNATURE")
if not delivered or not delivered.startswith("sha1="):
raise ApiError(404, "Missing or invalid X_HUB_SIGNATURE header")
if not (hmac.compare_digest(computed, delivered[5:])):
raise ApiError(403, "Invalid X_HUB_SIGNATURE")
def _validate_payload(trigger):
key = trigger.secret_data.get('webhook-key')
if not key:
raise ApiError(403, 'Trigger has no webhook-key secret defined')
computed = hmac.new(key.encode(), request.data, 'sha1').hexdigest()
delivered = request.headers.get('X_HUB_SIGNATURE')
if not delivered or not delivered.startswith('sha1='):
raise ApiError(404, 'Missing or invalid X_HUB_SIGNATURE header')
if not (hmac.compare_digest(computed, delivered[5:])):
raise ApiError(403, 'Invalid X_HUB_SIGNATURE')
def build_create(proj):
u = permissions.assert_can_build(proj)
p = Project.query.filter(Project.name == proj).first_or_404()
d = request.get_json() or {}
secrets = {}
# Check if the caller wants to inherit secrets from something like the
# "git-poller" trigger for the project.
trigger_type = d.get('trigger-type')
if trigger_type:
optional = trigger_type.endswith('-optional')
if optional:
trigger_type = trigger_type[:-9] # strip off the "-optional"
for t in p.triggers:
if TriggerTypes(t.type).name == trigger_type:
secrets = t.secret_data
break
else:
if not optional:
raise ApiError(400, 'No such trigger-type: %s' % trigger_type)
# Check if the caller wants to inherit secrets from a specific trigger
# definied for the project.
trigger_id = d.get('trigger-id')
if trigger_id:
for t in p.triggers:
if t.id == trigger_id:
secrets = t.secret_data
break
else:
raise ApiError(400, 'Unknown trigger-id: %s' % trigger_id)
secrets.update(d.get('secrets') or {})
if 'triggered-by' in secrets:
# Let's not allow triggered-by to be set externally.
del secrets['triggered-by']
if u:
secrets['triggered-by'] = str(u)
b = trigger_build(p, d.get('reason'), d.get('trigger-name'),
d.get('params'), secrets,
d.get('project-definition'),
d.get('queue-priority', 0))
url = url_for('api_build.build_get',
proj=p.name, build_id=b.build_id, _external=True)
weburl = None
if BUILD_URL_FMT:
weburl = BUILD_URL_FMT.format(
project=b.project.name, build=b.build_id)
return jsendify(
{'url': url, 'build_id': b.build_id, 'web_url': weburl}, 201)
def _filter_events(event):
ignores = (
"fork",
"ping",
"push",
"status",
"pull_request_review",
"pull_request_review_comment",
)
events = ignores + ("issue_comment", "pull_request")
if event not in events:
raise ApiError(400, "Invalid action: " + event)
if event in ignores:
raise ApiError(200, "OK, ignoring")
def _authenticate_runner(run):
key = request.args.get('apikey')
if key and key == run.api_key:
return
key = request.headers.get('Authorization', None)
if not key:
raise ApiError(401, {'message': 'No Authorization header provided'})
parts = key.split(' ')
if len(parts) != 2 or parts[0] != 'Token':
raise ApiError(401, {'message': 'Invalid Authorization header'})
if parts[1] != run.api_key:
raise ApiError(401, {'message': 'Incorrect API key'})
if run.complete:
raise ApiError(401, {'message': 'Run has already completed'})
def _authenticate_runner(run):
key = request.args.get("apikey")
if key and key == run.api_key:
return
key = request.headers.get("Authorization", None)
if not key:
raise ApiError(401, {"message": "No Authorization header provided"})
parts = key.split(" ")
if len(parts) != 2 or parts[0] != "Token":
raise ApiError(401, {"message": "Invalid Authorization header"})
if parts[1] != run.api_key:
raise ApiError(401, {"message": "Incorrect API key"})
if run.complete:
raise ApiError(401, {"message": "Run has already completed"})
def worker_create(name):
worker = request.get_json() or {}
required = (
"api_key",
"distro",
"mem_total",
"cpu_total",
"cpu_type",
"concurrent_runs",
"host_tags",
)
missing = []
for x in required:
if x not in worker:
missing.append(x)
if missing:
raise ApiError(400, "Missing required field(s): " + ", ".join(missing))
w = Worker(
name,
worker["distro"],
worker["mem_total"],
worker["cpu_total"],
worker["cpu_type"],
worker["api_key"],
worker["concurrent_runs"],
worker["host_tags"],
)
w.surges_only = worker.get("surges_only", False)
db.session.add(w)
db.session.commit()
return jsendify({}, 201)
def smtp_session():
s = smtplib.SMTP(SMTP_SERVER, 587)
rv, msg = s.starttls()
if rv != 220:
log.error('Unable to connect to SMTP server %s: %d %s', SMTP_SERVER,
rv, msg.decode())
raise ApiError(500, 'Unable to connect to SMTP server')
rv, msg = s.login(SMTP_USER, SMTP_PASSWORD)
if rv != 235:
log.error('Unable to authenticate with SMTP server %s: %d %s',
SMTP_SERVER, rv, msg.decode())
raise ApiError(500, 'Unable to authenticate with SMTP server')
try:
yield s
finally:
s.quit()
def run_get_progress_regex(proj, build_id, run):
r = _get_run(proj, build_id, run)
rundef = json.loads(Storage().get_run_definition(r))
progress = rundef.get('console-progress')
if progress:
return jsendify(progress)
raise ApiError(404, {'message': 'Run has not defined console-progress'})
def assert_internal_user():
'''A function that checks request headers to ensure the caller is a valid
internal user.'''
if not INTERNAL_API_KEY:
raise RuntimeError('JobServ missing INTERNAL_API_KEY')
sig = request.headers.get('X-JobServ-Sig')
ts = request.headers.get('X-Time')
if not sig:
raise ApiError(401, 'X-JobServ-Sig not provided')
if not ts:
raise ApiError(401, 'X-Time not provided')
msg = '%s,%s,%s' % (request.method, ts, request.base_url)
computed = hmac.new(INTERNAL_API_KEY, msg.encode(), 'sha1').hexdigest()
if not hmac.compare_digest(sig, computed):
raise ApiError(401, 'Invalid signature')
def assert_internal_user():
"""A function that checks request headers to ensure the caller is a valid
internal user."""
if not INTERNAL_API_KEY:
raise RuntimeError("JobServ missing INTERNAL_API_KEY")
sig = request.headers.get("X-JobServ-Sig")
ts = request.headers.get("X-Time")
if not sig:
raise ApiError(401, "X-JobServ-Sig not provided")
if not ts:
raise ApiError(401, "X-Time not provided")
msg = "%s,%s,%s" % (request.method, ts, request.base_url)
computed = hmac.new(INTERNAL_API_KEY, msg.encode(), "sha1").hexdigest()
if not hmac.compare_digest(sig, computed):
raise ApiError(401, "Invalid signature")
def _fail_unexpected(build, exception):
r = Run(build, "build-failure")
db.session.add(r)
r.set_status(BuildStatus.FAILED)
db.session.commit()
storage = Storage()
with storage.console_logfd(r, "a") as f:
f.write("Unexpected error prevented build from running:\n")
f.write(str(exception))
storage.copy_log(r)
if BUILD_URL_FMT:
url = BUILD_URL_FMT.format(project=build.project.name,
build=build.build_id)
else:
url = url_for(
"api_run.run_get_artifact",
proj=build.project.name,
build_id=build.build_id,
run=r.name,
path="console.log",
)
exception = ApiError(500, str(exception))
exception.resp.headers.extend({"Location": url})
return exception
def project_create_trigger(proj):
u = permissions.assert_internal_user()
p = get_or_404(Project.query.filter_by(name=proj))
d = request.get_json() or {}
ttype = d.pop('type')
if not ttype:
raise ApiError(401, 'Missing parameter: type')
ttype = TriggerTypes[ttype].value
try:
owner = d.pop('owner')
except KeyError:
owner = 'unknown-internal'
if u:
owner = str(u)
dr = df = None
try:
dr = d.pop('definition_repo')
df = d.pop('definition_file')
except KeyError:
pass
try:
secrets = d.pop('secrets')
# convert array of [{'name': <name>, 'value': <value>}, ...] dicts
# into dictionary of {<name>: <value>, ...}
secrets = {x['name']: x['value'] for x in secrets}
except KeyError:
secrets = d
db.session.add(ProjectTrigger(owner, ttype, p, dr, df, secrets))
db.session.commit()
return jsendify({}, 201)
def _register_github_hook(project, url, api_token, hook_token):
data = {
"name": "web",
"active": True,
"events": [
"pull_request",
"pull_request_review_comment",
"issue_comment",
],
"config": {
"url":
url_for("api_github.on_webhook", proj=project.name,
_external=True),
"content_type":
"json",
"secret":
hook_token,
},
}
headers = {
"Content-Type": "application/json",
"Authorization": "token " + api_token,
}
resp = requests.post(url, json=data, headers=headers)
if resp.status_code != 201:
raise ApiError(resp.status_code, resp.json())
def _get_params(owner, repo, pr_num, token):
headers = {
"Content-Type": "application/json",
"Authorization": "token " + token,
}
url = "https://api.github.com/repos/%s/%s/pulls/%d" % (owner, repo, pr_num)
for x in range(5):
r = requests.get(url, headers=headers)
if r.status_code == 200:
try:
data = r.json()
return {
"GH_PRNUM": int(pr_num),
"GH_OWNER": owner,
"GH_REPO": repo,
"GH_STATUS_URL": data["statuses_url"],
"GH_TARGET_REPO": data["base"]["repo"]["clone_url"],
"GIT_URL": data["head"]["repo"]["clone_url"],
"GIT_SHA_BASE": data["base"]["sha"],
"GIT_OLD_SHA": data["base"]["sha"],
"GIT_SHA": data["head"]["sha"],
}
except Exception:
logging.error("Error finding SHA: %d - %s", r.status_code,
r.text)
time.sleep(0.2)
raise ApiError(500, "Error finding SHA: %d - %s" % (r.status_code, r.text))
def run_get_artifact(proj, build_id, run, path):
r = _get_run(proj, build_id, run)
if r.complete:
storage = Storage()
if path.endswith('.html'):
# we are probably trying to render a static site like a build of
# ltd-docs. Return its content rather than a redirect so it will
# render in the browser
content = storage.get_artifact_content(r, path)
return content, 200, {'Content-Type': 'text/html'}
resp = storage.get_download_response(request, r, path)
resp.headers['X-RUN-STATUS'] = r.status.name
return resp
if path != 'console.log':
raise ApiError(
404, {'message': 'Run in progress, no artifacts available'})
if r.status == BuildStatus.QUEUED:
msg = '# Waiting for worker with tag: ' + r.host_tag
return (msg, 200,
{'Content-Type': 'text/plain', 'X-RUN-STATUS': r.status.name})
try:
fd = Storage().console_logfd(r, 'rb')
offset = request.headers.get('X-OFFSET')
if offset:
fd.seek(int(offset), 0)
resp = make_response(send_file(fd, mimetype='text/plain'))
resp.headers['X-RUN-STATUS'] = r.status.name
return resp
except FileNotFoundError:
# This is a race condition. The run completed while we were checking
return Storage().get_download_response(request, r, path)
def authenticate(clazz, request_headers):
auth = request_headers.get('Authorization')
if auth:
if auth and auth.startswith('Bearer '):
return clazz._authenticate_bearer(auth.split(' ', 1)[1])
else:
ApiError('400', 'Invalid Authorization header')
def _register_github_hook(project, url, api_token, hook_token):
data = {
'name': 'web',
'active': True,
'events': [
'pull_request',
'pull_request_review_comment',
'issue_comment',
],
'config': {
'url':
url_for('api_github.on_webhook', proj=project.name,
_external=True),
'content_type':
'json',
'secret':
hook_token,
}
}
headers = {
'Content-Type': 'application/json',
'Authorization': 'token ' + api_token,
}
resp = requests.post(url, json=data, headers=headers)
if resp.status_code != 201:
raise ApiError(resp.status_code, resp.json())
def _get_params(owner, repo, pr_num, token):
headers = {
'Content-Type': 'application/json',
'Authorization': 'token ' + token,
}
url = 'https://api.github.com/repos/%s/%s/pulls/%d' % (owner, repo, pr_num)
for x in range(5):
r = requests.get(url, headers=headers)
if r.status_code == 200:
try:
data = r.json()
return {
'GH_PRNUM': int(pr_num),
'GH_OWNER': owner,
'GH_REPO': repo,
'GH_STATUS_URL': data['statuses_url'],
'GH_TARGET_REPO': data['base']['repo']['clone_url'],
'GIT_URL': data['head']['repo']['clone_url'],
'GIT_SHA_BASE': data['base']['sha'],
'GIT_SHA': data['head']['sha'],
}
except Exception:
logging.error('Error finding SHA: %d - %s',
r.status_code, r.text)
time.sleep(0.2)
raise ApiError(500, 'Error finding SHA: %d - %s' % (r.status_code, r.text))
def _authenticate_bearer(bearer):
try:
bearer = jwt.decode(bearer, _jwt_secret(), algorithms=['HS256'])
return User(bearer['login'], bearer['email'], bearer['name'],
bearer['is_admin'])
except jwt.PyJWTError as e:
raise ApiError(401, str(e))
def create_webhook(proj):
u = permissions.assert_internal_user()
p = get_or_404(Project.query.filter_by(name=proj))
d = request.get_json() or {}
required = ('githubtok', 'owner', 'project')
missing = []
for x in required:
if x not in d:
missing.append(x)
if missing:
raise ApiError(401, 'Missing parameters: %s' % ','.join(missing))
owner = d.pop('owner')
hook_url = 'https://api.github.com/repos/%s/%s/hooks'
hook_url = hook_url % (owner, d.pop('project'))
d['webhook-key'] = secrets.token_urlsafe()
dr = df = None
try:
dr = d.pop('definition_repo')
df = d.pop('definition_file')
except KeyError:
pass
user = owner
if u:
user = str(u)
db.session.add(ProjectTrigger(
user, TriggerTypes.github_pr.value, p, dr, df, d))
_register_github_hook(p, hook_url, d['githubtok'], d['webhook-key'])
db.session.commit()
return jsendify({}, 201)
def build_promote(proj, build_id):
permissions.assert_can_promote(proj, build_id)
p = get_or_404(Project.query.filter_by(name=proj))
b = get_or_404(Build.query.filter_by(project=p, build_id=build_id))
if not b.complete:
raise ApiError(400, "Build is not yet complete")
data = request.get_json()
if not data:
raise ApiError(400, "Input data must be JSON")
b.status = BuildStatus.PROMOTED
b.name = data.get("name")
b.annotation = data.get("annotation")
db.session.commit()
return jsendify({}, 201)
def test_find():
permissions.assert_internal_user()
context = request.args.get('context')
if not context:
raise ApiError(401, {'message': 'Missing "context" query argument'})
tests = []
for t in Test.query.filter_by(context=context):
tests.append(t.as_json(detailed=True))
tests[-1]['metadata'] = t.run.meta
tests[-1]['api_key'] = t.run.api_key
return jsendify({'tests': tests})
def _set_base_sha(params, token):
url = params["GL_MR_API"] + "/versions"
headers = {
"Content-Type": "application/json",
"PRIVATE-TOKEN": token,
}
r = requests.get(url, headers=headers)
if r.status_code == 200:
params["GIT_SHA_BASE"] = r.json()[0]["base_commit_sha"]
else:
raise ApiError(
r.status_code,
"Unable to find base commit from %s:\n%s" % (url, r.text))
def project_create():
d = request.get_json() or {}
proj = d.get('name')
if not proj:
raise ApiError(401, 'Missing required parameter: "name"')
sync = d.get('synchronous-builds', False)
permissions.assert_internal_user()
db.session.add(Project(proj, sync))
db.session.commit()
url = url_for('api_project.project_get', proj=proj, _external=True)
return jsendify({'url': url}, 201)
