def _validate_payload(trigger): key = trigger.secret_data.get("webhook-key") if not key: raise ApiError(403, "Trigger has no webhook-key secret defined") if not hmac.compare_digest(key, request.headers["X-Gitlab-Token"]): raise ApiError(403, "Invalid X-Gitlab-Token")
def build_create(proj): u = permissions.assert_can_build(proj) p = Project.query.filter(Project.name == proj).first_or_404() d = request.get_json() or {} secrets = {} # Check if the caller wants to inherit secrets from something like the # "git-poller" trigger for the project. trigger_type = d.get("trigger-type") if trigger_type: optional = trigger_type.endswith("-optional") if optional: trigger_type = trigger_type[:-9] # strip off the "-optional" for t in p.triggers: if TriggerTypes(t.type).name == trigger_type: secrets.update(t.secret_data) break else: if not optional: raise ApiError(400, "No such trigger-type: %s" % trigger_type) # Check if the caller wants to inherit secrets from a specific trigger # definied for the project. trigger_id = d.get("trigger-id") if trigger_id: for t in p.triggers: if t.id == trigger_id: secrets.update(t.secret_data) break else: raise ApiError(400, "Unknown trigger-id: %s" % trigger_id) secrets.update(d.get("secrets") or {}) if "triggered-by" in secrets: # Let's not allow triggered-by to be set externally. del secrets["triggered-by"] if u: secrets["triggered-by"] = str(u) b = trigger_build( p, d.get("reason"), d.get("trigger-name"), d.get("params"), secrets, d.get("project-definition"), d.get("queue-priority", 0), ) url = url_for("api_build.build_get", proj=p.name, build_id=b.build_id, _external=True) weburl = None if BUILD_URL_FMT: weburl = BUILD_URL_FMT.format(project=b.project.name, build=b.build_id) return jsendify({ "url": url, "build_id": b.build_id, "web_url": weburl }, 201)
def _validate_payload(trigger): key = trigger.secret_data.get('webhook-key') if not key: raise ApiError(403, 'Trigger has no webhook-key secret defined') if not hmac.compare_digest(key, request.headers['X-Gitlab-Token']): raise ApiError(403, 'Invalid X-Gitlab-Token')
def _filter_events(event): ignores = ('fork', 'ping', 'push', 'status', 'pull_request_review', 'pull_request_review_comment') events = ignores + ('issue_comment', 'pull_request') if event not in events: raise ApiError(400, 'Invalid action: ' + event) if event in ignores: raise ApiError(200, 'OK, ignoring')
def simulator_validate(): data = request.get_json() if not data: raise ApiError(400, "run-definition must be posted as json data") try: ProjectDefinition.validate_data(data) except Exception as e: raise ApiError(400, str(e)) return jsendify({})
def _validate_payload(trigger): key = trigger.secret_data.get("webhook-key") if not key: raise ApiError(403, "Trigger has no webhook-key secret defined") computed = hmac.new(key.encode(), request.data, "sha1").hexdigest() delivered = request.headers.get("X_HUB_SIGNATURE") if not delivered or not delivered.startswith("sha1="): raise ApiError(404, "Missing or invalid X_HUB_SIGNATURE header") if not (hmac.compare_digest(computed, delivered[5:])): raise ApiError(403, "Invalid X_HUB_SIGNATURE")
def _validate_payload(trigger): key = trigger.secret_data.get('webhook-key') if not key: raise ApiError(403, 'Trigger has no webhook-key secret defined') computed = hmac.new(key.encode(), request.data, 'sha1').hexdigest() delivered = request.headers.get('X_HUB_SIGNATURE') if not delivered or not delivered.startswith('sha1='): raise ApiError(404, 'Missing or invalid X_HUB_SIGNATURE header') if not (hmac.compare_digest(computed, delivered[5:])): raise ApiError(403, 'Invalid X_HUB_SIGNATURE')
def build_create(proj): u = permissions.assert_can_build(proj) p = Project.query.filter(Project.name == proj).first_or_404() d = request.get_json() or {} secrets = {} # Check if the caller wants to inherit secrets from something like the # "git-poller" trigger for the project. trigger_type = d.get('trigger-type') if trigger_type: optional = trigger_type.endswith('-optional') if optional: trigger_type = trigger_type[:-9] # strip off the "-optional" for t in p.triggers: if TriggerTypes(t.type).name == trigger_type: secrets = t.secret_data break else: if not optional: raise ApiError(400, 'No such trigger-type: %s' % trigger_type) # Check if the caller wants to inherit secrets from a specific trigger # definied for the project. trigger_id = d.get('trigger-id') if trigger_id: for t in p.triggers: if t.id == trigger_id: secrets = t.secret_data break else: raise ApiError(400, 'Unknown trigger-id: %s' % trigger_id) secrets.update(d.get('secrets') or {}) if 'triggered-by' in secrets: # Let's not allow triggered-by to be set externally. del secrets['triggered-by'] if u: secrets['triggered-by'] = str(u) b = trigger_build(p, d.get('reason'), d.get('trigger-name'), d.get('params'), secrets, d.get('project-definition'), d.get('queue-priority', 0)) url = url_for('api_build.build_get', proj=p.name, build_id=b.build_id, _external=True) weburl = None if BUILD_URL_FMT: weburl = BUILD_URL_FMT.format( project=b.project.name, build=b.build_id) return jsendify( {'url': url, 'build_id': b.build_id, 'web_url': weburl}, 201)
def _filter_events(event): ignores = ( "fork", "ping", "push", "status", "pull_request_review", "pull_request_review_comment", ) events = ignores + ("issue_comment", "pull_request") if event not in events: raise ApiError(400, "Invalid action: " + event) if event in ignores: raise ApiError(200, "OK, ignoring")
def _authenticate_runner(run): key = request.args.get('apikey') if key and key == run.api_key: return key = request.headers.get('Authorization', None) if not key: raise ApiError(401, {'message': 'No Authorization header provided'}) parts = key.split(' ') if len(parts) != 2 or parts[0] != 'Token': raise ApiError(401, {'message': 'Invalid Authorization header'}) if parts[1] != run.api_key: raise ApiError(401, {'message': 'Incorrect API key'}) if run.complete: raise ApiError(401, {'message': 'Run has already completed'})
def _authenticate_runner(run): key = request.args.get("apikey") if key and key == run.api_key: return key = request.headers.get("Authorization", None) if not key: raise ApiError(401, {"message": "No Authorization header provided"}) parts = key.split(" ") if len(parts) != 2 or parts[0] != "Token": raise ApiError(401, {"message": "Invalid Authorization header"}) if parts[1] != run.api_key: raise ApiError(401, {"message": "Incorrect API key"}) if run.complete: raise ApiError(401, {"message": "Run has already completed"})
def worker_create(name): worker = request.get_json() or {} required = ( "api_key", "distro", "mem_total", "cpu_total", "cpu_type", "concurrent_runs", "host_tags", ) missing = [] for x in required: if x not in worker: missing.append(x) if missing: raise ApiError(400, "Missing required field(s): " + ", ".join(missing)) w = Worker( name, worker["distro"], worker["mem_total"], worker["cpu_total"], worker["cpu_type"], worker["api_key"], worker["concurrent_runs"], worker["host_tags"], ) w.surges_only = worker.get("surges_only", False) db.session.add(w) db.session.commit() return jsendify({}, 201)
def smtp_session(): s = smtplib.SMTP(SMTP_SERVER, 587) rv, msg = s.starttls() if rv != 220: log.error('Unable to connect to SMTP server %s: %d %s', SMTP_SERVER, rv, msg.decode()) raise ApiError(500, 'Unable to connect to SMTP server') rv, msg = s.login(SMTP_USER, SMTP_PASSWORD) if rv != 235: log.error('Unable to authenticate with SMTP server %s: %d %s', SMTP_SERVER, rv, msg.decode()) raise ApiError(500, 'Unable to authenticate with SMTP server') try: yield s finally: s.quit()
def run_get_progress_regex(proj, build_id, run): r = _get_run(proj, build_id, run) rundef = json.loads(Storage().get_run_definition(r)) progress = rundef.get('console-progress') if progress: return jsendify(progress) raise ApiError(404, {'message': 'Run has not defined console-progress'})
def assert_internal_user(): '''A function that checks request headers to ensure the caller is a valid internal user.''' if not INTERNAL_API_KEY: raise RuntimeError('JobServ missing INTERNAL_API_KEY') sig = request.headers.get('X-JobServ-Sig') ts = request.headers.get('X-Time') if not sig: raise ApiError(401, 'X-JobServ-Sig not provided') if not ts: raise ApiError(401, 'X-Time not provided') msg = '%s,%s,%s' % (request.method, ts, request.base_url) computed = hmac.new(INTERNAL_API_KEY, msg.encode(), 'sha1').hexdigest() if not hmac.compare_digest(sig, computed): raise ApiError(401, 'Invalid signature')
def assert_internal_user(): """A function that checks request headers to ensure the caller is a valid internal user.""" if not INTERNAL_API_KEY: raise RuntimeError("JobServ missing INTERNAL_API_KEY") sig = request.headers.get("X-JobServ-Sig") ts = request.headers.get("X-Time") if not sig: raise ApiError(401, "X-JobServ-Sig not provided") if not ts: raise ApiError(401, "X-Time not provided") msg = "%s,%s,%s" % (request.method, ts, request.base_url) computed = hmac.new(INTERNAL_API_KEY, msg.encode(), "sha1").hexdigest() if not hmac.compare_digest(sig, computed): raise ApiError(401, "Invalid signature")
def _fail_unexpected(build, exception): r = Run(build, "build-failure") db.session.add(r) r.set_status(BuildStatus.FAILED) db.session.commit() storage = Storage() with storage.console_logfd(r, "a") as f: f.write("Unexpected error prevented build from running:\n") f.write(str(exception)) storage.copy_log(r) if BUILD_URL_FMT: url = BUILD_URL_FMT.format(project=build.project.name, build=build.build_id) else: url = url_for( "api_run.run_get_artifact", proj=build.project.name, build_id=build.build_id, run=r.name, path="console.log", ) exception = ApiError(500, str(exception)) exception.resp.headers.extend({"Location": url}) return exception
def project_create_trigger(proj): u = permissions.assert_internal_user() p = get_or_404(Project.query.filter_by(name=proj)) d = request.get_json() or {} ttype = d.pop('type') if not ttype: raise ApiError(401, 'Missing parameter: type') ttype = TriggerTypes[ttype].value try: owner = d.pop('owner') except KeyError: owner = 'unknown-internal' if u: owner = str(u) dr = df = None try: dr = d.pop('definition_repo') df = d.pop('definition_file') except KeyError: pass try: secrets = d.pop('secrets') # convert array of [{'name': <name>, 'value': <value>}, ...] dicts # into dictionary of {<name>: <value>, ...} secrets = {x['name']: x['value'] for x in secrets} except KeyError: secrets = d db.session.add(ProjectTrigger(owner, ttype, p, dr, df, secrets)) db.session.commit() return jsendify({}, 201)
def _register_github_hook(project, url, api_token, hook_token): data = { "name": "web", "active": True, "events": [ "pull_request", "pull_request_review_comment", "issue_comment", ], "config": { "url": url_for("api_github.on_webhook", proj=project.name, _external=True), "content_type": "json", "secret": hook_token, }, } headers = { "Content-Type": "application/json", "Authorization": "token " + api_token, } resp = requests.post(url, json=data, headers=headers) if resp.status_code != 201: raise ApiError(resp.status_code, resp.json())
def _get_params(owner, repo, pr_num, token): headers = { "Content-Type": "application/json", "Authorization": "token " + token, } url = "https://api.github.com/repos/%s/%s/pulls/%d" % (owner, repo, pr_num) for x in range(5): r = requests.get(url, headers=headers) if r.status_code == 200: try: data = r.json() return { "GH_PRNUM": int(pr_num), "GH_OWNER": owner, "GH_REPO": repo, "GH_STATUS_URL": data["statuses_url"], "GH_TARGET_REPO": data["base"]["repo"]["clone_url"], "GIT_URL": data["head"]["repo"]["clone_url"], "GIT_SHA_BASE": data["base"]["sha"], "GIT_OLD_SHA": data["base"]["sha"], "GIT_SHA": data["head"]["sha"], } except Exception: logging.error("Error finding SHA: %d - %s", r.status_code, r.text) time.sleep(0.2) raise ApiError(500, "Error finding SHA: %d - %s" % (r.status_code, r.text))
def run_get_artifact(proj, build_id, run, path): r = _get_run(proj, build_id, run) if r.complete: storage = Storage() if path.endswith('.html'): # we are probably trying to render a static site like a build of # ltd-docs. Return its content rather than a redirect so it will # render in the browser content = storage.get_artifact_content(r, path) return content, 200, {'Content-Type': 'text/html'} resp = storage.get_download_response(request, r, path) resp.headers['X-RUN-STATUS'] = r.status.name return resp if path != 'console.log': raise ApiError( 404, {'message': 'Run in progress, no artifacts available'}) if r.status == BuildStatus.QUEUED: msg = '# Waiting for worker with tag: ' + r.host_tag return (msg, 200, {'Content-Type': 'text/plain', 'X-RUN-STATUS': r.status.name}) try: fd = Storage().console_logfd(r, 'rb') offset = request.headers.get('X-OFFSET') if offset: fd.seek(int(offset), 0) resp = make_response(send_file(fd, mimetype='text/plain')) resp.headers['X-RUN-STATUS'] = r.status.name return resp except FileNotFoundError: # This is a race condition. The run completed while we were checking return Storage().get_download_response(request, r, path)
def authenticate(clazz, request_headers): auth = request_headers.get('Authorization') if auth: if auth and auth.startswith('Bearer '): return clazz._authenticate_bearer(auth.split(' ', 1)[1]) else: ApiError('400', 'Invalid Authorization header')
def _register_github_hook(project, url, api_token, hook_token): data = { 'name': 'web', 'active': True, 'events': [ 'pull_request', 'pull_request_review_comment', 'issue_comment', ], 'config': { 'url': url_for('api_github.on_webhook', proj=project.name, _external=True), 'content_type': 'json', 'secret': hook_token, } } headers = { 'Content-Type': 'application/json', 'Authorization': 'token ' + api_token, } resp = requests.post(url, json=data, headers=headers) if resp.status_code != 201: raise ApiError(resp.status_code, resp.json())
def _get_params(owner, repo, pr_num, token): headers = { 'Content-Type': 'application/json', 'Authorization': 'token ' + token, } url = 'https://api.github.com/repos/%s/%s/pulls/%d' % (owner, repo, pr_num) for x in range(5): r = requests.get(url, headers=headers) if r.status_code == 200: try: data = r.json() return { 'GH_PRNUM': int(pr_num), 'GH_OWNER': owner, 'GH_REPO': repo, 'GH_STATUS_URL': data['statuses_url'], 'GH_TARGET_REPO': data['base']['repo']['clone_url'], 'GIT_URL': data['head']['repo']['clone_url'], 'GIT_SHA_BASE': data['base']['sha'], 'GIT_SHA': data['head']['sha'], } except Exception: logging.error('Error finding SHA: %d - %s', r.status_code, r.text) time.sleep(0.2) raise ApiError(500, 'Error finding SHA: %d - %s' % (r.status_code, r.text))
def _authenticate_bearer(bearer): try: bearer = jwt.decode(bearer, _jwt_secret(), algorithms=['HS256']) return User(bearer['login'], bearer['email'], bearer['name'], bearer['is_admin']) except jwt.PyJWTError as e: raise ApiError(401, str(e))
def create_webhook(proj): u = permissions.assert_internal_user() p = get_or_404(Project.query.filter_by(name=proj)) d = request.get_json() or {} required = ('githubtok', 'owner', 'project') missing = [] for x in required: if x not in d: missing.append(x) if missing: raise ApiError(401, 'Missing parameters: %s' % ','.join(missing)) owner = d.pop('owner') hook_url = 'https://api.github.com/repos/%s/%s/hooks' hook_url = hook_url % (owner, d.pop('project')) d['webhook-key'] = secrets.token_urlsafe() dr = df = None try: dr = d.pop('definition_repo') df = d.pop('definition_file') except KeyError: pass user = owner if u: user = str(u) db.session.add(ProjectTrigger( user, TriggerTypes.github_pr.value, p, dr, df, d)) _register_github_hook(p, hook_url, d['githubtok'], d['webhook-key']) db.session.commit() return jsendify({}, 201)
def build_promote(proj, build_id): permissions.assert_can_promote(proj, build_id) p = get_or_404(Project.query.filter_by(name=proj)) b = get_or_404(Build.query.filter_by(project=p, build_id=build_id)) if not b.complete: raise ApiError(400, "Build is not yet complete") data = request.get_json() if not data: raise ApiError(400, "Input data must be JSON") b.status = BuildStatus.PROMOTED b.name = data.get("name") b.annotation = data.get("annotation") db.session.commit() return jsendify({}, 201)
def test_find(): permissions.assert_internal_user() context = request.args.get('context') if not context: raise ApiError(401, {'message': 'Missing "context" query argument'}) tests = [] for t in Test.query.filter_by(context=context): tests.append(t.as_json(detailed=True)) tests[-1]['metadata'] = t.run.meta tests[-1]['api_key'] = t.run.api_key return jsendify({'tests': tests})
def _set_base_sha(params, token): url = params["GL_MR_API"] + "/versions" headers = { "Content-Type": "application/json", "PRIVATE-TOKEN": token, } r = requests.get(url, headers=headers) if r.status_code == 200: params["GIT_SHA_BASE"] = r.json()[0]["base_commit_sha"] else: raise ApiError( r.status_code, "Unable to find base commit from %s:\n%s" % (url, r.text))
def project_create(): d = request.get_json() or {} proj = d.get('name') if not proj: raise ApiError(401, 'Missing required parameter: "name"') sync = d.get('synchronous-builds', False) permissions.assert_internal_user() db.session.add(Project(proj, sync)) db.session.commit() url = url_for('api_project.project_get', proj=proj, _external=True) return jsendify({'url': url}, 201)