def get(self): auth = init_saml_auth() try: return redirect(auth.logout()) except Exception as e: logger.error("Could not initiate Single Logout: %s", e) return redirect(get_root_url("global"))
def get(self): auth = init_saml_auth() try: redirect_url = auth.process_slo() except Exception as e: logger.error("Single Logout failed: %s", e) return redirect(get_root_url("global")) errors = auth.get_errors() if len(errors) != 0: logger.error("Single Logout failed: %s", "; ".join(errors)) return redirect(get_root_url("global")) if redirect_url is None: redirect_url = redirect_url = get_root_url("global") response = redirect(redirect_url) response.set_cookie("token", "", expires=0) return response
def post(self): auth = init_saml_auth() auth.process_response() errors = auth.get_errors() logger.info("Request: %s %s", request, request.headers) if len(errors) != 0: logger.error("Authentication failed: %s", "; ".join(errors)) abort(500, "Authentication failed") if not auth.is_authenticated(): logger.error("User returned unauthorized from IdP") abort(401, "Unauthorized") attributes = get_attribute_dict(auth) logger.debug("User data: %s", attributes) email = format_user_field( get_env("INFRABOX_ACCOUNT_SAML_EMAIL_FORMAT"), attributes).lower() # Check if user already exists in database user = g.db.execute_one_dict( """ SELECT id FROM "user" WHERE email = %s """, [email]) if not user: name = format_user_field( get_env("INFRABOX_ACCOUNT_SAML_NAME_FORMAT"), attributes) username = format_user_field( get_env("INFRABOX_ACCOUNT_SAML_USERNAME_FORMAT"), attributes) user = g.db.execute_one_dict( """ INSERT INTO "user" (name, username, email) VALUES (%s, %s, %s) RETURNING id """, [name, username, email]) token = encode_user_token(user["id"]) g.db.commit() redirect_url = get_root_url("global") + "/dashboard/" logger.debug("Redirecting authenticated user to %s", redirect_url) response = redirect(redirect_url) response.set_cookie("token", token) return response
def get(self): ''' Returns the cluster settings ''' github_enabled = os.environ['INFRABOX_GITHUB_ENABLED'] == 'true' o = { 'INFRABOX_GITHUB_ENABLED': github_enabled, 'INFRABOX_SSO_LOGIN_ENABLED': os.environ['INFRABOX_ACCOUNT_SAML_ENABLED'] == 'true', 'INFRABOX_GERRIT_ENABLED': os.environ['INFRABOX_GERRIT_ENABLED'] == 'true', 'INFRABOX_ACCOUNT_SIGNUP_ENABLED': os.environ['INFRABOX_ACCOUNT_SIGNUP_ENABLED'] == 'true', 'INFRABOX_ACCOUNT_LDAP_ENABLED': os.environ['INFRABOX_ACCOUNT_LDAP_ENABLED'] == 'true', 'INFRABOX_ROOT_URL': get_root_url('global'), 'INFRABOX_GENERAL_REPORT_ISSUE_URL': os.environ['INFRABOX_GENERAL_REPORT_ISSUE_URL'], 'INFRABOX_CLUSTER_NAME': os.environ['INFRABOX_CLUSTER_NAME'], 'INFRABOX_LEGAL_PRIVACY_URL': os.environ['INFRABOX_LEGAL_PRIVACY_URL'], 'INFRABOX_LEGAL_TERMS_OF_USE_URL': os.environ['INFRABOX_LEGAL_TERMS_OF_USE_URL'], } if github_enabled: o['INFRABOX_GITHUB_LOGIN_ENABLED'] = os.environ['INFRABOX_GITHUB_LOGIN_ENABLED'] == 'true' return o
def post(self): ''' Create new project ''' user_id = g.token['user']['id'] b = request.get_json() name = b['name'] typ = b['type'] private = b['private'] projects = g.db.execute_one_dict( ''' SELECT COUNT(*) as cnt FROM project p INNER JOIN collaborator co ON p.id = co.project_id AND co.user_id = %s ''', [user_id]) if projects['cnt'] > 50: abort(400, 'too many projects') project = g.db.execute_one_dict( ''' SELECT * FROM project WHERE name = %s ''', [name]) if project: abort(400, 'A project with this name already exists') if typ == 'github': github_repo_name = b.get('github_repo_name', None) if not github_repo_name: abort(400, 'github_repo_name not set') split = github_repo_name.split('/') owner = split[0] repo_name = split[1] user = g.db.execute_one_dict( ''' SELECT github_api_token FROM "user" WHERE id = %s ''', [user_id]) if not user: abort(404) api_token = user['github_api_token'] headers = { "Authorization": "token " + api_token, "User-Agent": "InfraBox" } url = '%s/repos/%s/%s' % (os.environ['INFRABOX_GITHUB_API_URL'], owner, repo_name) # TODO(ib-steffen): allow custom ca bundles r = requests.get(url, headers=headers, verify=False) if r.status_code != 200: abort(400, 'Failed to get github repo') repo = r.json() if not repo['permissions']['admin']: abort(400, 'You are not allowed to connect this repo') r = g.db.execute_one_dict( ''' SELECT * FROM repository WHERE github_id = %s ''', [repo['id']]) if r: abort('Repo already connected') project = g.db.execute_one_dict( ''' INSERT INTO project (name, type, public) VALUES (%s, %s, %s) RETURNING id ''', [name, typ, not private]) project_id = project['id'] g.db.execute( ''' INSERT INTO collaborator (user_id, project_id, role) VALUES (%s, %s, 'Owner') ''', [user_id, project_id]) if typ == 'github': split = github_repo_name.split('/') owner = split[0] repo_name = split[1] clone_url = repo['clone_url'] if repo['private']: clone_url = repo['ssh_url'] g.db.execute( ''' INSERT INTO repository (name, html_url, clone_url, github_id, private, project_id, github_owner) VALUES (%s, %s, %s, %s, %s, %s, %s) ''', [ repo['name'], repo['html_url'], clone_url, repo['id'], repo['private'], project_id, repo['owner']['login'] ]) insecure_ssl = "0" if os.environ[ 'INFRABOX_GENERAL_DONT_CHECK_CERTIFICATES'] == 'true': insecure_ssl = "1" webhook_config = { 'name': "web", 'active': True, 'events': ["create", "delete", "public", "pull_request", "push"], 'config': { 'url': get_root_url('global') + '/github/hook', 'content_type': "json", 'secret': os.environ['INFRABOX_GITHUB_WEBHOOK_SECRET'], 'insecure_ssl': insecure_ssl } } headers = { "Authorization": "token " + api_token, "User-Agent": "InfraBox" } url = '%s/repos/%s/%s/hooks' % ( os.environ['INFRABOX_GITHUB_API_URL'], owner, repo_name) # TODO(ib-steffen): allow custom ca bundles r = requests.post(url, headers=headers, json=webhook_config, verify=False) if r.status_code != 201: abort(400, 'Failed to create github webhook') hook = r.json() g.db.execute( ''' UPDATE repository SET github_hook_id = %s WHERE github_id = %s ''', [hook['id'], repo['id']]) # deploy key key = RSA.generate(2048) private_key = key.exportKey('PEM') public_key = key.publickey().exportKey('OpenSSH') deploy_key_config = { 'title': "InfraBox", 'key': public_key, 'read_only': True } url = '%s/repos/%s/%s/keys' % ( os.environ['INFRABOX_GITHUB_API_URL'], owner, repo_name) # TODO(ib-steffen): allow custom ca bundles r = requests.post(url, headers=headers, json=deploy_key_config, verify=False) if r.status_code != 201: abort(400, 'Failed to create deploy key') g.db.execute( ''' UPDATE repository SET private_key = %s WHERE github_id = %s ''', [private_key, repo['id']]) elif typ == 'gerrit': g.db.execute( ''' INSERT INTO repository (name, private, project_id, html_url, clone_url, github_id) VALUES (%s, false, %s, '', '', 0) ''', [name, project_id]) g.db.commit() # Push updated collaborator and project data to Open Policy Agent opa_push_project_data(g.db) opa_push_collaborator_data(g.db) return OK('Project added')
def handle_job_update(conn, event): job_id = event['job_id'] jobs = execute_sql( conn, ''' SELECT id, state, name, project_id, build_id FROM job WHERE id = %s ''', [job_id]) if not jobs: return False job = jobs[0] project_id = job['project_id'] build_id = job['build_id'] projects = execute_sql( conn, ''' SELECT id, name, type FROM project WHERE id = %s ''', [project_id]) if not projects: return False project = projects[0] if project['type'] != 'github': return False builds = execute_sql( conn, ''' SELECT id, build_number, restart_counter, commit_id FROM build WHERE id = %s ''', [build_id]) if not builds: return False build = builds[0] project_name = project['name'] job_state = job['state'] job_name = job['name'].split(".")[0] commit_sha = build['commit_id'] build_id = build['id'] build_number = build['build_number'] build_restartCounter = build['restart_counter'] # determine github commit state state = 'success' if job_state in ('scheduled', 'running', 'queued'): state = 'pending' if job_state in ('failure', 'skipped', 'killed', 'unstable'): state = 'failure' if job_state == 'error': state = 'error' logger.info("") logger.info("Handle job %s", job_id) logger.info("Setting state to %s", state) token = execute_sql( conn, ''' SELECT github_api_token FROM "user" u INNER JOIN collaborator co ON co.role = 'Owner' AND co.project_id = %s AND co.user_id = u.id ''', [project_id]) if not token: logger.warn("No API token, not updating status") return False github_api_token = token[0]['github_api_token'] github_status_url = execute_sql( conn, ''' SELECT github_status_url FROM "commit" WHERE id = %s AND project_id = %s ''', [commit_sha, project_id])[0]['github_status_url'] ha_mode = os.environ.get('INFRABOX_HA_ENABLED') == 'true' if ha_mode: dashboard_url = get_root_url('global') else: dashboard_url = execute_sql( conn, ''' SELECT root_url FROM cluster WHERE name = 'master' ''', [])[0]['root_url'] target_url = '%s/dashboard/#/project/%s/build/%s/%s/job/%s' % ( dashboard_url, project_name, build_number, build_restartCounter, urllib.quote_plus(job_name).replace('+', '%20')) payload = { "state": state, "target_url": target_url, "description": "InfraBox", "context": "Job: %s" % job_name } headers = { "Authorization": "token " + github_api_token, "User-Agent": "InfraBox" } # TODO(ib-steffen): support ca bundles try: r = requests.post(github_status_url, data=json.dumps(payload), headers=headers, timeout=10, verify=False) if r.status_code != 201: logger.warn("Failed to update github status: %s", r.text) logger.warn(github_status_url) else: logger.info("Successfully updated github status") except Exception as e: logger.warn("Failed to update github status: %s", e) return False return True
def handle_job_update(conn, event): if event['type'] != 'UPDATE': return job_id = event['job_id'] c = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) c.execute( ''' SELECT id, state, name, project_id, build_id FROM job WHERE id = %s ''', [job_id]) job = c.fetchone() c.close() if not job: return project_id = job['project_id'] build_id = job['build_id'] c = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) c.execute( ''' SELECT id, name, type FROM project WHERE id = %s ''', [project_id]) project = c.fetchone() c.close() if not project: return if project['type'] != 'gerrit': return c = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) c.execute( ''' SELECT id, build_number, restart_counter, commit_id FROM build WHERE id = %s ''', [build_id]) build = c.fetchone() c.close() project_name = project['name'] project_id = project['id'] job_state = job['state'] job_name = job['name'] commit_sha = build['commit_id'] build_id = build['id'] build_number = build['build_number'] build_restart_counter = build['restart_counter'] if job_state in ('queued', 'scheduled', 'running'): return gerrit_port = int(get_env('INFRABOX_GERRIT_PORT')) gerrit_hostname = get_env('INFRABOX_GERRIT_HOSTNAME') gerrit_username = get_env('INFRABOX_GERRIT_USERNAME') gerrit_key_filename = get_env('INFRABOX_GERRIT_KEY_FILENAME') ha_mode = os.environ.get('INFRABOX_HA_ENABLED') == 'true' if ha_mode: dashboard_url = get_root_url('global') else: dashboard_url = execute_sql( conn, ''' SELECT root_url FROM cluster WHERE name = 'master' ''', [])[0]['root_url'] client = paramiko.SSHClient() client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(username=gerrit_username, hostname=gerrit_hostname, port=gerrit_port, key_filename=gerrit_key_filename) client.get_transport().set_keepalive(60) project_name_quote = urllib.quote_plus(project_name).replace('+', '%20') build_url = "%s/dashboard/#/project/%s/build/%s/%s" % ( dashboard_url, project_name_quote, build_number, build_restart_counter) c = conn.cursor() c.execute( ''' SELECT count(*) FROM build WHERE build_number = %s AND restart_counter > %s AND project_id = %s''', [build_number, build_restart_counter, project_id]) newer_builds = c.fetchone()[0] c.close() update_vote = True if newer_builds > 0: update_vote = False c = conn.cursor() c.execute( '''SELECT state, count(*) FROM job WHERE build_id = %s GROUP BY state''', [build_id]) states = c.fetchall() c.close() vote = None if len(states) == 1 and states[0][0] == 'finished': # all finished vote = "+1" message = "Build finished: %s" % build_url else: for s in states: if s[0] in ('running', 'scheduled', 'queued'): # still some running vote = "0" message = "Build running: %s" % build_url break elif s[0] != 'finished': # not successful vote = "-1" message = "Build failed: %s" % build_url if (job_name == 'Create Jobs' and vote == '0') or vote in ('-1', '+1'): logger.info('Setting InfraBox=%s for sha=%s', vote, commit_sha) cmd = 'gerrit review --project %s -m "%s" ' % (project_name, message) if update_vote: cmd += '--label InfraBox=%s ' % vote cmd += commit_sha execute_ssh_cmd(client, cmd) client.close()
def post(self, project_id): project = g.db.execute_one_dict( ''' SELECT type FROM project WHERE id = %s ''', [project_id]) if not project: abort(404, 'Project not found') if project['type'] != 'upload': abort(400, 'Project is not of type "upload"') build_id = str(uuid.uuid4()) key = '%s.zip' % build_id stream = request.files['project.zip'].stream storage.upload_project(stream, key) clusters = g.db.execute_many_dict( ''' SELECT root_url FROM cluster WHERE active = true AND enabled = true AND name != %s ''', [os.environ['INFRABOX_CLUSTER_NAME']]) for c in clusters: stream.seek(0) url = '%s/api/v1/projects/%s/upload/%s/' % ( c['root_url'], project_id, build_id) files = {'project.zip': stream} token = encode_project_token(g.token['id'], project_id, 'myproject') headers = {'Authorization': 'bearer ' + token} logger.info('Also uploading to %s', url) # TODO(ib-steffen): allow custom ca bundles r = requests.post(url, files=files, headers=headers, timeout=120, verify=False) if r.status_code != 200: abort(500, "Failed to upload data") build_number = g.db.execute_one_dict( ''' SELECT count(distinct build_number) + 1 AS build_number FROM build AS b WHERE b.project_id = %s ''', [project_id])['build_number'] source_upload_id = g.db.execute_one( ''' INSERT INTO source_upload(filename, project_id, filesize) VALUES (%s, %s, 0) RETURNING ID ''', [key, project_id])[0] g.db.execute( ''' INSERT INTO build (commit_id, build_number, project_id, source_upload_id, id) VALUES (null, %s, %s, %s, %s) ''', [build_number, project_id, source_upload_id, build_id]) definition = { 'build_only': False, 'resources': { 'limits': { 'cpu': 0.5, 'memory': 1024 } } } g.db.execute( ''' INSERT INTO job (id, state, build_id, type, name, project_id, dockerfile, definition, cluster_name) VALUES (gen_random_uuid(), 'queued', %s, 'create_job_matrix', 'Create Jobs', %s, '', %s, %s); ''', [build_id, project_id, json.dumps(definition), None]) project_name = g.db.execute_one( ''' SELECT name FROM project WHERE id = %s ''', [project_id])[0] root_url = get_root_url('global') url = '%s/dashboard/#/project/%s/build/%s/1' % ( root_url, project_name, build_number) data = { 'build': { 'id': build_id, 'number': build_number }, 'url': url } g.db.commit() return OK('successfully started build', data=data)
def get(self, project_id): job_name = request.args.get('job_name', None) filename = request.args.get('filename', None) branch = request.args.get('branch', None) if not filename or not job_name: abort(404) p = g.db.execute_one_dict( ''' SELECT type, name FROM project WHERE id = %s ''', [project_id]) project_type = p['type'] project_name = p['name'] result = None if branch and project_type in ('github', 'gerrit'): result = g.db.execute_one_dict( ''' SELECT j.id, name, c.id commit_id, b.build_number, b.restart_counter, c.branch FROM ( SELECT id, name, unnest(archive) AS archive, definition->>'name' AS realname, start_date, build_id FROM job WHERE project_id = %s ) j JOIN build b ON b.id = j.build_id JOIN "commit" c ON b.commit_id = c.id WHERE archive->>'filename' = %s AND realname = %s AND c.branch = %s ORDER BY start_date DESC; ''', [project_id, filename, job_name, branch]) else: result = g.db.execute_one_dict( ''' SELECT j.id, name, c.id commit_id, b.build_number, b.restart_counter, c.branch FROM ( SELECT id, name, unnest(archive) AS archive, definition->>'name' AS realname, start_date, build_id FROM job WHERE project_id = %s ) j JOIN build b ON b.id = j.build_id LEFT JOIN commit c ON b.commit_id = c.id WHERE archive->>'filename' = %s AND realname = %s ORDER BY start_date DESC; ''', [project_id, filename, job_name]) if not result: abort(404) job_id = result['id'] build_number = '{build_number}.{restart_counter}'.format( build_number=result['build_number'], restart_counter=result['restart_counter']) root_url = get_root_url('global') build_url = '%s/dashboard/#/project/%s/build/%s/%s' % ( root_url, project_name, result['build_number'], result['restart_counter']) job_url = '%s/job/%s' % (build_url, result['name']) headers = { 'Infrabox-Build-Number': build_number, 'Infrabox-Build-Url': build_url, 'Infrabox-Job-Url': job_url, } if result['branch']: headers['Infrabox-Branch'] = result['branch'] if result['commit_id']: headers['Infrabox-Commit'] = result['commit_id'] f = storage.download_archive('%s/%s' % (job_id, filename)) if not f: abort(404) filename = os.path.basename(filename) resp = make_response( send_file(f, attachment_filename=filename, mimetype=mimetypes.guess_type(filename)[0])) resp.headers = headers return resp
def get(self): job_id = g.token['job']['id'] data = {} # get all the job details r = g.db.execute_one(''' SELECT j.name, null, j.dockerfile, p.id, p.type, p.name, b.id, b.commit_id, b.source_upload_id, b.build_number, u.github_api_token, u.username, null, j.type, null, null, j.repo, null, j.state, null, j.env_var, j.env_var_ref, null, null, u.id, j.build_arg, j.deployment, null, b.restart_counter, j.definition FROM job j INNER JOIN build b ON j.build_id = b.id AND j.project_id = b.project_id INNER JOIN collaborator co ON co.project_id = j.project_id AND co.role = 'Owner' INNER JOIN "user" u ON co.user_id = u.id INNER JOIN project p ON co.project_id = p.id WHERE j.id = %s ''', [job_id]) limits = {} definition = r[29] build_only = True if definition: limits = definition['resources']['limits'] build_only = definition.get('build_only', True) data['job'] = { "id": job_id, "name": r[0], "dockerfile": r[2], "build_only": build_only, "type": r[13], "repo": r[16], "state": r[18], "cpu": limits.get('cpu', 1), "memory": limits.get('memory', 1024), "build_arguments": r[25], "definition": r[29] } state = data['job']['state'] if state in ("finished", "error", "failure", "skipped", "killed", "unstable"): abort(409, 'job not running anymore') env_vars = r[20] env_var_refs = r[21] deployments = r[26] data['project'] = { "id": r[3], "type": r[4], "name": r[5], } data['build'] = { "id": r[6], "commit_id": r[7], "source_upload_id": r[8], "build_number": r[9], "restart_counter": r[28] } data['repository'] = { "owner": r[11], "name": None, "github_api_token": r[10], "private": False } data['commit'] = { "branch": None, "tag": None } pull_request_id = None if data['project']['type'] == 'github' or data['project']['type'] == 'gerrit': r = g.db.execute_one(''' SELECT r.clone_url, r.name, r.private FROM repository r WHERE r.project_id = %s ''', [data['project']['id']]) data['repository']['clone_url'] = r[0] data['repository']['name'] = r[1] data['repository']['private'] = r[2] # A regular commit r = g.db.execute_one(''' SELECT c.branch, c.committer_name, c.tag, c.pull_request_id FROM commit c WHERE c.id = %s AND c.project_id = %s ''', [data['build']['commit_id'], data['project']['id']]) data['commit'] = { "id": data['build']['commit_id'], "branch": r[0], "committer_name": r[1], "tag": r[2] } pull_request_id = r[3] if data['project']['type'] == 'upload': r = g.db.execute_one(''' SELECT filename FROM source_upload WHERE id = %s ''', [data['build']['source_upload_id']]) data['source_upload'] = { "filename": r[0] } # get dependencies r = g.db.execute_many(''' WITH RECURSIVE next_job(id, parent) AS ( SELECT j.id, (p->>'job-id')::uuid parent FROM job j, jsonb_array_elements(j.dependencies) AS p WHERE j.id = %s UNION SELECT j.id, (p->>'job-id')::uuid parent FROM job j LEFT JOIN LATERAL jsonb_array_elements(j.dependencies) AS p ON true, next_job nj WHERE j.id = nj.parent ) SELECT id, name, state, start_date, end_date, dependencies FROM job WHERE id IN (SELECT distinct id FROM next_job WHERE id != %s) ''', [data['job']['id'], data['job']['id']]) data['dependencies'] = [] for d in r: data['dependencies'].append({ "id": d[0], "name": d[1], "state": d[2], "start_date": str(d[3]), "end_date": str(d[4]), "depends_on": d[5] }) # get parents r = g.db.execute_many(''' SELECT id, name FROM job where id IN (SELECT (deps->>'job-id')::uuid FROM job, jsonb_array_elements(job.dependencies) as deps WHERE id = %s) ''', [data['job']['id']]) data['parents'] = [] for d in r: data['parents'].append({ "id": d[0], "name": d[1] }) # get the secrets secrets = g.db.execute_many(''' SELECT name, value FROM secret WHERE project_id = %s ''', [data['project']['id']]) is_fork = data['job'].get('fork', False) def get_secret(name): if is_fork: abort(400, 'Access to secret %s is not allowed from a fork' % name) for ev in secrets: if ev[0] == name: return decrypt_secret(ev[1]) return None # Deployments data['deployments'] = [] if deployments: for dep in deployments: if dep['type'] == 'docker-registry': if 'password' not in dep: data['deployments'].append(dep) continue secret_name = dep['password']['$secret'] secret = get_secret(secret_name) if secret is None: abort(400, "Secret %s not found" % secret_name) dep['password'] = secret data['deployments'].append(dep) elif dep['type'] == 'gcr': service_account = dep['service_account']['$secret'] secret = get_secret(service_account) if secret is None: abort(400, "Secret %s not found" % service_account) dep['service_account'] = secret data['deployments'].append(dep) elif dep['type'] == 'ecr': access_key_id_name = dep['access_key_id']['$secret'] secret = get_secret(access_key_id_name) if secret is None: abort(400, "Secret %s not found" % access_key_id_name) dep['access_key_id'] = secret secret_access_key_name = dep['secret_access_key']['$secret'] secret = get_secret(secret_access_key_name) if secret is None: abort(400, "Secret %s not found" % secret_access_key_name) dep['secret_access_key'] = secret data['deployments'].append(dep) else: abort(400, "Unknown deployment type") # Registries data['registries'] = [] definition = data['job']['definition'] registries = None if definition: registries = definition.get('registries', None) if registries: for r in registries: if r['type'] == 'docker-registry': secret_name = r['password']['$secret'] secret = get_secret(secret_name) if secret is None: abort(400, "Secret %s not found" % secret_name) r['password'] = secret data['registries'].append(r) elif r['type'] == 'ecr': access_key_id_name = r['access_key_id']['$secret'] secret = get_secret(access_key_id_name) if secret is None: abort(400, "Secret %s not found" % access_key_id_name) r['access_key_id'] = secret secret_access_key_name = r['secret_access_key']['$secret'] secret = get_secret(secret_access_key_name) if secret is None: abort(400, "Secret %s not found" % secret_access_key_name) r['secret_access_key'] = secret data['registries'].append(r) else: abort(400, "Unknown deployment type") root_url = get_root_url("global") # Default env vars project_name = urllib.quote_plus(data['project']['name']).replace('+', '%20') job_name = urllib.quote_plus(data['job']['name']).replace('+', '%20') build_url = "%s/dashboard/#/project/%s/build/%s/%s" % (root_url, project_name, data['build']['build_number'], data['build']['restart_counter']) job_url = "%s/dashboard/#/project/%s/build/%s/%s/job/%s" % (root_url, project_name, data['build']['build_number'], data['build']['restart_counter'], job_name) job_api_url = "%s/api/v1/projects/%s/jobs/%s" % (root_url, data['project']['id'], data['job']['id']) build_api_url = "%s/api/v1/projects/%s/builds/%s" % (root_url, data['project']['id'], data['build']['id']) data['env_vars'] = { "TERM": "xterm-256color", "INFRABOX_JOB_ID": data['job']['id'], "INFRABOX_JOB_URL": job_url, "INFRABOX_JOB_API_URL": job_api_url, "INFRABOX_BUILD_API_URL": build_api_url, "INFRABOX_BUILD_NUMBER": "%s" % data['build']['build_number'], "INFRABOX_BUILD_RESTART_COUNTER": "%s" % data['build']['restart_counter'], "INFRABOX_BUILD_URL": build_url, } data['secrets'] = {} if data['commit']['branch']: data['env_vars']['INFRABOX_GIT_BRANCH'] = data['commit']['branch'] if data['commit']['tag']: data['env_vars']['INFRABOX_GIT_TAG'] = data['commit']['tag'] if pull_request_id: data['env_vars']['INFRABOX_GITHUB_PULL_REQUEST'] = "true" if env_vars: for name, value in env_vars.iteritems(): data['env_vars'][name] = str(value) if env_var_refs: for name, value in env_var_refs.iteritems(): secret = get_secret(value) if secret is None: abort(400, "Secret %s not found" % value) data['secrets'][name] = secret return jsonify(data)
def get(self): job_id = g.token['job']['id'] data = {} # get all the job details r = g.db.execute_one( ''' SELECT j.name, null, j.dockerfile, p.id, p.type, p.name, b.id, b.commit_id, b.source_upload_id, b.build_number, u.github_api_token, u.username, null, j.type, null, null, j.repo, null, j.state, null, j.env_var, j.env_var_ref, null, null, u.id, j.build_arg, j.deployment, null, b.restart_counter, j.definition FROM job j INNER JOIN build b ON j.build_id = b.id AND j.project_id = b.project_id INNER JOIN collaborator co ON co.project_id = j.project_id AND co.role = 'Owner' INNER JOIN "user" u ON co.user_id = u.id INNER JOIN project p ON co.project_id = p.id WHERE j.id = %s ''', [job_id]) limits = {} definition = r[29] build_only = True if definition: limits = definition['resources']['limits'] build_only = definition.get('build_only', True) data['job'] = { "id": job_id, "name": r[0], "dockerfile": r[2], "build_only": build_only, "type": r[13], "repo": r[16], "state": r[18], "cpu": limits.get('cpu', 1), "memory": limits.get('memory', 1024), "build_arguments": r[25], "definition": r[29] } state = data['job']['state'] if state in ("finished", "error", "failure", "skipped", "killed", "unstable"): abort(409, 'job not running anymore') env_vars = r[20] env_var_refs = r[21] deployments = r[26] data['project'] = { "id": r[3], "type": r[4], "name": r[5], } data['build'] = { "id": r[6], "commit_id": r[7], "source_upload_id": r[8], "build_number": r[9], "restart_counter": r[28] } data['repository'] = { "owner": r[11], "name": None, "github_api_token": r[10], "private": False } data['commit'] = {"branch": None, "tag": None} pull_request_id = None commit_env = None if data['project']['type'] == 'github' or data['project'][ 'type'] == 'gerrit': r = g.db.execute_one( ''' SELECT r.clone_url, r.name, r.private FROM repository r WHERE r.project_id = %s ''', [data['project']['id']]) data['repository']['clone_url'] = r[0] data['repository']['name'] = r[1] data['repository']['private'] = r[2] # A regular commit r = g.db.execute_one( ''' SELECT c.branch, c.committer_name, c.tag, c.pull_request_id, c.env FROM commit c WHERE c.id = %s AND c.project_id = %s ''', [data['build']['commit_id'], data['project']['id']]) data['commit'] = { "id": data['build']['commit_id'], "branch": r[0], "committer_name": r[1], "tag": r[2] } pull_request_id = r[3] commit_env = r[4] if data['project']['type'] == 'upload': r = g.db.execute_one( ''' SELECT filename FROM source_upload WHERE id = %s ''', [data['build']['source_upload_id']]) data['source_upload'] = {"filename": r[0]} # get dependencies r = g.db.execute_many( ''' WITH RECURSIVE next_job(id, parent) AS ( SELECT j.id, (p->>'job-id')::uuid parent FROM job j, jsonb_array_elements(j.dependencies) AS p WHERE j.id = %s UNION SELECT j.id, (p->>'job-id')::uuid parent FROM job j LEFT JOIN LATERAL jsonb_array_elements(j.dependencies) AS p ON true, next_job nj WHERE j.id = nj.parent ) SELECT id, name, state, start_date, end_date, dependencies FROM job WHERE id IN (SELECT distinct id FROM next_job WHERE id != %s) ''', [data['job']['id'], data['job']['id']]) data['dependencies'] = [] for d in r: data['dependencies'].append({ "id": d[0], "name": d[1], "state": d[2], "start_date": str(d[3]), "end_date": str(d[4]), "depends_on": d[5] }) # get parents r = g.db.execute_many( ''' SELECT id, name FROM job where id IN (SELECT (deps->>'job-id')::uuid FROM job, jsonb_array_elements(job.dependencies) as deps WHERE id = %s) ''', [data['job']['id']]) data['parents'] = [] for d in r: data['parents'].append({"id": d[0], "name": d[1]}) # get the secrets secrets = g.db.execute_many( ''' SELECT name, value FROM secret WHERE project_id = %s ''', [data['project']['id']]) is_fork = data['job'].get('fork', False) def get_secret_type(name): try: json.loads(name) return 'vault' except ValueError: return 'secret' def get_auth_type(res): token, role_id, secret_id = res[2], res[5], res[6] if token: validate_res = 'token' else: # validate appRole validate_res = 'appRole' if not role_id or not secret_id: validate_res = 'error' return validate_res def get_secret(name, project_id=None): secret_type = get_secret_type(name) if secret_type == 'vault': vault = json.loads(name) vault_name = vault['$vault'] secret_path = vault['$vault_secret_path'] secret_key = vault['$vault_secret_key'] if not project_id: abort( 400, "project_id is essential for getting Vault: '%s' " % vault_name) result = g.db.execute_one( """ SELECT url, version, token, ca, namespace, role_id, secret_id FROM vault WHERE name = %s and project_id = %s """, [vault_name, project_id]) if not result: abort( 400, "Cannot get Vault '%s' in project '%s' " % (vault_name, project_id)) url, version, token, ca, namespace, role_id, secret_id = result[ 0], result[1], result[2], result[3], result[4], result[ 5], result[6] if not namespace: namespace = '' if version == 'v1': url += '/v1/' + namespace + '/' + secret_path elif version == 'v2': paths = secret_path.split('/') url += '/v1/' + namespace + '/' + paths[ 0] + '/data/' + '/'.join(paths[1:]) # choose validate way validate_res = get_auth_type(result) if validate_res == 'token': app.logger.info('validate way is token') elif validate_res == 'appRole': app_role = {'role_id': role_id, 'secret_id': secret_id} json_data = json.dumps(app_role) app_role_url = result[ 0] + '/v1/' + namespace + '/auth/approle/login' res = requests.post(url=app_role_url, data=json_data, verify=False) if res.status_code == 200: json_res = json.loads(res.content) token = json_res['auth']['client_token'] else: abort( 400, "Getting value from vault error: url is '%s', validate way is appRole " % (url)) else: abort( 400, "Validate way is '%s' ! result is '%s' " % (validate_res, result)) if not ca: res = requests.get(url=url, headers={'X-Vault-Token': token}, verify=False) else: with tempfile.NamedTemporaryFile(delete=False) as f: f.write(ca) f.flush() # ensure all data written res = requests.get(url=url, headers={'X-Vault-Token': token}, verify=f.name) if res.status_code == 200: json_res = json.loads(res.content) if json_res['data'].get('data') and isinstance( json_res['data'].get('data'), dict): value = json_res['data'].get('data').get(secret_key) else: value = json_res['data'].get(secret_key) return value else: abort( 400, "Getting value from vault error: url is '%s', token is '%s' " % (url, result)) else: if is_fork: abort( 400, 'Access to secret %s is not allowed from a fork' % name) for ev in secrets: if ev[0] == name: return decrypt_secret(ev[1]) return None # Deployments data['deployments'] = [] if deployments: for dep in deployments: if dep['type'] == 'docker-registry': if 'password' not in dep: data['deployments'].append(dep) continue secret_name = dep['password']['$secret'] secret = get_secret(secret_name) if secret is None: abort(400, "Secret %s not found" % secret_name) dep['password'] = secret data['deployments'].append(dep) elif dep['type'] == 'gcr': service_account = dep['service_account']['$secret'] secret = get_secret(service_account) if secret is None: abort(400, "Secret %s not found" % service_account) dep['service_account'] = secret data['deployments'].append(dep) elif dep['type'] == 'ecr': access_key_id_name = dep['access_key_id']['$secret'] secret = get_secret(access_key_id_name) if secret is None: abort(400, "Secret %s not found" % access_key_id_name) dep['access_key_id'] = secret secret_access_key_name = dep['secret_access_key'][ '$secret'] secret = get_secret(secret_access_key_name) if secret is None: abort(400, "Secret %s not found" % secret_access_key_name) dep['secret_access_key'] = secret data['deployments'].append(dep) else: abort(400, "Unknown deployment type") # Registries data['registries'] = [] definition = data['job']['definition'] registries = None if definition: registries = definition.get('registries', None) if registries: for r in registries: if r['type'] == 'docker-registry': secret_name = r['password']['$secret'] secret = get_secret(secret_name) if secret is None: abort(400, "Secret %s not found" % secret_name) r['password'] = secret data['registries'].append(r) elif r['type'] == 'gcr': service_account = r['service_account']['$secret'] secret = get_secret(service_account) if secret is None: abort(400, "Secret %s not found" % service_account) r['service_account'] = secret data['registries'].append(r) elif r['type'] == 'ecr': access_key_id_name = r['access_key_id']['$secret'] secret = get_secret(access_key_id_name) if secret is None: abort(400, "Secret %s not found" % access_key_id_name) r['access_key_id'] = secret secret_access_key_name = r['secret_access_key']['$secret'] secret = get_secret(secret_access_key_name) if secret is None: abort(400, "Secret %s not found" % secret_access_key_name) r['secret_access_key'] = secret data['registries'].append(r) else: abort(400, "Unknown registry type") root_url = get_root_url("global") # Default env vars project_name = urllib.quote_plus(data['project']['name']).replace( '+', '%20') job_name = urllib.quote_plus(data['job']['name']).replace('+', '%20') build_url = "%s/dashboard/#/project/%s/build/%s/%s" % ( root_url, project_name, data['build']['build_number'], data['build']['restart_counter']) job_url = "%s/dashboard/#/project/%s/build/%s/%s/job/%s" % ( root_url, project_name, data['build']['build_number'], data['build']['restart_counter'], job_name) job_api_url = "%s/api/v1/projects/%s/jobs/%s" % ( root_url, data['project']['id'], data['job']['id']) build_api_url = "%s/api/v1/projects/%s/builds/%s" % ( root_url, data['project']['id'], data['build']['id']) data['env_vars'] = { "TERM": "xterm-256color", "INFRABOX_JOB_ID": data['job']['id'], "INFRABOX_JOB_URL": job_url, "INFRABOX_JOB_API_URL": job_api_url, "INFRABOX_BUILD_API_URL": build_api_url, "INFRABOX_BUILD_NUMBER": "%s" % data['build']['build_number'], "INFRABOX_BUILD_RESTART_COUNTER": "%s" % data['build']['restart_counter'], "INFRABOX_BUILD_URL": build_url, } data['secrets'] = {} if data['commit']['branch']: data['env_vars']['INFRABOX_GIT_BRANCH'] = data['commit']['branch'] if data['commit']['tag']: data['env_vars']['INFRABOX_GIT_TAG'] = data['commit']['tag'] if data['commit'].get('id', None): data['env_vars']['INFRABOX_COMMIT_ID'] = data['commit']['id'] if pull_request_id: data['env_vars']['INFRABOX_GITHUB_PULL_REQUEST'] = "true" if commit_env: data['env_vars'].update(commit_env) if env_vars: for name, value in env_vars.iteritems(): try: data['env_vars'][name] = str(value) except UnicodeEncodeError: data['env_vars'][name] = value.encode('utf-8') if env_var_refs: for name, value in env_var_refs.iteritems(): secret = get_secret(value, data['project']['id']) if secret is None: abort(400, "Secret %s not found" % value) data['secrets'][name] = secret return jsonify(data)
def get(self): state = request.args.get('state') code = request.args.get('code') t = request.args.get('t', None) if not states.get(state, None): abort(401) del states[state] # TODO(ib-steffen): allow custom ca bundles r = requests.post(GITHUB_TOKEN_URL, data={ 'client_id': GITHUB_CLIENT_ID, 'client_secret': GITHUB_CLIENT_SECRET, 'code': code, 'state': state }, headers={'Accept': 'application/json'}, verify=False) if r.status_code != 200: logger.error(r.text) abort(500) result = r.json() access_token = result['access_token'] check_org(access_token) # TODO(ib-steffen): allow custom ca bundles r = requests.get(GITHUB_USER_PROFILE_URL, headers={ 'Accept': 'application/json', 'Authorization': 'token %s' % access_token }, verify=False) gu = r.json() github_id = gu['id'] if os.environ['INFRABOX_GITHUB_LOGIN_ENABLED'] == 'true': user = g.db.execute_one_dict( ''' SELECT id FROM "user" WHERE github_id = %s ''', [github_id]) if not user: user = g.db.execute_one_dict( ''' INSERT INTO "user" (github_id, username, avatar_url, name) VALUES (%s, %s, %s, %s) RETURNING id ''', [github_id, gu['login'], gu['avatar_url'], gu['name']]) user_id = user['id'] else: if not t: abort(404) user = g.db.execute_one_dict( ''' SELECT id FROM "user" WHERE github_api_token = %s ''', [t]) if not user: abort(404) user_id = user['id'] g.db.execute( ''' UPDATE "user" SET github_api_token = %s, github_id = %s WHERE id = %s ''', [access_token, github_id, user_id]) g.db.commit() token = encode_user_token(user_id) url = get_root_url('global') + '/dashboard/' logger.debug("Redirecting GitHub user to %s", url) res = redirect(url) res.set_cookie('token', token) return res
from flask_restplus import Resource from pyinfraboxutils import get_logger, get_root_url from pyinfraboxutils.ibrestplus import api from pyinfraboxutils.token import encode_user_token logger = get_logger('github') GITHUB_CLIENT_ID = os.environ['INFRABOX_GITHUB_CLIENT_ID'] GITHUB_CLIENT_SECRET = os.environ['INFRABOX_GITHUB_CLIENT_SECRET'] GITHUB_AUTHORIZATION_URL = os.environ[ 'INFRABOX_GITHUB_LOGIN_URL'] + "/oauth/authorize" GITHUB_TOKEN_URL = os.environ[ 'INFRABOX_GITHUB_LOGIN_URL'] + "/oauth/access_token" GITHUB_USER_PROFILE_URL = os.environ['INFRABOX_GITHUB_API_URL'] + "/user" GITHUB_CALLBACK_URL = get_root_url('global') + "/github/auth/callback" # TODO(ib-steffen): move into DB states = {} def get_next_page(r): link = r.headers.get('Link', None) if not link: return None n1 = link.find('rel=\"next\"') if n1 < 0: return None