def maybe_get_image(source, target): assert isinstance(source, FQSHA) assert isinstance(target, FQSHA) d = os.getcwd() try: srepo = source.ref.repo trepo = target.ref.repo if not os.path.isdir(trepo.qname): os.makedirs(trepo.qname, exist_ok=True) os.chdir(trepo.qname) shell('git', 'clone', trepo.url, '.') else: os.chdir(trepo.qname) if sp.run(['/bin/sh', '-c', f'git remote | grep -q {srepo.qname}']).returncode != 0: shell('git', 'remote', 'add', srepo.qname, srepo.url) shell('git', 'fetch', 'origin') shell('git', 'fetch', srepo.qname) shell('git', 'checkout', target.sha) shell('git', 'config', 'user.email', '*****@*****.**') shell('git', 'config', 'user.name', 'hail-ci-leader') shell('git', 'merge', source.sha, '-m', 'foo') # a force push that removes refs could fail us... not sure what we # should do in that case. maybe 500'ing is OK? with open('hail-ci-build-image', 'r') as f: return f.read().strip() except (sp.CalledProcessError, FileNotFoundError) as e: log.exception(f'could not get hail-ci-build-image due to {e}') return None finally: shell('git', 'reset', '--merge') os.chdir(d)
def try_new_build(source, target): img = maybe_get_image(target, source) if img: attributes = { 'target': json.dumps(target.to_json()), 'source': json.dumps(source.to_json()), 'image': img, 'type': BUILD_JOB_TYPE } try: job = batch_client.create_job( img, command=['/bin/bash', '-c', PR_BUILD_SCRIPT], env={ 'SOURCE_REPO_URL': source.ref.repo.url, 'SOURCE_BRANCH': source.ref.name, 'SOURCE_SHA': source.sha, 'TARGET_REPO_URL': target.ref.repo.url, 'TARGET_BRANCH': target.ref.name, 'TARGET_SHA': target.sha }, resources={'requests': { 'cpu': '3.7', 'memory': '4G' }}, tolerations=[{ 'key': 'preemptible', 'value': 'true' }], service_account_name='test-svc', callback=SELF_HOSTNAME + '/ci_build_done', attributes=attributes, volumes=[{ 'volume': { 'name': f'hail-ci-{VERSION}-service-account-key', 'secret': { 'optional': False, 'secretName': f'hail-ci-{VERSION}-service-account-key' } }, 'volume_mount': { 'mountPath': '/secrets', 'name': f'hail-ci-{VERSION}-service-account-key', 'readOnly': True } }]) return Building(job, img, target.sha) except Exception as e: log.exception(f'could not start batch job due to {e}') return Buildable(img, target.sha) else: return NoImage(target.sha)
def refresh_github_state(): for target_repo in prs.watched_repos(): try: pulls = open_pulls(target_repo) pulls_by_target = collections.defaultdict(list) latest_target_shas = {} for pull in pulls: gh_pr = GitHubPR.from_gh_json(pull) if gh_pr.target_ref not in latest_target_shas: latest_target_shas[gh_pr.target_ref] = latest_sha_for_ref(gh_pr.target_ref) sha = latest_target_shas[gh_pr.target_ref] gh_pr.target_sha = sha pulls_by_target[gh_pr.target_ref].append(gh_pr) refresh_pulls(target_repo, pulls_by_target) refresh_reviews(pulls_by_target) except Exception as e: log.exception( f'could not refresh state for {target_repo.short_str()} due to {e}') return '', 200
def notify_github(self, build): log.info(f'notifying github of {build} for {self.short_str()}') json = { 'state': build.gh_state(), 'description': str(build), 'context': CONTEXT } if isinstance(build, Failure) or isinstance(build, Mergeable): json['target_url'] = \ f'https://storage.googleapis.com/{GCS_BUCKET}/ci/{self.source.sha}/{self.target.sha}/index.html' try: post_repo(self.target.ref.repo.qname, 'statuses/' + self.source.sha, json=json, status_code=201) except BadStatus as e: if e.status_code == 422: log.exception( f'Too many statuses applied to {self.source.sha}! This is a ' f'dangerous situation because I can no longer block merging ' f'of failing PRs.') else: raise e
def build_state_from_gh_json(d): assert isinstance(d, list), d assert all([isinstance(x, dict) for x in d]), d my_statuses = [status for status in d if status['context'] == CONTEXT] if len(my_statuses) != 0: latest_status = my_statuses[0] state = latest_status['state'] assert state in [ 'pending', 'failure', 'success' ], state # 'error' is allowed by github but not used by me description = latest_status['description'] try: matches = re.findall(r'({.*})$', description) assert len(matches) == 1, f'{d} {matches}' doc = json.loads(matches[0]) except Exception as e: log.exception( 'could not parse build state from description {latest_status}') return Unknown() return build_state_from_json(doc) else: return Unknown()
def refresh_github_state(): for target_repo in prs.watched_repos(): try: pulls = open_pulls(target_repo) pulls_by_target = collections.defaultdict(list) latest_target_shas = {} for pull in pulls: gh_pr = GitHubPR.from_gh_json(pull) if gh_pr.target_ref not in latest_target_shas: latest_target_shas[gh_pr.target_ref] = latest_sha_for_ref( gh_pr.target_ref) sha = latest_target_shas[gh_pr.target_ref] gh_pr.target_sha = sha pulls_by_target[gh_pr.target_ref].append(gh_pr) refresh_pulls(target_repo, pulls_by_target) refresh_reviews(pulls_by_target) # FIXME: I can't fit build state json in the status description # refresh_statuses(pulls_by_target) except Exception as e: log.exception( f'could not refresh state for {target_repo.short_str()} due to {e}' ) return '', 200
def build_state_from_gh_json(d): assert isinstance(d, list), d assert all([isinstance(x, dict) for x in d]), d my_statuses = [status for status in d if status['context'] == CONTEXT] if len(my_statuses) != 0: latest_status = my_statuses[0] state = latest_status['state'] assert state in [ 'pending', 'failure', 'success' ], state # 'error' is allowed by github but not used by me description = latest_status['description'] try: matches = re.findall(r'({.*})$', description) assert len(matches) == 1, f'{d} {matches}' doc = json.loads(matches[0]) except Exception as e: log.exception( 'could not parse build state from description {latest_status}') return Unknown() return build_state_from_json(doc) else: return Unknown()
def notify_github(self, build): log.info(f'notifying github of {build} for {self.short_str()}') json = { 'state': build.gh_state(), 'description': str(build), 'context': CONTEXT } if isinstance(build, Failure) or isinstance(build, Mergeable): json['target_url'] = \ f'https://storage.googleapis.com/{GCS_BUCKET}/ci/{self.source.sha}/{self.target.sha}/index.html' try: post_repo( self.target.ref.repo.qname, 'statuses/' + self.source.sha, json=json, status_code=201) except BadStatus as e: if e.status_code == 422: log.exception( f'Too many statuses applied to {self.source.sha}! This is a ' f'dangerous situation because I can no longer block merging ' f'of failing PRs.') else: raise e
def handle_invalid_usage(error): log.exception('bad status found when making request') return jsonify(error.data), error.status_code
def try_deploy(self, target_ref): assert isinstance(target_ref, FQRef) assert self.is_deployable_target_ref(target_ref), \ f'{target_ref} is non-deployable {[(ref.short_str(), deployable) for ref, deployable in self._watched_targets.items()]}' old_job = self.deploy_jobs.get(target_ref, None) if old_job is not None: log.info( f'will not deploy while deploy job {old_job.id} is running') return latest_sha = latest_sha_for_ref(target_ref) if latest_sha == self.latest_deployed[target_ref]: log.info(f'already deployed {latest_sha}') return try: img = get_image_for_target(target_ref) attributes = { 'target': json.dumps(FQSHA(target_ref, latest_sha).to_json()), 'image': img, 'type': DEPLOY_JOB_TYPE } env = { 'DEPLOY_REPO_URL': target_ref.repo.url, 'DEPLOY_BRANCH': target_ref.name, 'DEPLOY_SHA': latest_sha } volumes = [{ 'volume': { 'name': 'docker-sock-volume', 'hostPath': { 'path': '/var/run/docker.sock', 'type': 'File' } }, 'volume_mount': { 'mountPath': '/var/run/docker.sock', 'name': 'docker-sock-volume' } }] if target_ref.repo.owner == "hail-ci-test": # special case for test repos deploy_secret = f'ci-deploy-{VERSION}--hail-is-ci-test-service-account-key' else: deploy_secret = PRS._deploy_secrets.get(target_ref.repo, None) if deploy_secret: volumes.append({ 'volume': { 'name': f'{deploy_secret}', 'secret': { 'optional': False, 'secretName': f'{deploy_secret}' } }, 'volume_mount': { 'mountPath': '/secrets', 'name': f'{deploy_secret}', 'readOnly': True } }) job = batch_client.create_job( img, command=['/bin/bash', '-c', PR_DEPLOY_SCRIPT], env=env, resources={'requests': { 'cpu': '3.7', 'memory': '4G' }}, volumes=volumes, tolerations=[{ 'key': 'preemptible', 'value': 'true' }], security_context={ 'fsGroup': 412, }, attributes=attributes, callback=SELF_HOSTNAME + '/deploy_build_done') log.info( f'deploying {target_ref.short_str()}:{latest_sha} in job {job.id}' ) self.deploy_jobs[target_ref] = job except Exception as e: log.exception(f'could not start deploy job due to {e}')
def handle_invalid_usage(error): log.exception('bad status found when making request') return jsonify(error.data), error.status_code
def try_deploy(self, target_ref): assert isinstance(target_ref, FQRef) assert self.is_deployable_target_ref(target_ref), \ f'{target_ref} is non-deployable {[(ref.short_str(), deployable) for ref, deployable in self._watched_targets.items()]}' old_job = self.deploy_jobs.get(target_ref, None) if old_job is not None: log.info(f'will not deploy while deploy job {old_job.id} is running') return latest_sha = latest_sha_for_ref(target_ref) if latest_sha == self.latest_deployed[target_ref]: log.info(f'already deployed {latest_sha}') return try: img = get_image_for_target(target_ref) attributes = { 'target': json.dumps(FQSHA(target_ref, latest_sha).to_json()), 'image': img, 'type': DEPLOY_JOB_TYPE } env = { 'DEPLOY_REPO_URL': target_ref.repo.url, 'DEPLOY_BRANCH': target_ref.name, 'DEPLOY_SHA': latest_sha } volumes = [{ 'volume': { 'name': 'docker-sock-volume', 'hostPath': { 'path': '/var/run/docker.sock', 'type': 'File' } }, 'volume_mount': { 'mountPath': '/var/run/docker.sock', 'name': 'docker-sock-volume' } }] if target_ref.repo.owner == "hail-ci-test": # special case for test repos deploy_secret = f'ci-deploy-{VERSION}--hail-is-ci-test-service-account-key' else: deploy_secret = PRS._deploy_secrets.get(target_ref.repo, None) if deploy_secret: volumes.append({ 'volume': { 'name': f'{deploy_secret}', 'secret': { 'optional': False, 'secretName': f'{deploy_secret}' } }, 'volume_mount': { 'mountPath': '/secrets', 'name': f'{deploy_secret}', 'readOnly': True } }) job = batch_client.create_job( img, command=['/bin/bash', '-c', PR_DEPLOY_SCRIPT], env=env, resources={'requests': { 'cpu': '3.7', 'memory': '4G' }}, volumes=volumes, tolerations=[{ 'key': 'preemptible', 'value': 'true' }], security_context={ 'fsGroup': 412, }, service_account_name='deploy-svc', attributes=attributes, callback=SELF_HOSTNAME + '/deploy_build_done') log.info(f'deploying {target_ref.short_str()}:{latest_sha} in job {job.id}') self.deploy_jobs[target_ref] = job except Exception as e: log.exception(f'could not start deploy job due to {e}')