def maybe_get_image(source, target):
assert isinstance(source, FQSHA)
assert isinstance(target, FQSHA)
d = os.getcwd()
try:
srepo = source.ref.repo
trepo = target.ref.repo
if not os.path.isdir(trepo.qname):
os.makedirs(trepo.qname, exist_ok=True)
os.chdir(trepo.qname)
shell('git', 'clone', trepo.url, '.')
else:
os.chdir(trepo.qname)
if sp.run(['/bin/sh', '-c', f'git remote | grep -q {srepo.qname}']).returncode != 0:
shell('git', 'remote', 'add', srepo.qname, srepo.url)
shell('git', 'fetch', 'origin')
shell('git', 'fetch', srepo.qname)
shell('git', 'checkout', target.sha)
shell('git', 'config', 'user.email', '*****@*****.**')
shell('git', 'config', 'user.name', 'hail-ci-leader')
shell('git', 'merge', source.sha, '-m', 'foo')
# a force push that removes refs could fail us... not sure what we
# should do in that case. maybe 500'ing is OK?
with open('hail-ci-build-image', 'r') as f:
return f.read().strip()
except (sp.CalledProcessError, FileNotFoundError) as e:
log.exception(f'could not get hail-ci-build-image due to {e}')
return None
finally:
shell('git', 'reset', '--merge')
os.chdir(d)
def try_new_build(source, target):
img = maybe_get_image(target, source)
if img:
attributes = {
'target': json.dumps(target.to_json()),
'source': json.dumps(source.to_json()),
'image': img,
'type': BUILD_JOB_TYPE
}
try:
job = batch_client.create_job(
img,
command=['/bin/bash',
'-c',
PR_BUILD_SCRIPT],
env={
'SOURCE_REPO_URL': source.ref.repo.url,
'SOURCE_BRANCH': source.ref.name,
'SOURCE_SHA': source.sha,
'TARGET_REPO_URL': target.ref.repo.url,
'TARGET_BRANCH': target.ref.name,
'TARGET_SHA': target.sha
},
resources={'requests': {
'cpu': '3.7',
'memory': '4G'
}},
tolerations=[{
'key': 'preemptible',
'value': 'true'
}],
service_account_name='test-svc',
callback=SELF_HOSTNAME + '/ci_build_done',
attributes=attributes,
volumes=[{
'volume': {
'name': f'hail-ci-{VERSION}-service-account-key',
'secret': {
'optional': False,
'secretName':
f'hail-ci-{VERSION}-service-account-key'
}
},
'volume_mount': {
'mountPath': '/secrets',
'name': f'hail-ci-{VERSION}-service-account-key',
'readOnly': True
}
}])
return Building(job, img, target.sha)
except Exception as e:
log.exception(f'could not start batch job due to {e}')
return Buildable(img, target.sha)
else:
return NoImage(target.sha)
def refresh_github_state():
for target_repo in prs.watched_repos():
try:
pulls = open_pulls(target_repo)
pulls_by_target = collections.defaultdict(list)
latest_target_shas = {}
for pull in pulls:
gh_pr = GitHubPR.from_gh_json(pull)
if gh_pr.target_ref not in latest_target_shas:
latest_target_shas[gh_pr.target_ref] = latest_sha_for_ref(gh_pr.target_ref)
sha = latest_target_shas[gh_pr.target_ref]
gh_pr.target_sha = sha
pulls_by_target[gh_pr.target_ref].append(gh_pr)
refresh_pulls(target_repo, pulls_by_target)
refresh_reviews(pulls_by_target)
except Exception as e:
log.exception(
f'could not refresh state for {target_repo.short_str()} due to {e}')
return '', 200
def notify_github(self, build):
log.info(f'notifying github of {build} for {self.short_str()}')
json = {
'state': build.gh_state(),
'description': str(build),
'context': CONTEXT
}
if isinstance(build, Failure) or isinstance(build, Mergeable):
json['target_url'] = \
f'https://storage.googleapis.com/{GCS_BUCKET}/ci/{self.source.sha}/{self.target.sha}/index.html'
try:
post_repo(self.target.ref.repo.qname,
'statuses/' + self.source.sha,
json=json,
status_code=201)
except BadStatus as e:
if e.status_code == 422:
log.exception(
f'Too many statuses applied to {self.source.sha}! This is a '
f'dangerous situation because I can no longer block merging '
f'of failing PRs.')
else:
raise e
def build_state_from_gh_json(d):
assert isinstance(d, list), d
assert all([isinstance(x, dict) for x in d]), d
my_statuses = [status for status in d if status['context'] == CONTEXT]
if len(my_statuses) != 0:
latest_status = my_statuses[0]
state = latest_status['state']
assert state in [
'pending', 'failure', 'success'
], state # 'error' is allowed by github but not used by me
description = latest_status['description']
try:
matches = re.findall(r'({.*})$', description)
assert len(matches) == 1, f'{d} {matches}'
doc = json.loads(matches[0])
except Exception as e:
log.exception(
'could not parse build state from description {latest_status}')
return Unknown()
return build_state_from_json(doc)
else:
return Unknown()
def refresh_github_state():
for target_repo in prs.watched_repos():
try:
pulls = open_pulls(target_repo)
pulls_by_target = collections.defaultdict(list)
latest_target_shas = {}
for pull in pulls:
gh_pr = GitHubPR.from_gh_json(pull)
if gh_pr.target_ref not in latest_target_shas:
latest_target_shas[gh_pr.target_ref] = latest_sha_for_ref(
gh_pr.target_ref)
sha = latest_target_shas[gh_pr.target_ref]
gh_pr.target_sha = sha
pulls_by_target[gh_pr.target_ref].append(gh_pr)
refresh_pulls(target_repo, pulls_by_target)
refresh_reviews(pulls_by_target)
# FIXME: I can't fit build state json in the status description
# refresh_statuses(pulls_by_target)
except Exception as e:
log.exception(
f'could not refresh state for {target_repo.short_str()} due to {e}'
)
return '', 200
def build_state_from_gh_json(d):
assert isinstance(d, list), d
assert all([isinstance(x, dict) for x in d]), d
my_statuses = [status for status in d if status['context'] == CONTEXT]
if len(my_statuses) != 0:
latest_status = my_statuses[0]
state = latest_status['state']
assert state in [
'pending', 'failure', 'success'
], state # 'error' is allowed by github but not used by me
description = latest_status['description']
try:
matches = re.findall(r'({.*})$', description)
assert len(matches) == 1, f'{d} {matches}'
doc = json.loads(matches[0])
except Exception as e:
log.exception(
'could not parse build state from description {latest_status}')
return Unknown()
return build_state_from_json(doc)
else:
return Unknown()
def notify_github(self, build):
log.info(f'notifying github of {build} for {self.short_str()}')
json = {
'state': build.gh_state(),
'description': str(build),
'context': CONTEXT
}
if isinstance(build, Failure) or isinstance(build, Mergeable):
json['target_url'] = \
f'https://storage.googleapis.com/{GCS_BUCKET}/ci/{self.source.sha}/{self.target.sha}/index.html'
try:
post_repo(
self.target.ref.repo.qname,
'statuses/' + self.source.sha,
json=json,
status_code=201)
except BadStatus as e:
if e.status_code == 422:
log.exception(
f'Too many statuses applied to {self.source.sha}! This is a '
f'dangerous situation because I can no longer block merging '
f'of failing PRs.')
else:
raise e
def handle_invalid_usage(error):
log.exception('bad status found when making request')
return jsonify(error.data), error.status_code
def try_deploy(self, target_ref):
assert isinstance(target_ref, FQRef)
assert self.is_deployable_target_ref(target_ref), \
f'{target_ref} is non-deployable {[(ref.short_str(), deployable) for ref, deployable in self._watched_targets.items()]}'
old_job = self.deploy_jobs.get(target_ref, None)
if old_job is not None:
log.info(
f'will not deploy while deploy job {old_job.id} is running')
return
latest_sha = latest_sha_for_ref(target_ref)
if latest_sha == self.latest_deployed[target_ref]:
log.info(f'already deployed {latest_sha}')
return
try:
img = get_image_for_target(target_ref)
attributes = {
'target': json.dumps(FQSHA(target_ref, latest_sha).to_json()),
'image': img,
'type': DEPLOY_JOB_TYPE
}
env = {
'DEPLOY_REPO_URL': target_ref.repo.url,
'DEPLOY_BRANCH': target_ref.name,
'DEPLOY_SHA': latest_sha
}
volumes = [{
'volume': {
'name': 'docker-sock-volume',
'hostPath': {
'path': '/var/run/docker.sock',
'type': 'File'
}
},
'volume_mount': {
'mountPath': '/var/run/docker.sock',
'name': 'docker-sock-volume'
}
}]
if target_ref.repo.owner == "hail-ci-test":
# special case for test repos
deploy_secret = f'ci-deploy-{VERSION}--hail-is-ci-test-service-account-key'
else:
deploy_secret = PRS._deploy_secrets.get(target_ref.repo, None)
if deploy_secret:
volumes.append({
'volume': {
'name': f'{deploy_secret}',
'secret': {
'optional': False,
'secretName': f'{deploy_secret}'
}
},
'volume_mount': {
'mountPath': '/secrets',
'name': f'{deploy_secret}',
'readOnly': True
}
})
job = batch_client.create_job(
img,
command=['/bin/bash', '-c', PR_DEPLOY_SCRIPT],
env=env,
resources={'requests': {
'cpu': '3.7',
'memory': '4G'
}},
volumes=volumes,
tolerations=[{
'key': 'preemptible',
'value': 'true'
}],
security_context={
'fsGroup': 412,
},
attributes=attributes,
callback=SELF_HOSTNAME + '/deploy_build_done')
log.info(
f'deploying {target_ref.short_str()}:{latest_sha} in job {job.id}'
)
self.deploy_jobs[target_ref] = job
except Exception as e:
log.exception(f'could not start deploy job due to {e}')
def handle_invalid_usage(error):
log.exception('bad status found when making request')
return jsonify(error.data), error.status_code
def try_deploy(self, target_ref):
assert isinstance(target_ref, FQRef)
assert self.is_deployable_target_ref(target_ref), \
f'{target_ref} is non-deployable {[(ref.short_str(), deployable) for ref, deployable in self._watched_targets.items()]}'
old_job = self.deploy_jobs.get(target_ref, None)
if old_job is not None:
log.info(f'will not deploy while deploy job {old_job.id} is running')
return
latest_sha = latest_sha_for_ref(target_ref)
if latest_sha == self.latest_deployed[target_ref]:
log.info(f'already deployed {latest_sha}')
return
try:
img = get_image_for_target(target_ref)
attributes = {
'target': json.dumps(FQSHA(target_ref, latest_sha).to_json()),
'image': img,
'type': DEPLOY_JOB_TYPE
}
env = {
'DEPLOY_REPO_URL': target_ref.repo.url,
'DEPLOY_BRANCH': target_ref.name,
'DEPLOY_SHA': latest_sha
}
volumes = [{
'volume': {
'name': 'docker-sock-volume',
'hostPath': {
'path': '/var/run/docker.sock',
'type': 'File'
}
},
'volume_mount': {
'mountPath': '/var/run/docker.sock',
'name': 'docker-sock-volume'
}
}]
if target_ref.repo.owner == "hail-ci-test":
# special case for test repos
deploy_secret = f'ci-deploy-{VERSION}--hail-is-ci-test-service-account-key'
else:
deploy_secret = PRS._deploy_secrets.get(target_ref.repo, None)
if deploy_secret:
volumes.append({
'volume': {
'name': f'{deploy_secret}',
'secret': {
'optional': False,
'secretName':
f'{deploy_secret}'
}
},
'volume_mount': {
'mountPath': '/secrets',
'name': f'{deploy_secret}',
'readOnly': True
}
})
job = batch_client.create_job(
img,
command=['/bin/bash', '-c', PR_DEPLOY_SCRIPT],
env=env,
resources={'requests': {
'cpu': '3.7',
'memory': '4G'
}},
volumes=volumes,
tolerations=[{
'key': 'preemptible',
'value': 'true'
}],
security_context={
'fsGroup': 412,
},
service_account_name='deploy-svc',
attributes=attributes,
callback=SELF_HOSTNAME + '/deploy_build_done')
log.info(f'deploying {target_ref.short_str()}:{latest_sha} in job {job.id}')
self.deploy_jobs[target_ref] = job
except Exception as e:
log.exception(f'could not start deploy job due to {e}')