def try_new_build(source, target):
img = maybe_get_image(target, source)
if img:
attributes = {
'target': json.dumps(target.to_json()),
'source': json.dumps(source.to_json()),
'image': img,
'type': BUILD_JOB_TYPE
}
try:
job = batch_client.create_job(
img,
command=['/bin/bash',
'-c',
PR_BUILD_SCRIPT],
env={
'SOURCE_REPO_URL': source.ref.repo.url,
'SOURCE_BRANCH': source.ref.name,
'SOURCE_SHA': source.sha,
'TARGET_REPO_URL': target.ref.repo.url,
'TARGET_BRANCH': target.ref.name,
'TARGET_SHA': target.sha
},
resources={'requests': {
'cpu': '3.7',
'memory': '4G'
}},
tolerations=[{
'key': 'preemptible',
'value': 'true'
}],
service_account_name='test-svc',
callback=SELF_HOSTNAME + '/ci_build_done',
attributes=attributes,
volumes=[{
'volume': {
'name': f'hail-ci-{VERSION}-service-account-key',
'secret': {
'optional': False,
'secretName':
f'hail-ci-{VERSION}-service-account-key'
}
},
'volume_mount': {
'mountPath': '/secrets',
'name': f'hail-ci-{VERSION}-service-account-key',
'readOnly': True
}
}])
return Building(job, img, target.sha)
except Exception as e:
log.exception(f'could not start batch job due to {e}')
return Buildable(img, target.sha)
else:
return NoImage(target.sha)
def try_deploy(self, target_ref):
assert isinstance(target_ref, FQRef)
assert self.is_deployable_target_ref(target_ref), \
f'{target_ref} is non-deployable {[(ref.short_str(), deployable) for ref, deployable in self._watched_targets.items()]}'
old_job = self.deploy_jobs.get(target_ref, None)
if old_job is not None:
log.info(
f'will not deploy while deploy job {old_job.id} is running')
return
latest_sha = latest_sha_for_ref(target_ref)
if latest_sha == self.latest_deployed[target_ref]:
log.info(f'already deployed {latest_sha}')
return
try:
img = get_image_for_target(target_ref)
attributes = {
'target': json.dumps(FQSHA(target_ref, latest_sha).to_json()),
'image': img,
'type': DEPLOY_JOB_TYPE
}
env = {
'DEPLOY_REPO_URL': target_ref.repo.url,
'DEPLOY_BRANCH': target_ref.name,
'DEPLOY_SHA': latest_sha
}
volumes = [{
'volume': {
'name': 'docker-sock-volume',
'hostPath': {
'path': '/var/run/docker.sock',
'type': 'File'
}
},
'volume_mount': {
'mountPath': '/var/run/docker.sock',
'name': 'docker-sock-volume'
}
}]
if target_ref.repo.owner == "hail-ci-test":
# special case for test repos
deploy_secret = f'ci-deploy-{VERSION}--hail-is-ci-test-service-account-key'
else:
deploy_secret = PRS._deploy_secrets.get(target_ref.repo, None)
if deploy_secret:
volumes.append({
'volume': {
'name': f'{deploy_secret}',
'secret': {
'optional': False,
'secretName': f'{deploy_secret}'
}
},
'volume_mount': {
'mountPath': '/secrets',
'name': f'{deploy_secret}',
'readOnly': True
}
})
job = batch_client.create_job(
img,
command=['/bin/bash', '-c', PR_DEPLOY_SCRIPT],
env=env,
resources={'requests': {
'cpu': '3.7',
'memory': '4G'
}},
volumes=volumes,
tolerations=[{
'key': 'preemptible',
'value': 'true'
}],
security_context={
'fsGroup': 412,
},
attributes=attributes,
callback=SELF_HOSTNAME + '/deploy_build_done')
log.info(
f'deploying {target_ref.short_str()}:{latest_sha} in job {job.id}'
)
self.deploy_jobs[target_ref] = job
except Exception as e:
log.exception(f'could not start deploy job due to {e}')
def try_deploy(self, target_ref):
assert isinstance(target_ref, FQRef)
assert self.is_deployable_target_ref(target_ref), \
f'{target_ref} is non-deployable {[(ref.short_str(), deployable) for ref, deployable in self._watched_targets.items()]}'
old_job = self.deploy_jobs.get(target_ref, None)
if old_job is not None:
log.info(f'will not deploy while deploy job {old_job.id} is running')
return
latest_sha = latest_sha_for_ref(target_ref)
if latest_sha == self.latest_deployed[target_ref]:
log.info(f'already deployed {latest_sha}')
return
try:
img = get_image_for_target(target_ref)
attributes = {
'target': json.dumps(FQSHA(target_ref, latest_sha).to_json()),
'image': img,
'type': DEPLOY_JOB_TYPE
}
env = {
'DEPLOY_REPO_URL': target_ref.repo.url,
'DEPLOY_BRANCH': target_ref.name,
'DEPLOY_SHA': latest_sha
}
volumes = [{
'volume': {
'name': 'docker-sock-volume',
'hostPath': {
'path': '/var/run/docker.sock',
'type': 'File'
}
},
'volume_mount': {
'mountPath': '/var/run/docker.sock',
'name': 'docker-sock-volume'
}
}]
if target_ref.repo.owner == "hail-ci-test":
# special case for test repos
deploy_secret = f'ci-deploy-{VERSION}--hail-is-ci-test-service-account-key'
else:
deploy_secret = PRS._deploy_secrets.get(target_ref.repo, None)
if deploy_secret:
volumes.append({
'volume': {
'name': f'{deploy_secret}',
'secret': {
'optional': False,
'secretName':
f'{deploy_secret}'
}
},
'volume_mount': {
'mountPath': '/secrets',
'name': f'{deploy_secret}',
'readOnly': True
}
})
job = batch_client.create_job(
img,
command=['/bin/bash', '-c', PR_DEPLOY_SCRIPT],
env=env,
resources={'requests': {
'cpu': '3.7',
'memory': '4G'
}},
volumes=volumes,
tolerations=[{
'key': 'preemptible',
'value': 'true'
}],
security_context={
'fsGroup': 412,
},
service_account_name='deploy-svc',
attributes=attributes,
callback=SELF_HOSTNAME + '/deploy_build_done')
log.info(f'deploying {target_ref.short_str()}:{latest_sha} in job {job.id}')
self.deploy_jobs[target_ref] = job
except Exception as e:
log.exception(f'could not start deploy job due to {e}')