def do_job(hook_call_id):
"""A Celery task that does a job specified by a hook call.
Creates a :class:`Job` instance and executes a build script prescribed
by a triggered :class:`Hook`. Also sends job output to :attr:`Job.task_uuid`
Redis pub-sub channel and updates build status.
:param hook_call_id: int, :class:`HookCall` identifier
"""
hook_call = HookCall.query.get(hook_call_id)
assert hook_call, 'HookCall#{} does not exist.'.format(hook_call_id)
job = Job(
build=hook_call.build,
hook_call=hook_call,
task_uuid=do_job.request.id)
db.session.add(job)
job.started()
db.session.commit()
logger.info('start sleeping for some time...')
while(True):
sleep(5)
logger.info('zzzzzz...')
logger.info('done sleeping')
hook = hook_call.hook
project = hook.project
config = current_app.config
redis_client = redis.StrictRedis(host=config['KOZMIC_REDIS_HOST'],
port=config['KOZMIC_REDIS_PORT'],
db=config['KOZMIC_REDIS_DATABASE'])
publisher = Publisher(redis_client=redis_client, channel=job.task_uuid)
stdout = ''
try:
kwargs = dict(
publisher=publisher,
stall_timeout=config['KOZMIC_STALL_TIMEOUT'],
clone_url=(project.gh_https_clone_url if project.is_public else
project.gh_ssh_clone_url),
commit_sha=hook_call.build.gh_commit_sha)
message = 'Pulling "{}" Docker image...'.format(hook.docker_image)
logger.info(message)
publisher.publish(message)
stdout = message + '\n'
try:
logger.info(docker.base_url)
docker.pull(hook.docker_image)
# Make sure that image has been successfully pulled by calling
# `inspect_image` on it:
docker.inspect_image(hook.docker_image)
except DockerAPIError as e:
logger.info('Failed to pull %s: %s.', hook.docker_image, e)
job.finished(1)
job.stdout = str(e)
db.session.commit()
return
else:
message = '"{}" image has been pulled.'.format(hook.docker_image)
logger.info(message)
publisher.publish(message)
#if not project.is_public:
# project.deploy_key.ensure()
#if not project.is_public:
# kwargs['deploy_key'] = (
# project.deploy_key.rsa_private_key,
# project.passphrase)
if job.hook_call.hook.install_script:
cached_image = 'kozmic-cache/{}'.format(job.get_cache_id())
cached_image_tag = str(project.id)
if does_docker_image_exist(cached_image, cached_image_tag):
install_stdout = ('Skipping install script as tracked files '
'did not change...')
publisher.publish(install_stdout)
stdout += install_stdout + '\n'
else:
with _run(docker_image=hook.docker_image,
script=hook.install_script,
remove_container=False,
**kwargs) as (return_code, install_stdout, container):
stdout += install_stdout
if return_code == 0:
# Install script has finished successfully. So we
# promote the resulting container to an image that
# we will use for running the build script in
# this and consequent jobs
docker.commit(container['Id'], repository=cached_image,
tag=cached_image_tag)
docker.remove_container(container)
else:
job.finished(return_code)
job.stdout = stdout
db.session.commit()
return
assert docker.images(cached_image)
docker_image = cached_image + ':' + cached_image_tag
else:
docker_image = job.hook_call.hook.docker_image
with _run(docker_image=docker_image,
script=hook.build_script,
remove_container=True,
**kwargs) as (return_code, docker_stdout, container):
job.finished(return_code)
if len(docker_stdout) > 1000000:
build_stdout = docker_stdout[:200000] + docker_stdout[-800000:]
else:
build_stdout = docker_stdout
job.stdout = stdout + build_stdout
db.session.commit()
return
finally:
publisher.finish()
def _run(publisher, stall_timeout, clone_url, commit_sha,
docker_image, script, deploy_key=None, remove_container=True):
yielded = False
stdout = ''
try:
with create_temp_dir() as working_dir:
message_queue = Queue.Queue()
builder = Builder(
docker=docker._get_current_object(), # `docker` is a local proxy
deploy_key=deploy_key,
clone_url=clone_url,
commit_sha=commit_sha,
docker_image=docker_image,
script=script,
working_dir=working_dir,
message_queue=message_queue)
log_path = os.path.join(working_dir, 'script.log')
stop_reason = ''
try:
# Start Builder and wait until it will create the container
builder.start()
container = message_queue.get(block=True, timeout=60)
# Now the container id is known and we can pass it to Tailer
tailer = Tailer(
log_path=log_path,
publisher=publisher,
container=container,
kill_timeout=stall_timeout)
tailer.start()
try:
# Tell Builder to continue and wait for it to finish
message_queue.task_done()
builder.join()
finally:
tailer.stop()
if tailer.has_killed_container:
stop_reason = '\nSorry, your script has stalled and been killed.\n'
finally:
if builder.container and remove_container:
docker.remove_container(builder.container)
if os.path.exists(log_path):
with open(log_path, 'r') as log:
stdout = log.read()
assert ((builder.return_code is not None) ^
(builder.exc_info is not None))
if builder.exc_info:
# Re-raise exception happened in builder
# (it will be catched in the outer try-except)
raise builder.exc_info[1], None, builder.exc_info[2]
else:
try:
yield (builder.return_code,
stdout + stop_reason,
builder.container)
except:
raise
finally:
yielded = True # otherwise we get "generator didn't
# stop after throw()" error if nested
# code raised exception
except:
stdout += ('\nSorry, something went wrong. We are notified of '
'the issue and will fix it soon.')
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
stdout += ('\n'.join('!! ' + line for line in lines))
if not yielded:
yield 1, stdout, None
raise