def build_jupyter_task(task_uuid):
"""Function called by the celery task to build Jupyter image.
Builds a Jupyter (docker image) given the arguments, the logs
produced by the user provided script are forwarded to a SocketIO
server and namespace defined in the orchest internals config.
Args:
task_uuid:
Returns:
"""
with requests.sessions.Session() as session:
try:
update_jupyter_build_status("STARTED", session, task_uuid)
# Prepare the project snapshot with the correctly placed
# dockerfile, scripts, etc.
build_context = prepare_build_context(task_uuid)
# Use the agreed upon pattern for the docker image name.
docker_image_name = _config.JUPYTER_IMAGE_NAME
if not os.path.exists(__JUPYTER_BUILD_FULL_LOGS_DIRECTORY):
os.mkdir(__JUPYTER_BUILD_FULL_LOGS_DIRECTORY)
# place the logs in the celery container
complete_logs_path = os.path.join(
__JUPYTER_BUILD_FULL_LOGS_DIRECTORY, docker_image_name
)
status = SioStreamedTask.run(
# What we are actually running/doing in this task,
task_lambda=lambda user_logs_fo: build_docker_image(
docker_image_name,
build_context,
task_uuid,
user_logs_fo,
complete_logs_path,
),
identity="jupyter",
server=_config.ORCHEST_SOCKETIO_SERVER_ADDRESS,
namespace=_config.ORCHEST_SOCKETIO_JUPYTER_BUILDING_NAMESPACE,
# note: using task.is_aborted() could be an option but
# it was giving some issues related to
# multithreading/processing, moreover, also just passing
# the task_uuid to this function is less information to
# rely on, which is good.
abort_lambda=lambda: AbortableAsyncResult(task_uuid).is_aborted(),
)
# cleanup
os.system('rm -rf "%s"' % build_context["snapshot_path"])
update_jupyter_build_status(status, session, task_uuid)
# Catch all exceptions because we need to make sure to set the
# build state to failed.
except Exception as e:
update_jupyter_build_status("FAILURE", session, task_uuid)
raise e
finally:
filters = {
"label": [
"_orchest_jupyter_build_is_intermediate=1",
f"_orchest_jupyter_build_task_uuid={task_uuid}",
]
}
# Artifacts of this build (intermediate containers, images,
# etc.)
cleanup_docker_artifacts(filters)
# The status of the Celery task is SUCCESS since it has finished
# running. Not related to the actual state of the build, e.g.
# FAILURE.
return "SUCCESS"
def build_environment_task(task_uuid, project_uuid, environment_uuid,
project_path):
"""Function called by the celery task to build an environment.
Builds an environment (docker image) given the arguments, the logs produced by the user provided script
are forwarded to a SocketIO server and namespace defined in the orchest internals config.
Args:
task_uuid:
project_uuid:
environment_uuid:
project_path:
Returns:
"""
with requests.sessions.Session() as session:
try:
update_environment_build_status("STARTED", session, task_uuid)
# prepare the project snapshot with the correctly placed dockerfile, scripts, etc.
build_context = prepare_build_context(task_uuid, project_uuid,
environment_uuid,
project_path)
# use the agreed upon pattern for the docker image name
docker_image_name = _config.ENVIRONMENT_IMAGE_NAME.format(
project_uuid=project_uuid, environment_uuid=environment_uuid)
if not os.path.exists(__ENV_BUILD_FULL_LOGS_DIRECTORY):
os.mkdir(__ENV_BUILD_FULL_LOGS_DIRECTORY)
# place the logs in the celery container
complete_logs_path = os.path.join(__ENV_BUILD_FULL_LOGS_DIRECTORY,
docker_image_name)
status = SioStreamedTask.run(
# what we are actually running/doing in this task
task_lambda=lambda user_logs_fo: build_docker_image(
docker_image_name,
build_context,
task_uuid,
user_logs_fo,
complete_logs_path,
),
identity=f"{project_uuid}-{environment_uuid}",
server=_config.ORCHEST_SOCKETIO_SERVER_ADDRESS,
namespace=_config.ORCHEST_SOCKETIO_ENV_BUILDING_NAMESPACE,
# note: using task.is_aborted() could be an option but
# it was giving some issues related
# to multithreading/processing, moreover,
# also just passing the task_uuid to this function is less information
# to rely on, which is good
abort_lambda=lambda: AbortableAsyncResult(task_uuid).
is_aborted(),
)
# cleanup
os.system('rm -rf "%s"' % build_context)
update_environment_build_status(status, session, task_uuid)
# catch all exceptions because we need to make sure to set the build state to failed
except Exception as e:
update_environment_build_status("FAILURE", session, task_uuid)
raise e
finally:
filters = {
"label": [
"_orchest_env_build_is_intermediate=1",
f"_orchest_env_build_task_uuid={task_uuid}",
]
}
# artifacts of this build (intermediate containers, images, etc.)
cleanup_env_build_docker_artifacts(filters)
# see if outdated images of this environment can be cleaned up
url = (
f"{CONFIG_CLASS.ORCHEST_API_ADDRESS}"
f"/environment-images/dangling/{project_uuid}/{environment_uuid}"
)
session.delete(url)
return status
def build_jupyter_image_task(task_uuid: str, image_tag: str):
"""Function called by the celery task to build Jupyter image.
Builds a Jupyter image given the arguments, the logs produced by the
user provided script are forwarded to a SocketIO server and
namespace defined in the orchest internals config.
Args:
task_uuid:
image_tag:
Returns:
"""
with requests.sessions.Session() as session:
try:
update_jupyter_image_build_status("STARTED", session, task_uuid)
# Prepare the project snapshot with the correctly placed
# dockerfile, scripts, etc.
build_context = prepare_build_context(task_uuid)
# Use the agreed upon pattern for the image name.
image_name = _config.JUPYTER_IMAGE_NAME
if not os.path.exists(__JUPYTER_BUILD_FULL_LOGS_DIRECTORY):
os.mkdir(__JUPYTER_BUILD_FULL_LOGS_DIRECTORY)
# place the logs in the celery container
complete_logs_path = os.path.join(
__JUPYTER_BUILD_FULL_LOGS_DIRECTORY, image_name)
status = SioStreamedTask.run(
# What we are actually running/doing in this task,
task_lambda=lambda user_logs_fo: build_image(
task_uuid,
image_name,
image_tag,
build_context,
user_logs_fo,
complete_logs_path,
),
identity="jupyter",
server=_config.ORCHEST_SOCKETIO_SERVER_ADDRESS,
namespace=_config.
ORCHEST_SOCKETIO_JUPYTER_IMG_BUILDING_NAMESPACE,
# note: using task.is_aborted() could be an option but
# it was giving some issues related to
# multithreading/processing, moreover, also just passing
# the task_uuid to this function is less information to
# rely on, which is good.
abort_lambda=lambda: AbortableAsyncResult(task_uuid
).is_aborted(),
)
# cleanup
rmtree(build_context["snapshot_path"])
update_jupyter_image_build_status(status, session, task_uuid)
# Catch all exceptions because we need to make sure to set the
# build state to failed.
except Exception as e:
update_jupyter_image_build_status("FAILURE", session, task_uuid)
logger.error(e)
raise e
finally:
# We get here either because the task was successful or was
# aborted, in any case, delete the workflow.
k8s_custom_obj_api.delete_namespaced_custom_object(
"argoproj.io",
"v1alpha1",
_config.ORCHEST_NAMESPACE,
"workflows",
f"image-build-task-{task_uuid}",
)
# The status of the Celery task is SUCCESS since it has finished
# running. Not related to the actual state of the build, e.g.
# FAILURE.
return "SUCCESS"