def _cleanup(self):
"""
Prune failed dependencies older than DEPENDENCY_FAILURE_COOLDOWN seconds.
Limit the disk usage of the dependencies (both the bundle files and the serialized state file size)
Deletes oldest failed dependencies first and then oldest finished dependencies.
Doesn't touch downloading dependencies.
"""
self._prune_failed_dependencies()
# With all the locks (should be fast if no cleanup needed, otherwise make sure nothing is corrupted
while True:
with self._global_lock:
self._acquire_all_locks()
bytes_used = sum(dep_state.size_bytes
for dep_state in self._dependencies.values())
serialized_length = len(
codalab.worker.pyjson.dumps(self._dependencies))
if (bytes_used > self._max_cache_size_bytes
or serialized_length >
DependencyManager.MAX_SERIALIZED_LEN):
logger.debug(
'%d dependencies in cache, disk usage: %s (max %s), serialized size: %s (max %s)',
len(self._dependencies),
size_str(bytes_used),
size_str(self._max_cache_size_bytes),
size_str(serialized_length),
DependencyManager.MAX_SERIALIZED_LEN,
)
ready_deps = {
dep_key: dep_state
for dep_key, dep_state in self._dependencies.items()
if dep_state.stage == DependencyStage.READY
and not dep_state.dependents
}
failed_deps = {
dep_key: dep_state
for dep_key, dep_state in self._dependencies.items()
if dep_state.stage == DependencyStage.FAILED
}
if failed_deps:
dep_key_to_remove = min(
failed_deps.items(),
key=lambda dep: dep[1].last_used)[0]
elif ready_deps:
dep_key_to_remove = min(
ready_deps.items(),
key=lambda dep: dep[1].last_used)[0]
else:
logger.info(
'Dependency quota full but there are only downloading dependencies, not cleaning up until downloads are over'
)
self._release_all_locks()
break
if dep_key_to_remove:
self._delete_dependency(dep_key_to_remove)
self._release_all_locks()
else:
self._release_all_locks()
break
def check_resource_utilization(run_state: RunState):
logger.info(
f'Checking resource utilization for bundle. uuid: {run_state.bundle.uuid}'
)
cpu_usage, memory_usage = docker_utils.get_container_stats_with_docker_stats(
run_state.container)
run_state = run_state._replace(cpu_usage=cpu_usage,
memory_usage=memory_usage)
run_state = run_state._replace(memory_usage=memory_usage)
kill_messages = []
run_stats = docker_utils.get_container_stats(run_state.container)
run_state = run_state._replace(max_memory=max(
run_state.max_memory, run_stats.get('memory', 0)))
run_state = run_state._replace(
disk_utilization=self.disk_utilization[
run_state.bundle.uuid]['disk_utilization'])
container_time_total = docker_utils.get_container_running_time(
run_state.container)
run_state = run_state._replace(
container_time_total=container_time_total,
container_time_user=run_stats.get(
'container_time_user', run_state.container_time_user),
container_time_system=run_stats.get(
'container_time_system', run_state.container_time_system),
)
if run_state.resources.time and container_time_total > run_state.resources.time:
kill_messages.append(
'Time limit exceeded. (Container uptime %s > time limit %s)'
% (duration_str(container_time_total),
duration_str(run_state.resources.time)))
if run_state.max_memory > run_state.resources.memory or run_state.exitcode == 137:
kill_messages.append('Memory limit %s exceeded.' %
size_str(run_state.resources.memory))
if run_state.resources.disk and run_state.disk_utilization > run_state.resources.disk:
kill_messages.append('Disk limit %sb exceeded.' %
size_str(run_state.resources.disk))
if kill_messages:
run_state = run_state._replace(
kill_message=' '.join(kill_messages), is_killed=True)
return run_state
def apply_func(func, arg):
"""
Apply post-processing function |func| to |arg|.
|func| is a string representing a list of functions (which are to be
applied to |arg| in succession). Each function is either:
- 'duration', 'date', 'size' for special formatting
- '%...' for sprintf-style formatting
- s/.../... for regular expression substitution
- [a:b] for taking substrings
"""
FUNC_DELIM = ' | '
if isinstance(arg, tuple):
# tuples are (bundle_uuid, genpath) which have not been fleshed out
return arg + (func,)
try:
if func is None:
return arg
# String encoding of a function: size s/a/b
for f in func.split(FUNC_DELIM):
if f == 'date':
arg = formatting.date_str(float(arg)) if arg is not None else None
elif f == 'duration':
arg = formatting.duration_str(float(arg)) if arg is not None else None
elif f == 'size':
arg = formatting.size_str(float(arg)) if arg is not None else None
elif f.startswith('%'):
arg = (f % float(arg)) if arg is not None else None
elif f.startswith('s/'): # regular expression: s/<old string>/<new string>
esc_slash = '_ESC_SLASH_' # Assume this doesn't occur in s
# Preserve escaped characters: \/
tokens = f.replace('\\/', esc_slash).split('/')
if len(tokens) != 3:
return '<invalid regex: %s>' % f
s = tokens[1].replace(esc_slash, '/')
t = tokens[2].replace(esc_slash, '/')
arg = re.sub(s, t, arg)
elif f.startswith('['): # substring
m = re.match('\[(.*):(.*)\]', f)
if m:
start = int(m.group(1) or 0)
end = int(m.group(2) or len(arg))
arg = arg[start:end]
else:
return '<invalid function: %s>' % f
elif f.startswith('add '):
# 'add k v' checks if arg is a dictionary and updates it with arg[k] = v
if isinstance(arg, dict):
k, v = f.split(' ')[1:]
arg[k] = v
else:
return 'arg (%s) not a dictionary' % type(arg)
elif f.startswith('key '):
# 'key k' converts arg into a dictionary where arg[k] = arg
arg = {f.split(' ')[1]: arg}
else:
return '<invalid function: %s>' % f
return arg
except:
# Applying the function failed, so just return the arg.
return arg
def get_target_info(self, target, depth=1):
info = _call_with_retries(lambda: self.client.get_target_info(target, depth))
contents = info.get('contents')
# Render the sizes
if contents:
for item in contents:
if 'size' in item:
item['size_str'] = formatting.size_str(item['size'])
return info
def get_target_info(self, target, depth=1):
info = _call_with_retries(
lambda: self.client.get_target_info(target, depth))
contents = info.get('contents')
# Render the sizes
if contents:
for item in contents:
if 'size' in item:
item['size_str'] = formatting.size_str(item['size'])
return info
def check_resource_utilization(run_state):
kill_messages = []
run_stats = docker_utils.get_container_stats(run_state.container)
container_time_total = docker_utils.get_container_running_time(run_state.container)
run_state = run_state._replace(
container_time_total=container_time_total,
container_time_user=run_stats.get(
'container_time_user', run_state.container_time_user
),
container_time_system=run_stats.get(
'container_time_system', run_state.container_time_system
),
)
run_state = run_state._replace(
max_memory=max(run_state.max_memory, run_stats.get('memory', 0))
)
run_state = check_disk_utilization(run_state)
if run_state.resources.time and container_time_total > run_state.resources.time:
kill_messages.append(
'Time limit exceeded. (Container uptime %s > time limit %s)'
% (duration_str(container_time_total), duration_str(run_state.resources.time))
)
if run_state.max_memory > run_state.resources.memory or run_state.exitcode == 137:
kill_messages.append(
'Memory limit %s exceeded.' % size_str(run_state.resources.memory)
)
if run_state.resources.disk and run_state.disk_utilization > run_state.resources.disk:
kill_messages.append(
'Disk limit %sb exceeded.' % size_str(run_state.resources.disk)
)
if kill_messages:
run_state = run_state._replace(kill_message=' '.join(kill_messages), is_killed=True)
return run_state
def update_state_and_check_killed(bytes_downloaded):
"""
Callback method for bundle service client updates dependency state and
raises DownloadAbortedException if download is killed by dep. manager
"""
with self._dependency_locks[dependency_state.dependency_key]:
state = self._dependencies[dependency_state.dependency_key]
if state.killed:
raise DownloadAbortedException("Aborted by user")
self._dependencies[dependency_state.dependency_key] = state._replace(
size_bytes=bytes_downloaded,
message="Downloading dependency: %s downloaded"
% size_str(bytes_downloaded),
)
def apply_func(func, arg):
'''
Apply post-processing function |func| to |arg|.
|func| is a string representing a list of functions (which are to be
applied to |arg| in succession). Each function is either:
- 'duration', 'date', 'size' for special formatting
- '%...' for sprintf-style formatting
- s/... for regular expression substitution
- [a:b] for taking substrings
'''
FUNC_DELIM = ' | '
if isinstance(arg, tuple):
# tuples are (bundle_uuid, genpath) which have not been fleshed out
return arg + (func,)
try:
if func == None: return arg
# String encoding of a function: size s/a/b
for f in func.split(FUNC_DELIM):
if f == 'date':
arg = formatting.date_str(arg)
elif f == 'duration':
arg = formatting.duration_str(float(arg)) if arg != None else ''
elif f == 'size':
arg = formatting.size_str(arg)
elif f.startswith('%'):
arg = (f % float(arg)) if arg != None else ''
elif f.startswith('s/'): # regular expression
_, s, t = f.split("/")
arg = re.sub(s, t, arg)
elif f.startswith('['): # substring
m = re.match('\[(.*):(.*)\]', f)
if m:
start = int(m.group(1) or 0)
end = int(m.group(2) or -1)
arg = arg[start:end]
else:
return '<invalid function: %s>' % f
else:
return '<invalid function: %s>' % f
return arg
except:
# Can't apply the function, so just return the arg.
return arg
def get_top_level_contents(self, target):
info = self.client.get_target_info(target, 1)
if info is not None and info['type'] == 'directory':
for item in info['contents']:
item['size_str'] = formatting.size_str(item['size'])
return info
def _cleanup(self):
"""
Prune failed dependencies older than DEPENDENCY_FAILURE_COOLDOWN seconds.
Limit the disk usage of the dependencies (both the bundle files and the serialized state file size)
Deletes oldest failed dependencies first and then oldest finished dependencies.
Doesn't touch downloading dependencies.
"""
self._prune_failed_dependencies()
while True:
with self._state_lock:
try:
dependencies, paths = self._fetch_state()
except (ValueError, EnvironmentError):
# Do nothing if an error is thrown while reading from the state file
logging.exception(
"Error reading from state file when cleaning up dependencies. Trying again..."
)
continue
bytes_used = sum(dep_state.size_bytes for dep_state in dependencies.values())
serialized_length = len(codalab.worker.pyjson.dumps(dependencies))
if (
bytes_used > self._max_cache_size_bytes
or serialized_length > DependencyManager.MAX_SERIALIZED_LEN
):
logger.debug(
'%d dependencies, disk usage: %s (max %s), serialized size: %s (max %s)',
len(dependencies),
size_str(bytes_used),
size_str(self._max_cache_size_bytes),
size_str(serialized_length),
DependencyManager.MAX_SERIALIZED_LEN,
)
ready_deps = {
dep_key: dep_state
for dep_key, dep_state in dependencies.items()
if dep_state.stage == DependencyStage.READY and not dep_state.dependents
}
failed_deps = {
dep_key: dep_state
for dep_key, dep_state in dependencies.items()
if dep_state.stage == DependencyStage.FAILED
}
if failed_deps:
dep_key_to_remove = min(
failed_deps.items(), key=lambda dep: dep[1].last_used
)[0]
elif ready_deps:
dep_key_to_remove = min(
ready_deps.items(), key=lambda dep: dep[1].last_used
)[0]
else:
logger.info(
'Dependency quota full but there are only downloading dependencies, not cleaning up '
'until downloads are over.'
)
break
if dep_key_to_remove:
self._delete_dependency(dep_key_to_remove, dependencies, paths)
self._commit_state(dependencies, paths)
else:
break
def get_top_level_contents(self, target):
info = self.get_target_info(target, 1)
if info is not None and info['type'] == 'directory':
for item in info['contents']:
item['size_str'] = formatting.size_str(item['size'])
return info
def progress_callback(bytes_uploaded):
run_status = 'Uploading results: %s done (archived size)' % size_str(
bytes_uploaded)
self.uploading[
run_state.bundle.uuid]['run_status'] = run_status
return True
def _transition_from_PREPARING(self, run_state):
"""
1- Request the docker image from docker image manager
- if image is failed, move to CLEANING_UP state
2- Request the dependencies from dependency manager
- if any are failed, move to CLEANING_UP state
3- If all dependencies and docker image are ready:
- Set up the local filesystem for the run
- Create symlinks to dependencies
- Allocate resources and prepare the docker container
- Start the docker container
4- If all is successful, move to RUNNING state
"""
def mount_dependency(dependency, shared_file_system):
if not shared_file_system:
# Set up symlinks for the content at dependency path
Path(dependency.child_path).parent.mkdir(parents=True,
exist_ok=True)
os.symlink(dependency.docker_path, dependency.child_path)
# The following will be converted into a Docker volume binding like:
# dependency_path:docker_dependency_path:ro
docker_dependencies.append(
(dependency.parent_path, dependency.docker_path))
if run_state.is_killed or run_state.is_restaged:
log_bundle_transition(
bundle_uuid=run_state.bundle.uuid,
previous_stage=run_state.stage,
next_stage=RunStage.CLEANING_UP,
reason=
f'the bundle was {"killed" if run_state.is_killed else "restaged"}',
)
return run_state._replace(stage=RunStage.CLEANING_UP)
# Check CPU and GPU availability
try:
cpuset, gpuset = self.assign_cpu_and_gpu_sets_fn(
run_state.resources.cpus, run_state.resources.gpus)
except Exception as e:
message = "Unexpectedly unable to assign enough resources to bundle {}: {}".format(
run_state.bundle.uuid, str(e))
logger.error(message)
logger.error(traceback.format_exc())
return run_state._replace(run_status=message)
dependencies_ready = True
status_messages = []
if not self.shared_file_system:
# No need to download dependencies if we're in the shared FS,
# since they're already in our FS
for dep in run_state.bundle.dependencies:
dep_key = DependencyKey(dep.parent_uuid, dep.parent_path)
dependency_state = self.dependency_manager.get(
run_state.bundle.uuid, dep_key)
if dependency_state.stage == DependencyStage.DOWNLOADING:
status_messages.append(
'Downloading dependency %s: %s done (archived size)' %
(dep.child_path, size_str(
dependency_state.size_bytes)))
dependencies_ready = False
elif dependency_state.stage == DependencyStage.FAILED:
# Failed to download dependency; -> CLEANING_UP
log_bundle_transition(
bundle_uuid=run_state.bundle.uuid,
previous_stage=run_state.stage,
next_stage=RunStage.CLEANING_UP,
reason=
f'Dependency has failed for this bundle. Dependency child uuid: {dep.child_uuid}. Dependency child path: {dep.child_path}',
)
return run_state._replace(
stage=RunStage.CLEANING_UP,
failure_message='Failed to download dependency %s: %s'
% (dep.child_path, dependency_state.message),
)
# get the docker image
docker_image = run_state.resources.docker_image
image_state = self.image_manager.get(docker_image)
if image_state.stage == DependencyStage.DOWNLOADING:
status_messages.append('Pulling docker image %s %s' %
(docker_image, image_state.message))
dependencies_ready = False
elif image_state.stage == DependencyStage.FAILED:
# Failed to pull image; -> CLEANING_UP
message = 'Failed to download Docker image: %s' % image_state.message
logger.error(message)
return run_state._replace(stage=RunStage.CLEANING_UP,
failure_message=message)
# stop proceeding if dependency and image downloads aren't all done
if not dependencies_ready:
status_message = status_messages.pop()
if status_messages:
status_message += "(and downloading %d other dependencies and docker images)" % len(
status_messages)
logger.info(
f'bundle is not ready yet. uuid: {run_state.bundle.uuid}. status message: {status_message}'
)
return run_state._replace(run_status=status_message)
# All dependencies ready! Set up directories, symlinks and container. Start container.
# 1) Set up a directory to store the bundle.
if self.shared_file_system:
if not os.path.exists(run_state.bundle_path):
if run_state.bundle_dir_wait_num_tries == 0:
message = (
"Bundle directory cannot be found on the shared filesystem. "
"Please ensure the shared fileystem between the server and "
"your worker is mounted properly or contact your administrators."
)
log_bundle_transition(
bundle_uuid=run_state.bundle.uuid,
previous_stage=run_state.stage,
next_stage=RunStage.CLEANING_UP,
reason=
"Bundle directory cannot be found on the shared filesystem.",
)
return run_state._replace(stage=RunStage.CLEANING_UP,
failure_message=message)
next_bundle_dir_wait_num_tries = run_state.bundle_dir_wait_num_tries - 1
logger.info(
f'Waiting for bundle directory to be created by the server, uuid: {run_state.bundle.uuid}, bundle_dir_wait_num_tries: {next_bundle_dir_wait_num_tries}'
)
return run_state._replace(
run_status=
"Waiting for bundle directory to be created by the server",
bundle_dir_wait_num_tries=next_bundle_dir_wait_num_tries,
)
else:
remove_path(run_state.bundle_path)
os.makedirs(run_state.bundle_path)
# 2) Set up symlinks
docker_dependencies = []
docker_dependencies_path = (
RunStateMachine._ROOT + run_state.bundle.uuid +
('_dependencies' if not self.shared_file_system else ''))
for dep in run_state.bundle.dependencies:
full_child_path = os.path.normpath(
os.path.join(run_state.bundle_path, dep.child_path))
to_mount = []
dependency_path = self._get_dependency_path(run_state, dep)
if dep.child_path == RunStateMachine._CURRENT_DIRECTORY:
# Mount all the content of the dependency_path to the top-level of the bundle
for child in os.listdir(dependency_path):
child_path = os.path.normpath(
os.path.join(run_state.bundle_path, child))
to_mount.append(
DependencyToMount(
docker_path=os.path.join(docker_dependencies_path,
child),
child_path=child_path,
parent_path=os.path.join(dependency_path, child),
))
run_state = run_state._replace(
paths_to_remove=(run_state.paths_to_remove or []) +
[child_path])
else:
to_mount.append(
DependencyToMount(
docker_path=os.path.join(docker_dependencies_path,
dep.child_path),
child_path=full_child_path,
parent_path=dependency_path,
))
first_element_of_path = Path(dep.child_path).parts[0]
if first_element_of_path == RunStateMachine._ROOT:
run_state = run_state._replace(
paths_to_remove=(run_state.paths_to_remove or []) +
[full_child_path])
else:
# child_path can be a nested path, so later remove everything from the first element of the path
path_to_remove = os.path.join(run_state.bundle_path,
first_element_of_path)
run_state = run_state._replace(
paths_to_remove=(run_state.paths_to_remove or []) +
[path_to_remove])
for dependency in to_mount:
try:
mount_dependency(dependency, self.shared_file_system)
except OSError as e:
log_bundle_transition(
bundle_uuid=run_state.bundle.uuid,
previous_stage=run_state.stage,
next_stage=RunStage.CLEANING_UP,
reason=str(e.__class__),
level=logging.ERROR,
)
return run_state._replace(stage=RunStage.CLEANING_UP,
failure_message=str(e))
if run_state.resources.network:
docker_network = self.docker_network_external.name
else:
docker_network = self.docker_network_internal.name
# 3) Start container
try:
container = docker_utils.start_bundle_container(
run_state.bundle_path,
run_state.bundle.uuid,
docker_dependencies,
run_state.bundle.command,
run_state.resources.docker_image,
network=docker_network,
cpuset=cpuset,
gpuset=gpuset,
memory_bytes=run_state.resources.memory,
runtime=self.docker_runtime,
)
self.worker_docker_network.connect(container)
except docker_utils.DockerUserErrorException as e:
message = 'Cannot start Docker container: {}'.format(e)
log_bundle_transition(
bundle_uuid=run_state.bundle.uuid,
previous_stage=run_state.stage,
next_stage=RunStage.CLEANING_UP,
reason='Cannot start Docker container.',
level=logging.ERROR,
)
return run_state._replace(stage=RunStage.CLEANING_UP,
failure_message=message)
except Exception as e:
message = 'Cannot start container: {}'.format(e)
logger.error(message)
logger.error(traceback.format_exc())
raise
return run_state._replace(
stage=RunStage.RUNNING,
run_status='Running job in container',
container_id=container.id,
container=container,
docker_image=image_state.digest,
has_contents=True,
cpuset=cpuset,
gpuset=gpuset,
)
def get(self, image_spec):
"""
Always request the newest docker image from Dockerhub if it's not in downloading thread and return the current
downloading status(READY, FAILED, or DOWNLOADING).
When the requested image in the following states:
1. If it's not available on the platform, we download the image and return DOWNLOADING status.
2. If another thread is actively downloading it, we return DOWNLOADING status.
3. If another thread was downloading it but not active by the time the request was sent, we return the following status:
* READY if the image was downloaded successfully.
* FAILED if the image wasn't able to be downloaded due to any reason.
:param image_spec: Repo image_spec of docker image being requested
:returns: A DockerAvailabilityState object with the state of the docker image
"""
def image_availability_state(image_spec, success_message,
failure_message):
"""
Try to get the image specified by image_spec from host machine.
Return ImageAvailabilityState.
"""
try:
image = self._docker.images.get(image_spec)
digests = image.attrs.get('RepoDigests', [image_spec])
digest = digests[0] if len(digests) > 0 else None
new_timestamp = str(time.time())
image.tag(self.CACHE_TAG, tag=new_timestamp)
for tag in image.tags:
tag_label, timestamp = tag.split(":")
# remove any other timestamp but not the current one
if tag_label == self.CACHE_TAG and timestamp != new_timestamp:
try:
self._docker.images.remove(tag)
except docker.errors.NotFound as err:
# It's possible that we get a 404 not found error here when removing the image,
# since another worker on the same system has already done so. We just
# ignore this 404, since any extraneous tags will be removed during the next iteration.
logger.warning(
"Attempted to remove image %s from cache, but image was not found: %s",
tag,
err,
)
return ImageAvailabilityState(digest=digest,
stage=DependencyStage.READY,
message=success_message)
except Exception as ex:
if using_sentry():
capture_exception()
return ImageAvailabilityState(digest=None,
stage=DependencyStage.FAILED,
message=failure_message % ex)
if ':' not in image_spec:
# Both digests and repo:tag kind of specs include the : character. The only case without it is when
# a repo is specified without a tag (like 'latest')
# When this is the case, different images API methods act differently:
# - pull pulls all tags of the image
# - get tries to get `latest` by default
# That means if someone requests a docker image without a tag, and the image does not have a latest
# tag pushed to Dockerhub, pull will succeed since it will pull all other tags, but later get calls
# will fail since the `latest` tag won't be found on the system.
# We don't want to assume what tag the user wanted so we want the pull step to fail if no tag is specified
# and there's no latest tag on dockerhub.
# Hence, we append the latest tag to the image spec if there's no tag specified otherwise at the very beginning
image_spec += ':latest'
try:
if image_spec in self._downloading:
with self._downloading[image_spec]['lock']:
if self._downloading[image_spec].is_alive():
return ImageAvailabilityState(
digest=None,
stage=DependencyStage.DOWNLOADING,
message=self._downloading[image_spec]['status'],
)
else:
if self._downloading[image_spec]['success']:
status = image_availability_state(
image_spec,
success_message='Image ready',
failure_message=
'Image {} was downloaded successfully, '
'but it cannot be found locally due to unhandled error %s'
.format(image_spec),
)
else:
status = image_availability_state(
image_spec,
success_message=
'Image {} can not be downloaded from DockerHub '
'but it is found locally'.format(image_spec),
failure_message=self._downloading[image_spec]
['message'] + ": %s",
)
self._downloading.remove(image_spec)
return status
else:
def download():
logger.debug('Downloading Docker image %s', image_spec)
try:
self._docker.images.pull(image_spec)
logger.debug('Download for Docker image %s complete',
image_spec)
self._downloading[image_spec]['success'] = True
self._downloading[image_spec][
'message'] = "Downloading image"
except (docker.errors.APIError,
docker.errors.ImageNotFound) as ex:
logger.debug('Download for Docker image %s failed: %s',
image_spec, ex)
self._downloading[image_spec]['success'] = False
self._downloading[image_spec][
'message'] = "Can't download image: {}".format(ex)
# Check docker image size before pulling from Docker Hub.
# Do not download images larger than self._max_image_size
# Download images if size cannot be obtained
if self._max_image_size:
try:
image_size_bytes = docker_utils.get_image_size_without_pulling(
image_spec)
if image_size_bytes is None:
failure_msg = (
"Unable to find Docker image: {} from Docker HTTP Rest API V2. "
"Skipping Docker image size precheck.".format(
image_spec))
logger.info(failure_msg)
elif image_size_bytes > self._max_image_size:
failure_msg = (
"The size of " + image_spec +
": {} exceeds the maximum image size allowed {}."
.format(size_str(image_size_bytes),
size_str(self._max_image_size)))
return ImageAvailabilityState(
digest=None,
stage=DependencyStage.FAILED,
message=failure_msg)
except Exception as ex:
failure_msg = "Cannot fetch image size before pulling Docker image: {} from Docker Hub: {}.".format(
image_spec, ex)
logger.error(failure_msg)
return ImageAvailabilityState(
digest=None,
stage=DependencyStage.FAILED,
message=failure_msg)
self._downloading.add_if_new(
image_spec, threading.Thread(target=download, args=[]))
return ImageAvailabilityState(
digest=None,
stage=DependencyStage.DOWNLOADING,
message=self._downloading[image_spec]['status'],
)
except Exception as ex:
return ImageAvailabilityState(digest=None,
stage=DependencyStage.FAILED,
message=str(ex))
def _serialize(self, value, attr, obj):
return formatting.size_str(value)
def _transition_from_PREPARING(self, run_state):
"""
1- Request the docker image from docker image manager
- if image is failed, move to CLEANING_UP state
2- Request the dependencies from dependency manager
- if any are failed, move to CLEANING_UP state
3- If all dependencies and docker image are ready:
- Set up the local filesystem for the run
- Create symlinks to dependencies
- Allocate resources and prepare the docker container
- Start the docker container
4- If all is successful, move to RUNNING state
"""
if run_state.is_killed:
return run_state._replace(stage=RunStage.CLEANING_UP)
dependencies_ready = True
status_messages = []
if not self.shared_file_system:
# No need to download dependencies if we're in the shared FS since they're already in our FS
for dep_key, dep in run_state.bundle.dependencies.items():
dependency_state = self.dependency_manager.get(
run_state.bundle.uuid, dep_key)
if dependency_state.stage == DependencyStage.DOWNLOADING:
status_messages.append(
'Downloading dependency %s: %s done (archived size)' %
(dep.child_path, size_str(
dependency_state.size_bytes)))
dependencies_ready = False
elif dependency_state.stage == DependencyStage.FAILED:
# Failed to download dependency; -> CLEANING_UP
return run_state._replace(
stage=RunStage.CLEANING_UP,
failure_message='Failed to download dependency %s: %s'
% (dep.child_path, dependency_state.message),
)
# get the docker image
docker_image = run_state.resources.docker_image
image_state = self.docker_image_manager.get(docker_image)
if image_state.stage == DependencyStage.DOWNLOADING:
status_messages.append('Pulling docker image: ' +
(image_state.message or docker_image or ""))
dependencies_ready = False
elif image_state.stage == DependencyStage.FAILED:
# Failed to pull image; -> CLEANING_UP
message = 'Failed to download Docker image: %s' % image_state.message
logger.error(message)
return run_state._replace(stage=RunStage.CLEANING_UP,
failure_message=message)
# stop proceeding if dependency and image downloads aren't all done
if not dependencies_ready:
status_message = status_messages.pop()
if status_messages:
status_message += "(and downloading %d other dependencies and docker images)" % len(
status_messages)
return run_state._replace(run_status=status_message)
# All dependencies ready! Set up directories, symlinks and container. Start container.
# 1) Set up a directory to store the bundle.
if self.shared_file_system:
if not os.path.exists(run_state.bundle_path):
if run_state.bundle_dir_wait_num_tries == 0:
message = (
"Bundle directory cannot be found on the shared filesystem. "
"Please ensure the shared fileystem between the server and "
"your worker is mounted properly or contact your administrators."
)
logger.error(message)
return run_state._replace(stage=RunStage.CLEANING_UP,
failure_message=message)
return run_state._replace(
run_status=
"Waiting for bundle directory to be created by the server",
bundle_dir_wait_num_tries=run_state.
bundle_dir_wait_num_tries - 1,
)
else:
remove_path(run_state.bundle_path)
os.mkdir(run_state.bundle_path)
# 2) Set up symlinks
docker_dependencies = []
docker_dependencies_path = (
'/' + run_state.bundle.uuid +
('_dependencies' if not self.shared_file_system else ''))
for dep_key, dep in run_state.bundle.dependencies.items():
full_child_path = os.path.normpath(
os.path.join(run_state.bundle_path, dep.child_path))
if not full_child_path.startswith(run_state.bundle_path):
# Dependencies should end up in their bundles (ie prevent using relative paths like ..
# to get out of their parent bundles
message = 'Invalid key for dependency: %s' % (dep.child_path)
logger.error(message)
return run_state._replace(stage=RunStage.CLEANING_UP,
failure_message=message)
docker_dependency_path = os.path.join(docker_dependencies_path,
dep.child_path)
if self.shared_file_system:
# On a shared FS, we know where the dep is stored and can get the contents directly
dependency_path = os.path.realpath(
os.path.join(dep.location, dep.parent_path))
else:
# On a dependency_manager setup ask the manager where the dependency is
dependency_path = os.path.join(
self.dependency_manager.dependencies_dir,
self.dependency_manager.get(run_state.bundle.uuid,
dep_key).path,
)
os.symlink(docker_dependency_path, full_child_path)
# These are turned into docker volume bindings like:
# dependency_path:docker_dependency_path:ro
docker_dependencies.append(
(dependency_path, docker_dependency_path))
# 3) Set up container
if run_state.resources.network:
docker_network = self.docker_network_external.name
else:
docker_network = self.docker_network_internal.name
try:
cpuset, gpuset = self.assign_cpu_and_gpu_sets_fn(
run_state.resources.cpus, run_state.resources.gpus)
except Exception as e:
message = "Cannot assign enough resources: %s" % str(e)
logger.error(message)
logger.error(traceback.format_exc())
return run_state._replace(run_status=message)
# 4) Start container
try:
container = docker_utils.start_bundle_container(
run_state.bundle_path,
run_state.bundle.uuid,
docker_dependencies,
run_state.bundle.command,
run_state.resources.docker_image,
network=docker_network,
cpuset=cpuset,
gpuset=gpuset,
memory_bytes=run_state.resources.memory,
runtime=self.docker_runtime,
)
self.worker_docker_network.connect(container)
except Exception as e:
message = 'Cannot start Docker container: {}'.format(e)
logger.error(message)
logger.error(traceback.format_exc())
raise
return run_state._replace(
stage=RunStage.RUNNING,
run_status='Running job in Docker container',
container_id=container.id,
container=container,
docker_image=image_state.digest,
has_contents=True,
cpuset=cpuset,
gpuset=gpuset,
)
def apply_func(func, arg):
"""
Apply post-processing function |func| to |arg|.
|func| is a string representing a list of functions (which are to be
applied to |arg| in succession). Each function is either:
- 'duration', 'date', 'size' for special formatting
- '%...' for sprintf-style formatting
- s/.../... for regular expression substitution
- [a:b] for taking substrings
"""
FUNC_DELIM = ' | '
if isinstance(arg, tuple):
# tuples are (bundle_uuid, genpath) which have not been fleshed out
return arg + (func, )
try:
if func is None:
return arg
# String encoding of a function: size s/a/b
for f in func.split(FUNC_DELIM):
if f == 'str':
arg = str(arg)
elif f == 'date':
arg = formatting.date_str(
float(arg)) if arg is not None else None
elif f == 'duration':
arg = formatting.duration_str(
float(arg)) if arg is not None else None
elif f == 'size':
arg = formatting.size_str(
float(arg)) if arg is not None else None
elif f.startswith('%'):
arg = (f % float(arg)) if arg is not None else None
elif f.startswith(
's/'): # regular expression: s/<old string>/<new string>
esc_slash = '_ESC_SLASH_' # Assume this doesn't occur in s
# Preserve escaped characters: \/
tokens = f.replace('\\/', esc_slash).split('/')
if len(tokens) != 3:
return '<invalid regex: %s>' % f
s = tokens[1].replace(esc_slash, '/')
t = tokens[2].replace(esc_slash, '/')
arg = re.sub(s, t, arg)
elif f.startswith('['): # substring
m = re.match('\[(.*):(.*)\]', f)
if m:
start = int(m.group(1) or 0)
end = int(m.group(2) or len(arg))
arg = arg[start:end]
else:
return '<invalid function: %s>' % f
elif f.startswith('add '):
# 'add k v' checks if arg is a dictionary and updates it with arg[k] = v
if isinstance(arg, dict):
k, v = f.split(' ')[1:]
arg[k] = v
else:
return 'arg (%s) not a dictionary' % type(arg)
elif f.startswith('key '):
# 'key k' converts arg into a dictionary where arg[k] = arg
arg = {f.split(' ')[1]: arg}
else:
return '<invalid function: %s>' % f
return arg
except:
# Applying the function failed, so just return the arg.
return arg
def get(self, image_spec: str) -> ImageAvailabilityState:
"""
Always request the newest image from the cloud if it's not in downloading thread and return the current
downloading status(READY, FAILED, or DOWNLOADING).
When the requested image in the following states:
1. If it's not available on the platform, we download the image and return DOWNLOADING status.
2. If another thread is actively downloading it, we return DOWNLOADING status.
3. If another thread was downloading it but not active by the time the request was sent, we return the following status:
* READY if the image was downloaded successfully.
* FAILED if the image wasn't able to be downloaded due to any reason.
Args:
image_spec: the image that the requester needs.
The caller will need to determine the type of image they need before calling this function.
It is usually safe to prefix the image with the type of image.
For example, the docker image go would be docker://go
Returns:
ImageAvailabilityState of the image requested.
"""
try:
if image_spec in self._downloading:
with self._downloading[image_spec]['lock']:
if self._downloading[image_spec].is_alive():
return ImageAvailabilityState(
digest=None,
stage=DependencyStage.DOWNLOADING,
message=self._downloading[image_spec]['status'],
)
else:
if self._downloading[image_spec]['success']:
status = self._image_availability_state(
image_spec,
success_message='Image ready',
failure_message=
'Image {} was downloaded successfully, '
'but it cannot be found locally due to unhandled error %s'
.format(image_spec),
)
else:
status = self._image_availability_state(
image_spec,
success_message=
'Image {} can not be downloaded from the cloud '
'but it is found locally'.format(image_spec),
failure_message=self._downloading[image_spec]
['message'] + ": %s",
)
self._downloading.remove(image_spec)
return status
else:
if self._max_image_size:
try:
try:
image_size_bytes = self._image_size_without_pulling(
image_spec)
except NotImplementedError:
failure_msg = (
"Could not query size of {} from container runtime hub. "
"Skipping size precheck.".format(image_spec))
logger.info(failure_msg)
image_size_bytes = 0
if image_size_bytes > self._max_image_size:
failure_msg = (
"The size of " + image_spec +
": {} exceeds the maximum image size allowed {}."
.format(size_str(image_size_bytes),
size_str(self._max_image_size)))
logger.error(failure_msg)
return ImageAvailabilityState(
digest=None,
stage=DependencyStage.FAILED,
message=failure_msg)
except Exception as ex:
failure_msg = "Cannot fetch image size before pulling Docker image: {} from Docker Hub: {}.".format(
image_spec, ex)
logger.error(failure_msg)
return ImageAvailabilityState(
digest=None,
stage=DependencyStage.FAILED,
message=failure_msg)
self._downloading.add_if_new(
image_spec,
threading.Thread(target=self._download, args=[image_spec]))
return ImageAvailabilityState(
digest=None,
stage=DependencyStage.DOWNLOADING,
message=self._downloading[image_spec]['status'],
)
except Exception as ex:
logger.error(ex)
return ImageAvailabilityState(digest=None,
stage=DependencyStage.FAILED,
message=str(ex))