def validate(self, metadata_specs):
'''
Check that this metadata has the correct metadata keys and that it has
metadata values of the correct types.
'''
expected_keys = set(spec.key for spec in metadata_specs)
for key in self._metadata_keys:
if key not in expected_keys:
raise UsageError('Unexpected metadata key: %s' % (key,))
for spec in metadata_specs:
if spec.key in self._metadata_keys:
value = getattr(self, spec.key)
if spec.type is float and isinstance(value, int):
# cast int to float
value = float(value)
# Validate formatted string fields
if issubclass(spec.type, basestring) and spec.formatting is not None and value:
try:
if spec.formatting == 'duration':
formatting.parse_duration(value)
elif spec.formatting == 'size':
formatting.parse_size(value)
elif spec.formatting == 'date':
formatting.parse_datetime(value)
except ValueError as e:
raise UsageError(e.message)
if value is not None and not isinstance(value, spec.type):
raise UsageError(
'Metadata value for %s should be of type %s, was %s (type %s)'
% (spec.key, spec.type.__name__, value, type(value).__name__)
)
elif not spec.generated:
raise UsageError('Missing metadata key: %s' % (spec.key,))
def filter_bundles(self, bundles: BundlesPayload) -> BundlesPayload:
filtered_bundles: BundlesPayload = []
worker_memory_bytes: int = parse_size('{}m'.format(
self.args.memory_mb))
logger.info(
f"Current worker manager allocates {self.args.cpus} CPUs, {self.args.gpus} GPUs, "
f"and {worker_memory_bytes} bytes of RAM")
for bundle in bundles:
# Filter bundles based on the resources specified when creating the worker manager
if bundle['metadata']['request_cpus'] > self.args.cpus:
logger.info(
'Filtered out bundle {} based on unfulfillable resource requested: request_cpus={}'
.format(
bundle['uuid'],
bundle['metadata']['request_cpus'],
))
elif bundle['metadata']['request_gpus'] > self.args.gpus:
logger.info(
'Filtered out bundle {} based on unfulfillable resource requested: request_gpus={}'
.format(
bundle['uuid'],
bundle['metadata']['request_gpus'],
))
elif parse_size(bundle['metadata']
['request_memory']) > worker_memory_bytes:
logger.info(
'Filtered out bundle {} based on unfulfillable resource requested: request_memory={}'
.format(
bundle['uuid'],
bundle['metadata']['request_memory'],
))
else:
filtered_bundles.append(bundle)
return filtered_bundles
def validate(self, metadata_specs):
'''
Check that this metadata has the correct metadata keys and that it has
metadata values of the correct types.
'''
expected_keys = set(spec.key for spec in metadata_specs)
for key in self._metadata_keys:
if key not in expected_keys:
raise UsageError('Unexpected metadata key: %s' % (key, ))
for spec in metadata_specs:
if spec.key in self._metadata_keys:
value = getattr(self, spec.key)
if spec.type is float and isinstance(value, int):
# cast int to float
value = float(value)
# Validate formatted string fields
if issubclass(spec.type,
str) and spec.formatting is not None and value:
try:
if spec.formatting == 'duration':
formatting.parse_duration(value)
elif spec.formatting == 'size':
formatting.parse_size(value)
elif spec.formatting == 'date':
formatting.parse_datetime(value)
except ValueError as e:
raise UsageError(str(e))
if value is not None and not isinstance(value, spec.type):
raise UsageError(
'Metadata value for %s should be of type %s, was %s (type %s)'
% (spec.key, spec.type.__name__, value,
type(value).__name__))
elif not spec.generated and not spec.optional:
raise UsageError('Missing metadata key: %s' % (spec.key, ))
def _compute_request_disk(self, bundle):
"""
Compute the disk limit used for scheduling the run.
"""
#TODO: Remove this once we want to deprecate old versions
if not bundle.metadata.request_disk:
return formatting.parse_size('4g')
return formatting.parse_size(bundle.metadata.request_disk)
def _compute_request_memory(self, bundle):
"""
Compute the memory limit used for scheduling the run.
The default of 2g is for backwards compatibilty for
runs from before when we added client-side defaults
"""
if not bundle.metadata.request_memory:
return formatting.parse_size('2g')
return formatting.parse_size(bundle.metadata.request_memory)
def _compute_request_memory(self, bundle):
"""
Compute the memory limit used for scheduling the run.
The default of 2g is for backwards compatibilty for
runs from before when we added client-side defaults
"""
if not bundle.metadata.request_memory:
return formatting.parse_size('2g')
return formatting.parse_size(bundle.metadata.request_memory)
def _compute_request_memory(self, bundle):
"""
Compute the memory limit used for scheduling the run.
"""
if bundle.metadata.request_memory:
return formatting.parse_size(bundle.metadata.request_memory)
return self._default_request_memory
def _compute_request_memory(self, bundle):
"""
Compute the memory limit used for scheduling the run.
"""
if bundle.metadata.request_memory:
return formatting.parse_size(bundle.metadata.request_memory)
return self._default_request_memory
def default_user_info(self):
info = self.config['server'].get(
'default_user_info', {'time_quota': '1y', 'disk_quota': '1t'}
)
return {
'time_quota': formatting.parse_duration(info['time_quota']),
'disk_quota': formatting.parse_size(info['disk_quota']),
}
def default_user_info(self):
info = self.config['server'].get('default_user_info', {
'time_quota': '1y',
'disk_quota': '1t'
})
info['time_quota'] = formatting.parse_duration(info['time_quota'])
info['disk_quota'] = formatting.parse_size(info['disk_quota'])
return info
def default_user_info(self):
info = self.config['server'].get('default_user_info', {
'disk_quota': '1t',
'parallel_run_quota': 3
})
return {
'disk_quota': formatting.parse_size(info['disk_quota']),
'parallel_run_quota': info['parallel_run_quota'],
}
def _compute_request_disk(self, bundle):
"""
Compute the disk limit used for scheduling the run.
The default is min(disk quota the user has left, global max)
"""
if not bundle.metadata.request_disk:
return min(
self._model.get_user_disk_quota_left(bundle.owner_id) - 1,
self._max_request_disk)
return formatting.parse_size(bundle.metadata.request_disk)
def _compute_request_disk(self, bundle):
"""
Compute the disk limit used for scheduling the run.
The default is min(disk quota the user has left, global max)
"""
if not bundle.metadata.request_disk:
return min(
self._model.get_user_disk_quota_left(bundle.owner_id) - 1, self._max_request_disk
)
return formatting.parse_size(bundle.metadata.request_disk)
def filter_bundles(self, bundles: BundlesPayload) -> BundlesPayload:
filtered_bundles: BundlesPayload = []
for bundle in bundles:
# Filter bundles based on the resources specified when creating the worker manager
worker_memory_bytes: int = parse_size('{}m'.format(
self.args.memory_mb))
if (bundle['metadata']['request_cpus'] <= self.args.cpus
and bundle['metadata']['request_gpus'] <= self.args.gpus
and parse_size(bundle['metadata']['request_memory']) <=
worker_memory_bytes):
filtered_bundles.append(bundle)
else:
logger.info(
'Filtered out bundle {} based on resources requested: request_cpus={}, request_gpus={}, request_memory={}'
.format(
bundle['uuid'],
bundle['metadata']['request_cpus'],
bundle['metadata']['request_gpus'],
bundle['metadata']['request_memory'],
))
return filtered_bundles
def start_bundle_container(
bundle_path,
uuid,
dependencies,
command,
docker_image,
network=None,
cpuset=None,
gpuset=None,
memory_bytes=0,
detach=True,
tty=False,
runtime=DEFAULT_RUNTIME,
):
# Impose a minimum container request memory 4mb, same as docker's minimum allowed value
# https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory
# When using the REST api, it is allowed to set Memory to 0 but that means the container has unbounded
# access to the host machine's memory, which we have decided to not allow
if memory_bytes < parse_size('4m'):
raise DockerException('Minimum memory must be 4m ({} bytes)'.format(
parse_size('4m')))
if not command.endswith(';'):
command = '{};'.format(command)
docker_command = ['bash', '-c', '( %s ) >stdout 2>stderr' % command]
docker_bundle_path = '/' + uuid
volumes = get_bundle_container_volume_binds(bundle_path,
docker_bundle_path,
dependencies)
environment = {'HOME': docker_bundle_path}
working_dir = docker_bundle_path
# Unset entrypoint regardless of image
entrypoint = ''
cpuset_str = ','.join(cpuset) if cpuset else ''
# Get user/group that owns the bundle directory
# Then we can ensure that any created files are owned by the user/group
# that owns the bundle directory, not root.
bundle_stat = os.stat(bundle_path)
uid = bundle_stat.st_uid
gid = bundle_stat.st_gid
# TODO: Fix potential permissions issues arising from this setting
# This can cause problems if users expect to run as a specific user
user = '******' % (uid, gid)
if runtime == NVIDIA_RUNTIME:
# nvidia-docker runtime uses this env variable to allocate GPUs
environment['NVIDIA_VISIBLE_DEVICES'] = ','.join(
gpuset) if gpuset else 'all'
container = client.containers.run(
image=docker_image,
command=docker_command,
network=network,
mem_limit=memory_bytes,
cpuset_cpus=cpuset_str,
environment=environment,
working_dir=working_dir,
entrypoint=entrypoint,
volumes=volumes,
user=user,
detach=detach,
runtime=runtime,
tty=tty,
stdin_open=tty,
)
logger.debug('Started Docker container for UUID %s, container ID %s,',
uuid, container.id)
return container
def start_bundle(self, bundle, bundle_store, parent_dict, username):
'''
Sets up all the temporary files and then dispatches the job.
username: the username of the owner of the bundle
Returns the bundle information.
'''
# Create a temporary directory
temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid)
temp_dir = os.path.realpath(temp_dir) # Follow symlinks
path_util.make_directory(temp_dir)
# Copy all the dependencies to that temporary directory.
pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir)
print >>sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (bundle.uuid, temp_dir)
for (source, target) in pairs:
path_util.copy(source, target, follow_symlinks=False)
# Set docker image
docker_image = self.default_docker_image
if bundle.metadata.request_docker_image:
docker_image = bundle.metadata.request_docker_image
# Write the command to be executed to a script.
if docker_image:
container_file = temp_dir + '.cid' # contains the docker container id
action_file = temp_dir + '.action' # send actions to the container (e.g., kill)
status_dir = temp_dir + '.status' # receive information from the container (e.g., memory)
script_file = temp_dir + '.sh' # main entry point
internal_script_file = temp_dir + '-internal.sh' # run inside the docker container
# Names of file inside the docker container
docker_temp_dir = bundle.uuid
docker_internal_script_file = bundle.uuid + '-internal.sh'
# 1) script_file starts the docker container and runs internal_script_file in docker.
# --rm removes the docker container once the job terminates (note that this makes things slow)
# -v mounts the internal and user scripts and the temp directory
# Trap SIGTERM and forward it to docker.
with open(script_file, 'w') as f:
# trap doesn't quite work reliably with Torque, so don't use it
#f.write('trap \'echo Killing docker container $(cat %s); docker kill $(cat %s); echo Killed: $?; exit 143\' TERM\n' % (container_file, container_file))
# Inspect doesn't tell us a lot, so don't use it
#f.write('while [ -e %s ]; do docker inspect $(cat %s) > %s; sleep 1; done &\n' % (temp_dir, container_file, status_dir))
# Monitor CPU/memory/disk
monitor_commands = [
# Report on status
'mkdir -p %s' % status_dir,
'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is
'cp -f $cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat %s' % (container_file, status_dir),
'cp -f $cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes %s' % (container_file, status_dir),
'cp -f $cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes %s' % (container_file, status_dir),
# Respond to actions
'[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s' % (action_file, action_file, container_file, action_file),
]
f.write('while [ -e %s ]; do %s; sleep 1; done &\n' % (temp_dir, '; '. join(monitor_commands)))
# Constrain resources
resource_args = ''
if bundle.metadata.request_memory:
resource_args += ' -m %s' % int(formatting.parse_size(bundle.metadata.request_memory))
# TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids
f.write("docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s & wait $!\n" % (
resource_args,
container_file, os.geteuid(),
temp_dir, docker_temp_dir,
internal_script_file, docker_internal_script_file,
docker_image, docker_internal_script_file))
# 2) internal_script_file runs the actual command inside the docker container
with open(internal_script_file, 'w') as f:
# Make sure I have a username
f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (os.getlogin(), os.geteuid(), os.getgid()))
# Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...)
f.write(". .bashrc || exit 1\n")
# Go into the temp directory
f.write("cd %s &&\n" % docker_temp_dir)
# Run the actual command
f.write('(%s) > stdout 2>stderr\n' % bundle.command)
else:
# Just run the command regularly without docker
script_file = temp_dir + '.sh'
with open(script_file, 'w') as f:
f.write("cd %s &&\n" % temp_dir)
f.write('(%s) > stdout 2>stderr\n' % bundle.command)
# Determine resources to request
resource_args = []
if bundle.metadata.request_time:
resource_args.extend(['--request_time', formatting.parse_duration(bundle.metadata.request_time)])
if bundle.metadata.request_memory:
resource_args.extend(['--request_memory', formatting.parse_size(bundle.metadata.request_memory)])
if bundle.metadata.request_cpus:
resource_args.extend(['--request_cpus', bundle.metadata.request_cpus])
if bundle.metadata.request_gpus:
resource_args.extend(['--request_gpus', bundle.metadata.request_gpus])
if bundle.metadata.request_queue:
resource_args.extend(['--request_queue', bundle.metadata.request_queue])
if username:
resource_args.extend(['--username', username])
# Start the command
args = self.dispatch_command.split() + ['start'] + map(str, resource_args) + [script_file]
if self.verbose >= 1: print '=== start_bundle(): running %s' % args
result = json.loads(self.run_command_get_stdout(args))
if self.verbose >= 1: print '=== start_bundle(): got %s' % result
# Return the information about the job.
return {
'bundle': bundle,
'temp_dir': temp_dir,
'job_handle': result['handle'],
'docker_image': docker_image,
}
def main():
parser = argparse.ArgumentParser(description='CodaLab worker.')
parser.add_argument('--tag',
help='Tag that allows for scheduling runs on specific '
'workers.')
parser.add_argument(
'--server',
default='https://worksheets.codalab.org',
help='URL of the CodaLab server, in the format '
'<http|https>://<hostname>[:<port>] (e.g., https://worksheets.codalab.org)',
)
parser.add_argument(
'--work-dir',
default='codalab-worker-scratch',
help='Directory where to store temporary bundle data, '
'including dependencies and the data from run '
'bundles.',
)
parser.add_argument('--network-prefix',
default='codalab_worker_network',
help='Docker network name prefix')
parser.add_argument(
'--cpuset',
type=str,
metavar='CPUSET_STR',
default='ALL',
help='Comma-separated list of CPUs in which to allow bundle execution, '
'(e.g., \"0,2,3\", \"1\").',
)
parser.add_argument(
'--gpuset',
type=str,
metavar='GPUSET_STR',
default='ALL',
help='Comma-separated list of GPUs in which to allow bundle execution. '
'Each GPU can be specified by its index or UUID'
'(e.g., \"0,1\", \"1\", \"GPU-62casdfasd-asfas...\"',
)
parser.add_argument(
'--max-work-dir-size',
type=str,
metavar='SIZE',
default='10g',
help='Maximum size of the temporary bundle data '
'(e.g., 3, 3k, 3m, 3g, 3t).',
)
parser.add_argument(
'--max-image-cache-size',
type=str,
metavar='SIZE',
help='Limit the disk space used to cache Docker images '
'for worker jobs to the specified amount (e.g. '
'3, 3k, 3m, 3g, 3t). If the limit is exceeded, '
'the least recently used images are removed first. '
'Worker will not remove any images if this option '
'is not specified.',
)
parser.add_argument(
'--password-file',
help='Path to the file containing the username and '
'password for logging into the bundle service, '
'each on a separate line. If not specified, the '
'password is read from standard input.',
)
parser.add_argument('--verbose',
action='store_true',
help='Whether to output verbose log messages.')
parser.add_argument(
'--exit-when-idle',
action='store_true',
help=
'If specified the worker quits if it finds itself with no jobs after a checkin',
)
parser.add_argument(
'--id',
default='%s(%d)' % (socket.gethostname(), os.getpid()),
help='Internal use: ID to use for the worker.',
)
parser.add_argument(
'--shared-file-system',
action='store_true',
help='Internal use: Whether the file system containing '
'bundle data is shared between the bundle service '
'and the worker.',
)
args = parser.parse_args()
# Get the username and password.
logger.info('Connecting to %s' % args.server)
if args.password_file:
if os.stat(args.password_file).st_mode & (stat.S_IRWXG | stat.S_IRWXO):
print(
"""
Permissions on password file are too lax.
Only the user should be allowed to access the file.
On Linux, run:
chmod 600 %s""" % args.password_file,
file=sys.stderr,
)
exit(1)
with open(args.password_file) as f:
username = f.readline().strip()
password = f.readline().strip()
else:
username = os.environ.get('CODALAB_USERNAME')
if username is None:
username = input('Username: '******'CODALAB_PASSWORD')
if password is None:
password = getpass.getpass()
# Set up logging.
if args.verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
level=logging.INFO)
try:
bundle_service = BundleServiceClient(args.server, username, password)
except BundleAuthException as ex:
logger.error(
'Cannot log into the bundle service. Please check your worker credentials.\n'
)
logger.debug('Auth error: {}'.format(ex))
return
max_work_dir_size_bytes = parse_size(args.max_work_dir_size)
if args.max_image_cache_size is None:
max_images_bytes = None
else:
max_images_bytes = parse_size(args.max_image_cache_size)
if not os.path.exists(args.work_dir):
logging.debug('Work dir %s doesn\'t exist, creating.', args.work_dir)
os.makedirs(args.work_dir, 0o770)
def create_local_run_manager(worker):
"""
To avoid circular dependencies the Worker initializes takes a RunManager factory
to initilize its run manager. This method creates a LocalFilesystem-Docker RunManager
which is the default execution architecture Codalab uses
"""
docker_runtime = docker_utils.get_available_runtime()
cpuset = parse_cpuset_args(args.cpuset)
gpuset = parse_gpuset_args(args.gpuset)
dependency_manager = LocalFileSystemDependencyManager(
os.path.join(args.work_dir, 'dependencies-state.json'),
bundle_service,
args.work_dir,
max_work_dir_size_bytes,
)
image_manager = DockerImageManager(
os.path.join(args.work_dir, 'images-state.json'), max_images_bytes)
return LocalRunManager(
worker,
image_manager,
dependency_manager,
os.path.join(args.work_dir, 'run-state.json'),
cpuset,
gpuset,
args.work_dir,
docker_runtime=docker_runtime,
docker_network_prefix=args.network_prefix,
)
worker = Worker(
create_local_run_manager,
os.path.join(args.work_dir, 'worker-state.json'),
args.id,
args.tag,
args.work_dir,
args.exit_when_idle,
bundle_service,
)
# Register a signal handler to ensure safe shutdown.
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]:
signal.signal(sig, lambda signup, frame: worker.signal())
# BEGIN: DO NOT CHANGE THIS LINE UNLESS YOU KNOW WHAT YOU ARE DOING
# THIS IS HERE TO KEEP TEST-CLI FROM HANGING
logger.info('Worker started!')
# END
worker.start()
def _deserialize(self, value, attr, data):
return formatting.parse_size(value)
def _compute_request_disk(self, bundle):
"""
Compute the disk limit used for scheduling the run.
"""
return formatting.parse_size(bundle.metadata.request_disk)
def start_bundle(self, bundle, bundle_store, parent_dict, username):
'''
Sets up all the temporary files and then dispatches the job.
username: the username of the owner of the bundle
Returns the bundle information.
'''
# Create a temporary directory
temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid)
temp_dir = os.path.realpath(temp_dir) # Follow symlinks
path_util.make_directory(temp_dir)
# Copy all the dependencies to that temporary directory.
pairs = bundle.get_dependency_paths(bundle_store, parent_dict,
temp_dir)
print >> sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (
bundle.uuid, temp_dir)
for (source, target) in pairs:
path_util.copy(source, target, follow_symlinks=False)
# Set defaults for the dispatcher.
docker_image = self.default_docker_image
if bundle.metadata.request_docker_image:
docker_image = bundle.metadata.request_docker_image
request_time = self.default_request_time
if bundle.metadata.request_time:
request_time = bundle.metadata.request_time
request_memory = self.default_request_memory
if bundle.metadata.request_memory:
request_memory = bundle.metadata.request_memory
request_cpus = self.default_request_cpus
if bundle.metadata.request_cpus:
request_cpus = bundle.metadata.request_cpus
request_gpus = self.default_request_gpus
if bundle.metadata.request_gpus:
request_gpus = bundle.metadata.request_gpus
request_queue = self.default_request_queue
if bundle.metadata.request_queue:
request_queue = bundle.metadata.request_queue
request_priority = self.default_request_priority
if bundle.metadata.request_priority:
request_priority = bundle.metadata.request_priority
script_file = temp_dir + '.sh' # main entry point
ptr_temp_dir = '$temp_dir'
# 1) If no argument to script_file, use the temp_dir (e.g., Torque, master/worker share file system).
# 2) If argument is 'use_script_for_temp_dir', use the script to determine temp_dir (e.g., qsub, no master/worker do not share file system).
set_temp_dir_header = 'if [ -z "$1" ]; then temp_dir=' + temp_dir + '; else temp_dir=`readlink -f $0 | sed -e \'s/\\.sh$//\'`; fi\n'
# Write the command to be executed to a script.
if docker_image:
internal_script_file = temp_dir + '-internal.sh' # run inside the docker container
# These paths depend on $temp_dir, an environment variable which will be set (referenced inside script_file)
ptr_container_file = ptr_temp_dir + '.cid' # contains the docker container id
ptr_action_file = ptr_temp_dir + '.action' # send actions to the container (e.g., kill)
ptr_status_dir = ptr_temp_dir + '.status' # receive information from the container (e.g., memory)
ptr_script_file = ptr_temp_dir + '.sh' # main entry point
ptr_internal_script_file = ptr_temp_dir + '-internal.sh' # run inside the docker container
# Names of file inside the docker container
docker_temp_dir = bundle.uuid
docker_internal_script_file = bundle.uuid + '-internal.sh'
# 1) script_file starts the docker container and runs internal_script_file in docker.
# --rm removes the docker container once the job terminates (note that this makes things slow)
# -v mounts the internal and user scripts and the temp directory
# Trap SIGTERM and forward it to docker.
with open(script_file, 'w') as f:
f.write(set_temp_dir_header)
# Monitor CPU/memory/disk
def copy_if_exists(source_template, arg, target):
source = source_template % arg
# -f because target might be read-only
return 'if [ -e %s ] && [ -e %s ]; then cp -f %s %s; fi' % (
arg, source, source, target)
monitor_commands = [
# Report on status (memory, cpu, etc.)
'mkdir -p %s' % ptr_status_dir,
'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is
copy_if_exists(
'$cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat',
ptr_container_file, ptr_status_dir),
copy_if_exists(
'$cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes',
ptr_container_file, ptr_status_dir),
copy_if_exists(
'$cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes',
ptr_container_file, ptr_status_dir),
# Respond to kill action
'[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s'
% (ptr_action_file, ptr_action_file, ptr_container_file,
ptr_action_file),
# Sleep
'sleep 1',
]
f.write('while [ -e %s ]; do\n %s\ndone &\n' %
(ptr_temp_dir, '\n '.join(monitor_commands)))
# Tell docker to constrain resources (memory).
# Note: limiting memory is not always supported. See:
# http://programster.blogspot.com/2014/09/docker-implementing-container-memory.html
resource_args = ''
if bundle.metadata.request_memory:
resource_args += ' -m %s' % int(
formatting.parse_size(bundle.metadata.request_memory))
# TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids
f.write(
"docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s >%s/stdout 2>%s/stderr & wait $!\n"
%
(resource_args, ptr_container_file, os.geteuid(),
ptr_temp_dir, docker_temp_dir, ptr_internal_script_file,
docker_internal_script_file, docker_image,
docker_internal_script_file, ptr_temp_dir, ptr_temp_dir))
# 2) internal_script_file runs the actual command inside the docker container
with open(internal_script_file, 'w') as f:
# Make sure I have a username
username = pwd.getpwuid(os.getuid())[
0] # do this because os.getlogin() doesn't always work
f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" %
(username, os.geteuid(), os.getgid()))
# Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...)
f.write(". .bashrc || exit 1\n")
# Go into the temp directory
f.write("cd %s &&\n" % docker_temp_dir)
# Run the actual command
f.write('(%s) >>stdout 2>>stderr\n' % bundle.command)
else:
# Just run the command regularly without docker
with open(script_file, 'w') as f:
f.write(set_temp_dir_header)
f.write("cd %s &&\n" % ptr_temp_dir)
f.write('(%s) >stdout 2>stderr\n' % bundle.command)
# Determine resources to request
resource_args = []
if request_time:
resource_args.extend(
['--request_time',
formatting.parse_duration(request_time)])
if request_memory:
resource_args.extend(
['--request_memory',
formatting.parse_size(request_memory)])
if request_cpus:
resource_args.extend(['--request_cpus', request_cpus])
if request_gpus:
resource_args.extend(['--request_gpus', request_gpus])
if request_queue:
resource_args.extend(['--request_queue', request_queue])
if request_priority:
resource_args.extend(['--request_priority', request_priority])
if username:
resource_args.extend(['--username', username])
# Start the command
args = self.dispatch_command.split() + ['start'] + map(
str, resource_args) + [script_file]
if self.verbose >= 1: print '=== start_bundle(): running %s' % args
result = json.loads(self.run_command_get_stdout(args))
if self.verbose >= 1: print '=== start_bundle(): got %s' % result
# Return the information about the job.
return {
'bundle': bundle,
'temp_dir': temp_dir,
'job_handle': result['handle'],
'docker_image': docker_image,
}
def start_bundle(self, bundle, bundle_store, parent_dict, username):
'''
Sets up all the temporary files and then dispatches the job.
username: the username of the owner of the bundle
Returns the bundle information.
'''
# Create a temporary directory
temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid)
temp_dir = os.path.realpath(temp_dir) # Follow symlinks
path_util.make_directory(temp_dir)
# Copy all the dependencies to that temporary directory.
pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir)
print >>sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (bundle.uuid, temp_dir)
for (source, target) in pairs:
path_util.copy(source, target, follow_symlinks=False)
# Set defaults for the dispatcher.
docker_image = self.default_docker_image
if bundle.metadata.request_docker_image:
docker_image = bundle.metadata.request_docker_image
request_time = self.default_request_time
if bundle.metadata.request_time:
request_time = bundle.metadata.request_time
request_memory = self.default_request_memory
if bundle.metadata.request_memory:
request_memory = bundle.metadata.request_memory
request_cpus = self.default_request_cpus
if bundle.metadata.request_cpus:
request_cpus = bundle.metadata.request_cpus
request_gpus = self.default_request_gpus
if bundle.metadata.request_gpus:
request_gpus = bundle.metadata.request_gpus
request_queue = self.default_request_queue
if bundle.metadata.request_queue:
request_queue = bundle.metadata.request_queue
request_priority = self.default_request_priority
if bundle.metadata.request_priority:
request_priority = bundle.metadata.request_priority
script_file = temp_dir + '.sh' # main entry point
ptr_temp_dir = '$temp_dir'
# 1) If no argument to script_file, use the temp_dir (e.g., Torque, master/worker share file system).
# 2) If argument is 'use_script_for_temp_dir', use the script to determine temp_dir (e.g., qsub, no master/worker do not share file system).
set_temp_dir_header = 'if [ -z "$1" ]; then temp_dir=' + temp_dir + '; else temp_dir=`readlink -f $0 | sed -e \'s/\\.sh$//\'`; fi\n'
# Write the command to be executed to a script.
if docker_image:
internal_script_file = temp_dir + '-internal.sh' # run inside the docker container
# These paths depend on $temp_dir, an environment variable which will be set (referenced inside script_file)
ptr_container_file = ptr_temp_dir + '.cid' # contains the docker container id
ptr_action_file = ptr_temp_dir + '.action' # send actions to the container (e.g., kill)
ptr_status_dir = ptr_temp_dir + '.status' # receive information from the container (e.g., memory)
ptr_script_file = ptr_temp_dir + '.sh' # main entry point
ptr_internal_script_file = ptr_temp_dir + '-internal.sh' # run inside the docker container
# Names of file inside the docker container
docker_temp_dir = bundle.uuid
docker_internal_script_file = bundle.uuid + '-internal.sh'
# 1) script_file starts the docker container and runs internal_script_file in docker.
# --rm removes the docker container once the job terminates (note that this makes things slow)
# -v mounts the internal and user scripts and the temp directory
# Trap SIGTERM and forward it to docker.
with open(script_file, 'w') as f:
f.write(set_temp_dir_header)
# Monitor CPU/memory/disk
def copy_if_exists(source_template, arg, target):
source = source_template % arg
# -f because target might be read-only
return 'if [ -e %s ] && [ -e %s ]; then cp -f %s %s; fi' % (arg, source, source, target)
monitor_commands = [
# Report on status (memory, cpu, etc.)
'mkdir -p %s' % ptr_status_dir,
'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is
copy_if_exists('$cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat', ptr_container_file, ptr_status_dir),
copy_if_exists('$cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes', ptr_container_file, ptr_status_dir),
copy_if_exists('$cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes', ptr_container_file, ptr_status_dir),
# Respond to kill action
'[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s' % (ptr_action_file, ptr_action_file, ptr_container_file, ptr_action_file),
# Sleep
'sleep 1',
]
f.write('while [ -e %s ]; do\n %s\ndone &\n' % (ptr_temp_dir, '\n '. join(monitor_commands)))
# Tell docker to constrain resources (memory).
# Note: limiting memory is not always supported. See:
# http://programster.blogspot.com/2014/09/docker-implementing-container-memory.html
resource_args = ''
if bundle.metadata.request_memory:
resource_args += ' -m %s' % int(formatting.parse_size(bundle.metadata.request_memory))
# TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids
f.write("docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s & wait $!\n" % (
resource_args,
ptr_container_file,
os.geteuid(),
ptr_temp_dir, docker_temp_dir,
ptr_internal_script_file, docker_internal_script_file,
docker_image,
docker_internal_script_file))
# 2) internal_script_file runs the actual command inside the docker container
with open(internal_script_file, 'w') as f:
# Make sure I have a username
username = pwd.getpwuid(os.getuid())[0] # do this because os.getlogin() doesn't always work
f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (username, os.geteuid(), os.getgid()))
# Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...)
f.write(". .bashrc || exit 1\n")
# Go into the temp directory
f.write("cd %s &&\n" % docker_temp_dir)
# Run the actual command
f.write('(%s) > stdout 2>stderr\n' % bundle.command)
else:
# Just run the command regularly without docker
with open(script_file, 'w') as f:
f.write(set_temp_dir_header)
f.write("cd %s &&\n" % ptr_temp_dir)
f.write('(%s) > stdout 2>stderr\n' % bundle.command)
# Determine resources to request
resource_args = []
if request_time:
resource_args.extend(['--request_time', formatting.parse_duration(request_time)])
if request_memory:
resource_args.extend(['--request_memory', formatting.parse_size(request_memory)])
if request_cpus:
resource_args.extend(['--request_cpus', request_cpus])
if request_gpus:
resource_args.extend(['--request_gpus', request_gpus])
if request_queue:
resource_args.extend(['--request_queue', request_queue])
if request_priority:
resource_args.extend(['--request_priority', request_priority])
if username:
resource_args.extend(['--username', username])
# Start the command
args = self.dispatch_command.split() + ['start'] + map(str, resource_args) + [script_file]
if self.verbose >= 1: print '=== start_bundle(): running %s' % args
result = json.loads(self.run_command_get_stdout(args))
if self.verbose >= 1: print '=== start_bundle(): got %s' % result
# Return the information about the job.
return {
'bundle': bundle,
'temp_dir': temp_dir,
'job_handle': result['handle'],
'docker_image': docker_image,
}
