def validate(self, metadata_specs): ''' Check that this metadata has the correct metadata keys and that it has metadata values of the correct types. ''' expected_keys = set(spec.key for spec in metadata_specs) for key in self._metadata_keys: if key not in expected_keys: raise UsageError('Unexpected metadata key: %s' % (key,)) for spec in metadata_specs: if spec.key in self._metadata_keys: value = getattr(self, spec.key) if spec.type is float and isinstance(value, int): # cast int to float value = float(value) # Validate formatted string fields if issubclass(spec.type, basestring) and spec.formatting is not None and value: try: if spec.formatting == 'duration': formatting.parse_duration(value) elif spec.formatting == 'size': formatting.parse_size(value) elif spec.formatting == 'date': formatting.parse_datetime(value) except ValueError as e: raise UsageError(e.message) if value is not None and not isinstance(value, spec.type): raise UsageError( 'Metadata value for %s should be of type %s, was %s (type %s)' % (spec.key, spec.type.__name__, value, type(value).__name__) ) elif not spec.generated: raise UsageError('Missing metadata key: %s' % (spec.key,))
def filter_bundles(self, bundles: BundlesPayload) -> BundlesPayload: filtered_bundles: BundlesPayload = [] worker_memory_bytes: int = parse_size('{}m'.format( self.args.memory_mb)) logger.info( f"Current worker manager allocates {self.args.cpus} CPUs, {self.args.gpus} GPUs, " f"and {worker_memory_bytes} bytes of RAM") for bundle in bundles: # Filter bundles based on the resources specified when creating the worker manager if bundle['metadata']['request_cpus'] > self.args.cpus: logger.info( 'Filtered out bundle {} based on unfulfillable resource requested: request_cpus={}' .format( bundle['uuid'], bundle['metadata']['request_cpus'], )) elif bundle['metadata']['request_gpus'] > self.args.gpus: logger.info( 'Filtered out bundle {} based on unfulfillable resource requested: request_gpus={}' .format( bundle['uuid'], bundle['metadata']['request_gpus'], )) elif parse_size(bundle['metadata'] ['request_memory']) > worker_memory_bytes: logger.info( 'Filtered out bundle {} based on unfulfillable resource requested: request_memory={}' .format( bundle['uuid'], bundle['metadata']['request_memory'], )) else: filtered_bundles.append(bundle) return filtered_bundles
def validate(self, metadata_specs): ''' Check that this metadata has the correct metadata keys and that it has metadata values of the correct types. ''' expected_keys = set(spec.key for spec in metadata_specs) for key in self._metadata_keys: if key not in expected_keys: raise UsageError('Unexpected metadata key: %s' % (key, )) for spec in metadata_specs: if spec.key in self._metadata_keys: value = getattr(self, spec.key) if spec.type is float and isinstance(value, int): # cast int to float value = float(value) # Validate formatted string fields if issubclass(spec.type, str) and spec.formatting is not None and value: try: if spec.formatting == 'duration': formatting.parse_duration(value) elif spec.formatting == 'size': formatting.parse_size(value) elif spec.formatting == 'date': formatting.parse_datetime(value) except ValueError as e: raise UsageError(str(e)) if value is not None and not isinstance(value, spec.type): raise UsageError( 'Metadata value for %s should be of type %s, was %s (type %s)' % (spec.key, spec.type.__name__, value, type(value).__name__)) elif not spec.generated and not spec.optional: raise UsageError('Missing metadata key: %s' % (spec.key, ))
def _compute_request_disk(self, bundle): """ Compute the disk limit used for scheduling the run. """ #TODO: Remove this once we want to deprecate old versions if not bundle.metadata.request_disk: return formatting.parse_size('4g') return formatting.parse_size(bundle.metadata.request_disk)
def _compute_request_memory(self, bundle): """ Compute the memory limit used for scheduling the run. The default of 2g is for backwards compatibilty for runs from before when we added client-side defaults """ if not bundle.metadata.request_memory: return formatting.parse_size('2g') return formatting.parse_size(bundle.metadata.request_memory)
def _compute_request_memory(self, bundle): """ Compute the memory limit used for scheduling the run. """ if bundle.metadata.request_memory: return formatting.parse_size(bundle.metadata.request_memory) return self._default_request_memory
def default_user_info(self): info = self.config['server'].get( 'default_user_info', {'time_quota': '1y', 'disk_quota': '1t'} ) return { 'time_quota': formatting.parse_duration(info['time_quota']), 'disk_quota': formatting.parse_size(info['disk_quota']), }
def default_user_info(self): info = self.config['server'].get('default_user_info', { 'time_quota': '1y', 'disk_quota': '1t' }) info['time_quota'] = formatting.parse_duration(info['time_quota']) info['disk_quota'] = formatting.parse_size(info['disk_quota']) return info
def default_user_info(self): info = self.config['server'].get('default_user_info', { 'disk_quota': '1t', 'parallel_run_quota': 3 }) return { 'disk_quota': formatting.parse_size(info['disk_quota']), 'parallel_run_quota': info['parallel_run_quota'], }
def _compute_request_disk(self, bundle): """ Compute the disk limit used for scheduling the run. The default is min(disk quota the user has left, global max) """ if not bundle.metadata.request_disk: return min( self._model.get_user_disk_quota_left(bundle.owner_id) - 1, self._max_request_disk) return formatting.parse_size(bundle.metadata.request_disk)
def _compute_request_disk(self, bundle): """ Compute the disk limit used for scheduling the run. The default is min(disk quota the user has left, global max) """ if not bundle.metadata.request_disk: return min( self._model.get_user_disk_quota_left(bundle.owner_id) - 1, self._max_request_disk ) return formatting.parse_size(bundle.metadata.request_disk)
def filter_bundles(self, bundles: BundlesPayload) -> BundlesPayload: filtered_bundles: BundlesPayload = [] for bundle in bundles: # Filter bundles based on the resources specified when creating the worker manager worker_memory_bytes: int = parse_size('{}m'.format( self.args.memory_mb)) if (bundle['metadata']['request_cpus'] <= self.args.cpus and bundle['metadata']['request_gpus'] <= self.args.gpus and parse_size(bundle['metadata']['request_memory']) <= worker_memory_bytes): filtered_bundles.append(bundle) else: logger.info( 'Filtered out bundle {} based on resources requested: request_cpus={}, request_gpus={}, request_memory={}' .format( bundle['uuid'], bundle['metadata']['request_cpus'], bundle['metadata']['request_gpus'], bundle['metadata']['request_memory'], )) return filtered_bundles
def start_bundle_container( bundle_path, uuid, dependencies, command, docker_image, network=None, cpuset=None, gpuset=None, memory_bytes=0, detach=True, tty=False, runtime=DEFAULT_RUNTIME, ): # Impose a minimum container request memory 4mb, same as docker's minimum allowed value # https://docs.docker.com/config/containers/resource_constraints/#limit-a-containers-access-to-memory # When using the REST api, it is allowed to set Memory to 0 but that means the container has unbounded # access to the host machine's memory, which we have decided to not allow if memory_bytes < parse_size('4m'): raise DockerException('Minimum memory must be 4m ({} bytes)'.format( parse_size('4m'))) if not command.endswith(';'): command = '{};'.format(command) docker_command = ['bash', '-c', '( %s ) >stdout 2>stderr' % command] docker_bundle_path = '/' + uuid volumes = get_bundle_container_volume_binds(bundle_path, docker_bundle_path, dependencies) environment = {'HOME': docker_bundle_path} working_dir = docker_bundle_path # Unset entrypoint regardless of image entrypoint = '' cpuset_str = ','.join(cpuset) if cpuset else '' # Get user/group that owns the bundle directory # Then we can ensure that any created files are owned by the user/group # that owns the bundle directory, not root. bundle_stat = os.stat(bundle_path) uid = bundle_stat.st_uid gid = bundle_stat.st_gid # TODO: Fix potential permissions issues arising from this setting # This can cause problems if users expect to run as a specific user user = '******' % (uid, gid) if runtime == NVIDIA_RUNTIME: # nvidia-docker runtime uses this env variable to allocate GPUs environment['NVIDIA_VISIBLE_DEVICES'] = ','.join( gpuset) if gpuset else 'all' container = client.containers.run( image=docker_image, command=docker_command, network=network, mem_limit=memory_bytes, cpuset_cpus=cpuset_str, environment=environment, working_dir=working_dir, entrypoint=entrypoint, volumes=volumes, user=user, detach=detach, runtime=runtime, tty=tty, stdin_open=tty, ) logger.debug('Started Docker container for UUID %s, container ID %s,', uuid, container.id) return container
def start_bundle(self, bundle, bundle_store, parent_dict, username): ''' Sets up all the temporary files and then dispatches the job. username: the username of the owner of the bundle Returns the bundle information. ''' # Create a temporary directory temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid) temp_dir = os.path.realpath(temp_dir) # Follow symlinks path_util.make_directory(temp_dir) # Copy all the dependencies to that temporary directory. pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir) print >>sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (bundle.uuid, temp_dir) for (source, target) in pairs: path_util.copy(source, target, follow_symlinks=False) # Set docker image docker_image = self.default_docker_image if bundle.metadata.request_docker_image: docker_image = bundle.metadata.request_docker_image # Write the command to be executed to a script. if docker_image: container_file = temp_dir + '.cid' # contains the docker container id action_file = temp_dir + '.action' # send actions to the container (e.g., kill) status_dir = temp_dir + '.status' # receive information from the container (e.g., memory) script_file = temp_dir + '.sh' # main entry point internal_script_file = temp_dir + '-internal.sh' # run inside the docker container # Names of file inside the docker container docker_temp_dir = bundle.uuid docker_internal_script_file = bundle.uuid + '-internal.sh' # 1) script_file starts the docker container and runs internal_script_file in docker. # --rm removes the docker container once the job terminates (note that this makes things slow) # -v mounts the internal and user scripts and the temp directory # Trap SIGTERM and forward it to docker. with open(script_file, 'w') as f: # trap doesn't quite work reliably with Torque, so don't use it #f.write('trap \'echo Killing docker container $(cat %s); docker kill $(cat %s); echo Killed: $?; exit 143\' TERM\n' % (container_file, container_file)) # Inspect doesn't tell us a lot, so don't use it #f.write('while [ -e %s ]; do docker inspect $(cat %s) > %s; sleep 1; done &\n' % (temp_dir, container_file, status_dir)) # Monitor CPU/memory/disk monitor_commands = [ # Report on status 'mkdir -p %s' % status_dir, 'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is 'cp -f $cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat %s' % (container_file, status_dir), 'cp -f $cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes %s' % (container_file, status_dir), 'cp -f $cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes %s' % (container_file, status_dir), # Respond to actions '[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s' % (action_file, action_file, container_file, action_file), ] f.write('while [ -e %s ]; do %s; sleep 1; done &\n' % (temp_dir, '; '. join(monitor_commands))) # Constrain resources resource_args = '' if bundle.metadata.request_memory: resource_args += ' -m %s' % int(formatting.parse_size(bundle.metadata.request_memory)) # TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids f.write("docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s & wait $!\n" % ( resource_args, container_file, os.geteuid(), temp_dir, docker_temp_dir, internal_script_file, docker_internal_script_file, docker_image, docker_internal_script_file)) # 2) internal_script_file runs the actual command inside the docker container with open(internal_script_file, 'w') as f: # Make sure I have a username f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (os.getlogin(), os.geteuid(), os.getgid())) # Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...) f.write(". .bashrc || exit 1\n") # Go into the temp directory f.write("cd %s &&\n" % docker_temp_dir) # Run the actual command f.write('(%s) > stdout 2>stderr\n' % bundle.command) else: # Just run the command regularly without docker script_file = temp_dir + '.sh' with open(script_file, 'w') as f: f.write("cd %s &&\n" % temp_dir) f.write('(%s) > stdout 2>stderr\n' % bundle.command) # Determine resources to request resource_args = [] if bundle.metadata.request_time: resource_args.extend(['--request_time', formatting.parse_duration(bundle.metadata.request_time)]) if bundle.metadata.request_memory: resource_args.extend(['--request_memory', formatting.parse_size(bundle.metadata.request_memory)]) if bundle.metadata.request_cpus: resource_args.extend(['--request_cpus', bundle.metadata.request_cpus]) if bundle.metadata.request_gpus: resource_args.extend(['--request_gpus', bundle.metadata.request_gpus]) if bundle.metadata.request_queue: resource_args.extend(['--request_queue', bundle.metadata.request_queue]) if username: resource_args.extend(['--username', username]) # Start the command args = self.dispatch_command.split() + ['start'] + map(str, resource_args) + [script_file] if self.verbose >= 1: print '=== start_bundle(): running %s' % args result = json.loads(self.run_command_get_stdout(args)) if self.verbose >= 1: print '=== start_bundle(): got %s' % result # Return the information about the job. return { 'bundle': bundle, 'temp_dir': temp_dir, 'job_handle': result['handle'], 'docker_image': docker_image, }
def main(): parser = argparse.ArgumentParser(description='CodaLab worker.') parser.add_argument('--tag', help='Tag that allows for scheduling runs on specific ' 'workers.') parser.add_argument( '--server', default='https://worksheets.codalab.org', help='URL of the CodaLab server, in the format ' '<http|https>://<hostname>[:<port>] (e.g., https://worksheets.codalab.org)', ) parser.add_argument( '--work-dir', default='codalab-worker-scratch', help='Directory where to store temporary bundle data, ' 'including dependencies and the data from run ' 'bundles.', ) parser.add_argument('--network-prefix', default='codalab_worker_network', help='Docker network name prefix') parser.add_argument( '--cpuset', type=str, metavar='CPUSET_STR', default='ALL', help='Comma-separated list of CPUs in which to allow bundle execution, ' '(e.g., \"0,2,3\", \"1\").', ) parser.add_argument( '--gpuset', type=str, metavar='GPUSET_STR', default='ALL', help='Comma-separated list of GPUs in which to allow bundle execution. ' 'Each GPU can be specified by its index or UUID' '(e.g., \"0,1\", \"1\", \"GPU-62casdfasd-asfas...\"', ) parser.add_argument( '--max-work-dir-size', type=str, metavar='SIZE', default='10g', help='Maximum size of the temporary bundle data ' '(e.g., 3, 3k, 3m, 3g, 3t).', ) parser.add_argument( '--max-image-cache-size', type=str, metavar='SIZE', help='Limit the disk space used to cache Docker images ' 'for worker jobs to the specified amount (e.g. ' '3, 3k, 3m, 3g, 3t). If the limit is exceeded, ' 'the least recently used images are removed first. ' 'Worker will not remove any images if this option ' 'is not specified.', ) parser.add_argument( '--password-file', help='Path to the file containing the username and ' 'password for logging into the bundle service, ' 'each on a separate line. If not specified, the ' 'password is read from standard input.', ) parser.add_argument('--verbose', action='store_true', help='Whether to output verbose log messages.') parser.add_argument( '--exit-when-idle', action='store_true', help= 'If specified the worker quits if it finds itself with no jobs after a checkin', ) parser.add_argument( '--id', default='%s(%d)' % (socket.gethostname(), os.getpid()), help='Internal use: ID to use for the worker.', ) parser.add_argument( '--shared-file-system', action='store_true', help='Internal use: Whether the file system containing ' 'bundle data is shared between the bundle service ' 'and the worker.', ) args = parser.parse_args() # Get the username and password. logger.info('Connecting to %s' % args.server) if args.password_file: if os.stat(args.password_file).st_mode & (stat.S_IRWXG | stat.S_IRWXO): print( """ Permissions on password file are too lax. Only the user should be allowed to access the file. On Linux, run: chmod 600 %s""" % args.password_file, file=sys.stderr, ) exit(1) with open(args.password_file) as f: username = f.readline().strip() password = f.readline().strip() else: username = os.environ.get('CODALAB_USERNAME') if username is None: username = input('Username: '******'CODALAB_PASSWORD') if password is None: password = getpass.getpass() # Set up logging. if args.verbose: logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) else: logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO) try: bundle_service = BundleServiceClient(args.server, username, password) except BundleAuthException as ex: logger.error( 'Cannot log into the bundle service. Please check your worker credentials.\n' ) logger.debug('Auth error: {}'.format(ex)) return max_work_dir_size_bytes = parse_size(args.max_work_dir_size) if args.max_image_cache_size is None: max_images_bytes = None else: max_images_bytes = parse_size(args.max_image_cache_size) if not os.path.exists(args.work_dir): logging.debug('Work dir %s doesn\'t exist, creating.', args.work_dir) os.makedirs(args.work_dir, 0o770) def create_local_run_manager(worker): """ To avoid circular dependencies the Worker initializes takes a RunManager factory to initilize its run manager. This method creates a LocalFilesystem-Docker RunManager which is the default execution architecture Codalab uses """ docker_runtime = docker_utils.get_available_runtime() cpuset = parse_cpuset_args(args.cpuset) gpuset = parse_gpuset_args(args.gpuset) dependency_manager = LocalFileSystemDependencyManager( os.path.join(args.work_dir, 'dependencies-state.json'), bundle_service, args.work_dir, max_work_dir_size_bytes, ) image_manager = DockerImageManager( os.path.join(args.work_dir, 'images-state.json'), max_images_bytes) return LocalRunManager( worker, image_manager, dependency_manager, os.path.join(args.work_dir, 'run-state.json'), cpuset, gpuset, args.work_dir, docker_runtime=docker_runtime, docker_network_prefix=args.network_prefix, ) worker = Worker( create_local_run_manager, os.path.join(args.work_dir, 'worker-state.json'), args.id, args.tag, args.work_dir, args.exit_when_idle, bundle_service, ) # Register a signal handler to ensure safe shutdown. for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]: signal.signal(sig, lambda signup, frame: worker.signal()) # BEGIN: DO NOT CHANGE THIS LINE UNLESS YOU KNOW WHAT YOU ARE DOING # THIS IS HERE TO KEEP TEST-CLI FROM HANGING logger.info('Worker started!') # END worker.start()
def _deserialize(self, value, attr, data): return formatting.parse_size(value)
def _compute_request_disk(self, bundle): """ Compute the disk limit used for scheduling the run. """ return formatting.parse_size(bundle.metadata.request_disk)
def start_bundle(self, bundle, bundle_store, parent_dict, username): ''' Sets up all the temporary files and then dispatches the job. username: the username of the owner of the bundle Returns the bundle information. ''' # Create a temporary directory temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid) temp_dir = os.path.realpath(temp_dir) # Follow symlinks path_util.make_directory(temp_dir) # Copy all the dependencies to that temporary directory. pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir) print >> sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % ( bundle.uuid, temp_dir) for (source, target) in pairs: path_util.copy(source, target, follow_symlinks=False) # Set defaults for the dispatcher. docker_image = self.default_docker_image if bundle.metadata.request_docker_image: docker_image = bundle.metadata.request_docker_image request_time = self.default_request_time if bundle.metadata.request_time: request_time = bundle.metadata.request_time request_memory = self.default_request_memory if bundle.metadata.request_memory: request_memory = bundle.metadata.request_memory request_cpus = self.default_request_cpus if bundle.metadata.request_cpus: request_cpus = bundle.metadata.request_cpus request_gpus = self.default_request_gpus if bundle.metadata.request_gpus: request_gpus = bundle.metadata.request_gpus request_queue = self.default_request_queue if bundle.metadata.request_queue: request_queue = bundle.metadata.request_queue request_priority = self.default_request_priority if bundle.metadata.request_priority: request_priority = bundle.metadata.request_priority script_file = temp_dir + '.sh' # main entry point ptr_temp_dir = '$temp_dir' # 1) If no argument to script_file, use the temp_dir (e.g., Torque, master/worker share file system). # 2) If argument is 'use_script_for_temp_dir', use the script to determine temp_dir (e.g., qsub, no master/worker do not share file system). set_temp_dir_header = 'if [ -z "$1" ]; then temp_dir=' + temp_dir + '; else temp_dir=`readlink -f $0 | sed -e \'s/\\.sh$//\'`; fi\n' # Write the command to be executed to a script. if docker_image: internal_script_file = temp_dir + '-internal.sh' # run inside the docker container # These paths depend on $temp_dir, an environment variable which will be set (referenced inside script_file) ptr_container_file = ptr_temp_dir + '.cid' # contains the docker container id ptr_action_file = ptr_temp_dir + '.action' # send actions to the container (e.g., kill) ptr_status_dir = ptr_temp_dir + '.status' # receive information from the container (e.g., memory) ptr_script_file = ptr_temp_dir + '.sh' # main entry point ptr_internal_script_file = ptr_temp_dir + '-internal.sh' # run inside the docker container # Names of file inside the docker container docker_temp_dir = bundle.uuid docker_internal_script_file = bundle.uuid + '-internal.sh' # 1) script_file starts the docker container and runs internal_script_file in docker. # --rm removes the docker container once the job terminates (note that this makes things slow) # -v mounts the internal and user scripts and the temp directory # Trap SIGTERM and forward it to docker. with open(script_file, 'w') as f: f.write(set_temp_dir_header) # Monitor CPU/memory/disk def copy_if_exists(source_template, arg, target): source = source_template % arg # -f because target might be read-only return 'if [ -e %s ] && [ -e %s ]; then cp -f %s %s; fi' % ( arg, source, source, target) monitor_commands = [ # Report on status (memory, cpu, etc.) 'mkdir -p %s' % ptr_status_dir, 'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is copy_if_exists( '$cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat', ptr_container_file, ptr_status_dir), copy_if_exists( '$cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes', ptr_container_file, ptr_status_dir), copy_if_exists( '$cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes', ptr_container_file, ptr_status_dir), # Respond to kill action '[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s' % (ptr_action_file, ptr_action_file, ptr_container_file, ptr_action_file), # Sleep 'sleep 1', ] f.write('while [ -e %s ]; do\n %s\ndone &\n' % (ptr_temp_dir, '\n '.join(monitor_commands))) # Tell docker to constrain resources (memory). # Note: limiting memory is not always supported. See: # http://programster.blogspot.com/2014/09/docker-implementing-container-memory.html resource_args = '' if bundle.metadata.request_memory: resource_args += ' -m %s' % int( formatting.parse_size(bundle.metadata.request_memory)) # TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids f.write( "docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s >%s/stdout 2>%s/stderr & wait $!\n" % (resource_args, ptr_container_file, os.geteuid(), ptr_temp_dir, docker_temp_dir, ptr_internal_script_file, docker_internal_script_file, docker_image, docker_internal_script_file, ptr_temp_dir, ptr_temp_dir)) # 2) internal_script_file runs the actual command inside the docker container with open(internal_script_file, 'w') as f: # Make sure I have a username username = pwd.getpwuid(os.getuid())[ 0] # do this because os.getlogin() doesn't always work f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (username, os.geteuid(), os.getgid())) # Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...) f.write(". .bashrc || exit 1\n") # Go into the temp directory f.write("cd %s &&\n" % docker_temp_dir) # Run the actual command f.write('(%s) >>stdout 2>>stderr\n' % bundle.command) else: # Just run the command regularly without docker with open(script_file, 'w') as f: f.write(set_temp_dir_header) f.write("cd %s &&\n" % ptr_temp_dir) f.write('(%s) >stdout 2>stderr\n' % bundle.command) # Determine resources to request resource_args = [] if request_time: resource_args.extend( ['--request_time', formatting.parse_duration(request_time)]) if request_memory: resource_args.extend( ['--request_memory', formatting.parse_size(request_memory)]) if request_cpus: resource_args.extend(['--request_cpus', request_cpus]) if request_gpus: resource_args.extend(['--request_gpus', request_gpus]) if request_queue: resource_args.extend(['--request_queue', request_queue]) if request_priority: resource_args.extend(['--request_priority', request_priority]) if username: resource_args.extend(['--username', username]) # Start the command args = self.dispatch_command.split() + ['start'] + map( str, resource_args) + [script_file] if self.verbose >= 1: print '=== start_bundle(): running %s' % args result = json.loads(self.run_command_get_stdout(args)) if self.verbose >= 1: print '=== start_bundle(): got %s' % result # Return the information about the job. return { 'bundle': bundle, 'temp_dir': temp_dir, 'job_handle': result['handle'], 'docker_image': docker_image, }
def start_bundle(self, bundle, bundle_store, parent_dict, username): ''' Sets up all the temporary files and then dispatches the job. username: the username of the owner of the bundle Returns the bundle information. ''' # Create a temporary directory temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid) temp_dir = os.path.realpath(temp_dir) # Follow symlinks path_util.make_directory(temp_dir) # Copy all the dependencies to that temporary directory. pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir) print >>sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (bundle.uuid, temp_dir) for (source, target) in pairs: path_util.copy(source, target, follow_symlinks=False) # Set defaults for the dispatcher. docker_image = self.default_docker_image if bundle.metadata.request_docker_image: docker_image = bundle.metadata.request_docker_image request_time = self.default_request_time if bundle.metadata.request_time: request_time = bundle.metadata.request_time request_memory = self.default_request_memory if bundle.metadata.request_memory: request_memory = bundle.metadata.request_memory request_cpus = self.default_request_cpus if bundle.metadata.request_cpus: request_cpus = bundle.metadata.request_cpus request_gpus = self.default_request_gpus if bundle.metadata.request_gpus: request_gpus = bundle.metadata.request_gpus request_queue = self.default_request_queue if bundle.metadata.request_queue: request_queue = bundle.metadata.request_queue request_priority = self.default_request_priority if bundle.metadata.request_priority: request_priority = bundle.metadata.request_priority script_file = temp_dir + '.sh' # main entry point ptr_temp_dir = '$temp_dir' # 1) If no argument to script_file, use the temp_dir (e.g., Torque, master/worker share file system). # 2) If argument is 'use_script_for_temp_dir', use the script to determine temp_dir (e.g., qsub, no master/worker do not share file system). set_temp_dir_header = 'if [ -z "$1" ]; then temp_dir=' + temp_dir + '; else temp_dir=`readlink -f $0 | sed -e \'s/\\.sh$//\'`; fi\n' # Write the command to be executed to a script. if docker_image: internal_script_file = temp_dir + '-internal.sh' # run inside the docker container # These paths depend on $temp_dir, an environment variable which will be set (referenced inside script_file) ptr_container_file = ptr_temp_dir + '.cid' # contains the docker container id ptr_action_file = ptr_temp_dir + '.action' # send actions to the container (e.g., kill) ptr_status_dir = ptr_temp_dir + '.status' # receive information from the container (e.g., memory) ptr_script_file = ptr_temp_dir + '.sh' # main entry point ptr_internal_script_file = ptr_temp_dir + '-internal.sh' # run inside the docker container # Names of file inside the docker container docker_temp_dir = bundle.uuid docker_internal_script_file = bundle.uuid + '-internal.sh' # 1) script_file starts the docker container and runs internal_script_file in docker. # --rm removes the docker container once the job terminates (note that this makes things slow) # -v mounts the internal and user scripts and the temp directory # Trap SIGTERM and forward it to docker. with open(script_file, 'w') as f: f.write(set_temp_dir_header) # Monitor CPU/memory/disk def copy_if_exists(source_template, arg, target): source = source_template % arg # -f because target might be read-only return 'if [ -e %s ] && [ -e %s ]; then cp -f %s %s; fi' % (arg, source, source, target) monitor_commands = [ # Report on status (memory, cpu, etc.) 'mkdir -p %s' % ptr_status_dir, 'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is copy_if_exists('$cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat', ptr_container_file, ptr_status_dir), copy_if_exists('$cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes', ptr_container_file, ptr_status_dir), copy_if_exists('$cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes', ptr_container_file, ptr_status_dir), # Respond to kill action '[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s' % (ptr_action_file, ptr_action_file, ptr_container_file, ptr_action_file), # Sleep 'sleep 1', ] f.write('while [ -e %s ]; do\n %s\ndone &\n' % (ptr_temp_dir, '\n '. join(monitor_commands))) # Tell docker to constrain resources (memory). # Note: limiting memory is not always supported. See: # http://programster.blogspot.com/2014/09/docker-implementing-container-memory.html resource_args = '' if bundle.metadata.request_memory: resource_args += ' -m %s' % int(formatting.parse_size(bundle.metadata.request_memory)) # TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids f.write("docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s & wait $!\n" % ( resource_args, ptr_container_file, os.geteuid(), ptr_temp_dir, docker_temp_dir, ptr_internal_script_file, docker_internal_script_file, docker_image, docker_internal_script_file)) # 2) internal_script_file runs the actual command inside the docker container with open(internal_script_file, 'w') as f: # Make sure I have a username username = pwd.getpwuid(os.getuid())[0] # do this because os.getlogin() doesn't always work f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (username, os.geteuid(), os.getgid())) # Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...) f.write(". .bashrc || exit 1\n") # Go into the temp directory f.write("cd %s &&\n" % docker_temp_dir) # Run the actual command f.write('(%s) > stdout 2>stderr\n' % bundle.command) else: # Just run the command regularly without docker with open(script_file, 'w') as f: f.write(set_temp_dir_header) f.write("cd %s &&\n" % ptr_temp_dir) f.write('(%s) > stdout 2>stderr\n' % bundle.command) # Determine resources to request resource_args = [] if request_time: resource_args.extend(['--request_time', formatting.parse_duration(request_time)]) if request_memory: resource_args.extend(['--request_memory', formatting.parse_size(request_memory)]) if request_cpus: resource_args.extend(['--request_cpus', request_cpus]) if request_gpus: resource_args.extend(['--request_gpus', request_gpus]) if request_queue: resource_args.extend(['--request_queue', request_queue]) if request_priority: resource_args.extend(['--request_priority', request_priority]) if username: resource_args.extend(['--username', username]) # Start the command args = self.dispatch_command.split() + ['start'] + map(str, resource_args) + [script_file] if self.verbose >= 1: print '=== start_bundle(): running %s' % args result = json.loads(self.run_command_get_stdout(args)) if self.verbose >= 1: print '=== start_bundle(): got %s' % result # Return the information about the job. return { 'bundle': bundle, 'temp_dir': temp_dir, 'job_handle': result['handle'], 'docker_image': docker_image, }