def test_make_directory(self, mock_os): ''' Check that make_directory calls normalize and then creates the directory. ''' mock_os.path.join = os.path.join path_util.make_directory(self.test_path) self.assertEqual(mock_os.mkdir.call_args_list, self.mkdir_calls)
def start_bundle(self, bundle, bundle_store, parent_dict, username): """ Start a bundle in the background. """ if self.bundle != None: return None temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid) path_util.make_directory(temp_dir) # We don't follow symlinks (for consistency with remote # machine, where it is more secure, so people can't make us # copy random files on the system). Of course in local mode, # if some of those symlinks are absolute, the run can # read/write those locations. But we're not sandboxed, so # anything could happen. The dependencies are copied, so in # practice, this is not a bit worry. pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir) print >> sys.stderr, "LocalMachine.start_bundle: copying dependencies of %s to %s" % (bundle.uuid, temp_dir) for (source, target) in pairs: path_util.copy(source, target, follow_symlinks=False) script_file = temp_dir + ".sh" with open(script_file, "w") as f: f.write("cd %s &&\n" % temp_dir) f.write("(%s) > stdout 2>stderr\n" % bundle.command) # Use stdbuf (if it exists) to turn off buffering so we get real-time feedback. if os.path.exists("/usr/bin/stdbuf"): process = subprocess.Popen("/usr/bin/stdbuf -o0 bash " + script_file, shell=True) else: process = subprocess.Popen("bash " + script_file, shell=True) self.bundle = bundle self.temp_dir = temp_dir self.process = process return {"bundle": bundle, "temp_dir": temp_dir, "job_handle": str(process.pid)}
def codalab_home(self): from codalab.lib import path_util # Default to this directory in the user's home directory. # In the future, allow customization based on. result = path_util.normalize("~/.codalab") path_util.make_directory(result) return result
def start_bundle(self, bundle): """ Run the given bundle using an available Machine. Return whether something was started. """ # Check that we're running a bundle in the QUEUED state. state_message = "Unexpected bundle state: %s" % (bundle.state,) precondition(bundle.state == State.QUEUED, state_message) data_hash_message = "Unexpected bundle data_hash: %s" % (bundle.data_hash,) precondition(bundle.data_hash is None, data_hash_message) # Run the bundle. with self.profile("Running bundle..."): started = False if isinstance(bundle, RunBundle): try: # Get the username of the bundle results = self.auth_handler.get_users("ids", [bundle.owner_id]) if results.get(bundle.owner_id): username = results[bundle.owner_id].name else: username = str(bundle.owner_id) status = self.machine.start_bundle( bundle, self.bundle_store, self.get_parent_dict(bundle), username ) if status != None: status["started"] = int(time.time()) started = True except Exception as e: # If there's an exception, we just make the bundle fail # (even if it's not the bundle's fault). real_path = canonicalize.get_current_location(self.bundle_store, bundle.uuid) path_util.make_directory(real_path) status = { "bundle": bundle, "success": False, "failure_message": "Internal error: " + str(e), "temp_dir": real_path, } print "=== INTERNAL ERROR: %s" % e started = True # Force failing traceback.print_exc() else: # MakeBundle started = True if started: print "-- START BUNDLE: %s" % (bundle,) self._update_events_log("start_bundle", bundle, (bundle.uuid,)) # If we have a MakeBundle, then just process it immediately. if isinstance(bundle, MakeBundle): real_path = canonicalize.get_current_location(self.bundle_store, bundle.uuid) path_util.make_directory(real_path) status = {"bundle": bundle, "success": True, "temp_dir": real_path} # Update database if started: self.update_running_bundle(status) return started
def add_partition(self, target, new_partition_name): """ MultiDiskBundleStore specific method. Add a new partition to the bundle store, which is actually a symlink to the target directory, which the user has configured as the mountpoint for some desired partition. If `target` is None, then make the `new_partition_name` the actual directory. """ if target is not None: target = os.path.abspath(target) new_partition_location = os.path.join(self.partitions, new_partition_name) print("Adding new partition as %s..." % new_partition_location, file=sys.stderr) if target is None: path_util.make_directory(new_partition_location) else: path_util.soft_link(target, new_partition_location) # Where the bundles are stored mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY) try: path_util.make_directory(mdata) except Exception as e: print(e, file=sys.stderr) print( "Could not make directory %s on partition %s, aborting" % (mdata, target), file=sys.stderr, ) sys.exit(1) self.refresh_partitions() print( "Successfully added partition '%s' to the pool." % new_partition_name, file=sys.stderr )
def add_partition(self, target, new_partition_name): """ MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to the target directory, which the user has configured as the mountpoint for some desired partition. """ target = os.path.abspath(target) new_partition_location = os.path.join(self.partitions, new_partition_name) print >>sys.stderr, "Adding new partition as %s..." % new_partition_location path_util.soft_link(target, new_partition_location) mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY) try: path_util.make_directory(mdata) except Exception as e: print >>sys.stderr, e print >>sys.stderr, "Could not make directory %s on partition %s, aborting" % ( mdata, target, ) sys.exit(1) self.nodes.append(new_partition_name) print >>sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name
def add_partition(self, target, new_partition_name): """ MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to the target directory, which the user has configured as the mountpoint for some desired partition. """ target = os.path.abspath(target) new_partition_location = os.path.join(self.partitions, new_partition_name) print >> sys.stderr, "Adding new partition as %s..." % new_partition_location path_util.soft_link(target, new_partition_location) mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY) try: path_util.make_directory(mdata) except Exception as e: print >> sys.stderr, e print >> sys.stderr, "Could not make directory %s on partition %s, aborting" % ( mdata, target, ) sys.exit(1) self.nodes.append(new_partition_name) print >> sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name
def codalab_home(self): from codalab.lib import path_util # Default to this directory in the user's home directory. # In the future, allow customization based on. home = os.getenv('CODALAB_HOME', '~/.codalab') home = path_util.normalize(home) path_util.make_directory(home) return home
def setUpClass(cls): cls.test_root = path_util.normalize("~/.codalab_tests") path_util.make_directory(cls.test_root) cls.bundle_store = BundleStore(cls.test_root) cls.model = SQLiteModel(cls.test_root) users = [User('root', 0), User('user1', 1), User('user2', 2), User('user4', 4)] cls.auth_handler = MockAuthHandler(users) cls.client = LocalBundleClient('local', cls.bundle_store, cls.model, cls.auth_handler)
def setUpClass(cls): cls.test_root = path_util.normalize("~/.codalab_tests") path_util.make_directory(cls.test_root) cls.bundle_store = BundleStore(cls.test_root) cls.model = SQLiteModel("sqlite:///{}".format(os.path.join(cls.test_root, 'bundle.db')), {}) cls.model.root_user_id = '0' users = [User('root', '0'), User('user1', '1'), User('user2', '2'), User('user4', '4')] cls.auth_handler = MockAuthHandler(users) cls.client = LocalBundleClient('local', cls.bundle_store, cls.model, cls.auth_handler, verbose=1)
def __init__(self, bundle_model, codalab_home): BundleStore.__init__(self, bundle_model, codalab_home) self.partitions = os.path.join(self.codalab_home, 'partitions') path_util.make_directory(self.partitions) self.refresh_partitions() if self.__get_num_partitions() == 0: # Ensure at least one partition exists. self.add_partition(None, 'default') self.lru_cache = OrderedDict()
def add_partition(self, target, new_partition_name): """ MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to the target directory, which the user has configured as the mountpoint for some desired partition. First, all bundles that are to be relocated onto the new partition are copied to a temp location to be resilient against failures. After the copy is performed, the bundles are subsequently moved to the new partition, and finally the original copy of the bundles are deleted from their old locations """ target = os.path.abspath(target) new_partition_location = os.path.join(self.partitions, new_partition_name) mtemp = os.path.join(target, MultiDiskBundleStore.TEMP_SUBDIRECTORY) try: path_util.make_directory(mtemp) except: print >> sys.stderr, "Could not make directory %s on partition %s, aborting" % (mtemp, target) sys.exit(1) self.ring.add_node(new_partition_name) # Add the node to the partition locations delete_on_success = [] # Paths to bundles that will be deleted after the copy finishes successfully print >> sys.stderr, "Marking bundles for placement on new partition %s (might take a while)" % new_partition_name # For each bundle in the bundle store, check to see if any hash to the new partition. If so move them over partitions, _ = path_util.ls(self.partitions) for partition in partitions: partition_abs_path = os.path.join(self.partitions, partition, MultiDiskBundleStore.DATA_SUBDIRECTORY) bundles = reduce(lambda dirs, files: dirs + files, path_util.ls(partition_abs_path)) for bundle in bundles: correct_partition = self.ring.get_node(bundle) if correct_partition != partition: # Reposition the node to the correct partition from_path = os.path.join(self.partitions, partition, MultiDiskBundleStore.DATA_SUBDIRECTORY, bundle) to_path = os.path.join(mtemp, bundle) print >> sys.stderr, "copying %s to %s" % (from_path, to_path) path_util.copy(from_path, to_path) delete_on_success += [from_path] print >> sys.stderr, "Adding new partition as %s..." % new_partition_location path_util.soft_link(target, new_partition_location) # Atomically move the temp location to the new partition's mdata new_mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY) new_mtemp = os.path.join(new_partition_location, MultiDiskBundleStore.TEMP_SUBDIRECTORY) path_util.rename(new_mtemp, new_mdata) path_util.make_directory(new_mtemp) # Go through and purge all of the originals at this time print >> sys.stderr, "Cleaning up drives..." for to_delete in delete_on_success: path_util.remove(to_delete) print >> sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name
def codalab_home(self): from codalab.lib import path_util # Default to this directory in the user's home directory. # In the future, allow customization based on. home = os.getenv('CODALAB_HOME', '~/.codalab') home = path_util.normalize(home) path_util.make_directory(home) # Global setting! Make temp directory the same as the bundle store # temporary directory. The default /tmp generally doesn't have enough # space. tempfile.tempdir = os.path.join(home, BundleStore.TEMP_SUBDIRECTORY) return home
def start_bundle(self, bundle): ''' Run the given bundle using an available Machine. Return whether something was started. ''' # Check that we're running a bundle in the QUEUED state. state_message = 'Unexpected bundle state: %s' % (bundle.state,) precondition(bundle.state == State.QUEUED, state_message) data_hash_message = 'Unexpected bundle data_hash: %s' % (bundle.data_hash,) precondition(bundle.data_hash is None, data_hash_message) # Run the bundle. with self.profile('Running bundle...'): started = False if isinstance(bundle, RunBundle): try: # Get the username of the bundle results = self.auth_handler.get_users('ids', [bundle.owner_id]) if results.get(bundle.owner_id): username = results[bundle.owner_id].name else: username = str(bundle.owner_id) status = self.machine.start_bundle(bundle, self.bundle_store, self.get_parent_dict(bundle), username) if status != None: status['started'] = int(time.time()) started = True except Exception as e: # If there's an exception, we just make the bundle fail # (even if it's not the bundle's fault). real_path = canonicalize.get_current_location(self.bundle_store, bundle.uuid) path_util.make_directory(real_path) status = {'bundle': bundle, 'success': False, 'failure_message': 'Internal error: ' + str(e), 'temp_dir': real_path} print '=== INTERNAL ERROR: %s' % e started = True # Force failing traceback.print_exc() else: # MakeBundle started = True if started: print '-- START BUNDLE: %s' % (bundle,) self._update_events_log('start_bundle', bundle, (bundle.uuid,)) # If we have a MakeBundle, then just process it immediately. if isinstance(bundle, MakeBundle): real_path = canonicalize.get_current_location(self.bundle_store, bundle.uuid) path_util.make_directory(real_path) status = {'bundle': bundle, 'success': True, 'temp_dir': real_path} # Update database if started: self.update_running_bundle(status) return started
def setUpClass(cls): cls.test_root = path_util.normalize("~/.codalab_tests") path_util.make_directory(cls.test_root) cls.bundle_store = MultiDiskBundleStore(cls.test_root) cls.model = SQLiteModel("sqlite:///{}".format(os.path.join(cls.test_root, 'bundle.db')), {'time_quota': 1e12, 'disk_quota': 1e12}) cls.model.root_user_id = '0' users = [User('root', '0'), User('user1', '1'), User('user2', '2'), User('user4', '4')] cls.auth_handler = MockAuthHandler(users) for user in users: cls.model.add_user(user.name, user.name + '@codalab.org', '', user_id=user.unique_id, is_verified=True) cls.client = LocalBundleClient('local', cls.bundle_store, cls.model, None, None, None, cls.auth_handler, verbose=1)
def __init__(self, codalab_manager, torque_config): assert(codalab_manager.worker_model().shared_file_system) self._torque_ssh_host = torque_config['ssh_host'] self._torque_bundle_service_url = torque_config['bundle_service_url'] self._torque_password_file = torque_config['password_file'] self._torque_log_dir = torque_config['log_dir'] path_util.make_directory(self._torque_log_dir) if 'worker_code_dir' in torque_config: self._torque_worker_code_dir = torque_config['worker_code_dir'] else: codalab_cli = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) self._torque_worker_code_dir = os.path.join(codalab_cli, 'worker') self._last_delete_attempt = {}
def start_bundle(self, bundle): ''' Run the given bundle using an available Machine. Return whether something was started. ''' # Check that we're running a bundle in the RUNNING state. state_message = 'Unexpected bundle state: %s' % (bundle.state,) precondition(bundle.state == State.RUNNING, state_message) data_hash_message = 'Unexpected bundle data_hash: %s' % (bundle.data_hash,) precondition(bundle.data_hash is None, data_hash_message) # Run the bundle. with self.profile('Running bundle...'): started = False if isinstance(bundle, RunBundle): try: # Get the username of the bundle results = self.auth_handler.get_users('ids', [bundle.owner_id]) if results.get(bundle.owner_id): username = results[bundle.owner_id].name else: username = str(bundle.owner_id) status = self.machine.start_bundle(bundle, self.bundle_store, self.get_parent_dict(bundle), username) if status != None: started = True except Exception as e: # If there's an exception, we just make the bundle fail # (even if it's not the bundle's fault). temp_dir = canonicalize.get_current_location(self.bundle_store, bundle.uuid) path_util.make_directory(temp_dir) status = {'bundle': bundle, 'success': False, 'failure_message': str(e), 'temp_dir': temp_dir} print '=== INTERNAL ERROR: %s' % e started = True # Force failing traceback.print_exc() else: # MakeBundle started = True if started: print '-- START BUNDLE: %s' % (bundle,) # If we have a MakeBundle, then just process it immediately. if isinstance(bundle, MakeBundle): temp_dir = canonicalize.get_current_location(self.bundle_store, bundle.uuid) path_util.make_directory(temp_dir) status = {'bundle': bundle, 'success': True, 'temp_dir': temp_dir} # Update database if started: self.update_running_bundle(status) return started
def test_make_directory_if_exists(self, mock_os): ''' Check that make_directory still works if the directory exists. ''' mock_os.path.join = os.path.join failures = [0] def mkdir_when_directory_exists(path): failures[0] += 1 error = OSError() error.errno = errno.EEXIST raise error mock_os.mkdir.side_effect = mkdir_when_directory_exists path_util.make_directory(self.test_path) self.assertEqual(mock_os.mkdir.call_args_list, self.mkdir_calls) self.assertEqual(failures[0], 1)
def __init__(self, codalab_manager, torque_config): assert(codalab_manager.worker_model().shared_file_system) self._torque_ssh_host = torque_config['ssh_host'] self._torque_bundle_service_url = torque_config['bundle_service_url'] self._torque_password_file = torque_config['password_file'] self._torque_log_dir = torque_config['log_dir'] self._torque_min_seconds_between_qsub = torque_config.get('min_seconds_between_qsub', 0) path_util.make_directory(self._torque_log_dir) if 'worker_code_dir' in torque_config: self._torque_worker_code_dir = torque_config['worker_code_dir'] else: codalab_cli = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) self._torque_worker_code_dir = os.path.join(codalab_cli, 'worker') self._last_delete_attempt = {} self._last_qsub_time = 0
def setUpClass(cls): cls.test_root = path_util.normalize("~/.codalab_tests") path_util.make_directory(cls.test_root) cls.bundle_store = BundleStore(cls.test_root, []) cls.model = SQLiteModel(cls.test_root) cls.model.root_user_id = '0' users = [ User('root', '0'), User('user1', '1'), User('user2', '2'), User('user4', '4') ] cls.auth_handler = MockAuthHandler(users) cls.client = LocalBundleClient('local', cls.bundle_store, cls.model, cls.auth_handler, verbose=1)
def start_bundle(self, bundle, bundle_store, parent_dict, username): ''' Start a bundle in the background. ''' if self.bundle != None: return None temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid) path_util.make_directory(temp_dir) # We don't follow symlinks (for consistency with remote # machine, where it is more secure, so people can't make us # copy random files on the system). Of course in local mode, # if some of those symlinks are absolute, the run can # read/write those locations. But we're not sandboxed, so # anything could happen. The dependencies are copied, so in # practice, this is not a bit worry. pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir) print >> sys.stderr, 'LocalMachine.start_bundle: copying dependencies of %s to %s' % ( bundle.uuid, temp_dir) for (source, target) in pairs: path_util.copy(source, target, follow_symlinks=False) script_file = temp_dir + '.sh' with open(script_file, 'w') as f: f.write("cd %s &&\n" % temp_dir) f.write('(%s) > stdout 2>stderr\n' % bundle.command) # Use stdbuf (if it exists) to turn off buffering so we get real-time feedback. if os.path.exists('/usr/bin/stdbuf'): process = subprocess.Popen("/usr/bin/stdbuf -o0 bash " + script_file, shell=True) else: process = subprocess.Popen("bash " + script_file, shell=True) self.bundle = bundle self.temp_dir = temp_dir self.process = process return { 'bundle': bundle, 'temp_dir': temp_dir, 'job_handle': str(process.pid) }
def test_make_directory_with_failures(self, mock_os): ''' Check that make_directory still works if the directory exists. ''' mock_os.path.join = os.path.join def mkdir_with_other_failure(path): raise OSError() mock_os.mkdir.reset_mock() mock_os.mkdir.side_effect = mkdir_with_other_failure self.assertRaises(OSError, lambda: path_util.make_directory(self.test_path)) self.assertEqual(mock_os.mkdir.call_args_list, self.mkdir_calls)
def setUpClass(cls): cls.test_root = path_util.normalize("~/.codalab_tests") path_util.make_directory(cls.test_root) cls.bundle_store = MultiDiskBundleStore(cls.test_root) cls.model = SQLiteModel( "sqlite:///{}".format(os.path.join(cls.test_root, 'bundle.db')), {}) cls.model.root_user_id = '0' users = [ User('root', '0'), User('user1', '1'), User('user2', '2'), User('user4', '4') ] cls.auth_handler = MockAuthHandler(users) cls.client = LocalBundleClient('local', cls.bundle_store, cls.model, None, cls.auth_handler, verbose=1)
def initialize_store(self): """ Initializes the multi-disk bundle store. """ path_util.make_directory(self.partitions) path_util.make_directory(self.mtemp) # Create the default partition, if there are no partitions currently if self.__get_num_partitions() == 0: # Create a default partition that links to the codalab_home path_util.make_directory(os.path.join(self.codalab_home, MultiDiskBundleStore.DATA_SUBDIRECTORY)) path_util.make_directory(os.path.join(self.codalab_home, MultiDiskBundleStore.TEMP_SUBDIRECTORY)) default_partition = os.path.join(self.partitions, 'default') path_util.soft_link(self.codalab_home, default_partition)
def initialize_store(self): """ Initializes the multi-disk bundle store. """ path_util.make_directory(self.partitions) path_util.make_directory(self.mtemp) # Create the default partition, if there are no partitions currently if self.__get_num_partitions() == 0: # Create a default partition that links to the codalab_home path_util.make_directory( os.path.join(self.codalab_home, MultiDiskBundleStore.DATA_SUBDIRECTORY)) path_util.make_directory( os.path.join(self.codalab_home, MultiDiskBundleStore.TEMP_SUBDIRECTORY)) default_partition = os.path.join(self.partitions, 'default') path_util.soft_link(self.codalab_home, default_partition)
def make_directories(self): ''' Create the data, and temp directories for this BundleStore. ''' for path in (self.data, self.temp): path_util.make_directory(path)
def upload_to_bundle_store(self, bundle, sources, follow_symlinks, exclude_patterns, remove_sources, git, unpack, simplify_archives): """ Uploads contents for the given bundle to the bundle store. |sources|: specifies the locations of the contents to upload. Each element is either a URL, a local path or a tuple (filename, file-like object). |follow_symlinks|: for local path(s), whether to follow (resolve) symlinks, but only if remove_sources is False. |exclude_patterns|: for local path(s), don't upload these patterns (e.g., *.o), but only if remove_sources is False. |remove_sources|: for local path(s), whether |sources| should be removed |git|: for URLs, whether |source| is a git repo to clone. |unpack|: for each source in |sources|, whether to unpack it if it's an archive. |simplify_archives|: whether to simplify unpacked archives so that if they contain a single file, the final path is just that file, not a directory containing that file. If |sources| contains one source, then the bundle contents will be that source. Otherwise, the bundle contents will be a directory with each of the sources. Exceptions: - If |git|, then each source is replaced with the result of running 'git clone |source|' - If |unpack| is True or a source is an archive (zip, tar.gz, etc.), then unpack the source. """ bundle_path = self._bundle_store.get_bundle_location(bundle.uuid) try: path_util.make_directory(bundle_path) # Note that for uploads with a single source, the directory # structure is simplified at the end. for source in sources: is_url, is_local_path, is_fileobj, filename = self._interpret_source(source) source_output_path = os.path.join(bundle_path, filename) if is_url: if git: source_output_path = file_util.strip_git_ext(source_output_path) file_util.git_clone(source, source_output_path) else: file_util.download_url(source, source_output_path) if unpack and self._can_unpack_file(source_output_path): self._unpack_file( source_output_path, zip_util.strip_archive_ext(source_output_path), remove_source=True, simplify_archive=simplify_archives) elif is_local_path: source_path = path_util.normalize(source) path_util.check_isvalid(source_path, 'upload') if unpack and self._can_unpack_file(source_path): self._unpack_file( source_path, zip_util.strip_archive_ext(source_output_path), remove_source=remove_sources, simplify_archive=simplify_archives) elif remove_sources: path_util.rename(source_path, source_output_path) else: path_util.copy(source_path, source_output_path, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns) elif is_fileobj: if unpack and zip_util.path_is_archive(filename): self._unpack_fileobj( source[0], source[1], zip_util.strip_archive_ext(source_output_path), simplify_archive=simplify_archives) else: with open(source_output_path, 'wb') as out: shutil.copyfileobj(source[1], out) if len(sources) == 1: self._simplify_directory(bundle_path) except: if os.path.exists(bundle_path): path_util.remove(bundle_path) raise
from codalab.lib.codalab_manager import CodaLabManager from codalab.lib import path_util dry_run = False if len(sys.argv) > 1 and sys.argv[1] == '-f' else True manager = CodaLabManager() model = manager.model() CODALAB_HOME = manager.codalab_home """Move data/ directory over to a temp area, and create a staging tree for uuid-based storage""" DATA_DIR = os.path.join(CODALAB_HOME, 'data') FINAL_LOCATION = os.path.join(CODALAB_HOME, 'bundles') if not dry_run: path_util.make_directory(FINAL_LOCATION) """For each data hash, get a list of all bundles that have that hash, and make a copy of the bundle in the staging area under the UUID for the bundle.""" data_hashes = reduce(lambda x,y: x+y, path_util.ls(DATA_DIR)) for data_hash in data_hashes: orig_location = os.path.join(DATA_DIR, data_hash) bundles_with_hash = model.batch_get_bundles(data_hash=data_hash) # We'd prefer renaming bundles to making copies, but because we are converting from deduplicated storage # we need to make sure that we only perform renames if we map 1:1 UUID->Hash. rename_allowed = len(bundles_with_hash) <= 1 for bundle in bundles_with_hash: # Build the command to be executed in a subshell uuid = bundle.uuid copy_location = os.path.join(FINAL_LOCATION, uuid)
def upload_to_bundle_store( self, bundle, sources, follow_symlinks, exclude_patterns, remove_sources, git, unpack, simplify_archives, ): """ Uploads contents for the given bundle to the bundle store. |sources|: specifies the locations of the contents to upload. Each element is either a URL, a local path or a tuple (filename, binary file-like object). |follow_symlinks|: for local path(s), whether to follow (resolve) symlinks, but only if remove_sources is False. |exclude_patterns|: for local path(s), don't upload these patterns (e.g., *.o), but only if remove_sources is False. |remove_sources|: for local path(s), whether |sources| should be removed |git|: for URLs, whether |source| is a git repo to clone. |unpack|: for each source in |sources|, whether to unpack it if it's an archive. |simplify_archives|: whether to simplify unpacked archives so that if they contain a single file, the final path is just that file, not a directory containing that file. If |sources| contains one source, then the bundle contents will be that source. Otherwise, the bundle contents will be a directory with each of the sources. Exceptions: - If |git|, then each source is replaced with the result of running 'git clone |source|' - If |unpack| is True or a source is an archive (zip, tar.gz, etc.), then unpack the source. """ exclude_patterns = (self._default_exclude_patterns + exclude_patterns if exclude_patterns else self._default_exclude_patterns) bundle_link_url = getattr(bundle.metadata, "link_url", None) if bundle_link_url: # Don't do anything for linked bundles. return bundle_path = self._bundle_store.get_bundle_location(bundle.uuid) try: path_util.make_directory(bundle_path) # Note that for uploads with a single source, the directory # structure is simplified at the end. for source in sources: is_url, is_local_path, is_fileobj, filename = self._interpret_source( source) source_output_path = os.path.join(bundle_path, filename) if is_url: if git: source_output_path = file_util.strip_git_ext( source_output_path) file_util.git_clone(source, source_output_path) else: file_util.download_url(source, source_output_path) if unpack and self._can_unpack_file( source_output_path): self._unpack_file( source_output_path, zip_util.strip_archive_ext(source_output_path), remove_source=True, simplify_archive=simplify_archives, ) elif is_local_path: source_path = path_util.normalize(source) path_util.check_isvalid(source_path, 'upload') if unpack and self._can_unpack_file(source_path): self._unpack_file( source_path, zip_util.strip_archive_ext(source_output_path), remove_source=remove_sources, simplify_archive=simplify_archives, ) elif remove_sources: path_util.rename(source_path, source_output_path) else: path_util.copy( source_path, source_output_path, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns, ) elif is_fileobj: if unpack and zip_util.path_is_archive(filename): self._unpack_fileobj( source[0], source[1], zip_util.strip_archive_ext(source_output_path), simplify_archive=simplify_archives, ) else: with open(source_output_path, 'wb') as out: shutil.copyfileobj(source[1], out) if len(sources) == 1: self._simplify_directory(bundle_path) except: if os.path.exists(bundle_path): path_util.remove(bundle_path) raise
def add_partition(self, target, new_partition_name): """ MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to the target directory, which the user has configured as the mountpoint for some desired partition. First, all bundles that are to be relocated onto the new partition are copied to a temp location to be resilient against failures. After the copy is performed, the bundles are subsequently moved to the new partition, and finally the original copy of the bundles are deleted from their old locations """ target = os.path.abspath(target) new_partition_location = os.path.join(self.partitions, new_partition_name) mtemp = os.path.join(target, MultiDiskBundleStore.TEMP_SUBDIRECTORY) try: path_util.make_directory(mtemp) except: print >> sys.stderr, "Could not make directory %s on partition %s, aborting" % ( mtemp, target) sys.exit(1) self.ring.add_node( new_partition_name) # Add the node to the partition locations delete_on_success = [ ] # Paths to bundles that will be deleted after the copy finishes successfully print >> sys.stderr, "Marking bundles for placement on new partition %s (might take a while)" % new_partition_name # For each bundle in the bundle store, check to see if any hash to the new partition. If so move them over partitions, _ = path_util.ls(self.partitions) for partition in partitions: partition_abs_path = os.path.join( self.partitions, partition, MultiDiskBundleStore.DATA_SUBDIRECTORY) bundles = reduce(lambda dirs, files: dirs + files, path_util.ls(partition_abs_path)) for bundle in bundles: correct_partition = self.ring.get_node(bundle) if correct_partition != partition: # Reposition the node to the correct partition from_path = os.path.join( self.partitions, partition, MultiDiskBundleStore.DATA_SUBDIRECTORY, bundle) to_path = os.path.join(mtemp, bundle) print >> sys.stderr, "copying %s to %s" % (from_path, to_path) path_util.copy(from_path, to_path) delete_on_success += [from_path] print >> sys.stderr, "Adding new partition as %s..." % new_partition_location path_util.soft_link(target, new_partition_location) # Atomically move the temp location to the new partition's mdata new_mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY) new_mtemp = os.path.join(new_partition_location, MultiDiskBundleStore.TEMP_SUBDIRECTORY) path_util.rename(new_mtemp, new_mdata) path_util.make_directory(new_mtemp) # Go through and purge all of the originals at this time print >> sys.stderr, "Cleaning up drives..." for to_delete in delete_on_success: path_util.remove(to_delete) print >> sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name
def worker_socket_dir(self): from codalab.lib import path_util directory = os.path.join(self.codalab_home, 'worker_sockets') path_util.make_directory(directory) return directory
def make_temp_location(self, identifier): ''' Creates directory with given name under TEMP_SUBDIRECTORY ''' path_util.make_directory(self.get_temp_location(identifier))
def start_bundle(self, bundle, bundle_store, parent_dict, username): ''' Sets up all the temporary files and then dispatches the job. username: the username of the owner of the bundle Returns the bundle information. ''' # Create a temporary directory temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid) temp_dir = os.path.realpath(temp_dir) # Follow symlinks path_util.make_directory(temp_dir) # Copy all the dependencies to that temporary directory. pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir) print >>sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (bundle.uuid, temp_dir) for (source, target) in pairs: path_util.copy(source, target, follow_symlinks=False) # Set docker image docker_image = self.default_docker_image if bundle.metadata.request_docker_image: docker_image = bundle.metadata.request_docker_image # Write the command to be executed to a script. if docker_image: container_file = temp_dir + '.cid' # contains the docker container id action_file = temp_dir + '.action' # send actions to the container (e.g., kill) status_dir = temp_dir + '.status' # receive information from the container (e.g., memory) script_file = temp_dir + '.sh' # main entry point internal_script_file = temp_dir + '-internal.sh' # run inside the docker container # Names of file inside the docker container docker_temp_dir = bundle.uuid docker_internal_script_file = bundle.uuid + '-internal.sh' # 1) script_file starts the docker container and runs internal_script_file in docker. # --rm removes the docker container once the job terminates (note that this makes things slow) # -v mounts the internal and user scripts and the temp directory # Trap SIGTERM and forward it to docker. with open(script_file, 'w') as f: # trap doesn't quite work reliably with Torque, so don't use it #f.write('trap \'echo Killing docker container $(cat %s); docker kill $(cat %s); echo Killed: $?; exit 143\' TERM\n' % (container_file, container_file)) # Inspect doesn't tell us a lot, so don't use it #f.write('while [ -e %s ]; do docker inspect $(cat %s) > %s; sleep 1; done &\n' % (temp_dir, container_file, status_dir)) # Monitor CPU/memory/disk monitor_commands = [ # Report on status 'mkdir -p %s' % status_dir, 'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is 'cp -f $cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat %s' % (container_file, status_dir), 'cp -f $cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes %s' % (container_file, status_dir), 'cp -f $cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes %s' % (container_file, status_dir), # Respond to actions '[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s' % (action_file, action_file, container_file, action_file), ] f.write('while [ -e %s ]; do %s; sleep 1; done &\n' % (temp_dir, '; '. join(monitor_commands))) # Constrain resources resource_args = '' if bundle.metadata.request_memory: resource_args += ' -m %s' % int(formatting.parse_size(bundle.metadata.request_memory)) # TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids f.write("docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s & wait $!\n" % ( resource_args, container_file, os.geteuid(), temp_dir, docker_temp_dir, internal_script_file, docker_internal_script_file, docker_image, docker_internal_script_file)) # 2) internal_script_file runs the actual command inside the docker container with open(internal_script_file, 'w') as f: # Make sure I have a username f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (os.getlogin(), os.geteuid(), os.getgid())) # Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...) f.write(". .bashrc || exit 1\n") # Go into the temp directory f.write("cd %s &&\n" % docker_temp_dir) # Run the actual command f.write('(%s) > stdout 2>stderr\n' % bundle.command) else: # Just run the command regularly without docker script_file = temp_dir + '.sh' with open(script_file, 'w') as f: f.write("cd %s &&\n" % temp_dir) f.write('(%s) > stdout 2>stderr\n' % bundle.command) # Determine resources to request resource_args = [] if bundle.metadata.request_time: resource_args.extend(['--request_time', formatting.parse_duration(bundle.metadata.request_time)]) if bundle.metadata.request_memory: resource_args.extend(['--request_memory', formatting.parse_size(bundle.metadata.request_memory)]) if bundle.metadata.request_cpus: resource_args.extend(['--request_cpus', bundle.metadata.request_cpus]) if bundle.metadata.request_gpus: resource_args.extend(['--request_gpus', bundle.metadata.request_gpus]) if bundle.metadata.request_queue: resource_args.extend(['--request_queue', bundle.metadata.request_queue]) if username: resource_args.extend(['--username', username]) # Start the command args = self.dispatch_command.split() + ['start'] + map(str, resource_args) + [script_file] if self.verbose >= 1: print '=== start_bundle(): running %s' % args result = json.loads(self.run_command_get_stdout(args)) if self.verbose >= 1: print '=== start_bundle(): got %s' % result # Return the information about the job. return { 'bundle': bundle, 'temp_dir': temp_dir, 'job_handle': result['handle'], 'docker_image': docker_image, }
def upload(self, sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, uuid): """ |sources|: specifies the locations of the contents to upload. Each element is either a URL or a local path. |follow_symlinks|: for local path(s), whether to follow (resolve) symlinks |exclude_patterns|: for local path(s), don't upload these patterns (e.g., *.o) |git|: for URL, whether |source| is a git repo to clone. |unpack|: for each source in |sources|, whether to unpack it if it's an archive. |remove_sources|: remove |sources|. If |sources| contains one source, then the bundle contents will be that source. Otherwise, the bundle contents will be a directory with each of the sources. Exceptions: - If |git|, then each source is replaced with the result of running 'git clone |source|' - If |unpack| is True or a source is an archive (zip, tar.gz, etc.), then unpack the source. Install the contents of the directory at |source| into DATA_SUBDIRECTORY in a subdirectory named by a hash of the contents. Return a (data_hash, metadata) pair, where the metadata is a dict mapping keys to precomputed statistics about the new data directory. """ to_delete = [] # If just a single file, set the final path to be equal to that file single_path = len(sources) == 1 # Determine which disk this will go on disk_choice = self.ring.get_node(uuid) final_path = os.path.join(self.partitions, disk_choice, self.DATA_SUBDIRECTORY, uuid) if os.path.exists(final_path): raise UsageError('Path %s already present in bundle store' % final_path) # Only make if not there elif not single_path: path_util.make_directory(final_path) # Paths to resources subpaths = [] for source in sources: # Where to save |source| to (might change this value if we unpack). if not single_path: subpath = os.path.join(final_path, os.path.basename(source)) else: subpath = final_path if remove_sources: to_delete.append(source) source_unpack = unpack and zip_util.path_is_archive(source) if source_unpack and single_path: # Load the file into the bundle store under the given path subpath += zip_util.get_archive_ext(source) if path_util.path_is_url(source): # Download the URL. print_util.open_line('BundleStore.upload: downloading %s to %s' % (source, subpath)) if git: file_util.git_clone(source, subpath) else: file_util.download_url(source, subpath, print_status=True) if source_unpack: zip_util.unpack(subpath, zip_util.strip_archive_ext(subpath)) path_util.remove(subpath) subpath = zip_util.strip_archive_ext(subpath) print_util.clear_line() else: # Copy the local path. source_path = path_util.normalize(source) path_util.check_isvalid(source_path, 'upload') # Recursively copy the directory into the BundleStore print_util.open_line('BundleStore.upload: %s => %s' % (source_path, subpath)) if source_unpack: zip_util.unpack(source_path, zip_util.strip_archive_ext(subpath)) subpath = zip_util.strip_archive_ext(subpath) else: if remove_sources: path_util.rename(source_path, subpath) else: path_util.copy(source_path, subpath, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns) print_util.clear_line() subpaths.append(subpath) dirs_and_files = None if os.path.isdir(final_path): dirs_and_files = path_util.recursive_ls(final_path) else: dirs_and_files = [], [final_path] # Hash the contents of the bundle directory. Update the data_hash attribute # for the bundle print_util.open_line('BundleStore.upload: hashing %s' % final_path) data_hash = '0x%s' % (path_util.hash_directory(final_path, dirs_and_files)) print_util.clear_line() print_util.open_line('BundleStore.upload: computing size of %s' % final_path) data_size = path_util.get_size(final_path, dirs_and_files) print_util.clear_line() # Delete paths. for path in to_delete: if os.path.exists(path): path_util.remove(path) # After this operation there should always be a directory at the final path. assert (os.path.lexists(final_path)), 'Uploaded to %s failed!' % (final_path,) return (data_hash, {'data_size': data_size})
from codalab.lib.codalab_manager import CodaLabManager from codalab.lib import path_util dry_run = False if len(sys.argv) > 1 and sys.argv[1] == '-f' else True manager = CodaLabManager() model = manager.model() CODALAB_HOME = manager.codalab_home """Move data/ directory over to a temp area, and create a staging tree for uuid-based storage""" DATA_DIR = os.path.join(CODALAB_HOME, 'data') FINAL_LOCATION = os.path.join(CODALAB_HOME, 'bundles') if not dry_run: path_util.make_directory(FINAL_LOCATION) """For each data hash, get a list of all bundles that have that hash, and make a copy of the bundle in the staging area under the UUID for the bundle.""" data_hashes = reduce(lambda x, y: x + y, path_util.ls(DATA_DIR)) for data_hash in data_hashes: orig_location = os.path.join(DATA_DIR, data_hash) bundles_with_hash = model.batch_get_bundles(data_hash=data_hash) # We'd prefer renaming bundles to making copies, but because we are converting from deduplicated storage # we need to make sure that we only perform renames if we map 1:1 UUID->Hash. rename_allowed = len(bundles_with_hash) <= 1 for bundle in bundles_with_hash: # Build the command to be executed in a subshell uuid = bundle.uuid copy_location = os.path.join(FINAL_LOCATION, uuid) command = '%s %s %s' % ('mv' if rename_allowed else 'cp -a',
def make_temp_location(self, identifier): ''' Creates directory with given name under TEMP_SUBDIRECTORY ''' path_util.make_directory(self.get_temp_location(identifier));
def start_bundle(self, bundle, bundle_store, parent_dict, username): ''' Sets up all the temporary files and then dispatches the job. username: the username of the owner of the bundle Returns the bundle information. ''' # Create a temporary directory temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid) temp_dir = os.path.realpath(temp_dir) # Follow symlinks path_util.make_directory(temp_dir) # Copy all the dependencies to that temporary directory. pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir) print >>sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (bundle.uuid, temp_dir) for (source, target) in pairs: path_util.copy(source, target, follow_symlinks=False) # Set defaults for the dispatcher. docker_image = self.default_docker_image if bundle.metadata.request_docker_image: docker_image = bundle.metadata.request_docker_image request_time = self.default_request_time if bundle.metadata.request_time: request_time = bundle.metadata.request_time request_memory = self.default_request_memory if bundle.metadata.request_memory: request_memory = bundle.metadata.request_memory request_cpus = self.default_request_cpus if bundle.metadata.request_cpus: request_cpus = bundle.metadata.request_cpus request_gpus = self.default_request_gpus if bundle.metadata.request_gpus: request_gpus = bundle.metadata.request_gpus request_queue = self.default_request_queue if bundle.metadata.request_queue: request_queue = bundle.metadata.request_queue request_priority = self.default_request_priority if bundle.metadata.request_priority: request_priority = bundle.metadata.request_priority script_file = temp_dir + '.sh' # main entry point ptr_temp_dir = '$temp_dir' # 1) If no argument to script_file, use the temp_dir (e.g., Torque, master/worker share file system). # 2) If argument is 'use_script_for_temp_dir', use the script to determine temp_dir (e.g., qsub, no master/worker do not share file system). set_temp_dir_header = 'if [ -z "$1" ]; then temp_dir=' + temp_dir + '; else temp_dir=`readlink -f $0 | sed -e \'s/\\.sh$//\'`; fi\n' # Write the command to be executed to a script. if docker_image: internal_script_file = temp_dir + '-internal.sh' # run inside the docker container # These paths depend on $temp_dir, an environment variable which will be set (referenced inside script_file) ptr_container_file = ptr_temp_dir + '.cid' # contains the docker container id ptr_action_file = ptr_temp_dir + '.action' # send actions to the container (e.g., kill) ptr_status_dir = ptr_temp_dir + '.status' # receive information from the container (e.g., memory) ptr_script_file = ptr_temp_dir + '.sh' # main entry point ptr_internal_script_file = ptr_temp_dir + '-internal.sh' # run inside the docker container # Names of file inside the docker container docker_temp_dir = bundle.uuid docker_internal_script_file = bundle.uuid + '-internal.sh' # 1) script_file starts the docker container and runs internal_script_file in docker. # --rm removes the docker container once the job terminates (note that this makes things slow) # -v mounts the internal and user scripts and the temp directory # Trap SIGTERM and forward it to docker. with open(script_file, 'w') as f: f.write(set_temp_dir_header) # Monitor CPU/memory/disk def copy_if_exists(source_template, arg, target): source = source_template % arg # -f because target might be read-only return 'if [ -e %s ] && [ -e %s ]; then cp -f %s %s; fi' % (arg, source, source, target) monitor_commands = [ # Report on status (memory, cpu, etc.) 'mkdir -p %s' % ptr_status_dir, 'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is copy_if_exists('$cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat', ptr_container_file, ptr_status_dir), copy_if_exists('$cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes', ptr_container_file, ptr_status_dir), copy_if_exists('$cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes', ptr_container_file, ptr_status_dir), # Respond to kill action '[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s' % (ptr_action_file, ptr_action_file, ptr_container_file, ptr_action_file), # Sleep 'sleep 1', ] f.write('while [ -e %s ]; do\n %s\ndone &\n' % (ptr_temp_dir, '\n '. join(monitor_commands))) # Tell docker to constrain resources (memory). # Note: limiting memory is not always supported. See: # http://programster.blogspot.com/2014/09/docker-implementing-container-memory.html resource_args = '' if bundle.metadata.request_memory: resource_args += ' -m %s' % int(formatting.parse_size(bundle.metadata.request_memory)) # TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids f.write("docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s & wait $!\n" % ( resource_args, ptr_container_file, os.geteuid(), ptr_temp_dir, docker_temp_dir, ptr_internal_script_file, docker_internal_script_file, docker_image, docker_internal_script_file)) # 2) internal_script_file runs the actual command inside the docker container with open(internal_script_file, 'w') as f: # Make sure I have a username username = pwd.getpwuid(os.getuid())[0] # do this because os.getlogin() doesn't always work f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (username, os.geteuid(), os.getgid())) # Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...) f.write(". .bashrc || exit 1\n") # Go into the temp directory f.write("cd %s &&\n" % docker_temp_dir) # Run the actual command f.write('(%s) > stdout 2>stderr\n' % bundle.command) else: # Just run the command regularly without docker with open(script_file, 'w') as f: f.write(set_temp_dir_header) f.write("cd %s &&\n" % ptr_temp_dir) f.write('(%s) > stdout 2>stderr\n' % bundle.command) # Determine resources to request resource_args = [] if request_time: resource_args.extend(['--request_time', formatting.parse_duration(request_time)]) if request_memory: resource_args.extend(['--request_memory', formatting.parse_size(request_memory)]) if request_cpus: resource_args.extend(['--request_cpus', request_cpus]) if request_gpus: resource_args.extend(['--request_gpus', request_gpus]) if request_queue: resource_args.extend(['--request_queue', request_queue]) if request_priority: resource_args.extend(['--request_priority', request_priority]) if username: resource_args.extend(['--username', username]) # Start the command args = self.dispatch_command.split() + ['start'] + map(str, resource_args) + [script_file] if self.verbose >= 1: print '=== start_bundle(): running %s' % args result = json.loads(self.run_command_get_stdout(args)) if self.verbose >= 1: print '=== start_bundle(): got %s' % result # Return the information about the job. return { 'bundle': bundle, 'temp_dir': temp_dir, 'job_handle': result['handle'], 'docker_image': docker_image, }
def start_bundle(self, bundle, bundle_store, parent_dict, username): ''' Sets up all the temporary files and then dispatches the job. username: the username of the owner of the bundle Returns the bundle information. ''' # Create a temporary directory temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid) temp_dir = os.path.realpath(temp_dir) # Follow symlinks path_util.make_directory(temp_dir) # Copy all the dependencies to that temporary directory. pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir) print >>sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (bundle.uuid, temp_dir) for (source, target) in pairs: path_util.copy(source, target, follow_symlinks=False) # Set defaults for the dispatcher. docker_image = bundle.metadata.request_docker_image or self.default_docker_image # Parse |request_string| using |to_value|, but don't exceed |max_value|. def parse_and_min(to_value, request_string, default_value, max_value): # Use default if request value doesn't exist if request_string: request_value = to_value(request_string) else: request_value = default_value if request_value and max_value: return int(min(request_value, max_value)) elif request_value: return int(request_value) elif max_value: return int(max_value) else: return None request_time = parse_and_min(formatting.parse_duration, bundle.metadata.request_time, self.default_request_time, self.max_request_time) request_memory = parse_and_min(formatting.parse_size, bundle.metadata.request_memory, self.default_request_memory, self.max_request_memory) request_disk = parse_and_min(formatting.parse_size, bundle.metadata.request_disk, self.default_request_disk, self.max_request_disk) request_cpus = bundle.metadata.request_cpus or self.default_request_cpus request_gpus = bundle.metadata.request_gpus or self.default_request_gpus request_queue = bundle.metadata.request_queue or self.default_request_queue request_priority = bundle.metadata.request_priority or self.default_request_priority request_network = bundle.metadata.request_network or self.default_request_network script_file = temp_dir + '.sh' # main entry point ptr_temp_dir = '$temp_dir' # 1) If no argument to script_file, use the temp_dir (e.g., Torque, master/worker share file system). # 2) If argument is 'use_script_for_temp_dir', use the script to determine temp_dir (e.g., qsub, no master/worker do not share file system). set_temp_dir_header = 'if [ -z "$1" ]; then temp_dir=' + temp_dir + '; else temp_dir=`readlink -f $0 | sed -e \'s/\\.sh$//\'`; fi\n' # Write the command to be executed to a script. internal_script_file = temp_dir + '-internal.sh' # run inside the docker container # These paths depend on $temp_dir, an environment variable which will be set (referenced inside script_file) ptr_container_file = ptr_temp_dir + '.cid' # contains the docker container id ptr_action_file = ptr_temp_dir + '.action' # send actions to the container (e.g., kill) ptr_status_dir = ptr_temp_dir + '.status' # receive information from the container (e.g., memory) ptr_script_file = ptr_temp_dir + '.sh' # main entry point ptr_internal_script_file = ptr_temp_dir + '-internal.sh' # run inside the docker container # Names of file inside the docker container docker_temp_dir = '/' + bundle.uuid docker_internal_script_file = '/' + bundle.uuid + '-internal.sh' # 1) script_file starts the docker container and runs internal_script_file in docker. # --rm removes the docker container once the job terminates (note that this makes things slow) # -v mounts the internal and user scripts and the temp directory # Trap SIGTERM and forward it to docker. with open(script_file, 'w') as f: f.write(set_temp_dir_header) # Monitor CPU/memory/disk # Used to copy status about the docker container. def copy_if_exists(source_template, arg, target): source = source_template % arg # -f because target might be read-only return 'if [ -e %s ] && [ -e %s ]; then cp -f %s %s; fi' % (arg, source, source, target) def get_field(path, col): return 'cat %s | cut -f%s -d\'%s\'' % (path, col, BundleAction.SEPARATOR) monitor_commands = [ # Report on status (memory, cpu, etc.) 'mkdir -p %s' % ptr_status_dir, 'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is copy_if_exists('$cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat', ptr_container_file, ptr_status_dir), copy_if_exists('$cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes', ptr_container_file, ptr_status_dir), copy_if_exists('$cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes', ptr_container_file, ptr_status_dir), # Enforce memory limits '[ -e "%s/memory.usage_in_bytes" ] && mem=$(cat %s/memory.usage_in_bytes)' % (ptr_status_dir, ptr_status_dir), 'echo "memory: $mem (max %s)"' % request_memory, 'if [ -n "$mem" ] && [ "$mem" -gt "%s" ]; then echo "[CodaLab] Memory limit exceeded: $mem > %s, terminating." >> %s/stderr; docker kill $(cat %s); break; fi' % \ (request_memory, request_memory, ptr_temp_dir, ptr_container_file), # Enforce disk limits 'disk=$(du -sb %s | cut -f1)' % ptr_temp_dir, 'echo "disk: $disk (max %s)"' % request_disk, 'if [ -n "$disk" ] && [ "$disk" -gt "%s" ]; then echo "[CodaLab] Disk limit exceeded: $disk > %s, terminating." >> %s/stderr; docker kill $(cat %s); break; fi' % \ (request_disk, request_disk, ptr_temp_dir, ptr_container_file), # Execute "kill" 'if [ -e %s ] && [ "$(cat %s)" == "kill" ]; then echo "[CodaLab] Received kill command, terminating." >> %s/stderr; docker kill $(cat %s); rm %s; break; fi' % \ (ptr_action_file, ptr_action_file, ptr_temp_dir, ptr_container_file, ptr_action_file), # Execute "write <subpath> <contents>" 'if [ -e %s ] && [ "$(%s)" == "write" ]; then echo Writing...; %s > %s/$(%s); rm %s; fi' % \ (ptr_action_file, get_field(ptr_action_file, 1), get_field(ptr_action_file, '3-'), ptr_temp_dir, get_field(ptr_action_file, 2), ptr_action_file), # Sleep 'sleep 1', ] f.write('while [ -e %s ]; do\n %s\ndone &\n' % (ptr_temp_dir, '\n '. join(monitor_commands))) resource_args = '' # Limiting memory in docker is not (always) supported. So we rely on bash (see above). # http://programster.blogspot.com/2014/09/docker-implementing-container-memory.html #if request_memory: # resource_args += ' -m %s' % int(formatting.parse_size(request_memory)) # TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids # Attach all GPUs if any. Note that only the 64-bit version of # libcuda.so is picked up. f.write('devices=$(/bin/ls /dev/nvidia* 2>/dev/null)\n') f.write('if [ -n "$devices" ]; then devices=$(for d in $devices; do echo --device $d:$d; done); fi\n') f.write('libcuda=$(/sbin/ldconfig -p 2>/dev/null | grep "libcuda.so$" | grep "x86-64" | head -n 1 | cut -d " " -f 4)\n') f.write('if [ -n "$libcuda" ]; then libcuda=" -v $libcuda:/usr/lib/x86_64-linux-gnu/libcuda.so:ro"; fi\n') resource_args += ' $devices$libcuda' # Enable network? if not request_network: resource_args += ' --net=none' f.write("docker run%s --rm --cidfile %s -u %s -v %s:%s -v %s:%s -e HOME=%s %s bash %s >%s/stdout 2>%s/stderr & wait $!\n" % ( resource_args, ptr_container_file, os.geteuid(), ptr_temp_dir, docker_temp_dir, ptr_internal_script_file, docker_internal_script_file, docker_temp_dir, docker_image, docker_internal_script_file, ptr_temp_dir, ptr_temp_dir)) # 2) internal_script_file runs the actual command inside the docker container with open(internal_script_file, 'w') as f: # Make sure I have a username username = pwd.getpwuid(os.getuid())[0] # do this because os.getlogin() doesn't always work f.write("[ -w /etc/passwd ] && echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (username, os.geteuid(), os.getgid())) # Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...) f.write("[ -e .bashrc ] && . .bashrc\n") # Go into the temp directory f.write("cd %s &&\n" % docker_temp_dir) # Run the actual command f.write('(%s) >>stdout 2>>stderr\n' % bundle.command) # Determine resources to request resource_args = [] if request_time: resource_args.extend(['--request-time', request_time]) if request_memory: resource_args.extend(['--request-memory', request_memory]) if request_disk: resource_args.extend(['--request-disk', request_disk]) if request_cpus: resource_args.extend(['--request-cpus', request_cpus]) if request_gpus: resource_args.extend(['--request-gpus', request_gpus]) if request_queue: resource_args.extend(['--request-queue', request_queue]) if request_priority: resource_args.extend(['--request-priority', request_priority]) if username: resource_args.extend(['--username', username]) # Start the command args = self.dispatch_command.split() + ['start'] + map(str, resource_args) + [script_file] if self.verbose >= 1: print '=== start_bundle(): running %s' % args result = json.loads(self.run_command_get_stdout(args)) if self.verbose >= 1: print '=== start_bundle(): got %s' % result if not result['handle']: raise SystemError('Starting bundle failed') # Return the information about the job. return { 'bundle': bundle, 'temp_dir': temp_dir, 'job_handle': result['handle'], 'docker_image': docker_image, 'request_time': str(request_time) if request_time else None, 'request_memory': str(request_memory) if request_memory else None, 'request_disk': str(request_disk) if request_disk else None, 'request_cpus': request_cpus, 'request_gpus': request_gpus, 'request_queue': request_queue, 'request_priority': request_priority, 'request_network': request_network, }
def home(self): from codalab.lib import path_util result = path_util.normalize(self.config['home']) path_util.make_directory(result) return result
def start_bundle(self, bundle, bundle_store, parent_dict, username): ''' Sets up all the temporary files and then dispatches the job. username: the username of the owner of the bundle Returns the bundle information. ''' # Create a temporary directory temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid) temp_dir = os.path.realpath(temp_dir) # Follow symlinks path_util.make_directory(temp_dir) # Copy all the dependencies to that temporary directory. pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir) print >> sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % ( bundle.uuid, temp_dir) for (source, target) in pairs: path_util.copy(source, target, follow_symlinks=False) # Set defaults for the dispatcher. docker_image = self.default_docker_image if bundle.metadata.request_docker_image: docker_image = bundle.metadata.request_docker_image request_time = self.default_request_time if bundle.metadata.request_time: request_time = bundle.metadata.request_time request_memory = self.default_request_memory if bundle.metadata.request_memory: request_memory = bundle.metadata.request_memory request_cpus = self.default_request_cpus if bundle.metadata.request_cpus: request_cpus = bundle.metadata.request_cpus request_gpus = self.default_request_gpus if bundle.metadata.request_gpus: request_gpus = bundle.metadata.request_gpus request_queue = self.default_request_queue if bundle.metadata.request_queue: request_queue = bundle.metadata.request_queue request_priority = self.default_request_priority if bundle.metadata.request_priority: request_priority = bundle.metadata.request_priority script_file = temp_dir + '.sh' # main entry point ptr_temp_dir = '$temp_dir' # 1) If no argument to script_file, use the temp_dir (e.g., Torque, master/worker share file system). # 2) If argument is 'use_script_for_temp_dir', use the script to determine temp_dir (e.g., qsub, no master/worker do not share file system). set_temp_dir_header = 'if [ -z "$1" ]; then temp_dir=' + temp_dir + '; else temp_dir=`readlink -f $0 | sed -e \'s/\\.sh$//\'`; fi\n' # Write the command to be executed to a script. if docker_image: internal_script_file = temp_dir + '-internal.sh' # run inside the docker container # These paths depend on $temp_dir, an environment variable which will be set (referenced inside script_file) ptr_container_file = ptr_temp_dir + '.cid' # contains the docker container id ptr_action_file = ptr_temp_dir + '.action' # send actions to the container (e.g., kill) ptr_status_dir = ptr_temp_dir + '.status' # receive information from the container (e.g., memory) ptr_script_file = ptr_temp_dir + '.sh' # main entry point ptr_internal_script_file = ptr_temp_dir + '-internal.sh' # run inside the docker container # Names of file inside the docker container docker_temp_dir = bundle.uuid docker_internal_script_file = bundle.uuid + '-internal.sh' # 1) script_file starts the docker container and runs internal_script_file in docker. # --rm removes the docker container once the job terminates (note that this makes things slow) # -v mounts the internal and user scripts and the temp directory # Trap SIGTERM and forward it to docker. with open(script_file, 'w') as f: f.write(set_temp_dir_header) # Monitor CPU/memory/disk def copy_if_exists(source_template, arg, target): source = source_template % arg # -f because target might be read-only return 'if [ -e %s ] && [ -e %s ]; then cp -f %s %s; fi' % ( arg, source, source, target) monitor_commands = [ # Report on status (memory, cpu, etc.) 'mkdir -p %s' % ptr_status_dir, 'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is copy_if_exists( '$cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat', ptr_container_file, ptr_status_dir), copy_if_exists( '$cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes', ptr_container_file, ptr_status_dir), copy_if_exists( '$cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes', ptr_container_file, ptr_status_dir), # Respond to kill action '[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s' % (ptr_action_file, ptr_action_file, ptr_container_file, ptr_action_file), # Sleep 'sleep 1', ] f.write('while [ -e %s ]; do\n %s\ndone &\n' % (ptr_temp_dir, '\n '.join(monitor_commands))) # Tell docker to constrain resources (memory). # Note: limiting memory is not always supported. See: # http://programster.blogspot.com/2014/09/docker-implementing-container-memory.html resource_args = '' if bundle.metadata.request_memory: resource_args += ' -m %s' % int( formatting.parse_size(bundle.metadata.request_memory)) # TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids f.write( "docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s >%s/stdout 2>%s/stderr & wait $!\n" % (resource_args, ptr_container_file, os.geteuid(), ptr_temp_dir, docker_temp_dir, ptr_internal_script_file, docker_internal_script_file, docker_image, docker_internal_script_file, ptr_temp_dir, ptr_temp_dir)) # 2) internal_script_file runs the actual command inside the docker container with open(internal_script_file, 'w') as f: # Make sure I have a username username = pwd.getpwuid(os.getuid())[ 0] # do this because os.getlogin() doesn't always work f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (username, os.geteuid(), os.getgid())) # Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...) f.write(". .bashrc || exit 1\n") # Go into the temp directory f.write("cd %s &&\n" % docker_temp_dir) # Run the actual command f.write('(%s) >>stdout 2>>stderr\n' % bundle.command) else: # Just run the command regularly without docker with open(script_file, 'w') as f: f.write(set_temp_dir_header) f.write("cd %s &&\n" % ptr_temp_dir) f.write('(%s) >stdout 2>stderr\n' % bundle.command) # Determine resources to request resource_args = [] if request_time: resource_args.extend( ['--request_time', formatting.parse_duration(request_time)]) if request_memory: resource_args.extend( ['--request_memory', formatting.parse_size(request_memory)]) if request_cpus: resource_args.extend(['--request_cpus', request_cpus]) if request_gpus: resource_args.extend(['--request_gpus', request_gpus]) if request_queue: resource_args.extend(['--request_queue', request_queue]) if request_priority: resource_args.extend(['--request_priority', request_priority]) if username: resource_args.extend(['--username', username]) # Start the command args = self.dispatch_command.split() + ['start'] + map( str, resource_args) + [script_file] if self.verbose >= 1: print '=== start_bundle(): running %s' % args result = json.loads(self.run_command_get_stdout(args)) if self.verbose >= 1: print '=== start_bundle(): got %s' % result # Return the information about the job. return { 'bundle': bundle, 'temp_dir': temp_dir, 'job_handle': result['handle'], 'docker_image': docker_image, }