def _try_start_bundle(self, workers, worker, bundle): """ Tries to start running the bundle on the given worker, returning False if that failed. """ if self._model.set_starting_bundle(bundle, worker['user_id'], worker['worker_id']): workers.set_starting(bundle.uuid, worker) if self._worker_model.shared_file_system and worker[ 'user_id'] == self._model.root_user_id: # On a shared file system we create the path here to avoid NFS # directory cache issues. path = self._bundle_store.get_bundle_location(bundle.uuid) remove_path(path) os.mkdir(path) if self._worker_model.send_json_message( worker['socket_id'], self._construct_run_message(worker, bundle), 0.2): logger.info('Starting run bundle %s', bundle.uuid) return True else: self._model.restage_bundle(bundle) workers.restage(bundle.uuid) return False else: return False
def test_tar_empty(self): dir = tempfile.mkdtemp() self.addCleanup(lambda: remove_path(dir)) output_dir = tempfile.mkdtemp() self.addCleanup(lambda: remove_path(output_dir)) un_tar_gzip_directory(tar_gzip_directory(dir), output_dir) self.assertEquals(os.listdir(output_dir), [])
def test_tar_empty(self): dir = tempfile.mkdtemp() self.addCleanup(lambda: remove_path(dir)) temp_dir = tempfile.mkdtemp() self.addCleanup(lambda: remove_path(temp_dir)) output_dir = os.path.join(temp_dir, 'output') un_tar_directory(tar_gzip_directory(dir), output_dir, 'gz') self.assertEquals(os.listdir(output_dir), [])
def _make_bundle(self, bundle): try: path = os.path.normpath( self._bundle_store.get_bundle_location(bundle.uuid)) deps = [] for dep in bundle.dependencies: parent_bundle_path = os.path.normpath( self._bundle_store.get_bundle_location(dep.parent_uuid)) dependency_path = os.path.normpath( os.path.join(parent_bundle_path, dep.parent_path)) if (not dependency_path.startswith(parent_bundle_path) or (not os.path.islink(dependency_path) and not os.path.exists(dependency_path))): raise Exception('Invalid dependency %s' % (path_util.safe_join( dep.parent_uuid, dep.parent_path))) child_path = os.path.normpath( os.path.join(path, dep.child_path)) if not child_path.startswith(path): raise Exception('Invalid key for dependency: %s' % (dep.child_path)) deps.append((dependency_path, child_path)) remove_path(path) if len(deps) == 1 and deps[0][1] == path: path_util.copy(deps[0][0], path, follow_symlinks=False) else: os.mkdir(path) for dependency_path, child_path in deps: path_util.copy(dependency_path, child_path, follow_symlinks=False) self._upload_manager.update_metadata_and_save(bundle, new_bundle=False) logger.info('Finished making bundle %s', bundle.uuid) self._model.update_bundle(bundle, {'state': State.READY}) except Exception as e: logger.info('Failing bundle %s: %s', bundle.uuid, str(e)) self._model.update_bundle(bundle, { 'state': State.FAILED, 'metadata': { 'failure_message': str(e) } }) finally: with self._make_uuids_lock: self._make_uuids.remove(bundle.uuid)
def _make_bundle(self, bundle): try: path = os.path.normpath(self._bundle_store.get_bundle_location(bundle.uuid)) deps = [] for dep in bundle.dependencies: parent_bundle_path = os.path.normpath( self._bundle_store.get_bundle_location(dep.parent_uuid)) dependency_path = os.path.normpath( os.path.join(parent_bundle_path, dep.parent_path)) if (not dependency_path.startswith(parent_bundle_path) or (not os.path.islink(dependency_path) and not os.path.exists(dependency_path))): raise Exception('Invalid dependency %s' % ( path_util.safe_join(dep.parent_uuid, dep.parent_path))) child_path = os.path.normpath( os.path.join(path, dep.child_path)) if not child_path.startswith(path): raise Exception('Invalid key for dependency: %s' % ( dep.child_path)) deps.append((dependency_path, child_path)) remove_path(path) if len(deps) == 1 and deps[0][1] == path: path_util.copy(deps[0][0], path, follow_symlinks=False) else: os.mkdir(path) for dependency_path, child_path in deps: path_util.copy(dependency_path, child_path, follow_symlinks=False) self._upload_manager.update_metadata_and_save(bundle, new_bundle=False) logger.info('Finished making bundle %s', bundle.uuid) self._model.update_bundle(bundle, {'state': State.READY}) except Exception as e: logger.info('Failing bundle %s: %s', bundle.uuid, str(e)) self._model.update_bundle( bundle, {'state': State.FAILED, 'metadata': {'failure_message': str(e)}}) finally: with self._make_uuids_lock: self._make_uuids.remove(bundle.uuid)
def _try_start_bundle(self, workers, worker, bundle): """ Tries to start running the bundle on the given worker, returning False if that failed. """ if self._model.set_starting_bundle(bundle, worker['user_id'], worker['worker_id']): workers.set_starting(bundle.uuid, worker) if self._worker_model.shared_file_system and worker['user_id'] == self._model.root_user_id: # On a shared file system we create the path here to avoid NFS # directory cache issues. path = self._bundle_store.get_bundle_location(bundle.uuid) remove_path(path) os.mkdir(path) if self._worker_model.send_json_message( worker['socket_id'], self._construct_run_message(worker, bundle), 0.2): logger.info('Starting run bundle %s', bundle.uuid) return True else: self._model.restage_bundle(bundle) workers.restage(bundle.uuid) return False else: return False
def test_tar_has_files(self): dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'files') temp_dir = tempfile.mkdtemp() self.addCleanup(lambda: remove_path(temp_dir)) output_dir = os.path.join(temp_dir, 'output') un_tar_directory( tar_gzip_directory(dir, False, ['f2'], ['f1', 'b.txt']), output_dir, 'gz') output_dir_entries = os.listdir(output_dir) self.assertIn('dir1', output_dir_entries) self.assertIn('a.txt', output_dir_entries) self.assertNotIn('b.txt', output_dir_entries) self.assertTrue(os.path.exists(os.path.join(output_dir, 'dir1', 'f1'))) self.assertFalse(os.path.exists(os.path.join(output_dir, 'dir1', 'f2'))) self.assertTrue(os.path.islink(os.path.join(output_dir, 'a-symlink.txt')))
def tearDown(self): remove_path(self.work_dir)
def tearDown(self): remove_path(self.temp_dir)
def _clear_torque_logs(self, job_handle): remove_path(os.path.join(self._torque_log_dir, 'stdout.' + job_handle)) remove_path(os.path.join(self._torque_log_dir, 'stderr.' + job_handle))