def run_task(self,fw_spec=None): calc_loc = get_calc_loc(self['calc_loc'], fw_spec["calc_locs"]) calc_dir = calc_loc["path"] filesystem = calc_loc["filesystem"] fileclient = FileClient(filesystem=filesystem) calc_dir = fileclient.abspath(calc_dir) filenames = self.get('filenames') if filenames is None: files_to_copy = fileclient.listdir(calc_dir) elif isinstance(filenames, six.string_types): raise ValueError("filenames must be a list!") elif '$ALL_NO_SUBDIRS' in filenames: files_to_copy = fileclient.listdir(calc_dir) elif '$ALL' in filenames: if self.get('name_prepend') or self.get('name_append'): raise ValueError('name_prepend or name_append options not compatible with "$ALL" option') copy_r(calc_dir, os.getcwd()) return else: files_to_copy = filenames for f in files_to_copy: prev_path_full = os.path.join(calc_dir, f) dest_fname = self.get('name_prepend', "") + f + self.get( 'name_append', "") dest_path = os.path.join(os.getcwd(), dest_fname) fileclient.copy(prev_path_full, dest_path)
def run_task(self, fw_spec=None): calc_loc = get_calc_loc(self['calc_loc'], fw_spec["calc_locs"]) calc_dir = calc_loc["path"] filesystem = calc_loc["filesystem"] fileclient = FileClient(filesystem=filesystem) calc_dir = fileclient.abspath(calc_dir) filenames = self.get('filenames') if filenames is None: files_to_copy = fileclient.listdir(calc_dir) elif isinstance(filenames, str): raise ValueError("filenames must be a list!") elif '$ALL_NO_SUBDIRS' in filenames: files_to_copy = fileclient.listdir(calc_dir) elif '$ALL' in filenames: if self.get('name_prepend') or self.get('name_append'): raise ValueError( 'name_prepend or name_append options not compatible with "$ALL" option' ) copy_r(calc_dir, os.getcwd()) return else: files_to_copy = filenames for f in files_to_copy: prev_path_full = os.path.join(calc_dir, f) dest_fname = self.get('name_prepend', "") + f + self.get( 'name_append', "") dest_path = os.path.join(os.getcwd(), dest_fname) fileclient.copy(prev_path_full, dest_path)
def test_mongodb_more_files(self): try: VaspCalcDb.from_db_file(DB_FILE) except: raise unittest.SkipTest( "Cannot connect to MongoDB! Is the database server running? " "Are the credentials correct?") copy_r(self.vasp_dir, ".") with self.assertWarnsRegex(UserWarning, ".*wrong_file.*"): ft = LobsterRunToDb( calc_loc=True, db_file=DB_FILE, additional_outputs=[ "ICOHPLIST.lobster", "COOPCAR.lobster", "wrong_file", ], ) ft.run_task({"calc_locs": [{"path": "test"}]}) coll = self.get_task_collection("lobster") load_dict = coll.find_one({"formula_pretty": "K2Sn2O3"}) self.assertEqual(load_dict["formula_pretty"], "K2Sn2O3") self.assertListEqual(load_dict["output"]["chargespilling"], [0.008, 0.008]) db = self.get_task_database() gfs = gridfs.GridFS(db, "lobster_files") results = gfs.find({}).count() self.assertEqual(results, 2) for fn in ["ICOHPLIST", "COOPCAR"]: oid = load_dict[fn.lower() + "_id"] results = gfs.find({"_id": oid}).count() self.assertEqual(results, 1)
def __exit__(self, exc_type, exc_val, exc_tb): if self.rootpath is not None and os.path.exists(self.rootpath): if self.end_copy: copy_r(".", self.cwd) shutil.rmtree(self.tempdir) os.chdir(self.cwd) if self.create_symbolic_link: os.remove(ScratchDir.SCR_LINK)
def test_jsonfile_si(self): copy_r(self.vasp_si_dir, ".") ft = LobsterRunToDb(calc_loc=True) ft.run_task({"calc_locs": [{"path": "test"}]}) with open("task_lobster.json") as f: load_dict = json.load(f) self.assertEqual(load_dict["formula_pretty"], "Si") self.assertListEqual(load_dict["output"]["chargespilling"], [0.0147, 0.0147])
def __enter__(self): tempdir = self.cwd if self.rootpath is not None and os.path.exists(self.rootpath): tempdir = tempfile.mkdtemp(dir=self.rootpath) self.tempdir = os.path.abspath(tempdir) if self.start_copy: copy_r(".", tempdir) if self.create_symbolic_link: os.symlink(tempdir, ScratchDir.SCR_LINK) os.chdir(tempdir) return tempdir
def test_recursive_copy_and_compress(self): copy_r(os.path.join(test_dir, "cpr_src"), os.path.join(test_dir, "cpr_dst")) self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_dst", "test"))) self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_dst", "sub", "testr"))) compress_dir(os.path.join(test_dir, "cpr_src")) self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_src", "test.gz"))) self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_src", "sub", "testr.gz"))) decompress_dir(os.path.join(test_dir, "cpr_src")) self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_src", "test"))) self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_src", "sub", "testr"))) with open(os.path.join(test_dir, "cpr_src", "test")) as f: txt = f.read() self.assertEqual(txt, "what")
def __exit__(self, exc_type, exc_val, exc_tb): if self.rootpath is not None and os.path.exists(self.rootpath): if self.end_copy: files = set(os.listdir(self.tempdir)) orig_files = set(os.listdir(self.cwd)) # First copy files over copy_r(self.tempdir, self.cwd) # Delete any files that are now gone for f in orig_files - files: fpath = os.path.join(self.cwd, f) remove(fpath) os.chdir(self.cwd) remove(self.tempdir)
def test_mongodb(self): try: VaspCalcDb.from_db_file(DB_FILE) except: raise unittest.SkipTest( "Cannot connect to MongoDB! Is the database server running? " "Are the credentials correct?") copy_r(self.vasp_dir, ".") ft = LobsterRunToDb(calc_loc=True, db_file=DB_FILE) ft.run_task({"calc_locs": [{"path": "test"}]}) coll = self.get_task_collection("lobster") load_dict = coll.find_one({"formula_pretty": "K2Sn2O3"}) self.assertEqual(load_dict["formula_pretty"], "K2Sn2O3") self.assertListEqual(load_dict["output"]["chargespilling"], [0.008, 0.008]) self.assertNotIn("lobster_icohplist", load_dict)
def run_task(self, fw_spec=None): calc_loc = get_calc_loc(self["calc_loc"], fw_spec["calc_locs"]) calc_dir = calc_loc["path"] filesystem = calc_loc["filesystem"] fileclient = FileClient(filesystem=filesystem) calc_dir = fileclient.abspath(calc_dir) filenames = self.get("filenames") exclude_files = self.get("exclude_files", []) if filenames is None: files_to_copy = fileclient.listdir(calc_dir) elif isinstance(filenames, str): raise ValueError("filenames must be a list!") elif "$ALL_NO_SUBDIRS" in filenames: files_to_copy = fileclient.listdir(calc_dir) elif "$ALL" in filenames: if ( self.get("name_prepend") or self.get("name_append") or self.get("exclude_files") ): raise ValueError( 'name_prepend, name_append, and exclude_files \ options not compatible with "$ALL" option' ) copy_r(calc_dir, os.getcwd()) return else: files_to_copy = [] for fname in filenames: for f in glob.glob(os.path.join(calc_dir, fname)): files_to_copy.append(os.path.basename(f)) # delete any excluded files for fname in exclude_files: for f in glob.glob(os.path.join(calc_dir, fname)): if os.path.basename(f) in files_to_copy: files_to_copy.remove(os.path.basename(f)) for f in files_to_copy: prev_path_full = os.path.join(calc_dir, f) dest_fname = self.get("name_prepend", "") + f + self.get("name_append", "") dest_path = os.path.join(os.getcwd(), dest_fname) fileclient.copy(prev_path_full, dest_path)
def __exit__(self, exc_type, exc_val, exc_tb): if self.rootpath is not None and os.path.exists(self.rootpath): if self.end_copy: tempdir = tempfile.mkdtemp(dir=self.cwd) copy_r(self.cwd, tempdir) for f in os.listdir(self.cwd): fpath = os.path.join(self.cwd, f) try: if f != os.path.basename(tempdir): if os.path.isfile(fpath): os.remove(fpath) else: shutil.rmtree(fpath) except FileNotFoundError: # Ignore file not found. pass copy_r(".", self.cwd) shutil.rmtree(tempdir) os.chdir(self.cwd) shutil.rmtree(self.tempdir) if self.create_symbolic_link: os.remove(ScratchDir.SCR_LINK)
def test_recursive_copy_and_compress(self): copy_r(os.path.join(test_dir, "cpr_src"), os.path.join(test_dir, "cpr_dst")) self.assertTrue( os.path.exists(os.path.join(test_dir, "cpr_dst", "test"))) self.assertTrue( os.path.exists(os.path.join(test_dir, "cpr_dst", "sub", "testr"))) compress_dir(os.path.join(test_dir, "cpr_src")) self.assertTrue( os.path.exists(os.path.join(test_dir, "cpr_src", "test.gz"))) self.assertTrue( os.path.exists(os.path.join(test_dir, "cpr_src", "sub", "testr.gz"))) decompress_dir(os.path.join(test_dir, "cpr_src")) self.assertTrue( os.path.exists(os.path.join(test_dir, "cpr_src", "test"))) self.assertTrue( os.path.exists(os.path.join(test_dir, "cpr_src", "sub", "testr"))) with open(os.path.join(test_dir, "cpr_src", "test")) as f: txt = f.read() self.assertEqual(txt, "what")
def __enter__(self): tempdirs = [self.cwd] * self.n_dirs if self.rootpath is not None and os.path.exists(self.rootpath): tempdirs = [ tempfile.mkdtemp(dir=self.rootpath) for _ in range(self.n_dirs) ] self.tempdirs = [os.path.abspath(tempdir) for tempdir in tempdirs] if self.start_copy: [copy_r(".", tempdir) for tempdir in tempdirs] if self.create_symbolic_link: [ os.symlink(tempdir, '%s_%d' % (MultiScratchDir.SCR_LINK, i)) for i, tempdir in enumerate(tempdirs) ] return tempdirs
def copy_r(self, dst): """ Implements a recursive copy function similar to Unix's "cp -r" command. """ return copy_r(self.path, dst)