def run_task(self,fw_spec=None):
calc_loc = get_calc_loc(self['calc_loc'], fw_spec["calc_locs"])
calc_dir = calc_loc["path"]
filesystem = calc_loc["filesystem"]
fileclient = FileClient(filesystem=filesystem)
calc_dir = fileclient.abspath(calc_dir)
filenames = self.get('filenames')
if filenames is None:
files_to_copy = fileclient.listdir(calc_dir)
elif isinstance(filenames, six.string_types):
raise ValueError("filenames must be a list!")
elif '$ALL_NO_SUBDIRS' in filenames:
files_to_copy = fileclient.listdir(calc_dir)
elif '$ALL' in filenames:
if self.get('name_prepend') or self.get('name_append'):
raise ValueError('name_prepend or name_append options not compatible with "$ALL" option')
copy_r(calc_dir, os.getcwd())
return
else:
files_to_copy = filenames
for f in files_to_copy:
prev_path_full = os.path.join(calc_dir, f)
dest_fname = self.get('name_prepend', "") + f + self.get(
'name_append', "")
dest_path = os.path.join(os.getcwd(), dest_fname)
fileclient.copy(prev_path_full, dest_path)
def run_task(self, fw_spec=None):
calc_loc = get_calc_loc(self['calc_loc'], fw_spec["calc_locs"])
calc_dir = calc_loc["path"]
filesystem = calc_loc["filesystem"]
fileclient = FileClient(filesystem=filesystem)
calc_dir = fileclient.abspath(calc_dir)
filenames = self.get('filenames')
if filenames is None:
files_to_copy = fileclient.listdir(calc_dir)
elif isinstance(filenames, str):
raise ValueError("filenames must be a list!")
elif '$ALL_NO_SUBDIRS' in filenames:
files_to_copy = fileclient.listdir(calc_dir)
elif '$ALL' in filenames:
if self.get('name_prepend') or self.get('name_append'):
raise ValueError(
'name_prepend or name_append options not compatible with "$ALL" option'
)
copy_r(calc_dir, os.getcwd())
return
else:
files_to_copy = filenames
for f in files_to_copy:
prev_path_full = os.path.join(calc_dir, f)
dest_fname = self.get('name_prepend', "") + f + self.get(
'name_append', "")
dest_path = os.path.join(os.getcwd(), dest_fname)
fileclient.copy(prev_path_full, dest_path)
def test_mongodb_more_files(self):
try:
VaspCalcDb.from_db_file(DB_FILE)
except:
raise unittest.SkipTest(
"Cannot connect to MongoDB! Is the database server running? "
"Are the credentials correct?")
copy_r(self.vasp_dir, ".")
with self.assertWarnsRegex(UserWarning, ".*wrong_file.*"):
ft = LobsterRunToDb(
calc_loc=True,
db_file=DB_FILE,
additional_outputs=[
"ICOHPLIST.lobster",
"COOPCAR.lobster",
"wrong_file",
],
)
ft.run_task({"calc_locs": [{"path": "test"}]})
coll = self.get_task_collection("lobster")
load_dict = coll.find_one({"formula_pretty": "K2Sn2O3"})
self.assertEqual(load_dict["formula_pretty"], "K2Sn2O3")
self.assertListEqual(load_dict["output"]["chargespilling"],
[0.008, 0.008])
db = self.get_task_database()
gfs = gridfs.GridFS(db, "lobster_files")
results = gfs.find({}).count()
self.assertEqual(results, 2)
for fn in ["ICOHPLIST", "COOPCAR"]:
oid = load_dict[fn.lower() + "_id"]
results = gfs.find({"_id": oid}).count()
self.assertEqual(results, 1)
def __exit__(self, exc_type, exc_val, exc_tb):
if self.rootpath is not None and os.path.exists(self.rootpath):
if self.end_copy:
copy_r(".", self.cwd)
shutil.rmtree(self.tempdir)
os.chdir(self.cwd)
if self.create_symbolic_link:
os.remove(ScratchDir.SCR_LINK)
def __exit__(self, exc_type, exc_val, exc_tb):
if self.rootpath is not None and os.path.exists(self.rootpath):
if self.end_copy:
copy_r(".", self.cwd)
shutil.rmtree(self.tempdir)
os.chdir(self.cwd)
if self.create_symbolic_link:
os.remove(ScratchDir.SCR_LINK)
def test_jsonfile_si(self):
copy_r(self.vasp_si_dir, ".")
ft = LobsterRunToDb(calc_loc=True)
ft.run_task({"calc_locs": [{"path": "test"}]})
with open("task_lobster.json") as f:
load_dict = json.load(f)
self.assertEqual(load_dict["formula_pretty"], "Si")
self.assertListEqual(load_dict["output"]["chargespilling"],
[0.0147, 0.0147])
def __enter__(self):
tempdir = self.cwd
if self.rootpath is not None and os.path.exists(self.rootpath):
tempdir = tempfile.mkdtemp(dir=self.rootpath)
self.tempdir = os.path.abspath(tempdir)
if self.start_copy:
copy_r(".", tempdir)
if self.create_symbolic_link:
os.symlink(tempdir, ScratchDir.SCR_LINK)
os.chdir(tempdir)
return tempdir
def __enter__(self):
tempdir = self.cwd
if self.rootpath is not None and os.path.exists(self.rootpath):
tempdir = tempfile.mkdtemp(dir=self.rootpath)
self.tempdir = os.path.abspath(tempdir)
if self.start_copy:
copy_r(".", tempdir)
if self.create_symbolic_link:
os.symlink(tempdir, ScratchDir.SCR_LINK)
os.chdir(tempdir)
return tempdir
def test_recursive_copy_and_compress(self):
copy_r(os.path.join(test_dir, "cpr_src"), os.path.join(test_dir, "cpr_dst"))
self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_dst", "test")))
self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_dst", "sub", "testr")))
compress_dir(os.path.join(test_dir, "cpr_src"))
self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_src", "test.gz")))
self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_src", "sub", "testr.gz")))
decompress_dir(os.path.join(test_dir, "cpr_src"))
self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_src", "test")))
self.assertTrue(os.path.exists(os.path.join(test_dir, "cpr_src", "sub", "testr")))
with open(os.path.join(test_dir, "cpr_src", "test")) as f:
txt = f.read()
self.assertEqual(txt, "what")
def __exit__(self, exc_type, exc_val, exc_tb):
if self.rootpath is not None and os.path.exists(self.rootpath):
if self.end_copy:
files = set(os.listdir(self.tempdir))
orig_files = set(os.listdir(self.cwd))
# First copy files over
copy_r(self.tempdir, self.cwd)
# Delete any files that are now gone
for f in orig_files - files:
fpath = os.path.join(self.cwd, f)
remove(fpath)
os.chdir(self.cwd)
remove(self.tempdir)
def test_mongodb(self):
try:
VaspCalcDb.from_db_file(DB_FILE)
except:
raise unittest.SkipTest(
"Cannot connect to MongoDB! Is the database server running? "
"Are the credentials correct?")
copy_r(self.vasp_dir, ".")
ft = LobsterRunToDb(calc_loc=True, db_file=DB_FILE)
ft.run_task({"calc_locs": [{"path": "test"}]})
coll = self.get_task_collection("lobster")
load_dict = coll.find_one({"formula_pretty": "K2Sn2O3"})
self.assertEqual(load_dict["formula_pretty"], "K2Sn2O3")
self.assertListEqual(load_dict["output"]["chargespilling"],
[0.008, 0.008])
self.assertNotIn("lobster_icohplist", load_dict)
def run_task(self, fw_spec=None):
calc_loc = get_calc_loc(self["calc_loc"], fw_spec["calc_locs"])
calc_dir = calc_loc["path"]
filesystem = calc_loc["filesystem"]
fileclient = FileClient(filesystem=filesystem)
calc_dir = fileclient.abspath(calc_dir)
filenames = self.get("filenames")
exclude_files = self.get("exclude_files", [])
if filenames is None:
files_to_copy = fileclient.listdir(calc_dir)
elif isinstance(filenames, str):
raise ValueError("filenames must be a list!")
elif "$ALL_NO_SUBDIRS" in filenames:
files_to_copy = fileclient.listdir(calc_dir)
elif "$ALL" in filenames:
if (
self.get("name_prepend")
or self.get("name_append")
or self.get("exclude_files")
):
raise ValueError(
'name_prepend, name_append, and exclude_files \
options not compatible with "$ALL" option'
)
copy_r(calc_dir, os.getcwd())
return
else:
files_to_copy = []
for fname in filenames:
for f in glob.glob(os.path.join(calc_dir, fname)):
files_to_copy.append(os.path.basename(f))
# delete any excluded files
for fname in exclude_files:
for f in glob.glob(os.path.join(calc_dir, fname)):
if os.path.basename(f) in files_to_copy:
files_to_copy.remove(os.path.basename(f))
for f in files_to_copy:
prev_path_full = os.path.join(calc_dir, f)
dest_fname = self.get("name_prepend", "") + f + self.get("name_append", "")
dest_path = os.path.join(os.getcwd(), dest_fname)
fileclient.copy(prev_path_full, dest_path)
def __exit__(self, exc_type, exc_val, exc_tb):
if self.rootpath is not None and os.path.exists(self.rootpath):
if self.end_copy:
tempdir = tempfile.mkdtemp(dir=self.cwd)
copy_r(self.cwd, tempdir)
for f in os.listdir(self.cwd):
fpath = os.path.join(self.cwd, f)
try:
if f != os.path.basename(tempdir):
if os.path.isfile(fpath):
os.remove(fpath)
else:
shutil.rmtree(fpath)
except FileNotFoundError:
# Ignore file not found.
pass
copy_r(".", self.cwd)
shutil.rmtree(tempdir)
os.chdir(self.cwd)
shutil.rmtree(self.tempdir)
if self.create_symbolic_link:
os.remove(ScratchDir.SCR_LINK)
def __exit__(self, exc_type, exc_val, exc_tb):
if self.rootpath is not None and os.path.exists(self.rootpath):
if self.end_copy:
tempdir = tempfile.mkdtemp(dir=self.cwd)
copy_r(self.cwd, tempdir)
for f in os.listdir(self.cwd):
fpath = os.path.join(self.cwd, f)
try:
if f != os.path.basename(tempdir):
if os.path.isfile(fpath):
os.remove(fpath)
else:
shutil.rmtree(fpath)
except FileNotFoundError:
# Ignore file not found.
pass
copy_r(".", self.cwd)
shutil.rmtree(tempdir)
os.chdir(self.cwd)
shutil.rmtree(self.tempdir)
if self.create_symbolic_link:
os.remove(ScratchDir.SCR_LINK)
def test_recursive_copy_and_compress(self):
copy_r(os.path.join(test_dir, "cpr_src"),
os.path.join(test_dir, "cpr_dst"))
self.assertTrue(
os.path.exists(os.path.join(test_dir, "cpr_dst", "test")))
self.assertTrue(
os.path.exists(os.path.join(test_dir, "cpr_dst", "sub", "testr")))
compress_dir(os.path.join(test_dir, "cpr_src"))
self.assertTrue(
os.path.exists(os.path.join(test_dir, "cpr_src", "test.gz")))
self.assertTrue(
os.path.exists(os.path.join(test_dir, "cpr_src", "sub",
"testr.gz")))
decompress_dir(os.path.join(test_dir, "cpr_src"))
self.assertTrue(
os.path.exists(os.path.join(test_dir, "cpr_src", "test")))
self.assertTrue(
os.path.exists(os.path.join(test_dir, "cpr_src", "sub",
"testr")))
with open(os.path.join(test_dir, "cpr_src", "test")) as f:
txt = f.read()
self.assertEqual(txt, "what")
def __enter__(self):
tempdirs = [self.cwd] * self.n_dirs
if self.rootpath is not None and os.path.exists(self.rootpath):
tempdirs = [
tempfile.mkdtemp(dir=self.rootpath) for _ in range(self.n_dirs)
]
self.tempdirs = [os.path.abspath(tempdir) for tempdir in tempdirs]
if self.start_copy:
[copy_r(".", tempdir) for tempdir in tempdirs]
if self.create_symbolic_link:
[
os.symlink(tempdir,
'%s_%d' % (MultiScratchDir.SCR_LINK, i))
for i, tempdir in enumerate(tempdirs)
]
return tempdirs
def copy_r(self, dst):
"""
Implements a recursive copy function similar to Unix's "cp -r" command.
"""
return copy_r(self.path, dst)
def copy_r(self, dst):
"""
Implements a recursive copy function similar to Unix's "cp -r" command.
"""
return copy_r(self.path, dst)
