def setup_copy(self,
from_dir,
to_dir=None,
filesystem=None,
files_to_copy=None,
exclude_files=None,
from_path_dict=None):
"""
setup the copy i.e setup the from directory, filesystem, destination directory etc.
Args:
from_dir (str)
to_dir (str)
filesystem (str)
files_to_copy (list): if None all the files in the from_dir will be copied
exclude_files (list)
from_path_dict (dict): dict specification of the path. If specified must contain atleast
the key "path" that specifies the path to the from_dir.
"""
from_path_dict = from_path_dict or {}
from_dir = from_dir or from_path_dict.get("path", None)
filesystem = filesystem or from_path_dict.get("filesystem", None)
if from_dir is None:
raise ValueError("Must specify from_dir!")
self.fileclient = FileClient(filesystem=filesystem)
self.from_dir = self.fileclient.abspath(from_dir)
self.to_dir = to_dir or os.getcwd()
exclude_files = exclude_files or []
self.files_to_copy = files_to_copy or [
f for f in self.fileclient.listdir(self.from_dir)
if f not in exclude_files
]
def setup_copy(self, from_dir, to_dir=None, filesystem=None, files_to_copy=None,
exclude_files=None, from_path_dict=None, suffix=None,
fw_spec=None, continue_on_missing=False):
"""
setup the copy i.e setup the from directory, filesystem, destination directory etc.
Args:
from_dir (str)
to_dir (str)
filesystem (str)
files_to_copy (list): if None all the files in the from_dir will be copied
exclude_files (list): list of file names to be excluded.
suffix (str): suffix to append to each filename when copying
(e.g., rename 'INCAR' to 'INCAR.precondition')
continue_on_missing(bool): Whether to continue copying when a file
in filenames is missing. Defaults to False.
from_path_dict (dict): dict specification of the path. If specified must contain atleast
the key "path" that specifies the path to the from_dir.
"""
from_path_dict = from_path_dict or {}
from_dir = env_chk(from_dir, fw_spec, strict=False) or from_path_dict.get("path", None)
filesystem = filesystem or from_path_dict.get("filesystem", None)
if from_dir is None:
raise ValueError("Must specify from_dir!")
self.fileclient = FileClient(filesystem=filesystem)
self.from_dir = self.fileclient.abspath(from_dir)
self.to_dir = env_chk(to_dir, fw_spec, strict=False) or os.getcwd()
exclude_files = exclude_files or []
self.files_to_copy = files_to_copy or [f for f in self.fileclient.listdir(self.from_dir) if f not in exclude_files]
self.suffix = suffix
self.continue_on_missing = continue_on_missing
class CopyFiles(FiretaskBase):
"""
Task to copy the given list of files from the given directory to the destination directory.
To customize override the setup_copy and copy_files methods.
Optional params:
from_dir (str): path to the directory containing the files to be copied.
to_dir (str): path to the destination directory
filesystem (str)
files_to_copy (list): list of file names.
exclude_files (list): list of file names to be excluded.
"""
optional_params = ["from_dir", "to_dir", "filesystem", "files_to_copy", "exclude_files"]
def setup_copy(self, from_dir, to_dir=None, filesystem=None, files_to_copy=None, exclude_files=None,
from_path_dict=None):
"""
setup the copy i.e setup the from directory, filesystem, destination directory etc.
Args:
from_dir (str)
to_dir (str)
filesystem (str)
files_to_copy (list): if None all the files in the from_dir will be copied
exclude_files (list)
from_path_dict (dict): dict specification of the path. If specified must contain atleast
the key "path" that specifies the path to the from_dir.
"""
from_path_dict = from_path_dict or {}
from_dir = from_dir or from_path_dict.get("path", None)
filesystem = filesystem or from_path_dict.get("filesystem", None)
if from_dir is None:
raise ValueError("Must specify from_dir!")
self.fileclient = FileClient(filesystem=filesystem)
self.from_dir = self.fileclient.abspath(from_dir)
self.to_dir = to_dir or os.getcwd()
exclude_files = exclude_files or []
self.files_to_copy = files_to_copy or [f for f in self.fileclient.listdir(self.from_dir) if f not in exclude_files]
def copy_files(self):
"""
Defines the copy operation. Override this to customize copying.
"""
for f in self.files_to_copy:
prev_path_full = os.path.join(self.from_dir, f)
dest_path = os.path.join(self.to_dir, f)
self.fileclient.copy(prev_path_full, dest_path)
def run_task(self, fw_spec):
self.setup_copy(self.get("from_dir", None), to_dir=self.get("to_dir", None),
filesystem=self.get("filesystem", None),
files_to_copy=self.get("files_to_copy", None),
exclude_files=self.get("exclude_files", []))
self.copy_files()
class CopyFiles(FiretaskBase):
"""
Task to copy the given list of files from the given directory to the destination directory.
To customize override the setup_copy and copy_files methods.
Optional params:
from_dir (str): path to the directory containing the files to be copied.
to_dir (str): path to the destination directory
filesystem (str)
files_to_copy (list): list of file names.
exclude_files (list): list of file names to be excluded.
"""
optional_params = ["from_dir", "to_dir", "filesystem", "files_to_copy", "exclude_files"]
def setup_copy(self, from_dir, to_dir=None, filesystem=None, files_to_copy=None, exclude_files=None,
from_path_dict=None):
"""
setup the copy i.e setup the from directory, filesystem, destination directory etc.
Args:
from_dir (str)
to_dir (str)
filesystem (str)
files_to_copy (list): if None all the files in the from_dir will be copied
exclude_files (list)
from_path_dict (dict): dict specification of the path. If specified must contain atleast
the key "path" that specifies the path to the from_dir.
"""
from_path_dict = from_path_dict or {}
from_dir = from_dir or from_path_dict.get("path", None)
filesystem = filesystem or from_path_dict.get("filesystem", None)
if from_dir is None:
raise ValueError("Must specify from_dir!")
self.fileclient = FileClient(filesystem=filesystem)
self.from_dir = self.fileclient.abspath(from_dir)
self.to_dir = to_dir or os.getcwd()
exclude_files = exclude_files or []
self.files_to_copy = files_to_copy or [f for f in self.fileclient.listdir(self.from_dir) if f not in exclude_files]
def copy_files(self):
"""
Defines the copy operation. Override this to customize copying.
"""
for f in self.files_to_copy:
prev_path_full = os.path.join(self.from_dir, f)
dest_path = os.path.join(self.to_dir, f)
self.fileclient.copy(prev_path_full, dest_path)
def run_task(self, fw_spec):
self.setup_copy(self.get("from_dir", None), to_dir=self.get("to_dir", None),
filesystem=self.get("filesystem", None),
files_to_copy=self.get("files_to_copy", None),
exclude_files=self.get("exclude_files", []))
self.copy_files()
def run_task(self, fw_spec=None):
calc_loc = get_calc_loc(self['calc_loc'], fw_spec["calc_locs"])
calc_dir = calc_loc["path"]
filesystem = calc_loc["filesystem"]
fileclient = FileClient(filesystem=filesystem)
calc_dir = fileclient.abspath(calc_dir)
filenames = self.get('filenames')
if filenames is None:
files_to_copy = fileclient.listdir(calc_dir)
elif isinstance(filenames, str):
raise ValueError("filenames must be a list!")
elif '$ALL_NO_SUBDIRS' in filenames:
files_to_copy = fileclient.listdir(calc_dir)
elif '$ALL' in filenames:
if self.get('name_prepend') or self.get('name_append'):
raise ValueError(
'name_prepend or name_append options not compatible with "$ALL" option'
)
copy_r(calc_dir, os.getcwd())
return
else:
files_to_copy = filenames
for f in files_to_copy:
prev_path_full = os.path.join(calc_dir, f)
dest_fname = self.get('name_prepend', "") + f + self.get(
'name_append', "")
dest_path = os.path.join(os.getcwd(), dest_fname)
fileclient.copy(prev_path_full, dest_path)
def run_task(self,fw_spec=None):
calc_loc = get_calc_loc(self['calc_loc'], fw_spec["calc_locs"])
calc_dir = calc_loc["path"]
filesystem = calc_loc["filesystem"]
fileclient = FileClient(filesystem=filesystem)
calc_dir = fileclient.abspath(calc_dir)
filenames = self.get('filenames')
if filenames is None:
files_to_copy = fileclient.listdir(calc_dir)
elif isinstance(filenames, six.string_types):
raise ValueError("filenames must be a list!")
elif '$ALL_NO_SUBDIRS' in filenames:
files_to_copy = fileclient.listdir(calc_dir)
elif '$ALL' in filenames:
if self.get('name_prepend') or self.get('name_append'):
raise ValueError('name_prepend or name_append options not compatible with "$ALL" option')
copy_r(calc_dir, os.getcwd())
return
else:
files_to_copy = filenames
for f in files_to_copy:
prev_path_full = os.path.join(calc_dir, f)
dest_fname = self.get('name_prepend', "") + f + self.get(
'name_append', "")
dest_path = os.path.join(os.getcwd(), dest_fname)
fileclient.copy(prev_path_full, dest_path)
def run_task(self, fw_spec=None):
calc_loc = get_calc_loc(self["calc_loc"], fw_spec["calc_locs"])
calc_dir = calc_loc["path"]
filesystem = calc_loc["filesystem"]
fileclient = FileClient(filesystem=filesystem)
calc_dir = fileclient.abspath(calc_dir)
filenames = self.get("filenames")
exclude_files = self.get("exclude_files", [])
if filenames is None:
files_to_copy = fileclient.listdir(calc_dir)
elif isinstance(filenames, str):
raise ValueError("filenames must be a list!")
elif "$ALL_NO_SUBDIRS" in filenames:
files_to_copy = fileclient.listdir(calc_dir)
elif "$ALL" in filenames:
if (
self.get("name_prepend")
or self.get("name_append")
or self.get("exclude_files")
):
raise ValueError(
'name_prepend, name_append, and exclude_files \
options not compatible with "$ALL" option'
)
copy_r(calc_dir, os.getcwd())
return
else:
files_to_copy = []
for fname in filenames:
for f in glob.glob(os.path.join(calc_dir, fname)):
files_to_copy.append(os.path.basename(f))
# delete any excluded files
for fname in exclude_files:
for f in glob.glob(os.path.join(calc_dir, fname)):
if os.path.basename(f) in files_to_copy:
files_to_copy.remove(os.path.basename(f))
for f in files_to_copy:
prev_path_full = os.path.join(calc_dir, f)
dest_fname = self.get("name_prepend", "") + f + self.get("name_append", "")
dest_path = os.path.join(os.getcwd(), dest_fname)
fileclient.copy(prev_path_full, dest_path)
def setup_copy(self, from_dir, to_dir=None, filesystem=None, files_to_copy=None, exclude_files=None,
from_path_dict=None):
"""
setup the copy i.e setup the from directory, filesystem, destination directory etc.
Args:
from_dir (str)
to_dir (str)
filesystem (str)
files_to_copy (list): if None all the files in the from_dir will be copied
exclude_files (list)
from_path_dict (dict): dict specification of the path. If specified must contain atleast
the key "path" that specifies the path to the from_dir.
"""
from_path_dict = from_path_dict or {}
from_dir = from_dir or from_path_dict.get("path", None)
filesystem = filesystem or from_path_dict.get("filesystem", None)
if from_dir is None:
raise ValueError("Must specify from_dir!")
self.fileclient = FileClient(filesystem=filesystem)
self.from_dir = self.fileclient.abspath(from_dir)
self.to_dir = to_dir or os.getcwd()
exclude_files = exclude_files or []
self.files_to_copy = files_to_copy or [f for f in self.fileclient.listdir(self.from_dir) if f not in exclude_files]
def run_task(self,fw_spec=None):
calc_loc = get_calc_loc(self['calc_loc'], fw_spec["calc_locs"])
calc_dir = calc_loc["path"]
filesystem = calc_loc["filesystem"]
fileclient = FileClient(filesystem=filesystem)
calc_dir = fileclient.abspath(calc_dir)
if self.get('filenames'):
if isinstance(self["filenames"], six.string_types):
raise ValueError("filenames must be a list!")
files_to_copy = self['filenames']
else:
files_to_copy = fileclient.listdir(calc_dir)
for f in files_to_copy:
prev_path_full = os.path.join(calc_dir, f)
dest_fname = self.get('name_prepend', "") + f + self.get(
'name_append', "")
dest_path = os.path.join(os.getcwd(), dest_fname)
fileclient.copy(prev_path_full, dest_path)
def run_task(self, fw_spec):
if self.get("calc_dir"): # direct setting of calc dir - no calc_locs or filesystem!
calc_dir = self["calc_dir"]
filesystem = None
elif self.get("calc_loc"): # search for calc dir and filesystem within calc_locs
calc_loc = get_calc_loc(self["calc_loc"], fw_spec["calc_locs"])
calc_dir = calc_loc["path"]
filesystem = calc_loc["filesystem"]
else:
raise ValueError("Must specify either calc_dir or calc_loc!")
fileclient = FileClient(filesystem=filesystem)
calc_dir = fileclient.abspath(calc_dir)
contcar_to_poscar = self.get("contcar_to_poscar", True)
all_files = fileclient.listdir(calc_dir)
# determine what files need to be copied
if "$ALL" in self.get("additional_files", []):
files_to_copy = all_files
else:
files_to_copy = ['INCAR', 'POSCAR', 'KPOINTS', 'POTCAR', 'OUTCAR', 'vasprun.xml']
if self.get("additional_files"):
files_to_copy.extend(self["additional_files"])
if contcar_to_poscar and "CONTCAR" not in files_to_copy:
files_to_copy.append("CONTCAR")
files_to_copy = [f for f in files_to_copy if f != 'POSCAR'] # remove POSCAR
# start file copy
for f in files_to_copy:
prev_path_full = os.path.join(calc_dir, f)
# prev_path = os.path.join(os.path.split(calc_dir)[1], f)
dest_fname = 'POSCAR' if f == 'CONTCAR' and contcar_to_poscar else f
dest_path = os.path.join(os.getcwd(), dest_fname)
relax_ext = ""
relax_paths = sorted(fileclient.glob(prev_path_full+".relax*"), reverse=True)
if relax_paths:
if len(relax_paths) > 9:
raise ValueError("CopyVaspOutputs doesn't properly handle >9 relaxations!")
m = re.search('\.relax\d*', relax_paths[0])
relax_ext = m.group(0)
# detect .gz extension if needed - note that monty zpath() did not seem useful here
gz_ext = ""
if not (f + relax_ext) in all_files:
for possible_ext in [".gz", ".GZ"]:
if (f + relax_ext + possible_ext) in all_files:
gz_ext = possible_ext
if not (f + relax_ext + gz_ext) in all_files:
raise ValueError("Cannot find file: {}".format(f))
# copy the file (minus the relaxation extension)
fileclient.copy(prev_path_full + relax_ext + gz_ext, dest_path + gz_ext)
# unzip the .gz if needed
if gz_ext in ['.gz', ".GZ"]:
# unzip dest file
f = gzip.open(dest_path + gz_ext, 'rt')
file_content = f.read()
with open(dest_path, 'w') as f_out:
f_out.writelines(file_content)
f.close()
os.remove(dest_path + gz_ext)
class CopyFiles(FiretaskBase):
"""
Task to copy the given list of files from the given directory to the destination directory.
To customize override the setup_copy and copy_files methods.
Optional params:
from_dir (str): path to the directory containing the files to be copied. Supports env_chk.
to_dir (str): path to the destination directory. Supports env_chk.
filesystem (str)
files_to_copy (list): list of file names. Defaults to copying everything in from_dir.
exclude_files (list): list of file names to be excluded.
suffix (str): suffix to append to each filename when copying
(e.g., rename 'INCAR' to 'INCAR.precondition')
continue_on_missing(bool): Whether to continue copying when a file
in filenames is missing. Defaults to False.
"""
optional_params = [
"from_dir",
"to_dir",
"filesystem",
"files_to_copy",
"exclude_files",
"suffix",
"continue_on_missing",
]
def setup_copy(
self,
from_dir,
to_dir=None,
filesystem=None,
files_to_copy=None,
exclude_files=None,
from_path_dict=None,
suffix=None,
fw_spec=None,
continue_on_missing=False,
):
"""
setup the copy i.e setup the from directory, filesystem, destination directory etc.
Args:
from_dir (str)
to_dir (str)
filesystem (str)
files_to_copy (list): if None all the files in the from_dir will be copied
exclude_files (list): list of file names to be excluded.
suffix (str): suffix to append to each filename when copying
(e.g., rename 'INCAR' to 'INCAR.precondition')
continue_on_missing(bool): Whether to continue copying when a file
in filenames is missing. Defaults to False.
from_path_dict (dict): dict specification of the path. If specified must contain atleast
the key "path" that specifies the path to the from_dir.
"""
from_path_dict = from_path_dict or {}
from_dir = env_chk(from_dir, fw_spec, strict=False) or from_path_dict.get(
"path", None
)
filesystem = filesystem or from_path_dict.get("filesystem", None)
if from_dir is None:
raise ValueError("Must specify from_dir!")
self.fileclient = FileClient(filesystem=filesystem)
self.from_dir = self.fileclient.abspath(from_dir)
self.to_dir = env_chk(to_dir, fw_spec, strict=False) or os.getcwd()
exclude_files = exclude_files or []
self.files_to_copy = files_to_copy or [
f for f in self.fileclient.listdir(self.from_dir) if f not in exclude_files
]
self.suffix = suffix
self.continue_on_missing = continue_on_missing
def copy_files(self):
"""
Defines the copy operation. Override this to customize copying.
"""
for f in self.files_to_copy:
prev_path_full = os.path.join(self.from_dir, f)
if self.suffix:
dest_path = os.path.join(self.to_dir, f, self.suffix)
else:
dest_path = os.path.join(self.to_dir, f)
try:
self.fileclient.copy(prev_path_full, dest_path)
except FileNotFoundError as exc:
if continue_on_missing:
continue
else:
raise exc
def run_task(self, fw_spec):
self.setup_copy(
self.get("from_dir", None),
to_dir=self.get("to_dir", None),
filesystem=self.get("filesystem", None),
files_to_copy=self.get("files_to_copy", None),
exclude_files=self.get("exclude_files", []),
suffix=self.get("suffix", None),
fw_spec=fw_spec,
)
self.copy_files()