def setup_copy(self, from_dir, to_dir=None, filesystem=None, files_to_copy=None, exclude_files=None, from_path_dict=None): """ setup the copy i.e setup the from directory, filesystem, destination directory etc. Args: from_dir (str) to_dir (str) filesystem (str) files_to_copy (list): if None all the files in the from_dir will be copied exclude_files (list) from_path_dict (dict): dict specification of the path. If specified must contain atleast the key "path" that specifies the path to the from_dir. """ from_path_dict = from_path_dict or {} from_dir = from_dir or from_path_dict.get("path", None) filesystem = filesystem or from_path_dict.get("filesystem", None) if from_dir is None: raise ValueError("Must specify from_dir!") self.fileclient = FileClient(filesystem=filesystem) self.from_dir = self.fileclient.abspath(from_dir) self.to_dir = to_dir or os.getcwd() exclude_files = exclude_files or [] self.files_to_copy = files_to_copy or [ f for f in self.fileclient.listdir(self.from_dir) if f not in exclude_files ]
def setup_copy(self, from_dir, to_dir=None, filesystem=None, files_to_copy=None, exclude_files=None, from_path_dict=None, suffix=None, fw_spec=None, continue_on_missing=False): """ setup the copy i.e setup the from directory, filesystem, destination directory etc. Args: from_dir (str) to_dir (str) filesystem (str) files_to_copy (list): if None all the files in the from_dir will be copied exclude_files (list): list of file names to be excluded. suffix (str): suffix to append to each filename when copying (e.g., rename 'INCAR' to 'INCAR.precondition') continue_on_missing(bool): Whether to continue copying when a file in filenames is missing. Defaults to False. from_path_dict (dict): dict specification of the path. If specified must contain atleast the key "path" that specifies the path to the from_dir. """ from_path_dict = from_path_dict or {} from_dir = env_chk(from_dir, fw_spec, strict=False) or from_path_dict.get("path", None) filesystem = filesystem or from_path_dict.get("filesystem", None) if from_dir is None: raise ValueError("Must specify from_dir!") self.fileclient = FileClient(filesystem=filesystem) self.from_dir = self.fileclient.abspath(from_dir) self.to_dir = env_chk(to_dir, fw_spec, strict=False) or os.getcwd() exclude_files = exclude_files or [] self.files_to_copy = files_to_copy or [f for f in self.fileclient.listdir(self.from_dir) if f not in exclude_files] self.suffix = suffix self.continue_on_missing = continue_on_missing
class CopyFiles(FiretaskBase): """ Task to copy the given list of files from the given directory to the destination directory. To customize override the setup_copy and copy_files methods. Optional params: from_dir (str): path to the directory containing the files to be copied. to_dir (str): path to the destination directory filesystem (str) files_to_copy (list): list of file names. exclude_files (list): list of file names to be excluded. """ optional_params = ["from_dir", "to_dir", "filesystem", "files_to_copy", "exclude_files"] def setup_copy(self, from_dir, to_dir=None, filesystem=None, files_to_copy=None, exclude_files=None, from_path_dict=None): """ setup the copy i.e setup the from directory, filesystem, destination directory etc. Args: from_dir (str) to_dir (str) filesystem (str) files_to_copy (list): if None all the files in the from_dir will be copied exclude_files (list) from_path_dict (dict): dict specification of the path. If specified must contain atleast the key "path" that specifies the path to the from_dir. """ from_path_dict = from_path_dict or {} from_dir = from_dir or from_path_dict.get("path", None) filesystem = filesystem or from_path_dict.get("filesystem", None) if from_dir is None: raise ValueError("Must specify from_dir!") self.fileclient = FileClient(filesystem=filesystem) self.from_dir = self.fileclient.abspath(from_dir) self.to_dir = to_dir or os.getcwd() exclude_files = exclude_files or [] self.files_to_copy = files_to_copy or [f for f in self.fileclient.listdir(self.from_dir) if f not in exclude_files] def copy_files(self): """ Defines the copy operation. Override this to customize copying. """ for f in self.files_to_copy: prev_path_full = os.path.join(self.from_dir, f) dest_path = os.path.join(self.to_dir, f) self.fileclient.copy(prev_path_full, dest_path) def run_task(self, fw_spec): self.setup_copy(self.get("from_dir", None), to_dir=self.get("to_dir", None), filesystem=self.get("filesystem", None), files_to_copy=self.get("files_to_copy", None), exclude_files=self.get("exclude_files", [])) self.copy_files()
class CopyFiles(FiretaskBase): """ Task to copy the given list of files from the given directory to the destination directory. To customize override the setup_copy and copy_files methods. Optional params: from_dir (str): path to the directory containing the files to be copied. to_dir (str): path to the destination directory filesystem (str) files_to_copy (list): list of file names. exclude_files (list): list of file names to be excluded. """ optional_params = ["from_dir", "to_dir", "filesystem", "files_to_copy", "exclude_files"] def setup_copy(self, from_dir, to_dir=None, filesystem=None, files_to_copy=None, exclude_files=None, from_path_dict=None): """ setup the copy i.e setup the from directory, filesystem, destination directory etc. Args: from_dir (str) to_dir (str) filesystem (str) files_to_copy (list): if None all the files in the from_dir will be copied exclude_files (list) from_path_dict (dict): dict specification of the path. If specified must contain atleast the key "path" that specifies the path to the from_dir. """ from_path_dict = from_path_dict or {} from_dir = from_dir or from_path_dict.get("path", None) filesystem = filesystem or from_path_dict.get("filesystem", None) if from_dir is None: raise ValueError("Must specify from_dir!") self.fileclient = FileClient(filesystem=filesystem) self.from_dir = self.fileclient.abspath(from_dir) self.to_dir = to_dir or os.getcwd() exclude_files = exclude_files or [] self.files_to_copy = files_to_copy or [f for f in self.fileclient.listdir(self.from_dir) if f not in exclude_files] def copy_files(self): """ Defines the copy operation. Override this to customize copying. """ for f in self.files_to_copy: prev_path_full = os.path.join(self.from_dir, f) dest_path = os.path.join(self.to_dir, f) self.fileclient.copy(prev_path_full, dest_path) def run_task(self, fw_spec): self.setup_copy(self.get("from_dir", None), to_dir=self.get("to_dir", None), filesystem=self.get("filesystem", None), files_to_copy=self.get("files_to_copy", None), exclude_files=self.get("exclude_files", [])) self.copy_files()
def run_task(self, fw_spec=None): calc_loc = get_calc_loc(self['calc_loc'], fw_spec["calc_locs"]) calc_dir = calc_loc["path"] filesystem = calc_loc["filesystem"] fileclient = FileClient(filesystem=filesystem) calc_dir = fileclient.abspath(calc_dir) filenames = self.get('filenames') if filenames is None: files_to_copy = fileclient.listdir(calc_dir) elif isinstance(filenames, str): raise ValueError("filenames must be a list!") elif '$ALL_NO_SUBDIRS' in filenames: files_to_copy = fileclient.listdir(calc_dir) elif '$ALL' in filenames: if self.get('name_prepend') or self.get('name_append'): raise ValueError( 'name_prepend or name_append options not compatible with "$ALL" option' ) copy_r(calc_dir, os.getcwd()) return else: files_to_copy = filenames for f in files_to_copy: prev_path_full = os.path.join(calc_dir, f) dest_fname = self.get('name_prepend', "") + f + self.get( 'name_append', "") dest_path = os.path.join(os.getcwd(), dest_fname) fileclient.copy(prev_path_full, dest_path)
def run_task(self,fw_spec=None): calc_loc = get_calc_loc(self['calc_loc'], fw_spec["calc_locs"]) calc_dir = calc_loc["path"] filesystem = calc_loc["filesystem"] fileclient = FileClient(filesystem=filesystem) calc_dir = fileclient.abspath(calc_dir) filenames = self.get('filenames') if filenames is None: files_to_copy = fileclient.listdir(calc_dir) elif isinstance(filenames, six.string_types): raise ValueError("filenames must be a list!") elif '$ALL_NO_SUBDIRS' in filenames: files_to_copy = fileclient.listdir(calc_dir) elif '$ALL' in filenames: if self.get('name_prepend') or self.get('name_append'): raise ValueError('name_prepend or name_append options not compatible with "$ALL" option') copy_r(calc_dir, os.getcwd()) return else: files_to_copy = filenames for f in files_to_copy: prev_path_full = os.path.join(calc_dir, f) dest_fname = self.get('name_prepend', "") + f + self.get( 'name_append', "") dest_path = os.path.join(os.getcwd(), dest_fname) fileclient.copy(prev_path_full, dest_path)
def run_task(self, fw_spec=None): calc_loc = get_calc_loc(self["calc_loc"], fw_spec["calc_locs"]) calc_dir = calc_loc["path"] filesystem = calc_loc["filesystem"] fileclient = FileClient(filesystem=filesystem) calc_dir = fileclient.abspath(calc_dir) filenames = self.get("filenames") exclude_files = self.get("exclude_files", []) if filenames is None: files_to_copy = fileclient.listdir(calc_dir) elif isinstance(filenames, str): raise ValueError("filenames must be a list!") elif "$ALL_NO_SUBDIRS" in filenames: files_to_copy = fileclient.listdir(calc_dir) elif "$ALL" in filenames: if ( self.get("name_prepend") or self.get("name_append") or self.get("exclude_files") ): raise ValueError( 'name_prepend, name_append, and exclude_files \ options not compatible with "$ALL" option' ) copy_r(calc_dir, os.getcwd()) return else: files_to_copy = [] for fname in filenames: for f in glob.glob(os.path.join(calc_dir, fname)): files_to_copy.append(os.path.basename(f)) # delete any excluded files for fname in exclude_files: for f in glob.glob(os.path.join(calc_dir, fname)): if os.path.basename(f) in files_to_copy: files_to_copy.remove(os.path.basename(f)) for f in files_to_copy: prev_path_full = os.path.join(calc_dir, f) dest_fname = self.get("name_prepend", "") + f + self.get("name_append", "") dest_path = os.path.join(os.getcwd(), dest_fname) fileclient.copy(prev_path_full, dest_path)
def setup_copy(self, from_dir, to_dir=None, filesystem=None, files_to_copy=None, exclude_files=None, from_path_dict=None): """ setup the copy i.e setup the from directory, filesystem, destination directory etc. Args: from_dir (str) to_dir (str) filesystem (str) files_to_copy (list): if None all the files in the from_dir will be copied exclude_files (list) from_path_dict (dict): dict specification of the path. If specified must contain atleast the key "path" that specifies the path to the from_dir. """ from_path_dict = from_path_dict or {} from_dir = from_dir or from_path_dict.get("path", None) filesystem = filesystem or from_path_dict.get("filesystem", None) if from_dir is None: raise ValueError("Must specify from_dir!") self.fileclient = FileClient(filesystem=filesystem) self.from_dir = self.fileclient.abspath(from_dir) self.to_dir = to_dir or os.getcwd() exclude_files = exclude_files or [] self.files_to_copy = files_to_copy or [f for f in self.fileclient.listdir(self.from_dir) if f not in exclude_files]
def run_task(self,fw_spec=None): calc_loc = get_calc_loc(self['calc_loc'], fw_spec["calc_locs"]) calc_dir = calc_loc["path"] filesystem = calc_loc["filesystem"] fileclient = FileClient(filesystem=filesystem) calc_dir = fileclient.abspath(calc_dir) if self.get('filenames'): if isinstance(self["filenames"], six.string_types): raise ValueError("filenames must be a list!") files_to_copy = self['filenames'] else: files_to_copy = fileclient.listdir(calc_dir) for f in files_to_copy: prev_path_full = os.path.join(calc_dir, f) dest_fname = self.get('name_prepend', "") + f + self.get( 'name_append', "") dest_path = os.path.join(os.getcwd(), dest_fname) fileclient.copy(prev_path_full, dest_path)
def run_task(self, fw_spec): if self.get("calc_dir"): # direct setting of calc dir - no calc_locs or filesystem! calc_dir = self["calc_dir"] filesystem = None elif self.get("calc_loc"): # search for calc dir and filesystem within calc_locs calc_loc = get_calc_loc(self["calc_loc"], fw_spec["calc_locs"]) calc_dir = calc_loc["path"] filesystem = calc_loc["filesystem"] else: raise ValueError("Must specify either calc_dir or calc_loc!") fileclient = FileClient(filesystem=filesystem) calc_dir = fileclient.abspath(calc_dir) contcar_to_poscar = self.get("contcar_to_poscar", True) all_files = fileclient.listdir(calc_dir) # determine what files need to be copied if "$ALL" in self.get("additional_files", []): files_to_copy = all_files else: files_to_copy = ['INCAR', 'POSCAR', 'KPOINTS', 'POTCAR', 'OUTCAR', 'vasprun.xml'] if self.get("additional_files"): files_to_copy.extend(self["additional_files"]) if contcar_to_poscar and "CONTCAR" not in files_to_copy: files_to_copy.append("CONTCAR") files_to_copy = [f for f in files_to_copy if f != 'POSCAR'] # remove POSCAR # start file copy for f in files_to_copy: prev_path_full = os.path.join(calc_dir, f) # prev_path = os.path.join(os.path.split(calc_dir)[1], f) dest_fname = 'POSCAR' if f == 'CONTCAR' and contcar_to_poscar else f dest_path = os.path.join(os.getcwd(), dest_fname) relax_ext = "" relax_paths = sorted(fileclient.glob(prev_path_full+".relax*"), reverse=True) if relax_paths: if len(relax_paths) > 9: raise ValueError("CopyVaspOutputs doesn't properly handle >9 relaxations!") m = re.search('\.relax\d*', relax_paths[0]) relax_ext = m.group(0) # detect .gz extension if needed - note that monty zpath() did not seem useful here gz_ext = "" if not (f + relax_ext) in all_files: for possible_ext in [".gz", ".GZ"]: if (f + relax_ext + possible_ext) in all_files: gz_ext = possible_ext if not (f + relax_ext + gz_ext) in all_files: raise ValueError("Cannot find file: {}".format(f)) # copy the file (minus the relaxation extension) fileclient.copy(prev_path_full + relax_ext + gz_ext, dest_path + gz_ext) # unzip the .gz if needed if gz_ext in ['.gz', ".GZ"]: # unzip dest file f = gzip.open(dest_path + gz_ext, 'rt') file_content = f.read() with open(dest_path, 'w') as f_out: f_out.writelines(file_content) f.close() os.remove(dest_path + gz_ext)
class CopyFiles(FiretaskBase): """ Task to copy the given list of files from the given directory to the destination directory. To customize override the setup_copy and copy_files methods. Optional params: from_dir (str): path to the directory containing the files to be copied. Supports env_chk. to_dir (str): path to the destination directory. Supports env_chk. filesystem (str) files_to_copy (list): list of file names. Defaults to copying everything in from_dir. exclude_files (list): list of file names to be excluded. suffix (str): suffix to append to each filename when copying (e.g., rename 'INCAR' to 'INCAR.precondition') continue_on_missing(bool): Whether to continue copying when a file in filenames is missing. Defaults to False. """ optional_params = [ "from_dir", "to_dir", "filesystem", "files_to_copy", "exclude_files", "suffix", "continue_on_missing", ] def setup_copy( self, from_dir, to_dir=None, filesystem=None, files_to_copy=None, exclude_files=None, from_path_dict=None, suffix=None, fw_spec=None, continue_on_missing=False, ): """ setup the copy i.e setup the from directory, filesystem, destination directory etc. Args: from_dir (str) to_dir (str) filesystem (str) files_to_copy (list): if None all the files in the from_dir will be copied exclude_files (list): list of file names to be excluded. suffix (str): suffix to append to each filename when copying (e.g., rename 'INCAR' to 'INCAR.precondition') continue_on_missing(bool): Whether to continue copying when a file in filenames is missing. Defaults to False. from_path_dict (dict): dict specification of the path. If specified must contain atleast the key "path" that specifies the path to the from_dir. """ from_path_dict = from_path_dict or {} from_dir = env_chk(from_dir, fw_spec, strict=False) or from_path_dict.get( "path", None ) filesystem = filesystem or from_path_dict.get("filesystem", None) if from_dir is None: raise ValueError("Must specify from_dir!") self.fileclient = FileClient(filesystem=filesystem) self.from_dir = self.fileclient.abspath(from_dir) self.to_dir = env_chk(to_dir, fw_spec, strict=False) or os.getcwd() exclude_files = exclude_files or [] self.files_to_copy = files_to_copy or [ f for f in self.fileclient.listdir(self.from_dir) if f not in exclude_files ] self.suffix = suffix self.continue_on_missing = continue_on_missing def copy_files(self): """ Defines the copy operation. Override this to customize copying. """ for f in self.files_to_copy: prev_path_full = os.path.join(self.from_dir, f) if self.suffix: dest_path = os.path.join(self.to_dir, f, self.suffix) else: dest_path = os.path.join(self.to_dir, f) try: self.fileclient.copy(prev_path_full, dest_path) except FileNotFoundError as exc: if continue_on_missing: continue else: raise exc def run_task(self, fw_spec): self.setup_copy( self.get("from_dir", None), to_dir=self.get("to_dir", None), filesystem=self.get("filesystem", None), files_to_copy=self.get("files_to_copy", None), exclude_files=self.get("exclude_files", []), suffix=self.get("suffix", None), fw_spec=fw_spec, ) self.copy_files()