def copy_to_output(self): """Use gcp for transfer, since OUTPUT_DIR might be mounted read-only. Also has special logic to handle frepp_mode. """ if self.WK_DIR == self.OUT_DIR: return # no copying needed if self.frepp_mode: # only copy PODs that ran, whether they succeeded or not for pod in self.obj.iter_children(): if pod._has_placeholder: gfdl_util.gcp_wrapper(pod.POD_WK_DIR, pod.POD_OUT_DIR, log=pod.log) # copy all case-level files self.obj.log.debug("Copying case-level files in %s", self.WK_DIR) for f in os.listdir(self.WK_DIR): if os.path.isfile(os.path.join(self.WK_DIR, f)): self.obj.log.debug("Found case-level file %s", f) gfdl_util.gcp_wrapper(os.path.join(self.WK_DIR, f), self.OUT_DIR, log=self.obj.log) else: # copy everything at once if os.path.exists(self.OUT_DIR): if self.overwrite: try: self.obj.log.error('%s exists, attempting to remove.', self.OUT_DIR) gfdl_util.rmtree_wrapper(self.OUT_DIR) except OSError: # gcp will not overwrite dirs, so forced to save under # a different name despite overwrite=True self.obj.log.error( ("Couldn't remove %s (probably mounted read" "-only); will rename new directory."), self.OUT_DIR) else: self.obj.log.error("%s exists; will rename new directory.", self.OUT_DIR) try: if os.path.exists(self.OUT_DIR): # check again, since rmtree() might have succeeded self.OUT_DIR, version = \ util.bump_version(self.OUT_DIR) new_wkdir, _ = \ util.bump_version(self.WK_DIR, new_v=version) self.obj.log.debug("Move %s to %s", self.WK_DIR, new_wkdir) shutil.move(self.WK_DIR, new_wkdir) self.WK_DIR = new_wkdir gfdl_util.gcp_wrapper(self.WK_DIR, self.OUT_DIR, log=self.obj.log) except Exception: raise # only delete MODEL_WK_DIR if copied successfully self.obj.log.debug('Transfer succeeded; deleting directory %s', self.WK_DIR) gfdl_util.rmtree_wrapper(self.WK_DIR)
def backup_config_file(self, case): """Record settings in file config_save.json for rerunning. """ config = core.ConfigManager() out_file = os.path.join(self.WK_DIR, self._backup_config_file_name) if not self.file_overwrite: out_file, _ = util.bump_version(out_file) elif os.path.exists(out_file): _log.info("%s: Overwriting %s.", case.name, out_file) util.write_json(config.backup_config, out_file)
def backup_config_files(self): """Record settings in file config_save.json for rerunning. """ config = core.ConfigManager() for config_tup in config._configs.values(): if config_tup.backup_filename is None: continue out_file = os.path.join(self.WK_DIR, config_tup.backup_filename) if not self.file_overwrite: out_file, _ = util.bump_version(out_file) elif os.path.exists(out_file): self.obj.log.info("%s: Overwriting '%s'.", self.obj.full_name, out_file) util.write_json(config_tup.contents, out_file, log=self.obj.log)
def make_tar_file(self, case): """Make tar file of web/bitmap output. """ out_path = self._tarball_file_path if not self.file_overwrite: out_path, _ = util.bump_version(out_path) _log.info("%s: Creating %s.", case.name, out_path) elif os.path.exists(out_path): _log.info("%s: Overwriting %s.", case.name, out_path) tar_flags = [ f"--exclude=.{s}" for s in ('netCDF', 'nc', 'ps', 'PS', 'eps') ] tar_flags = ' '.join(tar_flags) util.run_shell_command( f'tar {tar_flags} -czf {out_path} -C {self.WK_DIR} .', dry_run=self.dry_run) return out_path
name = case['CASENAME'] yr1 = case['FIRSTYR'] yr2 = case['LASTYR'] else: name = case.name yr1 = case.attrs.date_range.start.format(precision=1) yr2 = case.attrs.date_range.end.format(precision=1) case_wk_dir = 'MDTF_{}_{}_{}'.format(name, yr1, yr2) d.MODEL_DATA_DIR = os.path.join(self.MODEL_DATA_ROOT, name) d.MODEL_WK_DIR = os.path.join(self.WORKING_DIR, case_wk_dir) d.MODEL_OUT_DIR = os.path.join(self.OUTPUT_DIR, case_wk_dir) if not overwrite: # bump both WK_DIR and OUT_DIR to same version because name of # former may be preserved when we copy to latter, depending on # copy method d.MODEL_WK_DIR, ver = util.bump_version( d.MODEL_WK_DIR, extra_dirs=[self.OUTPUT_DIR]) d.MODEL_OUT_DIR, _ = util.bump_version(d.MODEL_OUT_DIR, new_v=ver) return d def pod_paths(self, pod, case): d = util.NameSpace() d.POD_CODE_DIR = os.path.join(self.CODE_ROOT, 'diagnostics', pod.name) d.POD_OBS_DATA = os.path.join(self.OBS_DATA_ROOT, pod.name) d.POD_WK_DIR = os.path.join(case.MODEL_WK_DIR, pod.name) d.POD_OUT_DIR = os.path.join(case.MODEL_OUT_DIR, pod.name) return d class TempDirManager(util.Singleton): _prefix = 'MDTF_temp_'
) else: # copy everything at once if os.path.exists(self.OUT_DIR): if self.overwrite: try: _log.error('%s exists, attempting to remove.', self.OUT_DIR) gfdl_util.rmtree_wrapper(self.OUT_DIR) except OSError: # gcp will not overwrite dirs, so forced to save under # a different name despite overwrite=True _log.error(("Couldn't remove %s (probably mounted read" "-only); will rename new directory."), self.OUT_DIR) else: _log.error("%s exists; will rename new directory.", self.OUT_DIR) try: if os.path.exists(self.OUT_DIR): # check again, since rmtree() might have succeeded self.OUT_DIR, version = \ util.bump_version(self.OUT_DIR) new_wkdir, _ = \ util.bump_version(self.WK_DIR, new_v=version) _log.debug("Move %s to %s", self.WK_DIR, new_wkdir) shutil.move(self.WK_DIR, new_wkdir) self.WK_DIR = new_wkdir gfdl_util.gcp_wrapper(self.WK_DIR, self.OUT_DIR) except Exception: raise # only delete MODEL_WK_DIR if copied successfully _log.debug('Transfer succeeded; deleting directory %s', self.WK_DIR) gfdl_util.rmtree_wrapper(self.WK_DIR)