def copy_to_output(self):
"""Use gcp for transfer, since OUTPUT_DIR might be mounted read-only.
Also has special logic to handle frepp_mode.
"""
if self.WK_DIR == self.OUT_DIR:
return # no copying needed
if self.frepp_mode:
# only copy PODs that ran, whether they succeeded or not
for pod in self.obj.iter_children():
if pod._has_placeholder:
gfdl_util.gcp_wrapper(pod.POD_WK_DIR,
pod.POD_OUT_DIR,
log=pod.log)
# copy all case-level files
self.obj.log.debug("Copying case-level files in %s", self.WK_DIR)
for f in os.listdir(self.WK_DIR):
if os.path.isfile(os.path.join(self.WK_DIR, f)):
self.obj.log.debug("Found case-level file %s", f)
gfdl_util.gcp_wrapper(os.path.join(self.WK_DIR, f),
self.OUT_DIR,
log=self.obj.log)
else:
# copy everything at once
if os.path.exists(self.OUT_DIR):
if self.overwrite:
try:
self.obj.log.error('%s exists, attempting to remove.',
self.OUT_DIR)
gfdl_util.rmtree_wrapper(self.OUT_DIR)
except OSError:
# gcp will not overwrite dirs, so forced to save under
# a different name despite overwrite=True
self.obj.log.error(
("Couldn't remove %s (probably mounted read"
"-only); will rename new directory."),
self.OUT_DIR)
else:
self.obj.log.error("%s exists; will rename new directory.",
self.OUT_DIR)
try:
if os.path.exists(self.OUT_DIR):
# check again, since rmtree() might have succeeded
self.OUT_DIR, version = \
util.bump_version(self.OUT_DIR)
new_wkdir, _ = \
util.bump_version(self.WK_DIR, new_v=version)
self.obj.log.debug("Move %s to %s", self.WK_DIR, new_wkdir)
shutil.move(self.WK_DIR, new_wkdir)
self.WK_DIR = new_wkdir
gfdl_util.gcp_wrapper(self.WK_DIR,
self.OUT_DIR,
log=self.obj.log)
except Exception:
raise # only delete MODEL_WK_DIR if copied successfully
self.obj.log.debug('Transfer succeeded; deleting directory %s',
self.WK_DIR)
gfdl_util.rmtree_wrapper(self.WK_DIR)
def backup_config_file(self, case):
"""Record settings in file config_save.json for rerunning.
"""
config = core.ConfigManager()
out_file = os.path.join(self.WK_DIR, self._backup_config_file_name)
if not self.file_overwrite:
out_file, _ = util.bump_version(out_file)
elif os.path.exists(out_file):
_log.info("%s: Overwriting %s.", case.name, out_file)
util.write_json(config.backup_config, out_file)
def backup_config_files(self):
"""Record settings in file config_save.json for rerunning.
"""
config = core.ConfigManager()
for config_tup in config._configs.values():
if config_tup.backup_filename is None:
continue
out_file = os.path.join(self.WK_DIR, config_tup.backup_filename)
if not self.file_overwrite:
out_file, _ = util.bump_version(out_file)
elif os.path.exists(out_file):
self.obj.log.info("%s: Overwriting '%s'.", self.obj.full_name,
out_file)
util.write_json(config_tup.contents, out_file, log=self.obj.log)
def make_tar_file(self, case):
"""Make tar file of web/bitmap output.
"""
out_path = self._tarball_file_path
if not self.file_overwrite:
out_path, _ = util.bump_version(out_path)
_log.info("%s: Creating %s.", case.name, out_path)
elif os.path.exists(out_path):
_log.info("%s: Overwriting %s.", case.name, out_path)
tar_flags = [
f"--exclude=.{s}" for s in ('netCDF', 'nc', 'ps', 'PS', 'eps')
]
tar_flags = ' '.join(tar_flags)
util.run_shell_command(
f'tar {tar_flags} -czf {out_path} -C {self.WK_DIR} .',
dry_run=self.dry_run)
return out_path
name = case['CASENAME']
yr1 = case['FIRSTYR']
yr2 = case['LASTYR']
else:
name = case.name
yr1 = case.attrs.date_range.start.format(precision=1)
yr2 = case.attrs.date_range.end.format(precision=1)
case_wk_dir = 'MDTF_{}_{}_{}'.format(name, yr1, yr2)
d.MODEL_DATA_DIR = os.path.join(self.MODEL_DATA_ROOT, name)
d.MODEL_WK_DIR = os.path.join(self.WORKING_DIR, case_wk_dir)
d.MODEL_OUT_DIR = os.path.join(self.OUTPUT_DIR, case_wk_dir)
if not overwrite:
# bump both WK_DIR and OUT_DIR to same version because name of
# former may be preserved when we copy to latter, depending on
# copy method
d.MODEL_WK_DIR, ver = util.bump_version(
d.MODEL_WK_DIR, extra_dirs=[self.OUTPUT_DIR])
d.MODEL_OUT_DIR, _ = util.bump_version(d.MODEL_OUT_DIR, new_v=ver)
return d
def pod_paths(self, pod, case):
d = util.NameSpace()
d.POD_CODE_DIR = os.path.join(self.CODE_ROOT, 'diagnostics', pod.name)
d.POD_OBS_DATA = os.path.join(self.OBS_DATA_ROOT, pod.name)
d.POD_WK_DIR = os.path.join(case.MODEL_WK_DIR, pod.name)
d.POD_OUT_DIR = os.path.join(case.MODEL_OUT_DIR, pod.name)
return d
class TempDirManager(util.Singleton):
_prefix = 'MDTF_temp_'
)
else:
# copy everything at once
if os.path.exists(self.OUT_DIR):
if self.overwrite:
try:
_log.error('%s exists, attempting to remove.', self.OUT_DIR)
gfdl_util.rmtree_wrapper(self.OUT_DIR)
except OSError:
# gcp will not overwrite dirs, so forced to save under
# a different name despite overwrite=True
_log.error(("Couldn't remove %s (probably mounted read"
"-only); will rename new directory."), self.OUT_DIR)
else:
_log.error("%s exists; will rename new directory.", self.OUT_DIR)
try:
if os.path.exists(self.OUT_DIR):
# check again, since rmtree() might have succeeded
self.OUT_DIR, version = \
util.bump_version(self.OUT_DIR)
new_wkdir, _ = \
util.bump_version(self.WK_DIR, new_v=version)
_log.debug("Move %s to %s", self.WK_DIR, new_wkdir)
shutil.move(self.WK_DIR, new_wkdir)
self.WK_DIR = new_wkdir
gfdl_util.gcp_wrapper(self.WK_DIR, self.OUT_DIR)
except Exception:
raise # only delete MODEL_WK_DIR if copied successfully
_log.debug('Transfer succeeded; deleting directory %s', self.WK_DIR)
gfdl_util.rmtree_wrapper(self.WK_DIR)
