def verify_paths(self, config, p):
keep_temp = config.get('keep_temp', False)
# clean out WORKING_DIR if we're not keeping temp files:
if os.path.exists(p.WORKING_DIR) and not \
(keep_temp or p.WORKING_DIR == p.OUTPUT_DIR):
gfdl_util.rmtree_wrapper(p.WORKING_DIR)
try:
for dir_name, create_ in (('CODE_ROOT', False), ('OBS_DATA_REMOTE',
False),
('OBS_DATA_ROOT',
True), ('MODEL_DATA_ROOT',
True), ('WORKING_DIR', True)):
util.check_dir(p, dir_name, create=create_)
except Exception as exc:
_log.fatal((f"Input settings for {dir_name} mis-specified (caught "
f"{repr(exc)}.)"))
util.exit_handler(code=1)
# Use GCP to create OUTPUT_DIR on a volume that may be read-only
if not os.path.exists(p.OUTPUT_DIR):
gfdl_util.make_remote_dir(p.OUTPUT_DIR,
self.timeout,
self.dry_run,
log=_log)
def copy_to_output(self):
"""Use gcp for transfer, since OUTPUT_DIR might be mounted read-only.
Also has special logic to handle frepp_mode.
"""
if self.WK_DIR == self.OUT_DIR:
return # no copying needed
if self.frepp_mode:
# only copy PODs that ran, whether they succeeded or not
for pod in self.obj.iter_children():
if pod._has_placeholder:
gfdl_util.gcp_wrapper(pod.POD_WK_DIR,
pod.POD_OUT_DIR,
log=pod.log)
# copy all case-level files
self.obj.log.debug("Copying case-level files in %s", self.WK_DIR)
for f in os.listdir(self.WK_DIR):
if os.path.isfile(os.path.join(self.WK_DIR, f)):
self.obj.log.debug("Found case-level file %s", f)
gfdl_util.gcp_wrapper(os.path.join(self.WK_DIR, f),
self.OUT_DIR,
log=self.obj.log)
else:
# copy everything at once
if os.path.exists(self.OUT_DIR):
if self.overwrite:
try:
self.obj.log.error('%s exists, attempting to remove.',
self.OUT_DIR)
gfdl_util.rmtree_wrapper(self.OUT_DIR)
except OSError:
# gcp will not overwrite dirs, so forced to save under
# a different name despite overwrite=True
self.obj.log.error(
("Couldn't remove %s (probably mounted read"
"-only); will rename new directory."),
self.OUT_DIR)
else:
self.obj.log.error("%s exists; will rename new directory.",
self.OUT_DIR)
try:
if os.path.exists(self.OUT_DIR):
# check again, since rmtree() might have succeeded
self.OUT_DIR, version = \
util.bump_version(self.OUT_DIR)
new_wkdir, _ = \
util.bump_version(self.WK_DIR, new_v=version)
self.obj.log.debug("Move %s to %s", self.WK_DIR, new_wkdir)
shutil.move(self.WK_DIR, new_wkdir)
self.WK_DIR = new_wkdir
gfdl_util.gcp_wrapper(self.WK_DIR,
self.OUT_DIR,
log=self.obj.log)
except Exception:
raise # only delete MODEL_WK_DIR if copied successfully
self.obj.log.debug('Transfer succeeded; deleting directory %s',
self.WK_DIR)
gfdl_util.rmtree_wrapper(self.WK_DIR)
def verify_paths(self, config, p):
keep_temp = config.get('keep_temp', False)
# clean out WORKING_DIR if we're not keeping temp files:
if os.path.exists(p.WORKING_DIR) and not \
(keep_temp or p.WORKING_DIR == p.OUTPUT_DIR):
gfdl_util.rmtree_wrapper(p.WORKING_DIR)
util.check_dirs(p.CODE_ROOT, p.OBS_DATA_REMOTE, create=False)
util.check_dirs(p.MODEL_DATA_ROOT, p.OBS_DATA_ROOT, p.WORKING_DIR,
create=True)
# Use GCP to create OUTPUT_DIR on a volume that may be read-only
if not os.path.exists(p.OUTPUT_DIR):
gfdl_util.make_remote_dir(p.OUTPUT_DIR, self.timeout, self.dry_run)
gfdl_util.gcp_wrapper(pod.POD_WK_DIR, pod.POD_OUT_DIR)
# copy all case-level files
_log.debug("Copying case-level files in %s", self.WK_DIR)
for f in os.listdir(self.WK_DIR):
if os.path.isfile(os.path.join(self.WK_DIR, f)):
_log.debug("Found case-level file %s", f)
gfdl_util.gcp_wrapper(
os.path.join(self.WK_DIR, f), self.OUT_DIR,
)
else:
# copy everything at once
if os.path.exists(self.OUT_DIR):
if self.overwrite:
try:
_log.error('%s exists, attempting to remove.', self.OUT_DIR)
gfdl_util.rmtree_wrapper(self.OUT_DIR)
except OSError:
# gcp will not overwrite dirs, so forced to save under
# a different name despite overwrite=True
_log.error(("Couldn't remove %s (probably mounted read"
"-only); will rename new directory."), self.OUT_DIR)
else:
_log.error("%s exists; will rename new directory.", self.OUT_DIR)
try:
if os.path.exists(self.OUT_DIR):
# check again, since rmtree() might have succeeded
self.OUT_DIR, version = \
util.bump_version(self.OUT_DIR)
new_wkdir, _ = \
util.bump_version(self.WK_DIR, new_v=version)
_log.debug("Move %s to %s", self.WK_DIR, new_wkdir)