def make_tar_file(self):
"""Make the tar file locally in WK_DIR and gcp to destination,
since OUT_DIR might be mounted read-only.
"""
paths = core.PathManager()
out_path = super(GFDLHTMLOutputManager, self).make_tar_file()
_, file_name = os.path.split(out_path)
tar_dest_path = os.path.join(paths.OUTPUT_DIR, file_name)
gfdl_util.gcp_wrapper(out_path, tar_dest_path, log=self.obj.log)
return tar_dest_path
def copy_to_output(self, case):
"""Use gcp for transfer, since OUTPUT_DIR might be mounted read-only.
Also has special logic to handle frepp_mode.
"""
if self.WK_DIR == self.OUT_DIR:
return # no copying needed
if self.frepp_mode:
# only copy PODs that ran, whether they succeeded or not
for pod in case.pods.values():
if pod._has_placeholder:
gfdl_util.gcp_wrapper(pod.POD_WK_DIR, pod.POD_OUT_DIR)
# copy all case-level files
_log.debug("Copying case-level files in %s", self.WK_DIR)
for f in os.listdir(self.WK_DIR):
if os.path.isfile(os.path.join(self.WK_DIR, f)):
_log.debug("Found case-level file %s", f)
gfdl_util.gcp_wrapper(
os.path.join(self.WK_DIR, f), self.OUT_DIR,
)
def copy_to_output(self):
"""Use gcp for transfer, since OUTPUT_DIR might be mounted read-only.
Also has special logic to handle frepp_mode.
"""
if self.WK_DIR == self.OUT_DIR:
return # no copying needed
if self.frepp_mode:
# only copy PODs that ran, whether they succeeded or not
for pod in self.obj.iter_children():
if pod._has_placeholder:
gfdl_util.gcp_wrapper(pod.POD_WK_DIR,
pod.POD_OUT_DIR,
log=pod.log)
# copy all case-level files
self.obj.log.debug("Copying case-level files in %s", self.WK_DIR)
for f in os.listdir(self.WK_DIR):
if os.path.isfile(os.path.join(self.WK_DIR, f)):
self.obj.log.debug("Found case-level file %s", f)
gfdl_util.gcp_wrapper(os.path.join(self.WK_DIR, f),
self.OUT_DIR,
log=self.obj.log)
else:
# copy everything at once
if os.path.exists(self.OUT_DIR):
if self.overwrite:
try:
self.obj.log.error('%s exists, attempting to remove.',
self.OUT_DIR)
gfdl_util.rmtree_wrapper(self.OUT_DIR)
except OSError:
# gcp will not overwrite dirs, so forced to save under
# a different name despite overwrite=True
self.obj.log.error(
("Couldn't remove %s (probably mounted read"
"-only); will rename new directory."),
self.OUT_DIR)
else:
self.obj.log.error("%s exists; will rename new directory.",
self.OUT_DIR)
try:
if os.path.exists(self.OUT_DIR):
# check again, since rmtree() might have succeeded
self.OUT_DIR, version = \
util.bump_version(self.OUT_DIR)
new_wkdir, _ = \
util.bump_version(self.WK_DIR, new_v=version)
self.obj.log.debug("Move %s to %s", self.WK_DIR, new_wkdir)
shutil.move(self.WK_DIR, new_wkdir)
self.WK_DIR = new_wkdir
gfdl_util.gcp_wrapper(self.WK_DIR,
self.OUT_DIR,
log=self.obj.log)
except Exception:
raise # only delete MODEL_WK_DIR if copied successfully
self.obj.log.debug('Transfer succeeded; deleting directory %s',
self.WK_DIR)
gfdl_util.rmtree_wrapper(self.WK_DIR)
)
else:
# copy everything at once
if os.path.exists(self.OUT_DIR):
if self.overwrite:
try:
_log.error('%s exists, attempting to remove.', self.OUT_DIR)
gfdl_util.rmtree_wrapper(self.OUT_DIR)
except OSError:
# gcp will not overwrite dirs, so forced to save under
# a different name despite overwrite=True
_log.error(("Couldn't remove %s (probably mounted read"
"-only); will rename new directory."), self.OUT_DIR)
else:
_log.error("%s exists; will rename new directory.", self.OUT_DIR)
try:
if os.path.exists(self.OUT_DIR):
# check again, since rmtree() might have succeeded
self.OUT_DIR, version = \
util.bump_version(self.OUT_DIR)
new_wkdir, _ = \
util.bump_version(self.WK_DIR, new_v=version)
_log.debug("Move %s to %s", self.WK_DIR, new_wkdir)
shutil.move(self.WK_DIR, new_wkdir)
self.WK_DIR = new_wkdir
gfdl_util.gcp_wrapper(self.WK_DIR, self.OUT_DIR)
except Exception:
raise # only delete MODEL_WK_DIR if copied successfully
_log.debug('Transfer succeeded; deleting directory %s', self.WK_DIR)
gfdl_util.rmtree_wrapper(self.WK_DIR)