Esempio n. 1
0
    def verify_paths(self, config, p):
        keep_temp = config.get('keep_temp', False)
        # clean out WORKING_DIR if we're not keeping temp files:
        if os.path.exists(p.WORKING_DIR) and not \
            (keep_temp or p.WORKING_DIR == p.OUTPUT_DIR):
            gfdl_util.rmtree_wrapper(p.WORKING_DIR)

        try:
            for dir_name, create_ in (('CODE_ROOT', False), ('OBS_DATA_REMOTE',
                                                             False),
                                      ('OBS_DATA_ROOT',
                                       True), ('MODEL_DATA_ROOT',
                                               True), ('WORKING_DIR', True)):
                util.check_dir(p, dir_name, create=create_)
        except Exception as exc:
            _log.fatal((f"Input settings for {dir_name} mis-specified (caught "
                        f"{repr(exc)}.)"))
            util.exit_handler(code=1)

        # Use GCP to create OUTPUT_DIR on a volume that may be read-only
        if not os.path.exists(p.OUTPUT_DIR):
            gfdl_util.make_remote_dir(p.OUTPUT_DIR,
                                      self.timeout,
                                      self.dry_run,
                                      log=_log)
Esempio n. 2
0
 def copy_to_output(self):
     """Use gcp for transfer, since OUTPUT_DIR might be mounted read-only.
     Also has special logic to handle frepp_mode.
     """
     if self.WK_DIR == self.OUT_DIR:
         return  # no copying needed
     if self.frepp_mode:
         # only copy PODs that ran, whether they succeeded or not
         for pod in self.obj.iter_children():
             if pod._has_placeholder:
                 gfdl_util.gcp_wrapper(pod.POD_WK_DIR,
                                       pod.POD_OUT_DIR,
                                       log=pod.log)
         # copy all case-level files
         self.obj.log.debug("Copying case-level files in %s", self.WK_DIR)
         for f in os.listdir(self.WK_DIR):
             if os.path.isfile(os.path.join(self.WK_DIR, f)):
                 self.obj.log.debug("Found case-level file %s", f)
                 gfdl_util.gcp_wrapper(os.path.join(self.WK_DIR, f),
                                       self.OUT_DIR,
                                       log=self.obj.log)
     else:
         # copy everything at once
         if os.path.exists(self.OUT_DIR):
             if self.overwrite:
                 try:
                     self.obj.log.error('%s exists, attempting to remove.',
                                        self.OUT_DIR)
                     gfdl_util.rmtree_wrapper(self.OUT_DIR)
                 except OSError:
                     # gcp will not overwrite dirs, so forced to save under
                     # a different name despite overwrite=True
                     self.obj.log.error(
                         ("Couldn't remove %s (probably mounted read"
                          "-only); will rename new directory."),
                         self.OUT_DIR)
             else:
                 self.obj.log.error("%s exists; will rename new directory.",
                                    self.OUT_DIR)
         try:
             if os.path.exists(self.OUT_DIR):
                 # check again, since rmtree() might have succeeded
                 self.OUT_DIR, version = \
                     util.bump_version(self.OUT_DIR)
                 new_wkdir, _ = \
                     util.bump_version(self.WK_DIR, new_v=version)
                 self.obj.log.debug("Move %s to %s", self.WK_DIR, new_wkdir)
                 shutil.move(self.WK_DIR, new_wkdir)
                 self.WK_DIR = new_wkdir
             gfdl_util.gcp_wrapper(self.WK_DIR,
                                   self.OUT_DIR,
                                   log=self.obj.log)
         except Exception:
             raise  # only delete MODEL_WK_DIR if copied successfully
         self.obj.log.debug('Transfer succeeded; deleting directory %s',
                            self.WK_DIR)
         gfdl_util.rmtree_wrapper(self.WK_DIR)
Esempio n. 3
0
 def verify_paths(self, config, p):
     keep_temp = config.get('keep_temp', False)
     # clean out WORKING_DIR if we're not keeping temp files:
     if os.path.exists(p.WORKING_DIR) and not \
         (keep_temp or p.WORKING_DIR == p.OUTPUT_DIR):
         gfdl_util.rmtree_wrapper(p.WORKING_DIR)
     util.check_dirs(p.CODE_ROOT, p.OBS_DATA_REMOTE, create=False)
     util.check_dirs(p.MODEL_DATA_ROOT, p.OBS_DATA_ROOT, p.WORKING_DIR, 
         create=True)
     # Use GCP to create OUTPUT_DIR on a volume that may be read-only
     if not os.path.exists(p.OUTPUT_DIR):
         gfdl_util.make_remote_dir(p.OUTPUT_DIR, self.timeout, self.dry_run)
Esempio n. 4
0
             gfdl_util.gcp_wrapper(pod.POD_WK_DIR, pod.POD_OUT_DIR)
     # copy all case-level files
     _log.debug("Copying case-level files in %s", self.WK_DIR)
     for f in os.listdir(self.WK_DIR):
         if os.path.isfile(os.path.join(self.WK_DIR, f)):
             _log.debug("Found case-level file %s", f)
             gfdl_util.gcp_wrapper(
                 os.path.join(self.WK_DIR, f), self.OUT_DIR,
             )
 else:
     # copy everything at once
     if os.path.exists(self.OUT_DIR):
         if self.overwrite:
             try:
                 _log.error('%s exists, attempting to remove.', self.OUT_DIR)
                 gfdl_util.rmtree_wrapper(self.OUT_DIR)
             except OSError:
                 # gcp will not overwrite dirs, so forced to save under
                 # a different name despite overwrite=True
                 _log.error(("Couldn't remove %s (probably mounted read"
                     "-only); will rename new directory."), self.OUT_DIR)
         else:
             _log.error("%s exists; will rename new directory.", self.OUT_DIR)
     try:
         if os.path.exists(self.OUT_DIR):
             # check again, since rmtree() might have succeeded
             self.OUT_DIR, version = \
                 util.bump_version(self.OUT_DIR)
             new_wkdir, _ = \
                 util.bump_version(self.WK_DIR, new_v=version)
             _log.debug("Move %s to %s", self.WK_DIR, new_wkdir)