def fetch_dataset(self, var, d_key): """Copy files to temporary directory. (GCP can't copy to home dir, so always copy to a temp dir) """ tmpdir = core.TempDirManager().make_tempdir() self.log.debug("Created GCP fetch temp dir at %s.", tmpdir) (cp_command, smartsite) = self._get_fetch_method(self._fetch_method) paths = d_key.remote_data() if isinstance(paths, pd.Series): paths = paths.to_list() if not util.is_iterable(paths): paths = (paths, ) local_paths = [] for path in paths: # exceptions caught in parent loop in data_manager.DataSourceBase local_path = os.path.join(tmpdir, os.path.basename(path)) self.log.info(f"\tFetching {path[len(self.attrs.CASE_ROOT_DIR):]}") util.run_command( cp_command + [ smartsite + path, # gcp requires trailing slash, ln ignores it smartsite + tmpdir + os.sep ], timeout=self.timeout, dry_run=self.dry_run, log=self.log) local_paths.append(local_path) d_key.local_data = local_paths
def setUp_config_singletons(config=None, paths=None, pods=None, unittest=True): cwd = os.path.dirname(os.path.realpath(__file__)) code_root = os.path.dirname(os.path.dirname(cwd)) cli_obj = cli.MDTFTopLevelArgParser( code_root, skip_defaults=True, argv=f"-f {os.path.join(cwd, 'dummy_config.json')}") cli_obj.config = vars(cli_obj.parse_args()) if config: cli_obj.config.update(config) PodDataTuple = collections.namedtuple('PodDataTuple', 'sorted_lists pod_data realm_data') dummy_pod_data = PodDataTuple(pod_data=pods, realm_data=dict(), sorted_lists={ 'pods': [], 'realms': [] }) _ = core.ConfigManager(cli_obj, dummy_pod_data, unittest=unittest) pm = core.PathManager(cli_obj, unittest=unittest) if paths: pm.update(paths) _ = core.VariableTranslator(code_root, unittest=unittest) _ = core.TempDirManager(None, unittest=unittest)
def make_remote_dir(dest_dir, timeout=None, dry_run=None): """Workaround to create a directory on a remote filesystem by GCP'ing it. """ try: os.makedirs(dest_dir) except OSError as exc: # use GCP for this because output dir might be on a read-only filesystem. # apparently trying to test this with os.access is less robust than # just catching the error _log.debug("os.makedirs at %s failed (%r); trying GCP.", dest_dir, exc) tmpdirs = core.TempDirManager() work_dir = tmpdirs.make_tempdir() work_dir = os.path.join(work_dir, os.path.basename(dest_dir)) os.makedirs(work_dir) gcp_wrapper(work_dir, dest_dir, timeout=timeout, dry_run=dry_run)
def tearDown_config_singletons(): # clear Singletons try: temp = core.ConfigManager(unittest=True) temp._reset() except Exception: pass try: temp = core.PathManager(unittest=True) temp._reset() except Exception: pass try: temp = core.VariableTranslator(unittest=True) temp._reset() except Exception: pass try: temp = core.TempDirManager(unittest=True) temp._reset() except Exception: pass