def fetch_dataset(self, var, d_key):
"""Copy files to temporary directory.
(GCP can't copy to home dir, so always copy to a temp dir)
"""
tmpdir = core.TempDirManager().make_tempdir()
self.log.debug("Created GCP fetch temp dir at %s.", tmpdir)
(cp_command, smartsite) = self._get_fetch_method(self._fetch_method)
paths = d_key.remote_data()
if isinstance(paths, pd.Series):
paths = paths.to_list()
if not util.is_iterable(paths):
paths = (paths, )
local_paths = []
for path in paths:
# exceptions caught in parent loop in data_manager.DataSourceBase
local_path = os.path.join(tmpdir, os.path.basename(path))
self.log.info(f"\tFetching {path[len(self.attrs.CASE_ROOT_DIR):]}")
util.run_command(
cp_command + [
smartsite + path,
# gcp requires trailing slash, ln ignores it
smartsite + tmpdir + os.sep
],
timeout=self.timeout,
dry_run=self.dry_run,
log=self.log)
local_paths.append(local_path)
d_key.local_data = local_paths
def setUp_config_singletons(config=None, paths=None, pods=None, unittest=True):
cwd = os.path.dirname(os.path.realpath(__file__))
code_root = os.path.dirname(os.path.dirname(cwd))
cli_obj = cli.MDTFTopLevelArgParser(
code_root,
skip_defaults=True,
argv=f"-f {os.path.join(cwd, 'dummy_config.json')}")
cli_obj.config = vars(cli_obj.parse_args())
if config:
cli_obj.config.update(config)
PodDataTuple = collections.namedtuple('PodDataTuple',
'sorted_lists pod_data realm_data')
dummy_pod_data = PodDataTuple(pod_data=pods,
realm_data=dict(),
sorted_lists={
'pods': [],
'realms': []
})
_ = core.ConfigManager(cli_obj, dummy_pod_data, unittest=unittest)
pm = core.PathManager(cli_obj, unittest=unittest)
if paths:
pm.update(paths)
_ = core.VariableTranslator(code_root, unittest=unittest)
_ = core.TempDirManager(None, unittest=unittest)
def make_remote_dir(dest_dir, timeout=None, dry_run=None):
"""Workaround to create a directory on a remote filesystem by GCP'ing it.
"""
try:
os.makedirs(dest_dir)
except OSError as exc:
# use GCP for this because output dir might be on a read-only filesystem.
# apparently trying to test this with os.access is less robust than
# just catching the error
_log.debug("os.makedirs at %s failed (%r); trying GCP.", dest_dir, exc)
tmpdirs = core.TempDirManager()
work_dir = tmpdirs.make_tempdir()
work_dir = os.path.join(work_dir, os.path.basename(dest_dir))
os.makedirs(work_dir)
gcp_wrapper(work_dir, dest_dir, timeout=timeout, dry_run=dry_run)
def tearDown_config_singletons():
# clear Singletons
try:
temp = core.ConfigManager(unittest=True)
temp._reset()
except Exception:
pass
try:
temp = core.PathManager(unittest=True)
temp._reset()
except Exception:
pass
try:
temp = core.VariableTranslator(unittest=True)
temp._reset()
except Exception:
pass
try:
temp = core.TempDirManager(unittest=True)
temp._reset()
except Exception:
pass