def setup(self):
repo_path = tempfile.mkdtemp(**get_tempfile_kwargs(prefix="tree"))
create_tree(repo_path,
{'1.tar': {
'file.txt': 'load',
'1.dat': 'load2'
}})
self.ds = ds = Dataset(repo_path)
ds.create(force=True)
self.annex = ds.repo
# Let's add first archive to the annex so we could test
ds.save('1.tar', message="added 1.tar")
def __init__(self, toppath=None, persistent=False):
self._toppath = toppath
if toppath:
path = opj(toppath, ARCHIVES_TEMP_DIR)
if not persistent:
tempsuffix = "-" + _get_random_id()
lgr.debug("For non-persistent archives using %s suffix for path %s",
tempsuffix, path)
path += tempsuffix
# TODO: begging for a race condition
if not exists(path):
lgr.debug("Initiating clean cache for the archives under %s",
path)
try:
self._made_path = True
os.makedirs(path)
lgr.debug("Cache initialized")
except Exception:
lgr.error("Failed to initialize cached under %s", path)
raise
else:
lgr.debug(
"Not initiating existing cache for the archives under %s",
path)
self._made_path = False
else:
if persistent:
raise ValueError(
"%s cannot be persistent, because no toppath was provided"
% self)
path = tempfile.mkdtemp(**get_tempfile_kwargs())
self._made_path = True
self._path = path
self.persistent = persistent
# TODO? ensure that it is absent or we should allow for it to persist a bit?
#if exists(path):
# self._clean_cache()
self._archives = {}
# TODO: begging for a race condition
if not exists(path):
lgr.debug("Initiating clean cache for the archives under %s", self.path)
try:
self._made_path = True
os.makedirs(path)
lgr.debug("Cache initialized")
except Exception as e:
lgr.error("Failed to initialize cached under %s", path)
raise
else:
lgr.debug("Not initiating existing cache for the archives under %s", self.path)
self._made_path = False
def __init__(self, archive, path=None, persistent=False):
self._archive = archive
# TODO: bad location for extracted archive -- use tempfile
if not path:
path = tempfile.mktemp(**get_tempfile_kwargs(
prefix=_get_cached_filename(archive)))
if exists(path) and not persistent:
raise RuntimeError(
"Directory %s already exists whenever it should not "
"persist" % path)
self._persistent = persistent
self._path = path
def __call__(path=None, spec=None, seed=None):
levels = _parse_spec(spec)
if seed is not None:
# TODO: if to be used within a bigger project we shouldn't seed main RNG
random.seed(seed)
if path is None:
kw = get_tempfile_kwargs({}, prefix="ds")
path = tempfile.mkdtemp(**kw)
else:
# so we don't override anything
assert not exists(path)
os.makedirs(path)
# now we should just make it happen and return list of all the datasets
return list(_makeds(path, levels))
def setup(self):
self.log("Setup ran in %s, existing paths: %s", getpwd(), glob('*'))
tempdir = tempfile.mkdtemp(**get_tempfile_kwargs({}, prefix="bm"))
self.remove_paths.append(tempdir)
with tarfile.open(self.tarfile) as tar:
tar.extractall(tempdir)
# TODO -- remove this abomination after https://github.com/datalad/datalad/issues/1512 is fixed
epath = op.join(tempdir, 'testds1')
epath_unique = epath + str(self.__class__.ds_count)
os.rename(epath, epath_unique)
self.__class__.ds_count += 1
self.ds = Dataset(epath_unique)
self.repo = self.ds.repo
self.log("Finished setup for %s", tempdir)
def __call__(path=None, spec=None, seed=None):
if spec is None:
spec = "10/1-3/-2" # 10 on top level, some random number from 1 to 3 at the 2nd, up to 2 on 3rd
levels = _parse_spec(spec)
if seed is not None:
# TODO: if to be used within a bigger project we shouldn't seed main RNG
random.seed(seed)
if path is None:
kw = get_tempfile_kwargs({}, prefix="ds")
path = tempfile.mktemp(mkdir=True, **kw)
else:
# so we don't override anything
assert not exists(path)
os.makedirs(path)
# now we should just make it happen and return list of all the datasets
return list(_makeds(path, levels))
def setup(self):
self.log("Setup ran in %s, existing paths: %s", getpwd(), glob('*'))
tempdir = tempfile.mkdtemp(
**get_tempfile_kwargs({}, prefix="bm")
)
self.remove_paths.append(tempdir)
with tarfile.open(self.tarfile) as tar:
tar.extractall(tempdir)
# TODO -- remove this abomination after https://github.com/datalad/datalad/issues/1512 is fixed
epath = op.join(tempdir, 'testds1')
epath_unique = epath + str(self.__class__.ds_count)
os.rename(epath, epath_unique)
self.__class__.ds_count += 1
self.ds = Dataset(epath_unique)
self.repo = self.ds.repo
self.log("Finished setup for %s", tempdir)
def setup_class(cls):
mktmp_kws = get_tempfile_kwargs()
path = tempfile.mkdtemp(**mktmp_kws)
create_tree(path,
{"udir": {x + ".dat" + ver: x + " content"
for x in "abcd"
for ver in ["", ".v1"]}})
cls._hpath = HTTPPath(path)
cls._hpath.start()
cls.url = cls._hpath.url
cls.json_file = tempfile.mktemp(suffix=".json", **mktmp_kws)
with open(cls.json_file, "w") as jfh:
json.dump(
[{"url": cls.url + "udir/a.dat", "name": "a", "subdir": "foo"},
{"url": cls.url + "udir/b.dat", "name": "b", "subdir": "bar"},
{"url": cls.url + "udir/c.dat", "name": "c", "subdir": "foo"}],
jfh)
def setup_class(cls):
mktmp_kws = get_tempfile_kwargs()
path = tempfile.mkdtemp(**mktmp_kws)
create_tree(path,
{"udir": {x + ".dat" + ver: x + " content"
for x in "abcd"
for ver in ["", ".v1"]}})
cls._hpath = HTTPPath(path)
cls._hpath.start()
cls.url = cls._hpath.url
cls.json_file = tempfile.mktemp(suffix=".json", **mktmp_kws)
with open(cls.json_file, "w") as jfh:
json.dump(
[{"url": cls.url + "udir/a.dat", "name": "a", "subdir": "foo"},
{"url": cls.url + "udir/b.dat", "name": "b", "subdir": "bar"},
{"url": cls.url + "udir/c.dat", "name": "c", "subdir": "foo"}],
jfh)
def setup(self, exclude_metadata):
self.nfiles = 20
self.temp = Path(
tempfile.mkdtemp(**get_tempfile_kwargs({}, prefix='bm_addurls1')))
self.ds = dl.create(self.temp / "ds")
self.ds.config.set('annex.security.allowed-url-schemes',
'file',
where='local')
# populate list.csv and files
srcpath = PurePosixPath(self.temp)
rows = ["url,filename,bogus1,bogus2"]
for i in range(self.nfiles):
(self.temp / str(i)).write_text(str(i))
rows.append("file://{}/{},{},pocus,focus".format(srcpath, i, i))
self.listfile = self.temp / "list.csv"
self.listfile.write_text(os.linesep.join(rows))
def setup_class(cls):
mktmp_kws = get_tempfile_kwargs()
path = tempfile.mkdtemp(**mktmp_kws)
http_root = op.join(path, "srv")
create_tree(
http_root, {
"udir": {
x + ".dat" + ver: x + " content"
for x in "abcd" for ver in ["", ".v1"]
}
})
cls._hpath = HTTPPath(http_root)
cls._hpath.start()
cls.url = cls._hpath.url
cls.data = [{
"url": cls.url + "udir/a.dat",
"name": "a",
"subdir": "foo",
"md5sum": "3fb7c40c70b0ed19da713bd69ee12014",
"size": "9"
}, {
"url": cls.url + "udir/b.dat",
"name": "b",
"subdir": "bar",
"md5sum": "",
"size": ""
}, {
"url": cls.url + "udir/c.dat",
"name": "c",
"subdir": "foo",
"md5sum": "9b72648021b70b8c522642e4490d7ac3",
"size": "9"
}]
cls.json_file = op.join(path, "test_addurls.json")
with open(cls.json_file, "w") as jfh:
json.dump(cls.data, jfh)
cls.temp_dir = path
def setup(self):
self.path = tempfile.mkdtemp(**get_tempfile_kwargs({}, prefix='bm_forrest'))
def setup(self):
self.path = tempfile.mkdtemp(
**get_tempfile_kwargs({}, prefix='bm_forrest'))
