def setup(self): repo_path = tempfile.mkdtemp(**get_tempfile_kwargs(prefix="tree")) create_tree(repo_path, {'1.tar': { 'file.txt': 'load', '1.dat': 'load2' }}) self.ds = ds = Dataset(repo_path) ds.create(force=True) self.annex = ds.repo # Let's add first archive to the annex so we could test ds.save('1.tar', message="added 1.tar")
def __init__(self, toppath=None, persistent=False): self._toppath = toppath if toppath: path = opj(toppath, ARCHIVES_TEMP_DIR) if not persistent: tempsuffix = "-" + _get_random_id() lgr.debug("For non-persistent archives using %s suffix for path %s", tempsuffix, path) path += tempsuffix # TODO: begging for a race condition if not exists(path): lgr.debug("Initiating clean cache for the archives under %s", path) try: self._made_path = True os.makedirs(path) lgr.debug("Cache initialized") except Exception: lgr.error("Failed to initialize cached under %s", path) raise else: lgr.debug( "Not initiating existing cache for the archives under %s", path) self._made_path = False else: if persistent: raise ValueError( "%s cannot be persistent, because no toppath was provided" % self) path = tempfile.mkdtemp(**get_tempfile_kwargs()) self._made_path = True self._path = path self.persistent = persistent # TODO? ensure that it is absent or we should allow for it to persist a bit? #if exists(path): # self._clean_cache() self._archives = {} # TODO: begging for a race condition if not exists(path): lgr.debug("Initiating clean cache for the archives under %s", self.path) try: self._made_path = True os.makedirs(path) lgr.debug("Cache initialized") except Exception as e: lgr.error("Failed to initialize cached under %s", path) raise else: lgr.debug("Not initiating existing cache for the archives under %s", self.path) self._made_path = False
def __init__(self, archive, path=None, persistent=False): self._archive = archive # TODO: bad location for extracted archive -- use tempfile if not path: path = tempfile.mktemp(**get_tempfile_kwargs( prefix=_get_cached_filename(archive))) if exists(path) and not persistent: raise RuntimeError( "Directory %s already exists whenever it should not " "persist" % path) self._persistent = persistent self._path = path
def __call__(path=None, spec=None, seed=None): levels = _parse_spec(spec) if seed is not None: # TODO: if to be used within a bigger project we shouldn't seed main RNG random.seed(seed) if path is None: kw = get_tempfile_kwargs({}, prefix="ds") path = tempfile.mkdtemp(**kw) else: # so we don't override anything assert not exists(path) os.makedirs(path) # now we should just make it happen and return list of all the datasets return list(_makeds(path, levels))
def setup(self): self.log("Setup ran in %s, existing paths: %s", getpwd(), glob('*')) tempdir = tempfile.mkdtemp(**get_tempfile_kwargs({}, prefix="bm")) self.remove_paths.append(tempdir) with tarfile.open(self.tarfile) as tar: tar.extractall(tempdir) # TODO -- remove this abomination after https://github.com/datalad/datalad/issues/1512 is fixed epath = op.join(tempdir, 'testds1') epath_unique = epath + str(self.__class__.ds_count) os.rename(epath, epath_unique) self.__class__.ds_count += 1 self.ds = Dataset(epath_unique) self.repo = self.ds.repo self.log("Finished setup for %s", tempdir)
def __call__(path=None, spec=None, seed=None): if spec is None: spec = "10/1-3/-2" # 10 on top level, some random number from 1 to 3 at the 2nd, up to 2 on 3rd levels = _parse_spec(spec) if seed is not None: # TODO: if to be used within a bigger project we shouldn't seed main RNG random.seed(seed) if path is None: kw = get_tempfile_kwargs({}, prefix="ds") path = tempfile.mktemp(mkdir=True, **kw) else: # so we don't override anything assert not exists(path) os.makedirs(path) # now we should just make it happen and return list of all the datasets return list(_makeds(path, levels))
def setup(self): self.log("Setup ran in %s, existing paths: %s", getpwd(), glob('*')) tempdir = tempfile.mkdtemp( **get_tempfile_kwargs({}, prefix="bm") ) self.remove_paths.append(tempdir) with tarfile.open(self.tarfile) as tar: tar.extractall(tempdir) # TODO -- remove this abomination after https://github.com/datalad/datalad/issues/1512 is fixed epath = op.join(tempdir, 'testds1') epath_unique = epath + str(self.__class__.ds_count) os.rename(epath, epath_unique) self.__class__.ds_count += 1 self.ds = Dataset(epath_unique) self.repo = self.ds.repo self.log("Finished setup for %s", tempdir)
def setup_class(cls): mktmp_kws = get_tempfile_kwargs() path = tempfile.mkdtemp(**mktmp_kws) create_tree(path, {"udir": {x + ".dat" + ver: x + " content" for x in "abcd" for ver in ["", ".v1"]}}) cls._hpath = HTTPPath(path) cls._hpath.start() cls.url = cls._hpath.url cls.json_file = tempfile.mktemp(suffix=".json", **mktmp_kws) with open(cls.json_file, "w") as jfh: json.dump( [{"url": cls.url + "udir/a.dat", "name": "a", "subdir": "foo"}, {"url": cls.url + "udir/b.dat", "name": "b", "subdir": "bar"}, {"url": cls.url + "udir/c.dat", "name": "c", "subdir": "foo"}], jfh)
def setup(self, exclude_metadata): self.nfiles = 20 self.temp = Path( tempfile.mkdtemp(**get_tempfile_kwargs({}, prefix='bm_addurls1'))) self.ds = dl.create(self.temp / "ds") self.ds.config.set('annex.security.allowed-url-schemes', 'file', where='local') # populate list.csv and files srcpath = PurePosixPath(self.temp) rows = ["url,filename,bogus1,bogus2"] for i in range(self.nfiles): (self.temp / str(i)).write_text(str(i)) rows.append("file://{}/{},{},pocus,focus".format(srcpath, i, i)) self.listfile = self.temp / "list.csv" self.listfile.write_text(os.linesep.join(rows))
def setup_class(cls): mktmp_kws = get_tempfile_kwargs() path = tempfile.mkdtemp(**mktmp_kws) http_root = op.join(path, "srv") create_tree( http_root, { "udir": { x + ".dat" + ver: x + " content" for x in "abcd" for ver in ["", ".v1"] } }) cls._hpath = HTTPPath(http_root) cls._hpath.start() cls.url = cls._hpath.url cls.data = [{ "url": cls.url + "udir/a.dat", "name": "a", "subdir": "foo", "md5sum": "3fb7c40c70b0ed19da713bd69ee12014", "size": "9" }, { "url": cls.url + "udir/b.dat", "name": "b", "subdir": "bar", "md5sum": "", "size": "" }, { "url": cls.url + "udir/c.dat", "name": "c", "subdir": "foo", "md5sum": "9b72648021b70b8c522642e4490d7ac3", "size": "9" }] cls.json_file = op.join(path, "test_addurls.json") with open(cls.json_file, "w") as jfh: json.dump(cls.data, jfh) cls.temp_dir = path
def setup(self): self.path = tempfile.mkdtemp(**get_tempfile_kwargs({}, prefix='bm_forrest'))
def setup(self): self.path = tempfile.mkdtemp( **get_tempfile_kwargs({}, prefix='bm_forrest'))