Exemple #1
0
 def __init__(self, path: Path, *, location: Point5D = Point5D.zero(), filesystem: FS):
     try:
         raw_data = skimage.io.imread(filesystem.openbin(path.as_posix()))
     except ValueError:
         raise UnsupportedUrlException(path)
     axiskeys = "yxc"[: len(raw_data.shape)]
     super().__init__(url=filesystem.desc(path.as_posix()), data=raw_data, axiskeys=axiskeys, location=location)
Exemple #2
0
def _merge_primaries(changes_dict: ChangesDict, src_fs: FS,
                     dst_fs: FS) -> None:
    # TODO Not sure that this hits all cases, including removal of
    # files and directories.  Think about it.
    for dirpath in src_fs.walk.dirs(search="depth"):
        if _is_component_path(dirpath):
            lid = dirpath_to_lid(dirpath)
            changed = changes_dict.changed(lid)
            if changed:
                if not dst_fs.isdir(dirpath):
                    dst_fs.makedirs(dirpath)
                src_sub_fs = SubFS(src_fs, dirpath)
                dst_sub_fs = SubFS(dst_fs, dirpath)
                # delete directories in dst that don't exist in src
                for subdirpath in dst_sub_fs.walk.dirs(search="depth"):
                    if not src_sub_fs.isdir(subdirpath):
                        dst_sub_fs.removetree(subdirpath)
                # delete the files in the destination (if any)
                for filepath in component_files(dst_fs, dirpath):
                    dst_sub_fs.remove(filepath)
                # copy the new files across
                src_sub_fs = SubFS(src_fs, dirpath)
                for filepath in component_files(src_fs, dirpath):
                    fs.copy.copy_file(src_sub_fs, filepath, dst_sub_fs,
                                      filepath)
Exemple #3
0
    def openDataset(cls, path: Path, filesystem: FS) -> Tuple[h5py.Dataset, Path, Path]:
        outer_path = path
        dataset_path_components: List[str] = []
        while True:
            try:
                info = filesystem.getinfo(outer_path.as_posix())
                if not info.is_file:
                    raise UnsupportedUrlException(path.as_posix())
                break
            except ResourceNotFound as e:
                dataset_path_components.insert(0, outer_path.name)
                parent = outer_path.parent
                if parent == outer_path:
                    raise UnsupportedUrlException(path.as_posix())
                outer_path = parent

        try:
            binfile = filesystem.openbin(outer_path.as_posix())
            f = h5py.File(binfile, "r")
        except OSError as e:
            raise UnsupportedUrlException(path) from e

        try:
            inner_path = "/".join(dataset_path_components)
            dataset = f[inner_path]
            if not isinstance(dataset, h5py.Dataset):
                raise ValueError(f"{inner_path} is not a h5py.Dataset")
        except Exception as e:
            f.close()
            raise e

        return dataset, outer_path, Path(inner_path)
Exemple #4
0
def copy_fs(version_view: FS, deliverable: FS) -> None:
    # TODO The note below is probably obsolete.  It was written when
    # we were using the DeliverableFS to make the deliverable.  We've
    # now switched to using the DeliverableView. (The obsolete
    # DeliverableFS has been deleted from the repo.)  In any case,
    # this doesn't seem to hurt anything, but it can probably all be
    # replaced by fs.copy.copy_file()--see the last line.  Try it and
    # see.

    # TODO I could (and used to) just do a fs.copy.copy_fs() from the
    # version_view to a DeliverableFS.  I removed it to debug issues
    # with the validation tool.  Now I find this hack is just as easy
    # (though I wonder about efficiency).  It bothers me that this
    # visits hack parallels a visits hack in
    # plain_lidvid_to_visits_dirpath().  I should figure this out and
    # make it clean.  For now, though, this works.

    # Uses dollar-terminated paths
    for path, dirs, files in version_view.walk():
        parts = fs.path.parts(path)
        if len(parts) == 4:
            if len(parts[3]) == 10:
                visit = "visit_" + parts[3][4:6].lower() + "$"
                parts[3] = visit
        new_path = fs.path.join(*parts)
        if not deliverable.isdir(new_path):
            deliverable.makedir(new_path)
        for file in files:
            old_filepath = fs.path.join(path, file.name)
            new_filepath = fs.path.join(new_path, file.name)
            fs.copy.copy_file(version_view, old_filepath, deliverable,
                              new_filepath)
Exemple #5
0
    def write_to_bento(self, bento_fs: FS, build_ctx: str):
        docker_folder = fs.path.join("env", "docker")
        bento_fs.makedirs(docker_folder, recreate=True)
        dockerfile = fs.path.join(docker_folder, "Dockerfile")
        template_file = os.path.join(os.path.dirname(__file__), "docker",
                                     "Dockerfile.template")
        with open(template_file, "r", encoding="utf-8") as f:
            dockerfile_template = f.read()

        with bento_fs.open(dockerfile, "w") as dockerfile:
            dockerfile.write(
                dockerfile_template.format(
                    base_image=self.get_base_image_tag()))

        for filename in ["init.sh", "entrypoint.sh"]:
            copy_file_to_fs_folder(
                os.path.join(os.path.dirname(__file__), "docker", filename),
                bento_fs,
                docker_folder,
            )

        if self.setup_script:
            try:
                setup_script = resolve_user_filepath(self.setup_script,
                                                     build_ctx)
            except FileNotFoundError as e:
                raise InvalidArgument(f"Invalid setup_script file: {e}")
            copy_file_to_fs_folder(setup_script, bento_fs, docker_folder,
                                   "setup_script")
Exemple #6
0
    def __init__(self, host, user, passwd, port=22, compress=False, timeout=10, keepalive=10):
        FS.__init__(self)

        self._user = user
        self._host = host
        self._port = port
        self._client = client = paramiko.SSHClient()
        self._locale = None

        try:
            client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

            client.connect(
                socket.gethostbyname(host), port, user, passwd,
                compress=compress, timeout=timeout,
                allow_agent=False,look_for_keys=False
            )
            if keepalive > 0:
                client.get_transport().set_keepalive(keepalive)
            self._sftp = client.open_sftp()
            self._platform = None

        except (paramiko.ssh_exception.SSHException,            # protocol errors
                paramiko.ssh_exception.NoValidConnectionsError, # connexion errors
                socket.gaierror, socket.timeout) as e:          # TCP errors

            message = "Unable to create filesystem: {}".format(e)
            raise errors.CreateFailed(message)
Exemple #7
0
 def compile_fragment_files(
         self, write_fs: FS,
         found_fragments: Iterable[FoundFragment]) -> List[str]:
     """
     Compile fragment files into `parent_dir`.
     """
     outputs = []
     for version_fs, filename in found_fragments:
         try:
             fragment = self.load_fragment(version_fs.readtext(filename))
             fragment_type = fragment.get('type')
             showcontent = self.config.fragment_types.get(
                 fragment_type, {}).get('showcontent', True)
             section = fragment.get('section') or None
             rendered_content = render_fragment(
                 fragment, showcontent, self.config.changelog_output_type)
             if rendered_content.strip():
                 filename_stem = splitext(basename(filename))[0]
                 output_path = join(*filter(None, [
                     section, '{}.{}'.format(filename_stem, fragment_type)
                 ]))
                 log.info('Compiling {} -> {}'.format(
                     version_fs.getsyspath(filename),
                     write_fs.getsyspath(output_path)))
                 parent_dir = dirname(output_path)
                 if parent_dir:
                     write_fs.makedirs(parent_dir, recreate=True)
                 write_fs.writetext(output_path, rendered_content)
                 outputs.append(output_path)
         except Exception:
             raise FragmentCompilationError(filename)
     return outputs
Exemple #8
0
    def write_to_bento(self, bento_fs: FS, build_ctx: str):
        conda_folder = fs.path.join("env", "conda")
        bento_fs.makedirs(conda_folder, recreate=True)

        if self.environment_yml is not None:
            environment_yml_file = resolve_user_filepath(
                self.environment_yml, build_ctx)
            copy_file_to_fs_folder(
                environment_yml_file,
                bento_fs,
                conda_folder,
                dst_filename="environment_yml",
            )

            return

        deps_list = [] if self.dependencies is None else self.dependencies
        if self.pip is not None:
            deps_list.append(dict(pip=self.pip))  # type: ignore

        if not deps_list:
            return

        yaml_content = dict(dependencies=deps_list)
        yaml_content["channels"] = (["defaults"] if self.channels is None else
                                    self.channels)
        with bento_fs.open(fs.path.join(conda_folder, "environment_yml"),
                           "w") as f:
            yaml.dump(yaml_content, f)
Exemple #9
0
def cleantree(tgt_fs: FS, name: str):
    method = tgt_fs.remove if tgt_fs.isfile(name) else tgt_fs.removetree
    method(name)
    remaining_parts = Path(name).parts
    if len(remaining_parts) > 1:
        # if dst_path included more than 1 directory we are left with the top parent
        tgt_fs.removedir(remaining_parts[0])
Exemple #10
0
    def __init__(self,
                 path: Path,
                 *,
                 location: Point5D = Point5D.zero(),
                 filesystem: FS):
        url = filesystem.geturl(path.as_posix())
        match = re.search(r"[^/]+\.n5/.*$", url, re.IGNORECASE)
        if not match:
            raise UnsupportedUrlException(url)
        name = match.group(0)
        self.filesystem = filesystem.opendir(path.as_posix())

        with self.filesystem.openbin("attributes.json", "r") as f:
            attributes_json_bytes = f.read()
        attributes = json.loads(attributes_json_bytes.decode("utf8"))

        dimensions = attributes["dimensions"][::-1]
        blockSize = attributes["blockSize"][::-1]
        axiskeys = "".join(attributes["axes"]).lower(
        )[::-1] if "axes" in attributes else guess_axiskeys(dimensions)

        super().__init__(
            url=url,
            name=name,
            tile_shape=Shape5D.create(raw_shape=blockSize, axiskeys=axiskeys),
            shape=Shape5D.create(raw_shape=dimensions, axiskeys=axiskeys),
            dtype=np.dtype(attributes["dataType"]).newbyteorder(">"),
            location=location,
            axiskeys=axiskeys,
        )
        self.compression_type = attributes["compression"]["type"]
        if self.compression_type not in N5Block.DECOMPRESSORS.keys():
            raise NotImplementedError(
                f"Don't know how to decompress from {self.compression_type}")
Exemple #11
0
    def create_cowfs(base_fs: FS,
                     read_write_layer: FS,
                     recreate: bool = False) -> "COWFS":
        additions_fs = read_write_layer.makedir("/additions",
                                                recreate=recreate)
        deletions_fs = read_write_layer.makedir("/deletions",
                                                recreate=recreate)

        return COWFS(base_fs, additions_fs, deletions_fs)
Exemple #12
0
 def move(self, src, dst, **kwds):
     if self.getmeta("atomic.rename", False):
         if kwds.get("overwrite", False) or not self.exists(dst):
             try:
                 self.rename(src, dst)
                 return
             except FSError:
                 pass
     FS.move(self, src, dst, **kwds)
 def move(self, src, dst, **kwds):
     if self.getmeta("atomic.rename",False):
         if kwds.get("overwrite",False) or not self.exists(dst):
             try:
                 self.rename(src,dst)
                 return
             except FSError:
                 pass
     FS.move(self, src, dst, **kwds)
Exemple #14
0
 def copydir(self,
             src,
             dst,
             overwrite=False,
             ignore_errors=False,
             chunk_size=16384):
     if self.getmeta("read_only"):
         raise errors.UnsupportedError('read only filesystem')
     # FIXME: this is out of date; how to do native tahoe copy?
     # FIXME: Workaround because isfile() not exists on _TahoeLAFS
     FS.copydir(self, src, dst, overwrite, ignore_errors, chunk_size)
Exemple #15
0
def write_subdir_versions_to_directory(fs: FS, dir: str, d: Dict[str,
                                                                 str]) -> None:
    """
    Given the path to a directory, un-parse and write the contents of
    the given subdir-versions dictionary into a subdir-versions file
    in the directory.
    """
    SUBDIR_VERSIONS_FILEPATH = join(dir, SUBDIR_VERSIONS_FILENAME)
    fs.writetext(SUBDIR_VERSIONS_FILEPATH,
                 unparse_subdir_versions(d),
                 encoding="ascii")
Exemple #16
0
def write_dictionary_to_fs(fs: FS, dir_path: str, d: Dict[Any, Any]) -> None:
    for k, v in d.items():
        assert type(k) in [str, str]
        type_v = type(v)
        sub_path = join(dir_path, str(k))
        if type_v == dict:
            fs.makedir(sub_path)
            write_dictionary_to_fs(fs, sub_path, v)
        elif type_v in [str, str]:
            fs.writetext(sub_path, str(v))
        else:
            assert False, f"unexpected type {type_v} at {sub_path}"
Exemple #17
0
    def __init__(self,
                 base_fs: VersionView,
                 synth_files: Optional[Dict[str, bytes]] = None) -> None:
        FS.__init__(self)
        self.base_fs = base_fs
        self.path_dict: Dict[str, _Entry] = {"/": _DirInfo()}

        self._populate_path_dict_from_base_fs()

        # Insert the synthetic files
        synth_files = dict() if synth_files is None else synth_files
        self._populate_path_dict_from_synth_files(synth_files)
Exemple #18
0
def movefile(src_fs,
             src_path,
             dst_fs,
             dst_path,
             overwrite=True,
             chunk_size=64 * 1024):
    """Move a file from one filesystem to another. Will use system copyfile, if both files have a syspath.
    Otherwise file will be copied a chunk at a time.

    :param src_fs: Source filesystem object
    :param src_path: Source path
    :param dst_fs: Destination filesystem object
    :param dst_path: Destination filesystem object
    :param chunk_size: Size of chunks to move if system copyfile is not available (default 64K)

    """
    src_syspath = src_fs.getsyspath(src_path, allow_none=True)
    dst_syspath = dst_fs.getsyspath(dst_path, allow_none=True)

    if not overwrite and dst_fs.exists(dst_path):
        raise DestinationExistsError(dst_path)

    if src_fs is dst_fs:
        src_fs.move(src_path, dst_path, overwrite=overwrite)
        return

    # System copy if there are two sys paths
    if src_syspath is not None and dst_syspath is not None:
        FS._shutil_movefile(src_syspath, dst_syspath)
        return

    src_lock = getattr(src_fs, '_lock', None)

    if src_lock is not None:
        src_lock.acquire()

    try:
        src = None
        try:
            # Chunk copy
            src = src_fs.open(src_path, 'rb')
            dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)
        except:
            raise
        else:
            src_fs.remove(src_path)
        finally:
            if src is not None:
                src.close()
    finally:
        if src_lock is not None:
            src_lock.release()
Exemple #19
0
    def get_module_version(module_name: str, directory: FS) -> str:
        """
        Get the version of the module in the given directory
        :param module_name: name of the module
        :param directory: FS object pointing to the parent directory of the module
        :return: version of the module or None if it is not present in the directory
        """
        version = None
        if module_name in directory.listdir("."):
            manifest = directory.readtext(join(module_name, "__manifest__.py"))
            version = ast.literal_eval(manifest)["version"]

        return version
Exemple #20
0
def package_json(cwd_fs: FS):
    """
    Try guess a version from ``package.json``.
    """
    log.debug('Looking for package.json')
    if cwd_fs.exists('package.json'):
        log.debug('Guessing version with package.json')
        try:
            with cwd_fs.open('package.json', 'r') as fd:
                return json.load(fd).get('version')
        except json.JSONDecodeError:
            pass
    return None
Exemple #21
0
    def __init__(self, fs_prims: FSPrimitives) -> None:
        FS.__init__(self)
        self.prims = fs_prims

        _meta = self._meta = {
            "case_insensitive": os.path.normcase("Aa") != "aa",
            "network": False,
            "read_only": False,
            "supports_rename": False,
            "thread_safe": True,
            "unicode_paths": False,
            "virtual": False,
            "invalid_path_chars": "\0",
        }
Exemple #22
0
def file_reader(*,
                fs: FS,
                resource: str,
                mimetype: str,
                proxy: bool = False,
                mmap_mode: Union[str, None] = None,
                **kwargs) -> Any:
    if mimetype is None and resource.endswith(".npy"):
        mimetype = "application/octet-stream"
    elif mimetype is None:
        mimetype, _ = guess_type(resource)

    if isinstance(resource, Path):
        resource = str(resource)

    mapping = {
        "application/octet-stream": (
            np.load,
            "file",
            {
                "file": fs.open(resource, mode="rb"),
                "mmap_mode": mmap_mode,
                "allow_pickle": False,
            },
        ),
        "application/json": (
            json.load,
            "fp",
            {
                "fp": fs.open(resource, encoding="utf-8")
            },
        ),
        "text/csv": (
            pd.read_csv,
            "filepath_or_buffer",
            {
                "filepath_or_buffer": fs.open(resource)
            },
        ),
    }

    try:
        func, label, kwargs = mapping[mimetype]
    except KeyError:
        raise InvalidMimetype("Mimetype '{}' not understoof".format(mimetype))

    if proxy:
        return Proxy(func, label, kwargs)
    else:
        return func(**kwargs)
Exemple #23
0
 def __init__(
     self,
     contains_lidvids: bool,
     subcomponents: Set[S],
     fs: FS,
     filepaths: Set[str],
 ) -> None:
     """
     Create a 'VersionContents' object.  DO NOT USE this
     constructor.  Instead use the static methods
     VersionContents.create_from_lids() and
     VersionContents.create_from_lidvids().
     """
     if not _data_consistent(contains_lidvids, subcomponents):
         raise TypeError(
             "Subcomponents are not consistent, should be all "
             + "livids or all lids."
         )
     self.contains_lidvids = contains_lidvids
     self.subcomponents: Set[S] = subcomponents
     for filepath in filepaths:
         if not isabs(filepath):
             raise ValueError(f"{filepath} is not an absolute path.")
         if not fs.isfile(filepath):
             raise ValueError(f"{filepath} is not a file.")
     self.fs = fs
     self.filepaths = filepaths
Exemple #24
0
 def __init__(self, fs: FSBase):
     self._fs = fs
     try:
         self._iterations: List[str] = json.loads(
             fs.readtext(_CHECKPOINT_STATE_FILE))
     except Exception:
         self._iterations = []
 def run(
     self,
     trial: Trial,
     judge: TrialJudge,
     checkpoint_basedir_fs: FSBase,
 ) -> None:
     checkpoint = Checkpoint(
         checkpoint_basedir_fs.makedir(trial.trial_id, recreate=True))
     if not judge.can_accept(trial):
         return
     self._current_trial = trial
     self.initialize()
     try:
         if len(checkpoint) > 0:
             self._rung = int(checkpoint.latest.readtext("__RUNG__")) + 1
             self.load_checkpoint(checkpoint.latest)
         budget = judge.get_budget(trial, self.rung)
         while budget > 0:
             report = self.run_single_rung(budget)
             report = report.with_rung(self.rung).with_sort_metric(
                 self.generate_sort_metric(report.metric))
             decision = judge.judge(report)
             if decision.should_checkpoint:
                 with checkpoint.create() as fs:
                     fs.writetext("__RUNG__", str(self.rung))
                     self.save_checkpoint(fs)
             budget = decision.budget
             self._rung += 1
     finally:
         self.finalize()
Exemple #26
0
 def __init__(self,
              path: Path,
              *,
              location: Point5D = Point5D.zero(),
              filesystem: FS):
     self._dataset: Optional[h5py.Dataset] = None
     try:
         self._dataset, outer_path, inner_path = self.openDataset(
             path, filesystem=filesystem)
         axiskeys = self.getAxisKeys(self._dataset)
         tile_shape = Shape5D.create(raw_shape=self._dataset.chunks
                                     or self._dataset.shape,
                                     axiskeys=axiskeys)
         super().__init__(
             url=filesystem.desc(outer_path.as_posix()) + "/" +
             inner_path.as_posix(),
             tile_shape=tile_shape,
             shape=Shape5D.create(raw_shape=self._dataset.shape,
                                  axiskeys=axiskeys),
             dtype=self._dataset.dtype,
             name=self._dataset.file.filename.split("/")[-1] +
             self._dataset.name,
             location=location,
             axiskeys=axiskeys,
         )
     except Exception as e:
         if self._dataset:
             self._dataset.file.close()
         raise e
Exemple #27
0
def scan_vfs_dir(
    fs: FS,
    dir: str,
    namespaces: Tuple = None
) -> Tuple[List[Info], List[Info], List[Info], List[Info]]:
    """
    Returns a 4-tuple of (ordinary-file infos, ordinary-directory
    infos, subdir-versions-file infos, version-directory infos).  This
    lets us separate "real" files and dirs from the "bookkeeping"
    ones.
    """
    infos = list(fs.scandir(dir, namespaces=namespaces))
    file_infos = [info for info in infos if info.is_file]
    dir_infos = [info for info in infos if info.is_dir]

    ordinary_file_infos = [
        info for info in file_infos if info.name != SUBDIR_VERSIONS_FILENAME
    ]
    subdir_versions_file_infos = [
        info for info in file_infos if info.name == SUBDIR_VERSIONS_FILENAME
    ]
    ordinary_dir_infos = [
        info for info in dir_infos if info.name[0:2] != _VERSION_DIR_PREFIX
    ]
    version_dir_infos = [
        info for info in dir_infos if info.name[0:2] == _VERSION_DIR_PREFIX
    ]
    return (
        ordinary_file_infos,
        ordinary_dir_infos,
        subdir_versions_file_infos,
        version_dir_infos,
    )
Exemple #28
0
def movefile(src_fs, src_path, dst_fs, dst_path, overwrite=True, chunk_size=64*1024):
    """Move a file from one filesystem to another. Will use system copyfile, if both files have a syspath.
    Otherwise file will be copied a chunk at a time.

    :param src_fs: Source filesystem object
    :param src_path: Source path
    :param dst_fs: Destination filesystem object
    :param dst_path: Destination filesystem object
    :param chunk_size: Size of chunks to move if system copyfile is not available (default 64K)

    """
    src_syspath = src_fs.getsyspath(src_path, allow_none=True)
    dst_syspath = dst_fs.getsyspath(dst_path, allow_none=True)

    if not overwrite and dst_fs.exists(dst_path):
        raise DestinationExistsError(dst_path)

    if src_fs is dst_fs:
        src_fs.move(src_path, dst_path, overwrite=overwrite)
        return

    # System copy if there are two sys paths
    if src_syspath is not None and dst_syspath is not None:
        FS._shutil_movefile(src_syspath, dst_syspath)
        return

    src_lock = getattr(src_fs, '_lock', None)

    if src_lock is not None:
        src_lock.acquire()

    try:
        src = None
        try:
            # Chunk copy
            src = src_fs.open(src_path, 'rb')
            dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)
        except:
            raise
        else:
            src_fs.remove(src_path)
        finally:
            if src is not None:
                src.close()
    finally:
        if src_lock is not None:
            src_lock.release()
Exemple #29
0
def copyfile(src_fs, src_path, dst_fs, dst_path, overwrite=True, update=False,
             chunk_size=64*1024):
    """Copy a file from one filesystem to another. Will use system copyfile, if both files have a syspath.
    Otherwise file will be copied a chunk at a time. In the case where ``overwrite`` is False and
    ``update`` is True, the behaviour of ``update`` takes precedence.

    :param src_fs: Source filesystem object
    :param src_path: -- Source path
    :param dst_fs: Destination filesystem object
    :param dst_path: Destination filesystem object
    :param overwrite: Write to destination files even if they already exist
    :param update: Write to destination files only if the source is newer
    :param chunk_size: Size of chunks to move if system copyfile is not available (default 64K)

    """

    assert_write(src_fs, src_path, dst_fs, dst_path, overwrite, update)

    # If the src and dst fs objects are the same, then use a direct copy
    if src_fs is dst_fs:
        src_fs.copy(src_path, dst_path, overwrite=overwrite)
        return

    src_syspath = src_fs.getsyspath(src_path, allow_none=True)
    dst_syspath = dst_fs.getsyspath(dst_path, allow_none=True)

    # System copy if there are two sys paths
    if src_syspath is not None and dst_syspath is not None:
        FS._shutil_copyfile(src_syspath, dst_syspath)
        return

    src_lock = getattr(src_fs, '_lock', None)

    if src_lock is not None:
        src_lock.acquire()

    try:
        src = None
        try:
            src = src_fs.open(src_path, 'rb')
            dst_fs.setcontents(dst_path, src, chunk_size=chunk_size)
        finally:
            if src is not None:
                src.close()
    finally:
        if src_lock is not None:
            src_lock.release()
Exemple #30
0
def file_writer(*, data: Any, fs: FS, resource: str, mimetype: str,
                **kwargs) -> None:
    if isinstance(resource, Path):
        resource = str(resource)

    if mimetype == "application/octet-stream":
        return np.save(fs.open(resource, mode="wb"), data, allow_pickle=False)
    elif mimetype == "application/json":
        return json.dump(
            data,
            fs.open(resource, mode="w", encoding="utf-8"),
            indent=2,
            ensure_ascii=False,
        )
    elif mimetype == "text/csv":
        assert isinstance(data, pd.DataFrame)
        data.to_csv(fs.open(resource, mode="w", encoding="utf-8"))
Exemple #31
0
    def update_from_single_version(self, is_new: IS_NEW_TEST,
                                   single_version_fs: FS) -> bool:
        # TODO This import is circular; that's why I have it here
        # inside the function.  But there must be a better way to
        # structure.
        from pdart.fs.multiversioned.version_view import (
            is_segment,
            strip_segment,
            vv_lid_path,
        )

        # TODO Note that this makes assumptions about the source
        # filesystem format.  Document them.

        def update_from_lid(lid: LID) -> LIDVID:
            # Find the path corresponding to this LID.
            path = vv_lid_path(lid)

            # First, update all the children recursively.  Get their
            # LIDs by extending this LID with the names of the
            # subdirectories of path.  That handles directories.
            child_lidvids: Set[LIDVID] = {
                update_from_lid(lid.extend_lid(strip_segment(name)))
                for name in single_version_fs.listdir(path) if is_segment(name)
            }

            # Now look at files.  We create a VersionContents object
            # from the set of new LIDVIDs and all the files contained
            # in the component's directory.
            contents = VersionContents.create_from_lidvids_and_dirpath(
                child_lidvids, single_version_fs, path)

            # Now we ask the Multiversioned to insert these contents
            # as a new version if needed.  It returns the new LIDVID
            # if a new LIDVID is needed, otherwise it returns the old
            # one.
            return self.add_contents_if(is_new, lid, contents, False)

        bundle_segs = [
            strip_segment(name) for name in single_version_fs.listdir("/")
            if is_segment(name)
        ]

        # TODO I can't see any reason why there wouldn't be exactly a
        # single segment, but I'm throwing in an assert to let me know
        # if I'm wrong.
        if len(bundle_segs) != 1:
            raise ValueError(f"bundle_segs: {bundle_segs} is more than 1.")

        changed = False

        for bundle_seg in bundle_segs:
            lid = LID.create_from_parts([str(bundle_seg)])
            orig_lidvid: Optional[LIDVID] = self.latest_lidvid(lid)
            new_lidvid: LIDVID = update_from_lid(lid)
            changed = changed or new_lidvid != orig_lidvid

        return changed
Exemple #32
0
def categorize_filesystem(fs: FS) -> str:
    top_level_listing = fs.listdir("/")
    if not top_level_listing:
        return EMPTY_FS_TYPE
    elif any(name[-1] == "$" for name in top_level_listing):
        return SINGLE_VERSIONED_FS_TYPE
    elif any(is_version_dir(dir) for dir in fs.walk.dirs()):
        return MULTIVERSIONED_FS_TYPE
    else:
        return UNKNOWN_FS_TYPE
Exemple #33
0
 def makedirs(
     self,
     path: str,
     permissions: Optional[Permissions] = None,
     recreate: bool = False,
 ) -> SubFS[FS]:
     return FS.makedirs(self,
                        path,
                        permissions=permissions,
                        recreate=recreate)
 def copydir(self, src, dst, overwrite=False, ignore_errors=False, chunk_size=16384):
     if self.getmeta("read_only"):
         raise errors.UnsupportedError('read only filesystem')
     # FIXME: this is out of date; how to do native tahoe copy?
     # FIXME: Workaround because isfile() not exists on _TahoeLAFS
     FS.copydir(self, src, dst, overwrite, ignore_errors, chunk_size)
Exemple #35
0
 def copydir(self, src, dst, **kwds):
     FS.copydir(self,src,dst,**kwds)
Exemple #36
0
 def movedir(self, src, dst, **kwds):
     FS.movedir(self,src,dst,**kwds)
Exemple #37
0
 def move(self, src, dst, **kwds):
     FS.move(self,src,dst,**kwds)
     path = relpath(normpath(src))
     with self._size_lock:
         self._file_sizes.pop(path,None)
Exemple #38
0
 def close(self):
     self._olefile.close()
     FS.close(self)
Exemple #39
0
    def __init__(self):
        FS.__init__(self, thread_synchronize=True)

        self.fs_sequence = []
        self.fs_lookup =  {}