Exemplo n.º 1
0
def complete_file_input_values(input_key, input_value):
    """
    Completes the information inside a given file input value. Will alter the given input_value.
    Creates the following keys (if not already present): path, basename, dirname, nameroot, nameext

    :param input_key: An input key as string
    :param input_value: An input value with class 'File'
    """
    # define basename
    if 'basename' in input_value:
        basename = PurePosixPath(input_value['basename'])
    else:
        basename = PurePosixPath(input_key)
        input_value['basename'] = basename.as_posix()

    # define dirname
    if 'dirname' in input_value:
        dirname = PurePosixPath(input_value['dirname'])
    else:
        dirname = default_inputs_dirname()
        input_value['dirname'] = dirname.as_posix()

    # define nameroot, nameext
    # noinspection PyTypeChecker
    nameroot, nameext = os.path.splitext(basename.as_posix())
    input_value['nameroot'] = nameroot
    input_value['nameext'] = nameext

    # define path
    input_value['path'] = (dirname / basename).as_posix()
Exemplo n.º 2
0
 def tryLoad(cls, filesystem: JsonableFilesystem,
             path: PurePosixPath) -> "PrecomputedChunksInfo | Exception":
     url = filesystem.geturl(path.as_posix())
     if not filesystem.exists(path.as_posix()):
         return FileNotFoundError(f"Could not find info file at {url}")
     with filesystem.openbin(path.as_posix(), "r") as f:
         try:
             info_json = f.read().decode("utf8")
             return PrecomputedChunksInfo.from_json_value(
                 json.loads(info_json))
         except Exception:
             return ValueError(
                 f"Could not interpret json info file at {url}")
Exemplo n.º 3
0
class AutoEncoderDataSet(AbstractDataSet):
    def __init__(
        self,
        filepath: str,
        load_args: Optional[Dict[str, Any]] = None,
        save_args: Optional[Dict[str, Any]] = None,
    ) -> None:
        self._filepath = PurePosixPath(filepath)
        self._load_args = load_args if load_args else {}
        self._save_args = save_args if save_args else {}

    def _load(self) -> AutoEncoder:
        state_dict = torch.load(str(self._filepath))
        input_shape = state_dict["encoder_hidden_layer.weight"].shape[1]
        autoencoder = AutoEncoder(input_shape=input_shape)
        autoencoder.load_state_dict(state_dict)
        return autoencoder

    def _save(self, autoencoder: AutoEncoder) -> None:
        torch.save(autoencoder.state_dict(), str(self._filepath),
                   **self._save_args)

    def _exists(self) -> bool:
        return Path(self._filepath.as_posix()).exists()

    def _describe(self) -> Dict[str, Any]:
        return dict(
            filepath=self._filepath,
            load_args=self._load_args,
            save_args=self._save_args,
        )
Exemplo n.º 4
0
class GCNDataSet(AbstractDataSet):
    def __init__(
        self,
        filepath: str,
        load_args: Optional[Dict[str, Any]] = None,
        save_args: Optional[Dict[str, Any]] = None,
    ) -> None:
        self._filepath = PurePosixPath(filepath)
        self._load_args = load_args if load_args else {}
        self._save_args = save_args if save_args else {}

    def _load(self) -> Model:
        state_dict = torch.load(str(self._filepath))
        in_features, hidden_features, out_features = (
            state_dict["in_features.weight"].shape[1],
            state_dict["hidden_features.weight"].shape[1],
            state_dict["out_features.weight"].shape[1],
        )
        model = Model(in_features, hidden_features, out_features)
        model.load_state_dict(state_dict)
        return model

    def _save(self, model: Model) -> None:
        torch.save(model.state_dict(), str(self._filepath), **self._save_args)

    def _exists(self) -> bool:
        return Path(self._filepath.as_posix()).exists()

    def _describe(self) -> Dict[str, Any]:
        return dict(
            filepath=self._filepath,
            load_args=self._load_args,
            save_args=self._save_args,
        )
Exemplo n.º 5
0
    def __init__(
        self,
        *,
        path: PurePosixPath,
        location: Point5D = Point5D.zero(),
        filesystem: JsonableFilesystem,
        tile_shape: Optional[Shape5D] = None,
        spatial_resolution: Optional[Tuple[int, int, int]] = None,
    ):
        raw_data: "np.ndarray[Any, Any]" = skimage.io.imread(filesystem.openbin(path.as_posix())) # type: ignore
        c_axiskeys_on_disk = "yxc"[: len(raw_data.shape)]
        self._data = Array5D(raw_data, axiskeys=c_axiskeys_on_disk, location=location)

        if tile_shape is None:
            tile_shape = Shape5D.hypercube(256).to_interval5d().clamped(self._data.shape).shape

        super().__init__(
            c_axiskeys_on_disk=c_axiskeys_on_disk,
            filesystem=filesystem,
            path=path,
            dtype=self._data.dtype,
            interval=self._data.interval,
            tile_shape=tile_shape,
            spatial_resolution=spatial_resolution,
        )
Exemplo n.º 6
0
    def generate_lorenz(self, beta, rho, sigma, dt, size):
        """
        Generating Lorenz modeled time series
        :param size:
        :param beta:
        :param rho:
        :param sigma:
        :param dt:
        :return:
        """

        file_name = "lorenz_{rho}_{sigma}_{dt}_{size}.npy".format(rho=rho,
                                                                  sigma=sigma,
                                                                  dt=str(dt),
                                                                  size=size)
        path_to_file = PurePosixPath(self.PATH_TO_TS) / file_name
        self.path_to_data = path_to_file
        if path.exists(path_to_file):
            print("-> Time series with parameters rho={rho}, sigma={sigma}, dt={dt}, size={size} already exists". \
                  format(rho=rho, sigma=sigma, dt=str(dt), size=size))
        else:
            print("-> Creating time series with parameters rho={rho}, sigma={sigma}, dt={dt}, size={size}". \
                  format(rho=rho, sigma=sigma, dt=str(dt), size=size))
            rk = RungeKutta(beta=beta, rho=rho, sigma=sigma, dt=dt)
            ts = rk.get_series(n_iterations=size)
            with open(path_to_file.as_posix(), "wb") as f:
                np.save(file=f, arr=ts)
Exemplo n.º 7
0
 def __init__(self,
              *,
              outer_path: Path,
              inner_path: PurePosixPath,
              location: Point5D = Point5D.zero(),
              filesystem: JsonableFilesystem):
     self.outer_path = outer_path
     self.inner_path = inner_path
     self.filesystem = filesystem
     binfile = filesystem.openbin(outer_path.as_posix())
     f = h5py.File(binfile, "r")
     try:
         dataset = f[inner_path.as_posix()]
         if not isinstance(dataset, h5py.Dataset):
             raise ValueError(f"{inner_path} is not a Dataset")
         axiskeys = self.getAxisKeys(dataset)
         self._dataset = cast(h5py.Dataset, dataset)
         tile_shape = Shape5D.create(raw_shape=self._dataset.chunks
                                     or self._dataset.shape,
                                     axiskeys=axiskeys)
         super().__init__(
             tile_shape=tile_shape,
             interval=Shape5D.create(
                 raw_shape=self._dataset.shape,
                 axiskeys=axiskeys).to_interval5d(location),
             dtype=self._dataset.dtype,
             axiskeys=axiskeys,
         )
     except Exception as e:
         f.close()
         raise e
Exemplo n.º 8
0
def init():
    from datetime import datetime
    from flask import Flask
    from flask_bootstrap import Bootstrap
    from flask_nav import Nav, register_renderer
    from pathlib import PurePosixPath
    from .bootstrap import top_nav, CustomBootstrapRenderer
    from .config import PORTAL_NON_ROOT, SECRET_KEY, DEBUG, LAB_NAME, MAX_UPLOAD_SIZE, YANDEX_METRIKA
    from .views import view_bp

    app = Flask(__name__)

    app.config['DEBUG'] = DEBUG
    app.config['SECRET_KEY'] = SECRET_KEY
    app.config['BOOTSTRAP_SERVE_LOCAL'] = DEBUG
    app.config['ERROR_404_HELP'] = False
    app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD_SIZE

    app.jinja_env.globals.update(year=datetime.utcnow,
                                 laboratory=LAB_NAME,
                                 yandex=YANDEX_METRIKA)

    register_renderer(app, 'myrenderer', CustomBootstrapRenderer)
    nav = Nav(app)
    nav.register_element('top_nav', top_nav)
    Bootstrap(app)

    app_url = PurePosixPath('/') / (PORTAL_NON_ROOT or '')
    app.register_blueprint(
        view_bp, url_prefix=app_url.as_posix() if PORTAL_NON_ROOT else None)

    return app
Exemplo n.º 9
0
    def create(
        cls,
        *,
        outer_path: Path,
        inner_path: PurePosixPath,
        filesystem: JsonableFilesystem,
        attributes: N5DatasetAttributes,
    ) -> "N5DatasetSink":
        full_path = outer_path.joinpath(inner_path.as_posix().lstrip("/"))

        filesystem.makedirs(full_path.as_posix(), recreate=True)

        with filesystem.openbin(
                outer_path.joinpath("attributes.json").as_posix(), "w") as f:
            f.write(json.dumps({"n5": "2.0.0"}).encode("utf8"))

        with filesystem.openbin(
                full_path.joinpath("attributes.json").as_posix(), "w") as f:
            f.write(json.dumps(attributes.to_json_data()).encode("utf-8"))

        # create all directories in the constructor to avoid races when processing tiles
        created_dirs: Set[Path] = set()
        for tile in attributes.interval.split(attributes.blockSize):
            dir_path = full_path / attributes.get_tile_path(tile).parent
            if dir_path and dir_path not in created_dirs:
                # print(f"Will create dir at {dir_path}")
                filesystem.makedirs(dir_path.as_posix())
                created_dirs.add(dir_path)

        return N5DatasetSink(
            path=full_path,
            filesystem=filesystem,
            attributes=attributes,
        )
Exemplo n.º 10
0
def walk_launch_files(
    base: PurePosixPath, filter_path: Callable[[PurePosixPath], bool]
) -> Generator[PurePosixPath, Any, None]:
    for root, dirs, files in os.walk(base.as_posix()):
        for path in (os.path.join(root, name) for name in files):
            path: PurePosixPath = PurePosixPath(path)
            if filter_path(PurePosixPath(path)):
                yield path
Exemplo n.º 11
0
def init_app():
    from .config import (PORTAL_NON_ROOT, SECRET_KEY, DEBUG, LAB_NAME, RESIZE_URL, IMAGES_PATH, MAX_UPLOAD_SIZE,
                         YANDEX_METRIKA, VK_ENABLE, JOBS_ENABLE, CGRDB_ENABLE, VIEW_ENABLE)
    from .logins import load_user

    app = Flask(__name__)

    app.config['DEBUG'] = DEBUG
    app.config['SECRET_KEY'] = SECRET_KEY
    app.config['ERROR_404_HELP'] = False

    login_manager = LoginManager()
    login_manager.init_app(app)
    login_manager.login_view = '.login'
    login_manager.user_loader(load_user)

    app_url = PurePosixPath('/') / (PORTAL_NON_ROOT or '')

    if VIEW_ENABLE:
        from flask_bootstrap import Bootstrap
        from flask_misaka import Misaka
        from flask_nav import Nav, register_renderer
        from flask_resize import Resize
        from misaka import HTML_ESCAPE
        from .views import view_bp
        from .views.bootstrap import top_nav, CustomBootstrapRenderer, CustomMisakaRenderer

        app.config['BOOTSTRAP_SERVE_LOCAL'] = DEBUG
        app.config['RESIZE_URL'] = RESIZE_URL
        app.config['RESIZE_ROOT'] = IMAGES_PATH
        app.config['MAX_CONTENT_LENGTH'] = MAX_UPLOAD_SIZE

        app.jinja_env.globals.update(year=datetime.utcnow, laboratory=LAB_NAME, yandex=YANDEX_METRIKA)

        Resize(app)
        register_renderer(app, 'myrenderer', CustomBootstrapRenderer)
        nav = Nav(app)
        nav.register_element('top_nav', top_nav)
        Bootstrap(app)
        Misaka(app, renderer=CustomMisakaRenderer(flags=0 | HTML_ESCAPE), tables=True, autolink=True,
               underline=True, math=True, strikethrough=True, superscript=True, footnotes=True)

        app.register_blueprint(view_bp, url_prefix=app_url.as_posix() if PORTAL_NON_ROOT else None)

    if JOBS_ENABLE:
        from .API import load_jobs
        app.register_blueprint(load_jobs(), url_prefix=(app_url / 'api' / 'jobs').as_posix())
    if CGRDB_ENABLE:
        from .API import load_cgrdb
        app.register_blueprint(load_cgrdb(), url_prefix=(app_url / 'api' / 'db').as_posix())
    if VK_ENABLE:
        from .vk import vk_bp
        app.register_blueprint(vk_bp, url_prefix=(app_url / 'api' / 'vk').as_posix())

    return app
Exemplo n.º 12
0
    def __init__(
        self,
        *,
        datascheme: Optional[DataScheme] = None,
        protocol: Protocol,
        hostname: str,
        port: Optional[int] = None,
        path: PurePosixPath,
        search: Optional[Mapping[str, str]] = None,
        hash_: Optional[str] = None,
        search_quoting_method: SearchQuotingMethod = SearchQuotingMethod.
        QUOTE_PLUS,
    ):
        if not path.is_absolute():
            raise ValueError("Path '{path}' is not absolute")
        path_parts: List[str] = []
        for part in path.as_posix().split("/"):
            if part == "." or part == "":
                continue
            if part == "..":
                if len(path_parts) > 0:
                    _ = path_parts.pop()
            else:
                path_parts.append(part)

        self.datascheme = datascheme
        self.protocol = protocol
        self.hostname = hostname
        self.host = hostname + ("" if port is None else f":{port}")
        self.port = port
        self.path = PurePosixPath("/") / "/".join(path_parts)
        self.search = search or {}
        self.hash_ = hash_
        self.schemeless_raw = f"{protocol}://{self.host}"
        self.schemeless_raw += str(path)
        if self.search:
            if search_quoting_method == SearchQuotingMethod.QUOTE_PLUS:
                quote_via = quote_plus
            else:
                quote_via = quote
            self.schemeless_raw += "?" + urlencode(
                self.search, doseq=True, quote_via=quote_via)
        if self.hash_:
            self.schemeless_raw += "#" + self.hash_

        if self.datascheme:
            self.raw = f"{self.datascheme}+{self.schemeless_raw}"
            self.double_protocol_raw = f"{self.datascheme}://{self.schemeless_raw}"
        else:
            self.raw = self.schemeless_raw
            self.double_protocol_raw = self.raw

        if hostname == "" and protocol not in (Protocol.FILE, Protocol.MEMORY):
            raise ValueError(f"Missing hostname in {self.raw}")
        super().__init__()
Exemplo n.º 13
0
 def __init__(self, *, filesystem: JsonableFilesystem,
              outer_path: PurePosixPath, inner_path: PurePosixPath,
              attributes: N5DatasetAttributes):
     super().__init__(
         dtype=attributes.dataType,
         tile_shape=attributes.blockSize,
         interval=attributes.dimensions.to_interval5d(),
     )
     self.outer_path = outer_path
     self.inner_path = inner_path
     self.full_path = outer_path.joinpath(inner_path.as_posix().lstrip("/"))
     self.attributes = attributes
     self.filesystem = filesystem
Exemplo n.º 14
0
 def __init__(self,
              *,
              outer_path: PurePosixPath,
              inner_path: PurePosixPath,
              location: Point5D = Point5D.zero(),
              filesystem: JsonableFilesystem,
              spatial_resolution: Optional[Tuple[int, int, int]] = None):
     self.outer_path = outer_path
     self.inner_path = inner_path
     self.filesystem = filesystem
     binfile = filesystem.openbin(outer_path.as_posix())
     # FIXME: h5py might not like this if the filesystem isn't OSFS
     f = h5py.File(binfile, "r")  #type: ignore
     try:
         dataset = f[inner_path.as_posix()]
         if not isinstance(dataset, h5py.Dataset):
             raise ValueError(f"{inner_path} is not a Dataset")
         self.axiskeys = self.getAxisKeys(dataset)
         self._dataset = dataset
         tile_shape = Shape5D.create(raw_shape=self._dataset.chunks
                                     or self._dataset.shape,
                                     axiskeys=self.axiskeys)
         base_url = Url.parse(filesystem.geturl(outer_path.as_posix()))
         assert base_url is not None
         super().__init__(
             c_axiskeys_on_disk=self.axiskeys,
             tile_shape=tile_shape,
             interval=Shape5D.create(
                 raw_shape=self._dataset.shape,
                 axiskeys=self.axiskeys).to_interval5d(location),
             dtype=self._dataset.dtype,
             spatial_resolution=spatial_resolution or (1, 1, 1),  # FIXME
             filesystem=filesystem,
             path=self.outer_path)
     except Exception as e:
         f.close()
         raise e
Exemplo n.º 15
0
 def __init__(
     self,
     key: PurePosixPath,
     size: Tuple[int, int, int],
     resolution: Tuple[int, int, int],
     voxel_offset: Optional[Tuple[int, int, int]],
     chunk_sizes: Tuple[Tuple[int, int, int], ...],
     encoding: PrecomputedChunksEncoder,
 ) -> None:
     self.key = PurePosixPath(key.as_posix().lstrip("/"))
     self.size = size
     self.resolution = resolution
     self.voxel_offset = (0, 0, 0) if voxel_offset is None else voxel_offset
     self.chunk_sizes = chunk_sizes
     self.encoding = encoding
     super().__init__()
Exemplo n.º 16
0
class BetterCSVDataSet(AbstractDataSet):
    def __init__(self, filepath, load_args=None, save_args=None):
        self._filepath = PurePosixPath(filepath)
        self.load_args = load_args
        self.save_args = save_args

    def _load(self) -> pd.DataFrame:
        if self.load_args is None:
            self.load_args = {}
        return pd.read_csv(self._filepath, **self.load_args)

    def _save(self, df: pd.DataFrame) -> None:
        if self.save_args is None:
            self.save_args = {}
        df.to_csv(str(self._filepath), **self.save_args)

    def _exists(self) -> bool:
        return Path(self._filepath.as_posix()).exists()

    def _describe(self):
        return self.__dict__
Exemplo n.º 17
0
class BZ2TextDataSet(AbstractDataSet):
    def __init__(self, filepath, load_args=None, save_args=None):
        self._filepath = PurePosixPath(filepath)
        self.load_args = load_args

    def _load(self) -> pd.DataFrame:
        if self.load_args is None:
            self.load_args = {}

        with bz2.open(self._filepath, **self.load_args) as f:
            return f.readlines()

    def _save(self, obj):
        with open(self._filepath, "wb") as f:
            f.write(obj)

    def _exists(self) -> bool:
        return Path(self._filepath.as_posix()).exists()

    def _describe(self):
        return self.__dict__
Exemplo n.º 18
0
def create_n5(
    array: Array5D, *, axiskeys: Optional[str] = None, chunk_size: Shape5D, compression: N5Compressor = RawCompressor()
):
    path = PurePosixPath(tempfile.mkstemp()[1] + ".n5")
    sink = N5DatasetSink(
        outer_path=path,
        inner_path=PurePosixPath("/data"),
        filesystem=OsFs("/"),
        attributes=N5DatasetAttributes(
            dimensions=array.shape,
            blockSize=chunk_size,
            c_axiskeys=axiskeys or array.axiskeys,
            dataType=array.dtype,
            compression=compression,
        )
    )
    sink_writer = sink.create()
    assert not isinstance(sink_writer, Exception)

    for tile in array.split(chunk_size):
        sink_writer.write(tile)
    return path.as_posix()
Exemplo n.º 19
0
class AppendableCSVDataSet(AbstractDataSet):
    """``AppendableCSVDataSet`` loads/saves data from/to a local CSV file opened in
    append mode. It uses pandas to handle the CSV file.

    Example:
    ::

        >>> from kedro.extras.datasets.pandas import AppendableCSVDataSet
        >>> from kedro.extras.datasets.pandas import CSVDataSet
        >>> import pandas as pd
        >>>
        >>> data_1 = pd.DataFrame({'col1': [1, 2], 'col2': [4, 5],
        >>>                      'col3': [5, 6]})
        >>>
        >>> data_2 = pd.DataFrame({'col1': [7, 8], 'col2': [5, 7]})
        >>>
        >>> regular_ds = CSVDataSet(filepath="/tmp/test.csv")
        >>> appendable_ds = AppendableCSVDataSet(
        >>>     filepath="/tmp/test.csv",
        >>> )
        >>>
        >>> regular_ds.save(data_1)
        >>> appendable_ds.save(data_2)
        >>> reloaded = appendable_ds.load()
        >>> assert data_2.equals(reloaded)

    """
    DEFAULT_LOAD_ARGS = {}  # type: Dict[str, Any]
    DEFAULT_SAVE_ARGS = {"index": False}

    def __init__(
        self,
        filepath: str,
        load_args: Dict[str, Any] = None,
        save_args: Dict[str, Any] = None,
    ) -> None:
        """Creates a new instance of ``AppendableCSVDataSet`` pointing to an existing local
        CSV file to be opened in append mode.

        Args:
            filepath: Filepath in POSIX format to an existing local CSV file.
            load_args: Pandas options for loading CSV files.
                Here you can find all available arguments:
                https://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_csv.html
            save_args: Pandas options for saving CSV files.
                Here you can find all available arguments:
                https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_csv.html
                All defaults are preserved, but "index", which is set to False.
        """
        self._filepath = PurePosixPath(filepath)

        # Handle default load and save arguments
        self._load_args = deepcopy(self.DEFAULT_LOAD_ARGS)
        if load_args is not None:
            self._load_args.update(load_args)

        self._save_args = deepcopy(self.DEFAULT_SAVE_ARGS)
        if save_args is not None:
            self._save_args.update(save_args)

        # Use only append mode
        self._save_args["mode"] = "a"

    def _describe(self) -> Dict[str, Any]:
        return dict(filepath=self._filepath,
                    load_args=self._load_args,
                    save_args=self._save_args)

    def _load(self) -> pd.DataFrame:
        return pd.read_csv(str(self._filepath), **self._load_args)

    def _save(self, data: pd.DataFrame) -> None:
        # pylint: disable=abstract-class-instantiated
        try:
            if self._exists():
                self._save_args["header"] = False
            data.to_csv(str(self._filepath), **self._save_args)
        except FileNotFoundError:
            raise DataSetError(f"`{self._filepath}` CSV file not found. "
                               f"The file cannot be opened in "
                               f"append mode.")

    def _exists(self) -> bool:
        return Path(self._filepath.as_posix()).is_file()
Exemplo n.º 20
0
def simulate_fragmentation(rbc_path: Path,
                           mismatches: int,
                           seed_size: int,
                           subseed_size: int,
                           mode: str,
                           timeout: Optional[float] = None,
                           cutoff: bool = True,
                           threads: int = 0) -> Tuple[float, int]:
    user_id = uuid.uuid4()
    host_seed = secrets.token_bytes(seed_size)
    client_seed = corrupt_key(host_seed, mismatches)

    host_subseeds = [
        bytes(host_seed[index:index + subseed_size])
        for index in range(0, len(host_seed), subseed_size)
    ]
    client_subseeds = [
        bytes(client_seed[index:index + subseed_size])
        for index in range(0, len(client_seed), subseed_size)
    ]
    args = []

    for host_subseed, client_subseed in zip(host_subseeds, client_subseeds):
        nonce = secrets.token_bytes(len(host_seed) - subseed_size)
        host_subkey = host_subseed + nonce
        client_subkey = client_subseed + nonce

        subargs = [host_subkey.hex()]

        if mode == "aes":
            aes = AES.new(client_subkey, AES.MODE_ECB)
            subargs += [aes.encrypt(user_id.bytes).hex(), str(user_id)]
        elif mode == "chacha20":
            chacha20_nonce = secrets.token_bytes(CHACHA20_NONCE_SIZE)
            chacha20 = ChaCha20.new(key=client_subkey, nonce=chacha20_nonce)
            client_key = chacha20.encrypt(user_id.bytes)

            iv = chacha20.nonce
            iv = bytes(CHACHA20_OPENSSL_NONCE_SIZE - len(iv)) + iv

            subargs += [client_key.hex(), user_id.hex, iv.hex()]
        elif mode == "ecc":
            client_priv_key = ECC.construct(curve=EC_CURVE,
                                            d=int.from_bytes(
                                                client_subkey, "big"))
            subargs += [
                get_ec_public_key_bytes(client_priv_key, compress=False).hex()
            ]
        elif mode in hash_modes:
            h = hashlib.new(mode.replace('-', '_'), client_subkey)
            subargs += [h.hexdigest()]
        elif mode == "kang12":
            subargs += [KangarooTwelve(client_subkey, b'', KANG12_SIZE).hex()]
        else:
            print(f"Error: Mode '{mode}' is not recognized.", file=sys.stderr)
            sys.exit(1)

        args.append(subargs)

    duration = 0
    key_count = 0

    for subargs in args:
        # Call rbc_validator over wsl with the UUID, the full server subkey, the client cipher, and only
        # the necessary subkey size set
        # Only extract the stderr output in verbose mode to get the actual time taken searching in text
        # mode, and make sure to check if the return code was zero or not
        env_args = []

        rbc_path_abs = rbc_path.resolve()

        if sys.platform == "win32":
            env_args.append("wsl")
            rbc_path_abs = PurePosixPath("/mnt",
                                         rbc_path_abs.drive[:-1].lower(),
                                         *rbc_path_abs.parts[1:])

        env_args += [
            rbc_path_abs.as_posix(), f"--mode={mode}",
            f"--subkey={subseed_size * 8}", f"--threads={threads}", "-v", "-c"
        ]

        if not cutoff:
            env_args.append("-a")

        env_args += subargs

        try:
            validator_proc = subprocess.run(
                env_args,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.PIPE,
                universal_newlines=True,
                check=True,
                timeout=None if timeout is None else timeout - duration)
        except subprocess.TimeoutExpired:
            return float("inf"), key_count

        lines = [line for line in validator_proc.stderr.split("\n")]
        # Get the first line such that "Clock" is contained within it.
        clock_line = next(line for line in lines if re.search(r"Clock", line))
        count_line = next(line for line in lines
                          if re.search(r"searched", line))
        # Get only the decimal output (in seconds) and increment duration by its value
        duration += float(clock_line.split(" ")[3])
        key_count += int(count_line.split(" ")[3])

    return duration, key_count
Exemplo n.º 21
0
class AppendableExcelDataSet(AbstractDataSet):
    """``AppendableExcelDataSet`` loads/saves data from/to a local Excel file opened in
    append mode. It uses pandas to handle the Excel file.

    Example:
    ::

        >>> from kedro.extras.datasets.pandas import AppendableExcelDataSet
        >>> from kedro.extras.datasets.pandas import ExcelDataSet
        >>> import pandas as pd
        >>>
        >>> data_1 = pd.DataFrame({'col1': [1, 2], 'col2': [4, 5],
        >>>                      'col3': [5, 6]})
        >>>
        >>> data_2 = pd.DataFrame({'col1': [7, 8], 'col2': [5, 7]})
        >>>
        >>> regular_ds = ExcelDataSet(filepath="/tmp/test.xlsx")
        >>> appendable_ds = AppendableExcelDataSet(
        >>>     filepath="/tmp/test.xlsx",
        >>>     save_args={"sheet_name": "my_sheet"},
        >>>     load_args={"sheet_name": "my_sheet"}
        >>> )
        >>>
        >>> regular_ds.save(data_1)
        >>> appendable_ds.save(data_2)
        >>> reloaded = appendable_ds.load()
        >>> assert data_2.equals(reloaded)

    """

    DEFAULT_LOAD_ARGS = {"engine": "openpyxl"}
    DEFAULT_SAVE_ARGS = {"index": False}

    def __init__(
        self,
        filepath: str,
        load_args: Dict[str, Any] = None,
        save_args: Dict[str, Any] = None,
    ) -> None:
        """Creates a new instance of ``AppendableExcelDataSet`` pointing to an existing local
        Excel file to be opened in append mode.

        Args:
            filepath: Filepath in POSIX format to an existing local Excel file.
            load_args: Pandas options for loading Excel files.
                Here you can find all available arguments:
                https://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_excel.html
                All defaults are preserved, but "engine", which is set to "openpyxl".
            save_args: Pandas options for saving Excel files.
                Here you can find all available arguments:
                https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_excel.html
                All defaults are preserved, but "index", which is set to False.
                If you would like to specify options for the `ExcelWriter`,
                you can include them under "writer" key. Here you can
                find all available arguments:
                https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.ExcelWriter.html
                Note: `mode` option of `ExcelWriter` is set to `a` and it can not be overridden.
        """
        self._filepath = PurePosixPath(filepath)

        # Handle default load and save arguments
        self._load_args = deepcopy(self.DEFAULT_LOAD_ARGS)
        if load_args is not None:
            self._load_args.update(load_args)

        save_args = deepcopy(save_args) or {}
        self._save_args = deepcopy(self.DEFAULT_SAVE_ARGS)
        self._writer_args = save_args.pop("writer", {})  # type: Dict[str, Any]
        self._writer_args.setdefault("engine", "openpyxl")
        if save_args is not None:
            self._save_args.update(save_args)

        # Use only append mode
        self._writer_args["mode"] = "a"

    def _describe(self) -> Dict[str, Any]:
        return dict(
            filepath=self._filepath,
            load_args=self._load_args,
            save_args=self._save_args,
            writer_args=self._writer_args,
        )

    def _load(self) -> pd.DataFrame:
        return pd.read_excel(str(self._filepath), **self._load_args)

    def _save(self, data: pd.DataFrame) -> None:
        # pylint: disable=abstract-class-instantiated
        try:
            with pd.ExcelWriter(str(self._filepath),
                                **self._writer_args) as writer:
                data.to_excel(writer, **self._save_args)
        except FileNotFoundError as exc:
            raise DataSetError(
                f"`{self._filepath}` Excel file not found. The file cannot be opened in "
                f"append mode.") from exc

    def _exists(self) -> bool:
        return Path(self._filepath.as_posix()).is_file()
Exemplo n.º 22
0
class DataSaver:
    """
    Class that implements methods to save models to volumes and upload them to database
    """
    def __init__(self, local_path, base_name, template, **kwargs):
        """

        :param local_path:
        :param base_name:
        :param template:
        :param kwargs:
        """
        print("DEBUG", kwargs)
        print(template.astype(str))
        self.LOCAL_PATH = PurePosixPath(local_path)
        self.model_name = base_name + "_" + "_".join(template.astype(str))
        self.model_name = self.model_name + "_" + "_".join(
            list(map(str, kwargs.values())))
        self.local_path_to_model = self.LOCAL_PATH / self.model_name

    # TODO: Hash functions for model's names to distinguish them

    def _check_paths(self):
        """
        Checks local paths and tries to create them if they don't exist.
        Throws exception if failed.
        :return: True if path exists
        """
        if path.exists(self.LOCAL_PATH):
            return None
        else:
            print(
                "-> Creating local mounted path inside the container. Pay attention!"
            )
            try:
                makedirs(self.LOCAL_PATH.as_posix())
            except Exception:
                raise Exception("Could not create mounted path")

    def _checks_connections(self):
        """
        Checks
        :return:
        """
        return None

    def save_to_volume(self, obj):
        """
        Save to local disk to docker-volume
        :return:
        """
        self._check_paths()

        path_to_model = self.LOCAL_PATH / self.model_name
        with open(path_to_model, "wb") as f:
            dump(obj, f)

    def _load_to_database(self):
        """
        Loads data to database
        :return:
        """
        pass
Exemplo n.º 23
0
def do_save_project(filesystem: BucketFs, file_path: PurePosixPath,
                    workflow_contents: bytes):
    with filesystem.openbin(file_path.as_posix(), "w") as f:
        f.write(workflow_contents)
Exemplo n.º 24
0
 def absolute(self, path: PurePosixPath):
     path = '/' / path
     path = (self._basedir / path.as_posix()[1:])  #type: Path
     path.relative_to(self._basedir)
     return path
Exemplo n.º 25
0
 def save_project(self, fs: JsonableFilesystem, path: PurePosixPath) -> int:
     with fs.openbin(path.as_posix(), "w") as f:
         return f.write(self.get_ilp_contents())
Exemplo n.º 26
0
def do_load_project_bytes(filesystem: BucketFs,
                          file_path: PurePosixPath) -> bytes:
    with filesystem.openbin(file_path.as_posix(), "r") as f:
        return f.read()
Exemplo n.º 27
0
class ClusterRun:
    def __init__(self,
                 array_run_obj,
                 anat_file_path,
                 physio_file_path,
                 suffix=""):

        try:
            self.cluster_workspace = PurePosixPath(
                parameter_finder(array_run_obj.anatomy_df,
                                 'cluster_workspace'))
        except NameError:
            raise ParameterNotFoundError(
                "cluster_workspace is not defined for running CxSystem on cluster"
            )
        assert self.cluster_workspace.is_absolute(), \
            "cluster_workspace {} must be an absolute path with explicit [remote] home directory path".format(self.cluster_workspace.as_posix())

        try:
            self.cluster_address = parameter_finder(array_run_obj.anatomy_df,
                                                    'cluster_address')
        except NameError:
            raise ParameterNotFoundError(
                "cluster_address is not defined for running CxSystem on cluster"
            )

        try:
            self.cluster_login_node = parameter_finder(
                array_run_obj.anatomy_df, 'cluster_login_node')
        except NameError:
            print(
                " -  No cluster login node found. Directly conencting to cluster address {}"
                .format(self.cluster_address))
            self.cluster_login_node = '--'

        # # the following call will check if the cluster is available or not, but it needs root access
        # self.ping_cluster()

        try:
            self.cluster_username = parameter_finder(array_run_obj.anatomy_df,
                                                     'cluster_username')
            assert self.cluster_username != 'username', \
                "Cluster username must be changed in the configuration file, currently it is the default value 'username'"
            print(" -  Loggin in with user '%s'" % self.cluster_username)
        except NameError:
            self.cluster_username = input(' -  Enter cluster username: '******'password')
        except NameError:
            if 'CLUSTERPASS' in os.environ.keys():
                self.password = os.environ['CLUSTERPASS']
            else:
                self.password = getpass.getpass(
                    ' -  Enter password for user {}: '
                    ''.format(self.cluster_username))

        self.suffix = suffix
        print(" -  temp file suffix is %s" % self.suffix)
        self.client = paramiko.SSHClient()
        self.client.load_system_host_keys()
        self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        if self.cluster_login_node != '--':
            print(" -  Connecting to login node {}".format(
                self.cluster_login_node))
            sock = paramiko.ProxyCommand("ssh {}@{} nc {} 22".format(
                self.cluster_username, self.cluster_login_node,
                self.cluster_address))
            sock.settimeout(30)
            self.client.connect(self.cluster_address,
                                port=22,
                                username=self.cluster_username,
                                password=self.password,
                                sock=sock)
        else:
            self.client.connect(self.cluster_address,
                                port=22,
                                username=self.cluster_username,
                                password=self.password)

        print(" -  Connected to %s" % self.cluster_address)
        print(" -  Creating workspace folder if not exists")
        self.ssh_commander('mkdir -p {}'.format(
            self.cluster_workspace.as_posix()))
        scp = SCPClient(self.client.get_transport())

        if 'json' in anat_file_path.suffix.lower():
            converter = fileconverter.ConfigConverter(
                anat_file_path.as_posix())
            anat_file_path = Path(converter.save_as_csv(overwrite=True))
        if 'json' in physio_file_path.suffix.lower():
            converter = fileconverter.ConfigConverter(
                physio_file_path.as_posix())
            physio_file_path = Path(converter.save_as_csv(overwrite=True))

        print(" -  Transferring configuration files ...")

        self.remote_anat_filename = '_tmp_anat_config{}.csv'.format(
            self.suffix)
        self.remote_phys_filename = '_tmp_physio_config{}.csv'.format(
            self.suffix)
        self.local_workspace_unexpanded = Path(
            parameter_finder(array_run_obj.anatomy_df, 'workspace_path'))
        self.local_workspace = Path(
            parameter_finder(array_run_obj.anatomy_df,
                             'workspace_path')).expanduser()
        self.local_cluster_folder = self.local_workspace.joinpath(
            'cluster_run' + self.suffix)
        if not self.local_cluster_folder.is_dir():
            os.mkdir(self.local_cluster_folder.as_posix())
        try:
            imported_connections_file = Path(
                parameter_finder(array_run_obj.anatomy_df,
                                 'import_connections_from'))
            if imported_connections_file.is_file():
                scp.put(imported_connections_file.as_posix(),
                        self.cluster_workspace.as_posix())
                new_path = Path('./').joinpath(
                    imported_connections_file.name).as_posix()
                change_parameter_value_in_file(
                    anat_file_path.as_posix(),
                    self.local_cluster_folder.joinpath(anat_file_path.name),
                    'import_connections_from', new_path)
                anat_file_path = self.local_cluster_folder.joinpath(
                    anat_file_path.name)
        except TypeError:  # this is when the value is # or -- for instance
            pass
        scp.put(
            anat_file_path.as_posix(),
            self.cluster_workspace.joinpath(
                self.remote_anat_filename).as_posix())
        scp.put(
            physio_file_path.as_posix(),
            self.cluster_workspace.joinpath(
                self.remote_phys_filename).as_posix())

        # ask user to set the number of nodes, time and memory:
        print(
            " -  Please check the default csc_puhti.job file and set the time, memory and uncomment and enter email address if you wish."
            "\nNote that the number of nodes in default slurm file should always be set to 1."
            " Instead you should enter the number of nodes in the CxSystem network config file. "
            "\nAlso the default number of CPUs=16 does not need to be changed most of the times. "
        )

        self.slurm_file_path = Path(
            parameter_finder(array_run_obj.anatomy_df,
                             'cluster_job_file_path')).expanduser()
        if not self.slurm_file_path.is_file():
            if not self.slurm_file_path.is_absolute():
                raise RelativePathError(
                    "\nSlurm file {} not found in local workspace. Make sure the path to the file is "
                    "absolute".format(self.slurm_file_path.as_posix()))
            else:
                raise FileNotFoundError("\nSlurm file {} not found".format(
                    self.slurm_file_path.as_posix()))

        # updating remote cxsystem2
        self.update_remote_cxsystem2(self.slurm_file_path,
                                     self.cluster_workspace)

        # building slurm :
        for item_idx, item in enumerate(array_run_obj.clipping_indices):
            with open(self.slurm_file_path.as_posix(), 'r') as sl1:
                remote_slurm_filename = "_tmp_slurm{}_part{}.job".format(
                    self.suffix, item_idx)
                with open(
                        self.local_cluster_folder.joinpath(
                            remote_slurm_filename).as_posix(),
                        'w') as sl2:  # wb -> w
                    for line in sl1:
                        sl2.write(line)
                    try:
                        sl2.write(
                            "python -c "
                            "\"from cxsystem2.core.cxsystem import CxSystem as cxs; "
                            " cx = cxs('{anatomy}','{physio}', cluster_run_start_idx={cluster_start},cluster_run_step={cluster_step}); "
                            "cx.run()\"".format(
                                anatomy=self.remote_anat_filename,
                                physio=self.remote_phys_filename,
                                cluster_start=item,
                                cluster_step=array_run_obj.clipping_indices[
                                    item_idx + 1] -
                                array_run_obj.clipping_indices[item_idx]))
                    except IndexError:
                        sl2.write(
                            "python -c "
                            "\"from cxsystem2.core.cxsystem import CxSystem as cxs; "
                            " cx = cxs('{anatomy}','{physio}', cluster_run_start_idx={cluster_start},cluster_run_step={cluster_step}); "
                            "cx.run()\"".format(
                                anatomy=self.remote_anat_filename,
                                physio=self.remote_phys_filename,
                                cluster_start=item,
                                cluster_step=array_run_obj.total_configs -
                                array_run_obj.clipping_indices[item_idx]))

            scp.put(
                self.local_cluster_folder.joinpath(
                    remote_slurm_filename).as_posix(),
                self.cluster_workspace.joinpath(
                    remote_slurm_filename).as_posix())
        print(" -  Slurm file generated and copied to cluster")
        self.channel = self.client.invoke_shell()
        for item_idx, item in enumerate(array_run_obj.clipping_indices):
            remote_slurm_filename = "_tmp_slurm{}_part{}.job".format(
                self.suffix, item_idx)
            if platform == 'win32':
                print(" -  Converting the file using dos2unix")
                self.channel.send('cd {} && dos2unix {}\n'.format(
                    self.cluster_workspace.as_posix(), remote_slurm_filename))
                time.sleep(1)
            self.channel.send('cd {} && sbatch {}\n'.format(
                self.cluster_workspace.as_posix(), remote_slurm_filename))
            print(" -  Job file {} submitted".format(remote_slurm_filename))
            time.sleep(1)
        cluster_metadata = \
            {'cluster_address': self.cluster_address,
             'cluster_login_node': self.cluster_login_node,
             'cluster_username': self.cluster_username,
             'local_workspace_unexpanded': self.local_workspace_unexpanded.as_posix(),
             'local_workspace': self.local_workspace.as_posix(),
             'local_cluster_run_folder': self.local_cluster_folder.as_posix(),
             'local_cluster_run_download_folder': self.local_cluster_folder.joinpath('downloads'),
             'cluster_workspace': self.cluster_workspace.as_posix(),
             'cluster_simulation_folder': self.cluster_workspace.joinpath(parameter_finder(array_run_obj.anatomy_df, 'simulation_title')).as_posix(),
             'suffix': self.suffix,
             'timestamp': datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-7]}
        with open(
                self.local_cluster_folder.joinpath(
                    'cluster_metadata{}.pkl'.format(self.suffix)), 'wb') as ff:
            pickle.dump(cluster_metadata, ff)
        print(
            " -  Cluster metadata saved. To download the result and clean the environments after getting the email,"
            " run the following command in the terminal:\n")
        print("cxcluster " + self.local_cluster_folder.joinpath(
            'cluster_metadata{}.pkl'.format(self.suffix)).as_posix())

    def ping_cluster(self):
        try:  # check if the cluster address is ip or hostname
            socket.inet_aton(self.cluster_address)
            cluster_ip = self.cluster_address
        except OSError:
            cluster_ip = socket.gethostbyname(self.cluster_address)
        p = ping(cluster_ip, timeout=3)
        if not p:
            raise ClusterNotReachableError("Cluster node is not reachable")

    def ssh_commander(self, command, print_flag=False):
        stdin, stdout, stderr = self.client.exec_command(command, get_pty=True)
        out = stdout.read(),
        if print_flag is True:
            print(out[0])
        return out[0]

    def update_remote_cxsystem2(self, slurm_path, remote_workspace):
        slurm_path = Path(slurm_path)
        remote_workspace = Path(remote_workspace)
        module_name = self.find_remote_python_module(slurm_path)
        self.ssh_commander('mkdir -p {}'.format(
            self.cluster_workspace.as_posix()))

        # Query for valid cxsystem, install/update if necessary, report
        print(" -  Checking CxSystem2 on cluster")

        # Should be empty string for existing git repo
        git_repo_error_message = self.ssh_commander(
            'source ~/.bash_profile ; '
            'source ~/.bashrc ; '
            'cd {workspace} ; '
            'cd CxSystem2 ; '
            'git -C . rev-parse'.format(
                workspace=remote_workspace.as_posix())).decode('utf-8')

        if not git_repo_error_message:
            git_basename = self.ssh_commander(
                'cd {workspace}/CxSystem2 ; '
                'git rev-parse --show-toplevel'.format(
                    workspace=remote_workspace.as_posix())).decode('utf-8')
            commit_HEAD_hash = self.ssh_commander(
                'source ~/.bash_profile ; '
                'source ~/.bashrc ; '
                'cd {workspace} ; '
                'cd CxSystem2 ; '
                'git rev-parse --short HEAD'.format(
                    workspace=remote_workspace.as_posix())).decode('utf-8')
            git_branch = self.ssh_commander(
                'source ~/.bash_profile ; '
                'source ~/.bashrc ; '
                'cd {workspace} ; '
                'cd CxSystem2 ; '
                'git rev-parse --abbrev-ref HEAD'.format(
                    workspace=remote_workspace.as_posix())).decode('utf-8')
            print(
                f" -  The git repo is {git_basename}    branch is {git_branch}    commit HEAD hash is {commit_HEAD_hash}"
            )
            print(f" -  No need to download/install")
        else:
            print(" -  Updating CxSystem2 on cluster")
            print(
                self.ssh_commander(
                    'source ~/.bash_profile ; '
                    'source ~/.bashrc ; '
                    'cd {workspace} ; '
                    'git clone https://github.com/VisualNeuroscience-UH/CxSystem2 ; '
                    'cd CxSystem2 ; '
                    'git pull ; '.format(workspace=remote_workspace.as_posix()
                                         )).decode('utf-8'))
            print(
                self.ssh_commander(
                    'bash -lc \''
                    'source ~/.bash_profile ; '
                    'source ~/.bashrc ; '
                    'echo $PATH; '
                    'module load {module} ;'
                    'cd {cxfolder} ; '
                    'python -m pip install -Ue . --user\''.format(
                        module=module_name,
                        cxfolder=remote_workspace.joinpath(
                            'CxSystem2').as_posix())).decode('utf-8'))

    def find_remote_python_module(self, slurm_path):
        module_name = ''
        slurm_path = Path(slurm_path)
        with open(slurm_path.as_posix()) as f:
            for line in f:
                if 'module load' in line.lower() and 'python' in line.lower():
                    module_name = line.split(' ')[-1].strip('\n')
        print(" -  Remote module name is {}".format(module_name))
        return module_name
Exemplo n.º 28
0
class CmdRequestHandler(BaseRequestHandler):
    "Handle a FTP session on Command port"

    def __init__(self, request, client_address, server, filesystem):
        self._fs = filesystem
        self._cwd = PurePosixPath('/')
        BaseRequestHandler.__init__(self, request, client_address, server)

    def handle(self):
        self.request.settimeout(30)
        try:
            self.reply(220)  #Service ready for new user.
            while True:
                cmd = self.request.recv(8192).decode().strip().split(
                    maxsplit=1)  #type: List[str]
                if not cmd:
                    break
                print(cmd)
                try:
                    handler = self.__getattribute__('handle_' + cmd[0].upper())
                except AttributeError:
                    self.handle_unknown(*cmd)
                else:
                    handler(*cmd[1:])
                    # else:
                    #     print(handler.__code__.co_argcount)
                    #     self.reply(501)

        except socket.timeout:  #pylint: disable=E1101
            pass
        finally:
            print('closed\r\n')

    def handle_unknown(self, *args):
        if args[0] in ('ACCT', 'ALLO', 'SITE'):
            self.reply(
                202)  #Command not implemented, superfluous at this site.
        else:
            self.reply(502)  #Command not implemented.

    def handle_USER(self, user=None):  #Minimum implementation
        # self.reply(331) #User name okay, need password.
        self.reply(230)  #User logged in, proceed.

    # def handle_PASS(self, pwd=None):
    #     self.reply(230) #User logged in, proceed.

    def handle_PWD(self):
        self.reply(257, '"%s"' % self._cwd.as_posix())  #"PATHNAME" created.

    def handle_CWD(self, path):
        path = self._cwd / path

        if self._fs.is_dir(path):
            self._cwd = path
            self.reply(250)  #Requested file action okay, completed.
        else:
            self.reply(
                550
            )  #Requested action not taken. File unavailable (e.g., file not found, no access).

    def handle_MKD(self, path):
        path = self._cwd / path

        try:
            self._fs.mkdir(path)
            self.reply(257, '"%s"' % path.as_posix())  #"PATHNAME" created.
        except (ValueError, FileExistsError):
            self.reply(550)

    def handle_RMD(self, path):
        path = self._cwd / path

        self.reply(250)  #Requested file action okay, completed.

    # def handle_SYST(self):
    #     self.reply(215, "UNIX ")

    # def handle_FEAT(self):
    #     self.reply(211) #no-features

    def handle_TYPE(self, typecode):  #Minimum implementation
        self.reply(200)  #200 Command okay.

    # def handle_MODE(self, mode): #Minimum implementation
    #     pass
    # def handle_QUIT(self): #Minimum implementation
    #     pass
    def handle_PORT(self, port):  #Minimum implementation
        try:
            h1, h2, h3, h4, p1, p2 = map(int, port.split(','))
            data_host = ('.'.join(map(str, (h1, h2, h3, h4))), p1 * 256 + p2)
        except ValueError:
            self.reply(501)
        else:
            if data_host[0] != self.request.getpeername()[0]:
                self.reply(501)
            else:
                self._data_host = data_host
                self.reply(200)  #200 Command okay.

    # def handle_STRU(self, structure): #Minimum implementation
    #     pass

    def recv_data(self):
        self.reply(150)  #File status okay; about to open data connection.
        print(self._data_host)
        with socket.create_connection(self._data_host) as conn:
            while True:
                data = conn.recv(8192)
                if not data:
                    break
                yield data
        self.reply(
            226)  #Closing data connection. Requested file action successful

    def send_data(self, iterator):
        self.reply(150)  #File status okay; about to open data connection.
        with socket.create_connection(self._data_host) as conn:
            for data in iterator:
                print(data)
                conn.sendall(data)
        self.reply(
            226)  #Closing data connection. Requested file action successful

    def handle_STOR(self, path):  #Minimum implementation
        path = self._cwd / path
        self._fs.write_bytes(path, self.recv_data())

    def handle_RETR(self, path):  #Minimum implementation
        path = self._cwd / path
        self.send_data(self._fs.read_bytes(path))

    # def handle_NOOP(self): #Minimum implementation
    #     pass

    def handle_SIZE(self, path):
        path = self._cwd / path
        self.reply(312, '0')

    def handle_LIST(self, path='.'):
        path = self._cwd / path

        if not self._fs.is_dir(path):
            self.reply(550)
        else:
            self.send_data(((entry + '\r\n').encode()
                            for entry in self._fs.iterdir(path)))

    def reply(self, code, *args):
        print(code, args)
        self.request.send(("%d %s\r\n" % (code, " ".join(args))).encode())
Exemplo n.º 29
0
class PrecomputedChunksScale:
    def __init__(
        self,
        key: PurePosixPath,
        size: Tuple[int, int, int],
        resolution: Tuple[int, int, int],
        voxel_offset: Optional[Tuple[int, int, int]],
        chunk_sizes: Tuple[Tuple[int, int, int], ...],
        encoding: PrecomputedChunksEncoder,
    ) -> None:
        self.key = PurePosixPath(key.as_posix().lstrip("/"))
        self.size = size
        self.resolution = resolution
        self.voxel_offset = (0, 0, 0) if voxel_offset is None else voxel_offset
        self.chunk_sizes = chunk_sizes
        self.encoding = encoding
        super().__init__()

    @classmethod
    def from_datasource(
            cls, *, datasource: DataSource, key: PurePosixPath,
            encoding: PrecomputedChunksEncoder) -> "PrecomputedChunksScale":
        return PrecomputedChunksScale(
            key=key,
            chunk_sizes=tuple([
                (datasource.tile_shape.x, datasource.tile_shape.y,
                 datasource.tile_shape.z)
            ]),
            size=(datasource.shape.x, datasource.shape.y, datasource.shape.z),
            resolution=datasource.spatial_resolution,
            voxel_offset=(datasource.location.x, datasource.location.y,
                          datasource.location.z),
            encoding=encoding)

    def to_json_value(self) -> JsonObject:
        return {
            "key": self.key.as_posix(),
            "size": self.size,
            "resolution": self.resolution,
            "voxel_offset": self.voxel_offset,
            "chunk_sizes": self.chunk_sizes,
            "encoding": self.encoding.to_json_value(),
        }

    @classmethod
    def from_json_value(cls, value: JsonValue) -> "PrecomputedChunksScale":
        value_obj = ensureJsonObject(value)
        return PrecomputedChunksScale(
            key=PurePosixPath(ensureJsonString(value_obj.get("key"))),
            size=ensureJsonIntTripplet(value_obj.get("size")),
            resolution=ensureJsonIntTripplet(value_obj.get("resolution")),
            voxel_offset=ensureOptional(ensureJsonIntTripplet,
                                        value_obj.get("voxel_offset")),
            chunk_sizes=tuple([
                ensureJsonIntTripplet(v)
                for v in ensureJsonArray(value_obj.get("chunk_sizes"))
            ]),
            encoding=PrecomputedChunksEncoder.from_json_value(
                value_obj.get("encoding")),
        )

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, PrecomputedChunksScale):
            return False
        return (self.key == other.key and self.size == other.size
                and self.resolution == other.resolution
                and self.voxel_offset == other.voxel_offset
                and self.chunk_sizes == other.chunk_sizes
                and self.encoding == other.encoding)