示例#1
0
    def _save(
        self, data: Union[plt.figure, List[plt.figure],
                          Dict[str, plt.figure]]) -> None:
        save_path = self._get_save_path()

        if isinstance(data,
                      (list, dict)) and self._overwrite and self._exists():
            self._fs.rm(get_filepath_str(save_path, self._protocol),
                        recursive=True)

        if isinstance(data, list):
            for index, plot in enumerate(data):
                full_key_path = get_filepath_str(save_path / f"{index}.png",
                                                 self._protocol)
                self._save_to_fs(full_key_path=full_key_path, plot=plot)
        elif isinstance(data, dict):
            for plot_name, plot in data.items():
                full_key_path = get_filepath_str(save_path / plot_name,
                                                 self._protocol)
                self._save_to_fs(full_key_path=full_key_path, plot=plot)
        else:
            full_key_path = get_filepath_str(save_path, self._protocol)
            self._save_to_fs(full_key_path=full_key_path, plot=data)

        plt.close("all")

        self._invalidate_cache()
示例#2
0
    def _load(self) -> Any:
        load_path = get_filepath_str(self._get_load_path(), self._protocol)

        with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
            imported_backend = importlib.import_module(self._backend)
            return imported_backend.load(fs_file,
                                         **self._load_args)  # type: ignore
示例#3
0
 def _save(self, fig) -> None: 
     """Saves data to the specified filepath.
     """
     # using get_filepath_str ensures that the protocol and path are appended correctly for different filesystems
     save_path = get_filepath_str(self._get_save_path(), self._protocol)
     fig.save(f"{save_path}",**self._save_params)
     
示例#4
0
 def _load(self) -> graph_objects.Figure:
     load_path = get_filepath_str(self._get_load_path(), self._protocol)
     with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
         # read_json doesn't work correctly with file handler, so we have to read the file,
         # decode it manually and pass to the low-level from_json instead.
         return pio.from_json(str(fs_file.read(), "utf-8"),
                              **self._load_args)
示例#5
0
    def _load(self) -> DataLoader:
        load_path = get_filepath_str(self._get_load_path(), self._protocol)

        with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
            dataloader: DataLoader = self.BACKENDS[self._backend].load(
                fs_file, **self._load_args)  # nosec
            return dataloader
示例#6
0
    def _save(self, data: Message) -> None:
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
            Generator(fs_file, **self._generator_args).flatten(data, **self._save_args)

        self._invalidate_cache()
示例#7
0
    def _save(self, data: Image) -> None:
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
            data.save(fs_file, **self._save_args)

        self._invalidate_cache()
示例#8
0
    def _exists(self) -> bool:
        try:
            load_path = get_filepath_str(self._get_load_path(), self._protocol)
        except DataSetError:
            return False

        return self._fs.exists(load_path)
示例#9
0
    def _save(self, data: pd.DataFrame) -> None:
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
            data.to_json(path_or_buf=fs_file, **self._save_args)

        self._invalidate_cache()
    def _save(self, plot: str) -> None:
        save_path = Path(
            get_filepath_str(self._get_save_path(), self._protocol))
        save_path.parent.mkdir(parents=True, exist_ok=True)
        hv.save(plot, save_path, **self._save_args)

        self._invalidate_cache()
示例#11
0
    def _load(self) -> Any:
        load_path = get_filepath_str(self._get_load_path(), self._protocol)

        with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
            return self.BACKENDS[self._backend].load(
                fs_file, **self._load_args
            )  # nosec
示例#12
0
    def _save(self, data: str) -> None:
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
            fs_file.write(data)

        self._invalidate_cache()
示例#13
0
    def _load(self) -> pd.DataFrame:
        load_path = get_filepath_str(self._get_load_path(), self._protocol)

        with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
            data: pd.DataFrame = pd.read_csv(fs_file, **self._load_args)
            data.attrs = self.df_attrs if self.df_attrs else {}
            return data
示例#14
0
    def _load(self) -> (pd.DataFrame, np.ndarray):
        """Loads data from the DICOM file.

        Returns:
            Metadata from the DICOM file as a pandas Dataframe,
            Image data  as a numpy array
        """
        # using get_filepath_str ensures that the protocol and path are appended correctly for different filesystems
        load_path = get_filepath_str(self._filepath, self._protocol)
        with self._fs.open(load_path) as f:
            ds = pydicom.dcmread(f)

            #Convert metadata records to dataframe
            df = pd.DataFrame.from_records([
                (el.name, el.value) for el in ds if el.name not in
                ['Pixel Data', 'File Meta Information Version']
            ])
            #Convert row to columns
            df = df.T
            #Set first line as header
            df.columns = df.iloc[0]
            #Delete first line
            df = df.iloc[1:]
            pixel_array = ds.pixel_array
            return (df, pixel_array)
示例#15
0
    def _save(self, data: Dict) -> None:
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
            json.dump(data, fs_file, **self._save_args)

        self._invalidate_cache()
    def _load(self) -> str:
        load_path = get_filepath_str(self._get_load_path(), self._protocol)

        if str(load_path).endswith(".png"):
            return hv.RGB.load_image(load_path, **self._load_args)
        else:
            raise NotImplementedError("There is no way to convert from an\
                                    arbitrary saved image format to a plot.")
示例#17
0
    def _save(self, data: pd.DataFrame) -> None:
        plot_data = _plotly_express_wrapper(data, self._plotly_args)

        full_key_path = get_filepath_str(self._get_save_path(), self._protocol)
        with self._fs.open(full_key_path, **self._fs_open_args_save) as fs_file:
            plot_data.write_json(fs_file, **self._save_args)

        self._invalidate_cache()
示例#18
0
    def _save(self, data: DGLHeteroGraph) -> None:
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        dgl.save_graphs(save_path, data, **self._save_args)
        # with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
        #     json.dump(json_graph, fs_file)

        self._invalidate_cache()
示例#19
0
    def _save(self, data: networkx.Graph) -> None:
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        json_graph = networkx.node_link_data(data, **self._save_args)
        with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
            json.dump(json_graph, fs_file)

        self._invalidate_cache()
示例#20
0
    def _save(self, data: HoloViews) -> None:
        bytes_buffer = io.BytesIO()
        hv.save(data, bytes_buffer, **self._save_args)

        save_path = get_filepath_str(self._get_save_path(), self._protocol)
        with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
            fs_file.write(bytes_buffer.getvalue())

        self._invalidate_cache()
示例#21
0
    def _save(self, data: pd.DataFrame) -> None:
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        table = pa.Table.from_pandas(data)
        pq.write_table(
            table=table, where=save_path, filesystem=self._fs, **self._save_args
        )

        self.invalidate_cache()
示例#22
0
    def _save(self, data: Union[Dict, pd.DataFrame]) -> None:
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        if isinstance(data, pd.DataFrame):
            data = data.to_dict()

        with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
            yaml.dump(data, fs_file, **self._save_args)

        self._invalidate_cache()
示例#23
0
    def _save(self, data: Union[figure, List[figure], Dict[str,
                                                           figure]]) -> None:
        save_path = self._get_save_path()

        if isinstance(data, list):
            for index, plot in enumerate(data):
                full_key_path = get_filepath_str(save_path / f"{index}.png",
                                                 self._protocol)
                self._save_to_fs(full_key_path=full_key_path, plot=plot)
        elif isinstance(data, dict):
            for plot_name, plot in data.items():
                full_key_path = get_filepath_str(save_path / plot_name,
                                                 self._protocol)
                self._save_to_fs(full_key_path=full_key_path, plot=plot)
        else:
            full_key_path = get_filepath_str(save_path, self._protocol)
            self._save_to_fs(full_key_path=full_key_path, plot=data)

        self._invalidate_cache()
示例#24
0
    def _save(self, data: pd.DataFrame) -> None:
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        buf = BytesIO()
        data.to_feather(buf)

        with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
            fs_file.write(buf.getvalue())

        self._invalidate_cache()
示例#25
0
    def _load(self) -> pd.DataFrame:
        load_args = copy.deepcopy(self._load_args)
        engine = self.engines[self._connection_str]  # type: ignore

        if self._filepath:
            load_path = get_filepath_str(PurePosixPath(self._filepath),
                                         self._protocol)
            with self._fs.open(load_path, mode="r") as fs_file:
                load_args["sql"] = fs_file.read()

        return pd.read_sql_query(con=engine, **load_args)
示例#26
0
    def _load(self) -> tf.keras.Model:
        load_path = get_filepath_str(self._get_load_path(), self._protocol)

        with tempfile.TemporaryDirectory(prefix=self._tmp_prefix) as path:
            if self._is_h5:
                path = str(PurePath(path) / TEMPORARY_H5_FILE)
                self._fs.copy(load_path, path)
            else:
                self._fs.get(load_path, path, recursive=True)

            # Pass the local temporary directory/file path to keras.load_model
            return tf.keras.models.load_model(path, **self._load_args)
示例#27
0
    def _save(self, data: Any) -> None:
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
            try:
                self.BACKENDS[self._backend].dump(data, fs_file,
                                                  **self._save_args)
            except Exception as exc:
                raise DataSetError("{} was not serialized due to: {}".format(
                    str(data.__class__), str(exc))) from exc

        self._invalidate_cache()
示例#28
0
    def _save(self, data: pd.DataFrame) -> None:
        output = BytesIO()
        save_path = get_filepath_str(self._get_save_path(), self._protocol)

        # pylint: disable=abstract-class-instantiated
        with pd.ExcelWriter(output, **self._writer_args) as writer:
            data.to_excel(writer, **self._save_args)

        with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
            fs_file.write(output.getvalue())

        self._invalidate_cache()
示例#29
0
    def _save(self, data: Union[figure, List[figure], Dict[str, figure]]) -> None:
        if isinstance(data, list):
            for index, plot in enumerate(data):
                full_key_path = self._fs.pathsep.join(
                    [
                        get_filepath_str(self._filepath, self._protocol),
                        "{}.png".format(index),
                    ]
                )
                self._save_to_fs(full_key_path=full_key_path, plot=plot)

        elif isinstance(data, dict):
            for plot_name, plot in data.items():
                full_key_path = self._fs.pathsep.join(
                    [get_filepath_str(self._filepath, self._protocol), plot_name]
                )
                self._save_to_fs(full_key_path=full_key_path, plot=plot)

        else:
            full_key_path = get_filepath_str(self._filepath, self._protocol)
            self._save_to_fs(full_key_path=full_key_path, plot=data)

        self.invalidate_cache()
示例#30
0
    def _load(self) -> pd.DataFrame:
        load_args = copy.deepcopy(self._load_args)

        if self._filepath:
            load_path = get_filepath_str(PurePosixPath(self._filepath),
                                         self._protocol)
            with self._fs.open(load_path, mode="r") as fs_file:
                load_args["query"] = fs_file.read()

        return pd.read_gbq(
            project_id=self._project_id,
            credentials=self._credentials,
            **load_args,
        )