def _save(
self, data: Union[plt.figure, List[plt.figure],
Dict[str, plt.figure]]) -> None:
save_path = self._get_save_path()
if isinstance(data,
(list, dict)) and self._overwrite and self._exists():
self._fs.rm(get_filepath_str(save_path, self._protocol),
recursive=True)
if isinstance(data, list):
for index, plot in enumerate(data):
full_key_path = get_filepath_str(save_path / f"{index}.png",
self._protocol)
self._save_to_fs(full_key_path=full_key_path, plot=plot)
elif isinstance(data, dict):
for plot_name, plot in data.items():
full_key_path = get_filepath_str(save_path / plot_name,
self._protocol)
self._save_to_fs(full_key_path=full_key_path, plot=plot)
else:
full_key_path = get_filepath_str(save_path, self._protocol)
self._save_to_fs(full_key_path=full_key_path, plot=data)
plt.close("all")
self._invalidate_cache()
def _load(self) -> Any:
load_path = get_filepath_str(self._get_load_path(), self._protocol)
with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
imported_backend = importlib.import_module(self._backend)
return imported_backend.load(fs_file,
**self._load_args) # type: ignore
def _save(self, fig) -> None:
"""Saves data to the specified filepath.
"""
# using get_filepath_str ensures that the protocol and path are appended correctly for different filesystems
save_path = get_filepath_str(self._get_save_path(), self._protocol)
fig.save(f"{save_path}",**self._save_params)
def _load(self) -> graph_objects.Figure:
load_path = get_filepath_str(self._get_load_path(), self._protocol)
with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
# read_json doesn't work correctly with file handler, so we have to read the file,
# decode it manually and pass to the low-level from_json instead.
return pio.from_json(str(fs_file.read(), "utf-8"),
**self._load_args)
def _load(self) -> DataLoader:
load_path = get_filepath_str(self._get_load_path(), self._protocol)
with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
dataloader: DataLoader = self.BACKENDS[self._backend].load(
fs_file, **self._load_args) # nosec
return dataloader
def _save(self, data: Message) -> None:
save_path = get_filepath_str(self._get_save_path(), self._protocol)
with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
Generator(fs_file, **self._generator_args).flatten(data, **self._save_args)
self._invalidate_cache()
def _save(self, data: Image) -> None:
save_path = get_filepath_str(self._get_save_path(), self._protocol)
with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
data.save(fs_file, **self._save_args)
self._invalidate_cache()
def _exists(self) -> bool:
try:
load_path = get_filepath_str(self._get_load_path(), self._protocol)
except DataSetError:
return False
return self._fs.exists(load_path)
def _save(self, data: pd.DataFrame) -> None:
save_path = get_filepath_str(self._get_save_path(), self._protocol)
with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
data.to_json(path_or_buf=fs_file, **self._save_args)
self._invalidate_cache()
def _save(self, plot: str) -> None:
save_path = Path(
get_filepath_str(self._get_save_path(), self._protocol))
save_path.parent.mkdir(parents=True, exist_ok=True)
hv.save(plot, save_path, **self._save_args)
self._invalidate_cache()
def _load(self) -> Any:
load_path = get_filepath_str(self._get_load_path(), self._protocol)
with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
return self.BACKENDS[self._backend].load(
fs_file, **self._load_args
) # nosec
def _save(self, data: str) -> None:
save_path = get_filepath_str(self._get_save_path(), self._protocol)
with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
fs_file.write(data)
self._invalidate_cache()
def _load(self) -> pd.DataFrame:
load_path = get_filepath_str(self._get_load_path(), self._protocol)
with self._fs.open(load_path, **self._fs_open_args_load) as fs_file:
data: pd.DataFrame = pd.read_csv(fs_file, **self._load_args)
data.attrs = self.df_attrs if self.df_attrs else {}
return data
def _load(self) -> (pd.DataFrame, np.ndarray):
"""Loads data from the DICOM file.
Returns:
Metadata from the DICOM file as a pandas Dataframe,
Image data as a numpy array
"""
# using get_filepath_str ensures that the protocol and path are appended correctly for different filesystems
load_path = get_filepath_str(self._filepath, self._protocol)
with self._fs.open(load_path) as f:
ds = pydicom.dcmread(f)
#Convert metadata records to dataframe
df = pd.DataFrame.from_records([
(el.name, el.value) for el in ds if el.name not in
['Pixel Data', 'File Meta Information Version']
])
#Convert row to columns
df = df.T
#Set first line as header
df.columns = df.iloc[0]
#Delete first line
df = df.iloc[1:]
pixel_array = ds.pixel_array
return (df, pixel_array)
def _save(self, data: Dict) -> None:
save_path = get_filepath_str(self._get_save_path(), self._protocol)
with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
json.dump(data, fs_file, **self._save_args)
self._invalidate_cache()
def _load(self) -> str:
load_path = get_filepath_str(self._get_load_path(), self._protocol)
if str(load_path).endswith(".png"):
return hv.RGB.load_image(load_path, **self._load_args)
else:
raise NotImplementedError("There is no way to convert from an\
arbitrary saved image format to a plot.")
def _save(self, data: pd.DataFrame) -> None:
plot_data = _plotly_express_wrapper(data, self._plotly_args)
full_key_path = get_filepath_str(self._get_save_path(), self._protocol)
with self._fs.open(full_key_path, **self._fs_open_args_save) as fs_file:
plot_data.write_json(fs_file, **self._save_args)
self._invalidate_cache()
def _save(self, data: DGLHeteroGraph) -> None:
save_path = get_filepath_str(self._get_save_path(), self._protocol)
dgl.save_graphs(save_path, data, **self._save_args)
# with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
# json.dump(json_graph, fs_file)
self._invalidate_cache()
def _save(self, data: networkx.Graph) -> None:
save_path = get_filepath_str(self._get_save_path(), self._protocol)
json_graph = networkx.node_link_data(data, **self._save_args)
with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
json.dump(json_graph, fs_file)
self._invalidate_cache()
def _save(self, data: HoloViews) -> None:
bytes_buffer = io.BytesIO()
hv.save(data, bytes_buffer, **self._save_args)
save_path = get_filepath_str(self._get_save_path(), self._protocol)
with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
fs_file.write(bytes_buffer.getvalue())
self._invalidate_cache()
def _save(self, data: pd.DataFrame) -> None:
save_path = get_filepath_str(self._get_save_path(), self._protocol)
table = pa.Table.from_pandas(data)
pq.write_table(
table=table, where=save_path, filesystem=self._fs, **self._save_args
)
self.invalidate_cache()
def _save(self, data: Union[Dict, pd.DataFrame]) -> None:
save_path = get_filepath_str(self._get_save_path(), self._protocol)
if isinstance(data, pd.DataFrame):
data = data.to_dict()
with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
yaml.dump(data, fs_file, **self._save_args)
self._invalidate_cache()
def _save(self, data: Union[figure, List[figure], Dict[str,
figure]]) -> None:
save_path = self._get_save_path()
if isinstance(data, list):
for index, plot in enumerate(data):
full_key_path = get_filepath_str(save_path / f"{index}.png",
self._protocol)
self._save_to_fs(full_key_path=full_key_path, plot=plot)
elif isinstance(data, dict):
for plot_name, plot in data.items():
full_key_path = get_filepath_str(save_path / plot_name,
self._protocol)
self._save_to_fs(full_key_path=full_key_path, plot=plot)
else:
full_key_path = get_filepath_str(save_path, self._protocol)
self._save_to_fs(full_key_path=full_key_path, plot=data)
self._invalidate_cache()
def _save(self, data: pd.DataFrame) -> None:
save_path = get_filepath_str(self._get_save_path(), self._protocol)
buf = BytesIO()
data.to_feather(buf)
with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
fs_file.write(buf.getvalue())
self._invalidate_cache()
def _load(self) -> pd.DataFrame:
load_args = copy.deepcopy(self._load_args)
engine = self.engines[self._connection_str] # type: ignore
if self._filepath:
load_path = get_filepath_str(PurePosixPath(self._filepath),
self._protocol)
with self._fs.open(load_path, mode="r") as fs_file:
load_args["sql"] = fs_file.read()
return pd.read_sql_query(con=engine, **load_args)
def _load(self) -> tf.keras.Model:
load_path = get_filepath_str(self._get_load_path(), self._protocol)
with tempfile.TemporaryDirectory(prefix=self._tmp_prefix) as path:
if self._is_h5:
path = str(PurePath(path) / TEMPORARY_H5_FILE)
self._fs.copy(load_path, path)
else:
self._fs.get(load_path, path, recursive=True)
# Pass the local temporary directory/file path to keras.load_model
return tf.keras.models.load_model(path, **self._load_args)
def _save(self, data: Any) -> None:
save_path = get_filepath_str(self._get_save_path(), self._protocol)
with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
try:
self.BACKENDS[self._backend].dump(data, fs_file,
**self._save_args)
except Exception as exc:
raise DataSetError("{} was not serialized due to: {}".format(
str(data.__class__), str(exc))) from exc
self._invalidate_cache()
def _save(self, data: pd.DataFrame) -> None:
output = BytesIO()
save_path = get_filepath_str(self._get_save_path(), self._protocol)
# pylint: disable=abstract-class-instantiated
with pd.ExcelWriter(output, **self._writer_args) as writer:
data.to_excel(writer, **self._save_args)
with self._fs.open(save_path, **self._fs_open_args_save) as fs_file:
fs_file.write(output.getvalue())
self._invalidate_cache()
def _save(self, data: Union[figure, List[figure], Dict[str, figure]]) -> None:
if isinstance(data, list):
for index, plot in enumerate(data):
full_key_path = self._fs.pathsep.join(
[
get_filepath_str(self._filepath, self._protocol),
"{}.png".format(index),
]
)
self._save_to_fs(full_key_path=full_key_path, plot=plot)
elif isinstance(data, dict):
for plot_name, plot in data.items():
full_key_path = self._fs.pathsep.join(
[get_filepath_str(self._filepath, self._protocol), plot_name]
)
self._save_to_fs(full_key_path=full_key_path, plot=plot)
else:
full_key_path = get_filepath_str(self._filepath, self._protocol)
self._save_to_fs(full_key_path=full_key_path, plot=data)
self.invalidate_cache()
def _load(self) -> pd.DataFrame:
load_args = copy.deepcopy(self._load_args)
if self._filepath:
load_path = get_filepath_str(PurePosixPath(self._filepath),
self._protocol)
with self._fs.open(load_path, mode="r") as fs_file:
load_args["query"] = fs_file.read()
return pd.read_gbq(
project_id=self._project_id,
credentials=self._credentials,
**load_args,
)