def _save_filename(self, parent=None, filename='', caption='Save as...', selected_filter='', filters=None):
from spectrochempy.utils import pathclean
dftext = ''
directory = '.'
if filename:
filename = pathclean(filename)
directory = filename.parent
dftext = filename.suffix
if not dftext and selected_filter:
raise Exception("Save error")
if not dftext:
dftext = '.scp'
# -defaultextension, -filetypes, -initialdir, -initialfile, -message, -parent, -title, -typevariable,
# -command, or -confirmoverwrite
filename = filedialog.asksaveasfilename(parent=parent, title=caption, initialdir=str(directory),
initialfile=filename.name, defaultextension=dftext,
filetypes=self.filetypes(filters))
if parent is not None:
parent.destroy
if filename:
return pathclean(filename)
def _switch_protocol(self, key, files, **kwargs):
protocol = kwargs.get("protocol", None)
if protocol is not None and protocol != "ALL":
if not isinstance(protocol, list):
protocol = [protocol]
if key and key[1:] not in protocol and self.alias[
key[1:]] not in protocol:
return
datasets = []
for filename in files[key]:
filename = pathclean(filename)
read_ = getattr(self, f"_read_{key[1:]}")
try:
res = read_(self.objtype(), filename, **kwargs)
# sometimes read_ can return None (e.g. non labspec text file)
except FileNotFoundError as e:
# try to get the file from github
kwargs["read_method"] = read_
try:
res = _read_remote(self.objtype(), filename, **kwargs)
except OSError:
raise e
except IOError as e:
warning_(str(e))
res = None
except NotImplementedError as e:
warning_(str(e))
res = None
except Exception:
raise e
except IOError as e:
warning_(str(e))
res = None
except NotImplementedError as e:
warning_(str(e))
res = None
if res is not None:
if not isinstance(res, list):
datasets.append(res)
else:
datasets.extend(res)
if len(datasets) > 1:
datasets = self._do_merge(datasets, **kwargs)
if kwargs.get("merge", False):
datasets[0].name = pathclean(filename).stem
datasets[0].filename = pathclean(filename)
self.datasets.extend(datasets)
def makestyle(self, filename='mydefault', to_mpl=False):
if filename.startswith('scpy'):
error_('`scpy` is READ-ONLY. Please use an another style name.')
return
txt = ""
sline = ""
for key in mpl.rcParams.keys():
if key in ['animation.avconv_args', 'animation.avconv_path', 'animation.html_args', 'keymap.all_axes',
'mathtext.fallback_to_cm', 'validate_bool_maybe_none', 'savefig.jpeg_quality',
'text.latex.preview', 'backend', 'backend_fallback', 'date.epoch', 'docstring.hardcopy',
'figure.max_open_warning', 'figure.raise_window', 'interactive', 'savefig.directory', 'timezone',
'tk.window_focus', 'toolbar', 'webagg.address', 'webagg.open_in_browser', 'webagg.port',
'webagg.port_retries']:
continue
val = str(mpl.rcParams[key])
sav = ''
while val != sav:
sav = val
val = val.replace(' ', ' ')
line = f'{key:40s} : {val}\n'
if line[0] != sline:
txt += '\n'
sline = line[0]
if key not in ['axes.prop_cycle']:
line = line.replace('[', '').replace(']', "").replace('\'', '').replace('"', '')
if key == 'savefig.bbox':
line = f'{key:40s} : standard\n'
txt += line.replace("#", '')
# Non matplotlib parameters,
# some parameters are not saved in matplotlib style sheets so we willa dd them here
nonmplpars = ['method_1D', 'method_2D', 'method_3D', 'colorbar', 'show_projections', 'show_projection_x',
'show_projection_y', 'colormap', 'max_lines_in_stack', 'simplify', 'number_of_x_labels',
'number_of_y_labels', 'number_of_z_labels', 'number_of_contours', 'contour_alpha',
'contour_start', 'antialiased', 'rcount', 'ccount']
txt += '\n\n##\n## ADDITIONAL PARAMETERS FOR SPECTROCHEMPY\n##\n'
for par in nonmplpars:
txt += f"##@{par:37s} : {getattr(self, par)}\n"
stylesheet = (pathclean(self.stylesheets) / filename).with_suffix('.mplstyle')
stylesheet.write_text(txt)
if to_mpl:
# make it also accessible to pyplot
stylelib = (pathclean(mpl.get_configdir()) / 'stylelib' / filename).with_suffix('.mplstyle')
stylelib.write_text(txt)
# plot_preferences.traits()['style'].trait_types = plot_preferences.traits()['style'].trait_types +\
# (Unicode(filename),)
self.style = filename
return self.style
def _read_remote(*args, **kwargs):
from spectrochempy.core import preferences as prefs
datadir = prefs.datadir
dataset, path = args
# path of the required files
path = pathclean(path)
if _is_relative_to(path, datadir):
# try to make it relative for remote downloading
relative_path = _relative_to(path, datadir)
else:
# assume it is already relative
relative_path = path
# in principle the data came from github. Try to download it
replace = kwargs.pop("replace_existing", False)
dst = datadir / relative_path
_download_from_url(relative_path, dst, replace)
if not kwargs.pop("download_only", False):
read_method = kwargs.pop("read_method", read)
return read_method(dataset, dst, **kwargs)
def test_write():
nd = scp.read_omnic('irdata/nh4y-activation.spg')
# API write methods needs an instance of a NDDataset as the first argument
with pytest.raises(TypeError):
scp.write()
# the simplest way to save a dataset, is to use the function write with a filename as argument
filename = nd.write('essai.scp')
assert filename == cwd / 'essai.scp'
nd2 = NDDataset.load(filename)
testing.assert_dataset_equal(nd2, nd)
filename.unlink()
# if the filename is omitted, the a dialog is opened to select a name (and a protocol)
filename = nd.write()
assert filename is not None
assert filename.stem == nd.name
assert filename.suffix == '.scp'
filename.unlink()
# # a write protocole can be specified
# filename = nd.write(protocole='json')
# assert filename is not None
# assert filename.stem == nd.name
# assert filename.suffix == '.json'
# filename.unlink()
irdatadir = pathclean(prefs.datadir) / "irdata"
for f in ['essai.scp', 'nh4y-activation.scp']:
if (irdatadir / f).is_file():
(irdatadir / f).unlink()
def _relative_to(path, base):
pparts = path.parts
bparts = base.parts
if bparts[-1] in pparts:
idx = pparts.index(bparts[-1])
return pathclean("/".join(pparts[idx + 1:]))
raise ValueError(
f"'{path}' is not in the subpath of '{base}' OR one path is "
f"relative and the other absolute.")
def test_pathclean():
# Using unix/mac way to write paths
filename = pathclean("irdata/nh4y-activation.spg")
assert filename.suffix == ".spg"
assert filename.parent.name == "irdata"
# or Windows
filename = pathclean("irdata\\\\nh4y-activation.spg")
assert filename.parent.name == "irdata"
# Due to the escape character \\ in Unix, path string should be escaped \\\\
# or the raw-string prefix `r` must be used as shown below
filename = pathclean(r"irdata\\nh4y-activation.spg")
assert filename.suffix == ".spg"
assert filename.parent.name == "irdata"
# of course should work if input is alreadya Path
filename = pathclean(prefs.datadir / "irdata/nh4y-activation.spg")
assert filename.suffix == ".spg"
def directory(self):
"""
Current directory for this dataset.
ReadOnly property - automatically set when the filename is updated if
it contains a parent on its path.
"""
if self._filename:
return pathclean(self._filename).parent
else:
return None
def _listdir(s, initial, ns):
ns += 1
for f in pathclean(initial).glob(
'*'): # glob.glob(os.path.join(initial, '*')):
fb = f.name # os.path.basename(f)
if fb.startswith('.'):
continue
if not fb.startswith('acqu') and not fb.startswith(
'pulse') and fb not in ['ser', 'fid']:
s += " " * ns + "|__" + "%s\n" % fb
if f.is_dir():
s = _listdir(s, f, ns)
return s
def dialog_open(*args, **kwargs):
# mock opening a dialog
directory = kwargs.get("directory", None)
if directory is None:
directory = pathclean(DATADIR / "fakedir")
if kwargs.get("filters") == "directory":
return directory
if not args and not kwargs.get("single"):
return [DATADIR / "fakedir" / f"fake{i + 1}.fk" for i in range(2)]
return [DATADIR / "fakedir" / f"fake{i+1}.fk" for i in range(4)]
def save_dialog(parent=None, filename=None, caption='Save as...', selected_filter='', filters=("All Files (*)"),
**kwargs):
"""
Return a file where to save
"""
if USE_QT:
parent = kwargs.pop('Qt_parent', None) # in case this is launched from spectrochempy_gui
f = _QTFileDialogs._save_filename(parent=parent, filename=filename, caption=caption,
selected_filter=selected_filter, filters=filters)
else:
f = _TKFileDialogs()._save_filename(filename, caption, selected_filter, filters)
from spectrochempy.utils import pathclean
return pathclean(f)
def _listdir(strg, initial, nst):
nst += 1
for fil in pathclean(initial).glob(
"*"): # glob.glob(os.path.join(initial, '*')):
filename = fil.name # os.path.basename(f)
if filename.startswith("."): # pragma: no cover
continue
if (not filename.startswith("acqu")
and not filename.startswith("pulse")
and filename not in ["ser", "fid"]):
strg += " " * nst + f"|__{filename}\n"
if fil.is_dir():
strg = _listdir(strg, fil, nst)
return strg
def _switch_protocol(self, key, files, **kwargs):
protocol = kwargs.get('protocol', None)
if protocol is not None and protocol != 'ALL':
if not isinstance(protocol, list):
protocol = [protocol]
if key and key[1:] not in protocol and self.alias[key[1:]] not in protocol:
return
datasets = []
for filename in files[key]:
try:
read_ = getattr(self, f"_read_{key[1:]}")
except AttributeError:
warning_(f'a file with extension {key} was found in this directory but will be ignored')
try:
res = read_(self.objtype(), filename, **kwargs)
if not isinstance(res, list):
datasets.append(res)
else:
datasets.extend(res)
except FileNotFoundError:
warning_(f'No file with name `{filename}` could be found. Sorry! ')
except IOError as e:
warning_(str(e))
except Exception:
warning_(f'The file `{filename}` has a known extension but it could not be read. It is ignored!')
if len(datasets) > 1:
datasets = self._do_merge(datasets, **kwargs)
if kwargs.get('merge', False):
datasets[0].name = pathclean(filename).stem
datasets[0].filename = pathclean(filename)
self.datasets.extend(datasets)
def reset(self):
# remove the matplotlib_user json file to reset to defaults
config_dir = pathclean(preferences.cfg.config_dir)
f = config_dir / 'PlotPreferences.json'
if f.exists():
f.unlink()
plot_preferences._apply_style('scpy')
self.style = 'scpy'
# reset also non-matplolib preferences
nonmplpars = ['method_1D', 'method_2D', 'method_3D', 'colorbar', 'show_projections', 'show_projection_x',
'show_projection_y', 'colormap', 'max_lines_in_stack', 'simplify', 'number_of_x_labels',
'number_of_y_labels', 'number_of_z_labels', 'number_of_contours', 'contour_alpha',
'contour_start', 'antialiased', 'rcount', 'ccount']
for par in nonmplpars:
setattr(self, par, plot_preferences.traits()[par].default_value)
self._data = {}
def __call__(self, *args, **kwargs):
args = self._setup_object(*args)
try:
if 'filetypes' not in kwargs:
kwargs['filetypes'] = list(self.filetypes.values())
if args and args[0] is not None: # filename
protocol = self.protocols[pathclean(args[0]).suffix]
kwargs['filetypes'] = [self.filetypes[protocol]]
filename = check_filename_to_save(self.object, *args, **kwargs)
if kwargs.get('suffix', ''):
filename = filename.with_suffix(kwargs.get('suffix', ''))
protocol = self.protocols[filename.suffix]
write_ = getattr(self, f"_write_{protocol}")
write_(self.object, filename, **kwargs)
return filename
except Exception as e:
raise e
def test_write():
nd = scp.read_omnic("irdata/nh4y-activation.spg")
# API write methods needs an instance of a NDDataset as the first argument
with pytest.raises(TypeError):
scp.write()
# the simplest way to save a dataset, is to use the function write with a filename as argument
if (cwd / "essai.scp").exists():
(cwd / "essai.scp").unlink()
filename = nd.write("essai.scp") # should not open a DIALOG
assert filename == cwd / "essai.scp"
assert filename.exists()
# try to write it again
filename = nd.write("essai.scp") # should open a DIALOG to confirm
nd2 = NDDataset.load(filename)
testing.assert_dataset_equal(nd2, nd)
filename.unlink()
# if the filename is omitted, a dialog is opened to select a name (and a protocol)
filename = nd.write()
assert filename is not None
assert filename.stem == nd.name
assert filename.suffix == ".scp"
filename.unlink()
# # a write protocole can be specified
# filename = nd.write(protocole='json')
# assert filename is not None
# assert filename.stem == nd.name
# assert filename.suffix == '.json'
# filename.unlink()
irdatadir = pathclean(prefs.datadir) / "irdata"
for f in ["essai.scp", "nh4y-activation.scp"]:
if (irdatadir / f).is_file():
(irdatadir / f).unlink()
def reset(self):
"""
Remove the matplotlib_user json file to reset to defaults.
"""
config_dir = pathclean(preferences.cfg.config_dir)
f = config_dir / "PlotPreferences.json"
if f.exists():
f.unlink()
plot_preferences._apply_style("scpy")
self.style = "scpy"
# reset also non-matplolib preferences
nonmplpars = [
"method_1D",
"method_2D",
"method_3D",
"colorbar",
"show_projections",
"show_projection_x",
"show_projection_y",
"colormap",
"max_lines_in_stack",
"simplify",
"number_of_x_labels",
"number_of_y_labels",
"number_of_z_labels",
"number_of_contours",
"contour_alpha",
"contour_start",
"antialiased",
"rcount",
"ccount",
]
for par in nonmplpars:
setattr(self, par, plot_preferences.traits()[par].default_value)
self._data = {}
def dump(self, filename, **kwargs):
"""
Save the current object into compressed native spectrochempy format.
Parameters
----------
filename: str of `pathlib` object
File name where to save the current object.
"""
# Stage data in a temporary file on disk, before writing to zip.
import zipfile
import tempfile
# prepare the json data
try:
js = self.dumps(encoding="base64")
except Exception as e:
print(e)
# write in a temp file
_, tmpfile = tempfile.mkstemp(suffix="-spectrochempy")
tmpfile = pathclean(tmpfile)
tmpfile.write_bytes(js.encode("utf-8"))
# compress and write zip file
zipf = zipfile_factory(filename,
mode="w",
compression=zipfile.ZIP_DEFLATED)
zipf.write(tmpfile, arcname=f"{self.name}.json")
# tmpfile.unlink()
zipf.close()
self.filename = filename
self.name = filename.stem
return filename
def open_dialog(parent=None, single=True, directory=None, filters=("All Files (*)"), **kwargs):
"""
Return one or several files to open
"""
if USE_QT:
parent = kwargs.pop('Qt_parent', None) # in case this is launched from spectrochempy_gui
klass = _QTFileDialogs
else:
klass = _TKFileDialogs()
parent = klass.root
default_filter = kwargs.get('default_filter', None)
if directory is None:
directory = ''
if filters == 'directory':
caption = 'Select a folder'
f = klass._open_existing_directory(parent=parent, caption=caption, directory=str(directory))
elif single:
f = klass._open_filename(parent=parent, filters=filters, default_filter=default_filter)
else:
f = klass._open_multiple_filenames(parent=parent, filters=filters, default_filter=default_filter)
from spectrochempy.utils import pathclean
return pathclean(f)
""" Tests for the ndplugin module
"""
import pathlib
import pytest
from spectrochempy.utils.testing import assert_dataset_equal
from spectrochempy.core.dataset.nddataset import NDDataset
from spectrochempy.core import preferences as prefs
from spectrochempy.utils.testing import assert_array_equal
from spectrochempy.utils import pathclean
irdatadir = pathclean(prefs.datadir) / "irdata"
nmrdatadir = pathclean(prefs.datadir) / "nmrdata" / "bruker" / "tests" / "nmr"
cwd = pathlib.Path.cwd()
# Basic
# ------------------------------------------------------------------
def test_ndio_generic(NMR_dataset_1D):
nmr = NMR_dataset_1D
assert nmr.directory == nmrdatadir
# save with the default filename or open a dialog if it doesn't exists
# ------------------------------------------------------------------------
# save with the default name (equivalent to save_as in this case)
# as this file (IR_1D.scp) doesn't yet exist a confirmation dialog is opened
f = nmr.save()
def _get_databases_directory_default(self):
# the spectra path in package data
return pathclean(get_pkg_path("databases", "scp_data"))
if 'ipykernel_launcher' in sys.argv[0] and \
"--InlineBackend.rc={'figure.dpi': 96}" in sys.argv:
# We are running from NBSphinx - the plot must be inline to show up.
ip.magic('matplotlib inline')
else:
# Do not set the widget backend.... do not work most of the time after upbgrade of the various
# library and
# jupyter!!! ...
ip.magic('matplotlib inline') # widget
except Exception:
ip.magic('matplotlib qt')
# set_backend()
# a usefull utilities for dealing with path
from spectrochempy.utils import pathclean
DATADIR = pathclean(preferences.datadir)
__all__ = ['pathclean', 'DATADIR'] + ALL
import warnings
warnings.filterwarnings(action='ignore',
module='matplotlib') # , category=UserWarning)
# warnings.filterwarnings(action="error", category=DeprecationWarning)
# ==============================================================================
if __name__ == '__main__':
pass
try:
# work only if spectrochempy is installed
import spectrochempy
except ModuleNotFoundError: # pragma: no cover
raise ModuleNotFoundError(
"You must install spectrochempy and its dependencies before executing tests!"
)
import spectrochempy as scp
from spectrochempy import preferences as prefs
from spectrochempy.utils import pathclean
from spectrochempy.utils.testing import RandomSeedContext
# first download missing data
datadir = pathclean(prefs.datadir)
print("DATADIR: ", datadir)
# this process is relatively long, so we do not want to do it several time:
downloaded = datadir / "__downloaded__"
if not downloaded.exists():
scp.read_remote(datadir, download_only=True)
downloaded.write_text("")
# ======================================================================================================================
# FIXTURES
# ======================================================================================================================
# initialize a ipython session before calling spectrochempy
# ------------------------------------------------------------------
def _datadir_changed(self, change):
self.parent.datadir.path = pathclean(change['new'])
def _get_default_datadir(self):
return pathclean(self.parent.datadir.path)
def _data_validate(self, proposal):
# validation of the datadir attribute
datadir = proposal["value"]
if isinstance(datadir, str):
datadir = pathclean(datadir)
return datadir
def save(self, **kwargs):
"""
Save the current object in SpectroChemPy format.
Default extension is *.scp for |NDDataset|'s and *.pscp for
|Project|'s.
Parameters
----------
**kwargs
Optional keyword parameters (see Other Parameters).
Other Parameters
----------------
confirm : bool
See Also
---------
save_as : Save current object with a different name and/or directory.
write : Export current object to different format.
Examples
---------
Read some data from an OMNIC file
>>> nd = scp.read_omnic('wodger.spg')
>>> assert nd.name == 'wodger'
Write it in SpectroChemPy format (.scp)
(return a `pathlib` object)
>>> filename = nd.save()
Check the existence of the scp fie
>>> assert filename.is_file()
>>> assert filename.name == 'wodger.scp'
Remove this file
>>> filename.unlink()
"""
# By default we save the file in the self.directory and with the
# name + suffix depending
# on the current object type
if self.directory is None:
filename = pathclean(".") / self.name
else:
filename = pathclean(self.directory) / self.name
default_suffix = SCPY_SUFFIX[self.implements()]
filename = filename.with_suffix(default_suffix)
if not filename.exists() and kwargs.get("confirm", True):
# never saved
kwargs["caption"] = f"Save the current {self.implements()} as ... "
return self.save_as(filename, **kwargs)
# was already saved previously with this name,
# in this case we do not display a dialog and overwrite the same file
self.name = filename.stem
return self.dump(filename, **kwargs)
def save_as(self, filename="", **kwargs):
"""
Save the current |NDDataset| in SpectroChemPy format (*.scp).
Parameters
----------
filename : str
The filename of the file where to save the current dataset.
**kwargs
Optional keyword parameters (see Other Parameters).
Other Parameters
-----------------
directory : str, optional
If specified, the given `directory` and the `filename` will be
appended.
See Also
---------
save : Save current dataset.
write : Export current dataset to different format.
Notes
-----
Adapted from :class:`numpy.savez`.
Examples
--------
Read some data from an OMNIC file
>>> nd = scp.read_omnic('wodger.spg')
>>> assert nd.name == 'wodger'
Write it in SpectroChemPy format (.scp)
(return a `pathlib` object)
>>> filename = nd.save_as('new_wodger')
Check the existence of the scp fie
>>> assert filename.is_file()
>>> assert filename.name == 'new_wodger.scp'
Remove this file
>>> filename.unlink()
"""
if filename:
# we have a filename
# by default it use the saved directory
filename = pathclean(filename)
if self.directory and self.directory != filename.parent:
filename = self.directory / filename
else:
filename = self.directory
# suffix must be specified which correspond to the type of the
# object to save
default_suffix = SCPY_SUFFIX[self.implements()]
if filename is not None and not filename.is_dir():
filename = filename.with_suffix(default_suffix)
kwargs["filetypes"] = self.filetype
kwargs["caption"] = f"Save the current {self.implements()} as ... "
filename = check_filename_to_save(self,
filename,
save_as=True,
suffix=default_suffix,
**kwargs)
if filename:
self.filename = filename
return self.dump(filename, **kwargs)
def load(cls, filename, **kwargs):
"""
Open data from a '*.scp' (NDDataset) or '.pscp' (Project) file.
Parameters
----------
filename : `str`, `pathlib` or `file` objects
The name of the file to read (or a file objects.
**kwargs
Optional keyword parameters (see Other Parameters).
Other Parameters
----------------
content : str, optional
The optional content of the file(s) to be loaded as a binary string.
See Also
--------
read : Import dataset from various orgines.
save : Save the current dataset.
Notes
-----
Adapted from `numpy.load`.
Examples
--------
>>> nd1 = scp.read('irdata/nh4y-activation.spg')
>>> f = nd1.save()
>>> f.name
'nh4y-activation.scp'
>>> nd2 = scp.load(f)
Alternatively, this method can be called as a class method of NDDataset or Project object:
>>> from spectrochempy import *
>>> nd2 = NDDataset.load(f)
"""
content = kwargs.get("content", None)
if content:
fid = io.BytesIO(content)
else:
# be sure to convert filename to a pathlib object with the
# default suffix
filename = pathclean(filename)
suffix = cls().suffix
filename = filename.with_suffix(suffix)
if kwargs.get("directory", None) is not None:
filename = pathclean(kwargs.get("directory")) / filename
if not filename.exists():
raise FileNotFoundError(
f"No file with name {filename} could be found.")
# filename = check_filenames(filename, **kwargs)[0]
fid = open(filename, "rb")
# get zip file
try:
obj = ScpFile(fid)
except FileNotFoundError:
raise SpectroChemPyException(f"File {filename} doesn't exist!")
except Exception as e:
if str(e) == "File is not a zip file":
raise SpectroChemPyException(
"File not in 'scp' or 'pscp' format!")
raise SpectroChemPyException("Undefined error!")
js = obj[obj.files[0]]
if kwargs.get("json", False):
return js
new = cls.loads(js)
fid.close()
if filename:
filename = pathclean(filename)
new._filename = filename
new.name = filename.stem
return new
def filename(self, val):
self._filename = pathclean(val)
