def test_nddataset_create_from_complex_data():
# 1D (complex)
nd = scp.NDDataset([1.0 + 2.0j, 2.0 + 0j])
assert nd.data.size == 2
assert nd.size == 2
assert nd.data.shape == (2,)
assert nd.shape == (2,)
# 2D (complex in the last dimension - automatic detection)
nd = scp.NDDataset(
[[1.0 + 2.0j, 2.0 + 0j], [1.3 + 2.0j, 2.0 + 0.5j], [1.0 + 4.2j, 2.0 + 3j]]
)
assert nd.data.size == 6
assert nd.size == 6
assert nd.data.shape == (3, 2)
assert nd.shape == (3, 2)
# 2D quaternion
nd = scp.NDDataset(
[
[1.0, 2.0],
[1.3, 2.0],
[1.0, 2.0],
[1.0, 2.0],
],
dtype=typequaternion,
)
assert nd.data.size == 2
assert nd.size == 2
assert nd.data.shape == (2, 1)
assert nd.shape == (2, 1)
# take real part
ndr = nd.real
assert ndr.shape == (2, 1)
assert not ndr.is_quaternion
def test_nddataset_title():
ndd = scp.NDDataset([1., 2., 3.], title='xxxx')
assert ndd.title == 'xxxx'
ndd2 = scp.NDDataset(ndd, title='yyyy')
assert ndd2.title == 'yyyy'
ndd2.title = 'zzzz'
assert ndd2.title == 'zzzz'
def test_1D_NDDataset(a):
# 1D
for arr in [a, np.array(a)]:
ds = scp.NDDataset(arr)
assert ds.size == len(arr)
assert ds.shape == (ds.size,)
if ds.size == 0:
assert ds.dtype is None
assert ds.dims == []
else:
assert ds.dtype in [np.float64, np.complex128]
assert ds.dims == ['x']
# force dtype
ds = scp.NDDataset(arr, dtype=np.float32)
if ds.size == 0:
assert ds.dtype is None
else:
assert ds.dtype == np.float32
assert ds.title == "<untitled>"
assert ds.mask == scp.NOMASK
assert ds.meta == {}
assert ds.name.startswith("NDDataset")
assert ds.author == get_user_and_node()
assert ds.description == ""
assert ds.history == []
def test_nddataset_real_imag_quaternion():
na = np.array(
[[1.0 + 2.0j, 2.0 + 0j, 1.3 + 2.0j], [2.0 + 0.5j, 1.0 + 4.2j, 2.0 + 3j]]
)
nd = scp.NDDataset(na)
# in the last dimension
assert_array_equal(nd.real, na.real)
assert_array_equal(nd.imag, na.imag)
# in another dimension
nd.set_quaternion(inplace=True)
assert nd.is_quaternion
assert nd.shape == (1, 3)
na = np.array(
[
[1.0 + 2.0j, 2.0 + 0j],
[1.3 + 2.0j, 2.0 + 0.5j],
[1.0 + 4.2j, 2.0 + 3j],
[5.0 + 4.2j, 2.0 + 3j],
]
)
nd = scp.NDDataset(na)
nd.set_quaternion(inplace=True)
assert nd.is_quaternion
assert_array_equal(nd.real.data, na[::2, :].real)
nb = np.array(
[
[0.0 + 2.0j, 0.0 + 0j],
[1.3 + 2.0j, 2.0 + 0.5j],
[0.0 + 4.2j, 0.0 + 3j],
[5.0 + 4.2j, 2.0 + 3j],
]
)
ndj = scp.NDDataset(nb, dtype=quaternion)
assert nd.imag == ndj
def test_NNMF():
w1 = scp.NDDataset([[1, 2, 3], [4, 5, 6]])
h1 = scp.NDDataset([[1, 2], [3, 4], [5, 6]])
w2 = scp.NDDataset([[1, 1, 1], [4, 4, 4]])
h2 = scp.NDDataset([[1, 1], [3, 3], [5, 5]])
v = scp.dot(w1, h1)
nnmf = scp.NNMF(v,
w2,
h2,
tol=0.0001,
maxtime=60,
maxiter=100,
verbose=True)
scp.info_("------")
scp.info_(nnmf.C)
scp.info_("------")
scp.info_(nnmf.St)
assert_array_almost_equal(nnmf.C.data,
np.array([[1.4, 2.1, 2.9], [4.4, 5.2, 6.0]]),
decimal=1)
assert_array_almost_equal(nnmf.St.data,
[[0.8, 1.9], [2.9, 3.9], [5.1, 5.9]],
decimal=1)
def test_nddataset_title():
ndd = scp.NDDataset([1.0, 2.0, 3.0], title="xxxx")
assert ndd.title == "xxxx"
ndd2 = scp.NDDataset(ndd, title="yyyy")
assert ndd2.title == "yyyy"
ndd2.title = "zzzz"
assert ndd2.title == "zzzz"
def test_nddataset_mask_array_input():
marr = np.ma.array([1., 2., 5.]) # Masked array with no masked entries
nd = scp.NDDataset(marr)
assert not nd.is_masked
marr = np.ma.array([1., 2., 5.], mask=[True, False, False]) # Masked array
nd = scp.NDDataset(marr)
assert nd.is_masked
def test_issue_310():
import spectrochempy as scp
import numpy as np
D = scp.NDDataset(np.zeros((10, 5)))
c5 = scp.Coord.linspace(start=0.0, stop=1000.0, num=5, name="xcoord")
c10 = scp.Coord.linspace(start=0.0, stop=1000.0, num=10, name="ycoord")
assert c5.name == "xcoord"
D.set_coordset(x=c5, y=c10)
assert c5.name == "x"
assert str(D.coordset) == "CoordSet: [x:<untitled>, y:<untitled>]"
assert D.dims == ["y", "x"]
E = scp.NDDataset(np.ndarray((10, 5)))
E.dims = ["t", "s"]
E.set_coordset(s=c5, t=c10)
assert c5.name == "s"
assert str(E.coordset) == "CoordSet: [s:<untitled>, t:<untitled>]"
assert E.dims == ["t", "s"]
E = scp.NDDataset(np.ndarray((5, 10)))
E.dims = ["s", "t"]
E.set_coordset(s=c5, t=c10)
assert c5.name == "s"
assert str(E.coordset) == "CoordSet: [s:<untitled>, t:<untitled>]"
assert E.dims == ["s", "t"]
assert str(D.coordset) == "CoordSet: [x:<untitled>, y:<untitled>]"
assert D.dims == ["y", "x"]
assert str(D[:, 1]) == "NDDataset: [float64] unitless (shape: (y:10, x:1))"
def test_nddataset_coordset():
# init coordinates set at NDDataset initialization
dx = np.random.random((10, 7, 3))
coord0 = np.arange(10)
coord1 = np.arange(7)
coord2 = np.arange(3) * 100.
da = scp.NDDataset(dx, coordset=(coord0, coord1, coord2), title='absorbance',
coordtitles=['wavelength', 'time-on-stream', 'temperature'], coordunits=['cm^-1', 's', 'K'], )
assert da.shape == (10, 7, 3)
assert da.coordset.titles == ['temperature', 'time-on-stream', 'wavelength']
assert da.coordset.names == ['x', 'y', 'z']
assert da.coordunits == [ur.Unit('K'), ur.Unit('s'), ur.Unit('cm^-1')]
# order of dims follow data shape, but not necessarily the coord list (
# which is ordered by name)
assert da.dims == ['z', 'y', 'x']
assert da.coordset.names == sorted(da.dims)
# transpose
dat = da.T
assert dat.dims == ['x', 'y', 'z']
# dims changed but did not change coords order !
assert dat.coordset.names == sorted(dat.dims)
assert dat.coordtitles == da.coordset.titles
assert dat.coordunits == da.coordset.units
# too many coordinates
cadd = scp.Coord(labels=['d%d' % i for i in range(6)])
coordtitles = ['wavelength', 'time-on-stream', 'temperature']
coordunits = ['cm^-1', 's', None]
daa = scp.NDDataset(dx, coordset=[coord0, coord1, coord2, cadd, coord2.copy()], title='absorbance',
coordtitles=coordtitles, coordunits=coordunits, )
assert daa.coordset.titles == coordtitles[::-1]
assert daa.dims == ['z', 'y', 'x']
# with a CoordSet
c0, c1 = scp.Coord(labels=['d%d' % i for i in range(6)]), scp.Coord(data=[1, 2, 3, 4, 5, 6])
cc = scp.CoordSet(c0, c1)
cd = scp.CoordSet(x=cc, y=c1)
ds = scp.NDDataset([1, 2, 3, 6, 8, 0], coordset=cd, units='m')
assert ds.dims == ['x']
assert ds.x == cc
ds.history = 'essai: 1'
ds.history = 'try:2'
# wrong type
with pytest.raises(TypeError):
ds.coord[1.3]
# extra coordinates
with pytest.raises(AttributeError):
ds.y
# invalid_length
coord1 = scp.Coord(np.arange(9), title='wavelengths') # , units='m')
coord2 = scp.Coord(np.arange(20), title='time') # , units='s')
with pytest.raises(ValueError):
scp.NDDataset(np.random.random((10, 20)), coordset=(coord1, coord2))
def test_nddataset_repr_html():
dx = np.random.random((10, 100, 3))
coord0 = scp.Coord(
data=np.linspace(4000.0, 1000.0, 10),
labels="a b c d e f g h i j".split(),
mask=None,
units="cm^-1",
title="wavelength",
)
coord1 = scp.Coord(
data=np.linspace(0.0, 60.0, 100),
labels=None,
mask=None,
units="s",
title="time-on-stream",
)
coord2 = scp.Coord(
data=np.linspace(200.0, 300.0, 3),
labels=["cold", "normal", "hot"],
mask=None,
units="K",
title="temperature",
)
da = scp.NDDataset(
dx, coordset=[coord0, coord1, coord2], title="absorbance", units="absorbance"
)
da._repr_html_()
def test_nddataset_repr_html_bug_undesired_display_complex():
da = scp.NDDataset([1, 2, 3])
da.title = 'intensity'
da.description = 'Some experimental measurements'
da.units = 'dimensionless'
assert "(complex)" not in da._repr_html_()
pass
def test_bugs_units_change():
# check for bug on transmittance conversion
X = scp.NDDataset([0.0, 0.3, 1.3, 5.0], units="absorbance")
# A to T
X1 = X.to("transmittance")
assert_array_equal(X1.data, 10 ** -np.array([0.0, 0.3, 1.3, 5.0]) * 100)
assert X1.title == "transmittance"
# T to abs T
X2 = X1.to("absolute_transmittance")
assert_array_equal(X2.data, 10 ** -np.array([0.0, 0.3, 1.3, 5.0]))
assert X2.title == "transmittance"
# A to abs T
X2b = X.to("absolute_transmittance")
assert_array_equal(X2b, X2)
assert X2b.title == "transmittance"
# abs T to T
X3 = X2.to("transmittance")
assert_array_equal(X3, X1)
assert X3.title == "transmittance"
# T to A
X4 = X3.to("absorbance")
assert_array_almost_equal(X4, X)
assert X4.title == "absorbance"
# abs T to A
X5 = X2.to("absorbance")
assert_array_almost_equal(X5, X)
assert X5.title == "absorbance"
def test_nddataset_issue_29_mulitlabels():
DS = scp.NDDataset(np.random.rand(3, 4))
with pytest.raises(ValueError):
# shape data and label mismatch
DS.set_coordset(DS.y, scp.Coord(title='xaxis', units='s', data=[1, 2, 3, 4], labels=['a', 'b', 'c']))
c = scp.Coord(title='xaxis', units='s', data=[1, 2, 3, 4], labels=['a', 'b', 'c', 'd'])
DS.set_coordset(x=c)
c = scp.Coord(title='xaxis', units='s', data=[1, 2, 3, 4], labels=[['a', 'c', 'b', 'd'], ['e', 'f', 'g', 'h']])
d = DS.y
DS.set_coordset(d, c)
DS.x.labels = ['alpha', 'beta', 'omega', 'gamma']
assert DS.x.labels.shape == (4, 3)
# sort
DS1 = DS.sort(axis=1, by='value', descend=True)
assert_array_equal(DS1.x, [4, 3, 2, 1])
# sort
assert DS.dims == ['y', 'x']
DS1 = DS.sort(dim='x', by='label', descend=False)
assert_array_equal(DS1.x, [1, 3, 2, 4])
DS1 = DS.sort(dim='x', by='label', pos=2, descend=False)
assert_array_equal(DS1.x, [1, 2, 4, 3])
DS.sort(dim='y')
DS.y.labels = ['alpha', 'omega', 'gamma']
DS2 = DS.sort(dim='y')
assert_array_equal(DS2.y.labels, ['alpha', 'gamma', 'omega'])
def ds2():
# another dataset
return scp.NDDataset(
ref3d_2_data,
coordset=[coord0_2_, coord1_2_, coord2_2_],
title="absorbance",
units="absorbance",
).copy()
def test_nddataset_real_imag():
na = np.array(
[[1.0 + 2.0j, 2.0 + 0j], [1.3 + 2.0j, 2.0 + 0.5j], [1.0 + 4.2j, 2.0 + 3j]]
)
nd = scp.NDDataset(na)
# in the last dimension
assert_array_equal(nd.real, na.real)
assert_array_equal(nd.imag, na.imag)
def ds1():
# a dataset with coordinates
return scp.NDDataset(
ref3d_data,
coordset=[coord0_, coord1_, coord2_],
title="absorbance",
units="absorbance",
).copy()
def test_nddataset_copy_ref():
"""
Tests to ensure that creating a new NDDataset object copies by *reference*.
"""
a = np.ones((10, 10))
nd_ref = scp.NDDataset(a)
a[0, 0] = 0
assert nd_ref.data[0, 0] == 0
def test_nddataset_coords_indexer():
dx = np.random.random((10, 100, 10))
coord0 = np.linspace(4000, 1000, 10)
coord1 = np.linspace(0, 60, 10) # wrong length
coord2 = np.linspace(20, 30, 10)
with pytest.raises(ValueError): # wrong length
da = scp.NDDataset(
dx,
coordset=[coord0, coord1, coord2],
title="absorbance",
coordtitles=["wavelength", "time-on-stream", "temperature"],
coordunits=["cm^-1", "s", "K"],
)
coord1 = np.linspace(0, 60, 100)
da = scp.NDDataset(
dx,
coordset=[coord0, coord1, coord2],
title="absorbance",
coordtitles=["wavelength", "time-on-stream", "temperature"],
coordunits=["cm^-1", "s", "K"],
)
assert da.shape == (10, 100, 10)
coords = da.coordset
assert len(coords) == 3
assert_array_almost_equal(
da.coordset[2].data, coord0, decimal=2, err_msg="get axis by index " "failed"
)
# we use almost as SpectroChemPy round the coordinate numbers
assert_array_almost_equal(
da.coordset["wavelength"].data,
coord0,
decimal=2,
err_msg="get axis by title failed",
)
assert_array_almost_equal(
da.coordset["time-on-stream"].data,
coord1,
decimal=4,
err_msg="get axis by title failed",
)
assert_array_almost_equal(da.coordset["temperature"].data, coord2, decimal=3)
da.coordset["temperature"] += 273.15 * ur.K
assert_array_almost_equal(
da.coordset["temperature"].data, coord2 + 273.15, decimal=3
)
def test_nddataset_mask_invalid_shape(shape):
with pytest.raises(ValueError) as exc:
with RandomSeedContext(789):
scp.NDDataset(
np.random.random((10, 10)), mask=np.random.random(shape) > 0.5
)
assert exc.value.args[0] == "mask {} and data (10, 10) shape mismatch!".format(
shape
)
def test_nddataset_repr_html():
dx = np.random.random((10, 100, 3))
coord0 = scp.Coord(data=np.linspace(4000., 1000., 10), labels='a b c d e f g h i j'.split(), mask=None,
units="cm^-1", title='wavelength')
coord1 = scp.Coord(data=np.linspace(0., 60., 100), labels=None, mask=None, units="s", title='time-on-stream')
coord2 = scp.Coord(data=np.linspace(200., 300., 3), labels=['cold', 'normal', 'hot'], mask=None, units="K",
title='temperature')
da = scp.NDDataset(dx, coordset=[coord0, coord1, coord2], title='absorbance', units='absorbance')
da._repr_html_()
def test_nddataset_create_from_complex_data_with_units():
# 1D
nd = scp.NDDataset([1. + 2.j, 2. + 0j])
assert nd.data.size == 2
assert nd.size == 2
assert nd.data.shape == (2,)
assert nd.shape == (2,)
# add units
nd.units = 'm**-1'
nd.ito('cm^-1')
# 2D
nd2 = scp.NDDataset([[1. + 2.j, 2. + 0j], [1.3 + 2.j, 2. + 0.5j], [1. + 4.2j, 2. + 3j]])
assert nd2.data.size == 6
assert nd2.size == 6
assert nd2.data.shape == (3, 2)
assert nd2.shape == (3, 2)
# add units
nd2.units = 'm**-1'
nd2.ito('cm^-1')
def test_nddataset_set_complex_1D_during_math_op():
nd = scp.NDDataset([1., 2.], coordset=[scp.Coord([10, 20])], units='meter')
assert nd.data.size == 2
assert nd.size == 2
assert nd.shape == (2,)
assert nd.units == ur.meter
assert not nd.is_complex
ndj = nd * 1j
assert ndj.data.size == 2
assert ndj.is_complex
def test_nddataset_bug_par_arnaud():
import spectrochempy as scp
import numpy as np
x = scp.Coord(data=np.linspace(1000., 4000., num=6000), title='x')
y = scp.Coord(data=np.linspace(0., 10, num=5), title='y')
data = np.random.rand(x.size, y.size)
ds = scp.NDDataset(data, coordset=[x, y])
ds2 = ds[2000.0:3200.0, :]
assert ds2.coordset.y.data.shape[0] == 2400, 'taille axe 0 doit être 2400'
assert ds2.data.shape[0] == 2400, "taille dimension 0 doit être 2400"
def dsm():
# dataset with coords containing several axis and a mask
coordmultiple = scp.CoordSet(coord2_, coord2b_)
return scp.NDDataset(
ref3d_data,
coordset=[coord0_, coord1_, coordmultiple],
mask=ref3d_mask,
title="absorbance",
units="absorbance",
).copy()
def test_nddataset_complex_dataset_slicing_by_index():
na0 = np.array([1. + 2.j, 2., 0., 0., -1.j, 1j] * 4)
nd = scp.NDDataset(na0)
assert nd.shape == (24,)
assert nd.data.shape == (24,)
coords = (np.linspace(-10., 10., 24),)
nd.set_coordset(coords)
x1 = nd.x.copy()
nd.coordset = coords
x2 = nd.x.copy()
assert x1 == x2
# slicing
nd1 = nd[0]
assert nd1.shape == (1,)
assert nd1.data.shape == (1,)
# slicing range
nd2 = nd[1:6]
assert nd2.shape == (5,)
assert nd2.data.shape == (5,)
na0 = na0.reshape(6, 4)
nd = scp.NDDataset(na0)
coords = scp.CoordSet(np.linspace(-10., 10., 6), np.linspace(-1., 1., 4))
nd.set_coordset(**coords)
assert nd.shape == (6, 4)
assert nd.data.shape == (6, 4)
nd.coordset = coords
# slicing 2D
nd1 = nd[0]
assert nd1.shape == (1, 4)
assert nd1.data.shape == (1, 4)
# slicing range
nd1 = nd[1:3]
assert nd1.shape == (2, 4)
assert nd1.data.shape == (2, 4)
# slicing range
nd1 = nd[1:3, 0:2]
assert nd1.shape == (2, 2)
assert nd1.data.shape == (2, 2)
nd.set_complex()
assert nd.shape == (6, 4)
assert nd.data.shape == (6, 4)
def _make_concentrations_matrix(*profiles):
t = scp.LinearCoord(np.linspace(0, 10, 50), units="hour", title="time")
c = []
for p in profiles:
c.append(p(t.data))
ct = np.vstack(c)
ct = ct - np.min(ct)
if ct.shape[0] > 1:
ct = ct / np.sum(ct, axis=0)
ct = scp.NDDataset(data=ct, title="concentration", coordset=[range(len(ct)), t])
return ct
def test_nddataset_masked_array_input():
a = np.random.randn(100)
marr = np.ma.masked_where(a > 0, a)
nd = scp.NDDataset(marr)
# check that masks and data match
assert_array_equal(nd.mask, marr.mask)
assert_array_equal(nd.data, marr.data)
# check that they are both by reference
marr.mask[10] = ~marr.mask[10]
marr.data[11] = 123456789
assert_array_equal(nd.mask, marr.mask)
assert_array_equal(nd.data, marr.data)
def test_nddataset_simple_slicing():
d1 = scp.NDDataset(np.ones((5, 5)))
assert d1.data.shape == (5, 5)
assert d1.shape == (5, 5)
d2 = d1[2:3, 2:3]
assert d2.shape == (1, 1)
assert (d1 is not d2)
d3 = d1[2, 2]
assert d3.shape == (1, 1)
assert d3.squeeze().shape == ()
d3 = d1[0]
assert d3.shape == (1, 5)
def _make_spectra_matrix(modelname, ampl, pos, width, ratio=None, asym=None):
x = scp.Coord(np.linspace(6000.0, 1000.0, 4000), units="cm^-1", title="wavenumbers")
s = []
for arg in zip(modelname, ampl, pos, width, ratio, asym):
model = getattr(scp, arg[0] + "model")()
kwargs = {argname: arg[index + 1] for index, argname in enumerate(model.args)}
s.append(model.f(x.data, **kwargs))
st = np.vstack(s)
st = scp.NDDataset(
data=st, units="absorbance", title="absorbance", coordset=[range(len(st)), x]
)
return st
def simple_project():
proj = scp.Project(
# subprojects
scp.Project(name="P350", label=r"$\mathrm{M_P}\,(623\,K)$"),
scp.Project(name="A350", label=r"$\mathrm{M_A}\,(623\,K)$"),
scp.Project(name="B350", label=r"$\mathrm{M_B}\,(623\,K)$"),
# attributes
name="project_1",
label="main project",
)
assert proj.projects_names == ["P350", "A350", "B350"]
ir = scp.NDDataset([1.1, 2.2, 3.3], coordset=[[1, 2, 3]])
tg = scp.NDDataset([1, 3, 4], coordset=[[1, 2, 3]])
proj.A350["IR"] = ir
proj.A350["TG"] = tg
script_source = (
"set_loglevel(INFO)\n"
'info_(f"samples contained in the project are {proj.projects_names}")')
proj["print_info"] = scp.Script("print_info", script_source)
return proj
