def test_ipfp_no_values():
# 6, 12, 18
along_a = ndtest([(3, 'b')], start=1) * 6
# 6, 12, 18
along_b = ndtest([(3, 'a')], start=1) * 6
r = ipfp([along_a, along_b])
assert_array_equal(r, [[1.0, 2.0, 3.0], [2.0, 4.0, 6.0], [3.0, 6.0, 9.0]])
along_a = Array([2, 1], Axis(2, 'b'))
along_b = Array([1, 2], Axis(2, 'a'))
r = ipfp([along_a, along_b])
assert_array_equal(r, [[2 / 3, 1 / 3], [4 / 3, 2 / 3]])
def test_arrays():
h = ndtest(2)
_h = ndtest(3)
# exclude private arrays
s = arrays()
s_expected = Session([('e', e), ('e2', e2), ('f', f), ('g', g), ('h', h)])
assert s.equals(s_expected)
# all arrays
s = arrays(include_private=True)
s_expected = Session([('_e', _e), ('_h', _h), ('e', e), ('e2', e2), ('f', f), ('g', g), ('h', h)])
assert s.equals(s_expected)
def test_local_arrays():
h = ndtest(2)
_h = ndtest(3)
# exclude private local arrays
s = local_arrays()
s_expected = Session([('h', h)])
assert s.equals(s_expected)
# all local arrays
s = local_arrays(include_private=True)
s_expected = Session([('h', h), ('_h', _h)])
assert s.equals(s_expected)
def test_local_arrays():
h = ndtest(2)
_h = ndtest(3)
# exclude private local arrays
s = local_arrays()
s_expected = Session([('h', h)])
assert s.equals(s_expected)
# all local arrays
s = local_arrays(include_private=True)
s_expected = Session([('h', h), ('_h', _h)])
assert s.equals(s_expected)
def test_local_arrays(self):
local_arr1 = ndtest(2)
_local_arr2 = ndtest(3)
# exclude private local arrays
s = local_arrays()
s_expected = Session([('local_arr1', local_arr1)])
assert s.equals(s_expected)
# all local arrays
s = local_arrays(include_private=True)
s_expected = Session([('local_arr1', local_arr1),
('_local_arr2', _local_arr2)])
assert s.equals(s_expected)
def test_arrays():
i = ndtest(2)
_i = ndtest(3)
# exclude private arrays
s = arrays()
s_expected = Session([('e', e), ('f', f), ('g', g), ('h', h), ('i', i),
('k', k)])
assert s.equals(s_expected)
# all arrays
s = arrays(include_private=True)
s_expected = Session([('_e', _e), ('_i', _i), ('e', e), ('f', f), ('g', g),
('h', h), ('i', i), ('k', k)])
assert s.equals(s_expected)
def test_h5_io(tmpdir, session, meta):
fpath = tmp_path(tmpdir, 'test_session.h5')
session.meta = meta
session.save(fpath)
s = Session()
s.load(fpath)
# HDF does *not* keep ordering (ie, keys are always sorted +
# read Axis objects, then Groups objects and finally LArray objects)
assert list(s.keys()) == ['a', 'b', 'a01', 'b12', 'e', 'f', 'g']
assert s.meta == meta
# update a Group + an Axis + an array (overwrite=False)
a2 = Axis('a=0..2')
a2_01 = a2['0,1'] >> 'a01'
e2 = ndtest((a2, 'b=b0..b2'))
Session(a=a2, a01=a2_01, e=e2).save(fpath, overwrite=False)
s = Session()
s.load(fpath)
assert list(s.keys()) == ['a', 'b', 'a01', 'b12', 'e', 'f', 'g']
assert s['a'].equals(a2)
assert all(s['a01'] == a2_01)
assert_array_nan_equal(s['e'], e2)
assert s.meta == meta
# load only some objects
s = Session()
s.load(fpath, names=['a', 'a01', 'e', 'f'])
assert list(s.keys()) == ['a', 'a01', 'e', 'f']
assert s.meta == meta
def test_xlsx_pandas_io(tmpdir, session, meta):
fpath = tmp_path(tmpdir, 'test_session.xlsx')
session.meta = meta
session.save(fpath, engine='pandas_excel')
s = Session()
s.load(fpath, engine='pandas_excel')
assert list(s.keys()) == ['a', 'b', 'a01', 'b12', 'e', 'g', 'f']
assert s.meta == meta
# update a Group + an Axis + an array
# XXX: overwrite is not taken into account by the pandas_excel engine
a2 = Axis('a=0..2')
a2_01 = a2['0,1'] >> 'a01'
e2 = ndtest((a2, 'b=b0..b2'))
Session(a=a2, a01=a2_01, e=e2, meta=meta).save(fpath, engine='pandas_excel')
s = Session()
s.load(fpath, engine='pandas_excel')
assert list(s.keys()) == ['a', 'a01', 'e']
assert s['a'].equals(a2)
assert all(s['a01'] == a2_01)
assert_array_nan_equal(s['e'], e2)
assert s.meta == meta
# load only some objects
session.save(fpath, engine='pandas_excel')
s = Session()
s.load(fpath, names=['a', 'a01', 'e', 'f'], engine='pandas_excel')
assert list(s.keys()) == ['a', 'a01', 'e', 'f']
assert s.meta == meta
def test_xlsx_xlwings_io(tmpdir, session, meta):
fpath = tmp_path(tmpdir, 'test_session_xw.xlsx')
session.meta = meta
# test save when Excel file does not exist
session.save(fpath, engine='xlwings_excel')
s = Session()
s.load(fpath, engine='xlwings_excel')
# ordering is only kept if the file did not exist previously (otherwise the ordering is left intact)
assert list(s.keys()) == ['a', 'b', 'a01', 'b12', 'e', 'g', 'f']
assert s.meta == meta
# update a Group + an Axis + an array (overwrite=False)
a2 = Axis('a=0..2')
a2_01 = a2['0,1'] >> 'a01'
e2 = ndtest((a2, 'b=b0..b2'))
Session(a=a2, a01=a2_01, e=e2).save(fpath, engine='xlwings_excel', overwrite=False)
s = Session()
s.load(fpath, engine='xlwings_excel')
assert list(s.keys()) == ['a', 'b', 'a01', 'b12', 'e', 'g', 'f']
assert s['a'].equals(a2)
assert all(s['a01'] == a2_01)
assert_array_nan_equal(s['e'], e2)
assert s.meta == meta
# load only some objects
s = Session()
s.load(fpath, names=['a', 'a01', 'e', 'f'], engine='xlwings_excel')
assert list(s.keys()) == ['a', 'a01', 'e', 'f']
assert s.meta == meta
def test_pickle_io(tmpdir, session, meta):
fpath = tmp_path(tmpdir, 'test_session.pkl')
session.meta = meta
session.save(fpath)
s = Session()
s.load(fpath, engine='pickle')
assert list(s.keys()) == ['b', 'a', 'b12', 'a01', 'e', 'g', 'f']
assert s.meta == meta
# update a Group + an Axis + an array (overwrite=False)
a2 = Axis('a=0..2')
a2_01 = a2['0,1'] >> 'a01'
e2 = ndtest((a2, 'b=b0..b2'))
Session(a=a2, a01=a2_01, e=e2).save(fpath, overwrite=False)
s = Session()
s.load(fpath, engine='pickle')
assert list(s.keys()) == ['b', 'a', 'b12', 'a01', 'e', 'g', 'f']
assert s['a'].equals(a2)
assert isinstance(a2_01, Group)
assert isinstance(s['a01'], Group)
assert s['a01'].eval() == a2_01.eval()
assert_array_nan_equal(s['e'], e2)
assert s.meta == meta
# load only some objects
s = Session()
s.load(fpath, names=['a', 'a01', 'e', 'f'], engine='pickle')
assert list(s.keys()) == ['a', 'a01', 'e', 'f']
assert s.meta == meta
def test_ipfp_3d(self):
initial = ndtest((2, 2, 2))
initial_axes = initial.axes
# array sums already match target sums
targets = [initial.sum(axis) for axis in initial.axes]
r = ipfp(targets, initial)
assert_array_equal(r, initial)
self.assertEqual(r.axes, initial_axes)
# array sums do not match target sums (ie the usual case)
targets = [initial.sum(axis) + 1 for axis in initial.axes]
r = ipfp(targets, initial)
assert_array_equal(
r, [[[0.0, 2.0], [2.688963210702341, 3.311036789297659]],
[[4.551453540217585, 5.448546459782415],
[6.450132391879964, 7.549867608120035]]])
self.assertEqual(r.axes, initial_axes)
# same as above but using a more precise threshold
r = ipfp(targets, initial, threshold=0.01)
assert_array_equal(r, [[[0.0, 1.9999999999999998],
[2.994320023433978, 3.0056799765660225]],
[[4.990248916408187, 5.009751083591813],
[6.009541632308118, 7.990458367691883]]])
self.assertEqual(r.axes, initial_axes)
def test_setitem(self):
with open_excel(visible=False) as wb:
# sheet did not exist, str value
wb['sheet1'] = 'sheet1 content'
wb['sheet2'] = 'sheet2 content'
assert wb.sheet_names() == ['sheet1', 'sheet2']
# sheet did exist, str value
wb['sheet2'] = 'sheet2 content v2'
assert wb.sheet_names() == ['sheet1', 'sheet2']
assert wb['sheet2']['A1'].value == 'sheet2 content v2'
# sheet did not exist, Sheet value
wb['sheet3'] = wb['sheet1']
assert wb.sheet_names() == ['sheet1', 'sheet2', 'sheet3']
assert wb['sheet3']['A1'].value == 'sheet1 content'
# sheet did exist, Sheet value
wb['sheet2'] = wb['sheet1']
assert wb.sheet_names() == ['sheet1', 'sheet2', 'sheet3']
assert wb['sheet2']['A1'].value == 'sheet1 content'
with open_excel(visible=False, app="new") as wb2:
with pytest.raises(ValueError) as e_info:
wb2['sheet1'] = wb['sheet1']
assert e_info.value.args[
0] == "cannot copy a sheet from one instance of Excel to another"
# group key
arr = ndtest((3, 3))
for label in arr.b:
wb[label] = arr[label].dump()
assert arr[label].equals(wb[label].load())
def _test_io(tmpdir, session, meta, engine, ext):
filename = f"test_{engine}.{ext}" if 'csv' not in engine else f"test_{engine}{ext}"
fpath = tmp_path(tmpdir, filename)
is_excel_or_csv = 'excel' in engine or 'csv' in engine
kind = Array if is_excel_or_csv else (Axis, Group,
Array) + _supported_scalars_types
session = session.filter(kind=kind)
session.meta = meta
# save and load
session.save(fpath, engine=engine)
s = Session()
s.load(fpath, engine=engine)
# use Session.names instead of Session.keys because CSV, Excel and HDF do *not* keep ordering
assert s.names == session.names
assert s.equals(session)
if not is_excel_or_csv:
for key in s.filter(kind=Axis).keys():
assert s[key].dtype == session[key].dtype
if engine != 'pandas_excel':
assert s.meta == meta
# update a Group + an Axis + an array (overwrite=False)
a4 = Axis('a=0..3')
a4_01 = a3['0,1'] >> 'a01'
e2 = ndtest((a4, 'b=b0..b2'))
h2 = full_like(h, fill_value=10)
Session(a=a4, a01=a4_01, e=e2, h=h2).save(fpath,
overwrite=False,
engine=engine)
s = Session()
s.load(fpath, engine=engine)
if engine == 'pandas_excel':
# Session.save() via engine='pandas_excel' always overwrite the output Excel files
assert s.names == ['e', 'h']
elif is_excel_or_csv:
assert s.names == ['e', 'f', 'g', 'h']
else:
assert s.names == session.names
assert s['a'].equals(a4)
assert s['a01'].equals(a4_01)
assert_array_nan_equal(s['e'], e2)
if engine != 'pandas_excel':
assert s.meta == meta
# load only some objects
session.save(fpath, engine=engine)
s = Session()
names_to_load = ['e', 'f'] if is_excel_or_csv else [
'a', 'a01', 'a2', 'anonymous', 'e', 'f', 's_bool', 's_int'
]
s.load(fpath, names=names_to_load, engine=engine)
assert s.names == names_to_load
if engine != 'pandas_excel':
assert s.meta == meta
return fpath
def test_create_model():
# data = None --> empty array with shape (0,0) is generated
model = ArrayModel()
assert model.get_data_2D().shape == (0, 0)
# data = scalar
model = ArrayModel(LArray(5))
assert model.get_data_2D().shape == (1, 1)
# data = 1D array --> reshaped to 2D array with dim (1, len(data))
model = ArrayModel(ndtest(5))
assert model.get_data_2D().shape == (1, 5)
# data = 3D array --> reshaped to 2D array with dim (axis1*axis2, axis3)
model = ArrayModel(ndtest((5, 5, 5)))
assert model.get_data_2D().shape == (5 * 5, 5)
def test_setitem(self):
with open_excel(visible=False) as wb:
# sheet did not exist, str value
wb['sheet1'] = 'sheet1 content'
wb['sheet2'] = 'sheet2 content'
assert wb.sheet_names() == ['sheet1', 'sheet2']
# sheet did exist, str value
wb['sheet2'] = 'sheet2 content v2'
assert wb.sheet_names() == ['sheet1', 'sheet2']
assert wb['sheet2']['A1'].value == 'sheet2 content v2'
# sheet did not exist, Sheet value
wb['sheet3'] = wb['sheet1']
assert wb.sheet_names() == ['sheet1', 'sheet2', 'sheet3']
assert wb['sheet3']['A1'].value == 'sheet1 content'
# sheet did exist, Sheet value
wb['sheet2'] = wb['sheet1']
assert wb.sheet_names() == ['sheet1', 'sheet2', 'sheet3']
assert wb['sheet2']['A1'].value == 'sheet1 content'
with open_excel(visible=False, app="new") as wb2:
assert wb.app != wb2.app
with pytest.raises(ValueError) as e_info:
wb2['sheet1'] = wb['sheet1']
assert e_info.value.args[0] == "cannot copy a sheet from one instance of Excel to another"
# group key
arr = ndtest((3, 3))
for label in arr.b:
wb[label] = arr[label].dump()
assert arr[label].equals(wb[label].load())
def test_setattr_cs(checkedsession):
cs = checkedsession
# only change values of an array -> OK
cs.h = zeros_like(h)
# trying to add an undeclared variable -> prints a warning message
with must_warn(UserWarning, msg=f"'i' is not declared in '{cs.__class__.__name__}'"):
cs.i = ndtest((3, 3))
# trying to set a variable with an object of different type -> should fail
# a) type given explicitly
# -> Axis
with must_raise(TypeError, msg="instance of Axis expected"):
cs.a = 0
# -> CheckedArray
with must_raise(TypeError, msg="Expected object of type 'Array' or a scalar for the variable 'h' but got "
"object of type 'ndarray'"):
cs.h = h.data
# b) type deduced from the given default value
with must_raise(TypeError, msg="instance of Axis expected"):
cs.b = ndtest((3, 3))
# trying to set a CheckedArray variable using a scalar -> OK
cs.h = 5
# trying to set a CheckedArray variable using an array with axes in different order -> OK
cs.h = h.transpose()
assert cs.h.axes.names == h.axes.names
# broadcasting (missing axis) is allowed
cs.h = ndtest(a3)
assert_array_nan_equal(cs.h['b0'], cs.h['b1'])
# trying to set a CheckedArray variable using an array with wrong axes -> should fail
# a) extra axis
with must_raise(ValueError, msg="Array 'h' was declared with axes {a, b} but got array with axes {a, b, c} "
"(unexpected {c} axis)"):
cs.h = ndtest((a3, b2, 'c=c0..c2'))
# b) incompatible axis
msg = """\
Incompatible axis for array 'h':
Axis(['a0', 'a1', 'a2', 'a3', 'a4'], 'a')
vs
Axis(['a0', 'a1', 'a2', 'a3'], 'a')"""
with must_raise(ValueError, msg=msg):
cs.h = h.append('a', 0, 'a4')
def test_repr(self):
with open_excel(visible=False) as wb:
sheet = wb[0]
arr1 = ndtest((2, 3))
sheet['A1'] = arr1
res = repr(sheet['A1:C2'])
assert res == """\
def test_sub(session):
sess = session
# session - session
other = Session({'e': e, 'f': f})
other['e'] = e - 1
other['f'] = ones_like(f)
diff = sess - other
assert_array_nan_equal(diff['e'], np.full((2, 3), 1, dtype=np.int32))
assert_array_nan_equal(diff['f'], f - ones_like(f))
assert isnan(diff['g']).all()
assert diff.a is a
assert diff.a01 is a01
assert diff.c is c
# session - scalar
diff = sess - 2
assert_array_nan_equal(diff['e'], e - 2)
assert_array_nan_equal(diff['f'], f - 2)
assert_array_nan_equal(diff['g'], g - 2)
assert diff.a is a
assert diff.a01 is a01
assert diff.c is c
# session - dict(Array and scalar)
other = {'e': ones_like(e), 'f': 1}
diff = sess - other
assert_array_nan_equal(diff['e'], e - ones_like(e))
assert_array_nan_equal(diff['f'], f - 1)
assert isnan(diff['g']).all()
assert diff.a is a
assert diff.a01 is a01
assert diff.c is c
# session - array
axes = [a, b]
other = Session([('a', a), ('a01', a01), ('c', c), ('e', ndtest((a, b))),
('f', full((a, b), fill_value=3)),
('g', ndtest('c=c0..c2'))])
diff = other - ones(axes)
assert_array_nan_equal(diff['e'], other['e'] - ones(axes))
assert_array_nan_equal(diff['f'], other['f'] - ones(axes))
assert_array_nan_equal(diff['g'], other['g'] - ones(axes))
assert diff.a is a
assert diff.a01 is a01
assert diff.c is c
def test_asarray(self):
with open_excel(visible=False) as wb:
sheet = wb[0]
arr1 = ndtest((2, 3))
# no header so that we have an uniform dtype for the whole sheet
sheet['A1'] = arr1
res1 = np.asarray(sheet['A1:C2'])
assert np.array_equal(res1, arr1.data)
assert res1.dtype == arr1.dtype
arr1 = ndtest([Axis(2), Axis(3)])
# no header so that we have an uniform dtype for the whole sheet
sheet['A1'] = arr1
res1 = asarray(sheet['A1:C2'])
assert res1.equals(arr1)
assert res1.dtype == arr1.dtype
def test_repr(self):
with open_excel(visible=False) as wb:
sheet = wb[0]
arr1 = ndtest((2, 3))
sheet['A1'] = arr1
res = repr(sheet['A1:C2'])
assert res == """\
def test_aggregate(self):
with open_excel(visible=False) as wb:
sheet = wb[0]
arr1 = ndtest((2, 3))
# no header so that we have an uniform dtype for the whole sheet
sheet['A1'] = arr1
res = sheet['A1:C2'].sum()
assert res == 15
def test_aggregate(self):
with open_excel(visible=False) as wb:
sheet = wb[0]
arr1 = ndtest((2, 3))
# no header so that we have an uniform dtype for the whole sheet
sheet['A1'] = arr1
res = sheet['A1:C2'].sum()
assert res == 15
def setUp(self):
self.a = Axis([], 'a')
self.b = Axis([], 'b')
self.c = 'c'
self.d = {}
self.e = ndtest([(2, 'a0'), (3, 'a1')])
self.e2 = ndtest(('a=a0..a2', 'b=b0..b2'))
self.f = ndtest([(3, 'a0'), (2, 'a1')])
self.g = ndtest([(2, 'a0'), (4, 'a1')])
self.session = Session([
('b', self.b),
('a', self.a),
('c', self.c),
('d', self.d),
('e', self.e),
('g', self.g),
('f', self.f),
])
def test_aslarray(self):
with open_excel(visible=False) as wb:
sheet = wb[0]
arr1 = ndtest([Axis(2), Axis(3)])
# no header so that we have an uniform dtype for the whole sheet
sheet['A1'] = arr1
res1 = aslarray(sheet['A1:C2'])
assert res1.equals(arr1)
assert res1.dtype == arr1.dtype
def test_array_method(self):
with open_excel(visible=False) as wb:
sheet = wb[0]
# normal test array
arr1 = ndtest((2, 3))
sheet['A1'] = arr1.dump()
res1 = sheet.array('B2:D3', 'A2:A3', 'B1:D1', names=['a', 'b'])
assert arr1.equals(res1)
# array with int labels
arr2 = ndtest('0..1;0..2')
sheet['A1'] = arr2.dump()
res2 = sheet.array('B2:D3', 'A2:A3', 'B1:D1')
# larray_equal passes even if the labels are floats...
assert arr2.equals(res2)
# so we check the dtype explicitly
assert res2.axes[0].labels.dtype == arr2.axes[0].labels.dtype
assert res2.axes[1].labels.dtype == arr2.axes[1].labels.dtype
def test_asarray(self):
with open_excel(visible=False) as wb:
sheet = wb[0]
arr1 = ndtest((2, 3))
# no header so that we have an uniform dtype for the whole sheet
sheet['A1'] = arr1
res1 = np.asarray(sheet['A1:C2'])
assert np.array_equal(res1, arr1.data)
assert res1.dtype == arr1.dtype
def test_array_method(self):
with open_excel(visible=False) as wb:
sheet = wb[0]
# normal test array
arr1 = ndtest((2, 3))
sheet['A1'] = arr1.dump()
res1 = sheet.array('B2:D3', 'A2:A3', 'B1:D1', names=['a', 'b'])
assert arr1.equals(res1)
# array with int labels
arr2 = ndtest('0..1;0..2')
sheet['A1'] = arr2.dump()
res2 = sheet.array('B2:D3', 'A2:A3', 'B1:D1')
# larray_equal passes even if the labels are floats...
assert arr2.equals(res2)
# so we check the dtype explicitly
assert res2.axes[0].labels.dtype == arr2.axes[0].labels.dtype
assert res2.axes[1].labels.dtype == arr2.axes[1].labels.dtype
def test_getitem_lgroup():
axis = Axis("a=a0,a1")
assert axis['a0'][0] == 'a'
assert axis['a0'][1] == '0'
assert axis['a0':'a1'][1] == 'a1'
assert axis[:][1] == 'a1'
assert list(axis[:][0:2]) == ['a0', 'a1']
assert list((axis[:][[1, 0]])) == ['a1', 'a0']
assert axis[['a0', 'a1', 'a0']][2] == 'a0'
assert axis[('a0', 'a1', 'a0')][2] == 'a0'
assert axis[ndtest("a=a0,a1,a0")][2] == 2
def test_getitem(self):
axis = Axis("a=a0,a1")
assert axis['a0'][0] == 'a'
assert axis['a0'][1] == '0'
assert axis['a0':'a1'][1] == 'a1'
assert axis[:][1] == 'a1'
assert list(axis[:][0:2]) == ['a0', 'a1']
assert list((axis[:][[1, 0]])) == ['a1', 'a0']
assert axis[['a0', 'a1', 'a0']][2] == 'a0'
assert axis[('a0', 'a1', 'a0')][2] == 'a0'
assert axis[ndtest("a=a0,a1,a0")][2] == 2
def test_get_and_set_item(self):
arr = ndtest((2, 3))
with open_excel(visible=False) as wb:
sheet = wb[0]
# set a few values
sheet['A1'] = 1.5
sheet['A2'] = 2
sheet['A3'] = True
sheet['A4'] = 'toto'
# array without header
sheet['A5'] = arr
# array with header
sheet['A8'] = arr.dump()
# object array with a *numpy* NaN (dc2019 infamous 65535 bug)
obj_arr = ndtest((2, 3)).astype(object)
obj_arr['a0', 'b1'] = np.float64('nan')
assert type(obj_arr['a0', 'b1']) is np.float64
obj_arr_dump = obj_arr.dump()
# [['a\\b', 'b0', 'b1', 'b2'], ['a0', 0, nan, 2], ['a1', 3, 4, 5]]
# float and *not* np.float64, otherwise it gets converted to 65535 when written to Excel
assert type(obj_arr_dump[1][2]) is float
sheet['A12'] = obj_arr_dump
# read them back
assert sheet['A1'].value == 1.5
assert sheet['A2'].value == 2
assert sheet['A3'].value is True
assert sheet['A4'].value == 'toto'
# array without header
assert np.array_equal(sheet['A5:C6'].value, arr.data)
# array with header
assert sheet['A8:D10'].load().equals(arr)
assert sheet['A12:D14'].load().equals(obj_arr, nans_equal=True)
def test_ipfp_3d_with_axes(self):
initial = ndtest((2, 2, 2))
initial_axes = initial.axes
# array sums already match target sums (first axes)
axes = (X.a, X.b)
targets = [initial.sum(axis) for axis in axes]
r = ipfp(targets, initial, axes=axes)
assert_array_equal(r, initial)
self.assertEqual(r.axes, initial_axes)
# array sums already match target sums (other axes)
axes = (X.a, X.c)
targets = [initial.sum(axis) for axis in axes]
r = ipfp(targets, initial, axes=axes)
assert_array_equal(r, initial)
self.assertEqual(r.axes, initial_axes)
# array sums do not match target sums (ie the usual case) (first axes)
axes = (X.a, X.b)
targets = [initial.sum(axis) + 1 for axis in axes]
r = ipfp(targets, initial, axes=axes)
assert_array_equal(
r, [[[0.0, 1.3059701492537312], [3.0, 3.6940298507462686]],
[[4.680851063829787, 5.603448275862069],
[6.319148936170213, 7.3965517241379315]]])
self.assertEqual(r.axes, initial_axes)
# check that the result is the same as N 2D ipfp calls
assert_array_equal(r['c0'],
ipfp([t['c0'] for t in targets], initial['c0']))
assert_array_equal(r['c1'],
ipfp([t['c1'] for t in targets], initial['c1']))
# array sums do not match target sums (ie the usual case) (other axes)
axes = (X.a, X.c)
targets = [initial.sum(axis) + 1 for axis in axes]
r = ipfp(targets, initial, axes=axes)
assert_array_equal(
r, [[[0.0, 2.0], [2.432432432432432, 3.567567567567567]],
[[4.615384615384615, 5.384615384615385],
[6.539792387543252, 7.460207612456748]]])
self.assertEqual(r.axes, initial_axes)
# check that the result is the same as N 2D ipfp calls
assert_array_equal(r['b0'],
ipfp([t['b0'] for t in targets], initial['b0']))
assert_array_equal(r['b1'],
ipfp([t['b1'] for t in targets], initial['b1']))
def test_set_options_as_global():
original_ops = larray.get_options()
arr = larray.ndtest((500, 100))
larray.set_options(display_width=40, display_maxlines=10)
expected = """\
a\\b b0 b1 ... b98 b99
a0 0 1 ... 98 99
a1 100 101 ... 198 199
a2 200 201 ... 298 299
a3 300 301 ... 398 399
a4 400 401 ... 498 499
... ... ... ... ... ...
a495 49500 49501 ... 49598 49599
a496 49600 49601 ... 49698 49699
a497 49700 49701 ... 49798 49799
a498 49800 49801 ... 49898 49899
a499 49900 49901 ... 49998 49999"""
assert str(arr) == expected
larray.set_options(**original_ops)
def test_set_options_as_global():
original_ops = larray.get_options()
arr = larray.ndtest((500, 100))
larray.set_options(display_width=40, display_maxlines=10)
expected = """\
a\\b b0 b1 ... b98 b99
a0 0 1 ... 98 99
a1 100 101 ... 198 199
a2 200 201 ... 298 299
a3 300 301 ... 398 399
a4 400 401 ... 498 499
... ... ... ... ... ...
a495 49500 49501 ... 49598 49599
a496 49600 49601 ... 49698 49699
a497 49700 49701 ... 49798 49799
a498 49800 49801 ... 49898 49899
a499 49900 49901 ... 49998 49999"""
assert str(arr) == expected
larray.set_options(**original_ops)
def test_get_and_set_item(self):
arr = ndtest((2, 3))
with open_excel(visible=False) as wb:
sheet = wb[0]
# set a few values
sheet['A1'] = 1.5
sheet['A2'] = 2
sheet['A3'] = True
sheet['A4'] = 'toto'
# array without header
sheet['A5'] = arr
# array with header
sheet['A8'] = arr.dump()
# read them back
assert sheet['A1'].value == 1.5
assert sheet['A2'].value == 2
assert sheet['A3'].value == True
assert sheet['A4'].value == 'toto'
# array without header
assert np.array_equal(sheet['A5:C6'].value, arr.data)
# array with header
assert arr.equals(sheet['A8:D10'].load())
def test_get_and_set_item(self):
arr = ndtest((2, 3))
with open_excel(visible=False) as wb:
sheet = wb[0]
# set a few values
sheet['A1'] = 1.5
sheet['A2'] = 2
sheet['A3'] = True
sheet['A4'] = 'toto'
# array without header
sheet['A5'] = arr
# array with header
sheet['A8'] = arr.dump()
# read them back
assert sheet['A1'].value == 1.5
assert sheet['A2'].value == 2
assert sheet['A3'].value == True
assert sheet['A4'].value == 'toto'
# array without header
assert np.array_equal(sheet['A5:C6'].value, arr.data)
# array with header
assert arr.equals(sheet['A8:D10'].load())
# arr2.i[0, 0, 0, 0] = np.inf
# arr2.i[0, 0, 1, 1] = -np.inf
# arr2 = [0.0000111, 0.0000222]
# arr2 = [0.00001, 0.00002]
# edit(arr2, minvalue=-99, maxvalue=25.123456)
# print(arr2[0, 'A11', :, 'P01'])
# arr2 = la.random.normal(0, 10, axes="d0=0..4999;d1=0..19")
# edit(arr2)
# view(['a', 'bb', 5599])
# view(np.arange(12).reshape(2, 3, 2))
# view([])
data3 = np.random.normal(0, 1, size=(2, 15))
arr3 = la.ndtest((30, sex))
# data4 = np.random.normal(0, 1, size=(2, 15))
# arr4 = la.Array(data4, axes=(sex, lipro))
# arr4 = arr3.copy()
# arr4['F'] /= 2
arr4 = arr3.min(sex)
arr5 = arr3.max(sex)
arr6 = arr3.mean(sex)
# test isssue #35
arr7 = la.from_lists(
[['a', 1, 2, 3],
['', 1664780726569649730, -9196963249083393206, -7664327348053294350]])
def test_astype():
arr = ndtest(Axis('time=2015..2020,total')).drop('total')
time = arr.time
assert time.dtype.kind == 'U'
time = time.astype(int)
assert time.dtype.kind == 'i'
try:
import xlwings as xw
except ImportError:
xw = None
def equal(o1, o2):
if isinstance(o1, LArray) or isinstance(o2, LArray):
return o1.equals(o2)
elif isinstance(o1, Axis) or isinstance(o2, Axis):
return o1.equals(o2)
else:
return o1 == o2
global_arr1 = ndtest((2, 2))
_global_arr2 = ndtest((3, 3))
class TestSession(TestCase):
def setUp(self):
self.a = Axis([], 'a')
self.b = Axis([], 'b')
self.c = 'c'
self.d = {}
self.e = ndtest([(2, 'a0'), (3, 'a1')])
self.e2 = ndtest(('a=a0..a2', 'b=b0..b2'))
self.f = ndtest([(3, 'a0'), (2, 'a1')])
self.g = ndtest([(2, 'a0'), (4, 'a1')])
self.session = Session([
('b', self.b),
def data():
return ndtest((5, 5, 5, 5))
for e1, e2 in zip(got, expected):
assert equal(e1, e2), f"{e1} != {e2}"
a = Axis('a=a0..a2')
a2 = Axis('a=a0..a4')
a3 = Axis('a=a0..a3')
anonymous = Axis(4)
a01 = a['a0,a1'] >> 'a01'
ano01 = a['a0,a1']
b = Axis('b=0..4')
b2 = Axis('b=b0..b4')
b024 = b[[0, 2, 4]] >> 'b024'
c = 'c'
d = {}
e = ndtest([(2, 'a'), (3, 'b')])
_e = ndtest((3, 3))
f = ndtest((Axis(3), Axis(2)), dtype=float)
g = ndtest([(2, 'a'), (4, 'b')])
h = ndtest((a3, b2))
k = ndtest((3, 3))
# ########################### #
# SESSION #
# ########################### #
@pytest.fixture()
def session():
return Session([('b', b), ('b024', b024), ('a', a), ('a2', a2),
('anonymous', anonymous), ('a01', a01), ('ano01', ano01),
return o1 == o2
def assertObjListEqual(got, expected):
assert len(got) == len(expected)
for e1, e2 in zip(got, expected):
assert equal(e1, e2), "{} != {}".format(e1, e2)
a = Axis('a=a0..a2')
a01 = a['a0,a1'] >> 'a01'
b = Axis('b=b0..b2')
b12 = b['b1,b2'] >> 'b12'
c = 'c'
d = {}
e = ndtest([(2, 'a0'), (3, 'a1')])
_e = ndtest((3, 3))
e2 = ndtest(('a=a0..a2', 'b=b0..b2'))
f = ndtest([(3, 'a0'), (2, 'a1')])
g = ndtest([(2, 'a0'), (4, 'a1')])
@pytest.fixture()
def session():
return Session([('b', b), ('b12', b12), ('a', a), ('a01', a01),
('c', c), ('d', d), ('e', e), ('g', g), ('f', f)])
def test_init_session(meta):
s = Session(b, b12, a, a01, c=c, d=d, e=e, f=f, g=g)
assert s.names == ['a', 'a01', 'b', 'b12', 'c', 'd', 'e', 'f', 'g']
def test_sub_cs(checkedsession):
cs = checkedsession
session_cls = cs.__class__
# session - session
other = session_cls(a=a, a2=a2, a01=a01, e=e - 1, g=zeros_like(g), f=zeros_like(f), h=ones_like(h))
diff = cs - other
assert isinstance(diff, session_cls)
# --- non-array variables ---
assert diff.b is b
assert diff.b024 is b024
assert diff.a is a
assert diff.a2 is a2
assert diff.anonymous is anonymous
assert diff.a01 is a01
assert diff.ano01 is ano01
assert diff.c is c
assert diff.d is d
# --- array variables ---
assert_array_nan_equal(diff.e, np.full((2, 3), 1, dtype=np.int32))
assert_array_nan_equal(diff.g, g)
assert_array_nan_equal(diff.f, f)
assert_array_nan_equal(diff.h, h - ones_like(h))
# session - scalar
diff = cs - 2
assert isinstance(diff, session_cls)
# --- non-array variables ---
assert diff.b is b
assert diff.b024 is b024
assert diff.a is a
assert diff.a2 is a2
assert diff.anonymous is anonymous
assert diff.a01 is a01
assert diff.ano01 is ano01
assert diff.c is c
assert diff.d is d
# --- non constant arrays ---
assert_array_nan_equal(diff.e, e - 2)
assert_array_nan_equal(diff.g, g - 2)
assert_array_nan_equal(diff.f, f - 2)
assert_array_nan_equal(diff.h, h - 2)
# session - dict(Array and scalar)
other = {'e': ones_like(e), 'h': 1}
diff = cs - other
assert isinstance(diff, session_cls)
# --- non-array variables ---
assert diff.b is b
assert diff.b024 is b024
assert diff.a is a
assert diff.a2 is a2
assert diff.anonymous is anonymous
assert diff.a01 is a01
assert diff.ano01 is ano01
assert diff.c is c
assert diff.d is d
# --- non constant arrays ---
assert_array_nan_equal(diff.e, e - ones_like(e))
assert isnan(diff.g).all()
assert isnan(diff.f).all()
assert_array_nan_equal(diff.h, h - 1)
# session - array
axes = cs.h.axes
cs.e = ndtest(axes)
cs.g = ones_like(cs.h)
diff = cs - ones(axes)
assert isinstance(diff, session_cls)
# --- non-array variables ---
assert diff.b is b
assert diff.b024 is b024
assert diff.a is a
assert diff.a2 is a2
assert diff.anonymous is anonymous
assert diff.a01 is a01
assert diff.ano01 is ano01
assert diff.c is c
assert diff.d is d
# --- non constant arrays ---
assert_array_nan_equal(diff.e, cs.e - ones(axes))
assert_array_nan_equal(diff.g, cs.g - ones(axes))
assert isnan(diff.f).all()
assert_array_nan_equal(diff.h, cs.h - ones(axes))
def _test_io_cs(tmpdir, meta, engine, ext):
filename = f"test_{engine}.{ext}" if 'csv' not in engine else f"test_{engine}{ext}"
fpath = tmp_path(tmpdir, filename)
is_excel_or_csv = 'excel' in engine or 'csv' in engine
# Save and load
# -------------
# a) - all typed variables have a defined value
# - no extra variables are added
csession = TestCheckedSession(a=a, a2=a2, a01=a01, d=d, e=e, g=g, f=f, h=h, meta=meta)
csession.save(fpath, engine=engine)
cs = TestCheckedSession()
cs.load(fpath, engine=engine)
# --- keys ---
assert list(cs.keys()) == list(csession.keys())
# --- variables with default values ---
assert cs.b.equals(b)
assert cs.b024.equals(b024)
assert cs.anonymous.equals(anonymous)
assert cs.ano01.equals(ano01)
assert cs.d == d
# --- typed variables ---
# Array is support by all formats
assert cs.e.equals(e)
assert cs.g.equals(g)
assert cs.f.equals(f)
assert cs.h.equals(h)
# Axis and Group are not supported by the Excel and CSV formats
if is_excel_or_csv:
assert isinstance(cs.a, NotLoaded)
assert isinstance(cs.a2, NotLoaded)
assert isinstance(cs.a01, NotLoaded)
else:
assert cs.a.equals(a)
assert cs.a2.equals(a2)
assert cs.a01.equals(a01)
# --- dtype of Axis variables ---
if not is_excel_or_csv:
for key in cs.filter(kind=Axis).keys():
assert cs[key].dtype == csession[key].dtype
# --- metadata ---
if engine != 'pandas_excel':
assert cs.meta == meta
# b) - not all typed variables have a defined value
# - no extra variables are added
csession = TestCheckedSession(a=a, d=d, e=e, h=h, meta=meta)
if 'csv' in engine:
import shutil
shutil.rmtree(fpath)
csession.save(fpath, engine=engine)
cs = TestCheckedSession()
cs.load(fpath, engine=engine)
# --- keys ---
assert list(cs.keys()) == list(csession.keys())
# --- variables with default values ---
assert cs.b.equals(b)
assert cs.b024.equals(b024)
assert cs.anonymous.equals(anonymous)
assert cs.ano01.equals(ano01)
assert cs.d == d
# --- typed variables ---
# Array is support by all formats
assert cs.e.equals(e)
assert isinstance(cs.g, NotLoaded)
assert isinstance(cs.f, NotLoaded)
assert cs.h.equals(h)
# Axis and Group are not supported by the Excel and CSV formats
if is_excel_or_csv:
assert isinstance(cs.a, NotLoaded)
assert isinstance(cs.a2, NotLoaded)
assert isinstance(cs.a01, NotLoaded)
else:
assert cs.a.equals(a)
assert isinstance(cs.a2, NotLoaded)
assert isinstance(cs.a01, NotLoaded)
# c) - all typed variables have a defined value
# - extra variables are added
i = ndtest(6)
j = ndtest((3, 3))
k = ndtest((2, 2))
csession = TestCheckedSession(a=a, a2=a2, a01=a01, d=d, e=e, g=g, f=f, h=h, k=k, j=j, i=i, meta=meta)
csession.save(fpath, engine=engine)
cs = TestCheckedSession()
cs.load(fpath, engine=engine)
# --- names ---
# we do not use keys() since order of undeclared variables
# may not be preserved (at least for the HDF format)
assert cs.names == csession.names
# --- extra variable ---
assert cs.i.equals(i)
assert cs.j.equals(j)
assert cs.k.equals(k)
# Update a Group + an Axis + an array (overwrite=False)
# -----------------------------------------------------
csession = TestCheckedSession(a=a, a2=a2, a01=a01, d=d, e=e, g=g, f=f, h=h, meta=meta)
csession.save(fpath, engine=engine)
a4 = Axis('a=0..3')
a4_01 = a3['0,1'] >> 'a01'
e2 = ndtest((a4, 'b=b0..b2'))
h2 = full_like(h, fill_value=10)
TestCheckedSession(a=a4, a01=a4_01, e=e2, h=h2).save(fpath, overwrite=False, engine=engine)
cs = TestCheckedSession()
cs.load(fpath, engine=engine)
# --- variables with default values ---
assert cs.b.equals(b)
assert cs.b024.equals(b024)
assert cs.anonymous.equals(anonymous)
assert cs.ano01.equals(ano01)
# --- typed variables ---
# Array is support by all formats
assert cs.e.equals(e2)
assert cs.h.equals(h2)
if engine == 'pandas_excel':
# Session.save() via engine='pandas_excel' always overwrite the output Excel files
# arrays 'g' and 'f' have been dropped
assert isinstance(cs.g, NotLoaded)
assert isinstance(cs.f, NotLoaded)
# Axis and Group are not supported by the Excel and CSV formats
assert isinstance(cs.a, NotLoaded)
assert isinstance(cs.a2, NotLoaded)
assert isinstance(cs.a01, NotLoaded)
elif is_excel_or_csv:
assert cs.g.equals(g)
assert cs.f.equals(f)
# Axis and Group are not supported by the Excel and CSV formats
assert isinstance(cs.a, NotLoaded)
assert isinstance(cs.a2, NotLoaded)
assert isinstance(cs.a01, NotLoaded)
else:
assert list(cs.keys()) == list(csession.keys())
assert cs.a.equals(a4)
assert cs.a2.equals(a2)
assert cs.a01.equals(a4_01)
assert cs.g.equals(g)
assert cs.f.equals(f)
if engine != 'pandas_excel':
assert cs.meta == meta
# Load only some objects
# ----------------------
csession = TestCheckedSession(a=a, a2=a2, a01=a01, d=d, e=e, g=g, f=f, h=h, meta=meta)
csession.save(fpath, engine=engine)
cs = TestCheckedSession()
names_to_load = ['e', 'h'] if is_excel_or_csv else ['a', 'a01', 'a2', 'e', 'h']
cs.load(fpath, names=names_to_load, engine=engine)
# --- keys ---
assert list(cs.keys()) == list(csession.keys())
# --- variables with default values ---
assert cs.b.equals(b)
assert cs.b024.equals(b024)
assert cs.anonymous.equals(anonymous)
assert cs.ano01.equals(ano01)
assert cs.d == d
# --- typed variables ---
# Array is support by all formats
assert cs.e.equals(e)
assert isinstance(cs.g, NotLoaded)
assert isinstance(cs.f, NotLoaded)
assert cs.h.equals(h)
# Axis and Group are not supported by the Excel and CSV formats
if is_excel_or_csv:
assert isinstance(cs.a, NotLoaded)
assert isinstance(cs.a2, NotLoaded)
assert isinstance(cs.a01, NotLoaded)
else:
assert cs.a.equals(a)
assert cs.a2.equals(a2)
assert cs.a01.equals(a01)
return fpath
def generate_tests_files():
tests = {
'1d': 3,
'2d': "a=1..3; b=b0,b1",
'2d_classic': "a=a0..a2;b=b0..b2",
'3d': "a=1..3; b=b0,b1; c=c0..c2",
'int_labels': "a=0..2; b=0..2; c=0..2",
'missing_values': "a=1..3; b=b0,b1; c=c0..c2",
'unsorted': "a=3..1; b=b1,b0; c=c2..c0",
'position': "a=1..3; b=b0,b1; c=c0..c2"
}
wb = open_excel(os.path.join(DATA_DIR, 'test.xlsx'), overwrite_file=True)
wb_narrow = open_excel(os.path.join(DATA_DIR, 'test_narrow.xlsx'),
overwrite_file=True)
for name, dim in tests.items():
arr = ndtest(dim)
if name == '2d_classic':
df = arr.to_frame(fold_last_axis_name=False)
# wide format
df.to_csv(os.path.join(DATA_DIR, f'test{name}.csv'),
sep=',',
na_rep='')
wb[name] = ''
wb[name]['A1'].options().value = df
# narrow format
df = arr.to_series(name='value')
df.to_csv(os.path.join(DATA_DIR, f'test{name}_narrow.csv'),
sep=',',
na_rep='',
header=True)
wb_narrow[name] = ''
wb_narrow[name]['A1'].options().value = df
elif name == 'missing_values':
df = arr.to_frame(fold_last_axis_name=True)
# wide format
df = df.drop([(2, 'b0'), (3, 'b1')])
df.to_csv(os.path.join(DATA_DIR, f'test{name}.csv'),
sep=',',
na_rep='')
wb[name] = ''
wb[name]['A1'].options().value = df
# narrow format
df = arr.to_series(name='value')
df = df.drop([(2, 'b0'), (2, 'b1', 'c1'), (3, 'b1')])
df.to_csv(os.path.join(DATA_DIR, f'test{name}_narrow.csv'),
sep=',',
na_rep='',
header=True)
wb_narrow[name] = ''
wb_narrow[name]['A1'].options().value = df
elif name == 'position':
# wide format
wb[name] = ''
wb[name]['D3'] = arr.dump()
# narrow format
wb_narrow[name] = ''
wb_narrow[name]['D3'] = arr.dump(wide=False)
else:
# wide format
arr.to_csv(os.path.join(DATA_DIR, f'test{name}.csv'))
wb[name] = arr.dump()
# narrow format
arr.to_csv(os.path.join(DATA_DIR, f'test{name}_narrow.csv'),
wide=False)
wb_narrow[name] = arr.dump(wide=False)
wb.save()
wb.close()
wb_narrow.save()
wb_narrow.close()
def data():
return la.ndtest((5, 5, 5, 5))
