def setup(self):
# Generic setup
_a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
_b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
_c = ma.array(list(map(asbytes, ["one", "two", "three"])), mask=[0, 0, 1], dtype="|S8")
ddtype = [("a", int), ("b", float), ("c", "|S8")]
mrec = fromarrays([_a, _b, _c], dtype=ddtype, fill_value=(asbytes("99999"), asbytes("99999."), asbytes("N/A")))
nrec = recfromarrays((_a._data, _b._data, _c._data), dtype=ddtype)
self.data = (mrec, nrec, ddtype)
def setup(self):
"Generic setup"
_a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
_b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
_c = ma.array(["one", "two", "three"], mask=[0, 0, 1], dtype="|S8")
ddtype = [("a", int), ("b", float), ("c", "|S8")]
mrec = fromarrays([_a, _b, _c], dtype=ddtype, fill_value=(99999, 99999.0, "N/A"))
nrec = recfromarrays((_a.data, _b.data, _c.data), dtype=ddtype)
self.data = (mrec, nrec, ddtype)
def setup(self):
"Generic setup"
_a = ma.array([1,2,3],mask=[0,0,1],dtype=int)
_b = ma.array([1.1,2.2,3.3],mask=[0,0,1],dtype=float)
_c = ma.array(['one','two','three'],mask=[0,0,1],dtype='|S8')
ddtype = [('a',int),('b',float),('c','|S8')]
mrec = fromarrays([_a,_b,_c], dtype=ddtype,
fill_value=(99999,99999.,'N/A'))
nrec = recfromarrays((_a._data,_b._data,_c._data), dtype=ddtype)
self.data = (mrec, nrec, ddtype)
def fromarrays(
arraylist,
dtype=None,
shape=None,
formats=None,
names=None,
titles=None,
aligned=False,
byteorder=None,
fill_value=None,
):
"""
Creates a mrecarray from a (flat) list of masked arrays.
Parameters
----------
arraylist : sequence
A list of (masked) arrays. Each element of the sequence is first converted
to a masked array if needed. If a 2D array is passed as argument, it is
processed line by line
dtype : {None, dtype}, optional
Data type descriptor.
shape : {None, integer}, optional
Number of records. If None, shape is defined from the shape of the
first array in the list.
formats : {None, sequence}, optional
Sequence of formats for each individual field. If None, the formats will
be autodetected by inspecting the fields and selecting the highest dtype
possible.
names : {None, sequence}, optional
Sequence of the names of each field.
fill_value : {None, sequence}, optional
Sequence of data to be used as filling values.
Notes
-----
Lists of tuples should be preferred over lists of lists for faster processing.
"""
datalist = [getdata(x) for x in arraylist]
masklist = [np.atleast_1d(getmaskarray(x)) for x in arraylist]
_array = recfromarrays(
datalist,
dtype=dtype,
shape=shape,
formats=formats,
names=names,
titles=titles,
aligned=aligned,
byteorder=byteorder,
).view(mrecarray)
_array._mask.flat = list(zip(*masklist))
if fill_value is not None:
_array.fill_value = fill_value
return _array
def setup(self):
"Generic setup"
_a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
_b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
_c = ma.array(['one', 'two', 'three'], mask=[0, 0, 1], dtype='|S8')
ddtype = [('a', int), ('b', float), ('c', '|S8')]
mrec = fromarrays([_a, _b, _c],
dtype=ddtype,
fill_value=(99999, 99999., 'N/A'))
nrec = recfromarrays((_a._data, _b._data, _c._data), dtype=ddtype)
self.data = (mrec, nrec, ddtype)
def setup(self):
# Generic setup
_a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
_b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
_c = ma.array(list(map(asbytes, ['one', 'two', 'three'])),
mask=[0, 0, 1], dtype='|S8')
ddtype = [('a', int), ('b', float), ('c', '|S8')]
mrec = fromarrays([_a, _b, _c], dtype=ddtype,
fill_value=(asbytes('99999'), asbytes('99999.'),
asbytes('N/A')))
nrec = recfromarrays((_a._data, _b._data, _c._data), dtype=ddtype)
self.data = (mrec, nrec, ddtype)
def fromarrays(
arraylist,
dtype=None,
shape=None,
formats=None,
names=None,
titles=None,
aligned=False,
byteorder=None,
fill_value=None,
):
"""Creates a mrecarray from a (flat) list of masked arrays.
Parameters
----------
arraylist : sequence
A list of (masked) arrays. Each element of the sequence is first converted
to a masked array if needed. If a 2D array is passed as argument, it is
processed line by line
dtype : {None, dtype}, optional
Data type descriptor.
shape : {None, integer}, optional
Number of records. If None, shape is defined from the shape of the
first array in the list.
formats : {None, sequence}, optional
Sequence of formats for each individual field. If None, the formats will
be autodetected by inspecting the fields and selecting the highest dtype
possible.
names : {None, sequence}, optional
Sequence of the names of each field.
fill_value : {None, sequence}, optional
Sequence of data to be used as filling values.
Notes
-----
Lists of tuples should be preferred over lists of lists for faster processing.
"""
datalist = [getdata(x) for x in arraylist]
masklist = [np.atleast_1d(getmaskarray(x)) for x in arraylist]
_array = recfromarrays(
datalist,
dtype=dtype,
shape=shape,
formats=formats,
names=names,
titles=titles,
aligned=aligned,
byteorder=byteorder,
).view(mrecarray)
_array._mask.flat = zip(*masklist)
if fill_value is not None:
_array.fill_value = fill_value
return _array
def setup(self):
# Generic setup
_a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
_b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
_c = ma.array(list(map(asbytes, ['one', 'two', 'three'])),
mask=[0, 0, 1], dtype='|S8')
ddtype = [('a', int), ('b', float), ('c', '|S8')]
mrec = fromarrays([_a, _b, _c], dtype=ddtype,
fill_value=(asbytes('99999'), asbytes('99999.'),
asbytes('N/A')))
nrec = recfromarrays((_a._data, _b._data, _c._data), dtype=ddtype)
self.data = (mrec, nrec, ddtype)
class TestMRecordsImport(object):
_a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
_b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
_c = ma.array([b'one', b'two', b'three'], mask=[0, 0, 1], dtype='|S8')
ddtype = [('a', int), ('b', float), ('c', '|S8')]
mrec = fromarrays([_a, _b, _c],
dtype=ddtype,
fill_value=(b'99999', b'99999.', b'N/A'))
nrec = recfromarrays((_a._data, _b._data, _c._data), dtype=ddtype)
data = (mrec, nrec, ddtype)
def test_fromarrays(self):
_a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
_b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
_c = ma.array(['one', 'two', 'three'], mask=[0, 0, 1], dtype='|S8')
(mrec, nrec, _) = self.data
for (f, l) in zip(('a', 'b', 'c'), (_a, _b, _c)):
assert_equal(getattr(mrec, f)._mask, l._mask)
# One record only
_x = ma.array(
[1, 1.1, 'one'],
mask=[1, 0, 0],
)
assert_equal_records(fromarrays(_x, dtype=mrec.dtype), mrec[0])
def test_fromrecords(self):
# Test construction from records.
(mrec, nrec, ddtype) = self.data
#......
palist = [(1, 'abc', 3.7000002861022949, 0),
(2, 'xy', 6.6999998092651367, 1),
(0, ' ', 0.40000000596046448, 0)]
pa = recfromrecords(palist, names='c1, c2, c3, c4')
mpa = fromrecords(palist, names='c1, c2, c3, c4')
assert_equal_records(pa, mpa)
#.....
_mrec = fromrecords(nrec)
assert_equal(_mrec.dtype, mrec.dtype)
for field in _mrec.dtype.names:
assert_equal(getattr(_mrec, field), getattr(mrec._data, field))
_mrec = fromrecords(nrec.tolist(), names='c1,c2,c3')
assert_equal(_mrec.dtype, [('c1', int), ('c2', float), ('c3', '|S5')])
for (f, n) in zip(('c1', 'c2', 'c3'), ('a', 'b', 'c')):
assert_equal(getattr(_mrec, f), getattr(mrec._data, n))
_mrec = fromrecords(mrec)
assert_equal(_mrec.dtype, mrec.dtype)
assert_equal_records(_mrec._data, mrec.filled())
assert_equal_records(_mrec._mask, mrec._mask)
def test_fromrecords_wmask(self):
# Tests construction from records w/ mask.
(mrec, nrec, ddtype) = self.data
_mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=[
0,
1,
0,
])
assert_equal_records(_mrec._data, mrec._data)
assert_equal(_mrec._mask.tolist(), [(0, 0, 0), (1, 1, 1), (0, 0, 0)])
_mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=True)
assert_equal_records(_mrec._data, mrec._data)
assert_equal(_mrec._mask.tolist(), [(1, 1, 1), (1, 1, 1), (1, 1, 1)])
_mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=mrec._mask)
assert_equal_records(_mrec._data, mrec._data)
assert_equal(_mrec._mask.tolist(), mrec._mask.tolist())
_mrec = fromrecords(nrec.tolist(),
dtype=ddtype,
mask=mrec._mask.tolist())
assert_equal_records(_mrec._data, mrec._data)
assert_equal(_mrec._mask.tolist(), mrec._mask.tolist())
def test_fromtextfile(self):
# Tests reading from a text file.
fcontent = ("""#
'One (S)','Two (I)','Three (F)','Four (M)','Five (-)','Six (C)'
'strings',1,1.0,'mixed column',,1
'with embedded "double quotes"',2,2.0,1.0,,1
'strings',3,3.0E5,3,,1
'strings',4,-1e-10,,,1
""")
with temppath() as path:
with open(path, 'w') as f:
f.write(fcontent)
mrectxt = fromtextfile(path, delimitor=',', varnames='ABCDEFG')
assert_(isinstance(mrectxt, MaskedRecords))
assert_equal(mrectxt.F, [1, 1, 1, 1])
assert_equal(mrectxt.E._mask, [1, 1, 1, 1])
assert_equal(mrectxt.C, [1, 2, 3.e+5, -1e-10])
def test_addfield(self):
# Tests addfield
(mrec, nrec, ddtype) = self.data
(d, m) = ([100, 200, 300], [1, 0, 0])
mrec = addfield(mrec, ma.array(d, mask=m))
assert_equal(mrec.f3, d)
assert_equal(mrec.f3._mask, m)
class TestMRecordsImport(object):
_a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
_b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
_c = ma.array([b"one", b"two", b"three"], mask=[0, 0, 1], dtype="|S8")
ddtype = [("a", int), ("b", float), ("c", "|S8")]
mrec = fromarrays(
[_a, _b, _c], dtype=ddtype, fill_value=(b"99999", b"99999.", b"N/A")
)
nrec = recfromarrays((_a._data, _b._data, _c._data), dtype=ddtype)
data = (mrec, nrec, ddtype)
def test_fromarrays(self):
_a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
_b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
_c = ma.array(["one", "two", "three"], mask=[0, 0, 1], dtype="|S8")
(mrec, nrec, _) = self.data
for (f, l) in zip(("a", "b", "c"), (_a, _b, _c)):
assert_equal(getattr(mrec, f)._mask, l._mask)
# One record only
_x = ma.array([1, 1.1, "one"], mask=[1, 0, 0],)
assert_equal_records(fromarrays(_x, dtype=mrec.dtype), mrec[0])
def test_fromrecords(self):
# Test construction from records.
(mrec, nrec, ddtype) = self.data
# ......
palist = [
(1, "abc", 3.7000002861022949, 0),
(2, "xy", 6.6999998092651367, 1),
(0, " ", 0.40000000596046448, 0),
]
pa = recfromrecords(palist, names="c1, c2, c3, c4")
mpa = fromrecords(palist, names="c1, c2, c3, c4")
assert_equal_records(pa, mpa)
# .....
_mrec = fromrecords(nrec)
assert_equal(_mrec.dtype, mrec.dtype)
for field in _mrec.dtype.names:
assert_equal(getattr(_mrec, field), getattr(mrec._data, field))
_mrec = fromrecords(nrec.tolist(), names="c1,c2,c3")
assert_equal(_mrec.dtype, [("c1", int), ("c2", float), ("c3", "|S5")])
for (f, n) in zip(("c1", "c2", "c3"), ("a", "b", "c")):
assert_equal(getattr(_mrec, f), getattr(mrec._data, n))
_mrec = fromrecords(mrec)
assert_equal(_mrec.dtype, mrec.dtype)
assert_equal_records(_mrec._data, mrec.filled())
assert_equal_records(_mrec._mask, mrec._mask)
def test_fromrecords_wmask(self):
# Tests construction from records w/ mask.
(mrec, nrec, ddtype) = self.data
_mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=[0, 1, 0,])
assert_equal_records(_mrec._data, mrec._data)
assert_equal(_mrec._mask.tolist(), [(0, 0, 0), (1, 1, 1), (0, 0, 0)])
_mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=True)
assert_equal_records(_mrec._data, mrec._data)
assert_equal(_mrec._mask.tolist(), [(1, 1, 1), (1, 1, 1), (1, 1, 1)])
_mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=mrec._mask)
assert_equal_records(_mrec._data, mrec._data)
assert_equal(_mrec._mask.tolist(), mrec._mask.tolist())
_mrec = fromrecords(nrec.tolist(), dtype=ddtype, mask=mrec._mask.tolist())
assert_equal_records(_mrec._data, mrec._data)
assert_equal(_mrec._mask.tolist(), mrec._mask.tolist())
def test_fromtextfile(self):
# Tests reading from a text file.
fcontent = """#
'One (S)','Two (I)','Three (F)','Four (M)','Five (-)','Six (C)'
'strings',1,1.0,'mixed column',,1
'with embedded "double quotes"',2,2.0,1.0,,1
'strings',3,3.0E5,3,,1
'strings',4,-1e-10,,,1
"""
with temppath() as path:
with open(path, "w") as f:
f.write(fcontent)
mrectxt = fromtextfile(path, delimitor=",", varnames="ABCDEFG")
assert_(isinstance(mrectxt, MaskedRecords))
assert_equal(mrectxt.F, [1, 1, 1, 1])
assert_equal(mrectxt.E._mask, [1, 1, 1, 1])
assert_equal(mrectxt.C, [1, 2, 3.0e5, -1e-10])
def test_addfield(self):
# Tests addfield
(mrec, nrec, ddtype) = self.data
(d, m) = ([100, 200, 300], [1, 0, 0])
mrec = addfield(mrec, ma.array(d, mask=m))
assert_equal(mrec.f3, d)
assert_equal(mrec.f3._mask, m)