def test_pandas2dict():
r"""Test conversion of a Pandas data frame to a dictionary and back."""
assert_raises(TypeError, serialize.dict2pandas, None)
assert_raises(TypeError, serialize.pandas2dict, None)
nele = 5
names = ["complex", "name", "number", "value"]
dtypes = ['c16', 'S5', 'i8', 'f8']
dtype = np.dtype([(n, f) for n, f in zip(names, dtypes)])
arr_mix = np.zeros(nele, dtype)
arr_mix['name'][0] = 'hello'
arr_obj = np.array([list(), 'hello', 5], dtype='O')
test_arrs = [
arr_mix,
np.zeros(nele, 'float'), arr_mix['name'], arr_obj,
np.array([])
]
for ans in test_arrs:
frame = serialize.numpy2pandas(ans)
# Sorted
d = serialize.pandas2dict(frame)
res = serialize.dict2pandas(d)
np.testing.assert_array_equal(res, frame)
# Provided
d = serialize.pandas2dict(frame)
res = serialize.dict2pandas(d, order=ans.dtype.names)
np.testing.assert_array_equal(res, frame)
def test_numpy2pandas():
r"""Test conversion of a numpy array to a pandas data frame and back."""
with pytest.raises(TypeError):
serialize.numpy2pandas(None)
with pytest.raises(TypeError):
serialize.pandas2numpy(None)
nele = 5
names = ["name", "number", "value", "complex"]
dtypes = ['S5', 'i8', 'f8', 'c16']
dtype = np.dtype([(n, f) for n, f in zip(names, dtypes)])
arr_mix = np.zeros(nele, dtype)
arr_mix['name'][0] = 'hello'
arr_obj = np.array([list(), 'hello', 5], dtype='O')
test_arrs = [
arr_mix,
np.zeros(nele, 'float'), arr_mix['name'], arr_obj,
np.array([])
]
for ans in test_arrs:
frame = serialize.numpy2pandas(ans)
res = serialize.pandas2numpy(frame)
np.testing.assert_array_equal(ans, res)
def pandas_send_converter(obj):
r"""Performs conversion from a limited set of objects to a Pandas data frame
for sending to a file via PandasFileComm. Currently supports converting from
structured numpy arrays, lists/tuples of numpy arrays, and dictionaries.
Args:
obj (object): Object to convert.
Returns:
pandas.DataFrame: Converted data frame (or unmodified input if conversion
could not be completed.
"""
if isinstance(obj, (list, tuple)):
obj = serialize.list2pandas(obj)
elif isinstance(obj, np.ndarray):
obj = serialize.numpy2pandas(obj)
elif isinstance(obj, dict):
obj = serialize.dict2pandas(obj)
return obj
def evaluate_transform(self, x, no_copy=False):
r"""Call transform on the provided message.
Args:
x (object): Message object to transform.
no_copy (bool, optional): If True, the transformation occurs in
place. Otherwise a copy is created and transformed. Defaults
to False.
Returns:
object: The transformed message.
"""
if isinstance(x, pandas.DataFrame):
out = x
else:
out = super(PandasTransform,
self).evaluate_transform(x, no_copy=no_copy)
out = numpy2pandas(out)
return out
def get_testing_options(cls):
r"""Get testing options for the transform class.
Returns:
list: Multiple dictionaries of keywords and messages before/after
pairs that will result from the transform created by the provided
keywords.
"""
length = 5
t = {
'type':
'array',
'items': [{
'type': '1darray',
'subtype': 'bytes',
'precision': 40,
'length': length
}, {
'type': '1darray',
'subtype': 'int',
'precision': 64,
'length': length
}, {
'type': '1darray',
'subtype': 'float',
'precision': 64,
'length': length
}, {
'type': '1darray',
'subtype': 'complex',
'precision': 128,
'length': length
}]
}
dtype = np.dtype([(n, f) for n, f in zip(['f0', 'f1', 'f2', 'f3'],
['S5', 'i8', 'f8', 'c16'])])
x = np.zeros(length, dtype=dtype)
x[dtype.names[0]][0] = b'hello'
y = [x[n] for n in dtype.names]
x = numpy2pandas(x)
return [{
'kwargs': {
'original_datatype': t
},
'in/out': [(y, x)],
'in/out_t': [(t, t)]
}, {
'kwargs': {
'original_datatype': t
},
'in/out': [(x, x)],
'in/out_t': [(t, t)]
}, {
'kwargs': {
'original_datatype': t
},
'in/out': [(None, TypeError)]
}, {
'kwargs': {},
'in/out': [([0, 1, 2], AssertionError)]
}]
def get_testing_options(cls):
r"""Get testing options for the transform class.
Returns:
list: Multiple dictionaries of keywords and messages before/after
pairs that will result from the transform created by the provided
keywords.
"""
length = 5
dtype = np.dtype([('f%d' % i, f)
for i, f in enumerate(['S5', 'i8', 'f8', 'c16'])])
dtype_alt = np.dtype([
('alt%d' % i, f) for i, f in enumerate(['S5', 'i8', 'f8', 'c16'])
])
t = {
'type':
'array',
'items': [{
'type': '1darray',
'subtype': 'bytes',
'precision': 40,
'length': length
}, {
'type': '1darray',
'subtype': 'int',
'precision': 64,
'length': length
}, {
'type': '1darray',
'subtype': 'float',
'precision': 64,
'length': length
}, {
'type': '1darray',
'subtype': 'complex',
'precision': 128,
'length': length
}]
}
t_prec = {
'type':
'array',
'items': [{
'type': '1darray',
'subtype': 'bytes',
'length': length
}, {
'type': '1darray',
'subtype': 'int',
'precision': 64,
'length': length
}, {
'type': '1darray',
'subtype': 'float',
'precision': 64,
'length': length
}, {
'type': '1darray',
'subtype': 'complex',
'precision': 128,
'length': length
}]
}
t_arr = {
'type':
'array',
'items': [{
'type': 'array',
'items': [dict(i, type='scalar') for i in t['items']]
} for _ in range(length)]
}
t_arr_err = copy.deepcopy(t_arr)
t_arr_err['items'][0]['items'][0]['type'] = 'null'
t_obj = {
'type':
'array',
'items': [{
'type': 'object',
'properties': {
dtype_alt.names[i]: dict(t['items'][i], type='scalar')
for i in range(len(t['items']))
}
} for _ in range(length)]
}
t_arr_T = {
'type':
'array',
'items': [{
'type':
'array',
'items':
[dict(t['items'][i], type='scalar') for _ in range(length)]
} for i in range(len(t['items']))]
}
t_arr_prec = {
'type':
'array',
'items': [{
'type':
'array',
'items': [dict(i, type='scalar') for i in t_prec['items']]
} for _ in range(length)]
}
t_alt = {
'type':
'array',
'items': [
dict(x, title=dtype_alt.names[i])
for i, x in enumerate(t['items'])
]
}
x = np.zeros(length, dtype=dtype)
x[dtype.names[0]][0] = b'hello'
y = [x[n] for n in dtype.names]
x2 = np.zeros((length, length), dtype=dtype)
# y2 = [x2[n] for n in dtype2.names]
return [{
'kwargs': {
'original_datatype': t
},
'in/out': [(y, x), ([], np.zeros(0, dtype=dtype))],
'in/out_t':
[(t, t), (t_arr_prec, t_prec), (t_arr_T, t), (t_obj, t_alt),
({
'type': 'null'
}, AssertionError), (t['items'][0], t['items'][0]),
({
'type': 'array',
'items':
[dict(v, length=i) for i, v in enumerate(t['items'])]
}, AssertionError), (t_arr_err, AssertionError)]
}, {
'in/out': [(x, x)]
}, {
'in/out': [(x2, x2)]
}, {
'kwargs': {
'field_names': dtype_alt.names
},
'in/out': [(x, x.astype(dtype_alt, copy=True))],
'in/out_t': [(t, t_alt)]
}, {
'kwargs': {
'original_datatype': t
},
'in/out': [(numpy2pandas(x), x)],
'in/out_t': [(t, t)]
}, {
'kwargs': {
'original_datatype': t_arr
},
'in/out': [(x.tolist(), x)],
'in/out_t': [(t_arr, t),
({
'type': 'array',
'items': t_arr['items'][0]
}, t)]
}, {
'in/out': [({n: x[n]
for n in dtype.names}, x)],
'in/out_t': [({
'type': 'object',
'properties': {n: i
for n, i in zip(dtype.names, t['items'])}
}, {
'type':
'array',
'items':
[dict(i, title=n) for n, i in zip(dtype.names, t['items'])]
})]
}, {
'kwargs': {
'original_datatype': t_arr
},
'in/out': [(x.tolist(), x)],
'in/out_t': [(t_arr, t)]
}, {
'kwargs': {
'original_datatype': t
},
'in/out': [(None, TypeError)]
}, {
'kwargs': {},
'in/out': [([0, 1, 2], AssertionError)]
}]