def test_format_message():
r"""Test formatting message from a list or arguments and back."""
fmt = b'%5s\t%ld\t%lf\t%g%+gj\n'
dtype = serialize.cformat2nptype(fmt)
x_arr = np.ones(1, dtype)
x_tup = [x_arr[n][0] for n in x_arr.dtype.names]
flist = [fmt, "%ld"]
alist = [tuple(x_tup), 0]
for a, f in zip(alist, flist):
msg = serialize.format_message(a, f)
b = serialize.process_message(msg, f)
if not isinstance(a, tuple):
assert (b == (a, ))
else:
assert (b == a)
# Formats with mixed types
assert (serialize.format_message(b'hello', '%s') == 'hello')
assert (serialize.format_message('hello', b'%s') == b'hello')
# Errors
with pytest.raises(RuntimeError):
serialize.format_message((0, ), "%d %d")
with pytest.raises(TypeError):
serialize.process_message(0, "%d")
with pytest.raises(ValueError):
serialize.process_message(b'hello', "%d")
def test_format_header():
r"""Test formatting header."""
kws_all = dict(field_names=['name', 'number', 'value', 'complex'],
field_units=['n/a', 'g', 'cm', 'n/a'])
res_all = dict(names="# name\tnumber\tvalue\tcomplex\n",
units="# n/a\tg\tcm\tn/a\n")
if platform._is_win: # pragma: windows
kws_all['format_str'] = "%5s\t%l64d\t%g\t%g%+gj\n"
res_all['format'] = "# " + kws_all['format_str']
else:
kws_all['format_str'] = "%5s\t%ld\t%g\t%g%+gj\n"
res_all['format'] = "# " + kws_all['format_str']
kws_all['dtype'] = serialize.cformat2nptype(kws_all['format_str'],
names=kws_all['field_names'])
for x in [kws_all, res_all]:
for k, v in x.items():
if isinstance(v, str):
x[k] = backwards.as_bytes(v)
elif isinstance(v, list):
x[k] = [backwards.as_bytes(iv) for iv in v]
test_list = [(['format_str', 'field_names',
'field_units'], ['names', 'units', 'format']),
(['field_names', 'field_units',
'dtype'], ['names', 'units', 'format']),
(['field_units', 'dtype'], ['names', 'units', 'format']),
(['field_names'], ['names']), (['field_units'], ['units'])]
for kws_keys, res_keys in test_list:
kws = {k: kws_all[k] for k in kws_keys}
res = b''.join([res_all[k] for k in res_keys])
assert_equal(serialize.format_header(**kws), res)
assert_raises(ValueError, serialize.format_header)
def test_cformat2nptype_structured():
r"""Test conversion from C format string to numpy dtype for structured
data types."""
if platform._is_win: # pragma: debug
fmts = ["%5s", "%l64d", "%lf", "%g%+gj"]
else:
fmts = ["%5s", "%ld", "%lf", "%g%+gj"]
names0 = ['f0', 'f1', 'f2', 'f3']
names1 = ["name", "number", "value", "complex"]
dtypes = ['S5', 'i8', 'f8', 'c16']
dtype0 = np.dtype([(n, f) for n, f in zip(names0, dtypes)])
dtype1 = np.dtype([(n, f) for n, f in zip(names1, dtypes)])
alist = ["\t".join(fmts) + "\n", ''.join(fmts), fmts]
for a in alist:
b0 = serialize.cformat2nptype(a)
assert_equal(b0, dtype0)
b1 = serialize.cformat2nptype(a, names=names1)
assert_equal(b1, dtype1)
def test_array_to_table():
r"""Test conversion of arrays to ASCII table and back."""
flist = ['# %5s\t%ld\t%lf\t%g%+gj\n']
for use_astropy in [False, True]:
for f in flist:
dtype = serialize.cformat2nptype(f)
arr0 = np.ones(5, dtype)
arr0['f0'][0] = b'hello'
tab = serialize.array_to_table(arr0, f, use_astropy=use_astropy)
arr1 = serialize.table_to_array(tab, f, use_astropy=use_astropy)
np.testing.assert_array_equal(arr1, arr0)
def cformat2nptype(self, *args, **kwargs):
r"""Method to convert c format string to numpy data type.
Args:
*args: Arguments are passed to serialize.cformat2nptype.
**kwargs: Keyword arguments are passed to serialize.cformat2nptype.
Returns:
np.dtype: Corresponding numpy data type.
"""
return serialize.cformat2nptype(*args, **kwargs)
def test_format_message():
r"""Test formatting message from a list or arguments and back."""
fmt = b'%5s\t%ld\t%lf\t%g%+gj\n'
dtype = serialize.cformat2nptype(fmt)
x_arr = np.ones(1, dtype)
x_tup = [x_arr[n][0] for n in x_arr.dtype.names]
# x_tup[0] = backwards.as_str(x_tup[0])
flist = [fmt, "%ld"]
alist = [tuple(x_tup), 0]
for a, f in zip(alist, flist):
msg = serialize.format_message(a, f)
b = serialize.process_message(msg, f)
if not isinstance(a, tuple):
assert_equal(b, (a, ))
else:
assert_equal(b, a)
# Errors
assert_raises(RuntimeError, serialize.format_message, (0, ), "%d %d")
assert_raises(TypeError, serialize.process_message, 0, "%d")
assert_raises(ValueError, serialize.process_message, b'hello', "%d")
def test_cformat2nptype():
r"""Test conversion from C format string to numpy dtype."""
for a, b in map_cformat2nptype:
if isinstance(a, str):
a = [a]
for _ia in a:
if _ia.startswith(backwards.as_str(serialize._fmt_char)):
ia = backwards.as_bytes(_ia)
else:
ia = serialize._fmt_char + backwards.as_bytes(_ia)
assert_equal(serialize.cformat2nptype(ia), np.dtype(b)) # .str)
# assert_equal(serialize.cformat2nptype(ia), np.dtype(b).str)
assert_raises(TypeError, serialize.cformat2nptype, 0)
assert_raises(ValueError, serialize.cformat2nptype, b's')
assert_raises(ValueError, serialize.cformat2nptype, b'%')
assert_raises(ValueError,
serialize.cformat2nptype,
'%d\t%f',
names=['one'])
for a in unsupported_nptype:
assert_raises(ValueError, serialize.cformat2nptype,
backwards.as_bytes('%' + a))
def update_typedef_from_oldstyle(self, typedef):
r"""Update a given typedef using an old, table-style serialization spec.
Existing typedef values are not overwritten and warnings are raised if the
provided serialization spec is not compatible with the type definition.
Args:
typedef (dict): Type definition to update.
Returns:
dict: Updated typedef.
"""
for k in self._oldstyle_kws:
used = []
updated = []
v = self.extra_kwargs.get(k, getattr(self, k, None))
if v is None:
continue
# Check status
if ((k != 'format_str')
and (typedef.get('type', None) != 'array')):
continue
# Key specific changes to type
if k == 'format_str':
v = backwards.as_str(v)
fmts = extract_formats(v)
if 'type' in typedef:
if (typedef.get('type', None) == 'array'):
assert (len(typedef.get('items', [])) == len(fmts))
# if len(typedef.get('items', [])) != len(fmts):
# warnings.warn(("Number of items in typedef (%d) doesn't"
# + "match the number of formats (%d).")
# % (len(typedef.get('items', [])), len(fmts)))
continue
as_array = self.extra_kwargs.get(
'as_array', getattr(self, 'as_array', False))
typedef.update(type='array', items=[])
for i, fmt in enumerate(fmts):
nptype = cformat2nptype(fmt)
itype = OneDArrayMetaschemaType.encode_type(
np.ones(1, nptype))
itype = OneDArrayMetaschemaType.extract_typedef(itype)
if (fmt == '%s') and ('precision' in itype):
del itype['precision']
if as_array:
itype['type'] = '1darray'
else:
itype['type'] = itype.pop('subtype')
if (((itype['type'] in _flexible_types)
and ('precision' in itype))):
del itype['precision']
typedef['items'].append(itype)
used.append('as_array')
updated.append('format_str')
elif k == 'as_array':
# Can only be used in conjunction with format_str
pass
elif k in ['field_names', 'field_units']:
v = [backwards.as_str(x) for x in v]
if k == 'field_names':
tk = 'title'
else:
tk = 'units'
if isinstance(typedef['items'], dict):
typedef['items'] = [
copy.deepcopy(typedef['items']) for _ in range(len(v))
]
assert (len(v) == len(typedef.get('items', [])))
# if len(v) != len(typedef.get('items', [])):
# warnings.warn('%d %ss provided, but only %d items in typedef.'
# % (len(v), k, len(typedef.get('items', []))))
# continue
all_updated = True
for iv, itype in zip(v, typedef.get('items', [])):
if tk in itype:
all_updated = False
itype.setdefault(tk, iv)
if all_updated:
used.append(k)
updated.append(
k) # Won't change anything unless its an attribute
else: # pragma: debug
raise ValueError(
"Unrecognized table-style specification keyword: '%s'." %
k)
for rk in used:
if rk in self.extra_kwargs:
del self.extra_kwargs[rk]
for rk in updated:
if rk in self.extra_kwargs:
self.extra_kwargs[rk] = v
elif hasattr(self, rk):
setattr(self, rk, v)
return typedef
def test_cformat2nptype():
r"""Test conversion from C format string to numpy dtype."""
for a, b in map_cformat2nptype:
if isinstance(a, str):
a = [a]
for _ia in a:
if _ia.startswith(constants.FMT_CHAR_STR):
ia = _ia.encode("utf-8")
else:
ia = constants.FMT_CHAR + _ia.encode("utf-8")
assert (serialize.cformat2nptype(ia) == np.dtype(b)) # .str)
# assert(serialize.cformat2nptype(ia) == np.dtype(b).str)
with pytest.raises(TypeError):
serialize.cformat2nptype(0)
with pytest.raises(ValueError):
serialize.cformat2nptype(b's')
with pytest.raises(ValueError):
serialize.cformat2nptype(b'%')
with pytest.raises(ValueError):
serialize.cformat2nptype('%d\t%f', names=['one'])
for a in unsupported_nptype:
with pytest.raises(ValueError):
serialize.cformat2nptype(('%' + a).encode("utf-8"))
