def test_format_message(): r"""Test formatting message from a list or arguments and back.""" fmt = b'%5s\t%ld\t%lf\t%g%+gj\n' dtype = serialize.cformat2nptype(fmt) x_arr = np.ones(1, dtype) x_tup = [x_arr[n][0] for n in x_arr.dtype.names] flist = [fmt, "%ld"] alist = [tuple(x_tup), 0] for a, f in zip(alist, flist): msg = serialize.format_message(a, f) b = serialize.process_message(msg, f) if not isinstance(a, tuple): assert (b == (a, )) else: assert (b == a) # Formats with mixed types assert (serialize.format_message(b'hello', '%s') == 'hello') assert (serialize.format_message('hello', b'%s') == b'hello') # Errors with pytest.raises(RuntimeError): serialize.format_message((0, ), "%d %d") with pytest.raises(TypeError): serialize.process_message(0, "%d") with pytest.raises(ValueError): serialize.process_message(b'hello', "%d")
def test_format_header(): r"""Test formatting header.""" kws_all = dict(field_names=['name', 'number', 'value', 'complex'], field_units=['n/a', 'g', 'cm', 'n/a']) res_all = dict(names="# name\tnumber\tvalue\tcomplex\n", units="# n/a\tg\tcm\tn/a\n") if platform._is_win: # pragma: windows kws_all['format_str'] = "%5s\t%l64d\t%g\t%g%+gj\n" res_all['format'] = "# " + kws_all['format_str'] else: kws_all['format_str'] = "%5s\t%ld\t%g\t%g%+gj\n" res_all['format'] = "# " + kws_all['format_str'] kws_all['dtype'] = serialize.cformat2nptype(kws_all['format_str'], names=kws_all['field_names']) for x in [kws_all, res_all]: for k, v in x.items(): if isinstance(v, str): x[k] = backwards.as_bytes(v) elif isinstance(v, list): x[k] = [backwards.as_bytes(iv) for iv in v] test_list = [(['format_str', 'field_names', 'field_units'], ['names', 'units', 'format']), (['field_names', 'field_units', 'dtype'], ['names', 'units', 'format']), (['field_units', 'dtype'], ['names', 'units', 'format']), (['field_names'], ['names']), (['field_units'], ['units'])] for kws_keys, res_keys in test_list: kws = {k: kws_all[k] for k in kws_keys} res = b''.join([res_all[k] for k in res_keys]) assert_equal(serialize.format_header(**kws), res) assert_raises(ValueError, serialize.format_header)
def test_cformat2nptype_structured(): r"""Test conversion from C format string to numpy dtype for structured data types.""" if platform._is_win: # pragma: debug fmts = ["%5s", "%l64d", "%lf", "%g%+gj"] else: fmts = ["%5s", "%ld", "%lf", "%g%+gj"] names0 = ['f0', 'f1', 'f2', 'f3'] names1 = ["name", "number", "value", "complex"] dtypes = ['S5', 'i8', 'f8', 'c16'] dtype0 = np.dtype([(n, f) for n, f in zip(names0, dtypes)]) dtype1 = np.dtype([(n, f) for n, f in zip(names1, dtypes)]) alist = ["\t".join(fmts) + "\n", ''.join(fmts), fmts] for a in alist: b0 = serialize.cformat2nptype(a) assert_equal(b0, dtype0) b1 = serialize.cformat2nptype(a, names=names1) assert_equal(b1, dtype1)
def test_array_to_table(): r"""Test conversion of arrays to ASCII table and back.""" flist = ['# %5s\t%ld\t%lf\t%g%+gj\n'] for use_astropy in [False, True]: for f in flist: dtype = serialize.cformat2nptype(f) arr0 = np.ones(5, dtype) arr0['f0'][0] = b'hello' tab = serialize.array_to_table(arr0, f, use_astropy=use_astropy) arr1 = serialize.table_to_array(tab, f, use_astropy=use_astropy) np.testing.assert_array_equal(arr1, arr0)
def cformat2nptype(self, *args, **kwargs): r"""Method to convert c format string to numpy data type. Args: *args: Arguments are passed to serialize.cformat2nptype. **kwargs: Keyword arguments are passed to serialize.cformat2nptype. Returns: np.dtype: Corresponding numpy data type. """ return serialize.cformat2nptype(*args, **kwargs)
def test_format_message(): r"""Test formatting message from a list or arguments and back.""" fmt = b'%5s\t%ld\t%lf\t%g%+gj\n' dtype = serialize.cformat2nptype(fmt) x_arr = np.ones(1, dtype) x_tup = [x_arr[n][0] for n in x_arr.dtype.names] # x_tup[0] = backwards.as_str(x_tup[0]) flist = [fmt, "%ld"] alist = [tuple(x_tup), 0] for a, f in zip(alist, flist): msg = serialize.format_message(a, f) b = serialize.process_message(msg, f) if not isinstance(a, tuple): assert_equal(b, (a, )) else: assert_equal(b, a) # Errors assert_raises(RuntimeError, serialize.format_message, (0, ), "%d %d") assert_raises(TypeError, serialize.process_message, 0, "%d") assert_raises(ValueError, serialize.process_message, b'hello', "%d")
def test_cformat2nptype(): r"""Test conversion from C format string to numpy dtype.""" for a, b in map_cformat2nptype: if isinstance(a, str): a = [a] for _ia in a: if _ia.startswith(backwards.as_str(serialize._fmt_char)): ia = backwards.as_bytes(_ia) else: ia = serialize._fmt_char + backwards.as_bytes(_ia) assert_equal(serialize.cformat2nptype(ia), np.dtype(b)) # .str) # assert_equal(serialize.cformat2nptype(ia), np.dtype(b).str) assert_raises(TypeError, serialize.cformat2nptype, 0) assert_raises(ValueError, serialize.cformat2nptype, b's') assert_raises(ValueError, serialize.cformat2nptype, b'%') assert_raises(ValueError, serialize.cformat2nptype, '%d\t%f', names=['one']) for a in unsupported_nptype: assert_raises(ValueError, serialize.cformat2nptype, backwards.as_bytes('%' + a))
def update_typedef_from_oldstyle(self, typedef): r"""Update a given typedef using an old, table-style serialization spec. Existing typedef values are not overwritten and warnings are raised if the provided serialization spec is not compatible with the type definition. Args: typedef (dict): Type definition to update. Returns: dict: Updated typedef. """ for k in self._oldstyle_kws: used = [] updated = [] v = self.extra_kwargs.get(k, getattr(self, k, None)) if v is None: continue # Check status if ((k != 'format_str') and (typedef.get('type', None) != 'array')): continue # Key specific changes to type if k == 'format_str': v = backwards.as_str(v) fmts = extract_formats(v) if 'type' in typedef: if (typedef.get('type', None) == 'array'): assert (len(typedef.get('items', [])) == len(fmts)) # if len(typedef.get('items', [])) != len(fmts): # warnings.warn(("Number of items in typedef (%d) doesn't" # + "match the number of formats (%d).") # % (len(typedef.get('items', [])), len(fmts))) continue as_array = self.extra_kwargs.get( 'as_array', getattr(self, 'as_array', False)) typedef.update(type='array', items=[]) for i, fmt in enumerate(fmts): nptype = cformat2nptype(fmt) itype = OneDArrayMetaschemaType.encode_type( np.ones(1, nptype)) itype = OneDArrayMetaschemaType.extract_typedef(itype) if (fmt == '%s') and ('precision' in itype): del itype['precision'] if as_array: itype['type'] = '1darray' else: itype['type'] = itype.pop('subtype') if (((itype['type'] in _flexible_types) and ('precision' in itype))): del itype['precision'] typedef['items'].append(itype) used.append('as_array') updated.append('format_str') elif k == 'as_array': # Can only be used in conjunction with format_str pass elif k in ['field_names', 'field_units']: v = [backwards.as_str(x) for x in v] if k == 'field_names': tk = 'title' else: tk = 'units' if isinstance(typedef['items'], dict): typedef['items'] = [ copy.deepcopy(typedef['items']) for _ in range(len(v)) ] assert (len(v) == len(typedef.get('items', []))) # if len(v) != len(typedef.get('items', [])): # warnings.warn('%d %ss provided, but only %d items in typedef.' # % (len(v), k, len(typedef.get('items', [])))) # continue all_updated = True for iv, itype in zip(v, typedef.get('items', [])): if tk in itype: all_updated = False itype.setdefault(tk, iv) if all_updated: used.append(k) updated.append( k) # Won't change anything unless its an attribute else: # pragma: debug raise ValueError( "Unrecognized table-style specification keyword: '%s'." % k) for rk in used: if rk in self.extra_kwargs: del self.extra_kwargs[rk] for rk in updated: if rk in self.extra_kwargs: self.extra_kwargs[rk] = v elif hasattr(self, rk): setattr(self, rk, v) return typedef
def test_cformat2nptype(): r"""Test conversion from C format string to numpy dtype.""" for a, b in map_cformat2nptype: if isinstance(a, str): a = [a] for _ia in a: if _ia.startswith(constants.FMT_CHAR_STR): ia = _ia.encode("utf-8") else: ia = constants.FMT_CHAR + _ia.encode("utf-8") assert (serialize.cformat2nptype(ia) == np.dtype(b)) # .str) # assert(serialize.cformat2nptype(ia) == np.dtype(b).str) with pytest.raises(TypeError): serialize.cformat2nptype(0) with pytest.raises(ValueError): serialize.cformat2nptype(b's') with pytest.raises(ValueError): serialize.cformat2nptype(b'%') with pytest.raises(ValueError): serialize.cformat2nptype('%d\t%f', names=['one']) for a in unsupported_nptype: with pytest.raises(ValueError): serialize.cformat2nptype(('%' + a).encode("utf-8"))