def func_deserialize(self, msg):
r"""Deserialize a message.
Args:
msg (str, bytes): Message to be deserialized.
Returns:
obj: Deserialized Python object.
"""
fd = backwards.BytesIO(msg)
names = None
dtype = None
if self.initialized:
dtype = self.numpy_dtype
out = pandas.read_csv(fd,
sep=backwards.as_str(self.delimiter),
names=names,
dtype=dtype,
encoding='utf8')
fd.close()
if not backwards.PY2:
# For Python 3 and higher, make sure strings are bytes
for c, d in zip(out.columns, out.dtypes):
if (d == object) and isinstance(out[c][0],
backwards.unicode_type):
out[c] = out[c].apply(lambda s: s.encode('utf-8'))
# On windows, long != longlong and longlong requires special cformat
# For now, long will be used to preserve the use of %ld to match long
if platform._is_win: # pragma: windows
if np.dtype('longlong').itemsize == 8:
new_dtypes = dict()
for c, d in zip(out.columns, out.dtypes):
if d == np.dtype('longlong'):
new_dtypes[c] = np.int32
else:
new_dtypes[c] = d
out = out.astype(new_dtypes, copy=False)
# Reorder if necessary
out = self.apply_field_names(out, self.get_field_names())
if self.field_names is None:
self.field_names = out.columns.tolist()
if not self.initialized:
typedef = {'type': 'array', 'items': []}
np_out = serialize.pandas2numpy(out)
for n in self.get_field_names():
typedef['items'].append(
OneDArrayMetaschemaType.encode_type(np_out[n], title=n))
self.update_serializer(extract=True, **typedef)
return out
def update_typedef_from_oldstyle(self, typedef):
r"""Update a given typedef using an old, table-style serialization spec.
Existing typedef values are not overwritten and warnings are raised if the
provided serialization spec is not compatible with the type definition.
Args:
typedef (dict): Type definition to update.
Returns:
dict: Updated typedef.
"""
for k in self._oldstyle_kws:
used = []
updated = []
v = self.extra_kwargs.get(k, getattr(self, k, None))
if v is None:
continue
# Check status
if ((k != 'format_str')
and (typedef.get('type', None) != 'array')):
continue
# Key specific changes to type
if k == 'format_str':
v = tools.bytes2str(v)
fmts = serialize.extract_formats(v)
if 'type' in typedef:
if (typedef.get('type', None) == 'array'):
assert (len(typedef.get('items', [])) == len(fmts))
# This continue is covered, but the optimization
# causes it to be skipped at runtime
# https://bitbucket.org/ned/coveragepy/issues/198/
# continue-marked-as-not-covered
continue # pragma: no cover
as_array = self.extra_kwargs.get(
'as_array', getattr(self, 'as_array', False))
typedef.update(type='array', items=[])
for i, fmt in enumerate(fmts):
nptype = self.cformat2nptype(fmt)
itype = OneDArrayMetaschemaType.encode_type(
np.ones(1, nptype))
itype = OneDArrayMetaschemaType.extract_typedef(itype)
if (fmt == '%s') and ('precision' in itype):
del itype['precision']
if as_array:
itype['type'] = '1darray'
else:
itype['type'] = itype.pop('subtype')
if (((itype['type'] in _flexible_types)
and ('precision' in itype))):
del itype['precision']
typedef['items'].append(itype)
used.append('as_array')
updated.append('format_str')
elif k == 'as_array':
# Can only be used in conjunction with format_str
pass
elif k in ['field_names', 'field_units']:
v = tools.bytes2str(v, recurse=True)
if k == 'field_names':
tk = 'title'
else:
tk = 'units'
if isinstance(typedef['items'], dict):
typedef['items'] = [
copy.deepcopy(typedef['items']) for _ in range(len(v))
]
assert (len(v) == len(typedef.get('items', [])))
# if len(v) != len(typedef.get('items', [])):
# warnings.warn('%d %ss provided, but only %d items in typedef.'
# % (len(v), k, len(typedef.get('items', []))))
# continue
all_updated = True
for iv, itype in zip(v, typedef.get('items', [])):
if tk in itype:
all_updated = False
itype.setdefault(tk, iv)
if all_updated:
used.append(k)
updated.append(
k) # Won't change anything unless its an attribute
else: # pragma: debug
raise ValueError(
"Unrecognized table-style specification keyword: '%s'." %
k)
for rk in used:
if rk in self.extra_kwargs:
del self.extra_kwargs[rk]
for rk in updated:
if rk in self.extra_kwargs:
self.extra_kwargs[rk] = v
elif hasattr(self, rk):
setattr(self, rk, v)
return typedef