def func_deserialize(self, msg):
r"""Deserialize a message.
Args:
msg (str, bytes): Message to be deserialized.
Returns:
obj: Deserialized Python object.
"""
fd = backwards.BytesIO(msg)
names = None
dtype = None
if self.initialized:
dtype = self.numpy_dtype
out = pandas.read_csv(fd,
sep=backwards.as_str(self.delimiter),
names=names,
dtype=dtype,
encoding='utf8')
fd.close()
if not backwards.PY2:
# For Python 3 and higher, make sure strings are bytes
for c, d in zip(out.columns, out.dtypes):
if (d == object) and isinstance(out[c][0],
backwards.unicode_type):
out[c] = out[c].apply(lambda s: s.encode('utf-8'))
# On windows, long != longlong and longlong requires special cformat
# For now, long will be used to preserve the use of %ld to match long
if platform._is_win: # pragma: windows
if np.dtype('longlong').itemsize == 8:
new_dtypes = dict()
for c, d in zip(out.columns, out.dtypes):
if d == np.dtype('longlong'):
new_dtypes[c] = np.int32
else:
new_dtypes[c] = d
out = out.astype(new_dtypes, copy=False)
# Reorder if necessary
out = self.apply_field_names(out, self.get_field_names())
if self.field_names is None:
self.field_names = out.columns.tolist()
if not self.initialized:
typedef = {'type': 'array', 'items': []}
np_out = serialize.pandas2numpy(out)
for n in self.get_field_names():
typedef['items'].append(
OneDArrayMetaschemaType.encode_type(np_out[n], title=n))
self.update_serializer(extract=True, **typedef)
return out
def func_deserialize(self, msg):
r"""Deserialize a message.
Args:
msg (str, bytes): Message to be deserialized.
Returns:
obj: Deserialized Python object.
"""
fd = backwards.BytesIO(msg)
out = loadmat(fd, matlab_compatible=True)
mat_keys = ['__header__', '__globals__', '__version__']
for k in mat_keys:
del out[k]
fd.close()
return out
def array_to_table(arrs, fmt_str, use_astropy=False):
r"""Serialize an array as an ASCII table.
Args:
arrs (np.ndarray, list, tuple): Structured array or list/tuple of
arrays that contain table information.
fmt_str (str, bytes): Format string that should be used to structure
the ASCII array.
use_astropy (bool, optional): If True, astropy will be used to format
the table if it is installed. Defaults to False.
Returns:
bytes: ASCII table.
"""
if not _use_astropy:
use_astropy = False
dtype = cformat2nptype(fmt_str)
info = format2table(fmt_str)
comment = info.get('comment', None)
if comment is not None:
fmt_str = fmt_str.split(comment, 1)[-1]
arr1 = consolidate_array(arrs, dtype=dtype)
if use_astropy:
fd = backwards.StringIO()
table = apy_Table(arr1)
delimiter = info['delimiter']
delimiter = backwards.as_str(delimiter)
apy_ascii.write(table, fd, delimiter=delimiter,
format='no_header')
out = backwards.as_bytes(fd.getvalue())
else:
fd = backwards.BytesIO()
for ele in arr1:
line = format_message(ele.tolist(), backwards.as_bytes(fmt_str))
fd.write(line)
# fmt = fmt_str.split(info['newline'])[0]
# np.savetxt(fd, arr1,
# fmt=fmt, delimiter=info['delimiter'],
# newline=info['newline'], header='')
out = fd.getvalue()
fd.close()
return out
def get_first_frame(cls, msg):
r"""Extract one frame from the provided message that may contain one
or more frames.
Args:
msg (bytes): Message containing one or more frames.
Returns:
bytes: Portion of message containing the first frame. If no frames
are found, an empty string will be returned.
"""
fd = backwards.BytesIO(msg)
try:
backwards.pickle.load(fd)
used = fd.tell()
except BaseException:
used = 0
fd.close()
return msg[:used]
def func_serialize(self, args):
r"""Serialize a message.
Args:
args (obj): Python object to be serialized.
Returns:
bytes, str: Serialized message.
Raises:
TypeError: If args is not a dictionary.
"""
if not isinstance(args, dict):
raise TypeError('Object (type %s) is not a dictionary' %
type(args))
fd = backwards.BytesIO()
savemat(fd, args)
out = fd.getvalue()
fd.close()
return out
def _recv(self, timeout=0):
r"""Reads message from a file.
Args:
timeout (float, optional): Time in seconds to wait for a message.
Defaults to self.recv_timeout. Unused.
Returns:
tuple (bool, str): Success or failure of reading from the file and
the read messages as bytes.
"""
prev_pos = self.fd.tell()
flag, msg = super(PickleFileComm, self)._recv(timeout=timeout)
# Rewind file if message contains more than one pickle
if msg != self.eof_msg:
fd = backwards.BytesIO(msg)
backwards.pickle.load(fd)
used = fd.tell()
self.fd.seek(prev_pos + used)
msg = msg[:used]
fd.close()
return flag, msg
def table_to_array(msg,
fmt_str=None,
use_astropy=False,
names=None,
delimiter=None,
comment=None,
encoding='utf-8'):
r"""Extract information from an ASCII table as an array.
Args:
msg (bytes): ASCII table as bytes string.
fmt_str (bytes): Format string that should be used to parse the table.
If not provided, this will attempt to determine the types of columns
based on their contents.
use_astropy (bool, optional): If True, astropy will be used to parse
the table if it is installed. Defaults to False.
names (list, optional): Field names that should be used for the
structured data type of the output array. If not provided, names
are generated based on the order of the fields in the table.
delimiter (str, optional): String used to separate columns. Defaults to
None and is not used. This is only used if fmt_str is not provided.
comment (str, optional): String used to denote comments. Defaults to
None and is not used. This is only used if fmt_str is not provided.
encoding (str, optional): Encoding that should be used in Python 3 or
higher to extract information from the message. Defaults to 'utf-8'.
Returns:
np.ndarray: Table contents as an array.
"""
if not _use_astropy:
use_astropy = False
if fmt_str is None:
dtype = None
info = dict(delimiter=delimiter, comment=comment)
else:
dtype = cformat2nptype(fmt_str, names=names)
info = format2table(fmt_str)
names = dtype.names
fd = backwards.BytesIO(msg)
if names is not None:
names = [backwards.as_str(n) for n in names]
np_kws = dict()
if info.get('delimiter', None) is not None:
np_kws['delimiter'] = info['delimiter']
if info.get('comment', None) is not None:
np_kws['comments'] = info['comment']
for k, v in np_kws.items():
np_kws[k] = backwards.as_str(v)
if use_astropy:
# fd = backwards.StringIO(backwards.as_str(msg))
if 'comments' in np_kws:
np_kws['comment'] = np_kws.pop('comments')
tab = apy_ascii.read(fd,
names=names,
guess=True,
encoding=encoding,
format='no_header',
**np_kws)
arr = tab.as_array()
typs = [arr.dtype[i].str for i in range(len(arr.dtype))]
cols = [c for c in tab.columns]
# Convert type bytes if python 3
if not backwards.PY2: # pragma: Python 3
new_typs = copy.copy(typs)
convert = []
for i in range(len(arr.dtype)):
if np.issubdtype(arr.dtype[i], np.dtype('U')):
new_typs[i] = 'S' + typs[i].split('U')[-1]
convert.append(i)
if convert:
old_arr = arr
new_dtyp = np.dtype([(c, t) for c, t in zip(cols, new_typs)])
new_arr = np.zeros(arr.shape, new_dtyp)
for i in range(len(arr.dtype)):
if i in convert:
x = np.char.encode(old_arr[cols[i]], encoding='utf-8')
new_arr[cols[i]] = x
else:
new_arr[cols[i]] = old_arr[cols[i]]
arr = new_arr
typs = new_typs
# Convert complex type
for i in range(len(arr.dtype)):
if np.issubdtype(arr.dtype[i], np.dtype('S')):
new_typs = copy.copy(typs)
new_typs[i] = 'complex'
new_dtyp = np.dtype([(c, t) for c, t in zip(cols, new_typs)])
try:
arr = arr.astype(new_dtyp)
except ValueError:
pass
if dtype is not None:
arr = arr.astype(dtype)
else:
np_ver = tuple([float(x) for x in (np.__version__).split('.')])
if (np_ver >= (1.0, 14.0, 0.0)):
arr = np.genfromtxt(fd,
encoding='bytes',
autostrip=True,
dtype=None,
names=names,
**np_kws)
else:
arr = np.genfromtxt(fd,
autostrip=True,
dtype=None,
names=names,
**np_kws)
if dtype is not None:
arr = arr.astype(dtype)
fd.close()
return arr