def func_deserialize(self, msg):
r"""Deserialize a message.
Args:
msg (str, bytes): Message to be deserialized.
Returns:
obj: Deserialized Python object.
"""
if len(msg) == 0:
return dict()
fd = backwards.BytesIO(msg)
out = loadmat(fd, squeeze_me=False)
mat_keys = ['__header__', '__globals__', '__version__']
for k in mat_keys:
del out[k]
fd.close()
return out
def array_to_table(arrs, fmt_str, use_astropy=False):
r"""Serialize an array as an ASCII table.
Args:
arrs (np.ndarray, list, tuple): Structured array or list/tuple of
arrays that contain table information.
fmt_str (str, bytes): Format string that should be used to structure
the ASCII array.
use_astropy (bool, optional): If True, astropy will be used to format
the table if it is installed. Defaults to False.
Returns:
bytes: ASCII table.
"""
if not _use_astropy:
use_astropy = False
dtype = cformat2nptype(fmt_str)
info = format2table(fmt_str)
arr1 = consolidate_array(arrs, dtype=dtype)
if use_astropy:
fd = backwards.StringIO()
table = apy_Table(arr1)
delimiter = info['delimiter']
delimiter = backwards.bytes2unicode(delimiter)
apy_ascii.write(table, fd, delimiter=delimiter,
format='no_header')
out = backwards.unicode2bytes(fd.getvalue())
else:
fd = backwards.BytesIO()
for ele in arr1:
line = format_message(ele.tolist(), fmt_str)
fd.write(line)
# fmt = fmt_str.split(info['newline'])[0]
# np.savetxt(fd, arr1,
# fmt=fmt, delimiter=info['delimiter'],
# newline=info['newline'], header='')
out = fd.getvalue()
fd.close()
return out
def _recv(self, timeout=0):
r"""Reads message from a file.
Args:
timeout (float, optional): Time in seconds to wait for a message.
Defaults to self.recv_timeout. Unused.
Returns:
tuple (bool, str): Success or failure of reading from the file and
the read messages as bytes.
"""
prev_pos = self.fd.tell()
flag, msg = super(PickleFileComm, self)._recv(timeout=timeout)
if msg != self.eof_msg:
fd = backwards.BytesIO(msg)
backwards.pickle.load(fd)
used = fd.tell()
self.fd.seek(prev_pos + used)
msg = msg[:used]
fd.close()
return flag, msg
def func_serialize(self, args):
r"""Serialize a message.
Args:
args (obj): Python object to be serialized.
Returns:
bytes, str: Serialized message.
Raises:
TypeError: If args is not a dictionary.
"""
if isinstance(args, backwards.bytes_type) and (len(args) == 0):
return args
if not isinstance(args, dict):
raise TypeError('Object (type %s) is not a dictionary' %
type(args))
fd = backwards.BytesIO()
savemat(fd, args)
out = fd.getvalue()
fd.close()
return out
def func_deserialize(self, msg):
r"""Deserialize a message.
Args:
msg (str, bytes): Message to be deserialized.
Returns:
obj: Deserialized Python object.
"""
if len(msg) == 0:
out = self.empty_msg
else:
fd = backwards.BytesIO(msg)
out = pandas.read_csv(fd, sep=self.delimiter, encoding='utf8')
fd.close()
if not backwards.PY2:
# For Python 3 and higher, make sure strings are bytes
for c, d in zip(out.columns, out.dtypes):
if d == object:
out[c] = out[c].apply(lambda s: s.encode('utf-8'))
# On windows, long != longlong and longlong requires special cformat
# For now, long will be used to preserve the use of %ld to match long
if platform._is_win: # pragma: windows
if np.dtype('longlong').itemsize == 8:
new_dtypes = dict()
for c, d in zip(out.columns, out.dtypes):
if d == np.dtype('longlong'):
new_dtypes[c] = np.int32
else:
new_dtypes[c] = d
out = out.astype(new_dtypes, copy=False)
# for c, d in zip(out.columns, out.dtypes):
# if d == object:
# out[c] = out[c].apply(lambda s: s.strip())
return out
def table_to_array(msg, fmt_str=None, use_astropy=False, names=None,
delimiter=None, comment=None, encoding='utf-8'):
r"""Extract information from an ASCII table as an array.
Args:
msg (bytes): ASCII table as bytes string.
fmt_str (bytes): Format string that should be used to parse the table.
If not provided, this will attempt to determine the types of columns
based on their contents.
use_astropy (bool, optional): If True, astropy will be used to parse
the table if it is installed. Defaults to False.
names (list, optional): Field names that should be used for the
structured data type of the output array. If not provided, names
are generated based on the order of the fields in the table.
delimiter (str, optional): String used to separate columns. Defaults to
None and is not used. This is only used if fmt_str is not provided.
comment (str, optional): String used to denote comments. Defaults to
None and is not used. This is only used if fmt_str is not provided.
encoding (str, optional): Encoding that should be used in Python 3 or
higher to extract information from the message. Defaults to 'utf-8'.
Returns:
np.ndarray: Table contents as an array.
"""
if not _use_astropy:
use_astropy = False
if fmt_str is None:
dtype = None
info = dict(delimiter=delimiter, comment=comment)
else:
dtype = cformat2nptype(fmt_str, names=names)
info = format2table(fmt_str)
names = dtype.names
fd = backwards.BytesIO(msg)
if names is not None:
names = [backwards.bytes2unicode(n) for n in names]
np_kws = dict()
if info.get('delimiter', None) is not None:
np_kws['delimiter'] = info['delimiter']
if info.get('comment', None) is not None:
np_kws['comments'] = info['comment']
for k, v in np_kws.items():
np_kws[k] = backwards.bytes2unicode(v)
if use_astropy:
# fd = backwards.StringIO(backwards.bytes2unicode(msg))
if 'comments' in np_kws:
np_kws['comment'] = np_kws.pop('comments')
tab = apy_ascii.read(fd, names=names, guess=True,
encoding=encoding,
format='no_header', **np_kws)
arr = tab.as_array()
typs = [arr.dtype[i].str for i in range(len(arr.dtype))]
cols = [c for c in tab.columns]
# Convert type bytes if python 3
if not backwards.PY2: # pragma: Python 3
new_typs = copy.copy(typs)
convert = []
for i in range(len(arr.dtype)):
if np.issubdtype(arr.dtype[i], np.dtype('U')):
new_typs[i] = 'S' + typs[i].split('U')[-1]
convert.append(i)
if convert:
old_arr = arr
new_dtyp = np.dtype([(c, t) for c, t in zip(cols, new_typs)])
new_arr = np.zeros(arr.shape, new_dtyp)
for i in range(len(arr.dtype)):
if i in convert:
x = np.char.encode(old_arr[cols[i]], encoding='utf-8')
new_arr[cols[i]] = x
else:
new_arr[cols[i]] = old_arr[cols[i]]
arr = new_arr
typs = new_typs
# Convert complex type
for i in range(len(arr.dtype)):
if np.issubdtype(arr.dtype[i], np.dtype('S')):
new_typs = copy.copy(typs)
new_typs[i] = 'complex'
new_dtyp = np.dtype([(c, t) for c, t in zip(cols, new_typs)])
try:
arr = arr.astype(new_dtyp)
except ValueError:
pass
if dtype is not None:
arr = arr.astype(dtype)
else:
arr = np.genfromtxt(fd, autostrip=True, dtype=None,
names=names, **np_kws)
if dtype is not None:
arr = arr.astype(dtype)
fd.close()
return arr
def table_to_array(msg, fmt_str=None, use_astropy=False, names=None,
delimiter=None, comment=None):
r"""Extract information from an ASCII table as an array.
Args:
msg (bytes): ASCII table as bytes string.
fmt_str (bytes): Format string that should be used to parse the table.
If not provided, this will attempt to determine the types of columns
based on their contents.
use_astropy (bool, optional): If True, astropy will be used to parse
the table if it is installed. Defaults to False.
names (list, optional): Field names that should be used for the
structured data type of the output array. If not provided, names
are generated based on the order of the fields in the table.
delimiter (str, optional): String used to separate columns. Defaults to
None and is not used. This is only used if fmt_str is not provided.
comment (str, optional): String used to denote comments. Defaults to
None and is not used. This is only used if fmt_str is not provided.
Returns:
np.ndarray: Table contents as an array.
"""
if not _use_astropy:
use_astropy = False
if fmt_str is None:
dtype = None
info = dict(delimiter=delimiter, comment=comment)
else:
dtype = cformat2nptype(fmt_str, names=names)
info = format2table(fmt_str)
names = dtype.names
fd = backwards.BytesIO(msg)
np_kws = dict()
if info.get('delimiter', None) is not None:
np_kws['delimiter'] = info['delimiter']
if info.get('comment', None) is not None:
np_kws['comments'] = info['comment']
if use_astropy:
tab = apy_ascii.read(fd, names=names, guess=True,
format='no_header', **np_kws)
arr = tab.as_array()
typs = [arr.dtype[i].str for i in range(len(arr.dtype))]
cols = tab.columns
for i in range(len(arr.dtype)):
if np.issubdtype(arr.dtype[i], np.dtype('S')):
new_typs = copy.copy(typs)
new_typs[i] = 'complex'
new_dtyp = np.dtype([(c, t) for c, t in zip(cols, new_typs)])
try:
arr = arr.astype(new_dtyp)
except ValueError:
pass
if dtype is not None:
arr = arr.astype(dtype)
else:
if names is not None:
names = [backwards.bytes2unicode(n) for n in names]
for k, v in np_kws.items():
np_kws[k] = backwards.bytes2unicode(v)
arr = np.genfromtxt(fd, autostrip=True, dtype=None,
names=names, **np_kws)
if dtype is not None:
arr = arr.astype(dtype)
fd.close()
return arr