def parse_header(self, msg):
r"""Extract header info from a message.
Args:
msg (str): Message to extract header from.
Returns:
dict: Message properties.
"""
if CIS_MSG_HEAD not in msg:
out = dict(body=msg, size=len(msg))
return out
_, header, body = msg.split(CIS_MSG_HEAD)
out = dict(body=body)
for x in header.split(HEAD_KEY_SEP):
k, v = x.split(HEAD_VAL_SEP)
out[backwards.bytes2unicode(k)] = backwards.bytes2unicode(v)
for k in ['size', 'as_array', 'stype']:
if k in out:
out[k] = int(float(out[k]))
for k in ['format_str', 'field_names', 'field_units']:
if k in out:
out[k] = backwards.unicode2bytes(out[k])
if k in ['field_names', 'field_units']:
out[k] = out[k].split(backwards.unicode2bytes(','))
# for k in ['format_str']:
# if k in out:
# out[k] = backwards.unicode2bytes(out[k])
return out
def __init__(self, *args, **kwargs):
super(TestAsciiTableInputDriver_Array, self).__init__(*args, **kwargs)
self.inst_kwargs['as_array'] = True
names = [backwards.bytes2unicode(n) for n in self.field_names]
units = [backwards.bytes2unicode(n) for n in self.field_units]
self.inst_kwargs['column_names'] = names
self.inst_kwargs['column_units'] = units
self.inst_kwargs['use_astropy'] = False
def __init__(self, *args, **kwargs):
super(TestAsciiTableOutputDriver_Array, self).__init__(*args, **kwargs)
self.inst_kwargs['as_array'] = 'True'
names = [backwards.bytes2unicode(n) for n in self.field_names]
units = [backwards.bytes2unicode(n) for n in self.field_units]
self.inst_kwargs['column_names'] = ','.join(names)
self.inst_kwargs['column_units'] = ','.join(units)
self.inst_kwargs['use_astropy'] = 'False'
def read_array(self, names=None):
r"""Read the table in as an array.
Args:
names (list, optional): List of column names to label columns. If
not provided, existing names are used if they exist. Defaults
to None.
Returns:
np.ndarray: Array of table contents.
Raises:
ValueError: If names are provided, but not the same number as
there are columns.
"""
if names is None:
names = self.column_names
if (names is not None) and (len(names) != self.ncols):
raise ValueError(
"The number of names does not match the number of columns")
if hasattr(self, '_arr'):
return self._arr
openned = False
if not self.is_open:
self.open()
openned = True
if self.use_astropy:
tab = apy_ascii.read(self.fd,
names=names,
**getattr(self, 'astropy_kwargs', {}))
arr = tab.as_array()
typs = [arr.dtype[i].str for i in range(len(arr.dtype))]
cols = tab.columns
for i in range(len(arr.dtype)):
if np.issubdtype(arr.dtype[i], np.dtype('S')):
new_typs = copy.copy(typs)
new_typs[i] = 'complex'
new_dtyp = np.dtype([(c, t)
for c, t in zip(cols, new_typs)])
try:
arr = arr.astype(new_dtyp)
except ValueError:
pass
else:
arr = np.genfromtxt(self.fd,
comments=backwards.bytes2unicode(self.comment),
delimiter=backwards.bytes2unicode(self.column),
dtype=self.dtype,
autostrip=True,
names=names)
if openned:
self.close()
return arr
def format_header(self, header_info):
r"""Format header info to form a string that should prepend a message.
Args:
header_info (dict): Properties that should be included in the header.
Returns:
str: Message with header in front.
"""
header = backwards.bytes2unicode(CIS_MSG_HEAD)
header_str = {}
for k, v in header_info.items():
if isinstance(v, list):
header_str[k] = ','.join(
[backwards.bytes2unicode(x) for x in v])
elif isinstance(v, backwards.string_types):
header_str[k] = backwards.bytes2unicode(v)
else:
header_str[k] = str(v)
header += backwards.bytes2unicode(HEAD_KEY_SEP).join([
'%s%s%s' %
(backwards.bytes2unicode(k), backwards.bytes2unicode(HEAD_VAL_SEP),
backwards.bytes2unicode(v)) for k, v in header_str.items()
])
header += backwards.bytes2unicode(CIS_MSG_HEAD)
return backwards.unicode2bytes(header)
def run_cmake(self, target=None):
r"""Run the cmake command on the source.
Args:
target (str, optional): Target to build.
Raises:
RuntimeError: If there is an error in running cmake.
"""
curdir = os.getcwd()
os.chdir(self.sourcedir)
if not os.path.isfile('CMakeLists.txt'):
os.chdir(curdir)
self.cleanup()
raise IOError('No CMakeLists.txt file found in %s.' %
self.sourcedir)
# Configuration
if target != 'clean':
config_cmd = ['cmake'] + self.cmakeargs
config_cmd += ['-H.', self.sourcedir, '-B%s' % self.builddir]
self.debug(' '.join(config_cmd))
comp_process = tools.popen_nobuffer(config_cmd)
output, err = comp_process.communicate()
exit_code = comp_process.returncode
if exit_code != 0:
os.chdir(curdir)
self.cleanup()
self.error(backwards.bytes2unicode(output))
raise RuntimeError("CMake config failed with code %d." %
exit_code)
self.debug('Config output: \n%s' % output)
# Build
build_cmd = ['cmake', '--build', self.builddir, '--clean-first']
if self.target is not None:
build_cmd += ['--target', self.target]
self.info(' '.join(build_cmd))
comp_process = tools.popen_nobuffer(build_cmd)
output, err = comp_process.communicate()
exit_code = comp_process.returncode
if exit_code != 0:
os.chdir(curdir)
self.error(backwards.bytes2unicode(output))
self.cleanup()
raise RuntimeError("CMake build failed with code %d." % exit_code)
self.debug('Build output: \n%s' % output)
self.debug('Make complete')
os.chdir(curdir)
def check_reply_socket_recv(self, msg):
r"""Check incoming message for reply address.
Args:
msg (str): Incoming message to check.
Returns:
str: Messages with reply address removed if present.
"""
if self.direction == 'send':
return msg, None
prefix = backwards.format_bytes(backwards.unicode2bytes(':%s:'),
(_reply_msg, ))
if msg.startswith(prefix):
_, address, new_msg = msg.split(prefix)
if address not in self.reply_socket_recv:
self.reply_socket_recv[address] = self.context.socket(zmq.REQ)
self.reply_socket_recv[address].setsockopt(zmq.LINGER, 0)
self.reply_socket_recv[address].connect(address)
self.register_comm(
'REPLY_RECV_' + backwards.bytes2unicode(address),
self.reply_socket_recv[address])
self._n_reply_recv[address] = 0
self._n_zmq_recv[address] = 0
self.debug("new recv address: %s", address)
else: # pragma: debug
new_msg = msg
raise Exception("No reply socket address attached.")
return new_msg, address
def func_serialize(self, args):
r"""Serialize a message.
Args:
args (obj): Python object to be serialized.
Returns:
bytes, str: Serialized message.
"""
fd = backwards.StringIO()
if backwards.PY2:
args_ = args
else:
# For Python 3 and higher, bytes need to be encoded
args_ = copy.deepcopy(args)
for c in args.columns:
if isinstance(args_[c][0], backwards.bytes_type):
args_[c] = args_[c].apply(lambda s: s.decode('utf-8'))
if self.field_names is not None:
args_.columns = [
backwards.bytes2unicode(n) for n in self.field_names
]
args_.to_csv(fd,
index=False,
sep=self.delimiter,
mode='wb',
encoding='utf8',
header=self.write_header)
out = fd.getvalue()
fd.close()
return backwards.unicode2bytes(out)
def send_dict(self, args_dict, field_order=None, **kwargs):
r"""Send a message with fields specified in the input dictionary.
Args:
args_dict (dict): Dictionary with fields specifying output fields.
field_order (list, optional): List of fields in the order they
should be passed to send. If not provided, the fields from
the serializer are used. If the serializer dosn't have
field names an error will be raised.
**kwargs: Additiona keyword arguments are passed to send.
Returns:
bool: Success/failure of send.
Raises:
RuntimeError: If the field order can not be determined.
"""
if field_order is None:
if self.serializer.field_names is not None:
field_order = [
backwards.bytes2unicode(n) for n in self.serializer.field_names]
elif len(args_dict) <= 1:
field_order = [k for k in args_dict.keys()]
else: # pragma: debug
raise RuntimeError("Could not determine the field order.")
args = (serialize.dict2pandas(args_dict, order=field_order), )
return self.send(*args, **kwargs)
def test_format_bytes():
r"""Test formating of bytes string."""
s0 = "%s, %s"
ans = "one, one"
arg0 = "one"
args = (backwards.unicode2bytes(arg0), backwards.bytes2unicode(arg0))
for cvt in [backwards.unicode2bytes, backwards.bytes2unicode]:
res = backwards.format_bytes(cvt(s0), args)
nt.assert_equal(res, cvt(ans))
def test_send_recv_dict(self):
r"""Test send/recv Pandas data frame as dict."""
msg_send = serialize.pandas2dict(self.msg_short)
names = [backwards.bytes2unicode(n) for n in self.field_names]
flag = self.send_instance.send_dict(msg_send)
assert (flag)
flag, msg_recv = self.recv_instance.recv_dict()
assert (flag)
msg_recv = serialize.dict2pandas(msg_recv, order=names)
self.assert_msg_equal(msg_recv, self.msg_short)
def test_send_recv_dict(self):
r"""Test send/recv numpy array as dict."""
msg_send = serialize.numpy2dict(self.msg_short)
names = [backwards.bytes2unicode(n) for n in self.field_names]
flag = self.send_instance.send_dict(msg_send, field_order=names)
assert (flag)
flag, msg_recv = self.recv_instance.recv_dict()
assert (flag)
msg_recv = serialize.dict2numpy(msg_recv, order=names)
self.assert_msg_equal(msg_recv, self.msg_short)
def test_send_recv_dict(self):
r"""Test send/recv numpy array as dict."""
msg_send = {
backwards.bytes2unicode(k): v
for k, v in zip(self.field_names, self.msg_short)
}
flag = self.send_instance.send_dict(msg_send)
assert (flag)
flag, msg_recv = self.recv_instance.recv_dict()
assert (flag)
nt.assert_equal(msg_recv, msg_send)
def _close_backlog(self, wait=False):
r"""Close the backlog thread and the reply sockets."""
super(ZMQComm, self)._close_backlog(wait=wait)
if self.direction == 'send':
if (self.reply_socket_send is not None):
self.reply_socket_send.close(linger=0) # self.zmq_sleeptime)
self.unregister_comm("REPLY_SEND_" + self.reply_socket_address)
else:
for k, socket in self.reply_socket_recv.items():
socket.close(linger=0)
self.unregister_comm("REPLY_RECV_" +
backwards.bytes2unicode(k))
def set_reply_socket_recv(self, address):
r"""Set the recv reply socket if the address dosn't exist."""
if address not in self.reply_socket_recv:
s = self.context.socket(zmq.REQ)
s.setsockopt(zmq.LINGER, 0)
s.connect(address)
self.register_comm(
'REPLY_RECV_' + backwards.bytes2unicode(address), s)
with self.reply_socket_lock:
self._n_reply_recv[address] = 0
self._n_zmq_recv[address] = 0
self.reply_socket_recv[address] = s
self.debug("new recv address: %s", address)
return address
def cformat2pyscanf(cfmt):
r"""Convert a c format specification string to a version that the
python scanf module can use.
Args:
cfmt (str): C format specification string.
Returns:
str: Version of cfmt that can be parsed by scanf.
Raises:
TypeError: if cfmt is not a bytes/str.
ValueError: If the c format does not begin with '%'.
ValueError: If the c format does not contain type info.
"""
if not isinstance(cfmt, backwards.bytes_type):
raise TypeError("Input must be of type %s." % backwards.bytes_type)
elif not cfmt.startswith(_fmt_char):
raise ValueError("Provided C format string (%s) " % cfmt +
"does not start with '%'")
elif len(cfmt) == 1:
raise ValueError("Provided C format string (%s) " % cfmt +
"does not contain type info")
# Hacky, but necessary to handle concatenation of a single byte
cfmt_str = backwards.bytes2unicode(cfmt)
if cfmt_str[-1] == 'j':
# Handle complex format specifier
out = '%g%+gj'
else:
out = backwards.bytes2unicode(_fmt_char)
out += cfmt_str[-1]
out = out.replace('h', '')
out = out.replace('l', '')
out = out.replace('64', '')
return backwards.unicode2bytes(out)
def _send(self, msg):
r"""Write message to a file.
Args:
msg (bytes, str): Data to write to the file.
Returns:
bool: Success or failure of writing to the file.
"""
if msg != self.eof_msg:
if not self.open_as_binary:
msg = backwards.bytes2unicode(msg)
self.fd.write(msg)
self.fd.flush()
return True
def cformat2pyscanf(cfmt):
r"""Convert a c format specification string to a version that the
python scanf module can use.
Args:
cfmt (str, bytes, list): C format specification string or list of format
strings.
Returns:
str, bytes, list: Version of cfmt or list of cfmts that can be parsed by
scanf.
Raises:
TypeError: If cfmt is not a bytes/str/list.
ValueError: If there are not an format codes in the format string.
"""
if not (isinstance(cfmt, list) or isinstance(cfmt, backwards.string_types)):
raise TypeError("Input must be a string, bytes string, or list, not %s" %
type(cfmt))
if isinstance(cfmt, list):
return [cformat2pyscanf(f) for f in cfmt]
cfmt_out = backwards.bytes2unicode(cfmt)
fmt_list = extract_formats(cfmt_out)
if len(fmt_list) == 0:
raise ValueError("Could not locate any format codes in the "
+ "provided format string (%s)." % cfmt)
for cfmt_str in fmt_list:
# Hacky, but necessary to handle concatenation of a single byte
if cfmt_str[-1] == 's':
out = '%s'
else:
out = cfmt_str
# if cfmt_str[-1] == 'j':
# # Handle complex format specifier
# out = '%g%+gj'
# else:
# out = backwards.bytes2unicode(_fmt_char)
# out += cfmt_str[-1]
# out = out.replace('h', '')
# out = out.replace('l', '')
# out = out.replace('64', '')
cfmt_out = cfmt_out.replace(cfmt_str, out, 1)
if isinstance(cfmt, backwards.bytes_type):
cfmt_out = backwards.unicode2bytes(cfmt_out)
return cfmt_out
def is_unit(ustr):
r"""Determine if a string is a valid unit.
Args:
ustr: String representation to test.
Returns:
bool: True if the string is a valid unit. False otherwise.
"""
ustr = backwards.bytes2unicode(ustr)
if ustr == 'n/a':
return True
try:
_ureg(ustr)
except pint.errors.UndefinedUnitError:
return False
return True
def is_unit(ustr):
r"""Determine if a string is a valid unit.
Args:
ustr (str): String representation to test.
Returns:
bool: True if the string is a valid unit. False otherwise.
"""
ustr = backwards.bytes2unicode(ustr)
if is_null_unit(ustr):
return True
try:
as_unit(ustr)
except ValueError:
return False
return True
def extract_formats(fmt_str):
r"""Locate format codes within a format string.
Args:
fmt_str (str, bytes): Format string.
Returns:
list: List of identified format codes.
"""
fmt_regex = (
"%(?:\\d+\\$)?[+-]?(?:[ 0]|\'.{1})?-?\\d*(?:\\.\\d+)?"
+ "[lhjztL]*(?:64)?[bcdeEufFgGosxXi]"
+ "(?:%(?:\\d+\\$)?[+-](?:[ 0]|\'.{1})?-?\\d*(?:\\.\\d+)?"
+ "[lhjztL]*[eEfFgG]j)?")
out = re.findall(fmt_regex, backwards.bytes2unicode(fmt_str))
if isinstance(fmt_str, backwards.bytes_type):
out = [backwards.unicode2bytes(f) for f in out]
return out
def test_cformat2nptype():
r"""Test conversion from C format string to numpy dtype."""
for a, b in map_cformat2nptype:
if isinstance(a, str):
a = [a]
for _ia in a:
if _ia.startswith(backwards.bytes2unicode(AsciiTable._fmt_char)):
ia = backwards.unicode2bytes(_ia)
else:
ia = AsciiTable._fmt_char + backwards.unicode2bytes(_ia)
assert_equal(AsciiTable.cformat2nptype(ia), np.dtype(b).str)
assert_raises(TypeError, AsciiTable.cformat2nptype, 0)
assert_raises(ValueError, AsciiTable.cformat2nptype,
backwards.unicode2bytes('s'))
assert_raises(ValueError, AsciiTable.cformat2nptype,
backwards.unicode2bytes('%'))
for a in unsupported_nptype:
assert_raises(ValueError, AsciiTable.cformat2nptype,
backwards.unicode2bytes('%' + a))
def print_encoded(msg, *args, **kwargs):
r"""Print bytes to stdout, encoding if possible.
Args:
msg (str, bytes): Message to print.
*args: Additional arguments are passed to print.
**kwargs: Additional keyword arguments are passed to print.
"""
try:
print(backwards.bytes2unicode(msg), *args, **kwargs)
except UnicodeEncodeError: # pragma: debug
logging.debug("sys.stdout.encoding = %s, cannot print unicode",
sys.stdout.encoding)
kwargs.pop('end', None)
try:
print(msg, *args, **kwargs)
except UnicodeEncodeError: # pragma: debug
print(backwards.unicode2bytes(msg), *args, **kwargs)
def _send(self, msg):
r"""Write message to a file.
Args:
msg (bytes, str): Data to write to the file.
Returns:
bool: Success or failure of writing to the file.
"""
if msg != self.eof_msg:
if not self.open_as_binary:
msg = backwards.bytes2unicode(msg)
self.fd.write(msg)
if self.append == 'ow':
self.fd.truncate()
self.fd.flush()
if msg != self.eof_msg and self.is_series:
self.advance_in_series()
self.debug("Advanced to %d", self._series_index)
return True
def writeline_full(self, line):
r"""Write a line to the file in its present state. If it is not open,
nothing happens.
Args:
line (str/bytes): Line to be written.
Raises:
TypeError: If line is not the correct bytes type.
"""
if not self.is_open:
print("The file is not open. Nothing written.")
return
if self.open_as_binary:
line = backwards.unicode2bytes(line)
else:
line = backwards.bytes2unicode(line)
# if not isinstance(line, backwards.bytes_type):
# raise TypeError("Line must be of type %s" % backwards.bytes_type)
self.fd.write(line)
def array_to_table(arrs, fmt_str, use_astropy=False):
r"""Serialize an array as an ASCII table.
Args:
arrs (np.ndarray, list, tuple): Structured array or list/tuple of
arrays that contain table information.
fmt_str (str, bytes): Format string that should be used to structure
the ASCII array.
use_astropy (bool, optional): If True, astropy will be used to format
the table if it is installed. Defaults to False.
Returns:
bytes: ASCII table.
"""
if not _use_astropy:
use_astropy = False
dtype = cformat2nptype(fmt_str)
info = format2table(fmt_str)
arr1 = consolidate_array(arrs, dtype=dtype)
if use_astropy:
fd = backwards.StringIO()
table = apy_Table(arr1)
delimiter = info['delimiter']
delimiter = backwards.bytes2unicode(delimiter)
apy_ascii.write(table, fd, delimiter=delimiter,
format='no_header')
out = backwards.unicode2bytes(fd.getvalue())
else:
fd = backwards.BytesIO()
for ele in arr1:
line = format_message(ele.tolist(), fmt_str)
fd.write(line)
# fmt = fmt_str.split(info['newline'])[0]
# np.savetxt(fd, arr1,
# fmt=fmt, delimiter=info['delimiter'],
# newline=info['newline'], header='')
out = fd.getvalue()
fd.close()
return out
def test_cformat2nptype():
r"""Test conversion from C format string to numpy dtype."""
for a, b in map_cformat2nptype:
if isinstance(a, str):
a = [a]
for _ia in a:
if _ia.startswith(backwards.bytes2unicode(serialize._fmt_char)):
ia = backwards.unicode2bytes(_ia)
else:
ia = serialize._fmt_char + backwards.unicode2bytes(_ia)
nt.assert_equal(serialize.cformat2nptype(ia), np.dtype(b)) # .str)
# nt.assert_equal(serialize.cformat2nptype(ia), np.dtype(b).str)
nt.assert_raises(TypeError, serialize.cformat2nptype, 0)
nt.assert_raises(ValueError, serialize.cformat2nptype,
backwards.unicode2bytes('s'))
nt.assert_raises(ValueError, serialize.cformat2nptype,
backwards.unicode2bytes('%'))
nt.assert_raises(ValueError, serialize.cformat2nptype,
'%d\t%f', names=['one'])
for a in unsupported_nptype:
nt.assert_raises(ValueError, serialize.cformat2nptype,
backwards.unicode2bytes('%' + a))
def deserialize(self, msg):
r"""Deserialize a message.
Args:
msg (str, bytes): Message to be deserialized.
Returns:
tuple(obj, dict): Deserialized message and header information.
Raises:
TypeError: If msg is not bytes type (str on Python 2).
"""
if not isinstance(msg, backwards.bytes_type):
raise TypeError("Message to be deserialized is not bytes type.")
if len(msg) == 0:
obj = self._empty_msg
else:
metadata, data = msg.split(self.sep)
metadata = json.loads(backwards.bytes2unicode(metadata))
obj = self.__class__.decode(metadata, data, self._typedef)
return obj
def test_bytes2unicode():
r"""Ensure what results is proper bytes type."""
if backwards.PY2: # pragma: Python 2
res = backwards.unicode_type('hello')
backwards.assert_unicode(res)
nt.assert_equal(backwards.bytes2unicode('hello'), res)
nt.assert_equal(backwards.bytes2unicode(unicode('hello')), res)
nt.assert_equal(backwards.bytes2unicode(bytearray('hello', 'utf-8')),
res)
nt.assert_raises(TypeError, backwards.bytes2unicode, 1)
else: # pragma: Python 3
res = 'hello'
backwards.assert_unicode(res)
nt.assert_equal(backwards.bytes2unicode('hello'), res)
nt.assert_equal(backwards.bytes2unicode(b'hello'), res)
nt.assert_equal(backwards.bytes2unicode(bytearray('hello', 'utf-8')),
res)
nt.assert_raises(TypeError, backwards.bytes2unicode, 1)
def func_deserialize(self, msg):
r"""Deserialize a message.
Args:
msg (str, bytes): Message to be deserialized.
Returns:
dict: Deserialized Python dictionary.
"""
if len(msg) == 0:
out = self.empty_msg
else:
out = dict()
lines = backwards.bytes2unicode(msg).split(self.newline)
for l in lines:
kv = l.split(self.delimiter)
if len(kv) <= 1:
continue
elif len(kv) == 2:
out[kv[0]] = eval(kv[1])
else:
raise ValueError("Line has more than one delimiter: " + l)
return out