def encode_json(obj, fd=None, indent=None, sort_keys=True, **kwargs):
r"""Encode a Python object in JSON format.
Args:
obj (object): Python object to encode.
fd (file, optional): File descriptor for file that encoded object
should be written to. Defaults to None and string is returned.
indent (int, str, optional): Indentation for new lines in encoded
string. Defaults to None.
sort_keys (bool, optional): If True, the keys will be output in sorted
order. Defaults to True.
**kwargs: Additional keyword arguments are passed to json.dumps.
Returns:
str, bytes: Encoded object.
"""
if (indent is None) and (fd is not None):
indent = '\t'
# Character indents not allowed in Python 2 json
indent = indent_char2int(indent)
kwargs['indent'] = indent
kwargs['sort_keys'] = sort_keys
if 'cls' in kwargs:
kwargs.setdefault('default', kwargs.pop('cls')().default)
else:
kwargs.setdefault('default', JSONEncoder().default)
if fd is None:
return tools.str2bytes(json.dumps(obj, **kwargs))
else:
return json.dump(obj, fd, **kwargs)
def get_field_units(self, as_bytes=False):
r"""Get the field units for an array of fields.
Args:
as_bytes (bool, optional): If True, the field units will be returned
as bytes. If False the field units will be returned as unicode.
Defaults to False.
Returns:
list: Units for each field in the data type.
"""
if self.typedef['type'] != 'array':
return None
if getattr(self, 'field_units', None) is not None:
out = copy.deepcopy(self.field_units)
elif isinstance(self.typedef['items'], dict): # pragma: debug
raise Exception("Variable number of items not yet supported.")
elif isinstance(self.typedef['items'], list):
out = []
any_units = False
for i, x in enumerate(self.typedef['items']):
out.append(x.get('units', ''))
if len(x.get('units', '')) > 0:
any_units = True
# Don't use field units if they are all defaults
if not any_units:
out = None
if (out is not None):
if as_bytes:
out = tools.str2bytes(out, recurse=True)
else:
out = tools.bytes2str(out, recurse=True)
return out
def func_serialize(self, args):
r"""Serialize a message.
Args:
args (dict): Python dictionary to be serialized.
Returns:
bytes, str: Serialized message.
"""
lines = []
for k in args.keys():
v = args[k]
if not isinstance(k, (str, bytes)): # pragma: debug
raise ValueError(
"Serialization of non-string keys not supported.")
iline = tools.bytes2str(k) + self.delimiter
if isinstance(v, list):
indent = ' ' * len(iline)
arr_lines = []
assert (len(v) == 2)
assert (v[0].shape == v[1].shape)
for i in range(len(v[0])):
arr_lines.append(self._array_fmt % (v[0][i], v[1][i]))
v_units = [str(getattr(vv, 'units', '-')) for vv in v]
arr_lines[0] += f"\t! [{'; '.join(v_units)}]"
iline += (',\n' + indent).join(arr_lines)
elif isinstance(v, str):
iline += "\'%s\'" % v
else:
iline += json.dumps(v, cls=JSONReadableEncoder)
if hasattr(v, 'units'):
iline += f'\t! [{v.units}]'
lines.append(iline)
return tools.str2bytes('\n'.join(lines))
def _send_direct(self, msg, topic='', identity=None, **kwargs):
r"""Send a message.
Args:
msg (str, bytes): Message to be sent.
topic (str, optional): Filter that should be sent with the
message for 'PUB' sockets. Defaults to ''.
identity (str, optional): Identify of identified worker that
should be sent for 'ROUTER' sockets. Defaults to
self.dealer_identity.
**kwargs: Additional keyword arguments are passed to socket send.
Returns:
bool: Success or failure of send.
"""
if not self.is_open_direct: # pragma: debug
self.error("Socket closed")
return False
if identity is None:
identity = self.dealer_identity
topic = tools.str2bytes(topic)
identity = tools.str2bytes(identity)
if self.socket_type_name == 'PUB':
total_msg = topic + _flag_zmq_filter + msg
else:
total_msg = msg
total_msg = self.check_reply_socket_send(total_msg)
kwargs.setdefault('flags', zmq.NOBLOCK)
with self.socket_lock:
try:
if self.socket.closed: # pragma: debug
self.error("Socket closed")
return False
self.special_debug("Sending %d bytes to %s", len(total_msg), self.address)
if self.socket_type_name == 'ROUTER':
self.socket.send(identity, zmq.SNDMORE)
self.socket.send(total_msg, **kwargs)
self.special_debug("Sent %d bytes to %s", len(total_msg), self.address)
self._n_zmq_sent += 1
except zmq.ZMQError as e: # pragma: debug
if e.errno == zmq.EAGAIN:
raise AsyncComm.AsyncTryAgain("Socket not yet available.")
else:
self.special_debug("Socket could not send. (errno=%d)", e.errno)
return False
return True
def on_message(self, ch, method, props, body):
r"""Buffer received messages."""
body = tools.str2bytes(body)
if self.direction == 'send': # pragma: debug
raise Exception("Send comm received a message.")
with self.rmq_lock:
self._buffered_messages.append(body)
ch.basic_ack(delivery_tag=method.delivery_tag)
def array_to_table(arrs, fmt_str, use_astropy=False):
r"""Serialize an array as an ASCII table.
Args:
arrs (np.ndarray, list, tuple): Structured array or list/tuple of
arrays that contain table information.
fmt_str (str, bytes): Format string that should be used to structure
the ASCII array.
use_astropy (bool, optional): If True, astropy will be used to format
the table if it is installed. Defaults to False.
Returns:
bytes: ASCII table.
"""
if not _use_astropy:
use_astropy = False
dtype = cformat2nptype(fmt_str)
if len(dtype) == 0:
dtype = np.dtype([('f0', dtype)])
info = format2table(fmt_str)
comment = info.get('comment', None)
if comment is not None:
fmt_str = fmt_str.split(comment, 1)[-1]
arr1 = consolidate_array(arrs, dtype=dtype)
if use_astropy:
fd = sio.StringIO()
table = apy_Table(arr1)
delimiter = tools.bytes2str(info['delimiter'])
apy_ascii.write(table, fd, delimiter=delimiter,
format='no_header')
out = tools.str2bytes(fd.getvalue())
else:
fd = sio.BytesIO()
fmt_str = tools.str2bytes(fmt_str)
for ele in arr1:
line = format_message(ele.tolist(), fmt_str)
fd.write(line)
# fmt = fmt_str.split(info['newline'])[0]
# np.savetxt(fd, arr1,
# fmt=fmt, delimiter=info['delimiter'],
# newline=info['newline'], header='')
out = fd.getvalue()
fd.close()
return out
def test_str2bytes():
r"""Test str2bytes."""
vals = [('hello', b'hello'), (b'hello', b'hello'),
(('a', 'b'), (b'a', b'b')),
({
'a': 'a',
'b': 'b'
}, {
'a': b'a',
'b': b'b'
}), (['a', 'b'], [b'a', b'b']),
(['a', ['b', 'c']], [b'a', [b'b', b'c']])]
for x, exp in vals:
assert_equal(tools.str2bytes(x, recurse=True), exp)
def get_testing_options(cls, read_meth=None, **kwargs):
r"""Method to return a dictionary of testing options for this class.
Args:
read_meth (str, optional): Read method that will be used by the test
class. Defaults to None and will be set by the serialier.
**kwargs: Additional keyword arguments are passed to the parent
class's method and the serializers methods for determining the
default read_meth and concatenating the sent objects into the
objects that are expected to be received.
Returns:
dict: Dictionary of variables to use for testing. Key/value pairs:
kwargs (dict): Keyword arguments for comms tested with the
provided content.
send (list): List of objects to send to test file.
recv (list): List of objects that will be received from a test
file that was sent the messages in 'send'.
contents (bytes): Bytes contents of test file created by sending
the messages in 'send'.
"""
out = super(FileComm, cls).get_testing_options(**kwargs)
if 'read_meth' in cls._schema_properties:
if read_meth is None:
read_meth = cls._schema_properties['read_meth']['default']
out['kwargs']['read_meth'] = read_meth
if read_meth == 'readline':
out['recv_partial'] = [[x] for x in out['recv']]
if cls._default_serializer == 'direct':
comment = tools.str2bytes(
cls._schema_properties['comment']['default'] + 'Comment\n')
out['send'].append(comment)
out['contents'] += comment
out['recv_partial'].append([])
else:
seri_cls = cls._default_serializer_class
if seri_cls.concats_as_str:
out['recv_partial'] = [[x] for x in out['recv']]
out['recv'] = seri_cls.concatenate(out['recv'],
**out['kwargs'])
else:
out['recv_partial'] = [[out['recv'][0]]]
for i in range(1, len(out['recv'])):
out['recv_partial'].append(
seri_cls.concatenate(
out['recv_partial'][-1] + [out['recv'][i]],
**out['kwargs']))
out['recv'] = copy.deepcopy(out['recv_partial'][-1])
return out
def func_serialize(self, args):
r"""Serialize a message.
Args:
args: List of arguments to be formatted or numpy array to be
serialized.
Returns:
bytes, str: Serialized message.
"""
if (((self.extra_kwargs.get('format_str', None) is not None)
and isinstance(args, list))):
args = format_message(args, self.extra_kwargs['format_str'])
return tools.str2bytes(args)
def func_serialize(self, args):
r"""Serialize a message.
Args:
args: List of arguments to be formatted or numpy array to be
serialized.
Returns:
bytes, str: Serialized message.
"""
if self.format_str is None:
raise RuntimeError("Format string is not defined.")
args = self.datatype.coerce_type(args,
key_order=self.get_field_names())
if self.as_array:
out = serialize.array_to_table(args,
self.format_str,
use_astropy=self.use_astropy)
else:
out = serialize.format_message(args, self.format_str)
return tools.str2bytes(out)
def func_serialize(self, args):
r"""Serialize a message.
Args:
args (dict): Python dictionary to be serialized.
Returns:
bytes, str: Serialized message.
"""
out = ''
order = sorted([k for k in args.keys()])
newline_str = tools.bytes2str(self.newline)
for k in order:
v = args[k]
if not isinstance(k, (str, bytes)):
raise ValueError(
"Serialization of non-string keys not supported.")
out += tools.bytes2str(k)
out += self.delimiter
out += json.dumps(v, cls=JSONReadableEncoder)
out += newline_str
return tools.str2bytes(out)
def update_serializer(self, extract=False, skip_type=False, **kwargs):
r"""Update serializer with provided information.
Args:
extract (bool, optional): If True, the updated typedef will be
the bare minimum as extracted from total set of provided
keywords, otherwise the entire set will be sued. Defaults to
False.
skip_type (bool, optional): If True, everything is updated except
the data type. Defaults to False.
**kwargs: Additional keyword arguments are processed as part of
they type definition and are parsed for old-style keywords.
Raises:
RuntimeError: If there are keywords that are not valid typedef
keywords (currect or old-style).
"""
old_datatype = None
if self.initialized:
old_datatype = copy.deepcopy(self.datatype)
_metaschema = get_metaschema()
# Raise an error if the types are not compatible
seritype = kwargs.pop('seritype', self.seritype)
if (seritype != self._seritype) and (seritype !=
'default'): # pragma: debug
raise Exception("Cannot change types form %s to %s." %
(self._seritype, seritype))
# Remove metadata keywords unrelated to serialization
# TODO: Find a better way of tracking these
_remove_kws = [
'body', 'address', 'size', 'id', 'incomplete', 'raw', 'commtype',
'filetype', 'response_address', 'request_id', 'append', 'in_temp',
'is_series', 'working_dir', 'fmts', 'model_driver', 'env',
'send_converter', 'recv_converter', 'typedef_base'
]
kws = list(kwargs.keys())
for k in kws:
if (k in _remove_kws) or k.startswith('zmq'):
kwargs.pop(k)
# Set attributes and remove unused metadata keys
for k in self._schema_properties.keys():
if (k in kwargs) and (k != 'datatype'):
setattr(self, k, kwargs.pop(k))
# Create preliminary typedef
typedef = kwargs.pop('datatype', {})
for k in _metaschema['properties'].keys():
if k in kwargs:
typedef[k] = kwargs.pop(k)
# Update extra keywords
if (len(kwargs) > 0):
self.extra_kwargs.update(kwargs)
self.debug("Extra kwargs: %s" % str(self.extra_kwargs))
# Update type
if not skip_type:
# Update typedef from oldstyle keywords in extra_kwargs
typedef = self.update_typedef_from_oldstyle(typedef)
if typedef.get('type', None):
if extract:
cls = get_type_class(typedef['type'])
typedef = cls.extract_typedef(typedef)
self.datatype = get_type_from_def(typedef)
# Check to see if new datatype is compatible with new one
if old_datatype is not None:
errors = list(
compare_schema(self.typedef, old_datatype._typedef) or ())
if errors:
raise RuntimeError((
"Updated datatype is not compatible with the existing one."
+ " New:\n%s\nOld:\n%s\n") %
(pprint.pformat(self.typedef),
pprint.pformat(old_datatype._typedef)))
# Enfore that strings used with messages are in bytes
for k in self._attr_conv:
v = getattr(self, k, None)
if isinstance(v, (str, bytes)):
setattr(self, k, tools.str2bytes(v))
def get_testing_options(cls,
table_example=False,
array_columns=False,
include_oldkws=False,
table_string_type='bytes'):
r"""Method to return a dictionary of testing options for this class.
Arguments:
table_example (bool, optional): If True, the returned options will
be for an array of elements representing a table-like structure.
Defaults to False.
array_columns (bool, optional): If True, table_example is set to
True and the returned options will be for an array data type
where each element is an array representing a column Defaults to
False.
include_oldkws (bool, optional): If True, old-style keywords will be
added to the returned options. This will only have an effect if
table_example is True. Defaults to False.
table_string_type (str, optional): Type that should be used
for the string column in the table. Defaults to 'bytes'.
Returns:
dict: Dictionary of variables to use for testing. Key/value pairs:
* kwargs (dict): Keyword arguments for comms tested with the
provided content.
* empty (object): Object produced from deserializing an empty
message.
* objects (list): List of objects to be serialized/deserialized.
extra_kwargs (dict): Extra keyword arguments not used to
construct type definition.
* typedef (dict): Type definition resulting from the supplied
kwargs.
* dtype (np.dtype): Numpy data types that is consistent with the
determined type definition.
* contents (bytes): Concatenated serialization that will result from
deserializing the serialized objects.
* contents_recv (list): List of objects that would be deserialized
from contents.
"""
if array_columns:
table_example = True
if table_example:
assert (table_string_type in ['bytes', 'unicode', 'string'])
if table_string_type == 'string':
table_string_type = 'unicode'
if table_string_type == 'bytes':
np_dtype_str = 'S'
rows = [(b'one', np.int32(1), 1.0), (b'two', np.int32(2), 2.0),
(b'three', np.int32(3), 3.0)]
else:
np_dtype_str = 'U'
rows = [('one', np.int32(1), 1.0), ('two', np.int32(2), 2.0),
('three', np.int32(3), 3.0)]
out = {
'kwargs': {},
'empty': [],
'dtype':
None,
'extra_kwargs': {},
'typedef': {
'type':
'array',
'items': [{
'type': table_string_type,
'units': 'n/a',
'title': 'name'
}, {
'type': 'int',
'precision': 32,
'units': 'umol',
'title': 'count'
}, {
'type': 'float',
'precision': 64,
'units': 'cm',
'title': 'size'
}]
},
'contents':
(b'# name\tcount\tsize\n' + b'# n/a\tumol\tcm\n' +
b'# %5s\t%d\t%f\n' + b' one\t1\t1.000000\n' +
b' two\t2\t2.000000\n' + b'three\t3\t3.000000\n' +
b' one\t1\t1.000000\n' + b' two\t2\t2.000000\n' +
b'three\t3\t3.000000\n'),
'objects':
2 * rows,
'field_names': ['name', 'count', 'size'],
'field_units': ['n/a', 'umol', 'cm']
}
if include_oldkws:
out['kwargs'].update({
'format_str': '%5s\t%d\t%f\n',
'field_names': ['name', 'count', 'size'],
'field_units': ['n/a', 'umol', 'cm']
})
out['extra_kwargs']['format_str'] = out['kwargs']['format_str']
if 'format_str' in cls._attr_conv:
out['extra_kwargs']['format_str'] = tools.str2bytes(
out['extra_kwargs']['format_str'])
if array_columns:
out['kwargs']['as_array'] = True
dtype = np.dtype({
'names': out['field_names'],
'formats': ['%s5' % np_dtype_str, 'i4', 'f8']
})
out['dtype'] = dtype
arr = np.array(rows, dtype=dtype)
lst = [
units.add_units(arr[n], u)
for n, u in zip(out['field_names'], out['field_units'])
]
out['objects'] = [lst, lst]
for x in out['typedef']['items']:
x['subtype'] = x['type']
x['type'] = '1darray'
if x['title'] == 'name':
x['precision'] = 40
if x['subtype'] == 'unicode':
x['precision'] *= 4
else:
out = {
'kwargs': {},
'empty': b'',
'dtype': None,
'typedef': cls.default_datatype,
'extra_kwargs': {},
'objects': [b'Test message\n', b'Test message 2\n']
}
out['contents'] = b''.join(out['objects'])
return out
def _init_before_open(self,
context=None,
socket_type=None,
socket_action=None,
topic_filter='',
dealer_identity=None,
new_process=False,
reply_socket_address=None,
**kwargs):
r"""Initialize defaults for socket type/action based on direction."""
self.reply_socket_lock = multitasking.RLock()
self.socket_lock = multitasking.RLock()
# Client/Server things
if self.is_client:
socket_type = 'DEALER'
socket_action = 'connect'
self.direction = 'send'
if self.is_server:
socket_type = 'DEALER'
socket_action = 'connect'
self.direction = 'recv'
# Set defaults
if socket_type is None:
if self.direction == 'recv':
socket_type = _socket_recv_types[_default_socket_type]
elif self.direction == 'send':
socket_type = _socket_send_types[_default_socket_type]
if not (self.is_client or self.is_server):
if socket_type in ['PULL', 'SUB', 'REP', 'DEALER']:
self.direction = 'recv'
elif socket_type in ['PUSH', 'PUB', 'REQ', 'ROUTER']:
self.direction = 'send'
if socket_action is None:
if self.port in ['inproc', 'ipc']:
if socket_type in ['PULL', 'SUB', 'REQ', 'DEALER']:
socket_action = 'connect'
elif socket_type in ['PUSH', 'PUB', 'REP', 'ROUTER']:
socket_action = 'bind'
else:
if self.direction == 'recv':
socket_action = 'connect'
elif self.direction == 'send':
socket_action = 'bind'
elif self.port is None:
socket_action = 'bind'
else:
socket_action = 'connect'
if new_process:
self.context = zmq.Context()
else:
self.context = context or _global_context
self.socket_type_name = socket_type
self.socket_type = getattr(zmq, socket_type)
self.socket_action = socket_action
self.socket = self.context.socket(self.socket_type)
self.socket.setsockopt(zmq.LINGER, 0)
self.topic_filter = tools.str2bytes(topic_filter)
if dealer_identity is None:
dealer_identity = str(uuid.uuid4())
self.dealer_identity = tools.str2bytes(dealer_identity)
self._openned = False
self._bound = False
self._connected = False
self._recv_identities = set([])
# Reply socket attributes
self.zmq_sleeptime = int(10000 * self.sleeptime)
self.reply_socket_address = reply_socket_address
self.reply_socket_send = None
self.reply_socket_recv = {}
self._n_zmq_sent = 0
self._n_zmq_recv = {}
self._n_reply_sent = 0
self._n_reply_recv = {}
self._server_class = ZMQProxy
self._server_kwargs = dict(zmq_context=self.context,
nretry=4,
retry_timeout=2.0 * self.sleeptime)
super(ZMQComm, self)._init_before_open(**kwargs)
def format_header(format_str=None,
dtype=None,
comment=None,
delimiter=None,
newline=None,
field_names=None,
field_units=None):
r"""Get header lines for a table based on table information.
Args:
format_str (bytes, str, optional): Format string describing how the
table should be formatted. If not provided, information on the
formats is extracted from dtype.
dtype (np.dtype, optional): Structured data type specifying the types
of fields in the table. If not provided and format_str not
specified, the formats will not be part of the header.
comment (bytes, optional): String that should be used to comment the
header lines. If not provided and not in format_str, defaults to
_default_comment.
delimiter (bytes, optional): String that should be used to separate
columns. If not provided and not in format_str, defaults to
_default_delimiter.
newline (bytes, optional): String that should be used to end lines in
the table. If not provided and not in format_str, defaults to
_default_newline.
field_names (list, optional): List of field names that should be
included in the header. If not provided and dtype is None, names
will not be included in the header.
field_units (list, optional): List of field units that should be
included in the header. If not provided, units will not be
included.
Returns:
bytes: Bytes lines comprising a table header.
Raises:
ValueError: If there are not any format, names or units specified.
"""
# Set defaults
fmts = None
if format_str is not None:
info = format2table(format_str)
if len(info['fmts']) > 0:
fmts = info['fmts']
if delimiter is None:
delimiter = info['delimiter']
if newline is None:
newline = info.get('newline', None)
if comment is None:
comment = info.get('comment', None)
if dtype is not None:
if fmts is None:
fmts = nptype2cformat(dtype, asbytes=True)
if field_names is None:
field_names = [n.encode("utf-8") for n in dtype.names]
if delimiter is None:
delimiter = _default_delimiter
if comment is None:
comment = _default_comment
if newline is None:
newline = _default_newline
# Get count of fields
if fmts is not None:
nfld = len(fmts)
elif field_names is not None:
nfld = len(field_names)
elif field_units is not None:
nfld = len(field_units)
else:
raise ValueError("No formats, names, or units provided.")
# Create lines
out = []
for x in [field_names, field_units, fmts]:
if (x is not None) and (len(max(x, key=len)) > 0):
assert (len(x) == nfld)
x_bytes = tools.str2bytes(x, recurse=True)
out.append(comment + delimiter.join(x_bytes))
out = newline.join(out) + newline
return out
def on_message(self, _unused_channel, basic_deliver, properties, body):
r"""Buffer received messages."""
self.debug('Received message # %s from %s: %.100s',
basic_deliver.delivery_tag, properties.app_id, body)
body = tools.str2bytes(body)
self._buffered_messages.put(body)
