def encode_json(obj, fd=None, indent=None, sort_keys=True, **kwargs): r"""Encode a Python object in JSON format. Args: obj (object): Python object to encode. fd (file, optional): File descriptor for file that encoded object should be written to. Defaults to None and string is returned. indent (int, str, optional): Indentation for new lines in encoded string. Defaults to None. sort_keys (bool, optional): If True, the keys will be output in sorted order. Defaults to True. **kwargs: Additional keyword arguments are passed to json.dumps. Returns: str, bytes: Encoded object. """ if (indent is None) and (fd is not None): indent = '\t' # Character indents not allowed in Python 2 json indent = indent_char2int(indent) kwargs['indent'] = indent kwargs['sort_keys'] = sort_keys if 'cls' in kwargs: kwargs.setdefault('default', kwargs.pop('cls')().default) else: kwargs.setdefault('default', JSONEncoder().default) if fd is None: return tools.str2bytes(json.dumps(obj, **kwargs)) else: return json.dump(obj, fd, **kwargs)
def get_field_units(self, as_bytes=False): r"""Get the field units for an array of fields. Args: as_bytes (bool, optional): If True, the field units will be returned as bytes. If False the field units will be returned as unicode. Defaults to False. Returns: list: Units for each field in the data type. """ if self.typedef['type'] != 'array': return None if getattr(self, 'field_units', None) is not None: out = copy.deepcopy(self.field_units) elif isinstance(self.typedef['items'], dict): # pragma: debug raise Exception("Variable number of items not yet supported.") elif isinstance(self.typedef['items'], list): out = [] any_units = False for i, x in enumerate(self.typedef['items']): out.append(x.get('units', '')) if len(x.get('units', '')) > 0: any_units = True # Don't use field units if they are all defaults if not any_units: out = None if (out is not None): if as_bytes: out = tools.str2bytes(out, recurse=True) else: out = tools.bytes2str(out, recurse=True) return out
def func_serialize(self, args): r"""Serialize a message. Args: args (dict): Python dictionary to be serialized. Returns: bytes, str: Serialized message. """ lines = [] for k in args.keys(): v = args[k] if not isinstance(k, (str, bytes)): # pragma: debug raise ValueError( "Serialization of non-string keys not supported.") iline = tools.bytes2str(k) + self.delimiter if isinstance(v, list): indent = ' ' * len(iline) arr_lines = [] assert (len(v) == 2) assert (v[0].shape == v[1].shape) for i in range(len(v[0])): arr_lines.append(self._array_fmt % (v[0][i], v[1][i])) v_units = [str(getattr(vv, 'units', '-')) for vv in v] arr_lines[0] += f"\t! [{'; '.join(v_units)}]" iline += (',\n' + indent).join(arr_lines) elif isinstance(v, str): iline += "\'%s\'" % v else: iline += json.dumps(v, cls=JSONReadableEncoder) if hasattr(v, 'units'): iline += f'\t! [{v.units}]' lines.append(iline) return tools.str2bytes('\n'.join(lines))
def _send_direct(self, msg, topic='', identity=None, **kwargs): r"""Send a message. Args: msg (str, bytes): Message to be sent. topic (str, optional): Filter that should be sent with the message for 'PUB' sockets. Defaults to ''. identity (str, optional): Identify of identified worker that should be sent for 'ROUTER' sockets. Defaults to self.dealer_identity. **kwargs: Additional keyword arguments are passed to socket send. Returns: bool: Success or failure of send. """ if not self.is_open_direct: # pragma: debug self.error("Socket closed") return False if identity is None: identity = self.dealer_identity topic = tools.str2bytes(topic) identity = tools.str2bytes(identity) if self.socket_type_name == 'PUB': total_msg = topic + _flag_zmq_filter + msg else: total_msg = msg total_msg = self.check_reply_socket_send(total_msg) kwargs.setdefault('flags', zmq.NOBLOCK) with self.socket_lock: try: if self.socket.closed: # pragma: debug self.error("Socket closed") return False self.special_debug("Sending %d bytes to %s", len(total_msg), self.address) if self.socket_type_name == 'ROUTER': self.socket.send(identity, zmq.SNDMORE) self.socket.send(total_msg, **kwargs) self.special_debug("Sent %d bytes to %s", len(total_msg), self.address) self._n_zmq_sent += 1 except zmq.ZMQError as e: # pragma: debug if e.errno == zmq.EAGAIN: raise AsyncComm.AsyncTryAgain("Socket not yet available.") else: self.special_debug("Socket could not send. (errno=%d)", e.errno) return False return True
def on_message(self, ch, method, props, body): r"""Buffer received messages.""" body = tools.str2bytes(body) if self.direction == 'send': # pragma: debug raise Exception("Send comm received a message.") with self.rmq_lock: self._buffered_messages.append(body) ch.basic_ack(delivery_tag=method.delivery_tag)
def array_to_table(arrs, fmt_str, use_astropy=False): r"""Serialize an array as an ASCII table. Args: arrs (np.ndarray, list, tuple): Structured array or list/tuple of arrays that contain table information. fmt_str (str, bytes): Format string that should be used to structure the ASCII array. use_astropy (bool, optional): If True, astropy will be used to format the table if it is installed. Defaults to False. Returns: bytes: ASCII table. """ if not _use_astropy: use_astropy = False dtype = cformat2nptype(fmt_str) if len(dtype) == 0: dtype = np.dtype([('f0', dtype)]) info = format2table(fmt_str) comment = info.get('comment', None) if comment is not None: fmt_str = fmt_str.split(comment, 1)[-1] arr1 = consolidate_array(arrs, dtype=dtype) if use_astropy: fd = sio.StringIO() table = apy_Table(arr1) delimiter = tools.bytes2str(info['delimiter']) apy_ascii.write(table, fd, delimiter=delimiter, format='no_header') out = tools.str2bytes(fd.getvalue()) else: fd = sio.BytesIO() fmt_str = tools.str2bytes(fmt_str) for ele in arr1: line = format_message(ele.tolist(), fmt_str) fd.write(line) # fmt = fmt_str.split(info['newline'])[0] # np.savetxt(fd, arr1, # fmt=fmt, delimiter=info['delimiter'], # newline=info['newline'], header='') out = fd.getvalue() fd.close() return out
def test_str2bytes(): r"""Test str2bytes.""" vals = [('hello', b'hello'), (b'hello', b'hello'), (('a', 'b'), (b'a', b'b')), ({ 'a': 'a', 'b': 'b' }, { 'a': b'a', 'b': b'b' }), (['a', 'b'], [b'a', b'b']), (['a', ['b', 'c']], [b'a', [b'b', b'c']])] for x, exp in vals: assert_equal(tools.str2bytes(x, recurse=True), exp)
def get_testing_options(cls, read_meth=None, **kwargs): r"""Method to return a dictionary of testing options for this class. Args: read_meth (str, optional): Read method that will be used by the test class. Defaults to None and will be set by the serialier. **kwargs: Additional keyword arguments are passed to the parent class's method and the serializers methods for determining the default read_meth and concatenating the sent objects into the objects that are expected to be received. Returns: dict: Dictionary of variables to use for testing. Key/value pairs: kwargs (dict): Keyword arguments for comms tested with the provided content. send (list): List of objects to send to test file. recv (list): List of objects that will be received from a test file that was sent the messages in 'send'. contents (bytes): Bytes contents of test file created by sending the messages in 'send'. """ out = super(FileComm, cls).get_testing_options(**kwargs) if 'read_meth' in cls._schema_properties: if read_meth is None: read_meth = cls._schema_properties['read_meth']['default'] out['kwargs']['read_meth'] = read_meth if read_meth == 'readline': out['recv_partial'] = [[x] for x in out['recv']] if cls._default_serializer == 'direct': comment = tools.str2bytes( cls._schema_properties['comment']['default'] + 'Comment\n') out['send'].append(comment) out['contents'] += comment out['recv_partial'].append([]) else: seri_cls = cls._default_serializer_class if seri_cls.concats_as_str: out['recv_partial'] = [[x] for x in out['recv']] out['recv'] = seri_cls.concatenate(out['recv'], **out['kwargs']) else: out['recv_partial'] = [[out['recv'][0]]] for i in range(1, len(out['recv'])): out['recv_partial'].append( seri_cls.concatenate( out['recv_partial'][-1] + [out['recv'][i]], **out['kwargs'])) out['recv'] = copy.deepcopy(out['recv_partial'][-1]) return out
def func_serialize(self, args): r"""Serialize a message. Args: args: List of arguments to be formatted or numpy array to be serialized. Returns: bytes, str: Serialized message. """ if (((self.extra_kwargs.get('format_str', None) is not None) and isinstance(args, list))): args = format_message(args, self.extra_kwargs['format_str']) return tools.str2bytes(args)
def func_serialize(self, args): r"""Serialize a message. Args: args: List of arguments to be formatted or numpy array to be serialized. Returns: bytes, str: Serialized message. """ if self.format_str is None: raise RuntimeError("Format string is not defined.") args = self.datatype.coerce_type(args, key_order=self.get_field_names()) if self.as_array: out = serialize.array_to_table(args, self.format_str, use_astropy=self.use_astropy) else: out = serialize.format_message(args, self.format_str) return tools.str2bytes(out)
def func_serialize(self, args): r"""Serialize a message. Args: args (dict): Python dictionary to be serialized. Returns: bytes, str: Serialized message. """ out = '' order = sorted([k for k in args.keys()]) newline_str = tools.bytes2str(self.newline) for k in order: v = args[k] if not isinstance(k, (str, bytes)): raise ValueError( "Serialization of non-string keys not supported.") out += tools.bytes2str(k) out += self.delimiter out += json.dumps(v, cls=JSONReadableEncoder) out += newline_str return tools.str2bytes(out)
def update_serializer(self, extract=False, skip_type=False, **kwargs): r"""Update serializer with provided information. Args: extract (bool, optional): If True, the updated typedef will be the bare minimum as extracted from total set of provided keywords, otherwise the entire set will be sued. Defaults to False. skip_type (bool, optional): If True, everything is updated except the data type. Defaults to False. **kwargs: Additional keyword arguments are processed as part of they type definition and are parsed for old-style keywords. Raises: RuntimeError: If there are keywords that are not valid typedef keywords (currect or old-style). """ old_datatype = None if self.initialized: old_datatype = copy.deepcopy(self.datatype) _metaschema = get_metaschema() # Raise an error if the types are not compatible seritype = kwargs.pop('seritype', self.seritype) if (seritype != self._seritype) and (seritype != 'default'): # pragma: debug raise Exception("Cannot change types form %s to %s." % (self._seritype, seritype)) # Remove metadata keywords unrelated to serialization # TODO: Find a better way of tracking these _remove_kws = [ 'body', 'address', 'size', 'id', 'incomplete', 'raw', 'commtype', 'filetype', 'response_address', 'request_id', 'append', 'in_temp', 'is_series', 'working_dir', 'fmts', 'model_driver', 'env', 'send_converter', 'recv_converter', 'typedef_base' ] kws = list(kwargs.keys()) for k in kws: if (k in _remove_kws) or k.startswith('zmq'): kwargs.pop(k) # Set attributes and remove unused metadata keys for k in self._schema_properties.keys(): if (k in kwargs) and (k != 'datatype'): setattr(self, k, kwargs.pop(k)) # Create preliminary typedef typedef = kwargs.pop('datatype', {}) for k in _metaschema['properties'].keys(): if k in kwargs: typedef[k] = kwargs.pop(k) # Update extra keywords if (len(kwargs) > 0): self.extra_kwargs.update(kwargs) self.debug("Extra kwargs: %s" % str(self.extra_kwargs)) # Update type if not skip_type: # Update typedef from oldstyle keywords in extra_kwargs typedef = self.update_typedef_from_oldstyle(typedef) if typedef.get('type', None): if extract: cls = get_type_class(typedef['type']) typedef = cls.extract_typedef(typedef) self.datatype = get_type_from_def(typedef) # Check to see if new datatype is compatible with new one if old_datatype is not None: errors = list( compare_schema(self.typedef, old_datatype._typedef) or ()) if errors: raise RuntimeError(( "Updated datatype is not compatible with the existing one." + " New:\n%s\nOld:\n%s\n") % (pprint.pformat(self.typedef), pprint.pformat(old_datatype._typedef))) # Enfore that strings used with messages are in bytes for k in self._attr_conv: v = getattr(self, k, None) if isinstance(v, (str, bytes)): setattr(self, k, tools.str2bytes(v))
def get_testing_options(cls, table_example=False, array_columns=False, include_oldkws=False, table_string_type='bytes'): r"""Method to return a dictionary of testing options for this class. Arguments: table_example (bool, optional): If True, the returned options will be for an array of elements representing a table-like structure. Defaults to False. array_columns (bool, optional): If True, table_example is set to True and the returned options will be for an array data type where each element is an array representing a column Defaults to False. include_oldkws (bool, optional): If True, old-style keywords will be added to the returned options. This will only have an effect if table_example is True. Defaults to False. table_string_type (str, optional): Type that should be used for the string column in the table. Defaults to 'bytes'. Returns: dict: Dictionary of variables to use for testing. Key/value pairs: * kwargs (dict): Keyword arguments for comms tested with the provided content. * empty (object): Object produced from deserializing an empty message. * objects (list): List of objects to be serialized/deserialized. extra_kwargs (dict): Extra keyword arguments not used to construct type definition. * typedef (dict): Type definition resulting from the supplied kwargs. * dtype (np.dtype): Numpy data types that is consistent with the determined type definition. * contents (bytes): Concatenated serialization that will result from deserializing the serialized objects. * contents_recv (list): List of objects that would be deserialized from contents. """ if array_columns: table_example = True if table_example: assert (table_string_type in ['bytes', 'unicode', 'string']) if table_string_type == 'string': table_string_type = 'unicode' if table_string_type == 'bytes': np_dtype_str = 'S' rows = [(b'one', np.int32(1), 1.0), (b'two', np.int32(2), 2.0), (b'three', np.int32(3), 3.0)] else: np_dtype_str = 'U' rows = [('one', np.int32(1), 1.0), ('two', np.int32(2), 2.0), ('three', np.int32(3), 3.0)] out = { 'kwargs': {}, 'empty': [], 'dtype': None, 'extra_kwargs': {}, 'typedef': { 'type': 'array', 'items': [{ 'type': table_string_type, 'units': 'n/a', 'title': 'name' }, { 'type': 'int', 'precision': 32, 'units': 'umol', 'title': 'count' }, { 'type': 'float', 'precision': 64, 'units': 'cm', 'title': 'size' }] }, 'contents': (b'# name\tcount\tsize\n' + b'# n/a\tumol\tcm\n' + b'# %5s\t%d\t%f\n' + b' one\t1\t1.000000\n' + b' two\t2\t2.000000\n' + b'three\t3\t3.000000\n' + b' one\t1\t1.000000\n' + b' two\t2\t2.000000\n' + b'three\t3\t3.000000\n'), 'objects': 2 * rows, 'field_names': ['name', 'count', 'size'], 'field_units': ['n/a', 'umol', 'cm'] } if include_oldkws: out['kwargs'].update({ 'format_str': '%5s\t%d\t%f\n', 'field_names': ['name', 'count', 'size'], 'field_units': ['n/a', 'umol', 'cm'] }) out['extra_kwargs']['format_str'] = out['kwargs']['format_str'] if 'format_str' in cls._attr_conv: out['extra_kwargs']['format_str'] = tools.str2bytes( out['extra_kwargs']['format_str']) if array_columns: out['kwargs']['as_array'] = True dtype = np.dtype({ 'names': out['field_names'], 'formats': ['%s5' % np_dtype_str, 'i4', 'f8'] }) out['dtype'] = dtype arr = np.array(rows, dtype=dtype) lst = [ units.add_units(arr[n], u) for n, u in zip(out['field_names'], out['field_units']) ] out['objects'] = [lst, lst] for x in out['typedef']['items']: x['subtype'] = x['type'] x['type'] = '1darray' if x['title'] == 'name': x['precision'] = 40 if x['subtype'] == 'unicode': x['precision'] *= 4 else: out = { 'kwargs': {}, 'empty': b'', 'dtype': None, 'typedef': cls.default_datatype, 'extra_kwargs': {}, 'objects': [b'Test message\n', b'Test message 2\n'] } out['contents'] = b''.join(out['objects']) return out
def _init_before_open(self, context=None, socket_type=None, socket_action=None, topic_filter='', dealer_identity=None, new_process=False, reply_socket_address=None, **kwargs): r"""Initialize defaults for socket type/action based on direction.""" self.reply_socket_lock = multitasking.RLock() self.socket_lock = multitasking.RLock() # Client/Server things if self.is_client: socket_type = 'DEALER' socket_action = 'connect' self.direction = 'send' if self.is_server: socket_type = 'DEALER' socket_action = 'connect' self.direction = 'recv' # Set defaults if socket_type is None: if self.direction == 'recv': socket_type = _socket_recv_types[_default_socket_type] elif self.direction == 'send': socket_type = _socket_send_types[_default_socket_type] if not (self.is_client or self.is_server): if socket_type in ['PULL', 'SUB', 'REP', 'DEALER']: self.direction = 'recv' elif socket_type in ['PUSH', 'PUB', 'REQ', 'ROUTER']: self.direction = 'send' if socket_action is None: if self.port in ['inproc', 'ipc']: if socket_type in ['PULL', 'SUB', 'REQ', 'DEALER']: socket_action = 'connect' elif socket_type in ['PUSH', 'PUB', 'REP', 'ROUTER']: socket_action = 'bind' else: if self.direction == 'recv': socket_action = 'connect' elif self.direction == 'send': socket_action = 'bind' elif self.port is None: socket_action = 'bind' else: socket_action = 'connect' if new_process: self.context = zmq.Context() else: self.context = context or _global_context self.socket_type_name = socket_type self.socket_type = getattr(zmq, socket_type) self.socket_action = socket_action self.socket = self.context.socket(self.socket_type) self.socket.setsockopt(zmq.LINGER, 0) self.topic_filter = tools.str2bytes(topic_filter) if dealer_identity is None: dealer_identity = str(uuid.uuid4()) self.dealer_identity = tools.str2bytes(dealer_identity) self._openned = False self._bound = False self._connected = False self._recv_identities = set([]) # Reply socket attributes self.zmq_sleeptime = int(10000 * self.sleeptime) self.reply_socket_address = reply_socket_address self.reply_socket_send = None self.reply_socket_recv = {} self._n_zmq_sent = 0 self._n_zmq_recv = {} self._n_reply_sent = 0 self._n_reply_recv = {} self._server_class = ZMQProxy self._server_kwargs = dict(zmq_context=self.context, nretry=4, retry_timeout=2.0 * self.sleeptime) super(ZMQComm, self)._init_before_open(**kwargs)
def format_header(format_str=None, dtype=None, comment=None, delimiter=None, newline=None, field_names=None, field_units=None): r"""Get header lines for a table based on table information. Args: format_str (bytes, str, optional): Format string describing how the table should be formatted. If not provided, information on the formats is extracted from dtype. dtype (np.dtype, optional): Structured data type specifying the types of fields in the table. If not provided and format_str not specified, the formats will not be part of the header. comment (bytes, optional): String that should be used to comment the header lines. If not provided and not in format_str, defaults to _default_comment. delimiter (bytes, optional): String that should be used to separate columns. If not provided and not in format_str, defaults to _default_delimiter. newline (bytes, optional): String that should be used to end lines in the table. If not provided and not in format_str, defaults to _default_newline. field_names (list, optional): List of field names that should be included in the header. If not provided and dtype is None, names will not be included in the header. field_units (list, optional): List of field units that should be included in the header. If not provided, units will not be included. Returns: bytes: Bytes lines comprising a table header. Raises: ValueError: If there are not any format, names or units specified. """ # Set defaults fmts = None if format_str is not None: info = format2table(format_str) if len(info['fmts']) > 0: fmts = info['fmts'] if delimiter is None: delimiter = info['delimiter'] if newline is None: newline = info.get('newline', None) if comment is None: comment = info.get('comment', None) if dtype is not None: if fmts is None: fmts = nptype2cformat(dtype, asbytes=True) if field_names is None: field_names = [n.encode("utf-8") for n in dtype.names] if delimiter is None: delimiter = _default_delimiter if comment is None: comment = _default_comment if newline is None: newline = _default_newline # Get count of fields if fmts is not None: nfld = len(fmts) elif field_names is not None: nfld = len(field_names) elif field_units is not None: nfld = len(field_units) else: raise ValueError("No formats, names, or units provided.") # Create lines out = [] for x in [field_names, field_units, fmts]: if (x is not None) and (len(max(x, key=len)) > 0): assert (len(x) == nfld) x_bytes = tools.str2bytes(x, recurse=True) out.append(comment + delimiter.join(x_bytes)) out = newline.join(out) + newline return out
def on_message(self, _unused_channel, basic_deliver, properties, body): r"""Buffer received messages.""" self.debug('Received message # %s from %s: %.100s', basic_deliver.delivery_tag, properties.app_id, body) body = tools.str2bytes(body) self._buffered_messages.put(body)