def normalize(cls, obj): r"""Normalize an object, if possible, to conform to this type. Args: obj (object): Object to normalize. Returns: object: Normalized object. """ if cls.is_fixed and ('subtype' in cls.fixed_properties): if (cls.fixed_properties['subtype'] == 'bytes'): if isinstance(obj, backwards.string_types): obj = backwards.as_bytes(obj) else: obj = backwards.as_bytes(str(obj)) elif (cls.fixed_properties['subtype'] == 'unicode'): if isinstance(obj, backwards.string_types): obj = backwards.as_unicode(obj) else: obj = backwards.as_unicode(str(obj)) else: dtype = ScalarMetaschemaProperties._python_scalars[ cls.fixed_properties['subtype']][0] try: obj = dtype(obj) except (TypeError, ValueError): pass return obj
def test_format_header(): r"""Test formatting header.""" kws_all = dict(field_names=['name', 'number', 'value', 'complex'], field_units=['n/a', 'g', 'cm', 'n/a']) res_all = dict(names="# name\tnumber\tvalue\tcomplex\n", units="# n/a\tg\tcm\tn/a\n") if platform._is_win: # pragma: windows kws_all['format_str'] = "%5s\t%l64d\t%g\t%g%+gj\n" res_all['format'] = "# " + kws_all['format_str'] else: kws_all['format_str'] = "%5s\t%ld\t%g\t%g%+gj\n" res_all['format'] = "# " + kws_all['format_str'] kws_all['dtype'] = serialize.cformat2nptype(kws_all['format_str'], names=kws_all['field_names']) for x in [kws_all, res_all]: for k, v in x.items(): if isinstance(v, str): x[k] = backwards.as_bytes(v) elif isinstance(v, list): x[k] = [backwards.as_bytes(iv) for iv in v] test_list = [(['format_str', 'field_names', 'field_units'], ['names', 'units', 'format']), (['field_names', 'field_units', 'dtype'], ['names', 'units', 'format']), (['field_units', 'dtype'], ['names', 'units', 'format']), (['field_names'], ['names']), (['field_units'], ['units'])] for kws_keys, res_keys in test_list: kws = {k: kws_all[k] for k in kws_keys} res = b''.join([res_all[k] for k in res_keys]) assert_equal(serialize.format_header(**kws), res) assert_raises(ValueError, serialize.format_header)
def test_scanf(): r"""Test scanf.""" num_codes = 'dieEfgGouxX' fmt_list = [("%g%+gj", complex(1)), ("%s", "hello"), ("%5s", "the"), ("%5s\t%ld\t%lf\t%lf%+lfj\n", ("one", 1, 1.0, complex(1, 1)))] fmt_list += [("%" + s, 1) for s in num_codes] fmt_list += [("%5.2" + s, 1) for s in num_codes] fmt_list += [("%+5.2" + s, 1) for s in num_codes] fmt_list += [("%-5.2" + s, 1) for s in num_codes] fmt_list += [("% 5.2" + s, 1) for s in num_codes] fmt_list += [("%05.2" + s, 1) for s in num_codes] for fmt, val in fmt_list: if isinstance(val, tuple): val_tup = val else: val_tup = (val, ) new_tup = [] for v in val_tup: if isinstance(v, complex): new_tup += [v.real, v.imag] else: new_tup.append(v) val_str = fmt % tuple(new_tup) res = scanf.scanf(fmt, val_str) assert_equal(res, val_tup) # Test bytes version res = backwards.scanf_bytes(fmt, backwards.as_bytes(val_str)) assert_equal( res, backwards.as_bytes(val_tup, recurse=True, allow_pass=True)) # Test unicode version res = backwards.scanf_bytes(fmt, backwards.as_unicode(val_str)) assert_equal( res, backwards.as_unicode(val_tup, recurse=True, allow_pass=True))
def test_extract_formats(): r"""Test extract_formats.""" test_str = ['%10s\t%5.2f\t%4d\t%g%+gj'] test_fmt = [['%10s', '%5.2f', '%4d', '%g%+gj']] for s, f in zip(test_str, test_fmt): assert_equal(serialize.extract_formats(s), f) assert_equal(serialize.extract_formats(backwards.as_bytes(s)), [backwards.as_bytes(i) for i in f])
def __init__(self): self.field_names = ['name', 'count', 'size'] self.field_units = ['n/a', 'umol', 'cm'] self.nfields = len(self.field_names) self.comment = b'# ' self.delimiter = b'\t' self.newline = b'\n' self.field_names = [backwards.as_bytes(x) for x in self.field_names] self.field_units = [backwards.as_bytes(x) for x in self.field_units]
def _send_direct(self, msg, topic='', identity=None, **kwargs): r"""Send a message. Args: msg (str, bytes): Message to be sent. topic (str, optional): Filter that should be sent with the message for 'PUB' sockets. Defaults to ''. identity (str, optional): Identify of identified worker that should be sent for 'ROUTER' sockets. Defaults to self.dealer_identity. **kwargs: Additional keyword arguments are passed to socket send. Returns: bool: Success or failure of send. """ if not self.is_open_direct: # pragma: debug self.error("Socket closed") return False if identity is None: identity = self.dealer_identity topic = backwards.as_bytes(topic) identity = backwards.as_bytes(identity) if self.socket_type_name == 'PUB': total_msg = topic + _flag_zmq_filter + msg else: total_msg = msg total_msg = self.check_reply_socket_send(total_msg) kwargs.setdefault('flags', zmq.NOBLOCK) with self.socket_lock: try: if self.socket.closed: # pragma: debug self.error("Socket closed") return False self.special_debug("Sending %d bytes to %s", len(total_msg), self.address) if self.socket_type_name == 'ROUTER': self.socket.send(identity, zmq.SNDMORE) self.socket.send(total_msg, **kwargs) self.special_debug("Sent %d bytes to %s", len(total_msg), self.address) self._n_zmq_sent += 1 except zmq.ZMQError as e: # pragma: debug if e.errno == zmq.EAGAIN: raise AsyncComm.AsyncTryAgain("Socket not yet available.") else: self.special_debug("Socket could not send. (errno=%d)", e.errno) return False return True
def encode_json(obj, fd=None, indent=None, sort_keys=True, **kwargs): r"""Encode a Python object in JSON format. Args: obj (object): Python object to encode. fd (file, optional): File descriptor for file that encoded object should be written to. Defaults to None and string is returned. indent (int, str, optional): Indentation for new lines in encoded string. Defaults to None. sort_keys (bool, optional): If True, the keys will be output in sorted order. Defaults to True. **kwargs: Additional keyword arguments are passed to json.dumps. Returns: str, bytes: Encoded object. """ if (indent is None) and (fd is not None): indent = '\t' if backwards.PY2 or _use_rapidjson: # pragma: Python 2 # Character indents not allowed in Python 2 json indent = indent_char2int(indent) kwargs['indent'] = indent kwargs['sort_keys'] = sort_keys if fd is None: return backwards.as_bytes(json.dumps(obj, **kwargs)) else: return json.dump(obj, fd, **kwargs)
def _recv(self, timeout=0): r"""Reads message from a file. Args: timeout (float, optional): Time in seconds to wait for a message. Defaults to self.recv_timeout. Unused. Returns: tuple (bool, str): Success or failure of reading from the file and the read messages as bytes. """ flag = True try: if self.read_meth == 'read': out = self.fd.read() elif self.read_meth == 'readline': out = self.fd.readline() except BaseException: # pragma: debug # Use this to catch case where close called during receive. # In the future this should be handled via a lock. out = '' if len(out) == 0: if self.advance_in_series(): self.debug("Advanced to %d", self._series_index) flag, out = self._recv() else: out = self.eof_msg else: out = out.replace(self.platform_newline, self.newline) if not self.open_as_binary: out = backwards.as_bytes(out) return (flag, out)
def table2format(fmts=[], delimiter=None, newline=None, comment=None): r"""Create a format string from table information. Args: fmts (list, optional): List of format codes for each column. Defaults to []. delimiter (bytes, optional): String used to separate columns. Defaults to _default_delimiter. newline (bytes, optional): String used to indicate the end of a table line. Defaults to _default_newline. comment (bytes, optional): String that should be prepended to the format string to indicate a comment. Defaults to _default_comment. Returns: str, bytes: Table format string. """ if delimiter is None: delimiter = _default_delimiter if newline is None: newline = _default_newline if comment is None: comment = _default_comment if isinstance(fmts, np.dtype): fmts = nptype2cformat(fmts) bytes_fmts = [backwards.as_bytes(f) for f in fmts] fmt_str = comment + delimiter.join(bytes_fmts) + newline return fmt_str
def get_field_units(self, as_bytes=False): r"""Get the field units for an array of fields. Args: as_bytes (bool, optional): If True, the field units will be returned as bytes. If False the field units will be returned as unicode. Defaults to False. Returns: list: Units for each field in the data type. """ if self.typedef['type'] != 'array': return None if getattr(self, 'field_units', None) is not None: out = self.field_units elif isinstance(self.typedef['items'], dict): # pragma: debug raise Exception("Variable number of items not yet supported.") elif isinstance(self.typedef['items'], list): out = [] any_units = False for i, x in enumerate(self.typedef['items']): out.append(x.get('units', '')) if len(x.get('units', '')) > 0: any_units = True # Don't use field units if they are all defaults if not any_units: out = None if (out is not None): if as_bytes: out = [backwards.as_bytes(x) for x in out] else: out = [backwards.as_str(x) for x in out] return out
def on_message(self, ch, method, props, body): r"""Buffer received messages.""" if self.direction == 'send': # pragma: debug raise Exception("Send comm received a message.") with self.rmq_lock: self._buffered_messages.append(backwards.as_bytes(body)) ch.basic_ack(delivery_tag=method.delivery_tag)
def test_format_bytes(): r"""Test formating of bytes string.""" s0 = "%s, %s" ans = "one, one" arg0 = "one" args = (backwards.as_bytes(arg0), backwards.as_unicode(arg0)) for cvt in [backwards.as_bytes, backwards.as_unicode]: res = backwards.format_bytes(cvt(s0), args) assert_equal(res, cvt(ans))
def array_to_table(arrs, fmt_str, use_astropy=False): r"""Serialize an array as an ASCII table. Args: arrs (np.ndarray, list, tuple): Structured array or list/tuple of arrays that contain table information. fmt_str (str, bytes): Format string that should be used to structure the ASCII array. use_astropy (bool, optional): If True, astropy will be used to format the table if it is installed. Defaults to False. Returns: bytes: ASCII table. """ if not _use_astropy: use_astropy = False dtype = cformat2nptype(fmt_str) info = format2table(fmt_str) comment = info.get('comment', None) if comment is not None: fmt_str = fmt_str.split(comment, 1)[-1] arr1 = consolidate_array(arrs, dtype=dtype) if use_astropy: fd = backwards.StringIO() table = apy_Table(arr1) delimiter = info['delimiter'] delimiter = backwards.as_str(delimiter) apy_ascii.write(table, fd, delimiter=delimiter, format='no_header') out = backwards.as_bytes(fd.getvalue()) else: fd = backwards.BytesIO() for ele in arr1: line = format_message(ele.tolist(), backwards.as_bytes(fmt_str)) fd.write(line) # fmt = fmt_str.split(info['newline'])[0] # np.savetxt(fd, arr1, # fmt=fmt, delimiter=info['delimiter'], # newline=info['newline'], header='') out = fd.getvalue() fd.close() return out
def test_as_bytes(): r"""Test as_bytes.""" # Just strings res = b'hello' vals = ['hello', b'hello', u'hello', bytearray('hello', 'utf-8')] for v in vals: backwards.assert_bytes(res) assert_equal(backwards.as_bytes(v), res) assert_raises(TypeError, backwards.as_bytes, 1) # Recursive valpha = 'abcdef' ralpha = [backwards.as_bytes(a) for a in valpha] vals_rec = [vals, tuple(vals), {k: v for k, v in zip(valpha, vals)}] resl = [res for v in vals] resl_rec = [resl, tuple(resl), {k: v for k, v in zip(ralpha, resl)}] for v, r in zip(vals_rec, resl_rec): assert_equal(backwards.as_bytes(v, recurse=True), r) # Integer assert_equal(backwards.as_bytes(int(1), convert_types=(int, )), b'1') assert_equal(backwards.as_bytes(int(1), allow_pass=True), int(1))
def test_cformat2pyscanf(): r"""Test conversion of C format string to version for python scanf.""" all_a = [] all_b = [] for a, b in map_cformat2pyscanf: if isinstance(a, str): a = [a] for _ia in a: ia = backwards.as_bytes(_ia) ib = backwards.as_bytes(b) all_a.append(ia) all_b.append(ib) assert_equal(serialize.cformat2pyscanf(ia), ib) assert_raises(TypeError, serialize.cformat2pyscanf, 0) assert_raises(ValueError, serialize.cformat2pyscanf, b's') assert_raises(ValueError, serialize.cformat2pyscanf, b'%') fmt_a = b'\t'.join(all_a) fmt_b = b'\t'.join(all_b) assert_equal(serialize.cformat2pyscanf(all_a), all_b) assert_equal(serialize.cformat2pyscanf(fmt_a), fmt_b)
def func_serialize(self, args): r"""Serialize a message. Args: args (obj): Python object to be serialized. Returns: bytes, str: Serialized message. """ out = backwards.pickle.dumps(args) return backwards.as_bytes(out)
def func_serialize(self, args): r"""Serialize a message. Args: args: List of arguments to be formatted or numpy array to be serialized. Returns: bytes, str: Serialized message. """ return backwards.as_bytes(self.datatype.encode_data(args, self.typedef))
def test_cformat2nptype(): r"""Test conversion from C format string to numpy dtype.""" for a, b in map_cformat2nptype: if isinstance(a, str): a = [a] for _ia in a: if _ia.startswith(backwards.as_str(serialize._fmt_char)): ia = backwards.as_bytes(_ia) else: ia = serialize._fmt_char + backwards.as_bytes(_ia) assert_equal(serialize.cformat2nptype(ia), np.dtype(b)) # .str) # assert_equal(serialize.cformat2nptype(ia), np.dtype(b).str) assert_raises(TypeError, serialize.cformat2nptype, 0) assert_raises(ValueError, serialize.cformat2nptype, b's') assert_raises(ValueError, serialize.cformat2nptype, b'%') assert_raises(ValueError, serialize.cformat2nptype, '%d\t%f', names=['one']) for a in unsupported_nptype: assert_raises(ValueError, serialize.cformat2nptype, backwards.as_bytes('%' + a))
def test_AsciiTableComm_nofmt(): r"""Test read of asciitable without format.""" test_file = os.path.join(os.getcwd(), 'temp_file.txt') rows = [('one', 1, 1.0), ('two', 2, 2.0), ('three', 3, 3.0)] lines = [backwards.format_bytes('%5s\t%d\t%f\n', r) for r in rows] contents = backwards.as_bytes(''.join(lines)) with open(test_file, 'wb') as fd: fd.write(contents) inst = AsciiTableComm.AsciiTableComm('test', test_file, direction='recv') inst.open() for ans in rows: flag, x = inst.recv_dict() assert (flag) irow = [e for e in ans] irow[0] = backwards.as_bytes(irow[0]) idict = {'f%d' % i: irow[i] for i in range(len(irow))} # irow = tuple(irow) assert_equal(x, idict) flag, x = inst.recv() assert (not flag) inst.close() os.remove(test_file)
def test_parse_header(): r"""Test parsing header.""" header = [ "# name\tnumber\tvalue\tcomplex\n", "# n/a\tg\tcm\tn/a\n", "# %5s\t%ld\t%lf\t%g%+gj\n" ] res = dict(delimiter='\t', comment='# ', newline='\n', format_str="%5s\t%ld\t%lf\t%g%+gj\n", fmts=['%5s', '%ld', '%lf', '%g%+gj'], field_names=['name', 'number', 'value', 'complex'], field_units=['n/a', 'g', 'cm', 'n/a']) for i in range(len(header)): header[i] = backwards.as_bytes(header[i]) for k, v in res.items(): if isinstance(v, str): res[k] = backwards.as_bytes(v) elif isinstance(v, list): res[k] = [backwards.as_bytes(s) for s in v] assert_equal(serialize.parse_header(header), res) assert_equal(serialize.parse_header(header[::-1]), res) _empty = b'' assert_equal(serialize.parse_header(_empty.join(header)), res) # Test without formats header2 = header[:2] res2 = dict(**res) del res2['format_str'] res2['fmts'] = [] assert_equal(serialize.parse_header(header2), res2) # Test with explicit line numbers assert_equal( serialize.parse_header(header, lineno_names=0, lineno_units=1), res) # Test errors header3 = [header[0], header[0]] assert_raises(RuntimeError, serialize.parse_header, header3) header4 = [header[1], header[1]] assert_raises(RuntimeError, serialize.parse_header, header4)
def get_testing_options(cls, read_meth=None, **kwargs): r"""Method to return a dictionary of testing options for this class. Args: read_meth (str, optional): Read method that will be used by the test class. Defaults to None and will be set by the serialier. **kwargs: Additional keyword arguments are passed to the parent class's method and the serializers methods for determining the default read_meth and concatenating the sent objects into the objects that are expected to be received. Returns: dict: Dictionary of variables to use for testing. Key/value pairs: kwargs (dict): Keyword arguments for comms tested with the provided content. send (list): List of objects to send to test file. recv (list): List of objects that will be received from a test file that was sent the messages in 'send'. contents (bytes): Bytes contents of test file created by sending the messages in 'send'. """ out = super(FileComm, cls).get_testing_options(**kwargs) if 'read_meth' in cls._schema_properties: if read_meth is None: read_meth = cls._schema_properties['read_meth']['default'] out['kwargs']['read_meth'] = read_meth if read_meth == 'readline': out['recv_partial'] = [[x] for x in out['recv']] if cls._default_serializer == 'direct': comment = backwards.as_bytes( cls._schema_properties['comment']['default'] + 'Comment\n') out['send'].append(comment) out['contents'] += comment out['recv_partial'].append([]) else: seri_cls = cls._default_serializer_class if seri_cls.concats_as_str: out['recv_partial'] = [[x] for x in out['recv']] out['recv'] = seri_cls.concatenate(out['recv'], **out['kwargs']) else: out['recv_partial'] = [[out['recv'][0]]] for i in range(1, len(out['recv'])): out['recv_partial'].append( seri_cls.concatenate( out['recv_partial'][-1] + [out['recv'][i]], **out['kwargs'])) out['recv'] = copy.deepcopy(out['recv_partial'][-1]) return out
def func_serialize(self, args): r"""Serialize a message. Args: args: List of arguments to be formatted or numpy array to be serialized. Returns: bytes, str: Serialized message. """ if (((self.extra_kwargs.get('format_str', None) is not None) and isinstance(args, list))): args = format_message(args, self.extra_kwargs['format_str']) return backwards.as_bytes(args)
def test_field_specs(self): r"""Test field specifiers.""" super(TestAsciiTableSerialize, self).test_field_specs() # Specific to this class self.assert_equal( self.instance.format_str, backwards.as_bytes(self.testing_options['kwargs']['format_str'])) field_names = self.testing_options['kwargs'].get('field_names', None) if field_names is not None: field_names = [backwards.as_str(x) for x in field_names] self.assert_equal(self.instance.field_names, field_names) field_units = self.testing_options['kwargs'].get('field_units', None) if field_units is not None: field_units = [backwards.as_str(x) for x in field_units] self.assert_equal(self.instance.field_units, field_units)
def _recv(self, timeout=0): r"""Reads message from a file. Args: timeout (float, optional): Time in seconds to wait for a message. Defaults to self.recv_timeout. Unused. Returns: tuple (bool, str): Success or failure of reading from the file and the read messages as bytes. """ flag, msg = super(AsciiFileComm, self)._recv() if self.read_meth == 'readline': while flag and msg.startswith(backwards.as_bytes(self.comment)): flag, msg = super(AsciiFileComm, self)._recv() return flag, msg
def test_nptype2cformat_structured(): r"""Test conversion from structured numpy dtype to C format string.""" if platform._is_win: # pragma: windows fmts = ["%5s", "%l64d", "%g", "%g%+gj"] else: fmts = ["%5s", "%ld", "%g", "%g%+gj"] names0 = ['f0', 'f1', 'f2', 'f3'] names1 = ["name", "number", "value", "complex"] dtypes = ['S5', 'i8', 'f8', 'c16'] dtype0 = np.dtype([(n, f) for n, f in zip(names0, dtypes)]) dtype1 = np.dtype([(n, f) for n, f in zip(names1, dtypes)]) alist = [dtype0, dtype1] blist = [fmts, fmts] for a, b in zip(alist, blist): assert_equal(serialize.nptype2cformat(a), b) assert_equal(serialize.nptype2cformat(a, asbytes=True), [backwards.as_bytes(ib) for ib in b])
def func_serialize(self, args): r"""Serialize a message. Args: args (obj): Python object to be serialized. Returns: bytes, str: Serialized message. """ if not isinstance(args, pandas.DataFrame): raise TypeError(("Pandas DataFrame required. Invalid type " + "of '%s' provided.") % type(args)) fd = backwards.StringIO() if backwards.PY2: # pragma: Python 2 args_ = args else: # pragma: Python 3 # For Python 3 and higher, bytes need to be encoded args_ = copy.deepcopy(args) for c in args.columns: if isinstance(args_[c][0], backwards.bytes_type): args_[c] = args_[c].apply(lambda s: s.decode('utf-8')) if self.field_names is None: self.field_names = self.get_field_names() args_ = self.apply_field_names(args_, self.field_names) args_.to_csv( fd, index=False, # Not in pandas <0.24 # line_terminator=backwards.as_str(self.newline), sep=backwards.as_str(self.delimiter), mode='wb', encoding='utf8', header=(not self.dont_write_header)) if self.write_header_once: self.dont_write_header = True out = fd.getvalue() fd.close() # Required to change out \r\n for \n on windows out = out.replace(backwards.match_stype(out, platform._newline), backwards.match_stype(out, self.newline)) return backwards.as_bytes(out)
def print_encoded(msg, *args, **kwargs): r"""Print bytes to stdout, encoding if possible. Args: msg (str, bytes): Message to print. *args: Additional arguments are passed to print. **kwargs: Additional keyword arguments are passed to print. """ try: print(backwards.as_unicode(msg), *args, **kwargs) except (UnicodeEncodeError, UnicodeDecodeError): # pragma: debug logging.debug("sys.stdout.encoding = %s, cannot print unicode", sys.stdout.encoding) kwargs.pop('end', None) try: print(msg, *args, **kwargs) except UnicodeEncodeError: # pragma: debug print(backwards.as_bytes(msg), *args, **kwargs)
def language2python(cls, robj): r"""Prepare an R object for serialization in Python. Args: robj (object): Python object prepared in R. Returns: object: Python object in a form that is serialization friendly. """ logger.debug("language2python: %s, %s" % (robj, type(robj))) if isinstance(robj, tuple): return tuple([cls.language2python(x) for x in robj]) elif isinstance(robj, list): return [cls.language2python(x) for x in robj] elif isinstance(robj, dict): return {k: cls.language2python(v) for k, v in robj.items()} elif isinstance(robj, backwards.string_types): return backwards.as_bytes(robj) return robj
def update_serializer(self, *args, **kwargs): # Transform scalar into array for table if kwargs.get('type', 'array') != 'array': old_typedef = {} _metaschema = get_metaschema() for k in _metaschema['properties'].keys(): if k in kwargs: old_typedef[k] = kwargs.pop(k) new_typedef = {'type': 'array', 'items': [old_typedef]} kwargs.update(new_typedef) out = super(AsciiTableSerialize, self).update_serializer(*args, **kwargs) for k in ['format_str', 'delimiter', 'newline', 'comment']: v = getattr(self, k, None) if isinstance(v, backwards.string_types): setattr(self, k, backwards.as_bytes(v)) self.update_format_str() self.update_field_names() self.update_field_units() return out
def func_serialize(self, args): r"""Serialize a message. Args: args: List of arguments to be formatted or numpy array to be serialized. Returns: bytes, str: Serialized message. """ if self.format_str is None: raise RuntimeError("Format string is not defined.") args = self.datatype.coerce_type(args, key_order=self.get_field_names()) if self.as_array: out = serialize.array_to_table(args, self.format_str, use_astropy=self.use_astropy) else: out = serialize.format_message(args, self.format_str) return backwards.as_bytes(out)