def from_descriptor_string(self, s): """Set the attributes of this descriptor from a string generated by to_descriptor_string().""" for heap in iterheaps(TransportString(s)): items = heap.get_items() self.id = _spead.unpack(ID_FMT, items[ID_ID])[0][-1] shape = _spead.unpack(SHAPE_FMT, items[SHAPE_ID], cnt=-1) # Check if we have a dynamically sized value try: if shape[0][0] == 2: self.shape = -1 else: self.shape = [s[1] for s in shape] except(IndexError, TypeError): self.shape = [] self.format = items[FORMAT_ID] self.dtype_str = None self.dtype = None self.fortran_order = False if DTYPE_ID in items.keys(): self.dtype_str = ''.join(f[0] for f in items[DTYPE_ID]) self.shape, self.fortran_order, self.dtype = self._dtype_unpack(self.dtype_str) self.size = int(numpy.product(self.shape)) else: self._calcsize() self.name = ''.join([f[0] for f in items[NAME_ID]]) self.description = ''.join([f[0] for f in items[DESCRIPTION_ID]])
def iter_genpackets(heap, max_pkt_size=_spead.MAX_PACKET_LEN): """Provided a heap (dictionary of IDs and binary string values), iterate over the set of binary SPEAD packets that propagate this data to a receiver. The stream will be broken into packets of the specified maximum size.""" assert(_spead.HEAP_CNT_ID in heap.keys()) # Every heap has to have a HEAP_CNT pkt = _spead.SpeadPacket() descriptors = heap.pop(_spead.DESCRIPTOR_ID, []) items, heap_pyld, offset = [], [], 0 logger.info('itergenpackets: Converting a heap into packets') # Add descriptors for d in descriptors: if DEBUG: logger.debug('itergenpackets: Adding a descriptor to header') dlen = len(d) items.append((_spead.DIRECTADDR, _spead.DESCRIPTOR_ID, offset)) heap_pyld.append(d) offset += dlen # Add other items for id, (mode, val) in heap.iteritems(): vlen = len(val) if mode == _spead.DIRECTADDR: if DEBUG: logger.debug('itergenpackets: Adding extension item to header, id=%d, len(val)=%d' % (id, len(val))) items.append((_spead.DIRECTADDR, id, offset)) heap_pyld.append(val) offset += vlen # Pad out to _spead.ADDRLEN for bits < _spead.ADDRSIZE else: if DEBUG: logger.debug('itergenpackets: Adding standard item to header, id=%d, len(val)=%d' % (id, len(val))) items.append((_spead.IMMEDIATEADDR, id, _spead.unpack(DEFAULT_FMT, (ADDRNULL+val)[-_spead.ADDRLEN:])[0][0])) heap_pyld = ''.join(heap_pyld) heap_len, payload_cnt, offset = len(heap_pyld), 0, 0 while True: # The first packet contains all of the header entries for the items that changed # Subsequent packets are continuations that increment payload_cnt until all data is sent # XXX Need to check that # of changed items fits in _spead.MAX_PACKET_LEN. if payload_cnt == 0: h = items else: h = [(_spead.IMMEDIATEADDR, _spead.HEAP_CNT_ID, _spead.unpack(DEFAULT_FMT, heap[_spead.HEAP_CNT_ID][1])[0][0])] hlen = _spead.ITEMLEN * (len(h) + 4) # 4 for the spead hdr, heap_len, payload_len and payload_off payload_len = min(_spead.MAX_PACKET_LEN - hlen, heap_len - offset) h.append((_spead.IMMEDIATEADDR, _spead.HEAP_LEN_ID, heap_len)) h.append((_spead.IMMEDIATEADDR, _spead.PAYLOAD_LEN_ID, payload_len)) h.append((_spead.IMMEDIATEADDR, _spead.PAYLOAD_OFF_ID, offset)) pkt.items = h pkt.payload = heap_pyld[offset:offset+payload_len] if DEBUG: logger.debug('itergenpackets: Made packet with hlen=%d, payoff=%d, paylen=%d' % (len(h), offset, payload_len)) yield pkt.pack() offset += payload_len payload_cnt += 1 if offset >= heap_len: break logger.info('itergenpackets: Done converting a heap into packets') return
def readable_heap(heap, prepend=''): rv = ['', 'vvv HEAP ' + 'v'*(50-len(prepend))] for id in heap: try: name = NAME[id] if name == 'DESCRIPTOR': for cnt, d in enumerate(heap[id]): rv += readable_binpacket(d, prepend='DESCRIPTOR%d:' % cnt).split('\n') else: fmt, cnt = ITEM[name]['FMT'], ITEM[name]['CNT'] val = _spead.unpack(fmt, heap[id][1], cnt=cnt) try: while not isinstance(val, str) and len(val) == 1: val = val[0] except TypeError: pass s = '%16s: %s' % (name, val) if len(s) > (70 - len(prepend)): s = s[:(70-len(prepend)-3)]+'...' rv.append(s) except KeyError: s = '%16d: %s' % (id, '0x' + hexify(heap[id][1])) if len(s) > (70 - len(prepend)): s = s[:(70-len(prepend)-3)] + '...' rv.append(s) rv.append('^'*(60-len(prepend))) return '\n'.join([prepend+r for r in rv])
def unpack(self, s): """Convert a binary string into a value based on the format and shape of this Descriptor.""" logger.debug("Using traditional unpack") try: val = _spead.unpack(self.format, s[self._offset/8:], cnt=self.size, offset=self._offset % 8) except ValueError, e: raise ValueError(''.join(e.args) + ': ' 'Could not unpack %s: fmt=%s, size=%d, _offset=%d, but length of binary' ' string was %d' % (self.name, parsefmt(self.format), self.size, self._offset, len(s)))
def readable_header(h, prepend=''): rv = _spead.unpack(RAW_ITEM_FMT, h)[0] mode, id = rv[:2] raw_val = ''.join(rv[2:]) if mode == _spead.DIRECTADDR: val = 'OFF=%s' % (hexify(raw_val)) else: val = 'VAL=%s' % (hexify(raw_val)) try: return prepend + '[ MODE=%d | NAME=%16s | %s ]' % (mode, NAME[id], val) except KeyError: return prepend + '[ MODE=%d | ID=%16x | %s ]' % (mode, id, val)
def readable_header(h, prepend=''): rv = _spead.unpack(RAW_ITEM_FMT, h)[0] mode, id = rv[:2] raw_val = ''.join(rv[2:]) if mode == _spead.DIRECTADDR: val = 'OFF=%s' % (hexify(raw_val)) else: val = 'VAL=%s' % (hexify(raw_val)) try: return prepend+'[ MODE=%d | NAME=%16s | %s ]' % (mode, NAME[id], val) except KeyError: return prepend+'[ MODE=%d | ID=%16x | %s ]' % (mode, id, val)
def readable_binpacket(pkt, prepend='', show_payload=False): o, rv = 0, ['', 'vvv PACKET ' + 'v'*(50-len(prepend))] magic, version, itemsize, addrsize, junk, n_options = _spead.unpack(HDR_FMT, pkt[o:o+_spead.ITEMLEN])[0] o += _spead.ITEMLEN rv.append(' HEADER:[ SPEAD-CODE=%06x | VERSION=%d | N_OPTIONS=%d ]' % (magic, version, n_options)) for cnt in range(n_options): rv.append(readable_header(pkt[o:o+_spead.ITEMLEN], prepend='ITEM%02d:' % cnt)) o += _spead.ITEMLEN rv.append('PAYLOAD: BYTES=%d' % (len(pkt[o:]))) if show_payload: rv.append('PAYLOAD: ' + readable_payload(pkt[o:])) rv.append('^'*(60-len(prepend))) return '\n'.join([prepend+r for r in rv])
def unpack(self, s): """Convert a binary string into a value based on the format and shape of this Descriptor.""" logger.debug("Using traditional unpack") try: val = _spead.unpack(self.format, s[self._offset / 8:], cnt=self.size, offset=self._offset % 8) except ValueError, e: raise ValueError( ''.join(e.args) + ': ' 'Could not unpack %s: fmt=%s, size=%d, _offset=%d, but length of binary' ' string was %d' % (self.name, parsefmt(self.format), self.size, self._offset, len(s)))
def readable_binpacket(pkt, prepend='', show_payload=False): o, rv = 0, ['', 'vvv PACKET ' + 'v' * (50 - len(prepend))] magic, version, itemsize, addrsize, junk, n_options = _spead.unpack( HDR_FMT, pkt[o:o + _spead.ITEMLEN])[0] o += _spead.ITEMLEN rv.append(' HEADER:[ SPEAD-CODE=%06x | VERSION=%d | N_OPTIONS=%d ]' % (magic, version, n_options)) for cnt in range(n_options): rv.append( readable_header(pkt[o:o + _spead.ITEMLEN], prepend='ITEM%02d:' % cnt)) o += _spead.ITEMLEN rv.append('PAYLOAD: BYTES=%d' % (len(pkt[o:]))) if show_payload: rv.append('PAYLOAD: ' + readable_payload(pkt[o:])) rv.append('^' * (60 - len(prepend))) return '\n'.join([prepend + r for r in rv])
def calcsize(fmt): return sum([f[1] for f in _spead.unpack(FORMAT_FMT, fmt, cnt=-1)])
def parsefmt(fmt): return _spead.unpack(FORMAT_FMT, fmt, cnt=-1)
def iter_genpackets(heap, max_pkt_size=_spead.MAX_PACKET_LEN): """Provided a heap (dictionary of IDs and binary string values), iterate over the set of binary SPEAD packets that propagate this data to a receiver. The stream will be broken into packets of the specified maximum size.""" assert (_spead.HEAP_CNT_ID in heap.keys()) # Every heap has to have a HEAP_CNT pkt = _spead.SpeadPacket() descriptors = heap.pop(_spead.DESCRIPTOR_ID, []) items, heap_pyld, offset = [], [], 0 logger.info('itergenpackets: Converting a heap into packets') # Add descriptors for d in descriptors: if DEBUG: logger.debug('itergenpackets: Adding a descriptor to header') dlen = len(d) items.append((_spead.DIRECTADDR, _spead.DESCRIPTOR_ID, offset)) heap_pyld.append(d) offset += dlen # Add other items for id, (mode, val) in heap.iteritems(): vlen = len(val) if mode == _spead.DIRECTADDR: if DEBUG: logger.debug( 'itergenpackets: Adding extension item to header, id=%d, len(val)=%d' % (id, len(val))) items.append((_spead.DIRECTADDR, id, offset)) heap_pyld.append(val) offset += vlen # Pad out to _spead.ADDRLEN for bits < _spead.ADDRSIZE else: if DEBUG: logger.debug( 'itergenpackets: Adding standard item to header, id=%d, len(val)=%d' % (id, len(val))) items.append( (_spead.IMMEDIATEADDR, id, _spead.unpack(DEFAULT_FMT, (ADDRNULL + val)[-_spead.ADDRLEN:])[0][0])) heap_pyld = ''.join(heap_pyld) heap_len, payload_cnt, offset = len(heap_pyld), 0, 0 while True: # The first packet contains all of the header entries for the items that changed # Subsequent packets are continuations that increment payload_cnt until all data is sent # XXX Need to check that # of changed items fits in _spead.MAX_PACKET_LEN. if payload_cnt == 0: h = items else: h = [(_spead.IMMEDIATEADDR, _spead.HEAP_CNT_ID, _spead.unpack(DEFAULT_FMT, heap[_spead.HEAP_CNT_ID][1])[0][0])] hlen = _spead.ITEMLEN * ( len(h) + 4 ) # 4 for the spead hdr, heap_len, payload_len and payload_off payload_len = min(_spead.MAX_PACKET_LEN - hlen, heap_len - offset) h.append((_spead.IMMEDIATEADDR, _spead.HEAP_LEN_ID, heap_len)) h.append((_spead.IMMEDIATEADDR, _spead.PAYLOAD_LEN_ID, payload_len)) h.append((_spead.IMMEDIATEADDR, _spead.PAYLOAD_OFF_ID, offset)) pkt.items = h pkt.payload = heap_pyld[offset:offset + payload_len] if DEBUG: logger.debug( 'itergenpackets: Made packet with hlen=%d, payoff=%d, paylen=%d' % (len(h), offset, payload_len)) yield pkt.pack() offset += payload_len payload_cnt += 1 if offset >= heap_len: break logger.info('itergenpackets: Done converting a heap into packets') return