def from_descriptor_string(self, s):
"""Set the attributes of this descriptor from a string generated by to_descriptor_string()."""
for heap in iterheaps(TransportString(s)):
items = heap.get_items()
self.id = _spead.unpack(ID_FMT, items[ID_ID])[0][-1]
shape = _spead.unpack(SHAPE_FMT, items[SHAPE_ID], cnt=-1)
# Check if we have a dynamically sized value
try:
if shape[0][0] == 2:
self.shape = -1
else:
self.shape = [s[1] for s in shape]
except(IndexError, TypeError):
self.shape = []
self.format = items[FORMAT_ID]
self.dtype_str = None
self.dtype = None
self.fortran_order = False
if DTYPE_ID in items.keys():
self.dtype_str = ''.join(f[0] for f in items[DTYPE_ID])
self.shape, self.fortran_order, self.dtype = self._dtype_unpack(self.dtype_str)
self.size = int(numpy.product(self.shape))
else:
self._calcsize()
self.name = ''.join([f[0] for f in items[NAME_ID]])
self.description = ''.join([f[0] for f in items[DESCRIPTION_ID]])
def iter_genpackets(heap, max_pkt_size=_spead.MAX_PACKET_LEN):
"""Provided a heap (dictionary of IDs and binary string values),
iterate over the set of binary SPEAD packets that propagate this data
to a receiver. The stream will be broken into packets of the specified maximum size."""
assert(_spead.HEAP_CNT_ID in heap.keys()) # Every heap has to have a HEAP_CNT
pkt = _spead.SpeadPacket()
descriptors = heap.pop(_spead.DESCRIPTOR_ID, [])
items, heap_pyld, offset = [], [], 0
logger.info('itergenpackets: Converting a heap into packets')
# Add descriptors
for d in descriptors:
if DEBUG:
logger.debug('itergenpackets: Adding a descriptor to header')
dlen = len(d)
items.append((_spead.DIRECTADDR, _spead.DESCRIPTOR_ID, offset))
heap_pyld.append(d)
offset += dlen
# Add other items
for id, (mode, val) in heap.iteritems():
vlen = len(val)
if mode == _spead.DIRECTADDR:
if DEBUG:
logger.debug('itergenpackets: Adding extension item to header, id=%d, len(val)=%d' % (id, len(val)))
items.append((_spead.DIRECTADDR, id, offset))
heap_pyld.append(val)
offset += vlen
# Pad out to _spead.ADDRLEN for bits < _spead.ADDRSIZE
else:
if DEBUG:
logger.debug('itergenpackets: Adding standard item to header, id=%d, len(val)=%d' % (id, len(val)))
items.append((_spead.IMMEDIATEADDR, id,
_spead.unpack(DEFAULT_FMT, (ADDRNULL+val)[-_spead.ADDRLEN:])[0][0]))
heap_pyld = ''.join(heap_pyld)
heap_len, payload_cnt, offset = len(heap_pyld), 0, 0
while True:
# The first packet contains all of the header entries for the items that changed
# Subsequent packets are continuations that increment payload_cnt until all data is sent
# XXX Need to check that # of changed items fits in _spead.MAX_PACKET_LEN.
if payload_cnt == 0:
h = items
else:
h = [(_spead.IMMEDIATEADDR, _spead.HEAP_CNT_ID,
_spead.unpack(DEFAULT_FMT, heap[_spead.HEAP_CNT_ID][1])[0][0])]
hlen = _spead.ITEMLEN * (len(h) + 4) # 4 for the spead hdr, heap_len, payload_len and payload_off
payload_len = min(_spead.MAX_PACKET_LEN - hlen, heap_len - offset)
h.append((_spead.IMMEDIATEADDR, _spead.HEAP_LEN_ID, heap_len))
h.append((_spead.IMMEDIATEADDR, _spead.PAYLOAD_LEN_ID, payload_len))
h.append((_spead.IMMEDIATEADDR, _spead.PAYLOAD_OFF_ID, offset))
pkt.items = h
pkt.payload = heap_pyld[offset:offset+payload_len]
if DEBUG:
logger.debug('itergenpackets: Made packet with hlen=%d, payoff=%d, paylen=%d' %
(len(h), offset, payload_len))
yield pkt.pack()
offset += payload_len
payload_cnt += 1
if offset >= heap_len:
break
logger.info('itergenpackets: Done converting a heap into packets')
return
def readable_heap(heap, prepend=''):
rv = ['', 'vvv HEAP ' + 'v'*(50-len(prepend))]
for id in heap:
try:
name = NAME[id]
if name == 'DESCRIPTOR':
for cnt, d in enumerate(heap[id]):
rv += readable_binpacket(d, prepend='DESCRIPTOR%d:' % cnt).split('\n')
else:
fmt, cnt = ITEM[name]['FMT'], ITEM[name]['CNT']
val = _spead.unpack(fmt, heap[id][1], cnt=cnt)
try:
while not isinstance(val, str) and len(val) == 1:
val = val[0]
except TypeError:
pass
s = '%16s: %s' % (name, val)
if len(s) > (70 - len(prepend)):
s = s[:(70-len(prepend)-3)]+'...'
rv.append(s)
except KeyError:
s = '%16d: %s' % (id, '0x' + hexify(heap[id][1]))
if len(s) > (70 - len(prepend)):
s = s[:(70-len(prepend)-3)] + '...'
rv.append(s)
rv.append('^'*(60-len(prepend)))
return '\n'.join([prepend+r for r in rv])
def unpack(self, s):
"""Convert a binary string into a value based on the format and shape of this Descriptor."""
logger.debug("Using traditional unpack")
try:
val = _spead.unpack(self.format, s[self._offset/8:], cnt=self.size, offset=self._offset % 8)
except ValueError, e:
raise ValueError(''.join(e.args) + ': '
'Could not unpack %s: fmt=%s, size=%d, _offset=%d, but length of binary'
' string was %d' % (self.name, parsefmt(self.format),
self.size, self._offset, len(s)))
def readable_header(h, prepend=''):
rv = _spead.unpack(RAW_ITEM_FMT, h)[0]
mode, id = rv[:2]
raw_val = ''.join(rv[2:])
if mode == _spead.DIRECTADDR:
val = 'OFF=%s' % (hexify(raw_val))
else:
val = 'VAL=%s' % (hexify(raw_val))
try:
return prepend + '[ MODE=%d | NAME=%16s | %s ]' % (mode, NAME[id], val)
except KeyError:
return prepend + '[ MODE=%d | ID=%16x | %s ]' % (mode, id, val)
def readable_header(h, prepend=''):
rv = _spead.unpack(RAW_ITEM_FMT, h)[0]
mode, id = rv[:2]
raw_val = ''.join(rv[2:])
if mode == _spead.DIRECTADDR:
val = 'OFF=%s' % (hexify(raw_val))
else:
val = 'VAL=%s' % (hexify(raw_val))
try:
return prepend+'[ MODE=%d | NAME=%16s | %s ]' % (mode, NAME[id], val)
except KeyError:
return prepend+'[ MODE=%d | ID=%16x | %s ]' % (mode, id, val)
def readable_binpacket(pkt, prepend='', show_payload=False):
o, rv = 0, ['', 'vvv PACKET ' + 'v'*(50-len(prepend))]
magic, version, itemsize, addrsize, junk, n_options = _spead.unpack(HDR_FMT, pkt[o:o+_spead.ITEMLEN])[0]
o += _spead.ITEMLEN
rv.append(' HEADER:[ SPEAD-CODE=%06x | VERSION=%d | N_OPTIONS=%d ]' % (magic, version, n_options))
for cnt in range(n_options):
rv.append(readable_header(pkt[o:o+_spead.ITEMLEN], prepend='ITEM%02d:' % cnt))
o += _spead.ITEMLEN
rv.append('PAYLOAD: BYTES=%d' % (len(pkt[o:])))
if show_payload:
rv.append('PAYLOAD: ' + readable_payload(pkt[o:]))
rv.append('^'*(60-len(prepend)))
return '\n'.join([prepend+r for r in rv])
def unpack(self, s):
"""Convert a binary string into a value based on the format and shape of this Descriptor."""
logger.debug("Using traditional unpack")
try:
val = _spead.unpack(self.format,
s[self._offset / 8:],
cnt=self.size,
offset=self._offset % 8)
except ValueError, e:
raise ValueError(
''.join(e.args) + ': '
'Could not unpack %s: fmt=%s, size=%d, _offset=%d, but length of binary'
' string was %d' % (self.name, parsefmt(self.format),
self.size, self._offset, len(s)))
def readable_binpacket(pkt, prepend='', show_payload=False):
o, rv = 0, ['', 'vvv PACKET ' + 'v' * (50 - len(prepend))]
magic, version, itemsize, addrsize, junk, n_options = _spead.unpack(
HDR_FMT, pkt[o:o + _spead.ITEMLEN])[0]
o += _spead.ITEMLEN
rv.append(' HEADER:[ SPEAD-CODE=%06x | VERSION=%d | N_OPTIONS=%d ]' %
(magic, version, n_options))
for cnt in range(n_options):
rv.append(
readable_header(pkt[o:o + _spead.ITEMLEN],
prepend='ITEM%02d:' % cnt))
o += _spead.ITEMLEN
rv.append('PAYLOAD: BYTES=%d' % (len(pkt[o:])))
if show_payload:
rv.append('PAYLOAD: ' + readable_payload(pkt[o:]))
rv.append('^' * (60 - len(prepend)))
return '\n'.join([prepend + r for r in rv])
def calcsize(fmt):
return sum([f[1] for f in _spead.unpack(FORMAT_FMT, fmt, cnt=-1)])
def parsefmt(fmt):
return _spead.unpack(FORMAT_FMT, fmt, cnt=-1)
def iter_genpackets(heap, max_pkt_size=_spead.MAX_PACKET_LEN):
"""Provided a heap (dictionary of IDs and binary string values),
iterate over the set of binary SPEAD packets that propagate this data
to a receiver. The stream will be broken into packets of the specified maximum size."""
assert (_spead.HEAP_CNT_ID
in heap.keys()) # Every heap has to have a HEAP_CNT
pkt = _spead.SpeadPacket()
descriptors = heap.pop(_spead.DESCRIPTOR_ID, [])
items, heap_pyld, offset = [], [], 0
logger.info('itergenpackets: Converting a heap into packets')
# Add descriptors
for d in descriptors:
if DEBUG:
logger.debug('itergenpackets: Adding a descriptor to header')
dlen = len(d)
items.append((_spead.DIRECTADDR, _spead.DESCRIPTOR_ID, offset))
heap_pyld.append(d)
offset += dlen
# Add other items
for id, (mode, val) in heap.iteritems():
vlen = len(val)
if mode == _spead.DIRECTADDR:
if DEBUG:
logger.debug(
'itergenpackets: Adding extension item to header, id=%d, len(val)=%d'
% (id, len(val)))
items.append((_spead.DIRECTADDR, id, offset))
heap_pyld.append(val)
offset += vlen
# Pad out to _spead.ADDRLEN for bits < _spead.ADDRSIZE
else:
if DEBUG:
logger.debug(
'itergenpackets: Adding standard item to header, id=%d, len(val)=%d'
% (id, len(val)))
items.append(
(_spead.IMMEDIATEADDR, id,
_spead.unpack(DEFAULT_FMT,
(ADDRNULL + val)[-_spead.ADDRLEN:])[0][0]))
heap_pyld = ''.join(heap_pyld)
heap_len, payload_cnt, offset = len(heap_pyld), 0, 0
while True:
# The first packet contains all of the header entries for the items that changed
# Subsequent packets are continuations that increment payload_cnt until all data is sent
# XXX Need to check that # of changed items fits in _spead.MAX_PACKET_LEN.
if payload_cnt == 0:
h = items
else:
h = [(_spead.IMMEDIATEADDR, _spead.HEAP_CNT_ID,
_spead.unpack(DEFAULT_FMT,
heap[_spead.HEAP_CNT_ID][1])[0][0])]
hlen = _spead.ITEMLEN * (
len(h) + 4
) # 4 for the spead hdr, heap_len, payload_len and payload_off
payload_len = min(_spead.MAX_PACKET_LEN - hlen, heap_len - offset)
h.append((_spead.IMMEDIATEADDR, _spead.HEAP_LEN_ID, heap_len))
h.append((_spead.IMMEDIATEADDR, _spead.PAYLOAD_LEN_ID, payload_len))
h.append((_spead.IMMEDIATEADDR, _spead.PAYLOAD_OFF_ID, offset))
pkt.items = h
pkt.payload = heap_pyld[offset:offset + payload_len]
if DEBUG:
logger.debug(
'itergenpackets: Made packet with hlen=%d, payoff=%d, paylen=%d'
% (len(h), offset, payload_len))
yield pkt.pack()
offset += payload_len
payload_cnt += 1
if offset >= heap_len:
break
logger.info('itergenpackets: Done converting a heap into packets')
return
def parsefmt(fmt):
return _spead.unpack(FORMAT_FMT, fmt, cnt=-1)
def calcsize(fmt):
return sum([f[1] for f in _spead.unpack(FORMAT_FMT, fmt, cnt=-1)])
