class Node(model.Model):
"""
Stores the lightweight data associated with a node, and an index of heavy
data (which has to be downloaded separately). This includes:
- ``id``: self-descriptive. The ID of a node cannot be changed.
- ``parent``: the node parent's ID.
- ``pos_start`` and ``pos_end``: for nodes where it makes sense (mostly
chunka), the range of positions occupied by this object. ``pos_start``
is inclusive, while ``pos_end`` is exclusive.
- ``tags``: an unordered set of tags, which are simple strings.
- ``attr``: a string-keyed dict of attributes with arbitrary data.
- ``data``: an unordered set of keys with non-null heavyweight data.
The data has to be downloaded separately.
- ``bindata``: a dict with an index of available bindata. The keys in
this dict correspond to bindata keys, and the values are size of the
corresponding binary data in bytes. The actual bindata has to
be downloaded separately.
- ``triggers``: a dict containing active triggers for the given node.
The keys are trigger names, and they are mapped to the state of
the given trigger.
"""
id = fields.NodeID()
parent = fields.NodeID(default=NodeID.root_id)
pos_start = fields.Integer(optional=True)
pos_end = fields.Integer(optional=True)
tags = fields.Set(fields.String())
attr = fields.Map(fields.String(), fields.Any())
data = fields.Set(fields.String())
bindata = fields.Map(fields.String(), fields.SmallUnsignedInteger())
triggers = fields.Map(fields.String(), fields.Enum(TriggerState))
def test_enum(self):
a = fields.Enum(ZlewType)
a.validate(ZlewType.ZLEW)
with self.assertRaises(SchemaError):
a.validate('ZLEW')
with self.assertRaises(SchemaError):
a.validate('zlewzlewzlew')
self.assertEqual(a.dump(ZlewType.ZLEW), 'ZLEW')
self.assertIsInstance(a.dump(ZlewType.ZLEW), six.text_type)
self.assertIs(a.load('TURBOZLEW'), ZlewType.TURBOZLEW)
with self.assertRaises(SchemaError):
a.load(2)
with self.assertRaises(SchemaError):
a.load('zlewzlewzlew')
with self.assertRaises(SchemaError):
a.load('PIETROZLEW')
with self.assertRaises(TypeError):
fields.Enum(int)
class CheckTrigger(Check):
object_type = 'trigger'
node = fields.NodeID()
key = fields.String()
state = fields.Enum(TriggerState, optional=True)
class Repacker(Model):
"""
Repacks binary data to a different element width. Repacking conceptually
works as follows:
1. All source elements starting from the given index, are glued
together to form a big string of bits. If LITTLE endian is selected,
lower-indexed elements are less significant in the resulting
string, otherwise they are more significant.
2. Starting from LSB (for LITTLE endian) or MSB (for BIG
endian) of the resulting string, num_elements * (repacker.to_width +
repacker.low_pad + repacker.high_pad) bits are extracted and cut into
num_elements pieces (again ordered according to the selected
endian).
3. From each of the num_element pieces, high high_pad and low low_pad bits
are cut off, and the rest is returned as the result.
Of course, the actual repacking operation only reads as many source
elements as are actually necessary to determine the output. This number
can be determined by the repack_size() function. It is an error if fewer
elements than that are available in the source.
"""
endian = fields.Enum(Endian)
from_width = fields.SmallUnsignedInteger(minimum=1)
to_width = fields.SmallUnsignedInteger(minimum=1)
high_pad = fields.SmallUnsignedInteger(default=0)
low_pad = fields.SmallUnsignedInteger(default=0)
def __init__(self, endian, from_width, to_width, **kwargs):
super(Repacker, self).__init__(endian=endian,
from_width=from_width,
to_width=to_width,
**kwargs)
@property
def padded_width(self):
"""
Destination element width, including padding.
"""
return self.to_width + self.high_pad + self.low_pad
@property
def repack_unit(self):
"""
Returns repacking unit for given format, ie. lowest common multiple
of source and padded destination widths in bits.
"""
g = gcd(self.padded_width, self.from_width)
return self.padded_width * self.from_width // g
def repack_size(self, num_elements):
"""
Returns the number of source elements that would be read to perform
the given repacking. This is equal to
ceil(self.padded_width * num_elements / self.from_width).
"""
num_elements = operator.index(num_elements)
if num_elements < 0:
raise ValueError('number of elements cannot be negative')
bits = num_elements * self.padded_width
return (bits + self.from_width - 1) // self.from_width
def repackable_size(self, from_size):
"""
Returns the number of destination elements that can be retrieved
from the given source with the given repacking. This is equal to
floor(self.from_width * from_size / self.padded_width).
"""
if from_size < 0:
raise ValueError('source size cannot be negative')
bits = self.from_width * from_size
return bits // self.padded_width
def repack(self, src, start=0, num_elements=None):
"""
Repacks data from the given source BinData. The width of the source
must match the width specified at repacker construction. ``start``
specifies the source index (in source elements) where unpacking
should start. ``num_element`` specifies the number of destination
elements that should be returned (or None to repack as much data
as possible).
"""
if not isinstance(src, BinData):
raise TypeError('repack needs a BinData instance')
if self.from_width != src.width:
raise ValueError('repack source width mismatch')
start = operator.index(start)
if start < 0:
raise ValueError('start cannot be negative')
ru = self.repack_unit
spu = ru // self.from_width
dpu = ru // self.padded_width
if num_elements is None:
num_elements = self.repackable_size(len(src) - start)
rs = self.repack_size(num_elements)
if start + rs > len(src):
raise ValueError('not enough data in source')
units = (rs + spu - 1) // spu
res = BinData(self.to_width, num_elements)
mask = (1 << self.to_width) - 1
for u in range(units):
unit = 0
sp = start + u * spu
for i, x in enumerate(src[sp:sp + spu]):
if self.endian is Endian.LITTLE:
unit |= x << (i * self.from_width)
else:
unit |= x << ((spu - i - 1) * self.from_width)
for i in range(min(dpu, num_elements - u * dpu)):
if self.endian is Endian.LITTLE:
pos = i * self.padded_width
else:
pos = (dpu - i - 1) * self.padded_width
res[u * dpu + i] = unit >> (pos + self.low_pad) & mask
return res
@classmethod
def cpp_type(cls):
return 'RepackerModel', 'veles::data::Repacker', 'data/repack.h'
class EnumOptional(model.Model):
a = fields.Enum(TestEnum, optional=True)
class Enum(model.Model):
a = fields.Enum(TestEnum)
class FieldTypeString(FieldType):
mode = fields.Enum(FieldStringMode)
encoding = fields.Enum(FieldStringEncoding)
class FieldTypeFloat(FieldType):
mode = fields.Enum(FieldFloatMode)
complex = fields.Boolean()
class FieldTypeFixed(FieldType):
shift = fields.SmallInteger()
sign_mode = fields.Enum(FieldSignMode)
