def transcribe_value(B):
''' Transcribe this `Block`, the inverse of parse_value.
'''
transcription = []
is_indirect = B.indirect
span = B.span
if is_indirect:
# aside from the span, everything else comes from the superblock
B = B.superblock
block_type = B.type
assert block_type >= 0, "block_type(%s) => %d" % (B, B.type)
block_typed = block_type != BlockType.BT_HASHCODE
flags = ((F_BLOCK_INDIRECT if is_indirect else 0)
| (F_BLOCK_TYPED if block_typed else 0)
| 0 # no F_BLOCK_TYPE_FLAGS
)
transcription.append(BSUInt.transcribe_value(flags))
transcription.append(BSUInt.transcribe_value(span))
if block_typed:
transcription.append(BSUInt.transcribe_value(block_type))
# no block_type_flags
if block_type == BlockType.BT_HASHCODE:
transcription.append(B.hashcode.transcribe_b())
elif block_type == BlockType.BT_RLE:
transcription.append(B.octet)
elif block_type == BlockType.BT_LITERAL:
transcription.append(B.data)
elif block_type == BlockType.BT_SUBBLOCK:
transcription.append(BSUInt.transcribe_value(B.offset))
transcription.append(BlockRecord.transcribe_value(B.superblock))
else:
raise ValueError("unsupported Block type 0x%02x: %s" %
(block_type, B))
block_bs = b''.join(flatten_transcription(transcription))
return BSData(block_bs).transcribe()
def do(stream):
''' Return the number of indexed blocks in `local_store`.
'''
local_store = stream._local_store
if local_store is None:
raise ValueError("no local_store, request rejected")
return bytes(BSUInt(len(local_store)))
def parse_value(bfr):
''' Unserialise a serialised Dirent.
'''
type_ = BSUInt.parse_value(bfr)
flags = DirentFlags(BSUInt.parse_value(bfr))
if flags & DirentFlags.HASNAME:
flags ^= DirentFlags.HASNAME
name = BSString.parse_value(bfr)
else:
name = ""
if flags & DirentFlags.HASMETA:
flags ^= DirentFlags.HASMETA
metatext = BSString.parse_value(bfr)
else:
metatext = None
uu = None
if flags & DirentFlags.HASUUID:
flags ^= DirentFlags.HASUUID
uu = UUID(bytes=bfr.take(16))
if flags & DirentFlags.NOBLOCK:
flags ^= DirentFlags.NOBLOCK
block = None
else:
block = BlockRecord.parse_value(bfr)
if flags & DirentFlags.HASPREVDIRENT:
flags ^= DirentFlags.HASPREVDIRENT
prev_dirent_blockref = BlockRecord.parse_value(bfr)
else:
prev_dirent_blockref = None
if flags & DirentFlags.EXTENDED:
flags ^= DirentFlags.EXTENDED
extended_data = BSData.parse_value(bfr)
else:
extended_data = None
if flags:
warning("%s.parse_value: unexpected extra flags: 0x%02x",
cls.__name__, flags)
E = _Dirent.from_components(type_,
name,
meta=metatext,
uuid=uu,
block=block)
E._prev_dirent_blockref = prev_dirent_blockref
E.ingest_extended_data(extended_data)
return E
def transcribe(self):
''' Transcribe this packet.
'''
is_request = self.is_request
channel = self.channel
bss = [
BSUInt.transcribe_value(self.tag),
BSUInt.transcribe_value((0x01 if channel != 0 else 0x00)
| (0x02 if is_request else 0x00)
| (self.flags << 2)),
BSUInt.transcribe_value(channel) if channel != 0 else b'',
BSUInt.transcribe_value(self.rq_type) if is_request else b'',
self.payload
]
length = sum(len(bs) for bs in bss)
# spit out a BSData manually to avoid pointless bytes.join
yield BSUInt.transcribe_value(length)
yield bss
class hashclass(HashCode):
''' `HashCode` subclass.
'''
__slots__ = ()
HASHNAME = hashname
HASHFUNC = hashfunc
HASHLEN = hashlen
HASHENUM = hashenum
HASHENUM_BS = bytes(BSUInt(hashenum))
HASHLEN_ENCODED = len(HASHENUM_BS) + HASHLEN
def parse(cls, bfr):
''' Parse a `CompressibleDataRecord` from a buffer.
'''
flags = BSUInt.parse_value(bfr)
data = BSData.parse_value(bfr)
is_compressed = (flags & cls.FLAG_COMPRESSED) != 0
if is_compressed:
flags &= ~cls.FLAG_COMPRESSED
if flags:
raise ValueError("unsupported flags: 0x%02x" % (flags, ))
return cls(_data=data, is_compressed=is_compressed)
def __len__(self):
try:
flags, payload = self.do(LengthRequest())
except StoreError as e:
error("connection: %s", e)
return None
assert flags == 0
length, offset = BSUInt.parse_value_from_bytes(payload)
if offset < len(payload):
warning("unparsed bytes after BSUInt(length): %r",
payload[offset:])
return length
def parse(cls, bfr):
''' Parse a packet from a buffer.
'''
raw_payload = BSData.parse_value(bfr)
payload_bfr = CornuCopyBuffer([raw_payload])
self = cls()
# pylint: disable=attribute-defined-outside-init
self.tag = BSUInt.parse_value(payload_bfr)
flags = BSUInt.parse_value(payload_bfr)
has_channel = (flags & 0x01) != 0
self.is_request = (flags & 0x02) != 0
flags >>= 2
self.flags = flags
if has_channel:
self.channel = BSUInt.parse_value(payload_bfr)
else:
self.channel = 0
if self.is_request:
self.rq_type = BSUInt.parse_value(payload_bfr)
self.payload = b''.join(payload_bfr)
return self
def transcribe(self):
''' Serialise to binary format.
'''
E = self.dirent
flags = 0
type_ = E.type
if E.name:
flags |= DirentFlags.HASNAME
meta = None if E.isindirect else E.meta
if meta:
flags |= DirentFlags.HASMETA
if E.uuid:
flags |= DirentFlags.HASUUID
block = None if type_ is DirentType.INDIRECT else E.block
if block is None:
flags |= DirentFlags.NOBLOCK
if E._prev_dirent_blockref is not None:
flags |= DirentFlags.HASPREVDIRENT
extended_data = E.get_extended_data()
if extended_data:
flags |= DirentFlags.EXTENDED
yield BSUInt.transcribe_value(type_)
yield BSUInt.transcribe_value(flags)
if flags & DirentFlags.HASNAME:
yield BSString.transcribe_value(E.name)
if flags & DirentFlags.HASMETA:
yield BSString.transcribe_value(meta.textencode())
if flags & DirentFlags.HASUUID:
bs = E.uuid.bytes
if len(bs) != 16:
raise RuntimeError("len(E.uuid.bytes) != 16: %r" % (bs, ))
yield bs
if not flags & DirentFlags.NOBLOCK:
yield BlockRecord.transcribe_value(block)
if flags & DirentFlags.HASPREVDIRENT:
assert isinstance(E._prev_dirent_blockref, _Block)
yield BlockRecord.transcribe_value(E._prev_dirent_blockref)
if flags & DirentFlags.EXTENDED:
yield extended_data
def parse(cls, bfr, *, parse_flags):
''' Parse a HashCodesRequest from a buffer and construct.
'''
after = (parse_flags & 0x01) != 0
has_start_hashcode = (parse_flags & 0x02) != 0
extra_flags = parse_flags & ~0x03
if extra_flags:
raise ValueError("extra flags: 0x%02x" % (extra_flags, ))
hashname = BSString.parse_value(bfr)
hashclass = HashCode.by_index(hashname)
if has_start_hashcode:
start_hashcode = HashCodeField.parse_value(bfr)
if type(start_hashcode) is not hashclass: # pylint: disable=unidiomatic-typecheck
raise ValueError(
"request hashclass %s does not match start_hashcode class %s"
% (hashclass, type(start_hashcode)))
else:
start_hashcode = None
length = BSUInt.parse_value(bfr)
return cls(after=after,
hashclass=hashclass,
start_hashcode=start_hashcode,
length=length)
def parse_value(bfr):
''' Decode a Block reference from a buffer.
Format is a `BSData` holding this encoded data:
BS(flags)
0x01 indirect blockref
0x02 typed: type follows, otherwise BT_HASHCODE
0x04 type flags: per type flags follow type
BS(span)
[BS(type)]
[BS(type_flags)]
union {
type BT_HASHCODE: hash
type BT_RLE: octet-value (repeat span times to get data)
type BT_LITERAL: raw-data (span bytes)
type BT_SUBBLOCK: suboffset, super block
}
Even though this is all decodable without the leading length
we use a leading length so that future encodings do not
prevent parsing any following data.
'''
raw_encoding = BSData.parse_value(bfr)
blockref_bfr = CornuCopyBuffer.from_bytes(raw_encoding)
flags = BSUInt.parse_value(blockref_bfr)
is_indirect = bool(flags & F_BLOCK_INDIRECT)
is_typed = bool(flags & F_BLOCK_TYPED)
has_type_flags = bool(flags & F_BLOCK_TYPE_FLAGS)
unknown_flags = flags & ~(F_BLOCK_INDIRECT | F_BLOCK_TYPED
| F_BLOCK_TYPE_FLAGS)
if unknown_flags:
raise ValueError(
"unexpected flags value (0x%02x) with unsupported flags=0x%02x"
% (flags, unknown_flags))
span = BSUInt.parse_value(blockref_bfr)
if is_indirect:
# With indirect blocks, the span is of the implied data, not
# the referenced block's data. Therefore we build the referenced
# block with a span of None and store the span in the indirect
# block.
ispan = span
span = None
# block type, default BT_HASHCODE
if is_typed:
block_type = BlockType(BSUInt.parse_value(blockref_bfr))
else:
block_type = BlockType.BT_HASHCODE
if has_type_flags:
type_flags = BSUInt.parse_value(blockref_bfr)
if type_flags:
warning("nonzero type_flags: 0x%02x", type_flags)
else:
type_flags = 0x00
# instantiate type specific block ref
if block_type == BlockType.BT_HASHCODE:
hashcode = HashCode.from_buffer(blockref_bfr)
B = HashCodeBlock(hashcode=hashcode, span=span)
elif block_type == BlockType.BT_RLE:
octet = blockref_bfr.take(1)
B = RLEBlock(span, octet)
elif block_type == BlockType.BT_LITERAL:
data = blockref_bfr.take(span)
B = LiteralBlock(data)
elif block_type == BlockType.BT_SUBBLOCK:
suboffset = BSUInt.parse_value(blockref_bfr)
superB = BlockRecord.parse_value(blockref_bfr)
# wrap inner Block in subspan
B = SubBlock(superB, suboffset, span)
else:
raise ValueError("unsupported Block type 0x%02x" % (block_type, ))
if is_indirect:
B = IndirectBlock(B, span=ispan)
if not blockref_bfr.at_eof():
warning("unparsed data (%d bytes) follow Block %s",
len(raw_encoding) - blockref_bfr.offset, B)
assert isinstance(B, _Block)
return B
def transcribe(self):
''' Transcribe this data chunk as a data record.
'''
yield BSUInt.transcribe_value(self.flags)
yield BSData.transcribe_value(self._data)
def transcribe(self):
yield BSString.transcribe_value(self.hashclass.HASHNAME)
start_hashcode = self.start_hashcode
if start_hashcode is not None:
yield HashCodeField.transcribe_value(start_hashcode)
yield BSUInt.transcribe_value(self.length)
def transcribe_value(hashcode):
''' Serialise a hashcode.
'''
yield BSUInt.transcribe_value(hashcode.HASHENUM)
yield hashcode
def parse_value(bfr):
''' Decode a serialised hash from the CornuCopyBuffer `bfr`.
'''
hashenum = BSUInt.parse_value(bfr)
hashcls = HASHCLASS_BY_ENUM[hashenum]
return hashcls.from_hashbytes(bfr.take(hashcls.HASHLEN))
def data_offset(self):
''' The offset of the data chunk within the transcribed `DataRecord`.
'''
return (len(BSUInt.transcribe_value(self.flags)) +
BSData.data_offset_for(self._data))