def test_StoreBit(self):
bb = BitBucket()
expected_output = "|10111010|11111111|00001111|0101 [4]"
for c in expected_output:
if c == '1':
bb.StoreBit(1)
elif c == '0':
bb.StoreBit(0)
assert expected_output == str(bb)
def Encode(self, text, include_eof):
"""
Encodes 'text' using the pre-computed huffman coding, and returns it as
a tuple of (list-of-bytes, number-of-bits-as-int). If 'include_eof' is true,
then an EOF will be encoded at the end.
"""
bb = BitBucket()
self.EncodeToBB(bb, text, include_eof)
return bb.GetAllBits()
def RealOpsToOps(self, realops):
""" Unpacks wire format operations into in-memory format"""
bb = BitBucket()
bb.StoreBits((common_utils.StrToList(realops), len(realops) * 8))
seder = Spdy4SeDer()
(group_id,
ops) = seder.DeserializeInstructions(bb, self.packing_instructions,
self.huffman)
return (group_id, ops)
def test_GetBits4(self):
bb = BitBucket()
inp = [0xd, 0xd, 0xd, 0xd]
for offset in xrange(16):
bb.Clear()
StoreBitsFromString(bb, "1" * offset + "|11011101|11011101 [0]")
for i in xrange(offset):
bb.GetBit()
for i in xrange(len(inp)):
assert bb.GetBits4() == inp[i]
def OpsToRealOps(self, in_ops, header_group):
""" Packs in-memory format operations into wire format"""
data = BitBucket()
seder = Spdy4SeDer()
data.StoreBits(
seder.SerializeInstructions(in_ops, self.packing_instructions,
self.huffman, 0xffff, header_group,
True, self.options.verbose))
return common_utils.ListToStr(data.GetAllBits()[0])
def test_GetBits8(self):
bb = BitBucket()
inp = [0xdd, 0xee, 0xaa, 0xdd]
bitstr = "|11011101|11101110|10101010|11011101 [0]"
for offset in xrange(16):
bb.Clear()
StoreBitsFromString(bb, ("1" * offset) + bitstr)
for i in xrange(offset):
bb.GetBit()
for i in xrange(len(inp)):
assert bb.GetBits8() == inp[i]
def test_StoreBits8(self):
bb = BitBucket()
inp = [0xff, 0x00, 0x00, 0xdd]
orig_expected_str = "|11111111|00000000|00000000|11011101 [0]"
for offset in xrange(16):
bb.Clear()
for i in xrange(offset):
bb.StoreBit(1)
for n in inp:
bb.StoreBits8(n)
assert str(bb) == ReformatExpectedStr('1' * offset,
orig_expected_str)
def test_Clear(self):
bb = BitBucket()
bb.StoreBits(([0xff, 0xff, 0xff], 23))
if str(bb) != "|11111111|11111111|1111111 [7]":
self.fail("basic storing of bits didn't work!")
bb.Clear()
assert str(bb) == " [0]"
assert bb.output == []
assert bb.out_byte == 0
assert bb.out_boff == 0
assert bb.idx_byte == 0
assert bb.idx_boff == 0
def RebuildDecodeTreeFromCanonicalCodes(self):
root = [None, None, 0, [None, None]]
for sym in xrange(len(self.canonical_code_table)):
(code, _) = self.canonical_code_table[sym]
bb = BitBucket()
bb.StoreBits(code)
curr = root
while not bb.AllConsumed():
bit = bb.GetBits(1)[0][0] >> 7
if curr[3][bit] is None:
curr[3][bit] = [None, None, curr[2] + 1, [None, None]]
curr = curr[3][bit]
curr[1] = sym
curr[3] = []
self.code_tree = root
def OpsToRealOps(self, in_ops):
data = [BitBucket(), BitBucket(), BitBucket()]
PackOps(data, alt_packing_instructions, in_ops, self.huffman_table)
overall = BitBucket()
overall.StoreBits(data[0].GetAllBits())
overall.StoreBits(data[1].GetAllBits())
overall.StoreBits(data[2].GetAllBits())
return ListToStr(overall.GetAllBits()[0])
def main():
h = Huffman(request_freq_table)
for s in test_data:
print " encoding: ", s
sp = [ord(c) for c in s]
e_result = BitBucket()
h.EncodeToBB(e_result, sp, True)
print " e_result: ", FormatAsBits(e_result.GetAllBits())
d_result = ListToStr(h.DecodeFromBB(e_result, True, -1))
if d_result != s:
print "difference found: d_result(%s) vs orig(%s)" % (repr(d_result),
repr(s))
else:
print "It worked: ", s
print
def test_StoreBitse(self):
bb = BitBucket()
testcase_e = [
(([0, 52], 8 + 6), "|00000000|001101 [6]"),
(([185], 8), "|00000000|00110110|111001 [6]"),
]
RunTestCase(bb, testcase_e)
def test_AdvanceReadPtrToByteBoundary(self):
bb = BitBucket()
for offset in xrange(16):
bb.Clear()
StoreBitsFromString(bb, "1" * 32)
for i in xrange(offset):
bb.GetBit()
old_idx_byte = bb.idx_byte
old_idx_boff = bb.idx_boff
assert bb.idx_byte == offset / 8
assert bb.idx_boff == offset % 8
bb.AdvanceReadPtrToByteBoundary()
if old_idx_boff == 0:
assert bb.idx_boff == old_idx_boff
assert bb.idx_byte == old_idx_byte
else:
assert bb.idx_boff == 0
assert bb.idx_byte == (old_idx_byte + 1)
def test_StoreBitsd(self):
bb = BitBucket()
testcase_d = [
(([0xF0], 8), "|11110000 [0]"),
(([0xF0], 8), "|11110000|11110000 [0]"),
(([0xF0], 1), "|11110000|11110000|1 [1]"),
(([0x0F], 8), "|11110000|11110000|10000111|1 [1]"),
]
RunTestCase(bb, testcase_d)
def main():
bb = BitBucket()
testcase_a = [
(([0xFF,0],6+8), "|11111111|000000 [6]"),
(([0xFF], 3), "|11111111|00000011|1 [1]"),
(([0x00], 3), "|11111111|00000011|1000 [4]"),
(([0xFF,0], 8+6), "|11111111|00000011|10001111|11110000|00 [2]"),
(([0xFF], 4), "|11111111|00000011|10001111|11110000|001111 [6]"),
(([0x0], 4), "|11111111|00000011|10001111|11110000|00111100|00 [2]"),
]
RunTestCase(bb, testcase_a)
testcase_b = [
(([0xF0], 5), "|11110 [5]"),
(([0x0F], 5), "|11110000|01 [2]"),
(([0xF0], 5), "|11110000|0111110 [7]"),
(([0x0F], 5), "|11110000|01111100|0001 [4]"),
(([0xF0], 5), "|11110000|01111100|00011111|0 [1]"),
(([0x0F], 5), "|11110000|01111100|00011111|000001 [6]"),
(([0xF0], 5), "|11110000|01111100|00011111|00000111|110 [3]"),
(([0x0F], 5), "|11110000|01111100|00011111|00000111|11000001 [0]"),
(([0xF0], 5), "|11110000|01111100|00011111|00000111|11000001|11110 [5]"),
]
bb.Clear()
RunTestCase(bb, testcase_b)
testcase_c = [
(([0xF0], 1), "|1 [1]"),
(([0x0F], 1), "|10 [2]"),
(([0xF0], 1), "|101 [3]"),
(([0x0F], 1), "|1010 [4]"),
(([0xF0], 1), "|10101 [5]"),
(([0x0F], 1), "|101010 [6]"),
(([0xF0], 1), "|1010101 [7]"),
(([0x0F], 1), "|10101010 [0]"),
(([0xF0], 1), "|10101010|1 [1]"),
(([0x00,0xFF], 8+7), "|10101010|10000000|01111111 [0]"),
]
bb.Clear()
RunTestCase(bb, testcase_c)
testcase_d = [
(([0xF0], 8), "|11110000 [0]"),
(([0xF0], 8), "|11110000|11110000 [0]"),
(([0xF0], 1), "|11110000|11110000|1 [1]"),
(([0x0F], 8), "|11110000|11110000|10000111|1 [1]"),
]
bb.Clear()
RunTestCase(bb, testcase_d)
testcase_e = [
(([0,52], 8+6), "|00000000|001101 [6]"),
(([185], 8), "|00000000|00110110|111001 [6]"),
]
bb.Clear()
RunTestCase(bb, testcase_e)
print "Success!"
def test_StoreBitsa(self):
bb = BitBucket()
testcase_a = [
(([0xFF, 0], 6 + 8), "|11111111|000000 [6]"),
(([0xFF], 3), "|11111111|00000011|1 [1]"),
(([0x00], 3), "|11111111|00000011|1000 [4]"),
(([0xFF,
0], 8 + 6), "|11111111|00000011|10001111|11110000|00 [2]"),
(([0xFF], 4), "|11111111|00000011|10001111|11110000|001111 [6]"),
(([0x0], 4),
"|11111111|00000011|10001111|11110000|00111100|00 [2]"),
]
RunTestCase(bb, testcase_a)
def test_StoreBitsc(self):
bb = BitBucket()
testcase_c = [
(([0xF0], 1), "|1 [1]"),
(([0x0F], 1), "|10 [2]"),
(([0xF0], 1), "|101 [3]"),
(([0x0F], 1), "|1010 [4]"),
(([0xF0], 1), "|10101 [5]"),
(([0x0F], 1), "|101010 [6]"),
(([0xF0], 1), "|1010101 [7]"),
(([0x0F], 1), "|10101010 [0]"),
(([0xF0], 1), "|10101010|1 [1]"),
(([0x00, 0xFF], 8 + 7), "|10101010|10000000|01111111 [0]"),
]
RunTestCase(bb, testcase_c)
def test_StoreBitsb(self):
bb = BitBucket()
testcase_b = [
(([0xF0], 5), "|11110 [5]"),
(([0x0F], 5), "|11110000|01 [2]"),
(([0xF0], 5), "|11110000|0111110 [7]"),
(([0x0F], 5), "|11110000|01111100|0001 [4]"),
(([0xF0], 5), "|11110000|01111100|00011111|0 [1]"),
(([0x0F], 5), "|11110000|01111100|00011111|000001 [6]"),
(([0xF0], 5), "|11110000|01111100|00011111|00000111|110 [3]"),
(([0x0F], 5), "|11110000|01111100|00011111|00000111|11000001 [0]"),
(([0xF0], 5),
"|11110000|01111100|00011111|00000111|11000001|11110 [5]"),
]
RunTestCase(bb, testcase_b)
def DeserializeInstructions(self, frame, packing_instructions, huff):
""" Takes SPDY4 wire-format data and de-serializes it into in-memory
operations
It returns these operations.
"""
ops = []
bb = BitBucket()
bb.StoreBits(frame.GetAllBits())
flags = 0
#print 'DeserializeInstructions'
while flags == 0:
frame_len = bb.GetBits16() * 8
#print 'frame_len: ', frame_len
flags = bb.GetBits8()
#print 'flags: ', flags
stream_id = bb.GetBits32()
#print 'stream_id: ', stream_id
frame_type = bb.GetBits8()
#print 'frame_type: ', frame_type
while frame_len > 16: # 16 bits minimum for the opcode + count...
bits_remaining_at_start = bb.BitsRemaining()
opcode_val = bb.GetBits8()
#print 'opcode_val: ', opcode_val
op_count = bb.GetBits8() + 1
#print 'op_count: ', op_count
opcode_description = opcode_to_op[opcode_val]
opcode = opcode_description[0]
fields = opcode_description[1:]
for i in xrange(op_count):
op = {'opcode': opcode}
for field_name in packing_order:
if not field_name in fields:
continue
(params, _, unpack_fn) = packing_instructions[field_name]
val = unpack_fn(bb, params, huff)
#print val
op[field_name] = val
#print "BitsRemaining: %d (%d)" % (bb.BitsRemaining(), bb.BitsRemaining() % 8)
#print "Deser %d" % (bb.NumBits() - bb.BitsRemaining())
#print op
ops.append(op)
bits_consumed = (bits_remaining_at_start - bb.BitsRemaining())
#if not bits_consumed % 8 == 0:
# print "somehow didn't consume whole bytes..."
# print "Bits consumed: %d (%d)" % (bits_consumed, bits_consumed % 8)
# raise StandardError()
frame_len -= bits_consumed
#print 'ops: ', ops
return ops
def compress(self, inp_headers, host):
data = BitBucket()
res = ''
for k, v in inp_headers.items():
if k in bohe.ID_TABLE:
zz = data.NumBits()
# encode as registered header
data.StoreBits8(bohe.ID_TABLE.index(k) + 1)
l = 0
dohuff = True
# Set the binary flag
if k in bohe.ENCODERS:
data.StoreBit(1)
dohuff = False
# Set the multiple values flag...
if '\u00' in v:
data.StoreBit(1)
else:
data.StoreBit(0)
val = bohe.encode(k, v)
if dohuff:
val_as_list, len_in_bits = self.do_huff(self.huff, val)
else:
val_as_list = common_utils.StrToList(val)
len_in_bits = len(val_as_list) * 8
data.StoreBits22(len(val_as_list))
data.StoreBits((val_as_list, len_in_bits))
else:
data.StoreBits8(128 | len(k))
data.StoreBits((common_utils.StrToList(k), len(k) * 8))
data.StoreBit(0) # assume not binary value for now
if '\u00' in v:
data.StoreBit(1)
else:
data.StoreBit(0)
val_as_list, len_in_bits = self.do_huff(self.huff, v)
data.StoreBits22(len(val_as_list))
data.StoreBits((val_as_list, len_in_bits))
return ''.join(common_utils.ListToStr(data.GetAllBits()[0]))
def SerializeInstructions(self,
ops,
packing_instructions,
huff,
stream_id,
end_of_frame):
""" Serializes a set of instructions possibly containing many different
type of opcodes into SPDY4 wire format, discovers the resultant length,
computes the appropriate SPDY4 boilerplate, and then returns this
in a new BitBucket
"""
#print 'SerializeInstructions\n', ops
(ot, otr) = self.PreProcessToggles(ops)
payload_bb = BitBucket()
self.OutputOps(packing_instructions, huff, payload_bb, ot, 'toggl')
self.OutputOps(packing_instructions, huff, payload_bb, otr, 'trang')
self.OutputOps(packing_instructions, huff, payload_bb, ops['clone'],'clone')
self.OutputOps(packing_instructions, huff, payload_bb, ops['kvsto'],'kvsto')
self.OutputOps(packing_instructions, huff, payload_bb, ops['eref'], 'eref')
(payload, payload_len) = payload_bb.GetAllBits()
payload_len = (payload_len + 7) / 8 # partial bytes are counted as full
frame_bb = BitBucket()
self.WriteControlFrameBoilerplate(frame_bb, 0, 0, 0, 0)
boilerplate_length = frame_bb.BytesOfStorage()
frame_bb = BitBucket()
overall_bb = BitBucket()
bytes_allowed = 2**16 - boilerplate_length
while True:
#print 'payload_len: ', payload_len
bytes_to_consume = min(payload_len, bytes_allowed)
#print 'bytes_to_consume: ', bytes_to_consume
end_of_frame = (bytes_to_consume <= payload_len)
#print 'end_of_Frame: ', end_of_frame
self.WriteControlFrameBoilerplate(overall_bb, bytes_to_consume,
end_of_frame, stream_id, 0x8)
overall_bb.StoreBits( (payload, bytes_to_consume*8))
payload = payload[bytes_to_consume:]
payload_len -= bytes_allowed
if payload_len <= 0:
break
return overall_bb.GetAllBits()
def SerializeInstructions(self, ops, packing_instructions, huff, stream_id,
group_id, end_of_frame, verbose):
""" Serializes a set of instructions possibly containing many different
type of opcodes into SPDY4 wire format, discovers the resultant length,
computes the appropriate SPDY4 boilerplate, and then returns this
in a new BitBucket
"""
#print 'SerializeInstructions\n', ops
if verbose >= 5:
print
print "stream_id: %s group_id: %s" % (stream_id, group_id)
print FormatOps(ops)
print
payload_bb = BitBucket()
for opcode, oplist in ops.iteritems():
self.OutputOps(packing_instructions, huff, payload_bb, oplist,
opcode)
(payload, payload_len) = payload_bb.GetAllBits()
payload_len = (payload_len +
7) / 8 # partial bytes are counted as full
frame_bb = BitBucket()
self.WriteControlFrameBoilerplate(frame_bb, 0, 0, 0, group_id, 0)
boilerplate_length = frame_bb.BytesOfStorage()
frame_bb = BitBucket()
overall_bb = BitBucket()
bytes_allowed = 2**16 - boilerplate_length
while True:
#print 'payload_len: ', payload_len
bytes_to_consume = min(payload_len, bytes_allowed)
#print 'bytes_to_consume: ', bytes_to_consume
end_of_frame = (bytes_to_consume <= payload_len)
#print 'end_of_Frame: ', end_of_frame
self.WriteControlFrameBoilerplate(overall_bb, bytes_to_consume,
end_of_frame, stream_id,
group_id, 0x8)
overall_bb.StoreBits((payload, bytes_to_consume * 8))
payload = payload[bytes_to_consume:]
payload_len -= bytes_allowed
if payload_len <= 0:
break
if verbose >= 5:
PrintHex(overall_bb.GetAllBits()[0])
return overall_bb.GetAllBits()
def OpsToRealOps(self, in_ops, header_group):
""" Packs in-memory format operations into wire format"""
data = BitBucket()
PackOps(data, self.packing_instructions, in_ops, self.huffman,
header_group, self.options.verbose)
return common_utils.ListToStr(data.GetAllBits()[0])
def RealOpsToOps(self, realops):
bb = BitBucket()
bb.StoreBits((StrToList(realops), len(realops) * 8))
return UnpackOps([bb, BitBucket(), BitBucket()],
alt_packing_instructions, self.huffman_table)
def test_StoreBits4(self):
bb = BitBucket()
inp = [0xff, 0xf0, 0xf0, 0xfd]
for n in inp:
bb.StoreBits4(n)
assert str(bb) == "|11110000|00001101 [0]"
bb.Clear()
bb.StoreBit(0)
for n in inp:
bb.StoreBits4(n)
assert str(bb) == "|01111000|00000110|1 [1]"
bb.Clear()
bb.StoreBit(0)
bb.StoreBit(1)
for n in inp:
bb.StoreBits4(n)
assert str(bb) == "|01111100|00000011|01 [2]"
bb.Clear()
bb.StoreBit(0)
bb.StoreBit(1)
bb.StoreBit(0)
for n in inp:
bb.StoreBits4(n)
assert str(bb) == "|01011110|00000001|101 [3]"
bb.Clear()
bb.StoreBit(0)
bb.StoreBit(1)
bb.StoreBit(0)
bb.StoreBit(1)
for n in inp:
bb.StoreBits4(n)
assert str(bb) == "|01011111|00000000|1101 [4]"
def OpsToRealOps(self, in_ops): """ Packs in-memory format operations into wire format""" data = BitBucket() PackOps(data, packing_instructions, in_ops, self.huffman_table) return ListToStr(data.GetAllBits()[0])
def RealOpsToOps(self, realops): """ Unpacks wire format operations into in-memory format""" bb = BitBucket() bb.StoreBits((StrToList(realops), len(realops)*8)) return UnpackOps(bb, packing_instructions, self.huffman_table)
def Encode(self, text, include_eof):
bb = BitBucket()
self.EncodeToBB(bb, text, include_eof)
return bb.GetAllBits()