def test_round_trip():
freqs = {
'the': 10,
'quick': 3,
'brown': 4,
'fox': 1,
'jumped': 5,
'over': 8,
'lazy': 1,
'dog': 2,
'.': 9
}
codec = HuffmanCodec(freqs.items())
message = [
'the', 'quick', 'brown', 'fox', 'jumped', 'over', 'the', 'the', 'lazy',
'dog', '.'
]
strings = list(codec.strings)
codes = dict([(codec.leaves[i], strings[i])
for i in range(len(codec.leaves))])
bits = codec.encode(message)
string = ''.join('{0:b}'.format(c).rjust(8, '0')[::-1]
for c in bits.as_bytes())
for word in message:
code = codes[word]
assert string[:len(code)] == code
string = string[len(code):]
unpacked = [0] * len(message)
bits.seek(0)
codec.decode(bits, unpacked)
assert message == unpacked
def test_vocab(EN):
codec = HuffmanCodec([(w.orth, numpy.exp(w.prob)) for w in EN.vocab])
expected_length = 0
for i, code in enumerate(codec.strings):
leaf = codec.leaves[i]
expected_length += len(code) * numpy.exp(EN.vocab[leaf].prob)
assert 8 < expected_length < 15
def test_vocab_codec():
vocab = Vocab()
lex = vocab['dog']
lex = vocab['the']
lex = vocab['jumped']
codec = HuffmanCodec([(lex.orth, lex.prob) for lex in vocab])
bits = BitArray()
ids = [vocab[s].orth for s in ('the', 'dog', 'jumped')]
msg = numpy.array(ids, dtype=numpy.int32)
msg_list = list(msg)
codec.encode(msg, bits)
result = numpy.array(range(len(msg)), dtype=numpy.int32)
bits.seek(0)
codec.decode(bits, result)
assert msg_list == list(result)
def test_attribute():
freqs = {'the': 10, 'quick': 3, 'brown': 4, 'fox': 1, 'jumped': 5, 'over': 8,
'lazy': 1, 'dog': 2, '.': 9}
int_map = {'the': 0, 'quick': 1, 'brown': 2, 'fox': 3, 'jumped': 4, 'over': 5,
'lazy': 6, 'dog': 7, '.': 8}
codec = HuffmanCodec([(int_map[string], freq) for string, freq in freqs.items()])
bits = BitArray()
msg = numpy.array([1, 7], dtype=numpy.int32)
msg_list = list(msg)
codec.encode(msg, bits)
result = numpy.array([0, 0], dtype=numpy.int32)
bits.seek(0)
codec.decode(bits, result)
assert msg_list == list(result)
def test_round_trip():
freqs = {'the': 10, 'quick': 3, 'brown': 4, 'fox': 1, 'jumped': 5, 'over': 8,
'lazy': 1, 'dog': 2, '.': 9}
codec = HuffmanCodec(freqs.items())
message = ['the', 'quick', 'brown', 'fox', 'jumped', 'over', 'the',
'the', 'lazy', 'dog', '.']
strings = list(codec.strings)
codes = dict([(codec.leaves[i], strings[i]) for i in range(len(codec.leaves))])
bits = codec.encode(message)
string = ''.join('{0:b}'.format(c).rjust(8, '0')[::-1] for c in bits.as_bytes())
for word in message:
code = codes[word]
assert string[:len(code)] == code
string = string[len(code):]
unpacked = [0] * len(message)
bits.seek(0)
codec.decode(bits, unpacked)
assert message == unpacked
def test_vocab_codec():
def get_lex_props(string, prob):
return {
'flags': 0,
'length': len(string),
'orth': string,
'lower': string,
'norm': string,
'shape': string,
'prefix': string[0],
'suffix': string[-3:],
'cluster': 0,
'prob': prob,
'sentiment': 0
}
vocab = Vocab()
vocab['dog'] = get_lex_props('dog', 0.001)
vocab['the'] = get_lex_props('the', 0.05)
vocab['jumped'] = get_lex_props('jumped', 0.005)
codec = HuffmanCodec([(lex.orth, lex.prob) for lex in vocab])
bits = BitArray()
ids = [vocab[s].orth for s in ('the', 'dog', 'jumped')]
msg = numpy.array(ids, dtype=numpy.int32)
msg_list = list(msg)
codec.encode(msg, bits)
result = numpy.array(range(len(msg)), dtype=numpy.int32)
bits.seek(0)
codec.decode(bits, result)
assert msg_list == list(result)
def test_freqs():
freqs = []
words = []
for i, line in enumerate(open('freqs.txt')):
pieces = line.strip().split()
if len(pieces) != 2:
continue
freq, word = pieces
freqs.append(int(freq))
words.append(word)
total = float(sum(freqs))
codec = HuffmanCodec(zip(words, freqs))
expected_length = 0
for i, code in enumerate(codec.strings):
expected_length += len(code) * (freqs[i] / total)
assert 8 < expected_length < 14
def test1():
probs = numpy.zeros(shape=(10,), dtype=numpy.float32)
probs[0] = 0.3
probs[1] = 0.2
probs[2] = 0.15
probs[3] = 0.1
probs[4] = 0.06
probs[5] = 0.02
probs[6] = 0.01
probs[7] = 0.005
probs[8] = 0.0001
probs[9] = 0.000001
codec = HuffmanCodec(list(enumerate(probs)))
py_codes = py_encode(dict(enumerate(probs)))
py_codes = list(py_codes.items())
py_codes.sort()
assert codec.strings == [c for i, c in py_codes]
def test_rosetta():
txt = u"this is an example for huffman encoding"
symb2freq = defaultdict(int)
for ch in txt:
symb2freq[ch] += 1
by_freq = list(symb2freq.items())
by_freq.sort(reverse=True, key=lambda item: item[1])
symbols = [sym for sym, prob in by_freq]
codec = HuffmanCodec(symb2freq.items())
py_codec = py_encode(symb2freq)
codes = dict([(codec.leaves[i], codec.strings[i]) for i in range(len(codec.leaves))])
my_lengths = defaultdict(int)
py_lengths = defaultdict(int)
for symb, freq in symb2freq.items():
my = codes[symb]
my_lengths[len(my)] += freq
py_lengths[len(py_codec[symb])] += freq
my_exp_len = sum(length * weight for length, weight in my_lengths.items())
py_exp_len = sum(length * weight for length, weight in py_lengths.items())
assert my_exp_len == py_exp_len
def test_vocab_codec():
vocab = Vocab()
lex = vocab['dog']
lex = vocab['the']
lex = vocab['jumped']
codec = HuffmanCodec([(lex.orth, lex.prob) for lex in vocab])
bits = BitArray()
ids = [vocab[s].orth for s in ('the', 'dog', 'jumped')]
msg = numpy.array(ids, dtype=numpy.int32)
msg_list = list(msg)
codec.encode(msg, bits)
result = numpy.array(range(len(msg)), dtype=numpy.int32)
bits.seek(0)
codec.decode(bits, result)
assert msg_list == list(result)
def test_attribute():
freqs = {'the': 10, 'quick': 3, 'brown': 4, 'fox': 1, 'jumped': 5, 'over': 8,
'lazy': 1, 'dog': 2, '.': 9}
int_map = {'the': 0, 'quick': 1, 'brown': 2, 'fox': 3, 'jumped': 4, 'over': 5,
'lazy': 6, 'dog': 7, '.': 8}
codec = HuffmanCodec([(int_map[string], freq) for string, freq in freqs.items()])
bits = BitArray()
msg = numpy.array([1, 7], dtype=numpy.int32)
msg_list = list(msg)
codec.encode(msg, bits)
result = numpy.array([0, 0], dtype=numpy.int32)
bits.seek(0)
codec.decode(bits, result)
assert msg_list == list(result)
def test_empty():
codec = HuffmanCodec({})
assert codec.strings == []
