/
literalcoder.py
110 lines (100 loc) · 2.78 KB
/
literalcoder.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
import bitqueue
import arithmeticcodec
import plainstrcoder
import packedcoder
import zippedcoder
import numcoder
import gzip
from successordict import SuccessorDict as sdict
import json
import modelbuilder
import struct
import encoder
ac = None
class GILFprogram:
def __init__(self,rootnode):
self.root = rootnode
def run(self):
self.root()
def prepare():
global ac
with gzip.open('engmodel6.json.gz', 'rb') as f:
countdict = json.load(f,object_pairs_hook=sdict)
with open("symbols.json") as f:
symbols = json.load(f)
ac = ArithmeticCodec(modelbuilder.build_model(countdict,symbols))
def encode_list(data, bq=None):
if bq is None: bq=bitqueue.BitQueue()
if not isinstance(data, list): return encode(data, bq)
for item in data[:-1]:
encode(item, bq)
bq.pushBit(1)
encode(data[-1],bq)
bq.pushBit(0)
return bq
def encode(data, bq=None):
if bq is None: bq=bitqueue.BitQueue()
if isinstance(data, str) or isinstance(data, unicode):
bq.pushBits("0")
return encode_str(data,bq)
elif isinstance(data, int) or isinstance(data, long):
bq.pushBits("10")
return numcoder.encode(data,bq)
elif isinstance(data, float):
bq.pushBits("110")
bq.pushBytes(struct.pack('!f',data))
return bq
# elif type(data) is GILFprogram:
# bq.pushBits("111")
# return encoder.encode(data)
raise ValueError("Literals of type %s are not supported"%data.__class__.__name__)
def encode_str(s, bq=None):
global ac
if ac is None:
prepare()
if bq is None: bq=bitqueue.BitQueue()
try:
bq1 = packedcoder.encode(s)
encodetype = "01"
except:
pass
try:
bq2 = plainstrcoder.encode(s)
except:
pass
if bq1 is None or bq2 is not None and len(bq2)<len(bq1):
bq1 = bq2
encodetype = "00"
bq2 = zippedcoder.encode(s)
if bq1 is None or len(bq2)<len(bq1):
bq1 = bq2
encodetype = "10"
bq2 = ac.encode(teststring)
if len(bq2)<len(bq1):
bq1=bq2
encodetype = "11"
bq.pushBits(encodetype)
bq.append(bq1)
return bq
def decode_list(bq):
head = list(decode(bq))
if bq.nextBit():
remainder = decode_list(bq)
else:
remainder = []
head.extend(remainder)
return head
def decode(bq):
if not bq.nextBit():
return decode_str(bq)
if not bq.nextBit():
return numcoder.decode(bq)
if not bq.nextBit():
return struct.unpack('!f',bq.popBytes(4))[0]
return GILFprogram(encoder.decode(bq))
def decode_str(bq):
global ac
if ac is None:
prepare()
encodetype = bq.popBits(2)
return [plainstrcoder,packedcoder,zippedcoder,ac][encodetype].decode(bq)