def start(self, dictionary_size=256): self.dictionary_size = dictionary_size self.inp = open(self.inputfile, "rb") self.out = open(self.outputfile, "wb") self.bitin = arithmeticcoding.BitInputStream(self.inp) #self.freqsTable = arithmeticcoding.SimpleFrequencyTable([float(i % 8 + 1) for i in range(self.dictionary_size + 1)]) self.freqsTable = arithmeticcoding.FlatFrequencyTable( self.dictionary_size + 1) self.decoder = arithmeticcoding.ArithmeticDecoder(32, self.bitin)
def decompress(bitin, out): initfreqs = arithmeticcoding.FlatFrequencyTable(257) freqs = arithmeticcoding.SimpleFrequencyTable(initfreqs) dec = arithmeticcoding.ArithmeticDecoder(32, bitin) while True: # Decode and write one byte symbol = dec.read(freqs) if symbol == 256: # EOF symbol break out.write(bytes((symbol, ))) freqs.increment(symbol)
def decompress(freqs, bits_len, bitin, out): dec = arithmeticcoding.ArithmeticDecoder(bits_len, bitin) try: while True: symbol = dec.read(freqs) if symbol == 256: # EOF break # out.write(bytes((symbol,))) out.write(chr(symbol)) except ValueError as err: print(err)
def decompress(input_file): decode = bytearray() with open(input_file, "rb") as inp: bitin = arithmeticcoding.BitInputStream(inp) initfreqs = arithmeticcoding.FlatFrequencyTable(257) freqs = arithmeticcoding.SimpleFrequencyTable(initfreqs) dec = arithmeticcoding.ArithmeticDecoder(32, bitin) while True: # Decode and write one byte symbol = dec.read(freqs) if symbol == 256: # EOF symbol break decode.extend(bytes((symbol, ))) freqs.increment(symbol) return pickle.loads(decode)
def decompress_words(self, s, frequencies, num_symbols): dec = arithmeticcoding.ArithmeticDecoder(s) freqs = self.read_frequencies(frequencies) symbol = dec.read(freqs) self.a.append(symbol) return self.a # Reads an unsigned integer of the given bit width from the given stream. #def read_int(bitin, numbits): # result = 0 # for i in range(numbits): # result = (result << 1) | bitin.read_no_eof() # Big endian # return result # Main launcher #if __name__ == "__main__": # main(sys.argv[1 : ])
def decompress(bitin): pmf = [0.2, 0.1, 0.3, 0.2, 0.2] pmf_new = [0.2, 0.1, 0.1, 0.3, 0.3] pmf = pmf_quantization(pmf) pmf_new = pmf_quantization(pmf_new) freqs = arithmeticcoding.ContextFrequencyTable(pmf) dec = arithmeticcoding.ArithmeticDecoder(32, bitin) out = [] while True: # Decode and write one byte symbol = dec.read(freqs) if symbol == 4: # EOF symbol break #out.append(bytes((symbol,)) if python3 else chr(symbol)) out.append(symbol) freqs.increment(pmf_new) return out
def decompress(bitin, out): # Set up decoder and model dec = arithmeticcoding.ArithmeticDecoder(bitin) model = ppmmodel.PpmModel(MODEL_ORDER, 257, 256) history = [] while True: # Decode and write one byte symbol = decode_symbol(dec, model, history) if symbol == 256: # EOF symbol break out.write(bytes((symbol, )) if python3 else chr(symbol)) model.increment_contexts(history, symbol) if model.model_order >= 1: # Append current symbol or shift back by one if len(history) == model.model_order: del history[0] history.append(symbol)
def decomparess(inputfile, outfile, model): bitin = arithmeticcoding.BitInputStream(open(inputfile, "rb")) initfreqs = arithmeticcoding.FlatFrequencyTable(AE_SIZE) freqs = arithmeticcoding.SimpleFrequencyTable(initfreqs) dec = arithmeticcoding.ArithmeticDecoder(bitin) prev_chars = [] i = 0 with open(outfile, "w") as out: while (True): guesses = dec.read(freqs) freqs.increment(guesses) if guesses == MAGIC_EOF: break print('guesses', guesses) print(prev_chars) for _ in range(guesses): print(prev_chars) char = predict(prev_chars, model, indices_char) print('lit for guess', char) prev_chars.append(char) if len(prev_chars) > maxlen: prev_chars.pop(0) out.write(char) print("i", i) literal = dec.read(freqs) if literal == MAGIC_EOF: break print('lit', chr(literal)) out.write(chr(literal)) freqs.increment(literal) prev_chars.append(chr(literal)) if len(prev_chars) > maxlen: prev_chars.pop(0) i = i + 1 bitin.close()
def decompress(model): dec_char = '' bit_in = arithmeticcoding.BitInputStream(open('./result/data.bin', 'rb')) dec = arithmeticcoding.ArithmeticDecoder(bit_in) out_f = open('./result/recover.txt', 'w') index = 0 num_line = 0 hidden = None # context = [] while True: if index == 0: freq = generate_freqs(pro=1, first_step=True) dec_char = dec.read(freq) index += 1 # print(freq.frequencies, end='') # print(int2char[dec_char]) if dec_char == len(characters): break # context.append(int2char[dec_char]) out_f.write(int2char[dec_char]) else: out, hidden = predict(model, np.array(dec_char), hidden) out = out[0] # (35, ) freq = generate_freqs(pro=out, first_step=False) index += 1 dec_char = dec.read(freq) if dec_char == len(characters): break # context.append(int2char[dec_char]) out_f.write(int2char[dec_char]) if index == 100: index = 0 num_line += 1 hidden = None out_f.write('\n') if num_line % 100 == 0: print(num_line) # context = [] out_f.close()
def decompress(bitin, out): # Set up decoder and model. In this PPM model, symbol 256 represents EOF; # its frequency is 1 in the order -1 context but its frequency # is 0 in all other contexts (which have non-negative order). dec = arithmeticcoding.ArithmeticDecoder(32, bitin) model = ppmmodel.PpmModel(MODEL_ORDER, 257, 256) history = [] while True: # Decode and write one byte symbol = decode_symbol(dec, model, history) if symbol == 256: # EOF symbol break out.write(bytes((symbol,)) if python3 else chr(symbol)) model.increment_contexts(history, symbol) if model.model_order >= 1: # Prepend current symbol, dropping oldest symbol if necessary if len(history) == model.model_order: history.pop() history.insert(0, symbol)
def decode(self, compressed_file, recon_path): # with TOP N dimensions fileobj = open(compressed_file, mode='rb') fileobj.read(1) #dummy buf = fileobj.read(4) arr = np.frombuffer(buf, dtype=np.uint16) w = int(arr[0]) h = int(arr[1]) padded_w = int(math.ceil(w / 16) * 16) padded_h = int(math.ceil(h / 16) * 16) y_hat, z_hat, sigma_z = self.sess.run( [self.y_hat, self.z_hat, self.sigma_z], feed_dict={self.input_x: np.zeros( (1, 3, padded_h, padded_w))}) # NCHW padded_y1_hat = np.pad(y_hat[:, :self.M1, :, :], ((0, 0), (0, 0), (3, 0), (2, 1)), 'constant', constant_values=((0, 0), (0, 0), (0, 0), (0, 0))) ############### decode zhat #################################### bitin = arithmeticcoding.BitInputStream(fileobj) dec = arithmeticcoding.ArithmeticDecoder(bitin) printProgressBar(0, z_hat.shape[1], prefix='Decoding z_hat:', suffix='Complete', length=50) for ch_idx in range(z_hat.shape[1]): printProgressBar(ch_idx + 1, z_hat.shape[1], prefix='Decoding z_hat:', suffix='Complete', length=50) mu_val = 255 sigma_val = sigma_z[ch_idx] # exp_sigma_val = np.exp(sigma_val) freq = arithmeticcoding.ModelFrequencyTable(mu_val, sigma_val) for h_idx in range(z_hat.shape[2]): for w_idx in range(z_hat.shape[3]): symbol = dec.read(freq) if symbol == 512: # EOF symbol print("EOF symbol") break z_hat[:, ch_idx, h_idx, w_idx] = symbol - 255 # bitin.close() ################## ################################################# # Entropy decoding y # padded_z = np.zeros_like(padded_z, dtype = np.float32) h_s_out = self.sess.run(self.h_s_out, feed_dict={self.z_hat: z_hat}) c_prime = h_s_out[:, :self.M1, :, :] sigma2 = h_s_out[:, self.M1:, :, :] padded_c_prime = np.pad(c_prime, ((0, 0), (0, 0), (3, 0), (2, 1)), 'constant', constant_values=((0, 0), (0, 0), (0, 0), (0, 0))) padded_y1_hat[:, :, :, :] = 0.0 y_hat[:, :, :, :] = 0.0 # bitin = arithmeticcoding.BitInputStream(open(dec_inputfile, "rb")) # dec = arithmeticcoding.ArithmeticDecoder(bitin) printProgressBar(0, y_hat.shape[2], prefix='Decoding y_hat:', suffix='Complete', length=50) for h_idx in range(y_hat.shape[2]): printProgressBar(h_idx + 1, y_hat.shape[2], prefix='Decoding y_hat:', suffix='Complete', length=50) for w_idx in range(y_hat.shape[3]): c_prime_i = self.extractor_prime(padded_c_prime, h_idx, w_idx) c_doubleprime_i = self.extractor_doubleprime( padded_y1_hat, h_idx, w_idx) concatenated_c_i = np.concatenate([c_doubleprime_i, c_prime_i], axis=1) pred_mean, pred_sigma = self.sess.run( [self.pred_mean, self.pred_sigma], feed_dict={self.concatenated_c_i: concatenated_c_i}) zero_means = np.zeros([ pred_mean.shape[0], self.M2, pred_mean.shape[2], pred_mean.shape[3] ]) concat_pred_mean = np.concatenate([pred_mean, zero_means], axis=1) concat_pred_sigma = np.concatenate([ pred_sigma, sigma2[:, :, h_idx:h_idx + 1, w_idx:w_idx + 1] ], axis=1) for ch_idx in range(self.M): mu_val = concat_pred_mean[0, ch_idx, 0, 0] + 255 sigma_val = concat_pred_sigma[0, ch_idx, 0, 0] freq = arithmeticcoding.ModelFrequencyTable( mu_val, sigma_val) symbol = dec.read(freq) if symbol == 512: # EOF symbol print("EOF symbol") break if ch_idx < self.M1: padded_y1_hat[:, ch_idx, h_idx + 3, w_idx + 2] = symbol - 255 y_hat[:, ch_idx, h_idx, w_idx] = symbol - 255 bitin.close() ################################################# recon = self.sess.run(self.recon_image, {self.y_hat: y_hat}) recon = recon[0, -h:, -w:, :] im = Image.fromarray(recon.astype(np.uint8)) im.save(recon_path) return
def decode(self, compressed_file, recon_path): # with TOP N dimensions fileobj = open(compressed_file, mode='rb') fileobj.read(1) #dummy buf = fileobj.read(4) arr = np.frombuffer(buf, dtype=np.uint16) w = int(arr[0]) h = int(arr[1]) padded_w = int(math.ceil(w / 16) * 16) padded_h = int(math.ceil(h / 16) * 16) y_hat, z_hat, sigma_z = self.sess.run([self.y_hat, self.z_hat, self.sigma_z], feed_dict={self.input_x: np.zeros((1, 3, padded_h, padded_w))}) # NCHW ############### decode zhat #################################### bitin = arithmeticcoding.BitInputStream(fileobj) dec = arithmeticcoding.ArithmeticDecoder(bitin) z_hat[:, :, :, :] = 0.0 printProgressBar(0, z_hat.shape[1], prefix='Decoding z_hat:', suffix='Complete', length=50) for ch_idx in range(z_hat.shape[1]): printProgressBar(ch_idx + 1, z_hat.shape[1], prefix='Decoding z_hat:', suffix='Complete', length=50) mu_val = 255 sigma_val = sigma_z[ch_idx] freq = arithmeticcoding.ModelFrequencyTable(mu_val, sigma_val) for h_idx in range(z_hat.shape[2]): for w_idx in range(z_hat.shape[3]): symbol = dec.read(freq) if symbol == 512: # EOF symbol print("EOF symbol") break z_hat[:, ch_idx, h_idx, w_idx] = symbol - 255 ############### decode yhat #################################### c_prime = self.sess.run(self.c_prime, feed_dict={self.z_hat: z_hat}) # c_prime = np.round(c_prime, decimals=4) padded_c_prime = np.pad(c_prime, ((0, 0), (0, 0), (3, 0), (2, 1)), 'constant', constant_values=((0, 0), (0, 0), (0, 0), (0, 0))) padded_y_hat = np.pad(y_hat, ((0, 0), (0, 0), (3, 0), (2, 1)), 'constant', constant_values=((0, 0), (0, 0), (0, 0), (0, 0))) padded_y_hat[:, :, :, :] = 0.0 printProgressBar(0, y_hat.shape[2], prefix='Decoding y_hat:', suffix='Complete', length=50) for h_idx in range(y_hat.shape[2]): printProgressBar(h_idx + 1, y_hat.shape[2], prefix='Decoding y_hat:', suffix='Complete', length=50) for w_idx in range(y_hat.shape[3]): c_prime_i = self.extractor_prime(padded_c_prime, h_idx, w_idx) c_doubleprime_i = self.extractor_doubleprime(padded_y_hat, h_idx, w_idx) concatenated_c_i = np.concatenate([c_doubleprime_i, c_prime_i], axis=1) pred_mean, pred_sigma = self.sess.run( [self.pred_mean, self.pred_sigma], feed_dict={self.concatenated_c_i: concatenated_c_i}) for ch_idx in range(self.M): mu_val = pred_mean[0, ch_idx, 0, 0] + 255 sigma_val = pred_sigma[0, ch_idx, 0, 0] freq = arithmeticcoding.ModelFrequencyTable(mu_val, sigma_val) symbol = dec.read(freq) if symbol == 512: # EOF symbol print("EOF symbol") break padded_y_hat[:, ch_idx, h_idx + 3, w_idx + 2] = symbol - 255 bitin.close() y_hat = padded_y_hat[:, :, 3:, 2:-1] ################################################# recon = self.sess.run(self.recon_image, {self.y_hat: y_hat}) recon = recon[0, -h:, -w:, :] im = Image.fromarray(recon.astype(np.uint8)) im.save(recon_path) return
def decode(self, compressed_file, recon_path): # with TOP N dimensions fileobj = open(compressed_file, mode='rb') fileobj.read(1) #dummy buf = fileobj.read(4) arr = np.frombuffer(buf, dtype=np.uint16) w = int(arr[0]) h = int(arr[1]) new_w = int(math.ceil(float(w) / 2.0) * 2) new_h = int(math.ceil(float(h) / 2.0) * 2) pad_w_1 = int((float(new_w) / 2.0) % 2) pad_h_1 = int((float(new_h) / 2.0) % 2) res_w_1 = math.floor(float(new_w) / 2.0) + pad_w_1 res_h_1 = math.floor(float(new_h) / 2.0) + pad_h_1 pad_w_2 = int((float(res_w_1) / 2.0) % 2) pad_h_2 = int((float(res_h_1) / 2.0) % 2) res_w_2 = math.floor(float(res_w_1) / 2.0) + pad_w_2 res_h_2 = math.floor(float(res_h_1) / 2.0) + pad_h_2 pad_w_3 = int((float(res_w_2) / 2.0) % 2) pad_h_3 = int((float(res_h_2) / 2.0) % 2) res_w_3 = math.floor(float(res_w_2) / 2.0) + pad_w_3 res_h_3 = math.floor(float(res_h_2) / 2.0) + pad_h_3 pad_w = new_w - w pad_h = new_h - h sigma_z = self.sess.run(self.sigma_z) y_hat = np.zeros( (1, self.M, int(float(res_h_3) / 2.0), int(float(res_w_3) / 2.0)), dtype=np.float32) y_w = y_hat.shape[3] y_h = y_hat.shape[2] new_y_w = int(math.ceil(float(y_w) / 4.0) * 4) new_y_h = int(math.ceil(float(y_h) / 4.0) * 4) pad_y_w = new_y_w - y_w pad_y_h = new_y_h - y_h pad_y_hat = np.pad(y_hat, ((0, 0), (0, 0), (0, pad_y_h), (0, pad_y_w)), mode='edge') z_hat = self.sess.run(self.z_hat, feed_dict={self.y_hat: pad_y_hat}) # NCHW # y_hat, z_hat, sigma_z = self.sess.run([self.y_hat, self.z_hat, self.sigma_z], # feed_dict={ # self.input_x: np.zeros((1, 3, padded_h, padded_w))}) # NCHW padded_y1_hat = np.pad(y_hat[:, :self.M1, :, :], ((0, 0), (0, 0), (3, 0), (2, 1)), 'constant', constant_values=((0, 0), (0, 0), (0, 0), (0, 0))) ############### decode zhat #################################### bitin = arithmeticcoding.BitInputStream(fileobj) dec = arithmeticcoding.ArithmeticDecoder(bitin) printProgressBar(0, z_hat.shape[1], prefix='Decoding z_hat:', suffix='Complete', length=50) for ch_idx in range(z_hat.shape[1]): printProgressBar(ch_idx + 1, z_hat.shape[1], prefix='Decoding z_hat:', suffix='Complete', length=50) mu_val = 255 sigma_val = sigma_z[ch_idx] # exp_sigma_val = np.exp(sigma_val) freq = arithmeticcoding.ModelFrequencyTable(mu_val, sigma_val) for h_idx in range(z_hat.shape[2]): for w_idx in range(z_hat.shape[3]): symbol = dec.read(freq) if symbol == 512: # EOF symbol print("EOF symbol") break z_hat[:, ch_idx, h_idx, w_idx] = symbol - 255 # bitin.close() ################## ################################################# # Entropy decoding y # padded_z = np.zeros_like(padded_z, dtype = np.float32) h_s_out = self.sess.run(self.h_s_out, feed_dict={self.z_hat: z_hat}) c_prime = h_s_out[:, :self.M1, :, :] sigma2 = h_s_out[:, self.M1:, :, :] padded_c_prime = np.pad(c_prime, ((0, 0), (0, 0), (3, 0), (2, 1)), 'constant', constant_values=((0, 0), (0, 0), (0, 0), (0, 0))) padded_y1_hat[:, :, :, :] = 0.0 y_hat[:, :, :, :] = 0.0 # bitin = arithmeticcoding.BitInputStream(open(dec_inputfile, "rb")) # dec = arithmeticcoding.ArithmeticDecoder(bitin) printProgressBar(0, y_hat.shape[2], prefix='Decoding y_hat:', suffix='Complete', length=50) for h_idx in range(y_hat.shape[2]): printProgressBar(h_idx + 1, y_hat.shape[2], prefix='Decoding y_hat:', suffix='Complete', length=50) for w_idx in range(y_hat.shape[3]): c_prime_i = self.extractor_prime(padded_c_prime, h_idx, w_idx) c_doubleprime_i = self.extractor_doubleprime( padded_y1_hat, h_idx, w_idx) concatenated_c_i = np.concatenate([c_doubleprime_i, c_prime_i], axis=1) pred_mean, pred_sigma = self.sess.run( [self.pred_mean, self.pred_sigma], feed_dict={self.concatenated_c_i: concatenated_c_i}) zero_means = np.zeros([ pred_mean.shape[0], self.M2, pred_mean.shape[2], pred_mean.shape[3] ]) concat_pred_mean = np.concatenate([pred_mean, zero_means], axis=1) concat_pred_sigma = np.concatenate([ pred_sigma, sigma2[:, :, h_idx:h_idx + 1, w_idx:w_idx + 1] ], axis=1) for ch_idx in range(self.M): mu_val = concat_pred_mean[0, ch_idx, 0, 0] + 255 sigma_val = concat_pred_sigma[0, ch_idx, 0, 0] freq = arithmeticcoding.ModelFrequencyTable( mu_val, sigma_val) symbol = dec.read(freq) if symbol == 512: # EOF symbol print("EOF symbol") break if ch_idx < self.M1: padded_y1_hat[:, ch_idx, h_idx + 3, w_idx + 2] = symbol - 255 y_hat[:, ch_idx, h_idx, w_idx] = symbol - 255 bitin.close() ################################################# ############### gsh1 = self.sess.run(self.gsh1, feed_dict={self.y_hat: y_hat}) gsh1 = gsh1[:, :res_h_3 - pad_h_3, :res_w_3 - pad_w_3, :] gsh2 = self.sess.run(self.gsh2, feed_dict={self.gsh1: gsh1}) gsh2 = gsh2[:, :res_h_2 - pad_h_2, :res_w_2 - pad_w_2, :] gsh3 = self.sess.run(self.gsh3, feed_dict={self.gsh2: gsh2}) gsh3 = gsh3[:, :res_h_1 - pad_h_1, :res_w_1 - pad_w_1, :] recon = self.sess.run(self.recon_image, feed_dict={self.gsh3: gsh3}) recon = recon[0, :recon.shape[1] - pad_h, :recon.shape[2] - pad_w, :] ############### im = Image.fromarray(recon.astype(np.uint8)) im.save(recon_path) return
def inferenceNN( self, x, M, N, overall_freqs, L, activationFunction ): #N is the number of hidden nodes, the weights are of dimension MxN y = [0 for i in range(N)] enc = arithmeticcoding.ArithmeticEncoder() dec = arithmeticcoding.ArithmeticDecoder(L) q = deque([N]) #q_node = deque([node]) self.w = 0 tot_queue_length = floor(2 * log2(N + 1) + 1) max_queue_length = floor(2 * log2(N + 1) + 1) current_queue_length = floor(2 * log2(N + 1) + 1) j = 0 level = 0 flag = 0 flagp = 0 k = len(overall_freqs) print('M:', M, 'N:', N) while len(q) != 0 and level < M: currentNodeValue = q.popleft() current_queue_length -= floor(2 * log2(currentNodeValue + 1) + 1) if flagp == 0: print('current_queue_length', current_queue_length) flagp = 1 #currentnode = q_node.popleft() if currentNodeValue > 1: c = 0 #colour initialized with 0 while c <= k - 1 and currentNodeValue > 0: #kth colour need not be encoded binomial_frequencies = ec().binomial_encoder_frequencies( overall_freqs[c:], currentNodeValue) freqs = arithmeticcoding.SimpleFrequencyTable( binomial_frequencies) childNodeValue = dec.read(freqs) #if childNodeValue != currentnode.childNodes[c].v: # print('Not Matching!', childNodeValue, currentnode.childNodes[c].v) #else: # print('No problems here') enc.write(freqs, childNodeValue) currentNodeValue -= childNodeValue q.append(childNodeValue) current_queue_length += floor(2 * log2(childNodeValue + 1) + 1) max_queue_length = max(max_queue_length, current_queue_length) tot_queue_length += current_queue_length self.w += 1 #q_node.append(currentnode.childNodes[c]) #print('childNodeValue',childNodeValue) if childNodeValue > 0: flag = 1 for i in range(childNodeValue): # print('level:',level,'x[level]',x[level]) # print('Calculating Y....', level,':',self.w) y[j + i] += uc().index_to_weight(c) * x[level] #print(x[level], c) #y[j+i] += c*x[level] c = c + 1 j = (j + childNodeValue) % N if j == 0 and flag: level = level + 1 #print('level:',level) flag = 0 elif currentNodeValue == 1: freqs = arithmeticcoding.SimpleFrequencyTable(overall_freqs) c = dec.read(freqs) enc.write(freqs, c) q.append(1) current_queue_length += 3 max_queue_length = max(max_queue_length, current_queue_length) tot_queue_length += current_queue_length self.w += 1 y[j + i] += uc().index_to_weight(c) * x[level] j = (j + 1) % N if j == 0: level += 1 avg_queue_length = tot_queue_length / self.w L1 = enc.finish() #return L1 if needed y = np.array(y) if activationFunction == 'ReLU': y = uc().ReLU(y) elif activationFunction == 'sigmoid': y = uc().sigmoid(y) elif activationFunction == None: y = y return y, avg_queue_length, max_queue_length
def openFileRight(self): enc = arithmeticcoding.ArithmeticDecoder(self.bitoutR) return enc
def openFileLeft(self): enc = arithmeticcoding.ArithmeticDecoder(self.bitoutL) return enc