def decode(src, reader): with open(src, "rb") as f: data = f.read() assert data[0x00:0x02] == SOI, "SOI not recognized" assert data[0x02:0x14] == APP0, "APP 0 not recognized" assert data[0x14:0x59] == DQT0, "DQT 0 not recognized" assert data[0x59:0x9e] == DQT1, "DQT 1 not recognized" assert data[0x9e:0xa3] == SOF0_PREFIX, "SOF 0 not recognized" height, = unpack('>H', data[0xa3:0xa5]) width, = unpack('>H', data[0xa5:0xa7]) assert data[0xa7:0xb1] == SOF0_SUFFIX, "SOF 0 not recognized" assert data[0xb1:0x255] == DHT, "DHT not recognized" assert data[0x255:0x263] == SOS, "SOS not recognized" assert data[-2:] == EOI, "EOI not recognized" stream = BitStream() i = 0x263 while i < len(data) - 2: value = data[i:i + 1] stream.write(value, bytes) i += 1 if data[i] != 0xff else 2 bits = BitStream() for yAC, _, _ in huffmanDecode(stream, height // 8 * width // 8): read(np.array(yAC)[zigzagReverseOrder], bits, reader) length = unpack('>I', bits.read(bytes, 4))[0] if length > len(bits) / 8: print("{} of {} bytes are revealed.".format(len(bits) / 8, length)) length = len(bits) / 8 return bits.read(bytes, length)
def RTCM_converter_thread(server, port, username, password, mountpoint, rtcm_callback=None): import subprocess nt = subprocess.Popen(["./ntripclient", "--server", server, "--password", password, "--user", username, "--mountpoint", mountpoint], stdout=subprocess.PIPE) """nt = subprocess.Popen(["./ntrip.py", server, str(port), username, password, mountpoint], stdout=subprocess.PIPE)""" if nt is None or nt.stdout is None: indev = sys.stdin else: indev = nt.stdout print("RTCM using input {}".format(indev)) while True: sio = indev d = ord(sio.read(1)) if d != RTCMv3_PREAMBLE: continue pack_stream = BitStream() l1 = ord(sio.read(1)) l2 = ord(sio.read(1)) pack_stream.append(bs.pack('2*uint:8', l1, l2)) pack_stream.read(6) pkt_len = pack_stream.read(10).uint pkt = sio.read(pkt_len) parity = sio.read(3) if len(pkt) != pkt_len: print "Length error {} {}".format(len(pkt), pkt_len) continue if True: # TODO check parity for d in pkt: pack_stream.append(bs.pack('uint:8', ord(d))) msg = parse_rtcmv3(pack_stream) if msg is not None and rtcm_callback is not None: rtcm_callback(msg)
def count_bits(bit_stream: BitStream, which_bit: bool, max_count: int) -> int: """ count consecutive number of same bits :param bit_stream: a stream of bits :param which_bit: which bits to count :param max_count: max allowed number of same consecutive :return: count of same consecutive bits in given stream """ count = 0 which_bit = '1' if which_bit else '0' bit = str(bit_stream)[0] while bit == which_bit and count < max_count and len(bit_stream): bit_stream.read(bool, 1) count += 1 if len(bit_stream): bit = str(bit_stream)[0] return count
def decode(self, data): stream = BitStream(data) mode = MODE_NONE while True: first_bit = stream.read(bool) if first_bit: ones = leading_ones(stream) if mode == MODE_SCHEME1: if not first_bit: self.add_byte(ord(stream.read(bytes, 1))) continue elif ones < 8: cnt_field = stream.read( BitStream, ones + 1 if ones < 4 else 11 - ones) if ones > 4: cnt_field += (ones - 4) << (12 - ones) cnt = int(str(cnt_field), 2) + (2 << ones) loc = int(str(stream.read(BitStream, 10)), 2) for i in range(loc, loc + cnt): self.add_byte(self.buffer[i % 1024]) continue if first_bit and ones == 8: control = str(stream.read(BitStream, 3)) if control == '101': # Scheme 1 mode = MODE_SCHEME1 elif control == '100': # End of record (EOR) self.output.extend(self.buffer[:self.buffer_offset]) break else: raise ValueError( "Can't parse SLDC, unsupported control symbol: %s" % (control)) else: raise ValueError("Can't parse SLDC, unknown action") return self.output
def convert_bit_to_array(v_bits, values_type, nb_values): """ Convert a binary array into specific type array INPUT: v_bits: 1d-array, values_type: type of element in output array nb_values: number of elements to read OUTPUT: array with specific elements type indide """ stream = BitStream() stream.write(v_bits.astype(bool), bool) return stream.read(values_type, nb_values)
def decode_rtcm3_pack(buff): d = ord(buff[0]) if d != RTCMv3_PREAMBLE: print "RTCM3 preamble error!", d return pack_stream = BitStream() l1 = ord(buff[1]) l2 = ord(buff[2]) pack_stream.append(bs.pack('2*uint:8', l1, l2)) pack_stream.read(6) pkt_len = pack_stream.read(10).uint pkt = buff[3:3 + pkt_len] parity = buff[3 + pkt_len:-1] if True: # TODO check parity for d in pkt: pack_stream.append(bs.pack('uint:8', ord(d))) msg = parse_rtcmv3(pack_stream)
def fix_checksum(mnemonic): payload, checksum = split(mnemonic) # hash the payload hasher = sha256() hasher.update(as_bytes(payload)) long_checksum = BitStream(hasher.digest()) # append bits from the beginning of the SHA256 to the end of the starting entropy for bit in long_checksum.read(bool, len(payload) / 32): payload.write(bit) return bitstream_to_mnemonic(payload)
def expand(bit_stream: BitStream) -> BitStream: """ expand compressed bit stresm :param bit_stream: bitstream to expand :return: original stream before compression """ expanded = BitStream() bit = True while len(bit_stream): count = bit_stream.read(int8, 1)[0] for _ in range(count): expanded.write(bit, bool) bit = not bit return expanded
def pack(): contents = open(sys.argv[2], 'rb').read() packed_file = open(sys.argv[3], 'wb') bits = BitStream(contents) f = '' i = len(bits) while i > 0: b = bits.read(16) i -= 16 # dirty flip to deal with endianness b = str(b)[8:] + str(b)[:8] b = b[7:] f += b packed_bitstream = BitStream() for i in f: packed_bitstream.write(int(i), bool) for i in range(0, len(packed_bitstream) / 8): b = packed_bitstream.read(8) packed_file.write(struct.pack('B', int(str(b), 2)))
def unpack(): contents = open(sys.argv[2], 'rb').read() output_file = open(sys.argv[3], 'wb') bits = BitStream(contents) i = len(bits) while i > 0: b = bits.read(9) i -= 9 b = '0000000' + str(b) sh = int(b, 2) x = struct.pack("H", sh) output_file.write(x)
def checksum_is_good(mnemonic): payload, checksum = split(mnemonic) # hash the payload hasher = sha256() hasher.update(as_bytes(payload)) long_checksum = BitStream(hasher.digest()) new_checksum = BitStream() for bit in long_checksum.read(bool, len(payload) / 32): new_checksum.write(bit) print(checksum) print(new_checksum) return checksum == new_checksum
def _genEncodedData(self): """ Turns input data into a bit stream with headers but no error correction """ output = BitStream() # Add mode indicator. Using byte mode, so, write 0100 output.write([False, True, False, False], bool) # Add character count indicator. The size of the indicator depends on the QR version in use data_len = len(self.data_bytes) if self.version <= 9: output.write(data_len, uint8) else: output.write(data_len, uint16) # Add data output.write(self.data_bytes, bytes) # Get required size block_info = QRCode.BLOCK_LIST[self.version - 1][self.ecc] req_bytes = None if len(block_info) == 3: req_bytes = block_info[1] * block_info[2] else: req_bytes = block_info[1] * block_info[2] + block_info[ 3] * block_info[4] req_bits = req_bytes * 8 # Add 4-bit (or less) zero padding zero_count = min(4, req_bits - len(output)) output.write([False] * zero_count, bool) # Align data to bytes out_len = len(output) aligned_bits = ceil(out_len / 8) * 8 output.write([False] * (aligned_bits - out_len), bool) # Add padding bytes if necessary aligned_bytes = aligned_bits // 8 alternate = False for i in range(req_bytes - aligned_bytes): if alternate: output.write(17, uint8) else: output.write(236, uint8) alternate = not alternate return output.read(bytes)
def decode(bit_stream: BitStream, a_node: TrieNode): """ decode bitstream to a string :param bit_stream: bit stream to decode :param a_node: root of a trie representing computed huffman encoding :return: decoded string """ result = "" current = a_node while len(bit_stream): bit = bit_stream.read(bool, 1) if bit[0]: current = current.left else: current = current.right if current.is_leaf: result += current.char current = a_node return result
def LeftRotation(x, bits): stream = BitStream() stream.write(x, uint16) x = stream.read(bool, bits) stream.write(x, bool) return stream.read(uint16)
zCode = numpy.asarray(zCode) bitStream = BitStream() huffmanEncodeFunction.valueHuffmanEncode(zCode, bitStream) outputFile = open('./output/outputFile.b', 'wb+') # write encoded data bitLength = bitStream.__len__() filledNum = 8 - bitLength % 8 if(filledNum!=0): bitStream.write(numpy.ones([filledNum]).tolist(),bool) # 补全为字节(b的数量应该是8整数倍) sosBytes = bitStream.read(bytes) for i in range(len(sosBytes)): outputFile.write(bytes([sosBytes[i]])) #if(sosBytes[i]==255): #outputFile.write(bytes([0])) # FF to FF 00 outputFile.close()
#codedfile.write(rc) print("log2 of mean abs values of subbands: ", ricecoeff) """ import matplotlib.pyplot as plt plt.plot(ricecoeff) plt.show() """ for k in range(N): #loop across subbands: if (k % 100 == 0): print("Subband:", k) #m=2**b signedrice = rice(b=ricecoeff[k], signed=True) yrice = BitStream(ychan[chan, k, :].astype(np.int32), signedrice) #see: http://boisgera.github.io/bitstream/ #Turn bitstream format into sequence of bytes: ys = yrice.read(bytes, np.floor(len(yrice) / 8.0)) totalbytes += len(ys) pickle.dump(ys, codedfile, protocol=-1) #Rice coded subband samples #decoder: #yricedec = BitStream(); #yricedec.write(ys) #ychandec=yricedec.read(signedrice,100) #print("ychan[chan,k,:100].astype(np.int32)", ychan[chan,k,:100].astype(np.int32)) #print("ychandec=", ychandec) #print("len(yricedec)", len(yricedec)) """ numsamples=np.prod(x.shape) print("Total number of bytes=", totalbytes) print("Total number of samples:", numsamples)
fs, "channels=", channels, ) numblocks = pickle.load(codedfile) numblocks -= 4 #last encoded samples might be missing from rounding to bytes print("numblocks=", numblocks) for chan in range(channels): #loop over channels: print("channel ", chan) ricecoeffcomp = pickle.load(codedfile) ricecoeff = struct.unpack('B' * len(ricecoeffcomp), ricecoeffcomp) #print("ricecoeff=", ricecoeff) ychandec = np.zeros((N, numblocks)) for k in range(N): #loop across subbands: if (k % 100 == 0): print("Subband:", k) ys = pickle.load(codedfile) #Rice coded subband samples #m=2**b signedrice = rice(b=ricecoeff[k], signed=True) yricedec = BitStream() yricedec.write(ys) ychandec[k, :] = yricedec.read(signedrice, numblocks) if chan == 0: y0 = ychandec if chan == 1: y1 = ychandec print("Inverse IntMDCT:") xrek = IntMDCTsynfb(y0, y1, fb) xrek = np.clip(xrek, -2**15, 2**15 - 1) print("Write decoded signal to wav file ", decfile) wav.write(decfile, fs, np.int16(xrek))
class HuffmanDecoder(): def __init__(self, binary_path=None, decoded_file_path=None, symbols_amount=2**8): self.binary_path = binary_path self.decoded_file_path = decoded_file_path self.adaptative_binary_tree = AdaptativeBinaryTree(symbols_amount) def decode_binary(self): ##### Read bitstream from file. self.__read_binary_file() ##### Decode header self.__decode_header() ##### Decode remaining bitstream. self.decode_with_adaptative_hc() ##### Save decoded file self.__save_decoded_file() def read_bitstream(self, bitstream): if isinstance(bitstream, BitStream): self.bitstream = bitstream elif isinstance(bitstream, str): bool_list = list(map(lambda bit: bool(int(bit)), list(bitstream))) self.bitstream = BitStream() self.bitstream.write(bool_list) else: raise TypeError( "The provided bitstream should already be a bitstream.BitStream instance or a string." ) def decode_with_adaptative_hc(self, verbose=True): ##### Measure decoding time. decoding_start = time.time() ##### Create attribute to save decoded bytes. self.decoded_bytes = [] ##### Read first symbol and insert in the tree. symbol = self.bitstream.read(np.uint8, 1)[0] self.adaptative_binary_tree.insert_symbol(symbol) self.decoded_bytes.append(symbol) ##### Search for valid codeword. # NOTE: Since now the codeword do not have a fixed-lenght, we must look for a leaf node. bitstream_finished = False while bitstream_finished is not True: try: codeword_bool_list = self.bitstream.read(bool, 1) codeword = self.__convert_bool_list_to_str(codeword_bool_list) ##### Increase the codeword bit by bit, until reaching a valid symbol. symbol = None while symbol is None: symbol = self.adaptative_binary_tree.get_symbol_from_codeword( codeword) if symbol is None: codeword_bool_list += self.bitstream.read(bool, 1) codeword = self.__convert_bool_list_to_str( codeword_bool_list) else: if symbol == 'NYT': symbol = self.bitstream.read(np.uint8, 1)[0] ##### After finding the symbol, it can be added to the decoded_bytes_list. self.adaptative_binary_tree.insert_symbol(symbol) self.decoded_bytes.append(symbol) except ReadError: bitstream_finished = True decoding_finish = time.time() if verbose: self.__print_process_duration(decoding_start, decoding_finish, "Decoding Time") def get_decoded_bytes(self): return self.decoded_bytes ########## Private Methods def __read_binary_file(self): with open(self.binary_path, "rb") as bin_file: str_bitstream = bin_file.read().decode() bool_list = list( map(lambda bit: bool(int(bit)), list(str_bitstream))) self.bitstream = BitStream() self.bitstream.write(bool_list) def __decode_header(self): ##### Verify if file is image or text. self.image_file = self.bitstream.read(bool, 1)[0] # TODO: Test image header decoding. if self.image_file: ##### Check if image has 1 or 3 channels. self.three_channel_image = self.bitstream.read(bool, 1)[0] ##### Check if difference is positive positive_difference = self.bitstream.read(bool, 1)[0] ##### Read digits amount in difference value. digits_amount_bool_list = self.bitstream.read(bool, 4) digits_amount = int( self.__convert_bool_list_to_str(digits_amount_bool_list), 2) #### Read difference diff_str = '' for digit in range(digits_amount): digit_bool_list = self.bitstream.read(bool, 4) str_digit = str( int(self.__convert_bool_list_to_str(digit_bool_list))) diff_str += str_digit self.difference = int(diff_str) if positive_difference else ( -int(diff_str)) def __save_decoded_file(self): if self.image_file: ##### Get amount of pixels in the image. pixels_array = np.array(self.decoded_bytes) pixels_amount = len( pixels_array) / 3 if self.three_channel_image else len( pixels_array) ##### Get image dimensions second_degree_coeff = [1, np.abs(self.difference), -pixels_amount] height = int(np.around(np.roots(second_degree_coeff).max(), 0)) width = height + self.difference ##### Reshape Image channels = 3 if self.three_channel_image else 1 img = np.squeeze(pixels_array.reshape((height, width, channels))) ##### Include image extension if self.three_channel_image: self.decoded_file_path += '.png' file_format = 'PNG' else: self.decoded_file_path += '.bmp' file_format = 'BMP' ##### Save image image = Image.fromarray(img) image.save(self.decoded_file_path, format=file_format) else: ##### Convert bytes to string text_string = ''.join([chr(byte) for byte in self.decoded_bytes]) ##### Include extension in file path and save file. self.decoded_file_path += '.txt' with open(self.decoded_file_path, "wb") as decoded_file: decoded_file.write(text_string.encode()) decoded_file.close() def __print_process_duration(self, starting_time, ending_time, process_name): def check_plural(value, string): string += 's' if value > 1 else '' return string time_difference = int(ending_time - starting_time) seconds = time_difference % 60 minutes = (time_difference // 60) % 60 hours = (time_difference // 60) // 60 hours_string = check_plural(hours, f'{hours} hour') + ', ' if hours else '' minutes_string = check_plural( minutes, f'{minutes} minute') + ' and ' if minutes else '' seconds_string = check_plural( seconds, f'{seconds} second') if seconds else '0 second' print(process_name + ' took ' + hours_string + minutes_string + seconds_string + '.') def __convert_bool_list_to_str(self, bool_list): bs = ''.join(['1' if bit == True else '0' for bit in bool_list]) return bs
class FishStream: # init a BitStream to hold the bit values def __init__(self): self.stream = BitStream() self.fish_positions = [] self.zero_count = 0 self.one_count = 0 # helper function, add one to the stream and one count def add_zero(self): self.stream.write(False) self.zero_count += 1 # helper function, add zero to the stream and the zero count def add_one(self): self.stream.write(True) self.one_count += 1 def add_position(self, fish_id, x, y): # if velocity and acceleration is greater than these values # then flip the bit value. velocity_threshold = 5 acceleration_threshold = 5 # if the current index it empty this will throw # an indexException, and in which case, init the # current fish position at that index. try: # grab the past position of the current fish # - PREVIOUS - x_previous = self.fish_positions[fish_id][0] # position of x y_previous = self.fish_positions[fish_id][1] # position of y vx_previous = self.fish_positions[fish_id][2] # velocity of x vy_previous = self.fish_positions[fish_id][3] # velocity of y ax = ay = 0 # determine the current velocity if x_previous > x: vx = (x_previous - x) else: vx = (x - x_previous) if y_previous > y: vy = (y_previous - y) else: vy = (y - y_previous) # determine the current acceleration if vx_previous > vx: ax = vx_previous - vx else: ax = vx - vx_previous if vy_previous > vy: ay = vy_previous - vy else: ay = vy - vy_previous # current fish moved to the right if x_previous > x: if vx < velocity_threshold: if ax < acceleration_threshold: self.add_one() else: self.add_zero() else: if ax < acceleration_threshold: self.add_one() else: self.add_zero() # current fish moved to the left elif x_previous < x: if vx < velocity_threshold: if ax > acceleration_threshold: self.add_one() else: self.add_zero() else: if ax > acceleration_threshold: self.add_one() else: self.add_zero() # current fish moved up the screen if y_previous < y: if vy < velocity_threshold: if ay < acceleration_threshold: self.add_one() else: self.add_zero() else: if ay < acceleration_threshold: self.add_one() else: self.add_zero() # current fish moved down the screen elif y_previous > y: if vy < velocity_threshold: if ay > acceleration_threshold: self.add_one() else: self.add_zero() else: if ay > acceleration_threshold: self.add_one() else: self.add_zero() # overwrite previous positions with current self.fish_positions[fish_id] = [x, y, vx, vy] except IndexError: # new fish found, append is to the list self.fish_positions.append([x, y, 0, 0]) def print_stream(self): print self.stream # returns two values, probability of zero and one def get_probabilities(self): # calculate total (we use it twice) total = self.zero_count + self.one_count if total == 0: return 0, 0 # returns the probability of a zero and a one return float(self.zero_count) / total, \ float(self.one_count) / total def get_bits(self, length): while self.stream.__len__() < length: time.sleep(0.1) return_bits = self.stream.read(length) self.zero_count -= str(return_bits).count('0') self.one_count -= str(return_bits).count('1') return return_bits def get_length(self): return len(str(self.stream))
def decode(self, stream: BitStream): self._mean = stream.read(np.uint16, 1) self._p0 = np.frombuffer(stream.read(np.int8, 2), dtype=np.float16) self._p1 = np.frombuffer(stream.read(np.int8, 2), dtype=np.float16) self._p2 = np.frombuffer(stream.read(np.int8, 2), dtype=np.float16) return stream
np.int8) #integer, 8bit #print("ricecoeff=", ricecoeff) s = struct.pack('b' * int(len(ricecoeff)), *ricecoeff) pickle.dump(s, codedfile, protocol=-1) totalbytes += 2 * numblocks prederror = np.concatenate((prederror, np.zeros((4, numblocks))), axis=0) #add 4 zeros #to each block of prediction errors to possibly complete rice bytes. for k in range(numblocks): #loop across blocks: if (k % 100 == 0): print("block:", k) #Rice coding with m=2**b signedrice = rice(b=ricecoeff[k], signed=True) prederrorrice = BitStream(prederror[:, k].astype(np.int32), signedrice) #see: http://boisgera.github.io/bitstream/ #Turn bitstream format into sequence of bytes: prederrors = prederrorrice.read( bytes, np.floor(len(prederrorrice) / 8.0)) totalbytes += len(prederrors) pickle.dump(prederrors, codedfile, protocol=-1) #Rice coded block samples #""" numsamples = np.prod(x.shape) print("Total number of bytes=", totalbytes) print("Total number of samples:", numsamples) print("bytes per sample=", totalbytes * 1.0 / numsamples) print("Hence bis per sample=", 8 * 1.0 * totalbytes / numsamples) #"""
channels=pickle.load(codedfile) print("fs=", fs, "channels=", channels, ) numblocks=pickle.load(codedfile) #N-=4 #last encoded samples might be missing from rounding to bytes print("numblocks=", numblocks) xrek=np.zeros((numblocks*N, channels)) for chan in range(channels): #loop over channels: print("channel ", chan) ricecoeffcomp=pickle.load(codedfile) ricecoeff =struct.unpack( 'B' * len(ricecoeffcomp), ricecoeffcomp); print("len(ricecoeff)=", len(ricecoeff)) prederrordec=np.zeros(N*numblocks) for k in range(numblocks): #loop across blocks: if (k%100==0): print("Block number:",k) prederrors=pickle.load(codedfile) #Rice coded block samples #m=2**b signedrice=rice(b=ricecoeff[k],signed=True) prederrorrice = BitStream(); prederrorrice.write(prederrors) prederrordec[k*N:(k+1)*N]=prederrorrice.read(signedrice, N) print("NLMS prediction:") h=np.zeros(L) prederrordec=prederrordec*1.0 #convert to float to avoid overflow print("len(prederrordec)=", len(prederrordec)) xrek[:len(prederrordec),chan]=nlmslosslesspreddec(prederrordec,L,h) print("Write decoded signal to wav file ", decfile) wav.write(decfile,fs,np.int16(xrek))
def decode(self, stream: BitStream): self._max_val = stream.read(np.uint16, 1)[0] return stream
def encode(src, dst, message, writer): plain = pack( '>I', len(message)) + message # message is left-padded by it's length bits = BitStream() bits.write( # convert message to bitstream np.unpackbits(np.frombuffer(plain, dtype=np.uint8)).reshape(len(plain), 8).flatten(), bool) with Image.open(src).convert("YCbCr") as f: w, h = f.size # get width and height y, u, v = f.split() # get Y, U and V # split Y, U and V into blocks of 8*8 matrix y = toBlocks(np.array(y, dtype=np.double) - 128, 8, 8) u = toBlocks(np.array(u, dtype=np.double) - 128, 8, 8) v = toBlocks(np.array(v, dtype=np.double) - 128, 8, 8) prevYDC = prevUDC = prevVDC = 0 frequencyBefore = {} frequencyAfter = {} stream = BitStream() for yBlock, uBlock, vBlock in zip(y, u, v): # Y should be quantized yDCT = np.divide(cv2.dct(yBlock).flatten(), quantizationTable).round().astype(np.int) updateFrequency(yDCT, frequencyBefore) # update `frequencyBefore` write(yDCT, bits, writer) # write message bits to DCT updateFrequency(yDCT, frequencyAfter) # update `frequencyAfter` # U and V are not necessary to be quantized because all of them are formed with 0s uDCT = cv2.dct(uBlock).flatten().round().astype(np.int) vDCT = cv2.dct(vBlock).flatten().round().astype(np.int) # zigzag yAC = yDCT[zigzagOrder] uAC = uDCT[zigzagOrder] vAC = vDCT[zigzagOrder] # huffman encode huffmanEncode(stream, yAC[0] - prevYDC, yAC[1:], True) huffmanEncode(stream, uAC[0] - prevUDC, uAC[1:], False) huffmanEncode(stream, vAC[0] - prevVDC, vAC[1:], False) # store as previous DC value prevYDC = yAC[0] prevUDC = uAC[0] prevVDC = vAC[0] # pad by 1 stream.write(np.ones(8 - len(stream) % 8), bool) with open(dst, "wb+") as f: # write metadata f.write(SOI + APP0 + DQT0 + DQT1 + SOF0_PREFIX + pack(">H", h) + pack(">H", w) + SOF0_SUFFIX + DHT + SOS) # write image data for i in stream.read(bytes): f.write(pack("B", i)) if i == 0xff: f.write(pack("B", 0)) # write EOI f.write(EOI) return frequencyBefore, frequencyAfter
def main(inFileName='./output/output.npy', outFileName='./output/outputFile.b'): zigzagOrder32 = numpy.array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 516, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574, 575, 576, 577, 578, 579, 580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594, 595, 596, 597, 598, 599, 600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611, 612, 613, 614, 615, 616, 617, 618, 619, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654, 655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719, 720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 835, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 884, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 900, 901, 902, 903, 904, 905, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, 918, 919, 920, 921, 922, 923, 924, 925, 926, 927, 928, 929, 930, 931, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 942, 943, 944, 945, 946, 947, 948, 949, 950, 951, 952, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 975, 976, 977, 978, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023 ]) zigzagOrder16 = numpy.array([ 0, 1, 16, 32, 17, 2, 3, 18, 33, 48, 64, 49, 34, 19, 4, 5, 20, 35, 50, 65, 80, 96, 81, 66, 51, 36, 21, 6, 7, 22, 37, 52, 67, 82, 97, 112, 128, 113, 98, 83, 68, 53, 38, 23, 8, 9, 24, 39, 54, 69, 84, 99, 114, 129, 144, 160, 145, 130, 115, 100, 85, 70, 55, 40, 25, 10, 11, 26, 41, 56, 71, 86, 101, 116, 131, 146, 161, 176, 192, 177, 162, 147, 132, 117, 102, 87, 72, 57, 42, 27, 12, 13, 28, 43, 58, 73, 88, 103, 118, 133, 148, 163, 178, 193, 208, 224, 209, 194, 179, 164, 149, 134, 119, 104, 89, 74, 59, 44, 29, 14, 15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, 210, 225, 240, 241, 226, 211, 196, 181, 166, 151, 136, 121, 106, 91, 76, 61, 46, 31, 47, 62, 77, 92, 107, 122, 137, 152, 167, 182, 197, 212, 227, 242, 243, 228, 213, 198, 183, 168, 153, 138, 123, 108, 93, 78, 63, 79, 94, 109, 124, 139, 154, 169, 184, 199, 214, 229, 244, 245, 230, 215, 200, 185, 170, 155, 140, 125, 110, 95, 111, 126, 141, 156, 171, 186, 201, 216, 231, 246, 247, 232, 217, 202, 187, 172, 157, 142, 127, 143, 158, 173, 188, 203, 218, 233, 248, 249, 234, 219, 204, 189, 174, 159, 175, 190, 205, 220, 235, 250, 251, 236, 221, 206, 191, 207, 222, 237, 252, 253, 238, 223, 239, 254, 255 ]) zigzagOrder8 = numpy.array([ 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63 ]) zigzagOrder4 = numpy.array( [0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15]) # 不使用科学计数法输出 不输出省略号 浮点输出2位小数 numpy.set_printoptions(suppress=True, threshold=numpy.inf, precision=2) bitStream = BitStream() # 比特流 # 读取数据 inputData = numpy.load(inFileName).squeeze() minV = inputData.min() maxV = inputData.max() #print('原始数据的最小值最大值分别为', minV, maxV) ''' meanList = numpy.zeros(shape=[inputData.shape[0]], dtype=int) # 保存每个通道的均值 for i in range(inputData.shape[0]): # print('--------',i,'--------') meanV = int(inputData[i].mean()) inputData[i] = inputData[i] - meanV meanList[i] = meanV print('各个通道的均值', meanList) ''' ''' zCode = [] for i in range(inputData.shape[0]): if (inputData.shape[1] == 32): zCode.extend((inputData[i].flatten()[zigzagOrder32]).tolist()) if (inputData.shape[1] == 16): zCode.extend((inputData[i].flatten()[zigzagOrder16]).tolist()) if (inputData.shape[1] == 8): zCode.extend((inputData[i].flatten()[zigzagOrder8]).tolist()) if (inputData.shape[1] == 4): zCode.extend((inputData[i].flatten()[zigzagOrder4]).tolist()) zCode = numpy.asarray(zCode) ''' bitStream = BitStream() huffmanEncodeFunction.valueHuffmanEncode(inputData.flatten(), bitStream) outputFile = open(outFileName, 'wb+') # write encoded data bitLength = bitStream.__len__() filledNum = 8 - bitLength % 8 if (filledNum != 0): bitStream.write(numpy.ones([filledNum]).tolist(), bool) # 补全为字节(b的数量应该是8整数倍) sosBytes = bitStream.read(bytes) for i in range(len(sosBytes)): outputFile.write(bytes([sosBytes[i]])) # if(sosBytes[i]==255): # outputFile.write(bytes([0])) # FF to FF 00 outputFile.close()
#Demo of a rice endoder #Gerald Schuller, Aug. 2018 #for installation: sudo pip install audio.coders from audio.coders import rice from bitstream import BitStream import numpy as np origs = np.arange(-2, 6) print("Original= ", origs) #b: exponent of 2 ricecode = rice(b=1, signed=True) riceencoded = BitStream(origs.astype(np.int32), ricecode) print("rice encoded=", riceencoded) for index in origs: print("Index: ", index, "Rice code: ", BitStream(index, ricecode)) ricedecoded = riceencoded.read(ricecode, 8) print("rice decoded=", ricedecoded)
def main(): # inputBMPFileName outputJPEGFilename quality(from 1 to 100) DEBUGMODE(0 or 1) # example: # ./lena.bmp ./output.jpg 80 0 if (len(sys.argv) != 5): print( 'inputBMPFileName outputJPEGFilename quality(from 1 to 100) DEBUGMODE(0 or 1)' ) print('example:') print('./lena.bmp ./output.jpg 80 0') return srcFileName = sys.argv[1] outputJPEGFileName = sys.argv[2] quality = float(sys.argv[3]) DEBUG_MODE = int(sys.argv[4]) numpy.set_printoptions(threshold=numpy.inf) srcImage = Image.open(srcFileName) srcImageWidth, srcImageHeight = srcImage.size print('srcImageWidth = %d srcImageHeight = %d' % (srcImageWidth, srcImageHeight)) print('srcImage info:\n', srcImage) srcImageMatrix = numpy.asarray(srcImage) imageWidth = srcImageWidth imageHeight = srcImageHeight # add width and height to %8==0 if (srcImageWidth % 8 != 0): imageWidth = srcImageWidth // 8 * 8 + 8 if (srcImageHeight % 8 != 0): imageHeight = srcImageHeight // 8 * 8 + 8 print('added to: ', imageWidth, imageHeight) # copy data from srcImageMatrix to addedImageMatrix addedImageMatrix = numpy.zeros((imageHeight, imageWidth, 3), dtype=numpy.uint8) for y in range(srcImageHeight): for x in range(srcImageWidth): addedImageMatrix[y][x] = srcImageMatrix[y][x] # split y u v yImage, uImage, vImage = Image.fromarray(addedImageMatrix).convert( 'YCbCr').split() yImageMatrix = numpy.asarray(yImage).astype(int) uImageMatrix = numpy.asarray(uImage).astype(int) vImageMatrix = numpy.asarray(vImage).astype(int) if (DEBUG_MODE == 1): print('yImageMatrix:\n', yImageMatrix) print('uImageMatrix:\n', uImageMatrix) print('vImageMatrix:\n', vImageMatrix) yImageMatrix = yImageMatrix - 127 uImageMatrix = uImageMatrix - 127 vImageMatrix = vImageMatrix - 127 if (quality <= 0): quality = 1 if (quality > 100): quality = 100 if (quality < 50): qualityScale = 5000 / quality else: qualityScale = 200 - quality * 2 luminanceQuantTbl = numpy.array( numpy.floor((std_luminance_quant_tbl * qualityScale + 50) / 100)) luminanceQuantTbl[luminanceQuantTbl == 0] = 1 luminanceQuantTbl[luminanceQuantTbl > 255] = 255 luminanceQuantTbl = luminanceQuantTbl.reshape([8, 8]).astype(int) print('luminanceQuantTbl:\n', luminanceQuantTbl) chrominanceQuantTbl = numpy.array( numpy.floor((std_chrominance_quant_tbl * qualityScale + 50) / 100)) chrominanceQuantTbl[chrominanceQuantTbl == 0] = 1 chrominanceQuantTbl[chrominanceQuantTbl > 255] = 255 chrominanceQuantTbl = chrominanceQuantTbl.reshape([8, 8]).astype(int) print('chrominanceQuantTbl:\n', chrominanceQuantTbl) blockSum = imageWidth // 8 * imageHeight // 8 yDC = numpy.zeros([blockSum], dtype=int) uDC = numpy.zeros([blockSum], dtype=int) vDC = numpy.zeros([blockSum], dtype=int) dyDC = numpy.zeros([blockSum], dtype=int) duDC = numpy.zeros([blockSum], dtype=int) dvDC = numpy.zeros([blockSum], dtype=int) print('blockSum = ', blockSum) sosBitStream = BitStream() blockNum = 0 for y in range(0, imageHeight, 8): for x in range(0, imageWidth, 8): print('block (y,x): ', y, x, ' -> ', y + 8, x + 8) yDctMatrix = fftpack.dct(fftpack.dct(yImageMatrix[y:y + 8, x:x + 8], norm='ortho').T, norm='ortho').T uDctMatrix = fftpack.dct(fftpack.dct(uImageMatrix[y:y + 8, x:x + 8], norm='ortho').T, norm='ortho').T vDctMatrix = fftpack.dct(fftpack.dct(vImageMatrix[y:y + 8, x:x + 8], norm='ortho').T, norm='ortho').T if (blockSum <= 8): print('yDctMatrix:\n', yDctMatrix) print('uDctMatrix:\n', uDctMatrix) print('vDctMatrix:\n', vDctMatrix) yQuantMatrix = numpy.rint(yDctMatrix / luminanceQuantTbl) uQuantMatrix = numpy.rint(uDctMatrix / chrominanceQuantTbl) vQuantMatrix = numpy.rint(vDctMatrix / chrominanceQuantTbl) if (DEBUG_MODE == 1): print('yQuantMatrix:\n', yQuantMatrix) print('uQuantMatrix:\n', uQuantMatrix) print('vQuantMatrix:\n', vQuantMatrix) yZCode = yQuantMatrix.reshape([64])[zigzagOrder] uZCode = uQuantMatrix.reshape([64])[zigzagOrder] vZCode = vQuantMatrix.reshape([64])[zigzagOrder] yZCode = yZCode.astype(numpy.int) uZCode = uZCode.astype(numpy.int) vZCode = vZCode.astype(numpy.int) yDC[blockNum] = yZCode[0] uDC[blockNum] = uZCode[0] vDC[blockNum] = vZCode[0] if (blockNum == 0): dyDC[blockNum] = yDC[blockNum] duDC[blockNum] = uDC[blockNum] dvDC[blockNum] = vDC[blockNum] else: dyDC[blockNum] = yDC[blockNum] - yDC[blockNum - 1] duDC[blockNum] = uDC[blockNum] - uDC[blockNum - 1] dvDC[blockNum] = vDC[blockNum] - vDC[blockNum - 1] # huffman encode https://www.impulseadventure.com/photo/jpeg-huffman-coding.html # encode yDC if (DEBUG_MODE == 1): print("encode dyDC:", dyDC[blockNum]) sosBitStream.write( huffmanEncode.encodeDCToBoolList(dyDC[blockNum], 1, DEBUG_MODE), bool) # encode yAC if (DEBUG_MODE == 1): print("encode yAC:", yZCode[1:]) huffmanEncode.encodeACBlock(sosBitStream, yZCode[1:], 1, DEBUG_MODE) # encode uDC if (DEBUG_MODE == 1): print("encode duDC:", duDC[blockNum]) sosBitStream.write( huffmanEncode.encodeDCToBoolList(duDC[blockNum], 0, DEBUG_MODE), bool) # encode uAC if (DEBUG_MODE == 1): print("encode uAC:", uZCode[1:]) huffmanEncode.encodeACBlock(sosBitStream, uZCode[1:], 0, DEBUG_MODE) # encode vDC if (DEBUG_MODE == 1): print("encode dvDC:", dvDC[blockNum]) sosBitStream.write( huffmanEncode.encodeDCToBoolList(dvDC[blockNum], 0, DEBUG_MODE), bool) # encode uAC if (DEBUG_MODE == 1): print("encode vAC:", vZCode[1:]) huffmanEncode.encodeACBlock(sosBitStream, vZCode[1:], 0, DEBUG_MODE) blockNum = blockNum + 1 jpegFile = open(outputJPEGFileName, 'wb+') # write jpeg header jpegFile.write( huffmanEncode.hexToBytes('FFD8FFE000104A46494600010100000100010000')) # write y Quantization Table jpegFile.write(huffmanEncode.hexToBytes('FFDB004300')) luminanceQuantTbl = luminanceQuantTbl.reshape([64]) jpegFile.write(bytes(luminanceQuantTbl.tolist())) # write u/v Quantization Table jpegFile.write(huffmanEncode.hexToBytes('FFDB004301')) chrominanceQuantTbl = chrominanceQuantTbl.reshape([64]) jpegFile.write(bytes(chrominanceQuantTbl.tolist())) # write height and width jpegFile.write(huffmanEncode.hexToBytes('FFC0001108')) hHex = hex(srcImageHeight)[2:] while len(hHex) != 4: hHex = '0' + hHex jpegFile.write(huffmanEncode.hexToBytes(hHex)) wHex = hex(srcImageWidth)[2:] while len(wHex) != 4: wHex = '0' + wHex jpegFile.write(huffmanEncode.hexToBytes(wHex)) # 03 01 11 00 02 11 01 03 11 01 # 1:1 01 11 00 02 11 01 03 11 01 # 1:2 01 21 00 02 11 01 03 11 01 # 1:4 01 22 00 02 11 01 03 11 01 # write Subsamp jpegFile.write(huffmanEncode.hexToBytes('03011100021101031101')) #write huffman table jpegFile.write( huffmanEncode.hexToBytes( 'FFC401A20000000701010101010000000000000000040503020601000708090A0B0100020203010101010100000000000000010002030405060708090A0B1000020103030204020607030402060273010203110400052112314151061361227181143291A10715B14223C152D1E1331662F0247282F12543345392A2B26373C235442793A3B33617546474C3D2E2082683090A181984944546A4B456D355281AF2E3F3C4D4E4F465758595A5B5C5D5E5F566768696A6B6C6D6E6F637475767778797A7B7C7D7E7F738485868788898A8B8C8D8E8F82939495969798999A9B9C9D9E9F92A3A4A5A6A7A8A9AAABACADAEAFA110002020102030505040506040803036D0100021103042112314105511361220671819132A1B1F014C1D1E1234215526272F1332434438216925325A263B2C20773D235E2448317549308090A18192636451A2764745537F2A3B3C32829D3E3F38494A4B4C4D4E4F465758595A5B5C5D5E5F5465666768696A6B6C6D6E6F6475767778797A7B7C7D7E7F738485868788898A8B8C8D8E8F839495969798999A9B9C9D9E9F92A3A4A5A6A7A8A9AAABACADAEAFA' )) # SOS Start of Scan # yDC yAC uDC uAC vDC vAC sosLength = sosBitStream.__len__() filledNum = 8 - sosLength % 8 if (filledNum != 0): sosBitStream.write(numpy.ones([filledNum]).tolist(), bool) jpegFile.write(bytes([255, 218, 0, 12, 3, 1, 0, 2, 17, 3, 17, 0, 63, 0])) # FF DA 00 0C 03 01 00 02 11 03 11 00 3F 00 # write encoded data sosBytes = sosBitStream.read(bytes) for i in range(len(sosBytes)): jpegFile.write(bytes([sosBytes[i]])) if (sosBytes[i] == 255): jpegFile.write(bytes([0])) # FF to FF 00 # write end symbol jpegFile.write(bytes([255, 217])) # FF D9 jpegFile.close()