def decode(self, in_stream, out_stream): bs = BitStream() dq = deque() at_least_three = False for word in self.words_from_file(in_stream): if not word or word not in self.word_dict: continue #print >> sys.stderr, 'word:"', word, '"' dq.append(self.word_dict[word]) if at_least_three or len(dq) == 3: bs.append(pack(self.int_type, dq.popleft())) at_least_three = True if bs.len > self.bit_buffer: cut = 0 for byte in bs.cut(self.bit_buffer): cut += 1 byte.tofile(out_stream) del bs[:cut * self.bit_buffer] # dq has to have exactly 2 elements here, the last is the bit length of the first, unless it's 0 #print >> sys.stderr, 'dq:', dq extra_bits = dq.pop() bs.append(pack('uint:' + str(extra_bits), dq.popleft())) bs.tofile(out_stream)
def _blocksToBits(self, howMany, paletteType): '''This is an internal method that, based on how many blocks it was told to scan as well as the palette type used, will scan that amount on the image and return those converted bits. ''' bitString = BitStream() activeColorSet = self.paletteDict[paletteType].colorSet activePaletteDict = self.paletteConversionDict[paletteType] for block in range(howMany): blockCoords = next(self.nextBlock) rawRGB = scanBlock(self.image, self.pixelWidth, blockCoords[0], blockCoords[1]) if activeColorSet: bitString.append( activePaletteDict.getValue( colorSnap(rawRGB, activeColorSet))) else: bitString.append(activePaletteDict.getValue(rawRGB)) self.blockPosition += howMany config.statsHandler.blocksRead += howMany return bitString
def _decode_nalu(self, nalu_bytes): """ Returns nal_unit_type and RBSP payload from a NALU stream """ if "0x" + nalu_bytes[0: 4*8].hex == self.START_CODE_PREFIX: start_code = nalu_bytes.read('bytes:4') else: start_code = nalu_bytes.read('bytes:3') forbidden_zero_bit = nalu_bytes.read(1) nal_ref_idc = nalu_bytes.read('uint:2') nal_unit_type = nalu_bytes.read('uint:5') nal_unit_payload = nalu_bytes[nalu_bytes.pos:] rbsp_payload = BitStream() for i in range(int(len(nal_unit_payload)/8)): if len(nal_unit_payload) - nal_unit_payload.pos >= 24 and \ nal_unit_payload.peek('bits:24') == '0x000003': rbsp_payload.append(nal_unit_payload.read('bits:8')) rbsp_payload.append(nal_unit_payload.read('bits:8')) nal_unit_payload.read('bits:8') else: if nal_unit_payload.pos == len(nal_unit_payload): continue rbsp_payload.append(nal_unit_payload.read('bits:8')) return nal_unit_type, rbsp_payload
def encode(self, in_stream, out_stream): extra_bits = self.num_bits bs = BitStream() try: while True: chunk = in_stream.read(self.byte_buffer) #print >> sys.stderr, 'chunk:', chunk if(chunk): bs.append(BitStream(bytes=chunk)) else: while True: self.print_index(bs.read(self.int_type), out_stream) try: while True: self.print_index(bs.read(self.int_type), out_stream) except ReadError, e: #print >> sys.stderr, 'inner:', e pass except ReadError, e: #print >> sys.stderr, 'outer:', e extra_bits = bs.len - bs.bitpos if extra_bits > 0: #print >> sys.stderr, 'extra_bits:', extra_bits self.print_index(bs.read('uint:' + str(extra_bits)), out_stream) else: extra_bits = self.num_bits
def encode(self, in_stream, out_stream): extra_bits = self.num_bits bs = BitStream() try: while True: chunk = in_stream.read(self.byte_buffer) #print >> sys.stderr, 'chunk:', chunk if (chunk): bs.append(BitStream(bytes=chunk)) else: while True: self.print_index(bs.read(self.int_type), out_stream) try: while True: self.print_index(bs.read(self.int_type), out_stream) except ReadError, e: #print >> sys.stderr, 'inner:', e pass except ReadError, e: #print >> sys.stderr, 'outer:', e extra_bits = bs.len - bs.bitpos if extra_bits > 0: #print >> sys.stderr, 'extra_bits:', extra_bits self.print_index(bs.read('uint:' + str(extra_bits)), out_stream) else: extra_bits = self.num_bits
def returnFrameHeader(self, paletteType): '''This method returns the bits carrying the frame header for the frame, as well as the "carry over" bits, which were the excess capacity within those blocks, assuming there was extra space. ''' # always 608 bits, plus whatever remainder bits that may be present in the final block. Protocol v1 only! if paletteType != 'streamPalette' and paletteType != 'headerPalette' and paletteType != 'primaryPalette': raise ValueError( "FrameHandler.returnFrameHeader: invalid paletteType argument." ) fullBlockData = self._blocksToBits( math.ceil(608 / self.paletteDict[paletteType].bitLength), f'{paletteType}') carryOverBits = BitStream() if fullBlockData.len > 608: fullBlockData.pos = 608 carryOverBits.append( fullBlockData.read(f'bits : {fullBlockData.len - 608}')) fullBlockData.pos = 0 config.statsHandler.dataRead += carryOverBits.len return fullBlockData.read('bits : 608'), carryOverBits
def encode(block): """ Entropy of CAVLC Args: block: input macroblock, should be 4x4 intger matrix returns: A bitstream of CAVLC code """ #use Zigzag to scan zig = ZigZag.ZigzagMatrix() print("ZiaZag scan:") res = zig.matrix2zig(block) print(res) #Step1: get TotalCoeffs& T1s totalCoeffs = getTotalCoeffs(res) print("TotalCoeffs: ", totalCoeffs) if (totalCoeffs==0): NoFurther = BitStream('0b0') print("CAVLC: ", NoFurther.bin) return NoFurther t1s = getT1s(res) print("T1s: ", t1s) part1 = '0b' + vlc.coeff_token[0][totalCoeffs][t1s] print("coeff token:", part1) #step2: Encode the sign of each T1 part2, remains = encodeT1s(res, t1s) print("T1 sign:", part2) #step3: Encode the levels of the remaining non-zero coefficients part3 = encodeLevels(remains, totalCoeffs, t1s) print("encode Levels:", part3) #Step4: Encode the total number of zeros before the last coefficient totalZeros = getTotalZeros(res) print("TotalZeros: ", totalZeros) part4 = '' if ( totalCoeffs<block.size ): part4 = vlc.total_zeros[totalZeros][totalCoeffs] print("encode TotalZeros: ", part4) #step5: Encode each run of zeros part5 = encodeRunBefore(res, totalCoeffs, totalZeros) stream = BitStream() temp = part1 + part2 + part3 + part4 + part5 stream.append(temp) logging.debug("CAVLC: %s", stream.bin) # supplement zero at the end of stream, so we can print the hex code of the stream output_str = stream addon = 8 - len(stream.bin) % 8 for i in range(0, addon): output_str.append('0b0') logging.debug("CAVLC hex: %s", output_str.hex) return stream
class bit_file: def __init__(self, f, bits_per_op, mode='read', keep_open=False, read_size=MAX_ENCODING, read_count=1): if mode not in ['read', 'write']: raise Exception('bad bit_file mode. Try "read" or "write"') self.mode = mode if isinstance(f, str): fmode = 'wb' if mode == 'write' else 'rb' self.f = open(f, fmode) self.keep_open = False else: self.f = f self.keep_open = keep_open # determines whether __exit__ is a flush()+close(), or just a flush() self.bits_per_op = bits_per_op self.stream = BitStream() self.read_size = read_size self.read_count = read_count def __enter__(self): return self def __exit__(self, type, value, traceback): if self.mode == 'write' and not self.f.closed: self.save() if not self.keep_open and not self.f.closed: with self.f: # close file pass def write(self, bits): if isinstance(bits, bit_write_buffer): bits = bits.stream if not isinstance(bits, Bits): bits = Bits(uint=bits, length=self.bits_per_op) self.stream.append(bits) def read(self): if self.read_count and self.stream.bitpos == self.stream.length: self.stream.clear() self.stream.append(Bits(bytes=self.f.read(self.read_size))) self.read_count -= 1 try: bits = self.stream.read(f'uint:{self.bits_per_op}') except bitstring.ReadError: try: bits = self.stream.read('uint') except bitstring.InterpretError: bits = 0 return bits def save(self): self.f.write(self.stream.tobytes())
def decode(lst): q = BitStream() temp = BitArray('0b0') for bit, count in lst: for i in xrange(count): temp.bool = bit q.append(temp) return q
def sx(stat: EnergyProfileStat): buffer = BitStream() to_add = encode_7bit_long(round(stat.kw_in * 1000)) to_add.extend(encode_7bit_long(round(stat.kw_out * 1000))) to_add.extend(encode_7bit_long(round(stat.kw_net * 1000))) for thing in to_add: buffer.append(thing) buffer.bytepos = 0 return buffer
def RTCM_converter_thread(server, port, username, password, mountpoint, rtcm_callback=None): import subprocess nt = subprocess.Popen([ "./ntripclient", "--server", server, "--password", password, "--user", username, "--mountpoint", mountpoint ], stdout=subprocess.PIPE) """nt = subprocess.Popen(["./ntrip.py", server, str(port), username, password, mountpoint], stdout=subprocess.PIPE)""" if nt is None or nt.stdout is None: indev = sys.stdin else: indev = nt.stdout print("RTCM using input {}".format(indev)) while True: sio = indev d = ord(sio.read(1)) if d != RTCMv3_PREAMBLE: continue pack_stream = BitStream() l1 = ord(sio.read(1)) l2 = ord(sio.read(1)) pack_stream.append(bs.pack('2*uint:8', l1, l2)) pack_stream.read(6) pkt_len = pack_stream.read(10).uint pkt = sio.read(pkt_len) parity = sio.read(3) if len(pkt) != pkt_len: print("Length error {} {}".format(len(pkt), pkt_len)) continue if True: #TODO check parity for d in pkt: pack_stream.append(bs.pack('uint:8', ord(d))) msg = parse_rtcmv3(pack_stream) if msg is not None and rtcm_callback is not None: rtcm_callback(msg)
class bit_write_buffer(): def __init__(self, bits_per_op, **kwargs): super().__init__() self.stream = BitStream() self.bits_per_op = bits_per_op def write(self, bits): b1 = Bits(uint=bits, length=self.bits_per_op) prev = len(self.stream) self.stream.append(b1) return len(self.stream) - prev
def RTCM_converter_thread(server, port, username, password, mountpoint, rtcm_callback = None): import subprocess nt = subprocess.Popen(["./ntripclient", "--server", server, "--password", password, "--user", username, "--mountpoint", mountpoint ], stdout=subprocess.PIPE) """nt = subprocess.Popen(["./ntrip.py", server, str(port), username, password, mountpoint], stdout=subprocess.PIPE)""" if nt is None or nt.stdout is None: indev = sys.stdin else: indev = nt.stdout print("RTCM using input {}".format(indev)) while True: sio = indev d = ord(sio.read(1)) if d != RTCMv3_PREAMBLE: continue pack_stream = BitStream() l1 = ord(sio.read(1)) l2 = ord(sio.read(1)) pack_stream.append(bs.pack('2*uint:8', l1, l2)) pack_stream.read(6) pkt_len = pack_stream.read(10).uint pkt = sio.read(pkt_len) parity = sio.read(3) if len(pkt) != pkt_len: print "Length error {} {}".format(len(pkt), pkt_len) continue if True: #TODO check parity for d in pkt: pack_stream.append(bs.pack('uint:8',ord(d))) msg = parse_rtcmv3(pack_stream) if msg is not None and rtcm_callback is not None: rtcm_callback(msg)
def update(self): bits = BitStream() if self.value == "NORMAL": value = 1 elif self.value == "NIGHT": value = 2 elif self.value == "STANDBY": value = 3 elif self.value == "RESTART": value = 4 elif self.value == "RESTART": value = 5 bits.append(Bits(f"uint:8={value}")) self.rawdata = bits.bytes
def encoding16x16(block, QP): """ Encode a 16x16 macroblock Args: block: 16x16 matrix block QP: the QP value of quantization Returns: encoding of current macroblock """ size = block.shape step = 4 result = BitStream() # step1: Get the DC element of each 4x4 block DC_block = np.full((step, step), 0) for i in r_[:size[0]:step]: for j in r_[:size[1]:step]: x = int(i / step) y = int(j / step) DC_block[x][y] = block[i, j] # DC transorm coding logging.debug("16x16 block's DC transorm coding") dc_trans = tf.forwardHadamardAndScaling4x4(DC_block, QP) dc_code = cd.CAVLC(dc_trans) result.append(dc_code) # step2: 4x4 transform, quantization and coding current = np.full((step, step), 0) for m in r_[:size[0]:8]: for n in r_[:size[1]:8]: for i in r_[:8:step]: for j in r_[:8:step]: x = m + i y = n + j current = block[x:(x + step), y:(y + step)] logging.debug("4x4 block row %d column %d, pixel value:", x, y) logging.debug(current) temp = tf.forwardTransformAndScaling4x4(current, QP) logging.debug("coefficients:") logging.debug(temp) ac_code = cd.CAVLC(temp) result.append(ac_code) # step3: UV coding temp = encoding16x16UV(QP) # U result.append(temp) temp = encoding16x16UV(QP) # V result.append(temp) return result
def encode(data): stream = BitStream() stream.append('int:32=%d' % MAGIC_NUM) stream.append('int:8=%d' % data['version']) stream.append('int:8=%d' % data['command']) json_bytes = json.dumps(data).encode() stream.append('int:32=%d' % len(json_bytes)) stream.append(json_bytes) return stream.bytes
def sx(self): buffer = BitStream() buffer.append(f"uint:32={self.from_.year}") buffer.append(f"uint:8={self.from_.month}") buffer.append(f"uint:8={self.from_.day}") buffer.append(f"uint:32={self.to.year}") buffer.append(f"uint:8={self.to.month}") buffer.append(f"uint:8={self.to.day}") buffer.bytepos = 0 return buffer
def encoding16x16UV(QP): """ Encode a 8x8 U、V block Args: QP: the QP value of quantization Returns: encoding of current U、V macroblock """ block = np.full((8, 8), 0) # set all UV to zero,TODO:pass in real UV value step = 4 result = BitStream() size = block.shape # step1: Get the DC element of each 4x4 block DC_block = np.full((step, step), 0) for i in r_[:size[0]:step]: for j in r_[:size[1]:step]: x = int(i / step) y = int(j / step) DC_block[x][y] = block[i, j] # DC transorm coding logging.debug("8x8 UV block's DC transorm coding") dc_trans = tf.forwardHadamardAndScaling4x4(DC_block, QP) dc_code = cd.CAVLC(dc_trans) result.append(dc_code) # step2: 4x4 transform, quantization and coding current = np.full((step, step), 0) for i in r_[:size[0]:step]: for j in r_[:size[1]:step]: current = block[i:(i + step), j:(j + step)] logging.debug("4x4 UV block row %d column %d, pixel value:", i, j) logging.debug(current) temp = tf.forwardTransformAndScaling4x4(current, QP) logging.debug("coefficients:") logging.debug(temp) ac_code = cd.CAVLC(temp) result.append(ac_code) return result
def encode(word): # we build a dictionnary with all possible symbols and associate each of them with a number symbols ={} for letter in string.printable: bits = BitArray(BITS_USED) bits.uint = len(symbols) symbols[letter] = bits # the code we'll return code = BitStream() # a sequence of characters sequence = "" # for each character in the word we are encoding for c in word: # we add it to the sequence we are currently considering temp = sequence + c # if we know the result we just keep it and parse the next character if temp in symbols: sequence = temp # otherwise else: # we add the previously known sequence to the code code.append(symbols[sequence]) # and add the new sequence to our dictionnary of known sequences if len(symbols) < 1<<BITS_USED: bits = BitArray(BITS_USED) bits.uint = len(symbols) symbols[temp] = bits # the sequence is reset to only the new character sequence = c # there is usually a last unincoded sequence to be added code.append(symbols[sequence]) return code
def decrypt(ciphertext, key): bstream = BitStream() p, g, y = key[0] u = key[1] #trying to improve execution speed #a_pow_u = mod_exp(ciphertext[1][0], u, p) #inv_a_pow_u = modinv(a_pow_u, p) for block in ciphertext[1:]: #sys.stdout.write(".") #trying to improve execution speed a_pow_u = mod_exp(block[0], u, p) inv_a_pow_u = modinv(a_pow_u, p) x = (block[1] * inv_a_pow_u) % p block_size = math.floor(math.log(p, 2)) bstream.append('0b' + bin(x)[2:].zfill(int(block_size))) return bstream.read(ciphertext[0])
def decrypt(ciphertext, key): bstream = BitStream() p, g, y = key[0] u = key[1] #trying to improve execution speed #a_pow_u = mod_exp(ciphertext[1][0], u, p) #inv_a_pow_u = modinv(a_pow_u, p) for block in ciphertext[1:]: #sys.stdout.write(".") #trying to improve execution speed a_pow_u = mod_exp(block[0], u, p) inv_a_pow_u = modinv(a_pow_u, p) x = (block[1] * inv_a_pow_u) % p block_size = math.floor(math.log(p,2)) bstream.append('0b' + bin(x)[2:].zfill(int(block_size))) return bstream.read(ciphertext[0])
def pack_pak(dir, file_list = None, align_toc = 16, align_files = 16, eof = False): if file_list == None: file_list = sorted(os.listdir(dir)) num_files = len(file_list) toc_length = (num_files + 1) * 4 if eof: toc_length += 1 if toc_length % align_toc > 0: toc_length += align_toc - (toc_length % align_toc) archive_data = BitStream(uintle = 0, length = toc_length * 8) archive_data.overwrite(bitstring.pack("uintle:32", num_files), 0) for file_num, item in enumerate(file_list): full_path = os.path.join(dir, item) if os.path.isfile(full_path): data = pack_file(full_path) else: data = pack_dir(full_path, align_toc, align_files, eof) file_size = data.len / 8 padding = 0 if file_size % align_files > 0: padding = align_files - (file_size % align_files) data.append(BitStream(uintle = 0, length = padding * 8)) file_pos = archive_data.len / 8 archive_data.overwrite(bitstring.pack("uintle:32", file_pos), (file_num + 1) * 32) archive_data.append(data) del data if eof: archive_data.overwrite(bitstring.pack("uintle:32", archive_data.len / 8), (num_files + 1) * 32) return archive_data
def bit_rep(self, file_path): key = Fernet.generate_key() f = Fernet(key) offsite_bits = BitStream() with open (file_path,'rb') as file: token = f.encrypt(file.read()) data_stream = BitStream(bytes=token) # traverse all of the data stream in steps of size word_length # While data_stream.cut(word_length) is probably more efficient, value of n is useful here for n in range(0, len(data_stream), self.word_len): for m in range(0, self.word_len): if m in self.pos_list: if data_stream[n+m] is True: offsite_bits.append('0b1') else: offsite_bits.append('0b0') # Using urandom because of cryptographic security and because it is supported on major OSs # Replaces the bits at bit_pos (within each word) with randomly generated bits. data_stream.overwrite(bin(ord(urandom(1)) % 2), m+n) return data_stream, offsite_bits, key
def encode(im, QP): """ The core process of H264 encoding Args: im: the input frame returns: The binary code of the frame """ predict, residual, mode_map = prediction.IntraPrediction( im, 16) # 16x16 intra mode #according to page 133 Figure 8-6 totalMacro = BitStream() step = 16 imsize = residual.shape for i in r_[:imsize[0]:step]: for j in r_[:imsize[1]:step]: logging.debug("16x16 block index row %d, column %d", i, j) block16x16 = residual[i:(i + step), j:(j + step)] macro = encoding16x16(block16x16, QP) I_16x16_2_1_1 = 19 #temp code, TODO: should add some basic prediction type in nalutypes mb = ns.MacroblockLayer( I_16x16_2_1_1 ) #temp code, TODO: should use mode_map to reflect right predict mode mb.set__mb_pred( 0 ) #temp code, should input right parameter of intra_chroma_pred_mode mb.set__residual(macro) temp = mb.gen() totalMacro.append(temp) logging.debug("16x16 block macroblock: %s", temp.bin) logging.debug("-----------------------------------------") return totalMacro
class NaluStreamer(): def __init__(self, nalu_type): ''' init the value of nalu unit : param nalu_type: something like NAL_UNIT_TYPE_CODED_SLICE_IDR in nalutypes.py ''' if (nalu_type != nalutypes.NAL_UNIT_TYPE_UNSPECIFIED): # for specific nalutypes self.forbidden_zero_bit = '0b0' self.nal_ref_idc = '0b11' self.nal_unit_type = "0b" + "{0:05b}".format(nalu_type) self.stream = BitStream(START_CODE_PREFIX) self.stream.append(self.forbidden_zero_bit) self.stream.append(self.nal_ref_idc) self.stream.append(self.nal_unit_type) else: # for slice_data self.stream = BitStream() def rbsp_trailing_bits(self): ''' according to RBSP trainling bits syntax on page 35, and according to explanation page 49 ''' rbsp_stop_one_bit = '0b1' rbsp_alignment_zero_bit = '0b0' self.stream.append(rbsp_stop_one_bit) plus = 8 - (self.stream.length % 8) for x in range(0, plus): self.stream.append(rbsp_alignment_zero_bit) #print("length after plus:") #print(self.stream.length) def export(self, bitstream_output_handler): """ output the binary data into file """ self.stream.tofile(bitstream_output_handler)
def run(self): import subprocess server, port, username, password, mountpoint, rtcm_callback = self._Thread__args nt = subprocess.Popen([ "./ntripclient", "--server", server, "--password", password, "--user", username, "--mountpoint", mountpoint ], stdout=subprocess.PIPE) """nt = subprocess.Popen(["./ntrip.py", server, str(port), username, password, mountpoint], stdout=subprocess.PIPE)""" if nt is None or nt.stdout is None: indev = sys.stdin else: indev = nt.stdout sio = indev while not self._stopevent.isSet(): d = ord(sio.read(1)) if d != RTCMv3_PREAMBLE: continue pack_stream = BitStream() l1 = ord(sio.read(1)) l2 = ord(sio.read(1)) pack_stream.append(bs.pack('2*uint:8', l1, l2)) pack_stream.read(6) pkt_len = pack_stream.read(10).uint pkt = sio.read(pkt_len) parity = sio.read(3) if len(pkt) != pkt_len: print "Length error {} {}".format(len(pkt), pkt_len) continue if True: #TODO check parity for d in pkt: pack_stream.append(bs.pack('uint:8', ord(d))) msg = parse_rtcmv3(pack_stream) # MARCO forward unchanged pack_stream.append( bs.pack('3*uint:8', ord(parity[0]), ord(parity[1]), ord(parity[2]))) msg = struct.pack('<B', RTCMv3_PREAMBLE) + pack_stream.tobytes() if msg is not None and rtcm_callback is not None: rtcm_callback(msg) if nt is not None: nt.terminate() nt.wait() nt = None
def gen(self): """ generate the binary data of this macroblock The sequence here is very important, should be exact the same as SPECIFIC of H.264 """ stream = BitStream() stream.append(self.mb_type) #current we just handle 16x16 mode stream.append(self.mb_pred) stream.append(self.mb_qp_delta) stream.append(self.residual) return stream
def encode(word): # a dictionnary for all possible symbol symbols = {} # counts the number of occurences each symbol for letter in word: if letter in symbols: symbols[letter] += 1 else : symbols[letter] = 1 # here we construct the binary tree with tuples : tree = buildTree(symbols) # we have to return the tree too for decoding later # we could encode it better, but its native string representation is ok treeString = str(tree) treeString = treeString.replace(", ", ",") # the code we'll return, which first contains the length taken by the tree code = BitStream(12) code.int = len(treeString) # then we add the tree to the code codedTree = BitArray(8 * len(treeString)) codedTree.bytes = treeString code.append(codedTree) # a recursive traversal of the tree allows us to create a dictionnary of matching symbols and codes for encoding : symbols = buildEncodingDict(tree) # finally we encode the word codedWord = encodeWithDict(symbols, word) code.append(codedWord) return code
def _blocks_to_bits(self, how_many, palette_type): '''This is an internal method that, based on how many blocks it was told to scan as well as the palette type used, will scan that amount on the image and return those converted bits. ''' bit_string = BitStream() active_color_set = self.palette_dict[palette_type].color_set active_palette_dict = self.palette_conversion_dict[palette_type] for block in range(how_many): block_coords = next(self.next_block) raw_rgb = scan_block(self.image, self.pixel_width, block_coords[0], block_coords[1]) if active_color_set: bit_string.append(active_palette_dict.get_value(color_snap(raw_rgb, active_color_set))) else: bit_string.append(active_palette_dict.get_value(raw_rgb)) self.block_position += how_many config.stats_handler.blocks_read += how_many return bit_string
def dump_to_file(p_scale_vars, seeds, bits_per_block, out_file): """Write the p_scale and the seeds to file in a bit-efficient way Parameters --------- p_scale_vars: list seeds: list bits_per_block: int out_file: str """ # Create the bitstream, stream = BitStream() # Add the p_scale_vars to the bitstream for p_scale_var in p_scale_vars: stream.append(BitStream(float=p_scale_var, length=P_SCALE_LENGTH)) # Add the seeds to the bitstream for seed in seeds: stream.append(BitStream(uint=seed, length=bits_per_block)) # Dump the reads to file with open(out_file, mode='wb+') as f: stream.tofile(f)
def serialize(self): bitstream = BitStream() if self.leaf(): bitstream.append('bin=1') bitstream.append("{0:#0{1}x}".format(self.value, 4)) else: bitstream.append('bin=0') bitstream += self.left.serialize() + self.right.serialize() return bitstream
def update(self): bits = BitStream() for k, v in self.parsemap.items(): if isinstance(v, str): if "int" in v: bits.append(Bits(f"{v}={int(self.value)}")) else: bits.append(Bits(f"{v}={self.value}")) else: bits.append(Bits(bytes=self.value.toBytes())) self.rawdata = bits
def to_bin(commands): data = BitStream() lines = 0 for op, params in commands: if op == WRD_HEADER: continue if not op in OP_PARAMS: # raise Exception("Unknown op: 0x%02X" % op) print "Unknown op: 0x%02X" % op continue param_info = OP_PARAMS[op] # If it has a custom parsing function, use the equivalent packing function. if isinstance(param_info, basestring): command = globals()[OP_FUNCTIONS[op]](**params) data.append(command) else: if op == WRD_SHOW_LINE: lines += 1 data.append(bitstring.pack("uint:8, uint:8", CMD_MARKER, op)) unnamed_param_id = 0 for param_name, param_type in param_info: if param_name == None: data.append( bitstring.pack(param_type, params[param_name][unnamed_param_id])) unnamed_param_id += 1 else: data.append(bitstring.pack(param_type, params[param_name])) return bitstring.pack("uint:8, uint:8, uintle:16", CMD_MARKER, WRD_HEADER, lines) + data
def update(self): bits = BitStream() i = 0 for p in self.parsemap: if isinstance(p, str): ident = f"UNPARSED_{i}" bits.append(Bits(f"{p}={self.values[ident]}")) else: bits.append(Bits(self.values[p.name].toBits())) i += 1 if "UNPARSED_REST" in self.values: rest = self.values["UNPARSED_REST"] bits.append(Bits(f"bin={rest}")) self.rawdata = bits.bytes
def to_bin(commands): data = BitStream() lines = 0 for op, params in commands: if op == WRD_HEADER: continue if not op in OP_PARAMS: # raise Exception("Unknown op: 0x%02X" % op) print "Unknown op: 0x%02X" % op continue param_info = OP_PARAMS[op] # If it has a custom parsing function, use the equivalent packing function. if isinstance(param_info, basestring): command = globals()[OP_FUNCTIONS[op]](**params) data.append(command) else: if op == WRD_SHOW_LINE: lines += 1 data.append(bitstring.pack("uint:8, uint:8", CMD_MARKER, op)) unnamed_param_id = 0 for param_name, param_type in param_info: if param_name == None: data.append(bitstring.pack(param_type, params[param_name][unnamed_param_id])) unnamed_param_id += 1 else: data.append(bitstring.pack(param_type, params[param_name])) return bitstring.pack("uint:8, uint:8, uintle:16", CMD_MARKER, WRD_HEADER, lines) + data
def compress(self, data): weight = Counter(data) priority_queue = [HuffmanNode(value=byte, weight=weight[byte]) for byte in weight] heapify(priority_queue) while len(priority_queue) > 1: left = heappop(priority_queue) right = heappop(priority_queue) node = HuffmanNode(left, right, weight=left.weight + right.weight) heappush(priority_queue, node) root = heappop(priority_queue) dictionary = root.assign_codes() """ we need to add the tree to the compressed data, so that the decompressor can rebuild it in order do to it's work """ tree = root.serialize() result = BitStream() tree_len_bits = len(bin(len(tree))[2:]) if tree_len_bits > 16: raise ValueError("Huffman tree len is max 10*255-1 bit") # this converts len(tree) to hex with zero front pad to two bytes result.append("{0:#0{1}x}".format(len(tree), 6)) result += tree for byte in data: result.append('bin=' + dictionary[byte]) pad = 0 if len(result) % 8 != 0: pad = 8 - len(result) % 8 result.append('bin=' + '0' * pad) """ the compressed data layout is as follows: * 1B - number of pad bits (for byte align) * 2B- Huffman tree length (which btw = 10*num_of_chars_in_the_tree -1) * the Huffman tree itself (not byte aligned) * the encoded data paded with 0-7 bits at the end """ result = BitStream("{0:#0{1}x}".format(pad, 4)) + result return bytearray(result.bytes)
class PacketDecoder(object): HEADER = "0x59" FOOTER = "0x5254464d" LENGTH = 56 * 8 PATTERN = "2BH9I8B2I" packet_store = [] def __init__(self, listener_func): self.listener_func = listener_func self.stream = BitStream() def push(self, byte_array): self.stream.append(byte_array) def handle(self): while True: if len(self.stream) < self.LENGTH: break # Look for a header header_pos = self.stream.find(self.HEADER) if header_pos: header_pos = header_pos[0] else: self.stream.clear() break end_pos = header_pos + self.LENGTH if len(self.stream) < end_pos: # Not Enough Data break # Extract potential packet potential_packet = self.stream[header_pos:end_pos].bytes valid_packet = self.analyze_packet(potential_packet) if valid_packet: self.stream = self.stream[end_pos:] else: self.stream = self.stream[header_pos + 1 :] self.save_packets() self.packet_store = [] def analyze_packet(self, byte_string): if len(byte_string) != 56: return data = struct.unpack(self.PATTERN, byte_string) header = data[0] footer = data[21] if hex(header) == self.HEADER and hex(footer) == self.FOOTER: if self.validate_checksums(data): packet = self.create_packet(data) self.packet_store.append(packet) return True return False def validate_checksums(self, packet_data): # voltage_checksum = packet_data[12] # current_checksum = packet_data[13] # period_checksum = packet_data[14] # active_checksum = packet_data[15] # reactive_checksum = packet_data[16] # apparent_power = packet_data[17] # phase_angle_checksum = packet_data[18] # power_factor_checksum = packet_data[19] # checksum = packet_data[20] return True def create_packet(self, packet_data): flags = packet_data[2] epoch = packet_data[3] voltage = packet_data[4] * (2.37748 * 10 ** -4) + -0.14427 current = packet_data[5] * (113240.82786) + 953.97194 period = packet_data[6] active_power = packet_data[7] reactive_power = packet_data[8] apparent_power = packet_data[9] phase_angle = packet_data[10] power_factor = packet_data[11] return Packet( flags=flags, epoch=epoch, voltage=voltage, current=current, period=period, active_power=active_power, reactive_power=reactive_power, apparent_power=apparent_power, phase_angle=phase_angle, power_factor=power_factor, ) def save_packets(self): # submit packets to lib if self.packet_store: self.listener_func(self.packet_store)
def hash_string(k, p, plaintext): bstream = BitStream() for c in plaintext: bstream.append("0x" + c.encode("hex")) return AHash(k, p, bstream)
def encrypt_string(plaintext, pub_key): bstream = BitStream() for c in plaintext: bstream.append("0x" + c.encode("hex")) return encrypt(bstream, pub_key)
def dataCollect(self, args): """ init of _pf for further data analysis and presentation """ p_process_start= re.compile(r'Event log entries: [[]newest entry first[]]') p_process_found = False skip_lines = 4 self._dataDict = collections.OrderedDict() self._time = 0 self._dataDict['timestampHex'] = [] self._dataDict['timestamp'] = [] self._dataDict['timeLast'] = [] self._dataDict['timeLastInUs'] = [] self._dataDict['processId'] = [] self._dataDict['processName'] = [] self._dataDict['bfn'] = [] self._dataDict['ssn'] = [] self._dataDict['eventId'] = [] self._dataDict['eventIdHex'] = [] self._dataDict['eventName'] = [] self._dataDict['extraInfo'] = [] self._dataDict['extraInfo_parse'] = [] self._dataDict['note'] = [] self._dataDict['origin'] = [] entryList = [] entryIndex = 0 complete_block = False p_close = re.compile(r"^\n") p_data = re.compile(r'\w+\s+(.*)') currentLine = 0 endLine = 0 startLine = 0 if args.maxNumEventEntries > 0: assert(args.endEntry <= args.maxNumEventEntries) bufferEnd = None call("ag --numbers 'Max file size reached, dump aborted:' {} > temp".format(args.logFile), shell=True) if os.path.getsize("./temp") != 0: bufferEnd = check_output("ag --numbers 'Max file size reached, dump aborted:' {}".format(args.logFile), shell=True) os.remove("./temp") if bufferEnd: print('"Max file size reached, dump aborted" found') bufferEnd = bufferEnd.decode('utf-8') endLine = int(bufferEnd.split(":")[0]) - 2 # skip non-data lines directly if args.maxNumEventEntries == 0 and args.endEntry == 0: args.maxNumEventEntries = 30000 print("warning: Max file size reached in ecda dump {}, and dump aborted, filter out latest 30000 entries".format(args.logFile)) elif args.maxNumEventEntries and args.endEntry == 0: print("warning: Max file size reached in ecda dump {}, and dump aborted, filter out latest {} entries".format(args.logFile, args.maxNumEventEntries)) elif args.maxNumEventEntries == 0 and args.endEntry : print("warning: Max file size reached in ecda dump {}, and dump aborted, filter out entries from {} to {}".format(args.logFile, args.startEntry, args.endEntry)) elif args.maxNumEventEntries and args.endEntry : print("warning: Max file size reached in ecda dump {}, and dump aborted, filter out entries from {} to {}".format(args.logFile, args.startEntry, max(args.endEntry, args.startEntry+args.maxNumEventEntries))) bufferStart= None call("ag --numbers 'newest entry first' {} > temp".format(args.logFile), shell=True) if os.path.getsize("./temp") != 0: bufferStart= check_output("ag --numbers 'newest entry first' {}".format("temp"), shell=True) os.remove("./temp") if bufferStart: bufferStart= bufferStart.decode('utf-8') allLines = bufferStart.split("\n") startLine = int(allLines[1].split(":")[1]) + 5 #skip non-data ,include current line else: print("warning: no event log entry found in {}, return without parsing".format(args.logFile)) return if endLine == 0: bufferEnd= check_output("ag --numbers 'End of dump' {}".format(args.logFile), shell=True) assert(bufferEnd) #shall never be None print('"End of dump" found') bufferEnd = bufferEnd.decode('utf-8') endLine = int(bufferEnd.split(":")[0]) - 2 # skip non-data lines directly if args.startEntry < args.endEntry or args.maxNumEventEntries: print("info expect endEntry {}, maxNumEventEntries {} real entriesTotal {}".format(args.endEntry, args.maxNumEventEntries, (endLine-startLine)*2 )) endTemp = max(startLine + (args.endEntry>>1), startLine + (args.maxNumEventEntries>>1)) endLine = min(endTemp, endLine) startLine += (args.startEntry>>1) # skip non-data lines directly print("info: parse event entries from {} to {}".format(args.startEntry, (endLine-startLine)*2+args.startEntry)) eventLogEntryBitStream = BitStream() with open(args.logFile) as f: for line in f: #if p_process_start.search(line) and complete_block is False: # # skip part event log block # complete_block = True # continue ##if p_process_found is False and p_process_start.search(line): ## p_process_found = True ## continue ##if p_process_found and skip_lines > 0: ## skip_lines -= 1 ## continue ##if p_process_found and p_close.search(line): ## p_process_found = False ## break ##if args.maxNumEventEntries : ## if maxNumLines > 0: ## maxNumLines -= 1 ## else: ## break currentLine += 1 #ag counter from 1 if currentLine >= startLine and currentLine < endLine: #[startline,endline) if (currentLine - startLine)%1000 == 0: print("{} entries collected".format((currentLine - startLine)*2)) ##if p_process_found and skip_lines == 0: #m = p_data.search(line) ##print(line) #data = m.group(1).split(" ") #if len(data) == 4: # eventLogEntryBitStream.append('0x'+data[0]) # eventLogEntryBitStream.append('0x'+data[1]) # eventLogEntryBitStream.append('0x'+data[2]) # eventLogEntryBitStream.append('0x'+data[3]) # self.dataEntryAppend(eventLogEntryBitStream) # eventLogEntryBitStream.clear() # entryList.append(entryIndex) # entryIndex += 1 #else: # # two event entries of 64 bits for each # eventLogEntryBitStream.append('0x'+data[0]) # eventLogEntryBitStream.append('0x'+data[1]) # eventLogEntryBitStream.append('0x'+data[2]) # eventLogEntryBitStream.append('0x'+data[3]) # self.dataEntryAppend(eventLogEntryBitStream) # eventLogEntryBitStream.clear() # entryList.append(entryIndex) # entryIndex += 1 # eventLogEntryBitStream.append('0x'+data[4]) # eventLogEntryBitStream.append('0x'+data[5]) # eventLogEntryBitStream.append('0x'+data[6]) # eventLogEntryBitStream.append('0x'+data[7]) # self.dataEntryAppend(eventLogEntryBitStream) # eventLogEntryBitStream.clear() # entryList.append(entryIndex) # entryIndex += 1 data = line.strip().split(" ") # offset '' entries if len(data) == 6: eventLogEntryBitStream.append('0x'+data[2]) eventLogEntryBitStream.append('0x'+data[3]) eventLogEntryBitStream.append('0x'+data[4]) eventLogEntryBitStream.append('0x'+data[5]) self.dataEntryAppend(eventLogEntryBitStream) eventLogEntryBitStream.clear() entryList.append(entryIndex) entryIndex += 1 else: # two event entries of 64 bits for each eventLogEntryBitStream.append('0x'+data[2]) eventLogEntryBitStream.append('0x'+data[3]) eventLogEntryBitStream.append('0x'+data[4]) eventLogEntryBitStream.append('0x'+data[5]) self.dataEntryAppend(eventLogEntryBitStream) eventLogEntryBitStream.clear() entryList.append(entryIndex) entryIndex += 1 eventLogEntryBitStream.append('0x'+data[6]) eventLogEntryBitStream.append('0x'+data[7]) eventLogEntryBitStream.append('0x'+data[8]) eventLogEntryBitStream.append('0x'+data[9]) self.dataEntryAppend(eventLogEntryBitStream) eventLogEntryBitStream.clear() entryList.append(entryIndex) entryIndex += 1 print("--finish 64 bit eventlog entry collection") pass
class NALUnit: def __init__(self, filp, pos, utype, size, tid, nosetup=False): self.filp = filp self.pos = pos self.num_bytes_in_nalu = size self.nal_unit_type = utype self.temporal_id = tid self.rbsp_byte = BitStream() self.setup_rbsp() self.print_bin() if not nosetup: self.setup() def rbsp_read(self, fmt): return self.rbsp_byte.read(fmt) def next_bits(self, fmt, forward=False): try: if not forward: ret = self.filp.peek(fmt) else: ret = self.filp.read(fmt) except ReadError: return 0 return ret def setup_rbsp(self): self.filp.bytepos += 2 self.num_bytes_in_rbsp = 0 i = 2 while i < self.num_bytes_in_nalu: if i + 2 < self.num_bytes_in_nalu and self.next_bits('hex: 24') == '000003': self.rbsp_byte.append(self.filp.read(8)) self.num_bytes_in_rbsp += 1 self.rbsp_byte.append(self.filp.read(8)) self.num_bytes_in_rbsp += 1 # discard emulation_prevention_three_byte self.filp.read(8) i += 3 else: self.rbsp_byte.append(self.filp.read(8)) self.num_bytes_in_rbsp += 1 i += 1 self.filp.bytepos = self.pos def more_rbsp_data(self): try: while self.rbsp_read('uint: 1') != 1: pass return True except ReadError: return False def rbsp_trailing_bits(self): pos = self.rbsp_byte.pos bits = self.rbsp_byte.bytealign() self.rbsp_byte.pos = pos if bits == 0 or self.rbsp_read('uint: %d' % bits) != 1 << bits: print('Wrong rbsp_trailing_bits at NALU begined at bytes %d' % self.pos) exit(1) def profile_tier_level(self, ProfilePresentFlag, MaxNumSubLayersMinus1): if ProfilePresentFlag: self.general_profile_space = self.rbsp_read('uint: 2') self.general_tier_flag = self.rbsp_read('uint: 1') self.general_profile_idc = self.rbsp_read('uint: 5') self.general_profile_compatibility_flag = [0] * 32 for i in range(0, 32): self.general_profile_compatibility_flag[i] = self.rbsp_read("uint: 1") self.general_reserved_zero_16bits = self.rbsp_read('uint: 16') self.general_level_idc = self.rbsp_read('uint: 8') self.sub_layer_profile_present_flag = [0] * MaxNumSubLayersMinus1 self.sub_layer_level_present_flag = [0] * MaxNumSubLayersMinus1 self.sub_layer_profile_space = [0] * MaxNumSubLayersMinus1 self.sub_layer_tier_flag = [0] * MaxNumSubLayersMinus1 self.sub_layer_profile_idc = [0] * MaxNumSubLayersMinus1 self.sub_layer_reserved_zero_16bits = [0] * MaxNumSubLayersMinus1 self.sub_layer_level_idc = [0] * MaxNumSubLayersMinus1 self.sub_layer_profile_compatibility_flag = [[0] * 32] * MaxNumSubLayersMinus1 for i in range(0, MaxNumSubLayersMinus1): self.sub_layer_profile_present_flag[i] = self.rbsp_read('uint: 1') self.sub_layer_level_present_flag[i] = self.rbsp_read('uint: 1') if ProfilePresentFlag and self.sub_layer_profile_present_flag[i]: self.sub_layer_profile_space[i] = self.rbsp_read('uint: 2') self.sub_layer_tier_flag[i] = self.rbsp_read('uint: 1') self.sub_layer_profile_idc[i] = self.rbsp_read('uint: 5') for j in range(0, 32): self.sub_layer_profile_compatibility_flag[i][j] = self.rbsp_read('uint: 1') self.sub_layer_reserved_zero_16bits[i] = self.rbsp_read('uint: 16') if self.sub_layer_level_present_flag[i]: self.sub_layer_level_idc[i] = self.rbsp_read('uint: 8') def op_point(self, opIdx): self.op_num_layer_id_values_minus1 = [opIdx] = self.rbsp_read('ue') self.op_layer_id[opIdx] = [0] * self.op_num_layer_id_values_minus1 for i in range(0, self.op_num_layer_id_values_minus1): self.op_layer_id[opIdx][i] = self.rbsp_read('uint: 6') def short_term_ref_pic_set(self, idxRps): if idxRps != 0: self.inter_ref__pic_set_prediction_flag = self.rbsp_read('uint: 1') else: self.inter_ref__pic_set_prediction_flag = 0 if self.inter_ref__pic_set_prediction_flag: if idxRps == self.num_short_term_ref_pic_sets: self.delta_idx_minus1 = self.rbsp_read('ue') else: self.delta_idx_minus1 = 0 RIdx = idxRps - self.delta_idx_minus1 - 1 self.delta_rps_sign = self.rbsp_read('uint: 1') self.abs_delta_rps_minus1 = self.rbsp_read('ue') self.used_by_curr_pic_flag = [0] * (self.NumDeltaPocs[RIdx] + 1) self.use_delta_flag = [0] * (self.NumDeltaPocs[RIdx] + 1) for i in range(0, self.NumDeltaPocs[RIdx] + 1): self.used_by_curr_pic_flag[i] = self.rbsp_read('uint: 1') if not self.used_by_currpic_flag[i]: self.use_delta_flag[i] = self.rbsp_read('uint: 1') else: num_negative_pics = self.rbsp_read('ue') num_positive_pics = self.rbsp_read('ue') self.delta_poc_s0_minus1 = [0] * num_negative_pics self.used_by_curr_pic_s0_flag = [0] * num_negative_pics for i in range(0, num_negative_pics): self.delta_poc_s0_minus1[i] = self.rbsp_read('ue') self.used_by_curr_pic_s0_flag[i] = self.rbsp_read('uint: 1') self.delta_poc_s1_minus1 = [0] * num_positive_pics self.used_by_curr_pic_s1_flag = [0] * num_positive_pics for i in range(0, num_positive_pics): self.delta_poc_s1_minus1[i] = self.rbsp_read('ue') self.used_by_curr_pic_s1_flag[i] = self.rbsp_read('uint: 1') self.NumDeltaPocs[idxRps] = num_negative_pics + num_positive_pics def setup(self): pass def __str__(self): return 'NALU: pos=%d, length=%d, type=%s, tid=%d' % (self.pos, self.num_bytes_in_nalu, NAL_UNIT_TYPE[self.nal_unit_type], self.temporal_id) def print_bin(self): i = 1 for abyte in self.rbsp_byte.tobytes(): if i == 16: print('%3s' % hex(abyte)[2: ]) i = 1 else: print('%3s' % hex(abyte)[2: ], end=' ') i += 1 print('\n')
if( usbEndpoint != INCOMING_ENDPOINT and usbEndpoint != OUTGOING_ENDPOINT ): continue usbBuffer = BitStream('0x%s' % ( packet.data.usb_capdata.replace( ':', '' ) ) ) usbHeader = usbBuffer.readlist( 'bytes:3, uint:8' ) # Validate the header if( usbEndpoint == OUTGOING_ENDPOINT and usbHeader[0].encode( 'hex' ) != '000000' ): print 'Unexpected USB Header. Expected "0x000000", got "0x%s".' % ( usbHeader[0].encode( 'hex' ) ) raise Exception if( usbEndpoint == INCOMING_ENDPOINT and usbHeader[0] != 'ABC' ): print 'Unexpected USB Header. Expected "0x414243", got "0x%s".' % ( usbHeader[0].encode( 'hex' ) ) raise Exception messageBuffer.append( usbBuffer.read( usbHeader[1] * 8 ) ) # Clear the messageBuffer if we have a full message # TODO - we need to be able to figure out if all 60 bytes are conusumed, but it's the end of the message if( usbHeader[1] < USB_PACKET_SIZE ): print >> sys.stderr, 'Message %s' % ( 'OUT' if usbEndpoint == OUTGOING_ENDPOINT else 'IN' ) print >> sys.stderr, 'Hex: %s' % ( messageBuffer.hex ) # TODO - make a bayerMessage to also handle standard command sequences and ASTM messages if( messageBuffer.bytes[0:2] != 'Q\x03' ): print >> sys.stderr, 'String: %s\n' % ( messageBuffer.bytes ) else: msg = BayerBinaryMessage.MessageFactory( messageBuffer, BayerBinaryMessage.OUT if usbEndpoint == OUTGOING_ENDPOINT else BayerBinaryMessage.IN, pumpSession ) if( msg is not None ): if( isinstance( msg, MtGetAttachedPumpMessage ) ): pumpSession.pumpSerial = msg.pumpSerial
def encode(self, bit_strm=None, offset=0): if bit_strm is None: bit_strm = BitStream() if not isinstance(bit_strm, BitStream): return None bit_strm.append(BitArray(uint=self.syncword, length=12)) bit_strm.append(BitArray(int=self.ID, length=1)) bit_strm.append(BitArray(int=self.layer, length=2)) bit_strm.append(BitArray(int=self.protection_absent, length=1)) bit_strm.append(BitArray(int=self.profile, length=2)) bit_strm.append(BitArray(int=self.sampling_index, length=4)) bit_strm.append(BitArray(int=self.private_bit, length=1)) bit_strm.append(BitArray(int=self.chan_config, length=3)) bit_strm.append(BitArray(int=self.original_copy, length=1)) bit_strm.append(BitArray(int=self.home, length=1)) bit_strm.append(BitArray(int=self.copyright_id_bit, length=1)) bit_strm.append(BitArray(int=self.copyright_id_start, length=1)) bit_strm.append(BitArray(uint=self.aac_frame_length, length=13)) bit_strm.append(BitArray(uint=self.adts_buffer_fullness, length=11)) bit_strm.append(BitArray(int=self.number_of_raw_data_blocks_in_frame, length=2)) bit_strm.byteswap() return bit_strm