def split(mnemonic): final_bits = mnemonic_to_bitstream(mnemonic) # how long was the source entropy for this wordlist? # according to https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki#generating-the-mnemonic # CS = ENT / 32 # MS = (ENT + CS) / 11 # then... # MS = ENT(33/32) / 11 # so... # ENT = (352/33) * MS payload_len = (352 / 33) * len(mnemonic) if payload_len != int(payload_len): raise ValueError( "calculated {} bits of initial entropy".format(entropy_orig_len)) payload_len = int(payload_len) # transfer bits until we reach the end of the source entropy # all remaining bits will be the checksum payload = BitStream() for bit in final_bits.read(bool, payload_len): payload.write(bit) checksum = deepcopy(final_bits) return [payload, checksum]
def _interleaveGroups(self, dataGroups, ecGroups): """ Interleaves data groups into the final bitstream payload, with remainder bits """ def interleaveTwo(bitStream, first, second): # Interleaves two block groups together hasSecond = len(second) != 0 firstLen = len(first[0]) secondLen = len(second[0]) if hasSecond else 0 for cw in range(max(firstLen, secondLen)): if cw < firstLen: for block in first: bitStream.write(block[cw], uint8) if cw < secondLen: for block in second: bitStream.write(block[cw], uint8) # This is where the payload is stored payload = BitStream() # Interleave data blocks interleaveTwo(payload, dataGroups[0], dataGroups[1]) # Interleave EC blocks interleaveTwo(payload, ecGroups[0], ecGroups[1]) # Add remainder bits (zero-padding to match size constraints) payload.write([False] * QRCode.REMAINDER_LIST[self.version - 1], bool) return payload
def __read_binary_file(self): with open(self.binary_path, "rb") as bin_file: str_bitstream = bin_file.read().decode() bool_list = list( map(lambda bit: bool(int(bit)), list(str_bitstream))) self.bitstream = BitStream() self.bitstream.write(bool_list)
def slice_header(d: Table, bs: BitStream): d.add_field('first_mb_in_slice', bs.read_ue_golomb) d.add_field('slice_type', bs.read_ue_golomb) d.add_field('pic_parameter_set_id', bs.read_ue_golomb) pps = __find_pps_in_ctx(d.pic_parameter_set_id, d.context) if pps == None: raise Exception('no pps in ctx, abort!') sps = __find_sps_in_ctx(pps.seq_parameter_set_id, d.context) if pps == None: raise Exception('no sps in ctx, abort!') if sps.get_value('separate_colour_plane_flag', 0): d.add_field('colour_plane_id', bs.read_bits, count=2) try: nal_unit = d.context.nal_unit except AttributeError as e: raise Exception('no nal_unit in ctx, abort!') d.add_field('frame_num', bs.read_bits, count=(sps.log2_max_frame_num_minus4 + 4)) if not sps.frame_mbs_only_flag: d.add_field('field_pic_flag', bs.read_a_bit) if d.field_pic_flag: d.add_field('bottom_field_flag', bs.read_a_bit) if nal_unit.nal_unit_type == 5: #IdrPicFlag d.add_field('idr_pic_id', bs.read_ue_golomb) if sps.pic_order_cnt_type == 0: d.add_field('pic_order_cnt_lsb', bs.read_bits, count=(sps.log2_max_pic_order_cnt_lsb_minus4 + 4)) if pps.bottom_field_pic_order_in_frame_present_flag and not d.get_value( 'field_pic_flag', 0): bs.add_field('delta_pic_order_cnt_bottom', bs.read_se_golomb) d.add_field('delta_pic_order_cnt', __read_delta_pic_order_cnt, sps=sps, pps=pps, d=d, bs=bs) if pps.redundant_pic_cnt_present_flag: d.add_field('redundant_pic_cnt', bs.read_ue_golomb) slice_type = d.slice_type % 5 if slice_type == 1: #B d.add_field('direct_spatial_mv_pred_flag', bs.read_a_bit) if slice_type in [0, 1, 3]: #P SP B d.add_field('num_ref_idx_active_override_flag', bs.read_a_bit) if d.num_ref_idx_active_override_flag: d.add_field('num_ref_idx_l0_active_minus1', bs.read_ue_golomb) if slice_type == 1: #B d.add_field('num_ref_idx_l1_active_minus1', bs.read_ue_golomb) if nal_unit.nal_unit_type in [20, 21]: d.add_table(ref_pic_list_mvc_modification, type=slice_type) else: d.add_table(ref_pic_list_modification, type=slice_type)
def __fields_by_cpb_cnt_minus1(d: Table, bs: BitStream): d.bit_rate_value_minus1 = [0] * (d.cpb_cnt_minus1 + 1) d.cpb_size_value_minus1 = [0] * (d.cpb_cnt_minus1 + 1) d.cbr_flag = [0] * (d.cpb_cnt_minus1 + 1) for i in range(d.cpb_cnt_minus1 + 1): d.bit_rate_value_minus1[i] = bs.read_ue_golomb() d.cpb_size_value_minus1[i] = bs.read_ue_golomb() d.cbr_flag[i] = bs.read_a_bit()
def encode_node(self, node, stream: BitStream): is_leaf = node.is_leaf() stream.write(is_leaf) if is_leaf: node.get_leaf_data().encode(stream) else: for child in node.get_children(): self.encode_node(child, stream) return stream
def more_rbsp_data(bs: BitStream): n = bs.available() if n > 8: return True elif n <= 0: return False b = bs.read_bits(n, forward=False) return not b == (1 << (n - 1))
def seq_parameter_set_data(d: Table, bs: BitStream): d.add_field('profile_idc', bs.read_bits, count=8) d.add_field('constraint_set0_flag', bs.read_bits, count=1) d.add_field('constraint_set1_flag', bs.read_bits, count=1) d.add_field('constraint_set2_flag', bs.read_bits, count=1) d.add_field('constraint_set3_flag', bs.read_bits, count=1) d.add_field('constraint_set4_flag', bs.read_bits, count=1) d.add_field('constraint_set5_flag', bs.read_bits, count=1) bs.to_byte_end() # reserved_zero_2bits d.add_field('level_idc', bs.read_bits, count=8) d.add_field('seq_parameter_set_id', bs.read_ue_golomb) if d.profile_idc in [ 100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135 ]: d.add_field('chroma_format_idc', bs.read_ue_golomb) if d.chroma_format_idc == 3: d.add_field('separate_colour_plane_flag', bs.read_bits, count=1) d.add_field('bit_depth_luma_minus8', bs.read_ue_golomb) d.add_field('bit_depth_chroma_minus', bs.read_ue_golomb) d.add_field('qpprime_y_zero_transform_bypass_flag', bs.read_bits, count=1) d.add_field('seq_scaling_matrix_present_flag', bs.read_bits, count=1) if d.seq_scaling_matrix_present_flag: # todo test this branch d.add_fields(__fields_in_scaling_list, 'seq_scaling_list_present_flag', 'ScalingList4x4', 'UseDefaultScalingMatrix4x4Flag', 'ScalingList8x8', 'UseDefaultScalingMatrix8x8Flag') d.add_field('log2_max_frame_num_minus4', bs.read_ue_golomb) d.add_field('pic_order_cnt_type', bs.read_ue_golomb) if d.pic_order_cnt_type == 0: d.add_field('log2_max_pic_order_cnt_lsb_minus4', bs.read_ue_golomb) elif d.pic_order_cnt_type == 1: d.add_field('delta_pic_order_always_zero_flag', bs.read_bits, count=1) d.add_field('offset_for_non_ref_pic', bs.read_se_golomb) d.add_field('offset_for_top_to_bottom_field', bs.read_se_golomb) d.add_field('num_ref_frames_in_pic_order_cnt_cycle', bs.read_ue_golomb) d.add_field('offset_for_ref_fram', bs.read_se_golombs, count=d.num_ref_frames_in_pic_order_cnt_cycle) d.add_field('max_num_ref_frames', bs.read_ue_golomb) d.add_field('gaps_in_frame_num_value_allowed_flag', bs.read_bits, count=1) d.add_field('pic_width_in_mbs_minus1', bs.read_ue_golomb) d.add_field('pic_height_in_map_units_minus1', bs.read_ue_golomb) d.add_field('frame_mbs_only_flag', bs.read_bits, count=1) if d.frame_mbs_only_flag == 0: d.add_field('mb_adaptive_frame_field_flag', bs.read_bits, count=1) d.add_field('direct_8x8_inference_flag', bs.read_bits, count=1) d.add_field('frame_cropping_flag', bs.read_bits, count=1) if d.frame_cropping_flag == 1: d.add_field('frame_crop_left_offset', bs.read_ue_golomb) d.add_field('frame_crop_right_offset', bs.read_ue_golomb) d.add_field('frame_crop_top_offset', bs.read_ue_golomb) d.add_field('frame_crop_bottom_offset', bs.read_ue_golomb) d.add_field('vui_parameters_present_flag', bs.read_bits, count=1) if d.vui_parameters_present_flag: d.add_table(vui_parameters)
def test_writer(self, stream): assert stream == BitStream() stream.write(0) assert stream == BitStream() stream.write(1) assert stream == make_bitstream('1') stream.write(2) assert stream == make_bitstream('110') stream.write(3) assert stream == make_bitstream('11011')
def read_bitstream(self, bitstream): if isinstance(bitstream, BitStream): self.bitstream = bitstream elif isinstance(bitstream, str): bool_list = list(map(lambda bit: bool(int(bit)), list(bitstream))) self.bitstream = BitStream() self.bitstream.write(bool_list) else: raise TypeError( "The provided bitstream should already be a bitstream.BitStream instance or a string." )
def __read_ref_pic_list(d: Table, bs: BitStream): d.modification_of_pic_num_idc = [] d.abs_diff_pic_num_minus1 = [] d.long_term_pic_num = [] while True: d.modification_of_pic_num_idc.append(bs.read_se_golomb()) if d.modification_of_pic_num_idc[-1] in [0, 1]: d.abs_diff_pic_num_minus1.append(bs.read_se_golomb()) elif d.modification_of_pic_num_idc[-1] == 2: d.long_term_pic_num.append(bs.read_se_golomb()) elif d.modification_of_pic_num_idc[-1] == 3: break
def encode(a_text: str, encoding_table: dict) -> BitStream: """ encode given text :param a_text: text to encode :param encoding_table: encoding table build using huffman coding :return: encoded bitstream """ encoded = BitStream() for char in a_text: for bit in encoding_table[char]: encoded.write(bit, bool) return encoded
def convert_bit_to_array(v_bits, values_type, nb_values): """ Convert a binary array into specific type array INPUT: v_bits: 1d-array, values_type: type of element in output array nb_values: number of elements to read OUTPUT: array with specific elements type indide """ stream = BitStream() stream.write(v_bits.astype(bool), bool) return stream.read(values_type, nb_values)
def fix_checksum(mnemonic): payload, checksum = split(mnemonic) # hash the payload hasher = sha256() hasher.update(as_bytes(payload)) long_checksum = BitStream(hasher.digest()) # append bits from the beginning of the SHA256 to the end of the starting entropy for bit in long_checksum.read(bool, len(payload) / 32): payload.write(bit) return bitstream_to_mnemonic(payload)
def convert_array_to_bit(v_values, values_type): """ Convert an values-array with specific type to an binary array INPUT: v_values: 1d-array, values which be converted values_type: type of the v_values element. Usefull types; np.int8, np.int16, np.int32, np.float32 OUTPUT: 1d-array, return an integer array with binary values (0 or 1) """ stream = BitStream() stream.write(v_values, values_type) bit_str = str(stream) return np.array(list(bit_str), dtype=int), len(v_values)
def rbsp_data(bs: BitStream): from_byte = bs.pos().byte n = find_start_code(bs) to_byte = bs.pos().byte num = to_byte - from_byte - n bs.moveto(from_byte, 0) rbsp = bytearray() i = 0 while i < num: if i <= num - 3: next_3_byte = bs.read_bytes_to_array(3, forward=False) if next_3_byte[0] == 0 and next_3_byte[1] == 0 and next_3_byte[ 2] == 3: rbsp.extend([0, 0]) bs.skip(3, 0) i += 3 continue rbsp.extend(bs.read_bytes_to_array(1)) i += 1 return rbsp
def decode(src, reader): with open(src, "rb") as f: data = f.read() assert data[0x00:0x02] == SOI, "SOI not recognized" assert data[0x02:0x14] == APP0, "APP 0 not recognized" assert data[0x14:0x59] == DQT0, "DQT 0 not recognized" assert data[0x59:0x9e] == DQT1, "DQT 1 not recognized" assert data[0x9e:0xa3] == SOF0_PREFIX, "SOF 0 not recognized" height, = unpack('>H', data[0xa3:0xa5]) width, = unpack('>H', data[0xa5:0xa7]) assert data[0xa7:0xb1] == SOF0_SUFFIX, "SOF 0 not recognized" assert data[0xb1:0x255] == DHT, "DHT not recognized" assert data[0x255:0x263] == SOS, "SOS not recognized" assert data[-2:] == EOI, "EOI not recognized" stream = BitStream() i = 0x263 while i < len(data) - 2: value = data[i:i + 1] stream.write(value, bytes) i += 1 if data[i] != 0xff else 2 bits = BitStream() for yAC, _, _ in huffmanDecode(stream, height // 8 * width // 8): read(np.array(yAC)[zigzagReverseOrder], bits, reader) length = unpack('>I', bits.read(bytes, 4))[0] if length > len(bits) / 8: print("{} of {} bytes are revealed.".format(len(bits) / 8, length)) length = len(bits) / 8 return bits.read(bytes, length)
def expand(bit_stream: BitStream) -> BitStream: """ expand compressed bit stresm :param bit_stream: bitstream to expand :return: original stream before compression """ expanded = BitStream() bit = True while len(bit_stream): count = bit_stream.read(int8, 1)[0] for _ in range(count): expanded.write(bit, bool) bit = not bit return expanded
def unpack(): contents = open(sys.argv[2], 'rb').read() output_file = open(sys.argv[3], 'wb') bits = BitStream(contents) i = len(bits) while i > 0: b = bits.read(9) i -= 9 b = '0000000' + str(b) sh = int(b, 2) x = struct.pack("H", sh) output_file.write(x)
def load_plugin(name): mod = import_module('.main', 'plugins.'+name) bs = BitStream(bytes([])) t = Table('inspect', None, bs) plugin = Plugin() plugin.name = name for attr in dir(mod): if attr.startswith('__'): continue obj = getattr(mod, attr) if not inspect.isfunction(obj): continue sig = inspect.signature(obj) try: sig.bind(t, bs) if attr == 'default_parser': plugin.default = attr plugin.exports[attr] = obj except TypeError as e: continue if len(plugin.default) == 0 and len(plugin.exports) > 0: plugin.default = list(plugin.exports.keys())[0] plugin.doc.parse('plugins/'+name+'/doc.md') return plugin
def encode(self, source): """Koduje wejściowy ciąg danych przy pomocy wykładniczego kodu Golomba. Argumenty: source (List[int]): ciąg liczb naturalnych do zakodowania Zwraca: BitStream: strumień bitowy zawierający ciąg słów kodowych oraz opcjonalnie nagłówek (przy pośrednim trybie pracy kodera). """ stream = BitStream() self._source = source self._hist = histogram(source) # Utworzenie i zapisanie w nagłówku książki kodów (jeżeli wybrano tryb pośredni) if not self._direct: self._codebook = self._make_codebook(stream) header_len = len(stream) # Kodowanie danych źródłowych for word in source: self._encode_word(word, stream) # Obliczenie statystyk self._stream_len = len(stream) self._stream_data_len = len(stream) - header_len self._stats = Statistics(self) return stream
def sei_ff_coding(bs: BitStream): value = 0 next_byte = 0xFF while next_byte == 0xFF: next_byte = bs.read_bits(8) value += next_byte return value
def draw(self, cell): if self._depth_function is None: return bits = BitStream() bits = self._depth_function.encode(bits) self._depth_function.decode(bits) self._depth_function.uncompress(cell)
def load_in(file_name, fixed_width_length): with open(file_name, 'rb') as f: file_content = f.read() #Get input as stream of bits and split by 12 bit_stream_as_string = str(BitStream(file_content)) #If odd get last code odd = False if len(bit_stream_as_string) % fixed_width_length != 0: last_code = int(bit_stream_as_string[-fixed_width_length:], 2) bit_stream_as_string = bit_stream_as_string[:-16] odd = True code_array = [ bit_stream_as_string[i:i + fixed_width_length] for i in range(0, len(bit_stream_as_string), fixed_width_length) ] #Convert to codes for i in range(0, len(code_array)): code_array[i] = int(code_array[i], 2) if odd: code_array.append(last_code) return code_array
def test_encode_function_id(self): bits = encode_function_id(0, BitStream()) self.assertEqual(bits, BitStream([False, False])) bits = encode_function_id(1, BitStream()) self.assertEqual(bits, BitStream([False, True])) bits = encode_function_id(2, BitStream()) self.assertEqual(bits, BitStream([True, False])) bits = encode_function_id(3, BitStream()) self.assertEqual(bits, BitStream([True, True]))
def count_bits(bit_stream: BitStream, which_bit: bool, max_count: int) -> int: """ count consecutive number of same bits :param bit_stream: a stream of bits :param which_bit: which bits to count :param max_count: max allowed number of same consecutive :return: count of same consecutive bits in given stream """ count = 0 which_bit = '1' if which_bit else '0' bit = str(bit_stream)[0] while bit == which_bit and count < max_count and len(bit_stream): bit_stream.read(bool, 1) count += 1 if len(bit_stream): bit = str(bit_stream)[0] return count
def compress(bit_stream: BitStream) -> BitStream: """ compress a bit stream using run length encoding -> replace replace series of same consecutive bit by they number 00001110000111 -> 4343 -> 100011100011 f the bits are not in long consecutive series of same bits the compressed file can end up being bigger then original :param bit_stream: bit stream to compress :return: compressed stream """ compressed = BitStream() bit = True while len(bit_stream): count = count_bits(bit_stream, bit, 256) compressed.write(count, int8) bit = not bit return compressed
def test_writer(self, stream): assert stream == BitStream() write_unary(stream, 0) assert stream == make_bitstream('1') write_unary(stream, 1) assert stream == make_bitstream('101') write_unary(stream, 2) assert stream == make_bitstream('101001')
def test_compression(self): original_string = "this is some string to compress" byte_string = original_string.encode('ascii') original_stream = BitStream() original_stream.write(byte_string, bytes) original_stream_length = len(original_stream) compressed_stream = compress(original_stream) compressed_stream_length = len(compressed_stream) expanded_stream = expand(compressed_stream) retrieved_string = expanded_stream.read(bytes) retrieved_string = retrieved_string.decode('ascii') self.assertNotEqual(original_stream_length, compressed_stream_length) self.assertEqual(original_string, retrieved_string)
def test_encode_decode(self): fl = FunctionLeaf() fl2 = FunctionLeaf() fl.set_depth_function(MockF1()) bits = BitStream() bits = fl.encode(bits) fl2.decode(bits) self.assertAlmostEquals(fl2.get_depth_function().get_max_val(), fl.get_depth_function().get_max_val())
def RTCM_converter_thread(server, port, username, password, mountpoint, rtcm_callback=None): import subprocess nt = subprocess.Popen(["./ntripclient", "--server", server, "--password", password, "--user", username, "--mountpoint", mountpoint], stdout=subprocess.PIPE) """nt = subprocess.Popen(["./ntrip.py", server, str(port), username, password, mountpoint], stdout=subprocess.PIPE)""" if nt is None or nt.stdout is None: indev = sys.stdin else: indev = nt.stdout print("RTCM using input {}".format(indev)) while True: sio = indev d = ord(sio.read(1)) if d != RTCMv3_PREAMBLE: continue pack_stream = BitStream() l1 = ord(sio.read(1)) l2 = ord(sio.read(1)) pack_stream.append(bs.pack('2*uint:8', l1, l2)) pack_stream.read(6) pkt_len = pack_stream.read(10).uint pkt = sio.read(pkt_len) parity = sio.read(3) if len(pkt) != pkt_len: print "Length error {} {}".format(len(pkt), pkt_len) continue if True: # TODO check parity for d in pkt: pack_stream.append(bs.pack('uint:8', ord(d))) msg = parse_rtcmv3(pack_stream) if msg is not None and rtcm_callback is not None: rtcm_callback(msg)
def decode_rtcm3_pack(buff): d = ord(buff[0]) if d != RTCMv3_PREAMBLE: print "RTCM3 preamble error!", d return pack_stream = BitStream() l1 = ord(buff[1]) l2 = ord(buff[2]) pack_stream.append(bs.pack('2*uint:8', l1, l2)) pack_stream.read(6) pkt_len = pack_stream.read(10).uint pkt = buff[3:3 + pkt_len] parity = buff[3 + pkt_len:-1] if True: # TODO check parity for d in pkt: pack_stream.append(bs.pack('uint:8', ord(d))) msg = parse_rtcmv3(pack_stream)
def __init__(self): self.stream = BitStream() self.fish_positions = [] self.zero_count = 0 self.one_count = 0
class FishStream: # init a BitStream to hold the bit values def __init__(self): self.stream = BitStream() self.fish_positions = [] self.zero_count = 0 self.one_count = 0 # helper function, add one to the stream and one count def add_zero(self): self.stream.write(False) self.zero_count += 1 # helper function, add zero to the stream and the zero count def add_one(self): self.stream.write(True) self.one_count += 1 def add_position(self, fish_id, x, y): # if velocity and acceleration is greater than these values # then flip the bit value. velocity_threshold = 5 acceleration_threshold = 5 # if the current index it empty this will throw # an indexException, and in which case, init the # current fish position at that index. try: # grab the past position of the current fish # - PREVIOUS - x_previous = self.fish_positions[fish_id][0] # position of x y_previous = self.fish_positions[fish_id][1] # position of y vx_previous = self.fish_positions[fish_id][2] # velocity of x vy_previous = self.fish_positions[fish_id][3] # velocity of y ax = ay = 0 # determine the current velocity if x_previous > x: vx = (x_previous - x) else: vx = (x - x_previous) if y_previous > y: vy = (y_previous - y) else: vy = (y - y_previous) # determine the current acceleration if vx_previous > vx: ax = vx_previous - vx else: ax = vx - vx_previous if vy_previous > vy: ay = vy_previous - vy else: ay = vy - vy_previous # current fish moved to the right if x_previous > x: if vx < velocity_threshold: if ax < acceleration_threshold: self.add_one() else: self.add_zero() else: if ax < acceleration_threshold: self.add_one() else: self.add_zero() # current fish moved to the left elif x_previous < x: if vx < velocity_threshold: if ax > acceleration_threshold: self.add_one() else: self.add_zero() else: if ax > acceleration_threshold: self.add_one() else: self.add_zero() # current fish moved up the screen if y_previous < y: if vy < velocity_threshold: if ay < acceleration_threshold: self.add_one() else: self.add_zero() else: if ay < acceleration_threshold: self.add_one() else: self.add_zero() # current fish moved down the screen elif y_previous > y: if vy < velocity_threshold: if ay > acceleration_threshold: self.add_one() else: self.add_zero() else: if ay > acceleration_threshold: self.add_one() else: self.add_zero() # overwrite previous positions with current self.fish_positions[fish_id] = [x, y, vx, vy] except IndexError: # new fish found, append is to the list self.fish_positions.append([x, y, 0, 0]) def print_stream(self): print self.stream # returns two values, probability of zero and one def get_probabilities(self): # calculate total (we use it twice) total = self.zero_count + self.one_count if total == 0: return 0, 0 # returns the probability of a zero and a one return float(self.zero_count) / total, \ float(self.one_count) / total def get_bits(self, length): while self.stream.__len__() < length: time.sleep(0.1) return_bits = self.stream.read(length) self.zero_count -= str(return_bits).count('0') self.one_count -= str(return_bits).count('1') return return_bits def get_length(self): return len(str(self.stream))