def decode(src, reader): with open(src, "rb") as f: data = f.read() assert data[0x00:0x02] == SOI, "SOI not recognized" assert data[0x02:0x14] == APP0, "APP 0 not recognized" assert data[0x14:0x59] == DQT0, "DQT 0 not recognized" assert data[0x59:0x9e] == DQT1, "DQT 1 not recognized" assert data[0x9e:0xa3] == SOF0_PREFIX, "SOF 0 not recognized" height, = unpack('>H', data[0xa3:0xa5]) width, = unpack('>H', data[0xa5:0xa7]) assert data[0xa7:0xb1] == SOF0_SUFFIX, "SOF 0 not recognized" assert data[0xb1:0x255] == DHT, "DHT not recognized" assert data[0x255:0x263] == SOS, "SOS not recognized" assert data[-2:] == EOI, "EOI not recognized" stream = BitStream() i = 0x263 while i < len(data) - 2: value = data[i:i + 1] stream.write(value, bytes) i += 1 if data[i] != 0xff else 2 bits = BitStream() for yAC, _, _ in huffmanDecode(stream, height // 8 * width // 8): read(np.array(yAC)[zigzagReverseOrder], bits, reader) length = unpack('>I', bits.read(bytes, 4))[0] if length > len(bits) / 8: print("{} of {} bytes are revealed.".format(len(bits) / 8, length)) length = len(bits) / 8 return bits.read(bytes, length)
def test_writer(self, stream): assert stream == BitStream() stream.write(0) assert stream == BitStream() stream.write(1) assert stream == make_bitstream('1') stream.write(2) assert stream == make_bitstream('110') stream.write(3) assert stream == make_bitstream('11011')
def encode(self, source): """Koduje wejściowy ciąg danych przy pomocy wykładniczego kodu Golomba. Argumenty: source (List[int]): ciąg liczb naturalnych do zakodowania Zwraca: BitStream: strumień bitowy zawierający ciąg słów kodowych oraz opcjonalnie nagłówek (przy pośrednim trybie pracy kodera). """ stream = BitStream() self._source = source self._hist = histogram(source) # Utworzenie i zapisanie w nagłówku książki kodów (jeżeli wybrano tryb pośredni) if not self._direct: self._codebook = self._make_codebook(stream) header_len = len(stream) # Kodowanie danych źródłowych for word in source: self._encode_word(word, stream) # Obliczenie statystyk self._stream_len = len(stream) self._stream_data_len = len(stream) - header_len self._stats = Statistics(self) return stream
def __read_binary_file(self): with open(self.binary_path, "rb") as bin_file: str_bitstream = bin_file.read().decode() bool_list = list( map(lambda bit: bool(int(bit)), list(str_bitstream))) self.bitstream = BitStream() self.bitstream.write(bool_list)
def split(mnemonic): final_bits = mnemonic_to_bitstream(mnemonic) # how long was the source entropy for this wordlist? # according to https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki#generating-the-mnemonic # CS = ENT / 32 # MS = (ENT + CS) / 11 # then... # MS = ENT(33/32) / 11 # so... # ENT = (352/33) * MS payload_len = (352 / 33) * len(mnemonic) if payload_len != int(payload_len): raise ValueError( "calculated {} bits of initial entropy".format(entropy_orig_len)) payload_len = int(payload_len) # transfer bits until we reach the end of the source entropy # all remaining bits will be the checksum payload = BitStream() for bit in final_bits.read(bool, payload_len): payload.write(bit) checksum = deepcopy(final_bits) return [payload, checksum]
def load_plugin(name): mod = import_module('.main', 'plugins.'+name) bs = BitStream(bytes([])) t = Table('inspect', None, bs) plugin = Plugin() plugin.name = name for attr in dir(mod): if attr.startswith('__'): continue obj = getattr(mod, attr) if not inspect.isfunction(obj): continue sig = inspect.signature(obj) try: sig.bind(t, bs) if attr == 'default_parser': plugin.default = attr plugin.exports[attr] = obj except TypeError as e: continue if len(plugin.default) == 0 and len(plugin.exports) > 0: plugin.default = list(plugin.exports.keys())[0] plugin.doc.parse('plugins/'+name+'/doc.md') return plugin
def checksum_is_good(mnemonic): payload, checksum = split(mnemonic) # hash the payload hasher = sha256() hasher.update(as_bytes(payload)) long_checksum = BitStream(hasher.digest()) new_checksum = BitStream() for bit in long_checksum.read(bool, len(payload) / 32): new_checksum.write(bit) print(checksum) print(new_checksum) return checksum == new_checksum
def load_in(file_name, fixed_width_length): with open(file_name, 'rb') as f: file_content = f.read() #Get input as stream of bits and split by 12 bit_stream_as_string = str(BitStream(file_content)) #If odd get last code odd = False if len(bit_stream_as_string) % fixed_width_length != 0: last_code = int(bit_stream_as_string[-fixed_width_length:], 2) bit_stream_as_string = bit_stream_as_string[:-16] odd = True code_array = [ bit_stream_as_string[i:i + fixed_width_length] for i in range(0, len(bit_stream_as_string), fixed_width_length) ] #Convert to codes for i in range(0, len(code_array)): code_array[i] = int(code_array[i], 2) if odd: code_array.append(last_code) return code_array
def _interleaveGroups(self, dataGroups, ecGroups): """ Interleaves data groups into the final bitstream payload, with remainder bits """ def interleaveTwo(bitStream, first, second): # Interleaves two block groups together hasSecond = len(second) != 0 firstLen = len(first[0]) secondLen = len(second[0]) if hasSecond else 0 for cw in range(max(firstLen, secondLen)): if cw < firstLen: for block in first: bitStream.write(block[cw], uint8) if cw < secondLen: for block in second: bitStream.write(block[cw], uint8) # This is where the payload is stored payload = BitStream() # Interleave data blocks interleaveTwo(payload, dataGroups[0], dataGroups[1]) # Interleave EC blocks interleaveTwo(payload, ecGroups[0], ecGroups[1]) # Add remainder bits (zero-padding to match size constraints) payload.write([False] * QRCode.REMAINDER_LIST[self.version - 1], bool) return payload
def draw(self, cell): if self._depth_function is None: return bits = BitStream() bits = self._depth_function.encode(bits) self._depth_function.decode(bits) self._depth_function.uncompress(cell)
def test_encode_function_id(self): bits = encode_function_id(0, BitStream()) self.assertEqual(bits, BitStream([False, False])) bits = encode_function_id(1, BitStream()) self.assertEqual(bits, BitStream([False, True])) bits = encode_function_id(2, BitStream()) self.assertEqual(bits, BitStream([True, False])) bits = encode_function_id(3, BitStream()) self.assertEqual(bits, BitStream([True, True]))
def test_writer(self, stream): assert stream == BitStream() write_unary(stream, 0) assert stream == make_bitstream('1') write_unary(stream, 1) assert stream == make_bitstream('101') write_unary(stream, 2) assert stream == make_bitstream('101001')
def __init__(self, buffer=BitStream(), state=DataBufferState.BUFFER_ENABLED, flush=False, watch=False): self.buffer = buffer self.state = state self.flush = flush self.watch = watch
def test_encode_decode(self): fl = FunctionLeaf() fl2 = FunctionLeaf() fl.set_depth_function(MockF1()) bits = BitStream() bits = fl.encode(bits) fl2.decode(bits) self.assertAlmostEquals(fl2.get_depth_function().get_max_val(), fl.get_depth_function().get_max_val())
def read_bitstream(self, bitstream): if isinstance(bitstream, BitStream): self.bitstream = bitstream elif isinstance(bitstream, str): bool_list = list(map(lambda bit: bool(int(bit)), list(bitstream))) self.bitstream = BitStream() self.bitstream.write(bool_list) else: raise TypeError( "The provided bitstream should already be a bitstream.BitStream instance or a string." )
def get_peak_signal_to_noise(image: np.ndarray, f: DepthFunction): uncompressed = np.zeros(image.shape) f.compress(image) bits = BitStream() bits = f.encode(bits) f.decode(bits) f.uncompress(uncompressed) if metrics.mean_squared_error(image, uncompressed) == 0: return float('Inf') sig = peak_signal_to_noise(image, uncompressed) return sig
def encode(a_text: str, encoding_table: dict) -> BitStream: """ encode given text :param a_text: text to encode :param encoding_table: encoding table build using huffman coding :return: encoded bitstream """ encoded = BitStream() for char in a_text: for bit in encoding_table[char]: encoded.write(bit, bool) return encoded
def convert_bit_to_array(v_bits, values_type, nb_values): """ Convert a binary array into specific type array INPUT: v_bits: 1d-array, values_type: type of element in output array nb_values: number of elements to read OUTPUT: array with specific elements type indide """ stream = BitStream() stream.write(v_bits.astype(bool), bool) return stream.read(values_type, nb_values)
def mnemonic_to_bitstream(mnemonic): stream = BitStream() for word in mnemonic: value = words.index(word) pointer = 0x1 for _ in range(11): if pointer & value: stream.write(True, bool) else: stream.write(False, bool) pointer <<= 1 return stream
def convert_array_to_bit(v_values, values_type): """ Convert an values-array with specific type to an binary array INPUT: v_values: 1d-array, values which be converted values_type: type of the v_values element. Usefull types; np.int8, np.int16, np.int32, np.float32 OUTPUT: 1d-array, return an integer array with binary values (0 or 1) """ stream = BitStream() stream.write(v_values, values_type) bit_str = str(stream) return np.array(list(bit_str), dtype=int), len(v_values)
def fix_checksum(mnemonic): payload, checksum = split(mnemonic) # hash the payload hasher = sha256() hasher.update(as_bytes(payload)) long_checksum = BitStream(hasher.digest()) # append bits from the beginning of the SHA256 to the end of the starting entropy for bit in long_checksum.read(bool, len(payload) / 32): payload.write(bit) return bitstream_to_mnemonic(payload)
def expand(bit_stream: BitStream) -> BitStream: """ expand compressed bit stresm :param bit_stream: bitstream to expand :return: original stream before compression """ expanded = BitStream() bit = True while len(bit_stream): count = bit_stream.read(int8, 1)[0] for _ in range(count): expanded.write(bit, bool) bit = not bit return expanded
def pack(): contents = open(sys.argv[2], 'rb').read() packed_file = open(sys.argv[3], 'wb') bits = BitStream(contents) f = '' i = len(bits) while i > 0: b = bits.read(16) i -= 16 # dirty flip to deal with endianness b = str(b)[8:] + str(b)[:8] b = b[7:] f += b packed_bitstream = BitStream() for i in f: packed_bitstream.write(int(i), bool) for i in range(0, len(packed_bitstream) / 8): b = packed_bitstream.read(8) packed_file.write(struct.pack('B', int(str(b), 2)))
def unpack(): contents = open(sys.argv[2], 'rb').read() output_file = open(sys.argv[3], 'wb') bits = BitStream(contents) i = len(bits) while i > 0: b = bits.read(9) i -= 9 b = '0000000' + str(b) sh = int(b, 2) x = struct.pack("H", sh) output_file.write(x)
def _genEncodedData(self): """ Turns input data into a bit stream with headers but no error correction """ output = BitStream() # Add mode indicator. Using byte mode, so, write 0100 output.write([False, True, False, False], bool) # Add character count indicator. The size of the indicator depends on the QR version in use data_len = len(self.data_bytes) if self.version <= 9: output.write(data_len, uint8) else: output.write(data_len, uint16) # Add data output.write(self.data_bytes, bytes) # Get required size block_info = QRCode.BLOCK_LIST[self.version - 1][self.ecc] req_bytes = None if len(block_info) == 3: req_bytes = block_info[1] * block_info[2] else: req_bytes = block_info[1] * block_info[2] + block_info[ 3] * block_info[4] req_bits = req_bytes * 8 # Add 4-bit (or less) zero padding zero_count = min(4, req_bits - len(output)) output.write([False] * zero_count, bool) # Align data to bytes out_len = len(output) aligned_bits = ceil(out_len / 8) * 8 output.write([False] * (aligned_bits - out_len), bool) # Add padding bytes if necessary aligned_bytes = aligned_bits // 8 alternate = False for i in range(req_bytes - aligned_bytes): if alternate: output.write(17, uint8) else: output.write(236, uint8) alternate = not alternate return output.read(bytes)
def test_compression(self): original_string = "this is some string to compress" byte_string = original_string.encode('ascii') original_stream = BitStream() original_stream.write(byte_string, bytes) original_stream_length = len(original_stream) compressed_stream = compress(original_stream) compressed_stream_length = len(compressed_stream) expanded_stream = expand(compressed_stream) retrieved_string = expanded_stream.read(bytes) retrieved_string = retrieved_string.decode('ascii') self.assertNotEqual(original_stream_length, compressed_stream_length) self.assertEqual(original_string, retrieved_string)
def compress(bit_stream: BitStream) -> BitStream: """ compress a bit stream using run length encoding -> replace replace series of same consecutive bit by they number 00001110000111 -> 4343 -> 100011100011 f the bits are not in long consecutive series of same bits the compressed file can end up being bigger then original :param bit_stream: bit stream to compress :return: compressed stream """ compressed = BitStream() bit = True while len(bit_stream): count = count_bits(bit_stream, bit, 256) compressed.write(count, int8) bit = not bit return compressed
def decode(self, data): stream = BitStream(data) mode = MODE_NONE while True: first_bit = stream.read(bool) if first_bit: ones = leading_ones(stream) if mode == MODE_SCHEME1: if not first_bit: self.add_byte(ord(stream.read(bytes, 1))) continue elif ones < 8: cnt_field = stream.read( BitStream, ones + 1 if ones < 4 else 11 - ones) if ones > 4: cnt_field += (ones - 4) << (12 - ones) cnt = int(str(cnt_field), 2) + (2 << ones) loc = int(str(stream.read(BitStream, 10)), 2) for i in range(loc, loc + cnt): self.add_byte(self.buffer[i % 1024]) continue if first_bit and ones == 8: control = str(stream.read(BitStream, 3)) if control == '101': # Scheme 1 mode = MODE_SCHEME1 elif control == '100': # End of record (EOR) self.output.extend(self.buffer[:self.buffer_offset]) break else: raise ValueError( "Can't parse SLDC, unsupported control symbol: %s" % (control)) else: raise ValueError("Can't parse SLDC, unknown action") return self.output
def default_parser(d: Table, bs: BitStream): analyse_nalu_header(d, bs) rbsp_bs = BitStream(d.nal_unit.rbsp) d.context.add_exist(d.nal_unit) if d.nal_unit.nal_unit_type == 9: d.add_table(access_unit_delimiter_rbsp, bs=rbsp_bs) elif d.nal_unit.nal_unit_type == 6: d.add_table(sei_rbsp, bs=rbsp_bs) elif d.nal_unit.nal_unit_type in [1, 5]: d.add_table(slice_layer_without_partitioning_rbsp, bs=rbsp_bs) elif d.nal_unit.nal_unit_type == 7: d.add_table(seq_parameter_set_rbsp, bs=rbsp_bs) elif d.nal_unit.nal_unit_type == 8: d.add_table(pic_parameter_set_rbsp, bs=rbsp_bs) elif d.nal_unit.nal_unit_type == 15: d.add_table(subset_seq_parameter_set_rbsp, bs=rbsp_bs) elif d.nal_unit.nal_unit_type == 20: d.add_table(slice_layer_extension_rbsp, bs=rbsp_bs)
def toCode(r): stream = bitarray() for i in range(0, len(r)): tempStream = BitStream() if ',' in r[i]: temp = r[i].split(',') else: temp = r[i] if temp == 'True': tempStream.write(True, bool) elif temp == 'False': tempStream.write(False, bool) else: for j in range(0, len(temp)): tempStream.write(str2bool(temp[j]), bool) stream = numpy.append(stream, tempStream) return stream