def decode(src, reader):
with open(src, "rb") as f:
data = f.read()
assert data[0x00:0x02] == SOI, "SOI not recognized"
assert data[0x02:0x14] == APP0, "APP 0 not recognized"
assert data[0x14:0x59] == DQT0, "DQT 0 not recognized"
assert data[0x59:0x9e] == DQT1, "DQT 1 not recognized"
assert data[0x9e:0xa3] == SOF0_PREFIX, "SOF 0 not recognized"
height, = unpack('>H', data[0xa3:0xa5])
width, = unpack('>H', data[0xa5:0xa7])
assert data[0xa7:0xb1] == SOF0_SUFFIX, "SOF 0 not recognized"
assert data[0xb1:0x255] == DHT, "DHT not recognized"
assert data[0x255:0x263] == SOS, "SOS not recognized"
assert data[-2:] == EOI, "EOI not recognized"
stream = BitStream()
i = 0x263
while i < len(data) - 2:
value = data[i:i + 1]
stream.write(value, bytes)
i += 1 if data[i] != 0xff else 2
bits = BitStream()
for yAC, _, _ in huffmanDecode(stream, height // 8 * width // 8):
read(np.array(yAC)[zigzagReverseOrder], bits, reader)
length = unpack('>I', bits.read(bytes, 4))[0]
if length > len(bits) / 8:
print("{} of {} bytes are revealed.".format(len(bits) / 8, length))
length = len(bits) / 8
return bits.read(bytes, length)
def test_writer(self, stream):
assert stream == BitStream()
stream.write(0)
assert stream == BitStream()
stream.write(1)
assert stream == make_bitstream('1')
stream.write(2)
assert stream == make_bitstream('110')
stream.write(3)
assert stream == make_bitstream('11011')
def encode(self, source):
"""Koduje wejściowy ciąg danych przy pomocy wykładniczego kodu Golomba.
Argumenty:
source (List[int]): ciąg liczb naturalnych do zakodowania
Zwraca:
BitStream: strumień bitowy zawierający ciąg słów kodowych oraz opcjonalnie
nagłówek (przy pośrednim trybie pracy kodera).
"""
stream = BitStream()
self._source = source
self._hist = histogram(source)
# Utworzenie i zapisanie w nagłówku książki kodów (jeżeli wybrano tryb pośredni)
if not self._direct:
self._codebook = self._make_codebook(stream)
header_len = len(stream)
# Kodowanie danych źródłowych
for word in source:
self._encode_word(word, stream)
# Obliczenie statystyk
self._stream_len = len(stream)
self._stream_data_len = len(stream) - header_len
self._stats = Statistics(self)
return stream
def __read_binary_file(self):
with open(self.binary_path, "rb") as bin_file:
str_bitstream = bin_file.read().decode()
bool_list = list(
map(lambda bit: bool(int(bit)), list(str_bitstream)))
self.bitstream = BitStream()
self.bitstream.write(bool_list)
def split(mnemonic):
final_bits = mnemonic_to_bitstream(mnemonic)
# how long was the source entropy for this wordlist?
# according to https://github.com/bitcoin/bips/blob/master/bip-0039.mediawiki#generating-the-mnemonic
# CS = ENT / 32
# MS = (ENT + CS) / 11
# then...
# MS = ENT(33/32) / 11
# so...
# ENT = (352/33) * MS
payload_len = (352 / 33) * len(mnemonic)
if payload_len != int(payload_len):
raise ValueError(
"calculated {} bits of initial entropy".format(entropy_orig_len))
payload_len = int(payload_len)
# transfer bits until we reach the end of the source entropy
# all remaining bits will be the checksum
payload = BitStream()
for bit in final_bits.read(bool, payload_len):
payload.write(bit)
checksum = deepcopy(final_bits)
return [payload, checksum]
def load_plugin(name):
mod = import_module('.main', 'plugins.'+name)
bs = BitStream(bytes([]))
t = Table('inspect', None, bs)
plugin = Plugin()
plugin.name = name
for attr in dir(mod):
if attr.startswith('__'):
continue
obj = getattr(mod, attr)
if not inspect.isfunction(obj):
continue
sig = inspect.signature(obj)
try:
sig.bind(t, bs)
if attr == 'default_parser':
plugin.default = attr
plugin.exports[attr] = obj
except TypeError as e:
continue
if len(plugin.default) == 0 and len(plugin.exports) > 0:
plugin.default = list(plugin.exports.keys())[0]
plugin.doc.parse('plugins/'+name+'/doc.md')
return plugin
def checksum_is_good(mnemonic):
payload, checksum = split(mnemonic)
# hash the payload
hasher = sha256()
hasher.update(as_bytes(payload))
long_checksum = BitStream(hasher.digest())
new_checksum = BitStream()
for bit in long_checksum.read(bool, len(payload) / 32):
new_checksum.write(bit)
print(checksum)
print(new_checksum)
return checksum == new_checksum
def load_in(file_name, fixed_width_length):
with open(file_name, 'rb') as f:
file_content = f.read()
#Get input as stream of bits and split by 12
bit_stream_as_string = str(BitStream(file_content))
#If odd get last code
odd = False
if len(bit_stream_as_string) % fixed_width_length != 0:
last_code = int(bit_stream_as_string[-fixed_width_length:], 2)
bit_stream_as_string = bit_stream_as_string[:-16]
odd = True
code_array = [
bit_stream_as_string[i:i + fixed_width_length]
for i in range(0, len(bit_stream_as_string), fixed_width_length)
]
#Convert to codes
for i in range(0, len(code_array)):
code_array[i] = int(code_array[i], 2)
if odd:
code_array.append(last_code)
return code_array
def _interleaveGroups(self, dataGroups, ecGroups):
""" Interleaves data groups into the final bitstream payload, with remainder bits """
def interleaveTwo(bitStream, first, second):
# Interleaves two block groups together
hasSecond = len(second) != 0
firstLen = len(first[0])
secondLen = len(second[0]) if hasSecond else 0
for cw in range(max(firstLen, secondLen)):
if cw < firstLen:
for block in first:
bitStream.write(block[cw], uint8)
if cw < secondLen:
for block in second:
bitStream.write(block[cw], uint8)
# This is where the payload is stored
payload = BitStream()
# Interleave data blocks
interleaveTwo(payload, dataGroups[0], dataGroups[1])
# Interleave EC blocks
interleaveTwo(payload, ecGroups[0], ecGroups[1])
# Add remainder bits (zero-padding to match size constraints)
payload.write([False] * QRCode.REMAINDER_LIST[self.version - 1], bool)
return payload
def draw(self, cell):
if self._depth_function is None:
return
bits = BitStream()
bits = self._depth_function.encode(bits)
self._depth_function.decode(bits)
self._depth_function.uncompress(cell)
def test_encode_function_id(self):
bits = encode_function_id(0, BitStream())
self.assertEqual(bits, BitStream([False, False]))
bits = encode_function_id(1, BitStream())
self.assertEqual(bits, BitStream([False, True]))
bits = encode_function_id(2, BitStream())
self.assertEqual(bits, BitStream([True, False]))
bits = encode_function_id(3, BitStream())
self.assertEqual(bits, BitStream([True, True]))
def test_writer(self, stream):
assert stream == BitStream()
write_unary(stream, 0)
assert stream == make_bitstream('1')
write_unary(stream, 1)
assert stream == make_bitstream('101')
write_unary(stream, 2)
assert stream == make_bitstream('101001')
def __init__(self,
buffer=BitStream(),
state=DataBufferState.BUFFER_ENABLED,
flush=False,
watch=False):
self.buffer = buffer
self.state = state
self.flush = flush
self.watch = watch
def test_encode_decode(self):
fl = FunctionLeaf()
fl2 = FunctionLeaf()
fl.set_depth_function(MockF1())
bits = BitStream()
bits = fl.encode(bits)
fl2.decode(bits)
self.assertAlmostEquals(fl2.get_depth_function().get_max_val(),
fl.get_depth_function().get_max_val())
def read_bitstream(self, bitstream):
if isinstance(bitstream, BitStream):
self.bitstream = bitstream
elif isinstance(bitstream, str):
bool_list = list(map(lambda bit: bool(int(bit)), list(bitstream)))
self.bitstream = BitStream()
self.bitstream.write(bool_list)
else:
raise TypeError(
"The provided bitstream should already be a bitstream.BitStream instance or a string."
)
def get_peak_signal_to_noise(image: np.ndarray, f: DepthFunction):
uncompressed = np.zeros(image.shape)
f.compress(image)
bits = BitStream()
bits = f.encode(bits)
f.decode(bits)
f.uncompress(uncompressed)
if metrics.mean_squared_error(image, uncompressed) == 0:
return float('Inf')
sig = peak_signal_to_noise(image, uncompressed)
return sig
def encode(a_text: str, encoding_table: dict) -> BitStream:
"""
encode given text
:param a_text: text to encode
:param encoding_table: encoding table build using huffman coding
:return: encoded bitstream
"""
encoded = BitStream()
for char in a_text:
for bit in encoding_table[char]:
encoded.write(bit, bool)
return encoded
def convert_bit_to_array(v_bits, values_type, nb_values):
"""
Convert a binary array into specific type array
INPUT:
v_bits: 1d-array,
values_type: type of element in output array
nb_values: number of elements to read
OUTPUT:
array with specific elements type indide
"""
stream = BitStream()
stream.write(v_bits.astype(bool), bool)
return stream.read(values_type, nb_values)
def mnemonic_to_bitstream(mnemonic):
stream = BitStream()
for word in mnemonic:
value = words.index(word)
pointer = 0x1
for _ in range(11):
if pointer & value:
stream.write(True, bool)
else:
stream.write(False, bool)
pointer <<= 1
return stream
def convert_array_to_bit(v_values, values_type):
"""
Convert an values-array with specific type to an binary array
INPUT:
v_values: 1d-array, values which be converted
values_type: type of the v_values element.
Usefull types; np.int8, np.int16, np.int32, np.float32
OUTPUT:
1d-array, return an integer array with binary values (0 or 1)
"""
stream = BitStream()
stream.write(v_values, values_type)
bit_str = str(stream)
return np.array(list(bit_str), dtype=int), len(v_values)
def fix_checksum(mnemonic):
payload, checksum = split(mnemonic)
# hash the payload
hasher = sha256()
hasher.update(as_bytes(payload))
long_checksum = BitStream(hasher.digest())
# append bits from the beginning of the SHA256 to the end of the starting entropy
for bit in long_checksum.read(bool, len(payload) / 32):
payload.write(bit)
return bitstream_to_mnemonic(payload)
def expand(bit_stream: BitStream) -> BitStream:
"""
expand compressed bit stresm
:param bit_stream: bitstream to expand
:return: original stream before compression
"""
expanded = BitStream()
bit = True
while len(bit_stream):
count = bit_stream.read(int8, 1)[0]
for _ in range(count):
expanded.write(bit, bool)
bit = not bit
return expanded
def pack():
contents = open(sys.argv[2], 'rb').read()
packed_file = open(sys.argv[3], 'wb')
bits = BitStream(contents)
f = ''
i = len(bits)
while i > 0:
b = bits.read(16)
i -= 16
# dirty flip to deal with endianness
b = str(b)[8:] + str(b)[:8]
b = b[7:]
f += b
packed_bitstream = BitStream()
for i in f:
packed_bitstream.write(int(i), bool)
for i in range(0, len(packed_bitstream) / 8):
b = packed_bitstream.read(8)
packed_file.write(struct.pack('B', int(str(b), 2)))
def unpack():
contents = open(sys.argv[2], 'rb').read()
output_file = open(sys.argv[3], 'wb')
bits = BitStream(contents)
i = len(bits)
while i > 0:
b = bits.read(9)
i -= 9
b = '0000000' + str(b)
sh = int(b, 2)
x = struct.pack("H", sh)
output_file.write(x)
def _genEncodedData(self):
""" Turns input data into a bit stream with headers but no error correction """
output = BitStream()
# Add mode indicator. Using byte mode, so, write 0100
output.write([False, True, False, False], bool)
# Add character count indicator. The size of the indicator depends on the QR version in use
data_len = len(self.data_bytes)
if self.version <= 9:
output.write(data_len, uint8)
else:
output.write(data_len, uint16)
# Add data
output.write(self.data_bytes, bytes)
# Get required size
block_info = QRCode.BLOCK_LIST[self.version - 1][self.ecc]
req_bytes = None
if len(block_info) == 3:
req_bytes = block_info[1] * block_info[2]
else:
req_bytes = block_info[1] * block_info[2] + block_info[
3] * block_info[4]
req_bits = req_bytes * 8
# Add 4-bit (or less) zero padding
zero_count = min(4, req_bits - len(output))
output.write([False] * zero_count, bool)
# Align data to bytes
out_len = len(output)
aligned_bits = ceil(out_len / 8) * 8
output.write([False] * (aligned_bits - out_len), bool)
# Add padding bytes if necessary
aligned_bytes = aligned_bits // 8
alternate = False
for i in range(req_bytes - aligned_bytes):
if alternate:
output.write(17, uint8)
else:
output.write(236, uint8)
alternate = not alternate
return output.read(bytes)
def test_compression(self):
original_string = "this is some string to compress"
byte_string = original_string.encode('ascii')
original_stream = BitStream()
original_stream.write(byte_string, bytes)
original_stream_length = len(original_stream)
compressed_stream = compress(original_stream)
compressed_stream_length = len(compressed_stream)
expanded_stream = expand(compressed_stream)
retrieved_string = expanded_stream.read(bytes)
retrieved_string = retrieved_string.decode('ascii')
self.assertNotEqual(original_stream_length, compressed_stream_length)
self.assertEqual(original_string, retrieved_string)
def compress(bit_stream: BitStream) -> BitStream:
"""
compress a bit stream using run length encoding
-> replace replace series of same consecutive bit by they number
00001110000111 -> 4343 -> 100011100011
f the bits are not in long consecutive series of same bits the
compressed file can end up being bigger then original
:param bit_stream: bit stream to compress
:return: compressed stream
"""
compressed = BitStream()
bit = True
while len(bit_stream):
count = count_bits(bit_stream, bit, 256)
compressed.write(count, int8)
bit = not bit
return compressed
def decode(self, data):
stream = BitStream(data)
mode = MODE_NONE
while True:
first_bit = stream.read(bool)
if first_bit:
ones = leading_ones(stream)
if mode == MODE_SCHEME1:
if not first_bit:
self.add_byte(ord(stream.read(bytes, 1)))
continue
elif ones < 8:
cnt_field = stream.read(
BitStream, ones + 1 if ones < 4 else 11 - ones)
if ones > 4:
cnt_field += (ones - 4) << (12 - ones)
cnt = int(str(cnt_field), 2) + (2 << ones)
loc = int(str(stream.read(BitStream, 10)), 2)
for i in range(loc, loc + cnt):
self.add_byte(self.buffer[i % 1024])
continue
if first_bit and ones == 8:
control = str(stream.read(BitStream, 3))
if control == '101': # Scheme 1
mode = MODE_SCHEME1
elif control == '100': # End of record (EOR)
self.output.extend(self.buffer[:self.buffer_offset])
break
else:
raise ValueError(
"Can't parse SLDC, unsupported control symbol: %s" %
(control))
else:
raise ValueError("Can't parse SLDC, unknown action")
return self.output
def default_parser(d: Table, bs: BitStream):
analyse_nalu_header(d, bs)
rbsp_bs = BitStream(d.nal_unit.rbsp)
d.context.add_exist(d.nal_unit)
if d.nal_unit.nal_unit_type == 9:
d.add_table(access_unit_delimiter_rbsp, bs=rbsp_bs)
elif d.nal_unit.nal_unit_type == 6:
d.add_table(sei_rbsp, bs=rbsp_bs)
elif d.nal_unit.nal_unit_type in [1, 5]:
d.add_table(slice_layer_without_partitioning_rbsp, bs=rbsp_bs)
elif d.nal_unit.nal_unit_type == 7:
d.add_table(seq_parameter_set_rbsp, bs=rbsp_bs)
elif d.nal_unit.nal_unit_type == 8:
d.add_table(pic_parameter_set_rbsp, bs=rbsp_bs)
elif d.nal_unit.nal_unit_type == 15:
d.add_table(subset_seq_parameter_set_rbsp, bs=rbsp_bs)
elif d.nal_unit.nal_unit_type == 20:
d.add_table(slice_layer_extension_rbsp, bs=rbsp_bs)
def toCode(r):
stream = bitarray()
for i in range(0, len(r)):
tempStream = BitStream()
if ',' in r[i]:
temp = r[i].split(',')
else:
temp = r[i]
if temp == 'True':
tempStream.write(True, bool)
elif temp == 'False':
tempStream.write(False, bool)
else:
for j in range(0, len(temp)):
tempStream.write(str2bool(temp[j]), bool)
stream = numpy.append(stream, tempStream)
return stream
