def trace(request):
from random import randrange
from bitstring import BitArray
trace_recs_num = randrange(2, 5)
trace_recs = []
for i in range(trace_recs_num):
peer_ID = BitArray(int=randrange(0, 1000), length=256)
KX_port = randrange(10000, 20000)
IPv4_address = BitArray(int=randrange(0, 1000), length=32)
IPv6_address = BitArray(int=randrange(0, 1000), length=128)
trace_rec = { 'peer_ID': peer_ID.tobytes(),
'KX_port': KX_port,
'IPv4_address': IPv4_address.tobytes(),
'IPv6_address': IPv6_address.tobytes() }
trace_recs.append(trace_rec)
self.logger.debug('process_dht_request->trace.str(peer_ID): %s' % str(peer_ID))
self.logger.debug('process_dht_request->trace.str(KX_port): %s' % str(KX_port))
self.logger.debug('process_dht_request->trace.str(IPv4_address): %s' % str(IPv4_address))
self.logger.debug('process_dht_request->trace.str(IPv6_address): %s' % str(IPv6_address))
response = { 'trace': trace_recs,
'all_ok': True }
self.logger.debug('process_dht_request->find_value.len(response["trace"]): %s' % len(response['trace']))
return response
def encode_block(in_bytes):
if len(in_bytes) == 1:
block = BitArray()
block.append(Bits('0b1'))
block.append(in_bytes)
return block.tobytes()
bw_xf, eof_idx = burrows_wheeler_transform(in_bytes)
front_xf = move_to_front_transform(bw_xf)
rle_data = run_length_encode(front_xf)
huff_data, huff_symbols, serialized_tree = huffman_encode(rle_data,
symbol_bits=8)
huff_len = len(huff_data)
tree_len = len(serialized_tree)
block = BitArray()
block.append(Bits('0b0'))
block.append(Bits(uint=tree_len, length=16))
block.append(Bits(serialized_tree))
block.append(Bits(uint=huff_symbols, length=16))
block.append(Bits(uint=huff_len, length=16))
block.append(Bits(huff_data))
block.append(Bits(uint=eof_idx, length=BLOCK_SIZE_BITS))
return block.tobytes()
def generateConfirmationKey(identitySecret,time,tag=""):
identitysecret = b64decode(identitySecret)
secret = BitArray(bytes=identitysecret,length=len(identitysecret)*8)
if tag != "":
tagBuff = BitArray(bytes=tag,length=len(tag)*8)
buff = BitArray(4*8)
time = int(time)
buff.append(BitArray(int=time,length=32))
if tag != "":
buff.append(tagBuff)
conf_hmac = hmac.new(secret.tobytes(),buff.tobytes(),hashlib.sha1)
return b64encode(conf_hmac.digest())
def exec(self):
message = f'Hello World at {date.today()}'
message_bytes = bytes(message, encoding='utf-8')
bit_array = BitArray(bytes=message_bytes)
pr('original', message)
pr('\t==> hex ', bit_array.hex)
pr('\t==> bin ', bit_array.bin)
pr('\t==> bytes ', bit_array.tobytes())
pr('\t==> str ', str(bit_array.tobytes(), encoding='utf-8'))
pr('\t==> length ', bit_array.length // 8)
pr('\t==> 1st byte ', bit_array[:8].tobytes())
pr('\t==> 2nd byte ', bit_array[8:8 * 2].tobytes())
pr('\t==> last byte', bit_array[-8:].tobytes())
def to_send(self ,s):
#make str bits to bytes
data_in_bit = BitArray(bin=s)
print s
after_dewhite_bin = data_in_bit.tobytes()
after_dewhite_str = array.array( "B",(after_dewhite_bin))
self.message_port_pub(pmt.intern('out'), pmt.cons(pmt.PMT_NIL, pmt.init_u8vector(len(after_dewhite_bin), bytearray(after_dewhite_bin))))
def sendFirst(id, socket, error):
#Input
num1 = int(input("Number: "))
while num1 < -2147483648 or num1 > 2147483647:
print("Wrong number! Input integer in range ",
"from -2,147,483,648 to 2,147,483,647 ")
num1 = int(input("First number: "))
#Pack
packg = BitArray()
packg = pack("+", num1, 0, id, 0, 1, 0)
#Send package
print(
f"Sending first: {packg.bin[0:3]} {packg.bin[3:35]} {packg.bin[35:37]}",
f" {packg.bin[37:45]} {packg.bin[45:46]} {packg.bin[46:47]} {packg.bin[47:]}"
)
socket.send(packg.tobytes())
#Recieve package
data = BitArray()
data.append(socket.recv(1024))
print(
f"Recieved first: {data.bin[0:3]} {data.bin[3:35]} {data.bin[35:37]}",
f" {data.bin[37:45]} {data.bin[45:46]} {data.bin[46:47]} {data.bin[47:]}"
)
print("Sending next")
def start_fuzz():
# rtp = b'\x80\x63\x78\xa6\x00\x00\x17\x65\xe6\x89\xe9\x31' + (b'\x61' * 1000)
rtp_packet = BitArray()
# rtp_packet += 'uint:2=%d' % rtp_packet_types['version']
# rtp_packet += 'uint:1=%d' % rtp_packet_types['padding']
# rtp_packet += 'uint:1=%d' % rtp_packet_types['extension']
# rtp_packet += 'uint:4=%d' % rtp_packet_types['crsc_count']
# rtp_packet += 'uint:1=%d' % rtp_packet_types['marker']
# raw_input(rtp_packet)
# rtp_packet += 'A'
# raw_input(rtp_packet)
# rtp_packet += 'uint:7=%d' % rtp_packet_types['payload_type']
test_case_count = 0
fuzz_payloadtype(128)
print "[*] Test Function : fuzz_payloadtype End"
# raw_input(TEST_CASE_ID)
fuzz_timestamp(40000)
print "[*] Test Function : fuzz_timestamp End"
fuzz_sequence_number(20000)
print "[*] Test Function : fuzz_sequence_number End"
fuzz_ssrc_id(40000)
print "[*] Test Function : fuzz_ssrc_id End"
# rtp_packet += 'uint:16=%d' % rtp_packet_types['sequence_number']
# rtp_packet += 'uint:32=%d' % rtp_packet_types['timev']
# rtp_packet += 'uint:32=%d' % rtp_packet_types['ssrc_id']
# rtp_packet.append('0x41'*80)
# print(rtp_packet)
# print(rtp_packet[4:8])
# raw_input(len(rtp_packet[4:8]))
# print(rtp_packet)
# raw_input(len(rtp_packet))
# for count in range(STOPAFTER):
pkt = rtp_packet.tobytes()
def get_bytes(self):
bitstream = BitArray()
current_address = self.imagebase
labels_to_addresses_map = {}
for ast_node in self.ast:
ast_node.set_node_bitfields(self.compute_node_bitfields(ast_node))
ast_node.set_node_address(current_address)
for lbl in ast_node.labels:
labels_to_addresses_map[lbl] = ast_node.address
# TODO: What happens with addressing in non-standard word sizes?
bit_length = self.bitfields_to_bitarray(
ast_node.node_bitfields).length
byte_length = int(bit_length / DEFAULT_BYTE_BITSIZE)
if bit_length % DEFAULT_BYTE_BITSIZE != 0:
byte_length += 1
current_address += byte_length
for ast_node in self.ast:
self.update_label_placeholders(ast_node, labels_to_addresses_map)
for ast_node in self.ast:
ast_node.set_node_bitfields(self.compute_node_bitfields(ast_node))
node_bitarray = self.bitfields_to_bitarray(ast_node.node_bitfields)
bitstream.append(node_bitarray)
return bitstream.tobytes()
def initFile(self): file_name = self.file_to_stream['name'] fout = open(file_name, "wb") data = BitArray(int(self.file_to_stream_length)*8) data = data.tobytes() fout.write(data) fout.close()
def lzwv_encode(in_bytes):
max_entries = 2**MAX_CODE_LEN
in_array = bytearray(in_bytes)
dictionary = {bytes([i]): i for i in range(256)}
code_len = 9
out_array = BitArray()
while len(in_array) > 1:
# Find string s that's not in the dictionary
s = b''
while (s in dictionary or s == b'') and len(in_array) > 0:
s = s + bytes([in_array.pop(0)])
in_array.insert(0, s[-1])
# Emit code for the s[:-1], which is in the dictionary
out_array.append(Bits(uint=dictionary[s[:-1]], length=code_len))
# If there's room in the dictionary, add s
c = len(dictionary)
if c < max_entries:
dictionary[s] = c
# If just added max value for code len, increment it.
if c == 2**code_len - 1:
code_len = min(MAX_CODE_LEN, code_len + 1)
# Emit code for final value
out_array.append(
Bits(uint=dictionary[bytes([in_array.pop(0)])], length=code_len))
return out_array.tobytes()
def test_for_strict_avalanche():
"""
Function to apply the SAC test defined in https://eprint.iacr.org/2010/564.pdf.
"""
with open('key.txt', 'rb') as file:
key = file.read()
associated_data = generate_nonce(16).encode(encoding='utf-8')
sample_size = 32768
plaintext_bit_size = 64
plaintext_byte_length = 8
algorithms = [
"ascon128.so", "ascon128a.so", "isap_a_128.so", "isap_k_128.so",
"oribatida_128.so", "locus_64.so", "lotus_64.so", "spoc_64.so"
]
print(algorithms)
for alg in algorithms:
print(alg)
if "128" in alg:
tagsize = 16
else:
tagsize = 8
matrix = np.zeros(
(plaintext_bit_size, plaintext_bit_size + (tagsize * 8)),
dtype=int)
for _ in range(0, sample_size):
plaintext = random.getrandbits(plaintext_bit_size)
nonce = generate_nonce(16).encode(encoding='utf-8')
variant = "encryption_interfaces/" + alg
original_cipher = encrypt(
key, nonce, associated_data,
plaintext.to_bytes(plaintext_byte_length, sys.byteorder),
variant, tagsize)
for bit_position in range(0, plaintext_bit_size):
temp_plaintext = BitArray(
plaintext.to_bytes(plaintext_byte_length, sys.byteorder))
temp_plaintext.invert(bit_position)
nonce = generate_nonce(16).encode(encoding='utf-8')
cipher = encrypt(key, nonce, associated_data,
temp_plaintext.tobytes(), variant, tagsize)
cipher_array = BitArray(cipher) ^ BitArray(original_cipher)
for cipher_position in range(
0, plaintext_bit_size + (tagsize * 8)):
matrix[bit_position][cipher_position] += int(
cipher_array.bin[cipher_position])
flip_probabilty = 0.5
expected_count = sample_size * flip_probabilty
with open('sac_results.txt', 'a') as f:
f.write(variant + "\n")
for row in range(0, (plaintext_bit_size)):
for column in range(0, plaintext_bit_size + (tagsize * 8)):
statistics, pvalue = chisquare([
int(matrix[row][column]),
int(sample_size - matrix[row][column])
], [expected_count, expected_count])
f.write(str(pvalue) + "\n")
print("Values Stat & P-Value")
print(statistics)
print(str(pvalue))
def translate():
data = input()
bin_data = BitArray(hex=data)
print(bin_data.bin)
packet = Turbo(parse=bin_data.tobytes())
print(packet.print())
def amrPayload2Storage_EfficientMode(payload):
if (amr_type == 'nb'):
# AMR-NR
bitlen = [95, 103, 118, 134, 148, 159, 204, 244, 39]
# print "selected codec is amr nb"
elif (amr_type == 'wb'):
# AMR-WB TS 26.201 - total bits
bitlen = [132, 177, 253, 285, 317, 365, 397, 461, 477, 40]
# print "selected codec is amr wb"
else:
exit(1)
amr = BitArray(bytes=payload)
cmr = amr[0:4]
mode = amr[5:9]
#print(mode.uint)
#assert mode.uint >=0 and mode.uint <=8
if not (mode.uint >= 0 and mode.uint <= 8):
return
else:
qual = amr[9:10]
voice = amr[10:10 + bitlen[mode.uint]]
#print("cmr=%d\tmod=%d\tqual=%d\tvoicebits=%d" % (cmr.uint,mode.uint,qual.uint,voice.len))
storage = BitArray(bin='0')
storage.append(mode)
storage.append(qual)
storage.append('0b00') # padding
assert storage.len == 8, "check length of storage header is one byte"
storage.append(voice)
return storage.tobytes()
def gen_frame_header(self, craft_id, virtual_channel_id,
virtual_channel_count, is_encrypt):
# 输入类型检查
# 范围检查, 首先定义了bit流的长度, 用overwrite函数时可保证长度(范围)安全
h_craft_id = BitArray(8)
h_virtual_channel_id = BitArray(6)
h_virtual_channel_count = BitArray(24)
h_is_encrypt = BitArray(1)
h_craft_id.overwrite(craft_id, 0)
h_virtual_channel_id.overwrite(
BitArray(uint=virtual_channel_id, length=6), 0)
h_virtual_channel_count.overwrite(
BitArray(uint=virtual_channel_count, length=24), 0)
h_is_encrypt.overwrite(is_encrypt, 0)
header = BitArray()
header.append('0b00')
header.append(h_craft_id)
header.append(h_virtual_channel_id)
header.append(h_virtual_channel_count)
header.append('0b00')
header.append(h_is_encrypt)
header.append('0b00000')
if is_encrypt == '0b0':
self.__is_encrypt = False
else:
self.__is_encrypt = True
self.__frame_header = header.tobytes()
def fuzz_SR_packet_basis(count):
global TEST_CASE_ID
ssrc_id_max = 2**32 - 1
for i in range(count):
sr_packet = BitArray()
rtcp_SR_types['length'] = 6
sr_packet += 'uint:2=%d' % rtcp_SR_types['version']
sr_packet += 'uint:1=%d' % rtcp_SR_types['padding']
sr_packet += 'uint:5=%d' % rtcp_SR_types['rc']
sr_packet += 'uint:8=%d' % rtcp_SR_types['packet_type']
sr_packet += 'uint:16=%d' % rtcp_SR_types['length']
print("[*] Test case ID:%d" % (TEST_CASE_ID) + " START")
TEST_CASE_ID += 1
rand_ssrc_id = random.randint(0, ssrc_id_max)
rand_ntp_timestamp_MSL = random.randint(0, ssrc_id_max)
rand_ntp_timestamp_LSW = random.randint(0, ssrc_id_max)
rand_rtp_timestamp = random.randint(0, ssrc_id_max)
rand_ntp_timestamp_LSW = random.randint(0, ssrc_id_max)
rand_sender_packet_count = random.randint(0, ssrc_id_max)
rand_sender_octet_count = random.randint(0, ssrc_id_max)
print(
"[*] Test type : Basis SR packet mix fuzzing Test , Payload : %d "
% (rand_ssrc_id))
sr_packet += 'uint:32=%d' % rand_ssrc_id
sr_packet += 'uint:32=%d' % rand_ntp_timestamp_MSL
sr_packet += 'uint:32=%d' % rand_ntp_timestamp_LSW
sr_packet += 'uint:32=%d' % rand_rtp_timestamp
sr_packet += 'uint:32=%d' % rand_sender_packet_count
sr_packet += 'uint:32=%d' % rand_sender_octet_count
send_to_target(sr_packet.tobytes())
def toCharset(charset, string, toUpper=False):
"""Convert unicode text to various charsets using a lookup table
Args:
charset (string): lookup table
string (string): unicode string to convert
toUpper (bool, optional): Make chars uppercase before converting
(intended for 6 bit charsets or charsets that do not support uppercase).
Defaults to False.
Returns:
bytes: sequence of bytes (split into bits of length
"""
asciibits = BitArray()
cut = int(8 - log2(len(charset)))
if toUpper:
chars = chars.upper()
for char in string:
bits = charset.find(char)
if bits < 0:
if charset == GOST:
bits = 25 # use '*' for GOST
else:
bits = charset.find("?")
asciibits.append(BitArray(bytes([bits]))[cut:])
return asciibits.tobytes()
def get_bits_from_file(filepath, key):
print('Reading file...')
bitarray = BitArray(filename=filepath)
# adding a delimiter to know when the file ends to avoid corrupted files
# when retrieving
cipher = AES.new(key, AES.MODE_EAX, nonce=SALT)
ciphertext, tag = cipher.encrypt_and_digest(bitarray.tobytes())
filename = os.path.basename(filepath)
pickled = pickle.dumps({'tag':tag,
'data':ciphertext,
'filename':filepath})
print('Zipping...')
#zip
out = io.BytesIO()
with gzip.GzipFile(fileobj=out, mode='w') as fo:
fo.write(pickled)
zip = out.getvalue()
#zip
del bitarray
del pickled
bitarray = BitArray(zip)
return bitarray.bin
def build_cmd(self, command, arg, uid, data_type):
cmd = command.encode(encoding="utf-8")
if len(command) < 10:
cmd += bytes(10 - len(command))
# add data padding
if arg is None:
cmd += int(0).to_bytes(8, byteorder='big')
elif data_type is self.CmdDataType.NUMBER or data_type is self.CmdDataType.RECORD:
cmd += int(arg).to_bytes(8, byteorder='big')
elif data_type is self.CmdDataType.SIGNED:
cmd += int(arg).to_bytes(8, byteorder='big', signed=True)
elif data_type is self.CmdDataType.STRING:
cmd += self.str2bit(arg)
elif data_type is self.CmdDataType.STORED:
ba = BitArray()
ba += [0] * 38
ba.append(
BitArray(int(arg['resolution']).to_bytes(1,
byteorder='big'))[2:])
ba.append(
BitArray(int(arg['offset']).to_bytes(3, byteorder='big'))[4:])
cmd += ba.tobytes()
else:
self.logger.debug("Unhandled data type: %s" % (str(data_type)))
raise self.UnknownDataType("Unhandled data type: %s" %
str(data_type))
# add index
cmd += uid.to_bytes(2, byteorder='big')
return cmd
def golomb_encode(hashes_list, hash_length, M):
hash_len_bits = hash_length * 8
# Must be sorted for delta diff to make sense
assert sorted(hashes_list)
# Must not contain duplicates.
assert len(hashes_list) == len(set(hashes_list))
m_bits = int(math.log(M, 2))
assert abs(math.log(M, 2) - float(m_bits)) < 0.00001
outarray = BitArray(bytes = hashes_list[0], length=hash_len_bits)
min_is_zero = False
prev = BitArray(bytes = hashes_list[0], length=hash_len_bits)
for curr_hash in hashes_list[1:]:
curr = BitArray(bytes=curr_hash, length=hash_len_bits)
N = curr.uint - prev.uint
q = int(math.floor(N / M))
r = N % M
if q == 0:
outarray.append(Bits(bin='0b0'))
min_is_zero = True
else:
outarray.append(Bits(bin=bin(2**q - 1) + '0'))
outarray.append(Bits(uint=r, length=m_bits))
prev = curr
if not min_is_zero:
print "Inefficient encoding: Minimum is not zero."
return outarray.tobytes()
def hf_compress(byte_array):
MAX_SIZE = 256
freq_table = [0 for i in range(MAX_SIZE)]
for b in byte_array:
freq_table[b] += 1
hf_tree = create_huffman_tree(freq_table)
hf_codes = parse_tree(hf_tree)
result = BitArray(bin_pad(len(hf_codes), 8))
for n in hf_codes:
byte_key = bytes([n[0]])
result.append(byte_key)
code_length = bin_pad(len(n[1]), 4)
result.append(code_length)
result.append('0b' + n[1])
for b in byte_array:
code = [c[1] for c in hf_codes if c[0] == b].pop()
result.append('0b' + code)
# find required padding to make bits go into bytes
padding_size = 8 - len(result.bin) % 8
result.prepend(bin_pad(padding_size, 8))
return result.tobytes()
def scramble(data, offset, reverse=False) -> bytes:
"""
Scrambles/descrambles bytes with specified offset.
"""
a = BitArray(data, endian="little")
b = BitArray(length=len(a))
if len(offset) != 2:
raise ValueError("The offset must be a tuple with two values!")
p1, p2 = offset
if p1 >= p2:
raise ValueError("The first index must be less than the second index!")
for i in range(len(a)):
if (i < p1):
b[i] = a[i]
elif (i < p2):
x = (a[i - p1] if reverse else b[i - p1])
b[i] = a[i] ^ x
else:
x = (a[i - p1] if reverse else b[i - p1])
y = (a[i - p2] if reverse else b[i - p2])
b[i] = a[i] ^ x ^ y
return b.tobytes()
def compress_file(self, input_file_name, output_file_name):
tree: HuffmanTree = HuffmanTree(input_file_name)
root: Node = tree.construct_tree()
if root is not None:
padding: int = 0
with open(input_file_name) as file_in:
with open(output_file_name, 'wb') as file_out:
# Write Place Holder for Padding
file_out.write(BitArray('0b0000').tobytes())
current: str = file_in.readline()
bits: BitArray = BitArray()
bits += tree.to_bits()
while current != '':
for char in current:
bits += tree.get_character(char)
if(len(bits) % 8 == 0):
file_out.write(bits.tobytes())
bits = BitArray()
current = file_in.readline()
file_out.write(bits.tobytes())
padding = 8 - (len(bits) % 8)
# Write the Padding Number to the start of the file.
with open(output_file_name, 'rb+') as fp:
fp.seek(0, 0)
fp.write(padding.to_bytes(1, byteorder="little"))
else:
with open(output_file_name, 'w'):
pass
def fuzz_ssrc_id(count):
global TEST_CASE_ID
ssrc_id_max = 2**32 - 1
for i in range(count):
rtp_packet = BitArray()
rtp_packet += 'uint:2=%d' % rtp_packet_types['version']
rtp_packet += 'uint:1=%d' % rtp_packet_types['padding']
rtp_packet += 'uint:1=%d' % rtp_packet_types['extension']
rtp_packet += 'uint:4=%d' % rtp_packet_types['crsc_count']
rtp_packet += 'uint:1=%d' % rtp_packet_types['marker']
rtp_packet += 'uint:7=%d' % rtp_packet_types['payload_type']
rtp_packet += 'uint:16=%d' % rtp_packet_types['sequence_number']
rtp_packet += 'uint:32=%d' % rtp_packet_types['timev']
print("[*] Test case ID:%d" % (TEST_CASE_ID) + " START")
TEST_CASE_ID += 1
rand_number = random.randint(0, ssrc_id_max)
print("[*] Test type : Basis SSRC identifier Test , Payload : %d " %
(rand_number))
rtp_packet += 'uint:32=%d' % rand_number
# rtp_packet += 'uint:32=%d' % rtp_packet_types['ssrc_id']
send_to_target(rtp_packet.tobytes())
def compress(self):
compressed_filepath = self.destination_path + "/" + self.filename + "-" + self.file_ext[
1:] + ".bin"
# Coding for .txt and .docx files.
if self.file_ext == ".txt" or ".docx":
self.char_to_code = self.encode()
self.code_to_char = {
value: key
for key, value in self.char_to_code.items()
}
encoded_content = "".join(
[self.char_to_code[a] for a in self.content])
self.bit_length = len(encoded_content)
bit_array = BitArray(bin=encoded_content)
with open(compressed_filepath, 'wb') as compressed_file:
compressed_file.write(bit_array.tobytes())
compressed_file.close()
return compressed_filepath
else:
raise ValueError("HuffmanCoder Compression Error: Wrong File Type")
return
def seed_to_signing_key(seed, index=0):
"""
from raiblocks seed to private key (ed25519 seed)
:param str seed: the hex string of raiblocks seed, 64 characters long
:param int index: the index of wallet address
:rtype: bytes
:return: signing key
Code Example::
seed = 'CCC020CAF01C98B6B076A9F00573503E0D7FBA85BC7CA21AF3B3C02A2DDF5326'
sk = seed_to_signing_key(seed, 1)
print(sk.hex())
"""
h = blake2b(digest_size=32)
seed_bytes = bytes.fromhex(seed)
h.update(seed_bytes)
index_bits = BitArray(uint=index, length=32)
index_bytes = index_bits.tobytes()
h.update(index_bytes)
return h.digest()
def send_rtp_packet(header, payload, ip, port, packets_in_payload=2, number=0):
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as my_socket:
my_socket.connect((ip, port))
if (number == 0):
number = 100000000
try:
for i in range(number):
packet = BitArray()
packet.append(header.version)
packet.append(header.pad_flag)
packet.append(header.ext_flag)
packet.append(header.cc)
packet.append(header.marker)
packet.append(header.payload_type)
packet.append(BitArray(uint=header.seq_number, length=16))
packet.append(BitArray(uint=header.timestamp, length=32))
packet.append(header.ssrc)
packet.append(header.csrc)
if header.ext_flag.bin == '1':
print('aqui va la extension')
print('Tamaño de la cabecera RTP: ' + str(len(packet.bin)))
# Cuantos paquetes mp3 metemos en el mismo paquete RTP
for j in range(packets_in_payload):
payload._take_mp3_frame()
packet.append(BitArray(bin=payload.frame))
print(payload.frame[0:32])
packetBytes = packet.tobytes()
my_socket.send(packetBytes)
header._next(payload.frameTimeMs)
except IndexError:
pass
def coding_with_hamming_code(data):
msg = BitArray()
packet_list = HammingCode.split_message(
BitArray(data), HammingCode.SIZE_OF_INFORMATION_PART)
for i in packet_list:
msg += HammingCode.insert_control_bits(i)
return msg.tobytes()
def LZCompress(filepath, wBits=16, lBits=8):
filepath = "Files/" + filepath
#window size is 2^number of bits allocated to it - 1
windowSize = 2**wBits - 1
lookaheadSize = 2**lBits - 1
try:
file = open(filepath, "r")
except:
print("Failed to open file, please check filepath is correct.")
return False
data = file.read()
file.close()
#tracking the file size to compare on compression ratio
originalFileSize = os.path.getsize(filepath)
outputBitString = BitArray()
i = 0
#loop through the entire data string
while (i < len(data)):
match = findLargestMatch(data, i, windowSize, lookaheadSize)
#create bit strings representing integers in the tuple
distance = BitArray(uint=match[0], length=wBits)
bitsToCopy = BitArray(uint=match[1], length=lBits)
outputBitString.append(distance)
outputBitString.append(bitsToCopy)
if match[2] != '-1':
#create bit string to represent character, using 8 bits for each character
nextChar = BitArray(uint=ord(match[2]), length=8)
outputBitString.append(nextChar)
i += match[1] + 1
#setting up the file paths
compressedFilePath = filepath.split('.')[0] + "Compressed"
decompressedFilePath = filepath.split(
'.')[0] + "Decompressed" + '.' + filepath.split('.')[1]
#writing the compressed file
compressedFile = open(compressedFilePath, 'wb')
compressedFile.write(outputBitString.tobytes())
compressedFile.close()
#calculate and output the compression ratio
compressedFileSize = os.path.getsize(compressedFilePath)
print("Compression ratio: " +
str(round(compressedFileSize / originalFileSize, 3)))
#read compressed file, decode and output decoding results to a new file
decompressedFile = open(decompressedFilePath, 'w')
decompressedFile.write(LZDecompress(compressedFilePath, wBits, lBits))
decompressedFile.close()
def lz77_encode(input_data, window_bits=DEFAULT_WINDOW_BITS):
tokens = lz77_encode_to_tokens(input_data, window_bits)
out = BitArray()
for t in tokens:
out.append(Bits(uint=t[0], length=window_bits))
out.append(Bits(uint=t[1], length=REFERENCE_SIZE_BITS))
out.append(Bits(uint=t[2], length=8))
return out.tobytes()
def test_encode_bitfield(self):
bitfield_length = 50
bitfield = BitArray([i % 2 for i in range(bitfield_length)])
bitfield_as_bytes = bitfield.tobytes()
truth = struct.pack('>Ib', 1 + len(bitfield_as_bytes),
peer.BitField.ID) + bitfield_as_bytes
message = peer.BitField(bitfield)
self.assertEqual(message.encode(), truth)
def send_bitfield(self):
"""
Tell the peer all the pieces you have in a really compact form.
"""
# the pieces are compacted in sequence in to bits
field = BitArray(
map(lambda sha, piece: sha == piece.digest,
self.torrent.pieces.items()))
self.send_payload(5, field.tobytes())
def int_to_bytes(i, length=64):
"""
Convert int to bytes, length is bit length.
"""
if not isinstance(i, int):
raise ValueError('int_to_bytes: data is not int')
i_bits = BitArray(uint=i, length=length)
return i_bits.tobytes()
def colors_to_text(colors, n_tons, coding):
n_colors = n_tons**3
bits_array = base_change(colors, n_colors, 2)
bit_string = ''.join(map(str, bits_array))
bits = BitArray('0b' + bit_string)
encoded = bits.tobytes()
text = decode(encoded, coding)
return text
def lz77huff_encode(input_data, window_bits=DEFAULT_WINDOW_BITS):
tokens = lz77_encode_to_tokens(input_data, window_bits)
symbols = [s for tok in tokens for s in tok]
bits = BitArray()
for s in symbols:
bits.append(Bits(uint=s, length=window_bits))
encoded_data, num_symbols, serialized_tree = huffman_encode(bits.tobytes(),
symbol_bits=window_bits)
return encoded_data, num_symbols, serialized_tree
def generateAuthCode(secret,offset=0):
secret = b64decode(secret)
secret = BitArray(bytes=secret,length=len(secret)*8)
buff = BitArray(8*8)
timestamp = timeOffset(offset)
buff[4*8:] = int(timestamp/30)
auth_hmac = hmac.new(secret.tobytes(),buff.tobytes(),hashlib.sha1)
hashed = auth_hmac.digest()
hashed = BitArray(bytes=hashed,length=len(hashed)*8)
start = hashed[(19*8):(19*8)+8] & BitArray('0x0f')
hash_slice = hashed[start.int*8:(start.int*8)+(4*8)]
fullcode = hash_slice & BitArray("0x7fffffff")
fullcode = fullcode.int
chars = '23456789BCDFGHJKMNPQRTVWXY'
code = ""
for x in range(5):
code += chars[int(fullcode % len(chars))]
fullcode = fullcode/len(chars)
return code
def readAmbeFrameFromUDP( self, _sock ):
_ambeAll = BitArray() # Start with an empty array
for i in range(0, 3):
_ambe = self.readSock(_sock,7) # Read AMBE from the socket
if _ambe:
_ambe1 = BitArray('0x'+h(_ambe[0:49]))
_ambeAll += _ambe1[0:50] # Append the 49 bits to the string
else:
break
return _ambeAll.tobytes() # Return the 49 * 3 as an array of bytes
def send_bitfield(self):
"""
Tell the peer all the pieces you have in a really compact form.
"""
# the pieces are compacted in sequence in to bits
field = BitArray(
map(
lambda (sha, piece): sha == piece.digest,
self.torrent.pieces.items()
)
)
self.send_payload(5, field.tobytes())
def initFile(self): file_name = self.file_to_stream['name'] # Check if File Already Exists, If yes, then Read it. if os.path.isfile(file_name) == True: # File Exists print "File Already Exists. Checking For Partial Streaming" self.reCheck(file_name) else : fout = open(file_name, "wb") data = BitArray(int(self.file_to_stream_length)*8) data = data.tobytes() fout.write(data) fout.close()
def find_value(request):
# TODO: Do the same as in store request above, and delete the lines below
# TODO: in the algorithm of this function, the peer must decide if it is already storing the key/value pair in question
# by issuing a self.data_server.get(key), and checking the result
from random import randrange
from bitstring import BitArray
content = BitArray(int=randrange(0, 1000), length=256)
self.logger.debug('process_dht_request->find_value.str(content): %s' % str(content))
self.logger.debug('process_dht_request->find_value.len(str(content)): %s' % len(str(content)))
response = { 'content': content.tobytes(),
'all_ok': True }
self.logger.debug('process_dht_request->find_value.len(response["content"]): %s' % len(response['content']))
return response
def send(self):
toSend = BitArray()
for i in range(0,3):
frequency = BitArray(uint=self.frequencies[i], length=3)
intensity = BitArray(uint=self.intensities[i], length=3)
state = BitArray(uint=1, length=3)
toSend.append(intensity)
toSend.append(frequency)
toSend.append(state)
toSend.append('0b00000')
toSend.append('0x00')
if self.enable_bluetooth:
rospy.loginfo("sending msg(hex): %s" %(toSend))
rospy.loginfo("sending msg(bin): %s"%(toSend.bin))
self.bt.send(self.toConnect, str(toSend.tobytes()))
else:
rospy.logdebug("sending msg: %s" %(toSend))
rospy.logdebug(str(toSend.bin))
def double_DES(key, plain): keybits = BitArray(hex = key) plainbits= BitArray(hex = plain) if len(keybits) != 112: print "length of double DES key is not 112 bits!" return if len(plainbits) != 64: print "length of double DES plain text is not 64 bits!" return rev_keybytes = '' for k in keybits.cut(7): rev_keybytes += (odd_parity[k.bin]) obj1 = DES.new(rev_keybytes[0:8], DES.MODE_ECB) midbytes = obj1.encrypt(plainbits.tobytes()) obj2 = DES.new(rev_keybytes[8:16], DES.MODE_ECB) cipherbytes = obj2.encrypt(midbytes) return BitArray(bytes = cipherbytes).hex
def golomb_encode(hashes_list, hash_length, M):
"""Given a sorted list of fixed-size values, compress it by
using Golomb coding to represent the difference between the values."""
hash_len_bits = hash_length * 8
# Must be sorted for deltas to be small and easily compressable.
assert sorted(hashes_list) == hashes_list
# Must not contain duplicates.
assert len(hashes_list) == len(set(hashes_list))
# M is the tunable parameter.
m_bits = int(math.log(M, 2))
# Make sure that M is a power of 2.
assert M > 0 and not (M & (M - 1))
# First item in the output bit array is the first hash value.
outarray = BitArray(bytes = hashes_list[0], length=hash_len_bits)
# Set to true when the diff value / M == 0.
# If no such value exists then the chosen M is too small, so warn.
min_is_zero = False
prev = BitArray(bytes = hashes_list[0], length=hash_len_bits)
for curr_hash in hashes_list[1:]:
curr = BitArray(bytes=curr_hash, length=hash_len_bits)
N = curr.uint - prev.uint
q = int(math.floor(N / M))
r = N % M
# Unary-encode q.
if q == 0:
outarray.append(Bits(bin='0b0'))
min_is_zero = True
else:
outarray.append(Bits(bin=bin(2**q - 1) + '0'))
# Write r using plain binary representation.
outarray.append(Bits(uint=r, length=m_bits))
prev = curr
if not min_is_zero:
print "Inefficient encoding: Minimum is not zero."
return outarray.tobytes()
def optimiseScr(data):
dataBits = BitArray(bytes=data)
for block in range(0,3):
addrAttr = 0x1800 + 0x100 * block
addrPixel = 0x0800 * block
for char in range(0,256):
addrAttr2 = 8*(addrAttr + char)
addrPixel2 = 8*(addrPixel + char)
pixelData = dataBits[addrPixel2:addrPixel2+8]
pixelData.append(dataBits[addrPixel2+0x0800:addrPixel2+0x0808])
pixelData.append(dataBits[addrPixel2+0x1000:addrPixel2+0x1008])
pixelData.append(dataBits[addrPixel2+0x1800:addrPixel2+0x1808])
pixelData.append(dataBits[addrPixel2+0x2000:addrPixel2+0x2008])
pixelData.append(dataBits[addrPixel2+0x2800:addrPixel2+0x2808])
pixelData.append(dataBits[addrPixel2+0x3000:addrPixel2+0x3008])
pixelData.append(dataBits[addrPixel2+0x3800:addrPixel2+0x3808])
if pixelData.count(1) >= 32:
# More INK than PAPER in this square
# Flip pixels
pixelData = ~pixelData
dataBits[addrPixel2:addrPixel2+8]=pixelData[0:8]
dataBits[addrPixel2+0x0800:addrPixel2+0x0808]=pixelData[8:16]
dataBits[addrPixel2+0x1000:addrPixel2+0x1008]=pixelData[16:24]
dataBits[addrPixel2+0x1800:addrPixel2+0x1808]=pixelData[24:32]
dataBits[addrPixel2+0x2000:addrPixel2+0x2008]=pixelData[32:40]
dataBits[addrPixel2+0x2800:addrPixel2+0x2808]=pixelData[40:48]
dataBits[addrPixel2+0x3000:addrPixel2+0x3008]=pixelData[48:56]
dataBits[addrPixel2+0x3800:addrPixel2+0x3808]=pixelData[56:64]
# Flip attributes
attribute = dataBits[addrAttr2:addrAttr2+8]
dataBits[addrAttr2:addrAttr2+8]=[
attribute[0],attribute[1],
attribute[5],attribute[6],attribute[7],
attribute[2],attribute[3],attribute[4]
]
data = dataBits.tobytes()
return data
def encode(self):
"""encoding our file"""
self.map = {}
tree = self.tree_maker()
for i in self.data.keys():
current = tree
code = []
while not (current.getLeftChild() is None and current.getRightChild() is None):
if i in current.getLeftChild().getValue()[1]:
code.append('0')
current = current.getLeftChild()
elif i in current.getRightChild().getValue()[1]:
code.append('1')
current = current.getRightChild()
self.map[i] = ''.join(code)
coded_file = open('map.txt', 'w')
coded_file.write(str(self.map))
coded_file.close()
for i in self.file:
self.coded += self.map[i]
with open('file.bin', 'wb') as f:
b = BitArray(bin=self.coded)
f.write(b.tobytes())
def create_delta_diff(hashes_list, hash_length):
# Must be sorted for delta diff to make sense
assert sorted(hashes_list)
# Must not contain duplicates.
assert len(hashes_list) == len(set(hashes_list))
hash_len_bits = hash_length * 8
outarray = BitArray(bytes = hashes_list[0], length=hash_len_bits)
prev = BitArray(bytes = hashes_list[0], length=hash_len_bits)
for curr_hash in hashes_list[1:]:
curr = BitArray(bytes=curr_hash, length=hash_len_bits)
i = 0
while prev[i] == curr[i]:
i += 1
assert i < 64
bits_differ = hash_len_bits - i
assert bits_differ > 0 and bits_differ < 65
outarray.append(Bits(uint=(bits_differ - 1), length=6))
diferring_bits = curr[i:]
#print i, bits_differ, diferring_bits.length
assert bits_differ == diferring_bits.length
outarray.append(diferring_bits)
prev = curr
return outarray.tobytes()
def issue_dht_trace(logger):
def clean_up():
# Clean up
logger.debug('closing socket')
s.close()
logger.debug('done')
# Test DHT-server request handling from KX
address = (config.HOSTNAME, config.PORT)
logger.info('server on %s:%s', address[0], address[1])
# Connect to the server
logger.debug('creating socket')
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
logger.debug('connecting to server')
s.connect(address)
# DHT TRACE
key = BitArray(int=randrange(0, 100), length=256)
pack_str = 'HH32s'
size = calcsize(pack_str)
dht_get = pack(pack_str,
socket.htons(size),
socket.htons(config.MSG_DHT_TRACE),
key.tobytes())
logger.debug('key: %s' % str(key))
logger.debug('total DHT TRACE request size: %s' % size)
# Send the data
logger.debug('sending the DHT TRACE request')
s.send(dht_get)
# Receive a response
logger.debug('waiting for response')
res_header = s.recv(4)
logger.debug('response length: %s' % len(res_header))
if len(res_header) == 0:
clean_up()
else:
amount_received = len(res_header)
unpacked_header = unpack('HH', res_header)
res_size = socket.ntohs(unpacked_header[0])
res_type = socket.ntohs(unpacked_header[1])
logger.debug('response size: %s' % res_size)
logger.debug('response type: %s' % res_type)
if res_type == config.MSG_DHT_ERROR:
clean_up()
amount_expected = res_size
res_body = ''
while amount_received < amount_expected:
res_body += s.recv(16384)
amount_received = len(res_header) + len(res_body)
trace_recs_num = (len(res_body)-calcsize('32s'))/calcsize('32sHH4s16s')
logger.debug('trace_recs_num: %s' % trace_recs_num)
pack_str = '32s' + trace_recs_num * '32sHH4s16s'
unpacked_response = unpack(pack_str, res_body)
logger.debug('pack_str: %s' % pack_str)
logger.debug('key: %s' % '0x' + ''.join([hex(ord(i)) for i in unpacked_response[0]]).replace('x00', '00').replace('0x', ''))
logger.debug('len(unpacked_response): %s' % str(len(unpacked_response)))
for i in range(trace_recs_num):
logger.debug('peer_ID: %s' % '0x' + ''.join([hex(ord(j)) for j in unpacked_response[i*5+1]]).replace('x00', '00').replace('0x', ''))
logger.debug('KX_port: %s' % socket.ntohs(unpacked_response[i*5+2]))
logger.debug('IPv4_address: %s' % '0x' + ''.join([hex(ord(j)) for j in unpacked_response[i*5+4]]).replace('x00', '00').replace('0x', ''))
logger.debug('IPv6_address: %s' % '0x' + ''.join([hex(ord(j)) for j in unpacked_response[i*5+5]]).replace('x00', '00').replace('0x', ''))
import serial
import math
from bitstring import BitArray
import socket
import argparse
parser = argparse.ArgumentParser(description='Transmit PPM from serial.')
parser.add_argument('serial', help='the serial port to pull PPM from')
parser.add_argument('serial_rate', help='the rate in bps to connect to the serial port at', type=int)
parser.add_argument('destination_ip', help='the IP address to send PPM to')
parser.add_argument('port', help='the port to use', type=int)
args = parser.parse_args()
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
ser = serial.Serial(args.serial, args.serial_rate)
while True:
bits = BitArray()
line = ser.readline().strip()
for pulse_length in line.split():
bits.append('uint:16=' + pulse_length)
s.sendto(bits.tobytes(), (args.destination_ip, args.port))
print line
def writeFileBIN(self):
f = open(self.file_name_bin, 'wb')
s = ''.join(self.bin_list)
b = BitArray(bin=s)
f.write(b.tobytes())
f.close()
def issue_dht_put(logger):
def clean_up():
# Clean up
logger.debug('closing socket')
s.close()
logger.debug('done')
# Test DHT-server request handling from KX
address = (config.HOSTNAME, config.PORT)
logger.info('server on %s:%s', address[0], address[1])
# Connect to the server
logger.debug('creating socket')
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
logger.debug('connecting to server')
s.connect(address)
# DHT PUT
key = BitArray(int=randrange(0, 100), length=256)
ttl = 12
replication = 4
reserved = BitArray(int=0, length=32)
content = BitArray(int=randrange(0, 1000), length=256)
pack_str = 'HH32sHH4s%ss' % str(len(content)/8)
size = calcsize(pack_str)
logger.debug('key: %s' % str(key))
logger.debug('ttl: %s' % ttl)
logger.debug('replication: %s (in 8 MSBits: %s)' % (replication, replication << 8))
logger.debug('reserved: %s' % str(reserved))
logger.debug('content: %s' % str(content))
logger.debug('total DHT PUT request size: %s' % size)
dht_put = pack(pack_str,
socket.htons(size),
socket.htons(config.MSG_DHT_PUT),
key.tobytes(),
socket.htons(ttl),
socket.htons(replication << 8),
reserved.tobytes(),
content.tobytes())
# Send the data
logger.debug('sending the DHT PUT request')
s.send(dht_put)
# Receive a response
logger.debug('waiting for response')
res_header = s.recv(4)
logger.debug('response length: %s' % len(res_header))
if len(res_header) == 0:
clean_up()
else:
amount_received = len(res_header)
unpacked_header = unpack('HH', res_header)
res_size = socket.ntohs(unpacked_header[0])
res_type = socket.ntohs(unpacked_header[1])
logger.debug('response size: %s' % res_size)
logger.debug('response type: %s' % res_type)
clean_up()
class AdaptiveCoder:
def encode_file(self, input_filepath, output_filepath):
"""Kódování souboru"""
self.init_frequencies()
self.size = 0
self.lp = 0
self.up = self.M
self.c = 0
self.res = BitArray()
filesize = os.path.getsize(input_filepath)
with open(input_filepath, 'rb') as f:
# Soubor čtu po bytech, výstupní kod se průběžne ukládá do self.res
byte = f.read(1)
while byte != b'':
if self.size % 1000 == 0:
print("{0}/{1}".format(self.size, filesize))
self.encode_integer(ord(byte))
byte = f.read(1)
self.res.append(bin(1))
with open(output_filepath, 'wb') as fo:
order = floor(log(self.size, 2)) + 1
fo.write((order).to_bytes(16, byteorder="big", signed=False))
fo.write(self.size.to_bytes(order, byteorder="big", signed=False))
fo.write(self.res.tobytes())
return (self.size, self.res.bin)
def encode_integer(self, character):
""" Kodování znaku v modelu"""
self.size += 1
# přičteme jedničku kvůli posunutí indexu
new_up = self.lp + ((self.up - self.lp + 1) *
self.cumul_freq[character + 1]) // self.cumul_freq[self.symbols] - 1
new_lp = self.lp + ((self.up - self.lp + 1) *
self.cumul_freq[character]) // self.cumul_freq[self.symbols]
self.up = new_up
self.lp = new_lp
#print(self.up, self.lp)
# print(self.M)
self.increment_freq(character)
if self.up < self.lp:
print(self.size)
raise Exception("Lower bound is bigger than upper.")
# škálování
while (self.up < self.H) or (self.lp >= self.H) or ((self.lp >= self.Q1) and (self.up < self.Q3)):
d = 0
#print((self.lp, self.up))
if ((self.lp >= self.Q1) and (self.up < self.Q3)):
self.c += 1
d = self.Q1
else:
if self.up < self.H:
b = 0
d = 0
else:
b = 1
d = self.H
self.res.append(bin(b))
while self.c > 0:
self.res.append(bin(not(b)))
self.c -= 1
self.lp = 2 * (self.lp - d)
self.up = 2 * (self.up - d) + 1
def decode_file(self, input_filepath, output_filepath):
""" Dekódování souboru"""
self.init_frequencies()
self.lp = 0
self.up = self.M
with open(input_filepath, 'rb') as fo:
order = int.from_bytes(fo.read(16), byteorder='big', signed=False)
size = int.from_bytes(
fo.read(order), byteorder='big', signed=False)
encoded = fo.read()
if self.k - len(encoded) > 0:
"Doplnění nulových bytů"
encoded += bytes(self.k - len(encoded))
a = BitStream(encoded)
with open(output_filepath, 'wb') as fi:
self.decode_stream(size, a, fi)
def decode_stream(self, size, encoded, output_file):
"""Dekódování streamu"""
j = 0
x = encoded.read("uint:%s" % self.k)
decoded = bytearray(self.BUF_SIZE)
pos = 0
while True:
if (j % 1000) == 0:
print("{0}/{1}".format(j, size))
sym = None
# kumulativní četnost x
c = (self.cumul_freq[self.symbols] *
(x - self.lp + 1)) // (self.up - self.lp + 1)
# hledání intervalu, kam patří x
# for i in range(0, self.symbols):
# low = self.cumul_freq[i]
# high = self.cumul_freq[i + 1]
# if low <= c < high:
# sym = i
# break
# optimálnější hledání intervalu
u = 0
v = self.symbols - 1
while u <= v:
s = (u + v) // 2
low = self.cumul_freq[s]
high = self.cumul_freq[s + 1]
if low <= c < high:
sym = s
break
if c < low:
v = s - 1
else:
u = s + 1
# Chyba, kumulativní četnost x nezařazena
if sym is None:
print("(lp, up) : (%s, %s)" % (self.lp, self.up))
print("Current position: %s" % j)
print("x value: %s" % x)
print("X cumulative value: %s" % c)
tmp = self.cumul_freq[self.symbols] * \
(x - self.lp + 1) / (self.up - self.lp + 1)
print("X unrounded cumulative value: %s" % tmp)
print(self.cumul_freq)
raise Exception("Symbol not found")
# symbol do bufferu
decoded[pos] = sym
pos += 1
# vyprázdnění bufferu
if pos == self.BUF_SIZE:
pos = 0
output_file.write(decoded)
j += 1
if (j == size):
if pos > 0:
output_file.write(decoded[:pos])
break
new_up = self.lp + \
((self.up - self.lp + 1) *
self.cumul_freq[sym + 1]) // self.cumul_freq[self.symbols] - 1
new_lp = self.lp + \
((self.up - self.lp + 1) *
self.cumul_freq[sym]) // self.cumul_freq[self.symbols]
self.lp = new_lp
self.up = new_up
self.increment_freq(sym)
#print((self.lp, self.up))
# škálování
while (self.up < self.H) or (self.lp >= self.H) or ((self.lp >= self.Q1) and (self.up < self.Q3)):
if ((self.lp >= self.Q1) and (self.up < self.Q3)):
d = self.Q1
else:
if self.up < self.H:
d = 0
else:
d = self.H
self.lp = 2 * (self.lp - d)
self.up = 2 * (self.up - d) + 1
if encoded.pos < encoded.len:
b = encoded.read("uint:1")
else:
b = 0
x = ((2 * (x - d)) + b)
if not (0 <= x < self.M):
raise Exception("Variable x is out of range: %s" % x)
def init_frequencies(self):
# inicializujeme počtu výskytu znaků
self.freq = [0] + self.symbols * [1]
self.cumul_freq = list(range(0, 257))
def increment_cumul_from(self, i_from=1):
for i in range(i_from, 257):
self.cumul_freq[i] += 1
def refrest_cumul_freq(self):
print("adjusting frequencies")
acc = 0
tmp = 0
for i in range(1, 257):
tmp = self.freq[i]
if tmp > 1:
tmp = tmp // 2
self.freq[i] = tmp
acc += tmp
self.cumul_freq[i] = acc
def increment_freq(self, pos):
""" Inkrementace frekvence výskytu symbolu"""
# pos+1 kvůli g_{i-1}, na nultém indexu je vždy nula
self.freq[pos + 1] += 1
self.increment_cumul_from(pos + 1)
# Ošetření P_min
if self.cumul_freq[self.symbols] == self.Q1:
self.refrest_cumul_freq()
def __init__(self, k=32):
# Inicializace parametrů kodéru
self.k = k
self.M = 2**self.k - 1
self.symbols = 256
self.Q1 = 2**(self.k - 2)
self.H = 2 * self.Q1
self.Q3 = 3 * self.Q1
# Velikost bufferu pro zápis, nutno optimalizovat
self.BUF_SIZE = 2**10
class TestPeerWireTranslator(object):
def setup_class(self):
self.translator = PeerWireTranslator()
self.msg_keep_alive = struct.pack('B', 0)
self.msg_choke = struct.pack('>IB', 1, _MSG_CHOKE)
self.msg_unchoke = struct.pack('>IB', 1, _MSG_UNCHOKE)
self.msg_interested = struct.pack('>IB', 1, _MSG_INTERESTED)
self.msg_not_interested = struct.pack('>IB', 1, _MSG_NOT_INTERESTED)
self.have_index = 55
self.msg_have = struct.pack('>IBI', 5, _MSG_HAVE, self.have_index)
self.bits = BitArray('0x123456789abcdef')
bitsasbytes = self.bits.tobytes()
length = len(bitsasbytes)
self.msg_bitfield = struct.pack('>IB{}s'.format(length),
1+length, _MSG_BITFIELD, bitsasbytes)
self.request_index = 10
self.request_begin = 60
self.request_length = 255
self.msg_request = struct.pack('>IB3I', 13, _MSG_REQUEST, self.request_index,
self.request_begin, self.request_length)
self.piece_index = 17
self.piece_begin = 245
self.piece_block = """Four score and seven years ago our fathers brought forth on this
continent a new nation, conceived in liberty, and dedicated to the
proposition that all men are created equal.
Now we are engaged in a great civil war, testing whether that nation,
or any nation so conceived and so dedicated, can long endure. We are
met on a great battle-field of that war. We have come to dedicate a
portion of that field, as a final resting place for those who here gave
their lives that that nation might live. It is altogether fitting and
proper that we should do this.
But, in a larger sense, we can not dedicate, we can not consecrate, we
can not hallow this ground. The brave men, living and dead, who
struggled here, have consecrated it, far above our poor power to add or
detract. The world will little note, nor long remember what we say
here, but it can never forget what they did here. It is for us the
living, rather, to be dedicated here to the unfinished work which they
who fought here have thus far so nobly advanced. It is rather for us to
be here dedicated to the great task remaining before us—that from these
honored dead we take increased devotion to that cause for which they
gave the last full measure of devotion—that we here highly resolve that
these dead shall not have died in vain—that this nation, under God,
shall have a new birth of freedom—and that government of the people, by
the people, for the people, shall not perish from the earth."""
length = len(self.piece_block)
self.msg_piece = struct.pack('>IB2I{}s'.format(length), 9+length,
_MSG_PIECE, self.piece_index,
self.piece_begin, self.piece_block)
self.cancel_index = 99
self.cancel_begin = 128
self.cancel_length = 13
self.msg_cancel = struct.pack('>IB3I', 13, _MSG_CANCEL, self.cancel_index,
self.cancel_begin, self.cancel_length)
self.junk = "abcdefghijklmnop"
length = len(self.junk)
self.msg_invalid = struct.pack('>IB{}s'.format(length), 1+length,
_MSG_INVALID, self.junk)
def rx_bytes(self, string):
message = bytearray(string)
sent = 0
while sent != len(message):
buf, needed = self.translator.get_rx_buffer()
buf[:needed] = message[sent:sent+needed]
sent = sent+needed
self.translator.rx_bytes(needed)
def rx_bytes_in_parts(self, string):
message = bytearray(string)
sent = 0
while sent != len(message):
buf, needed = self.translator.get_rx_buffer()
if needed > 1:
buf[:needed-1] = message[sent:sent+needed-1]
sent = sent+needed-1
self.translator.rx_bytes(needed-1)
else:
buf[:needed] = message[sent:sent+needed]
sent = sent+needed
self.translator.rx_bytes(needed)
"""
def test_keep_alive(self):
self.rx_bytes(self.msg_keep_alive)
self.translator.set_receiver(self.receiver)
self.rx_bytes(self.msg_keep_alive)
assert self.receiver.rx_keep_alive.called
call_count = self.receiver.rx_keep_alive.call_count
self.translator.unset_receiver()
self.rx_bytes(self.msg_keep_alive)
assert self.receiver.rx_keep_alive.call_count == call_count
"""
def test_rx_choke(self):
self.rx_bytes(self.msg_choke)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes(self.msg_choke)
assert self.receiver.rx_choke.called
call_count = self.receiver.rx_choke.call_count
self.rx_bytes_in_parts(self.msg_choke)
assert self.receiver.rx_choke.call_count == call_count+1
call_count = self.receiver.rx_choke.call_count
self.translator.unset_receiver()
self.rx_bytes(self.msg_choke)
assert self.receiver.rx_choke.call_count == call_count
def test_rx_unchoke(self):
self.rx_bytes(self.msg_unchoke)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes(self.msg_unchoke)
assert self.receiver.rx_unchoke.called
call_count = self.receiver.rx_unchoke.call_count
self.rx_bytes_in_parts(self.msg_unchoke)
assert self.receiver.rx_unchoke.call_count == call_count+1
call_count = self.receiver.rx_unchoke.call_count
self.translator.unset_receiver()
self.rx_bytes(self.msg_unchoke)
assert self.receiver.rx_unchoke.call_count == call_count
def test_rx_interested(self):
self.rx_bytes(self.msg_interested)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes(self.msg_interested)
assert self.receiver.rx_interested.called
call_count = self.receiver.rx_interested.call_count
self.rx_bytes_in_parts(self.msg_interested)
assert self.receiver.rx_interested.call_count == call_count+1
call_count = self.receiver.rx_interested.call_count
self.translator.unset_receiver()
self.rx_bytes(self.msg_interested)
assert self.receiver.rx_interested.call_count == call_count
def test_rx_not_interested(self):
self.rx_bytes(self.msg_not_interested)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes(self.msg_not_interested)
assert self.receiver.rx_not_interested.called
call_count = self.receiver.rx_not_interested.call_count
self.rx_bytes_in_parts(self.msg_not_interested)
assert self.receiver.rx_not_interested.call_count == call_count+1
call_count = self.receiver.rx_not_interested.call_count
self.translator.unset_receiver()
self.rx_bytes(self.msg_not_interested)
assert self.receiver.rx_not_interested.call_count == call_count
def test_rx_have(self):
self.rx_bytes(self.msg_have)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes(self.msg_have)
self.receiver.rx_have.assert_called_with(self.have_index)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes_in_parts(self.msg_have)
self.receiver.rx_have.assert_called_with(self.have_index)
call_count = self.receiver.rx_have.call_count
self.translator.unset_receiver()
self.rx_bytes(self.msg_have)
assert self.receiver.rx_have.call_count == call_count
def test_rx_bitfield(self):
self.rx_bytes(self.msg_bitfield)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes(self.msg_bitfield)
bits = self.receiver.rx_bitfield.call_args_list[0][0][0]
assert bits.tobytes() == self.bits.tobytes()
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes_in_parts(self.msg_bitfield)
bits = self.receiver.rx_bitfield.call_args_list[0][0][0]
assert bits.tobytes() == self.bits.tobytes()
call_count = self.receiver.rx_bitfield.call_count
self.translator.unset_receiver()
self.rx_bytes(self.msg_bitfield)
assert self.receiver.rx_bitfield.call_count == call_count
def test_rx_request(self):
self.rx_bytes(self.msg_request)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes(self.msg_request)
self.receiver.rx_request.assert_called_with(self.request_index,
self.request_begin, self.request_length)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes_in_parts(self.msg_request)
self.receiver.rx_request.assert_called_with(self.request_index,
self.request_begin, self.request_length)
call_count = self.receiver.rx_request.call_count
self.translator.unset_receiver()
self.rx_bytes(self.msg_request)
assert self.receiver.rx_request.call_count == call_count
def test_rx_piece(self):
self.rx_bytes(self.msg_piece)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes(self.msg_piece)
args = self.receiver.rx_piece.call_args_list[0][0]
assert args[0] == self.piece_index
assert args[1] == self.piece_begin
for i in range(len(args[2])):
assert args[2][i] == self.piece_block[i]
print self.piece_block
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes_in_parts(self.msg_piece)
args = self.receiver.rx_piece.call_args_list[0][0]
assert args[0] == self.piece_index
assert args[1] == self.piece_begin
for i in range(len(args[2])):
assert args[2][i] == self.piece_block[i]
call_count = self.receiver.rx_piece.call_count
self.translator.unset_receiver()
self.rx_bytes(self.msg_piece)
assert self.receiver.rx_piece.call_count == call_count
def test_rx_cancel(self):
self.rx_bytes(self.msg_cancel)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes(self.msg_cancel)
self.receiver.rx_cancel.assert_called_with(self.cancel_index,
self.cancel_begin, self.cancel_length)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes_in_parts(self.msg_cancel)
self.receiver.rx_cancel.assert_called_with(self.cancel_index,
self.cancel_begin, self.cancel_length)
call_count = self.receiver.rx_cancel.call_count
self.translator.unset_receiver()
self.rx_bytes(self.msg_cancel)
assert self.receiver.rx_cancel.call_count == call_count
def test_rx_invalid(self):
self.rx_bytes(self.msg_invalid)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes(self.msg_invalid)
self.receiver = Mock()
self.translator.set_receiver(self.receiver)
self.rx_bytes_in_parts(self.msg_invalid)
self.translator.unset_receiver()
self.rx_bytes(self.msg_invalid)
def test_tx_keep_alive(self):
self.translator.tx_keep_alive()
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_keep_alive()
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_keep_alive
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_keep_alive()
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_choke(self):
self.translator.tx_choke()
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_choke()
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_choke
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_choke()
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_unchoke(self):
self.translator.tx_unchoke()
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_unchoke()
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_unchoke
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_unchoke()
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_interested(self):
self.translator.tx_interested()
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_interested()
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_interested
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_interested()
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_not_interested(self):
self.translator.tx_not_interested()
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_not_interested()
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_not_interested
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_not_interested()
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_have(self):
self.translator.tx_have(self.have_index)
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_have(self.have_index)
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_have
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_have(self.have_index)
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_bitfield(self):
self.translator.tx_bitfield(self.bits)
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_bitfield(self.bits)
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_bitfield
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_bitfield(self.bits)
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_request(self):
self.translator.tx_request(self.request_index, self.request_begin, self.request_length)
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_request(self.request_index, self.request_begin, self.request_length)
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_request
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_request(self.request_index, self.request_begin, self.request_length)
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_piece(self):
self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block)
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block)
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_piece
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block)
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_cancel(self):
self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length)
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length)
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_cancel
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length)
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_keep_alive(self):
self.translator.tx_keep_alive()
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_keep_alive()
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_keep_alive
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_keep_alive()
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_choke(self):
self.translator.tx_choke()
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_choke()
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_choke
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_choke()
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_unchoke(self):
self.translator.tx_unchoke()
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_unchoke()
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_unchoke
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_unchoke()
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_interested(self):
self.translator.tx_interested()
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_interested()
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_interested
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_interested()
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_not_interested(self):
self.translator.tx_not_interested()
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_not_interested()
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_not_interested
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_not_interested()
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_have(self):
self.translator.tx_have(self.have_index)
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_have(self.have_index)
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_have
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_have(self.have_index)
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_bitfield(self):
self.translator.tx_bitfield(self.bits)
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_bitfield(self.bits)
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_bitfield
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_bitfield(self.bits)
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_request(self):
self.translator.tx_request(self.request_index, self.request_begin, self.request_length)
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_request(self.request_index, self.request_begin, self.request_length)
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_request
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_request(self.request_index, self.request_begin, self.request_length)
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_piece(self):
self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block)
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block)
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_piece
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block)
assert self.readerwriter.tx_bytes.call_count == call_count
def test_tx_cancel(self):
self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length)
self.readerwriter = Mock()
self.translator.set_readerwriter(self.readerwriter)
self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length)
calls = self.readerwriter.tx_bytes.call_args_list
assert "".join([arg[0][0] for arg in calls]) == self.msg_cancel
call_count = self.readerwriter.tx_bytes.call_count
self.translator.unset_readerwriter()
self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length)
assert self.readerwriter.tx_bytes.call_count == call_count
class Option(object):
"""
Represent a CoAP option.
"""
def __init__(self):
"""
Initialize an option.
"""
self._number = None
self._value = None
@property
def number(self):
"""
Get the option number.
:return: the option number
"""
return self._number
@number.setter
def number(self, number):
"""
Set the option number.
:param number: the number
"""
self._number = number
@property
def value(self):
"""
Get the option value.
:return: the option value as bytes
"""
if type(self._value) is None:
self._value = BitArray()
name, opt_type, repeatable, defaults = defines.options[self._number]
if opt_type == defines.INTEGER:
if self._value.len > 0:
return self._value.uint
else:
return defaults
return self._value.tobytes()
@value.setter
def value(self, val):
"""
Sets the option value.
:param val: the value
"""
if type(val) is str:
val = BitArray(bytes=val, length=len(val) * 8)
if type(val) is int and bit_len(val) != 0:
val = BitArray(uint=val, length=bit_len(val) * 8)
if type(val) is int and bit_len(val) == 0:
val = BitArray()
assert(type(val) is BitArray)
self._value = val
@property
def raw_value(self):
"""
Get the option value.
:return: the option value as BitArray
"""
if type(self._value) is None:
self._value = BitArray()
return self._value
@property
def length(self):
"""
Get the len of the option value
:return: the len of the option value
"""
assert(type(self._value) is BitArray)
return len(self._value.tobytes())
@property
def safe(self):
"""
Check if option is safe.
:return: True if safe, False otherwise
"""
if self._number == defines.inv_options["Uri-Host"] or self._number == defines.inv_options["Uri-Port"] \
or self._number == defines.inv_options["Uri-Path"] or self._number == defines.inv_options["Max-Age"] \
or self._number == defines.inv_options["Uri-Query"] \
or self._number == defines.inv_options["Proxy-Uri"] \
or self._number == defines.inv_options["Proxy-Scheme"]:
return False
return True
@property
def name(self):
"""
Gets option name.
:return: the name of the oprion
"""
return defines.options[self._number][0]
def __str__(self):
"""
Return the option as a formatted string.
:return: the string representing the option
"""
name, opt_type, repeatable, defaults = defines.options[self._number]
if name == "ETag":
return name + ": " + str(self.raw_value) + "\n"
else:
return name + ": " + str(self.value) + "\n"
def __eq__(self, other):
"""
Compare options.
:param other: the option to compare
:return: True if equal
"""
return self.__dict__ == other.__dict__
def compress(source_filepath, dest):
try:
source_file = open(source_filepath, "rb").read().decode('utf-8')
except IOError as e:
logging.critical("I/O error({}): {}".format(e.errno, e.strerror))
except:
logging.critical("{} is not a valid encoded utf-8 file".
format(source_file))
# ----------------------------------------------------------------------- #
# Get information on the source file:
# - Alphabet and weight of each symbol from the alphabet
# ----------------------------------------------------------------------- #
huff_prob = huffprob.HuffProb(source_file)
stats = huff_prob.get_stats()
get_process_time("Get stats")
# ----------------------------------------------------------------------- #
# Build the Huffman tree and the prefix-free binary code for each symbol
# ----------------------------------------------------------------------- #
huff_tree = hufftree.HuffTree(stats)
get_process_time("Build HuffTree")
# ----------------------------------------------------------------------- #
# Encode the source file using the prefix-free binary code
# ----------------------------------------------------------------------- #
codewords = huff_tree.get_codewords()
buf = BitArray()
for char in source_file:
buf += codewords[char]
# Convert "01101..." BitArray to a bytearray
encoded_data = buf.tobytes()
get_process_time("Encode file")
# ----------------------------------------------------------------------- #
# Create the output file:
# - Add a header with the tree and symbol list
# - Add the encoded data
# ----------------------------------------------------------------------- #
tree_struct = huff_tree.get_bitstring_struct()
tree_symbols = huff_tree.get_symbols()
header = tree_struct.tobytes()
header += bytearray(''.join(tree_symbols).encode('utf-8'))
get_process_time("Build header")
dest_filepath = "{}.huf".format(os.path.splitext(source_filepath)[0])
dest_file = open(dest_filepath, "wb")
dest_file.write(header + encoded_data)
dest_file.close()
get_process_time("Write .huf file")
# ----------------------------------------------------------------------- #
# Performance report
# ----------------------------------------------------------------------- #
source_size = os.path.getsize(source_filepath)
dest_size = os.path.getsize(dest_filepath)
rate = 1 - (dest_size / source_size)
logging.info("{} ({}b) -> {} ({}b)".format(source_filepath,
source_size,
dest_filepath,
dest_size
))
logging.info("Compression rate: {:.2%}".format(rate))