def trace(request): from random import randrange from bitstring import BitArray trace_recs_num = randrange(2, 5) trace_recs = [] for i in range(trace_recs_num): peer_ID = BitArray(int=randrange(0, 1000), length=256) KX_port = randrange(10000, 20000) IPv4_address = BitArray(int=randrange(0, 1000), length=32) IPv6_address = BitArray(int=randrange(0, 1000), length=128) trace_rec = { 'peer_ID': peer_ID.tobytes(), 'KX_port': KX_port, 'IPv4_address': IPv4_address.tobytes(), 'IPv6_address': IPv6_address.tobytes() } trace_recs.append(trace_rec) self.logger.debug('process_dht_request->trace.str(peer_ID): %s' % str(peer_ID)) self.logger.debug('process_dht_request->trace.str(KX_port): %s' % str(KX_port)) self.logger.debug('process_dht_request->trace.str(IPv4_address): %s' % str(IPv4_address)) self.logger.debug('process_dht_request->trace.str(IPv6_address): %s' % str(IPv6_address)) response = { 'trace': trace_recs, 'all_ok': True } self.logger.debug('process_dht_request->find_value.len(response["trace"]): %s' % len(response['trace'])) return response
def encode_block(in_bytes): if len(in_bytes) == 1: block = BitArray() block.append(Bits('0b1')) block.append(in_bytes) return block.tobytes() bw_xf, eof_idx = burrows_wheeler_transform(in_bytes) front_xf = move_to_front_transform(bw_xf) rle_data = run_length_encode(front_xf) huff_data, huff_symbols, serialized_tree = huffman_encode(rle_data, symbol_bits=8) huff_len = len(huff_data) tree_len = len(serialized_tree) block = BitArray() block.append(Bits('0b0')) block.append(Bits(uint=tree_len, length=16)) block.append(Bits(serialized_tree)) block.append(Bits(uint=huff_symbols, length=16)) block.append(Bits(uint=huff_len, length=16)) block.append(Bits(huff_data)) block.append(Bits(uint=eof_idx, length=BLOCK_SIZE_BITS)) return block.tobytes()
def generateConfirmationKey(identitySecret,time,tag=""): identitysecret = b64decode(identitySecret) secret = BitArray(bytes=identitysecret,length=len(identitysecret)*8) if tag != "": tagBuff = BitArray(bytes=tag,length=len(tag)*8) buff = BitArray(4*8) time = int(time) buff.append(BitArray(int=time,length=32)) if tag != "": buff.append(tagBuff) conf_hmac = hmac.new(secret.tobytes(),buff.tobytes(),hashlib.sha1) return b64encode(conf_hmac.digest())
def exec(self): message = f'Hello World at {date.today()}' message_bytes = bytes(message, encoding='utf-8') bit_array = BitArray(bytes=message_bytes) pr('original', message) pr('\t==> hex ', bit_array.hex) pr('\t==> bin ', bit_array.bin) pr('\t==> bytes ', bit_array.tobytes()) pr('\t==> str ', str(bit_array.tobytes(), encoding='utf-8')) pr('\t==> length ', bit_array.length // 8) pr('\t==> 1st byte ', bit_array[:8].tobytes()) pr('\t==> 2nd byte ', bit_array[8:8 * 2].tobytes()) pr('\t==> last byte', bit_array[-8:].tobytes())
def to_send(self ,s): #make str bits to bytes data_in_bit = BitArray(bin=s) print s after_dewhite_bin = data_in_bit.tobytes() after_dewhite_str = array.array( "B",(after_dewhite_bin)) self.message_port_pub(pmt.intern('out'), pmt.cons(pmt.PMT_NIL, pmt.init_u8vector(len(after_dewhite_bin), bytearray(after_dewhite_bin))))
def sendFirst(id, socket, error): #Input num1 = int(input("Number: ")) while num1 < -2147483648 or num1 > 2147483647: print("Wrong number! Input integer in range ", "from -2,147,483,648 to 2,147,483,647 ") num1 = int(input("First number: ")) #Pack packg = BitArray() packg = pack("+", num1, 0, id, 0, 1, 0) #Send package print( f"Sending first: {packg.bin[0:3]} {packg.bin[3:35]} {packg.bin[35:37]}", f" {packg.bin[37:45]} {packg.bin[45:46]} {packg.bin[46:47]} {packg.bin[47:]}" ) socket.send(packg.tobytes()) #Recieve package data = BitArray() data.append(socket.recv(1024)) print( f"Recieved first: {data.bin[0:3]} {data.bin[3:35]} {data.bin[35:37]}", f" {data.bin[37:45]} {data.bin[45:46]} {data.bin[46:47]} {data.bin[47:]}" ) print("Sending next")
def start_fuzz(): # rtp = b'\x80\x63\x78\xa6\x00\x00\x17\x65\xe6\x89\xe9\x31' + (b'\x61' * 1000) rtp_packet = BitArray() # rtp_packet += 'uint:2=%d' % rtp_packet_types['version'] # rtp_packet += 'uint:1=%d' % rtp_packet_types['padding'] # rtp_packet += 'uint:1=%d' % rtp_packet_types['extension'] # rtp_packet += 'uint:4=%d' % rtp_packet_types['crsc_count'] # rtp_packet += 'uint:1=%d' % rtp_packet_types['marker'] # raw_input(rtp_packet) # rtp_packet += 'A' # raw_input(rtp_packet) # rtp_packet += 'uint:7=%d' % rtp_packet_types['payload_type'] test_case_count = 0 fuzz_payloadtype(128) print "[*] Test Function : fuzz_payloadtype End" # raw_input(TEST_CASE_ID) fuzz_timestamp(40000) print "[*] Test Function : fuzz_timestamp End" fuzz_sequence_number(20000) print "[*] Test Function : fuzz_sequence_number End" fuzz_ssrc_id(40000) print "[*] Test Function : fuzz_ssrc_id End" # rtp_packet += 'uint:16=%d' % rtp_packet_types['sequence_number'] # rtp_packet += 'uint:32=%d' % rtp_packet_types['timev'] # rtp_packet += 'uint:32=%d' % rtp_packet_types['ssrc_id'] # rtp_packet.append('0x41'*80) # print(rtp_packet) # print(rtp_packet[4:8]) # raw_input(len(rtp_packet[4:8])) # print(rtp_packet) # raw_input(len(rtp_packet)) # for count in range(STOPAFTER): pkt = rtp_packet.tobytes()
def get_bytes(self): bitstream = BitArray() current_address = self.imagebase labels_to_addresses_map = {} for ast_node in self.ast: ast_node.set_node_bitfields(self.compute_node_bitfields(ast_node)) ast_node.set_node_address(current_address) for lbl in ast_node.labels: labels_to_addresses_map[lbl] = ast_node.address # TODO: What happens with addressing in non-standard word sizes? bit_length = self.bitfields_to_bitarray( ast_node.node_bitfields).length byte_length = int(bit_length / DEFAULT_BYTE_BITSIZE) if bit_length % DEFAULT_BYTE_BITSIZE != 0: byte_length += 1 current_address += byte_length for ast_node in self.ast: self.update_label_placeholders(ast_node, labels_to_addresses_map) for ast_node in self.ast: ast_node.set_node_bitfields(self.compute_node_bitfields(ast_node)) node_bitarray = self.bitfields_to_bitarray(ast_node.node_bitfields) bitstream.append(node_bitarray) return bitstream.tobytes()
def initFile(self): file_name = self.file_to_stream['name'] fout = open(file_name, "wb") data = BitArray(int(self.file_to_stream_length)*8) data = data.tobytes() fout.write(data) fout.close()
def lzwv_encode(in_bytes): max_entries = 2**MAX_CODE_LEN in_array = bytearray(in_bytes) dictionary = {bytes([i]): i for i in range(256)} code_len = 9 out_array = BitArray() while len(in_array) > 1: # Find string s that's not in the dictionary s = b'' while (s in dictionary or s == b'') and len(in_array) > 0: s = s + bytes([in_array.pop(0)]) in_array.insert(0, s[-1]) # Emit code for the s[:-1], which is in the dictionary out_array.append(Bits(uint=dictionary[s[:-1]], length=code_len)) # If there's room in the dictionary, add s c = len(dictionary) if c < max_entries: dictionary[s] = c # If just added max value for code len, increment it. if c == 2**code_len - 1: code_len = min(MAX_CODE_LEN, code_len + 1) # Emit code for final value out_array.append( Bits(uint=dictionary[bytes([in_array.pop(0)])], length=code_len)) return out_array.tobytes()
def test_for_strict_avalanche(): """ Function to apply the SAC test defined in https://eprint.iacr.org/2010/564.pdf. """ with open('key.txt', 'rb') as file: key = file.read() associated_data = generate_nonce(16).encode(encoding='utf-8') sample_size = 32768 plaintext_bit_size = 64 plaintext_byte_length = 8 algorithms = [ "ascon128.so", "ascon128a.so", "isap_a_128.so", "isap_k_128.so", "oribatida_128.so", "locus_64.so", "lotus_64.so", "spoc_64.so" ] print(algorithms) for alg in algorithms: print(alg) if "128" in alg: tagsize = 16 else: tagsize = 8 matrix = np.zeros( (plaintext_bit_size, plaintext_bit_size + (tagsize * 8)), dtype=int) for _ in range(0, sample_size): plaintext = random.getrandbits(plaintext_bit_size) nonce = generate_nonce(16).encode(encoding='utf-8') variant = "encryption_interfaces/" + alg original_cipher = encrypt( key, nonce, associated_data, plaintext.to_bytes(plaintext_byte_length, sys.byteorder), variant, tagsize) for bit_position in range(0, plaintext_bit_size): temp_plaintext = BitArray( plaintext.to_bytes(plaintext_byte_length, sys.byteorder)) temp_plaintext.invert(bit_position) nonce = generate_nonce(16).encode(encoding='utf-8') cipher = encrypt(key, nonce, associated_data, temp_plaintext.tobytes(), variant, tagsize) cipher_array = BitArray(cipher) ^ BitArray(original_cipher) for cipher_position in range( 0, plaintext_bit_size + (tagsize * 8)): matrix[bit_position][cipher_position] += int( cipher_array.bin[cipher_position]) flip_probabilty = 0.5 expected_count = sample_size * flip_probabilty with open('sac_results.txt', 'a') as f: f.write(variant + "\n") for row in range(0, (plaintext_bit_size)): for column in range(0, plaintext_bit_size + (tagsize * 8)): statistics, pvalue = chisquare([ int(matrix[row][column]), int(sample_size - matrix[row][column]) ], [expected_count, expected_count]) f.write(str(pvalue) + "\n") print("Values Stat & P-Value") print(statistics) print(str(pvalue))
def translate(): data = input() bin_data = BitArray(hex=data) print(bin_data.bin) packet = Turbo(parse=bin_data.tobytes()) print(packet.print())
def amrPayload2Storage_EfficientMode(payload): if (amr_type == 'nb'): # AMR-NR bitlen = [95, 103, 118, 134, 148, 159, 204, 244, 39] # print "selected codec is amr nb" elif (amr_type == 'wb'): # AMR-WB TS 26.201 - total bits bitlen = [132, 177, 253, 285, 317, 365, 397, 461, 477, 40] # print "selected codec is amr wb" else: exit(1) amr = BitArray(bytes=payload) cmr = amr[0:4] mode = amr[5:9] #print(mode.uint) #assert mode.uint >=0 and mode.uint <=8 if not (mode.uint >= 0 and mode.uint <= 8): return else: qual = amr[9:10] voice = amr[10:10 + bitlen[mode.uint]] #print("cmr=%d\tmod=%d\tqual=%d\tvoicebits=%d" % (cmr.uint,mode.uint,qual.uint,voice.len)) storage = BitArray(bin='0') storage.append(mode) storage.append(qual) storage.append('0b00') # padding assert storage.len == 8, "check length of storage header is one byte" storage.append(voice) return storage.tobytes()
def gen_frame_header(self, craft_id, virtual_channel_id, virtual_channel_count, is_encrypt): # 输入类型检查 # 范围检查, 首先定义了bit流的长度, 用overwrite函数时可保证长度(范围)安全 h_craft_id = BitArray(8) h_virtual_channel_id = BitArray(6) h_virtual_channel_count = BitArray(24) h_is_encrypt = BitArray(1) h_craft_id.overwrite(craft_id, 0) h_virtual_channel_id.overwrite( BitArray(uint=virtual_channel_id, length=6), 0) h_virtual_channel_count.overwrite( BitArray(uint=virtual_channel_count, length=24), 0) h_is_encrypt.overwrite(is_encrypt, 0) header = BitArray() header.append('0b00') header.append(h_craft_id) header.append(h_virtual_channel_id) header.append(h_virtual_channel_count) header.append('0b00') header.append(h_is_encrypt) header.append('0b00000') if is_encrypt == '0b0': self.__is_encrypt = False else: self.__is_encrypt = True self.__frame_header = header.tobytes()
def fuzz_SR_packet_basis(count): global TEST_CASE_ID ssrc_id_max = 2**32 - 1 for i in range(count): sr_packet = BitArray() rtcp_SR_types['length'] = 6 sr_packet += 'uint:2=%d' % rtcp_SR_types['version'] sr_packet += 'uint:1=%d' % rtcp_SR_types['padding'] sr_packet += 'uint:5=%d' % rtcp_SR_types['rc'] sr_packet += 'uint:8=%d' % rtcp_SR_types['packet_type'] sr_packet += 'uint:16=%d' % rtcp_SR_types['length'] print("[*] Test case ID:%d" % (TEST_CASE_ID) + " START") TEST_CASE_ID += 1 rand_ssrc_id = random.randint(0, ssrc_id_max) rand_ntp_timestamp_MSL = random.randint(0, ssrc_id_max) rand_ntp_timestamp_LSW = random.randint(0, ssrc_id_max) rand_rtp_timestamp = random.randint(0, ssrc_id_max) rand_ntp_timestamp_LSW = random.randint(0, ssrc_id_max) rand_sender_packet_count = random.randint(0, ssrc_id_max) rand_sender_octet_count = random.randint(0, ssrc_id_max) print( "[*] Test type : Basis SR packet mix fuzzing Test , Payload : %d " % (rand_ssrc_id)) sr_packet += 'uint:32=%d' % rand_ssrc_id sr_packet += 'uint:32=%d' % rand_ntp_timestamp_MSL sr_packet += 'uint:32=%d' % rand_ntp_timestamp_LSW sr_packet += 'uint:32=%d' % rand_rtp_timestamp sr_packet += 'uint:32=%d' % rand_sender_packet_count sr_packet += 'uint:32=%d' % rand_sender_octet_count send_to_target(sr_packet.tobytes())
def toCharset(charset, string, toUpper=False): """Convert unicode text to various charsets using a lookup table Args: charset (string): lookup table string (string): unicode string to convert toUpper (bool, optional): Make chars uppercase before converting (intended for 6 bit charsets or charsets that do not support uppercase). Defaults to False. Returns: bytes: sequence of bytes (split into bits of length """ asciibits = BitArray() cut = int(8 - log2(len(charset))) if toUpper: chars = chars.upper() for char in string: bits = charset.find(char) if bits < 0: if charset == GOST: bits = 25 # use '*' for GOST else: bits = charset.find("?") asciibits.append(BitArray(bytes([bits]))[cut:]) return asciibits.tobytes()
def get_bits_from_file(filepath, key): print('Reading file...') bitarray = BitArray(filename=filepath) # adding a delimiter to know when the file ends to avoid corrupted files # when retrieving cipher = AES.new(key, AES.MODE_EAX, nonce=SALT) ciphertext, tag = cipher.encrypt_and_digest(bitarray.tobytes()) filename = os.path.basename(filepath) pickled = pickle.dumps({'tag':tag, 'data':ciphertext, 'filename':filepath}) print('Zipping...') #zip out = io.BytesIO() with gzip.GzipFile(fileobj=out, mode='w') as fo: fo.write(pickled) zip = out.getvalue() #zip del bitarray del pickled bitarray = BitArray(zip) return bitarray.bin
def build_cmd(self, command, arg, uid, data_type): cmd = command.encode(encoding="utf-8") if len(command) < 10: cmd += bytes(10 - len(command)) # add data padding if arg is None: cmd += int(0).to_bytes(8, byteorder='big') elif data_type is self.CmdDataType.NUMBER or data_type is self.CmdDataType.RECORD: cmd += int(arg).to_bytes(8, byteorder='big') elif data_type is self.CmdDataType.SIGNED: cmd += int(arg).to_bytes(8, byteorder='big', signed=True) elif data_type is self.CmdDataType.STRING: cmd += self.str2bit(arg) elif data_type is self.CmdDataType.STORED: ba = BitArray() ba += [0] * 38 ba.append( BitArray(int(arg['resolution']).to_bytes(1, byteorder='big'))[2:]) ba.append( BitArray(int(arg['offset']).to_bytes(3, byteorder='big'))[4:]) cmd += ba.tobytes() else: self.logger.debug("Unhandled data type: %s" % (str(data_type))) raise self.UnknownDataType("Unhandled data type: %s" % str(data_type)) # add index cmd += uid.to_bytes(2, byteorder='big') return cmd
def golomb_encode(hashes_list, hash_length, M): hash_len_bits = hash_length * 8 # Must be sorted for delta diff to make sense assert sorted(hashes_list) # Must not contain duplicates. assert len(hashes_list) == len(set(hashes_list)) m_bits = int(math.log(M, 2)) assert abs(math.log(M, 2) - float(m_bits)) < 0.00001 outarray = BitArray(bytes = hashes_list[0], length=hash_len_bits) min_is_zero = False prev = BitArray(bytes = hashes_list[0], length=hash_len_bits) for curr_hash in hashes_list[1:]: curr = BitArray(bytes=curr_hash, length=hash_len_bits) N = curr.uint - prev.uint q = int(math.floor(N / M)) r = N % M if q == 0: outarray.append(Bits(bin='0b0')) min_is_zero = True else: outarray.append(Bits(bin=bin(2**q - 1) + '0')) outarray.append(Bits(uint=r, length=m_bits)) prev = curr if not min_is_zero: print "Inefficient encoding: Minimum is not zero." return outarray.tobytes()
def hf_compress(byte_array): MAX_SIZE = 256 freq_table = [0 for i in range(MAX_SIZE)] for b in byte_array: freq_table[b] += 1 hf_tree = create_huffman_tree(freq_table) hf_codes = parse_tree(hf_tree) result = BitArray(bin_pad(len(hf_codes), 8)) for n in hf_codes: byte_key = bytes([n[0]]) result.append(byte_key) code_length = bin_pad(len(n[1]), 4) result.append(code_length) result.append('0b' + n[1]) for b in byte_array: code = [c[1] for c in hf_codes if c[0] == b].pop() result.append('0b' + code) # find required padding to make bits go into bytes padding_size = 8 - len(result.bin) % 8 result.prepend(bin_pad(padding_size, 8)) return result.tobytes()
def scramble(data, offset, reverse=False) -> bytes: """ Scrambles/descrambles bytes with specified offset. """ a = BitArray(data, endian="little") b = BitArray(length=len(a)) if len(offset) != 2: raise ValueError("The offset must be a tuple with two values!") p1, p2 = offset if p1 >= p2: raise ValueError("The first index must be less than the second index!") for i in range(len(a)): if (i < p1): b[i] = a[i] elif (i < p2): x = (a[i - p1] if reverse else b[i - p1]) b[i] = a[i] ^ x else: x = (a[i - p1] if reverse else b[i - p1]) y = (a[i - p2] if reverse else b[i - p2]) b[i] = a[i] ^ x ^ y return b.tobytes()
def compress_file(self, input_file_name, output_file_name): tree: HuffmanTree = HuffmanTree(input_file_name) root: Node = tree.construct_tree() if root is not None: padding: int = 0 with open(input_file_name) as file_in: with open(output_file_name, 'wb') as file_out: # Write Place Holder for Padding file_out.write(BitArray('0b0000').tobytes()) current: str = file_in.readline() bits: BitArray = BitArray() bits += tree.to_bits() while current != '': for char in current: bits += tree.get_character(char) if(len(bits) % 8 == 0): file_out.write(bits.tobytes()) bits = BitArray() current = file_in.readline() file_out.write(bits.tobytes()) padding = 8 - (len(bits) % 8) # Write the Padding Number to the start of the file. with open(output_file_name, 'rb+') as fp: fp.seek(0, 0) fp.write(padding.to_bytes(1, byteorder="little")) else: with open(output_file_name, 'w'): pass
def fuzz_ssrc_id(count): global TEST_CASE_ID ssrc_id_max = 2**32 - 1 for i in range(count): rtp_packet = BitArray() rtp_packet += 'uint:2=%d' % rtp_packet_types['version'] rtp_packet += 'uint:1=%d' % rtp_packet_types['padding'] rtp_packet += 'uint:1=%d' % rtp_packet_types['extension'] rtp_packet += 'uint:4=%d' % rtp_packet_types['crsc_count'] rtp_packet += 'uint:1=%d' % rtp_packet_types['marker'] rtp_packet += 'uint:7=%d' % rtp_packet_types['payload_type'] rtp_packet += 'uint:16=%d' % rtp_packet_types['sequence_number'] rtp_packet += 'uint:32=%d' % rtp_packet_types['timev'] print("[*] Test case ID:%d" % (TEST_CASE_ID) + " START") TEST_CASE_ID += 1 rand_number = random.randint(0, ssrc_id_max) print("[*] Test type : Basis SSRC identifier Test , Payload : %d " % (rand_number)) rtp_packet += 'uint:32=%d' % rand_number # rtp_packet += 'uint:32=%d' % rtp_packet_types['ssrc_id'] send_to_target(rtp_packet.tobytes())
def compress(self): compressed_filepath = self.destination_path + "/" + self.filename + "-" + self.file_ext[ 1:] + ".bin" # Coding for .txt and .docx files. if self.file_ext == ".txt" or ".docx": self.char_to_code = self.encode() self.code_to_char = { value: key for key, value in self.char_to_code.items() } encoded_content = "".join( [self.char_to_code[a] for a in self.content]) self.bit_length = len(encoded_content) bit_array = BitArray(bin=encoded_content) with open(compressed_filepath, 'wb') as compressed_file: compressed_file.write(bit_array.tobytes()) compressed_file.close() return compressed_filepath else: raise ValueError("HuffmanCoder Compression Error: Wrong File Type") return
def seed_to_signing_key(seed, index=0): """ from raiblocks seed to private key (ed25519 seed) :param str seed: the hex string of raiblocks seed, 64 characters long :param int index: the index of wallet address :rtype: bytes :return: signing key Code Example:: seed = 'CCC020CAF01C98B6B076A9F00573503E0D7FBA85BC7CA21AF3B3C02A2DDF5326' sk = seed_to_signing_key(seed, 1) print(sk.hex()) """ h = blake2b(digest_size=32) seed_bytes = bytes.fromhex(seed) h.update(seed_bytes) index_bits = BitArray(uint=index, length=32) index_bytes = index_bits.tobytes() h.update(index_bytes) return h.digest()
def send_rtp_packet(header, payload, ip, port, packets_in_payload=2, number=0): with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as my_socket: my_socket.connect((ip, port)) if (number == 0): number = 100000000 try: for i in range(number): packet = BitArray() packet.append(header.version) packet.append(header.pad_flag) packet.append(header.ext_flag) packet.append(header.cc) packet.append(header.marker) packet.append(header.payload_type) packet.append(BitArray(uint=header.seq_number, length=16)) packet.append(BitArray(uint=header.timestamp, length=32)) packet.append(header.ssrc) packet.append(header.csrc) if header.ext_flag.bin == '1': print('aqui va la extension') print('Tamaño de la cabecera RTP: ' + str(len(packet.bin))) # Cuantos paquetes mp3 metemos en el mismo paquete RTP for j in range(packets_in_payload): payload._take_mp3_frame() packet.append(BitArray(bin=payload.frame)) print(payload.frame[0:32]) packetBytes = packet.tobytes() my_socket.send(packetBytes) header._next(payload.frameTimeMs) except IndexError: pass
def coding_with_hamming_code(data): msg = BitArray() packet_list = HammingCode.split_message( BitArray(data), HammingCode.SIZE_OF_INFORMATION_PART) for i in packet_list: msg += HammingCode.insert_control_bits(i) return msg.tobytes()
def LZCompress(filepath, wBits=16, lBits=8): filepath = "Files/" + filepath #window size is 2^number of bits allocated to it - 1 windowSize = 2**wBits - 1 lookaheadSize = 2**lBits - 1 try: file = open(filepath, "r") except: print("Failed to open file, please check filepath is correct.") return False data = file.read() file.close() #tracking the file size to compare on compression ratio originalFileSize = os.path.getsize(filepath) outputBitString = BitArray() i = 0 #loop through the entire data string while (i < len(data)): match = findLargestMatch(data, i, windowSize, lookaheadSize) #create bit strings representing integers in the tuple distance = BitArray(uint=match[0], length=wBits) bitsToCopy = BitArray(uint=match[1], length=lBits) outputBitString.append(distance) outputBitString.append(bitsToCopy) if match[2] != '-1': #create bit string to represent character, using 8 bits for each character nextChar = BitArray(uint=ord(match[2]), length=8) outputBitString.append(nextChar) i += match[1] + 1 #setting up the file paths compressedFilePath = filepath.split('.')[0] + "Compressed" decompressedFilePath = filepath.split( '.')[0] + "Decompressed" + '.' + filepath.split('.')[1] #writing the compressed file compressedFile = open(compressedFilePath, 'wb') compressedFile.write(outputBitString.tobytes()) compressedFile.close() #calculate and output the compression ratio compressedFileSize = os.path.getsize(compressedFilePath) print("Compression ratio: " + str(round(compressedFileSize / originalFileSize, 3))) #read compressed file, decode and output decoding results to a new file decompressedFile = open(decompressedFilePath, 'w') decompressedFile.write(LZDecompress(compressedFilePath, wBits, lBits)) decompressedFile.close()
def lz77_encode(input_data, window_bits=DEFAULT_WINDOW_BITS): tokens = lz77_encode_to_tokens(input_data, window_bits) out = BitArray() for t in tokens: out.append(Bits(uint=t[0], length=window_bits)) out.append(Bits(uint=t[1], length=REFERENCE_SIZE_BITS)) out.append(Bits(uint=t[2], length=8)) return out.tobytes()
def test_encode_bitfield(self): bitfield_length = 50 bitfield = BitArray([i % 2 for i in range(bitfield_length)]) bitfield_as_bytes = bitfield.tobytes() truth = struct.pack('>Ib', 1 + len(bitfield_as_bytes), peer.BitField.ID) + bitfield_as_bytes message = peer.BitField(bitfield) self.assertEqual(message.encode(), truth)
def send_bitfield(self): """ Tell the peer all the pieces you have in a really compact form. """ # the pieces are compacted in sequence in to bits field = BitArray( map(lambda sha, piece: sha == piece.digest, self.torrent.pieces.items())) self.send_payload(5, field.tobytes())
def int_to_bytes(i, length=64): """ Convert int to bytes, length is bit length. """ if not isinstance(i, int): raise ValueError('int_to_bytes: data is not int') i_bits = BitArray(uint=i, length=length) return i_bits.tobytes()
def colors_to_text(colors, n_tons, coding): n_colors = n_tons**3 bits_array = base_change(colors, n_colors, 2) bit_string = ''.join(map(str, bits_array)) bits = BitArray('0b' + bit_string) encoded = bits.tobytes() text = decode(encoded, coding) return text
def lz77huff_encode(input_data, window_bits=DEFAULT_WINDOW_BITS): tokens = lz77_encode_to_tokens(input_data, window_bits) symbols = [s for tok in tokens for s in tok] bits = BitArray() for s in symbols: bits.append(Bits(uint=s, length=window_bits)) encoded_data, num_symbols, serialized_tree = huffman_encode(bits.tobytes(), symbol_bits=window_bits) return encoded_data, num_symbols, serialized_tree
def generateAuthCode(secret,offset=0): secret = b64decode(secret) secret = BitArray(bytes=secret,length=len(secret)*8) buff = BitArray(8*8) timestamp = timeOffset(offset) buff[4*8:] = int(timestamp/30) auth_hmac = hmac.new(secret.tobytes(),buff.tobytes(),hashlib.sha1) hashed = auth_hmac.digest() hashed = BitArray(bytes=hashed,length=len(hashed)*8) start = hashed[(19*8):(19*8)+8] & BitArray('0x0f') hash_slice = hashed[start.int*8:(start.int*8)+(4*8)] fullcode = hash_slice & BitArray("0x7fffffff") fullcode = fullcode.int chars = '23456789BCDFGHJKMNPQRTVWXY' code = "" for x in range(5): code += chars[int(fullcode % len(chars))] fullcode = fullcode/len(chars) return code
def readAmbeFrameFromUDP( self, _sock ): _ambeAll = BitArray() # Start with an empty array for i in range(0, 3): _ambe = self.readSock(_sock,7) # Read AMBE from the socket if _ambe: _ambe1 = BitArray('0x'+h(_ambe[0:49])) _ambeAll += _ambe1[0:50] # Append the 49 bits to the string else: break return _ambeAll.tobytes() # Return the 49 * 3 as an array of bytes
def send_bitfield(self): """ Tell the peer all the pieces you have in a really compact form. """ # the pieces are compacted in sequence in to bits field = BitArray( map( lambda (sha, piece): sha == piece.digest, self.torrent.pieces.items() ) ) self.send_payload(5, field.tobytes())
def initFile(self): file_name = self.file_to_stream['name'] # Check if File Already Exists, If yes, then Read it. if os.path.isfile(file_name) == True: # File Exists print "File Already Exists. Checking For Partial Streaming" self.reCheck(file_name) else : fout = open(file_name, "wb") data = BitArray(int(self.file_to_stream_length)*8) data = data.tobytes() fout.write(data) fout.close()
def find_value(request): # TODO: Do the same as in store request above, and delete the lines below # TODO: in the algorithm of this function, the peer must decide if it is already storing the key/value pair in question # by issuing a self.data_server.get(key), and checking the result from random import randrange from bitstring import BitArray content = BitArray(int=randrange(0, 1000), length=256) self.logger.debug('process_dht_request->find_value.str(content): %s' % str(content)) self.logger.debug('process_dht_request->find_value.len(str(content)): %s' % len(str(content))) response = { 'content': content.tobytes(), 'all_ok': True } self.logger.debug('process_dht_request->find_value.len(response["content"]): %s' % len(response['content'])) return response
def send(self): toSend = BitArray() for i in range(0,3): frequency = BitArray(uint=self.frequencies[i], length=3) intensity = BitArray(uint=self.intensities[i], length=3) state = BitArray(uint=1, length=3) toSend.append(intensity) toSend.append(frequency) toSend.append(state) toSend.append('0b00000') toSend.append('0x00') if self.enable_bluetooth: rospy.loginfo("sending msg(hex): %s" %(toSend)) rospy.loginfo("sending msg(bin): %s"%(toSend.bin)) self.bt.send(self.toConnect, str(toSend.tobytes())) else: rospy.logdebug("sending msg: %s" %(toSend)) rospy.logdebug(str(toSend.bin))
def double_DES(key, plain): keybits = BitArray(hex = key) plainbits= BitArray(hex = plain) if len(keybits) != 112: print "length of double DES key is not 112 bits!" return if len(plainbits) != 64: print "length of double DES plain text is not 64 bits!" return rev_keybytes = '' for k in keybits.cut(7): rev_keybytes += (odd_parity[k.bin]) obj1 = DES.new(rev_keybytes[0:8], DES.MODE_ECB) midbytes = obj1.encrypt(plainbits.tobytes()) obj2 = DES.new(rev_keybytes[8:16], DES.MODE_ECB) cipherbytes = obj2.encrypt(midbytes) return BitArray(bytes = cipherbytes).hex
def golomb_encode(hashes_list, hash_length, M): """Given a sorted list of fixed-size values, compress it by using Golomb coding to represent the difference between the values.""" hash_len_bits = hash_length * 8 # Must be sorted for deltas to be small and easily compressable. assert sorted(hashes_list) == hashes_list # Must not contain duplicates. assert len(hashes_list) == len(set(hashes_list)) # M is the tunable parameter. m_bits = int(math.log(M, 2)) # Make sure that M is a power of 2. assert M > 0 and not (M & (M - 1)) # First item in the output bit array is the first hash value. outarray = BitArray(bytes = hashes_list[0], length=hash_len_bits) # Set to true when the diff value / M == 0. # If no such value exists then the chosen M is too small, so warn. min_is_zero = False prev = BitArray(bytes = hashes_list[0], length=hash_len_bits) for curr_hash in hashes_list[1:]: curr = BitArray(bytes=curr_hash, length=hash_len_bits) N = curr.uint - prev.uint q = int(math.floor(N / M)) r = N % M # Unary-encode q. if q == 0: outarray.append(Bits(bin='0b0')) min_is_zero = True else: outarray.append(Bits(bin=bin(2**q - 1) + '0')) # Write r using plain binary representation. outarray.append(Bits(uint=r, length=m_bits)) prev = curr if not min_is_zero: print "Inefficient encoding: Minimum is not zero." return outarray.tobytes()
def optimiseScr(data): dataBits = BitArray(bytes=data) for block in range(0,3): addrAttr = 0x1800 + 0x100 * block addrPixel = 0x0800 * block for char in range(0,256): addrAttr2 = 8*(addrAttr + char) addrPixel2 = 8*(addrPixel + char) pixelData = dataBits[addrPixel2:addrPixel2+8] pixelData.append(dataBits[addrPixel2+0x0800:addrPixel2+0x0808]) pixelData.append(dataBits[addrPixel2+0x1000:addrPixel2+0x1008]) pixelData.append(dataBits[addrPixel2+0x1800:addrPixel2+0x1808]) pixelData.append(dataBits[addrPixel2+0x2000:addrPixel2+0x2008]) pixelData.append(dataBits[addrPixel2+0x2800:addrPixel2+0x2808]) pixelData.append(dataBits[addrPixel2+0x3000:addrPixel2+0x3008]) pixelData.append(dataBits[addrPixel2+0x3800:addrPixel2+0x3808]) if pixelData.count(1) >= 32: # More INK than PAPER in this square # Flip pixels pixelData = ~pixelData dataBits[addrPixel2:addrPixel2+8]=pixelData[0:8] dataBits[addrPixel2+0x0800:addrPixel2+0x0808]=pixelData[8:16] dataBits[addrPixel2+0x1000:addrPixel2+0x1008]=pixelData[16:24] dataBits[addrPixel2+0x1800:addrPixel2+0x1808]=pixelData[24:32] dataBits[addrPixel2+0x2000:addrPixel2+0x2008]=pixelData[32:40] dataBits[addrPixel2+0x2800:addrPixel2+0x2808]=pixelData[40:48] dataBits[addrPixel2+0x3000:addrPixel2+0x3008]=pixelData[48:56] dataBits[addrPixel2+0x3800:addrPixel2+0x3808]=pixelData[56:64] # Flip attributes attribute = dataBits[addrAttr2:addrAttr2+8] dataBits[addrAttr2:addrAttr2+8]=[ attribute[0],attribute[1], attribute[5],attribute[6],attribute[7], attribute[2],attribute[3],attribute[4] ] data = dataBits.tobytes() return data
def encode(self): """encoding our file""" self.map = {} tree = self.tree_maker() for i in self.data.keys(): current = tree code = [] while not (current.getLeftChild() is None and current.getRightChild() is None): if i in current.getLeftChild().getValue()[1]: code.append('0') current = current.getLeftChild() elif i in current.getRightChild().getValue()[1]: code.append('1') current = current.getRightChild() self.map[i] = ''.join(code) coded_file = open('map.txt', 'w') coded_file.write(str(self.map)) coded_file.close() for i in self.file: self.coded += self.map[i] with open('file.bin', 'wb') as f: b = BitArray(bin=self.coded) f.write(b.tobytes())
def create_delta_diff(hashes_list, hash_length): # Must be sorted for delta diff to make sense assert sorted(hashes_list) # Must not contain duplicates. assert len(hashes_list) == len(set(hashes_list)) hash_len_bits = hash_length * 8 outarray = BitArray(bytes = hashes_list[0], length=hash_len_bits) prev = BitArray(bytes = hashes_list[0], length=hash_len_bits) for curr_hash in hashes_list[1:]: curr = BitArray(bytes=curr_hash, length=hash_len_bits) i = 0 while prev[i] == curr[i]: i += 1 assert i < 64 bits_differ = hash_len_bits - i assert bits_differ > 0 and bits_differ < 65 outarray.append(Bits(uint=(bits_differ - 1), length=6)) diferring_bits = curr[i:] #print i, bits_differ, diferring_bits.length assert bits_differ == diferring_bits.length outarray.append(diferring_bits) prev = curr return outarray.tobytes()
def issue_dht_trace(logger): def clean_up(): # Clean up logger.debug('closing socket') s.close() logger.debug('done') # Test DHT-server request handling from KX address = (config.HOSTNAME, config.PORT) logger.info('server on %s:%s', address[0], address[1]) # Connect to the server logger.debug('creating socket') s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) logger.debug('connecting to server') s.connect(address) # DHT TRACE key = BitArray(int=randrange(0, 100), length=256) pack_str = 'HH32s' size = calcsize(pack_str) dht_get = pack(pack_str, socket.htons(size), socket.htons(config.MSG_DHT_TRACE), key.tobytes()) logger.debug('key: %s' % str(key)) logger.debug('total DHT TRACE request size: %s' % size) # Send the data logger.debug('sending the DHT TRACE request') s.send(dht_get) # Receive a response logger.debug('waiting for response') res_header = s.recv(4) logger.debug('response length: %s' % len(res_header)) if len(res_header) == 0: clean_up() else: amount_received = len(res_header) unpacked_header = unpack('HH', res_header) res_size = socket.ntohs(unpacked_header[0]) res_type = socket.ntohs(unpacked_header[1]) logger.debug('response size: %s' % res_size) logger.debug('response type: %s' % res_type) if res_type == config.MSG_DHT_ERROR: clean_up() amount_expected = res_size res_body = '' while amount_received < amount_expected: res_body += s.recv(16384) amount_received = len(res_header) + len(res_body) trace_recs_num = (len(res_body)-calcsize('32s'))/calcsize('32sHH4s16s') logger.debug('trace_recs_num: %s' % trace_recs_num) pack_str = '32s' + trace_recs_num * '32sHH4s16s' unpacked_response = unpack(pack_str, res_body) logger.debug('pack_str: %s' % pack_str) logger.debug('key: %s' % '0x' + ''.join([hex(ord(i)) for i in unpacked_response[0]]).replace('x00', '00').replace('0x', '')) logger.debug('len(unpacked_response): %s' % str(len(unpacked_response))) for i in range(trace_recs_num): logger.debug('peer_ID: %s' % '0x' + ''.join([hex(ord(j)) for j in unpacked_response[i*5+1]]).replace('x00', '00').replace('0x', '')) logger.debug('KX_port: %s' % socket.ntohs(unpacked_response[i*5+2])) logger.debug('IPv4_address: %s' % '0x' + ''.join([hex(ord(j)) for j in unpacked_response[i*5+4]]).replace('x00', '00').replace('0x', '')) logger.debug('IPv6_address: %s' % '0x' + ''.join([hex(ord(j)) for j in unpacked_response[i*5+5]]).replace('x00', '00').replace('0x', ''))
import serial import math from bitstring import BitArray import socket import argparse parser = argparse.ArgumentParser(description='Transmit PPM from serial.') parser.add_argument('serial', help='the serial port to pull PPM from') parser.add_argument('serial_rate', help='the rate in bps to connect to the serial port at', type=int) parser.add_argument('destination_ip', help='the IP address to send PPM to') parser.add_argument('port', help='the port to use', type=int) args = parser.parse_args() s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) ser = serial.Serial(args.serial, args.serial_rate) while True: bits = BitArray() line = ser.readline().strip() for pulse_length in line.split(): bits.append('uint:16=' + pulse_length) s.sendto(bits.tobytes(), (args.destination_ip, args.port)) print line
def writeFileBIN(self): f = open(self.file_name_bin, 'wb') s = ''.join(self.bin_list) b = BitArray(bin=s) f.write(b.tobytes()) f.close()
def issue_dht_put(logger): def clean_up(): # Clean up logger.debug('closing socket') s.close() logger.debug('done') # Test DHT-server request handling from KX address = (config.HOSTNAME, config.PORT) logger.info('server on %s:%s', address[0], address[1]) # Connect to the server logger.debug('creating socket') s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) logger.debug('connecting to server') s.connect(address) # DHT PUT key = BitArray(int=randrange(0, 100), length=256) ttl = 12 replication = 4 reserved = BitArray(int=0, length=32) content = BitArray(int=randrange(0, 1000), length=256) pack_str = 'HH32sHH4s%ss' % str(len(content)/8) size = calcsize(pack_str) logger.debug('key: %s' % str(key)) logger.debug('ttl: %s' % ttl) logger.debug('replication: %s (in 8 MSBits: %s)' % (replication, replication << 8)) logger.debug('reserved: %s' % str(reserved)) logger.debug('content: %s' % str(content)) logger.debug('total DHT PUT request size: %s' % size) dht_put = pack(pack_str, socket.htons(size), socket.htons(config.MSG_DHT_PUT), key.tobytes(), socket.htons(ttl), socket.htons(replication << 8), reserved.tobytes(), content.tobytes()) # Send the data logger.debug('sending the DHT PUT request') s.send(dht_put) # Receive a response logger.debug('waiting for response') res_header = s.recv(4) logger.debug('response length: %s' % len(res_header)) if len(res_header) == 0: clean_up() else: amount_received = len(res_header) unpacked_header = unpack('HH', res_header) res_size = socket.ntohs(unpacked_header[0]) res_type = socket.ntohs(unpacked_header[1]) logger.debug('response size: %s' % res_size) logger.debug('response type: %s' % res_type) clean_up()
class AdaptiveCoder: def encode_file(self, input_filepath, output_filepath): """Kódování souboru""" self.init_frequencies() self.size = 0 self.lp = 0 self.up = self.M self.c = 0 self.res = BitArray() filesize = os.path.getsize(input_filepath) with open(input_filepath, 'rb') as f: # Soubor čtu po bytech, výstupní kod se průběžne ukládá do self.res byte = f.read(1) while byte != b'': if self.size % 1000 == 0: print("{0}/{1}".format(self.size, filesize)) self.encode_integer(ord(byte)) byte = f.read(1) self.res.append(bin(1)) with open(output_filepath, 'wb') as fo: order = floor(log(self.size, 2)) + 1 fo.write((order).to_bytes(16, byteorder="big", signed=False)) fo.write(self.size.to_bytes(order, byteorder="big", signed=False)) fo.write(self.res.tobytes()) return (self.size, self.res.bin) def encode_integer(self, character): """ Kodování znaku v modelu""" self.size += 1 # přičteme jedničku kvůli posunutí indexu new_up = self.lp + ((self.up - self.lp + 1) * self.cumul_freq[character + 1]) // self.cumul_freq[self.symbols] - 1 new_lp = self.lp + ((self.up - self.lp + 1) * self.cumul_freq[character]) // self.cumul_freq[self.symbols] self.up = new_up self.lp = new_lp #print(self.up, self.lp) # print(self.M) self.increment_freq(character) if self.up < self.lp: print(self.size) raise Exception("Lower bound is bigger than upper.") # škálování while (self.up < self.H) or (self.lp >= self.H) or ((self.lp >= self.Q1) and (self.up < self.Q3)): d = 0 #print((self.lp, self.up)) if ((self.lp >= self.Q1) and (self.up < self.Q3)): self.c += 1 d = self.Q1 else: if self.up < self.H: b = 0 d = 0 else: b = 1 d = self.H self.res.append(bin(b)) while self.c > 0: self.res.append(bin(not(b))) self.c -= 1 self.lp = 2 * (self.lp - d) self.up = 2 * (self.up - d) + 1 def decode_file(self, input_filepath, output_filepath): """ Dekódování souboru""" self.init_frequencies() self.lp = 0 self.up = self.M with open(input_filepath, 'rb') as fo: order = int.from_bytes(fo.read(16), byteorder='big', signed=False) size = int.from_bytes( fo.read(order), byteorder='big', signed=False) encoded = fo.read() if self.k - len(encoded) > 0: "Doplnění nulových bytů" encoded += bytes(self.k - len(encoded)) a = BitStream(encoded) with open(output_filepath, 'wb') as fi: self.decode_stream(size, a, fi) def decode_stream(self, size, encoded, output_file): """Dekódování streamu""" j = 0 x = encoded.read("uint:%s" % self.k) decoded = bytearray(self.BUF_SIZE) pos = 0 while True: if (j % 1000) == 0: print("{0}/{1}".format(j, size)) sym = None # kumulativní četnost x c = (self.cumul_freq[self.symbols] * (x - self.lp + 1)) // (self.up - self.lp + 1) # hledání intervalu, kam patří x # for i in range(0, self.symbols): # low = self.cumul_freq[i] # high = self.cumul_freq[i + 1] # if low <= c < high: # sym = i # break # optimálnější hledání intervalu u = 0 v = self.symbols - 1 while u <= v: s = (u + v) // 2 low = self.cumul_freq[s] high = self.cumul_freq[s + 1] if low <= c < high: sym = s break if c < low: v = s - 1 else: u = s + 1 # Chyba, kumulativní četnost x nezařazena if sym is None: print("(lp, up) : (%s, %s)" % (self.lp, self.up)) print("Current position: %s" % j) print("x value: %s" % x) print("X cumulative value: %s" % c) tmp = self.cumul_freq[self.symbols] * \ (x - self.lp + 1) / (self.up - self.lp + 1) print("X unrounded cumulative value: %s" % tmp) print(self.cumul_freq) raise Exception("Symbol not found") # symbol do bufferu decoded[pos] = sym pos += 1 # vyprázdnění bufferu if pos == self.BUF_SIZE: pos = 0 output_file.write(decoded) j += 1 if (j == size): if pos > 0: output_file.write(decoded[:pos]) break new_up = self.lp + \ ((self.up - self.lp + 1) * self.cumul_freq[sym + 1]) // self.cumul_freq[self.symbols] - 1 new_lp = self.lp + \ ((self.up - self.lp + 1) * self.cumul_freq[sym]) // self.cumul_freq[self.symbols] self.lp = new_lp self.up = new_up self.increment_freq(sym) #print((self.lp, self.up)) # škálování while (self.up < self.H) or (self.lp >= self.H) or ((self.lp >= self.Q1) and (self.up < self.Q3)): if ((self.lp >= self.Q1) and (self.up < self.Q3)): d = self.Q1 else: if self.up < self.H: d = 0 else: d = self.H self.lp = 2 * (self.lp - d) self.up = 2 * (self.up - d) + 1 if encoded.pos < encoded.len: b = encoded.read("uint:1") else: b = 0 x = ((2 * (x - d)) + b) if not (0 <= x < self.M): raise Exception("Variable x is out of range: %s" % x) def init_frequencies(self): # inicializujeme počtu výskytu znaků self.freq = [0] + self.symbols * [1] self.cumul_freq = list(range(0, 257)) def increment_cumul_from(self, i_from=1): for i in range(i_from, 257): self.cumul_freq[i] += 1 def refrest_cumul_freq(self): print("adjusting frequencies") acc = 0 tmp = 0 for i in range(1, 257): tmp = self.freq[i] if tmp > 1: tmp = tmp // 2 self.freq[i] = tmp acc += tmp self.cumul_freq[i] = acc def increment_freq(self, pos): """ Inkrementace frekvence výskytu symbolu""" # pos+1 kvůli g_{i-1}, na nultém indexu je vždy nula self.freq[pos + 1] += 1 self.increment_cumul_from(pos + 1) # Ošetření P_min if self.cumul_freq[self.symbols] == self.Q1: self.refrest_cumul_freq() def __init__(self, k=32): # Inicializace parametrů kodéru self.k = k self.M = 2**self.k - 1 self.symbols = 256 self.Q1 = 2**(self.k - 2) self.H = 2 * self.Q1 self.Q3 = 3 * self.Q1 # Velikost bufferu pro zápis, nutno optimalizovat self.BUF_SIZE = 2**10
class TestPeerWireTranslator(object): def setup_class(self): self.translator = PeerWireTranslator() self.msg_keep_alive = struct.pack('B', 0) self.msg_choke = struct.pack('>IB', 1, _MSG_CHOKE) self.msg_unchoke = struct.pack('>IB', 1, _MSG_UNCHOKE) self.msg_interested = struct.pack('>IB', 1, _MSG_INTERESTED) self.msg_not_interested = struct.pack('>IB', 1, _MSG_NOT_INTERESTED) self.have_index = 55 self.msg_have = struct.pack('>IBI', 5, _MSG_HAVE, self.have_index) self.bits = BitArray('0x123456789abcdef') bitsasbytes = self.bits.tobytes() length = len(bitsasbytes) self.msg_bitfield = struct.pack('>IB{}s'.format(length), 1+length, _MSG_BITFIELD, bitsasbytes) self.request_index = 10 self.request_begin = 60 self.request_length = 255 self.msg_request = struct.pack('>IB3I', 13, _MSG_REQUEST, self.request_index, self.request_begin, self.request_length) self.piece_index = 17 self.piece_begin = 245 self.piece_block = """Four score and seven years ago our fathers brought forth on this continent a new nation, conceived in liberty, and dedicated to the proposition that all men are created equal. Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting place for those who here gave their lives that that nation might live. It is altogether fitting and proper that we should do this. But, in a larger sense, we can not dedicate, we can not consecrate, we can not hallow this ground. The brave men, living and dead, who struggled here, have consecrated it, far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us—that from these honored dead we take increased devotion to that cause for which they gave the last full measure of devotion—that we here highly resolve that these dead shall not have died in vain—that this nation, under God, shall have a new birth of freedom—and that government of the people, by the people, for the people, shall not perish from the earth.""" length = len(self.piece_block) self.msg_piece = struct.pack('>IB2I{}s'.format(length), 9+length, _MSG_PIECE, self.piece_index, self.piece_begin, self.piece_block) self.cancel_index = 99 self.cancel_begin = 128 self.cancel_length = 13 self.msg_cancel = struct.pack('>IB3I', 13, _MSG_CANCEL, self.cancel_index, self.cancel_begin, self.cancel_length) self.junk = "abcdefghijklmnop" length = len(self.junk) self.msg_invalid = struct.pack('>IB{}s'.format(length), 1+length, _MSG_INVALID, self.junk) def rx_bytes(self, string): message = bytearray(string) sent = 0 while sent != len(message): buf, needed = self.translator.get_rx_buffer() buf[:needed] = message[sent:sent+needed] sent = sent+needed self.translator.rx_bytes(needed) def rx_bytes_in_parts(self, string): message = bytearray(string) sent = 0 while sent != len(message): buf, needed = self.translator.get_rx_buffer() if needed > 1: buf[:needed-1] = message[sent:sent+needed-1] sent = sent+needed-1 self.translator.rx_bytes(needed-1) else: buf[:needed] = message[sent:sent+needed] sent = sent+needed self.translator.rx_bytes(needed) """ def test_keep_alive(self): self.rx_bytes(self.msg_keep_alive) self.translator.set_receiver(self.receiver) self.rx_bytes(self.msg_keep_alive) assert self.receiver.rx_keep_alive.called call_count = self.receiver.rx_keep_alive.call_count self.translator.unset_receiver() self.rx_bytes(self.msg_keep_alive) assert self.receiver.rx_keep_alive.call_count == call_count """ def test_rx_choke(self): self.rx_bytes(self.msg_choke) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes(self.msg_choke) assert self.receiver.rx_choke.called call_count = self.receiver.rx_choke.call_count self.rx_bytes_in_parts(self.msg_choke) assert self.receiver.rx_choke.call_count == call_count+1 call_count = self.receiver.rx_choke.call_count self.translator.unset_receiver() self.rx_bytes(self.msg_choke) assert self.receiver.rx_choke.call_count == call_count def test_rx_unchoke(self): self.rx_bytes(self.msg_unchoke) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes(self.msg_unchoke) assert self.receiver.rx_unchoke.called call_count = self.receiver.rx_unchoke.call_count self.rx_bytes_in_parts(self.msg_unchoke) assert self.receiver.rx_unchoke.call_count == call_count+1 call_count = self.receiver.rx_unchoke.call_count self.translator.unset_receiver() self.rx_bytes(self.msg_unchoke) assert self.receiver.rx_unchoke.call_count == call_count def test_rx_interested(self): self.rx_bytes(self.msg_interested) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes(self.msg_interested) assert self.receiver.rx_interested.called call_count = self.receiver.rx_interested.call_count self.rx_bytes_in_parts(self.msg_interested) assert self.receiver.rx_interested.call_count == call_count+1 call_count = self.receiver.rx_interested.call_count self.translator.unset_receiver() self.rx_bytes(self.msg_interested) assert self.receiver.rx_interested.call_count == call_count def test_rx_not_interested(self): self.rx_bytes(self.msg_not_interested) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes(self.msg_not_interested) assert self.receiver.rx_not_interested.called call_count = self.receiver.rx_not_interested.call_count self.rx_bytes_in_parts(self.msg_not_interested) assert self.receiver.rx_not_interested.call_count == call_count+1 call_count = self.receiver.rx_not_interested.call_count self.translator.unset_receiver() self.rx_bytes(self.msg_not_interested) assert self.receiver.rx_not_interested.call_count == call_count def test_rx_have(self): self.rx_bytes(self.msg_have) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes(self.msg_have) self.receiver.rx_have.assert_called_with(self.have_index) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes_in_parts(self.msg_have) self.receiver.rx_have.assert_called_with(self.have_index) call_count = self.receiver.rx_have.call_count self.translator.unset_receiver() self.rx_bytes(self.msg_have) assert self.receiver.rx_have.call_count == call_count def test_rx_bitfield(self): self.rx_bytes(self.msg_bitfield) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes(self.msg_bitfield) bits = self.receiver.rx_bitfield.call_args_list[0][0][0] assert bits.tobytes() == self.bits.tobytes() self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes_in_parts(self.msg_bitfield) bits = self.receiver.rx_bitfield.call_args_list[0][0][0] assert bits.tobytes() == self.bits.tobytes() call_count = self.receiver.rx_bitfield.call_count self.translator.unset_receiver() self.rx_bytes(self.msg_bitfield) assert self.receiver.rx_bitfield.call_count == call_count def test_rx_request(self): self.rx_bytes(self.msg_request) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes(self.msg_request) self.receiver.rx_request.assert_called_with(self.request_index, self.request_begin, self.request_length) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes_in_parts(self.msg_request) self.receiver.rx_request.assert_called_with(self.request_index, self.request_begin, self.request_length) call_count = self.receiver.rx_request.call_count self.translator.unset_receiver() self.rx_bytes(self.msg_request) assert self.receiver.rx_request.call_count == call_count def test_rx_piece(self): self.rx_bytes(self.msg_piece) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes(self.msg_piece) args = self.receiver.rx_piece.call_args_list[0][0] assert args[0] == self.piece_index assert args[1] == self.piece_begin for i in range(len(args[2])): assert args[2][i] == self.piece_block[i] print self.piece_block self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes_in_parts(self.msg_piece) args = self.receiver.rx_piece.call_args_list[0][0] assert args[0] == self.piece_index assert args[1] == self.piece_begin for i in range(len(args[2])): assert args[2][i] == self.piece_block[i] call_count = self.receiver.rx_piece.call_count self.translator.unset_receiver() self.rx_bytes(self.msg_piece) assert self.receiver.rx_piece.call_count == call_count def test_rx_cancel(self): self.rx_bytes(self.msg_cancel) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes(self.msg_cancel) self.receiver.rx_cancel.assert_called_with(self.cancel_index, self.cancel_begin, self.cancel_length) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes_in_parts(self.msg_cancel) self.receiver.rx_cancel.assert_called_with(self.cancel_index, self.cancel_begin, self.cancel_length) call_count = self.receiver.rx_cancel.call_count self.translator.unset_receiver() self.rx_bytes(self.msg_cancel) assert self.receiver.rx_cancel.call_count == call_count def test_rx_invalid(self): self.rx_bytes(self.msg_invalid) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes(self.msg_invalid) self.receiver = Mock() self.translator.set_receiver(self.receiver) self.rx_bytes_in_parts(self.msg_invalid) self.translator.unset_receiver() self.rx_bytes(self.msg_invalid) def test_tx_keep_alive(self): self.translator.tx_keep_alive() self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_keep_alive() calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_keep_alive call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_keep_alive() assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_choke(self): self.translator.tx_choke() self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_choke() calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_choke call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_choke() assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_unchoke(self): self.translator.tx_unchoke() self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_unchoke() calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_unchoke call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_unchoke() assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_interested(self): self.translator.tx_interested() self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_interested() calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_interested call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_interested() assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_not_interested(self): self.translator.tx_not_interested() self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_not_interested() calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_not_interested call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_not_interested() assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_have(self): self.translator.tx_have(self.have_index) self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_have(self.have_index) calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_have call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_have(self.have_index) assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_bitfield(self): self.translator.tx_bitfield(self.bits) self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_bitfield(self.bits) calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_bitfield call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_bitfield(self.bits) assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_request(self): self.translator.tx_request(self.request_index, self.request_begin, self.request_length) self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_request(self.request_index, self.request_begin, self.request_length) calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_request call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_request(self.request_index, self.request_begin, self.request_length) assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_piece(self): self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block) self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block) calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_piece call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block) assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_cancel(self): self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length) self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length) calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_cancel call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length) assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_keep_alive(self): self.translator.tx_keep_alive() self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_keep_alive() calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_keep_alive call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_keep_alive() assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_choke(self): self.translator.tx_choke() self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_choke() calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_choke call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_choke() assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_unchoke(self): self.translator.tx_unchoke() self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_unchoke() calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_unchoke call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_unchoke() assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_interested(self): self.translator.tx_interested() self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_interested() calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_interested call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_interested() assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_not_interested(self): self.translator.tx_not_interested() self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_not_interested() calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_not_interested call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_not_interested() assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_have(self): self.translator.tx_have(self.have_index) self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_have(self.have_index) calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_have call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_have(self.have_index) assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_bitfield(self): self.translator.tx_bitfield(self.bits) self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_bitfield(self.bits) calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_bitfield call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_bitfield(self.bits) assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_request(self): self.translator.tx_request(self.request_index, self.request_begin, self.request_length) self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_request(self.request_index, self.request_begin, self.request_length) calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_request call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_request(self.request_index, self.request_begin, self.request_length) assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_piece(self): self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block) self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block) calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_piece call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_piece(self.piece_index, self.piece_begin, self.piece_block) assert self.readerwriter.tx_bytes.call_count == call_count def test_tx_cancel(self): self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length) self.readerwriter = Mock() self.translator.set_readerwriter(self.readerwriter) self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length) calls = self.readerwriter.tx_bytes.call_args_list assert "".join([arg[0][0] for arg in calls]) == self.msg_cancel call_count = self.readerwriter.tx_bytes.call_count self.translator.unset_readerwriter() self.translator.tx_cancel(self.cancel_index, self.cancel_begin, self.cancel_length) assert self.readerwriter.tx_bytes.call_count == call_count
class Option(object): """ Represent a CoAP option. """ def __init__(self): """ Initialize an option. """ self._number = None self._value = None @property def number(self): """ Get the option number. :return: the option number """ return self._number @number.setter def number(self, number): """ Set the option number. :param number: the number """ self._number = number @property def value(self): """ Get the option value. :return: the option value as bytes """ if type(self._value) is None: self._value = BitArray() name, opt_type, repeatable, defaults = defines.options[self._number] if opt_type == defines.INTEGER: if self._value.len > 0: return self._value.uint else: return defaults return self._value.tobytes() @value.setter def value(self, val): """ Sets the option value. :param val: the value """ if type(val) is str: val = BitArray(bytes=val, length=len(val) * 8) if type(val) is int and bit_len(val) != 0: val = BitArray(uint=val, length=bit_len(val) * 8) if type(val) is int and bit_len(val) == 0: val = BitArray() assert(type(val) is BitArray) self._value = val @property def raw_value(self): """ Get the option value. :return: the option value as BitArray """ if type(self._value) is None: self._value = BitArray() return self._value @property def length(self): """ Get the len of the option value :return: the len of the option value """ assert(type(self._value) is BitArray) return len(self._value.tobytes()) @property def safe(self): """ Check if option is safe. :return: True if safe, False otherwise """ if self._number == defines.inv_options["Uri-Host"] or self._number == defines.inv_options["Uri-Port"] \ or self._number == defines.inv_options["Uri-Path"] or self._number == defines.inv_options["Max-Age"] \ or self._number == defines.inv_options["Uri-Query"] \ or self._number == defines.inv_options["Proxy-Uri"] \ or self._number == defines.inv_options["Proxy-Scheme"]: return False return True @property def name(self): """ Gets option name. :return: the name of the oprion """ return defines.options[self._number][0] def __str__(self): """ Return the option as a formatted string. :return: the string representing the option """ name, opt_type, repeatable, defaults = defines.options[self._number] if name == "ETag": return name + ": " + str(self.raw_value) + "\n" else: return name + ": " + str(self.value) + "\n" def __eq__(self, other): """ Compare options. :param other: the option to compare :return: True if equal """ return self.__dict__ == other.__dict__
def compress(source_filepath, dest): try: source_file = open(source_filepath, "rb").read().decode('utf-8') except IOError as e: logging.critical("I/O error({}): {}".format(e.errno, e.strerror)) except: logging.critical("{} is not a valid encoded utf-8 file". format(source_file)) # ----------------------------------------------------------------------- # # Get information on the source file: # - Alphabet and weight of each symbol from the alphabet # ----------------------------------------------------------------------- # huff_prob = huffprob.HuffProb(source_file) stats = huff_prob.get_stats() get_process_time("Get stats") # ----------------------------------------------------------------------- # # Build the Huffman tree and the prefix-free binary code for each symbol # ----------------------------------------------------------------------- # huff_tree = hufftree.HuffTree(stats) get_process_time("Build HuffTree") # ----------------------------------------------------------------------- # # Encode the source file using the prefix-free binary code # ----------------------------------------------------------------------- # codewords = huff_tree.get_codewords() buf = BitArray() for char in source_file: buf += codewords[char] # Convert "01101..." BitArray to a bytearray encoded_data = buf.tobytes() get_process_time("Encode file") # ----------------------------------------------------------------------- # # Create the output file: # - Add a header with the tree and symbol list # - Add the encoded data # ----------------------------------------------------------------------- # tree_struct = huff_tree.get_bitstring_struct() tree_symbols = huff_tree.get_symbols() header = tree_struct.tobytes() header += bytearray(''.join(tree_symbols).encode('utf-8')) get_process_time("Build header") dest_filepath = "{}.huf".format(os.path.splitext(source_filepath)[0]) dest_file = open(dest_filepath, "wb") dest_file.write(header + encoded_data) dest_file.close() get_process_time("Write .huf file") # ----------------------------------------------------------------------- # # Performance report # ----------------------------------------------------------------------- # source_size = os.path.getsize(source_filepath) dest_size = os.path.getsize(dest_filepath) rate = 1 - (dest_size / source_size) logging.info("{} ({}b) -> {} ({}b)".format(source_filepath, source_size, dest_filepath, dest_size )) logging.info("Compression rate: {:.2%}".format(rate))