예제 #1
0
def store_data(filename_base, code_page, num_chunks, chunks):
    int_chunks = [int(chunk, 2) for chunk in chunks]
    out_file_path = make_path("%s%s_encoded%s" % (settings.OUT_FOLDER, filename_base, settings.HUFFMAN_DATAFILE_EXTENSION))
    with open(out_file_path, "wb") as encoded_file:
        encoded_file.write(struct.pack('I', num_chunks))
        encoded_file.write(bytearray(int_chunks))
    out_file_path = make_path("%s%s_encoding%s" % (settings.OUT_FOLDER, filename_base, settings.HUFFMAN_DATAFILE_EXTENSION))
    json.dump(code_page, open(out_file_path, 'w'))
예제 #2
0
def load_data(filename_base):
    in_file_path = make_path("%s%s_encoding%s" % (settings.OUT_FOLDER, filename_base, settings.HUFFMAN_DATAFILE_EXTENSION))
    code_page = json.load(open(in_file_path))
    in_file_path = make_path("%s%s_encoded%s" % (settings.OUT_FOLDER, filename_base, settings.HUFFMAN_DATAFILE_EXTENSION))
    file_size = os.path.getsize(in_file_path)
    with open(in_file_path, "rb") as encoded_file:
        (num_chars,) = struct.unpack('I', encoded_file.read(4))
        n_bytes = file_size-4
        bytes = list(struct.unpack('%dB' % n_bytes, encoded_file.read(n_bytes)))
    to_bin = lambda i: format(i, '08b')
    encoded_string = ''.join(map(to_bin, bytes))

    # print encoded_string

    return code_page, num_chars, encoded_string
예제 #3
0
def huffman(filename_base, chunk_size):
    encoded_string = ""
    in_file_path = make_path("%s%s%s" % (settings.DATA_FOLDER, filename_base, settings.HUFFMAN_DATAFILE_EXTENSION))
    with open(in_file_path) as f:
        for line in f:
            encoded_string += unicode(line, errors='replace') + u'\n'

    if settings.DEBUG:
        encoded_string = u"TO BE OR\n NOT TO BE"
    frequencies = count_occurences(encoded_string, chunk_size)
    tree = construct_huffman_tree(frequencies)
    code_page = construct_code_page(tree)
    if not settings.SURPRESS_OUTPUT:
        print code_page

    num_chunks, chunks = encode(code_page, encoded_string, chunk_size)

    store_data(filename_base, code_page, num_chunks, chunks)
    code_page, num_chars, encoded_string = load_data(filename_base)
    decoded_string = decode(code_page, num_chars, encoded_string)

    if not settings.SURPRESS_OUTPUT:
        print ">> decoded string:", decoded_string

    out_file_path = make_path("%s%s_decoded%s" % (settings.OUT_FOLDER, filename_base, settings.HUFFMAN_DATAFILE_EXTENSION))
    with open(out_file_path, 'w') as f:
        f.write(decoded_string.encode('utf8'))

    file_path = make_path("%s%s%s" % (settings.DATA_FOLDER, filename_base, settings.HUFFMAN_DATAFILE_EXTENSION))
    decoded_file_size = os.path.getsize(file_path)
    file_path = make_path("%s%s_encoded%s" % (settings.OUT_FOLDER, filename_base, settings.HUFFMAN_DATAFILE_EXTENSION))
    encoded_file_size = os.path.getsize(file_path)
    file_path = make_path("%s%s_encoding%s" % (settings.OUT_FOLDER, filename_base, settings.HUFFMAN_DATAFILE_EXTENSION))
    encoding_file_size = os.path.getsize(file_path)

    print "Encoded size:", encoded_file_size
    print "Encodingsize:", encoding_file_size
    print "Decoded size:", decoded_file_size

    print "Compression rate:", float(encoded_file_size + encoding_file_size) / float(decoded_file_size) * 100.0, "%"