Beispiel #1
0
 def startCompressMessage(self):
    if self._isServer:
       if self._compressor is None:
          self._compressor = bz2.BZ2Compressor(self.server_max_compress_level)
    else:
       if self._compressor is None:
          self._compressor = bz2.BZ2Compressor(self.client_max_compress_level)
Beispiel #2
0
    def _write(self, raw_data, filename) :
        """
        Write raw data to a compressed file.

        @arg raw_data: The raw_data to be compressed and written
        @type raw_data: byte string
        @arg filename: The intended name of the outfile
        @type filename: unicode

        @return: outfile ; The full path and name of the file written
        @rtype: unicode
        """
        result = chardet.detect(raw_data)
        if result['confidence'] > 0.5:
            encoding = result['encoding']
        else:
            encoding = 'utf-8'

        if not util.is_utf8_alias(encoding):
            raw_data = raw_data.decode(encoding).encode('utf-8')

        # Compress the data to save disk space.
        comp = bz2.BZ2Compressor()
        data = comp.compress(raw_data)
        data += comp.flush()
        out_handle = open(self._nametofile(filename), "wb")
        out_handle.write(data)
        out_handle.close()

        return out_handle.name      # return the full path to the file
Beispiel #3
0
    def handle(self):
        compressor = bz2.BZ2Compressor()

        # Find out what file the client wants
        filename = self.request.recv(1024)
        self.logger.debug('client asked for: "%s"', filename)

        # Send chunks of the file as they are compressed
        with open(filename, 'rb') as input:
            while True:
                block = input.read(BLOCK_SIZE)
                if not block:
                    break
                self.logger.debug('RAW "%s"', block)
                compressed = compressor.compress(block)
                if compressed:
                    self.logger.debug('SENDING "%s"',
                                      binascii.hexlify(compressed))
                    self.request.send(compressed)
                else:
                    self.logger.debug('BUFFERING')

        # Send any data being buffered by the compressor
        remaining = compressor.flush()
        while remaining:
            to_send = remaining[:BLOCK_SIZE]
            remaining = remaining[BLOCK_SIZE:]
            self.logger.debug('FLUSHING "%s"', binascii.hexlify(to_send))
            self.request.send(to_send)
        return
Beispiel #4
0
 def get(cls):
     if support['lzma']:
         return b'LZ00', lzma.LZMACompressor()
     elif support['bz2']:
         return b'BZ00', bz2.BZ2Compressor()
     else:
         return b'ZL00', zlib.compressobj()
Beispiel #5
0
    def handle(self):
        compressor = bz2.BZ2Compressor()

        # Cosa vuole il client?
        filename = self.request.recv(1024).decode('utf-8')
        self.logger.debug('il client richiede: "%s"', filename)

        # Invio di pezzi del file mentre si stanno comprimento
        with open(filename, 'rb') as input:
            while True:
                block = input.read(BLOCK_SIZE)
                if not block:
                    break
                self.logger.debug('GREZZI %r', block)
                compressed = compressor.compress(block)
                if compressed:
                    self.logger.debug('IN INVIO %r',
                                      binascii.hexlify(compressed))
                    self.request.send(compressed)
                else:
                    self.logger.debug('ACCUMULO')

        # Invia tutti i dati accumulati al compressore
        remaining = compressor.flush()
        while remaining:
            to_send = remaining[:BLOCK_SIZE]
            remaining = remaining[BLOCK_SIZE:]
            self.logger.debug('SCARICAMENTO %r', binascii.hexlify(to_send))
            self.request.send(to_send)
        return
Beispiel #6
0
def preprocess_data(path: str,
                    output: Optional[str] = None,
                    *,
                    delete_original: bool = True) -> str:
    """Extracts moves from file on given path using :func extract_games: and saves it to compressed file
     in .bz2 format to original location or ``output`` if provided

    :param path: path to a file
    :param output: optional path to save location
    :param delete_original: wheather to delete original uncompressed file, default `True`

    :return: path to compressed file
    """
    compressor = bz2.BZ2Compressor()
    output = output if output else os.path.join(
        os.path.dirname(path), 'proc_' + os.path.basename(path))
    # modifying path is necessary, because otherwise we read and write to the same file
    if not output.endswith('.bz2'):
        output += '.bz2'
    with open(output, 'wb') as f:
        for game in extract_games(
                path):  # extract and compress games one by one
            game_str = bytes(' '.join(game) + '\n', encoding='utf-8')
            data = compressor.compress(game_str)
            f.write(data)
        f.write(compressor.flush())

    if delete_original:
        os.remove(path)
    return output
Beispiel #7
0
def _write_data(data, filehandle, options):
    # Now write data directly
    rawdata = data.tostring(order='F')
    if options['encoding'] == 'raw':
        filehandle.write(rawdata)
    elif options['encoding'].lower() in ['ascii', 'text', 'txt']:
        # savetxt only works for 1D and 2D arrays, so reshape any > 2 dim arrays into one long 1D array
        if data.ndim > 2:
            np.savetxt(filehandle, data.ravel(order='F'), '%.17g')
        else:
            np.savetxt(filehandle, data.T, '%.17g')
    else:
        if options['encoding'] == 'gzip':
            comp_obj = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16)
        elif options['encoding'] == 'bzip2':
            comp_obj = bz2.BZ2Compressor()
        else:
            raise NrrdError('Unsupported encoding: "%s"' % options['encoding'])

        # write data in chunks
        start_index = 0
        while start_index < len(rawdata):
            end_index = start_index + _WRITE_CHUNKSIZE
            if end_index > len(rawdata):
                end_index = len(rawdata)
            filehandle.write(comp_obj.compress(rawdata[start_index:end_index]))
            start_index = end_index
        filehandle.write(comp_obj.flush())
        filehandle.flush()
Beispiel #8
0
 def _initialize_compressor(self):
     if self._compression_type == CompressionTypes.BZIP2:
         self._compressor = bz2.BZ2Compressor()
     else:
         assert self._compression_type == CompressionTypes.GZIP
         self._compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION,
                                             zlib.DEFLATED, self._gzip_mask)
Beispiel #9
0
def _get_encoder(compression):
    if compression == 'zlib':
        try:
            import zlib
        except ImportError:
            raise ImportError("Your Python does not have the zlib library, "
                              "therefore the block in this ASDF file "
                              "can not be compressed.")
        return zlib.compressobj()
    elif compression == 'bzp2':
        try:
            import bz2
        except ImportError:
            raise ImportError("Your Python does not have the bz2 library, "
                              "therefore the block in this ASDF file "
                              "can not be compressed.")
        return bz2.BZ2Compressor()
    elif compression == 'lz4':
        try:
            import lz4.block
        except ImportError:
            raise ImportError(
                "lz4 library in not installed in your Python environment, "
                "therefore the block in this ASDF file "
                "can not be compressed.")
        return Lz4Compressor(lz4.block)
    else:
        raise ValueError("Unknown compression type: '{0}'".format(compression))
Beispiel #10
0
def bz2_demo():
    # 单次压缩
    bytes_com = bz2.compress(content)
    print("单次压缩: ", bytes_com)
    bytes_decom = bz2.decompress(bytes_com)
    print("单次解压: ", bytes_decom)

    # 增量压缩
    bzcom = bz2.BZ2Compressor()
    bzdecom = bz2.BZ2Decompressor()

    bytes_com = bzcom.compress(content)
    bytes_com += bzcom.flush()
    print("增量压缩: ", bytes_com)

    bytes_decom = bzdecom.decompress(bytes_com)
    print("增量解压: ", bytes_decom)

    # 读写压缩
    with open('file.txt', 'rb') as read, bz2.open('file.txt.gz',
                                                  'wb') as write:
        shutil.copyfileobj(read, write)
    with bz2.open('file.txt.gz', 'rb') as read, open('temp.txt',
                                                     'wb') as write:
        shutil.copyfileobj(read, write)
Beispiel #11
0
    def format(self, tokensource, outfile):
        if self.compress == 'gz':
            import gzip
            outfile = gzip.GzipFile('', 'wb', 9, outfile)
            write = outfile.write
            flush = outfile.flush
        elif self.compress == 'bz2':
            import bz2
            compressor = bz2.BZ2Compressor(9)

            def write(text):
                outfile.write(compressor.compress(text))

            def flush():
                outfile.write(compressor.flush())
                outfile.flush()
        else:
            write = outfile.write
            flush = outfile.flush

        lasttype = None
        lastval = u''
        if self.error_color:
            for ttype, value in tokensource:
                line = "%s\t%r\n" % (ttype, value)
                if ttype is Token.Error:
                    write(colorize(self.error_color, line))
                else:
                    write(line)
        else:
            for ttype, value in tokensource:
                write("%s\t%r\n" % (ttype, value))
        flush()
Beispiel #12
0
def compress(method, data_in):
    '''
  Compress data in one of a set of methods.
  '''
    data = None

    if method == 'gzip':
        import gzip
        data = gzip.compress(data_in)

    elif method == 'bz2':
        import bz2
        c = bz2.BZ2Compressor()
        data = c.compress(data_in)
        data += c.flush()

    elif method == 'xz':
        import lzma
        c = lzma.LZMACompressor(format=lzma.FORMAT_XZ)
        data = c.compress(data_in)
        data += c.flush()

    elif method == 'none':
        data = data_in

    else:
        raise Exception('Invalid compression method')

    return data
Beispiel #13
0
def bz2_func():
    '''
    使用bzip2压缩算法压缩和解压
    该模块的所有类都是线程安全的
    读取和写入压缩文件: open() BZ2File
    增量压缩: BZ2Compressor BZ2Decopressor
    单次压缩: compress() decompress()
    '''

    # 读写压缩文件
    # 二进制或文本模式打开bzip2压缩文件
    # bz2.open(filename, mode='r', compresslevel=9, encoding=None, errors=None, newline=None)
    file = bz2.open("box.bz2")

    # 单次压缩
    # bz2.compress(data, compresslevel=9)
    bytes_com = bz2.compress(b'luzhuo.me')
    # bz2.decompress(data)
    bytes_decom = bz2.decompress(bytes_com)

    # 增量压缩
    # class bz2.BZ2Compressor(compresslevel=9)
    bzcom = bz2.BZ2Compressor()

    bytes_com = bzcom.compress(b'luzhuo.me')  # 压缩
    bytes_com = bzcom.flush()  # 完成压缩, 返回剩余的压缩数据

    # 增量压缩的解压缩
    # class bz2.BZ2Decompressor
    bzdecom = bz2.BZ2Decompressor()

    # decompress(data, max_length=-1) // 解压
    bytes_decom = bzdecom.decompress(bytes_com)
    boolean = bzdecom.eof  # 是否到达流结束标记
    def handle(self):
        compressor = bz2.BZ2Compressor()

        # Find out what the client wants
        filename = self.request.recv(1024)
        self.logger.debug("client asked for '%s'", filename)

        # Send chunks of the file as they are compressed
        with open(filename, 'rb') as input:
            while True:
                block = input.read(BLOCK_SIZE)
                if not block:
                    break
                self.logger.debug("RAW '%s'", block)
                compressed = compressor.compress(block)
                if compressed:
                    self.logger.debug("SENDING '%s'",
                                      binascii.hexlify(compressed))
                    s.request.send(compressed)
                else:
                    self.logger.debug("Buffering...")

        # Send any data left in the buffer
        remaining = compressor.flush()
        while remaining:
            to_send = remaining[:BLOCK_SIZE]
            remaining = remaining[BLOCK_SIZE:]
            self.logger.debug("FLUSHING '%s'", to_send)
            self.request.send(to_send)
        return
    def generate_and_write(self):
        prev_point = None
        i = 0

        if self.strm == "cfile":
            self.compressor = bz2.BZ2Compressor()

        if self.output_format == "gjson":
            self.write_out('{"type": "FeatureCollection", "features": [')
        if self.output_format == "wkt":
            self.write_out('MULTIPOINT (')

        while i < self.card:
            point = self.generate_point(i, prev_point)
            if self.is_valid_point(point):
                prev_point = point
                data = prev_point.to_string(self.output_format) + '\n'
                self.write_out(data)
                i = i + 1
                if self.output_format == "gjson" and i < self.card:
                    self.write_out(',')
                if self.output_format == "wkt" and i < self.card:
                    self.write_out(',')

        if self.output_format == "gjson":
            self.write_out(']}')
        if self.output_format == "wkt":
            self.write_out(')')

        if self.strm == "cfile":
            data = self.compressor.flush(
            )  # Get the last bits of data remaining in the compressor
            sys.stdout.buffer.write(data)
Beispiel #16
0
						def cereal(self):
							bz2.BZ2Compressor(compresslevel=[1-9])
							commands.getstatusoutput(1,null)
							commands.getstatus(0,null)
							#bz2.BZ2File(Medussa[mode=self.terminating, compress = true, compresslevel=[1-9]])
							def user(self):
								automota.self(classmethod)
								world("\t\t\z\y %z /x5z/").author(serialize(self(setattr)))
								while world < destroyed:
									#create argus Cluster
									Paradox.resolved == true
								#end
									def create(self):
										HTMLParser
										def solid(self):
											#crypt.crypt.__doc__
											def state(self):
												ConfigParser
												def cluster(self):
													ColorPicker("0x1239879")
													commands
													#continue
													def paradox(self):
														#crypt.crypt.__doc__
														def select(self):
															#crypt.crypt.__doc__
															def anomaly(self):
																crypt.crypt(self)
																def space(self):
																	#crypt.crypt.__doc__
																	def dimension(self):#buildtools,afxres,ConfigParser,Cookie
																		print "World"
Beispiel #17
0
    def __init__(self, *args, **kwds):
        if not hasattr(self, "compresslevel"):
            self.compresslevel = 9
        # Compression function with flush and reset.
        c = [bz2.BZ2Compressor()]

        def compress(data):
            if data == "":
                return ""
            return c[0].compress(data)

        def c_flush():
            return c[0].flush()

        def c_reset():
            c[0] = bz2.BZ2Compressor()

        compress.flush = c_flush
        compress.reset = c_reset
        self.compress = compress
        # Decompression funtion with reset
        d = [bz2.BZ2Decompressor()]

        def decompress(data):
            if data == "":
                return ""
            return d[0].decompress(data)

        def d_reset():
            d[0] = bz2.BZ2Decompressor()

        decompress.reset = d_reset
        self.decompress = decompress
        # These can now be used by superclass constructors
        super(BZip2Mixin, self).__init__(*args, **kwds)
Beispiel #18
0
    def wrapper(*args, **kwargs):
        result_iter = f(*args, **kwargs)
        assert isinstance(result_iter, GeneratorType), repr(result_iter)

        compressor = bz2.BZ2Compressor(BZ2_COMPRESSLEVEL)

        size_unpacked = 0
        size_packed = 0

        # Compress the input data in an iterative way.
        for chunk in result_iter:
            size_unpacked += len(chunk)
            compressed = compressor.compress(chunk)
            size_packed += len(compressed)
            if compressed:
                yield compressed

        # Finalize
        compressed = compressor.flush()
        size_packed += len(compressed)
        if compressed:
            yield compressed

        logger_bzip2.debug(
            'Compressed %i/%i (%.2f%%)', size_packed, size_unpacked, 100.0 *
            size_packed / size_unpacked if size_unpacked else float('inf'))
Beispiel #19
0
def bz2_nnrd(img_list, outfile, scan_name, update, center=''):
    """
    Given a list of 2D image paths create nrrd in a temp file and then write this to a bz2 compressed nrrd.
    """
    reader = Imreader(img_list)
    first_image = reader.imread(img_list[0])
    shape = list(first_image.shape)
    shape = [shape[1], shape[0]]

    if center.lower() == 'tcp':
        # The X and Y are determined from the dimensions of a Z slice
        # TCP data is on it's side so we need to reverese these shape dimensions
        shape.reverse()

    shape.append(len(img_list))

    compressor = bz2.BZ2Compressor()
    compressed_name = outfile + '.bz2'

    with open(compressed_name, 'wb') as fh_w:

        header_str = nrrd_header_from_dict(
            shape,
            first_image.dtype,
            3,
            header=orientations.RAS_HEADER_OPTIONS)
        fh_w.write(compressor.compress(header_str))

        print(center)

        for i, f in enumerate(img_list):

            if i % 20 == 0:
                done = int((100.0 / len(img_list)) * i)
                update.emit('{} {}%'.format(scan_name, done))

            img_arr = reader.imread(f)

            if center.lower() == 'tcp':
                img_arr = np.rot90(img_arr, k=3)
                img_arr = np.fliplr(img_arr)

            elif center.lower() == 'ucd':
                img_arr = np.rot90(img_arr, k=2)

            elif center.lower() == 'ccp':
                pass  # No flip needed for compressor

            else:  # Get the rest ofthe data into RAS format
                img_arr = np.fliplr(img_arr)

            rawdata = img_arr.T.tostring(order='F')

            compressed = compressor.compress(rawdata)
            fh_w.write(compressed)
        # Send any data being buffered by the compressor
        remaining = compressor.flush()
        if remaining:
            fh_w.write(remaining)
Beispiel #20
0
 def transfer(self):
     engine = bz2.BZ2Compressor()
     while 1:
         data = self.read()
         if not data:
             break
         self.write(engine.compress(data))
     self.write(engine.flush())
Beispiel #21
0
 def __init__(self, stream, compression_level=9):
     assert isinstance(compression_level, int)
     assert 1 <= compression_level <= 9
     self.stream = stream
     self.EOF = False
     self.compressor = bz2.BZ2Compressor(compression_level)
     self.buffer = b''  # TODO: bytearray would be more efficient?
     self.pos = 0
Beispiel #22
0
def test_smart_open_for_bz2_file():
    fd, path = tempfile.mkstemp()
    comp = bz2.BZ2Compressor()
    with os.fdopen(fd, 'wb') as f:
        f.write(comp.compress('abcdef'.encode()))
        f.write(comp.flush())
    assert not isinstance(dumper.smart_open(path), bz2.BZ2File)
    os.unlink(path)
Beispiel #23
0
def bz2_nnrd(img_list, outfile, scan_name, update):
    """
    """
    reader = Imreader(img_list)
    first_image = reader.imread(img_list[0])
    shape = list(first_image.shape)
    shape = [shape[1], shape[0]]
    shape.append(len(img_list))
    print '++++==== bzp'
    tempnrrd = tempfile.TemporaryFile(mode="wb+")
    nrrd.write_nrrd_header(tempnrrd, shape, first_image.dtype, 3)

    compressor = bz2.BZ2Compressor()

    for i, f in enumerate(img_list):
        if i % 20 == 0:
            done = int((50.0 / len(img_list)) * i)
            update.emit('{} {}%'.format(scan_name, done))
        img_arr = reader.imread(f)
        rawdata = img_arr.T.tostring(order='F')
        tempnrrd.write(rawdata)

    BLOCK_SIZE = 52428800  # Around 20 MB in memory at one time
    # TODO: Check its smaller than image size
    compressed_name = outfile + '.bz2'

    file_size = os.fstat(tempnrrd.fileno()).st_size
    tempnrrd.seek(0)
    bytes_read = 0

    with open(compressed_name, 'wb') as fh_w:
        while True:
            block = tempnrrd.read(BLOCK_SIZE)
            bytes_read += BLOCK_SIZE
            done = int(50 + (50.0 / file_size) * bytes_read)
            if done >= 100:  # The getsize might not be accurate?
                done = 99
            update.emit('{} {}%'.format(scan_name, done))

            if not block:
                break
            compressed = compressor.compress(block)
            if compressed:

                try:
                    fh_w.write(compressed)
                except IOError:
                    update.emit("Error in compression - job terminated")
                    print('failed to write bzp chunk')
                    return

        # Send any data being buffered by the compressor
        remaining = compressor.flush()
        while remaining:
            to_send = remaining[:BLOCK_SIZE]
            remaining = remaining[BLOCK_SIZE:]

            fh_w.write(to_send)
Beispiel #24
0
    def compressstream(self, it, opts=None):
        opts = opts or {}
        z = bz2.BZ2Compressor(opts.get('level', 9))
        for chunk in it:
            data = z.compress(chunk)
            if data:
                yield data

        yield z.flush()
Beispiel #25
0
def test_resultstream(bzip2resultstream):
    """Test result stream serializer."""
    c = bz2.BZ2Compressor()
    c.compress(b'test 1')
    c.compress(b'test 2')
    data = c.flush()

    assert bzip2resultstream.read() == data
    assert bzip2resultstream.read() == b''
Beispiel #26
0
 def __init__(self, complevel=5, filename="file.log.bz2"):
     self.compressor = bz2.BZ2Compressor(complevel)
     self.stringlist = []
     self.outputopenedok = False
     self.filename = filename
     try:
         self.fh = open(self.filename, "wb")
         self.outputopenedok = True
     except:
         self.outputopenedok = False
Beispiel #27
0
def get_phone_number(data):
    comp = bz2.BZ2Compressor()
    decomp = bz2.BZ2Decompressor()
    unparsed = cookies_data.replace('+', ' ')
    parsed = urllib.parse.unquote_to_bytes(unparsed)
    print(f'data from urllib.parse: {parsed}')
    print(f'data from bz2.decomp: {bz2.decompress(parsed).decode()}')
    sp = xmlrpc.client.ServerProxy(
        'http://www.pythonchallenge.com/pc/phonebook.php')
    return sp.phone('Leopold')
Beispiel #28
0
def WriteGraph(graph, output_filename):
    """
    Write a graph to disk for later I/O access

    @param graph: the graph data structure to save to disk
    @param output_filename: the location to save the graph data structure
    """
    assert (output_filename.endswith('.graph.bz2'))

    # create a new compression object
    compressor = bz2.BZ2Compressor()

    # write the basic attributes for the graph to disk
    nvertices = graph.NVertices()
    nedges = graph.NEdges()
    directed = graph.directed
    colored = graph.colored
    prefix = graph.prefix

    # create an empty byte array which we will concatenate later
    compressed_graph = []

    compressed_graph.append(
        compressor.compress(
            struct.pack('qq??', nvertices, nedges, directed, colored)))
    compressed_graph.append(
        compressor.compress(struct.pack('128s', prefix.encode())))

    # write all of the vertices and their attributes
    for vertex in graph.vertices.values():
        compressed_graph.append(
            compressor.compress(
                struct.pack('qqqq', vertex.index, vertex.enumeration_index,
                            vertex.community, vertex.color)))

    # write all of the edges and their attributes
    for edge in graph.edges:
        compressed_graph.append(
            compressor.compress(
                struct.pack('qqd', edge.source_index, edge.destination_index,
                            edge.weight)))

    # flush the data
    compressed_graph.append(compressor.flush())

    # convert the array into a binary string - faster than native implementation
    compressed_graph = b''.join(compressed_graph)

    # write the compressed string to file
    with open(output_filename, 'wb') as fd:
        fd.write(compressed_graph)
Beispiel #29
0
def stream_write_bz2(ifh, ofh):
    '''Compress *ifh* into *ofh* using bz2 compression'''

    compr = bz2.BZ2Compressor(9)
    while True:
        buf = ifh.read(BUFSIZE)
        if not buf:
            break
        buf = compr.compress(buf)
        if buf:
            ofh.write(buf)
    buf = compr.flush()
    if buf:
        ofh.write(buf)
Beispiel #30
0
    def SendReport(self, xmldoc):
        if xmldoc.type != 'document_xml':
            raise Exception("Input is not XML document")

        fbuf = io.StringIO()
        xmlbuf = libxml2.createOutputBuffer(fbuf, 'UTF-8')
        doclen = xmldoc.saveFileTo(xmlbuf, 'UTF-8')

        compr = bz2.BZ2Compressor(9)
        cmpr = compr.compress(fbuf.getvalue())
        data = base64.b64encode(cmpr + compr.flush())
        ret = self.srv.SendReport(self.hostname, data)
        print("rtevalclient::SendReport() - Sent %i bytes (XML document length: %i bytes, compression ratio: %.02f%%)" % (len(data), doclen, (1-(float(len(data)) / float(doclen)))*100 ))
        return ret