Пример #1
0
    def run(self, obj, config):
        self.config = config
        self.obj = obj
        user = self.current_task.user
        data = io.BytesIO(obj.filedata.read())
        swf = bytearray()
        try:
            comp = data.read(3)
            header = data.read(5)
            if comp == 'CWS':
                swf = 'FWS' + header + zlib.decompress(data.read())
            if comp == 'ZWS':
                data.seek(12) # seek to LZMA props
                swf = 'FWS' + header + pylzma.decompress(data.read())
        except Exception as exc:
                self._error("unswf: (%s)." % exc)
                return

        if swf:
            h = md5(str(swf)).hexdigest()
            name = h
            if not user.has_access_to(SampleACL.WRITE):
                self._info("User does not have permission to add Samples to CRITs")
                self._add_result("Extract Canceled", "User does not have permission to add Samples to CRITs")
                return
                
            self._info("New file: %s (%d bytes, %s)" % (name, len(swf), h))
            handle_file(name, swf, self.obj.source,
                related_id=str(self.obj.id),
                related_type=str(self.obj._meta['crits_type']),
                campaign=self.obj.campaign,
                method=self.name,
                relationship=RelationshipTypes.RELATED_TO,
                user=self.current_task.user)
            self._add_result("file_added", name, {'md5': h})
Пример #2
0
 def _decompressSWF(f, swf_size):
     magic = f.read(3)
     if magic == "CWS":
         try:
             header = "FWS" + f.read(5)
             data = zlib.decompress(f.read())[:swf_size-8]
             return header + data 
         except (QuitScanException, GlobalScanTimeoutError, GlobalModuleTimeoutError):
             raise
         except Exception:
             return "ERROR"
         finally:
             logging.debug("extract_swf - closing stringio handle in decompress")
             f.close()
     elif magic == "ZWS":
         try:
             header = "FWS" + f.read(5)
             f.seek(12)
             data = pylzma.decompress(f.read())[:swf_size-8]
             return header + data
         except (QuitScanException, GlobalScanTimeoutError, GlobalModuleTimeoutError):
             raise
         except Exception:
             return "ERROR"
         finally:
             logging.debug("extract_swf - closing stringio handle in decompress")
             f.close()
     else:
         return None
Пример #3
0
 def test_compression_decompression_noeos(self):
     # call compression and decompression on random data of various sizes
     for i in range(18):
         size = 1 << i
         original = generate_random(size)
         result = pylzma.decompress(pylzma.compress(original, eos=0), maxlength=size)
         self.assertEqual(md5(original).hexdigest(), md5(result).hexdigest())
Пример #4
0
    def read(self, filename):
        if filename in self.nsis_header.files:
            data = None
            (foff, ftime, extract_type) = self.nsis_header.files[filename]

            if self.case_type == 1:  # case 1: 설치 파일 전부를 압축한 경우
                # print '#Case 1'
                # print hex(foff)
                # print hex(kavutil.get_uint32(self.body_data, foff) & 0x7fffffff)
                fsize = kavutil.get_uint32(self.body_data, foff) & 0x7fffffff
                return self.body_data[foff + 4:foff + 4 + fsize]
            elif self.case_type == 2:  # case 2: 개별로 압축한 경우
                # print '#Case 2'
                # print hex(foff)
                # print hex(kavutil.get_uint32(self.body_data, foff) & 0x7fffffff)
                fsize = kavutil.get_uint32(self.body_data, foff) & 0x7fffffff
                fdata = self.body_data[foff + 4:foff + 4 + fsize]
                comp_type = self.__get_comp_type(kavutil.get_uint32(fdata, 0))
                # print comp_type
                if comp_type == self.TYPE_LZMA:
                    try:  # 전체 압축한 경우인지 확인해 본다.
                        data = pylzma.decompress(fdata)
                    except TypeError:
                        pass
                elif comp_type == self.TYPE_ZLIB:
                    try:
                        data = zlib.decompress(fdata, -15)
                    except zlib.error:
                        pass
            return data
        else:
            return None
Пример #5
0
    def load(path_stack, compression="gzip"):
        """
        Load Stack instance from .stack file.

        Parameters
        ----------
        path_stack : str
            The full path to the .stack file that should be created, including
            the extension.
        compression : {'gzip', 'lzma'}, default 'gzip'
            The compression type that has been used saving the file.
        
        Returns
        -------
        None
        """

        if not path_stack.endswith('.stack'):
            raise ValueError(
                "To avoid ambiguity, when using Stack.load() you must provide the full path to "
                "the stack file you want to create, including the file extension. For example: "
                "stack.load(path_stack='./output/MyStack.stack'). Your call looks like this: "
                "stack.load(path_stack='%s', ...)" % (path_stack)
            )

        if compression is None:
            f = open(path_stack, 'rb')
        elif compression.lower() == "lzma":
            f = pylzma.decompress(open(path_stack, 'rb'))  # there seems to be a problem here!
        else:
            f = gzip.open(path_stack, 'rb')

        new_stack = cPickle.load(f)
        f.close()
        return new_stack
Пример #6
0
def unzlib(id):
    body, sz = get_response_and_size(id, "all")
    obj_num = -1
    name = ""
    if check_errors():
        return
    name = get_name(id)
    # decomp = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(body))
    # page = decomp.read()
    inData = body
    if inData[0] == 'C':
        # zlib SWF
        decompressData = zlib.decompress(inData[8:])
    elif inData[0] == 'Z':
        # lzma SWF
        decompressData = pylzma.decompress(inData[12:])
    elif inData[0] == 'F':
        # uncompressed SWF
        decompressData = inData[8:]
    else:
        print 'not a SWF file'
        return obj_num, name

    sigSize = struct.unpack("<I", inData[4:8])[0]
    decompressSize = len(decompressData) + 8
    if sigSize != decompressSize:
        print 'Length not correct, decompression failed'
    else:
        header = list(struct.unpack("<8B", inData[0:8]))
        header[0] = ord('F')
        page = struct.pack("<8B", *header) + decompressData

        obj_num = add_object("unzlib", page, id=id)
    return obj_num, name
Пример #7
0
    def read(self, filename):
        if filename in self.nsis_header.files:
            data = None
            (foff, ftime, extract_type) = self.nsis_header.files[filename]

            if self.case_type == 1:  # case 1: 설치 파일 전부를 압축한 경우
                # print '#Case 1'
                # print hex(foff)
                # print hex(kavutil.get_uint32(self.body_data, foff) & 0x7fffffff)
                fsize = kavutil.get_uint32(self.body_data, foff) & 0x7fffffff
                return self.body_data[foff+4:foff+4+fsize]
            elif self.case_type == 2:  # case 2: 개별로 압축한 경우
                # print '#Case 2'
                # print hex(foff)
                # print hex(kavutil.get_uint32(self.body_data, foff) & 0x7fffffff)
                fsize = kavutil.get_uint32(self.body_data, foff) & 0x7fffffff
                fdata = self.body_data[foff+4:foff+4+fsize]
                comp_type = self.__get_comp_type(kavutil.get_uint32(fdata, 0))
                # print comp_type
                if comp_type == self.TYPE_LZMA:
                    try:  # 전체 압축한 경우인지 확인해 본다.
                        data = pylzma.decompress(fdata)
                    except TypeError:
                        pass
                elif comp_type == self.TYPE_ZLIB:
                    try:
                        data = zlib.decompress(fdata, -15)
                    except zlib.error:
                        pass
            return data
        else:
            return None
Пример #8
0
def unpack(data):
    """
    Unpack data from ddsx and returns it. If data have wrong header, prints error
    and exit(1). Return unpacked dds data, ready for saving.

    :param data: ddsx data
    """
    header_format = struct.unpack_from('4s', data, 0x4)[0]
    if header_format not in ddsx_types:
        print 'wrong ddsx type:', header_format
        exit(1)

    dds_height = struct.unpack_from('H', data, 0xc)[0]
    dds_width = struct.unpack_from('H', data, 0xe)[0]
    dds_mipmapcount = struct.unpack_from('B', data, 0x10)[0]
    dds_unpacked_body_size = struct.unpack_from('I', data, 0x18)[0]
    dds_body_size = struct.unpack_from('I', data, 0x1c)[0]
    ddsx_unknown_flag_0 = struct.unpack_from('B', data, 0xa)
    if ddsx_unknown_flag_0 in [0, 1]:
        pass  # all unpack ok 11 c0 01 40, 11 40 01 40, 11 40 00 40

    dds_data = ctypes.create_string_buffer(0x80)
    struct.pack_into('128B', dds_data, 0, *dds_header)
    struct.pack_into('I', dds_data, 0xc, dds_width)
    struct.pack_into('I', dds_data, 0x10, dds_height)
    struct.pack_into('I', dds_data, 0x14, dds_unpacked_body_size)
    struct.pack_into('B', dds_data, 0x1c, dds_mipmapcount)
    struct.pack_into('4s', dds_data, 0x54, header_format)

    if dds_body_size == 0:  # not packed
        return dds_data.raw + data[0x20:]
    elif struct.unpack_from('I', data, 0x20)[0] == 0x1000005d:  # packed with lzma
        return dds_data.raw + pylzma.decompress(data[0x20:], maxlength=dds_unpacked_body_size)
    else:  # packed with zlib
        return dds_data.raw + zlib.decompress(data[0x20:])
Пример #9
0
def unzip(inData):
    if inData[0] == 'C':
        # zlib SWF
        debug('zlib compressed swf detected.')
        decompressData = zlib.decompress(inData[8:])
    elif inData[0] == 'Z':
        # lzma SWF
        debug('lzma compressed swf detected.')
        decompressData = pylzma.decompress(inData[12:])
    elif inData[0] == 'F':
        # uncompressed SWF
        debug('Uncompressed swf detected.')
        decompressData = inData[8:]
    else:
        exit('not a SWF file')

    sigSize = struct.unpack("<I", inData[4:8])[0]
    debug('Filesize in signature: %s' % sigSize)

    decompressSize = len(decompressData) + 8
    debug('Filesize decompressed: %s' % decompressSize)

    check((sigSize == decompressSize),
          'Length not correct, decompression failed')
    header = list(struct.unpack("<8B", inData[0:8]))
    header[0] = ord('F')

    debug('Generating uncompressed data')
    return struct.pack("<8B", *header) + decompressData
Пример #10
0
 def parse(self, data):
     """ 
     Parses the SWF.
     
     The @data parameter can be a file object or a SWFStream
     """
     self._data = data = data if isinstance(data,
                                            SWFStream) else SWFStream(data)
     self._header = SWFHeader(self._data)
     if self._header.compressed:
         temp = StringIO.StringIO()
         if self._header.compressed_zlib:
             import zlib
             data = data.f.read()
             zip = zlib.decompressobj()
             temp.write(zip.decompress(data))
         else:
             import pylzma
             data.readUI32()  #consume compressed length
             data = data.f.read()
             temp.write(pylzma.decompress(data))
         temp.seek(0)
         data = SWFStream(temp)
     self._header._frame_size = data.readRECT()
     self._header._frame_rate = data.readFIXED8()
     self._header._frame_count = data.readUI16()
     self.parse_tags(data)
Пример #11
0
 def parse(self, data):
     """ 
     Parses the SWF.
     
     The @data parameter can be a file object or a SWFStream
     """
     self._data = data = data if isinstance(data, SWFStream) else SWFStream(data)
     self._header = SWFHeader(self._data)
     if self._header.compressed:
         temp = StringIO.StringIO()
         if self._header.compressed_zlib:
             import zlib
             data = data.f.read()
             zip = zlib.decompressobj()
             temp.write(zip.decompress(data))
         else:
             import pylzma
             data.readUI32() #consume compressed length
             data = data.f.read()
             temp.write(pylzma.decompress(data))
         temp.seek(0)
         data = SWFStream(temp)
     self._header._frame_size = data.readRECT()
     self._header._frame_rate = data.readFIXED8()
     self._header._frame_count = data.readUI16()
     self.parse_tags(data)
def unzip(inData):
    if inData[0] == 'C':
        # zlib SWF
        debug('zlib compressed swf detected.')
        decompressData = zlib.decompress(inData[8:])
    elif inData[0] == 'Z':
        # lzma SWF
        debug('lzma compressed swf detected.')
        decompressData = pylzma.decompress(inData[12:])
    elif inData[0] == 'F':
        # uncompressed SWF
        debug('Uncompressed swf detected.')
        decompressData = inData[8:]
    else:
        print('not a SWF file')

    sigSize = struct.unpack("<I", inData[4:8])[0]
    debug('Filesize in signature: %s' % sigSize)

    decompressSize = len(decompressData) +8
    debug('Filesize decompressed: %s' % decompressSize)

    check((sigSize == decompressSize), 'Length not correct, decompression failed')
    header = list(struct.unpack("<8B", inData[0:8]))
    header[0] = ord('F')

    debug('Generating uncompressed data')
    return struct.pack("<8B", *header)+decompressData
Пример #13
0
    def load(path_stack, compression="gzip"):
        """
        Load Stack instance from .stack file.

        Parameters
        ----------
        path_stack : str
            The full path to the .stack file that should be created, including
            the extension.
        compression : {'gzip', 'lzma'}, default 'gzip'
            The compression type that has been used saving the file.
        
        Returns
        -------
        None
        """

        if not path_stack.endswith('.stack'):
            raise ValueError(
                "To avoid ambiguity, when using Stack.load() you must provide the full path to "
                "the stack file you want to create, including the file extension. For example: "
                "stack.load(path_stack='./output/MyStack.stack'). Your call looks like this: "
                "stack.load(path_stack='%s', ...)" % (path_stack))

        if compression is None:
            f = open(path_stack, 'rb')
        elif compression.lower() == "lzma":
            f = pylzma.decompress(open(
                path_stack, 'rb'))  # there seems to be a problem here!
        else:
            f = gzip.open(path_stack, 'rb')

        new_stack = cPickle.load(f)
        f.close()
        return new_stack
Пример #14
0
def unzlib(id):
    body, sz = get_response_and_size(id, "all")
    obj_num = -1
    name = ""
    if check_errors():
        return
    name = get_name(id)
    # decomp = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(body))
    # page = decomp.read()
    inData = body
    if inData[0] == 'C':
        # zlib SWF
        decompressData = zlib.decompress(inData[8:])
    elif inData[0] == 'Z':
        # lzma SWF
        decompressData = pylzma.decompress(inData[12:])
    elif inData[0] == 'F':
        # uncompressed SWF
        decompressData = inData[8:]
    else:
        print 'not a SWF file'
        return obj_num, name

    sigSize = struct.unpack("<I", inData[4:8])[0]
    decompressSize = len(decompressData) +8
    if sigSize != decompressSize:
        print 'Length not correct, decompression failed'
    else:
        header = list(struct.unpack("<8B", inData[0:8]))
        header[0] = ord('F')
        page = struct.pack("<8B", *header)+decompressData

        obj_num = add_object("unzlib",page,id=id)
    return obj_num, name
Пример #15
0
def unpackHistogramCollection(buf, decode=True, decompress=True):
	"""
	Unpack a collection of histograms
	"""

	# Dencode if asked
	if decode:
		buf = base64.b64decode(buf)
	# Deompress if asked
	if decompress:
		buf = pylzma.decompress(buf)

	# Read header
	(ver, numHistograms) = struct.unpack("<BI", buf[:5])
	p = 5

	# Start piling histograms
	hc = HistogramCollection()
	for i in range(0,numHistograms):

		# Read histogram and offset position
		(histo, p) = unpackHistogram(buf,p)

		# Append histogram in collection
		hc.append( histo )

	# Return collection
	return hc
Пример #16
0
    def run(self, obj, config):
        self.config = config
        self.obj = obj
        data = io.BytesIO(obj.filedata.read())
        swf = bytearray()
        try:
            comp = data.read(3)
            header = data.read(5)
            if comp == 'CWS':
                swf = 'FWS' + header + zlib.decompress(data.read())
            if comp == 'ZWS':
                data.seek(12) # seek to LZMA props
                swf = 'FWS' + header + pylzma.decompress(data.read())
        except Exception as exc:
                self._error("unswf: (%s)." % exc)
                return

        if swf:
            h = md5(str(swf)).hexdigest()
            name = h
            self._info("New file: %s (%d bytes, %s)" % (name, len(swf), h))
            handle_file(name, swf, self.obj.source,
                related_id=str(self.obj.id),
                related_type=str(self.obj._meta['crits_type']),
                campaign=self.obj.campaign,
                method=self.name,
                relationship=RelationshipTypes.RELATED_TO,
                user=self.current_task.username)
            self._add_result("file_added", name, {'md5': h})
Пример #17
0
    def run(self, obj, config):
        self.config = config
        self.obj = obj
        data = io.BytesIO(obj.filedata.read())
        swf = bytearray()
        try:
            comp = data.read(3)
            header = data.read(5)
            if comp == 'CWS':
                swf = 'FWS' + header + zlib.decompress(data.read())
            if comp == 'ZWS':
                data.seek(12) # seek to LZMA props
                swf = 'FWS' + header + pylzma.decompress(data.read())
        except Exception as exc:
                self._error("unswf: (%s)." % exc)
                return

        if swf:
            h = md5(str(swf)).hexdigest()
            name = h
            self._info("New file: %s (%d bytes, %s)" % (name, len(swf), h))
            handle_file(name, swf, self.obj.source,
                related_id=str(self.obj.id),
                campaign=self.obj.campaign,
                method=self.name,
                relationship='Related_To',
                user=self.current_task.username)
            self._add_result("file_added", name, {'md5': h})
Пример #18
0
 def __getitem__(self, key):
     from vyperlogix.crypto.Encryptors import Encryptors
     try:
         val = self.__db[key]
         if (not self.isPickleMethodUseSafeSerializer):
             if (self.isPickleMethodUseZLIB):
                 val = zlib.decompress(val)
             elif (self.isPickleMethodUseLZMA):
                 val = pylzma.decompress(val)
         if (self.isPickleMethodUseStrings):
             val = val.split(',')
             if (len(val) > 1):
                 val = [
                     tuple(x.split('|')) if x.find('|') > -1 else x
                     for x in val
                 ]
         elif (self.isPickleMethodUseBsdDbShelf):
             val = self.unPickleItem(val)
         elif (self.isPickleMethodUseMarshal):
             val = marshal.loads(hexToStr(val))
         elif (self.isPickleMethodUseSafeSerializer):
             try:
                 val = loads(val, beSilent=True)
             except:
                 pass
             if (isinstance(val, dict)):
                 d_val = HashedLists2()
                 d_val.fromDict(val)
                 val = d_val
         elif (self.isPickleMethodUseCerealizer):
             val = cerealizer.loads(val)
     except Exception as details:
         val = 'UNKNOWN value for key (%s) of type "%s" due to ERROR "%s".' % (
             key, str(key.__class__), str(details))
     return val
Пример #19
0
	def uncompress(self, filename=None, data=None):
		# if data has already been uncompressed, return it
		if hasattr(self, '__data__'):
			return self.__data__

		if filename == None and data == None:
			return None

		if not filename == None:
			self.__data__ = open(filename, 'rb').read()
		else:
			self.__data__ = data

		if self.__data__[:3] == 'FWS':
			self.compressed = False
			return self.__data__
		if self.__data__[:3] == 'ZWS':
			self.compressed = True
			rawdata = pylzma.decompress(self.__data__[12:])
		elif self.__data__[:3] == 'CWS':
			self.compressed = True
			rawdata = zlib.decompress(self.__data__[8:])
		else:
			raise SwfFormatError('Unexpected magic string, not a Flash file.')
			
		
		swfdata = 'FWS' + self.__data__[3] + pack('I', len(rawdata) + 8) + rawdata

		return swfdata
Пример #20
0
 def _decompressSWF(f, swf_size):
     magic = f.read(3)
     if magic == "CWS":
         try:
             header = "FWS" + f.read(5)
             data = zlib.decompress(f.read())[:swf_size - 8]
             return header + data
         except (QuitScanException, GlobalScanTimeoutError,
                 GlobalModuleTimeoutError):
             raise
         except Exception:
             return "ERROR"
         finally:
             logging.debug(
                 "extract_swf - closing stringio handle in decompress")
             f.close()
     elif magic == "ZWS":
         try:
             header = "FWS" + f.read(5)
             f.seek(12)
             data = pylzma.decompress(f.read())[:swf_size - 8]
             return header + data
         except (QuitScanException, GlobalScanTimeoutError,
                 GlobalModuleTimeoutError):
             raise
         except Exception:
             return "ERROR"
         finally:
             logging.debug(
                 "extract_swf - closing stringio handle in decompress")
             f.close()
     else:
         return None
Пример #21
0
 def test_compression_decompression_eos(self):
     # call compression and decompression on random data of various sizes
     for i in xrange(18):
         size = 1 << i
         original = generate_random(size)
         result = pylzma.decompress(pylzma.compress(original, eos=1))
         self.assertEqual(len(result), size)
         self.assertEqual(md5.new(original).hexdigest(), md5.new(result).hexdigest())
Пример #22
0
def decom_lzma_algm(file_name):
    import pylzma
    outfilename = file_name + '.decomp'
    try:
        with open(file_name, 'rb') as f, open(outfilename, 'wb') as out:
            out.write(pylzma.decompress(f.read()))
    finally:
        out.close()
Пример #23
0
def unLZMAzipfile(file):
    f = open(file, 'rb+')
    line = f.read()
    result = pylzma.decompress(line)
    f.seek(0)
    f.truncate()
    f.write(result)
    f.close()
Пример #24
0
	def _read_config(self):
		"""Fetch the device configuration descriptor from the device."""
		#Is the crc necessary here?
		self._ser.write(b'\\#\x00\x00\x00\x00')
		(clen,) = struct.unpack(">H", self._my_ser_read(2))
		cbytes = self._my_ser_read(clen)
		self._my_ser_read(2) #read and ignore the not-yet-crc
		return json.JSONDecoder().decode(str(pylzma.decompress(cbytes), "UTF-8"))
Пример #25
0
def lzma_decompress(data):
    """
        LZMA decompression using pylzma.
        The LZMA header consists of 5 + 8 bytes.
        The first 5 bytes are compression parameters, the 8 following bytes specify
        the length of the data.
    """
    # unpack the data length from the LZMA header (bytes # 6-13 inclusively/unsigned long long int)
    coded_length = struct.unpack('<Q', data[5:13])[0]
    if coded_length == '\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF':
        # if the length is -1 (in two's complement), there is an EOS marker
        # marking the end of the data and pylzma doesn't need the coded_length
        return pylzma.decompress(data[:5] + data[13:])
    else:
        # if the length is specified, pylzma needs the coded_length since there probably
        # is no EOS marker
        return pylzma.decompress(data[:5] + data[13:], maxlength=coded_length)
Пример #26
0
def lzma_decompress(data):
    """
        LZMA decompression using pylzma.
        The LZMA header consists of 5 + 8 bytes.
        The first 5 bytes are compression parameters, the 8 following bytes specify
        the length of the data.
    """
    # unpack the data length from the LZMA header (bytes # 6-13 inclusively/unsigned long long int)
    coded_length = struct.unpack('<Q', data[5:13])[0]
    if coded_length == '\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF':
        # if the length is -1 (in two's complement), there is an EOS marker
        # marking the end of the data and pylzma doesn't need the coded_length
        return pylzma.decompress(data[:5] + data[13:])
    else:
        # if the length is specified, pylzma needs the coded_length since there probably
        # is no EOS marker
        return pylzma.decompress(data[:5] + data[13:], maxlength=coded_length)
Пример #27
0
 def run(self):
   if self.compresslib=="lzma":
     self.datadecompressed=pylzma.decompress(self.data)
   elif self.compresslib=="zlib":
     self.datadecompressed=zlib.decompress(self.data)
   elif self.compresslib=="bz2":
     self.datadecompressed=bz2.decompress(self.data)
   elif self.compresslib=="none":
     self.datadecompressed=self.data
Пример #28
0
def checkCompression(file):
    con = dbapi2.connect(file)
    con.text_factory = str

    cur = con.execute('SELECT id, contents FROM articles')
    for id, contents in cur:
        #print contents
        dummy = decompress(contents)
        print dummy
Пример #29
0
def finish():
    time.sleep(0.5)
    try:
        decompressed = pylzma.decompress(working_on)
    except TypeError, e:
        save_to.close()
        rospy.logerr("Failed to decompress file: {}".format(e.message))
        saved.publish("")
        return False
Пример #30
0
 def run(self):
     if self.compresslib == "lzma":
         self.datadecompressed = pylzma.decompress(self.data)
     elif self.compresslib == "zlib":
         self.datadecompressed = zlib.decompress(self.data)
     elif self.compresslib == "bz2":
         self.datadecompressed = bz2.decompress(self.data)
     elif self.compresslib == "none":
         self.datadecompressed = self.data
Пример #31
0
def checkCompression(file):
  con=dbapi2.connect(file)
  con.text_factory=str

  cur=con.execute('SELECT id, contents FROM articles')
  for id, contents in cur:
    #print contents
    dummy=decompress(contents)
    print dummy
Пример #32
0
 def test_matchfinders(self):
     # use different matchfinder algorithms for compression
     matchfinders = ['bt2', 'bt3', 'hc4']
     original = 'hello world'
     for mf in matchfinders:
         result = pylzma.decompress(pylzma.compress(original, matchfinder=mf))
         self.assertEqual(original, result)
         
     self.failUnlessRaises(TypeError, pylzma.compress, original, matchfinder='1234')
Пример #33
0
def pickleLoad(fname):
    '''
    General routine for restoring pickled and lzma-compressed data.
    '''
    with open(fname, 'rb') as f:
        pickled = f.read()
        pickled = pylzma.decompress(pickled)
        df = cPickle.loads(pickled)
    return df
Пример #34
0
 def test_compression_decompression_eos(self):
     # call compression and decompression on random data of various sizes
     for i in range(18):
         size = 1 << i
         original = generate_random(size)
         result = pylzma.decompress(pylzma.compress(original, eos=1))
         self.assertEqual(len(result), size)
         self.assertEqual(
             md5(original).hexdigest(),
             md5(result).hexdigest())
Пример #35
0
 def decompress(flag, compressed):
     if flag == CFLCOMPRESS_NONE:
         return compressed
     if flag == CFLCOMPRESS_LZMA:
         try:
             return pylzma.decompress(compressed)
         except (TypeError, ValueError) as e:
             raise InvalidCFLError(e)
     else:
         raise InvalidCFLError('Unsupported flag %r' % (flag,))
Пример #36
0
 def test_compression_decompression_noeos(self):
     # call compression and decompression on random data of various sizes
     for i in xrange(18):
         size = 1 << i
         original = generate_random(size)
         result = pylzma.decompress(pylzma.compress(original, eos=0),
                                    maxlength=size)
         self.assertEqual(
             md5.new(original).hexdigest(),
             md5.new(result).hexdigest())
Пример #37
0
 def updateDB(self, updatedDB, type):
     from time import time
     buffer = pylzma.decompress(updatedDB)
     f = file("./%s/data/%s"%(GAMEROOT,self.auctionDB),"wb")
     f.write(buffer)
     f.close()
     self.lastUpdate = time()
     if (type == 1): #refresh search results
         self.Search(100)
     else: #refresh selling list results
         self.getCharacterSellList(100)
Пример #38
0
 def test_compression_file(self):
     # test compressing from file-like object (C class)
     infile = BytesIO(self.plain)
     outfile = BytesIO()
     compress = pylzma.compressfile(infile, eos=1)
     while 1:
         data = compress.read(1)
         if not data: break
         outfile.write(data)
     check = pylzma.decompress(outfile.getvalue())
     self.assertEqual(check, self.plain)
Пример #39
0
def decode_post(data):
	js = pylzma.decompress(data).decode("utf-8")
	np = json.loads(js)
	mape = dict((v,k) for k,v in __pmap().items())
	post = dict()
	for k,v in np.items():
		nk = k
		if(k in mape):
			nk = mape[k]
		post[nk] = v
	return post
Пример #40
0
 def test_compression_file_python(self):
     # test compressing from file-like object (Python class)
     infile = PyStringIO(self.plain)
     outfile = PyStringIO()
     compress = pylzma.compressfile(infile, eos=1)
     while 1:
         data = compress.read(1)
         if not data: break
         outfile.write(data)
     check = pylzma.decompress(outfile.getvalue())
     self.assertEqual(check, self.plain)
Пример #41
0
 def updateDB(self, updatedDB, type):
     from time import time
     buffer = pylzma.decompress(updatedDB)
     f = file("./%s/data/%s" % (GAMEROOT, self.auctionDB), "wb")
     f.write(buffer)
     f.close()
     self.lastUpdate = time()
     if (type == 1):  #refresh search results
         self.Search(100)
     else:  #refresh selling list results
         self.getCharacterSellList(100)
Пример #42
0
 def get(self, request, format=None):
     filename = request.GET['filename']
     image = Images.objects.filter(filename=filename).first()
     if image is None:
         return Response({'detail': 'Invalid Filename'}, status=404)
     namef = open(settings.IMAGES_ROOT + '/' + filename + '.' + 'compr',
                  'r+')
     filecontent = namef.read()
     uncomp = pylzma.decompress(filecontent)
     ext = filename.rsplit('.', 1)[1].lower()
     return HttpResponse(uncomp, content_type='image/' + ext)
 def uncompress_lzma(self, data):
     """uncompress lzma compressed stream"""
     if not self.lzma_install:
         if self.show_errors:
             print("\t[ERROR] pylzma module not installed - aborting validation/decompression")
         return None
     else:
         data = data[4:]
         try:
             return pylzma.decompress(data)
         except Exception:
             return None
Пример #44
0
def LzmaDec(cmpData, orgDataSize, orgBinSize):
    # cut binary data tail & decompress
    try:
        orgData = decompress(cmpData)[:orgDataSize]
    except:
        GlobalMsg.panic('LZMA internal error, decompression failed')

    # cut bin string tail
    binStr = BitArray(bytes = orgData).bin[2:2+orgBinSize]

    # return original bin string
    return binStr
Пример #45
0
def decompress_json(data):
    """Replaces all short abbreviations with real field names"""
    try:
        data = pylzma.decompress( base64.urlsafe_b64decode( str(data) ) )
    except (py7zlib.ArchiveError, TypeError):
        return
    else:
        #Extend field names
        for full_field, cut_field in COMPRESS_MAPPING:
            data = data.replace(cut_field, full_field)

    return data 
Пример #46
0
 def test_compression_file_python(self):
     # test compressing from file-like object (Python class)
     from StringIO import StringIO as PyStringIO
     infile = PyStringIO(self.plain)
     outfile = PyStringIO()
     compress = pylzma.compressfile(infile, eos=1)
     while 1:
         data = compress.read(1)
         if not data: break
         outfile.write(data)
     check = pylzma.decompress(outfile.getvalue())
     self.assertEqual(check, self.plain)
Пример #47
0
 def _test_compression_streaming(self):
     # test compressing with one byte at a time...
     # XXX: disabled as LZMA doesn't support streaming compression yet
     compress = pylzma.compressobj(eos=1)
     infile = BytesIO(self.plain)
     outfile = BytesIO()
     while 1:
         data = infile.read(1)
         if not data: break
         outfile.write(compress.compress(data, 1))
     outfile.write(compress.flush())
     check = pylzma.decompress(outfile.getvalue())
     self.assertEqual(check, self.plain)
Пример #48
0
 def _test_compression_streaming(self):
     # test compressing with one byte at a time...
     # XXX: disabled as LZMA doesn't support streaming compression yet
     compress = pylzma.compressobj(eos=1)
     infile = BytesIO(self.plain)
     outfile = BytesIO()
     while 1:
         data = infile.read(1)
         if not data: break
         outfile.write(compress.compress(data, 1))
     outfile.write(compress.flush())
     check = pylzma.decompress(outfile.getvalue())
     self.assertEqual(check, self.plain)
Пример #49
0
    def get(self):
        '''
        读取解压数据并返回使用时间

        'return'第一个为数据,第二个为时间
        '''
        logging.info('start decompressing')
        start_time=time.time()
        data=pylzma.decompress(self.__file)
        end_time=time.time()
        use_time=end_time-start_time
        logging.info('decompress complete,using time:{}'.format(use_time))
        return data,use_time
Пример #50
0
 def uncompress_lzma(self, data):
     'uncompress lzma compressed stream'
     if self.lzma_installed == False:
         if self.show_errors:
             print "\t[ERROR] pylzma module not installed - aborting validation/decompression"
         return None
     else:
         data = data[4:]
         try:
             import pylzma
             return pylzma.decompress(data)
         except:
             return None
Пример #51
0
 def uncompress_lzma(self, data):
     'uncompress lzma compressed stream'
     if self.lzma_installed == False:
         if self.show_errors:
             print "\t[ERROR] pylzma module not installed - aborting validation/decompression"
         return None
     else:
         data = data[4:]
         try:
             import pylzma
             return pylzma.decompress(data)
         except:
             return None
Пример #52
0
 def _test_compression_streaming(self):
     # XXX: this doesn't work, yet
     # test compressing with one byte at a time...
     compress = pylzma.compressobj(eos=1)
     infile = StringIO(self.plain)
     outfile = StringIO()
     while 1:
         data = infile.read(1)
         if not data: break
         outfile.write(compress.compress(data, 1))
     outfile.write(compress.flush())
     check = pylzma.decompress(outfile.getvalue())
     self.assertEqual(check, self.plain)
Пример #53
0
    def test_matchfinders(self):
        # use different matchfinder algorithms for compression
        matchfinders = ['bt2', 'bt3', 'hc4']
        original = 'hello world'
        for mf in matchfinders:
            result = pylzma.decompress(
                pylzma.compress(original, matchfinder=mf))
            self.assertEqual(original, result)

        self.failUnlessRaises(TypeError,
                              pylzma.compress,
                              original,
                              matchfinder='1234')
Пример #54
0
 def _test_compression_streaming(self):
     # XXX: this doesn't work, yet
     # test compressing with one byte at a time...
     compress = pylzma.compressobj(eos=1)
     infile = StringIO(self.plain)
     outfile = StringIO()
     while 1:
         data = infile.read(1)
         if not data: break
         outfile.write(compress.compress(data, 1))
     outfile.write(compress.flush())
     check = pylzma.decompress(outfile.getvalue())
     self.assertEqual(check, self.plain)
    def decompress_lzma(self):

        self.flash.seek(12)
        self.lzma_decompressed = pylzma.decompress(self.flash.read1(self.file_factor))

        c_lzma_file = io.BytesIO()
        c_lzma_file.write(b'FWS')
        c_lzma_file.write(struct.pack("<B", self.version))
        c_lzma_file.write(struct.pack("<I", self.file_size))
        c_lzma_file.write(self.lzma_decompressed)

        c_lzma_file.seek(0)

        return c_lzma_file
Пример #56
0
def EXTRACT_SWF(s, buff):

   SWF = {}

   magic = buff[:3]
   data = ''

   if magic == 'CWS':
      SWF['Buffer'] = 'FWS' + buff[3:8] + zlib.decompress(buff[8:])
   elif magic == 'ZWS':
      SWF['Buffer'] = 'FWS' + buff[3:8] + pylzma.decompress(buff[12:])
   elif magic == 'FWS':
      SWF['Version'] = ord(buff[3])

   return SWF
 def ConvertTextMessageToMavlink(self, TextMessage):
     ######################################################################################
     #
     #  Summary:  Takes a text message's payload text, unBase64's it, decompresses it with LZMA,
     #  parses out the multiple Mavlink messages within the buffer, and a returns a list of
     #  decoded Mavlink messages.  Base64 is used to make the text buffer url/sms/email safe.
     #
     ######################################################################################
     try:
         DecodedMavlinkBuffer = base64.b64decode(TextMessage)
         DecompressedMavlinkBuffer = pylzma.decompress(DecodedMavlinkBuffer)
         ListOfMavlinkMessages = self._MavlinkHelperObject.parse_buffer(DecompressedMavlinkBuffer)
         return ListOfMavlinkMessages
     except Exception, e:
         self.Logger("Exception in ConvertTextMessagetoMavlink: "+str(e), message_importance=1)
         return None
Пример #58
0
 def _file_decode(cls, data):
     """{z,b,l}{j,p}<data_len><pad><DATA>"""
     alg = data[0]
     header = data[1]
     idx = data.find(cls.PADDING)
     size = int(data[2:idx])
     data = data[idx + 1:idx + 1 + size]
     if alg == cls.GZIP:
         data = data.decode('zlib')
     elif alg == cls.BZ2:
         data = data.decode('bz2')
     elif alg == cls.LZMA:
         data = pylzma.decompress(data)
     if header == cls.JSON:
         data = json.loads(data)
     if header == cls.PICKLE:
         data = pickle.loads(data)
     return data