def import_data( self, buffer, parent=None ): assert utils.is_bytes( buffer ) result = [] buf_out = [] i = 0 while i < len( buffer ): # 0x00 <= n < 0x80: copy next n+1 bytes to output stream if buffer[i] in range( 0x00, 0x80 ): count = buffer[i]+1 buf_out.append( buffer[i+1:i+1+count] ) i += count+1 # n == 0x80: end of segment elif buffer[i] == 0x80: product = b''.join( buf_out ) if len( product ) != self.DECOMPRESSED_SIZE: logger.warning( '{}: was expecting {} bytes of data, got {}'.format( self, self.DECOMPRESSED_SIZE, len( product ) ) ) result.append( product ) buf_out = [] i += 1 # 0x81 <= n < 0xff: repeat next byte (257-n) times else: count = 257-buffer[i] buf_out.append( buffer[i+1:i+2]*count ) i += 2 if buf_out: logger.warning( '{}: EOF reached before last RLE block closed'.format( self ) ) result.append( b''.join( buf_out ) ) # result is a 960x160 3bpp image, divided into 4x 40 scanline segments unpack = (self.plan.import_data( x ).payload for x in result) return mrc.TransformResult( payload=bytes( itertools.chain( *unpack ) ), end_offset=i )
def export_data( self, buffer, parent=None ): assert utils.is_bytes( buffer ) assert len( buffer ) == 960*160 segments = (buffer[960*40*i:960*40*(i+1)] for i in range(4)) segments = (self.plan.export_data( x ).payload for x in segments) result = bytearray() for segment in segments: pointer = 0 while pointer < len( segment ): start = pointer end = pointer+1 if end >= len( segment ): result.append( 0x00 ) result.append( segment[start] ) pointer += 1 elif segment[end] == segment[start]: while ((end+1) < len( segment )) and (segment[end+1] == segment[end]) and (end-start < 127): end += 1 result.append( 257-(end+1-start) ) result.append( segment[start] ) pointer = end+1 else: while ((end+1) < len( segment )) and (segment[end+1] != segment[end]) and (end-1-start < 128): end += 1 result.append( end-1-start ) result.extend( segment[start:end] ) pointer = end result.append( 0x80 ) return mrc.TransformResult( payload=bytes( result ) )
def import_data( self, buffer, parent=None ): assert utils.is_bytes( buffer ) pointer = 0 result = bytearray() while pointer < len( buffer ): test = buffer[pointer] pointer += 1 if test & 0x80: result += buffer[pointer:pointer+(test & 0x7f)] pointer += (test & 0x7f) else: result += buffer[pointer:pointer+1]*test pointer += 1 return mrc.TransformResult( payload=bytes( result ), end_offset=pointer )
def import_data( self, buffer, parent=None ): assert utils.is_bytes( buffer ) inline_copy_bits = buffer[0] high_priority_flag = mrc.Bits( 0x01, 0b00010000 ).get_from_buffer( buffer ) palette_offset = mrc.Bits( 0x01, 0b00001100 ).get_from_buffer( buffer ) flip_horiz = mrc.Bits( 0x01, 0b00000010 ).get_from_buffer( buffer ) flip_vert = mrc.Bits( 0x01, 0b00000001 ).get_from_buffer( buffer ) incremental_copy = mrc.UInt16_BE( 0x02 ).get_from_buffer( buffer ) literal_copy = mrc.UInt16_BE( 0x04 ).get_from_buffer( buffer ) bs = BitStream( buffer[0x06:], 0, bits_reverse=True ) output = bytearray()
def import_data(self, buffer, parent=None): assert utils.is_bytes(buffer) pointer = 0 result = bytearray() while pointer < len(buffer): test = buffer[pointer] pointer += 1 if test & 0x80: result += buffer[pointer:pointer + (test & 0x7f)] pointer += (test & 0x7f) else: result += buffer[pointer:pointer + 1] * test pointer += 1 return mrc.TransformResult(payload=bytes(result), end_offset=pointer)
def get_from_buffer(self, buffer, parent=None): assert utils.is_bytes(buffer) offset = mrc.property_get(self.offset, parent) pointer = offset total = 0 shift = 0 while pointer < len(buffer): test = buffer[pointer] pointer += 1 total += (test & 0x7f) << shift shift += 7 if test & 0x80: break total += 1 << shift return total
def get_from_buffer( self, buffer, parent=None ): assert utils.is_bytes( buffer ) offset = mrc.property_get( self.offset, parent ) pointer = offset total = 0 shift = 0 while pointer < len( buffer ): test = buffer[pointer] pointer += 1 total += (test & 0x7f) << shift shift += 7 if test & 0x80: break total += 1 << shift return total
def export_data(self, buffer, parent=None): assert utils.is_bytes(buffer) assert len(buffer) == 960 * 160 segments = (buffer[960 * 40 * i:960 * 40 * (i + 1)] for i in range(4)) segments = (self.plan.export_data(x).payload for x in segments) result = bytearray() for segment in segments: pointer = 0 while pointer < len(segment): start = pointer end = pointer + 1 if end >= len(segment): result.append(0x00) result.append(segment[start]) pointer += 1 elif segment[end] == segment[start]: while ((end + 1) < len(segment)) and ( segment[end + 1] == segment[end]) and (end - start < 127): end += 1 result.append(257 - (end + 1 - start)) result.append(segment[start]) pointer = end + 1 else: while ((end + 1) < len(segment)) and ( segment[end + 1] != segment[end]) and (end - 1 - start < 128): end += 1 result.append(end - 1 - start) result.extend(segment[start:end]) pointer = end result.append(0x80) return mrc.TransformResult(payload=bytes(result))
def import_data(self, buffer, parent=None): assert utils.is_bytes(buffer) result = [] buf_out = [] i = 0 while i < len(buffer): # 0x00 <= n < 0x80: copy next n+1 bytes to output stream if buffer[i] in range(0x00, 0x80): count = buffer[i] + 1 buf_out.append(buffer[i + 1:i + 1 + count]) i += count + 1 # n == 0x80: end of segment elif buffer[i] == 0x80: product = b''.join(buf_out) if len(product) != self.DECOMPRESSED_SIZE: logger.warning( '{}: was expecting {} bytes of data, got {}'.format( self, self.DECOMPRESSED_SIZE, len(product))) result.append(product) buf_out = [] i += 1 # 0x81 <= n < 0xff: repeat next byte (257-n) times else: count = 257 - buffer[i] buf_out.append(buffer[i + 1:i + 2] * count) i += 2 if buf_out: logger.warning( '{}: EOF reached before last RLE block closed'.format(self)) result.append(b''.join(buf_out)) # result is a 960x160 3bpp image, divided into 4x 40 scanline segments unpack = (self.plan.import_data(x).payload for x in result) return mrc.TransformResult(payload=bytes(itertools.chain(*unpack)), end_offset=i)
def import_data( self, buffer, parent=None ): assert utils.is_bytes( buffer ) pattern_count = mrc.UInt16_BE( 0x0000 ).get_from_buffer( buffer ) xor_mode = (pattern_count & 0x8000) != 0 pattern_count &= 0x7fff index = 2 lut = {} prev_pal_index = 0 while index < len( buffer ): test = buffer[index] if test == 0xff: break elif test & 0x80: code_raw = buffer[index+2] bit_count = buffer[index+1] & 0x0f code = ''.join(['1' if (code_raw & (1<<i) ) else '0' for i in range( bit_count-1, -1, -1 )]) lut[code] = { 'pal_index': buffer[index] & 0x0f, 'copy_count': ((buffer[index+1] & 0xf0) >> 4) + 1, } prev_pal_index = lut[code]['pal_index'] index += 3 else: code_raw = buffer[index+1] bit_count = buffer[index] & 0x0f code = ''.join(['1' if (code_raw & (1<<i) ) else '0' for i in range( bit_count-1, -1, -1 )]) lut[code] = { 'pal_index': prev_pal_index, 'copy_count': ((buffer[index] & 0xf0) >> 4) + 1, } index += 2 bs = BitReader( buffer[index+1:], 0, bits_reverse=True ) state = { 'output': bytearray( 64*pattern_count ), 'output_index': 0, 'current_row': [], 'prev_row': bytearray( 8 ) } def push_pal( pal, state ): state['current_row'].append( pal ) if len( state['current_row'] ) == 8: output_index = state['output_index'] for i in range( 8 ): state['output'][output_index+i] = state['current_row'][i] if xor_mode: for i in range( 8 ): state['output'][output_index+i] ^= state['prev_row'][i] prev_row = state['output'][output_index:output_index+8] state['output_index'] += 8 state['current_row'].clear() return max_key_size = max( [len(x) for x in lut.keys()] ) while state['output_index'] < 64*pattern_count: test = '' for i in range( max_key_size ): test += '1' if bs.get_bits( 1 ) else '0' if test in lut or test == '111111': break if test in lut: for i in range( lut[test]['copy_count'] ): push_pal( lut[test]['pal_index'], state ) elif test == '111111': copy_count = bs.get_bits( 3 ) pal_index = bs.get_bits( 4 ) for i in range( copy_count ): push_pal( pal_index, state ) else: raise Exception( 'Invalid code found in data stream, aborting' ) return bytes( state['output'] )
def import_data(self, buffer, parent=None): assert utils.is_bytes(buffer) pointer = 0 total_num_bytes = len(buffer) bit_count = utils.from_uint8(buffer[pointer:pointer + 1]) checksum = utils.from_uint8(buffer[pointer + 1:pointer + 2]) decompressed_size = utils.from_uint32_be(buffer[pointer + 2:pointer + 6]) compressed_size = utils.from_uint32_be(buffer[pointer + 6:pointer + 10]) pointer += 10 total_num_bytes -= 10 compressed_size -= 10 compressed_data = bytearray(buffer[pointer:pointer + compressed_size]) if checksum != self._xor_checksum(compressed_data): logger.warning('{}: Checksum doesn\'t match header'.format(self)) pointer += compressed_size total_num_bytes -= compressed_size # first byte of compressed data is shifted wrongly, fix compressed_data[-1] = (compressed_data[-1] << (8 - bit_count)) & 0xff bs = bits.BitStream(compressed_data, start_offset=(compressed_size - 1, bit_count - 1), bytes_reverse=True, bit_endian='little', io_endian='big') def copy_prev_data(blocklen, offset_size, state): offset = bs.read(offset_size) for i in range(blocklen): state['dptr'] -= 1 state['ddata'][state['dptr']] = state['ddata'][state['dptr'] + offset + 1] return def dump_data(num_bytes, state): for i in range(num_bytes): state['dptr'] -= 1 state['ddata'][state['dptr']] = bs.read(8) return state = { 'dptr': decompressed_size, 'ddata': bytearray(decompressed_size), } while True: if bs.read(1) == 1: test = bs.read(2) if test == 0: copy_prev_data(3, 9, state) elif test == 1: copy_prev_data(4, 10, state) elif test == 2: copy_prev_data(bs.read(8) + 1, 12, state) elif test == 3: dump_data(bs.read(8) + 9, state) else: test = bs.read(1) if test == 0: dump_data(bs.read(3) + 1, state) elif test == 1: copy_prev_data(2, 8, state) if not (state['dptr'] > 0): break return mrc.TransformResult(payload=bytes(state['ddata']), end_offset=pointer)
def export_data(self, buffer: bytes, parent=None): assert utils.is_bytes(buffer) # load in constructor properties bpp = mrc.property_get(self.bpp, parent) width = mrc.property_get(self.width, parent) height = mrc.property_get(self.height, parent) plane_size = mrc.property_get(self.plane_size, parent) plane_padding = mrc.property_get(self.plane_padding, parent) frame_offset = mrc.property_get(self.frame_offset, parent) frame_count = mrc.property_get(self.frame_count, parent) frame_stride = mrc.property_get(self.frame_stride, parent) assert (bpp >= 0) and (bpp <= 8) if (width or height): assert (width * height) % 8 == 0 if plane_size: raise Exception( 'Can\'t define plane_size when either width or height is defined.' ) elif plane_size is None and frame_count == 1: # for a single frame without a plane size, assume the buffer contains everything assert len(buffer) % bpp == 0 plane_size = len(buffer) // bpp else: assert plane_size is not None if not plane_size: plane_size = math.ceil(width * height / 8) assert (frame_count >= 1) if frame_count >= 2 and frame_stride is None: frame_stride = bpp * (plane_size + plane_padding) else: frame_stride = frame_stride if frame_stride is not None else 0 if frame_count == 1: assert len(buffer) >= frame_offset + plane_size * 8 else: assert len(buffer) >= frame_offset + frame_count * frame_stride # this method just does the opposite of the above; split chunky pixels back into planes. planes = array('Q') segment_size = plane_size + plane_padding if frame_count == 1: raw_planes = bytearray(frame_offset + segment_size * bpp) else: raw_planes = bytearray(frame_offset + frame_count * frame_stride) for f in range(frame_count): pointer = frame_offset + f * frame_stride planes = planes[0:0] # load our chunky pixels into the 64-bit int array planes.frombytes(buffer[f * plane_size * 8:(f + 1) * plane_size * 8]) # check for endianness! if sys.byteorder == 'little': planes.byteswap() for b in range(bpp): for i in range(plane_size): # for each group of 8 chunky pixels, use pack_bits to fill up 8 bits # of the relevant bitplane raw_planes[pointer + b * segment_size + i] = bits.pack_bits((planes[i] >> b)) return mrc.TransformResult(payload=raw_planes)
def import_data(self, buffer: bytes, parent=None): assert utils.is_bytes(buffer) # load in constructor properties bpp = mrc.property_get(self.bpp, parent) width = mrc.property_get(self.width, parent) height = mrc.property_get(self.height, parent) plane_size = mrc.property_get(self.plane_size, parent) plane_padding = mrc.property_get(self.plane_padding, parent) frame_offset = mrc.property_get(self.frame_offset, parent) frame_count = mrc.property_get(self.frame_count, parent) frame_stride = mrc.property_get(self.frame_stride, parent) row_planar_size = mrc.property_get(self.row_planar_size, parent) plane_order = mrc.property_get(self.plane_order, parent) assert (bpp >= 0) and (bpp <= 8) if (width or height): assert (width * height) % 8 == 0 if plane_size: raise Exception( 'Can\'t define plane_size when either width or height is defined.' ) elif plane_size is None and frame_count == 1: # for a single frame without a plane size, assume the buffer contains everything assert len(buffer) % bpp == 0 plane_size = len(buffer) // bpp else: assert plane_size is not None assert (frame_count >= 1) if plane_size is None: plane_size = math.ceil(width * height / 8) if frame_count >= 2 and frame_stride is None: frame_stride = bpp * (plane_size + plane_padding) else: frame_stride = frame_stride if frame_stride is not None else 0 if row_planar_size: assert row_planar_size >= 1 if not plane_order: plane_order = range(bpp) else: assert all([y in range(bpp) for y in plane_order]) assert len(plane_order) == len(set(plane_order)) # because frame_stride can potentially read past the buffer, only worry about measuring # the last n-1 strides + one frame assert len(buffer) >= frame_offset + ( frame_count - 1) * frame_stride + bpp * plane_size # our output is going to be "chunky"; each byte is a pixel (8-bit or 256 colour mode) raw_image = bytearray(plane_size * frame_count) # the input is planar. this is a packed format found occasionally in old graphics hardware, # and in old image formats where space was paramount. # the trick is you can have less than 8 bits in your colourspace! # e.g. if you only need 8 colours, you can get away with a 3-bit colourspace and save 62.5% space. # instead of each byte being a pixel, each byte stores 8 pixels worth of data for a single plane. # there is one plane per bit of colourspace, and the planes are stored one after another. # in order for the calculations to be fast, planar graphics are pretty much always divisible by 8. # we're going to abuse this and unpack our bitplanes using 64-bit integers. # let's make a big array of them. planes = array('Q', (0, ) * (plane_size)) segment_size = plane_size + plane_padding for f in range(frame_count): pointer = frame_offset + f * frame_stride for bi, b in enumerate(plane_order): for i in range(plane_size): # for the first iteration, clear the plane if bi == 0: planes[i] = 0 if row_planar_size is None: address = pointer + b * segment_size + i else: address = pointer + (row_planar_size * bpp) * ( i // row_planar_size) + row_planar_size * b + ( i % row_planar_size) # bits.unpack_bits is a helper method which converts a 1-byte bitfield # into 8 bool bytes (i.e. 1 or 0) stored as a 64-bit int. # we can effectively work on 8 chunky pixels at once! # because the chunky pixels are bitfields, combining planes is an easy # left shift (i.e. move all the bits up by [plane ID] places) and bitwise OR planes[i] |= bits.unpack_bits(buffer[address]) << bi # check for endianness! for most intel and ARM chips the order of bytes in hardware is reversed, # so we need to flip it around for the bytes to be sequential. if sys.byteorder == 'little': planes.byteswap() # convert our planes array to bytes, and you have your chunky pixels raw_image[f * plane_size * 8:(f + 1) * plane_size * 8] = planes.tobytes() if frame_count > 1: end_offset = frame_offset + frame_count * frame_stride else: plane_bits = plane_size * 8 * bpp end_offset = frame_offset + (plane_bits) // 8 + (1 if (plane_bits % 8) else 0) return mrc.TransformResult(payload=bytes(raw_image), end_offset=end_offset)
def decode_nfo(buffer): """Decodes a byte string in NFO format (beloved by PC scener groups) from DOS Code Page 437 to Unicode.""" assert utils.is_bytes(buffer) return '\n'.join( [''.join([CP437[y] for y in x]) for x in buffer.split(b'\r\n')])
def export_data( self, buffer: bytes, parent=None ): assert utils.is_bytes( buffer ) # load in constructor properties bpp = mrc.property_get( self.bpp, parent ) width = mrc.property_get( self.width, parent ) height = mrc.property_get( self.height, parent ) plane_size = mrc.property_get( self.plane_size, parent ) plane_padding = mrc.property_get( self.plane_padding, parent ) frame_offset = mrc.property_get( self.frame_offset, parent ) frame_count = mrc.property_get( self.frame_count, parent ) frame_stride = mrc.property_get( self.frame_stride, parent ) assert (bpp >= 0) and (bpp <= 8) if (width or height): assert (width*height) % 8 == 0 if plane_size: raise Exception( 'Can\'t define plane_size when either width or height is defined.' ) elif plane_size is None and frame_count == 1: # for a single frame without a plane size, assume the buffer contains everything assert len( buffer ) % bpp == 0 plane_size = len( buffer ) // bpp else: assert plane_size is not None if not plane_size: plane_size = math.ceil( width*height/8 ) assert (frame_count >= 1) if frame_count >= 2 and frame_stride is None: frame_stride = bpp*(plane_size+plane_padding) else: frame_stride = frame_stride if frame_stride is not None else 0 if frame_count == 1: assert len( buffer ) >= frame_offset + plane_size*8 else: assert len( buffer ) >= frame_offset + frame_count*frame_stride # this method just does the opposite of the above; split chunky pixels back into planes. planes = array( 'Q' ) segment_size = plane_size+plane_padding if frame_count == 1: raw_planes = bytearray( frame_offset+segment_size*bpp ) else: raw_planes = bytearray( frame_offset+frame_count*frame_stride ) for f in range( frame_count ): pointer = frame_offset+f*frame_stride planes = planes[0:0] # load our chunky pixels into the 64-bit int array planes.frombytes( buffer[f*plane_size*8:(f+1)*plane_size*8] ) # check for endianness! if sys.byteorder == 'little': planes.byteswap() for b in range( bpp ): for i in range( plane_size ): # for each group of 8 chunky pixels, use pack_bits to fill up 8 bits # of the relevant bitplane raw_planes[pointer+b*segment_size+i] = utils.pack_bits( (planes[i] >> b) ) return mrc.TransformResult( payload=raw_planes )
def import_data( self, buffer: bytes, parent=None ): assert utils.is_bytes( buffer ) # load in constructor properties bpp = mrc.property_get( self.bpp, parent ) width = mrc.property_get( self.width, parent ) height = mrc.property_get( self.height, parent ) plane_size = mrc.property_get( self.plane_size, parent ) plane_padding = mrc.property_get( self.plane_padding, parent ) frame_offset = mrc.property_get( self.frame_offset, parent ) frame_count = mrc.property_get( self.frame_count, parent ) frame_stride = mrc.property_get( self.frame_stride, parent ) row_planar_size = mrc.property_get( self.row_planar_size, parent ) plane_order = mrc.property_get( self.plane_order, parent ) assert (bpp >= 0) and (bpp <= 8) if (width or height): assert (width*height) % 8 == 0 if plane_size: raise Exception( 'Can\'t define plane_size when either width or height is defined.' ) elif plane_size is None and frame_count == 1: # for a single frame without a plane size, assume the buffer contains everything assert len( buffer ) % bpp == 0 plane_size = len( buffer ) // bpp else: assert plane_size is not None assert (frame_count >= 1) if plane_size is None: plane_size = math.ceil( width*height/8 ) if frame_count >= 2 and frame_stride is None: frame_stride = bpp*(plane_size+plane_padding) else: frame_stride = frame_stride if frame_stride is not None else 0 if row_planar_size: assert row_planar_size >= 1 if not plane_order: plane_order = range( bpp ) else: assert all( [y in range( bpp ) for y in plane_order] ) assert len( plane_order ) == len( set( plane_order ) ) # because frame_stride can potentially read past the buffer, only worry about measuring # the last n-1 strides + one frame assert len( buffer ) >= frame_offset + (frame_count-1)*frame_stride + bpp*plane_size # our output is going to be "chunky"; each byte is a pixel (8-bit or 256 colour mode) raw_image = bytearray( plane_size*frame_count ) # the input is planar. this is a packed format found occasionally in old graphics hardware, # and in old image formats where space was paramount. # the trick is you can have less than 8 bits in your colourspace! # e.g. if you only need 8 colours, you can get away with a 3-bit colourspace and save 62.5% space. # instead of each byte being a pixel, each byte stores 8 pixels worth of data for a single plane. # there is one plane per bit of colourspace, and the planes are stored one after another. # in order for the calculations to be fast, planar graphics are pretty much always divisible by 8. # we're going to abuse this and unpack our bitplanes using 64-bit integers. # let's make a big array of them. planes = array( 'Q', (0,)*(plane_size) ) segment_size = plane_size+plane_padding for f in range( frame_count ): pointer = frame_offset+f*frame_stride for bi, b in enumerate( plane_order ): for i in range( plane_size ): # for the first iteration, clear the plane if bi==0: planes[i] = 0 if row_planar_size is None: address = pointer+b*segment_size+i else: address = pointer + (row_planar_size*bpp)*(i // row_planar_size) + row_planar_size*b + (i % row_planar_size) # utils.unpack_bits is a helper method which converts a 1-byte bitfield # into 8 bool bytes (i.e. 1 or 0) stored as a 64-bit int. # we can effectively work on 8 chunky pixels at once! # because the chunky pixels are bitfields, combining planes is an easy # left shift (i.e. move all the bits up by [plane ID] places) and bitwise OR planes[i] |= utils.unpack_bits( buffer[address] ) << bi # check for endianness! for most intel and ARM chips the order of bytes in hardware is reversed, # so we need to flip it around for the bytes to be sequential. if sys.byteorder == 'little': planes.byteswap() # convert our planes array to bytes, and you have your chunky pixels raw_image[f*plane_size*8:(f+1)*plane_size*8] = planes.tobytes() if frame_count > 1: end_offset = frame_offset + frame_count*frame_stride else: bits = plane_size*8*bpp end_offset = frame_offset + (bits)//8 + (1 if (bits % 8) else 0) return mrc.TransformResult( payload=bytes( raw_image ), end_offset=end_offset )
def import_data(self, buffer, parent=None): assert utils.is_bytes(buffer) decompressed_size = ( (buffer[0] & 0x0f) << 16) + (buffer[2] << 8) + buffer[3]
def import_data( self, buffer, parent=None ): assert utils.is_bytes( buffer ) decompressed_size = ((buffer[0] & 0x0f) << 16) + (buffer[2] << 8) + buffer[3]
def export_data( self, buffer, parent=None ): assert utils.is_bytes( buffer ) decompressed_size = len( buffer ) bs = utils.BitWriter( bits_reverse=True ) pointer = 0 def encode_raw_data( length, bs ): assert length <= 255+9 if length > 8: bs.put_bits( length-9, 8 ) bs.put_bits( 0x7, 3 ) elif length > 0: bs.put_bits( length-1, 3 ) bs.put_bits( 0x0, 2 ) def find_reference(): # main form of compression is of the form: # - while decompressing from end to start # - look forward [up to max_offset] bytes in the decompressed data # - copy [up to max_length] bytes to the current decompression position # the largest offset supported by the file format is 4096, but this means # every call to find_reference loops 4096 times. # this takes foreeeever in Python! # because the compression is worthless and time is money, max_offset has # been slashed to 16 to speed up proceedings. #max_offset = (1 << 12) + 1 max_offset = (1 << 4) + 1 # largest length supported by the file format is 256 max_length = (1 << 8) + 1 length = 4 # throw away short references offset = 0 short_offset = [0, 0, 0] for i in range( pointer+1, pointer+max_offset ): temp_len = 0 while (temp_len < max_length) and (i+temp_len < decompressed_size): # record short references if (temp_len >= 2) and (temp_len <= 4): if short_offset[temp_len-2] == 0: short_offset[temp_len-2] = i-pointer if buffer[pointer+temp_len] != buffer[i+temp_len]: break temp_len += 1 if temp_len == max_length: temp_len -= 1 # largest reference so far? use it if temp_len > length: length = temp_len offset = i-pointer assert length < max_length assert offset < max_offset # no long references? try short if (offset == 0): for i in (2, 1, 0): max_short_offset = (1 << (i+8))+1 if (short_offset[i] > 0) and (short_offset[i] < max_short_offset): length = i+2 offset = short_offset[i] break return length, offset raw = 0 while pointer < decompressed_size: length, ref = find_reference() if ref > 0: if raw > 0: encode_raw_data( raw, bs ) raw = 0 if length > 4: bs.put_bits( ref-1, 12 ) bs.put_bits( length-1, 8 ) bs.put_bits( 0x6, 3 ) elif length == 4: bs.put_bits( ref-1, 10 ) bs.put_bits( 0x5, 3 ) elif length == 3: bs.put_bits( ref-1, 9 ) bs.put_bits( 0x4, 3 ) elif length == 2: bs.put_bits( ref-1, 8 ) bs.put_bits( 0x1, 2 ) pointer += length else: bs.put_bits( buffer[pointer], 8 ) raw += 1 if raw == 264: encode_raw_data( raw, bs ) raw = 0 pointer += 1 encode_raw_data( raw, bs ) compressed_data = bs.get_buffer() compressed_size = len( compressed_data ) + 10 checksum = self._xor_checksum( compressed_data ) output = bytearray( 6 ) output[0:1] = utils.to_uint8( 8-(bs.bits_remaining % 8) ) output[1:2] = utils.to_uint8( checksum ) output[2:6] = utils.to_uint32_be( decompressed_size ) output[6:10] = utils.to_uint32_be( compressed_size ) output.extend( compressed_data ) return mrc.TransformResult( payload=bytes( output ) )
def import_data( self, buffer, parent=None ): assert utils.is_bytes( buffer ) stage_1 = self.rle.import_data( buffer ) stage_2 = self.plan.import_data( stage_1.payload ) return mrc.TransformResult( payload=stage_2.payload, end_offset=stage_1.end_offset )
def export_data(self, buffer, parent=None): assert utils.is_bytes(buffer) decompressed_size = len(buffer) bs = utils.BitWriter(bits_reverse=True) pointer = 0 def encode_raw_data(length, bs): assert length <= 255 + 9 if length > 8: bs.put_bits(length - 9, 8) bs.put_bits(0x7, 3) elif length > 0: bs.put_bits(length - 1, 3) bs.put_bits(0x0, 2) def find_reference(): # main form of compression is of the form: # - while decompressing from end to start # - look forward [up to max_offset] bytes in the decompressed data # - copy [up to max_length] bytes to the current decompression position # the largest offset supported by the file format is 4096, but this means # every call to find_reference loops 4096 times. # this takes foreeeever in Python! # because the compression is worthless and time is money, max_offset has # been slashed to 16 to speed up proceedings. #max_offset = (1 << 12) + 1 max_offset = (1 << 4) + 1 # largest length supported by the file format is 256 max_length = (1 << 8) + 1 length = 4 # throw away short references offset = 0 short_offset = [0, 0, 0] for i in range(pointer + 1, pointer + max_offset): temp_len = 0 while (temp_len < max_length) and (i + temp_len < decompressed_size): # record short references if (temp_len >= 2) and (temp_len <= 4): if short_offset[temp_len - 2] == 0: short_offset[temp_len - 2] = i - pointer if buffer[pointer + temp_len] != buffer[i + temp_len]: break temp_len += 1 if temp_len == max_length: temp_len -= 1 # largest reference so far? use it if temp_len > length: length = temp_len offset = i - pointer assert length < max_length assert offset < max_offset # no long references? try short if (offset == 0): for i in (2, 1, 0): max_short_offset = (1 << (i + 8)) + 1 if (short_offset[i] > 0) and (short_offset[i] < max_short_offset): length = i + 2 offset = short_offset[i] break return length, offset raw = 0 while pointer < decompressed_size: length, ref = find_reference() if ref > 0: if raw > 0: encode_raw_data(raw, bs) raw = 0 if length > 4: bs.put_bits(ref - 1, 12) bs.put_bits(length - 1, 8) bs.put_bits(0x6, 3) elif length == 4: bs.put_bits(ref - 1, 10) bs.put_bits(0x5, 3) elif length == 3: bs.put_bits(ref - 1, 9) bs.put_bits(0x4, 3) elif length == 2: bs.put_bits(ref - 1, 8) bs.put_bits(0x1, 2) pointer += length else: bs.put_bits(buffer[pointer], 8) raw += 1 if raw == 264: encode_raw_data(raw, bs) raw = 0 pointer += 1 encode_raw_data(raw, bs) compressed_data = bs.get_buffer() compressed_size = len(compressed_data) + 10 checksum = self._xor_checksum(compressed_data) output = bytearray(6) output[0:1] = utils.to_uint8(8 - (bs.bits_remaining % 8)) output[1:2] = utils.to_uint8(checksum) output[2:6] = utils.to_uint32_be(decompressed_size) output[6:10] = utils.to_uint32_be(compressed_size) output.extend(compressed_data) return mrc.TransformResult(payload=bytes(output))
def import_data( self, buffer, parent=None ): assert utils.is_bytes( buffer ) pointer = 0; total_num_bytes = len( buffer ) bit_count = utils.from_uint8( buffer[pointer:pointer+1] ) checksum = utils.from_uint8( buffer[pointer+1:pointer+2] ) decompressed_size = utils.from_uint32_be( buffer[pointer+2:pointer+6] ) compressed_size = utils.from_uint32_be( buffer[pointer+6:pointer+10] ) pointer += 10 total_num_bytes -= 10 compressed_size -= 10 compressed_data = buffer[pointer:pointer+compressed_size] if checksum != self._xor_checksum( compressed_data ): logger.warning( '{}: Checksum doesn\'t match header'.format( self ) ) pointer += compressed_size total_num_bytes -= compressed_size bs = utils.BitReader( compressed_data, compressed_size-1, bytes_reverse=True, output_reverse=True ) bs.bits_remaining = bit_count def copy_prev_data( blocklen, offset_size, state ): offset = bs.get_bits( offset_size ) for i in range( blocklen ): state['dptr'] -= 1 state['ddata'][state['dptr']] = state['ddata'][state['dptr']+offset+1] return def dump_data( num_bytes, state ): for i in range( num_bytes ): state['dptr'] -= 1 state['ddata'][state['dptr']] = bs.get_bits( 8 ) return state = { 'dptr': decompressed_size, 'ddata': bytearray( decompressed_size ), } while True: if bs.get_bits( 1 ) == 1: test = bs.get_bits( 2 ) if test==0: copy_prev_data( 3, 9, state ) elif test==1: copy_prev_data( 4, 10, state ) elif test==2: copy_prev_data( bs.get_bits( 8 )+1, 12, state ) elif test==3: dump_data( bs.get_bits( 8 )+9, state ) else: test = bs.get_bits( 1 ) if test==0: dump_data( bs.get_bits( 3 )+1, state ) elif test==1: copy_prev_data( 2, 8, state ) if not (state['dptr'] > 0): break return mrc.TransformResult( payload=bytes( state['ddata'] ), end_offset=pointer )
def decode_nfo( buffer ): """Decodes a byte string in NFO format (beloved by PC scener groups) from DOS Code Page 437 to Unicode.""" assert utils.is_bytes( buffer ) return '\n'.join( [''.join( [CP437[y] for y in x] ) for x in buffer.split( b'\r\n' )] )
def import_data(self, buffer, parent=None): assert utils.is_bytes(buffer) pointer = 0 total_num_bytes = len(buffer) bit_count = utils.from_uint8(buffer[pointer:pointer + 1]) checksum = utils.from_uint8(buffer[pointer + 1:pointer + 2]) decompressed_size = utils.from_uint32_be(buffer[pointer + 2:pointer + 6]) compressed_size = utils.from_uint32_be(buffer[pointer + 6:pointer + 10]) pointer += 10 total_num_bytes -= 10 compressed_size -= 10 compressed_data = buffer[pointer:pointer + compressed_size] if checksum != self._xor_checksum(compressed_data): logger.warning('{}: Checksum doesn\'t match header'.format(self)) pointer += compressed_size total_num_bytes -= compressed_size bs = utils.BitReader(compressed_data, compressed_size - 1, bytes_reverse=True, output_reverse=True) bs.bits_remaining = bit_count def copy_prev_data(blocklen, offset_size, state): offset = bs.get_bits(offset_size) for i in range(blocklen): state['dptr'] -= 1 state['ddata'][state['dptr']] = state['ddata'][state['dptr'] + offset + 1] return def dump_data(num_bytes, state): for i in range(num_bytes): state['dptr'] -= 1 state['ddata'][state['dptr']] = bs.get_bits(8) return state = { 'dptr': decompressed_size, 'ddata': bytearray(decompressed_size), } while True: if bs.get_bits(1) == 1: test = bs.get_bits(2) if test == 0: copy_prev_data(3, 9, state) elif test == 1: copy_prev_data(4, 10, state) elif test == 2: copy_prev_data(bs.get_bits(8) + 1, 12, state) elif test == 3: dump_data(bs.get_bits(8) + 9, state) else: test = bs.get_bits(1) if test == 0: dump_data(bs.get_bits(3) + 1, state) elif test == 1: copy_prev_data(2, 8, state) if not (state['dptr'] > 0): break return mrc.TransformResult(payload=bytes(state['ddata']), end_offset=pointer)