def md5(self, filename): compressed_blocks = None compress = Compress(self.source_filename) if compress.get_format(): compressed_blocks = self.__block_list( os.path.getsize(self.source_filename) ) compress.uncompress(temporary=True) blocks = self.__block_list( os.path.getsize(compress.uncompressed_filename) ) else: blocks = self.__block_list( os.path.getsize(self.source_filename) ) with open(self.source_filename) as source: checksum = hashlib.md5(source.read()).hexdigest() with open(filename, 'w') as md5: if compressed_blocks: md5.write( '%s %s %s %s %s\n' % ( checksum, blocks.blocks, blocks.blocksize, compressed_blocks.blocks, compressed_blocks.blocksize ) ) else: md5.write( '%s %s %s\n' % ( checksum, blocks.blocks, blocks.blocksize ) )
def create_initrd(self, mbrid=None): if self.is_prepared(): log.info('Creating initrd cpio archive') initrd_file_name = ''.join( [ self.target_dir, '/', self.xml_state.xml_data.get_name(), '.initrd' ] ) # we can't simply exclude boot when building the archive # because the file boot/mbrid must be preserved. Because of # that we create a copy of the boot directory and remove # everything in boot/ except for boot/mbrid. The original # boot directory should not be changed because we rely # on other data in boot/ e.g the kernel to be available # for the entire image building process self.temp_boot_root_directory = mkdtemp() Command.run( [ 'rsync', '-zav', self.boot_root_directory + '/', self.temp_boot_root_directory ] ) boot_directory = self.temp_boot_root_directory + '/boot' Path.wipe(boot_directory) if mbrid: log.info( '--> Importing mbrid: %s', mbrid.get_id() ) Path.create(boot_directory) image_identifier = boot_directory + '/mbrid' mbrid.write(image_identifier) cpio = ArchiveCpio(initrd_file_name) # the following is a list of directories which were needed # during the process of creating an image but not when the # image is actually booting with this initrd exclude_from_archive = [ '/var/cache', '/image', '/usr/lib/grub2' ] cpio.create( source_dir=self.temp_boot_root_directory, exclude=exclude_from_archive ) log.info( '--> xz compressing archive' ) compress = Compress(initrd_file_name) compress.xz() self.initrd_filename = compress.compressed_filename
def open_file(): filename = "water-stays-saved.jpeg" try: image = Compress(filename) seam = image.best_seam() image.remove_color_seam(seam) image.save("water.jpeg") except Exception: raise
def create_install_pxe_archive(self): """ Create an oem install tar archive suitable for installing a disk image via the network using the PXE boot protocol. The archive contains the raw disk image and its checksum as well as an install initrd and kernel plus the required kernel commandline information which needs to be added as append line in the pxelinux config file on the boot server """ self.pxe_dir = mkdtemp(prefix="pxe-install-media.", dir=self.target_dir) # the system image is transfered as xz compressed variant log.info("xz compressing disk image") pxe_image_filename = "".join([self.pxe_dir, "/", self.xml_state.xml_data.get_name(), ".xz"]) compress = Compress(source_filename=self.diskname, keep_source_on_compress=True) compress.xz() Command.run(["mv", compress.compressed_filename, pxe_image_filename]) # the system image transfer is checked against a checksum log.info("Creating disk image checksum") pxe_md5_filename = "".join([self.pxe_dir, "/", self.xml_state.xml_data.get_name(), ".md5"]) checksum = Checksum(pxe_image_filename) checksum.md5(pxe_md5_filename) # create pxe config append information # this information helps to configure the boot server correctly append_filename = "".join([self.pxe_dir, "/", self.xml_state.xml_data.get_name(), ".append"]) cmdline = "pxe=1" custom_cmdline = self.xml_state.build_type.get_kernelcmdline() if custom_cmdline: cmdline += " " + custom_cmdline with open(append_filename, "w") as append: append.write("%s\n" % cmdline) # create initrd for pxe install log.info("Creating pxe install boot image") self.__create_pxe_install_kernel_and_initrd() # create pxe install tarball log.info("Creating pxe install archive") archive = ArchiveTar(self.pxename.replace(".xz", "")) archive.create_xz_compressed(self.pxe_dir)
def md5(self, filename): compressed_blocks = None compress = Compress(self.source_filename) if compress.get_format(): compressed_blocks = self.__block_list( os.path.getsize(self.source_filename)) compress.uncompress(temporary=True) blocks = self.__block_list( os.path.getsize(compress.uncompressed_filename)) else: blocks = self.__block_list(os.path.getsize(self.source_filename)) with open(self.source_filename) as source: checksum = hashlib.md5(source.read()).hexdigest() with open(filename, 'w') as md5: if compressed_blocks: md5.write( '%s %s %s %s %s\n' % (checksum, blocks.blocks, blocks.blocksize, compressed_blocks.blocks, compressed_blocks.blocksize)) else: md5.write('%s %s %s\n' % (checksum, blocks.blocks, blocks.blocksize))
def create_initrd(self, mbrid=None): if self.is_prepared(): log.info('Creating initrd cpio archive') initrd_file_name = ''.join([ self.target_dir, '/', self.xml_state.xml_data.get_name(), '.initrd' ]) # we can't simply exclude boot when building the archive # because the file boot/mbrid must be preserved. Because of # that we create a copy of the boot directory and remove # everything in boot/ except for boot/mbrid. The original # boot directory should not be changed because we rely # on other data in boot/ e.g the kernel to be available # for the entire image building process self.temp_boot_root_directory = mkdtemp() Command.run([ 'rsync', '-zav', self.boot_root_directory + '/', self.temp_boot_root_directory ]) boot_directory = self.temp_boot_root_directory + '/boot' Path.wipe(boot_directory) if mbrid: log.info('--> Importing mbrid: %s', mbrid.get_id()) Path.create(boot_directory) image_identifier = boot_directory + '/mbrid' mbrid.write(image_identifier) cpio = ArchiveCpio(initrd_file_name) # the following is a list of directories which were needed # during the process of creating an image but not when the # image is actually booting with this initrd exclude_from_archive = ['/var/cache', '/image', '/usr/lib/grub2'] cpio.create(source_dir=self.temp_boot_root_directory, exclude=exclude_from_archive) log.info('--> xz compressing archive') compress = Compress(initrd_file_name) compress.xz() self.initrd_filename = compress.compressed_filename
def compress(): if request.method == "POST": if request.files: content_type = request.mimetype image = request.files['image'] compression_rate = request.form['compression_rate'] image.save(os.path.join(app.config['uploads'], image.filename)) # print(compression_rate) path = Compress( os.path.join(app.config['uploads'], image.filename), int(compression_rate)) encoded_img = get_response_image( os.path.join(app.config['compress_path'], path)) uncompress_img = get_response_image( os.path.join(app.config['compress_path'], "uncompressed.bmp")) response = { "message": "saved", "encoded_img": encoded_img, "uncompress_img": uncompress_img } return response, 200
def setUp(self): self.c = Compress()
def create(self): log.info('Creating PXE root filesystem image') self.filesystem.create() self.image = self.filesystem.filename if self.compressed: log.info('xz compressing root filesystem image') compress = Compress(self.image) compress.xz() self.image = compress.compressed_filename log.info('Creating PXE root filesystem MD5 checksum') self.filesystem_checksum = self.filesystem.filename + '.md5' checksum = Checksum(self.image) checksum.md5(self.filesystem_checksum) # prepare boot(initrd) root system log.info('Creating PXE boot image') self.boot_image_task.prepare() # export modprobe configuration to boot image self.system_setup.export_modprobe_setup( self.boot_image_task.boot_root_directory ) # extract kernel from boot(initrd) root system kernel = Kernel(self.boot_image_task.boot_root_directory) kernel_data = kernel.get_kernel() if kernel_data: self.kernel_filename = ''.join( [self.image_name, '-', kernel_data.version, '.kernel'] ) kernel.copy_kernel( self.target_dir, self.kernel_filename ) else: raise KiwiPxeBootImageError( 'No kernel in boot image tree %s found' % self.boot_image_task.boot_root_directory ) # extract hypervisor from boot(initrd) root system if self.machine and self.machine.get_domain() == 'dom0': kernel_data = kernel.get_xen_hypervisor() if kernel_data: self.hypervisor_filename = ''.join( [self.image_name, '-', kernel_data.name] ) kernel.copy_xen_hypervisor( self.target_dir, self.hypervisor_filename ) self.result.add( 'xen_hypervisor', self.hypervisor_filename ) else: raise KiwiPxeBootImageError( 'No hypervisor in boot image tree %s found' % self.boot_image_task.boot_root_directory ) # create initrd for pxe boot self.boot_image_task.create_initrd() self.result.add( 'kernel', self.kernel_filename ) self.result.add( 'initrd', self.boot_image_task.initrd_filename ) self.result.add( 'filesystem_image', self.image ) self.result.add( 'filesystem_md5', self.filesystem_checksum ) if self.pxedeploy: log.warning( 'Creation of client config file from pxedeploy not implemented' ) return self.result
def create(self): log.info('Creating PXE root filesystem image') self.filesystem.create() self.image = self.filesystem.filename if self.compressed: log.info('xz compressing root filesystem image') compress = Compress(self.image) compress.xz() self.image = compress.compressed_filename log.info('Creating PXE root filesystem MD5 checksum') self.filesystem_checksum = self.filesystem.filename + '.md5' checksum = Checksum(self.image) checksum.md5(self.filesystem_checksum) # prepare boot(initrd) root system log.info('Creating PXE boot image') self.boot_image_task.prepare() # export modprobe configuration to boot image self.system_setup.export_modprobe_setup( self.boot_image_task.boot_root_directory) # extract kernel from boot(initrd) root system kernel = Kernel(self.boot_image_task.boot_root_directory) kernel_data = kernel.get_kernel() if kernel_data: self.kernel_filename = ''.join( [self.image_name, '-', kernel_data.version, '.kernel']) kernel.copy_kernel(self.target_dir, self.kernel_filename) else: raise KiwiPxeBootImageError( 'No kernel in boot image tree %s found' % self.boot_image_task.boot_root_directory) # extract hypervisor from boot(initrd) root system if self.machine and self.machine.get_domain() == 'dom0': kernel_data = kernel.get_xen_hypervisor() if kernel_data: self.hypervisor_filename = ''.join( [self.image_name, '-', kernel_data.name]) kernel.copy_xen_hypervisor(self.target_dir, self.hypervisor_filename) self.result.add('xen_hypervisor', self.hypervisor_filename) else: raise KiwiPxeBootImageError( 'No hypervisor in boot image tree %s found' % self.boot_image_task.boot_root_directory) # create initrd for pxe boot self.boot_image_task.create_initrd() self.result.add('kernel', self.kernel_filename) self.result.add('initrd', self.boot_image_task.initrd_filename) self.result.add('filesystem_image', self.image) self.result.add('filesystem_md5', self.filesystem_checksum) if self.pxedeploy: log.warning( 'Creation of client config file from pxedeploy not implemented' ) return self.result
class TestCompress(unittest.TestCase): def setUp(self): self.c = Compress() def test_encode_word_the(self): enc = self.c.encode_symbol('the') binary = int(enc, 2) self.assertEquals(binary, 19) def test_encode_word_from(self): enc = self.c.encode_symbol('from') binary = int(enc, 2) self.assertEquals(binary, 109) def test_encode_word_home(self): enc = self.c.encode_symbol('home') binary = int(enc, 2) self.assertEquals(binary, 233) def test_encode_nonexistent_word(self): enc = self.c.encode_symbol('no-such-word') self.assertEquals(None, enc) def test_encode_char_space(self): enc = self.c.encode_symbol(' ') binary = int(enc, 2) self.assertEquals(binary, 7) def test_encode_char_e(self): enc = self.c.encode_symbol('e') binary = int(enc, 2) self.assertEquals(binary, 2) def test_encode_char_w(self): enc = self.c.encode_symbol('w') binary = int(enc, 2) self.assertEquals(binary, 48) def test_encode_text(self): enc = self.c.encode('this is') binary = int(enc, 2) self.assertEquals(binary, 183) def test_encode_text_with_punctuation(self): enc = self.c.encode('foo? is this bar!') binary = int(enc, 2) self.assertEquals(binary, 29686222816609210748006L) def test_encode_decode_1(self): s = 'you are really cool!' enc = self.c.encode(s) dec = self.c.decode(enc) self.assertEquals(s, dec) def test_encode_decode_2(self): s = '? i wonder how . compressed it ! will, be !' enc = self.c.encode(s) dec = self.c.decode(enc) self.assertEquals(s, dec) def test_encode_decode_3(self): s = 'xaajj tiko ax auui ahj nn nana nib' enc = self.c.encode(s) dec = self.c.decode(enc) self.assertEquals(s, dec)
def test_run(self): Compress('test/ori', 'test/res/' + now_str).run()
from compress import Compress path = Compress('harry.jpg', 1) print(path)
def create_recovery_archive(self): """ create a compressed recovery archive from the root tree for use with kiwi's recvoery system. The method creates additional data into the image root filesystem which is deleted prior to the creation of a new recovery data set """ # cleanup bash_comand = [ 'rm', '-f', self.root_dir + '/recovery.*' ] Command.run(['bash', '-c', ' '.join(bash_comand)]) if not self.oemconfig['recovery']: return # recovery.tar log.info('Creating recovery tar archive') metadata = { 'archive_name': self.root_dir + '/recovery.tar', 'archive_filecount': self.root_dir + '/recovery.tar.files', 'archive_size': self.root_dir + '/recovery.tar.size', 'partition_size': self.root_dir + '/recovery.partition.size', 'partition_filesystem': self.root_dir + '/recovery.tar.filesystem' } recovery_archive = NamedTemporaryFile( delete=False ) archive = ArchiveTar( filename=recovery_archive.name, create_from_file_list=False ) archive.create( source_dir=self.root_dir, exclude=['dev', 'proc', 'sys'], options=[ '--numeric-owner', '--hard-dereference', '--preserve-permissions' ] ) Command.run( ['mv', recovery_archive.name, metadata['archive_name']] ) # recovery.tar.filesystem recovery_filesystem = self.xml_state.build_type.get_filesystem() with open(metadata['partition_filesystem'], 'w') as partfs: partfs.write('%s' % recovery_filesystem) log.info( '--> Recovery partition filesystem: %s', recovery_filesystem ) # recovery.tar.files bash_comand = [ 'tar', '-tf', metadata['archive_name'], '|', 'wc', '-l' ] tar_files_call = Command.run( ['bash', '-c', ' '.join(bash_comand)] ) tar_files_count = int(tar_files_call.output.rstrip('\n')) with open(metadata['archive_filecount'], 'w') as files: files.write('%d\n' % tar_files_count) log.info( '--> Recovery file count: %d files', tar_files_count ) # recovery.tar.size recovery_archive_size_bytes = os.path.getsize(metadata['archive_name']) with open(metadata['archive_size'], 'w') as size: size.write('%d' % recovery_archive_size_bytes) log.info( '--> Recovery uncompressed size: %d mbytes', int(recovery_archive_size_bytes / 1048576) ) # recovery.tar.gz log.info('--> Compressing recovery archive') compress = Compress(self.root_dir + '/recovery.tar') compress.gzip() # recovery.partition.size recovery_archive_gz_size_mbytes = int( os.path.getsize(metadata['archive_name'] + '.gz') / 1048576 ) recovery_partition_mbytes = recovery_archive_gz_size_mbytes \ + Defaults.get_recovery_spare_mbytes() with open(metadata['partition_size'], 'w') as gzsize: gzsize.write('%d' % recovery_partition_mbytes) log.info( '--> Recovery partition size: %d mbytes', recovery_partition_mbytes ) # delete recovery archive if inplace recovery is requested # In this mode the recovery archive is created at install time # and not at image creation time. However the recovery metadata # is preserved in order to be able to check if enough space # is available on the disk to create the recovery archive. if self.oemconfig['recovery_inplace']: log.info( '--> Inplace recovery requested, deleting archive' ) Path.wipe(metadata['archive_name'] + '.gz')
import sys sys.path.append('..') from compress import Compress files = [ 'pat-0.pat', #'pat-1.pat', #'pat-2.pat', #'pat-3.pat', #'pat-4.pat', 'pat-5.pat', 'pat-6.pat', 'pat-7.pat', 'pat-8.pat' ] print files for file in files: print '\n\n----------------' + file + '----------------' with open(file, 'r') as fh: comp = Compress(fh.readlines()) print 'COMPRESSED:' for line in comp.compress(): print line
def add(filename: str, path: pathlib.Path, compresstype: int, offset: int): version = 46 # Bzip2 flags = 0 extra = b"" centraldirectory = [] fileinfo = os.stat(filename) f = open(filename, "rb").read() filename = os.path.join(path, os.path.basename(filename)) os.path.abspath(filename) # info("Compressing " + filename + "...") modtime, moddate = mkdostime(time.localtime(fileinfo.st_ctime)) compressed = Compress(f, compresstype) # Create the checksum for the file checksum = crc32(f) # Only set compressed if lower if len(compressed) < len(f): data = compressed # Otherwise just store it else: data = f compresstype = CompressionTypes.STORE.value # Create the file header header = structs.headerStruct.pack(b"\x50\x4b\x03\x04", version, flags, compresstype, modtime, moddate, checksum, len(data), fileinfo.st_size, len(filename), len(extra)) # Create the file header for the central directory centralheader = structs.centralHeader( b"\x50\x4b\x01\x02", 3 << 8 | 23, version, flags, compresstype, modtime, moddate, checksum, len(data), fileinfo.st_size, len(filename), len(extra), len("Comment"), 0, 1, 0, offset, ) # Create the central directory object with all metadata centraldirectory = { "centralheader": centralheader, "filename": filename, "extra": extra, "comment": "Comment" } # Return data file = header + bytes(filename, 'utf-8') + extra + data return (file, centraldirectory)
def create_install_pxe_archive(self): """ Create an oem install tar archive suitable for installing a disk image via the network using the PXE boot protocol. The archive contains the raw disk image and its checksum as well as an install initrd and kernel plus the required kernel commandline information which needs to be added as append line in the pxelinux config file on the boot server """ self.pxe_dir = mkdtemp( prefix='pxe-install-media.', dir=self.target_dir ) # the system image is transfered as xz compressed variant log.info('xz compressing disk image') pxe_image_filename = ''.join( [ self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.xz' ] ) compress = Compress( source_filename=self.diskname, keep_source_on_compress=True ) compress.xz() Command.run( ['mv', compress.compressed_filename, pxe_image_filename] ) # the system image transfer is checked against a checksum log.info('Creating disk image checksum') pxe_md5_filename = ''.join( [ self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.md5' ] ) checksum = Checksum(pxe_image_filename) checksum.md5(pxe_md5_filename) # create pxe config append information # this information helps to configure the boot server correctly append_filename = ''.join( [ self.pxe_dir, '/', self.xml_state.xml_data.get_name(), '.append' ] ) cmdline = 'pxe=1' custom_cmdline = self.xml_state.build_type.get_kernelcmdline() if custom_cmdline: cmdline += ' ' + custom_cmdline with open(append_filename, 'w') as append: append.write('%s\n' % cmdline) # create initrd for pxe install log.info('Creating pxe install boot image') self.__create_pxe_install_kernel_and_initrd() # create pxe install tarball log.info('Creating pxe install archive') archive = ArchiveTar( self.pxename.replace('.xz', '') ) archive.create_xz_compressed( self.pxe_dir )
def step_impl(context, extension): try: context.object = Compress(extension, context.filename) except Exception as e: context.exc = e