def md5(self, filename):
compressed_blocks = None
compress = Compress(self.source_filename)
if compress.get_format():
compressed_blocks = self.__block_list(
os.path.getsize(self.source_filename)
)
compress.uncompress(temporary=True)
blocks = self.__block_list(
os.path.getsize(compress.uncompressed_filename)
)
else:
blocks = self.__block_list(
os.path.getsize(self.source_filename)
)
with open(self.source_filename) as source:
checksum = hashlib.md5(source.read()).hexdigest()
with open(filename, 'w') as md5:
if compressed_blocks:
md5.write(
'%s %s %s %s %s\n' % (
checksum, blocks.blocks, blocks.blocksize,
compressed_blocks.blocks, compressed_blocks.blocksize
)
)
else:
md5.write(
'%s %s %s\n' % (
checksum, blocks.blocks, blocks.blocksize
)
)
def create_initrd(self, mbrid=None):
if self.is_prepared():
log.info('Creating initrd cpio archive')
initrd_file_name = ''.join(
[
self.target_dir, '/',
self.xml_state.xml_data.get_name(), '.initrd'
]
)
# we can't simply exclude boot when building the archive
# because the file boot/mbrid must be preserved. Because of
# that we create a copy of the boot directory and remove
# everything in boot/ except for boot/mbrid. The original
# boot directory should not be changed because we rely
# on other data in boot/ e.g the kernel to be available
# for the entire image building process
self.temp_boot_root_directory = mkdtemp()
Command.run(
[
'rsync', '-zav', self.boot_root_directory + '/',
self.temp_boot_root_directory
]
)
boot_directory = self.temp_boot_root_directory + '/boot'
Path.wipe(boot_directory)
if mbrid:
log.info(
'--> Importing mbrid: %s', mbrid.get_id()
)
Path.create(boot_directory)
image_identifier = boot_directory + '/mbrid'
mbrid.write(image_identifier)
cpio = ArchiveCpio(initrd_file_name)
# the following is a list of directories which were needed
# during the process of creating an image but not when the
# image is actually booting with this initrd
exclude_from_archive = [
'/var/cache', '/image', '/usr/lib/grub2'
]
cpio.create(
source_dir=self.temp_boot_root_directory,
exclude=exclude_from_archive
)
log.info(
'--> xz compressing archive'
)
compress = Compress(initrd_file_name)
compress.xz()
self.initrd_filename = compress.compressed_filename
def open_file():
filename = "water-stays-saved.jpeg"
try:
image = Compress(filename)
seam = image.best_seam()
image.remove_color_seam(seam)
image.save("water.jpeg")
except Exception:
raise
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains the raw disk image and its checksum
as well as an install initrd and kernel plus the required
kernel commandline information which needs to be added
as append line in the pxelinux config file on the boot
server
"""
self.pxe_dir = mkdtemp(prefix="pxe-install-media.", dir=self.target_dir)
# the system image is transfered as xz compressed variant
log.info("xz compressing disk image")
pxe_image_filename = "".join([self.pxe_dir, "/", self.xml_state.xml_data.get_name(), ".xz"])
compress = Compress(source_filename=self.diskname, keep_source_on_compress=True)
compress.xz()
Command.run(["mv", compress.compressed_filename, pxe_image_filename])
# the system image transfer is checked against a checksum
log.info("Creating disk image checksum")
pxe_md5_filename = "".join([self.pxe_dir, "/", self.xml_state.xml_data.get_name(), ".md5"])
checksum = Checksum(pxe_image_filename)
checksum.md5(pxe_md5_filename)
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = "".join([self.pxe_dir, "/", self.xml_state.xml_data.get_name(), ".append"])
cmdline = "pxe=1"
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += " " + custom_cmdline
with open(append_filename, "w") as append:
append.write("%s\n" % cmdline)
# create initrd for pxe install
log.info("Creating pxe install boot image")
self.__create_pxe_install_kernel_and_initrd()
# create pxe install tarball
log.info("Creating pxe install archive")
archive = ArchiveTar(self.pxename.replace(".xz", ""))
archive.create_xz_compressed(self.pxe_dir)
def md5(self, filename):
compressed_blocks = None
compress = Compress(self.source_filename)
if compress.get_format():
compressed_blocks = self.__block_list(
os.path.getsize(self.source_filename))
compress.uncompress(temporary=True)
blocks = self.__block_list(
os.path.getsize(compress.uncompressed_filename))
else:
blocks = self.__block_list(os.path.getsize(self.source_filename))
with open(self.source_filename) as source:
checksum = hashlib.md5(source.read()).hexdigest()
with open(filename, 'w') as md5:
if compressed_blocks:
md5.write(
'%s %s %s %s %s\n' %
(checksum, blocks.blocks, blocks.blocksize,
compressed_blocks.blocks, compressed_blocks.blocksize))
else:
md5.write('%s %s %s\n' %
(checksum, blocks.blocks, blocks.blocksize))
def create_initrd(self, mbrid=None):
if self.is_prepared():
log.info('Creating initrd cpio archive')
initrd_file_name = ''.join([
self.target_dir, '/',
self.xml_state.xml_data.get_name(), '.initrd'
])
# we can't simply exclude boot when building the archive
# because the file boot/mbrid must be preserved. Because of
# that we create a copy of the boot directory and remove
# everything in boot/ except for boot/mbrid. The original
# boot directory should not be changed because we rely
# on other data in boot/ e.g the kernel to be available
# for the entire image building process
self.temp_boot_root_directory = mkdtemp()
Command.run([
'rsync', '-zav', self.boot_root_directory + '/',
self.temp_boot_root_directory
])
boot_directory = self.temp_boot_root_directory + '/boot'
Path.wipe(boot_directory)
if mbrid:
log.info('--> Importing mbrid: %s', mbrid.get_id())
Path.create(boot_directory)
image_identifier = boot_directory + '/mbrid'
mbrid.write(image_identifier)
cpio = ArchiveCpio(initrd_file_name)
# the following is a list of directories which were needed
# during the process of creating an image but not when the
# image is actually booting with this initrd
exclude_from_archive = ['/var/cache', '/image', '/usr/lib/grub2']
cpio.create(source_dir=self.temp_boot_root_directory,
exclude=exclude_from_archive)
log.info('--> xz compressing archive')
compress = Compress(initrd_file_name)
compress.xz()
self.initrd_filename = compress.compressed_filename
def compress():
if request.method == "POST":
if request.files:
content_type = request.mimetype
image = request.files['image']
compression_rate = request.form['compression_rate']
image.save(os.path.join(app.config['uploads'], image.filename))
# print(compression_rate)
path = Compress(
os.path.join(app.config['uploads'], image.filename),
int(compression_rate))
encoded_img = get_response_image(
os.path.join(app.config['compress_path'], path))
uncompress_img = get_response_image(
os.path.join(app.config['compress_path'], "uncompressed.bmp"))
response = {
"message": "saved",
"encoded_img": encoded_img,
"uncompress_img": uncompress_img
}
return response, 200
def setUp(self):
self.c = Compress()
def create(self):
log.info('Creating PXE root filesystem image')
self.filesystem.create()
self.image = self.filesystem.filename
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz()
self.image = compress.compressed_filename
log.info('Creating PXE root filesystem MD5 checksum')
self.filesystem_checksum = self.filesystem.filename + '.md5'
checksum = Checksum(self.image)
checksum.md5(self.filesystem_checksum)
# prepare boot(initrd) root system
log.info('Creating PXE boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[self.image_name, '-', kernel_data.version, '.kernel']
)
kernel.copy_kernel(
self.target_dir, self.kernel_filename
)
else:
raise KiwiPxeBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# extract hypervisor from boot(initrd) root system
if self.machine and self.machine.get_domain() == 'dom0':
kernel_data = kernel.get_xen_hypervisor()
if kernel_data:
self.hypervisor_filename = ''.join(
[self.image_name, '-', kernel_data.name]
)
kernel.copy_xen_hypervisor(
self.target_dir, self.hypervisor_filename
)
self.result.add(
'xen_hypervisor', self.hypervisor_filename
)
else:
raise KiwiPxeBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# create initrd for pxe boot
self.boot_image_task.create_initrd()
self.result.add(
'kernel', self.kernel_filename
)
self.result.add(
'initrd', self.boot_image_task.initrd_filename
)
self.result.add(
'filesystem_image', self.image
)
self.result.add(
'filesystem_md5', self.filesystem_checksum
)
if self.pxedeploy:
log.warning(
'Creation of client config file from pxedeploy not implemented'
)
return self.result
def create(self):
log.info('Creating PXE root filesystem image')
self.filesystem.create()
self.image = self.filesystem.filename
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz()
self.image = compress.compressed_filename
log.info('Creating PXE root filesystem MD5 checksum')
self.filesystem_checksum = self.filesystem.filename + '.md5'
checksum = Checksum(self.image)
checksum.md5(self.filesystem_checksum)
# prepare boot(initrd) root system
log.info('Creating PXE boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[self.image_name, '-', kernel_data.version, '.kernel'])
kernel.copy_kernel(self.target_dir, self.kernel_filename)
else:
raise KiwiPxeBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory)
# extract hypervisor from boot(initrd) root system
if self.machine and self.machine.get_domain() == 'dom0':
kernel_data = kernel.get_xen_hypervisor()
if kernel_data:
self.hypervisor_filename = ''.join(
[self.image_name, '-', kernel_data.name])
kernel.copy_xen_hypervisor(self.target_dir,
self.hypervisor_filename)
self.result.add('xen_hypervisor', self.hypervisor_filename)
else:
raise KiwiPxeBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory)
# create initrd for pxe boot
self.boot_image_task.create_initrd()
self.result.add('kernel', self.kernel_filename)
self.result.add('initrd', self.boot_image_task.initrd_filename)
self.result.add('filesystem_image', self.image)
self.result.add('filesystem_md5', self.filesystem_checksum)
if self.pxedeploy:
log.warning(
'Creation of client config file from pxedeploy not implemented'
)
return self.result
class TestCompress(unittest.TestCase):
def setUp(self):
self.c = Compress()
def test_encode_word_the(self):
enc = self.c.encode_symbol('the')
binary = int(enc, 2)
self.assertEquals(binary, 19)
def test_encode_word_from(self):
enc = self.c.encode_symbol('from')
binary = int(enc, 2)
self.assertEquals(binary, 109)
def test_encode_word_home(self):
enc = self.c.encode_symbol('home')
binary = int(enc, 2)
self.assertEquals(binary, 233)
def test_encode_nonexistent_word(self):
enc = self.c.encode_symbol('no-such-word')
self.assertEquals(None, enc)
def test_encode_char_space(self):
enc = self.c.encode_symbol(' ')
binary = int(enc, 2)
self.assertEquals(binary, 7)
def test_encode_char_e(self):
enc = self.c.encode_symbol('e')
binary = int(enc, 2)
self.assertEquals(binary, 2)
def test_encode_char_w(self):
enc = self.c.encode_symbol('w')
binary = int(enc, 2)
self.assertEquals(binary, 48)
def test_encode_text(self):
enc = self.c.encode('this is')
binary = int(enc, 2)
self.assertEquals(binary, 183)
def test_encode_text_with_punctuation(self):
enc = self.c.encode('foo? is this bar!')
binary = int(enc, 2)
self.assertEquals(binary, 29686222816609210748006L)
def test_encode_decode_1(self):
s = 'you are really cool!'
enc = self.c.encode(s)
dec = self.c.decode(enc)
self.assertEquals(s, dec)
def test_encode_decode_2(self):
s = '? i wonder how . compressed it ! will, be !'
enc = self.c.encode(s)
dec = self.c.decode(enc)
self.assertEquals(s, dec)
def test_encode_decode_3(self):
s = 'xaajj tiko ax auui ahj nn nana nib'
enc = self.c.encode(s)
dec = self.c.decode(enc)
self.assertEquals(s, dec)
def setUp(self):
self.c = Compress()
class TestCompress(unittest.TestCase):
def setUp(self):
self.c = Compress()
def test_encode_word_the(self):
enc = self.c.encode_symbol('the')
binary = int(enc, 2)
self.assertEquals(binary, 19)
def test_encode_word_from(self):
enc = self.c.encode_symbol('from')
binary = int(enc, 2)
self.assertEquals(binary, 109)
def test_encode_word_home(self):
enc = self.c.encode_symbol('home')
binary = int(enc, 2)
self.assertEquals(binary, 233)
def test_encode_nonexistent_word(self):
enc = self.c.encode_symbol('no-such-word')
self.assertEquals(None, enc)
def test_encode_char_space(self):
enc = self.c.encode_symbol(' ')
binary = int(enc, 2)
self.assertEquals(binary, 7)
def test_encode_char_e(self):
enc = self.c.encode_symbol('e')
binary = int(enc, 2)
self.assertEquals(binary, 2)
def test_encode_char_w(self):
enc = self.c.encode_symbol('w')
binary = int(enc, 2)
self.assertEquals(binary, 48)
def test_encode_text(self):
enc = self.c.encode('this is')
binary = int(enc, 2)
self.assertEquals(binary, 183)
def test_encode_text_with_punctuation(self):
enc = self.c.encode('foo? is this bar!')
binary = int(enc, 2)
self.assertEquals(binary, 29686222816609210748006L)
def test_encode_decode_1(self):
s = 'you are really cool!'
enc = self.c.encode(s)
dec = self.c.decode(enc)
self.assertEquals(s, dec)
def test_encode_decode_2(self):
s = '? i wonder how . compressed it ! will, be !'
enc = self.c.encode(s)
dec = self.c.decode(enc)
self.assertEquals(s, dec)
def test_encode_decode_3(self):
s = 'xaajj tiko ax auui ahj nn nana nib'
enc = self.c.encode(s)
dec = self.c.decode(enc)
self.assertEquals(s, dec)
def test_run(self):
Compress('test/ori', 'test/res/' + now_str).run()
from compress import Compress
path = Compress('harry.jpg', 1)
print(path)
def create_recovery_archive(self):
"""
create a compressed recovery archive from the root tree
for use with kiwi's recvoery system. The method creates
additional data into the image root filesystem which is
deleted prior to the creation of a new recovery data set
"""
# cleanup
bash_comand = [
'rm', '-f', self.root_dir + '/recovery.*'
]
Command.run(['bash', '-c', ' '.join(bash_comand)])
if not self.oemconfig['recovery']:
return
# recovery.tar
log.info('Creating recovery tar archive')
metadata = {
'archive_name':
self.root_dir + '/recovery.tar',
'archive_filecount':
self.root_dir + '/recovery.tar.files',
'archive_size':
self.root_dir + '/recovery.tar.size',
'partition_size':
self.root_dir + '/recovery.partition.size',
'partition_filesystem':
self.root_dir + '/recovery.tar.filesystem'
}
recovery_archive = NamedTemporaryFile(
delete=False
)
archive = ArchiveTar(
filename=recovery_archive.name,
create_from_file_list=False
)
archive.create(
source_dir=self.root_dir,
exclude=['dev', 'proc', 'sys'],
options=[
'--numeric-owner',
'--hard-dereference',
'--preserve-permissions'
]
)
Command.run(
['mv', recovery_archive.name, metadata['archive_name']]
)
# recovery.tar.filesystem
recovery_filesystem = self.xml_state.build_type.get_filesystem()
with open(metadata['partition_filesystem'], 'w') as partfs:
partfs.write('%s' % recovery_filesystem)
log.info(
'--> Recovery partition filesystem: %s', recovery_filesystem
)
# recovery.tar.files
bash_comand = [
'tar', '-tf', metadata['archive_name'], '|', 'wc', '-l'
]
tar_files_call = Command.run(
['bash', '-c', ' '.join(bash_comand)]
)
tar_files_count = int(tar_files_call.output.rstrip('\n'))
with open(metadata['archive_filecount'], 'w') as files:
files.write('%d\n' % tar_files_count)
log.info(
'--> Recovery file count: %d files', tar_files_count
)
# recovery.tar.size
recovery_archive_size_bytes = os.path.getsize(metadata['archive_name'])
with open(metadata['archive_size'], 'w') as size:
size.write('%d' % recovery_archive_size_bytes)
log.info(
'--> Recovery uncompressed size: %d mbytes',
int(recovery_archive_size_bytes / 1048576)
)
# recovery.tar.gz
log.info('--> Compressing recovery archive')
compress = Compress(self.root_dir + '/recovery.tar')
compress.gzip()
# recovery.partition.size
recovery_archive_gz_size_mbytes = int(
os.path.getsize(metadata['archive_name'] + '.gz') / 1048576
)
recovery_partition_mbytes = recovery_archive_gz_size_mbytes \
+ Defaults.get_recovery_spare_mbytes()
with open(metadata['partition_size'], 'w') as gzsize:
gzsize.write('%d' % recovery_partition_mbytes)
log.info(
'--> Recovery partition size: %d mbytes',
recovery_partition_mbytes
)
# delete recovery archive if inplace recovery is requested
# In this mode the recovery archive is created at install time
# and not at image creation time. However the recovery metadata
# is preserved in order to be able to check if enough space
# is available on the disk to create the recovery archive.
if self.oemconfig['recovery_inplace']:
log.info(
'--> Inplace recovery requested, deleting archive'
)
Path.wipe(metadata['archive_name'] + '.gz')
import sys
sys.path.append('..')
from compress import Compress
files = [
'pat-0.pat',
#'pat-1.pat',
#'pat-2.pat',
#'pat-3.pat',
#'pat-4.pat',
'pat-5.pat',
'pat-6.pat',
'pat-7.pat',
'pat-8.pat'
]
print files
for file in files:
print '\n\n----------------' + file + '----------------'
with open(file, 'r') as fh:
comp = Compress(fh.readlines())
print 'COMPRESSED:'
for line in comp.compress():
print line
def add(filename: str, path: pathlib.Path, compresstype: int, offset: int):
version = 46 # Bzip2
flags = 0
extra = b""
centraldirectory = []
fileinfo = os.stat(filename)
f = open(filename, "rb").read()
filename = os.path.join(path, os.path.basename(filename))
os.path.abspath(filename)
# info("Compressing " + filename + "...")
modtime, moddate = mkdostime(time.localtime(fileinfo.st_ctime))
compressed = Compress(f, compresstype)
# Create the checksum for the file
checksum = crc32(f)
# Only set compressed if lower
if len(compressed) < len(f):
data = compressed
# Otherwise just store it
else:
data = f
compresstype = CompressionTypes.STORE.value
# Create the file header
header = structs.headerStruct.pack(b"\x50\x4b\x03\x04", version, flags,
compresstype, modtime, moddate,
checksum, len(data), fileinfo.st_size,
len(filename), len(extra))
# Create the file header for the central directory
centralheader = structs.centralHeader(
b"\x50\x4b\x01\x02",
3 << 8 | 23,
version,
flags,
compresstype,
modtime,
moddate,
checksum,
len(data),
fileinfo.st_size,
len(filename),
len(extra),
len("Comment"),
0,
1,
0,
offset,
)
# Create the central directory object with all metadata
centraldirectory = {
"centralheader": centralheader,
"filename": filename,
"extra": extra,
"comment": "Comment"
}
# Return data
file = header + bytes(filename, 'utf-8') + extra + data
return (file, centraldirectory)
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains the raw disk image and its checksum
as well as an install initrd and kernel plus the required
kernel commandline information which needs to be added
as append line in the pxelinux config file on the boot
server
"""
self.pxe_dir = mkdtemp(
prefix='pxe-install-media.', dir=self.target_dir
)
# the system image is transfered as xz compressed variant
log.info('xz compressing disk image')
pxe_image_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.xz'
]
)
compress = Compress(
source_filename=self.diskname,
keep_source_on_compress=True
)
compress.xz()
Command.run(
['mv', compress.compressed_filename, pxe_image_filename]
)
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
pxe_md5_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.md5'
]
)
checksum = Checksum(pxe_image_filename)
checksum.md5(pxe_md5_filename)
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.append'
]
)
cmdline = 'pxe=1'
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += ' ' + custom_cmdline
with open(append_filename, 'w') as append:
append.write('%s\n' % cmdline)
# create initrd for pxe install
log.info('Creating pxe install boot image')
self.__create_pxe_install_kernel_and_initrd()
# create pxe install tarball
log.info('Creating pxe install archive')
archive = ArchiveTar(
self.pxename.replace('.xz', '')
)
archive.create_xz_compressed(
self.pxe_dir
)
def step_impl(context, extension):
try:
context.object = Compress(extension, context.filename)
except Exception as e:
context.exc = e
