def test_differenceInRow(self):
input_data = ("5 1 9 5\r\n"
"7 5 3\r\n"
"2 4 6 8")
c = Checksum(input_data)
self.assertEqual(8, c.line_difference(0))
self.assertEqual(4, c.line_difference(1))
self.assertEqual(6, c.line_difference(2))
def create_install_iso(self):
"""
Create an install ISO from the disk_image as hybrid ISO
bootable via legacy BIOS, EFI and as disk from Stick
"""
self.media_dir = mkdtemp(prefix="install-media.", dir=self.target_dir)
# custom iso metadata
self.custom_iso_args = ["-V", '"KIWI Installation System"', "-A", self.mbrid.get_id()]
# the system image transfer is checked against a checksum
log.info("Creating disk image checksum")
self.squashed_contents = mkdtemp(prefix="install-squashfs.", dir=self.target_dir)
checksum = Checksum(self.diskname)
checksum.md5(self.squashed_contents + "/" + self.md5name)
# the kiwi initrd code triggers the install by trigger files
self.__create_iso_install_trigger_files()
# the system image is stored as squashfs embedded file
log.info("Creating squashfs embedded disk image")
Command.run(["cp", "-l", self.diskname, self.squashed_contents + "/" + self.squashed_diskname])
squashed_image_file = "".join([self.target_dir, "/", self.squashed_diskname, ".squashfs"])
squashed_image = FileSystemSquashFs(device_provider=None, root_dir=self.squashed_contents)
squashed_image.create_on_file(squashed_image_file)
Command.run(["mv", squashed_image_file, self.media_dir])
# setup bootloader config to boot the ISO via isolinux
log.info("Setting up install image bootloader configuration")
bootloader_config_isolinux = BootLoaderConfig("isolinux", self.xml_state, self.media_dir)
bootloader_config_isolinux.setup_install_boot_images(
mbrid=None, lookup_path=self.boot_image_task.boot_root_directory
)
bootloader_config_isolinux.setup_install_image_config(mbrid=None)
bootloader_config_isolinux.write()
# setup bootloader config to boot the ISO via EFI
bootloader_config_grub = BootLoaderConfig("grub2", self.xml_state, self.media_dir)
bootloader_config_grub.setup_install_boot_images(
mbrid=self.mbrid, lookup_path=self.boot_image_task.boot_root_directory
)
bootloader_config_grub.setup_install_image_config(mbrid=self.mbrid)
bootloader_config_grub.write()
# create initrd for install image
log.info("Creating install image boot image")
self.__create_iso_install_kernel_and_initrd()
# create iso filesystem from media_dir
log.info("Creating ISO filesystem")
iso_image = FileSystemIsoFs(device_provider=None, root_dir=self.media_dir, custom_args=self.custom_iso_args)
iso_header_offset = iso_image.create_on_file(self.isoname)
# make it hybrid
Iso.create_hybrid(iso_header_offset, self.mbrid, self.isoname)
def create(self):
supported_archives = Defaults.get_archive_image_types()
if self.requested_archive_type not in supported_archives:
raise KiwiArchiveSetupError('Unknown archive type: %s' %
self.requested_archive_type)
if self.requested_archive_type == 'tbz':
log.info('Creating XZ compressed tar archive')
archive = ArchiveTar(self.__target_file_for('tar'))
archive.create_xz_compressed(self.root_dir)
checksum = Checksum(self.filename)
log.info('--> Creating archive checksum')
checksum.md5(self.checksum)
self.result.add('root_archive', self.filename)
self.result.add('root_archive_checksum', self.checksum)
return self.result
def evaluate_checksum(test_name, plotfile, rtol=1.e-9, atol=1.e-40,
do_fields=True, do_particles=True):
'''Compare plotfile checksum with benchmark.
Read checksum from input plotfile, read benchmark
corresponding to test_name, and assert their equality.
@param test_name Name of test, as found between [] in .ini file.
@param plotfile Plotfile from which the checksum is computed.
@param rtol Relative tolerance for the comparison.
@param atol Absolute tolerance for the comparison.
@param do_fields Whether to compare fields in the checksum.
@param do_particles Whether to compare particles in the checksum.
'''
test_checksum = Checksum(test_name, plotfile, do_fields=do_fields,
do_particles=do_particles)
test_checksum.evaluate(rtol=rtol, atol=atol)
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains the raw disk image and its checksum
as well as an install initrd and kernel plus the required
kernel commandline information which needs to be added
as append line in the pxelinux config file on the boot
server
"""
self.pxe_dir = mkdtemp(prefix="pxe-install-media.", dir=self.target_dir)
# the system image is transfered as xz compressed variant
log.info("xz compressing disk image")
pxe_image_filename = "".join([self.pxe_dir, "/", self.xml_state.xml_data.get_name(), ".xz"])
compress = Compress(source_filename=self.diskname, keep_source_on_compress=True)
compress.xz()
Command.run(["mv", compress.compressed_filename, pxe_image_filename])
# the system image transfer is checked against a checksum
log.info("Creating disk image checksum")
pxe_md5_filename = "".join([self.pxe_dir, "/", self.xml_state.xml_data.get_name(), ".md5"])
checksum = Checksum(pxe_image_filename)
checksum.md5(pxe_md5_filename)
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = "".join([self.pxe_dir, "/", self.xml_state.xml_data.get_name(), ".append"])
cmdline = "pxe=1"
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += " " + custom_cmdline
with open(append_filename, "w") as append:
append.write("%s\n" % cmdline)
# create initrd for pxe install
log.info("Creating pxe install boot image")
self.__create_pxe_install_kernel_and_initrd()
# create pxe install tarball
log.info("Creating pxe install archive")
archive = ArchiveTar(self.pxename.replace(".xz", ""))
archive.create_xz_compressed(self.pxe_dir)
def send_packet(self, seq_num):
print("Sending ", self.get_next_seq_num(seq_num), "; Timer started")
if (not self.queue[seq_num]['data']):
self.queue[seq_num]['data'] = Util.randomString(self.segment_size)
checksum = Checksum.compute(self.queue[seq_num]['data'])
if (seq_num in self.checkEPackets):
checksum = "avalakipavalaki"
self.checkEPackets.pop(self.checkEPackets.index(seq_num))
print("Simulating wrong checksum for packet: ", seq_num)
packet = Packet(self.queue[seq_num]['data'], checksum, seq_num, False)
self.udp_helper.send(packet.getSerializedPacket(), self)
def test_parseInput(self):
input_data = ("5 1 9 5\r\n"
"7 5 3\r\n"
"2 4 6 8")
c = Checksum(input_data)
expected_data = [[5, 1, 9, 5],
[7, 5, 3],
[2, 4, 6, 8]]
self.assertEqual(expected_data, c.matrix)
def compute_checksum(self, datasetpath):
"""Compute checksum for file using specified data set path."""
try:
pathname = os.path.join(datasetpath, self.name)
result = Checksum(pathname)
self.checksum = result.checksum
self.size = result.size
except Exception: # pylint: disable=broad-except
# Just allow silent failure here - error is evident by checksum
# remaining None
pass
def create(self):
supported_archives = Defaults.get_archive_image_types()
if self.requested_archive_type not in supported_archives:
raise KiwiArchiveSetupError(
'Unknown archive type: %s' % self.requested_archive_type
)
if self.requested_archive_type == 'tbz':
log.info('Creating XZ compressed tar archive')
archive = ArchiveTar(
self.__target_file_for('tar')
)
archive.create_xz_compressed(self.root_dir)
checksum = Checksum(self.filename)
log.info('--> Creating archive checksum')
checksum.md5(self.checksum)
self.result.add(
'root_archive', self.filename
)
self.result.add(
'root_archive_checksum', self.checksum
)
return self.result
def reset_benchmark(test_name, plotfile, do_fields=True, do_particles=True):
'''Update the benchmark (overwrites reference json file).
Overwrite value of benchmark corresponding to
test_name with checksum read from input plotfile.
@param test_name Name of test, as found between [] in .ini file.
@param plotfile Plotfile from which the checksum is computed.
@param do_fields Whether to write field checksums in the benchmark.
@param do_particles Whether to write particles checksums in the benchmark.
'''
ref_checksum = Checksum(test_name, plotfile, do_fields=do_fields,
do_particles=do_particles)
ref_benchmark = Benchmark(test_name, ref_checksum.data)
ref_benchmark.reset()
def send_ack(self, seq_num, client_addr):
# Prepare Acknowledgement
packet = struct.pack('IH', seq_num, self.header)
# Calculate checksum
checksum = Checksum.calculate(packet)
# Add Checksum to the packet
packet = ctypes.create_string_buffer(8)
struct.pack_into('IHH', packet, 0, seq_num, checksum, self.header)
# Corrupt packet based on the probability
if random.random() <= self.packet_corrupt_probability:
packet[random.randint(0, len(packet) - 1)] = 0
# Send ACK
time.sleep(1)
self.server_socket.sendto(packet, client_addr)
self.logger.info('Sequence Number %d - Acknowledgement Sent', seq_num)
def execute(self):
# Read packets and process
while True:
message, client_addr = self.server_socket.recvfrom(5000)
packet = struct.unpack('IHH' + str(len(message) - 8) + 's', message)
# Drop packet
if random.random() <= self.packet_drop_probability:
self.logger.info('Sequence Number %d - Packet Dropped', packet[0])
continue
self.logger.info('Sequence Number %d - Packet Received', packet[0])
# Verify packet checksum
if not Checksum.verify(message):
self.logger.info('Sequence Number %d - Packet is Corrupt', packet[0])
continue
self.process(packet, client_addr)
def get_next_ack(self):
message = self.socket.recv(self.mss)
packet = struct.unpack('IHH', message)
# Drop ack based on probability
if random.random() <= self.packet_drop_probability:
self.logger.info(
'Acknowledgement Number %d - Acknowledgement Dropped',
packet[0])
return False
self.logger.info(
'Acknowledgement Number %d - Acknowledgement Received', packet[0])
# Packet corrupted, ignore
if not Checksum.verify(message):
self.logger.info(
'Acknowledgement Number %d - Acknowledgement Corrupted',
packet[0])
return False
return packet
def get_next_packet(self, seq_num):
if self.num_packets == 0:
return None
self.num_packets -= 1
# Create packet
data = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(self.mss - 8)).encode('ascii')
length = len(data)
# Create packet
packet = struct.pack('IH' + str(length) + 's', seq_num, self.header,
data)
# Calculate checksum
checksum = Checksum.calculate(packet)
# Add Checksum to the packet
packet = ctypes.create_string_buffer(self.mss)
struct.pack_into('IHH' + str(length) + 's', packet, 0, seq_num,
checksum, self.header, data)
return packet
def startListening(self):
while True:
try:
data, addr = self.serverSock.recvfrom(1024)
packet = Packet()
packet.deserializePacket(data)
# print('addr: ', addr)
if (self.simulatePacketLoss()):
print("Simulating packet loss for packet with seq no: ",
packet.seq_num)
continue
if (Checksum.verify(packet.data, packet.checksum)):
ack_num, discard = self.protocol.receive_packet(packet)
if (discard):
print("Discarding packet with sequence number " +
str(packet.seq_num))
else:
print("Received Segment: ", str(packet.seq_num))
_ = self.serverSock.sendto(str(ack_num).encode(),
addr) #sending ack with ack_num
print("ACK Sent: ", str(ack_num))
else:
print(
"Discarding packet with invalid checksum, packet no: ",
packet.seq_num)
except KeyboardInterrupt:
print('Interrupted')
os._exit(0)
except ConnectionResetError:
pass # Do nothing.
except Exception:
raise Exception
return
def parse_pkg(dt, dd, has_ver=True):
# Term: element <span class="term">
k = dt[0]
s = k.text.replace('\n', '')
# packages have a version number, patches do not
pat = r'([^(]+)\(([^)]+)\)' if has_ver else r'(.*) -'
m = re.match(pat, s)
if m:
name = m.group(1).strip()
version = None
if has_ver:
version = m.group(2)
else:
print(f'Term: unable to parse "{s}"')
# Element <span class="token">
k = k[0]
s = k.text
m = re.match(r'([0-9,]+)', s)
if m:
size = int(m.group(1).replace(',', ''))
else:
print(f'Token: unable to parse "{s}"')
# Definition
for k in dd:
if k.tag == 'p':
s = k.text.strip()
if s.startswith('Download:'):
kk = k[0]
url = kk.attrib['href']
if s.startswith('MD5 sum:'):
kk = k[0]
md5 = Checksum('md5', kk.text)
return Pkg(name, size, url, md5, version)
def test_totalChecksum(self):
input_data = ("5 1 9 5\r\n"
"7 5 3\r\n"
"2 4 6 8")
c = Checksum(input_data)
self.assertEqual(18, c.solve())
def create(self):
log.info('Creating PXE root filesystem image')
self.filesystem.create()
self.image = self.filesystem.filename
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz()
self.image = compress.compressed_filename
log.info('Creating PXE root filesystem MD5 checksum')
self.filesystem_checksum = self.filesystem.filename + '.md5'
checksum = Checksum(self.image)
checksum.md5(self.filesystem_checksum)
# prepare boot(initrd) root system
log.info('Creating PXE boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[self.image_name, '-', kernel_data.version, '.kernel']
)
kernel.copy_kernel(
self.target_dir, self.kernel_filename
)
else:
raise KiwiPxeBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# extract hypervisor from boot(initrd) root system
if self.machine and self.machine.get_domain() == 'dom0':
kernel_data = kernel.get_xen_hypervisor()
if kernel_data:
self.hypervisor_filename = ''.join(
[self.image_name, '-', kernel_data.name]
)
kernel.copy_xen_hypervisor(
self.target_dir, self.hypervisor_filename
)
self.result.add(
'xen_hypervisor', self.hypervisor_filename
)
else:
raise KiwiPxeBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory
)
# create initrd for pxe boot
self.boot_image_task.create_initrd()
self.result.add(
'kernel', self.kernel_filename
)
self.result.add(
'initrd', self.boot_image_task.initrd_filename
)
self.result.add(
'filesystem_image', self.image
)
self.result.add(
'filesystem_md5', self.filesystem_checksum
)
if self.pxedeploy:
log.warning(
'Creation of client config file from pxedeploy not implemented'
)
return self.result
def test_emptyInput(self):
input_data = ''
c = Checksum(input_data)
self.assertEqual(0, c.solve())
def create_install_pxe_archive(self):
"""
Create an oem install tar archive suitable for installing a
disk image via the network using the PXE boot protocol.
The archive contains the raw disk image and its checksum
as well as an install initrd and kernel plus the required
kernel commandline information which needs to be added
as append line in the pxelinux config file on the boot
server
"""
self.pxe_dir = mkdtemp(
prefix='pxe-install-media.', dir=self.target_dir
)
# the system image is transfered as xz compressed variant
log.info('xz compressing disk image')
pxe_image_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.xz'
]
)
compress = Compress(
source_filename=self.diskname,
keep_source_on_compress=True
)
compress.xz()
Command.run(
['mv', compress.compressed_filename, pxe_image_filename]
)
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
pxe_md5_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.md5'
]
)
checksum = Checksum(pxe_image_filename)
checksum.md5(pxe_md5_filename)
# create pxe config append information
# this information helps to configure the boot server correctly
append_filename = ''.join(
[
self.pxe_dir, '/',
self.xml_state.xml_data.get_name(), '.append'
]
)
cmdline = 'pxe=1'
custom_cmdline = self.xml_state.build_type.get_kernelcmdline()
if custom_cmdline:
cmdline += ' ' + custom_cmdline
with open(append_filename, 'w') as append:
append.write('%s\n' % cmdline)
# create initrd for pxe install
log.info('Creating pxe install boot image')
self.__create_pxe_install_kernel_and_initrd()
# create pxe install tarball
log.info('Creating pxe install archive')
archive = ArchiveTar(
self.pxename.replace('.xz', '')
)
archive.create_xz_compressed(
self.pxe_dir
)
def create(self):
log.info('Creating PXE root filesystem image')
self.filesystem.create()
self.image = self.filesystem.filename
if self.compressed:
log.info('xz compressing root filesystem image')
compress = Compress(self.image)
compress.xz()
self.image = compress.compressed_filename
log.info('Creating PXE root filesystem MD5 checksum')
self.filesystem_checksum = self.filesystem.filename + '.md5'
checksum = Checksum(self.image)
checksum.md5(self.filesystem_checksum)
# prepare boot(initrd) root system
log.info('Creating PXE boot image')
self.boot_image_task.prepare()
# export modprobe configuration to boot image
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory)
# extract kernel from boot(initrd) root system
kernel = Kernel(self.boot_image_task.boot_root_directory)
kernel_data = kernel.get_kernel()
if kernel_data:
self.kernel_filename = ''.join(
[self.image_name, '-', kernel_data.version, '.kernel'])
kernel.copy_kernel(self.target_dir, self.kernel_filename)
else:
raise KiwiPxeBootImageError(
'No kernel in boot image tree %s found' %
self.boot_image_task.boot_root_directory)
# extract hypervisor from boot(initrd) root system
if self.machine and self.machine.get_domain() == 'dom0':
kernel_data = kernel.get_xen_hypervisor()
if kernel_data:
self.hypervisor_filename = ''.join(
[self.image_name, '-', kernel_data.name])
kernel.copy_xen_hypervisor(self.target_dir,
self.hypervisor_filename)
self.result.add('xen_hypervisor', self.hypervisor_filename)
else:
raise KiwiPxeBootImageError(
'No hypervisor in boot image tree %s found' %
self.boot_image_task.boot_root_directory)
# create initrd for pxe boot
self.boot_image_task.create_initrd()
self.result.add('kernel', self.kernel_filename)
self.result.add('initrd', self.boot_image_task.initrd_filename)
self.result.add('filesystem_image', self.image)
self.result.add('filesystem_md5', self.filesystem_checksum)
if self.pxedeploy:
log.warning(
'Creation of client config file from pxedeploy not implemented'
)
return self.result
def create_install_iso(self):
"""
Create an install ISO from the disk_image as hybrid ISO
bootable via legacy BIOS, EFI and as disk from Stick
"""
self.media_dir = mkdtemp(
prefix='install-media.', dir=self.target_dir
)
# custom iso metadata
self.custom_iso_args = [
'-V', '"KIWI Installation System"',
'-A', self.mbrid.get_id()
]
# the system image transfer is checked against a checksum
log.info('Creating disk image checksum')
self.squashed_contents = mkdtemp(
prefix='install-squashfs.', dir=self.target_dir
)
checksum = Checksum(self.diskname)
checksum.md5(self.squashed_contents + '/' + self.md5name)
# the kiwi initrd code triggers the install by trigger files
self.__create_iso_install_trigger_files()
# the system image is stored as squashfs embedded file
log.info('Creating squashfs embedded disk image')
Command.run(
[
'cp', '-l', self.diskname,
self.squashed_contents + '/' + self.squashed_diskname
]
)
squashed_image_file = ''.join(
[
self.target_dir, '/', self.squashed_diskname, '.squashfs'
]
)
squashed_image = FileSystemSquashFs(
device_provider=None, root_dir=self.squashed_contents
)
squashed_image.create_on_file(squashed_image_file)
Command.run(
['mv', squashed_image_file, self.media_dir]
)
# setup bootloader config to boot the ISO via isolinux
log.info('Setting up install image bootloader configuration')
bootloader_config_isolinux = BootLoaderConfig(
'isolinux', self.xml_state, self.media_dir
)
bootloader_config_isolinux.setup_install_boot_images(
mbrid=None,
lookup_path=self.boot_image_task.boot_root_directory
)
bootloader_config_isolinux.setup_install_image_config(
mbrid=None
)
bootloader_config_isolinux.write()
# setup bootloader config to boot the ISO via EFI
bootloader_config_grub = BootLoaderConfig(
'grub2', self.xml_state, self.media_dir
)
bootloader_config_grub.setup_install_boot_images(
mbrid=self.mbrid,
lookup_path=self.boot_image_task.boot_root_directory
)
bootloader_config_grub.setup_install_image_config(
mbrid=self.mbrid
)
bootloader_config_grub.write()
# create initrd for install image
log.info('Creating install image boot image')
self.__create_iso_install_kernel_and_initrd()
# create iso filesystem from media_dir
log.info('Creating ISO filesystem')
iso_image = FileSystemIsoFs(
device_provider=None,
root_dir=self.media_dir,
custom_args=self.custom_iso_args
)
iso_header_offset = iso_image.create_on_file(self.isoname)
# make it hybrid
Iso.create_hybrid(
iso_header_offset, self.mbrid, self.isoname
)
