def test_clonezilla_image(self):
image = ClonezillaImage(
"/mnt/backup/clonezilla.focal/2020-08-30-15-img_mbr_many_different_fs/clonezilla-img"
)
# assert_true("message", False)
# assert_false("message", True)
image = ClonezillaImage(
"/mnt/backup/clonezilla.focal/2020-09-02-07-img_ntfsclone_partimage/clonezilla-img"
)
# sdf2.aa : Partimage
# sdf6.ntfs-img.aa : NTFS Clone
# sdf13.ext4-ptcl-img.gz.aa : Partclone ext4
# sdf13.dd-ptcl-img.gz.aa : Partclone dd (same as regular dd data, it would appear)
print("looking at " + str(image.dev_fs_dict))
def test_chs_sf_parsing(self):
chs_sf_string = """cylinders=12336
heads=255
sectors=2"""
chs_sf_dict = ClonezillaImage.parse_chs_sf_output(chs_sf_string)
expected_dict = {'cylinders': 12336, 'heads': 255, 'sectors': 2}
self.assertEqual(chs_sf_dict, expected_dict)
def test_compression_detection(self):
self.assertEqual(
"gzip",
ClonezillaImage.extract_image_compression_from_file_utility(
"sdf1.dd-ptcl-img.gz.aa: gzip compressed data, max speed, from Unix, original size modulo 2^32 268435456 gzip compressed data, reserved method, from FAT filesystem (MS-DOS, OS/2, NT), original size modulo 2^32 268435456"
))
self.assertEqual(
"bzip2",
ClonezillaImage.extract_image_compression_from_file_utility(
"sdd1.ext4-ptcl-img.bz2.aa: bzip2 compressed data, block size = 300k"
))
self.assertEqual(
"lzo",
ClonezillaImage.extract_image_compression_from_file_utility(
"sdb1.ext4-ptcl-img.lzo.aa: lzop compressed data - version 1.040, LZO1X-1, os: Unix"
))
self.assertEqual(
"lzma",
ClonezillaImage.extract_image_compression_from_file_utility(
"sdd1.ext4-ptcl-img.lzma.aa: LZMA compressed data, streamed"))
self.assertEqual(
"xz",
ClonezillaImage.extract_image_compression_from_file_utility(
"sdb1.ext4-ptcl-img.xz.aa: XZ compressed data:"))
self.assertEqual(
"lzip",
ClonezillaImage.extract_image_compression_from_file_utility(
"sdb1.ext4-ptcl-img.lzip.aa: lzip compressed data, version: 1")
)
self.assertEqual(
"lrzip",
ClonezillaImage.extract_image_compression_from_file_utility(
"sdb1.ext4-ptcl-img.lrz.aa: LRZIP compressed data - version 0.6"
))
self.assertEqual(
"lz4",
ClonezillaImage.extract_image_compression_from_file_utility(
"sdb1.ext4-ptcl-img.lz4.aa: LZ4 compressed data (v1.4+)"))
self.assertEqual(
"zstd",
ClonezillaImage.extract_image_compression_from_file_utility(
"sdb1.ext4-ptcl-img.zst.aa: Zstandard compressed data (v0.8+), Dictionary ID: None"
))
self.assertEqual(
"uncompressed",
ClonezillaImage.extract_image_compression_from_file_utility(
"sdb1.ext4-ptcl-img.uncomp.aa: data"))
def test_dev_fs_list_parsing(self):
dev_fs_list_string = """# This is a comment line
# Another comment line
/dev/sda3 ntfs
/dev/sda7 ext4
"""
dev_fs_dict = ClonezillaImage.parse_dev_fs_list_output(
dev_fs_list_string)
expected_dict = {"/dev/sda3": "ntfs", "/dev/sda7": "ext4"}
self.assertEqual(dev_fs_dict, expected_dict)
def scan_dummy_images_and_annotate(self, dir):
# Loops over the partitions listed in the 'parts' file
for image_key in self.image_format_dict_dict.keys():
short_partition_key = re.sub('/dev/', '', image_key)
# For standard MBR and GPT partitions, the partition key listed in the 'parts' file has a directly
# associated backup image, so check for this.
image_format_dict = ClonezillaImage.scan_backup_image(dir, short_partition_key, False)
if len(image_format_dict) > 0:
self.image_format_dict_dict[image_key].update(image_format_dict)
else:
# Expected for eg, extended partition.
print("Could not find " + short_partition_key + " in " + dir)
def scan_file(self, absolute_path, filename, enduser_filename):
print("Scan file " + absolute_path)
is_image = False
try:
image = None
if isfile(absolute_path):
# Identify Clonezilla images by presence of a file named "parts". Cannot use "clonezilla-img" or
# "dev-fs.list" because these files were not created by in earlier Clonezilla versions. Cannot use
# "disk" as Clonezilla's 'saveparts' function does not create it. But both 'savedisk' and 'saveparts'
# always creates a file named 'parts' across every version of Clonezilla tested.
error_suffix = ""
if absolute_path.endswith("parts"):
print("Found Clonezilla image " + filename)
image = ClonezillaImage(absolute_path, enduser_filename)
error_suffix = _(
"This can happen when loading images which Clonezilla was unable to completely backup. Any other filesystems within the image should be restorable as normal."
)
is_image = True
elif absolute_path.endswith(".backup"):
print(
"Found a legacy Redo Backup / Rescuezilla v1.0.5 image "
+ filename)
image = RedoBackupLegacyImage(absolute_path,
enduser_filename, filename)
error_suffix = _(
"Any other filesystems within the image should be restorable as normal."
)
is_image = True
if is_image:
image_warning_message = ""
for short_partition_key in image.warning_dict.keys():
image_warning_message += " " + short_partition_key + ": " + image.warning_dict[
short_partition_key] + "\n"
if len(image_warning_message) > 0:
self.failed_to_read_image_dict[absolute_path] = _(
"Unable to fully process the image associated with the following partitions:"
) + "\n" + image_warning_message + error_suffix
if image is not None:
self.image_dict[image.absolute_path] = image
except Exception as e:
print("Failed to read: " + absolute_path)
tb = traceback.format_exc()
self.failed_to_read_image_dict[enduser_filename] = tb
traceback.print_exc()
return is_image
def test_new_dev_fs_list_parsing(self):
dev_fs_list_string = """# <Device name> <File system> <Size>
# File system is got from ocs-get-part-info. It might be different from that of blkid or parted.
/dev/sda1 vfat 512M
/dev/sda3 swap 15.9G"""
dev_fs_dict = ClonezillaImage.parse_dev_fs_list_output(
dev_fs_list_string)
expected_dict = {
'/dev/sda1': {
'filesystem': "vfat",
'size': "512M"
},
'/dev/sda3': {
'filesystem': "swap",
'size': "15.9G"
}
}
self.assertEqual(dev_fs_dict, expected_dict)
def scan_file(self, absolute_path, enduser_filename):
print("Scan file " + absolute_path)
is_image = False
try:
temp_image_dict = {}
dirname = os.path.dirname(absolute_path)
if isfile(absolute_path) and dirname not in self.ignore_folder_set:
head, filename = os.path.split(absolute_path)
# Identify Clonezilla images by presence of a file named "parts". Cannot use "clonezilla-img" or
# "dev-fs.list" because these files were not created by in earlier Clonezilla versions. Cannot use
# "disk" as Clonezilla's 'saveparts' function does not create it. But both 'savedisk' and 'saveparts'
# always creates a file named 'parts' across every version of Clonezilla tested.
error_suffix = ""
# Ignore [/mnt/backup/]/bin/parts and [/mnt/backup/]/sbin/parts
if filename == "parts" and not filename == "bin" and not filename == "sbin":
print("Found Clonezilla image " + filename)
GLib.idle_add(
self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(
filename=absolute_path))
temp_image_dict = ClonezillaImage.get_clonezilla_image_dict(
absolute_path, enduser_filename)
error_suffix = _(
"This can happen when loading images which Clonezilla was unable to completely backup."
)
error_suffix += " " + _(
"Any other filesystems within the image should be restorable as normal."
)
# Only 1 Clonezilla image per folder, so consider the image scanned
is_image = True
elif absolute_path.endswith(".backup"):
# The legacy Redo Backup and Recovery v0.9.3-v1.0.4 format was adapted and extended Foxclone, so
# care is taken here to delineate the image formats by a simple heuristic: the existence of Foxclone's MBR backup.
foxclone_mbr = absolute_path.split(".backup")[0] + ".grub"
if os.path.exists(foxclone_mbr):
print("Found a Foxclone image " + filename)
GLib.idle_add(
self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(
filename=absolute_path))
temp_image_dict = {
absolute_path:
FoxcloneImage(absolute_path, enduser_filename,
filename)
}
error_suffix = _(
"Any other filesystems within the image should be restorable as normal."
)
is_image = True
else:
print(
"Found a legacy Redo Backup / Rescuezilla v1.0.5 image "
+ filename)
GLib.idle_add(
self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(
filename=absolute_path))
temp_image_dict = {
absolute_path:
RedoBackupLegacyImage(absolute_path,
enduser_filename, filename)
}
error_suffix = _(
"Any other filesystems within the image should be restorable as normal."
)
is_image = True
elif absolute_path.endswith(".redo"):
# The Redo Rescue format's metadata is a JSON file ending in .redo. Unfortunately this conflicts
# with the legacy Redo Backup and Recovery 0.9.2 format, which also uses a metadata file ending in
# .redo, so care is taken here to delineate the image formats by a simple heuristic: whether or not
# the file is valid JSON.
if RedoRescueImage.is_valid_json(absolute_path):
# ".redo" is used for Redo Rescue format and Redo Backup and Recovery 0.9.2 format
print("Found Redo Rescue image " + filename)
GLib.idle_add(
self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(
filename=absolute_path))
temp_image_dict = {
absolute_path:
RedoRescueImage(absolute_path, enduser_filename,
filename)
}
error_suffix = _(
"Any other filesystems within the image should be restorable as normal."
)
is_image = True
else:
print(
"Found a legacy Redo Backup and Recovery v0.9.2 image "
+ filename)
GLib.idle_add(
self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(
filename=absolute_path))
temp_image_dict = {
absolute_path:
RedoBackupLegacyImage(absolute_path,
enduser_filename, filename)
}
error_suffix = _(
"Any other filesystems within the image should be restorable as normal."
)
is_image = True
elif absolute_path.endswith(
".partitions"
) and not absolute_path.endswith(".minimum.partitions"):
print("Found FOG Project image " + filename)
GLib.idle_add(
self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(
filename=absolute_path))
temp_image_dict = {
absolute_path:
FogProjectImage(absolute_path, enduser_filename,
filename)
}
error_suffix = _(
"Any other filesystems within the image should be restorable as normal."
)
is_image = True
elif absolute_path.endswith(".fsa"):
print("Found FSArchiver image " + filename)
GLib.idle_add(
self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(
filename=absolute_path))
temp_image_dict = {
absolute_path:
FsArchiverImage(absolute_path, enduser_filename,
filename)
}
error_suffix = ""
is_image = True
elif ".apt." in absolute_path:
# Apart GTK images within a single folder are combined into one ApartGTKImage instance, so ensure
# the folder hasn't already been scanned.
print("Found Apart GTK image " + filename +
" (will include other images in the same folder)")
GLib.idle_add(
self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(
filename=absolute_path))
temp_image_dict = {
absolute_path: ApartGtkImage(absolute_path)
}
error_suffix = _(
"Any other filesystems within the image should be restorable as normal."
)
# Only 1 Apart GTK image per folder (which may contain a huge number of images, often of the
# same partition). Need to add image to the ignore fodler set to prevent double scanning
self.ignore_folder_set.add(dirname)
is_image = True
# If haven't found an image for this file, try scanning for QemuImages. Due to slow scan, do not look
# in subfolders
else:
is_qemu_candidate, extension = QemuImage.is_supported_extension(
filename)
if is_qemu_candidate:
# TODO: Considering skipping raw images, for speedup.
# is_raw = QemuImage.does_file_extension_refer_to_raw_image(extension)
if QemuImage.has_conflict_img_format_in_same_folder(
absolute_path, extension):
print(
"Not considering " + filename +
" as QemuImage as found exiting image it probably belongs to"
)
else:
print(
"Found an extension that should be compatible with qemu-nbd: "
+ filename)
timeout_seconds = 10
GLib.idle_add(
self.please_wait_popup.
set_secondary_label_text,
_("Scanning: {filename}").format(
filename=absolute_path) + " " +
_("({timeout_seconds} second timeout)").format(
timeout_seconds=timeout_seconds))
qemu_img = QemuImage(absolute_path,
enduser_filename,
timeout_seconds)
if qemu_img.has_initialized:
temp_image_dict = {absolute_path: qemu_img}
error_suffix = _(
"Support for virtual machine images is still experimental."
)
is_image = True
if is_image:
image_warning_message = ""
for key in temp_image_dict.keys():
for warning_dict_key in temp_image_dict[
key].warning_dict.keys():
image_warning_message += " " + warning_dict_key + ": "\
+ temp_image_dict[key].warning_dict[warning_dict_key] + "\n"
if len(image_warning_message) > 0:
self.failed_to_read_image_dict[absolute_path] = _(
"Unable to fully process the image associated with the following partitions:"
) + "\n" + image_warning_message + error_suffix
for key in temp_image_dict.keys():
self.image_dict[key] = temp_image_dict[key]
except Exception as e:
print("Failed to read: " + absolute_path)
tb = traceback.format_exc()
self.failed_to_read_image_dict[enduser_filename] = tb
traceback.print_exc()
return is_image
def scan_file(self, absolute_path, filename, enduser_filename):
print("Scan file " + absolute_path)
is_image = False
try:
temp_image_dict = {}
if isfile(absolute_path):
# Identify Clonezilla images by presence of a file named "parts". Cannot use "clonezilla-img" or
# "dev-fs.list" because these files were not created by in earlier Clonezilla versions. Cannot use
# "disk" as Clonezilla's 'saveparts' function does not create it. But both 'savedisk' and 'saveparts'
# always creates a file named 'parts' across every version of Clonezilla tested.
error_suffix = ""
if absolute_path.endswith("parts"):
print("Found Clonezilla image " + filename)
GLib.idle_add(self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(filename=absolute_path))
temp_image_dict = ClonezillaImage.get_clonezilla_image_dict(absolute_path, enduser_filename)
error_suffix = _(
"This can happen when loading images which Clonezilla was unable to completely backup. Any other filesystems within the image should be restorable as normal.")
is_image = True
elif absolute_path.endswith(".backup"):
# The legacy Redo Backup and Recovery v0.9.3-v1.0.4 format was adapted and extended Foxclone, so
# care is taken here to delineate the image formats by a simple heuristic: the existence of Foxclone's MBR backup.
foxclone_mbr = absolute_path.split(".backup")[0] + ".grub"
if os.path.exists(foxclone_mbr):
print("Found a Foxclone image " + filename)
GLib.idle_add(self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(filename=absolute_path))
temp_image_dict = {absolute_path: FoxcloneImage(absolute_path, enduser_filename, filename)}
error_suffix = _("Any other filesystems within the image should be restorable as normal.")
is_image = True
else:
print("Found a legacy Redo Backup / Rescuezilla v1.0.5 image " + filename)
GLib.idle_add(self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(filename=absolute_path))
temp_image_dict = {absolute_path: RedoBackupLegacyImage(absolute_path, enduser_filename, filename)}
error_suffix = _("Any other filesystems within the image should be restorable as normal.")
is_image = True
elif absolute_path.endswith(".redo"):
# The Redo Rescue format's metadata is a JSON file ending in .redo. Unfortunately this conflicts
# with the legacy Redo Backup and Recovery 0.9.2 format, which also uses a metadata file ending in
# .redo, so care is taken here to delineate the image formats by a simple heuristic: whether or not
# the file is valid JSON.
if RedoRescueImage.is_valid_json(absolute_path):
# ".redo" is used for Redo Rescue format and Redo Backup and Recovery 0.9.2 format
print("Found Redo Rescue image " + filename)
GLib.idle_add(self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(filename=absolute_path))
temp_image_dict = {absolute_path: RedoRescueImage(absolute_path, enduser_filename, filename)}
error_suffix = _("Any other filesystems within the image should be restorable as normal.")
is_image = True
else:
print("Found a legacy Redo Backup and Recovery v0.9.2 image " + filename)
GLib.idle_add(self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(filename=absolute_path))
temp_image_dict = {absolute_path: RedoBackupLegacyImage(absolute_path, enduser_filename, filename)}
error_suffix = _("Any other filesystems within the image should be restorable as normal.")
is_image = True
elif absolute_path.endswith(".partitions") and not absolute_path.endswith(".minimum.partitions"):
print("Found FOG Project image " + filename)
GLib.idle_add(self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(filename=absolute_path))
temp_image_dict = {absolute_path: FogProjectImage(absolute_path, enduser_filename, filename)}
error_suffix = _("Any other filesystems within the image should be restorable as normal.")
is_image = True
elif absolute_path.endswith(".fsa"):
print("Found FSArchiver image " + filename)
GLib.idle_add(self.please_wait_popup.set_secondary_label_text,
_("Scanning: {filename}").format(filename=absolute_path))
temp_image_dict = {absolute_path: FsArchiverImage(absolute_path, enduser_filename, filename)}
error_suffix = ""
is_image = True
elif QemuImage.is_supported_extension(filename):
print("Found an extension that should be compatible with qemu-nbd: " + filename)
print("Skipping: " + filename)
timeout_seconds = 10
GLib.idle_add(self.please_wait_popup.set_secondary_label_text,
_(f"Scanning: {filename} ({timeout_seconds} second timeout)").format(filename=absolute_path, timeout_seconds=timeout_seconds))
temp_image_dict = {absolute_path: QemuImage(absolute_path, enduser_filename, timeout_seconds)}
error_suffix = _("Support for virtual machine images is still experimental.")
is_image = True
if is_image:
image_warning_message = ""
for key in temp_image_dict.keys():
for warning_dict_key in temp_image_dict[key].warning_dict.keys():
image_warning_message += " " + warning_dict_key + ": "\
+ temp_image_dict[key].warning_dict[warning_dict_key] + "\n"
if len(image_warning_message) > 0:
self.failed_to_read_image_dict[absolute_path] = _(
"Unable to fully process the image associated with the following partitions:") + "\n" + image_warning_message + error_suffix
for key in temp_image_dict.keys():
self.image_dict[key] = temp_image_dict[key]
except Exception as e:
print("Failed to read: " + absolute_path)
tb = traceback.format_exc()
self.failed_to_read_image_dict[enduser_filename] = tb
traceback.print_exc()
return is_image