def _parse_bmap(self): """ Parse the bmap file and initialize corresponding class instance attributs. """ try: self._xml = ElementTree.parse(self._f_bmap) except ElementTree.ParseError as err: raise Error("cannot parse the bmap file '%s' which should be a " \ "proper XML file: %s" % (self._bmap_path, err)) xml = self._xml self.bmap_version = str(xml.getroot().attrib.get('version')) # Make sure we support this version major = int(self.bmap_version.split('.', 1)[0]) if major > SUPPORTED_BMAP_VERSION: raise Error("only bmap format version up to %d is supported, " \ "version %d is not supported" \ % (SUPPORTED_BMAP_VERSION, major)) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) self.blocks_cnt = int(xml.find("BlocksCount").text.strip()) self.mapped_cnt = int(xml.find("MappedBlocksCount").text.strip()) self.image_size = int(xml.find("ImageSize").text.strip()) self.image_size_human = human_size(self.image_size) self.mapped_size = self.mapped_cnt * self.block_size self.mapped_size_human = human_size(self.mapped_size) self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt blocks_cnt = (self.image_size + self.block_size - 1) // self.block_size if self.blocks_cnt != blocks_cnt: raise Error("Inconsistent bmap - image size does not match " \ "blocks count (%d bytes != %d blocks * %d bytes)" \ % (self.image_size, self.blocks_cnt, self.block_size))
def _parse_bmap(self): """ Parse the bmap file and initialize corresponding class instance attributs. """ try: self._xml = ElementTree.parse(self._f_bmap) except ElementTree.ParseError as err: raise Error("cannot parse the bmap file '%s' which should be a " \ "proper XML file: %s" % (self._bmap_path, err)) xml = self._xml self.bmap_version = str(xml.getroot().attrib.get('version')) # Make sure we support this version major = int(self.bmap_version.split('.', 1)[0]) if major > SUPPORTED_BMAP_VERSION: raise Error("only bmap format version up to %d is supported, " \ "version %d is not supported" \ % (SUPPORTED_BMAP_VERSION, major)) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) self.blocks_cnt = int(xml.find("BlocksCount").text.strip()) self.mapped_cnt = int(xml.find("MappedBlocksCount").text.strip()) self.image_size = int(xml.find("ImageSize").text.strip()) self.image_size_human = human_size(self.image_size) self.mapped_size = self.mapped_cnt * self.block_size self.mapped_size_human = human_size(self.mapped_size) self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size if self.blocks_cnt != blocks_cnt: raise Error("Inconsistent bmap - image size does not match " \ "blocks count (%d bytes != %d blocks * %d bytes)" \ % (self.image_size, self.blocks_cnt, self.block_size))
def _initialize_sizes(self, image_size): """ This function is only used when the there is no bmap. It initializes attributes like 'blocks_cnt', 'mapped_cnt', etc. Normally, the values are read from the bmap file, but in this case they are just set to something reasonable. """ self.image_size = image_size self.image_size_human = human_size(image_size) self.blocks_cnt = self.image_size + self.block_size - 1 self.blocks_cnt /= self.block_size self.mapped_cnt = self.blocks_cnt self.mapped_size = self.image_size self.mapped_size_human = self.image_size_human
def __init__(self, image, dest, bmap=None): """ The same as the constructor of the 'BmapCopy' base class, but adds useful guard-checks specific to block devices. """ # Call the base class constructor first BmapCopy.__init__(self, image, dest, bmap) self._batch_bytes = 1024 * 1024 self._batch_blocks = self._batch_bytes / self.block_size self._batch_queue_len = 6 self._dest_fsync_watermark = (6 * 1024 * 1024) / self.block_size self._sysfs_base = None self._sysfs_scheduler_path = None self._sysfs_max_ratio_path = None self._old_scheduler_value = None self._old_max_ratio_value = None # If the image size is known (i.e., it is not compressed) - check that # it fits the block device. if self.image_size: try: bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: raise Error("cannot seed block device '%s': %s " \ % (self._dest_path, err.strerror)) if bdev_size < self.image_size: raise Error("the image file '%s' has size %s and it will not " \ "fit the block device '%s' which has %s capacity" \ % (self._image_path, self.image_size_human, self._dest_path, human_size(bdev_size))) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev self._sysfs_base = "/sys/dev/block/%s:%s/" \ % (os.major(st_rdev), os.minor(st_rdev)) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we # need to go one level up in the sysfs hierarchy. try: if not os.path.exists(self._sysfs_base + "queue"): self._sysfs_base = self._sysfs_base + "../" except OSError: # No problem, this is just an optimization. pass self._sysfs_scheduler_path = self._sysfs_base + "queue/scheduler" self._sysfs_max_ratio_path = self._sysfs_base + "bdi/max_ratio"
def __init__(self, image, dest, bmap = None): """ The same as the constructor of the 'BmapCopy' base class, but adds useful guard-checks specific to block devices. """ # Call the base class constructor first BmapCopy.__init__(self, image, dest, bmap) self._batch_bytes = 1024 * 1024 self._batch_blocks = self._batch_bytes / self.block_size self._batch_queue_len = 6 self._dest_fsync_watermark = (6 * 1024 * 1024) / self.block_size self._sysfs_base = None self._sysfs_scheduler_path = None self._sysfs_max_ratio_path = None self._old_scheduler_value = None self._old_max_ratio_value = None # If the image size is known (i.e., it is not compressed) - check that # it fits the block device. if self.image_size: try: bdev_size = os.lseek(self._f_dest.fileno(), 0, os.SEEK_END) os.lseek(self._f_dest.fileno(), 0, os.SEEK_SET) except OSError as err: raise Error("cannot seed block device '%s': %s " \ % (self._dest_path, err.strerror)) if bdev_size < self.image_size: raise Error("the image file '%s' has size %s and it will not " \ "fit the block device '%s' which has %s capacity" \ % (self._image_path, self.image_size_human, self._dest_path, human_size(bdev_size))) # Construct the path to the sysfs directory of our block device st_rdev = os.fstat(self._f_dest.fileno()).st_rdev self._sysfs_base = "/sys/dev/block/%s:%s/" \ % (os.major(st_rdev), os.minor(st_rdev)) # Check if the 'queue' sub-directory exists. If yes, then our block # device is entire disk. Otherwise, it is a partition, in which case we # need to go one level up in the sysfs hierarchy. try: if not os.path.exists(self._sysfs_base + "queue"): self._sysfs_base = self._sysfs_base + "../" except OSError: # No problem, this is just an optimization. pass self._sysfs_scheduler_path = self._sysfs_base + "queue/scheduler" self._sysfs_max_ratio_path = self._sysfs_base + "bdi/max_ratio"
def _set_image_size(self, image_size): """ Set image size and initialize various other geometry-related attributes. """ if self.image_size is not None and self.image_size != image_size: raise Error("cannot set image size to %d bytes, it is known to " \ "be %d bytes (%s)" % (image_size, self.image_size, self.image_size_human)) self.image_size = image_size self.image_size_human = human_size(image_size) self.blocks_cnt = (self.image_size + self.block_size - 1) // self.block_size if self.mapped_cnt is None: self.mapped_cnt = self.blocks_cnt self.mapped_size = self.image_size self.mapped_size_human = self.image_size_human
def generate(self, include_checksums=True): """ Generate bmap for the image file. If 'include_checksums' is 'True', also generate checksums for block ranges. """ # Save image file position in order to restore it at the end image_pos = self._f_image.tell() self._bmap_file_start() # Generate the block map and write it to the XML block map # file as we go. self.mapped_cnt = 0 for first, last in self.filemap.get_mapped_ranges(0, self.blocks_cnt): self.mapped_cnt += last - first + 1 if include_checksums: chksum = self._calculate_chksum(first, last) chksum = " chksum=\"%s\"" % chksum else: chksum = "" if first != last: self._f_bmap.write(" <Range%s> %s-%s </Range>\n" % (chksum, first, last)) else: self._f_bmap.write(" <Range%s> %s </Range>\n" % (chksum, first)) self.mapped_size = self.mapped_cnt * self.block_size self.mapped_size_human = human_size(self.mapped_size) self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt self._bmap_file_end() try: self._f_bmap.flush() except IOError as err: raise Error("cannot flush the bmap file '%s': %s" % (self._bmap_path, err)) self._f_image.seek(image_pos)
def __init__(self, image, bmap, chksum_type="sha256"): """ Initialize a class instance: * image - full path or a file-like object of the image to create bmap for * bmap - full path or a file object to use for writing the resulting bmap to * chksum - type of the check sum to use in the bmap file (all checksum types which python's "hashlib" module supports are allowed). """ self.image_size = None self.image_size_human = None self.block_size = None self.blocks_cnt = None self.mapped_cnt = None self.mapped_size = None self.mapped_size_human = None self.mapped_percent = None self._mapped_count_pos1 = None self._mapped_count_pos2 = None self._chksum_pos = None self._f_image_needs_close = False self._f_bmap_needs_close = False self._cs_type = chksum_type.lower() try: self._cs_len = len(hashlib.new(self._cs_type).hexdigest()) except ValueError as err: raise Error("cannot initialize hash function \"%s\": %s" % (self._cs_type, err)) if hasattr(image, "read"): self._f_image = image self._image_path = image.name else: self._image_path = image self._open_image_file() if hasattr(bmap, "read"): self._f_bmap = bmap self._bmap_path = bmap.name else: self._bmap_path = bmap self._open_bmap_file() try: self.filemap = Filemap.filemap(self._f_image) except (Filemap.Error, Filemap.ErrorNotSupp) as err: raise Error("cannot generate bmap for file '%s': %s" % (self._image_path, err)) self.image_size = self.filemap.image_size self.image_size_human = human_size(self.image_size) if self.image_size == 0: raise Error("cannot generate bmap for zero-sized image file '%s'" % self._image_path) self.block_size = self.filemap.block_size self.blocks_cnt = self.filemap.blocks_cnt
def _parse_bmap(self): """ Parse the bmap file and initialize corresponding class instance attributs. """ try: self._xml = ElementTree.parse(self._f_bmap) except ElementTree.ParseError as err: # Extrace the erroneous line with some context self._f_bmap.seek(0) xml_extract = "" for num, line in enumerate(self._f_bmap): if num >= err.position[0] - 4 and num <= err.position[0] + 4: xml_extract += "Line %d: %s" % (num, line) raise Error("cannot parse the bmap file '%s' which should be a " "proper XML file: %s, the XML extract:\n%s" % (self._bmap_path, err, xml_extract)) xml = self._xml self.bmap_version = str(xml.getroot().attrib.get('version')) # Make sure we support this version self.bmap_version_major = int(self.bmap_version.split('.', 1)[0]) self.bmap_version_minor = int(self.bmap_version.split('.', 1)[1]) if self.bmap_version_major > SUPPORTED_BMAP_VERSION: raise Error("only bmap format version up to %d is supported, " "version %d is not supported" % (SUPPORTED_BMAP_VERSION, self.bmap_version_major)) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) self.blocks_cnt = int(xml.find("BlocksCount").text.strip()) self.mapped_cnt = int(xml.find("MappedBlocksCount").text.strip()) self.image_size = int(xml.find("ImageSize").text.strip()) self.image_size_human = human_size(self.image_size) self.mapped_size = self.mapped_cnt * self.block_size self.mapped_size_human = human_size(self.mapped_size) self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size if self.blocks_cnt != blocks_cnt: raise Error("Inconsistent bmap - image size does not match " "blocks count (%d bytes != %d blocks * %d bytes)" % (self.image_size, self.blocks_cnt, self.block_size)) if self.bmap_version_major > 1 or \ (self.bmap_version_major == 1 and self.bmap_version_minor == 4): # In bmap format version 1.0-1.3 the only supported checksum type # was SHA1. Version 2.0 started supporting arbitrary checksum # types. A new "ChecksumType" tag was introduce to specify the # checksum function name. And all XML tags which contained "sha1" # in their name were renamed to something more neutral. This was an # change incompatible with previous formats. # # There is a special format version 1.4, which should not have been # ever issued, but was released by a mistake. The mistake was that # when implementing version 2.0 support we mistakenly gave it # version number 1.4. This was later on fixed and format version # 1.4 became version 2.0. So 1.4 and 2.0 formats are identical. # # Note, bmap files did not contain checksums prior to version 1.3. self._cs_type = xml.find("ChecksumType").text.strip() self._cs_attrib_name = "chksum" self._bmap_cs_attrib_name = "BmapFileChecksum" elif self.bmap_version_minor == 3: self._cs_type = "sha1" self._cs_attrib_name = "sha1" self._bmap_cs_attrib_name = "BmapFileSHA1" if self._cs_type: try: self._cs_len = len(hashlib.new(self._cs_type).hexdigest()) except ValueError as err: raise Error("cannot initialize hash function \"%s\": %s" % (self._cs_type, err)) self._verify_bmap_checksum()
def _parse_bmap(self): """ Parse the bmap file and initialize corresponding class instance attributs. """ try: self._xml = ElementTree.parse(self._f_bmap) except ElementTree.ParseError as err: # Extrace the erroneous line with some context self._f_bmap.seek(0) xml_extract = "" for num, line in enumerate(self._f_bmap): if num >= err.position[0] - 4 and num <= err.position[0] + 4: xml_extract += "Line %d: %s" % (num, line) raise Error("cannot parse the bmap file '%s' which should be a " "proper XML file: %s, the XML extract:\n%s" % (self._bmap_path, err, xml_extract)) xml = self._xml self.bmap_version = str(xml.getroot().attrib.get('version')) # Make sure we support this version self.bmap_version_major = int(self.bmap_version.split('.', 1)[0]) self.bmap_version_minor = int(self.bmap_version.split('.', 1)[1]) if self.bmap_version_major > SUPPORTED_BMAP_VERSION: raise Error("only bmap format version up to %d is supported, " "version %d is not supported" % (SUPPORTED_BMAP_VERSION, self.bmap_version_major)) # Fetch interesting data from the bmap XML file self.block_size = int(xml.find("BlockSize").text.strip()) self.blocks_cnt = int(xml.find("BlocksCount").text.strip()) self.mapped_cnt = int(xml.find("MappedBlocksCount").text.strip()) self.image_size = int(xml.find("ImageSize").text.strip()) self.image_size_human = human_size(self.image_size) self.mapped_size = self.mapped_cnt * self.block_size self.mapped_size_human = human_size(self.mapped_size) self.mapped_percent = (self.mapped_cnt * 100.0) / self.blocks_cnt blocks_cnt = (self.image_size + self.block_size - 1) / self.block_size if self.blocks_cnt != blocks_cnt: raise Error("Inconsistent bmap - image size does not match " "blocks count (%d bytes != %d blocks * %d bytes)" % (self.image_size, self.blocks_cnt, self.block_size)) if self.bmap_version_major >= 1 and self.bmap_version_minor >= 3: # Bmap file checksum appeard in format 1.3 and the only supported # checksum type was SHA1. Version 1.4 started supporting arbitrary # checksum types. A new "ChecksumType" tag was introduce to specify # the checksum function name. And all XML tags which contained # "sha1" in their name were renamed to something more neutral. if self.bmap_version_minor == 3: self._cs_type = "sha1" self._cs_attrib_name = "sha1" else: self._cs_type = xml.find("ChecksumType").text.strip() self._cs_attrib_name = "chksum" try: self._cs_len = len(hashlib.new(self._cs_type).hexdigest()) except ValueError as err: raise Error("cannot initialize hash function \"%s\": %s" % (self._cs_type, err)) self._verify_bmap_checksum()
def __init__(self, image, bmap, chksum_type="sha256", log=None): """ Initialize a class instance: * image - full path or a file-like object of the image to create bmap for * bmap - full path or a file object to use for writing the resulting bmap to * chksum - type of the check sum to use in the bmap file (all checksum types which python's "hashlib" module supports are allowed). * log - the logger object to use for printing messages. """ self._log = log if self._log is None: self._log = logging.getLogger(__name__) self.image_size = None self.image_size_human = None self.block_size = None self.blocks_cnt = None self.mapped_cnt = None self.mapped_size = None self.mapped_size_human = None self.mapped_percent = None self._mapped_count_pos1 = None self._mapped_count_pos2 = None self._chksum_pos = None self._f_image_needs_close = False self._f_bmap_needs_close = False self._cs_type = chksum_type.lower() try: self._cs_len = len(hashlib.new(self._cs_type).hexdigest()) except ValueError as err: raise Error("cannot initialize hash function \"%s\": %s" % (self._cs_type, err)) if hasattr(image, "read"): self._f_image = image self._image_path = image.name else: self._image_path = image self._open_image_file() if hasattr(bmap, "read"): self._f_bmap = bmap self._bmap_path = bmap.name else: self._bmap_path = bmap self._open_bmap_file() try: self.filemap = Filemap.filemap(self._f_image, self._log) except (Filemap.Error, Filemap.ErrorNotSupp) as err: raise Error("cannot generate bmap for file '%s': %s" % (self._image_path, err)) self.image_size = self.filemap.image_size self.image_size_human = human_size(self.image_size) if self.image_size == 0: raise Error("cannot generate bmap for zero-sized image file '%s'" % self._image_path) self.block_size = self.filemap.block_size self.blocks_cnt = self.filemap.blocks_cnt