def add_disk(self, disk_type, device_type, disk, qemu_type="raw"): """ Add a disk to the libvirt file :param disk_type: A string containing the type of disk (file, block) :param device_type: A string containing the type of device (disk, cdrom) :param disk: The path to the disk :param qemu_type: A string containing the format of disk (e.g., raw) :return: """ xml_type = "%s_%s" % (disk_type, device_type) if device_type in self.disk_ids: self.disk_ids[device_type] += 1 else: self.disk_ids[device_type] = ord('a') f = os.path.join( self.config_dir, "%s-%s-%s" % (disk_type, device_type, LibvirtFile.TEMPLATE_FILE)) if not os.path.exists(f): cziso.abort("Unable to find libvirt file %s" % f) values = { 'id': chr(self.disk_ids[device_type]), xml_type: os.path.realpath(disk), 'qemu_type': qemu_type } self.disk_xmls.append(cziso.fill_template(f, **values))
def _get_disk_info(self): """ Find the provided image partitions on local host and disk size :return: A tuple containing the disk size and string array containing location of image partitions """ mount = self.get_mount() if self.get_mount() is None: self.logger.error("Disk is not mounted") return None, None out, rc = cziso.run_command("fdisk -l %s" % mount) if rc != 0: cziso.abort("Unable to run fdisk command") for line in out: if self.size_gb == 0: matcher = re.search("^Disk \S+: ([\d\.]+) GB", line) if matcher is not None: size_gb_float = float(matcher.group(1)) self.size_gb = int(math.ceil(size_gb_float)) self.logger.info("Disk size is %i GB" % self.size_gb) matcher = re.match("^(%s\S+).*\s+(\S+)$" % mount, line) if matcher: if matcher.group(2) == "Linux": self.partitions.append(matcher.group(1)) if self.size_gb == 0: cziso.abort("Unable to find disk size of %s" % self) return self.size_gb, self.partitions
def create_dir_md5sums_file(self, folder_id, name): """ Create a new md5sum file for the specified directory :param folder_id: The Google drive id for the folder :return: A string containing the path to a newly created md5sum file """ query = "(not name contains 'md5sums.txt') and mimeType != 'application/vnd.google-apps.folder' and '%s' in parents" % folder_id results = self.drive.files().list( q=query, fields=GdriveAuth.FILE_FIELDS).execute() items = results.get('files', []) if not items: return None md5sum_filename = "%s-md5sums.txt" % name md5sum_filepath = os.path.join(self.temp_dir, md5sum_filename) self.logger.debug("Creating temporary md5sum file %s" % md5sum_filepath) f = open(md5sum_filepath, "w") if f is None: cziso.abort("Unable to write md5sums to file %s" % md5sum_filepath) for item in items: self.logger.debug('Adding %s %s' % (item['md5Checksum'], item['name'])) f.write("%s %s\n" % (item['md5Checksum'], item['name'])) f.close() return (md5sum_filepath, md5sum_filename)
def __init__(self, config): self.logger = logging.getLogger(self.__module__) self.temp_dir = config.get("cziso", "temp_directory") self.service_account_credentials = config.get_path( "google", "service_account_credentials") self.chunk_size = int(config.get("google", "chunk_size")) self.default_drive_dir_id = config.get("google", "default_drive_id") try: import apiclient.discovery from oauth2client.service_account import ServiceAccountCredentials self.credentials = ServiceAccountCredentials.from_json_keyfile_name( self.service_account_credentials, GdriveAuth.SCOPE) http_auth = self.credentials.authorize(httplib2.Http()) self.drive = apiclient.discovery.build('drive', 'v3', http=http_auth, cache_discovery=False) except Exception as e: error = """ Problem authenticating to Google Drive: %s To use the upload function, you must have the Python Google APIs installed. Please see the following URL to see how to install the APIs: https://developers.google.com/api-client-library/python/start/installation """ cziso.abort(error % str(e)) self.logger.debug("Successfully imported Google API")
def add_to_libvirt(self, libvirt): """ Mount the image if necessary and add disk to libvirt config file :param libvirt: An object of type virtualmachine.LibvirtFile """ if not self.mount(libvirt=True): cziso.abort("Unable to mount image %s" % self) libvirt.add_disk(self.get_disk_type(), "disk", self.get_mount(libvirt=True), self.get_qemu_type())
def parse_image_size_from_iso_filename(filename): """ Parse and return the original image size from a restore ISO filename. :param filename: The restore ISO filename :return: An integer representing the original image size in GB """ matcher = re.search("\.(\d+)G\.", filename) if matcher is None: cziso.abort("Unable to parse image size from restore ISO filename") return int(matcher.group(1))
def run(self, config, args): arg_vals = self.parse_args(args) cziso.abort_if_no_x() image = cziso.image.Image.factory(arg_vals["image"]) target_image = None if arg_vals["target-image"] is not None: target_image = cziso.image.Image.factory(arg_vals["target-image"]) if not image.exists(): cziso.abort("Image %s does not exists" % image) cz = cziso.clonezilla.Clonezilla(config) cz.modify_image(image, target_image)
def run(self, config, args): arg_vals = self.parse_args(args) try: __import__("cziso.gdrive") except ImportError as e: cziso.abort("Missing %s" % str(e)) gdrive = cziso.gdrive.GdriveAuth(config) gdrive.upload(arg_vals["file"], arg_vals["filename"], arg_vals["gdrive_folder"], self.is_arg_true(arg_vals["revision"]), arg_vals["description"])
def is_mapped(self): out, rc = cziso.run_command("rocks list host storagemap %s" % self.nas) if rc != 0: cziso.abort("Unable to list current storagemap") mapped_pattern = "^%s\s+%s.*\s+(\S*mapped)\s+.*" % (self.vol, self.pool) self.logger.debug("Looking for pattern '%s'" % mapped_pattern) for line in out: matcher = re.search(mapped_pattern, line) if matcher is not None: mapped_status = matcher.group(1) self.logger.debug("Status of vol %s is %s" % (self.vol, mapped_status)) return mapped_status == "mapped" return False
def factory(image): """ Return the correct subclass instance for specific image type. Currently supports RAW and ZFS vol images. :param image: A string containing an image URI :return: """ if ZfsVol.match(image): return ZfsVol(image) elif QemuImg.match(image): return QemuImg(image) else: cziso.abort("Image type of %s is not supported" % image)
def fsck(self): """ Runs fsck on disk to repair any issues :return: True if successful; otherwise False """ self.logger.info("Running fsck on disk partitions") self._get_disk_info() if not self.partitions: cziso.abort("Unable to find any partitions on disk") for partition in self.partitions: out, rc = cziso.run_command("fsck -y %s" % partition) if rc != 0: self.unmount() cziso.abort("Problem running fsck -y on partition: %s" % "\n".join(out)) self.logger.debug("fsck output: %s" % "\n".join(out))
def update(self, zip_path, out_dir=os.getcwd()): """ Generate new Clonezilla Live ISOs for both the custom and regular cases. :param zip_path: A string containing the path to a Clonezilla zip release :param out_dir: A string containing a path to where the ISOs should be written :return: A tuple containing the path to the regular and custom ISOs """ self.logger.info("Generating custom ISO for %s" % zip_path) cz_version = os.path.splitext(os.path.basename(zip_path))[0] tmp = self.create_temp_directory() self.logger.debug("Created temporary directory %s" % tmp) # unpack zip zip_dir = os.path.join(tmp, "zip") out, rc = cziso.run_command("unzip %s -d %s" % (zip_path, zip_dir)) if rc != 0: cziso.abort("Unable to unzip %s: %s" % (zip_path, "\n".join(out))) # generate regular ISO regular_iso = os.path.join(out_dir, "%s-regular.iso" % cz_version) cziso.generate_iso(self.genisoimage_command, zip_dir, regular_iso) # customize file isolinux_file = os.path.join(zip_dir, "syslinux", "isolinux.cfg") if not os.path.exists(isolinux_file): cziso.abort("Unable to find %s" % isolinux_file) self.logger.info("Editing %s" % isolinux_file) cziso.file_edit(isolinux_file, Clonezilla.CUSTOMIZATIONS) # generate custom ISO custom_iso = os.path.join(out_dir, "%s-custom.iso" % cz_version) cziso.generate_iso(self.genisoimage_command, zip_dir, custom_iso) # cleanup self.logger.debug("Removing temporary directory %s" % tmp) shutil.rmtree(tmp) return regular_iso, custom_iso
def __init__(self, image): """ Constructor for ZFS vol backed VM image. :param image: The URI of the image of zfs://nas/pool/vol format """ Image.__init__(self, image, "block", "raw") matcher = ZfsVol.match(image) self.nas = matcher.group(1) self.pool = matcher.group(2) self.vol = matcher.group(3) self.logger.debug( "Creating ZfsVol instance for vol %s in pool %s at %s" % (self.vol, self.pool, self.nas)) out, rc = cziso.run_command( "rocks report host attr localhost attr=hostname") if rc != 0: cziso.abort("Unable to determine physical hostname") self.hostname = out[0] self.mountpoint = None
def restore_clonezilla_iso(self, iso_file, image): """ Restpre a Clonezilla VM ISO file :param iso_file: Path to Clonezilla ISO file to restore :param image: Destination for restored image of type Image :return: Returns if successful; otherwise aborts """ self.logger.info("Restoring image %s to image %s" % (iso_file, image)) if not os.path.exists(iso_file): cziso.abort("ISO file %s does not exist" % iso_file) # launch Clonezilla libvirt_file = cziso.virtualmachine.LibvirtFile(self.config.config_dir) libvirt_file.add_disk("file", "cdrom", iso_file) image.add_to_libvirt(libvirt_file) vm = cziso.virtualmachine.VM() status = vm.launch(libvirt_file.get_xml()) if status != 0: cziso.abort("Unable to launch Clonezilla Live VM") # run restore iso script expect_path = cziso.fill_template(self.restore_expect, tmp_dir=self.temp_dir, vm_name=libvirt_file.get_name()) self.logger.info("""Running restore expect script -- it may take a few mins to boot the Clonezilla Live VM before you see any output""") subprocess.call("expect %s" % expect_path, shell=True) # cleanup vm.clean() image.unmount() os.remove(expect_path) self.logger.info("Restored image %s is now ready" % image)
def convert_to_clonezilla_iso(self, image, out_dir, network): """ Create a Clonezilla ISO file from specified image :param image: Path to raw image file to convert :param ip: IP address to use for Clonezilla VM (default: from Rocks) :param netmask: Netmask to use for Clonezilla VM (default: from Rocks) :return: Returns if successful; otherwise aborts """ if not image.exists(): cziso.abort("Image file %s does not exist" % image) if out_dir is not None and not os.path.exists(out_dir): cziso.abort("Output directory %s does not exist" % out_dir) self.logger.info("Converting image %s to iso" % image) # mount raw image and check it if not image.mount(): cziso.abort("Unable to mount input image %s" % image) image.fsck() image.unmount() # mount temp directory to place iso when complete tmp = self.create_temp_directory() ip, netmask = None, None if network is None: ip, netmask = cziso.get_free_ip(self.priv_interface) else: ip, netmask = network.split(":") if netmask is None or ip is None: cziso.abort("Unable to create a NFS export. No ip or netmask") cziso.create_nfs_export(tmp, ip) # check that we don't overwrite an existing ISO file generated_iso_filename = Clonezilla.get_cz_restore_iso_filename(image) generated_iso_path = os.path.join(tmp, generated_iso_filename) # insert the disk size into the file name new_iso_filename = Clonezilla.get_cziso_restore_iso_filename(image) candidate_dst_file = os.path.join(self.temp_dir, new_iso_filename) if out_dir is not None: candidate_dst_file = os.path.join(out_dir, new_iso_filename) dst_file = cziso.increment_filename(candidate_dst_file) # launch Clonezilla libvirt_file = cziso.virtualmachine.LibvirtFile(self.config.config_dir) libvirt_file.add_disk("file", "cdrom", self.clonezilla_custom.get_or_download()) image.add_to_libvirt(libvirt_file) libvirt_file.set_interface(self.priv_interface) vm = cziso.virtualmachine.VM() status = vm.launch(libvirt_file.get_xml()) if status != 0: cziso.abort("Unable to launch Clonezilla Live VM") # run create iso script expect_path = cziso.fill_template(self.create_expect, tmp_dir=tmp, temp_dir=tmp, vm_name=libvirt_file.get_name(), ip=ip, netmask=netmask, vm_id=image.get_image_id()) self.logger.info( """Running expect script to execute gen-rec-iso script -- it may take a few mins to boot the Clonezilla Live VM before you see any output""") subprocess.call("expect %s" % expect_path, shell=True) if os.path.exists(generated_iso_path): self.logger.debug("Moving ISO file %s to %s" % (generated_iso_path, dst_file)) os.rename(generated_iso_path, dst_file) self.logger.info("Clonezilla restore ISO file is now ready at %s" % dst_file) else: self.logger.error("Clonezilla did not generate ISO file") # cleanup vm.clean() cziso.remove_nfs_export(tmp, ip) shutil.rmtree(tmp) image.unmount()
def upload(self, file_path, filename=None, folder_id=None, revision=False, description=None): """ Upload specified file to Google drive folder :param file_path: A string containing the path to the file to upload :param filename: A string containing a different name of the file on Google drive (default: filename of uploading file) :param folder_id: The Google drive id for the folder to upload to :param revision: A boolean value that is True if the file being uploaded already exists in Google drive and this is a new revision of file; otherwise False. :param description: A description for the Google drive file :return: A string containing the Google drive id for file """ if not os.path.exists(file_path): cziso.abort("File %s does not exist" % file_path) if filename is None: filename = os.path.basename(file_path) if folder_id is None: folder_id = self.default_drive_dir_id folder_metadata = self.get_metadata(folder_id) if folder_metadata is None: cziso.abort("Google Drive folder %s does not exist" % folder_id) self.logger.info("Uploading file %s to Gdrive %s" % (file_path, folder_id)) existing_file = self.get_file(filename, folder_id) if existing_file is not None and revision is False: cziso.abort( "File %s already exists in drive folder %s. %s" % (filename, folder_id, "Please re-run with revision=true to upload as new revision")) request, media = self._request_create_or_update( existing_file, file_path, folder_id, filename, description) id = self._upload_file(request, media) self.logger.info("Upload Complete!") self.logger.info("Google drive id for %s is %s" % (file_path, id)) self.logger.info("Generating new md5sum for directory %s" % folder_id) (md5sum_filepath, md5sum_filename) = self.create_dir_md5sums_file( folder_id, folder_metadata['name']) existing_file = self.get_file(md5sum_filename, folder_id) request, media = self._request_create_or_update( existing_file, md5sum_filepath, folder_id, md5sum_filename) md5sum_id = self._upload_file(request, media) if md5sum_id is not None: self.logger.info("md5sum file uploaded to directory %s" % folder_id) else: self.logger.error("Unable to upload md5sum file to directory %s" % folder_id) os.remove(md5sum_filepath) return id