def add_disk(self, disk_type, device_type, disk, qemu_type="raw"):
"""
Add a disk to the libvirt file
:param disk_type: A string containing the type of disk (file, block)
:param device_type: A string containing the type of device (disk, cdrom)
:param disk: The path to the disk
:param qemu_type: A string containing the format of disk (e.g., raw)
:return:
"""
xml_type = "%s_%s" % (disk_type, device_type)
if device_type in self.disk_ids:
self.disk_ids[device_type] += 1
else:
self.disk_ids[device_type] = ord('a')
f = os.path.join(
self.config_dir,
"%s-%s-%s" % (disk_type, device_type, LibvirtFile.TEMPLATE_FILE))
if not os.path.exists(f):
cziso.abort("Unable to find libvirt file %s" % f)
values = {
'id': chr(self.disk_ids[device_type]),
xml_type: os.path.realpath(disk),
'qemu_type': qemu_type
}
self.disk_xmls.append(cziso.fill_template(f, **values))
def _get_disk_info(self):
"""
Find the provided image partitions on local host and disk size
:return: A tuple containing the disk size and string array containing
location of image partitions
"""
mount = self.get_mount()
if self.get_mount() is None:
self.logger.error("Disk is not mounted")
return None, None
out, rc = cziso.run_command("fdisk -l %s" % mount)
if rc != 0:
cziso.abort("Unable to run fdisk command")
for line in out:
if self.size_gb == 0:
matcher = re.search("^Disk \S+: ([\d\.]+) GB", line)
if matcher is not None:
size_gb_float = float(matcher.group(1))
self.size_gb = int(math.ceil(size_gb_float))
self.logger.info("Disk size is %i GB" % self.size_gb)
matcher = re.match("^(%s\S+).*\s+(\S+)$" % mount, line)
if matcher:
if matcher.group(2) == "Linux":
self.partitions.append(matcher.group(1))
if self.size_gb == 0:
cziso.abort("Unable to find disk size of %s" % self)
return self.size_gb, self.partitions
def create_dir_md5sums_file(self, folder_id, name):
"""
Create a new md5sum file for the specified directory
:param folder_id: The Google drive id for the folder
:return: A string containing the path to a newly created md5sum file
"""
query = "(not name contains 'md5sums.txt') and mimeType != 'application/vnd.google-apps.folder' and '%s' in parents" % folder_id
results = self.drive.files().list(
q=query, fields=GdriveAuth.FILE_FIELDS).execute()
items = results.get('files', [])
if not items:
return None
md5sum_filename = "%s-md5sums.txt" % name
md5sum_filepath = os.path.join(self.temp_dir, md5sum_filename)
self.logger.debug("Creating temporary md5sum file %s" %
md5sum_filepath)
f = open(md5sum_filepath, "w")
if f is None:
cziso.abort("Unable to write md5sums to file %s" % md5sum_filepath)
for item in items:
self.logger.debug('Adding %s %s' %
(item['md5Checksum'], item['name']))
f.write("%s %s\n" % (item['md5Checksum'], item['name']))
f.close()
return (md5sum_filepath, md5sum_filename)
def __init__(self, config):
self.logger = logging.getLogger(self.__module__)
self.temp_dir = config.get("cziso", "temp_directory")
self.service_account_credentials = config.get_path(
"google", "service_account_credentials")
self.chunk_size = int(config.get("google", "chunk_size"))
self.default_drive_dir_id = config.get("google", "default_drive_id")
try:
import apiclient.discovery
from oauth2client.service_account import ServiceAccountCredentials
self.credentials = ServiceAccountCredentials.from_json_keyfile_name(
self.service_account_credentials, GdriveAuth.SCOPE)
http_auth = self.credentials.authorize(httplib2.Http())
self.drive = apiclient.discovery.build('drive',
'v3',
http=http_auth,
cache_discovery=False)
except Exception as e:
error = """
Problem authenticating to Google Drive: %s
To use the upload function, you must have the Python Google APIs installed.
Please see the following URL to see how to install the APIs:
https://developers.google.com/api-client-library/python/start/installation
"""
cziso.abort(error % str(e))
self.logger.debug("Successfully imported Google API")
def add_to_libvirt(self, libvirt):
"""
Mount the image if necessary and add disk to libvirt config file
:param libvirt: An object of type virtualmachine.LibvirtFile
"""
if not self.mount(libvirt=True):
cziso.abort("Unable to mount image %s" % self)
libvirt.add_disk(self.get_disk_type(), "disk",
self.get_mount(libvirt=True), self.get_qemu_type())
def parse_image_size_from_iso_filename(filename):
"""
Parse and return the original image size from a restore ISO filename.
:param filename: The restore ISO filename
:return: An integer representing the original image size in GB
"""
matcher = re.search("\.(\d+)G\.", filename)
if matcher is None:
cziso.abort("Unable to parse image size from restore ISO filename")
return int(matcher.group(1))
def run(self, config, args):
arg_vals = self.parse_args(args)
cziso.abort_if_no_x()
image = cziso.image.Image.factory(arg_vals["image"])
target_image = None
if arg_vals["target-image"] is not None:
target_image = cziso.image.Image.factory(arg_vals["target-image"])
if not image.exists():
cziso.abort("Image %s does not exists" % image)
cz = cziso.clonezilla.Clonezilla(config)
cz.modify_image(image, target_image)
def run(self, config, args):
arg_vals = self.parse_args(args)
try:
__import__("cziso.gdrive")
except ImportError as e:
cziso.abort("Missing %s" % str(e))
gdrive = cziso.gdrive.GdriveAuth(config)
gdrive.upload(arg_vals["file"], arg_vals["filename"],
arg_vals["gdrive_folder"],
self.is_arg_true(arg_vals["revision"]),
arg_vals["description"])
def is_mapped(self):
out, rc = cziso.run_command("rocks list host storagemap %s" % self.nas)
if rc != 0:
cziso.abort("Unable to list current storagemap")
mapped_pattern = "^%s\s+%s.*\s+(\S*mapped)\s+.*" % (self.vol,
self.pool)
self.logger.debug("Looking for pattern '%s'" % mapped_pattern)
for line in out:
matcher = re.search(mapped_pattern, line)
if matcher is not None:
mapped_status = matcher.group(1)
self.logger.debug("Status of vol %s is %s" %
(self.vol, mapped_status))
return mapped_status == "mapped"
return False
def factory(image):
"""
Return the correct subclass instance for specific image type.
Currently supports RAW and ZFS vol images.
:param image: A string containing an image URI
:return:
"""
if ZfsVol.match(image):
return ZfsVol(image)
elif QemuImg.match(image):
return QemuImg(image)
else:
cziso.abort("Image type of %s is not supported" % image)
def fsck(self):
"""
Runs fsck on disk to repair any issues
:return: True if successful; otherwise False
"""
self.logger.info("Running fsck on disk partitions")
self._get_disk_info()
if not self.partitions:
cziso.abort("Unable to find any partitions on disk")
for partition in self.partitions:
out, rc = cziso.run_command("fsck -y %s" % partition)
if rc != 0:
self.unmount()
cziso.abort("Problem running fsck -y on partition: %s" %
"\n".join(out))
self.logger.debug("fsck output: %s" % "\n".join(out))
def update(self, zip_path, out_dir=os.getcwd()):
"""
Generate new Clonezilla Live ISOs for both the custom and regular
cases.
:param zip_path: A string containing the path to a Clonezilla zip
release
:param out_dir: A string containing a path to where the ISOs should
be written
:return: A tuple containing the path to the regular and custom ISOs
"""
self.logger.info("Generating custom ISO for %s" % zip_path)
cz_version = os.path.splitext(os.path.basename(zip_path))[0]
tmp = self.create_temp_directory()
self.logger.debug("Created temporary directory %s" % tmp)
# unpack zip
zip_dir = os.path.join(tmp, "zip")
out, rc = cziso.run_command("unzip %s -d %s" % (zip_path, zip_dir))
if rc != 0:
cziso.abort("Unable to unzip %s: %s" % (zip_path, "\n".join(out)))
# generate regular ISO
regular_iso = os.path.join(out_dir, "%s-regular.iso" % cz_version)
cziso.generate_iso(self.genisoimage_command, zip_dir, regular_iso)
# customize file
isolinux_file = os.path.join(zip_dir, "syslinux", "isolinux.cfg")
if not os.path.exists(isolinux_file):
cziso.abort("Unable to find %s" % isolinux_file)
self.logger.info("Editing %s" % isolinux_file)
cziso.file_edit(isolinux_file, Clonezilla.CUSTOMIZATIONS)
# generate custom ISO
custom_iso = os.path.join(out_dir, "%s-custom.iso" % cz_version)
cziso.generate_iso(self.genisoimage_command, zip_dir, custom_iso)
# cleanup
self.logger.debug("Removing temporary directory %s" % tmp)
shutil.rmtree(tmp)
return regular_iso, custom_iso
def __init__(self, image):
"""
Constructor for ZFS vol backed VM image.
:param image: The URI of the image of zfs://nas/pool/vol format
"""
Image.__init__(self, image, "block", "raw")
matcher = ZfsVol.match(image)
self.nas = matcher.group(1)
self.pool = matcher.group(2)
self.vol = matcher.group(3)
self.logger.debug(
"Creating ZfsVol instance for vol %s in pool %s at %s" %
(self.vol, self.pool, self.nas))
out, rc = cziso.run_command(
"rocks report host attr localhost attr=hostname")
if rc != 0:
cziso.abort("Unable to determine physical hostname")
self.hostname = out[0]
self.mountpoint = None
def restore_clonezilla_iso(self, iso_file, image):
"""
Restpre a Clonezilla VM ISO file
:param iso_file: Path to Clonezilla ISO file to restore
:param image: Destination for restored image of type Image
:return: Returns if successful; otherwise aborts
"""
self.logger.info("Restoring image %s to image %s" % (iso_file, image))
if not os.path.exists(iso_file):
cziso.abort("ISO file %s does not exist" % iso_file)
# launch Clonezilla
libvirt_file = cziso.virtualmachine.LibvirtFile(self.config.config_dir)
libvirt_file.add_disk("file", "cdrom", iso_file)
image.add_to_libvirt(libvirt_file)
vm = cziso.virtualmachine.VM()
status = vm.launch(libvirt_file.get_xml())
if status != 0:
cziso.abort("Unable to launch Clonezilla Live VM")
# run restore iso script
expect_path = cziso.fill_template(self.restore_expect,
tmp_dir=self.temp_dir,
vm_name=libvirt_file.get_name())
self.logger.info("""Running restore expect script -- it may take a few
mins to boot the Clonezilla Live VM before you see any output""")
subprocess.call("expect %s" % expect_path, shell=True)
# cleanup
vm.clean()
image.unmount()
os.remove(expect_path)
self.logger.info("Restored image %s is now ready" % image)
def convert_to_clonezilla_iso(self, image, out_dir, network):
"""
Create a Clonezilla ISO file from specified image
:param image: Path to raw image file to convert
:param ip: IP address to use for Clonezilla VM (default: from Rocks)
:param netmask: Netmask to use for Clonezilla VM (default: from Rocks)
:return: Returns if successful; otherwise aborts
"""
if not image.exists():
cziso.abort("Image file %s does not exist" % image)
if out_dir is not None and not os.path.exists(out_dir):
cziso.abort("Output directory %s does not exist" % out_dir)
self.logger.info("Converting image %s to iso" % image)
# mount raw image and check it
if not image.mount():
cziso.abort("Unable to mount input image %s" % image)
image.fsck()
image.unmount()
# mount temp directory to place iso when complete
tmp = self.create_temp_directory()
ip, netmask = None, None
if network is None:
ip, netmask = cziso.get_free_ip(self.priv_interface)
else:
ip, netmask = network.split(":")
if netmask is None or ip is None:
cziso.abort("Unable to create a NFS export. No ip or netmask")
cziso.create_nfs_export(tmp, ip)
# check that we don't overwrite an existing ISO file
generated_iso_filename = Clonezilla.get_cz_restore_iso_filename(image)
generated_iso_path = os.path.join(tmp, generated_iso_filename)
# insert the disk size into the file name
new_iso_filename = Clonezilla.get_cziso_restore_iso_filename(image)
candidate_dst_file = os.path.join(self.temp_dir, new_iso_filename)
if out_dir is not None:
candidate_dst_file = os.path.join(out_dir, new_iso_filename)
dst_file = cziso.increment_filename(candidate_dst_file)
# launch Clonezilla
libvirt_file = cziso.virtualmachine.LibvirtFile(self.config.config_dir)
libvirt_file.add_disk("file", "cdrom",
self.clonezilla_custom.get_or_download())
image.add_to_libvirt(libvirt_file)
libvirt_file.set_interface(self.priv_interface)
vm = cziso.virtualmachine.VM()
status = vm.launch(libvirt_file.get_xml())
if status != 0:
cziso.abort("Unable to launch Clonezilla Live VM")
# run create iso script
expect_path = cziso.fill_template(self.create_expect,
tmp_dir=tmp,
temp_dir=tmp,
vm_name=libvirt_file.get_name(),
ip=ip,
netmask=netmask,
vm_id=image.get_image_id())
self.logger.info(
"""Running expect script to execute gen-rec-iso script -- it may
take a few mins to boot the Clonezilla Live VM before you see any output""")
subprocess.call("expect %s" % expect_path, shell=True)
if os.path.exists(generated_iso_path):
self.logger.debug("Moving ISO file %s to %s" %
(generated_iso_path, dst_file))
os.rename(generated_iso_path, dst_file)
self.logger.info("Clonezilla restore ISO file is now ready at %s" %
dst_file)
else:
self.logger.error("Clonezilla did not generate ISO file")
# cleanup
vm.clean()
cziso.remove_nfs_export(tmp, ip)
shutil.rmtree(tmp)
image.unmount()
def upload(self,
file_path,
filename=None,
folder_id=None,
revision=False,
description=None):
"""
Upload specified file to Google drive folder
:param file_path: A string containing the path to the file to upload
:param filename: A string containing a different name of the file on
Google drive (default: filename of uploading file)
:param folder_id: The Google drive id for the folder to upload to
:param revision: A boolean value that is True if the file being uploaded
already exists in Google drive and this is a new revision of file;
otherwise False.
:param description: A description for the Google drive file
:return: A string containing the Google drive id for file
"""
if not os.path.exists(file_path):
cziso.abort("File %s does not exist" % file_path)
if filename is None:
filename = os.path.basename(file_path)
if folder_id is None:
folder_id = self.default_drive_dir_id
folder_metadata = self.get_metadata(folder_id)
if folder_metadata is None:
cziso.abort("Google Drive folder %s does not exist" % folder_id)
self.logger.info("Uploading file %s to Gdrive %s" %
(file_path, folder_id))
existing_file = self.get_file(filename, folder_id)
if existing_file is not None and revision is False:
cziso.abort(
"File %s already exists in drive folder %s. %s" %
(filename, folder_id,
"Please re-run with revision=true to upload as new revision"))
request, media = self._request_create_or_update(
existing_file, file_path, folder_id, filename, description)
id = self._upload_file(request, media)
self.logger.info("Upload Complete!")
self.logger.info("Google drive id for %s is %s" % (file_path, id))
self.logger.info("Generating new md5sum for directory %s" % folder_id)
(md5sum_filepath, md5sum_filename) = self.create_dir_md5sums_file(
folder_id, folder_metadata['name'])
existing_file = self.get_file(md5sum_filename, folder_id)
request, media = self._request_create_or_update(
existing_file, md5sum_filepath, folder_id, md5sum_filename)
md5sum_id = self._upload_file(request, media)
if md5sum_id is not None:
self.logger.info("md5sum file uploaded to directory %s" %
folder_id)
else:
self.logger.error("Unable to upload md5sum file to directory %s" %
folder_id)
os.remove(md5sum_filepath)
return id
