class IndirectionCloud(object): zope.interface.implements(CloudDelegate) def __init__(self): super(IndirectionCloud, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.pim = PersistentImageManager.default_manager() self.res_mgr = ReservationManager() def builder_should_create_target_image(self, builder, target, image_id, template, parameters): # This plugin wants to be the only thing operating on the input image # We do all our work here and then return False which stops any additional activity # User may specify a utility image - if they do not we assume we can use the input image utility_image_id = parameters.get('utility_image', image_id) # The utility image is what we actually re-animate with Oz # We borrow these variable names from code that is very similar to the Oz/TinMan OS plugin self.active_image = self.pim.image_with_id(utility_image_id) if not self.active_image: raise Exception("Could not find utility image with ID (%s)" % (utility_image_id) ) self.tdlobj = oz.TDL.TDL(xmlstring=self.active_image.template) # Later on, we will either copy in the base_image content as a file, or expose it as a device # to the utility VM. We cannot do both. Detect invalid input here before doing any long running # work input_image_device = parameters.get('input_image_device', None) input_image_file = parameters.get('input_image_filename', None) if input_image_device and input_image_file: raise Exception("You can specify either an input_image_device or an input_image_file but not both") if (not input_image_device) and (not input_image_file): input_image_file="/input_image.raw" # We remove any packages, commands and files from the original TDL - these have already been # installed/executed. We leave the repos in place, as it is possible that commands executed # later may depend on them self.tdlobj.packages = [ ] self.tdlobj.commands = { } self.tdlobj.files = { } # This creates a new Oz object - replaces the auto-generated disk file location with # the copy of the utility image made above, and prepares an initial libvirt_xml string self._init_oz() utility_image_tmp = self.app_config['imgdir'] + "/tmp-utility-image-" + str(builder.target_image.identifier) self.guest.diskimage = utility_image_tmp # Below we will create this file as a qcow2 image using the original utility image as # a backing store - For the follow-on XML generation to work correctly, we need to force # Oz to use qcow2 as the image type self.guest.image_type = 'qcow2' if 'utility_cpus' in parameters: self.guest.install_cpus = int(parameters['utility_cpus']) libvirt_xml = self.guest._generate_xml("hd", None) libvirt_doc = libxml2.parseDoc(libvirt_xml) # Now we create a second disk image as working/scratch space # Hardcode at 30G # TODO: Make configurable # Make it, format it, copy in the base_image working_space_image = self.app_config['imgdir'] + "/working-space-image-" + str(builder.target_image.identifier) self.create_ext2_image(working_space_image) # Modify the libvirt_xml used with Oz to contain a reference to a second "working space" disk image working_space_device = parameters.get('working_space_device', 'vdb') self.add_disk(libvirt_doc, working_space_image, working_space_device) self.log.debug("Updated domain XML with working space image:\n%s" % (libvirt_xml)) # We expect to find a partial TDL document in this parameter - this is what drives the # tasks performed by the utility image if 'utility_customizations' in parameters: self.oz_refresh_customizations(parameters['utility_customizations']) else: self.log.info('No additional repos, packages, files or commands specified for utility tasks') # Create a qcow2 image using the original utility image file (which may be read-only) as a # backing store. self.log.debug("Creating temporary writeable qcow2 working copy of utlity image (%s) as (%s)" % (self.active_image.data, utility_image_tmp)) self.guest._internal_generate_diskimage(image_filename=utility_image_tmp, backing_filename=self.active_image.data) if input_image_file: # Here we finally involve the actual Base Image content - it is made available for the utlity image to modify self.copy_content_to_image(builder.base_image.data, working_space_image, input_image_file) else: # Note that we know that one or the other of these are set because of code earlier self.add_disk(libvirt_doc, builder.base_image.data, input_image_device) # Run all commands, repo injection, etc specified try: self.log.debug("Launching utility image and running any customizations specified") libvirt_xml = libvirt_doc.serialize(None, 1) self.guest.customize(libvirt_xml) self.log.debug("Utility image tasks complete") finally: self.log.debug("Cleaning up install artifacts") self.guest.cleanup_install() # After shutdown, extract the results results_location = parameters.get('results_location', "/results/images/boot.iso") self.copy_content_from_image(results_location, working_space_image, builder.target_image.data) # TODO: Remove working_space image and utility_image_tmp return False def add_disk(self, libvirt_doc, disk_image_file, device_name): devices = libvirt_doc.xpathEval("/domain/devices")[0] new_dev = devices.newChild(None, "disk", None) new_dev.setProp("type", "file") new_dev.setProp("device", "disk") source = new_dev.newChild(None, "source", None) source.setProp("file", disk_image_file) target = new_dev.newChild(None, "target", None) target.setProp("dev", device_name) target.setProp("bus", self.guest.disk_bus) def oz_refresh_customizations(self, partial_tdl): # This takes our already created and well formed TDL object with already blank customizations # and attempts to add in any additional customizations found in partial_tdl # partial_tdl need not contain the <os>, <name> or <description> sections # if it does they will be ignored # TODO: Submit an Oz patch to make this shorter or a utility function within the TDL class doc = lxml.etree.fromstring(partial_tdl) self.tdlobj.doc = doc packageslist = doc.xpath('/template/packages/package') self.tdlobj._add_packages(packageslist) for afile in doc.xpath('/template/files/file'): name = afile.get('name') if name is None: raise Exception("File without a name was given") contenttype = afile.get('type') if contenttype is None: contenttype = 'raw' content = afile.text if content: content = content.strip() else: content = '' self.tdlobj.files[name] = data_from_type(name, contenttype, content) repositorieslist = doc.xpath('/template/repositories/repository') self.tdlobj._add_repositories(repositorieslist) self.tdlobj.commands = self.tdlobj._parse_commands() def _init_oz(self): # populate a config object to pass to OZ; this allows us to specify our # own output dir but inherit other Oz behavior self.oz_config = ConfigParser.SafeConfigParser() if self.oz_config.read("/etc/oz/oz.cfg") != []: self.oz_config.set('paths', 'output_dir', self.app_config["imgdir"]) if "oz_data_dir" in self.app_config: self.oz_config.set('paths', 'data_dir', self.app_config["oz_data_dir"]) if "oz_screenshot_dir" in self.app_config: self.oz_config.set('paths', 'screenshot_dir', self.app_config["oz_screenshot_dir"]) else: raise ImageFactoryException("No Oz config file found. Can't continue.") # Use the factory function from Oz directly try: # Force uniqueness by overriding the name in the TDL self.tdlobj.name = "factory-build-" + self.active_image.identifier self.guest = oz.GuestFactory.guest_factory(self.tdlobj, self.oz_config, None) # Oz just selects a random port here - This could potentially collide if we are unlucky self.guest.listen_port = self.res_mgr.get_next_listen_port() except libvirtError, e: raise ImageFactoryException("Cannot connect to libvirt. Make sure libvirt is running. [Original message: %s]" % e.message) except OzException, e: if "Unsupported" in e.message: raise ImageFactoryException("TinMan plugin does not support distro (%s) update (%s) in TDL" % (self.tdlobj.distro, self.tdlobj.update) ) else: raise e
class IndirectionCloud(object): zope.interface.implements(CloudDelegate) def __init__(self): super(IndirectionCloud, self).__init__() self.app_config = ApplicationConfiguration().configuration self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.pim = PersistentImageManager.default_manager() self.res_mgr = ReservationManager() def builder_should_create_target_image(self, builder, target, image_id, template, parameters): # This plugin wants to be the only thing operating on the input image # We do all our work here and then return False which stops any additional activity self.working_space_image = None self.utility_image_tmp = None try: self._builder_should_create_target_image(builder, target, image_id, template, parameters) finally: self.log.debug("Cleaning up temporary utility image and working space image") for fname in [ self.working_space_image, self.utility_image_tmp ]: if fname and os.path.isfile(fname): os.unlink(fname) return False def _builder_should_create_target_image(self, builder, target, image_id, template, parameters): # User may specify a utility image - if they do not we assume we can use the input image utility_image_id = parameters.get('utility_image', image_id) # The utility image is what we actually re-animate with Oz # We borrow these variable names from code that is very similar to the Oz/TinMan OS plugin self.active_image = self.pim.image_with_id(utility_image_id) if not self.active_image: raise Exception("Could not find utility image with ID (%s)" % (utility_image_id) ) self.tdlobj = oz.TDL.TDL(xmlstring=self.active_image.template) # Later on, we will either copy in the base_image content as a file, or expose it as a device # to the utility VM. We cannot do both. Detect invalid input here before doing any long running # work input_image_device = parameters.get('input_image_device', None) input_image_file = parameters.get('input_image_filename', None) if input_image_device and input_image_file: raise Exception("You can specify either an input_image_device or an input_image_file but not both") if (not input_image_device) and (not input_image_file): input_image_file="/input_image.raw" # We remove any packages, commands and files from the original TDL - these have already been # installed/executed. We leave the repos in place, as it is possible that commands executed # later may depend on them self.tdlobj.packages = [ ] self.tdlobj.commands = { } self.tdlobj.files = { } # This creates a new Oz object - replaces the auto-generated disk file location with # a copy of the utility image we will make later, and prepares an initial libvirt_xml string self._init_oz() self.utility_image_tmp = self.app_config['imgdir'] + "/tmp-utility-image-" + str(builder.target_image.identifier) self.guest.diskimage = self.utility_image_tmp # Below we will create this file as a qcow2 image using the original utility image as # a backing store - For the follow-on XML generation to work correctly, we need to force # Oz to use qcow2 as the image type self.guest.image_type = 'qcow2' if 'utility_cpus' in parameters: self.guest.install_cpus = int(parameters['utility_cpus']) libvirt_xml = self.guest._generate_xml("hd", None) libvirt_doc = libxml2.parseDoc(libvirt_xml) # Now we create a second disk image as working/scratch space # Hardcode at 30G # TODO: Make configurable # Make it, format it, copy in the base_image self.working_space_image = self.app_config['imgdir'] + "/working-space-image-" + str(builder.target_image.identifier) self.create_ext2_image(self.working_space_image) # Modify the libvirt_xml used with Oz to contain a reference to a second "working space" disk image working_space_device = parameters.get('working_space_device', 'vdb') self.add_disk(libvirt_doc, self.working_space_image, working_space_device) self.log.debug("Updated domain XML with working space image:\n%s" % (libvirt_xml)) # We expect to find a partial TDL document in this parameter - this is what drives the # tasks performed by the utility image if 'utility_customizations' in parameters: self.oz_refresh_customizations(parameters['utility_customizations']) else: self.log.info('No additional repos, packages, files or commands specified for utility tasks') # Create a qcow2 image using the original utility image file (which may be read-only) as a # backing store. self.log.debug("Creating temporary writeable qcow2 working copy of utlity image (%s) as (%s)" % (self.active_image.data, self.utility_image_tmp)) self.guest._internal_generate_diskimage(image_filename=self.utility_image_tmp, backing_filename=self.active_image.data) if input_image_file: # Here we finally involve the actual Base Image content - it is made available for the utlity image to modify self.copy_content_to_image(builder.base_image.data, self.working_space_image, input_image_file) else: # Note that we know that one or the other of these are set because of code earlier self.add_disk(libvirt_doc, builder.base_image.data, input_image_device) # Run all commands, repo injection, etc specified try: self.log.debug("Launching utility image and running any customizations specified") libvirt_xml = libvirt_doc.serialize(None, 1) self.guest.customize(libvirt_xml) self.log.debug("Utility image tasks complete") finally: self.log.debug("Cleaning up install artifacts") self.guest.cleanup_install() # After shutdown, extract the results results_location = parameters.get('results_location', "/results/images/boot.iso") self.copy_content_from_image(results_location, self.working_space_image, builder.target_image.data) def add_disk(self, libvirt_doc, disk_image_file, device_name): devices = libvirt_doc.xpathEval("/domain/devices")[0] new_dev = devices.newChild(None, "disk", None) new_dev.setProp("type", "file") new_dev.setProp("device", "disk") source = new_dev.newChild(None, "source", None) source.setProp("file", disk_image_file) target = new_dev.newChild(None, "target", None) target.setProp("dev", device_name) target.setProp("bus", self.guest.disk_bus) def oz_refresh_customizations(self, partial_tdl): # This takes our already created and well formed TDL object with already blank customizations # and attempts to add in any additional customizations found in partial_tdl # partial_tdl need not contain the <os>, <name> or <description> sections # if it does they will be ignored # TODO: Submit an Oz patch to make this shorter or a utility function within the TDL class doc = lxml.etree.fromstring(partial_tdl) self.tdlobj.doc = doc packageslist = doc.xpath('/template/packages/package') self.tdlobj._add_packages(packageslist) for afile in doc.xpath('/template/files/file'): name = afile.get('name') if name is None: raise Exception("File without a name was given") contenttype = afile.get('type') if contenttype is None: contenttype = 'raw' content = afile.text if content: content = content.strip() else: content = '' self.tdlobj.files[name] = data_from_type(name, contenttype, content) repositorieslist = doc.xpath('/template/repositories/repository') self.tdlobj._add_repositories(repositorieslist) self.tdlobj.commands = self.tdlobj._parse_commands() def _init_oz(self): # populate a config object to pass to OZ; this allows us to specify our # own output dir but inherit other Oz behavior self.oz_config = ConfigParser.SafeConfigParser() if self.oz_config.read("/etc/oz/oz.cfg") != []: self.oz_config.set('paths', 'output_dir', self.app_config["imgdir"]) if "oz_data_dir" in self.app_config: self.oz_config.set('paths', 'data_dir', self.app_config["oz_data_dir"]) if "oz_screenshot_dir" in self.app_config: self.oz_config.set('paths', 'screenshot_dir', self.app_config["oz_screenshot_dir"]) else: raise ImageFactoryException("No Oz config file found. Can't continue.") # Use the factory function from Oz directly try: # Force uniqueness by overriding the name in the TDL self.tdlobj.name = "factory-build-" + self.active_image.identifier self.guest = oz.GuestFactory.guest_factory(self.tdlobj, self.oz_config, None) # Oz just selects a random port here - This could potentially collide if we are unlucky self.guest.listen_port = self.res_mgr.get_next_listen_port() except libvirtError, e: raise ImageFactoryException("Cannot connect to libvirt. Make sure libvirt is running. [Original message: %s]" % e.message) except OzException, e: if "Unsupported" in e.message: raise ImageFactoryException("TinMan plugin does not support distro (%s) update (%s) in TDL" % (self.tdlobj.distro, self.tdlobj.update) ) else: raise e
class TinMan(object): def activity(self, activity): # Simple helper function # Activity should be a one line human-readable string indicating the task in progress # We log it at DEBUG and also set it as the status_detail on our active image self.log.debug(activity) self.active_image.status_detail['activity'] = activity ## INTERFACE METHOD def create_target_image(self, builder, target, base_image, parameters): self.log.info( 'create_target_image() called for TinMan plugin - creating a TargetImage' ) self.active_image = builder.target_image self.target = target self.base_image = builder.base_image # populate our target_image bodyfile with the original base image # which we do not want to modify in place self.activity("Copying BaseImage to modifiable TargetImage") self.log.debug( "Copying base_image file (%s) to new target_image file (%s)" % (builder.base_image.data, builder.target_image.data)) oz.ozutil.copyfile_sparse(builder.base_image.data, builder.target_image.data) self.image = builder.target_image.data # Merge together any TDL-style customizations requested via our plugin-to-plugin interface # with any target specific packages, repos and commands and then run a second Oz customization # step. self.tdlobj = oz.TDL.TDL( xmlstring=builder.base_image.template, rootpw_required=self.app_config["tdl_require_root_pw"]) # We remove any packages, commands and files from the original TDL - these have already been # installed/executed. We leave the repos in place, as it is possible that the target # specific packages or commands may require them. self.tdlobj.packages = [] self.tdlobj.commands = {} self.tdlobj.files = {} # This is user-defined target-specific packages and repos in a local config file self.add_target_content() # This is content deposited by cloud plugins - typically commands to run to prep the image further self.merge_cloud_plugin_content() # If there are no new commands, packages or files, we can stop here - there is no need to run Oz again if (len(self.tdlobj.packages) + len(self.tdlobj.commands) + len(self.tdlobj.files)) == 0: self.log.debug( "No further modification of the TargetImage to perform in the OS Plugin - returning" ) return # We have some additional work to do - create a new Oz guest object that we can use to run the guest # customization a second time self._init_oz() self.guest.diskimage = builder.target_image.data libvirt_xml = self.guest._generate_xml("hd", None) # One last step is required here - The persistent net rules in some Fedora and RHEL versions # Will cause our new incarnation of the image to fail to get network - fix that here # We unfortunately end up having to duplicate this a second time in the cloud plugins # when we are done with our second stage customizations # TODO: Consider moving all of that back here guestfs_handle = launch_inspect_and_mount(builder.target_image.data) remove_net_persist(guestfs_handle) shutdown_and_close(guestfs_handle) try: self.log.debug( "Doing second-stage target_image customization and ICICLE generation" ) #self.percent_complete = 30 builder.target_image.icicle = self.guest.customize_and_generate_icicle( libvirt_xml) self.log.debug("Customization and ICICLE generation complete") #self.percent_complete = 50 finally: self.activity("Cleaning up install artifacts") self.guest.cleanup_install() def add_cloud_plugin_content(self, content): # This is a method that cloud plugins can call to deposit content/commands to be run # during the OS-specific first stage of the Target Image creation. # The expected input is a dict containing commands and files # No support for repos at the moment as these introduce external deps that we may not be able to count on # Add this to an array which will later be merged into the TDL object used to drive Oz self.cloud_plugin_content.append(content) def merge_cloud_plugin_content(self): for content in self.cloud_plugin_content: if 'files' in content: for fileentry in content['files']: if not 'name' in fileentry: raise ImageFactoryException( "File given without a name") if not 'type' in fileentry: raise ImageFactoryException( "File given without a type") if not 'file' in fileentry: raise ImageFactoryException( "File given without any content") if fileentry['type'] == 'raw': self.tdlobj.files[ fileentry['name']] = fileentry['file'] elif fileentry['type'] == 'base64': if len(fileentry['file']) == 0: self.tdlobj.files[fileentry['name']] = "" else: self.tdlobj.files[ fileentry['name']] = base64.b64decode( fileentry['file']) else: raise ImageFactoryException( "File given with invalid type (%s)" % (fileentry['type'])) if 'commands' in content: for command in content['commands']: if not 'name' in command: raise ImageFactoryException( "Command given without a name") if not 'type' in command: raise ImageFactoryException( "Command given without a type") if not 'command' in command: raise ImageFactoryException( "Command given without any content") if command['type'] == 'raw': self.tdlobj.commands[ command['name']] = command['command'] elif command['type'] == 'base64': if len(command['command']) == 0: self.log.warning("Command with zero length given") self.tdlobj.commands[command['name']] = "" else: self.tdlobj.commands[ command['name']] = base64.b64decode( command['command']) else: raise ImageFactoryException( "Command given with invalid type (%s)" % (command['type'])) def add_target_content(self): """Merge in target specific package and repo content. TDL object must already exist as self.tdlobj""" doc = None if isfile("/etc/imagefactory/target_content.xml"): doc = libxml2.parseFile("/etc/imagefactory/target_content.xml") else: self.log.debug( "Found neither a call-time config nor a config file - doing nothing" ) return # Purely to make the xpath statements below a tiny bit shorter target = self.target os = self.tdlobj.distro version = self.tdlobj.update arch = self.tdlobj.arch # We go from most to least specific in this order: # arch -> version -> os-> target # Note that at the moment we even allow an include statment that covers absolutely everything. # That is, one that doesn't even specify a target - this is to support a very simple call-time syntax include = doc.xpathEval( "/template_includes/include[@target='%s' and @os='%s' and @version='%s' and @arch='%s']" % (target, os, version, arch)) if len(include) == 0: include = doc.xpathEval( "/template_includes/include[@target='%s' and @os='%s' and @version='%s' and not(@arch)]" % (target, os, version)) if len(include) == 0: include = doc.xpathEval( "/template_includes/include[@target='%s' and @os='%s' and not(@version) and not(@arch)]" % (target, os)) if len(include) == 0: include = doc.xpathEval( "/template_includes/include[@target='%s' and not(@os) and not(@version) and not(@arch)]" % (target)) if len(include) == 0: include = doc.xpathEval( "/template_includes/include[not(@target) and not(@os) and not(@version) and not(@arch)]" ) if len(include) == 0: self.log.debug( "cannot find a config section that matches our build details - doing nothing" ) return # OK - We have at least one config block that matches our build - take the first one, merge it and be done # TODO: Merge all of them? Err out if there is more than one? Warn? include = include[0] packages = include.xpathEval("packages") if len(packages) > 0: self.tdlobj.merge_packages(str(packages[0])) repositories = include.xpathEval("repositories") if len(repositories) > 0: self.tdlobj.merge_repositories(str(repositories[0])) def __init__(self): super(TinMan, self).__init__() self.cloud_plugin_content = [] config_obj = ApplicationConfiguration() self.app_config = config_obj.configuration self.res_mgr = ReservationManager() self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) self.parameters = None self.install_script_object = None self.guest = None def abort(self): self.log.debug("ABORT called in TinMan plugin") # If we have an active Oz VM destroy it - if not do nothing but log why we did nothing if not self.guest: self.log.debug("No Oz guest object present - nothing to do") return try: # Oz doesn't keep the active domain object as an instance variable so we have to look it up guest_dom = self.guest.libvirt_conn.lookupByName(self.tdlobj.name) except Exception as e: self.log.exception(e) self.log.debug("No Oz VM found with name (%s) - nothing to do" % (self.tdlobj.name)) self.log.debug( "This likely means the local VM has already been destroyed or never started" ) return try: self.log.debug("Attempting to destroy local guest/domain (%s)" % (self.tdlobj.name)) guest_dom.destroy() except Exception as e: self.log.exception(e) self.log.warning( "Exception encountered while destroying domain - it may still exist" ) def _init_oz(self): # TODO: This is a convenience variable for refactoring - rename self.new_image_id = self.active_image.identifier # Create a name combining the TDL name and the UUID for use when tagging EC2 AMIs self.longname = self.tdlobj.name + "-" + self.new_image_id # Oz assumes unique names - TDL built for multiple backends guarantees they are not unique # We don't really care about the name so just force uniqueness # 18-Jul-2011 - Moved to constructor and modified to change TDL object name itself # Oz now uses the tdlobject name property directly in several places so we must change it self.tdlobj.name = "factory-build-" + self.new_image_id # populate a config object to pass to OZ; this allows us to specify our # own output dir but inherit other Oz behavior self.oz_config = configparser.SafeConfigParser() if self.oz_config.read("/etc/oz/oz.cfg") != []: if self.parameters.get("oz_overrides", None) != None: oz_overrides = json.loads( self.parameters.get("oz_overrides", None).replace("'", "\"")) for i in oz_overrides: for key, val in oz_overrides[i].items(): self.oz_config.set(i, key, str(val)) self.oz_config.set('paths', 'output_dir', self.app_config["imgdir"]) if "oz_data_dir" in self.app_config: self.oz_config.set('paths', 'data_dir', self.app_config["oz_data_dir"]) if "oz_screenshot_dir" in self.app_config: self.oz_config.set('paths', 'screenshot_dir', self.app_config["oz_screenshot_dir"]) print("=============== Final Oz Config ================") for section in self.oz_config.sections(): print("[ {0} ]".format(section)) for option in self.oz_config.options(section): print(" {0} = {1}".format( option, self.oz_config.get(section, option))) else: raise ImageFactoryException( "No Oz config file found. Can't continue.") # make this a property to enable quick cleanup on abort self.instance = None # Here we are always dealing with a local install self.init_guest() ## INTERFACE METHOD def create_base_image(self, builder, template, parameters): self.log.info( 'create_base_image() called for TinMan plugin - creating a BaseImage' ) self.tdlobj = oz.TDL.TDL( xmlstring=template.xml, rootpw_required=self.app_config["tdl_require_root_pw"]) if parameters: self.parameters = parameters else: self.parameters = {} # TODO: Standardize reference scheme for the persistent image objects in our builder # Having local short-name copies like this may well be a good idea though they # obscure the fact that these objects are in a container "upstream" of our plugin object self.base_image = builder.base_image # Set to the image object that is actively being created or modified # Used in the logging helper function above self.active_image = self.base_image try: self._init_oz() self.guest.diskimage = self.base_image.data self.activity("Cleaning up any old Oz guest") self.guest.cleanup_old_guest() self.activity("Generating JEOS install media") self.threadsafe_generate_install_media(self.guest) self.percent_complete = 10 # We want to save this later for use by RHEV-M and Condor clouds libvirt_xml = "" gfs = None try: self.activity("Generating JEOS disk image") # Newer Oz versions introduce a configurable disk size in TDL # We must still detect that it is present and pass it in this call try: disksize = getattr(self.guest, "disksize") except AttributeError: disksize = 10 self.guest.generate_diskimage(size=disksize) # TODO: If we already have a base install reuse it # subject to some rules about updates to underlying repo self.activity("Execute JEOS install") libvirt_xml = self.guest.install(self.app_config["timeout"]) self.base_image.parameters['libvirt_xml'] = libvirt_xml self.image = self.guest.diskimage self.log.debug( "Base install complete - Doing customization and ICICLE generation" ) self.percent_complete = 30 # Power users may wish to avoid ever booting the guest after the installer is finished # They can do so by passing in a { "generate_icicle": False } KV pair in the parameters dict if parameter_cast_to_bool( self.parameters.get("generate_icicle", True)): if parameter_cast_to_bool( self.parameters.get("offline_icicle", False)): self.guest.customize(libvirt_xml) gfs = launch_inspect_and_mount(self.image, readonly=True) # Monkey-patching is bad # TODO: Work with Chris to incorporate a more elegant version of this into Oz itself def libguestfs_execute_command(gfs, cmd, timeout): stdout = gfs.sh(cmd) return (stdout, None, 0) self.guest.guest_execute_command = libguestfs_execute_command builder.base_image.icicle = self.guest.do_icicle(gfs) else: builder.base_image.icicle = self.guest.customize_and_generate_icicle( libvirt_xml) else: # koji errs out if this value is None - set to an empty ICICLE instead builder.base_image.icicle = "<icicle></icicle>" self.guest.customize(libvirt_xml) self.log.debug("Customization and ICICLE generation complete") self.percent_complete = 50 finally: self.activity("Cleaning up install artifacts") if self.guest: self.guest.cleanup_install() if self.install_script_object: # NamedTemporaryFile - removed on close self.install_script_object.close() if gfs: shutdown_and_close(gfs) self.log.debug("Generated disk image (%s)" % (self.guest.diskimage)) # OK great, we now have a customized KVM image finally: pass # TODO: Create the base_image object representing this # TODO: Create the base_image object at the beginning and then set the diskimage accordingly def init_guest(self): # Use the factory function from Oz directly # This raises an exception if the TDL contains an unsupported distro or version # Cloud plugins that use KVM directly, such as RHEV-M and openstack-kvm can accept # any arbitrary guest that Oz is capable of producing install_script_name = None install_script = self.parameters.get("install_script", None) if install_script: self.install_script_object = NamedTemporaryFile(mode='w') self.install_script_object.write(install_script) self.install_script_object.flush() install_script_name = self.install_script_object.name try: self.guest = oz.GuestFactory.guest_factory(self.tdlobj, self.oz_config, install_script_name) # Oz just selects a random port here - This could potentially collide if we are unlucky self.guest.listen_port = self.res_mgr.get_next_listen_port() except libvirtError as e: raise ImageFactoryException( "Cannot connect to libvirt. Make sure libvirt is running. [Original message: %s]" % e.message) except OzException as e: if "Unsupported" in e.message: raise ImageFactoryException( "TinMan plugin does not support distro (%s) update (%s) in TDL" % (self.tdlobj.distro, self.tdlobj.update)) else: raise e def log_exc(self): self.log.debug("Exception caught in ImageFactory") self.log.debug(traceback.format_exc()) self.active_image.status_detal['error'] = traceback.format_exc() def threadsafe_generate_install_media(self, guest): # Oz caching of install media and modified install media is not thread safe # Make it safe here using some locks # We can only have one active generate_install_media() call for each unique tuple: # (OS, update, architecture, installtype) tdl = guest.tdl queue_name = "%s-%s-%s-%s" % (tdl.distro, tdl.update, tdl.arch, tdl.installtype) self.res_mgr.get_named_lock(queue_name) try: guest.generate_install_media(force_download=False) finally: self.res_mgr.release_named_lock(queue_name)
class FedoraOS(object): zope.interface.implements(OSDelegate) def activity(self, activity): # Simple helper function # Activity should be a one line human-readable string indicating the task in progress # We log it at DEBUG and also set it as the status_detail on our active image self.log.debug(activity) self.active_image.status_detail['activity'] = activity ## INTERFACE METHOD def create_target_image(self, builder, target, base_image, parameters): self.log.info('create_target_image() called for FedoraOS plugin - creating a TargetImage') self.active_image = builder.target_image self.target = target self.base_image = builder.base_image # populate our target_image bodyfile with the original base image # which we do not want to modify in place self.activity("Copying BaseImage to modifiable TargetImage") self.log.debug("Copying base_image file (%s) to new target_image file (%s)" % (builder.base_image.data, builder.target_image.data)) oz.ozutil.copyfile_sparse(builder.base_image.data, builder.target_image.data) self.image = builder.target_image.data # Merge together any TDL-style customizations requested via our plugin-to-plugin interface # with any target specific packages, repos and commands and then run a second Oz customization # step. self.tdlobj = oz.TDL.TDL(xmlstring=builder.base_image.template, rootpw_required=self.app_config["tdl_require_root_pw"]) # We remove any packages, commands and files from the original TDL - these have already been # installed/executed. We leave the repos in place, as it is possible that the target # specific packages or commands may require them. self.tdlobj.packages = [ ] self.tdlobj.commands = { } self.tdlobj.files = { } # This is user-defined target-specific packages and repos in a local config file self.add_target_content() # This is content deposited by cloud plugins - typically commands to run to prep the image further self.merge_cloud_plugin_content() # If there are no new commands, packages or files, we can stop here - there is no need to run Oz again if (len(self.tdlobj.packages) + len(self.tdlobj.commands) + len(self.tdlobj.files)) == 0: self.log.debug("No further modification of the TargetImage to perform in the OS Plugin - returning") return # We have some additional work to do - create a new Oz guest object that we can use to run the guest # customization a second time self._init_oz() self.guest.diskimage = builder.target_image.data libvirt_xml = self.guest._generate_xml("hd", None) # One last step is required here - The persistent net rules in some Fedora and RHEL versions # Will cause our new incarnation of the image to fail to get network - fix that here # We unfortunately end up having to duplicate this a second time in the cloud plugins # when we are done with our second stage customizations # TODO: Consider moving all of that back here guestfs_handle = launch_inspect_and_mount(builder.target_image.data) remove_net_persist(guestfs_handle) shutdown_and_close(guestfs_handle) try: self.log.debug("Doing second-stage target_image customization and ICICLE generation") #self.percent_complete = 30 self.output_descriptor = self.guest.customize_and_generate_icicle(libvirt_xml) self.log.debug("Customization and ICICLE generation complete") #self.percent_complete = 50 finally: self.activity("Cleaning up install artifacts") self.guest.cleanup_install() def add_cloud_plugin_content(self, content): # This is a method that cloud plugins can call to deposit content/commands to be run # during the OS-specific first stage of the Target Image creation. # The expected input is a dict containing commands and files # No support for repos at the moment as these introduce external deps that we may not be able to count on # Add this to an array which will later be merged into the TDL object used to drive Oz self.cloud_plugin_content.append(content) def merge_cloud_plugin_content(self): for content in self.cloud_plugin_content: if 'files' in content: for fileentry in content['files']: if not 'name' in fileentry: raise ImageFactoryException("File given without a name") if not 'type' in fileentry: raise ImageFactoryException("File given without a type") if not 'file' in fileentry: raise ImageFactoryException("File given without any content") if fileentry['type'] == 'raw': self.tdlobj.files[fileentry['name']] = fileentry['file'] elif fileentry['type'] == 'base64': if len(fileentry['file']) == 0: self.tdlobj.files[fileentry['name']] = "" else: self.tdlobj.files[fileentry['name']] = base64.b64decode(fileentry['file']) else: raise ImageFactoryException("File given with invalid type (%s)" % (file['type'])) if 'commands' in content: for command in content['commands']: if not 'name' in command: raise ImageFactoryException("Command given without a name") if not 'type' in command: raise ImageFactoryException("Command given without a type") if not 'command' in command: raise ImageFactoryException("Command given without any content") if command['type'] == 'raw': self.tdlobj.commands[command['name']] = command['command'] elif command['type'] == 'base64': if len(command['command']) == 0: self.log.warning("Command with zero length given") self.tdlobj.commands[command['name']] = "" else: self.tdlobj.commandss[command['name']] = base64.b64decode(command['command']) else: raise ImageFactoryException("Command given with invalid type (%s)" % (command['type'])) def add_target_content(self): """Merge in target specific package and repo content. TDL object must already exist as self.tdlobj""" doc = None if isfile("/etc/imagefactory/target_content.xml"): doc = libxml2.parseFile("/etc/imagefactory/target_content.xml") else: self.log.debug("Found neither a call-time config nor a config file - doing nothing") return # Purely to make the xpath statements below a tiny bit shorter target = self.target os=self.tdlobj.distro version=self.tdlobj.update arch=self.tdlobj.arch # We go from most to least specific in this order: # arch -> version -> os-> target # Note that at the moment we even allow an include statment that covers absolutely everything. # That is, one that doesn't even specify a target - this is to support a very simple call-time syntax include = doc.xpathEval("/template_includes/include[@target='%s' and @os='%s' and @version='%s' and @arch='%s']" % (target, os, version, arch)) if len(include) == 0: include = doc.xpathEval("/template_includes/include[@target='%s' and @os='%s' and @version='%s' and not(@arch)]" % (target, os, version)) if len(include) == 0: include = doc.xpathEval("/template_includes/include[@target='%s' and @os='%s' and not(@version) and not(@arch)]" % (target, os)) if len(include) == 0: include = doc.xpathEval("/template_includes/include[@target='%s' and not(@os) and not(@version) and not(@arch)]" % (target)) if len(include) == 0: include = doc.xpathEval("/template_includes/include[not(@target) and not(@os) and not(@version) and not(@arch)]") if len(include) == 0: self.log.debug("cannot find a config section that matches our build details - doing nothing") return # OK - We have at least one config block that matches our build - take the first one, merge it and be done # TODO: Merge all of them? Err out if there is more than one? Warn? include = include[0] packages = include.xpathEval("packages") if len(packages) > 0: self.tdlobj.merge_packages(str(packages[0])) repositories = include.xpathEval("repositories") if len(repositories) > 0: self.tdlobj.merge_repositories(str(repositories[0])) def __init__(self): super(FedoraOS, self).__init__() self.cloud_plugin_content = [ ] config_obj = ApplicationConfiguration() self.app_config = config_obj.configuration self.res_mgr = ReservationManager() self.log = logging.getLogger('%s.%s' % (__name__, self.__class__.__name__)) def _init_oz(self): # TODO: This is a convenience variable for refactoring - rename self.new_image_id = self.active_image.identifier # Create a name combining the TDL name and the UUID for use when tagging EC2 AMIs self.longname = self.tdlobj.name + "-" + self.new_image_id # Oz assumes unique names - TDL built for multiple backends guarantees they are not unique # We don't really care about the name so just force uniqueness # 18-Jul-2011 - Moved to constructor and modified to change TDL object name itself # Oz now uses the tdlobject name property directly in several places so we must change it self.tdlobj.name = "factory-build-" + self.new_image_id # populate a config object to pass to OZ; this allows us to specify our # own output dir but inherit other Oz behavior self.oz_config = ConfigParser.SafeConfigParser() if self.oz_config.read("/etc/oz/oz.cfg") != []: self.oz_config.set('paths', 'output_dir', self.app_config["imgdir"]) else: raise ImageFactoryException("No Oz config file found. Can't continue.") # make this a property to enable quick cleanup on abort self.instance = None # Here we are always dealing with a local install self.init_guest() ## INTERFACE METHOD def create_base_image(self, builder, template, parameters): self.log.info('create_base_image() called for FedoraOS plugin - creating a BaseImage') self.tdlobj = oz.TDL.TDL(xmlstring=template.xml, rootpw_required=self.app_config["tdl_require_root_pw"]) # TODO: Standardize reference scheme for the persistent image objects in our builder # Having local short-name copies like this may well be a good idea though they # obscure the fact that these objects are in a container "upstream" of our plugin object self.base_image = builder.base_image # Set to the image object that is actively being created or modified # Used in the logging helper function above self.active_image = self.base_image self._init_oz() self.guest.diskimage = self.base_image.data # The remainder comes from the original build_upload(self, build_id) self.status="BUILDING" try: self.activity("Cleaning up any old Oz guest") self.guest.cleanup_old_guest() self.activity("Generating JEOS install media") self.threadsafe_generate_install_media(self.guest) self.percent_complete=10 # We want to save this later for use by RHEV-M and Condor clouds libvirt_xml="" try: self.activity("Generating JEOS disk image") self.guest.generate_diskimage() # TODO: If we already have a base install reuse it # subject to some rules about updates to underlying repo self.activity("Execute JEOS install") libvirt_xml = self.guest.install(self.app_config["timeout"]) self.base_image.parameters['libvirt_xml'] = libvirt_xml self.image = self.guest.diskimage self.log.debug("Base install complete - Doing customization and ICICLE generation") self.percent_complete = 30 self.output_descriptor = self.guest.customize_and_generate_icicle(libvirt_xml) self.log.debug("Customization and ICICLE generation complete") self.percent_complete = 50 finally: self.activity("Cleaning up install artifacts") self.guest.cleanup_install() self.log.debug("Generated disk image (%s)" % (self.guest.diskimage)) # OK great, we now have a customized KVM image finally: pass # TODO: Create the base_image object representing this # TODO: Create the base_image object at the beginning and then set the diskimage accordingly def init_guest(self): # Use the factory function from Oz directly # This raises an exception if the TDL contains an unsupported distro or version # Cloud plugins that use KVM directly, such as RHEV-M and openstack-kvm can accept # any arbitrary guest that Oz is capable of producing try: self.guest = oz.GuestFactory.guest_factory(self.tdlobj, self.oz_config, None) # Oz just selects a random port here - This could potentially collide if we are unlucky self.guest.listen_port = self.res_mgr.get_next_listen_port() except: raise ImageFactoryException("OS plugin does not support distro (%s) update (%s) in TDL" % (self.tdlobj.distro, self.tdlobj.update) ) def log_exc(self): self.log.debug("Exception caught in ImageFactory") self.log.debug(traceback.format_exc()) self.active_image.status_detal['error'] = traceback.format_exc() def threadsafe_generate_install_media(self, guest): # Oz caching of install media and modified install media is not thread safe # Make it safe here using some locks # We can only have one active generate_install_media() call for each unique tuple: # (OS, update, architecture, installtype) tdl = guest.tdl queue_name = "%s-%s-%s-%s" % (tdl.distro, tdl.update, tdl.arch, tdl.installtype) self.res_mgr.get_named_lock(queue_name) try: guest.generate_install_media(force_download=False) finally: self.res_mgr.release_named_lock(queue_name)