def _start_image_for_build(self, user_info, node_instance): self._prepare_machine_for_build_image(user_info) manifest_downloader = ManifestDownloader(self.slConfigHolder) image_id = node_instance.get_image_id() node_instance.set_image_attributes({ 'imageVersion': manifest_downloader.getImageVersion(imageId=image_id) }) self._update_stratuslab_config_holder_for_build_image( user_info, node_instance) self.creator = Creator(image_id, self.slConfigHolder) self.creator.setListener(self._get_listener()) createImageTemplateDict = self.creator._getCreateImageTemplateDict() def our_create_template_dict(): createImageTemplateDict.update({}) return createImageTemplateDict self.creator._getCreateImageTemplateDict = our_create_template_dict self.creator.createStep1() vm = self.creator.runner return vm
def startImage(self, user_info, image_info): self._prepareMachineForBuildImage() self.slConfigHolder.set('marketplaceEndpoint', user_info.get_cloud('marketplace.endpoint')) manifestDownloader = ManifestDownloader(self.slConfigHolder) imageId = self.getImageId(image_info) image_info['imageVersion'] = manifestDownloader.getImageVersion(imageId=imageId) self._updateStratuslabConfigHolderForBuildImage(user_info, image_info) self.creator = Creator(imageId, self.slConfigHolder) self.creator.setListener(self.listener) createImageTemplateDict = self.creator._getCreateImageTemplateDict() msgData = StratuslabClientCloud._getCreateImageTemplateMessaging(image_info, self._getCloudInstanceName()) def ourCreateTemplateDict(): createImageTemplateDict.update(msgData) return createImageTemplateDict self.creator._getCreateImageTemplateDict = ourCreateTemplateDict self.creator.createStep1() self.addVm(NodeDecorator.MACHINE_NAME, self.creator.runner) return self.getVmsDetails()
def doWork(self): configHolder = ConfigHolder(self.options.__dict__, self.config) if self.options.saveDisk: creator = Creator(self.image, configHolder) creator.createRunner() runner = creator.runner else: runner = VmManagerFactory.create(self.image, configHolder) if self.options.listType: print runner.listInstanceTypes() else: vmInfo = runner.runInstance(details=True) # TODO: This is anti-intuitive and should be reviewed. # # To keep compatibility with the previous output, this will # only print something when quiet is set. The 'normal' output # will be printed via INFO logging statements in the code. if self.options.quiet > 0: self._printShortResults(vmInfo)
class StratusLabClientCloud(BaseCloudConnector): RUNINSTANCE_RETRY_TIMEOUT = 3 POLL_STORAGE_FOR_IMAGE_ID_TIMEOUT_MIN = 30 POLL_STORAGE_FOR_IMAGE_ID_SLEEP_MIN = 1 cloudName = 'stratuslab' @staticmethod def _wait_vm_in_state(states, runner, vm_id, counts=3, sleep=2, throw=False): counter = 1 while counter <= counts: state = runner.getVmState(vm_id) if state in states: return state time.sleep(sleep) counter += 1 if throw: raise Exception('Timed out while waiting for states: %s' % states) def __init__(self, configHolder): self.creator = None super(StratusLabClientCloud, self).__init__(configHolder) self.slConfigHolder = StratuslabConfigHolder(configHolder.options, configHolder.config) self._set_listener( CreatorBaseListener(verbose=(self.verboseLevel > 1))) self._set_capabilities(contextualization=True, direct_ip_assignment=True, orchestrator_can_kill_itself_or_its_vapp=True) if self.verboseLevel > 2: LogUtil.set_logger_level(level=logging.DEBUG) # Temporary workaround: Try to increase to the maximum the limit of the number of open file descriptors. # This is a workaround to a bug where some connections to the StratusLab frontend stays in CLOSE_WAIT. # The standard limit is hit when the Run contains a huge amount of VMs (> 1000). try: import resource l = resource.getrlimit(resource.RLIMIT_NOFILE) resource.setrlimit(resource.RLIMIT_NOFILE, (l[1], l[1])) except: pass def _start_image_for_build(self, user_info, node_instance): self._prepare_machine_for_build_image(user_info) manifest_downloader = ManifestDownloader(self.slConfigHolder) image_id = node_instance.get_image_id() node_instance.set_image_attributes({ 'imageVersion': manifest_downloader.getImageVersion(imageId=image_id) }) self._update_stratuslab_config_holder_for_build_image( user_info, node_instance) self.creator = Creator(image_id, self.slConfigHolder) self.creator.setListener(self._get_listener()) createImageTemplateDict = self.creator._getCreateImageTemplateDict() def our_create_template_dict(): createImageTemplateDict.update({}) return createImageTemplateDict self.creator._getCreateImageTemplateDict = our_create_template_dict self.creator.createStep1() vm = self.creator.runner return vm def list_instances(self): self.slConfigHolder.set('ipToHostname', False) vms = Monitor(self.slConfigHolder).listVms() populate_vms_with_disk_sizes(vms, self.slConfigHolder.deepcopy()) return vms def _vm_get_ip_from_list_instances(self, vm_instance): return vm_instance.template_nic_ip def _vm_get_cpu(self, vm_instance): return vm_instance.template_vcpu def _vm_get_ram(self, vm_instance): return vm_instance.template_memory def _vm_get_root_disk(self, vm_instance): try: return vm_instance.template_disk_0_size except AttributeError: try: return get_root_disk_size_from_disk_source( vm_instance.template_disk_source, self.slConfigHolder.deepcopy()) except UnknownRootDiskSizeSourceError: return super(BaseCloudConnector, self)._vm_get_root_disk() @override def _build_image(self, user_info, node_instance): machine_name = node_instance.get_name() vm = self._get_vm(machine_name) vm_ip = self._vm_get_ip(vm) self._build_image_increment(user_info, node_instance, vm_ip) self.creator.createStep2() image_id = self._search_storage_for_new_image(self.slConfigHolder) if not image_id: util.printDetail( 'WARNING: Failed to get image ID from StratusLab storage!', verboseThreshold=0) else: util.printDetail('New built image ID %s' % image_id, verboseThreshold=0) return image_id def _search_storage_for_new_image(self, slConfigHolder): warn_msg = "WARNING: Unable to search for new image ID. %s env.var is not set." pdisk_endpoint = os.environ.get('SLIPSTREAM_PDISK_ENDPOINT', None) if not pdisk_endpoint: print >> sys.stdout, warn_msg % 'SLIPSTREAM_PDISK_ENDPOINT' sys.stdout.flush() return '' diid = os.environ.get('SLIPSTREAM_DIID', None) if not diid: print >> sys.stdout, warn_msg % 'SLIPSTREAM_DIID' sys.stdout.flush() return '' return self._poll_storage_for_new_image(pdisk_endpoint, diid, slConfigHolder) def _poll_storage_for_new_image(self, pdisk_endpoint, diid, slConfigHolder): # TODO: Introduce checking for the state of the VM. Bail out on Failed or Unknown. tag = "SlipStream-%s" % diid filters = { 'tag': [ tag, ] } slConfigHolder.set('pdiskEndpoint', pdisk_endpoint) pdisk = VolumeManagerFactory.create(slConfigHolder) print >> sys.stdout, "Searching on %s for disk with tag %s." % \ (pdisk_endpoint, tag) sys.stdout.flush() new_image_id = '' poll_duration = self._get_poll_storage_for_image_id_timeout() time_stop = time.time() + poll_duration time_sleep = self._get_poll_storage_for_image_id_sleep() print >> sys.stdout, "Sleeping for %s min with %s min intervals." % \ (poll_duration / 60, time_sleep / 60) while time.time() <= time_stop: volumes = pdisk.describeVolumes(filters) if len(volumes) > 0: try: new_image_id = volumes[0]['identifier'] except Exception as ex: print "Exception occurred looking for volume: %s" % ex break time.sleep(time_sleep) print >> sys.stdout, "Time left for search %d min." % ( (time_stop - time.time()) / 60) sys.stdout.flush() return new_image_id def _get_poll_storage_for_image_id_timeout(self): "Returns the timeout in seconds." return self.POLL_STORAGE_FOR_IMAGE_ID_TIMEOUT_MIN * 60 def _get_poll_storage_for_image_id_sleep(self): "Returns the sleep time in seconds." return self.POLL_STORAGE_FOR_IMAGE_ID_SLEEP_MIN * 60 @staticmethod def _get_create_image_messaging_message(image_resource_uri): return base64.b64encode('{"uri":"%s", "imageid":""}' % image_resource_uri) @override def _initialization(self, user_info, **kwargs): self.slConfigHolder.options.update(Runner.defaultRunOptions()) self._set_user_info_on_stratuslab_config_holder( user_info, run_instance=kwargs.get('run_instance', True)) @override def _start_image(self, user_info, node_instance, vm_name): if self.is_build_image(): return self._start_image_for_build(user_info, node_instance) else: return self._start_image_for_deployment(node_instance, vm_name) def _start_image_for_deployment(self, node_instance, vm_name): slConfigHolder = self.slConfigHolder.deepcopy() self._set_instance_params_on_config_holder(slConfigHolder, node_instance) image_id = node_instance.get_image_id() self._set_extra_context_data_on_config_holder(slConfigHolder, node_instance) self._set_vm_name_on_config_holder(slConfigHolder, vm_name) runner = self._run_instance(image_id, slConfigHolder) return runner @override def _vm_get_ip(self, vm): if isinstance(vm, CloudInfo): return getattr(vm, Monitor.TEMPLATE_NIC_IP) else: # Runner return vm.instancesDetail[0]['ip'] @override def _vm_get_id(self, vm): if isinstance(vm, CloudInfo): return vm.id else: # Runner return vm.instancesDetail[0]['id'] @override def _vm_get_state(self, vm): if isinstance(vm, CloudInfo): return vm.state_summary else: # Runner return vm.instancesDetail[0]['state'] def _set_instance_params_on_config_holder(self, slConfigHolder, node_instance): self._set_instance_size_on_config_holder(slConfigHolder, node_instance) self._set_extra_disks_on_config_holder(slConfigHolder, node_instance) self._set_network_type_on_config_holder(slConfigHolder, node_instance) def _set_instance_size_on_config_holder(self, slConfigHolder, node_instance): self._set_instance_type_on_configholder(slConfigHolder, node_instance) self._set_cpu_ram_on_config_holder(slConfigHolder, node_instance) def _set_instance_type_on_configholder(self, slConfigHolder, node_instance): instance_type = node_instance.get_instance_type() if instance_type: slConfigHolder.instanceType = instance_type def _set_cpu_ram_on_config_holder(self, slConfigHolder, node_instance): slConfigHolder.vmCpu = node_instance.get_cpu() or None vm_ram_gb = node_instance.get_ram() or None if vm_ram_gb: try: # StratusLab needs value in MB slConfigHolder.vmRam = str(int(vm_ram_gb.strip()) * 1024) except: pass def _set_extra_disks_on_config_holder(self, slConfigHolder, node_instance): # 'extra_disk_volatile' is given in GB - 'extraDiskSize' needs to be in MB slConfigHolder.extraDiskSize = int( node_instance.get_volatile_extra_disk_size() or 0) * 1024 slConfigHolder.persistentDiskUUID = node_instance.get_cloud_parameter( 'extra_disk_persistent', '') slConfigHolder.readonlyDiskId = node_instance.get_cloud_parameter( 'extra_disk_readonly', '') def _set_extra_context_data_on_config_holder(self, slConfigHolder, node_instance): node_instance_name = node_instance.get_name() regex = 'SLIPSTREAM_' if self.is_start_orchestrator(): regex += '|CLOUDCONNECTOR_' env_matcher = re.compile(regex) slConfigHolder.extraContextData = '#'.join([ '%s=%s' % (k, v) for (k, v) in os.environ.items() if env_matcher.match(k) ]) slConfigHolder.extraContextData += '#%s=%s' % ( util.ENV_NODE_INSTANCE_NAME, node_instance_name) slConfigHolder.extraContextData += '#SCRIPT_EXEC=%s' % self._build_slipstream_bootstrap_command( node_instance) def _set_vm_name_on_config_holder(self, slConfigHolder, vm_name): slConfigHolder.vmName = vm_name def _run_instance(self, image_id, slConfigHolder, max_attempts=3): if max_attempts <= 0: max_attempts = 1 attempt = 1 while True: try: runner = self._do_run_instance(image_id, slConfigHolder) except socket.error, ex: if attempt >= max_attempts: import traceback cause = '' cause_lines = traceback.format_exception(*sys.exc_info()) for line in cause_lines: cause += line raise Exceptions.ExecutionException( "Failed to launch instance after %i attempts: %s \nCaused by :\n%s" % (attempt, str(ex), cause)) time.sleep(self.RUNINSTANCE_RETRY_TIMEOUT) attempt += 1 else: return runner
class MainProgram(Runnable): """A command-line program to create StratusLab image.""" parser_usage = '''%prog [options] base-image-id''' parser_description = ''' Create a new image from an existing base image by adding packages and running configuration scripts. The base-image-id argument is the Marketplace identifier for the initial base image. ''' def parse(self): self.parser.add_option('-n', '--name', dest='newInstalledSoftwareName', help='name of the new image', default='', metavar='FILE') self.parser.add_option('--name-version', dest='newInstalledSoftwareVersion', help='version of installed software', default='', metavar='VERSION') self.parser.add_option('--image-group', dest='newImageGroupName', help='group for the new image (this corresponds to the base image folder - e.g. base, grid). ' 'In case of structured groups use "." as delimiter. Eg. group.subgroup.etc', default='', metavar='NAME') self.parser.add_option('--image-version', dest='newImageGroupVersion', help='version for the new image. By default a minor version of the base image is incremented.', default='', metavar='VERSION') self.parser.add_option('--author', dest='author', help='author of the new image', default='', metavar='NAME') self.parser.add_option('--author-email', dest='authorEmail', help='Email address of the author of the new image', default='', metavar='EMAIL') self.parser.add_option('--marketplace-endpoint-newimage', dest='marketplaceEndpointNewimage', help='Marketplace to register the new image manifest in. ' 'No default. If not provided, either base image Marketplace or defined ' 'by cloud site will be used.', default='', metavar='URL') self.parser.add_option('--title', dest='title', help='title of the new image', default='', metavar='TEXT') self.parser.add_option('--comment', dest='comment', help='description of the new image', default='', metavar='TEXT') self.parser.add_option('-s', '--scripts', dest='scripts', help='scripts to execute on the VM (comma separated)', default='', metavar='FILE') self.parser.add_option('-a', '--packages', dest='packages', help='packages to install on the machine (comma separated)', default='', metavar='PACKAGE') self.parser.add_option('--os', dest='os', help='operation system. By default is taken from Manifest.', default='', metavar='OS') self.parser.add_option('--installer', dest='installer', help='package installer. By default recovered based on OS.', default='', metavar='NAME') self.parser.add_option('--extra-os-repos', dest='extraOsReposUrls', help='extra repositories to install [--packages] from (comma separated). For apt ' "\nbase_uri distribution [component1] ...", default='', metavar='URL') self.parser.add_option('--exclude', dest='excludeFromCreatedImage', help='exclude file/dir from new image (comma separated). Removed by default: %s' % ', '.join(Creator.excludeFromCreatedImageDefault), default='', metavar='FILE') self.parser.add_option('--no-shutdown', dest='shutdownVm', help='leave the VM running. A public IP will be assigned.', default=True, action='store_false') self.parser.add_option('--no-upload', dest='noUpload', help='do not upload the new image to an appliances repository. ' "\nRequires --no-shutdown (otherwise it makes little sense ;-)", default=False, action='store_true') self.parser.add_option('--persistent-disk', dest='persistentDiskUUID', help='persistent disk UUID', default=None) self.parser.add_option('--vm-start-timeout', dest='vmStartTimeout', help='seconds to wait for VM to become available. Default: %i' % Creator.VM_START_TIMEOUT, default=Creator.VM_START_TIMEOUT, type='int', metavar='TIMEOUT') self.parser.add_option('--vm-ping-timeout', dest='vmPingTimeout', help='seconds to wait for VM\'s network to become available. Default: %i' % Creator.VM_PING_TIMEOUT, default=Creator.VM_PING_TIMEOUT, type='int', metavar='TIMEOUT') super(MainProgram, self).parse() def diskSizeOptionCallback(self, option, opt_str, value, parser): setattr(parser.values, option.dest, 1024 * value) def checkOptions(self): super(MainProgram, self).checkOptions() if self.options.noUpload and self.options.shutdownVm: self.parser.error('If you specify --no-upload you also need to specify --no-shutdown, otherwise \ you\'ll never be able to retrieve the new image') if not self.options.title: self.parser.error('Provide a title describing the new image') if not self.options.comment: self.parser.error('Provide a comment describing the new image') if not self.options.author: self.parser.error('Provide an author for the new image') if not self.options.authorEmail: self.parser.error('Provide a valid Email address of the author of the image.') def doWork(self): configHolder = ConfigHolder(self.options.__dict__, self.config or {}) self.creator = Creator(self.image, configHolder) self.creator.create()
def doWork(self): configHolder = ConfigHolder(self.options.__dict__, self.config or {}) self.creator = Creator(self.image, configHolder) self.creator.create()
class StratuslabClientCloud(BaseCloudConnector): RUNINSTANCE_RETRY_TIMEOUT = 3 cloudName = 'stratuslab' def __init__(self, slipstreamConfigHolder): super(StratuslabClientCloud, self).__init__(slipstreamConfigHolder) self.slConfigHolder = StratuslabConfigHolder(slipstreamConfigHolder.options, slipstreamConfigHolder.config) self.listener = CreatorBaseListener(verbose=(self.verboseLevel > 1)) self.setCapabilities(contextualization=True, direct_ip_assignment=True, orchestrator_can_kill_itself_or_its_vapp=True) patchStratuslab() def startImage(self, user_info, image_info): self._prepareMachineForBuildImage() self.slConfigHolder.set('marketplaceEndpoint', user_info.get_cloud('marketplace.endpoint')) manifestDownloader = ManifestDownloader(self.slConfigHolder) imageId = self.getImageId(image_info) image_info['imageVersion'] = manifestDownloader.getImageVersion(imageId=imageId) self._updateStratuslabConfigHolderForBuildImage(user_info, image_info) self.creator = Creator(imageId, self.slConfigHolder) self.creator.setListener(self.listener) createImageTemplateDict = self.creator._getCreateImageTemplateDict() msgData = StratuslabClientCloud._getCreateImageTemplateMessaging(image_info, self._getCloudInstanceName()) def ourCreateTemplateDict(): createImageTemplateDict.update(msgData) return createImageTemplateDict self.creator._getCreateImageTemplateDict = ourCreateTemplateDict self.creator.createStep1() self.addVm(NodeDecorator.MACHINE_NAME, self.creator.runner) return self.getVmsDetails() def _buildImage(self, userInfo, imageInfo): #self.creator.create() self.creator.createStep2() # # if messaging is set to 'pdisk', then try polling for the new image # identifier from the storage system; otherwise will just return empty # string # self._newImageId = self._pollStorageForNewImage(self.slConfigHolder) def _pollStorageForNewImage(self, slConfigHolder): newImageId = '' msg_type = os.environ.get('SLIPSTREAM_MESSAGING_TYPE', None) msg_endpoint = os.environ.get('SLIPSTREAM_MESSAGING_ENDPOINT', None) if msg_type and msg_endpoint: if msg_type == 'pdisk': diid = os.environ.get('SLIPSTREAM_DIID', None) if diid: tag = "SlipStream-%s" % diid filters = {'tag': [tag, ]} slConfigHolder.set('pdiskEndpoint', msg_endpoint) pdisk = VolumeManagerFactory.create(slConfigHolder) print >> sys.stdout, "Searching on %s for disk with tag %s." % (msg_endpoint, tag) sys.stdout.flush() # hardcoded polling for 30' at 1' intervals for i in range(30): print >> sys.stdout, "Search iteration %d" % i sys.stdout.flush() volumes = pdisk.describeVolumes(filters) if len(volumes) > 0: try: newImageId = volumes[0]['identifier'] except Exception as e: print "Exception occurred looking for volume: %s" % e pass break time.sleep(60) print "Returning new image ID value: %s" % newImageId return newImageId @staticmethod def _getCreateImageTemplateMessaging(imageInfo, cloud_instance_name): msg_type = os.environ.get('SLIPSTREAM_MESSAGING_TYPE', None) if msg_type: imageResourceUri = BaseCloudConnector.getResourceUri(imageInfo) + '/' + cloud_instance_name message = StratuslabClientCloud._getCreateImageMessagingMessage(imageResourceUri) msgData = {Runner.CREATE_IMAGE_KEY_MSG_TYPE: msg_type, Runner.CREATE_IMAGE_KEY_MSG_ENDPOINT: os.environ['SLIPSTREAM_MESSAGING_ENDPOINT'], Runner.CREATE_IMAGE_KEY_MSG_MESSAGE: message} if msg_type in ('amazonsqs', 'dirq'): msgData.update({Runner.CREATE_IMAGE_KEY_MSG_QUEUE: os.environ['SLIPSTREAM_MESSAGING_QUEUE']}) elif msg_type == 'rest': msgData.update({Runner.CREATE_IMAGE_KEY_MSG_QUEUE: imageResourceUri}) elif msg_type == 'pdisk': msgData = {} else: raise Exceptions.ExecutionException('Unsupported messaging type: %s' % msg_type) else: msgData = {} return msgData @staticmethod def _getCreateImageMessagingMessage(imageResourceUri): return base64.b64encode('{"uri":"%s", "imageid":""}' % imageResourceUri) def initialization(self, user_info): self.slConfigHolder.options.update(Runner.defaultRunOptions()) self._setUserInfoOnStratuslabConfigHolder(user_info) def _startImage(self, user_info, image_info, instance_name, cloudSpecificData=None): configHolder = self.slConfigHolder.deepcopy() self._setInstanceParamsOnConfigHolder(configHolder, image_info) imageId = self.getImageId(image_info) self._setExtraContextDataOnConfigHolder(configHolder, cloudSpecificData) self._setVmNameOnConfigHolder(configHolder, instance_name) runner = self._runInstance(imageId, configHolder) return runner def _getCloudSpecificData(self, node_info, node_number, nodename): return nodename def vmGetIp(self, runner): return runner.instancesDetail[0]['ip'] def vmGetId(self, runner): return runner.instancesDetail[0]['id'] def _setInstanceParamsOnConfigHolder(self, configHolder, image): self._setInstanceSizeOnConfigHolder(configHolder, image) self._setExtraDisksOnConfigHolder(configHolder, image) self._setNetworkTypeOnConfigHolder(configHolder, image) def _setInstanceSizeOnConfigHolder(self, configHolder, image): self._setInstanceTypeOnConfigHolder(configHolder, image) self._setCpuRamOnConfigHolder(configHolder, image) def _setInstanceTypeOnConfigHolder(self, configHolder, image): configHolder.instanceType = self._getInstanceType(image) def _setCpuRamOnConfigHolder(self, configHolder, image): configHolder.vmCpu = self._getImageCpu(image) or None vmRamGb = self._getImageRam(image) or None if vmRamGb: try: # StratusLab needs value in MB configHolder.vmRam = str(int(vmRamGb.strip()) * 1024) except: pass def _setExtraDisksOnConfigHolder(self, configHolder, image): extra_disks = self.getExtraDisks(image) # 'extra_disk_volatile' is given in GB - 'extraDiskSize' needs to be in MB configHolder.extraDiskSize = int(extra_disks.get('extra.disk.volatile', 0) or 0) * 1024 configHolder.persistentDiskUUID = extra_disks.get('extra_disk_persistent', '') configHolder.readonlyDiskId = extra_disks.get('extra_disk_readonly', '') def _setExtraContextDataOnConfigHolder(self, configHolder, nodename): configHolder.extraContextData = '#'.join( ['%s=%s' % (k, v) for (k, v) in os.environ.items() if k.startswith('SLIPSTREAM_')]) configHolder.extraContextData += '#SLIPSTREAM_NODENAME=%s' % nodename configHolder.extraContextData += '#SCRIPT_EXEC=%s' % self._buildSlipStreamBootstrapCommand(nodename) def _setVmNameOnConfigHolder(self, configHolder, nodename): configHolder.vmName = nodename def _runInstance(self, imageId, configHolder, max_attempts=3): if max_attempts <= 0: max_attempts = 1 attempt = 1 while True: try: runner = self._doRunInstance(imageId, configHolder) except socket.error, ex: if attempt >= max_attempts: raise Exceptions.ExecutionException( "Failed to launch instance after %i attempts: %s" % (attempt, str(ex))) time.sleep(self.RUNINSTANCE_RETRY_TIMEOUT) attempt += 1 else: return runner