class StratusLabClientCloud(BaseCloudConnector): RUNINSTANCE_RETRY_TIMEOUT = 3 POLL_STORAGE_FOR_IMAGE_ID_TIMEOUT_MIN = 30 POLL_STORAGE_FOR_IMAGE_ID_SLEEP_MIN = 1 cloudName = 'stratuslab' @staticmethod def _wait_vm_in_state(states, runner, vm_id, counts=3, sleep=2, throw=False): counter = 1 while counter <= counts: state = runner.getVmState(vm_id) if state in states: return state time.sleep(sleep) counter += 1 if throw: raise Exception('Timed out while waiting for states: %s' % states) def __init__(self, configHolder): self.creator = None super(StratusLabClientCloud, self).__init__(configHolder) self.slConfigHolder = StratuslabConfigHolder(configHolder.options, configHolder.config) self._set_listener( CreatorBaseListener(verbose=(self.verboseLevel > 1))) self._set_capabilities(contextualization=True, direct_ip_assignment=True, orchestrator_can_kill_itself_or_its_vapp=True) if self.verboseLevel > 2: LogUtil.set_logger_level(level=logging.DEBUG) # Temporary workaround: Try to increase to the maximum the limit of the number of open file descriptors. # This is a workaround to a bug where some connections to the StratusLab frontend stays in CLOSE_WAIT. # The standard limit is hit when the Run contains a huge amount of VMs (> 1000). try: import resource l = resource.getrlimit(resource.RLIMIT_NOFILE) resource.setrlimit(resource.RLIMIT_NOFILE, (l[1], l[1])) except: pass def _start_image_for_build(self, user_info, node_instance): self._prepare_machine_for_build_image(user_info) manifest_downloader = ManifestDownloader(self.slConfigHolder) image_id = node_instance.get_image_id() node_instance.set_image_attributes({ 'imageVersion': manifest_downloader.getImageVersion(imageId=image_id) }) self._update_stratuslab_config_holder_for_build_image( user_info, node_instance) self.creator = Creator(image_id, self.slConfigHolder) self.creator.setListener(self._get_listener()) createImageTemplateDict = self.creator._getCreateImageTemplateDict() def our_create_template_dict(): createImageTemplateDict.update({}) return createImageTemplateDict self.creator._getCreateImageTemplateDict = our_create_template_dict self.creator.createStep1() vm = self.creator.runner return vm def list_instances(self): self.slConfigHolder.set('ipToHostname', False) vms = Monitor(self.slConfigHolder).listVms() populate_vms_with_disk_sizes(vms, self.slConfigHolder.deepcopy()) return vms def _vm_get_ip_from_list_instances(self, vm_instance): return vm_instance.template_nic_ip def _vm_get_cpu(self, vm_instance): return vm_instance.template_vcpu def _vm_get_ram(self, vm_instance): return vm_instance.template_memory def _vm_get_root_disk(self, vm_instance): try: return vm_instance.template_disk_0_size except AttributeError: try: return get_root_disk_size_from_disk_source( vm_instance.template_disk_source, self.slConfigHolder.deepcopy()) except UnknownRootDiskSizeSourceError: return super(BaseCloudConnector, self)._vm_get_root_disk() @override def _build_image(self, user_info, node_instance): machine_name = node_instance.get_name() vm = self._get_vm(machine_name) vm_ip = self._vm_get_ip(vm) self._build_image_increment(user_info, node_instance, vm_ip) self.creator.createStep2() image_id = self._search_storage_for_new_image(self.slConfigHolder) if not image_id: util.printDetail( 'WARNING: Failed to get image ID from StratusLab storage!', verboseThreshold=0) else: util.printDetail('New built image ID %s' % image_id, verboseThreshold=0) return image_id def _search_storage_for_new_image(self, slConfigHolder): warn_msg = "WARNING: Unable to search for new image ID. %s env.var is not set." pdisk_endpoint = os.environ.get('SLIPSTREAM_PDISK_ENDPOINT', None) if not pdisk_endpoint: print >> sys.stdout, warn_msg % 'SLIPSTREAM_PDISK_ENDPOINT' sys.stdout.flush() return '' diid = os.environ.get('SLIPSTREAM_DIID', None) if not diid: print >> sys.stdout, warn_msg % 'SLIPSTREAM_DIID' sys.stdout.flush() return '' return self._poll_storage_for_new_image(pdisk_endpoint, diid, slConfigHolder) def _poll_storage_for_new_image(self, pdisk_endpoint, diid, slConfigHolder): # TODO: Introduce checking for the state of the VM. Bail out on Failed or Unknown. tag = "SlipStream-%s" % diid filters = { 'tag': [ tag, ] } slConfigHolder.set('pdiskEndpoint', pdisk_endpoint) pdisk = VolumeManagerFactory.create(slConfigHolder) print >> sys.stdout, "Searching on %s for disk with tag %s." % \ (pdisk_endpoint, tag) sys.stdout.flush() new_image_id = '' poll_duration = self._get_poll_storage_for_image_id_timeout() time_stop = time.time() + poll_duration time_sleep = self._get_poll_storage_for_image_id_sleep() print >> sys.stdout, "Sleeping for %s min with %s min intervals." % \ (poll_duration / 60, time_sleep / 60) while time.time() <= time_stop: volumes = pdisk.describeVolumes(filters) if len(volumes) > 0: try: new_image_id = volumes[0]['identifier'] except Exception as ex: print "Exception occurred looking for volume: %s" % ex break time.sleep(time_sleep) print >> sys.stdout, "Time left for search %d min." % ( (time_stop - time.time()) / 60) sys.stdout.flush() return new_image_id def _get_poll_storage_for_image_id_timeout(self): "Returns the timeout in seconds." return self.POLL_STORAGE_FOR_IMAGE_ID_TIMEOUT_MIN * 60 def _get_poll_storage_for_image_id_sleep(self): "Returns the sleep time in seconds." return self.POLL_STORAGE_FOR_IMAGE_ID_SLEEP_MIN * 60 @staticmethod def _get_create_image_messaging_message(image_resource_uri): return base64.b64encode('{"uri":"%s", "imageid":""}' % image_resource_uri) @override def _initialization(self, user_info, **kwargs): self.slConfigHolder.options.update(Runner.defaultRunOptions()) self._set_user_info_on_stratuslab_config_holder( user_info, run_instance=kwargs.get('run_instance', True)) @override def _start_image(self, user_info, node_instance, vm_name): if self.is_build_image(): return self._start_image_for_build(user_info, node_instance) else: return self._start_image_for_deployment(node_instance, vm_name) def _start_image_for_deployment(self, node_instance, vm_name): slConfigHolder = self.slConfigHolder.deepcopy() self._set_instance_params_on_config_holder(slConfigHolder, node_instance) image_id = node_instance.get_image_id() self._set_extra_context_data_on_config_holder(slConfigHolder, node_instance) self._set_vm_name_on_config_holder(slConfigHolder, vm_name) runner = self._run_instance(image_id, slConfigHolder) return runner @override def _vm_get_ip(self, vm): if isinstance(vm, CloudInfo): return getattr(vm, Monitor.TEMPLATE_NIC_IP) else: # Runner return vm.instancesDetail[0]['ip'] @override def _vm_get_id(self, vm): if isinstance(vm, CloudInfo): return vm.id else: # Runner return vm.instancesDetail[0]['id'] @override def _vm_get_state(self, vm): if isinstance(vm, CloudInfo): return vm.state_summary else: # Runner return vm.instancesDetail[0]['state'] def _set_instance_params_on_config_holder(self, slConfigHolder, node_instance): self._set_instance_size_on_config_holder(slConfigHolder, node_instance) self._set_extra_disks_on_config_holder(slConfigHolder, node_instance) self._set_network_type_on_config_holder(slConfigHolder, node_instance) def _set_instance_size_on_config_holder(self, slConfigHolder, node_instance): self._set_instance_type_on_configholder(slConfigHolder, node_instance) self._set_cpu_ram_on_config_holder(slConfigHolder, node_instance) def _set_instance_type_on_configholder(self, slConfigHolder, node_instance): instance_type = node_instance.get_instance_type() if instance_type: slConfigHolder.instanceType = instance_type def _set_cpu_ram_on_config_holder(self, slConfigHolder, node_instance): slConfigHolder.vmCpu = node_instance.get_cpu() or None vm_ram_gb = node_instance.get_ram() or None if vm_ram_gb: try: # StratusLab needs value in MB slConfigHolder.vmRam = str(int(vm_ram_gb.strip()) * 1024) except: pass def _set_extra_disks_on_config_holder(self, slConfigHolder, node_instance): # 'extra_disk_volatile' is given in GB - 'extraDiskSize' needs to be in MB slConfigHolder.extraDiskSize = int( node_instance.get_volatile_extra_disk_size() or 0) * 1024 slConfigHolder.persistentDiskUUID = node_instance.get_cloud_parameter( 'extra_disk_persistent', '') slConfigHolder.readonlyDiskId = node_instance.get_cloud_parameter( 'extra_disk_readonly', '') def _set_extra_context_data_on_config_holder(self, slConfigHolder, node_instance): node_instance_name = node_instance.get_name() regex = 'SLIPSTREAM_' if self.is_start_orchestrator(): regex += '|CLOUDCONNECTOR_' env_matcher = re.compile(regex) slConfigHolder.extraContextData = '#'.join([ '%s=%s' % (k, v) for (k, v) in os.environ.items() if env_matcher.match(k) ]) slConfigHolder.extraContextData += '#%s=%s' % ( util.ENV_NODE_INSTANCE_NAME, node_instance_name) slConfigHolder.extraContextData += '#SCRIPT_EXEC=%s' % self._build_slipstream_bootstrap_command( node_instance) def _set_vm_name_on_config_holder(self, slConfigHolder, vm_name): slConfigHolder.vmName = vm_name def _run_instance(self, image_id, slConfigHolder, max_attempts=3): if max_attempts <= 0: max_attempts = 1 attempt = 1 while True: try: runner = self._do_run_instance(image_id, slConfigHolder) except socket.error, ex: if attempt >= max_attempts: import traceback cause = '' cause_lines = traceback.format_exception(*sys.exc_info()) for line in cause_lines: cause += line raise Exceptions.ExecutionException( "Failed to launch instance after %i attempts: %s \nCaused by :\n%s" % (attempt, str(ex), cause)) time.sleep(self.RUNINSTANCE_RETRY_TIMEOUT) attempt += 1 else: return runner
class StratuslabClientCloud(BaseCloudConnector): RUNINSTANCE_RETRY_TIMEOUT = 3 cloudName = 'stratuslab' def __init__(self, slipstreamConfigHolder): super(StratuslabClientCloud, self).__init__(slipstreamConfigHolder) self.slConfigHolder = StratuslabConfigHolder(slipstreamConfigHolder.options, slipstreamConfigHolder.config) self.listener = CreatorBaseListener(verbose=(self.verboseLevel > 1)) self.setCapabilities(contextualization=True, direct_ip_assignment=True, orchestrator_can_kill_itself_or_its_vapp=True) patchStratuslab() def startImage(self, user_info, image_info): self._prepareMachineForBuildImage() self.slConfigHolder.set('marketplaceEndpoint', user_info.get_cloud('marketplace.endpoint')) manifestDownloader = ManifestDownloader(self.slConfigHolder) imageId = self.getImageId(image_info) image_info['imageVersion'] = manifestDownloader.getImageVersion(imageId=imageId) self._updateStratuslabConfigHolderForBuildImage(user_info, image_info) self.creator = Creator(imageId, self.slConfigHolder) self.creator.setListener(self.listener) createImageTemplateDict = self.creator._getCreateImageTemplateDict() msgData = StratuslabClientCloud._getCreateImageTemplateMessaging(image_info, self._getCloudInstanceName()) def ourCreateTemplateDict(): createImageTemplateDict.update(msgData) return createImageTemplateDict self.creator._getCreateImageTemplateDict = ourCreateTemplateDict self.creator.createStep1() self.addVm(NodeDecorator.MACHINE_NAME, self.creator.runner) return self.getVmsDetails() def _buildImage(self, userInfo, imageInfo): #self.creator.create() self.creator.createStep2() # # if messaging is set to 'pdisk', then try polling for the new image # identifier from the storage system; otherwise will just return empty # string # self._newImageId = self._pollStorageForNewImage(self.slConfigHolder) def _pollStorageForNewImage(self, slConfigHolder): newImageId = '' msg_type = os.environ.get('SLIPSTREAM_MESSAGING_TYPE', None) msg_endpoint = os.environ.get('SLIPSTREAM_MESSAGING_ENDPOINT', None) if msg_type and msg_endpoint: if msg_type == 'pdisk': diid = os.environ.get('SLIPSTREAM_DIID', None) if diid: tag = "SlipStream-%s" % diid filters = {'tag': [tag, ]} slConfigHolder.set('pdiskEndpoint', msg_endpoint) pdisk = VolumeManagerFactory.create(slConfigHolder) print >> sys.stdout, "Searching on %s for disk with tag %s." % (msg_endpoint, tag) sys.stdout.flush() # hardcoded polling for 30' at 1' intervals for i in range(30): print >> sys.stdout, "Search iteration %d" % i sys.stdout.flush() volumes = pdisk.describeVolumes(filters) if len(volumes) > 0: try: newImageId = volumes[0]['identifier'] except Exception as e: print "Exception occurred looking for volume: %s" % e pass break time.sleep(60) print "Returning new image ID value: %s" % newImageId return newImageId @staticmethod def _getCreateImageTemplateMessaging(imageInfo, cloud_instance_name): msg_type = os.environ.get('SLIPSTREAM_MESSAGING_TYPE', None) if msg_type: imageResourceUri = BaseCloudConnector.getResourceUri(imageInfo) + '/' + cloud_instance_name message = StratuslabClientCloud._getCreateImageMessagingMessage(imageResourceUri) msgData = {Runner.CREATE_IMAGE_KEY_MSG_TYPE: msg_type, Runner.CREATE_IMAGE_KEY_MSG_ENDPOINT: os.environ['SLIPSTREAM_MESSAGING_ENDPOINT'], Runner.CREATE_IMAGE_KEY_MSG_MESSAGE: message} if msg_type in ('amazonsqs', 'dirq'): msgData.update({Runner.CREATE_IMAGE_KEY_MSG_QUEUE: os.environ['SLIPSTREAM_MESSAGING_QUEUE']}) elif msg_type == 'rest': msgData.update({Runner.CREATE_IMAGE_KEY_MSG_QUEUE: imageResourceUri}) elif msg_type == 'pdisk': msgData = {} else: raise Exceptions.ExecutionException('Unsupported messaging type: %s' % msg_type) else: msgData = {} return msgData @staticmethod def _getCreateImageMessagingMessage(imageResourceUri): return base64.b64encode('{"uri":"%s", "imageid":""}' % imageResourceUri) def initialization(self, user_info): self.slConfigHolder.options.update(Runner.defaultRunOptions()) self._setUserInfoOnStratuslabConfigHolder(user_info) def _startImage(self, user_info, image_info, instance_name, cloudSpecificData=None): configHolder = self.slConfigHolder.deepcopy() self._setInstanceParamsOnConfigHolder(configHolder, image_info) imageId = self.getImageId(image_info) self._setExtraContextDataOnConfigHolder(configHolder, cloudSpecificData) self._setVmNameOnConfigHolder(configHolder, instance_name) runner = self._runInstance(imageId, configHolder) return runner def _getCloudSpecificData(self, node_info, node_number, nodename): return nodename def vmGetIp(self, runner): return runner.instancesDetail[0]['ip'] def vmGetId(self, runner): return runner.instancesDetail[0]['id'] def _setInstanceParamsOnConfigHolder(self, configHolder, image): self._setInstanceSizeOnConfigHolder(configHolder, image) self._setExtraDisksOnConfigHolder(configHolder, image) self._setNetworkTypeOnConfigHolder(configHolder, image) def _setInstanceSizeOnConfigHolder(self, configHolder, image): self._setInstanceTypeOnConfigHolder(configHolder, image) self._setCpuRamOnConfigHolder(configHolder, image) def _setInstanceTypeOnConfigHolder(self, configHolder, image): configHolder.instanceType = self._getInstanceType(image) def _setCpuRamOnConfigHolder(self, configHolder, image): configHolder.vmCpu = self._getImageCpu(image) or None vmRamGb = self._getImageRam(image) or None if vmRamGb: try: # StratusLab needs value in MB configHolder.vmRam = str(int(vmRamGb.strip()) * 1024) except: pass def _setExtraDisksOnConfigHolder(self, configHolder, image): extra_disks = self.getExtraDisks(image) # 'extra_disk_volatile' is given in GB - 'extraDiskSize' needs to be in MB configHolder.extraDiskSize = int(extra_disks.get('extra.disk.volatile', 0) or 0) * 1024 configHolder.persistentDiskUUID = extra_disks.get('extra_disk_persistent', '') configHolder.readonlyDiskId = extra_disks.get('extra_disk_readonly', '') def _setExtraContextDataOnConfigHolder(self, configHolder, nodename): configHolder.extraContextData = '#'.join( ['%s=%s' % (k, v) for (k, v) in os.environ.items() if k.startswith('SLIPSTREAM_')]) configHolder.extraContextData += '#SLIPSTREAM_NODENAME=%s' % nodename configHolder.extraContextData += '#SCRIPT_EXEC=%s' % self._buildSlipStreamBootstrapCommand(nodename) def _setVmNameOnConfigHolder(self, configHolder, nodename): configHolder.vmName = nodename def _runInstance(self, imageId, configHolder, max_attempts=3): if max_attempts <= 0: max_attempts = 1 attempt = 1 while True: try: runner = self._doRunInstance(imageId, configHolder) except socket.error, ex: if attempt >= max_attempts: raise Exceptions.ExecutionException( "Failed to launch instance after %i attempts: %s" % (attempt, str(ex))) time.sleep(self.RUNINSTANCE_RETRY_TIMEOUT) attempt += 1 else: return runner