def remoteRunCommand(user, host, command, sshKey=None, password='', nohup=False): sudo = 'sudo' if user != 'root' else '' nohup_cmd = (nohup is True) and 'at now -f %s' or '%s' cmd = ('%s %s' % (sudo, nohup_cmd % command)).strip() rc, stderr = sshCmdWithStderr(cmd, host, user=user, sshKey=sshKey, password=password) if rc != 0: if nohup and (re.search('.* (command )?not found.*', stderr, re.MULTILINE) or not _remote_command_exists('at', host, user, sshKey, password)): nohup_cmd = _remote_command_exists('nohup', host, user, sshKey, password) and 'nohup %s' or '%s' cmd = ('%s %s >/dev/null 2>&1 </dev/null &' % (sudo, nohup_cmd % command)).strip() rc, stderr = sshCmdWithStderr(cmd, host, user=user, sshKey=sshKey, password=password) if rc != 0: raise Exceptions.ExecutionException("An error occurred while executing the command: %s\n%s." % (command, stderr)) else: raise Exceptions.ExecutionException("An error occurred while executing the command: %s\n%s." % (command, stderr)) else: # Check stderr as atd may not be running, though return code was 0. # Starting atd service will start the command. if re.search('.*No atd running\?', stderr, re.MULTILINE): remote_start_service('atd', user, host, sshKey, password) return rc, stderr
def _initialization(self, user_info): util.printStep('Initialize the OpenStack connector.') self._thread_local.driver = self._get_driver(user_info) try: self.flavors = self._thread_local.driver.list_sizes() except InvalidCredsError as e: raise Exceptions.ValidationException("%s: Invalid Cloud credentials. " "Please check your Cloud username, password, project name and domain " "(if applicable) in your SlipStream user profile." % self.get_cloud_service_name()) except LibcloudError as e: if e.message == 'Could not find specified endpoint': raise Exceptions.ValidationException("%s: Invalid Cloud configuration. " "Please ask your SlipStream administrator to check the region, " "service-type and service-name." % self.get_cloud_service_name()) else: raise if self.is_deployment(): self._get_iaas_attributes() self._import_keypair(user_info) elif self.is_build_image(): self._get_iaas_attributes() self._create_keypair_and_set_on_user_info(user_info)
def _handle5xx(resp): if resp.status_code == SERVICE_UNAVAILABLE_ERROR: raise Exceptions.ServiceUnavailableError( "SlipStream is in maintenance.") else: raise Exceptions.ServerError( "Failed calling method %s on url %s, with reason: %d: %s" % (method, url, resp.status_code, resp.reason))
def _request(headers): try: return self.session.request(method, url, data=body, headers=headers, verify=self.host_cert_verify) except requests.exceptions.InvalidSchema as ex: raise Exceptions.ClientError("Malformed URL: %s" % ex) except httplib.BadStatusLine: raise Exceptions.NetworkError( "Error: BadStatusLine contacting: %s" % url)
def _wait_and_get_instance_ip_address(self, vm): # vm = # { # 'ip': '', # 'instance': <slipstream.cloudconnectors.okeanos.NodeDetails object at 0x0000000>, # 'id': '549929', # 'networkType': 'Public' # } self.log("vm = %s" % vm) nodeDetails = vm['instance'] nodeId = nodeDetails.id nodeDetailsActive = self.okeanosClient.waitNodeStatus(nodeId, NodeStatus.ACTIVE) nodeDetails.updateIPsAndStatusFrom(nodeDetailsActive) ip = nodeDetails.ipv4s[0] # Wait for SSH connectivity remoteUsername = "******" sshTimeout = 7.0 self.okeanosClient.waitSshOnNode(nodeDetails, username=remoteUsername, timeout=sshTimeout) self.log("id = %s, ip = %s, adminPass = %s" % (nodeId, ip, nodeDetails.adminPass)) if ip: vm['ip'] = ip self.log("vm = %s" % vm) return vm raise Exceptions.ExecutionException('Timed out while waiting for IPs to be assigned to instances: %s' % nodeId)
def _set_user_info_on_stratuslab_config_holder(self, user_info, build_image=False, run_instance=True): try: self.slConfigHolder.set('endpoint', user_info.get_cloud_endpoint()) self.slConfigHolder.set('username', user_info.get_cloud_username()) self.slConfigHolder.set('password', user_info.get_cloud_password()) sshPubKeysFile = self.__populate_ssh_pub_keys_file(user_info) self.slConfigHolder.set('userPublicKeyFile', sshPubKeysFile) if run_instance or build_image: self.slConfigHolder.set( 'marketplaceEndpoint', user_info.get_cloud('marketplace.endpoint')) if build_image: self.slConfigHolder.set( 'author', '%s %s' % (user_info.get_first_name(), user_info.get_last_name())) self.slConfigHolder.set('authorEmail', user_info.get_email()) self.slConfigHolder.set('saveDisk', True) except KeyError, ex: raise Exceptions.ExecutionException( 'Error bootstrapping from User Parameters. %s' % str(ex))
def remoteRunScript(user, host, script, sshKey=None, password='', nohup=False): fd, scriptFile = tempfile.mkstemp() try: os.write(fd, script) os.close(fd) os.chmod(scriptFile, 0755) dstScriptFile = '/tmp/%s' % os.path.basename(scriptFile) retry_count = 3 while True: rc, stderr = scp(scriptFile, dstScriptFile, user, host, sshKey=sshKey, password=password, withStderr=True) if rc != 0: if retry_count <= 0: raise Exceptions.ExecutionException("An error occurred while uploading " "script to %s: %s" % (host, stderr)) else: time.sleep(5) retry_count -= 1 else: break sshCmdWithStderr('chmod 0755 %s' % dstScriptFile, host, sshKey=sshKey, user=user, password=password) finally: try: os.unlink(scriptFile) except: pass return remoteRunCommand(user, host, dstScriptFile, sshKey, password, nohup)
def get_cloudconnector_modulename_by_cloudname(cloudname): for module_name in get_cloudconnector_modulenames(): connector_class = loadModule(module_name).getConnectorClass() if getattr(connector_class, 'cloudName') == cloudname: return module_name raise Exceptions.NotFoundError( "Failed to find cloud connector module for cloud %s." % cloudname)
def _ensure_pdisk_endpoint_is_set(self): slipstream_pdisk_endpoint = os.environ.get('SLIPSTREAM_PDISK_ENDPOINT') if slipstream_pdisk_endpoint: self.slConfigHolder.set('pdiskEndpoint', slipstream_pdisk_endpoint) if not self.slConfigHolder.pdiskEndpoint: raise Exceptions.ExecutionException( 'PDisk endpoint should be set on StratusLab ConfigHolder')
def getRunState(self, uuid=None, ignoreAbort=True): if not uuid and not self.diid: raise Exceptions.ExecutionException("Run ID should be provided " "to get state.") state_key = NodeDecorator.globalNamespacePrefix + NodeDecorator.STATE_KEY self.run_url = self.runEndpoint + '/' + (uuid or self.diid) return self.getRuntimeParameter(state_key, ignoreAbort=ignoreAbort)
def _stop_deployment(self): errors = [] for nodename, runner in self.get_vms().items(): vm_id = runner.vmIds[0] try: vm_info = runner.cloud._vmInfo(vm_id) uuids_list = self._get_volatile_disk_ids_to_delete( etree.fromstring(vm_info)) runner.killInstances() self._wait_vm_in_state(['Done', 'Failed'], runner, vm_id) time.sleep(2) self._delete_volatile_disks(uuids_list) except Exception as ex: traceback.print_exc() try: time.sleep(2) runner.killInstances() self._delete_volatile_disks(uuids_list) except Exception as ex: traceback.print_exc() errors.append('Error killing node %s\n%s' % (nodename, str(ex))) if errors: raise Exceptions.CloudError('Failed stopping following instances. ' 'Details: %s' % '\n -> '.join(errors))
def _waitCanConnectWithWinrmOrAbort(self, winrm): try: self._waitCanConnectWithWinrmOrTimeout(winrm, self.TIMEOUT_CONNECT) except Exception as ex: raise Exceptions.ExecutionException("Failed to connect to " "%s: %s" % (winrm.endpoint, str(ex)))
def validate_ram_cpu_size(auth_parms, prod_offer, cpu_size, ram_size): # Check if the cpu and ram input is negative if ((int(cpu_size) <= 0) or ((int(ram_size) <= 0))): raise Exceptions.ExecutionException( "CPU and RAM amount should be greater than 0") prod_offer = get_prod_offer(auth_parms, prod_offer) prod_component = prod_offer['componentConfig'] print prod_component prod_conf_cpu = prod_component[0]['productConfiguredValues'] validator_cpu = prod_conf_cpu[0] validateString_cpu = validator_cpu['validator']['validateString'] validateStringList_cpu = validateString_cpu.split("-") print 'Valid CPU Core for product offer: ' print validateString_cpu cpu_allowed = False # For the Standard disk PO the validateString returned is: 1-10. # Validation will fail if the format changes. if (int(cpu_size) >= int(validateStringList_cpu[0]) and int(cpu_size) <= int(validateStringList_cpu[1])): cpu_allowed = True if (cpu_allowed != True): raise Exceptions.ExecutionException( "Invalid disk size for the product offer. Valid Disk sizes: %s" % validateString_cpu) prod_conf_ram = prod_component[1]['productConfiguredValues'] validator_ram = prod_conf_ram[0] validateString_ram = validator_ram['validator']['validateString'] validateStringList_ram = validateString_ram.split(",") print 'Valid RAM amount for product offer: ' print validateString_ram ram_allowed = False for size in validateStringList_ram: if (int(size) == int(ram_size)): ram_allowed = True if (ram_allowed != True): raise Exceptions.ExecutionException( "Invalid disk size for the product offer. Valid Disk sizes: %s" % validateString_ram)
def _start_image_on_cloudstack(self, user_info, node_instance, vm_name): instance_name = self.format_instance_name(vm_name) instance_type = node_instance.get_instance_type() ip_type = node_instance.get_network_type() keypair = None contextualization_script = None if not node_instance.is_windows(): keypair = user_info.get_keypair_name() contextualization_script = self._get_bootstrap_script_if_not_build_image(node_instance) security_groups = node_instance.get_security_groups() security_groups = (len(security_groups) > 0) and security_groups or None try: size = [i for i in self.sizes if i.name == instance_type][0] except IndexError: raise Exceptions.ParameterNotFoundException("Couldn't find the specified instance type: %s" % instance_type) image = self._get_image(node_instance) if node_instance.is_windows(): instance = self.libcloud_driver.create_node( name=instance_name, size=size, image=image, location=self.zone, ex_security_groups=security_groups) else: instance = self.libcloud_driver.create_node( name=instance_name, size=size, image=image, location=self.zone, ex_keyname=keypair, ex_userdata=contextualization_script, ex_security_groups=security_groups) ip = self._get_instance_ip_address(instance, ip_type) if not ip: raise Exceptions.ExecutionException("Couldn't find a '%s' IP" % ip_type) vm = dict(instance=instance, ip=ip, id=instance.id) return vm
def _rpc_execute(self, command, *args): proxy = self._create_rpc_connection() remote_function = getattr(proxy, command) success, output_or_error_msg, err_code = \ remote_function(self._create_session_string(), *args) if not success: raise Exceptions.ExecutionException(output_or_error_msg) return output_or_error_msg
def _attach_disk_and_report(self, node_instance, reporter): attached_disk = self._attach_disk(node_instance) if not attached_disk: raise Exceptions.ExecutionException( 'Attached disk name not provided by connector after disk attach operation.' ) if hasattr(reporter, '__call__'): reporter(node_instance, attached_disk)
def waitUntilSshCanConnectOrTimeout(host, timeout, user='******', password='', sshKey=None, **kwargs): """Returns True on success or raises on any unhandled failures. """ kind = password and 'api' or 'cli' time_stop = time.time() + timeout timeout_connect = 3 auth_failures = 20 reason = 'Unknown' while (time_stop - time.time()) >= 0: kwargs_ = copy.copy(kwargs) try: if True == globals()['_ssh_can_connect_' + kind](host, user, sshKey=sshKey, password=password, timeout=timeout_connect, **kwargs_): return True except SshConnectionRefused as ex: _printDetail(str(ex), kwargs_) reason = 'SshConnectionRefused' time.sleep(5) except SshHostUnreachable as ex: _printDetail(str(ex), kwargs_) reason = 'SshHostUnreachable' time.sleep(10) except SshConnectionResetByPeer as ex: _printDetail(str(ex), kwargs_) reason = 'SshConnectionResetByPeer' time.sleep(5) except SshConnectionTimedOut as ex: _printDetail(str(ex), kwargs_) reason = 'SshConnectionTimedOut' timeout_connect *= 2 except SshServerNameNotKnown as ex: _printDetail(str(ex), kwargs_) raise except SshAuthFailed as ex: _printDetail('%s - %i attempts left.' % (str(ex).strip('\n\r\t '), auth_failures), kwargs_) reason = 'SshAuthFailed' if auth_failures <= 0: raise auth_failures -= 1 time.sleep(5) except SshFailedToConnect as ex: reason = 'SshFailedToConnect' _printDetail(str(ex), kwargs_) time.sleep(5) except paramiko.SSHException as ex: _printDetail(str(ex), kwargs_) reason = 'paramiko.SSHException:', str(ex) time.sleep(5) except exceptions.EOFError as ex: _printDetail(str(ex), kwargs_) reason = 'exceptions.EOFError' time.sleep(5) raise Exceptions.TimeoutException('Failed to connect after %s sec. \nReason: %s' % (timeout, reason))
def _import_keypair(self, user_info): kp_name = 'ss-key-%i' % int(time.time()) public_key = user_info.get_public_keys() try: kp = self._thread_local.driver.ex_import_keypair_from_string(kp_name, public_key) except Exception as ex: raise Exceptions.ExecutionException('Cannot import the public key. Reason: %s' % ex) kp_name = kp.name user_info.set_keypair_name(kp_name) return kp_name
def _wait_vm_in_state(self, vm_id, state, time_out, time_sleep=30): time_stop = time.time() + time_out current_state = self._get_vm_state(vm_id) while current_state != self.VM_STATE.index(state): if time.time() > time_stop: raise Exceptions.ExecutionException( 'Timed out while waiting VM {0} to enter in state {1}'.format(vm_id, state)) time.sleep(time_sleep) current_state = self._get_vm_state(vm_id) return current_state
def _wait_image_not_in_state(self, image_id, state, time_out, time_sleep=30): time_stop = time.time() + time_out current_state = self._get_image_state(image_id) while current_state == self.IMAGE_STATE.index(state): if time.time() > time_stop: raise Exceptions.ExecutionException( 'Timed out while waiting for image {0} to be in state {1}'.format(image_id, state)) time.sleep(time_sleep) current_state = self._get_image_state(image_id) return current_state
def _import_keypair(self, user_info): kp_name = 'ss-key-%i' % int(time.time() * 1000000) public_key = user_info.get_public_keys() try: kp = self.libcloud_driver.import_key_pair_from_string(kp_name, public_key) except Exception as e: user_info.set_keypair_name(None) raise Exceptions.ExecutionException('Cannot import the public key. Reason: %s' % e) kp_name = kp.name user_info.set_keypair_name(kp_name) return kp_name
def _getConfigFile(self): for location in LocalContextualizer.LOCAL_CONTEXTUALIZATION_LOCATIONS: filename = os.path.join( location, LocalContextualizer.LOCAL_CONTEXTUALIZATION_FILENAME) if os.path.exists(filename): util.printDetail( 'Using local contextualization file: %s' % filename, self.verboseLevel) return filename raise Exceptions.ConfigurationError( 'Failed to find local contextualization file.')
def getRuntimeParameter(self, key, ignoreAbort=False): url = self.run_url + '/' + key if self.ignoreAbort or ignoreAbort: url += SlipStreamHttpClient.URL_IGNORE_ABORT_ATTRIBUTE_QUERY try: _, content = self._httpGet(url, accept='text/plain') except Exceptions.NotFoundError as ex: raise Exceptions.NotFoundError('"%s" for %s' % (str(ex), key)) return content.strip().strip('"').strip("'")
def remove_bad_char_in_instance_name(self, name): try: newname = re.sub(r'[^a-zA-Z0-9-]', '', name) m = re.search('[a-zA-Z]([a-zA-Z0-9-]*[a-zA-Z0-9]+)?', newname) return m.string[m.start():m.end()] except: raise Exceptions.ExecutionException( 'Cannot handle the instance name "%s". Instance name can ' 'contain ASCII letters "a" through "z", the digits "0" ' 'through "9", and the hyphen ("-"), must be between 1 and 63 ' 'characters long, and can\'t start or end with "-" ' 'and can\'t start with digit' % name)
def _wait_image_creation_completed(self, image_id): time_wait = 600 time_stop = time.time() + time_wait img = None while not img: if time.time() > time_stop: raise Exceptions.ExecutionException( 'Timed out while waiting for image "%s" to be created' % image_id) time.sleep(1) images = self._thread_local.driver.list_images() img = searchInObjectList(images, 'id', image_id)
def createExecutor(configHolder): cloudWrapper = CloudWrapper(configHolder) category = cloudWrapper.get_run_category() configHolder.set(KEY_RUN_CATEGORY, category) cloudWrapper.initCloudConnector(configHolder) if category == RUN_CATEGORY_IMAGE: return OrchestratorImageBuildExecutor(cloudWrapper, configHolder) elif category == RUN_CATEGORY_DEPLOYMENT: return OrchestratorDeploymentExecutor(cloudWrapper, configHolder) else: raise Exceptions.ClientError("Unknown category: %s" % category)
def _vm_get_root_disk_size_from_pdisk(disk_source, config_holder): disk_source = disk_source.replace('/', ':') pdisk_endpoint = ':'.join(disk_source.split(':')[1:3]) config_holder.set('pdiskUsername', config_holder.username) config_holder.set('pdiskPassword', config_holder.password) config_holder.set('pdiskEndpoint', pdisk_endpoint) pdisk = VolumeManagerFactory.create(config_holder) image_uuid = _disk_source_get_image_id(disk_source) volume = pdisk.describeVolumes({'uuid': ['^%s$' % image_uuid]}) if len(volume) == 0: raise Exceptions.ExecutionException('Failed to describe volume in %s with UUID %s' % (pdisk_endpoint, image_uuid)) return int(volume[0]['size'])
def resize_cpu_ram(auth_parms, vm_uuid, serverName, clusterUUID, vdcUUID, cpu, ram): """ Function to resize the server """ product_offer = 'Standard Server' server_po_uuid = get_prod_offer_uuid(auth_parms, product_offer) if (server_po_uuid == ""): raise Exceptions.ExecutionException("No '" + product_offer + "' Product Offer found") if (server_po_uuid != ""): # Modify the server modify_cpu_ram(auth_parms, vm_uuid, serverName, clusterUUID, vdcUUID, cpu, ram, server_po_uuid)
def _wait_instance_in_running_state(self, instance_id): time_wait = 600 time_stop = time.time() + time_wait state = '' while state != NodeState.RUNNING: if time.time() > time_stop: raise Exceptions.ExecutionException( 'Timed out while waiting for instance "%s" enter in running state' % instance_id) time.sleep(1) nodes = self._thread_local.driver.list_nodes() node = searchInObjectList(nodes, 'id', instance_id) self._raise_on_failed_instance(node) state = node.state
def _stop_deployment(self): errors = [] for nodename, runner in self.get_vms().items(): try: runner.killInstances() except Exception: # Retry killing instances. try: time.sleep(2) runner.killInstances() except Exception as ex: errors.append('Error killing node %s\n%s' % (nodename, str(ex))) if errors: raise Exceptions.CloudError('Failed stopping following instances. ' 'Details: %s' % '\n -> '.join(errors))