def remoteRunCommand(user, host, command, sshKey=None, password='', nohup=False): sudo = 'sudo' if user != 'root' else '' nohup_cmd = (nohup is True) and 'at now -f %s' or '%s' cmd = ('%s %s' % (sudo, nohup_cmd % command)).strip() rc, stderr = sshCmdWithStderr(cmd, host, user=user, sshKey=sshKey, password=password) if rc != 0: if nohup and (re.search('.* (command )?not found.*', stderr, re.MULTILINE) or not _remote_command_exists('at', host, user, sshKey, password)): nohup_cmd = _remote_command_exists('nohup', host, user, sshKey, password) and 'nohup %s' or '%s' cmd = ('%s %s >/dev/null 2>&1 </dev/null &' % (sudo, nohup_cmd % command)).strip() rc, stderr = sshCmdWithStderr(cmd, host, user=user, sshKey=sshKey, password=password) if rc != 0: raise Exceptions.ExecutionException("An error occurred while executing the command: %s\n%s." % (command, stderr)) else: raise Exceptions.ExecutionException("An error occurred while executing the command: %s\n%s." % (command, stderr)) else: # Check stderr as atd may not be running, though return code was 0. # Starting atd service will start the command. if re.search('.*No atd running\?', stderr, re.MULTILINE): remote_start_service('atd', user, host, sshKey, password) return rc, stderr
def getRunState(self, uuid=None, ignoreAbort=True): if not uuid and not self.diid: raise Exceptions.ExecutionException("Run ID should be provided " "to get state.") state_key = NodeDecorator.globalNamespacePrefix + NodeDecorator.STATE_KEY self.run_url = self.runEndpoint + '/' + (uuid or self.diid) return self.getRuntimeParameter(state_key, ignoreAbort=ignoreAbort)
def _set_user_info_on_stratuslab_config_holder(self, user_info, build_image=False, run_instance=True): try: self.slConfigHolder.set('endpoint', user_info.get_cloud_endpoint()) self.slConfigHolder.set('username', user_info.get_cloud_username()) self.slConfigHolder.set('password', user_info.get_cloud_password()) sshPubKeysFile = self.__populate_ssh_pub_keys_file(user_info) self.slConfigHolder.set('userPublicKeyFile', sshPubKeysFile) if run_instance or build_image: self.slConfigHolder.set( 'marketplaceEndpoint', user_info.get_cloud('marketplace.endpoint')) if build_image: self.slConfigHolder.set( 'author', '%s %s' % (user_info.get_first_name(), user_info.get_last_name())) self.slConfigHolder.set('authorEmail', user_info.get_email()) self.slConfigHolder.set('saveDisk', True) except KeyError, ex: raise Exceptions.ExecutionException( 'Error bootstrapping from User Parameters. %s' % str(ex))
def _wait_and_get_instance_ip_address(self, vm): # vm = # { # 'ip': '', # 'instance': <slipstream.cloudconnectors.okeanos.NodeDetails object at 0x0000000>, # 'id': '549929', # 'networkType': 'Public' # } self.log("vm = %s" % vm) nodeDetails = vm['instance'] nodeId = nodeDetails.id nodeDetailsActive = self.okeanosClient.waitNodeStatus(nodeId, NodeStatus.ACTIVE) nodeDetails.updateIPsAndStatusFrom(nodeDetailsActive) ip = nodeDetails.ipv4s[0] # Wait for SSH connectivity remoteUsername = "******" sshTimeout = 7.0 self.okeanosClient.waitSshOnNode(nodeDetails, username=remoteUsername, timeout=sshTimeout) self.log("id = %s, ip = %s, adminPass = %s" % (nodeId, ip, nodeDetails.adminPass)) if ip: vm['ip'] = ip self.log("vm = %s" % vm) return vm raise Exceptions.ExecutionException('Timed out while waiting for IPs to be assigned to instances: %s' % nodeId)
def remoteRunScript(user, host, script, sshKey=None, password='', nohup=False): fd, scriptFile = tempfile.mkstemp() try: os.write(fd, script) os.close(fd) os.chmod(scriptFile, 0755) dstScriptFile = '/tmp/%s' % os.path.basename(scriptFile) retry_count = 3 while True: rc, stderr = scp(scriptFile, dstScriptFile, user, host, sshKey=sshKey, password=password, withStderr=True) if rc != 0: if retry_count <= 0: raise Exceptions.ExecutionException("An error occurred while uploading " "script to %s: %s" % (host, stderr)) else: time.sleep(5) retry_count -= 1 else: break sshCmdWithStderr('chmod 0755 %s' % dstScriptFile, host, sshKey=sshKey, user=user, password=password) finally: try: os.unlink(scriptFile) except: pass return remoteRunCommand(user, host, dstScriptFile, sshKey, password, nohup)
def _waitCanConnectWithWinrmOrAbort(self, winrm): try: self._waitCanConnectWithWinrmOrTimeout(winrm, self.TIMEOUT_CONNECT) except Exception as ex: raise Exceptions.ExecutionException("Failed to connect to " "%s: %s" % (winrm.endpoint, str(ex)))
def _ensure_pdisk_endpoint_is_set(self): slipstream_pdisk_endpoint = os.environ.get('SLIPSTREAM_PDISK_ENDPOINT') if slipstream_pdisk_endpoint: self.slConfigHolder.set('pdiskEndpoint', slipstream_pdisk_endpoint) if not self.slConfigHolder.pdiskEndpoint: raise Exceptions.ExecutionException( 'PDisk endpoint should be set on StratusLab ConfigHolder')
def validate_ram_cpu_size(auth_parms, prod_offer, cpu_size, ram_size): # Check if the cpu and ram input is negative if ((int(cpu_size) <= 0) or ((int(ram_size) <= 0))): raise Exceptions.ExecutionException( "CPU and RAM amount should be greater than 0") prod_offer = get_prod_offer(auth_parms, prod_offer) prod_component = prod_offer['componentConfig'] print prod_component prod_conf_cpu = prod_component[0]['productConfiguredValues'] validator_cpu = prod_conf_cpu[0] validateString_cpu = validator_cpu['validator']['validateString'] validateStringList_cpu = validateString_cpu.split("-") print 'Valid CPU Core for product offer: ' print validateString_cpu cpu_allowed = False # For the Standard disk PO the validateString returned is: 1-10. # Validation will fail if the format changes. if (int(cpu_size) >= int(validateStringList_cpu[0]) and int(cpu_size) <= int(validateStringList_cpu[1])): cpu_allowed = True if (cpu_allowed != True): raise Exceptions.ExecutionException( "Invalid disk size for the product offer. Valid Disk sizes: %s" % validateString_cpu) prod_conf_ram = prod_component[1]['productConfiguredValues'] validator_ram = prod_conf_ram[0] validateString_ram = validator_ram['validator']['validateString'] validateStringList_ram = validateString_ram.split(",") print 'Valid RAM amount for product offer: ' print validateString_ram ram_allowed = False for size in validateStringList_ram: if (int(size) == int(ram_size)): ram_allowed = True if (ram_allowed != True): raise Exceptions.ExecutionException( "Invalid disk size for the product offer. Valid Disk sizes: %s" % validateString_ram)
def _rpc_execute(self, command, *args): proxy = self._create_rpc_connection() remote_function = getattr(proxy, command) success, output_or_error_msg, err_code = \ remote_function(self._create_session_string(), *args) if not success: raise Exceptions.ExecutionException(output_or_error_msg) return output_or_error_msg
def _attach_disk_and_report(self, node_instance, reporter): attached_disk = self._attach_disk(node_instance) if not attached_disk: raise Exceptions.ExecutionException( 'Attached disk name not provided by connector after disk attach operation.' ) if hasattr(reporter, '__call__'): reporter(node_instance, attached_disk)
def _wait_vm_in_state(self, vm_id, state, time_out, time_sleep=30): time_stop = time.time() + time_out current_state = self._get_vm_state(vm_id) while current_state != self.VM_STATE.index(state): if time.time() > time_stop: raise Exceptions.ExecutionException( 'Timed out while waiting VM {0} to enter in state {1}'.format(vm_id, state)) time.sleep(time_sleep) current_state = self._get_vm_state(vm_id) return current_state
def _wait_image_not_in_state(self, image_id, state, time_out, time_sleep=30): time_stop = time.time() + time_out current_state = self._get_image_state(image_id) while current_state == self.IMAGE_STATE.index(state): if time.time() > time_stop: raise Exceptions.ExecutionException( 'Timed out while waiting for image {0} to be in state {1}'.format(image_id, state)) time.sleep(time_sleep) current_state = self._get_image_state(image_id) return current_state
def _import_keypair(self, user_info): kp_name = 'ss-key-%i' % int(time.time()) public_key = user_info.get_public_keys() try: kp = self._thread_local.driver.ex_import_keypair_from_string(kp_name, public_key) except Exception as ex: raise Exceptions.ExecutionException('Cannot import the public key. Reason: %s' % ex) kp_name = kp.name user_info.set_keypair_name(kp_name) return kp_name
def _import_keypair(self, user_info): kp_name = 'ss-key-%i' % int(time.time() * 1000000) public_key = user_info.get_public_keys() try: kp = self.libcloud_driver.import_key_pair_from_string(kp_name, public_key) except Exception as e: user_info.set_keypair_name(None) raise Exceptions.ExecutionException('Cannot import the public key. Reason: %s' % e) kp_name = kp.name user_info.set_keypair_name(kp_name) return kp_name
def _wait_image_creation_completed(self, image_id): time_wait = 600 time_stop = time.time() + time_wait img = None while not img: if time.time() > time_stop: raise Exceptions.ExecutionException( 'Timed out while waiting for image "%s" to be created' % image_id) time.sleep(1) images = self._thread_local.driver.list_images() img = searchInObjectList(images, 'id', image_id)
def remove_bad_char_in_instance_name(self, name): try: newname = re.sub(r'[^a-zA-Z0-9-]', '', name) m = re.search('[a-zA-Z]([a-zA-Z0-9-]*[a-zA-Z0-9]+)?', newname) return m.string[m.start():m.end()] except: raise Exceptions.ExecutionException( 'Cannot handle the instance name "%s". Instance name can ' 'contain ASCII letters "a" through "z", the digits "0" ' 'through "9", and the hyphen ("-"), must be between 1 and 63 ' 'characters long, and can\'t start or end with "-" ' 'and can\'t start with digit' % name)
def _vm_get_root_disk_size_from_pdisk(disk_source, config_holder): disk_source = disk_source.replace('/', ':') pdisk_endpoint = ':'.join(disk_source.split(':')[1:3]) config_holder.set('pdiskUsername', config_holder.username) config_holder.set('pdiskPassword', config_holder.password) config_holder.set('pdiskEndpoint', pdisk_endpoint) pdisk = VolumeManagerFactory.create(config_holder) image_uuid = _disk_source_get_image_id(disk_source) volume = pdisk.describeVolumes({'uuid': ['^%s$' % image_uuid]}) if len(volume) == 0: raise Exceptions.ExecutionException('Failed to describe volume in %s with UUID %s' % (pdisk_endpoint, image_uuid)) return int(volume[0]['size'])
def resize_cpu_ram(auth_parms, vm_uuid, serverName, clusterUUID, vdcUUID, cpu, ram): """ Function to resize the server """ product_offer = 'Standard Server' server_po_uuid = get_prod_offer_uuid(auth_parms, product_offer) if (server_po_uuid == ""): raise Exceptions.ExecutionException("No '" + product_offer + "' Product Offer found") if (server_po_uuid != ""): # Modify the server modify_cpu_ram(auth_parms, vm_uuid, serverName, clusterUUID, vdcUUID, cpu, ram, server_po_uuid)
def _wait_instance_in_running_state(self, instance_id): time_wait = 600 time_stop = time.time() + time_wait state = '' while state != NodeState.RUNNING: if time.time() > time_stop: raise Exceptions.ExecutionException( 'Timed out while waiting for instance "%s" enter in running state' % instance_id) time.sleep(1) nodes = self._thread_local.driver.list_nodes() node = searchInObjectList(nodes, 'id', instance_id) self._raise_on_failed_instance(node) state = node.state
def _wait_can_connect_with_ssh_or_abort(self, host, username='', password='', sshKey=None): self._print_detail('Check if we can connect to %s' % host) try: waitUntilSshCanConnectOrTimeout(host, self.TIMEOUT_CONNECT, sshKey=sshKey, user=username, password=password, verboseLevel=self.verboseLevel) except Exception as ex: raise Exceptions.ExecutionException("Failed to connect to " "%s: %s, %s" % (host, type(ex), str(ex)))
def _start_image_on_cloudstack(self, user_info, node_instance, vm_name): instance_name = self.format_instance_name(vm_name) instance_type = node_instance.get_instance_type() ip_type = node_instance.get_network_type() keypair = None contextualization_script = None if not node_instance.is_windows(): keypair = user_info.get_keypair_name() contextualization_script = self._get_bootstrap_script_if_not_build_image(node_instance) security_groups = node_instance.get_security_groups() security_groups = (len(security_groups) > 0) and security_groups or None try: size = [i for i in self.sizes if i.name == instance_type][0] except IndexError: raise Exceptions.ParameterNotFoundException("Couldn't find the specified instance type: %s" % instance_type) image = self._get_image(node_instance) if node_instance.is_windows(): instance = self.libcloud_driver.create_node( name=instance_name, size=size, image=image, location=self.zone, ex_security_groups=security_groups) else: instance = self.libcloud_driver.create_node( name=instance_name, size=size, image=image, location=self.zone, ex_keyname=keypair, ex_userdata=contextualization_script, ex_security_groups=security_groups) ip = self._get_instance_ip_address(instance, ip_type) if not ip: raise Exceptions.ExecutionException("Couldn't find a '%s' IP" % ip_type) vm = dict(instance=instance, ip=ip, id=instance.id) return vm
def _remove_additional_disk(self, vm_instance): additional_disk_id = vm_instance.extra.get('metadata', {}).get('additional_disk_id') if additional_disk_id: DISK_STATE_AVAILABLE = 0 time_wait = 60 time_stop = time.time() + time_wait additional_disk = self._thread_local.driver.ex_get_volume(additional_disk_id) diskState = additional_disk.state while diskState != DISK_STATE_AVAILABLE: if time.time() > time_stop: raise Exceptions.ExecutionException( 'Timed out while waiting for disk "%s" to be deleted' % additional_disk_id) time.sleep(5) diskState = self._thread_local.driver.ex_get_volume(additional_disk_id).state self._thread_local.driver.destroy_volume(additional_disk)
def create_disk(auth_parms, prod_offer, disk_size, disk_name, vdc_uuid): """ Function to create disk """ # get product offer uuid for the disk in question prod_offer_uuid = get_prod_offer_uuid(auth_parms, prod_offer) disk_job = rest_create_disk(auth_parms, vdc_uuid, prod_offer_uuid, disk_name, disk_size) disk_uuid = disk_job['itemUUID'] print("our newly created disk UUID=" + disk_uuid) # Check the job completes status = wait_for_job(auth_parms, disk_job['resourceUUID'], "SUCCESSFUL", 90) if (status != 0): raise Exceptions.ExecutionException( "Failed to add create disk (uuid=" + disk_uuid + ")") return disk_uuid
def _run_instance(self, image_id, slConfigHolder, max_attempts=3): if max_attempts <= 0: max_attempts = 1 attempt = 1 while True: try: runner = self._do_run_instance(image_id, slConfigHolder) except socket.error, ex: if attempt >= max_attempts: import traceback cause = '' cause_lines = traceback.format_exception(*sys.exc_info()) for line in cause_lines: cause += line raise Exceptions.ExecutionException( "Failed to launch instance after %i attempts: %s \nCaused by :\n%s" % (attempt, str(ex), cause)) time.sleep(self.RUNINSTANCE_RETRY_TIMEOUT) attempt += 1 else: return runner
def validate_disk_size(auth_parms, prod_offer, disk_size, disk_name, vdc_uuid): prod_offer = get_prod_offer(auth_parms, prod_offer) prod_component = prod_offer['componentConfig'] print prod_component prod_conf = prod_component[0]['productConfiguredValues'] validator = prod_conf[0] validateString = validator['validator']['validateString'] validateStringList = validateString.split(",") print 'Valid Disk Size for product offer: ' print validateString allowed = False for size in validateStringList: if (int(size) == disk_size): allowed = True if (allowed != True): raise Exceptions.ExecutionException( "Invalid disk size for the product offer. Valid Disk sizes: %s" % validateString)
def _wait_and_get_instance_ip_address(self, vm): if vm.get('ip'): return vm time_wait = 180 time_stop = time.time() + time_wait instance = vm['instance'] ipType = vm['networkType'] strict = vm['strict_ip_retrival'] vmId = vm['id'] ip = vm['ip'] while time.time() < time_stop: time.sleep(1) instances = self._thread_local.driver.list_nodes() instance = searchInObjectList(instances, 'id', vmId) self._raise_on_failed_instance(instance) ip = self._get_instance_ip_address(instance, ipType or '', strict) if ip: vm['ip'] = ip return vm try: ip = self._get_instance_ip_address(instance, ipType or '', False) # pylint: disable=W0703 except Exception: pass if ip: vm['ip'] = ip return vm raise Exceptions.ExecutionException( 'Timed out after %s sec, while waiting for IPs to be assigned to instances: %s' % (time_wait, vmId))
def start_server(auth_parms, server_data): """Function to start server, uuid in server_data""" server_uuid = server_data[0] server_state = get_server_state(auth_parms, server_uuid) if server_state == 'STOPPED': rc = change_server_status(auth_parms=auth_parms, server_uuid=server_uuid, state='RUNNING') # change_server_status() waits on the server getting to the requested state, so we don't # need to call wait_for_server() here. However, we do (1) need to check the status and (2) # wait on the server actually being accessible (as opposed to having a RUNNING state in # FCO, which really just means that the underlying kvm process has started). # # 1. Check rc (0 is good) if (rc != 0): raise Exceptions.ExecutionException("Failed to put server " + server_uuid + " in to running state") server_resultset = list_resource_by_uuid(auth_parms, uuid=server_uuid, res_type='SERVER') print("Server result set is:") print server_resultset server_ip = server_resultset['list'][0]['nics'][0]['ipAddresses'][0][ 'ipAddress'] # yuk print("server IP=" + server_ip) # Step 2. Wait on it being accessible. It is possible that the server doesn't have ssh installed, # or it is firewalled, so don't fail here if we can't connect, just carry on and let # the caller deal with any potential issue. The alternative is a hard-coded sleep, or # trying a ping (platform specific and/or root privs needed). is_ssh_port_open(server_ip, 30) server_data.append(server_ip) return server_data
def append_ssh_pubkey_to_authorized_keys(pubkey, user=''): if is_windows(): return if not user: user = getpass.getuser() if not user_exists(user): raise Exceptions.ExecutionException('User %s not found.' % user) dot_ssh_path = os.path.expanduser('~' + user) + '/.ssh' try: os.mkdir(dot_ssh_path) os.chmod(dot_ssh_path, stat.S_IRWXU) except: pass file_content = '\n# Keys added by SlipStream\n%s\n# End of keys added by SlipStream\n' % pubkey authorized_keys_path = dot_ssh_path + '/authorized_keys' fileAppendContent(authorized_keys_path, file_content) os.chmod(authorized_keys_path, stat.S_IRUSR | stat.S_IWUSR) execute('chown -R %(user)s:$(id -g %(user)s) %(ssh_path)s' % { 'user': user, 'ssh_path': dot_ssh_path }, noWait=True, shell=True, withStderr=True, withOutput=True) execute('restorecon -R %s || true;' % dot_ssh_path, noWait=True, shell=True, withStderr=True, withOutput=True)
def _get_my_node_instance_name(config_holder): try: return config_holder.node_instance_name except Exception: raise Exceptions.ExecutionException( 'Failed to get the node instance name of the the current VM')
def _install_packages_local(packages): cmd = 'apt-get -y install %s' % ' '.join(packages) rc, output = commands.getstatusoutput(cmd) if rc != 0: raise Exceptions.ExecutionException( 'Could not install required packages: %s\n%s' % (cmd, output))