def remoteRunCommand(user, host, command, sshKey=None, password='', nohup=False):
sudo = 'sudo' if user != 'root' else ''
nohup_cmd = (nohup is True) and 'at now -f %s' or '%s'
cmd = ('%s %s' % (sudo, nohup_cmd % command)).strip()
rc, stderr = sshCmdWithStderr(cmd, host, user=user, sshKey=sshKey, password=password)
if rc != 0:
if nohup and (re.search('.* (command )?not found.*', stderr, re.MULTILINE) or
not _remote_command_exists('at', host, user, sshKey, password)):
nohup_cmd = _remote_command_exists('nohup', host, user, sshKey, password) and 'nohup %s' or '%s'
cmd = ('%s %s >/dev/null 2>&1 </dev/null &' % (sudo, nohup_cmd % command)).strip()
rc, stderr = sshCmdWithStderr(cmd, host, user=user, sshKey=sshKey, password=password)
if rc != 0:
raise Exceptions.ExecutionException("An error occurred while executing the command: %s\n%s." %
(command, stderr))
else:
raise Exceptions.ExecutionException("An error occurred while executing the command: %s\n%s." %
(command, stderr))
else:
# Check stderr as atd may not be running, though return code was 0.
# Starting atd service will start the command.
if re.search('.*No atd running\?', stderr, re.MULTILINE):
remote_start_service('atd', user, host, sshKey, password)
return rc, stderr
def getRunState(self, uuid=None, ignoreAbort=True):
if not uuid and not self.diid:
raise Exceptions.ExecutionException("Run ID should be provided "
"to get state.")
state_key = NodeDecorator.globalNamespacePrefix + NodeDecorator.STATE_KEY
self.run_url = self.runEndpoint + '/' + (uuid or self.diid)
return self.getRuntimeParameter(state_key, ignoreAbort=ignoreAbort)
def _set_user_info_on_stratuslab_config_holder(self,
user_info,
build_image=False,
run_instance=True):
try:
self.slConfigHolder.set('endpoint', user_info.get_cloud_endpoint())
self.slConfigHolder.set('username', user_info.get_cloud_username())
self.slConfigHolder.set('password', user_info.get_cloud_password())
sshPubKeysFile = self.__populate_ssh_pub_keys_file(user_info)
self.slConfigHolder.set('userPublicKeyFile', sshPubKeysFile)
if run_instance or build_image:
self.slConfigHolder.set(
'marketplaceEndpoint',
user_info.get_cloud('marketplace.endpoint'))
if build_image:
self.slConfigHolder.set(
'author', '%s %s' %
(user_info.get_first_name(), user_info.get_last_name()))
self.slConfigHolder.set('authorEmail', user_info.get_email())
self.slConfigHolder.set('saveDisk', True)
except KeyError, ex:
raise Exceptions.ExecutionException(
'Error bootstrapping from User Parameters. %s' % str(ex))
def _wait_and_get_instance_ip_address(self, vm):
# vm =
# {
# 'ip': '',
# 'instance': <slipstream.cloudconnectors.okeanos.NodeDetails object at 0x0000000>,
# 'id': '549929',
# 'networkType': 'Public'
# }
self.log("vm = %s" % vm)
nodeDetails = vm['instance']
nodeId = nodeDetails.id
nodeDetailsActive = self.okeanosClient.waitNodeStatus(nodeId, NodeStatus.ACTIVE)
nodeDetails.updateIPsAndStatusFrom(nodeDetailsActive)
ip = nodeDetails.ipv4s[0]
# Wait for SSH connectivity
remoteUsername = "******"
sshTimeout = 7.0
self.okeanosClient.waitSshOnNode(nodeDetails, username=remoteUsername, timeout=sshTimeout)
self.log("id = %s, ip = %s, adminPass = %s" % (nodeId, ip, nodeDetails.adminPass))
if ip:
vm['ip'] = ip
self.log("vm = %s" % vm)
return vm
raise Exceptions.ExecutionException('Timed out while waiting for IPs to be assigned to instances: %s' % nodeId)
def remoteRunScript(user, host, script, sshKey=None, password='', nohup=False):
fd, scriptFile = tempfile.mkstemp()
try:
os.write(fd, script)
os.close(fd)
os.chmod(scriptFile, 0755)
dstScriptFile = '/tmp/%s' % os.path.basename(scriptFile)
retry_count = 3
while True:
rc, stderr = scp(scriptFile, dstScriptFile, user, host,
sshKey=sshKey, password=password, withStderr=True)
if rc != 0:
if retry_count <= 0:
raise Exceptions.ExecutionException("An error occurred while uploading "
"script to %s: %s" % (host, stderr))
else:
time.sleep(5)
retry_count -= 1
else:
break
sshCmdWithStderr('chmod 0755 %s' % dstScriptFile, host, sshKey=sshKey,
user=user, password=password)
finally:
try:
os.unlink(scriptFile)
except:
pass
return remoteRunCommand(user, host, dstScriptFile, sshKey, password, nohup)
def _waitCanConnectWithWinrmOrAbort(self, winrm):
try:
self._waitCanConnectWithWinrmOrTimeout(winrm, self.TIMEOUT_CONNECT)
except Exception as ex:
raise Exceptions.ExecutionException("Failed to connect to "
"%s: %s" %
(winrm.endpoint, str(ex)))
def _ensure_pdisk_endpoint_is_set(self):
slipstream_pdisk_endpoint = os.environ.get('SLIPSTREAM_PDISK_ENDPOINT')
if slipstream_pdisk_endpoint:
self.slConfigHolder.set('pdiskEndpoint', slipstream_pdisk_endpoint)
if not self.slConfigHolder.pdiskEndpoint:
raise Exceptions.ExecutionException(
'PDisk endpoint should be set on StratusLab ConfigHolder')
def validate_ram_cpu_size(auth_parms, prod_offer, cpu_size, ram_size):
# Check if the cpu and ram input is negative
if ((int(cpu_size) <= 0) or ((int(ram_size) <= 0))):
raise Exceptions.ExecutionException(
"CPU and RAM amount should be greater than 0")
prod_offer = get_prod_offer(auth_parms, prod_offer)
prod_component = prod_offer['componentConfig']
print prod_component
prod_conf_cpu = prod_component[0]['productConfiguredValues']
validator_cpu = prod_conf_cpu[0]
validateString_cpu = validator_cpu['validator']['validateString']
validateStringList_cpu = validateString_cpu.split("-")
print 'Valid CPU Core for product offer: '
print validateString_cpu
cpu_allowed = False
# For the Standard disk PO the validateString returned is: 1-10.
# Validation will fail if the format changes.
if (int(cpu_size) >= int(validateStringList_cpu[0])
and int(cpu_size) <= int(validateStringList_cpu[1])):
cpu_allowed = True
if (cpu_allowed != True):
raise Exceptions.ExecutionException(
"Invalid disk size for the product offer. Valid Disk sizes: %s" %
validateString_cpu)
prod_conf_ram = prod_component[1]['productConfiguredValues']
validator_ram = prod_conf_ram[0]
validateString_ram = validator_ram['validator']['validateString']
validateStringList_ram = validateString_ram.split(",")
print 'Valid RAM amount for product offer: '
print validateString_ram
ram_allowed = False
for size in validateStringList_ram:
if (int(size) == int(ram_size)):
ram_allowed = True
if (ram_allowed != True):
raise Exceptions.ExecutionException(
"Invalid disk size for the product offer. Valid Disk sizes: %s" %
validateString_ram)
def _rpc_execute(self, command, *args):
proxy = self._create_rpc_connection()
remote_function = getattr(proxy, command)
success, output_or_error_msg, err_code = \
remote_function(self._create_session_string(), *args)
if not success:
raise Exceptions.ExecutionException(output_or_error_msg)
return output_or_error_msg
def _attach_disk_and_report(self, node_instance, reporter):
attached_disk = self._attach_disk(node_instance)
if not attached_disk:
raise Exceptions.ExecutionException(
'Attached disk name not provided by connector after disk attach operation.'
)
if hasattr(reporter, '__call__'):
reporter(node_instance, attached_disk)
def _wait_vm_in_state(self, vm_id, state, time_out, time_sleep=30):
time_stop = time.time() + time_out
current_state = self._get_vm_state(vm_id)
while current_state != self.VM_STATE.index(state):
if time.time() > time_stop:
raise Exceptions.ExecutionException(
'Timed out while waiting VM {0} to enter in state {1}'.format(vm_id, state))
time.sleep(time_sleep)
current_state = self._get_vm_state(vm_id)
return current_state
def _wait_image_not_in_state(self, image_id, state, time_out, time_sleep=30):
time_stop = time.time() + time_out
current_state = self._get_image_state(image_id)
while current_state == self.IMAGE_STATE.index(state):
if time.time() > time_stop:
raise Exceptions.ExecutionException(
'Timed out while waiting for image {0} to be in state {1}'.format(image_id, state))
time.sleep(time_sleep)
current_state = self._get_image_state(image_id)
return current_state
def _import_keypair(self, user_info):
kp_name = 'ss-key-%i' % int(time.time())
public_key = user_info.get_public_keys()
try:
kp = self._thread_local.driver.ex_import_keypair_from_string(kp_name, public_key)
except Exception as ex:
raise Exceptions.ExecutionException('Cannot import the public key. Reason: %s' % ex)
kp_name = kp.name
user_info.set_keypair_name(kp_name)
return kp_name
def _import_keypair(self, user_info):
kp_name = 'ss-key-%i' % int(time.time() * 1000000)
public_key = user_info.get_public_keys()
try:
kp = self.libcloud_driver.import_key_pair_from_string(kp_name, public_key)
except Exception as e:
user_info.set_keypair_name(None)
raise Exceptions.ExecutionException('Cannot import the public key. Reason: %s' % e)
kp_name = kp.name
user_info.set_keypair_name(kp_name)
return kp_name
def _wait_image_creation_completed(self, image_id):
time_wait = 600
time_stop = time.time() + time_wait
img = None
while not img:
if time.time() > time_stop:
raise Exceptions.ExecutionException(
'Timed out while waiting for image "%s" to be created' % image_id)
time.sleep(1)
images = self._thread_local.driver.list_images()
img = searchInObjectList(images, 'id', image_id)
def remove_bad_char_in_instance_name(self, name):
try:
newname = re.sub(r'[^a-zA-Z0-9-]', '', name)
m = re.search('[a-zA-Z]([a-zA-Z0-9-]*[a-zA-Z0-9]+)?', newname)
return m.string[m.start():m.end()]
except:
raise Exceptions.ExecutionException(
'Cannot handle the instance name "%s". Instance name can '
'contain ASCII letters "a" through "z", the digits "0" '
'through "9", and the hyphen ("-"), must be between 1 and 63 '
'characters long, and can\'t start or end with "-" '
'and can\'t start with digit' % name)
def _vm_get_root_disk_size_from_pdisk(disk_source, config_holder):
disk_source = disk_source.replace('/', ':')
pdisk_endpoint = ':'.join(disk_source.split(':')[1:3])
config_holder.set('pdiskUsername', config_holder.username)
config_holder.set('pdiskPassword', config_holder.password)
config_holder.set('pdiskEndpoint', pdisk_endpoint)
pdisk = VolumeManagerFactory.create(config_holder)
image_uuid = _disk_source_get_image_id(disk_source)
volume = pdisk.describeVolumes({'uuid': ['^%s$' % image_uuid]})
if len(volume) == 0:
raise Exceptions.ExecutionException('Failed to describe volume in %s with UUID %s' %
(pdisk_endpoint, image_uuid))
return int(volume[0]['size'])
def resize_cpu_ram(auth_parms, vm_uuid, serverName, clusterUUID, vdcUUID, cpu,
ram):
""" Function to resize the server """
product_offer = 'Standard Server'
server_po_uuid = get_prod_offer_uuid(auth_parms, product_offer)
if (server_po_uuid == ""):
raise Exceptions.ExecutionException("No '" + product_offer +
"' Product Offer found")
if (server_po_uuid != ""):
# Modify the server
modify_cpu_ram(auth_parms, vm_uuid, serverName, clusterUUID, vdcUUID,
cpu, ram, server_po_uuid)
def _wait_instance_in_running_state(self, instance_id):
time_wait = 600
time_stop = time.time() + time_wait
state = ''
while state != NodeState.RUNNING:
if time.time() > time_stop:
raise Exceptions.ExecutionException(
'Timed out while waiting for instance "%s" enter in running state'
% instance_id)
time.sleep(1)
nodes = self._thread_local.driver.list_nodes()
node = searchInObjectList(nodes, 'id', instance_id)
self._raise_on_failed_instance(node)
state = node.state
def _wait_can_connect_with_ssh_or_abort(self,
host,
username='',
password='',
sshKey=None):
self._print_detail('Check if we can connect to %s' % host)
try:
waitUntilSshCanConnectOrTimeout(host,
self.TIMEOUT_CONNECT,
sshKey=sshKey,
user=username,
password=password,
verboseLevel=self.verboseLevel)
except Exception as ex:
raise Exceptions.ExecutionException("Failed to connect to "
"%s: %s, %s" %
(host, type(ex), str(ex)))
def _start_image_on_cloudstack(self, user_info, node_instance, vm_name):
instance_name = self.format_instance_name(vm_name)
instance_type = node_instance.get_instance_type()
ip_type = node_instance.get_network_type()
keypair = None
contextualization_script = None
if not node_instance.is_windows():
keypair = user_info.get_keypair_name()
contextualization_script = self._get_bootstrap_script_if_not_build_image(node_instance)
security_groups = node_instance.get_security_groups()
security_groups = (len(security_groups) > 0) and security_groups or None
try:
size = [i for i in self.sizes if i.name == instance_type][0]
except IndexError:
raise Exceptions.ParameterNotFoundException("Couldn't find the specified instance type: %s" % instance_type)
image = self._get_image(node_instance)
if node_instance.is_windows():
instance = self.libcloud_driver.create_node(
name=instance_name,
size=size,
image=image,
location=self.zone,
ex_security_groups=security_groups)
else:
instance = self.libcloud_driver.create_node(
name=instance_name,
size=size,
image=image,
location=self.zone,
ex_keyname=keypair,
ex_userdata=contextualization_script,
ex_security_groups=security_groups)
ip = self._get_instance_ip_address(instance, ip_type)
if not ip:
raise Exceptions.ExecutionException("Couldn't find a '%s' IP" % ip_type)
vm = dict(instance=instance,
ip=ip,
id=instance.id)
return vm
def _remove_additional_disk(self, vm_instance):
additional_disk_id = vm_instance.extra.get('metadata', {}).get('additional_disk_id')
if additional_disk_id:
DISK_STATE_AVAILABLE = 0
time_wait = 60
time_stop = time.time() + time_wait
additional_disk = self._thread_local.driver.ex_get_volume(additional_disk_id)
diskState = additional_disk.state
while diskState != DISK_STATE_AVAILABLE:
if time.time() > time_stop:
raise Exceptions.ExecutionException(
'Timed out while waiting for disk "%s" to be deleted' % additional_disk_id)
time.sleep(5)
diskState = self._thread_local.driver.ex_get_volume(additional_disk_id).state
self._thread_local.driver.destroy_volume(additional_disk)
def create_disk(auth_parms, prod_offer, disk_size, disk_name, vdc_uuid):
""" Function to create disk """
# get product offer uuid for the disk in question
prod_offer_uuid = get_prod_offer_uuid(auth_parms, prod_offer)
disk_job = rest_create_disk(auth_parms, vdc_uuid, prod_offer_uuid,
disk_name, disk_size)
disk_uuid = disk_job['itemUUID']
print("our newly created disk UUID=" + disk_uuid)
# Check the job completes
status = wait_for_job(auth_parms, disk_job['resourceUUID'], "SUCCESSFUL",
90)
if (status != 0):
raise Exceptions.ExecutionException(
"Failed to add create disk (uuid=" + disk_uuid + ")")
return disk_uuid
def _run_instance(self, image_id, slConfigHolder, max_attempts=3):
if max_attempts <= 0:
max_attempts = 1
attempt = 1
while True:
try:
runner = self._do_run_instance(image_id, slConfigHolder)
except socket.error, ex:
if attempt >= max_attempts:
import traceback
cause = ''
cause_lines = traceback.format_exception(*sys.exc_info())
for line in cause_lines:
cause += line
raise Exceptions.ExecutionException(
"Failed to launch instance after %i attempts: %s \nCaused by :\n%s"
% (attempt, str(ex), cause))
time.sleep(self.RUNINSTANCE_RETRY_TIMEOUT)
attempt += 1
else:
return runner
def validate_disk_size(auth_parms, prod_offer, disk_size, disk_name, vdc_uuid):
prod_offer = get_prod_offer(auth_parms, prod_offer)
prod_component = prod_offer['componentConfig']
print prod_component
prod_conf = prod_component[0]['productConfiguredValues']
validator = prod_conf[0]
validateString = validator['validator']['validateString']
validateStringList = validateString.split(",")
print 'Valid Disk Size for product offer: '
print validateString
allowed = False
for size in validateStringList:
if (int(size) == disk_size):
allowed = True
if (allowed != True):
raise Exceptions.ExecutionException(
"Invalid disk size for the product offer. Valid Disk sizes: %s" %
validateString)
def _wait_and_get_instance_ip_address(self, vm):
if vm.get('ip'):
return vm
time_wait = 180
time_stop = time.time() + time_wait
instance = vm['instance']
ipType = vm['networkType']
strict = vm['strict_ip_retrival']
vmId = vm['id']
ip = vm['ip']
while time.time() < time_stop:
time.sleep(1)
instances = self._thread_local.driver.list_nodes()
instance = searchInObjectList(instances, 'id', vmId)
self._raise_on_failed_instance(instance)
ip = self._get_instance_ip_address(instance, ipType or '', strict)
if ip:
vm['ip'] = ip
return vm
try:
ip = self._get_instance_ip_address(instance, ipType or '', False)
# pylint: disable=W0703
except Exception:
pass
if ip:
vm['ip'] = ip
return vm
raise Exceptions.ExecutionException(
'Timed out after %s sec, while waiting for IPs to be assigned to instances: %s' % (time_wait, vmId))
def start_server(auth_parms, server_data):
"""Function to start server, uuid in server_data"""
server_uuid = server_data[0]
server_state = get_server_state(auth_parms, server_uuid)
if server_state == 'STOPPED':
rc = change_server_status(auth_parms=auth_parms,
server_uuid=server_uuid,
state='RUNNING')
# change_server_status() waits on the server getting to the requested state, so we don't
# need to call wait_for_server() here. However, we do (1) need to check the status and (2)
# wait on the server actually being accessible (as opposed to having a RUNNING state in
# FCO, which really just means that the underlying kvm process has started).
#
# 1. Check rc (0 is good)
if (rc != 0):
raise Exceptions.ExecutionException("Failed to put server " +
server_uuid +
" in to running state")
server_resultset = list_resource_by_uuid(auth_parms,
uuid=server_uuid,
res_type='SERVER')
print("Server result set is:")
print server_resultset
server_ip = server_resultset['list'][0]['nics'][0]['ipAddresses'][0][
'ipAddress'] # yuk
print("server IP=" + server_ip)
# Step 2. Wait on it being accessible. It is possible that the server doesn't have ssh installed,
# or it is firewalled, so don't fail here if we can't connect, just carry on and let
# the caller deal with any potential issue. The alternative is a hard-coded sleep, or
# trying a ping (platform specific and/or root privs needed).
is_ssh_port_open(server_ip, 30)
server_data.append(server_ip)
return server_data
def append_ssh_pubkey_to_authorized_keys(pubkey, user=''):
if is_windows():
return
if not user:
user = getpass.getuser()
if not user_exists(user):
raise Exceptions.ExecutionException('User %s not found.' % user)
dot_ssh_path = os.path.expanduser('~' + user) + '/.ssh'
try:
os.mkdir(dot_ssh_path)
os.chmod(dot_ssh_path, stat.S_IRWXU)
except:
pass
file_content = '\n# Keys added by SlipStream\n%s\n# End of keys added by SlipStream\n' % pubkey
authorized_keys_path = dot_ssh_path + '/authorized_keys'
fileAppendContent(authorized_keys_path, file_content)
os.chmod(authorized_keys_path, stat.S_IRUSR | stat.S_IWUSR)
execute('chown -R %(user)s:$(id -g %(user)s) %(ssh_path)s' % {
'user': user,
'ssh_path': dot_ssh_path
},
noWait=True,
shell=True,
withStderr=True,
withOutput=True)
execute('restorecon -R %s || true;' % dot_ssh_path,
noWait=True,
shell=True,
withStderr=True,
withOutput=True)
def _get_my_node_instance_name(config_holder):
try:
return config_holder.node_instance_name
except Exception:
raise Exceptions.ExecutionException(
'Failed to get the node instance name of the the current VM')
def _install_packages_local(packages):
cmd = 'apt-get -y install %s' % ' '.join(packages)
rc, output = commands.getstatusoutput(cmd)
if rc != 0:
raise Exceptions.ExecutionException(
'Could not install required packages: %s\n%s' % (cmd, output))