def remoteRunCommand(user, host, command, sshKey=None, password='', nohup=False):
sudo = 'sudo' if user != 'root' else ''
nohup_cmd = (nohup is True) and 'at now -f %s' or '%s'
cmd = ('%s %s' % (sudo, nohup_cmd % command)).strip()
rc, stderr = sshCmdWithStderr(cmd, host, user=user, sshKey=sshKey, password=password)
if rc != 0:
if nohup and (re.search('.* (command )?not found.*', stderr, re.MULTILINE) or
not _remote_command_exists('at', host, user, sshKey, password)):
nohup_cmd = _remote_command_exists('nohup', host, user, sshKey, password) and 'nohup %s' or '%s'
cmd = ('%s %s >/dev/null 2>&1 </dev/null &' % (sudo, nohup_cmd % command)).strip()
rc, stderr = sshCmdWithStderr(cmd, host, user=user, sshKey=sshKey, password=password)
if rc != 0:
raise Exceptions.ExecutionException("An error occurred while executing the command: %s\n%s." %
(command, stderr))
else:
raise Exceptions.ExecutionException("An error occurred while executing the command: %s\n%s." %
(command, stderr))
else:
# Check stderr as atd may not be running, though return code was 0.
# Starting atd service will start the command.
if re.search('.*No atd running\?', stderr, re.MULTILINE):
remote_start_service('atd', user, host, sshKey, password)
return rc, stderr
def _initialization(self, user_info):
util.printStep('Initialize the OpenStack connector.')
self._thread_local.driver = self._get_driver(user_info)
try:
self.flavors = self._thread_local.driver.list_sizes()
except InvalidCredsError as e:
raise Exceptions.ValidationException("%s: Invalid Cloud credentials. "
"Please check your Cloud username, password, project name and domain "
"(if applicable) in your SlipStream user profile."
% self.get_cloud_service_name())
except LibcloudError as e:
if e.message == 'Could not find specified endpoint':
raise Exceptions.ValidationException("%s: Invalid Cloud configuration. "
"Please ask your SlipStream administrator to check the region, "
"service-type and service-name."
% self.get_cloud_service_name())
else:
raise
if self.is_deployment():
self._get_iaas_attributes()
self._import_keypair(user_info)
elif self.is_build_image():
self._get_iaas_attributes()
self._create_keypair_and_set_on_user_info(user_info)
def _handle5xx(resp):
if resp.status_code == SERVICE_UNAVAILABLE_ERROR:
raise Exceptions.ServiceUnavailableError(
"SlipStream is in maintenance.")
else:
raise Exceptions.ServerError(
"Failed calling method %s on url %s, with reason: %d: %s" %
(method, url, resp.status_code, resp.reason))
def _request(headers):
try:
return self.session.request(method,
url,
data=body,
headers=headers,
verify=self.host_cert_verify)
except requests.exceptions.InvalidSchema as ex:
raise Exceptions.ClientError("Malformed URL: %s" % ex)
except httplib.BadStatusLine:
raise Exceptions.NetworkError(
"Error: BadStatusLine contacting: %s" % url)
def _wait_and_get_instance_ip_address(self, vm):
# vm =
# {
# 'ip': '',
# 'instance': <slipstream.cloudconnectors.okeanos.NodeDetails object at 0x0000000>,
# 'id': '549929',
# 'networkType': 'Public'
# }
self.log("vm = %s" % vm)
nodeDetails = vm['instance']
nodeId = nodeDetails.id
nodeDetailsActive = self.okeanosClient.waitNodeStatus(nodeId, NodeStatus.ACTIVE)
nodeDetails.updateIPsAndStatusFrom(nodeDetailsActive)
ip = nodeDetails.ipv4s[0]
# Wait for SSH connectivity
remoteUsername = "******"
sshTimeout = 7.0
self.okeanosClient.waitSshOnNode(nodeDetails, username=remoteUsername, timeout=sshTimeout)
self.log("id = %s, ip = %s, adminPass = %s" % (nodeId, ip, nodeDetails.adminPass))
if ip:
vm['ip'] = ip
self.log("vm = %s" % vm)
return vm
raise Exceptions.ExecutionException('Timed out while waiting for IPs to be assigned to instances: %s' % nodeId)
def _set_user_info_on_stratuslab_config_holder(self,
user_info,
build_image=False,
run_instance=True):
try:
self.slConfigHolder.set('endpoint', user_info.get_cloud_endpoint())
self.slConfigHolder.set('username', user_info.get_cloud_username())
self.slConfigHolder.set('password', user_info.get_cloud_password())
sshPubKeysFile = self.__populate_ssh_pub_keys_file(user_info)
self.slConfigHolder.set('userPublicKeyFile', sshPubKeysFile)
if run_instance or build_image:
self.slConfigHolder.set(
'marketplaceEndpoint',
user_info.get_cloud('marketplace.endpoint'))
if build_image:
self.slConfigHolder.set(
'author', '%s %s' %
(user_info.get_first_name(), user_info.get_last_name()))
self.slConfigHolder.set('authorEmail', user_info.get_email())
self.slConfigHolder.set('saveDisk', True)
except KeyError, ex:
raise Exceptions.ExecutionException(
'Error bootstrapping from User Parameters. %s' % str(ex))
def remoteRunScript(user, host, script, sshKey=None, password='', nohup=False):
fd, scriptFile = tempfile.mkstemp()
try:
os.write(fd, script)
os.close(fd)
os.chmod(scriptFile, 0755)
dstScriptFile = '/tmp/%s' % os.path.basename(scriptFile)
retry_count = 3
while True:
rc, stderr = scp(scriptFile, dstScriptFile, user, host,
sshKey=sshKey, password=password, withStderr=True)
if rc != 0:
if retry_count <= 0:
raise Exceptions.ExecutionException("An error occurred while uploading "
"script to %s: %s" % (host, stderr))
else:
time.sleep(5)
retry_count -= 1
else:
break
sshCmdWithStderr('chmod 0755 %s' % dstScriptFile, host, sshKey=sshKey,
user=user, password=password)
finally:
try:
os.unlink(scriptFile)
except:
pass
return remoteRunCommand(user, host, dstScriptFile, sshKey, password, nohup)
def get_cloudconnector_modulename_by_cloudname(cloudname):
for module_name in get_cloudconnector_modulenames():
connector_class = loadModule(module_name).getConnectorClass()
if getattr(connector_class, 'cloudName') == cloudname:
return module_name
raise Exceptions.NotFoundError(
"Failed to find cloud connector module for cloud %s." % cloudname)
def _ensure_pdisk_endpoint_is_set(self):
slipstream_pdisk_endpoint = os.environ.get('SLIPSTREAM_PDISK_ENDPOINT')
if slipstream_pdisk_endpoint:
self.slConfigHolder.set('pdiskEndpoint', slipstream_pdisk_endpoint)
if not self.slConfigHolder.pdiskEndpoint:
raise Exceptions.ExecutionException(
'PDisk endpoint should be set on StratusLab ConfigHolder')
def getRunState(self, uuid=None, ignoreAbort=True):
if not uuid and not self.diid:
raise Exceptions.ExecutionException("Run ID should be provided "
"to get state.")
state_key = NodeDecorator.globalNamespacePrefix + NodeDecorator.STATE_KEY
self.run_url = self.runEndpoint + '/' + (uuid or self.diid)
return self.getRuntimeParameter(state_key, ignoreAbort=ignoreAbort)
def _stop_deployment(self):
errors = []
for nodename, runner in self.get_vms().items():
vm_id = runner.vmIds[0]
try:
vm_info = runner.cloud._vmInfo(vm_id)
uuids_list = self._get_volatile_disk_ids_to_delete(
etree.fromstring(vm_info))
runner.killInstances()
self._wait_vm_in_state(['Done', 'Failed'], runner, vm_id)
time.sleep(2)
self._delete_volatile_disks(uuids_list)
except Exception as ex:
traceback.print_exc()
try:
time.sleep(2)
runner.killInstances()
self._delete_volatile_disks(uuids_list)
except Exception as ex:
traceback.print_exc()
errors.append('Error killing node %s\n%s' %
(nodename, str(ex)))
if errors:
raise Exceptions.CloudError('Failed stopping following instances. '
'Details: %s' %
'\n -> '.join(errors))
def _waitCanConnectWithWinrmOrAbort(self, winrm):
try:
self._waitCanConnectWithWinrmOrTimeout(winrm, self.TIMEOUT_CONNECT)
except Exception as ex:
raise Exceptions.ExecutionException("Failed to connect to "
"%s: %s" %
(winrm.endpoint, str(ex)))
def validate_ram_cpu_size(auth_parms, prod_offer, cpu_size, ram_size):
# Check if the cpu and ram input is negative
if ((int(cpu_size) <= 0) or ((int(ram_size) <= 0))):
raise Exceptions.ExecutionException(
"CPU and RAM amount should be greater than 0")
prod_offer = get_prod_offer(auth_parms, prod_offer)
prod_component = prod_offer['componentConfig']
print prod_component
prod_conf_cpu = prod_component[0]['productConfiguredValues']
validator_cpu = prod_conf_cpu[0]
validateString_cpu = validator_cpu['validator']['validateString']
validateStringList_cpu = validateString_cpu.split("-")
print 'Valid CPU Core for product offer: '
print validateString_cpu
cpu_allowed = False
# For the Standard disk PO the validateString returned is: 1-10.
# Validation will fail if the format changes.
if (int(cpu_size) >= int(validateStringList_cpu[0])
and int(cpu_size) <= int(validateStringList_cpu[1])):
cpu_allowed = True
if (cpu_allowed != True):
raise Exceptions.ExecutionException(
"Invalid disk size for the product offer. Valid Disk sizes: %s" %
validateString_cpu)
prod_conf_ram = prod_component[1]['productConfiguredValues']
validator_ram = prod_conf_ram[0]
validateString_ram = validator_ram['validator']['validateString']
validateStringList_ram = validateString_ram.split(",")
print 'Valid RAM amount for product offer: '
print validateString_ram
ram_allowed = False
for size in validateStringList_ram:
if (int(size) == int(ram_size)):
ram_allowed = True
if (ram_allowed != True):
raise Exceptions.ExecutionException(
"Invalid disk size for the product offer. Valid Disk sizes: %s" %
validateString_ram)
def _start_image_on_cloudstack(self, user_info, node_instance, vm_name):
instance_name = self.format_instance_name(vm_name)
instance_type = node_instance.get_instance_type()
ip_type = node_instance.get_network_type()
keypair = None
contextualization_script = None
if not node_instance.is_windows():
keypair = user_info.get_keypair_name()
contextualization_script = self._get_bootstrap_script_if_not_build_image(node_instance)
security_groups = node_instance.get_security_groups()
security_groups = (len(security_groups) > 0) and security_groups or None
try:
size = [i for i in self.sizes if i.name == instance_type][0]
except IndexError:
raise Exceptions.ParameterNotFoundException("Couldn't find the specified instance type: %s" % instance_type)
image = self._get_image(node_instance)
if node_instance.is_windows():
instance = self.libcloud_driver.create_node(
name=instance_name,
size=size,
image=image,
location=self.zone,
ex_security_groups=security_groups)
else:
instance = self.libcloud_driver.create_node(
name=instance_name,
size=size,
image=image,
location=self.zone,
ex_keyname=keypair,
ex_userdata=contextualization_script,
ex_security_groups=security_groups)
ip = self._get_instance_ip_address(instance, ip_type)
if not ip:
raise Exceptions.ExecutionException("Couldn't find a '%s' IP" % ip_type)
vm = dict(instance=instance,
ip=ip,
id=instance.id)
return vm
def _rpc_execute(self, command, *args):
proxy = self._create_rpc_connection()
remote_function = getattr(proxy, command)
success, output_or_error_msg, err_code = \
remote_function(self._create_session_string(), *args)
if not success:
raise Exceptions.ExecutionException(output_or_error_msg)
return output_or_error_msg
def _attach_disk_and_report(self, node_instance, reporter):
attached_disk = self._attach_disk(node_instance)
if not attached_disk:
raise Exceptions.ExecutionException(
'Attached disk name not provided by connector after disk attach operation.'
)
if hasattr(reporter, '__call__'):
reporter(node_instance, attached_disk)
def waitUntilSshCanConnectOrTimeout(host, timeout, user='******', password='',
sshKey=None, **kwargs):
"""Returns True on success or raises on any unhandled failures.
"""
kind = password and 'api' or 'cli'
time_stop = time.time() + timeout
timeout_connect = 3
auth_failures = 20
reason = 'Unknown'
while (time_stop - time.time()) >= 0:
kwargs_ = copy.copy(kwargs)
try:
if True == globals()['_ssh_can_connect_' + kind](host, user,
sshKey=sshKey, password=password,
timeout=timeout_connect, **kwargs_):
return True
except SshConnectionRefused as ex:
_printDetail(str(ex), kwargs_)
reason = 'SshConnectionRefused'
time.sleep(5)
except SshHostUnreachable as ex:
_printDetail(str(ex), kwargs_)
reason = 'SshHostUnreachable'
time.sleep(10)
except SshConnectionResetByPeer as ex:
_printDetail(str(ex), kwargs_)
reason = 'SshConnectionResetByPeer'
time.sleep(5)
except SshConnectionTimedOut as ex:
_printDetail(str(ex), kwargs_)
reason = 'SshConnectionTimedOut'
timeout_connect *= 2
except SshServerNameNotKnown as ex:
_printDetail(str(ex), kwargs_)
raise
except SshAuthFailed as ex:
_printDetail('%s - %i attempts left.' % (str(ex).strip('\n\r\t '), auth_failures), kwargs_)
reason = 'SshAuthFailed'
if auth_failures <= 0:
raise
auth_failures -= 1
time.sleep(5)
except SshFailedToConnect as ex:
reason = 'SshFailedToConnect'
_printDetail(str(ex), kwargs_)
time.sleep(5)
except paramiko.SSHException as ex:
_printDetail(str(ex), kwargs_)
reason = 'paramiko.SSHException:', str(ex)
time.sleep(5)
except exceptions.EOFError as ex:
_printDetail(str(ex), kwargs_)
reason = 'exceptions.EOFError'
time.sleep(5)
raise Exceptions.TimeoutException('Failed to connect after %s sec. \nReason: %s'
% (timeout, reason))
def _import_keypair(self, user_info):
kp_name = 'ss-key-%i' % int(time.time())
public_key = user_info.get_public_keys()
try:
kp = self._thread_local.driver.ex_import_keypair_from_string(kp_name, public_key)
except Exception as ex:
raise Exceptions.ExecutionException('Cannot import the public key. Reason: %s' % ex)
kp_name = kp.name
user_info.set_keypair_name(kp_name)
return kp_name
def _wait_vm_in_state(self, vm_id, state, time_out, time_sleep=30):
time_stop = time.time() + time_out
current_state = self._get_vm_state(vm_id)
while current_state != self.VM_STATE.index(state):
if time.time() > time_stop:
raise Exceptions.ExecutionException(
'Timed out while waiting VM {0} to enter in state {1}'.format(vm_id, state))
time.sleep(time_sleep)
current_state = self._get_vm_state(vm_id)
return current_state
def _wait_image_not_in_state(self, image_id, state, time_out, time_sleep=30):
time_stop = time.time() + time_out
current_state = self._get_image_state(image_id)
while current_state == self.IMAGE_STATE.index(state):
if time.time() > time_stop:
raise Exceptions.ExecutionException(
'Timed out while waiting for image {0} to be in state {1}'.format(image_id, state))
time.sleep(time_sleep)
current_state = self._get_image_state(image_id)
return current_state
def _import_keypair(self, user_info):
kp_name = 'ss-key-%i' % int(time.time() * 1000000)
public_key = user_info.get_public_keys()
try:
kp = self.libcloud_driver.import_key_pair_from_string(kp_name, public_key)
except Exception as e:
user_info.set_keypair_name(None)
raise Exceptions.ExecutionException('Cannot import the public key. Reason: %s' % e)
kp_name = kp.name
user_info.set_keypair_name(kp_name)
return kp_name
def _getConfigFile(self):
for location in LocalContextualizer.LOCAL_CONTEXTUALIZATION_LOCATIONS:
filename = os.path.join(
location, LocalContextualizer.LOCAL_CONTEXTUALIZATION_FILENAME)
if os.path.exists(filename):
util.printDetail(
'Using local contextualization file: %s' % filename,
self.verboseLevel)
return filename
raise Exceptions.ConfigurationError(
'Failed to find local contextualization file.')
def getRuntimeParameter(self, key, ignoreAbort=False):
url = self.run_url + '/' + key
if self.ignoreAbort or ignoreAbort:
url += SlipStreamHttpClient.URL_IGNORE_ABORT_ATTRIBUTE_QUERY
try:
_, content = self._httpGet(url, accept='text/plain')
except Exceptions.NotFoundError as ex:
raise Exceptions.NotFoundError('"%s" for %s' % (str(ex), key))
return content.strip().strip('"').strip("'")
def remove_bad_char_in_instance_name(self, name):
try:
newname = re.sub(r'[^a-zA-Z0-9-]', '', name)
m = re.search('[a-zA-Z]([a-zA-Z0-9-]*[a-zA-Z0-9]+)?', newname)
return m.string[m.start():m.end()]
except:
raise Exceptions.ExecutionException(
'Cannot handle the instance name "%s". Instance name can '
'contain ASCII letters "a" through "z", the digits "0" '
'through "9", and the hyphen ("-"), must be between 1 and 63 '
'characters long, and can\'t start or end with "-" '
'and can\'t start with digit' % name)
def _wait_image_creation_completed(self, image_id):
time_wait = 600
time_stop = time.time() + time_wait
img = None
while not img:
if time.time() > time_stop:
raise Exceptions.ExecutionException(
'Timed out while waiting for image "%s" to be created' % image_id)
time.sleep(1)
images = self._thread_local.driver.list_images()
img = searchInObjectList(images, 'id', image_id)
def createExecutor(configHolder):
cloudWrapper = CloudWrapper(configHolder)
category = cloudWrapper.get_run_category()
configHolder.set(KEY_RUN_CATEGORY, category)
cloudWrapper.initCloudConnector(configHolder)
if category == RUN_CATEGORY_IMAGE:
return OrchestratorImageBuildExecutor(cloudWrapper, configHolder)
elif category == RUN_CATEGORY_DEPLOYMENT:
return OrchestratorDeploymentExecutor(cloudWrapper, configHolder)
else:
raise Exceptions.ClientError("Unknown category: %s" % category)
def _vm_get_root_disk_size_from_pdisk(disk_source, config_holder):
disk_source = disk_source.replace('/', ':')
pdisk_endpoint = ':'.join(disk_source.split(':')[1:3])
config_holder.set('pdiskUsername', config_holder.username)
config_holder.set('pdiskPassword', config_holder.password)
config_holder.set('pdiskEndpoint', pdisk_endpoint)
pdisk = VolumeManagerFactory.create(config_holder)
image_uuid = _disk_source_get_image_id(disk_source)
volume = pdisk.describeVolumes({'uuid': ['^%s$' % image_uuid]})
if len(volume) == 0:
raise Exceptions.ExecutionException('Failed to describe volume in %s with UUID %s' %
(pdisk_endpoint, image_uuid))
return int(volume[0]['size'])
def resize_cpu_ram(auth_parms, vm_uuid, serverName, clusterUUID, vdcUUID, cpu,
ram):
""" Function to resize the server """
product_offer = 'Standard Server'
server_po_uuid = get_prod_offer_uuid(auth_parms, product_offer)
if (server_po_uuid == ""):
raise Exceptions.ExecutionException("No '" + product_offer +
"' Product Offer found")
if (server_po_uuid != ""):
# Modify the server
modify_cpu_ram(auth_parms, vm_uuid, serverName, clusterUUID, vdcUUID,
cpu, ram, server_po_uuid)
def _wait_instance_in_running_state(self, instance_id):
time_wait = 600
time_stop = time.time() + time_wait
state = ''
while state != NodeState.RUNNING:
if time.time() > time_stop:
raise Exceptions.ExecutionException(
'Timed out while waiting for instance "%s" enter in running state'
% instance_id)
time.sleep(1)
nodes = self._thread_local.driver.list_nodes()
node = searchInObjectList(nodes, 'id', instance_id)
self._raise_on_failed_instance(node)
state = node.state
def _stop_deployment(self):
errors = []
for nodename, runner in self.get_vms().items():
try:
runner.killInstances()
except Exception:
# Retry killing instances.
try:
time.sleep(2)
runner.killInstances()
except Exception as ex:
errors.append('Error killing node %s\n%s' %
(nodename, str(ex)))
if errors:
raise Exceptions.CloudError('Failed stopping following instances. '
'Details: %s' %
'\n -> '.join(errors))
