def testQueueExceptionsRaises(self):
tr = TasksRunner(Mock(side_effect=EXCEPTION(EXCEPTION_MESSAGE)))
ntasks = 3
for _ in range(ntasks):
tr.put_task()
tr.run_tasks()
self.failUnlessRaises(EXCEPTION, tr.wait_tasks_processed)
def _stopInstances(self, instances):
tasksRunnner = TasksRunner()
for instance in instances:
driver = self._getDriver(self.user_info)
tasksRunnner.run_task(driver.destroy_node, (instance,))
tasksRunnner.wait_tasks_finished()
def __start_nodes_instances_and_clients(self, user_info, nodes_instances):
self.__tasks_runnner = TasksRunner(
self.__start_node_instance_and_client,
max_workers=self.max_iaas_workers,
verbose=self.verboseLevel)
for node_instance in nodes_instances.values():
self.__tasks_runnner.put_task(user_info, node_instance)
self.__tasks_runnner.run_tasks()
def testQueueExceptionsManualCheck(self):
tr = TasksRunner()
ntasks = 3
for _ in range(ntasks):
tr.run_task(Mock(side_effect=EXCEPTION(EXCEPTION_MESSAGE)))
while not tr._tasks_finished():
time.sleep(1)
assert tr._tasks_finished() is True
assert len(tr.threads) == ntasks
assert tr.exc_queue.qsize() == ntasks
for _ in range(ntasks):
_assert_exc_info(tr.exc_queue.get())
def testRunTaskNoExceptions(self):
tr = TasksRunner(Mock())
ntasks = 5
for _ in range(ntasks):
tr.put_task(None)
tr.run_tasks()
tr.wait_tasks_processed()
assert 0 == tr.tasks_queue.qsize()
assert tr._tasks_finished() is True
assert len(tr.workers) == ntasks
def set_tasks_and_run(self, nodes_instances, done_reporter):
"""
:param nodes_instances: list of node instances
:type nodes_instances: list [NodeInstance, ]
:param done_reporter: function that reports back to SlipStream
:type done_reporter: callable with signature `done_reporter(<NoneInstance>)`
"""
self._tasks_runner = TasksRunner(self._task_executor,
max_workers=self._max_workers,
verbose=self._verbose_level)
for node_instance in nodes_instances:
self._tasks_runner.put_task(node_instance, done_reporter)
self._tasks_runner.run_tasks()
def testMtasksNworkers(self):
max_workers = 10
tr = TasksRunner(Mock(), max_workers=10)
ntasks = 25
for _ in range(ntasks):
tr.put_task(None)
tr.run_tasks()
tr.wait_tasks_processed()
assert 0 == tr.tasks_queue.qsize()
assert tr._tasks_finished() is True
assert len(tr.workers) == max_workers
def _stop_instances(self, instances):
tasksRunnner = TasksRunner(self._stop_instance,
max_workers=self.max_iaas_workers,
verbose=self.verboseLevel)
for instance in instances:
tasksRunnner.put_task(instance)
tasksRunnner.run_tasks()
tasksRunnner.wait_tasks_processed()
def _stop_instances(self, instances):
max_workers = self._get_max_workers(self.configHolder)
tasksRunnner = TasksRunner(self._stop_instance,
max_workers=max_workers,
verbose=self.verboseLevel)
for instance in instances:
tasksRunnner.put_task(instance)
tasksRunnner.run_tasks()
tasksRunnner.wait_tasks_processed()
def testRunTaskNoExceptions(self):
tr = TasksRunner(Mock())
ntasks = 5
for _ in range(ntasks):
tr.put_task(None)
tr.run_tasks()
tr.wait_tasks_processed()
assert 0 == tr.tasks_queue.qsize()
assert tr._tasks_finished() is True
assert len(tr.workers) == ntasks
def __start_nodes_instances_and_clients(self, user_info, nodes_instances):
self.__tasks_runnner = TasksRunner(self.__start_node_instance_and_client,
max_workers=self.max_iaas_workers,
verbose=self.verboseLevel)
for node_instance in nodes_instances.values():
self.__tasks_runnner.put_task(user_info, node_instance)
self.__tasks_runnner.run_tasks()
def testMtasksNworkers(self):
max_workers = 10
tr = TasksRunner(Mock(), max_workers=10)
ntasks = 25
for _ in range(ntasks):
tr.put_task(None)
tr.run_tasks()
tr.wait_tasks_processed()
assert 0 == tr.tasks_queue.qsize()
assert tr._tasks_finished() is True
assert len(tr.workers) == max_workers
def _stop_instances(self, instances):
tasksRunnner = TasksRunner(self._stop_instance,
max_workers=self.max_iaas_workers,
verbose=self.verboseLevel)
for instance in instances:
tasksRunnner.put_task(instance)
tasksRunnner.run_tasks()
tasksRunnner.wait_tasks_processed()
def testRunTaskNoExceptions(self):
tr = TasksRunner()
tr.QUEUE_GET_TIMEOUT = 1
ntasks = 3
for _ in range(ntasks):
tr.run_task(Mock())
tr.wait_tasks_finished()
assert tr._tasks_finished() is True
assert len(tr.threads) == ntasks
def testQueueExceptionsManualCheck(self):
tr = TasksRunner(Mock(side_effect=EXCEPTION(EXCEPTION_MESSAGE)))
ntasks = 3
for _ in range(ntasks):
tr.put_task(None)
tr.run_tasks()
while not tr._tasks_finished():
time.sleep(1)
assert tr._tasks_finished() is True
assert len(tr.workers) == ntasks
assert tr.exc_queue.qsize() == ntasks
for _ in range(ntasks):
_assert_exc_info(tr.exc_queue.get())
class VmScaler(object):
def __init__(self, task_executor, max_workers, verbose_level):
self._task_executor = task_executor
self._max_workers = max_workers
self._verbose_level = verbose_level
self._tasks_runner = None
def set_tasks_and_run(self, nodes_instances, done_reporter):
"""
:param nodes_instances: list of node instances
:type nodes_instances: list [NodeInstance, ]
:param done_reporter: function that reports back to SlipStream
:type done_reporter: callable with signature `done_reporter(<NoneInstance>)`
"""
self._tasks_runner = TasksRunner(self._task_executor,
max_workers=self._max_workers,
verbose=self._verbose_level)
for node_instance in nodes_instances:
self._tasks_runner.put_task(node_instance, done_reporter)
self._tasks_runner.run_tasks()
def wait_tasks_finished(self):
if self._tasks_runner is not None:
self._tasks_runner.wait_tasks_processed()
class VmScaler(object):
def __init__(self, task_executor, max_workers, verbose_level):
self._task_executor = task_executor
self._max_workers = max_workers
self._verbose_level = verbose_level
self._tasks_runner = None
def set_tasks_and_run(self, nodes_instances, done_reporter):
"""
:param nodes_instances: list of node instances
:type nodes_instances: list [NodeInstance, ]
:param done_reporter: function that reports back to SlipStream
:type done_reporter: callable with signature `done_reporter(<NoneInstance>)`
"""
self._tasks_runner = TasksRunner(self._task_executor,
max_workers=self._max_workers,
verbose=self._verbose_level)
for node_instance in nodes_instances:
self._tasks_runner.put_task(node_instance, done_reporter)
self._tasks_runner.run_tasks()
def wait_tasks_finished(self):
if self._tasks_runner is not None:
self._tasks_runner.wait_tasks_processed()
def testQueueExceptionsRaises(self):
tr = TasksRunner(Mock(side_effect=EXCEPTION(EXCEPTION_MESSAGE)))
ntasks = 3
for _ in range(ntasks):
tr.put_task()
tr.run_tasks()
self.failUnlessRaises(EXCEPTION, tr.wait_tasks_processed)
def set_tasks_and_run(self, nodes_instances, done_reporter):
"""
:param nodes_instances: list of node instances
:type nodes_instances: list [NodeInstance, ]
:param done_reporter: function that reports back to SlipStream
:type done_reporter: callable with signature `done_reporter(<NoneInstance>)`
"""
self._tasks_runner = TasksRunner(self._task_executor,
max_workers=self._max_workers,
verbose=self._verbose_level)
for node_instance in nodes_instances:
self._tasks_runner.put_task(node_instance, done_reporter)
self._tasks_runner.run_tasks()
class BaseCloudConnector(object):
# ----- METHODS THAT CAN/SHOULD BE IMPLEMENTED IN CONNECTORS -----
def _initialization(self, user_info):
"""This method is called once before calling any others methods of the connector.
This method can be used to some initialization tasks like configuring the Cloud driver."""
pass
def _finalization(self, user_info):
"""This method is called once when all instances have been started or if an exception has occurred."""
pass
def _start_image(self, user_info, node_instance, vm_name):
"""Cloud specific VM provisioning.
Returns: node - cloud specific representation of a started VM."""
raise NotImplementedError()
def _build_image(self, user_info, node_instance):
"""This method is called during the Executing state of a build image. In most cases this method should call
self._build_image_increment() and then ask the Cloud to create an image from the instance.
The return value should be the new Cloud image id as a string."""
raise NotImplementedError()
def _wait_and_get_instance_ip_address(self, vm):
"""This method needs to be implemented by the connector if the latter not define the capability
'direct_ip_assignment'."""
return vm
def _stop_deployment(self):
"""This method should terminate the full vapp if the connector has the vapp capability.
This method should terminate all instances except the orchestrator if the connector doesn't have the vapp
capability."""
pass
def _stop_vms_by_ids(self, ids):
"""This method should destroy all VMs corresponding to VMs IDs of the list."""
raise NotImplementedError()
def _stop_vapps_by_ids(self, ids):
"""This method is used to destroy a full vApp if the capability 'vapp' is set."""
pass
def list_instances(self):
"""Returns list of VMs consumable by the connector's _vm_get_id(vm) and _vm_get_ip(vm).
"""
raise NotImplementedError()
def _vm_get_id(self, vm_instance):
"""Retrieve the VM ID from the vm_instance object returned by _start_image().
Returns: cloud ID of the instance."""
raise NotImplementedError()
def _vm_get_ip(self, vm_instance):
"""Retrieve an IP from the vm_instance object returned by _start_image().
Returns: one IP - Public or Private."""
raise NotImplementedError()
def _vm_get_ports_mapping(self, vm_instance):
"""Retrieve ports mapping from the vm_instance object returned by _start_image().
Returns: [<protocol>:<published_port>:<target_port> ...] e.g. tcp:8080:80 ."""
pass
def _vm_get_password(self, vm_instance):
"""Retrieve the password of the VM from the vm_instance object returned by _start_image().
Returns: the password needed to connect to the VM"""
pass
def _vm_get_state(self, vm_instance):
"""Retrieve VM state from the vm_instance object returned by list_instances().
Returns: VM cloud state."""
return ""
def _create_allow_all_security_group(self):
"""If the conncector support security groups, this method should create a security group named
NodeDecorator.SECURITY_GROUP_ALLOW_ALL_NAME with everything allowed if it doesn't exist yet."""
pass
''' The methods below are used to generate the reply of *-describe-instances
'''
def _vm_get_ip_from_list_instances(self, vm_instance):
"""Retrieve an IP from the vm_instance object returned by list_instances().
Returns: one IP - Public or Private."""
pass
def _vm_get_id_from_list_instances(self, vm_instance):
"""Retrieve an ID from the vm_instance object returned by list_instances().
Returns: one ID."""
pass
def _vm_get_cpu(self, vm_instance):
"""Retrieve the number of CPU of the VM from the vm_instance object returned by list_instances().
Returns: the number of CPU of VM"""
pass
def _vm_get_ram(self, vm_instance):
"""Retrieve the amount of RAM memory of the VM from the vm_instance object returned by list_instances().
Returns: the amount of RAM memory of the VM in MB"""
pass
def _vm_get_root_disk(self, vm_instance):
"""Retrieve the size of the root disk of the VM from the vm_instance object returned by list_instances().
Returns: the size of the root disk of the VM in GB"""
pass
def _vm_get_instance_type(self, vm_instance):
"""Retrieve the instance type of the VM from the vm_instance object returned by list_instances().
Returns: the name of the instance type of the VM"""
pass
def _get_vm_failed_states(self):
"""Override the method or provide cloud specific list of VM_FAILED_STATES.
"""
return self.VM_FAILED_STATES
''' The methods below are used to generate the reply of *-service-offers
'''
def _list_vm_sizes(self):
"""
Return a list of available VM sizes.
The data structure of 'size' element is left to the connector implementation.
Methods '_size_get_...(vm_size) are used to retrieve specific data from a 'size' element.
"""
pass
def _size_get_cpu(self, vm_size):
"""
Extract and return the amount of vCPU from the specified vm_size.
:param vm_size: A 'size' object as in the list returned by _list_vm_sizes().
:rtype int
"""
pass
def _size_get_ram(self, vm_size):
"""
Extract and return the size of the RAM memory in MB from the specified vm_size.
:param vm_size: A 'size' object as in the list returned by _list_vm_sizes().
:rtype float
"""
pass
def _size_get_disk(self, vm_size):
"""
Extract and return the size of the root disk in GB from the specified vm_size.
:param vm_size: A 'size' object as in the list returned by _list_vm_sizes().
:rtype float
"""
pass
def _size_get_instance_type(self, vm_size):
"""
Extract and return the instance type from the specified vm_size.
:param vm_size: A 'size' object as in the list returned by _list_vm_sizes().
:rtype int
"""
pass
"""IaaS actions for VM vertical scalability.
If the VM needs to be restarted, the implementation should
* wait until VM enters Running state, and
* assert that the action was correctly fulfilled by IaaS.
If the action on the VM can be applied w/o restart, the implementation should
* wait until action is applied by IaaS, and
* assert that the action was correctly fulfilled by IaaS.
Raise `slipstream.exceptions.Exceptions.ExecutionException`
on any handled error.
"""
def _resize(self, node_instance):
"""
:param node_instance: node instance object
:type node_instance: <NodeInstance>
"""
# Example code
# Cloud VM id.
# vm_id = node_instance.get_instance_id()
# RAM in GB.
# ram = node_instance.get_ram()
# Number of CPUs.
# cpu = node_instance.get_cpu()
# In case cloud uses T-short sizes.
# instance_type = node_instance.get_instance_type()
# IaaS calls go here.
raise NotImplementedError()
def _attach_disk(self, node_instance):
"""Attach extra disk to the VM.
:param node_instance: node instance object
:type node_instance: <NodeInstance>
:return: name of the device that was attached
:rtype: string
"""
# Example code
# device_name = ''
# Cloud VM id.
# vm_id = node_instance.get_instance_id()
# Size of the disk to attach (in GB).
# disk_size_GB = node_instance.get_disk_attach_size()
# IaaS calls go here.
# return device_name
raise NotImplementedError()
def _detach_disk(self, node_instance):
"""Detach disk from the VM.
:param node_instance: node instance object
:type node_instance: <NodeInstance>
"""
# Example code
# Cloud VM id.
# vm_id = node_instance.get_instance_id()
# Name of the block device to detach (/dev/XYZ).
# device = node_instance.get_disk_detach_device()
# IaaS calls go here.
raise NotImplementedError()
# ----------------------------------------------------------------
TIMEOUT_CONNECT = 10 * 60
RUN_BOOTSTRAP_SCRIPT = False
WAIT_IP = False
# CAPABILITIES
CAPABILITY_VAPP = 'vapp'
CAPABILITY_BUILD_IN_SINGLE_VAPP = 'buildInSingleVapp'
CAPABILITY_CONTEXTUALIZATION = 'contextualization'
CAPABILITY_WINDOWS_CONTEXTUALIZATION = 'windowsContextualization'
CAPABILITY_GENERATE_PASSWORD = '******'
CAPABILITY_DIRECT_IP_ASSIGNMENT = 'directIpAssignment'
CAPABILITY_ORCHESTRATOR_CAN_KILL_ITSELF_OR_ITS_VAPP = 'orchestratorCanKillItselfOrItsVapp'
VM_FAILED_STATES = ['failed', 'error']
def __init__(self, configHolder):
"""Constructor.
All connectors need to call this constructor from their own constructor.
Moreover all connectors need to define their capabilities with the method _set_capabilities().
"""
self.verboseLevel = 0
configHolder.assign(self)
self.configHolder = configHolder
self.run_category = getattr(configHolder, KEY_RUN_CATEGORY, None)
self.max_iaas_workers = None
self.sshPrivKeyFile = '%s/.ssh/id_rsa' % os.path.expanduser("~")
self.sshPubKeyFile = self.sshPrivKeyFile + '.pub'
self.__listener = SimplePrintListener(verbose=(self.verboseLevel > 1))
self.__vms = {}
self.__cloud = os.environ.get(util.ENV_CONNECTOR_INSTANCE)
self.__init_threading_related()
self.tempPrivateKeyFileName = ''
self.tempPublicKey = ''
self.__capabilities = []
self.__already_published = defaultdict(set)
# Ensure that httplib use the "old" default for certificates validation
# since libcloud doesn't work with the new default.
try:
ssl._https_verify_certificates(False)
except:
pass
self._user_info = None
self.cimi_deployment_prototype = False
@property
def user_info(self):
if self._user_info is None:
raise ValueError('Object not initialized. Please call _initialization first.')
return self._user_info
@user_info.setter
def user_info(self, value):
self._user_info = value
def __init_threading_related(self):
self.__tasks_runnner = None
# This parameter is thread local
self._thread_local = local()
def _set_capabilities(self, vapp=False, build_in_single_vapp=False,
contextualization=False,
windows_contextualization=False,
generate_password=False,
direct_ip_assignment=False,
orchestrator_can_kill_itself_or_its_vapp=False):
if vapp:
self.__capabilities.append(self.CAPABILITY_VAPP)
if build_in_single_vapp:
self.__capabilities.append(self.CAPABILITY_BUILD_IN_SINGLE_VAPP)
if contextualization:
self.__capabilities.append(self.CAPABILITY_CONTEXTUALIZATION)
if windows_contextualization:
self.__capabilities.append(self.CAPABILITY_WINDOWS_CONTEXTUALIZATION)
if generate_password:
self.__capabilities.append(self.CAPABILITY_GENERATE_PASSWORD)
if direct_ip_assignment:
self.__capabilities.append(self.CAPABILITY_DIRECT_IP_ASSIGNMENT)
if orchestrator_can_kill_itself_or_its_vapp:
self.__capabilities.append(
self.CAPABILITY_ORCHESTRATOR_CAN_KILL_ITSELF_OR_ITS_VAPP)
def _reset_capabilities(self):
self.__capabilities = []
def has_capability(self, capability):
return capability in self.__capabilities
def __set_contextualization_capabilities(self, user_info):
native_contextualization = user_info.get_cloud(NodeDecorator.NATIVE_CONTEXTUALIZATION_KEY, None)
if native_contextualization:
try:
self.__capabilities.remove(self.CAPABILITY_CONTEXTUALIZATION)
self.__capabilities.remove(self.CAPABILITY_WINDOWS_CONTEXTUALIZATION)
except ValueError:
pass
if native_contextualization == 'always':
self.__capabilities.append(self.CAPABILITY_CONTEXTUALIZATION)
self.__capabilities.append(self.CAPABILITY_WINDOWS_CONTEXTUALIZATION)
elif native_contextualization == 'linux-only':
self.__capabilities.append(self.CAPABILITY_CONTEXTUALIZATION)
elif native_contextualization == 'windows-only':
self.__capabilities.append(self.CAPABILITY_WINDOWS_CONTEXTUALIZATION)
def is_build_image(self):
return self.run_category == NodeDecorator.IMAGE
def is_deployment(self):
return self.run_category == NodeDecorator.DEPLOYMENT
def stop_deployment(self):
self._stop_deployment()
def stop_vms_by_ids(self, ids):
self._stop_vms_by_ids(ids)
def stop_vapps_by_ids(self, ids):
self._stop_vapps_by_ids(ids)
def stop_node_instances(self, node_instances):
ids = []
for node_instance in node_instances:
ids.append(node_instance.get_instance_id())
self.__del_vm(node_instance.get_name())
if len(ids) > 0:
self.stop_vms_by_ids(ids)
def __create_allow_all_security_group_if_needed(self, nodes_instances):
allow_all = NodeDecorator.SECURITY_GROUP_ALLOW_ALL_NAME
for ni in nodes_instances.itervalues():
if allow_all in ni.get_security_groups():
self._create_allow_all_security_group()
break
def start_nodes_and_clients(self, user_info, nodes_instances, init_extra_kwargs={}):
self._initialization(user_info, **init_extra_kwargs)
self.__set_contextualization_capabilities(user_info)
self.__create_allow_all_security_group_if_needed(nodes_instances)
self.cimi_deployment_prototype = bool(nodes_instances.values()[0].get_deployment_context())
try:
self.__start_nodes_instantiation_tasks_wait_finished(user_info,
nodes_instances)
finally:
self._finalization(user_info)
def __start_nodes_instantiation_tasks_wait_finished(self, user_info,
nodes_instances):
self.__start_nodes_instances_and_clients(user_info, nodes_instances)
self.__wait_nodes_startup_tasks_finished()
def __start_nodes_instances_and_clients(self, user_info, nodes_instances):
self.__tasks_runnner = TasksRunner(self.__start_node_instance_and_client,
max_workers=self.max_iaas_workers,
verbose=self.verboseLevel)
for node_instance in nodes_instances.values():
self.__tasks_runnner.put_task(user_info, node_instance)
self.__tasks_runnner.run_tasks()
def __wait_nodes_startup_tasks_finished(self):
if self.__tasks_runnner != None:
self.__tasks_runnner.wait_tasks_processed()
def __start_node_instance_and_client(self, user_info, node_instance):
node_instance_name = node_instance.get_name()
self._print_detail("Starting instance: %s" % node_instance_name)
node_context = node_instance.get_deployment_context()
self.cimi_deployment_prototype = bool(node_context)
if self.cimi_deployment_prototype:
vm_name = node_instance_name + '--' + node_context.get('SLIPSTREAM_DIID', '').replace('deployment/', '')
else:
vm_name = self._generate_vm_name(node_instance_name)
vm = self._start_image(user_info,
node_instance,
vm_name)
self.__add_vm(vm, node_instance)
if not self.has_capability(self.CAPABILITY_DIRECT_IP_ASSIGNMENT):
vm = self._wait_and_get_instance_ip_address(vm)
self.__add_vm(vm, node_instance)
if not self.is_build_image():
if not node_instance.is_windows() and not self.has_capability(self.CAPABILITY_CONTEXTUALIZATION):
self.__secure_ssh_access_and_run_bootstrap_script(user_info, node_instance, self._vm_get_ip(vm))
elif node_instance.is_windows() and not self.has_capability(self.CAPABILITY_WINDOWS_CONTEXTUALIZATION):
self.__launch_windows_bootstrap_script(node_instance, self._vm_get_ip(vm))
def get_vms(self):
return self.__vms
def _get_vm(self, name):
try:
return self.__vms[name]
except KeyError:
raise Exceptions.NotFoundError("VM '%s' not found." % name)
def __add_vm(self, vm, node_instance):
name = node_instance.get_name()
with lock:
self.__vms[name] = vm
self._publish_vm_info(vm, node_instance)
def __del_vm(self, name):
try:
del self.__vms[name]
except KeyError:
util.printDetail("Failed locally removing VM '%s'. Not found." % name)
def _publish_vm_info(self, vm, node_instance):
instance_name = node_instance.get_name()
vm_id = self._vm_get_id(vm)
vm_ip = self._vm_get_ip(vm)
vm_ports_mapping = self._vm_get_ports_mapping(vm)
with lock:
already_published = self.__already_published[instance_name]
if self.cimi_deployment_prototype:
if vm_id and 'id' not in already_published:
node_instance.set_instance_id(vm_id)
already_published.add('id')
if vm_ip and 'ip' not in already_published:
node_instance.set_cloud_node_ip(vm_ip)
already_published.add('ip')
if node_instance and vm_ip and 'ssh' not in already_published:
if node_instance:
vm_ip = self._vm_get_ip(vm) or ''
ssh_username, ssh_password = self.__get_vm_username_password(node_instance)
node_instance.set_cloud_node_ssh_url('ssh://%s@%s' % (ssh_username.strip(), vm_ip.strip()))
node_instance.set_cloud_node_ssh_password(ssh_password)
if ssh_username and ssh_password:
already_published.add('ssh')
if vm_ports_mapping and 'vm_ports_mapping' not in already_published:
node_instance.set_cloud_node_ports_mapping(vm_ports_mapping)
ssh_found = re.search('tcp:(\d+):22', str(vm_ports_mapping))
if ssh_found:
ssh_username, ssh_password = self.__get_vm_username_password(node_instance)
node_instance.set_cloud_node_ssh_url('ssh://{}@{}:{}'.format(ssh_username.strip(),
vm_ip.strip(),
ssh_found.group(1)))
already_published.add('vm_ports_mapping')
else:
if vm_id and 'id' not in already_published:
self._publish_vm_id(instance_name, vm_id)
already_published.add('id')
if vm_ip and 'ip' not in already_published:
self._publish_vm_ip(instance_name, vm_ip)
already_published.add('ip')
if node_instance and vm_ip and 'ssh' not in already_published:
self._publish_url_ssh(vm, node_instance)
already_published.add('ssh')
def _publish_vm_id(self, instance_name, vm_id):
# Needed for thread safety
NodeInfoPublisher(self.configHolder).publish_instanceid(instance_name,
str(vm_id))
def _publish_vm_ip(self, instance_name, vm_ip):
# Needed for thread safety
NodeInfoPublisher(self.configHolder).publish_hostname(instance_name,
vm_ip)
def _publish_url_ssh(self, vm, node_instance):
if not node_instance:
return
instance_name = node_instance.get_name()
vm_ip = self._vm_get_ip(vm) or ''
ssh_username, _ = self.__get_vm_username_password(node_instance)
# Needed for thread safety
NodeInfoPublisher(self.configHolder).publish_url_ssh(instance_name,
vm_ip, ssh_username)
def _set_temp_private_key_and_public_key(self, privateKeyFileName, publicKey):
self.tempPrivateKeyFileName = privateKeyFileName
self.tempPublicKey = publicKey
def _get_temp_private_key_file_name_and_public_key(self):
return self.tempPrivateKeyFileName, self.tempPublicKey
def build_image(self, user_info, node_instance):
if not self.has_capability(self.CAPABILITY_CONTEXTUALIZATION):
username, password = self.__get_vm_username_password(node_instance)
privateKey, publicKey = generate_keypair()
privateKey = util.file_put_content_in_temp_file(privateKey)
self._set_temp_private_key_and_public_key(privateKey, publicKey)
ip = self._vm_get_ip(self._get_vm(NodeDecorator.MACHINE_NAME))
self._secure_ssh_access(ip, username, password, publicKey)
new_id = self._build_image(user_info, node_instance)
return new_id
def get_creator_vm_id(self):
vm = self._get_vm(NodeDecorator.MACHINE_NAME)
return self._vm_get_id(vm)
def _remote_run_build_target(self, node_instance, host, target, username, password, private_key_file):
for subtarget in target:
target_name = subtarget.get('name')
full_target_name = '%s.%s' % (subtarget.get('module'), target_name)
if not MachineExecutor.need_to_execute_build_step(node_instance, target, subtarget):
message = 'Component already built. Nothing to do on target "%s" for node "%s"\n' \
% (full_target_name, node_instance.get_name())
util.printAndFlush(message)
continue
script = subtarget.get('script')
if script:
message = 'Executing target "%s" on remote host "%s"' % (full_target_name, node_instance.get_name())
self._print_step(message)
remoteRunScript(username, host, script, sshKey=private_key_file, password=password)
else:
message = 'Nothing to do for target "%s" on node "%s""\n' % (full_target_name, node_instance.get_name())
util.printAndFlush(message)
def _build_image_increment(self, user_info, node_instance, host):
prerecipe = node_instance.get_prerecipe()
recipe = node_instance.get_recipe()
packages = node_instance.get_packages()
try:
machine_name = node_instance.get_name()
username, password, ssh_private_key_file = self._get_ssh_credentials(node_instance, user_info)
if not self.has_capability(self.CAPABILITY_CONTEXTUALIZATION) and not ssh_private_key_file:
password = ''
ssh_private_key_file, publicKey = self._get_temp_private_key_file_name_and_public_key()
self._wait_can_connect_with_ssh_or_abort(host, username=username,
password=password,
sshKey=ssh_private_key_file)
if prerecipe:
self._print_step('Running Pre-recipe', machine_name)
self._remote_run_build_target(node_instance, host, prerecipe, username, password, ssh_private_key_file)
if packages:
self._print_step('Installing Packages', machine_name)
platform = node_instance.get_platform()
remoteRunCommand(command=util.get_packages_install_command(platform, packages),
host=host, user=username, sshKey=ssh_private_key_file, password=password)
if recipe:
self._print_step('Running Recipe', machine_name)
self._remote_run_build_target(node_instance, host, recipe, username, password, ssh_private_key_file)
if not self.has_capability(self.CAPABILITY_CONTEXTUALIZATION):
self._revert_ssh_security(host, username, ssh_private_key_file, publicKey)
finally:
try:
os.unlink(ssh_private_key_file)
except: # pylint: disable=bare-except
pass
def get_cloud_service_name(self):
return self.__cloud
def _get_ssh_credentials(self, node_instance, user_info):
username, password = self.__get_vm_username_password(node_instance)
if password:
ssh_private_key_file = None
else:
ssh_private_key_file = self.__get_ssh_private_key_file(user_info)
return username, password, ssh_private_key_file
def __get_ssh_private_key_file(self, user_info):
fd, ssh_private_key_file = tempfile.mkstemp()
os.write(fd, user_info.get_private_key())
os.close(fd)
os.chmod(ssh_private_key_file, 0400)
return ssh_private_key_file
def __get_vm_username_password(self, node_instance, default_user='******'):
user = node_instance.get_username(default_user)
password = self.__get_vm_password(node_instance)
return user, password
def __get_vm_password(self, node_instance):
password = node_instance.get_password()
if not password:
instance_name = node_instance.get_name()
try:
vm = self._get_vm(instance_name)
password = self._vm_get_password(vm)
except: # pylint: disable=bare-except
pass
return password
@staticmethod
def _get_run_uuid(default=None):
return os.environ.get('SLIPSTREAM_DIID', default)
@classmethod
def _generate_vm_name(cls, instance_name):
vm_name = instance_name
run_id = cls._get_run_uuid()
if run_id:
vm_name = vm_name + NodeDecorator.NODE_PROPERTY_SEPARATOR + run_id
return vm_name
@staticmethod
def is_start_orchestrator():
return os.environ.get('IS_ORCHESTRATOR', 'False') == 'True'
def __secure_ssh_access_and_run_bootstrap_script(self, user_info, node_instance, ip):
username, password = self.__get_vm_username_password(node_instance)
privateKey, publicKey = generate_keypair()
privateKey = util.file_put_content_in_temp_file(privateKey)
self._secure_ssh_access(ip, username, password, publicKey, user_info)
self.__launch_bootstrap_script(node_instance, ip, username, privateKey)
def _secure_ssh_access(self, ip, username, password, publicKey, user_info=None):
self._wait_can_connect_with_ssh_or_abort(ip, username, password)
script = self.__get_obfuscation_script(publicKey, username, user_info)
self._print_detail("Securing SSH access to %s with:\n%s\n" % (ip,
script))
_, output = self._run_script(ip, username, script, password=password)
self._print_detail("Secured SSH access to %s. Output:\n%s\n" % (ip,
output))
def _revert_ssh_security(self, ip, username, privateKey, orchestratorPublicKey):
self._wait_can_connect_with_ssh_or_abort(ip, username, sshKey=privateKey)
script = "#!/bin/bash -xe\n"
script += "set +e\n"
script += "grep -vF '" + orchestratorPublicKey + "' ~/.ssh/authorized_keys > ~/.ssh/authorized_keys.tmp\n"
script += "set -e\n"
script += "sleep 2\nmv ~/.ssh/authorized_keys.tmp ~/.ssh/authorized_keys\n"
script += "chown -R " + username + ":$(id -g " + username + ")" + " ~/.ssh\n"
script += "restorecon -Rv ~/.ssh || true\n"
script += "sed -i -r 's/^#?[\\t ]*(PasswordAuthentication[\\t ]+)((yes)|(no))/\\1yes/' /etc/ssh/sshd_config\n"
script += "sync\nsleep 2\n"
script += "[ -x /etc/init.d/sshd ] && { service sshd reload; } || { service ssh reload || /etc/init.d/ssh reload; }\n"
self._print_detail("Reverting security of SSH access to %s with:\n%s\n" % (ip, script))
_, output = self._run_script(ip, username, script, sshKey=privateKey)
self._print_detail("Reverted security of SSH access to %s. Output:\n%s\n" % (ip, output))
def __launch_bootstrap_script(self, node_instance, ip, username, privateKey):
self._wait_can_connect_with_ssh_or_abort(ip, username, sshKey=privateKey)
script = self._get_bootstrap_script(node_instance)
self._print_detail("Launching bootstrap script on %s:\n%s\n" % (ip,
script))
_, output = self._run_script_on_backgroud(ip, username, script,
sshKey=privateKey)
self._print_detail("Launched bootstrap script on %s:\n%s\n" % (ip,
output))
def __launch_windows_bootstrap_script(self, node_instance, ip):
username, password = self.__get_vm_username_password(node_instance, 'administrator')
script = self._get_bootstrap_script(node_instance, username=username)
winrm = self._getWinrm(ip, username, password)
self._waitCanConnectWithWinrmOrAbort(winrm)
self._print_detail("Launching bootstrap script on %s:\n%s\n" % (ip,
script))
util.printAndFlush(script)
winrm.timeout = winrm.set_timeout(600)
output = self._runScriptWithWinrm(winrm, script)
self._print_detail("Launched bootstrap script on %s:\n%s\n" % (ip,
output))
def _getWinrm(self, ip, username, password):
return WinRMWebService(endpoint='http://%s:5985/wsman' % ip,
transport='plaintext', username=username,
password=password)
def _runScriptWithWinrm(self, winrm, script):
shellId = winrm.open_shell()
commands = ''
for command in script.splitlines():
if command and not command.startswith('rem '):
commands += command + '& '
commands += 'echo "Bootstrap Finished"'
stdout, stderr, returnCode = self._runCommandWithWinrm(winrm, commands,
shellId,
runAndContinue=True)
# winrm.close_shell(shellId)
return stdout, stderr, returnCode
def _waitCanConnectWithWinrmOrAbort(self, winrm):
try:
self._waitCanConnectWithWinrmOrTimeout(winrm, self.TIMEOUT_CONNECT)
except Exception as ex:
raise Exceptions.ExecutionException("Failed to connect to "
"%s: %s" % (winrm.endpoint,
str(ex)))
def _waitCanConnectWithWinrmOrTimeout(self, winrm, timeout):
time_stop = time.time() + timeout
while (time_stop - time.time()) >= 0:
try:
_, _, returnCode = self._runCommandWithWinrm(winrm, 'exit 0')
if returnCode == 0:
return
except WinRMTransportError as ex:
util.printDetail(str(ex))
time.sleep(5)
def _runCommandWithWinrm(self, winrm, command, shellId=None,
runAndContinue=False):
if shellId:
_shellId = shellId
else:
_shellId = winrm.open_shell()
util.printAndFlush('\nwinrm.run_command\n')
commandId = winrm.run_command(_shellId, command, [])
stdout, stderr, returnCode = ('N/A', 'N/A', 0)
if not runAndContinue:
util.printAndFlush('\nwinrm.get_command_output\n')
try:
stdout, stderr, returnCode = winrm.get_command_output(_shellId, commandId)
except Exception as e: # pylint: disable=broad-except
print 'WINRM Exception: %s' % str(e)
util.printAndFlush('\nwinrm.cleanup_command\n')
winrm.cleanup_command(_shellId, commandId)
if not shellId:
winrm.close_shell(_shellId)
return stdout, stderr, returnCode
def __get_obfuscation_script(self, orchestrator_public_key, username, user_info=None):
command = "#!/bin/bash -xe\n"
# command += "sed -r -i 's/# *(account +required +pam_access\\.so).*/\\1/' /etc/pam.d/login\n"
# command += "echo '-:ALL:LOCAL' >> /etc/security/access.conf\n"
command += "sed -i -r '/^[\\t ]*RSAAuthentication/d;/^[\\t ]*PubkeyAuthentication/d;/^[\\t ]*PasswordAuthentication/d' /etc/ssh/sshd_config\n"
command += "echo 'RSAAuthentication yes\nPubkeyAuthentication yes\nPasswordAuthentication no\n' >> /etc/ssh/sshd_config\n"
# command += "sed -i -r 's/^#?[\\t ]*(RSAAuthentication[\\t ]+)((yes)|(no))/\\1yes/' /etc/ssh/sshd_config\n"
# command += "sed -i -r 's/^#?[\\t ]*(PubkeyAuthentication[\\t ]+)((yes)|(no))/\\1yes/' /etc/ssh/sshd_config\n"
# command += "sed -i -r 's/^#?[\t ]*(PasswordAuthentication[\\t ]+)((yes)|(no))/\\1no/' /etc/ssh/sshd_config\n"
command += "umask 077\n"
command += "mkdir -p ~/.ssh\n"
if user_info:
command += "echo '" + user_info.get_public_keys() + "' >> ~/.ssh/authorized_keys\n"
command += "echo '" + orchestrator_public_key + "' >> ~/.ssh/authorized_keys\n"
command += "chown -R " + username + ":$(id -g " + username + ")" + " ~/.ssh\n"
command += "restorecon -Rv ~/.ssh || true\n"
command += "[ -x /etc/init.d/sshd ] && { service sshd reload; } || { service ssh reload; }\n"
return command
def _get_bootstrap_script_if_not_build_image(self, node_instance, pre_export=None,
pre_bootstrap=None, post_bootstrap=None,
username=None):
return self._get_bootstrap_script(node_instance, pre_export, pre_bootstrap, post_bootstrap, username) \
if not self.is_build_image() else ''
def _get_bootstrap_script(self, node_instance, pre_export=None,
pre_bootstrap=None, post_bootstrap=None,
username=None):
"""This method can be redefined by connectors if they need a specific bootstrap script
with the SSH contextualization."""
script = ''
addEnvironmentVariableCommand = ''
node_instance_name = node_instance.get_name()
if node_instance.is_windows():
addEnvironmentVariableCommand = 'set'
script += 'rem cmd\n'
else:
addEnvironmentVariableCommand = 'export'
script += '#!/bin/sh -ex\n'
if pre_export:
script += '%s\n' % pre_export
if self.cimi_deployment_prototype:
for var, val in node_instance.get_deployment_context().items():
if re.search(' ', val):
val = '"%s"' % val
script += '%s %s=%s\n' % (addEnvironmentVariableCommand, var, val)
else:
regex = 'SLIPSTREAM_'
if self.is_start_orchestrator():
regex += '|CLOUDCONNECTOR_'
env_matcher = re.compile(regex)
if pre_export:
script += '%s\n' % pre_export
for var, val in os.environ.items():
if env_matcher.match(var) and var != util.ENV_NODE_INSTANCE_NAME:
if re.search(' ', val):
val = '"%s"' % val
script += '%s %s=%s\n' % (addEnvironmentVariableCommand, var,
val)
script += '%s %s=%s\n' % (addEnvironmentVariableCommand,
util.ENV_NODE_INSTANCE_NAME,
node_instance_name)
if pre_bootstrap:
script += '%s\n' % pre_bootstrap
script += '%s\n' % self._build_slipstream_bootstrap_command(node_instance,
username)
if post_bootstrap:
script += '%s\n' % post_bootstrap
return script
def _build_slipstream_bootstrap_command(self, node_instance, username=None):
instance_name = node_instance.get_name()
if self.cimi_deployment_prototype:
bootstrap_url = node_instance.get_deployment_context().get('SLIPSTREAM_BOOTSTRAP_BIN')
else:
bootstrap_url = util.get_required_envvar('SLIPSTREAM_BOOTSTRAP_BIN')
if node_instance.is_windows():
return self.__build_slipstream_bootstrap_command_for_windows(instance_name, bootstrap_url)
else:
return self.__build_slipstream_bootstrap_command_for_linux(instance_name, bootstrap_url)
def __build_slipstream_bootstrap_command_for_windows(self, instance_name, bootstrap_url):
command = 'If Not Exist %(reports)s mkdir %(reports)s\n'
command += 'If Not Exist %(ss_home)s mkdir %(ss_home)s\n'
command += 'If Not Exist "C:\\Python27\\python.exe" ( '
command += ' powershell -Command "$wc = New-Object System.Net.WebClient; $wc.DownloadFile(\'https://www.python.org/ftp/python/2.7.13/python-2.7.13.amd64.msi\', $env:temp+\'\\python.msi\')"\n'
command += ' start /wait msiexec /i %%TMP%%\\python.msi /qn /quiet /norestart /log log.txt TARGETDIR=C:\\Python27\\ ALLUSERS=1 '
command += ')\n'
command += 'setx path "%%path%%;C:\\Python27;C:\\opt\\slipstream\\client\\bin;C:\\opt\\slipstream\\client\\sbin" /M\n'
command += 'setx PYTHONPATH "%%PYTHONPATH%%;C:\\opt\\slipstream\\client\\lib" /M\n'
command += 'powershell -Command "[System.Net.ServicePointManager]::ServerCertificateValidationCallback = {$true}; $wc = New-Object System.Net.WebClient; $wc.Headers.Add(\'User-Agent\',\'PowerShell\'); $wc.DownloadFile(\'%(bootstrapUrl)s\', \'%(bootstrap)s\')" > %(reports)s\\%(nodename)s.slipstream.log 2>&1\n'
command += 'set PATH=%%PATH%%;C:\\Python27;C:\\opt\\slipstream\\client\\bin;C:\\opt\\slipstream\\client\\sbin\n'
command += 'set PYTHONPATH=C:\\opt\\slipstream\\client\\lib\n'
command += 'start "test" "%%SystemRoot%%\System32\cmd.exe" /C "C:\\Python27\\python %(bootstrap)s %(machine_executor)s >> %(reports)s\\%(nodename)s.slipstream.log 2>&1"\n'
return command % self._get_bootstrap_command_replacements_for_windows(instance_name, bootstrap_url)
def __build_slipstream_bootstrap_command_for_linux(self, instance_name, bootstrap_url):
command = 'mkdir -p %(reports)s %(ss_home)s; '
command += '(wget --timeout=60 --retry-connrefused --no-check-certificate -O %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1 '
command += '|| curl --retry 20 -k -f -o %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1) '
command += '&& chmod 0755 %(bootstrap)s; %(bootstrap)s %(machine_executor)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1'
return command % self._get_bootstrap_command_replacements_for_linux(instance_name, bootstrap_url)
def _get_bootstrap_command_replacements_for_linux(self, instance_name, bootstrap_url):
return {
'reports': util.REPORTSDIR,
'bootstrap': os.path.join(util.SLIPSTREAM_HOME, 'slipstream.bootstrap'),
'bootstrapUrl': bootstrap_url,
'ss_home': util.SLIPSTREAM_HOME,
'nodename': instance_name,
'machine_executor': 'node' if self.cimi_deployment_prototype else self._get_machine_executor_type()
}
def _get_bootstrap_command_replacements_for_windows(self, instance_name, bootstrap_url):
return {
'reports': util.WINDOWS_REPORTSDIR,
'bootstrap': '\\'.join([util.WINDOWS_SLIPSTREAM_HOME, 'slipstream.bootstrap']),
'bootstrapUrl': bootstrap_url,
'ss_home': util.WINDOWS_SLIPSTREAM_HOME,
'nodename': instance_name,
'machine_executor': 'node' if self.cimi_deployment_prototype else self._get_machine_executor_type()
}
def _get_machine_executor_type(self):
return self.is_start_orchestrator() and 'orchestrator' or 'node'
def _wait_can_connect_with_ssh_or_abort(self, host, username='', password='', sshKey=None):
self._print_detail('Check if we can connect to %s' % host)
try:
waitUntilSshCanConnectOrTimeout(host,
self.TIMEOUT_CONNECT,
sshKey=sshKey,
user=username, password=password,
verboseLevel=self.verboseLevel)
except Exception as ex:
raise Exceptions.ExecutionException("Failed to connect to "
"%s: %s, %s" % (host, type(ex),
str(ex)))
def _run_script(self, ip, username, script, password='', sshKey=None):
return remoteRunScript(username, ip, script, sshKey=sshKey, password=password)
def _run_script_on_backgroud(self, ip, username, script, password='', sshKey=None):
return remoteRunScriptNohup(username, ip, script, sshKey=sshKey, password=password)
def set_slipstream_client_as_listener(self, client):
self._set_listener(SlipStreamClientListenerAdapter(client))
def _set_listener(self, listener):
self.__listener = listener
def _get_listener(self):
return self.__listener
def _print_detail(self, message):
util.printDetail(message, self.verboseLevel)
def _print_step(self, message, write_for=None):
util.printStep(message)
if write_for:
self._get_listener().write_for(write_for, message)
def get_vms_details(self):
vms_details = []
for name, vm in self.get_vms().items():
vms_details.append({name: {'id': self._vm_get_id(vm),
'ip': self._vm_get_ip(vm)}})
return vms_details
def _has_vm_failed(self, vm_instance):
"""Check if VM failed on the cloud level. vm_instance as returned by _start_image().
Returns: True or False."""
vm_state = self._vm_get_state(vm_instance).lower()
return vm_state in [fstate.lower() for fstate in self._get_vm_failed_states()]
def resize(self, node_instances, done_reporter=None):
self._scale_action_runner(self._resize_and_report, node_instances, done_reporter)
# TODO: use decorator for reporter.
def _resize_and_report(self, node_instance, reporter):
self._resize(node_instance)
if hasattr(reporter, '__call__'):
reporter(node_instance)
def attach_disk(self, node_instances, done_reporter=None):
self._scale_action_runner(self._attach_disk_and_report, node_instances, done_reporter)
def _attach_disk_and_report(self, node_instance, reporter):
attached_disk = self._attach_disk(node_instance)
if not attached_disk:
raise Exceptions.ExecutionException(
'Attached disk name not provided by connector after disk attach operation.')
if hasattr(reporter, '__call__'):
reporter(node_instance, attached_disk)
def detach_disk(self, node_instances, done_reporter=None):
self._scale_action_runner(self._detach_disk_and_report, node_instances, done_reporter)
def _detach_disk_and_report(self, node_instance, reporter):
self._detach_disk(node_instance)
if hasattr(reporter, '__call__'):
reporter(node_instance)
def _scale_action_runner(self, scale_action, node_instances, done_reporter):
"""
:param scale_action: task executor
:type scale_action: callable
:param node_instances: list of node instances
:type node_instances: list [NodeInstance, ]
:param done_reporter: function that reports back to SlipStream
:type done_reporter: callable with signature `done_reporter(<NoneInstance>)`
"""
scaler = VmScaler(scale_action, self.max_iaas_workers, self.verboseLevel)
scaler.set_tasks_and_run(node_instances, done_reporter)
scaler.wait_tasks_finished()
def _init_threading_related(self):
self.tasksRunnner = TasksRunner()
# This parameter is thread local
self._thread_local = local()
self._thread_local.isWindows = False
class BaseCloudConnector(object):
TIMEOUT_CONNECT = 10 * 60
DISK_VOLATILE_PARAMETER_NAME = (SlipStreamHttpClient.DomExtractor.EXTRADISK_PREFIX +
'.volatile')
DISK_PERSISTENT_PARAMETER_NAME = (SlipStreamHttpClient.DomExtractor.EXTRADISK_PREFIX +
'.persistent')
RUN_BOOTSTRAP_SCRIPT = False
WAIT_IP = False
ADD_SSHPUBKEY_ON_NODE = False
# CAPABILITIES
CAPABILITY_VAPP = 'vapp'
CAPABILITY_BUILD_IN_SINGLE_VAPP = 'buildInSingleVapp'
CAPABILITY_CONTEXTUALIZATION = 'contextualization'
CAPABILITY_WINDOWS_CONTEXTUALIZATION = 'windowsContextualization'
CAPABILITY_GENERATE_PASSWORD = '******'
CAPABILITY_DIRECT_IP_ASSIGNMENT = 'directIpAssignment'
CAPABILITY_ADD_SHH_PUBLIC_KEY_ON_NODE = 'addSshPublicKeyOnNode'
CAPABILITY_ORCHESTRATOR_CAN_KILL_ITSELF_OR_ITS_VAPP = 'orchestratorCanKillItselfOrItsVapp'
def __init__(self, configHolder):
self.verboseLevel = 0
configHolder.assign(self)
self.configHolder = configHolder
self.sshPrivKeyFile = '%s/.ssh/id_rsa' % os.path.expanduser("~")
self.sshPubKeyFile = self.sshPrivKeyFile + '.pub'
self.listener = SimplePrintListener(verbose=(self.verboseLevel > 1))
self._vms = {}
self.cloud = os.environ['SLIPSTREAM_CONNECTOR_INSTANCE']
# For image creation.
self._newImageId = '' # created image ID on a Cloud
self._creatorVmId = '' # image ID of creator instance
self._terminateRunServerSide = False
self._init_threading_related()
self.tempPrivateKeyFileName = ''
self.tempPublicKey = ''
self._capabilities = []
def _init_threading_related(self):
self.tasksRunnner = TasksRunner()
# This parameter is thread local
self._thread_local = local()
self._thread_local.isWindows = False
def setCapabilities(self, vapp=False, build_in_single_vapp=False,
contextualization=False,
windows_contextualization=False,
generate_password=False,
direct_ip_assignment=False,
add_ssh_public_key_on_node=False,
orchestrator_can_kill_itself_or_its_vapp=False):
if vapp:
self._capabilities.append(self.CAPABILITY_VAPP)
if build_in_single_vapp:
self._capabilities.append(self.CAPABILITY_BUILD_IN_SINGLE_VAPP)
if contextualization:
self._capabilities.append(self.CAPABILITY_CONTEXTUALIZATION)
if windows_contextualization:
self._capabilities.append(self.CAPABILITY_WINDOWS_CONTEXTUALIZATION)
if generate_password:
self._capabilities.append(self.CAPABILITY_GENERATE_PASSWORD)
if direct_ip_assignment:
self._capabilities.append(self.CAPABILITY_DIRECT_IP_ASSIGNMENT)
if add_ssh_public_key_on_node:
self._capabilities.append(self.CAPABILITY_ADD_SHH_PUBLIC_KEY_ON_NODE)
if orchestrator_can_kill_itself_or_its_vapp:
self._capabilities.append(self.CAPABILITY_ORCHESTRATOR_CAN_KILL_ITSELF_OR_ITS_VAPP)
def hasCapability(self, capability):
return capability in self._capabilities
@staticmethod
def extractAllTargets(imageInfo):
"""Return: three tuple of strings (prerecipe, recipe, packages).
packages - space separated list of packages to install.
"""
imageTargets = BaseCloudConnector.getTargets(imageInfo)
prerecipe = imageTargets.get('prerecipe', '').strip()
recipe = imageTargets.get('recipe', '').strip()
packages = ' '.join(imageTargets.get('packages', '')).strip()
return prerecipe, recipe, packages
@staticmethod
def getResourceUri(imageInfo):
return imageInfo['attributes']['resourceUri']
@staticmethod
def getAttributes(imageInfo):
return imageInfo['attributes']
@staticmethod
def getImageUser(imageInfo):
return imageInfo['attributes']['loginUser']
@staticmethod
def getCloudParameters(image):
return image['cloud_parameters']['Cloud']
@staticmethod
def getCloudNetworkType(image_info):
return BaseCloudConnector.getCloudParameters(image_info)['network']
@staticmethod
def getTargets(imageInfo):
return imageInfo['targets']
@staticmethod
def _getSshPrivateKey(userInfo):
return userInfo.get_cloud('private.key')
@staticmethod
def getImageId(image_info):
imageId = image_info['attributes']['imageId']
if not imageId:
raise Exceptions.ExecutionException("Couldn't get image ID.")
return imageId
@staticmethod
def setPlatform(imageInfo, platform):
imageInfo['attributes']['platform'] = platform
@staticmethod
def getPlatform(image_info):
return image_info['attributes']['platform']
@staticmethod
def setArchitecture(imageInfo, arch):
imageInfo['attributes']['arch'] = arch
@staticmethod
def getArchitecture(image_info):
return image_info['attributes']['arch']
@staticmethod
def getExtraDisks(image_info):
return image_info['extra_disks']
def _userInfoGetKeypairName(self, user_info):
return user_info.get_cloud('keypair.name')
def _userInfoSetPrivateKey(self, user_info, privkey):
user_info[user_info.cloud + 'private.key'] = privkey
def _userInfoSetKeypairName(self, user_info, kp_name):
user_info[user_info.cloud + 'keypair.name'] = kp_name
def isWindows(self):
return self._thread_local.isWindows
def _setIsWindows(self, image_info):
self._thread_local.isWindows = self.getPlatform(image_info).lower() == 'windows'
@deprecated
def setRunBootstrapScript(self, run=True):
self.RUN_BOOTSTRAP_SCRIPT = run
@deprecated
def needRunBootstrapScript(self):
return self.RUN_BOOTSTRAP_SCRIPT is True
@deprecated
def setWaitIp(self):
self.WAIT_IP = True
@deprecated
def needWaitIp(self):
return self.WAIT_IP is True
@deprecated
def setNeedToAddSshPubkeyOnNode(self):
self.ADD_SSHPUBKEY_ON_NODE = True
@deprecated
def needToAddSshPubkeyOnNode(self):
return self.ADD_SSHPUBKEY_ON_NODE
def startImage(self, user_info, image_info):
name = NodeDecorator.MACHINE_NAME
self.initialization(user_info)
try:
vm = self._startImage(user_info, image_info, name)
self.addVm(name, vm)
self._creatorVmId = self.vmGetId(vm)
if not self.hasCapability(self.CAPABILITY_DIRECT_IP_ASSIGNMENT):
vm = self._waitAndGetInstanceIpAddress(vm)
self.addVm(name, vm)
finally:
self.finalization(user_info)
return self.getVmsDetails()
def startNodesAndClients(self, user_info, nodes_info):
self.initialization(user_info)
try:
self._startNodeInstantiationTasksWaitFinished(user_info, nodes_info)
finally:
self.finalization(user_info)
return self.getVmsDetails()
def _startNodeInstantiationTasksWaitFinished(self, user_info, nodes_info):
for node_info in nodes_info:
self._startNodeInstancesAndClients(user_info, node_info)
self._waitNodesStartupTasksFinshed()
def _startNodeInstancesAndClients(self, user_info, node_info):
for node_number in range(1, int(node_info['multiplicity']) + 1):
self.tasksRunnner.run_task(self._startNodeInstanceAndClient,
(user_info, node_info, node_number))
def _waitNodesStartupTasksFinshed(self):
self.tasksRunnner.wait_tasks_finished()
def _startNodeInstanceAndClient(self, user_info, node_info, node_number):
image_info = self._extractImageInfoFromNodeInfo(node_info)
nodename = self._generateNodeName(node_info['nodename'], node_number)
self._setIsWindows(image_info)
self._printDetail("Starting node: %s" % nodename)
cloudSpecificData = self._getCloudSpecificData(node_info,
node_number, nodename)
vm = self._startImage(user_info,
image_info,
self._generateInstanceName(nodename),
cloudSpecificData)
self.addVm(nodename, vm)
if not self.hasCapability(self.CAPABILITY_DIRECT_IP_ASSIGNMENT):
vm = self._waitAndGetInstanceIpAddress(vm)
self.addVm(nodename, vm)
if not self.hasCapability(self.CAPABILITY_CONTEXTUALIZATION) and not self.isWindows():
self._secureSshAccessAndRunBootstrapScript(user_info, image_info,
nodename,
self.vmGetIp(vm))
elif not self.hasCapability(self.CAPABILITY_WINDOWS_CONTEXTUALIZATION) and self.isWindows():
self._launchWindowsBootstrapScript(image_info, nodename,
self.vmGetIp(vm))
def _startImage(self, user_info, image_info, instance_name, cloudSpecificData=None):
"""Cloud specific VM provisioning.
Returns: node - cloud specific representation of a started VM."""
raise NotImplementedError()
def initialization(self, user_info):
pass
def finalization(self, user_info):
"""This method is called once when all instances have been started or
if an exception has occurred."""
pass
def _getCloudSpecificData(self, node_info, node_number, nodename):
return None
def _waitAndGetInstanceIpAddress(self, vm):
"""This method needs to be implemented by the connector if the latter
not define the capability 'direct_ip_assignment'."""
return vm
def vmGetIp(self, vm_instance):
"""Cloud specific getter.
Returns: one IP - Public or Private."""
raise NotImplementedError()
def vmGetId(self, vm_instance):
"""Cloud specific getter.
Returns: cloud ID of the instance."""
raise NotImplementedError()
def vmGetPassword(self, vm_name):
return None
def stopDeployment(self):
"""This method should terminate all instances except the orchestrator if
the connector don't has the vapp capability.
This method should terminate the full vapp if the connector has the vapp
capability."""
pass
# please use stopDeployment instead
@deprecated
def stopImages(self):
"""Please use stopDeployment instead"""
return self.stopDeployment()
@deprecated
def stopImagesByIds(self, ids):
"""Please use stopVmsByIds instead"""
return self.stopVmsByIds(ids)
def stopVmsByIds(self, ids):
pass
def stopVappsByIds(self, ids):
pass
@deprecated
def setTerminateRunServerSide(self):
self._terminateRunServerSide = True
@deprecated
def isTerminateRunServerSide(self):
return self._terminateRunServerSide
def addVm(self, name, vm):
self._vms[name] = vm
self.publishVmInfo(name, vm)
def publishVmInfo(self, nodename, vm):
self.publishVmId(nodename, self.vmGetId(vm))
self.publishVmIp(nodename, self.vmGetIp(vm))
def publishVmId(self, nodename, vm_id):
# Needed for thread safety
NodeInfoPublisher(self.configHolder).publish_instanceid(nodename, str(vm_id))
def publishVmIp(self, nodename, vm_ip):
# Needed for thread safety
NodeInfoPublisher(self.configHolder).publish_hostname(nodename, vm_ip)
def getVms(self):
return self._vms
def getVm(self, name):
try:
return self._vms[name]
except KeyError:
raise Exceptions.NotFoundError("VM '%s' not found." % name)
def _setTempPrivateKeyAndPublicKey(self, privateKeyFileName, publicKey):
self.tempPrivateKeyFileName = privateKeyFileName
self.tempPublicKey = publicKey
def _getTempPrivateKeyFileNameAndPublicKey(self):
return self.tempPrivateKeyFileName, self.tempPublicKey
def _buildImage(self, user_info, image_info):
pass
def buildImage(self, user_info, image_info):
if not self.hasCapability(self.CAPABILITY_CONTEXTUALIZATION):
username, password = self._getSshUsernamePassword(image_info)
privateKey, publicKey = generateKeyPair()
privateKey = util.filePutContentInTempFile(privateKey)
self._setTempPrivateKeyAndPublicKey(privateKey, publicKey)
ip = self.vmGetIp(self.getVm(NodeDecorator.MACHINE_NAME))
self._secureSshAccess(ip, username, password, publicKey)
new_id = self._buildImage(user_info, image_info)
if new_id:
self.setNewImageId(new_id)
return new_id
def getNewImageId(self):
return self._newImageId
def setNewImageId(self, image_id):
self._newImageId = image_id
def getCreatorVmId(self):
return self._creatorVmId
def _buildImageIncrement(self, user_info, imageInfo, host):
prerecipe, recipe, packages = self.extractAllTargets(imageInfo)
try:
username, password, sshPrivateKeyFile = \
self._getSshCredentials(imageInfo, user_info,
NodeDecorator.MACHINE_NAME)
if not self.hasCapability(self.CAPABILITY_CONTEXTUALIZATION) and not sshPrivateKeyFile:
password = ''
sshPrivateKeyFile, publicKey = self._getTempPrivateKeyFileNameAndPublicKey()
self._waitCanConnectWithSshOrAbort(host, username=username,
password=password,
sshKey=sshPrivateKeyFile)
if prerecipe:
util.printStep('Running Pre-recipe')
remoteRunScript(username, host, prerecipe,
sshKey=sshPrivateKeyFile, password=password)
if packages:
util.printStep('Installing Packages')
remoteInstallPackages(username, host, packages,
self.getPlatform(imageInfo),
sshKey=sshPrivateKeyFile, password=password)
if recipe:
util.printStep('Running Recipe')
remoteRunScript(username, host, recipe,
sshKey=sshPrivateKeyFile, password=password)
if not self.hasCapability(self.CAPABILITY_CONTEXTUALIZATION):
self._revertSshSecurity(host, username, sshPrivateKeyFile, publicKey)
finally:
try:
os.unlink(sshPrivateKeyFile)
except:
pass
def _getCloudInstanceName(self):
return self.cloud
def _getSshCredentials(self, imageInfo, user_info, vm_name=None):
username, password = self._getSshUsernamePassword(imageInfo, vm_name)
if password:
sshPrivateKeyFile = None
else:
sshPrivateKeyFile = self._getSshPrivateKeyFile(user_info)
return username, password, sshPrivateKeyFile
def _getSshPrivateKeyFile(self, user_info):
fd, sshPrivateKeyFile = tempfile.mkstemp()
os.write(fd, self._getSshPrivateKey(user_info))
os.close(fd)
os.chmod(sshPrivateKeyFile, 0400)
return sshPrivateKeyFile
def _getInstanceType(self, image):
return self._getCloudParameter(image, 'instance.type')
def _getImageCpu(self, image):
return self._getCloudParameter(image, 'cpu')
def _getImageRam(self, image):
return self._getCloudParameter(image, 'ram')
def _getImageSmp(self, image):
return self._getCloudParameter(image, 'smp')
def _getImageNetworkType(self, image):
return self._getCloudParameter(image, 'network')
def _getSshUsernamePassword(self, image, vm_name=None):
attributes = image['attributes']
user = attributes.get('loginUser', '')
if not user:
user = '******'
password = None
try:
password = self._getCloudParameter(image, 'login.password')
except Exceptions.ParameterNotFoundException:
try:
password = self.vmGetPassword(vm_name)
except:
pass
return user, password
def _getCloudParameter(self, image, parameter):
params = image['cloud_parameters'][self.cloud]
param = '%s.%s' % (self.cloud, parameter)
try:
return params[param]
except KeyError:
raise Exceptions.ParameterNotFoundException("Cloud parameter '%s' not found" % param)
def _generateNodeName(self, nodename, node_number):
return nodename + NodeDecorator.NODE_MULTIPLICITY_SEPARATOR + str(node_number)
def _generateInstanceName(self, nodename):
return nodename + ':%s' % self._getRunId()
def _getRunId(self):
return os.environ.get('SLIPSTREAM_DIID', '???')
def _secureSshAccessAndRunBootstrapScript(self, userInfo, image_info, nodename, ip):
username, password = self._getSshUsernamePassword(image_info, nodename)
privateKey, publicKey = generateKeyPair()
privateKey = util.filePutContentInTempFile(privateKey)
self._secureSshAccess(ip, username, password, publicKey, userInfo)
self._launchBootstrapScript(nodename, ip, username, privateKey)
def _secureSshAccess(self, ip, username, password, publicKey, userInfo=None):
self._waitCanConnectWithSshOrAbort(ip, username, password)
script = self.getObfuscationScript(publicKey, username, userInfo)
self._printDetail("Securing SSH access to %s with:\n%s\n" % (ip, script))
_, output = self._runScript(ip, username, script, password=password)
self._printDetail("Secured SSH access to %s. Output:\n%s\n" % (ip, output))
def _revertSshSecurity(self, ip, username, privateKey, orchestratorPublicKey):
self._waitCanConnectWithSshOrAbort(ip, username, sshKey=privateKey)
script = "#!/bin/bash -xe\n"
script += "set +e\n"
script += "grep -vF '" + orchestratorPublicKey + "' ~/.ssh/authorized_keys > ~/.ssh/authorized_keys.tmp\n"
script += "set -e\n"
script += "sleep 2\nmv ~/.ssh/authorized_keys.tmp ~/.ssh/authorized_keys\n"
script += "chown -R " + username + ":$(id -g " + username + ")" + " ~/.ssh\n"
script += "restorecon -Rv ~/.ssh || true\n"
script += "sed -i -r 's/^#?[\\t ]*(PasswordAuthentication[\\t ]+)((yes)|(no))/\\1yes/' /etc/ssh/sshd_config\n"
script += "sync\nsleep 2\n"
script += "[ -x /etc/init.d/sshd ] && { service sshd reload; } || { service ssh reload; }\n"
self._printDetail("Reverting security of SSH access to %s with:\n%s\n" % (ip, script))
_, output = self._runScript(ip, username, script, sshKey=privateKey)
self._printDetail("Reverted security of SSH access to %s. Output:\n%s\n" % (ip, output))
def _launchBootstrapScript(self, nodename, ip, username, privateKey):
self._waitCanConnectWithSshOrAbort(ip, username, sshKey=privateKey)
script = self._getBootstrapScript(nodename)
self._printDetail("Launching bootstrap script on %s:\n%s\n" % (ip, script))
_, output = self._runScriptOnBackgroud(ip, username, script, sshKey=privateKey)
self._printDetail("Launched bootstrap script on %s:\n%s\n" % (ip, output))
def _launchWindowsBootstrapScript(self, image_info, nodename, ip):
username, password = self._getSshUsernamePassword(image_info, nodename)
script = self._getBootstrapScript(nodename, username=username)
winrm = self._getWinrm(ip, username, password)
self._waitCanConnectWithWinrmOrAbort(winrm)
self._printDetail("Launching bootstrap script on %s:\n%s\n" % (ip, script))
util.printAndFlush(script)
winrm.timeout = winrm.set_timeout(600)
output = self._runScriptWithWinrm(winrm, script)
self._printDetail("Launched bootstrap script on %s:\n%s\n" % (ip, output))
def _getWinrm(self, ip, username, password):
return WinRMWebService(endpoint='http://%s:5985/wsman' % ip, transport='plaintext',
username=username, password=password)
def _runScriptWithWinrm(self, winrm, script):
shellId = winrm.open_shell()
commands = ''
for command in script.splitlines():
if command:
commands += command + '& '
commands += 'echo "Bootstrap Finished"'
stdout, stderr, returnCode = self._runCommandWithWinrm(winrm, commands, shellId,
runAndContinue=True)
#winrm.close_shell(shellId)
return stdout, stderr, returnCode
def _waitCanConnectWithWinrmOrAbort(self, winrm):
try:
self._waitCanConnectWithWinrmOrTimeout(winrm, self.TIMEOUT_CONNECT)
except Exception as ex:
raise Exceptions.ExecutionException("Failed to connect to %s: %s" % (winrm.endpoint,
str(ex)))
def _waitCanConnectWithWinrmOrTimeout(self, winrm, timeout):
time_stop = time.time() + timeout
while (time_stop - time.time()) >= 0:
try:
_, _, returnCode = self._runCommandWithWinrm(winrm, 'exit 0')
if returnCode == 0:
return
except WinRMTransportError as ex:
util.printDetail(str(ex))
time.sleep(5)
def _runCommandWithWinrm(self, winrm, command, shellId=None, runAndContinue=False):
if shellId:
_shellId = shellId
else:
_shellId = winrm.open_shell()
util.printAndFlush('\nwinrm.run_command\n')
commandId = winrm.run_command(_shellId, command, [])
stdout, stderr, returnCode = ('N/A', 'N/A', 0)
util.printAndFlush('\nwinrm.get_command_output\n')
if not runAndContinue:
try:
stdout, stderr, returnCode = winrm.get_command_output(_shellId, commandId)
except Exception as e:
print 'WINRM Exception: %s' % str(e)
util.printAndFlush('\nwinrm.cleanup_command\n')
if not runAndContinue:
winrm.cleanup_command(_shellId, commandId)
if not shellId:
winrm.close_shell(_shellId)
return stdout, stderr, returnCode
def getObfuscationScript(self, orchestratorPublicKey, username, userInfo=None):
command = "#!/bin/bash -xe\n"
#command += "sed -r -i 's/# *(account +required +pam_access\\.so).*/\\1/' /etc/pam.d/login\n"
#command += "echo '-:ALL:LOCAL' >> /etc/security/access.conf\n"
command += "sed -i -r '/^[\\t ]*RSAAuthentication/d;/^[\\t ]*PubkeyAuthentication/d;/^[\\t ]*PasswordAuthentication/d' /etc/ssh/sshd_config\n"
command += "echo 'RSAAuthentication yes\nPubkeyAuthentication yes\nPasswordAuthentication no\n' >> /etc/ssh/sshd_config\n"
#command += "sed -i -r 's/^#?[\\t ]*(RSAAuthentication[\\t ]+)((yes)|(no))/\\1yes/' /etc/ssh/sshd_config\n"
#command += "sed -i -r 's/^#?[\\t ]*(PubkeyAuthentication[\\t ]+)((yes)|(no))/\\1yes/' /etc/ssh/sshd_config\n"
#command += "sed -i -r 's/^#?[\t ]*(PasswordAuthentication[\\t ]+)((yes)|(no))/\\1no/' /etc/ssh/sshd_config\n"
command += "umask 077\n"
command += "mkdir -p ~/.ssh\n"
if userInfo:
command += "echo '" + self._getPublicSshKey(userInfo) + "' >> ~/.ssh/authorized_keys\n"
command += "echo '" + orchestratorPublicKey + "' >> ~/.ssh/authorized_keys\n"
command += "chown -R " + username + ":$(id -g " + username + ")" + " ~/.ssh\n"
command += "restorecon -Rv ~/.ssh || true\n"
command += "[ -x /etc/init.d/sshd ] && { service sshd reload; } || { service ssh reload; }\n"
return command
def _getPublicSshKey(self, userInfo):
return userInfo.get_general('ssh.public.key') or ''
def _getBootstrapScript(self, nodename, preExport=None, preBootstrap=None, postBootstrap=None,
username=None):
script = ''
addEnvironmentVariableCommand = ''
if self.isWindows():
addEnvironmentVariableCommand = 'set'
else:
addEnvironmentVariableCommand = 'export'
script += '#!/bin/sh -ex\n'
if preExport:
script += '%s\n' % preExport
util.printAndFlush(str(os.environ))
for var, val in os.environ.items():
if var.startswith('SLIPSTREAM_') and var != 'SLIPSTREAM_NODENAME':
if var == 'SLIPSTREAM_REPORT_DIR' and self.isWindows():
val = Client.WINDOWS_REPORTSDIR
if re.search(' ', val):
val = '"%s"' % val
script += '%s %s=%s\n' % (addEnvironmentVariableCommand, var, val)
script += '%s SLIPSTREAM_NODENAME=%s\n' % (addEnvironmentVariableCommand, nodename)
script += '%s %s=%s\n' % (addEnvironmentVariableCommand,
util.ENV_NEED_TO_ADD_SSHPUBKEY,
self.hasCapability(self.ADD_SSHPUBKEY_ON_NODE))
if preBootstrap:
script += '%s\n' % preBootstrap
script += '%s\n' % self._buildSlipStreamBootstrapCommand(nodename, username)
if postBootstrap:
script += '%s\n' % postBootstrap
return script
def _buildSlipStreamBootstrapCommand(self, nodename, username=None):
if self.isWindows():
return self._buildSlipStreamBootstrapCommandForWindows(nodename, username)
else:
return self._buildSlipStreamBootstrapCommandForLinux(nodename)
def _buildSlipStreamBootstrapCommandForWindows(self, nodename, username):
if not username:
username = '******'
bootstrap = 'slipstream.bootstrap'
reportdir = Client.WINDOWS_REPORTSDIR
command = 'mkdir %(reports)s\n'
command += 'powershell -Command "$wc = New-Object System.Net.WebClient; $wc.DownloadFile(\'http://www.python.org/ftp/python/2.7.4/python-2.7.4.msi\', $env:temp+\'\\python.msi\')"\n'
command += 'start /wait msiexec /i %%TMP%%\\python.msi /qn /quiet /norestart /log log.txt TARGETDIR=C:\\Python27\\ ALLUSERS=1\n'
command += 'setx path "%%path%%;C:\\Python27" /M\n'
command += 'powershell -Command "[System.Net.ServicePointManager]::ServerCertificateValidationCallback = {$true}; $wc = New-Object System.Net.WebClient; $wc.Headers.Add(\'User-Agent\',\'PowerShell\'); $wc.DownloadFile(\'%(bootstrapUrl)s\', $env:temp+\'\\%(bootstrap)s\')" > %(reports)s\\%(nodename)s.slipstream.log 2>&1\n'
command += 'set PATH=%%PATH%%;C:\\Python27;C:\\opt\\slipstream\\client\\bin\n'
command += 'set PYTHONPATH=C:\\opt\\slipstream\\client\\lib\n'
password = ''
if not self.hasCapability(self.CAPABILITY_GENERATE_PASSWORD):
password = ''.join(random.choice(string.ascii_letters + string.digits)
for _ in range(10))
command += 'set pass=%(password)s\n'
command += 'net user %(username)s %%pass%%\n'
command += 'ss-get nodename > tmp.txt\n'
command += 'set /p nodename= < tmp.txt\n'
command += 'ss-get index > tmp.txt\n'
command += 'set /p index= < tmp.txt\n'
command += 'ss-get cloudservice > tmp.txt\n'
command += 'set /p cloudservice= < tmp.txt\n'
command += 'del tmp.txt\n'
command += 'ss-set %%nodename%%.%%index%%:%%cloudservice%%.login.password %%pass%%\n'
#command += 'C:\\Python27\\python %%TMP%%\\%(bootstrap)s >> %(reports)s\%(nodename)s.slipstream.log 2>&1\n'
command += 'start "test" "%%SystemRoot%%\System32\cmd.exe" /C "C:\\Python27\\python %%TMP%%\\%(bootstrap)s >> %(reports)s\%(nodename)s.slipstream.log 2>&1"\n'
return command % {
'bootstrap': bootstrap,
'bootstrapUrl': os.environ['SLIPSTREAM_BOOTSTRAP_BIN'],
'reports': reportdir,
'nodename': nodename,
'username': username,
'password': password
}
def _buildSlipStreamBootstrapCommandForLinux(self, nodename):
bootstrap = os.path.join(tempfile.gettempdir(), 'slipstream.bootstrap')
reportdir = Client.REPORTSDIR
command = 'mkdir -p %(reports)s; wget --no-check-certificate --secure-protocol=SSLv3 -O %(bootstrap)s %(bootstrapUrl)s >%(reports)s/%(nodename)s.slipstream.log 2>&1 && chmod 0755 %(bootstrap)s; %(bootstrap)s >>%(reports)s/%(nodename)s.slipstream.log 2>&1'
return command % {
'bootstrap': bootstrap,
'bootstrapUrl': os.environ['SLIPSTREAM_BOOTSTRAP_BIN'],
'reports': reportdir,
'nodename': nodename
}
def _waitCanConnectWithSshOrAbort(self, host, username='', password='', sshKey=None):
self._printDetail('Check if we can connect to %s' % host)
try:
waitUntilSshCanConnectOrTimeout(host,
self.TIMEOUT_CONNECT,
sshKey=sshKey,
user=username, password=password)
except Exception as ex:
raise Exceptions.ExecutionException("Failed to connect to %s: %s" % (host, str(ex)))
def _runScript(self, ip, username, script, password='', sshKey=None):
return remoteRunScript(username, ip, script, sshKey=sshKey,
password=password)
def _runScriptOnBackgroud(self, ip, username, script, password='', sshKey=None):
return remoteRunScriptNohup(username, ip, script,
sshKey=sshKey, password=password)
def _extractImageInfoFromNodeInfo(self, node_info):
return node_info['image']
def setSlipStreamClientAsListener(self, client):
self.listener = SlipStreamClientListenerAdapter(client)
def _printDetail(self, message):
util.printDetail(message, self.verboseLevel)
def getVmsDetails(self):
vms_details = []
for name, vm in self.getVms().items():
vms_details.append({name: {'id': self.vmGetId(vm),
'ip': self.vmGetIp(vm)}})
return vms_details
class BaseCloudConnector(object):
# ----- METHODS THAT CAN/SHOULD BE IMPLEMENTED IN CONNECTORS -----
def _initialization(self, user_info):
"""This method is called once before calling any others methods of the connector.
This method can be used to some initialization tasks like configuring the Cloud driver."""
pass
def _finalization(self, user_info):
"""This method is called once when all instances have been started or if an exception has occurred."""
pass
def _start_image(self, user_info, node_instance, vm_name):
"""Cloud specific VM provisioning.
Returns: node - cloud specific representation of a started VM."""
raise NotImplementedError()
def _build_image(self, user_info, node_instance):
"""This method is called during the Executing state of a build image. In most cases this method should call
self._build_image_increment() and then ask the Cloud to create an image from the instance.
The return value should be the new Cloud image id as a string."""
raise NotImplementedError()
def _wait_and_get_instance_ip_address(self, vm):
"""This method needs to be implemented by the connector if the latter not define the capability
'direct_ip_assignment'."""
return vm
def _stop_deployment(self):
"""This method should terminate the full vapp if the connector has the vapp capability.
This method should terminate all instances except the orchestrator if the connector doesn't have the vapp
capability."""
pass
def _stop_vms_by_ids(self, ids):
"""This method should destroy all VMs corresponding to VMs IDs of the list."""
raise NotImplementedError()
def _stop_vapps_by_ids(self, ids):
"""This method is used to destroy a full vApp if the capability 'vapp' is set."""
pass
def list_instances(self):
"""Returns list of VMs consumable by the connector's _vm_get_id(vm) and _vm_get_ip(vm).
"""
raise NotImplementedError()
def _vm_get_id(self, vm_instance):
"""Retrieve the VM ID from the vm_instance object returned by _start_image().
Returns: cloud ID of the instance."""
raise NotImplementedError()
def _vm_get_ip(self, vm_instance):
"""Retrieve an IP from the vm_instance object returned by _start_image().
Returns: one IP - Public or Private."""
raise NotImplementedError()
def _vm_get_ports_mapping(self, vm_instance):
"""Retrieve ports mapping from the vm_instance object returned by _start_image().
Returns: [<protocol>:<published_port>:<target_port> ...] e.g. tcp:8080:80 ."""
pass
def _vm_get_password(self, vm_instance):
"""Retrieve the password of the VM from the vm_instance object returned by _start_image().
Returns: the password needed to connect to the VM"""
pass
def _vm_get_state(self, vm_instance):
"""Retrieve VM state from the vm_instance object returned by list_instances().
Returns: VM cloud state."""
return ""
def _create_allow_all_security_group(self):
"""If the conncector support security groups, this method should create a security group named
NodeDecorator.SECURITY_GROUP_ALLOW_ALL_NAME with everything allowed if it doesn't exist yet."""
pass
''' The methods below are used to generate the reply of *-describe-instances
'''
def _vm_get_ip_from_list_instances(self, vm_instance):
"""Retrieve an IP from the vm_instance object returned by list_instances().
Returns: one IP - Public or Private."""
pass
def _vm_get_id_from_list_instances(self, vm_instance):
"""Retrieve an ID from the vm_instance object returned by list_instances().
Returns: one ID."""
pass
def _vm_get_cpu(self, vm_instance):
"""Retrieve the number of CPU of the VM from the vm_instance object returned by list_instances().
Returns: the number of CPU of VM"""
pass
def _vm_get_ram(self, vm_instance):
"""Retrieve the amount of RAM memory of the VM from the vm_instance object returned by list_instances().
Returns: the amount of RAM memory of the VM in MB"""
pass
def _vm_get_root_disk(self, vm_instance):
"""Retrieve the size of the root disk of the VM from the vm_instance object returned by list_instances().
Returns: the size of the root disk of the VM in GB"""
pass
def _vm_get_instance_type(self, vm_instance):
"""Retrieve the instance type of the VM from the vm_instance object returned by list_instances().
Returns: the name of the instance type of the VM"""
pass
def _get_vm_failed_states(self):
"""Override the method or provide cloud specific list of VM_FAILED_STATES.
"""
return self.VM_FAILED_STATES
''' The methods below are used to generate the reply of *-service-offers
'''
def _list_vm_sizes(self):
"""
Return a list of available VM sizes.
The data structure of 'size' element is left to the connector implementation.
Methods '_size_get_...(vm_size) are used to retrieve specific data from a 'size' element.
"""
pass
def _size_get_cpu(self, vm_size):
"""
Extract and return the amount of vCPU from the specified vm_size.
:param vm_size: A 'size' object as in the list returned by _list_vm_sizes().
:rtype int
"""
pass
def _size_get_ram(self, vm_size):
"""
Extract and return the size of the RAM memory in MB from the specified vm_size.
:param vm_size: A 'size' object as in the list returned by _list_vm_sizes().
:rtype float
"""
pass
def _size_get_disk(self, vm_size):
"""
Extract and return the size of the root disk in GB from the specified vm_size.
:param vm_size: A 'size' object as in the list returned by _list_vm_sizes().
:rtype float
"""
pass
def _size_get_instance_type(self, vm_size):
"""
Extract and return the instance type from the specified vm_size.
:param vm_size: A 'size' object as in the list returned by _list_vm_sizes().
:rtype int
"""
pass
"""IaaS actions for VM vertical scalability.
If the VM needs to be restarted, the implementation should
* wait until VM enters Running state, and
* assert that the action was correctly fulfilled by IaaS.
If the action on the VM can be applied w/o restart, the implementation should
* wait until action is applied by IaaS, and
* assert that the action was correctly fulfilled by IaaS.
Raise `slipstream.exceptions.Exceptions.ExecutionException`
on any handled error.
"""
def _resize(self, node_instance):
"""
:param node_instance: node instance object
:type node_instance: <NodeInstance>
"""
# Example code
# Cloud VM id.
# vm_id = node_instance.get_instance_id()
# RAM in GB.
# ram = node_instance.get_ram()
# Number of CPUs.
# cpu = node_instance.get_cpu()
# In case cloud uses T-short sizes.
# instance_type = node_instance.get_instance_type()
# IaaS calls go here.
raise NotImplementedError()
def _attach_disk(self, node_instance):
"""Attach extra disk to the VM.
:param node_instance: node instance object
:type node_instance: <NodeInstance>
:return: name of the device that was attached
:rtype: string
"""
# Example code
# device_name = ''
# Cloud VM id.
# vm_id = node_instance.get_instance_id()
# Size of the disk to attach (in GB).
# disk_size_GB = node_instance.get_disk_attach_size()
# IaaS calls go here.
# return device_name
raise NotImplementedError()
def _detach_disk(self, node_instance):
"""Detach disk from the VM.
:param node_instance: node instance object
:type node_instance: <NodeInstance>
"""
# Example code
# Cloud VM id.
# vm_id = node_instance.get_instance_id()
# Name of the block device to detach (/dev/XYZ).
# device = node_instance.get_disk_detach_device()
# IaaS calls go here.
raise NotImplementedError()
# ----------------------------------------------------------------
TIMEOUT_CONNECT = 10 * 60
RUN_BOOTSTRAP_SCRIPT = False
WAIT_IP = False
# CAPABILITIES
CAPABILITY_VAPP = 'vapp'
CAPABILITY_BUILD_IN_SINGLE_VAPP = 'buildInSingleVapp'
CAPABILITY_CONTEXTUALIZATION = 'contextualization'
CAPABILITY_WINDOWS_CONTEXTUALIZATION = 'windowsContextualization'
CAPABILITY_GENERATE_PASSWORD = '******'
CAPABILITY_DIRECT_IP_ASSIGNMENT = 'directIpAssignment'
CAPABILITY_ORCHESTRATOR_CAN_KILL_ITSELF_OR_ITS_VAPP = 'orchestratorCanKillItselfOrItsVapp'
VM_FAILED_STATES = ['failed', 'error']
def __init__(self, configHolder):
"""Constructor.
All connectors need to call this constructor from their own constructor.
Moreover all connectors need to define their capabilities with the method _set_capabilities().
"""
self.verboseLevel = 0
configHolder.assign(self)
self.configHolder = configHolder
self.run_category = getattr(configHolder, KEY_RUN_CATEGORY, None)
self.max_iaas_workers = None
self.sshPrivKeyFile = '%s/.ssh/id_rsa' % os.path.expanduser("~")
self.sshPubKeyFile = self.sshPrivKeyFile + '.pub'
self.__listener = SimplePrintListener(verbose=(self.verboseLevel > 1))
self.__vms = {}
self.__cloud = os.environ.get(util.ENV_CONNECTOR_INSTANCE)
self.__init_threading_related()
self.tempPrivateKeyFileName = ''
self.tempPublicKey = ''
self.__capabilities = []
self.__already_published = defaultdict(set)
# Ensure that httplib use the "old" default for certificates validation
# since libcloud doesn't work with the new default.
try:
ssl._https_verify_certificates(False)
except:
pass
self._user_info = None
self.cimi_deployment_prototype = False
@property
def user_info(self):
if self._user_info is None:
raise ValueError(
'Object not initialized. Please call _initialization first.')
return self._user_info
@user_info.setter
def user_info(self, value):
self._user_info = value
def __init_threading_related(self):
self.__tasks_runnner = None
# This parameter is thread local
self._thread_local = local()
def _set_capabilities(self,
vapp=False,
build_in_single_vapp=False,
contextualization=False,
windows_contextualization=False,
generate_password=False,
direct_ip_assignment=False,
orchestrator_can_kill_itself_or_its_vapp=False):
if vapp:
self.__capabilities.append(self.CAPABILITY_VAPP)
if build_in_single_vapp:
self.__capabilities.append(self.CAPABILITY_BUILD_IN_SINGLE_VAPP)
if contextualization:
self.__capabilities.append(self.CAPABILITY_CONTEXTUALIZATION)
if windows_contextualization:
self.__capabilities.append(
self.CAPABILITY_WINDOWS_CONTEXTUALIZATION)
if generate_password:
self.__capabilities.append(self.CAPABILITY_GENERATE_PASSWORD)
if direct_ip_assignment:
self.__capabilities.append(self.CAPABILITY_DIRECT_IP_ASSIGNMENT)
if orchestrator_can_kill_itself_or_its_vapp:
self.__capabilities.append(
self.CAPABILITY_ORCHESTRATOR_CAN_KILL_ITSELF_OR_ITS_VAPP)
def _reset_capabilities(self):
self.__capabilities = []
def has_capability(self, capability):
return capability in self.__capabilities
def __set_contextualization_capabilities(self, user_info):
native_contextualization = user_info.get_cloud(
NodeDecorator.NATIVE_CONTEXTUALIZATION_KEY, None)
if native_contextualization:
try:
self.__capabilities.remove(self.CAPABILITY_CONTEXTUALIZATION)
self.__capabilities.remove(
self.CAPABILITY_WINDOWS_CONTEXTUALIZATION)
except ValueError:
pass
if native_contextualization == 'always':
self.__capabilities.append(self.CAPABILITY_CONTEXTUALIZATION)
self.__capabilities.append(
self.CAPABILITY_WINDOWS_CONTEXTUALIZATION)
elif native_contextualization == 'linux-only':
self.__capabilities.append(self.CAPABILITY_CONTEXTUALIZATION)
elif native_contextualization == 'windows-only':
self.__capabilities.append(
self.CAPABILITY_WINDOWS_CONTEXTUALIZATION)
def is_build_image(self):
return self.run_category == NodeDecorator.IMAGE
def is_deployment(self):
return self.run_category == NodeDecorator.DEPLOYMENT
def stop_deployment(self):
self._stop_deployment()
def stop_vms_by_ids(self, ids):
self._stop_vms_by_ids(ids)
def stop_vapps_by_ids(self, ids):
self._stop_vapps_by_ids(ids)
def stop_node_instances(self, node_instances):
ids = []
for node_instance in node_instances:
ids.append(node_instance.get_instance_id())
self.__del_vm(node_instance.get_name())
if len(ids) > 0:
self.stop_vms_by_ids(ids)
def __create_allow_all_security_group_if_needed(self, nodes_instances):
allow_all = NodeDecorator.SECURITY_GROUP_ALLOW_ALL_NAME
for ni in nodes_instances.itervalues():
if allow_all in ni.get_security_groups():
self._create_allow_all_security_group()
break
def start_nodes_and_clients(self,
user_info,
nodes_instances,
init_extra_kwargs={}):
self._initialization(user_info, **init_extra_kwargs)
self.__set_contextualization_capabilities(user_info)
self.__create_allow_all_security_group_if_needed(nodes_instances)
self.cimi_deployment_prototype = bool(
nodes_instances.values()[0].get_deployment_context())
try:
self.__start_nodes_instantiation_tasks_wait_finished(
user_info, nodes_instances)
finally:
self._finalization(user_info)
def __start_nodes_instantiation_tasks_wait_finished(
self, user_info, nodes_instances):
self.__start_nodes_instances_and_clients(user_info, nodes_instances)
self.__wait_nodes_startup_tasks_finished()
def __start_nodes_instances_and_clients(self, user_info, nodes_instances):
self.__tasks_runnner = TasksRunner(
self.__start_node_instance_and_client,
max_workers=self.max_iaas_workers,
verbose=self.verboseLevel)
for node_instance in nodes_instances.values():
self.__tasks_runnner.put_task(user_info, node_instance)
self.__tasks_runnner.run_tasks()
def __wait_nodes_startup_tasks_finished(self):
if self.__tasks_runnner != None:
self.__tasks_runnner.wait_tasks_processed()
def __start_node_instance_and_client(self, user_info, node_instance):
node_instance_name = node_instance.get_name()
self._print_detail("Starting instance: %s" % node_instance_name)
node_context = node_instance.get_deployment_context()
self.cimi_deployment_prototype = bool(node_context)
if self.cimi_deployment_prototype:
vm_name = node_instance_name + '--' + node_context.get(
'SLIPSTREAM_DIID', '').replace('deployment/', '')
else:
vm_name = self._generate_vm_name(node_instance_name)
vm = self._start_image(user_info, node_instance, vm_name)
self.__add_vm(vm, node_instance)
if not self.has_capability(self.CAPABILITY_DIRECT_IP_ASSIGNMENT):
vm = self._wait_and_get_instance_ip_address(vm)
self.__add_vm(vm, node_instance)
if not self.is_build_image():
if not node_instance.is_windows() and not self.has_capability(
self.CAPABILITY_CONTEXTUALIZATION):
self.__secure_ssh_access_and_run_bootstrap_script(
user_info, node_instance, self._vm_get_ip(vm))
elif node_instance.is_windows() and not self.has_capability(
self.CAPABILITY_WINDOWS_CONTEXTUALIZATION):
self.__launch_windows_bootstrap_script(node_instance,
self._vm_get_ip(vm))
def get_vms(self):
return self.__vms
def _get_vm(self, name):
try:
return self.__vms[name]
except KeyError:
raise Exceptions.NotFoundError("VM '%s' not found." % name)
def __add_vm(self, vm, node_instance):
name = node_instance.get_name()
with lock:
self.__vms[name] = vm
self._publish_vm_info(vm, node_instance)
def __del_vm(self, name):
try:
del self.__vms[name]
except KeyError:
util.printDetail("Failed locally removing VM '%s'. Not found." %
name)
def _publish_vm_info(self, vm, node_instance):
instance_name = node_instance.get_name()
vm_id = self._vm_get_id(vm)
vm_ip = self._vm_get_ip(vm)
vm_ports_mapping = self._vm_get_ports_mapping(vm)
with lock:
already_published = self.__already_published[instance_name]
if self.cimi_deployment_prototype:
if vm_id and 'id' not in already_published:
node_instance.set_instance_id(vm_id)
already_published.add('id')
if vm_ip and 'ip' not in already_published:
node_instance.set_cloud_node_ip(vm_ip)
already_published.add('ip')
if node_instance and vm_ip and 'ssh' not in already_published:
if node_instance:
vm_ip = self._vm_get_ip(vm) or ''
ssh_username, ssh_password = self.__get_vm_username_password(
node_instance)
node_instance.set_cloud_node_ssh_url(
'ssh://%s@%s' %
(ssh_username.strip(), vm_ip.strip()))
node_instance.set_cloud_node_ssh_password(ssh_password)
if ssh_username and ssh_password:
already_published.add('ssh')
if vm_ports_mapping and 'vm_ports_mapping' not in already_published:
node_instance.set_cloud_node_ports_mapping(
vm_ports_mapping)
ssh_found = re.search('tcp:(\d+):22',
str(vm_ports_mapping))
if ssh_found:
ssh_username, ssh_password = self.__get_vm_username_password(
node_instance)
node_instance.set_cloud_node_ssh_url(
'ssh://{}@{}:{}'.format(ssh_username.strip(),
vm_ip.strip(),
ssh_found.group(1)))
already_published.add('vm_ports_mapping')
else:
if vm_id and 'id' not in already_published:
self._publish_vm_id(instance_name, vm_id)
already_published.add('id')
if vm_ip and 'ip' not in already_published:
self._publish_vm_ip(instance_name, vm_ip)
already_published.add('ip')
if node_instance and vm_ip and 'ssh' not in already_published:
self._publish_url_ssh(vm, node_instance)
already_published.add('ssh')
def _publish_vm_id(self, instance_name, vm_id):
# Needed for thread safety
NodeInfoPublisher(self.configHolder).publish_instanceid(
instance_name, str(vm_id))
def _publish_vm_ip(self, instance_name, vm_ip):
# Needed for thread safety
NodeInfoPublisher(self.configHolder).publish_hostname(
instance_name, vm_ip)
def _publish_url_ssh(self, vm, node_instance):
if not node_instance:
return
instance_name = node_instance.get_name()
vm_ip = self._vm_get_ip(vm) or ''
ssh_username, _ = self.__get_vm_username_password(node_instance)
# Needed for thread safety
NodeInfoPublisher(self.configHolder).publish_url_ssh(
instance_name, vm_ip, ssh_username)
def _set_temp_private_key_and_public_key(self, privateKeyFileName,
publicKey):
self.tempPrivateKeyFileName = privateKeyFileName
self.tempPublicKey = publicKey
def _get_temp_private_key_file_name_and_public_key(self):
return self.tempPrivateKeyFileName, self.tempPublicKey
def build_image(self, user_info, node_instance):
if not self.has_capability(self.CAPABILITY_CONTEXTUALIZATION):
username, password = self.__get_vm_username_password(node_instance)
privateKey, publicKey = generate_keypair()
privateKey = util.file_put_content_in_temp_file(privateKey)
self._set_temp_private_key_and_public_key(privateKey, publicKey)
ip = self._vm_get_ip(self._get_vm(NodeDecorator.MACHINE_NAME))
self._secure_ssh_access(ip, username, password, publicKey)
new_id = self._build_image(user_info, node_instance)
return new_id
def get_creator_vm_id(self):
vm = self._get_vm(NodeDecorator.MACHINE_NAME)
return self._vm_get_id(vm)
def _remote_run_build_target(self, node_instance, host, target, username,
password, private_key_file):
for subtarget in target:
target_name = subtarget.get('name')
full_target_name = '%s.%s' % (subtarget.get('module'), target_name)
if not MachineExecutor.need_to_execute_build_step(
node_instance, target, subtarget):
message = 'Component already built. Nothing to do on target "%s" for node "%s"\n' \
% (full_target_name, node_instance.get_name())
util.printAndFlush(message)
continue
script = subtarget.get('script')
if script:
message = 'Executing target "%s" on remote host "%s"' % (
full_target_name, node_instance.get_name())
self._print_step(message)
remoteRunScript(username,
host,
script,
sshKey=private_key_file,
password=password)
else:
message = 'Nothing to do for target "%s" on node "%s""\n' % (
full_target_name, node_instance.get_name())
util.printAndFlush(message)
def _build_image_increment(self, user_info, node_instance, host):
prerecipe = node_instance.get_prerecipe()
recipe = node_instance.get_recipe()
packages = node_instance.get_packages()
try:
machine_name = node_instance.get_name()
username, password, ssh_private_key_file = self._get_ssh_credentials(
node_instance, user_info)
if not self.has_capability(self.CAPABILITY_CONTEXTUALIZATION
) and not ssh_private_key_file:
password = ''
ssh_private_key_file, publicKey = self._get_temp_private_key_file_name_and_public_key(
)
self._wait_can_connect_with_ssh_or_abort(
host,
username=username,
password=password,
sshKey=ssh_private_key_file)
if prerecipe:
self._print_step('Running Pre-recipe', machine_name)
self._remote_run_build_target(node_instance, host, prerecipe,
username, password,
ssh_private_key_file)
if packages:
self._print_step('Installing Packages', machine_name)
platform = node_instance.get_platform()
remoteRunCommand(command=util.get_packages_install_command(
platform, packages),
host=host,
user=username,
sshKey=ssh_private_key_file,
password=password)
if recipe:
self._print_step('Running Recipe', machine_name)
self._remote_run_build_target(node_instance, host, recipe,
username, password,
ssh_private_key_file)
if not self.has_capability(self.CAPABILITY_CONTEXTUALIZATION):
self._revert_ssh_security(host, username, ssh_private_key_file,
publicKey)
finally:
try:
os.unlink(ssh_private_key_file)
except: # pylint: disable=bare-except
pass
def get_cloud_service_name(self):
return self.__cloud
def _get_ssh_credentials(self, node_instance, user_info):
username, password = self.__get_vm_username_password(node_instance)
if password:
ssh_private_key_file = None
else:
ssh_private_key_file = self.__get_ssh_private_key_file(user_info)
return username, password, ssh_private_key_file
def __get_ssh_private_key_file(self, user_info):
fd, ssh_private_key_file = tempfile.mkstemp()
os.write(fd, user_info.get_private_key())
os.close(fd)
os.chmod(ssh_private_key_file, 0400)
return ssh_private_key_file
def __get_vm_username_password(self, node_instance, default_user='******'):
user = node_instance.get_username(default_user)
password = self.__get_vm_password(node_instance)
return user, password
def __get_vm_password(self, node_instance):
password = node_instance.get_password()
if not password:
instance_name = node_instance.get_name()
try:
vm = self._get_vm(instance_name)
password = self._vm_get_password(vm)
except: # pylint: disable=bare-except
pass
return password
@staticmethod
def _get_run_uuid(default=None):
return os.environ.get('SLIPSTREAM_DIID', default)
@classmethod
def _generate_vm_name(cls, instance_name):
vm_name = instance_name
run_id = cls._get_run_uuid()
if run_id:
vm_name = vm_name + NodeDecorator.NODE_PROPERTY_SEPARATOR + run_id
return vm_name
@staticmethod
def is_start_orchestrator():
return os.environ.get('IS_ORCHESTRATOR', 'False') == 'True'
def __secure_ssh_access_and_run_bootstrap_script(self, user_info,
node_instance, ip):
username, password = self.__get_vm_username_password(node_instance)
privateKey, publicKey = generate_keypair()
privateKey = util.file_put_content_in_temp_file(privateKey)
self._secure_ssh_access(ip, username, password, publicKey, user_info)
self.__launch_bootstrap_script(node_instance, ip, username, privateKey)
def _secure_ssh_access(self,
ip,
username,
password,
publicKey,
user_info=None):
self._wait_can_connect_with_ssh_or_abort(ip, username, password)
script = self.__get_obfuscation_script(publicKey, username, user_info)
self._print_detail("Securing SSH access to %s with:\n%s\n" %
(ip, script))
_, output = self._run_script(ip, username, script, password=password)
self._print_detail("Secured SSH access to %s. Output:\n%s\n" %
(ip, output))
def _revert_ssh_security(self, ip, username, privateKey,
orchestratorPublicKey):
self._wait_can_connect_with_ssh_or_abort(ip,
username,
sshKey=privateKey)
script = "#!/bin/bash -xe\n"
script += "set +e\n"
script += "grep -vF '" + orchestratorPublicKey + "' ~/.ssh/authorized_keys > ~/.ssh/authorized_keys.tmp\n"
script += "set -e\n"
script += "sleep 2\nmv ~/.ssh/authorized_keys.tmp ~/.ssh/authorized_keys\n"
script += "chown -R " + username + ":$(id -g " + username + ")" + " ~/.ssh\n"
script += "restorecon -Rv ~/.ssh || true\n"
script += "sed -i -r 's/^#?[\\t ]*(PasswordAuthentication[\\t ]+)((yes)|(no))/\\1yes/' /etc/ssh/sshd_config\n"
script += "sync\nsleep 2\n"
script += "[ -x /etc/init.d/sshd ] && { service sshd reload; } || { service ssh reload || /etc/init.d/ssh reload; }\n"
self._print_detail(
"Reverting security of SSH access to %s with:\n%s\n" %
(ip, script))
_, output = self._run_script(ip, username, script, sshKey=privateKey)
self._print_detail(
"Reverted security of SSH access to %s. Output:\n%s\n" %
(ip, output))
def __launch_bootstrap_script(self, node_instance, ip, username,
privateKey):
self._wait_can_connect_with_ssh_or_abort(ip,
username,
sshKey=privateKey)
script = self._get_bootstrap_script(node_instance)
self._print_detail("Launching bootstrap script on %s:\n%s\n" %
(ip, script))
_, output = self._run_script_on_backgroud(ip,
username,
script,
sshKey=privateKey)
self._print_detail("Launched bootstrap script on %s:\n%s\n" %
(ip, output))
def __launch_windows_bootstrap_script(self, node_instance, ip):
username, password = self.__get_vm_username_password(
node_instance, 'administrator')
script = self._get_bootstrap_script(node_instance, username=username)
winrm = self._getWinrm(ip, username, password)
self._waitCanConnectWithWinrmOrAbort(winrm)
self._print_detail("Launching bootstrap script on %s:\n%s\n" %
(ip, script))
util.printAndFlush(script)
winrm.timeout = winrm.set_timeout(600)
output = self._runScriptWithWinrm(winrm, script)
self._print_detail("Launched bootstrap script on %s:\n%s\n" %
(ip, output))
def _getWinrm(self, ip, username, password):
return WinRMWebService(endpoint='http://%s:5985/wsman' % ip,
transport='plaintext',
username=username,
password=password)
def _runScriptWithWinrm(self, winrm, script):
shellId = winrm.open_shell()
commands = ''
for command in script.splitlines():
if command and not command.startswith('rem '):
commands += command + '& '
commands += 'echo "Bootstrap Finished"'
stdout, stderr, returnCode = self._runCommandWithWinrm(
winrm, commands, shellId, runAndContinue=True)
# winrm.close_shell(shellId)
return stdout, stderr, returnCode
def _waitCanConnectWithWinrmOrAbort(self, winrm):
try:
self._waitCanConnectWithWinrmOrTimeout(winrm, self.TIMEOUT_CONNECT)
except Exception as ex:
raise Exceptions.ExecutionException("Failed to connect to "
"%s: %s" %
(winrm.endpoint, str(ex)))
def _waitCanConnectWithWinrmOrTimeout(self, winrm, timeout):
time_stop = time.time() + timeout
while (time_stop - time.time()) >= 0:
try:
_, _, returnCode = self._runCommandWithWinrm(winrm, 'exit 0')
if returnCode == 0:
return
except WinRMTransportError as ex:
util.printDetail(str(ex))
time.sleep(5)
def _runCommandWithWinrm(self,
winrm,
command,
shellId=None,
runAndContinue=False):
if shellId:
_shellId = shellId
else:
_shellId = winrm.open_shell()
util.printAndFlush('\nwinrm.run_command\n')
commandId = winrm.run_command(_shellId, command, [])
stdout, stderr, returnCode = ('N/A', 'N/A', 0)
if not runAndContinue:
util.printAndFlush('\nwinrm.get_command_output\n')
try:
stdout, stderr, returnCode = winrm.get_command_output(
_shellId, commandId)
except Exception as e: # pylint: disable=broad-except
print 'WINRM Exception: %s' % str(e)
util.printAndFlush('\nwinrm.cleanup_command\n')
winrm.cleanup_command(_shellId, commandId)
if not shellId:
winrm.close_shell(_shellId)
return stdout, stderr, returnCode
def __get_obfuscation_script(self,
orchestrator_public_key,
username,
user_info=None):
command = "#!/bin/bash -xe\n"
# command += "sed -r -i 's/# *(account +required +pam_access\\.so).*/\\1/' /etc/pam.d/login\n"
# command += "echo '-:ALL:LOCAL' >> /etc/security/access.conf\n"
command += "sed -i -r '/^[\\t ]*RSAAuthentication/d;/^[\\t ]*PubkeyAuthentication/d;/^[\\t ]*PasswordAuthentication/d' /etc/ssh/sshd_config\n"
command += "echo 'RSAAuthentication yes\nPubkeyAuthentication yes\nPasswordAuthentication no\n' >> /etc/ssh/sshd_config\n"
# command += "sed -i -r 's/^#?[\\t ]*(RSAAuthentication[\\t ]+)((yes)|(no))/\\1yes/' /etc/ssh/sshd_config\n"
# command += "sed -i -r 's/^#?[\\t ]*(PubkeyAuthentication[\\t ]+)((yes)|(no))/\\1yes/' /etc/ssh/sshd_config\n"
# command += "sed -i -r 's/^#?[\t ]*(PasswordAuthentication[\\t ]+)((yes)|(no))/\\1no/' /etc/ssh/sshd_config\n"
command += "umask 077\n"
command += "mkdir -p ~/.ssh\n"
if user_info:
command += "echo '" + user_info.get_public_keys(
) + "' >> ~/.ssh/authorized_keys\n"
command += "echo '" + orchestrator_public_key + "' >> ~/.ssh/authorized_keys\n"
command += "chown -R " + username + ":$(id -g " + username + ")" + " ~/.ssh\n"
command += "restorecon -Rv ~/.ssh || true\n"
command += "[ -x /etc/init.d/sshd ] && { service sshd reload; } || { service ssh reload; }\n"
return command
def _get_bootstrap_script_if_not_build_image(self,
node_instance,
pre_export=None,
pre_bootstrap=None,
post_bootstrap=None,
username=None):
return self._get_bootstrap_script(node_instance, pre_export, pre_bootstrap, post_bootstrap, username) \
if not self.is_build_image() else ''
def _get_bootstrap_script(self,
node_instance,
pre_export=None,
pre_bootstrap=None,
post_bootstrap=None,
username=None):
"""This method can be redefined by connectors if they need a specific bootstrap script
with the SSH contextualization."""
script = ''
addEnvironmentVariableCommand = ''
node_instance_name = node_instance.get_name()
if node_instance.is_windows():
addEnvironmentVariableCommand = 'set'
script += 'rem cmd\n'
else:
addEnvironmentVariableCommand = 'export'
script += '#!/bin/sh -ex\n'
if pre_export:
script += '%s\n' % pre_export
if self.cimi_deployment_prototype:
for var, val in node_instance.get_deployment_context().items():
if re.search(' ', val):
val = '"%s"' % val
script += '%s %s=%s\n' % (addEnvironmentVariableCommand, var,
val)
else:
regex = 'SLIPSTREAM_'
if self.is_start_orchestrator():
regex += '|CLOUDCONNECTOR_'
env_matcher = re.compile(regex)
if pre_export:
script += '%s\n' % pre_export
for var, val in os.environ.items():
if env_matcher.match(
var) and var != util.ENV_NODE_INSTANCE_NAME:
if re.search(' ', val):
val = '"%s"' % val
script += '%s %s=%s\n' % (addEnvironmentVariableCommand,
var, val)
script += '%s %s=%s\n' % (addEnvironmentVariableCommand,
util.ENV_NODE_INSTANCE_NAME,
node_instance_name)
if pre_bootstrap:
script += '%s\n' % pre_bootstrap
script += '%s\n' % self._build_slipstream_bootstrap_command(
node_instance, username)
if post_bootstrap:
script += '%s\n' % post_bootstrap
return script
def _build_slipstream_bootstrap_command(self,
node_instance,
username=None):
instance_name = node_instance.get_name()
if self.cimi_deployment_prototype:
bootstrap_url = node_instance.get_deployment_context().get(
'SLIPSTREAM_BOOTSTRAP_BIN')
else:
bootstrap_url = util.get_required_envvar(
'SLIPSTREAM_BOOTSTRAP_BIN')
if node_instance.is_windows():
return self.__build_slipstream_bootstrap_command_for_windows(
instance_name, bootstrap_url)
else:
return self.__build_slipstream_bootstrap_command_for_linux(
instance_name, bootstrap_url)
def __build_slipstream_bootstrap_command_for_windows(
self, instance_name, bootstrap_url):
command = 'If Not Exist %(reports)s mkdir %(reports)s\n'
command += 'If Not Exist %(ss_home)s mkdir %(ss_home)s\n'
command += 'If Not Exist "C:\\Python27\\python.exe" ( '
command += ' powershell -Command "$wc = New-Object System.Net.WebClient; $wc.DownloadFile(\'https://www.python.org/ftp/python/2.7.13/python-2.7.13.amd64.msi\', $env:temp+\'\\python.msi\')"\n'
command += ' start /wait msiexec /i %%TMP%%\\python.msi /qn /quiet /norestart /log log.txt TARGETDIR=C:\\Python27\\ ALLUSERS=1 '
command += ')\n'
command += 'setx path "%%path%%;C:\\Python27;C:\\opt\\slipstream\\client\\bin;C:\\opt\\slipstream\\client\\sbin" /M\n'
command += 'setx PYTHONPATH "%%PYTHONPATH%%;C:\\opt\\slipstream\\client\\lib" /M\n'
command += 'powershell -Command "[System.Net.ServicePointManager]::ServerCertificateValidationCallback = {$true}; $wc = New-Object System.Net.WebClient; $wc.Headers.Add(\'User-Agent\',\'PowerShell\'); $wc.DownloadFile(\'%(bootstrapUrl)s\', \'%(bootstrap)s\')" > %(reports)s\\%(nodename)s.slipstream.log 2>&1\n'
command += 'set PATH=%%PATH%%;C:\\Python27;C:\\opt\\slipstream\\client\\bin;C:\\opt\\slipstream\\client\\sbin\n'
command += 'set PYTHONPATH=C:\\opt\\slipstream\\client\\lib\n'
command += 'start "test" "%%SystemRoot%%\System32\cmd.exe" /C "C:\\Python27\\python %(bootstrap)s %(machine_executor)s >> %(reports)s\\%(nodename)s.slipstream.log 2>&1"\n'
return command % self._get_bootstrap_command_replacements_for_windows(
instance_name, bootstrap_url)
def __build_slipstream_bootstrap_command_for_linux(self, instance_name,
bootstrap_url):
command = 'mkdir -p %(reports)s %(ss_home)s; '
command += '(wget --timeout=60 --retry-connrefused --no-check-certificate -O %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1 '
command += '|| curl --retry 20 -k -f -o %(bootstrap)s %(bootstrapUrl)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1) '
command += '&& chmod 0755 %(bootstrap)s; %(bootstrap)s %(machine_executor)s >> %(reports)s/%(nodename)s.slipstream.log 2>&1'
return command % self._get_bootstrap_command_replacements_for_linux(
instance_name, bootstrap_url)
def _get_bootstrap_command_replacements_for_linux(self, instance_name,
bootstrap_url):
return {
'reports':
util.REPORTSDIR,
'bootstrap':
os.path.join(util.SLIPSTREAM_HOME, 'slipstream.bootstrap'),
'bootstrapUrl':
bootstrap_url,
'ss_home':
util.SLIPSTREAM_HOME,
'nodename':
instance_name,
'machine_executor':
'node' if self.cimi_deployment_prototype else
self._get_machine_executor_type()
}
def _get_bootstrap_command_replacements_for_windows(
self, instance_name, bootstrap_url):
return {
'reports':
util.WINDOWS_REPORTSDIR,
'bootstrap':
'\\'.join([util.WINDOWS_SLIPSTREAM_HOME, 'slipstream.bootstrap']),
'bootstrapUrl':
bootstrap_url,
'ss_home':
util.WINDOWS_SLIPSTREAM_HOME,
'nodename':
instance_name,
'machine_executor':
'node' if self.cimi_deployment_prototype else
self._get_machine_executor_type()
}
def _get_machine_executor_type(self):
return self.is_start_orchestrator() and 'orchestrator' or 'node'
def _wait_can_connect_with_ssh_or_abort(self,
host,
username='',
password='',
sshKey=None):
self._print_detail('Check if we can connect to %s' % host)
try:
waitUntilSshCanConnectOrTimeout(host,
self.TIMEOUT_CONNECT,
sshKey=sshKey,
user=username,
password=password,
verboseLevel=self.verboseLevel)
except Exception as ex:
raise Exceptions.ExecutionException("Failed to connect to "
"%s: %s, %s" %
(host, type(ex), str(ex)))
def _run_script(self, ip, username, script, password='', sshKey=None):
return remoteRunScript(username,
ip,
script,
sshKey=sshKey,
password=password)
def _run_script_on_backgroud(self,
ip,
username,
script,
password='',
sshKey=None):
return remoteRunScriptNohup(username,
ip,
script,
sshKey=sshKey,
password=password)
def set_slipstream_client_as_listener(self, client):
self._set_listener(SlipStreamClientListenerAdapter(client))
def _set_listener(self, listener):
self.__listener = listener
def _get_listener(self):
return self.__listener
def _print_detail(self, message):
util.printDetail(message, self.verboseLevel)
def _print_step(self, message, write_for=None):
util.printStep(message)
if write_for:
self._get_listener().write_for(write_for, message)
def get_vms_details(self):
vms_details = []
for name, vm in self.get_vms().items():
vms_details.append(
{name: {
'id': self._vm_get_id(vm),
'ip': self._vm_get_ip(vm)
}})
return vms_details
def _has_vm_failed(self, vm_instance):
"""Check if VM failed on the cloud level. vm_instance as returned by _start_image().
Returns: True or False."""
vm_state = self._vm_get_state(vm_instance).lower()
return vm_state in [
fstate.lower() for fstate in self._get_vm_failed_states()
]
def resize(self, node_instances, done_reporter=None):
self._scale_action_runner(self._resize_and_report, node_instances,
done_reporter)
# TODO: use decorator for reporter.
def _resize_and_report(self, node_instance, reporter):
self._resize(node_instance)
if hasattr(reporter, '__call__'):
reporter(node_instance)
def attach_disk(self, node_instances, done_reporter=None):
self._scale_action_runner(self._attach_disk_and_report, node_instances,
done_reporter)
def _attach_disk_and_report(self, node_instance, reporter):
attached_disk = self._attach_disk(node_instance)
if not attached_disk:
raise Exceptions.ExecutionException(
'Attached disk name not provided by connector after disk attach operation.'
)
if hasattr(reporter, '__call__'):
reporter(node_instance, attached_disk)
def detach_disk(self, node_instances, done_reporter=None):
self._scale_action_runner(self._detach_disk_and_report, node_instances,
done_reporter)
def _detach_disk_and_report(self, node_instance, reporter):
self._detach_disk(node_instance)
if hasattr(reporter, '__call__'):
reporter(node_instance)
def _scale_action_runner(self, scale_action, node_instances,
done_reporter):
"""
:param scale_action: task executor
:type scale_action: callable
:param node_instances: list of node instances
:type node_instances: list [NodeInstance, ]
:param done_reporter: function that reports back to SlipStream
:type done_reporter: callable with signature `done_reporter(<NoneInstance>)`
"""
scaler = VmScaler(scale_action, self.max_iaas_workers,
self.verboseLevel)
scaler.set_tasks_and_run(node_instances, done_reporter)
scaler.wait_tasks_finished()