def onFinalizing(self):
util.printAction('Finalizing')
if self.wrapper.isAbort():
util.printError("Failed")
else:
util.printAction('Done!')
def _launch_process(self, target_script, exports=None, name=None):
'''Returns launched process as subprocess.Popen instance.
'''
try:
fn = self._write_target_script_to_file(target_script, name)
except Exception as e:
util.printError('Writing script "%s" to file failed with: "%s". Retrying with random filename.' % (name, e))
fn = self._write_target_script_to_file(target_script)
current_dir = os.getcwd()
new_dir = util.get_temporary_storage_dir()
os.chdir(new_dir)
if 'HOME' not in os.environ:
if exports is None:
exports = {}
exports['HOME'] = os.path.expanduser('~')
try:
process = util.execute(fn, noWait=True, extra_env=exports, withStderr=True)
finally:
os.chdir(current_dir)
return process
def _write_target_script_to_file(self, target_script, name=None):
file_suffix = ''
if util.is_windows():
file_suffix = '.ps1'
directory = None
try:
directory = util.get_state_storage_dir()
except Exception as e:
util.printError('Creating script storage directory failed with: "%s"' % (e,))
if name is None or directory is None:
fn = tempfile.mktemp(suffix=file_suffix, dir=directory)
else:
filename = re.sub(r'[^0-9a-z._-]', '', name.replace('/', '_').replace(' ', '-').replace(':', '__').lower())
if file_suffix:
filename += file_suffix
fn = os.path.join(directory, filename)
if isinstance(target_script, unicode):
with codecs.open(fn, 'w', 'utf8') as fh:
fh.write(target_script)
else:
with open(fn, 'w') as fh:
fh.write(target_script)
os.chmod(fn, 0755)
return fn
def _execute_state(self, state):
if not state:
raise ExecutionException('ERROR: Machine executor: No state to execute '
'specified.')
try:
self._set_state_start_time()
method_name = 'on' + state
if hasattr(self, method_name):
getattr(self, method_name)()
else:
self._state_not_implemented(state)
except AbortException as ex:
util.printError('Abort flag raised: %s' % ex)
except TerminalStateException:
return
except KeyboardInterrupt:
raise
except (SystemExit, Exception) as ex:
if isinstance(ex, SystemExit) and str(ex).startswith('Terminating on signal'):
self._log_and_set_statecustom('Machine executor is stopping with: %s' % ex)
else:
util.printError('Error executing node, with detail: %s' % ex)
traceback.print_exc()
self._fail(ex)
self.onSendingReports()
def _initialization(self, user_info):
util.printStep('Initialize the CloudStack connector.')
self.user_info = user_info
if self.is_build_image():
raise NotImplementedError('The run category "%s" is not yet implemented' % self.run_category)
elif self.is_deployment():
try:
self._import_keypair(user_info)
except Exceptions.ExecutionException as e:
util.printError(e)
def _start_image_on_cloudstack(self, user_info, node_instance, vm_name):
try:
kp_name = self._import_keypair(user_info)
node_instance.set_cloud_node_ssh_keypair_name(kp_name)
except Exceptions.ExecutionException as e:
util.printError(e)
instance_name = self.format_instance_name(vm_name)
instance_type = node_instance.get_instance_type()
ip_type = node_instance.get_network_type()
keypair = None
contextualization_script = None
if not node_instance.is_windows():
keypair = user_info.get_keypair_name()
contextualization_script = self._get_bootstrap_script_if_not_build_image(node_instance)
security_groups = node_instance.get_security_groups()
security_groups = (len(security_groups) > 0) and security_groups or None
try:
size = [i for i in self.sizes if i.name == instance_type][0]
except IndexError:
raise Exceptions.ParameterNotFoundException("Couldn't find the specified instance type: %s" % instance_type)
image = self._get_image(node_instance)
disk_size = node_instance.get_cloud_parameter('disk')
kwargs = dict(name=instance_name,
size=size,
image=image,
location=self.zone,
ex_security_groups=security_groups)
if disk_size:
kwargs['ex_rootdisksize'] = disk_size.replace('G', '')
if not node_instance.is_windows():
kwargs.update(dict(ex_keyname= keypair,
ex_userdata=contextualization_script))
instance = self.libcloud_driver.create_node(**kwargs)
ip = self._get_instance_ip_address(instance, ip_type)
if not ip:
raise Exceptions.ExecutionException("Couldn't find a '%s' IP" % ip_type)
vm = dict(instance=instance,
ip=ip,
id=instance.id)
return vm
def onInitializing(self):
util.printAction('Initializing')
util.printStep('Starting instances')
try:
self.wrapper.startImages()
except Exceptions.AbortException:
pass
except Exception as ex:
util.printError('Error starting instances with error: %s' % ex)
raise
util.printStep('Publishing instance initialization information')
self.wrapper.publishDeploymentInitializationInfo()
def onProvisioning(self):
super(OrchestratorImageBuildExecutor, self).onProvisioning()
util.printStep('Starting instance')
try:
self.wrapper.start_node_instances()
except Exceptions.AbortException:
pass
except Exception as ex:
util.printError('Error starting instance with error: %s' % ex)
raise
finally:
self._complete_machine_state()
def onProvisioning(self):
super(OrchestratorImageBuildExecutor, self).onProvisioning()
util.printStep('Starting instance')
try:
self.wrapper.start_node_instances()
except Exceptions.AbortException:
pass
except Exception as ex:
util.printError('Error starting instance with error: %s' % ex)
raise
finally:
self._complete_machine_state()
def _run_instances_action(action, action_name):
"""
:param action: action to run
:type action: callable
:param action_name: name of the action
:type: string
"""
util.printStep('%s instances' % action_name.capitalize())
try:
action()
except Exceptions.AbortException:
pass
except Exception as ex:
util.printError('Error %s instances: %s' % (action_name, ex))
raise
def onExecuting(self):
super(OrchestratorImageBuildExecutor, self).onExecuting()
if self.wrapper.isAbort():
util.printError('Abort set, skipping Running')
self._complete_machine_state()
return
util.printStep('Building new image')
try:
self.wrapper.build_image()
except KeyboardInterrupt:
raise
except Exception as ex:
self.wrapper.fail(str(ex))
finally:
self._complete_machine_state()
def onExecuting(self):
super(OrchestratorImageBuildExecutor, self).onExecuting()
if self.wrapper.isAbort():
util.printError('Abort set, skipping Running')
self._complete_machine_state()
return
util.printStep('Building new image')
try:
self.wrapper.build_image()
except KeyboardInterrupt:
raise
except Exception as ex:
self.wrapper.fail(str(ex))
finally:
self._complete_machine_state()
def _initialization(self, user_info):
util.printStep('Initialize the CloudStack connector.')
self._thread_local.driver = self._get_driver(user_info)
self.zones = self._thread_local.driver.list_locations()
self.zone = self._get_zone(user_info)
self.sizes = self._thread_local.driver.list_sizes(location=self.zone)
self.images = self._thread_local.driver.list_images(location=self.zone)
self.user_info = user_info
if self.is_build_image():
raise NotImplementedError('The run category "%s" is not yet implemented' % self.run_category)
elif self.is_deployment():
try:
self._import_keypair(user_info)
except Exceptions.ExecutionException as e:
util.printError(e)
def onTerminal(self):
util.printAction('Terminating')
util.printStep('Stopping instances')
try:
self.wrapper.stopNodes()
except Exceptions.AbortException:
pass
except Exception as ex:
util.printError('Error stopping instances: %s' % ex)
raise
util.printStep('Publishing instance termination information')
self.wrapper.publishDeploymentTerminateInfo()
super(OrchestratorDeploymentExecutor, self).onTerminal()
self._killItself()
def _ssh_execute_cli(cmd, host, user, sshKey, tcp_timeout, **kwargs):
def _appendToSshCommandFromKwargs(keyword, append):
if kwargs.get(keyword, False):
sshCmd.append(append)
try:
del kwargs[keyword]
except:
pass
def _removeInvalidExecuteKwargs():
for keyword in ['verboseLevel', 'verboseThreshold', 'password']:
try:
del kwargs[keyword]
except:
pass
sshCmd = ['ssh', '-p', SSH_PORT, '-o', 'ConnectTimeout=%s' % tcp_timeout,
'-o', 'StrictHostKeyChecking=no']
if sshKey and os.path.isfile(sshKey):
sshCmd.append('-i')
sshCmd.append(sshKey)
tty = kwargs.get('pseudoTTY', False)
for keyAppend in [('sshVerb', '-v'), ('sshQuiet', '-q'), ('pseudoTTY', '-t -t')]:
_appendToSshCommandFromKwargs(*keyAppend)
sshCmd.append('--')
sshCmd.append('%s@%s' % (user, host))
sshCmd.append(cmd)
_removeInvalidExecuteKwargs()
rc, stderr = execute(sshCmd, **kwargs)
if not tty and 'sudo' in stderr and 'tty' in stderr:
printError('sudo require tty: "%s". Retrying with tty' % stderr)
sshCmd.insert(1, '-tt')
return execute(sshCmd, **kwargs)
return rc, stderr
def need_to_stop_images(self, ignore_on_success_run_forever=False):
runParameters = self._get_run_parameters()
keep_running = runParameters.get('General.keep-running')
if not self._check_keep_running(keep_running):
message = 'Wrong value for "keep-running" (%s). Should be one of the following: %s. Using the default value (%s).'\
% (keep_running, self.KEEP_RUNNING_VALUES, self.KEEP_RUNNING_DEFAULT)
util.printError(message)
self.fail(message)
keep_running = self.KEEP_RUNNING_DEFAULT
stop = True
if self.isAbort():
if keep_running in [self.KEEP_RUNNING_ALWAYS, self.KEEP_RUNNING_ON_ERROR]:
stop = False
elif keep_running in [self.KEEP_RUNNING_ALWAYS, self.KEEP_RUNNING_ON_SUCCESS] and not ignore_on_success_run_forever:
stop = False
return stop
def _execute(self, state=None):
state = (state and state) or self.wrapper.getState()
if not state:
raise ExecutionException('Machine executor: No state to execute specified.')
try:
getattr(self, 'on' + state)()
except AbortException as ex:
util.printError('Abort flag raised: %s' % ex)
except TerminalStateException:
return
except KeyboardInterrupt:
raise
except (SystemExit, Exception) as ex:
util.printError('Error executing node, with detail: %s' % ex)
traceback.print_exc()
self.wrapper.fail(str(ex))
self.wrapper.advance()
state = self._waitForNextState(state)
self._execute(state)
def _test_start_stop_images(self):
"Live test that starts and stops VMs on a cloud."
self.client.run_category = RUN_CATEGORY_DEPLOYMENT
success = True
error = ''
try:
self.client.start_nodes_and_clients(self.user_info,
self.node_instances)
vms = self.client.get_vms()
assert len(vms) == self.multiplicity
util.printAction('Instances started.')
pp(vms)
except Exception as ex:
success = False
error = self._get_ex_msg(ex)
util.printError("Exception caught while starting instances!")
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
finally:
util.printAction("Stopping deployment.")
self.client.stop_deployment()
self.assertEquals(success, True, error)
def _test_start_stop_images(self):
"Live test that starts and stops VMs on a cloud."
self.client.run_category = RUN_CATEGORY_DEPLOYMENT
success = True
error = ''
try:
self.client.start_nodes_and_clients(self.user_info,
self.node_instances)
vms = self.client.get_vms()
assert len(vms) == self.multiplicity
util.printAction('Instances started.')
pp(vms)
except Exception as ex:
success = False
error = self._get_ex_msg(ex)
util.printError("Exception caught while starting instances!")
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
finally:
util.printAction("Stopping deployment.")
self.client.stop_deployment()
self.assertEquals(success, True, error)
def need_to_stop_images(self, ignore_on_success_run_forever=False):
runParameters = self._get_run_parameters()
keep_running = runParameters.get('General.keep-running')
if not self._check_keep_running(keep_running):
message = 'Wrong value for "keep-running" (%s). Should be one of the following: %s. Using the default value (%s).'\
% (keep_running, self.KEEP_RUNNING_VALUES, self.KEEP_RUNNING_DEFAULT)
util.printError(message)
self.fail(message)
keep_running = self.KEEP_RUNNING_DEFAULT
stop = True
if self.isAbort():
if keep_running in [
self.KEEP_RUNNING_ALWAYS, self.KEEP_RUNNING_ON_ERROR
]:
stop = False
elif keep_running in [
self.KEEP_RUNNING_ALWAYS, self.KEEP_RUNNING_ON_SUCCESS
] and not ignore_on_success_run_forever:
stop = False
return stop
def fail(self, message):
util.printError('Failing... %s' % message)
traceback.print_exc()
abort = self._qualifyKey(NodeDecorator.ABORT_KEY)
self.clientSlipStream.setRuntimeParameter(abort, message)
def _fail(self, key, message):
util.printError('Failing... %s' % message)
traceback.print_exc()
value = util.truncate_middle(Client.VALUE_LENGTH_LIMIT, message,
'\n(truncated)\n')
self._ss_client.setRuntimeParameter(key, value)
def _fail(self, key, message):
util.printError('Failing... %s' % message)
traceback.print_exc()
value = util.truncate_middle(Client.VALUE_LENGTH_LIMIT, message, '\n(truncated)\n')
self._ss_client.setRuntimeParameter(key, value)
def _publish_abort_and_fail(self, message, exception):
util.printError('Failing... %s: %s' % (message, str(exception)))
traceback.print_exc()
AbortExceptionPublisher(self.configHolder).publish(message, sys.exc_info())
raise exception
def _publish_abort_and_fail(self, message, exception):
util.printError('Failing... %s: %s' % (message, str(exception)))
traceback.print_exc()
AbortExceptionPublisher(self.configHolder).publish(
message, sys.exc_info())
raise exception
def onTerminal(self):
if self.wrapper.isAbort():
util.printError("Failed")
else:
util.printAction('Done!')