def images(self):
if self._images is None:
with self._lock_images:
if self._images is None:
util.printDetail('Getting images')
self._images = self.libcloud_driver.list_images(location=self.zone)
return self._images
def zones(self):
if self._zones is None:
with self._lock_zones:
if self._zones is None:
util.printDetail('Getting zones')
self._zones = self.libcloud_driver.list_locations()
return self._zones
def images(self):
if self._images is None:
with self._lock_images:
if self._images is None:
util.printDetail('Getting images')
self._images = self.libcloud_driver.list_images(location=self.zone)
return self._images
def volumes(self):
if self._volumes is None:
with self._lock_volumes:
if self._volumes is None:
util.printDetail('Getting volumes')
self._volumes = self.libcloud_driver.list_volumes()
return self._volumes
def zones(self):
if self._zones is None:
with self._lock_zones:
if self._zones is None:
util.printDetail('Getting zones')
self._zones = self.libcloud_driver.list_locations()
return self._zones
def zone(self):
if self._zone is None:
with self._lock_zone:
if self._zone is None:
util.printDetail('Getting zone')
self._zone = self._get_zone(self.user_info)
return self._zone
def _is_horizontal_scale_down(self):
try:
return self.wrapper.is_horizontal_scale_down_vm()
except InconsistentScaleStateError as ex:
util.printDetail("Machine Executor. Ignoring exception: %s" %
str(ex))
return False
def zone(self):
if self._zone is None:
with self._lock_zone:
if self._zone is None:
util.printDetail('Getting zone')
self._zone = self._get_zone(self.user_info)
return self._zone
def _getConfigFile(self):
for location in LocalContextualizer.LOCAL_CONTEXTUALIZATION_LOCATIONS:
filename = os.path.join(location, LocalContextualizer.LOCAL_CONTEXTUALIZATION_FILENAME)
if os.path.exists(filename):
util.printDetail('Using local contextualization file: %s' % filename, self.verboseLevel)
return filename
raise Exceptions.ConfigurationError('Failed to find local contextualization file.')
def onSendingReports(self):
util.printAction('Sending report')
if self._need_to_send_reports():
self._execute_report_target_and_send_reports()
self._unset_need_to_send_reports()
else:
util.printDetail('INFO: Conditionally skipped sending reports.',
verboseThreshold=util.VERBOSE_LEVEL_QUIET)
self.wrapper.set_scale_state_operational()
def _waitCanConnectWithWinrmOrTimeout(self, winrm, timeout):
time_stop = time.time() + timeout
while (time_stop - time.time()) >= 0:
try:
_, _, returnCode = self._runCommandWithWinrm(winrm, 'exit 0')
if returnCode == 0:
return
except WinRMTransportError as ex:
util.printDetail(str(ex))
time.sleep(5)
def _waitCanConnectWithWinrmOrTimeout(self, winrm, timeout):
time_stop = time.time() + timeout
while (time_stop - time.time()) >= 0:
try:
_, _, returnCode = self._runCommandWithWinrm(winrm, 'exit 0')
if returnCode == 0:
return
except WinRMTransportError as ex:
util.printDetail(str(ex))
time.sleep(5)
def _getConfigFile(self):
for location in LocalContextualizer.LOCAL_CONTEXTUALIZATION_LOCATIONS:
filename = os.path.join(
location, LocalContextualizer.LOCAL_CONTEXTUALIZATION_FILENAME)
if os.path.exists(filename):
util.printDetail(
'Using local contextualization file: %s' % filename,
self.verboseLevel)
return filename
raise Exceptions.ConfigurationError(
'Failed to find local contextualization file.')
def onSendingFinalReport(self):
util.printAction('Sending report')
try:
self._executeTarget('report')
except ExecutionException as ex:
util.printDetail("Failed executing 'report' with: \n%s" % str(ex),
verboseLevel=self.verboseLevel,
verboseThreshold=util.VERBOSE_LEVEL_NORMAL)
raise
finally:
super(NodeDeploymentExecutor, self).onSendingFinalReport()
def _set_root_disk_sizes_on_vms(sources_and_vms, config_holder):
sources_and_sizes = {}
for source in sources_and_vms.keys():
try:
sources_and_sizes[source] = get_root_disk_size_from_disk_source(source, config_holder)
except Exception as ex:
util.printDetail('WARNING: Failed to get root disk size for image %s. Reason: %s' %
(source, str(ex)))
for source, size in sources_and_sizes.iteritems():
for vm in sources_and_vms[source]:
vm.template_disk_0_size = size
def _waitForNextState(self, state):
timeSleep = 5
timeMax = time.time() + float(self.timeout)
util.printDetail('Waiting for next state transition, currently in %s' %
state, self.verboseLevel, util.VERBOSE_LEVEL_NORMAL)
while time.time() <= timeMax:
newState = self.wrapper.getState()
if state != newState:
return newState
else:
time.sleep(timeSleep)
raise TimeoutException('Timeout reached waiting for next state, current state: %s' % state)
def _get_state(self):
state = self.wrapper.getState()
if state:
return state
else:
for stime in self._get_state_retry_sleep_times():
util.printDetail('WARNING: Got no state. Retrying after %s sec.' % stime)
self._sleep(stime)
state = self.wrapper.getState()
if state:
return state
raise ExecutionException('ERROR: Machine executor: Got no state from server.')
def _wait_for_next_state(self, state):
"""Returns the next state after waiting (polling is used) for the state
transition from the server.
"""
util.printDetail('Waiting for the next state transition, currently in %s' % state,
self.verboseLevel, util.VERBOSE_LEVEL_NORMAL)
while True:
new_state = self._get_state()
if state != new_state:
return new_state
self._sleep(self._get_sleep_time(state))
def onExecuting(self):
super(OrchestratorDeploymentExecutor, self).onExecuting()
self._complete_state_for_failed_node_instances()
self.wrapper.check_scale_state_consistency()
self._get_recovery_mode()
if self._is_recovery_mode():
util.printDetail("Recovery mode enabled, recipes will not be executed.",
verboseThreshold=util.VERBOSE_LEVEL_QUIET)
return
self._execute_execute_target()
def _execute_report_target_and_send_reports(self):
exports = {'SLIPSTREAM_REPORT_DIR': util.get_platform_reports_dir()}
try:
self._execute_target('report',
exports=exports,
ssdisplay=False,
ignore_abort=True)
except ExecutionException as ex:
util.printDetail("Failed executing 'report' with: \n%s" % str(ex),
verboseLevel=self.verboseLevel,
verboseThreshold=util.VERBOSE_LEVEL_NORMAL)
raise
finally:
super(NodeDeploymentExecutor, self).onSendingReports()
def _execute_scale_action_target(self):
scale_action = self._get_global_scale_action()
if scale_action:
target = self._get_target_on_scale_action(scale_action)
if target:
exports = self._get_scaling_exports()
self._execute_target(target, exports)
self._set_need_to_send_reports()
else:
util.printDetail("Deployment is scaling. No target to "
"execute on action %s" % scale_action)
else:
util.printDetail("WARNING: deployment is scaling, but no "
"scaling action defined.")
def _run_target_script(self, target_script, exports=None, ignore_abort=False, name=None):
'''Return exit code of the user script and the last line of stderr
Output of the script goes to stdout/err and will end up in the node executor's log file.
'''
_name = self._get_script_name(name)
if not target_script:
util.printAndFlush('Script "%s" is empty\n' % (_name,))
return self.SCRIPT_EXIT_SUCCESS
if not isinstance(target_script, basestring):
raise ExecutionException('Not a string buffer provided as target for script "%s". Type is: %s'
% (_name, type(target_script)))
process = self._launch_process(target_script, exports, name)
result = Queue()
t = Thread(target=self.print_and_keep_last_stderr, args=(process.stderr, result))
t.daemon = True # thread dies with the program
t.start()
try:
# The process is still working on the background.
while process.poll() is None:
# Ask server whether the abort flag is set. If so, kill the
# process and exit. Otherwise, sleep for some time.
if not ignore_abort and self.wrapper.isAbort():
try:
util.printDetail('Abort flag detected. '
'Terminating execution of script "%s"...' % (_name,))
process.terminate()
util.sleep(5)
if process.poll() is None:
util.printDetail('Termination is taking too long. '
'Killing the script "%s"...' % (_name,))
process.kill()
except OSError:
pass
break
util.sleep(self.TARGET_POLL_INTERVAL)
except IOError as e:
if e.errno != errno.EINTR:
raise
else:
util.printDetail('Signal EINTR detected. Ignoring it.')
return 0
util.printDetail("End of the script '%s'" % (_name,))
stderr_last_line = ''
try:
stderr_last_line = result.get(timeout=60)
except Empty:
pass
return process.returncode, stderr_last_line
def __init__(self, dummy):
self.verboseLevel = 0
self.options = None
self.args = None
self.parser = None
self._setParserAndParse()
self.userProperties = {}
self.userEnv = {}
self.version = False
self.noemail = False
util.PRINT_TO_STDERR_ONLY = True
util.printDetail("Calling: '%s'" % ' '.join(sys.argv), self.verboseLevel)
self._callAndHandleErrorsForCommands(self.doWork.__name__)
util.PRINT_TO_STDERR_ONLY = False
def _build_image(self, user_info, node_instance):
machine_name = node_instance.get_name()
vm = self._get_vm(machine_name)
vm_ip = self._vm_get_ip(vm)
self._build_image_increment(user_info, node_instance, vm_ip)
self.creator.createStep2()
image_id = self._search_storage_for_new_image(self.slConfigHolder)
if not image_id:
util.printDetail(
'WARNING: Failed to get image ID from StratusLab storage!',
verboseThreshold=0)
else:
util.printDetail('New built image ID %s' % image_id,
verboseThreshold=0)
return image_id
def __init__(self, dummy):
self.verboseLevel = 0
self.options = None
self.args = None
self.parser = None
self._setParserAndParse()
self.userProperties = {}
self.userEnv = {}
self.version = False
self.noemail = False
util.PRINT_TO_STDERR_ONLY = True
util.printDetail("Calling: '%s'" % ' '.join(sys.argv),
self.verboseLevel)
self._callAndHandleErrorsForCommands(self.doWork.__name__)
util.PRINT_TO_STDERR_ONLY = False
def onExecuting(self):
util.printAction('Executing')
self._get_recovery_mode()
if self._is_recovery_mode():
util.printDetail(
"Recovery mode enabled, recipes will not be executed.",
verboseThreshold=util.VERBOSE_LEVEL_QUIET)
return
if self._skip_execute_due_to_vertical_scaling:
util.printDetail(
"Vertical scaling: skipping execution of execute targets.",
verboseThreshold=util.VERBOSE_LEVEL_QUIET)
self._skip_execute_due_to_vertical_scaling = False
return
if not self.wrapper.is_scale_state_operational():
self._execute_build_recipes()
self._execute_execute_target()
else:
self._execute_scale_action_target()
def onInitializing(self):
util.printAction('Initializing')
self._addSshPubkeyIfNeeded()
util.printStep('Getting deployment targets')
self.targets = self.wrapper.getTargets()
util.printDetail('Deployment targets:')
for target, script in self.targets.items():
util.printAndFlush('-' * 25)
util.printDetail('Target: %s' % target)
util.printDetail('Script:\n%s\n' % script[0])
def libcloud_driver(self):
with self._lock_libcloud_driver:
if not hasattr(self._thread_local, '_libcloud_driver'):
util.printDetail('Initializing libcloud driver')
self._thread_local._libcloud_driver = self._get_driver(self.user_info)
return self._thread_local._libcloud_driver
def __del_vm(self, name):
try:
del self.__vms[name]
except KeyError:
util.printDetail("Failed locally removing VM '%s'. Not found." %
name)
def _log(msg):
util.printDetail(msg, verboseThreshold=0)
def onProvisioning(self):
util.printAction('Node image creation sequence')
util.printDetail(
"Nothing to do from within the image, "
"the orchestrator will do the work.", 0)
def libcloud_driver(self):
with self._lock_libcloud_driver:
if not hasattr(self._thread_local, '_libcloud_driver'):
util.printDetail('Initializing libcloud driver')
self._thread_local._libcloud_driver = self._get_driver(self.user_info)
return self._thread_local._libcloud_driver
def _printDetail(self, message):
util.printDetail(message, self.verboseLevel, util.VERBOSE_LEVEL_DETAILED)
def __del_vm(self, name):
try:
del self.__vms[name]
except KeyError:
util.printDetail("Failed locally removing VM '%s'. Not found." % name)
def _log_normal(self, message):
util.printDetail(message, self.verboseLevel,
util.VERBOSE_LEVEL_NORMAL)
def _log_debug(self, message):
util.printDetail(message, self.verboseLevel,
util.VERBOSE_LEVEL_DETAILED)
def _is_horizontal_scale_down(self):
try:
return self.wrapper.is_horizontal_scale_down()
except Exceptions.InconsistentScaleStateError as ex:
util.printDetail("Machine Executor. Ignoring exception: %s" % str(ex))
return False
def _printDetail(self, message):
util.printDetail(message, self.verboseLevel, self.verboseThreshold)
def log(self, message):
util.printDetail(message, self.verboseLevel)
def onInitializing(self):
util.printAction('Node image creation sequence')
util.printDetail("Nothing to do from within the image, "
"the orchestrator will do the work.", 0)
def log(self, message):
util.printDetail(message, self.verboseLevel)
def _print_detail(self, message):
util.printDetail(message, self.verboseLevel)
def _print_detail(self, message):
util.printDetail(message, self.verboseLevel)
def _printDetail(self, message):
util.printDetail(message, self.verboseLevel, self.verboseThreshold)
def _execute_build_recipes(self):
util.printDetail('Executing build recipes')
self._execute_target(NodeDecorator.NODE_PRERECIPE, abort_on_err=True)
self._install_user_packages()
self._execute_target(NodeDecorator.NODE_RECIPE, abort_on_err=True)
def _log_normal(self, message):
util.printDetail(message, self.verboseLevel, util.VERBOSE_LEVEL_NORMAL)
def _log(msg):
util.printDetail(msg, verboseThreshold=0)
