def images(self): if self._images is None: with self._lock_images: if self._images is None: util.printDetail('Getting images') self._images = self.libcloud_driver.list_images(location=self.zone) return self._images
def zones(self): if self._zones is None: with self._lock_zones: if self._zones is None: util.printDetail('Getting zones') self._zones = self.libcloud_driver.list_locations() return self._zones
def images(self): if self._images is None: with self._lock_images: if self._images is None: util.printDetail('Getting images') self._images = self.libcloud_driver.list_images(location=self.zone) return self._images
def volumes(self): if self._volumes is None: with self._lock_volumes: if self._volumes is None: util.printDetail('Getting volumes') self._volumes = self.libcloud_driver.list_volumes() return self._volumes
def zones(self): if self._zones is None: with self._lock_zones: if self._zones is None: util.printDetail('Getting zones') self._zones = self.libcloud_driver.list_locations() return self._zones
def zone(self): if self._zone is None: with self._lock_zone: if self._zone is None: util.printDetail('Getting zone') self._zone = self._get_zone(self.user_info) return self._zone
def _is_horizontal_scale_down(self): try: return self.wrapper.is_horizontal_scale_down_vm() except InconsistentScaleStateError as ex: util.printDetail("Machine Executor. Ignoring exception: %s" % str(ex)) return False
def zone(self): if self._zone is None: with self._lock_zone: if self._zone is None: util.printDetail('Getting zone') self._zone = self._get_zone(self.user_info) return self._zone
def _getConfigFile(self): for location in LocalContextualizer.LOCAL_CONTEXTUALIZATION_LOCATIONS: filename = os.path.join(location, LocalContextualizer.LOCAL_CONTEXTUALIZATION_FILENAME) if os.path.exists(filename): util.printDetail('Using local contextualization file: %s' % filename, self.verboseLevel) return filename raise Exceptions.ConfigurationError('Failed to find local contextualization file.')
def onSendingReports(self): util.printAction('Sending report') if self._need_to_send_reports(): self._execute_report_target_and_send_reports() self._unset_need_to_send_reports() else: util.printDetail('INFO: Conditionally skipped sending reports.', verboseThreshold=util.VERBOSE_LEVEL_QUIET) self.wrapper.set_scale_state_operational()
def _waitCanConnectWithWinrmOrTimeout(self, winrm, timeout): time_stop = time.time() + timeout while (time_stop - time.time()) >= 0: try: _, _, returnCode = self._runCommandWithWinrm(winrm, 'exit 0') if returnCode == 0: return except WinRMTransportError as ex: util.printDetail(str(ex)) time.sleep(5)
def _waitCanConnectWithWinrmOrTimeout(self, winrm, timeout): time_stop = time.time() + timeout while (time_stop - time.time()) >= 0: try: _, _, returnCode = self._runCommandWithWinrm(winrm, 'exit 0') if returnCode == 0: return except WinRMTransportError as ex: util.printDetail(str(ex)) time.sleep(5)
def _getConfigFile(self): for location in LocalContextualizer.LOCAL_CONTEXTUALIZATION_LOCATIONS: filename = os.path.join( location, LocalContextualizer.LOCAL_CONTEXTUALIZATION_FILENAME) if os.path.exists(filename): util.printDetail( 'Using local contextualization file: %s' % filename, self.verboseLevel) return filename raise Exceptions.ConfigurationError( 'Failed to find local contextualization file.')
def onSendingFinalReport(self): util.printAction('Sending report') try: self._executeTarget('report') except ExecutionException as ex: util.printDetail("Failed executing 'report' with: \n%s" % str(ex), verboseLevel=self.verboseLevel, verboseThreshold=util.VERBOSE_LEVEL_NORMAL) raise finally: super(NodeDeploymentExecutor, self).onSendingFinalReport()
def _set_root_disk_sizes_on_vms(sources_and_vms, config_holder): sources_and_sizes = {} for source in sources_and_vms.keys(): try: sources_and_sizes[source] = get_root_disk_size_from_disk_source(source, config_holder) except Exception as ex: util.printDetail('WARNING: Failed to get root disk size for image %s. Reason: %s' % (source, str(ex))) for source, size in sources_and_sizes.iteritems(): for vm in sources_and_vms[source]: vm.template_disk_0_size = size
def _waitForNextState(self, state): timeSleep = 5 timeMax = time.time() + float(self.timeout) util.printDetail('Waiting for next state transition, currently in %s' % state, self.verboseLevel, util.VERBOSE_LEVEL_NORMAL) while time.time() <= timeMax: newState = self.wrapper.getState() if state != newState: return newState else: time.sleep(timeSleep) raise TimeoutException('Timeout reached waiting for next state, current state: %s' % state)
def _get_state(self): state = self.wrapper.getState() if state: return state else: for stime in self._get_state_retry_sleep_times(): util.printDetail('WARNING: Got no state. Retrying after %s sec.' % stime) self._sleep(stime) state = self.wrapper.getState() if state: return state raise ExecutionException('ERROR: Machine executor: Got no state from server.')
def _wait_for_next_state(self, state): """Returns the next state after waiting (polling is used) for the state transition from the server. """ util.printDetail('Waiting for the next state transition, currently in %s' % state, self.verboseLevel, util.VERBOSE_LEVEL_NORMAL) while True: new_state = self._get_state() if state != new_state: return new_state self._sleep(self._get_sleep_time(state))
def onExecuting(self): super(OrchestratorDeploymentExecutor, self).onExecuting() self._complete_state_for_failed_node_instances() self.wrapper.check_scale_state_consistency() self._get_recovery_mode() if self._is_recovery_mode(): util.printDetail("Recovery mode enabled, recipes will not be executed.", verboseThreshold=util.VERBOSE_LEVEL_QUIET) return self._execute_execute_target()
def _execute_report_target_and_send_reports(self): exports = {'SLIPSTREAM_REPORT_DIR': util.get_platform_reports_dir()} try: self._execute_target('report', exports=exports, ssdisplay=False, ignore_abort=True) except ExecutionException as ex: util.printDetail("Failed executing 'report' with: \n%s" % str(ex), verboseLevel=self.verboseLevel, verboseThreshold=util.VERBOSE_LEVEL_NORMAL) raise finally: super(NodeDeploymentExecutor, self).onSendingReports()
def _execute_scale_action_target(self): scale_action = self._get_global_scale_action() if scale_action: target = self._get_target_on_scale_action(scale_action) if target: exports = self._get_scaling_exports() self._execute_target(target, exports) self._set_need_to_send_reports() else: util.printDetail("Deployment is scaling. No target to " "execute on action %s" % scale_action) else: util.printDetail("WARNING: deployment is scaling, but no " "scaling action defined.")
def _run_target_script(self, target_script, exports=None, ignore_abort=False, name=None): '''Return exit code of the user script and the last line of stderr Output of the script goes to stdout/err and will end up in the node executor's log file. ''' _name = self._get_script_name(name) if not target_script: util.printAndFlush('Script "%s" is empty\n' % (_name,)) return self.SCRIPT_EXIT_SUCCESS if not isinstance(target_script, basestring): raise ExecutionException('Not a string buffer provided as target for script "%s". Type is: %s' % (_name, type(target_script))) process = self._launch_process(target_script, exports, name) result = Queue() t = Thread(target=self.print_and_keep_last_stderr, args=(process.stderr, result)) t.daemon = True # thread dies with the program t.start() try: # The process is still working on the background. while process.poll() is None: # Ask server whether the abort flag is set. If so, kill the # process and exit. Otherwise, sleep for some time. if not ignore_abort and self.wrapper.isAbort(): try: util.printDetail('Abort flag detected. ' 'Terminating execution of script "%s"...' % (_name,)) process.terminate() util.sleep(5) if process.poll() is None: util.printDetail('Termination is taking too long. ' 'Killing the script "%s"...' % (_name,)) process.kill() except OSError: pass break util.sleep(self.TARGET_POLL_INTERVAL) except IOError as e: if e.errno != errno.EINTR: raise else: util.printDetail('Signal EINTR detected. Ignoring it.') return 0 util.printDetail("End of the script '%s'" % (_name,)) stderr_last_line = '' try: stderr_last_line = result.get(timeout=60) except Empty: pass return process.returncode, stderr_last_line
def __init__(self, dummy): self.verboseLevel = 0 self.options = None self.args = None self.parser = None self._setParserAndParse() self.userProperties = {} self.userEnv = {} self.version = False self.noemail = False util.PRINT_TO_STDERR_ONLY = True util.printDetail("Calling: '%s'" % ' '.join(sys.argv), self.verboseLevel) self._callAndHandleErrorsForCommands(self.doWork.__name__) util.PRINT_TO_STDERR_ONLY = False
def _build_image(self, user_info, node_instance): machine_name = node_instance.get_name() vm = self._get_vm(machine_name) vm_ip = self._vm_get_ip(vm) self._build_image_increment(user_info, node_instance, vm_ip) self.creator.createStep2() image_id = self._search_storage_for_new_image(self.slConfigHolder) if not image_id: util.printDetail( 'WARNING: Failed to get image ID from StratusLab storage!', verboseThreshold=0) else: util.printDetail('New built image ID %s' % image_id, verboseThreshold=0) return image_id
def __init__(self, dummy): self.verboseLevel = 0 self.options = None self.args = None self.parser = None self._setParserAndParse() self.userProperties = {} self.userEnv = {} self.version = False self.noemail = False util.PRINT_TO_STDERR_ONLY = True util.printDetail("Calling: '%s'" % ' '.join(sys.argv), self.verboseLevel) self._callAndHandleErrorsForCommands(self.doWork.__name__) util.PRINT_TO_STDERR_ONLY = False
def onExecuting(self): util.printAction('Executing') self._get_recovery_mode() if self._is_recovery_mode(): util.printDetail( "Recovery mode enabled, recipes will not be executed.", verboseThreshold=util.VERBOSE_LEVEL_QUIET) return if self._skip_execute_due_to_vertical_scaling: util.printDetail( "Vertical scaling: skipping execution of execute targets.", verboseThreshold=util.VERBOSE_LEVEL_QUIET) self._skip_execute_due_to_vertical_scaling = False return if not self.wrapper.is_scale_state_operational(): self._execute_build_recipes() self._execute_execute_target() else: self._execute_scale_action_target()
def onInitializing(self): util.printAction('Initializing') self._addSshPubkeyIfNeeded() util.printStep('Getting deployment targets') self.targets = self.wrapper.getTargets() util.printDetail('Deployment targets:') for target, script in self.targets.items(): util.printAndFlush('-' * 25) util.printDetail('Target: %s' % target) util.printDetail('Script:\n%s\n' % script[0])
def libcloud_driver(self): with self._lock_libcloud_driver: if not hasattr(self._thread_local, '_libcloud_driver'): util.printDetail('Initializing libcloud driver') self._thread_local._libcloud_driver = self._get_driver(self.user_info) return self._thread_local._libcloud_driver
def __del_vm(self, name): try: del self.__vms[name] except KeyError: util.printDetail("Failed locally removing VM '%s'. Not found." % name)
def _log(msg): util.printDetail(msg, verboseThreshold=0)
def onProvisioning(self): util.printAction('Node image creation sequence') util.printDetail( "Nothing to do from within the image, " "the orchestrator will do the work.", 0)
def libcloud_driver(self): with self._lock_libcloud_driver: if not hasattr(self._thread_local, '_libcloud_driver'): util.printDetail('Initializing libcloud driver') self._thread_local._libcloud_driver = self._get_driver(self.user_info) return self._thread_local._libcloud_driver
def _printDetail(self, message): util.printDetail(message, self.verboseLevel, util.VERBOSE_LEVEL_DETAILED)
def __del_vm(self, name): try: del self.__vms[name] except KeyError: util.printDetail("Failed locally removing VM '%s'. Not found." % name)
def _log_normal(self, message): util.printDetail(message, self.verboseLevel, util.VERBOSE_LEVEL_NORMAL)
def _log_debug(self, message): util.printDetail(message, self.verboseLevel, util.VERBOSE_LEVEL_DETAILED)
def _is_horizontal_scale_down(self): try: return self.wrapper.is_horizontal_scale_down() except Exceptions.InconsistentScaleStateError as ex: util.printDetail("Machine Executor. Ignoring exception: %s" % str(ex)) return False
def _printDetail(self, message): util.printDetail(message, self.verboseLevel, self.verboseThreshold)
def log(self, message): util.printDetail(message, self.verboseLevel)
def onInitializing(self): util.printAction('Node image creation sequence') util.printDetail("Nothing to do from within the image, " "the orchestrator will do the work.", 0)
def log(self, message): util.printDetail(message, self.verboseLevel)
def _print_detail(self, message): util.printDetail(message, self.verboseLevel)
def _print_detail(self, message): util.printDetail(message, self.verboseLevel)
def _printDetail(self, message): util.printDetail(message, self.verboseLevel, self.verboseThreshold)
def _execute_build_recipes(self): util.printDetail('Executing build recipes') self._execute_target(NodeDecorator.NODE_PRERECIPE, abort_on_err=True) self._install_user_packages() self._execute_target(NodeDecorator.NODE_RECIPE, abort_on_err=True)
def _log_normal(self, message): util.printDetail(message, self.verboseLevel, util.VERBOSE_LEVEL_NORMAL)
def _log(msg): util.printDetail(msg, verboseThreshold=0)