def __load_module(self, alias): """ Load module class by alias :param alias: str :return: class """ if alias in self.modules: return self.modules[alias] mod_conf = self.config.get('modules') if alias not in mod_conf: msg = "Module '%s' not found in list of available aliases %s" % (alias, sorted(mod_conf.keys())) raise TaurusConfigError(msg) settings = ensure_is_dict(mod_conf, alias, "class") acopy = copy.deepcopy(settings) BetterDict.traverse(acopy, Configuration.masq_sensitive) self.log.debug("Module config: %s %s", alias, acopy) err = TaurusConfigError("Class name for alias '%s' is not found in module settings: %s" % (alias, settings)) clsname = settings.get('class', err) self.modules[alias] = load_class(clsname) if not issubclass(self.modules[alias], EngineModule): # raise TaurusInternalException("Module class does not inherit from EngineModule: %s" % clsname) pass return self.modules[alias]
def __apply_single_override(self, dest, name, value): """ Apply single override :type name: str :type value: str """ self.log.debug("Applying %s=%s", name, value) parts = [(int(x) if is_int(x) else x) for x in name.split(".")] pointer = dest for index, part in enumerate(parts[:-1]): self.__ensure_list_capacity(pointer, part, parts[index + 1]) if isinstance(part, integer_types): if part < 0: if isinstance(parts[index + 1], integer_types): pointer.append([]) else: pointer.append(BetterDict()) pointer = pointer[-1] else: pointer = pointer[part] elif isinstance(parts[index + 1], integer_types) and isinstance( pointer, dict): pointer = pointer.get(part, [], force_set=True) else: pointer = pointer.get(part, force_set=True) self.__ensure_list_capacity(pointer, parts[-1]) self.log.debug("Applying: [%s]=%s", parts[-1], value) if isinstance(parts[-1], string_types) and parts[-1][0] == '^': item = parts[-1][1:] if isinstance(pointer, list): item = int(item) if -len(pointer) <= item < len(pointer): del pointer[item] else: self.log.debug("No value to delete: %s", item) elif isinstance(pointer, dict): if item in pointer: del pointer[item] else: self.log.debug("No value to delete: %s", item) else: raise ValueError( "Cannot handle override %s in non-iterable type %s" % (item, pointer)) else: parsed_value = self.__parse_override_value(value) self.log.debug("Parsed override value: %r -> %r (%s)", value, parsed_value, type(parsed_value)) if isinstance(parsed_value, dict): parsed_value = BetterDict.from_dict(parsed_value) if isinstance(pointer, list) and parts[-1] < 0: pointer.append(parsed_value) else: pointer[parts[-1]] = parsed_value
def __init__(self): super(ScenarioExecutor, self).__init__() self.provisioning = None self.execution = BetterDict() # FIXME: why have this field if we have `parameters` from base class? self.__scenario = None self.label = None self.widget = None self.reader = None self.delay = None self.start_time = None self.env = None self.preprocess_args = lambda x: None
def __ensure_list_capacity(self, pointer, part, next_part=None): """ Extend pointer list to hold additional item :type pointer: list :type part: int """ if isinstance(pointer, list) and isinstance(part, integer_types): while len(pointer) <= part: self.log.debug("Len %s less than %s", len(pointer), part) if isinstance(next_part, integer_types): pointer.append([]) else: pointer.append(BetterDict())
def dump(self, filename=None, fmt=None): """ Dump current state of dict into file. If no filename or format specified, defaults are used :type filename: str or NoneType :type fmt: str or NoneType """ if not filename: filename = self.dump_filename if filename: if not fmt: self.dump(filename + ".yml", self.YAML) self.dump(filename + ".json", self.JSON) return acopy = copy.deepcopy(self) BetterDict.traverse(acopy, self.masq_sensitive) BetterDict.traverse(acopy, self.replace_infinities) with open(filename, "wb") as fhd: self.log.debug("Dumping %s config into %s", fmt, filename) acopy.write(fhd, fmt)
def eval_env(self): """ Should be done after `configure` """ envs = self.config.get(SETTINGS, force_set=True).get("env", force_set=True) envs[TAURUS_ARTIFACTS_DIR] = self.artifacts_dir for varname in envs: if envs[varname]: envs[varname] = str(envs[varname]) envs[varname] = os.path.expandvars(envs[varname]) for varname in envs: self.env.set({varname: envs[varname]}) if envs[varname] is None: if varname in os.environ: os.environ.pop(varname) else: os.environ[varname] = str(envs[varname]) def custom_expandvars(value): parts = re.split(r'(\$\{.*?\})', value) value = '' for item in parts: if item and item.startswith("${") and item.endswith("}"): key = item[2:-1] if key in envs: item = envs[key] if item is not None: value += text_type(item) return value def apply_env(value, key, container): if isinstance(value, string_types): container[key] = custom_expandvars(value) BetterDict.traverse(self.config, apply_env)
def get_scenario(self, name=None, cache_scenario=True): """ Returns scenario dict, extract if scenario is inlined :return: DictOfDicts """ if name is None and self.__scenario is not None: return self.__scenario scenarios = self.engine.config.get("scenarios", force_set=True) if name is None: # get current scenario exc = TaurusConfigError("Scenario is not found in execution: %s" % self.execution) label = self.execution.get('scenario', exc) is_script = isinstance(label, string_types) and label not in scenarios and \ os.path.exists(self.engine.find_file(label)) if isinstance(label, list): msg = "Invalid content of scenario, list type instead of dict or string: %s" raise TaurusConfigError(msg % label) if isinstance(label, dict) or is_script: self.log.debug("Extract %s into scenarios" % label) if isinstance(label, string_types): scenario = BetterDict.from_dict({Scenario.SCRIPT: label}) else: scenario = label path = self.get_script_path(scenario=Scenario(self.engine, scenario)) if path: label = os.path.basename(path) if not path or label in scenarios: hash_str = str(hashlib.md5(to_json(scenario).encode()).hexdigest()) label = 'autogenerated_' + hash_str[-10:] scenarios[label] = scenario self.execution['scenario'] = label self.label = label else: # get scenario by name label = name exc = TaurusConfigError("Scenario '%s' not found in scenarios: %s" % (label, scenarios.keys())) scenario = scenarios.get(label, exc) scenario_obj = Scenario(self.engine, scenario) if name is None and cache_scenario: self.__scenario = scenario_obj return scenario_obj
class ScenarioExecutor(EngineModule): """ :type provisioning: engine.Provisioning :type execution: BetterDict """ RAMP_UP = "ramp-up" HOLD_FOR = "hold-for" CONCURR = "concurrency" THRPT = "throughput" EXEC = "execution" STEPS = "steps" LOAD_FMT = namedtuple("LoadSpec", "concurrency throughput ramp_up hold iterations duration steps") def __init__(self): super(ScenarioExecutor, self).__init__() self.provisioning = None self.execution = BetterDict() # FIXME: why have this field if we have `parameters` from base class? self.__scenario = None self.label = None self.widget = None self.reader = None self.delay = None self.start_time = None self.env = None self.preprocess_args = lambda x: None def has_results(self): if self.reader and self.reader.buffer: return True else: return False def get_script_path(self, required=False, scenario=None): """ :type required: bool :type scenario: Scenario """ if scenario is None: scenario = self.get_scenario() if required: exc = TaurusConfigError("You must provide script for %s" % self) script = scenario.get(Scenario.SCRIPT, exc) else: script = scenario.get(Scenario.SCRIPT) if script: script = self.engine.find_file(script) scenario[Scenario.SCRIPT] = script return script def get_scenario(self, name=None, cache_scenario=True): """ Returns scenario dict, extract if scenario is inlined :return: DictOfDicts """ if name is None and self.__scenario is not None: return self.__scenario scenarios = self.engine.config.get("scenarios", force_set=True) if name is None: # get current scenario exc = TaurusConfigError("Scenario is not found in execution: %s" % self.execution) label = self.execution.get('scenario', exc) is_script = isinstance(label, string_types) and label not in scenarios and \ os.path.exists(self.engine.find_file(label)) if isinstance(label, list): msg = "Invalid content of scenario, list type instead of dict or string: %s" raise TaurusConfigError(msg % label) if isinstance(label, dict) or is_script: self.log.debug("Extract %s into scenarios" % label) if isinstance(label, string_types): scenario = BetterDict.from_dict({Scenario.SCRIPT: label}) else: scenario = label path = self.get_script_path(scenario=Scenario(self.engine, scenario)) if path: label = os.path.basename(path) if not path or label in scenarios: hash_str = str(hashlib.md5(to_json(scenario).encode()).hexdigest()) label = 'autogenerated_' + hash_str[-10:] scenarios[label] = scenario self.execution['scenario'] = label self.label = label else: # get scenario by name label = name exc = TaurusConfigError("Scenario '%s' not found in scenarios: %s" % (label, scenarios.keys())) scenario = scenarios.get(label, exc) scenario_obj = Scenario(self.engine, scenario) if name is None and cache_scenario: self.__scenario = scenario_obj return scenario_obj def get_load(self): """ Helper method to read load specification """ def eval_int(value): try: return int(value) except (ValueError, TypeError): return value def eval_float(value): try: return int(value) except (ValueError, TypeError): return value prov_type = self.engine.config.get(Provisioning.PROV) ensure_is_dict(self.execution, ScenarioExecutor.THRPT, prov_type) throughput = eval_float(self.execution[ScenarioExecutor.THRPT].get(prov_type, 0)) ensure_is_dict(self.execution, ScenarioExecutor.CONCURR, prov_type) concurrency = eval_int(self.execution[ScenarioExecutor.CONCURR].get(prov_type, 0)) iterations = eval_int(self.execution.get("iterations", None)) ramp_up = self.execution.get(ScenarioExecutor.RAMP_UP, None) steps = eval_int(self.execution.get(ScenarioExecutor.STEPS, None)) hold = dehumanize_time(self.execution.get(ScenarioExecutor.HOLD_FOR, 0)) if ramp_up is None: duration = hold else: ramp_up = dehumanize_time(ramp_up) duration = hold + ramp_up if duration and not iterations: iterations = 0 # which means infinite msg = '' if not isinstance(concurrency, numeric_types + (type(None),)): msg += "Invalid concurrency value[%s]: %s " % (type(concurrency).__name__, concurrency) if not isinstance(throughput, numeric_types + (type(None),)): msg += "Invalid throughput value[%s]: %s " % (type(throughput).__name__, throughput) if not isinstance(steps, numeric_types + (type(None),)): msg += "Invalid throughput value[%s]: %s " % (type(steps).__name__, steps) if not isinstance(iterations, numeric_types + (type(None),)): msg += "Invalid throughput value[%s]: %s " % (type(iterations).__name__, iterations) if msg: raise TaurusConfigError(msg) return self.LOAD_FMT(concurrency=concurrency, ramp_up=ramp_up, throughput=throughput, hold=hold, iterations=iterations, duration=duration, steps=steps) def get_resource_files(self): files_list = [] if isinstance(self, FileLister): files_list.extend(self.resource_files()) files_list.extend(self.execution.get("files", [])) return files_list def __repr__(self): return "%s/%s" % (self.execution.get("executor", None), self.label if self.label else id(self)) def execute(self, args, cwd=None, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=False): self.preprocess_args(args) return self.engine.start_subprocess(args=args, cwd=cwd, stdout=stdout, stderr=stderr, stdin=stdin, shell=shell, env=self.env)
def __init__(self): self.log = logging.getLogger('') self.engine = None self.settings = BetterDict() self.parameters = BetterDict()
class EngineModule(object): """ Base class for any BZT engine module :type engine: Engine :type settings: BetterDict """ def __init__(self): self.log = logging.getLogger('') self.engine = None self.settings = BetterDict() self.parameters = BetterDict() def prepare(self): """ Preparation stage, at which configuration is being read, configs and tools being prepared. All long preparations and checks should be made here, to make `startup` stage as fast as possible. """ pass def startup(self): """ Startup should be as fast as possible. Launch background processes, do some API calls for initiation of actual work. Consider making all checks and preparations on `prepare` stage. """ pass def check(self): """ Check if work should be finished :rtype: bool :return: True if should be finished """ return False def shutdown(self): """ Stop all processes that were started in `startup` stage. Should also be as fast as possible, deferring all long operations to `post_process` stage. """ pass def post_process(self): """ Do all possibly long analysis and processing on run results """ pass def _should_run(self): """ Returns True if provisioning matches run-at """ prov = self.engine.config.get(Provisioning.PROV) runat = self.parameters.get("run-at", None) if runat is not None and prov != runat: self.log.debug("Should not run because of non-matching prov: %s != %s", prov, runat) return False return True