def __init__(self, config, scenario, engine):
super(HierarchicHTTPRequest, self).__init__(config, scenario, engine)
self.upload_files = self.config.get("upload-files", [])
if self.method == "PUT" and len(self.upload_files) > 1:
self.upload_files = self.upload_files[:1]
for file_dict in self.upload_files:
param = file_dict.get("param", None)
if self.method == "PUT":
file_dict["param"] = ""
if self.method == "POST" and not param:
raise TaurusConfigError(
"Items from upload-files must specify parameter name")
path_exc = TaurusConfigError(
"Items from upload-files must specify path to file")
path = str(file_dict.get("path", path_exc))
if not has_variable_pattern(path): # exclude variables
path = self.engine.find_file(path) # prepare full path for jmx
else:
msg = "Path '%s' contains variable and can't be expanded. Don't use relative paths in 'upload-files'!"
self.log.warning(msg % path)
file_dict["path"] = path
mime = mimetypes.guess_type(
file_dict["path"])[0] or "application/octet-stream"
file_dict.get("mime-type", mime, force_set=True)
self.content_encoding = self.config.get('content-encoding', None)
def __load_module(self, alias):
"""
Load module class by alias
:param alias: str
:return: class
"""
if alias in self.modules:
return self.modules[alias]
mod_conf = self.config.get('modules')
if alias not in mod_conf:
msg = "Module '%s' not found in list of available aliases %s" % (alias, sorted(mod_conf.keys()))
raise TaurusConfigError(msg)
settings = ensure_is_dict(mod_conf, alias, "class")
acopy = copy.deepcopy(settings)
BetterDict.traverse(acopy, Configuration.masq_sensitive)
self.log.debug("Module config: %s %s", alias, acopy)
err = TaurusConfigError("Class name for alias '%s' is not found in module settings: %s" % (alias, settings))
clsname = settings.get('class', err)
self.modules[alias] = load_class(clsname)
if not issubclass(self.modules[alias], EngineModule):
# raise TaurusInternalException("Module class does not inherit from EngineModule: %s" % clsname)
pass
return self.modules[alias]
def get_scenario(self, name=None, cache_scenario=True):
"""
Returns scenario dict, extract if scenario is inlined
:return: DictOfDicts
"""
if name is None and self.__scenario is not None:
return self.__scenario
scenarios = self.engine.config.get("scenarios", force_set=True)
if name is None: # get current scenario
exc = TaurusConfigError("Scenario is not found in execution: %s" % self.execution)
label = self.execution.get('scenario', exc)
is_script = isinstance(label, string_types) and label not in scenarios and \
os.path.exists(self.engine.find_file(label))
if isinstance(label, list):
msg = "Invalid content of scenario, list type instead of dict or string: %s"
raise TaurusConfigError(msg % label)
if isinstance(label, dict) or is_script:
self.log.debug("Extract %s into scenarios" % label)
if isinstance(label, string_types):
scenario = BetterDict.from_dict({Scenario.SCRIPT: label})
else:
scenario = label
path = self.get_script_path(scenario=Scenario(self.engine, scenario))
if path:
label = os.path.basename(path)
if not path or label in scenarios:
hash_str = str(hashlib.md5(to_json(scenario).encode()).hexdigest())
label = 'autogenerated_' + hash_str[-10:]
scenarios[label] = scenario
self.execution['scenario'] = label
self.label = label
else: # get scenario by name
label = name
exc = TaurusConfigError("Scenario '%s' not found in scenarios: %s" % (label, scenarios.keys()))
scenario = scenarios.get(label, exc)
scenario_obj = Scenario(self.engine, scenario)
if name is None and cache_scenario:
self.__scenario = scenario_obj
return scenario_obj
def get_data_sources(self):
data_sources = self.get(self.FIELD_DATA_SOURCES, [])
if not isinstance(data_sources, list):
raise TaurusConfigError("data-sources '%s' is not a list" % data_sources)
for index, _ in enumerate(data_sources):
ensure_is_dict(data_sources, index, "path")
return self.get(self.FIELD_DATA_SOURCES, [])
def load(self, config_files, callback=None):
"""
Load and merge JSON/YAML files into current dict
:type callback: callable
:type config_files: list[str]
"""
self.log.debug("Configs: %s", config_files)
for config_file in config_files:
try:
configs = []
with codecs.open(config_file, 'r', encoding='utf-8') as fds:
if self.tab_replacement_spaces:
contents = self._replace_tabs(fds.readlines(), config_file)
else:
contents = fds.read()
self._read_yaml_or_json(config_file, configs, contents)
for config in configs:
self.merge(config)
except KeyboardInterrupt:
raise
except InvalidTaurusConfiguration:
raise
except BaseException as exc:
raise TaurusConfigError("Error when reading config file '%s': %s" % (config_file, exc))
if callback is not None:
callback(config_file)
def prepare(self):
"""
Preparation in provisioning begins with reading executions list
and instantiating ScenarioExecutor classes for them
"""
super(Provisioning, self).prepare()
esettings = self.engine.config.get(SETTINGS)
default_executor = esettings.get("default-executor", None)
exc = TaurusConfigError("No 'execution' is configured. Did you forget to pass config files?")
if ScenarioExecutor.EXEC not in self.engine.config and self.disallow_empty_execution:
# raise exc
pass
executions = self.engine.config.get(ScenarioExecutor.EXEC, [])
if not executions and self.disallow_empty_execution:
# raise exc
pass
if isinstance(executions, dict):
executions = [executions]
for execution in executions:
executor = execution.get("executor", default_executor)
if not executor:
msg = "Cannot determine executor type and no default executor in %s"
# raise TaurusConfigError(msg % execution)
pass
instance = self.engine.instantiate_module(executor)
instance.provisioning = self
instance.execution = execution
# assert isinstance(instance, ScenarioExecutor)
self.executors.append(instance)
def visit_includescenarioblock(self, block):
scenario_name = block.scenario_name
if scenario_name in self.path:
msg = "Mutual recursion detected in include-scenario blocks (scenario %s)"
raise TaurusConfigError(msg % scenario_name)
self.record_path(scenario_name)
scenario = self.executor.get_scenario(name=block.scenario_name)
return self.executor.res_files_from_scenario(scenario)
def __prepare_provisioning(self):
"""
Instantiate provisioning class
"""
err = TaurusConfigError("Please check global config availability or configure provisioning settings")
cls = self.config.get(Provisioning.PROV, err)
self.provisioning = self.instantiate_module(cls)
self.prepared.append(self.provisioning)
self.provisioning.prepare()
def get_load(self):
"""
Helper method to read load specification
"""
def eval_int(value):
try:
return int(value)
except (ValueError, TypeError):
return value
def eval_float(value):
try:
return int(value)
except (ValueError, TypeError):
return value
prov_type = self.engine.config.get(Provisioning.PROV)
ensure_is_dict(self.execution, ScenarioExecutor.THRPT, prov_type)
throughput = eval_float(self.execution[ScenarioExecutor.THRPT].get(prov_type, 0))
ensure_is_dict(self.execution, ScenarioExecutor.CONCURR, prov_type)
concurrency = eval_int(self.execution[ScenarioExecutor.CONCURR].get(prov_type, 0))
iterations = eval_int(self.execution.get("iterations", None))
ramp_up = self.execution.get(ScenarioExecutor.RAMP_UP, None)
steps = eval_int(self.execution.get(ScenarioExecutor.STEPS, None))
hold = dehumanize_time(self.execution.get(ScenarioExecutor.HOLD_FOR, 0))
if ramp_up is None:
duration = hold
else:
ramp_up = dehumanize_time(ramp_up)
duration = hold + ramp_up
if duration and not iterations:
iterations = 0 # which means infinite
msg = ''
if not isinstance(concurrency, numeric_types + (type(None),)):
msg += "Invalid concurrency value[%s]: %s " % (type(concurrency).__name__, concurrency)
if not isinstance(throughput, numeric_types + (type(None),)):
msg += "Invalid throughput value[%s]: %s " % (type(throughput).__name__, throughput)
if not isinstance(steps, numeric_types + (type(None),)):
msg += "Invalid throughput value[%s]: %s " % (type(steps).__name__, steps)
if not isinstance(iterations, numeric_types + (type(None),)):
msg += "Invalid throughput value[%s]: %s " % (type(iterations).__name__, iterations)
if msg:
raise TaurusConfigError(msg)
return self.LOAD_FMT(concurrency=concurrency, ramp_up=ramp_up, throughput=throughput, hold=hold,
iterations=iterations, duration=duration, steps=steps)
def __parse_requests(self, raw_requests, require_url=True):
requests = []
for key in range(len(raw_requests)): # pylint: disable=consider-using-enumerate
req = ensure_is_dict(raw_requests, key, "url")
if not require_url and "url" not in req:
req["url"] = None
try:
requests.append(self.__parse_request(req))
except BaseException as exc:
logging.debug("%s\n%s" % (exc, traceback.format_exc()))
raise TaurusConfigError("Wrong request:\n %s" % req)
return requests
def prepare(self):
"""
Read aggregation options
"""
super(ConsolidatingAggregator, self).prepare()
# make unique & sort
self.track_percentiles = self.settings.get("percentiles",
self.track_percentiles)
self.track_percentiles = list(set(self.track_percentiles))
self.track_percentiles.sort()
self.settings["percentiles"] = self.track_percentiles
self.ignored_labels = self.settings.get("ignore-labels",
self.ignored_labels)
self.generalize_labels = self.settings.get("generalize-labels",
self.generalize_labels)
self.min_buffer_len = dehumanize_time(
self.settings.get("min-buffer-len", self.min_buffer_len))
max_buffer_len = self.settings.get("max-buffer-len",
self.max_buffer_len)
try:
self.max_buffer_len = dehumanize_time(max_buffer_len)
except TaurusInternalException as exc:
self.log.debug("Exception in dehumanize_time(%s): %s",
max_buffer_len, exc)
raise TaurusConfigError("Wrong 'max-buffer-len' value: %s" %
max_buffer_len)
self.buffer_multiplier = self.settings.get("buffer-multiplier",
self.buffer_multiplier)
count = len(self.track_percentiles)
if count == 1:
self.buffer_scale_idx = str(float(self.track_percentiles[0]))
if count > 1:
percentile = self.settings.get("buffer-scale-choice", 0.5)
percentiles = [i / (count - 1.0) for i in range(count)]
distances = [
abs(percentile - percentiles[i]) for i in range(count)
]
index_position = distances.index(min(distances))
self.buffer_scale_idx = str(
float(self.track_percentiles[index_position]))
debug_str = 'Buffer scaling setup: percentile %s from %s selected'
self.log.debug(debug_str, self.buffer_scale_idx,
self.track_percentiles)
self.rtimes_len = self.settings.get("rtimes-len", self.rtimes_len)
self.max_error_count = self.settings.get("max-error-variety",
self.max_error_count)
def __init__(self, config, scenario, engine):
self.engine = engine
self.log = self.engine.log.getChild(self.__class__.__name__)
super(HTTPRequest, self).__init__(config, scenario)
msg = "Option 'url' is mandatory for request but not found in %s" % config
self.url = self.config.get("url", TaurusConfigError(msg))
self.label = self.config.get("label", self.url)
self.method = self.config.get("method", "GET")
if not has_variable_pattern(self.method):
self.method = self.method.upper()
# TODO: add method to join dicts/lists from scenario/request level?
self.headers = self.config.get("headers", {})
self.keepalive = self.config.get('keepalive', None)
self.timeout = self.config.get('timeout', None)
self.think_time = self.config.get('think-time', None)
self.follow_redirects = self.config.get('follow-redirects', None)
self.body = self.__get_body()
def get_script_path(self, required=False, scenario=None):
"""
:type required: bool
:type scenario: Scenario
"""
if scenario is None:
scenario = self.get_scenario()
if required:
exc = TaurusConfigError("You must provide script for %s" % self)
script = scenario.get(Scenario.SCRIPT, exc)
else:
script = scenario.get(Scenario.SCRIPT)
if script:
script = self.engine.find_file(script)
scenario[Scenario.SCRIPT] = script
return script
def __configure(self, configs):
self.log.info("Starting with configs: %s", configs)
if self.options.no_system_configs is None:
self.options.no_system_configs = False
bzt_rc = os.path.expanduser(os.path.join('~', ".bzt-rc"))
if os.path.exists(bzt_rc):
self.log.debug("Using personal config: %s" % bzt_rc)
else:
self.log.debug("Adding personal config: %s", bzt_rc)
self.log.info("No personal config found, creating one at %s",
bzt_rc)
shutil.copy(
os.path.join(get_full_path(__file__, step_up=1), 'resources',
'base-bzt-rc.yml'), bzt_rc)
merged_config = self.engine.configure(
[bzt_rc] + configs, not self.options.no_system_configs)
# apply aliases
for alias in self.options.aliases:
cli_aliases = self.engine.config.get('cli-aliases')
keys = sorted(cli_aliases.keys())
err = TaurusConfigError(
"'%s' not found in aliases. Available aliases are: %s" %
(alias, ", ".join(keys)))
self.engine.config.merge(cli_aliases.get(alias, err))
if self.options.option:
overrider = ConfigOverrider(self.log)
overrider.apply_overrides(self.options.option, self.engine.config)
if self.__is_verbose():
CLI.console_handler.setLevel(logging.DEBUG)
self.engine.create_artifacts_dir(configs, merged_config)
self.engine.default_cwd = os.getcwd()
self.engine.eval_env(
) # yacky, I don't like having it here, but how to apply it after aliases and artif dir?
def __lint_config(self):
settings = self.engine.config.get(CLI.CLI_SETTINGS).get("linter")
self.log.debug("Linting config")
self.warn_on_unfamiliar_fields = settings.get(
"warn-on-unfamiliar-fields", True)
config_copy = copy.deepcopy(self.engine.config)
ignored_warnings = settings.get("ignored-warnings", [])
self.linter = ConfigurationLinter(config_copy, ignored_warnings,
self.log)
self.linter.register_checkers()
self.linter.lint()
warnings = self.linter.get_warnings()
for warning in warnings:
self.log.warning(str(warning))
if settings.get("lint-and-exit", False):
if warnings:
raise TaurusConfigError(
"Errors were found in the configuration")
else:
raise NormalShutdown(
"Linting has finished, no errors were found")
def __prepare_reporters(self):
"""
Instantiate reporters, then prepare them in case they would like to interact
"""
reporting = self.config.get(Reporter.REP, [])
for index, reporter in enumerate(reporting):
reporter = ensure_is_dict(reporting, index, "module")
msg = "reporter 'module' field isn't recognized: %s"
cls = reporter.get('module', TaurusConfigError(msg % reporter))
instance = self.instantiate_module(cls)
instance.parameters = reporter
if self.__singletone_exists(instance, self.reporters):
continue
# assert isinstance(instance, Reporter)
self.reporters.append(instance)
for reporter in self.reporters[:]:
if not reporter.should_run():
self.reporters.remove(reporter)
# prepare reporters
for module in self.reporters:
self.prepared.append(module)
module.prepare()
def __parse_request(self, req):
if 'if' in req:
condition = req.get("if")
# TODO: apply some checks to `condition`?
then_clause = req.get(
"then",
TaurusConfigError(
"'then' clause is mandatory for 'if' blocks"))
then_requests = self.__parse_requests(then_clause)
else_clause = req.get("else", [])
else_requests = self.__parse_requests(else_clause)
return IfBlock(condition, then_requests, else_requests, req)
elif 'once' in req:
do_block = req.get(
"once",
TaurusConfigError(
"operation list is mandatory for 'once' blocks"))
do_requests = self.__parse_requests(do_block)
return OnceBlock(do_requests, req)
elif 'loop' in req:
loops = req.get("loop")
do_block = req.get(
"do",
TaurusConfigError(
"'do' option is mandatory for 'loop' blocks"))
do_requests = self.__parse_requests(do_block)
return LoopBlock(loops, do_requests, req)
elif 'while' in req:
condition = req.get("while")
do_block = req.get(
"do",
TaurusConfigError(
"'do' option is mandatory for 'while' blocks"))
do_requests = self.__parse_requests(do_block)
return WhileBlock(condition, do_requests, req)
elif 'foreach' in req:
iteration_str = req.get("foreach")
match = re.match(r'(.+) in (.+)', iteration_str)
if not match:
msg = "'foreach' value should be in format '<elementName> in <collection>' but '%s' found"
raise TaurusConfigError(msg % iteration_str)
loop_var, input_var = match.groups()
do_block = req.get(
"do",
TaurusConfigError(
"'do' field is mandatory for 'foreach' blocks"))
do_requests = self.__parse_requests(do_block)
return ForEachBlock(input_var, loop_var, do_requests, req)
elif 'transaction' in req:
name = req.get('transaction')
do_block = req.get(
'do',
TaurusConfigError(
"'do' field is mandatory for transaction blocks"))
do_requests = self.__parse_requests(do_block)
include_timers = req.get('include-timers')
return TransactionBlock(name, do_requests, include_timers, req,
self.scenario)
elif 'include-scenario' in req:
name = req.get('include-scenario')
return IncludeScenarioBlock(name, req)
elif 'action' in req:
action = req.get('action')
if action not in ('pause', 'stop', 'stop-now', 'continue'):
raise TaurusConfigError(
"Action should be either 'pause', 'stop', 'stop-now' or 'continue'"
)
target = req.get('target', 'current-thread')
if target not in ('current-thread', 'all-threads'):
msg = "Target for action should be either 'current-thread' or 'all-threads' but '%s' found"
raise TaurusConfigError(msg % target)
duration = req.get('pause-duration', None)
if duration is not None:
duration = dehumanize_time(duration)
return ActionBlock(action, target, duration, req)
elif 'set-variables' in req:
mapping = req.get('set-variables')
return SetVariables(mapping, req)
else:
return HierarchicHTTPRequest(req, self.scenario, self.engine)