def _set_env(self):
props = BetterDict()
props.merge(self.settings.get('properties'))
props.merge(self.get_scenario().get("properties"))
props['gatling.core.outputDirectoryBaseName'] = self.dir_prefix
props['gatling.core.directory.resources'] = self.engine.artifacts_dir
props['gatling.core.directory.results'] = self.engine.artifacts_dir
props.merge(self._get_simulation_props())
props.merge(self._get_load_props())
props.merge(self._get_scenario_props())
for key in sorted(props.keys()):
prop = props[key]
val_tpl = "%s"
if isinstance(prop, string_types):
if not is_windows(
): # extend properties support (contained separators/quotes/etc.) on lin/mac
val_tpl = "%r"
if PY2:
prop = prop.encode(
"utf-8", 'ignore') # to convert from unicode into str
self.env.add_java_param(
{"JAVA_OPTS": ("-D%s=" + val_tpl) % (key, prop)})
self.env.set({"NO_PAUSE": "TRUE"})
self.env.add_java_param(
{"JAVA_OPTS": self.settings.get("java-opts", None)})
self.log.debug('JAVA_OPTS: "%s"', self.env.get("JAVA_OPTS"))
def execute(self, args, cwd=None, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=False, env=None):
if cwd is None:
cwd = self.engine.default_cwd
environ = BetterDict()
environ.merge(dict(os.environ))
if env is not None:
if is_windows():
# as variables in windows are case insensitive we should provide correct merging
cur_env = {name.upper(): environ[name] for name in environ}
old_keys = set(env.keys())
env = {name.upper(): env[name] for name in env}
new_keys = set(env.keys())
if old_keys != new_keys:
msg = 'Some taurus environment variables might be been lost: %s'
self.log.debug(msg, list(old_keys - new_keys))
environ = BetterDict()
environ.merge(cur_env)
environ.merge(env)
environ.merge({"TAURUS_ARTIFACTS_DIR": self.engine.artifacts_dir})
environ = {key: environ[key] for key in environ.keys() if environ[key] is not None}
self.log.debug("Executing shell from %s: %s", cwd, args)
return shell_exec(args, cwd=cwd, stdout=stdout, stderr=stderr, stdin=stdin, shell=shell, env=environ)
def execute(self, args, cwd=None, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=False, env=None):
if cwd is None:
cwd = self.engine.default_cwd
aliases = self.get_hostaliases()
hosts_file = None
if aliases:
hosts_file = self.engine.create_artifact("hostaliases", "")
with open(hosts_file, 'w') as fds:
for key, value in iteritems(aliases):
fds.write("%s %s\n" % (key, value))
environ = BetterDict()
environ.merge(dict(os.environ))
if aliases:
environ["HOSTALIASES"] = hosts_file
if env is not None:
if is_windows:
# as variables in windows are case insensitive we should provide correct merging
cur_env = {name.upper(): environ[name] for name in environ}
old_keys = set(env.keys())
env = {name.upper(): env[name] for name in env}
new_keys = set(env.keys())
if old_keys != new_keys:
msg = 'Some taurus environment variables has been lost: %s'
self.log.warning(msg, list(old_keys - new_keys))
environ = BetterDict()
environ.merge(cur_env)
environ.merge(env)
environ.merge({"TAURUS_ARTIFACTS_DIR": self.engine.artifacts_dir})
environ = {key: environ[key] for key in environ.keys() if environ[key] is not None}
return shell_exec(args, cwd=cwd, stdout=stdout, stderr=stderr, stdin=stdin, shell=shell, env=environ)
def _set_env(self):
props = BetterDict()
props.merge(self.settings.get('properties'))
props.merge(self.get_scenario().get("properties"))
props['gatling.core.outputDirectoryBaseName'] = self.dir_prefix
props['gatling.core.directory.resources'] = self.engine.artifacts_dir
props['gatling.core.directory.results'] = self.engine.artifacts_dir
props.merge(self._get_simulation_props())
props.merge(self._get_load_props())
props.merge(self._get_scenario_props())
for key in sorted(props.keys()):
prop = props[key]
val_tpl = "%s"
if isinstance(prop, string_types):
val_tpl = "%r"
self.env.add_java_param(
{"JAVA_OPTS": ("-D%s=" + val_tpl) % (key, prop)})
self.env.set({"NO_PAUSE": "TRUE"})
self.env.add_java_param(
{"JAVA_OPTS": self.settings.get("java-opts", None)})
self.log.debug('JAVA_OPTS: "%s"', self.env.get("JAVA_OPTS"))
def get_kpi_body(self, data_buffer, is_final):
# - reporting format:
# {labels: <data>, # see below
# sourceID: <id of BlazeMeterClient object>,
# [is_final: True]} # for last report
#
# - elements of 'data' are described in __get_label()
#
# - elements of 'intervals' are described in __get_interval()
# every interval contains info about response codes that were received on it.
report_items = BetterDict()
if data_buffer:
self.owner.first_ts = min(self.owner.first_ts,
data_buffer[0][DataPoint.TIMESTAMP])
self.owner.last_ts = max(self.owner.last_ts,
data_buffer[-1][DataPoint.TIMESTAMP])
# following data is received in the cumulative way
for label, kpi_set in iteritems(
data_buffer[-1][DataPoint.CUMULATIVE]):
if self.owner.extend_report:
report_item = {}
for state in kpi_set:
report_item[state] = self.__get_label(
label, kpi_set[state])
self.__add_errors(report_item[state], kpi_set[state])
else:
report_item = self.__get_label(label, kpi_set)
self.__add_errors(report_item, kpi_set) # 'Errors' tab
report_items[label] = report_item
# fill 'Timeline Report' tab with intervals data
# intervals are received in the additive way
if report_items:
for dpoint in data_buffer:
time_stamp = dpoint[DataPoint.TIMESTAMP]
for label, kpi_set in iteritems(dpoint[DataPoint.CURRENT]):
exc = TaurusInternalException(
'Cumulative KPISet is non-consistent')
report_item = report_items.get(label, exc)
if self.owner.extend_report:
for state in report_item:
if state in kpi_set:
report_item[state]['intervals'].append(
self.__get_interval(
kpi_set[state], time_stamp))
else:
report_item['intervals'].append(
self.__get_interval(kpi_set, time_stamp))
report_items = [
report_items[key] for key in sorted(report_items.keys())
] # convert dict to list
data = {"labels": report_items, "sourceID": id(self.owner)}
if is_final:
data['final'] = True
return to_json(data)
def _extract_rest_request(self, test_step):
label = test_step.get('name')
config = test_step.find('./con:config', namespaces=self.NAMESPACES)
method = config.get('method')
method_name = config.get('methodName')
method_obj = self.interface.find('.//con:method[@name="%s"]' % method_name, namespaces=self.NAMESPACES)
params = BetterDict()
if method_obj is not None:
parent = method_obj.getparent()
while parent.tag.endswith('resource'):
for param in parent.findall('./con:parameters/con:parameter', namespaces=self.NAMESPACES):
param_name = param.findtext('./con:name', namespaces=self.NAMESPACES)
param_value = param.findtext('./con:value', namespaces=self.NAMESPACES)
def_value = param.findtext('./con:default', namespaces=self.NAMESPACES)
if param_value:
params[param_name] = param_value
elif def_value:
params[param_name] = def_value
parent = parent.getparent()
url = self._calc_base_address(test_step) + config.get('resourcePath')
headers = self._extract_headers(config)
assertions = self._extract_assertions(config)
params.merge({
entry.get("key"): entry.get("value")
for entry in config.findall('./con:restRequest/con:parameters/con:entry', namespaces=self.NAMESPACES)
})
for param_name in copy.copy(list(params.keys())):
template = "{" + param_name + "}"
if template in url:
param_value = params.pop(param_name)
url = url.replace(template, param_value)
request = {"url": url, "label": label}
if method is not None and method != "GET":
request["method"] = method
if headers:
request["headers"] = headers
if assertions:
request["assert"] = assertions
body = {}
for key, value in iteritems(params):
body[key] = value
if body:
request["body"] = body
return request
def _set_env(self):
props = BetterDict()
props.merge(self.settings.get('properties'))
props.merge(self.get_scenario().get("properties"))
props['gatling.core.outputDirectoryBaseName'] = self.dir_prefix
props['gatling.core.directory.resources'] = self.engine.artifacts_dir
props['gatling.core.directory.results'] = self.engine.artifacts_dir
props.merge(self._get_simulation_props())
props.merge(self._get_load_props())
props.merge(self._get_scenario_props())
for key in sorted(props.keys()):
prop = props[key]
val_tpl = "%s"
if isinstance(prop, string_types):
if not is_windows(
): # extend properties support (contained separators/quotes/etc.) on lin/mac
val_tpl = "%r"
if PY2:
prop = prop.encode(
"utf-8", 'ignore') # to convert from unicode into str
if is_gatling2(self.tool.version) or not key.startswith(
'gatling.'): # send param through java_opts
self.env.add_java_param(
{"JAVA_OPTS": ("-D%s=" + val_tpl) % (key, prop)})
self.env.set({"NO_PAUSE": "TRUE"})
self.env.add_java_param(
{"JAVA_OPTS": self.settings.get("java-opts", None)})
self.log.debug('JAVA_OPTS: "%s"', self.env.get("JAVA_OPTS"))
if not is_gatling2(self.tool.version): # cook prop file
prop_lines = []
for key in props:
if key.startswith("gatling."):
prop_lines.append("%s = %s" % (key, props[key]))
conf_dir = self.engine.create_artifact("conf", "")
os.mkdir(conf_dir)
with open(os.path.join(conf_dir, "gatling.conf"),
'w') as conf_file:
conf_file.write('\n'.join(prop_lines))
self.env.add_path({"GATLING_CONF": conf_dir})
def execute(self,
args,
cwd=None,
stdout=PIPE,
stderr=PIPE,
stdin=PIPE,
shell=False,
env=None):
if cwd is None:
cwd = self.engine.default_cwd
aliases = self.get_hostaliases()
hosts_file = None
if aliases:
hosts_file = self.engine.create_artifact("hostaliases", "")
with open(hosts_file, 'w') as fds:
for key, value in iteritems(aliases):
fds.write("%s %s\n" % (key, value))
environ = BetterDict()
environ.merge(dict(os.environ))
if aliases:
environ["HOSTALIASES"] = hosts_file
if env is not None:
environ.merge(env)
environ.merge({"TAURUS_ARTIFACTS_DIR": self.engine.artifacts_dir})
environ = {
key: environ[key]
for key in environ.keys() if environ[key] is not None
}
return shell_exec(args,
cwd=cwd,
stdout=stdout,
stderr=stderr,
stdin=stdin,
shell=shell,
env=environ)
class ConsolidatingAggregator(EngineModule, ResultsProvider):
"""
:type underlings: list[bzt.modules.aggregator.ResultsProvider]
"""
# FIXME: it was oscillating with remote test of 100 servers
def __init__(self):
EngineModule.__init__(self)
ResultsProvider.__init__(self)
self.generalize_labels = True
self.ignored_labels = []
self.underlings = []
self.buffer = BetterDict()
self.buffer_len = 2
def prepare(self):
"""
Read aggregation options
"""
super(ConsolidatingAggregator, self).prepare()
self.track_percentiles = self.settings.get("percentiles", self.track_percentiles)
self.buffer_len = self.settings.get("buffer-seconds", self.buffer_len)
self.ignored_labels = self.settings.get("ignore-labels", self.ignored_labels)
self.generalize_labels = self.settings.get("generalize-labels", self.generalize_labels)
def add_underling(self, underling):
"""
Add source for aggregating
:type underling: ResultsProvider
"""
underling.track_percentiles = self.track_percentiles
if isinstance(underling, ResultsReader):
underling.ignored_labels = self.ignored_labels
underling.generalize_labels = self.generalize_labels
# underling.buffer_len = self.buffer_len # NOTE: is it ok for underling to have the same buffer len?
self.underlings.append(underling)
def check(self):
"""
Check if there is next aggregate data present
:rtype: bool
"""
for point in self.datapoints():
self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID])
return super(ConsolidatingAggregator, self).check()
def post_process(self):
"""
Process all remaining aggregate data
"""
super(ConsolidatingAggregator, self).post_process()
for point in self.datapoints(True):
self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID])
def _process_underlings(self, final_pass):
for underling in self.underlings:
for data in [x for x in underling.datapoints(final_pass)]:
tstamp = data[DataPoint.TIMESTAMP]
if self.buffer:
mints = min(self.buffer.keys())
if tstamp < mints:
self.log.warning("Putting datapoint %s into %s", tstamp, mints)
data[DataPoint.TIMESTAMP] = mints
tstamp = mints
self.buffer.get(tstamp, []).append(data)
def _calculate_datapoints(self, final_pass=False):
"""
Override ResultsProvider._calculate_datapoints
"""
self._process_underlings(final_pass)
self.log.debug("Consolidator buffer[%s]: %s", len(self.buffer), self.buffer.keys())
if not self.buffer:
return
timestamps = sorted(self.buffer.keys())
while timestamps and (final_pass or (timestamps[-1] >= timestamps[0] + self.buffer_len)):
tstamp = timestamps.pop(0)
self.log.debug("Merging into %s", tstamp)
points_to_consolidate = self.buffer.pop(tstamp)
point = DataPoint(tstamp, self.track_percentiles)
for subresult in points_to_consolidate:
self.log.debug("Merging %s", subresult[DataPoint.TIMESTAMP])
point.merge_point(subresult)
point.recalculate()
yield point
class ConsolidatingAggregator(Aggregator, ResultsProvider):
"""
:type underlings: list[bzt.modules.aggregator.ResultsProvider]
"""
# TODO: switch to underling-count-based completeness criteria
def __init__(self):
Aggregator.__init__(self, is_functional=False)
ResultsProvider.__init__(self)
self.generalize_labels = False
self.ignored_labels = ["ignore"]
self.underlings = []
self.buffer = BetterDict()
self.rtimes_len = 1000
def prepare(self):
"""
Read aggregation options
"""
super(ConsolidatingAggregator, self).prepare()
# make unique & sort
self.track_percentiles = self.settings.get("percentiles",
self.track_percentiles)
self.track_percentiles = list(set(self.track_percentiles))
self.track_percentiles.sort()
self.settings["percentiles"] = self.track_percentiles
self.ignored_labels = self.settings.get("ignore-labels",
self.ignored_labels)
self.generalize_labels = self.settings.get("generalize-labels",
self.generalize_labels)
self.min_buffer_len = dehumanize_time(
self.settings.get("min-buffer-len", self.min_buffer_len))
max_buffer_len = self.settings.get("max-buffer-len",
self.max_buffer_len)
try:
self.max_buffer_len = dehumanize_time(max_buffer_len)
except TaurusInternalException as exc:
self.log.debug("Exception in dehumanize_time(%s): %s",
max_buffer_len, exc)
raise TaurusConfigError("Wrong 'max-buffer-len' value: %s" %
max_buffer_len)
self.buffer_multiplier = self.settings.get("buffer-multiplier",
self.buffer_multiplier)
count = len(self.track_percentiles)
if count == 1:
self.buffer_scale_idx = str(float(self.track_percentiles[0]))
if count > 1:
percentile = self.settings.get("buffer-scale-choice", 0.5)
percentiles = [i / (count - 1.0) for i in range(count)]
distances = [
abs(percentile - percentiles[i]) for i in range(count)
]
index_position = distances.index(min(distances))
self.buffer_scale_idx = str(
float(self.track_percentiles[index_position]))
debug_str = 'Buffer scaling setup: percentile %s from %s selected'
self.log.debug(debug_str, self.buffer_scale_idx,
self.track_percentiles)
self.rtimes_len = self.settings.get("rtimes-len", self.rtimes_len)
def add_underling(self, underling):
"""
Add source for aggregating
:type underling: ResultsProvider
"""
underling.track_percentiles = self.track_percentiles
if isinstance(underling, ResultsReader):
underling.ignored_labels = self.ignored_labels
underling.generalize_labels = self.generalize_labels
underling.min_buffer_len = self.min_buffer_len
underling.max_buffer_len = self.max_buffer_len
underling.buffer_multiplier = self.buffer_multiplier
underling.buffer_scale_idx = self.buffer_scale_idx
underling.rtimes_len = self.rtimes_len
self.underlings.append(underling)
def check(self):
"""
Check if there is next aggregate data present
:rtype: bool
"""
for point in self.datapoints():
self.log.debug("Processed datapoint: %s/%s",
point[DataPoint.TIMESTAMP],
point[DataPoint.SOURCE_ID])
return super(ConsolidatingAggregator, self).check()
def post_process(self):
"""
Process all remaining aggregate data
"""
super(ConsolidatingAggregator, self).post_process()
for point in self.datapoints(True):
self.log.debug("Processed datapoint: %s/%s",
point[DataPoint.TIMESTAMP],
point[DataPoint.SOURCE_ID])
def _process_underlings(self, final_pass):
for underling in self.underlings:
for data in underling.datapoints(final_pass):
tstamp = data[DataPoint.TIMESTAMP]
if self.buffer:
mints = min(self.buffer.keys())
if tstamp < mints:
self.log.debug("Putting datapoint %s into %s", tstamp,
mints)
data[DataPoint.TIMESTAMP] = mints
tstamp = mints
self.buffer.get(tstamp, [], force_set=True).append(data)
def _calculate_datapoints(self, final_pass=False):
"""
Override ResultsProvider._calculate_datapoints
"""
self._process_underlings(final_pass)
self.log.debug("Consolidator buffer[%s]: %s", len(self.buffer),
self.buffer.keys())
if not self.buffer:
return
timestamps = sorted(self.buffer.keys())
while timestamps and (
final_pass or
(timestamps[-1] >= timestamps[0] + self.buffer_len)):
tstamp = timestamps.pop(0)
self.log.debug("Merging into %s", tstamp)
points_to_consolidate = self.buffer.pop(tstamp)
point = DataPoint(tstamp, self.track_percentiles)
for subresult in points_to_consolidate:
self.log.debug("Merging %s", subresult[DataPoint.TIMESTAMP])
point.merge_point(subresult)
point.recalculate()
yield point
class ConsolidatingAggregator(EngineModule, ResultsProvider):
"""
:type underlings: list[bzt.modules.aggregator.ResultsProvider]
"""
# TODO: switch to underling-count-based completeness criteria
def __init__(self):
EngineModule.__init__(self)
ResultsProvider.__init__(self)
self.generalize_labels = False
self.ignored_labels = []
self.underlings = []
self.buffer = BetterDict()
def prepare(self):
"""
Read aggregation options
"""
super(ConsolidatingAggregator, self).prepare()
# make unique & sort
percentiles = self.settings.get("percentiles", self.track_percentiles)
percentiles = list(set(percentiles))
percentiles.sort()
self.track_percentiles = percentiles
self.settings['percentiles'] = percentiles
self.ignored_labels = self.settings.get("ignore-labels", self.ignored_labels)
self.generalize_labels = self.settings.get("generalize-labels", self.generalize_labels)
self.min_buffer_len = dehumanize_time(self.settings.get("min-buffer-len", self.min_buffer_len))
max_buffer_len = self.settings.get("max-buffer-len", self.max_buffer_len)
try: # for max_buffer_len == float('inf')
self.max_buffer_len = dehumanize_time(max_buffer_len)
except ValueError as verr:
if str(verr).find('inf') != -1:
self.max_buffer_len = max_buffer_len
else:
raise
self.buffer_multiplier = self.settings.get("buffer-multiplier", self.buffer_multiplier)
percentile = self.settings.get("buffer-scale-choice", 0.5)
count = len(self.track_percentiles)
if count == 1:
self.buffer_scale_idx = str(float(self.track_percentiles[0]))
if count > 1:
percentiles = [i / (count - 1.0) for i in range(count)]
distances = [abs(percentile - percentiles[i]) for i in range(count)]
index_position = distances.index(min(distances))
self.buffer_scale_idx = str(float(self.track_percentiles[index_position]))
debug_str = 'Buffer scaling setup: percentile %s from %s selected'
self.log.debug(debug_str, self.buffer_scale_idx, self.track_percentiles)
def add_underling(self, underling):
"""
Add source for aggregating
:type underling: ResultsProvider
"""
underling.track_percentiles = self.track_percentiles
if isinstance(underling, ResultsReader):
underling.ignored_labels = self.ignored_labels
underling.generalize_labels = self.generalize_labels
underling.min_buffer_len = self.min_buffer_len
underling.max_buffer_len = self.max_buffer_len
underling.buffer_multiplier = self.buffer_multiplier
underling.buffer_scale_idx = self.buffer_scale_idx
self.underlings.append(underling)
def check(self):
"""
Check if there is next aggregate data present
:rtype: bool
"""
for point in self.datapoints():
self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID])
return super(ConsolidatingAggregator, self).check()
def post_process(self):
"""
Process all remaining aggregate data
"""
super(ConsolidatingAggregator, self).post_process()
for point in self.datapoints(True):
self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID])
def _process_underlings(self, final_pass):
for underling in self.underlings:
for data in underling.datapoints(final_pass):
tstamp = data[DataPoint.TIMESTAMP]
if self.buffer:
mints = min(self.buffer.keys())
if tstamp < mints:
self.log.warning("Putting datapoint %s into %s", tstamp, mints)
data[DataPoint.TIMESTAMP] = mints
tstamp = mints
self.buffer.get(tstamp, []).append(data)
def _calculate_datapoints(self, final_pass=False):
"""
Override ResultsProvider._calculate_datapoints
"""
self._process_underlings(final_pass)
self.log.debug("Consolidator buffer[%s]: %s", len(self.buffer), self.buffer.keys())
if not self.buffer:
return
timestamps = sorted(self.buffer.keys())
while timestamps and (final_pass or (timestamps[-1] >= timestamps[0] + self.buffer_len)):
tstamp = timestamps.pop(0)
self.log.debug("Merging into %s", tstamp)
points_to_consolidate = self.buffer.pop(tstamp)
point = DataPoint(tstamp, self.track_percentiles)
for subresult in points_to_consolidate:
self.log.debug("Merging %s", subresult[DataPoint.TIMESTAMP])
point.merge_point(subresult)
point.recalculate()
yield point
class ConsolidatingAggregator(EngineModule, ResultsProvider):
"""
:type underlings: list[bzt.modules.aggregator.ResultsProvider]
"""
# TODO: switch to underling-count-based completeness criteria
def __init__(self):
EngineModule.__init__(self)
ResultsProvider.__init__(self)
self.generalize_labels = False
self.ignored_labels = []
self.underlings = []
self.buffer = BetterDict()
self.buffer_len = 2
def prepare(self):
"""
Read aggregation options
"""
super(ConsolidatingAggregator, self).prepare()
self.track_percentiles = self.settings.get("percentiles",
self.track_percentiles)
self.buffer_len = self.settings.get("buffer-seconds", self.buffer_len)
self.ignored_labels = self.settings.get("ignore-labels",
self.ignored_labels)
self.generalize_labels = self.settings.get("generalize-labels",
self.generalize_labels)
def add_underling(self, underling):
"""
Add source for aggregating
:type underling: ResultsProvider
"""
underling.track_percentiles = self.track_percentiles
if isinstance(underling, ResultsReader):
underling.ignored_labels = self.ignored_labels
underling.generalize_labels = self.generalize_labels
# underling.buffer_len = self.buffer_len # NOTE: is it ok for underling to have the same buffer len?
self.underlings.append(underling)
def check(self):
"""
Check if there is next aggregate data present
:rtype: bool
"""
for point in self.datapoints():
self.log.debug("Processed datapoint: %s/%s",
point[DataPoint.TIMESTAMP],
point[DataPoint.SOURCE_ID])
return super(ConsolidatingAggregator, self).check()
def post_process(self):
"""
Process all remaining aggregate data
"""
super(ConsolidatingAggregator, self).post_process()
for point in self.datapoints(True):
self.log.debug("Processed datapoint: %s/%s",
point[DataPoint.TIMESTAMP],
point[DataPoint.SOURCE_ID])
def _process_underlings(self, final_pass):
for underling in self.underlings:
for data in [x for x in underling.datapoints(final_pass)]:
tstamp = data[DataPoint.TIMESTAMP]
if self.buffer:
mints = min(self.buffer.keys())
if tstamp < mints:
self.log.warning("Putting datapoint %s into %s",
tstamp, mints)
data[DataPoint.TIMESTAMP] = mints
tstamp = mints
self.buffer.get(tstamp, []).append(data)
def _calculate_datapoints(self, final_pass=False):
"""
Override ResultsProvider._calculate_datapoints
"""
self._process_underlings(final_pass)
self.log.debug("Consolidator buffer[%s]: %s", len(self.buffer),
self.buffer.keys())
if not self.buffer:
return
timestamps = sorted(self.buffer.keys())
while timestamps and (
final_pass or
(timestamps[-1] >= timestamps[0] + self.buffer_len)):
tstamp = timestamps.pop(0)
self.log.debug("Merging into %s", tstamp)
points_to_consolidate = self.buffer.pop(tstamp)
point = DataPoint(tstamp, self.track_percentiles)
for subresult in points_to_consolidate:
self.log.debug("Merging %s", subresult[DataPoint.TIMESTAMP])
point.merge_point(subresult)
point.recalculate()
yield point
class PipInstaller(Service):
def __init__(self, packages=None, temp_flag=True):
super(PipInstaller, self).__init__()
self.packages = packages or []
self.versions = BetterDict()
self.engine = None
self.temp = temp_flag
self.target_dir = None
self.interpreter = sys.executable
self.pip_cmd = [self.interpreter, "-m", "pip"]
def _check_pip(self):
cmdline = self.pip_cmd + ["--version"]
try:
exec_and_communicate(cmdline)
except TaurusCalledProcessError as exc:
self.log.debug(exc)
raise TaurusInternalException(
"pip module not found for interpreter %s" % self.interpreter)
def _get_installed(self):
cmdline = self.pip_cmd + ["list"]
out, _ = exec_and_communicate(cmdline)
out = out.split('\n')[2:-1]
return dict(
zip([line.split(' ')[0] for line in out],
[line.strip().split(' ')[-1] for line in out]))
def _missed(self, packages):
installed = self._get_installed()
missed = []
for package in packages:
if package not in installed or package in self.versions and installed[
package] != self.versions[package]:
missed.append(package)
return missed
def _convert_config_versions(self):
"""
extract from packages config:
packages:
- one
- two==0.0.0
- name: three
version: 0.0.0
and add to self.packages and self.versions
"""
packages_list = self.parameters.get("packages", None)
if not packages_list:
return
for package_data in packages_list:
package, version = None, None
if isinstance(package_data, dict):
package, version = package_data['name'], package_data.get(
"version", None)
elif isinstance(package_data, str):
package_params = package_data.split("==")
package, version = package_params[0], package_params[1] if len(
package_params) > 1 else None
self.packages.append(package)
if version:
self.versions[package] = version
def prepare_pip(self):
"""
pip-installer expect follow definition:
- service pip-install
temp: false # install to ~/.bzt instead of artifacts dir
packages:
- first_pkg
- second_pkg
"""
self._check_pip()
self._convert_config_versions()
if not self.packages:
return
# install into artifacts dir if temp, otherwise into .bzt
self.temp = self.settings.get("temp", self.temp)
self.temp = self.parameters.get("temp", self.temp)
self.target_dir = self.engine.temp_pythonpath if self.temp else self.engine.user_pythonpath
if not os.path.exists(self.target_dir):
os.makedirs(get_full_path(self.target_dir), exist_ok=True)
def prepare(self):
self.prepare_pip()
if not self.all_packages_installed():
self.install()
def all_packages_installed(self):
self.packages = self._missed(self.packages)
self.versions = {
package: self.versions[package]
for package in self.versions.keys() if package in self.packages
}
return False if self.packages else True
def install(self):
if not self.packages:
self.log.debug("Nothing to install")
return
cmdline = self.pip_cmd + ["install", "-t", self.target_dir]
for package in self.packages:
version = self.versions.get(package, None)
cmdline += [f"{package}=={version}"] if version else [package]
cmdline += ["--upgrade"]
self.log.debug("pip-installer cmdline: '%s'" % ' '.join(cmdline))
try:
out, err = exec_and_communicate(cmdline)
except TaurusCalledProcessError as exc:
self.log.debug(exc)
for line in exc.output.split('\n'):
if line.startswith("ERROR"):
self.log.error(" ".join(line.split(" ")[1:]))
return
if "Successfully installed" in out:
self.log.info(out.split("\n")[-2])
for err_line in err.split("\n"):
if err_line.startswith('WARNING'):
self.log.warning(" ".join(err_line.split(" ")[1:]))
if err_line.startswith('ERROR'):
self.log.error(" ".join(err_line.split(" ")[1:]))
self.log.debug("pip-installer stdout: \n%s" % out)
if err:
self.log.debug("pip-installer stderr:\n%s" % err)
def get_version(self, package):
installed = self._get_installed()
return installed[package]
def post_process(self):
# might be forbidden on win as tool still work
if self.packages and self.temp and not is_windows() and os.path.exists(
self.target_dir):
self.log.debug("remove packages: %s" % self.packages)
shutil.rmtree(
self.target_dir
) # it removes all content of directory in reality, not only self.packages