def _extract_rest_request(self, test_step): label = test_step.get('name') config = test_step.find('./con:config', namespaces=self.NAMESPACES) method = config.get('method') method_name = config.get('methodName') method_obj = self.interface.find('.//con:method[@name="%s"]' % method_name, namespaces=self.NAMESPACES) params = BetterDict() if method_obj is not None: parent = method_obj.getparent() while parent.tag.endswith('resource'): for param in parent.findall('./con:parameters/con:parameter', namespaces=self.NAMESPACES): param_name = param.findtext('./con:name', namespaces=self.NAMESPACES) param_value = param.findtext('./con:value', namespaces=self.NAMESPACES) def_value = param.findtext('./con:default', namespaces=self.NAMESPACES) if param_value: params[param_name] = param_value elif def_value: params[param_name] = def_value parent = parent.getparent() url = self._calc_base_address(test_step) + config.get('resourcePath') headers = self._extract_headers(config) assertions = self._extract_assertions(config) params.merge({ entry.get("key"): entry.get("value") for entry in config.findall('./con:restRequest/con:parameters/con:entry', namespaces=self.NAMESPACES) }) for param_name in copy.copy(list(params.keys())): template = "{" + param_name + "}" if template in url: param_value = params.pop(param_name) url = url.replace(template, param_value) request = {"url": url, "label": label} if method is not None and method != "GET": request["method"] = method if headers: request["headers"] = headers if assertions: request["assert"] = assertions body = {} for key, value in iteritems(params): body[key] = value if body: request["body"] = body return request
class ConsolidatingAggregator(EngineModule, ResultsProvider): """ :type underlings: list[bzt.modules.aggregator.ResultsProvider] """ # FIXME: it was oscillating with remote test of 100 servers def __init__(self): EngineModule.__init__(self) ResultsProvider.__init__(self) self.generalize_labels = True self.ignored_labels = [] self.underlings = [] self.buffer = BetterDict() self.buffer_len = 2 def prepare(self): """ Read aggregation options """ super(ConsolidatingAggregator, self).prepare() self.track_percentiles = self.settings.get("percentiles", self.track_percentiles) self.buffer_len = self.settings.get("buffer-seconds", self.buffer_len) self.ignored_labels = self.settings.get("ignore-labels", self.ignored_labels) self.generalize_labels = self.settings.get("generalize-labels", self.generalize_labels) def add_underling(self, underling): """ Add source for aggregating :type underling: ResultsProvider """ underling.track_percentiles = self.track_percentiles if isinstance(underling, ResultsReader): underling.ignored_labels = self.ignored_labels underling.generalize_labels = self.generalize_labels # underling.buffer_len = self.buffer_len # NOTE: is it ok for underling to have the same buffer len? self.underlings.append(underling) def check(self): """ Check if there is next aggregate data present :rtype: bool """ for point in self.datapoints(): self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID]) return super(ConsolidatingAggregator, self).check() def post_process(self): """ Process all remaining aggregate data """ super(ConsolidatingAggregator, self).post_process() for point in self.datapoints(True): self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID]) def _process_underlings(self, final_pass): for underling in self.underlings: for data in [x for x in underling.datapoints(final_pass)]: tstamp = data[DataPoint.TIMESTAMP] if self.buffer: mints = min(self.buffer.keys()) if tstamp < mints: self.log.warning("Putting datapoint %s into %s", tstamp, mints) data[DataPoint.TIMESTAMP] = mints tstamp = mints self.buffer.get(tstamp, []).append(data) def _calculate_datapoints(self, final_pass=False): """ Override ResultsProvider._calculate_datapoints """ self._process_underlings(final_pass) self.log.debug("Consolidator buffer[%s]: %s", len(self.buffer), self.buffer.keys()) if not self.buffer: return timestamps = sorted(self.buffer.keys()) while timestamps and (final_pass or (timestamps[-1] >= timestamps[0] + self.buffer_len)): tstamp = timestamps.pop(0) self.log.debug("Merging into %s", tstamp) points_to_consolidate = self.buffer.pop(tstamp) point = DataPoint(tstamp, self.track_percentiles) for subresult in points_to_consolidate: self.log.debug("Merging %s", subresult[DataPoint.TIMESTAMP]) point.merge_point(subresult) point.recalculate() yield point
class ConsolidatingAggregator(Aggregator, ResultsProvider): """ :type underlings: list[bzt.modules.aggregator.ResultsProvider] """ # TODO: switch to underling-count-based completeness criteria def __init__(self): Aggregator.__init__(self, is_functional=False) ResultsProvider.__init__(self) self.generalize_labels = False self.ignored_labels = ["ignore"] self.underlings = [] self.buffer = BetterDict() self.rtimes_len = 1000 def prepare(self): """ Read aggregation options """ super(ConsolidatingAggregator, self).prepare() # make unique & sort self.track_percentiles = self.settings.get("percentiles", self.track_percentiles) self.track_percentiles = list(set(self.track_percentiles)) self.track_percentiles.sort() self.settings["percentiles"] = self.track_percentiles self.ignored_labels = self.settings.get("ignore-labels", self.ignored_labels) self.generalize_labels = self.settings.get("generalize-labels", self.generalize_labels) self.min_buffer_len = dehumanize_time( self.settings.get("min-buffer-len", self.min_buffer_len)) max_buffer_len = self.settings.get("max-buffer-len", self.max_buffer_len) try: self.max_buffer_len = dehumanize_time(max_buffer_len) except TaurusInternalException as exc: self.log.debug("Exception in dehumanize_time(%s): %s", max_buffer_len, exc) raise TaurusConfigError("Wrong 'max-buffer-len' value: %s" % max_buffer_len) self.buffer_multiplier = self.settings.get("buffer-multiplier", self.buffer_multiplier) count = len(self.track_percentiles) if count == 1: self.buffer_scale_idx = str(float(self.track_percentiles[0])) if count > 1: percentile = self.settings.get("buffer-scale-choice", 0.5) percentiles = [i / (count - 1.0) for i in range(count)] distances = [ abs(percentile - percentiles[i]) for i in range(count) ] index_position = distances.index(min(distances)) self.buffer_scale_idx = str( float(self.track_percentiles[index_position])) debug_str = 'Buffer scaling setup: percentile %s from %s selected' self.log.debug(debug_str, self.buffer_scale_idx, self.track_percentiles) self.rtimes_len = self.settings.get("rtimes-len", self.rtimes_len) def add_underling(self, underling): """ Add source for aggregating :type underling: ResultsProvider """ underling.track_percentiles = self.track_percentiles if isinstance(underling, ResultsReader): underling.ignored_labels = self.ignored_labels underling.generalize_labels = self.generalize_labels underling.min_buffer_len = self.min_buffer_len underling.max_buffer_len = self.max_buffer_len underling.buffer_multiplier = self.buffer_multiplier underling.buffer_scale_idx = self.buffer_scale_idx underling.rtimes_len = self.rtimes_len self.underlings.append(underling) def check(self): """ Check if there is next aggregate data present :rtype: bool """ for point in self.datapoints(): self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID]) return super(ConsolidatingAggregator, self).check() def post_process(self): """ Process all remaining aggregate data """ super(ConsolidatingAggregator, self).post_process() for point in self.datapoints(True): self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID]) def _process_underlings(self, final_pass): for underling in self.underlings: for data in underling.datapoints(final_pass): tstamp = data[DataPoint.TIMESTAMP] if self.buffer: mints = min(self.buffer.keys()) if tstamp < mints: self.log.debug("Putting datapoint %s into %s", tstamp, mints) data[DataPoint.TIMESTAMP] = mints tstamp = mints self.buffer.get(tstamp, [], force_set=True).append(data) def _calculate_datapoints(self, final_pass=False): """ Override ResultsProvider._calculate_datapoints """ self._process_underlings(final_pass) self.log.debug("Consolidator buffer[%s]: %s", len(self.buffer), self.buffer.keys()) if not self.buffer: return timestamps = sorted(self.buffer.keys()) while timestamps and ( final_pass or (timestamps[-1] >= timestamps[0] + self.buffer_len)): tstamp = timestamps.pop(0) self.log.debug("Merging into %s", tstamp) points_to_consolidate = self.buffer.pop(tstamp) point = DataPoint(tstamp, self.track_percentiles) for subresult in points_to_consolidate: self.log.debug("Merging %s", subresult[DataPoint.TIMESTAMP]) point.merge_point(subresult) point.recalculate() yield point
class ConsolidatingAggregator(EngineModule, ResultsProvider): """ :type underlings: list[bzt.modules.aggregator.ResultsProvider] """ # TODO: switch to underling-count-based completeness criteria def __init__(self): EngineModule.__init__(self) ResultsProvider.__init__(self) self.generalize_labels = False self.ignored_labels = [] self.underlings = [] self.buffer = BetterDict() def prepare(self): """ Read aggregation options """ super(ConsolidatingAggregator, self).prepare() # make unique & sort percentiles = self.settings.get("percentiles", self.track_percentiles) percentiles = list(set(percentiles)) percentiles.sort() self.track_percentiles = percentiles self.settings['percentiles'] = percentiles self.ignored_labels = self.settings.get("ignore-labels", self.ignored_labels) self.generalize_labels = self.settings.get("generalize-labels", self.generalize_labels) self.min_buffer_len = dehumanize_time(self.settings.get("min-buffer-len", self.min_buffer_len)) max_buffer_len = self.settings.get("max-buffer-len", self.max_buffer_len) try: # for max_buffer_len == float('inf') self.max_buffer_len = dehumanize_time(max_buffer_len) except ValueError as verr: if str(verr).find('inf') != -1: self.max_buffer_len = max_buffer_len else: raise self.buffer_multiplier = self.settings.get("buffer-multiplier", self.buffer_multiplier) percentile = self.settings.get("buffer-scale-choice", 0.5) count = len(self.track_percentiles) if count == 1: self.buffer_scale_idx = str(float(self.track_percentiles[0])) if count > 1: percentiles = [i / (count - 1.0) for i in range(count)] distances = [abs(percentile - percentiles[i]) for i in range(count)] index_position = distances.index(min(distances)) self.buffer_scale_idx = str(float(self.track_percentiles[index_position])) debug_str = 'Buffer scaling setup: percentile %s from %s selected' self.log.debug(debug_str, self.buffer_scale_idx, self.track_percentiles) def add_underling(self, underling): """ Add source for aggregating :type underling: ResultsProvider """ underling.track_percentiles = self.track_percentiles if isinstance(underling, ResultsReader): underling.ignored_labels = self.ignored_labels underling.generalize_labels = self.generalize_labels underling.min_buffer_len = self.min_buffer_len underling.max_buffer_len = self.max_buffer_len underling.buffer_multiplier = self.buffer_multiplier underling.buffer_scale_idx = self.buffer_scale_idx self.underlings.append(underling) def check(self): """ Check if there is next aggregate data present :rtype: bool """ for point in self.datapoints(): self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID]) return super(ConsolidatingAggregator, self).check() def post_process(self): """ Process all remaining aggregate data """ super(ConsolidatingAggregator, self).post_process() for point in self.datapoints(True): self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID]) def _process_underlings(self, final_pass): for underling in self.underlings: for data in underling.datapoints(final_pass): tstamp = data[DataPoint.TIMESTAMP] if self.buffer: mints = min(self.buffer.keys()) if tstamp < mints: self.log.warning("Putting datapoint %s into %s", tstamp, mints) data[DataPoint.TIMESTAMP] = mints tstamp = mints self.buffer.get(tstamp, []).append(data) def _calculate_datapoints(self, final_pass=False): """ Override ResultsProvider._calculate_datapoints """ self._process_underlings(final_pass) self.log.debug("Consolidator buffer[%s]: %s", len(self.buffer), self.buffer.keys()) if not self.buffer: return timestamps = sorted(self.buffer.keys()) while timestamps and (final_pass or (timestamps[-1] >= timestamps[0] + self.buffer_len)): tstamp = timestamps.pop(0) self.log.debug("Merging into %s", tstamp) points_to_consolidate = self.buffer.pop(tstamp) point = DataPoint(tstamp, self.track_percentiles) for subresult in points_to_consolidate: self.log.debug("Merging %s", subresult[DataPoint.TIMESTAMP]) point.merge_point(subresult) point.recalculate() yield point
class ConsolidatingAggregator(EngineModule, ResultsProvider): """ :type underlings: list[bzt.modules.aggregator.ResultsProvider] """ # TODO: switch to underling-count-based completeness criteria def __init__(self): EngineModule.__init__(self) ResultsProvider.__init__(self) self.generalize_labels = False self.ignored_labels = [] self.underlings = [] self.buffer = BetterDict() self.buffer_len = 2 def prepare(self): """ Read aggregation options """ super(ConsolidatingAggregator, self).prepare() self.track_percentiles = self.settings.get("percentiles", self.track_percentiles) self.buffer_len = self.settings.get("buffer-seconds", self.buffer_len) self.ignored_labels = self.settings.get("ignore-labels", self.ignored_labels) self.generalize_labels = self.settings.get("generalize-labels", self.generalize_labels) def add_underling(self, underling): """ Add source for aggregating :type underling: ResultsProvider """ underling.track_percentiles = self.track_percentiles if isinstance(underling, ResultsReader): underling.ignored_labels = self.ignored_labels underling.generalize_labels = self.generalize_labels # underling.buffer_len = self.buffer_len # NOTE: is it ok for underling to have the same buffer len? self.underlings.append(underling) def check(self): """ Check if there is next aggregate data present :rtype: bool """ for point in self.datapoints(): self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID]) return super(ConsolidatingAggregator, self).check() def post_process(self): """ Process all remaining aggregate data """ super(ConsolidatingAggregator, self).post_process() for point in self.datapoints(True): self.log.debug("Processed datapoint: %s/%s", point[DataPoint.TIMESTAMP], point[DataPoint.SOURCE_ID]) def _process_underlings(self, final_pass): for underling in self.underlings: for data in [x for x in underling.datapoints(final_pass)]: tstamp = data[DataPoint.TIMESTAMP] if self.buffer: mints = min(self.buffer.keys()) if tstamp < mints: self.log.warning("Putting datapoint %s into %s", tstamp, mints) data[DataPoint.TIMESTAMP] = mints tstamp = mints self.buffer.get(tstamp, []).append(data) def _calculate_datapoints(self, final_pass=False): """ Override ResultsProvider._calculate_datapoints """ self._process_underlings(final_pass) self.log.debug("Consolidator buffer[%s]: %s", len(self.buffer), self.buffer.keys()) if not self.buffer: return timestamps = sorted(self.buffer.keys()) while timestamps and ( final_pass or (timestamps[-1] >= timestamps[0] + self.buffer_len)): tstamp = timestamps.pop(0) self.log.debug("Merging into %s", tstamp) points_to_consolidate = self.buffer.pop(tstamp) point = DataPoint(tstamp, self.track_percentiles) for subresult in points_to_consolidate: self.log.debug("Merging %s", subresult[DataPoint.TIMESTAMP]) point.merge_point(subresult) point.recalculate() yield point