def normalize_output(self):
logger.debug("Normalizing output from %s instances" % len(self.collector_instances))
instances = list()
for instance in self.collector_instances:
instances.append(instance.measurements)
aggregated = self.aggregate_results(instances)
normalized_mean_result = dict()
# calculate statistics and remove anomalies (everything outside one std from mean)
for key, value in aggregated.iteritems():
measurements = aggregated[key]
std = numbers.pstdev(measurements)
mean = numbers.get_mean(measurements)
logger.debug(
"Evaluating std for '{}': " "mean: {}, std: {}, measurements: {}".format(key, mean, std, measurements)
)
for measurement in measurements:
if numbers.is_out_of_bounds(measurement, mean, std):
aggregated[key].remove(measurement)
logger.debug(
"'{}': {} is more than one std ({}) from the mean ({}), "
"removing from measurements".format(key, measurement, std, mean)
)
normalized_mean_result[key] = numbers.get_mean(aggregated[key])
logger.debug("Normalized result: %s" % PrettyLog(normalized_mean_result))
return normalized_mean_result
def normalize_output(self):
logger.debug('Normalizing output from %s instances' %
len(self.collector_instances))
instances = list()
for instance in self.collector_instances:
instances.append(instance.measurements)
aggregated = self.aggregate_results(instances)
normalized_mean_result = dict()
# calculate statistics and remove anomalies (everything outside one std from mean)
for key, value in aggregated.iteritems():
measurements = aggregated[key]
std = numbers.pstdev(measurements)
mean = numbers.get_mean(measurements)
logger.debug('Evaluating std for \'{}\': '
'mean: {}, std: {}, measurements: {}'.format(
key, mean, std, measurements))
for measurement in measurements:
if numbers.is_out_of_bounds(measurement, mean, std):
aggregated[key].remove(measurement)
logger.debug(
'\'{}\': {} is more than one std ({}) from the mean ({}), '
'removing from measurements'.format(
key, measurement, std, mean))
normalized_mean_result[key] = numbers.get_mean(aggregated[key])
logger.debug('Normalized result: %s' %
PrettyLog(normalized_mean_result))
return normalized_mean_result
def compare_with_previous_results(self, outputs_list, end_result):
"""Compare results with previous runs"""
test_successful = True
error_log = ''
logger.debug(
'Probe: calculating previous runs statistics, will fail this run if an anomaly is found, '
'Grace factor is: {}'.format(self.GRACE_FACTOR))
logger.debug(
'================================================================================\n'
)
aggregated = self.aggregate_results(outputs_list)
# calculate statistics and fail build if an anomaly was found (everything outside one std from mean)
for key, value in aggregated.iteritems():
measurements = aggregated[key]
# skip empty results
if not measurements:
logger.debug(
"Empty aggregated results for {}, skipping!".format(
measurements))
continue
std = numbers.pstdev(measurements)
mean = numbers.get_mean(measurements)
logger.debug('Evaluating std for \'{}\': '
'mean: {}, std: {}, previous results: {}'.format(
key, "%.2f" % mean, "%.2f" % std, measurements))
if key in end_result:
measurement = end_result[key]
if numbers.is_out_of_upper_bound(measurement,
mean * self.GRACE_FACTOR,
std):
error = '--> Run failed: \'{}\': {} is more than one std ({}) from the mean ({})'.format(
key, measurement, "%.2f" % std, "%.2f" % mean)
error_log += error + '\n'
logger.error(error)
test_successful = False
logger.debug(
'================================================================================\n'
)
return test_successful, error_log
def compare_with_previous_results(self, outputs_list, end_result):
"""Compare results with previous runs"""
test_successful = True
error_log = ""
logger.debug(
"Probe: calculating previous runs statistics, will fail this run if an anomaly is found, "
"Grace factor is: {}".format(self.GRACE_FACTOR)
)
logger.debug("================================================================================\n")
aggregated = self.aggregate_results(outputs_list)
# calculate statistics and fail build if an anomaly was found (everything outside one std from mean)
for key, value in aggregated.iteritems():
measurements = aggregated[key]
# skip empty results
if not measurements:
logger.debug("Empty aggregated results for {}, skipping!".format(measurements))
continue
std = numbers.pstdev(measurements)
mean = numbers.get_mean(measurements)
logger.debug(
"Evaluating std for '{}': "
"mean: {}, std: {}, previous results: {}".format(key, "%.2f" % mean, "%.2f" % std, measurements)
)
if key in end_result:
measurement = end_result[key]
if numbers.is_out_of_upper_bound(measurement, mean * self.GRACE_FACTOR, std):
error = "--> Run failed: '{}': {} is more than one std ({}) from the mean ({})".format(
key, measurement, "%.2f" % std, "%.2f" % mean
)
error_log += error + "\n"
logger.error(error)
test_successful = False
logger.debug("================================================================================\n")
return test_successful, error_log