def encode_bins(self, p_output): p_output = json.loads(p_output) p_output['jobs'][0].pop('trim') test_list = ['read', 'write'] for test in test_list: histogram = HdrHistogram(1, 5 * 3600 * 1000, 3) clat = p_output['jobs'][0][test]['clat']['bins'] total_buckets = clat['FIO_IO_U_PLAT_NR'] grp_msb_bits = clat['FIO_IO_U_PLAT_BITS'] buckets_per_grp = clat['FIO_IO_U_PLAT_VAL'] for bucket in xrange(total_buckets): if clat[str(bucket)]: grp = bucket / buckets_per_grp subbucket = bucket % buckets_per_grp if grp == 0: val = subbucket - 1 else: base = 2 ** (grp_msb_bits + grp - 1) val = int(base + (base / buckets_per_grp) * (subbucket - 0.5)) histogram.record_value(val, clat[str(bucket)]) p_output['jobs'][0][test]['clat']['hist'] = histogram.encode() p_output['jobs'][0][test]['clat'].pop('bins') p_output['jobs'][0][test]['clat'].pop('percentile') return json.dumps(p_output)
def generate_histogram(cls, jitter_measurements, offset): histogram = HdrHistogram(cls.HISTOGRAM_MIN, cls.HISTOGRAM_MAX, cls.SIG_FIGS) for m in jitter_measurements: if (m + offset) >= 1: histogram.record_value((m + offset)) return histogram
def encode_bins(self, p_output): p_output = json.loads(p_output) p_output['jobs'][0].pop('trim') test_list = ['read', 'write'] for test in test_list: histogram = HdrHistogram(1, 5 * 3600 * 1000, 3) clat = p_output['jobs'][0][test]['clat']['bins'] total_buckets = clat['FIO_IO_U_PLAT_NR'] grp_msb_bits = clat['FIO_IO_U_PLAT_BITS'] buckets_per_grp = clat['FIO_IO_U_PLAT_VAL'] for bucket in xrange(total_buckets): if clat[str(bucket)]: grp = bucket / buckets_per_grp subbucket = bucket % buckets_per_grp if grp == 0: val = subbucket - 1 else: base = 2**(grp_msb_bits + grp - 1) val = int(base + (base / buckets_per_grp) * (subbucket - 0.5)) histogram.record_value(val, clat[str(bucket)]) p_output['jobs'][0][test]['clat']['hist'] = histogram.encode() p_output['jobs'][0][test]['clat'].pop('bins') p_output['jobs'][0][test]['clat'].pop('percentile') return json.dumps(p_output)
def consolidate_results(results): err_flag = False all_res = {'tool': 'wrk2'} total_count = len(results) if not total_count: return all_res for key in ['http_rps', 'http_total_req', 'http_sock_err', 'http_sock_timeout', 'http_throughput_kbytes']: all_res[key] = 0 for item in results: all_res[key] += item['results'].get(key, 0) all_res[key] = int(all_res[key]) if 'latency_stats' in results[0]['results']: # for item in results: # print item['results']['latency_stats'] all_res['latency_stats'] = [] histogram = HdrHistogram(1, 24 * 3600 * 1000 * 1000, 2) for item in results: if 'latency_stats' in item['results']: histogram.decode_and_add(item['results']['latency_stats']) else: err_flag = True perc_list = [50, 75, 90, 99, 99.9, 99.99, 99.999] latency_dict = histogram.get_percentile_to_value_dict(perc_list) for key, value in latency_dict.iteritems(): all_res['latency_stats'].append([key, value]) all_res['latency_stats'].sort() if err_flag: LOG.warning('Unable to find latency_stats from the result dictionary, this ' 'may indicate that the test application on VM exited abnormally.') return all_res
def __init__(self, latency_list=None): """Create a latency instance. latency_list: aggregate all latency values from list if not None """ self.min_usec = sys.maxsize self.max_usec = 0 self.avg_usec = 0 self.hdrh = None if latency_list: hdrh_list = [] for lat in latency_list: if lat.available(): self.min_usec = min(self.min_usec, lat.min_usec) self.max_usec = max(self.max_usec, lat.max_usec) self.avg_usec += lat.avg_usec if lat.hdrh_available(): hdrh_list.append(HdrHistogram.decode(lat.hdrh)) # aggregate histograms if any if hdrh_list: def add_hdrh(x, y): x.add(y) return x decoded_hdrh = reduce(add_hdrh, hdrh_list) self.hdrh = HdrHistogram.encode(decoded_hdrh).decode('utf-8') # round to nearest usec self.avg_usec = int(round(float(self.avg_usec) / len(latency_list)))
def consolidate_results(results): all_res = {'tool': 'wrk2'} total_count = len(results) if not total_count: return all_res for key in ['http_rps', 'http_total_req', 'http_sock_err', 'http_sock_timeout', 'http_throughput_kbytes']: all_res[key] = 0 for item in results: if (key in item['results']): all_res[key] += item['results'][key] all_res[key] = int(all_res[key]) if 'latency_stats' in results[0]['results']: # for item in results: # print item['results']['latency_stats'] all_res['latency_stats'] = [] histogram = HdrHistogram(1, 24 * 3600 * 1000 * 1000, 2) for item in results: histogram.decode_and_add(item['results']['latency_stats']) perc_list = [50, 75, 90, 99, 99.9, 99.99, 99.999] latency_dict = histogram.get_percentile_to_value_dict(perc_list) for key, value in latency_dict.iteritems(): all_res['latency_stats'].append([key, value]) all_res['latency_stats'].sort() return all_res
def test_mean_stddev(): # fill up a histogram with the values in the list histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) for value in VALUES_LIST: histogram.record_value(value) assert(histogram.get_mean_value() == 2000.5) assert(histogram.get_stddev() == 1000.5)
def test_scaled_highest_equiv_value(): histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) assert histogram.get_highest_equivalent_value(8180) == 8183 assert histogram.get_highest_equivalent_value(8191) == 8191 assert histogram.get_highest_equivalent_value(8193) == 8199 assert histogram.get_highest_equivalent_value(9995) == 9999 assert histogram.get_highest_equivalent_value(10007) == 10007 assert histogram.get_highest_equivalent_value(10008) == 10015
def test_scaled_highest_equiv_value(): histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) assert 8183 == histogram.get_highest_equivalent_value(8180) assert 8191 == histogram.get_highest_equivalent_value(8191) assert 8199 == histogram.get_highest_equivalent_value(8193) assert 9999 == histogram.get_highest_equivalent_value(9995) assert 10007 == histogram.get_highest_equivalent_value(10007) assert 10015 == histogram.get_highest_equivalent_value(10008)
def test_highest_equivalent_value(): histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) assert 8183 * 1024 + 1023 == histogram.get_highest_equivalent_value(8180 * 1024) assert 8191 * 1024 + 1023 == histogram.get_highest_equivalent_value(8191 * 1024) assert 8199 * 1024 + 1023 == histogram.get_highest_equivalent_value(8193 * 1024) assert 9999 * 1024 + 1023 == histogram.get_highest_equivalent_value(9995 * 1024) assert 10007 * 1024 + 1023 == histogram.get_highest_equivalent_value(10007 * 1024) assert 10015 * 1024 + 1023 == histogram.get_highest_equivalent_value(10008 * 1024)
class Histogram(object): def __init__(self, num_histograms, cores, flow_config, opts): self.histograms = [ HdrHistogram(1, 1000 * 1000, 2) for i in range(num_histograms) ] self.global_histogram = HdrHistogram(1, 1000 * 1000, 2) self.cores = cores self.flow_config = flow_config self.violations = [0 for i in range(len(flow_config))] self.dropped = [0 for i in range(len(flow_config))] self.print_values = opts.print_values if self.print_values: self.print_files = [ open(opts.output_file + '_flow' + str(flow), 'w+') for flow in range(len(flow_config)) ] def record_value(self, flow, value): self.global_histogram.record_value(value) self.histograms[flow].record_value(value) if self.flow_config[flow].get('slo'): if value > self.flow_config[flow].get('slo'): self.violations[flow] += 1 if self.print_values: self.print_files[flow].write(str(value) + '\n') def print_info(self): info = [] for i in range(len(self.histograms)): # Add the dropped requests as max time max_value = self.histograms[i].get_max_value() for j in range(self.dropped[i]): self.histograms[i].record_value(max_value) # Get the total count of received requests total_count = self.histograms[i].get_total_count() # Get the 99th latency latency = self.histograms[i].get_value_at_percentile(99) # Prepare the json for output new_value = { 'latency': latency, 'per_core_through': (1.0 * (total_count - self.dropped[i]) / self.cores), 'slo_success': 1.0 - (1.0 * self.violations[i] / total_count), 'dropped_requests': self.dropped[i] } info.append(new_value) print json.dumps(info) def drop_request(self, flow_id): self.dropped[flow_id] += 1 self.violations[flow_id] += 1
def test_dump_histogram(): samples = [ # standard Hdr test histogram 'HISTFAAAACF4nJNpmSzMwMDAzAABMJoRTM6Y1mD/ASLwN5oJAFuQBYU=', 'HISTFAAAACh4nJNpmSzMwMDAyQABzFCaEUzOmNZg/wEisL2Kaasc00ImJgCC8Qbe' ] for hdrh in samples: HdrHistogram.dump(hdrh, output=open(os.devnull, 'wb')) HdrHistogram.dump(hdrh)
def log_stats(self): stats_snapshot = self.stats.copy() tmp_stats = OrderedDict() if self.stats_snapshot_previous: for k in stats_snapshot.keys(): tmp_stats[k+'_rate'] = (float(stats_snapshot[k]) - float(self.stats_snapshot_previous[k])) / \ float(self.stats_interval) else: for k in stats_snapshot.keys(): tmp_stats[k + '_rate'] = stats_snapshot[k] self.stats_snapshot_previous = stats_snapshot # copy and reset latency histogram ih = copy.copy(self.interval_histogram) self.interval_histogram = HdrHistogram(1, 10000000, 3) latency = OrderedDict() latency['latency_max'] = ih.get_max_value() latency['latency_min'] = ih.get_min_value() latency['latency_mean'] = ih.get_mean_value() for x in [99.90, 99.00]: latency['latency_{0:.2f}'.format(x)] = ih.get_value_at_percentile( x) # copy and reset ttl histogram th = copy.copy(self.ttl_histogram) self.ttl_histogram = HdrHistogram(1, 10000000, 3) ttl = OrderedDict() ttl['ttl_max'] = th.get_max_value() ttl['ttl_min'] = th.get_min_value() ttl['ttl_mean'] = th.get_mean_value() for x in [99.90, 99.00]: ttl['ttl_{0:.2f}'.format(x)] = th.get_value_at_percentile(x) # copy and reset ttl histogram #ml = copy.copy(self.msglag_histogram) #self.msglag_histogram = HdrHistogram(1, 10000000, 3) #mlag = OrderedDict() #mlag['message_lag_max'] = ml.get_max_value() #mlag['message_lag_min'] = ml.get_min_value() #mlag['message_lag_mean'] = ml.get_mean_value() #mlag['message_lag_stddev'] = ml.get_stddev() #for x in [99.99, 99.95, 99.00, 95.00, 90.00]: # mlag['message_lag_%.2f' % x] = ml.get_value_at_percentile(x) data = OrderedDict() data.update(stats_snapshot) data.update(tmp_stats) data.update(latency) data.update(ttl) #data.update(mlag) data.update({'timestamp': format_time(datetime.utcnow())}) self._stats_logger.info("{data}", data=json.dumps(data))
def main(): args = sys.argv[1:] if args: encoded_histograms = args for hdrh in encoded_histograms: print('\nDumping histogram: ' + hdrh + '\n') HdrHistogram.dump(hdrh) else: print('\nUsage: %s [<string encoded hdr histogram>]*\n' % (sys.argv[0]))
def test_basic(): histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) expected_bucket_count = 22 if BITNESS == 64 else 21 expected_counts_len = 23552 if BITNESS == 64 else 22528 assert histogram.bucket_count == expected_bucket_count assert histogram.sub_bucket_count == 2048 assert histogram.counts_len == expected_counts_len assert histogram.unit_magnitude == 0 assert histogram.sub_bucket_half_count_magnitude == 10 assert histogram.get_count_at_sub_bucket(0, 0) == 0 assert histogram.equals(histogram)
def test_invalid_significant_figures(): try: HdrHistogram(LOWEST, HIGHEST, -1) assert False except ValueError: pass try: HdrHistogram(LOWEST, HIGHEST, 6) assert False except ValueError: pass
def __init__(self, total, concurrency): self._total = total self._concurrency = concurrency self._active = 0 self._done = 0 self._error = 0 self._condition = threading.Condition() self._histogram = HdrHistogram(1, 300000000000, 4) self._minLatency = sys.maxsize self._maxLatency = int('-inf')
def check_cod_perf(): histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, 2) fill_start_index = (20 * histogram.counts_len) // 100 fill_to_index = fill_start_index + (30 * histogram.counts_len) // 100 fill_hist_counts(histogram, fill_to_index, fill_start_index) # encode 1000 times start = datetime.datetime.now() for _ in range(1000): histogram.encode() delta = datetime.datetime.now() - start print(delta)
def check_dec_perf(): histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, 2) fill_start_index = (20 * histogram.counts_len) // 100 fill_to_index = fill_start_index + (30 * histogram.counts_len) // 100 fill_hist_counts(histogram, fill_to_index, fill_start_index) b64 = histogram.encode() # decode and add to self 1000 times start = datetime.datetime.now() for _ in range(1000): histogram.decode_and_add(b64) delta = datetime.datetime.now() - start print(delta)
def check_hist_encode(word_size, digits, expected_compressed_length, fill_start_percent, fill_count_percent): histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, digits, word_size=word_size) if fill_count_percent: fill_start_index = (fill_start_percent * histogram.counts_len) // 100 fill_to_index = fill_start_index + (fill_count_percent * histogram.counts_len) // 100 fill_hist_counts(histogram, fill_to_index, fill_start_index) b64 = histogram.encode() assert len(b64) == expected_compressed_length
def check_hist_encode(word_size, digits, expected_compressed_length, fill_start_percent, fill_count_percent): histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, digits, word_size=word_size) if fill_count_percent: fill_start_index = (fill_start_percent * histogram.counts_len) // 100 fill_to_index = fill_start_index + (fill_count_percent * histogram.counts_len) // 100 fill_hist_counts(histogram, fill_to_index, fill_start_index) b64 = histogram.encode() assert(len(b64) == expected_compressed_length)
def test_hist_codec_partial(): histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, SIGNIFICANT) partial_histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, SIGNIFICANT) # put some known numbers in the first half buckets half_count = partial_histogram.counts_len fill_hist_counts(partial_histogram, half_count) encoded = partial_histogram.encode() histogram.decode_and_add(encoded) # now verify that the partial counters are identical to the original check_hist_counts(histogram, half_count, multiplier=1) check_hist_counts(histogram, histogram.counts_len, start=half_count + 1, multiplier=0)
def __init__(self, num_histograms, cores, flow_config, opts): self.histograms = [ HdrHistogram(1, 1000 * 1000, 2) for i in range(num_histograms) ] self.global_histogram = HdrHistogram(1, 1000 * 1000, 2) self.cores = cores self.flow_config = flow_config self.violations = [0 for i in range(len(flow_config))] self.dropped = [0 for i in range(len(flow_config))] self.print_values = opts.print_values if self.print_values: self.print_files = [ open(opts.output_file + '_flow' + str(flow), 'w+') for flow in range(len(flow_config)) ]
def test_tagged_v2_log_add(): accumulated_histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) log_reader = HistogramLogReader(TAGGED_V2_LOG, accumulated_histogram) while 1: decoded_histogram = log_reader.add_next_interval_histogram() if not decoded_histogram: break
def test_basic(): histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) assert histogram.bucket_count == 22 assert histogram.sub_bucket_count == 2048 assert histogram.counts_len == 23552 assert histogram.unit_magnitude == 0 assert histogram.sub_bucket_half_count_magnitude == 10
def test_hdr_interop(): # decode and add the encoded histograms histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) corrected_histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) histogram.decode_and_add(ENCODE_SAMPLES_HDRHISTOGRAM_C[0]) corrected_histogram.decode_and_add(ENCODE_SAMPLES_HDRHISTOGRAM_C[1]) # check the percentiles. min, max values match check_percentiles(histogram, corrected_histogram)
def __call__( self, *, started_workunits: tuple[Workunit, ...], completed_workunits: tuple[Workunit, ...], finished: bool, context: StreamingWorkunitContext, ) -> None: if not self.enabled: return # Aggregate counters on completed workunits. for workunit in completed_workunits: if "counters" in workunit: for name, value in workunit["counters"].items(): self.counters[name] += value if not finished: return # Add any counters with a count of 0. for counter in context.run_tracker.counter_names: if counter not in self.counters: self.counters[counter] = 0 # Log aggregated counters. counter_lines = "\n".join( f" {name}: {count}" for name, count in sorted(self.counters.items())) logger.info(f"Counters:\n{counter_lines}") if not self.has_histogram_module: return from hdrh.histogram import HdrHistogram histograms = context.get_observation_histograms()["histograms"] if not histograms: logger.info("No observation histogram were recorded.") return logger.info("Observation histogram summaries:") for name, encoded_histogram in histograms.items(): # Note: The Python library for HDR Histogram will only decode compressed histograms # that are further encoded with base64. See # https://github.com/HdrHistogram/HdrHistogram_py/issues/29. histogram = HdrHistogram.decode( base64.b64encode(encoded_histogram)) percentile_to_vals = "\n".join( f" p{percentile}: {value}" for percentile, value in histogram. get_percentile_to_value_dict([25, 50, 75, 90, 95, 99]).items()) logger.info(f"Summary of `{name}` observation histogram:\n" f" min: {histogram.get_min_value()}\n" f" max: {histogram.get_max_value()}\n" f" mean: {histogram.get_mean_value():.3f}\n" f" std dev: {histogram.get_stddev():.3f}\n" f" total observations: {histogram.total_count}\n" f"{percentile_to_vals}")
def dump(args=None): """ Dump a list of Hdr histograms encodings args: list of strings, each string representing an Hdr encoding """ if not args: args = sys.argv[1:] res = 1 if args: encoded_histograms = args for hdrh in encoded_histograms: print('\nDumping histogram: ' + hdrh + '\n') HdrHistogram.dump(hdrh) res = 0 else: print('\nUsage: %s [<string encoded hdr histogram>]*\n' % (sys.argv[0])) return res
def record_latency(name): try: start = time.time() yield finally: elapsed = time.time() - start if name not in hists: hists[name] = HdrHistogram(1, 30 * 1000, 2) # 1ms-30sec, 2 sig figs hists[name].record_value(elapsed * 1000) # ms
def dumpHdrhLog(file, data): histogram = HdrHistogram( MIN_LATEMCY_USECS, MAX_LATENCY_USECS, LATENCY_SIGNIFICANT_DIGITS ) for d in data: histogram.record_value(d) histogram.output_percentile_distribution( open(file, "wb"), USEC_PER_SEC, TICKS_PER_HALF_DISTANCE )
def test_highest_equivalent_value(): histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) assert 8183 * 1024 + 1023 == histogram.get_highest_equivalent_value(8180 * 1024) assert 8191 * 1024 + 1023 == histogram.get_highest_equivalent_value(8191 * 1024) assert 8199 * 1024 + 1023 == histogram.get_highest_equivalent_value(8193 * 1024) assert 9999 * 1024 + 1023 == histogram.get_highest_equivalent_value(9995 * 1024) assert 10007 * 1024 + 1023 == histogram.get_highest_equivalent_value( 10007 * 1024) assert 10015 * 1024 + 1023 == histogram.get_highest_equivalent_value( 10008 * 1024)
def test_mean_stddev(): # fill up a histogram with the values in the list histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) for value in VALUES_LIST: histogram.record_value(value) assert histogram.get_mean_value() == 2000.5 assert histogram.get_stddev() == 1000.5
def test_jHiccup_v2_log(): accumulated_histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) for checklist in JHICCUP_CHECKLISTS: accumulated_histogram.reset() log_reader = HistogramLogReader(JHICCUP_V2_LOG_NAME, accumulated_histogram) histogram_count = 0 total_count = 0 target_numbers = checklist.pop('target') while 1: decoded_histogram = log_reader.get_next_interval_histogram( **checklist) if not decoded_histogram: break histogram_count += 1 total_count += decoded_histogram.get_total_count() accumulated_histogram.add(decoded_histogram) # These logs use 8 byte counters assert decoded_histogram.get_word_size() == 8 # These logs use the default 1.0 conversion ratio assert decoded_histogram.get_int_to_double_conversion_ratio( ) == 1.0 for statement in target_numbers: assert eval(statement) == target_numbers[statement] log_reader.close()
def test_jHiccup_v2_log(): accumulated_histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) for checklist in JHICCUP_CHECKLISTS: accumulated_histogram.reset() log_reader = HistogramLogReader(JHICCUP_V2_LOG_NAME, accumulated_histogram) histogram_count = 0 total_count = 0 target_numbers = checklist.pop('target') while 1: decoded_histogram = log_reader.get_next_interval_histogram(**checklist) if not decoded_histogram: break histogram_count += 1 total_count += decoded_histogram.get_total_count() accumulated_histogram.add(decoded_histogram) # These logs use 8 byte counters assert(decoded_histogram.get_word_size() == 8) for statement in target_numbers: assert(eval(statement) == target_numbers[statement]) log_reader.close()
def from_row(cls, row, prefix=""): if row is None: return None else: raw_histo = row[prefix + "jitterHistogram"] return cls( row[prefix + "measurementID"], row[prefix + "serviceSetID"], row[prefix + "minJitter"], row[prefix + "maxJitter"], row[prefix + "avgJitter"], row[prefix + "stdDevJitter"], HdrHistogram.decode(raw_histo, b64_wrap=False) if raw_histo else None, row[prefix + "jitterHistogramOffset"], row[prefix + "interval"], cls._json_loads(row[prefix + "extraJSONData"]))
def consolidate_results(results): err_flag = False all_res = {'tool': 'wrk2'} total_count = len(results) if not total_count: return all_res for key in [ 'http_rps', 'http_total_req', 'http_sock_err', 'http_sock_timeout', 'http_throughput_kbytes' ]: all_res[key] = 0 for item in results: all_res[key] += item['results'].get(key, 0) all_res[key] = int(all_res[key]) if 'latency_stats' in results[0]['results']: # for item in results: # print item['results']['latency_stats'] all_res['latency_stats'] = [] histogram = HdrHistogram(1, 24 * 3600 * 1000 * 1000, 2) for item in results: if 'latency_stats' in item['results']: histogram.decode_and_add(item['results']['latency_stats']) else: err_flag = True perc_list = [50, 75, 90, 99, 99.9, 99.99, 99.999] latency_dict = histogram.get_percentile_to_value_dict(perc_list) for key, value in latency_dict.iteritems(): all_res['latency_stats'].append([key, value]) all_res['latency_stats'].sort() if err_flag: LOG.warning( 'Unable to find latency_stats from the result dictionary, this ' 'may indicate that the test application on VM exited abnormally.' ) return all_res
def consolidate_results(results): total_count = len(results) if not total_count: return {'tool': 'fio'} all_res = {} for key in [ 'read_iops', 'read_bw', 'write_iops', 'write_bw', 'read_runtime_ms', 'write_runtime_ms', 'read_KB', 'write_KB' ]: total = 0 for item in results: total += item['results'].get(key, 0) if total: all_res[key] = int(total) all_res['tool'] = results[0]['results']['tool'] clat_list = [] # perc_list = [1, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 99, 99.5, 99.9, 99.95, 99.99] perc_list = [50, 75, 90, 99, 99.9, 99.99, 99.999] if 'read_hist' in results[0]['results']: clat_list.append('read_hist') if 'write_hist' in results[0]['results']: clat_list.append('write_hist') for clat in clat_list: all_res[clat] = [] histogram = HdrHistogram(1, 5 * 3600 * 1000, 3) for item in results: histogram.decode_and_add(item['results'][clat]) latency_dict = histogram.get_percentile_to_value_dict(perc_list) for key, value in latency_dict.iteritems(): all_res[clat].append([key, value]) all_res[clat].sort() return all_res
def test_tagged_v2_log(): histogram_count = 0 total_count = 0 accumulated_histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) accumulated_histogram_tags = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) log_reader = HistogramLogReader(TAGGED_V2_LOG, accumulated_histogram) while 1: decoded_histogram = log_reader.get_next_interval_histogram() if not decoded_histogram: break histogram_count += 1 total_count += decoded_histogram.get_total_count() if decoded_histogram.get_tag() == 'A': accumulated_histogram_tags.add(decoded_histogram) else: assert decoded_histogram.get_tag() is None accumulated_histogram.add(decoded_histogram) assert accumulated_histogram.equals(accumulated_histogram_tags) assert total_count == 32290
def check_hist_codec_b64(word_size, b64_wrap): histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, SIGNIFICANT, b64_wrap=b64_wrap, word_size=word_size) # encode with all zero counters encoded = histogram.encode() # add back same histogram histogram.decode_and_add(encoded) # counters should remain zero check_hist_counts(histogram, histogram.counts_len, multiplier=0) # fill up the histogram fill_hist_counts(histogram, histogram.counts_len) encoded = histogram.encode() histogram.decode_and_add(encoded) check_hist_counts(histogram, histogram.counts_len, multiplier=2)
def test_record_value(): histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) histogram.record_value(TEST_VALUE_LEVEL) assert(histogram.get_count_at_value(TEST_VALUE_LEVEL) == 1) assert(histogram.get_total_count() == 1)
def _decode_next_interval_histogram(self, dest_histogram, range_start_time_sec=0.0, range_end_time_sec=sys.maxint, absolute=False): '''Read the next interval histogram from the log, if interval falls within an absolute or relative time range. Timestamps are assumed to appear in order in the log file, and as such this method will return a null upon encountering a timestamp larger than range_end_time_sec. Relative time range: the range is assumed to be in seconds relative to the actual timestamp value found in each interval line in the log Absolute time range: Absolute timestamps are calculated by adding the timestamp found with the recorded interval to the [latest, optional] start time found in the log. The start time is indicated in the log with a "#[StartTime: " followed by the start time in seconds. Params: dest_histogram if None, created a new histogram, else adds the new interval histogram to it range_start_time_sec The absolute or relative start of the expected time range, in seconds. range_start_time_sec The absolute or relative end of the expected time range, in seconds. absolute Defines if the passed range is absolute or relative Return: Returns an histogram object if an interval line was found with an associated start timestamp value that falls between start_time_sec and end_time_sec, or null if no such interval line is found. Upon encountering any unexpected format errors in reading the next interval from the file, this method will return None. The histogram returned will have it's timestamp set to the absolute timestamp calculated from adding the interval's indicated timestamp value to the latest [optional] start time found in the log. Exceptions: ValueError if there is a syntax error in one of the float fields ''' while 1: line = self.input_file.readline() if not line: return None if line[0] == '#': match_res = re_start_time.match(line) if match_res: self.start_time_sec = float(match_res.group(1)) self.observed_start_time = True continue match_res = re_base_time.match(line) if match_res: self.base_time = float(match_res.group(1)) self.observed_base_time = True continue match_res = re_histogram_interval.match(line) if not match_res: # probably a legend line that starts with "\"StartTimestamp" continue # Decode: startTimestamp, intervalLength, maxTime, histogramPayload # Timestamp is expected to be in seconds log_time_stamp_in_sec = float(match_res.group(1)) interval_length_sec = float(match_res.group(2)) cpayload = match_res.group(4) if not self.observed_start_time: # No explicit start time noted. Use 1st observed time: self.start_time_sec = log_time_stamp_in_sec self.observed_start_time = True if not self.observed_base_time: # No explicit base time noted. # Deduce from 1st observed time (compared to start time): if log_time_stamp_in_sec < self.start_time_sec - (365 * 24 * 3600.0): # Criteria Note: if log timestamp is more than a year in # the past (compared to StartTime), # we assume that timestamps in the log are not absolute self.base_time_sec = self.start_time_sec else: # Timestamps are absolute self.base_time_sec = 0.0 self.observed_base_time = True absolute_start_time_stamp_sec = \ log_time_stamp_in_sec + self.base_time_sec offset_start_time_stamp_sec = \ absolute_start_time_stamp_sec - self.start_time_sec # Timestamp length is expect to be in seconds absolute_end_time_stamp_sec = \ absolute_start_time_stamp_sec + interval_length_sec if absolute: start_time_stamp_to_check_range_on = absolute_start_time_stamp_sec else: start_time_stamp_to_check_range_on = offset_start_time_stamp_sec if start_time_stamp_to_check_range_on < range_start_time_sec: continue if start_time_stamp_to_check_range_on > range_end_time_sec: return None if dest_histogram: # add the interval histogram to the destination histogram histogram = dest_histogram histogram.decode_and_add(cpayload) else: histogram = HdrHistogram.decode(cpayload) histogram.set_start_time_stamp(absolute_start_time_stamp_sec * 1000.0) histogram.set_end_time_stamp(absolute_end_time_stamp_sec * 1000.0) return histogram
def test_large_numbers(): histogram = HdrHistogram(20000000, 100000000, 17) histogram.record_value(100000000) histogram.record_value(20000000) histogram.record_value(30000000) assert(histogram.values_are_equivalent(20000000, histogram.get_value_at_percentile(50.0))) assert(histogram.values_are_equivalent(30000000, histogram.get_value_at_percentile(83.33))) assert(histogram.values_are_equivalent(100000000, histogram.get_value_at_percentile(83.34))) assert(histogram.values_are_equivalent(100000000, histogram.get_value_at_percentile(99.0)))
def test_empty_histogram(): histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) assert(histogram.get_min_value() == 0) assert(histogram.get_max_value() == 0) assert(histogram.get_mean_value() == 0) assert(histogram.get_stddev() == 0)
def load_corrected_histogram(): histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT) # record this value with a count of 10,000 histogram.record_corrected_value(1000, INTERVAL, 10000) histogram.record_corrected_value(100000000, INTERVAL) return histogram
def test_out_of_range_values(): histogram = HdrHistogram(1, 1000, 14) assert(histogram.record_value(32767)) assert(histogram.record_value(32768) is False)