def diff(scenarios, loops, old, new): old_data = {} new_data = {} # collect data for sn in scenarios: old_data[sn] = [] new_data[sn] = [] for i in range(loops): old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i))) new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i))) # crunch data headers = ['Benchmark', 'qps'] rows = [] for sn in scenarios: mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn])) print('%s: %s=%r %s=%r mdn_diff=%r' % (sn, new, new_data[sn], old, old_data[sn], mdn_diff)) s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5) if abs(s) > 3 and mdn_diff > 0.5: rows.append([sn, '%+d%%' % s]) if rows: return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') else: return None
def diff(scenarios, loops, old, new): old_data = {} new_data = {} # collect data for sn in scenarios: old_data[sn] = [] new_data[sn] = [] for i in range(loops): old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i))) new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i))) # crunch data headers = ['Benchmark', 'qps'] rows = [] for sn in scenarios: mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn])) print(('%s: %s=%r %s=%r mdn_diff=%r' % (sn, new, new_data[sn], old, old_data[sn], mdn_diff))) s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5) if abs(s) > 3 and mdn_diff > 0.5: rows.append([sn, '%+d%%' % s]) if rows: return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') else: return None
def process(self, track, new_name, old_name): for f in sorted(track): new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue mdn_diff = abs(_median(new) - _median(old)) _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' % (f, new_name, new, old_name, old, mdn_diff)) s = bm_speedup.speedup(new, old) if abs(s) > 3 and mdn_diff > 0.5: self.final[f] = '%+d%%' % s return self.final.keys()
def process(self, track, new_name, old_name): for f in sorted(track): new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue mdn_diff = abs(_median(new) - _median(old)) _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' % (f, new_name, new, old_name, old, mdn_diff)) s = bm_speedup.speedup(new, old, 1e-5) if abs(s) > 3: if mdn_diff > 0.5 or 'trickle' in f: self.final[f] = '%+d%%' % s return self.final.keys()