def main(args):
bm_build.build('new', args.benchmarks, args.jobs, args.counters)
old = args.old
if args.diff_base:
old = 'old'
where_am_i = subprocess.check_output(
['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
subprocess.check_call(['git', 'checkout', args.diff_base])
try:
bm_build.build(old, args.benchmarks, args.jobs, args.counters)
finally:
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
jobs_list = []
jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops,
args.regex, args.counters)
jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops,
args.regex, args.counters)
# shuffle all jobs to eliminate noise from GCE CPU drift
random.shuffle(jobs_list, random.SystemRandom().random)
jobset.run(jobs_list, maxjobs=args.jobs)
diff, note, significance = bm_diff.diff(args.benchmarks, args.loops,
args.regex, args.track, old, 'new',
args.counters)
if diff:
text = '[%s] Performance differences noted:\n%s' % (
args.pr_comment_name, diff)
else:
text = '[%s] No significant performance differences' % args.pr_comment_name
if note:
text = note + '\n\n' + text
print('%s' % text)
check_on_pr.check_on_pr('Benchmark', '```\n%s\n```' % text)
check_on_pr.label_significance_on_pr('perf-change', significance)
def main(args):
bm_build.build('new', args.benchmarks, args.jobs, args.counters)
old = args.old
if args.diff_base:
old = 'old'
where_am_i = subprocess.check_output(
['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
subprocess.check_call(['git', 'checkout', args.diff_base])
try:
bm_build.build(old, args.benchmarks, args.jobs, args.counters)
finally:
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
jobs_list = []
jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops,
args.regex, args.counters)
jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops,
args.regex, args.counters)
# shuffle all jobs to eliminate noise from GCE CPU drift
random.shuffle(jobs_list, random.SystemRandom().random)
jobset.run(jobs_list, maxjobs=args.jobs)
diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex,
args.track, old, 'new', args.counters)
if diff:
text = '[%s] Performance differences noted:\n%s' % (
args.pr_comment_name, diff)
else:
text = '[%s] No significant performance differences' % args.pr_comment_name
if note:
text = note + '\n\n' + text
print('%s' % text)
check_on_pr.check_on_pr('Benchmark', '```\n%s\n```' % text)
def main(args):
build('new', args.jobs)
if args.diff_base:
where_am_i = subprocess.check_output(
['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode().strip()
subprocess.check_call(['git', 'checkout', args.diff_base])
try:
build('old', args.jobs)
finally:
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
run('new', qps_scenarios._SCENARIOS, args.loops)
run('old', qps_scenarios._SCENARIOS, args.loops)
diff_output = diff(qps_scenarios._SCENARIOS, args.loops, 'old', 'new')
if diff_output:
text = '[qps] Performance differences noted:\n%s' % diff_output
else:
text = '[qps] No significant performance differences'
print(('%s' % text))
check_on_pr.check_on_pr('QPS', '```\n%s\n```' % text)
subprocess.check_call(['git', 'checkout', args.diff_base])
try:
old = _run()
finally:
# restore the original revision (="cur")
subprocess.check_call(['git', 'checkout', where_am_i])
text = ''
if old is None:
print(cur)
for key, value in sorted(cur.items()):
text += '{}: {}\n'.format(key, value)
else:
print(cur, old)
diff_size = 0
for scenario in _SCENARIOS.keys():
for key, value in sorted(_INTERESTING.items()):
key = scenario + ': ' + key
if key in cur:
if key not in old:
text += '{}: {}\n'.format(key, cur[key])
else:
diff_size += cur[key] - old[key]
text += '{}: {} -> {}\n'.format(key, old[key], cur[key])
print("DIFF_SIZE: %f" % diff_size)
check_on_pr.label_increase_decrease_on_pr('per-call-memory', diff_size, 64)
print(text)
check_on_pr.check_on_pr('Memory Difference', '```\n%s\n```' % text)
else:
has_diff = False
for i in range(0, len(size_labels) - 1):
if abs(new_size[i] - old_size[i]) < diff_threshold:
continue
if new_size[i] > old_size[i]:
diff_sign = ' (>)'
else:
diff_sign = ' (<)'
has_diff = True
text += row_format.format('{:,}'.format(new_size[i]),
size_labels[i] + diff_sign, '{:,}'.format(
old_size[i]))
i = len(size_labels) - 1
if new_size[i] > old_size[i]:
diff_sign = ' (>)'
elif new_size[i] < old_size[i]:
diff_sign = ' (<)'
else:
diff_sign = ' (=)'
text += ('\n' if has_diff else '') + row_format.format(
'{:,}'.format(new_size[i]), size_labels[i] + diff_sign,
'{:,}'.format(old_size[i]))
if not has_diff:
text += '\n No significant differences in binary sizes\n'
text += '\n'
print text
check_on_pr.check_on_pr('Binary Size', '```\n%s\n```' % text)
assert len(new_version) == 1
cmd = 'bloaty-build/bloaty -d compileunits,symbols'
if old_version:
assert len(old_version) == 1
text += subprocess.check_output(
'%s --debug-file=%s --debug-file=%s %s.stripped -- %s.stripped' %
(cmd, new_version[0], old_version[0], new_version[0],
old_version[0]),
shell=True).decode()
sections = [
x for x in csv.reader(
subprocess.check_output('bloaty-build/bloaty --csv %s -- %s' %
(new_version[0], old_version[0]),
shell=True).decode().splitlines())
]
print(sections)
for section in sections[1:]:
diff_size += int(section[2])
else:
text += subprocess.check_output('%s %s.stripped --debug-file=%s' %
(cmd, new_version[0], new_version[0]),
shell=True).decode()
text += '\n\n'
severity = _rank_diff_bytes(diff_size)
print("SEVERITY: %d" % severity)
print(text)
check_on_pr.check_on_pr('Bloat Difference', '```\n%s\n```' % text)
check_on_pr.label_significance_on_pr('bloat', severity)
if i == len(_SIZE_LABELS) - 1:
# skip line before rendering "Total"
text += '\n'
text += _render_row(new_size[i], _SIZE_LABELS[i], '')
else:
has_diff = False
# go through all labels but "Total"
for i in range(0, len(_SIZE_LABELS) - 1):
if abs(new_size[i] - old_size[i]) >= _DIFF_THRESHOLD:
has_diff = True
diff_sign = _diff_sign(new_size[i],
old_size[i],
diff_threshold=_DIFF_THRESHOLD)
text += _render_row(new_size[i], _SIZE_LABELS[i] + diff_sign,
old_size[i])
# render the "Total"
i = len(_SIZE_LABELS) - 1
diff_sign = _diff_sign(new_size[i], old_size[i])
# skip line before rendering "Total"
text += '\n'
text += _render_row(new_size[i], _SIZE_LABELS[i] + diff_sign,
old_size[i])
if not has_diff:
text += '\n No significant differences in binary sizes\n'
text += '\n'
print(text)
check_on_pr.check_on_pr('ObjC Binary Size', '```\n%s\n```' % text)
build('old')
except subprocess.CalledProcessError, e:
subprocess.check_call(['make', 'clean'])
build('old')
finally:
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
subprocess.check_call(
'make -j%d' % args.jobs, shell=True, cwd='third_party/bloaty')
text = ''
for lib in LIBS:
text += '****************************************************************\n\n'
text += lib + '\n\n'
old_version = glob.glob('bloat_diff_old/opt/%s' % lib)
new_version = glob.glob('bloat_diff_new/opt/%s' % lib)
assert len(new_version) == 1
cmd = 'third_party/bloaty/bloaty -d compileunits,symbols'
if old_version:
assert len(old_version) == 1
text += subprocess.check_output(
'%s %s -- %s' % (cmd, new_version[0], old_version[0]), shell=True)
else:
text += subprocess.check_output(
'%s %s' % (cmd, new_version[0]), shell=True)
text += '\n\n'
print text
check_on_pr.check_on_pr('Bloat Difference', '```\n%s\n```' % text)
