def run_process():
runner = BenchmarkRunner(
benchmarks, REPO_PATH, REPO_URL, BUILD, DB_PATH, TMP_DIR, PREPARE,
always_clean=True, run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
runner.run()
generate_rst_files(runner.benchmarks, DB_PATH, RST_BASE, """LONG DESC.""")
def main(args=None):
import argparse
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=__doc__ +
"For more information, see the docstring of `BenchmarkRunner`:\n" +
BenchmarkRunner.__doc__)
parser.add_argument('--run-option',
default='eod',
help="one of {'eod', 'all', 'last', integer}")
ns = parser.parse_args(args)
run_option = ns.run_option
if run_option.isdigit():
run_option = int(run_option)
runner = BenchmarkRunner(benchmarks,
REPO_PATH,
REPO_URL,
BUILD,
DB_PATH,
TMP_DIR,
PREPARE,
run_option=run_option,
start_date=START_DATE)
runner.run()
return runner
def run_process():
runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
BUILD, DB_PATH, TMP_DIR, PREPARE,
always_clean=True,
run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
runner.run()
def run_process(run_option='eod'):
runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
BUILD, DB_PATH, TMP_DIR, PREPARE,
always_clean=True,
run_option=run_option, start_date=START_DATE,
module_dependencies=dependencies)
runner.run()
def run_process():
runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
BUILD, DB_PATH, TMP_DIR, PREPARE,
clean_cmd=PREPARE,
run_option='eod', run_order='multires',
start_date=START_DATE,
module_dependencies=dependencies,
verify=True)
runner.run()
def run_process(existing='min', run_order='multires', run_limit=None):
runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
BUILD, DB_PATH, TMP_DIR, PREPARE,
branches=BRANCHES,
clean_cmd=PREPARE,
run_option='all', run_order=run_order, run_limit=run_limit,
start_date=START_DATE,
existing=existing,
module_dependencies=dependencies,
verify=True)
runner.run()
def run_process(existing='min', run_order='multires', run_limit=None):
runner = BenchmarkRunner(suite.benchmarks,
suite.REPO_PATH,
suite.REPO_URL,
suite.BUILD,
suite.DB_PATH,
suite.TMP_DIR,
suite.PREPARE,
branches=suite.BRANCHES,
clean_cmd=suite.PREPARE,
run_option='all',
run_order=run_order,
run_limit=run_limit,
start_date=suite.START_DATE,
existing=existing,
module_dependencies=suite.dependencies,
verify=True)
runner.run()
def run_process(existing='min', run_order='multires', run_limit=None):
runner = BenchmarkRunner(suite.benchmarks,
suite.REPO_PATH,
suite.REPO_URL,
suite.BUILD,
suite.DB_PATH,
suite.TMP_DIR,
suite.PREPARE,
branches=suite.BRANCHES,
clean_cmd=suite.PREPARE,
run_option='all',
run_order=run_order,
run_limit=run_limit,
start_date=suite.START_DATE,
existing=existing,
module_dependencies=suite.dependencies,
verify=True)
runner.run()
def main(args=None):
import argparse
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=__doc__
+ "For more information, see the docstring of `BenchmarkRunner`:\n"
+ BenchmarkRunner.__doc__)
parser.add_argument(
'--run-option', default='eod',
help="one of {'eod', 'all', 'last', integer}")
ns = parser.parse_args(args)
run_option = ns.run_option
if run_option.isdigit():
run_option = int(run_option)
runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
BUILD, DB_PATH, TMP_DIR, PREPARE,
run_option=run_option, start_date=START_DATE)
runner.run()
return runner
def main():
import shutil
import tempfile
import warnings
from pandas import Series
from vbench.api import BenchmarkRunner
from suite import (REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies,
benchmarks)
from memory_profiler import memory_usage
warnings.filterwarnings('ignore', category=FutureWarning)
try:
TMP_DIR = tempfile.mkdtemp()
runner = BenchmarkRunner(
benchmarks,
REPO_PATH,
REPO_PATH,
BUILD,
DB_PATH,
TMP_DIR,
PREPARE,
always_clean=True,
# run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
results = {}
for b in runner.benchmarks:
k = b.name
try:
vs = memory_usage((b.run, ))
v = max(vs)
# print(k, v)
results[k] = v
except Exception as e:
print("Exception caught in %s\n" % k)
print(str(e))
s = Series(results)
s.sort()
print(s)
finally:
shutil.rmtree(TMP_DIR)
def test_benchmarkrunner():
from suite import *
# Just to make sure there are no left-overs
shutil.rmtree(TMP_DIR)
if exists(DB_PATH):
os.unlink(DB_PATH)
ok_(not exists(DB_PATH))
runner = BenchmarkRunner(benchmarks,
REPO_PATH,
REPO_URL,
BUILD,
DB_PATH,
TMP_DIR,
PREPARE,
clean_cmd=CLEAN,
run_option='all',
run_order='normal',
start_date=START_DATE,
module_dependencies=DEPENDENCIES)
revisions_to_run = runner._get_revisions_to_run()
eq_(len(revisions_to_run), 6) # we have 6 now
revisions_ran = runner.run()
# print "D1: ", revisions_ran
# for this test we should inject our "failed to build revision"
# Since no tests were ran for it -- it is not reported
revisions_ran_ = [x[0] for x in revisions_ran]
revisions_ran_.insert(4, 'e83ffa5')
assert_array_equal(revisions_ran_, revisions_to_run)
# First revision
eq_(revisions_ran[0][1],
(False, 3)) # no functions were available at that point
eq_(revisions_ran[1][1],
(True, 3)) # all 3 tests were available in the first rev
ok_(exists(TMP_DIR))
ok_(exists(DB_PATH))
eq_(runner.blacklist, set(['e83ffa5'])) # one which failed to build
# Run 2nd time and verify that all are still listed BUT none new succeeds
revisions_ran = runner.run()
#print "D2: ", revisions_ran
for rev, v in revisions_ran:
eq_(v, (False, 0))
# What if we expand list of benchmarks and run 3rd time
runner.benchmarks = collect_benchmarks(['vb_sins', 'vb_sins2'])
revisions_ran = runner.run()
# for that single added benchmark there still were no function
eq_(revisions_ran[0][1], (False, 1))
# all others should have "succeeded" on that single one
for rev, v in revisions_ran[1:]:
eq_(v, (True, 1))
# and on 4th run -- nothing new
revisions_ran = runner.run()
for rev, v in revisions_ran:
eq_(v, (False, 0))
# Now let's smoke test generation of the .rst files
from vbench.reports import generate_rst_files
rstdir = pjoin(TMP_DIR, 'sources')
generate_rst_files(runner.benchmarks, DB_PATH, rstdir,
"""VERY LONG DESCRIPTION""")
# Verify that it all looks close to the desired
image_files = [
basename(x) for x in glob(pjoin(rstdir, 'vbench/figures/*.png'))
]
target_image_files = [b.name + '.png' for b in runner.benchmarks]
eq_(set(image_files), set(target_image_files))
rst_files = [basename(x) for x in glob(pjoin(rstdir, 'vbench/*.rst'))]
target_rst_files = [b.name + '.rst' for b in runner.benchmarks]
eq_(set(rst_files), set(target_rst_files))
module_files = [basename(x) for x in glob(pjoin(rstdir, '*.rst'))]
target_module_files = list(
set(['vb_' + b.module_name + '.rst' for b in runner.benchmarks]))
eq_(set(module_files), set(target_module_files + ['index.rst']))
#print TMP_DIR
shutil.rmtree(TMP_DIR)
shutil.rmtree(dirname(DB_PATH))
def run_process():
runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
'', DB_PATH, TMP_DIR, '',
run_option='all', start_date=START_DATE,
module_dependencies=[])
runner.run()
def profile_comparative(benchmarks):
from vbench.api import BenchmarkRunner
from vbench.db import BenchmarkDB
from vbench.git import GitRepo
from suite import BUILD, DB_PATH, PREPARE, dependencies
TMP_DIR = tempfile.mkdtemp()
try:
prprint("Opening DB at '%s'...\n" % DB_PATH)
db = BenchmarkDB(DB_PATH)
prprint("Initializing Runner...")
# all in a good cause...
GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)
runner = BenchmarkRunner(
benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
TMP_DIR, PREPARE, always_clean=True,
# run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
repo = runner.repo # (steal the parsed git repo used by runner)
h_head = args.target_commit or repo.shas[-1]
h_baseline = args.base_commit
# ARGH. reparse the repo, without discarding any commits,
# then overwrite the previous parse results
# prprint("Slaughtering kittens...")
(repo.shas, repo.messages,
repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
args.base_commit)
prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
prprint('Baseline [%s] : %s\n' % (h_baseline,
repo.messages.get(h_baseline, "")))
prprint("Removing any previous measurements for the commits.")
db.delete_rev_results(h_baseline)
db.delete_rev_results(h_head)
# TODO: we could skip this, but we need to make sure all
# results are in the DB, which is a little tricky with
# start dates and so on.
prprint("Running benchmarks for baseline [%s]" % h_baseline)
runner._run_and_write_results(h_baseline)
prprint("Running benchmarks for target [%s]" % h_head)
runner._run_and_write_results(h_head)
prprint('Processing results...')
head_res = get_results_df(db, h_head)
baseline_res = get_results_df(db, h_baseline)
report_comparative(head_res,baseline_res)
finally:
# print("Disposing of TMP_DIR: %s" % TMP_DIR)
shutil.rmtree(TMP_DIR)
def main():
TMP_DIR = tempfile.mkdtemp()
prprint("TMP_DIR = %s" % TMP_DIR)
prprint("LOG_FILE = %s\n" % LOG_FILE)
try:
logfile = open(LOG_FILE, 'w')
prprint("Processing Repo at '%s'..." % REPO_PATH)
repo = GitRepo(REPO_PATH)
# get hashes of baseline and current head
h_head = repo.shas[-1]
h_baseline = BASELINE_COMMIT
prprint("Opening DB at '%s'...\n" % DB_PATH)
db = BenchmarkDB(DB_PATH)
prprint('Comparing Head [%s] : %s ' %
(h_head, repo.messages.get(h_head, "")))
prprint('Against baseline [%s] : %s \n' %
(h_baseline, repo.messages.get(h_baseline, "")))
prprint("Initializing Runner...")
runner = BenchmarkRunner(
benchmarks,
REPO_PATH,
REPO_PATH,
BUILD,
DB_PATH,
TMP_DIR,
PREPARE,
always_clean=True,
# run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
prprint("removing any previous measurements for the commits.")
db.delete_rev_results(h_baseline)
db.delete_rev_results(h_head)
# TODO: we could skip this, but we need to make sure all
# results are in the DB, which is a little tricky with
# start dates and so on.
prprint("Running benchmarks for baseline commit '%s'" % h_baseline)
runner._run_and_write_results(h_baseline)
prprint("Running benchmarks for current HEAD '%s'" % h_head)
runner._run_and_write_results(h_head)
prprint('Processing results...')
head_res = get_results_df(db, h_head)
baseline_res = get_results_df(db, h_baseline)
ratio = head_res['timing'] / baseline_res['timing']
totals = DataFrame(dict(t_head=head_res['timing'],
t_baseline=baseline_res['timing'],
ratio=ratio,
name=baseline_res.name),
columns=["t_head", "t_baseline", "ratio", "name"])
totals = totals.ix[totals.t_head > 0.010] # ignore sub 10micros
totals = totals.dropna().sort("ratio").set_index(
'name') # sort in ascending order
s = "\n\nResults:\n" + totals.to_string(
float_format=lambda x: "%0.4f" % x) + "\n\n"
s += "Columns: test_name | head_time [ms] | baseline_time [ms] | ratio\n\n"
s += "- a Ratio of 1.30 means HEAD is 30% slower then the Baseline.\n\n"
s += 'Head [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
s += 'Baseline [%s] : %s\n\n' % (h_baseline,
repo.messages.get(h_baseline, ""))
logfile.write(s)
logfile.close()
prprint(s)
prprint("Results were also written to the logfile at '%s'\n" %
LOG_FILE)
finally:
# print("Disposing of TMP_DIR: %s" % TMP_DIR)
shutil.rmtree(TMP_DIR)
logfile.close()
def profile_comparative(benchmarks):
from vbench.api import BenchmarkRunner
from vbench.db import BenchmarkDB
from vbench.git import GitRepo
from suite import BUILD, DB_PATH, PREPARE, dependencies
TMP_DIR = tempfile.mkdtemp()
try:
prprint("Opening DB at '%s'...\n" % DB_PATH)
db = BenchmarkDB(DB_PATH)
prprint("Initializing Runner...")
# all in a good cause...
GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)
runner = BenchmarkRunner(
benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
TMP_DIR, PREPARE, always_clean=True,
# run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
repo = runner.repo # (steal the parsed git repo used by runner)
h_head = args.target_commit or repo.shas[-1]
h_baseline = args.base_commit
# ARGH. reparse the repo, without discarding any commits,
# then overwrite the previous parse results
# prprint ("Slaughtering kittens..." )
(repo.shas, repo.messages,
repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
args.base_commit)
prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
prprint('Baseline [%s] : %s\n' % (h_baseline,
repo.messages.get(h_baseline, "")))
prprint("Removing any previous measurements for the commits.")
db.delete_rev_results(h_baseline)
db.delete_rev_results(h_head)
# TODO: we could skip this, but we need to make sure all
# results are in the DB, which is a little tricky with
# start dates and so on.
prprint("Running benchmarks for baseline [%s]" % h_baseline)
runner._run_and_write_results(h_baseline)
prprint("Running benchmarks for target [%s]" % h_head)
runner._run_and_write_results(h_head)
prprint('Processing results...')
head_res = get_results_df(db, h_head)
baseline_res = get_results_df(db, h_baseline)
ratio = head_res['timing'] / baseline_res['timing']
totals = DataFrame({HEAD_COL:head_res['timing'],
BASE_COL:baseline_res['timing'],
'ratio':ratio,
'name':baseline_res.name},
columns=[HEAD_COL, BASE_COL, "ratio", "name"])
totals = totals.ix[totals[HEAD_COL] > args.min_duration]
# ignore below threshold
totals = totals.dropna(
).sort("ratio").set_index('name') # sort in ascending order
h_msg = repo.messages.get(h_head, "")
b_msg = repo.messages.get(h_baseline, "")
print_report(totals,h_head=h_head,h_msg=h_msg,
h_baseline=h_baseline,b_msg=b_msg)
if args.outdf:
prprint("The results DataFrame was written to '%s'\n" % args.outdf)
totals.save(args.outdf)
finally:
# print("Disposing of TMP_DIR: %s" % TMP_DIR)
shutil.rmtree(TMP_DIR)
def run_process(run_option):
runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
BUILD, DB_PATH, TMP_DIR, PREPARE,
run_option=run_option, start_date=START_DATE,
module_dependencies=dependencies)
runner.run()
def main():
TMP_DIR = tempfile.mkdtemp()
prprint("TMP_DIR = %s" % TMP_DIR)
prprint("LOG_FILE = %s\n" % LOG_FILE)
try:
logfile = open(LOG_FILE, 'w')
prprint( "Processing Repo at '%s'..." % REPO_PATH)
repo = GitRepo(REPO_PATH)
# get hashes of baseline and current head
h_head = repo.shas[-1]
h_baseline = BASELINE_COMMIT
prprint( "Opening DB at '%s'...\n" % DB_PATH)
db = BenchmarkDB(DB_PATH)
prprint( 'Comparing Head [%s] : %s ' % (h_head, repo.messages.get(h_head,"")))
prprint( 'Against baseline [%s] : %s \n' % (h_baseline,
repo.messages.get(h_baseline,"")))
prprint("Initializing Runner...")
runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
TMP_DIR, PREPARE, always_clean=True,
# run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
prprint ("removing any previous measurements for the commits." )
db.delete_rev_results(h_baseline)
db.delete_rev_results(h_head)
# TODO: we could skip this, but we need to make sure all
# results are in the DB, which is a little tricky with
# start dates and so on.
prprint( "Running benchmarks for baseline commit '%s'" % h_baseline)
runner._run_and_write_results(h_baseline)
prprint ("Running benchmarks for current HEAD '%s'" % h_head)
runner._run_and_write_results(h_head)
prprint( 'Processing results...')
head_res = get_results_df(db,h_head)
baseline_res = get_results_df(db,h_baseline)
ratio = head_res['timing']/baseline_res['timing']
totals = DataFrame(dict(t_head=head_res['timing'],
t_baseline=baseline_res['timing'],
ratio=ratio,
name=baseline_res.name),columns=["t_head","t_baseline","ratio","name"])
totals = totals.ix[totals.t_head > 0.010] # ignore sub 10micros
totals = totals.dropna().sort("ratio").set_index('name') # sort in ascending order
s = "\n\nResults:\n" + totals.to_string(float_format=lambda x: "%0.4f" %x) + "\n\n"
s += "Columns: test_name | head_time [ms] | baseline_time [ms] | ratio\n\n"
s += "- a Ratio of 1.30 means HEAD is 30% slower then the Baseline.\n\n"
s += 'Head [%s] : %s\n' % (h_head, repo.messages.get(h_head,""))
s += 'Baseline [%s] : %s\n\n' % (h_baseline,repo.messages.get(h_baseline,""))
logfile.write(s)
logfile.close()
prprint(s )
prprint("Results were also written to the logfile at '%s'\n" % LOG_FILE)
finally:
# print("Disposing of TMP_DIR: %s" % TMP_DIR)
shutil.rmtree(TMP_DIR)
logfile.close()
def main():
from pandas import DataFrame
from vbench.api import BenchmarkRunner
from vbench.db import BenchmarkDB
from vbench.git import GitRepo
from suite import REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies, benchmarks
# GitRepo wants exactly 7 character hash?
args.base_commit = args.base_commit[:7]
if args.target_commit:
args.target_commit = args.target_commit[:7]
if not args.log_file:
args.log_file = os.path.abspath(
os.path.join(REPO_PATH, 'vb_suite.log'))
random.seed(args.seed)
np.random.seed(args.seed)
TMP_DIR = tempfile.mkdtemp()
prprint("TMP_DIR = %s" % TMP_DIR)
prprint("LOG_FILE = %s\n" % args.log_file)
benchmarks = [x for x in benchmarks if re.search(args.regex,x.name)]
try:
logfile = open(args.log_file, 'w')
prprint("Opening DB at '%s'...\n" % DB_PATH)
db = BenchmarkDB(DB_PATH)
prprint("Initializing Runner...")
# all in a good cause...
GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)
runner = BenchmarkRunner(
benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
TMP_DIR, PREPARE, always_clean=True,
# run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
repo = runner.repo # (steal the parsed git repo used by runner)
# ARGH. reparse the repo, without discarding any commits,
# then overwrite the previous parse results
# prprint ("Slaughtering kittens..." )
(repo.shas, repo.messages,
repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
args.base_commit)
h_head = args.target_commit or repo.shas[-1]
h_baseline = args.base_commit
prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
prprint('Baseline [%s] : %s\n' % (h_baseline,
repo.messages.get(h_baseline, "")))
prprint("removing any previous measurements for the commits.")
db.delete_rev_results(h_baseline)
db.delete_rev_results(h_head)
# TODO: we could skip this, but we need to make sure all
# results are in the DB, which is a little tricky with
# start dates and so on.
prprint("Running benchmarks for baseline [%s]" % h_baseline)
runner._run_and_write_results(h_baseline)
prprint("Running benchmarks for target [%s]" % h_head)
runner._run_and_write_results(h_head)
prprint('Processing results...')
head_res = get_results_df(db, h_head)
baseline_res = get_results_df(db, h_baseline)
ratio = head_res['timing'] / baseline_res['timing']
totals = DataFrame(dict(t_head=head_res['timing'],
t_baseline=baseline_res['timing'],
ratio=ratio,
name=baseline_res.name), columns=["t_head", "t_baseline", "ratio", "name"])
totals = totals.ix[totals.t_head > args.min_duration]
# ignore below threshold
totals = totals.dropna(
).sort("ratio").set_index('name') # sort in ascending order
hdr = ftr = """
-----------------------------------------------------------------------
Test name | target[ms] | base[ms] | ratio |
-----------------------------------------------------------------------
""".strip() +"\n"
s = "\n"
s += hdr
for i in range(len(totals)):
t,b,r = totals.irow(i).values
s += "{0:30s} {1: 12.4f} {2: 12.4f} {3: 12.4f}\n".format(totals.index[i],t,b,r)
s+= ftr + "\n"
s += "Ratio < 1.0 means the target commit is faster then the baseline.\n"
s += "Seed used: %d\n\n" % args.seed
s += 'Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
s += 'Base [%s] : %s\n\n' % (
h_baseline, repo.messages.get(h_baseline, ""))
logfile.write(s)
logfile.close()
prprint(s)
prprint("Results were also written to the logfile at '%s'\n" %
args.log_file)
finally:
# print("Disposing of TMP_DIR: %s" % TMP_DIR)
shutil.rmtree(TMP_DIR)
logfile.close()
def profile_comparative(benchmarks):
from vbench.api import BenchmarkRunner
from vbench.db import BenchmarkDB
from vbench.git import GitRepo
from suite import BUILD, DB_PATH, PREPARE, dependencies
TMP_DIR = tempfile.mkdtemp()
try:
prprint("Opening DB at '%s'...\n" % DB_PATH)
db = BenchmarkDB(DB_PATH)
prprint("Initializing Runner...")
# all in a good cause...
GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)
runner = BenchmarkRunner(
benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
TMP_DIR, PREPARE, always_clean=True,
# run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
repo = runner.repo # (steal the parsed git repo used by runner)
h_head = args.target_commit or repo.shas[-1]
h_baseline = args.base_commit
# ARGH. reparse the repo, without discarding any commits,
# then overwrite the previous parse results
# prprint ("Slaughtering kittens..." )
(repo.shas, repo.messages,
repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
args.base_commit)
prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
prprint('Baseline [%s] : %s\n' % (h_baseline,
repo.messages.get(h_baseline, "")))
prprint("removing any previous measurements for the commits.")
db.delete_rev_results(h_baseline)
db.delete_rev_results(h_head)
# TODO: we could skip this, but we need to make sure all
# results are in the DB, which is a little tricky with
# start dates and so on.
prprint("Running benchmarks for baseline [%s]" % h_baseline)
runner._run_and_write_results(h_baseline)
prprint("Running benchmarks for target [%s]" % h_head)
runner._run_and_write_results(h_head)
prprint('Processing results...')
head_res = get_results_df(db, h_head)
baseline_res = get_results_df(db, h_baseline)
ratio = head_res['timing'] / baseline_res['timing']
totals = DataFrame({HEAD_COL:head_res['timing'],
BASE_COL:baseline_res['timing'],
'ratio':ratio,
'name':baseline_res.name},
columns=[HEAD_COL, BASE_COL, "ratio", "name"])
totals = totals.ix[totals[HEAD_COL] > args.min_duration]
# ignore below threshold
totals = totals.dropna(
).sort("ratio").set_index('name') # sort in ascending order
h_msg = repo.messages.get(h_head, "")
b_msg = repo.messages.get(h_baseline, "")
print_report(totals,h_head=h_head,h_msg=h_msg,
h_baseline=h_baseline,b_msg=b_msg)
if args.outdf:
prprint("The results DataFrame was written to '%s'\n" % args.outdf)
totals.save(args.outdf)
finally:
# print("Disposing of TMP_DIR: %s" % TMP_DIR)
shutil.rmtree(TMP_DIR)
def test_benchmarkrunner():
from vbench.api import BenchmarkRunner
# Just to make sure there are no left-overs
shutil.rmtree(TMP_DIR)
if exists(DB_PATH):
os.unlink(DB_PATH)
ok_(not exists(DB_PATH))
runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
BUILD, DB_PATH, TMP_DIR, PREPARE,
clean_cmd=CLEAN,
run_option='all', run_order='normal',
start_date=START_DATE,
module_dependencies=DEPENDENCIES)
revisions_to_run = runner._get_revisions_to_run()
eq_(len(revisions_to_run), 4) # we had 4 so far
revisions_ran = runner.run()
# print "D1: ", revisions_ran
assert_array_equal([x[0] for x in revisions_ran],
revisions_to_run)
# First revision
eq_(revisions_ran[0][1], (False, 3)) # no functions were available at that point
eq_(revisions_ran[1][1], (True, 3)) # all 3 tests were available in the first rev
ok_(exists(TMP_DIR))
ok_(exists(DB_PATH))
eq_(len(runner.blacklist), 0)
# Run 2nd time and verify that all are still listed BUT none new succeeds
revisions_ran = runner.run()
#print "D2: ", revisions_ran
for rev, v in revisions_ran:
eq_(v, (False, 0))
# What if we expand list of benchmarks and run 3rd time
runner.benchmarks = collect_benchmarks(['vb_sins', 'vb_sins2'])
revisions_ran = runner.run()
# for that single added benchmark there still were no function
eq_(revisions_ran[0][1], (False, 1))
# all others should have "succeeded" on that single one
for rev, v in revisions_ran[1:]:
eq_(v, (True, 1))
# and on 4th run -- nothing new
revisions_ran = runner.run()
for rev, v in revisions_ran:
eq_(v, (False, 0))
# Now let's smoke test generation of the .rst files
from vbench.reports import generate_rst_files
rstdir = pjoin(TMP_DIR, 'sources')
generate_rst_files(runner.benchmarks, DB_PATH, rstdir, """VERY LONG DESCRIPTION""")
# Verify that it all looks close to the desired
image_files = [basename(x) for x in glob(pjoin(rstdir, 'vbench/figures/*.png'))]
target_image_files = [b.name + '.png' for b in runner.benchmarks]
eq_(set(image_files), set(target_image_files))
rst_files = [basename(x) for x in glob(pjoin(rstdir, 'vbench/*.rst'))]
target_rst_files = [b.name + '.rst' for b in runner.benchmarks]
eq_(set(rst_files), set(target_rst_files))
module_files = [basename(x) for x in glob(pjoin(rstdir, '*.rst'))]
target_module_files = list(set(['vb_' + b.module_name + '.rst' for b in runner.benchmarks]))
eq_(set(module_files), set(target_module_files + ['index.rst']))
#print TMP_DIR
shutil.rmtree(TMP_DIR)
shutil.rmtree(dirname(DB_PATH))
def profile_comparative(benchmarks):
from vbench.api import BenchmarkRunner
from vbench.db import BenchmarkDB
from vbench.git import GitRepo
from suite import BUILD, DB_PATH, PREPARE, dependencies
TMP_DIR = args.temp_dir or tempfile.mkdtemp()
try:
prprint("Opening DB at '%s'...\n" % DB_PATH)
db = BenchmarkDB(DB_PATH)
prprint("Initializing Runner...")
# all in a good cause...
GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)
runner = BenchmarkRunner(
benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
TMP_DIR, PREPARE, always_clean=True,
# run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
repo = runner.repo # (steal the parsed git repo used by runner)
h_head = args.target_commit or repo.shas[-1]
h_baseline = args.base_commit
# ARGH. reparse the repo, without discarding any commits,
# then overwrite the previous parse results
# prprint("Slaughtering kittens...")
(repo.shas, repo.messages,
repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
args.base_commit)
prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
prprint('Baseline [%s] : %s\n' % (h_baseline,
repo.messages.get(h_baseline, "")))
prprint("Removing any previous measurements for the commits.")
db.delete_rev_results(h_baseline)
db.delete_rev_results(h_head)
# TODO: we could skip this, but we need to make sure all
# results are in the DB, which is a little tricky with
# start dates and so on.
prprint("Running benchmarks for baseline [%s]" % h_baseline)
runner._run_and_write_results(h_baseline)
prprint("Running benchmarks for target [%s]" % h_head)
runner._run_and_write_results(h_head)
prprint('Processing results...')
head_res = get_results_df(db, h_head)
baseline_res = get_results_df(db, h_baseline)
report_comparative(head_res,baseline_res)
finally:
# print("Disposing of TMP_DIR: %s" % TMP_DIR)
shutil.rmtree(TMP_DIR)
SVN_URL = 'https://neuralensemble.org/svn/brian/trunk'
REPO_PATH = os.path.join(PATH, 'brian-trunk')
if not os.path.exists(REPO_PATH):
# create the repository
os.makedirs(REPO_PATH)
os.system('git svn clone %s %s' % (SVN_URL, REPO_PATH))
else:
# update the repository (can only be called from the directory...)
os.chdir(REPO_PATH)
os.system('git svn rebase')
os.system('git svn fetch')
REPO_URL = 'file://' + REPO_PATH
TMP_DIR = tempfile.mkdtemp(suffix='vbench')
# Those two are not really needed at the moment as no C extensions are
# compiled by default
# TODO: Does using sys.executable here work on Windows?
PREPARE = "%s setup.py clean" % sys.executable
BUILD = "%s setup.py build_ext" % sys.executable
START_DATE = datetime(2008, 9, 23) # Brian version 1.0.0
repo = GitRepo(REPO_PATH)
runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL, BUILD, DB_PATH,
TMP_DIR, PREPARE, run_option=run_option,
start_date=START_DATE)
runner.run()
def main():
from pandas import DataFrame
from vbench.api import BenchmarkRunner
from vbench.db import BenchmarkDB
from suite import REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies, benchmarks
if not args.base_commit:
args.base_commit = BASELINE_COMMIT
# GitRepo wants exactly 7 character hash?
args.base_commit = args.base_commit[:7]
if args.target_commit:
args.target_commit = args.target_commit[:7]
if not args.log_file:
args.log_file = os.path.abspath(os.path.join(REPO_PATH,
'vb_suite.log'))
TMP_DIR = tempfile.mkdtemp()
prprint("TMP_DIR = %s" % TMP_DIR)
prprint("LOG_FILE = %s\n" % args.log_file)
try:
logfile = open(args.log_file, 'w')
prprint("Opening DB at '%s'...\n" % DB_PATH)
db = BenchmarkDB(DB_PATH)
prprint("Initializing Runner...")
runner = BenchmarkRunner(
benchmarks,
REPO_PATH,
REPO_PATH,
BUILD,
DB_PATH,
TMP_DIR,
PREPARE,
always_clean=True,
# run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
repo = runner.repo # (steal the parsed git repo used by runner)
# ARGH. reparse the repo, without discarding any commits,
# then overwrite the previous parse results
# prprint ("Slaughtering kittens..." )
(repo.shas, repo.messages, repo.timestamps,
repo.authors) = _parse_commit_log(REPO_PATH)
h_head = args.target_commit or repo.shas[-1]
h_baseline = args.base_commit
prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
prprint('Baseline [%s] : %s\n' %
(h_baseline, repo.messages.get(h_baseline, "")))
prprint("removing any previous measurements for the commits.")
db.delete_rev_results(h_baseline)
db.delete_rev_results(h_head)
# TODO: we could skip this, but we need to make sure all
# results are in the DB, which is a little tricky with
# start dates and so on.
prprint("Running benchmarks for baseline [%s]" % h_baseline)
runner._run_and_write_results(h_baseline)
prprint("Running benchmarks for target [%s]" % h_head)
runner._run_and_write_results(h_head)
prprint('Processing results...')
head_res = get_results_df(db, h_head)
baseline_res = get_results_df(db, h_baseline)
ratio = head_res['timing'] / baseline_res['timing']
totals = DataFrame(dict(t_head=head_res['timing'],
t_baseline=baseline_res['timing'],
ratio=ratio,
name=baseline_res.name),
columns=["t_head", "t_baseline", "ratio", "name"])
totals = totals.ix[totals.t_head > args.min_duration]
# ignore below threshold
totals = totals.dropna().sort("ratio").set_index(
'name') # sort in ascending order
s = "\n\nResults:\n"
s += totals.to_string(
float_format=lambda x: "{:4.4f}".format(x).rjust(10))
s += "\n\n"
s += "Columns: test_name | target_duration [ms] | baseline_duration [ms] | ratio\n\n"
s += "- a Ratio of 1.30 means the target commit is 30% slower then the baseline.\n\n"
s += 'Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
s += 'Baseline [%s] : %s\n\n' % (h_baseline,
repo.messages.get(h_baseline, ""))
logfile.write(s)
logfile.close()
prprint(s)
prprint("Results were also written to the logfile at '%s'\n" %
args.log_file)
finally:
# print("Disposing of TMP_DIR: %s" % TMP_DIR)
shutil.rmtree(TMP_DIR)
logfile.close()
def main():
from pandas import DataFrame
from vbench.api import BenchmarkRunner
from vbench.db import BenchmarkDB
from suite import REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies, benchmarks
if not args.base_commit:
args.base_commit = BASELINE_COMMIT
# GitRepo wants exactly 7 character hash?
args.base_commit = args.base_commit[:7]
if args.target_commit:
args.target_commit = args.target_commit[:7]
if not args.log_file:
args.log_file = os.path.abspath(
os.path.join(REPO_PATH, 'vb_suite.log'))
TMP_DIR = tempfile.mkdtemp()
prprint("TMP_DIR = %s" % TMP_DIR)
prprint("LOG_FILE = %s\n" % args.log_file)
try:
logfile = open(args.log_file, 'w')
prprint("Opening DB at '%s'...\n" % DB_PATH)
db = BenchmarkDB(DB_PATH)
prprint("Initializing Runner...")
runner = BenchmarkRunner(
benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
TMP_DIR, PREPARE, always_clean=True,
# run_option='eod', start_date=START_DATE,
module_dependencies=dependencies)
repo = runner.repo # (steal the parsed git repo used by runner)
# ARGH. reparse the repo, without discarding any commits,
# then overwrite the previous parse results
# prprint ("Slaughtering kittens..." )
(repo.shas, repo.messages,
repo.timestamps, repo.authors) = _parse_commit_log(REPO_PATH)
h_head = args.target_commit or repo.shas[-1]
h_baseline = args.base_commit
prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
prprint('Baseline [%s] : %s\n' % (h_baseline,
repo.messages.get(h_baseline, "")))
prprint("removing any previous measurements for the commits.")
db.delete_rev_results(h_baseline)
db.delete_rev_results(h_head)
# TODO: we could skip this, but we need to make sure all
# results are in the DB, which is a little tricky with
# start dates and so on.
prprint("Running benchmarks for baseline [%s]" % h_baseline)
runner._run_and_write_results(h_baseline)
prprint("Running benchmarks for target [%s]" % h_head)
runner._run_and_write_results(h_head)
prprint('Processing results...')
head_res = get_results_df(db, h_head)
baseline_res = get_results_df(db, h_baseline)
ratio = head_res['timing'] / baseline_res['timing']
totals = DataFrame(dict(t_head=head_res['timing'],
t_baseline=baseline_res['timing'],
ratio=ratio,
name=baseline_res.name), columns=["t_head", "t_baseline", "ratio", "name"])
totals = totals.ix[totals.t_head > args.min_duration]
# ignore below threshold
totals = totals.dropna(
).sort("ratio").set_index('name') # sort in ascending order
s = "\n\nResults:\n"
s += totals.to_string(
float_format=lambda x: "{:4.4f}".format(x).rjust(10))
s += "\n\n"
s += "Columns: test_name | target_duration [ms] | baseline_duration [ms] | ratio\n\n"
s += "- a Ratio of 1.30 means the target commit is 30% slower then the baseline.\n\n"
s += 'Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
s += 'Baseline [%s] : %s\n\n' % (
h_baseline, repo.messages.get(h_baseline, ""))
logfile.write(s)
logfile.close()
prprint(s)
prprint("Results were also written to the logfile at '%s'\n" %
args.log_file)
finally:
# print("Disposing of TMP_DIR: %s" % TMP_DIR)
shutil.rmtree(TMP_DIR)
logfile.close()
