コード例 #1
0
ファイル: run_suite.py プロジェクト: useric/bquery
def run_process():
    runner = BenchmarkRunner(
        benchmarks, REPO_PATH, REPO_URL, BUILD, DB_PATH, TMP_DIR, PREPARE,
        always_clean=True, run_option='eod', start_date=START_DATE,
        module_dependencies=dependencies)
    runner.run()
    generate_rst_files(runner.benchmarks, DB_PATH, RST_BASE, """LONG DESC.""")
コード例 #2
0
ファイル: run_benchmark.py プロジェクト: s-zeng/sexpdata
def main(args=None):
    import argparse
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=__doc__ +
        "For more information, see the docstring of `BenchmarkRunner`:\n" +
        BenchmarkRunner.__doc__)
    parser.add_argument('--run-option',
                        default='eod',
                        help="one of {'eod', 'all', 'last', integer}")
    ns = parser.parse_args(args)

    run_option = ns.run_option
    if run_option.isdigit():
        run_option = int(run_option)

    runner = BenchmarkRunner(benchmarks,
                             REPO_PATH,
                             REPO_URL,
                             BUILD,
                             DB_PATH,
                             TMP_DIR,
                             PREPARE,
                             run_option=run_option,
                             start_date=START_DATE)
    runner.run()
    return runner
コード例 #3
0
ファイル: run_suite.py プロジェクト: statsmodels/vbench
def run_process():
    runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
                             BUILD, DB_PATH, TMP_DIR, PREPARE,
                             always_clean=True,
                             run_option='eod', start_date=START_DATE,
                             module_dependencies=dependencies)
    runner.run()
コード例 #4
0
ファイル: run_suite.py プロジェクト: zhanghx17/minpower
def run_process(run_option='eod'):
    runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
                             BUILD, DB_PATH, TMP_DIR, PREPARE,
                             always_clean=True,
                             run_option=run_option, start_date=START_DATE,
                             module_dependencies=dependencies)
    runner.run()
コード例 #5
0
ファイル: run_suite.py プロジェクト: arinkverma/numpy-vbench
def run_process():
    runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
                             BUILD, DB_PATH, TMP_DIR, PREPARE,
                             clean_cmd=PREPARE,
                             run_option='eod', run_order='multires',
                             start_date=START_DATE,
                             module_dependencies=dependencies,
                             verify=True)
    runner.run()
コード例 #6
0
ファイル: run_suite.py プロジェクト: yarikoptic/numpy-vbench
def run_process(existing='min', run_order='multires', run_limit=None):
    runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
                             BUILD, DB_PATH, TMP_DIR, PREPARE,
                             branches=BRANCHES,
                             clean_cmd=PREPARE,
                             run_option='all', run_order=run_order, run_limit=run_limit,
                             start_date=START_DATE,
                             existing=existing,
                             module_dependencies=dependencies,
                             verify=True)
    runner.run()
コード例 #7
0
ファイル: run_suite.py プロジェクト: dit/dit-vbench
def run_process(existing='min', run_order='multires', run_limit=None):
    runner = BenchmarkRunner(suite.benchmarks,
                             suite.REPO_PATH,
                             suite.REPO_URL,
                             suite.BUILD,
                             suite.DB_PATH,
                             suite.TMP_DIR,
                             suite.PREPARE,
                             branches=suite.BRANCHES,
                             clean_cmd=suite.PREPARE,
                             run_option='all',
                             run_order=run_order,
                             run_limit=run_limit,
                             start_date=suite.START_DATE,
                             existing=existing,
                             module_dependencies=suite.dependencies,
                             verify=True)
    runner.run()
コード例 #8
0
ファイル: run_suite.py プロジェクト: chebee7i/buhmm-vbench
def run_process(existing='min', run_order='multires', run_limit=None):
    runner = BenchmarkRunner(suite.benchmarks,
                             suite.REPO_PATH,
                             suite.REPO_URL,
                             suite.BUILD,
                             suite.DB_PATH,
                             suite.TMP_DIR,
                             suite.PREPARE,
                             branches=suite.BRANCHES,
                             clean_cmd=suite.PREPARE,
                             run_option='all',
                             run_order=run_order,
                             run_limit=run_limit,
                             start_date=suite.START_DATE,
                             existing=existing,
                             module_dependencies=suite.dependencies,
                             verify=True)
    runner.run()
コード例 #9
0
ファイル: run_benchmark.py プロジェクト: EdTsft/sexpdata
def main(args=None):
    import argparse
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=__doc__
        + "For more information, see the docstring of `BenchmarkRunner`:\n"
        + BenchmarkRunner.__doc__)
    parser.add_argument(
        '--run-option', default='eod',
        help="one of {'eod', 'all', 'last', integer}")
    ns = parser.parse_args(args)

    run_option = ns.run_option
    if run_option.isdigit():
        run_option = int(run_option)

    runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
                             BUILD, DB_PATH, TMP_DIR, PREPARE,
                             run_option=run_option, start_date=START_DATE)
    runner.run()
    return runner
コード例 #10
0
def main():
    import shutil
    import tempfile
    import warnings

    from pandas import Series

    from vbench.api import BenchmarkRunner
    from suite import (REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies,
                       benchmarks)

    from memory_profiler import memory_usage

    warnings.filterwarnings('ignore', category=FutureWarning)

    try:
        TMP_DIR = tempfile.mkdtemp()
        runner = BenchmarkRunner(
            benchmarks,
            REPO_PATH,
            REPO_PATH,
            BUILD,
            DB_PATH,
            TMP_DIR,
            PREPARE,
            always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)
        results = {}
        for b in runner.benchmarks:
            k = b.name
            try:
                vs = memory_usage((b.run, ))
                v = max(vs)
                # print(k, v)
                results[k] = v
            except Exception as e:
                print("Exception caught in %s\n" % k)
                print(str(e))

        s = Series(results)
        s.sort()
        print(s)

    finally:
        shutil.rmtree(TMP_DIR)
コード例 #11
0
def test_benchmarkrunner():
    from suite import *

    # Just to make sure there are no left-overs
    shutil.rmtree(TMP_DIR)
    if exists(DB_PATH):
        os.unlink(DB_PATH)
    ok_(not exists(DB_PATH))

    runner = BenchmarkRunner(benchmarks,
                             REPO_PATH,
                             REPO_URL,
                             BUILD,
                             DB_PATH,
                             TMP_DIR,
                             PREPARE,
                             clean_cmd=CLEAN,
                             run_option='all',
                             run_order='normal',
                             start_date=START_DATE,
                             module_dependencies=DEPENDENCIES)
    revisions_to_run = runner._get_revisions_to_run()
    eq_(len(revisions_to_run), 6)  # we have 6 now

    revisions_ran = runner.run()
    # print "D1: ", revisions_ran
    # for this test we should inject our "failed to build revision"
    # Since no tests were ran for it -- it is not reported
    revisions_ran_ = [x[0] for x in revisions_ran]
    revisions_ran_.insert(4, 'e83ffa5')
    assert_array_equal(revisions_ran_, revisions_to_run)

    # First revision
    eq_(revisions_ran[0][1],
        (False, 3))  # no functions were available at that point
    eq_(revisions_ran[1][1],
        (True, 3))  # all 3 tests were available in the first rev

    ok_(exists(TMP_DIR))
    ok_(exists(DB_PATH))

    eq_(runner.blacklist, set(['e83ffa5']))  # one which failed to build

    # Run 2nd time and verify that all are still listed BUT none new succeeds
    revisions_ran = runner.run()
    #print "D2: ", revisions_ran
    for rev, v in revisions_ran:
        eq_(v, (False, 0))

    # What if we expand list of benchmarks and run 3rd time
    runner.benchmarks = collect_benchmarks(['vb_sins', 'vb_sins2'])
    revisions_ran = runner.run()
    # for that single added benchmark there still were no function
    eq_(revisions_ran[0][1], (False, 1))
    # all others should have "succeeded" on that single one
    for rev, v in revisions_ran[1:]:
        eq_(v, (True, 1))

    # and on 4th run -- nothing new
    revisions_ran = runner.run()
    for rev, v in revisions_ran:
        eq_(v, (False, 0))

    # Now let's smoke test generation of the .rst files
    from vbench.reports import generate_rst_files
    rstdir = pjoin(TMP_DIR, 'sources')
    generate_rst_files(runner.benchmarks, DB_PATH, rstdir,
                       """VERY LONG DESCRIPTION""")

    # Verify that it all looks close to the desired
    image_files = [
        basename(x) for x in glob(pjoin(rstdir, 'vbench/figures/*.png'))
    ]
    target_image_files = [b.name + '.png' for b in runner.benchmarks]
    eq_(set(image_files), set(target_image_files))

    rst_files = [basename(x) for x in glob(pjoin(rstdir, 'vbench/*.rst'))]
    target_rst_files = [b.name + '.rst' for b in runner.benchmarks]
    eq_(set(rst_files), set(target_rst_files))

    module_files = [basename(x) for x in glob(pjoin(rstdir, '*.rst'))]
    target_module_files = list(
        set(['vb_' + b.module_name + '.rst' for b in runner.benchmarks]))
    eq_(set(module_files), set(target_module_files + ['index.rst']))

    #print TMP_DIR
    shutil.rmtree(TMP_DIR)
    shutil.rmtree(dirname(DB_PATH))
コード例 #12
0
ファイル: suite.py プロジェクト: acdha/pymaging
def run_process():
    runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
                             '', DB_PATH, TMP_DIR, '',
                             run_option='all', start_date=START_DATE,
                             module_dependencies=[])
    runner.run()
コード例 #13
0
ファイル: test_perf.py プロジェクト: wudcwctw/pandas
def profile_comparative(benchmarks):

    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from vbench.git import GitRepo
    from suite import BUILD, DB_PATH, PREPARE, dependencies
    TMP_DIR = tempfile.mkdtemp()

    try:

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")

        # all in a good cause...
        GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)

        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)
        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint("Slaughtering kittens...")
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
                                                                args.base_commit)

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("Removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)

        report_comparative(head_res,baseline_res)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
コード例 #14
0
ファイル: test_perf.py プロジェクト: KWMalik/pandas
def main():
    TMP_DIR = tempfile.mkdtemp()
    prprint("TMP_DIR = %s" % TMP_DIR)
    prprint("LOG_FILE = %s\n" % LOG_FILE)

    try:
        logfile = open(LOG_FILE, 'w')

        prprint("Processing Repo at '%s'..." % REPO_PATH)
        repo = GitRepo(REPO_PATH)

        # get hashes of baseline and current head
        h_head = repo.shas[-1]
        h_baseline = BASELINE_COMMIT

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint('Comparing Head [%s] : %s ' %
                (h_head, repo.messages.get(h_head, "")))
        prprint('Against baseline [%s] : %s \n' %
                (h_baseline, repo.messages.get(h_baseline, "")))

        prprint("Initializing Runner...")
        runner = BenchmarkRunner(
            benchmarks,
            REPO_PATH,
            REPO_PATH,
            BUILD,
            DB_PATH,
            TMP_DIR,
            PREPARE,
            always_clean=True,
            #                             run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        prprint("removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline commit '%s'" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for current HEAD '%s'" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame(dict(t_head=head_res['timing'],
                                t_baseline=baseline_res['timing'],
                                ratio=ratio,
                                name=baseline_res.name),
                           columns=["t_head", "t_baseline", "ratio", "name"])
        totals = totals.ix[totals.t_head > 0.010]  # ignore sub 10micros
        totals = totals.dropna().sort("ratio").set_index(
            'name')  # sort in ascending order

        s = "\n\nResults:\n" + totals.to_string(
            float_format=lambda x: "%0.4f" % x) + "\n\n"
        s += "Columns: test_name | head_time [ms] | baseline_time [ms] | ratio\n\n"
        s += "- a Ratio of 1.30 means HEAD is 30% slower then the Baseline.\n\n"

        s += 'Head [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
        s += 'Baseline [%s] : %s\n\n' % (h_baseline,
                                         repo.messages.get(h_baseline, ""))

        logfile.write(s)
        logfile.close()

        prprint(s)
        prprint("Results were also written to the logfile at '%s'\n" %
                LOG_FILE)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
        logfile.close()
コード例 #15
0
ファイル: test_perf.py プロジェクト: esc/pandas
def profile_comparative(benchmarks):

    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from vbench.git import GitRepo
    from suite import BUILD, DB_PATH, PREPARE, dependencies
    TMP_DIR = tempfile.mkdtemp()

    try:

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")

        # all in a good cause...
        GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)

        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)
        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint ("Slaughtering kittens..." )
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
                                                                args.base_commit)

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("Removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame({HEAD_COL:head_res['timing'],
                                BASE_COL:baseline_res['timing'],
                                'ratio':ratio,
                                'name':baseline_res.name},
                                columns=[HEAD_COL, BASE_COL, "ratio", "name"])
        totals = totals.ix[totals[HEAD_COL] > args.min_duration]
            # ignore below threshold
        totals = totals.dropna(
        ).sort("ratio").set_index('name')  # sort in ascending order

        h_msg =   repo.messages.get(h_head, "")
        b_msg =   repo.messages.get(h_baseline, "")

        print_report(totals,h_head=h_head,h_msg=h_msg,
                     h_baseline=h_baseline,b_msg=b_msg)

        if args.outdf:
            prprint("The results DataFrame was written to '%s'\n" %  args.outdf)
            totals.save(args.outdf)
    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
コード例 #16
0
def run_process(run_option):
    runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
                             BUILD, DB_PATH, TMP_DIR, PREPARE,
                             run_option=run_option, start_date=START_DATE,
                             module_dependencies=dependencies)
    runner.run()
コード例 #17
0
ファイル: test_perf.py プロジェクト: KWMalik/pandas
def main():
    TMP_DIR =  tempfile.mkdtemp()
    prprint("TMP_DIR = %s" % TMP_DIR)
    prprint("LOG_FILE = %s\n" % LOG_FILE)

    try:
        logfile = open(LOG_FILE, 'w')

        prprint( "Processing Repo at '%s'..." % REPO_PATH)
        repo = GitRepo(REPO_PATH)

        # get hashes of baseline and current head
        h_head = repo.shas[-1]
        h_baseline = BASELINE_COMMIT

        prprint( "Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint( 'Comparing Head [%s] : %s ' % (h_head, repo.messages.get(h_head,"")))
        prprint( 'Against baseline [%s] : %s \n' % (h_baseline,
                repo.messages.get(h_baseline,"")))

        prprint("Initializing Runner...")
        runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
                                 TMP_DIR, PREPARE, always_clean=True,
            #                             run_option='eod', start_date=START_DATE,
                                 module_dependencies=dependencies)

        prprint ("removing any previous measurements for the commits." )
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint( "Running benchmarks for baseline commit '%s'" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint ("Running benchmarks for current HEAD '%s'" % h_head)
        runner._run_and_write_results(h_head)

        prprint( 'Processing results...')

        head_res = get_results_df(db,h_head)
        baseline_res = get_results_df(db,h_baseline)
        ratio = head_res['timing']/baseline_res['timing']
        totals = DataFrame(dict(t_head=head_res['timing'],
                                t_baseline=baseline_res['timing'],
                                ratio=ratio,
                                name=baseline_res.name),columns=["t_head","t_baseline","ratio","name"])
        totals = totals.ix[totals.t_head > 0.010] # ignore sub 10micros
        totals = totals.dropna().sort("ratio").set_index('name') # sort in ascending order

        s = "\n\nResults:\n" + totals.to_string(float_format=lambda x: "%0.4f" %x) + "\n\n"
        s += "Columns: test_name | head_time [ms] | baseline_time [ms] | ratio\n\n"
        s += "- a Ratio of 1.30 means HEAD is 30% slower then the Baseline.\n\n"

        s += 'Head [%s] : %s\n' % (h_head, repo.messages.get(h_head,""))
        s += 'Baseline [%s] : %s\n\n' % (h_baseline,repo.messages.get(h_baseline,""))

        logfile.write(s)
        logfile.close()

        prprint(s )
        prprint("Results were also written to the logfile at '%s'\n" % LOG_FILE)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
        logfile.close()
コード例 #18
0
ファイル: test_perf.py プロジェクト: da415/pandas
def main():
    from pandas import DataFrame
    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from vbench.git import GitRepo
    from suite import REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies, benchmarks

    # GitRepo wants exactly 7 character hash?
    args.base_commit = args.base_commit[:7]
    if args.target_commit:
        args.target_commit = args.target_commit[:7]

    if not args.log_file:
        args.log_file = os.path.abspath(
            os.path.join(REPO_PATH, 'vb_suite.log'))

    random.seed(args.seed)
    np.random.seed(args.seed)

    TMP_DIR = tempfile.mkdtemp()
    prprint("TMP_DIR = %s" % TMP_DIR)
    prprint("LOG_FILE = %s\n" % args.log_file)

    benchmarks = [x for x in benchmarks if re.search(args.regex,x.name)]

    try:
        logfile = open(args.log_file, 'w')

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")

        # all in a good cause...
        GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)

        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint ("Slaughtering kittens..." )
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
                                                                args.base_commit)

        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame(dict(t_head=head_res['timing'],
                                t_baseline=baseline_res['timing'],
                                ratio=ratio,
                                name=baseline_res.name), columns=["t_head", "t_baseline", "ratio", "name"])
        totals = totals.ix[totals.t_head > args.min_duration]
            # ignore below threshold
        totals = totals.dropna(
        ).sort("ratio").set_index('name')  # sort in ascending order


        hdr = ftr = """
-----------------------------------------------------------------------
Test name                      | target[ms] |  base[ms]  |   ratio    |
-----------------------------------------------------------------------
""".strip() +"\n"

        s = "\n"
        s += hdr
        for i in range(len(totals)):
            t,b,r = totals.irow(i).values
            s += "{0:30s} {1: 12.4f} {2: 12.4f} {3: 12.4f}\n".format(totals.index[i],t,b,r)
        s+= ftr + "\n"

        s += "Ratio < 1.0 means the target commit is faster then the baseline.\n"
        s += "Seed used: %d\n\n" % args.seed

        s += 'Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
        s += 'Base   [%s] : %s\n\n' % (
            h_baseline, repo.messages.get(h_baseline, ""))

        logfile.write(s)
        logfile.close()

        prprint(s)
        prprint("Results were also written to the logfile at '%s'\n" %
                args.log_file)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
        logfile.close()
コード例 #19
0
ファイル: test_perf.py プロジェクト: tongming/pandas
def profile_comparative(benchmarks):

    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from vbench.git import GitRepo
    from suite import BUILD, DB_PATH, PREPARE, dependencies
    TMP_DIR = tempfile.mkdtemp()

    try:

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")

        # all in a good cause...
        GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)

        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)
        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint ("Slaughtering kittens..." )
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
                                                                args.base_commit)

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame({HEAD_COL:head_res['timing'],
                                BASE_COL:baseline_res['timing'],
                                'ratio':ratio,
                                'name':baseline_res.name},
                                columns=[HEAD_COL, BASE_COL, "ratio", "name"])
        totals = totals.ix[totals[HEAD_COL] > args.min_duration]
            # ignore below threshold
        totals = totals.dropna(
        ).sort("ratio").set_index('name')  # sort in ascending order

        h_msg =   repo.messages.get(h_head, "")
        b_msg =   repo.messages.get(h_baseline, "")

        print_report(totals,h_head=h_head,h_msg=h_msg,
                     h_baseline=h_baseline,b_msg=b_msg)

        if args.outdf:
            prprint("The results DataFrame was written to '%s'\n" %  args.outdf)
            totals.save(args.outdf)
    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
コード例 #20
0
ファイル: test_bench.py プロジェクト: grinner/vbench
def test_benchmarkrunner():
    from vbench.api import BenchmarkRunner
    

    # Just to make sure there are no left-overs
    shutil.rmtree(TMP_DIR)
    if exists(DB_PATH):
        os.unlink(DB_PATH)
    ok_(not exists(DB_PATH))

    runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL,
                             BUILD, DB_PATH, TMP_DIR, PREPARE,
                             clean_cmd=CLEAN,
                             run_option='all', run_order='normal',
                             start_date=START_DATE,
                             module_dependencies=DEPENDENCIES)
    revisions_to_run = runner._get_revisions_to_run()
    eq_(len(revisions_to_run), 4)                # we had 4 so far

    revisions_ran = runner.run()
    # print "D1: ", revisions_ran
    assert_array_equal([x[0] for x in revisions_ran],
                       revisions_to_run)
    # First revision
    eq_(revisions_ran[0][1], (False, 3))    # no functions were available at that point
    eq_(revisions_ran[1][1], (True, 3))     # all 3 tests were available in the first rev

    ok_(exists(TMP_DIR))
    ok_(exists(DB_PATH))

    eq_(len(runner.blacklist), 0)

    # Run 2nd time and verify that all are still listed BUT none new succeeds
    revisions_ran = runner.run()
    #print "D2: ", revisions_ran
    for rev, v in revisions_ran:
        eq_(v, (False, 0))

    # What if we expand list of benchmarks and run 3rd time
    runner.benchmarks = collect_benchmarks(['vb_sins', 'vb_sins2'])
    revisions_ran = runner.run()
    # for that single added benchmark there still were no function
    eq_(revisions_ran[0][1], (False, 1))
    # all others should have "succeeded" on that single one
    for rev, v in revisions_ran[1:]:
        eq_(v, (True, 1))

    # and on 4th run -- nothing new
    revisions_ran = runner.run()
    for rev, v in revisions_ran:
        eq_(v, (False, 0))

    # Now let's smoke test generation of the .rst files
    from vbench.reports import generate_rst_files
    rstdir = pjoin(TMP_DIR, 'sources')
    generate_rst_files(runner.benchmarks, DB_PATH, rstdir, """VERY LONG DESCRIPTION""")

    # Verify that it all looks close to the desired
    image_files = [basename(x) for x in glob(pjoin(rstdir, 'vbench/figures/*.png'))]
    target_image_files = [b.name + '.png' for b in runner.benchmarks]
    eq_(set(image_files), set(target_image_files))

    rst_files = [basename(x) for x in glob(pjoin(rstdir, 'vbench/*.rst'))]
    target_rst_files = [b.name + '.rst' for b in runner.benchmarks]
    eq_(set(rst_files), set(target_rst_files))

    module_files = [basename(x) for x in glob(pjoin(rstdir, '*.rst'))]
    target_module_files = list(set(['vb_' + b.module_name + '.rst' for b in runner.benchmarks]))
    eq_(set(module_files), set(target_module_files + ['index.rst']))

    #print TMP_DIR
    shutil.rmtree(TMP_DIR)
    shutil.rmtree(dirname(DB_PATH))
コード例 #21
0
ファイル: test_perf.py プロジェクト: AbhijitBadve/pandas
def profile_comparative(benchmarks):

    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from vbench.git import GitRepo
    from suite import BUILD, DB_PATH, PREPARE, dependencies

    TMP_DIR = args.temp_dir or tempfile.mkdtemp()

    try:

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")

        # all in a good cause...
        GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)

        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)
        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint("Slaughtering kittens...")
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
                                                                args.base_commit)

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("Removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)

        report_comparative(head_res,baseline_res)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
コード例 #22
0
ファイル: run_benchmarks.py プロジェクト: yzerlaut/brian
    SVN_URL = 'https://neuralensemble.org/svn/brian/trunk'
    REPO_PATH = os.path.join(PATH, 'brian-trunk')
    if not os.path.exists(REPO_PATH):
        # create the repository
        os.makedirs(REPO_PATH)
        os.system('git svn clone %s %s' % (SVN_URL, REPO_PATH))
    else:
        # update the repository (can only be called from the directory...) 
        os.chdir(REPO_PATH)
        os.system('git svn rebase')
        os.system('git svn fetch')

    REPO_URL = 'file://' + REPO_PATH

    TMP_DIR = tempfile.mkdtemp(suffix='vbench')

    # Those two are not really needed at the moment as no C extensions are
    # compiled by default
    # TODO: Does using sys.executable here work on Windows? 
    PREPARE = "%s setup.py clean" % sys.executable
    BUILD = "%s setup.py build_ext" % sys.executable

    START_DATE = datetime(2008, 9, 23) # Brian version 1.0.0

    repo = GitRepo(REPO_PATH)

    runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_URL, BUILD, DB_PATH,
                             TMP_DIR, PREPARE, run_option=run_option, 
                             start_date=START_DATE)
    runner.run()
コード例 #23
0
ファイル: test_perf.py プロジェクト: trwhitcomb/pandas
def main():
    from pandas import DataFrame
    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from suite import REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies, benchmarks

    if not args.base_commit:
        args.base_commit = BASELINE_COMMIT

    # GitRepo wants exactly 7 character hash?
    args.base_commit = args.base_commit[:7]
    if args.target_commit:
        args.target_commit = args.target_commit[:7]

    if not args.log_file:
        args.log_file = os.path.abspath(os.path.join(REPO_PATH,
                                                     'vb_suite.log'))

    TMP_DIR = tempfile.mkdtemp()
    prprint("TMP_DIR = %s" % TMP_DIR)
    prprint("LOG_FILE = %s\n" % args.log_file)

    try:
        logfile = open(args.log_file, 'w')

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")
        runner = BenchmarkRunner(
            benchmarks,
            REPO_PATH,
            REPO_PATH,
            BUILD,
            DB_PATH,
            TMP_DIR,
            PREPARE,
            always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint ("Slaughtering kittens..." )
        (repo.shas, repo.messages, repo.timestamps,
         repo.authors) = _parse_commit_log(REPO_PATH)

        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' %
                (h_baseline, repo.messages.get(h_baseline, "")))

        prprint("removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame(dict(t_head=head_res['timing'],
                                t_baseline=baseline_res['timing'],
                                ratio=ratio,
                                name=baseline_res.name),
                           columns=["t_head", "t_baseline", "ratio", "name"])
        totals = totals.ix[totals.t_head > args.min_duration]
        # ignore below threshold
        totals = totals.dropna().sort("ratio").set_index(
            'name')  # sort in ascending order

        s = "\n\nResults:\n"
        s += totals.to_string(
            float_format=lambda x: "{:4.4f}".format(x).rjust(10))
        s += "\n\n"
        s += "Columns: test_name | target_duration [ms] | baseline_duration [ms] | ratio\n\n"
        s += "- a Ratio of 1.30 means the target commit is 30% slower then the baseline.\n\n"

        s += 'Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
        s += 'Baseline [%s] : %s\n\n' % (h_baseline,
                                         repo.messages.get(h_baseline, ""))

        logfile.write(s)
        logfile.close()

        prprint(s)
        prprint("Results were also written to the logfile at '%s'\n" %
                args.log_file)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
        logfile.close()
コード例 #24
0
ファイル: test_perf.py プロジェクト: JuliusDegesys/pandas
def main():
    from pandas import DataFrame
    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from suite import REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies, benchmarks

    if not args.base_commit:
        args.base_commit = BASELINE_COMMIT

    # GitRepo wants exactly 7 character hash?
    args.base_commit = args.base_commit[:7]
    if args.target_commit:
        args.target_commit = args.target_commit[:7]

    if not args.log_file:
        args.log_file = os.path.abspath(
            os.path.join(REPO_PATH, 'vb_suite.log'))

    TMP_DIR = tempfile.mkdtemp()
    prprint("TMP_DIR = %s" % TMP_DIR)
    prprint("LOG_FILE = %s\n" % args.log_file)

    try:
        logfile = open(args.log_file, 'w')

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")
        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint ("Slaughtering kittens..." )
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(REPO_PATH)

        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame(dict(t_head=head_res['timing'],
                                t_baseline=baseline_res['timing'],
                                ratio=ratio,
                                name=baseline_res.name), columns=["t_head", "t_baseline", "ratio", "name"])
        totals = totals.ix[totals.t_head > args.min_duration]
            # ignore below threshold
        totals = totals.dropna(
        ).sort("ratio").set_index('name')  # sort in ascending order

        s = "\n\nResults:\n"
        s += totals.to_string(
            float_format=lambda x: "{:4.4f}".format(x).rjust(10))
        s += "\n\n"
        s += "Columns: test_name | target_duration [ms] | baseline_duration [ms] | ratio\n\n"
        s += "- a Ratio of 1.30 means the target commit is 30% slower then the baseline.\n\n"

        s += 'Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
        s += 'Baseline [%s] : %s\n\n' % (
            h_baseline, repo.messages.get(h_baseline, ""))

        logfile.write(s)
        logfile.close()

        prprint(s)
        prprint("Results were also written to the logfile at '%s'\n" %
                args.log_file)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
        logfile.close()