示例#1
0
文件: runner.py 项目: peterjc/vbench
    def __init__(self,
                 benchmarks,
                 repo_path,
                 repo_url,
                 build_cmd,
                 db_path,
                 tmp_dir,
                 preparation_cmd,
                 run_option='end_of_day',
                 start_date=None,
                 overwrite=False):

        self.benchmarks = benchmarks
        self.checksums = [b.checksum for b in benchmarks]

        self.start_date = start_date
        self.run_option = run_option

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url, self.tmp_dir, build_cmd,
                                    preparation_cmd)

        self._register_benchmarks()
示例#2
0
    def __init__(self,
                 benchmarks,
                 repo_path,
                 repo_url,
                 build_cmd,
                 db_path,
                 tmp_dir,
                 prep_cmd,
                 clean_cmd=None,
                 run_option='eod',
                 run_order='normal',
                 start_date=None,
                 overwrite=False,
                 module_dependencies=None,
                 always_clean=False,
                 use_blacklist=True,
                 verify=False):
        log.info("Initializing benchmark runner for %d benchmarks" %
                 (len(benchmarks)))
        self._benchmarks = None
        self._checksums = None

        if verify:
            verify_benchmarks(benchmarks, raise_=True)

        self.start_date = start_date
        self.run_option = run_option
        self.run_order = run_order

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url,
                                    self.tmp_dir,
                                    build_cmd,
                                    prep_cmd,
                                    clean_cmd,
                                    always_clean=always_clean,
                                    dependencies=module_dependencies)

        self.benchmarks = benchmarks
示例#3
0
文件: runner.py 项目: uckelman/vbench
    def __init__(self, benchmarks, repo_path, repo_url,
                 build_cmd, db_path, tmp_dir,
                 preparation_cmd,
                 run_option='eod', start_date=None, overwrite=False,
                 module_dependencies=None,
                 always_clean=False,
                 use_blacklist=True,
                 time=True):

        self.benchmarks = benchmarks
        self.checksums = [b.checksum for b in benchmarks]

        self.start_date = start_date
        self.run_option = run_option

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        self.blacklist = set(self.db.get_rev_blacklist())
        self.time = time
        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url, self.tmp_dir, build_cmd,
                                    preparation_cmd,
                                    always_clean=always_clean,
                                    dependencies=module_dependencies)
        self._register_benchmarks()
        self._python = os.environ.get('VBENCH_PYTHON', 'python')
示例#4
0
文件: runner.py 项目: peterjc/vbench
    def __init__(
        self,
        benchmarks,
        repo_path,
        repo_url,
        build_cmd,
        db_path,
        tmp_dir,
        preparation_cmd,
        run_option="end_of_day",
        start_date=None,
        overwrite=False,
    ):

        self.benchmarks = benchmarks
        self.checksums = [b.checksum for b in benchmarks]

        self.start_date = start_date
        self.run_option = run_option

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url, self.tmp_dir, build_cmd, preparation_cmd)

        self._register_benchmarks()
示例#5
0
    def __init__(self,
                 benchmarks,
                 repo_path,
                 repo_url,
                 build_cmd,
                 db_path,
                 tmp_dir,
                 preparation_cmd,
                 run_option='eod',
                 start_date=None,
                 overwrite=False,
                 module_dependencies=None,
                 always_clean=False,
                 use_blacklist=True,
                 time=True):

        self.benchmarks = benchmarks
        self.checksums = [b.checksum for b in benchmarks]

        self.start_date = start_date
        self.run_option = run_option

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        self.blacklist = set(self.db.get_rev_blacklist())
        self.time = time
        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url,
                                    self.tmp_dir,
                                    build_cmd,
                                    preparation_cmd,
                                    always_clean=always_clean,
                                    dependencies=module_dependencies)
        self._register_benchmarks()
        self._python = os.environ.get('VBENCH_PYTHON', 'python')
示例#6
0
文件: runner.py 项目: B-Rich/vbench
    def __init__(
        self,
        benchmarks,
        repo_path,
        repo_url,
        build_cmd,
        db_path,
        tmp_dir,
        prep_cmd,
        clean_cmd=None,
        run_option="eod",
        run_order="normal",
        start_date=None,
        overwrite=False,
        module_dependencies=None,
        always_clean=False,
        use_blacklist=True,
    ):
        log.info("Initializing benchmark runner for %d benchmarks" % (len(benchmarks)))
        self._benchmarks = None
        self._checksums = None

        self.start_date = start_date
        self.run_option = run_option
        self.run_order = run_order

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        self.blacklist = set(self.db.get_rev_blacklist())

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(
            repo_url,
            self.tmp_dir,
            build_cmd,
            prep_cmd,
            clean_cmd,
            always_clean=always_clean,
            dependencies=module_dependencies,
        )

        self.benchmarks = benchmarks
示例#7
0
    def __init__(
        self,
        benchmarks,
        repo_path,
        repo_url,
        build_cmd,
        db_path,
        tmp_dir,
        preparation_cmd,
        run_option="end_of_day",
        start_date=None,
        overwrite=False,
        module_dependencies=None,
        use_blacklist=True,
    ):

        self.benchmarks = benchmarks
        self.checksums = [b.checksum for b in benchmarks]

        self.start_date = start_date
        self.run_option = run_option

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        self.blacklist = set(self.db.get_rev_blacklist())

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(
            repo_url, self.tmp_dir, build_cmd, preparation_cmd, dependencies=module_dependencies
        )
        self._register_benchmarks()
示例#8
0
 def get_results(self, db_path):
     from vbench.db import BenchmarkDB
     db = BenchmarkDB.get_instance(db_path)
     return db.get_benchmark_results(self.checksum)
示例#9
0
def main():
    TMP_DIR = tempfile.mkdtemp()
    prprint("TMP_DIR = %s" % TMP_DIR)
    prprint("LOG_FILE = %s\n" % LOG_FILE)

    try:
        logfile = open(LOG_FILE, 'w')

        prprint("Processing Repo at '%s'..." % REPO_PATH)
        repo = GitRepo(REPO_PATH)

        # get hashes of baseline and current head
        h_head = repo.shas[-1]
        h_baseline = BASELINE_COMMIT

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint('Comparing Head [%s] : %s ' %
                (h_head, repo.messages.get(h_head, "")))
        prprint('Against baseline [%s] : %s \n' %
                (h_baseline, repo.messages.get(h_baseline, "")))

        prprint("Initializing Runner...")
        runner = BenchmarkRunner(
            benchmarks,
            REPO_PATH,
            REPO_PATH,
            BUILD,
            DB_PATH,
            TMP_DIR,
            PREPARE,
            always_clean=True,
            #                             run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        prprint("removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline commit '%s'" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for current HEAD '%s'" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame(dict(t_head=head_res['timing'],
                                t_baseline=baseline_res['timing'],
                                ratio=ratio,
                                name=baseline_res.name),
                           columns=["t_head", "t_baseline", "ratio", "name"])
        totals = totals.ix[totals.t_head > 0.010]  # ignore sub 10micros
        totals = totals.dropna().sort("ratio").set_index(
            'name')  # sort in ascending order

        s = "\n\nResults:\n" + totals.to_string(
            float_format=lambda x: "%0.4f" % x) + "\n\n"
        s += "Columns: test_name | head_time [ms] | baseline_time [ms] | ratio\n\n"
        s += "- a Ratio of 1.30 means HEAD is 30% slower then the Baseline.\n\n"

        s += 'Head [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
        s += 'Baseline [%s] : %s\n\n' % (h_baseline,
                                         repo.messages.get(h_baseline, ""))

        logfile.write(s)
        logfile.close()

        prprint(s)
        prprint("Results were also written to the logfile at '%s'\n" %
                LOG_FILE)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
        logfile.close()
示例#10
0
文件: runner.py 项目: uckelman/vbench
class BenchmarkRunner(object):
    """

    Parameters
    ----------
    benchmarks : list of Benchmark objects
    repo_path
    build_cmd
    db_path
    run_option : {'eod', 'all', 'last', integer}, default: 'eod'
        eod: use the last revision for each calendar day
        all: benchmark every revision
        last: only try to run the last revision
        some integer N: run each N revisions
    overwrite : boolean
    dependencies : list or None
        should be list of modules visible in cwd
    time : boolean
        whether to measure how much running the benchmarks takes
    """

    def __init__(self, benchmarks, repo_path, repo_url,
                 build_cmd, db_path, tmp_dir,
                 preparation_cmd,
                 run_option='eod', start_date=None, overwrite=False,
                 module_dependencies=None,
                 always_clean=False,
                 use_blacklist=True,
                 time=True):

        self.benchmarks = benchmarks
        self.checksums = [b.checksum for b in benchmarks]

        self.start_date = start_date
        self.run_option = run_option

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        self.blacklist = set(self.db.get_rev_blacklist())
        self.time = time
        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url, self.tmp_dir, build_cmd,
                                    preparation_cmd,
                                    always_clean=always_clean,
                                    dependencies=module_dependencies)
        self._register_benchmarks()
        self._python = os.environ.get('VBENCH_PYTHON', 'python')

    def run(self):
        revisions = self._get_revisions_to_run()

        for rev in revisions:
            if self.use_blacklist and rev in self.blacklist:
                print 'SKIPPING BLACKLISTED %s' % rev
                continue

            any_succeeded, n_active = self._run_and_write_results(rev)
            if not any_succeeded and n_active > 0:
                self.bench_repo.hard_clean()

                any_succeeded2, n_active = self._run_and_write_results(rev)

                # just guessing that this revision is broken, should stop
                # wasting our time
                if (not any_succeeded2 and n_active > 5
                    and self.use_blacklist):
                    print 'BLACKLISTING %s' % rev
                    self.db.add_rev_blacklist(rev)

    def _run_and_write_results(self, rev):
        """
        Returns True if any runs succeeded
        """
        n_active_benchmarks, results = self._run_revision(rev)
        tracebacks = []

        any_succeeded = False

        for checksum, timing in results.iteritems():
            if 'traceback' in timing:
                tracebacks.append(timing['traceback'])

            timestamp = self.repo.timestamps[rev]

            any_succeeded = any_succeeded or 'timing' in timing

            self.db.write_result(checksum, rev, timestamp,
                                 timing.get('loops'),
                                 timing.get('timing'),
                                 timing.get('memory'),
                                 timing.get('traceback'))

        return any_succeeded, n_active_benchmarks

    def _register_benchmarks(self):
        ex_benchmarks = self.db.get_benchmarks()
        db_checksums = set(ex_benchmarks.index)
        for bm in self.benchmarks:
            if bm.checksum in db_checksums:
                self.db.update_name(bm)
            else:
                print 'Writing new benchmark %s, %s' % (bm.name, bm.checksum)
                self.db.write_benchmark(bm)

    def _run_revision(self, rev):
        need_to_run = self._get_benchmarks_for_rev(rev)

        if not need_to_run:
            print 'No benchmarks need running at %s' % rev
            return 0, {}

        print 'Running %d benchmarks for revision %s' % (len(need_to_run), rev)
        for bm in need_to_run:
            print bm.name

        self.bench_repo.switch_to_revision(rev)

        pickle_path = os.path.join(self.tmp_dir, 'benchmarks.pickle')
        results_path = os.path.join(self.tmp_dir, 'results.pickle')
        if os.path.exists(results_path):
            os.remove(results_path)
        pickle.dump(need_to_run, open(pickle_path, 'w'))

        # run the process
        cmd = '%s vb_run_benchmarks.py %s %s' % (self._python, pickle_path,
                                                 results_path)
        if self.time:
            cmd = 'time ' + cmd
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                shell=True,
                                cwd=self.tmp_dir)
        stdout, stderr = proc.communicate()

        print 'stdout: %s' % stdout

        if stderr:
            if ("object has no attribute" in stderr or
                'ImportError' in stderr):
                print stderr
                print 'HARD CLEANING!'
                self.bench_repo.hard_clean()
            print stderr

        if not os.path.exists(results_path):
            print 'Failed for revision %s' % rev
            return len(need_to_run), {}

        results = pickle.load(open(results_path, 'r'))

        try:
            os.remove(pickle_path)
        except OSError:
            pass

        return len(need_to_run), results

    def _get_benchmarks_for_rev(self, rev):
        existing_results = self.db.get_rev_results(rev)
        need_to_run = []

        timestamp = self.repo.timestamps[rev]

        for b in self.benchmarks:
            if b.start_date is not None and b.start_date > timestamp:
                continue

            if b.checksum not in existing_results:
                need_to_run.append(b)

        return need_to_run

    def _get_revisions_to_run(self):

        # TODO generalize someday to other vcs...git only for now

        rev_by_timestamp = self.repo.shas.sort_index()

        # # assume they're in order, but check for now
        # assert(rev_by_timestamp.index.is_monotonic)

        if self.start_date is not None:
            rev_by_timestamp = rev_by_timestamp.ix[self.start_date:]

        if self.run_option == 'eod':
            grouped = rev_by_timestamp.groupby(datetime.date)
            revs_to_run = grouped.apply(lambda x: x[-1]).values
        elif self.run_option == 'all':
            revs_to_run = rev_by_timestamp.values
        elif self.run_option == 'last':
            revs_to_run = rev_by_timestamp.values[-1:]
            # TODO: if the very last revision fails, there should be a way
            # to look for the second last, etc, until the last one that was run
        elif isinstance(self.run_option, int):
            revs_to_run = rev_by_timestamp.values[::self.run_option]
        else:
            raise Exception('unrecognized run_option %s' % self.run_option)

        return revs_to_run
示例#11
0
class BenchmarkRunner(object):
    """

    Parameters
    ----------
    benchmarks : list of Benchmark objects
    repo_path
    build_cmd
    db_path
    run_option : {'eod', 'all', integer}
        eod: use the last revision for each calendar day
        all: benchmark every revision
        some integer N: run each N revisions
    overwrite : boolean
    dependencies : list or None
        should be list of modules visible in cwd
    """

    def __init__(
        self,
        benchmarks,
        repo_path,
        repo_url,
        build_cmd,
        db_path,
        tmp_dir,
        preparation_cmd,
        run_option="end_of_day",
        start_date=None,
        overwrite=False,
        module_dependencies=None,
        use_blacklist=True,
    ):

        self.benchmarks = benchmarks
        self.checksums = [b.checksum for b in benchmarks]

        self.start_date = start_date
        self.run_option = run_option

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        self.blacklist = set(self.db.get_rev_blacklist())

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(
            repo_url, self.tmp_dir, build_cmd, preparation_cmd, dependencies=module_dependencies
        )
        self._register_benchmarks()

    def run(self):
        revisions = self._get_revisions_to_run()

        for rev in revisions:
            if self.use_blacklist and rev in self.blacklist:
                print "SKIPPING BLACKLISTED %s" % rev
                continue

            any_succeeded, n_active = self._run_and_write_results(rev)
            if not any_succeeded and n_active > 0:
                self.bench_repo.hard_clean()

                any_succeeded2, n_active = self._run_and_write_results(rev)

                # just guessing that this revision is broken, should stop
                # wasting our time
                if not any_succeeded2 and n_active > 5 and self.use_blacklist:
                    print "BLACKLISTING %s" % rev
                    self.db.add_rev_blacklist(rev)

    def _run_and_write_results(self, rev):
        """
        Returns True if any runs succeeded
        """
        n_active_benchmarks, results = self._run_revision(rev)
        tracebacks = []

        any_succeeded = False

        for checksum, timing in results.iteritems():
            if "traceback" in timing:
                tracebacks.append(timing["traceback"])

            timestamp = self.repo.timestamps[rev]

            any_succeeded = any_succeeded or "timing" in timing

            self.db.write_result(
                checksum, rev, timestamp, timing.get("loops"), timing.get("timing"), timing.get("traceback")
            )

        return any_succeeded, n_active_benchmarks

    def _register_benchmarks(self):
        ex_benchmarks = self.db.get_benchmarks()
        db_checksums = set(ex_benchmarks.index)
        for bm in self.benchmarks:
            if bm.checksum in db_checksums:
                self.db.update_name(bm)
            else:
                print "Writing new benchmark %s, %s" % (bm.name, bm.checksum)
                self.db.write_benchmark(bm)

    def _run_revision(self, rev):
        need_to_run = self._get_benchmarks_for_rev(rev)

        if not need_to_run:
            print "No benchmarks need running at %s" % rev
            return 0, {}

        print "Running %d benchmarks for revision %s" % (len(need_to_run), rev)
        for bm in need_to_run:
            print bm.name

        self.bench_repo.switch_to_revision(rev)

        pickle_path = os.path.join(self.tmp_dir, "benchmarks.pickle")
        results_path = os.path.join(self.tmp_dir, "results.pickle")
        if os.path.exists(results_path):
            os.remove(results_path)
        pickle.dump(need_to_run, open(pickle_path, "w"))

        # run the process
        cmd = "python vb_run_benchmarks.py %s %s" % (pickle_path, results_path)
        print cmd
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=self.tmp_dir)
        stdout, stderr = proc.communicate()

        print "stdout: %s" % stdout

        if stderr:
            if "object has no attribute" in stderr or "ImportError" in stderr:
                print stderr
                print "HARD CLEANING!"
                self.bench_repo.hard_clean()
            print stderr

        if not os.path.exists(results_path):
            print "Failed for revision %s" % rev
            return len(need_to_run), {}

        results = pickle.load(open(results_path, "r"))

        try:
            os.remove(pickle_path)
        except OSError:
            pass

        return len(need_to_run), results

    def _get_benchmarks_for_rev(self, rev):
        existing_results = self.db.get_rev_results(rev)
        need_to_run = []

        timestamp = self.repo.timestamps[rev]

        for b in self.benchmarks:
            if b.start_date is not None and b.start_date > timestamp:
                continue

            if b.checksum not in existing_results:
                need_to_run.append(b)

        return need_to_run

    def _get_revisions_to_run(self):

        # TODO generalize someday to other vcs...git only for now

        rev_by_timestamp = self.repo.shas.sort_index()

        # # assume they're in order, but check for now
        # assert(rev_by_timestamp.index.is_monotonic)

        if self.start_date is not None:
            rev_by_timestamp = rev_by_timestamp.ix[self.start_date :]

        if self.run_option == "eod":
            grouped = rev_by_timestamp.groupby(datetime.date)
            revs_to_run = grouped.apply(lambda x: x[-1]).values
        elif self.run_option == "all":
            revs_to_run = rev_by_timestamp.values
        elif isinstance(self.run_option, int):
            revs_to_run = rev_by_timestamp.values[:: self.run_option]
        else:
            raise Exception("unrecognized run_option %s" % self.run_option)

        return revs_to_run
示例#12
0
class BenchmarkRunner(object):
    """

    Parameters
    ----------
    benchmarks : list of Benchmark objects
    repo_path
    build_cmd
    db_path
    run_option : {'eod', 'all', 'last', integer}, default: 'eod'
        eod: use the last revision for each calendar day
        all: benchmark every revision
        last: only try to run the last revision
        some integer N: run each N revisions
    run_order :
        normal : original order (default)
        reverse: in reverse order (latest first)
        multires: cover all revisions but in the order increasing
                  temporal detail
    overwrite : boolean
    dependencies : list or None
        should be list of modules visible in cwd
    """

    def __init__(self, benchmarks, repo_path, repo_url,
                 build_cmd, db_path, tmp_dir,
                 prep_cmd,
                 clean_cmd=None,
                 run_option='eod', run_order='normal',
                 start_date=None, overwrite=False,
                 module_dependencies=None,
                 always_clean=False,
                 use_blacklist=True):
        log.info("Initializing benchmark runner for %d benchmarks" % (len(benchmarks)))
        self._benchmarks = None
        self._checksums = None

        self.start_date = start_date
        self.run_option = run_option
        self.run_order = run_order

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        self.blacklist = set(self.db.get_rev_blacklist())

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url, self.tmp_dir, build_cmd,
                                    prep_cmd,
                                    clean_cmd,
                                    always_clean=always_clean,
                                    dependencies=module_dependencies)

        self.benchmarks = benchmarks

    def _get_benchmarks(self):
        return self._benchmarks

    def _set_benchmarks(self, benchmarks):
        self._benchmarks = benchmarks
        self._checksums = [b.checksum for b in benchmarks]
        self._register_benchmarks()

    benchmarks = property(fget=_get_benchmarks, fset=_set_benchmarks)
    checksums = property(fget=lambda self:self._checksums)

    def run(self):
        log.info("Collecting revisions to run")
        revisions = self._get_revisions_to_run()
        ran_revisions = []
        log.info("Running benchmarks for %d revisions" % (len(revisions),))
        for rev in revisions:
            if self.use_blacklist and rev in self.blacklist:
                log.warn('Skipping blacklisted %s' % rev)
                continue

            any_succeeded, n_active = self._run_and_write_results(rev)
            ran_revisions.append((rev, (any_succeeded, n_active)))
            log.debug("%s succeeded among %d active benchmarks",
                      {True: "Some", False: "None"}[any_succeeded],
                      n_active)
            if not any_succeeded and n_active > 0:
                self.bench_repo.hard_clean()

                any_succeeded2, n_active = self._run_and_write_results(rev)

                # just guessing that this revision is broken, should stop
                # wasting our time
                if (not any_succeeded2 and n_active > 5
                    and self.use_blacklist):
                    log.warn('Blacklisting %s' % rev)
                    self.db.add_rev_blacklist(rev)
        return ran_revisions

    def _run_and_write_results(self, rev):
        """
        Returns True if any runs succeeded
        """
        n_active_benchmarks, results = self._run_revision(rev)
        tracebacks = []

        any_succeeded = False

        for checksum, timing in results.iteritems():
            if 'traceback' in timing:
                tracebacks.append(timing['traceback'])

            timestamp = self.repo.timestamps[rev]

            any_succeeded = any_succeeded or 'timing' in timing

            self.db.write_result(checksum, rev, timestamp,
                                 timing.get('loops'),
                                 timing.get('timing'),
                                 timing.get('traceback'))

        return any_succeeded, n_active_benchmarks

    def _register_benchmarks(self):
        log.info('Getting benchmarks')
        ex_benchmarks = self.db.get_benchmarks()
        db_checksums = set(ex_benchmarks.index)
        log.info("Registering %d benchmarks" % len(ex_benchmarks))
        for bm in self.benchmarks:
            if bm.checksum in db_checksums:
                self.db.update_name(bm)
            else:
                log.info('Writing new benchmark %s, %s' % (bm.name, bm.checksum))
                self.db.write_benchmark(bm)

    def _run_revision(self, rev):
        need_to_run = self._get_benchmarks_for_rev(rev)

        if not need_to_run:
            log.info('No benchmarks need running at %s' % rev)
            return 0, {}

        log.info('Running %d benchmarks for revision %s' % (len(need_to_run), rev))
        for bm in need_to_run:
            log.debug(bm.name)

        self.bench_repo.switch_to_revision(rev)

        pickle_path = os.path.join(self.tmp_dir, 'benchmarks.pickle')
        results_path = os.path.join(self.tmp_dir, 'results.pickle')
        if os.path.exists(results_path):
            os.remove(results_path)
        pickle.dump(need_to_run, open(pickle_path, 'w'))

        # run the process
        cmd = 'python vb_run_benchmarks.py %s %s' % (pickle_path, results_path)
        log.debug("CMD: %s" % cmd)
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                shell=True,
                                cwd=self.tmp_dir)
        stdout, stderr = proc.communicate()

        if stdout:
            log.debug('stdout: %s' % stdout)

        if proc.returncode:
            log.warn("Returned with non-0 code: %d" % proc.returncode)

        if stderr:
            log.warn("stderr: %s" % stderr)
            if ("object has no attribute" in stderr or
                'ImportError' in stderr):
                log.warn('HARD CLEANING!')
                self.bench_repo.hard_clean()

        if not os.path.exists(results_path):
            log.warn('Failed for revision %s' % rev)
            return len(need_to_run), {}

        results = pickle.load(open(results_path, 'r'))

        try:
            os.remove(pickle_path)
        except OSError:
            pass

        return len(need_to_run), results

    def _get_benchmarks_for_rev(self, rev):
        existing_results = self.db.get_rev_results(rev)
        need_to_run = []

        timestamp = self.repo.timestamps[rev]

        for b in self.benchmarks:
            if b.start_date is not None and b.start_date > timestamp:
                continue

            if b.checksum not in existing_results:
                need_to_run.append(b)

        return need_to_run

    def _get_revisions_to_run(self):

        # TODO generalize someday to other vcs...git only for now

        rev_by_timestamp = self.repo.shas.sort_index()

        # # assume they're in order, but check for now
        # assert(rev_by_timestamp.index.is_monotonic)

        if self.start_date is not None:
            rev_by_timestamp = rev_by_timestamp.ix[self.start_date:]

        if self.run_option == 'eod':
            grouped = rev_by_timestamp.groupby(datetime.date)
            revs_to_run = grouped.apply(lambda x: x[-1]).values
        elif self.run_option == 'all':
            revs_to_run = rev_by_timestamp.values
        elif self.run_option == 'last':
            revs_to_run = rev_by_timestamp.values[-1:]
            # TODO: if the very last revision fails, there should be a way
            # to look for the second last, etc, until the last one that was run
        elif isinstance(self.run_option, int):
            revs_to_run = rev_by_timestamp.values[::self.run_option]
        else:
            raise ValueError('unrecognized run_option=%r' % self.run_option)

        if not self.run_order in _RUN_ORDERS:
            raise ValueError('unrecognized run_order=%r. Must be among %s'
                             % (self.run_order, _RUN_ORDERS.keys()))
        revs_to_run = _RUN_ORDERS[self.run_order](revs_to_run)

        return revs_to_run
示例#13
0
def profile_comparative(benchmarks):

    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from vbench.git import GitRepo
    from suite import BUILD, DB_PATH, PREPARE, dependencies

    TMP_DIR = args.temp_dir or tempfile.mkdtemp()

    try:

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")

        # all in a good cause...
        GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)

        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)
        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint("Slaughtering kittens...")
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
                                                                args.base_commit)

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("Removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)

        report_comparative(head_res,baseline_res)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
示例#14
0
class BenchmarkRunner(object):
    """

    Parameters
    ----------
    benchmarks : list of Benchmark objects
    repo_path
    build_cmd
    db_path
    run_option : {'eod', 'all', 'last', integer}, default: 'eod'
        eod: use the last revision for each calendar day
        all: benchmark every revision
        last: only try to run the last revision
        some integer N: run each N revisions
    run_order :
        normal : original order (default)
        reverse: in reverse order (latest first)
        multires: cover all revisions but in the order increasing
                  temporal detail
    overwrite : boolean
    dependencies : list or None
        should be list of modules visible in cwd
    """
    def __init__(self,
                 benchmarks,
                 repo_path,
                 repo_url,
                 build_cmd,
                 db_path,
                 tmp_dir,
                 prep_cmd,
                 clean_cmd=None,
                 run_option='eod',
                 run_order='normal',
                 start_date=None,
                 overwrite=False,
                 module_dependencies=None,
                 always_clean=False,
                 use_blacklist=True,
                 verify=False):
        log.info("Initializing benchmark runner for %d benchmarks" %
                 (len(benchmarks)))
        self._benchmarks = None
        self._checksums = None

        if verify:
            verify_benchmarks(benchmarks, raise_=True)

        self.start_date = start_date
        self.run_option = run_option
        self.run_order = run_order

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url,
                                    self.tmp_dir,
                                    build_cmd,
                                    prep_cmd,
                                    clean_cmd,
                                    always_clean=always_clean,
                                    dependencies=module_dependencies)

        self.benchmarks = benchmarks

    def _get_benchmarks(self):
        return self._benchmarks

    def _set_benchmarks(self, benchmarks):
        self._benchmarks = benchmarks
        self._checksums = [b.checksum for b in benchmarks]
        self._register_benchmarks()

    benchmarks = property(fget=_get_benchmarks, fset=_set_benchmarks)
    checksums = property(fget=lambda self: self._checksums)

    @property
    def blacklist(self):
        return set(self.db.get_rev_blacklist())

    def _blacklist_rev(self, rev, msg=""):
        if self.use_blacklist:
            log.warn(('Blacklisting %s' % rev) + ": %s" % msg if msg else ".")
            self.db.add_rev_blacklist(rev)

    def run(self):
        log.info("Collecting revisions to run")
        revisions = self._get_revisions_to_run()
        ran_revisions = []
        log.info("Running benchmarks for %d revisions" % (len(revisions), ))
        # get the current black list (might be a different one on a next .run())
        blacklist = self.blacklist
        for rev in revisions:
            if self.use_blacklist and rev in blacklist:
                log.warn('Skipping blacklisted %s' % rev)
                continue

            try:
                any_succeeded, n_active = self._run_and_write_results(rev)
            except FailedToBuildError, e:
                self._blacklist_rev(rev, msg=str(e))
                continue

            # All the rerunning below somewhat obscures the destiny of
            # ran_revisions. TODO: make it clear(er)
            ran_revisions.append((rev, (any_succeeded, n_active)))

            if n_active:
                log.debug("%s succeeded among %d active benchmarks", {
                    True: "Some",
                    False: "None"
                }[any_succeeded], n_active)
                if not any_succeeded:
                    # Give them a second chance
                    self.bench_repo.hard_clean()
                    try:
                        any_succeeded2, n_active2 = self._run_and_write_results(
                            rev)
                    except FailedToBuildError, e:
                        log.warn(
                            "Failed to build upon 2nd attempt to benchmark, "
                            "verify build infrastructure. Skipping for now: %s"
                            % e)
                        continue

                    assert (
                        n_active == n_active2,
                        "Since not any_succeeded, number of benchmarks should remain the same"
                    )
                    # just guessing that this revision is broken, should stop
                    # wasting our time
                    if (not any_succeeded2 and n_active > 5):
                        self._blacklist_rev(
                            rev,
                            "None benchmark among %d has succeeded" % n_active)
示例#15
0
class BenchmarkRunner(object):
    """

    Parameters
    ----------
    benchmarks : list of Benchmark objects
    repo_path
    build_cmd
    db_path
    run_option : {'eod', 'all', 'last', integer}, default: 'eod'
        eod: use the last revision for each calendar day
        all: benchmark every revision
        last: only try to run the last revision
        some integer N: run each N revisions
    run_order :
        normal : original order (default)
        reverse: in reverse order (latest first)
        multires: cover all revisions but in the order increasing
                  temporal detail
    overwrite : boolean
    dependencies : list or None
        should be list of modules visible in cwd
    """

    def __init__(self, benchmarks, repo_path, repo_url,
                 build_cmd, db_path, tmp_dir,
                 prep_cmd,
                 clean_cmd=None,
                 run_option='eod', run_order='normal',
                 start_date=None, overwrite=False,
                 module_dependencies=None,
                 always_clean=False,
                 use_blacklist=True):
        ### @end
        log.info("Initializing benchmark runner for %d benchmarks" % (len(benchmarks)))
        self._benchmarks = None
        self._checksums = None

        self.start_date = start_date
        self.run_option = run_option
        self.run_order = run_order

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        self.blacklist = set(self.db.get_rev_blacklist())

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url, self.tmp_dir, build_cmd,
                                    prep_cmd,
                                    clean_cmd,
                                    always_clean=always_clean,
                                    dependencies=module_dependencies)

        self.benchmarks = benchmarks

    def _get_benchmarks(self):
        return self._benchmarks

    def _set_benchmarks(self, benchmarks):
        self._benchmarks = benchmarks
        self._checksums = [b.checksum for b in benchmarks]
        self._register_benchmarks()

    benchmarks = property(fget=_get_benchmarks, fset=_set_benchmarks)
    checksums = property(fget=lambda self:self._checksums)

    def run(self):
        log.info("Collecting revisions to run")
        revisions = self._get_revisions_to_run()
        ran_revisions = []
        log.info("Running benchmarks for %d revisions" % (len(revisions),))
        for rev in revisions:
            if self.use_blacklist and rev in self.blacklist:
                log.warn('Skipping blacklisted %s' % rev)
                continue

            any_succeeded, n_active = self._run_and_write_results(rev)
            ran_revisions.append((rev, (any_succeeded, n_active)))
            log.debug("%s succeeded among %d active benchmarks",
                      {True: "Some", False: "None"}[any_succeeded],
                      n_active)
            if not any_succeeded and n_active > 0:
                self.bench_repo.hard_clean()

                any_succeeded2, n_active = self._run_and_write_results(rev)

                # just guessing that this revision is broken, should stop
                # wasting our time
                if (not any_succeeded2 and n_active > 5
                    and self.use_blacklist):
                    log.warn('Blacklisting %s' % rev)
                    self.db.add_rev_blacklist(rev)
        return ran_revisions

    def _run_and_write_results(self, rev):
        """
        Returns True if any runs succeeded
        """
        n_active_benchmarks, results = self._run_revision(rev)
        tracebacks = []

        any_succeeded = False

        for checksum, timing in results.iteritems():
            if 'traceback' in timing:
                tracebacks.append(timing['traceback'])

            timestamp = self.repo.timestamps[rev]

            any_succeeded = any_succeeded or 'timing' in timing

            self.db.write_result(checksum, rev, timestamp,
                                 timing.get('loops'),
                                 timing.get('timing'),
                                 timing.get('traceback'))

        return any_succeeded, n_active_benchmarks

    def _register_benchmarks(self):
        log.info('Getting benchmarks')
        ex_benchmarks = self.db.get_benchmarks()
        db_checksums = set(ex_benchmarks.index)
        log.info("Registering %d benchmarks" % len(ex_benchmarks))
        for bm in self.benchmarks:
            if bm.checksum in db_checksums:
                self.db.update_name(bm)
            else:
                log.info('Writing new benchmark %s, %s' % (bm.name, bm.checksum))
                self.db.write_benchmark(bm)

    def _run_revision(self, rev):
        need_to_run = self._get_benchmarks_for_rev(rev)

        if not need_to_run:
            log.info('No benchmarks need running at %s' % rev)
            return 0, {}

        log.info('Running %d benchmarks for revision %s' % (len(need_to_run), rev))
        for bm in need_to_run:
            log.debug(bm.name)

        self.bench_repo.switch_to_revision(rev)

        pickle_path = os.path.join(self.tmp_dir, 'benchmarks.pickle')
        results_path = os.path.join(self.tmp_dir, 'results.pickle')
        if os.path.exists(results_path):
            os.remove(results_path)
        pickle.dump(need_to_run, open(pickle_path, 'w'))

        # run the process
        cmd = 'python vb_run_benchmarks.py %s %s' % (pickle_path, results_path)
        log.debug("CMD: %s" % cmd)
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                shell=True,
                                cwd=self.tmp_dir)
        stdout, stderr = proc.communicate()

        if stdout:
            log.debug('stdout: %s' % stdout)

        if proc.returncode:
            log.warn("Returned with non-0 code: %d" % proc.returncode)

        if stderr:
            log.warn("stderr: %s" % stderr)
            if ("object has no attribute" in stderr or
                'ImportError' in stderr):
                log.warn('HARD CLEANING!')
                self.bench_repo.hard_clean()

        if not os.path.exists(results_path):
            log.warn('Failed for revision %s' % rev)
            return len(need_to_run), {}

        results = pickle.load(open(results_path, 'r'))

        try:
            os.remove(pickle_path)
        except OSError:
            pass

        return len(need_to_run), results

    def _get_benchmarks_for_rev(self, rev):
        existing_results = self.db.get_rev_results(rev)
        need_to_run = []

        timestamp = self.repo.timestamps[rev]

        for b in self.benchmarks:
            if b.start_date is not None and b.start_date > timestamp:
                continue

            if b.checksum not in existing_results:
                need_to_run.append(b)

        return need_to_run

    def _get_revisions_to_run(self):

        # TODO generalize someday to other vcs...git only for now

        rev_by_timestamp = self.repo.shas.sort_index()

        # # assume they're in order, but check for now
        # assert(rev_by_timestamp.index.is_monotonic)

        if self.start_date is not None:
            rev_by_timestamp = rev_by_timestamp.ix[self.start_date:]

        if self.run_option == 'eod':
            grouped = rev_by_timestamp.groupby(datetime.date)
            revs_to_run = grouped.apply(lambda x: x[-1]).values
        elif self.run_option == 'all':
            revs_to_run = rev_by_timestamp.values
        elif self.run_option == 'last':
            revs_to_run = rev_by_timestamp.values[-1:]
            # TODO: if the very last revision fails, there should be a way
            # to look for the second last, etc, until the last one that was run
        elif isinstance(self.run_option, int):
            revs_to_run = rev_by_timestamp.values[::self.run_option]
        else:
            raise ValueError('unrecognized run_option=%r' % self.run_option)

        if not self.run_order in _RUN_ORDERS:
            raise ValueError('unrecognized run_order=%r. Must be among %s'
                             % (self.run_order, _RUN_ORDERS.keys()))
        revs_to_run = _RUN_ORDERS[self.run_order](revs_to_run)

        return revs_to_run
示例#16
0
文件: runner.py 项目: peterjc/vbench
class BenchmarkRunner(object):
    """

    Parameters
    ----------
    benchmarks : list of Benchmark objects
    repo_path
    build_cmd
    db_path
    run_option : {'eod', 'all', integer}
        eod: use the last revision for each calendar day
        all: benchmark every revision
        some integer N: run each N revisions
    overwrite : boolean
    """
    def __init__(self,
                 benchmarks,
                 repo_path,
                 repo_url,
                 build_cmd,
                 db_path,
                 tmp_dir,
                 preparation_cmd,
                 run_option='end_of_day',
                 start_date=None,
                 overwrite=False):

        self.benchmarks = benchmarks
        self.checksums = [b.checksum for b in benchmarks]

        self.start_date = start_date
        self.run_option = run_option

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url, self.tmp_dir, build_cmd,
                                    preparation_cmd)

        self._register_benchmarks()

    def run(self):
        revisions = self._get_revisions_to_run()

        for rev in revisions:
            results = self._run_revision(rev)
            tracebacks = []

            for checksum, timing in results.iteritems():
                if 'traceback' in timing:
                    tracebacks.append(timing['traceback'])

                self.db.write_result(checksum, rev, timing.get('loops'),
                                     timing.get('timing'),
                                     timing.get('traceback'))

            for tb in tracebacks:
                if 'object has no attribute' in tb:
                    print 'HARD CLEANING because of %s' % tb
                    self.bench_repo.hard_clean()
                    break

    def _register_benchmarks(self):
        db_checksums = set(v.checksum for v in self.db.get_benchmarks())
        for bm in self.benchmarks:
            if bm.checksum in db_checksums:
                continue
            print 'Writing new benchmark %s, %s' % (bm.name, bm.checksum)
            self.db.write_benchmark(bm)

    def _run_revision(self, rev):
        need_to_run = self._get_benchmarks_for_rev(rev)

        if not need_to_run:
            print 'No benchmarks need running at %s' % rev
            return {}

        print 'Running %d benchmarks for revision %s' % (len(need_to_run), rev)

        self.bench_repo.switch_to_revision(rev)

        pickle_path = os.path.join(self.tmp_dir, 'benchmarks.pickle')
        results_path = os.path.join(self.tmp_dir, 'results.pickle')
        if os.path.exists(results_path):
            os.remove(results_path)
        pickle.dump(need_to_run, open(pickle_path, 'w'))

        # run the process
        cmd = 'python vb_run_benchmarks.py %s %s' % (pickle_path, results_path)
        print cmd
        proc = subprocess.Popen(cmd,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                shell=True,
                                cwd=self.tmp_dir)
        stdout, stderr = proc.communicate()

        print 'stdout: %s' % stdout

        if stderr:
            if ("object has no attribute" in stderr
                    or 'ImportError' in stderr):
                print 'HARD CLEANING!'
                self.bench_repo.hard_clean()
            print stderr

        if not os.path.exists(results_path):
            print 'Failed for revision %s' % rev
            return {}

        results = pickle.load(open(results_path, 'r'))

        try:
            os.remove(pickle_path)
        except OSError:
            pass

        return results

    def _get_benchmarks_for_rev(self, rev):
        existing_results = self.db.get_rev_results(rev)
        need_to_run = []

        timestamp = self.repo.timestamps[rev]

        for b in self.benchmarks:
            if b.start_date is not None and b.start_date > timestamp:
                continue

            if b.checksum not in existing_results:
                need_to_run.append(b)

        return need_to_run

    def _get_revisions_to_run(self):

        # TODO generalize someday to other vcs...git only for now

        rev_by_timestamp = self.repo.shas.sort_index()

        # # assume they're in order, but check for now
        # assert(rev_by_timestamp.index.is_monotonic)

        if self.start_date is not None:
            rev_by_timestamp = rev_by_timestamp.ix[self.start_date:]

        if self.run_option == 'eod':
            grouped = rev_by_timestamp.groupby(datetime.date)
            revs_to_run = grouped.apply(lambda x: x[-1]).values
        elif self.run_option == 'all':
            revs_to_run = rev_by_timestamp.values
        elif isinstance(self.run_option, int):
            revs_to_run = rev_by_timestamp.values[::self.run_option]
        else:
            raise Exception('unrecognized run_method %s' % self.run_method)

        return revs_to_run
示例#17
0
def profile_comparative(benchmarks):

    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from vbench.git import GitRepo
    from suite import BUILD, DB_PATH, PREPARE, dependencies
    TMP_DIR = tempfile.mkdtemp()

    try:

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")

        # all in a good cause...
        GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)

        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)
        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint("Slaughtering kittens...")
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
                                                                args.base_commit)

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("Removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)

        report_comparative(head_res,baseline_res)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
示例#18
0
def main():
    from pandas import DataFrame
    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from suite import REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies, benchmarks

    if not args.base_commit:
        args.base_commit = BASELINE_COMMIT

    # GitRepo wants exactly 7 character hash?
    args.base_commit = args.base_commit[:7]
    if args.target_commit:
        args.target_commit = args.target_commit[:7]

    if not args.log_file:
        args.log_file = os.path.abspath(
            os.path.join(REPO_PATH, 'vb_suite.log'))

    TMP_DIR = tempfile.mkdtemp()
    prprint("TMP_DIR = %s" % TMP_DIR)
    prprint("LOG_FILE = %s\n" % args.log_file)

    try:
        logfile = open(args.log_file, 'w')

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")
        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint ("Slaughtering kittens..." )
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(REPO_PATH)

        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame(dict(t_head=head_res['timing'],
                                t_baseline=baseline_res['timing'],
                                ratio=ratio,
                                name=baseline_res.name), columns=["t_head", "t_baseline", "ratio", "name"])
        totals = totals.ix[totals.t_head > args.min_duration]
            # ignore below threshold
        totals = totals.dropna(
        ).sort("ratio").set_index('name')  # sort in ascending order

        s = "\n\nResults:\n"
        s += totals.to_string(
            float_format=lambda x: "{:4.4f}".format(x).rjust(10))
        s += "\n\n"
        s += "Columns: test_name | target_duration [ms] | baseline_duration [ms] | ratio\n\n"
        s += "- a Ratio of 1.30 means the target commit is 30% slower then the baseline.\n\n"

        s += 'Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
        s += 'Baseline [%s] : %s\n\n' % (
            h_baseline, repo.messages.get(h_baseline, ""))

        logfile.write(s)
        logfile.close()

        prprint(s)
        prprint("Results were also written to the logfile at '%s'\n" %
                args.log_file)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
        logfile.close()
示例#19
0
class BenchmarkRunner(object):
    """

    Parameters
    ----------
    benchmarks : list of Benchmark objects
    repo_path
    build_cmd
    db_path
    run_option : {'eod', 'all', 'last', integer}, default: 'eod'
        eod: use the last revision for each calendar day
        all: benchmark every revision
        last: only try to run the last revision
        some integer N: run each N revisions
    overwrite : boolean
    dependencies : list or None
        should be list of modules visible in cwd
    """
    def __init__(self,
                 benchmarks,
                 repo_path,
                 repo_url,
                 build_cmd,
                 db_path,
                 tmp_dir,
                 preparation_cmd,
                 run_option='eod',
                 start_date=None,
                 overwrite=False,
                 module_dependencies=None,
                 always_clean=False,
                 use_blacklist=True):

        self.benchmarks = benchmarks
        self.checksums = [b.checksum for b in benchmarks]

        self.start_date = start_date
        self.run_option = run_option

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        self.blacklist = set(self.db.get_rev_blacklist())

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url,
                                    self.tmp_dir,
                                    build_cmd,
                                    preparation_cmd,
                                    always_clean=always_clean,
                                    dependencies=module_dependencies)
        self._register_benchmarks()

    def run(self):
        revisions = self._get_revisions_to_run()

        for rev in revisions:
            if self.use_blacklist and rev in self.blacklist:
                print 'SKIPPING BLACKLISTED %s' % rev
                continue

            any_succeeded, n_active = self._run_and_write_results(rev)
            if not any_succeeded and n_active > 0:
                self.bench_repo.hard_clean()

                any_succeeded2, n_active = self._run_and_write_results(rev)

                # just guessing that this revision is broken, should stop
                # wasting our time
                if (not any_succeeded2 and n_active > 5
                        and self.use_blacklist):
                    print 'BLACKLISTING %s' % rev
                    self.db.add_rev_blacklist(rev)

    def _run_and_write_results(self, rev):
        """
        Returns True if any runs succeeded
        """
        n_active_benchmarks, results = self._run_revision(rev)
        tracebacks = []

        any_succeeded = False

        for checksum, timing in results.iteritems():
            if 'traceback' in timing:
                tracebacks.append(timing['traceback'])

            timestamp = self.repo.timestamps[rev]

            any_succeeded = any_succeeded or 'timing' in timing

            self.db.write_result(checksum, rev, timestamp, timing.get('loops'),
                                 timing.get('timing'), timing.get('traceback'))

        return any_succeeded, n_active_benchmarks

    def _register_benchmarks(self):
        ex_benchmarks = self.db.get_benchmarks()
        db_checksums = set(ex_benchmarks.index)
        for bm in self.benchmarks:
            if bm.checksum in db_checksums:
                self.db.update_name(bm)
            else:
                print 'Writing new benchmark %s, %s' % (bm.name, bm.checksum)
                self.db.write_benchmark(bm)

    def _run_revision(self, rev):
        need_to_run = self._get_benchmarks_for_rev(rev)

        if not need_to_run:
            print 'No benchmarks need running at %s' % rev
            return 0, {}

        print 'Running %d benchmarks for revision %s' % (len(need_to_run), rev)
        for bm in need_to_run:
            print bm.name

        self.bench_repo.switch_to_revision(rev)

        pickle_path = os.path.join(self.tmp_dir, 'benchmarks.pickle')
        results_path = os.path.join(self.tmp_dir, 'results.pickle')
        if os.path.exists(results_path):
            os.remove(results_path)
        pickle.dump(need_to_run, open(pickle_path, 'w'))

        # run the process
        cmd = 'python vb_run_benchmarks.py %s %s' % (pickle_path, results_path)
        print cmd
        proc = subprocess.Popen(cmd,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE,
                                shell=True,
                                cwd=self.tmp_dir)
        stdout, stderr = proc.communicate()

        print 'stdout: %s' % stdout

        if stderr:
            if ("object has no attribute" in stderr
                    or 'ImportError' in stderr):
                print stderr
                print 'HARD CLEANING!'
                self.bench_repo.hard_clean()
            print stderr

        if not os.path.exists(results_path):
            print 'Failed for revision %s' % rev
            return len(need_to_run), {}

        results = pickle.load(open(results_path, 'r'))

        try:
            os.remove(pickle_path)
        except OSError:
            pass

        return len(need_to_run), results

    def _get_benchmarks_for_rev(self, rev):
        existing_results = self.db.get_rev_results(rev)
        need_to_run = []

        timestamp = self.repo.timestamps[rev]

        for b in self.benchmarks:
            if b.start_date is not None and b.start_date > timestamp:
                continue

            if b.checksum not in existing_results:
                need_to_run.append(b)

        return need_to_run

    def _get_revisions_to_run(self):

        # TODO generalize someday to other vcs...git only for now

        rev_by_timestamp = self.repo.shas.sort_index()

        # # assume they're in order, but check for now
        # assert(rev_by_timestamp.index.is_monotonic)

        if self.start_date is not None:
            rev_by_timestamp = rev_by_timestamp.ix[self.start_date:]

        if self.run_option == 'eod':
            grouped = rev_by_timestamp.groupby(datetime.date)
            revs_to_run = grouped.apply(lambda x: x[-1]).values
        elif self.run_option == 'all':
            revs_to_run = rev_by_timestamp.values
        elif self.run_option == 'last':
            revs_to_run = rev_by_timestamp.values[-1:]
            # TODO: if the very last revision fails, there should be a way
            # to look for the second last, etc, until the last one that was run
        elif isinstance(self.run_option, int):
            revs_to_run = rev_by_timestamp.values[::self.run_option]
        else:
            raise Exception('unrecognized run_option %s' % self.run_option)

        return revs_to_run
示例#20
0
def profile_comparative(benchmarks):

    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from vbench.git import GitRepo
    from suite import BUILD, DB_PATH, PREPARE, dependencies
    TMP_DIR = tempfile.mkdtemp()

    try:

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")

        # all in a good cause...
        GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)

        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)
        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint ("Slaughtering kittens..." )
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
                                                                args.base_commit)

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame({HEAD_COL:head_res['timing'],
                                BASE_COL:baseline_res['timing'],
                                'ratio':ratio,
                                'name':baseline_res.name},
                                columns=[HEAD_COL, BASE_COL, "ratio", "name"])
        totals = totals.ix[totals[HEAD_COL] > args.min_duration]
            # ignore below threshold
        totals = totals.dropna(
        ).sort("ratio").set_index('name')  # sort in ascending order

        h_msg =   repo.messages.get(h_head, "")
        b_msg =   repo.messages.get(h_baseline, "")

        print_report(totals,h_head=h_head,h_msg=h_msg,
                     h_baseline=h_baseline,b_msg=b_msg)

        if args.outdf:
            prprint("The results DataFrame was written to '%s'\n" %  args.outdf)
            totals.save(args.outdf)
    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
示例#21
0
文件: test_perf.py 项目: da415/pandas
def main():
    from pandas import DataFrame
    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from vbench.git import GitRepo
    from suite import REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies, benchmarks

    # GitRepo wants exactly 7 character hash?
    args.base_commit = args.base_commit[:7]
    if args.target_commit:
        args.target_commit = args.target_commit[:7]

    if not args.log_file:
        args.log_file = os.path.abspath(
            os.path.join(REPO_PATH, 'vb_suite.log'))

    random.seed(args.seed)
    np.random.seed(args.seed)

    TMP_DIR = tempfile.mkdtemp()
    prprint("TMP_DIR = %s" % TMP_DIR)
    prprint("LOG_FILE = %s\n" % args.log_file)

    benchmarks = [x for x in benchmarks if re.search(args.regex,x.name)]

    try:
        logfile = open(args.log_file, 'w')

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")

        # all in a good cause...
        GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)

        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint ("Slaughtering kittens..." )
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
                                                                args.base_commit)

        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame(dict(t_head=head_res['timing'],
                                t_baseline=baseline_res['timing'],
                                ratio=ratio,
                                name=baseline_res.name), columns=["t_head", "t_baseline", "ratio", "name"])
        totals = totals.ix[totals.t_head > args.min_duration]
            # ignore below threshold
        totals = totals.dropna(
        ).sort("ratio").set_index('name')  # sort in ascending order


        hdr = ftr = """
-----------------------------------------------------------------------
Test name                      | target[ms] |  base[ms]  |   ratio    |
-----------------------------------------------------------------------
""".strip() +"\n"

        s = "\n"
        s += hdr
        for i in range(len(totals)):
            t,b,r = totals.irow(i).values
            s += "{0:30s} {1: 12.4f} {2: 12.4f} {3: 12.4f}\n".format(totals.index[i],t,b,r)
        s+= ftr + "\n"

        s += "Ratio < 1.0 means the target commit is faster then the baseline.\n"
        s += "Seed used: %d\n\n" % args.seed

        s += 'Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
        s += 'Base   [%s] : %s\n\n' % (
            h_baseline, repo.messages.get(h_baseline, ""))

        logfile.write(s)
        logfile.close()

        prprint(s)
        prprint("Results were also written to the logfile at '%s'\n" %
                args.log_file)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
        logfile.close()
示例#22
0
文件: test_perf.py 项目: esc/pandas
def profile_comparative(benchmarks):

    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from vbench.git import GitRepo
    from suite import BUILD, DB_PATH, PREPARE, dependencies
    TMP_DIR = tempfile.mkdtemp()

    try:

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")

        # all in a good cause...
        GitRepo._parse_commit_log = _parse_wrapper(args.base_commit)

        runner = BenchmarkRunner(
            benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
            TMP_DIR, PREPARE, always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)
        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint ("Slaughtering kittens..." )
        (repo.shas, repo.messages,
         repo.timestamps, repo.authors) = _parse_commit_log(None,REPO_PATH,
                                                                args.base_commit)

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' % (h_baseline,
                repo.messages.get(h_baseline, "")))

        prprint("Removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame({HEAD_COL:head_res['timing'],
                                BASE_COL:baseline_res['timing'],
                                'ratio':ratio,
                                'name':baseline_res.name},
                                columns=[HEAD_COL, BASE_COL, "ratio", "name"])
        totals = totals.ix[totals[HEAD_COL] > args.min_duration]
            # ignore below threshold
        totals = totals.dropna(
        ).sort("ratio").set_index('name')  # sort in ascending order

        h_msg =   repo.messages.get(h_head, "")
        b_msg =   repo.messages.get(h_baseline, "")

        print_report(totals,h_head=h_head,h_msg=h_msg,
                     h_baseline=h_baseline,b_msg=b_msg)

        if args.outdf:
            prprint("The results DataFrame was written to '%s'\n" %  args.outdf)
            totals.save(args.outdf)
    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
示例#23
0
 def get_results(self, db_path):
     from vbench.db import BenchmarkDB
     db = BenchmarkDB.get_instance(db_path)
     return db.get_benchmark_results(self.checksum)
示例#24
0
def main():
    from pandas import DataFrame
    from vbench.api import BenchmarkRunner
    from vbench.db import BenchmarkDB
    from suite import REPO_PATH, BUILD, DB_PATH, PREPARE, dependencies, benchmarks

    if not args.base_commit:
        args.base_commit = BASELINE_COMMIT

    # GitRepo wants exactly 7 character hash?
    args.base_commit = args.base_commit[:7]
    if args.target_commit:
        args.target_commit = args.target_commit[:7]

    if not args.log_file:
        args.log_file = os.path.abspath(os.path.join(REPO_PATH,
                                                     'vb_suite.log'))

    TMP_DIR = tempfile.mkdtemp()
    prprint("TMP_DIR = %s" % TMP_DIR)
    prprint("LOG_FILE = %s\n" % args.log_file)

    try:
        logfile = open(args.log_file, 'w')

        prprint("Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint("Initializing Runner...")
        runner = BenchmarkRunner(
            benchmarks,
            REPO_PATH,
            REPO_PATH,
            BUILD,
            DB_PATH,
            TMP_DIR,
            PREPARE,
            always_clean=True,
            # run_option='eod', start_date=START_DATE,
            module_dependencies=dependencies)

        repo = runner.repo  # (steal the parsed git repo used by runner)

        # ARGH. reparse the repo, without discarding any commits,
        # then overwrite the previous parse results
        # prprint ("Slaughtering kittens..." )
        (repo.shas, repo.messages, repo.timestamps,
         repo.authors) = _parse_commit_log(REPO_PATH)

        h_head = args.target_commit or repo.shas[-1]
        h_baseline = args.base_commit

        prprint('Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, "")))
        prprint('Baseline [%s] : %s\n' %
                (h_baseline, repo.messages.get(h_baseline, "")))

        prprint("removing any previous measurements for the commits.")
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint("Running benchmarks for baseline [%s]" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint("Running benchmarks for target [%s]" % h_head)
        runner._run_and_write_results(h_head)

        prprint('Processing results...')

        head_res = get_results_df(db, h_head)
        baseline_res = get_results_df(db, h_baseline)
        ratio = head_res['timing'] / baseline_res['timing']
        totals = DataFrame(dict(t_head=head_res['timing'],
                                t_baseline=baseline_res['timing'],
                                ratio=ratio,
                                name=baseline_res.name),
                           columns=["t_head", "t_baseline", "ratio", "name"])
        totals = totals.ix[totals.t_head > args.min_duration]
        # ignore below threshold
        totals = totals.dropna().sort("ratio").set_index(
            'name')  # sort in ascending order

        s = "\n\nResults:\n"
        s += totals.to_string(
            float_format=lambda x: "{:4.4f}".format(x).rjust(10))
        s += "\n\n"
        s += "Columns: test_name | target_duration [ms] | baseline_duration [ms] | ratio\n\n"
        s += "- a Ratio of 1.30 means the target commit is 30% slower then the baseline.\n\n"

        s += 'Target [%s] : %s\n' % (h_head, repo.messages.get(h_head, ""))
        s += 'Baseline [%s] : %s\n\n' % (h_baseline,
                                         repo.messages.get(h_baseline, ""))

        logfile.write(s)
        logfile.close()

        prprint(s)
        prprint("Results were also written to the logfile at '%s'\n" %
                args.log_file)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
        logfile.close()
示例#25
0
def main():
    TMP_DIR =  tempfile.mkdtemp()
    prprint("TMP_DIR = %s" % TMP_DIR)
    prprint("LOG_FILE = %s\n" % LOG_FILE)

    try:
        logfile = open(LOG_FILE, 'w')

        prprint( "Processing Repo at '%s'..." % REPO_PATH)
        repo = GitRepo(REPO_PATH)

        # get hashes of baseline and current head
        h_head = repo.shas[-1]
        h_baseline = BASELINE_COMMIT

        prprint( "Opening DB at '%s'...\n" % DB_PATH)
        db = BenchmarkDB(DB_PATH)

        prprint( 'Comparing Head [%s] : %s ' % (h_head, repo.messages.get(h_head,"")))
        prprint( 'Against baseline [%s] : %s \n' % (h_baseline,
                repo.messages.get(h_baseline,"")))

        prprint("Initializing Runner...")
        runner = BenchmarkRunner(benchmarks, REPO_PATH, REPO_PATH, BUILD, DB_PATH,
                                 TMP_DIR, PREPARE, always_clean=True,
            #                             run_option='eod', start_date=START_DATE,
                                 module_dependencies=dependencies)

        prprint ("removing any previous measurements for the commits." )
        db.delete_rev_results(h_baseline)
        db.delete_rev_results(h_head)

        # TODO: we could skip this, but we need to make sure all
        # results are in the DB, which is a little tricky with
        # start dates and so on.
        prprint( "Running benchmarks for baseline commit '%s'" % h_baseline)
        runner._run_and_write_results(h_baseline)

        prprint ("Running benchmarks for current HEAD '%s'" % h_head)
        runner._run_and_write_results(h_head)

        prprint( 'Processing results...')

        head_res = get_results_df(db,h_head)
        baseline_res = get_results_df(db,h_baseline)
        ratio = head_res['timing']/baseline_res['timing']
        totals = DataFrame(dict(t_head=head_res['timing'],
                                t_baseline=baseline_res['timing'],
                                ratio=ratio,
                                name=baseline_res.name),columns=["t_head","t_baseline","ratio","name"])
        totals = totals.ix[totals.t_head > 0.010] # ignore sub 10micros
        totals = totals.dropna().sort("ratio").set_index('name') # sort in ascending order

        s = "\n\nResults:\n" + totals.to_string(float_format=lambda x: "%0.4f" %x) + "\n\n"
        s += "Columns: test_name | head_time [ms] | baseline_time [ms] | ratio\n\n"
        s += "- a Ratio of 1.30 means HEAD is 30% slower then the Baseline.\n\n"

        s += 'Head [%s] : %s\n' % (h_head, repo.messages.get(h_head,""))
        s += 'Baseline [%s] : %s\n\n' % (h_baseline,repo.messages.get(h_baseline,""))

        logfile.write(s)
        logfile.close()

        prprint(s )
        prprint("Results were also written to the logfile at '%s'\n" % LOG_FILE)

    finally:
        #        print("Disposing of TMP_DIR: %s" % TMP_DIR)
        shutil.rmtree(TMP_DIR)
        logfile.close()
示例#26
0
文件: runner.py 项目: chebee7i/vbench
class BenchmarkRunner(object):
    """

    Parameters
    ----------
    benchmarks : list of Benchmark objects
    repo_path
    build_cmd
    db_path
    run_option : {'eod', 'all', 'last', integer}, default: 'eod'
        eod: use the last revision for each calendar day
        all: benchmark every revision
        last: only try to run the last revision
        some integer N: run each N revisions
    run_order :
        normal : original order (default)
        reverse: in reverse order (latest first)
        multires: cover all revisions but in the order increasing
                  temporal detail
    overwrite : boolean
    dependencies : list or None
        should be list of modules visible in cwd
    """

    def __init__(self, benchmarks, repo_path, repo_url,
                 build_cmd, db_path, tmp_dir,
                 prep_cmd,
                 clean_cmd=None,
                 run_option='eod', run_order='normal',
                 start_date=None, overwrite=False,
                 module_dependencies=None,
                 always_clean=False,
                 use_blacklist=True,
                 verify=False):
        log.info("Initializing benchmark runner for %d benchmarks" % (len(benchmarks)))
        self._benchmarks = None
        self._checksums = None

        if verify:
            verify_benchmarks(benchmarks, raise_=True)

        self.start_date = start_date
        self.run_option = run_option
        self.run_order = run_order

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        self.use_blacklist = use_blacklist

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url, self.tmp_dir, build_cmd,
                                    prep_cmd,
                                    clean_cmd,
                                    always_clean=always_clean,
                                    dependencies=module_dependencies)

        self.benchmarks = benchmarks

    def _get_benchmarks(self):
        return self._benchmarks

    def _set_benchmarks(self, benchmarks):
        self._benchmarks = benchmarks
        self._checksums = [b.checksum for b in benchmarks]
        self._register_benchmarks()

    benchmarks = property(fget=_get_benchmarks, fset=_set_benchmarks)
    checksums = property(fget=lambda self:self._checksums)

    @property
    def blacklist(self):
        return set(self.db.get_rev_blacklist())

    def _blacklist_rev(self, rev, msg=""):
        if self.use_blacklist:
            log.warn(('Blacklisting %s' % rev) + ": %s" % msg if msg else ".")
            self.db.add_rev_blacklist(rev)

    def run(self):
        log.info("Collecting revisions to run")
        revisions = self._get_revisions_to_run()
        ran_revisions = []
        log.info("Running benchmarks for %d revisions" % (len(revisions),))
        # get the current black list (might be a different one on a next .run())
        blacklist = self.blacklist
        for rev in revisions:
            if self.use_blacklist and rev in blacklist:
                log.warn('Skipping blacklisted %s' % rev)
                continue

            try:
                any_succeeded, n_active = self._run_and_write_results(rev)
            except FailedToBuildError, e:
                self._blacklist_rev(rev, msg=str(e))
                continue

            # All the rerunning below somewhat obscures the destiny of
            # ran_revisions. TODO: make it clear(er)
            ran_revisions.append((rev, (any_succeeded, n_active)))

            if n_active:
                log.debug("%s succeeded among %d active benchmarks",
                          {True: "Some", False: "None"}[any_succeeded],
                          n_active)
                if not any_succeeded:
                    # Give them a second chance
                    self.bench_repo.hard_clean()
                    try:
                        any_succeeded2, n_active2 = self._run_and_write_results(rev)
                    except FailedToBuildError, e:
                        log.warn("Failed to build upon 2nd attempt to benchmark, "
                                 "verify build infrastructure. Skipping for now: %s" % e)
                        continue

                    assert(n_active == n_active2,
                           "Since not any_succeeded, number of benchmarks should remain the same")
                    # just guessing that this revision is broken, should stop
                    # wasting our time
                    if (not any_succeeded2 and n_active > 5):
                        self._blacklist_rev(rev, "None benchmark among %d has succeeded" % n_active)
示例#27
0
文件: runner.py 项目: peterjc/vbench
class BenchmarkRunner(object):
    """

    Parameters
    ----------
    benchmarks : list of Benchmark objects
    repo_path
    build_cmd
    db_path
    run_option : {'eod', 'all', integer}
        eod: use the last revision for each calendar day
        all: benchmark every revision
        some integer N: run each N revisions
    overwrite : boolean
    """

    def __init__(
        self,
        benchmarks,
        repo_path,
        repo_url,
        build_cmd,
        db_path,
        tmp_dir,
        preparation_cmd,
        run_option="end_of_day",
        start_date=None,
        overwrite=False,
    ):

        self.benchmarks = benchmarks
        self.checksums = [b.checksum for b in benchmarks]

        self.start_date = start_date
        self.run_option = run_option

        self.repo_path = repo_path
        self.db_path = db_path

        self.repo = GitRepo(self.repo_path)
        self.db = BenchmarkDB(db_path)

        # where to copy the repo
        self.tmp_dir = tmp_dir
        self.bench_repo = BenchRepo(repo_url, self.tmp_dir, build_cmd, preparation_cmd)

        self._register_benchmarks()

    def run(self):
        revisions = self._get_revisions_to_run()

        for rev in revisions:
            results = self._run_revision(rev)
            tracebacks = []

            for checksum, timing in results.iteritems():
                if "traceback" in timing:
                    tracebacks.append(timing["traceback"])

                self.db.write_result(checksum, rev, timing.get("loops"), timing.get("timing"), timing.get("traceback"))

            for tb in tracebacks:
                if "object has no attribute" in tb:
                    print "HARD CLEANING because of %s" % tb
                    self.bench_repo.hard_clean()
                    break

    def _register_benchmarks(self):
        db_checksums = set(v.checksum for v in self.db.get_benchmarks())
        for bm in self.benchmarks:
            if bm.checksum in db_checksums:
                continue
            print "Writing new benchmark %s, %s" % (bm.name, bm.checksum)
            self.db.write_benchmark(bm)

    def _run_revision(self, rev):
        need_to_run = self._get_benchmarks_for_rev(rev)

        if not need_to_run:
            print "No benchmarks need running at %s" % rev
            return {}

        print "Running %d benchmarks for revision %s" % (len(need_to_run), rev)

        self.bench_repo.switch_to_revision(rev)

        pickle_path = os.path.join(self.tmp_dir, "benchmarks.pickle")
        results_path = os.path.join(self.tmp_dir, "results.pickle")
        if os.path.exists(results_path):
            os.remove(results_path)
        pickle.dump(need_to_run, open(pickle_path, "w"))

        # run the process
        cmd = "python vb_run_benchmarks.py %s %s" % (pickle_path, results_path)
        print cmd
        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, cwd=self.tmp_dir)
        stdout, stderr = proc.communicate()

        print "stdout: %s" % stdout

        if stderr:
            if "object has no attribute" in stderr or "ImportError" in stderr:
                print "HARD CLEANING!"
                self.bench_repo.hard_clean()
            print stderr

        if not os.path.exists(results_path):
            print "Failed for revision %s" % rev
            return {}

        results = pickle.load(open(results_path, "r"))

        try:
            os.remove(pickle_path)
        except OSError:
            pass

        return results

    def _get_benchmarks_for_rev(self, rev):
        existing_results = self.db.get_rev_results(rev)
        need_to_run = []

        timestamp = self.repo.timestamps[rev]

        for b in self.benchmarks:
            if b.start_date is not None and b.start_date > timestamp:
                continue

            if b.checksum not in existing_results:
                need_to_run.append(b)

        return need_to_run

    def _get_revisions_to_run(self):

        # TODO generalize someday to other vcs...git only for now

        rev_by_timestamp = self.repo.shas.sort_index()

        # # assume they're in order, but check for now
        # assert(rev_by_timestamp.index.is_monotonic)

        if self.start_date is not None:
            rev_by_timestamp = rev_by_timestamp.ix[self.start_date :]

        if self.run_option == "eod":
            grouped = rev_by_timestamp.groupby(datetime.date)
            revs_to_run = grouped.apply(lambda x: x[-1]).values
        elif self.run_option == "all":
            revs_to_run = rev_by_timestamp.values
        elif isinstance(self.run_option, int):
            revs_to_run = rev_by_timestamp.values[:: self.run_option]
        else:
            raise Exception("unrecognized run_method %s" % self.run_method)

        return revs_to_run