def get_django_version(loc, vcs=None): if vcs: switch_to_branch(vcs, loc, do_cleanup=True) pythonpath = Path.cwd() else: pythonpath = Path(loc).absolute() out, err, _ = perf.CallAndCaptureOutput( [sys.executable, "-c" "import django; print django.get_version()"], env={"PYTHONPATH": pythonpath} ) return out.strip()
def get_widgy_version(loc, vcs=None): if vcs: switch_to_branch(vcs, loc, do_cleanup=True) pythonpath = Path.cwd() else: pythonpath = Path(loc).absolute() out, err, _ = perf.CallAndCaptureOutput( [sys.executable, '-c' 'import widgy; print widgy.get_version()'], env={'PYTHONPATH': pythonpath}) return out.strip()
def get_widgy_version(loc, vcs=None): if vcs: switch_to_branch(vcs, loc, do_cleanup=True) pythonpath = Path.cwd() else: pythonpath = Path(loc).absolute() out, err, _ = perf.CallAndCaptureOutput( [sys.executable, '-c' 'import widgy; print widgy.get_version()'], env = {'PYTHONPATH': pythonpath} ) return out.strip()
def _make_cli(tmpdir, request): """ Set up a CLI instance pointing to a tmpdir for a library. This can't be in the body of cli() itself because we want to call it both from the cli fixture and from TestBuild.build. """ cli = TestCLI( stdout = io.StringIO(), stderr = io.StringIO(), libpath = str(tmpdir), ) # Chdir to the temp dir, remembering to restore when we're done. oldcwd = Path.cwd() os.chdir(str(tmpdir)) request.addfinalizer(lambda: os.chdir(oldcwd)) return cli
def run_benchmarks(control, experiment, benchmark_dir, benchmarks, trials, vcs=None, record_dir=None, profile_dir=None, continue_on_error=False): if benchmarks: print "Running benchmarks: %s" % " ".join(benchmarks) else: print "Running all benchmarks" if record_dir: record_dir = Path(record_dir).expand().absolute() if not record_dir.exists(): raise ValueError('Recording directory "%s" does not exist' % record_dir) print "Recording data to '%s'" % record_dir control_label = get_widgy_version(control, vcs=vcs) experiment_label = get_widgy_version(experiment, vcs=vcs) branch_info = "%s branch " % vcs if vcs else "" print "Control: Widgy %s (in %s%s)" % (control_label, branch_info, control) print "Experiment: Widgy %s (in %s%s)" % (experiment_label, branch_info, experiment) print # Calculate the subshell envs that we'll use to execute the # benchmarks in. if vcs: control_env = { 'PYTHONPATH': '%s:%s' % (Path.cwd().absolute(), Path(benchmark_dir)), } experiment_env = control_env.copy() else: control_env = { 'PYTHONPATH': '%s:%s' % (Path(control).absolute(), Path(benchmark_dir)) } experiment_env = { 'PYTHONPATH': '%s:%s' % (Path(experiment).absolute(), Path(benchmark_dir)) } for benchmark in discover_benchmarks(benchmark_dir): if not benchmarks or benchmark.name in benchmarks: print "Running '%s' benchmark ..." % benchmark.name settings_mod = '%s.settings' % benchmark.name control_env['DJANGO_SETTINGS_MODULE'] = settings_mod experiment_env['DJANGO_SETTINGS_MODULE'] = settings_mod if profile_dir is not None: control_env['DJANGOBENCH_PROFILE_FILE'] = Path( profile_dir, "con-%s" % benchmark.name) experiment_env['DJANGOBENCH_PROFILE_FILE'] = Path( profile_dir, "exp-%s" % benchmark.name) try: if vcs: switch_to_branch(vcs, control) control_data = run_benchmark(benchmark, trials, control_env) if vcs: switch_to_branch(vcs, experiment) experiment_data = run_benchmark(benchmark, trials, experiment_env) except SkipBenchmark, reason: print "Skipped: %s\n" % reason continue except RuntimeError, error: if continue_on_error: print "Failed: %s\n" % error continue raise options = argparse.Namespace( track_memory=False, diff_instrumentation=False, benchmark_name=benchmark.name, disable_timelines=True, control_label=control_label, experiment_label=experiment_label, ) result = perf.CompareBenchmarkData(control_data, experiment_data, options) if record_dir: record_benchmark_results( dest=record_dir.child('%s.json' % benchmark.name), name=benchmark.name, result=result, control=control_label, experiment=experiment_label, control_data=control_data, experiment_data=experiment_data, ) print format_benchmark_result(result, len(control_data.runtimes)) print
def run_benchmarks(control, experiment, benchmark_dir, benchmarks, trials, vcs=None, record_dir=None, profile_dir=None): if benchmarks: print "Running benchmarks: %s" % " ".join(benchmarks) else: print "Running all benchmarks" if record_dir: record_dir = Path(record_dir).expand().absolute() if not record_dir.exists(): raise ValueError('Recording directory "%s" does not exist' % record_dir) print "Recording data to '%s'" % record_dir control_label = get_django_version(control, vcs=vcs) experiment_label = get_django_version(experiment, vcs=vcs) branch_info = "%s branch " % vcs if vcs else "" print "Control: Django %s (in %s%s)" % (control_label, branch_info, control) print "Experiment: Django %s (in %s%s)" % (experiment_label, branch_info, experiment) print # Calculate the subshell envs that we'll use to execute the # benchmarks in. if vcs: control_env = experiment_env = { 'PYTHONPATH': '%s:%s' % (Path.cwd().absolute(), Path(benchmark_dir)), } else: control_env = {'PYTHONPATH': '%s:%s' % (Path(control).absolute(), Path(benchmark_dir))} experiment_env = {'PYTHONPATH': '%s:%s' % (Path(experiment).absolute(), Path(benchmark_dir))} for benchmark in discover_benchmarks(benchmark_dir): if not benchmarks or benchmark.name in benchmarks: print "Running '%s' benchmark ..." % benchmark.name settings_mod = '%s.settings' % benchmark.name control_env['DJANGO_SETTINGS_MODULE'] = settings_mod experiment_env['DJANGO_SETTINGS_MODULE'] = settings_mod if profile_dir is not None: control_env['DJANGOBENCH_PROFILE_FILE'] = Path(profile_dir, "con-%s" % benchmark.name) experiment_env['DJANGOBENCH_PROFILE_FILE'] = Path(profile_dir, "exp-%s" % benchmark.name) try: if vcs: switch_to_branch(vcs, control) control_data = run_benchmark(benchmark, trials, control_env) if vcs: switch_to_branch(vcs, experiment) experiment_data = run_benchmark(benchmark, trials, experiment_env) except SkipBenchmark, reason: print "Skipped: %s\n" % reason continue options = argparse.Namespace( track_memory = False, diff_instrumentation = False, benchmark_name = benchmark.name, disable_timelines = True, control_label = control_label, experiment_label = experiment_label, ) result = perf.CompareBenchmarkData(control_data, experiment_data, options) if record_dir: record_benchmark_results( dest = record_dir.child('%s.json' % benchmark.name), name = benchmark.name, result = result, control = control_label, experiment = experiment_label, control_data = control_data, experiment_data = experiment_data, ) print format_benchmark_result(result, len(control_data.runtimes)) print
def __init__(self, cwd): self.prev_cwd = FSPath.cwd() self.cwd = Path(cwd) if not self.cwd.exists(): self.cwd.mkdir(parents=True)
def run_benchmarks(control, experiment, benchmark_dir, benchmarks, trials, record_dir, profile_dir): if benchmarks: print "Running benchmarks: %s" % " ".join(benchmarks) else: print "Running all benchmarks" if record_dir: record_dir = Path(record_dir).expand().absolute() if not record_dir.exists(): raise ValueError('Recording directory "%s" does not exist' % record_dir) print "Recording data to '%s'" % record_dir print "Control: %s" % control print "Experiment: %s" % experiment print control_env_dir = tempfile.mkdtemp() experiment_env_dir = tempfile.mkdtemp() # create envs virtualenv.create_environment(control_env_dir, False) virtualenv.create_environment(experiment_env_dir, False) control_python = setup_env(control_env_dir, control) experiment_python = setup_env(experiment_env_dir, experiment) control_env = { 'PYTHONPATH': '%s:%s:%s' % (Path.cwd().absolute(), Path(benchmark_dir), Path(__file__).parent.parent.absolute()), } experiment_env = control_env.copy() for benchmark in discover_benchmarks(benchmark_dir): if not benchmarks or benchmark.name in benchmarks: print "Running '%s' benchmark ..." % benchmark.name settings_mod = '%s.settings' % benchmark.name control_env['DJANGO_SETTINGS_MODULE'] = settings_mod experiment_env['DJANGO_SETTINGS_MODULE'] = settings_mod if profile_dir is not None: control_env['DJANGOBENCH_PROFILE_FILE'] = Path(profile_dir, "con-%s" % benchmark.name) experiment_env['DJANGOBENCH_PROFILE_FILE'] = Path(profile_dir, "exp-%s" % benchmark.name) try: control_data = run_benchmark(benchmark, trials, control_env, control_python) experiment_data = run_benchmark(benchmark, trials, experiment_env, experiment_python) except SkipBenchmark, reason: print "Skipped: %s\n" % reason continue options = argparse.Namespace( track_memory = False, diff_instrumentation = False, benchmark_name = benchmark.name, disable_timelines = True, control_label = control, experiment_label = experiment, ) result = perf.CompareBenchmarkData(control_data, experiment_data, options) if record_dir: record_benchmark_results( dest = record_dir.child('%s.json' % benchmark.name), name = benchmark.name, result = result, control = control, experiment = experiment, control_data = control_data, experiment_data = experiment_data, ) print format_benchmark_result(result, len(control_data.runtimes)) print