def test_find_branch_tips(dummy_runs: List[Run], stored_runs: Storage):
tips = stored_runs.find_branch_tips()
assert tips["main"] == dummy_runs[int(len(dummy_runs) / 2 - 1)].commit
assert tips["feature"] == dummy_runs[-1].commit
assert stored_runs.find_branch_tips() == stored_runs.find_branch_tips_slow(
)
def make_test_data(spec_file: typer.FileText, n: int = typer.Option(1, "-n")):
runs = generate_dummy_data(42, n, ["main", "feature_a", "feature_b"])
spec = load_spec(spec_file)
storage = Storage(spec.storage_dir)
for run in runs:
storage.store_run(run)
def stored_runs(dummy_runs, tmp_path) -> Storage:
storage_dir = tmp_path / "storage"
storage_dir.mkdir()
storage = Storage(storage_dir)
for run in dummy_runs:
storage.store_run(run)
return storage
def test_iterate(dummy_runs: List[Run], stored_runs: Storage):
mid = int(len(dummy_runs) / 2)
act = list(stored_runs.iterate(dummy_runs[mid - 1].commit))
exp = list(reversed(dummy_runs[:mid]))
assert exp == act
act = list(stored_runs.iterate(dummy_runs[-1].commit))
exp = list(reversed(dummy_runs[mid:]))
assert exp == act
def do_list(
spec_file: typer.FileText,
) -> None:
spec = load_spec(spec_file)
storage = Storage(spec.storage_dir)
tips = storage.find_branch_tips()
for key, tip in tips.items():
print(key)
for i in storage.iterate(tip):
print("-", i.commit.hash[:8], i.commit.date, i.commit.message)
def test_get_branch_tip(stored_runs: Storage, dummy_runs: List[Run]) -> None:
mid = int(len(dummy_runs) / 2)
exp1 = dummy_runs[mid - 1]
exp2 = dummy_runs[-1]
act1 = stored_runs.get_branch_tip(exp1.branch)
assert act1 == exp1.commit
act2 = stored_runs.get_branch_tip(exp2.branch)
assert act2 == exp2.commit
slow = stored_runs.find_branch_tips_slow()
for branch, exp in slow.items():
act = stored_runs.get_branch_tip(branch)
assert act == exp
def collect_cmd(
spec_file: typer.FileText,
jobs: int = typer.Option(1, "--jobs", "-j"),
commit_in: str = typer.Option(
get_current_commit().hash, "--commit", show_default=True
),
branch: str = typer.Option(get_branch(), "--branch", show_default=True),
) -> None:
spec = load_spec(spec_file)
storage = Storage(spec.storage_dir)
commit = Commit(
hash=str(commit_in),
date=get_commit_date(commit_in),
message=get_commit_message(commit_in),
)
# parent = Commit(hash=str(parent_in), date=get_commit_date(parent_in))
parent = storage.get_branch_tip(get_branch())
assert commit != parent, "We ran on this commit before it seems"
msg.info(f"#jobs: {jobs}")
msg.info(f"on commit: {commit}")
msg.info(f"parent commit: {parent}")
if jobs > 1:
msg.warn(
"If you're running benchmarks from the collect call,"
" concurrency can affect results"
)
assert jobs > 0, "Jobs value must be positive"
msg.good("Spec loaded successfully")
msg.divider()
try:
results = run_collectors(spec.collectors, jobs=jobs)
except CollectorError as e:
msg.fail("Collector returned invalid format")
typer.echo(str(e.exc))
return
# raise e
msg.good("Collection completed")
# print(results)
run = Run(
commit=commit,
parent=parent,
branch=branch,
date=datetime.now(),
results=sum((r.metrics for r in results), []),
context={},
)
# print(run)
storage = Storage(spec.storage_dir)
storage.store_run(run)
def publish(
spec_file: typer.FileText,
output: Path,
) -> None:
spec = load_spec(spec_file)
storage = Storage(spec.storage_dir)
if not output.exists():
output.mkdir(parents=True)
make_report(spec, storage, output)
def test_run(stored_runs: Storage, dummy_runs: List[Run]):
# assert len(dummy_runs) == 2*10
mid = int(len(dummy_runs) / 2)
tips = stored_runs.find_branch_tips()
loaded_runs = sum(
[list(reversed(list(stored_runs.iterate(t)))) for t in tips.values()],
[])
print()
print(len(loaded_runs))
assert loaded_runs[0].parent is None
assert loaded_runs[mid].parent is None
for run_prev, run in zip(loaded_runs[:mid], loaded_runs[1:mid]):
assert run_prev.commit == run.parent
assert run.parent is not None
for run_prev, run in zip(loaded_runs[mid:], loaded_runs[mid + 1:]):
assert run_prev.commit == run.parent
assert run.parent is not None
def test_storage(dummy_run: Run, tmp_path: Path) -> None:
storage_dir = tmp_path / "storage"
storage_dir.mkdir()
storage = Storage(storage_dir)
storage.store_run(dummy_run)
exp_file = storage_dir / Storage._make_filename(dummy_run.commit)
assert exp_file.exists()
raw = exp_file.read_text()
assert raw == dummy_run.json(indent=2)
run_read = storage.get(dummy_run.commit)
assert run_read == dummy_run
with pytest.raises(AssertionError):
storage.get(Commit(hash="J" * 40, date=datetime.now(),
message="blubb"))
def test_metrics(stored_runs: Storage, dummy_runs: List[Run]) -> None:
exp = {}
for run in dummy_runs:
for m in run.results:
m2 = m.copy()
m2.value = None
if m2.group is not None:
exp.setdefault(m2.group, set())
exp[m2.group].add(m2)
else:
exp.setdefault("other", set())
exp["other"].add(m2)
# exp = sorted(exp, key=lambda m: m.name)
exp = {k: sorted(v, key=lambda m: m.name) for k, v in exp.items()}
act = stored_runs.get_metrics()
# act = sorted(act, key=lambda m: m.name)
assert exp == act
def test_get_branches(stored_runs: Storage) -> None:
assert sorted(stored_runs.get_branches()) == sorted(["main", "feature"])
def test_make_filename() -> None:
act = Storage._make_filename(
Commit(hash="ABCDEFG" + "Y" * 33, date=datetime.now(), message="blub"))
assert "ABCDEFG" + "Y" * 33 + ".json" == act
def test_iterate_all(dummy_runs: List[Run], stored_runs: Storage):
exp = sorted(dummy_runs, key=lambda r: r.commit.hash)
act = list(stored_runs.iterate_all())
assert exp == act
def test_count(stored_runs: Storage, dummy_runs: List[Run]) -> None:
assert stored_runs.num_runs() == len(dummy_runs)
def make_report(spec: Spec, storage: Storage, output: Path) -> None:
print(storage.get_branches())
msg.info("Begin report generation")
global github_project
github_project = spec.github_project
with rich.progress.Progress() as progress:
task = progress.add_task("Creating dataframe",
total=storage.num_runs())
def update():
progress.advance(task)
df, metrics_by_group = storage.dataframe(with_metrics=True,
progress_callback=update)
metrics_by_group = {
g: list(filter(lambda m: spec.report_filter(m, df), ms))
for g, ms in metrics_by_group.items()
}
msg.good("Dataframe created")
env = make_environment()
env.globals["metrics"] = metrics_by_group
env.globals["github_project"] = spec.github_project
copy_static(output)
global current_url
# start page
tpl = env.get_template("index.html.j2")
current_url = "/"
(output / "index.html").write_text(tpl.render())
group_tpl = env.get_template("group.html.j2")
for group, metrics in metrics_by_group.items():
msg.info(f"Group: {group}")
with ProcessPoolExecutor() as ex:
futures = [
ex.submit(
process_metric,
m,
df,
output,
metrics_by_group,
spec.github_project,
spec.report_num_commits,
) for m in metrics
]
for f in rich.progress.track(as_completed(futures),
total=len(futures)):
metric = f.result()
print(metric.name)
msg.good(f"Completed group {group}")
# for metric in rich.progress.track(metrics):
# process_metric(metric, df, output, env)
url = group_url(group)
page = output / url / "index.html"
with push_url(url):
page.write_text(group_tpl.render(group=group))
def test_dataframe(dummy_runs: List[Run], stored_runs: Storage):
print()
df = stored_runs.dataframe()
print(df.head())
print(df.tail())