def dummy_run() -> Run:
return Run(
commit=Commit(hash="X" * 40, date=datetime.now(), message="blubb"),
parent=Commit(hash="X" * 40, date=datetime.now(), message="blubb"),
branch="main",
date=datetime.now(),
results=[Metric(name="a.metric", value=42, unit="X")],
)
def collect_cmd(
spec_file: typer.FileText,
jobs: int = typer.Option(1, "--jobs", "-j"),
commit_in: str = typer.Option(
get_current_commit().hash, "--commit", show_default=True
),
branch: str = typer.Option(get_branch(), "--branch", show_default=True),
) -> None:
spec = load_spec(spec_file)
storage = Storage(spec.storage_dir)
commit = Commit(
hash=str(commit_in),
date=get_commit_date(commit_in),
message=get_commit_message(commit_in),
)
# parent = Commit(hash=str(parent_in), date=get_commit_date(parent_in))
parent = storage.get_branch_tip(get_branch())
assert commit != parent, "We ran on this commit before it seems"
msg.info(f"#jobs: {jobs}")
msg.info(f"on commit: {commit}")
msg.info(f"parent commit: {parent}")
if jobs > 1:
msg.warn(
"If you're running benchmarks from the collect call,"
" concurrency can affect results"
)
assert jobs > 0, "Jobs value must be positive"
msg.good("Spec loaded successfully")
msg.divider()
try:
results = run_collectors(spec.collectors, jobs=jobs)
except CollectorError as e:
msg.fail("Collector returned invalid format")
typer.echo(str(e.exc))
return
# raise e
msg.good("Collection completed")
# print(results)
run = Run(
commit=commit,
parent=parent,
branch=branch,
date=datetime.now(),
results=sum((r.metrics for r in results), []),
context={},
)
# print(run)
storage = Storage(spec.storage_dir)
storage.store_run(run)
def get_parent_commit() -> Optional[Commit]:
try:
return Commit(
hash=_get_hash("HEAD~"),
date=get_commit_date("HEAD~"),
message=get_commit_message("HEAD~"),
)
except subprocess.CalledProcessError as e:
if e.returncode == 128:
return None
raise e
def test_run() -> None:
# VALID
Run(
commit=Commit(hash="X" * 40, date=datetime.now(), message="blubb"),
parent=Commit(hash="X" * 40, date=datetime.now(), message="blubb"),
branch="main",
date=datetime.now(),
results=[Metric(name="a.metric", value=42, unit="X")],
)
with pytest.raises(SpecValidationError):
# duplicate name
Run(
commit=Commit(hash="X" * 40),
parent=Commit(hash="X" * 40),
branch="main",
date=datetime.now(),
results=[
Metric(name="a.metric", value=42, unit="X"),
Metric(name="a.metric", value=42, unit="X"),
],
)
def test_storage(dummy_run: Run, tmp_path: Path) -> None:
storage_dir = tmp_path / "storage"
storage_dir.mkdir()
storage = Storage(storage_dir)
storage.store_run(dummy_run)
exp_file = storage_dir / Storage._make_filename(dummy_run.commit)
assert exp_file.exists()
raw = exp_file.read_text()
assert raw == dummy_run.json(indent=2)
run_read = storage.get(dummy_run.commit)
assert run_read == dummy_run
with pytest.raises(AssertionError):
storage.get(Commit(hash="J" * 40, date=datetime.now(),
message="blubb"))
def get_branch_tip(self, branch: str) -> Optional[Commit]:
filename = self._get_branch_file(branch)
if not filename.exists():
return None
with filename.open("r") as fh:
return Commit(**json.load(fh))
def test_make_filename() -> None:
act = Storage._make_filename(
Commit(hash="ABCDEFG" + "Y" * 33, date=datetime.now(), message="blub"))
assert "ABCDEFG" + "Y" * 33 + ".json" == act
def make_commit() -> Commit:
return Commit(
date=datetime.now(),
hash=hashlib.sha1(str(random.random()).encode("utf8")).hexdigest(),
message=" ".join([random.choice(_words) for _ in range(5)]),
)
def get_current_commit() -> Commit:
return Commit(
hash=_get_hash("HEAD"),
date=get_commit_date("HEAD"),
message=get_commit_message("HEAD"),
)