def test_failing_metrics_ignore_skipped():
start_time = datetime(year=2017, month=12, day=11, hour=8)
run1 = PipelineRun(
start_time=start_time,
end_time=start_time + timedelta(minutes=10),
stage_results=[StageRun(StageStatus.ok)],
)
run2 = PipelineRun(
start_time=start_time + timedelta(minutes=10),
end_time=start_time + timedelta(minutes=20),
stage_results=[StageRun(StageStatus.fail)],
)
run3 = PipelineRun(
start_time=start_time + timedelta(minutes=20),
end_time=start_time + timedelta(minutes=30),
stage_results=[StageRun(StageStatus.ok)],
)
run4 = PipelineRun(
start_time=start_time + timedelta(minutes=30),
end_time=start_time + timedelta(minutes=40),
stage_results=[StageRun(StageStatus.skip)],
)
runs = [run1, run2, run3, run4]
metrics = pipeline_metrics(runs)
assert metrics.pipeline_lead_time == timedelta(minutes=10)
assert metrics.pipeline_failure_rate == pytest.approx(0.25, 0.01)
assert metrics.pipeline_interval == timedelta(minutes=20)
assert metrics.pipeline_recovery_time == timedelta(minutes=10)
def test_deploy_metrics_recovery_doesnt_ignore_nights_and_weekends():
start_time = datetime(year=2018, month=6, day=14, hour=17, minute=45)
run1 = PipelineRun(start_time=start_time,
end_time=start_time + timedelta(minutes=10),
stage_results=[StageRun(StageStatus.ok)],
deploy_time=start_time + timedelta(minutes=11))
run2 = PipelineRun(
start_time=start_time + timedelta(minutes=10),
end_time=start_time + timedelta(minutes=20),
stage_results=[StageRun(StageStatus.fail)],
)
run3 = PipelineRun(start_time=start_time + timedelta(hours=14, minutes=20),
end_time=start_time + timedelta(hours=14, minutes=30),
stage_results=[StageRun(StageStatus.ok)],
deploy_time=start_time +
timedelta(hours=14, minutes=31))
runs = [run1, run2, run3]
metrics = pipeline_metrics(runs)
assert metrics.deployment_lead_time == timedelta(minutes=11)
assert metrics.deployment_failure_rate == 0
assert metrics.deployment_interval == timedelta(hours=14, minutes=20)
assert metrics.deployment_recovery_time == None
def test_release_candidate_metrics_ignore_weekends():
start_time = datetime(year=2018, month=6, day=15, hour=17, minute=50)
run1 = PipelineRun(
start_time=start_time,
end_time=start_time + timedelta(minutes=10),
stage_results=[StageRun(StageStatus.ok)],
)
run2 = PipelineRun(
start_time=start_time + timedelta(minutes=10),
end_time=start_time + timedelta(minutes=20),
stage_results=[StageRun(StageStatus.fail)],
)
run3 = PipelineRun(
start_time=start_time + timedelta(days=2, hours=14, minutes=20),
end_time=start_time + timedelta(days=2, hours=14, minutes=30),
stage_results=[StageRun(StageStatus.ok)],
)
runs = [run1, run2, run3]
metrics = pipeline_metrics(runs)
assert metrics.pipeline_lead_time == timedelta(minutes=10)
assert metrics.pipeline_failure_rate == pytest.approx(0.333, 0.01)
assert metrics.pipeline_interval == timedelta(minutes=20)
assert metrics.pipeline_recovery_time == timedelta(minutes=10)
def test_passing_metrics():
start_time = datetime(year=2017, month=12, day=11, hour=8)
run1 = PipelineRun(
start_time=start_time,
end_time=start_time + timedelta(minutes=10),
stage_results=[StageRun(StageStatus.ok)],
)
run2 = PipelineRun(
start_time=start_time + timedelta(minutes=10),
end_time=start_time + timedelta(minutes=20),
stage_results=[StageRun(StageStatus.ok)],
)
run3 = PipelineRun(
start_time=start_time + timedelta(minutes=20),
end_time=start_time + timedelta(minutes=30),
stage_results=[StageRun(StageStatus.ok)],
)
runs = [run1, run2, run3]
metrics = pipeline_metrics(runs)
assert metrics.pipeline_lead_time == timedelta(minutes=10)
assert metrics.pipeline_failure_rate == 0.0
assert metrics.pipeline_interval == timedelta(minutes=10)
assert metrics.pipeline_recovery_time is None
from metrics import pipeline_metrics
from simulation import run_simulation, print_runs
from stages import Stage
stages = [
Stage("Build & Unit Test",
duration=timedelta(minutes=10),
failure_rate=0.01),
Stage("Acceptance Test", duration=timedelta(minutes=20),
failure_rate=0.02),
Stage("Manual Test",
duration=timedelta(minutes=120),
failure_rate=0.05,
manual_stage=True),
]
start_time = datetime(year=2017, month=6, day=19, hour=8)
commits = generate_commits(100, start_time, offset=1000, max_interval=122)
deployer = Deployer(duration=timedelta(minutes=20),
deploy_policy=DeployPolicy.EveryPassing,
deploy_hour=8)
runs = run_simulation(start_time, stages, commits=commits, deployer=deployer)
print_runs("simulation2", stages, runs)
metrics = pipeline_metrics(runs)
print_metrics("simulation2", metrics)
print(metrics.pretty_print())