def test_failing_metrics_ignore_skipped():
start_time = datetime(year=2017, month=12, day=11, hour=8)
run1 = PipelineRun(
start_time=start_time,
end_time=start_time + timedelta(minutes=10),
stage_results=[StageRun(StageStatus.ok)],
)
run2 = PipelineRun(
start_time=start_time + timedelta(minutes=10),
end_time=start_time + timedelta(minutes=20),
stage_results=[StageRun(StageStatus.fail)],
)
run3 = PipelineRun(
start_time=start_time + timedelta(minutes=20),
end_time=start_time + timedelta(minutes=30),
stage_results=[StageRun(StageStatus.ok)],
)
run4 = PipelineRun(
start_time=start_time + timedelta(minutes=30),
end_time=start_time + timedelta(minutes=40),
stage_results=[StageRun(StageStatus.skip)],
)
runs = [run1, run2, run3, run4]
metrics = pipeline_metrics(runs)
assert metrics.pipeline_lead_time == timedelta(minutes=10)
assert metrics.pipeline_failure_rate == pytest.approx(0.25, 0.01)
assert metrics.pipeline_interval == timedelta(minutes=20)
assert metrics.pipeline_recovery_time == timedelta(minutes=10)
def test_choose_runs_to_deploy_once_a_week():
deploy_delay = timedelta(hours=1)
ten = timedelta(minutes=10)
deployer = Deployer(duration=deploy_delay, deploy_policy=DeployPolicy.OnceAWeek, deploy_hour=9, deploy_day=6)
runs = [StubPipelineRun(now, now, [StageRun(StageStatus.ok, end_time=now)]),
StubPipelineRun(now + ten, now + ten, [StageRun(StageStatus.ok, end_time=(now + ten))])]
deployer.add_deployments(runs)
assert runs[0].deploy_time == ""
# deploy on Saturday
assert runs[1].deploy_time == datetime(year=2018, month=6, day=2, hour=9) + deploy_delay
def test_choose_runs_to_deploy_same_day():
now = datetime(year=2018,month=6,day=1,hour=15)
deploy_delay = timedelta(hours=1)
ten = timedelta(minutes=10)
deployer = Deployer(duration=deploy_delay, deploy_policy=DeployPolicy.OnceADay, deploy_hour=17)
runs = [StubPipelineRun(now, now, [StageRun(StageStatus.ok, end_time=now)]),
StubPipelineRun(now + ten, now + ten, [StageRun(StageStatus.ok, end_time=(now + ten))])]
deployer.add_deployments(runs)
assert runs[0].deploy_time == ""
# deploy same day since deploy time is after run is ready
assert runs[1].deploy_time == datetime(year=2018, month=6, day=1, hour=17) + deploy_delay
def test_choose_runs_to_deploy():
deploy_delay = timedelta(minutes=1)
deployer = Deployer(duration=deploy_delay, deploy_policy=DeployPolicy.EveryPassing)
runs = [StubPipelineRun(now, now, [StageRun(StageStatus.ok, end_time=now)])]
deployer.add_deployments(runs)
assert runs[0].deploy_time == now + deploy_delay
def test_str():
assert "Pipeline Run(2018-04-03 08:00:00, 2018-04-03 08:10:00, [Commit(name='#001', counter=1, time=datetime.datetime(2018, 4, 3, 7, 50))], [ok], )"\
== str(PipelineRun(start_time=now,
end_time=now + timedelta(minutes=10),
changes_included=[commit1],
stage_results=[StageRun(StageStatus.ok)],
deploy_time=None,
))
def test_realistic_failure_recovery(passing_stage, failing_stage):
end_of_first_stage_time = now + passing_stage.duration
assert passing_stage.add_result(now,
now) == StageRun(StageStatus.ok, now,
end_of_first_stage_time)
assert failing_stage.add_result(end_of_first_stage_time, now) == StageRun(
StageStatus.fail, end_of_first_stage_time,
end_of_first_stage_time + failing_stage.duration)
start_of_next_pipeline = now + passing_stage.duration
end_of_next_pipeline_first_stage = start_of_next_pipeline + passing_stage.duration
assert passing_stage.add_result(start_of_next_pipeline,
start_of_next_pipeline) == StageRun(
StageStatus.ok, start_of_next_pipeline,
end_of_next_pipeline_first_stage)
assert failing_stage.add_result(
end_of_next_pipeline_first_stage, start_of_next_pipeline) == StageRun(
StageStatus.repeat_fail, end_of_next_pipeline_first_stage,
end_of_next_pipeline_first_stage + failing_stage.duration)
def test_deploy_metrics_recovery_doesnt_ignore_nights_and_weekends():
start_time = datetime(year=2018, month=6, day=14, hour=17, minute=45)
run1 = PipelineRun(start_time=start_time,
end_time=start_time + timedelta(minutes=10),
stage_results=[StageRun(StageStatus.ok)],
deploy_time=start_time + timedelta(minutes=11))
run2 = PipelineRun(
start_time=start_time + timedelta(minutes=10),
end_time=start_time + timedelta(minutes=20),
stage_results=[StageRun(StageStatus.fail)],
)
run3 = PipelineRun(start_time=start_time + timedelta(hours=14, minutes=20),
end_time=start_time + timedelta(hours=14, minutes=30),
stage_results=[StageRun(StageStatus.ok)],
deploy_time=start_time +
timedelta(hours=14, minutes=31))
runs = [run1, run2, run3]
metrics = pipeline_metrics(runs)
assert metrics.deployment_lead_time == timedelta(minutes=11)
assert metrics.deployment_failure_rate == 0
assert metrics.deployment_interval == timedelta(hours=14, minutes=20)
assert metrics.deployment_recovery_time == None
def test_one_stage(passing_stage):
stages = [passing_stage]
commits = [commit1]
pipeline = Pipeline(stages, trigger="commits")
runs = pipeline.simulation(now, commits, timedelta(minutes=60))
assert len(runs) == 1
assert str(
PipelineRun(
start_time=now,
end_time=now + timedelta(minutes=10),
changes_included=commits,
stage_results=[StageRun(StageStatus.ok)],
deploy_time=None,
)) == str(runs[0])
def test_release_candidate_metrics_ignore_weekends():
start_time = datetime(year=2018, month=6, day=15, hour=17, minute=50)
run1 = PipelineRun(
start_time=start_time,
end_time=start_time + timedelta(minutes=10),
stage_results=[StageRun(StageStatus.ok)],
)
run2 = PipelineRun(
start_time=start_time + timedelta(minutes=10),
end_time=start_time + timedelta(minutes=20),
stage_results=[StageRun(StageStatus.fail)],
)
run3 = PipelineRun(
start_time=start_time + timedelta(days=2, hours=14, minutes=20),
end_time=start_time + timedelta(days=2, hours=14, minutes=30),
stage_results=[StageRun(StageStatus.ok)],
)
runs = [run1, run2, run3]
metrics = pipeline_metrics(runs)
assert metrics.pipeline_lead_time == timedelta(minutes=10)
assert metrics.pipeline_failure_rate == pytest.approx(0.333, 0.01)
assert metrics.pipeline_interval == timedelta(minutes=20)
assert metrics.pipeline_recovery_time == timedelta(minutes=10)
def test_passing_metrics():
start_time = datetime(year=2017, month=12, day=11, hour=8)
run1 = PipelineRun(
start_time=start_time,
end_time=start_time + timedelta(minutes=10),
stage_results=[StageRun(StageStatus.ok)],
)
run2 = PipelineRun(
start_time=start_time + timedelta(minutes=10),
end_time=start_time + timedelta(minutes=20),
stage_results=[StageRun(StageStatus.ok)],
)
run3 = PipelineRun(
start_time=start_time + timedelta(minutes=20),
end_time=start_time + timedelta(minutes=30),
stage_results=[StageRun(StageStatus.ok)],
)
runs = [run1, run2, run3]
metrics = pipeline_metrics(runs)
assert metrics.pipeline_lead_time == timedelta(minutes=10)
assert metrics.pipeline_failure_rate == 0.0
assert metrics.pipeline_interval == timedelta(minutes=10)
assert metrics.pipeline_recovery_time is None
def test_deploy_metrics():
start_time = datetime(year=2017, month=12, day=11, hour=8)
run1 = PipelineRun(start_time=start_time,
end_time=start_time + timedelta(minutes=10),
stage_results=[StageRun(StageStatus.ok)],
deploy_time=start_time + timedelta(minutes=11))
run2 = PipelineRun(
start_time=start_time + timedelta(minutes=10),
end_time=start_time + timedelta(minutes=20),
stage_results=[StageRun(StageStatus.fail)],
)
run3 = PipelineRun(start_time=start_time + timedelta(minutes=20),
end_time=start_time + timedelta(minutes=30),
stage_results=[StageRun(StageStatus.ok)],
deploy_time=start_time + timedelta(minutes=31))
runs = [run1, run2, run3]
calculator = MetricsCalculator(runs)
metrics = calculator.metrics()
assert metrics.deployment_lead_time == timedelta(minutes=11)
assert metrics.deployment_failure_rate == 0
assert metrics.deployment_interval == timedelta(minutes=20)
assert metrics.deployment_recovery_time == None
assert len(calculator.deploys) == 2
def test_stage_status_strings():
assert str(StageStatus.ok) == "ok"
assert ["ok", "fail", "foobar"
] == list(map(str, [StageStatus.ok, StageStatus.fail, "foobar"]))
assert str(StageRun(status=StageStatus.ok, start_time=now,
end_time=now)) == "ok"
def test_stage_result(passing_stage, failing_stage):
assert passing_stage.add_result(now, now) == StageRun(
StageStatus.ok, now, now + passing_stage.duration)
assert failing_stage.add_result(now, now) == StageRun(
StageStatus.fail, now, now + failing_stage.duration)
