def test_failing_metrics_ignore_skipped(): start_time = datetime(year=2017, month=12, day=11, hour=8) run1 = PipelineRun( start_time=start_time, end_time=start_time + timedelta(minutes=10), stage_results=[StageRun(StageStatus.ok)], ) run2 = PipelineRun( start_time=start_time + timedelta(minutes=10), end_time=start_time + timedelta(minutes=20), stage_results=[StageRun(StageStatus.fail)], ) run3 = PipelineRun( start_time=start_time + timedelta(minutes=20), end_time=start_time + timedelta(minutes=30), stage_results=[StageRun(StageStatus.ok)], ) run4 = PipelineRun( start_time=start_time + timedelta(minutes=30), end_time=start_time + timedelta(minutes=40), stage_results=[StageRun(StageStatus.skip)], ) runs = [run1, run2, run3, run4] metrics = pipeline_metrics(runs) assert metrics.pipeline_lead_time == timedelta(minutes=10) assert metrics.pipeline_failure_rate == pytest.approx(0.25, 0.01) assert metrics.pipeline_interval == timedelta(minutes=20) assert metrics.pipeline_recovery_time == timedelta(minutes=10)
def test_choose_runs_to_deploy_once_a_week(): deploy_delay = timedelta(hours=1) ten = timedelta(minutes=10) deployer = Deployer(duration=deploy_delay, deploy_policy=DeployPolicy.OnceAWeek, deploy_hour=9, deploy_day=6) runs = [StubPipelineRun(now, now, [StageRun(StageStatus.ok, end_time=now)]), StubPipelineRun(now + ten, now + ten, [StageRun(StageStatus.ok, end_time=(now + ten))])] deployer.add_deployments(runs) assert runs[0].deploy_time == "" # deploy on Saturday assert runs[1].deploy_time == datetime(year=2018, month=6, day=2, hour=9) + deploy_delay
def test_choose_runs_to_deploy_same_day(): now = datetime(year=2018,month=6,day=1,hour=15) deploy_delay = timedelta(hours=1) ten = timedelta(minutes=10) deployer = Deployer(duration=deploy_delay, deploy_policy=DeployPolicy.OnceADay, deploy_hour=17) runs = [StubPipelineRun(now, now, [StageRun(StageStatus.ok, end_time=now)]), StubPipelineRun(now + ten, now + ten, [StageRun(StageStatus.ok, end_time=(now + ten))])] deployer.add_deployments(runs) assert runs[0].deploy_time == "" # deploy same day since deploy time is after run is ready assert runs[1].deploy_time == datetime(year=2018, month=6, day=1, hour=17) + deploy_delay
def test_choose_runs_to_deploy(): deploy_delay = timedelta(minutes=1) deployer = Deployer(duration=deploy_delay, deploy_policy=DeployPolicy.EveryPassing) runs = [StubPipelineRun(now, now, [StageRun(StageStatus.ok, end_time=now)])] deployer.add_deployments(runs) assert runs[0].deploy_time == now + deploy_delay
def test_str(): assert "Pipeline Run(2018-04-03 08:00:00, 2018-04-03 08:10:00, [Commit(name='#001', counter=1, time=datetime.datetime(2018, 4, 3, 7, 50))], [ok], )"\ == str(PipelineRun(start_time=now, end_time=now + timedelta(minutes=10), changes_included=[commit1], stage_results=[StageRun(StageStatus.ok)], deploy_time=None, ))
def test_realistic_failure_recovery(passing_stage, failing_stage): end_of_first_stage_time = now + passing_stage.duration assert passing_stage.add_result(now, now) == StageRun(StageStatus.ok, now, end_of_first_stage_time) assert failing_stage.add_result(end_of_first_stage_time, now) == StageRun( StageStatus.fail, end_of_first_stage_time, end_of_first_stage_time + failing_stage.duration) start_of_next_pipeline = now + passing_stage.duration end_of_next_pipeline_first_stage = start_of_next_pipeline + passing_stage.duration assert passing_stage.add_result(start_of_next_pipeline, start_of_next_pipeline) == StageRun( StageStatus.ok, start_of_next_pipeline, end_of_next_pipeline_first_stage) assert failing_stage.add_result( end_of_next_pipeline_first_stage, start_of_next_pipeline) == StageRun( StageStatus.repeat_fail, end_of_next_pipeline_first_stage, end_of_next_pipeline_first_stage + failing_stage.duration)
def test_deploy_metrics_recovery_doesnt_ignore_nights_and_weekends(): start_time = datetime(year=2018, month=6, day=14, hour=17, minute=45) run1 = PipelineRun(start_time=start_time, end_time=start_time + timedelta(minutes=10), stage_results=[StageRun(StageStatus.ok)], deploy_time=start_time + timedelta(minutes=11)) run2 = PipelineRun( start_time=start_time + timedelta(minutes=10), end_time=start_time + timedelta(minutes=20), stage_results=[StageRun(StageStatus.fail)], ) run3 = PipelineRun(start_time=start_time + timedelta(hours=14, minutes=20), end_time=start_time + timedelta(hours=14, minutes=30), stage_results=[StageRun(StageStatus.ok)], deploy_time=start_time + timedelta(hours=14, minutes=31)) runs = [run1, run2, run3] metrics = pipeline_metrics(runs) assert metrics.deployment_lead_time == timedelta(minutes=11) assert metrics.deployment_failure_rate == 0 assert metrics.deployment_interval == timedelta(hours=14, minutes=20) assert metrics.deployment_recovery_time == None
def test_one_stage(passing_stage): stages = [passing_stage] commits = [commit1] pipeline = Pipeline(stages, trigger="commits") runs = pipeline.simulation(now, commits, timedelta(minutes=60)) assert len(runs) == 1 assert str( PipelineRun( start_time=now, end_time=now + timedelta(minutes=10), changes_included=commits, stage_results=[StageRun(StageStatus.ok)], deploy_time=None, )) == str(runs[0])
def test_release_candidate_metrics_ignore_weekends(): start_time = datetime(year=2018, month=6, day=15, hour=17, minute=50) run1 = PipelineRun( start_time=start_time, end_time=start_time + timedelta(minutes=10), stage_results=[StageRun(StageStatus.ok)], ) run2 = PipelineRun( start_time=start_time + timedelta(minutes=10), end_time=start_time + timedelta(minutes=20), stage_results=[StageRun(StageStatus.fail)], ) run3 = PipelineRun( start_time=start_time + timedelta(days=2, hours=14, minutes=20), end_time=start_time + timedelta(days=2, hours=14, minutes=30), stage_results=[StageRun(StageStatus.ok)], ) runs = [run1, run2, run3] metrics = pipeline_metrics(runs) assert metrics.pipeline_lead_time == timedelta(minutes=10) assert metrics.pipeline_failure_rate == pytest.approx(0.333, 0.01) assert metrics.pipeline_interval == timedelta(minutes=20) assert metrics.pipeline_recovery_time == timedelta(minutes=10)
def test_passing_metrics(): start_time = datetime(year=2017, month=12, day=11, hour=8) run1 = PipelineRun( start_time=start_time, end_time=start_time + timedelta(minutes=10), stage_results=[StageRun(StageStatus.ok)], ) run2 = PipelineRun( start_time=start_time + timedelta(minutes=10), end_time=start_time + timedelta(minutes=20), stage_results=[StageRun(StageStatus.ok)], ) run3 = PipelineRun( start_time=start_time + timedelta(minutes=20), end_time=start_time + timedelta(minutes=30), stage_results=[StageRun(StageStatus.ok)], ) runs = [run1, run2, run3] metrics = pipeline_metrics(runs) assert metrics.pipeline_lead_time == timedelta(minutes=10) assert metrics.pipeline_failure_rate == 0.0 assert metrics.pipeline_interval == timedelta(minutes=10) assert metrics.pipeline_recovery_time is None
def test_deploy_metrics(): start_time = datetime(year=2017, month=12, day=11, hour=8) run1 = PipelineRun(start_time=start_time, end_time=start_time + timedelta(minutes=10), stage_results=[StageRun(StageStatus.ok)], deploy_time=start_time + timedelta(minutes=11)) run2 = PipelineRun( start_time=start_time + timedelta(minutes=10), end_time=start_time + timedelta(minutes=20), stage_results=[StageRun(StageStatus.fail)], ) run3 = PipelineRun(start_time=start_time + timedelta(minutes=20), end_time=start_time + timedelta(minutes=30), stage_results=[StageRun(StageStatus.ok)], deploy_time=start_time + timedelta(minutes=31)) runs = [run1, run2, run3] calculator = MetricsCalculator(runs) metrics = calculator.metrics() assert metrics.deployment_lead_time == timedelta(minutes=11) assert metrics.deployment_failure_rate == 0 assert metrics.deployment_interval == timedelta(minutes=20) assert metrics.deployment_recovery_time == None assert len(calculator.deploys) == 2
def test_stage_status_strings(): assert str(StageStatus.ok) == "ok" assert ["ok", "fail", "foobar" ] == list(map(str, [StageStatus.ok, StageStatus.fail, "foobar"])) assert str(StageRun(status=StageStatus.ok, start_time=now, end_time=now)) == "ok"
def test_stage_result(passing_stage, failing_stage): assert passing_stage.add_result(now, now) == StageRun( StageStatus.ok, now, now + passing_stage.duration) assert failing_stage.add_result(now, now) == StageRun( StageStatus.fail, now, now + failing_stage.duration)