def resolve_runs(self, graphene_info, **kwargs): return [ graphene_info.schema.type_named('PipelineRun')(r) for r in graphene_info.context.instance.get_runs( filters=PipelineRunsFilter.for_schedule(self._schedule), limit=kwargs.get('limit'), ) ]
def resolve_runs(self, graphene_info, **kwargs): from .pipelines.pipeline import GrapheneRun if kwargs.get("limit") and self._batch_loader: limit = kwargs["limit"] records = ( self._batch_loader.get_run_records_for_sensor(self._instigator_state.name, limit) if self._instigator_state.instigator_type == InstigatorType.SENSOR else self._batch_loader.get_run_records_for_schedule( self._instigator_state.name, limit ) ) return [GrapheneRun(record) for record in records] if self._instigator_state.instigator_type == InstigatorType.SENSOR: filters = PipelineRunsFilter.for_sensor(self._instigator_state) else: filters = PipelineRunsFilter.for_schedule(self._instigator_state) return [ GrapheneRun(record) for record in graphene_info.context.instance.get_run_records( filters=filters, limit=kwargs.get("limit"), ) ]
def resolve_runs(self, graphene_info, **kwargs): if self._job_state.job_type == JobType.SENSOR: filters = PipelineRunsFilter.for_sensor(self._job_state) else: filters = PipelineRunsFilter.for_schedule(self._job_state) return [ graphene_info.schema.type_named("PipelineRun")(r) for r in graphene_info.context.instance.get_runs( filters=filters, limit=kwargs.get("limit"), ) ]
def resolve_runs(self, graphene_info, **kwargs): from .pipelines.pipeline import GrapheneRun if self._job_state.job_type == InstigatorType.SENSOR: filters = PipelineRunsFilter.for_sensor(self._job_state) else: filters = PipelineRunsFilter.for_schedule(self._job_state) return [ GrapheneRun(r) for r in graphene_info.context.instance.get_runs( filters=filters, limit=kwargs.get("limit"), ) ]
def resolve_runsCount(self, graphene_info): if self._job_state.job_type == JobType.SENSOR: filters = PipelineRunsFilter.for_sensor(self._job_state) else: filters = PipelineRunsFilter.for_schedule(self._job_state) return graphene_info.context.instance.get_runs_count(filters=filters)
def resolve_runs_count(self, graphene_info): return graphene_info.context.instance.get_runs_count( filters=PipelineRunsFilter.for_schedule(self._schedule) )
def test_bad_schedules_mixed_with_good_schedule(external_repo_context, capfd): with instance_with_schedules(external_repo_context) as (instance, external_repo): good_schedule = external_repo.get_external_schedule("simple_schedule") bad_schedule = external_repo.get_external_schedule( "bad_should_execute_schedule_on_odd_days") good_origin = good_schedule.get_external_origin() bad_origin = bad_schedule.get_external_origin() unloadable_origin = _get_unloadable_schedule_origin() initial_datetime = pendulum.datetime( year=2019, month=2, day=27, hour=0, minute=0, second=0, ) with pendulum.test(initial_datetime): instance.start_schedule_and_update_storage_state(good_schedule) instance.start_schedule_and_update_storage_state(bad_schedule) unloadable_schedule_state = JobState( unloadable_origin, JobType.SCHEDULE, JobStatus.RUNNING, ScheduleJobData("0 0 * * *", pendulum.now("UTC").timestamp()), ) instance.add_job_state(unloadable_schedule_state) launch_scheduled_runs(instance, logger(), pendulum.now("UTC")) assert instance.get_runs_count() == 1 wait_for_all_runs_to_start(instance) validate_run_started( instance.get_runs()[0], execution_time=initial_datetime, partition_time=pendulum.datetime(2019, 2, 26), ) good_ticks = instance.get_job_ticks(good_origin.get_id()) assert len(good_ticks) == 1 validate_tick( good_ticks[0], good_schedule, initial_datetime, JobTickStatus.SUCCESS, instance.get_runs()[0].run_id, ) bad_ticks = instance.get_job_ticks(bad_origin.get_id()) assert len(bad_ticks) == 1 assert bad_ticks[0].status == JobTickStatus.FAILURE assert ("Error occurred during the execution of should_execute " "for schedule bad_should_execute_schedule" in bad_ticks[0].error.message) unloadable_ticks = instance.get_job_ticks( unloadable_origin.get_id()) assert len(unloadable_ticks) == 0 captured = capfd.readouterr() assert "Scheduler failed for doesnt_exist" in captured.out assert "doesnt_exist not found at module scope" in captured.out initial_datetime = initial_datetime.add(days=1) with pendulum.test(initial_datetime): new_now = pendulum.now("UTC") launch_scheduled_runs(instance, logger(), new_now) assert instance.get_runs_count() == 3 wait_for_all_runs_to_start(instance) good_schedule_runs = instance.get_runs( filters=PipelineRunsFilter.for_schedule(good_schedule)) assert len(good_schedule_runs) == 2 validate_run_started( good_schedule_runs[0], execution_time=new_now, partition_time=pendulum.datetime(2019, 2, 27), ) good_ticks = instance.get_job_ticks(good_origin.get_id()) assert len(good_ticks) == 2 validate_tick( good_ticks[0], good_schedule, new_now, JobTickStatus.SUCCESS, good_schedule_runs[0].run_id, ) bad_schedule_runs = instance.get_runs( filters=PipelineRunsFilter.for_schedule(bad_schedule)) assert len(bad_schedule_runs) == 1 validate_run_started( bad_schedule_runs[0], execution_time=new_now, partition_time=pendulum.datetime(2019, 2, 27), ) bad_ticks = instance.get_job_ticks(bad_origin.get_id()) assert len(bad_ticks) == 2 validate_tick( bad_ticks[0], bad_schedule, new_now, JobTickStatus.SUCCESS, bad_schedule_runs[0].run_id, ) unloadable_ticks = instance.get_job_ticks( unloadable_origin.get_id()) assert len(unloadable_ticks) == 0 captured = capfd.readouterr() assert "Scheduler failed for doesnt_exist" in captured.out assert "doesnt_exist not found at module scope" in captured.out
def test_bad_schedule_mixed_with_good_schedule(external_repo_context): with instance_with_schedules(external_repo_context) as (instance, external_repo): good_schedule = external_repo.get_external_schedule("simple_schedule") bad_schedule = external_repo.get_external_schedule( "bad_should_execute_schedule_on_odd_days") good_origin = good_schedule.get_origin() bad_origin = bad_schedule.get_origin() initial_datetime = datetime( year=2019, month=2, day=27, hour=0, minute=0, second=0, tzinfo=get_utc_timezone(), ) with freeze_time(initial_datetime) as frozen_datetime: instance.start_schedule_and_update_storage_state(good_schedule) instance.start_schedule_and_update_storage_state(bad_schedule) launch_scheduled_runs(instance, get_default_scheduler_logger(), get_current_datetime_in_utc()) assert instance.get_runs_count() == 1 wait_for_all_runs_to_start(instance) validate_run_started(instance.get_runs()[0], initial_datetime, "2019-02-26") good_ticks = instance.get_schedule_ticks(good_origin.get_id()) assert len(good_ticks) == 1 validate_tick( good_ticks[0], good_schedule, initial_datetime, ScheduleTickStatus.SUCCESS, instance.get_runs()[0].run_id, ) bad_ticks = instance.get_schedule_ticks(bad_origin.get_id()) assert len(bad_ticks) == 1 assert bad_ticks[0].status == ScheduleTickStatus.FAILURE assert ("Error occurred during the execution of should_execute " "for schedule bad_should_execute_schedule" in bad_ticks[0].error.message) frozen_datetime.tick(delta=timedelta(days=1)) new_now = get_current_datetime_in_utc() launch_scheduled_runs(instance, get_default_scheduler_logger(), get_current_datetime_in_utc()) assert instance.get_runs_count() == 3 wait_for_all_runs_to_start(instance) good_schedule_runs = instance.get_runs( filters=PipelineRunsFilter.for_schedule(good_schedule)) assert len(good_schedule_runs) == 2 validate_run_started(good_schedule_runs[0], new_now, "2019-02-27") good_ticks = instance.get_schedule_ticks(good_origin.get_id()) assert len(good_ticks) == 2 validate_tick( good_ticks[0], good_schedule, new_now, ScheduleTickStatus.SUCCESS, good_schedule_runs[0].run_id, ) bad_schedule_runs = instance.get_runs( filters=PipelineRunsFilter.for_schedule(bad_schedule)) assert len(bad_schedule_runs) == 1 validate_run_started(bad_schedule_runs[0], new_now, "2019-02-27") bad_ticks = instance.get_schedule_ticks(bad_origin.get_id()) assert len(bad_ticks) == 2 validate_tick( bad_ticks[0], bad_schedule, new_now, ScheduleTickStatus.SUCCESS, bad_schedule_runs[0].run_id, )