def resolve_runs(self, graphene_info, **kwargs):
return [
graphene_info.schema.type_named('PipelineRun')(r)
for r in graphene_info.context.instance.get_runs(
filters=PipelineRunsFilter.for_schedule(self._schedule), limit=kwargs.get('limit'),
)
]
def resolve_runs(self, graphene_info, **kwargs):
from .pipelines.pipeline import GrapheneRun
if kwargs.get("limit") and self._batch_loader:
limit = kwargs["limit"]
records = (
self._batch_loader.get_run_records_for_sensor(self._instigator_state.name, limit)
if self._instigator_state.instigator_type == InstigatorType.SENSOR
else self._batch_loader.get_run_records_for_schedule(
self._instigator_state.name, limit
)
)
return [GrapheneRun(record) for record in records]
if self._instigator_state.instigator_type == InstigatorType.SENSOR:
filters = PipelineRunsFilter.for_sensor(self._instigator_state)
else:
filters = PipelineRunsFilter.for_schedule(self._instigator_state)
return [
GrapheneRun(record)
for record in graphene_info.context.instance.get_run_records(
filters=filters,
limit=kwargs.get("limit"),
)
]
def resolve_runs(self, graphene_info, **kwargs):
if self._job_state.job_type == JobType.SENSOR:
filters = PipelineRunsFilter.for_sensor(self._job_state)
else:
filters = PipelineRunsFilter.for_schedule(self._job_state)
return [
graphene_info.schema.type_named("PipelineRun")(r)
for r in graphene_info.context.instance.get_runs(
filters=filters, limit=kwargs.get("limit"),
)
]
def resolve_runs(self, graphene_info, **kwargs):
from .pipelines.pipeline import GrapheneRun
if self._job_state.job_type == InstigatorType.SENSOR:
filters = PipelineRunsFilter.for_sensor(self._job_state)
else:
filters = PipelineRunsFilter.for_schedule(self._job_state)
return [
GrapheneRun(r) for r in graphene_info.context.instance.get_runs(
filters=filters,
limit=kwargs.get("limit"),
)
]
def resolve_runsCount(self, graphene_info):
if self._job_state.job_type == JobType.SENSOR:
filters = PipelineRunsFilter.for_sensor(self._job_state)
else:
filters = PipelineRunsFilter.for_schedule(self._job_state)
return graphene_info.context.instance.get_runs_count(filters=filters)
def resolve_runs_count(self, graphene_info):
return graphene_info.context.instance.get_runs_count(
filters=PipelineRunsFilter.for_schedule(self._schedule)
)
def test_bad_schedules_mixed_with_good_schedule(external_repo_context, capfd):
with instance_with_schedules(external_repo_context) as (instance,
external_repo):
good_schedule = external_repo.get_external_schedule("simple_schedule")
bad_schedule = external_repo.get_external_schedule(
"bad_should_execute_schedule_on_odd_days")
good_origin = good_schedule.get_external_origin()
bad_origin = bad_schedule.get_external_origin()
unloadable_origin = _get_unloadable_schedule_origin()
initial_datetime = pendulum.datetime(
year=2019,
month=2,
day=27,
hour=0,
minute=0,
second=0,
)
with pendulum.test(initial_datetime):
instance.start_schedule_and_update_storage_state(good_schedule)
instance.start_schedule_and_update_storage_state(bad_schedule)
unloadable_schedule_state = JobState(
unloadable_origin,
JobType.SCHEDULE,
JobStatus.RUNNING,
ScheduleJobData("0 0 * * *",
pendulum.now("UTC").timestamp()),
)
instance.add_job_state(unloadable_schedule_state)
launch_scheduled_runs(instance, logger(), pendulum.now("UTC"))
assert instance.get_runs_count() == 1
wait_for_all_runs_to_start(instance)
validate_run_started(
instance.get_runs()[0],
execution_time=initial_datetime,
partition_time=pendulum.datetime(2019, 2, 26),
)
good_ticks = instance.get_job_ticks(good_origin.get_id())
assert len(good_ticks) == 1
validate_tick(
good_ticks[0],
good_schedule,
initial_datetime,
JobTickStatus.SUCCESS,
instance.get_runs()[0].run_id,
)
bad_ticks = instance.get_job_ticks(bad_origin.get_id())
assert len(bad_ticks) == 1
assert bad_ticks[0].status == JobTickStatus.FAILURE
assert ("Error occurred during the execution of should_execute "
"for schedule bad_should_execute_schedule"
in bad_ticks[0].error.message)
unloadable_ticks = instance.get_job_ticks(
unloadable_origin.get_id())
assert len(unloadable_ticks) == 0
captured = capfd.readouterr()
assert "Scheduler failed for doesnt_exist" in captured.out
assert "doesnt_exist not found at module scope" in captured.out
initial_datetime = initial_datetime.add(days=1)
with pendulum.test(initial_datetime):
new_now = pendulum.now("UTC")
launch_scheduled_runs(instance, logger(), new_now)
assert instance.get_runs_count() == 3
wait_for_all_runs_to_start(instance)
good_schedule_runs = instance.get_runs(
filters=PipelineRunsFilter.for_schedule(good_schedule))
assert len(good_schedule_runs) == 2
validate_run_started(
good_schedule_runs[0],
execution_time=new_now,
partition_time=pendulum.datetime(2019, 2, 27),
)
good_ticks = instance.get_job_ticks(good_origin.get_id())
assert len(good_ticks) == 2
validate_tick(
good_ticks[0],
good_schedule,
new_now,
JobTickStatus.SUCCESS,
good_schedule_runs[0].run_id,
)
bad_schedule_runs = instance.get_runs(
filters=PipelineRunsFilter.for_schedule(bad_schedule))
assert len(bad_schedule_runs) == 1
validate_run_started(
bad_schedule_runs[0],
execution_time=new_now,
partition_time=pendulum.datetime(2019, 2, 27),
)
bad_ticks = instance.get_job_ticks(bad_origin.get_id())
assert len(bad_ticks) == 2
validate_tick(
bad_ticks[0],
bad_schedule,
new_now,
JobTickStatus.SUCCESS,
bad_schedule_runs[0].run_id,
)
unloadable_ticks = instance.get_job_ticks(
unloadable_origin.get_id())
assert len(unloadable_ticks) == 0
captured = capfd.readouterr()
assert "Scheduler failed for doesnt_exist" in captured.out
assert "doesnt_exist not found at module scope" in captured.out
def test_bad_schedule_mixed_with_good_schedule(external_repo_context):
with instance_with_schedules(external_repo_context) as (instance,
external_repo):
good_schedule = external_repo.get_external_schedule("simple_schedule")
bad_schedule = external_repo.get_external_schedule(
"bad_should_execute_schedule_on_odd_days")
good_origin = good_schedule.get_origin()
bad_origin = bad_schedule.get_origin()
initial_datetime = datetime(
year=2019,
month=2,
day=27,
hour=0,
minute=0,
second=0,
tzinfo=get_utc_timezone(),
)
with freeze_time(initial_datetime) as frozen_datetime:
instance.start_schedule_and_update_storage_state(good_schedule)
instance.start_schedule_and_update_storage_state(bad_schedule)
launch_scheduled_runs(instance, get_default_scheduler_logger(),
get_current_datetime_in_utc())
assert instance.get_runs_count() == 1
wait_for_all_runs_to_start(instance)
validate_run_started(instance.get_runs()[0], initial_datetime,
"2019-02-26")
good_ticks = instance.get_schedule_ticks(good_origin.get_id())
assert len(good_ticks) == 1
validate_tick(
good_ticks[0],
good_schedule,
initial_datetime,
ScheduleTickStatus.SUCCESS,
instance.get_runs()[0].run_id,
)
bad_ticks = instance.get_schedule_ticks(bad_origin.get_id())
assert len(bad_ticks) == 1
assert bad_ticks[0].status == ScheduleTickStatus.FAILURE
assert ("Error occurred during the execution of should_execute "
"for schedule bad_should_execute_schedule"
in bad_ticks[0].error.message)
frozen_datetime.tick(delta=timedelta(days=1))
new_now = get_current_datetime_in_utc()
launch_scheduled_runs(instance, get_default_scheduler_logger(),
get_current_datetime_in_utc())
assert instance.get_runs_count() == 3
wait_for_all_runs_to_start(instance)
good_schedule_runs = instance.get_runs(
filters=PipelineRunsFilter.for_schedule(good_schedule))
assert len(good_schedule_runs) == 2
validate_run_started(good_schedule_runs[0], new_now, "2019-02-27")
good_ticks = instance.get_schedule_ticks(good_origin.get_id())
assert len(good_ticks) == 2
validate_tick(
good_ticks[0],
good_schedule,
new_now,
ScheduleTickStatus.SUCCESS,
good_schedule_runs[0].run_id,
)
bad_schedule_runs = instance.get_runs(
filters=PipelineRunsFilter.for_schedule(bad_schedule))
assert len(bad_schedule_runs) == 1
validate_run_started(bad_schedule_runs[0], new_now, "2019-02-27")
bad_ticks = instance.get_schedule_ticks(bad_origin.get_id())
assert len(bad_ticks) == 2
validate_tick(
bad_ticks[0],
bad_schedule,
new_now,
ScheduleTickStatus.SUCCESS,
bad_schedule_runs[0].run_id,
)
