def weekday_partition_selector(
ctx: ScheduleExecutionContext, partition_set: PartitionSetDefinition
) -> Union[Partition, List[Partition]]:
"""Maps a schedule execution time to the corresponding partition or list of partitions that
should be executed at that time"""
partitions = partition_set.get_partitions(ctx.scheduled_execution_time)
weekday = ctx.scheduled_execution_time.weekday(
) if ctx.scheduled_execution_time else 0
return partitions[weekday]
def backfilling_partition_selector(context: ScheduleExecutionContext,
partition_set_def: PartitionSetDefinition):
runs_by_partition = _fetch_runs_by_partition(context.instance,
partition_set_def)
selected = None
for partition in partition_set_def.get_partitions():
runs = runs_by_partition[partition.name]
selected = partition
# break when we find the first empty partition
if len(runs) == 0:
break
# may return an already satisfied final partition - bank on should_execute to prevent firing in schedule
return selected
def backfilling_partition_selector(
context: ScheduleExecutionContext, partition_set_def: PartitionSetDefinition, retry_failed=False
):
status_filters = [PipelineRunStatus.SUCCESS] if retry_failed else None
runs_by_partition = _fetch_runs_by_partition(
context.instance, partition_set_def, status_filters
)
selected = None
for partition in partition_set_def.get_partitions():
runs = runs_by_partition[partition.name]
selected = partition
# break when we find the first empty partition
if len(runs) == 0:
break
# may return an already satisfied final partition - bank on should_execute to prevent firing in schedule
return selected