def weekday_partition_selector( ctx: ScheduleExecutionContext, partition_set: PartitionSetDefinition ) -> Union[Partition, List[Partition]]: """Maps a schedule execution time to the corresponding partition or list of partitions that should be executed at that time""" partitions = partition_set.get_partitions(ctx.scheduled_execution_time) weekday = ctx.scheduled_execution_time.weekday( ) if ctx.scheduled_execution_time else 0 return partitions[weekday]
def backfilling_partition_selector(context: ScheduleExecutionContext, partition_set_def: PartitionSetDefinition): runs_by_partition = _fetch_runs_by_partition(context.instance, partition_set_def) selected = None for partition in partition_set_def.get_partitions(): runs = runs_by_partition[partition.name] selected = partition # break when we find the first empty partition if len(runs) == 0: break # may return an already satisfied final partition - bank on should_execute to prevent firing in schedule return selected
def backfilling_partition_selector( context: ScheduleExecutionContext, partition_set_def: PartitionSetDefinition, retry_failed=False ): status_filters = [PipelineRunStatus.SUCCESS] if retry_failed else None runs_by_partition = _fetch_runs_by_partition( context.instance, partition_set_def, status_filters ) selected = None for partition in partition_set_def.get_partitions(): runs = runs_by_partition[partition.name] selected = partition # break when we find the first empty partition if len(runs) == 0: break # may return an already satisfied final partition - bank on should_execute to prevent firing in schedule return selected