def inner(fn): check.callable_param(fn, "fn") schedule_name = name or fn.__name__ tags_fn_for_partition_value = lambda partition: {} if tags_fn_for_date: tags_fn_for_partition_value = lambda partition: tags_fn_for_date( partition.value) partition_set = PartitionSetDefinition( name="{}_partitions".format(schedule_name), pipeline_name=pipeline_name, partition_fn=partition_fn, run_config_fn_for_partition=lambda partition: fn(partition.value), solid_selection=solid_selection, tags_fn_for_partition=tags_fn_for_partition_value, mode=mode, ) return partition_set.create_schedule_definition( schedule_name, cron_schedule, should_execute=should_execute, environment_vars=environment_vars, partition_selector=create_offset_partition_selector( execution_time_to_partition_fn=execution_time_to_partition_fn, ), execution_timezone=execution_timezone, )
def inner(fn: Callable[..., Dict[str, Any]]) -> ScheduleDefinition: check.callable_param(fn, "fn") schedule_name = name or fn.__name__ tags_fn_for_partition_value: Callable[["Partition"], Optional[Dict[ str, str]]] = lambda partition: {} if tags_fn_for_date: tags_fn = cast( Callable[[datetime.datetime], Optional[Dict[str, str]]], tags_fn_for_date) tags_fn_for_partition_value = lambda partition: tags_fn(partition. value) partition_set = PartitionSetDefinition( name="{}_partitions".format(schedule_name), pipeline_name=pipeline_name, partition_fn=partition_fn, run_config_fn_for_partition=lambda partition: fn(partition.value), solid_selection=solid_selection, tags_fn_for_partition=tags_fn_for_partition_value, mode=mode, ) return partition_set.create_schedule_definition( schedule_name, cron_schedule, should_execute=should_execute, environment_vars=environment_vars, partition_selector=create_offset_partition_selector( execution_time_to_partition_fn=execution_time_to_partition_fn, ), execution_timezone=execution_timezone, )
def create_hourly_hn_download_schedule(): pipeline_name = "download_pipeline" schedule_name = "hourly_hn_download_schedule" start_date = datetime.datetime(2021, 1, 1) execution_time = datetime.time(0, 0) partitions_def = ScheduleTimeBasedPartitionsDefinition( schedule_type=ScheduleType.HOURLY, start=start_date, execution_time=execution_time, fmt=DEFAULT_HOURLY_FORMAT_WITH_TIMEZONE, timezone="UTC", ) partition_set = PartitionSetDefinition( name="{}_partitions".format(schedule_name), pipeline_name=pipeline_name, # type: ignore[arg-type] run_config_fn_for_partition=lambda partition: get_hourly_download_def_schedule_config(partition.value), mode="prod", partitions_def=partitions_def, ) schedule_def = partition_set.create_schedule_definition( schedule_name, partitions_def.get_cron_schedule(), partition_selector=create_offset_partition_selector( execution_time_to_partition_fn=partitions_def. get_execution_time_to_partition_fn(), ), execution_timezone="UTC", decorated_fn=get_hourly_download_def_schedule_config, job=download_comments_and_stories_prod, ) return schedule_def
def inner(fn: Callable[[datetime.datetime], Dict[str, Any]]) -> PartitionScheduleDefinition: check.callable_param(fn, "fn") schedule_name = name or fn.__name__ tags_fn_for_partition_value: Callable[ ["Partition"], Optional[Dict[str, str]] ] = lambda partition: {} if tags_fn_for_date: tags_fn = cast( Callable[[datetime.datetime], Optional[Dict[str, str]]], tags_fn_for_date ) tags_fn_for_partition_value = lambda partition: tags_fn(partition.value) fmt = ( DEFAULT_HOURLY_FORMAT_WITH_TIMEZONE if execution_timezone else DEFAULT_HOURLY_FORMAT_WITHOUT_TIMEZONE ) partitions_def = ScheduleTimeBasedPartitionsDefinition( schedule_type=ScheduleType.HOURLY, start=start_date, execution_time=execution_time, end=end_date, fmt=fmt, timezone=execution_timezone, offset=partition_hours_offset, ) partition_set = PartitionSetDefinition( name="{}_partitions".format(schedule_name), pipeline_name=pipeline_name, # type: ignore[arg-type] run_config_fn_for_partition=lambda partition: fn(partition.value), solid_selection=solid_selection, tags_fn_for_partition=tags_fn_for_partition_value, mode=mode, partitions_def=partitions_def, ) schedule_def = partition_set.create_schedule_definition( schedule_name, partitions_def.get_cron_schedule(), should_execute=should_execute, environment_vars=environment_vars, partition_selector=create_offset_partition_selector( execution_time_to_partition_fn=partitions_def.get_execution_time_to_partition_fn(), ), execution_timezone=execution_timezone, description=description, decorated_fn=fn, default_status=default_status, ) update_wrapper(schedule_def, wrapped=fn) return schedule_def