def materialization_schedule(): # create weekly partition set schedule_name = "many_events_partitioned" partition_set = PartitionSetDefinition( name="many_events_minutely", pipeline_name="many_events", partition_fn=date_partition_range(start=datetime.datetime(2020, 1, 1)), run_config_fn_for_partition=lambda _: {"storage": { "filesystem": {} }}, ) def _should_execute(context): return backfill_should_execute(context, partition_set) return partition_set.create_schedule_definition( schedule_name=schedule_name, cron_schedule="* * * * *", # tick every minute partition_selector=backfilling_partition_selector, should_execute=_should_execute, )
def define_partitions(): integer_set = PartitionSetDefinition( name="integer_partition", pipeline_name="no_config_pipeline", solid_selection=['return_hello'], mode="default", partition_fn=lambda: [Partition(i) for i in range(10)], run_config_fn_for_partition=lambda _: {"storage": { "filesystem": {} }}, ) enum_set = PartitionSetDefinition( name="enum_partition", pipeline_name="noop_pipeline", partition_fn=lambda: ["one", "two", "three"], run_config_fn_for_partition=lambda _: {"storage": { "filesystem": {} }}, ) return [integer_set, enum_set]
def materialization_schedule(): # create weekly partition set schedule_name = 'many_events_partitioned' partition_set = PartitionSetDefinition( name='many_events_minutely', pipeline_name='many_events', partition_fn=date_partition_range(start=datetime.datetime(2020, 1, 1)), environment_dict_fn_for_partition=lambda _: {'storage': { 'filesystem': {} }}, ) def _should_execute(context): return backfill_should_execute(context, partition_set, schedule_name) return partition_set.create_schedule_definition( schedule_name=schedule_name, cron_schedule="* * * * *", # tick every minute partition_selector=backfilling_partition_selector, should_execute=_should_execute, )
def define_bar_partitions(): def error_name(): raise Exception("womp womp") def error_config(_): raise Exception("womp womp") return { "baz_partitions": PartitionSetDefinition( name="baz_partitions", pipeline_name="baz", partition_fn=lambda: string.digits, run_config_fn_for_partition=lambda partition: { "solids": { "do_input": { "inputs": { "x": { "value": partition.value } } } } }, ), "error_name_partitions": PartitionSetDefinition( name="error_name_partitions", pipeline_name="baz", partition_fn=error_name, ), "error_config_partitions": PartitionSetDefinition( name="error_config_partitions", pipeline_name="baz", partition_fn=error_config, ), }
def backfill_test_schedule(): schedule_name = 'backfill_unreliable_weekly' # create weekly partition set partition_set = PartitionSetDefinition( name='unreliable_weekly', pipeline_name='unreliable_pipeline', partition_fn=date_partition_range( # first sunday of the year start=datetime.datetime(2020, 1, 5), delta=datetime.timedelta(weeks=1), ), run_config_fn_for_partition=lambda _: {'storage': {'filesystem': {}}}, ) def _should_execute(context): return backfill_should_execute(context, partition_set) return partition_set.create_schedule_definition( schedule_name=schedule_name, cron_schedule="* * * * *", # tick every minute partition_selector=backfilling_partition_selector, should_execute=_should_execute, )
def backfill_test_schedule(): schedule_name = "backfill_unreliable_weekly" # create weekly partition set partition_set = PartitionSetDefinition( name="unreliable_weekly", pipeline_name="unreliable_pipeline", partition_fn=date_partition_range( # first sunday of the year start=datetime.datetime(2020, 1, 5), delta_range="weeks", ), ) def _should_execute(context): return backfill_should_execute(context, partition_set) return partition_set.create_schedule_definition( schedule_name=schedule_name, cron_schedule="* * * * *", # tick every minute partition_selector=backfilling_partition_selector, should_execute=_should_execute, execution_timezone=_toys_tz_info(), )
def longitudinal_schedule(): from .longitudinal import longitudinal_config schedule_name = 'longitudinal_demo' partition_set = PartitionSetDefinition( name='ingest_and_train', pipeline_name='longitudinal_pipeline', partition_fn=date_partition_range(start=datetime.datetime(2020, 1, 1)), run_config_fn_for_partition=longitudinal_config, ) def _should_execute(context): return backfill_should_execute(context, partition_set, retry_failed=True) def _partition_selector(context, partition_set): return backfilling_partition_selector(context, partition_set, retry_failed=True) return partition_set.create_schedule_definition( schedule_name=schedule_name, cron_schedule="*/5 * * * *", # tick every 5 minutes partition_selector=_partition_selector, should_execute=_should_execute, )
def define_partitions(): integer_set = PartitionSetDefinition( name="integer_partition", pipeline_name="no_config_pipeline", solid_selection=["return_hello"], mode="default", partition_fn=lambda: [Partition(i) for i in range(10)], run_config_fn_for_partition=lambda _: {"intermediate_storage": { "filesystem": {} }}, tags_fn_for_partition=lambda partition: {"foo": partition.name}, ) enum_set = PartitionSetDefinition( name="enum_partition", pipeline_name="noop_pipeline", partition_fn=lambda: ["one", "two", "three"], run_config_fn_for_partition=lambda _: {"intermediate_storage": { "filesystem": {} }}, ) chained_partition_set = PartitionSetDefinition( name="chained_integer_partition", pipeline_name="chained_failure_pipeline", mode="default", partition_fn=lambda: [Partition(i) for i in range(10)], run_config_fn_for_partition=lambda _: {"intermediate_storage": { "filesystem": {} }}, ) return [integer_set, enum_set, chained_partition_set]
def backfilling_partition_selector(context: ScheduleExecutionContext, partition_set_def: PartitionSetDefinition): runs_by_partition = _fetch_runs_by_partition(context.instance, partition_set_def) selected = None for partition in partition_set_def.get_partitions(): runs = runs_by_partition[partition.name] selected = partition # break when we find the first empty partition if len(runs) == 0: break # may return an already satisfied final partition - bank on should_execute to prevent firing in schedule return selected
def define_baz_partitions(): return [ PartitionSetDefinition( name='baz_partitions', pipeline_name='baz', partition_fn=lambda: string.ascii_lowercase, environment_dict_fn_for_partition=lambda partition: {'solids': { 'do_input': { 'inputs': { 'x': { 'value': partition } } } }}, ) ]
def define_baz_partitions(): return { "baz_partitions": PartitionSetDefinition( name="baz_partitions", pipeline_name="baz", partition_fn=lambda: string.ascii_lowercase, run_config_fn_for_partition=lambda partition: { "solids": { "do_input": { "inputs": { "x": { "value": partition.value } } } } }, ) }
def backfilling_partition_selector( context: ScheduleExecutionContext, partition_set_def: PartitionSetDefinition, retry_failed=False ): status_filters = [PipelineRunStatus.SUCCESS] if retry_failed else None runs_by_partition = _fetch_runs_by_partition( context.instance, partition_set_def, status_filters ) selected = None for partition in partition_set_def.get_partitions(): runs = runs_by_partition[partition.name] selected = partition # break when we find the first empty partition if len(runs) == 0: break # may return an already satisfied final partition - bank on should_execute to prevent firing in schedule return selected
def define_baz_partitions(): return { 'baz_partitions': PartitionSetDefinition( name='baz_partitions', pipeline_name='baz', partition_fn=lambda: string.ascii_lowercase, run_config_fn_for_partition=lambda partition: { 'solids': { 'do_input': { 'inputs': { 'x': { 'value': partition.value } } } } }, ) }
def define_schedules(): integer_partition_set = PartitionSetDefinition( name='scheduled_integer_partitions', pipeline_name='no_config_pipeline', partition_fn=lambda: [Partition(x) for x in range(1, 10)], environment_dict_fn_for_partition=lambda _partition: {"storage": { "filesystem": {} }}, tags_fn_for_partition=lambda _partition: {"test": "1234"}, ) no_config_pipeline_hourly_schedule = ScheduleDefinition( name="no_config_pipeline_hourly_schedule", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", environment_dict={"storage": { "filesystem": {} }}, ) no_config_pipeline_hourly_schedule_with_config_fn = ScheduleDefinition( name="no_config_pipeline_hourly_schedule_with_config_fn", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", environment_dict_fn=lambda _context: {"storage": { "filesystem": {} }}, ) no_config_should_execute = ScheduleDefinition( name="no_config_should_execute", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", environment_dict={"storage": { "filesystem": {} }}, should_execute=lambda _context: False, ) dynamic_config = ScheduleDefinition( name="dynamic_config", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", environment_dict_fn=lambda _context: {"storage": { "filesystem": {} }}, ) partition_based = integer_partition_set.create_schedule_definition( schedule_name="partition_based", cron_schedule="0 0 * * *", ) partition_based_custom_selector = integer_partition_set.create_schedule_definition( schedule_name="partition_based_custom_selector", cron_schedule="0 0 * * *", partition_selector=last_empty_partition, ) @daily_schedule( pipeline_name='no_config_pipeline', start_date=datetime.datetime.now() - datetime.timedelta(days=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=2)).time(), ) def partition_based_decorator(_date): return {"storage": {"filesystem": {}}} @daily_schedule( pipeline_name='multi_mode_with_loggers', start_date=datetime.datetime.now() - datetime.timedelta(days=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=2)).time(), mode='foo_mode', ) def partition_based_multi_mode_decorator(_date): return {"storage": {"filesystem": {}}} @hourly_schedule( pipeline_name='no_config_chain_pipeline', start_date=datetime.datetime.now() - datetime.timedelta(days=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=2)).time(), solid_selection=['return_foo'], ) def solid_selection_hourly_decorator(_date): return {"storage": {"filesystem": {}}} @daily_schedule( pipeline_name='no_config_chain_pipeline', start_date=datetime.datetime.now() - datetime.timedelta(days=2), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=3)).time(), solid_selection=['return_foo'], ) def solid_selection_daily_decorator(_date): return {"storage": {"filesystem": {}}} @monthly_schedule( pipeline_name='no_config_chain_pipeline', start_date=datetime.datetime.now() - datetime.timedelta(days=100), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=4)).time(), solid_selection=['return_foo'], ) def solid_selection_monthly_decorator(_date): return {"storage": {"filesystem": {}}} @weekly_schedule( pipeline_name='no_config_chain_pipeline', start_date=datetime.datetime.now() - datetime.timedelta(days=50), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=5)).time(), solid_selection=['return_foo'], ) def solid_selection_weekly_decorator(_date): return {"storage": {"filesystem": {}}} # Schedules for testing the user error boundary @daily_schedule( pipeline_name='no_config_pipeline', start_date=datetime.datetime.now() - datetime.timedelta(days=1), should_execute=lambda _: asdf, # pylint: disable=undefined-variable ) def should_execute_error_schedule(_date): return {"storage": {"filesystem": {}}} @daily_schedule( pipeline_name='no_config_pipeline', start_date=datetime.datetime.now() - datetime.timedelta(days=1), tags_fn_for_date=lambda _: asdf, # pylint: disable=undefined-variable ) def tags_error_schedule(_date): return {"storage": {"filesystem": {}}} @daily_schedule( pipeline_name='no_config_pipeline', start_date=datetime.datetime.now() - datetime.timedelta(days=1), ) def environment_dict_error_schedule(_date): return asdf # pylint: disable=undefined-variable tagged_pipeline_schedule = ScheduleDefinition( name="tagged_pipeline_schedule", cron_schedule="0 0 * * *", pipeline_name="tagged_pipeline", environment_dict={"storage": { "filesystem": {} }}, ) tagged_pipeline_override_schedule = ScheduleDefinition( name="tagged_pipeline_override_schedule", cron_schedule="0 0 * * *", pipeline_name="tagged_pipeline", environment_dict={"storage": { "filesystem": {} }}, tags={'foo': 'notbar'}, ) invalid_config_schedule = ScheduleDefinition( name="invalid_config_schedule", cron_schedule="0 0 * * *", pipeline_name="pipeline_with_enum_config", environment_dict={"solids": { "takes_an_enum": { 'config': "invalid" } }}, ) return [ environment_dict_error_schedule, no_config_pipeline_hourly_schedule, no_config_pipeline_hourly_schedule_with_config_fn, no_config_should_execute, dynamic_config, partition_based, partition_based_custom_selector, partition_based_decorator, partition_based_multi_mode_decorator, solid_selection_hourly_decorator, solid_selection_daily_decorator, solid_selection_monthly_decorator, solid_selection_weekly_decorator, should_execute_error_schedule, tagged_pipeline_schedule, tagged_pipeline_override_schedule, tags_error_schedule, invalid_config_schedule, ]
import datetime from dagster import Partition, PartitionSetDefinition, ScheduleDefinition, daily_schedule, schedules from dagster.core.definitions.partition import last_empty_partition from dagster.utils.test import FilesytemTestScheduler integer_partition_set = PartitionSetDefinition( name='scheduled_integer_partitions', pipeline_name='no_config_pipeline', partition_fn=lambda: [Partition(x) for x in range(1, 10)], environment_dict_fn_for_partition=lambda _partition: {"storage": { "filesystem": {} }}, tags_fn_for_partition=lambda _partition: {"test": "1234"}, ) @schedules(scheduler=FilesytemTestScheduler) def define_scheduler(): no_config_pipeline_hourly_schedule = ScheduleDefinition( name="no_config_pipeline_hourly_schedule", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", environment_dict={"storage": { "filesystem": {} }}, ) no_config_pipeline_hourly_schedule_with_config_fn = ScheduleDefinition(
} }, "daily_top_action": { "outputs": { "result": { "partitions": [date] } } }, } } asset_lineage_partition_set = PartitionSetDefinition( name="date_partition_set", pipeline_name="asset_lineage_pipeline", partition_fn=get_date_partitions, run_config_fn_for_partition=run_config_for_date_partition, ) def metadata_for_actions(df): return { "min_score": int(df["score"].min()), "max_score": int(df["score"].max()), "sample rows": EventMetadata.md(df[:5].to_markdown()), } class MyDatabaseIOManager(PickledObjectFilesystemIOManager): def _get_path(self, context): keys = context.get_run_scoped_output_identifier()
Partition("GOOG"), Partition("MSFT"), Partition("TSLA"), ] def environment_dict_for_ticker_partition(partition): ticker_symbol = partition.value return { 'solids': { 'query_historical_stock_data': { 'config': { 'symbol': ticker_symbol } } } } stock_ticker_partition_sets = PartitionSetDefinition( name="stock_ticker_partition_sets", pipeline_name="compute_total_stock_volume", partition_fn=get_stock_ticker_partitions, environment_dict_fn_for_partition=environment_dict_for_ticker_partition, ) def define_partitions(): return [stock_ticker_partition_sets]
# start_def def get_date_partitions(): """Every day in the month of May, 2020""" return [Partition(f"2020-05-{str(day).zfill(2)}") for day in range(1, 32)] def run_config_for_date_partition(partition): date = partition.value return {"solids": {"process_data_for_date": {"config": {"date": date}}}} date_partition_set = PartitionSetDefinition( name="date_partition_set", pipeline_name="my_data_pipeline", partition_fn=get_date_partitions, run_config_fn_for_partition=run_config_for_date_partition, ) # end_def @solid def my_solid(_): pass @pipeline def my_data_pipeline(): my_solid()
def partition_config_fn(partition): return { 'solids': { 'announce_partition': { 'config': { 'partition': "Partition is: " + str(partition.value) } } } } ten_days_ago = date.today() - timedelta(days=10) log_date_set = PartitionSetDefinition( name='date_partitions', pipeline_name='log_partitions', partition_fn=date_partition_range(ten_days_ago), environment_dict_fn_for_partition=partition_config_fn, ) us_states_set = PartitionSetDefinition( name='state_partitions', pipeline_name='log_partitions', partition_fn=us_states, environment_dict_fn_for_partition=partition_config_fn, ) @repository_partitions def define_repository_partitions(): return [log_date_set, us_states_set]
return [ Partition("M"), Partition("Tu"), Partition("W"), Partition("Th"), Partition("F"), Partition("Sa"), Partition("Su"), ] def run_config_for_day_partition(partition): day = partition.value return { 'solids': { 'process_data_for_day': { 'config': { 'day_of_week': day } } } } day_partition_set = PartitionSetDefinition( name="day_partition_set", pipeline_name="my_pipeline", partition_fn=get_day_partition, run_config_fn_for_partition=run_config_for_day_partition, )
def partition_config_fn(partition): return { 'solids': { 'announce_partition': { 'config': { 'partition': "Partition is: " + str(partition.value) } } } } ten_days_ago = date.today() - timedelta(days=10) log_date_set = PartitionSetDefinition( name='date_partitions', pipeline_name='log_partitions', partition_fn=date_partition_range(ten_days_ago), environment_dict_fn_for_partition=partition_config_fn, ) def dash_stats_datetime_partition_config(partition): current_date = partition.value yesterday = current_date - timedelta(days=1) date_string = yesterday.strftime("%Y-%m-%d") return { 'resources': { 'bigquery': None, 'slack': { 'config': { 'token': os.getenv('SLACK_TOKEN')
previous_month_last_day = date - datetime.timedelta(days=1) previous_month_first_day = previous_month_last_day.replace(day=1) return { 'solids': { 'query_historical_stock_data': { 'config': { 'ds_start': previous_month_first_day.strftime("%Y-%m-%d"), 'ds_end': previous_month_last_day.strftime("%Y-%m-%d"), 'symbol': 'AAPL', } } } } stock_data_partitions_set = PartitionSetDefinition( name="stock_data_partitions_set", pipeline_name="compute_total_stock_volume", partition_fn=date_partition_range( start=datetime.datetime(2018, 1, 1), end=datetime.datetime(2019, 1, 1), delta=relativedelta(months=1), ), run_config_fn_for_partition=run_config_fn_for_date, ) def define_partitions(): return [stock_data_partitions_set]
start_date = start_date + timedelta(days=1) return partitions def run_config_for_date_partition(partition): date = partition.value config = yaml.load(open(Path(__file__).parent / "realized_trips.yaml", "r")) config["solids"]["download_brt_raw_realized_trips"]["config"]["date"] = date return config daily_partition_set = PartitionSetDefinition( name="daily_partitions", pipeline_name="br_rj_riodejaneiro_gtfs_realized_trips", partition_fn=get_date_partitions, run_config_fn_for_partition=run_config_for_date_partition, mode="dev", ) def daily_partition_selector(context, partition_set): partitions = partition_set.get_partitions(context.scheduled_execution_time) return partitions[-2] # one day before run day (today) daily_schedule = daily_partition_set.create_schedule_definition( "br_rj_riodejaneiro_gtfs_realized_trips", "0 1 * * *", partition_selector=daily_partition_selector,
def define_schedules(): integer_partition_set = PartitionSetDefinition( name="scheduled_integer_partitions", pipeline_name="no_config_pipeline", partition_fn=lambda: [Partition(x) for x in range(1, 10)], tags_fn_for_partition=lambda _partition: {"test": "1234"}, ) no_config_pipeline_hourly_schedule = ScheduleDefinition( name="no_config_pipeline_hourly_schedule", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", ) no_config_pipeline_hourly_schedule_with_config_fn = ScheduleDefinition( name="no_config_pipeline_hourly_schedule_with_config_fn", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", ) no_config_should_execute = ScheduleDefinition( name="no_config_should_execute", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", should_execute=lambda _context: False, ) dynamic_config = ScheduleDefinition( name="dynamic_config", cron_schedule="0 0 * * *", pipeline_name="no_config_pipeline", ) partition_based = integer_partition_set.create_schedule_definition( schedule_name="partition_based", cron_schedule="0 0 * * *", partition_selector=last_empty_partition, ) @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight().subtract(days=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=2)).time(), ) def partition_based_decorator(_date): return {} @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight().subtract(days=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=2)).time(), default_status=DefaultScheduleStatus.RUNNING, ) def running_in_code_schedule(_date): return {} @daily_schedule( pipeline_name="multi_mode_with_loggers", start_date=today_at_midnight().subtract(days=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=2)).time(), mode="foo_mode", ) def partition_based_multi_mode_decorator(_date): return {} @hourly_schedule( pipeline_name="no_config_chain_pipeline", start_date=today_at_midnight().subtract(days=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=2)).time(), solid_selection=["return_foo"], ) def solid_selection_hourly_decorator(_date): return {} @daily_schedule( pipeline_name="no_config_chain_pipeline", start_date=today_at_midnight().subtract(days=2), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=3)).time(), solid_selection=["return_foo"], ) def solid_selection_daily_decorator(_date): return {} @monthly_schedule( pipeline_name="no_config_chain_pipeline", start_date=(today_at_midnight().subtract(days=100)).replace(day=1), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=4)).time(), solid_selection=["return_foo"], ) def solid_selection_monthly_decorator(_date): return {} @weekly_schedule( pipeline_name="no_config_chain_pipeline", start_date=today_at_midnight().subtract(days=50), execution_time=(datetime.datetime.now() + datetime.timedelta(hours=5)).time(), solid_selection=["return_foo"], ) def solid_selection_weekly_decorator(_date): return {} # Schedules for testing the user error boundary @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight().subtract(days=1), should_execute=lambda _: asdf, # pylint: disable=undefined-variable ) def should_execute_error_schedule(_date): return {} @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight().subtract(days=1), tags_fn_for_date=lambda _: asdf, # pylint: disable=undefined-variable ) def tags_error_schedule(_date): return {} @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight().subtract(days=1), ) def run_config_error_schedule(_date): return asdf # pylint: disable=undefined-variable @daily_schedule( pipeline_name="no_config_pipeline", start_date=today_at_midnight("US/Central") - datetime.timedelta(days=1), execution_timezone="US/Central", ) def timezone_schedule(_date): return {} tagged_pipeline_schedule = ScheduleDefinition( name="tagged_pipeline_schedule", cron_schedule="0 0 * * *", pipeline_name="tagged_pipeline", ) tagged_pipeline_override_schedule = ScheduleDefinition( name="tagged_pipeline_override_schedule", cron_schedule="0 0 * * *", pipeline_name="tagged_pipeline", tags={"foo": "notbar"}, ) invalid_config_schedule = ScheduleDefinition( name="invalid_config_schedule", cron_schedule="0 0 * * *", pipeline_name="pipeline_with_enum_config", run_config={"solids": { "takes_an_enum": { "config": "invalid" } }}, ) return [ run_config_error_schedule, no_config_pipeline_hourly_schedule, no_config_pipeline_hourly_schedule_with_config_fn, no_config_should_execute, dynamic_config, partition_based, partition_based_decorator, partition_based_multi_mode_decorator, solid_selection_hourly_decorator, solid_selection_daily_decorator, solid_selection_monthly_decorator, solid_selection_weekly_decorator, should_execute_error_schedule, tagged_pipeline_schedule, tagged_pipeline_override_schedule, tags_error_schedule, timezone_schedule, invalid_config_schedule, running_in_code_schedule, ]
Partition("GOOG"), Partition("MSFT"), Partition("TSLA"), ] def run_config_for_ticker_partition(partition): ticker_symbol = partition.value return { "solids": { "query_historical_stock_data": { "config": { "symbol": ticker_symbol } } } } stock_ticker_partition_sets = PartitionSetDefinition( name="stock_ticker_partition_sets", pipeline_name="compute_total_stock_volume", partition_fn=get_stock_ticker_partitions, run_config_fn_for_partition=run_config_for_ticker_partition, ) def define_partitions(): return [stock_ticker_partition_sets]