import os from datetime import datetime, timedelta from dagster_examples.bay_bikes.pipelines import generate_training_set_and_train_model from dagster import daily_schedule from dagster.core.definitions.decorators import monthly_schedule from dagster.utils.merger import deep_merge_dicts weather_etl_environment = generate_training_set_and_train_model.get_preset( "prod_weather_etl" ).run_config trip_etl_environment = generate_training_set_and_train_model.get_preset("prod_trip_etl").run_config now = datetime.now() @daily_schedule( pipeline_name="generate_training_set_and_train_model", start_date=datetime(year=2019, month=1, day=1), execution_time=(now + timedelta(minutes=1)).time(), solid_selection=["weather_etl"], mode="production", environment_vars={ "POSTGRES_USERNAME": os.environ.get("POSTGRES_USERNAME", ""), "POSTGRES_PASSWORD": os.environ.get("POSTGRES_PASSWORD", ""), "POSTGRES_HOST": os.environ.get("POSTGRES_HOST", ""), "POSTGRES_DB": os.environ.get("POSTGRES_DB", ""), }, ) def daily_weather_ingest_schedule(date):
import os from datetime import datetime, timedelta from dagster_examples.bay_bikes.pipelines import generate_training_set_and_train_model from dagster import daily_schedule from dagster.core.definitions.decorators import monthly_schedule from dagster.utils.merger import deep_merge_dicts weather_etl_environment = generate_training_set_and_train_model.get_preset( 'prod_weather_etl').environment_dict trip_etl_environment = generate_training_set_and_train_model.get_preset( 'prod_trip_etl').environment_dict now = datetime.now() @daily_schedule( pipeline_name='generate_training_set_and_train_model', start_date=datetime(year=2019, month=1, day=1), execution_time=(now + timedelta(minutes=1)).time(), solid_selection=['weather_etl'], mode='production', environment_vars={ 'POSTGRES_USERNAME': os.environ.get('POSTGRES_USERNAME', ''), 'POSTGRES_PASSWORD': os.environ.get('POSTGRES_PASSWORD', ''), 'POSTGRES_HOST': os.environ.get('POSTGRES_HOST', ''), 'POSTGRES_DB': os.environ.get('POSTGRES_DB', ''), }, ) def daily_weather_ingest_schedule(date):