import os
from datetime import datetime, timedelta
from dagster_examples.bay_bikes.pipelines import generate_training_set_and_train_model
from dagster import daily_schedule
from dagster.core.definitions.decorators import monthly_schedule
from dagster.utils.merger import deep_merge_dicts
weather_etl_environment = generate_training_set_and_train_model.get_preset(
"prod_weather_etl"
).run_config
trip_etl_environment = generate_training_set_and_train_model.get_preset("prod_trip_etl").run_config
now = datetime.now()
@daily_schedule(
pipeline_name="generate_training_set_and_train_model",
start_date=datetime(year=2019, month=1, day=1),
execution_time=(now + timedelta(minutes=1)).time(),
solid_selection=["weather_etl"],
mode="production",
environment_vars={
"POSTGRES_USERNAME": os.environ.get("POSTGRES_USERNAME", ""),
"POSTGRES_PASSWORD": os.environ.get("POSTGRES_PASSWORD", ""),
"POSTGRES_HOST": os.environ.get("POSTGRES_HOST", ""),
"POSTGRES_DB": os.environ.get("POSTGRES_DB", ""),
},
)
def daily_weather_ingest_schedule(date):
import os
from datetime import datetime, timedelta
from dagster_examples.bay_bikes.pipelines import generate_training_set_and_train_model
from dagster import daily_schedule
from dagster.core.definitions.decorators import monthly_schedule
from dagster.utils.merger import deep_merge_dicts
weather_etl_environment = generate_training_set_and_train_model.get_preset(
'prod_weather_etl').environment_dict
trip_etl_environment = generate_training_set_and_train_model.get_preset(
'prod_trip_etl').environment_dict
now = datetime.now()
@daily_schedule(
pipeline_name='generate_training_set_and_train_model',
start_date=datetime(year=2019, month=1, day=1),
execution_time=(now + timedelta(minutes=1)).time(),
solid_selection=['weather_etl'],
mode='production',
environment_vars={
'POSTGRES_USERNAME': os.environ.get('POSTGRES_USERNAME', ''),
'POSTGRES_PASSWORD': os.environ.get('POSTGRES_PASSWORD', ''),
'POSTGRES_HOST': os.environ.get('POSTGRES_HOST', ''),
'POSTGRES_DB': os.environ.get('POSTGRES_DB', ''),
},
)
def daily_weather_ingest_schedule(date):