def test_presets():
@solid(config={'error': Bool})
def can_fail(context):
if context.solid_config['error']:
raise Exception('I did an error')
return 'cool'
@lambda_solid
def always_fail():
raise Exception('I always do this')
pipeline = PipelineDefinition(
name='simple',
solid_defs=[can_fail, always_fail],
preset_defs=[
PresetDefinition.from_files(
'passing',
environment_files=[file_relative_path(__file__, 'pass_env.yaml')],
solid_subset=['can_fail'],
),
PresetDefinition(
'passing_direct_dict',
environment_dict={'solids': {'can_fail': {'config': {'error': False}}}},
solid_subset=['can_fail'],
),
PresetDefinition.from_files(
'failing_1',
environment_files=[file_relative_path(__file__, 'fail_env.yaml')],
solid_subset=['can_fail'],
),
PresetDefinition.from_files(
'failing_2', environment_files=[file_relative_path(__file__, 'pass_env.yaml')]
),
],
)
with pytest.raises(DagsterInvalidDefinitionError):
PresetDefinition.from_files(
'invalid_1', environment_files=[file_relative_path(__file__, 'not_a_file.yaml')]
)
with pytest.raises(DagsterInvariantViolationError):
PresetDefinition.from_files(
'invalid_2',
environment_files=[file_relative_path(__file__, 'test_repository_definition.py')],
)
assert execute_pipeline_with_preset(pipeline, 'passing').success
assert execute_pipeline_with_preset(pipeline, 'passing_direct_dict').success
with pytest.raises(Exception):
execute_pipeline_with_preset(pipeline, 'failing_1')
with pytest.raises(Exception):
execute_pipeline_with_preset(pipeline, 'failing_2')
with pytest.raises(Exception, match="Could not find preset"):
execute_pipeline_with_preset(pipeline, 'not_failing')
def define_multi_mode_with_resources_pipeline():
# API red alert. One has to wrap a type in Field because it is callable
@resource(config_schema=Int)
def adder_resource(init_context):
return lambda x: x + init_context.resource_config
@resource(config_schema=Int)
def multer_resource(init_context):
return lambda x: x * init_context.resource_config
@resource(config_schema={"num_one": Int, "num_two": Int})
def double_adder_resource(init_context):
return (
lambda x: x
+ init_context.resource_config["num_one"]
+ init_context.resource_config["num_two"]
)
@solid(required_resource_keys={"op"})
def apply_to_three(context):
return context.resources.op(3)
return PipelineDefinition(
name="multi_mode_with_resources",
solid_defs=[apply_to_three],
mode_defs=[
ModeDefinition(
name="add_mode", resource_defs={"op": adder_resource, "io_manager": fs_io_manager}
),
ModeDefinition(name="mult_mode", resource_defs={"op": multer_resource}),
ModeDefinition(
name="double_adder_mode",
resource_defs={"op": double_adder_resource},
description="Mode that adds two numbers to thing",
),
],
preset_defs=[
PresetDefinition.from_files(
"add",
mode="add_mode",
config_files=[
file_relative_path(
__file__, "../environments/multi_mode_with_resources/add_mode.yaml"
)
],
),
PresetDefinition(
"multiproc",
mode="add_mode",
run_config={
"resources": {"op": {"config": 2}},
"execution": {"multiprocess": {}},
},
),
],
)
def define_multi_mode_with_resources_pipeline():
# API red alert. One has to wrap a type in Field because it is callable
@resource(config=Int)
def adder_resource(init_context):
return lambda x: x + init_context.resource_config
@resource(config=Int)
def multer_resource(init_context):
return lambda x: x * init_context.resource_config
@resource(config={'num_one': Int, 'num_two': Int})
def double_adder_resource(init_context):
return (
lambda x: x
+ init_context.resource_config['num_one']
+ init_context.resource_config['num_two']
)
@solid(required_resource_keys={'op'})
def apply_to_three(context):
return context.resources.op(3)
return PipelineDefinition(
name='multi_mode_with_resources',
solid_defs=[apply_to_three],
mode_defs=[
ModeDefinition(name='add_mode', resource_defs={'op': adder_resource}),
ModeDefinition(name='mult_mode', resource_defs={'op': multer_resource}),
ModeDefinition(
name='double_adder_mode',
resource_defs={'op': double_adder_resource},
description='Mode that adds two numbers to thing',
),
],
preset_defs=[
PresetDefinition.from_files(
'add',
mode='add_mode',
environment_files=[
file_relative_path(
__file__, './environments/multi_mode_with_resources/add_mode.yaml'
)
],
),
PresetDefinition(
'multiproc',
mode='add_mode',
environment_dict={
'resources': {'op': {'config': 2}},
'execution': {'multiprocess': {}},
'storage': {'filesystem': {}},
},
),
],
)
def define_multi_mode_with_resources_pipeline():
@resource(config_field=Field(Int))
def adder_resource(init_context):
return lambda x: x + init_context.resource_config
@resource(config_field=Field(Int))
def multer_resource(init_context):
return lambda x: x * init_context.resource_config
@resource(config={'num_one': Field(Int), 'num_two': Field(Int)})
def double_adder_resource(init_context):
return (lambda x: x + init_context.resource_config['num_one'] +
init_context.resource_config['num_two'])
@solid
def apply_to_three(context):
return context.resources.op(3)
return PipelineDefinition(
name='multi_mode_with_resources',
solid_defs=[apply_to_three],
mode_defs=[
ModeDefinition(name='add_mode',
resource_defs={'op': adder_resource}),
ModeDefinition(name='mult_mode',
resource_defs={'op': multer_resource}),
ModeDefinition(
name='double_adder_mode',
resource_defs={'op': double_adder_resource},
description='Mode that adds two numbers to thing',
),
],
preset_defs=[
PresetDefinition.from_files(
'add',
mode='add_mode',
environment_files=[
script_relative_path(
'./environments/multi_mode_with_resources/add_mode.yaml'
)
],
)
],
)
if context.solid_config['return_wrong_type']:
return string + string
return int(string)
@pipeline(
mode_defs=[
ModeDefinition(
name='errorable_mode', resource_defs={'errorable_resource': define_errorable_resource()}
)
],
preset_defs=[
PresetDefinition.from_files(
'passing',
environment_files=[file_relative_path(__file__, 'environments/error.yaml')],
mode='errorable_mode',
)
],
)
def error_monster():
start = emit_num.alias('start')()
middle = num_to_str.alias('middle')(num=start)
str_to_num.alias('end')(string=middle)
if __name__ == '__main__':
result = execute_pipeline(
error_monster,
{
'solids': {
'db_info': redshift_db_info_resource,
'tempfile': tempfile_resource,
'file_cache': s3_file_cache,
},
system_storage_defs=s3_plus_default_storage_defs,
)
@pipeline(
# ordered so the local is first and therefore the default
mode_defs=[local_mode, test_mode, prod_mode],
preset_defs=[
PresetDefinition.from_files(
name='local_fast',
mode='local',
environment_files=[
file_relative_path(__file__, 'environments/local_base.yaml'),
file_relative_path(__file__, 'environments/local_fast_ingest.yaml'),
],
),
PresetDefinition.from_files(
name='local_full',
mode='local',
environment_files=[
file_relative_path(__file__, 'environments/local_base.yaml'),
file_relative_path(__file__, 'environments/local_full_ingest.yaml'),
],
),
],
)
def airline_demo_ingest_pipeline():
# on time data
def test_presets():
@solid(config_schema={"error": Bool})
def can_fail(context):
if context.solid_config["error"]:
raise Exception("I did an error")
return "cool"
@lambda_solid
def always_fail():
raise Exception("I always do this")
pipe = PipelineDefinition(
name="simple",
solid_defs=[can_fail, always_fail],
preset_defs=[
PresetDefinition.from_files(
"passing",
config_files=[file_relative_path(__file__, "pass_env.yaml")],
solid_selection=["can_fail"],
),
PresetDefinition.from_files(
"passing_overide_to_fail",
config_files=[file_relative_path(__file__, "pass_env.yaml")],
solid_selection=["can_fail"],
).with_additional_config(
{"solids": {
"can_fail": {
"config": {
"error": True
}
}
}}),
PresetDefinition(
"passing_direct_dict",
run_config={
"solids": {
"can_fail": {
"config": {
"error": False
}
}
}
},
solid_selection=["can_fail"],
),
PresetDefinition.from_files(
"failing_1",
config_files=[file_relative_path(__file__, "fail_env.yaml")],
solid_selection=["can_fail"],
),
PresetDefinition.from_files(
"failing_2",
config_files=[file_relative_path(__file__, "pass_env.yaml")]),
PresetDefinition(
"subset",
solid_selection=["can_fail"],
),
],
)
with pytest.raises(DagsterInvariantViolationError):
PresetDefinition.from_files(
"invalid_1",
config_files=[file_relative_path(__file__, "not_a_file.yaml")])
with pytest.raises(DagsterInvariantViolationError):
PresetDefinition.from_files(
"invalid_2",
config_files=[
file_relative_path(__file__, "test_repository_definition.py")
],
)
assert execute_pipeline(pipe, preset="passing").success
assert execute_pipeline(pipe, preset="passing_direct_dict").success
assert execute_pipeline(pipe, preset="failing_1",
raise_on_error=False).success == False
assert execute_pipeline(pipe, preset="failing_2",
raise_on_error=False).success == False
with pytest.raises(DagsterInvariantViolationError,
match="Could not find preset"):
execute_pipeline(pipe, preset="not_failing", raise_on_error=False)
assert (execute_pipeline(pipe,
preset="passing_overide_to_fail",
raise_on_error=False).success == False)
assert execute_pipeline(
pipe,
preset="passing",
run_config={
"solids": {
"can_fail": {
"config": {
"error": False
}
}
}
},
).success
with pytest.raises(
check.CheckError,
match=re.escape(
"The environment set in preset 'passing' does not agree with the environment passed "
"in the `run_config` argument."),
):
execute_pipeline(
pipe,
preset="passing",
run_config={"solids": {
"can_fail": {
"config": {
"error": True
}
}
}},
)
assert execute_pipeline(
pipe,
preset="subset",
run_config={
"solids": {
"can_fail": {
"config": {
"error": False
}
}
}
},
).success
output_def=OutputDefinition(dagster_pd.DataFrame),
)
def always_fails_solid(**_kwargs):
raise Exception('I am a programmer and I make error')
@pipeline
def pandas_hello_world_fails():
always_fails_solid(sum_sq_solid=sum_sq_solid(sum_df=sum_solid()) # pylint: disable=no-value-for-parameter
)
@pipeline(preset_defs=[
PresetDefinition.from_files(
'test',
environment_files=[
file_relative_path(__file__,
'environments/pandas_hello_world_test.yaml')
],
),
PresetDefinition.from_files(
'prod',
environment_files=[
file_relative_path(__file__,
'environments/pandas_hello_world_prod.yaml')
],
),
])
def pandas_hello_world():
sum_sq_solid(sum_solid()) # pylint: disable=no-value-for-parameter
],
)
def merge_staging_to_delta_table(properties):
prop_s3_coordinate = upload_to_s3(cache_properies_from_rest_api(properties))
# return assets for property
return merge_property_delta(input_dataframe=flatten_json(s3_to_df(prop_s3_coordinate)))
@pipeline(
mode_defs=[local_mode],
preset_defs=[
PresetDefinition.from_files(
name='local',
mode='local',
config_files=[
file_relative_path(__file__, 'config_environments/local_base.yaml'),
file_relative_path(__file__, 'config_pipelines/scrape_realestate.yaml'),
],
),
],
)
def scrape_realestate():
merge_staging_to_delta_table.alias('merge_SO_buy')(
list_changed_properties.alias('list_SO_buy_flat')()
)
data_exploration()
list_changed_properties.alias('list_BE_buy_flat')()
)
return people.count()
mode = ModeDefinition(
name='prod',
resource_defs={
'pyspark_step_launcher': emr_pyspark_step_launcher,
'pyspark': pyspark_resource,
's3': s3_resource,
},
system_storage_defs=s3_plus_default_storage_defs,
)
preset = PresetDefinition.from_files(
name='prod',
mode='prod',
environment_files=['prod_resources.yaml', 's3_storage.yaml'],
)
@pipeline(
mode_defs=[mode],
preset_defs=[preset],
)
def my_pipeline():
count_people(filter_over_50(make_people()))
def define_repository():
return RepositoryDefinition('emr_pyspark', pipeline_defs=[my_pipeline])
def test_presets():
@solid(config_schema={'error': Bool})
def can_fail(context):
if context.solid_config['error']:
raise Exception('I did an error')
return 'cool'
@lambda_solid
def always_fail():
raise Exception('I always do this')
pipeline = PipelineDefinition(
name='simple',
solid_defs=[can_fail, always_fail],
preset_defs=[
PresetDefinition.from_files(
'passing',
environment_files=[
file_relative_path(__file__, 'pass_env.yaml')
],
solid_selection=['can_fail'],
),
PresetDefinition.from_files(
'passing_overide_to_fail',
environment_files=[
file_relative_path(__file__, 'pass_env.yaml')
],
solid_selection=['can_fail'],
).with_additional_config(
{'solids': {
'can_fail': {
'config': {
'error': True
}
}
}}),
PresetDefinition(
'passing_direct_dict',
run_config={
'solids': {
'can_fail': {
'config': {
'error': False
}
}
}
},
solid_selection=['can_fail'],
),
PresetDefinition.from_files(
'failing_1',
environment_files=[
file_relative_path(__file__, 'fail_env.yaml')
],
solid_selection=['can_fail'],
),
PresetDefinition.from_files('failing_2',
environment_files=[
file_relative_path(
__file__, 'pass_env.yaml')
]),
PresetDefinition(
'subset',
solid_selection=['can_fail'],
),
],
)
with pytest.raises(DagsterInvalidDefinitionError):
PresetDefinition.from_files('invalid_1',
environment_files=[
file_relative_path(
__file__, 'not_a_file.yaml')
])
with pytest.raises(DagsterInvariantViolationError):
PresetDefinition.from_files(
'invalid_2',
environment_files=[
file_relative_path(__file__, 'test_repository_definition.py')
],
)
assert execute_pipeline(pipeline, preset='passing').success
assert execute_pipeline(pipeline, preset='passing_direct_dict').success
assert execute_pipeline(pipeline, preset='failing_1',
raise_on_error=False).success == False
assert execute_pipeline(pipeline, preset='failing_2',
raise_on_error=False).success == False
with pytest.raises(DagsterInvariantViolationError,
match='Could not find preset'):
execute_pipeline(pipeline, preset='not_failing', raise_on_error=False)
assert (execute_pipeline(pipeline,
preset='passing_overide_to_fail',
raise_on_error=False).success == False)
assert execute_pipeline(
pipeline,
preset='passing',
run_config={
'solids': {
'can_fail': {
'config': {
'error': False
}
}
}
},
).success
with pytest.raises(
check.CheckError,
match=re.escape(
'The environment set in preset \'passing\' does not agree with the environment passed '
'in the `run_config` argument.'),
):
execute_pipeline(
pipeline,
preset='passing',
run_config={'solids': {
'can_fail': {
'config': {
'error': True
}
}
}},
)
assert execute_pipeline(
pipeline,
preset='subset',
run_config={
'solids': {
'can_fail': {
'config': {
'error': False
}
}
}
},
).success
}
}
},
'resources': {
'warehouse': {
'config': {
'conn_str': ':memory:'
}
}
},
},
mode='unittest',
),
PresetDefinition.from_files(
'dev',
config_files=[
file_relative_path(__file__, 'presets_dev_warehouse.yaml'),
file_relative_path(__file__, 'presets_csv.yaml'),
],
mode='dev',
),
],
)
def presets_pipeline():
normalize_calories(read_csv())
if __name__ == '__main__':
result = execute_pipeline(presets_pipeline, preset='unittest')
assert result.success
@solid
def make_burger(_):
return 'cheese burger'
@solid
def add_sauce(_, food, sauce: Sauce):
return '{food} with {flavor} sauce'.format(food=food, flavor=sauce.flavor)
@solid
def inspect_sauce(context, sauce: Sauce) -> Sauce:
context.log.info('The sauce tastes {flavor}'.format(flavor=sauce.flavor))
return sauce
@pipeline(preset_defs=[
PresetDefinition.from_files(
'test_input', [script_relative_path('./custom_type_input.yaml')]),
PresetDefinition.from_files(
'test_output',
[
script_relative_path('./custom_type_output.yaml'),
script_relative_path('./custom_type_input.yaml'),
],
),
])
def burger_time():
inspected_sauce = inspect_sauce()
add_sauce(make_burger(), inspected_sauce)
@pipeline(
mode_defs=[
ModeDefinition(
intermediate_storage_defs=s3_plus_default_intermediate_storage_defs,
resource_defs={'s3': s3_resource},
executor_defs=default_executors + [celery_k8s_job_executor],
)
],
preset_defs=[
PresetDefinition.from_files(
'example',
config_files=[
file_relative_path(
__file__,
os.path.join('..', 'run_config', 'celery_k8s.yaml')),
file_relative_path(
__file__, os.path.join('..', 'run_config',
'pipeline.yaml')),
],
mode='default',
),
],
)
def example_pipe():
count_letters(multiply_the_word())
@repository
def example_repo():
return [example_pipe]
},
),
ModeDefinition(
name='production',
resource_defs={
'credentials_vault': credentials_vault,
'transporter': production_transporter,
'volume': mount,
},
),
],
preset_defs=[
PresetDefinition.from_files(
'dev',
mode='local',
environment_files=[
file_relative_path(__file__, 'environments/weather_base.yaml'),
file_relative_path(__file__, 'environments/weather_dev.yaml'),
],
),
PresetDefinition.from_files(
'production',
mode='production',
environment_files=[
file_relative_path(__file__, 'environments/weather_base.yaml'),
file_relative_path(__file__, 'environments/weather_production.yaml'),
],
),
],
)
def extract_daily_weather_data_pipeline():
upload_weather_report = upload_file_to_bucket.alias('upload_weather_report')
def define_papermill_pandas_hello_world_solid():
return dagstermill.define_dagstermill_solid(
name='papermill_pandas_hello_world',
notebook_path=nb_test_path('papermill_pandas_hello_world'),
input_defs=[InputDefinition(name='df', dagster_type=DataFrame)],
output_defs=[OutputDefinition(DataFrame)],
)
@pipeline(preset_defs=[
PresetDefinition.from_files(
'test',
config_files=[
file_relative_path(
__file__,
'pandas_hello_world/environments/papermill_pandas_hello_world_test.yaml',
)
],
),
PresetDefinition.from_files(
'prod',
config_files=[
file_relative_path(
__file__,
'pandas_hello_world/environments/papermill_pandas_hello_world_prod.yaml',
)
],
),
])
def papermill_pandas_hello_world_pipeline():
}
}
},
'resources': {
'warehouse': {
'config': {
'conn_str': ':memory:'
}
}
},
},
mode='unittest',
),
PresetDefinition.from_files(
'dev',
environment_files=[
script_relative_path('presets_dev_warehouse.yaml'),
script_relative_path('presets_csv.yaml'),
],
mode='dev',
),
],
)
def presets_pipeline():
normalize_calories(read_csv())
if __name__ == '__main__':
result = execute_pipeline_with_preset(presets_pipeline, 'unittest')
assert result.success
# (use local disk instead of S3 for local runs, sqlite instead of Snowflake, etc)
prod_mode = ModeDefinition(
name='prod',
resource_defs={
's3': s3_resource,
'snowflake': snowflake_resource
},
system_storage_defs=s3_plus_default_storage_defs,
)
# Presets are a type of configuration where each file can overwrite values in the next, this is where per-environment
# settings and, I guess, secrets go?
preset_defs = PresetDefinition.from_files(
name='prod',
mode='prod',
environment_files=[
file_relative_path(__file__, 'environments/shared.yaml'),
],
)
# Solids are the units of execution that are steps in a pipeline. Lambda solids are simpler and do not require
# configuration or context.
@lambda_solid(
output_def=OutputDefinition(DataFrame), # Describes what this solid will return
)
def generate_test_data():
"""
Generate random data
"""
df = pd.DataFrame(np.random.randint(0, 100, size=(100, 4)),
input_defs=[InputDefinition("sum_sq_solid", dagster_pd.DataFrame)],
output_def=OutputDefinition(dagster_pd.DataFrame),
)
def always_fails_solid(**_kwargs):
raise Exception("I am a programmer and I make error")
@pipeline
def pandas_hello_world_fails():
always_fails_solid(sum_sq_solid=sum_sq_solid(sum_df=sum_solid()))
@pipeline(preset_defs=[
PresetDefinition.from_files(
"test",
config_files=[
file_relative_path(__file__,
"environments/pandas_hello_world_test.yaml")
],
),
PresetDefinition.from_files(
"prod",
config_files=[
file_relative_path(__file__,
"environments/pandas_hello_world_prod.yaml")
],
),
])
def pandas_hello_world():
sum_sq_solid(sum_solid())
@pipeline(
mode_defs=[
ModeDefinition(
name='development',
resource_defs={
'credentials_vault': credentials_vault,
'postgres_db': postgres_db_info_resource,
},
),
],
preset_defs=[
PresetDefinition.from_files(
'development',
mode='development',
environment_files=[
file_relative_path(__file__,
'environments/dev_database_resources.yaml'),
file_relative_path(__file__, 'environments/weather.yaml'),
],
),
],
)
def extract_daily_weather_data_pipeline():
insert_weather_report_into_table = insert_into_table.alias(
'insert_weather_report_into_table')
insert_weather_report_into_table(
download_weather_report_from_weather_api())
@pipeline(
mode_defs=[
from dagster_aws.s3 import s3_system_storage, s3_resource
import numpy as np
celery_mode_defs = [
ModeDefinition(name='default',
resource_defs={'s3': s3_resource},
executor_defs=default_executors + [celery_executor],
system_storage_defs=default_system_storage_defs +
[s3_system_storage])
]
celery_yaml_path = Path(__file__).parent / 'celery_execution.yaml'
presets = [
PresetDefinition.from_files(name='celery',
environment_files=[str(celery_yaml_path)],
mode='default')
]
@solid
def generate_number(_):
print('hello')
num = np.random.randint(low=1, high=100)
return num
@solid
def square_number(_, number):
square = np.power(number, 2)
return square
@pipeline(
mode_defs=[
ModeDefinition(
name='default',
resource_defs={
's3': s3_resource,
'snowflake': snowflake_resource,
'spark': spark_resource,
},
)
],
preset_defs=[
PresetDefinition.from_files(
name='default',
mode='default',
environment_files=[file_relative_path(__file__, 'environments/default.yaml')],
)
],
)
def event_ingest_pipeline():
event_ingest = create_spark_solid(
name='event_ingest',
main_class='io.dagster.events.EventPipeline',
description='Ingest events from JSON to Parquet',
)
@solid(input_defs=[InputDefinition('start', Nothing)], required_resource_keys={'snowflake'})
def snowflake_load(context):
# TODO: express dependency of this solid on event_ingest
context.resources.snowflake.load_table_from_local_parquet(
"io_manager": s3_pickle_io_manager
},
executor_defs=default_executors + [celery_k8s_job_executor],
),
ModeDefinition(
name="test",
executor_defs=default_executors + [celery_k8s_job_executor],
),
],
preset_defs=[
PresetDefinition.from_files(
"celery_k8s",
config_files=[
file_relative_path(
__file__,
os.path.join("..", "run_config", "celery_k8s.yaml")),
file_relative_path(
__file__, os.path.join("..", "run_config",
"pipeline.yaml")),
],
mode="default",
),
PresetDefinition.from_files(
"default",
config_files=[
file_relative_path(
__file__, os.path.join("..", "run_config",
"pipeline.yaml")),
],
mode="default",
),
],
@pipeline(
mode_defs=[
ModeDefinition(
name='development',
resource_defs={
'credentials_vault': credentials_vault,
'postgres_db': postgres_db_info_resource,
},
),
],
preset_defs=[
PresetDefinition.from_files(
'development',
mode='development',
environment_files=[
file_relative_path(__file__, 'environments/dev_resources.yaml'),
file_relative_path(__file__, 'environments/weather.yaml'),
],
),
],
)
def extract_daily_weather_data_pipeline():
insert_weather_report_into_table = insert_row_into_table.alias(
'insert_weather_report_into_table'
)
insert_weather_report_into_table(download_weather_report_from_weather_api())
# TODO: Add Local Mode when tests are written
@pipeline(
mode_defs=[
}
}
},
"resources": {
"warehouse": {
"config": {
"conn_str": ":memory:"
}
}
},
},
mode="unittest",
),
PresetDefinition.from_files(
"dev",
config_files=[
file_relative_path(__file__, "presets_dev_warehouse.yaml"),
file_relative_path(__file__, "presets_csv.yaml"),
],
mode="dev",
),
],
)
def presets_pipeline():
normalize_calories(read_csv())
if __name__ == "__main__":
result = execute_pipeline(presets_pipeline, preset="unittest")
assert result.success
@pipeline(
mode_defs=[
ModeDefinition(
name='development',
resource_defs={
'credentials_vault': credentials_vault,
'postgres_db': postgres_db_info_resource,
},
),
],
preset_defs=[
PresetDefinition.from_files(
'development',
mode='development',
environment_files=[
file_relative_path(__file__,
'environments/dev_resources.yaml'),
file_relative_path(__file__, 'environments/weather.yaml'),
],
),
],
)
def extract_daily_weather_data_pipeline():
insert_weather_report_into_table = insert_into_table.alias(
'insert_weather_report_into_table')
insert_weather_report_into_table(
download_weather_report_from_weather_api())
@pipeline(
mode_defs=[
'field_six_nullable_int_list':
Field([Noneable(int)], is_optional=True),
},
},
)
def a_solid_with_multilayered_config(_):
return None
return a_solid_with_multilayered_config()
@pipeline(preset_defs=[
PresetDefinition.from_files(
name='prod',
environment_files=[
file_relative_path(__file__,
'../environments/csv_hello_world_prod.yaml')
],
),
PresetDefinition.from_files(
name='test',
environment_files=[
file_relative_path(__file__,
'../environments/csv_hello_world_test.yaml')
],
),
PresetDefinition(
name='test_inline',
environment_dict={
'solids': {
'sum_solid': {
@pipeline(
mode_defs=[
ModeDefinition(
"dev",
resource_defs={
"basedosdados_config": basedosdados_config,
"timezone_config": timezone_config,
"discord_webhook": discord_webhook,
"keepalive_key": keepalive_key,
},
),
],
preset_defs=[
PresetDefinition.from_files(
"realized_trips",
config_files=[str(Path(__file__).parent / "realized_trips.yaml")],
mode="dev",
),
],
tags={
"pipeline": "br_rj_riodejaneiro_gtfs_realized_trips",
"dagster-k8s/config": {
"container_config": {
"resources": {
"requests": {"cpu": "250m", "memory": "500Mi"},
"limits": {"cpu": "1500m", "memory": "1Gi"},
},
}
},
},
)
},
},
)
def a_solid_with_multilayered_config(_):
return None
a_solid_with_multilayered_config()
@pipeline(
mode_defs=[default_mode_def_for_test],
preset_defs=[
PresetDefinition.from_files(
name="prod",
config_files=[
file_relative_path(
__file__, "../environments/csv_hello_world_prod.yaml")
],
),
PresetDefinition.from_files(
name="test",
config_files=[
file_relative_path(
__file__, "../environments/csv_hello_world_test.yaml")
],
),
PresetDefinition(
name="test_inline",
run_config={
"solids": {
"sum_solid": {
@pipeline(
mode_defs=[
ModeDefinition(
name='local',
resource_defs={'transporter': local_transporter, 'volume': temporary_directory_mount},
),
ModeDefinition(
name='production',
resource_defs={'transporter': production_transporter, 'volume': mount},
),
],
preset_defs=[
PresetDefinition.from_files(
'dev',
mode='local',
environment_files=[
file_relative_path(__file__, 'environments/bay_bike_pipeline_base.yaml'),
file_relative_path(__file__, 'environments/bay_bike_pipeline_dev.yaml'),
],
),
PresetDefinition.from_files(
'production',
mode='production',
environment_files=[
file_relative_path(__file__, 'environments/bay_bike_pipeline_base.yaml'),
file_relative_path(__file__, 'environments/bay_bike_pipeline_production.yaml'),
],
),
],
)
def extract_monthly_bay_bike_pipeline():
upload_consolidated_csv = upload_file_to_bucket.alias('upload_consolidated_csv')
