def test_two_modes(snapshot):
pipeline_def = PipelineDefinition(
name='TwoModePipelines',
solid_defs=[],
mode_defs=[
ModeDefinition(
'mode_one',
resource_defs={
'value': dummy_resource({'mode_one_field': String})
},
),
ModeDefinition(
'mode_two',
resource_defs={
'value': dummy_resource({'mode_two_field': Int})
},
),
],
)
snapshot.assert_match(
scaffold_pipeline_config(pipeline_def, mode='mode_one'))
snapshot.assert_match(
scaffold_pipeline_config(pipeline_def,
mode='mode_one',
skip_optional=False))
snapshot.assert_match(
scaffold_pipeline_config(pipeline_def, mode='mode_two'))
snapshot.assert_match(
scaffold_pipeline_config(pipeline_def,
mode='mode_two',
skip_optional=False))
def test_two_modes(snapshot):
pipeline_def = PipelineDefinition(
name="TwoModePipelines",
solid_defs=[],
mode_defs=[
ModeDefinition(
"mode_one",
resource_defs={
"value": dummy_resource({"mode_one_field": String})
},
),
ModeDefinition(
"mode_two",
resource_defs={
"value": dummy_resource({"mode_two_field": Int})
},
),
],
)
snapshot.assert_match(
scaffold_pipeline_config(pipeline_def, mode="mode_one"))
snapshot.assert_match(
scaffold_pipeline_config(pipeline_def,
mode="mode_one",
skip_non_required=False))
snapshot.assert_match(
scaffold_pipeline_config(pipeline_def, mode="mode_two"))
snapshot.assert_match(
scaffold_pipeline_config(pipeline_def,
mode="mode_two",
skip_non_required=False))
def test_two_contexts():
pipeline_def = PipelineDefinition(
name='TwoContextsPipeline',
solids=[],
context_definitions={
'context_one': PipelineContextDefinition(
context_fn=lambda *args: fail_me(),
config_field=Field(Dict({'context_one_field': Field(String)})),
),
'context_two': PipelineContextDefinition(
context_fn=lambda *args: fail_me(),
config_field=Field(Dict({'context_two_field': Field(Int)})),
),
},
)
assert scaffold_pipeline_config(pipeline_def) == {'context': {}}
assert scaffold_pipeline_config(pipeline_def, skip_optional=False) == {
'context': {
'context_one': {'config': {'context_one_field': ''}, 'resources': {}},
'context_two': {'config': {'context_two_field': 0}, 'resources': {}},
},
'solids': {},
'expectations': {'evaluate': True},
'execution': {},
}
def test_basic_solids_config(snapshot):
pipeline_def = PipelineDefinition(
name='BasicSolidsConfigPipeline',
solid_defs=[
SolidDefinition(
name='required_field_solid',
input_defs=[],
output_defs=[],
config={'required_int': Int},
compute_fn=lambda *_args: fail_me(),
)
],
)
env_config_type = create_environment_type(pipeline_def)
assert env_config_type.fields['solids'].is_optional is False
solids_config_type = env_config_type.fields['solids'].config_type
assert solids_config_type.fields[
'required_field_solid'].is_optional is False
required_solid_config_type = solids_config_type.fields[
'required_field_solid'].config_type
assert required_solid_config_type.fields['config'].is_optional is False
assert set(
env_config_type.fields['loggers'].config_type.fields.keys()) == set(
['console'])
console_logger_config_type = env_config_type.fields[
'loggers'].config_type.fields['console']
assert set(console_logger_config_type.config_type.fields.keys()) == set(
['config'])
assert console_logger_config_type.config_type.fields['config'].is_optional
console_logger_config_config_type = console_logger_config_type.config_type.fields[
'config'].config_type
assert set(console_logger_config_config_type.fields.keys()) == set(
['log_level', 'name'])
snapshot.assert_match(
scaffold_pipeline_config(pipeline_def, skip_optional=False))
def test_basic_solids_config(snapshot):
pipeline_def = PipelineDefinition(
name="BasicSolidsConfigPipeline",
solid_defs=[
SolidDefinition(
name="required_field_solid",
input_defs=[],
output_defs=[],
config_schema={"required_int": Int},
compute_fn=lambda *_args: fail_me(),
)
],
)
env_config_type = create_environment_type(pipeline_def)
assert env_config_type.fields["solids"].is_required
solids_config_type = env_config_type.fields["solids"].config_type
assert solids_config_type.fields["required_field_solid"].is_required
required_solid_config_type = solids_config_type.fields[
"required_field_solid"].config_type
assert required_solid_config_type.fields["config"].is_required
assert set(
env_config_type.fields["loggers"].config_type.fields.keys()) == set(
["console"])
console_logger_config_type = env_config_type.fields[
"loggers"].config_type.fields["console"]
assert set(console_logger_config_type.config_type.fields.keys()) == set(
["config"])
assert console_logger_config_type.config_type.fields[
"config"].is_required is False
console_logger_config_config_type = console_logger_config_type.config_type.fields[
"config"].config_type
assert set(console_logger_config_config_type.fields.keys()) == set(
["log_level", "name"])
snapshot.assert_match(
scaffold_pipeline_config(pipeline_def, skip_non_required=False))
def test_basic_solids_config():
pipeline_def = PipelineDefinition(
name='BasicSolidsConfigPipeline',
solids=[
SolidDefinition(
name='required_field_solid',
inputs=[],
outputs=[],
config_field=Field(Dict(fields={'required_int': Field(Int)})),
transform_fn=lambda *_args: fail_me(),
)
],
)
env_config_type = pipeline_def.environment_type
assert env_config_type.fields['solids'].is_optional is False
solids_config_type = env_config_type.fields['solids'].config_type
assert solids_config_type.fields['required_field_solid'].is_optional is False
required_solid_config_type = solids_config_type.fields['required_field_solid'].config_type
assert required_solid_config_type.fields['config'].is_optional is False
context_config_type = env_config_type.fields['context'].config_type
assert 'default' in context_config_type.fields
assert context_config_type.fields['default'].is_optional
default_context_config_type = context_config_type.fields['default'].config_type
assert set(default_context_config_type.fields.keys()) == set(['config', 'resources'])
default_context_user_config_type = default_context_config_type.fields['config'].config_type
assert set(default_context_user_config_type.fields.keys()) == set(['log_level'])
assert scaffold_pipeline_config(pipeline_def, skip_optional=False) == {
'context': {'default': {'config': {'log_level': ''}, 'resources': {}}},
'solids': {'required_field_solid': {'config': {'required_int': 0}}},
'expectations': {'evaluate': True},
'execution': {},
}
def test_basic_solids_config():
pipeline_def = PipelineDefinition(
name='BasicSolidsConfigPipeline',
solids=[
SolidDefinition(
name='required_field_solid',
inputs=[],
outputs=[],
config_field=Field(Dict(fields={'required_int': Field(Int)})),
compute_fn=lambda *_args: fail_me(),
)
],
)
env_config_type = create_environment_type(pipeline_def)
assert env_config_type.fields['solids'].is_optional is False
solids_config_type = env_config_type.fields['solids'].config_type
assert solids_config_type.fields[
'required_field_solid'].is_optional is False
required_solid_config_type = solids_config_type.fields[
'required_field_solid'].config_type
assert required_solid_config_type.fields['config'].is_optional is False
assert set(
env_config_type.fields['loggers'].config_type.fields.keys()) == set(
['console'])
console_logger_config_type = env_config_type.fields[
'loggers'].config_type.fields['console']
assert set(console_logger_config_type.config_type.fields.keys()) == set(
['config'])
assert console_logger_config_type.config_type.fields['config'].is_optional
console_logger_config_config_type = console_logger_config_type.config_type.fields[
'config'].config_type
assert set(console_logger_config_config_type.fields.keys()) == set(
['log_level', 'name'])
assert scaffold_pipeline_config(pipeline_def, skip_optional=False) == {
'loggers': {
'console': {
'config': {
'log_level': '',
'name': ''
}
}
},
'solids': {
'required_field_solid': {
'config': {
'required_int': 0
}
}
},
'expectations': {
'evaluate': True
},
'execution': {},
'resources': {},
'storage': {
'filesystem': {
'base_dir': ''
},
'in_memory': {},
's3': {
's3_bucket': ''
}
},
}
def test_two_modes():
pipeline_def = PipelineDefinition(
name='TwoModePipelines',
solids=[],
mode_definitions=[
ModeDefinition(
'mode_one',
resources={
'value':
dummy_resource(
Field(Dict({'mode_one_field': Field(String)})))
},
),
ModeDefinition(
'mode_two',
resources={
'value':
dummy_resource(Field(Dict({'mode_two_field': Field(Int)})))
},
),
],
)
assert scaffold_pipeline_config(pipeline_def, mode='mode_one') == {
'resources': {
'value': {
'config': {
'mode_one_field': ''
}
}
}
}
assert scaffold_pipeline_config(pipeline_def,
mode='mode_one',
skip_optional=False) == {
'loggers': {
'console': {
'config': {
'log_level': '',
'name': ''
}
}
},
'solids': {},
'expectations': {
'evaluate': True
},
'storage': {
'in_memory': {},
'filesystem': {
'base_dir': ''
},
's3': {
's3_bucket': ''
}
},
'execution': {},
'resources': {
'value': {
'config': {
'mode_one_field': ''
}
}
},
}
assert scaffold_pipeline_config(pipeline_def, mode='mode_two') == {
'resources': {
'value': {
'config': {
'mode_two_field': 0
}
}
}
}
assert scaffold_pipeline_config(pipeline_def,
mode='mode_two',
skip_optional=False) == {
'solids': {},
'expectations': {
'evaluate': True
},
'storage': {
'in_memory': {},
'filesystem': {
'base_dir': ''
},
's3': {
's3_bucket': ''
}
},
'execution': {},
'resources': {
'value': {
'config': {
'mode_two_field': 0
}
}
},
'loggers': {
'console': {
'config': {
'log_level': '',
'name': ''
}
}
},
}
