def test_solid_dictionary_some_no_config():
@solid(name='int_config_solid', config=Int, input_defs=[], output_defs=[])
def int_config_solid(_):
return None
@solid(name='no_config_solid', input_defs=[], output_defs=[])
def no_config_solid(_):
return None
@pipeline
def pipeline_def():
int_config_solid()
no_config_solid()
env = EnvironmentConfig.build(
pipeline_def, {'solids': {
'int_config_solid': {
'config': 1
}
}})
assert {'int_config_solid', 'no_config_solid'} == set(env.solids.keys())
assert env.solids == {
'int_config_solid': SolidConfig(1),
'no_config_solid': SolidConfig()
}
def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
env_obj = EnvironmentConfig.build(
pipeline_def,
{
'solids': {
'int_config_solid': {
'config': 1
},
'string_config_solid': {
'config': 'bar'
}
},
},
)
value = env_obj.solids
assert set(['int_config_solid',
'string_config_solid']) == set(value.keys())
assert value == {
'int_config_solid': SolidConfig(1),
'string_config_solid': SolidConfig('bar')
}
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
mode_defs=[
ModeDefinition(
name='test_mode',
resource_defs={
'test_resource':
ResourceDefinition(resource_fn=lambda: None,
config_schema=Any)
},
)
],
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_schema=Int,
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None),
],
)
env = EnvironmentConfig.build(
pipeline_def,
{
'resources': {
'test_resource': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
},
},
)
assert isinstance(env, EnvironmentConfig)
assert env.solids == {
'int_config_solid': SolidConfig(123),
'no_config_solid': SolidConfig()
}
assert env.resources == {'test_resource': {'config': 1}}
def test_whole_environment():
pipeline_def = PipelineDefinition(
name="some_pipeline",
mode_defs=[
ModeDefinition(
name="test_mode",
resource_defs={
"test_resource":
ResourceDefinition(resource_fn=lambda _: None,
config_schema=Any)
},
)
],
solid_defs=[
SolidDefinition(
name="int_config_solid",
config_schema=Int,
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(name="no_config_solid",
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None),
],
)
env = EnvironmentConfig.build(
pipeline_def,
{
"resources": {
"test_resource": {
"config": 1
}
},
"solids": {
"int_config_solid": {
"config": 123
}
},
},
)
assert isinstance(env, EnvironmentConfig)
assert env.solids == {
"int_config_solid": SolidConfig(123),
"no_config_solid": SolidConfig()
}
assert env.resources == {"test_resource": {"config": 1}}
def test_solid_dictionary_some_no_config():
pipeline_def = PipelineDefinition(
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(
name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None
),
]
)
solid_dict_type = define_solid_dictionary_cls(
'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name
).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': {'config': 1}})
)
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
mode_definitions=[
ModeDefinition(
name='test_mode',
resources={
'test_resource':
ResourceDefinition(resource_fn=lambda: None,
config_field=Field(Any))
},
)
],
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
compute_fn=lambda *args: None),
],
)
environment_type = create_environment_type(pipeline_def)
assert (environment_type.fields['resources'].config_type.name ==
'SomePipeline.Mode.TestMode.Resources')
solids_type = environment_type.fields['solids'].config_type
assert solids_type.name == 'SomePipeline.SolidsConfigDictionary'
assert (solids_type.fields['int_config_solid'].config_type.name ==
'SomePipeline.SolidConfig.IntConfigSolid')
assert (environment_type.fields['expectations'].config_type.name ==
'SomePipeline.ExpectationsConfig')
env = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(
environment_type,
{
'resources': {
'test_resource': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
},
},
))
assert isinstance(env, EnvironmentConfig)
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.expectations == ExpectationsConfig(evaluate=True)
assert env.resources == {'test_resource': {'config': 1}}
def test_solid_dictionary_some_no_config():
pipeline_def = PipelineDefinition(solids=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
transform_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None),
])
solid_dict_type = define_solid_dictionary_cls(
'foobar', create_creation_data(pipeline_def)).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type,
{'int_config_solid': {
'config': 1
}}))
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_some_no_config():
@solid(name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[])
def int_config_solid(_):
return None
@solid(name='no_config_solid', input_defs=[], output_defs=[])
def no_config_solid(_):
return None
@pipeline
def pipeline_def():
int_config_solid()
no_config_solid()
solid_dict_type = define_solid_dictionary_cls(
'foobar', pipeline_def.solids, pipeline_def.dependency_structure,
pipeline_def.name).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type,
{'int_config_solid': {
'config': 1
}}))
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def construct_solid_dictionary(solid_dict_value):
return {
key: SolidConfig(
config=value.get('config'),
inputs=value.get('inputs', {}),
outputs=value.get('outputs', []),
)
for key, value in solid_dict_value.items()
}
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
context_definitions={
'test':
PipelineContextDefinition(
config_field=Field(Any),
context_fn=lambda *args: ExecutionContext())
},
solids=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
transform_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None),
],
)
environment_type = pipeline_def.environment_type
assert environment_type.fields[
'context'].config_type.name == 'SomePipeline.ContextConfig'
solids_type = environment_type.fields['solids'].config_type
assert solids_type.name == 'SomePipeline.SolidsConfigDictionary'
assert (solids_type.fields['int_config_solid'].config_type.name ==
'SomePipeline.SolidConfig.IntConfigSolid')
assert (environment_type.fields['expectations'].config_type.name ==
'SomePipeline.ExpectationsConfig')
env = construct_environment_config(
throwing_evaluate_config_value(
pipeline_def.environment_type,
{
'context': {
'test': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
}
},
))
assert isinstance(env, EnvironmentConfig)
assert env.context == ContextConfig('test', 1, persistence={'file': {}})
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.expectations == ExpectationsConfig(evaluate=True)
def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls('foobar', pipeline_def).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(
solid_dict_type,
{'int_config_solid': {'config': 1}, 'string_config_solid': {'config': 'bar'}},
)
)
assert set(['int_config_solid', 'string_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1), 'string_config_solid': SolidConfig('bar')}
assert solid_dict_type.type_attributes.is_system_config
for specific_solid_config_field in solid_dict_type.fields.values():
specific_solid_config_type = specific_solid_config_field.config_type
assert specific_solid_config_type.type_attributes.is_system_config
user_config_field = specific_solid_config_field.config_type.fields['config']
assert user_config_field.config_type.type_attributes.is_system_config is False
def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
pipeline_def.solids, pipeline_def.dependency_structure)
env_obj = EnvironmentConfig.build(
pipeline_def,
{
'solids': {
'int_config_solid': {
'config': 1
},
'string_config_solid': {
'config': 'bar'
}
},
},
)
value = env_obj.solids
assert set(['int_config_solid',
'string_config_solid']) == set(value.keys())
assert value == {
'int_config_solid': SolidConfig(1),
'string_config_solid': SolidConfig('bar')
}
assert solid_dict_type.type_attributes.is_system_config
for specific_solid_config_field in solid_dict_type.fields.values():
specific_solid_config_type = specific_solid_config_field.config_type
assert specific_solid_config_type.type_attributes.is_system_config
user_config_field = specific_solid_config_field.config_type.fields[
'config']
assert user_config_field.config_type.type_attributes.is_system_config is False
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
mode_defs=[
ModeDefinition(
name='test_mode',
resource_defs={
'test_resource': ResourceDefinition(
resource_fn=lambda: None, config_field=Field(Any)
)
},
)
],
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(
name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None
),
],
)
environment_type = create_environment_type(pipeline_def)
env = EnvironmentConfig.from_config_value(
throwing_evaluate_config_value(
environment_type,
{
'resources': {'test_resource': {'config': 1}},
'solids': {'int_config_solid': {'config': 123}},
},
),
{
'resources': {'test_resource': {'config': 1}},
'solids': {'int_config_solid': {'config': 123}},
},
)
assert isinstance(env, EnvironmentConfig)
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.resources == {'test_resource': {'config': 1}}
