def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
env_obj = EnvironmentConfig.build(
pipeline_def,
{
"solids": {
"int_config_solid": {
"config": 1
},
"string_config_solid": {
"config": "bar"
}
},
},
)
value = env_obj.solids
assert set(["int_config_solid",
"string_config_solid"]) == set(value.keys())
assert value == {
"int_config_solid": SolidConfig(1),
"string_config_solid": SolidConfig("bar")
}
def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
env_obj = ResolvedRunConfig.build(
pipeline_def,
{
"solids": {
"int_config_solid": {
"config": 1
},
"string_config_solid": {
"config": "bar"
}
},
},
)
value = env_obj.solids
assert set(["int_config_solid",
"string_config_solid"]) == set(value.keys())
assert value == {
"int_config_solid": SolidConfig.from_dict({"config": 1}),
"string_config_solid": SolidConfig.from_dict({"config": "bar"}),
}
def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
env_obj = EnvironmentConfig.build(
pipeline_def,
{
'solids': {
'int_config_solid': {
'config': 1
},
'string_config_solid': {
'config': 'bar'
}
},
},
)
value = env_obj.solids
assert set(['int_config_solid',
'string_config_solid']) == set(value.keys())
assert value == {
'int_config_solid': SolidConfig(1),
'string_config_solid': SolidConfig('bar')
}
def test_solid_dictionary_some_no_config():
@solid(name='int_config_solid',
config_schema=Int,
input_defs=[],
output_defs=[])
def int_config_solid(_):
return None
@solid(name='no_config_solid', input_defs=[], output_defs=[])
def no_config_solid(_):
return None
@pipeline
def pipeline_def():
int_config_solid()
no_config_solid()
env = EnvironmentConfig.build(
pipeline_def, {'solids': {
'int_config_solid': {
'config': 1
}
}})
assert {'int_config_solid', 'no_config_solid'} == set(env.solids.keys())
assert env.solids == {
'int_config_solid': SolidConfig(1),
'no_config_solid': SolidConfig()
}
def test_solid_dictionary_some_no_config():
@solid(name="int_config_solid",
config_schema=Int,
input_defs=[],
output_defs=[])
def int_config_solid(_):
return None
@solid(name="no_config_solid", input_defs=[], output_defs=[])
def no_config_solid(_):
return None
@pipeline
def pipeline_def():
int_config_solid()
no_config_solid()
env = ResolvedRunConfig.build(
pipeline_def, {"solids": {
"int_config_solid": {
"config": 1
}
}})
assert {"int_config_solid", "no_config_solid"} == set(env.solids.keys())
assert env.solids == {
"int_config_solid": SolidConfig.from_dict({"config": 1}),
"no_config_solid": SolidConfig.from_dict({}),
}
def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
'foobar', pipeline_def.solids, pipeline_def.dependency_structure,
pipeline_def.name).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(
solid_dict_type,
{
'int_config_solid': {
'config': 1
},
'string_config_solid': {
'config': 'bar'
}
},
))
assert set(['int_config_solid',
'string_config_solid']) == set(value.keys())
assert value == {
'int_config_solid': SolidConfig(1),
'string_config_solid': SolidConfig('bar')
}
assert solid_dict_type.type_attributes.is_system_config
for specific_solid_config_field in solid_dict_type.fields.values():
specific_solid_config_type = specific_solid_config_field.config_type
assert specific_solid_config_type.type_attributes.is_system_config
user_config_field = specific_solid_config_field.config_type.fields[
'config']
assert user_config_field.config_type.type_attributes.is_system_config is False
def _composite_descent(parent_stack, solids_config_dict, resource_defs):
"""
The core implementation of composite_descent. This yields a stream of
SolidConfigEntry. This is used by composite_descent to construct a
dictionary.
It descends over the entire solid hierarchy, constructing an entry
for every handle. If it encounters a composite solid instance
with a config mapping, it will invoke that config mapping fn,
producing the config that is necessary to configure the child solids.
This process unrolls recursively as you descend down the tree.
"""
for solid in parent_stack.current_container.solids:
current_stack = parent_stack.descend(solid)
current_handle = current_stack.handle
current_solid_config = solids_config_dict.get(solid.name, {})
# the base case
if isinstance(solid.definition, SolidDefinition):
config_mapped_solid_config = solid.definition.apply_config_mapping(
{"config": current_solid_config.get("config")})
if not config_mapped_solid_config.success:
raise DagsterInvalidConfigError(
"Error in config for solid {}".format(solid.name),
config_mapped_solid_config.errors,
config_mapped_solid_config,
)
complete_config_object = merge_dicts(
current_solid_config, config_mapped_solid_config.value)
yield SolidConfigEntry(
current_handle, SolidConfig.from_dict(complete_config_object))
continue
graph_def = check.inst(solid.definition, GraphDefinition)
yield SolidConfigEntry(
current_handle,
SolidConfig.from_dict({
"inputs":
current_solid_config.get("inputs"),
"outputs":
current_solid_config.get("outputs"),
}),
)
# If there is a config mapping, invoke it and get the descendent solids
# config that way. Else just grabs the solids entry of the current config
solids_dict = (_get_mapped_solids_dict(
solid, graph_def, current_stack, current_solid_config,
resource_defs) if graph_def.config_mapping else
current_solid_config.get("solids", {}))
yield from _composite_descent(current_stack, solids_dict,
resource_defs)
def _composite_descent(parent_stack, solids_config_dict):
'''
The core implementation of composite_descent. This yields a stream of
SolidConfigEntry. This is used by composite_descent to construct a
dictionary.
It descends over the entire solid hierarchy, constructing an entry
for every handle. If it encounters a composite solid instance
with a config mapping, it will invoke that config mapping fn,
producing the config that is necessary to configure the child solids.
This process unrolls recursively as you descend down the tree.
'''
for solid in parent_stack.current_container.solids:
current_stack = parent_stack.descend(solid)
current_handle = current_stack.handle
current_solid_config = solids_config_dict.get(solid.name, {})
# the base case
if not isinstance(solid.definition, CompositeSolidDefinition):
transformed_current_solid_config = solid.definition.apply_config_mapping(
current_solid_config)
if not transformed_current_solid_config.success:
raise DagsterInvalidConfigError(
'Error in config for solid {}'.format(solid.name),
transformed_current_solid_config.errors,
transformed_current_solid_config,
)
yield SolidConfigEntry(
current_handle,
SolidConfig.from_dict(transformed_current_solid_config.value))
continue
composite_def = check.inst(solid.definition, CompositeSolidDefinition)
yield SolidConfigEntry(
current_handle,
SolidConfig.from_dict({
'inputs':
current_solid_config.get('inputs'),
'outputs':
current_solid_config.get('outputs'),
}),
)
# If there is a config mapping, invoke it and get the descendent solids
# config that way. Else just grabs the solids entry of the current config
solids_dict = (_get_mapped_solids_dict(composite_def, current_stack,
current_solid_config)
if composite_def.config_mapping else
current_solid_config.get('solids', {}))
for sce in _composite_descent(current_stack, solids_dict):
yield sce
def test_whole_environment():
pipeline_def = PipelineDefinition(
name="some_pipeline",
mode_defs=[
ModeDefinition(
name="test_mode",
resource_defs={
"test_resource":
ResourceDefinition(resource_fn=lambda _: None,
config_schema=Any)
},
)
],
solid_defs=[
SolidDefinition(
name="int_config_solid",
config_schema=Int,
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(name="no_config_solid",
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None),
],
)
env = EnvironmentConfig.build(
pipeline_def,
{
"resources": {
"test_resource": {
"config": 1
}
},
"solids": {
"int_config_solid": {
"config": 123
}
},
},
)
assert isinstance(env, EnvironmentConfig)
assert env.solids == {
"int_config_solid": SolidConfig.from_dict({"config": 123}),
"no_config_solid": SolidConfig.from_dict({}),
}
assert env.resources == {
"test_resource": {
"config": 1
},
"object_manager": {}
}
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
mode_defs=[
ModeDefinition(
name='test_mode',
resource_defs={
'test_resource':
ResourceDefinition(resource_fn=lambda: None,
config_schema=Any)
},
)
],
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_schema=Int,
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None),
],
)
env = EnvironmentConfig.build(
pipeline_def,
{
'resources': {
'test_resource': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
},
},
)
assert isinstance(env, EnvironmentConfig)
assert env.solids == {
'int_config_solid': SolidConfig(123),
'no_config_solid': SolidConfig()
}
assert env.resources == {'test_resource': {'config': 1}}
def test_solid_dictionary_some_no_config():
pipeline_def = PipelineDefinition(
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(
name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None
),
]
)
solid_dict_type = define_solid_dictionary_cls(
'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name
).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': {'config': 1}})
)
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_some_no_config():
@solid(name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[])
def int_config_solid(_):
return None
@solid(name='no_config_solid', input_defs=[], output_defs=[])
def no_config_solid(_):
return None
@pipeline
def pipeline_def():
int_config_solid()
no_config_solid()
solid_dict_type = define_solid_dictionary_cls(
'foobar', pipeline_def.solids, pipeline_def.dependency_structure,
pipeline_def.name).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type,
{'int_config_solid': {
'config': 1
}}))
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_some_no_config():
pipeline_def = PipelineDefinition(solids=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
transform_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None),
])
solid_dict_type = define_solid_dictionary_cls(
'foobar', create_creation_data(pipeline_def)).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type,
{'int_config_solid': {
'config': 1
}}))
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
mode_definitions=[
ModeDefinition(
name='test_mode',
resources={
'test_resource':
ResourceDefinition(resource_fn=lambda: None,
config_field=Field(Any))
},
)
],
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
compute_fn=lambda *args: None),
],
)
environment_type = create_environment_type(pipeline_def)
assert (environment_type.fields['resources'].config_type.name ==
'SomePipeline.Mode.TestMode.Resources')
solids_type = environment_type.fields['solids'].config_type
assert solids_type.name == 'SomePipeline.SolidsConfigDictionary'
assert (solids_type.fields['int_config_solid'].config_type.name ==
'SomePipeline.SolidConfig.IntConfigSolid')
assert (environment_type.fields['expectations'].config_type.name ==
'SomePipeline.ExpectationsConfig')
env = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(
environment_type,
{
'resources': {
'test_resource': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
},
},
))
assert isinstance(env, EnvironmentConfig)
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.expectations == ExpectationsConfig(evaluate=True)
assert env.resources == {'test_resource': {'config': 1}}
def construct_solid_dictionary(solid_dict_value):
return {
key: SolidConfig(
config=value.get('config'),
inputs=value.get('inputs', {}),
outputs=value.get('outputs', []),
)
for key, value in solid_dict_value.items()
}
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
context_definitions={
'test':
PipelineContextDefinition(
config_field=Field(Any),
context_fn=lambda *args: ExecutionContext())
},
solids=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
transform_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None),
],
)
environment_type = pipeline_def.environment_type
assert environment_type.fields[
'context'].config_type.name == 'SomePipeline.ContextConfig'
solids_type = environment_type.fields['solids'].config_type
assert solids_type.name == 'SomePipeline.SolidsConfigDictionary'
assert (solids_type.fields['int_config_solid'].config_type.name ==
'SomePipeline.SolidConfig.IntConfigSolid')
assert (environment_type.fields['expectations'].config_type.name ==
'SomePipeline.ExpectationsConfig')
env = construct_environment_config(
throwing_evaluate_config_value(
pipeline_def.environment_type,
{
'context': {
'test': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
}
},
))
assert isinstance(env, EnvironmentConfig)
assert env.context == ContextConfig('test', 1, persistence={'file': {}})
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.expectations == ExpectationsConfig(evaluate=True)
def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
pipeline_def.solids, pipeline_def.dependency_structure)
env_obj = EnvironmentConfig.build(
pipeline_def,
{
'solids': {
'int_config_solid': {
'config': 1
},
'string_config_solid': {
'config': 'bar'
}
},
},
)
value = env_obj.solids
assert set(['int_config_solid',
'string_config_solid']) == set(value.keys())
assert value == {
'int_config_solid': SolidConfig(1),
'string_config_solid': SolidConfig('bar')
}
assert solid_dict_type.type_attributes.is_system_config
for specific_solid_config_field in solid_dict_type.fields.values():
specific_solid_config_type = specific_solid_config_field.config_type
assert specific_solid_config_type.type_attributes.is_system_config
user_config_field = specific_solid_config_field.config_type.fields[
'config']
assert user_config_field.config_type.type_attributes.is_system_config is False
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
mode_defs=[
ModeDefinition(
name='test_mode',
resource_defs={
'test_resource': ResourceDefinition(
resource_fn=lambda: None, config_field=Field(Any)
)
},
)
],
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(
name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None
),
],
)
environment_type = create_environment_type(pipeline_def)
env = EnvironmentConfig.from_config_value(
throwing_evaluate_config_value(
environment_type,
{
'resources': {'test_resource': {'config': 1}},
'solids': {'int_config_solid': {'config': 123}},
},
),
{
'resources': {'test_resource': {'config': 1}},
'solids': {'int_config_solid': {'config': 123}},
},
)
assert isinstance(env, EnvironmentConfig)
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.resources == {'test_resource': {'config': 1}}
