def test_optional_solid_with_optional_subfield(): pipeline_def = PipelineDefinition( name="some_pipeline", solid_defs=[ SolidDefinition( name="int_config_solid", config_schema=Field( {"optional_field": Field(String, is_required=False)}, is_required=False), input_defs=[], output_defs=[], compute_fn=lambda *_args: None, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields["solids"].is_required is False assert env_type.fields["execution"].is_required is False
def test_solid_configs_defaults(): env_type = create_environment_type(define_test_solids_config_pipeline()) solids_field = env_type.fields['solids'] assert_has_fields(solids_field.config_type, 'int_config_solid', 'string_config_solid') int_solid_field = solids_field.config_type.fields['int_config_solid'] assert int_solid_field.is_optional # TODO: this is the test case the exposes the default dodginess assert int_solid_field.default_provided assert_has_fields(int_solid_field.config_type, 'config') int_solid_config_field = int_solid_field.config_type.fields['config'] assert int_solid_config_field.is_optional assert not int_solid_config_field.default_provided
def test_optional_solid_with_optional_subfield(): pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[ SolidDefinition( name='int_config_solid', config=Field( {'optional_field': Field(String, is_optional=True)}, is_optional=True), input_defs=[], output_defs=[], compute_fn=lambda *_args: None, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_optional assert env_type.fields['execution'].is_optional
def test_provided_default_on_resources_config(): @solid(name='some_solid', input_defs=[], output_defs=[]) def some_solid(_): return None @pipeline( mode_defs=[ ModeDefinition( name='some_mode', resource_defs={ 'some_resource': ResourceDefinition( resource_fn=lambda: None, config_field=Field( Dict( { 'with_default_int': Field( Int, is_optional=True, default_value=23434 ) } ) ), ) }, ) ] ) def pipeline_def(): some_solid() env_type = create_environment_type(pipeline_def) assert env_type.type_attributes.is_system_config some_resource_field = env_type.fields['resources'].config_type.fields['some_resource'] assert some_resource_field.is_optional some_resource_config_field = some_resource_field.config_type.fields['config'] assert some_resource_config_field.is_optional assert some_resource_config_field.default_value == {'with_default_int': 23434} assert some_resource_field.default_value == {'config': {'with_default_int': 23434}} value = EnvironmentConfig.from_config_value(throwing_evaluate_config_value(env_type, {}), {}) assert value.resources == {'some_resource': {'config': {'with_default_int': 23434}}}
def test_solid_configs_defaults(): env_type = create_environment_type(define_test_solids_config_pipeline()) solids_field = env_type.fields["solids"] assert_has_fields(solids_field.config_type, "int_config_solid", "string_config_solid") int_solid_field = solids_field.config_type.fields["int_config_solid"] assert int_solid_field.is_required is False # TODO: this is the test case the exposes the default dodginess # https://github.com/dagster-io/dagster/issues/1990 assert int_solid_field.default_provided assert_has_fields(int_solid_field.config_type, "config") int_solid_config_field = int_solid_field.config_type.fields["config"] assert int_solid_config_field.is_required is False assert not int_solid_config_field.default_provided
def scaffold_pipeline_config(pipeline_def, skip_optional=True, mode=None): check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) check.bool_param(skip_optional, 'skip_optional') env_config_type = create_environment_type(pipeline_def, mode=mode) env_dict = {} for env_field_name, env_field in env_config_type.fields.items(): if skip_optional and env_field.is_optional: continue # unfortunately we have to treat this special for now if env_field_name == 'context': if skip_optional and env_config_type.fields['context'].is_optional: continue env_dict[env_field_name] = scaffold_type(env_field.config_type, skip_optional) return env_dict
def create_environment_config(pipeline, environment_dict=None, run_config=None): check.inst_param(pipeline, 'pipeline', PipelineDefinition) check.opt_dict_param(environment_dict, 'environment') run_config = check.opt_inst_param(run_config, 'run_config', RunConfig, default=RunConfig()) mode = run_config.mode or pipeline.get_default_mode_name() environment_type = create_environment_type(pipeline, mode) result = evaluate_config(environment_type, environment_dict, pipeline, run_config) if not result.success: raise DagsterInvalidConfigError(pipeline, result.errors, environment_dict) return EnvironmentConfig.from_dict(result.value)
def test_optional_solid_with_required_scalar_config(): def _assert_config_none(context, value): assert context.solid_config is value pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[ SolidDefinition( name='int_config_solid', config_schema=Int, input_defs=[], output_defs=[], compute_fn=lambda context, _inputs: _assert_config_none( context, 234), ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_required is True solids_type = env_type.fields['solids'].config_type assert solids_type.fields['int_config_solid'].is_required is True int_config_solid_type = solids_type.fields['int_config_solid'].config_type assert_has_fields(int_config_solid_type, 'config') int_config_solid_config_field = int_config_solid_type.fields['config'] assert int_config_solid_config_field.is_required is True execute_pipeline(pipeline_def, {'solids': { 'int_config_solid': { 'config': 234 } }})
def test_provided_default_on_resources_config(): @solid( name="some_solid", input_defs=[], output_defs=[], required_resource_keys={"some_resource"} ) def some_solid(_): return None @pipeline( mode_defs=[ ModeDefinition( name="some_mode", resource_defs={ "some_resource": ResourceDefinition( resource_fn=lambda _: None, config_schema={ "with_default_int": Field(Int, is_required=False, default_value=23434) }, ) }, ) ] ) def pipeline_def(): some_solid() env_type = create_environment_type(pipeline_def) some_resource_field = env_type.fields["resources"].config_type.fields["some_resource"] assert some_resource_field.is_required is False some_resource_config_field = some_resource_field.config_type.fields["config"] assert some_resource_config_field.is_required is False assert some_resource_config_field.default_value == {"with_default_int": 23434} assert some_resource_field.default_value == {"config": {"with_default_int": 23434}} value = EnvironmentConfig.build(pipeline_def, {}) assert value.resources == { "some_resource": ResourceConfig({"with_default_int": 23434}), "io_manager": ResourceConfig(None), }
def test_required_solid_with_required_subfield(): pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[ SolidDefinition( name='int_config_solid', config_field=Field(Dict({'required_field': Field(String)})), input_defs=[], output_defs=[], compute_fn=lambda *_args: None, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_optional is False assert env_type.fields['solids'].config_type solids_type = env_type.fields['solids'].config_type assert solids_type.fields['int_config_solid'].is_optional is False int_config_solid_type = solids_type.fields['int_config_solid'].config_type assert int_config_solid_type.fields['config'].is_optional is False assert env_type.fields['execution'].is_optional assert env_type.fields['expectations'].is_optional env_obj = EnvironmentConfig.from_dict( throwing_evaluate_config_value( env_type, {'solids': {'int_config_solid': {'config': {'required_field': 'foobar'}}}} ) ) assert env_obj.solids['int_config_solid'].config['required_field'] == 'foobar' with pytest.raises(DagsterEvaluateConfigValueError): throwing_evaluate_config_value(env_type, {'solids': {}}) with pytest.raises(DagsterEvaluateConfigValueError): throwing_evaluate_config_value(env_type, {})
def scaffold_pipeline_config(pipeline_def, skip_non_required=True, mode=None): check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition) check.bool_param(skip_non_required, "skip_non_required") env_config_type = create_environment_type(pipeline_def, mode=mode) env_dict = {} for env_field_name, env_field in env_config_type.fields.items(): if skip_non_required and not env_field.is_required: continue # unfortunately we have to treat this special for now if env_field_name == "context": if skip_non_required and not env_config_type.fields[ "context"].is_required: continue env_dict[env_field_name] = scaffold_type(env_field.config_type, skip_non_required) return env_dict
def test_required_solid_with_required_subfield(): pipeline_def = PipelineDefinition( name="some_pipeline", solid_defs=[ SolidDefinition( name="int_config_solid", config_schema={"required_field": String}, input_defs=[], output_defs=[], compute_fn=lambda *_args: None, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields["solids"].is_required is True assert env_type.fields["solids"].config_type solids_type = env_type.fields["solids"].config_type assert solids_type.fields["int_config_solid"].is_required is True int_config_solid_type = solids_type.fields["int_config_solid"].config_type assert int_config_solid_type.fields["config"].is_required is True assert env_type.fields["execution"].is_required is False env_obj = EnvironmentConfig.build( pipeline_def, {"solids": {"int_config_solid": {"config": {"required_field": "foobar"}}}}, ) assert env_obj.solids["int_config_solid"].config["required_field"] == "foobar" res = process_config(env_type, {"solids": {}}) assert not res.success res = process_config(env_type, {}) assert not res.success
def test_required_solid_with_required_subfield(): pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[ SolidDefinition( name='int_config_solid', config_schema={'required_field': String}, input_defs=[], output_defs=[], compute_fn=lambda *_args: None, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_required is True assert env_type.fields['solids'].config_type solids_type = env_type.fields['solids'].config_type assert solids_type.fields['int_config_solid'].is_required is True int_config_solid_type = solids_type.fields['int_config_solid'].config_type assert int_config_solid_type.fields['config'].is_required is True assert env_type.fields['execution'].is_required is False env_obj = EnvironmentConfig.build( pipeline_def, {'solids': {'int_config_solid': {'config': {'required_field': 'foobar'}}}}, ) assert env_obj.solids['int_config_solid'].config['required_field'] == 'foobar' res = process_config(env_type, {'solids': {}}) assert not res.success res = process_config(env_type, {}) assert not res.success
def test_required_solid_with_required_subfield(): pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[ SolidDefinition( name='int_config_solid', config={'required_field': String}, input_defs=[], output_defs=[], compute_fn=lambda *_args: None, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_required is True assert env_type.fields['solids'].config_type solids_type = env_type.fields['solids'].config_type assert solids_type.fields['int_config_solid'].is_required is True int_config_solid_type = solids_type.fields['int_config_solid'].config_type assert int_config_solid_type.fields['config'].is_required is True assert env_type.fields['execution'].is_required is False env_obj = EnvironmentConfig.build( pipeline_def, {'solids': {'int_config_solid': {'config': {'required_field': 'foobar'}}}}, ) assert env_obj.solids['int_config_solid'].config['required_field'] == 'foobar' with pytest.raises(DagsterEvaluateConfigValueError): throwing_validate_config_value(env_type, {'solids': {}}) with pytest.raises(DagsterEvaluateConfigValueError): throwing_validate_config_value(env_type, {})
def test_required_resource_with_required_subfield(): pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[], mode_defs=[ ModeDefinition( resource_defs={ 'with_required': ResourceDefinition( resource_fn=lambda _: None, config_schema={'required_field': String}, ) } ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_required is False assert env_type.fields['execution'].is_required is False assert env_type.fields['resources'].is_required assert nested_field(env_type, 'resources', 'with_required').is_required assert nested_field(env_type, 'resources', 'with_required', 'config').is_required assert nested_field( env_type, 'resources', 'with_required', 'config', 'required_field' ).is_required
def test_bad_config(): configs_and_expected_errors = [ ( # Create disposition must match enum values { 'create_disposition': 'this is not a valid create disposition' }, 'Value not in enum type BQCreateDisposition', ), ( # Dataset must be of form project_name.dataset_name { 'default_dataset': 'this is not a valid dataset' }, 'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "Dataset"', ), ( # Table must be of form project_name.dataset_name.table_name { 'destination': 'this is not a valid table' }, 'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "Table"', ), ( # Priority must match enum values { 'priority': 'this is not a valid priority' }, 'Value not in enum type BQPriority', ), ( # Schema update options must be a list { 'schema_update_options': 'this is not valid schema update options' }, 'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]', ), ( { 'schema_update_options': ['this is not valid schema update options'] }, 'Value not in enum type BQSchemaUpdateOption', ), ( { 'write_disposition': 'this is not a valid write disposition' }, 'Value not in enum type BQWriteDisposition', ), ] pipeline_def = PipelineDefinition( name='test_config_pipeline', solids=[BigQuerySolidDefinition('test', ['SELECT 1'])], mode_definitions=bq_modes(), ) env_type = create_environment_type(pipeline_def) for config_fragment, error_message in configs_and_expected_errors: config = { 'solids': { 'test': { 'config': { 'query_job_config': config_fragment } } } } result = evaluate_config_value(env_type, config) assert result.errors[0].message == error_message
def test_bad_config(): configs_and_expected_errors = [ ( # Create disposition must match enum values { "create_disposition": "this is not a valid create disposition" }, "Value at path root:solids:test:config:query_job_config:create_disposition not in enum type BQCreateDisposition", ), ( # Priority must match enum values { "priority": "this is not a valid priority" }, "Value at path root:solids:test:config:query_job_config:priority not in enum type BQPriority got this is not a valid priority", ), ( # Schema update options must be a list { "schema_update_options": "this is not valid schema update options" }, 'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: "[BQSchemaUpdateOption]"', ), ( { "schema_update_options": ["this is not valid schema update options"] }, "Value at path root:solids:test:config:query_job_config:schema_update_options[0] not in enum type BQSchemaUpdateOption", ), ( { "write_disposition": "this is not a valid write disposition" }, "Value at path root:solids:test:config:query_job_config:write_disposition not in enum type BQWriteDisposition", ), ] @pipeline(mode_defs=bq_modes()) def test_config_pipeline(): bq_solid_for_queries(["SELECT 1"]).alias("test")() env_type = create_environment_type(test_config_pipeline) for config_fragment, error_message in configs_and_expected_errors: config = { "solids": { "test": { "config": { "query_job_config": config_fragment } } } } result = validate_config(env_type, config) assert error_message in result.errors[0].message configs_and_expected_validation_errors = [ ( { "default_dataset": "this is not a valid dataset" }, "Datasets must be of the form", # project_name.dataset_name', ), ( { "destination": "this is not a valid table" }, "Tables must be of the form", # project_name.dataset_name.table_name' ), ] for config_fragment, error_message in configs_and_expected_validation_errors: config = { "solids": { "test": { "config": { "query_job_config": config_fragment } } } } result = process_config(env_type, config) assert error_message in result.errors[0].message
def test_bad_config(): configs_and_expected_errors = [ ( # Create disposition must match enum values { 'create_disposition': 'this is not a valid create disposition' }, 'Value not in enum type BQCreateDisposition', ), ( # Dataset must be of form project_name.dataset_name { 'default_dataset': 'this is not a valid dataset' }, 'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "Dataset"', ), ( # Table must be of form project_name.dataset_name.table_name { 'destination': 'this is not a valid table' }, 'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "Table"', ), ( # Priority must match enum values { 'priority': 'this is not a valid priority' }, 'Value not in enum type BQPriority', ), ( # Schema update options must be a list { 'schema_update_options': 'this is not valid schema update options' }, 'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]', ), ( { 'schema_update_options': ['this is not valid schema update options'] }, 'Value not in enum type BQSchemaUpdateOption', ), ( { 'write_disposition': 'this is not a valid write disposition' }, 'Value not in enum type BQWriteDisposition', ), ] @pipeline(mode_defs=bq_modes()) def test_config_pipeline(): bq_solid_for_queries(['SELECT 1']).alias('test')() # pylint: disable=no-value-for-parameter env_type = create_environment_type(test_config_pipeline) for config_fragment, error_message in configs_and_expected_errors: config = { 'solids': { 'test': { 'config': { 'query_job_config': config_fragment } } } } result = evaluate_config(env_type, config, test_config_pipeline) assert result.errors[0].message == error_message
def test_optional_and_required_context(): pipeline_def = PipelineDefinition( name="some_pipeline", solid_defs=[], mode_defs=[ ModeDefinition( name="mixed", resource_defs={ "optional_resource": ResourceDefinition( lambda _: None, config_schema={ "optional_field": Field(String, is_required=False) }, ), "required_resource": ResourceDefinition( lambda _: None, config_schema={"required_field": String}, ), }, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields["solids"].is_required is False assert env_type.fields["execution"].is_required is False assert nested_field(env_type, "resources").is_required assert nested_field(env_type, "resources", "optional_resource").is_required is False assert nested_field(env_type, "resources", "optional_resource", "config").is_required is False assert (nested_field(env_type, "resources", "optional_resource", "config", "optional_field").is_required is False) assert nested_field(env_type, "resources", "required_resource").is_required assert nested_field(env_type, "resources", "required_resource", "config").is_required assert nested_field(env_type, "resources", "required_resource", "config", "required_field").is_required env_obj = EnvironmentConfig.build( pipeline_def, { "resources": { "required_resource": { "config": { "required_field": "foo" } } } }, ) assert env_obj.resources == { "optional_resource": { "config": {} }, "required_resource": { "config": { "required_field": "foo" } }, "object_manager": {}, }
def test_optional_and_required_context(): pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[], mode_definitions=[ ModeDefinition( name='mixed', resources={ 'optional_resource': ResourceDefinition( lambda: None, config_field=Field(dagster_type=Dict( fields={ 'optional_field': Field(String, is_optional=True) })), ), 'required_resource': ResourceDefinition( lambda: None, config_field=Field(dagster_type=Dict( fields={'required_field': Field(String)})), ), }, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_optional assert env_type.fields['execution'].is_optional assert env_type.fields['expectations'].is_optional assert nested_field(env_type, 'resources').is_required assert nested_field(env_type, 'resources', 'optional_resource').is_optional assert nested_field(env_type, 'resources', 'optional_resource', 'config').is_optional assert nested_field(env_type, 'resources', 'optional_resource', 'config', 'optional_field').is_optional assert nested_field(env_type, 'resources', 'required_resource').is_required assert nested_field(env_type, 'resources', 'required_resource', 'config').is_required assert nested_field(env_type, 'resources', 'required_resource', 'config', 'required_field').is_required env_obj = EnvironmentConfig.from_dict( throwing_evaluate_config_value( env_type, { 'resources': { 'required_resource': { 'config': { 'required_field': 'foo' } } } })) assert env_obj.resources == { 'optional_resource': { 'config': {} }, 'required_resource': { 'config': { 'required_field': 'foo' } }, }
def test_whole_environment(): pipeline_def = PipelineDefinition( name='some_pipeline', mode_defs=[ ModeDefinition( name='test_mode', resource_defs={ 'test_resource': ResourceDefinition(resource_fn=lambda: None, config_field=Field(Any)) }, ) ], solid_defs=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), input_defs=[], output_defs=[], compute_fn=lambda *args: None, ), SolidDefinition(name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None), ], ) environment_type = create_environment_type(pipeline_def) assert (environment_type.fields['resources'].config_type.name == 'SomePipeline.Mode.TestMode.Resources') solids_type = environment_type.fields['solids'].config_type assert solids_type.name == 'SomePipeline.SolidsConfigDictionary' assert (solids_type.fields['int_config_solid'].config_type.name == 'SomePipeline.SolidConfig.IntConfigSolid') env = EnvironmentConfig.from_config_value( throwing_evaluate_config_value( environment_type, { 'resources': { 'test_resource': { 'config': 1 } }, 'solids': { 'int_config_solid': { 'config': 123 } }, }, ), { 'resources': { 'test_resource': { 'config': 1 } }, 'solids': { 'int_config_solid': { 'config': 123 } }, }, ) assert isinstance(env, EnvironmentConfig) assert env.solids == {'int_config_solid': SolidConfig(123)} assert env.resources == {'test_resource': {'config': 1}}
def test_optional_and_required_context(): pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[], mode_defs=[ ModeDefinition( name='mixed', resource_defs={ 'optional_resource': ResourceDefinition( lambda: None, config_schema={ 'optional_field': Field(String, is_required=False) }, ), 'required_resource': ResourceDefinition( lambda: None, config_schema={'required_field': String}, ), }, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_required is False assert env_type.fields['execution'].is_required is False assert nested_field(env_type, 'resources').is_required assert nested_field(env_type, 'resources', 'optional_resource').is_required is False assert nested_field(env_type, 'resources', 'optional_resource', 'config').is_required is False assert (nested_field(env_type, 'resources', 'optional_resource', 'config', 'optional_field').is_required is False) assert nested_field(env_type, 'resources', 'required_resource').is_required assert nested_field(env_type, 'resources', 'required_resource', 'config').is_required assert nested_field(env_type, 'resources', 'required_resource', 'config', 'required_field').is_required env_obj = EnvironmentConfig.build( pipeline_def, { 'resources': { 'required_resource': { 'config': { 'required_field': 'foo' } } } }, ) assert env_obj.resources == { 'optional_resource': { 'config': {} }, 'required_resource': { 'config': { 'required_field': 'foo' } }, }
def test_bad_config(): configs_and_expected_errors = [ ( # Create disposition must match enum values { 'create_disposition': 'this is not a valid create disposition' }, 'Value at path root:solids:test:config:query_job_config:create_disposition not in enum type BQCreateDisposition', ), ( # Priority must match enum values { 'priority': 'this is not a valid priority' }, 'Value at path root:solids:test:config:query_job_config:priority not in enum type BQPriority got this is not a valid priority', ), ( # Schema update options must be a list { 'schema_update_options': 'this is not valid schema update options' }, 'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]', ), ( { 'schema_update_options': ['this is not valid schema update options'] }, 'Value at path root:solids:test:config:query_job_config:schema_update_options[0] not in enum type BQSchemaUpdateOption', ), ( { 'write_disposition': 'this is not a valid write disposition' }, 'Value at path root:solids:test:config:query_job_config:write_disposition not in enum type BQWriteDisposition', ), ] @pipeline(mode_defs=bq_modes()) def test_config_pipeline(): bq_solid_for_queries(['SELECT 1']).alias('test')() env_type = create_environment_type(test_config_pipeline) for config_fragment, error_message in configs_and_expected_errors: config = { 'solids': { 'test': { 'config': { 'query_job_config': config_fragment } } } } result = validate_config(env_type, config) assert error_message in result.errors[0].message configs_and_expected_validation_errors = [ ( { 'default_dataset': 'this is not a valid dataset' }, 'Datasets must be of the form', # project_name.dataset_name', ), ( { 'destination': 'this is not a valid table' }, 'Tables must be of the form', # project_name.dataset_name.table_name' ), ] for config_fragment, error_message in configs_and_expected_validation_errors: config = { 'solids': { 'test': { 'config': { 'query_job_config': config_fragment } } } } result = process_config(env_type, config) assert error_message in result.errors[0].message
def resolve_environment_type(self, _graphene_info, mode=None): return to_dauphin_config_type( create_environment_type(self._pipeline, mode))
def test_basic_solids_config(): pipeline_def = PipelineDefinition( name='BasicSolidsConfigPipeline', solids=[ SolidDefinition( name='required_field_solid', inputs=[], outputs=[], config_field=Field(Dict(fields={'required_int': Field(Int)})), compute_fn=lambda *_args: fail_me(), ) ], ) env_config_type = create_environment_type(pipeline_def) assert env_config_type.fields['solids'].is_optional is False solids_config_type = env_config_type.fields['solids'].config_type assert solids_config_type.fields[ 'required_field_solid'].is_optional is False required_solid_config_type = solids_config_type.fields[ 'required_field_solid'].config_type assert required_solid_config_type.fields['config'].is_optional is False assert set( env_config_type.fields['loggers'].config_type.fields.keys()) == set( ['console']) console_logger_config_type = env_config_type.fields[ 'loggers'].config_type.fields['console'] assert set(console_logger_config_type.config_type.fields.keys()) == set( ['config']) assert console_logger_config_type.config_type.fields['config'].is_optional console_logger_config_config_type = console_logger_config_type.config_type.fields[ 'config'].config_type assert set(console_logger_config_config_type.fields.keys()) == set( ['log_level', 'name']) assert scaffold_pipeline_config(pipeline_def, skip_optional=False) == { 'loggers': { 'console': { 'config': { 'log_level': '', 'name': '' } } }, 'solids': { 'required_field_solid': { 'config': { 'required_int': 0 } } }, 'expectations': { 'evaluate': True }, 'execution': {}, 'resources': {}, 'storage': { 'filesystem': { 'base_dir': '' }, 'in_memory': {}, 's3': { 's3_bucket': '' } }, }