def test_two_modes(snapshot): pipeline_def = PipelineDefinition( name='TwoModePipelines', solid_defs=[], mode_defs=[ ModeDefinition( 'mode_one', resource_defs={ 'value': dummy_resource({'mode_one_field': String}) }, ), ModeDefinition( 'mode_two', resource_defs={ 'value': dummy_resource({'mode_two_field': Int}) }, ), ], ) snapshot.assert_match( scaffold_pipeline_config(pipeline_def, mode='mode_one')) snapshot.assert_match( scaffold_pipeline_config(pipeline_def, mode='mode_one', skip_optional=False)) snapshot.assert_match( scaffold_pipeline_config(pipeline_def, mode='mode_two')) snapshot.assert_match( scaffold_pipeline_config(pipeline_def, mode='mode_two', skip_optional=False))
def test_two_modes(snapshot): pipeline_def = PipelineDefinition( name="TwoModePipelines", solid_defs=[], mode_defs=[ ModeDefinition( "mode_one", resource_defs={ "value": dummy_resource({"mode_one_field": String}) }, ), ModeDefinition( "mode_two", resource_defs={ "value": dummy_resource({"mode_two_field": Int}) }, ), ], ) snapshot.assert_match( scaffold_pipeline_config(pipeline_def, mode="mode_one")) snapshot.assert_match( scaffold_pipeline_config(pipeline_def, mode="mode_one", skip_non_required=False)) snapshot.assert_match( scaffold_pipeline_config(pipeline_def, mode="mode_two")) snapshot.assert_match( scaffold_pipeline_config(pipeline_def, mode="mode_two", skip_non_required=False))
def test_two_contexts(): pipeline_def = PipelineDefinition( name='TwoContextsPipeline', solids=[], context_definitions={ 'context_one': PipelineContextDefinition( context_fn=lambda *args: fail_me(), config_field=Field(Dict({'context_one_field': Field(String)})), ), 'context_two': PipelineContextDefinition( context_fn=lambda *args: fail_me(), config_field=Field(Dict({'context_two_field': Field(Int)})), ), }, ) assert scaffold_pipeline_config(pipeline_def) == {'context': {}} assert scaffold_pipeline_config(pipeline_def, skip_optional=False) == { 'context': { 'context_one': {'config': {'context_one_field': ''}, 'resources': {}}, 'context_two': {'config': {'context_two_field': 0}, 'resources': {}}, }, 'solids': {}, 'expectations': {'evaluate': True}, 'execution': {}, }
def test_basic_solids_config(snapshot): pipeline_def = PipelineDefinition( name='BasicSolidsConfigPipeline', solid_defs=[ SolidDefinition( name='required_field_solid', input_defs=[], output_defs=[], config={'required_int': Int}, compute_fn=lambda *_args: fail_me(), ) ], ) env_config_type = create_environment_type(pipeline_def) assert env_config_type.fields['solids'].is_optional is False solids_config_type = env_config_type.fields['solids'].config_type assert solids_config_type.fields[ 'required_field_solid'].is_optional is False required_solid_config_type = solids_config_type.fields[ 'required_field_solid'].config_type assert required_solid_config_type.fields['config'].is_optional is False assert set( env_config_type.fields['loggers'].config_type.fields.keys()) == set( ['console']) console_logger_config_type = env_config_type.fields[ 'loggers'].config_type.fields['console'] assert set(console_logger_config_type.config_type.fields.keys()) == set( ['config']) assert console_logger_config_type.config_type.fields['config'].is_optional console_logger_config_config_type = console_logger_config_type.config_type.fields[ 'config'].config_type assert set(console_logger_config_config_type.fields.keys()) == set( ['log_level', 'name']) snapshot.assert_match( scaffold_pipeline_config(pipeline_def, skip_optional=False))
def test_basic_solids_config(snapshot): pipeline_def = PipelineDefinition( name="BasicSolidsConfigPipeline", solid_defs=[ SolidDefinition( name="required_field_solid", input_defs=[], output_defs=[], config_schema={"required_int": Int}, compute_fn=lambda *_args: fail_me(), ) ], ) env_config_type = create_environment_type(pipeline_def) assert env_config_type.fields["solids"].is_required solids_config_type = env_config_type.fields["solids"].config_type assert solids_config_type.fields["required_field_solid"].is_required required_solid_config_type = solids_config_type.fields[ "required_field_solid"].config_type assert required_solid_config_type.fields["config"].is_required assert set( env_config_type.fields["loggers"].config_type.fields.keys()) == set( ["console"]) console_logger_config_type = env_config_type.fields[ "loggers"].config_type.fields["console"] assert set(console_logger_config_type.config_type.fields.keys()) == set( ["config"]) assert console_logger_config_type.config_type.fields[ "config"].is_required is False console_logger_config_config_type = console_logger_config_type.config_type.fields[ "config"].config_type assert set(console_logger_config_config_type.fields.keys()) == set( ["log_level", "name"]) snapshot.assert_match( scaffold_pipeline_config(pipeline_def, skip_non_required=False))
def test_basic_solids_config(): pipeline_def = PipelineDefinition( name='BasicSolidsConfigPipeline', solids=[ SolidDefinition( name='required_field_solid', inputs=[], outputs=[], config_field=Field(Dict(fields={'required_int': Field(Int)})), transform_fn=lambda *_args: fail_me(), ) ], ) env_config_type = pipeline_def.environment_type assert env_config_type.fields['solids'].is_optional is False solids_config_type = env_config_type.fields['solids'].config_type assert solids_config_type.fields['required_field_solid'].is_optional is False required_solid_config_type = solids_config_type.fields['required_field_solid'].config_type assert required_solid_config_type.fields['config'].is_optional is False context_config_type = env_config_type.fields['context'].config_type assert 'default' in context_config_type.fields assert context_config_type.fields['default'].is_optional default_context_config_type = context_config_type.fields['default'].config_type assert set(default_context_config_type.fields.keys()) == set(['config', 'resources']) default_context_user_config_type = default_context_config_type.fields['config'].config_type assert set(default_context_user_config_type.fields.keys()) == set(['log_level']) assert scaffold_pipeline_config(pipeline_def, skip_optional=False) == { 'context': {'default': {'config': {'log_level': ''}, 'resources': {}}}, 'solids': {'required_field_solid': {'config': {'required_int': 0}}}, 'expectations': {'evaluate': True}, 'execution': {}, }
def test_basic_solids_config(): pipeline_def = PipelineDefinition( name='BasicSolidsConfigPipeline', solids=[ SolidDefinition( name='required_field_solid', inputs=[], outputs=[], config_field=Field(Dict(fields={'required_int': Field(Int)})), compute_fn=lambda *_args: fail_me(), ) ], ) env_config_type = create_environment_type(pipeline_def) assert env_config_type.fields['solids'].is_optional is False solids_config_type = env_config_type.fields['solids'].config_type assert solids_config_type.fields[ 'required_field_solid'].is_optional is False required_solid_config_type = solids_config_type.fields[ 'required_field_solid'].config_type assert required_solid_config_type.fields['config'].is_optional is False assert set( env_config_type.fields['loggers'].config_type.fields.keys()) == set( ['console']) console_logger_config_type = env_config_type.fields[ 'loggers'].config_type.fields['console'] assert set(console_logger_config_type.config_type.fields.keys()) == set( ['config']) assert console_logger_config_type.config_type.fields['config'].is_optional console_logger_config_config_type = console_logger_config_type.config_type.fields[ 'config'].config_type assert set(console_logger_config_config_type.fields.keys()) == set( ['log_level', 'name']) assert scaffold_pipeline_config(pipeline_def, skip_optional=False) == { 'loggers': { 'console': { 'config': { 'log_level': '', 'name': '' } } }, 'solids': { 'required_field_solid': { 'config': { 'required_int': 0 } } }, 'expectations': { 'evaluate': True }, 'execution': {}, 'resources': {}, 'storage': { 'filesystem': { 'base_dir': '' }, 'in_memory': {}, 's3': { 's3_bucket': '' } }, }
def test_two_modes(): pipeline_def = PipelineDefinition( name='TwoModePipelines', solids=[], mode_definitions=[ ModeDefinition( 'mode_one', resources={ 'value': dummy_resource( Field(Dict({'mode_one_field': Field(String)}))) }, ), ModeDefinition( 'mode_two', resources={ 'value': dummy_resource(Field(Dict({'mode_two_field': Field(Int)}))) }, ), ], ) assert scaffold_pipeline_config(pipeline_def, mode='mode_one') == { 'resources': { 'value': { 'config': { 'mode_one_field': '' } } } } assert scaffold_pipeline_config(pipeline_def, mode='mode_one', skip_optional=False) == { 'loggers': { 'console': { 'config': { 'log_level': '', 'name': '' } } }, 'solids': {}, 'expectations': { 'evaluate': True }, 'storage': { 'in_memory': {}, 'filesystem': { 'base_dir': '' }, 's3': { 's3_bucket': '' } }, 'execution': {}, 'resources': { 'value': { 'config': { 'mode_one_field': '' } } }, } assert scaffold_pipeline_config(pipeline_def, mode='mode_two') == { 'resources': { 'value': { 'config': { 'mode_two_field': 0 } } } } assert scaffold_pipeline_config(pipeline_def, mode='mode_two', skip_optional=False) == { 'solids': {}, 'expectations': { 'evaluate': True }, 'storage': { 'in_memory': {}, 'filesystem': { 'base_dir': '' }, 's3': { 's3_bucket': '' } }, 'execution': {}, 'resources': { 'value': { 'config': { 'mode_two_field': 0 } } }, 'loggers': { 'console': { 'config': { 'log_level': '', 'name': '' } } }, }