def test_custom_contexts(): @solid(inputs=[], outputs=[OutputDefinition()]) def custom_context_transform(info): assert info.context.resources == {'field_one': 'value_two'} pipeline = PipelineDefinition( solids=[custom_context_transform], context_definitions={ 'custom_one': PipelineContextDefinition( config_def=ConfigDefinition.config_dict( {'field_one': Field(dagster_type=types.String)}), context_fn=lambda info: ExecutionContext(resources=info.config ), ), 'custom_two': PipelineContextDefinition( config_def=ConfigDefinition.config_dict( {'field_one': Field(dagster_type=types.String)}), context_fn=lambda info: ExecutionContext(resources=info.config ), ) }, ) environment_one = config.Environment( context=config.Context('custom_one', {'field_one': 'value_two'})) execute_pipeline(pipeline, environment=environment_one) environment_two = config.Environment( context=config.Context('custom_two', {'field_one': 'value_two'})) execute_pipeline(pipeline, environment=environment_two)
def test_invalid_context(): @lambda_solid def never_transform(): raise Exception('should never execute') default_context_pipeline = PipelineDefinition(solids=[never_transform]) environment_context_not_found = config.Environment( context=config.Context('not_found', {})) with pytest.raises(DagsterInvariantViolationError, message='Context not_found not found'): execute_pipeline(default_context_pipeline, environment=environment_context_not_found, throw_on_error=True) environment_field_name_mismatch = config.Environment( context=config.Context(config={'unexpected': 'value'})) with pytest.raises(DagsterTypeError, message='Argument mismatch in context default'): execute_pipeline(default_context_pipeline, environment=environment_field_name_mismatch, throw_on_error=True) with_argful_context_pipeline = PipelineDefinition( solids=[never_transform], context_definitions={ 'default': PipelineContextDefinition( config_def=ConfigDefinition.config_dict( {'string_field': Field(types.String)}), context_fn=lambda info: info.config, ) }) environment_no_config_error = config.Environment(context=config.Context( config={})) with pytest.raises(DagsterTypeError, message='Argument mismatch in context default'): execute_pipeline(with_argful_context_pipeline, environment=environment_no_config_error, throw_on_error=True) environment_type_mismatch_error = config.Environment( context=config.Context(config={'string_field': 1})) with pytest.raises(DagsterTypeError, message='Argument mismatch in context default'): execute_pipeline(with_argful_context_pipeline, environment=environment_type_mismatch_error, throw_on_error=True)
def test_default_value(): def _get_config_test_solid(config_key, config_value): @solid(inputs=[], outputs=[OutputDefinition()]) def config_test(info): assert info.context.resources == {config_key: config_value} return config_test pipeline = PipelineDefinition( solids=[_get_config_test_solid('field_one', 'heyo')], context_definitions={ 'custom_one': PipelineContextDefinition( config_def=ConfigDefinition.config_dict({ 'field_one': Field( dagster_type=types.String, is_optional=True, default_value='heyo', ) }), context_fn=lambda info: ExecutionContext(resources=info.config ), ), }) execute_pipeline(pipeline, environment=config.Environment( context=config.Context('custom_one', {})))
def test_default_context_with_log_level(): @solid(inputs=[], outputs=[OutputDefinition()]) def default_context_transform(info): for logger in info.context._logger.loggers: assert logger.level == INFO pipeline = PipelineDefinition(solids=[default_context_transform]) execute_pipeline(pipeline, environment=config.Environment(context=config.Context( config={'log_level': 'INFO'}))) with pytest.raises(DagsterTypeError, message='Argument mismatch in context default'): execute_pipeline(pipeline, environment=config.Environment(context=config.Context( config={'log_level': 2})))
def test_yield_context(): events = [] @solid(inputs=[], outputs=[OutputDefinition()]) def custom_context_transform(info): assert info.context.resources == {'field_one': 'value_two'} assert info.context._context_dict['foo'] == 'bar' # pylint: disable=W0212 events.append('during') def _yield_context(info): events.append('before') context = ExecutionContext(resources=info.config) with context.value('foo', 'bar'): yield context events.append('after') pipeline = PipelineDefinition( solids=[custom_context_transform], context_definitions={ 'custom_one': PipelineContextDefinition( config_def=ConfigDefinition.config_dict( {'field_one': Field(dagster_type=types.String)}), context_fn=_yield_context, ), }) environment_one = config.Environment( context=config.Context('custom_one', {'field_one': 'value_two'})) execute_pipeline(pipeline, environment=environment_one) assert events == ['before', 'during', 'after']
def test_construct_full_environment_default_context_name(): document = ''' context: config: context_arg: context_value ''' environment = config.construct_environment(yaml.load(document)) assert environment == config.Environment(context=config.Context( 'default', {'context_arg': 'context_value'}), )
def test_intro_tutorial_part_nine_final_error(): with pytest.raises(DagsterTypeError, match='Field username not found'): execute_pipeline( define_part_nine_final(), config.Environment( solids={ 'injest_a': config.Solid(2), 'injest_b': config.Solid(3), }, context=config.Context( name='cloud', config={ 'credentials': { 'username': '******', 'pass': '******', }, }, ), ), )
def test_intro_tutorial_part_nine_final_cloud_success(): result = execute_pipeline( define_part_nine_final(), config.Environment( solids={ 'injest_a': config.Solid(2), 'injest_b': config.Solid(3), }, context=config.Context( name='cloud', config={ 'credentials': { 'user': '******', 'pass': '******', }, }, ), ), ) assert result.success
def test_intro_tutorial_part_nine_final_local_success(): result = execute_pipeline( define_part_nine_final(), config.Environment(solids={ 'injest_a': config.Solid(2), 'injest_b': config.Solid(3), }, context=config.Context(name='local'))) assert result.success assert result.result_for_solid('injest_a').transformed_value() == 2 assert result.result_for_solid('injest_b').transformed_value() == 3 assert result.result_for_solid('add_ints').transformed_value() == 5 assert result.result_for_solid('mult_ints').transformed_value() == 6 assert result.context.resources.store.values == { 'a': 2, 'b': 3, 'add': 5, 'mult': 6, }