def test_custom_contexts():
@solid(inputs=[], outputs=[OutputDefinition()])
def custom_context_transform(info):
assert info.context.resources == {'field_one': 'value_two'}
pipeline = PipelineDefinition(
solids=[custom_context_transform],
context_definitions={
'custom_one':
PipelineContextDefinition(
config_def=ConfigDefinition.config_dict(
{'field_one': Field(dagster_type=types.String)}),
context_fn=lambda info: ExecutionContext(resources=info.config
),
),
'custom_two':
PipelineContextDefinition(
config_def=ConfigDefinition.config_dict(
{'field_one': Field(dagster_type=types.String)}),
context_fn=lambda info: ExecutionContext(resources=info.config
),
)
},
)
environment_one = config.Environment(
context=config.Context('custom_one', {'field_one': 'value_two'}))
execute_pipeline(pipeline, environment=environment_one)
environment_two = config.Environment(
context=config.Context('custom_two', {'field_one': 'value_two'}))
execute_pipeline(pipeline, environment=environment_two)
def test_invalid_context():
@lambda_solid
def never_transform():
raise Exception('should never execute')
default_context_pipeline = PipelineDefinition(solids=[never_transform])
environment_context_not_found = config.Environment(
context=config.Context('not_found', {}))
with pytest.raises(DagsterInvariantViolationError,
message='Context not_found not found'):
execute_pipeline(default_context_pipeline,
environment=environment_context_not_found,
throw_on_error=True)
environment_field_name_mismatch = config.Environment(
context=config.Context(config={'unexpected': 'value'}))
with pytest.raises(DagsterTypeError,
message='Argument mismatch in context default'):
execute_pipeline(default_context_pipeline,
environment=environment_field_name_mismatch,
throw_on_error=True)
with_argful_context_pipeline = PipelineDefinition(
solids=[never_transform],
context_definitions={
'default':
PipelineContextDefinition(
config_def=ConfigDefinition.config_dict(
{'string_field': Field(types.String)}),
context_fn=lambda info: info.config,
)
})
environment_no_config_error = config.Environment(context=config.Context(
config={}))
with pytest.raises(DagsterTypeError,
message='Argument mismatch in context default'):
execute_pipeline(with_argful_context_pipeline,
environment=environment_no_config_error,
throw_on_error=True)
environment_type_mismatch_error = config.Environment(
context=config.Context(config={'string_field': 1}))
with pytest.raises(DagsterTypeError,
message='Argument mismatch in context default'):
execute_pipeline(with_argful_context_pipeline,
environment=environment_type_mismatch_error,
throw_on_error=True)
def test_default_value():
def _get_config_test_solid(config_key, config_value):
@solid(inputs=[], outputs=[OutputDefinition()])
def config_test(info):
assert info.context.resources == {config_key: config_value}
return config_test
pipeline = PipelineDefinition(
solids=[_get_config_test_solid('field_one', 'heyo')],
context_definitions={
'custom_one':
PipelineContextDefinition(
config_def=ConfigDefinition.config_dict({
'field_one':
Field(
dagster_type=types.String,
is_optional=True,
default_value='heyo',
)
}),
context_fn=lambda info: ExecutionContext(resources=info.config
),
),
})
execute_pipeline(pipeline,
environment=config.Environment(
context=config.Context('custom_one', {})))
def test_default_context_with_log_level():
@solid(inputs=[], outputs=[OutputDefinition()])
def default_context_transform(info):
for logger in info.context._logger.loggers:
assert logger.level == INFO
pipeline = PipelineDefinition(solids=[default_context_transform])
execute_pipeline(pipeline,
environment=config.Environment(context=config.Context(
config={'log_level': 'INFO'})))
with pytest.raises(DagsterTypeError,
message='Argument mismatch in context default'):
execute_pipeline(pipeline,
environment=config.Environment(context=config.Context(
config={'log_level': 2})))
def test_yield_context():
events = []
@solid(inputs=[], outputs=[OutputDefinition()])
def custom_context_transform(info):
assert info.context.resources == {'field_one': 'value_two'}
assert info.context._context_dict['foo'] == 'bar' # pylint: disable=W0212
events.append('during')
def _yield_context(info):
events.append('before')
context = ExecutionContext(resources=info.config)
with context.value('foo', 'bar'):
yield context
events.append('after')
pipeline = PipelineDefinition(
solids=[custom_context_transform],
context_definitions={
'custom_one':
PipelineContextDefinition(
config_def=ConfigDefinition.config_dict(
{'field_one': Field(dagster_type=types.String)}),
context_fn=_yield_context,
),
})
environment_one = config.Environment(
context=config.Context('custom_one', {'field_one': 'value_two'}))
execute_pipeline(pipeline, environment=environment_one)
assert events == ['before', 'during', 'after']
def test_construct_full_environment_default_context_name():
document = '''
context:
config:
context_arg: context_value
'''
environment = config.construct_environment(yaml.load(document))
assert environment == config.Environment(context=config.Context(
'default', {'context_arg': 'context_value'}), )
def test_intro_tutorial_part_nine_final_error():
with pytest.raises(DagsterTypeError, match='Field username not found'):
execute_pipeline(
define_part_nine_final(),
config.Environment(
solids={
'injest_a': config.Solid(2),
'injest_b': config.Solid(3),
},
context=config.Context(
name='cloud',
config={
'credentials': {
'username': '******',
'pass': '******',
},
},
),
),
)
def test_intro_tutorial_part_nine_final_cloud_success():
result = execute_pipeline(
define_part_nine_final(),
config.Environment(
solids={
'injest_a': config.Solid(2),
'injest_b': config.Solid(3),
},
context=config.Context(
name='cloud',
config={
'credentials': {
'user': '******',
'pass': '******',
},
},
),
),
)
assert result.success
def test_intro_tutorial_part_nine_final_local_success():
result = execute_pipeline(
define_part_nine_final(),
config.Environment(solids={
'injest_a': config.Solid(2),
'injest_b': config.Solid(3),
},
context=config.Context(name='local')))
assert result.success
assert result.result_for_solid('injest_a').transformed_value() == 2
assert result.result_for_solid('injest_b').transformed_value() == 3
assert result.result_for_solid('add_ints').transformed_value() == 5
assert result.result_for_solid('mult_ints').transformed_value() == 6
assert result.context.resources.store.values == {
'a': 2,
'b': 3,
'add': 5,
'mult': 6,
}
