def test_double_type_key():
class KeyOneNameOneType(ConfigType):
def __init__(self):
super(KeyOneNameOneType, self).__init__(key='KeyOne',
name='NameOne')
class KeyOneNameTwoType(ConfigType):
def __init__(self):
super(KeyOneNameTwoType, self).__init__(key='KeyOne',
name='NameTwo')
@solid(config_field=Field(KeyOneNameOneType))
def solid_one(_context):
raise Exception('should not execute')
@solid(config_field=Field(KeyOneNameTwoType))
def solid_two(_context):
raise Exception('should not execute')
with pytest.raises(DagsterInvalidDefinitionError) as exc_info:
create_environment_schema(
PipelineDefinition(solid_defs=[solid_one, solid_two]))
assert str(exc_info.value) == (
'Type keys must be unique. You have constructed two different instances of types '
'with the same key "KeyOne".')
def test_double_type_key():
class NameOneType(_ConfigHasFields):
def __init__(self):
super(NameOneType, self).__init__(key='Key',
name='NameOne',
kind=ConfigTypeKind.DICT,
fields={'foo': Field(Int)})
class NameOneTypePrime(_ConfigHasFields):
def __init__(self):
super(NameOneTypePrime, self).__init__(key='Key',
name='NameOnePrime',
kind=ConfigTypeKind.DICT,
fields={'bar': Field(Int)})
@solid(config_field=Field(NameOneType))
def solid_one(_context):
raise Exception('should not execute')
@solid(config_field=Field(NameOneTypePrime))
def solid_two(_context):
raise Exception('should not execute')
with pytest.raises(DagsterInvalidDefinitionError) as exc_info:
create_environment_schema(
PipelineDefinition(solid_defs=[solid_one, solid_two]))
assert str(exc_info.value) == (
'Type keys must be unique. You have constructed two different instances of types '
'with the same key "Key".')
def test_double_type_name():
@solid(config_field=Field(NamedDict('SomeTypeName', {'some_field': Field(String)})))
def solid_one(_context):
raise Exception('should not execute')
@solid(config_field=Field(NamedDict('SomeTypeName', {'another_field': Field(String)})))
def solid_two(_context):
raise Exception('should not execute')
with pytest.raises(DagsterInvalidDefinitionError) as exc_info:
create_environment_schema(PipelineDefinition(solid_defs=[solid_one, solid_two]))
assert str(exc_info.value) == (
'Type names must be unique. You have constructed two different instances of '
'types with the same name "SomeTypeName".'
)
def test_all_types_provided():
pipeline_def = PipelineDefinition(
name='pipeline',
solid_defs=[],
mode_defs=[
ModeDefinition(
name='SomeMode',
resource_defs={
'some_resource': ResourceDefinition(
lambda: None,
config_field=Field(
NamedDict(
'SomeModeNamedDict',
{
'with_default_int': Field(
Int, is_optional=True, default_value=23434
)
},
)
),
)
},
)
],
)
environment_schema = create_environment_schema(pipeline_def)
all_types = list(environment_schema.all_config_types())
type_names = set(t.name for t in all_types)
assert 'SomeModeNamedDict' in type_names
assert 'Pipeline.Mode.SomeMode.Environment' in type_names
assert 'Pipeline.Mode.SomeMode.Resources.SomeResource' in type_names
def test_secret_field():
@solid(config_field=Field(
Dict({
'password': Field(String, is_secret=True),
'notpassword': Field(String)
})))
def solid_with_secret(_context):
pass
pipeline_def = PipelineDefinition(name='secret_pipeline',
solid_defs=[solid_with_secret])
environment_schema = create_environment_schema(pipeline_def)
config_type = environment_schema.config_type_named(
'SecretPipeline.SolidConfig.SolidWithSecret')
assert config_type
password_field = config_type.fields['config'].config_type.fields[
'password']
assert password_field.is_secret
notpassword_field = config_type.fields['config'].config_type.fields[
'notpassword']
assert not notpassword_field.is_secret
def get_validated_config(graphene_info, dauphin_pipeline, environment_dict,
mode):
check.opt_dict_param(environment_dict, 'environment_dict', key_type=str)
check.str_param(mode, 'mode')
pipeline = dauphin_pipeline.get_dagster_pipeline()
environment_schema = create_environment_schema(pipeline, mode)
validated_config = evaluate_config(environment_schema.environment_type,
environment_dict, pipeline)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=dauphin_pipeline,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
graphene_info, err)
for err in validated_config.errors
],
))
return validated_config
def test_pipeline_types():
@lambda_solid
def produce_string():
return 'foo'
@solid(
input_defs=[InputDefinition('input_one', String)],
output_defs=[OutputDefinition(Any)],
config={'another_field': Int},
)
def solid_one(_context, input_one):
raise Exception('should not execute')
pipeline_def = PipelineDefinition(
solid_defs=[produce_string, solid_one],
dependencies={
'solid_one': {
'input_one': DependencyDefinition('produce_string')
}
},
)
environment_schema = create_environment_schema(pipeline_def)
assert environment_schema.has_config_type('String')
assert environment_schema.has_config_type('Int')
assert not environment_schema.has_config_type('SomeName')
def test_all_types_provided():
pipeline_def = PipelineDefinition(
name='pipeline',
solid_defs=[],
mode_defs=[
ModeDefinition(
name='SomeMode',
resource_defs={
'some_resource':
ResourceDefinition(
lambda: None,
config={
'with_default_int':
Field(Int, is_optional=True, default_value=23434)
},
)
},
)
],
)
environment_schema = create_environment_schema(pipeline_def)
all_types = list(environment_schema.all_config_types())
matching_types = [
tt for tt in all_types if tt.kind == ConfigTypeKind.STRICT_SHAPE
and 'with_default_int' in tt.fields.keys()
]
assert len(matching_types) == 1
def is_config_valid(pipeline_def, environment_dict, mode):
check.str_param(mode, 'mode')
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
environment_schema = create_environment_schema(pipeline_def, mode)
validated_config = validate_config(environment_schema.environment_type, environment_dict)
return validated_config.success
def resolve_config_types(self, _graphene_info, mode=None):
environment_schema = create_environment_schema(self._pipeline, mode)
return sorted(
list(
map(to_dauphin_config_type,
environment_schema.all_config_types())),
key=lambda config_type: config_type.key,
)
def get_config_type(graphene_info, pipeline_name, config_type_name, mode):
check.str_param(pipeline_name, 'pipeline_name')
check.str_param(config_type_name, 'config_type_name')
check.opt_str_param(mode, 'mode')
pipeline = get_dagster_pipeline_from_selector(graphene_info, ExecutionSelector(pipeline_name))
environment_schema = create_environment_schema(pipeline, mode)
if not environment_schema.has_config_type(config_type_name):
raise UserFacingGraphQLError(
graphene_info.schema.type_named('ConfigTypeNotFoundError')(
pipeline=pipeline, config_type_name=config_type_name
)
)
return to_dauphin_config_type(environment_schema.config_type_named(config_type_name))
def test_materialization_schema_types():
@lambda_solid(output_def=OutputDefinition(String))
def return_one():
return 1
pipeline_def = PipelineDefinition(name='test_materialization_schema_types',
solid_defs=[return_one])
environment_schema = create_environment_schema(pipeline_def)
string_mat_schema = environment_schema.config_type_named(
'String.MaterializationSchema')
string_json_mat_schema = string_mat_schema.fields['json'].config_type
assert environment_schema.config_type_keyed(string_json_mat_schema.key)
def get_validated_config(pipeline_def, environment_dict, mode):
from dagster_graphql.schema.errors import DauphinPipelineConfigValidationInvalid
check.str_param(mode, 'mode')
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
environment_schema = create_environment_schema(pipeline_def, mode)
validated_config = validate_config(environment_schema.environment_type,
environment_dict)
if not validated_config.success:
raise UserFacingGraphQLError(
DauphinPipelineConfigValidationInvalid.for_validation_errors(
pipeline_def.get_pipeline_index(), validated_config.errors))
return validated_config
def get_validated_config(graphene_info, pipeline_def, environment_dict, mode):
check.str_param(mode, 'mode')
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
environment_schema = create_environment_schema(pipeline_def, mode)
validated_config = validate_config(environment_schema.environment_type,
environment_dict)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=DauphinPipeline.from_pipeline_def(pipeline_def),
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
pipeline_def.get_config_schema_snapshot(), err)
for err in validated_config.errors
],
))
return validated_config
