def test_double_type_key(): class KeyOneNameOneType(ConfigType): def __init__(self): super(KeyOneNameOneType, self).__init__(key='KeyOne', name='NameOne') class KeyOneNameTwoType(ConfigType): def __init__(self): super(KeyOneNameTwoType, self).__init__(key='KeyOne', name='NameTwo') @solid(config_field=Field(KeyOneNameOneType)) def solid_one(_context): raise Exception('should not execute') @solid(config_field=Field(KeyOneNameTwoType)) def solid_two(_context): raise Exception('should not execute') with pytest.raises(DagsterInvalidDefinitionError) as exc_info: create_environment_schema( PipelineDefinition(solid_defs=[solid_one, solid_two])) assert str(exc_info.value) == ( 'Type keys must be unique. You have constructed two different instances of types ' 'with the same key "KeyOne".')
def test_double_type_key(): class NameOneType(_ConfigHasFields): def __init__(self): super(NameOneType, self).__init__(key='Key', name='NameOne', kind=ConfigTypeKind.DICT, fields={'foo': Field(Int)}) class NameOneTypePrime(_ConfigHasFields): def __init__(self): super(NameOneTypePrime, self).__init__(key='Key', name='NameOnePrime', kind=ConfigTypeKind.DICT, fields={'bar': Field(Int)}) @solid(config_field=Field(NameOneType)) def solid_one(_context): raise Exception('should not execute') @solid(config_field=Field(NameOneTypePrime)) def solid_two(_context): raise Exception('should not execute') with pytest.raises(DagsterInvalidDefinitionError) as exc_info: create_environment_schema( PipelineDefinition(solid_defs=[solid_one, solid_two])) assert str(exc_info.value) == ( 'Type keys must be unique. You have constructed two different instances of types ' 'with the same key "Key".')
def test_double_type_name(): @solid(config_field=Field(NamedDict('SomeTypeName', {'some_field': Field(String)}))) def solid_one(_context): raise Exception('should not execute') @solid(config_field=Field(NamedDict('SomeTypeName', {'another_field': Field(String)}))) def solid_two(_context): raise Exception('should not execute') with pytest.raises(DagsterInvalidDefinitionError) as exc_info: create_environment_schema(PipelineDefinition(solid_defs=[solid_one, solid_two])) assert str(exc_info.value) == ( 'Type names must be unique. You have constructed two different instances of ' 'types with the same name "SomeTypeName".' )
def test_all_types_provided(): pipeline_def = PipelineDefinition( name='pipeline', solid_defs=[], mode_defs=[ ModeDefinition( name='SomeMode', resource_defs={ 'some_resource': ResourceDefinition( lambda: None, config_field=Field( NamedDict( 'SomeModeNamedDict', { 'with_default_int': Field( Int, is_optional=True, default_value=23434 ) }, ) ), ) }, ) ], ) environment_schema = create_environment_schema(pipeline_def) all_types = list(environment_schema.all_config_types()) type_names = set(t.name for t in all_types) assert 'SomeModeNamedDict' in type_names assert 'Pipeline.Mode.SomeMode.Environment' in type_names assert 'Pipeline.Mode.SomeMode.Resources.SomeResource' in type_names
def test_secret_field(): @solid(config_field=Field( Dict({ 'password': Field(String, is_secret=True), 'notpassword': Field(String) }))) def solid_with_secret(_context): pass pipeline_def = PipelineDefinition(name='secret_pipeline', solid_defs=[solid_with_secret]) environment_schema = create_environment_schema(pipeline_def) config_type = environment_schema.config_type_named( 'SecretPipeline.SolidConfig.SolidWithSecret') assert config_type password_field = config_type.fields['config'].config_type.fields[ 'password'] assert password_field.is_secret notpassword_field = config_type.fields['config'].config_type.fields[ 'notpassword'] assert not notpassword_field.is_secret
def get_validated_config(graphene_info, dauphin_pipeline, environment_dict, mode): check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) check.str_param(mode, 'mode') pipeline = dauphin_pipeline.get_dagster_pipeline() environment_schema = create_environment_schema(pipeline, mode) validated_config = evaluate_config(environment_schema.environment_type, environment_dict, pipeline) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=dauphin_pipeline, errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( graphene_info, err) for err in validated_config.errors ], )) return validated_config
def test_pipeline_types(): @lambda_solid def produce_string(): return 'foo' @solid( input_defs=[InputDefinition('input_one', String)], output_defs=[OutputDefinition(Any)], config={'another_field': Int}, ) def solid_one(_context, input_one): raise Exception('should not execute') pipeline_def = PipelineDefinition( solid_defs=[produce_string, solid_one], dependencies={ 'solid_one': { 'input_one': DependencyDefinition('produce_string') } }, ) environment_schema = create_environment_schema(pipeline_def) assert environment_schema.has_config_type('String') assert environment_schema.has_config_type('Int') assert not environment_schema.has_config_type('SomeName')
def test_all_types_provided(): pipeline_def = PipelineDefinition( name='pipeline', solid_defs=[], mode_defs=[ ModeDefinition( name='SomeMode', resource_defs={ 'some_resource': ResourceDefinition( lambda: None, config={ 'with_default_int': Field(Int, is_optional=True, default_value=23434) }, ) }, ) ], ) environment_schema = create_environment_schema(pipeline_def) all_types = list(environment_schema.all_config_types()) matching_types = [ tt for tt in all_types if tt.kind == ConfigTypeKind.STRICT_SHAPE and 'with_default_int' in tt.fields.keys() ] assert len(matching_types) == 1
def is_config_valid(pipeline_def, environment_dict, mode): check.str_param(mode, 'mode') check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) environment_schema = create_environment_schema(pipeline_def, mode) validated_config = validate_config(environment_schema.environment_type, environment_dict) return validated_config.success
def resolve_config_types(self, _graphene_info, mode=None): environment_schema = create_environment_schema(self._pipeline, mode) return sorted( list( map(to_dauphin_config_type, environment_schema.all_config_types())), key=lambda config_type: config_type.key, )
def get_config_type(graphene_info, pipeline_name, config_type_name, mode): check.str_param(pipeline_name, 'pipeline_name') check.str_param(config_type_name, 'config_type_name') check.opt_str_param(mode, 'mode') pipeline = get_dagster_pipeline_from_selector(graphene_info, ExecutionSelector(pipeline_name)) environment_schema = create_environment_schema(pipeline, mode) if not environment_schema.has_config_type(config_type_name): raise UserFacingGraphQLError( graphene_info.schema.type_named('ConfigTypeNotFoundError')( pipeline=pipeline, config_type_name=config_type_name ) ) return to_dauphin_config_type(environment_schema.config_type_named(config_type_name))
def test_materialization_schema_types(): @lambda_solid(output_def=OutputDefinition(String)) def return_one(): return 1 pipeline_def = PipelineDefinition(name='test_materialization_schema_types', solid_defs=[return_one]) environment_schema = create_environment_schema(pipeline_def) string_mat_schema = environment_schema.config_type_named( 'String.MaterializationSchema') string_json_mat_schema = string_mat_schema.fields['json'].config_type assert environment_schema.config_type_keyed(string_json_mat_schema.key)
def get_validated_config(pipeline_def, environment_dict, mode): from dagster_graphql.schema.errors import DauphinPipelineConfigValidationInvalid check.str_param(mode, 'mode') check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) environment_schema = create_environment_schema(pipeline_def, mode) validated_config = validate_config(environment_schema.environment_type, environment_dict) if not validated_config.success: raise UserFacingGraphQLError( DauphinPipelineConfigValidationInvalid.for_validation_errors( pipeline_def.get_pipeline_index(), validated_config.errors)) return validated_config
def get_validated_config(graphene_info, pipeline_def, environment_dict, mode): check.str_param(mode, 'mode') check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) environment_schema = create_environment_schema(pipeline_def, mode) validated_config = validate_config(environment_schema.environment_type, environment_dict) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=DauphinPipeline.from_pipeline_def(pipeline_def), errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( pipeline_def.get_config_schema_snapshot(), err) for err in validated_config.errors ], )) return validated_config