def test_scalar_or_selector(): int_or_selector = ScalarUnion( scalar_type=int, non_scalar_schema=Selector({ 'a_string': str, 'an_int': int }), ) assert validate_config(int_or_selector, 2).success assert not validate_config(int_or_selector, '2').success assert not validate_config(int_or_selector, False).success assert validate_config(int_or_selector, {'a_string': 'kjdfk'}).success assert validate_config(int_or_selector, {'an_int': 2}).success assert not validate_config(int_or_selector, {}).success assert not validate_config(int_or_selector, { 'a_string': 'kjdfk', 'an_int': 2 }).success assert not validate_config(int_or_selector, {'wrong_key': 'kjdfd'}).success assert not validate_config(int_or_selector, {'a_string': 2}).success assert not validate_config(int_or_selector, { 'a_string': 'kjdfk', 'extra_field': 'kd' }).success
def test_scalar_or_selector(): int_or_selector = ScalarUnion( scalar_type=int, non_scalar_schema=Selector({ "a_string": str, "an_int": int }), ) assert validate_config(int_or_selector, 2).success assert not validate_config(int_or_selector, "2").success assert not validate_config(int_or_selector, False).success assert validate_config(int_or_selector, {"a_string": "kjdfk"}).success assert validate_config(int_or_selector, {"an_int": 2}).success assert not validate_config(int_or_selector, {}).success assert not validate_config(int_or_selector, { "a_string": "kjdfk", "an_int": 2 }).success assert not validate_config(int_or_selector, {"wrong_key": "kjdfd"}).success assert not validate_config(int_or_selector, {"a_string": 2}).success assert not validate_config(int_or_selector, { "a_string": "kjdfk", "extra_field": "kd" }).success
def dagster_instance_config( base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None, ): check.str_param(base_dir, "base_dir") check.invariant(os.path.isdir(base_dir), "base_dir should be a directory") overrides = check.opt_dict_param(overrides, "overrides") config_yaml_path = os.path.join(base_dir, config_filename) if not os.path.exists(config_yaml_path): warnings.warn(( "The dagster instance configuration file ({config_filename}) is not present at " "{base_dir}. Dagster uses this file to know where and how to store " "local artifacts, information about past runs, and structured events.\n" "If nothing is specified, Dagster will store this information " "in the local filesystem in the {base_dir} directory.").format( config_filename=config_filename, base_dir=base_dir)) dagster_config_dict = merge_dicts(load_yaml_from_globs(config_yaml_path), overrides) if "custom_instance_class" in dagster_config_dict: custom_instance_class_data = dagster_config_dict[ "custom_instance_class"] validate_custom_config = validate_config( configurable_class_schema(), custom_instance_class_data, ) if not validate_custom_config.success: raise DagsterInvalidConfigError( "Errors whilst loading dagster custom class config at {}". format(config_filename), validate_custom_config.errors, custom_instance_class_data, ) custom_instance_class = class_from_code_pointer( custom_instance_class_data["module"], custom_instance_class_data["class"]) schema = merge_dicts(dagster_instance_config_schema(), custom_instance_class.config_schema()) else: custom_instance_class = None schema = dagster_instance_config_schema() dagster_config = validate_config(schema, dagster_config_dict) if not dagster_config.success: raise DagsterInvalidConfigError( "Errors whilst loading dagster instance config at {}.".format( config_filename), dagster_config.errors, dagster_config_dict, ) return (dagster_config.value, custom_instance_class)
def test_example_selector_success(): result = validate_config(ExampleSelector, {'option_one': 'foo'}) assert result.success assert result.value == {'option_one': 'foo'} result = validate_config(ExampleSelector, {'option_two': 'foo'}) assert result.success assert result.value == {'option_two': 'foo'}
def test_example_selector_success(): result = validate_config(ExampleSelector, {"option_one": "foo"}) assert result.success assert result.value == {"option_one": "foo"} result = validate_config(ExampleSelector, {"option_two": "foo"}) assert result.success assert result.value == {"option_two": "foo"}
def test_permissive_dict_with_fields(): perm_dict_with_field = Permissive({"a_key": Field(str)}) assert validate_config(perm_dict_with_field, {"a_key": "djfkdjkfd"}).success assert validate_config( perm_dict_with_field, {"a_key": "djfkdjkfd", "extra_key": "kdjkfd"} ).success assert not validate_config(perm_dict_with_field, {"a_key": 2}).success assert not validate_config(perm_dict_with_field, {}).success
def dagster_instance_config( base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None, ): check.str_param(base_dir, "base_dir") check.invariant(os.path.isdir(base_dir), "base_dir should be a directory") overrides = check.opt_dict_param(overrides, "overrides") config_yaml_path = os.path.join(base_dir, config_filename) if not os.path.exists(config_yaml_path) and is_dagster_home_set(): warnings.warn( f"No dagster instance configuration file ({config_filename}) found at " f"{base_dir}. Defaulting to loading and storing all metadata with {base_dir}. " f"If this is the desired behavior, create an empty {config_filename} file in {base_dir}." ) dagster_config_dict = merge_dicts(load_yaml_from_globs(config_yaml_path), overrides) if "instance_class" in dagster_config_dict: custom_instance_class_data = dagster_config_dict["instance_class"] validate_custom_config = validate_config( configurable_class_schema(), custom_instance_class_data, ) if not validate_custom_config.success: raise DagsterInvalidConfigError( "Errors whilst loading dagster custom class config at {}". format(config_filename), validate_custom_config.errors, custom_instance_class_data, ) custom_instance_class = class_from_code_pointer( custom_instance_class_data["module"], custom_instance_class_data["class"]) schema = merge_dicts(dagster_instance_config_schema(), custom_instance_class.config_schema()) else: custom_instance_class = None schema = dagster_instance_config_schema() dagster_config = validate_config(schema, dagster_config_dict) if not dagster_config.success: raise DagsterInvalidConfigError( "Errors whilst loading dagster instance config at {}.".format( config_filename), dagster_config.errors, dagster_config_dict, ) return (dagster_config.value, custom_instance_class)
def test_nested_success(): value = { "level_one": { "string_field": "skdsjfkdj", "int_field": 123, "bool_field": True } } assert validate_config(DoubleLevelShape, value).success assert not validate_config(DoubleLevelShape, None).success
def test_nested_success(): value = { 'level_one': { 'string_field': 'skdsjfkdj', 'int_field': 123, 'bool_field': True } } assert validate_config(DoubleLevelShape, value).success assert not validate_config(DoubleLevelShape, None).success
def test_float_field(): config_field = convert_potential_field({"float_field": Float}) assert validate_config(config_field.config_type, {"float_field": 1.0}).value == { "float_field": 1.0 } assert process_config(config_field.config_type, {"float_field": 1.0}).value == { "float_field": 1.0 } assert validate_config(config_field.config_type, {"float_field": 1}).value == {"float_field": 1} assert process_config(config_field.config_type, {"float_field": 1}).value == { "float_field": 1.0 }
def test_permissive_dict_with_fields(): perm_dict_with_field = Permissive({'a_key': Field(str)}) assert validate_config(perm_dict_with_field, { 'a_key': 'djfkdjkfd' }).success assert validate_config(perm_dict_with_field, { 'a_key': 'djfkdjkfd', 'extra_key': 'kdjkfd' }).success assert not validate_config(perm_dict_with_field, {'a_key': 2}).success assert not validate_config(perm_dict_with_field, {}).success
def test_config_double_list(): nested_lists = {'nested_list_one': [int], 'nested_list_two': [str]} value = {'nested_list_one': [1, 2, 3], 'nested_list_two': ['foo', 'bar']} result = validate_config(nested_lists, value) assert result.success assert result.value == value error_value = {'nested_list_one': 'kjdfkdj', 'nested_list_two': ['bar']} error_result = validate_config(nested_lists, error_value) assert not error_result.success
def test_config_double_list(): nested_lists = {"nested_list_one": [int], "nested_list_two": [str]} value = {"nested_list_one": [1, 2, 3], "nested_list_two": ["foo", "bar"]} result = validate_config(nested_lists, value) assert result.success assert result.value == value error_value = {"nested_list_one": "kjdfkdj", "nested_list_two": ["bar"]} error_result = validate_config(nested_lists, error_value) assert not error_result.success
def test_config_double_list_double_error(): nested_lists = {"nested_list_one": [int], "nested_list_two": [str]} error_value = {"nested_list_one": "kjdfkdj", "nested_list_two": ["bar", 2]} error_result = validate_config(nested_lists, error_value) assert not error_result.success assert len(error_result.errors) == 2
def test_config_double_list_double_error(): nested_lists = {'nested_list_one': [int], 'nested_list_two': [str]} error_value = {'nested_list_one': 'kjdfkdj', 'nested_list_two': ['bar', 2]} error_result = validate_config(nested_lists, error_value) assert not error_result.success assert len(error_result.errors) == 2
def test_config_list_in_dict(): nested_list_type = {'nested_list': [int]} value = {'nested_list': [1, 2, 3]} result = validate_config(nested_list_type, value) assert result.success assert result.value == value
def is_config_valid(pipeline_def, environment_dict, mode): check.str_param(mode, 'mode') check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) environment_schema = create_environment_schema(pipeline_def, mode) validated_config = validate_config(environment_schema.environment_type, environment_dict) return validated_config.success
def test_example_selector_wrong_field(): result = validate_config(ExampleSelector, {'nope': 234}) assert not result.success assert result.value is None assert len(result.errors) == 1 assert result.errors[ 0].reason == DagsterEvaluationErrorReason.FIELD_NOT_DEFINED
def test_example_selector_error_top_level_type(): result = validate_config(ExampleSelector, 'kjsdkf') assert not result.success assert result.value is None assert len(result.errors) == 1 assert result.errors[ 0].reason == DagsterEvaluationErrorReason.RUNTIME_TYPE_MISMATCH
def resolve_from_unvalidated_config(self, config: Any) -> Any: """Validates config against outer config schema, and calls mapping against validated config.""" receive_processed_config_values = check.opt_bool_param( self.receive_processed_config_values, "receive_processed_config_values", default=True) if receive_processed_config_values: outer_evr = process_config( self.config_schema.config_type, config, ) else: outer_evr = validate_config( self.config_schema.config_type, config, ) if not outer_evr.success: raise DagsterInvalidConfigError( "Error in config mapping ", outer_evr.errors, config, ) outer_config = outer_evr.value if not receive_processed_config_values: outer_config = resolve_defaults( cast(ConfigType, self.config_schema.config_type), outer_config, ).value return self.config_fn(outer_config)
def test_deep_scalar(): value = { 'level_one_string_field': 'foo', 'level_two_dict': { 'level_two_int_field': 234234, 'level_three_dict': { 'level_three_string': 123 }, }, } result = validate_config(MultiLevelShapeType, value) assert not result.success assert len(result.errors) == 1 error = result.errors[0] assert error.reason == DagsterEvaluationErrorReason.RUNTIME_TYPE_MISMATCH assert error.error_data.config_type_snap.given_name == 'String' assert error.error_data.value_rep == '123' assert len(error.stack.entries) == 3 assert [entry.field_name for entry in error.stack.entries] == [ 'level_two_dict', 'level_three_dict', 'level_three_string', ] assert not result.errors_at_level('level_one_string_field') assert not result.errors_at_level('level_two_dict') assert not result.errors_at_level('level_two_dict', 'level_three_dict') assert (len( result.errors_at_level('level_two_dict', 'level_three_dict', 'level_three_string')) == 1)
def get_user_defined_k8s_config(tags): check.inst_param(tags, "tags", frozentags) if not any(key in tags for key in [K8S_RESOURCE_REQUIREMENTS_KEY, USER_DEFINED_K8S_CONFIG_KEY]): return UserDefinedDagsterK8sConfig() user_defined_k8s_config = {} if USER_DEFINED_K8S_CONFIG_KEY in tags: user_defined_k8s_config_value = json.loads(tags[USER_DEFINED_K8S_CONFIG_KEY]) result = validate_config(USER_DEFINED_K8S_CONFIG_SCHEMA, user_defined_k8s_config_value) if not result.success: raise DagsterInvalidConfigError( "Error in tags for {}".format(USER_DEFINED_K8S_CONFIG_KEY), result.errors, result, ) user_defined_k8s_config = result.value container_config = user_defined_k8s_config.get("container_config", {}) # Backcompat for resource requirements key if K8S_RESOURCE_REQUIREMENTS_KEY in tags: resource_requirements_config = get_k8s_resource_requirements(tags) container_config = merge_dicts( container_config, {"resources": resource_requirements_config} ) return UserDefinedDagsterK8sConfig( container_config=container_config, pod_template_spec_metadata=user_defined_k8s_config.get("pod_template_spec_metadata"), pod_spec_config=user_defined_k8s_config.get("pod_spec_config"), job_config=user_defined_k8s_config.get("job_config"), job_spec_config=user_defined_k8s_config.get("job_spec_config"), )
def is_config_valid(pipeline_def, run_config, mode): check.str_param(mode, "mode") check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition) run_config_schema = create_run_config_schema(pipeline_def, mode) validated_config = validate_config(run_config_schema.environment_type, run_config) return validated_config.success
def get_config_dir(config_yaml=None): instance = DagsterInstance.get() config_type = celery_executor.config_field.config_type config_value = get_config_value_from_yaml(config_yaml) config_module_name = 'dagster_celery_config' config_dir = os.path.join(instance.root_directory, 'dagster_celery', 'config', str(uuid.uuid4())) mkdir_p(config_dir) config_path = os.path.join( config_dir, '{config_module_name}.py'.format( config_module_name=config_module_name)) validated_config = validate_config(config_type, config_value).value with open(config_path, 'w') as fd: if 'broker' in validated_config: fd.write('broker_url = \'{broker_url}\'\n'.format( broker_url=str(validated_config['broker']))) if 'backend' in validated_config: fd.write('result_backend = \'{result_backend}\'\n'.format( result_backend=str(validated_config['backend']))) if 'config_source' in validated_config: for key, value in validated_config['config_source'].items(): fd.write('{key} = {value}\n'.format(key=key, value=repr(value))) # n.b. right now we don't attempt to clean up this cache, but it might make sense to delete # any files older than some time if there are more than some number of files present, etc. return config_dir
def test_deep_scalar(): value = { "level_one_string_field": "foo", "level_two_dict": { "level_two_int_field": 234234, "level_three_dict": { "level_three_string": 123 }, }, } result = validate_config(MultiLevelShapeType, value) assert not result.success assert len(result.errors) == 1 error = result.errors[0] assert error.reason == DagsterEvaluationErrorReason.RUNTIME_TYPE_MISMATCH assert error.error_data.config_type_snap.given_name == "String" assert error.error_data.value_rep == "123" assert len(error.stack.entries) == 3 assert [entry.field_name for entry in error.stack.entries] == [ "level_two_dict", "level_three_dict", "level_three_string", ] assert not result.errors_at_level("level_one_string_field") assert not result.errors_at_level("level_two_dict") assert not result.errors_at_level("level_two_dict", "level_three_dict") assert (len( result.errors_at_level("level_two_dict", "level_three_dict", "level_three_string")) == 1)
def resolve_is_environment_config_valid(graphene_info, environment_schema, dagster_pipeline, environment_dict): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(environment_schema, 'environment_schema', EnvironmentSchema) check.inst_param(dagster_pipeline, 'dagster_pipeline', PipelineDefinition) check.dict_param(environment_dict, 'environment_dict', key_type=str) validated_config = validate_config(environment_schema.environment_type, environment_dict) dauphin_pipeline = graphene_info.schema.type_named('Pipeline')( dagster_pipeline) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=dauphin_pipeline, errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( graphene_info, err) for err in validated_config.errors ], )) return graphene_info.schema.type_named('PipelineConfigValidationValid')( dauphin_pipeline)
def test_int_field(): config_field = convert_potential_field({'int_field': Int}) assert validate_config(config_field.config_type, { 'int_field': 1 }).value == { 'int_field': 1 }
def _check_invocation_requirements( solid_def: "SolidDefinition", context: Optional["DirectSolidExecutionContext"] ) -> "DirectSolidExecutionContext": """Ensure that provided context fulfills requirements of solid definition. If no context was provided, then construct an enpty DirectSolidExecutionContext """ from dagster.core.execution.context.invocation import DirectSolidExecutionContext from dagster.config.validate import validate_config # Check resource requirements if solid_def.required_resource_keys and context is None: raise DagsterInvalidInvocationError( f'Solid "{solid_def.name}" has required resources, but no context was provided. Use the' "`build_solid_context` function to construct a context with the required " "resources." ) if context is not None and solid_def.required_resource_keys: resources_dict = cast( # type: ignore[attr-defined] "DirectSolidExecutionContext", context, ).resources._asdict() for resource_key in solid_def.required_resource_keys: if resource_key not in resources_dict: raise DagsterInvalidInvocationError( f'Solid "{solid_def.name}" requires resource "{resource_key}", but no resource ' "with that key was found on the context." ) # Check config requirements if not context and solid_def.config_schema.as_field().is_required: raise DagsterInvalidInvocationError( f'Solid "{solid_def.name}" has required config schema, but no context was provided. ' "Use the `build_solid_context` function to create a context with config." ) config = None if solid_def.config_field: solid_config = check.opt_dict_param( context.solid_config if context else None, "solid_config" ) config_evr = validate_config(solid_def.config_field.config_type, solid_config) if not config_evr.success: raise DagsterInvalidConfigError( "Error in config for solid ", config_evr.errors, solid_config ) mapped_config_evr = solid_def.apply_config_mapping({"config": solid_config}) if not mapped_config_evr.success: raise DagsterInvalidConfigError( "Error in config for solid ", mapped_config_evr.errors, solid_config ) config = mapped_config_evr.value.get("config", {}) return ( context if context else DirectSolidExecutionContext(solid_config=config, resources_dict=None, instance=None) )
def test_shape_with_field_substitutions_collisions(): value = {"foo_field": {}, "bar_field": "world", "foo": {}} result = validate_config(FieldSubShape, value) assert not result.success assert len(result.errors) == 1 collision_error = result.errors[0] assert collision_error.reason == DagsterEvaluationErrorReason.FIELD_ALIAS_COLLISION
def test_selector_within_dict_no_subfields(): result = validate_config(Shape({'selector': Field(ExampleSelector)}), {'selector': {}}) assert not result.success assert len(result.errors) == 1 assert result.errors[0].message == ( "Must specify a field at path root:selector if more than one field " "is defined. Defined fields: ['option_one', 'option_two']")