def test_scalar_or_selector():
int_or_selector = ScalarUnion(
scalar_type=int,
non_scalar_schema=Selector({
'a_string': str,
'an_int': int
}),
)
assert validate_config(int_or_selector, 2).success
assert not validate_config(int_or_selector, '2').success
assert not validate_config(int_or_selector, False).success
assert validate_config(int_or_selector, {'a_string': 'kjdfk'}).success
assert validate_config(int_or_selector, {'an_int': 2}).success
assert not validate_config(int_or_selector, {}).success
assert not validate_config(int_or_selector, {
'a_string': 'kjdfk',
'an_int': 2
}).success
assert not validate_config(int_or_selector, {'wrong_key': 'kjdfd'}).success
assert not validate_config(int_or_selector, {'a_string': 2}).success
assert not validate_config(int_or_selector, {
'a_string': 'kjdfk',
'extra_field': 'kd'
}).success
def test_scalar_or_selector():
int_or_selector = ScalarUnion(
scalar_type=int,
non_scalar_schema=Selector({
"a_string": str,
"an_int": int
}),
)
assert validate_config(int_or_selector, 2).success
assert not validate_config(int_or_selector, "2").success
assert not validate_config(int_or_selector, False).success
assert validate_config(int_or_selector, {"a_string": "kjdfk"}).success
assert validate_config(int_or_selector, {"an_int": 2}).success
assert not validate_config(int_or_selector, {}).success
assert not validate_config(int_or_selector, {
"a_string": "kjdfk",
"an_int": 2
}).success
assert not validate_config(int_or_selector, {"wrong_key": "kjdfd"}).success
assert not validate_config(int_or_selector, {"a_string": 2}).success
assert not validate_config(int_or_selector, {
"a_string": "kjdfk",
"extra_field": "kd"
}).success
def dagster_instance_config(
base_dir,
config_filename=DAGSTER_CONFIG_YAML_FILENAME,
overrides=None,
):
check.str_param(base_dir, "base_dir")
check.invariant(os.path.isdir(base_dir), "base_dir should be a directory")
overrides = check.opt_dict_param(overrides, "overrides")
config_yaml_path = os.path.join(base_dir, config_filename)
if not os.path.exists(config_yaml_path):
warnings.warn((
"The dagster instance configuration file ({config_filename}) is not present at "
"{base_dir}. Dagster uses this file to know where and how to store "
"local artifacts, information about past runs, and structured events.\n"
"If nothing is specified, Dagster will store this information "
"in the local filesystem in the {base_dir} directory.").format(
config_filename=config_filename, base_dir=base_dir))
dagster_config_dict = merge_dicts(load_yaml_from_globs(config_yaml_path),
overrides)
if "custom_instance_class" in dagster_config_dict:
custom_instance_class_data = dagster_config_dict[
"custom_instance_class"]
validate_custom_config = validate_config(
configurable_class_schema(),
custom_instance_class_data,
)
if not validate_custom_config.success:
raise DagsterInvalidConfigError(
"Errors whilst loading dagster custom class config at {}".
format(config_filename),
validate_custom_config.errors,
custom_instance_class_data,
)
custom_instance_class = class_from_code_pointer(
custom_instance_class_data["module"],
custom_instance_class_data["class"])
schema = merge_dicts(dagster_instance_config_schema(),
custom_instance_class.config_schema())
else:
custom_instance_class = None
schema = dagster_instance_config_schema()
dagster_config = validate_config(schema, dagster_config_dict)
if not dagster_config.success:
raise DagsterInvalidConfigError(
"Errors whilst loading dagster instance config at {}.".format(
config_filename),
dagster_config.errors,
dagster_config_dict,
)
return (dagster_config.value, custom_instance_class)
def test_example_selector_success():
result = validate_config(ExampleSelector, {'option_one': 'foo'})
assert result.success
assert result.value == {'option_one': 'foo'}
result = validate_config(ExampleSelector, {'option_two': 'foo'})
assert result.success
assert result.value == {'option_two': 'foo'}
def test_example_selector_success():
result = validate_config(ExampleSelector, {"option_one": "foo"})
assert result.success
assert result.value == {"option_one": "foo"}
result = validate_config(ExampleSelector, {"option_two": "foo"})
assert result.success
assert result.value == {"option_two": "foo"}
def test_permissive_dict_with_fields():
perm_dict_with_field = Permissive({"a_key": Field(str)})
assert validate_config(perm_dict_with_field, {"a_key": "djfkdjkfd"}).success
assert validate_config(
perm_dict_with_field, {"a_key": "djfkdjkfd", "extra_key": "kdjkfd"}
).success
assert not validate_config(perm_dict_with_field, {"a_key": 2}).success
assert not validate_config(perm_dict_with_field, {}).success
def dagster_instance_config(
base_dir,
config_filename=DAGSTER_CONFIG_YAML_FILENAME,
overrides=None,
):
check.str_param(base_dir, "base_dir")
check.invariant(os.path.isdir(base_dir), "base_dir should be a directory")
overrides = check.opt_dict_param(overrides, "overrides")
config_yaml_path = os.path.join(base_dir, config_filename)
if not os.path.exists(config_yaml_path) and is_dagster_home_set():
warnings.warn(
f"No dagster instance configuration file ({config_filename}) found at "
f"{base_dir}. Defaulting to loading and storing all metadata with {base_dir}. "
f"If this is the desired behavior, create an empty {config_filename} file in {base_dir}."
)
dagster_config_dict = merge_dicts(load_yaml_from_globs(config_yaml_path),
overrides)
if "instance_class" in dagster_config_dict:
custom_instance_class_data = dagster_config_dict["instance_class"]
validate_custom_config = validate_config(
configurable_class_schema(),
custom_instance_class_data,
)
if not validate_custom_config.success:
raise DagsterInvalidConfigError(
"Errors whilst loading dagster custom class config at {}".
format(config_filename),
validate_custom_config.errors,
custom_instance_class_data,
)
custom_instance_class = class_from_code_pointer(
custom_instance_class_data["module"],
custom_instance_class_data["class"])
schema = merge_dicts(dagster_instance_config_schema(),
custom_instance_class.config_schema())
else:
custom_instance_class = None
schema = dagster_instance_config_schema()
dagster_config = validate_config(schema, dagster_config_dict)
if not dagster_config.success:
raise DagsterInvalidConfigError(
"Errors whilst loading dagster instance config at {}.".format(
config_filename),
dagster_config.errors,
dagster_config_dict,
)
return (dagster_config.value, custom_instance_class)
def test_nested_success():
value = {
"level_one": {
"string_field": "skdsjfkdj",
"int_field": 123,
"bool_field": True
}
}
assert validate_config(DoubleLevelShape, value).success
assert not validate_config(DoubleLevelShape, None).success
def test_nested_success():
value = {
'level_one': {
'string_field': 'skdsjfkdj',
'int_field': 123,
'bool_field': True
}
}
assert validate_config(DoubleLevelShape, value).success
assert not validate_config(DoubleLevelShape, None).success
def test_float_field():
config_field = convert_potential_field({"float_field": Float})
assert validate_config(config_field.config_type, {"float_field": 1.0}).value == {
"float_field": 1.0
}
assert process_config(config_field.config_type, {"float_field": 1.0}).value == {
"float_field": 1.0
}
assert validate_config(config_field.config_type, {"float_field": 1}).value == {"float_field": 1}
assert process_config(config_field.config_type, {"float_field": 1}).value == {
"float_field": 1.0
}
def test_permissive_dict_with_fields():
perm_dict_with_field = Permissive({'a_key': Field(str)})
assert validate_config(perm_dict_with_field, {
'a_key': 'djfkdjkfd'
}).success
assert validate_config(perm_dict_with_field, {
'a_key': 'djfkdjkfd',
'extra_key': 'kdjkfd'
}).success
assert not validate_config(perm_dict_with_field, {'a_key': 2}).success
assert not validate_config(perm_dict_with_field, {}).success
def test_config_double_list():
nested_lists = {'nested_list_one': [int], 'nested_list_two': [str]}
value = {'nested_list_one': [1, 2, 3], 'nested_list_two': ['foo', 'bar']}
result = validate_config(nested_lists, value)
assert result.success
assert result.value == value
error_value = {'nested_list_one': 'kjdfkdj', 'nested_list_two': ['bar']}
error_result = validate_config(nested_lists, error_value)
assert not error_result.success
def test_config_double_list():
nested_lists = {"nested_list_one": [int], "nested_list_two": [str]}
value = {"nested_list_one": [1, 2, 3], "nested_list_two": ["foo", "bar"]}
result = validate_config(nested_lists, value)
assert result.success
assert result.value == value
error_value = {"nested_list_one": "kjdfkdj", "nested_list_two": ["bar"]}
error_result = validate_config(nested_lists, error_value)
assert not error_result.success
def test_config_double_list_double_error():
nested_lists = {"nested_list_one": [int], "nested_list_two": [str]}
error_value = {"nested_list_one": "kjdfkdj", "nested_list_two": ["bar", 2]}
error_result = validate_config(nested_lists, error_value)
assert not error_result.success
assert len(error_result.errors) == 2
def test_config_double_list_double_error():
nested_lists = {'nested_list_one': [int], 'nested_list_two': [str]}
error_value = {'nested_list_one': 'kjdfkdj', 'nested_list_two': ['bar', 2]}
error_result = validate_config(nested_lists, error_value)
assert not error_result.success
assert len(error_result.errors) == 2
def test_config_list_in_dict():
nested_list_type = {'nested_list': [int]}
value = {'nested_list': [1, 2, 3]}
result = validate_config(nested_list_type, value)
assert result.success
assert result.value == value
def is_config_valid(pipeline_def, environment_dict, mode):
check.str_param(mode, 'mode')
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
environment_schema = create_environment_schema(pipeline_def, mode)
validated_config = validate_config(environment_schema.environment_type, environment_dict)
return validated_config.success
def test_example_selector_wrong_field():
result = validate_config(ExampleSelector, {'nope': 234})
assert not result.success
assert result.value is None
assert len(result.errors) == 1
assert result.errors[
0].reason == DagsterEvaluationErrorReason.FIELD_NOT_DEFINED
def test_example_selector_error_top_level_type():
result = validate_config(ExampleSelector, 'kjsdkf')
assert not result.success
assert result.value is None
assert len(result.errors) == 1
assert result.errors[
0].reason == DagsterEvaluationErrorReason.RUNTIME_TYPE_MISMATCH
def resolve_from_unvalidated_config(self, config: Any) -> Any:
"""Validates config against outer config schema, and calls mapping against validated config."""
receive_processed_config_values = check.opt_bool_param(
self.receive_processed_config_values,
"receive_processed_config_values",
default=True)
if receive_processed_config_values:
outer_evr = process_config(
self.config_schema.config_type,
config,
)
else:
outer_evr = validate_config(
self.config_schema.config_type,
config,
)
if not outer_evr.success:
raise DagsterInvalidConfigError(
"Error in config mapping ",
outer_evr.errors,
config,
)
outer_config = outer_evr.value
if not receive_processed_config_values:
outer_config = resolve_defaults(
cast(ConfigType, self.config_schema.config_type),
outer_config,
).value
return self.config_fn(outer_config)
def test_deep_scalar():
value = {
'level_one_string_field': 'foo',
'level_two_dict': {
'level_two_int_field': 234234,
'level_three_dict': {
'level_three_string': 123
},
},
}
result = validate_config(MultiLevelShapeType, value)
assert not result.success
assert len(result.errors) == 1
error = result.errors[0]
assert error.reason == DagsterEvaluationErrorReason.RUNTIME_TYPE_MISMATCH
assert error.error_data.config_type_snap.given_name == 'String'
assert error.error_data.value_rep == '123'
assert len(error.stack.entries) == 3
assert [entry.field_name for entry in error.stack.entries] == [
'level_two_dict',
'level_three_dict',
'level_three_string',
]
assert not result.errors_at_level('level_one_string_field')
assert not result.errors_at_level('level_two_dict')
assert not result.errors_at_level('level_two_dict', 'level_three_dict')
assert (len(
result.errors_at_level('level_two_dict', 'level_three_dict',
'level_three_string')) == 1)
def get_user_defined_k8s_config(tags):
check.inst_param(tags, "tags", frozentags)
if not any(key in tags for key in [K8S_RESOURCE_REQUIREMENTS_KEY, USER_DEFINED_K8S_CONFIG_KEY]):
return UserDefinedDagsterK8sConfig()
user_defined_k8s_config = {}
if USER_DEFINED_K8S_CONFIG_KEY in tags:
user_defined_k8s_config_value = json.loads(tags[USER_DEFINED_K8S_CONFIG_KEY])
result = validate_config(USER_DEFINED_K8S_CONFIG_SCHEMA, user_defined_k8s_config_value)
if not result.success:
raise DagsterInvalidConfigError(
"Error in tags for {}".format(USER_DEFINED_K8S_CONFIG_KEY), result.errors, result,
)
user_defined_k8s_config = result.value
container_config = user_defined_k8s_config.get("container_config", {})
# Backcompat for resource requirements key
if K8S_RESOURCE_REQUIREMENTS_KEY in tags:
resource_requirements_config = get_k8s_resource_requirements(tags)
container_config = merge_dicts(
container_config, {"resources": resource_requirements_config}
)
return UserDefinedDagsterK8sConfig(
container_config=container_config,
pod_template_spec_metadata=user_defined_k8s_config.get("pod_template_spec_metadata"),
pod_spec_config=user_defined_k8s_config.get("pod_spec_config"),
job_config=user_defined_k8s_config.get("job_config"),
job_spec_config=user_defined_k8s_config.get("job_spec_config"),
)
def is_config_valid(pipeline_def, run_config, mode):
check.str_param(mode, "mode")
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
run_config_schema = create_run_config_schema(pipeline_def, mode)
validated_config = validate_config(run_config_schema.environment_type, run_config)
return validated_config.success
def get_config_dir(config_yaml=None):
instance = DagsterInstance.get()
config_type = celery_executor.config_field.config_type
config_value = get_config_value_from_yaml(config_yaml)
config_module_name = 'dagster_celery_config'
config_dir = os.path.join(instance.root_directory, 'dagster_celery',
'config', str(uuid.uuid4()))
mkdir_p(config_dir)
config_path = os.path.join(
config_dir, '{config_module_name}.py'.format(
config_module_name=config_module_name))
validated_config = validate_config(config_type, config_value).value
with open(config_path, 'w') as fd:
if 'broker' in validated_config:
fd.write('broker_url = \'{broker_url}\'\n'.format(
broker_url=str(validated_config['broker'])))
if 'backend' in validated_config:
fd.write('result_backend = \'{result_backend}\'\n'.format(
result_backend=str(validated_config['backend'])))
if 'config_source' in validated_config:
for key, value in validated_config['config_source'].items():
fd.write('{key} = {value}\n'.format(key=key,
value=repr(value)))
# n.b. right now we don't attempt to clean up this cache, but it might make sense to delete
# any files older than some time if there are more than some number of files present, etc.
return config_dir
def test_deep_scalar():
value = {
"level_one_string_field": "foo",
"level_two_dict": {
"level_two_int_field": 234234,
"level_three_dict": {
"level_three_string": 123
},
},
}
result = validate_config(MultiLevelShapeType, value)
assert not result.success
assert len(result.errors) == 1
error = result.errors[0]
assert error.reason == DagsterEvaluationErrorReason.RUNTIME_TYPE_MISMATCH
assert error.error_data.config_type_snap.given_name == "String"
assert error.error_data.value_rep == "123"
assert len(error.stack.entries) == 3
assert [entry.field_name for entry in error.stack.entries] == [
"level_two_dict",
"level_three_dict",
"level_three_string",
]
assert not result.errors_at_level("level_one_string_field")
assert not result.errors_at_level("level_two_dict")
assert not result.errors_at_level("level_two_dict", "level_three_dict")
assert (len(
result.errors_at_level("level_two_dict", "level_three_dict",
"level_three_string")) == 1)
def resolve_is_environment_config_valid(graphene_info, environment_schema,
dagster_pipeline, environment_dict):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(environment_schema, 'environment_schema',
EnvironmentSchema)
check.inst_param(dagster_pipeline, 'dagster_pipeline', PipelineDefinition)
check.dict_param(environment_dict, 'environment_dict', key_type=str)
validated_config = validate_config(environment_schema.environment_type,
environment_dict)
dauphin_pipeline = graphene_info.schema.type_named('Pipeline')(
dagster_pipeline)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=dauphin_pipeline,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
graphene_info, err)
for err in validated_config.errors
],
))
return graphene_info.schema.type_named('PipelineConfigValidationValid')(
dauphin_pipeline)
def test_int_field():
config_field = convert_potential_field({'int_field': Int})
assert validate_config(config_field.config_type, {
'int_field': 1
}).value == {
'int_field': 1
}
def _check_invocation_requirements(
solid_def: "SolidDefinition", context: Optional["DirectSolidExecutionContext"]
) -> "DirectSolidExecutionContext":
"""Ensure that provided context fulfills requirements of solid definition.
If no context was provided, then construct an enpty DirectSolidExecutionContext
"""
from dagster.core.execution.context.invocation import DirectSolidExecutionContext
from dagster.config.validate import validate_config
# Check resource requirements
if solid_def.required_resource_keys and context is None:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" has required resources, but no context was provided. Use the'
"`build_solid_context` function to construct a context with the required "
"resources."
)
if context is not None and solid_def.required_resource_keys:
resources_dict = cast( # type: ignore[attr-defined]
"DirectSolidExecutionContext",
context,
).resources._asdict()
for resource_key in solid_def.required_resource_keys:
if resource_key not in resources_dict:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" requires resource "{resource_key}", but no resource '
"with that key was found on the context."
)
# Check config requirements
if not context and solid_def.config_schema.as_field().is_required:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" has required config schema, but no context was provided. '
"Use the `build_solid_context` function to create a context with config."
)
config = None
if solid_def.config_field:
solid_config = check.opt_dict_param(
context.solid_config if context else None, "solid_config"
)
config_evr = validate_config(solid_def.config_field.config_type, solid_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for solid ", config_evr.errors, solid_config
)
mapped_config_evr = solid_def.apply_config_mapping({"config": solid_config})
if not mapped_config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for solid ", mapped_config_evr.errors, solid_config
)
config = mapped_config_evr.value.get("config", {})
return (
context
if context
else DirectSolidExecutionContext(solid_config=config, resources_dict=None, instance=None)
)
def test_shape_with_field_substitutions_collisions():
value = {"foo_field": {}, "bar_field": "world", "foo": {}}
result = validate_config(FieldSubShape, value)
assert not result.success
assert len(result.errors) == 1
collision_error = result.errors[0]
assert collision_error.reason == DagsterEvaluationErrorReason.FIELD_ALIAS_COLLISION
def test_selector_within_dict_no_subfields():
result = validate_config(Shape({'selector': Field(ExampleSelector)}),
{'selector': {}})
assert not result.success
assert len(result.errors) == 1
assert result.errors[0].message == (
"Must specify a field at path root:selector if more than one field "
"is defined. Defined fields: ['option_one', 'option_two']")
