def test_int_source():
assert process_config(IntSource, 1).success
assert not process_config(IntSource, "foo").success
assert not process_config(IntSource, {"env": 1}).success
assert "DAGSTER_TEST_ENV_VAR" not in os.environ
assert not process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).success
assert (
'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" '
"which is not set. In order for this execution to succeed it must be set in "
"this environment."
) in process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).errors[0].message
with environ({"DAGSTER_TEST_ENV_VAR": "4"}):
assert process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).success
assert process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).value == 4
with environ({"DAGSTER_TEST_ENV_VAR": "four"}):
assert not process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).success
assert (
'Value "four" stored in env variable "DAGSTER_TEST_ENV_VAR" cannot '
"be coerced into an int."
) in process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).errors[0].message
def test_int_source():
assert process_config(IntSource, 1).success
assert not process_config(IntSource, 'foo').success
assert not process_config(IntSource, {'env': 1}).success
assert 'DAGSTER_TEST_ENV_VAR' not in os.environ
assert not process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).success
assert (
'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" '
'which is not set. In order for this execution to succeed it must be set in '
'this environment.'
) in process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).errors[0].message
with environ({'DAGSTER_TEST_ENV_VAR': '4'}):
assert process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).success
assert process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).value == 4
with environ({'DAGSTER_TEST_ENV_VAR': 'four'}):
assert not process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).success
assert (
'Value "four" stored in env variable "DAGSTER_TEST_ENV_VAR" cannot '
'be coerced into an int.'
) in process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).errors[0].message
def test_int_source():
assert process_config(IntSource, 1).success
assert not process_config(IntSource, 'foo').success
assert not process_config(IntSource, {'env': 1}).success
assert 'DAGSTER_TEST_ENV_VAR' not in os.environ
assert not process_config(IntSource, {
'env': 'DAGSTER_TEST_ENV_VAR'
}).success
assert ('Environment variable "DAGSTER_TEST_ENV_VAR" is not set'
in process_config(IntSource, {
'env': 'DAGSTER_TEST_ENV_VAR'
}).errors[0].message)
with environ({'DAGSTER_TEST_ENV_VAR': '4'}):
assert process_config(IntSource, {
'env': 'DAGSTER_TEST_ENV_VAR'
}).success
assert process_config(IntSource, {
'env': 'DAGSTER_TEST_ENV_VAR'
}).value == 4
with environ({'DAGSTER_TEST_ENV_VAR': 'four'}):
assert not process_config(IntSource, {
'env': 'DAGSTER_TEST_ENV_VAR'
}).success
assert ("invalid literal for int() with base 10: 'four'"
in process_config(IntSource, {
'env': 'DAGSTER_TEST_ENV_VAR'
}).errors[0].message)
def test_float_field():
config_field = convert_potential_field({"float_field": Float})
assert validate_config(config_field.config_type, {"float_field": 1.0}).value == {
"float_field": 1.0
}
assert process_config(config_field.config_type, {"float_field": 1.0}).value == {
"float_field": 1.0
}
assert validate_config(config_field.config_type, {"float_field": 1}).value == {"float_field": 1}
assert process_config(config_field.config_type, {"float_field": 1}).value == {
"float_field": 1.0
}
def test_solid_config_error():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
pipeline_def.solids, pipeline_def.dependency_structure,
)
int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type
res = process_config(int_solid_config_type, {'notconfig': 1})
assert not res.success
assert re.match('Undefined field "notconfig"', res.errors[0].message)
res = process_config(int_solid_config_type, 1)
assert not res.success
def build(pipeline_def, run_config=None, mode=None):
'''This method validates a given run config against the pipeline config schema. If
successful, we instantiate an EnvironmentConfig object.
In case the run_config is invalid, this method raises a DagsterInvalidConfigError
'''
from dagster.config.validate import process_config
from .composite_descent import composite_descent
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
run_config = check.opt_dict_param(run_config, 'run_config')
check.opt_str_param(mode, 'mode')
mode = mode or pipeline_def.get_default_mode_name()
environment_type = create_environment_type(pipeline_def, mode)
config_evr = process_config(environment_type, run_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
'Error in config for pipeline {}'.format(pipeline_def.name),
config_evr.errors,
run_config,
)
config_value = config_evr.value
mode_def = pipeline_def.get_mode_definition(mode)
resource_configs = config_value.get('resources', {})
processed_resource_configs = {}
for resource_key, resource_def in mode_def.resource_defs.items():
resource_config = resource_configs.get(resource_key, {})
resource_config_evr = resource_def.process_config(resource_config)
if not resource_config_evr.success:
raise DagsterInvalidConfigError(
'Error in config for resource {}'.format(resource_key),
resource_config_evr.errors,
resource_config,
)
else:
processed_resource_configs[
resource_key] = resource_config_evr.value
solid_config_dict = composite_descent(pipeline_def,
config_value.get('solids', {}))
# TODO: replace this with a simple call to from_dict of the config.get when ready to fully deprecate
temp_intermed = config_value.get('intermediate_storage')
if config_value.get('storage'):
if temp_intermed is None:
temp_intermed = {EmptyIntermediateStoreBackcompatConfig(): {}}
return EnvironmentConfig(
solids=solid_config_dict,
execution=ExecutionConfig.from_dict(config_value.get('execution')),
storage=StorageConfig.from_dict(config_value.get('storage')),
intermediate_storage=IntermediateStorageConfig.from_dict(
temp_intermed),
loggers=config_value.get('loggers'),
original_config_dict=run_config,
resources=processed_resource_configs,
)
def _resolve_bound_config(solid_config: Any,
solid_def: SolidDefinition) -> Any:
"""Validate config against config schema, and return validated config."""
from dagster.config.validate import process_config
# Config processing system expects the top level config schema to be a dictionary, but solid
# config schema can be scalar. Thus, we wrap it in another layer of indirection.
outer_config_shape = Shape({"config": solid_def.get_config_field()})
config_evr = process_config(
outer_config_shape, {"config": solid_config} if solid_config else {})
if not config_evr.success:
raise DagsterInvalidConfigError(
f"Error in config for {solid_def.node_type_str} ",
config_evr.errors,
solid_config,
)
validated_config = cast(Dict, config_evr.value).get("config")
mapped_config_evr = solid_def.apply_config_mapping(
{"config": validated_config})
if not mapped_config_evr.success:
raise DagsterInvalidConfigError(
f"Error in config for {solid_def.node_type_str} ",
mapped_config_evr.errors,
solid_config,
)
validated_config = cast(Dict, mapped_config_evr.value).get("config")
return validated_config
def _get_host_mode_executor(recon_pipeline, run_config, get_executor_def_fn,
instance):
execution_config = run_config.get("execution")
if execution_config:
executor_name, executor_config = ensure_single_item(execution_config)
else:
executor_name = None
executor_config = {}
executor_def = get_executor_def_fn(executor_name)
executor_config_type = def_config_field(executor_def).config_type
config_evr = process_config(executor_config_type, executor_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error in executor config for executor {}".format(
executor_def.name),
config_evr.errors,
executor_config,
)
executor_config_value = config_evr.value
init_context = InitExecutorContext(
pipeline=recon_pipeline,
executor_def=executor_def,
executor_config=executor_config_value["config"],
instance=instance,
)
check_cross_process_constraints(init_context)
return executor_def.executor_creation_fn(init_context)
def rehydrate(self):
from dagster.config.field import resolve_to_config_type
from dagster.config.validate import process_config
from dagster.core.errors import DagsterInvalidConfigError
try:
module = importlib.import_module(self.module_name)
except ModuleNotFoundError:
check.failed(
f"Couldn't import module {self.module_name} when attempting to load the "
f"configurable class {self.module_name}.{self.class_name}")
try:
klass = getattr(module, self.class_name)
except AttributeError:
check.failed(
f"Couldn't find class {self.class_name} in module when attempting to load the "
f"configurable class {self.module_name}.{self.class_name}")
if not issubclass(klass, ConfigurableClass):
raise check.CheckError(
klass,
f"class {self.class_name} in module {self.module_name}",
ConfigurableClass,
)
config_dict = self.config_dict
result = process_config(resolve_to_config_type(klass.config_type()),
config_dict)
if not result.success:
raise DagsterInvalidConfigError(
f"Errors whilst loading configuration for {klass.config_type()}.",
result.errors,
config_dict,
)
return klass.from_config_value(self, result.value)
def _get_validated_celery_k8s_executor_config(environment_dict):
check.dict_param(environment_dict, 'environment_dict')
try:
from dagster_celery.executor_k8s import CELERY_K8S_CONFIG_KEY, celery_k8s_config
except ImportError:
raise DagsterK8sError(
'To use the CeleryK8sRunLauncher, dagster_celery must be'
' installed in your Python environment.'
)
check.invariant(
CELERY_K8S_CONFIG_KEY in environment_dict.get('execution', {}),
'{} execution must be configured in pipeline execution config to launch runs with '
'CeleryK8sRunLauncher'.format(CELERY_K8S_CONFIG_KEY),
)
execution_config_schema = resolve_to_config_type(celery_k8s_config())
execution_run_config = environment_dict['execution'][CELERY_K8S_CONFIG_KEY].get('config', {})
res = process_config(execution_config_schema, execution_run_config)
check.invariant(
res.success, 'Incorrect {} execution schema provided'.format(CELERY_K8S_CONFIG_KEY)
)
return res.value
def create_from_config(run_container_context):
run_docker_container_context = (run_container_context.get(
"docker", {}) if run_container_context else {})
if not run_docker_container_context:
return DockerContainerContext()
processed_container_context = process_config(
DOCKER_CONTAINER_CONTEXT_SCHEMA, run_docker_container_context)
if not processed_container_context.success:
raise DagsterInvalidConfigError(
"Errors while parsing Docker container context",
processed_container_context.errors,
run_docker_container_context,
)
processed_context_value = processed_container_context.value
return DockerContainerContext(
registry=processed_context_value.get("registry"),
env_vars=processed_context_value.get("env_vars", []),
networks=processed_context_value.get("networks", []),
container_kwargs=processed_context_value.get("container_kwargs"),
)
def _get_validated_celery_k8s_executor_config(run_config):
check.dict_param(run_config, "run_config")
executor_config = run_config.get("execution", {})
execution_config_schema = resolve_to_config_type(celery_k8s_config())
# In run config on jobs, we don't have an executor key
if not CELERY_K8S_CONFIG_KEY in executor_config:
execution_run_config = executor_config.get("config", {})
else:
execution_run_config = (run_config["execution"][CELERY_K8S_CONFIG_KEY]
or {}).get("config", {})
res = process_config(execution_config_schema, execution_run_config)
check.invariant(
res.success,
"Incorrect execution schema provided. Note: You may also be seeing this error "
"because you are using the configured API. "
"Using configured with the {config_key} executor is not supported at this time, "
"and all executor config must be directly in the run config without using configured."
.format(config_key=CELERY_K8S_CONFIG_KEY, ),
)
return res.value
def _get_host_mode_executor(recon_pipeline, run_config, executor_defs,
instance):
execution_config = run_config.get("execution", {})
execution_config_type = Field(selector_for_named_defs(executor_defs),
default_value={
executor_defs[0].name: {}
}).config_type
config_evr = process_config(execution_config_type, execution_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error processing execution config {}".format(execution_config),
config_evr.errors,
execution_config,
)
execution_config_value = config_evr.value
executor_name, executor_config = ensure_single_item(execution_config_value)
executor_defs_by_name = {
executor_def.name: executor_def
for executor_def in executor_defs
}
executor_def = executor_defs_by_name[executor_name]
init_context = InitExecutorContext(
job=recon_pipeline,
executor_def=executor_def,
executor_config=executor_config["config"],
instance=instance,
)
check_cross_process_constraints(init_context)
return executor_def.executor_creation_fn(init_context)
def test_noneable_string_source_array():
assert process_config(Noneable(Array(StringSource)), []).success
assert process_config(Noneable(Array(StringSource)), None).success
assert (
'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" '
"which is not set. In order for this execution to succeed it must be set in "
"this environment.") in process_config(Noneable(
Array(StringSource)), ["test", {
"env": "DAGSTER_TEST_ENV_VAR"
}]).errors[0].message
with environ({"DAGSTER_TEST_ENV_VAR": "baz"}):
assert process_config(Noneable(Array(StringSource)),
["test", {
"env": "DAGSTER_TEST_ENV_VAR"
}]).success
def _get_mapped_solids_dict(composite_def, current_stack,
current_solid_config):
# the spec of the config mapping function is that it takes the dictionary at:
# solid_name:
# config: {dict_passed_to_user}
# and it returns the dictionary rooted at solids
# solid_name:
# solids: {return_value_of_config_fn}
# We must call the config mapping function and then validate it against
# the child schema.
with user_code_error_boundary(DagsterConfigMappingFunctionError,
_get_error_lambda(current_stack)):
mapped_solids_config = composite_def.config_mapping.config_fn(
current_solid_config.get('config', {}))
# Dynamically construct the type that the output of the config mapping function will
# be evaluated against
type_to_evaluate_against = define_solid_dictionary_cls(
composite_def.solids, composite_def.dependency_structure,
current_stack.handle)
# process against that new type
evr = process_config(type_to_evaluate_against, mapped_solids_config)
if not evr.success:
raise_composite_descent_config_error(current_stack,
mapped_solids_config, evr)
return evr.value
def resolve_from_unvalidated_config(self, config: Any) -> Any:
"""Validates config against outer config schema, and calls mapping against validated config."""
receive_processed_config_values = check.opt_bool_param(
self.receive_processed_config_values,
"receive_processed_config_values",
default=True)
if receive_processed_config_values:
outer_evr = process_config(
self.config_schema.config_type,
config,
)
else:
outer_evr = validate_config(
self.config_schema.config_type,
config,
)
if not outer_evr.success:
raise DagsterInvalidConfigError(
"Error in config mapping ",
outer_evr.errors,
config,
)
outer_config = outer_evr.value
if not receive_processed_config_values:
outer_config = resolve_defaults(
cast(ConfigType, self.config_schema.config_type),
outer_config,
).value
return self.config_fn(outer_config)
def create_from_config(run_container_context) -> "K8sContainerContext":
run_k8s_container_context = (run_container_context.get("k8s", {})
if run_container_context else {})
if not run_k8s_container_context:
return K8sContainerContext()
processed_container_context = process_config(
DagsterK8sJobConfig.config_type_container_context(),
run_k8s_container_context)
if not processed_container_context.success:
raise DagsterInvalidConfigError(
"Errors while parsing k8s container context",
processed_container_context.errors,
run_k8s_container_context,
)
processed_context_value = processed_container_context.value
return K8sContainerContext(
image_pull_policy=processed_context_value.get("image_pull_policy"),
image_pull_secrets=processed_context_value.get(
"image_pull_secrets"),
service_account_name=processed_context_value.get(
"service_account_name"),
env_config_maps=processed_context_value.get("env_config_maps"),
env_secrets=processed_context_value.get("env_secrets"),
env_vars=processed_context_value.get("env_vars"),
volume_mounts=processed_context_value.get("volume_mounts"),
volumes=processed_context_value.get("volumes"),
labels=processed_context_value.get("labels"),
namespace=processed_context_value.get("namespace"),
)
def test_required_solid_with_required_subfield():
pipeline_def = PipelineDefinition(
name="some_pipeline",
solid_defs=[
SolidDefinition(
name="int_config_solid",
config_schema={"required_field": String},
input_defs=[],
output_defs=[],
compute_fn=lambda *_args: None,
)
],
)
env_type = create_run_config_schema_type(pipeline_def)
assert env_type.fields["solids"].is_required is True
assert env_type.fields["solids"].config_type
solids_type = env_type.fields["solids"].config_type
assert solids_type.fields["int_config_solid"].is_required is True
int_config_solid_type = solids_type.fields["int_config_solid"].config_type
assert int_config_solid_type.fields["config"].is_required is True
assert env_type.fields["execution"].is_required is False
env_obj = ResolvedRunConfig.build(
pipeline_def,
{
"solids": {
"int_config_solid": {
"config": {
"required_field": "foobar"
}
}
}
},
)
assert env_obj.solids["int_config_solid"].config[
"required_field"] == "foobar"
res = process_config(env_type, {"solids": {}})
assert not res.success
res = process_config(env_type, {})
assert not res.success
def test_required_solid_with_required_subfield():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_schema={'required_field': String},
input_defs=[],
output_defs=[],
compute_fn=lambda *_args: None,
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_required is True
assert env_type.fields['solids'].config_type
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_required is True
int_config_solid_type = solids_type.fields['int_config_solid'].config_type
assert int_config_solid_type.fields['config'].is_required is True
assert env_type.fields['execution'].is_required is False
env_obj = EnvironmentConfig.build(
pipeline_def,
{
'solids': {
'int_config_solid': {
'config': {
'required_field': 'foobar'
}
}
}
},
)
assert env_obj.solids['int_config_solid'].config[
'required_field'] == 'foobar'
res = process_config(env_type, {'solids': {}})
assert not res.success
res = process_config(env_type, {})
assert not res.success
def wrapped_process_config_fn(config):
config_evr = process_config({'config': config_schema or {}},
config)
if config_evr.success:
return self.process_config(
{'config': config_fn(config_evr.value['config'])})
else:
return config_evr
def test_noneable_string_source_array():
assert process_config(Noneable(Array(StringSource)), []).success
assert process_config(Noneable(Array(StringSource)), None).success
assert ((
'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" '
'which is not set. In order for this execution to succeed it must be set in '
'this environment.')
in process_config(Noneable(Array(StringSource)),
['test', {
'env': 'DAGSTER_TEST_ENV_VAR'
}]).errors[0].message)
with environ({'DAGSTER_TEST_ENV_VAR': 'baz'}):
assert process_config(Noneable(Array(StringSource)),
['test', {
'env': 'DAGSTER_TEST_ENV_VAR'
}]).success
def test_string_source():
assert process_config(StringSource, "foo").success
assert not process_config(StringSource, 1).success
assert not process_config(StringSource, {"env": 1}).success
assert "DAGSTER_TEST_ENV_VAR" not in os.environ
assert not process_config(StringSource, {
"env": "DAGSTER_TEST_ENV_VAR"
}).success
assert (
'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" '
"which is not set. In order for this execution to succeed it must be set in "
"this environment.") in process_config(StringSource, {
"env": "DAGSTER_TEST_ENV_VAR"
}).errors[0].message
with environ({"DAGSTER_TEST_ENV_VAR": "baz"}):
assert process_config(StringSource, {
"env": "DAGSTER_TEST_ENV_VAR"
}).success
assert process_config(StringSource, {
"env": "DAGSTER_TEST_ENV_VAR"
}).value == "baz"
def test_string_source():
assert process_config(StringSource, 'foo').success
assert not process_config(StringSource, 1).success
assert not process_config(StringSource, {'env': 1}).success
assert 'DAGSTER_TEST_ENV_VAR' not in os.environ
assert not process_config(StringSource, {
'env': 'DAGSTER_TEST_ENV_VAR'
}).success
assert (
'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" '
'which is not set. In order for this execution to succeed it must be set in '
'this environment.') in process_config(StringSource, {
'env': 'DAGSTER_TEST_ENV_VAR'
}).errors[0].message
with environ({'DAGSTER_TEST_ENV_VAR': 'baz'}):
assert process_config(StringSource, {
'env': 'DAGSTER_TEST_ENV_VAR'
}).success
assert process_config(StringSource, {
'env': 'DAGSTER_TEST_ENV_VAR'
}).value == 'baz'
def test_solid_config_error():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
solids=pipeline_def.solids,
ignored_solids=None,
dependency_structure=pipeline_def.dependency_structure,
parent_handle=None,
resource_defs={},
)
int_solid_config_type = solid_dict_type.fields["int_config_solid"].config_type
res = process_config(int_solid_config_type, {"notconfig": 1})
assert not res.success
assert re.match('Received unexpected config entry "notconfig"', res.errors[0].message)
res = process_config(int_solid_config_type, 1)
assert not res.success
def _get_mapped_solids_dict(
composite,
graph_def,
current_stack,
current_solid_config,
resource_defs,
is_using_graph_job_op_apis,
):
# the spec of the config mapping function is that it takes the dictionary at:
# solid_name:
# config: {dict_passed_to_user}
# and it returns the dictionary rooted at solids
# solid_name:
# solids: {return_value_of_config_fn}
# We must call the config mapping function and then validate it against
# the child schema.
# apply @configured config mapping to the composite's incoming config before we get to the
# composite's own config mapping process
config_mapped_solid_config = graph_def.apply_config_mapping(
current_solid_config)
if not config_mapped_solid_config.success:
raise DagsterInvalidConfigError(
"Error in config for composite solid {}".format(composite.name),
config_mapped_solid_config.errors,
config_mapped_solid_config,
)
with user_code_error_boundary(DagsterConfigMappingFunctionError,
_get_error_lambda(current_stack)):
mapped_solids_config = graph_def.config_mapping.resolve_from_validated_config(
config_mapped_solid_config.value.get("config", {}))
# Dynamically construct the type that the output of the config mapping function will
# be evaluated against
type_to_evaluate_against = define_solid_dictionary_cls(
solids=graph_def.solids,
ignored_solids=None,
dependency_structure=graph_def.dependency_structure,
parent_handle=current_stack.handle,
resource_defs=resource_defs,
is_using_graph_job_op_apis=is_using_graph_job_op_apis,
)
# process against that new type
evr = process_config(type_to_evaluate_against, mapped_solids_config)
if not evr.success:
raise_composite_descent_config_error(current_stack,
mapped_solids_config, evr)
return evr.value
def test_storage_in_memory_config():
pipeline_def = PipelineDefinition(name='pipeline', solid_defs=[])
env_type = create_environment_type(pipeline_def)
assert 'storage' in env_type.fields
config_value = process_config(env_type, {'storage': {'in_memory': {}}})
assert config_value.success
assert config_value.value['storage'] == {'in_memory': {}}
def test_storage_in_memory_config():
pipeline_def = PipelineDefinition(name="pipeline", solid_defs=[])
env_type = create_environment_type(pipeline_def)
assert "storage" in env_type.fields
config_value = process_config(env_type, {"intermediate_storage": {"in_memory": {}}})
assert config_value.success
assert config_value.value["intermediate_storage"] == {"in_memory": {}}
def test_files_default_config():
pipeline_def = PipelineDefinition(name="pipeline", solid_defs=[])
env_type = create_environment_type(pipeline_def)
assert "storage" in env_type.fields
config_value = process_config(env_type, {})
assert config_value.success
assert "storage" not in config_value
def resolve_config(self, processed_config: Dict[str, Any]) -> EvaluateValueResult:
check.dict_param(processed_config, "processed_config")
# Validate resolved config against the inner definitions's config_schema (on self).
config_evr = process_config(
{"config": self.parent_def.config_field or {}},
self._invoke_user_config_fn(processed_config),
)
if config_evr.success:
return self.parent_def.apply_config_mapping(config_evr.value) # Recursive step
else:
return config_evr # Bubble up the errors
def test_bad_config():
configs_and_expected_errors = [
(
# Create disposition must match enum values
{"create_disposition": "this is not a valid create disposition"},
"Value at path root:ops:test:config:query_job_config:create_disposition not in enum type BQCreateDisposition",
),
(
# Priority must match enum values
{"priority": "this is not a valid priority"},
"Value at path root:ops:test:config:query_job_config:priority not in enum type BQPriority got this is not a valid priority",
),
(
# Schema update options must be a list
{"schema_update_options": "this is not valid schema update options"},
'Value at path root:ops:test:config:query_job_config:schema_update_options must be list. Expected: "[BQSchemaUpdateOption]"',
),
(
{"schema_update_options": ["this is not valid schema update options"]},
"Value at path root:ops:test:config:query_job_config:schema_update_options[0] not in enum type BQSchemaUpdateOption",
),
(
{"write_disposition": "this is not a valid write disposition"},
"Value at path root:ops:test:config:query_job_config:write_disposition not in enum type BQWriteDisposition",
),
]
@job(resource_defs={"bigquery": bigquery_resource})
def test_config():
bq_op_for_queries(["SELECT 1"]).alias("test")()
env_type = create_run_config_schema(test_config).config_type
for config_fragment, error_message in configs_and_expected_errors:
config = {"ops": {"test": {"config": {"query_job_config": config_fragment}}}}
result = validate_config(env_type, config)
assert error_message in result.errors[0].message
configs_and_expected_validation_errors = [
(
{"default_dataset": "this is not a valid dataset"},
"Datasets must be of the form", # project_name.dataset_name',
),
(
{"destination": "this is not a valid table"},
"Tables must be of the form", # project_name.dataset_name.table_name'
),
]
for config_fragment, error_message in configs_and_expected_validation_errors:
config = {"ops": {"test": {"config": {"query_job_config": config_fragment}}}}
result = process_config(env_type, config)
assert error_message in result.errors[0].message
