def test_int_source(): assert process_config(IntSource, 1).success assert not process_config(IntSource, "foo").success assert not process_config(IntSource, {"env": 1}).success assert "DAGSTER_TEST_ENV_VAR" not in os.environ assert not process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).success assert ( 'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" ' "which is not set. In order for this execution to succeed it must be set in " "this environment." ) in process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).errors[0].message with environ({"DAGSTER_TEST_ENV_VAR": "4"}): assert process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).success assert process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).value == 4 with environ({"DAGSTER_TEST_ENV_VAR": "four"}): assert not process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).success assert ( 'Value "four" stored in env variable "DAGSTER_TEST_ENV_VAR" cannot ' "be coerced into an int." ) in process_config(IntSource, {"env": "DAGSTER_TEST_ENV_VAR"}).errors[0].message
def test_int_source(): assert process_config(IntSource, 1).success assert not process_config(IntSource, 'foo').success assert not process_config(IntSource, {'env': 1}).success assert 'DAGSTER_TEST_ENV_VAR' not in os.environ assert not process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).success assert ( 'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" ' 'which is not set. In order for this execution to succeed it must be set in ' 'this environment.' ) in process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).errors[0].message with environ({'DAGSTER_TEST_ENV_VAR': '4'}): assert process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).success assert process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).value == 4 with environ({'DAGSTER_TEST_ENV_VAR': 'four'}): assert not process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).success assert ( 'Value "four" stored in env variable "DAGSTER_TEST_ENV_VAR" cannot ' 'be coerced into an int.' ) in process_config(IntSource, {'env': 'DAGSTER_TEST_ENV_VAR'}).errors[0].message
def test_int_source(): assert process_config(IntSource, 1).success assert not process_config(IntSource, 'foo').success assert not process_config(IntSource, {'env': 1}).success assert 'DAGSTER_TEST_ENV_VAR' not in os.environ assert not process_config(IntSource, { 'env': 'DAGSTER_TEST_ENV_VAR' }).success assert ('Environment variable "DAGSTER_TEST_ENV_VAR" is not set' in process_config(IntSource, { 'env': 'DAGSTER_TEST_ENV_VAR' }).errors[0].message) with environ({'DAGSTER_TEST_ENV_VAR': '4'}): assert process_config(IntSource, { 'env': 'DAGSTER_TEST_ENV_VAR' }).success assert process_config(IntSource, { 'env': 'DAGSTER_TEST_ENV_VAR' }).value == 4 with environ({'DAGSTER_TEST_ENV_VAR': 'four'}): assert not process_config(IntSource, { 'env': 'DAGSTER_TEST_ENV_VAR' }).success assert ("invalid literal for int() with base 10: 'four'" in process_config(IntSource, { 'env': 'DAGSTER_TEST_ENV_VAR' }).errors[0].message)
def test_float_field(): config_field = convert_potential_field({"float_field": Float}) assert validate_config(config_field.config_type, {"float_field": 1.0}).value == { "float_field": 1.0 } assert process_config(config_field.config_type, {"float_field": 1.0}).value == { "float_field": 1.0 } assert validate_config(config_field.config_type, {"float_field": 1}).value == {"float_field": 1} assert process_config(config_field.config_type, {"float_field": 1}).value == { "float_field": 1.0 }
def test_solid_config_error(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( pipeline_def.solids, pipeline_def.dependency_structure, ) int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type res = process_config(int_solid_config_type, {'notconfig': 1}) assert not res.success assert re.match('Undefined field "notconfig"', res.errors[0].message) res = process_config(int_solid_config_type, 1) assert not res.success
def build(pipeline_def, run_config=None, mode=None): '''This method validates a given run config against the pipeline config schema. If successful, we instantiate an EnvironmentConfig object. In case the run_config is invalid, this method raises a DagsterInvalidConfigError ''' from dagster.config.validate import process_config from .composite_descent import composite_descent check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) run_config = check.opt_dict_param(run_config, 'run_config') check.opt_str_param(mode, 'mode') mode = mode or pipeline_def.get_default_mode_name() environment_type = create_environment_type(pipeline_def, mode) config_evr = process_config(environment_type, run_config) if not config_evr.success: raise DagsterInvalidConfigError( 'Error in config for pipeline {}'.format(pipeline_def.name), config_evr.errors, run_config, ) config_value = config_evr.value mode_def = pipeline_def.get_mode_definition(mode) resource_configs = config_value.get('resources', {}) processed_resource_configs = {} for resource_key, resource_def in mode_def.resource_defs.items(): resource_config = resource_configs.get(resource_key, {}) resource_config_evr = resource_def.process_config(resource_config) if not resource_config_evr.success: raise DagsterInvalidConfigError( 'Error in config for resource {}'.format(resource_key), resource_config_evr.errors, resource_config, ) else: processed_resource_configs[ resource_key] = resource_config_evr.value solid_config_dict = composite_descent(pipeline_def, config_value.get('solids', {})) # TODO: replace this with a simple call to from_dict of the config.get when ready to fully deprecate temp_intermed = config_value.get('intermediate_storage') if config_value.get('storage'): if temp_intermed is None: temp_intermed = {EmptyIntermediateStoreBackcompatConfig(): {}} return EnvironmentConfig( solids=solid_config_dict, execution=ExecutionConfig.from_dict(config_value.get('execution')), storage=StorageConfig.from_dict(config_value.get('storage')), intermediate_storage=IntermediateStorageConfig.from_dict( temp_intermed), loggers=config_value.get('loggers'), original_config_dict=run_config, resources=processed_resource_configs, )
def _resolve_bound_config(solid_config: Any, solid_def: SolidDefinition) -> Any: """Validate config against config schema, and return validated config.""" from dagster.config.validate import process_config # Config processing system expects the top level config schema to be a dictionary, but solid # config schema can be scalar. Thus, we wrap it in another layer of indirection. outer_config_shape = Shape({"config": solid_def.get_config_field()}) config_evr = process_config( outer_config_shape, {"config": solid_config} if solid_config else {}) if not config_evr.success: raise DagsterInvalidConfigError( f"Error in config for {solid_def.node_type_str} ", config_evr.errors, solid_config, ) validated_config = cast(Dict, config_evr.value).get("config") mapped_config_evr = solid_def.apply_config_mapping( {"config": validated_config}) if not mapped_config_evr.success: raise DagsterInvalidConfigError( f"Error in config for {solid_def.node_type_str} ", mapped_config_evr.errors, solid_config, ) validated_config = cast(Dict, mapped_config_evr.value).get("config") return validated_config
def _get_host_mode_executor(recon_pipeline, run_config, get_executor_def_fn, instance): execution_config = run_config.get("execution") if execution_config: executor_name, executor_config = ensure_single_item(execution_config) else: executor_name = None executor_config = {} executor_def = get_executor_def_fn(executor_name) executor_config_type = def_config_field(executor_def).config_type config_evr = process_config(executor_config_type, executor_config) if not config_evr.success: raise DagsterInvalidConfigError( "Error in executor config for executor {}".format( executor_def.name), config_evr.errors, executor_config, ) executor_config_value = config_evr.value init_context = InitExecutorContext( pipeline=recon_pipeline, executor_def=executor_def, executor_config=executor_config_value["config"], instance=instance, ) check_cross_process_constraints(init_context) return executor_def.executor_creation_fn(init_context)
def rehydrate(self): from dagster.config.field import resolve_to_config_type from dagster.config.validate import process_config from dagster.core.errors import DagsterInvalidConfigError try: module = importlib.import_module(self.module_name) except ModuleNotFoundError: check.failed( f"Couldn't import module {self.module_name} when attempting to load the " f"configurable class {self.module_name}.{self.class_name}") try: klass = getattr(module, self.class_name) except AttributeError: check.failed( f"Couldn't find class {self.class_name} in module when attempting to load the " f"configurable class {self.module_name}.{self.class_name}") if not issubclass(klass, ConfigurableClass): raise check.CheckError( klass, f"class {self.class_name} in module {self.module_name}", ConfigurableClass, ) config_dict = self.config_dict result = process_config(resolve_to_config_type(klass.config_type()), config_dict) if not result.success: raise DagsterInvalidConfigError( f"Errors whilst loading configuration for {klass.config_type()}.", result.errors, config_dict, ) return klass.from_config_value(self, result.value)
def _get_validated_celery_k8s_executor_config(environment_dict): check.dict_param(environment_dict, 'environment_dict') try: from dagster_celery.executor_k8s import CELERY_K8S_CONFIG_KEY, celery_k8s_config except ImportError: raise DagsterK8sError( 'To use the CeleryK8sRunLauncher, dagster_celery must be' ' installed in your Python environment.' ) check.invariant( CELERY_K8S_CONFIG_KEY in environment_dict.get('execution', {}), '{} execution must be configured in pipeline execution config to launch runs with ' 'CeleryK8sRunLauncher'.format(CELERY_K8S_CONFIG_KEY), ) execution_config_schema = resolve_to_config_type(celery_k8s_config()) execution_run_config = environment_dict['execution'][CELERY_K8S_CONFIG_KEY].get('config', {}) res = process_config(execution_config_schema, execution_run_config) check.invariant( res.success, 'Incorrect {} execution schema provided'.format(CELERY_K8S_CONFIG_KEY) ) return res.value
def create_from_config(run_container_context): run_docker_container_context = (run_container_context.get( "docker", {}) if run_container_context else {}) if not run_docker_container_context: return DockerContainerContext() processed_container_context = process_config( DOCKER_CONTAINER_CONTEXT_SCHEMA, run_docker_container_context) if not processed_container_context.success: raise DagsterInvalidConfigError( "Errors while parsing Docker container context", processed_container_context.errors, run_docker_container_context, ) processed_context_value = processed_container_context.value return DockerContainerContext( registry=processed_context_value.get("registry"), env_vars=processed_context_value.get("env_vars", []), networks=processed_context_value.get("networks", []), container_kwargs=processed_context_value.get("container_kwargs"), )
def _get_validated_celery_k8s_executor_config(run_config): check.dict_param(run_config, "run_config") executor_config = run_config.get("execution", {}) execution_config_schema = resolve_to_config_type(celery_k8s_config()) # In run config on jobs, we don't have an executor key if not CELERY_K8S_CONFIG_KEY in executor_config: execution_run_config = executor_config.get("config", {}) else: execution_run_config = (run_config["execution"][CELERY_K8S_CONFIG_KEY] or {}).get("config", {}) res = process_config(execution_config_schema, execution_run_config) check.invariant( res.success, "Incorrect execution schema provided. Note: You may also be seeing this error " "because you are using the configured API. " "Using configured with the {config_key} executor is not supported at this time, " "and all executor config must be directly in the run config without using configured." .format(config_key=CELERY_K8S_CONFIG_KEY, ), ) return res.value
def _get_host_mode_executor(recon_pipeline, run_config, executor_defs, instance): execution_config = run_config.get("execution", {}) execution_config_type = Field(selector_for_named_defs(executor_defs), default_value={ executor_defs[0].name: {} }).config_type config_evr = process_config(execution_config_type, execution_config) if not config_evr.success: raise DagsterInvalidConfigError( "Error processing execution config {}".format(execution_config), config_evr.errors, execution_config, ) execution_config_value = config_evr.value executor_name, executor_config = ensure_single_item(execution_config_value) executor_defs_by_name = { executor_def.name: executor_def for executor_def in executor_defs } executor_def = executor_defs_by_name[executor_name] init_context = InitExecutorContext( job=recon_pipeline, executor_def=executor_def, executor_config=executor_config["config"], instance=instance, ) check_cross_process_constraints(init_context) return executor_def.executor_creation_fn(init_context)
def test_noneable_string_source_array(): assert process_config(Noneable(Array(StringSource)), []).success assert process_config(Noneable(Array(StringSource)), None).success assert ( 'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" ' "which is not set. In order for this execution to succeed it must be set in " "this environment.") in process_config(Noneable( Array(StringSource)), ["test", { "env": "DAGSTER_TEST_ENV_VAR" }]).errors[0].message with environ({"DAGSTER_TEST_ENV_VAR": "baz"}): assert process_config(Noneable(Array(StringSource)), ["test", { "env": "DAGSTER_TEST_ENV_VAR" }]).success
def _get_mapped_solids_dict(composite_def, current_stack, current_solid_config): # the spec of the config mapping function is that it takes the dictionary at: # solid_name: # config: {dict_passed_to_user} # and it returns the dictionary rooted at solids # solid_name: # solids: {return_value_of_config_fn} # We must call the config mapping function and then validate it against # the child schema. with user_code_error_boundary(DagsterConfigMappingFunctionError, _get_error_lambda(current_stack)): mapped_solids_config = composite_def.config_mapping.config_fn( current_solid_config.get('config', {})) # Dynamically construct the type that the output of the config mapping function will # be evaluated against type_to_evaluate_against = define_solid_dictionary_cls( composite_def.solids, composite_def.dependency_structure, current_stack.handle) # process against that new type evr = process_config(type_to_evaluate_against, mapped_solids_config) if not evr.success: raise_composite_descent_config_error(current_stack, mapped_solids_config, evr) return evr.value
def resolve_from_unvalidated_config(self, config: Any) -> Any: """Validates config against outer config schema, and calls mapping against validated config.""" receive_processed_config_values = check.opt_bool_param( self.receive_processed_config_values, "receive_processed_config_values", default=True) if receive_processed_config_values: outer_evr = process_config( self.config_schema.config_type, config, ) else: outer_evr = validate_config( self.config_schema.config_type, config, ) if not outer_evr.success: raise DagsterInvalidConfigError( "Error in config mapping ", outer_evr.errors, config, ) outer_config = outer_evr.value if not receive_processed_config_values: outer_config = resolve_defaults( cast(ConfigType, self.config_schema.config_type), outer_config, ).value return self.config_fn(outer_config)
def create_from_config(run_container_context) -> "K8sContainerContext": run_k8s_container_context = (run_container_context.get("k8s", {}) if run_container_context else {}) if not run_k8s_container_context: return K8sContainerContext() processed_container_context = process_config( DagsterK8sJobConfig.config_type_container_context(), run_k8s_container_context) if not processed_container_context.success: raise DagsterInvalidConfigError( "Errors while parsing k8s container context", processed_container_context.errors, run_k8s_container_context, ) processed_context_value = processed_container_context.value return K8sContainerContext( image_pull_policy=processed_context_value.get("image_pull_policy"), image_pull_secrets=processed_context_value.get( "image_pull_secrets"), service_account_name=processed_context_value.get( "service_account_name"), env_config_maps=processed_context_value.get("env_config_maps"), env_secrets=processed_context_value.get("env_secrets"), env_vars=processed_context_value.get("env_vars"), volume_mounts=processed_context_value.get("volume_mounts"), volumes=processed_context_value.get("volumes"), labels=processed_context_value.get("labels"), namespace=processed_context_value.get("namespace"), )
def test_required_solid_with_required_subfield(): pipeline_def = PipelineDefinition( name="some_pipeline", solid_defs=[ SolidDefinition( name="int_config_solid", config_schema={"required_field": String}, input_defs=[], output_defs=[], compute_fn=lambda *_args: None, ) ], ) env_type = create_run_config_schema_type(pipeline_def) assert env_type.fields["solids"].is_required is True assert env_type.fields["solids"].config_type solids_type = env_type.fields["solids"].config_type assert solids_type.fields["int_config_solid"].is_required is True int_config_solid_type = solids_type.fields["int_config_solid"].config_type assert int_config_solid_type.fields["config"].is_required is True assert env_type.fields["execution"].is_required is False env_obj = ResolvedRunConfig.build( pipeline_def, { "solids": { "int_config_solid": { "config": { "required_field": "foobar" } } } }, ) assert env_obj.solids["int_config_solid"].config[ "required_field"] == "foobar" res = process_config(env_type, {"solids": {}}) assert not res.success res = process_config(env_type, {}) assert not res.success
def test_required_solid_with_required_subfield(): pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[ SolidDefinition( name='int_config_solid', config_schema={'required_field': String}, input_defs=[], output_defs=[], compute_fn=lambda *_args: None, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_required is True assert env_type.fields['solids'].config_type solids_type = env_type.fields['solids'].config_type assert solids_type.fields['int_config_solid'].is_required is True int_config_solid_type = solids_type.fields['int_config_solid'].config_type assert int_config_solid_type.fields['config'].is_required is True assert env_type.fields['execution'].is_required is False env_obj = EnvironmentConfig.build( pipeline_def, { 'solids': { 'int_config_solid': { 'config': { 'required_field': 'foobar' } } } }, ) assert env_obj.solids['int_config_solid'].config[ 'required_field'] == 'foobar' res = process_config(env_type, {'solids': {}}) assert not res.success res = process_config(env_type, {}) assert not res.success
def wrapped_process_config_fn(config): config_evr = process_config({'config': config_schema or {}}, config) if config_evr.success: return self.process_config( {'config': config_fn(config_evr.value['config'])}) else: return config_evr
def test_noneable_string_source_array(): assert process_config(Noneable(Array(StringSource)), []).success assert process_config(Noneable(Array(StringSource)), None).success assert (( 'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" ' 'which is not set. In order for this execution to succeed it must be set in ' 'this environment.') in process_config(Noneable(Array(StringSource)), ['test', { 'env': 'DAGSTER_TEST_ENV_VAR' }]).errors[0].message) with environ({'DAGSTER_TEST_ENV_VAR': 'baz'}): assert process_config(Noneable(Array(StringSource)), ['test', { 'env': 'DAGSTER_TEST_ENV_VAR' }]).success
def test_string_source(): assert process_config(StringSource, "foo").success assert not process_config(StringSource, 1).success assert not process_config(StringSource, {"env": 1}).success assert "DAGSTER_TEST_ENV_VAR" not in os.environ assert not process_config(StringSource, { "env": "DAGSTER_TEST_ENV_VAR" }).success assert ( 'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" ' "which is not set. In order for this execution to succeed it must be set in " "this environment.") in process_config(StringSource, { "env": "DAGSTER_TEST_ENV_VAR" }).errors[0].message with environ({"DAGSTER_TEST_ENV_VAR": "baz"}): assert process_config(StringSource, { "env": "DAGSTER_TEST_ENV_VAR" }).success assert process_config(StringSource, { "env": "DAGSTER_TEST_ENV_VAR" }).value == "baz"
def test_string_source(): assert process_config(StringSource, 'foo').success assert not process_config(StringSource, 1).success assert not process_config(StringSource, {'env': 1}).success assert 'DAGSTER_TEST_ENV_VAR' not in os.environ assert not process_config(StringSource, { 'env': 'DAGSTER_TEST_ENV_VAR' }).success assert ( 'You have attempted to fetch the environment variable "DAGSTER_TEST_ENV_VAR" ' 'which is not set. In order for this execution to succeed it must be set in ' 'this environment.') in process_config(StringSource, { 'env': 'DAGSTER_TEST_ENV_VAR' }).errors[0].message with environ({'DAGSTER_TEST_ENV_VAR': 'baz'}): assert process_config(StringSource, { 'env': 'DAGSTER_TEST_ENV_VAR' }).success assert process_config(StringSource, { 'env': 'DAGSTER_TEST_ENV_VAR' }).value == 'baz'
def test_solid_config_error(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( solids=pipeline_def.solids, ignored_solids=None, dependency_structure=pipeline_def.dependency_structure, parent_handle=None, resource_defs={}, ) int_solid_config_type = solid_dict_type.fields["int_config_solid"].config_type res = process_config(int_solid_config_type, {"notconfig": 1}) assert not res.success assert re.match('Received unexpected config entry "notconfig"', res.errors[0].message) res = process_config(int_solid_config_type, 1) assert not res.success
def _get_mapped_solids_dict( composite, graph_def, current_stack, current_solid_config, resource_defs, is_using_graph_job_op_apis, ): # the spec of the config mapping function is that it takes the dictionary at: # solid_name: # config: {dict_passed_to_user} # and it returns the dictionary rooted at solids # solid_name: # solids: {return_value_of_config_fn} # We must call the config mapping function and then validate it against # the child schema. # apply @configured config mapping to the composite's incoming config before we get to the # composite's own config mapping process config_mapped_solid_config = graph_def.apply_config_mapping( current_solid_config) if not config_mapped_solid_config.success: raise DagsterInvalidConfigError( "Error in config for composite solid {}".format(composite.name), config_mapped_solid_config.errors, config_mapped_solid_config, ) with user_code_error_boundary(DagsterConfigMappingFunctionError, _get_error_lambda(current_stack)): mapped_solids_config = graph_def.config_mapping.resolve_from_validated_config( config_mapped_solid_config.value.get("config", {})) # Dynamically construct the type that the output of the config mapping function will # be evaluated against type_to_evaluate_against = define_solid_dictionary_cls( solids=graph_def.solids, ignored_solids=None, dependency_structure=graph_def.dependency_structure, parent_handle=current_stack.handle, resource_defs=resource_defs, is_using_graph_job_op_apis=is_using_graph_job_op_apis, ) # process against that new type evr = process_config(type_to_evaluate_against, mapped_solids_config) if not evr.success: raise_composite_descent_config_error(current_stack, mapped_solids_config, evr) return evr.value
def test_storage_in_memory_config(): pipeline_def = PipelineDefinition(name='pipeline', solid_defs=[]) env_type = create_environment_type(pipeline_def) assert 'storage' in env_type.fields config_value = process_config(env_type, {'storage': {'in_memory': {}}}) assert config_value.success assert config_value.value['storage'] == {'in_memory': {}}
def test_storage_in_memory_config(): pipeline_def = PipelineDefinition(name="pipeline", solid_defs=[]) env_type = create_environment_type(pipeline_def) assert "storage" in env_type.fields config_value = process_config(env_type, {"intermediate_storage": {"in_memory": {}}}) assert config_value.success assert config_value.value["intermediate_storage"] == {"in_memory": {}}
def test_files_default_config(): pipeline_def = PipelineDefinition(name="pipeline", solid_defs=[]) env_type = create_environment_type(pipeline_def) assert "storage" in env_type.fields config_value = process_config(env_type, {}) assert config_value.success assert "storage" not in config_value
def resolve_config(self, processed_config: Dict[str, Any]) -> EvaluateValueResult: check.dict_param(processed_config, "processed_config") # Validate resolved config against the inner definitions's config_schema (on self). config_evr = process_config( {"config": self.parent_def.config_field or {}}, self._invoke_user_config_fn(processed_config), ) if config_evr.success: return self.parent_def.apply_config_mapping(config_evr.value) # Recursive step else: return config_evr # Bubble up the errors
def test_bad_config(): configs_and_expected_errors = [ ( # Create disposition must match enum values {"create_disposition": "this is not a valid create disposition"}, "Value at path root:ops:test:config:query_job_config:create_disposition not in enum type BQCreateDisposition", ), ( # Priority must match enum values {"priority": "this is not a valid priority"}, "Value at path root:ops:test:config:query_job_config:priority not in enum type BQPriority got this is not a valid priority", ), ( # Schema update options must be a list {"schema_update_options": "this is not valid schema update options"}, 'Value at path root:ops:test:config:query_job_config:schema_update_options must be list. Expected: "[BQSchemaUpdateOption]"', ), ( {"schema_update_options": ["this is not valid schema update options"]}, "Value at path root:ops:test:config:query_job_config:schema_update_options[0] not in enum type BQSchemaUpdateOption", ), ( {"write_disposition": "this is not a valid write disposition"}, "Value at path root:ops:test:config:query_job_config:write_disposition not in enum type BQWriteDisposition", ), ] @job(resource_defs={"bigquery": bigquery_resource}) def test_config(): bq_op_for_queries(["SELECT 1"]).alias("test")() env_type = create_run_config_schema(test_config).config_type for config_fragment, error_message in configs_and_expected_errors: config = {"ops": {"test": {"config": {"query_job_config": config_fragment}}}} result = validate_config(env_type, config) assert error_message in result.errors[0].message configs_and_expected_validation_errors = [ ( {"default_dataset": "this is not a valid dataset"}, "Datasets must be of the form", # project_name.dataset_name', ), ( {"destination": "this is not a valid table"}, "Tables must be of the form", # project_name.dataset_name.table_name' ), ] for config_fragment, error_message in configs_and_expected_validation_errors: config = {"ops": {"test": {"config": {"query_job_config": config_fragment}}}} result = process_config(env_type, config) assert error_message in result.errors[0].message