def test_config_int():
int_inst = resolve_to_config_type(Int)
assert evaluate_config(int_inst, 1).success
assert not evaluate_config(int_inst, None).success
assert not evaluate_config(int_inst, 'r').success
assert not int_inst.is_list
assert not int_inst.is_nullable
assert not (int_inst.is_nullable or int_inst.is_list)
def test_config_any():
any_inst = resolve_to_config_type(None)
assert evaluate_config(any_inst, 1).success
assert evaluate_config(any_inst, None).success
assert evaluate_config(any_inst, 'r').success
assert not any_inst.is_list
assert not any_inst.is_nullable
assert any_inst.is_any
def test_resource_def_config_errors():
takes_int_resource_def = ResourceDefinition(
resource_fn=lambda: None, config_field=Field(Dict({'required_int': Field(Int)}))
)
resource_type = define_resource_cls('Parent', 'takes_int', takes_int_resource_def).inst()
assert not evaluate_config(resource_type, 1).success
assert not evaluate_config(resource_type, {}).success
assert not evaluate_config(resource_type, {'config': {}}).success
assert evaluate_config(resource_type, {'config': {'required_int': 2}}).success
assert not evaluate_config(resource_type, {'config': {'required_int': 'kdjfkd'}}).success
def resolve_is_environment_config_valid(graphene_info, environment_schema,
dagster_pipeline, environment_dict):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(environment_schema, 'environment_schema',
EnvironmentSchema)
check.inst_param(dagster_pipeline, 'dagster_pipeline', PipelineDefinition)
check.opt_dict_param(environment_dict, 'environment_dict', key_type=str)
validated_config = evaluate_config(environment_schema.environment_type,
environment_dict, dagster_pipeline)
dauphin_pipeline = graphene_info.schema.type_named('Pipeline')(
dagster_pipeline)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=dauphin_pipeline,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
graphene_info, err)
for err in validated_config.errors
],
))
return graphene_info.schema.type_named('PipelineConfigValidationValid')(
dauphin_pipeline)
def get_validated_config(graphene_info, dauphin_pipeline, environment_dict,
mode):
check.opt_dict_param(environment_dict, 'environment_dict', key_type=str)
check.str_param(mode, 'mode')
pipeline = dauphin_pipeline.get_dagster_pipeline()
environment_schema = create_environment_schema(pipeline, mode)
validated_config = evaluate_config(environment_schema.environment_type,
environment_dict, pipeline)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=dauphin_pipeline,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
graphene_info, err)
for err in validated_config.errors
],
))
return validated_config
def test_int_field(snapshot):
config_field = Field(Dict({'int_field': Field(Int)}))
assert_config_type_snapshot(snapshot, config_field)
assert evaluate_config(config_field.config_type, {
'int_field': 1
}).value == {
'int_field': 1
}
def test_int_field():
config_field = Field(Dict({'int_field': Field(Int)}))
assert evaluate_config(config_field.config_type, {
'int_field': 1
}).value == {
'int_field': 1
}
def dagster_instance_config(base_dir):
dagster_config_dict = load_yaml_from_globs(
os.path.join(base_dir, DAGSTER_CONFIG_YAML_FILENAME))
dagster_config_type = define_dagster_config_cls().inst()
dagster_config = evaluate_config(dagster_config_type, dagster_config_dict)
if not dagster_config.success:
raise DagsterInvalidConfigError(None, dagster_config.errors,
dagster_config_dict)
return dagster_config.value
def dagster_instance_config(base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None):
overrides = check.opt_dict_param(overrides, 'overrides')
dagster_config_dict = merge_dicts(
load_yaml_from_globs(os.path.join(base_dir, config_filename)), overrides
)
dagster_config_type = define_dagster_config_cls().inst()
dagster_config = evaluate_config(dagster_config_type, dagster_config_dict)
if not dagster_config.success:
raise DagsterInvalidConfigError(None, dagster_config.errors, dagster_config_dict)
return dagster_config.value
def test_list_nullable_int():
lni = resolve_to_config_type(List[Optional[Int]])
assert evaluate_config(lni, [1]).success
assert evaluate_config(lni, [1, 2]).success
assert evaluate_config(lni, []).success
assert evaluate_config(lni, [None]).success
assert evaluate_config(lni, [1, None]).success
assert not evaluate_config(lni, None).success
assert not evaluate_config(lni, [1, 'absdf']).success
def test_list_int():
list_int = resolve_to_config_type(List[Int])
assert evaluate_config(list_int, [1]).success
assert evaluate_config(list_int, [1, 2]).success
assert evaluate_config(list_int, []).success
assert not evaluate_config(list_int, [None]).success
assert not evaluate_config(list_int, [1, None]).success
assert not evaluate_config(list_int, None).success
assert not evaluate_config(list_int, [1, 'absdf']).success
def create_environment_config(pipeline, environment_dict=None, run_config=None):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
check.opt_dict_param(environment_dict, 'environment')
run_config = check.opt_inst_param(run_config, 'run_config', RunConfig, default=RunConfig())
mode = run_config.mode or pipeline.get_default_mode_name()
environment_type = create_environment_type(pipeline, mode)
result = evaluate_config(environment_type, environment_dict, pipeline, run_config)
if not result.success:
raise DagsterInvalidConfigError(pipeline, result.errors, environment_dict)
return EnvironmentConfig.from_config_value(result.value, original_config_dict=environment_dict)
def test_bad_config():
configs_and_expected_errors = [
(
# Create disposition must match enum values
{'create_disposition': 'this is not a valid create disposition'},
'Value not in enum type BQCreateDisposition',
),
(
# Dataset must be of form project_name.dataset_name
{'default_dataset': 'this is not a valid dataset'},
'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "Dataset"',
),
(
# Table must be of form project_name.dataset_name.table_name
{'destination': 'this is not a valid table'},
'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "Table"',
),
(
# Priority must match enum values
{'priority': 'this is not a valid priority'},
'Value not in enum type BQPriority',
),
(
# Schema update options must be a list
{'schema_update_options': 'this is not valid schema update options'},
'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]',
),
(
{'schema_update_options': ['this is not valid schema update options']},
'Value not in enum type BQSchemaUpdateOption',
),
(
{'write_disposition': 'this is not a valid write disposition'},
'Value not in enum type BQWriteDisposition',
),
]
@pipeline(mode_defs=bq_modes())
def test_config_pipeline():
bq_solid_for_queries(['SELECT 1']).alias('test')() # pylint: disable=no-value-for-parameter
env_type = create_environment_type(test_config_pipeline)
for config_fragment, error_message in configs_and_expected_errors:
config = {'solids': {'test': {'config': {'query_job_config': config_fragment}}}}
result = evaluate_config(env_type, config, test_config_pipeline)
assert result.errors[0].message == error_message
def rehydrate(self, **constructor_kwargs):
from dagster.core.errors import DagsterInvalidConfigError
from dagster.core.types.evaluator import evaluate_config
module = importlib.import_module(self.module_name)
klass = getattr(module, self.class_name)
check.subclass_param(
klass,
'class {class_name} in module {module_name}'.format(
class_name=self.class_name, module_name=self.module_name),
ConfigurableClass,
)
config_dict = yaml.load(self.config_yaml)
result = evaluate_config(klass.config_type().inst(), config_dict)
if not result.success:
raise DagsterInvalidConfigError(None, result.errors, config_dict)
constructor_kwargs['inst_data'] = self
return klass.from_config_value(result.value, **constructor_kwargs)
def rehydrate(self, **constructor_kwargs):
from dagster.core.errors import DagsterInvalidConfigError
from dagster.core.types.evaluator import evaluate_config
try:
module = importlib.import_module(self.module_name)
except seven.ModuleNotFoundError:
check.invariant(
False,
'Couldn\'t import module {module_name} when attempting to rehydrate the '
'configurable class {configurable_class}'.format(
module_name=self.module_name,
configurable_class=self.module_name + '.' +
self.class_name,
),
)
try:
klass = getattr(module, self.class_name)
except AttributeError:
check.invariant(
False,
'Couldn\'t find class {class_name} in module when attempting to rehydrate the '
'configurable class {configurable_class}'.format(
class_name=self.class_name,
configurable_class=self.module_name + '.' +
self.class_name,
),
)
check.subclass_param(
klass,
'class {class_name} in module {module_name}'.format(
class_name=self.class_name, module_name=self.module_name),
ConfigurableClass,
)
config_dict = yaml.load(self.config_yaml)
result = evaluate_config(klass.config_type().inst(), config_dict)
if not result.success:
raise DagsterInvalidConfigError(None, result.errors, config_dict)
constructor_kwargs['inst_data'] = self
return klass.from_config_value(result.value, **constructor_kwargs)
def test_optional_int():
optional_int_inst = resolve_to_config_type(Optional[Int])
assert evaluate_config(optional_int_inst, 1).success
assert evaluate_config(optional_int_inst, None).success
assert not evaluate_config(optional_int_inst, 'r').success
def throwing_evaluate_config_value(config_type, config_value):
result = evaluate_config(config_type, config_value)
if not result.success:
raise DagsterEvaluateConfigValueError(result.errors[0].stack, result.errors[0].message)
return result.value
def assert_eval_failure(config_type, value):
assert not evaluate_config(config_type, value).success
def eval_config_value_from_dagster_type(dagster_type, value):
return evaluate_config(resolve_to_config_type(dagster_type), value)
def assert_config_value_success(config_type, config_value, expected):
result = evaluate_config(config_type, config_value)
assert result.success
assert result.value == expected
def test_config_enums():
assert evaluate_config(define_test_enum_type(), 'VALUE_ONE').success
def test_config_enum_error_none():
assert not evaluate_config(define_test_enum_type(), None).success
def test_config_enum_error_wrong_type():
assert not evaluate_config(define_test_enum_type(), 384934).success
def test_config_enum_error():
assert not evaluate_config(define_test_enum_type(), 'NOT_PRESENT').success
