def test_config_int():
int_inst = resolve_to_config_type(Int)
assert evaluate_config(int_inst, 1).success
assert not evaluate_config(int_inst, None).success
assert not evaluate_config(int_inst, 'r').success
assert not int_inst.is_list
assert not int_inst.is_nullable
assert not (int_inst.is_nullable or int_inst.is_list)
def test_config_any():
any_inst = resolve_to_config_type(None)
assert evaluate_config(any_inst, 1).success
assert evaluate_config(any_inst, None).success
assert evaluate_config(any_inst, 'r').success
assert not any_inst.is_list
assert not any_inst.is_nullable
assert any_inst.is_any
def resolve_is_environment_config_valid(graphene_info, environment_schema,
dagster_pipeline, environment_dict):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(environment_schema, 'environment_schema',
EnvironmentSchema)
check.inst_param(dagster_pipeline, 'dagster_pipeline', PipelineDefinition)
check.opt_dict_param(environment_dict, 'environment_dict', key_type=str)
validated_config = evaluate_config(environment_schema.environment_type,
environment_dict, dagster_pipeline)
dauphin_pipeline = graphene_info.schema.type_named('Pipeline')(
dagster_pipeline)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=dauphin_pipeline,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
graphene_info, err)
for err in validated_config.errors
],
))
return graphene_info.schema.type_named('PipelineConfigValidationValid')(
dauphin_pipeline)
def test_int_field(snapshot):
config_field = _to_field({'int_field': Int})
assert_config_type_snapshot(snapshot, config_field)
assert evaluate_config(config_field.config_type, {
'int_field': 1
}).value == {
'int_field': 1
}
def test_list_nullable_int():
lni = resolve_to_config_type(List[Optional[Int]])
assert evaluate_config(lni, [1]).success
assert evaluate_config(lni, [1, 2]).success
assert evaluate_config(lni, []).success
assert evaluate_config(lni, [None]).success
assert evaluate_config(lni, [1, None]).success
assert not evaluate_config(lni, None).success
assert not evaluate_config(lni, [1, 'absdf']).success
def test_list_int():
list_int = resolve_to_config_type(List[Int])
assert evaluate_config(list_int, [1]).success
assert evaluate_config(list_int, [1, 2]).success
assert evaluate_config(list_int, []).success
assert not evaluate_config(list_int, [None]).success
assert not evaluate_config(list_int, [1, None]).success
assert not evaluate_config(list_int, None).success
assert not evaluate_config(list_int, [1, 'absdf']).success
def dagster_instance_config(base_dir,
config_filename=DAGSTER_CONFIG_YAML_FILENAME,
overrides=None):
overrides = check.opt_dict_param(overrides, 'overrides')
dagster_config_dict = merge_dicts(
load_yaml_from_globs(os.path.join(base_dir, config_filename)),
overrides)
dagster_config_type = define_dagster_config_cls()
dagster_config = evaluate_config(dagster_config_type, dagster_config_dict)
if not dagster_config.success:
raise DagsterInvalidConfigError(None, dagster_config.errors,
dagster_config_dict)
return dagster_config.value
def build(pipeline, environment_dict=None, run_config=None):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
check.opt_dict_param(environment_dict, 'environment')
run_config = check.opt_inst_param(run_config,
'run_config',
IRunConfig,
default=RunConfig())
mode = run_config.mode or pipeline.get_default_mode_name()
environment_type = create_environment_type(pipeline, mode)
result = evaluate_config(environment_type, environment_dict, pipeline,
run_config)
if not result.success:
raise DagsterInvalidConfigError(pipeline, result.errors,
environment_dict)
return EnvironmentConfig.from_config_value(result.value,
environment_dict)
def rehydrate(self, **constructor_kwargs):
from dagster.core.errors import DagsterInvalidConfigError
from dagster.core.types.config.evaluator import evaluate_config
try:
module = importlib.import_module(self.module_name)
except seven.ModuleNotFoundError:
check.failed(
'Couldn\'t import module {module_name} when attempting to rehydrate the '
'configurable class {configurable_class}'.format(
module_name=self.module_name,
configurable_class=self.module_name + '.' + self.class_name,
)
)
try:
klass = getattr(module, self.class_name)
except AttributeError:
check.failed(
'Couldn\'t find class {class_name} in module when attempting to rehydrate the '
'configurable class {configurable_class}'.format(
class_name=self.class_name,
configurable_class=self.module_name + '.' + self.class_name,
)
)
if not issubclass(klass, ConfigurableClass):
raise check.CheckError(
klass,
'class {class_name} in module {module_name}'.format(
class_name=self.class_name, module_name=self.module_name
),
ConfigurableClass,
)
config_dict = yaml.load(self.config_yaml)
result = evaluate_config(klass.config_type(), config_dict)
if not result.success:
raise DagsterInvalidConfigError(None, result.errors, config_dict)
return klass.from_config_value(self, result.value, **constructor_kwargs)
def get_validated_config(graphene_info, dauphin_pipeline, environment_dict,
mode):
check.str_param(mode, 'mode')
pipeline = dauphin_pipeline.get_dagster_pipeline()
environment_schema = create_environment_schema(pipeline, mode)
validated_config = evaluate_config(environment_schema.environment_type,
environment_dict, pipeline)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=dauphin_pipeline,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
graphene_info, err)
for err in validated_config.errors
],
))
return validated_config
def throwing_evaluate_config_value(config_type, config_value):
result = evaluate_config(config_type, config_value)
if not result.success:
raise DagsterEvaluateConfigValueError(result.errors[0].stack,
result.errors[0].message)
return result.value
def test_optional_int():
optional_int_inst = resolve_to_config_type(Optional[Int])
assert evaluate_config(optional_int_inst, 1).success
assert evaluate_config(optional_int_inst, None).success
assert not evaluate_config(optional_int_inst, 'r').success
def eval_config_value_from_dagster_type(dagster_type, value):
return evaluate_config(resolve_to_config_type(dagster_type), value)
def assert_eval_failure(config_type, value):
assert not evaluate_config(config_type, value).success
def assert_config_value_success(config_type, config_value, expected):
result = evaluate_config(config_type, config_value)
assert result.success
assert result.value == expected
def test_bad_config():
configs_and_expected_errors = [
(
# Create disposition must match enum values
{
'create_disposition': 'this is not a valid create disposition'
},
'Value not in enum type BQCreateDisposition',
),
(
# Dataset must be of form project_name.dataset_name
{
'default_dataset': 'this is not a valid dataset'
},
'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "_Dataset"',
),
(
# Table must be of form project_name.dataset_name.table_name
{
'destination': 'this is not a valid table'
},
'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "_Table"',
),
(
# Priority must match enum values
{
'priority': 'this is not a valid priority'
},
'Value not in enum type BQPriority',
),
(
# Schema update options must be a list
{
'schema_update_options':
'this is not valid schema update options'
},
'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]',
),
(
{
'schema_update_options':
['this is not valid schema update options']
},
'Value not in enum type BQSchemaUpdateOption',
),
(
{
'write_disposition': 'this is not a valid write disposition'
},
'Value not in enum type BQWriteDisposition',
),
]
@pipeline(mode_defs=bq_modes())
def test_config_pipeline():
bq_solid_for_queries(['SELECT 1']).alias('test')()
env_type = create_environment_type(test_config_pipeline)
for config_fragment, error_message in configs_and_expected_errors:
config = {
'solids': {
'test': {
'config': {
'query_job_config': config_fragment
}
}
}
}
result = evaluate_config(env_type, config, test_config_pipeline)
assert result.errors[0].message == error_message