def resolve_is_environment_config_valid(
graphene_info, environment_schema, dagster_pipeline, environment_dict
):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(environment_schema, 'environment_schema', EnvironmentSchema)
check.inst_param(dagster_pipeline, 'dagster_pipeline', PipelineDefinition)
check.dict_param(environment_dict, 'environment_dict', key_type=str)
validated_config = validate_config(environment_schema.environment_type, environment_dict)
dauphin_pipeline = graphene_info.schema.type_named('Pipeline')(dagster_pipeline)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=dauphin_pipeline,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError'
).from_dagster_error(graphene_info, err)
for err in validated_config.errors
],
)
)
return graphene_info.schema.type_named('PipelineConfigValidationValid')(dauphin_pipeline)
def test_scalar_or_selector():
int_or_selector = ScalarUnion(
scalar_type=resolve_to_config_type(int),
non_scalar_type=Selector({'a_string': str, 'an_int': int}),
)
assert validate_config(int_or_selector, 2).success
assert not validate_config(int_or_selector, '2').success
assert not validate_config(int_or_selector, False).success
assert validate_config(int_or_selector, {'a_string': 'kjdfk'}).success
assert validate_config(int_or_selector, {'an_int': 2}).success
assert not validate_config(int_or_selector, {}).success
assert not validate_config(int_or_selector, {'a_string': 'kjdfk', 'an_int': 2}).success
assert not validate_config(int_or_selector, {'wrong_key': 'kjdfd'}).success
assert not validate_config(int_or_selector, {'a_string': 2}).success
assert not validate_config(int_or_selector, {'a_string': 'kjdfk', 'extra_field': 'kd'}).success
def test_list_int():
list_int = resolve_to_config_type(List[Int])
assert validate_config(list_int, [1]).success
assert validate_config(list_int, [1, 2]).success
assert validate_config(list_int, []).success
assert not validate_config(list_int, [None]).success
assert not validate_config(list_int, [1, None]).success
assert not validate_config(list_int, None).success
assert not validate_config(list_int, [1, 'absdf']).success
def test_list_nullable_int():
lni = resolve_to_config_type(List[Optional[Int]])
assert validate_config(lni, [1]).success
assert validate_config(lni, [1, 2]).success
assert validate_config(lni, []).success
assert validate_config(lni, [None]).success
assert validate_config(lni, [1, None]).success
assert not validate_config(lni, None).success
assert not validate_config(lni, [1, 'absdf']).success
def test_scalar_or_dict():
int_or_dict = ScalarUnion(
scalar_type=resolve_to_config_type(int), non_scalar_type=Dict({'a_string': str})
)
assert validate_config(int_or_dict, 2).success
assert not validate_config(int_or_dict, '2').success
assert not validate_config(int_or_dict, False).success
assert validate_config(int_or_dict, {'a_string': 'kjdfk'}).success
assert not validate_config(int_or_dict, {}).success
assert not validate_config(int_or_dict, {'wrong_key': 'kjdfd'}).success
assert not validate_config(int_or_dict, {'a_string': 2}).success
assert not validate_config(int_or_dict, {'a_string': 'kjdfk', 'extra_field': 'kd'}).success
def test_scalar_or_list():
int_or_list = ScalarUnion(
scalar_type=resolve_to_config_type(int), non_scalar_type=resolve_to_config_type(List[str])
)
assert validate_config(int_or_list, 2).success
assert not validate_config(int_or_list, '2').success
assert not validate_config(int_or_list, False).success
assert validate_config(int_or_list, []).success
assert validate_config(int_or_list, ['ab']).success
assert not validate_config(int_or_list, [2]).success
assert not validate_config(int_or_list, {}).success
def test_bad_config():
configs_and_expected_errors = [
(
# Create disposition must match enum values
{'create_disposition': 'this is not a valid create disposition'},
'Value not in enum type BQCreateDisposition',
),
(
# Dataset must be of form project_name.dataset_name
{'default_dataset': 'this is not a valid dataset'},
'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "_Dataset"',
),
(
# Table must be of form project_name.dataset_name.table_name
{'destination': 'this is not a valid table'},
'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "_Table"',
),
(
# Priority must match enum values
{'priority': 'this is not a valid priority'},
'Value not in enum type BQPriority',
),
(
# Schema update options must be a list
{'schema_update_options': 'this is not valid schema update options'},
'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]',
),
(
{'schema_update_options': ['this is not valid schema update options']},
'Value not in enum type BQSchemaUpdateOption',
),
(
{'write_disposition': 'this is not a valid write disposition'},
'Value not in enum type BQWriteDisposition',
),
]
@pipeline(mode_defs=bq_modes())
def test_config_pipeline():
bq_solid_for_queries(['SELECT 1']).alias('test')()
env_type = create_environment_type(test_config_pipeline)
for config_fragment, error_message in configs_and_expected_errors:
config = {'solids': {'test': {'config': {'query_job_config': config_fragment}}}}
result = validate_config(env_type, config)
assert result.errors[0].message == error_message
def dagster_instance_config(base_dir,
config_filename=DAGSTER_CONFIG_YAML_FILENAME,
overrides=None):
overrides = check.opt_dict_param(overrides, 'overrides')
dagster_config_dict = merge_dicts(
load_yaml_from_globs(os.path.join(base_dir, config_filename)),
overrides)
dagster_config_type = define_dagster_config_cls()
dagster_config = validate_config(dagster_config_type, dagster_config_dict)
if not dagster_config.success:
raise DagsterInvalidConfigError(
'Errors whilst loading dagster instance config at {}.'.format(
config_filename),
dagster_config.errors,
dagster_config_dict,
)
return dagster_config.value
def test_list_of_scalar_or_dict():
int_or_dict_list = resolve_to_config_type(
List[
ScalarUnion(
scalar_type=resolve_to_config_type(int), non_scalar_type=Dict({'a_string': str})
)
]
)
assert validate_config(int_or_dict_list, []).success
assert validate_config(int_or_dict_list, [2]).success
assert validate_config(int_or_dict_list, [{'a_string': 'kjdfd'}]).success
assert validate_config(int_or_dict_list, [2, {'a_string': 'kjdfd'}]).success
assert not validate_config(int_or_dict_list, [2, {'wrong_key': 'kjdfd'}]).success
assert not validate_config(int_or_dict_list, [2, {'a_string': 2343}]).success
assert not validate_config(int_or_dict_list, ['kjdfkd', {'a_string': 'kjdfd'}]).success
def celery_worker_config_args(config_yaml, config_module, config_file):
check.invariant(
not (sum(int(bool(x)) for x in [config_module, config_file, config_yaml]) > 1),
'Can only set one of --config-yaml/-y --config-module/-m or --config-file/-f',
)
if config_yaml:
instance = DagsterInstance.get()
config_type = celery_executor.config_field.config_type
config_value = get_config_value_from_yaml(config_yaml)
config_hash = hash(yaml.dump(config_value))
config_dir = os.path.join(instance.root_directory, 'dagster_celery', 'config')
mkdir_p(config_dir)
config_path = os.path.join(config_dir, '{config_hash}.py'.format(config_hash=config_hash))
if not os.path.exists(config_path):
validated_config = validate_config(config_type, config_value).value
with open(config_path, 'w') as fd:
printer = IndentingPrinter(printer=fd.write)
if 'broker' in validated_config:
printer.line(
'broker_url = {broker_url}'.format(
broker_url=str(validated_config['broker'])
)
)
if 'backend' in validated_config:
printer.line(
'result_backend = {result_backend}'.format(
result_backend=str(validated_config['backend'])
)
)
if 'config_source' in validated_config:
for key, value in validated_config['config_source'].items():
printer.line('{key} = {value}'.format(key=key, value=str(value)))
# n.b. right now we don't attempt to clean up this cache, but it might make sense to delete
# any files older than some time if there are more than some number of files present, etc.
return ['--config', config_path]
elif config_module or config_file:
return ['--config', config_module or config_file]
else:
return []
def get_validated_config(graphene_info, dauphin_pipeline, environment_dict,
mode):
check.str_param(mode, 'mode')
pipeline = dauphin_pipeline.get_dagster_pipeline()
environment_schema = create_environment_schema(pipeline, mode)
validated_config = validate_config(environment_schema.environment_type,
environment_dict)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=dauphin_pipeline,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
graphene_info, err)
for err in validated_config.errors
],
))
return validated_config
def test_config_enum_error():
assert not validate_config(define_test_enum_type(), 'NOT_PRESENT').success
def test_config_enum_error_none():
assert not validate_config(define_test_enum_type(), None).success
def test_config_enum_error_wrong_type():
assert not validate_config(define_test_enum_type(), 384934).success
def test_config_any():
any_inst = resolve_to_config_type(None)
assert validate_config(any_inst, 1).success
assert validate_config(any_inst, None).success
assert validate_config(any_inst, 'r').success
assert any_inst.kind == ConfigTypeKind.ANY
def test_config_enums():
assert validate_config(define_test_enum_type(), 'VALUE_ONE').success
def _validate(dagster_type, config_value):
return validate_config(resolve_to_config_type(dagster_type), config_value)
def test_int_field():
config_field = _to_field({'int_field': Int})
assert validate_config(config_field.config_type, {'int_field': 1}).value == {'int_field': 1}
def test_optional_int():
optional_int_inst = resolve_to_config_type(Optional[Int])
assert validate_config(optional_int_inst, 1).success
assert validate_config(optional_int_inst, None).success
assert not validate_config(optional_int_inst, 'r').success
def test_config_int():
int_inst = resolve_to_config_type(Int)
assert validate_config(int_inst, 1).success
assert not validate_config(int_inst, None).success
assert not validate_config(int_inst, 'r').success
assert int_inst.kind == ConfigTypeKind.SCALAR
def assert_eval_failure(config_type, value):
assert not validate_config(config_type, value).success
