def resolve_is_environment_config_valid( graphene_info, environment_schema, dagster_pipeline, environment_dict ): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(environment_schema, 'environment_schema', EnvironmentSchema) check.inst_param(dagster_pipeline, 'dagster_pipeline', PipelineDefinition) check.dict_param(environment_dict, 'environment_dict', key_type=str) validated_config = validate_config(environment_schema.environment_type, environment_dict) dauphin_pipeline = graphene_info.schema.type_named('Pipeline')(dagster_pipeline) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=dauphin_pipeline, errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError' ).from_dagster_error(graphene_info, err) for err in validated_config.errors ], ) ) return graphene_info.schema.type_named('PipelineConfigValidationValid')(dauphin_pipeline)
def test_scalar_or_selector(): int_or_selector = ScalarUnion( scalar_type=resolve_to_config_type(int), non_scalar_type=Selector({'a_string': str, 'an_int': int}), ) assert validate_config(int_or_selector, 2).success assert not validate_config(int_or_selector, '2').success assert not validate_config(int_or_selector, False).success assert validate_config(int_or_selector, {'a_string': 'kjdfk'}).success assert validate_config(int_or_selector, {'an_int': 2}).success assert not validate_config(int_or_selector, {}).success assert not validate_config(int_or_selector, {'a_string': 'kjdfk', 'an_int': 2}).success assert not validate_config(int_or_selector, {'wrong_key': 'kjdfd'}).success assert not validate_config(int_or_selector, {'a_string': 2}).success assert not validate_config(int_or_selector, {'a_string': 'kjdfk', 'extra_field': 'kd'}).success
def test_list_int(): list_int = resolve_to_config_type(List[Int]) assert validate_config(list_int, [1]).success assert validate_config(list_int, [1, 2]).success assert validate_config(list_int, []).success assert not validate_config(list_int, [None]).success assert not validate_config(list_int, [1, None]).success assert not validate_config(list_int, None).success assert not validate_config(list_int, [1, 'absdf']).success
def test_list_nullable_int(): lni = resolve_to_config_type(List[Optional[Int]]) assert validate_config(lni, [1]).success assert validate_config(lni, [1, 2]).success assert validate_config(lni, []).success assert validate_config(lni, [None]).success assert validate_config(lni, [1, None]).success assert not validate_config(lni, None).success assert not validate_config(lni, [1, 'absdf']).success
def test_scalar_or_dict(): int_or_dict = ScalarUnion( scalar_type=resolve_to_config_type(int), non_scalar_type=Dict({'a_string': str}) ) assert validate_config(int_or_dict, 2).success assert not validate_config(int_or_dict, '2').success assert not validate_config(int_or_dict, False).success assert validate_config(int_or_dict, {'a_string': 'kjdfk'}).success assert not validate_config(int_or_dict, {}).success assert not validate_config(int_or_dict, {'wrong_key': 'kjdfd'}).success assert not validate_config(int_or_dict, {'a_string': 2}).success assert not validate_config(int_or_dict, {'a_string': 'kjdfk', 'extra_field': 'kd'}).success
def test_scalar_or_list(): int_or_list = ScalarUnion( scalar_type=resolve_to_config_type(int), non_scalar_type=resolve_to_config_type(List[str]) ) assert validate_config(int_or_list, 2).success assert not validate_config(int_or_list, '2').success assert not validate_config(int_or_list, False).success assert validate_config(int_or_list, []).success assert validate_config(int_or_list, ['ab']).success assert not validate_config(int_or_list, [2]).success assert not validate_config(int_or_list, {}).success
def test_bad_config(): configs_and_expected_errors = [ ( # Create disposition must match enum values {'create_disposition': 'this is not a valid create disposition'}, 'Value not in enum type BQCreateDisposition', ), ( # Dataset must be of form project_name.dataset_name {'default_dataset': 'this is not a valid dataset'}, 'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "_Dataset"', ), ( # Table must be of form project_name.dataset_name.table_name {'destination': 'this is not a valid table'}, 'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "_Table"', ), ( # Priority must match enum values {'priority': 'this is not a valid priority'}, 'Value not in enum type BQPriority', ), ( # Schema update options must be a list {'schema_update_options': 'this is not valid schema update options'}, 'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]', ), ( {'schema_update_options': ['this is not valid schema update options']}, 'Value not in enum type BQSchemaUpdateOption', ), ( {'write_disposition': 'this is not a valid write disposition'}, 'Value not in enum type BQWriteDisposition', ), ] @pipeline(mode_defs=bq_modes()) def test_config_pipeline(): bq_solid_for_queries(['SELECT 1']).alias('test')() env_type = create_environment_type(test_config_pipeline) for config_fragment, error_message in configs_and_expected_errors: config = {'solids': {'test': {'config': {'query_job_config': config_fragment}}}} result = validate_config(env_type, config) assert result.errors[0].message == error_message
def dagster_instance_config(base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None): overrides = check.opt_dict_param(overrides, 'overrides') dagster_config_dict = merge_dicts( load_yaml_from_globs(os.path.join(base_dir, config_filename)), overrides) dagster_config_type = define_dagster_config_cls() dagster_config = validate_config(dagster_config_type, dagster_config_dict) if not dagster_config.success: raise DagsterInvalidConfigError( 'Errors whilst loading dagster instance config at {}.'.format( config_filename), dagster_config.errors, dagster_config_dict, ) return dagster_config.value
def test_list_of_scalar_or_dict(): int_or_dict_list = resolve_to_config_type( List[ ScalarUnion( scalar_type=resolve_to_config_type(int), non_scalar_type=Dict({'a_string': str}) ) ] ) assert validate_config(int_or_dict_list, []).success assert validate_config(int_or_dict_list, [2]).success assert validate_config(int_or_dict_list, [{'a_string': 'kjdfd'}]).success assert validate_config(int_or_dict_list, [2, {'a_string': 'kjdfd'}]).success assert not validate_config(int_or_dict_list, [2, {'wrong_key': 'kjdfd'}]).success assert not validate_config(int_or_dict_list, [2, {'a_string': 2343}]).success assert not validate_config(int_or_dict_list, ['kjdfkd', {'a_string': 'kjdfd'}]).success
def celery_worker_config_args(config_yaml, config_module, config_file): check.invariant( not (sum(int(bool(x)) for x in [config_module, config_file, config_yaml]) > 1), 'Can only set one of --config-yaml/-y --config-module/-m or --config-file/-f', ) if config_yaml: instance = DagsterInstance.get() config_type = celery_executor.config_field.config_type config_value = get_config_value_from_yaml(config_yaml) config_hash = hash(yaml.dump(config_value)) config_dir = os.path.join(instance.root_directory, 'dagster_celery', 'config') mkdir_p(config_dir) config_path = os.path.join(config_dir, '{config_hash}.py'.format(config_hash=config_hash)) if not os.path.exists(config_path): validated_config = validate_config(config_type, config_value).value with open(config_path, 'w') as fd: printer = IndentingPrinter(printer=fd.write) if 'broker' in validated_config: printer.line( 'broker_url = {broker_url}'.format( broker_url=str(validated_config['broker']) ) ) if 'backend' in validated_config: printer.line( 'result_backend = {result_backend}'.format( result_backend=str(validated_config['backend']) ) ) if 'config_source' in validated_config: for key, value in validated_config['config_source'].items(): printer.line('{key} = {value}'.format(key=key, value=str(value))) # n.b. right now we don't attempt to clean up this cache, but it might make sense to delete # any files older than some time if there are more than some number of files present, etc. return ['--config', config_path] elif config_module or config_file: return ['--config', config_module or config_file] else: return []
def get_validated_config(graphene_info, dauphin_pipeline, environment_dict, mode): check.str_param(mode, 'mode') pipeline = dauphin_pipeline.get_dagster_pipeline() environment_schema = create_environment_schema(pipeline, mode) validated_config = validate_config(environment_schema.environment_type, environment_dict) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=dauphin_pipeline, errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( graphene_info, err) for err in validated_config.errors ], )) return validated_config
def test_config_enum_error(): assert not validate_config(define_test_enum_type(), 'NOT_PRESENT').success
def test_config_enum_error_none(): assert not validate_config(define_test_enum_type(), None).success
def test_config_enum_error_wrong_type(): assert not validate_config(define_test_enum_type(), 384934).success
def test_config_any(): any_inst = resolve_to_config_type(None) assert validate_config(any_inst, 1).success assert validate_config(any_inst, None).success assert validate_config(any_inst, 'r').success assert any_inst.kind == ConfigTypeKind.ANY
def test_config_enums(): assert validate_config(define_test_enum_type(), 'VALUE_ONE').success
def _validate(dagster_type, config_value): return validate_config(resolve_to_config_type(dagster_type), config_value)
def test_int_field(): config_field = _to_field({'int_field': Int}) assert validate_config(config_field.config_type, {'int_field': 1}).value == {'int_field': 1}
def test_optional_int(): optional_int_inst = resolve_to_config_type(Optional[Int]) assert validate_config(optional_int_inst, 1).success assert validate_config(optional_int_inst, None).success assert not validate_config(optional_int_inst, 'r').success
def test_config_int(): int_inst = resolve_to_config_type(Int) assert validate_config(int_inst, 1).success assert not validate_config(int_inst, None).success assert not validate_config(int_inst, 'r').success assert int_inst.kind == ConfigTypeKind.SCALAR
def assert_eval_failure(config_type, value): assert not validate_config(config_type, value).success