def test_resource_def_config_errors(): takes_int_resource_def = ResourceDefinition( resource_fn=lambda: None, config_field=Field(Dict({'required_int': Field(Int)})) ) resource_type = define_resource_cls('Parent', 'takes_int', takes_int_resource_def).inst() assert not evaluate_config_value(resource_type, 1).success assert not evaluate_config_value(resource_type, {}).success assert not evaluate_config_value(resource_type, {'config': {}}).success assert evaluate_config_value(resource_type, {'config': {'required_int': 2}}).success assert not evaluate_config_value(resource_type, {'config': {'required_int': 'kdjfkd'}}).success
def resolve_is_environment_config_valid(graphene_info, environment_schema, dagster_pipeline, environment_dict): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(environment_schema, 'environment_schema', EnvironmentSchema) check.inst_param(dagster_pipeline, 'dagster_pipeline', PipelineDefinition) check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) validated_config = evaluate_config_value( environment_schema.environment_type, environment_dict) dauphin_pipeline = graphene_info.schema.type_named('Pipeline')( dagster_pipeline) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=dauphin_pipeline, errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( graphene_info, err) for err in validated_config.errors ], )) return graphene_info.schema.type_named('PipelineConfigValidationValid')( dauphin_pipeline)
def test_int_field(): config_field = Field(Dict({'int_field': Field(Int)})) assert evaluate_config_value(config_field.config_type, { 'int_field': 1 }).value == { 'int_field': 1 }
def test_errors(): context_defs = { 'test': PipelineContextDefinition( config_field=Field(Dict({'required_int': Field(Int)})), context_fn=lambda *args: ExecutionContext(), ) } context_config_type = define_context_context_cls('something', context_defs).inst() assert not evaluate_config_value(context_config_type, 1).success assert not evaluate_config_value(context_config_type, {}).success invalid_value = {'context_one': 1, 'context_two': 2} result = evaluate_config_value(context_config_type, invalid_value) assert not result.success assert len(result.errors) == 1
def create_environment_config(pipeline, environment_dict=None, mode=None): check.inst_param(pipeline, 'pipeline', PipelineDefinition) check.opt_dict_param(environment_dict, 'environment') mode = check.opt_str_param(mode, 'mode', default=pipeline.get_default_mode_name()) environment_type = create_environment_type(pipeline, mode) result = evaluate_config_value(environment_type, environment_dict) if not result.success: raise PipelineConfigEvaluationError(pipeline, result.errors, environment_dict) return EnvironmentConfig.from_dict(result.value)
def _config_or_error_from_pipeline(info, pipeline, env_config): pipeline_env_type = pipeline.get_dagster_pipeline().environment_type validated_config = evaluate_config_value(pipeline_env_type, env_config) if not validated_config.success: return EitherError( info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=pipeline, errors=[ info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( info, err) for err in validated_config.errors ], )) else: return EitherValue(validated_config)
def execute_pipeline_through_queue( repository_info, pipeline_name, pipeline_solid_subset, config, run_id, message_queue ): """ Execute pipeline using message queue as a transport """ message_queue.put(ProcessStartedSentinel(os.getpid())) execution_metadata = ExecutionMetadata(run_id, event_callback=message_queue.put) from .app import RepositoryContainer repository_container = RepositoryContainer(repository_info) if repository_container.repo_error: message_queue.put( MultiprocessingError( serializable_error_info_from_exc_info(repository_container.repo_error) ) ) return pipeline = repository_container.repository.get_pipeline(pipeline_name) pipeline = get_subset_pipeline(pipeline, pipeline_solid_subset) typed_environment = construct_environment_config( evaluate_config_value(pipeline.environment_type, config).value ) try: result = execute_pipeline_with_metadata( pipeline, typed_environment, execution_metadata=execution_metadata, throw_on_user_error=False, ) return result except: # pylint: disable=W0702 message_queue.put( MultiprocessingError(serializable_error_info_from_exc_info(sys.exc_info())) ) finally: message_queue.put(MultiprocessingDone())
def get_validated_config(graphene_info, dauphin_pipeline, environment_dict, mode): check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) check.str_param(mode, 'mode') environment_schema = create_environment_schema( dauphin_pipeline.get_dagster_pipeline(), mode) validated_config = evaluate_config_value( environment_schema.environment_type, environment_dict) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=dauphin_pipeline, errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( graphene_info, err) for err in validated_config.errors ], )) return validated_config
def throwing_evaluate_config_value(config_type, config_value): result = evaluate_config_value(config_type, config_value) if not result.success: raise DagsterEvaluateConfigValueError(result.errors[0].stack, result.errors[0].message) return result.value
def eval_config_value_from_dagster_type(dagster_type, value): return evaluate_config_value(resolve_to_config_type(dagster_type), value)
def assert_eval_failure(config_type, value): assert not evaluate_config_value(config_type, value).success
def assert_config_value_success(config_type, config_value, expected): result = evaluate_config_value(config_type, config_value) assert result.success assert result.value == expected
def test_bad_config(): configs_and_expected_errors = [ ( # Create disposition must match enum values { 'create_disposition': 'this is not a valid create disposition' }, 'Value not in enum type BQCreateDisposition', ), ( # Dataset must be of form project_name.dataset_name { 'default_dataset': 'this is not a valid dataset' }, 'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "Dataset"', ), ( # Table must be of form project_name.dataset_name.table_name { 'destination': 'this is not a valid table' }, 'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "Table"', ), ( # Priority must match enum values { 'priority': 'this is not a valid priority' }, 'Value not in enum type BQPriority', ), ( # Schema update options must be a list { 'schema_update_options': 'this is not valid schema update options' }, 'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]', ), ( { 'schema_update_options': ['this is not valid schema update options'] }, 'Value not in enum type BQSchemaUpdateOption', ), ( { 'write_disposition': 'this is not a valid write disposition' }, 'Value not in enum type BQWriteDisposition', ), ] pipeline_def = PipelineDefinition( name='test_config_pipeline', solids=[BigQuerySolidDefinition('test', ['SELECT 1'])], mode_definitions=bq_modes(), ) env_type = create_environment_type(pipeline_def) for config_fragment, error_message in configs_and_expected_errors: config = { 'solids': { 'test': { 'config': { 'query_job_config': config_fragment } } } } result = evaluate_config_value(env_type, config) assert result.errors[0].message == error_message