def test_pipeline_types(): @lambda_solid def produce_string(): return 'foo' @solid( input_defs=[InputDefinition('input_one', String)], output_defs=[OutputDefinition(Any)], config_schema={'another_field': Int}, ) def solid_one(_context, input_one): raise Exception('should not execute') pipeline_def = PipelineDefinition( solid_defs=[produce_string, solid_one], dependencies={ 'solid_one': { 'input_one': DependencyDefinition('produce_string') } }, ) run_config_schema = create_run_config_schema(pipeline_def) assert run_config_schema.has_config_type('String') assert run_config_schema.has_config_type('Int') assert not run_config_schema.has_config_type('SomeName')
def test_all_types_provided(): pipeline_def = PipelineDefinition( name="pipeline", solid_defs=[], mode_defs=[ ModeDefinition( name="SomeMode", resource_defs={ "some_resource": ResourceDefinition( lambda _: None, config_schema={ "with_default_int": Field(Int, is_required=False, default_value=23434) }, ) }, ) ], ) run_config_schema = create_run_config_schema(pipeline_def) all_types = list(run_config_schema.all_config_types()) matching_types = [ tt for tt in all_types if tt.kind == ConfigTypeKind.STRICT_SHAPE and "with_default_int" in tt.fields.keys() ] assert len(matching_types) == 1
def test_pipeline_types(): @lambda_solid def produce_string(): return "foo" @solid( input_defs=[InputDefinition("input_one", String)], output_defs=[OutputDefinition(Any)], config_schema={"another_field": Int}, ) def solid_one(_context, input_one): raise Exception("should not execute") pipeline_def = PipelineDefinition( solid_defs=[produce_string, solid_one], dependencies={ "solid_one": { "input_one": DependencyDefinition("produce_string") } }, ) run_config_schema = create_run_config_schema(pipeline_def) assert run_config_schema.has_config_type("String") assert run_config_schema.has_config_type("Int") assert not run_config_schema.has_config_type("SomeName")
def is_config_valid(pipeline_def, run_config, mode): check.str_param(mode, "mode") check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition) run_config_schema = create_run_config_schema(pipeline_def, mode) validated_config = validate_config(run_config_schema.environment_type, run_config) return validated_config.success
def scaffold_pipeline_config(pipeline_def: PipelineDefinition, skip_non_required: bool = True, mode: Optional[str] = None): check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition) check.bool_param(skip_non_required, "skip_non_required") env_config_type = create_run_config_schema(pipeline_def, mode=mode).config_type env_dict = {} for env_field_name, env_field in env_config_type.fields.items( ): # type: ignore if skip_non_required and not env_field.is_required: continue # unfortunately we have to treat this special for now if env_field_name == "context": if skip_non_required and not env_config_type.fields[ "context"].is_required: # type: ignore continue env_dict[env_field_name] = scaffold_type(env_field.config_type, skip_non_required) return env_dict
def is_config_valid(pipeline_def, environment_dict, mode): check.str_param(mode, 'mode') check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) run_config_schema = create_run_config_schema(pipeline_def, mode) validated_config = validate_config(run_config_schema.environment_type, environment_dict) return validated_config.success
def test_bad_config(): configs_and_expected_errors = [ ( # Create disposition must match enum values {"create_disposition": "this is not a valid create disposition"}, "Value at path root:ops:test:config:query_job_config:create_disposition not in enum type BQCreateDisposition", ), ( # Priority must match enum values {"priority": "this is not a valid priority"}, "Value at path root:ops:test:config:query_job_config:priority not in enum type BQPriority got this is not a valid priority", ), ( # Schema update options must be a list {"schema_update_options": "this is not valid schema update options"}, 'Value at path root:ops:test:config:query_job_config:schema_update_options must be list. Expected: "[BQSchemaUpdateOption]"', ), ( {"schema_update_options": ["this is not valid schema update options"]}, "Value at path root:ops:test:config:query_job_config:schema_update_options[0] not in enum type BQSchemaUpdateOption", ), ( {"write_disposition": "this is not a valid write disposition"}, "Value at path root:ops:test:config:query_job_config:write_disposition not in enum type BQWriteDisposition", ), ] @job(resource_defs={"bigquery": bigquery_resource}) def test_config(): bq_op_for_queries(["SELECT 1"]).alias("test")() env_type = create_run_config_schema(test_config).config_type for config_fragment, error_message in configs_and_expected_errors: config = {"ops": {"test": {"config": {"query_job_config": config_fragment}}}} result = validate_config(env_type, config) assert error_message in result.errors[0].message configs_and_expected_validation_errors = [ ( {"default_dataset": "this is not a valid dataset"}, "Datasets must be of the form", # project_name.dataset_name', ), ( {"destination": "this is not a valid table"}, "Tables must be of the form", # project_name.dataset_name.table_name' ), ] for config_fragment, error_message in configs_and_expected_validation_errors: config = {"ops": {"test": {"config": {"query_job_config": config_fragment}}}} result = process_config(env_type, config) assert error_message in result.errors[0].message
def get_validated_config(pipeline_def, environment_dict, mode): from dagster_graphql.schema.errors import DauphinPipelineConfigValidationInvalid check.str_param(mode, 'mode') check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) run_config_schema = create_run_config_schema(pipeline_def, mode) validated_config = validate_config(run_config_schema.environment_type, environment_dict) if not validated_config.success: raise UserFacingGraphQLError( DauphinPipelineConfigValidationInvalid.for_validation_errors( pipeline_def.get_external_pipeline(), validated_config.errors)) return validated_config
def get_validated_config(pipeline_def, run_config, mode): from ..schema.pipelines.config import GrapheneRunConfigValidationInvalid check.str_param(mode, "mode") check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition) run_config_schema = create_run_config_schema(pipeline_def, mode) validated_config = validate_config(run_config_schema.config_type, run_config) if not validated_config.success: raise UserFacingGraphQLError( GrapheneRunConfigValidationInvalid.for_validation_errors( pipeline_def.get_external_pipeline(), validated_config.errors)) return validated_config
def test_basic_solids_config(snapshot): pipeline_def = PipelineDefinition( name="BasicSolidsConfigPipeline", solid_defs=[ SolidDefinition( name="required_field_solid", input_defs=[], output_defs=[], config_schema={"required_int": Int}, compute_fn=lambda *_args: fail_me(), ) ], ) env_config_type = create_run_config_schema(pipeline_def).config_type assert env_config_type.fields["solids"].is_required solids_config_type = env_config_type.fields["solids"].config_type assert solids_config_type.fields["required_field_solid"].is_required required_solid_config_type = solids_config_type.fields[ "required_field_solid"].config_type assert required_solid_config_type.fields["config"].is_required assert set( env_config_type.fields["loggers"].config_type.fields.keys()) == set( ["console"]) console_logger_config_type = env_config_type.fields[ "loggers"].config_type.fields["console"] assert set(console_logger_config_type.config_type.fields.keys()) == set( ["config"]) assert console_logger_config_type.config_type.fields[ "config"].is_required is False console_logger_config_config_type = console_logger_config_type.config_type.fields[ "config"].config_type assert set(console_logger_config_config_type.fields.keys()) == set( ["log_level", "name"]) snapshot.assert_match( scaffold_pipeline_config(pipeline_def, skip_non_required=False))
def create_run_config_schema_type(pipeline_def): schema = create_run_config_schema(pipeline_def=pipeline_def, mode=None) return schema.config_type