def test_pipeline_types():
@lambda_solid
def produce_string():
return 'foo'
@solid(
input_defs=[InputDefinition('input_one', String)],
output_defs=[OutputDefinition(Any)],
config_schema={'another_field': Int},
)
def solid_one(_context, input_one):
raise Exception('should not execute')
pipeline_def = PipelineDefinition(
solid_defs=[produce_string, solid_one],
dependencies={
'solid_one': {
'input_one': DependencyDefinition('produce_string')
}
},
)
run_config_schema = create_run_config_schema(pipeline_def)
assert run_config_schema.has_config_type('String')
assert run_config_schema.has_config_type('Int')
assert not run_config_schema.has_config_type('SomeName')
def test_all_types_provided():
pipeline_def = PipelineDefinition(
name="pipeline",
solid_defs=[],
mode_defs=[
ModeDefinition(
name="SomeMode",
resource_defs={
"some_resource": ResourceDefinition(
lambda _: None,
config_schema={
"with_default_int": Field(Int, is_required=False, default_value=23434)
},
)
},
)
],
)
run_config_schema = create_run_config_schema(pipeline_def)
all_types = list(run_config_schema.all_config_types())
matching_types = [
tt
for tt in all_types
if tt.kind == ConfigTypeKind.STRICT_SHAPE and "with_default_int" in tt.fields.keys()
]
assert len(matching_types) == 1
def test_pipeline_types():
@lambda_solid
def produce_string():
return "foo"
@solid(
input_defs=[InputDefinition("input_one", String)],
output_defs=[OutputDefinition(Any)],
config_schema={"another_field": Int},
)
def solid_one(_context, input_one):
raise Exception("should not execute")
pipeline_def = PipelineDefinition(
solid_defs=[produce_string, solid_one],
dependencies={
"solid_one": {
"input_one": DependencyDefinition("produce_string")
}
},
)
run_config_schema = create_run_config_schema(pipeline_def)
assert run_config_schema.has_config_type("String")
assert run_config_schema.has_config_type("Int")
assert not run_config_schema.has_config_type("SomeName")
def is_config_valid(pipeline_def, run_config, mode):
check.str_param(mode, "mode")
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
run_config_schema = create_run_config_schema(pipeline_def, mode)
validated_config = validate_config(run_config_schema.environment_type, run_config)
return validated_config.success
def scaffold_pipeline_config(pipeline_def: PipelineDefinition,
skip_non_required: bool = True,
mode: Optional[str] = None):
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
check.bool_param(skip_non_required, "skip_non_required")
env_config_type = create_run_config_schema(pipeline_def,
mode=mode).config_type
env_dict = {}
for env_field_name, env_field in env_config_type.fields.items(
): # type: ignore
if skip_non_required and not env_field.is_required:
continue
# unfortunately we have to treat this special for now
if env_field_name == "context":
if skip_non_required and not env_config_type.fields[
"context"].is_required: # type: ignore
continue
env_dict[env_field_name] = scaffold_type(env_field.config_type,
skip_non_required)
return env_dict
def is_config_valid(pipeline_def, environment_dict, mode):
check.str_param(mode, 'mode')
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
run_config_schema = create_run_config_schema(pipeline_def, mode)
validated_config = validate_config(run_config_schema.environment_type,
environment_dict)
return validated_config.success
def test_bad_config():
configs_and_expected_errors = [
(
# Create disposition must match enum values
{"create_disposition": "this is not a valid create disposition"},
"Value at path root:ops:test:config:query_job_config:create_disposition not in enum type BQCreateDisposition",
),
(
# Priority must match enum values
{"priority": "this is not a valid priority"},
"Value at path root:ops:test:config:query_job_config:priority not in enum type BQPriority got this is not a valid priority",
),
(
# Schema update options must be a list
{"schema_update_options": "this is not valid schema update options"},
'Value at path root:ops:test:config:query_job_config:schema_update_options must be list. Expected: "[BQSchemaUpdateOption]"',
),
(
{"schema_update_options": ["this is not valid schema update options"]},
"Value at path root:ops:test:config:query_job_config:schema_update_options[0] not in enum type BQSchemaUpdateOption",
),
(
{"write_disposition": "this is not a valid write disposition"},
"Value at path root:ops:test:config:query_job_config:write_disposition not in enum type BQWriteDisposition",
),
]
@job(resource_defs={"bigquery": bigquery_resource})
def test_config():
bq_op_for_queries(["SELECT 1"]).alias("test")()
env_type = create_run_config_schema(test_config).config_type
for config_fragment, error_message in configs_and_expected_errors:
config = {"ops": {"test": {"config": {"query_job_config": config_fragment}}}}
result = validate_config(env_type, config)
assert error_message in result.errors[0].message
configs_and_expected_validation_errors = [
(
{"default_dataset": "this is not a valid dataset"},
"Datasets must be of the form", # project_name.dataset_name',
),
(
{"destination": "this is not a valid table"},
"Tables must be of the form", # project_name.dataset_name.table_name'
),
]
for config_fragment, error_message in configs_and_expected_validation_errors:
config = {"ops": {"test": {"config": {"query_job_config": config_fragment}}}}
result = process_config(env_type, config)
assert error_message in result.errors[0].message
def get_validated_config(pipeline_def, environment_dict, mode):
from dagster_graphql.schema.errors import DauphinPipelineConfigValidationInvalid
check.str_param(mode, 'mode')
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
run_config_schema = create_run_config_schema(pipeline_def, mode)
validated_config = validate_config(run_config_schema.environment_type,
environment_dict)
if not validated_config.success:
raise UserFacingGraphQLError(
DauphinPipelineConfigValidationInvalid.for_validation_errors(
pipeline_def.get_external_pipeline(), validated_config.errors))
return validated_config
def get_validated_config(pipeline_def, run_config, mode):
from ..schema.pipelines.config import GrapheneRunConfigValidationInvalid
check.str_param(mode, "mode")
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
run_config_schema = create_run_config_schema(pipeline_def, mode)
validated_config = validate_config(run_config_schema.config_type,
run_config)
if not validated_config.success:
raise UserFacingGraphQLError(
GrapheneRunConfigValidationInvalid.for_validation_errors(
pipeline_def.get_external_pipeline(), validated_config.errors))
return validated_config
def test_basic_solids_config(snapshot):
pipeline_def = PipelineDefinition(
name="BasicSolidsConfigPipeline",
solid_defs=[
SolidDefinition(
name="required_field_solid",
input_defs=[],
output_defs=[],
config_schema={"required_int": Int},
compute_fn=lambda *_args: fail_me(),
)
],
)
env_config_type = create_run_config_schema(pipeline_def).config_type
assert env_config_type.fields["solids"].is_required
solids_config_type = env_config_type.fields["solids"].config_type
assert solids_config_type.fields["required_field_solid"].is_required
required_solid_config_type = solids_config_type.fields[
"required_field_solid"].config_type
assert required_solid_config_type.fields["config"].is_required
assert set(
env_config_type.fields["loggers"].config_type.fields.keys()) == set(
["console"])
console_logger_config_type = env_config_type.fields[
"loggers"].config_type.fields["console"]
assert set(console_logger_config_type.config_type.fields.keys()) == set(
["config"])
assert console_logger_config_type.config_type.fields[
"config"].is_required is False
console_logger_config_config_type = console_logger_config_type.config_type.fields[
"config"].config_type
assert set(console_logger_config_config_type.fields.keys()) == set(
["log_level", "name"])
snapshot.assert_match(
scaffold_pipeline_config(pipeline_def, skip_non_required=False))
def create_run_config_schema_type(pipeline_def):
schema = create_run_config_schema(pipeline_def=pipeline_def, mode=None)
return schema.config_type
