def test_resource_def_config_errors():
takes_int_resource_def = ResourceDefinition(
resource_fn=lambda: None, config_field=Field(Dict({'required_int': Field(Int)}))
)
resource_type = define_resource_cls('Parent', 'takes_int', takes_int_resource_def).inst()
assert not evaluate_config_value(resource_type, 1).success
assert not evaluate_config_value(resource_type, {}).success
assert not evaluate_config_value(resource_type, {'config': {}}).success
assert evaluate_config_value(resource_type, {'config': {'required_int': 2}}).success
assert not evaluate_config_value(resource_type, {'config': {'required_int': 'kdjfkd'}}).success
def resolve_is_environment_config_valid(graphene_info, environment_schema,
dagster_pipeline, environment_dict):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(environment_schema, 'environment_schema',
EnvironmentSchema)
check.inst_param(dagster_pipeline, 'dagster_pipeline', PipelineDefinition)
check.opt_dict_param(environment_dict, 'environment_dict', key_type=str)
validated_config = evaluate_config_value(
environment_schema.environment_type, environment_dict)
dauphin_pipeline = graphene_info.schema.type_named('Pipeline')(
dagster_pipeline)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=dauphin_pipeline,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
graphene_info, err)
for err in validated_config.errors
],
))
return graphene_info.schema.type_named('PipelineConfigValidationValid')(
dauphin_pipeline)
def test_int_field():
config_field = Field(Dict({'int_field': Field(Int)}))
assert evaluate_config_value(config_field.config_type, {
'int_field': 1
}).value == {
'int_field': 1
}
def test_errors():
context_defs = {
'test': PipelineContextDefinition(
config_field=Field(Dict({'required_int': Field(Int)})),
context_fn=lambda *args: ExecutionContext(),
)
}
context_config_type = define_context_context_cls('something', context_defs).inst()
assert not evaluate_config_value(context_config_type, 1).success
assert not evaluate_config_value(context_config_type, {}).success
invalid_value = {'context_one': 1, 'context_two': 2}
result = evaluate_config_value(context_config_type, invalid_value)
assert not result.success
assert len(result.errors) == 1
def create_environment_config(pipeline, environment_dict=None, mode=None):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
check.opt_dict_param(environment_dict, 'environment')
mode = check.opt_str_param(mode, 'mode', default=pipeline.get_default_mode_name())
environment_type = create_environment_type(pipeline, mode)
result = evaluate_config_value(environment_type, environment_dict)
if not result.success:
raise PipelineConfigEvaluationError(pipeline, result.errors, environment_dict)
return EnvironmentConfig.from_dict(result.value)
def _config_or_error_from_pipeline(info, pipeline, env_config):
pipeline_env_type = pipeline.get_dagster_pipeline().environment_type
validated_config = evaluate_config_value(pipeline_env_type, env_config)
if not validated_config.success:
return EitherError(
info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=pipeline,
errors=[
info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
info, err) for err in validated_config.errors
],
))
else:
return EitherValue(validated_config)
def execute_pipeline_through_queue(
repository_info, pipeline_name, pipeline_solid_subset, config, run_id, message_queue
):
"""
Execute pipeline using message queue as a transport
"""
message_queue.put(ProcessStartedSentinel(os.getpid()))
execution_metadata = ExecutionMetadata(run_id, event_callback=message_queue.put)
from .app import RepositoryContainer
repository_container = RepositoryContainer(repository_info)
if repository_container.repo_error:
message_queue.put(
MultiprocessingError(
serializable_error_info_from_exc_info(repository_container.repo_error)
)
)
return
pipeline = repository_container.repository.get_pipeline(pipeline_name)
pipeline = get_subset_pipeline(pipeline, pipeline_solid_subset)
typed_environment = construct_environment_config(
evaluate_config_value(pipeline.environment_type, config).value
)
try:
result = execute_pipeline_with_metadata(
pipeline,
typed_environment,
execution_metadata=execution_metadata,
throw_on_user_error=False,
)
return result
except: # pylint: disable=W0702
message_queue.put(
MultiprocessingError(serializable_error_info_from_exc_info(sys.exc_info()))
)
finally:
message_queue.put(MultiprocessingDone())
def get_validated_config(graphene_info, dauphin_pipeline, environment_dict,
mode):
check.opt_dict_param(environment_dict, 'environment_dict', key_type=str)
check.str_param(mode, 'mode')
environment_schema = create_environment_schema(
dauphin_pipeline.get_dagster_pipeline(), mode)
validated_config = evaluate_config_value(
environment_schema.environment_type, environment_dict)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=dauphin_pipeline,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
graphene_info, err)
for err in validated_config.errors
],
))
return validated_config
def throwing_evaluate_config_value(config_type, config_value):
result = evaluate_config_value(config_type, config_value)
if not result.success:
raise DagsterEvaluateConfigValueError(result.errors[0].stack, result.errors[0].message)
return result.value
def eval_config_value_from_dagster_type(dagster_type, value):
return evaluate_config_value(resolve_to_config_type(dagster_type), value)
def assert_eval_failure(config_type, value):
assert not evaluate_config_value(config_type, value).success
def assert_config_value_success(config_type, config_value, expected):
result = evaluate_config_value(config_type, config_value)
assert result.success
assert result.value == expected
def test_bad_config():
configs_and_expected_errors = [
(
# Create disposition must match enum values
{
'create_disposition': 'this is not a valid create disposition'
},
'Value not in enum type BQCreateDisposition',
),
(
# Dataset must be of form project_name.dataset_name
{
'default_dataset': 'this is not a valid dataset'
},
'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "Dataset"',
),
(
# Table must be of form project_name.dataset_name.table_name
{
'destination': 'this is not a valid table'
},
'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "Table"',
),
(
# Priority must match enum values
{
'priority': 'this is not a valid priority'
},
'Value not in enum type BQPriority',
),
(
# Schema update options must be a list
{
'schema_update_options':
'this is not valid schema update options'
},
'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]',
),
(
{
'schema_update_options':
['this is not valid schema update options']
},
'Value not in enum type BQSchemaUpdateOption',
),
(
{
'write_disposition': 'this is not a valid write disposition'
},
'Value not in enum type BQWriteDisposition',
),
]
pipeline_def = PipelineDefinition(
name='test_config_pipeline',
solids=[BigQuerySolidDefinition('test', ['SELECT 1'])],
mode_definitions=bq_modes(),
)
env_type = create_environment_type(pipeline_def)
for config_fragment, error_message in configs_and_expected_errors:
config = {
'solids': {
'test': {
'config': {
'query_job_config': config_fragment
}
}
}
}
result = evaluate_config_value(env_type, config)
assert result.errors[0].message == error_message
