def test_optional_solid_with_optional_subfield():
pipeline_def = PipelineDefinition(
name="some_pipeline",
solid_defs=[
SolidDefinition(
name="int_config_solid",
config_schema=Field(
{"optional_field": Field(String, is_required=False)},
is_required=False),
input_defs=[],
output_defs=[],
compute_fn=lambda *_args: None,
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields["solids"].is_required is False
assert env_type.fields["execution"].is_required is False
def test_solid_configs_defaults():
env_type = create_environment_type(define_test_solids_config_pipeline())
solids_field = env_type.fields['solids']
assert_has_fields(solids_field.config_type, 'int_config_solid', 'string_config_solid')
int_solid_field = solids_field.config_type.fields['int_config_solid']
assert int_solid_field.is_optional
# TODO: this is the test case the exposes the default dodginess
assert int_solid_field.default_provided
assert_has_fields(int_solid_field.config_type, 'config')
int_solid_config_field = int_solid_field.config_type.fields['config']
assert int_solid_config_field.is_optional
assert not int_solid_config_field.default_provided
def test_optional_solid_with_optional_subfield():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[
SolidDefinition(
name='int_config_solid',
config=Field(
{'optional_field': Field(String, is_optional=True)},
is_optional=True),
input_defs=[],
output_defs=[],
compute_fn=lambda *_args: None,
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_optional
assert env_type.fields['execution'].is_optional
def test_provided_default_on_resources_config():
@solid(name='some_solid', input_defs=[], output_defs=[])
def some_solid(_):
return None
@pipeline(
mode_defs=[
ModeDefinition(
name='some_mode',
resource_defs={
'some_resource': ResourceDefinition(
resource_fn=lambda: None,
config_field=Field(
Dict(
{
'with_default_int': Field(
Int, is_optional=True, default_value=23434
)
}
)
),
)
},
)
]
)
def pipeline_def():
some_solid()
env_type = create_environment_type(pipeline_def)
assert env_type.type_attributes.is_system_config
some_resource_field = env_type.fields['resources'].config_type.fields['some_resource']
assert some_resource_field.is_optional
some_resource_config_field = some_resource_field.config_type.fields['config']
assert some_resource_config_field.is_optional
assert some_resource_config_field.default_value == {'with_default_int': 23434}
assert some_resource_field.default_value == {'config': {'with_default_int': 23434}}
value = EnvironmentConfig.from_config_value(throwing_evaluate_config_value(env_type, {}), {})
assert value.resources == {'some_resource': {'config': {'with_default_int': 23434}}}
def test_solid_configs_defaults():
env_type = create_environment_type(define_test_solids_config_pipeline())
solids_field = env_type.fields["solids"]
assert_has_fields(solids_field.config_type, "int_config_solid", "string_config_solid")
int_solid_field = solids_field.config_type.fields["int_config_solid"]
assert int_solid_field.is_required is False
# TODO: this is the test case the exposes the default dodginess
# https://github.com/dagster-io/dagster/issues/1990
assert int_solid_field.default_provided
assert_has_fields(int_solid_field.config_type, "config")
int_solid_config_field = int_solid_field.config_type.fields["config"]
assert int_solid_config_field.is_required is False
assert not int_solid_config_field.default_provided
def scaffold_pipeline_config(pipeline_def, skip_optional=True, mode=None):
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
check.bool_param(skip_optional, 'skip_optional')
env_config_type = create_environment_type(pipeline_def, mode=mode)
env_dict = {}
for env_field_name, env_field in env_config_type.fields.items():
if skip_optional and env_field.is_optional:
continue
# unfortunately we have to treat this special for now
if env_field_name == 'context':
if skip_optional and env_config_type.fields['context'].is_optional:
continue
env_dict[env_field_name] = scaffold_type(env_field.config_type, skip_optional)
return env_dict
def create_environment_config(pipeline,
environment_dict=None,
run_config=None):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
check.opt_dict_param(environment_dict, 'environment')
run_config = check.opt_inst_param(run_config,
'run_config',
RunConfig,
default=RunConfig())
mode = run_config.mode or pipeline.get_default_mode_name()
environment_type = create_environment_type(pipeline, mode)
result = evaluate_config(environment_type, environment_dict, pipeline,
run_config)
if not result.success:
raise DagsterInvalidConfigError(pipeline, result.errors,
environment_dict)
return EnvironmentConfig.from_dict(result.value)
def test_optional_solid_with_required_scalar_config():
def _assert_config_none(context, value):
assert context.solid_config is value
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_schema=Int,
input_defs=[],
output_defs=[],
compute_fn=lambda context, _inputs: _assert_config_none(
context, 234),
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_required is True
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_required is True
int_config_solid_type = solids_type.fields['int_config_solid'].config_type
assert_has_fields(int_config_solid_type, 'config')
int_config_solid_config_field = int_config_solid_type.fields['config']
assert int_config_solid_config_field.is_required is True
execute_pipeline(pipeline_def,
{'solids': {
'int_config_solid': {
'config': 234
}
}})
def test_provided_default_on_resources_config():
@solid(
name="some_solid", input_defs=[], output_defs=[], required_resource_keys={"some_resource"}
)
def some_solid(_):
return None
@pipeline(
mode_defs=[
ModeDefinition(
name="some_mode",
resource_defs={
"some_resource": ResourceDefinition(
resource_fn=lambda _: None,
config_schema={
"with_default_int": Field(Int, is_required=False, default_value=23434)
},
)
},
)
]
)
def pipeline_def():
some_solid()
env_type = create_environment_type(pipeline_def)
some_resource_field = env_type.fields["resources"].config_type.fields["some_resource"]
assert some_resource_field.is_required is False
some_resource_config_field = some_resource_field.config_type.fields["config"]
assert some_resource_config_field.is_required is False
assert some_resource_config_field.default_value == {"with_default_int": 23434}
assert some_resource_field.default_value == {"config": {"with_default_int": 23434}}
value = EnvironmentConfig.build(pipeline_def, {})
assert value.resources == {
"some_resource": ResourceConfig({"with_default_int": 23434}),
"io_manager": ResourceConfig(None),
}
def test_required_solid_with_required_subfield():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Dict({'required_field': Field(String)})),
input_defs=[],
output_defs=[],
compute_fn=lambda *_args: None,
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_optional is False
assert env_type.fields['solids'].config_type
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_optional is False
int_config_solid_type = solids_type.fields['int_config_solid'].config_type
assert int_config_solid_type.fields['config'].is_optional is False
assert env_type.fields['execution'].is_optional
assert env_type.fields['expectations'].is_optional
env_obj = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(
env_type, {'solids': {'int_config_solid': {'config': {'required_field': 'foobar'}}}}
)
)
assert env_obj.solids['int_config_solid'].config['required_field'] == 'foobar'
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(env_type, {'solids': {}})
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(env_type, {})
def scaffold_pipeline_config(pipeline_def, skip_non_required=True, mode=None):
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
check.bool_param(skip_non_required, "skip_non_required")
env_config_type = create_environment_type(pipeline_def, mode=mode)
env_dict = {}
for env_field_name, env_field in env_config_type.fields.items():
if skip_non_required and not env_field.is_required:
continue
# unfortunately we have to treat this special for now
if env_field_name == "context":
if skip_non_required and not env_config_type.fields[
"context"].is_required:
continue
env_dict[env_field_name] = scaffold_type(env_field.config_type,
skip_non_required)
return env_dict
def test_required_solid_with_required_subfield():
pipeline_def = PipelineDefinition(
name="some_pipeline",
solid_defs=[
SolidDefinition(
name="int_config_solid",
config_schema={"required_field": String},
input_defs=[],
output_defs=[],
compute_fn=lambda *_args: None,
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields["solids"].is_required is True
assert env_type.fields["solids"].config_type
solids_type = env_type.fields["solids"].config_type
assert solids_type.fields["int_config_solid"].is_required is True
int_config_solid_type = solids_type.fields["int_config_solid"].config_type
assert int_config_solid_type.fields["config"].is_required is True
assert env_type.fields["execution"].is_required is False
env_obj = EnvironmentConfig.build(
pipeline_def,
{"solids": {"int_config_solid": {"config": {"required_field": "foobar"}}}},
)
assert env_obj.solids["int_config_solid"].config["required_field"] == "foobar"
res = process_config(env_type, {"solids": {}})
assert not res.success
res = process_config(env_type, {})
assert not res.success
def test_required_solid_with_required_subfield():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_schema={'required_field': String},
input_defs=[],
output_defs=[],
compute_fn=lambda *_args: None,
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_required is True
assert env_type.fields['solids'].config_type
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_required is True
int_config_solid_type = solids_type.fields['int_config_solid'].config_type
assert int_config_solid_type.fields['config'].is_required is True
assert env_type.fields['execution'].is_required is False
env_obj = EnvironmentConfig.build(
pipeline_def, {'solids': {'int_config_solid': {'config': {'required_field': 'foobar'}}}},
)
assert env_obj.solids['int_config_solid'].config['required_field'] == 'foobar'
res = process_config(env_type, {'solids': {}})
assert not res.success
res = process_config(env_type, {})
assert not res.success
def test_required_solid_with_required_subfield():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[
SolidDefinition(
name='int_config_solid',
config={'required_field': String},
input_defs=[],
output_defs=[],
compute_fn=lambda *_args: None,
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_required is True
assert env_type.fields['solids'].config_type
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_required is True
int_config_solid_type = solids_type.fields['int_config_solid'].config_type
assert int_config_solid_type.fields['config'].is_required is True
assert env_type.fields['execution'].is_required is False
env_obj = EnvironmentConfig.build(
pipeline_def, {'solids': {'int_config_solid': {'config': {'required_field': 'foobar'}}}},
)
assert env_obj.solids['int_config_solid'].config['required_field'] == 'foobar'
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_validate_config_value(env_type, {'solids': {}})
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_validate_config_value(env_type, {})
def test_required_resource_with_required_subfield():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[],
mode_defs=[
ModeDefinition(
resource_defs={
'with_required': ResourceDefinition(
resource_fn=lambda _: None, config_schema={'required_field': String},
)
}
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_required is False
assert env_type.fields['execution'].is_required is False
assert env_type.fields['resources'].is_required
assert nested_field(env_type, 'resources', 'with_required').is_required
assert nested_field(env_type, 'resources', 'with_required', 'config').is_required
assert nested_field(
env_type, 'resources', 'with_required', 'config', 'required_field'
).is_required
def test_bad_config():
configs_and_expected_errors = [
(
# Create disposition must match enum values
{
'create_disposition': 'this is not a valid create disposition'
},
'Value not in enum type BQCreateDisposition',
),
(
# Dataset must be of form project_name.dataset_name
{
'default_dataset': 'this is not a valid dataset'
},
'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "Dataset"',
),
(
# Table must be of form project_name.dataset_name.table_name
{
'destination': 'this is not a valid table'
},
'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "Table"',
),
(
# Priority must match enum values
{
'priority': 'this is not a valid priority'
},
'Value not in enum type BQPriority',
),
(
# Schema update options must be a list
{
'schema_update_options':
'this is not valid schema update options'
},
'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]',
),
(
{
'schema_update_options':
['this is not valid schema update options']
},
'Value not in enum type BQSchemaUpdateOption',
),
(
{
'write_disposition': 'this is not a valid write disposition'
},
'Value not in enum type BQWriteDisposition',
),
]
pipeline_def = PipelineDefinition(
name='test_config_pipeline',
solids=[BigQuerySolidDefinition('test', ['SELECT 1'])],
mode_definitions=bq_modes(),
)
env_type = create_environment_type(pipeline_def)
for config_fragment, error_message in configs_and_expected_errors:
config = {
'solids': {
'test': {
'config': {
'query_job_config': config_fragment
}
}
}
}
result = evaluate_config_value(env_type, config)
assert result.errors[0].message == error_message
def test_bad_config():
configs_and_expected_errors = [
(
# Create disposition must match enum values
{
"create_disposition": "this is not a valid create disposition"
},
"Value at path root:solids:test:config:query_job_config:create_disposition not in enum type BQCreateDisposition",
),
(
# Priority must match enum values
{
"priority": "this is not a valid priority"
},
"Value at path root:solids:test:config:query_job_config:priority not in enum type BQPriority got this is not a valid priority",
),
(
# Schema update options must be a list
{
"schema_update_options":
"this is not valid schema update options"
},
'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: "[BQSchemaUpdateOption]"',
),
(
{
"schema_update_options":
["this is not valid schema update options"]
},
"Value at path root:solids:test:config:query_job_config:schema_update_options[0] not in enum type BQSchemaUpdateOption",
),
(
{
"write_disposition": "this is not a valid write disposition"
},
"Value at path root:solids:test:config:query_job_config:write_disposition not in enum type BQWriteDisposition",
),
]
@pipeline(mode_defs=bq_modes())
def test_config_pipeline():
bq_solid_for_queries(["SELECT 1"]).alias("test")()
env_type = create_environment_type(test_config_pipeline)
for config_fragment, error_message in configs_and_expected_errors:
config = {
"solids": {
"test": {
"config": {
"query_job_config": config_fragment
}
}
}
}
result = validate_config(env_type, config)
assert error_message in result.errors[0].message
configs_and_expected_validation_errors = [
(
{
"default_dataset": "this is not a valid dataset"
},
"Datasets must be of the form", # project_name.dataset_name',
),
(
{
"destination": "this is not a valid table"
},
"Tables must be of the form", # project_name.dataset_name.table_name'
),
]
for config_fragment, error_message in configs_and_expected_validation_errors:
config = {
"solids": {
"test": {
"config": {
"query_job_config": config_fragment
}
}
}
}
result = process_config(env_type, config)
assert error_message in result.errors[0].message
def test_bad_config():
configs_and_expected_errors = [
(
# Create disposition must match enum values
{
'create_disposition': 'this is not a valid create disposition'
},
'Value not in enum type BQCreateDisposition',
),
(
# Dataset must be of form project_name.dataset_name
{
'default_dataset': 'this is not a valid dataset'
},
'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "Dataset"',
),
(
# Table must be of form project_name.dataset_name.table_name
{
'destination': 'this is not a valid table'
},
'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "Table"',
),
(
# Priority must match enum values
{
'priority': 'this is not a valid priority'
},
'Value not in enum type BQPriority',
),
(
# Schema update options must be a list
{
'schema_update_options':
'this is not valid schema update options'
},
'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]',
),
(
{
'schema_update_options':
['this is not valid schema update options']
},
'Value not in enum type BQSchemaUpdateOption',
),
(
{
'write_disposition': 'this is not a valid write disposition'
},
'Value not in enum type BQWriteDisposition',
),
]
@pipeline(mode_defs=bq_modes())
def test_config_pipeline():
bq_solid_for_queries(['SELECT 1']).alias('test')() # pylint: disable=no-value-for-parameter
env_type = create_environment_type(test_config_pipeline)
for config_fragment, error_message in configs_and_expected_errors:
config = {
'solids': {
'test': {
'config': {
'query_job_config': config_fragment
}
}
}
}
result = evaluate_config(env_type, config, test_config_pipeline)
assert result.errors[0].message == error_message
def test_optional_and_required_context():
pipeline_def = PipelineDefinition(
name="some_pipeline",
solid_defs=[],
mode_defs=[
ModeDefinition(
name="mixed",
resource_defs={
"optional_resource":
ResourceDefinition(
lambda _: None,
config_schema={
"optional_field": Field(String, is_required=False)
},
),
"required_resource":
ResourceDefinition(
lambda _: None,
config_schema={"required_field": String},
),
},
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields["solids"].is_required is False
assert env_type.fields["execution"].is_required is False
assert nested_field(env_type, "resources").is_required
assert nested_field(env_type, "resources",
"optional_resource").is_required is False
assert nested_field(env_type, "resources", "optional_resource",
"config").is_required is False
assert (nested_field(env_type, "resources", "optional_resource", "config",
"optional_field").is_required is False)
assert nested_field(env_type, "resources", "required_resource").is_required
assert nested_field(env_type, "resources", "required_resource",
"config").is_required
assert nested_field(env_type, "resources", "required_resource", "config",
"required_field").is_required
env_obj = EnvironmentConfig.build(
pipeline_def,
{
"resources": {
"required_resource": {
"config": {
"required_field": "foo"
}
}
}
},
)
assert env_obj.resources == {
"optional_resource": {
"config": {}
},
"required_resource": {
"config": {
"required_field": "foo"
}
},
"object_manager": {},
}
def test_optional_and_required_context():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[],
mode_definitions=[
ModeDefinition(
name='mixed',
resources={
'optional_resource':
ResourceDefinition(
lambda: None,
config_field=Field(dagster_type=Dict(
fields={
'optional_field': Field(String,
is_optional=True)
})),
),
'required_resource':
ResourceDefinition(
lambda: None,
config_field=Field(dagster_type=Dict(
fields={'required_field': Field(String)})),
),
},
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_optional
assert env_type.fields['execution'].is_optional
assert env_type.fields['expectations'].is_optional
assert nested_field(env_type, 'resources').is_required
assert nested_field(env_type, 'resources', 'optional_resource').is_optional
assert nested_field(env_type, 'resources', 'optional_resource',
'config').is_optional
assert nested_field(env_type, 'resources', 'optional_resource', 'config',
'optional_field').is_optional
assert nested_field(env_type, 'resources', 'required_resource').is_required
assert nested_field(env_type, 'resources', 'required_resource',
'config').is_required
assert nested_field(env_type, 'resources', 'required_resource', 'config',
'required_field').is_required
env_obj = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(
env_type, {
'resources': {
'required_resource': {
'config': {
'required_field': 'foo'
}
}
}
}))
assert env_obj.resources == {
'optional_resource': {
'config': {}
},
'required_resource': {
'config': {
'required_field': 'foo'
}
},
}
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
mode_defs=[
ModeDefinition(
name='test_mode',
resource_defs={
'test_resource':
ResourceDefinition(resource_fn=lambda: None,
config_field=Field(Any))
},
)
],
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None),
],
)
environment_type = create_environment_type(pipeline_def)
assert (environment_type.fields['resources'].config_type.name ==
'SomePipeline.Mode.TestMode.Resources')
solids_type = environment_type.fields['solids'].config_type
assert solids_type.name == 'SomePipeline.SolidsConfigDictionary'
assert (solids_type.fields['int_config_solid'].config_type.name ==
'SomePipeline.SolidConfig.IntConfigSolid')
env = EnvironmentConfig.from_config_value(
throwing_evaluate_config_value(
environment_type,
{
'resources': {
'test_resource': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
},
},
),
{
'resources': {
'test_resource': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
},
},
)
assert isinstance(env, EnvironmentConfig)
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.resources == {'test_resource': {'config': 1}}
def test_optional_and_required_context():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[],
mode_defs=[
ModeDefinition(
name='mixed',
resource_defs={
'optional_resource':
ResourceDefinition(
lambda: None,
config_schema={
'optional_field': Field(String, is_required=False)
},
),
'required_resource':
ResourceDefinition(
lambda: None,
config_schema={'required_field': String},
),
},
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_required is False
assert env_type.fields['execution'].is_required is False
assert nested_field(env_type, 'resources').is_required
assert nested_field(env_type, 'resources',
'optional_resource').is_required is False
assert nested_field(env_type, 'resources', 'optional_resource',
'config').is_required is False
assert (nested_field(env_type, 'resources', 'optional_resource', 'config',
'optional_field').is_required is False)
assert nested_field(env_type, 'resources', 'required_resource').is_required
assert nested_field(env_type, 'resources', 'required_resource',
'config').is_required
assert nested_field(env_type, 'resources', 'required_resource', 'config',
'required_field').is_required
env_obj = EnvironmentConfig.build(
pipeline_def,
{
'resources': {
'required_resource': {
'config': {
'required_field': 'foo'
}
}
}
},
)
assert env_obj.resources == {
'optional_resource': {
'config': {}
},
'required_resource': {
'config': {
'required_field': 'foo'
}
},
}
def test_bad_config():
configs_and_expected_errors = [
(
# Create disposition must match enum values
{
'create_disposition': 'this is not a valid create disposition'
},
'Value at path root:solids:test:config:query_job_config:create_disposition not in enum type BQCreateDisposition',
),
(
# Priority must match enum values
{
'priority': 'this is not a valid priority'
},
'Value at path root:solids:test:config:query_job_config:priority not in enum type BQPriority got this is not a valid priority',
),
(
# Schema update options must be a list
{
'schema_update_options':
'this is not valid schema update options'
},
'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]',
),
(
{
'schema_update_options':
['this is not valid schema update options']
},
'Value at path root:solids:test:config:query_job_config:schema_update_options[0] not in enum type BQSchemaUpdateOption',
),
(
{
'write_disposition': 'this is not a valid write disposition'
},
'Value at path root:solids:test:config:query_job_config:write_disposition not in enum type BQWriteDisposition',
),
]
@pipeline(mode_defs=bq_modes())
def test_config_pipeline():
bq_solid_for_queries(['SELECT 1']).alias('test')()
env_type = create_environment_type(test_config_pipeline)
for config_fragment, error_message in configs_and_expected_errors:
config = {
'solids': {
'test': {
'config': {
'query_job_config': config_fragment
}
}
}
}
result = validate_config(env_type, config)
assert error_message in result.errors[0].message
configs_and_expected_validation_errors = [
(
{
'default_dataset': 'this is not a valid dataset'
},
'Datasets must be of the form', # project_name.dataset_name',
),
(
{
'destination': 'this is not a valid table'
},
'Tables must be of the form', # project_name.dataset_name.table_name'
),
]
for config_fragment, error_message in configs_and_expected_validation_errors:
config = {
'solids': {
'test': {
'config': {
'query_job_config': config_fragment
}
}
}
}
result = process_config(env_type, config)
assert error_message in result.errors[0].message
def resolve_environment_type(self, _graphene_info, mode=None):
return to_dauphin_config_type(
create_environment_type(self._pipeline, mode))
def test_basic_solids_config():
pipeline_def = PipelineDefinition(
name='BasicSolidsConfigPipeline',
solids=[
SolidDefinition(
name='required_field_solid',
inputs=[],
outputs=[],
config_field=Field(Dict(fields={'required_int': Field(Int)})),
compute_fn=lambda *_args: fail_me(),
)
],
)
env_config_type = create_environment_type(pipeline_def)
assert env_config_type.fields['solids'].is_optional is False
solids_config_type = env_config_type.fields['solids'].config_type
assert solids_config_type.fields[
'required_field_solid'].is_optional is False
required_solid_config_type = solids_config_type.fields[
'required_field_solid'].config_type
assert required_solid_config_type.fields['config'].is_optional is False
assert set(
env_config_type.fields['loggers'].config_type.fields.keys()) == set(
['console'])
console_logger_config_type = env_config_type.fields[
'loggers'].config_type.fields['console']
assert set(console_logger_config_type.config_type.fields.keys()) == set(
['config'])
assert console_logger_config_type.config_type.fields['config'].is_optional
console_logger_config_config_type = console_logger_config_type.config_type.fields[
'config'].config_type
assert set(console_logger_config_config_type.fields.keys()) == set(
['log_level', 'name'])
assert scaffold_pipeline_config(pipeline_def, skip_optional=False) == {
'loggers': {
'console': {
'config': {
'log_level': '',
'name': ''
}
}
},
'solids': {
'required_field_solid': {
'config': {
'required_int': 0
}
}
},
'expectations': {
'evaluate': True
},
'execution': {},
'resources': {},
'storage': {
'filesystem': {
'base_dir': ''
},
'in_memory': {},
's3': {
's3_bucket': ''
}
},
}
