def define_executor_config_cls(mode_definition):
check.inst_param(mode_definition, "mode_definition", ModeDefinition)
fields = {}
for executor_def in mode_definition.executor_defs:
fields[executor_def.name] = Field(
Shape(fields={"config": executor_def.config_schema}
if executor_def.config_schema else {}, ))
return Selector(fields)
def pg_config():
return Selector({
'postgres_url': str,
'postgres_db': {
'username': str,
'password': StringSource,
'hostname': str,
'db_name': str,
'port': Field(int, is_required=False, default_value=5432),
},
})
def define_intermediate_storage_config_cls(mode_definition):
check.inst_param(mode_definition, "mode_definition", ModeDefinition)
fields = {}
for intermediate_storage_def in mode_definition.intermediate_storage_defs:
fields[intermediate_storage_def.name] = Field(
Shape(fields={"config": intermediate_storage_def.config_schema}
if intermediate_storage_def.config_schema else {}, ))
return Selector(fields)
def pg_config():
return Selector({
'postgres_url': str,
'postgres_db': {
'username': str,
'password': str,
'hostname': str,
'db_name': str,
'port': Field(int, is_optional=True, default_value=5432),
},
})
def define_executor_config_cls(mode_definition):
check.inst_param(mode_definition, 'mode_definition', ModeDefinition)
fields = {}
for executor_def in mode_definition.executor_defs:
fields[executor_def.name] = Field(
Shape(fields={'config': executor_def.config_field}
if executor_def.config_field else {}, ))
return Selector(fields)
def define_storage_config_cls(mode_definition):
check.inst_param(mode_definition, 'mode_definition', ModeDefinition)
fields = {}
for storage_def in mode_definition.system_storage_defs:
fields[storage_def.name] = Field(
Shape(fields={'config': storage_def.config_field}
if storage_def.config_field else {}, ))
return Selector(fields)
WORKSPACE_CONFIG_SCHEMA = {
"load_from":
Field(
[
Selector(
merge_dicts(
_get_target_config(include_executable_path=True),
{
"python_environment": {
"executable_path":
StringSource,
"target":
Selector(
_get_target_config(
include_executable_path=False)),
},
"grpc_server": {
"host": Field(str, is_required=False),
"socket": Field(str, is_required=False),
"port": Field(int, is_required=False),
"location_name": Field(str, is_required=False),
},
},
))
],
is_required=False,
),
}
WORKSPACE_CONFIG_SCHEMA_WITH_LEGACY = merge_dicts(
def selector_for_named_defs(named_defs):
return Selector({
named_def.name: def_config_field(named_def)
for named_def in named_defs
})
"attribute": Field(StringSource, is_required=False),
"location_name": Field(StringSource, is_required=False),
"executable_path": Field(StringSource, is_required=False),
},
),
}
WORKSPACE_CONFIG_SCHEMA = {
"load_from":
Field(
[
Selector(
merge_dicts(
_get_target_config(),
{
"grpc_server": {
"host": Field(StringSource, is_required=False),
"socket": Field(StringSource, is_required=False),
"port": Field(IntSource, is_required=False),
"location_name": Field(StringSource,
is_required=False),
"ssl": Field(bool, is_required=False),
},
},
))
],
is_required=False,
),
}
assert level_two_error.error_data.field_name == 'level_two_int_field'
assert not result.errors_at_level('level_two_dict', 'level_three_dict')
final_level_errors = result.errors_at_level('level_two_dict',
'level_three_dict',
'level_three_string')
assert len(final_level_errors) == 1
final_level_error = final_level_errors[0]
assert final_level_error.reason == DagsterEvaluationErrorReason.RUNTIME_TYPE_MISMATCH
ExampleSelector = Selector({
'option_one': Field(String),
'option_two': Field(String)
})
def test_example_selector_success():
result = eval_config_value_from_dagster_type(ExampleSelector,
{'option_one': 'foo'})
assert result.success
assert result.value == {'option_one': 'foo'}
result = eval_config_value_from_dagster_type(ExampleSelector,
{'option_two': 'foo'})
assert result.success
assert result.value == {'option_two': 'foo'}
non_scalar_schema={
'package_name': str,
'attribute': Field(str, is_required=False),
'location_name': Field(str, is_required=False),
},
),
}
WORKSPACE_CONFIG_SCHEMA = {
'load_from': [
Selector(
merge_dicts(
_get_target_config(),
{
'python_environment': {
'executable_path': StringSource,
'target': Selector(_get_target_config()),
},
},
))
],
}
WORKSPACE_CONFIG_SCHEMA_WITH_LEGACY = Selector(
merge_dicts(
{
'repository': {
'module': Field(str, is_required=False),
'file': Field(str, is_required=False),
'fn': Field(str),
},
explicit_forkserver_preload=start_cfg.get("preload_modules"),
)
MULTI_PROC_CONFIG = {
"max_concurrent":
Field(Int, is_required=False, default_value=0),
"start_method":
Field(
Selector({
"spawn": {},
"forkserver": {
"preload_modules":
Field(
[str],
is_required=False,
description=
"Explicit modules to preload in the forkserver.",
),
},
# fork currently unsupported due to threads usage
}),
is_required=False,
description=
("Select how subprocesses are created. Defaults to spawn.\n"
"When forkserver is selected, set_forkserver_preload will be called with either:\n"
"* the preload_modules list if provided by config\n"
"* the module containing the Job if it was loaded from a module\n"
"* dagster\n"
"https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods"
),
assert level_two_error.error_data.field_name == 'level_two_int_field'
assert not result.errors_at_level('level_two_dict', 'level_three_dict')
final_level_errors = result.errors_at_level('level_two_dict',
'level_three_dict',
'level_three_string')
assert len(final_level_errors) == 1
final_level_error = final_level_errors[0]
assert final_level_error.reason == DagsterEvaluationErrorReason.RUNTIME_TYPE_MISMATCH
ExampleSelector = Selector({
'option_one': Field(String),
'option_two': Field(String)
})
def test_example_selector_success():
result = eval_config_value_from_dagster_type(ExampleSelector,
{'option_one': 'foo'})
assert result.success
assert result.value == {'option_one': 'foo'}
result = eval_config_value_from_dagster_type(ExampleSelector,
{'option_two': 'foo'})
assert result.success
assert result.value == {'option_two': 'foo'}
),
}
WORKSPACE_CONFIG_SCHEMA = {
'load_from':
Field(
[
Selector(
merge_dicts(
_get_target_config(),
{
'python_environment': {
'executable_path': StringSource,
'target': Selector(_get_target_config()),
},
'grpc_server': {
'host': Field(str, is_required=False),
'socket': Field(str, is_required=False),
'port': Field(int, is_required=False),
'location_name': Field(str, is_required=False),
},
},
))
],
is_required=False,
),
'opt_in':
Field([Enum('WorkspaceOptInFeature', [EnumValue('grpc')])],
is_required=False),
}
