def define_executor_config_cls(mode_definition): check.inst_param(mode_definition, "mode_definition", ModeDefinition) fields = {} for executor_def in mode_definition.executor_defs: fields[executor_def.name] = Field( Shape(fields={"config": executor_def.config_schema} if executor_def.config_schema else {}, )) return Selector(fields)
def pg_config(): return Selector({ 'postgres_url': str, 'postgres_db': { 'username': str, 'password': StringSource, 'hostname': str, 'db_name': str, 'port': Field(int, is_required=False, default_value=5432), }, })
def define_intermediate_storage_config_cls(mode_definition): check.inst_param(mode_definition, "mode_definition", ModeDefinition) fields = {} for intermediate_storage_def in mode_definition.intermediate_storage_defs: fields[intermediate_storage_def.name] = Field( Shape(fields={"config": intermediate_storage_def.config_schema} if intermediate_storage_def.config_schema else {}, )) return Selector(fields)
def pg_config(): return Selector({ 'postgres_url': str, 'postgres_db': { 'username': str, 'password': str, 'hostname': str, 'db_name': str, 'port': Field(int, is_optional=True, default_value=5432), }, })
def define_executor_config_cls(mode_definition): check.inst_param(mode_definition, 'mode_definition', ModeDefinition) fields = {} for executor_def in mode_definition.executor_defs: fields[executor_def.name] = Field( Shape(fields={'config': executor_def.config_field} if executor_def.config_field else {}, )) return Selector(fields)
def define_storage_config_cls(mode_definition): check.inst_param(mode_definition, 'mode_definition', ModeDefinition) fields = {} for storage_def in mode_definition.system_storage_defs: fields[storage_def.name] = Field( Shape(fields={'config': storage_def.config_field} if storage_def.config_field else {}, )) return Selector(fields)
WORKSPACE_CONFIG_SCHEMA = { "load_from": Field( [ Selector( merge_dicts( _get_target_config(include_executable_path=True), { "python_environment": { "executable_path": StringSource, "target": Selector( _get_target_config( include_executable_path=False)), }, "grpc_server": { "host": Field(str, is_required=False), "socket": Field(str, is_required=False), "port": Field(int, is_required=False), "location_name": Field(str, is_required=False), }, }, )) ], is_required=False, ), } WORKSPACE_CONFIG_SCHEMA_WITH_LEGACY = merge_dicts(
def selector_for_named_defs(named_defs): return Selector({ named_def.name: def_config_field(named_def) for named_def in named_defs })
"attribute": Field(StringSource, is_required=False), "location_name": Field(StringSource, is_required=False), "executable_path": Field(StringSource, is_required=False), }, ), } WORKSPACE_CONFIG_SCHEMA = { "load_from": Field( [ Selector( merge_dicts( _get_target_config(), { "grpc_server": { "host": Field(StringSource, is_required=False), "socket": Field(StringSource, is_required=False), "port": Field(IntSource, is_required=False), "location_name": Field(StringSource, is_required=False), "ssl": Field(bool, is_required=False), }, }, )) ], is_required=False, ), }
assert level_two_error.error_data.field_name == 'level_two_int_field' assert not result.errors_at_level('level_two_dict', 'level_three_dict') final_level_errors = result.errors_at_level('level_two_dict', 'level_three_dict', 'level_three_string') assert len(final_level_errors) == 1 final_level_error = final_level_errors[0] assert final_level_error.reason == DagsterEvaluationErrorReason.RUNTIME_TYPE_MISMATCH ExampleSelector = Selector({ 'option_one': Field(String), 'option_two': Field(String) }) def test_example_selector_success(): result = eval_config_value_from_dagster_type(ExampleSelector, {'option_one': 'foo'}) assert result.success assert result.value == {'option_one': 'foo'} result = eval_config_value_from_dagster_type(ExampleSelector, {'option_two': 'foo'}) assert result.success assert result.value == {'option_two': 'foo'}
non_scalar_schema={ 'package_name': str, 'attribute': Field(str, is_required=False), 'location_name': Field(str, is_required=False), }, ), } WORKSPACE_CONFIG_SCHEMA = { 'load_from': [ Selector( merge_dicts( _get_target_config(), { 'python_environment': { 'executable_path': StringSource, 'target': Selector(_get_target_config()), }, }, )) ], } WORKSPACE_CONFIG_SCHEMA_WITH_LEGACY = Selector( merge_dicts( { 'repository': { 'module': Field(str, is_required=False), 'file': Field(str, is_required=False), 'fn': Field(str), },
explicit_forkserver_preload=start_cfg.get("preload_modules"), ) MULTI_PROC_CONFIG = { "max_concurrent": Field(Int, is_required=False, default_value=0), "start_method": Field( Selector({ "spawn": {}, "forkserver": { "preload_modules": Field( [str], is_required=False, description= "Explicit modules to preload in the forkserver.", ), }, # fork currently unsupported due to threads usage }), is_required=False, description= ("Select how subprocesses are created. Defaults to spawn.\n" "When forkserver is selected, set_forkserver_preload will be called with either:\n" "* the preload_modules list if provided by config\n" "* the module containing the Job if it was loaded from a module\n" "* dagster\n" "https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods" ),
assert level_two_error.error_data.field_name == 'level_two_int_field' assert not result.errors_at_level('level_two_dict', 'level_three_dict') final_level_errors = result.errors_at_level('level_two_dict', 'level_three_dict', 'level_three_string') assert len(final_level_errors) == 1 final_level_error = final_level_errors[0] assert final_level_error.reason == DagsterEvaluationErrorReason.RUNTIME_TYPE_MISMATCH ExampleSelector = Selector({ 'option_one': Field(String), 'option_two': Field(String) }) def test_example_selector_success(): result = eval_config_value_from_dagster_type(ExampleSelector, {'option_one': 'foo'}) assert result.success assert result.value == {'option_one': 'foo'} result = eval_config_value_from_dagster_type(ExampleSelector, {'option_two': 'foo'}) assert result.success assert result.value == {'option_two': 'foo'}
), } WORKSPACE_CONFIG_SCHEMA = { 'load_from': Field( [ Selector( merge_dicts( _get_target_config(), { 'python_environment': { 'executable_path': StringSource, 'target': Selector(_get_target_config()), }, 'grpc_server': { 'host': Field(str, is_required=False), 'socket': Field(str, is_required=False), 'port': Field(int, is_required=False), 'location_name': Field(str, is_required=False), }, }, )) ], is_required=False, ), 'opt_in': Field([Enum('WorkspaceOptInFeature', [EnumValue('grpc')])], is_required=False), }