def build(pipeline_def, run_config=None, mode=None):
'''This method validates a given run config against the pipeline config schema. If
successful, we instantiate an EnvironmentConfig object.
In case the run_config is invalid, this method raises a DagsterInvalidConfigError
'''
from dagster.config.validate import process_config
from .composite_descent import composite_descent
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
run_config = check.opt_dict_param(run_config, 'run_config')
check.opt_str_param(mode, 'mode')
mode = mode or pipeline_def.get_default_mode_name()
environment_type = create_environment_type(pipeline_def, mode)
config_evr = process_config(environment_type, run_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
'Error in config for pipeline {}'.format(pipeline_def.name),
config_evr.errors,
run_config,
)
config_value = config_evr.value
mode_def = pipeline_def.get_mode_definition(mode)
resource_configs = config_value.get('resources', {})
processed_resource_configs = {}
for resource_key, resource_def in mode_def.resource_defs.items():
resource_config = resource_configs.get(resource_key, {})
resource_config_evr = resource_def.process_config(resource_config)
if not resource_config_evr.success:
raise DagsterInvalidConfigError(
'Error in config for resource {}'.format(resource_key),
resource_config_evr.errors,
resource_config,
)
else:
processed_resource_configs[
resource_key] = resource_config_evr.value
solid_config_dict = composite_descent(pipeline_def,
config_value.get('solids', {}))
# TODO: replace this with a simple call to from_dict of the config.get when ready to fully deprecate
temp_intermed = config_value.get('intermediate_storage')
if config_value.get('storage'):
if temp_intermed is None:
temp_intermed = {EmptyIntermediateStoreBackcompatConfig(): {}}
return EnvironmentConfig(
solids=solid_config_dict,
execution=ExecutionConfig.from_dict(config_value.get('execution')),
storage=StorageConfig.from_dict(config_value.get('storage')),
intermediate_storage=IntermediateStorageConfig.from_dict(
temp_intermed),
loggers=config_value.get('loggers'),
original_config_dict=run_config,
resources=processed_resource_configs,
)
def build(pipeline_def, environment_dict=None, mode=None):
'''This method validates a given environment dict against the pipeline config schema. If
successful, we instantiate an EnvironmentConfig object.
In case the environment_dict is invalid, this method raises a DagsterInvalidConfigError
'''
from dagster.config.validate import process_config
from .composite_descent import composite_descent
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
environment_dict = check.opt_dict_param(environment_dict,
'environment_dict')
check.opt_str_param(mode, 'mode')
mode = mode or pipeline_def.get_default_mode_name()
environment_type = create_environment_type(pipeline_def, mode)
config_evr = process_config(environment_type, environment_dict)
if not config_evr.success:
raise DagsterInvalidConfigError(
'Error in config for pipeline {}'.format(pipeline_def.name),
config_evr.errors,
environment_dict,
)
config_value = config_evr.value
solid_config_dict = composite_descent(pipeline_def,
config_value.get('solids', {}))
return EnvironmentConfig(
solids=solid_config_dict,
execution=ExecutionConfig.from_dict(config_value.get('execution')),
storage=StorageConfig.from_dict(config_value.get('storage')),
loggers=config_value.get('loggers'),
original_config_dict=environment_dict,
resources=config_value.get('resources'),
)
def build(pipeline_def, run_config=None, mode=None):
"""This method validates a given run config against the pipeline config schema. If
successful, we instantiate an EnvironmentConfig object.
In case the run_config is invalid, this method raises a DagsterInvalidConfigError
"""
from dagster.config.validate import process_config
from dagster.core.definitions.executor import ExecutorDefinition
from dagster.core.definitions.intermediate_storage import IntermediateStorageDefinition
from .composite_descent import composite_descent
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
run_config = check.opt_dict_param(run_config, "run_config")
check.opt_str_param(mode, "mode")
mode = mode or pipeline_def.get_default_mode_name()
environment_type = create_environment_type(pipeline_def, mode)
config_evr = process_config(
environment_type, run_config_storage_field_backcompat(run_config))
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for pipeline {}".format(pipeline_def.name),
config_evr.errors,
run_config,
)
config_value = config_evr.value
mode_def = pipeline_def.get_mode_definition(mode)
config_mapped_intermediate_storage_configs = config_map_objects(
config_value,
mode_def.intermediate_storage_defs,
"intermediate_storage",
IntermediateStorageDefinition,
"intermediate storage",
)
config_mapped_execution_configs = config_map_objects(
config_value, mode_def.executor_defs, "execution",
ExecutorDefinition, "executor")
config_mapped_resource_configs = config_map_resources(
pipeline_def, config_value, mode)
config_mapped_logger_configs = config_map_loggers(
pipeline_def, config_value, mode)
solid_config_dict = composite_descent(pipeline_def,
config_value.get("solids", {}),
mode_def.resource_defs)
return EnvironmentConfig(
solids=solid_config_dict,
execution=ExecutionConfig.from_dict(
config_mapped_execution_configs),
intermediate_storage=IntermediateStorageConfig.from_dict(
config_mapped_intermediate_storage_configs),
loggers=config_mapped_logger_configs,
original_config_dict=run_config,
resources=config_mapped_resource_configs,
mode=mode,
)
def build(pipeline_def, run_config=None, mode=None):
"""This method validates a given run config against the pipeline config schema. If
successful, we instantiate an EnvironmentConfig object.
In case the run_config is invalid, this method raises a DagsterInvalidConfigError
"""
from dagster.config.validate import process_config
from dagster.core.definitions.executor import ExecutorDefinition
from dagster.core.definitions.intermediate_storage import IntermediateStorageDefinition
from dagster.core.definitions.system_storage import SystemStorageDefinition
from .composite_descent import composite_descent
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
run_config = check.opt_dict_param(run_config, "run_config")
check.opt_str_param(mode, "mode")
mode = mode or pipeline_def.get_default_mode_name()
environment_type = create_environment_type(pipeline_def, mode)
config_evr = process_config(environment_type, run_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for pipeline {}".format(pipeline_def.name),
config_evr.errors,
run_config,
)
config_value = config_evr.value
mode_def = pipeline_def.get_mode_definition(mode)
config_mapped_intermediate_storage_configs = config_map_objects(
config_value,
mode_def.intermediate_storage_defs,
"intermediate_storage",
IntermediateStorageDefinition,
"intermediate storage",
)
# TODO: replace this with a simple call to from_dict of config_mapped_intermediate_storage_configs when ready to fully deprecate
# TODO: tracking: https://github.com/dagster-io/dagster/issues/2705
temp_intermed = config_mapped_intermediate_storage_configs
if config_value.get("storage") and temp_intermed is None:
temp_intermed = {EmptyIntermediateStoreBackcompatConfig(): {}}
config_mapped_execution_configs = config_map_objects(
config_value, mode_def.executor_defs, "execution",
ExecutorDefinition, "executor")
config_mapped_system_storage_configs = config_map_objects(
config_value,
mode_def.system_storage_defs,
"storage",
SystemStorageDefinition,
"system storage",
)
config_mapped_resource_configs = config_map_resources(
pipeline_def, config_value, mode)
config_mapped_logger_configs = config_map_loggers(
pipeline_def, config_value, mode)
solid_config_dict = composite_descent(pipeline_def,
config_value.get("solids", {}))
return EnvironmentConfig(
solids=solid_config_dict,
execution=ExecutionConfig.from_dict(
config_mapped_execution_configs),
storage=StorageConfig.from_dict(
config_mapped_system_storage_configs),
intermediate_storage=IntermediateStorageConfig.from_dict(
temp_intermed),
loggers=config_mapped_logger_configs,
original_config_dict=run_config,
resources=config_mapped_resource_configs,
)
