def __init__( self, name=None, resource_defs=None, logger_defs=None, system_storage_defs=None, description=None, ): from .system_storage import SystemStorageDefinition, mem_system_storage, fs_system_storage self.name = check.opt_str_param(name, 'name', DEFAULT_MODE_NAME) self.resource_defs = check.opt_dict_param( resource_defs, 'resource_defs', key_type=str, value_type=ResourceDefinition) self.loggers = (check.opt_dict_param(logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition) or default_loggers()) self.system_storage_defs = check.list_param( system_storage_defs if system_storage_defs else [mem_system_storage, fs_system_storage], 'system_storage_def', of_type=SystemStorageDefinition, ) self.description = check.opt_str_param(description, 'description')
def create_creation_data(pipeline_def): return EnvironmentClassCreationData( pipeline_def.name, pipeline_def.solids, pipeline_def.dependency_structure, mode_definition=None, logger_defs=default_loggers(), )
def __new__( cls, name=None, resource_defs=None, logger_defs=None, system_storage_defs=None, executor_defs=None, description=None, intermediate_storage_defs=None, ): from dagster.core.storage.system_storage import ( default_system_storage_defs, default_intermediate_storage_defs, ) from .system_storage import SystemStorageDefinition from .intermediate_storage import IntermediateStorageDefinition if system_storage_defs is not None and intermediate_storage_defs is None: warnings.warn( "system_storage_defs are deprecated and will be removed in 0.10.0 " "and should be replaced with " "intermediate_storage_defs for intermediates and resource_defs for files" ) return super(ModeDefinition, cls).__new__( cls, name=check_for_invalid_name_and_warn(name) if name else DEFAULT_MODE_NAME, resource_defs=check.opt_dict_param(resource_defs, "resource_defs", key_type=str, value_type=ResourceDefinition), loggers=(check.opt_dict_param(logger_defs, "logger_defs", key_type=str, value_type=LoggerDefinition) or default_loggers()), system_storage_defs=check.list_param( system_storage_defs if system_storage_defs else default_system_storage_defs, "system_storage_defs", of_type=SystemStorageDefinition, ), intermediate_storage_defs=check.list_param( intermediate_storage_defs if intermediate_storage_defs else default_intermediate_storage_defs, "intermediate_storage_defs", of_type=IntermediateStorageDefinition, ), executor_defs=check.list_param( executor_defs if executor_defs else default_executors, "executor_defs", of_type=ExecutorDefinition, ), description=check.opt_str_param(description, "description"), )
def __new__( cls, name=None, resource_defs=None, logger_defs=None, system_storage_defs=None, executor_defs=None, description=None, intermediate_storage_defs=None, ): from dagster.core.storage.system_storage import ( default_system_storage_defs, default_intermediate_storage_defs, ) from .system_storage import SystemStorageDefinition from .intermediate_storage import IntermediateStorageDefinition if system_storage_defs is not None and intermediate_storage_defs is None: warnings.warn( 'system_storage_defs are deprecated and will be removed in 0.10.0 ' 'and should be replaced with ' 'intermediate_storage_defs for intermediates and resource_defs for files' ) return super(ModeDefinition, cls).__new__( cls, name=check.opt_str_param(name, 'name', DEFAULT_MODE_NAME), resource_defs=check.opt_dict_param(resource_defs, 'resource_defs', key_type=str, value_type=ResourceDefinition), loggers=(check.opt_dict_param(logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition) or default_loggers()), system_storage_defs=check.list_param( system_storage_defs if system_storage_defs else default_system_storage_defs, 'system_storage_defs', of_type=SystemStorageDefinition, ), intermediate_storage_defs=check.list_param( intermediate_storage_defs if intermediate_storage_defs else default_intermediate_storage_defs, 'intermediate_storage_defs', of_type=IntermediateStorageDefinition, ), executor_defs=check.list_param( executor_defs if executor_defs else default_executors, 'executor_defs', of_type=ExecutorDefinition, ), description=check.opt_str_param(description, 'description'), )
def create_creation_data(pipeline_def): return EnvironmentClassCreationData( pipeline_def.name, pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.mode_definition, logger_defs=default_loggers(), ignored_solids=[], required_resources=set(), )
def create_creation_data(pipeline_def): return RunConfigSchemaCreationData( pipeline_def.name, pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.mode_definition, logger_defs=default_loggers(), ignored_solids=[], required_resources=set(), is_using_graph_job_op_apis=pipeline_def.is_job, )
def __new__( cls, name: Optional[str] = None, resource_defs: Optional[Dict[str, ResourceDefinition]] = None, logger_defs: Optional[Dict[str, LoggerDefinition]] = None, executor_defs: Optional[List[ExecutorDefinition]] = None, description: Optional[str] = None, _config_mapping: Optional[ConfigMapping] = None, _partitioned_config: Optional["PartitionedConfig"] = None, ): from .partition import PartitionedConfig resource_defs = check.opt_dict_param(resource_defs, "resource_defs", key_type=str, value_type=ResourceDefinition) for key in resource_defs: if not key.isidentifier(): check.failed( f"Resource key '{key}' must be a valid Python identifier.") if resource_defs and "io_manager" in resource_defs: resource_defs_with_defaults = resource_defs else: from dagster.core.storage.mem_io_manager import mem_io_manager resource_defs_with_defaults = merge_dicts( {"io_manager": mem_io_manager}, resource_defs or {}) return super(ModeDefinition, cls).__new__( cls, name=check_valid_name(name) if name else DEFAULT_MODE_NAME, resource_defs=resource_defs_with_defaults, loggers=(check.opt_dict_param(logger_defs, "logger_defs", key_type=str, value_type=LoggerDefinition) or default_loggers()), executor_defs=check.list_param( executor_defs if executor_defs else default_executors, "executor_defs", of_type=ExecutorDefinition, ), description=check.opt_str_param(description, "description"), config_mapping=check.opt_inst_param(_config_mapping, "_config_mapping", ConfigMapping), partitioned_config=check.opt_inst_param(_partitioned_config, "_partitioned_config", PartitionedConfig), )
def create_log_manager( context_creation_data: ContextCreationData) -> DagsterLogManager: check.inst_param(context_creation_data, "context_creation_data", ContextCreationData) pipeline_def, mode_def, environment_config, pipeline_run = ( context_creation_data.pipeline_def, context_creation_data.mode_def, context_creation_data.environment_config, context_creation_data.pipeline_run, ) # The following logic is tightly coupled to the processing of logger config in # python_modules/dagster/dagster/core/system_config/objects.py#config_map_loggers # Changes here should be accompanied checked against that function, which applies config mapping # via ConfigurableDefinition (@configured) to incoming logger configs. See docstring for more details. loggers = [] for logger_key, logger_def in mode_def.loggers.items() or default_loggers( ).items(): if logger_key in environment_config.loggers: loggers.append( logger_def.logger_fn( InitLoggerContext( environment_config.loggers.get(logger_key, {}).get("config"), logger_def, pipeline_def=pipeline_def, run_id=pipeline_run.run_id, ))) if not loggers: for (logger_def, logger_config) in default_system_loggers(): loggers.append( logger_def.logger_fn( InitLoggerContext( logger_config, logger_def, pipeline_def=pipeline_def, run_id=pipeline_run.run_id, ))) # should this be first in loggers list? loggers.append(context_creation_data.instance.get_logger()) return DagsterLogManager( run_id=pipeline_run.run_id, logging_tags=get_logging_tags(pipeline_run), loggers=loggers, )
def create_log_manager(context_creation_data): check.inst_param(context_creation_data, 'context_creation_data', ContextCreationData) pipeline_def, mode_def, environment_config, run_config = ( context_creation_data.pipeline_def, context_creation_data.mode_def, context_creation_data.environment_config, context_creation_data.run_config, ) loggers = [] for logger_key, logger_def in mode_def.loggers.items() or default_loggers( ).items(): if logger_key in environment_config.loggers: loggers.append( logger_def.logger_fn( InitLoggerContext( environment_config.loggers.get(logger_key, {}).get('config'), pipeline_def, logger_def, run_config.run_id, ))) if run_config.loggers: for logger in run_config.loggers: loggers.append(logger) if not loggers: for (logger_def, logger_config) in default_system_loggers(): loggers.append( logger_def.logger_fn( InitLoggerContext(logger_config, pipeline_def, logger_def, run_config.run_id))) if run_config.event_callback: init_logger_context = InitLoggerContext({}, pipeline_def, logger_def, run_config.run_id) loggers.append( construct_event_logger( run_config.event_callback).logger_fn(init_logger_context)) return DagsterLogManager( run_id=run_config.run_id, logging_tags=get_logging_tags(context_creation_data.run_config, context_creation_data.pipeline_def), loggers=loggers, )
def __init__(self, name=DEFAULT_MODE_NAME, resources=None, loggers=None, description=None): self.name = check.str_param(name, 'name') self.resource_defs = check.opt_dict_param( resources, 'resources', key_type=str, value_type=ResourceDefinition) self.loggers = (check.opt_dict_param( loggers, 'loggers', key_type=str, value_type=LoggerDefinition) or default_loggers()) self.description = check.opt_str_param(description, 'description')
def __new__( cls, name=None, resource_defs=None, logger_defs=None, executor_defs=None, description=None, intermediate_storage_defs=None, ): from dagster.core.storage.system_storage import default_intermediate_storage_defs from .intermediate_storage import IntermediateStorageDefinition check.opt_dict_param(resource_defs, "resource_defs", key_type=str, value_type=ResourceDefinition) if resource_defs and "io_manager" in resource_defs: resource_defs_with_defaults = resource_defs else: from dagster.core.storage.mem_io_manager import mem_io_manager resource_defs_with_defaults = merge_dicts( {"io_manager": mem_io_manager}, resource_defs or {}) return super(ModeDefinition, cls).__new__( cls, name=check_valid_name(name) if name else DEFAULT_MODE_NAME, resource_defs=resource_defs_with_defaults, loggers=(check.opt_dict_param(logger_defs, "logger_defs", key_type=str, value_type=LoggerDefinition) or default_loggers()), intermediate_storage_defs=check.list_param( intermediate_storage_defs if intermediate_storage_defs else default_intermediate_storage_defs, "intermediate_storage_defs", of_type=IntermediateStorageDefinition, ), executor_defs=check.list_param( executor_defs if executor_defs else default_executors, "executor_defs", of_type=ExecutorDefinition, ), description=check.opt_str_param(description, "description"), )
def create_log_manager(context_creation_data): check.inst_param(context_creation_data, 'context_creation_data', ContextCreationData) pipeline_def, mode_def, environment_config, run_config = ( context_creation_data.pipeline_def, context_creation_data.mode_def, context_creation_data.environment_config, context_creation_data.run_config, ) loggers = [] for logger_key, logger_def in mode_def.loggers.items() or default_loggers().items(): if logger_key in environment_config.loggers: loggers.append( logger_def.logger_fn( InitLoggerContext( environment_config.loggers.get(logger_key, {}).get('config'), pipeline_def, logger_def, run_config.run_id, ) ) ) if not loggers: for (logger_def, logger_config) in default_system_loggers(): loggers.append( logger_def.logger_fn( InitLoggerContext(logger_config, pipeline_def, logger_def, run_config.run_id) ) ) # should this be first in loggers list? loggers.append(context_creation_data.instance.get_event_listener()) return DagsterLogManager( run_id=run_config.run_id, logging_tags=get_logging_tags( context_creation_data.run_config, context_creation_data.pipeline_def ), loggers=loggers, )
def _create_loggers(environment_config, run_config, pipeline_def, mode_def): check.inst_param(environment_config, 'environment_config', EnvironmentConfig) check.inst_param(run_config, 'run_config', RunConfig) check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) check.inst_param(mode_def, 'mode_def', ModeDefinition) loggers = [] for logger_key, logger_def in mode_def.loggers.items() or default_loggers().items(): if logger_key in environment_config.loggers: loggers.append( logger_def.logger_fn( InitLoggerContext( environment_config.loggers.get(logger_key, {}).get('config'), pipeline_def, logger_def, run_config.run_id, ) ) ) if run_config.loggers: for logger in run_config.loggers: loggers.append(logger) if not loggers: for (logger_def, logger_config) in default_system_loggers(): loggers.append( logger_def.logger_fn( InitLoggerContext(logger_config, pipeline_def, logger_def, run_config.run_id) ) ) if run_config.event_callback: init_logger_context = InitLoggerContext({}, pipeline_def, logger_def, run_config.run_id) loggers.append( construct_event_logger(run_config.event_callback).logger_fn(init_logger_context) ) return loggers
def __new__( cls, name=None, resource_defs=None, logger_defs=None, system_storage_defs=None, executor_defs=None, description=None, ): from dagster.core.storage.system_storage import default_system_storage_defs from .system_storage import SystemStorageDefinition return super(ModeDefinition, cls).__new__( cls, name=check.opt_str_param(name, 'name', DEFAULT_MODE_NAME), resource_defs=check.opt_dict_param(resource_defs, 'resource_defs', key_type=str, value_type=ResourceDefinition), loggers=(check.opt_dict_param(logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition) or default_loggers()), system_storage_defs=check.list_param( system_storage_defs if system_storage_defs else default_system_storage_defs, 'system_storage_defs', of_type=SystemStorageDefinition, ), executor_defs=check.list_param( executor_defs if executor_defs else default_executors, 'executor_defs', of_type=ExecutorDefinition, ), description=check.opt_str_param(description, 'description'), )
def __new__( cls, name: Optional[str] = None, resource_defs: Optional[Dict[str, ResourceDefinition]] = None, logger_defs: Optional[Dict[str, LoggerDefinition]] = None, executor_defs: Optional[List[ExecutorDefinition]] = None, description: Optional[str] = None, intermediate_storage_defs: Optional[ List["IntermediateStorageDefinition"]] = None, _config_mapping: Optional[ConfigMapping] = None, _partitioned_config: Optional["PartitionedConfig"] = None, ): from dagster.core.storage.system_storage import default_intermediate_storage_defs from .intermediate_storage import IntermediateStorageDefinition from .partition import PartitionedConfig check.opt_dict_param(resource_defs, "resource_defs", key_type=str, value_type=ResourceDefinition) if resource_defs and "io_manager" in resource_defs: resource_defs_with_defaults = resource_defs else: from dagster.core.storage.mem_io_manager import mem_io_manager resource_defs_with_defaults = merge_dicts( {"io_manager": mem_io_manager}, resource_defs or {}) if _config_mapping: experimental_arg_warning("_config_mapping", "ModeDefinition.__new__") if _partitioned_config: experimental_arg_warning("_partitioned_config", "ModeDefinition.__new__") return super(ModeDefinition, cls).__new__( cls, name=check_valid_name(name) if name else DEFAULT_MODE_NAME, resource_defs=resource_defs_with_defaults, loggers=(check.opt_dict_param(logger_defs, "logger_defs", key_type=str, value_type=LoggerDefinition) or default_loggers()), intermediate_storage_defs=check.list_param( intermediate_storage_defs if intermediate_storage_defs else default_intermediate_storage_defs, "intermediate_storage_defs", of_type=IntermediateStorageDefinition, ), executor_defs=check.list_param( executor_defs if executor_defs else default_executors, "executor_defs", of_type=ExecutorDefinition, ), description=check.opt_str_param(description, "description"), config_mapping=check.opt_inst_param(_config_mapping, "_config_mapping", ConfigMapping), partitioned_config=check.opt_inst_param(_partitioned_config, "_partitioned_config", PartitionedConfig), )