def define_structured_logger(name, callback, level): check.str_param(name, "name") check.callable_param(callback, "callback") level = coerce_valid_log_level(level) return construct_single_handler_logger(name, level, StructuredLoggerHandler(callback))
def colored_console_logger(init_context): """This logger provides support for sending Dagster logs to stdout in a colored format. It is included by default on jobs which do not otherwise specify loggers. """ level = coerce_valid_log_level(init_context.logger_config["log_level"]) name = init_context.logger_config["name"] klass = logging.getLoggerClass() logger_ = klass(name, level=level) coloredlogs.install( logger=logger_, level=level, fmt=default_format_string(), datefmt=default_date_format_string(), field_styles={ "levelname": { "color": "blue" }, "asctime": { "color": "green" } }, level_styles={ "debug": {}, "error": { "color": "red" } }, ) return logger_
def bar_logger(init_context): logger_ = logging.Logger("bar") logger_.log = lambda level, msg, **kwargs: bar_logger_captured_results.append( (level, msg)) logger_.setLevel( coerce_valid_log_level(init_context.logger_config["log_level"])) return logger_
def __new__( cls, error_info, level, user_message, run_id, timestamp, step_key=None, pipeline_name=None, dagster_event=None, job_name=None, ): if pipeline_name and job_name: raise DagsterInvariantViolationError( "Provided both `pipeline_name` and `job_name` parameters to `EventLogEntry` " "initialization. Please provide only one or the other.") pipeline_name = pipeline_name or job_name return super(EventLogEntry, cls).__new__( cls, check.opt_inst_param(error_info, "error_info", SerializableErrorInfo), coerce_valid_log_level(level), check.str_param(user_message, "user_message"), check.str_param(run_id, "run_id"), check.float_param(timestamp, "timestamp"), check.opt_str_param(step_key, "step_key"), check.opt_str_param(pipeline_name, "pipeline_name"), check.opt_inst_param(dagster_event, "dagster_event", DagsterEvent), )
def test_single_step_resource_event_logs(): # Test to attribute logs for single-step plans which are often the representation of # sub-plans in a multiprocessing execution environment. Most likely will need to be rewritten # with the refactor detailed in https://github.com/dagster-io/dagster/issues/2239 USER_SOLID_MESSAGE = "I AM A SOLID" USER_RESOURCE_MESSAGE = "I AM A RESOURCE" events = [] def event_callback(record): assert isinstance(record, EventLogEntry) events.append(record) @solid(required_resource_keys={"a"}) def resource_solid(context): context.log.info(USER_SOLID_MESSAGE) @resource def resource_a(context): context.log.info(USER_RESOURCE_MESSAGE) return "A" the_pipeline = PipelineDefinition( name="resource_logging_pipeline", solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={"a": resource_a}, logger_defs={"callback": construct_event_logger(event_callback)}, ) ], ) with instance_for_test() as instance: pipeline_run = instance.create_run_for_pipeline( the_pipeline, run_config={"loggers": {"callback": {}}}, solids_to_execute={"resource_solid"}, ) result = execute_run(InMemoryPipeline(the_pipeline), pipeline_run, instance) assert result.success log_messages = [ event for event in events if isinstance(event, EventLogEntry) and event.level == coerce_valid_log_level("INFO") ] assert len(log_messages) == 2 resource_log_message = next( iter( [ message for message in log_messages if message.user_message == USER_RESOURCE_MESSAGE ] ) ) assert resource_log_message.step_key == "resource_solid"
def define_json_file_logger(name, json_path, level): check.str_param(name, "name") check.str_param(json_path, "json_path") level = coerce_valid_log_level(level) stream_handler = JsonFileHandler(json_path) stream_handler.setFormatter(define_default_formatter()) return construct_single_handler_logger(name, level, stream_handler)
def __new__(cls, name, message, level, meta, record): return super(StructuredLoggerMessage, cls).__new__( cls, check.str_param(name, "name"), check.str_param(message, "message"), coerce_valid_log_level(level), check.dict_param(meta, "meta"), check.inst_param(record, "record", logging.LogRecord), )
def __init__( self, dagster_handler: DagsterLogHandler, level: int = logging.NOTSET, managed_loggers: List[logging.Logger] = None, ): super().__init__(name="dagster", level=coerce_valid_log_level(level)) self._managed_loggers = check.opt_list_param(managed_loggers, "managed_loggers", of_type=logging.Logger) self._dagster_handler = dagster_handler self.addHandler(dagster_handler)
def cloudwatch_logger(init_context): """This logger provides support for sending Dagster logs to AWS CloudWatch. Example: .. code-block:: python from dagster import job, op from dagster_aws.cloudwatch import cloudwatch_logger @op def hello_op(context): context.log.info('Hello, Cloudwatch!') context.log.error('This is an error') @job(logger_defs={'cloudwatch': cloudwatch_logger}) def hello_cloudwatch(): hello_op() hello_cloudwatch.execute_in_process( run_config={ 'loggers': { 'cloudwatch': { 'config': { 'log_group_name': '/dagster-test/test-cloudwatch-logging', 'log_stream_name': 'test-logging', 'aws_region': 'us-west-1' } } } } ) """ level = coerce_valid_log_level(init_context.logger_config["log_level"]) name = init_context.logger_config["name"] klass = logging.getLoggerClass() logger_ = klass(name, level=level) logger_.addHandler( CloudwatchLogsHandler( init_context.logger_config["log_group_name"], init_context.logger_config["log_stream_name"], aws_region=init_context.logger_config.get("aws_region"), aws_secret_access_key=init_context.logger_config.get( "aws_secret_access_key"), aws_access_key_id=init_context.logger_config.get( "aws_access_key_id"), )) return logger_
def log(self, level, msg, *args, **kwargs): """Log a message at the given level. Attributes about the context it was logged in (such as the solid name or pipeline name) will be automatically attached to the created record. Args: level (str, int): either a string representing the desired log level ("INFO", "WARN"), or an integer level such as logging.INFO or logging.DEBUG. msg (str): the message to be logged *args: the logged message will be msg % args """ level = coerce_valid_log_level(level) # log DagsterEvents regardless of level if self.isEnabledFor(level) or ("extra" in kwargs and DAGSTER_META_KEY in kwargs["extra"]): self._log(level, msg, args, **kwargs)
def create( cls, loggers: List[logging.Logger], handlers: List[logging.Handler] = None, instance: Optional["DagsterInstance"] = None, pipeline_run: Optional["PipelineRun"] = None, ) -> "DagsterLogManager": """Create a DagsterLogManager with a set of subservient loggers.""" handlers = check.opt_list_param(handlers, "handlers", of_type=logging.Handler) managed_loggers = [get_dagster_logger()] python_log_level = logging.NOTSET if instance: handlers += instance.get_handlers() managed_loggers += [ logging.getLogger(lname) if lname != "root" else logging.getLogger() for lname in instance.managed_python_loggers ] if instance.python_log_level is not None: python_log_level = coerce_valid_log_level( instance.python_log_level) # set all loggers to the declared logging level for logger in managed_loggers: logger.setLevel(python_log_level) if pipeline_run: logging_metadata = DagsterLoggingMetadata( run_id=pipeline_run.run_id, pipeline_name=pipeline_run.pipeline_name, pipeline_tags=pipeline_run.tags, ) else: logging_metadata = DagsterLoggingMetadata() return cls( dagster_handler=DagsterLogHandler( logging_metadata=logging_metadata, loggers=loggers, handlers=handlers, ), level=python_log_level, managed_loggers=managed_loggers, )
def construct_single_handler_logger(name, level, handler): check.str_param(name, "name") check.inst_param(handler, "handler", logging.Handler) level = coerce_valid_log_level(level) @logger def single_handler_logger(_init_context): klass = logging.getLoggerClass() logger_ = klass(name, level=level) logger_.addHandler(handler) handler.setLevel(level) return logger_ return single_handler_logger
def __new__( cls, name: str, message: str, level: int, meta: Dict[object, object], record: logging.LogRecord, ): return super(StructuredLoggerMessage, cls).__new__( cls, check.str_param(name, "name"), check.str_param(message, "message"), coerce_valid_log_level(level), check.dict_param(meta, "meta"), check.inst_param(record, "record", logging.LogRecord), )
def _get_result(self, data: str = None) -> DbtRpcOutput: """Sends a request to the dbt RPC server and continuously polls for the status of a request until the state is ``success``.""" out = super()._get_result(data) request_token = out.result.get("request_token") logs_start = 0 elapsed_time = -1 current_state = None while True: out = self.poll( request_token=request_token, logs=True, logs_start=logs_start, ) logs = out.result.get("logs", []) for log in logs: self.logger.log( msg=log["message"], level=coerce_valid_log_level(log.get("levelname", "INFO")), extra=log.get("extra"), ) logs_start += len(logs) current_state = out.result.get("state") # Stop polling if request's state is no longer "running". if current_state != "running": break elapsed_time = out.result.get("elapsed", 0) # Sleep for the configured time interval before polling again. time.sleep(self.poll_interval) if current_state != "success": raise Failure(description=( f"Request {request_token} finished with state '{current_state}' in " f"{elapsed_time} seconds"), ) return out
def json_console_logger(init_context): """This logger provides support for sending Dagster logs to stdout in json format. Example: .. code-block:: python from dagster import op, job from dagster.loggers import json_console_logger @op def hello_op(context): context.log.info('Hello, world!') context.log.error('This is an error') @job(logger_defs={'json_logger': json_console_logger})]) def json_logged_job(): hello_op() """ level = coerce_valid_log_level(init_context.logger_config["log_level"]) name = init_context.logger_config["name"] klass = logging.getLoggerClass() logger_ = klass(name, level=level) handler = coloredlogs.StandardErrorHandler() class JsonFormatter(logging.Formatter): def format(self, record): return seven.json.dumps(record.__dict__) handler.setFormatter(JsonFormatter()) logger_.addHandler(handler) return logger_
def test_logger(init_context): assert init_context.logger_config == "secret testing value!!" it["ran"] = True logger_ = logging.Logger("test", level=coerce_valid_log_level("INFO")) return logger_
def grpc_command( port=None, socket=None, host=None, max_workers=None, heartbeat=False, heartbeat_timeout=30, lazy_load_user_code=False, ipc_output_file=None, fixed_server_id=None, override_system_timezone=None, log_level="INFO", use_python_environment_entry_point=False, container_context=None, **kwargs, ): if seven.IS_WINDOWS and port is None: raise click.UsageError( "You must pass a valid --port/-p on Windows: --socket/-s not supported." ) if not (port or socket and not (port and socket)): raise click.UsageError( "You must pass one and only one of --port/-p or --socket/-s.") configure_loggers(log_level=coerce_valid_log_level(log_level)) logger = logging.getLogger("dagster.code_server") loadable_target_origin = None if any(kwargs[key] for key in [ "attribute", "working_directory", "module_name", "package_name", "python_file", "empty_working_directory", ]): loadable_target_origin = LoadableTargetOrigin( executable_path=sys.executable, attribute=kwargs["attribute"], working_directory=(None if kwargs.get("empty_working_directory") else get_working_directory_from_kwargs(kwargs)), module_name=kwargs["module_name"], python_file=kwargs["python_file"], package_name=kwargs["package_name"], ) with (mock_system_timezone(override_system_timezone) if override_system_timezone else nullcontext()): server = DagsterGrpcServer( port=port, socket=socket, host=host, loadable_target_origin=loadable_target_origin, max_workers=max_workers, heartbeat=heartbeat, heartbeat_timeout=heartbeat_timeout, lazy_load_user_code=lazy_load_user_code, ipc_output_file=ipc_output_file, fixed_server_id=fixed_server_id, entry_point=(get_python_environment_entry_point(sys.executable) if use_python_environment_entry_point else DEFAULT_DAGSTER_ENTRY_POINT), container_context=json.loads(container_context) if container_context != None else None, ) code_desc = " " if loadable_target_origin: if loadable_target_origin.python_file: code_desc = f" for file {loadable_target_origin.python_file} " elif loadable_target_origin.package_name: code_desc = f" for package {loadable_target_origin.package_name} " elif loadable_target_origin.module_name: code_desc = f" for module {loadable_target_origin.module_name} " server_desc = ( f"Dagster code server{code_desc}on port {port} in process {os.getpid()}" if port else f"Dagster code server{code_desc}in process {os.getpid()}") logger.info("Started {server_desc}".format(server_desc=server_desc)) try: server.serve() finally: logger.info( "Shutting down {server_desc}".format(server_desc=server_desc))
def test_logger(init_context): assert init_context.logger_config["enum"] == TestPythonEnum.OTHER it["ran test_logger"] = True logger_ = logging.Logger("test", level=coerce_valid_log_level("INFO")) return logger_
def int_logger(init_context): logger_ = logging.Logger("foo") logger_.setLevel(coerce_valid_log_level(init_context.logger_config)) return logger_
def execute_cli( executable: str, command: str, flags_dict: Dict[str, Any], log: Any, warn_error: bool, ignore_handled_error: bool, target_path: str, ) -> DbtCliOutput: """Executes a command on the dbt CLI in a subprocess.""" check.str_param(executable, "executable") check.str_param(command, "command") check.dict_param(flags_dict, "flags_dict", key_type=str) check.bool_param(warn_error, "warn_error") check.bool_param(ignore_handled_error, "ignore_handled_error") # Format the dbt CLI flags in the command.. warn_error = ["--warn-error"] if warn_error else [] command_list = [ executable, "--log-format", "json", *warn_error, *command.split(" ") ] for flag, value in flags_dict.items(): if not value: continue command_list.append(f"--{flag}") if isinstance(value, bool): # If a bool flag (and is True), the presence of the flag itself is enough. continue if isinstance(value, list): check.list_param(value, f"config.{flag}", of_type=str) command_list += value continue if isinstance(value, dict): command_list.append(json.dumps(value)) continue command_list.append(str(value)) # Execute the dbt CLI command in a subprocess. full_command = " ".join(command_list) log.info(f"Executing command: {full_command}") return_code = 0 logs = [] output = [] process = subprocess.Popen(command_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for raw_line in process.stdout or []: line = raw_line.decode("utf-8") output.append(line) try: json_line = json.loads(line) except json.JSONDecodeError: log.info(line.rstrip()) else: logs.append(json_line) level = coerce_valid_log_level( json_line.get("levelname", json_line.get("level", "info"))) log.log( level, json_line.get("message", json_line.get("msg", line.rstrip()))) process.wait() return_code = process.returncode log.info("dbt exited with return code {return_code}".format( return_code=return_code)) raw_output = "\n".join(output) if return_code == 2: raise DagsterDbtCliFatalRuntimeError(logs=logs, raw_output=raw_output) if return_code == 1 and not ignore_handled_error: raise DagsterDbtCliHandledRuntimeError(logs=logs, raw_output=raw_output) run_results = (parse_run_results(flags_dict["project-dir"], target_path) if command in DBT_RUN_RESULTS_COMMANDS else {}) return DbtCliOutput( command=full_command, return_code=return_code, raw_output=raw_output, logs=logs, result=run_results, )
def basic_logger(context): called["basic_logger"] = context.logger_config return logging.Logger("test", level=coerce_valid_log_level("INFO"))