def define_structured_logger(name, callback, level):
check.str_param(name, "name")
check.callable_param(callback, "callback")
level = coerce_valid_log_level(level)
return construct_single_handler_logger(name, level,
StructuredLoggerHandler(callback))
def colored_console_logger(init_context):
"""This logger provides support for sending Dagster logs to stdout in a colored format. It is
included by default on jobs which do not otherwise specify loggers.
"""
level = coerce_valid_log_level(init_context.logger_config["log_level"])
name = init_context.logger_config["name"]
klass = logging.getLoggerClass()
logger_ = klass(name, level=level)
coloredlogs.install(
logger=logger_,
level=level,
fmt=default_format_string(),
datefmt=default_date_format_string(),
field_styles={
"levelname": {
"color": "blue"
},
"asctime": {
"color": "green"
}
},
level_styles={
"debug": {},
"error": {
"color": "red"
}
},
)
return logger_
def bar_logger(init_context):
logger_ = logging.Logger("bar")
logger_.log = lambda level, msg, **kwargs: bar_logger_captured_results.append(
(level, msg))
logger_.setLevel(
coerce_valid_log_level(init_context.logger_config["log_level"]))
return logger_
def __new__(
cls,
error_info,
level,
user_message,
run_id,
timestamp,
step_key=None,
pipeline_name=None,
dagster_event=None,
job_name=None,
):
if pipeline_name and job_name:
raise DagsterInvariantViolationError(
"Provided both `pipeline_name` and `job_name` parameters to `EventLogEntry` "
"initialization. Please provide only one or the other.")
pipeline_name = pipeline_name or job_name
return super(EventLogEntry, cls).__new__(
cls,
check.opt_inst_param(error_info, "error_info",
SerializableErrorInfo),
coerce_valid_log_level(level),
check.str_param(user_message, "user_message"),
check.str_param(run_id, "run_id"),
check.float_param(timestamp, "timestamp"),
check.opt_str_param(step_key, "step_key"),
check.opt_str_param(pipeline_name, "pipeline_name"),
check.opt_inst_param(dagster_event, "dagster_event", DagsterEvent),
)
def test_single_step_resource_event_logs():
# Test to attribute logs for single-step plans which are often the representation of
# sub-plans in a multiprocessing execution environment. Most likely will need to be rewritten
# with the refactor detailed in https://github.com/dagster-io/dagster/issues/2239
USER_SOLID_MESSAGE = "I AM A SOLID"
USER_RESOURCE_MESSAGE = "I AM A RESOURCE"
events = []
def event_callback(record):
assert isinstance(record, EventLogEntry)
events.append(record)
@solid(required_resource_keys={"a"})
def resource_solid(context):
context.log.info(USER_SOLID_MESSAGE)
@resource
def resource_a(context):
context.log.info(USER_RESOURCE_MESSAGE)
return "A"
the_pipeline = PipelineDefinition(
name="resource_logging_pipeline",
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={"a": resource_a},
logger_defs={"callback": construct_event_logger(event_callback)},
)
],
)
with instance_for_test() as instance:
pipeline_run = instance.create_run_for_pipeline(
the_pipeline,
run_config={"loggers": {"callback": {}}},
solids_to_execute={"resource_solid"},
)
result = execute_run(InMemoryPipeline(the_pipeline), pipeline_run, instance)
assert result.success
log_messages = [
event
for event in events
if isinstance(event, EventLogEntry) and event.level == coerce_valid_log_level("INFO")
]
assert len(log_messages) == 2
resource_log_message = next(
iter(
[
message
for message in log_messages
if message.user_message == USER_RESOURCE_MESSAGE
]
)
)
assert resource_log_message.step_key == "resource_solid"
def define_json_file_logger(name, json_path, level):
check.str_param(name, "name")
check.str_param(json_path, "json_path")
level = coerce_valid_log_level(level)
stream_handler = JsonFileHandler(json_path)
stream_handler.setFormatter(define_default_formatter())
return construct_single_handler_logger(name, level, stream_handler)
def __new__(cls, name, message, level, meta, record):
return super(StructuredLoggerMessage, cls).__new__(
cls,
check.str_param(name, "name"),
check.str_param(message, "message"),
coerce_valid_log_level(level),
check.dict_param(meta, "meta"),
check.inst_param(record, "record", logging.LogRecord),
)
def __init__(
self,
dagster_handler: DagsterLogHandler,
level: int = logging.NOTSET,
managed_loggers: List[logging.Logger] = None,
):
super().__init__(name="dagster", level=coerce_valid_log_level(level))
self._managed_loggers = check.opt_list_param(managed_loggers,
"managed_loggers",
of_type=logging.Logger)
self._dagster_handler = dagster_handler
self.addHandler(dagster_handler)
def cloudwatch_logger(init_context):
"""This logger provides support for sending Dagster logs to AWS CloudWatch.
Example:
.. code-block:: python
from dagster import job, op
from dagster_aws.cloudwatch import cloudwatch_logger
@op
def hello_op(context):
context.log.info('Hello, Cloudwatch!')
context.log.error('This is an error')
@job(logger_defs={'cloudwatch': cloudwatch_logger})
def hello_cloudwatch():
hello_op()
hello_cloudwatch.execute_in_process(
run_config={
'loggers': {
'cloudwatch': {
'config': {
'log_group_name': '/dagster-test/test-cloudwatch-logging',
'log_stream_name': 'test-logging',
'aws_region': 'us-west-1'
}
}
}
}
)
"""
level = coerce_valid_log_level(init_context.logger_config["log_level"])
name = init_context.logger_config["name"]
klass = logging.getLoggerClass()
logger_ = klass(name, level=level)
logger_.addHandler(
CloudwatchLogsHandler(
init_context.logger_config["log_group_name"],
init_context.logger_config["log_stream_name"],
aws_region=init_context.logger_config.get("aws_region"),
aws_secret_access_key=init_context.logger_config.get(
"aws_secret_access_key"),
aws_access_key_id=init_context.logger_config.get(
"aws_access_key_id"),
))
return logger_
def log(self, level, msg, *args, **kwargs):
"""Log a message at the given level. Attributes about the context it was logged in (such as
the solid name or pipeline name) will be automatically attached to the created record.
Args:
level (str, int): either a string representing the desired log level ("INFO", "WARN"),
or an integer level such as logging.INFO or logging.DEBUG.
msg (str): the message to be logged
*args: the logged message will be msg % args
"""
level = coerce_valid_log_level(level)
# log DagsterEvents regardless of level
if self.isEnabledFor(level) or ("extra" in kwargs and DAGSTER_META_KEY in kwargs["extra"]):
self._log(level, msg, args, **kwargs)
def create(
cls,
loggers: List[logging.Logger],
handlers: List[logging.Handler] = None,
instance: Optional["DagsterInstance"] = None,
pipeline_run: Optional["PipelineRun"] = None,
) -> "DagsterLogManager":
"""Create a DagsterLogManager with a set of subservient loggers."""
handlers = check.opt_list_param(handlers,
"handlers",
of_type=logging.Handler)
managed_loggers = [get_dagster_logger()]
python_log_level = logging.NOTSET
if instance:
handlers += instance.get_handlers()
managed_loggers += [
logging.getLogger(lname)
if lname != "root" else logging.getLogger()
for lname in instance.managed_python_loggers
]
if instance.python_log_level is not None:
python_log_level = coerce_valid_log_level(
instance.python_log_level)
# set all loggers to the declared logging level
for logger in managed_loggers:
logger.setLevel(python_log_level)
if pipeline_run:
logging_metadata = DagsterLoggingMetadata(
run_id=pipeline_run.run_id,
pipeline_name=pipeline_run.pipeline_name,
pipeline_tags=pipeline_run.tags,
)
else:
logging_metadata = DagsterLoggingMetadata()
return cls(
dagster_handler=DagsterLogHandler(
logging_metadata=logging_metadata,
loggers=loggers,
handlers=handlers,
),
level=python_log_level,
managed_loggers=managed_loggers,
)
def construct_single_handler_logger(name, level, handler):
check.str_param(name, "name")
check.inst_param(handler, "handler", logging.Handler)
level = coerce_valid_log_level(level)
@logger
def single_handler_logger(_init_context):
klass = logging.getLoggerClass()
logger_ = klass(name, level=level)
logger_.addHandler(handler)
handler.setLevel(level)
return logger_
return single_handler_logger
def __new__(
cls,
name: str,
message: str,
level: int,
meta: Dict[object, object],
record: logging.LogRecord,
):
return super(StructuredLoggerMessage, cls).__new__(
cls,
check.str_param(name, "name"),
check.str_param(message, "message"),
coerce_valid_log_level(level),
check.dict_param(meta, "meta"),
check.inst_param(record, "record", logging.LogRecord),
)
def _get_result(self, data: str = None) -> DbtRpcOutput:
"""Sends a request to the dbt RPC server and continuously polls for the status of a request until the state is ``success``."""
out = super()._get_result(data)
request_token = out.result.get("request_token")
logs_start = 0
elapsed_time = -1
current_state = None
while True:
out = self.poll(
request_token=request_token,
logs=True,
logs_start=logs_start,
)
logs = out.result.get("logs", [])
for log in logs:
self.logger.log(
msg=log["message"],
level=coerce_valid_log_level(log.get("levelname", "INFO")),
extra=log.get("extra"),
)
logs_start += len(logs)
current_state = out.result.get("state")
# Stop polling if request's state is no longer "running".
if current_state != "running":
break
elapsed_time = out.result.get("elapsed", 0)
# Sleep for the configured time interval before polling again.
time.sleep(self.poll_interval)
if current_state != "success":
raise Failure(description=(
f"Request {request_token} finished with state '{current_state}' in "
f"{elapsed_time} seconds"), )
return out
def json_console_logger(init_context):
"""This logger provides support for sending Dagster logs to stdout in json format.
Example:
.. code-block:: python
from dagster import op, job
from dagster.loggers import json_console_logger
@op
def hello_op(context):
context.log.info('Hello, world!')
context.log.error('This is an error')
@job(logger_defs={'json_logger': json_console_logger})])
def json_logged_job():
hello_op()
"""
level = coerce_valid_log_level(init_context.logger_config["log_level"])
name = init_context.logger_config["name"]
klass = logging.getLoggerClass()
logger_ = klass(name, level=level)
handler = coloredlogs.StandardErrorHandler()
class JsonFormatter(logging.Formatter):
def format(self, record):
return seven.json.dumps(record.__dict__)
handler.setFormatter(JsonFormatter())
logger_.addHandler(handler)
return logger_
def test_logger(init_context):
assert init_context.logger_config == "secret testing value!!"
it["ran"] = True
logger_ = logging.Logger("test", level=coerce_valid_log_level("INFO"))
return logger_
def grpc_command(
port=None,
socket=None,
host=None,
max_workers=None,
heartbeat=False,
heartbeat_timeout=30,
lazy_load_user_code=False,
ipc_output_file=None,
fixed_server_id=None,
override_system_timezone=None,
log_level="INFO",
use_python_environment_entry_point=False,
container_context=None,
**kwargs,
):
if seven.IS_WINDOWS and port is None:
raise click.UsageError(
"You must pass a valid --port/-p on Windows: --socket/-s not supported."
)
if not (port or socket and not (port and socket)):
raise click.UsageError(
"You must pass one and only one of --port/-p or --socket/-s.")
configure_loggers(log_level=coerce_valid_log_level(log_level))
logger = logging.getLogger("dagster.code_server")
loadable_target_origin = None
if any(kwargs[key] for key in [
"attribute",
"working_directory",
"module_name",
"package_name",
"python_file",
"empty_working_directory",
]):
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
attribute=kwargs["attribute"],
working_directory=(None if kwargs.get("empty_working_directory")
else get_working_directory_from_kwargs(kwargs)),
module_name=kwargs["module_name"],
python_file=kwargs["python_file"],
package_name=kwargs["package_name"],
)
with (mock_system_timezone(override_system_timezone)
if override_system_timezone else nullcontext()):
server = DagsterGrpcServer(
port=port,
socket=socket,
host=host,
loadable_target_origin=loadable_target_origin,
max_workers=max_workers,
heartbeat=heartbeat,
heartbeat_timeout=heartbeat_timeout,
lazy_load_user_code=lazy_load_user_code,
ipc_output_file=ipc_output_file,
fixed_server_id=fixed_server_id,
entry_point=(get_python_environment_entry_point(sys.executable)
if use_python_environment_entry_point else
DEFAULT_DAGSTER_ENTRY_POINT),
container_context=json.loads(container_context)
if container_context != None else None,
)
code_desc = " "
if loadable_target_origin:
if loadable_target_origin.python_file:
code_desc = f" for file {loadable_target_origin.python_file} "
elif loadable_target_origin.package_name:
code_desc = f" for package {loadable_target_origin.package_name} "
elif loadable_target_origin.module_name:
code_desc = f" for module {loadable_target_origin.module_name} "
server_desc = (
f"Dagster code server{code_desc}on port {port} in process {os.getpid()}"
if port else
f"Dagster code server{code_desc}in process {os.getpid()}")
logger.info("Started {server_desc}".format(server_desc=server_desc))
try:
server.serve()
finally:
logger.info(
"Shutting down {server_desc}".format(server_desc=server_desc))
def test_logger(init_context):
assert init_context.logger_config["enum"] == TestPythonEnum.OTHER
it["ran test_logger"] = True
logger_ = logging.Logger("test", level=coerce_valid_log_level("INFO"))
return logger_
def int_logger(init_context):
logger_ = logging.Logger("foo")
logger_.setLevel(coerce_valid_log_level(init_context.logger_config))
return logger_
def execute_cli(
executable: str,
command: str,
flags_dict: Dict[str, Any],
log: Any,
warn_error: bool,
ignore_handled_error: bool,
target_path: str,
) -> DbtCliOutput:
"""Executes a command on the dbt CLI in a subprocess."""
check.str_param(executable, "executable")
check.str_param(command, "command")
check.dict_param(flags_dict, "flags_dict", key_type=str)
check.bool_param(warn_error, "warn_error")
check.bool_param(ignore_handled_error, "ignore_handled_error")
# Format the dbt CLI flags in the command..
warn_error = ["--warn-error"] if warn_error else []
command_list = [
executable, "--log-format", "json", *warn_error, *command.split(" ")
]
for flag, value in flags_dict.items():
if not value:
continue
command_list.append(f"--{flag}")
if isinstance(value, bool):
# If a bool flag (and is True), the presence of the flag itself is enough.
continue
if isinstance(value, list):
check.list_param(value, f"config.{flag}", of_type=str)
command_list += value
continue
if isinstance(value, dict):
command_list.append(json.dumps(value))
continue
command_list.append(str(value))
# Execute the dbt CLI command in a subprocess.
full_command = " ".join(command_list)
log.info(f"Executing command: {full_command}")
return_code = 0
logs = []
output = []
process = subprocess.Popen(command_list,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
for raw_line in process.stdout or []:
line = raw_line.decode("utf-8")
output.append(line)
try:
json_line = json.loads(line)
except json.JSONDecodeError:
log.info(line.rstrip())
else:
logs.append(json_line)
level = coerce_valid_log_level(
json_line.get("levelname", json_line.get("level", "info")))
log.log(
level,
json_line.get("message", json_line.get("msg", line.rstrip())))
process.wait()
return_code = process.returncode
log.info("dbt exited with return code {return_code}".format(
return_code=return_code))
raw_output = "\n".join(output)
if return_code == 2:
raise DagsterDbtCliFatalRuntimeError(logs=logs, raw_output=raw_output)
if return_code == 1 and not ignore_handled_error:
raise DagsterDbtCliHandledRuntimeError(logs=logs,
raw_output=raw_output)
run_results = (parse_run_results(flags_dict["project-dir"], target_path)
if command in DBT_RUN_RESULTS_COMMANDS else {})
return DbtCliOutput(
command=full_command,
return_code=return_code,
raw_output=raw_output,
logs=logs,
result=run_results,
)
def basic_logger(context):
called["basic_logger"] = context.logger_config
return logging.Logger("test", level=coerce_valid_log_level("INFO"))
