def __init__(self, task):
super(TaskCtrl, self).__init__(task)
from dbnd._core.task.task import Task
assert isinstance(task, Task)
from dbnd._core.task_ctrl.task_relations import TaskRelations # noqa: F811
from dbnd._core.task_ctrl.task_dag import _TaskDagNode # noqa: F811
from dbnd._core.task_ctrl.task_visualiser import TaskVisualiser # noqa: F811
from dbnd._core.task_ctrl.task_output_builder import TaskOutputBuilder
from dbnd._core.task_ctrl.task_dag_describe import DescribeDagCtrl
from dbnd._core.task_ctrl.task_validator import TaskValidator
self._relations = TaskRelations(task)
self.task_validator = TaskValidator(task)
self._task_dag = _TaskDagNode(task)
self.outputs = TaskOutputBuilder(task) # type: TaskOutputBuilder
self._visualiser = TaskVisualiser(task)
self.describe_dag = DescribeDagCtrl(task)
self._should_run = self.task_meta.task_enabled and self.task._should_run(
)
# will be assigned by the latest Run
self.last_task_run = None # type: Optional[TaskRun]
self.force_task_run_uid = None # force task run uid
def test_parameter_value_stays_hidden_in_banner(self):
my_task = t_f.t("test_string")
my_task.dbnd_run()
banner = TaskVisualiser(my_task).banner("some_msg")
print(banner)
assert "***" in banner and "test_string" not in banner
def test_in_memory_dump(self):
s = t_very_long_params.task(t_param="long_string" * 1000)
assert len(s.t_param) > _MAX_VALUE_SIZE * 3
actual = TaskVisualiser(s).banner("Running task")
logger.warning(actual)
assert len(actual) < _MAX_VALUE_SIZE * 3
def test_in_memory_dump(self):
s = t_very_long_params.task(t_param="long_string" * 1000)
assert len(
s.t_param) > s.settings.describe.console_value_preview_size * 3
actual = TaskVisualiser(s).banner("Running task")
logger.warning(actual)
assert len(actual) < s.settings.describe.console_value_preview_size * 3
def test_exception(self):
s = TTask(t_param="my_param")
try:
raise Exception("MyException")
except Exception:
actual = TaskVisualiser(s).banner("Runinng task",
exc_info=sys.exc_info())
assert actual
assert "MyException" in actual
def __init__(self, task):
super(_BaseTaskCtrl, self).__init__(task)
from dbnd._core.task_ctrl.task_dag import _TaskDagNode # noqa: F811
from dbnd._core.task_ctrl.task_dag_describe import DescribeDagCtrl
from dbnd._core.task_ctrl.task_descendant import (
TaskDescendants as _TaskDescendants, )
from dbnd._core.task_ctrl.task_repr import TaskRepr
from dbnd._core.task_ctrl.task_visualiser import TaskVisualiser # noqa: F811
self._task_dag = _TaskDagNode(task)
self.descendants = _TaskDescendants(task)
self._visualiser = TaskVisualiser(task)
self.describe_dag = DescribeDagCtrl(task)
self.task_repr = TaskRepr(self.task)
# will be assigned by the latest Run
self.last_task_run = None # type: Optional[TaskRun]
self.force_task_run_uid = None # force task run uid
def test_simple_dump(self):
s = TTask(t_param="my_param")
actual = TaskVisualiser(s).banner("Runinng task")
assert "my_param" in actual
def logging_status():
# type: ()->str
"""
Shows the status of the logging system
All known loggers, logging configuration and so on.
:return:
"""
report = DoctorStatusReportBuilder("Logging Status")
report.log("logging.root", logging.root)
report.log("logging.root.handlers", logging.root.handlers)
report.log("logger", logger)
report.log("logger.handlers", logger.handlers)
# airflow usually alternate stderr/stdout
report.log("sys.stderr", sys.stderr)
report.log("sys.stderr[close]", hasattr(sys.stderr, "close"))
report.log("sys.stderr", sys.__stderr__)
report.log("sys.__stderr__[close]", hasattr(sys.__stderr__, "close"))
dbnd_context = try_get_databand_context()
if dbnd_context:
from dbnd._core.task_ctrl.task_visualiser import TaskVisualiser
report.add_sub_report(
TaskVisualiser(dbnd_context.settings.log).banner("Log Config"))
# check airflow logging
try:
from logging import Logger
airflow_task_logger = Logger.manager.loggerDict.get("airflow.task")
if airflow_task_logger:
report.log("Airlfow task logger", airflow_task_logger)
report.log("Airlfow task logger handlers",
airflow_task_logger.handlers)
else:
report.log("Airlfow task logger", "not found")
except Exception as ex:
ex_msg = "Failed to get airlfow.task logger status: %s" % ex
report.log("Airflow task logger", ex_msg)
logger.exception(ex_msg)
logging_status = report.get_status_str()
logging_status = "\n{sep}\n{msg}\n{sep}s\n".format(msg=logging_status,
sep="*" * 40)
logger.info(logging_status)
# if we run this check we might have a problem with logs, we don't know how we are going to see the message
print("\n\nLogging Status (via __stderr__)%s" % logging_status,
file=sys.__stderr__)
logger.info(
"Running logging validation.. (you will see a lot of messages)")
# now we can print things, it might be that one of them will "kill the process"
# because of some weird log handlers loop
print("Message via print")
print("Message via print stderr", file=sys.stderr)
print("Message via print __stderr__", file=sys.__stderr__)
logging.info("Message via logging root")
logger.info("Message via logger")
return logging_status