def test_run_naive_taskinstance(self, mock_local_job):
"""
Test that we can run naive (non-localized) task instances
"""
naive_date = datetime(2016, 1, 1)
dag_id = 'test_run_ignores_all_dependencies'
dag = self.dagbag.get_dag('test_run_ignores_all_dependencies')
task0_id = 'test_run_dependent_task'
args0 = [
'tasks',
'run',
'--ignore-all-dependencies',
'--local',
dag_id,
task0_id,
naive_date.isoformat(),
]
task_command.task_run(self.parser.parse_args(args0), dag=dag)
mock_local_job.assert_called_once_with(
task_instance=mock.ANY,
mark_success=False,
ignore_all_deps=True,
ignore_depends_on_past=False,
ignore_task_deps=False,
ignore_ti_state=False,
pickle_id=None,
pool=None,
)
def test_disable_handler_modifying(self, run_task_mock):
"""If [core] donot_modify_handlers is set to True, the root logger is untouched"""
from airflow import settings
logger = logging.getLogger("foo.bar")
def task_inner(*args, **kwargs):
logger.warning("not redirected")
run_task_mock.side_effect = task_inner
config = {
('core', 'dags_folder'): self.dag_path,
('logging', 'logging_level'): "INFO",
}
old_value = settings.DONOT_MODIFY_HANDLERS
settings.DONOT_MODIFY_HANDLERS = True
with conf_vars(config):
with self.assertLogs(level=logging.WARNING) as captured:
task_command.task_run(self.parser.parse_args(self.task_args))
assert captured.output == ["WARNING:foo.bar:not redirected"]
settings.DONOT_MODIFY_HANDLERS = old_value
def test_log_file_template_with_run_task(self):
"""Verify that the taskinstance has the right context for log_filename_template"""
with mock.patch.object(task_command, "_run_task_by_selected_method"):
with conf_vars({('core', 'dags_folder'): self.dag_path}):
# increment the try_number of the task to be run
dag = DagBag().get_dag(self.dag_id)
task = dag.get_task(self.task_id)
with create_session() as session:
dag.create_dagrun(
execution_date=self.execution_date,
start_date=timezone.utcnow(),
state=State.RUNNING,
run_type=DagRunType.MANUAL,
session=session,
)
ti = TaskInstance(task, self.execution_date)
ti.refresh_from_db(session=session, lock_for_update=True)
ti.try_number = 1 # not running, so starts at 0
session.merge(ti)
log_file_path = os.path.join(
os.path.dirname(self.ti_log_file_path), "2.log")
try:
task_command.task_run(
self.parser.parse_args(self.task_args))
assert os.path.exists(log_file_path)
finally:
try:
os.remove(log_file_path)
except OSError:
pass
def test_logging_with_run_task(self):
# We are not using self.assertLogs as we want to verify what actually is stored in the Log file
# as that is what gets displayed
with conf_vars({('core', 'dags_folder'): os.path.join(ROOT_FOLDER, f"tests/dags/{self.dag_id}")}):
task_command.task_run(self.parser.parse_args([
'tasks', 'run', self.dag_id, self.task_id, '--local', self.execution_date_str]))
with open(self.ti_log_file_path) as l_file:
logs = l_file.read()
print(logs) # In case of a test failures this line would show detailed log
logs_list = logs.splitlines()
self.assertIn("INFO - Started process", logs)
self.assertIn(f"Subtask {self.task_id}", logs)
self.assertIn("standard_task_runner.py", logs)
self.assertIn(f"INFO - Running: ['airflow', 'tasks', 'run', '{self.dag_id}', "
f"'{self.task_id}', '{self.execution_date_str}',", logs)
self.assert_log_line("Log from DAG Logger", logs_list)
self.assert_log_line("Log from TI Logger", logs_list)
self.assert_log_line("Log from Print statement", logs_list, expect_from_logging_mixin=True)
self.assertIn(f"INFO - Marking task as SUCCESS.dag_id={self.dag_id}, "
f"task_id={self.task_id}, execution_date=20170101T000000", logs)
def test_logging_with_run_task_subprocess(self):
# We are not using self.assertLogs as we want to verify what actually is stored in the Log file
# as that is what gets displayed
with conf_vars({('core', 'dags_folder'): self.dag_path}):
task_command.task_run(self.parser.parse_args(self.task_args))
with open(self.ti_log_file_path) as l_file:
logs = l_file.read()
print(logs
) # In case of a test failures this line would show detailed log
logs_list = logs.splitlines()
assert f"Subtask {self.task_id}" in logs
assert "base_task_runner.py" in logs
self.assert_log_line("Log from DAG Logger", logs_list)
self.assert_log_line("Log from TI Logger", logs_list)
self.assert_log_line("Log from Print statement",
logs_list,
expect_from_logging_mixin=True)
assert (
f"INFO - Running: ['airflow', 'tasks', 'run', '{self.dag_id}', "
f"'{self.task_id}', '{self.execution_date_str}'," in logs)
assert (f"INFO - Marking task as SUCCESS. dag_id={self.dag_id}, "
f"task_id={self.task_id}, execution_date=20170101T000000"
in logs)
def test_cli_run_invalid_raw_option(self, option: str):
with self.assertRaisesRegex(
AirflowException,
"Option --raw does not work with some of the other options on this command."
):
task_command.task_run(self.parser.parse_args([ # type: ignore
'tasks', 'run', 'example_bash_operator', 'runme_0', DEFAULT_DATE.isoformat(), '--raw', option
]))
def test_cli_run_mutually_exclusive(self):
with self.assertRaisesRegex(
AirflowException,
"Option --raw and --local are mutually exclusive."
):
task_command.task_run(self.parser.parse_args([
'tasks', 'run', 'example_bash_operator', 'runme_0', DEFAULT_DATE.isoformat(), '--raw',
'--local'
]))
def test_local_run(self):
args = self.parser.parse_args([
'tasks', 'run', 'example_python_operator', 'print_the_context',
'2018-04-27T08:39:51.298439+00:00', '--interactive', '--subdir',
'/root/dags/example_python_operator.py'
])
dag = get_dag(args.subdir, args.dag_id)
reset(dag.dag_id)
task_command.task_run(args)
task = dag.get_task(task_id=args.task_id)
ti = TaskInstance(task, args.execution_date)
ti.refresh_from_db()
state = ti.current_state()
self.assertEqual(state, State.SUCCESS)
def test_local_run(self):
args = create_mock_args(
task_id='print_the_context',
dag_id='example_python_operator',
subdir='/root/dags/example_python_operator.py',
interactive=True,
execution_date=timezone.parse('2018-04-27T08:39:51.298439+00:00'))
dag = get_dag(args)
reset(dag.dag_id)
task_command.task_run(args)
task = dag.get_task(task_id=args.task_id)
ti = TaskInstance(task, args.execution_date)
ti.refresh_from_db()
state = ti.current_state()
self.assertEqual(state, State.SUCCESS)
def test_root_logger_restored(self, run_task_mock):
"""Verify that the root logging context is restored"""
logger = logging.getLogger("foo.bar")
def task_inner(*args, **kwargs):
logger.warning("redirected log message")
run_task_mock.side_effect = task_inner
config = {
('core', 'dags_folder'): self.dag_path,
('logging', 'logging_level'): "INFO",
}
with conf_vars(config):
with self.assertLogs(level=logging.WARNING) as captured:
logger.warning("not redirected")
task_command.task_run(self.parser.parse_args(self.task_args))
assert captured.output == ["WARNING:foo.bar:not redirected"]
assert self.root_logger.level == logging.WARNING
assert self.root_logger.handlers == self.root_handlers
def test_run_ignores_all_dependencies(self):
"""
Test that run respects ignore_all_dependencies
"""
dag_id = 'test_run_ignores_all_dependencies'
dag = self.dagbag.get_dag('test_run_ignores_all_dependencies')
dag.clear()
task0_id = 'test_run_dependent_task'
args0 = [
'tasks', 'run', '--ignore-all-dependencies', dag_id, task0_id,
DEFAULT_DATE.isoformat()
]
task_command.task_run(self.parser.parse_args(args0))
ti_dependent0 = TaskInstance(task=dag.get_task(task0_id),
execution_date=DEFAULT_DATE)
ti_dependent0.refresh_from_db()
assert ti_dependent0.state == State.FAILED
task1_id = 'test_run_dependency_task'
args1 = [
'tasks',
'run',
'--ignore-all-dependencies',
dag_id,
task1_id,
(DEFAULT_DATE + timedelta(days=1)).isoformat(),
]
task_command.task_run(self.parser.parse_args(args1))
ti_dependency = TaskInstance(task=dag.get_task(task1_id),
execution_date=DEFAULT_DATE +
timedelta(days=1))
ti_dependency.refresh_from_db()
assert ti_dependency.state == State.FAILED
task2_id = 'test_run_dependent_task'
args2 = [
'tasks',
'run',
'--ignore-all-dependencies',
dag_id,
task2_id,
(DEFAULT_DATE + timedelta(days=1)).isoformat(),
]
task_command.task_run(self.parser.parse_args(args2))
ti_dependent = TaskInstance(task=dag.get_task(task2_id),
execution_date=DEFAULT_DATE +
timedelta(days=1))
ti_dependent.refresh_from_db()
assert ti_dependent.state == State.SUCCESS
def test_cli_run(self):
task_command.task_run(
self.parser.parse_args([
'tasks', 'run', 'example_bash_operator', 'runme_0', '--local',
DEFAULT_DATE.isoformat()
]))
