def test_run_naive_taskinstance(self, mock_local_job):
"""
Test that we can run naive (non-localized) task instances
"""
NAIVE_DATE = datetime(2016, 1, 1)
dag_id = 'test_run_ignores_all_dependencies'
dag = self.dagbag.get_dag('test_run_ignores_all_dependencies')
task0_id = 'test_run_dependent_task'
args0 = ['run',
'-A',
'--local',
dag_id,
task0_id,
NAIVE_DATE.isoformat()]
cli.run(self.parser.parse_args(args0), dag=dag)
mock_local_job.assert_called_with(
task_instance=mock.ANY,
mark_success=False,
ignore_all_deps=True,
ignore_depends_on_past=False,
ignore_task_deps=False,
ignore_ti_state=False,
pickle_id=None,
pool=None,
)
def test_run_naive_taskinstance(self, mock_local_job):
"""
Test that we can run naive (non-localized) task instances
"""
NAIVE_DATE = datetime(2016, 1, 1)
dag_id = 'test_run_ignores_all_dependencies'
dag = self.dagbag.get_dag('test_run_ignores_all_dependencies')
task0_id = 'test_run_dependent_task'
args0 = [
'tasks', 'run', '-A', '--local', dag_id, task0_id,
NAIVE_DATE.isoformat()
]
cli.run(self.parser.parse_args(args0), dag=dag)
mock_local_job.assert_called_once_with(
task_instance=mock.ANY,
mark_success=False,
ignore_all_deps=True,
ignore_depends_on_past=False,
ignore_task_deps=False,
ignore_ti_state=False,
pickle_id=None,
pool=None,
)
def test_local_run(self):
args = create_mock_args(
task_id='print_the_context',
dag_id='example_python_operator',
subdir='/root/dags/example_python_operator.py',
interactive=True,
execution_date=timezone.parse('2018-04-27T08:39:51.298439+00:00'))
reset(args.dag_id)
run(args)
dag = get_dag(args)
task = dag.get_task(task_id=args.task_id)
ti = TaskInstance(task, args.execution_date)
ti.refresh_from_db()
state = ti.current_state()
self.assertEqual(state, State.SUCCESS)
def test_local_run(self):
args = create_mock_args(
task_id='print_the_context',
dag_id='example_python_operator',
subdir='/root/dags/example_python_operator.py',
interactive=True,
execution_date=timezone.parse('2018-04-27T08:39:51.298439+00:00')
)
reset(args.dag_id)
with patch('argparse.Namespace', args) as mock_args:
run(mock_args)
dag = get_dag(mock_args)
task = dag.get_task(task_id=args.task_id)
ti = TaskInstance(task, args.execution_date)
ti.refresh_from_db()
state = ti.current_state()
self.assertEqual(state, State.SUCCESS)
def test_logging_with_run_task(self):
# We are not using self.assertLogs as we want to verify what actually is stored in the Log file
# as that is what gets displayed
with conf_vars({
('core', 'dags_folder'):
os.path.join(TEST_DAG_FOLDER, self.dag_id)
}):
cli.run(
self.parser.parse_args([
'run', self.dag_id, self.task_id, '--local',
self.execution_date_str
]))
with open(self.ti_log_file_path) as l_file:
logs = l_file.read()
print(logs
) # In case of a test failures this line would show detailed log
logs_list = logs.splitlines()
self.assertIn("INFO - Started process", logs)
self.assertIn("Subtask {}".format(self.task_id), logs)
self.assertIn("standard_task_runner.py", logs)
self.assertIn(
"INFO - Running: ['airflow', 'run', '{}', "
"'{}', '{}',".format(self.dag_id, self.task_id,
self.execution_date_str), logs)
self.assert_log_line("Log from DAG Logger", logs_list)
self.assert_log_line("Log from TI Logger", logs_list)
self.assert_log_line("Log from Print statement",
logs_list,
expect_from_logging_mixin=True)
self.assertIn(
"INFO - Marking task as SUCCESS.dag_id={}, task_id={}, "
"execution_date=20170101T000000".format(self.dag_id, self.task_id),
logs)
def test_run_ignores_all_dependencies(self):
"""
Test that run respects ignore_all_dependencies
"""
dag_id = 'test_run_ignores_all_dependencies'
dag = self.dagbag.get_dag('test_run_ignores_all_dependencies')
dag.clear()
task0_id = 'test_run_dependent_task'
args0 = ['tasks',
'run',
'-A',
dag_id,
task0_id,
DEFAULT_DATE.isoformat()]
cli.run(self.parser.parse_args(args0))
ti_dependent0 = TI(
task=dag.get_task(task0_id),
execution_date=DEFAULT_DATE)
ti_dependent0.refresh_from_db()
self.assertEqual(ti_dependent0.state, State.FAILED)
task1_id = 'test_run_dependency_task'
args1 = ['tasks',
'run',
'-A',
dag_id,
task1_id,
(DEFAULT_DATE + datetime.timedelta(days=1)).isoformat()]
cli.run(self.parser.parse_args(args1))
ti_dependency = TI(
task=dag.get_task(task1_id),
execution_date=DEFAULT_DATE + datetime.timedelta(days=1))
ti_dependency.refresh_from_db()
self.assertEqual(ti_dependency.state, State.FAILED)
task2_id = 'test_run_dependent_task'
args2 = ['tasks',
'run',
'-A',
dag_id,
task2_id,
(DEFAULT_DATE + datetime.timedelta(days=1)).isoformat()]
cli.run(self.parser.parse_args(args2))
ti_dependent = TI(
task=dag.get_task(task2_id),
execution_date=DEFAULT_DATE + datetime.timedelta(days=1))
ti_dependent.refresh_from_db()
self.assertEqual(ti_dependent.state, State.SUCCESS)
def test_cli_run(self):
cli.run(self.parser.parse_args([
'run', 'example_bash_operator', 'runme_0', '-l',
DEFAULT_DATE.isoformat()]))
return "get_dataset: result"
op_get_variable = PythonOperator(
task_id='get_variable',
provide_context=True,
python_callable=get_variable,
dag=dag,
)
op_print_context >> op_get_dataset >> op_get_variable
if __name__ == '__main__':
from airflow.bin.cli import run, get_dag
from unittest.mock import patch
run_args = dict(
task_id='get_variable',
dag_id='ILTest-python',
interactive=False,
cfg_path = "",
execution_date=timezone.parse('2018-04-27T08:39:51.298439+00:00')
)
run( Namespace(**run_args) )
def test_cli_run(self):
cli.run(
self.parser.parse_args([
'run', 'example_bash_operator', 'runme_0', '-l',
DEFAULT_DATE.isoformat()
]))
def test_cli_run(self):
cli.run(self.parser.parse_args(["run", "example_bash_operator", "runme_0", "-l", DEFAULT_DATE.isoformat()]))
