def test_cli_backfill_depends_on_past(self):
"""
Test that CLI respects -I argument
"""
dag_id = 'test_dagrun_states_deadlock'
run_date = DEFAULT_DATE + datetime.timedelta(days=1)
args = [
'backfill',
dag_id,
'-l',
'-s',
run_date.isoformat(),
]
dag = self.dagbag.get_dag(dag_id)
dag.clear()
self.assertRaisesRegexp(
AirflowException,
'BackfillJob is deadlocked',
cli.backfill,
self.parser.parse_args(args))
cli.backfill(self.parser.parse_args(args + ['-I']))
ti = TI(dag.get_task('test_depends_on_past'), run_date)
ti.refresh_from_db()
# task ran
self.assertEqual(ti.state, State.SUCCESS)
dag.clear()
def test_cli_backfill_depends_on_past(self):
dag_id = 'test_dagrun_states_deadlock'
run_date = DEFAULT_DATE + datetime.timedelta(days=1)
args = [
'backfill',
dag_id,
'-l',
'-sd',
TEST_DAGS_FOLDER,
'-s',
run_date.isoformat(),
]
dag = get_dag(dag_id, TEST_DAGS_FOLDER)
cli.backfill(self.parser.parse_args(args))
ti = TI(dag.get_task('test_depends_on_past'), run_date)
ti.refresh_from_db()
# task did not run
self.assertEqual(ti.state, State.NONE)
cli.backfill(self.parser.parse_args(args + ['-I']))
ti = TI(dag.get_task('test_depends_on_past'), run_date)
ti.refresh_from_db()
# task ran
self.assertEqual(ti.state, State.SUCCESS)
def test_cli_backfill_depends_on_past(self):
"""
Test that CLI respects -I argument
"""
dag_id = 'test_dagrun_states_deadlock'
run_date = DEFAULT_DATE + datetime.timedelta(days=1)
args = [
'backfill',
dag_id,
'-l',
'-s',
run_date.isoformat(),
]
dag = self.dagbag.get_dag(dag_id)
dag.clear()
self.assertRaisesRegexp(AirflowException, 'BackfillJob is deadlocked',
cli.backfill, self.parser.parse_args(args))
cli.backfill(self.parser.parse_args(args + ['-I']))
ti = TI(dag.get_task('test_depends_on_past'), run_date)
ti.refresh_from_db()
# task ran
self.assertEqual(ti.state, State.SUCCESS)
dag.clear()
def test_cli_backfill_depends_on_past_backwards(self, mock_run):
"""
Test that CLI respects -B argument and raises on interaction with depends_on_past
"""
dag_id = 'test_depends_on_past'
start_date = DEFAULT_DATE + timedelta(days=1)
end_date = start_date + timedelta(days=1)
args = [
'dags',
'backfill',
dag_id,
'-l',
'-s',
start_date.isoformat(),
'-e',
end_date.isoformat(),
'-I',
'-B',
]
dag = self.dagbag.get_dag(dag_id)
cli.backfill(self.parser.parse_args(args), dag=dag)
mock_run.assert_called_once_with(
start_date=start_date,
end_date=end_date,
conf=None,
delay_on_limit_secs=1.0,
donot_pickle=False,
ignore_first_depends_on_past=True,
ignore_task_deps=False,
local=True,
mark_success=False,
pool=None,
rerun_failed_tasks=False,
run_backwards=True,
verbose=False,
)
def test_cli_backfill_depends_on_past(self, mock_run):
"""
Test that CLI respects -I argument
We just check we call dag.run() right. The behaviour of that kwarg is
tested in test_jobs
"""
dag_id = 'test_dagrun_states_deadlock'
run_date = DEFAULT_DATE + timedelta(days=1)
args = [
'dags',
'backfill',
dag_id,
'-l',
'-s',
run_date.isoformat(),
'-I',
]
dag = self.dagbag.get_dag(dag_id)
cli.backfill(self.parser.parse_args(args), dag=dag)
mock_run.assert_called_once_with(
start_date=run_date,
end_date=run_date,
conf=None,
delay_on_limit_secs=1.0,
donot_pickle=False,
ignore_first_depends_on_past=True,
ignore_task_deps=False,
local=True,
mark_success=False,
pool=None,
rerun_failed_tasks=False,
run_backwards=False,
verbose=False,
)
def test_cli_backfill_depends_on_past_backwards(self, mock_run):
"""
Test that CLI respects -B argument and raises on interaction with depends_on_past
"""
dag_id = 'test_depends_on_past'
start_date = DEFAULT_DATE + timedelta(days=1)
end_date = start_date + timedelta(days=1)
args = [
'backfill',
dag_id,
'-l',
'-s',
start_date.isoformat(),
'-e',
end_date.isoformat(),
'-I',
'-B',
]
dag = self.dagbag.get_dag(dag_id)
cli.backfill(self.parser.parse_args(args), dag=dag)
mock_run.assert_called_with(
start_date=start_date,
end_date=end_date,
conf=None,
delay_on_limit_secs=1.0,
donot_pickle=False,
ignore_first_depends_on_past=True,
ignore_task_deps=False,
local=True,
mark_success=False,
pool=None,
rerun_failed_tasks=False,
run_backwards=True,
verbose=False,
)
def test_cli_backfill_depends_on_past(self, mock_run):
"""
Test that CLI respects -I argument
We just check we call dag.run() right. The behaviour of that kwarg is
tested in test_jobs
"""
dag_id = 'test_dagrun_states_deadlock'
run_date = DEFAULT_DATE + timedelta(days=1)
args = [
'backfill',
dag_id,
'-l',
'-s',
run_date.isoformat(),
'-I',
]
dag = self.dagbag.get_dag(dag_id)
cli.backfill(self.parser.parse_args(args), dag=dag)
mock_run.assert_called_with(
start_date=run_date,
end_date=run_date,
conf=None,
delay_on_limit_secs=1.0,
donot_pickle=False,
ignore_first_depends_on_past=True,
ignore_task_deps=False,
local=True,
mark_success=False,
pool=None,
rerun_failed_tasks=False,
run_backwards=False,
verbose=False,
)
def test_backfill(self):
cli.backfill(self.parser.parse_args(["backfill", "example_bash_operator", "-s", DEFAULT_DATE.isoformat()]))
cli.backfill(
self.parser.parse_args(["backfill", "example_bash_operator", "--dry_run", "-s", DEFAULT_DATE.isoformat()])
)
cli.backfill(
self.parser.parse_args(["backfill", "example_bash_operator", "-l", "-s", DEFAULT_DATE.isoformat()])
)
def test_backfill(self):
cli.backfill(self.parser.parse_args([
'backfill', 'example_bash_operator',
'-s', DEFAULT_DATE.isoformat()]))
cli.backfill(self.parser.parse_args([
'backfill', 'example_bash_operator', '--dry_run',
'-s', DEFAULT_DATE.isoformat()]))
cli.backfill(self.parser.parse_args([
'backfill', 'example_bash_operator', '-l',
'-s', DEFAULT_DATE.isoformat()]))
def test_backfill(self):
cli.backfill(self.parser.parse_args([
'backfill', 'example_bash_operator',
'-s', DEFAULT_DATE.isoformat()]))
cli.backfill(self.parser.parse_args([
'backfill', 'example_bash_operator', '--dry_run',
'-s', DEFAULT_DATE.isoformat()]))
cli.backfill(self.parser.parse_args([
'backfill', 'example_bash_operator', '-l',
'-s', DEFAULT_DATE.isoformat()]))
def test_backfill(self, mock_run):
cli.backfill(
self.parser.parse_args([
'dags', 'backfill', 'example_bash_operator', '-s',
DEFAULT_DATE.isoformat()
]))
mock_run.assert_called_once_with(
start_date=DEFAULT_DATE,
end_date=DEFAULT_DATE,
conf=None,
delay_on_limit_secs=1.0,
donot_pickle=False,
ignore_first_depends_on_past=False,
ignore_task_deps=False,
local=False,
mark_success=False,
pool=None,
rerun_failed_tasks=False,
run_backwards=False,
verbose=False,
)
mock_run.reset_mock()
dag = self.dagbag.get_dag('example_bash_operator')
with mock.patch('sys.stdout', new_callable=io.StringIO) as mock_stdout:
cli.backfill(self.parser.parse_args([
'dags', 'backfill', 'example_bash_operator', '-t', 'runme_0',
'--dry_run', '-s',
DEFAULT_DATE.isoformat()
]),
dag=dag)
mock_stdout.seek(0, 0)
self.assertListEqual([
"Dry run of DAG example_bash_operator on {}\n".format(
DEFAULT_DATE.isoformat()),
"Task runme_0\n",
], mock_stdout.readlines())
mock_run.assert_not_called() # Dry run shouldn't run the backfill
cli.backfill(self.parser.parse_args([
'dags', 'backfill', 'example_bash_operator', '--dry_run', '-s',
DEFAULT_DATE.isoformat()
]),
dag=dag)
mock_run.assert_not_called() # Dry run shouldn't run the backfill
cli.backfill(self.parser.parse_args([
'dags', 'backfill', 'example_bash_operator', '-l', '-s',
DEFAULT_DATE.isoformat()
]),
dag=dag)
mock_run.assert_called_once_with(
start_date=DEFAULT_DATE,
end_date=DEFAULT_DATE,
conf=None,
delay_on_limit_secs=1.0,
donot_pickle=False,
ignore_first_depends_on_past=False,
ignore_task_deps=False,
local=True,
mark_success=False,
pool=None,
rerun_failed_tasks=False,
run_backwards=False,
verbose=False,
)
mock_run.reset_mock()
def test_backfill(self, mock_run):
cli.backfill(self.parser.parse_args([
'backfill', 'example_bash_operator',
'-s', DEFAULT_DATE.isoformat()]))
mock_run.assert_called_with(
start_date=DEFAULT_DATE,
end_date=DEFAULT_DATE,
conf=None,
delay_on_limit_secs=1.0,
donot_pickle=False,
ignore_first_depends_on_past=False,
ignore_task_deps=False,
local=False,
mark_success=False,
pool=None,
rerun_failed_tasks=False,
run_backwards=False,
verbose=False,
)
mock_run.reset_mock()
dag = self.dagbag.get_dag('example_bash_operator')
with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout:
cli.backfill(self.parser.parse_args([
'backfill', 'example_bash_operator', '-t', 'runme_0', '--dry_run',
'-s', DEFAULT_DATE.isoformat()]), dag=dag)
mock_stdout.seek(0, 0)
self.assertListEqual(
[
"Dry run of DAG example_bash_operator on {}\n".format(DEFAULT_DATE.isoformat()),
"Task runme_0\n",
],
mock_stdout.readlines()
)
mock_run.assert_not_called() # Dry run shouldn't run the backfill
cli.backfill(self.parser.parse_args([
'backfill', 'example_bash_operator', '--dry_run',
'-s', DEFAULT_DATE.isoformat()]), dag=dag)
mock_run.assert_not_called() # Dry run shouldn't run the backfill
cli.backfill(self.parser.parse_args([
'backfill', 'example_bash_operator', '-l',
'-s', DEFAULT_DATE.isoformat()]), dag=dag)
mock_run.assert_called_with(
start_date=DEFAULT_DATE,
end_date=DEFAULT_DATE,
conf=None,
delay_on_limit_secs=1.0,
donot_pickle=False,
ignore_first_depends_on_past=False,
ignore_task_deps=False,
local=True,
mark_success=False,
pool=None,
rerun_failed_tasks=False,
run_backwards=False,
verbose=False,
)
mock_run.reset_mock()
