def test_cli_backfill_depends_on_past(self): """ Test that CLI respects -I argument """ dag_id = 'test_dagrun_states_deadlock' run_date = DEFAULT_DATE + datetime.timedelta(days=1) args = [ 'backfill', dag_id, '-l', '-s', run_date.isoformat(), ] dag = self.dagbag.get_dag(dag_id) dag.clear() self.assertRaisesRegexp( AirflowException, 'BackfillJob is deadlocked', cli.backfill, self.parser.parse_args(args)) cli.backfill(self.parser.parse_args(args + ['-I'])) ti = TI(dag.get_task('test_depends_on_past'), run_date) ti.refresh_from_db() # task ran self.assertEqual(ti.state, State.SUCCESS) dag.clear()
def test_cli_backfill_depends_on_past(self): dag_id = 'test_dagrun_states_deadlock' run_date = DEFAULT_DATE + datetime.timedelta(days=1) args = [ 'backfill', dag_id, '-l', '-sd', TEST_DAGS_FOLDER, '-s', run_date.isoformat(), ] dag = get_dag(dag_id, TEST_DAGS_FOLDER) cli.backfill(self.parser.parse_args(args)) ti = TI(dag.get_task('test_depends_on_past'), run_date) ti.refresh_from_db() # task did not run self.assertEqual(ti.state, State.NONE) cli.backfill(self.parser.parse_args(args + ['-I'])) ti = TI(dag.get_task('test_depends_on_past'), run_date) ti.refresh_from_db() # task ran self.assertEqual(ti.state, State.SUCCESS)
def test_cli_backfill_depends_on_past(self): """ Test that CLI respects -I argument """ dag_id = 'test_dagrun_states_deadlock' run_date = DEFAULT_DATE + datetime.timedelta(days=1) args = [ 'backfill', dag_id, '-l', '-s', run_date.isoformat(), ] dag = self.dagbag.get_dag(dag_id) dag.clear() self.assertRaisesRegexp(AirflowException, 'BackfillJob is deadlocked', cli.backfill, self.parser.parse_args(args)) cli.backfill(self.parser.parse_args(args + ['-I'])) ti = TI(dag.get_task('test_depends_on_past'), run_date) ti.refresh_from_db() # task ran self.assertEqual(ti.state, State.SUCCESS) dag.clear()
def test_cli_backfill_depends_on_past_backwards(self, mock_run): """ Test that CLI respects -B argument and raises on interaction with depends_on_past """ dag_id = 'test_depends_on_past' start_date = DEFAULT_DATE + timedelta(days=1) end_date = start_date + timedelta(days=1) args = [ 'dags', 'backfill', dag_id, '-l', '-s', start_date.isoformat(), '-e', end_date.isoformat(), '-I', '-B', ] dag = self.dagbag.get_dag(dag_id) cli.backfill(self.parser.parse_args(args), dag=dag) mock_run.assert_called_once_with( start_date=start_date, end_date=end_date, conf=None, delay_on_limit_secs=1.0, donot_pickle=False, ignore_first_depends_on_past=True, ignore_task_deps=False, local=True, mark_success=False, pool=None, rerun_failed_tasks=False, run_backwards=True, verbose=False, )
def test_cli_backfill_depends_on_past(self, mock_run): """ Test that CLI respects -I argument We just check we call dag.run() right. The behaviour of that kwarg is tested in test_jobs """ dag_id = 'test_dagrun_states_deadlock' run_date = DEFAULT_DATE + timedelta(days=1) args = [ 'dags', 'backfill', dag_id, '-l', '-s', run_date.isoformat(), '-I', ] dag = self.dagbag.get_dag(dag_id) cli.backfill(self.parser.parse_args(args), dag=dag) mock_run.assert_called_once_with( start_date=run_date, end_date=run_date, conf=None, delay_on_limit_secs=1.0, donot_pickle=False, ignore_first_depends_on_past=True, ignore_task_deps=False, local=True, mark_success=False, pool=None, rerun_failed_tasks=False, run_backwards=False, verbose=False, )
def test_cli_backfill_depends_on_past_backwards(self, mock_run): """ Test that CLI respects -B argument and raises on interaction with depends_on_past """ dag_id = 'test_depends_on_past' start_date = DEFAULT_DATE + timedelta(days=1) end_date = start_date + timedelta(days=1) args = [ 'backfill', dag_id, '-l', '-s', start_date.isoformat(), '-e', end_date.isoformat(), '-I', '-B', ] dag = self.dagbag.get_dag(dag_id) cli.backfill(self.parser.parse_args(args), dag=dag) mock_run.assert_called_with( start_date=start_date, end_date=end_date, conf=None, delay_on_limit_secs=1.0, donot_pickle=False, ignore_first_depends_on_past=True, ignore_task_deps=False, local=True, mark_success=False, pool=None, rerun_failed_tasks=False, run_backwards=True, verbose=False, )
def test_cli_backfill_depends_on_past(self, mock_run): """ Test that CLI respects -I argument We just check we call dag.run() right. The behaviour of that kwarg is tested in test_jobs """ dag_id = 'test_dagrun_states_deadlock' run_date = DEFAULT_DATE + timedelta(days=1) args = [ 'backfill', dag_id, '-l', '-s', run_date.isoformat(), '-I', ] dag = self.dagbag.get_dag(dag_id) cli.backfill(self.parser.parse_args(args), dag=dag) mock_run.assert_called_with( start_date=run_date, end_date=run_date, conf=None, delay_on_limit_secs=1.0, donot_pickle=False, ignore_first_depends_on_past=True, ignore_task_deps=False, local=True, mark_success=False, pool=None, rerun_failed_tasks=False, run_backwards=False, verbose=False, )
def test_backfill(self): cli.backfill(self.parser.parse_args(["backfill", "example_bash_operator", "-s", DEFAULT_DATE.isoformat()])) cli.backfill( self.parser.parse_args(["backfill", "example_bash_operator", "--dry_run", "-s", DEFAULT_DATE.isoformat()]) ) cli.backfill( self.parser.parse_args(["backfill", "example_bash_operator", "-l", "-s", DEFAULT_DATE.isoformat()]) )
def test_backfill(self): cli.backfill(self.parser.parse_args([ 'backfill', 'example_bash_operator', '-s', DEFAULT_DATE.isoformat()])) cli.backfill(self.parser.parse_args([ 'backfill', 'example_bash_operator', '--dry_run', '-s', DEFAULT_DATE.isoformat()])) cli.backfill(self.parser.parse_args([ 'backfill', 'example_bash_operator', '-l', '-s', DEFAULT_DATE.isoformat()]))
def test_backfill(self): cli.backfill(self.parser.parse_args([ 'backfill', 'example_bash_operator', '-s', DEFAULT_DATE.isoformat()])) cli.backfill(self.parser.parse_args([ 'backfill', 'example_bash_operator', '--dry_run', '-s', DEFAULT_DATE.isoformat()])) cli.backfill(self.parser.parse_args([ 'backfill', 'example_bash_operator', '-l', '-s', DEFAULT_DATE.isoformat()]))
def test_backfill(self, mock_run): cli.backfill( self.parser.parse_args([ 'dags', 'backfill', 'example_bash_operator', '-s', DEFAULT_DATE.isoformat() ])) mock_run.assert_called_once_with( start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, conf=None, delay_on_limit_secs=1.0, donot_pickle=False, ignore_first_depends_on_past=False, ignore_task_deps=False, local=False, mark_success=False, pool=None, rerun_failed_tasks=False, run_backwards=False, verbose=False, ) mock_run.reset_mock() dag = self.dagbag.get_dag('example_bash_operator') with mock.patch('sys.stdout', new_callable=io.StringIO) as mock_stdout: cli.backfill(self.parser.parse_args([ 'dags', 'backfill', 'example_bash_operator', '-t', 'runme_0', '--dry_run', '-s', DEFAULT_DATE.isoformat() ]), dag=dag) mock_stdout.seek(0, 0) self.assertListEqual([ "Dry run of DAG example_bash_operator on {}\n".format( DEFAULT_DATE.isoformat()), "Task runme_0\n", ], mock_stdout.readlines()) mock_run.assert_not_called() # Dry run shouldn't run the backfill cli.backfill(self.parser.parse_args([ 'dags', 'backfill', 'example_bash_operator', '--dry_run', '-s', DEFAULT_DATE.isoformat() ]), dag=dag) mock_run.assert_not_called() # Dry run shouldn't run the backfill cli.backfill(self.parser.parse_args([ 'dags', 'backfill', 'example_bash_operator', '-l', '-s', DEFAULT_DATE.isoformat() ]), dag=dag) mock_run.assert_called_once_with( start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, conf=None, delay_on_limit_secs=1.0, donot_pickle=False, ignore_first_depends_on_past=False, ignore_task_deps=False, local=True, mark_success=False, pool=None, rerun_failed_tasks=False, run_backwards=False, verbose=False, ) mock_run.reset_mock()
def test_backfill(self, mock_run): cli.backfill(self.parser.parse_args([ 'backfill', 'example_bash_operator', '-s', DEFAULT_DATE.isoformat()])) mock_run.assert_called_with( start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, conf=None, delay_on_limit_secs=1.0, donot_pickle=False, ignore_first_depends_on_past=False, ignore_task_deps=False, local=False, mark_success=False, pool=None, rerun_failed_tasks=False, run_backwards=False, verbose=False, ) mock_run.reset_mock() dag = self.dagbag.get_dag('example_bash_operator') with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: cli.backfill(self.parser.parse_args([ 'backfill', 'example_bash_operator', '-t', 'runme_0', '--dry_run', '-s', DEFAULT_DATE.isoformat()]), dag=dag) mock_stdout.seek(0, 0) self.assertListEqual( [ "Dry run of DAG example_bash_operator on {}\n".format(DEFAULT_DATE.isoformat()), "Task runme_0\n", ], mock_stdout.readlines() ) mock_run.assert_not_called() # Dry run shouldn't run the backfill cli.backfill(self.parser.parse_args([ 'backfill', 'example_bash_operator', '--dry_run', '-s', DEFAULT_DATE.isoformat()]), dag=dag) mock_run.assert_not_called() # Dry run shouldn't run the backfill cli.backfill(self.parser.parse_args([ 'backfill', 'example_bash_operator', '-l', '-s', DEFAULT_DATE.isoformat()]), dag=dag) mock_run.assert_called_with( start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, conf=None, delay_on_limit_secs=1.0, donot_pickle=False, ignore_first_depends_on_past=False, ignore_task_deps=False, local=True, mark_success=False, pool=None, rerun_failed_tasks=False, run_backwards=False, verbose=False, ) mock_run.reset_mock()