def test_clear_task_instances_without_task(self): dag = DAG('test_clear_task_instances_without_task', start_date=DEFAULT_DATE, end_date=DEFAULT_DATE + datetime.timedelta(days=10)) task0 = DummyOperator(task_id='task0', owner='test', dag=dag) task1 = DummyOperator(task_id='task1', owner='test', dag=dag, retries=2) ti0 = TI(task=task0, execution_date=DEFAULT_DATE) ti1 = TI(task=task1, execution_date=DEFAULT_DATE) ti0.run() ti1.run() # Remove the task from dag. dag.task_dict = {} self.assertFalse(dag.has_task(task0.task_id)) self.assertFalse(dag.has_task(task1.task_id)) session = settings.Session() qry = session.query(TI).filter(TI.dag_id == dag.dag_id).all() clear_task_instances(qry, session) session.commit() # When dag is None, max_tries will be maximum of original max_tries or try_number. ti0.refresh_from_db() ti1.refresh_from_db() # Next try to run will be try 2 self.assertEqual(ti0.try_number, 2) self.assertEqual(ti0.max_tries, 1) self.assertEqual(ti1.try_number, 2) self.assertEqual(ti1.max_tries, 2)
def test_clear_task_instances_without_task(self): dag = DAG('test_clear_task_instances_without_task', start_date=DEFAULT_DATE, end_date=DEFAULT_DATE + datetime.timedelta(days=10)) task0 = DummyOperator(task_id='task0', owner='test', dag=dag) task1 = DummyOperator(task_id='task1', owner='test', dag=dag, retries=2) ti0 = TI(task=task0, execution_date=DEFAULT_DATE) ti1 = TI(task=task1, execution_date=DEFAULT_DATE) ti0.run() ti1.run() # Remove the task from dag. dag.task_dict = {} self.assertFalse(dag.has_task(task0.task_id)) self.assertFalse(dag.has_task(task1.task_id)) session = settings.Session() qry = session.query(TI).filter( TI.dag_id == dag.dag_id).all() clear_task_instances(qry, session) session.commit() # When dag is None, max_tries will be maximum of original max_tries or try_number. ti0.refresh_from_db() ti1.refresh_from_db() # Next try to run will be try 2 self.assertEqual(ti0.try_number, 2) self.assertEqual(ti0.max_tries, 1) self.assertEqual(ti1.try_number, 2) self.assertEqual(ti1.max_tries, 2)
def test_clear_task_instances_without_task(self): dag = DAG( 'test_clear_task_instances_without_task', start_date=DEFAULT_DATE, end_date=DEFAULT_DATE + datetime.timedelta(days=10), ) task0 = DummyOperator(task_id='task0', owner='test', dag=dag) task1 = DummyOperator(task_id='task1', owner='test', dag=dag, retries=2) ti0 = TI(task=task0, execution_date=DEFAULT_DATE) ti1 = TI(task=task1, execution_date=DEFAULT_DATE) dag.create_dagrun( execution_date=ti0.execution_date, state=State.RUNNING, run_type=DagRunType.SCHEDULED, ) ti0.run() ti1.run() # Remove the task from dag. dag.task_dict = {} assert not dag.has_task(task0.task_id) assert not dag.has_task(task1.task_id) with create_session() as session: qry = session.query(TI).filter(TI.dag_id == dag.dag_id).all() clear_task_instances(qry, session) # When dag is None, max_tries will be maximum of original max_tries or try_number. ti0.refresh_from_db() ti1.refresh_from_db() # Next try to run will be try 2 assert ti0.try_number == 2 assert ti0.max_tries == 1 assert ti1.try_number == 2 assert ti1.max_tries == 2
# ) #for i in df_independecy['taskname']: # tasktable=df_independecy[df_independecy['taskname']==i] # task = BashOperator( # task_id=i, # bash_command=('python '+tasktable['path'].values[0]+i+'.py'), # owner=tasktable['owner'].values[0], # start_date=datetime.strptime(tasktable['start_date'].values[0],'%Y-%m-%d %H:%M:%S'), # priority_weight=tasktable['priority_weight'].values[0], # dag=dag) # task.set_upstream(rely_task_done) for i in df_dependecy['taskname']: tasktable = df_dependecy[df_dependecy['taskname'] == i] if not dag.has_task(i): task = BashOperator( task_id=i, bash_command=('python ' + tasktable['path'].values[0] + i + '.py'), owner=tasktable['owner'].values[0], start_date=datetime.strptime(tasktable['start_date'].values[0], '%Y-%m-%d %H:%M:%S'), priority_weight=tasktable['priority_weight'].values[0], dag=dag) else: task = dag.get_task(i) dep = tasktable['rely_on'].values[0] #if i in top_task.values: # task.set_downstream(rely_task_done) if dep <> '': depend = dep.split(',')