def get_dynim_dag_list(**self): clients_file = open('/root/integration_engine/dags/clients.json', 'r') data =json.load(clients_file) data = json.loads(data) listofdags = [] for clientid, client in data.iteritems(): default_args['owner'] = client['owner'] startdate = parser.parse(client['start_date']) dag = af_models.DAG(dag_id=str(client['name']).replace(' ', '_').replace('&', '_'), start_date = startdate, schedule_interval=client['schedule_time'], default_args=default_args) ingestions = client['ingestions'] for ingid, ingestion in ingestions.iteritems(): taskid = 'task' + clientid + '_' + ingid exchange = ingestion['exchange'] task_store = Task_Store(ingid, exchange) task_validation = Task_Validate(ingid, exchange) #tasks task1 = af_op.PythonOperator(task_id = ingestion['name'].replace(' ', '_'), provide_context=True, \ python_callable=Task, dag=dag) #subtasks st1 = af_op.PythonOperator(task_id = ingestion['name'].replace(' ', '_') + '_store', provide_context=True, \ python_callable=task_store, dag=dag) st2 = af_op.PythonOperator(task_id = ingestion['name'].replace(' ', '_') + '_validate', provide_context=True, \ python_callable=task_validation, dag=dag) st2.set_upstream(st1) task1.set_upstream(st2) listofdags.append(dag) return listofdags
def test_timeout(self): t = operators.PythonOperator(task_id='test_timeout', execution_timeout=timedelta(seconds=2), python_callable=lambda: sleep(10), dag=self.dag) self.assertRaises(utils.AirflowTaskTimeout, t.run, start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, force=True)
def test_python_op(self): def test_py_op(templates_dict, ds, **kwargs): if not templates_dict['ds'] == ds: raise Exception("failure") t = operators.PythonOperator(task_id='test_py_op', provide_context=True, python_callable=test_py_op, templates_dict={'ds': "{{ ds }}"}, dag=self.dag) t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, force=True)