def get_dynim_dag_list(**self):
clients_file = open('/root/integration_engine/dags/clients.json', 'r')
data =json.load(clients_file)
data = json.loads(data)
listofdags = []
for clientid, client in data.iteritems():
default_args['owner'] = client['owner']
startdate = parser.parse(client['start_date'])
dag = af_models.DAG(dag_id=str(client['name']).replace(' ', '_').replace('&', '_'), start_date = startdate, schedule_interval=client['schedule_time'], default_args=default_args)
ingestions = client['ingestions']
for ingid, ingestion in ingestions.iteritems():
taskid = 'task' + clientid + '_' + ingid
exchange = ingestion['exchange']
task_store = Task_Store(ingid, exchange)
task_validation = Task_Validate(ingid, exchange)
#tasks
task1 = af_op.PythonOperator(task_id = ingestion['name'].replace(' ', '_'), provide_context=True, \
python_callable=Task, dag=dag)
#subtasks
st1 = af_op.PythonOperator(task_id = ingestion['name'].replace(' ', '_') + '_store', provide_context=True, \
python_callable=task_store, dag=dag)
st2 = af_op.PythonOperator(task_id = ingestion['name'].replace(' ', '_') + '_validate', provide_context=True, \
python_callable=task_validation, dag=dag)
st2.set_upstream(st1)
task1.set_upstream(st2)
listofdags.append(dag)
return listofdags
def test_timeout(self):
t = operators.PythonOperator(task_id='test_timeout',
execution_timeout=timedelta(seconds=2),
python_callable=lambda: sleep(10),
dag=self.dag)
self.assertRaises(utils.AirflowTaskTimeout,
t.run,
start_date=DEFAULT_DATE,
end_date=DEFAULT_DATE,
force=True)
def test_python_op(self):
def test_py_op(templates_dict, ds, **kwargs):
if not templates_dict['ds'] == ds:
raise Exception("failure")
t = operators.PythonOperator(task_id='test_py_op',
provide_context=True,
python_callable=test_py_op,
templates_dict={'ds': "{{ ds }}"},
dag=self.dag)
t.run(start_date=DEFAULT_DATE, end_date=DEFAULT_DATE, force=True)
