def trigger_loop(self, submission, task_list, mdata):
# loop_num = None
c = Client(None, None)
submission_dict = submission.serialize()
task_dict_list = [task.serialize() for task in task_list]
submission_hash = submission.submission_hash
self.run_id = f"dag_run_{submission_hash}"
try:
c.trigger_dag(dag_id=self.dag_name,
run_id=self.run_id,
conf={
'submission_dict': submission_dict,
'task_dict_list': task_dict_list,
'mdata': mdata
})
except DagRunAlreadyExists:
dag_run_state = self.get_dag_run_state()
if dag_run_state == State.FAILED:
raise AirflowFailException(
f"subdag dag_run fail dag_id:{self.dag_name}; run_id:{self.run_id};"
)
else:
print(f"continue from old dag_run {self.run_id}")
loop_return = self.wait_until_end()
return loop_return
async def execute(param: PredictParam):
try:
c = Client(None, None)
c.trigger_dag(
dag_id="trial_operation",
conf={
"target_date": param.target_date,
"work_dir":
"~/Code/between_poc_and_production/notebooks/trial/",
},
)
return {"result": "Success", "message": "successfully triggered"}
except Exception as e:
return {"result": "Failed", "message": f"error occured: {e}"}
def setUp(self):
super().setUp()
clear_db_pools()
self.client = Client(api_base_url=None, auth=None)
class TestLocalClient(unittest.TestCase):
@classmethod
def setUpClass(cls):
super(TestLocalClient, cls).setUpClass()
DagBag(example_bash_operator.__file__).get_dag("example_bash_operator").sync_to_db()
def setUp(self):
super().setUp()
clear_db_pools()
self.client = Client(api_base_url=None, auth=None)
def tearDown(self):
clear_db_pools()
super().tearDown()
@patch.object(models.DAG, 'create_dagrun')
def test_trigger_dag(self, mock):
test_dag_id = "example_bash_operator"
models.DagBag(include_examples=True)
# non existent
with self.assertRaises(AirflowException):
self.client.trigger_dag(dag_id="blablabla")
with freeze_time(EXECDATE):
# no execution date, execution date should be set automatically
self.client.trigger_dag(dag_id=test_dag_id)
mock.assert_called_once_with(run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# execution date with microseconds cutoff
self.client.trigger_dag(dag_id=test_dag_id, execution_date=EXECDATE)
mock.assert_called_once_with(run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# run id
run_id = "my_run_id"
self.client.trigger_dag(dag_id=test_dag_id, run_id=run_id)
mock.assert_called_once_with(run_id=run_id,
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# test conf
conf = '{"name": "John"}'
self.client.trigger_dag(dag_id=test_dag_id, conf=conf)
mock.assert_called_once_with(run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=json.loads(conf),
external_trigger=True)
mock.reset_mock()
def test_delete_dag(self):
key = "my_dag_id"
with create_session() as session:
self.assertEqual(session.query(DagModel).filter(DagModel.dag_id == key).count(), 0)
session.add(DagModel(dag_id=key))
with create_session() as session:
self.assertEqual(session.query(DagModel).filter(DagModel.dag_id == key).count(), 1)
self.client.delete_dag(dag_id=key)
self.assertEqual(session.query(DagModel).filter(DagModel.dag_id == key).count(), 0)
def test_get_pool(self):
self.client.create_pool(name='foo', slots=1, description='')
pool = self.client.get_pool(name='foo')
self.assertEqual(pool, ('foo', 1, ''))
def test_get_pools(self):
self.client.create_pool(name='foo1', slots=1, description='')
self.client.create_pool(name='foo2', slots=2, description='')
pools = sorted(self.client.get_pools(), key=lambda p: p[0])
self.assertEqual(pools, [('foo1', 1, ''), ('foo2', 2, '')])
def test_create_pool(self):
pool = self.client.create_pool(name='foo', slots=1, description='')
self.assertEqual(pool, ('foo', 1, ''))
with create_session() as session:
self.assertEqual(session.query(models.Pool).count(), 1)
def test_delete_pool(self):
self.client.create_pool(name='foo', slots=1, description='')
with create_session() as session:
self.assertEqual(session.query(models.Pool).count(), 1)
self.client.delete_pool(name='foo')
with create_session() as session:
self.assertEqual(session.query(models.Pool).count(), 0)
def setUp(self):
self.client = Client(api_base_url=None, auth=None)
def setUp(self):
super(TestLocalClient, self).setUp()
self.client = Client(api_base_url=None, auth=None)
self.session = settings.Session()
class TestLocalClient(unittest.TestCase):
@classmethod
def setUpClass(cls):
super(TestLocalClient, cls).setUpClass()
session = settings.Session()
session.query(models.Pool).delete()
session.commit()
session.close()
def setUp(self):
super(TestLocalClient, self).setUp()
self.client = Client(api_base_url=None, auth=None)
self.session = settings.Session()
def tearDown(self):
self.session.query(models.Pool).delete()
self.session.commit()
self.session.close()
super(TestLocalClient, self).tearDown()
@patch.object(models.DAG, 'create_dagrun')
def test_trigger_dag(self, mock):
client = self.client
# non existent
with self.assertRaises(AirflowException):
client.trigger_dag(dag_id="blablabla")
with freeze_time(EXECDATE):
# no execution date, execution date should be set automatically
client.trigger_dag(dag_id="test_start_date_scheduling")
mock.assert_called_once_with(run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# execution date with microseconds cutoff
client.trigger_dag(dag_id="test_start_date_scheduling", execution_date=EXECDATE)
mock.assert_called_once_with(run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# run id
run_id = "my_run_id"
client.trigger_dag(dag_id="test_start_date_scheduling", run_id=run_id)
mock.assert_called_once_with(run_id=run_id,
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# test conf
conf = '{"name": "John"}'
client.trigger_dag(dag_id="test_start_date_scheduling", conf=conf)
mock.assert_called_once_with(run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=json.loads(conf),
external_trigger=True)
mock.reset_mock()
def test_get_pool(self):
self.client.create_pool(name='foo', slots=1, description='')
pool = self.client.get_pool(name='foo')
self.assertEqual(pool, ('foo', 1, ''))
def test_get_pools(self):
self.client.create_pool(name='foo1', slots=1, description='')
self.client.create_pool(name='foo2', slots=2, description='')
pools = sorted(self.client.get_pools(), key=lambda p: p[0])
self.assertEqual(pools, [('foo1', 1, ''), ('foo2', 2, '')])
def test_create_pool(self):
pool = self.client.create_pool(name='foo', slots=1, description='')
self.assertEqual(pool, ('foo', 1, ''))
self.assertEqual(self.session.query(models.Pool).count(), 1)
def test_delete_pool(self):
self.client.create_pool(name='foo', slots=1, description='')
self.assertEqual(self.session.query(models.Pool).count(), 1)
self.client.delete_pool(name='foo')
self.assertEqual(self.session.query(models.Pool).count(), 0)
def setUp(self):
super().setUp()
clear_db_pools()
self.client = Client(api_base_url=None, auth=None)
class TestLocalClient(unittest.TestCase):
@classmethod
def setUpClass(cls):
super(TestLocalClient, cls).setUpClass()
DagBag(example_bash_operator.__file__).get_dag("example_bash_operator").sync_to_db()
def setUp(self):
super().setUp()
clear_db_pools()
self.client = Client(api_base_url=None, auth=None)
def tearDown(self):
clear_db_pools()
super().tearDown()
@patch.object(models.DAG, 'create_dagrun')
def test_trigger_dag(self, mock):
test_dag_id = "example_bash_operator"
models.DagBag(include_examples=True)
# non existent
with self.assertRaises(AirflowException):
self.client.trigger_dag(dag_id="blablabla")
with freeze_time(EXECDATE):
# no execution date, execution date should be set automatically
self.client.trigger_dag(dag_id=test_dag_id)
mock.assert_called_once_with(run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# execution date with microseconds cutoff
self.client.trigger_dag(dag_id=test_dag_id, execution_date=EXECDATE)
mock.assert_called_once_with(run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# run id
run_id = "my_run_id"
self.client.trigger_dag(dag_id=test_dag_id, run_id=run_id)
mock.assert_called_once_with(run_id=run_id,
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# test conf
conf = '{"name": "John"}'
self.client.trigger_dag(dag_id=test_dag_id, conf=conf)
mock.assert_called_once_with(run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=json.loads(conf),
external_trigger=True)
mock.reset_mock()
def test_delete_dag(self):
key = "my_dag_id"
with create_session() as session:
self.assertEqual(session.query(DagModel).filter(DagModel.dag_id == key).count(), 0)
session.add(DagModel(dag_id=key))
with create_session() as session:
self.assertEqual(session.query(DagModel).filter(DagModel.dag_id == key).count(), 1)
self.client.delete_dag(dag_id=key)
self.assertEqual(session.query(DagModel).filter(DagModel.dag_id == key).count(), 0)
def test_get_pool(self):
self.client.create_pool(name='foo', slots=1, description='')
pool = self.client.get_pool(name='foo')
self.assertEqual(pool, ('foo', 1, ''))
def test_get_pools(self):
self.client.create_pool(name='foo1', slots=1, description='')
self.client.create_pool(name='foo2', slots=2, description='')
pools = sorted(self.client.get_pools(), key=lambda p: p[0])
self.assertEqual(pools, [('foo1', 1, ''), ('foo2', 2, '')])
def test_create_pool(self):
pool = self.client.create_pool(name='foo', slots=1, description='')
self.assertEqual(pool, ('foo', 1, ''))
with create_session() as session:
self.assertEqual(session.query(models.Pool).count(), 1)
def test_delete_pool(self):
self.client.create_pool(name='foo', slots=1, description='')
with create_session() as session:
self.assertEqual(session.query(models.Pool).count(), 1)
self.client.delete_pool(name='foo')
with create_session() as session:
self.assertEqual(session.query(models.Pool).count(), 0)
class TestLocalClient(unittest.TestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
DagBag(example_bash_operator.__file__).get_dag(
"example_bash_operator").sync_to_db()
def setUp(self):
super().setUp()
clear_db_pools()
self.client = Client(api_base_url=None, auth=None)
def tearDown(self):
clear_db_pools()
super().tearDown()
@patch.object(DAG, 'create_dagrun')
def test_trigger_dag(self, mock):
test_dag_id = "example_bash_operator"
run_id = DagRun.generate_run_id(DagRunType.MANUAL,
EXECDATE_NOFRACTIONS)
DagBag(include_examples=True)
# non existent
with pytest.raises(AirflowException):
self.client.trigger_dag(dag_id="blablabla")
with freeze_time(EXECDATE):
# no execution date, execution date should be set automatically
self.client.trigger_dag(dag_id=test_dag_id)
mock.assert_called_once_with(
run_id=run_id,
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True,
dag_hash=ANY,
)
mock.reset_mock()
# execution date with microseconds cutoff
self.client.trigger_dag(dag_id=test_dag_id,
execution_date=EXECDATE)
mock.assert_called_once_with(
run_id=run_id,
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True,
dag_hash=ANY,
)
mock.reset_mock()
# run id
custom_run_id = "my_run_id"
self.client.trigger_dag(dag_id=test_dag_id, run_id=custom_run_id)
mock.assert_called_once_with(
run_id=custom_run_id,
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True,
dag_hash=ANY,
)
mock.reset_mock()
# test conf
conf = '{"name": "John"}'
self.client.trigger_dag(dag_id=test_dag_id, conf=conf)
mock.assert_called_once_with(
run_id=run_id,
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=json.loads(conf),
external_trigger=True,
dag_hash=ANY,
)
mock.reset_mock()
def test_delete_dag(self):
key = "my_dag_id"
with create_session() as session:
assert session.query(DagModel).filter(
DagModel.dag_id == key).count() == 0
session.add(DagModel(dag_id=key))
with create_session() as session:
assert session.query(DagModel).filter(
DagModel.dag_id == key).count() == 1
self.client.delete_dag(dag_id=key)
assert session.query(DagModel).filter(
DagModel.dag_id == key).count() == 0
def test_get_pool(self):
self.client.create_pool(name='foo', slots=1, description='')
pool = self.client.get_pool(name='foo')
assert pool == ('foo', 1, '')
def test_get_pools(self):
self.client.create_pool(name='foo1', slots=1, description='')
self.client.create_pool(name='foo2', slots=2, description='')
pools = sorted(self.client.get_pools(), key=lambda p: p[0])
assert pools == [('default_pool', 128, 'Default pool'),
('foo1', 1, ''), ('foo2', 2, '')]
def test_create_pool(self):
pool = self.client.create_pool(name='foo', slots=1, description='')
assert pool == ('foo', 1, '')
with create_session() as session:
assert session.query(Pool).count() == 2
def test_delete_pool(self):
self.client.create_pool(name='foo', slots=1, description='')
with create_session() as session:
assert session.query(Pool).count() == 2
self.client.delete_pool(name='foo')
with create_session() as session:
assert session.query(Pool).count() == 1
def setUp(self):
super(TestLocalClient, self).setUp()
self.client = Client(api_base_url=None, auth=None)
self.session = settings.Session()
class TestLocalClient(unittest.TestCase):
@classmethod
def setUpClass(cls):
super(TestLocalClient, cls).setUpClass()
session = settings.Session()
session.query(models.Pool).delete()
session.commit()
session.close()
def setUp(self):
super(TestLocalClient, self).setUp()
self.client = Client(api_base_url=None, auth=None)
self.session = settings.Session()
def tearDown(self):
self.session.query(models.Pool).delete()
self.session.commit()
self.session.close()
super(TestLocalClient, self).tearDown()
@patch.object(models.DAG, 'create_dagrun')
def test_trigger_dag(self, mock):
client = self.client
# non existent
with self.assertRaises(AirflowException):
client.trigger_dag(dag_id="blablabla")
with freeze_time(EXECDATE):
# no execution date, execution date should be set automatically
client.trigger_dag(dag_id="test_start_date_scheduling")
mock.assert_called_once_with(
run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# execution date with microseconds cutoff
client.trigger_dag(dag_id="test_start_date_scheduling",
execution_date=EXECDATE)
mock.assert_called_once_with(
run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# run id
run_id = "my_run_id"
client.trigger_dag(dag_id="test_start_date_scheduling",
run_id=run_id)
mock.assert_called_once_with(run_id=run_id,
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=None,
external_trigger=True)
mock.reset_mock()
# test conf
conf = '{"name": "John"}'
client.trigger_dag(dag_id="test_start_date_scheduling", conf=conf)
mock.assert_called_once_with(
run_id="manual__{0}".format(EXECDATE_ISO),
execution_date=EXECDATE_NOFRACTIONS,
state=State.RUNNING,
conf=json.loads(conf),
external_trigger=True)
mock.reset_mock()
def test_delete_dag(self):
key = "my_dag_id"
session = settings.Session()
DM = models.DagModel
self.assertEqual(session.query(DM).filter(DM.dag_id == key).count(), 0)
session.add(DM(dag_id=key))
session.commit()
self.assertEqual(session.query(DM).filter(DM.dag_id == key).count(), 1)
self.client.delete_dag(dag_id=key)
self.assertEqual(session.query(DM).filter(DM.dag_id == key).count(), 0)
def test_get_pool(self):
self.client.create_pool(name='foo', slots=1, description='')
pool = self.client.get_pool(name='foo')
self.assertEqual(pool, ('foo', 1, ''))
def test_get_pools(self):
self.client.create_pool(name='foo1', slots=1, description='')
self.client.create_pool(name='foo2', slots=2, description='')
pools = sorted(self.client.get_pools(), key=lambda p: p[0])
self.assertEqual(pools, [('foo1', 1, ''), ('foo2', 2, '')])
def test_create_pool(self):
pool = self.client.create_pool(name='foo', slots=1, description='')
self.assertEqual(pool, ('foo', 1, ''))
self.assertEqual(self.session.query(models.Pool).count(), 1)
def test_delete_pool(self):
self.client.create_pool(name='foo', slots=1, description='')
self.assertEqual(self.session.query(models.Pool).count(), 1)
self.client.delete_pool(name='foo')
self.assertEqual(self.session.query(models.Pool).count(), 0)
def trigger_etl_dags():
c = Client(None, None)
dag_ids = get_etl_dags()
for dag_id in dag_ids:
c.trigger_dag(dag_id=dag_id)
def runConvert(conf):
c = Client(None, None)
timestamp = datetime.timestamp(datetime.now())
c.trigger_dag(dag_id='conversion_flow',
run_id='conversion_flow' + str(timestamp),
conf=json.dumps(conf))