def test_commit_multiple_objects(self):
db_api.start_tx()
try:
created = db_api.create_workflow_execution(WF_EXECS[0])
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(created, fetched)
created_wb = db_api.create_workbook(WORKBOOKS[0])
fetched_wb = db_api.get_workbook(created_wb.name)
self.assertEqual(created_wb, fetched_wb)
self.assertTrue(self.is_db_session_open())
db_api.commit_tx()
finally:
db_api.end_tx()
self.assertFalse(self.is_db_session_open())
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(created, fetched)
fetched_wb = db_api.get_workbook(created_wb.name)
self.assertEqual(created_wb, fetched_wb)
self.assertFalse(self.is_db_session_open())
def test_commit_multiple_objects(self):
db_api.start_tx()
try:
created = db_api.create_workflow_execution(WF_EXECS[0])
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(created, fetched)
created_wb = db_api.create_workbook(WORKBOOKS[0])
fetched_wb = db_api.get_workbook(created_wb.name)
self.assertEqual(created_wb, fetched_wb)
self.assertTrue(self.is_db_session_open())
db_api.commit_tx()
finally:
db_api.end_tx()
self.assertFalse(self.is_db_session_open())
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(created, fetched)
fetched_wb = db_api.get_workbook(created_wb.name)
self.assertEqual(created_wb, fetched_wb)
self.assertFalse(self.is_db_session_open())
def _run_correct_locking(self, wf_ex):
self._random_sleep()
with db_api.transaction():
# Lock workflow execution and get the most up-to-date object.
wf_ex = db_api.acquire_lock(db_models.WorkflowExecution, wf_ex.id)
# Refresh the object.
db_api.get_workflow_execution(wf_ex.id)
wf_ex.name = str(int(wf_ex.name) + 1)
return wf_ex.name
def _run_correct_locking(self, wf_ex):
self._random_sleep()
with db_api.transaction():
# Lock workflow execution and get the most up-to-date object.
wf_ex = db_api.acquire_lock(db_models.WorkflowExecution, wf_ex.id)
# Refresh the object.
db_api.get_workflow_execution(wf_ex.id)
wf_ex.name = str(int(wf_ex.name) + 1)
return wf_ex.name
def test_rollback_multiple_objects(self):
db_api.start_tx()
try:
created = db_api.create_workflow_execution(WF_EXECS[0])
fetched = db_api.get_workflow_execution(created['id'])
self.assertEqual(created, fetched)
created_wb = db_api.create_workbook(WORKBOOKS[0])
fetched_wb = db_api.get_workbook(created_wb.name)
self.assertEqual(created_wb, fetched_wb)
self.assertTrue(self.is_db_session_open())
db_api.rollback_tx()
finally:
db_api.end_tx()
self.assertFalse(self.is_db_session_open())
self.assertRaises(
exc.NotFoundException,
db_api.get_workflow_execution,
created.id
)
self.assertRaises(
exc.NotFoundException,
db_api.get_workbook,
created_wb.name
)
self.assertFalse(self.is_db_session_open())
def test_rollback_multiple_objects(self):
db_api.start_tx()
try:
created = db_api.create_workflow_execution(WF_EXECS[0])
fetched = db_api.get_workflow_execution(created['id'])
self.assertEqual(created, fetched)
created_wb = db_api.create_workbook(WORKBOOKS[0])
fetched_wb = db_api.get_workbook(created_wb.name)
self.assertEqual(created_wb, fetched_wb)
self.assertTrue(self.is_db_session_open())
db_api.rollback_tx()
finally:
db_api.end_tx()
self.assertFalse(self.is_db_session_open())
self.assertRaises(exc.NotFoundException, db_api.get_workflow_execution,
created.id)
self.assertRaises(exc.NotFoundException, db_api.get_workbook,
created_wb.name)
self.assertFalse(self.is_db_session_open())
def _run_correct_locking(self, wf_ex):
# Set context info for the thread.
auth_context.set_ctx(test_base.get_context())
self._random_sleep()
with db_api.transaction():
# Lock workflow execution and get the most up-to-date object.
wf_ex = db_api.acquire_lock(db_models.WorkflowExecution, wf_ex.id)
# Refresh the object.
db_api.get_workflow_execution(wf_ex.id)
wf_ex.name = str(int(wf_ex.name) + 1)
return wf_ex.name
def _run_correct_locking(self, wf_ex):
# Set context info for the thread.
auth_context.set_ctx(test_base.get_context())
self._random_sleep()
with db_api.transaction():
# Lock workflow execution and get the most up-to-date object.
wf_ex = db_api.acquire_lock(db_models.WorkflowExecution, wf_ex.id)
# Refresh the object.
db_api.get_workflow_execution(wf_ex.id)
wf_ex.name = str(int(wf_ex.name) + 1)
return wf_ex.name
def test_delete_workflow_execution(self):
created = db_api.create_workflow_execution(WF_EXECS[0])
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(created, fetched)
db_api.delete_workflow_execution(created.id)
self.assertRaises(exc.NotFoundException, db_api.get_workflow_execution,
created.id)
def test_create_and_get_and_load_workflow_execution(self):
created = db_api.create_workflow_execution(WF_EXECS[0])
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(created, fetched)
fetched = db_api.load_workflow_execution(created.id)
self.assertEqual(created, fetched)
self.assertIsNone(db_api.load_workflow_execution("not-existing-id"))
def test_create_and_get_and_load_workflow_execution(self):
created = db_api.create_workflow_execution(WF_EXECS[0])
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(created, fetched)
fetched = db_api.load_workflow_execution(created.id)
self.assertEqual(created, fetched)
self.assertIsNone(db_api.load_workflow_execution("not-existing-id"))
def test_update_workflow_execution_env_wrong_state(self):
wf_exec_template = {
'spec': {},
'start_params': {
'task': 'my_task1'
},
'state': 'PAUSED',
'state_info': None,
'params': {
'env': {
'k1': 'abc'
}
},
'created_at': None,
'updated_at': None,
'context': {
'__env': {
'k1': 'fee fi fo fum'
}
},
'task_id': None,
'trust_id': None,
'description': None,
'output': None
}
states_not_permitted = [
states.RUNNING, states.RUNNING_DELAYED, states.SUCCESS,
states.WAITING
]
update_env = {'k1': 'foobar'}
for state in states_not_permitted:
wf_exec = copy.deepcopy(wf_exec_template)
wf_exec['state'] = state
with db_api.transaction():
created = db_api.create_workflow_execution(wf_exec)
self.assertIsNone(created.updated_at)
self.assertRaises(exc.NotAllowedException,
wf_service.update_workflow_execution_env,
created, update_env)
fetched = db_api.get_workflow_execution(created.id)
self.assertDictEqual(wf_exec['params']['env'],
fetched.params['env'])
self.assertDictEqual(wf_exec['context']['__env'],
fetched.context['__env'])
def test_task_executions(self):
# Add an associated object into collection.
with db_api.transaction():
wf_ex = db_api.create_workflow_execution(WF_EXECS[0])
self.assertEqual(0, len(wf_ex.task_executions))
wf_ex.task_executions.append(
db_models.TaskExecution(**TASK_EXECS[0])
)
# Make sure task execution has been saved.
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertIsNotNone(wf_ex)
self.assertEqual(1, len(wf_ex.task_executions))
task_ex = wf_ex.task_executions[0]
self.assertEqual(TASK_EXECS[0]['name'], task_ex.name)
# Make sure that polymorphic load works correctly.
self.assertEqual(2, len(db_api.get_executions()))
self.assertEqual(1, len(db_api.get_workflow_executions()))
self.assertEqual(1, len(db_api.get_task_executions()))
# Remove task execution from collection.
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
del wf_ex.task_executions[:]
# Make sure task execution has been removed.
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(0, len(wf_ex.task_executions))
self.assertIsNone(db_api.load_task_execution(task_ex.id))
def test_update_workflow_execution_env_wrong_state(self):
wf_exec_template = {
'spec': {},
'start_params': {'task': 'my_task1'},
'state': 'PAUSED',
'state_info': None,
'params': {'env': {'k1': 'abc'}},
'created_at': None,
'updated_at': None,
'context': {'__env': {'k1': 'fee fi fo fum'}},
'task_id': None,
'trust_id': None,
'description': None,
'output': None
}
states_not_permitted = [
states.RUNNING,
states.RUNNING_DELAYED,
states.SUCCESS,
states.WAITING
]
update_env = {'k1': 'foobar'}
for state in states_not_permitted:
wf_exec = copy.deepcopy(wf_exec_template)
wf_exec['state'] = state
with db_api.transaction():
created = db_api.create_workflow_execution(wf_exec)
self.assertIsNone(created.updated_at)
self.assertRaises(
exc.NotAllowedException,
wf_service.update_workflow_execution_env,
created,
update_env
)
fetched = db_api.get_workflow_execution(created.id)
self.assertDictEqual(
wf_exec['params']['env'],
fetched.params['env']
)
self.assertDictEqual(
wf_exec['context']['__env'],
fetched.context['__env']
)
def test_task_executions(self):
# Add an associated object into collection.
with db_api.transaction():
wf_ex = db_api.create_workflow_execution(WF_EXECS[0])
self.assertEqual(0, len(wf_ex.task_executions))
wf_ex.task_executions.append(
db_models.TaskExecution(**TASK_EXECS[0]))
# Make sure task execution has been saved.
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertIsNotNone(wf_ex)
self.assertEqual(1, len(wf_ex.task_executions))
task_ex = wf_ex.task_executions[0]
self.assertEqual(TASK_EXECS[0]['name'], task_ex.name)
# Make sure that polymorphic load works correctly.
self.assertEqual(2, len(db_api.get_executions()))
self.assertEqual(1, len(db_api.get_workflow_executions()))
self.assertEqual(1, len(db_api.get_task_executions()))
# Remove task execution from collection.
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
del wf_ex.task_executions[:]
# Make sure task execution has been removed.
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(0, len(wf_ex.task_executions))
self.assertIsNone(db_api.load_task_execution(task_ex.id))
def test_delete_workflow_execution(self):
created = db_api.create_workflow_execution(WF_EXECS[0])
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(created, fetched)
db_api.delete_workflow_execution(created.id)
self.assertRaises(
exc.NotFoundException,
db_api.get_workflow_execution,
created.id
)
def test_update_workflow_execution_env(self):
wf_exec_template = {
'spec': {},
'start_params': {
'task': 'my_task1'
},
'state': 'PAUSED',
'state_info': None,
'params': {
'env': {
'k1': 'abc'
}
},
'created_at': None,
'updated_at': None,
'context': {
'__env': {
'k1': 'fee fi fo fum'
}
},
'task_id': None,
'trust_id': None,
'description': None,
'output': None
}
states_permitted = [states.IDLE, states.PAUSED, states.ERROR]
update_env = {'k1': 'foobar'}
for state in states_permitted:
wf_exec = copy.deepcopy(wf_exec_template)
wf_exec['state'] = state
with db_api.transaction():
created = db_api.create_workflow_execution(wf_exec)
self.assertIsNone(created.updated_at)
updated = wf_service.update_workflow_execution_env(
created, update_env)
self.assertDictEqual(update_env, updated.params['env'])
self.assertDictEqual(update_env, updated.context['__env'])
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(updated, fetched)
self.assertIsNotNone(fetched.updated_at)
def _run_invalid_locking(self, wf_ex):
self._random_sleep()
with db_api.transaction():
# Load object into the session (transaction).
wf_ex = db_api.get_workflow_execution(wf_ex.id)
# It's too late to lock the object here because it's already
# been loaded into the session so there should be multiple
# threads that read the same object state so they write the
# same value into DB. As a result we won't get a result
# (object name) equal to a number of transactions.
db_api.acquire_lock(db_models.WorkflowExecution, wf_ex.id)
wf_ex.name = str(int(wf_ex.name) + 1)
return wf_ex.name
def _run_invalid_locking(self, wf_ex):
self._random_sleep()
with db_api.transaction():
# Load object into the session (transaction).
wf_ex = db_api.get_workflow_execution(wf_ex.id)
# It's too late to lock the object here because it's already
# been loaded into the session so there should be multiple
# threads that read the same object state so they write the
# same value into DB. As a result we won't get a result
# (object name) equal to a number of transactions.
db_api.acquire_lock(db_models.WorkflowExecution, wf_ex.id)
wf_ex.name = str(int(wf_ex.name) + 1)
return wf_ex.name
def test_update_workflow_execution_env(self):
wf_exec_template = {
'spec': {},
'start_params': {'task': 'my_task1'},
'state': 'PAUSED',
'state_info': None,
'params': {'env': {'k1': 'abc'}},
'created_at': None,
'updated_at': None,
'context': {'__env': {'k1': 'fee fi fo fum'}},
'task_id': None,
'trust_id': None,
'description': None,
'output': None
}
states_permitted = [
states.IDLE,
states.PAUSED,
states.ERROR
]
update_env = {'k1': 'foobar'}
for state in states_permitted:
wf_exec = copy.deepcopy(wf_exec_template)
wf_exec['state'] = state
with db_api.transaction():
created = db_api.create_workflow_execution(wf_exec)
self.assertIsNone(created.updated_at)
updated = wf_service.update_workflow_execution_env(
created,
update_env
)
self.assertDictEqual(update_env, updated.params['env'])
self.assertDictEqual(update_env, updated.context['__env'])
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(updated, fetched)
self.assertIsNotNone(fetched.updated_at)
def _run_correct_locking(self, wf_ex):
self._random_sleep()
with db_api.transaction():
# Here we lock the object before it gets loaded into the
# session and prevent reading the same object state by
# multiple transactions. Hence the rest of the transaction
# body works atomically (in a serialized manner) and the
# result (object name) must be equal to a number of
# transactions.
db_api.acquire_lock(db_models.WorkflowExecution, wf_ex.id)
# Refresh the object.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
wf_ex.name = str(int(wf_ex.name) + 1)
return wf_ex.name
def test_update_workflow_execution(self):
created = db_api.create_workflow_execution(WF_EXECS[0])
self.assertIsNone(created.updated_at)
updated = db_api.update_execution(created.id, {
'state': 'RUNNING',
'state_info': "Running..."
})
self.assertEqual('RUNNING', updated.state)
self.assertEqual('RUNNING',
db_api.load_workflow_execution(updated.id).state)
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(updated, fetched)
self.assertIsNotNone(fetched.updated_at)
def _run_correct_locking(self, wf_ex):
self._random_sleep()
with db_api.transaction():
# Here we lock the object before it gets loaded into the
# session and prevent reading the same object state by
# multiple transactions. Hence the rest of the transaction
# body works atomically (in a serialized manner) and the
# result (object name) must be equal to a number of
# transactions.
db_api.acquire_lock(db_models.WorkflowExecution, wf_ex.id)
# Refresh the object.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
wf_ex.name = str(int(wf_ex.name) + 1)
return wf_ex.name
def test_correct_locking(self):
wf_ex = db_api.create_workflow_execution(WF_EXEC)
threads = []
number = 500
for i in range(1, number):
threads.append(eventlet.spawn(self._run_correct_locking, wf_ex))
[t.wait() for t in threads]
[t.kill() for t in threads]
wf_ex = db_api.get_workflow_execution(wf_ex.id)
print("Correct locking test gave object name: %s" % wf_ex.name)
self.assertEqual(str(number), wf_ex.name)
def test_update_workflow_execution(self):
created = db_api.create_workflow_execution(WF_EXECS[0])
self.assertIsNone(created.updated_at)
updated = db_api.update_execution(
created.id,
{'state': 'RUNNING', 'state_info': "Running..."}
)
self.assertEqual('RUNNING', updated.state)
self.assertEqual(
'RUNNING',
db_api.load_workflow_execution(updated.id).state
)
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(updated, fetched)
self.assertIsNotNone(fetched.updated_at)
def test_create_or_update_workflow_execution(self):
id = 'not-existing-id'
self.assertIsNone(db_api.load_workflow_execution(id))
created = db_api.create_or_update_workflow_execution(id, WF_EXECS[0])
self.assertIsNotNone(created)
self.assertIsNotNone(created.id)
updated = db_api.create_or_update_workflow_execution(
created.id, {'state': 'RUNNING'})
self.assertEqual('RUNNING', updated.state)
self.assertEqual('RUNNING',
db_api.load_workflow_execution(updated.id).state)
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(updated, fetched)
def test_correct_locking(self):
wf_ex = db_api.create_workflow_execution(WF_EXEC)
threads = []
number = 500
for i in range(1, number):
threads.append(
eventlet.spawn(self._run_correct_locking, wf_ex)
)
[t.wait() for t in threads]
[t.kill() for t in threads]
wf_ex = db_api.get_workflow_execution(wf_ex.id)
print("Correct locking test gave object name: %s" % wf_ex.name)
self.assertEqual(str(number), wf_ex.name)
def test_create_or_update_workflow_execution(self):
id = 'not-existing-id'
self.assertIsNone(db_api.load_workflow_execution(id))
created = db_api.create_or_update_workflow_execution(id, WF_EXECS[0])
self.assertIsNotNone(created)
self.assertIsNotNone(created.id)
updated = db_api.create_or_update_workflow_execution(
created.id,
{'state': 'RUNNING'}
)
self.assertEqual('RUNNING', updated.state)
self.assertEqual(
'RUNNING',
db_api.load_workflow_execution(updated.id).state
)
fetched = db_api.get_workflow_execution(created.id)
self.assertEqual(updated, fetched)