def test_prepare_tasks(self):
wb = workbook.WorkbookSpec(WORKBOOK)
tasks = [
db_api.task_create(EXEC_ID, TASK.copy()),
db_api.task_create(EXEC_ID, TASK2.copy())
]
executables = data_flow.prepare_tasks(tasks, CONTEXT, wb)
self.assertEqual(2, len(executables))
self.assertEqual(tasks[0]['id'], executables[0][0])
self.assertEqual('std.echo', executables[0][1])
self.assertDictEqual({'p2': 'val32', 'p3': '', 'p1': 'My string'},
executables[0][2])
self.assertEqual(tasks[1]['id'], executables[1][0])
self.assertEqual('std.echo', executables[1][1])
self.assertDictEqual({'output': 'My string val32'},
executables[1][2])
for task in tasks:
db_task = db_api.task_get(task['id'])
self.assertDictEqual(CONTEXT, db_task['in_context'])
self.assertDictEqual({'p1': 'My string',
'p2': 'val32',
'p3': ''},
db_task['parameters'])
self.assertEqual(states.RUNNING, db_task['state'])
def _create_tasks(cls, task_list, workbook, workbook_name, execution_id):
tasks = []
for task in task_list:
state, task_runtime_context = retry.get_task_runtime(task)
action_ns = workbook.namespaces.get(task.get_action_namespace())
action_spec = None
if action_ns:
action_spec = \
action_ns.actions.get(task.get_action_name())
db_task = db_api.task_create(workbook_name, execution_id, {
"name": task.name,
"requires": task.requires,
"task_spec": task.to_dict(),
"action_spec": {} if not action_spec
else action_spec.to_dict(),
"state": state,
"tags": task.get_property("tags", None),
"task_runtime_context": task_runtime_context
})
tasks.append(db_task)
return tasks
def test_handle_task(self):
# Create a new workbook.
workbook = db_api.workbook_create(SAMPLE_WORKBOOK)
self.assertIsInstance(workbook, dict)
# Create a new execution.
execution = db_api.execution_create(SAMPLE_EXECUTION['workbook_name'],
SAMPLE_EXECUTION)
self.assertIsInstance(execution, dict)
# Create a new task.
SAMPLE_TASK['execution_id'] = execution['id']
task = db_api.task_create(SAMPLE_TASK['workbook_name'],
SAMPLE_TASK['execution_id'],
SAMPLE_TASK)
self.assertIsInstance(task, dict)
self.assertIn('id', task)
# Send the task request to the Executor.
ex_client = executor.ExecutorClient(self.transport)
ex_client.handle_task(SAMPLE_CONTEXT, task=task)
# Check task execution state.
db_task = db_api.task_get(task['workbook_name'],
task['execution_id'],
task['id'])
self.assertEqual(db_task['state'], states.SUCCESS)
def test_handle_task(self):
# Create a new workbook.
workbook = db_api.workbook_create(SAMPLE_WORKBOOK)
self.assertIsInstance(workbook, dict)
# Create a new execution.
execution = db_api.execution_create(SAMPLE_EXECUTION['workbook_name'],
SAMPLE_EXECUTION)
self.assertIsInstance(execution, dict)
# Create a new task.
SAMPLE_TASK['execution_id'] = execution['id']
task = db_api.task_create(SAMPLE_TASK['workbook_name'],
SAMPLE_TASK['execution_id'],
SAMPLE_TASK)
self.assertIsInstance(task, dict)
self.assertIn('id', task)
# Send the task request to the Executor.
ex_client = executor.ExecutorClient(self.transport)
ex_client.handle_task(SAMPLE_CONTEXT, task=task)
# Check task execution state.
db_task = db_api.task_get(task['workbook_name'],
task['execution_id'],
task['id'])
self.assertEqual(db_task['state'], states.SUCCESS)
def _create_tasks(cls, task_list, workbook, workbook_name, execution_id):
tasks = []
for task in task_list:
state, task_runtime_context = retry.get_task_runtime(task)
action_ns = workbook.namespaces.get(task.get_action_namespace())
action_spec = None
if action_ns:
action_spec = \
action_ns.actions.get(task.get_action_name())
db_task = db_api.task_create(
workbook_name, execution_id, {
"name": task.name,
"requires": task.requires,
"task_spec": task.to_dict(),
"action_spec":
{} if not action_spec else action_spec.to_dict(),
"state": state,
"tags": task.get_property("tags", None),
"task_runtime_context": task_runtime_context
})
tasks.append(db_task)
return tasks
def setUp(self):
super(TestExecutor, self).setUp()
# Create a new workbook.
self.workbook = db_api.workbook_create(SAMPLE_WORKBOOK)
# Create a new execution.
self.execution = db_api.execution_create(
SAMPLE_EXECUTION['workbook_name'], SAMPLE_EXECUTION)
# Create a new task.
SAMPLE_TASK['execution_id'] = self.execution['id']
self.task = db_api.task_create(
SAMPLE_TASK['execution_id'], SAMPLE_TASK)
def test_prepare_tasks(self):
task = db_api.task_create(WB_NAME, EXEC_ID, TASK.copy())
tasks = [task]
data_flow.prepare_tasks(tasks, CONTEXT)
db_task = db_api.task_get(WB_NAME, EXEC_ID, tasks[0]['id'])
self.assertDictEqual(db_task['in_context'], CONTEXT)
self.assertDictEqual(db_task['input'], {
'p1': 'My string',
'p2': 'val32'
})
def test_prepare_tasks(self):
task = db_api.task_create(EXEC_ID, TASK.copy())
tasks = [task]
data_flow.prepare_tasks(tasks, CONTEXT)
db_task = db_api.task_get(tasks[0]['id'])
self.assertDictEqual(CONTEXT, db_task['in_context'])
self.assertDictEqual({'p1': 'My string',
'p2': 'val32',
'p3': ''},
db_task['parameters'])
def test_prepare_tasks(self):
task = db_api.task_create(WB_NAME, EXEC_ID, TASK.copy())
tasks = [task]
data_flow.prepare_tasks(tasks, CONTEXT)
db_task = db_api.task_get(WB_NAME, EXEC_ID, tasks[0]['id'])
self.assertDictEqual(CONTEXT, db_task['in_context'])
self.assertDictEqual({
'p1': 'My string',
'p2': 'val32',
'p3': ''
}, db_task['parameters'])
def _create_tasks(cls, task_list, workbook, workbook_name, execution_id):
tasks = []
for task in task_list:
db_task = db_api.task_create(workbook_name, execution_id, {
"name": task.name,
"requires": task.requires,
"task_spec": task.to_dict(),
"service_spec": workbook.services.get(
task.get_action_service()).to_dict(),
"state": states.IDLE,
"tags": task.get_property("tags", None)
})
tasks.append(db_task)
return tasks
def test_handle_task(self):
# Mock the RestAction
mock_rest_action = self.mock_action_run()
# Create a new workbook.
workbook = db_api.workbook_create(SAMPLE_WORKBOOK)
self.assertIsInstance(workbook, dict)
# Create a new execution.
execution = db_api.execution_create(SAMPLE_EXECUTION['workbook_name'],
SAMPLE_EXECUTION)
self.assertIsInstance(execution, dict)
# Create a new task.
SAMPLE_TASK['execution_id'] = execution['id']
task = db_api.task_create(SAMPLE_TASK['workbook_name'],
SAMPLE_TASK['execution_id'],
SAMPLE_TASK)
self.assertIsInstance(task, dict)
self.assertIn('id', task)
# Send the task request to the Executor.
transport = self.server.transport
ex_client = client.ExecutorClient(transport)
ex_client.handle_task(SAMPLE_CONTEXT, task=task)
# Check task execution state. There is no timeout mechanism in
# unittest. There is an example to add a custom timeout decorator that
# can wrap this test function in another process and then manage the
# process time. However, it seems more straightforward to keep the
# loop finite.
for i in range(0, 50):
db_task = db_api.task_get(task['workbook_name'],
task['execution_id'],
task['id'])
# Ensure the request reached the executor and the action has ran.
if db_task['state'] != states.IDLE:
mock_rest_action.assert_called_once_with()
self.assertIn(db_task['state'],
[states.RUNNING, states.SUCCESS, states.ERROR])
return
time.sleep(0.1)
# Task is not being processed. Throw an exception here.
raise Exception('Timed out waiting for task to be processed.')
def _create_tasks(cls, task_list, workbook, workbook_name, execution_id):
tasks = []
for task in task_list:
state, task_runtime_context = retry.get_task_runtime(task)
action_spec = workbook.get_action(task.get_full_action_name())
db_task = db_api.task_create(execution_id, {
"name": task.name,
"requires": task.get_requires(),
"task_spec": task.to_dict(),
"action_spec": {} if not action_spec
else action_spec.to_dict(),
"state": state,
"tags": task.get_property("tags", None),
"task_runtime_context": task_runtime_context,
"workbook_name": workbook_name
})
tasks.append(db_task)
return tasks
