def test_execute_parallel_tasks_duplicate_id(self, m_sleep):
m_sleep.return_value = 0
m_vmm_client = mock.Mock(spec=VmmClient)
# Just using create_vm as an example.
m_vmm_client.create_vm.__name__ = "create_vm"
m_vmm_client.create_vm.side_effect = [
[{"task_id": "1", "task_type": "vmm"}],
[{"task_id": "1", "task_type": "vmm"}],
]
task_desc_list = [
HypervTaskDescriptor(
pre_task_msg="Doing Task 1",
post_task_msg="Finished Task 1",
create_task_func=m_vmm_client.create_vm),
HypervTaskDescriptor(
pre_task_msg="Doing Task 2",
post_task_msg="Finished Task 2",
create_task_func=m_vmm_client.create_vm),
]
with self.assertRaises(CurieTestException) as ar:
HypervTaskPoller.execute_parallel_tasks(task_descriptors=task_desc_list,
vmm=m_vmm_client)
self.assertEqual(
str(ar.exception),
"Unhandled exception occurred in HypervTaskPoller while waiting for "
"tasks: Duplicate task ID '1' found in response to 'create_vm'")
def test_execute_parallel_task_failure_and_restart_failure(
self, m_log, m_sleep, m_time):
m_sleep.return_value = 0
start_time = time.time()
m_time.time.side_effect = lambda: start_time + m_time.time.call_count
m_vmm_client = mock.Mock(spec=VmmClient)
# Just using create_vm as an example.
m_vmm_client.create_vm.side_effect = [
[{"task_id": "1", "task_type": "vmm"}],
]
m_vmm_client.vm_restart_vmm_job.side_effect = [
[{"task_id": "2", "task_type": "vmm"}],
[{"task_id": "3", "task_type": "vmm"}],
]
m_vmm_client.vm_get_job_status.side_effect = [
[{"task_id": "1", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "10",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "failed",
"status": "Failed", "completed": True, "progress": "100",
"error": "Hyper-V task 1 hit by a PowerShell",
},
],
# after task_id '1' is restarted we get new task_id '2'
[{"task_id": "2", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "90",
"error": "",
},
],
[{"task_id": "2", "task_type": "vmm", "state": "failed",
"status": "Failed", "completed": True, "progress": "100",
"error": "Hyper-V task 2 hit by a PowerShell",
},
],
]
m_vmm_client.vm_stop_job.side_effect = [[]]
task_desc_list = [
HypervTaskDescriptor(
pre_task_msg="Doing Task 1",
post_task_msg="Finished Task 1",
create_task_func=m_vmm_client.create_vm,
vmm_restart_count=1),
]
with self.assertRaises(CurieTestException) as ar:
HypervTaskPoller.execute_parallel_tasks(task_descriptors=task_desc_list,
vmm=m_vmm_client)
self.assertEqual(
str(ar.exception),
"1 of 1 tasks failed. See log for more details (most recent error "
"message: 'Hyper-V task 2 hit by a PowerShell')")
def test_execute_parallel_tasks_failure_and_restart(
self, m_log, m_sleep, m_time):
m_sleep.return_value = 0
start_time = time.time()
m_time.time.side_effect = lambda: start_time + m_time.time.call_count
m_vmm_client = mock.Mock(spec=VmmClient)
# Just using create_vm as an example.
m_vmm_client.create_vm.side_effect = [
[{"task_id": "1", "task_type": "vmm"}],
[{"task_id": "2", "task_type": "vmm"}],
[{"task_id": "3", "task_type": "vmm"}],
]
m_vmm_client.vm_restart_vmm_job.side_effect = [
[{"task_id": "4", "task_type": "vmm"}],
[{"task_id": "5", "task_type": "vmm"}],
]
m_vmm_client.vm_get_job_status.side_effect = [
[{"task_id": "1", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "10",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "20",
"error": "",
},
{"task_id": "3", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "20",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "90",
"error": "",
},
{"task_id": "3", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "90",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "failed",
"status": "Failed", "completed": True, "progress": "100",
"error": "Hyper-V task hit by a PowerShell",
},
{"task_id": "3", "task_type": "vmm", "state": "failed",
"status": "Failed", "completed": True, "progress": "100",
"error": "Hyper-V task hit by a PowerShell",
},
],
# after task_id '2' & '3' are restarted we get new task_id '4' & '5'
[{"task_id": "4", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "90",
"error": "",
},
{"task_id": "5", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "90",
"error": "",
},
],
[{"task_id": "4", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "5", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
],
]
m_vmm_client.vm_stop_job.side_effect = [[]]
task_desc_list = [
HypervTaskDescriptor(
pre_task_msg="Doing Task 1",
post_task_msg="Finished Task 1",
create_task_func=m_vmm_client.create_vm,
vmm_restart_count=1),
HypervTaskDescriptor(
pre_task_msg="Doing Task 2",
post_task_msg="Finished Task 2",
create_task_func=m_vmm_client.create_vm,
vmm_restart_count=1),
HypervTaskDescriptor(
pre_task_msg="Doing Task 3",
post_task_msg="Finished Task 3",
create_task_func=m_vmm_client.create_vm,
vmm_restart_count=1),
]
HypervTaskPoller.execute_parallel_tasks(task_descriptors=task_desc_list,
vmm=m_vmm_client)
self.assertIn(
mock.call("Polling %d active tasks... (%d / %d tasks remaining)", 3, 3, 3),
m_log.info.mock_calls)
self.assertIn(mock.call("Task: 1 running (10%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Task: 2 running (20%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Task: 3 running (20%)"), m_log.info.mock_calls)
self.assertIn(
mock.call("Polling %d active tasks... (%d / %d tasks remaining)", 3, 3, 3),
m_log.info.mock_calls)
self.assertIn(mock.call("Task: 1 completed (100%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Task: 2 running (90%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Task: 3 running (90%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Finished Task 1"), m_log.info.mock_calls)
self.assertIn(
mock.call("Polling %d active tasks... (%d / %d tasks remaining)", 2, 2, 3),
m_log.info.mock_calls)
self.assertIn(mock.call("Task: 2 failed (100%)"), m_log.error.mock_calls)
self.assertIn(mock.call("Task: 3 failed (100%)"), m_log.error.mock_calls)
self.assertIn(
mock.call("Restart %d of task %s successful. New task is %s.", 1, '2', '5'),
m_log.info.mock_calls)
self.assertIn(
mock.call("Restart %d of task %s successful. New task is %s.", 1, '3', '4'),
m_log.info.mock_calls)
self.assertIn(
mock.call("Polling %d active tasks... (%d / %d tasks remaining)", 2, 2, 3),
m_log.info.mock_calls)
self.assertIn(mock.call("Task: 5 running (90%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Task: 4 running (90%)"), m_log.info.mock_calls)
self.assertIn(
mock.call("Polling %d active tasks... (%d / %d tasks remaining)", 2, 2, 3),
m_log.info.mock_calls)
self.assertIn(mock.call("Task: 5 completed (100%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Task: 4 completed (100%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Finished Task 2"), m_log.info.mock_calls)
self.assertIn(mock.call("Finished Task 3"), m_log.info.mock_calls)
def test_execute_parallel_tasks_failure(self, m_sleep):
m_sleep.return_value = 0
m_vmm_client = mock.Mock(spec=VmmClient)
# Just using create_vm as an example.
m_vmm_client.create_vm.side_effect = [
[{"task_id": "1", "task_type": "vmm"}],
[{"task_id": "2", "task_type": "vmm"}],
]
m_vmm_client.vm_get_job_status.side_effect = [
[{"task_id": "1", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "10",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "20",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "50",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "60",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "90",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "failed",
"status": "Failed", "completed": True, "progress": "100",
"error": "Hyper-V task hit by a PowerShell",
},
],
]
m_vmm_client.vm_stop_job.side_effect = [[]]
task_desc_list = [
HypervTaskDescriptor(
pre_task_msg="Doing Task 1",
post_task_msg="Finished Task 1",
create_task_func=m_vmm_client.create_vm),
HypervTaskDescriptor(
pre_task_msg="Doing Task 2",
post_task_msg="Finished Task 2",
create_task_func=m_vmm_client.create_vm),
]
with self.assertRaises(CurieTestException) as ar:
HypervTaskPoller.execute_parallel_tasks(task_descriptors=task_desc_list,
vmm=m_vmm_client)
self.assertEqual(
str(ar.exception),
"1 of 2 tasks failed. See log for more details (most recent error "
"message: 'Hyper-V task hit by a PowerShell')")
def test_execute_parallel_tasks_in_order(self, m_log, m_sleep):
m_sleep.return_value = 0
m_vmm_client = mock.Mock(spec=VmmClient)
# Just using create_vm as an example.
m_vmm_client.create_vm.side_effect = [
[{"task_id": "1", "task_type": "vmm"}],
[{"task_id": "2", "task_type": "vmm"}],
[{"task_id": "3", "task_type": "vmm"}],
[{"task_id": "4", "task_type": "vmm"}],
]
m_vmm_client.vm_get_job_status.side_effect = [
[{"task_id": "1", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "10",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "10",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "3", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "10",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "3", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "3", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "4", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "10",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "3", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "4", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
],
]
task_desc_list = [
HypervTaskDescriptor(
pre_task_msg="Doing Task 1",
post_task_msg="Finished Task 1",
create_task_func=m_vmm_client.create_vm),
HypervTaskDescriptor(
pre_task_msg="Doing Task 2",
post_task_msg="Finished Task 2",
create_task_func=m_vmm_client.create_vm),
HypervTaskDescriptor(
pre_task_msg="Doing Task 3",
post_task_msg="Finished Task 3",
create_task_func=m_vmm_client.create_vm),
HypervTaskDescriptor(
pre_task_msg="Doing Task 4",
post_task_msg="Finished Task 4",
create_task_func=m_vmm_client.create_vm),
]
HypervTaskPoller.execute_parallel_tasks(task_descriptors=task_desc_list,
vmm=m_vmm_client,
max_parallel=1)
m_vmm_client.vm_get_job_status.assert_has_calls([
mock.call([{"task_id": "1", "task_type": "vmm"}]),
mock.call([{"task_id": "1", "task_type": "vmm"}]),
mock.call([{"task_id": "2", "task_type": "vmm"}]),
mock.call([{"task_id": "2", "task_type": "vmm"}]),
mock.call([{"task_id": "3", "task_type": "vmm"}]),
mock.call([{"task_id": "3", "task_type": "vmm"}]),
mock.call([{"task_id": "4", "task_type": "vmm"}]),
mock.call([{"task_id": "4", "task_type": "vmm"}]),
])
self.assertEqual(m_log.info.mock_calls[0], mock.call('Started %s: %s', '1', 'Doing Task 1'))
self.assertEqual(m_log.info.mock_calls[1], mock.call('Polling %d active tasks... (%d / %d tasks remaining)', 1, 4, 4))
self.assertEqual(m_log.info.mock_calls[2], mock.call('Task: 1 running (10%)'))
self.assertEqual(m_log.info.mock_calls[3], mock.call('Polling %d active tasks... (%d / %d tasks remaining)', 1, 4, 4))
self.assertEqual(m_log.info.mock_calls[4], mock.call('Task: 1 completed (100%)'))
self.assertEqual(m_log.info.mock_calls[6], mock.call('Finished Task 1'))
self.assertEqual(m_log.info.mock_calls[7], mock.call('Started %s: %s', '2', 'Doing Task 2'))
self.assertEqual(m_log.info.mock_calls[8], mock.call('Polling %d active tasks... (%d / %d tasks remaining)', 1, 3, 4))
self.assertEqual(m_log.info.mock_calls[9], mock.call('Task: 2 running (10%)'))
self.assertEqual(m_log.info.mock_calls[10], mock.call('Polling %d active tasks... (%d / %d tasks remaining)', 1, 3, 4))
self.assertEqual(m_log.info.mock_calls[11], mock.call('Task: 2 completed (100%)'))
self.assertEqual(m_log.info.mock_calls[13], mock.call('Finished Task 2'))
self.assertEqual(m_log.info.mock_calls[14], mock.call('Started %s: %s', '3', 'Doing Task 3'))
self.assertEqual(m_log.info.mock_calls[15], mock.call('Polling %d active tasks... (%d / %d tasks remaining)', 1, 2, 4))
self.assertEqual(m_log.info.mock_calls[16], mock.call('Task: 3 running (10%)'))
self.assertEqual(m_log.info.mock_calls[17], mock.call('Polling %d active tasks... (%d / %d tasks remaining)', 1, 2, 4))
self.assertEqual(m_log.info.mock_calls[18], mock.call('Task: 3 completed (100%)'))
self.assertEqual(m_log.info.mock_calls[20], mock.call('Finished Task 3'))
self.assertEqual(m_log.info.mock_calls[21], mock.call('Started %s: %s', '4', 'Doing Task 4'))
self.assertEqual(m_log.info.mock_calls[22], mock.call('Polling %d active tasks... (%d / %d tasks remaining)', 1, 1, 4))
self.assertEqual(m_log.info.mock_calls[23], mock.call('Task: 4 running (10%)'))
self.assertEqual(m_log.info.mock_calls[24], mock.call('Polling %d active tasks... (%d / %d tasks remaining)', 1, 1, 4))
self.assertEqual(m_log.info.mock_calls[25], mock.call('Task: 4 completed (100%)'))
self.assertEqual(m_log.info.mock_calls[27], mock.call('Finished Task 4'))
def test_execute_parallel_tasks_defaults(self, m_log, m_sleep):
m_sleep.return_value = 0
m_vmm_client = mock.Mock(spec=VmmClient)
# Just using create_vm as an example.
m_vmm_client.create_vm.side_effect = [
[{"task_id": "1", "task_type": "vmm"}],
[{"task_id": "2", "task_type": "vmm"}],
]
m_vmm_client.vm_get_job_status.side_effect = [
[{"task_id": "1", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "10",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "20",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "50",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "60",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "running",
"status": "Running", "completed": False, "progress": "90",
"error": "",
},
],
[{"task_id": "1", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
{"task_id": "2", "task_type": "vmm", "state": "completed",
"status": "Completed", "completed": True, "progress": "100",
"error": "",
},
],
]
task_desc_list = [
HypervTaskDescriptor(
pre_task_msg="Doing Task 1",
post_task_msg="Finished Task 1",
create_task_func=m_vmm_client.create_vm),
HypervTaskDescriptor(
pre_task_msg="Doing Task 2",
post_task_msg="Finished Task 2",
create_task_func=m_vmm_client.create_vm),
]
HypervTaskPoller.execute_parallel_tasks(task_descriptors=task_desc_list,
vmm=m_vmm_client)
m_vmm_client.vm_get_job_status.assert_has_calls([
mock.call([{"task_id": "1", "task_type": "vmm"},
{"task_id": "2", "task_type": "vmm"},
]),
mock.call([{"task_id": "1", "task_type": "vmm"},
{"task_id": "2", "task_type": "vmm"},
]),
mock.call([{"task_id": "1", "task_type": "vmm"},
{"task_id": "2", "task_type": "vmm"},
]),
mock.call([{"task_id": "2", "task_type": "vmm"},
]),
])
self.assertIn(
mock.call("Polling %d active tasks... (%d / %d tasks remaining)", 2, 2, 2),
m_log.info.mock_calls)
self.assertIn(mock.call("Task: 1 running (10%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Task: 2 running (20%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Task: 1 running (50%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Task: 2 running (60%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Task: 1 completed (100%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Finished Task 1"), m_log.info.mock_calls)
self.assertIn(mock.call("Task: 2 running (90%)"), m_log.info.mock_calls)
self.assertIn(
mock.call("Polling %d active tasks... (%d / %d tasks remaining)", 1, 1, 2),
m_log.info.mock_calls)
self.assertIn(mock.call("Task: 2 completed (100%)"), m_log.info.mock_calls)
self.assertIn(mock.call("Finished Task 2"), m_log.info.mock_calls)