def test_execute_parallel_vcenter_tasks_failed(
self, m_get_deadline_secs, m_time):
start_time = time.time()
m_time.time.side_effect = lambda: start_time + m_time.time.call_count
m_get_deadline_secs.side_effect = lambda x: start_time + 30
num_tasks = 20
mock_task_descs = []
for index in xrange(num_tasks):
if index < 5:
dummy_task = DummyVimTask(num_ticks=1, fail=False, tid=index)
else:
dummy_task = DummyVimTask(num_ticks=1, fail=True, tid=index)
def return_dummy_task(dummy_task):
return dummy_task
dummy_task_desc = VsphereTaskDescriptor(
pre_task_msg="Pre-dummy %d" % index,
post_task_msg="Post-dummy %d" % index,
create_task_func=partial(return_dummy_task, dummy_task))
mock_task_descs.append(dummy_task_desc)
with self.assertRaises(CurieTestException) as ar:
TaskPoller.execute_parallel_tasks(task_descriptors=mock_task_descs,
poll_secs=0, timeout_secs=30)
self.assertEqual("15 of 20 tasks failed. See log for more details (most "
"recent error message: 'Induced error')",
str(ar.exception))
def test_vcenter_task_timeout_descriptors(self, m_get_deadline_secs, m_time):
start_time = time.time()
m_time.time.side_effect = lambda: start_time + m_time.time.call_count
m_get_deadline_secs.side_effect = lambda x: start_time
num_tasks = 20
mock_task_descs = []
for index in xrange(num_tasks):
if index < 5:
dummy_task = DummyVimTask(num_ticks=1, tid=index) # Completed
else:
dummy_task = DummyVimTask(num_ticks=1e6, tid=index) # In progress
def return_dummy_task(dummy_task):
return dummy_task
dummy_task_desc = VsphereTaskDescriptor(
pre_task_msg="Pre-dummy %d" % index,
post_task_msg="Post-dummy %d" % index,
create_task_func=partial(return_dummy_task, dummy_task))
mock_task_descs.append(dummy_task_desc)
with self.assertRaises(CurieTestException) as ar:
TaskPoller.execute_parallel_tasks(task_descriptors=mock_task_descs,
poll_secs=0, timeout_secs=0)
self.assertEqual("Tasks timed out after '0' seconds (5/20 succeeded)",
str(ar.exception))
def test_get_deadline_secs(self, mock_time):
mock_time.return_value = self.sample_timestamp
self.assertEqual(
TaskPoller.get_deadline_secs(None),
TaskPoller.get_default_timeout_secs() + self.sample_timestamp)
self.assertEqual(TaskPoller.get_deadline_secs(10),
self.sample_timestamp + 10)
def deploy_goldimage_image_service(self, goldimages_directory,
goldimage_name):
"""
Deploy a gold image to the image service.
Args:
goldimage_name (str): Name of the gold image to deploy.
Returns:
str: ID of the created disk image.
"""
arch = self.get_cluster_architecture()
# Select a vdisk format to use. Currently PPC64LE goldimages are only built
# using qcow2 format and the x86_64 in vmdk. We could have the manager
# perform a conversion, but acropolis can already do the image conversion
# for us.
if arch == GoldImageManager.ARCH_PPC64LE:
disk_format = GoldImageManager.FORMAT_QCOW2
else:
disk_format = GoldImageManager.FORMAT_VMDK
# Use the GoldImage manager to get a path to our appropriate goldimage
goldimage_manager = GoldImageManager(goldimages_directory)
goldimage_path = goldimage_manager.get_goldimage_path(
goldimage_name, format_str=disk_format, arch=arch)
log.debug("Deploying %s to cluster", goldimage_path)
# Deploy the image to service
disk_name = os.path.splitext(os.path.basename(goldimage_path))[0]
img_uuid, tid, _ = self._prism_client.images_create(
NameUtil.goldimage_vmdisk_name(disk_name, "os"), goldimage_path,
self._container_id)
TaskPoller.execute_parallel_tasks(tasks=PrismTask.from_task_id(
self._prism_client, tid),
timeout_secs=3600)
# NB: Required due to possible AHV bug. See XRAY-225.
num_images_get_retries = 5
for attempt_num in xrange(num_images_get_retries):
images_get_data = self._prism_client.images_get(image_id=img_uuid)
image_state = images_get_data["image_state"]
if image_state.lower() == "active":
# Return the disk image
return images_get_data["vm_disk_id"]
else:
log.info(
"Waiting for created image to become active "
"(imageState: %s, retry %d of %d)", image_state,
attempt_num + 1, num_images_get_retries)
log.debug(images_get_data)
time.sleep(1)
else:
raise CurieException(
CurieError.kInternalError,
"Created image failed to become active within "
"%d attempts" % num_images_get_retries)
def create_vm(self,
goldimages_directory,
goldimage_name,
vm_name,
vcpus=1,
ram_mb=1024,
node_id=None,
datastore_name=None,
data_disks=()):
"""
See 'Cluster.create_vm' for documentation.
"""
log.info(
"Creating VM %s based on %s with %d vCPUs, %d MB RAM and %s "
"disks on node %s in datastore %s ", vm_name, goldimage_name,
vcpus, ram_mb, str(data_disks), str(node_id), datastore_name)
image_uuid = self.deploy_goldimage_image_service(
goldimages_directory, goldimage_name)
# This namedtuple hackery is to handle the expectations in vm.py which
# expects information directly parsed from an OVF file.
Units = namedtuple("Units", ["multiplier"])
Disk = namedtuple("Disk", ["capacity", "units"])
attach_disks = [
Disk(gb, Units(1024 * 1024 * 1024)) for gb in data_disks
]
vm_desc = VmDescriptor(name=vm_name,
memory_mb=ram_mb,
num_vcpus=vcpus,
vmdisk_uuid_list=[image_uuid],
attached_disks=attach_disks,
container_uuid=self._container_id)
# Create the VM
log.info("Creating VM '%s' with %s MB RAM and %s vCPUs", vm_desc.name,
vm_desc.memory_mb, vm_desc.num_vcpus)
nic_specs = \
[vm_desc.to_ahv_vm_nic_create_spec(self._network_id)["specList"][0]]
resp = self._prism_client.vms_create(vm_desc, nic_specs)
tid = resp.get("taskUuid")
if not tid:
raise CurieException(CurieError.kManagementServerApiError,
"Failed to deploy VM: %s" % resp)
TaskPoller.execute_parallel_tasks(tasks=PrismTask.from_task_id(
self._prism_client, tid),
timeout_secs=60)
task_json = self._prism_client.tasks_get_by_id(tid)
vm_uuid = task_json["entityList"][0]["uuid"]
# Make a Curie VM descriptor and assign it to the requested node
vm = self.__vm_json_to_curie_vm(
self._prism_client.vms_get_by_id(vm_uuid))
vm._node_id = node_id
return vm
def test_execute_parallel_vcenter_tasks_failed_no_raise(
self, m_get_deadline_secs, m_time):
start_time = time.time()
m_time.time.side_effect = lambda: start_time + m_time.time.call_count
m_get_deadline_secs.side_effect = lambda x: start_time + 30
num_tasks = 20
mock_task_descs = []
for index in xrange(num_tasks):
if index < 5:
dummy_task = DummyVimTask(num_ticks=1, fail=False, tid=index)
else:
dummy_task = DummyVimTask(num_ticks=1, fail=True, tid=index)
def return_dummy_task(dummy_task):
return dummy_task
dummy_task_desc = VsphereTaskDescriptor(
pre_task_msg="Pre-dummy %d" % index,
post_task_msg="Post-dummy %d" % index,
create_task_func=partial(return_dummy_task, dummy_task))
mock_task_descs.append(dummy_task_desc)
task_map = TaskPoller.execute_parallel_tasks(
task_descriptors=mock_task_descs, poll_secs=0, timeout_secs=30,
raise_on_failure=False)
tasks = task_map.values()
for task in tasks[:5]:
self.assertEqual("success", task._state)
for task in tasks[5:]:
self.assertEqual("error", task._state)
def cleanup_images(self):
"""
Cleans up image service, removing any images associated with curie.
"""
images = self._prism_client.images_get().get("entities", {})
to_delete_image_uuids = []
for image in images:
if image["name"].startswith(CURIE_GOLDIMAGE_VM_DISK_PREFIX):
to_delete_image_uuids.append(image["uuid"])
log.info("Deleting images %s",
", ".join([i for i in to_delete_image_uuids]))
task_map = self._prism_client.images_delete(to_delete_image_uuids)
image_id_tid_map = {}
for image_id, tid in task_map.iteritems():
image_id_tid_map[image_id] = PrismTask.from_task_id(
self._prism_client, tid)
TaskPoller.execute_parallel_tasks(tasks=image_id_tid_map.values(),
timeout_secs=300)
def test_vcenter_task_timeout(self, m_get_deadline_secs, m_time):
start_time = time.time()
m_time.time.side_effect = lambda: start_time + m_time.time.call_count
m_get_deadline_secs.side_effect = lambda x: start_time
num_tasks = 20
mock_tasks = []
for index in xrange(num_tasks):
if index < 5:
dummy_task = DummyVimTask(num_ticks=1, tid=index) # Completed
else:
dummy_task = DummyVimTask(num_ticks=1e6, tid=index) # In progress
vsphere_task = VsphereTask.from_vim_task(dummy_task)
mock_tasks.append(vsphere_task)
with self.assertRaises(CurieTestException) as ar:
TaskPoller.execute_parallel_tasks(tasks=mock_tasks,
poll_secs=0, timeout_secs=0)
self.assertEqual("Tasks timed out after '0' seconds (5/20 succeeded)",
str(ar.exception))
def test_vcenter_task_base_exception(self):
dummy_task = DummyVimTaskBaseExc(not_found=True)
poller = TaskPoller(10, poll_interval_secs=0)
vsphere_task = VsphereTask.from_vim_task(dummy_task)
poller.add_task(vsphere_task)
poller.start()
ret = poller.wait_for()
self.assertEqual(vsphere_task.get_status(), TaskStatus.kInternalError)
def test_vcenter_task_from_vim_task(self):
dummy_task = DummyVimTask()
poller = TaskPoller(10, poll_interval_secs=0)
vsphere_task = VsphereTask.from_vim_task(dummy_task)
poller.add_task(vsphere_task)
poller.start()
ret = poller.wait_for()
self.assertNotEqual(ret, None)
def test_vcenter_task_not_found(self):
dummy_task = DummyVimTask(not_found=True)
poller = TaskPoller(10, poll_interval_secs=0)
vsphere_task = VsphereTask.from_vim_task(dummy_task)
poller.add_task(vsphere_task)
poller.start()
ret = poller.wait_for()
self.assertNotEqual(ret, None)
self.assertTrue(vsphere_task.is_terminal())
self.assertEqual(vsphere_task.get_status(), TaskStatus.kNotFound)
def test_prism_task_from_id(self, mock_tasks_get_by_id):
dummy_task = DummyPrismTask()
mock_tasks_get_by_id.side_effect = MockTasksGetById()
poller = TaskPoller(10, poll_interval_secs=0)
prism_task = PrismTask.from_task_id(self.prism, dummy_task.id())
mock_tasks_get_by_id.side_effect.add_task(dummy_task)
poller.add_task(prism_task)
poller.start()
ret = poller.wait_for()
self.assertNotEqual(ret, None)
def _test_task_multiple(self, create_task_funcs):
poller = TaskPoller(2, poll_interval_secs=0)
for ii in range(4):
task_func_index = ii % len(create_task_funcs)
create_task_funcs[task_func_index].tids.append(str(uuid.uuid4()))
task = create_task_funcs[task_func_index].create_curie_task_instance()
poller.add_task(task)
poller.start()
tid = create_task_funcs[0].tids[0]
log.info("Waiting for %s", tid)
ret = poller.wait_for(task_id=tid)
log.info("Done waiting for %s: %s", tid, ret)
self.assert_(tid in ret)
self.assert_(poller._remaining_task_count <= 2)
log.info("Waiting for remaining tasks...")
poller.wait_for_all()