def _sample_vm(self): flavor = Flavor(name="flavor", cost=[QuotaLineItem("a", "b", 1)]) disks = [ Disk(stable_uuid("%s-1" % id), flavor, False, False, 1, datastore=Datastore("ds1")), Disk(stable_uuid("%s-2" % id), flavor, False, False, 1, datastore=Datastore("ds2")), ] vm = Vm() vm.id = id vm.flavor = "flavor" vm.flavor_info = flavor vm.disks = disks vm.state = State().STARTED vm.datastore = Datastore("ds1") return vm
def create_resource(self, id): flavor = Flavor(name="flavor", cost=[QuotaLineItem("a", "b", 1)]) disks = [ Disk(stable_uuid("%s-1" % id), flavor, False, False, 1, datastore=Datastore("ds1")), Disk(stable_uuid("%s-2" % id), flavor, False, False, 1, datastore=Datastore("ds2")), ] vm = Vm() vm.id = id vm.flavor = "flavor" vm.flavor_info = flavor vm.state = State().STARTED vm.datastore = Datastore("ds1") resource = Resource(vm, disks) return resource
def test_place_on_multiple_datastores(self): """ Test placement can actually place vm to datastores without image. """ host_datastores = self.vim_client.get_all_datastores() image_datastore = self._find_configured_datastore_in_host_config() dest_datastore = None for ds in host_datastores: if ds.id != image_datastore.id: dest_datastore = ds break if not dest_datastore: raise SkipTest() # Test only 2 datastores, with one image datastore and another # datastore. self.provision_hosts( datastores=[image_datastore.name, dest_datastore.name], used_for_vms=False) self.client_connections() concurrency = 3 atmoic_lock = threading.Lock() results = {"count": 0} # Only copy image to datastore[0] new_image_id = str(uuid.uuid4()) datastore = Datastore(id=image_datastore.name) src_image = Image("ttylinux", datastore) dst_image = Image(new_image_id, datastore) request = Host.CopyImageRequest(src_image, dst_image) response = self.host_client.copy_image(request) assert_that(response.result, is_(CopyImageResultCode.OK)) def _thread(): self._test_create_vm_with_ephemeral_disks(new_image_id, concurrent=True, new_client=True) with atmoic_lock: results["count"] += 1 threads = [] for i in range(concurrency): thread = threading.Thread(target=_thread) threads.append(thread) thread.start() for thread in threads: thread.join() # Make sure the new image is copied to both datastores, and clean # them up. for ds in (image_datastore, dest_datastore): image = Image(datastore=Datastore(id=ds.name), id=new_image_id) self._delete_image(image) assert_that(results["count"], is_(concurrency))
def _to_thrift_datastore(self, ds): """ From vim.Datastore to gen.resource.ttypes.Datastore """ uuid = ds.info.url.rsplit("/", 1)[1] name = ds.name type = ds.summary.type system_tag = None tags = [] if type == "VMFS": if ds.info.vmfs.local: thrift_type = DatastoreType.LOCAL_VMFS system_tag = LOCAL_VMFS_TAG else: thrift_type = DatastoreType.SHARED_VMFS system_tag = SHARED_VMFS_TAG elif type == "NFS": thrift_type = DatastoreType.NFS_3 system_tag = NFS_TAG elif type == "NFSV41": thrift_type = DatastoreType.NFS_41 system_tag = NFS_TAG elif type == "vsan": thrift_type = DatastoreType.VSAN else: thrift_type = DatastoreType.OTHER # Set datastore tags if system_tag: tags.append(system_tag) return Datastore(uuid, name, thrift_type, frozenset(tags))
def _to_thrift_datastore(self, ds): """ From VimDatastore to gen.resource.ttypes.Datastore """ if not ds.id: return None system_tag = None tags = [] if ds.type == "VMFS": if ds.local: thrift_type = DatastoreType.LOCAL_VMFS system_tag = LOCAL_VMFS_TAG else: thrift_type = DatastoreType.SHARED_VMFS system_tag = SHARED_VMFS_TAG elif ds.type == "NFS": thrift_type = DatastoreType.NFS_3 system_tag = NFS_TAG elif ds.type == "NFSV41": thrift_type = DatastoreType.NFS_41 system_tag = NFS_TAG elif ds.type == "vsan": thrift_type = DatastoreType.VSAN system_tag = VSAN_TAG else: thrift_type = DatastoreType.OTHER # Set datastore tags if system_tag: tags.append(system_tag) return Datastore(ds.id, ds.name, thrift_type, frozenset(tags))
def test_generated_events(self): chairman_list = ["%s:%s" % (self.chairman_host, self.chairman_port)] chairman_clients_num = 100 host_num = 50 self.agent_host = "localhost" self.agent_port = 20000 ds_id1 = str(uuid.uuid4()) ds_id2 = str(uuid.uuid4()) ds_id3 = str(uuid.uuid4()) datastores = [ Datastore(ds_id1, "ds1", DatastoreType.SHARED_VMFS), Datastore(ds_id2, "ds2", DatastoreType.SHARED_VMFS), Datastore(ds_id3, "ds3", DatastoreType.SHARED_VMFS) ] availability_zones = [str(uuid.uuid4()), str(uuid.uuid4())] networks = [Network("nw1", [NetworkType.VM])] sim1 = Simulator(chairman_list, chairman_clients_num, host_num, self.agent_host, self.agent_port, datastores, availability_zones, networks) self.hosts = sim1.hosts sim1.init() self._write_root_address(self.agent_host, self.agent_port) for x in xrange(100): thread = threading.Thread(target=sim1.run_batch, args=(100, )) thread.daemon = True thread.start() thread.join() time.sleep(45) print "sleeping before sync" sim1.sync_machines() print "finished sync" root = sim1.hosts[Simulator.ROOT_HOST_ID] self._check_internal_consistancy(sim1.hosts, root) self._check_chairmans_consistancy() self._check_sim_state_with_chairman(root, sim1.hosts)
def test_copy_image(self): """ Copy image unit tests """ handler = HostHandler(MagicMock()) image_mgr = MagicMock() image_mgr.check_image = MagicMock(return_value=False) handler.hypervisor.image_manager = image_mgr # Simulates ds name-to-id normalization by assuming all parameters are # datastore ids, exception those ending in "_name" are treated as names # which normalizes to an id with suffix removed. def fake_normalize(ds_name_or_id): if ds_name_or_id.endswith("_name"): return ds_name_or_id[:-5] return ds_name_or_id mock_ds_manager = handler.hypervisor.datastore_manager mock_ds_manager.datastore_type.return_value = DatastoreType.EXT3 mock_ds_manager.normalize.side_effect = fake_normalize # Check self copy is a no-op. src_image = Image("id1", Datastore("datastore1")) dst_image = Image("id1", Datastore("datastore1")) req = CopyImageRequest(src_image, dst_image) response = handler.copy_image(req) self.assertEqual(response.result, CopyImageResultCode.OK) # Still a self copy if datastores normalizes to the same id. src_image = Image("id1", Datastore("datastore1_name")) req = CopyImageRequest(src_image, dst_image) response = handler.copy_image(req) self.assertEqual(response.result, CopyImageResultCode.OK) # Check that if image is not found we return the correct code. dst_image = Image("id1", Datastore("datastore2")) req = CopyImageRequest(src_image, dst_image) response = handler.copy_image(req) self.assertEqual(response.result, CopyImageResultCode.IMAGE_NOT_FOUND) # Test destination image already exists. image_mgr.check_image = MagicMock(return_value=True) image_mgr.copy_image = MagicMock(side_effect=DiskAlreadyExistException) req = CopyImageRequest(src_image, dst_image) response = handler.copy_image(req) self.assertEqual(response.result, CopyImageResultCode.DESTINATION_ALREADY_EXIST) # Happy path. src_image = Image("id1", Datastore("datastore1_name")) dst_image = Image("id1", Datastore("datastore2_name")) image_mgr.check_image = MagicMock(return_value=True) image_mgr.copy_image = MagicMock() req = CopyImageRequest(src_image, dst_image) response = handler.copy_image(req) self.assertEqual(response.result, CopyImageResultCode.OK) image_mgr.check_image.assert_called_once_with("id1", "datastore1") image_mgr.copy_image.assert_called_once_with("datastore1", "id1", "datastore2", "id1")
def __init__(self, system, datastores, image_datastore): self._system = system self._datastores = [] self._datastore_names = datastores self._datastore_id_to_name = {} self._image_datastore = None for name in datastores: ds_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, str(name))) ds = Datastore(ds_id, name, DatastoreType.EXT3, []) self._datastores.append(ds) self._datastore_id_to_name[ds_id] = name if name == image_datastore: self._image_datastore = ds_id self._system.add_datastore_gb(ds_id)
def __init__(self, system, datastores, image_datastore): self._system = system self._datastores = [] self._datastore_names = datastores self._datastore_id_to_name = {} self._image_datastore = None self.ds_user_tags = common.services.get(ServiceName.DATASTORE_TAGS) for name in datastores: ds_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, str(name))) ds = Datastore(ds_id, name, DatastoreType.EXT3, []) self._datastores.append(ds) self._datastore_id_to_name[ds_id] = name if name == image_datastore: self._image_datastore = ds_id self._system.add_datastore_gb(ds_id)
def _to_thrift_datastore(self, ds): """ From vim.Datastore to gen.resource.ttypes.Datastore """ # Ignore this datastore if it has no url if not ds.info.url: self.logger.critical("Ignoring %s because info.url of this datastore is empty" % ds.name) return None uuid = ds.info.url.rsplit("/", 1)[1] name = ds.name type = ds.summary.type system_tag = None tags = [] if type == "VMFS": # if 'local' property is not available then we fall back to old # way of getting to know local/shared access which was used in API versions before 5.5. if hasattr(ds.info.vmfs, 'local'): shared = not ds.info.vmfs.local else: shared = ds.summary.multipleHostAccess if not shared: thrift_type = DatastoreType.LOCAL_VMFS system_tag = LOCAL_VMFS_TAG else: thrift_type = DatastoreType.SHARED_VMFS system_tag = SHARED_VMFS_TAG elif type == "NFS": thrift_type = DatastoreType.NFS_3 system_tag = NFS_TAG elif type == "NFSV41": thrift_type = DatastoreType.NFS_41 system_tag = NFS_TAG elif type == "vsan": thrift_type = DatastoreType.VSAN system_tag = VSAN_TAG else: thrift_type = DatastoreType.OTHER # Set datastore tags if system_tag: tags.append(system_tag) return Datastore(uuid, name, thrift_type, frozenset(tags))
def get_register_host_request(self, port=8080): """ Generates a random register host request which has same datastore and same availability zone """ host_id = str(uuid.uuid4()) if not hasattr(self, "image_datastore"): self.image_datastore = str(uuid.uuid4()) datastores = [Datastore(self.image_datastore)] networks = [Network("nw1", [NetworkType.VM])] host_config = HostConfig(agent_id=host_id, datastores=datastores, address=ServerAddress("127.0.0.1", port=port), networks=networks) host_config.availability_zone = "foo" host_config.image_datastore_ids = set(self.image_datastore) return RegisterHostRequest(host_id, host_config)
def test_get_hosts_from_zk(self): hosts = get_hosts_from_zk(self.zk_client) assert_that(len(hosts), is_(0)) networks = [Network("nw1", [NetworkType.VM])] dsid = str(uuid.uuid4()) datastores = [Datastore(dsid, "ds1", DatastoreType.SHARED_VMFS)] # Register two hosts agent_host = "localhost" agent1_port = 12345 req1 = get_register_host_request(agent_host, agent1_port, agent_id="host1", networks=networks, datastores=datastores, image_datastore=dsid, availability_zone="az1") agent2_port = 12346 req2 = get_register_host_request(agent_host, agent2_port, agent_id="host2", networks=networks, datastores=datastores, image_datastore=dsid, availability_zone="az1") # Register two hosts resp = self.chairman_client.register_host(req1) assert_that(resp.result, is_(RegisterHostResultCode.OK)) resp = self.chairman_client.register_host(req2) assert_that(resp.result, is_(RegisterHostResultCode.OK)) hosts = get_hosts_from_zk(self.zk_client) # map list to dict indexed by host id hosts = dict((h.id, h) for h in hosts) assert_that(len(hosts), is_(2)) _h1 = hosts[req1.config.agent_id] _h2 = hosts[req2.config.agent_id] # Verify that the requests match the hosts that were # constructed by get_hosts_from_zk assert_that(req1.config.agent_id, _h1.id) assert_that(req2.config.agent_id, _h2.id) assert_that(req1.config.address.host, _h1.address) assert_that(req2.config.address.port, _h2.port)
def setUp(self): self.hostname = "localhost" self.host_port = 1234 self.availability_zone_id = "test" self.host_addr = ServerAddress(self.hostname, self.host_port) self.chairman_list = [] self.agent_id = "foo" self.host_config = HostConfig(self.agent_id, self.availability_zone_id, [Datastore("bar")], self.host_addr, [Network("nw1")]) self.registrant = ChairmanRegistrant(self.chairman_list) host_handler = MagicMock() host_handler.get_host_config_no_logging.return_value = \ GetConfigResponse(hostConfig=self.host_config) common.services.register(Host.Iface, host_handler) self.request = RegisterHostRequest("foo", self.host_config) self.state_file = tempfile.mktemp() common.services.register(ServiceName.MODE, Mode(State(self.state_file)))
def build(self): hypervisor = MagicMock() hypervisor.datastore_manager = MagicMock() hypervisor.datastore_manager.vm_datastores.return_value = \ [ds for ds in self.ds_map.keys() if ds != self.image_ds] hypervisor.datastore_manager.image_datastore.return_value = \ self.image_ds hypervisor.datastore_manager.get_datastore_ids.return_value = \ self.ds_map.keys() hypervisor.datastore_manager.datastore_info = self.datastore_info hypervisor.datastore_manager.normalize.side_effect = self.normalize hypervisor.datastore_manager.get_datastores.return_value = \ [Datastore(id=ds_id, tags=self.ds_map[ds_id][1]) for ds_id in self.ds_map.keys()] hypervisor.network_manager.get_vm_networks.return_value = \ self.vm_networks hypervisor.system = MagicMock() hypervisor.system.total_vmusable_memory_mb.return_value = \ self.total_mem hypervisor.system.host_version.return_value = \ self.host_version hypervisor.system.num_physical_cpus.return_value = 1 hypervisor.image_manager = MagicMock() hypervisor.image_manager.get_image_id_from_disks.return_value = \ self.image_id hypervisor.image_manager.check_image = self.check_image hypervisor.image_manager.image_size.return_value = self.image_size hypervisor.vm_manager = MagicMock() hypervisor.vm_manager.get_used_memory_mb.return_value = 0 placement_option = PlacementOption(self.mem_overcommit, self.cpu_overcommit, self.image_datastores) return PlacementManager(hypervisor, placement_option)
def test_get_datastores(self, mkdir_mock): """ Test esx datastore manager with different datastore types. Verify the datastore types are correctly parsed and all the datastores are populated. """ hypervisor = MagicMock() vim_client = MagicMock() vim_client.get_all_datastores.return_value = self.get_datastore_mock([ # name, url, type, local ["datastore1", "/vmfs/volumes/id-1", "VMFS", True], ["datastore2", "/vmfs/volumes/id-2", "VMFS", False], ["datastore3", "/vmfs/volumes/id-3", "NFS", None], ["datastore4", "/vmfs/volumes/id-4", "NFSV41", None], ["datastore5", "/vmfs/volumes/id-5", "vsan", None], ["datastore6", "/vmfs/volumes/id-6", "VFFS", None], ]) hypervisor.vim_client = vim_client ds_list = [ "datastore1", "datastore2", "datastore3", "datastore4", "datastore5", "datastore6" ] image_ds = [{"name": "datastore2", "used_for_vms": False}] ds_manager = EsxDatastoreManager(hypervisor, ds_list, image_ds) expected_call_args = [] for ds in ds_list: for folder in [ DISK_FOLDER_NAME, VM_FOLDER_NAME, IMAGE_FOLDER_NAME, TMP_IMAGE_FOLDER_NAME ]: expected_call_args.append('/vmfs/volumes/%s/%s' % (ds, folder)) called_args = [c[0][0] for c in mkdir_mock.call_args_list] assert_that(called_args, contains_inanyorder(*expected_call_args)) assert_that( ds_manager.get_datastore_ids(), contains_inanyorder("id-1", "id-2", "id-3", "id-4", "id-5", "id-6")) assert_that( ds_manager.vm_datastores(), contains_inanyorder("id-1", "id-3", "id-4", "id-5", "id-6")) datastores = ds_manager.get_datastores() assert_that( datastores, contains_inanyorder( Datastore("id-1", "datastore1", type=DSType.LOCAL_VMFS, tags=set([LOCAL_VMFS_TAG])), Datastore("id-2", "datastore2", type=DSType.SHARED_VMFS, tags=set([SHARED_VMFS_TAG])), Datastore("id-3", "datastore3", type=DSType.NFS_3, tags=set([NFS_TAG])), Datastore("id-4", "datastore4", type=DSType.NFS_41, tags=set([NFS_TAG])), Datastore("id-5", "datastore5", type=DSType.VSAN, tags=set()), Datastore("id-6", "datastore6", type=DSType.OTHER, tags=set()))) assert_that(ds_manager.image_datastores(), is_(["id-2"])) assert_that(ds_manager.datastore_type("id-1"), is_(DSType.LOCAL_VMFS)) assert_that(ds_manager.datastore_type("id-2"), is_(DSType.SHARED_VMFS)) assert_that(ds_manager.datastore_type("id-3"), is_(DSType.NFS_3)) assert_that(ds_manager.datastore_type("id-4"), is_(DSType.NFS_41)) assert_that(ds_manager.datastore_type("id-5"), is_(DSType.VSAN)) assert_that(ds_manager.datastore_type("id-6"), is_(DSType.OTHER)) # test normalize assert_that(ds_manager.normalize("id-1"), is_("id-1")) assert_that(ds_manager.normalize("datastore1"), is_("id-1"))
def test_get_datastores(self, mkdir_mock): """ Test esx datastore manager with different datastore types. Verify the datastore types are correctly parsed and all the datastores are populated. """ hypervisor = MagicMock() vim_client = MagicMock() dstags = MagicMock() dstags.get.return_value = [] common.services.register(ServiceName.DATASTORE_TAGS, dstags) vim_client.get_datastore.side_effect = self._get_datastore hypervisor.vim_client = vim_client ds_list = [ "datastore1", "datastore2", "datastore3", "datastore4", "datastore5", "datastore6" ] ds_manager = EsxDatastoreManager(hypervisor, ds_list, set(["datastore2"])) expected_call_args = [] for ds in ds_list: for folder in [ DISK_FOLDER_NAME, VM_FOLDER_NAME, IMAGE_FOLDER_NAME, TMP_IMAGE_FOLDER_NAME ]: expected_call_args.append('/vmfs/volumes/%s/%s' % (ds, folder)) called_args = [c[0][0] for c in mkdir_mock.call_args_list] assert_that(called_args, equal_to(expected_call_args)) assert_that(ds_manager.get_datastore_ids(), has_length(6)) assert_that( ds_manager.get_datastore_ids(), contains_inanyorder("id-1", "id-2", "id-3", "id-4", "id-5", "id-6")) assert_that(ds_manager.vm_datastores(), has_length(5)) assert_that( ds_manager.vm_datastores(), contains_inanyorder("id-1", "id-3", "id-4", "id-5", "id-6")) datastores = ds_manager.get_datastores() assert_that( datastores[0], is_( Datastore("id-1", "datastore1", type=DSType.LOCAL_VMFS, tags=[LOCAL_VMFS_TAG]))) assert_that( datastores[1], is_( Datastore("id-2", "datastore2", type=DSType.SHARED_VMFS, tags=[SHARED_VMFS_TAG]))) assert_that( datastores[2], is_( Datastore("id-3", "datastore3", type=DSType.NFS_3, tags=[NFS_TAG]))) assert_that( datastores[3], is_( Datastore("id-4", "datastore4", type=DSType.NFS_41, tags=[NFS_TAG]))) assert_that( datastores[4], is_(Datastore("id-5", "datastore5", type=DSType.VSAN, tags=[]))) assert_that( datastores[5], is_(Datastore("id-6", "datastore6", type=DSType.OTHER, tags=[]))) assert_that(ds_manager.image_datastores(), is_(set(["id-2"]))) assert_that(ds_manager.datastore_type("id-1"), is_(DSType.LOCAL_VMFS)) assert_that(ds_manager.datastore_type("id-2"), is_(DSType.SHARED_VMFS)) assert_that(ds_manager.datastore_type("id-3"), is_(DSType.NFS_3)) assert_that(ds_manager.datastore_type("id-4"), is_(DSType.NFS_41)) assert_that(ds_manager.datastore_type("id-5"), is_(DSType.VSAN)) assert_that(ds_manager.datastore_type("id-6"), is_(DSType.OTHER)) # test normalize assert_that(ds_manager.normalize("id-1"), is_("id-1")) assert_that(ds_manager.normalize("datastore1"), is_("id-1"))
def test_get_datastores(self): """ Test esx datastore manager with different datastore types. Verify the datastore types are correctly parsed and all the datastores are populated. """ hypervisor = MagicMock() vim_client = MagicMock() vim_client.get_all_datastores.return_value = self.get_datastore_mock([ # name, url, type, local ["datastore1", "/vmfs/volumes/id-1", "VMFS", True], ["datastore2", "/vmfs/volumes/id-2", "VMFS", False], ["datastore3", "/vmfs/volumes/id-3", "NFS", None], ["datastore4", "/vmfs/volumes/id-4", "NFSV41", None], ["datastore5", "/vmfs/volumes/id-5", "vsan", None], ["datastore6", "/vmfs/volumes/id-6", "VFFS", None], ]) hypervisor.vim_client = vim_client ds_list = [ "datastore1", "datastore2", "datastore3", "datastore4", "datastore5", "datastore6" ] image_ds = [{"name": "datastore2", "used_for_vms": False}] ds_manager = EsxDatastoreManager(hypervisor, ds_list, image_ds) assert_that( ds_manager.get_datastore_ids(), contains_inanyorder("id-1", "id-2", "id-3", "id-4", "id-5", "id-6")) assert_that( ds_manager.vm_datastores(), contains_inanyorder("id-1", "id-3", "id-4", "id-5", "id-6")) datastores = ds_manager.get_datastores() assert_that( datastores, contains_inanyorder( Datastore("id-1", "datastore1", type=DSType.LOCAL_VMFS, tags=set([LOCAL_VMFS_TAG])), Datastore("id-2", "datastore2", type=DSType.SHARED_VMFS, tags=set([SHARED_VMFS_TAG])), Datastore("id-3", "datastore3", type=DSType.NFS_3, tags=set([NFS_TAG])), Datastore("id-4", "datastore4", type=DSType.NFS_41, tags=set([NFS_TAG])), Datastore("id-5", "datastore5", type=DSType.VSAN, tags=set([VSAN_TAG])), Datastore("id-6", "datastore6", type=DSType.OTHER, tags=set()))) assert_that(ds_manager.image_datastores(), is_(["id-2"])) assert_that(ds_manager.datastore_type("id-1"), is_(DSType.LOCAL_VMFS)) assert_that(ds_manager.datastore_type("id-2"), is_(DSType.SHARED_VMFS)) assert_that(ds_manager.datastore_type("id-3"), is_(DSType.NFS_3)) assert_that(ds_manager.datastore_type("id-4"), is_(DSType.NFS_41)) assert_that(ds_manager.datastore_type("id-5"), is_(DSType.VSAN)) assert_that(ds_manager.datastore_type("id-6"), is_(DSType.OTHER)) # test normalize assert_that(ds_manager.normalize("id-1"), is_("id-1")) assert_that(ds_manager.normalize("datastore1"), is_("id-1"))