def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
self.host_client = Hypervisor.create_host_client(
errback=lambda: suicide())
self.host_client.connect_local()
atexit.register(lambda client: client.disconnect(), self.host_client)
self.datastore_manager = DatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.host_client.add_update_listener(self.datastore_manager)
self.vm_manager = VmManager(self.host_client, self.datastore_manager)
self.disk_manager = DiskManager(self.host_client,
self.datastore_manager)
self.image_manager = ImageManager(self.host_client,
self.datastore_manager)
self.network_manager = NetworkManager(self.host_client)
self.system = System(self.host_client)
options = PlacementOption(agent_config.memory_overcommit,
agent_config.cpu_overcommit,
agent_config.image_datastores)
self.placement_manager = PlacementManager(self, options)
self.image_monitor = ImageMonitor(self.datastore_manager,
self.image_manager, self.vm_manager)
self.image_manager.monitor_for_cleanup()
self.image_transferer = NfcImageTransferer(self.host_client)
atexit.register(self.image_manager.cleanup)
def test_single_datastore(self, mkdir_mock):
"""Test that datastore manager works with a single datastore."""
hypervisor = MagicMock()
host_client = MagicMock()
host_client.get_all_datastores.return_value = self.get_datastore_mock([
# name, url, type, local
["datastore1", "id-1", "VMFS", True],
])
hypervisor.host_client = host_client
# No valid datastore. One image datastore for cloud VMs.
ds_list = []
image_ds = [{"name": "datastore1", "used_for_vms": True}]
ds_manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(ds_manager.initialized, is_(True))
assert_that(ds_manager.get_datastore_ids(), is_(["id-1"]))
assert_that(ds_manager.vm_datastores(), is_([]))
assert_that(ds_manager.image_datastores(), is_(["id-1"]))
# No valid datastore. No image datastore for cloud VMs.
ds_list = ["bad-ds"]
image_ds = [{"name": "datastore1", "used_for_vms": False}]
ds_manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(ds_manager.initialized, is_(False))
def test_nonexistent_datastores(self, mkdir_mock):
"""Test that non-existent datastore get filtered out."""
host_client = MagicMock()
host_client.get_all_datastores.return_value = self.get_datastore_mock([
["datastore1", "id-1", "VMFS", True],
["datastore2", "id-2", "VMFS", True],
["datastore3", "id-3", "VMFS", True],
])
hypervisor = MagicMock()
hypervisor.host_client = host_client
ds_list = ["datastore1", "bad-datastore1"]
image_ds = [
{
"name": "datastore2",
"used_for_vms": True
},
{
"name": "datastore3",
"used_for_vms": False
},
{
"name": "bad-datastores2",
"used_for_vms": False
},
]
manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(manager.get_datastore_ids(),
contains_inanyorder("id-1", "id-2", "id-3"))
assert_that(manager.vm_datastores(), is_(["id-1"]))
assert_that(manager.image_datastores(),
contains_inanyorder("id-2", "id-3"))
assert_that(manager.initialized, is_(True))
def test_multiple_image_datastores(self, mkdir_mock):
"""Test that datastore manager works with multiple image datastores."""
host_client = MagicMock()
host_client.get_all_datastores.return_value = self.get_datastore_mock([
["datastore1", "id-1", "VMFS", True],
["datastore2", "id-2", "VMFS", True],
["datastore3", "id-3", "VMFS", True],
])
hypervisor = MagicMock()
hypervisor.host_client = host_client
ds_list = ["datastore1"]
image_ds = [
{
"name": "datastore2",
"used_for_vms": True
},
{
"name": "datastore3",
"used_for_vms": False
},
]
ds_manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(ds_manager.get_datastore_ids(),
contains_inanyorder("id-1", "id-2", "id-3"))
assert_that(ds_manager.vm_datastores(), is_(["id-1"]))
assert_that(ds_manager.image_datastores(),
contains_inanyorder("id-2", "id-3"))
assert_that(ds_manager.initialized, is_(True))
def test_empty_url_datastores(self, mkdir_mock):
"""Test that datastores with empty url get filtered out."""
host_client = MagicMock()
host_client.get_all_datastores.return_value = self.get_datastore_mock([
["datastore1", "", "VMFS", True],
["datastore2", None, "VMFS", True],
["datastore3", "id-3", "VMFS", True],
])
hypervisor = MagicMock()
hypervisor.host_client = host_client
ds_list = ["datastore1", "datastore2", "datastore3"]
image_ds = [{"name": "datastore3", "used_for_vms": True}]
manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(manager.get_datastore_ids(), contains("id-3"))
def test_empty_url_datastores(self, mkdir_mock):
"""Test that datastores with empty url get filtered out."""
host_client = MagicMock()
host_client.get_all_datastores.return_value = self.get_datastore_mock([
["datastore1", "", "VMFS", True],
["datastore2", None, "VMFS", True],
["datastore3", "id-3", "VMFS", True],
])
hypervisor = MagicMock()
hypervisor.host_client = host_client
ds_list = ["datastore1", "datastore2", "datastore3"]
image_ds = [{"name": "datastore3", "used_for_vms": True}]
manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(manager.get_datastore_ids(), contains("id-3"))
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
self.host_client = Hypervisor.create_host_client(errback=lambda: suicide())
self.host_client.connect_local()
atexit.register(lambda client: client.disconnect(), self.host_client)
self.datastore_manager = DatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.host_client.add_update_listener(self.datastore_manager)
self.vm_manager = VmManager(self.host_client, self.datastore_manager)
self.disk_manager = DiskManager(self.host_client, self.datastore_manager)
self.image_manager = ImageManager(self.host_client, self.datastore_manager)
self.network_manager = NetworkManager(self.host_client)
self.system = System(self.host_client)
options = PlacementOption(agent_config.memory_overcommit,
agent_config.cpu_overcommit,
agent_config.image_datastores)
self.placement_manager = PlacementManager(self, options)
self.image_monitor = ImageMonitor(self.datastore_manager,
self.image_manager,
self.vm_manager)
self.image_manager.monitor_for_cleanup()
self.image_transferer = NfcImageTransferer(self.host_client)
atexit.register(self.image_manager.cleanup)
def test_single_datastore(self, mkdir_mock):
"""Test that datastore manager works with a single datastore."""
hypervisor = MagicMock()
host_client = MagicMock()
host_client.get_all_datastores.return_value = self.get_datastore_mock([
# name, url, type, local
["datastore1", "id-1", "VMFS", True],
])
hypervisor.host_client = host_client
# No valid datastore. One image datastore for cloud VMs.
ds_list = []
image_ds = [{"name": "datastore1", "used_for_vms": True}]
ds_manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(ds_manager.initialized, is_(True))
assert_that(ds_manager.get_datastore_ids(), is_(["id-1"]))
assert_that(ds_manager.vm_datastores(), is_([]))
assert_that(ds_manager.image_datastores(), is_(["id-1"]))
# No valid datastore. No image datastore for cloud VMs.
ds_list = ["bad-ds"]
image_ds = [{"name": "datastore1", "used_for_vms": False}]
ds_manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(ds_manager.initialized, is_(False))
def test_multiple_image_datastores(self, mkdir_mock):
"""Test that datastore manager works with multiple image datastores."""
host_client = MagicMock()
host_client.get_all_datastores.return_value = self.get_datastore_mock([
["datastore1", "id-1", "VMFS", True],
["datastore2", "id-2", "VMFS", True],
["datastore3", "id-3", "VMFS", True],
])
hypervisor = MagicMock()
hypervisor.host_client = host_client
ds_list = ["datastore1"]
image_ds = [
{"name": "datastore2", "used_for_vms": True},
{"name": "datastore3", "used_for_vms": False},
]
ds_manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(ds_manager.get_datastore_ids(),
contains_inanyorder("id-1", "id-2", "id-3"))
assert_that(ds_manager.vm_datastores(), is_(["id-1"]))
assert_that(ds_manager.image_datastores(),
contains_inanyorder("id-2", "id-3"))
assert_that(ds_manager.initialized, is_(True))
def test_nonexistent_datastores(self, mkdir_mock):
"""Test that non-existent datastore get filtered out."""
host_client = MagicMock()
host_client.get_all_datastores.return_value = self.get_datastore_mock([
["datastore1", "id-1", "VMFS", True],
["datastore2", "id-2", "VMFS", True],
["datastore3", "id-3", "VMFS", True],
])
hypervisor = MagicMock()
hypervisor.host_client = host_client
ds_list = ["datastore1", "bad-datastore1"]
image_ds = [
{"name": "datastore2", "used_for_vms": True},
{"name": "datastore3", "used_for_vms": False},
{"name": "bad-datastores2", "used_for_vms": False},
]
manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(manager.get_datastore_ids(),
contains_inanyorder("id-1", "id-2", "id-3"))
assert_that(manager.vm_datastores(), is_(["id-1"]))
assert_that(manager.image_datastores(),
contains_inanyorder("id-2", "id-3"))
assert_that(manager.initialized, is_(True))
class Hypervisor(object):
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
self.host_client = Hypervisor.create_host_client(errback=lambda: suicide())
self.host_client.connect_local()
atexit.register(lambda client: client.disconnect(), self.host_client)
self.datastore_manager = DatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.host_client.add_update_listener(self.datastore_manager)
self.vm_manager = VmManager(self.host_client, self.datastore_manager)
self.disk_manager = DiskManager(self.host_client, self.datastore_manager)
self.image_manager = ImageManager(self.host_client, self.datastore_manager)
self.network_manager = NetworkManager(self.host_client)
self.system = System(self.host_client)
options = PlacementOption(agent_config.memory_overcommit,
agent_config.cpu_overcommit,
agent_config.image_datastores)
self.placement_manager = PlacementManager(self, options)
self.image_monitor = ImageMonitor(self.datastore_manager,
self.image_manager,
self.vm_manager)
self.image_manager.monitor_for_cleanup()
self.image_transferer = NfcImageTransferer(self.host_client)
atexit.register(self.image_manager.cleanup)
@staticmethod
def create_host_client(auto_sync=True, errback=None):
try:
# check whether attache is installed. If not, find_module will throw ImportError.
from host.hypervisor.esx.attache_client import AttacheClient
return AttacheClient(auto_sync, errback)
except ImportError:
from host.hypervisor.esx.vim_client import VimClient
return VimClient(auto_sync, errback)
def check_image(self, image_id, datastore_id):
return self.image_manager.check_image(
image_id, self.datastore_manager.datastore_name(datastore_id)
)
def get_resources(self):
return self.vm_manager.get_resources()
def get_vm_resource(self, vm_id):
vm = self.vm_manager.get_resource(vm_id)
resource = Resource(vm=vm)
return resource
def acquire_vim_ticket(self):
return self.host_client.get_vim_ticket()
def add_update_listener(self, listener):
self.host_client.add_update_listener(listener)
def remove_update_listener(self, listener):
self.host_client.remove_update_listener(listener)
def transfer_image(self, source_image_id, source_datastore,
destination_image_id, destination_datastore,
host, port):
return self.image_transferer.send_image_to_host(
source_image_id, source_datastore,
destination_image_id, destination_datastore, host, port)
@property
def memory_overcommit(self):
return self.placement_manager.memory_overcommit
def set_memory_overcommit(self, value):
self.placement_manager.memory_overcommit = value
@property
def cpu_overcommit(self):
return self.placement_manager.cpu_overcommit
def set_cpu_overcommit(self, value):
self.placement_manager.cpu_overcommit = value
class Hypervisor(object):
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
# If VimClient's housekeeping thread failed to update its own cache,
# call errback to commit suicide. Watchdog will bring up the agent
# again.
self.host_client = Hypervisor.create_host_client(
errback=lambda: suicide())
self.host_client.connect_local()
atexit.register(lambda client: client.disconnect(), self.host_client)
self.datastore_manager = DatastoreManager(
self, agent_config.datastores, agent_config.image_datastores)
# datastore manager needs to update the cache when there is a change.
self.host_client.add_update_listener(self.datastore_manager)
self.vm_manager = VmManager(self.host_client, self.datastore_manager)
self.disk_manager = DiskManager(self.host_client,
self.datastore_manager)
self.image_manager = ImageManager(self.host_client,
self.datastore_manager)
self.network_manager = NetworkManager(self.host_client)
self.system = System(self.host_client)
options = PlacementOption(agent_config.memory_overcommit,
agent_config.cpu_overcommit,
agent_config.image_datastores)
self.placement_manager = PlacementManager(self, options)
self.image_monitor = ImageMonitor(self.datastore_manager,
self.image_manager, self.vm_manager)
self.image_manager.monitor_for_cleanup()
self.image_transferer = NfcImageTransferer(self.host_client)
atexit.register(self.image_manager.cleanup)
@staticmethod
def create_host_client(auto_sync=True, errback=None):
try:
# check whether attache is installed. If not, find_module will throw ImportError.
from host.hypervisor.esx.attache_client import AttacheClient
return AttacheClient(auto_sync, errback)
except ImportError:
from host.hypervisor.esx.vim_client import VimClient
return VimClient(auto_sync, errback)
def check_image(self, image_id, datastore_id):
return self.image_manager.check_image(
image_id, self.datastore_manager.datastore_name(datastore_id))
def get_resources(self):
return self.vm_manager.get_resources()
def get_vm_resource(self, vm_id):
vm = self.vm_manager.get_resource(vm_id)
resource = Resource(vm=vm)
return resource
def acquire_vim_ticket(self):
return self.host_client.get_vim_ticket()
def add_update_listener(self, listener):
self.host_client.add_update_listener(listener)
def remove_update_listener(self, listener):
self.host_client.remove_update_listener(listener)
def transfer_image(self, source_image_id, source_datastore,
destination_image_id, destination_datastore, host,
port):
return self.image_transferer.send_image_to_host(
source_image_id, source_datastore, destination_image_id,
destination_datastore, host, port)
@property
def memory_overcommit(self):
return self.placement_manager.memory_overcommit
def set_memory_overcommit(self, value):
self.placement_manager.memory_overcommit = value
@property
def cpu_overcommit(self):
return self.placement_manager.cpu_overcommit
def set_cpu_overcommit(self, value):
self.placement_manager.cpu_overcommit = value
def test_get_datastores(self):
""" Test esx datastore manager with different datastore types.
Verify the datastore types are correctly parsed and all the
datastores are populated.
"""
hypervisor = MagicMock()
host_client = MagicMock()
host_client.get_all_datastores.return_value = self.get_datastore_mock([
# name, url, type, local
["datastore1", "id-1", "VMFS", True],
["datastore2", "id-2", "VMFS", False],
["datastore3", "id-3", "NFS", None],
["datastore4", "id-4", "NFSV41", None],
["datastore5", "id-5", "vsan", None],
["datastore6", "id-6", "VFFS", None],
])
hypervisor.host_client = host_client
ds_list = ["datastore1", "datastore2", "datastore3",
"datastore4", "datastore5", "datastore6"]
image_ds = [{"name": "datastore2", "used_for_vms": False}]
ds_manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(ds_manager.get_datastore_ids(),
contains_inanyorder("id-1", "id-2", "id-3", "id-4",
"id-5", "id-6"))
assert_that(ds_manager.vm_datastores(),
contains_inanyorder("id-1", "id-3", "id-4", "id-5",
"id-6"))
datastores = ds_manager.get_datastores()
assert_that(datastores, contains_inanyorder(
Datastore("id-1", "datastore1", type=DSType.LOCAL_VMFS,
tags=set([LOCAL_VMFS_TAG])),
Datastore("id-2", "datastore2", type=DSType.SHARED_VMFS,
tags=set([SHARED_VMFS_TAG])),
Datastore("id-3", "datastore3", type=DSType.NFS_3,
tags=set([NFS_TAG])),
Datastore("id-4", "datastore4", type=DSType.NFS_41,
tags=set([NFS_TAG])),
Datastore("id-5", "datastore5", type=DSType.VSAN, tags=set([VSAN_TAG])),
Datastore("id-6", "datastore6", type=DSType.OTHER, tags=set())))
assert_that(ds_manager.image_datastores(), is_(["id-2"]))
assert_that(ds_manager.datastore_type("id-1"),
is_(DSType.LOCAL_VMFS))
assert_that(ds_manager.datastore_type("id-2"),
is_(DSType.SHARED_VMFS))
assert_that(ds_manager.datastore_type("id-3"),
is_(DSType.NFS_3))
assert_that(ds_manager.datastore_type("id-4"),
is_(DSType.NFS_41))
assert_that(ds_manager.datastore_type("id-5"),
is_(DSType.VSAN))
assert_that(ds_manager.datastore_type("id-6"),
is_(DSType.OTHER))
# test normalize
assert_that(ds_manager.normalize("id-1"), is_("id-1"))
assert_that(ds_manager.normalize("datastore1"), is_("id-1"))
def test_get_datastores(self):
""" Test esx datastore manager with different datastore types.
Verify the datastore types are correctly parsed and all the
datastores are populated.
"""
hypervisor = MagicMock()
host_client = MagicMock()
host_client.get_all_datastores.return_value = self.get_datastore_mock([
# name, url, type, local
["datastore1", "id-1", "VMFS", True],
["datastore2", "id-2", "VMFS", False],
["datastore3", "id-3", "NFS", None],
["datastore4", "id-4", "NFSV41", None],
["datastore5", "id-5", "vsan", None],
["datastore6", "id-6", "VFFS", None],
])
hypervisor.host_client = host_client
ds_list = [
"datastore1", "datastore2", "datastore3", "datastore4",
"datastore5", "datastore6"
]
image_ds = [{"name": "datastore2", "used_for_vms": False}]
ds_manager = DatastoreManager(hypervisor, ds_list, image_ds)
assert_that(
ds_manager.get_datastore_ids(),
contains_inanyorder("id-1", "id-2", "id-3", "id-4", "id-5",
"id-6"))
assert_that(
ds_manager.vm_datastores(),
contains_inanyorder("id-1", "id-3", "id-4", "id-5", "id-6"))
datastores = ds_manager.get_datastores()
assert_that(
datastores,
contains_inanyorder(
Datastore("id-1",
"datastore1",
type=DSType.LOCAL_VMFS,
tags=set([LOCAL_VMFS_TAG])),
Datastore("id-2",
"datastore2",
type=DSType.SHARED_VMFS,
tags=set([SHARED_VMFS_TAG])),
Datastore("id-3",
"datastore3",
type=DSType.NFS_3,
tags=set([NFS_TAG])),
Datastore("id-4",
"datastore4",
type=DSType.NFS_41,
tags=set([NFS_TAG])),
Datastore("id-5",
"datastore5",
type=DSType.VSAN,
tags=set([VSAN_TAG])),
Datastore("id-6", "datastore6", type=DSType.OTHER,
tags=set())))
assert_that(ds_manager.image_datastores(), is_(["id-2"]))
assert_that(ds_manager.datastore_type("id-1"), is_(DSType.LOCAL_VMFS))
assert_that(ds_manager.datastore_type("id-2"), is_(DSType.SHARED_VMFS))
assert_that(ds_manager.datastore_type("id-3"), is_(DSType.NFS_3))
assert_that(ds_manager.datastore_type("id-4"), is_(DSType.NFS_41))
assert_that(ds_manager.datastore_type("id-5"), is_(DSType.VSAN))
assert_that(ds_manager.datastore_type("id-6"), is_(DSType.OTHER))
# test normalize
assert_that(ds_manager.normalize("id-1"), is_("id-1"))
assert_that(ds_manager.normalize("datastore1"), is_("id-1"))
