def test_gc_image_dir(self, dst_path, src_path):
""" Test that we move the directory correctly to the GC location """
src_dir = file_util.mkdtemp(delete=True)
dst_dir = file_util.mkdtemp(delete=True)
src_path.return_value = os.path.join(src_dir, "test.vmdk")
dst_path.return_value = dst_dir
self.image_manager._gc_image_dir("ds1", "foo")
uuid_dir = os.path.join(dst_dir, os.listdir(dst_dir)[0])
# Verify the src directory has been moved into the garbage dir.
self.assertEqual(os.listdir(uuid_dir), [os.path.basename(src_dir)])
src_path.assert_called_once_with("ds1", "foo", IMAGE_FOLDER_NAME)
dst_path.assert_called_once_with("ds1", GC_IMAGE_FOLDER)
def test_gc_image_dir(self, dst_path, src_path):
""" Test that we move the directory correctly to the GC location """
src_dir = file_util.mkdtemp(delete=True)
dst_dir = file_util.mkdtemp(delete=True)
src_path.return_value = os.path.join(src_dir, "test.vmdk")
dst_path.return_value = dst_dir
self.image_manager._gc_image_dir("ds1", "foo")
uuid_dir = os.path.join(dst_dir, os.listdir(dst_dir)[0])
# Verify the src directory has been moved into the garbage dir.
self.assertEqual(os.listdir(uuid_dir), [os.path.basename(src_dir)])
src_path.assert_called_once_with("ds1", "foo", IMAGE_FOLDER_NAME)
dst_path.assert_called_once_with("ds1", GC_IMAGE_FOLDER)
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
prefix = socket.gethostname()
suffix = str(agent_config.host_port)
self._uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, prefix + suffix))
tempdir = mkdtemp(prefix='disk', delete=True)
self.disk_manager = FakeDiskManager(self,
os.path.join(tempdir, 'disk'))
self.vm_manager = FakeVmManager(self)
self.network_manager = FakeNetworkManager(self, agent_config.networks)
self.system = FakeSystem(self)
datastores = agent_config.datastores
# For fake hypervisor, we assume there is always one image datastore.
if agent_config.image_datastores:
image_datastore = list(agent_config.image_datastores)[0]["name"]
else:
image_datastore = None
self.datastore_manager = FakeDatastoreManager(self.system, datastores,
image_datastore)
self.image_manager = FakeImageManager(self, image_datastore)
self.image_manager.copy_to_datastores(
"ttylinux",
self.datastore_manager.get_datastore_ids())
def test_reap_tmp_images(self, _os_datastore_path, _uuid):
""" Test that stray images are found and deleted by the reaper """
def _fake_ds_folder(datastore, folder):
return "%s__%s" % (datastore, folder)
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_images_folder = _fake_ds_folder(ds.id, TMP_IMAGE_FOLDER_NAME)
tmp_images_dir = os.path.join(tmpdir, tmp_images_folder)
tmp_image_dir = os.path.join(tmp_images_dir, "stray_image")
os.mkdir(tmp_images_dir)
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_path(datastore, folder):
return os.path.join(tmpdir, _fake_ds_folder(datastore, folder))
_os_datastore_path.side_effect = _fake_os_datastore_path
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = EsxImageManager(self.vim_client, ds_manager)
image_manager.reap_tmp_images()
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
def __init__(self, agent_config):
self.logger = logging.getLogger(__name__)
self._multi_agent_id = agent_config.multi_agent_id
prefix = socket.gethostname()
suffix = str(agent_config.host_port)
self._uuid = str(uuid.uuid5(uuid.NAMESPACE_DNS, prefix + suffix))
tempdir = mkdtemp(prefix='disk', delete=True)
self.disk_manager = FakeDiskManager(self,
os.path.join(tempdir, 'disk'))
self.vm_manager = FakeVmManager(self)
self.network_manager = FakeNetworkManager(self, agent_config.networks)
self.system = FakeSystem(self)
datastores = agent_config.datastores
# For fake hypervisor, we assume there is always one image datastore.
if agent_config.image_datastores:
image_datastore = list(agent_config.image_datastores)[0]["name"]
else:
image_datastore = None
self.datastore_manager = FakeDatastoreManager(self.system, datastores,
image_datastore)
self.image_manager = FakeImageManager(self, image_datastore)
self.image_manager.copy_to_datastores(
"ttylinux",
self.datastore_manager.get_datastore_ids())
def __init__(self, id, networks, datastores, cpu, mem, disk, overcommit):
self.id = id
self.cpu = cpu
self.mem = mem
self.disk = disk
self.parent = ""
self.constraints = set()
host_constraint = ResourceConstraint(ResourceConstraintType.HOST,
["host-" + str(id)])
self.constraints.add(host_constraint)
[self.constraints.add(net) for net in networks]
[self.constraints.add(ds) for ds in datastores]
self.address = ""
self.port = ""
conf_dir = mkdtemp(delete=True)
state = State(os.path.join(conf_dir, "state.json"))
common.services.register(ServiceName.MODE, Mode(state))
self.hv = self._get_hypervisor_instance(
id, cpu, mem, disk, [ds.values[0] for ds in datastores],
[network.values[0] for network in networks], overcommit)
# need agent_config for create/delete vm.
agent_config = AgentConfig([
"--config-path", conf_dir, "--hostname", "localhost", "--port",
"1234", "--host-id", id
])
common.services.register(ServiceName.AGENT_CONFIG, agent_config)
super(Host, self).__init__(self.hv)
def setUp(self):
self._test_dir = "/tmp"
self._fs_type = DatastoreType.EXT3
tempdir = mkdtemp(delete=True, dir=self._test_dir)
self._file_name = os.path.join(tempdir, "fileio_lock_test")
self._lock_path = "%s.%s" % (self._file_name, LOCK_EXTENSION)
def setUp(self):
self.agent_conf_dir = mkdtemp(delete=True)
state = State(os.path.join(self.agent_conf_dir, "state.json"))
common.services.register(ServiceName.MODE, Mode(state))
common.services.register(ServiceName.DATASTORE_TAGS,
DatastoreTags(state))
self.agent = AgentConfig(["--config-path", self.agent_conf_dir])
def setUp(self):
self._test_dir = "/tmp"
self._fs_type = DatastoreType.EXT3
tempdir = mkdtemp(delete=True, dir=self._test_dir)
self._file_name = os.path.join(tempdir, "fileio_lock_test")
self._lock_path = "%s.%s" % (self._file_name, LOCK_EXTENSION)
def __init__(self, id, networks, datastores, cpu, mem, disk, overcommit):
self.id = id
self.cpu = cpu
self.mem = mem
self.disk = disk
self.parent = ""
self.constraints = set()
host_constraint = ResourceConstraint(ResourceConstraintType.HOST,
["host-" + str(id)])
self.constraints.add(host_constraint)
[self.constraints.add(net) for net in networks]
[self.constraints.add(ds) for ds in datastores]
self.address = ""
self.port = ""
conf_dir = mkdtemp(delete=True)
state = State(os.path.join(conf_dir, "state.json"))
common.services.register(ServiceName.MODE, Mode(state))
self.hv = self._get_hypervisor_instance(id,
cpu,
mem,
disk,
[ds.values[0] for ds in
datastores],
[network.values[0] for
network in networks],
overcommit)
# need agent_config for create/delete vm.
agent_config = AgentConfig(["--config-path", conf_dir,
"--hostname", "localhost",
"--port", "1234",
"--host-id", id])
common.services.register(ServiceName.AGENT_CONFIG, agent_config)
super(Host, self).__init__(self.hv)
def setUp(self):
self.agent_conf_dir = mkdtemp(delete=True)
state = State(os.path.join(self.agent_conf_dir, "state.json"))
common.services.register(ServiceName.MODE, Mode(state))
common.services.register(ServiceName.DATASTORE_TAGS,
DatastoreTags(state))
self.agent = AgentConfig(["--config-path", self.agent_conf_dir])
def setUp(self):
self.services_helper = ServicesHelper()
self.mock_options = MagicMock()
self.agent_config_dir = mkdtemp(delete=True)
self.agent_config = AgentConfig(["--config-path",
self.agent_config_dir,
"--hypervisor", "esx"])
def test_reap_tmp_images(self, _os_datastore_path, _uuid):
""" Test that stray images are found and deleted by the reaper """
def _fake_ds_folder(datastore, folder):
return "%s__%s" % (datastore, folder)
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_images_folder = _fake_ds_folder(ds.id, TMP_IMAGE_FOLDER_NAME)
tmp_images_dir = os.path.join(tmpdir, tmp_images_folder)
tmp_image_dir = os.path.join(tmp_images_dir, "stray_image")
os.mkdir(tmp_images_dir)
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_path(datastore, folder):
return os.path.join(tmpdir, _fake_ds_folder(datastore, folder))
_os_datastore_path.side_effect = _fake_os_datastore_path
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = EsxImageManager(self.vim_client, ds_manager)
image_manager.reap_tmp_images()
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
def test_rm_rf(self):
tempdir = mkdtemp()
assert_that(os.path.exists(tempdir), is_(True))
rm_rf(tempdir)
assert_that(os.path.exists(tempdir), is_(False))
# rm_rf a dir that doesn't exist shouldn't raise exception
rm_rf(tempdir)
def __init__(self, hypervisor, disk_path=None, capacity_map=None):
self.hypervisor = hypervisor
self.capacity_map = capacity_map
self.disk_path = disk_path or mkdtemp(delete=True)
self._logger = logging.getLogger(__name__)
self._logger.debug("Fake disk manager uses tempdir %s" %
self.disk_path)
mkdir_p(self.disk_path)
def test_rm_rf(self):
tempdir = mkdtemp()
assert_that(os.path.exists(tempdir), is_(True))
rm_rf(tempdir)
assert_that(os.path.exists(tempdir), is_(False))
# rm_rf a dir that doesn't exist shouldn't raise exception
rm_rf(tempdir)
def __init__(self, hypervisor, disk_path=None, capacity_map=None):
self.hypervisor = hypervisor
self.capacity_map = capacity_map
self.disk_path = disk_path or mkdtemp(delete=True)
self._logger = logging.getLogger(__name__)
self._logger.debug("Fake disk manager uses tempdir %s" %
self.disk_path)
mkdir_p(self.disk_path)
def test_image_size(self):
self.ds_manager.image_datastores.return_value = ["ds1", "ds2"]
with patch("host.image.image_manager.os_vmdk_flat_path"
"") as image_path:
tmpdir = file_util.mkdtemp(delete=True)
image_path.return_value = tmpdir
size = self.image_manager.image_size("image_id")
self.assertTrue(size > 0)
def test_image_size(self):
self.ds_manager.image_datastores.return_value = ["ds1", "ds2"]
with patch("host.hypervisor.esx.image_manager.os_vmdk_flat_path"
"") as image_path:
tmpdir = file_util.mkdtemp(delete=True)
image_path.return_value = tmpdir
size = self.image_manager.image_size("image_id")
self.assertTrue(size > 0)
def setUp(self):
self.hypervisor = MagicMock()
self.datastore_id = "ds_id_1"
self.datastore_name = "ds_name_1"
self.hypervisor.datastore_manager.get_datastore_ids.return_value = \
[self.datastore_id]
self.hypervisor.datastore_manager.datastore_name.return_value = \
self.datastore_name
self.im = Fim(self.hypervisor, "ds_id_1", mkdtemp(delete=True))
self.hypervisor.image_manager = self.im
def setUp(self):
self.hypervisor = MagicMock()
self.datastore_id = "ds_id_1"
self.datastore_name = "ds_name_1"
self.hypervisor.datastore_manager.get_datastore_ids.return_value = \
[self.datastore_id]
self.hypervisor.datastore_manager.datastore_name.return_value = \
self.datastore_name
self.im = Fim(self.hypervisor, "ds_id_1", mkdtemp(delete=True))
self.hypervisor.image_manager = self.im
def setUp(self):
self.agent_conf_dir = mkdtemp(delete=True)
state = State(os.path.join(self.agent_conf_dir, "state.json"))
common.services.register(ServiceName.MODE, Mode(state))
common.services.register(ServiceName.DATASTORE_TAGS,
DatastoreTags(state))
self.multi_agent = MultiAgent(2200, AgentConfig.DEFAULT_CONFIG_PATH,
AgentConfig.DEFAULT_CONFIG_FILE)
self.agent = AgentConfig(["--config-path", self.agent_conf_dir])
def setUp(self):
self.agent_conf_dir = mkdtemp(delete=True)
state = State(os.path.join(self.agent_conf_dir, "state.json"))
common.services.register(ServiceName.MODE, Mode(state))
common.services.register(ServiceName.DATASTORE_TAGS,
DatastoreTags(state))
self.multi_agent = MultiAgent(2200,
AgentConfig.DEFAULT_CONFIG_PATH,
AgentConfig.DEFAULT_CONFIG_FILE)
self.agent = AgentConfig(["--config-path", self.agent_conf_dir])
def test_atomic_write_file(self):
"""Test atomic write file"""
tempdir = mkdtemp(delete=True)
self.file_name = os.path.join(tempdir, "atomic_tmp_file")
with atomic_write_file(self.file_name) as fd:
fd.write("testing..")
file_state = os.stat(self.file_name)
self.assertTrue(file_state.st_size == 9)
def tmp_func():
with atomic_write_file(self.file_name):
raise Exception("Ouch~")
self.assertRaises(Exception, tmp_func)
def test_atomic_write_file(self):
"""Test atomic write file"""
tempdir = mkdtemp(delete=True)
self.file_name = os.path.join(tempdir, "atomic_tmp_file")
with atomic_write_file(self.file_name) as fd:
fd.write("testing..")
file_state = os.stat(self.file_name)
self.assertTrue(file_state.st_size == 9)
def tmp_func():
with atomic_write_file(self.file_name):
raise Exception("Ouch~")
self.assertRaises(Exception, tmp_func)
def test_image_type(self, type, replication, expected_type,
expected_replication):
self.ds_manager.image_datastores.return_value = ["ds1", "ds2"]
with patch("host.hypervisor.esx.image_manager.os_image_manifest_path"
"") as manifest_path:
tmpdir = file_util.mkdtemp(delete=True)
tmpfile = os.path.join(tmpdir, "ds1.manifest")
manifest_path.return_value = tmpfile
with open(tmpfile, 'w+') as f:
f.write('{"imageType":"%s","imageReplication":"%s"}' % (
type, replication))
type, replication = self.image_manager.get_image_manifest(
"image_id")
self.assertEqual(type, expected_type)
self.assertEqual(replication, expected_replication)
def test_image_type(self, type, replication, expected_type,
expected_replication):
self.ds_manager.image_datastores.return_value = "ds1"
with patch("host.hypervisor.esx.image_manager.os_image_manifest_path"
"") as manifest_path:
tmpdir = file_util.mkdtemp(delete=True)
tmpfile = os.path.join(tmpdir, "ds1.manifest")
manifest_path.return_value = tmpfile
with open(tmpfile, 'w+') as f:
f.write('{"imageType":"%s","imageReplication":"%s"}' % (
type, replication))
type, replication = self.image_manager.get_image_manifest(
"image_id")
self.assertEqual(type, expected_type)
self.assertEqual(replication, expected_replication)
def setUp(self):
self.state_file = tempfile.mktemp()
self.state = CommonState(self.state_file)
common.services.register(ServiceName.REQUEST_ID, threading.local())
common.services.register(ServiceName.LOCKED_VMS, ExclusiveSet())
common.services.register(ServiceName.MODE, Mode(self.state))
self.agent_conf_dir = mkdtemp(delete=True)
self.hostname = "localhost"
self._config = MagicMock()
self._config.agent_id = stable_uuid("agent_id")
self._config.hostname = "localhost"
self._config.host_port = 1234
self._config.host_version = "X.X.X"
self._config.reboot_required = False
self._config.image_datastores = []
common.services.register(ServiceName.AGENT_CONFIG, self._config)
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_path,
_uuid):
""" Test that stray images are found and deleted by the reaper """
def _fake_ds_folder(datastore, folder):
return "%s/%s" % (datastore, folder)
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_ds_dir = os.path.join(tmpdir, ds.id)
os.mkdir(tmp_ds_dir)
tmp_image_dir = os.path.join(
tmp_ds_dir,
compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image"))
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_path(datastore, folder):
return os.path.join(tmpdir, _fake_ds_folder(datastore, folder))
_os_datastore_path.side_effect = _fake_os_datastore_path
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = EsxImageManager(self.vim_client, ds_manager)
if not _allow_grace_period:
image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0
time.sleep(0.1)
image_manager.reap_tmp_images()
if _allow_grace_period:
# verify stray image is not deleted due to grace period
self.assertTrue(os.path.exists(path))
else:
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
def test_reap_tmp_images(self, _allow_grace_period, _os_datastore_path,
_uuid):
""" Test that stray images are found and deleted by the reaper """
def _fake_ds_folder(datastore, folder):
return "%s/%s" % (datastore, folder)
ds = MagicMock()
ds.id = "dsid"
ds.type = DatastoreType.EXT3
# In a random transient directory, set up a directory to act as the
# tmp images folder and to contain a stray image folder with a file.
tmpdir = file_util.mkdtemp(delete=True)
tmp_ds_dir = os.path.join(tmpdir, ds.id)
os.mkdir(tmp_ds_dir)
tmp_image_dir = os.path.join(tmp_ds_dir, compond_path_join(TMP_IMAGE_FOLDER_NAME_PREFIX, "stray_image"))
os.mkdir(tmp_image_dir)
(fd, path) = tempfile.mkstemp(prefix='strayimage_', dir=tmp_image_dir)
self.assertTrue(os.path.exists(path))
def _fake_os_datastore_path(datastore, folder):
return os.path.join(tmpdir, _fake_ds_folder(datastore, folder))
_os_datastore_path.side_effect = _fake_os_datastore_path
ds_manager = MagicMock()
ds_manager.get_datastores.return_value = [ds]
image_manager = EsxImageManager(self.vim_client, ds_manager)
if not _allow_grace_period:
image_manager.REAP_TMP_IMAGES_GRACE_PERIOD = 0.0
time.sleep(0.1)
image_manager.reap_tmp_images()
if _allow_grace_period:
# verify stray image is not deleted due to grace period
self.assertTrue(os.path.exists(path))
else:
# verify stray image is deleted
self.assertFalse(os.path.exists(path))
def setUp(self):
self._threadpool = ThreadPoolExecutor(16)
self.state_file = tempfile.mktemp()
self.state = CommonState(self.state_file)
common.services.register(ThreadPoolExecutor, self._threadpool)
common.services.register(ServiceName.REQUEST_ID, threading.local())
common.services.register(ServiceName.LOCKED_VMS, ExclusiveSet())
common.services.register(ServiceName.MODE, Mode(self.state))
self.agent_conf_dir = mkdtemp(delete=True)
self.hostname = "localhost"
self._config = MagicMock()
self._config.hypervisor = "fake"
self._config.agent_id = stable_uuid("agent_id")
self._config.hostname = "localhost"
self._config.host_port = 1234
self._config.host_version = "X.X.X"
self._config.reboot_required = False
self._config.image_datastores = []
common.services.register(ServiceName.AGENT_CONFIG, self._config)
def init(self):
tempdir = mkdtemp(delete=True, dir=self._test_dir)
self._file_name = os.path.join(tempdir, "fileio_lock_test")
self._lock_path = "%s.%s" % (self._file_name,
FileBackedLock.LOCK_EXTENSION)
def setUp(self):
self.services_helper = ServicesHelper()
self.hv = None
self.agent_config_dir = mkdtemp(delete=True)
self.agent_config = AgentConfig(["--config-path",
self.agent_config_dir])
def test_mkdtemp(self):
# Register first so it runs after tempdir is deleted
atexit.register(self._verify_dir, False)
self.tempdir = mkdtemp(delete=True)
def test_mkdtemp_normal(self):
# Register first so it runs after tempdir is deleted
atexit.register(self._verify_dir, True)
self.tempdir = mkdtemp()
def test_mkdtemp(self):
# Register first so it runs after tempdir is deleted
atexit.register(self._verify_dir, False)
self.tempdir = mkdtemp(delete=True)
def test_mkdtemp_normal(self):
# Register first so it runs after tempdir is deleted
atexit.register(self._verify_dir, True)
self.tempdir = mkdtemp()
def setUp(self):
vmfs_dir = "/vmfs/volumes/datastore1/"
tempdir = mkdtemp(delete=False, dir=vmfs_dir)
self._file_name = os.path.join(tempdir, "ref_count.ref")
def init(self):
tempdir = mkdtemp(delete=True, dir=self._test_dir)
self._file_name = os.path.join(tempdir, "fileio_lock_test")
self._lock_path = "%s.%s" % (self._file_name,
FileBackedLock.LOCK_EXTENSION)
def setUp(self):
self.agent_conf_dir = mkdtemp(delete=True)
state = State(os.path.join(self.agent_conf_dir, "state.json"))
common.services.register(ServiceName.MODE, Mode(state))
common.services.register(ServiceName.REGISTRANT, mock.MagicMock())
self.agent = AgentConfig(["--config-path", self.agent_conf_dir])
def setUp(self):
self.agent_conf_dir = mkdtemp(delete=True)
state = State(os.path.join(self.agent_conf_dir, "state.json"))
common.services.register(ServiceName.MODE, Mode(state))
common.services.register(ServiceName.REGISTRANT, mock.MagicMock())
self.agent = AgentConfig(["--config-path", self.agent_conf_dir])
def setUp(self):
self.services_helper = ServicesHelper()
self.hv = None
self.agent_config_dir = mkdtemp(delete=True)
self.agent_config = AgentConfig(
["--config-path", self.agent_config_dir])
def setUp(self):
vmfs_dir = "/vmfs/volumes/datastore1/"
tempdir = mkdtemp(delete=False, dir=vmfs_dir)
self._file_name = os.path.join(tempdir, "ref_count.ref")
def setUp(self):
self.services_helper = ServicesHelper()
self.mock_options = MagicMock()
self.agent_config_dir = mkdtemp(delete=True)
self.agent_config = AgentConfig(
["--config-path", self.agent_config_dir, "--hypervisor", "esx"])