def test_list_backing_images_resized(self): self.stubs.Set( os, 'listdir', lambda x: [ '_base', 'instance-00000001', 'instance-00000002', 'instance-00000003' ]) self.stubs.Set(os.path, 'exists', lambda x: x.find('instance-') != -1) self.stubs.Set( libvirt_utils, 'get_disk_backing_file', lambda x: ('e97222e91fc4241f49a7f520d1dcf446751129b3_' '10737418240')) found = os.path.join( CONF.instances_path, CONF.image_cache_subdirectory_name, 'e97222e91fc4241f49a7f520d1dcf446751129b3_' '10737418240') image_cache_manager = imagecache.ImageCacheManager() image_cache_manager.unexplained_images = [found] image_cache_manager.instance_names = self.stock_instance_names inuse_images = image_cache_manager._list_backing_images() self.assertEqual(inuse_images, [found]) self.assertEqual(len(image_cache_manager.unexplained_images), 0)
def test_age_and_verify_swap_images(self, mock_remove, mock_getmtime, mock_utime, mock_exist, mock_chown): image_cache_manager = imagecache.ImageCacheManager() expected_remove = set() expected_exist = set(['swap_128', 'swap_256']) image_cache_manager.back_swap_images.add('swap_128') image_cache_manager.back_swap_images.add('swap_256') image_cache_manager.used_swap_images.add('swap_128') def getmtime(path): return time.time() - 1000000 mock_getmtime.side_effect = getmtime def removefile(path): if not path.startswith('/tmp_age_test'): return os.remove(path) fn = os.path.split(path)[-1] expected_remove.add(fn) expected_exist.remove(fn) mock_remove.side_effect = removefile image_cache_manager._age_and_verify_swap_images(None, '/tmp_age_test') self.assertEqual(1, len(expected_exist)) self.assertEqual(1, len(expected_remove)) self.assertIn('swap_128', expected_exist) self.assertIn('swap_256', expected_remove)
def test_list_base_images(self): listing = [ '00000001', 'ephemeral_0_20_None', '17d1b00b81642842e514494a78e804e9a511637c_5368709120.info', '00000004', 'swap_1000' ] images = [ 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm', 'e09c675c2d1cfac32dae3c2d83689c8c94bc693b_sm', 'e97222e91fc4241f49a7f520d1dcf446751129b3', '17d1b00b81642842e514494a78e804e9a511637c', '17d1b00b81642842e514494a78e804e9a511637c_5368709120', '17d1b00b81642842e514494a78e804e9a511637c_10737418240' ] listing.extend(images) self.stubs.Set(os, 'listdir', lambda x: listing) self.stubs.Set(os.path, 'isfile', lambda x: True) base_dir = '/var/lib/patron/instances/_base' self.flags(instances_path='/var/lib/patron/instances') image_cache_manager = imagecache.ImageCacheManager() image_cache_manager._list_base_images(base_dir) sanitized = [] for ent in image_cache_manager.unexplained_images: sanitized.append(ent.replace(base_dir + '/', '')) self.assertEqual(sorted(sanitized), sorted(images)) expected = os.path.join(base_dir, 'e97222e91fc4241f49a7f520d1dcf446751129b3') self.assertIn(expected, image_cache_manager.unexplained_images) expected = os.path.join( base_dir, '17d1b00b81642842e514494a78e804e9a511637c_' '10737418240') self.assertIn(expected, image_cache_manager.unexplained_images) unexpected = os.path.join(base_dir, '00000004') self.assertNotIn(unexpected, image_cache_manager.unexplained_images) for ent in image_cache_manager.unexplained_images: self.assertTrue(ent.startswith(base_dir)) self.assertEqual(len(image_cache_manager.originals), 2) expected = os.path.join(base_dir, '17d1b00b81642842e514494a78e804e9a511637c') self.assertIn(expected, image_cache_manager.originals) unexpected = os.path.join( base_dir, '17d1b00b81642842e514494a78e804e9a511637c_' '10737418240') self.assertNotIn(unexpected, image_cache_manager.originals) self.assertEqual(1, len(image_cache_manager.back_swap_images)) self.assertIn('swap_1000', image_cache_manager.back_swap_images)
def test_find_base_file_nothing(self): self.stubs.Set(os.path, 'exists', lambda x: False) base_dir = '/var/lib/patron/instances/_base' fingerprint = '549867354867' image_cache_manager = imagecache.ImageCacheManager() res = list(image_cache_manager._find_base_file(base_dir, fingerprint)) self.assertEqual(0, len(res))
def _check_body(self, tmpdir, info_attr): self.flags(instances_path=tmpdir) self.flags(image_info_filename_pattern=('$instances_path/' '%(image)s.info'), group='libvirt') fname, info_fname, testdata = self._make_checksum(tmpdir) self._write_file(info_fname, info_attr, testdata) image_cache_manager = imagecache.ImageCacheManager() return image_cache_manager, fname
def test_find_base_file_small(self): fingerprint = '968dd6cc49e01aaa044ed11c0cce733e0fa44a6a' self.stubs.Set(os.path, 'exists', lambda x: x.endswith('%s_sm' % fingerprint)) base_dir = '/var/lib/patron/instances/_base' image_cache_manager = imagecache.ImageCacheManager() res = list(image_cache_manager._find_base_file(base_dir, fingerprint)) base_file = os.path.join(base_dir, fingerprint + '_sm') self.assertEqual(res, [(base_file, True, False)])
def test_store_swap_image(self): image_cache_manager = imagecache.ImageCacheManager() image_cache_manager._store_swap_image('swap_') image_cache_manager._store_swap_image('swap_123') image_cache_manager._store_swap_image('swap_456') image_cache_manager._store_swap_image('swap_abc') image_cache_manager._store_swap_image('123_swap') image_cache_manager._store_swap_image('swap_129_') self.assertEqual(len(image_cache_manager.back_swap_images), 2) expect_set = set(['swap_123', 'swap_456']) self.assertEqual(image_cache_manager.back_swap_images, expect_set)
def test_remove_base_file_dne(self): # This test is solely to execute the "does not exist" code path. We # don't expect the method being tested to do anything in this case. with utils.tempdir() as tmpdir: self.flags(instances_path=tmpdir) self.flags(image_info_filename_pattern=('$instances_path/' '%(image)s.info'), group='libvirt') fname = os.path.join(tmpdir, 'aaa') image_cache_manager = imagecache.ImageCacheManager() image_cache_manager._remove_base_file(fname)
def test_handle_base_image_absent(self): img = '123' with intercept_log_messages() as stream: image_cache_manager = imagecache.ImageCacheManager() image_cache_manager.used_images = {'123': (1, 0, ['banana-42'])} image_cache_manager._handle_base_image(img, None) self.assertEqual(image_cache_manager.unexplained_images, []) self.assertEqual(image_cache_manager.removable_base_files, []) self.assertEqual(image_cache_manager.corrupt_base_files, []) self.assertNotEqual(stream.getvalue().find('an absent base file'), -1)
def test_handle_base_image_unused(self): img = '123' with self._make_base_file() as fname: os.utime(fname, (-1, time.time() - 3601)) image_cache_manager = imagecache.ImageCacheManager() image_cache_manager.unexplained_images = [fname] image_cache_manager._handle_base_image(img, fname) self.assertEqual(image_cache_manager.unexplained_images, []) self.assertEqual(image_cache_manager.removable_base_files, [fname]) self.assertEqual(image_cache_manager.corrupt_base_files, [])
def test_verify_checksum_file_missing(self): with utils.tempdir() as tmpdir: self.flags(instances_path=tmpdir) self.flags(image_info_filename_pattern=('$instances_path/' '%(image)s.info'), group='libvirt') fname, info_fname, testdata = self._make_checksum(tmpdir) image_cache_manager = imagecache.ImageCacheManager() res = image_cache_manager._verify_checksum('aaa', fname) self.assertIsNone(res) # Checksum requests for a file with no checksum now have the # side effect of creating the checksum self.assertTrue(os.path.exists(info_fname))
def test_handle_base_image_used_remotely(self): self.stubs.Set(libvirt_utils, 'chown', lambda x, y: None) img = '123' with self._make_base_file() as fname: os.utime(fname, (-1, time.time() - 3601)) image_cache_manager = imagecache.ImageCacheManager() image_cache_manager.unexplained_images = [fname] image_cache_manager.used_images = {'123': (0, 1, ['banana-42'])} image_cache_manager._handle_base_image(img, fname) self.assertEqual(image_cache_manager.unexplained_images, []) self.assertEqual(image_cache_manager.removable_base_files, []) self.assertEqual(image_cache_manager.corrupt_base_files, [])
def test_list_backing_images_disk_notexist(self): self.stubs.Set(os, 'listdir', lambda x: ['_base', 'banana-42-hamster']) self.stubs.Set(os.path, 'exists', lambda x: x.find('banana-42-hamster') != -1) def fake_get_disk(disk_path): raise processutils.ProcessExecutionError() self.stubs.Set(libvirt_utils, 'get_disk_backing_file', fake_get_disk) image_cache_manager = imagecache.ImageCacheManager() image_cache_manager.unexplained_images = [] image_cache_manager.instance_names = self.stock_instance_names self.assertRaises(processutils.ProcessExecutionError, image_cache_manager._list_backing_images)
def test_find_base_file_resized(self): fingerprint = '968dd6cc49e01aaa044ed11c0cce733e0fa44a6a' listing = [ '00000001', 'ephemeral_0_20_None', '968dd6cc49e01aaa044ed11c0cce733e0fa44a6a_10737418240', '00000004' ] self.stubs.Set(os, 'listdir', lambda x: listing) self.stubs.Set(os.path, 'exists', lambda x: x.endswith('%s_10737418240' % fingerprint)) self.stubs.Set(os.path, 'isfile', lambda x: True) base_dir = '/var/lib/patron/instances/_base' image_cache_manager = imagecache.ImageCacheManager() image_cache_manager._list_base_images(base_dir) res = list(image_cache_manager._find_base_file(base_dir, fingerprint)) base_file = os.path.join(base_dir, fingerprint + '_10737418240') self.assertEqual(res, [(base_file, False, True)])
def test_handle_base_image_used_missing(self): img = '123' with utils.tempdir() as tmpdir: self.flags(instances_path=tmpdir) self.flags(image_info_filename_pattern=('$instances_path/' '%(image)s.info'), group='libvirt') fname = os.path.join(tmpdir, 'aaa') image_cache_manager = imagecache.ImageCacheManager() image_cache_manager.unexplained_images = [fname] image_cache_manager.used_images = {'123': (1, 0, ['banana-42'])} image_cache_manager._handle_base_image(img, fname) self.assertEqual(image_cache_manager.unexplained_images, []) self.assertEqual(image_cache_manager.removable_base_files, []) self.assertEqual(image_cache_manager.corrupt_base_files, [])
def test_remove_base_file_oserror(self): with intercept_log_messages() as stream: with utils.tempdir() as tmpdir: self.flags(instances_path=tmpdir) self.flags(image_info_filename_pattern=('$instances_path/' '%(image)s.info'), group='libvirt') fname = os.path.join(tmpdir, 'aaa') os.mkdir(fname) os.utime(fname, (-1, time.time() - 3601)) # This will raise an OSError because of file permissions image_cache_manager = imagecache.ImageCacheManager() image_cache_manager._remove_base_file(fname) self.assertTrue(os.path.exists(fname)) self.assertNotEqual(stream.getvalue().find('Failed to remove'), -1)
def test_remove_base_file(self): with self._make_base_file() as fname: image_cache_manager = imagecache.ImageCacheManager() image_cache_manager._remove_base_file(fname) info_fname = imagecache.get_info_filename(fname) lock_name = 'patron-' + os.path.split(fname)[-1] lock_dir = os.path.join(CONF.instances_path, 'locks') lock_file = os.path.join(lock_dir, lock_name) # Files are initially too new to delete self.assertTrue(os.path.exists(fname)) self.assertTrue(os.path.exists(info_fname)) self.assertTrue(os.path.exists(lock_file)) # Old files get cleaned up though os.utime(fname, (-1, time.time() - 3601)) image_cache_manager._remove_base_file(fname) self.assertFalse(os.path.exists(fname)) self.assertFalse(os.path.exists(info_fname)) self.assertFalse(os.path.exists(lock_file))
def test_handle_base_image_checksum_fails(self): self.flags(checksum_base_images=True, group='libvirt') self.stubs.Set(libvirt_utils, 'chown', lambda x, y: None) img = '123' with self._make_base_file() as fname: with open(fname, 'w') as f: f.write('banana') d = {'sha1': '21323454'} with open('%s.info' % fname, 'w') as f: f.write(jsonutils.dumps(d)) image_cache_manager = imagecache.ImageCacheManager() image_cache_manager.unexplained_images = [fname] image_cache_manager.used_images = {'123': (1, 0, ['banana-42'])} image_cache_manager._handle_base_image(img, fname) self.assertEqual(image_cache_manager.unexplained_images, []) self.assertEqual(image_cache_manager.removable_base_files, []) self.assertEqual(image_cache_manager.corrupt_base_files, [fname])
def test_remove_base_file_original(self): with self._make_base_file() as fname: image_cache_manager = imagecache.ImageCacheManager() image_cache_manager.originals = [fname] image_cache_manager._remove_base_file(fname) info_fname = imagecache.get_info_filename(fname) # Files are initially too new to delete self.assertTrue(os.path.exists(fname)) self.assertTrue(os.path.exists(info_fname)) # This file should stay longer than a resized image os.utime(fname, (-1, time.time() - 3601)) image_cache_manager._remove_base_file(fname) self.assertTrue(os.path.exists(fname)) self.assertTrue(os.path.exists(info_fname)) # Originals don't stay forever though os.utime(fname, (-1, time.time() - 3600 * 25)) image_cache_manager._remove_base_file(fname) self.assertFalse(os.path.exists(fname)) self.assertFalse(os.path.exists(info_fname))
def test_get_age_of_file_not_exists(self, mock_exists): image_cache_manager = imagecache.ImageCacheManager() exists, age = image_cache_manager._get_age_of_file('/tmp') self.assertFalse(exists) self.assertEqual(0, age)
def test_configured_checksum_path(self): with utils.tempdir() as tmpdir: self.flags(instances_path=tmpdir) self.flags(image_info_filename_pattern=('$instances_path/' '%(image)s.info'), group='libvirt') # Ensure there is a base directory os.mkdir(os.path.join(tmpdir, '_base')) # Fake the database call which lists running instances instances = [{ 'image_ref': '1', 'host': CONF.host, 'name': 'instance-1', 'uuid': '123', 'vm_state': '', 'task_state': '' }, { 'image_ref': '1', 'host': CONF.host, 'name': 'instance-2', 'uuid': '456', 'vm_state': '', 'task_state': '' }] all_instances = [] for instance in instances: all_instances.append( fake_instance.fake_instance_obj(None, **instance)) def touch(filename): f = open(filename, 'w') f.write('Touched') f.close() old = time.time() - (25 * 3600) hashed = 'e97222e91fc4241f49a7f520d1dcf446751129b3' base_filename = os.path.join(tmpdir, hashed) touch(base_filename) touch(base_filename + '.info') os.utime(base_filename + '.info', (old, old)) touch(base_filename + '.info') os.utime(base_filename + '.info', (old, old)) self.mox.StubOutWithMock( objects.block_device.BlockDeviceMappingList, 'get_by_instance_uuid') ctxt = context.get_admin_context() objects.block_device.BlockDeviceMappingList.get_by_instance_uuid( ctxt, '123').AndReturn(None) objects.block_device.BlockDeviceMappingList.get_by_instance_uuid( ctxt, '456').AndReturn(None) self.mox.ReplayAll() image_cache_manager = imagecache.ImageCacheManager() image_cache_manager.update(ctxt, all_instances) self.assertTrue(os.path.exists(base_filename)) self.assertTrue(os.path.exists(base_filename + '.info'))
def test_verify_base_images_no_base(self): self.flags(instances_path='/tmp/no/such/dir/name/please') image_cache_manager = imagecache.ImageCacheManager() image_cache_manager.update(None, [])
def test_get_age_of_file(self, mock_getmtime, mock_time, mock_exists): image_cache_manager = imagecache.ImageCacheManager() exists, age = image_cache_manager._get_age_of_file('/tmp') self.assertTrue(exists) self.assertEqual(1000000, age)
def test_verify_base_images(self): hashed_1 = '356a192b7913b04c54574d18c28d46e6395428ab' hashed_21 = '472b07b9fcf2c2451e8781e944bf5f77cd8457c8' hashed_22 = '12c6fc06c99a462375eeb3f43dfd832b08ca9e17' hashed_42 = '92cfceb39d57d914ed8b14d0e37643de0797ae56' self.flags(instances_path='/instance_path', image_cache_subdirectory_name='_base') base_file_list = [ '00000001', 'ephemeral_0_20_None', 'e97222e91fc4241f49a7f520d1dcf446751129b3_sm', 'e09c675c2d1cfac32dae3c2d83689c8c94bc693b_sm', hashed_42, hashed_1, hashed_21, hashed_22, '%s_5368709120' % hashed_1, '%s_10737418240' % hashed_1, '00000004' ] def fq_path(path): return os.path.join('/instance_path/_base/', path) # Fake base directory existence orig_exists = os.path.exists def exists(path): # The python coverage tool got angry with my overly broad mocks if not path.startswith('/instance_path'): return orig_exists(path) if path in [ '/instance_path', '/instance_path/_base', '/instance_path/instance-1/disk', '/instance_path/instance-2/disk', '/instance_path/instance-3/disk', '/instance_path/_base/%s.info' % hashed_42 ]: return True for p in base_file_list: if path == fq_path(p): return True if path == fq_path(p) + '.info': return False if path in [ '/instance_path/_base/%s_sm' % i for i in [hashed_1, hashed_21, hashed_22, hashed_42] ]: return False self.fail('Unexpected path existence check: %s' % path) self.stubs.Set(os.path, 'exists', lambda x: exists(x)) self.stubs.Set(libvirt_utils, 'chown', lambda x, y: None) # We need to stub utime as well self.stubs.Set(os, 'utime', lambda x, y: None) # Fake up some instances in the instances directory orig_listdir = os.listdir def listdir(path): # The python coverage tool got angry with my overly broad mocks if not path.startswith('/instance_path'): return orig_listdir(path) if path == '/instance_path': return ['instance-1', 'instance-2', 'instance-3', '_base'] if path == '/instance_path/_base': return base_file_list self.fail('Unexpected directory listed: %s' % path) self.stubs.Set(os, 'listdir', lambda x: listdir(x)) # Fake isfile for these faked images in _base orig_isfile = os.path.isfile def isfile(path): # The python coverage tool got angry with my overly broad mocks if not path.startswith('/instance_path'): return orig_isfile(path) for p in base_file_list: if path == fq_path(p): return True self.fail('Unexpected isfile call: %s' % path) self.stubs.Set(os.path, 'isfile', lambda x: isfile(x)) # Fake the database call which lists running instances instances = [{ 'image_ref': '1', 'host': CONF.host, 'name': 'instance-1', 'uuid': '123', 'vm_state': '', 'task_state': '' }, { 'image_ref': '1', 'kernel_id': '21', 'ramdisk_id': '22', 'host': CONF.host, 'name': 'instance-2', 'uuid': '456', 'vm_state': '', 'task_state': '' }] all_instances = [ fake_instance.fake_instance_obj(None, **instance) for instance in instances ] image_cache_manager = imagecache.ImageCacheManager() # Fake the utils call which finds the backing image def get_disk_backing_file(path): if path in [ '/instance_path/instance-1/disk', '/instance_path/instance-2/disk' ]: return fq_path('%s_5368709120' % hashed_1) self.fail('Unexpected backing file lookup: %s' % path) self.stubs.Set(libvirt_utils, 'get_disk_backing_file', lambda x: get_disk_backing_file(x)) # Fake out verifying checksums, as that is tested elsewhere self.stubs.Set(image_cache_manager, '_verify_checksum', lambda x, y: True) # Fake getmtime as well orig_getmtime = os.path.getmtime def getmtime(path): if not path.startswith('/instance_path'): return orig_getmtime(path) return 1000000 self.stubs.Set(os.path, 'getmtime', lambda x: getmtime(x)) # Make sure we don't accidentally remove a real file orig_remove = os.remove def remove(path): if not path.startswith('/instance_path'): return orig_remove(path) # Don't try to remove fake files return self.stubs.Set(os, 'remove', lambda x: remove(x)) self.mox.StubOutWithMock(objects.block_device.BlockDeviceMappingList, 'get_by_instance_uuid') ctxt = context.get_admin_context() objects.block_device.BlockDeviceMappingList.get_by_instance_uuid( ctxt, '123').AndReturn(None) objects.block_device.BlockDeviceMappingList.get_by_instance_uuid( ctxt, '456').AndReturn(None) self.mox.ReplayAll() # And finally we can make the call we're actually testing... # The argument here should be a context, but it is mocked out image_cache_manager.update(ctxt, all_instances) # Verify active = [ fq_path(hashed_1), fq_path('%s_5368709120' % hashed_1), fq_path(hashed_21), fq_path(hashed_22) ] for act in active: self.assertIn(act, image_cache_manager.active_base_files) self.assertEqual(len(image_cache_manager.active_base_files), len(active)) for rem in [ fq_path('e97222e91fc4241f49a7f520d1dcf446751129b3_sm'), fq_path('e09c675c2d1cfac32dae3c2d83689c8c94bc693b_sm'), fq_path(hashed_42), fq_path('%s_10737418240' % hashed_1) ]: self.assertIn(rem, image_cache_manager.removable_base_files) # Ensure there are no "corrupt" images as well self.assertEqual(len(image_cache_manager.corrupt_base_files), 0)