def setUp(self): self.opts = _config = minion_config(None) self.tmp_dir = tempfile.mkdtemp(dir=integration.TMP) os.makedirs(self.module_dir) self.lib_count = collections.defaultdict(int) # mapping of path -> count # bootstrap libs with open(os.path.join(self.module_dir, '__init__.py'), 'w') as fh: # No .decode() needed here as deep_init_base is defined as str and # not bytes. fh.write(deep_init_base) fh.flush() os.fsync(fh.fileno()) # flush to disk self.lib_paths = {} dir_path = self.module_dir for lib_name in self.libs: dir_path = os.path.join(dir_path, lib_name) self.lib_paths[lib_name] = dir_path os.makedirs(dir_path) self.update_lib(lib_name) dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.utils = utils(self.opts) self.proxy = proxy(self.opts) self.minion_mods = minion_mods(self.opts) self.loader = LazyLoader(dirs, self.opts, tag='module', pack={'__utils__': self.utils, '__proxy__': self.proxy, '__salt__': self.minion_mods} )
def setUp(self): self.opts = _config = minion_config(None) self.tmp_dir = tempfile.mkdtemp(dir=integration.TMP) os.makedirs(self.module_dir) self.lib_count = collections.defaultdict(int) # mapping of path -> count # bootstrap libs with open(os.path.join(self.module_dir, '__init__.py'), 'w') as fh: fh.write(deep_init_base) fh.flush() os.fsync(fh.fileno()) # flush to disk self.lib_paths = {} dir_path = self.module_dir for lib_name in self.libs: dir_path = os.path.join(dir_path, lib_name) self.lib_paths[lib_name] = dir_path os.makedirs(dir_path) self.update_lib(lib_name) dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.loader = LazyLoader(dirs, self.opts, tag='module')
def setUp(self): self.opts = _config = minion_config(None) self.opts['grains'] = grains(self.opts) self.tmp_dir = tempfile.mkdtemp(dir=integration.TMP) dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.loader = LazyLoader(dirs, self.opts, tag='module')
def setUp(self): self.opts = _config = minion_config(None) self.tmp_dir = tempfile.mkdtemp(dir=integration.TMP) os.makedirs(self.module_dir) self.count = 0 self.lib_count = 0 dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.loader = LazyLoader(dirs, self.opts, tag='module')
def setUp(self): self.opts = _config = minion_config(None) self.opts['grains'] = grains(self.opts) self.tmp_dir = tempfile.mkdtemp(dir=integration.TMP) self.count = 0 dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.utils = utils(self.opts) self.proxy = proxy(self.opts) self.minion_mods = minion_mods(self.opts) self.loader = LazyLoader(dirs, self.opts, tag='module', pack={'__utils__': self.utils, '__proxy__': self.proxy, '__salt__': self.minion_mods})
class LazyLoaderDeepSubmodReloadingTest(TestCase): module_name = 'loadertestsubmoddeep' libs = ('top_lib', 'mid_lib', 'bot_lib') def setUp(self): self.opts = _config = minion_config(None) self.tmp_dir = tempfile.mkdtemp(dir=integration.TMP) os.makedirs(self.module_dir) self.lib_count = collections.defaultdict(int) # mapping of path -> count # bootstrap libs with open(os.path.join(self.module_dir, '__init__.py'), 'w') as fh: fh.write(deep_init_base) fh.flush() os.fsync(fh.fileno()) # flush to disk self.lib_paths = {} dir_path = self.module_dir for lib_name in self.libs: dir_path = os.path.join(dir_path, lib_name) self.lib_paths[lib_name] = dir_path os.makedirs(dir_path) self.update_lib(lib_name) dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.loader = LazyLoader(dirs, self.opts, tag='module') @property def module_dir(self): return os.path.join(self.tmp_dir, self.module_name) def update_lib(self, lib_name): path = os.path.join(self.lib_paths[lib_name], '__init__.py') self.lib_count[lib_name] += 1 with open(path, 'wb') as fh: fh.write(submodule_lib_template.format(count=self.lib_count[lib_name])) fh.flush() os.fsync(fh.fileno()) # flush to disk # pyc files don't like it when we change the original quickly # since the header bytes only contain the timestamp (granularity of seconds) # TODO: don't write them? Is *much* slower on re-load (~3x) # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode try: os.unlink(path + 'c') except OSError: pass def tearDown(self): shutil.rmtree(self.tmp_dir) def test_basic(self): self.assertIn('{0}.top'.format(self.module_name), self.loader) def _verify_libs(self): for lib in self.libs: self.assertEqual(self.loader['{0}.{1}'.format(self.module_name, lib.replace('_lib', ''))](), self.lib_count[lib]) def test_reload(self): ''' Make sure that we can reload all libraries of arbitrary depth ''' self._verify_libs() # update them all for lib in self.libs: for x in xrange(5): self.update_lib(lib) self.loader.clear() self._verify_libs()
class LazyLoaderSubmodReloadingTest(TestCase): ''' Test the loader of salt with changing modules ''' module_name = 'loadertestsubmod' module_key = 'loadertestsubmod.test' def setUp(self): self.opts = _config = minion_config(None) self.tmp_dir = tempfile.mkdtemp(dir=integration.TMP) os.makedirs(self.module_dir) self.count = 0 self.lib_count = 0 dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.loader = LazyLoader(dirs, self.opts, tag='module') def tearDown(self): shutil.rmtree(self.tmp_dir) def update_module(self): self.count += 1 with open(self.module_path, 'wb') as fh: fh.write(submodule_template.format(count=self.count)) fh.flush() os.fsync(fh.fileno()) # flush to disk # pyc files don't like it when we change the original quickly # since the header bytes only contain the timestamp (granularity of seconds) # TODO: don't write them? Is *much* slower on re-load (~3x) # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode try: os.unlink(self.module_path + 'c') except OSError: pass def rm_module(self): os.unlink(self.module_path) os.unlink(self.module_path + 'c') def update_lib(self): self.lib_count += 1 with open(self.lib_path, 'wb') as fh: fh.write(submodule_lib_template.format(count=self.lib_count)) fh.flush() os.fsync(fh.fileno()) # flush to disk # pyc files don't like it when we change the original quickly # since the header bytes only contain the timestamp (granularity of seconds) # TODO: don't write them? Is *much* slower on re-load (~3x) # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode try: os.unlink(self.lib_path + 'c') except OSError: pass def rm_lib(self): os.unlink(self.lib_path) os.unlink(self.lib_path + 'c') @property def module_dir(self): return os.path.join(self.tmp_dir, self.module_name) @property def module_path(self): return os.path.join(self.module_dir, '__init__.py') @property def lib_path(self): return os.path.join(self.module_dir, 'lib.py') def test_basic(self): # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) self.update_module() self.update_lib() self.loader.clear() self.assertIn(self.module_key, self.loader) def test_reload(self): # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) # update both the module and the lib for x in range(1, 3): self.update_module() self.update_lib() self.loader.clear() self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) # update just the module for x in range(1, 3): self.update_module() self.loader.clear() self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) # update just the lib for x in range(1, 3): self.update_lib() self.loader.clear() self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) self.rm_module() # make sure that even if we remove the module, its still loaded until a clear self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) self.loader.clear() self.assertNotIn(self.module_key, self.loader) def test_reload_missing_lib(self): # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) # update both the module and the lib self.update_module() self.update_lib() self.loader.clear() self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) # remove the lib, this means we should fail to load the module next time self.rm_lib() self.loader.clear() self.assertNotIn(self.module_key, self.loader)
class LazyLoaderReloadingTest(TestCase): ''' Test the loader of salt with changing modules ''' module_name = 'loadertest' module_key = 'loadertest.test' def setUp(self): self.opts = _config = minion_config(None) self.tmp_dir = tempfile.mkdtemp(dir=integration.TMP) self.count = 0 dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.loader = LazyLoader(dirs, self.opts, tag='module') def tearDown(self): shutil.rmtree(self.tmp_dir) def update_module(self): self.count += 1 with open(self.module_path, 'wb') as fh: fh.write(module_template.format(count=self.count)) fh.flush() os.fsync(fh.fileno()) # flush to disk # pyc files don't like it when we change the original quickly # since the header bytes only contain the timestamp (granularity of seconds) # TODO: don't write them? Is *much* slower on re-load (~3x) # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode try: os.unlink(self.module_path + 'c') except OSError: pass def rm_module(self): os.unlink(self.module_path) os.unlink(self.module_path + 'c') @property def module_path(self): return os.path.join(self.tmp_dir, '{0}.py'.format(self.module_name)) def test_alias(self): ''' Make sure that you can access alias-d modules ''' # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) self.update_module() self.assertNotIn('{0}.test_alias'.format(self.module_name), self.loader) self.assertTrue(inspect.isfunction(self.loader['{0}.working_alias'.format(self.module_name)])) def test_clear(self): self.assertTrue(inspect.isfunction(self.loader['test.ping'])) self.update_module() # write out out custom module self.loader.clear() # clear the loader dict # force a load of our module self.assertTrue(inspect.isfunction(self.loader[self.module_key])) # make sure we only loaded our custom module # which means that we did correctly refresh the file mapping for k, v in six.iteritems(self.loader._dict): self.assertTrue(k.startswith(self.module_name)) def test_load(self): # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) self.update_module() self.assertTrue(inspect.isfunction(self.loader[self.module_key])) def test__load__(self): ''' If a module specifies __load__ we should only load/expose those modules ''' self.update_module() # ensure it doesn't exist self.assertNotIn(self.module_key + '2', self.loader) def test__load__and_depends(self): ''' If a module specifies __load__ we should only load/expose those modules ''' self.update_module() # ensure it doesn't exist self.assertNotIn(self.module_key + '3', self.loader) self.assertNotIn(self.module_key + '4', self.loader) def test_reload(self): # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) # make sure it updates correctly for x in range(1, 3): self.update_module() self.loader.clear() self.assertEqual(self.loader[self.module_key](), self.count) self.rm_module() # make sure that even if we remove the module, its still loaded until a clear self.assertEqual(self.loader[self.module_key](), self.count) self.loader.clear() self.assertNotIn(self.module_key, self.loader)
class LazyLoaderModulePackageTest(TestCase): ''' Test the loader of salt with changing modules ''' module_name = 'loadertestmodpkg' module_key = 'loadertestmodpkg.test' def setUp(self): self.opts = _config = minion_config(None) self.opts['grains'] = grains(self.opts) self.tmp_dir = tempfile.mkdtemp(dir=integration.TMP) dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.loader = LazyLoader(dirs, self.opts, tag='module') def tearDown(self): shutil.rmtree(self.tmp_dir) def update_pyfile(self, pyfile, contents): dirname = os.path.dirname(pyfile) if not os.path.exists(dirname): os.makedirs(dirname) with open(pyfile, 'wb') as fh: fh.write(salt.utils.to_bytes(contents)) fh.flush() os.fsync(fh.fileno()) # flush to disk # pyc files don't like it when we change the original quickly # since the header bytes only contain the timestamp (granularity of seconds) # TODO: don't write them? Is *much* slower on re-load (~3x) # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode try: os.unlink(pyfile + 'c') except OSError: pass def rm_pyfile(self, pyfile): os.unlink(pyfile) os.unlink(pyfile + 'c') def update_module(self, relative_path, contents): self.update_pyfile(os.path.join(self.tmp_dir, relative_path), contents) def rm_module(self, relative_path): self.rm_pyfile(os.path.join(self.tmp_dir, relative_path)) def test_module(self): # ensure it doesn't exist self.assertNotIn('foo', self.loader) self.assertNotIn('foo.test', self.loader) self.update_module('foo.py', mod_template.format(val=1)) self.loader.clear() self.assertIn('foo.test', self.loader) self.assertEqual(self.loader['foo.test'](), 1) def test_package(self): # ensure it doesn't exist self.assertNotIn('foo', self.loader) self.assertNotIn('foo.test', self.loader) self.update_module('foo/__init__.py', mod_template.format(val=2)) self.loader.clear() self.assertIn('foo.test', self.loader) self.assertEqual(self.loader['foo.test'](), 2) def test_module_package_collision(self): # ensure it doesn't exist self.assertNotIn('foo', self.loader) self.assertNotIn('foo.test', self.loader) self.update_module('foo.py', mod_template.format(val=3)) self.loader.clear() self.assertIn('foo.test', self.loader) self.assertEqual(self.loader['foo.test'](), 3) self.update_module('foo/__init__.py', mod_template.format(val=4)) self.loader.clear() self.assertIn('foo.test', self.loader) self.assertEqual(self.loader['foo.test'](), 4)
def setUp(self): self.opts = _config = minion_config(None) self.loader = LazyLoader(_module_dirs(self.opts, 'modules', 'module'), self.opts, tag='module', whitelist=['test', 'pillar'])
class LazyLoaderDeepSubmodReloadingTest(TestCase): module_name = 'loadertestsubmoddeep' libs = ('top_lib', 'mid_lib', 'bot_lib') def setUp(self): self.opts = _config = minion_config(None) self.tmp_dir = tempfile.mkdtemp(dir=tests.integration.TMP) os.makedirs(self.module_dir) self.lib_count = collections.defaultdict( int) # mapping of path -> count # bootstrap libs with open(os.path.join(self.module_dir, '__init__.py'), 'w') as fh: fh.write(deep_init_base) fh.flush() os.fsync(fh.fileno()) # flush to disk self.lib_paths = {} dir_path = self.module_dir for lib_name in self.libs: dir_path = os.path.join(dir_path, lib_name) self.lib_paths[lib_name] = dir_path os.makedirs(dir_path) self.update_lib(lib_name) dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.loader = LazyLoader(dirs, self.opts, tag='module') @property def module_dir(self): return os.path.join(self.tmp_dir, self.module_name) def update_lib(self, lib_name): path = os.path.join(self.lib_paths[lib_name], '__init__.py') self.lib_count[lib_name] += 1 with open(path, 'wb') as fh: fh.write( submodule_lib_template.format(count=self.lib_count[lib_name])) fh.flush() os.fsync(fh.fileno()) # flush to disk # pyc files don't like it when we change the original quickly # since the header bytes only contain the timestamp (granularity of seconds) # TODO: don't write them? Is *much* slower on re-load (~3x) # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode try: os.unlink(path + 'c') except OSError: pass def tearDown(self): shutil.rmtree(self.tmp_dir) def test_basic(self): self.assertIn('{0}.top'.format(self.module_name), self.loader) def _verify_libs(self): for lib in self.libs: self.assertEqual( self.loader['{0}.{1}'.format(self.module_name, lib.replace('_lib', ''))](), self.lib_count[lib]) def test_reload(self): ''' Make sure that we can reload all libraries of arbitrary depth ''' self._verify_libs() # update them all for lib in self.libs: for x in xrange(5): self.update_lib(lib) self.loader.clear() self._verify_libs()
class LazyLoaderSubmodReloadingTest(TestCase): ''' Test the loader of salt with changing modules ''' module_name = 'loadertestsubmod' module_key = 'loadertestsubmod.test' def setUp(self): self.opts = _config = minion_config(None) self.tmp_dir = tempfile.mkdtemp(dir=tests.integration.TMP) os.makedirs(self.module_dir) self.count = 0 self.lib_count = 0 dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.loader = LazyLoader(dirs, self.opts, tag='module') def tearDown(self): shutil.rmtree(self.tmp_dir) def update_module(self): self.count += 1 with open(self.module_path, 'wb') as fh: fh.write(submodule_template.format(count=self.count)) fh.flush() os.fsync(fh.fileno()) # flush to disk # pyc files don't like it when we change the original quickly # since the header bytes only contain the timestamp (granularity of seconds) # TODO: don't write them? Is *much* slower on re-load (~3x) # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode try: os.unlink(self.module_path + 'c') except OSError: pass def rm_module(self): os.unlink(self.module_path) os.unlink(self.module_path + 'c') def update_lib(self): self.lib_count += 1 with open(self.lib_path, 'wb') as fh: fh.write(submodule_lib_template.format(count=self.lib_count)) fh.flush() os.fsync(fh.fileno()) # flush to disk # pyc files don't like it when we change the original quickly # since the header bytes only contain the timestamp (granularity of seconds) # TODO: don't write them? Is *much* slower on re-load (~3x) # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode try: os.unlink(self.lib_path + 'c') except OSError: pass def rm_lib(self): os.unlink(self.lib_path) os.unlink(self.lib_path + 'c') @property def module_dir(self): return os.path.join(self.tmp_dir, self.module_name) @property def module_path(self): return os.path.join(self.module_dir, '__init__.py') @property def lib_path(self): return os.path.join(self.module_dir, 'lib.py') def test_basic(self): # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) self.update_module() self.update_lib() self.loader.clear() self.assertIn(self.module_key, self.loader) def test_reload(self): # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) # update both the module and the lib for x in range(1, 3): self.update_module() self.update_lib() self.loader.clear() self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) # update just the module for x in range(1, 3): self.update_module() self.loader.clear() self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) # update just the lib for x in range(1, 3): self.update_lib() self.loader.clear() self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) self.rm_module() # make sure that even if we remove the module, its still loaded until a clear self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) self.loader.clear() self.assertNotIn(self.module_key, self.loader) def test_reload_missing_lib(self): # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) # update both the module and the lib self.update_module() self.update_lib() self.loader.clear() self.assertEqual(self.loader[self.module_key](), (self.count, self.lib_count)) # remove the lib, this means we should fail to load the module next time self.rm_lib() self.loader.clear() self.assertNotIn(self.module_key, self.loader)
class LazyLoaderReloadingTest(TestCase): ''' Test the loader of salt with changing modules ''' module_name = 'loadertest' module_key = 'loadertest.test' def setUp(self): self.opts = _config = minion_config(None) self.tmp_dir = tempfile.mkdtemp(dir=tests.integration.TMP) self.count = 0 dirs = _module_dirs(self.opts, 'modules', 'module') dirs.append(self.tmp_dir) self.loader = LazyLoader(dirs, self.opts, tag='module') def tearDown(self): shutil.rmtree(self.tmp_dir) def update_module(self): self.count += 1 with open(self.module_path, 'wb') as fh: fh.write(module_template.format(count=self.count)) fh.flush() os.fsync(fh.fileno()) # flush to disk # pyc files don't like it when we change the original quickly # since the header bytes only contain the timestamp (granularity of seconds) # TODO: don't write them? Is *much* slower on re-load (~3x) # https://docs.python.org/2/library/sys.html#sys.dont_write_bytecode try: os.unlink(self.module_path + 'c') except OSError: pass def rm_module(self): os.unlink(self.module_path) os.unlink(self.module_path + 'c') @property def module_path(self): return os.path.join(self.tmp_dir, '{0}.py'.format(self.module_name)) def test_alias(self): ''' Make sure that you can access alias-d modules ''' # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) self.update_module() self.assertNotIn('{0}.test_alias'.format(self.module_name), self.loader) self.assertTrue( inspect.isfunction(self.loader['{0}.working_alias'.format( self.module_name)])) def test_clear(self): self.assertTrue(inspect.isfunction(self.loader['test.ping'])) self.update_module() # write out out custom module self.loader.clear() # clear the loader dict # force a load of our module self.assertTrue(inspect.isfunction(self.loader[self.module_key])) # make sure we only loaded our custom module # which means that we did correctly refresh the file mapping for k, v in six.iteritems(self.loader._dict): self.assertTrue(k.startswith(self.module_name)) def test_load(self): # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) self.update_module() self.assertTrue(inspect.isfunction(self.loader[self.module_key])) def test__load__(self): ''' If a module specifies __load__ we should only load/expose those modules ''' self.update_module() # ensure it doesn't exist self.assertNotIn(self.module_key + '2', self.loader) def test__load__and_depends(self): ''' If a module specifies __load__ we should only load/expose those modules ''' self.update_module() # ensure it doesn't exist self.assertNotIn(self.module_key + '3', self.loader) self.assertNotIn(self.module_key + '4', self.loader) def test_reload(self): # ensure it doesn't exist self.assertNotIn(self.module_key, self.loader) # make sure it updates correctly for x in range(1, 3): self.update_module() self.loader.clear() self.assertEqual(self.loader[self.module_key](), self.count) self.rm_module() # make sure that even if we remove the module, its still loaded until a clear self.assertEqual(self.loader[self.module_key](), self.count) self.loader.clear() self.assertNotIn(self.module_key, self.loader)
def setUp(self): self.opts = _config = minion_config(None) self.loader = LazyLoader(_module_dirs(self.opts, 'modules', 'module'), self.opts, tag='module', virtual_enable=False)