def testModuleLoad(self): logging.basicConfig() manager['server.startup'] = ('tests.gensrc.testmodule1.TestModule1', 'tests.gensrc.testmodule2.TestModule2') s = Server() import tests.gensrc basedir = tests.gensrc.__path__[0] with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f: f.write(module1) with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f: f.write(module2) # Run unittest discover may already load the module, reload it import tests.gensrc.testmodule1 import tests.gensrc.testmodule2 removeCache(tests.gensrc.testmodule1) removeCache(tests.gensrc.testmodule2) reload(tests.gensrc.testmodule1) reload(tests.gensrc.testmodule2) # Sometimes the timestamp is not working, make sure python re-compile the source file r = RoutineContainer(s.scheduler) apiResults = [] def testproc(): yield (ModuleLoadStateChanged.createMatcher(),) for m in callAPI(r, "testmodule1", "method1", {}): yield m apiResults.append(r.retvalue) for m in callAPI(r, "testmodule1", "method2", {'a' : 1, 'b' : 2}): yield m apiResults.append(r.retvalue) try: for m in callAPI(r, "testmodule1", "method4", {}): yield m apiResults.append(None) except ValueError as exc: apiResults.append(exc.args[0]) from .gensrc.testmodule2 import ModuleTestEvent2 matcher = ModuleTestEvent2.createMatcher() self.event = False def proc2(): for m in callAPI(r, "testmodule1", "method3", {'a' : 1, 'b' : 2}): yield m def callback(event, matcher): self.event = event if False: yield for m in r.withCallback(proc2(), callback, matcher): yield m if not self.event: for m in r.waitWithTimeout(0.1, matcher): yield m if not r.timeout: self.event = r.event if self.event: apiResults.append((self.event.result, self.event.version)) else: apiResults.append(False) for m in callAPI(r, "testmodule1", "discover", {}): yield m apiResults.append(r.retvalue) with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f: f.write(module1v2) for m in s.moduleloader.delegate(s.moduleloader.reloadModules(['tests.gensrc.testmodule1.TestModule1'])): yield m for m in callAPI(r, "testmodule1", "method1", {}): yield m apiResults.append(r.retvalue) matcher = ModuleTestEvent2.createMatcher() self.event = False def proc2_2(): for m in callAPI(r, "testmodule1", "method3", {'a' : 1, 'b' : 2}): yield m def callback_2(event, matcher): self.event = event if False: yield for m in r.withCallback(proc2_2(), callback_2, matcher): yield m if not self.event: for m in r.waitWithTimeout(0.1, matcher): yield m if not r.timeout: self.event = r.event if self.event: apiResults.append((self.event.result, self.event.version)) else: apiResults.append(False) with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f: f.write(module2v2) for m in s.moduleloader.delegate(s.moduleloader.reloadModules(['tests.gensrc.testmodule2.TestModule2'])): yield m matcher = ModuleTestEvent2.createMatcher() self.event = False def proc2_3(): for m in callAPI(r, "testmodule1", "method3", {'a' : 1, 'b' : 2}): yield m def callback_3(event, matcher): self.event = event if False: yield for m in r.withCallback(proc2_3(), callback_3, matcher): yield m if not self.event: for m in r.waitWithTimeout(0.1, matcher): yield m if not r.timeout: self.event = r.event if self.event: apiResults.append((self.event.result, self.event.version)) else: apiResults.append(False) with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f: f.write(module1v3) with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f: f.write(module2v3) for m in s.moduleloader.delegate(s.moduleloader.reloadModules(['tests.gensrc.testmodule1.TestModule1','tests.gensrc.testmodule2.TestModule2'])): yield m for m in callAPI(r, "testmodule1", "method1", {}): yield m apiResults.append(r.retvalue) matcher = ModuleTestEvent2.createMatcher() self.event = False def proc2_4(): for m in callAPI(r, "testmodule1", "method3", {'a' : 1, 'b' : 2}): yield m def callback_4(event, matcher): self.event = event if False: yield for m in r.withCallback(proc2_4(), callback_4, matcher): yield m if not self.event: for m in r.waitWithTimeout(0.1, matcher): yield m if not r.timeout: self.event = r.event if self.event: apiResults.append((self.event.result, self.event.version)) else: apiResults.append(False) try: for m in r.executeWithTimeout(1.0, callAPI(r, "testmodule1", "notexists", {})): yield m except ValueError: apiResults.append(True) except: apiResults.append(False) else: apiResults.append(False) for m in s.moduleloader.delegate(s.moduleloader.unloadByPath("tests.gensrc.testmodule1.TestModule1")): yield m r.main = testproc r.start() s.serve() print(repr(apiResults)) self.assertEqual(apiResults, ['version1', 3, 'test', (3, 'version1'), {'method1':'Run method1', 'method2':'Run method2', 'method3':'Run method3', 'method4': 'Run method4', 'discover':'Discover API definitions. Set details=true to show details'}, 'version2', (3, 'version1'), (3, 'version2'), 'version3', (3, 'version3'), True])
def reloadModules(self, pathlist): """ Reload modules with a full path in the pathlist """ loadedModules = [] failures = [] for path in pathlist: p, module = findModule(path, False) if module is not None and hasattr(module, '_instance') and module._instance.state != ModuleLoadStateChanged.UNLOADED: loadedModules.append(module) # Unload all modules ums = [ModuleLoadStateChanged.createMatcher(m, ModuleLoadStateChanged.UNLOADED) for m in loadedModules] for m in loadedModules: # Only unload the module itself, not its dependencies, since we will restart the module soon enough self.subroutine(self.unloadmodule(m, True), False) while ums: yield tuple(ums) ums.remove(self.matcher) # Group modules by package grouped = {} for path in pathlist: dotpos = path.rfind('.') if dotpos == -1: raise ModuleLoadException('Must specify module with full path, including package name') package = path[:dotpos] classname = path[dotpos + 1:] mlist = grouped.setdefault(package, []) p, module = findModule(path, False) mlist.append((classname, module)) for package, mlist in grouped.items(): # Reload each package only once try: p = sys.modules[package] # Remove cache to ensure a clean import from source file removeCache(p) p = reload(p) except KeyError: try: p = __import__(package, fromlist=[m[0] for m in mlist]) except: self._logger.warning('Failed to import a package: %r, resume others', package, exc_info = True) failures.append('Failed to import: ' + package) continue except: self._logger.warning('Failed to import a package: %r, resume others', package, exc_info = True) failures.append('Failed to import: ' + package) continue for cn, module in mlist: try: module2 = getattr(p, cn) except AttributeError: self._logger.warning('Cannot find module %r in package %r, resume others', package, cn) failures.append('Failed to import: ' + package + '.' + cn) continue if module is not None and module is not module2: # Update the references try: lpos = loadedModules.index(module) loaded = True except: loaded = False for d in module.depends: # The new reference is automatically added on import, only remove the old reference d.referencedBy.remove(module) if loaded and hasattr(d, '_instance'): try: d._instance.dependedBy.remove(module) d._instance.dependedBy.add(module2) except ValueError: pass if hasattr(module, 'referencedBy'): for d in module.referencedBy: pos = d.depends.index(module) d.depends[pos] = module2 if not hasattr(module2, 'referencedBy'): module2.referencedBy = [] module2.referencedBy.append(d) if loaded: loadedModules[lpos] = module2 # Start the uploaded modules for m in loadedModules: self.subroutine(self.loadmodule(m)) if failures: raise ModuleLoadException('Following errors occurred during reloading, check log for more details:\n' + '\n'.join(failures))
async def reload_modules(self, pathlist): """ Reload modules with a full path in the pathlist """ loadedModules = [] failures = [] for path in pathlist: p, module = findModule(path, False) if module is not None and hasattr(module, '_instance') and module._instance.state != ModuleLoadStateChanged.UNLOADED: loadedModules.append(module) # Unload all modules ums = [ModuleLoadStateChanged.createMatcher(m, ModuleLoadStateChanged.UNLOADED) for m in loadedModules] for m in loadedModules: # Only unload the module itself, not its dependencies, since we will restart the module soon enough self.subroutine(self.unloadmodule(m, True), False) await self.wait_for_all(*ums) # Group modules by package grouped = {} for path in pathlist: dotpos = path.rfind('.') if dotpos == -1: raise ModuleLoadException('Must specify module with full path, including package name') package = path[:dotpos] classname = path[dotpos + 1:] mlist = grouped.setdefault(package, []) p, module = findModule(path, False) mlist.append((classname, module)) for package, mlist in grouped.items(): # Reload each package only once try: p = sys.modules[package] # Remove cache to ensure a clean import from source file removeCache(p) p = reload(p) except KeyError: try: p = __import__(package, fromlist=[m[0] for m in mlist]) except Exception: self._logger.warning('Failed to import a package: %r, resume others', package, exc_info = True) failures.append('Failed to import: ' + package) continue except Exception: self._logger.warning('Failed to import a package: %r, resume others', package, exc_info = True) failures.append('Failed to import: ' + package) continue for cn, module in mlist: try: module2 = getattr(p, cn) except AttributeError: self._logger.warning('Cannot find module %r in package %r, resume others', package, cn) failures.append('Failed to import: ' + package + '.' + cn) continue if module is not None and module is not module2: # Update the references try: lpos = loadedModules.index(module) loaded = True except Exception: loaded = False for d in module.depends: # The new reference is automatically added on import, only remove the old reference d.referencedBy.remove(module) if loaded and hasattr(d, '_instance'): try: d._instance.dependedBy.remove(module) d._instance.dependedBy.add(module2) except ValueError: pass if hasattr(module, 'referencedBy'): for d in module.referencedBy: pos = d.depends.index(module) d.depends[pos] = module2 if not hasattr(module2, 'referencedBy'): module2.referencedBy = [] module2.referencedBy.append(d) if loaded: loadedModules[lpos] = module2 # Start the uploaded modules for m in loadedModules: self.subroutine(self.loadmodule(m)) if failures: raise ModuleLoadException('Following errors occurred during reloading, check log for more details:\n' + '\n'.join(failures))
def testModuleLoad(self): logging.basicConfig() manager['server.startup'] = ('tests.gensrc.testmodule1.TestModule1', 'tests.gensrc.testmodule2.TestModule2') s = Server() import tests.gensrc basedir = tests.gensrc.__path__[0] with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f: f.write(module1) with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f: f.write(module2) # Run unittest discover may already load the module, reload it import tests.gensrc.testmodule1 import tests.gensrc.testmodule2 removeCache(tests.gensrc.testmodule1) removeCache(tests.gensrc.testmodule2) reload(tests.gensrc.testmodule1) reload(tests.gensrc.testmodule2) # Sometimes the timestamp is not working, make sure python re-compile the source file r = RoutineContainer(s.scheduler) apiResults = [] def testproc(): yield (ModuleLoadStateChanged.createMatcher(), ) for m in callAPI(r, "testmodule1", "method1", {}): yield m apiResults.append(r.retvalue) for m in callAPI(r, "testmodule1", "method2", {'a': 1, 'b': 2}): yield m apiResults.append(r.retvalue) try: for m in callAPI(r, "testmodule1", "method4", {}): yield m apiResults.append(None) except ValueError as exc: apiResults.append(exc.args[0]) from .gensrc.testmodule2 import ModuleTestEvent2 matcher = ModuleTestEvent2.createMatcher() self.event = False def proc2(): for m in callAPI(r, "testmodule1", "method3", { 'a': 1, 'b': 2 }): yield m def callback(event, matcher): self.event = event if False: yield for m in r.withCallback(proc2(), callback, matcher): yield m if not self.event: for m in r.waitWithTimeout(0.1, matcher): yield m if not r.timeout: self.event = r.event if self.event: apiResults.append((self.event.result, self.event.version)) else: apiResults.append(False) for m in callAPI(r, "testmodule1", "discover", {}): yield m apiResults.append(r.retvalue) with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f: f.write(module1v2) for m in s.moduleloader.delegate( s.moduleloader.reloadModules( ['tests.gensrc.testmodule1.TestModule1'])): yield m for m in callAPI(r, "testmodule1", "method1", {}): yield m apiResults.append(r.retvalue) matcher = ModuleTestEvent2.createMatcher() self.event = False def proc2_2(): for m in callAPI(r, "testmodule1", "method3", { 'a': 1, 'b': 2 }): yield m def callback_2(event, matcher): self.event = event if False: yield for m in r.withCallback(proc2_2(), callback_2, matcher): yield m if not self.event: for m in r.waitWithTimeout(0.1, matcher): yield m if not r.timeout: self.event = r.event if self.event: apiResults.append((self.event.result, self.event.version)) else: apiResults.append(False) with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f: f.write(module2v2) for m in s.moduleloader.delegate( s.moduleloader.reloadModules( ['tests.gensrc.testmodule2.TestModule2'])): yield m matcher = ModuleTestEvent2.createMatcher() self.event = False def proc2_3(): for m in callAPI(r, "testmodule1", "method3", { 'a': 1, 'b': 2 }): yield m def callback_3(event, matcher): self.event = event if False: yield for m in r.withCallback(proc2_3(), callback_3, matcher): yield m if not self.event: for m in r.waitWithTimeout(0.1, matcher): yield m if not r.timeout: self.event = r.event if self.event: apiResults.append((self.event.result, self.event.version)) else: apiResults.append(False) with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f: f.write(module1v3) with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f: f.write(module2v3) for m in s.moduleloader.delegate( s.moduleloader.reloadModules([ 'tests.gensrc.testmodule1.TestModule1', 'tests.gensrc.testmodule2.TestModule2' ])): yield m for m in callAPI(r, "testmodule1", "method1", {}): yield m apiResults.append(r.retvalue) matcher = ModuleTestEvent2.createMatcher() self.event = False def proc2_4(): for m in callAPI(r, "testmodule1", "method3", { 'a': 1, 'b': 2 }): yield m def callback_4(event, matcher): self.event = event if False: yield for m in r.withCallback(proc2_4(), callback_4, matcher): yield m if not self.event: for m in r.waitWithTimeout(0.1, matcher): yield m if not r.timeout: self.event = r.event if self.event: apiResults.append((self.event.result, self.event.version)) else: apiResults.append(False) try: for m in r.executeWithTimeout( 1.0, callAPI(r, "testmodule1", "notexists", {})): yield m except ValueError: apiResults.append(True) except Exception: apiResults.append(False) else: apiResults.append(False) for m in s.moduleloader.delegate( s.moduleloader.unloadByPath( "tests.gensrc.testmodule1.TestModule1")): yield m r.main = testproc r.start() s.serve() print(repr(apiResults)) self.assertEqual(apiResults, [ 'version1', 3, 'test', (3, 'version1'), { 'method1': 'Run method1', 'method2': 'Run method2', 'method3': 'Run method3', 'method4': 'Run method4', 'discover': 'Discover API definitions. Set details=true to show details' }, 'version2', (3, 'version1'), (3, 'version2'), 'version3', (3, 'version3'), True ])