def testModuleLoad(self):
logging.basicConfig()
manager['server.startup'] = ('tests.gensrc.testmodule1.TestModule1', 'tests.gensrc.testmodule2.TestModule2')
s = Server()
import tests.gensrc
basedir = tests.gensrc.__path__[0]
with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f:
f.write(module1)
with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f:
f.write(module2)
# Run unittest discover may already load the module, reload it
import tests.gensrc.testmodule1
import tests.gensrc.testmodule2
removeCache(tests.gensrc.testmodule1)
removeCache(tests.gensrc.testmodule2)
reload(tests.gensrc.testmodule1)
reload(tests.gensrc.testmodule2)
# Sometimes the timestamp is not working, make sure python re-compile the source file
r = RoutineContainer(s.scheduler)
apiResults = []
def testproc():
yield (ModuleLoadStateChanged.createMatcher(),)
for m in callAPI(r, "testmodule1", "method1", {}):
yield m
apiResults.append(r.retvalue)
for m in callAPI(r, "testmodule1", "method2", {'a' : 1, 'b' : 2}):
yield m
apiResults.append(r.retvalue)
try:
for m in callAPI(r, "testmodule1", "method4", {}):
yield m
apiResults.append(None)
except ValueError as exc:
apiResults.append(exc.args[0])
from .gensrc.testmodule2 import ModuleTestEvent2
matcher = ModuleTestEvent2.createMatcher()
self.event = False
def proc2():
for m in callAPI(r, "testmodule1", "method3", {'a' : 1, 'b' : 2}):
yield m
def callback(event, matcher):
self.event = event
if False:
yield
for m in r.withCallback(proc2(), callback, matcher):
yield m
if not self.event:
for m in r.waitWithTimeout(0.1, matcher):
yield m
if not r.timeout:
self.event = r.event
if self.event:
apiResults.append((self.event.result, self.event.version))
else:
apiResults.append(False)
for m in callAPI(r, "testmodule1", "discover", {}):
yield m
apiResults.append(r.retvalue)
with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f:
f.write(module1v2)
for m in s.moduleloader.delegate(s.moduleloader.reloadModules(['tests.gensrc.testmodule1.TestModule1'])):
yield m
for m in callAPI(r, "testmodule1", "method1", {}):
yield m
apiResults.append(r.retvalue)
matcher = ModuleTestEvent2.createMatcher()
self.event = False
def proc2_2():
for m in callAPI(r, "testmodule1", "method3", {'a' : 1, 'b' : 2}):
yield m
def callback_2(event, matcher):
self.event = event
if False:
yield
for m in r.withCallback(proc2_2(), callback_2, matcher):
yield m
if not self.event:
for m in r.waitWithTimeout(0.1, matcher):
yield m
if not r.timeout:
self.event = r.event
if self.event:
apiResults.append((self.event.result, self.event.version))
else:
apiResults.append(False)
with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f:
f.write(module2v2)
for m in s.moduleloader.delegate(s.moduleloader.reloadModules(['tests.gensrc.testmodule2.TestModule2'])):
yield m
matcher = ModuleTestEvent2.createMatcher()
self.event = False
def proc2_3():
for m in callAPI(r, "testmodule1", "method3", {'a' : 1, 'b' : 2}):
yield m
def callback_3(event, matcher):
self.event = event
if False:
yield
for m in r.withCallback(proc2_3(), callback_3, matcher):
yield m
if not self.event:
for m in r.waitWithTimeout(0.1, matcher):
yield m
if not r.timeout:
self.event = r.event
if self.event:
apiResults.append((self.event.result, self.event.version))
else:
apiResults.append(False)
with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f:
f.write(module1v3)
with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f:
f.write(module2v3)
for m in s.moduleloader.delegate(s.moduleloader.reloadModules(['tests.gensrc.testmodule1.TestModule1','tests.gensrc.testmodule2.TestModule2'])):
yield m
for m in callAPI(r, "testmodule1", "method1", {}):
yield m
apiResults.append(r.retvalue)
matcher = ModuleTestEvent2.createMatcher()
self.event = False
def proc2_4():
for m in callAPI(r, "testmodule1", "method3", {'a' : 1, 'b' : 2}):
yield m
def callback_4(event, matcher):
self.event = event
if False:
yield
for m in r.withCallback(proc2_4(), callback_4, matcher):
yield m
if not self.event:
for m in r.waitWithTimeout(0.1, matcher):
yield m
if not r.timeout:
self.event = r.event
if self.event:
apiResults.append((self.event.result, self.event.version))
else:
apiResults.append(False)
try:
for m in r.executeWithTimeout(1.0, callAPI(r, "testmodule1", "notexists", {})):
yield m
except ValueError:
apiResults.append(True)
except:
apiResults.append(False)
else:
apiResults.append(False)
for m in s.moduleloader.delegate(s.moduleloader.unloadByPath("tests.gensrc.testmodule1.TestModule1")):
yield m
r.main = testproc
r.start()
s.serve()
print(repr(apiResults))
self.assertEqual(apiResults, ['version1', 3, 'test', (3, 'version1'),
{'method1':'Run method1', 'method2':'Run method2', 'method3':'Run method3', 'method4': 'Run method4', 'discover':'Discover API definitions. Set details=true to show details'},
'version2', (3, 'version1'), (3, 'version2'), 'version3', (3, 'version3'), True])
def reloadModules(self, pathlist):
"""
Reload modules with a full path in the pathlist
"""
loadedModules = []
failures = []
for path in pathlist:
p, module = findModule(path, False)
if module is not None and hasattr(module, '_instance') and module._instance.state != ModuleLoadStateChanged.UNLOADED:
loadedModules.append(module)
# Unload all modules
ums = [ModuleLoadStateChanged.createMatcher(m, ModuleLoadStateChanged.UNLOADED) for m in loadedModules]
for m in loadedModules:
# Only unload the module itself, not its dependencies, since we will restart the module soon enough
self.subroutine(self.unloadmodule(m, True), False)
while ums:
yield tuple(ums)
ums.remove(self.matcher)
# Group modules by package
grouped = {}
for path in pathlist:
dotpos = path.rfind('.')
if dotpos == -1:
raise ModuleLoadException('Must specify module with full path, including package name')
package = path[:dotpos]
classname = path[dotpos + 1:]
mlist = grouped.setdefault(package, [])
p, module = findModule(path, False)
mlist.append((classname, module))
for package, mlist in grouped.items():
# Reload each package only once
try:
p = sys.modules[package]
# Remove cache to ensure a clean import from source file
removeCache(p)
p = reload(p)
except KeyError:
try:
p = __import__(package, fromlist=[m[0] for m in mlist])
except:
self._logger.warning('Failed to import a package: %r, resume others', package, exc_info = True)
failures.append('Failed to import: ' + package)
continue
except:
self._logger.warning('Failed to import a package: %r, resume others', package, exc_info = True)
failures.append('Failed to import: ' + package)
continue
for cn, module in mlist:
try:
module2 = getattr(p, cn)
except AttributeError:
self._logger.warning('Cannot find module %r in package %r, resume others', package, cn)
failures.append('Failed to import: ' + package + '.' + cn)
continue
if module is not None and module is not module2:
# Update the references
try:
lpos = loadedModules.index(module)
loaded = True
except:
loaded = False
for d in module.depends:
# The new reference is automatically added on import, only remove the old reference
d.referencedBy.remove(module)
if loaded and hasattr(d, '_instance'):
try:
d._instance.dependedBy.remove(module)
d._instance.dependedBy.add(module2)
except ValueError:
pass
if hasattr(module, 'referencedBy'):
for d in module.referencedBy:
pos = d.depends.index(module)
d.depends[pos] = module2
if not hasattr(module2, 'referencedBy'):
module2.referencedBy = []
module2.referencedBy.append(d)
if loaded:
loadedModules[lpos] = module2
# Start the uploaded modules
for m in loadedModules:
self.subroutine(self.loadmodule(m))
if failures:
raise ModuleLoadException('Following errors occurred during reloading, check log for more details:\n' + '\n'.join(failures))
async def reload_modules(self, pathlist):
"""
Reload modules with a full path in the pathlist
"""
loadedModules = []
failures = []
for path in pathlist:
p, module = findModule(path, False)
if module is not None and hasattr(module, '_instance') and module._instance.state != ModuleLoadStateChanged.UNLOADED:
loadedModules.append(module)
# Unload all modules
ums = [ModuleLoadStateChanged.createMatcher(m, ModuleLoadStateChanged.UNLOADED) for m in loadedModules]
for m in loadedModules:
# Only unload the module itself, not its dependencies, since we will restart the module soon enough
self.subroutine(self.unloadmodule(m, True), False)
await self.wait_for_all(*ums)
# Group modules by package
grouped = {}
for path in pathlist:
dotpos = path.rfind('.')
if dotpos == -1:
raise ModuleLoadException('Must specify module with full path, including package name')
package = path[:dotpos]
classname = path[dotpos + 1:]
mlist = grouped.setdefault(package, [])
p, module = findModule(path, False)
mlist.append((classname, module))
for package, mlist in grouped.items():
# Reload each package only once
try:
p = sys.modules[package]
# Remove cache to ensure a clean import from source file
removeCache(p)
p = reload(p)
except KeyError:
try:
p = __import__(package, fromlist=[m[0] for m in mlist])
except Exception:
self._logger.warning('Failed to import a package: %r, resume others', package, exc_info = True)
failures.append('Failed to import: ' + package)
continue
except Exception:
self._logger.warning('Failed to import a package: %r, resume others', package, exc_info = True)
failures.append('Failed to import: ' + package)
continue
for cn, module in mlist:
try:
module2 = getattr(p, cn)
except AttributeError:
self._logger.warning('Cannot find module %r in package %r, resume others', package, cn)
failures.append('Failed to import: ' + package + '.' + cn)
continue
if module is not None and module is not module2:
# Update the references
try:
lpos = loadedModules.index(module)
loaded = True
except Exception:
loaded = False
for d in module.depends:
# The new reference is automatically added on import, only remove the old reference
d.referencedBy.remove(module)
if loaded and hasattr(d, '_instance'):
try:
d._instance.dependedBy.remove(module)
d._instance.dependedBy.add(module2)
except ValueError:
pass
if hasattr(module, 'referencedBy'):
for d in module.referencedBy:
pos = d.depends.index(module)
d.depends[pos] = module2
if not hasattr(module2, 'referencedBy'):
module2.referencedBy = []
module2.referencedBy.append(d)
if loaded:
loadedModules[lpos] = module2
# Start the uploaded modules
for m in loadedModules:
self.subroutine(self.loadmodule(m))
if failures:
raise ModuleLoadException('Following errors occurred during reloading, check log for more details:\n' + '\n'.join(failures))
def testModuleLoad(self):
logging.basicConfig()
manager['server.startup'] = ('tests.gensrc.testmodule1.TestModule1',
'tests.gensrc.testmodule2.TestModule2')
s = Server()
import tests.gensrc
basedir = tests.gensrc.__path__[0]
with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f:
f.write(module1)
with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f:
f.write(module2)
# Run unittest discover may already load the module, reload it
import tests.gensrc.testmodule1
import tests.gensrc.testmodule2
removeCache(tests.gensrc.testmodule1)
removeCache(tests.gensrc.testmodule2)
reload(tests.gensrc.testmodule1)
reload(tests.gensrc.testmodule2)
# Sometimes the timestamp is not working, make sure python re-compile the source file
r = RoutineContainer(s.scheduler)
apiResults = []
def testproc():
yield (ModuleLoadStateChanged.createMatcher(), )
for m in callAPI(r, "testmodule1", "method1", {}):
yield m
apiResults.append(r.retvalue)
for m in callAPI(r, "testmodule1", "method2", {'a': 1, 'b': 2}):
yield m
apiResults.append(r.retvalue)
try:
for m in callAPI(r, "testmodule1", "method4", {}):
yield m
apiResults.append(None)
except ValueError as exc:
apiResults.append(exc.args[0])
from .gensrc.testmodule2 import ModuleTestEvent2
matcher = ModuleTestEvent2.createMatcher()
self.event = False
def proc2():
for m in callAPI(r, "testmodule1", "method3", {
'a': 1,
'b': 2
}):
yield m
def callback(event, matcher):
self.event = event
if False:
yield
for m in r.withCallback(proc2(), callback, matcher):
yield m
if not self.event:
for m in r.waitWithTimeout(0.1, matcher):
yield m
if not r.timeout:
self.event = r.event
if self.event:
apiResults.append((self.event.result, self.event.version))
else:
apiResults.append(False)
for m in callAPI(r, "testmodule1", "discover", {}):
yield m
apiResults.append(r.retvalue)
with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f:
f.write(module1v2)
for m in s.moduleloader.delegate(
s.moduleloader.reloadModules(
['tests.gensrc.testmodule1.TestModule1'])):
yield m
for m in callAPI(r, "testmodule1", "method1", {}):
yield m
apiResults.append(r.retvalue)
matcher = ModuleTestEvent2.createMatcher()
self.event = False
def proc2_2():
for m in callAPI(r, "testmodule1", "method3", {
'a': 1,
'b': 2
}):
yield m
def callback_2(event, matcher):
self.event = event
if False:
yield
for m in r.withCallback(proc2_2(), callback_2, matcher):
yield m
if not self.event:
for m in r.waitWithTimeout(0.1, matcher):
yield m
if not r.timeout:
self.event = r.event
if self.event:
apiResults.append((self.event.result, self.event.version))
else:
apiResults.append(False)
with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f:
f.write(module2v2)
for m in s.moduleloader.delegate(
s.moduleloader.reloadModules(
['tests.gensrc.testmodule2.TestModule2'])):
yield m
matcher = ModuleTestEvent2.createMatcher()
self.event = False
def proc2_3():
for m in callAPI(r, "testmodule1", "method3", {
'a': 1,
'b': 2
}):
yield m
def callback_3(event, matcher):
self.event = event
if False:
yield
for m in r.withCallback(proc2_3(), callback_3, matcher):
yield m
if not self.event:
for m in r.waitWithTimeout(0.1, matcher):
yield m
if not r.timeout:
self.event = r.event
if self.event:
apiResults.append((self.event.result, self.event.version))
else:
apiResults.append(False)
with open(os.path.join(basedir, 'testmodule1.py'), 'wb') as f:
f.write(module1v3)
with open(os.path.join(basedir, 'testmodule2.py'), 'wb') as f:
f.write(module2v3)
for m in s.moduleloader.delegate(
s.moduleloader.reloadModules([
'tests.gensrc.testmodule1.TestModule1',
'tests.gensrc.testmodule2.TestModule2'
])):
yield m
for m in callAPI(r, "testmodule1", "method1", {}):
yield m
apiResults.append(r.retvalue)
matcher = ModuleTestEvent2.createMatcher()
self.event = False
def proc2_4():
for m in callAPI(r, "testmodule1", "method3", {
'a': 1,
'b': 2
}):
yield m
def callback_4(event, matcher):
self.event = event
if False:
yield
for m in r.withCallback(proc2_4(), callback_4, matcher):
yield m
if not self.event:
for m in r.waitWithTimeout(0.1, matcher):
yield m
if not r.timeout:
self.event = r.event
if self.event:
apiResults.append((self.event.result, self.event.version))
else:
apiResults.append(False)
try:
for m in r.executeWithTimeout(
1.0, callAPI(r, "testmodule1", "notexists", {})):
yield m
except ValueError:
apiResults.append(True)
except Exception:
apiResults.append(False)
else:
apiResults.append(False)
for m in s.moduleloader.delegate(
s.moduleloader.unloadByPath(
"tests.gensrc.testmodule1.TestModule1")):
yield m
r.main = testproc
r.start()
s.serve()
print(repr(apiResults))
self.assertEqual(apiResults, [
'version1', 3, 'test', (3, 'version1'), {
'method1':
'Run method1',
'method2':
'Run method2',
'method3':
'Run method3',
'method4':
'Run method4',
'discover':
'Discover API definitions. Set details=true to show details'
}, 'version2', (3, 'version1'), (3, 'version2'), 'version3',
(3, 'version3'), True
])
