def _autoreload(self):
th = self.scheduler.setTimer(self.checkinterval, self.checkinterval)
try:
tm = TimerEvent.createMatcher(th)
while True:
yield (tm,)
t = time()
reloads = []
loaded = self.activeModules().values()
self._logger.debug('Checking loaded modules: %r', loaded)
for k in loaded:
p, _ = findModule(k, False)
if not p or not hasattr(p, '__file__') or not p.__file__:
continue
if p.__file__.endswith('.pyc'):
source = p.__file__[:-1]
else:
source = p.__file__
try:
mtime = os.path.getmtime(source)
if mtime <= t and mtime > self._lastcheck:
reloads.append(k)
except:
pass
if reloads:
self._logger.warning('Auto reload following modules: %r', reloads)
try:
for m in self.reloadmodules(reloads):
yield m
except:
self._logger.warning('Exception occurs on auto reload', exc_info=True)
self._lastcheck = t
finally:
self.scheduler.cancelTimer(th)
def _autoreload(self):
th = self.scheduler.setTimer(self.checkinterval, self.checkinterval)
try:
tm = TimerEvent.createMatcher(th)
while True:
yield (tm,)
t = time()
reloads = []
loaded = self.activeModules().values()
self._logger.debug("Checking loaded modules: %r", loaded)
for k in loaded:
p, _ = findModule(k, False)
if not p or not hasattr(p, "__file__") or not p.__file__:
continue
if p.__file__.endswith(".pyc"):
source = p.__file__[:-1]
else:
source = p.__file__
try:
mtime = os.path.getmtime(source)
if mtime <= t and mtime > self._lastcheck:
reloads.append(k)
except:
pass
if reloads:
self._logger.warning("Auto reload following modules: %r", reloads)
try:
for m in self.reloadmodules(reloads):
yield m
except:
self._logger.warning("Exception occurs on auto reload", exc_info=True)
self._lastcheck = t
finally:
self.scheduler.cancelTimer(th)
async def run(self, src, dst, clean=None):
if clean is not None:
clean = True
else:
clean = False
src_module, sep, src_vhost = src.partition(':')
dst_module, sep, dst_vhost = dst.partition(':')
load_modules = [
self.knownmodules.get(m, m) for m in set((src_module, dst_module))
]
await self.server.moduleloader.execute_all(
[self.server.moduleloader.loadByPath(m) for m in load_modules])
src_service = findModule(self.knownmodules.get(
src_module, src_module))[1]._instance.getServiceName()
dst_service = findModule(self.knownmodules.get(
dst_module, dst_module))[1]._instance.getServiceName()
if clean:
src_keys, dst_keys = await self.apiroutine.execute_all([
call_api(self.apiroutine, src_service, 'listallkeys',
{'vhost': src_vhost}),
call_api(self.apiroutine, dst_service, 'listallkeys',
{'vhost': dst_vhost})
])
else:
src_keys = await call_api(self.apiroutine, src_service,
'listallkeys', {'vhost': src_vhost})
dst_keys = []
delete_keys = set(dst_keys)
delete_keys.difference_update(src_keys)
print('Migrating from %s (vhost = %r) to %s (vhost = %r)' %
(src_service, src_vhost, dst_service, dst_vhost))
print('Moving %d keys, deleting %d keys' %
(len(src_keys), len(delete_keys)))
print(
'Please check. Migrating starts in 5 seconds, press Ctrl+C to cancel...'
)
await self.apiroutine.wait_with_timeout(5)
for i in range(0, len(src_keys), 100):
move_keys = tuple(src_keys[i:i + 100])
try:
values = await call_api(self.apiroutine, src_service, 'mget', {
'keys': move_keys,
'vhost': src_vhost
})
except ValueError:
# There might be illegal keys, try them one by one
async def move_key(key):
try:
(value, ) = await call_api(self.apiroutine,
src_service, 'mget', {
'keys': (key, ),
'vhost': src_vhost
})
except ValueError:
print(
'Key %r is not valid, it cannot be loaded. Ignore this key.'
% (key, ))
else:
await call_api(self.apiroutine, dst_service, 'mset', {
'kvpairs': ((key, value), ),
'vhost': dst_vhost
})
await self.apiroutine.execute_all(
[move_key(k) for k in move_keys])
else:
await call_api(self.apiroutine, dst_service, 'mset', {
'kvpairs': tuple(zip(move_keys, values)),
'vhost': dst_vhost
})
print('%d keys moved' % (i, ))
print('Deleting old keys...')
if delete_keys:
for k in delete_keys:
await call_api(self.apiroutine, dst_service, 'delete', {
'key': k,
'vhost': dst_vhost
})
print('All jobs done.')
def run(self, src, dst, clean=None):
if clean is not None:
clean = True
else:
clean = False
src_module, sep, src_vhost = src.partition(":")
dst_module, sep, dst_vhost = dst.partition(":")
load_modules = [self.knownmodules.get(m, m) for m in set((src_module, dst_module))]
with closing(
self.apiroutine.executeAll(
[self.server.moduleloader.loadByPath(m) for m in load_modules], self.server.moduleloader, ()
)
) as g:
for m in g:
yield m
src_service = findModule(self.knownmodules.get(src_module, src_module))[1]._instance.getServiceName()
dst_service = findModule(self.knownmodules.get(dst_module, dst_module))[1]._instance.getServiceName()
if clean:
with closing(
self.apiroutine.executeAll(
[
callAPI(self.apiroutine, src_service, "listallkeys", {"vhost": src_vhost}),
callAPI(self.apiroutine, dst_service, "listallkeys", {"vhost": dst_vhost}),
]
)
) as g:
for m in g:
yield m
(src_keys,), (dst_keys,) = self.apiroutine.retvalue
else:
for m in callAPI(self.apiroutine, src_service, "listallkeys", {"vhost": src_vhost}):
yield m
src_keys = self.apiroutine.retvalue
dst_keys = []
delete_keys = set(dst_keys)
delete_keys.difference_update(src_keys)
print("Migrating from %s (vhost = %r) to %s (vhost = %r)" % (src_service, src_vhost, dst_service, dst_vhost))
print("Moving %d keys, deleting %d keys" % (len(src_keys), len(delete_keys)))
print("Please check. Migrating starts in 5 seconds, press Ctrl+C to cancel...")
for m in self.apiroutine.waitWithTimeout(5):
yield m
for i in range(0, len(src_keys), 100):
move_keys = tuple(src_keys[i : i + 100])
try:
for m in callAPI(self.apiroutine, src_service, "mget", {"keys": move_keys, "vhost": src_vhost}):
yield m
values = self.apiroutine.retvalue
except ValueError:
# There might be illegal keys, try them one by one
def move_key(key):
try:
for m in callAPI(self.apiroutine, src_service, "mget", {"keys": (key,), "vhost": src_vhost}):
yield m
except ValueError:
print("Key %r is not valid, it cannot be loaded. Ignore this key." % (key,))
else:
for m in callAPI(
self.apiroutine,
dst_service,
"mset",
{"kvpairs": ((key, self.apiroutine.retvalue[0]),), "vhost": dst_vhost},
):
yield m
with closing(self.apiroutine.executeAll([move_key(k) for k in move_keys], retnames=())) as g:
for m in g:
yield m
else:
for m in callAPI(
self.apiroutine, dst_service, "mset", {"kvpairs": tuple(zip(move_keys, values)), "vhost": dst_vhost}
):
yield m
print("%d keys moved" % (i,))
print("Deleting old keys...")
if delete_keys:
for k in delete_keys:
for m in callAPI(self.apiroutine, dst_service, "delete", {"key": k, "vhost": dst_vhost}):
yield m
print("All jobs done.")
