Ejemplo n.º 1
0
 def _autoreload(self):
     th = self.scheduler.setTimer(self.checkinterval, self.checkinterval)
     try:
         tm = TimerEvent.createMatcher(th)
         while True:
             yield (tm,)
             t = time()
             reloads = []
             loaded = self.activeModules().values()
             self._logger.debug('Checking loaded modules: %r', loaded)
             for k in loaded:
                 p, _ = findModule(k, False)
                 if not p or not hasattr(p, '__file__') or not p.__file__:
                     continue
                 if p.__file__.endswith('.pyc'):
                     source = p.__file__[:-1]
                 else:
                     source = p.__file__
                 try:
                     mtime = os.path.getmtime(source)
                     if mtime <= t and mtime > self._lastcheck:
                         reloads.append(k)
                 except:
                     pass
             if reloads:
                 self._logger.warning('Auto reload following modules: %r', reloads)
                 try:
                     for m in self.reloadmodules(reloads):
                         yield m
                 except:
                     self._logger.warning('Exception occurs on auto reload', exc_info=True)
             self._lastcheck = t
     finally:
         self.scheduler.cancelTimer(th)
Ejemplo n.º 2
0
 def _autoreload(self):
     th = self.scheduler.setTimer(self.checkinterval, self.checkinterval)
     try:
         tm = TimerEvent.createMatcher(th)
         while True:
             yield (tm,)
             t = time()
             reloads = []
             loaded = self.activeModules().values()
             self._logger.debug("Checking loaded modules: %r", loaded)
             for k in loaded:
                 p, _ = findModule(k, False)
                 if not p or not hasattr(p, "__file__") or not p.__file__:
                     continue
                 if p.__file__.endswith(".pyc"):
                     source = p.__file__[:-1]
                 else:
                     source = p.__file__
                 try:
                     mtime = os.path.getmtime(source)
                     if mtime <= t and mtime > self._lastcheck:
                         reloads.append(k)
                 except:
                     pass
             if reloads:
                 self._logger.warning("Auto reload following modules: %r", reloads)
                 try:
                     for m in self.reloadmodules(reloads):
                         yield m
                 except:
                     self._logger.warning("Exception occurs on auto reload", exc_info=True)
             self._lastcheck = t
     finally:
         self.scheduler.cancelTimer(th)
Ejemplo n.º 3
0
    async def run(self, src, dst, clean=None):
        if clean is not None:
            clean = True
        else:
            clean = False
        src_module, sep, src_vhost = src.partition(':')
        dst_module, sep, dst_vhost = dst.partition(':')
        load_modules = [
            self.knownmodules.get(m, m) for m in set((src_module, dst_module))
        ]
        await self.server.moduleloader.execute_all(
            [self.server.moduleloader.loadByPath(m) for m in load_modules])
        src_service = findModule(self.knownmodules.get(
            src_module, src_module))[1]._instance.getServiceName()
        dst_service = findModule(self.knownmodules.get(
            dst_module, dst_module))[1]._instance.getServiceName()
        if clean:
            src_keys, dst_keys = await self.apiroutine.execute_all([
                call_api(self.apiroutine, src_service, 'listallkeys',
                         {'vhost': src_vhost}),
                call_api(self.apiroutine, dst_service, 'listallkeys',
                         {'vhost': dst_vhost})
            ])
        else:
            src_keys = await call_api(self.apiroutine, src_service,
                                      'listallkeys', {'vhost': src_vhost})
            dst_keys = []
        delete_keys = set(dst_keys)
        delete_keys.difference_update(src_keys)
        print('Migrating from %s (vhost = %r) to %s (vhost = %r)' %
              (src_service, src_vhost, dst_service, dst_vhost))
        print('Moving %d keys, deleting %d keys' %
              (len(src_keys), len(delete_keys)))
        print(
            'Please check. Migrating starts in 5 seconds, press Ctrl+C to cancel...'
        )
        await self.apiroutine.wait_with_timeout(5)
        for i in range(0, len(src_keys), 100):
            move_keys = tuple(src_keys[i:i + 100])
            try:
                values = await call_api(self.apiroutine, src_service, 'mget', {
                    'keys': move_keys,
                    'vhost': src_vhost
                })
            except ValueError:
                # There might be illegal keys, try them one by one
                async def move_key(key):
                    try:
                        (value, ) = await call_api(self.apiroutine,
                                                   src_service, 'mget', {
                                                       'keys': (key, ),
                                                       'vhost': src_vhost
                                                   })
                    except ValueError:
                        print(
                            'Key %r is not valid, it cannot be loaded. Ignore this key.'
                            % (key, ))
                    else:
                        await call_api(self.apiroutine, dst_service, 'mset', {
                            'kvpairs': ((key, value), ),
                            'vhost': dst_vhost
                        })

                await self.apiroutine.execute_all(
                    [move_key(k) for k in move_keys])
            else:
                await call_api(self.apiroutine, dst_service, 'mset', {
                    'kvpairs': tuple(zip(move_keys, values)),
                    'vhost': dst_vhost
                })
            print('%d keys moved' % (i, ))
        print('Deleting old keys...')
        if delete_keys:
            for k in delete_keys:
                await call_api(self.apiroutine, dst_service, 'delete', {
                    'key': k,
                    'vhost': dst_vhost
                })
        print('All jobs done.')
Ejemplo n.º 4
0
    def run(self, src, dst, clean=None):
        if clean is not None:
            clean = True
        else:
            clean = False
        src_module, sep, src_vhost = src.partition(":")
        dst_module, sep, dst_vhost = dst.partition(":")
        load_modules = [self.knownmodules.get(m, m) for m in set((src_module, dst_module))]
        with closing(
            self.apiroutine.executeAll(
                [self.server.moduleloader.loadByPath(m) for m in load_modules], self.server.moduleloader, ()
            )
        ) as g:
            for m in g:
                yield m
        src_service = findModule(self.knownmodules.get(src_module, src_module))[1]._instance.getServiceName()
        dst_service = findModule(self.knownmodules.get(dst_module, dst_module))[1]._instance.getServiceName()
        if clean:
            with closing(
                self.apiroutine.executeAll(
                    [
                        callAPI(self.apiroutine, src_service, "listallkeys", {"vhost": src_vhost}),
                        callAPI(self.apiroutine, dst_service, "listallkeys", {"vhost": dst_vhost}),
                    ]
                )
            ) as g:
                for m in g:
                    yield m
            (src_keys,), (dst_keys,) = self.apiroutine.retvalue
        else:
            for m in callAPI(self.apiroutine, src_service, "listallkeys", {"vhost": src_vhost}):
                yield m
            src_keys = self.apiroutine.retvalue
            dst_keys = []
        delete_keys = set(dst_keys)
        delete_keys.difference_update(src_keys)
        print("Migrating from %s (vhost = %r) to %s (vhost = %r)" % (src_service, src_vhost, dst_service, dst_vhost))
        print("Moving %d keys, deleting %d keys" % (len(src_keys), len(delete_keys)))
        print("Please check. Migrating starts in 5 seconds, press Ctrl+C to cancel...")
        for m in self.apiroutine.waitWithTimeout(5):
            yield m
        for i in range(0, len(src_keys), 100):
            move_keys = tuple(src_keys[i : i + 100])
            try:
                for m in callAPI(self.apiroutine, src_service, "mget", {"keys": move_keys, "vhost": src_vhost}):
                    yield m
                values = self.apiroutine.retvalue
            except ValueError:
                # There might be illegal keys, try them one by one
                def move_key(key):
                    try:
                        for m in callAPI(self.apiroutine, src_service, "mget", {"keys": (key,), "vhost": src_vhost}):
                            yield m
                    except ValueError:
                        print("Key %r is not valid, it cannot be loaded. Ignore this key." % (key,))
                    else:
                        for m in callAPI(
                            self.apiroutine,
                            dst_service,
                            "mset",
                            {"kvpairs": ((key, self.apiroutine.retvalue[0]),), "vhost": dst_vhost},
                        ):
                            yield m

                with closing(self.apiroutine.executeAll([move_key(k) for k in move_keys], retnames=())) as g:
                    for m in g:
                        yield m
            else:
                for m in callAPI(
                    self.apiroutine, dst_service, "mset", {"kvpairs": tuple(zip(move_keys, values)), "vhost": dst_vhost}
                ):
                    yield m
            print("%d keys moved" % (i,))
        print("Deleting old keys...")
        if delete_keys:
            for k in delete_keys:
                for m in callAPI(self.apiroutine, dst_service, "delete", {"key": k, "vhost": dst_vhost}):
                    yield m
        print("All jobs done.")