Ejemplo n.º 1
0
def start_server():
    """start the server for all general fetchers"""
    import_all_modules(__file__, __name__)

    if not register_fetcher.fetcher_list:
        uklogger.log_err('no general fetcher available, exit')
        return

    def worker(fetcher):
        try:
            while True:
                uklogger.log_info('run fetcher {}'.format(
                    fetcher.fetcher_name))
                fetcher.run()
                gevent.sleep(fetcher.sleep_time)
        except KeyboardInterrupt:
            uklogger.log_info('worker {} got KeyboardInterrupt, exit'.format(
                fetcher.fetcher_name))

    jobs = [gevent.spawn(worker, i) for i in register_fetcher.fetcher_list]
    try:
        for i in jobs:
            i.join()
    except KeyboardInterrupt:
        uklogger.log_info('got KeyboardInterrupt, exit')
Ejemplo n.º 2
0
    @property
    def disp_name(self):
        """display name on webpage"""
        return self._get_disp_name()

    @property
    def config_type(self):
        """get user configurable fields"""
        return self._get_config_type()

    def _get_mongo_collection(self):
        """:return a mongo collection for this fetcher"""
        return get_mongo('postfilter_' + self.name)

    def _save_config(self, user_id, config):
        """save the config using default method"""
        self._get_mongo_collection().save({'_id': user_id, 'config': config})

    def _del_config(self, user_id):
        """remove config for a user"""
        self._get_mongo_collection().remove({'_id': user_id})

    def _load_config(self, user_id):
        """load the config using default method"""
        rst = list(self._get_mongo_collection().find({'_id': user_id}))
        if rst:
            return rst[0]['config']


import_all_modules(__file__, __name__)
Ejemplo n.º 3
0
def load_all_fetcher():
    if not getattr(load_all_fetcher, 'done', False):
        import_all_modules(__file__, __name__)
        load_all_fetcher.done = True