def start_server(): """start the server for all general fetchers""" import_all_modules(__file__, __name__) if not register_fetcher.fetcher_list: uklogger.log_err('no general fetcher available, exit') return def worker(fetcher): try: while True: uklogger.log_info('run fetcher {}'.format( fetcher.fetcher_name)) fetcher.run() gevent.sleep(fetcher.sleep_time) except KeyboardInterrupt: uklogger.log_info('worker {} got KeyboardInterrupt, exit'.format( fetcher.fetcher_name)) jobs = [gevent.spawn(worker, i) for i in register_fetcher.fetcher_list] try: for i in jobs: i.join() except KeyboardInterrupt: uklogger.log_info('got KeyboardInterrupt, exit')
@property def disp_name(self): """display name on webpage""" return self._get_disp_name() @property def config_type(self): """get user configurable fields""" return self._get_config_type() def _get_mongo_collection(self): """:return a mongo collection for this fetcher""" return get_mongo('postfilter_' + self.name) def _save_config(self, user_id, config): """save the config using default method""" self._get_mongo_collection().save({'_id': user_id, 'config': config}) def _del_config(self, user_id): """remove config for a user""" self._get_mongo_collection().remove({'_id': user_id}) def _load_config(self, user_id): """load the config using default method""" rst = list(self._get_mongo_collection().find({'_id': user_id})) if rst: return rst[0]['config'] import_all_modules(__file__, __name__)
def load_all_fetcher(): if not getattr(load_all_fetcher, 'done', False): import_all_modules(__file__, __name__) load_all_fetcher.done = True