def start_server():
"""start the server for all general fetchers"""
import_all_modules(__file__, __name__)
if not register_fetcher.fetcher_list:
uklogger.log_err('no general fetcher available, exit')
return
def worker(fetcher):
try:
while True:
uklogger.log_info('run fetcher {}'.format(
fetcher.fetcher_name))
fetcher.run()
gevent.sleep(fetcher.sleep_time)
except KeyboardInterrupt:
uklogger.log_info('worker {} got KeyboardInterrupt, exit'.format(
fetcher.fetcher_name))
jobs = [gevent.spawn(worker, i) for i in register_fetcher.fetcher_list]
try:
for i in jobs:
i.join()
except KeyboardInterrupt:
uklogger.log_info('got KeyboardInterrupt, exit')
@property
def disp_name(self):
"""display name on webpage"""
return self._get_disp_name()
@property
def config_type(self):
"""get user configurable fields"""
return self._get_config_type()
def _get_mongo_collection(self):
""":return a mongo collection for this fetcher"""
return get_mongo('postfilter_' + self.name)
def _save_config(self, user_id, config):
"""save the config using default method"""
self._get_mongo_collection().save({'_id': user_id, 'config': config})
def _del_config(self, user_id):
"""remove config for a user"""
self._get_mongo_collection().remove({'_id': user_id})
def _load_config(self, user_id):
"""load the config using default method"""
rst = list(self._get_mongo_collection().find({'_id': user_id}))
if rst:
return rst[0]['config']
import_all_modules(__file__, __name__)
def load_all_fetcher():
if not getattr(load_all_fetcher, 'done', False):
import_all_modules(__file__, __name__)
load_all_fetcher.done = True