def start_server():
"""start the server for all general fetchers"""
import_all_modules(__file__, __name__)
if not register_fetcher.fetcher_list:
uklogger.log_err('no general fetcher available, exit')
return
def worker(fetcher):
try:
while True:
uklogger.log_info('run fetcher {}'.format(
fetcher.fetcher_name))
fetcher.run()
gevent.sleep(fetcher.sleep_time)
except KeyboardInterrupt:
uklogger.log_info('worker {} got KeyboardInterrupt, exit'.format(
fetcher.fetcher_name))
jobs = [gevent.spawn(worker, i) for i in register_fetcher.fetcher_list]
try:
for i in jobs:
i.join()
except KeyboardInterrupt:
uklogger.log_info('got KeyboardInterrupt, exit')
def on_user_activated(user_id):
try:
user_fetcher = get_db_set(user_id, 'fetcher')
for i in user_fetcher:
fetcher = register_fetcher.fetcher_map.get(i)
if fetcher is None:
uklogger.log_err(
'fetcher {} not exist, requested by user {}'.format(
i, user_id))
else:
uklogger.log_info('run fetcher {} for user {}'.format(
i, user_id))
fetcher.run(user_id)
except Exception as ex:
uklogger.log_exc(ex)
def apply(cls, ctx, doc):
"""apply all prefilters; invoked by :meth:`FetcherContext.new_item`
note that user-defined prefilters would be first invoked, and then
system-wide ones"""
if ctx.user_id is not None:
for i in get_db_set(ctx.user_id, 'prefilter'):
flt = cls.user_customizable_map.get(i)
if flt is None:
uklogger.log_err(
'prefilter {} not exist, requested by user {}'.format(
i, ctx.user_id))
else:
flt.func(ctx, doc)
for i in cls.prefilter_list:
i.func(ctx, doc)
