def wrapper():
ctx = self._create_fetcher_context()
assert isinstance(ctx, FetcherContext)
try:
return func(ctx)
except AbortItemProcessing:
pass
except KeyboardInterrupt:
raise
except Exception as exc:
uklogger.log_exc(exc)
def on_user_activated(user_id):
try:
user_fetcher = get_db_set(user_id, 'fetcher')
for i in user_fetcher:
fetcher = register_fetcher.fetcher_map.get(i)
if fetcher is None:
uklogger.log_err(
'fetcher {} not exist, requested by user {}'.format(
i, user_id))
else:
uklogger.log_info('run fetcher {} for user {}'.format(
i, user_id))
fetcher.run(user_id)
except Exception as ex:
uklogger.log_exc(ex)
def run(cls, ctx):
conf = cls.load_config(ctx.user_id)
if not conf:
ctx.new_item(TextOnlyItem(u'网络学堂验证失败', ''), ['THU learn'])
return
coll = ctx.get_mongo_collection()
if is_in_unittest():
entries = [{'id': 'test-{}'.format(uuid.uuid4()),
'title': 'thu learn in testcase',
'content': '{}@{}'.format(conf['username'],
conf['password']),
'create_time': '2013-12-14'}]
else:
try:
entries = fetch(conf['username'], conf['password'])
print entries
except Exception as e:
print e
ctx.new_item(TextOnlyItem(u'网络学堂抓取失败:' +
str(e), ''),
['THU learn'])
log_exc(e)
return
for entry in entries:
try:
coll.insert({'_id': str(ctx.user_id) + entry['id']})
except DuplicateKeyError:
continue
ctx.new_item(
TextOnlyItem(entry['title'], entry['content']),
['THU learn'], time.strptime(entry['create_time'], '%Y-%m-%d'),
{'id': entry['id']})
log_info(u'THU learn: new entry: {} {}'.format(entry['id'],
entry['title']))