def get_celery_task():
"""get celery task, which takes user id as its sole argument"""
global _celery_app
global _celery_task
if _celery_task:
return _celery_task
load_all_fetcher()
_celery_app = Celery('ukfetcher', broker=ukconfig.celery_broker)
_celery_app.conf.update(
CELERY_ACCEPT_CONTENT=['pickle', 'json', 'msgpack', 'yaml'])
@_celery_app.task
def on_user_activated(user_id):
try:
user_fetcher = get_db_set(user_id, 'fetcher')
for i in user_fetcher:
fetcher = register_fetcher.fetcher_map.get(i)
if fetcher is None:
uklogger.log_err(
'fetcher {} not exist, requested by user {}'.format(
i, user_id))
else:
uklogger.log_info('run fetcher {} for user {}'.format(
i, user_id))
fetcher.run(user_id)
except Exception as ex:
uklogger.log_exc(ex)
if is_in_unittest():
_celery_task = on_user_activated
else:
_celery_task = on_user_activated.delay
return _celery_task
def run(cls, ctx):
conf = cls.load_config(ctx.user_id)
if not conf:
ctx.new_item(TextOnlyItem(u'网络学堂验证失败', ''), ['THU learn'])
return
coll = ctx.get_mongo_collection()
if is_in_unittest():
entries = [{'id': 'test-{}'.format(uuid.uuid4()),
'title': 'thu learn in testcase',
'content': '{}@{}'.format(conf['username'],
conf['password']),
'create_time': '2013-12-14'}]
else:
try:
entries = fetch(conf['username'], conf['password'])
print entries
except Exception as e:
print e
ctx.new_item(TextOnlyItem(u'网络学堂抓取失败:' +
str(e), ''),
['THU learn'])
log_exc(e)
return
for entry in entries:
try:
coll.insert({'_id': str(ctx.user_id) + entry['id']})
except DuplicateKeyError:
continue
ctx.new_item(
TextOnlyItem(entry['title'], entry['content']),
['THU learn'], time.strptime(entry['create_time'], '%Y-%m-%d'),
{'id': entry['id']})
log_info(u'THU learn: new entry: {} {}'.format(entry['id'],
entry['title']))
"""log a message from api-website"""
print colored('API', 'green'), msg
# TODO: use log util, log to file, including time, module, etc.
def log_fetcher(msg):
"""log a message from fetcher"""
print colored('FETCHER', 'yellow'), msg
# TODO: use log util, log to file, including time, module, etc.
def log_info(msg):
"""log an info message"""
print colored('INFO', 'blue'), msg
# TODO: use log util, log to file, including time, module, etc.
def log_err(msg):
"""log an err message"""
print colored('ERR', 'red', attrs=['blink']), msg
# TODO: use log util, log to file, including time, module, etc.
def log_exc(exc):
"""log an unexpected exception"""
log_err('Caught unexpected exception: {}\n{}'.format(
exc, traceback.format_exc()))
if is_in_unittest():
log_api = log_fetcher = log_info = lambda msg: None
from ukutil import is_in_unittest
FRONTEND_PORT = 4999
API_PORT = 5000
API_RUN_OPTIONS = {
'debug': True,
'use_reloader': not is_in_unittest()
}
FRONTEND_RUN_OPTIONS = {
'debug': True
}