Пример #1
0
def get_celery_task():
    """get celery task, which takes user id as its sole argument"""
    global _celery_app
    global _celery_task
    if _celery_task:
        return _celery_task
    load_all_fetcher()
    _celery_app = Celery('ukfetcher', broker=ukconfig.celery_broker)
    _celery_app.conf.update(
        CELERY_ACCEPT_CONTENT=['pickle', 'json', 'msgpack', 'yaml'])

    @_celery_app.task
    def on_user_activated(user_id):
        try:
            user_fetcher = get_db_set(user_id, 'fetcher')
            for i in user_fetcher:
                fetcher = register_fetcher.fetcher_map.get(i)
                if fetcher is None:
                    uklogger.log_err(
                        'fetcher {} not exist, requested by user {}'.format(
                            i, user_id))
                else:
                    uklogger.log_info('run fetcher {} for user {}'.format(
                        i, user_id))
                    fetcher.run(user_id)
        except Exception as ex:
            uklogger.log_exc(ex)

    if is_in_unittest():
        _celery_task = on_user_activated
    else:
        _celery_task = on_user_activated.delay
    return _celery_task
Пример #2
0
    def run(cls, ctx):
        conf = cls.load_config(ctx.user_id)
        if not conf:
            ctx.new_item(TextOnlyItem(u'网络学堂验证失败', ''), ['THU learn'])
            return

        coll = ctx.get_mongo_collection()
        if is_in_unittest():
            entries = [{'id': 'test-{}'.format(uuid.uuid4()),
                        'title': 'thu learn in testcase',
                        'content': '{}@{}'.format(conf['username'],
                                                  conf['password']),
                        'create_time': '2013-12-14'}]
        else:
            try:
                entries = fetch(conf['username'], conf['password'])
                print entries
            except Exception as e:
                print e
                ctx.new_item(TextOnlyItem(u'网络学堂抓取失败:' +
                                          str(e), ''),
                             ['THU learn'])
                log_exc(e)
                return

        for entry in entries:
            try:
                coll.insert({'_id': str(ctx.user_id) + entry['id']})
            except DuplicateKeyError:
                continue
            ctx.new_item(
                TextOnlyItem(entry['title'], entry['content']),
                ['THU learn'], time.strptime(entry['create_time'], '%Y-%m-%d'),
                {'id': entry['id']})
            log_info(u'THU learn: new entry: {} {}'.format(entry['id'],
                                                           entry['title']))
Пример #3
0
    """log a message from api-website"""
    print colored('API', 'green'), msg
    # TODO: use log util, log to file, including time, module, etc.


def log_fetcher(msg):
    """log a message from fetcher"""
    print colored('FETCHER', 'yellow'), msg
    # TODO: use log util, log to file, including time, module, etc.


def log_info(msg):
    """log an info message"""
    print colored('INFO', 'blue'), msg
    # TODO: use log util, log to file, including time, module, etc.


def log_err(msg):
    """log an err message"""
    print colored('ERR', 'red', attrs=['blink']), msg
    # TODO: use log util, log to file, including time, module, etc.


def log_exc(exc):
    """log an unexpected exception"""
    log_err('Caught unexpected exception: {}\n{}'.format(
        exc, traceback.format_exc()))

if is_in_unittest():
    log_api = log_fetcher = log_info = lambda msg: None
Пример #4
0
from ukutil import is_in_unittest

FRONTEND_PORT = 4999
API_PORT = 5000

API_RUN_OPTIONS = {
    'debug': True,
    'use_reloader': not is_in_unittest()
}

FRONTEND_RUN_OPTIONS = {
    'debug': True
}