コード例 #1
0
ファイル: tag.py プロジェクト: blxlrsmb/uknow
def set_tag():
    """
    {
     name: ['tagname'],
     tab: 'tabname'
    }
    """
    try:
        data = json.loads(request.data)
        tags = data['name']
        tabname = data['tab']
        assert isinstance(tabname, basestring)
        assert isinstance(tags, list)
        for tag in tags:
            assert isinstance(tag, basestring)
    except:
        return {'error': 'illegal format'}
    u = get_user(current_user.username)
    for tab in u['tab']:
        if tabname == tab['name']:
            tab['tags'] = tags
            get_mongo('user').save(u)
            log_info('user {0} set tag to {1} on tab \
                     {2}'.format(current_user.username, tags, tabname))
            return {'tabs': u['tab']}
    return {'error': 'no such tab'}
コード例 #2
0
ファイル: tab.py プロジェクト: blxlrsmb/uknow
def add_tab():
    """add a new tab to current user
    {
        name: 'string'      # name of the tab
        priority: integer   # bigger, better. default to be 0
    }
    """
    try:
        data = json.loads(request.data)
        name = data['name']
        priority = int(data.get('priority', 0))
        assert isinstance(name, basestring)
    except:
        return {'error': 'illegal format'}
    username = current_user.username
    doc = get_user(username)
    for tab in doc['tab']:
        if tab['name'] == name:
            return {'error': 'tab with this name already exists!'}
    doc['tab'].append({
        'name': name,
        'priority': priority,
        'tags': []
    })
    get_mongo('user').save(doc)
    log_info('user {0} add tab {1}'.format(username, name))
    return {'success': 1}
コード例 #3
0
ファイル: tag.py プロジェクト: blxlrsmb/uknow
def del_tag():
    """
    GET /del_tag?name=xxx&tab=xxx
    """
    try:
        data = request.args
        tagname = data['name']
        tabname = data['tab']
        assert isinstance(tagname, basestring) \
            and isinstance(tabname, basestring)
    except:
        return {'error': 'illegal format'}
    u = get_user(current_user.username)
    for tab in u['tab']:
        if tabname == tab['name']:
            try:
                tab['tags'].remove(tagname)
            except ValueError:
                return {'error': 'tag {0} not in tab {1}'.format(tagname,
                                                                 tabname)}
            get_mongo('user').save(u)
            log_info('user {0} del tag {1} in tab \
                     {2}'.format(current_user.username, tagname, tabname))
            return {'tabs': u['tab']}
    return {'error': 'no such tab'}
コード例 #4
0
ファイル: __init__.py プロジェクト: blxlrsmb/uknow
def start_server():
    """start the server for all general fetchers"""
    import_all_modules(__file__, __name__)

    if not register_fetcher.fetcher_list:
        uklogger.log_err('no general fetcher available, exit')
        return

    def worker(fetcher):
        try:
            while True:
                uklogger.log_info('run fetcher {}'.format(
                    fetcher.fetcher_name))
                fetcher.run()
                gevent.sleep(fetcher.sleep_time)
        except KeyboardInterrupt:
            uklogger.log_info('worker {} got KeyboardInterrupt, exit'.format(
                fetcher.fetcher_name))

    jobs = [gevent.spawn(worker, i) for i in register_fetcher.fetcher_list]
    try:
        for i in jobs:
            i.join()
    except KeyboardInterrupt:
        uklogger.log_info('got KeyboardInterrupt, exit')
コード例 #5
0
ファイル: user.py プロジェクト: blxlrsmb/uknow
def register():
    """user registration api.
        username: string
        password: string        XXX TODO
    """
    try:
        username = request.values['username']
        password = request.values['password']
        assert isinstance(username, basestring)
        assert isinstance(password, basestring)
    except:
        return {'error': 'illegal format'}
    if len(username) < 3 or len(password) < 3:
        return {'error':
                'length of username and password must' +
                ' be at least 3 characters'}

    exist = get_user(username)
    if exist:
        return {"error": "user {0} already exists".format(username)}

    db = get_mongo('user')
    db.insert({
        'username': username,
        'password': password,
        'tab': []
    })
    log_info('new user: {0}:{1}'.format(username, password))
    return {'success': 1}
コード例 #6
0
ファイル: tag.py プロジェクト: blxlrsmb/uknow
def add_tag():
    """
    {
     name: 'tagname',
     tab: 'tabname'
    }
    """
    try:
        data = json.loads(request.data)
        tagname = data['name']
        tabname = data['tab']
        assert isinstance(tagname, basestring) \
            and isinstance(tabname, basestring)
    except:
        return {'error': 'illegal format'}
    u = get_user(current_user.username)
    for tab in u['tab']:
        if tabname == tab['name']:
            l = tab['tags']
            l.append(tagname)
            l = list(set(l))
            tab['tags'] = l
            get_mongo('user').save(u)
            log_info('user {0} add tag {1} to tab \
                     {2}'.format(current_user.username, tagname, tabname))
            return {'tabs': u['tab']}
    return {'error': 'no such tab'}
コード例 #7
0
ファイル: __init__.py プロジェクト: blxlrsmb/uknow
 def worker(fetcher):
     try:
         while True:
             uklogger.log_info('run fetcher {}'.format(
                 fetcher.fetcher_name))
             fetcher.run()
             gevent.sleep(fetcher.sleep_time)
     except KeyboardInterrupt:
         uklogger.log_info('worker {} got KeyboardInterrupt, exit'.format(
             fetcher.fetcher_name))
コード例 #8
0
ファイル: xkcd.py プロジェクト: blxlrsmb/uknow
def xkcd_rss_fetcher(ctx):
    """fetcher xkcd.com/rss.xml, save each title with tag `xkcd`"""
    URL = 'http://www.xkcd.com/rss.xml'
    coll = ctx.get_mongo_collection()

    for entry in fetch_rss(URL).entries:
        try:
            coll.insert({'_id': entry.id})
        except DuplicateKeyError:
            continue
        ctx.new_item(TextOnlyItem(entry.title, entry.description),
                     ['xkcd'], parse_entry_time(entry),
                     {'id': entry.id})
        log_info(u'xkcd rss: new entry: {} {}'.format(entry.id,
                                                      entry.title))
コード例 #9
0
ファイル: __init__.py プロジェクト: blxlrsmb/uknow
 def on_user_activated(user_id):
     try:
         user_fetcher = get_db_set(user_id, 'fetcher')
         for i in user_fetcher:
             fetcher = register_fetcher.fetcher_map.get(i)
             if fetcher is None:
                 uklogger.log_err(
                     'fetcher {} not exist, requested by user {}'.format(
                         i, user_id))
             else:
                 uklogger.log_info('run fetcher {} for user {}'.format(
                     i, user_id))
                 fetcher.run(user_id)
     except Exception as ex:
         uklogger.log_exc(ex)
コード例 #10
0
ファイル: zhihu.py プロジェクト: blxlrsmb/uknow
def zhihu_rss_fetcher(ctx):
    """fetcher zhihu.com/rss/, save each title with tag `zhihu`"""
    URL = 'http://www.zhihu.com/rss'
    coll = ctx.get_mongo_collection()

    for entry in fetch_rss(URL).entries:
        try:
            coll.insert({'_id': entry.link})
        except DuplicateKeyError:
            continue
        ctx.new_item(TextOnlyItem(entry.title, entry.description), ['zhihu'],
                     parse_entry_time(entry),
                     {'id': entry.link})
        log_info(u'zhihu: new entry: {} {}'.format(entry.link,
                                                   entry.title))
コード例 #11
0
ファイル: stackoverflow.py プロジェクト: blxlrsmb/uknow
def stackoverflow_rss_fetcher(ctx):
    """fetcher stackoverflow.com/feeds,
    save each title with tag `stackoverflow`"""
    URL = 'http://stackoverflow.com/feeds'
    coll = ctx.get_mongo_collection()

    for entry in fetch_rss(URL).entries:
        try:
            coll.insert({'_id': entry.id})
        except DuplicateKeyError:
            continue
        ctx.new_item(TextOnlyItem(entry.title, entry.summary),
                     ['Stackoverflow'], parse_entry_time(entry),
                     {'id': entry.id})
        log_info(u'stackoverflow: new entry: {} {}'.format(entry.id,
                                                           entry.title))
コード例 #12
0
ファイル: user.py プロジェクト: blxlrsmb/uknow
def login():
    """login api"""
    try:
        username = request.values['username']
        password = request.values['password']
        assert isinstance(username, basestring) \
            and isinstance(password, basestring)
    except Exception as e:
        return {'error': 'illegal login format!' + str(e)}

    auth = User(username, password)
    err = auth.get_error()
    if err:
        return err
    login_user(auth)
    log_info('user {0} succesfully logged in'.format(username))
    return {'success': 1}
コード例 #13
0
ファイル: quora.py プロジェクト: blxlrsmb/uknow
def quora_rss_fetcher(ctx):
    """fetcher quora.com/rss/, save each title with tag `quora`"""
    URL = 'https://www.quora.com/rss'
    coll = ctx.get_mongo_collection()

    for entry in fetch_rss(URL).entries:
        try:
            coll.insert({'_id': entry.id})
        except DuplicateKeyError:
            continue
        try:
            content = entry.content[0].value
        except:
            content = entry.summary
        ctx.new_item(TextOnlyItem(entry.title, content), ['Quora'],
                     parse_entry_time(entry), {'id': entry.id})
        log_info(u'quora rss: new entry: {} {}'.format(entry.id, entry.title))
コード例 #14
0
ファイル: tsinghua.py プロジェクト: blxlrsmb/uknow
def tsinghua_portal_fetcher(ctx):
    """fetcher portal.tsinghua.edu.cn, save each title with tag `portal`"""
    coll = ctx.get_mongo_collection()

    for entry in Tsinghua.getInfoNotices():
        try:
            coll.insert({'_id': entry['link']})
        except DuplicateKeyError:
            continue
        ctx.new_item(TextOnlyItem(entry['title'], ""), ['Tsinghua info'],
                     other={"id": entry['link']})
        log_info(u'Tsinghua Info: new entry: {} {}'.format(entry['link'],
                                                           entry['title']))
    for entry in Tsinghua.getLibNotices():
        try:
            coll.insert({'_id': entry['link']})
        except DuplicateKeyError:
            continue
        ctx.new_item(TextOnlyItem(entry['title'], ""), ['Tsinghua library'],
                     other={"id": entry['link']})
        log_info(u'Tsinghua Library: new entry: {} {}'.format(entry['link'],
                                                              entry['title']))
コード例 #15
0
ファイル: thu_learn.py プロジェクト: blxlrsmb/uknow
    def run(cls, ctx):
        conf = cls.load_config(ctx.user_id)
        if not conf:
            ctx.new_item(TextOnlyItem(u'网络学堂验证失败', ''), ['THU learn'])
            return

        coll = ctx.get_mongo_collection()
        if is_in_unittest():
            entries = [{'id': 'test-{}'.format(uuid.uuid4()),
                        'title': 'thu learn in testcase',
                        'content': '{}@{}'.format(conf['username'],
                                                  conf['password']),
                        'create_time': '2013-12-14'}]
        else:
            try:
                entries = fetch(conf['username'], conf['password'])
                print entries
            except Exception as e:
                print e
                ctx.new_item(TextOnlyItem(u'网络学堂抓取失败:' +
                                          str(e), ''),
                             ['THU learn'])
                log_exc(e)
                return

        for entry in entries:
            try:
                coll.insert({'_id': str(ctx.user_id) + entry['id']})
            except DuplicateKeyError:
                continue
            ctx.new_item(
                TextOnlyItem(entry['title'], entry['content']),
                ['THU learn'], time.strptime(entry['create_time'], '%Y-%m-%d'),
                {'id': entry['id']})
            log_info(u'THU learn: new entry: {} {}'.format(entry['id'],
                                                           entry['title']))
コード例 #16
0
ファイル: autotagging.py プロジェクト: blxlrsmb/uknow
def auto_tagging(ctx, doc):
    """auto tagging an item.
        It will load tagger model from `ukconfig.tagger_path`.
        Model should be trained prior to make this function work"""
    global _tagger
    if _tagger is None:
        try:
            log_info('loading tagger ...')
            _tagger = TextTagger.load(ukconfig.tagger_path)
        except IOError:
            log_info('tagger model not found.')
            return

    tags = _tagger.predict_one(doc['desc'].render_content())
    declare_tag(tags)
    log_info('original tag: ' + str(doc['tag']))
    log_info('autotagging: ' + str(tags))
    doc['tag'] = list(set(doc['tag'] + tags))
    """auto tag """
コード例 #17
0
ファイル: user.py プロジェクト: blxlrsmb/uknow
def logout():
    """ logout api"""
    log_info('user {0} logged out'.format(current_user.username))
    logout_user()
    return {'success': 1}
コード例 #18
0
ファイル: bitcoin.py プロジェクト: blxlrsmb/uknow
def bitcoin_fetcher(ctx):
    """fetcher btcchina.com, save with tag `bitcoin`"""
    price = getPrice()
    ctx.new_item(TextOnlyItem("Last Bitcoin Price: " +
                              u"¥" + str(price), ""), ['Bitcoin'])
    log_info(u'bitcoin price update: {}'.format(price))