Esempio n. 1
0
def cleanup():
    """
    reindexes knowls, also the list of categories. prunes history.
    this is an internal task just for admins!
    """
    from knowl import refresh_knowl_categories, extract_cat, make_keywords, get_knowls
    cats = refresh_knowl_categories()
    knowls = get_knowls()
    q_knowls = knowls.find({}, ['content', 'title'])
    for k in q_knowls:
        kid = k['_id']
        cat = extract_cat(kid)
        search_keywords = make_keywords(k['content'], kid, k['title'])
        knowls.update({'_id': kid},
                      {"$set": {
                          'cat': cat,
                          '_keywords': search_keywords
                      }})

    hcount = 0
    # max allowed history length
    max_h = 50
    q_knowls = knowls.find({'history': {'$exists': True}}, ['history'])
    for k in q_knowls:
        if len(k['history']) <= max_h:
            continue
        hcount += 1
        knowls.update({'_id': k['_id']},
                      {'$set': {
                          'history': k['history'][-max_h:]
                      }})

    return "categories: %s <br/>reindexed %s knowls<br/>pruned %s histories" % (
        cats, q_knowls.count(), hcount)
Esempio n. 2
0
def cleanup():
    """
    reindexes knowls, also the list of categories. prunes history.
    this is an internal task just for admins!
    """
    from knowl import refresh_knowl_categories, extract_cat, make_keywords, get_knowls
    cats = refresh_knowl_categories()
    knowls = get_knowls()
    q_knowls = knowls.find({}, ['content', 'title'])
    for k in q_knowls:
        kid = k['_id']
        cat = extract_cat(kid)
        search_keywords = make_keywords(k['content'], kid, k['title'])
        knowls.update({'_id': kid},
                      {"$set": {
                       'cat': cat,
                       '_keywords': search_keywords
                       }})

    hcount = 0
    # max allowed history length
    max_h = 50
    q_knowls = knowls.find({'history': {'$exists': True}}, ['history'])
    for k in q_knowls:
        if len(k['history']) <= max_h:
            continue
        hcount += 1
        knowls.update({'_id': k['_id']}, {'$set': {'history': k['history'][-max_h:]}})

    return "categories: %s <br/>reindexed %s knowls<br/>pruned %s histories" % (cats, q_knowls.count(), hcount)
Esempio n. 3
0
def cleanup():
    """
    reindexes knowls, also the list of categories. prunes history.
    this is an internal task just for admins!
    """
    from knowl import refresh_knowl_categories, extract_cat, make_keywords, get_knowls

    cats = refresh_knowl_categories()
    knowls = get_knowls()
    q_knowls = knowls.find(fields=["content", "title"])
    for k in q_knowls:
        kid = k["_id"]
        cat = extract_cat(kid)
        search_keywords = make_keywords(k["content"], kid, k["title"])
        knowls.update({"_id": kid}, {"$set": {"cat": cat, "_keywords": search_keywords}})

    hcount = 0
    # max allowed history length
    max_h = 50
    q_knowls = knowls.find({"history": {"$exists": True}}, fields=["history"])
    for k in q_knowls:
        if len(k["history"]) <= max_h:
            continue
        hcount += 1
        knowls.update({"_id": k["_id"]}, {"$set": {"history": k["history"][-max_h:]}})

    return "categories: %s <br/>reindexed %s knowls<br/>pruned %s histories" % (cats, q_knowls.count(), hcount)
Esempio n. 4
0
def index():
    # bypassing the Knowl objects to speed things up
    from knowl import get_knowls
    # See issue #1169
    #    try:
    #        get_knowls().ensure_index('_keywords')
    #        get_knowls().ensure_index('cat')
    #    except pymongo.errors.OperationFailure:
    #        pass

    cur_cat = request.args.get("category", "")

    qualities = []
    defaults = "filter" not in request.args
    filtermode = "filter" in request.args
    searchmode = "search" in request.args
    categorymode = "category" in request.args

    from knowl import knowl_qualities
    # TODO wrap this into a loop:
    reviewed = request.args.get("reviewed", "") == "on" or defaults
    ok = request.args.get("ok", "") == "on" or defaults
    beta = request.args.get("beta", "") == "on" or defaults

    if reviewed:
        qualities.append("reviewed")
    if ok:
        qualities.append("ok")
    if beta:
        qualities.append("beta")

    s_query = {}

    if filtermode:
        quality_q = {'$in': qualities}
        s_query['quality'] = quality_q

    keyword = request.args.get("search", "").lower()
    if searchmode and keyword:
        keywords = filter(lambda _: len(_) >= 3, keyword.split(" "))
        # logger.debug("keywords: %s" % keywords)
        keyword_q = {'_keywords': {"$all": keywords}}
        s_query.update(keyword_q)

    if categorymode:
        s_query.update({'cat': cur_cat})  # { "$regex" : r"^%s\..+" % cur_cat }

    logger.debug("search query: %s" % s_query)
    knowls = get_knowls().find(s_query, ['title'])

    def first_char(k):
        t = k['title']
        if len(t) == 0:
            return "?"
        if t[0] not in string.ascii_letters:
            return "?"
        return t[0].upper()

    # way to additionally narrow down the search
    # def incl(knwl):
    #   if keyword in knwl['_id'].lower():   return True
    #   if keyword in knwl['title'].lower(): return True
    #   return False
    # if keyword: knowls = filter(incl, knowls)

    from knowl import get_categories
    cats = get_categories()

    def knowl_sort_key(knowl):
        '''sort knowls, special chars at the end'''
        title = knowl['title']
        if title and title[0] in string.ascii_letters:
            return (0, title.lower())
        else:
            return (1, title.lower())

    knowls = sorted(knowls, key=knowl_sort_key)
    from itertools import groupby
    knowls = groupby(knowls, first_char)
    return render_template("knowl-index.html",
                           title="Knowledge Database",
                           bread=get_bread(),
                           knowls=knowls,
                           search=keyword,
                           searchbox=searchbox(request.args.get("search", ""),
                                               searchmode),
                           knowl_qualities=knowl_qualities,
                           searchmode=searchmode,
                           filters=(beta, ok, reviewed),
                           categories=cats,
                           cur_cat=cur_cat,
                           categorymode=categorymode)
Esempio n. 5
0
def index():
  # bypassing the Knowl objects to speed things up
  from knowl import get_knowls
  get_knowls().ensure_index('_keywords')
  keyword = request.args.get("search", "").lower()
  keywords = filter(lambda _:len(_) >= 3, keyword.split(" "))
  #logger.debug("keywords: %s" % keywords)
  keyword_q = {'_keywords' : { "$all" : keywords}}

  cur_cat = request.args.get("category", "")
  
  qualities = []
  defaults = "filter" not in request.args
  searchmode = "search" in request.args
  categorymode = "category" in request.args

  from knowl import knowl_qualities
  # TODO wrap this into a loop:
  reviewed = request.args.get("reviewed", "") == "on" or defaults or searchmode
  ok       = request.args.get("ok", "") == "on"       or defaults or searchmode
  beta     = request.args.get("beta", "") == "on"     or defaults or searchmode

  if reviewed: qualities.append("reviewed")
  if ok:       qualities.append("ok")
  if beta:     qualities.append("beta")

  quality_q = { '$in' : qualities }

  s_query = {}
  s_query['title'] = { "$exists" : True }
  if searchmode:
    s_query['quality'] = quality_q

  if categorymode:
    s_query['_id'] = { "$regex" : r"^%s\..+" % cur_cat }

  logger.debug("search query: %s" % s_query)
  knowls = get_knowls().find(s_query, fields=['title'])

  def first_char(k):
    t = k['title']
    if len(t) == 0: return "?"
    if t[0] not in string.ascii_letters: return "?"
    return t[0].upper()

  # way to additionally narrow down the search
  # def incl(knwl):
  #   if keyword in knwl['_id'].lower():   return True
  #   if keyword in knwl['title'].lower(): return True
  #   return False
  # if keyword: knowls = filter(incl, knowls)
  
  from knowl import get_categories 
  cats = get_categories()

  knowls = sorted(knowls, key = lambda x : x['title'].lower())
  from itertools import groupby
  knowls = groupby(knowls, first_char)
  return render_template("knowl-index.html", 
         title  = "Knowledge Database",
         bread  = get_bread(),
         knowls = knowls,
         search = keyword,
         navi_raw = searchbox(request.args.get("search", ""), searchmode),
         knowl_qualities = knowl_qualities,
         searchmode = searchmode,
         filters = (beta, ok, reviewed),
         categories = cats,
         cur_cat = cur_cat,
         categorymode = categorymode)
Esempio n. 6
0
def index():
    # bypassing the Knowl objects to speed things up
    from knowl import get_knowls
    get_knowls().ensure_index('_keywords')
    keyword = request.args.get("search", "").lower()
    keywords = filter(lambda _: len(_) >= 3, keyword.split(" "))
    #logger.debug("keywords: %s" % keywords)
    keyword_q = {'_keywords': {"$all": keywords}}

    cur_cat = request.args.get("category", "")

    qualities = []
    defaults = "filter" not in request.args
    searchmode = "search" in request.args
    categorymode = "category" in request.args

    from knowl import knowl_qualities
    # TODO wrap this into a loop:
    reviewed = request.args.get("reviewed",
                                "") == "on" or defaults or searchmode
    ok = request.args.get("ok", "") == "on" or defaults or searchmode
    beta = request.args.get("beta", "") == "on" or defaults or searchmode

    if reviewed: qualities.append("reviewed")
    if ok: qualities.append("ok")
    if beta: qualities.append("beta")

    quality_q = {'$in': qualities}

    s_query = {}
    s_query['title'] = {"$exists": True}
    if searchmode:
        s_query['quality'] = quality_q

    if categorymode:
        s_query['_id'] = {"$regex": r"^%s\..+" % cur_cat}

    logger.debug("search query: %s" % s_query)
    knowls = get_knowls().find(s_query, fields=['title'])

    def first_char(k):
        t = k['title']
        if len(t) == 0: return "?"
        if t[0] not in string.ascii_letters: return "?"
        return t[0].upper()

    # way to additionally narrow down the search
    # def incl(knwl):
    #   if keyword in knwl['_id'].lower():   return True
    #   if keyword in knwl['title'].lower(): return True
    #   return False
    # if keyword: knowls = filter(incl, knowls)

    from knowl import get_categories
    cats = get_categories()

    knowls = sorted(knowls, key=lambda x: x['title'].lower())
    from itertools import groupby
    knowls = groupby(knowls, first_char)
    return render_template("knowl-index.html",
                           title="Knowledge Database",
                           bread=get_bread(),
                           knowls=knowls,
                           search=keyword,
                           navi_raw=searchbox(request.args.get("search", ""),
                                              searchmode),
                           knowl_qualities=knowl_qualities,
                           searchmode=searchmode,
                           filters=(beta, ok, reviewed),
                           categories=cats,
                           cur_cat=cur_cat,
                           categorymode=categorymode)
Esempio n. 7
0
def index():
    # bypassing the Knowl objects to speed things up
    from knowl import get_knowls
    try:
        get_knowls().ensure_index('_keywords')
        get_knowls().ensure_index('cat')
    except pymongo.errors.OperationFailure:
        pass

    cur_cat = request.args.get("category", "")

    qualities = []
    defaults = "filter" not in request.args
    filtermode = "filter" in request.args
    searchmode = "search" in request.args
    categorymode = "category" in request.args

    from knowl import knowl_qualities
    # TODO wrap this into a loop:
    reviewed = request.args.get("reviewed", "") == "on" or defaults
    ok = request.args.get("ok", "") == "on" or defaults
    beta = request.args.get("beta", "") == "on" or defaults

    if reviewed:
        qualities.append("reviewed")
    if ok:
        qualities.append("ok")
    if beta:
        qualities.append("beta")

    s_query = {}

    if filtermode:
        quality_q = {'$in': qualities}
        s_query['quality'] = quality_q

    keyword = request.args.get("search", "").lower()
    if searchmode and keyword:
        keywords = filter(lambda _: len(_) >= 3, keyword.split(" "))
        # logger.debug("keywords: %s" % keywords)
        keyword_q = {'_keywords': {"$all": keywords}}
        s_query.update(keyword_q)

    if categorymode:
        s_query.update({'cat': cur_cat})  # { "$regex" : r"^%s\..+" % cur_cat }

    logger.debug("search query: %s" % s_query)
    knowls = get_knowls().find(s_query, ['title'])

    def first_char(k):
        t = k['title']
        if len(t) == 0:
            return "?"
        if t[0] not in string.ascii_letters:
            return "?"
        return t[0].upper()

    # way to additionally narrow down the search
    # def incl(knwl):
    #   if keyword in knwl['_id'].lower():   return True
    #   if keyword in knwl['title'].lower(): return True
    #   return False
    # if keyword: knowls = filter(incl, knowls)

    from knowl import get_categories
    cats = get_categories()

    def knowl_sort_key(knowl):
        '''sort knowls, special chars at the end'''
        title = knowl['title']
        if title and title[0] in string.ascii_letters:
            return (0, title.lower())
        else:
            return (1, title.lower())

    knowls = sorted(knowls, key=knowl_sort_key)
    from itertools import groupby
    knowls = groupby(knowls, first_char)
    return render_template("knowl-index.html",
                           title="Knowledge Database",
                           bread=get_bread(),
                           knowls=knowls,
                           search=keyword,
                           searchbox=searchbox(request.args.get("search", ""), searchmode),
                           knowl_qualities=knowl_qualities,
                           searchmode=searchmode,
                           filters=(beta, ok, reviewed),
                           categories = cats,
                           cur_cat = cur_cat,
                           categorymode = categorymode)