コード例 #1
0
def kb_add(req, ln=CFG_SITE_LANG, sortby="to", kbtype=""):
    """
    Adds a new kb
    @param req the request
    @param ln language
    @param sortby to or from
    @param kbtype type of knowledge base. one of: "", taxonomy, dynamic
    """
    ln = wash_language(ln)
    _ = gettext_set_language(ln)

    navtrail_previous_links = ''' &gt; <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases"))

    try:
        dummy = getUid(req)
    except:
        return error_page('Error', req)

    (auth_code, auth_msg) = check_user(req, 'cfgbibknowledge')
    if not auth_code:
        name = "Untitled"
        if kbtype == "taxonomy":
            name = "Untitled Taxonomy"
        if kbtype == "dynamic":
            name = "Untitled dynamic"
        kb_id = bibknowledge.add_kb(kb_name=name, kb_type=kbtype)
        redirect_to_url(req, "kb?ln=%(ln)s&amp;action=attributes&amp;kb=%(kb)s" % {'ln':ln, 'kb':kb_id, 'sortby':sortby})
    else:
        navtrail_previous_links = ''' &gt; <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases"))

        return page_not_authorized(req=req,
                                   text=auth_msg,
                                   navtrail=navtrail_previous_links)
コード例 #2
0
def kb_add(req, ln=CFG_SITE_LANG, sortby="to", kbtype=""):
    """
    Adds a new kb
    @param req the request
    @param ln language
    @param sortby to or from
    @param kbtype type of knowledge base. one of: "", taxonomy, dynamic
    """
    ln = wash_language(ln)
    _ = gettext_set_language(ln)

    navtrail_previous_links = ''' &gt; <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases"))

    try:
        dummy = getUid(req)
    except:
        return error_page('Error', req)

    (auth_code, auth_msg) = check_user(req, 'cfgbibknowledge')
    if not auth_code:
        name = "Untitled"
        if kbtype == "taxonomy":
            name = "Untitled Taxonomy"
        if kbtype == "dynamic":
            name = "Untitled dynamic"
        kb_id = bibknowledge.add_kb(kb_name=name, kb_type=kbtype)
        redirect_to_url(req, "kb?ln=%(ln)s&amp;action=attributes&amp;kb=%(kb)s" % {'ln':ln, 'kb':kb_id, 'sortby':sortby})
    else:
        navtrail_previous_links = ''' &gt; <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases"))

        return page_not_authorized(req=req,
                                   text=auth_msg,
                                   navtrail=navtrail_previous_links)
コード例 #3
0
 def test_taxonomy(self):
     """bibknowledge - test a taxonomy (must run as bibsched user)"""
     import mechanize
     from os import remove
     from invenio.bibknowledge import get_kbt_items_for_bibedit, add_kb, \
         get_kb_name, delete_kb, kb_exists
     username = "******"
     password = "******"
     #create a new taxonomy kb
     new_kb_id = add_kb("testtaxonomy", "taxonomy")
     #what was the name?
     new_kb_name = get_kb_name(new_kb_id)
     #get the taxonomy file
     response = mechanize.urlopen(
         "http://invenio-software.org/download/invenio-demo-site-files/HEP.rdf"
     )
     content = response.read()
     f = open(CFG_TMPDIR + "/HEP.rdf", "w")
     f.write(content)
     f.close()
     #upload it to the right destination, but log in first
     browser = mechanize.Browser()
     browser.open(CFG_SITE_SECURE_URL + "/youraccount/login")
     browser.select_form(nr=0)
     browser['nickname'] = username
     browser['password'] = password
     browser.submit()
     #go to upload page
     uploadpage = browser.open(CFG_SITE_URL + "/kb?kb=" + str(new_kb_id))
     #check that we are there
     content = uploadpage.read()
     namethere = content.count("testtaxonomy")
     assert namethere > 0
     #upload
     browser.open(CFG_SITE_URL + "/kb?kb=" + str(new_kb_id))
     browser.select_form(name="upload")
     browser.form["kb"] = str(new_kb_id)  #force the id
     browser.form.add_file(open(CFG_TMPDIR + "/HEP.rdf"),
                           content_type='text/plain',
                           filename="HEP.rdf",
                           name="file")
     browser.submit()
     #check that we can get an item from the kb
     items = get_kbt_items_for_bibedit(new_kb_name, "prefLabel",
                                       "Altarelli")
     #item should contain 1 string: 'Altarelli-Parisi equation'
     self.assertEqual(1, len(items))
     #delete the temp file
     remove(CFG_TMPDIR + "/HEP.rdf")
     #delete the test odf the DB
     delete_kb(new_kb_name)
     still_there = kb_exists(new_kb_name)
     self.assertEqual(False, still_there)
コード例 #4
0
 def test_add_get_remove(self):
     """bibknowledge - test creating a kb, adding a mapping, removing it, removing kb"""
     new_kb_id = add_kb()
     new_name = get_kb_name(new_kb_id)
     add_kb_mapping(new_name, "foobar", "barfoo")
     fbexists = kb_mapping_exists(new_name, "foobar")
     self.assertEqual(True, fbexists)
     remove_kb_mapping(new_name, "foobar")
     fbexists = kb_mapping_exists(new_name, "foobar")
     self.assertEqual(False, fbexists)
     delete_kb(new_name)
     still_there = kb_exists(new_name)
     self.assertEqual(False, still_there)
コード例 #5
0
 def test_add_get_remove(self):
     """bibknowledge - test creating a kb, adding a mapping, removing it, removing kb"""
     new_kb_id = add_kb()
     new_name = get_kb_name(new_kb_id)
     add_kb_mapping(new_name, "foobar", "barfoo")
     fbexists = kb_mapping_exists(new_name, "foobar")
     self.assertEqual(True, fbexists)
     remove_kb_mapping(new_name, "foobar")
     fbexists = kb_mapping_exists(new_name, "foobar")
     self.assertEqual(False, fbexists)
     delete_kb(new_name)
     still_there = kb_exists(new_name)
     self.assertEqual(False, still_there)
コード例 #6
0
 def test_taxonomy(self):
     """bibknowledge - test a taxonomy (must run as bibsched user)"""
     import mechanize
     from os import remove
     username = "******"
     password = "******"
     #create a new taxonomy kb
     new_kb_id = add_kb("testtaxonomy","taxonomy")
     #what was the name?
     new_kb_name = get_kb_name(new_kb_id)
     #get the taxonomy file
     response = mechanize.urlopen("http://invenio-software.org/download/invenio-demo-site-files/HEP.rdf")
     content = response.read()
     f = open(CFG_TMPDIR+"/HEP.rdf","w")
     f.write(content)
     f.close()
     #upload it to the right destination, but log in first
     browser = mechanize.Browser()
     browser.open(CFG_SITE_SECURE_URL + "/youraccount/login")
     browser.select_form(nr=0)
     browser['p_un'] = username
     browser['p_pw'] = password
     browser.submit()
     #go to upload page
     uploadpage = browser.open(CFG_SITE_URL+"/kb?kb="+str(new_kb_id))
     #check that we are there
     content = uploadpage.read()
     namethere = content.count("testtaxonomy")
     assert namethere > 0
     #upload
     browser.open(CFG_SITE_URL+"/kb?kb="+str(new_kb_id))
     browser.select_form(name="upload")
     browser.form["kb"] = str(new_kb_id) #force the id
     browser.form.add_file(open(CFG_TMPDIR+"/HEP.rdf"), content_type='text/plain', filename="HEP.rdf", name="file")
     browser.submit()
     #check that we can get an item from the kb
     items = get_kbt_items_for_bibedit(new_kb_name, "prefLabel", "Altarelli")
     #item should contain 1 string: 'Altarelli-Parisi equation'
     self.assertEqual(1, len(items))
     #delete the temp file
     remove(CFG_TMPDIR+"/HEP.rdf")
     #delete the test odf the DB
     delete_kb(new_kb_name)
     still_there = kb_exists(new_kb_name)
     self.assertEqual(False, still_there)
コード例 #7
0
def load_kbs(cfg, run_sql, in_task=False):
    for kb, query in cfg.iteritems():
        task_sleep_now_if_required(can_stop_too=True)
        if not kb_exists(kb):
            add_kb(kb)
        if in_task:
            write_message("Updating %s KB..." % kb)
        try:
            if not in_task:
                print "kb:", kb
                print "kb beginning:", len(get_kb_mappings(kb))
            if kb.startswith('json_'):
                encoder = ComplexEncoder()
                mapping, description = run_sql(query, with_desc=True)
                if kb in CFG_ADDITIONAL_ENTRIES:
                    mapping += CFG_ADDITIONAL_ENTRIES[kb]
                    if not in_task:
                        print CFG_ADDITIONAL_ENTRIES[kb]
                column_counter = {}
                new_description = []
                for column in description[1:]:
                    column = column[0]
                    counter = column_counter[column] = column_counter.get(
                        column, 0) + 1
                    if counter > 1:
                        new_description.append('%s%d' % (column, counter))
                    else:
                        new_description.append(column)
                description = new_description
            else:
                mapping = run_sql(query)
                if kb in CFG_ADDITIONAL_ENTRIES:
                    mapping += CFG_ADDITIONAL_ENTRIES[kb]
                    if not in_task:
                        print CFG_ADDITIONAL_ENTRIES[kb]
                if not in_task:
                    print "mapping:", len(mapping)
                if kb == 'projects':
                    mapping += [('000000', 'NO PROJECT')]
            original_keys = set([key[0] for key in get_kbr_keys(kb)])
            if not in_task:
                print "original_keys before:", len(original_keys)

            updated = 0
            added = 0
            for i, row in enumerate(mapping):
                key, value = row[0], row[1:]
                if kb.startswith('json_'):
                    value = encoder.encode(dict(zip(description, value)))
                else:
                    value = value[0]
                if value:
                    if key in original_keys:
                        original_keys.remove(key)
                    if in_task:
                        task_update_progress("%s - %s%%" %
                                             (kb, i * 100 / len(mapping)))
                    if kb_mapping_exists(kb, key):
                        updated += 1
                        update_kb_mapping(kb, key, key, value)
                    else:
                        added += 1
                        add_kb_mapping(kb, key, value)
            if not in_task:
                print "updated:", updated, "added:", added
                print "kb after update:", len(get_kb_mappings(kb))
                print "original_keys after:", len(original_keys)
            if in_task:
                task_update_progress("Cleaning %s" % kb)
            for key in original_keys:
                remove_kb_mapping(kb, key)
            if not in_task:
                print "kb after remove:", len(get_kb_mappings(kb))
        except:
            register_exception(alert_admin=True,
                               prefix="Error when updating KB %s" % kb)
            continue
コード例 #8
0
def load_kbs(cfg, run_sql, in_task=False):
    for kb, query in cfg.iteritems():
        task_sleep_now_if_required(can_stop_too=True)
        if not kb_exists(kb):
            add_kb(kb)
        if in_task:
            write_message("Updating %s KB..." % kb)
        try:
            if not in_task:
                print "kb:", kb
                print "kb beginning:", len(get_kb_mappings(kb))
            if kb.startswith('json_'):
                encoder = ComplexEncoder()
                mapping, description = run_sql(query, with_desc=True)
                if kb in CFG_ADDITIONAL_ENTRIES:
                    mapping += CFG_ADDITIONAL_ENTRIES[kb]
                    if not in_task:
                        print CFG_ADDITIONAL_ENTRIES[kb]
                column_counter = {}
                new_description = []
                for column in description[1:]:
                    column = column[0]
                    counter = column_counter[column] = column_counter.get(column, 0) + 1
                    if counter > 1:
                        new_description.append('%s%d' % (column, counter))
                    else:
                        new_description.append(column)
                description = new_description
            else:
                mapping = run_sql(query)
                if kb in CFG_ADDITIONAL_ENTRIES:
                    mapping += CFG_ADDITIONAL_ENTRIES[kb]
                    if not in_task:
                        print CFG_ADDITIONAL_ENTRIES[kb]
                if not in_task:
                    print "mapping:", len(mapping)
                if kb == 'projects':
                    mapping += [('000000', 'NO PROJECT')]
            original_keys = set([key[0] for key in get_kbr_keys(kb)])
            if not in_task:
                print "original_keys before:", len(original_keys)

            updated = 0
            added = 0
            for i, row in enumerate(mapping):
                key, value = row[0], row[1:]
                if kb.startswith('json_'):
                    value = encoder.encode(dict(zip(description, value)))
                else:
                    value = value[0]
                if value:
                    if key in original_keys:
                        original_keys.remove(key)
                    if in_task:
                        task_update_progress("%s - %s%%" % (kb, i * 100 / len(mapping)))
                    if kb_mapping_exists(kb, key):
                        updated += 1
                        update_kb_mapping(kb, key, key, value)
                    else:
                        added += 1
                        add_kb_mapping(kb, key, value)
            if not in_task:
                print "updated:", updated, "added:", added
                print "kb after update:", len(get_kb_mappings(kb))
                print "original_keys after:", len(original_keys)
            if in_task:
                task_update_progress("Cleaning %s" % kb)
            for key in original_keys:
                remove_kb_mapping(kb, key)
            if not in_task:
                print "kb after remove:", len(get_kb_mappings(kb))
        except:
            register_exception(alert_admin=True, prefix="Error when updating KB %s" % kb)
            continue