def _get_json_dump_of_codens(): """Dump codens to a global variable where our javascript can get it. In our BFT we will want to have an onSubmit() handler which substitutes every short title for a coden; this makes the data for that available""" return '\n<script type="text/javascript">gCODENS = %s</script>\n' % simplejson.dumps( dict([(x['value'], x['key']) for x in get_kb_mappings('CODENS')]))
def format_element(bfo, separator=' | ', link="yes"): """ Prints PACS info as best is possible. @param link if yes (default) prints link to search for this item in Inspire @param separator separates multiple items """ fields = bfo.fields('084__') output = [] output1 = [] pacs_count = 0 link = "" for item in fields: if item.get('2') == 'PACS': pacs_code = item.get('a') if pacs_code: pacs_kb_mapping = get_kb_mappings('PACS', key=pacs_code, match_type="e") title = 'Translation not available' if pacs_kb_mapping: title = pacs_kb_mapping[0]['value'] search_link = "<a href='/search?" + urlencode( {'p': '084__:"' + pacs_code + '"' }) + "' title='" + title + "'>" + pacs_code + "</a>" if pacs_count < 25: output.append(search_link) else: output1.append(search_link) pacs_count += 1 if len(output1): link = """ | <a href="#" onclick="toggle2('content', this); return false;" style="color:green;background:white;"><i>More</i></a> <div id="content" style="display:none; padding-left:42px;"> %(content)s </div> <script type="text/javascript"> function toggle2(id, link) { var e = document.getElementById(id); if (e.style.display == '') { e.style.display = 'none'; link.innerHTML = '<i>More</i>'; } else { e.style.display = ''; link.innerHTML = '<i>Less</i>'; } } </script> """ % { 'content': separator.join(output1) } return separator.join(output) + link
class JournalForm(WTFormDefault): name = QuerySelectField('', get_pk=lambda i: i['key'], get_label=lambda i: i['value'], query_factory=lambda: [{ 'key': '', 'value': _('Any journal') }] + get_kb_mappings('EJOURNALS')) vol = TextField(_('Vol')) page = TextField(_('Pg'))
def _kb_license_choices(domain_data=True, domain_content=True, domain_software=True): def _mapper(x): license = json.loads(x["value"]) if ( (license["domain_data"] and domain_data) or (license["domain_content"] and domain_content) or (license["domain_software"] and domain_software) ): return (x["key"], license["title"]) else: return None return filter(lambda x: x is not None, map(_mapper, get_kb_mappings("licenses", "", "")))
def get_journal_info(recid, tags): record_info = [] # TODO: handle recors with multiple journals tagsvalues = {} # we store the tags and their values here # like c->444 y->1999 p->"journal of foo", # v->20 tmp = get_fieldvalues(recid, tags['publication']['journal']) if tmp: tagsvalues["p"] = tmp[0] tmp = get_fieldvalues(recid, tags['publication']['volume']) if tmp: tagsvalues["v"] = tmp[0] tmp = get_fieldvalues(recid, tags['publication']['year']) if tmp: tagsvalues["y"] = tmp[0] tmp = get_fieldvalues(recid, tags['publication']['pages']) if tmp: # if the page numbers have "x-y" take just x pages = tmp[0] hpos = pages.find("-") if hpos > 0: pages = pages[:hpos] tagsvalues["c"] = pages # check if we have the required data ok = True for c in tags['publication_format']: if c in ('p', 'v', 'y', 'c'): if c not in tagsvalues: ok = False if ok: publ = format_journal(tags['publication_format'], tagsvalues) record_info += [publ] alt_volume = get_alt_volume(tagsvalues['v']) if alt_volume: tagsvalues2 = tagsvalues.copy() tagsvalues2['v'] = alt_volume publ = format_journal(tags['publication_format'], tagsvalues2) record_info += [publ] # Add codens for coden in get_kb_mappings('CODENS', value=tagsvalues['p']): tagsvalues2 = tagsvalues.copy() tagsvalues2['p'] = coden['key'] publ = format_journal(tags['publication_format'], tagsvalues2) record_info += [publ] return record_info
def get_journal_info(recid, tags): record_info = [] # TODO: handle recors with multiple journals tagsvalues = {} # we store the tags and their values here # like c->444 y->1999 p->"journal of foo", # v->20 tmp = get_fieldvalues(recid, tags['publication']['journal']) if tmp: tagsvalues["p"] = tmp[0] tmp = get_fieldvalues(recid, tags['publication']['volume']) if tmp: tagsvalues["v"] = tmp[0] tmp = get_fieldvalues(recid, tags['publication']['year']) if tmp: tagsvalues["y"] = tmp[0] tmp = get_fieldvalues(recid, tags['publication']['pages']) if tmp: # if the page numbers have "x-y" take just x pages = tmp[0] hpos = pages.find("-") if hpos > 0: pages = pages[:hpos] tagsvalues["c"] = pages # check if we have the required data ok = True for c in tags['publication_format']: if c in ('p', 'v', 'y', 'c'): if c not in tagsvalues: ok = False if ok: publ = format_journal(tags['publication_format'], tagsvalues) record_info += [publ] alt_volume = get_alt_volume(tagsvalues['v']) if alt_volume: tagsvalues2 = tagsvalues.copy() tagsvalues2['v'] = alt_volume publ = format_journal(tags['publication_format'], tagsvalues2) record_info += [publ] # Add codens for coden in get_kb_mappings('CODENS', value=tagsvalues['p']): tagsvalues2 = tagsvalues.copy() tagsvalues2['p'] = coden['key'] publ = format_journal(tags['publication_format'], tagsvalues2) record_info += [publ] return record_info
def format_element(bfo, separator=' | ', link="yes"): """ Prints PACS info as best is possible. @param link if yes (default) prints link to search for this item in Inspire @param separator separates multiple items """ fields = bfo.fields('084__') output = [] output1 = [] pacs_count = 0 link = "" for item in fields: if item.get('2') == 'PACS': pacs_code = item.get('a') if pacs_code: pacs_kb_mapping = get_kb_mappings('PACS', key=pacs_code, match_type="e") title = 'Translation not available' if pacs_kb_mapping: title = pacs_kb_mapping[0]['value'] search_link = "<a href='/search?" + urlencode({'p': '084__:"' + pacs_code + '"'}) + "' title='" + title + "'>" + pacs_code + "</a>" if pacs_count < 25: output.append(search_link) else: output1.append(search_link) pacs_count += 1 if len(output1): link = """ | <a href="#" onclick="toggle2('content', this); return false;" style="color:green;background:white;"><i>More</i></a> <div id="content" style="display:none; padding-left:42px;"> %(content)s </div> <script type="text/javascript"> function toggle2(id, link) { var e = document.getElementById(id); if (e.style.display == '') { e.style.display = 'none'; link.innerHTML = '<i>More</i>'; } else { e.style.display = ''; link.innerHTML = '<i>Less</i>'; } } </script> """ % {'content': separator.join(output1)} return separator.join(output) + link
def _kb_license_choices(domain_data=True, domain_content=True, domain_software=True): def _mapper(x): license = json.loads(x['value']) if (license['domain_data'] and domain_data) or \ (license['domain_content'] and domain_content) or \ (license['domain_software'] and domain_software): return (x['key'], license['title']) else: return None return filter(lambda x: x is not None, map(_mapper, get_kb_mappings('licenses', '', '')))
def prepare_data_cache(self): """ "Index" knowledge base and cache it. """ cache = {} for mapping in get_kb_mappings(self.get_kbname()): key = mapping['key'] value = mapping['value'] words = clean_and_split_words_and_stem(key, CFG_SITE_LANG, stem_p=True) for word in words: if not cache.has_key(word): cache[word] = [] if not value in cache[word]: cache[word].append(value) return cache
def prepare_data_cache(self): """ "Index" knowledge base and cache it. """ cache = {} for mapping in get_kb_mappings(self.get_kbname()): key = mapping['key'] value = mapping['value'] words = clean_and_split_words_and_stem(key, CFG_SITE_LANG, stem_p=True) for word in words: if not cache.has_key(word): cache[word] = [] if not value in cache[word]: cache[word].append(value) return cache
def _get_json_dump_of_codens(): """Dump codens to a global variable where our javascript can get it. In our BFT we will want to have an onSubmit() handler which substitutes every short title for a coden; this makes the data for that available""" return '\n<script type="text/javascript">gCODENS = %s</script>\n' % simplejson.dumps(dict([(x['value'], x['key']) for x in get_kb_mappings('CODENS')]))
def kb_export(req, kbname="", format="kbr", searchkey="", searchvalue="", searchtype="s", ln=CFG_SITE_LANG): """ Exports the given kb so that it is listed in stdout (the browser). @param req the request @param kbname knowledge base name @param format kba for an authority file, kbr for a leftside-rightside kb @param searchkey include only lines that match this on the left side @param searchvalue include only lines that match this on the right side @param searchtype s = substring match, e = exact match @param ln language """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_URL, ln, _("Manage Knowledge Bases")) if not kbname: return page(title=_("Knowledge base name missing"), body = """Required parameter kbname is missing.""", language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) #in order to make 'wget' downloads easy we do not require authorization #first check the type of the KB kbtype = None kbinfo = None kbid = None kbinfos = bibknowledge.get_kbs_info("", kbname) if kbinfos: kbinfo = kbinfos[0] kbtype = kbinfo['kbtype'] kbid = kbinfo['id'] else: return page(title=_("Unknown knowledge base"), body = _("There is no knowledge base with that name."), language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) if not kbtype or kbtype == 'w': #get the kb and print it mappings = bibknowledge.get_kb_mappings(kbname, searchkey, \ searchvalue, searchtype) if format == 'jquery': ret = [] for m in mappings: label = m['value'] or m['key'] value = m['key'] or m['value'] ret.append({'label': label, 'value': value}) req.content_type = 'application/json' return json.dumps(ret) if not mappings: body = "There is no knowledge base named "+kbname+" or it is empty", return page(title=_("No such knowledge base"), body=body, language=ln, navtrail=navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: #there were mappings seq = [] #sequence: right sides need to made unique for m in mappings: mkey = m['key'] mvalue = m['value'] if format == "right" or format == "kba": seq.append(mvalue) else: req.write(mkey+"---"+mvalue+"\n") #make unique seq and print it useq = uniq(seq) for s in useq: req.write(s+"\n") if kbtype == 'd': #dynamic type: call export #ok, let's search.. res = bibknowledge.get_kbd_values(kbname, searchvalue) if not res: req.write("\n") #in order to say something for r in res: if not searchvalue: req.write(r+"\n") #output all if searchvalue and r.count(searchvalue) > 0: req.write(r+"\n") #output matching lines if kbtype == 't': #taxonomy: output the file kbfilename = CFG_WEBDIR+"/kbfiles/"+str(kbid)+".rdf" try: f = open(kbfilename, 'r') for line in f: req.write(line) f.close() except: req.write("Reading the file "+kbfilename+" failed.")
def load_kbs(cfg, run_sql, in_task=False): for kb, query in cfg.iteritems(): task_sleep_now_if_required(can_stop_too=True) if not kb_exists(kb): add_kb(kb) if in_task: write_message("Updating %s KB..." % kb) try: if not in_task: print "kb:", kb print "kb beginning:", len(get_kb_mappings(kb)) if kb.startswith('json_'): encoder = ComplexEncoder() mapping, description = run_sql(query, with_desc=True) if kb in CFG_ADDITIONAL_ENTRIES: mapping += CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print CFG_ADDITIONAL_ENTRIES[kb] column_counter = {} new_description = [] for column in description[1:]: column = column[0] counter = column_counter[column] = column_counter.get(column, 0) + 1 if counter > 1: new_description.append('%s%d' % (column, counter)) else: new_description.append(column) description = new_description else: mapping = run_sql(query) if kb in CFG_ADDITIONAL_ENTRIES: mapping += CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print "mapping:", len(mapping) if kb == 'projects': mapping += [('000000', 'NO PROJECT')] original_keys = set([key[0] for key in get_kbr_keys(kb)]) if not in_task: print "original_keys before:", len(original_keys) updated = 0 added = 0 for i, row in enumerate(mapping): key, value = row[0], row[1:] if kb.startswith('json_'): value = encoder.encode(dict(zip(description, value))) else: value = value[0] if value: if key in original_keys: original_keys.remove(key) if in_task: task_update_progress("%s - %s%%" % (kb, i * 100 / len(mapping))) if kb_mapping_exists(kb, key): updated += 1 update_kb_mapping(kb, key, key, value) else: added += 1 add_kb_mapping(kb, key, value) if not in_task: print "updated:", updated, "added:", added print "kb after update:", len(get_kb_mappings(kb)) print "original_keys after:", len(original_keys) if in_task: task_update_progress("Cleaning %s" % kb) for key in original_keys: remove_kb_mapping(kb, key) if not in_task: print "kb after remove:", len(get_kb_mappings(kb)) except: register_exception(alert_admin=True, prefix="Error when updating KB %s" % kb) continue
## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Expand single letter subjects - get mapping from KB """ from invenio.bibknowledge import get_kb_mappings from invenio.config import CFG_BIBEDIT_KB_SUBJECTS subjects = {} for item in get_kb_mappings(kb_name=CFG_BIBEDIT_KB_SUBJECTS): subjects[item['key']] = item['value'] def check_record(record): """ Expand single letter subjects """ for position, value in record.iterfield("65017a", ("2", "INSPIRE")): if len(value) == 1: if subjects.has_key(value): record.amend_field(position[:3], subjects[value]) else: record.warn('single letter subject without mapping: %s' % value)
def format_element(bfo, separator=' | ', link="yes"): """ Prints Conference info as best is possible. @param link if yes (default) prints link to SPIRES conference info @param separator separates multiple conferences """ authors = bfo.fields('084__') # Process authors to add link, highlight and format affiliation output = [] output1 = [] pdgcount = 0 link = "" pdgcode = "" for exp in authors: if exp.get('9') == 'PDG' and 'a' in exp: values = get_kb_mappings('PDG', key=exp['a'], match_type="e") pdgcode = exp['a'] for ch in [':', '=']: if ch in pdgcode: pdgcode = pdgcode.replace(ch, '/') if values: search_link = '<a href="http://pdglive.lbl.gov/view/{0}">'.format( pdgcode) if values[0]['value'] == "THE TEXT IS MISSING FOR THIS NODE.": search_link += pdgcode + ' (Title Unknown)' else: search_link += cgi.escape(values[0]['value']) search_link += '</a>' pdgcount += 1 if pdgcount < 3: output.append(search_link) else: output1.append(search_link) if len(output1): link = """ | <a href="#" style="color:green;background:white;" onclick="toggle2('content', this)"><i>More</i></a> <div id="content" style="display:none; padding-left:36px;"> %(content)s </div> <script type="text/javascript"> function toggle2(id, link) { var e = document.getElementById(id); if (e.style.display == '') { e.style.display = 'none'; link.innerHTML = '<i>More</i>'; } else { e.style.display = ''; link.innerHTML = '<i>Less</i>'; } } </script> """ % { 'content': separator.join(output1) } return separator.join(output) + link
def kb_export(req, kbname="", format="kbr", searchkey="", searchvalue="", searchtype="s", ln=CFG_SITE_LANG): """ Exports the given kb so that it is listed in stdout (the browser). @param req the request @param kbname knowledge base name @param format kba for an authority file, kbr for a leftside-rightside kb @param searchkey include only lines that match this on the left side @param searchvalue include only lines that match this on the right side @param searchtype s = substring match, e = exact match @param ln language """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_URL, ln, _("Manage Knowledge Bases")) if not kbname: return page(title=_("Knowledge base name missing"), body = """Required parameter kbname is missing.""", language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) #in order to make 'wget' downloads easy we do not require authorization #first check the type of the KB kbtype = None kbinfo = None kbid = None kbinfos = bibknowledge.get_kbs_info("", kbname) if kbinfos: kbinfo = kbinfos[0] kbtype = kbinfo['kbtype'] kbid = kbinfo['id'] else: return page(title=_("Unknown knowledge base"), body = _("There is no knowledge base with that name."), language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) if kbtype == None or kbtype == 'w': #get the kb and print it mappings = bibknowledge.get_kb_mappings(kbname, searchkey, \ searchvalue, searchtype) if format == 'jquery': ret = [] for m in mappings: label = m['value'] or m['key'] value = m['key'] or m['value'] ret.append({'label': label, 'value': value}) req.content_type = 'application/json' return json.dumps(ret) if not mappings: body = "There is no knowledge base named "+kbname+" or it is empty", return page(title=_("No such knowledge base"), body=body, language=ln, navtrail=navtrail_previous_links, lastupdated=__lastupdated__, req=req) else: #there were mappings seq = [] #sequence: right sides need to made unique for m in mappings: mkey = m['key'] mvalue = m['value'] if format == "right" or format == "kba": seq.append(mvalue) else: req.write(mkey+"---"+mvalue+"\n") #make unique seq and print it useq = uniq(seq) for s in useq: req.write(s+"\n") if kbtype == 'd': #dynamic type: call export #ok, let's search.. res = bibknowledge.get_kbd_values(kbname, searchvalue) if not res: req.write("\n") #in order to say something for r in res: if not searchvalue: req.write(r+"\n") #output all if searchvalue and r.count(searchvalue) > 0: req.write(r+"\n") #output matching lines if kbtype == 't': #taxonomy: output the file kbfilename = CFG_WEBDIR+"/kbfiles/"+str(kbid)+".rdf" try: f = open(kbfilename, 'r') for line in f: req.write(line) f.close() except: req.write("Reading the file "+kbfilename+" failed.")
def kb_add_mapping(req, kb, mapFrom, mapTo, sortby="to", ln=CFG_SITE_LANG, forcetype=None, replacements=None, kb_type=None): """ Adds a new mapping to a kb. @param ln language @param kb the kb id to show @param sortby the sorting criteria ('from' or 'to') @param forcetype indicates if this function should ask about replacing left/right sides (None or 'no') replace in current kb ('curr') or in all ('all') @param replacements an object containing kbname+++left+++right strings. Can be a string or an array of strings @param kb_type None for normal from-to kb's, 't' for taxonomies """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases")) try: dummy = getUid(req) except: return error_page('Error', req) (auth_code, auth_msg) = check_user(req, 'cfgbibknowledge') if not auth_code: kb_id = wash_url_argument(kb, 'int') kb_name = bibknowledge.get_kb_name(kb_id) if kb_name is None: return page(title=_("Unknown Knowledge Base"), body = "", language=ln, navtrail = navtrail_previous_links, errors = [("ERR_KB_ID_UNKNOWN", kb)], lastupdated=__lastupdated__, req=req) key = wash_url_argument(mapFrom, 'str') value = wash_url_argument(mapTo, 'str') #check if key or value already exists in some KB left_sides_match = bibknowledge.get_kb_mappings("", key, "") #check that the match is exact left_sides = [] for m in left_sides_match: if m['key'] == key: left_sides.append(m) right_sides_match = bibknowledge.get_kb_mappings("", "", value) right_sides = [] for m in right_sides_match: if m['value'] == value: right_sides.append(m) if (len(right_sides) == 0) and (len(left_sides) == 0): #no problems, just add in current forcetype = "curr" #likewise, if this is a taxonomy, just pass on if kb_type == 't': forcetype = "curr" if forcetype and not forcetype == "no": pass else: if len(left_sides) > 0: return page(title=_("Left side exists"), body = bibknowledgeadminlib.perform_request_verify_rule(ln, kb_id, key, value, "left", kb_name, left_sides), language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) if len(right_sides) > 0: return page(title=_("Right side exists"), body = bibknowledgeadminlib.perform_request_verify_rule(ln, kb_id, key, value, "right", kb_name, right_sides), language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) if forcetype == "curr": bibknowledge.add_kb_mapping(kb_name, key, value) if forcetype == "all": #a bit tricky.. remove the rules given in param replacement and add the current #rule in the same kb's if replacements: #"replacements" can be either a string or an array. Let's make it always an array if type(replacements) == type("this is a string"): mystr = replacements replacements = [] replacements.append(mystr) for r in replacements: if r.find("++++") > 0: (rkbname, rleft, dummy) = r.split('++++') bibknowledge.remove_kb_mapping(rkbname, rleft) #add only if this is not yet there.. if not bibknowledge.kb_mapping_exists(rkbname, key): bibknowledge.add_kb_mapping(rkbname, key, value) redirect_to_url(req, "kb?ln=%(ln)s&kb=%(kb)s&sortby=%(sortby)s&kb_type=%(kb_type)s" % {'ln':ln, 'kb':kb_id, 'sortby':sortby, 'kb_type':kb_type}) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links)
def inner(dummy_form, dummy_field, term, limit=50): result = get_kb_mappings(name, '', term)[:limit] return map(mapper, result) if mapper is not None else result
def load_kbs(cfg, run_sql, in_task=False): for kb, query in cfg.iteritems(): task_sleep_now_if_required(can_stop_too=True) if not kb_exists(kb): add_kb(kb) if in_task: write_message("Updating %s KB..." % kb) try: if not in_task: print "kb:", kb print "kb beginning:", len(get_kb_mappings(kb)) if kb.startswith('json_'): encoder = ComplexEncoder() mapping, description = run_sql(query, with_desc=True) if kb in CFG_ADDITIONAL_ENTRIES: mapping += CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print CFG_ADDITIONAL_ENTRIES[kb] column_counter = {} new_description = [] for column in description[1:]: column = column[0] counter = column_counter[column] = column_counter.get( column, 0) + 1 if counter > 1: new_description.append('%s%d' % (column, counter)) else: new_description.append(column) description = new_description else: mapping = run_sql(query) if kb in CFG_ADDITIONAL_ENTRIES: mapping += CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print CFG_ADDITIONAL_ENTRIES[kb] if not in_task: print "mapping:", len(mapping) if kb == 'projects': mapping += [('000000', 'NO PROJECT')] original_keys = set([key[0] for key in get_kbr_keys(kb)]) if not in_task: print "original_keys before:", len(original_keys) updated = 0 added = 0 for i, row in enumerate(mapping): key, value = row[0], row[1:] if kb.startswith('json_'): value = encoder.encode(dict(zip(description, value))) else: value = value[0] if value: if key in original_keys: original_keys.remove(key) if in_task: task_update_progress("%s - %s%%" % (kb, i * 100 / len(mapping))) if kb_mapping_exists(kb, key): updated += 1 update_kb_mapping(kb, key, key, value) else: added += 1 add_kb_mapping(kb, key, value) if not in_task: print "updated:", updated, "added:", added print "kb after update:", len(get_kb_mappings(kb)) print "original_keys after:", len(original_keys) if in_task: task_update_progress("Cleaning %s" % kb) for key in original_keys: remove_kb_mapping(kb, key) if not in_task: print "kb after remove:", len(get_kb_mappings(kb)) except: register_exception(alert_admin=True, prefix="Error when updating KB %s" % kb) continue
## published by the Free Software Foundation; either version 2 of the ## License, or (at your option) any later version. ## ## Invenio is distributed in the hope that it will be useful, but ## WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ## General Public License for more details. ## ## You should have received a copy of the GNU General Public License ## along with Invenio; if not, write to the Free Software Foundation, Inc., ## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Expand single letter subjects - get mapping from KB """ from invenio.bibknowledge import get_kb_mappings from invenio.config import CFG_BIBEDIT_KB_SUBJECTS subjects = {} for item in get_kb_mappings(kb_name=CFG_BIBEDIT_KB_SUBJECTS): subjects[item['key']] = item['value'] def check_record(record): """ Expand single letter subjects """ for position, value in record.iterfield("65017a", ("2", "INSPIRE")): if len(value) == 1: if subjects.has_key(value): record.amend_field(position[:3], subjects[value]) else: record.warn('single letter subject without mapping: %s' % value)
def format_element(bfo, separator=' | ', link="yes"): """ Prints Conference info as best is possible. @param link if yes (default) prints link to SPIRES conference info @param separator separates multiple conferences """ authors = bfo.fields('084__') # Process authors to add link, highlight and format affiliation output = [] output1 = [] pdgcount = 0 link = "" pdgcode = "" for exp in authors: if exp.get('9') == 'PDG' and 'a' in exp: values = get_kb_mappings('PDG', key=exp['a'], match_type="e") pdgcode = exp['a'] for ch in [':', '=']: if ch in pdgcode: pdgcode = pdgcode.replace(ch, '/') if values: search_link = '<a href="http://pdglive.lbl.gov/view/{0}">'.format(pdgcode) if values[0]['value'] == "THE TEXT IS MISSING FOR THIS NODE.": search_link += pdgcode + ' (Title Unknown)' else: search_link += cgi.escape(values[0]['value']) search_link += '</a>' pdgcount += 1 if pdgcount < 3: output.append(search_link) else: output1.append(search_link) if len(output1): link = """ | <a href="#" style="color:green;background:white;" onclick="toggle2('content', this)"><i>More</i></a> <div id="content" style="display:none; padding-left:36px;"> %(content)s </div> <script type="text/javascript"> function toggle2(id, link) { var e = document.getElementById(id); if (e.style.display == '') { e.style.display = 'none'; link.innerHTML = '<i>More</i>'; } else { e.style.display = ''; link.innerHTML = '<i>Less</i>'; } } </script> """ % {'content': separator.join(output1)} return separator.join(output) + link
def kb_add_mapping(req, kb, mapFrom, mapTo, sortby="to", ln=CFG_SITE_LANG, forcetype=None, replacements=None, kb_type=None): """ Adds a new mapping to a kb. @param ln language @param kb the kb id to show @param sortby the sorting criteria ('from' or 'to') @param forcetype indicates if this function should ask about replacing left/right sides (None or 'no') replace in current kb ('curr') or in all ('all') @param replacements an object containing kbname+++left+++right strings. Can be a string or an array of strings @param kb_type None for normal from-to kb's, 't' for taxonomies """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % ( CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases")) try: dummy = getUid(req) except MySQLdb.Error: return error_page(req) (auth_code, auth_msg) = check_user(req, 'cfgbibknowledge') if not auth_code: kb_id = wash_url_argument(kb, 'int') kb_name = bibknowledge.get_kb_name(kb_id) if kb_name is None: return page(title=_("Unknown Knowledge Base"), body="", language=ln, navtrail=navtrail_previous_links, errors=[("ERR_KB_ID_UNKNOWN", kb)], lastupdated=__lastupdated__, req=req) key = wash_url_argument(mapFrom, 'str') value = wash_url_argument(mapTo, 'str') #check if key or value already exists in some KB left_sides_match = bibknowledge.get_kb_mappings("", key, "") #check that the match is exact left_sides = [] for m in left_sides_match: if m['key'] == key: left_sides.append(m) right_sides_match = bibknowledge.get_kb_mappings("", "", value) right_sides = [] for m in right_sides_match: if m['value'] == value: right_sides.append(m) if (len(right_sides) == 0) and (len(left_sides) == 0): #no problems, just add in current forcetype = "curr" #likewise, if this is a taxonomy, just pass on if kb_type == 't': forcetype = "curr" if forcetype and not forcetype == "no": pass else: if len(left_sides) > 0: return page( title=_("Left side exists"), body=bibknowledgeadminlib.perform_request_verify_rule( ln, kb_id, key, value, "left", kb_name, left_sides), language=ln, navtrail=navtrail_previous_links, lastupdated=__lastupdated__, req=req) if len(right_sides) > 0: return page( title=_("Right side exists"), body=bibknowledgeadminlib.perform_request_verify_rule( ln, kb_id, key, value, "right", kb_name, right_sides), language=ln, navtrail=navtrail_previous_links, lastupdated=__lastupdated__, req=req) if forcetype == "curr": bibknowledge.add_kb_mapping(kb_name, key, value) if forcetype == "all": #a bit tricky.. remove the rules given in param replacement and add the current #rule in the same kb's if replacements: #"replacements" can be either a string or an array. Let's make it always an array if type(replacements) == type("this is a string"): mystr = replacements replacements = [] replacements.append(mystr) for r in replacements: if r.find("++++") > 0: (rkbname, rleft, dummy) = r.split('++++') bibknowledge.remove_kb_mapping(rkbname, rleft) #add only if this is not yet there.. if not bibknowledge.kb_mapping_exists(rkbname, key): bibknowledge.add_kb_mapping(rkbname, key, value) redirect_to_url( req, "kb?ln=%(ln)s&kb=%(kb)s&sortby=%(sortby)s&kb_type=%(kb_type)s" % { 'ln': ln, 'kb': kb_id, 'sortby': sortby, 'kb_type': kb_type }) else: return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links)
def kb_export(req, kbname="", format="kbr", searchkey="", searchvalue="", searchtype="s", limit=None, ln=CFG_SITE_LANG): """ Exports the given kb so that it is listed in stdout (the browser). @param req the request @param kbname knowledge base name @param expression evaluate this for the returned lines @param format 'kba' for authority file, 'kbr' for leftside-rightside, json for json-formatted dictionaries @param searchkey include only lines that match this on the left side @param searchvalue include only lines that match this on the right side @param searchtype s = substring match, e = exact match @param limit how many results to return. None means all @param ln language """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % ( CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases")) if not kbname: return page(title=_("Knowledge base name missing"), body="""Required parameter kbname is missing.""", language=ln, navtrail=navtrail_previous_links, lastupdated=__lastupdated__, req=req) #in order to make 'wget' downloads easy we do not require authorization #first check the type of the KB kbtype = None kbinfo = None kbid = None kbinfos = bibknowledge.get_kbs_info("", kbname) if kbinfos: kbinfo = kbinfos[0] kbtype = kbinfo['kbtype'] kbid = kbinfo['id'] else: return page(title=_("Unknown knowledge base"), body=_("There is no knowledge base with that name."), language=ln, navtrail=navtrail_previous_links, lastupdated=__lastupdated__, req=req) if kbtype == None or kbtype == 'w': if format and format == "ejson": req.content_type = 'application/json' return bibknowledge.get_kb_mappings_embedded_json(kbname, searchkey, \ searchvalue, searchtype, limit) elif format and format[0] == 'j': # as JSON formatted string req.content_type = 'application/json' return bibknowledge.get_kb_mappings_json(kbname, searchkey, \ searchvalue, searchtype, limit) # left side / right side KB mappings = bibknowledge.get_kb_mappings(kbname, searchkey, \ searchvalue, searchtype) if format == 'right' or format == 'kba': # as authority sequence seq = [m['value'] for m in mappings] seq = uniq(sorted(seq)) for s in seq: req.write(s + "\n") return else: # as regularly formatted left-right mapping for m in mappings: req.write(m['key'] + '---' + m['value'] + '\n') return elif kbtype == 'd': # dynamic kb, another interface for perform_request_search if format and format[0] == 'j': req.content_type = "application/json" return bibknowledge.get_kbd_values_json(kbname, searchvalue) else: # print it as a list of values for hit in bibknowledge.get_kbd_values(kbname, searchvalue): req.write(hit + '\n') req.write('\n') return elif kbtype == 't': #taxonomy: output the file kbfilename = CFG_WEBDIR + "/kbfiles/" + str(kbid) + ".rdf" try: f = open(kbfilename, 'r') for line in f: req.write(line) f.close() except: req.write("Reading the file " + kbfilename + " failed.") else: # This situation should never happen raise ValueError, "Unsupported KB Type: %s" % kbtype
def get_journal_info(record, tags): """Fetch journal info from given record""" record_info = [] journals_fields = record.find_fields(tags['publication']['journal'][:5]) for field in journals_fields: # we store the tags and their values here # like c->444 y->1999 p->"journal of foo", # v->20 tagsvalues = {} try: tmp = field.get_subfield_values(tags['publication']['journal'][5])[0] except IndexError: pass else: tagsvalues["p"] = tmp try: tmp = field.get_subfield_values(tags['publication']['volume'][5])[0] except IndexError: pass else: tagsvalues["v"] = tmp try: tmp = field.get_subfield_values(tags['publication']['year'][5])[0] except IndexError: pass else: tagsvalues["y"] = tmp try: tmp = field.get_subfield_values(tags['publication']['pages'][5])[0] except IndexError: pass else: # if the page numbers have "x-y" take just x tagsvalues["c"] = tmp.split('-', 1)[0] # check if we have the required data ok = True for c in tags['publication_format']: if c in ('p', 'v', 'y', 'c'): if c not in tagsvalues: ok = False if ok: publ = format_journal(tags['publication_format'], tagsvalues) record_info += [publ] alt_volume = get_alt_volume(tagsvalues['v']) if alt_volume: tagsvalues2 = tagsvalues.copy() tagsvalues2['v'] = alt_volume publ = format_journal(tags['publication_format'], tagsvalues2) record_info += [publ] # Add codens for coden in get_kb_mappings('CODENS', value=tagsvalues['p']): tagsvalues2 = tagsvalues.copy() tagsvalues2['p'] = coden['key'] publ = format_journal(tags['publication_format'], tagsvalues2) record_info += [publ] return record_info
def kb_export(req, kbname="", format="kbr", searchkey="", searchvalue="", searchtype="s", limit=None, ln=CFG_SITE_LANG): """ Exports the given kb so that it is listed in stdout (the browser). @param req the request @param kbname knowledge base name @param expression evaluate this for the returned lines @param format 'kba' for authority file, 'kbr' for leftside-rightside, json for json-formatted dictionaries @param searchkey include only lines that match this on the left side @param searchvalue include only lines that match this on the right side @param searchtype s = substring match, e = exact match @param limit how many results to return. None means all @param ln language """ ln = wash_language(ln) _ = gettext_set_language(ln) navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases")) if not kbname: return page(title=_("Knowledge base name missing"), body = """Required parameter kbname is missing.""", language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) #in order to make 'wget' downloads easy we do not require authorization #first check the type of the KB kbtype = None kbinfo = None kbid = None kbinfos = bibknowledge.get_kbs_info("", kbname) if kbinfos: kbinfo = kbinfos[0] kbtype = kbinfo['kbtype'] kbid = kbinfo['id'] else: return page(title=_("Unknown knowledge base"), body = _("There is no knowledge base with that name."), language=ln, navtrail = navtrail_previous_links, lastupdated=__lastupdated__, req=req) if not kbtype or kbtype == 'w': if format and format == "ejson": req.content_type = 'application/json' return bibknowledge.get_kb_mappings_embedded_json(kbname, searchkey, \ searchvalue, searchtype, limit) elif format and format[0] == 'j': # as JSON formatted string req.content_type = 'application/json' return bibknowledge.get_kb_mappings_json(kbname, searchkey, \ searchvalue, searchtype, limit) # left side / right side KB mappings = bibknowledge.get_kb_mappings(kbname, searchkey, \ searchvalue, searchtype) if format == 'right' or format == 'kba': # as authority sequence seq = [m['value'] for m in mappings] seq = uniq(sorted(seq)) for s in seq: req.write(s+"\n"); return else: # as regularly formatted left-right mapping for m in mappings: req.write(m['key'] + '---' + m['value'] + '\n') return elif kbtype == 'd': # dynamic kb, another interface for perform_request_search if format and format[0] == 'j': req.content_type = "application/json" return bibknowledge.get_kbd_values_json(kbname, searchvalue) else: # print it as a list of values for hit in bibknowledge.get_kbd_values(kbname, searchvalue): req.write(hit + '\n') req.write('\n') return elif kbtype == 't': #taxonomy: output the file kbfilename = CFG_WEBDIR+"/kbfiles/"+str(kbid)+".rdf" try: f = open(kbfilename, 'r') for line in f: req.write(line) f.close() except: req.write("Reading the file "+kbfilename+" failed.") else: # This situation should never happen raise ValueError, "Unsupported KB Type: %s" % kbtype
def get_journal_info(record, tags): """Fetch journal info from given record""" record_info = [] journals_fields = record.find_fields(tags['publication']['journal'][:5]) for field in journals_fields: # we store the tags and their values here # like c->444 y->1999 p->"journal of foo", # v->20 tagsvalues = {} try: tmp = field.get_subfield_values( tags['publication']['journal'][5])[0] except IndexError: pass else: tagsvalues["p"] = tmp try: tmp = field.get_subfield_values( tags['publication']['volume'][5])[0] except IndexError: pass else: tagsvalues["v"] = tmp try: tmp = field.get_subfield_values(tags['publication']['year'][5])[0] except IndexError: pass else: tagsvalues["y"] = tmp try: tmp = field.get_subfield_values(tags['publication']['pages'][5])[0] except IndexError: pass else: # if the page numbers have "x-y" take just x tagsvalues["c"] = tmp.split('-', 1)[0] # check if we have the required data ok = True for c in tags['publication_format']: if c in ('p', 'v', 'y', 'c'): if c not in tagsvalues: ok = False if ok: publ = format_journal(tags['publication_format'], tagsvalues) record_info += [publ] alt_volume = get_alt_volume(tagsvalues['v']) if alt_volume: tagsvalues2 = tagsvalues.copy() tagsvalues2['v'] = alt_volume publ = format_journal(tags['publication_format'], tagsvalues2) record_info += [publ] # Add codens for coden in get_kb_mappings('CODENS', value=tagsvalues['p']): tagsvalues2 = tagsvalues.copy() tagsvalues2['p'] = coden['key'] publ = format_journal(tags['publication_format'], tagsvalues2) record_info += [publ] return record_info