def _get_json_dump_of_codens():
"""Dump codens to a global variable where our javascript can get it.
In our BFT we will want to have an onSubmit() handler which substitutes
every short title for a coden; this makes the data for that available"""
return '\n<script type="text/javascript">gCODENS = %s</script>\n' % simplejson.dumps(
dict([(x['value'], x['key']) for x in get_kb_mappings('CODENS')]))
def format_element(bfo, separator=' | ', link="yes"):
"""
Prints PACS info as best is possible.
@param link if yes (default) prints link to search for this item in Inspire
@param separator separates multiple items
"""
fields = bfo.fields('084__')
output = []
output1 = []
pacs_count = 0
link = ""
for item in fields:
if item.get('2') == 'PACS':
pacs_code = item.get('a')
if pacs_code:
pacs_kb_mapping = get_kb_mappings('PACS',
key=pacs_code,
match_type="e")
title = 'Translation not available'
if pacs_kb_mapping:
title = pacs_kb_mapping[0]['value']
search_link = "<a href='/search?" + urlencode(
{'p': '084__:"' + pacs_code + '"'
}) + "' title='" + title + "'>" + pacs_code + "</a>"
if pacs_count < 25:
output.append(search_link)
else:
output1.append(search_link)
pacs_count += 1
if len(output1):
link = """ | <a href="#" onclick="toggle2('content', this); return false;" style="color:green;background:white;"><i>More</i></a>
<div id="content" style="display:none; padding-left:42px;">
%(content)s
</div>
<script type="text/javascript">
function toggle2(id, link) {
var e = document.getElementById(id);
if (e.style.display == '') {
e.style.display = 'none';
link.innerHTML = '<i>More</i>';
}
else {
e.style.display = '';
link.innerHTML = '<i>Less</i>';
}
}
</script>
""" % {
'content': separator.join(output1)
}
return separator.join(output) + link
class JournalForm(WTFormDefault):
name = QuerySelectField('',
get_pk=lambda i: i['key'],
get_label=lambda i: i['value'],
query_factory=lambda: [{
'key': '',
'value': _('Any journal')
}] + get_kb_mappings('EJOURNALS'))
vol = TextField(_('Vol'))
page = TextField(_('Pg'))
def _kb_license_choices(domain_data=True, domain_content=True, domain_software=True):
def _mapper(x):
license = json.loads(x["value"])
if (
(license["domain_data"] and domain_data)
or (license["domain_content"] and domain_content)
or (license["domain_software"] and domain_software)
):
return (x["key"], license["title"])
else:
return None
return filter(lambda x: x is not None, map(_mapper, get_kb_mappings("licenses", "", "")))
def get_journal_info(recid, tags):
record_info = []
# TODO: handle recors with multiple journals
tagsvalues = {} # we store the tags and their values here
# like c->444 y->1999 p->"journal of foo",
# v->20
tmp = get_fieldvalues(recid, tags['publication']['journal'])
if tmp:
tagsvalues["p"] = tmp[0]
tmp = get_fieldvalues(recid, tags['publication']['volume'])
if tmp:
tagsvalues["v"] = tmp[0]
tmp = get_fieldvalues(recid, tags['publication']['year'])
if tmp:
tagsvalues["y"] = tmp[0]
tmp = get_fieldvalues(recid, tags['publication']['pages'])
if tmp:
# if the page numbers have "x-y" take just x
pages = tmp[0]
hpos = pages.find("-")
if hpos > 0:
pages = pages[:hpos]
tagsvalues["c"] = pages
# check if we have the required data
ok = True
for c in tags['publication_format']:
if c in ('p', 'v', 'y', 'c'):
if c not in tagsvalues:
ok = False
if ok:
publ = format_journal(tags['publication_format'], tagsvalues)
record_info += [publ]
alt_volume = get_alt_volume(tagsvalues['v'])
if alt_volume:
tagsvalues2 = tagsvalues.copy()
tagsvalues2['v'] = alt_volume
publ = format_journal(tags['publication_format'], tagsvalues2)
record_info += [publ]
# Add codens
for coden in get_kb_mappings('CODENS',
value=tagsvalues['p']):
tagsvalues2 = tagsvalues.copy()
tagsvalues2['p'] = coden['key']
publ = format_journal(tags['publication_format'], tagsvalues2)
record_info += [publ]
return record_info
def get_journal_info(recid, tags):
record_info = []
# TODO: handle recors with multiple journals
tagsvalues = {} # we store the tags and their values here
# like c->444 y->1999 p->"journal of foo",
# v->20
tmp = get_fieldvalues(recid, tags['publication']['journal'])
if tmp:
tagsvalues["p"] = tmp[0]
tmp = get_fieldvalues(recid, tags['publication']['volume'])
if tmp:
tagsvalues["v"] = tmp[0]
tmp = get_fieldvalues(recid, tags['publication']['year'])
if tmp:
tagsvalues["y"] = tmp[0]
tmp = get_fieldvalues(recid, tags['publication']['pages'])
if tmp:
# if the page numbers have "x-y" take just x
pages = tmp[0]
hpos = pages.find("-")
if hpos > 0:
pages = pages[:hpos]
tagsvalues["c"] = pages
# check if we have the required data
ok = True
for c in tags['publication_format']:
if c in ('p', 'v', 'y', 'c'):
if c not in tagsvalues:
ok = False
if ok:
publ = format_journal(tags['publication_format'], tagsvalues)
record_info += [publ]
alt_volume = get_alt_volume(tagsvalues['v'])
if alt_volume:
tagsvalues2 = tagsvalues.copy()
tagsvalues2['v'] = alt_volume
publ = format_journal(tags['publication_format'], tagsvalues2)
record_info += [publ]
# Add codens
for coden in get_kb_mappings('CODENS',
value=tagsvalues['p']):
tagsvalues2 = tagsvalues.copy()
tagsvalues2['p'] = coden['key']
publ = format_journal(tags['publication_format'], tagsvalues2)
record_info += [publ]
return record_info
def format_element(bfo, separator=' | ', link="yes"):
"""
Prints PACS info as best is possible.
@param link if yes (default) prints link to search for this item in Inspire
@param separator separates multiple items
"""
fields = bfo.fields('084__')
output = []
output1 = []
pacs_count = 0
link = ""
for item in fields:
if item.get('2') == 'PACS':
pacs_code = item.get('a')
if pacs_code:
pacs_kb_mapping = get_kb_mappings('PACS', key=pacs_code, match_type="e")
title = 'Translation not available'
if pacs_kb_mapping:
title = pacs_kb_mapping[0]['value']
search_link = "<a href='/search?" + urlencode({'p': '084__:"' + pacs_code + '"'}) + "' title='" + title + "'>" + pacs_code + "</a>"
if pacs_count < 25:
output.append(search_link)
else:
output1.append(search_link)
pacs_count += 1
if len(output1):
link = """ | <a href="#" onclick="toggle2('content', this); return false;" style="color:green;background:white;"><i>More</i></a>
<div id="content" style="display:none; padding-left:42px;">
%(content)s
</div>
<script type="text/javascript">
function toggle2(id, link) {
var e = document.getElementById(id);
if (e.style.display == '') {
e.style.display = 'none';
link.innerHTML = '<i>More</i>';
}
else {
e.style.display = '';
link.innerHTML = '<i>Less</i>';
}
}
</script>
""" % {'content': separator.join(output1)}
return separator.join(output) + link
def _kb_license_choices(domain_data=True,
domain_content=True,
domain_software=True):
def _mapper(x):
license = json.loads(x['value'])
if (license['domain_data'] and domain_data) or \
(license['domain_content'] and domain_content) or \
(license['domain_software'] and domain_software):
return (x['key'], license['title'])
else:
return None
return filter(lambda x: x is not None,
map(_mapper, get_kb_mappings('licenses', '', '')))
def prepare_data_cache(self):
"""
"Index" knowledge base and cache it.
"""
cache = {}
for mapping in get_kb_mappings(self.get_kbname()):
key = mapping['key']
value = mapping['value']
words = clean_and_split_words_and_stem(key, CFG_SITE_LANG, stem_p=True)
for word in words:
if not cache.has_key(word):
cache[word] = []
if not value in cache[word]:
cache[word].append(value)
return cache
def prepare_data_cache(self):
"""
"Index" knowledge base and cache it.
"""
cache = {}
for mapping in get_kb_mappings(self.get_kbname()):
key = mapping['key']
value = mapping['value']
words = clean_and_split_words_and_stem(key,
CFG_SITE_LANG,
stem_p=True)
for word in words:
if not cache.has_key(word):
cache[word] = []
if not value in cache[word]:
cache[word].append(value)
return cache
def _get_json_dump_of_codens():
"""Dump codens to a global variable where our javascript can get it.
In our BFT we will want to have an onSubmit() handler which substitutes
every short title for a coden; this makes the data for that available"""
return '\n<script type="text/javascript">gCODENS = %s</script>\n' % simplejson.dumps(dict([(x['value'], x['key']) for x in get_kb_mappings('CODENS')]))
def kb_export(req, kbname="", format="kbr", searchkey="", searchvalue="", searchtype="s", ln=CFG_SITE_LANG):
"""
Exports the given kb so that it is listed in stdout (the browser).
@param req the request
@param kbname knowledge base name
@param format kba for an authority file, kbr for a leftside-rightside kb
@param searchkey include only lines that match this on the left side
@param searchvalue include only lines that match this on the right side
@param searchtype s = substring match, e = exact match
@param ln language
"""
ln = wash_language(ln)
_ = gettext_set_language(ln)
navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_URL, ln, _("Manage Knowledge Bases"))
if not kbname:
return page(title=_("Knowledge base name missing"),
body = """Required parameter kbname
is missing.""",
language=ln,
navtrail = navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
#in order to make 'wget' downloads easy we do not require authorization
#first check the type of the KB
kbtype = None
kbinfo = None
kbid = None
kbinfos = bibknowledge.get_kbs_info("", kbname)
if kbinfos:
kbinfo = kbinfos[0]
kbtype = kbinfo['kbtype']
kbid = kbinfo['id']
else:
return page(title=_("Unknown knowledge base"),
body = _("There is no knowledge base with that name."),
language=ln,
navtrail = navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
if not kbtype or kbtype == 'w':
#get the kb and print it
mappings = bibknowledge.get_kb_mappings(kbname, searchkey, \
searchvalue, searchtype)
if format == 'jquery':
ret = []
for m in mappings:
label = m['value'] or m['key']
value = m['key'] or m['value']
ret.append({'label': label, 'value': value})
req.content_type = 'application/json'
return json.dumps(ret)
if not mappings:
body = "There is no knowledge base named "+kbname+" or it is empty",
return page(title=_("No such knowledge base"),
body=body,
language=ln,
navtrail=navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
else: #there were mappings
seq = [] #sequence: right sides need to made unique
for m in mappings:
mkey = m['key']
mvalue = m['value']
if format == "right" or format == "kba":
seq.append(mvalue)
else:
req.write(mkey+"---"+mvalue+"\n")
#make unique seq and print it
useq = uniq(seq)
for s in useq:
req.write(s+"\n")
if kbtype == 'd': #dynamic type: call export
#ok, let's search..
res = bibknowledge.get_kbd_values(kbname, searchvalue)
if not res:
req.write("\n") #in order to say something
for r in res:
if not searchvalue:
req.write(r+"\n") #output all
if searchvalue and r.count(searchvalue) > 0:
req.write(r+"\n") #output matching lines
if kbtype == 't': #taxonomy: output the file
kbfilename = CFG_WEBDIR+"/kbfiles/"+str(kbid)+".rdf"
try:
f = open(kbfilename, 'r')
for line in f:
req.write(line)
f.close()
except:
req.write("Reading the file "+kbfilename+" failed.")
def load_kbs(cfg, run_sql, in_task=False):
for kb, query in cfg.iteritems():
task_sleep_now_if_required(can_stop_too=True)
if not kb_exists(kb):
add_kb(kb)
if in_task:
write_message("Updating %s KB..." % kb)
try:
if not in_task:
print "kb:", kb
print "kb beginning:", len(get_kb_mappings(kb))
if kb.startswith('json_'):
encoder = ComplexEncoder()
mapping, description = run_sql(query, with_desc=True)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
column_counter = {}
new_description = []
for column in description[1:]:
column = column[0]
counter = column_counter[column] = column_counter.get(column, 0) + 1
if counter > 1:
new_description.append('%s%d' % (column, counter))
else:
new_description.append(column)
description = new_description
else:
mapping = run_sql(query)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print "mapping:", len(mapping)
if kb == 'projects':
mapping += [('000000', 'NO PROJECT')]
original_keys = set([key[0] for key in get_kbr_keys(kb)])
if not in_task:
print "original_keys before:", len(original_keys)
updated = 0
added = 0
for i, row in enumerate(mapping):
key, value = row[0], row[1:]
if kb.startswith('json_'):
value = encoder.encode(dict(zip(description, value)))
else:
value = value[0]
if value:
if key in original_keys:
original_keys.remove(key)
if in_task:
task_update_progress("%s - %s%%" % (kb, i * 100 / len(mapping)))
if kb_mapping_exists(kb, key):
updated += 1
update_kb_mapping(kb, key, key, value)
else:
added += 1
add_kb_mapping(kb, key, value)
if not in_task:
print "updated:", updated, "added:", added
print "kb after update:", len(get_kb_mappings(kb))
print "original_keys after:", len(original_keys)
if in_task:
task_update_progress("Cleaning %s" % kb)
for key in original_keys:
remove_kb_mapping(kb, key)
if not in_task:
print "kb after remove:", len(get_kb_mappings(kb))
except:
register_exception(alert_admin=True, prefix="Error when updating KB %s" % kb)
continue
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Expand single letter subjects - get mapping from KB
"""
from invenio.bibknowledge import get_kb_mappings
from invenio.config import CFG_BIBEDIT_KB_SUBJECTS
subjects = {}
for item in get_kb_mappings(kb_name=CFG_BIBEDIT_KB_SUBJECTS):
subjects[item['key']] = item['value']
def check_record(record):
"""
Expand single letter subjects
"""
for position, value in record.iterfield("65017a", ("2", "INSPIRE")):
if len(value) == 1:
if subjects.has_key(value):
record.amend_field(position[:3], subjects[value])
else:
record.warn('single letter subject without mapping: %s' %
value)
def format_element(bfo, separator=' | ', link="yes"):
"""
Prints Conference info as best is possible.
@param link if yes (default) prints link to SPIRES conference info
@param separator separates multiple conferences
"""
authors = bfo.fields('084__')
# Process authors to add link, highlight and format affiliation
output = []
output1 = []
pdgcount = 0
link = ""
pdgcode = ""
for exp in authors:
if exp.get('9') == 'PDG' and 'a' in exp:
values = get_kb_mappings('PDG', key=exp['a'], match_type="e")
pdgcode = exp['a']
for ch in [':', '=']:
if ch in pdgcode:
pdgcode = pdgcode.replace(ch, '/')
if values:
search_link = '<a href="http://pdglive.lbl.gov/view/{0}">'.format(
pdgcode)
if values[0]['value'] == "THE TEXT IS MISSING FOR THIS NODE.":
search_link += pdgcode + ' (Title Unknown)'
else:
search_link += cgi.escape(values[0]['value'])
search_link += '</a>'
pdgcount += 1
if pdgcount < 3:
output.append(search_link)
else:
output1.append(search_link)
if len(output1):
link = """ | <a href="#" style="color:green;background:white;" onclick="toggle2('content', this)"><i>More</i></a>
<div id="content" style="display:none; padding-left:36px;">
%(content)s
</div>
<script type="text/javascript">
function toggle2(id, link) {
var e = document.getElementById(id);
if (e.style.display == '') {
e.style.display = 'none';
link.innerHTML = '<i>More</i>';
}
else
{
e.style.display = '';
link.innerHTML = '<i>Less</i>';
}
}
</script>
""" % {
'content': separator.join(output1)
}
return separator.join(output) + link
def kb_export(req, kbname="", format="kbr", searchkey="", searchvalue="", searchtype="s", ln=CFG_SITE_LANG):
"""
Exports the given kb so that it is listed in stdout (the browser).
@param req the request
@param kbname knowledge base name
@param format kba for an authority file, kbr for a leftside-rightside kb
@param searchkey include only lines that match this on the left side
@param searchvalue include only lines that match this on the right side
@param searchtype s = substring match, e = exact match
@param ln language
"""
ln = wash_language(ln)
_ = gettext_set_language(ln)
navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_URL, ln, _("Manage Knowledge Bases"))
if not kbname:
return page(title=_("Knowledge base name missing"),
body = """Required parameter kbname
is missing.""",
language=ln,
navtrail = navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
#in order to make 'wget' downloads easy we do not require authorization
#first check the type of the KB
kbtype = None
kbinfo = None
kbid = None
kbinfos = bibknowledge.get_kbs_info("", kbname)
if kbinfos:
kbinfo = kbinfos[0]
kbtype = kbinfo['kbtype']
kbid = kbinfo['id']
else:
return page(title=_("Unknown knowledge base"),
body = _("There is no knowledge base with that name."),
language=ln,
navtrail = navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
if kbtype == None or kbtype == 'w':
#get the kb and print it
mappings = bibknowledge.get_kb_mappings(kbname, searchkey, \
searchvalue, searchtype)
if format == 'jquery':
ret = []
for m in mappings:
label = m['value'] or m['key']
value = m['key'] or m['value']
ret.append({'label': label, 'value': value})
req.content_type = 'application/json'
return json.dumps(ret)
if not mappings:
body = "There is no knowledge base named "+kbname+" or it is empty",
return page(title=_("No such knowledge base"),
body=body,
language=ln,
navtrail=navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
else: #there were mappings
seq = [] #sequence: right sides need to made unique
for m in mappings:
mkey = m['key']
mvalue = m['value']
if format == "right" or format == "kba":
seq.append(mvalue)
else:
req.write(mkey+"---"+mvalue+"\n")
#make unique seq and print it
useq = uniq(seq)
for s in useq:
req.write(s+"\n")
if kbtype == 'd': #dynamic type: call export
#ok, let's search..
res = bibknowledge.get_kbd_values(kbname, searchvalue)
if not res:
req.write("\n") #in order to say something
for r in res:
if not searchvalue:
req.write(r+"\n") #output all
if searchvalue and r.count(searchvalue) > 0:
req.write(r+"\n") #output matching lines
if kbtype == 't': #taxonomy: output the file
kbfilename = CFG_WEBDIR+"/kbfiles/"+str(kbid)+".rdf"
try:
f = open(kbfilename, 'r')
for line in f:
req.write(line)
f.close()
except:
req.write("Reading the file "+kbfilename+" failed.")
def kb_add_mapping(req, kb, mapFrom, mapTo, sortby="to", ln=CFG_SITE_LANG,
forcetype=None, replacements=None, kb_type=None):
"""
Adds a new mapping to a kb.
@param ln language
@param kb the kb id to show
@param sortby the sorting criteria ('from' or 'to')
@param forcetype indicates if this function should ask about replacing left/right sides (None or 'no')
replace in current kb ('curr') or in all ('all')
@param replacements an object containing kbname+++left+++right strings.
Can be a string or an array of strings
@param kb_type None for normal from-to kb's, 't' for taxonomies
"""
ln = wash_language(ln)
_ = gettext_set_language(ln)
navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases"))
try:
dummy = getUid(req)
except:
return error_page('Error', req)
(auth_code, auth_msg) = check_user(req, 'cfgbibknowledge')
if not auth_code:
kb_id = wash_url_argument(kb, 'int')
kb_name = bibknowledge.get_kb_name(kb_id)
if kb_name is None:
return page(title=_("Unknown Knowledge Base"),
body = "",
language=ln,
navtrail = navtrail_previous_links,
errors = [("ERR_KB_ID_UNKNOWN", kb)],
lastupdated=__lastupdated__,
req=req)
key = wash_url_argument(mapFrom, 'str')
value = wash_url_argument(mapTo, 'str')
#check if key or value already exists in some KB
left_sides_match = bibknowledge.get_kb_mappings("", key, "")
#check that the match is exact
left_sides = []
for m in left_sides_match:
if m['key'] == key:
left_sides.append(m)
right_sides_match = bibknowledge.get_kb_mappings("", "", value)
right_sides = []
for m in right_sides_match:
if m['value'] == value:
right_sides.append(m)
if (len(right_sides) == 0) and (len(left_sides) == 0):
#no problems, just add in current
forcetype = "curr"
#likewise, if this is a taxonomy, just pass on
if kb_type == 't':
forcetype = "curr"
if forcetype and not forcetype == "no":
pass
else:
if len(left_sides) > 0:
return page(title=_("Left side exists"),
body = bibknowledgeadminlib.perform_request_verify_rule(ln, kb_id, key, value, "left", kb_name, left_sides),
language=ln,
navtrail = navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
if len(right_sides) > 0:
return page(title=_("Right side exists"),
body = bibknowledgeadminlib.perform_request_verify_rule(ln, kb_id, key, value, "right", kb_name, right_sides),
language=ln,
navtrail = navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
if forcetype == "curr":
bibknowledge.add_kb_mapping(kb_name, key, value)
if forcetype == "all":
#a bit tricky.. remove the rules given in param replacement and add the current
#rule in the same kb's
if replacements:
#"replacements" can be either a string or an array. Let's make it always an array
if type(replacements) == type("this is a string"):
mystr = replacements
replacements = []
replacements.append(mystr)
for r in replacements:
if r.find("++++") > 0:
(rkbname, rleft, dummy) = r.split('++++')
bibknowledge.remove_kb_mapping(rkbname, rleft)
#add only if this is not yet there..
if not bibknowledge.kb_mapping_exists(rkbname, key):
bibknowledge.add_kb_mapping(rkbname, key, value)
redirect_to_url(req, "kb?ln=%(ln)s&kb=%(kb)s&sortby=%(sortby)s&kb_type=%(kb_type)s" % {'ln':ln,
'kb':kb_id,
'sortby':sortby,
'kb_type':kb_type})
else:
return page_not_authorized(req=req,
text=auth_msg,
navtrail=navtrail_previous_links)
def inner(dummy_form, dummy_field, term, limit=50):
result = get_kb_mappings(name, '', term)[:limit]
return map(mapper, result) if mapper is not None else result
def load_kbs(cfg, run_sql, in_task=False):
for kb, query in cfg.iteritems():
task_sleep_now_if_required(can_stop_too=True)
if not kb_exists(kb):
add_kb(kb)
if in_task:
write_message("Updating %s KB..." % kb)
try:
if not in_task:
print "kb:", kb
print "kb beginning:", len(get_kb_mappings(kb))
if kb.startswith('json_'):
encoder = ComplexEncoder()
mapping, description = run_sql(query, with_desc=True)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
column_counter = {}
new_description = []
for column in description[1:]:
column = column[0]
counter = column_counter[column] = column_counter.get(
column, 0) + 1
if counter > 1:
new_description.append('%s%d' % (column, counter))
else:
new_description.append(column)
description = new_description
else:
mapping = run_sql(query)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print "mapping:", len(mapping)
if kb == 'projects':
mapping += [('000000', 'NO PROJECT')]
original_keys = set([key[0] for key in get_kbr_keys(kb)])
if not in_task:
print "original_keys before:", len(original_keys)
updated = 0
added = 0
for i, row in enumerate(mapping):
key, value = row[0], row[1:]
if kb.startswith('json_'):
value = encoder.encode(dict(zip(description, value)))
else:
value = value[0]
if value:
if key in original_keys:
original_keys.remove(key)
if in_task:
task_update_progress("%s - %s%%" %
(kb, i * 100 / len(mapping)))
if kb_mapping_exists(kb, key):
updated += 1
update_kb_mapping(kb, key, key, value)
else:
added += 1
add_kb_mapping(kb, key, value)
if not in_task:
print "updated:", updated, "added:", added
print "kb after update:", len(get_kb_mappings(kb))
print "original_keys after:", len(original_keys)
if in_task:
task_update_progress("Cleaning %s" % kb)
for key in original_keys:
remove_kb_mapping(kb, key)
if not in_task:
print "kb after remove:", len(get_kb_mappings(kb))
except:
register_exception(alert_admin=True,
prefix="Error when updating KB %s" % kb)
continue
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
"""
Expand single letter subjects - get mapping from KB
"""
from invenio.bibknowledge import get_kb_mappings
from invenio.config import CFG_BIBEDIT_KB_SUBJECTS
subjects = {}
for item in get_kb_mappings(kb_name=CFG_BIBEDIT_KB_SUBJECTS):
subjects[item['key']] = item['value']
def check_record(record):
"""
Expand single letter subjects
"""
for position, value in record.iterfield("65017a", ("2", "INSPIRE")):
if len(value) == 1:
if subjects.has_key(value):
record.amend_field(position[:3], subjects[value])
else:
record.warn('single letter subject without mapping: %s' % value)
def format_element(bfo, separator=' | ', link="yes"):
"""
Prints Conference info as best is possible.
@param link if yes (default) prints link to SPIRES conference info
@param separator separates multiple conferences
"""
authors = bfo.fields('084__')
# Process authors to add link, highlight and format affiliation
output = []
output1 = []
pdgcount = 0
link = ""
pdgcode = ""
for exp in authors:
if exp.get('9') == 'PDG' and 'a' in exp:
values = get_kb_mappings('PDG', key=exp['a'], match_type="e")
pdgcode = exp['a']
for ch in [':', '=']:
if ch in pdgcode:
pdgcode = pdgcode.replace(ch, '/')
if values:
search_link = '<a href="http://pdglive.lbl.gov/view/{0}">'.format(pdgcode)
if values[0]['value'] == "THE TEXT IS MISSING FOR THIS NODE.":
search_link += pdgcode + ' (Title Unknown)'
else:
search_link += cgi.escape(values[0]['value'])
search_link += '</a>'
pdgcount += 1
if pdgcount < 3:
output.append(search_link)
else:
output1.append(search_link)
if len(output1):
link = """ | <a href="#" style="color:green;background:white;" onclick="toggle2('content', this)"><i>More</i></a>
<div id="content" style="display:none; padding-left:36px;">
%(content)s
</div>
<script type="text/javascript">
function toggle2(id, link) {
var e = document.getElementById(id);
if (e.style.display == '') {
e.style.display = 'none';
link.innerHTML = '<i>More</i>';
}
else
{
e.style.display = '';
link.innerHTML = '<i>Less</i>';
}
}
</script>
""" % {'content': separator.join(output1)}
return separator.join(output) + link
def kb_add_mapping(req,
kb,
mapFrom,
mapTo,
sortby="to",
ln=CFG_SITE_LANG,
forcetype=None,
replacements=None,
kb_type=None):
"""
Adds a new mapping to a kb.
@param ln language
@param kb the kb id to show
@param sortby the sorting criteria ('from' or 'to')
@param forcetype indicates if this function should ask about replacing left/right sides (None or 'no')
replace in current kb ('curr') or in all ('all')
@param replacements an object containing kbname+++left+++right strings.
Can be a string or an array of strings
@param kb_type None for normal from-to kb's, 't' for taxonomies
"""
ln = wash_language(ln)
_ = gettext_set_language(ln)
navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (
CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases"))
try:
dummy = getUid(req)
except MySQLdb.Error:
return error_page(req)
(auth_code, auth_msg) = check_user(req, 'cfgbibknowledge')
if not auth_code:
kb_id = wash_url_argument(kb, 'int')
kb_name = bibknowledge.get_kb_name(kb_id)
if kb_name is None:
return page(title=_("Unknown Knowledge Base"),
body="",
language=ln,
navtrail=navtrail_previous_links,
errors=[("ERR_KB_ID_UNKNOWN", kb)],
lastupdated=__lastupdated__,
req=req)
key = wash_url_argument(mapFrom, 'str')
value = wash_url_argument(mapTo, 'str')
#check if key or value already exists in some KB
left_sides_match = bibknowledge.get_kb_mappings("", key, "")
#check that the match is exact
left_sides = []
for m in left_sides_match:
if m['key'] == key:
left_sides.append(m)
right_sides_match = bibknowledge.get_kb_mappings("", "", value)
right_sides = []
for m in right_sides_match:
if m['value'] == value:
right_sides.append(m)
if (len(right_sides) == 0) and (len(left_sides) == 0):
#no problems, just add in current
forcetype = "curr"
#likewise, if this is a taxonomy, just pass on
if kb_type == 't':
forcetype = "curr"
if forcetype and not forcetype == "no":
pass
else:
if len(left_sides) > 0:
return page(
title=_("Left side exists"),
body=bibknowledgeadminlib.perform_request_verify_rule(
ln, kb_id, key, value, "left", kb_name, left_sides),
language=ln,
navtrail=navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
if len(right_sides) > 0:
return page(
title=_("Right side exists"),
body=bibknowledgeadminlib.perform_request_verify_rule(
ln, kb_id, key, value, "right", kb_name, right_sides),
language=ln,
navtrail=navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
if forcetype == "curr":
bibknowledge.add_kb_mapping(kb_name, key, value)
if forcetype == "all":
#a bit tricky.. remove the rules given in param replacement and add the current
#rule in the same kb's
if replacements:
#"replacements" can be either a string or an array. Let's make it always an array
if type(replacements) == type("this is a string"):
mystr = replacements
replacements = []
replacements.append(mystr)
for r in replacements:
if r.find("++++") > 0:
(rkbname, rleft, dummy) = r.split('++++')
bibknowledge.remove_kb_mapping(rkbname, rleft)
#add only if this is not yet there..
if not bibknowledge.kb_mapping_exists(rkbname, key):
bibknowledge.add_kb_mapping(rkbname, key, value)
redirect_to_url(
req,
"kb?ln=%(ln)s&kb=%(kb)s&sortby=%(sortby)s&kb_type=%(kb_type)s" % {
'ln': ln,
'kb': kb_id,
'sortby': sortby,
'kb_type': kb_type
})
else:
return page_not_authorized(req=req,
text=auth_msg,
navtrail=navtrail_previous_links)
def kb_export(req,
kbname="",
format="kbr",
searchkey="",
searchvalue="",
searchtype="s",
limit=None,
ln=CFG_SITE_LANG):
"""
Exports the given kb so that it is listed in stdout (the browser).
@param req the request
@param kbname knowledge base name
@param expression evaluate this for the returned lines
@param format 'kba' for authority file, 'kbr' for leftside-rightside, json
for json-formatted dictionaries
@param searchkey include only lines that match this on the left side
@param searchvalue include only lines that match this on the right side
@param searchtype s = substring match, e = exact match
@param limit how many results to return. None means all
@param ln language
"""
ln = wash_language(ln)
_ = gettext_set_language(ln)
navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (
CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases"))
if not kbname:
return page(title=_("Knowledge base name missing"),
body="""Required parameter kbname
is missing.""",
language=ln,
navtrail=navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
#in order to make 'wget' downloads easy we do not require authorization
#first check the type of the KB
kbtype = None
kbinfo = None
kbid = None
kbinfos = bibknowledge.get_kbs_info("", kbname)
if kbinfos:
kbinfo = kbinfos[0]
kbtype = kbinfo['kbtype']
kbid = kbinfo['id']
else:
return page(title=_("Unknown knowledge base"),
body=_("There is no knowledge base with that name."),
language=ln,
navtrail=navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
if kbtype == None or kbtype == 'w':
if format and format == "ejson":
req.content_type = 'application/json'
return bibknowledge.get_kb_mappings_embedded_json(kbname, searchkey, \
searchvalue, searchtype, limit)
elif format and format[0] == 'j':
# as JSON formatted string
req.content_type = 'application/json'
return bibknowledge.get_kb_mappings_json(kbname, searchkey, \
searchvalue, searchtype, limit)
# left side / right side KB
mappings = bibknowledge.get_kb_mappings(kbname, searchkey, \
searchvalue, searchtype)
if format == 'right' or format == 'kba':
# as authority sequence
seq = [m['value'] for m in mappings]
seq = uniq(sorted(seq))
for s in seq:
req.write(s + "\n")
return
else:
# as regularly formatted left-right mapping
for m in mappings:
req.write(m['key'] + '---' + m['value'] + '\n')
return
elif kbtype == 'd':
# dynamic kb, another interface for perform_request_search
if format and format[0] == 'j':
req.content_type = "application/json"
return bibknowledge.get_kbd_values_json(kbname, searchvalue)
else:
# print it as a list of values
for hit in bibknowledge.get_kbd_values(kbname, searchvalue):
req.write(hit + '\n')
req.write('\n')
return
elif kbtype == 't': #taxonomy: output the file
kbfilename = CFG_WEBDIR + "/kbfiles/" + str(kbid) + ".rdf"
try:
f = open(kbfilename, 'r')
for line in f:
req.write(line)
f.close()
except:
req.write("Reading the file " + kbfilename + " failed.")
else:
# This situation should never happen
raise ValueError, "Unsupported KB Type: %s" % kbtype
def get_journal_info(record, tags):
"""Fetch journal info from given record"""
record_info = []
journals_fields = record.find_fields(tags['publication']['journal'][:5])
for field in journals_fields:
# we store the tags and their values here
# like c->444 y->1999 p->"journal of foo",
# v->20
tagsvalues = {}
try:
tmp = field.get_subfield_values(tags['publication']['journal'][5])[0]
except IndexError:
pass
else:
tagsvalues["p"] = tmp
try:
tmp = field.get_subfield_values(tags['publication']['volume'][5])[0]
except IndexError:
pass
else:
tagsvalues["v"] = tmp
try:
tmp = field.get_subfield_values(tags['publication']['year'][5])[0]
except IndexError:
pass
else:
tagsvalues["y"] = tmp
try:
tmp = field.get_subfield_values(tags['publication']['pages'][5])[0]
except IndexError:
pass
else:
# if the page numbers have "x-y" take just x
tagsvalues["c"] = tmp.split('-', 1)[0]
# check if we have the required data
ok = True
for c in tags['publication_format']:
if c in ('p', 'v', 'y', 'c'):
if c not in tagsvalues:
ok = False
if ok:
publ = format_journal(tags['publication_format'], tagsvalues)
record_info += [publ]
alt_volume = get_alt_volume(tagsvalues['v'])
if alt_volume:
tagsvalues2 = tagsvalues.copy()
tagsvalues2['v'] = alt_volume
publ = format_journal(tags['publication_format'], tagsvalues2)
record_info += [publ]
# Add codens
for coden in get_kb_mappings('CODENS',
value=tagsvalues['p']):
tagsvalues2 = tagsvalues.copy()
tagsvalues2['p'] = coden['key']
publ = format_journal(tags['publication_format'], tagsvalues2)
record_info += [publ]
return record_info
def kb_export(req, kbname="", format="kbr", searchkey="", searchvalue="", searchtype="s", limit=None, ln=CFG_SITE_LANG):
"""
Exports the given kb so that it is listed in stdout (the browser).
@param req the request
@param kbname knowledge base name
@param expression evaluate this for the returned lines
@param format 'kba' for authority file, 'kbr' for leftside-rightside, json
for json-formatted dictionaries
@param searchkey include only lines that match this on the left side
@param searchvalue include only lines that match this on the right side
@param searchtype s = substring match, e = exact match
@param limit how many results to return. None means all
@param ln language
"""
ln = wash_language(ln)
_ = gettext_set_language(ln)
navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases"))
if not kbname:
return page(title=_("Knowledge base name missing"),
body = """Required parameter kbname
is missing.""",
language=ln,
navtrail = navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
#in order to make 'wget' downloads easy we do not require authorization
#first check the type of the KB
kbtype = None
kbinfo = None
kbid = None
kbinfos = bibknowledge.get_kbs_info("", kbname)
if kbinfos:
kbinfo = kbinfos[0]
kbtype = kbinfo['kbtype']
kbid = kbinfo['id']
else:
return page(title=_("Unknown knowledge base"),
body = _("There is no knowledge base with that name."),
language=ln,
navtrail = navtrail_previous_links,
lastupdated=__lastupdated__,
req=req)
if not kbtype or kbtype == 'w':
if format and format == "ejson":
req.content_type = 'application/json'
return bibknowledge.get_kb_mappings_embedded_json(kbname, searchkey, \
searchvalue, searchtype, limit)
elif format and format[0] == 'j':
# as JSON formatted string
req.content_type = 'application/json'
return bibknowledge.get_kb_mappings_json(kbname, searchkey, \
searchvalue, searchtype, limit)
# left side / right side KB
mappings = bibknowledge.get_kb_mappings(kbname, searchkey, \
searchvalue, searchtype)
if format == 'right' or format == 'kba':
# as authority sequence
seq = [m['value'] for m in mappings]
seq = uniq(sorted(seq))
for s in seq:
req.write(s+"\n");
return
else:
# as regularly formatted left-right mapping
for m in mappings:
req.write(m['key'] + '---' + m['value'] + '\n')
return
elif kbtype == 'd':
# dynamic kb, another interface for perform_request_search
if format and format[0] == 'j':
req.content_type = "application/json"
return bibknowledge.get_kbd_values_json(kbname, searchvalue)
else:
# print it as a list of values
for hit in bibknowledge.get_kbd_values(kbname, searchvalue):
req.write(hit + '\n')
req.write('\n')
return
elif kbtype == 't': #taxonomy: output the file
kbfilename = CFG_WEBDIR+"/kbfiles/"+str(kbid)+".rdf"
try:
f = open(kbfilename, 'r')
for line in f:
req.write(line)
f.close()
except:
req.write("Reading the file "+kbfilename+" failed.")
else:
# This situation should never happen
raise ValueError, "Unsupported KB Type: %s" % kbtype
def get_journal_info(record, tags):
"""Fetch journal info from given record"""
record_info = []
journals_fields = record.find_fields(tags['publication']['journal'][:5])
for field in journals_fields:
# we store the tags and their values here
# like c->444 y->1999 p->"journal of foo",
# v->20
tagsvalues = {}
try:
tmp = field.get_subfield_values(
tags['publication']['journal'][5])[0]
except IndexError:
pass
else:
tagsvalues["p"] = tmp
try:
tmp = field.get_subfield_values(
tags['publication']['volume'][5])[0]
except IndexError:
pass
else:
tagsvalues["v"] = tmp
try:
tmp = field.get_subfield_values(tags['publication']['year'][5])[0]
except IndexError:
pass
else:
tagsvalues["y"] = tmp
try:
tmp = field.get_subfield_values(tags['publication']['pages'][5])[0]
except IndexError:
pass
else:
# if the page numbers have "x-y" take just x
tagsvalues["c"] = tmp.split('-', 1)[0]
# check if we have the required data
ok = True
for c in tags['publication_format']:
if c in ('p', 'v', 'y', 'c'):
if c not in tagsvalues:
ok = False
if ok:
publ = format_journal(tags['publication_format'], tagsvalues)
record_info += [publ]
alt_volume = get_alt_volume(tagsvalues['v'])
if alt_volume:
tagsvalues2 = tagsvalues.copy()
tagsvalues2['v'] = alt_volume
publ = format_journal(tags['publication_format'], tagsvalues2)
record_info += [publ]
# Add codens
for coden in get_kb_mappings('CODENS', value=tagsvalues['p']):
tagsvalues2 = tagsvalues.copy()
tagsvalues2['p'] = coden['key']
publ = format_journal(tags['publication_format'], tagsvalues2)
record_info += [publ]
return record_info
