def kb_edit_mapping(req, kb, key, mapFrom, mapTo, update="", delete="", sortby="to", ln=CFG_SITE_LANG):
"""
Edit a mapping to in kb. Edit can be "update old value" or "delete existing value"
@param kb the knowledge base id to edit
@param key the key of the mapping that will be modified
@param mapFrom the new key of the mapping
@param mapTo the new value of the mapping
@param update contains a value if the mapping is to be updated
@param delete contains a value if the mapping is to be deleted
@param sortby the sorting criteria ('from' or 'to')
"""
ln = wash_language(ln)
_ = gettext_set_language(ln)
navtrail_previous_links = """ > <a class="navtrail" href="%s/kb?ln=%s">%s</a>""" % (
CFG_SITE_SECURE_URL,
ln,
_("Manage Knowledge Bases"),
)
try:
dummy = getUid(req)
except:
return error_page("Error", req)
(auth_code, auth_msg) = check_user(req, "cfgbibknowledge")
if not auth_code:
kb_id = wash_url_argument(kb, "int")
kb_name = bibknowledge.get_kb_name(kb_id)
if kb_name is None:
return page(
title=_("Unknown Knowledge Base"),
body="",
language=ln,
navtrail=navtrail_previous_links,
errors=[("ERR_KB_ID_UNKNOWN", kb)],
lastupdated=__lastupdated__,
req=req,
)
key = wash_url_argument(key, "str")
if delete != "":
# Delete
bibknowledge.remove_kb_mapping(kb_name, key)
if update != "":
# Update
new_key = wash_url_argument(mapFrom, "str")
new_value = wash_url_argument(mapTo, "str")
bibknowledge.update_kb_mapping(kb_name, key, new_key, new_value)
redirect_to_url(req, "kb?ln=%(ln)s&kb=%(kb)s&sortby=%(sortby)s" % {"ln": ln, "kb": kb_id, "sortby": sortby})
else:
return page_not_authorized(req=req, text=auth_msg, navtrail=navtrail_previous_links)
def update_kb(kb_name, data, key_fun, value_fun=lambda x: x, update=False):
"""Update a knowledge base from data."""
# Memory greedy, but faster than many individual SQL queries.
mappings = dict([(d['key'], d['value']) for d in get_kb_mappings(kb_name)])
for item in data:
k = key_fun(item)
v = json.dumps(value_fun(item))
if k not in mappings:
add_kb_mapping(kb_name, k, v)
elif update and mappings[k] != v:
update_kb_mapping(kb_name, k, k, v)
def update_kb(kb_name, data, key_fun, value_fun=lambda x: x, update=False):
"""Update a knowledge base from data."""
# Memory greedy, but faster than many individual SQL queries.
mappings = dict([
(d['key'], d['value']) for d in get_kb_mappings(kb_name)
])
for item in data:
k = key_fun(item)
v = json.dumps(value_fun(item))
if k not in mappings:
add_kb_mapping(kb_name, k, v)
elif update and mappings[k] != v:
update_kb_mapping(kb_name, k, k, v)
def load_kbs(cfg, run_sql, in_task=False):
for kb, query in cfg.iteritems():
task_sleep_now_if_required(can_stop_too=True)
if not kb_exists(kb):
add_kb(kb)
if in_task:
write_message("Updating %s KB..." % kb)
try:
if not in_task:
print "kb:", kb
print "kb beginning:", len(get_kb_mappings(kb))
if kb.startswith('json_'):
encoder = ComplexEncoder()
mapping, description = run_sql(query, with_desc=True)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
column_counter = {}
new_description = []
for column in description[1:]:
column = column[0]
counter = column_counter[column] = column_counter.get(
column, 0) + 1
if counter > 1:
new_description.append('%s%d' % (column, counter))
else:
new_description.append(column)
description = new_description
else:
mapping = run_sql(query)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print "mapping:", len(mapping)
if kb == 'projects':
mapping += [('000000', 'NO PROJECT')]
original_keys = set([key[0] for key in get_kbr_keys(kb)])
if not in_task:
print "original_keys before:", len(original_keys)
updated = 0
added = 0
for i, row in enumerate(mapping):
key, value = row[0], row[1:]
if kb.startswith('json_'):
value = encoder.encode(dict(zip(description, value)))
else:
value = value[0]
if value:
if key in original_keys:
original_keys.remove(key)
if in_task:
task_update_progress("%s - %s%%" %
(kb, i * 100 / len(mapping)))
if kb_mapping_exists(kb, key):
updated += 1
update_kb_mapping(kb, key, key, value)
else:
added += 1
add_kb_mapping(kb, key, value)
if not in_task:
print "updated:", updated, "added:", added
print "kb after update:", len(get_kb_mappings(kb))
print "original_keys after:", len(original_keys)
if in_task:
task_update_progress("Cleaning %s" % kb)
for key in original_keys:
remove_kb_mapping(kb, key)
if not in_task:
print "kb after remove:", len(get_kb_mappings(kb))
except:
register_exception(alert_admin=True,
prefix="Error when updating KB %s" % kb)
continue
def load_kbs(cfg, run_sql, in_task=False):
for kb, query in cfg.iteritems():
task_sleep_now_if_required(can_stop_too=True)
if not kb_exists(kb):
add_kb(kb)
if in_task:
write_message("Updating %s KB..." % kb)
try:
if not in_task:
print "kb:", kb
print "kb beginning:", len(get_kb_mappings(kb))
if kb.startswith('json_'):
encoder = ComplexEncoder()
mapping, description = run_sql(query, with_desc=True)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
column_counter = {}
new_description = []
for column in description[1:]:
column = column[0]
counter = column_counter[
column] = column_counter.get(column, 0) + 1
if counter > 1:
new_description.append('%s%d' % (column, counter))
else:
new_description.append(column)
description = new_description
else:
mapping = run_sql(query)
if kb in CFG_ADDITIONAL_ENTRIES:
mapping += CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print CFG_ADDITIONAL_ENTRIES[kb]
if not in_task:
print "mapping:", len(mapping)
if kb == 'projects':
mapping += [('000000', 'NO PROJECT')]
original_keys = set([key[0] for key in get_kbr_keys(kb)])
if not in_task:
print "original_keys before:", len(original_keys)
updated = 0
added = 0
for i, row in enumerate(mapping):
key, value = row[0], row[1:]
if kb.startswith('json_'):
value = encoder.encode(dict(zip(description, value)))
else:
value = value[0]
if value:
if key in original_keys:
original_keys.remove(key)
if in_task:
task_update_progress(
"%s - %s%%" % (kb, i * 100 / len(mapping)))
if kb_mapping_exists(kb, key):
updated += 1
update_kb_mapping(kb, key, key, value)
else:
added += 1
add_kb_mapping(kb, key, value)
if not in_task:
print "updated:", updated, "added:", added
print "kb after update:", len(get_kb_mappings(kb))
print "original_keys after:", len(original_keys)
if in_task:
task_update_progress("Cleaning %s" % kb)
for key in original_keys:
remove_kb_mapping(kb, key)
if not in_task:
print "kb after remove:", len(get_kb_mappings(kb))
except:
register_exception(
alert_admin=True, prefix="Error when updating KB %s" % kb)
continue
def kb_edit_mapping(req,
kb,
key,
mapFrom,
mapTo,
update="",
delete="",
sortby="to",
ln=CFG_SITE_LANG):
"""
Edit a mapping to in kb. Edit can be "update old value" or "delete existing value"
@param kb the knowledge base id to edit
@param key the key of the mapping that will be modified
@param mapFrom the new key of the mapping
@param mapTo the new value of the mapping
@param update contains a value if the mapping is to be updated
@param delete contains a value if the mapping is to be deleted
@param sortby the sorting criteria ('from' or 'to')
"""
ln = wash_language(ln)
_ = gettext_set_language(ln)
navtrail_previous_links = ''' > <a class="navtrail" href="%s/kb?ln=%s">%s</a>''' % (
CFG_SITE_SECURE_URL, ln, _("Manage Knowledge Bases"))
try:
dummy = getUid(req)
except:
return error_page('Error', req)
(auth_code, auth_msg) = check_user(req, 'cfgbibknowledge')
if not auth_code:
kb_id = wash_url_argument(kb, 'int')
kb_name = bibknowledge.get_kb_name(kb_id)
if kb_name is None:
return page(title=_("Unknown Knowledge Base"),
body="",
language=ln,
navtrail=navtrail_previous_links,
errors=[("ERR_KB_ID_UNKNOWN", kb)],
lastupdated=__lastupdated__,
req=req)
key = wash_url_argument(key, 'str')
if delete != "":
#Delete
bibknowledge.remove_kb_mapping(kb_name, key)
if update != "":
#Update
new_key = wash_url_argument(mapFrom, 'str')
new_value = wash_url_argument(mapTo, 'str')
bibknowledge.update_kb_mapping(kb_name, key, new_key, new_value)
redirect_to_url(
req, "kb?ln=%(ln)s&kb=%(kb)s&sortby=%(sortby)s" % {
'ln': ln,
'kb': kb_id,
'sortby': sortby
})
else:
return page_not_authorized(req=req,
text=auth_msg,
navtrail=navtrail_previous_links)