def write_to_methoddata_table(id_method, data_dict, data_dict_ordered, data_list_sorted, update_timestamp=True):
"""Serialize the date and write it to the bsrMETHODDATA"""
write_message('Starting serializing the data..', verbose=5)
serialized_data_dict = serialize_via_marshal(data_dict)
serialized_data_dict_ordered = serialize_via_marshal(data_dict_ordered)
serialized_data_list_sorted = serialize_via_marshal(data_list_sorted)
write_message('Serialization completed.', verbose=5)
date = strftime("%Y-%m-%d %H:%M:%S", time.localtime())
if not update_timestamp:
try:
date = run_sql('SELECT last_updated from bsrMETHODDATA WHERE id_bsrMETHOD = %s', (id_method, ))[0][0]
except IndexError:
pass # keep the generated date
write_message("Starting writing the data for method_id=%s " \
"to the database (table bsrMETHODDATA)" %id_method, verbose=5)
try:
write_message('Deleting old data..', verbose=5)
run_sql("DELETE FROM bsrMETHODDATA WHERE id_bsrMETHOD = %s", (id_method, ))
write_message('Inserting new data..', verbose=5)
run_sql("INSERT into bsrMETHODDATA \
(id_bsrMETHOD, data_dict, data_dict_ordered, data_list_sorted, last_updated) \
VALUES (%s, %s, %s, %s, %s)", \
(id_method, serialized_data_dict, serialized_data_dict_ordered, \
serialized_data_list_sorted, date, ))
except Error as err:
write_message("The error [%s] occured when inserting new bibsort data "\
"into bsrMETHODATA table" %err, sys.stderr)
return False
write_message('Writing to the bsrMETHODDATA successfully completed.', \
verbose=5)
return True
def insert_into_cit_db(dic, name):
"""Stores citation dictionary in the database"""
ndate = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
s = serialize_via_marshal(dic)
write_message("size of %s %s" % (name, len(s)))
# check that this column really exists
run_sql("""REPLACE INTO rnkCITATIONDATA(object_name, object_value,
last_updated) VALUES (%s, %s, %s)""", (name, s, ndate))
def get_recstruct_record(recid):
value = serialize_via_marshal(get_record(recid))
b = Bibfmt(id_bibrec=recid,
format='recstruct',
last_updated=db.func.now(),
value=value)
db.session.add(b)
db.session.commit()
def intoDB(dict, date, rank_method_code):
"""Insert the rank method data into the database"""
mid = run_sql("SELECT id from rnkMETHOD where name=%s", (rank_method_code, ))
del_rank_method_codeDATA(rank_method_code)
serdata = serialize_via_marshal(dict);
midstr = str(mid[0][0]);
run_sql("INSERT INTO rnkMETHODDATA(id_rnkMETHOD, relevance_data) VALUES (%s,%s)", (midstr, serdata,))
if date:
run_sql("UPDATE rnkMETHOD SET last_updated=%s WHERE name=%s", (date, rank_method_code))
def do_upgrade():
rows_to_change = run_sql("SELECT id, arguments FROM oaiHARVEST", with_dict=True)
# Move away from old columns
for row in rows_to_change:
if row['arguments']:
arguments = deserialize_via_marshal(row['arguments'])
if "c_cfg-file" in arguments:
arguments['c_stylesheet'] = arguments['c_cfg-file']
del arguments['c_cfg-file']
run_sql("UPDATE oaiHARVEST set arguments=%s WHERE id=%s",
(serialize_via_marshal(arguments), row['id']))
def into_db(dict_of_ranks, rank_method_code):
"""Writes into the rnkMETHODDATA table the ranking results"""
method_id = run_sql("SELECT id from rnkMETHOD where name=%s", \
(rank_method_code, ))
del_rank_method_data(rank_method_code)
serialized_data = serialize_via_marshal(dict_of_ranks)
method_id_str = str(method_id[0][0])
run_sql("INSERT INTO rnkMETHODDATA(id_rnkMETHOD, relevance_data) \
VALUES(%s, %s) ", (method_id_str, serialized_data, ))
date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
run_sql("UPDATE rnkMETHOD SET last_updated=%s WHERE name=%s", \
(date, rank_method_code))
write_message("Finished writing the ranks into rnkMETHOD table", verbose=5)
def write_to_methoddata_table(id_method,
data_dict,
data_dict_ordered,
data_list_sorted,
update_timestamp=True):
"""Serialize the date and write it to the bsrMETHODDATA"""
write_message('Starting serializing the data..', verbose=5)
serialized_data_dict = serialize_via_marshal(data_dict)
serialized_data_dict_ordered = serialize_via_marshal(data_dict_ordered)
serialized_data_list_sorted = serialize_via_marshal(data_list_sorted)
write_message('Serialization completed.', verbose=5)
date = strftime("%Y-%m-%d %H:%M:%S", time.localtime())
if not update_timestamp:
try:
date = run_sql(
'SELECT last_updated from bsrMETHODDATA WHERE id_bsrMETHOD = %s',
(id_method, ))[0][0]
except IndexError:
pass # keep the generated date
write_message("Starting writing the data for method_id=%s " \
"to the database (table bsrMETHODDATA)" %id_method, verbose=5)
try:
write_message('Deleting old data..', verbose=5)
run_sql("DELETE FROM bsrMETHODDATA WHERE id_bsrMETHOD = %s",
(id_method, ))
write_message('Inserting new data..', verbose=5)
run_sql("INSERT into bsrMETHODDATA \
(id_bsrMETHOD, data_dict, data_dict_ordered, data_list_sorted, last_updated) \
VALUES (%s, %s, %s, %s, %s)" , \
(id_method, serialized_data_dict, serialized_data_dict_ordered, \
serialized_data_list_sorted, date, ))
except Error as err:
write_message("The error [%s] occured when inserting new bibsort data "\
"into bsrMETHODATA table" %err, sys.stderr)
return False
write_message('Writing to the bsrMETHODDATA successfully completed.', \
verbose=5)
return True
def intoDB(dic, date, rank_method_code):
"""Insert the rank method data into the database"""
mid = run_sql("SELECT id from rnkMETHOD where name=%s",
(rank_method_code, ))
del_rank_method_codeDATA(rank_method_code)
serdata = serialize_via_marshal(dic)
midstr = str(mid[0][0])
run_sql(
"INSERT INTO rnkMETHODDATA(id_rnkMETHOD, relevance_data) VALUES (%s,%s)",
(
midstr,
serdata,
))
if date:
run_sql("UPDATE rnkMETHOD SET last_updated=%s WHERE name=%s",
(date, rank_method_code))
def do_upgrade():
create_statement = run_sql('SHOW CREATE TABLE oaiHARVEST')[0][1]
if '`arguments` text' in create_statement:
run_sql("ALTER TABLE oaiHARVEST CHANGE arguments arguments blob")
# translate old values
if '`bibconvertcfgfile`' in create_statement:
rows_to_change = run_sql("SELECT id, bibconvertcfgfile, bibfilterprogram, arguments FROM oaiHARVEST", with_dict=True)
# Move away from old columns
for row in rows_to_change:
if row['arguments']:
arguments = deserialize_via_marshal(row['arguments'])
else:
arguments = {}
arguments['c_cfg-file'] = row['bibconvertcfgfile']
arguments['f_filter-file'] = row['bibfilterprogram']
run_sql("UPDATE oaiHARVEST set arguments=%s WHERE id=%s", (serialize_via_marshal(arguments), row['id']))
run_sql("ALTER TABLE oaiHARVEST DROP COLUMN bibconvertcfgfile")
run_sql("ALTER TABLE oaiHARVEST DROP COLUMN bibfilterprogram")
def do_upgrade():
create_statement = run_sql('SHOW CREATE TABLE oaiHARVEST')[0][1]
if '`arguments` text' in create_statement:
run_sql("ALTER TABLE oaiHARVEST CHANGE arguments arguments blob")
# translate old values
if '`bibconvertcfgfile`' in create_statement:
rows_to_change = run_sql(
"SELECT id, bibconvertcfgfile, bibfilterprogram, arguments FROM oaiHARVEST",
with_dict=True)
# Move away from old columns
for row in rows_to_change:
if row['arguments']:
arguments = deserialize_via_marshal(row['arguments'])
else:
arguments = {}
arguments['c_cfg-file'] = row['bibconvertcfgfile']
arguments['f_filter-file'] = row['bibfilterprogram']
run_sql("UPDATE oaiHARVEST set arguments=%s WHERE id=%s",
(serialize_via_marshal(arguments), row['id']))
run_sql("ALTER TABLE oaiHARVEST DROP COLUMN bibconvertcfgfile")
run_sql("ALTER TABLE oaiHARVEST DROP COLUMN bibfilterprogram")
def store_weights_cache(weights):
"""Store into key/value store"""
cache.set('citations_weights', serialize_via_marshal(weights))
def store_weights_cache(weights):
"""Store into key/value store"""
cache.set('selfcites_weights', serialize_via_marshal(weights))
def store_weights_cache(weights):
"""Store into key/value store"""
redis = get_redis()
redis.set("citations_weights", serialize_via_marshal(weights))
def get_recstruct_record(recid):
value = serialize_via_marshal(get_record(recid))
b = Bibfmt(id_bibrec=recid, format='recstruct',
last_updated=db.func.now(), value=value)
db.session.add(b)
db.session.commit()
def store_weights_cache(weights):
"""Store into key/value store"""
redis = get_redis()
redis.set('selfcites_weights', serialize_via_marshal(weights))
