def write_to_buckets_table(id_method,
bucket_no,
bucket_data,
bucket_last_value,
update_timestamp=True):
"""Serialize the date and write it to the bsrMEHODDATA_BUCKETS"""
write_message('Writing the data for bucket number %s for ' \
'method_id=%s to the database' \
%(bucket_no, id_method), verbose=5)
write_message('Serializing data for bucket number %s' % bucket_no,
verbose=5)
serialized_bucket_data = bucket_data.fastdump()
date = strftime("%Y-%m-%d %H:%M:%S", time.localtime())
if not update_timestamp:
try:
date = run_sql('SELECT last_update from bsrMETHODDATABUCKET WHERE id_bsrMETHOD = %s and bucket_no = %s', \
(id_method, bucket_no))[0][0]
except IndexError:
pass # keep the generated date
try:
write_message('Deleting old data.', verbose=5)
run_sql("DELETE FROM bsrMETHODDATABUCKET \
WHERE id_bsrMETHOD = %s AND bucket_no = %s" , \
(id_method, bucket_no, ))
write_message('Inserting new data.', verbose=5)
run_sql("INSERT into bsrMETHODDATABUCKET \
(id_bsrMETHOD, bucket_no, bucket_data, bucket_last_value, last_updated) \
VALUES (%s, %s, %s, %s, %s)" , \
(id_method, bucket_no, serialized_bucket_data, bucket_last_value, date, ))
except Error, err:
write_message("The error [%s] occured when inserting new bibsort data " \
"into bsrMETHODATA_BUCKETS table" %err, sys.stderr)
return False
def write_to_methoddata_table(id_method,
data_dict,
data_dict_ordered,
data_list_sorted,
update_timestamp=True):
"""Serialize the date and write it to the bsrMETHODDATA"""
write_message('Starting serializing the data..', verbose=5)
serialized_data_dict = serialize_via_marshal(data_dict)
serialized_data_dict_ordered = serialize_via_marshal(data_dict_ordered)
serialized_data_list_sorted = serialize_via_marshal(data_list_sorted)
write_message('Serialization completed.', verbose=5)
date = strftime("%Y-%m-%d %H:%M:%S", time.localtime())
if not update_timestamp:
try:
date = run_sql(
'SELECT last_update from bsrMETHODDATA WHERE id_bsrMETHOD = %s',
(id_method, ))[0][0]
except IndexError:
pass # keep the generated date
write_message("Starting writing the data for method_id=%s " \
"to the database (table bsrMETHODDATA)" %id_method, verbose=5)
try:
write_message('Deleting old data..', verbose=5)
run_sql("DELETE FROM bsrMETHODDATA WHERE id_bsrMETHOD = %s",
(id_method, ))
write_message('Inserting new data..', verbose=5)
run_sql("INSERT into bsrMETHODDATA \
(id_bsrMETHOD, data_dict, data_dict_ordered, data_list_sorted, last_updated) \
VALUES (%s, %s, %s, %s, %s)" , \
(id_method, serialized_data_dict, serialized_data_dict_ordered, \
serialized_data_list_sorted, date, ))
except Error, err:
write_message("The error [%s] occured when inserting new bibsort data "\
"into bsrMETHODATA table" %err, sys.stderr)
return False
def write_to_buckets_table(id_method, bucket_no, bucket_data, bucket_last_value, update_timestamp=True):
"""Serialize the date and write it to the bsrMEHODDATA_BUCKETS"""
write_message('Writing the data for bucket number %s for ' \
'method_id=%s to the database' \
%(bucket_no, id_method), verbose=5)
write_message('Serializing data for bucket number %s' %bucket_no, verbose=5)
serialized_bucket_data = bucket_data.fastdump()
date = strftime("%Y-%m-%d %H:%M:%S", time.localtime())
if not update_timestamp:
try:
date = run_sql('SELECT last_update from bsrMETHODDATABUCKET WHERE id_bsrMETHOD = %s and bucket_no = %s', \
(id_method, bucket_no))[0][0]
except IndexError:
pass # keep the generated date
try:
write_message('Deleting old data.', verbose=5)
run_sql("DELETE FROM bsrMETHODDATABUCKET \
WHERE id_bsrMETHOD = %s AND bucket_no = %s", \
(id_method, bucket_no, ))
write_message('Inserting new data.', verbose=5)
run_sql("INSERT into bsrMETHODDATABUCKET \
(id_bsrMETHOD, bucket_no, bucket_data, bucket_last_value, last_updated) \
VALUES (%s, %s, %s, %s, %s)", \
(id_method, bucket_no, serialized_bucket_data, bucket_last_value, date, ))
except Error, err:
write_message("The error [%s] occured when inserting new bibsort data " \
"into bsrMETHODATA_BUCKETS table" %err, sys.stderr)
return False
def write_to_methoddata_table(id_method, data_dict, data_dict_ordered, data_list_sorted, update_timestamp=True):
"""Serialize the date and write it to the bsrMETHODDATA"""
write_message('Starting serializing the data..', verbose=5)
serialized_data_dict = serialize_via_marshal(data_dict)
serialized_data_dict_ordered = serialize_via_marshal(data_dict_ordered)
serialized_data_list_sorted = serialize_via_marshal(data_list_sorted)
write_message('Serialization completed.', verbose=5)
date = strftime("%Y-%m-%d %H:%M:%S", time.localtime())
if not update_timestamp:
try:
date = run_sql('SELECT last_update from bsrMETHODDATA WHERE id_bsrMETHOD = %s', (id_method, ))[0][0]
except IndexError:
pass # keep the generated date
write_message("Starting writing the data for method_id=%s " \
"to the database (table bsrMETHODDATA)" %id_method, verbose=5)
try:
write_message('Deleting old data..', verbose=5)
run_sql("DELETE FROM bsrMETHODDATA WHERE id_bsrMETHOD = %s", (id_method, ))
write_message('Inserting new data..', verbose=5)
run_sql("INSERT into bsrMETHODDATA \
(id_bsrMETHOD, data_dict, data_dict_ordered, data_list_sorted, last_updated) \
VALUES (%s, %s, %s, %s, %s)", \
(id_method, serialized_data_dict, serialized_data_dict_ordered, \
serialized_data_list_sorted, date, ))
except Error, err:
write_message("The error [%s] occured when inserting new bibsort data "\
"into bsrMETHODATA table" %err, sys.stderr)
return False
def add_person_comment(person_id, message):
'''
Adds a comment to a person after enriching it with meta-data (date+time)
@param person_id: person id to assign the comment to
@type person_id: int
@param message: defines the comment to set
@type message: string
@return the message incl. the metadata if everything was fine, False on err
@rtype: string or boolean
'''
msg = ""
pid = -1
try:
msg = str(message)
pid = int(person_id)
except (ValueError, TypeError):
return False
strtimestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime())
msg = escape(msg, quote=True)
dbmsg = "%s;;;%s" % (strtimestamp, msg)
dbapi.set_person_data(pid, "comment", dbmsg)
return dbmsg
def check_records(records, date_fields=CFG_DEFAULT_DATE_FIELDS):
"""Corrects all dates!"""
for record in records:
for position, value in record.iterfields(date_fields):
for date_format in CFG_POSSIBLE_DATE_FORMATS:
try:
parsed_date = strftime("%Y-%m-%d",
(strptime(value, date_format)))
if position[0] in CFG_ONLY_YEAR_FIELDS:
parsed_date = parsed_date[:4]
if parsed_date != value:
record.amend_field(position, parsed_date)
break
except ValueError:
pass
else:
for date_format in CFG_POSSIBLE_DATE_FORMATS_ONLY_YEAR_MONTH:
try:
parsed_date = strftime("%Y-%m",
(strptime(value, date_format)))
if position[0] in CFG_ONLY_YEAR_FIELDS:
parsed_date = parsed_date[:4]
if parsed_date != value:
record.amend_field(position, parsed_date)
break
except ValueError:
pass
else:
for date_format in CFG_POSSIBLE_DATE_FORMATS_ONLY_YEAR:
try:
parsed_date = strftime(
"%Y", (strptime(value, date_format)))
if parsed_date != value:
record.amend_field(position, parsed_date)
break
except ValueError:
pass
else:
record.set_invalid(
"Cannot recognize date %s in position %s" %
(value, position))
def check_records(records, date_fields=CFG_DEFAULT_DATE_FIELDS):
"""Corrects all dates!"""
for record in records:
for position, value in record.iterfields(date_fields):
for date_format in CFG_POSSIBLE_DATE_FORMATS:
try:
parsed_date = strftime("%Y-%m-%d", (strptime(value, date_format)))
if position[0] in CFG_ONLY_YEAR_FIELDS:
parsed_date = parsed_date[:4]
if parsed_date != value:
record.amend_field(position, parsed_date)
break
except ValueError:
pass
else:
for date_format in CFG_POSSIBLE_DATE_FORMATS_ONLY_YEAR_MONTH:
try:
parsed_date = strftime("%Y-%m", (strptime(value, date_format)))
if position[0] in CFG_ONLY_YEAR_FIELDS:
parsed_date = parsed_date[:4]
if parsed_date != value:
record.amend_field(position, parsed_date)
break
except ValueError:
pass
else:
for date_format in CFG_POSSIBLE_DATE_FORMATS_ONLY_YEAR:
try:
parsed_date = strftime("%Y", (strptime(value, date_format)))
if parsed_date != value:
record.amend_field(position, parsed_date)
break
except ValueError:
pass
else:
record.set_invalid("Cannot recognize date %s in position %s" % (value, position))
def check_records(records, date_fields=CFG_DEFAULT_DATE_FIELDS):
"""
Backdate the earliest_date of a record to reflect the earliest date information
available.
Note in case of partial dates (e.g. just the year or just year-month), if there
is another valid date within the
"""
for record in records:
dates = []
recid = int(record["001"][0][3])
creation_date, modification_date, earliest_date = run_sql("SELECT creation_date, modification_date, earliest_date FROM bibrec WHERE id=%s", (recid, ))[0]
creation_date = strftime("%Y-%m-%d %H:%M:%S", creation_date)
modification_date = strftime("%Y-%m-%d %H:%M:%S", modification_date)
earliest_date = strftime("%Y-%m-%d %H:%M:%S", earliest_date)
dates.append(creation_date)
dates.append(modification_date)
if '005' in record:
dates.append(strftime("%Y-%m-%d %H:%M:%S", strptime(record["005"][0][3], "%Y%m%d%H%M%S.0")))
for position, value in record.iterfields(date_fields):
for format in CFG_POSSIBLE_DATE_FORMATS:
try:
parsed_date = strftime("%Y-%m-%d 00:00:00", (strptime(value, format)))
dates.append(parsed_date)
break
except ValueError:
pass
else:
for format in CFG_POSSIBLE_DATE_FORMATS_ONLY_YEAR_MONTH:
try:
parsed_date = strftime("%Y-%m-99 00:00:00", (strptime(value, format)))
dates.append(parsed_date)
break
except ValueError:
pass
else:
for format in CFG_POSSIBLE_DATE_FORMATS_ONLY_YEAR:
try:
parsed_date = strftime("%Y-99-99 00:00:00", (strptime(value, format)))
dates.append(parsed_date)
break
except ValueError:
pass
min_date = min(dates)
## Let's restore meaningful first month and day
min_date = min_date.replace("-99", "-01")
if min_date != earliest_date:
run_sql("UPDATE bibrec SET earliest_date=%s WHERE id=%s", (min_date, recid))
record.warn("record earliest_date amended from %s to %s" % (earliest_date, min_date))
def _sort_dates(self, val):
"""
Convert:
'8 nov 2010' => '2010-11-08'
'nov 2010' => '2010-11-01'
'2010' => '2010-01-01'
"""
datetext_format = "%Y-%m-%d"
try:
datestruct = strptime(val, datetext_format)
except ValueError:
try:
datestruct = strptime(val, "%d %b %Y")
except ValueError:
try:
datestruct = strptime(val, "%b %Y")
except ValueError:
try:
datestruct = strptime(val, "%Y")
except ValueError:
return val
return strftime(datetext_format, datestruct)
write_message(
"The error: [%s] occured while trying to get \
the bibsort data from the database." % err, sys.stderr)
return False
write_message('The bibsort data has been read from the database.',
verbose=5)
if results:
config = ConfigParser.ConfigParser()
for item in results:
section = "sort_field_%s" % item[0]
config.add_section(section)
config.set(section, "name", item[1])
config.set(section, "definition", item[2])
config.set(section, "washer", item[3])
output_file_name = CFG_ETCDIR + '/bibsort/bibsort_db_dump_%s.cfg' % \
strftime("%d%m%Y%H%M%S", time.localtime())
write_message('Opening the output file %s' % output_file_name)
try:
output_file = open(output_file_name, 'w')
config.write(output_file)
output_file.close()
except Error, err:
write_message('Can not operate on the configuration file %s [%s].' \
%(output_file_name, err), stream=sys.stderr)
return False
write_message('Configuration data dumped to file.')
else:
write_message("The bsrMETHOD table does not contain any data.")
return True
def test_strftime_date_under_1900_object(self):
expected = "Thu, 03 Jan 1765 00:00:00 +0000"
result = dateutils.strftime("%a, %d %b %Y %H:%M:%S +0000", datetime.date(1765, 1, 3))
self.assertEqual(expected, result)
def perform_update_buckets(recids_current_ordered, recids_to_insert, recids_old_ordered, method_id, update_timestamp = True):
"""Updates the buckets"""
bucket_insert = {}
bucket_delete = {}
write_message("Updating the buckets for method_id = %s" %method_id, verbose=5)
buckets = run_sql("SELECT bucket_no, bucket_last_value \
FROM bsrMETHODDATABUCKET \
WHERE id_bsrMETHOD = %s", (method_id, ))
if not buckets:
write_message("No bucket data found for method_id %s." \
%method_id, sys.stderr)
raise Exception
#sort the buckets to be sure we are iterating them in order(1 to max):
buckets_dict = dict(buckets)
for recid in recids_to_insert:
for bucket_no in buckets_dict:
if recids_current_ordered[recid] <= buckets_dict[bucket_no]:
bucket_insert.setdefault(bucket_no, []).append(recid)
break
for recid in recids_old_ordered:
record_inserted = 0
record_deleted = 0
for bucket_no in buckets_dict:
bucket_value = int(buckets_dict[bucket_no])
if record_inserted and record_deleted:
#both insertion and deletion have been registered
break
if recids_current_ordered[recid] <= bucket_value and \
recids_old_ordered[recid] <= bucket_value and \
not record_inserted and \
not record_deleted:
#both before and after the modif,
#recid should be in the same bucket -> nothing to do
break
if recids_current_ordered[recid] <= bucket_value and not record_inserted:
#recid should be, after the modif, here, so insert
bucket_insert.setdefault(bucket_no, []).append(recid)
record_inserted = 1
if recids_old_ordered[recid] <= bucket_value and not record_deleted:
#recid was here before modif, must be removed
bucket_delete.setdefault(bucket_no, []).append(recid)
record_deleted = 1
for bucket_no in buckets_dict:
if (bucket_no in bucket_insert) or (bucket_no in bucket_delete):
res = run_sql("SELECT bucket_data FROM bsrMETHODDATABUCKET \
where id_bsrMETHOD = %s AND bucket_no = %s", \
(method_id, bucket_no, ))
bucket_data = intbitset(res[0][0])
for recid in bucket_insert.get(bucket_no, []):
bucket_data.add(recid)
for recid in bucket_delete.get(bucket_no, []):
bucket_data.remove(recid)
if update_timestamp:
date = strftime("%Y-%m-%d %H:%M:%S", time.localtime())
run_sql("UPDATE bsrMETHODDATABUCKET \
SET bucket_data = %s, last_updated = %s \
WHERE id_bsrMETHOD = %s AND bucket_no = %s", \
(bucket_data.fastdump(), date, method_id, bucket_no, ))
else:
run_sql("UPDATE bsrMETHODDATABUCKET \
SET bucket_data = %s \
WHERE id_bsrMETHOD = %s AND bucket_no = %s", \
(bucket_data.fastdump(), method_id, bucket_no, ))
write_message("Updating bucket %s for method %s." %(bucket_no, method_id), verbose=5)
def test_strftime_date_under_1900(self):
test_date = "3.1.1765"
expected = "Thu, 03 Jan 1765 00:00:00 +0000"
result = dateutils.strftime("%a, %d %b %Y %H:%M:%S +0000", strptime(test_date, "%d.%m.%Y"))
self.assertEqual(expected, result)
def test_strftime_date_over_1900_object(self):
expected = "Thu, 12 Mar 1908 00:00:00 +0000"
result = dateutils.strftime("%a, %d %b %Y %H:%M:%S +0000", datetime.date(1908, 3, 12))
self.assertEqual(expected, result)
def perform_update_buckets(recids_current_ordered,
recids_to_insert,
recids_old_ordered,
method_id,
update_timestamp=True):
"""Updates the buckets"""
bucket_insert = {}
bucket_delete = {}
write_message("Updating the buckets for method_id = %s" % method_id,
verbose=5)
buckets = run_sql(
"SELECT bucket_no, bucket_last_value \
FROM bsrMETHODDATABUCKET \
WHERE id_bsrMETHOD = %s", (method_id, ))
if not buckets:
write_message("No bucket data found for method_id %s." \
%method_id, sys.stderr)
raise Exception
#sort the buckets to be sure we are iterating them in order(1 to max):
buckets_dict = dict(buckets)
for recid in recids_to_insert:
for bucket_no in buckets_dict:
if recids_current_ordered[recid] <= buckets_dict[bucket_no]:
bucket_insert.setdefault(bucket_no, []).append(recid)
break
for recid in recids_old_ordered:
record_inserted = 0
record_deleted = 0
for bucket_no in buckets_dict:
bucket_value = int(buckets_dict[bucket_no])
if record_inserted and record_deleted:
#both insertion and deletion have been registered
break
if recids_current_ordered[recid] <= bucket_value and \
recids_old_ordered[recid] <= bucket_value and \
not record_inserted and \
not record_deleted:
#both before and after the modif,
#recid should be in the same bucket -> nothing to do
break
if recids_current_ordered[
recid] <= bucket_value and not record_inserted:
#recid should be, after the modif, here, so insert
bucket_insert.setdefault(bucket_no, []).append(recid)
record_inserted = 1
if recids_old_ordered[recid] <= bucket_value and not record_deleted:
#recid was here before modif, must be removed
bucket_delete.setdefault(bucket_no, []).append(recid)
record_deleted = 1
for bucket_no in buckets_dict:
if (bucket_no in bucket_insert) or (bucket_no in bucket_delete):
res = run_sql("SELECT bucket_data FROM bsrMETHODDATABUCKET \
where id_bsrMETHOD = %s AND bucket_no = %s" , \
(method_id, bucket_no, ))
bucket_data = intbitset(res[0][0])
for recid in bucket_insert.get(bucket_no, []):
bucket_data.add(recid)
for recid in bucket_delete.get(bucket_no, []):
if recid in bucket_data:
bucket_data.remove(recid)
if update_timestamp:
date = strftime("%Y-%m-%d %H:%M:%S", time.localtime())
run_sql("UPDATE bsrMETHODDATABUCKET \
SET bucket_data = %s, last_updated = %s \
WHERE id_bsrMETHOD = %s AND bucket_no = %s" , \
(bucket_data.fastdump(), date, method_id, bucket_no, ))
else:
run_sql("UPDATE bsrMETHODDATABUCKET \
SET bucket_data = %s \
WHERE id_bsrMETHOD = %s AND bucket_no = %s" , \
(bucket_data.fastdump(), method_id, bucket_no, ))
write_message("Updating bucket %s for method %s." %
(bucket_no, method_id),
verbose=5)
def test_strftime_date_over_1900(self):
test_date = "12.03.1908"
expected = "Thu, 12 Mar 1908 00:00:00 +0000"
result = dateutils.strftime("%a, %d %b %Y %H:%M:%S +0000", strptime(test_date, "%d.%m.%Y"))
self.assertEqual(expected, result)
def test_strftime_date_over_1900(self):
test_date = "12.03.1908"
expected = "Thu, 12 Mar 1908 00:00:00 +0000"
result = dateutils.strftime("%a, %d %b %Y %H:%M:%S +0000",
strptime(test_date, "%d.%m.%Y"))
self.assertEqual(expected, result)
def test_strftime_date_under_1900(self):
test_date = "3.1.1765"
expected = "Thu, 03 Jan 1765 00:00:00 +0000"
result = dateutils.strftime("%a, %d %b %Y %H:%M:%S +0000",
strptime(test_date, "%d.%m.%Y"))
self.assertEqual(expected, result)
results = run_sql("SELECT id, name, definition, washer FROM bsrMETHOD")
except Error, err:
write_message("The error: [%s] occured while trying to get \
the bibsort data from the database." %err, sys.stderr)
return False
write_message('The bibsort data has been read from the database.', verbose=5)
if results:
config = ConfigParser.ConfigParser()
for item in results:
section = "sort_field_%s" % item[0]
config.add_section(section)
config.set(section, "name", item[1])
config.set(section, "definition", item[2])
config.set(section, "washer", item[3])
output_file_name = CFG_ETCDIR + '/bibsort/bibsort_db_dump_%s.cfg' % \
strftime("%d%m%Y%H%M%S", time.localtime())
write_message('Opening the output file %s' %output_file_name)
try:
output_file = open(output_file_name, 'w')
config.write(output_file)
output_file.close()
except Error, err:
write_message('Can not operate on the configuration file %s [%s].' \
%(output_file_name, err), stream=sys.stderr)
return False
write_message('Configuration data dumped to file.')
else:
write_message("The bsrMETHOD table does not contain any data.")
return True
def test_strftime_date_over_1900_object(self):
test_date = "12.03.1908"
expected = "Thu, 12 Mar 1908 00:00:00 +0000"
result = dateutils.strftime("%a, %d %b %Y %H:%M:%S +0000",
datetime.date(1908, 3, 12))
self.assertEqual(expected, result)
def test_strftime_date_under_1900_object(self):
test_date = "3.1.1765"
expected = "Thu, 03 Jan 1765 00:00:00 +0000"
result = dateutils.strftime("%a, %d %b %Y %H:%M:%S +0000",
datetime.date(1765, 1, 3))
self.assertEqual(expected, result)
