def test_legacy_export_marcxml(self):
"""Record - legacy export marxml."""
# FIXME: use a better way to compare
from invenio.legacy.bibrecord import create_record, records_identical
blob = '''
<record>
<controlfield tag="001">8</controlfield>
<datafield tag="100" ind1=" " ind2=" ">
<subfield code="a">Efstathiou, G P</subfield>
<subfield code="u">Cambridge University</subfield>
</datafield>
<datafield tag="245" ind1=" " ind2=" ">
<subfield code="a">Title</subfield>
<subfield code="b">SubTitle</subfield>
</datafield>
<datafield tag="700" ind1=" " ind2=" ">
<subfield code="a">Lasenby, A N</subfield>
</datafield>
<datafield tag="980" ind1=" " ind2=" ">
<subfield code="a">Articles</subfield>
</datafield>
</record>
'''
rec = Record.create(blob, master_format='marc', namespace='testsuite')
recstruct, _, _ = create_record(blob)
json_recstruct, _, _ = create_record(rec.legacy_export_as_marc())
self.assertTrue(records_identical(json_recstruct, recstruct,
ignore_subfield_order=True))
def test_legacy_export_marcxml(self):
"""Record - legacy export marxml."""
# FIXME: use a better way to compare
from invenio.legacy.bibrecord import create_record, records_identical
blob = '''
<record>
<controlfield tag="001">8</controlfield>
<datafield tag="100" ind1=" " ind2=" ">
<subfield code="a">Efstathiou, G P</subfield>
<subfield code="u">Cambridge University</subfield>
</datafield>
<datafield tag="245" ind1=" " ind2=" ">
<subfield code="a">Title</subfield>
<subfield code="b">SubTitle</subfield>
</datafield>
<datafield tag="700" ind1=" " ind2=" ">
<subfield code="a">Lasenby, A N</subfield>
</datafield>
<datafield tag="980" ind1=" " ind2=" ">
<subfield code="a">Articles</subfield>
</datafield>
</record>
'''
rec = Record.create(blob, master_format='marc', namespace='testsuite')
recstruct, _, _ = create_record(blob)
json_recstruct, _, _ = create_record(rec.legacy_export_as_marc())
self.assertTrue(records_identical(json_recstruct, recstruct,
ignore_subfield_order=True))
def compare_revisions(rev1, rev2, recid):
"""Compare two revisions of a record."""
from invenio.legacy.bibedit.engine import (get_marcxml_of_revision_id,
re_revdate_split)
from invenio.legacy.bibrecord.xmlmarc2textmarc import create_marc_record
from invenio.legacy.bibrecord import create_record
from invenio.legacy.bibedit.utils import record_revision_exists
from invenio.utils.text import show_diff
person1 = ""
person2 = ""
if (not record_revision_exists(recid, rev1)) or \
(not record_revision_exists(recid, rev2)):
return render_template("editor/revision_comparison_error.html")
else:
xml1 = get_marcxml_of_revision_id(recid, rev1)
xml2 = get_marcxml_of_revision_id(recid, rev2)
# Create MARC representations of the records
marc1 = create_marc_record(
create_record(xml1)[0], '', {
"text-marc": 1,
"aleph-marc": 0
})
marc2 = create_marc_record(
create_record(xml2)[0], '', {
"text-marc": 1,
"aleph-marc": 0
})
comparison = show_diff(
marc1,
marc2,
prefix="<pre>",
suffix="</pre>",
prefix_removed='<strong class="diff_field_deleted">',
suffix_removed='</strong>',
prefix_added='<strong class="diff_field_added">',
suffix_added='</strong>')
job_date1 = "%s-%s-%s %s:%s:%s" % re_revdate_split.search(
rev1).groups()
job_date2 = "%s-%s-%s %s:%s:%s" % re_revdate_split.search(
rev2).groups()
# Getting the author of each revision
info1 = get_info_of_record_revision(recid, job_date1)
info2 = get_info_of_record_revision(recid, job_date2)
if info1:
person1 = info1[0][1]
if info2:
person2 = info2[0][1]
ctx = {
"job_date1": job_date1,
"job_date2": job_date2,
"person1": person1,
"person2": person2,
"comparison": comparison
}
return render_template("editor/revision_comparison.html", **ctx)
def compare_references(test, a, b):
from invenio.legacy.bibrecord import create_record, record_xml_output, \
record_delete_field
## Let's normalize records to remove the Invenio refextract signature
a = create_record(a)[0]
b = create_record(b)[0]
record_delete_field(a, '999', 'C', '6')
a = record_xml_output(a)
b = record_xml_output(b)
test.assertEqual(a, b)
def compare_references(test, a, b):
from invenio.legacy.bibrecord import create_record, record_xml_output, \
record_delete_field
## Let's normalize records to remove the Invenio refextract signature
a = create_record(a)[0]
b = create_record(b)[0]
record_delete_field(a, '999', 'C', '6')
a = record_xml_output(a)
b = record_xml_output(b)
test.assertEqual(a, b)
def _get_record_slave(recid, result, mode=None, uid=None):
"""Check if record exists and return it in dictionary format.
If any kind of error occurs returns None.
If mode=='revision' then recid parameter is considered as revid."""
record = None
if recid == 'none':
mode = 'none'
if mode == 'recid':
record_status = record_exists(recid)
#check for errors
if record_status == 0:
result['resultCode'], result[
'resultText'] = 1, 'Non-existent record: %s' % recid
elif record_status == -1:
result['resultCode'], result[
'resultText'] = 1, 'Deleted record: %s' % recid
elif record_locked_by_queue(recid):
result['resultCode'], result[
'resultText'] = 1, 'Record %s locked by queue' % recid
else:
record = create_record(print_record(recid, 'xm'))[0]
elif mode == 'tmpfile':
file_path = '%s_%s.xml' % (_get_file_path(
recid, uid), cfg['CFG_BIBEDIT_TO_MERGE_SUFFIX'])
if not os.path.isfile(file_path): #check if file doesn't exist
result['resultCode'], result[
'resultText'] = 1, 'Temporary file doesnt exist'
else: #open file
tmpfile = open(file_path, 'r')
record = create_record(tmpfile.read())[0]
tmpfile.close()
elif mode == 'revision':
if revision_format_valid_p(recid):
marcxml = get_marcxml_of_revision_id(recid)
if marcxml:
record = create_record(marcxml)[0]
else:
result['resultCode'], result[
'resultText'] = 1, 'The specified revision does not exist'
else:
result['resultCode'], result[
'resultText'] = 1, 'Invalid revision id'
elif mode == 'none':
return {}
else:
result['resultCode'], result[
'resultText'] = 1, 'Invalid record mode for record2'
record_order_subfields(record)
return record
def compare_revisions(rev1, rev2, recid):
"""Compare two revisions of a record."""
from invenio.legacy.bibedit.engine import (get_marcxml_of_revision_id,
re_revdate_split)
from invenio.legacy.bibrecord.xmlmarc2textmarc import create_marc_record
from invenio.legacy.bibrecord import create_record
from invenio.legacy.bibedit.utils import record_revision_exists
from invenio.utils.text import show_diff
person1 = ""
person2 = ""
if (not record_revision_exists(recid, rev1)) or \
(not record_revision_exists(recid, rev2)):
return render_template("editor/revision_comparison_error.html")
else:
xml1 = get_marcxml_of_revision_id(recid, rev1)
xml2 = get_marcxml_of_revision_id(recid, rev2)
# Create MARC representations of the records
marc1 = create_marc_record(
create_record(xml1)[0], '', {"text-marc": 1, "aleph-marc": 0})
marc2 = create_marc_record(
create_record(xml2)[0], '', {"text-marc": 1, "aleph-marc": 0})
comparison = show_diff(
marc1,
marc2,
prefix="<pre>", suffix="</pre>",
prefix_removed='<strong class="diff_field_deleted">',
suffix_removed='</strong>',
prefix_added='<strong class="diff_field_added">',
suffix_added='</strong>')
job_date1 = "%s-%s-%s %s:%s:%s" % re_revdate_split.search(rev1
).groups()
job_date2 = "%s-%s-%s %s:%s:%s" % re_revdate_split.search(rev2
).groups()
# Getting the author of each revision
info1 = get_info_of_record_revision(recid, job_date1)
info2 = get_info_of_record_revision(recid, job_date2)
if info1:
person1 = info1[0][1]
if info2:
person2 = info2[0][1]
ctx = {
"job_date1": job_date1,
"job_date2": job_date2,
"person1": person1,
"person2": person2,
"comparison": comparison
}
return render_template("editor/revision_comparison.html", **ctx)
def doilookup(self, req, form):
"""
Returns the metadata from the crossref website based on the DOI.
"""
args = wash_urlargd(form, {'doi': (str, '')})
response = defaultdict(list)
if args['doi']:
doi = args['doi']
try:
marcxml_template = get_marcxml_for_doi(doi)
except CrossrefError:
# Just ignore Crossref errors
pass
else:
record = create_record(marcxml_template)[0]
if record:
# We need to convert this record structure to a simple dictionary
for key, value in record.items(
): # key, value = (773, [([('0', 'PER:64142'), ...], ' ', ' ', '', 47)])
for val in value: # val = ([('0', 'PER:64142'), ...], ' ', ' ', '', 47)
ind1 = val[1].replace(" ", "_")
ind2 = val[2].replace(" ", "_")
for (k, v) in val[0]: # k, v = ('0', 'PER:5409')
response[key + ind1 + ind2 + k].append(v)
# The output dictionary is something like:
# {"100__a": ['Smith, J.'],
# "700__a": ['Anderson, J.', 'Someoneelse, E.'],
# "700__u": ['University1', 'University2']}
# return dictionary as JSON
return json.dumps(response)
def merge_record_with_template(rec, template_name, is_hp_record=False):
""" Extend the record rec with the contents of the template and return it"""
template = get_record_template(template_name)
if not template:
return
template_bibrec = create_record(template)[0]
# if the record is a holding pen record make all subfields volatile
if is_hp_record:
record_make_all_subfields_volatile(template_bibrec)
for field_tag in template_bibrec:
if not record_has_field(rec, field_tag):
for field_instance in template_bibrec[field_tag]:
record_add_field(rec,
field_tag,
field_instance[1],
field_instance[2],
subfields=field_instance[0])
else:
for template_field_instance in template_bibrec[field_tag]:
subfield_codes_template = field_get_subfield_codes(
template_field_instance)
for field_instance in rec[field_tag]:
subfield_codes = field_get_subfield_codes(field_instance)
for code in subfield_codes_template:
if code not in subfield_codes:
field_add_subfield(
field_instance, code,
field_get_subfield_values(
template_field_instance, code)[0])
record_order_subfields(rec)
return rec
def replace_references(recid, uid=None, txt=None, url=None):
"""Replace references for a record
The record itself is not updated, the marc xml of the document with updated
references is returned
Parameters:
* recid: the id of the record
* txt: references in text mode
* inspire: format of ther references
"""
# Parse references
if txt is not None:
references_xml = extract_references_from_string_xml(txt, is_only_references=True)
elif url is not None:
references_xml = extract_references_from_url_xml(url)
else:
references_xml = extract_references_from_record_xml(recid)
references = create_record(references_xml)
dummy1, dummy2, record, dummy3, dummy4, dummy5, dummy6 = get_cache_contents(recid, uid)
out_xml = None
references_to_add = record_get_field_instances(references[0], tag="999", ind1="C", ind2="5")
refextract_status = record_get_field_instances(references[0], tag="999", ind1="C", ind2="6")
if references_to_add:
# Replace 999 fields
record_delete_fields(record, "999")
record_add_fields(record, "999", references_to_add)
record_add_fields(record, "999", refextract_status)
# Update record references
out_xml = record_xml_output(record)
return out_xml
def replace_references(recid):
"""Replace references for a record
The record itself is not updated, the marc xml of the document with updated
references is returned
Parameters:
* recid: the id of the record
"""
# Parse references
references_xml = extract_references_from_record_xml(recid)
references = create_record(references_xml)
# Record marc xml
record = get_record(recid)
if references[0]:
fields_to_add = record_get_field_instances(references[0],
tag='999',
ind1='%',
ind2='%')
# Replace 999 fields
record_delete_fields(record, '999')
record_add_fields(record, '999', fields_to_add)
# Update record references
out_xml = record_xml_output(record)
else:
out_xml = None
return out_xml
def test_legacy_create_recstruct(self):
"""Record - create recstruct."""
from invenio.legacy.bibrecord import create_record, records_identical
blob = """
<record>
<controlfield tag="001">8</controlfield>
<datafield tag="100" ind1=" " ind2=" ">
<subfield code="a">Efstathiou, G P</subfield>
<subfield code="u">Cambridge University</subfield>
</datafield>
<datafield tag="245" ind1=" " ind2=" ">
<subfield code="a">Title</subfield>
<subfield code="b">SubTitle</subfield>
</datafield>
<datafield tag="700" ind1=" " ind2=" ">
<subfield code="a">Lasenby, A N</subfield>
</datafield>
<datafield tag="980" ind1=" " ind2=" ">
<subfield code="a">Articles</subfield>
</datafield>
</record>
"""
rec = Record.create(blob, master_format="marc", namespace="testsuite")
json_recstruct = rec.legacy_create_recstruct()
recstruct, _, _ = create_record(blob)
self.assertTrue(records_identical(json_recstruct, recstruct, ignore_subfield_order=True))
def merge_record_with_template(rec, template_name, is_hp_record=False):
""" Extend the record rec with the contents of the template and return it"""
template = get_record_template(template_name)
if not template:
return
template_bibrec = create_record(template)[0]
# if the record is a holding pen record make all subfields volatile
if is_hp_record:
record_make_all_subfields_volatile(template_bibrec)
for field_tag in template_bibrec:
if not record_has_field(rec, field_tag):
for field_instance in template_bibrec[field_tag]:
record_add_field(rec, field_tag, field_instance[1], field_instance[2], subfields=field_instance[0])
else:
for template_field_instance in template_bibrec[field_tag]:
subfield_codes_template = field_get_subfield_codes(template_field_instance)
for field_instance in rec[field_tag]:
subfield_codes = field_get_subfield_codes(field_instance)
for code in subfield_codes_template:
if code not in subfield_codes:
field_add_subfield(
field_instance, code, field_get_subfield_values(template_field_instance, code)[0]
)
record_order_subfields(rec)
return rec
def doilookup(self, req, form):
"""
Returns the metadata from the crossref website based on the DOI.
"""
args = wash_urlargd(form, {
'doi': (str, '')})
response = defaultdict(list)
if args['doi']:
doi = args['doi']
try:
marcxml_template = get_marcxml_for_doi(doi)
except CrossrefError:
# Just ignore Crossref errors
pass
else:
record = create_record(marcxml_template)[0]
if record:
# We need to convert this record structure to a simple dictionary
for key, value in record.items(): # key, value = (773, [([('0', 'PER:64142'), ...], ' ', ' ', '', 47)])
for val in value: # val = ([('0', 'PER:64142'), ...], ' ', ' ', '', 47)
ind1 = val[1].replace(" ", "_")
ind2 = val[2].replace(" ", "_")
for (k, v) in val[0]: # k, v = ('0', 'PER:5409')
response[key+ind1+ind2+k].append(v)
# The output dictionary is something like:
# {"100__a": ['Smith, J.'],
# "700__a": ['Anderson, J.', 'Someoneelse, E.'],
# "700__u": ['University1', 'University2']}
# return dictionary as JSON
return json.dumps(response)
def cli_clean_revisions(recid, dry_run=True, verbose=True):
"""Clean revisions of the given recid, by removing duplicate revisions
that do not change the content of the record."""
if recid == '*':
recids = intbitset(run_sql("SELECT DISTINCT id_bibrec FROM hstRECORD"))
else:
try:
recids = [int(recid)]
except ValueError:
print('ERROR: record ID must be integer, not %s.' % recid)
sys.exit(1)
for recid in recids:
all_revisions = run_sql("SELECT marcxml, job_id, job_name, job_person, job_date FROM hstRECORD WHERE id_bibrec=%s ORDER BY job_date ASC", (recid,))
previous_rec = {}
deleted_revisions = 0
for marcxml, job_id, job_name, job_person, job_date in all_revisions:
try:
current_rec = create_record(zlib.decompress(marcxml))[0]
except Exception:
print("ERROR: corrupted revisions found. Please run %s --fix-revisions '*'" % sys.argv[0], file=sys.stderr)
sys.exit(1)
if records_identical(current_rec, previous_rec):
deleted_revisions += 1
if not dry_run:
run_sql("DELETE FROM hstRECORD WHERE id_bibrec=%s AND job_id=%s AND job_name=%s AND job_person=%s AND job_date=%s", (recid, job_id, job_name, job_person, job_date))
previous_rec = current_rec
if verbose and deleted_revisions:
print("record %s: deleted %s duplicate revisions out of %s" % (recid, deleted_revisions, len(all_revisions)))
if verbose:
print("DONE")
def get_templates(templatesDir, tmpl_name, tmpl_description, extractContent = False):
"""Return list of templates [filename, name, description, content*]
the extractContent variable indicated if the parsed content should
be included"""
template_fnames = fnmatch.filter(templatesDir, '*.xml')
templates = []
for filepath in template_fnames:
template_file = open(filepath,'r')
template = template_file.read()
template_file.close()
fname = os.path.basename(filepath)
fname_stripped = os.path.splitext(fname)[0]
mo_name = tmpl_name.search(template)
mo_description = tmpl_description.search(template)
date_modified = time.ctime(os.path.getmtime(filepath))
if mo_name:
name = mo_name.group(1)
else:
name = fname_stripped
if mo_description:
description = mo_description.group(1)
else:
description = ''
if (extractContent):
parsedTemplate = create_record(template)[0]
if parsedTemplate != None:
# If the template was correct
templates.append([fname_stripped, name, description, parsedTemplate])
else:
raise "Problem when parsing the template %s" % (fname, )
else:
templates.append([fname_stripped, name, description, date_modified])
return templates
def test_marc_export(self):
from invenio.modules.records.api import Record
from invenio.legacy.bibrecord import create_record, record_xml_output
rec = Record(json=test_record, master_format='marc')
# Needed to properly set authors when generating MARC
first = rec['authors'][0]
additional = rec['authors'][1:]
rec['_first_author'] = first
rec['_additional_authors'] = additional
output_marc = record_xml_output(
create_record(rec.legacy_export_as_marc())[0]
)
try:
self.assertEqual(test_marc, output_marc)
except AssertionError:
# Print diff in case of errors.
import difflib
diff = "".join(difflib.unified_diff(
test_marc.splitlines(1),
output_marc.splitlines(1)
))
raise AssertionError(diff)
form_json = rec.produce('json_for_form')
for k, v in test_form_json.items():
self.assertEqual(form_json[k], test_form_json[k])
def test_marc_export(self):
from invenio.modules.records.api import Record
from invenio.legacy.bibrecord import create_record, record_xml_output
rec = Record(json=test_record, master_format='marc')
# Needed to properly set authors when generating MARC
first = rec['authors'][0]
additional = rec['authors'][1:]
rec['_first_author'] = first
rec['_additional_authors'] = additional
output_marc = record_xml_output(
create_record(rec.legacy_export_as_marc())[0]
)
try:
self.assertEqual(test_marc, output_marc)
except AssertionError:
# Print diff in case of errors.
import difflib
diff = "".join(difflib.unified_diff(
test_marc.splitlines(1),
output_marc.splitlines(1)
))
raise AssertionError(diff)
form_json = rec.produce('json_for_form')
for k, v in test_form_json.items():
self.assertEqual(form_json[k], test_form_json[k])
def _get_record_slave(recid, result, mode=None, uid=None):
"""Check if record exists and return it in dictionary format.
If any kind of error occurs returns None.
If mode=='revision' then recid parameter is considered as revid."""
record = None
if recid == 'none':
mode = 'none'
if mode == 'recid':
record_status = record_exists(recid)
#check for errors
if record_status == 0:
result['resultCode'], result['resultText'] = 1, 'Non-existent record: %s' % recid
elif record_status == -1:
result['resultCode'], result['resultText'] = 1, 'Deleted record: %s' % recid
elif record_locked_by_queue(recid):
result['resultCode'], result['resultText'] = 1, 'Record %s locked by queue' % recid
else:
record = create_record( print_record(recid, 'xm') )[0]
record_order_subfields(record)
elif mode == 'tmpfile':
file_path = '%s_%s.xml' % (_get_file_path(recid, uid),
CFG_BIBEDIT_TO_MERGE_SUFFIX)
if not os.path.isfile(file_path): #check if file doesn't exist
result['resultCode'], result['resultText'] = 1, 'Temporary file doesnt exist'
else: #open file
tmpfile = open(file_path, 'r')
record = create_record( tmpfile.read() )[0]
tmpfile.close()
elif mode == 'revision':
if revision_format_valid_p(recid):
marcxml = get_marcxml_of_revision_id(recid)
if marcxml:
record = create_record(marcxml)[0]
else:
result['resultCode'], result['resultText'] = 1, 'The specified revision does not exist'
else:
result['resultCode'], result['resultText'] = 1, 'Invalid revision id'
elif mode == 'none':
return {}
else:
result['resultCode'], result['resultText'] = 1, 'Invalid record mode for record2'
return record
def get_bibrecord(recid):
"""Return record in BibRecord wrapping."""
if record_exists(recid):
record_revision_ids = get_record_revision_ids(recid)
if record_revision_ids:
return create_record(get_marcxml_of_revision_id(max(record_revision_ids)))[0]
else:
return get_record(recid)
def get_bibrecord(recid):
"""Return record in BibRecord wrapping."""
if record_exists(recid):
record_revision_ids = get_record_revision_ids(recid)
if record_revision_ids:
return create_record(get_marcxml_of_revision_id(max(record_revision_ids)))[0]
else:
return get_record(recid)
def save_xml_record(recid, uid, xml_record='', to_upload=True, to_merge=False,
task_name="bibedit", sequence_id=None):
"""Write XML record to file. Default behaviour is to read the record from
a BibEdit cache file, filter out the unchanged volatile subfields,
write it back to an XML file and then pass this file to BibUpload.
@param xml_record: give XML as string in stead of reading cache file
@param to_upload: pass the XML file to BibUpload
@param to_merge: prepare an XML file for BibMerge to use
"""
if not xml_record:
# Read record from cache file.
cache = get_cache_contents(recid, uid)
if cache:
record = cache[2]
used_changes = cache[4]
xml_record = record_xml_output(record)
delete_cache(recid, uid)
delete_disabled_changes(used_changes)
else:
record = create_record(xml_record)[0]
# clean the record from unfilled volatile fields
record_strip_empty_volatile_subfields(record)
record_strip_empty_fields(record)
# order subfields alphabetically before saving the record
record_order_subfields(record)
xml_to_write = wash_for_xml(record_xml_output(record))
# Write XML file.
if not to_merge:
fd, file_path = tempfile.mkstemp(dir=cfg['CFG_BIBEDIT_CACHEDIR'],
prefix="%s_" % cfg['CFG_BIBEDIT_FILENAME'],
suffix="_%s_%s.xml" % (recid, uid))
f = os.fdopen(fd, 'w')
f.write(xml_to_write)
f.close()
else:
file_path = '%s_%s.xml' % (_get_file_path(recid, uid),
cfg['CFG_BIBEDIT_TO_MERGE_SUFFIX'])
xml_file = open(file_path, 'w')
xml_file.write(xml_to_write)
xml_file.close()
user_name = get_user_info(uid)[1]
if to_upload:
args = ['bibupload', user_name, '-P', '5', '-r',
file_path, '-u', user_name]
if task_name == "bibedit":
args.extend(['--name', 'bibedit'])
if sequence_id:
args.extend(["-I", sequence_id])
args.extend(['--email-logs-on-error'])
task_low_level_submission(*args)
return True
def save_xml_record(recid, uid, xml_record='', to_upload=True, to_merge=False,
task_name="bibedit", sequence_id=None):
"""Write XML record to file. Default behaviour is to read the record from
a BibEdit cache file, filter out the unchanged volatile subfields,
write it back to an XML file and then pass this file to BibUpload.
@param xml_record: give XML as string in stead of reading cache file
@param to_upload: pass the XML file to BibUpload
@param to_merge: prepare an XML file for BibMerge to use
"""
if not xml_record:
# Read record from cache file.
cache = get_cache_contents(recid, uid)
if cache:
record = cache[2]
used_changes = cache[4]
xml_record = record_xml_output(record)
delete_cache(recid, uid)
delete_disabled_changes(used_changes)
else:
record = create_record(xml_record)[0]
# clean the record from unfilled volatile fields
record_strip_empty_volatile_subfields(record)
record_strip_empty_fields(record)
# order subfields alphabetically before saving the record
record_order_subfields(record)
xml_to_write = wash_for_xml(record_xml_output(record))
# Write XML file.
if not to_merge:
fd, file_path = tempfile.mkstemp(dir=cfg['CFG_BIBEDIT_CACHEDIR'],
prefix="%s_" % cfg['CFG_BIBEDIT_FILENAME'],
suffix="_%s_%s.xml" % (recid, uid))
f = os.fdopen(fd, 'w')
f.write(xml_to_write)
f.close()
else:
file_path = '%s_%s.xml' % (_get_file_path(recid, uid),
cfg['CFG_BIBEDIT_TO_MERGE_SUFFIX'])
xml_file = open(file_path, 'w')
xml_file.write(xml_to_write)
xml_file.close()
user_name = get_user_info(uid)[1]
if to_upload:
args = ['bibupload', user_name, '-P', '5', '-r',
file_path, '-u', user_name]
if task_name == "bibedit":
args.extend(['--name', 'bibedit'])
if sequence_id:
args.extend(["-I", sequence_id])
args.extend(['--email-logs-on-error'])
task_low_level_submission(*args)
return True
def __init__(self, recID, ln=CFG_SITE_LANG, search_pattern=None,
xml_record=None, user_info=None, output_format=''):
"""
Creates a new bibformat object, with given record.
You can either specify an record ID to format, or give its xml representation.
if 'xml_record' is not None, use 'xml_record' instead of recID for the record.
'user_info' allows to grant access to some functionalities on
a page depending on the user's priviledges. It is a dictionary
in the following form::
user_info = {
'remote_ip' : '',
'remote_host' : '',
'referer' : '',
'uri' : '',
'agent' : '',
'uid' : -1,
'nickname' : '',
'email' : '',
'group' : [],
'guest' : '1'
}
:param recID: the id of a record
:param ln: the language in which the record has to be formatted
:param search_pattern: list of string representing the request used by the user in web interface
:param xml_record: a xml string of the record to format
:param user_info: the information of the user who will view the formatted page
:param output_format: the output_format used for formatting this record
"""
self.xml_record = None # *Must* remain empty if recid is given
if xml_record is not None:
# If record is given as parameter
self.xml_record = xml_record
self.record = create_record(xml_record)[0]
recID = record_get_field_value(self.record, "001") or None
recID = int(recID) if recID is not None else recID
try:
assert isinstance(recID, (int, long, type(None))), 'Argument of wrong type!'
except AssertionError:
register_exception(prefix="recid needs to be an integer in BibFormatObject",
alert_admin=True)
recID = int(recID)
self.recID = recID
self.lang = wash_language(ln)
if search_pattern is None:
search_pattern = []
self.search_pattern = search_pattern
self.output_format = output_format
self.user_info = user_info
if self.user_info is None:
from invenio.ext.login.legacy_user import UserInfo
self.user_info = UserInfo(None)
def test_marc_export(self):
from invenio.modules.records.api import Record
from invenio.legacy.bibrecord import create_record
r = Record(json=test_record, master_format='marc')
self.assertEqual(
r.legacy_create_recstruct(),
create_record(test_marc)[0],
)
def _get_record_linking_fields(recid_b, recid_a, tag, ind1, ind2):
"""
Returns the fields (defined by tag, ind1, ind2) in record (given
by recid_b) that do not link to another given record (recid_a).
"""
fields = []
rec = create_record(format_record(recid_b, "xm"))[0]
for field_instance in record_get_field_instances(rec, tag=tag, ind1=ind1, ind2=ind2):
if not ('w', str(recid_a)) in field_instance[0]:
fields.append(field_instance)
return fields
def crossref_process_template(template, change=False):
"""
Creates record from template based on xml template
@param change: if set to True, makes changes to the record (translating the
title, unifying autroh names etc.), if not - returns record without
any changes
@return: record
"""
record = create_record(template)[0]
if change:
crossref_translate_title(record)
crossref_normalize_name(record)
return record
def crossref_process_template(template, change=False):
"""
Creates record from template based on xml template
@param change: if set to True, makes changes to the record (translating the
title, unifying autroh names etc.), if not - returns record without
any changes
@return: record
"""
record = create_record(template)[0]
if change:
crossref_translate_title(record)
crossref_normalize_name(record)
return record
def modify_record_timestamp(revision_xml, last_revision_ts):
""" Modify tag 005 to add the revision passed as parameter.
@param revision_xml: marcxml representation of the record to modify
@type revision_xml: string
@param last_revision_ts: timestamp to add to 005 tag
@type last_revision_ts: string
@return: marcxml with 005 tag modified
"""
recstruct = create_record(revision_xml)[0]
record_modify_controlfield(recstruct, "005", last_revision_ts,
field_position_local=0)
return record_xml_output(recstruct)
def _get_record_linking_fields(recid_b, recid_a, tag, ind1, ind2):
"""
Returns the fields (defined by tag, ind1, ind2) in record (given
by recid_b) that do not link to another given record (recid_a).
"""
fields = []
rec = create_record(format_record(recid_b, "xm"))[0]
for field_instance in record_get_field_instances(rec,
tag=tag,
ind1=ind1,
ind2=ind2):
if not ('w', str(recid_a)) in field_instance[0]:
fields.append(field_instance)
return fields
def _next_value(self, recid=None, xml_record=None, start_date=None):
"""
Returns the next cnum for the given recid
@param recid: id of the record where the cnum will be generated
@type recid: int
@param xml_record: record in xml format
@type xml_record: string
@param start_date: use given start date
@type start_date: string
@return: next cnum for the given recid. Format is Cyy-mm-dd.[.1n]
@rtype: string
@raises ConferenceNoStartDateError: No date information found in the
given recid
"""
bibrecord = None
if recid is None and xml_record is not None:
bibrecord = create_record(xml_record)[0]
elif recid is not None:
bibrecord = get_bibrecord(recid)
if start_date is None and bibrecord is not None:
start_date = record_get_field_value(bibrecord,
tag="111",
ind1="",
ind2="",
code="x")
if not start_date:
raise ConferenceNoStartDateError
base_cnum = "C" + start_date[2:]
record_cnums = self._get_record_cnums(base_cnum)
if not record_cnums:
new_cnum = base_cnum
else:
# Get the max current revision, cnums are in format Cyy-mm-dd,
# Cyy-mm-dd.1, Cyy-mm-dd.2
highest_revision = max([0] + [
int(rev[0].split('.')[1])
for rev in record_cnums if '.' in rev[0]
])
new_cnum = base_cnum + '.' + str(highest_revision + 1)
return new_cnum
def modify_record_timestamp(revision_xml, last_revision_ts):
""" Modify tag 005 to add the revision passed as parameter.
@param revision_xml: marcxml representation of the record to modify
@type revision_xml: string
@param last_revision_ts: timestamp to add to 005 tag
@type last_revision_ts: string
@return: marcxml with 005 tag modified
"""
recstruct = create_record(revision_xml)[0]
if "005" in recstruct:
record_modify_controlfield(recstruct, "005", last_revision_ts,
field_position_local=0)
else:
record_add_field(recstruct, '005', controlfield_value=last_revision_ts)
return record_xml_output(recstruct)
def legacy_create_recstruct(self):
"""Create the `recstruct` representation.
It uses the producer rules and
:func:`~invenio.legacy.bibrecord.create_record`.
"""
# FIXME: it might be a bit overkilling
from invenio.legacy.bibrecord import create_record
record, status_code, errors = create_record(self.legacy_export_as_marc())
if status_code == 0:
# There was an error
if isinstance(errors, list):
errors = "\n".join(errors)
raise ReaderException("There was an error while parsing MARCXML: %s" % (errors,))
return record
def _next_value(self, recid=None, xml_record=None, start_date=None):
"""
Returns the next cnum for the given recid
@param recid: id of the record where the cnum will be generated
@type recid: int
@param xml_record: record in xml format
@type xml_record: string
@param start_date: use given start date
@type start_date: string
@return: next cnum for the given recid. Format is Cyy-mm-dd.[.1n]
@rtype: string
@raises ConferenceNoStartDateError: No date information found in the
given recid
"""
bibrecord = None
if recid is None and xml_record is not None:
bibrecord = create_record(xml_record)[0]
elif recid is not None:
bibrecord = get_bibrecord(recid)
if start_date is None and bibrecord is not None:
start_date = record_get_field_value(bibrecord,
tag="111",
ind1="",
ind2="",
code="x")
if not start_date:
raise ConferenceNoStartDateError
base_cnum = "C" + start_date[2:]
record_cnums = self._get_record_cnums(base_cnum)
if not record_cnums:
new_cnum = base_cnum
else:
# Get the max current revision, cnums are in format Cyy-mm-dd,
# Cyy-mm-dd.1, Cyy-mm-dd.2
highest_revision = max([0] + [int(rev[0].split('.')[1]) for rev in record_cnums if '.' in rev[0]])
new_cnum = base_cnum + '.' + str(highest_revision + 1)
return new_cnum
def save_xml_record(recid, uid, xml_record='', to_upload=True, to_merge=False):
"""Write XML record to file. Default behaviour is to read the record from
a BibEdit cache file, filter out the unchanged volatile subfields,
write it back to an XML file and then pass this file to BibUpload.
@param xml_record: give XML as string in stead of reading cache file
@param to_upload: pass the XML file to BibUpload
@param to_merge: prepare an XML file for BibMerge to use
"""
if not xml_record:
# Read record from cache file.
cache = get_cache_file_contents(recid, uid)
if cache:
record = cache[2]
used_changes = cache[4]
xml_record = record_xml_output(record)
delete_cache_file(recid, uid)
delete_disabled_changes(used_changes)
else:
record = create_record(xml_record)[0]
# clean the record from unfilled volatile fields
record_strip_empty_volatile_subfields(record)
record_strip_empty_fields(record)
# order subfields alphabetically before saving the record
record_order_subfields(record)
xml_to_write = wash_for_xml(record_xml_output(record))
# Write XML file.
if not to_merge:
file_path = '%s.xml' % _get_file_path(recid, uid)
else:
file_path = '%s_%s.xml' % (_get_file_path(recid, uid),
CFG_BIBEDIT_TO_MERGE_SUFFIX)
xml_file = open(file_path, 'w')
xml_file.write(xml_to_write)
xml_file.close()
user_name = get_user_info(uid)[1]
if to_upload:
# Pass XML file to BibUpload.
task_low_level_submission('bibupload', 'bibedit', '-P', '5', '-r',
file_path, '-u', user_name)
return True
def legacy_create_recstruct(self):
"""Create the `recstruct` representation.
It uses the producer rules and
:func:`~invenio.legacy.bibrecord.create_record`.
"""
# FIXME: it might be a bit overkilling
from invenio.legacy.bibrecord import create_record
record, status_code, errors = create_record(
self.legacy_export_as_marc())
if status_code == 0:
# There was an error
if isinstance(errors, list):
errors = "\n".join(errors)
raise ReaderException(
"There was an error while parsing MARCXML: %s" % (errors, ))
return record
def _prepare_blob(self, *args, **kwargs):
"""Transform the incoming blob into recstruct.
FIXME: stop using recstruct!
"""
from invenio.legacy.bibrecord import create_record
class SaveDict(dict):
__getitem__ = dict.get
def dict_extend_helper(d, key, value):
"""Helper function.
If the key is present inside the dictionary it creates a list (if
not present) and extends it with the new value.
Almost as in `list.extend`
"""
if key in d:
current_value = d.get(key)
if not isinstance(current_value, list):
current_value = [current_value]
current_value.append(value)
value = current_value
d[key] = value
self.rec_tree = SaveDict()
record, status_code, errors = create_record(self._blob)
if status_code == 0:
if isinstance(errors, list):
errors = "\n".join(errors)
# There was an error
raise ReaderException(
"There was an error while parsing MARCXML: %s" % (errors, ))
for key, values in iteritems(record):
if key < '010' and key.isdigit():
self.rec_tree[key] = [value[3] for value in values]
else:
for value in values:
field = SaveDict()
for subfield in value[0]:
dict_extend_helper(field, subfield[0], subfield[1])
dict_extend_helper(self.rec_tree,
(key + value[1] + value[2]).replace(
' ', '_'), field)
def _prepare_blob(self, *args, **kwargs):
"""Transform the incoming blob into recstruct.
FIXME: stop using recstruct!
"""
from invenio.legacy.bibrecord import create_record
class SaveDict(dict):
__getitem__ = dict.get
def dict_extend_helper(d, key, value):
"""Helper function.
If the key is present inside the dictionary it creates a list (if
not present) and extends it with the new value.
Almost as in `list.extend`
"""
if key in d:
current_value = d.get(key)
if not isinstance(current_value, list):
current_value = [current_value]
current_value.append(value)
value = current_value
d[key] = value
self.rec_tree = SaveDict()
record, status_code, errors = create_record(self._blob)
if status_code == 0:
if isinstance(errors, list):
errors = "\n".join(errors)
# There was an error
raise ReaderException(
"There was an error while parsing MARCXML: %s" % (errors,))
for key, values in iteritems(record):
if key < '010' and key.isdigit():
self.rec_tree[key] = [value[3] for value in values]
else:
for value in values:
field = SaveDict()
for subfield in value[0]:
dict_extend_helper(field, subfield[0], subfield[1])
dict_extend_helper(
self.rec_tree,
(key + value[1] + value[2]).replace(' ', '_'), field)
def replace_references(recid, uid=None, txt=None, url=None):
"""Replace references for a record
The record itself is not updated, the marc xml of the document with updated
references is returned
Parameters:
* recid: the id of the record
* txt: references in text mode
* inspire: format of ther references
"""
# Parse references
if txt is not None:
references_xml = extract_references_from_string_xml(
txt, is_only_references=True)
elif url is not None:
references_xml = extract_references_from_url_xml(url)
else:
references_xml = extract_references_from_record_xml(recid)
references = create_record(references_xml)
dummy1, dummy2, record, dummy3, dummy4, dummy5, dummy6 = get_cache_contents(
recid, uid)
out_xml = None
references_to_add = record_get_field_instances(references[0],
tag='999',
ind1='C',
ind2='5')
refextract_status = record_get_field_instances(references[0],
tag='999',
ind1='C',
ind2='6')
if references_to_add:
# Replace 999 fields
record_delete_fields(record, '999')
record_add_fields(record, '999', references_to_add)
record_add_fields(record, '999', refextract_status)
# Update record references
out_xml = record_xml_output(record)
return out_xml
def main():
from invenio.legacy.search_engine import get_record
from invenio.legacy.bibupload.engine import (
bibupload,
)
from invenio.legacy.bibrecord import (
create_record,
)
from invenio.legacy.bibedit.db_layer import get_record_revisions
from invenio.legacy.bibedit.utils import (
get_record_revision_ids,
get_marcxml_of_revision,
)
# Loop through list of records
for r in RECORDS:
rec = get_record(r)
if not rec:
break
print('Processing record: {0}'.format(r))
# pprint(rec)
print(get_record_revision_ids(r))
print
revs = get_record_revisions(r)
print(revs)
print
for id, rev in revs[0:1]:
marcxml = get_marcxml_of_revision(r, rev)
# print(marcxml)
# print
rec = create_record(marcxml)[0]
pprint(rec)
if raw_input('Bibupload (y/n)? ') == 'y':
# bibupload(rec, 'delete')
# sleep(5)
bibupload(rec, 'replace')
def get_templates(templatesDir,
tmpl_name,
tmpl_description,
extractContent=False):
"""Return list of templates [filename, name, description, content*]
the extractContent variable indicated if the parsed content should
be included"""
template_fnames = fnmatch.filter(templatesDir, '*.xml')
templates = []
for fname in template_fnames:
filepath = fname
template_file = open(filepath, 'r')
template = template_file.read()
template_file.close()
fname = os.path.basename(filepath)
fname_stripped = os.path.splitext(fname)[0]
mo_name = tmpl_name.search(template)
mo_description = tmpl_description.search(template)
date_modified = time.ctime(os.path.getmtime(filepath))
if mo_name:
name = mo_name.group(1)
else:
name = fname_stripped
if mo_description:
description = mo_description.group(1)
else:
description = ''
if (extractContent):
parsedTemplate = create_record(template)[0]
if parsedTemplate is not None:
# If the template was correct
templates.append(
[fname_stripped, name, description, parsedTemplate])
else:
raise Exception("Problem when parsing the template %s" %
(fname, ))
else:
templates.append(
[fname_stripped, name, description, date_modified])
return templates
def main():
from invenio.legacy.search_engine import get_record
from invenio.legacy.bibupload.engine import (
bibupload, )
from invenio.legacy.bibrecord import (
create_record, )
from invenio.legacy.bibedit.db_layer import get_record_revisions
from invenio.legacy.bibedit.utils import (
get_record_revision_ids,
get_marcxml_of_revision,
)
# Loop through list of records
for r in RECORDS:
rec = get_record(r)
if not rec:
break
print('Processing record: {0}'.format(r))
# pprint(rec)
print(get_record_revision_ids(r))
print
revs = get_record_revisions(r)
print(revs)
print
for id, rev in revs[0:1]:
marcxml = get_marcxml_of_revision(r, rev)
# print(marcxml)
# print
rec = create_record(marcxml)[0]
pprint(rec)
if raw_input('Bibupload (y/n)? ') == 'y':
# bibupload(rec, 'delete')
# sleep(5)
bibupload(rec, 'replace')
def cli_clean_revisions(recid, dry_run=True, verbose=True):
"""Clean revisions of the given recid, by removing duplicate revisions
that do not change the content of the record."""
if recid == '*':
recids = intbitset(
run_sql("""SELECT DISTINCT id_bibrec FROM "hstRECORD" """))
else:
try:
recids = [int(recid)]
except ValueError:
print('ERROR: record ID must be integer, not %s.' % recid)
sys.exit(1)
for recid in recids:
all_revisions = run_sql(
"""SELECT marcxml, job_id, job_name, job_person, job_date FROM "hstRECORD" WHERE id_bibrec=%s ORDER BY job_date ASC""",
(recid, ))
previous_rec = {}
deleted_revisions = 0
for marcxml, job_id, job_name, job_person, job_date in all_revisions:
try:
current_rec = create_record(zlib.decompress(str(marcxml)))[0]
except Exception:
print(
"ERROR: corrupted revisions found. Please run %s --fix-revisions '*'"
% sys.argv[0],
file=sys.stderr)
sys.exit(1)
if records_identical(current_rec, previous_rec):
deleted_revisions += 1
if not dry_run:
run_sql(
"""DELETE FROM "hstRECORD" WHERE id_bibrec=%s AND job_id=%s AND job_name=%s AND job_person=%s AND job_date=%s""",
(recid, job_id, job_name, job_person, job_date))
previous_rec = current_rec
if verbose and deleted_revisions:
print("record %s: deleted %s duplicate revisions out of %s" %
(recid, deleted_revisions, len(all_revisions)))
if verbose:
print("DONE")
def merge_record_with_template(rec, template_name):
""" Extend the record rec with the contents of the template and return it"""
template = get_record_template(template_name)
if not template:
return
template_bibrec = create_record(template)[0]
for field_tag in template_bibrec:
if not record_has_field(rec, field_tag):
for field_instance in template_bibrec[field_tag]:
record_add_field(rec, field_tag, field_instance[1],
field_instance[2], subfields=field_instance[0])
else:
for template_field_instance in template_bibrec[field_tag]:
subfield_codes_template = field_get_subfield_codes(template_field_instance)
for field_instance in rec[field_tag]:
subfield_codes = field_get_subfield_codes(field_instance)
for code in subfield_codes_template:
if code not in subfield_codes:
field_add_subfield(field_instance, code,
field_get_subfield_values(template_field_instance,
code)[0])
return rec
def verify_revision(self, verify_record, original_record, opt_mode=None):
"""
Compares the upload record with the same 005 record from archive.
Once the changes are identified, The latest revision of the record is fetched
from the system and the identified changes are applied over the latest.
Returns record patch in case of non-conflicting addition/modification/deletion
Conflicting records raise Error and stops the bibupload process
"""
upload_rev = ''
original_rev = ''
r_date = ''
record_patch = {}
# No need for revision check for other operations
if opt_mode not in ['replace', 'correct']:
return
if '001' in verify_record:
self.rec_id = record_get_field_value(verify_record, '001')
# Retrieving Revision tags for comparison
if '005' in verify_record:
upload_rev = record_get_field_value(verify_record, '005')
r_date = upload_rev.split('.')[0]
if r_date not in [k[1] for k in get_record_revisions(self.rec_id)]:
raise InvenioBibUploadInvalidRevisionError(self.rec_id, r_date)
else:
raise InvenioBibUploadMissing005Error(self.rec_id)
if '005' in original_record:
original_rev = record_get_field_value(original_record, '005')
else:
raise InvenioBibUploadMissing005Error(self.rec_id)
# Retrieving the archived version
marc_xml = get_marcxml_of_record_revision(self.rec_id, r_date)
res = create_record(zlib.decompress(marc_xml[0][0]))
archived_record = res[0]
# Comparing Upload and Archive record
curr_patch = self.compare_records(verify_record, archived_record,
opt_mode)
# No changes in Upload Record compared to Archived Revision
# Raising Error to skip the bibupload for the record
if not curr_patch:
raise InvenioBibUploadUnchangedRecordError(self.rec_id, upload_rev)
if original_rev == upload_rev:
# Upload, Archive and Original Records have same Revisions.
affected_tags = self.retrieve_affected_tags_with_ind(curr_patch)
return ('correct',
self.generate_final_patch(curr_patch,
self.rec_id), affected_tags)
# Comparing Original and Archive record
orig_patch = self.compare_records(original_record, archived_record,
opt_mode)
# Checking for conflicts
# If no original patch - Original Record same as Archived Record
if orig_patch:
curr_patch = self.detect_conflict(verify_record, curr_patch, upload_rev, \
original_record, orig_patch, original_rev)
record_patch = self.generate_final_patch(curr_patch, self.rec_id)
affected_tags = self.retrieve_affected_tags_with_ind(curr_patch)
# Returning patch in case of no conflicting fields
return ('correct', record_patch, affected_tags)
def check_doi_status_after_merge(original_recid1, original_recid2, final_record1, final_record_2, record2_marked_as_duplicate_p=False, submit_confirmed_p=False):
"""
Check that the result of the merge does not removed DOIs managed
by the system, and that not duplicate DOI would be
created. Returns a tuple(error_code, message).
@param original_recid1: the record ID of the original record 1 (master)
@type original_recid1: int
@param original_recid2: the record ID of the original record 2 (slave)
@type original_recid2: int
@param final_record1: the resulting merged record
@type final_record1: BibRecord object
@param final_record_2: the resulting slave "merged" record (optional when record2_marked_as_duplicate_p is False)
@type final_record_2: BibRecord object
@param record2_marked_as_duplicate_p: True if the record 2 will be marked as duplicate (and deleted)
@type record2_marked_as_duplicate_p: bool
@param submit_confirmed_p: if the user has already confirmed to proceed with submission, according to previous messages displayed. If True, do not ask again confirmation and proceed if all tests pass.
@type submit_confirmed_p: bool
"""
errcode = 0
message = ''
new_record1_dois = get_dois(final_record1)
new_record1_managed_dois = get_dois(final_record1, internal_only_p=True)
original_record1_managed_dois = get_dois(create_record(print_record(original_recid1, 'xm'))[0],
internal_only_p=True)
original_record2_dois = get_dois(create_record(print_record(original_recid2, 'xm'))[0])
# Are there any DOI from record 1 (master) lost in the merging?
lost_dois_in_record1 = [doi for doi in original_record1_managed_dois \
if not doi in new_record1_managed_dois]
# Enough to check for duplicate DOI creation in this record,
# not whole DB
duplicate_dois_after_merge = [doi for doi in new_record1_dois if new_record1_dois.count(doi) > 1]
if record2_marked_as_duplicate_p:
new_record2_managed_dois = get_dois(final_record_2, internal_only_p=True)
original_record2_managed_dois = get_dois(create_record(print_record(original_recid2, 'xm'))[0],
internal_only_p=True)
# Are there any DOI from record 2 (slave) lost in the merging?
lost_dois_in_record2 = [doi for doi in original_record2_managed_dois \
if not doi in new_record1_managed_dois]
else:
lost_dois_in_record2 = []
duplicate_dois_after_merge += [doi for doi in new_record1_dois if doi in original_record2_dois]
if ((lost_dois_in_record1 or lost_dois_in_record2) and \
CFG_BIBEDIT_INTERNAL_DOI_PROTECTION_LEVEL > 0) or \
duplicate_dois_after_merge:
if CFG_BIBEDIT_INTERNAL_DOI_PROTECTION_LEVEL == 1 and \
not duplicate_dois_after_merge and \
not submit_confirmed_p:
errcode = 1
message = 'The resulting merged record misses DOI(s) managed by the system.<script type="text/javascript">%(check_duplicate_box)sif (confirm(\'The resulting merged record will lose DOI(s) managed by the system.\\n' + \
'The following DOI(s) were in the original record (#1) but are not in the final merged one:\\n' + '\\n'.join(lost_dois_in_record1) + \
'\\nAre you sure that you want to submit the merged records without the DOI(s)?\')) {onclickSubmitButton(confirm_p=false, additional_data={\'confirmed_submit\': true})}</script>'
elif duplicate_dois_after_merge and lost_dois_in_record1:
errcode = 1
message = 'The changes cannot be submitted because the resulting merged record (a) misses DOI(s) managed by the system and/or (b) will create duplicate DOIs.<script type="text/javascript">%(check_duplicate_box)salert(\'The changes cannot be submitted because the resulting merged record (a) misses DOI(s) managed by the system and (b) will create duplicate DOIs.\\n' + \
'The following DOI(s) were in the original record (#1) but are not in the final merged one:\\n' + '\\n'.join(lost_dois_in_record1) + \
'\\nThe following DOI(s) would be duplicate after merge:\\n' + '\\n'.join(duplicate_dois_after_merge) + \
'\\nMake sure that the mentionned DOI(s) are included in the final merged record and/or no duplicate DOIs are created (suggestion: merge in the other way around).\');</script>'
elif duplicate_dois_after_merge:
errcode = 1
message = 'The changes cannot be submitted because the resulting merged record will create a duplicate DOI.<script type="text/javascript">%(check_duplicate_box)salert(\'The changes cannot be submitted because the resulting merged record will create a duplicate DOI.\\n' + \
'The following DOI(s) would be duplicate after merge:\\n' + '\\n'.join(duplicate_dois_after_merge) + \
'\\nMake sure that the mentionned DOI(s) are not duplicated (suggestion: merge in the other way around).\');</script>'
elif not (CFG_BIBEDIT_INTERNAL_DOI_PROTECTION_LEVEL == 1 and submit_confirmed_p):
# lost DOIs after merge
errcode = 1
message = 'The changes cannot be submitted because the resulting merged record misses DOI(s) managed by the system.<script type="text/javascript">%(check_duplicate_box)salert(\'The changes cannot be submitted because the resulting merged record misses the DOI(s) managed by the system.\\n' + \
'The following DOI(s) were in the original record (#1) but are not in the final merged one:\\n' + '\\n'.join(lost_dois_in_record1) + \
'\\nMake sure that the mentionned DOI(s) are included in the final merged record.\');</script>'
message = message % {'check_duplicate_box': record2_marked_as_duplicate_p and '$(\'#bibMergeDupeCheckbox\').attr(\'checked\', true);' or ''}
return (errcode, message)
def move_drafts_articles_to_ready(journal_name, issue):
"""
Move draft articles to their final "collection".
To do so we rely on the convention that an admin-chosen keyword
must be removed from the metadata
"""
protected_datafields = ['100', '245', '246', '520', '590', '700']
keyword_to_remove = get_journal_draft_keyword_to_remove(journal_name)
collections_to_refresh = {}
indexes_to_refresh = get_journal_index_to_refresh_on_release(journal_name)
bibindex_indexes_params = []
if indexes_to_refresh:
bibindex_indexes_params = ['-w', ','.join(indexes_to_refresh)]
categories = get_journal_categories(journal_name, issue)
task_sequence_id = str(bibtask_allocate_sequenceid())
for category in categories:
articles = get_journal_articles(journal_name, issue, category)
for order, recids in iteritems(articles):
for recid in recids:
record_xml = format_record(recid, of='xm')
if not record_xml:
continue
new_record_xml_path = os.path.join(CFG_TMPSHAREDDIR,
'webjournal_publish_' + \
str(recid) + '.xml')
if os.path.exists(new_record_xml_path):
# Do not modify twice
continue
record_struc = create_record(record_xml)
record = record_struc[0]
new_record = update_draft_record_metadata(record,
protected_datafields,
keyword_to_remove)
new_record_xml = print_rec(new_record)
if new_record_xml.find(keyword_to_remove) >= 0:
new_record_xml = new_record_xml.replace(keyword_to_remove, '')
# Write to file
new_record_xml_file = file(new_record_xml_path, 'w')
new_record_xml_file.write(new_record_xml)
new_record_xml_file.close()
# Submit
task_low_level_submission('bibupload',
'WebJournal',
'-c', new_record_xml_path,
'-I', task_sequence_id)
task_low_level_submission('bibindex',
'WebJournal',
'-i', str(recid),
'-I', task_sequence_id,
*bibindex_indexes_params)
for collection in get_all_collections_of_a_record(recid):
collections_to_refresh[collection] = ''
# Refresh collections
collections_to_refresh.update([(c, '') for c in get_journal_collection_to_refresh_on_release(journal_name)])
for collection in collections_to_refresh.keys():
task_low_level_submission('webcoll',
'WebJournal',
'-f', '-P', '2', '-p', '1', '-c', collection,
'-I', task_sequence_id)
def get_bibrecord(recid):
"""Return record in BibRecord wrapping."""
if record_exists(recid):
return create_record(print_record(recid, 'xm'))[0]
def verify_revision(self, verify_record, original_record, opt_mode=None):
"""
Compares the upload record with the same 005 record from archive.
Once the changes are identified, The latest revision of the record is fetched
from the system and the identified changes are applied over the latest.
Returns record patch in case of non-conflicting addition/modification/deletion
Conflicting records raise Error and stops the bibupload process
"""
upload_rev = ''
original_rev = ''
r_date = ''
record_patch = {}
# No need for revision check for other operations
if opt_mode not in ['replace', 'correct']:
return
if '001' in verify_record:
self.rec_id = record_get_field_value(verify_record, '001')
# Retrieving Revision tags for comparison
if '005' in verify_record:
upload_rev = record_get_field_value(verify_record, '005')
r_date = upload_rev.split('.')[0]
if r_date not in [k[1] for k in get_record_revisions(self.rec_id)]:
raise InvenioBibUploadInvalidRevisionError(self.rec_id, r_date)
else:
raise InvenioBibUploadMissing005Error(self.rec_id)
if '005' in original_record:
original_rev = record_get_field_value(original_record, '005')
else:
raise InvenioBibUploadMissing005Error(self.rec_id)
# Retrieving the archived version
marc_xml = get_marcxml_of_record_revision(self.rec_id, r_date)
res = create_record(zlib.decompress(marc_xml[0][0]))
archived_record = res[0]
# Comparing Upload and Archive record
curr_patch = self.compare_records(verify_record, archived_record, opt_mode)
# No changes in Upload Record compared to Archived Revision
# Raising Error to skip the bibupload for the record
if not curr_patch:
raise InvenioBibUploadUnchangedRecordError(self.rec_id, upload_rev)
if original_rev == upload_rev:
# Upload, Archive and Original Records have same Revisions.
affected_tags = self.retrieve_affected_tags_with_ind(curr_patch)
return ('correct', self.generate_final_patch(curr_patch, self.rec_id), affected_tags)
# Comparing Original and Archive record
orig_patch = self.compare_records(original_record, archived_record, opt_mode)
# Checking for conflicts
# If no original patch - Original Record same as Archived Record
if orig_patch:
curr_patch = self.detect_conflict(verify_record, curr_patch, upload_rev, \
original_record, orig_patch, original_rev)
record_patch = self.generate_final_patch(curr_patch, self.rec_id)
affected_tags = self.retrieve_affected_tags_with_ind(curr_patch)
# Returning patch in case of no conflicting fields
return ('correct', record_patch, affected_tags)
def move_drafts_articles_to_ready(journal_name, issue):
"""
Move draft articles to their final "collection".
To do so we rely on the convention that an admin-chosen keyword
must be removed from the metadata
"""
protected_datafields = ['100', '245', '246', '520', '590', '700']
keyword_to_remove = get_journal_draft_keyword_to_remove(journal_name)
collections_to_refresh = {}
indexes_to_refresh = get_journal_index_to_refresh_on_release(journal_name)
bibindex_indexes_params = []
if indexes_to_refresh:
bibindex_indexes_params = ['-w', ','.join(indexes_to_refresh)]
categories = get_journal_categories(journal_name, issue)
task_sequence_id = str(bibtask_allocate_sequenceid())
for category in categories:
articles = get_journal_articles(journal_name, issue, category)
for order, recids in iteritems(articles):
for recid in recids:
record_xml = format_record(recid, of='xm')
if not record_xml:
continue
new_record_xml_path = os.path.join(CFG_TMPSHAREDDIR,
'webjournal_publish_' + \
str(recid) + '.xml')
if os.path.exists(new_record_xml_path):
# Do not modify twice
continue
record_struc = create_record(record_xml)
record = record_struc[0]
new_record = update_draft_record_metadata(record,
protected_datafields,
keyword_to_remove)
new_record_xml = print_rec(new_record)
if new_record_xml.find(keyword_to_remove) >= 0:
new_record_xml = new_record_xml.replace(keyword_to_remove, '')
# Write to file
new_record_xml_file = file(new_record_xml_path, 'w')
new_record_xml_file.write(new_record_xml)
new_record_xml_file.close()
# Submit
task_low_level_submission('bibupload',
'WebJournal',
'-c', new_record_xml_path,
'-I', task_sequence_id)
task_low_level_submission('bibindex',
'WebJournal',
'-i', str(recid),
'-I', task_sequence_id,
*bibindex_indexes_params)
for collection in get_all_collections_of_a_record(recid):
collections_to_refresh[collection] = ''
# Refresh collections
collections_to_refresh.update([(c, '') for c in get_journal_collection_to_refresh_on_release(journal_name)])
for collection in collections_to_refresh.keys():
task_low_level_submission('webcoll',
'WebJournal',
'-f', '-P', '2', '-p', '1', '-c', collection,
'-I', task_sequence_id)
def check_doi_status_after_merge(original_recid1,
original_recid2,
final_record1,
final_record_2,
record2_marked_as_duplicate_p=False,
submit_confirmed_p=False):
"""
Check that the result of the merge does not removed DOIs managed
by the system, and that not duplicate DOI would be
created. Returns a tuple(error_code, message).
@param original_recid1: the record ID of the original record 1 (master)
@type original_recid1: int
@param original_recid2: the record ID of the original record 2 (slave)
@type original_recid2: int
@param final_record1: the resulting merged record
@type final_record1: BibRecord object
@param final_record_2: the resulting slave "merged" record (optional when record2_marked_as_duplicate_p is False)
@type final_record_2: BibRecord object
@param record2_marked_as_duplicate_p: True if the record 2 will be marked as duplicate (and deleted)
@type record2_marked_as_duplicate_p: bool
@param submit_confirmed_p: if the user has already confirmed to proceed with submission, according to previous messages displayed. If True, do not ask again confirmation and proceed if all tests pass.
@type submit_confirmed_p: bool
"""
errcode = 0
message = ''
new_record1_dois = get_dois(final_record1)
new_record1_managed_dois = get_dois(final_record1, internal_only_p=True)
original_record1_managed_dois = get_dois(create_record(
print_record(original_recid1, 'xm'))[0],
internal_only_p=True)
original_record2_dois = get_dois(
create_record(print_record(original_recid2, 'xm'))[0])
# Are there any DOI from record 1 (master) lost in the merging?
lost_dois_in_record1 = [doi for doi in original_record1_managed_dois \
if not doi in new_record1_managed_dois]
# Enough to check for duplicate DOI creation in this record,
# not whole DB
duplicate_dois_after_merge = [
doi for doi in new_record1_dois if new_record1_dois.count(doi) > 1
]
if record2_marked_as_duplicate_p:
new_record2_managed_dois = get_dois(final_record_2,
internal_only_p=True)
original_record2_managed_dois = get_dois(create_record(
print_record(original_recid2, 'xm'))[0],
internal_only_p=True)
# Are there any DOI from record 2 (slave) lost in the merging?
lost_dois_in_record2 = [doi for doi in original_record2_managed_dois \
if not doi in new_record1_managed_dois]
else:
lost_dois_in_record2 = []
duplicate_dois_after_merge += [
doi for doi in new_record1_dois if doi in original_record2_dois
]
if ((lost_dois_in_record1 or lost_dois_in_record2) and \
CFG_BIBEDIT_INTERNAL_DOI_PROTECTION_LEVEL > 0) or \
duplicate_dois_after_merge:
if CFG_BIBEDIT_INTERNAL_DOI_PROTECTION_LEVEL == 1 and \
not duplicate_dois_after_merge and \
not submit_confirmed_p:
errcode = 1
message = 'The resulting merged record misses DOI(s) managed by the system.<script type="text/javascript">%(check_duplicate_box)sif (confirm(\'The resulting merged record will lose DOI(s) managed by the system.\\n' + \
'The following DOI(s) were in the original record (#1) but are not in the final merged one:\\n' + '\\n'.join(lost_dois_in_record1) + \
'\\nAre you sure that you want to submit the merged records without the DOI(s)?\')) {onclickSubmitButton(confirm_p=false, additional_data={\'confirmed_submit\': true})}</script>'
elif duplicate_dois_after_merge and lost_dois_in_record1:
errcode = 1
message = 'The changes cannot be submitted because the resulting merged record (a) misses DOI(s) managed by the system and/or (b) will create duplicate DOIs.<script type="text/javascript">%(check_duplicate_box)salert(\'The changes cannot be submitted because the resulting merged record (a) misses DOI(s) managed by the system and (b) will create duplicate DOIs.\\n' + \
'The following DOI(s) were in the original record (#1) but are not in the final merged one:\\n' + '\\n'.join(lost_dois_in_record1) + \
'\\nThe following DOI(s) would be duplicate after merge:\\n' + '\\n'.join(duplicate_dois_after_merge) + \
'\\nMake sure that the mentionned DOI(s) are included in the final merged record and/or no duplicate DOIs are created (suggestion: merge in the other way around).\');</script>'
elif duplicate_dois_after_merge:
errcode = 1
message = 'The changes cannot be submitted because the resulting merged record will create a duplicate DOI.<script type="text/javascript">%(check_duplicate_box)salert(\'The changes cannot be submitted because the resulting merged record will create a duplicate DOI.\\n' + \
'The following DOI(s) would be duplicate after merge:\\n' + '\\n'.join(duplicate_dois_after_merge) + \
'\\nMake sure that the mentionned DOI(s) are not duplicated (suggestion: merge in the other way around).\');</script>'
elif not (CFG_BIBEDIT_INTERNAL_DOI_PROTECTION_LEVEL == 1
and submit_confirmed_p):
# lost DOIs after merge
errcode = 1
message = 'The changes cannot be submitted because the resulting merged record misses DOI(s) managed by the system.<script type="text/javascript">%(check_duplicate_box)salert(\'The changes cannot be submitted because the resulting merged record misses the DOI(s) managed by the system.\\n' + \
'The following DOI(s) were in the original record (#1) but are not in the final merged one:\\n' + '\\n'.join(lost_dois_in_record1) + \
'\\nMake sure that the mentionned DOI(s) are included in the final merged record.\');</script>'
message = message % {
'check_duplicate_box':
record2_marked_as_duplicate_p
and '$(\'#bibMergeDupeCheckbox\').attr(\'checked\', true);' or ''
}
return (errcode, message)
def _next_value(self, recid=None, xml_record=None, bibrecord=None):
"""Return the next texkey for the given recid.
:param recid: id of the record where the texkey will be generated
:type recid: int
:param xml_record: record in xml format
:type xml_record: string
:return: next texkey for the given recid.
:rtype: string
:raises TexkeyNoAuthorError: No main author (100__a) or collaboration
(710__g) in the given recid
"""
if recid is None and xml_record is not None:
bibrecord = create_record(xml_record)[0]
elif bibrecord is None:
bibrecord = get_bibrecord(recid)
main_author = record_get_field_value(bibrecord,
tag="100",
ind1="",
ind2="",
code="a")
if not main_author:
# Try with collaboration name
main_author = record_get_field_value(bibrecord,
tag="710",
ind1="",
ind2="",
code="g")
main_author = "".join([p for p in main_author.split()
if p.lower() != "collaboration"])
if not main_author:
# Try with corporate author
main_author = record_get_field_value(bibrecord,
tag="100",
ind1="",
ind2="",
code="a")
if not main_author:
raise TexkeyNoAuthorError
# Remove utf-8 special characters
main_author = unidecode(main_author.decode('utf-8'))
try:
texkey_first_part = main_author.split(',')[0].replace(" ", "")
except KeyError:
texkey_first_part = ""
year = record_get_field_value(bibrecord,
tag="269",
ind1="",
ind2="",
code="c")
if not year:
year = record_get_field_value(bibrecord,
tag="260",
ind1="",
ind2="",
code="c")
if not year:
year = record_get_field_value(bibrecord,
tag="773",
ind1="",
ind2="",
code="y")
if not year:
year = record_get_field_value(bibrecord,
tag="502",
ind1="",
ind2="",
code="d")
if not year:
raise TexkeyNoYearError
try:
texkey_second_part = year.split("-")[0]
except KeyError:
texkey_second_part = ""
texkey_third_part = _texkey_random_chars(recid)
texkey = texkey_first_part + ":" + \
texkey_second_part + texkey_third_part
tries = 0
while self._value_exists(texkey) and tries < TEXKEY_MAXTRIES:
# Key is already in the DB, generate a new one
texkey_third_part = _texkey_random_chars(recid, use_random=True)
texkey = texkey_first_part + ":" + \
texkey_second_part + texkey_third_part
tries += 1
return texkey
def _next_value(self, recid=None, xml_record=None, bibrecord=None):
"""
Returns the next texkey for the given recid
@param recid: id of the record where the texkey will be generated
@type recid: int
@param xml_record: record in xml format
@type xml_record: string
@return: next texkey for the given recid.
@rtype: string
@raises TexkeyNoAuthorError: No main author (100__a) or collaboration
(710__g) in the given recid
"""
if recid is None and xml_record is not None:
bibrecord = create_record(xml_record)[0]
elif bibrecord is None:
bibrecord = get_bibrecord(recid)
main_author = record_get_field_value(bibrecord,
tag="100",
ind1="",
ind2="",
code="a")
if not main_author:
# Try with collaboration name
main_author = record_get_field_value(bibrecord,
tag="710",
ind1="",
ind2="",
code="g")
main_author = "".join([
p for p in main_author.split() if p.lower() != "collaboration"
])
if not main_author:
# Try with corporate author
main_author = record_get_field_value(bibrecord,
tag="100",
ind1="",
ind2="",
code="a")
if not main_author:
raise TexkeyNoAuthorError
# Remove utf-8 special characters
main_author = unidecode(main_author.decode('utf-8'))
try:
texkey_first_part = main_author.split(',')[0].replace(" ", "")
except KeyError:
texkey_first_part = ""
year = record_get_field_value(bibrecord,
tag="269",
ind1="",
ind2="",
code="c")
if not year:
year = record_get_field_value(bibrecord,
tag="260",
ind1="",
ind2="",
code="c")
if not year:
year = record_get_field_value(bibrecord,
tag="773",
ind1="",
ind2="",
code="y")
if not year:
year = record_get_field_value(bibrecord,
tag="502",
ind1="",
ind2="",
code="d")
if not year:
raise TexkeyNoYearError
try:
texkey_second_part = year.split("-")[0]
except KeyError:
texkey_second_part = ""
texkey_third_part = _texkey_random_chars(recid)
texkey = texkey_first_part + ":" + texkey_second_part + texkey_third_part
tries = 0
while self._value_exists(texkey) and tries < TEXKEY_MAXTRIES:
# Key is already in the DB, generate a new one
texkey_third_part = _texkey_random_chars(recid, use_random=True)
texkey = texkey_first_part + ":" + texkey_second_part + texkey_third_part
tries += 1
return texkey