def tearDown(self):
"""Helper function that restores recID 3 MARCXML"""
recs = bibupload.xml_marc_to_records(self.bibupload_xml)
bibupload.bibupload(recs[0], opt_mode='delete')
oai_harvest_daemon.oai_harvest_get = self.oai_harvest_get
oai_harvest_dblayer.get_oai_src = self.get_oai_src
run_sql("UPDATE bibrec SET modification_date=%s WHERE id=%s", (self.original_modification_date, self.recid,))
def perform_upload_check(xml_record, mode):
""" Performs a upload simulation with the given record and mode
@return: string describing errors
@rtype: string
"""
error_cache = []
def my_writer(msg, stream=sys.stdout, verbose=1):
if verbose == 1:
if 'DONE' not in msg:
error_cache.append(msg.strip())
orig_writer = bibupload_module.write_message
bibupload_module.write_message = my_writer
error_cache.extend(perform_basic_upload_checks(xml_record))
if error_cache:
# There has been some critical error
return '\n'.join(error_cache)
recs = xml_marc_to_records(xml_record)
try:
upload_mode = mode[2:]
# Adapt input data for bibupload function
if upload_mode == "r insert-or-replace":
upload_mode = "replace_or_insert"
for record in recs:
if record:
record_strip_empty_volatile_subfields(record)
record_strip_empty_fields(record)
bibupload(record, opt_mode=upload_mode, pretend=True)
finally:
bibupload_module.write_message = orig_writer
return '\n'.join(error_cache)
def bst_openaire_altmetric():
"""
"""
recids = search_pattern(p="0->Z", f="0247_a")
a = Altmetric()
for recid in recids:
try:
# Check if we already have an Altmetric id
sysno_inst = get_fieldvalues(recid, "035__9")
if ['Altmetric'] in sysno_inst:
continue
doi_val = get_fieldvalues(recid, "0247_a")[0]
json_res = a.doi(doi_val)
rec = {}
record_add_field(rec, "001", controlfield_value=str(recid))
if json_res:
record_add_field(rec,
'035',
subfields=[('a',
str(json_res['altmetric_id'])),
('9', 'Altmetric')])
bibupload(rec, opt_mode='correct')
except AltmetricHTTPException, e:
register_exception(prefix='Altmetric error (status code %s): %s' %
(e.status_code, str(e)),
alert_admin=False)
def bst_openaire_altmetric():
"""
"""
recids = search_pattern(p="0->Z", f="0247_a")
a = Altmetric()
for recid in recids:
try:
# Check if we already have an Altmetric id
sysno_inst = get_fieldvalues(recid, "035__9")
if ['Altmetric'] in sysno_inst:
continue
doi_val = get_fieldvalues(recid, "0247_a")[0]
json_res = a.doi(doi_val)
rec = {}
record_add_field(rec, "001", controlfield_value=str(recid))
if json_res:
record_add_field(rec, '035', subfields=[('a',
str(json_res['altmetric_id'])), ('9', 'Altmetric')])
bibupload(rec, opt_mode='correct')
except AltmetricHTTPException, e:
register_exception(prefix='Altmetric error (status code %s): %s' %
(e.status_code, str(e)), alert_admin=False)
def perform_upload_check(xml_record, mode):
""" Performs a upload simulation with the given record and mode
@return: string describing errors
@rtype: string
"""
error_cache = []
def my_writer(msg, stream=sys.stdout, verbose=1):
if verbose == 1:
if 'DONE' not in msg:
error_cache.append(msg.strip())
orig_writer = bibupload_module.write_message
bibupload_module.write_message = my_writer
error_cache.extend(perform_basic_upload_checks(xml_record))
if error_cache:
# There has been some critical error
return '\n'.join(error_cache)
recs = xml_marc_to_records(xml_record)
try:
upload_mode = mode[2:]
# Adapt input data for bibupload function
if upload_mode == "r insert-or-replace":
upload_mode = "replace_or_insert"
for record in recs:
if record:
record_strip_empty_volatile_subfields(record)
record_strip_empty_fields(record)
bibupload(record, opt_mode=upload_mode, pretend=True)
finally:
bibupload_module.write_message = orig_writer
return '\n'.join(error_cache)
def tearDown(self):
"""Helper function that restores recID 3 MARCXML"""
recs = bibupload.xml_marc_to_records(self.bibupload_xml)
bibupload.bibupload(recs[0], opt_mode='delete')
oai_harvest_daemon.oai_harvest_get = self.oai_harvest_get
oai_harvest_dblayer.get_oai_src = self.get_oai_src
run_sql("UPDATE bibrec SET modification_date=%s WHERE id=%s", (
self.original_modification_date,
self.recid,
))
def migrate_record(recid, substitutions=[], additions=[]):
from invenio.search_engine import get_record
from invenio.bibrecord import record_add_field
from invenio.bibupload import bibupload
from invenio.search_engine import get_fieldvalues
rec = get_record(recid)
if get_fieldvalues(recid, '536__c'):
additions = additions + [
[('a', 'user-ecfunded')],
]
if recid == 941:
additions = additions + [
[('a', 'publication'), ('b', 'workingpaper')],
]
for k in rec.keys():
if k not in ['001', '980']:
del rec[k]
if substitutions:
try:
newcolls = []
colls = rec['980']
for c in colls:
try:
# We are only interested in subfield 'a'
code, val = c[0][0]
for old, new in substitutions:
if val == old:
c[0][0] = (code, new)
break
except IndexError:
pass
newcolls.append(c)
del rec['980']
for c in newcolls:
record_add_field(rec, '980', subfields=c[0])
except KeyError:
warnings.warn("Could not migrate record %s" % recid)
return
if additions:
for a in additions:
record_add_field(rec, '980', subfields=a)
print rec
bibupload(rec, opt_mode="correct")
def bibupload_merger(merged_bibrecords, logger, opt_mode="replace_or_insert", pretend=False):
"""Function to upload directly in the Invenio DB"""
def write_message(msg, stream=sys.stdout, verbose=False):
"""Custom definition of write_message
to override the Invenio log"""
#logger.info(msg)
pass
#I override the function inside Invenio
bibupload.write_message = write_message
for bibrecord in merged_bibrecords:
bibupload.bibupload(bibrecord, opt_tag=None, opt_mode=opt_mode,
opt_stage_to_start_from=1, opt_notimechange=0, oai_rec_id = "", pretend=pretend)
def migrate_record(recid):
"""
Migrate a record from Orphan to ZENODO
"""
from invenio.bibrecord import record_add_field
from invenio.bibupload import bibupload
rec = {}
record_add_field(rec, '001', controlfield_value=str(recid))
(rec, is_public) = migrate_980__ab(recid, rec)
(rec, accessright) = migrate_542__l(recid, rec)
migrate_bibdoc_status(recid, is_public, accessright)
bibupload(rec, opt_mode="correct")
def migrate_record(recid):
"""
Migrate a record from Orphan to ZENODO
"""
from invenio.bibrecord import record_add_field
from invenio.bibupload import bibupload
rec = {}
record_add_field(rec, '001', controlfield_value=str(recid))
(rec, is_public) = migrate_980__ab(recid, rec)
(rec, accessright) = migrate_542__l(recid, rec)
migrate_bibdoc_status(recid, is_public, accessright)
bibupload(rec, opt_mode="correct")
def migrate_record(recid, substitutions=[], additions=[]):
from invenio.search_engine import get_record
from invenio.bibrecord import record_add_field
from invenio.bibupload import bibupload
from invenio.search_engine import get_fieldvalues
rec = get_record(recid)
if get_fieldvalues(recid, '536__c'):
additions = additions + [[('a', 'user-ecfunded')], ]
if recid == 941:
additions = additions + [[('a', 'publication'), ('b', 'workingpaper')], ]
for k in rec.keys():
if k not in ['001', '980']:
del rec[k]
if substitutions:
try:
newcolls = []
colls = rec['980']
for c in colls:
try:
# We are only interested in subfield 'a'
code, val = c[0][0]
for old, new in substitutions:
if val == old:
c[0][0] = (code, new)
break
except IndexError:
pass
newcolls.append(c)
del rec['980']
for c in newcolls:
record_add_field(rec, '980', subfields=c[0])
except KeyError:
warnings.warn("Could not migrate record %s" % recid)
return
if additions:
for a in additions:
record_add_field(rec, '980', subfields=a)
print rec
bibupload(rec, opt_mode="correct")
def setUp(self, recid=RECID, arxiv_id=ARXIV_ID):
self.recid = recid
self.arxiv_id = arxiv_id
self.arxiv_version = 1
self.bibupload_xml = """<record>
<controlfield tag="001">%s</controlfield>
<datafield tag="037" ind1=" " ind2=" ">
<subfield code="a">arXiv:%s</subfield>
<subfield code="9">arXiv</subfield>
<subfield code="c">hep-ph</subfield>
</datafield>
</record>""" % (recid, arxiv_id)
bibtask.setup_loggers()
bibtask.task_set_task_param('verbose', 0)
recs = bibupload.xml_marc_to_records(self.bibupload_xml)
status, dummy, err = bibupload.bibupload(recs[0], opt_mode='correct')
assert status == 0, err.strip()
assert len(get_fieldvalues(recid, '037__a')) == 1
def mocked_oai_harvest_get(prefix, baseurl, harvestpath,
verb, identifier):
temp_fd, temp_path = mkstemp()
os.write(temp_fd, ARXIV_OAI_RESPONSE % self.arxiv_version)
os.close(temp_fd)
return [temp_path]
self.oai_harvest_get = oai_harvest_daemon.oai_harvest_get
oai_harvest_daemon.oai_harvest_get = mocked_oai_harvest_get
def mocked_get_oai_src(params={}):
return [{'baseurl': ''}]
self.get_oai_src = oai_harvest_dblayer.get_oai_src
oai_harvest_dblayer.get_oai_src = mocked_get_oai_src
def main(argv):
recID=0
opts,pargs=getopt.getopt(argv,'di:')
verbose = False
for opt, arg in opts:
if opt == '-i':
recID=arg
if opt == '-d':
verbose = True
result=format_record(recID=recID,of='xm')
if result:
#change the result to MARC by applying a template
if verbose:
print result
raw_input("go on?")
result = bibconvert_xslt_engine.convert(result, "marcxmltoplain.xsl")
#call a sub that changes the stuff to editable form, calls editor,
#returns a string
new = convert_edit(result)
newr = to_marc(new)
if verbose:
#debug
f=open('/tmp/debug', 'w')
f.write(new)
f.write(newr)
f.close()
print newr
if upper(raw_input("Save to DB Y/N:")) =='Y':
recs=xml_marc_to_records(''.join(newr))
response=bibupload(recs[0],opt_mode='replace')
if response[0]:print "Error updating record: "+response[0]
def create_delete(message):
"""creates and deletes the record"""
diff = message.getParam('diff')
if diff:
diff = int(str(diff))
else:
diff = 5 # 5 secs older
recid = bibupload.create_new_record()
#rec_id = bibupload.create_new_record()
#expected_rec_id = dbquery.run_sql("SELECT MAX(id) FROM bibrec")[0][0] + 1
xml_to_delete = """
<record>
<datafield tag="100" ind1=" " ind2=" ">
<subfield code="a">Test, Jane</subfield>
<subfield code="u">Test Institute</subfield>
</datafield>
<datafield tag="100" ind1="4" ind2="7">
<subfield code="a">Test, Johnson</subfield>
<subfield code="u">Test University</subfield>
</datafield>
<datafield tag="980" ind1="" ind2="">
<subfield code="c">DELETED</subfield>
</datafield>
</record>
"""
recs = bibupload.xml_marc_to_records(xml_to_delete)
ret = bibupload.bibupload(recs[0], opt_mode='insert')
recid = ret[1]
message.setResults(Integer(recid))
change_date(recid, diff=diff)
def main(argv):
recID=0
opts,pargs=getopt.getopt(argv,'di:')
verbose = False
for opt, arg in opts:
if opt == '-i':
recID=arg
if opt == '-d':
verbose = True
result=format_record(recID=recID,of='xm')
if result:
#change the result to MARC by applying a template
if verbose:
print result
raw_input("go on?")
result = bibconvert_xslt_engine.convert(result, "marcxmltoplain.xsl")
#call a sub that changes the stuff to editable form, calls editor,
#returns a string
new = convert_edit(result)
newr = to_marc(new)
if verbose:
#debug
f=open('/tmp/debug', 'w')
f.write(new)
f.write(newr)
f.close()
print newr
if upper(raw_input("Save to DB Y/N:")) =='Y':
recs=xml_marc_to_records(''.join(newr))
response=bibupload(recs[0],opt_mode='replace')
if response[0]:print "Error updating record: "+response[0]
def get_bibrec_for_record(marcxml, opt_mode):
'''
A record is uploaded to the system using mainly functionality
of the bibupload module. Then a bibrec is returned for the record.
'''
recs = create_record(marcxml, parser='lxml')
_, recid, _ = bibupload(recs[0], opt_mode=opt_mode)
return recid
def test_BibUpload_revision_verifier(self):
""" BibUpload Revision Verifier - Called from BibUpload Operation - Patch & Conflict Scenarios"""
recs = xml_marc_to_records(self.rev1)
# --> Revision 1 submitted
error, self.recid, dummy_msg = bibupload(recs[0], opt_mode='insert')
self.check_record_consistency(self.recid)
record = get_record(self.recid)
rev = record_get_field_value(record, '005', '', '')
recs = xml_marc_to_records(self.rev1)
self.rev2 = self.rev2.replace('123456789', str(self.recid))
self.rev2 = self.rev2.replace('20110101000000.0', rev)
self.rev1_modified = self.rev1_modified.replace(
'123456789', str(self.recid))
self.rev1_modified = self.rev1_modified.replace(
'20110101000000.0', rev)
self.final_xm = self.final_xm.replace('123456789', str(self.recid))
recs = xml_marc_to_records(self.rev1)
recs = xml_marc_to_records(self.rev2)
# --> Revision 2 submitted
error, self.recid, dummy_msg = bibupload(recs[0], opt_mode='replace')
self.check_record_consistency(self.recid)
record = get_record(self.recid)
self.rev2 = self.rev2.replace(
rev, record_get_field_value(record, '005', '', ''))
self.rev2_modified = self.rev2_modified.replace(
'123456789', str(self.recid))
self.rev2_modified = self.rev2_modified.replace(
'20110101000000.0', record_get_field_value(record, '005', '', ''))
# --> Revision 1 modified submitted
recs = xml_marc_to_records(self.rev1_modified)
error, self.recid, dummy_msg = bibupload(recs[0], opt_mode='replace')
self.check_record_consistency(self.recid)
record = get_record(self.recid)
rev = record_get_field_value(record, '005', '', '')
self.final_xm = self.final_xm.replace('20110101000000.0', rev)
self.assertEqual(
compare_xmbuffers(self.final_xm, print_record(self.recid, 'xm')),
'')
# --> Revision 2 modified submitted
recs = xml_marc_to_records(self.rev2_modified)
error, self.recid, dummy_msg = bibupload(recs[0], opt_mode='replace')
self.check_record_consistency(self.recid)
self.assertEquals(error, 2)
def test_BibUpload_revision_verifier(self):
""" BibUpload Revision Verifier - Called from BibUpload Operation - Patch & Conflict Scenarios"""
recs = xml_marc_to_records(self.rev1)
# --> Revision 1 submitted
error, self.recid, dummy_msg = bibupload(recs[0], opt_mode="insert")
self.check_record_consistency(self.recid)
record = get_record(self.recid)
rev = record_get_field_value(record, "005", "", "")
recs = xml_marc_to_records(self.rev1)
self.rev2 = self.rev2.replace("123456789", str(self.recid))
self.rev2 = self.rev2.replace("20110101000000.0", rev)
self.rev1_modified = self.rev1_modified.replace("123456789", str(self.recid))
self.rev1_modified = self.rev1_modified.replace("20110101000000.0", rev)
self.final_xm = self.final_xm.replace("123456789", str(self.recid))
recs = xml_marc_to_records(self.rev1)
recs = xml_marc_to_records(self.rev2)
# --> Revision 2 submitted
error, self.recid, dummy_msg = bibupload(recs[0], opt_mode="replace")
self.check_record_consistency(self.recid)
record = get_record(self.recid)
self.rev2 = self.rev2.replace(rev, record_get_field_value(record, "005", "", ""))
self.rev2_modified = self.rev2_modified.replace("123456789", str(self.recid))
self.rev2_modified = self.rev2_modified.replace(
"20110101000000.0", record_get_field_value(record, "005", "", "")
)
# --> Revision 1 modified submitted
recs = xml_marc_to_records(self.rev1_modified)
error, self.recid, dummy_msg = bibupload(recs[0], opt_mode="replace")
self.check_record_consistency(self.recid)
record = get_record(self.recid)
rev = record_get_field_value(record, "005", "", "")
self.final_xm = self.final_xm.replace("20110101000000.0", rev)
self.assertEqual(compare_xmbuffers(self.final_xm, print_record(self.recid, "xm")), "")
# --> Revision 2 modified submitted
recs = xml_marc_to_records(self.rev2_modified)
error, self.recid, dummy_msg = bibupload(recs[0], opt_mode="replace")
self.check_record_consistency(self.recid)
self.assertEquals(error, 2)
def delete_record(message):
"""deletes record"""
diff = message.getParam('diff')
if diff:
diff = int(str(diff))
else:
diff = 5 # 5 secs older
recid = int(Integer.cast_(message.getParam('recid')).intValue())
record = search_engine.get_record(recid)
bibrecord.record_add_field(record, "980", subfields=[("c", "DELETED")])
ret = bibupload.bibupload(record, opt_mode='replace')
recid = ret[1]
message.setResults(Integer(recid))
# extra query needed because invenio sets modification date=NOW()
# and so the deleted recs have modification_date<creation_date
#dbquery.run_sql("UPDATE bibrec SET modification_date=NOW() + %s WHERE id=%s" % (diff, recid))
change_date(recid, diff=diff)
def create_record(message):
"""creates record"""
diff = message.getParam('diff')
if diff:
diff = int(str(diff))
else:
diff = 5 # 5 secs older
recid = bibupload.create_new_record()
message.setResults(Integer(int(recid)))
#dbquery.run_sql("UPDATE bibrec SET modification_date=NOW() + %s, creation_date=NOW() + %s WHERE id=%s" % (diff, diff,recid))
data = message.getParam('data')
if data:
record = search_engine.get_record(recid)
for field_data in str(data).split("|"):
points = field_data.split(":", 3)
bibrecord.record_add_field(record, points[0], subfields=[(points[1], points[2])])
ret = bibupload.bibupload(record, opt_mode='replace')
change_date(recid, diff=diff, cdiff=diff)
def init_test_records():
"""
Initializes test records for revision verifying scenarios
Inserts 1st version and then appends new field every 1 sec
to create 2nd and 3rd version of the record
Returns a dict of following format :
{'id':recid,
'rev1':(rev1_rec, rev1_005),
'rev2':(rev2_rec, rev2_005tag),
'rev3':(rev3_rec, rev3_005tag)}
"""
# Rev 1 -- tag 100
rev1 = """ <record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester, T</subfield>
<subfield code="u">DESY</subfield>
</datafield>
</record>"""
# Append 970 to Rev1
rev1_append = """<record>
<controlfield tag="001">123456789</controlfield>
<datafield tag ="970" ind1=" " ind2=" ">
<subfield code="a">0003719PHOPHO</subfield>
</datafield>
</record>"""
# Rev 2 -- Rev 1 + tag 970
rev2 = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester, T</subfield>
<subfield code="u">DESY</subfield>
</datafield>
<datafield tag ="970" ind1=" " ind2=" ">
<subfield code="a">0003719PHOPHO</subfield>
</datafield>
</record>"""
# Append 888 to Rev2
rev2_append = """<record>
<controlfield tag="001">123456789</controlfield>
<datafield tag="888" ind1=" " ind2=" ">
<subfield code="a">dumb text</subfield>
</datafield>
</record>"""
# Rev 3 -- Rev 2 + tag 888
rev3 = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester, T</subfield>
<subfield code="u">DESY</subfield>
</datafield>
<datafield tag ="970" ind1=" " ind2=" ">
<subfield code="a">0003719PHOPHO</subfield>
</datafield>
<datafield tag="888" ind1=" " ind2=" ">
<subfield code="a">dumb text</subfield>
</datafield>
</record>"""
init_details = {}
insert_record = rev1.replace(
'<controlfield tag="001">123456789</controlfield>', '')
insert_record = insert_record.replace(
'<controlfield tag="005">20110101000000.0</controlfield>', '')
recs = xml_marc_to_records(insert_record)
# --> Revision 1 submitted
res = bibupload(recs[0], opt_mode='insert')
recid = res[1]
init_details['id'] = (str(recid), )
rec = get_record(recid)
rev_tag = record_get_field_value(rec, '005', '', '')
# update the test data
rev1 = rev1.replace('123456789', str(recid))
rev1 = rev1.replace('20110101000000.0', rev_tag)
rev1_append = rev1_append.replace('123456789', str(recid))
rev2 = rev2.replace('123456789', str(recid))
rev2 = rev2.replace('20110101000000.0', rev_tag)
rev2_append = rev2_append.replace('123456789', str(recid))
rev3 = rev3.replace('123456789', str(recid))
init_details['rev1'] = (rev1, rev_tag)
old_rev_tag = rev_tag
# --> Revision 2 submitted
recs = xml_marc_to_records(rev1_append)
res = bibupload(recs[0], opt_mode='append')
rec = get_record(recid)
rev_tag = record_get_field_value(rec, '005')
rev2 = rev2.replace(old_rev_tag, rev_tag)
rev3 = rev3.replace('20110101000000.0', rev_tag)
init_details['rev2'] = (rev2, rev_tag)
old_rev_tag = rev_tag
# --> Revision 3 submitted
recs = xml_marc_to_records(rev2_append)
res = bibupload(recs[0], opt_mode='append')
rec = get_record(recid)
rev_tag = record_get_field_value(rec, '005')
rev3 = rev3.replace(old_rev_tag, rev_tag)
init_details['rev3'] = (rev3, rev_tag)
return init_details
def setUp(self):
""" Sets up sample records for Modified Fields Scenarios."""
GenericBibUploadTest.setUp(self)
# Rev 1
self.rev1 = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester, T</subfield>
<subfield code="u">DESY</subfield>
</datafield>
<datafield tag ="300" ind1=" " ind2=" ">
<subfield code="a">Test, Field-1</subfield>
</datafield>
<datafield tag ="300" ind1=" " ind2=" ">
<subfield code="a">Test, Field-2</subfield>
</datafield>
<datafield tag ="300" ind1="C" ind2="P">
<subfield code="a">Test, Field-3</subfield>
</datafield>
</record>"""
# Rev 1 -- To Replace
self.rev1_mod = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester, T</subfield>
<subfield code="u">DESY</subfield>
</datafield>
</record>"""
# Patch with SPECIAL DELETE FIELD-1
self.patch_1 = """<record>
<controlfield tag="001">123456789</controlfield>
<datafield tag ="300" ind1=" " ind2=" ">
<subfield code="0">__DELETE_FIELDS__</subfield>
</datafield>
<datafield tag ="300" ind1="C" ind2="P">
<subfield code="0">__DELETE_FIELDS__</subfield>
</datafield>
</record>"""
# Patch with SPECIAL DELETE FIELD-2
self.patch_2 = """<record>
<controlfield tag="001">123456789</controlfield>
<datafield tag ="300" ind1="C" ind2="P">
<subfield code="0">__DELETE_FIELDS__</subfield>
</datafield>
<datafield tag ="300" ind1=" " ind2=" ">
<subfield code="0">__DELETE_FIELDS__</subfield>
</datafield>
</record>"""
self.rev_to_insert = self.rev1.replace('<controlfield tag="001">123456789</controlfield>', '')
self.rev_to_insert = self.rev_to_insert.replace('<controlfield tag="005">20110101000000.0</controlfield>','')
rec = xml_marc_to_records(self.rev_to_insert)
dummy_error, self.recid, dummy_msg = bibupload(rec[0], opt_mode='insert')
self.check_record_consistency(self.recid)
self.rev1 = self.rev1.replace('123456789', str(self.recid))
self.rev1_mod = self.rev1_mod.replace('123456789', str(self.recid))
self.patch_1 = self.patch_1.replace('123456789', str(self.recid))
self.patch_2 = self.patch_2.replace('123456789', str(self.recid))
record = get_record(self.recid)
rev = record_get_field_value(record, '005')
self.rev1 = self.rev1.replace('20110101000000.0', rev)
self.rev1_mod = self.rev1_mod.replace('20110101000000.0', rev)
def setUp(self):
""" Sets up sample records for Modified Fields Scenarios."""
GenericBibUploadTest.setUp(self)
# Rev 1 -- 100-1/100-2/100-3
self.rev1 = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester1, T</subfield>
<subfield code="u">DESY1</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester2, T</subfield>
<subfield code="u">DESY2</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester3, T</subfield>
<subfield code="u">DESY3</subfield>
</datafield>
<datafield tag ="970" ind1=" " ind2=" ">
<subfield code="a">0003719PHYPHY</subfield>
</datafield>
<datafield tag="888" ind1=" " ind2=" ">
<subfield code="a">dumb text</subfield>
</datafield>
</record>"""
# Rev 1 Modified -- 100-2/100-3/100-1
self.rev1_mod = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester2, T</subfield>
<subfield code="u">DESY2</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester3, T</subfield>
<subfield code="u">DESY3</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester1, T</subfield>
<subfield code="u">DESY1</subfield>
</datafield>
<datafield tag ="970" ind1=" " ind2=" ">
<subfield code="a">0003719PHYPHY</subfield>
</datafield>
<datafield tag="888" ind1=" " ind2=" ">
<subfield code="a">dumb text</subfield>
</datafield>
</record>"""
self.patch = """<record>
<controlfield tag="001">123456789</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester2, T</subfield>
<subfield code="u">DESY2</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester3, T</subfield>
<subfield code="u">DESY3</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester1, T</subfield>
<subfield code="u">DESY1</subfield>
</datafield>
</record>"""
insert_record = self.rev1.replace(
'<controlfield tag="001">123456789</controlfield>', '')
insert_record = insert_record.replace(
'<controlfield tag="005">20110101000000.0</controlfield>', '')
recs = xml_marc_to_records(insert_record)
# --> Revision 1 submitted
res = bibupload(recs[0], opt_mode='insert')
self.recid = res[1]
self.check_record_consistency(self.recid)
rec = get_record(self.recid)
rev_tag = record_get_field_value(rec, '005', '', '')
# update the test data
self.rev1 = self.rev1.replace('123456789', str(self.recid))
self.rev1 = self.rev1.replace('20110101000000.0', rev_tag)
self.rev1_mod = self.rev1_mod.replace('123456789', str(self.recid))
self.rev1_mod = self.rev1_mod.replace('20110101000000.0', rev_tag)
self.patch = self.patch.replace('123456789', str(self.recid))
def init_test_records():
"""
Initializes test records for revision verifying scenarios
Inserts 1st version and then appends new field every 1 sec
to create 2nd and 3rd version of the record
Returns a dict of following format :
{'id':recid,
'rev1':(rev1_rec, rev1_005),
'rev2':(rev2_rec, rev2_005tag),
'rev3':(rev3_rec, rev3_005tag)}
"""
# Rev 1 -- tag 100
rev1 = """ <record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester, T</subfield>
<subfield code="u">DESY</subfield>
</datafield>
</record>"""
# Append 970 to Rev1
rev1_append = """<record>
<controlfield tag="001">123456789</controlfield>
<datafield tag ="970" ind1=" " ind2=" ">
<subfield code="a">0003719PHOPHO</subfield>
</datafield>
</record>"""
# Rev 2 -- Rev 1 + tag 970
rev2 = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester, T</subfield>
<subfield code="u">DESY</subfield>
</datafield>
<datafield tag ="970" ind1=" " ind2=" ">
<subfield code="a">0003719PHOPHO</subfield>
</datafield>
</record>"""
# Append 888 to Rev2
rev2_append = """<record>
<controlfield tag="001">123456789</controlfield>
<datafield tag="888" ind1=" " ind2=" ">
<subfield code="a">dumb text</subfield>
</datafield>
</record>"""
# Rev 3 -- Rev 2 + tag 888
rev3 = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester, T</subfield>
<subfield code="u">DESY</subfield>
</datafield>
<datafield tag ="970" ind1=" " ind2=" ">
<subfield code="a">0003719PHOPHO</subfield>
</datafield>
<datafield tag="888" ind1=" " ind2=" ">
<subfield code="a">dumb text</subfield>
</datafield>
</record>"""
init_details = {}
insert_record = rev1.replace(
'<controlfield tag="001">123456789</controlfield>', '')
insert_record = insert_record.replace(
'<controlfield tag="005">20110101000000.0</controlfield>', '')
recs = xml_marc_to_records(insert_record)
# --> Revision 1 submitted
res = bibupload(recs[0], opt_mode='insert')
recid = res[1]
init_details['id'] = (str(recid), )
rec = get_record(recid)
rev_tag = record_get_field_value(rec, '005', '', '')
# update the test data
rev1 = rev1.replace('123456789', str(recid))
rev1 = rev1.replace('20110101000000.0', rev_tag)
rev1_append = rev1_append.replace('123456789', str(recid))
rev2 = rev2.replace('123456789', str(recid))
rev2 = rev2.replace('20110101000000.0', rev_tag)
rev2_append = rev2_append.replace('123456789', str(recid))
rev3 = rev3.replace('123456789', str(recid))
init_details['rev1'] = (rev1, rev_tag)
old_rev_tag = rev_tag
# --> Revision 2 submitted
recs = xml_marc_to_records(rev1_append)
res = bibupload(recs[0], opt_mode='append')
rec = get_record(recid)
rev_tag = record_get_field_value(rec, '005')
rev2 = rev2.replace(old_rev_tag, rev_tag)
rev3 = rev3.replace('20110101000000.0', rev_tag)
init_details['rev2'] = (rev2, rev_tag)
old_rev_tag = rev_tag
# --> Revision 3 submitted
recs = xml_marc_to_records(rev2_append)
res = bibupload(recs[0], opt_mode='append')
rec = get_record(recid)
rev_tag = record_get_field_value(rec, '005')
rev3 = rev3.replace(old_rev_tag, rev_tag)
init_details['rev3'] = (rev3, rev_tag)
return init_details
def setUp(self):
""" Sets up sample records for Modified Fields Scenarios."""
GenericBibUploadTest.setUp(self)
# Rev 1 -- 100-1/100-2/100-3
self.rev1 = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester1, T</subfield>
<subfield code="u">DESY1</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester2, T</subfield>
<subfield code="u">DESY2</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester3, T</subfield>
<subfield code="u">DESY3</subfield>
</datafield>
<datafield tag ="970" ind1=" " ind2=" ">
<subfield code="a">0003719PHYPHY</subfield>
</datafield>
<datafield tag="888" ind1=" " ind2=" ">
<subfield code="a">dumb text</subfield>
</datafield>
</record>"""
# Rev 1 Modified -- 100-2/100-3/100-1
self.rev1_mod = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester2, T</subfield>
<subfield code="u">DESY2</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester3, T</subfield>
<subfield code="u">DESY3</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester1, T</subfield>
<subfield code="u">DESY1</subfield>
</datafield>
<datafield tag ="970" ind1=" " ind2=" ">
<subfield code="a">0003719PHYPHY</subfield>
</datafield>
<datafield tag="888" ind1=" " ind2=" ">
<subfield code="a">dumb text</subfield>
</datafield>
</record>"""
self.patch = """<record>
<controlfield tag="001">123456789</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester2, T</subfield>
<subfield code="u">DESY2</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester3, T</subfield>
<subfield code="u">DESY3</subfield>
</datafield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester1, T</subfield>
<subfield code="u">DESY1</subfield>
</datafield>
</record>"""
insert_record = self.rev1.replace(
'<controlfield tag="001">123456789</controlfield>', '')
insert_record = insert_record.replace(
'<controlfield tag="005">20110101000000.0</controlfield>', '')
recs = xml_marc_to_records(insert_record)
# --> Revision 1 submitted
res = bibupload(recs[0], opt_mode='insert')
self.recid = res[1]
self.check_record_consistency(self.recid)
rec = get_record(self.recid)
rev_tag = record_get_field_value(rec, '005', '', '')
# update the test data
self.rev1 = self.rev1.replace('123456789', str(self.recid))
self.rev1 = self.rev1.replace('20110101000000.0', rev_tag)
self.rev1_mod = self.rev1_mod.replace('123456789', str(self.recid))
self.rev1_mod = self.rev1_mod.replace('20110101000000.0', rev_tag)
self.patch = self.patch.replace('123456789', str(self.recid))
def tearDown(self):
"""Helper function that restores recID 3 MARCXML"""
recs = bibupload.xml_marc_to_records(self.bibupload_xml)
bibupload.bibupload(recs[0], opt_mode='delete')
oai_harvest_daemon.oai_harvest_get = self.oai_harvest_get
oai_harvest_dblayer.get_oai_src = self.get_oai_src
def setUp(self):
""" Sets up sample records for Modified Fields Scenarios."""
GenericBibUploadTest.setUp(self)
# Rev 1
self.rev1 = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester, T</subfield>
<subfield code="u">DESY</subfield>
</datafield>
<datafield tag ="300" ind1=" " ind2=" ">
<subfield code="a">Test, Field-1</subfield>
</datafield>
<datafield tag ="300" ind1=" " ind2=" ">
<subfield code="a">Test, Field-2</subfield>
</datafield>
<datafield tag ="300" ind1="C" ind2="P">
<subfield code="a">Test, Field-3</subfield>
</datafield>
</record>"""
# Rev 1 -- To Replace
self.rev1_mod = """<record>
<controlfield tag="001">123456789</controlfield>
<controlfield tag="005">20110101000000.0</controlfield>
<datafield tag ="100" ind1=" " ind2=" ">
<subfield code="a">Tester, T</subfield>
<subfield code="u">DESY</subfield>
</datafield>
</record>"""
# Patch with SPECIAL DELETE FIELD-1
self.patch_1 = """<record>
<controlfield tag="001">123456789</controlfield>
<datafield tag ="300" ind1=" " ind2=" ">
<subfield code="0">__DELETE_FIELDS__</subfield>
</datafield>
<datafield tag ="300" ind1="C" ind2="P">
<subfield code="0">__DELETE_FIELDS__</subfield>
</datafield>
</record>"""
# Patch with SPECIAL DELETE FIELD-2
self.patch_2 = """<record>
<controlfield tag="001">123456789</controlfield>
<datafield tag ="300" ind1="C" ind2="P">
<subfield code="0">__DELETE_FIELDS__</subfield>
</datafield>
<datafield tag ="300" ind1=" " ind2=" ">
<subfield code="0">__DELETE_FIELDS__</subfield>
</datafield>
</record>"""
self.rev_to_insert = self.rev1.replace(
'<controlfield tag="001">123456789</controlfield>', '')
self.rev_to_insert = self.rev_to_insert.replace(
'<controlfield tag="005">20110101000000.0</controlfield>', '')
rec = xml_marc_to_records(self.rev_to_insert)
dummy_error, self.recid, dummy_msg = bibupload(rec[0],
opt_mode='insert')
self.check_record_consistency(self.recid)
self.rev1 = self.rev1.replace('123456789', str(self.recid))
self.rev1_mod = self.rev1_mod.replace('123456789', str(self.recid))
self.patch_1 = self.patch_1.replace('123456789', str(self.recid))
self.patch_2 = self.patch_2.replace('123456789', str(self.recid))
record = get_record(self.recid)
rev = record_get_field_value(record, '005')
self.rev1 = self.rev1.replace('20110101000000.0', rev)
self.rev1_mod = self.rev1_mod.replace('20110101000000.0', rev)
def tearDown(self):
"""Helper function that restores recID 3 MARCXML"""
recs = bibupload.xml_marc_to_records(self.bibupload_xml)
bibupload.bibupload(recs[0], opt_mode='delete')
oai_harvest_daemon.oai_harvest_get = self.oai_harvest_get
oai_harvest_dblayer.get_oai_src = self.get_oai_src
