def test_12b(self):
args = {}
args['authority'] = 1
tag, code = '991', 'd'
report = ReportList.get_by_name('speech_missing_' + tag + code)
Bib({
'_id': 1
}).set_values(('791', 'a', 'GOOD'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'ITS'), (tag, code, 3)).commit()
Bib({
'_id': 2
}).set_values(
#('001', None, '2'),
('791', 'a', 'BAD'),
('791', 'b', 1),
('791', 'c', 1),
('930', 'a', 'ITS'),
(tag, 'z', 'WRONG')).commit()
Bib({
'_id': 3
}).set_values(('791', 'a', 'GOOD'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'ITS')).commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][0:3], ['ITS', '2', 'BAD'])
def db():
from dlx import DB
from dlx.marc import Bib, Auth
from dlx.file import S3, File, Identifier
from tempfile import TemporaryFile
DB.connect(
'mongomock://localhost') # ? does mock connection create a fresh db ?
DB.bibs.drop()
DB.auths.drop()
DB.files.drop()
DB.handle['dlx_dl_log'].drop()
Auth().set('100', 'a', 'name_1').commit()
Auth().set('100', 'a', 'name_2').commit()
Bib().set('191', 'a', 'TEST/1').set('245', 'a',
'title_1').set('700', 'a', 1).commit()
Bib().set('245', 'a', 'title_2').set('700', 'a', 2).commit()
S3.connect(access_key='key', access_key_id='key_id', bucket='mock_bucket')
S3.client.create_bucket(Bucket=S3.bucket)
handle = TemporaryFile()
handle.write(b'some data')
handle.seek(0)
File.import_from_handle(handle,
filename='',
identifiers=[Identifier('symbol', 'TEST/1')],
languages=['EN'],
mimetype='text/plain',
source='test')
return DB.client
def test_1_2_13(self):
args = {}
args['authority'] = 1
for params in (['991', 'z', 'I'], ['999', 'c',
't'], ['991', 'f', 'X27']):
report = ReportList.get_by_name(
'bib_missing_subfield_value_{}_{}_{}'.format(*params))
Bib({
'_id': 1
}).set_values(('191', 'a', 'GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
(params)).commit()
params[2] = 'x'
Bib({
'_id': 1
}).set_values(('191', 'a', 'BAD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('991', 'e', 'Participation'), (params)).commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][1], 'BAD')
def test_set():
from dlx.marc import Bib
bib = Bib()
bib.set('245', 'a', 'Edited')
assert bib.get_value('245', 'a') == 'Edited'
bib.set('245', 'a', 'Repeated field', address=['+'])
assert bib.get_values('245', 'a') == ['Edited', 'Repeated field']
bib.set('245', 'a', 'Repeated field edited', address=[1])
assert bib.get_value('245', 'a', address=[1, 0]) == 'Repeated field edited'
bib.set('245', 'a', 'Repeated subfield', address=[1, '+'])
assert bib.get_value('245', 'a', address=[1, 1]) == 'Repeated subfield'
bib.set('651', 'a', 9)
assert bib.get_xref('651', 'a') == 9
bib = Bib().set_values(
('245', 'a', 'yet another'),
('245', 'b', 'title'),
('500', 'a', 'desc'),
('500', 'a', 'desc', {'address': ['+']}),
)
assert bib.get_values('245', 'a', 'b') == ['yet another', 'title']
assert bib.get_values('500', 'a') == ['desc', 'desc']
def test_7a_12a(self):
args = {}
args['authority'] = 1
for t in [('991', 'd')]:
tag, code = t[0], t[1]
report = ReportList.get_by_name('bib_missing_' + tag + code)
Bib({
'_id': 1
}).set_values(('001', None, '2'), ('191', 'a', 'GOOD'),
('191', 'b', 1), ('191', 'c', 1),
('930', 'a', 'UND'), (tag, code, 3)).commit()
Bib({
'_id': 2
}).set_values(('001', None, '2'), ('191', 'a', 'BAD'),
('191', 'b', 1), ('191', 'c', 1),
('930', 'a', 'UND'), (tag, 'z', 'WRONG')).commit()
Bib({
'_id': 3
}).set_values(('001', None, '3'), ('191', 'a', 'GOOD'),
('191', 'b', 1), ('191', 'c', 1),
('930', 'a', 'UND')).commit()
results = report.execute(args)
expected = 2 if tag == '191' else 1
self.assertEqual(len(results), expected)
def test_commit(db, bibs, auths):
from dlx import DB
from dlx.marc import Bib, Auth
from datetime import datetime
from bson import ObjectId
from jsonschema.exceptions import ValidationError
with pytest.raises(Exception):
Bib({'_id': 'I am invalid'}).commit()
for bib in [Bib(x) for x in bibs]:
assert bib.commit().acknowledged
for auth in [Auth(x) for x in auths]:
assert auth.commit().acknowledged
bib = Bib({'_id': 3})
assert bib.commit().acknowledged
assert isinstance(bib.updated, datetime)
assert bib.user == 'admin'
assert bib.history()[0].to_dict() == bib.to_dict()
assert bib.history()[0].user == 'admin'
assert Bib.max_id() == 3
Bib().commit()
Bib().commit()
assert Bib.max_id() == 5
def test_15(self):
report = ReportList.get_by_name('bib_incorrect_793_committees')
Bib({
'_id': 1
}).set_values(('191', 'a', 'A/C.6/GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', '06')).commit()
Bib({
'_id': 2
}).set_values(('191', 'a', 'A/C.2/BAD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', 'WRONG')).commit()
Bib({
'_id': 3
}).set_values(('191', 'a', 'A/C.9/GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', '09')).commit()
args = {}
args['authority'] = 1
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][0], 'A/C.2/BAD')
args['authority'] = 2
results = report.execute(args)
self.assertEqual(len(results), 0)
def test_14b(self):
report = ReportList.get_by_name('bib_incorrect_793_plenary')
Bib({
'_id': 1
}).set_values(('191', 'a', 'A/RES/GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', 'PL')).commit()
Bib({
'_id': 2
}).set_values(('191', 'a', 'A/SESSION_1/L.GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', 'PL')).commit()
Bib({
'_id': 3
}).set_values(('191', 'a', 'A/RES/BAD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', 'WRONG')).commit()
Bib({
'_id': 4
}).set_values(('191', 'a', 'A/SESSION_1/L.BAD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', 'WRONG')).commit()
args = {}
args['authority'] = 1
results = report.execute(args)
self.assertEqual(len(results), 2)
self.assertEqual(results[0][0], 'A/RES/BAD')
self.assertEqual(results[1][0], 'A/SESSION_1/L.BAD')
def test_merge():
from dlx.marc import Bib
bib1 = Bib().set('000', None, 'leader').set('245', 'a', 'Title')
bib2 = Bib().set('000', None, '|eade|').set('269', 'a', 'Date')
bib1.merge(bib2)
assert bib1.get_value('269', 'a') == 'Date'
assert bib1.get_value('000') == 'leader'
def test_21(self):
report = ReportList.get_by_name('speech_incorrect_field_992')
Bib({
'_id': 1
}).set_values(('791', 'a', 'TEST'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'ITS'), ('992', 'a', 'z')).commit()
Bib({
'_id': 2
}).set_values(('191', 'a', 'TEST'), ('992', 'a', 'y')).commit()
results = report.execute({'authority': 1})
self.assertEqual(len(results), 1)
self.assertEqual(results[0][0], 'TEST')
def test_modified_since_log(db, capsys):
from http.server import HTTPServer
from xmldiff.main import diff_texts
from dlx import DB
from dlx.marc import Bib
server = HTTPServer(('127.0.0.1', 9090), None)
responses.add(responses.POST, 'http://127.0.0.1:9090', body='test OK')
dlx_dl.API_URL = 'http://127.0.0.1:9090'
dlx_dl.run(connect=db,
source='test',
type='bib',
modified_within=100,
use_api=True,
api_key='x')
capsys.readouterr().out # clear stdout
Bib().set('999', 'a', 'new').commit()
dlx_dl.run(connect=db,
source='test',
type='bib',
modified_since_log=True,
use_api=True,
api_key='x')
entry = db['dummy']['dlx_dl_log'].find_one({'record_id': 3})
control = '<record><datafield tag="035" ind1=" " ind2=" "><subfield code="a">(DHL)3</subfield></datafield><datafield tag="980" ind1=" " ind2=" "><subfield code="a">BIB</subfield></datafield><datafield tag="999" ind1=" " ind2=" "><subfield code="a">new</subfield></datafield></record>'
assert diff_texts(entry['xml'], control) == []
def test_get_xref(db, bibs):
from dlx.marc import Bib
bib = Bib(bibs[0])
assert bib.get_xref('650', 'a') == 1
bib.set('710', 'a', 3, address='+')
assert bib.get_xrefs('710') == [2,3]
def test_xml_encoding():
from dlx.marc import Bib
from xmldiff import main
control = '<record><datafield ind1=" " ind2=" " tag="245"><subfield code="a">Title with an é</subfield></datafield></record>'
bib = Bib().set('245', 'a', 'Title with an é')
assert main.diff_texts(bib.to_xml(), control) == []
def test_delete(db, capsys):
import json
from http.server import HTTPServer
from xmldiff.main import diff_texts
from dlx.marc import Bib
server = HTTPServer(('127.0.0.1', 9090), None)
responses.add(responses.POST, 'http://127.0.0.1:9090', body='test OK')
dlx_dl.API_URL = 'http://127.0.0.1:9090'
bib = Bib().set('245', 'a', 'Will self destruct')
bib.commit()
bib.delete()
dlx_dl.run(connect=db,
source='test',
type='bib',
modified_within=100,
use_api=True,
api_key='x')
data = list(filter(None, capsys.readouterr().out.split('\n')))
assert len(data) == 3
assert json.loads(data[2])['record_id'] == 3
### end
def test_field_get_value(bibs):
from dlx.marc import Bib
bib = Bib(bibs[0])
field = bib.get_field('245')
assert field.get_value('a') == 'This'
assert field.get_values('a', 'b') == ['This', 'is the']
def test_get_field(bibs):
from dlx.marc import Bib, Field, Controlfield, Datafield
bib = Bib(bibs[0])
assert isinstance(bib.get_field('000'), Controlfield)
assert isinstance(bib.get_field('245'), Datafield)
assert bib.get_field('245').tag == '245'
fields = bib.get_fields('245', '520')
for field in fields:
assert isinstance(field, Field)
bib = Bib()
for tag in ('400', '100', '500', '300', '200'):
bib.set(tag, 'a', 'test')
assert [field.tag for field in bib.get_fields()] == ['100', '200', '300', '400', '500']
def test_23a(self):
report = ReportList.get_by_name('speech_field_mismatch_269_992')
Bib({
'_id': 1
}).set_values(('791', 'a', 'A/x'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'ITS'), ('269', 'a', 'x'),
('992', 'a', 'z')).commit()
results = report.execute({'authority': 1})
self.assertEqual(len(results), 1)
def test_26(self):
report = ReportList.get_by_name('speech_incomplete_authority')
Bib({
'_id': 1
}).set_values(('791', 'b', 1), ('791', 'c', 1), ('930', 'a', 'ITS'),
('700', 'a', 6)).commit()
results = report.execute({'authority': 1})
self.assertEqual(len(results), 1)
self.assertEqual(results[0][1], 'Person, A.')
def test_8a_14a_16(self):
args = {}
args['authority'] = 1
for tag in ('991', '992'):
report = ReportList.get_by_name('bib_missing_' + tag)
Bib({
'_id': 1
}).set_values(('191', 'a', 'GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
(tag, 'a', 3)).commit()
Bib({
'_id': 2
}).set_values(('191', 'a', 'X/SR.1/BAD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND')).commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][2], 'X/SR.1/BAD')
def test_25(self):
report = ReportList.get_by_name('speech_duplicate_record')
for x in [str(x) for x in range(1, 3)]:
Bib({
'_id': x
}).set_values(('791', 'a', 'A/dupe'), ('791', 'b', 1),
('791', 'c', 1), ('930', 'a', 'ITS'),
('700', 'a', 6)).commit()
results = report.execute({'authority': 1})
self.assertEqual(len(results), 1)
def test_7b(self):
report = ReportList.get_by_name('bib_incorrect_subfield_191_9')
for body, code in [('A', 'G'), ('E', 'C'), ('S', 'X')]:
Bib({
'_id': 1
}).set_values(('191', 'a', body + '/GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('191', '9', code),
('930', 'a', 'UND')).commit()
Bib({
'_id': 2
}).set_values(('191', 'a', body + '/BAD'), ('191', 'b', 1),
('191', 'c', 1), ('191', '9', 'z'),
('930', 'a', 'UND')).commit()
args = {}
args['authority'] = 1
results = report.execute(args)
self.assertEqual(len(results), 1)
def test_12c_17b_18b_20b(self):
args = {}
args['authority'] = 1
for tag in ('039', '856', '991', '992'):
report = ReportList.get_by_name('vote_missing_' + tag)
Bib({
'_id': 1
}).set_values(('791', 'a', 'GOOD'), ('791', 'b', 1),
('791', 'c', 1), ('930', 'a', 'VOT'),
(tag, 'a', 9, {
'auth_control': False
})).commit()
Bib({
'_id': 2
}).set_values(('791', 'a', 'BAD'), ('791', 'b', 1),
('791', 'c', 1), ('930', 'a', 'VOT')).commit()
results = report.execute(args)
self.assertEqual(results[0][2], 'BAD')
def test_get_value(bibs):
from dlx.marc import Bib
bib = Bib(bibs[0])
assert bib.get_value('000') == 'leader'
assert bib.get_value('245', 'a') == 'This'
assert bib.get_values('245', 'a', 'b') == ['This', 'is the']
assert bib.get_value('520', 'a', address=[1, 1]) == 'Repeated subfield'
assert bib.get_values('520', 'a', place=1) == ['Another description', 'Repeated subfield']
assert bib.get_value('999', 'a') == ''
assert bib.get_values('999', 'a') == []
assert bib.get_value('245', 'a') == bib.get('245', 'a')
assert bib.get_values('520', 'a') == bib.gets('520', 'a')
def test_init(bibs, auths):
from dlx.marc import BibSet, Bib, AuthSet, Auth
records = [Bib(x) for x in bibs]
bibset = BibSet(records)
assert isinstance(bibset, BibSet)
assert len(bibset.records) == 2
assert bibset.count == 2
records = [Auth(x) for x in auths]
authset = AuthSet(records)
assert isinstance(authset, AuthSet)
assert len(authset.records) == 2
assert authset.count == 2
def test_delete(db):
from dlx import DB
from dlx.marc import Bib
from datetime import datetime
bib = Bib().set('245', 'a', 'This record will self-destruct')
bib.commit()
bib.delete()
assert Bib.match_id(bib.id) == None
history = DB.handle['bib_history'].find_one({'_id': bib.id})
assert history['deleted']['user'] == 'admin'
assert isinstance(history['deleted']['time'], datetime)
def test_12c(self):
args = {}
args['authority'] = 1
tag, code = '991', 'd'
report = ReportList.get_by_name('vote_missing_' + tag + code)
Bib({
'_id': 1
}).set_values(('791', 'a', 'GOOD'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'VOT'), (tag, code, 3)).commit()
Bib({
'_id': 2
}).set_values(('791', 'a', 'BAD'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'VOT'), (tag, 'z', 'WRONG')).commit()
Bib({
'_id': 3
}).set_values(('791', 'a', 'GOOD'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'VOT')).commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
def test_set_008(bibs):
from dlx.marc import Bib
from dlx.config import Config
import time
bib = Bib(bibs[0])
date_tag, date_code = Config.date_field
bib.set(date_tag, date_code, '19991231')
with pytest.raises(Exception):
bib.set('008', None, 'already set')
bib.set_008()
bib.set('008', None, '')
bib.set_008();
assert bib.get_value('008')[0:6] == time.strftime('%y%m%d')
assert bib.get_value('008')[7:11] == '1999'
def test_init_bib(db, bibs):
from dlx.marc import Marc, Bib, Controlfield, Datafield, Subfield, Literal, Linked
bib = Bib(bibs[0])
assert isinstance(bib, Marc)
assert isinstance(bib, Bib)
assert len(bib.controlfields) == 2
assert len(bib.datafields) == 5
assert len(bib.fields) == 7
for field in bib.controlfields:
assert isinstance(field, Controlfield)
for field in bib.datafields:
assert isinstance(field, Datafield)
assert field.record_type == 'bib'
assert field.tag and field.ind1 and field.ind2
for subfield in field.subfields:
assert isinstance(subfield, Subfield)
assert isinstance(subfield, (Literal, Linked))
assert subfield.code and subfield.value
def test_4_5_6(self):
for rtype, code, tag in [('bib', 'UND', '191'),
('speech', 'ITS', '791'),
('vote', 'VOT', '791')]:
report = ReportList.get_by_name('{}_incorrect_session_{}'.format(
rtype, tag))
args = {}
args['authority'] = 1
Bib({
'_id': 1
}).set_values(
(tag, 'a', 'A/SESSION_1/GOOD'),
(tag, 'b', 1),
(tag, 'c', 1),
(tag, 'r', 'ASESSION_1'),
('930', 'a', code),
).commit()
Bib({
'_id': 2
}).set_values(
(tag, 'a', 'A/SESSION_1/BAD'),
(tag, 'b', 1),
(tag, 'c', 1),
(tag, 'r', 'x'),
('930', 'a', code),
).commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][1], 'A/SESSION_1/BAD')
for x in ['C.1', 'RES', 'INF', 'BUR']:
Bib({
'_id': 1
}).set_values((tag, 'a', 'A/{}/SESSION_1/GOOD'.format(x)),
(tag, 'b', 1), (tag, 'c', 1),
(tag, 'r', 'ASESSION_1'), ('930', 'a', code),
('991', 'a', 3)).commit()
Bib({
'_id': 2
}).set_values((tag, 'a', 'A/{}/SESSION_1/BAD'.format(x)),
(tag, 'b', 1), (tag, 'c', 1), (tag, 'r', 'x'),
('930', 'a', code), ('991', 'a', 4)).commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertRegex(results[0][1], r'^A/.+/SESSION_1/BAD')
Bib.match_id(1).set(tag, 'a', 'S/2018/GOOD').set(tag, 'r',
'S73').commit()
Bib.match_id(2).set(tag, 'a', 'S/2018/BAD').set(tag, 'r',
'x').commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][1], 'S/2018/BAD')
Bib.match_id(1).set(tag, 'a', 'S/RES/GOOD(2018)').commit()
Bib.match_id(2).set(tag, 'a', 'S/RES/BAD(2018)').commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][1], 'S/RES/BAD(2018)')
Bib.match_id(1).set(tag, 'a', 'S/PRST/2018/GOOD').commit()
Bib.match_id(2).set(tag, 'a', 'S/PRST/2018/BAD').commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][1], 'S/PRST/2018/BAD')
Bib.match_id(1).set(tag, 'a',
'E/RES/SESSION_1').set(tag, 'r',
'ESESSION_1').commit()
Bib.match_id(2).set(tag, 'a',
'E/RES/SESSION_1/BAD').set(tag, 'r',
'x').commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][1], 'E/RES/SESSION_1/BAD')
def test_language(db):
from dlx.marc import Bib, Auth
Auth({'_id': 3}).set('150', 'a', 'Text').set('994', 'a', 'Texto').commit()
bib = Bib({'_id': 3}).set('650', 'a', 3)
assert bib.get('650', 'a', language='es') == 'Texto'