def test_12b(self):
args = {}
args['authority'] = 1
tag, code = '991', 'd'
report = ReportList.get_by_name('speech_missing_' + tag + code)
Bib({
'_id': 1
}).set_values(('791', 'a', 'GOOD'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'ITS'), (tag, code, 3)).commit()
Bib({
'_id': 2
}).set_values(
#('001', None, '2'),
('791', 'a', 'BAD'),
('791', 'b', 1),
('791', 'c', 1),
('930', 'a', 'ITS'),
(tag, 'z', 'WRONG')).commit()
Bib({
'_id': 3
}).set_values(('791', 'a', 'GOOD'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'ITS')).commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][0:3], ['ITS', '2', 'BAD'])
def db():
from dlx import DB
from dlx.marc import Bib, Auth
from dlx.file import S3, File, Identifier
from tempfile import TemporaryFile
DB.connect(
'mongomock://localhost') # ? does mock connection create a fresh db ?
DB.bibs.drop()
DB.auths.drop()
DB.files.drop()
DB.handle['dlx_dl_log'].drop()
Auth().set('100', 'a', 'name_1').commit()
Auth().set('100', 'a', 'name_2').commit()
Bib().set('191', 'a', 'TEST/1').set('245', 'a',
'title_1').set('700', 'a', 1).commit()
Bib().set('245', 'a', 'title_2').set('700', 'a', 2).commit()
S3.connect(access_key='key', access_key_id='key_id', bucket='mock_bucket')
S3.client.create_bucket(Bucket=S3.bucket)
handle = TemporaryFile()
handle.write(b'some data')
handle.seek(0)
File.import_from_handle(handle,
filename='',
identifiers=[Identifier('symbol', 'TEST/1')],
languages=['EN'],
mimetype='text/plain',
source='test')
return DB.client
def test_find(db):
from dlx.marc import Bib, Auth
bibs = Bib.find({})
assert inspect.isgenerator(bibs)
assert len(list(bibs)) == 2
for bib in Bib.find({}):
assert isinstance(bib, Bib)
auths = Auth.find({})
assert inspect.isgenerator(auths)
assert len(list(auths)) == 2
for auth in Auth.find({}):
assert isinstance(auth, Auth)
auths = Auth.find({'_id': 1})
assert len(list(auths)) == 1
auths = Auth.find({}, limit=1)
assert len(list(auths)) == 1
auths = Auth.find({}, limit=0, skip=1, projection={})
assert len(list(auths)) == 1
def test_field_get_value(bibs):
from dlx.marc import Bib
bib = Bib(bibs[0])
field = bib.get_field('245')
assert field.get_value('a') == 'This'
assert field.get_values('a', 'b') == ['This', 'is the']
def test_1_2_13(self):
args = {}
args['authority'] = 1
for params in (['991', 'z', 'I'], ['999', 'c',
't'], ['991', 'f', 'X27']):
report = ReportList.get_by_name(
'bib_missing_subfield_value_{}_{}_{}'.format(*params))
Bib({
'_id': 1
}).set_values(('191', 'a', 'GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
(params)).commit()
params[2] = 'x'
Bib({
'_id': 1
}).set_values(('191', 'a', 'BAD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('991', 'e', 'Participation'), (params)).commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][1], 'BAD')
def test_7a_12a(self):
args = {}
args['authority'] = 1
for t in [('991', 'd')]:
tag, code = t[0], t[1]
report = ReportList.get_by_name('bib_missing_' + tag + code)
Bib({
'_id': 1
}).set_values(('001', None, '2'), ('191', 'a', 'GOOD'),
('191', 'b', 1), ('191', 'c', 1),
('930', 'a', 'UND'), (tag, code, 3)).commit()
Bib({
'_id': 2
}).set_values(('001', None, '2'), ('191', 'a', 'BAD'),
('191', 'b', 1), ('191', 'c', 1),
('930', 'a', 'UND'), (tag, 'z', 'WRONG')).commit()
Bib({
'_id': 3
}).set_values(('001', None, '3'), ('191', 'a', 'GOOD'),
('191', 'b', 1), ('191', 'c', 1),
('930', 'a', 'UND')).commit()
results = report.execute(args)
expected = 2 if tag == '191' else 1
self.assertEqual(len(results), expected)
def test_xml_encoding():
from dlx.marc import Bib
from xmldiff import main
control = '<record><datafield ind1=" " ind2=" " tag="245"><subfield code="a">Title with an é</subfield></datafield></record>'
bib = Bib().set('245', 'a', 'Title with an é')
assert main.diff_texts(bib.to_xml(), control) == []
def test_15(self):
report = ReportList.get_by_name('bib_incorrect_793_committees')
Bib({
'_id': 1
}).set_values(('191', 'a', 'A/C.6/GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', '06')).commit()
Bib({
'_id': 2
}).set_values(('191', 'a', 'A/C.2/BAD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', 'WRONG')).commit()
Bib({
'_id': 3
}).set_values(('191', 'a', 'A/C.9/GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', '09')).commit()
args = {}
args['authority'] = 1
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][0], 'A/C.2/BAD')
args['authority'] = 2
results = report.execute(args)
self.assertEqual(len(results), 0)
def test_14b(self):
report = ReportList.get_by_name('bib_incorrect_793_plenary')
Bib({
'_id': 1
}).set_values(('191', 'a', 'A/RES/GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', 'PL')).commit()
Bib({
'_id': 2
}).set_values(('191', 'a', 'A/SESSION_1/L.GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', 'PL')).commit()
Bib({
'_id': 3
}).set_values(('191', 'a', 'A/RES/BAD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', 'WRONG')).commit()
Bib({
'_id': 4
}).set_values(('191', 'a', 'A/SESSION_1/L.BAD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
('793', 'a', 'WRONG')).commit()
args = {}
args['authority'] = 1
results = report.execute(args)
self.assertEqual(len(results), 2)
self.assertEqual(results[0][0], 'A/RES/BAD')
self.assertEqual(results[1][0], 'A/SESSION_1/L.BAD')
def test_querystring(db):
from dlx.marc import Bib, Auth, QueryDocument
query = QueryDocument.from_string('{"245": {"a": "/^(This|Another)/", "b": "is the", "c": "title"}}')
assert len(list(Bib.find(query.compile()))) == 2
query = QueryDocument.from_string('{"OR": {"650": 0, "710": 0}}')
assert len(list(Bib.find(query.compile()))) == 1
query = QueryDocument.from_string('{"110": {"a": "Another header"}}')
assert Auth.find_one(query.compile()).id == 2
def test_21(self):
report = ReportList.get_by_name('speech_incorrect_field_992')
Bib({
'_id': 1
}).set_values(('791', 'a', 'TEST'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'ITS'), ('992', 'a', 'z')).commit()
Bib({
'_id': 2
}).set_values(('191', 'a', 'TEST'), ('992', 'a', 'y')).commit()
results = report.execute({'authority': 1})
self.assertEqual(len(results), 1)
self.assertEqual(results[0][0], 'TEST')
def test_to_mrk(bibs):
from dlx.marc import Bib
control = '000 leader\n008 controlfield\n245 \\\\$aThis$bis the$ctitle\n520 \\\\$aDescription\n520 \\\\$aAnother description$aRepeated subfield\n650 \\\\$aHeader\n710 \\\\$aAnother header\n'
bib = Bib.find_one({'_id': 1})
assert bib.to_mrk() == control
def test_querydocument(db):
from dlx.marc import Bib, Auth, QueryDocument, Condition, Or
from bson import SON
from json import loads
import re
query = QueryDocument(Condition(tag='245', subfields={'a': 'This'}))
assert isinstance(query.compile(), SON)
qjson = query.to_json()
qdict = loads(qjson)
assert qdict['245']['$elemMatch']['subfields']['$elemMatch']['code'] == 'a'
assert qdict['245']['$elemMatch']['subfields']['$elemMatch']['value'] == 'This'
query = QueryDocument(
Condition(tag='245', subfields={'a': re.compile(r'(This|Another)'), 'b': 'is the', 'c': 'title'}),
Condition(tag='650', modifier='exists'),
Or(
Condition(tag='710', modifier='exists'),
Condition(tag='520', modifier='not_exists')
)
)
assert len(list(Bib.find(query.compile()))) == 2
query = QueryDocument(
Condition(tag='110', subfields={'a': 'Another header'}),
)
assert len(list(Auth.find(query.compile()))) == 1
assert Auth.find_one(query.compile()).id == 2
def test_to_xml(db):
from dlx.marc import Bib
from xmldiff import main
control = '<record><controlfield tag="000">leader</controlfield><controlfield tag="008">controlfield</controlfield><datafield ind1=" " ind2=" " tag="245"><subfield code="a">This</subfield><subfield code="b">is the</subfield><subfield code="c">title</subfield></datafield><datafield ind1=" " ind2=" " tag="520"><subfield code="a">Description</subfield></datafield><datafield ind1=" " ind2=" " tag="520"><subfield code="a">Another description</subfield><subfield code="a">Repeated subfield</subfield></datafield><datafield ind1=" " ind2=" " tag="650"><subfield code="a">Header</subfield><subfield code="0">1</subfield></datafield><datafield ind1=" " ind2=" " tag="710"><subfield code="a">Another header</subfield><subfield code="0">2</subfield></datafield></record>'
bib = Bib.find_one({'_id': 1})
assert main.diff_texts(bib.to_xml(), control) == []
def test_modified_since_log(db, capsys):
from http.server import HTTPServer
from xmldiff.main import diff_texts
from dlx import DB
from dlx.marc import Bib
server = HTTPServer(('127.0.0.1', 9090), None)
responses.add(responses.POST, 'http://127.0.0.1:9090', body='test OK')
dlx_dl.API_URL = 'http://127.0.0.1:9090'
dlx_dl.run(connect=db,
source='test',
type='bib',
modified_within=100,
use_api=True,
api_key='x')
capsys.readouterr().out # clear stdout
Bib().set('999', 'a', 'new').commit()
dlx_dl.run(connect=db,
source='test',
type='bib',
modified_since_log=True,
use_api=True,
api_key='x')
entry = db['dummy']['dlx_dl_log'].find_one({'record_id': 3})
control = '<record><datafield tag="035" ind1=" " ind2=" "><subfield code="a">(DHL)3</subfield></datafield><datafield tag="980" ind1=" " ind2=" "><subfield code="a">BIB</subfield></datafield><datafield tag="999" ind1=" " ind2=" "><subfield code="a">new</subfield></datafield></record>'
assert diff_texts(entry['xml'], control) == []
def test_get_field(bibs):
from dlx.marc import Bib, Field, Controlfield, Datafield
bib = Bib(bibs[0])
assert isinstance(bib.get_field('000'), Controlfield)
assert isinstance(bib.get_field('245'), Datafield)
assert bib.get_field('245').tag == '245'
fields = bib.get_fields('245', '520')
for field in fields:
assert isinstance(field, Field)
bib = Bib()
for tag in ('400', '100', '500', '300', '200'):
bib.set(tag, 'a', 'test')
assert [field.tag for field in bib.get_fields()] == ['100', '200', '300', '400', '500']
def test_merge():
from dlx.marc import Bib
bib1 = Bib().set('000', None, 'leader').set('245', 'a', 'Title')
bib2 = Bib().set('000', None, '|eade|').set('269', 'a', 'Date')
bib1.merge(bib2)
assert bib1.get_value('269', 'a') == 'Date'
assert bib1.get_value('000') == 'leader'
def test_auth_lookup(db):
from dlx.marc import Bib, Auth
bib = Bib.find_one({'_id': 1})
assert bib.get_xref('650', 'a') == 1
assert bib.get_value('650', 'a') == 'Header'
auth = Auth.find_one({'_id': 1})
auth.set('150', 'a', 'Changed').commit()
assert bib.get_value('650', 'a') == 'Changed'
def test_find_one(db, bibs, auths):
from dlx.marc import Bib, Auth
bib = Bib.find_one({'_id': 1})
assert bib.id == 1
assert isinstance(bib, Bib)
auth = Auth.find_one({'_id': 1})
assert auth.id == 1
assert isinstance(auth, Auth)
def test_23a(self):
report = ReportList.get_by_name('speech_field_mismatch_269_992')
Bib({
'_id': 1
}).set_values(('791', 'a', 'A/x'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'ITS'), ('269', 'a', 'x'),
('992', 'a', 'z')).commit()
results = report.execute({'authority': 1})
self.assertEqual(len(results), 1)
def test_26(self):
report = ReportList.get_by_name('speech_incomplete_authority')
Bib({
'_id': 1
}).set_values(('791', 'b', 1), ('791', 'c', 1), ('930', 'a', 'ITS'),
('700', 'a', 6)).commit()
results = report.execute({'authority': 1})
self.assertEqual(len(results), 1)
self.assertEqual(results[0][1], 'Person, A.')
def test_25(self):
report = ReportList.get_by_name('speech_duplicate_record')
for x in [str(x) for x in range(1, 3)]:
Bib({
'_id': x
}).set_values(('791', 'a', 'A/dupe'), ('791', 'b', 1),
('791', 'c', 1), ('930', 'a', 'ITS'),
('700', 'a', 6)).commit()
results = report.execute({'authority': 1})
self.assertEqual(len(results), 1)
def test_8a_14a_16(self):
args = {}
args['authority'] = 1
for tag in ('991', '992'):
report = ReportList.get_by_name('bib_missing_' + tag)
Bib({
'_id': 1
}).set_values(('191', 'a', 'GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND'),
(tag, 'a', 3)).commit()
Bib({
'_id': 2
}).set_values(('191', 'a', 'X/SR.1/BAD'), ('191', 'b', 1),
('191', 'c', 1), ('930', 'a', 'UND')).commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
self.assertEqual(results[0][2], 'X/SR.1/BAD')
def test_delete_field(bibs):
from dlx.marc import Bib
bib = Bib.find_one({'_id': 1})
bib.delete_field('008')
assert list(bib.get_fields('008')) == []
bib.delete_field('500')
assert list(bib.get_fields('500')) == []
bib.delete_field('520', place=1)
assert len(list(bib.get_fields('520'))) == 1
assert bib.get_values('520', 'a') == ['Description']
def test_get_xref(db, bibs):
from dlx.marc import Bib
bib = Bib(bibs[0])
assert bib.get_xref('650', 'a') == 1
bib.set('710', 'a', 3, address='+')
assert bib.get_xrefs('710') == [2,3]
def test_delete(db, capsys):
import json
from http.server import HTTPServer
from xmldiff.main import diff_texts
from dlx.marc import Bib
server = HTTPServer(('127.0.0.1', 9090), None)
responses.add(responses.POST, 'http://127.0.0.1:9090', body='test OK')
dlx_dl.API_URL = 'http://127.0.0.1:9090'
bib = Bib().set('245', 'a', 'Will self destruct')
bib.commit()
bib.delete()
dlx_dl.run(connect=db,
source='test',
type='bib',
modified_within=100,
use_api=True,
api_key='x')
data = list(filter(None, capsys.readouterr().out.split('\n')))
assert len(data) == 3
assert json.loads(data[2])['record_id'] == 3
### end
def test_7b(self):
report = ReportList.get_by_name('bib_incorrect_subfield_191_9')
for body, code in [('A', 'G'), ('E', 'C'), ('S', 'X')]:
Bib({
'_id': 1
}).set_values(('191', 'a', body + '/GOOD'), ('191', 'b', 1),
('191', 'c', 1), ('191', '9', code),
('930', 'a', 'UND')).commit()
Bib({
'_id': 2
}).set_values(('191', 'a', body + '/BAD'), ('191', 'b', 1),
('191', 'c', 1), ('191', '9', 'z'),
('930', 'a', 'UND')).commit()
args = {}
args['authority'] = 1
results = report.execute(args)
self.assertEqual(len(results), 1)
def test_12c_17b_18b_20b(self):
args = {}
args['authority'] = 1
for tag in ('039', '856', '991', '992'):
report = ReportList.get_by_name('vote_missing_' + tag)
Bib({
'_id': 1
}).set_values(('791', 'a', 'GOOD'), ('791', 'b', 1),
('791', 'c', 1), ('930', 'a', 'VOT'),
(tag, 'a', 9, {
'auth_control': False
})).commit()
Bib({
'_id': 2
}).set_values(('791', 'a', 'BAD'), ('791', 'b', 1),
('791', 'c', 1), ('930', 'a', 'VOT')).commit()
results = report.execute(args)
self.assertEqual(results[0][2], 'BAD')
def test_12c(self):
args = {}
args['authority'] = 1
tag, code = '991', 'd'
report = ReportList.get_by_name('vote_missing_' + tag + code)
Bib({
'_id': 1
}).set_values(('791', 'a', 'GOOD'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'VOT'), (tag, code, 3)).commit()
Bib({
'_id': 2
}).set_values(('791', 'a', 'BAD'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'VOT'), (tag, 'z', 'WRONG')).commit()
Bib({
'_id': 3
}).set_values(('791', 'a', 'GOOD'), ('791', 'b', 1), ('791', 'c', 1),
('930', 'a', 'VOT')).commit()
results = report.execute(args)
self.assertEqual(len(results), 1)
def test_init(bibs, auths):
from dlx.marc import BibSet, Bib, AuthSet, Auth
records = [Bib(x) for x in bibs]
bibset = BibSet(records)
assert isinstance(bibset, BibSet)
assert len(bibset.records) == 2
assert bibset.count == 2
records = [Auth(x) for x in auths]
authset = AuthSet(records)
assert isinstance(authset, AuthSet)
assert len(authset.records) == 2
assert authset.count == 2