def setUp(self):
self.v_id = str(uuid.uuid4())
self.v_url = "http://localhost:8888/misirlou/tests/fixtures/manifest.json"
with open("misirlou/tests/fixtures/manifest.json") as f:
self.w_valid = WIPManifest(self.v_url,
self.v_id,
prefetched_data=f.read())
def test_file_retrieval(self):
"""Test full creation from local manifest doc."""
w = WIPManifest(
"http://localhost:8888/misirlou/tests/fixtures/manifest.json",
str(uuid.uuid4()))
w.create()
self.assertDictEqual(self.w_valid.json, w.json)
def test_suggestions(self):
# Create and index a manifest.
v_id = str(uuid.uuid4())
v_url = "http://localhost:8888/misirlou/tests/fixtures/manifest.json"
with open("misirlou/tests/fixtures/manifest.json") as f:
w_valid = WIPManifest(v_url, v_id, prefetched_data=f.read())
w_valid.create()
self.solr_con.commit()
# Some queries with known responses given our test manifest.
resp = self.client.get("/suggest/?q=serm")
expected = {'suggestions': ['sermons', 'sermon']}
self.assertDictEqual(resp.data, expected)
resp = self.client.get("/suggest/?q=des")
expected = {'suggestions': ['destiné']}
self.assertDictEqual(resp.data, expected)
# A query which should have no suggestions, as it is a complete word.
resp = self.client.get("/suggest/?q=dimanches")
expected = {'suggestions': []}
self.assertDictEqual(resp.data, expected)
# Forgetting the query parameter is handled silently.
resp = self.client.get("/suggest/")
expected = {'suggestions': []}
self.assertDictEqual(resp.data, expected)
def test_commit_solr(self):
rem_url = "http://localhost:8888/misirlou/tests/fixtures/manifest.json"
wman = WIPManifest(rem_url, str(uuid.uuid4()))
wman.create()
commit_solr()
res = self.client.get("/?q=Maria&format=json")
search_data = res.data['search']
self.assertEqual(search_data['num_found'], 1)
def test_commit_solr(self):
rem_url = "http://localhost:8888/misirlou/tests/fixtures/manifest.json"
wman = WIPManifest(rem_url, str(uuid.uuid4()))
wman.create()
commit_solr()
res = self.client.get("/?q=Maria&format=json")
search_data = res.data['search']
self.assertEqual(search_data['num_found'], 1)
def test_get(self):
# Create and index a manifest.
v_id = str(uuid.uuid4())
v_url = "http://localhost:8888/misirlou/tests/fixtures/manifest.json"
with open("misirlou/tests/fixtures/manifest.json") as f:
w_valid = WIPManifest(v_url, v_id, prefetched_data=f.read())
w_valid.create()
self.solr_con.commit()
with open('misirlou/tests/fixtures/recent_manifests.json') as f:
expected = json.load(f)
actual = self.client.get('/manifests/recent/').json()
for result in itertools.chain(expected['results'], actual['results']):
del result['local_id']
self.assertDictEqual(actual, expected)
def test_get(self):
# Create and index a manifest.
v_id = str(uuid.uuid4())
v_url = "http://localhost:8888/misirlou/tests/fixtures/manifest.json"
with open("misirlou/tests/fixtures/manifest.json") as f:
w_valid = WIPManifest(v_url, v_id, prefetched_data=f.read())
w_valid.create()
self.solr_con.commit()
with open('misirlou/tests/fixtures/recent_manifests.json') as f:
expected = json.load(f)
actual = self.client.get('/manifests/recent/').json()
for result in itertools.chain(expected['results'], actual['results']):
del result['local_id']
self.assertDictEqual(actual, expected)
def test_is_duplicates(self):
"""Test the duplicate finder is working.
Ensure that, while creating manifest X, if manifest Y exists
already and is exactly the same as X, then X will get the same id
as Y (leading to an over-write, rather than a duplication).
"""
temp_id = str(uuid.uuid4())
WIPManifest(remote_url=self.v_url, shared_id=temp_id).create()
self.w_valid._remove_db_duplicates()
# found the duplicate in the database.
self.assertEqual(self.w_valid.id, temp_id)
self.w_valid.id = self.v_id
def setUp(self):
self.v_id = str(uuid.uuid4())
self.v_url = "http://localhost:8888/misirlou/tests/fixtures/manifest.json"
with open("misirlou/tests/fixtures/manifest.json") as f:
self.w_valid = WIPManifest(self.v_url, self.v_id, prefetched_data=f.read())
class WIPManifestTestCase(MisirlouTestSetup):
def setUp(self):
self.v_id = str(uuid.uuid4())
self.v_url = "http://*****:*****@language': "en", "@value": "one"},
{'@language': "fr", "@value": "deux"}],
'multiple-unknown-no-en': [{'@language': "de", "@value": "eins"},
{'@language': "it", "@value": "dos"}],
}
self.w_valid.doc = {'metadata': []}
for k, v in metadata.items():
self.w_valid._add_metadata(k, v)
# Needed as hashed keys may appear in any order.
self.w_valid.doc['metadata'] = sorted(self.w_valid.doc['metadata'])
correct = {'metadata': ['one', 'one two'],
'metadata_txt_de': ['eins'],
'metadata_txt_fr': ['deux'],
'metadata_txt_en': ['one'],
'metadata_txt_it': ['dos']}
self.assertDictEqual(self.w_valid.doc, correct)
def test_add_metadata_known_keys(self):
"""Test parsing metadata with known keys.
When keys are known, the parser should add them as independent keys
in the document, preserve language data, and set defaults when
English values are found.
"""
metadata = {
'title': "The title",
'author': [{'@language': "en", "@value": "The Author"},
{'@language': "fr", "@value": "Le Author"}],
'location': [{'@language': 'fr', "@value": "Le Monde"}],
'period': "Around 14th Century"
}
self.w_valid.doc = {'metadata': []}
for k, v in metadata.items():
self.w_valid._add_metadata(k, v)
correct = {'title': 'The title',
'author': 'The Author',
'author_txt_fr': 'Le Author',
'location_txt_fr': 'Le Monde',
'date': 'Around 14th Century',
'metadata': []}
self.assertDictEqual(self.w_valid.doc, correct)
def test_label_normalizer(self):
"""Test behaviour of label normalizer (choosing a label).
Label normalizer will compare labels against the map defined in
setings.py. The goal is to match similar words to one indexed field
(e.g. 'period' -> 'date', 'Title(s)' -> 'title'"
It should follow this priority in choosing a label
1) An english label that can be normalized
2) Any label that can be normalized
3) Return None (as signifier of no good choices)
"""
# An english, normalizable label gets priority.
eng_priority = [{"@language": "fr", "@value": "date"},
{"@language": "en", "@value": "title(s)"}]
l = self.w_valid._meta_label_normalizer(eng_priority)
self.assertEqual(l, "title")
# A normalizable label get's priority over an unknown english label.
norm_priority = [{"@language": "fr", "@value": "publication date"},
{"@language": "en", "@value": "Unknown"}]
l = self.w_valid._meta_label_normalizer(norm_priority)
self.assertEqual(l, 'date')
# With no normalization possible, return None.
no_eng = [{"@language": "fr", "@value": "je ne sais pas"},
{"@language": "it", "@value": "unknown"}]
l = self.w_valid._meta_label_normalizer(no_eng)
self.assertEqual(l, None)
# If label is a normalizable string, get it's normalization
str_test = "publication date"
l = self.w_valid._meta_label_normalizer(str_test)
self.assertEqual(l, "date")
# If label is unknown string, return nothing.
str_test = "Unknown"
l = self.w_valid._meta_label_normalizer(str_test)
self.assertEqual(l, None)
# Lack of a label returns None
l = self.w_valid._meta_label_normalizer(None)
self.assertEqual(l, None)
def test_file_retrieval(self):
"""Test full creation from local manifest doc."""
w = WIPManifest("http://localhost:8888/misirlou/tests/fixtures/manifest.json",
str(uuid.uuid4()))
w.create()
self.assertDictEqual(self.w_valid.json, w.json)
def test_file_retrieval(self):
"""Test full creation from local manifest doc."""
w = WIPManifest("http://localhost:8888/misirlou/tests/fixtures/manifest.json",
str(uuid.uuid4()))
w.create()
self.assertDictEqual(self.w_valid.json, w.json)
class WIPManifestTestCase(MisirlouTestSetup):
def setUp(self):
self.v_id = str(uuid.uuid4())
self.v_url = "http://*****:*****@language': "en",
"@value": "one"
}, {
'@language': "fr",
"@value": "deux"
}],
'multiple-unknown-no-en': [{
'@language': "de",
"@value": "eins"
}, {
'@language': "it",
"@value": "dos"
}],
}
self.w_valid.doc = {'metadata': []}
for k, v in metadata.items():
self.w_valid._add_metadata(k, v)
# Needed as hashed keys may appear in any order.
self.w_valid.doc['metadata'] = sorted(self.w_valid.doc['metadata'])
correct = {
'metadata': ['one', 'one two'],
'metadata_txt_de': ['eins'],
'metadata_txt_fr': ['deux'],
'metadata_txt_en': ['one'],
'metadata_txt_it': ['dos']
}
self.assertDictEqual(self.w_valid.doc, correct)
def test_add_metadata_known_keys(self):
"""Test parsing metadata with known keys.
When keys are known, the parser should add them as independent keys
in the document, preserve language data, and set defaults when
English values are found.
"""
metadata = {
'title':
"The title",
'author': [{
'@language': "en",
"@value": "The Author"
}, {
'@language': "fr",
"@value": "Le Author"
}],
'location': [{
'@language': 'fr',
"@value": "Le Monde"
}],
'period':
"Around 14th Century"
}
self.w_valid.doc = {'metadata': []}
for k, v in metadata.items():
self.w_valid._add_metadata(k, v)
correct = {
'title': 'The title',
'author': 'The Author',
'author_txt_fr': 'Le Author',
'location_txt_fr': 'Le Monde',
'date': 'Around 14th Century',
'metadata': []
}
self.assertDictEqual(self.w_valid.doc, correct)
def test_label_normalizer(self):
"""Test behaviour of label normalizer (choosing a label).
Label normalizer will compare labels against the map defined in
setings.py. The goal is to match similar words to one indexed field
(e.g. 'period' -> 'date', 'Title(s)' -> 'title'"
It should follow this priority in choosing a label
1) An english label that can be normalized
2) Any label that can be normalized
3) Return None (as signifier of no good choices)
"""
# An english, normalizable label gets priority.
eng_priority = [{
"@language": "fr",
"@value": "date"
}, {
"@language": "en",
"@value": "title(s)"
}]
l = self.w_valid._meta_label_normalizer(eng_priority)
self.assertEqual(l, "title")
# A normalizable label get's priority over an unknown english label.
norm_priority = [{
"@language": "fr",
"@value": "publication date"
}, {
"@language": "en",
"@value": "Unknown"
}]
l = self.w_valid._meta_label_normalizer(norm_priority)
self.assertEqual(l, 'date')
# With no normalization possible, return None.
no_eng = [{
"@language": "fr",
"@value": "je ne sais pas"
}, {
"@language": "it",
"@value": "unknown"
}]
l = self.w_valid._meta_label_normalizer(no_eng)
self.assertEqual(l, None)
# If label is a normalizable string, get it's normalization
str_test = "publication date"
l = self.w_valid._meta_label_normalizer(str_test)
self.assertEqual(l, "date")
# If label is unknown string, return nothing.
str_test = "Unknown"
l = self.w_valid._meta_label_normalizer(str_test)
self.assertEqual(l, None)
# Lack of a label returns None
l = self.w_valid._meta_label_normalizer(None)
self.assertEqual(l, None)
def test_file_retrieval(self):
"""Test full creation from local manifest doc."""
w = WIPManifest(
"http://localhost:8888/misirlou/tests/fixtures/manifest.json",
str(uuid.uuid4()))
w.create()
self.assertDictEqual(self.w_valid.json, w.json)
def re_index(self, **kwargs):
from misirlou.helpers.IIIFImporter import WIPManifest
text_id = str(self.id)
wip = WIPManifest(self.remote_url, text_id)
wip.create()
def test_results_search(self):
# Create and index a manifest.
v_id = str(uuid.uuid4())
v_url = "http://*****:*****@id':
'http://testserver/?q=Maria&format=json',
'results': [{
'label':
['Luzern, Zentral- und Hochschulbibliothek, KB 35 4°'],
'logo':
None,
'description': [
'In addition to sermons and sermon-related material pertaining to Sundays, saints’ days and feast-days dedicated to Mary, the manuscript contains part of S. Bonaventure’s (1221-1274) commentary on the four books of the Sentences of Peter Lombard, and the treatise De arca Noe by Marquard of Lindau (d. 1392). '
],
'thumbnail': {
'height':
6496,
'format':
'image/jpeg',
'@type':
'dctypes:Image',
'width':
4872,
'service': {
'profile':
'http://library.stanford.edu/iiif/image-api/compliance.html#level1',
'@context':
'http://iiif.io/api/image/2/context.json',
'@id':
'http://www.e-codices.unifr.ch/loris/zhl/zhl-0035-4/zhl-0035-4_e001.jp2'
},
'@id':
'http://www.e-codices.unifr.ch/loris/zhl/zhl-0035-4/zhl-0035-4_e001.jp2/full/full/0/default.jpg'
},
'local_id':
'183072c6-2eba-4faf-af1c-87f77b7aa229',
'hits': [{
'field':
'description_txt_it',
'parsed': [
' dedicate a ', 'Maria',
', il manoscritto contiene parti del commento di s. Bonaventura (1221-1274) al libro'
]
}],
'@id':
'http://localhost:8888/misirlou/tests/fixtures/manifest.json',
'attribution':
['e-codices - Virtual Manuscript Library of Switzerland']
}],
'num_found':
1
},
'routes': {
'manifests': 'http://testserver/manifests/?format=json'
}
}
self.solr_con.delete_all()
self.solr_con.commit()
def re_index(self, **kwargs):
from misirlou.helpers.IIIFImporter import WIPManifest
text_id = str(self.id)
wip = WIPManifest(self.remote_url, text_id)
wip.create()