def test_remote():
"""Test a list or remote Manifest URLs."""
urls = [
#"http://dms-data.stanford.edu/data/manifests/Walters/qm670kv1873/manifest.json",
#"http://manifests.ydc2.yale.edu/manifest/Admont23.json",
#"http://oculus-dev.lib.harvard.edu/manifests/drs:5981093",
#"http://iiif-dev.bodleian.ox.ac.uk/metadata/bib_germ_1485_d1/bib_germ_1485_d1.json",
#"http://iiif-dev.bodleian.ox.ac.uk/metadata/ms_auct_t_inf_2_1/ms_auct_t_inf_2_1.json",
#"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b84473026/manifest.json",
#"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b84473026/manifest-ranges.json",
#"http://demos.biblissima-condorcet.fr/mirador/data/add_ms_10289_edited_8v-9r.json",
#"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b8438674r/manifest.json",
"http://demos.biblissima-condorcet.fr/iiif/metadata/BVH/B410186201_LI03/manifest.json",
"http://sanddragon.bl.uk/IIIFMetadataService/add_ms_10289.json",
"http://sr-svx-93.unifr.ch/metadata/iiif/bbb-0218/manifest.json",
#"http://www.shared-canvas.org/impl/demo1d/res/manifest.json"
]
for u in urls:
fh = urllib.urlopen(u)
data = fh.read()
fh.close()
try:
print("------")
print(u)
reader = ManifestReader(data)
nmfst = reader.read()
js = nmfst.toJSON()
except Exception as e:
print(" => %s: %s" % (e.__class__.__name__, e))
def check_manifest(self, data, version, url=None, warnings=[]):
"""Check manifest data at version, return JSON."""
infojson = {}
# Check if 3.0 if so run through schema rather than this version...
if version == '3.0':
infojson = schemavalidator.validate(data, version, url)
else:
reader = ManifestReader(data, version=version)
err = None
try:
mf = reader.read()
mf.toJSON()
# Passed!
okay = 1
except Exception as e:
# Failed
err = e
okay = 0
warnings.extend(reader.get_warnings())
infojson = {
'received': data,
'okay': okay,
'warnings': warnings,
'error': str(err),
'url': url
}
return self.return_json(infojson)
def test07_every_value_its_list(self):
fh = open('tests/everything_a_list_fixture.json')
data = fh.read()
fh.close()
js = json.loads(data)
mr = ManifestReader(js)
doc = mr.read()
def do_test(data, excexp):
debug_tests = 0
reader = ManifestReader(data)
if excexp == None:
# Should pass
try:
what = reader.read()
js = what.toJSON()
print_warnings(reader)
return 1
except Exception as e:
print_warnings(reader)
if debug_tests:
print("%s: %s" % (e.__class__.__name__, e))
return 0
else:
# Should raise exception excexp
try:
what = reader.read()
js = what.toJSON()
if debug_tests:
print("Loaded okay, should have failed")
print(json.dumps(js, sort_keys=True, indent=2))
print_warnings(reader)
return "no exception"
except excexp as e:
# Got expected exception
return None
except Exception as e:
# Not the exception we expected
return "bad exception, got %s (%s)" % (e.__class__,str(e))
def do_test(data, excexp):
debug_tests = 0
reader = ManifestReader(data)
if excexp == None:
# Should pass
try:
what = reader.read()
js = what.toJSON()
print_warnings(reader)
return 1
except Exception as e:
print_warnings(reader)
if debug_tests:
print("%s: %s" % (e.__class__.__name__, e))
return 0
else:
# Should raise exception excexp
try:
what = reader.read()
js = what.toJSON()
if debug_tests:
print("Loaded okay, should have failed")
print(json.dumps(js, sort_keys=True, indent=2))
print_warnings(reader)
return "no exception"
except excexp as e:
# Got expected exception
return None
except Exception as e:
# Not the exception we expected
return "bad exception, got %s (%s)" % (e.__class__, str(e))
def test07_label_value_language(self):
fh = open('tests/label_value_language_fixture.json')
data = fh.read()
fh.close()
js = json.loads(data)
mr = ManifestReader(js)
doc = mr.read()
def test06_range_range(self):
fh = open('tests/range_range_fixture.json')
data = fh.read()
fh.close()
js = json.loads(data)
mr = ManifestReader(js)
doc = mr.read()
def test05_multipleContexts(self):
fh = open('tests/multiple_contexts_fixture.json')
data = fh.read()
fh.close()
js = json.loads(data)
mr = ManifestReader(js)
doc = mr.read()
def test_remote():
"""Test a list or remote Manifest URLs."""
urls = [
#"http://dms-data.stanford.edu/data/manifests/Walters/qm670kv1873/manifest.json",
#"http://manifests.ydc2.yale.edu/manifest/Admont23.json",
#"http://oculus-dev.lib.harvard.edu/manifests/drs:5981093",
#"http://iiif-dev.bodleian.ox.ac.uk/metadata/bib_germ_1485_d1/bib_germ_1485_d1.json",
#"http://iiif-dev.bodleian.ox.ac.uk/metadata/ms_auct_t_inf_2_1/ms_auct_t_inf_2_1.json",
#"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b84473026/manifest.json",
#"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b84473026/manifest-ranges.json",
#"http://demos.biblissima-condorcet.fr/mirador/data/add_ms_10289_edited_8v-9r.json",
#"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b8438674r/manifest.json",
"http://demos.biblissima-condorcet.fr/iiif/metadata/BVH/B410186201_LI03/manifest.json",
"http://sanddragon.bl.uk/IIIFMetadataService/add_ms_10289.json",
"http://sr-svx-93.unifr.ch/metadata/iiif/bbb-0218/manifest.json",
#"http://www.shared-canvas.org/impl/demo1d/res/manifest.json"
]
for u in urls:
fh = urllib.urlopen(u)
data = fh.read()
fh.close()
try:
print("------")
print(u)
reader = ManifestReader(data)
nmfst = reader.read()
js = nmfst.toJSON()
except Exception, e:
print(" => %s: %s" % (e.__class__.__name__, e))
def test01_fixtures(self):
top = 'tests/testdata/2.0/example/fixtures/collection.json'
fh = file(top)
data = fh.read()
fh.close()
reader = ManifestReader(data)
ncoll = reader.read()
# And walk the manifests
for manifest in ncoll.manifests:
mfid = manifest.id
fn = mfid.replace('http://iiif.io/api/presentation/', 'tests/testdata/')
fh = file(fn)
data = fh.read()
fh.close()
#print "Manifest: %s" % mfid
diff = ''
js2 = ''
try:
js = json.loads(data)
reader = ManifestReader(data)
nman = reader.read()
js2 = nman.toJSON(top=True)
# Construct helpful diff...
if js != js2:
data2 = nman.toString(compact=False)
diff += "diff:\nin: %s out: %s" % (len(data), len(data2))
diff += "- is in, + is out"
for x in difflib.unified_diff(data.split('\n'), data2.split('\n')):
diff += x
except Exception, e:
diff = "Read failed with exception: %s" % (str(e))
self.assertEqual( js, js2, "Manifest %s: %s" % (mfid,diff) )
def get_manifest(url):
"""Download the manifest from url"""
try:
with request.urlopen(url) as response:
reader = ManifestReader(response.read())
return reader.read()
except Exception:
return None
def test04_buildFactory(self):
mr = ManifestReader('dataaa')
mf = mr.buildFactory('1.0')
self.assertEqual( mf.context_uri, 'http://www.shared-canvas.org/ns/context.json' )
mf = mr.buildFactory('2')
self.assertEqual( mf.context_uri, 'http://iiif.io/api/presentation/2/context.json' )
# setting in require_version overrides buildFactory() arg
mr = ManifestReader('dataaa', version='1.0')
mf = mr.buildFactory('2')
self.assertEqual( mf.context_uri, 'http://www.shared-canvas.org/ns/context.json' )
def error(num):
# Load error manifest number num
mfid = "http://iiif.io/api/presentation/2.0/example/errors/%d/manifest.json" % (num)
fn = mfid.replace('http://iiif.io/api/presentation/', 'tests/testdata/' )
fh = file(fn)
data = fh.read()
fh.close()
reader = ManifestReader(data)
what = reader.read()
js = what.toJSON()
return js
def error(num):
# Load error manifest number num
mfid = "http://iiif.io/api/presentation/2.0/example/errors/%d/manifest.json" % (
num)
fn = mfid.replace('http://iiif.io/api/presentation/', 'tests/testdata/')
fh = open(fn)
data = fh.read()
fh.close()
reader = ManifestReader(data)
what = reader.read()
js = what.toJSON()
return js
def _get_validated_manifest(self, manifest_uri, version):
"""Return a validated manifest."""
r = requests.get(manifest_uri)
if r.status_code != 200:
err_msg = 'Invalid manifest URI: {} error'.format(r.status_code)
raise BulkImportException(err_msg)
reader = ManifestReader(r.text, version=version)
try:
mf = reader.read()
mf_json = mf.toJSON()
except Exception as e:
raise BulkImportException(str(e))
return mf_json
def _get_validated_manifest(self, manifest_uri, version):
"""Return a validated manifest."""
r = requests.get(manifest_uri)
if r.status_code != 200:
err_msg = 'Invalid manifest URI: {} error'.format(r.status_code)
raise BulkImportException(err_msg)
reader = ManifestReader(r.text, version=version)
try:
mf = reader.read()
mf_json = mf.toJSON()
except Exception as e:
raise BulkImportException(str(e))
return mf_json
def test_remote(urls):
"""Test a list or remote Manifest URLs."""
for u in urls:
fh = urlopen(u)
data = fh.read()
fh.close()
try:
print("------")
print(u)
reader = ManifestReader(data)
nmfst = reader.read()
js = nmfst.toJSON()
except Exception as e:
print(" => %s: %s" % (e.__class__.__name__, e))
def do_test(data, excexp):
debug_tests = 0
reader = ManifestReader(data)
if excexp == None:
# Should pass
try:
what = reader.read()
js = what.toJSON()
print_warnings(reader)
return 1
except Exception, e:
print_warnings(reader)
if debug_tests:
print "%s: %s" % (e.__class__.__name__, e)
return 0
def test05_jsonld_to_langhash(self):
mr = ManifestReader('ab')
# Errors
self.assertRaises(DataError, mr.jsonld_to_langhash, [])
# should this one be caught as a DataError?
self.assertRaises(TypeError, mr.jsonld_to_langhash, ['@value'])
self.assertRaises(DataError, mr.jsonld_to_langhash, {})
self.assertRaises(DataError, mr.jsonld_to_langhash, {'a': 'b'})
#
lh = mr.jsonld_to_langhash('a')
self.assertEqual(lh, 'a')
lh = mr.jsonld_to_langhash({'@value': 'av'})
self.assertEqual(lh, 'av')
lh = mr.jsonld_to_langhash({'@value': 'avz', '@language': 'zz'})
self.assertEqual(lh, {'zz': 'avz'})
def process_manifest(url, path, include_label=False):
"""Process and download manifest by url"""
# Download manifest
with request.urlopen(url) as response:
assert response.getcode() == 200
reader = ManifestReader(response.read)
manifest = reader.read()
# Should we go through every sequence or only the first?
assert len(manifest.sequences) > 0
sequence = manifest.sequences[0]
canvases = sequence.canvases
prefix = manifest.label if include_label else ""
for canvas in canvases:
download_canvas(canvas, path, prefix)
def test06_labels_and_values(self):
mr = ManifestReader('ab')
# Must have label and value
self.assertRaises(KeyError, mr.labels_and_values, {})
self.assertRaises(KeyError, mr.labels_and_values, {'label': None})
self.assertRaises(KeyError, mr.labels_and_values, {'value': None})
# All possible forms...
# simple
lv = mr.labels_and_values({'label': 'l', 'value': 'v'})
self.assertEqual(lv, {'l': 'v'})
# dict and list label
lv = mr.labels_and_values({'label': {'@language': 'en',
'@value': 'lab-en'},
'value': 'v'})
self.assertEqual(lv, {'label': {'en': 'lab-en'},
'value': 'v'})
lv = mr.labels_and_values({'label': ['label1', 'label2'],
'value': 'v'})
self.assertEqual(lv, {'label': ['label1', 'label2'],
'value': 'v'})
# dict and list value
lv = mr.labels_and_values({'label': 'l',
'value': {'@language': 'cz',
'@value': 'val-cz'}})
self.assertEqual(lv, {'l': {'cz': 'val-cz'}})
lv = mr.labels_and_values({'label': 'l',
'value': ['val1', 'val2']})
self.assertEqual(lv, {'l': ['val1', 'val2']})
def check_manifest(self, data, version, warnings=[]):
# Now check data
reader = ManifestReader(data, version=version)
err = None
try:
mf = reader.read()
mf.toJSON()
# Passed!
okay = 1
except Exception as e:
# Failed
err = e
okay = 0
warnings.extend(reader.get_warnings())
infojson = {'received': data, 'okay': okay, 'warnings': warnings, \
'error': str(err)}
return self.return_json(infojson)
def do_GET_test(self):
url = request.query.get('url', '')
version = request.query.get('version', '2.0')
url = url.strip()
parsed_url = urlparse(url)
if not parsed_url.scheme.startswith('http'):
return self.format_response({'okay': 0, 'error': 'URLs must use HTTP or HTTPS', 'url': url})
try:
(data, webhandle) = self.fetch(url)
except:
return self.format_response({'okay': 0, 'error': 'Cannot fetch url', 'url': url})
# First check HTTP level
ct = webhandle.headers.get('content-type', '')
cors = webhandle.headers.get('access-control-allow-origin', '')
warnings = []
if not ct.startswith('application/json') and not ct.startswith('application/ld+json'):
# not json
warnings.append("URL does not have correct content-type header: got \"%s\", expected JSON" % ct)
if cors != "*":
warnings.append("URL does not have correct access-control-allow-origin header:"
" got \"%s\", expected *" % cors)
# Now check data
reader = ManifestReader(data, version=version)
err = None
try:
mf = reader.read()
mf.toJSON()
# Passed!
okay = 1
except Exception as err:
# Failed
okay = 0
warnings.extend(reader.get_warnings())
infojson = {'url': url, 'okay': okay, 'warnings': warnings, 'error': str(err)}
return self.format_response(infojson)
def do_POST_test(self):
data = request.json
version = '2.0'
warnings = []
# Now check data
reader = ManifestReader(data, version=version)
err = None
try:
mf = reader.read()
mf.toJSON()
# Passed!
okay = 1
except Exception as err:
# Failed
okay = 0
warnings.extend(reader.get_warnings())
infojson = {'received': data, 'okay': okay, 'warnings': warnings, 'error': str(err)}
return self.format_response(infojson)
def test01_fixtures(self):
top = 'tests/testdata/2.0/example/fixtures/collection.json'
fh = open(top)
data = fh.read()
fh.close()
reader = ManifestReader(data)
ncoll = reader.read()
# And walk the manifests
for manifest in ncoll.manifests:
mfid = manifest.id
fn = mfid.replace('http://iiif.io/api/presentation/',
'tests/testdata/')
fh = open(fn)
data = fh.read()
fh.close()
#print("Manifest: %s" % mfid)
diff = ''
js2 = ''
try:
js = json.loads(data)
reader = ManifestReader(data)
nman = reader.read()
js2 = nman.toJSON(top=True)
# Construct helpful diff...
if js != js2:
data2 = nman.toString(compact=False)
diff += "diff:\nin: %s out: %s" % (len(data), len(data2))
diff += "- is in, + is out"
for x in difflib.unified_diff(data.split('\n'),
data2.split('\n')):
diff += x
except Exception as e:
diff = "Read failed with exception: %s" % (str(e))
self.assertEqual(js, js2, "Manifest %s: %s" % (mfid, diff))
def check_manifest(self, data, version, url=None, warnings=[]):
"""Check manifest data at version, return JSON."""
reader = ManifestReader(data, version=version)
err = None
try:
mf = reader.read()
mf.toJSON()
# Passed!
okay = 1
except Exception as e:
# Failed
err = e
okay = 0
warnings.extend(reader.get_warnings())
infojson = {
'received': data,
'okay': okay,
'warnings': warnings,
'error': str(err),
'url': url
}
return self.return_json(infojson)
def test_validate_iiif(self):
view = ManifestDetail.as_view()
volume = Manifest.objects.all().first()
kwargs = {'pid': self.volume.pid, 'version': 'v2'}
url = reverse('ManifestRender', kwargs=kwargs)
response = self.client.get(url)
manifest = json.loads(response.content.decode('UTF-8-sig'))
reader = ManifestReader(response.content, version='2.1')
try:
mf = reader.read()
assert mf.toJSON()
except Exception as error:
raise Exception(error)
assert manifest['@id'] == "%s/manifest" % (self.volume.baseurl)
assert manifest['label'] == self.volume.label
assert manifest['description'] == volume.summary
assert manifest['thumbnail'][
'@id'] == "%s/%s/full/600,/0/default.jpg" % (
self.volume.canvas_set.all().first().IIIF_IMAGE_SERVER_BASE,
self.start_canvas.pid)
assert manifest['sequences'][0][
'startCanvas'] == self.volume.start_canvas
def check_manifest(self, data, version, url=None, warnings=[]):
"""Check manifest data at version, return JSON."""
reader = ManifestReader(data, version=version)
err = None
try:
mf = reader.read()
mf.toJSON()
# Passed!
okay = 1
except Exception as e:
# Failed
err = e
okay = 0
warnings.extend(reader.get_warnings())
infojson = {
'received': data,
'okay': okay,
'warnings': warnings,
'error': str(err),
'url': url
}
return self.return_json(infojson)
def test04_buildFactory(self):
mr = ManifestReader('dataaa')
mf = mr.buildFactory('1.0')
self.assertEqual(
mf.context_uri, 'http://www.shared-canvas.org/ns/context.json')
mf = mr.buildFactory('2')
self.assertEqual(
mf.context_uri, 'http://iiif.io/api/presentation/2/context.json')
# setting in require_version overrides buildFactory() arg
mr = ManifestReader('dataaa', version='1.0')
mf = mr.buildFactory('2')
self.assertEqual(
mf.context_uri, 'http://www.shared-canvas.org/ns/context.json')
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Download images from IIIF Presentation Manifest")
parser.add_argument("url", help="URL to manifest")
parser.add_argument("-p",
"--path",
help="location to download images",
default=".")
args = parser.parse_args()
# Get the manifest
with request.urlopen(args.url) as response:
assert response.getcode() == 200
data = response.read()
reader = ManifestReader(data)
manifest = reader.read()
# Get the sequence
assert len(manifest.sequences) > 0
sequence = manifest.sequences[0]
canvases = sequence.canvases
print("Downloading {} images of manifest \"{}\" to {}.".format(
len(canvases), manifest.label, args.path))
for canvas in canvases:
label = canvas.label
print("Downloading {}...".format(label))
annot = canvas.images[0]
extension = mimetypes.guess_extension(annot.resource.format)
if filename is not '-':
fh.close()
if __name__ == "__main__":
if len(sys.argv) > 1:
args = sys.argv[1]
else:
args = '-'
with _smart_open(args) as handle:
content = handle.read()
# shoving these to fix stuff
json_manifest = json.loads(content)
json_manifest['logo'] = LOGO_URI
json_manifest['@context'] = MANIFEST_CONTEXT[VERSION]
#reader = ManifestReader(content, version=VERSION)
reader = ManifestReader(json.dumps(json_manifest), version=VERSION)
try:
manifest = reader.read()
manifest.toJSON() # passed!
except Exception as e:
if VERBOSE:
print('failed ({}): {}'.format(args, e))
exit(1)
else:
exit(0)
def check_manifest(self, data, version, url=None, warnings=[]):
"""Check manifest data at version, return JSON."""
infojson = {}
# Check if 3.0 if so run through schema rather than this version...
if version == '3.0':
try:
infojson = schemavalidator.validate(data, version, url)
for error in infojson['errorList']:
error.pop('error', None)
mf = json.loads(data)
if url and 'id' in mf and mf['id'] != url:
raise ValidationError(
"The manifest id ({}) should be the same as the URL it is published at ({})."
.format(mf["id"], url))
except ValidationError as e:
if infojson:
infojson['errorList'].append({
'title':
'Resolve Error',
'detail':
str(e),
'description':
'',
'path':
'/id',
'context':
'{ \'id\': \'...\'}'
})
else:
infojson = {
'received': data,
'okay': 0,
'error': str(e),
'url': url,
'warnings': []
}
except Exception as e:
traceback.print_exc()
infojson = {
'received':
data,
'okay':
0,
'error':
'Presentation Validator bug: "{}". Please create a <a href="https://github.com/IIIF/presentation-validator/issues">Validator Issue</a>, including a link to the manifest.'
.format(e),
'url':
url,
'warnings': []
}
else:
reader = ManifestReader(data, version=version)
err = None
try:
mf = reader.read()
mf.toJSON()
if url and mf.id != url:
raise ValidationError(
"Manifest @id ({}) is different to the location where it was retrieved ({})"
.format(mf.id, url))
# Passed!
okay = 1
except KeyError as e:
print('Failed falidation due to:')
traceback.print_exc()
err = 'Failed due to KeyError {}, check trace for details'.format(
e)
okay = 0
except Exception as e:
# Failed
print('Failed falidation due to:')
traceback.print_exc()
err = e
okay = 0
warnings.extend(reader.get_warnings())
infojson = {
'received': data,
'okay': okay,
'warnings': warnings,
'error': str(err),
'url': url
}
return self.return_json(infojson)
def test03_init(self):
mr = ManifestReader('oopsee')
self.assertEqual(mr.data, 'oopsee')
self.assertEqual(mr.require_version, None)