def test_remote(): """Test a list or remote Manifest URLs.""" urls = [ #"http://dms-data.stanford.edu/data/manifests/Walters/qm670kv1873/manifest.json", #"http://manifests.ydc2.yale.edu/manifest/Admont23.json", #"http://oculus-dev.lib.harvard.edu/manifests/drs:5981093", #"http://iiif-dev.bodleian.ox.ac.uk/metadata/bib_germ_1485_d1/bib_germ_1485_d1.json", #"http://iiif-dev.bodleian.ox.ac.uk/metadata/ms_auct_t_inf_2_1/ms_auct_t_inf_2_1.json", #"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b84473026/manifest.json", #"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b84473026/manifest-ranges.json", #"http://demos.biblissima-condorcet.fr/mirador/data/add_ms_10289_edited_8v-9r.json", #"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b8438674r/manifest.json", "http://demos.biblissima-condorcet.fr/iiif/metadata/BVH/B410186201_LI03/manifest.json", "http://sanddragon.bl.uk/IIIFMetadataService/add_ms_10289.json", "http://sr-svx-93.unifr.ch/metadata/iiif/bbb-0218/manifest.json", #"http://www.shared-canvas.org/impl/demo1d/res/manifest.json" ] for u in urls: fh = urllib.urlopen(u) data = fh.read() fh.close() try: print("------") print(u) reader = ManifestReader(data) nmfst = reader.read() js = nmfst.toJSON() except Exception as e: print(" => %s: %s" % (e.__class__.__name__, e))
def check_manifest(self, data, version, url=None, warnings=[]): """Check manifest data at version, return JSON.""" infojson = {} # Check if 3.0 if so run through schema rather than this version... if version == '3.0': infojson = schemavalidator.validate(data, version, url) else: reader = ManifestReader(data, version=version) err = None try: mf = reader.read() mf.toJSON() # Passed! okay = 1 except Exception as e: # Failed err = e okay = 0 warnings.extend(reader.get_warnings()) infojson = { 'received': data, 'okay': okay, 'warnings': warnings, 'error': str(err), 'url': url } return self.return_json(infojson)
def test07_every_value_its_list(self): fh = open('tests/everything_a_list_fixture.json') data = fh.read() fh.close() js = json.loads(data) mr = ManifestReader(js) doc = mr.read()
def do_test(data, excexp): debug_tests = 0 reader = ManifestReader(data) if excexp == None: # Should pass try: what = reader.read() js = what.toJSON() print_warnings(reader) return 1 except Exception as e: print_warnings(reader) if debug_tests: print("%s: %s" % (e.__class__.__name__, e)) return 0 else: # Should raise exception excexp try: what = reader.read() js = what.toJSON() if debug_tests: print("Loaded okay, should have failed") print(json.dumps(js, sort_keys=True, indent=2)) print_warnings(reader) return "no exception" except excexp as e: # Got expected exception return None except Exception as e: # Not the exception we expected return "bad exception, got %s (%s)" % (e.__class__,str(e))
def do_test(data, excexp): debug_tests = 0 reader = ManifestReader(data) if excexp == None: # Should pass try: what = reader.read() js = what.toJSON() print_warnings(reader) return 1 except Exception as e: print_warnings(reader) if debug_tests: print("%s: %s" % (e.__class__.__name__, e)) return 0 else: # Should raise exception excexp try: what = reader.read() js = what.toJSON() if debug_tests: print("Loaded okay, should have failed") print(json.dumps(js, sort_keys=True, indent=2)) print_warnings(reader) return "no exception" except excexp as e: # Got expected exception return None except Exception as e: # Not the exception we expected return "bad exception, got %s (%s)" % (e.__class__, str(e))
def test07_label_value_language(self): fh = open('tests/label_value_language_fixture.json') data = fh.read() fh.close() js = json.loads(data) mr = ManifestReader(js) doc = mr.read()
def test06_range_range(self): fh = open('tests/range_range_fixture.json') data = fh.read() fh.close() js = json.loads(data) mr = ManifestReader(js) doc = mr.read()
def test05_multipleContexts(self): fh = open('tests/multiple_contexts_fixture.json') data = fh.read() fh.close() js = json.loads(data) mr = ManifestReader(js) doc = mr.read()
def test_remote(): """Test a list or remote Manifest URLs.""" urls = [ #"http://dms-data.stanford.edu/data/manifests/Walters/qm670kv1873/manifest.json", #"http://manifests.ydc2.yale.edu/manifest/Admont23.json", #"http://oculus-dev.lib.harvard.edu/manifests/drs:5981093", #"http://iiif-dev.bodleian.ox.ac.uk/metadata/bib_germ_1485_d1/bib_germ_1485_d1.json", #"http://iiif-dev.bodleian.ox.ac.uk/metadata/ms_auct_t_inf_2_1/ms_auct_t_inf_2_1.json", #"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b84473026/manifest.json", #"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b84473026/manifest-ranges.json", #"http://demos.biblissima-condorcet.fr/mirador/data/add_ms_10289_edited_8v-9r.json", #"http://demos.biblissima-condorcet.fr/iiif/metadata/ark:/12148/btv1b8438674r/manifest.json", "http://demos.biblissima-condorcet.fr/iiif/metadata/BVH/B410186201_LI03/manifest.json", "http://sanddragon.bl.uk/IIIFMetadataService/add_ms_10289.json", "http://sr-svx-93.unifr.ch/metadata/iiif/bbb-0218/manifest.json", #"http://www.shared-canvas.org/impl/demo1d/res/manifest.json" ] for u in urls: fh = urllib.urlopen(u) data = fh.read() fh.close() try: print("------") print(u) reader = ManifestReader(data) nmfst = reader.read() js = nmfst.toJSON() except Exception, e: print(" => %s: %s" % (e.__class__.__name__, e))
def test01_fixtures(self): top = 'tests/testdata/2.0/example/fixtures/collection.json' fh = file(top) data = fh.read() fh.close() reader = ManifestReader(data) ncoll = reader.read() # And walk the manifests for manifest in ncoll.manifests: mfid = manifest.id fn = mfid.replace('http://iiif.io/api/presentation/', 'tests/testdata/') fh = file(fn) data = fh.read() fh.close() #print "Manifest: %s" % mfid diff = '' js2 = '' try: js = json.loads(data) reader = ManifestReader(data) nman = reader.read() js2 = nman.toJSON(top=True) # Construct helpful diff... if js != js2: data2 = nman.toString(compact=False) diff += "diff:\nin: %s out: %s" % (len(data), len(data2)) diff += "- is in, + is out" for x in difflib.unified_diff(data.split('\n'), data2.split('\n')): diff += x except Exception, e: diff = "Read failed with exception: %s" % (str(e)) self.assertEqual( js, js2, "Manifest %s: %s" % (mfid,diff) )
def get_manifest(url): """Download the manifest from url""" try: with request.urlopen(url) as response: reader = ManifestReader(response.read()) return reader.read() except Exception: return None
def test04_buildFactory(self): mr = ManifestReader('dataaa') mf = mr.buildFactory('1.0') self.assertEqual( mf.context_uri, 'http://www.shared-canvas.org/ns/context.json' ) mf = mr.buildFactory('2') self.assertEqual( mf.context_uri, 'http://iiif.io/api/presentation/2/context.json' ) # setting in require_version overrides buildFactory() arg mr = ManifestReader('dataaa', version='1.0') mf = mr.buildFactory('2') self.assertEqual( mf.context_uri, 'http://www.shared-canvas.org/ns/context.json' )
def error(num): # Load error manifest number num mfid = "http://iiif.io/api/presentation/2.0/example/errors/%d/manifest.json" % (num) fn = mfid.replace('http://iiif.io/api/presentation/', 'tests/testdata/' ) fh = file(fn) data = fh.read() fh.close() reader = ManifestReader(data) what = reader.read() js = what.toJSON() return js
def error(num): # Load error manifest number num mfid = "http://iiif.io/api/presentation/2.0/example/errors/%d/manifest.json" % ( num) fn = mfid.replace('http://iiif.io/api/presentation/', 'tests/testdata/') fh = open(fn) data = fh.read() fh.close() reader = ManifestReader(data) what = reader.read() js = what.toJSON() return js
def _get_validated_manifest(self, manifest_uri, version): """Return a validated manifest.""" r = requests.get(manifest_uri) if r.status_code != 200: err_msg = 'Invalid manifest URI: {} error'.format(r.status_code) raise BulkImportException(err_msg) reader = ManifestReader(r.text, version=version) try: mf = reader.read() mf_json = mf.toJSON() except Exception as e: raise BulkImportException(str(e)) return mf_json
def test_remote(urls): """Test a list or remote Manifest URLs.""" for u in urls: fh = urlopen(u) data = fh.read() fh.close() try: print("------") print(u) reader = ManifestReader(data) nmfst = reader.read() js = nmfst.toJSON() except Exception as e: print(" => %s: %s" % (e.__class__.__name__, e))
def do_test(data, excexp): debug_tests = 0 reader = ManifestReader(data) if excexp == None: # Should pass try: what = reader.read() js = what.toJSON() print_warnings(reader) return 1 except Exception, e: print_warnings(reader) if debug_tests: print "%s: %s" % (e.__class__.__name__, e) return 0
def test05_jsonld_to_langhash(self): mr = ManifestReader('ab') # Errors self.assertRaises(DataError, mr.jsonld_to_langhash, []) # should this one be caught as a DataError? self.assertRaises(TypeError, mr.jsonld_to_langhash, ['@value']) self.assertRaises(DataError, mr.jsonld_to_langhash, {}) self.assertRaises(DataError, mr.jsonld_to_langhash, {'a': 'b'}) # lh = mr.jsonld_to_langhash('a') self.assertEqual(lh, 'a') lh = mr.jsonld_to_langhash({'@value': 'av'}) self.assertEqual(lh, 'av') lh = mr.jsonld_to_langhash({'@value': 'avz', '@language': 'zz'}) self.assertEqual(lh, {'zz': 'avz'})
def process_manifest(url, path, include_label=False): """Process and download manifest by url""" # Download manifest with request.urlopen(url) as response: assert response.getcode() == 200 reader = ManifestReader(response.read) manifest = reader.read() # Should we go through every sequence or only the first? assert len(manifest.sequences) > 0 sequence = manifest.sequences[0] canvases = sequence.canvases prefix = manifest.label if include_label else "" for canvas in canvases: download_canvas(canvas, path, prefix)
def test06_labels_and_values(self): mr = ManifestReader('ab') # Must have label and value self.assertRaises(KeyError, mr.labels_and_values, {}) self.assertRaises(KeyError, mr.labels_and_values, {'label': None}) self.assertRaises(KeyError, mr.labels_and_values, {'value': None}) # All possible forms... # simple lv = mr.labels_and_values({'label': 'l', 'value': 'v'}) self.assertEqual(lv, {'l': 'v'}) # dict and list label lv = mr.labels_and_values({'label': {'@language': 'en', '@value': 'lab-en'}, 'value': 'v'}) self.assertEqual(lv, {'label': {'en': 'lab-en'}, 'value': 'v'}) lv = mr.labels_and_values({'label': ['label1', 'label2'], 'value': 'v'}) self.assertEqual(lv, {'label': ['label1', 'label2'], 'value': 'v'}) # dict and list value lv = mr.labels_and_values({'label': 'l', 'value': {'@language': 'cz', '@value': 'val-cz'}}) self.assertEqual(lv, {'l': {'cz': 'val-cz'}}) lv = mr.labels_and_values({'label': 'l', 'value': ['val1', 'val2']}) self.assertEqual(lv, {'l': ['val1', 'val2']})
def check_manifest(self, data, version, warnings=[]): # Now check data reader = ManifestReader(data, version=version) err = None try: mf = reader.read() mf.toJSON() # Passed! okay = 1 except Exception as e: # Failed err = e okay = 0 warnings.extend(reader.get_warnings()) infojson = {'received': data, 'okay': okay, 'warnings': warnings, \ 'error': str(err)} return self.return_json(infojson)
def do_GET_test(self): url = request.query.get('url', '') version = request.query.get('version', '2.0') url = url.strip() parsed_url = urlparse(url) if not parsed_url.scheme.startswith('http'): return self.format_response({'okay': 0, 'error': 'URLs must use HTTP or HTTPS', 'url': url}) try: (data, webhandle) = self.fetch(url) except: return self.format_response({'okay': 0, 'error': 'Cannot fetch url', 'url': url}) # First check HTTP level ct = webhandle.headers.get('content-type', '') cors = webhandle.headers.get('access-control-allow-origin', '') warnings = [] if not ct.startswith('application/json') and not ct.startswith('application/ld+json'): # not json warnings.append("URL does not have correct content-type header: got \"%s\", expected JSON" % ct) if cors != "*": warnings.append("URL does not have correct access-control-allow-origin header:" " got \"%s\", expected *" % cors) # Now check data reader = ManifestReader(data, version=version) err = None try: mf = reader.read() mf.toJSON() # Passed! okay = 1 except Exception as err: # Failed okay = 0 warnings.extend(reader.get_warnings()) infojson = {'url': url, 'okay': okay, 'warnings': warnings, 'error': str(err)} return self.format_response(infojson)
def do_POST_test(self): data = request.json version = '2.0' warnings = [] # Now check data reader = ManifestReader(data, version=version) err = None try: mf = reader.read() mf.toJSON() # Passed! okay = 1 except Exception as err: # Failed okay = 0 warnings.extend(reader.get_warnings()) infojson = {'received': data, 'okay': okay, 'warnings': warnings, 'error': str(err)} return self.format_response(infojson)
def test01_fixtures(self): top = 'tests/testdata/2.0/example/fixtures/collection.json' fh = open(top) data = fh.read() fh.close() reader = ManifestReader(data) ncoll = reader.read() # And walk the manifests for manifest in ncoll.manifests: mfid = manifest.id fn = mfid.replace('http://iiif.io/api/presentation/', 'tests/testdata/') fh = open(fn) data = fh.read() fh.close() #print("Manifest: %s" % mfid) diff = '' js2 = '' try: js = json.loads(data) reader = ManifestReader(data) nman = reader.read() js2 = nman.toJSON(top=True) # Construct helpful diff... if js != js2: data2 = nman.toString(compact=False) diff += "diff:\nin: %s out: %s" % (len(data), len(data2)) diff += "- is in, + is out" for x in difflib.unified_diff(data.split('\n'), data2.split('\n')): diff += x except Exception as e: diff = "Read failed with exception: %s" % (str(e)) self.assertEqual(js, js2, "Manifest %s: %s" % (mfid, diff))
def check_manifest(self, data, version, url=None, warnings=[]): """Check manifest data at version, return JSON.""" reader = ManifestReader(data, version=version) err = None try: mf = reader.read() mf.toJSON() # Passed! okay = 1 except Exception as e: # Failed err = e okay = 0 warnings.extend(reader.get_warnings()) infojson = { 'received': data, 'okay': okay, 'warnings': warnings, 'error': str(err), 'url': url } return self.return_json(infojson)
def test_validate_iiif(self): view = ManifestDetail.as_view() volume = Manifest.objects.all().first() kwargs = {'pid': self.volume.pid, 'version': 'v2'} url = reverse('ManifestRender', kwargs=kwargs) response = self.client.get(url) manifest = json.loads(response.content.decode('UTF-8-sig')) reader = ManifestReader(response.content, version='2.1') try: mf = reader.read() assert mf.toJSON() except Exception as error: raise Exception(error) assert manifest['@id'] == "%s/manifest" % (self.volume.baseurl) assert manifest['label'] == self.volume.label assert manifest['description'] == volume.summary assert manifest['thumbnail'][ '@id'] == "%s/%s/full/600,/0/default.jpg" % ( self.volume.canvas_set.all().first().IIIF_IMAGE_SERVER_BASE, self.start_canvas.pid) assert manifest['sequences'][0][ 'startCanvas'] == self.volume.start_canvas
def test04_buildFactory(self): mr = ManifestReader('dataaa') mf = mr.buildFactory('1.0') self.assertEqual( mf.context_uri, 'http://www.shared-canvas.org/ns/context.json') mf = mr.buildFactory('2') self.assertEqual( mf.context_uri, 'http://iiif.io/api/presentation/2/context.json') # setting in require_version overrides buildFactory() arg mr = ManifestReader('dataaa', version='1.0') mf = mr.buildFactory('2') self.assertEqual( mf.context_uri, 'http://www.shared-canvas.org/ns/context.json')
if __name__ == "__main__": parser = argparse.ArgumentParser( description="Download images from IIIF Presentation Manifest") parser.add_argument("url", help="URL to manifest") parser.add_argument("-p", "--path", help="location to download images", default=".") args = parser.parse_args() # Get the manifest with request.urlopen(args.url) as response: assert response.getcode() == 200 data = response.read() reader = ManifestReader(data) manifest = reader.read() # Get the sequence assert len(manifest.sequences) > 0 sequence = manifest.sequences[0] canvases = sequence.canvases print("Downloading {} images of manifest \"{}\" to {}.".format( len(canvases), manifest.label, args.path)) for canvas in canvases: label = canvas.label print("Downloading {}...".format(label)) annot = canvas.images[0] extension = mimetypes.guess_extension(annot.resource.format)
if filename is not '-': fh.close() if __name__ == "__main__": if len(sys.argv) > 1: args = sys.argv[1] else: args = '-' with _smart_open(args) as handle: content = handle.read() # shoving these to fix stuff json_manifest = json.loads(content) json_manifest['logo'] = LOGO_URI json_manifest['@context'] = MANIFEST_CONTEXT[VERSION] #reader = ManifestReader(content, version=VERSION) reader = ManifestReader(json.dumps(json_manifest), version=VERSION) try: manifest = reader.read() manifest.toJSON() # passed! except Exception as e: if VERBOSE: print('failed ({}): {}'.format(args, e)) exit(1) else: exit(0)
def check_manifest(self, data, version, url=None, warnings=[]): """Check manifest data at version, return JSON.""" infojson = {} # Check if 3.0 if so run through schema rather than this version... if version == '3.0': try: infojson = schemavalidator.validate(data, version, url) for error in infojson['errorList']: error.pop('error', None) mf = json.loads(data) if url and 'id' in mf and mf['id'] != url: raise ValidationError( "The manifest id ({}) should be the same as the URL it is published at ({})." .format(mf["id"], url)) except ValidationError as e: if infojson: infojson['errorList'].append({ 'title': 'Resolve Error', 'detail': str(e), 'description': '', 'path': '/id', 'context': '{ \'id\': \'...\'}' }) else: infojson = { 'received': data, 'okay': 0, 'error': str(e), 'url': url, 'warnings': [] } except Exception as e: traceback.print_exc() infojson = { 'received': data, 'okay': 0, 'error': 'Presentation Validator bug: "{}". Please create a <a href="https://github.com/IIIF/presentation-validator/issues">Validator Issue</a>, including a link to the manifest.' .format(e), 'url': url, 'warnings': [] } else: reader = ManifestReader(data, version=version) err = None try: mf = reader.read() mf.toJSON() if url and mf.id != url: raise ValidationError( "Manifest @id ({}) is different to the location where it was retrieved ({})" .format(mf.id, url)) # Passed! okay = 1 except KeyError as e: print('Failed falidation due to:') traceback.print_exc() err = 'Failed due to KeyError {}, check trace for details'.format( e) okay = 0 except Exception as e: # Failed print('Failed falidation due to:') traceback.print_exc() err = e okay = 0 warnings.extend(reader.get_warnings()) infojson = { 'received': data, 'okay': okay, 'warnings': warnings, 'error': str(err), 'url': url } return self.return_json(infojson)
def test03_init(self): mr = ManifestReader('oopsee') self.assertEqual(mr.data, 'oopsee') self.assertEqual(mr.require_version, None)