def freeze(options, args): rosrs = ROSRS_Session(options["rosrs_uri"], options["rosrs_access_token"]) service_uri = urljoin(options["rosrs_uri"], "../evo/finalize/") body = { 'target': args[2], } body = json.dumps(body) reqheaders = {} (status, reason, headers, data) = response = rosrs.doRequest(uripath=service_uri, method="POST", body=body, ctype="application/json", reqheaders=reqheaders) if "location" in headers: while print_job_status(parse_job(rosrs, headers['location']), options, True): time.sleep(1) print "freeze operation finished successfully" return 0 else: print status print reason print headers print data print "Given URI isn't correct" return -1
def copy_operation(options, args, ro_type): options["rosrs_access_token"] rosrs = ROSRS_Session(options["rosrs_uri"], options["rosrs_access_token"]) service_uri = urljoin(options["rosrs_uri"], "../evo/copy/") body = { 'copyfrom': args[2], 'type': ro_type, 'finalize': ("%s" % options['freeze']).lower() } body = json.dumps(body) reqheaders = {'Slug': args[3]} response = rosrs.doRequest(uripath=service_uri, method="POST", body=body, ctype="application/json", reqheaders=reqheaders) if response[0] != 201: return handle_copy_error(options, rosrs, response, ro_type) if not options["asynchronous"]: return handle_synchronous_copy_operation_with_esc_option( options, rosrs, response, ro_type) if options["asynchronous"]: return handle_asynchronous_copy_operation(options, rosrs, response, ro_type) return 0
def testArchive(self): rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN) (copy_status, archiveUri) = self.createArchive(self.TEST_CREATED_RO_ID, self.TEST_SNAPHOT_ID, False) assert copy_status == "DONE" (status, reason, data, evo_type) = rosrs.getROEvolution(archiveUri) assert evo_type == 3 self.freeze(archiveUri) (status, reason, data, evo_type) = rosrs.getROEvolution(archiveUri) assert evo_type == 2 (status, reason) = self.rosrs.deleteRO(self.TEST_CREATED_RO_ID) (status, reason) = self.rosrs.deleteRO(archiveUri)
def freeze(self, ro_uri): rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN) service_uri = urljoin(ro_test_config.ROSRS_URI, "../evo/finalize/") body = { 'target': ro_uri, } body = json.dumps(body) reqheaders = {} (status, reason, headers, data) = rosrs.doRequest(uripath=service_uri, method="POST", body=body, ctype="application/json", reqheaders=reqheaders) job_location = get_location(headers) status = "RUNNING" while status == "RUNNING": (status, id) = parse_job(rosrs, job_location) return status
def setUp(self): super(TestROSRSMetadata, self).setUp() self.rosrs = ROSRS_Session(Config.ROSRS_API_URI, accesskey=Config.AUTHORIZATION) # Clean up from previous runs self.rosrs.deleteRO(Config.TEST_RO_PATH) return
def __init__(self, roconfig, roref, dummysetupfortest=False): """ Initialize: read manifest from object at given directory into local RDF graph roconfig is the research object manager configuration, supplied as a dictionary roref a URI reference that refers to the Research Object to be accessed, or relative path name (see ro_uriutils.resolveFileAsUri for interpretation) dummysetupfortest is an optional parameter that, if True, suppresses some aspects of the setup (does not attempt to read a RO manifest) for isolated testing. """ self.roconfig = roconfig self.roref = roref self.dummyfortest = dummysetupfortest self.manifestgraph = None self.roannotations = None self.registries = None uri = resolveFileAsUri(roref) if not uri.endswith("/"): uri += "/" self.rouri = rdflib.URIRef(uri) if self._isLocal(): self.rosrs = None else: self.rosrs = ROSRS_Session( self.roconfig["rosrs_uri"], self.roconfig["rosrs_access_token"] ) self._loadManifest() # Get RO URI from manifest # May be different from computed value if manifest has absolute URI self.rouri = self.manifestgraph.value(None, RDF.type, RO.ResearchObject) # Check that the manifest contained at least one RO URI assert self.rouri is not None return
def setUp(self): super(TestEvoCommands, self).setUp() self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN) (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID, "Test RO for ROEVO", "Test Creator", "2012-09-06") self.CREATED_RO = rouri; return
def setUp(self): super(TestROSRS_Session, self).setUp() self.rosrs = ROSRS_Session(Config.ROSRS_API_URI, accesskey=Config.AUTHORIZATION) # Clean up from previous runs self.rosrs.deleteRO(Config.TEST_RO_PATH, purge=True) self.createdTestRO = None return
def setUp(self): super(TestROSRSMetadata, self).setUp() self.rosrs = ROSRS_Session(Config.ROSRS_API_URI, accesskey=Config.AUTHORIZATION) self.roname = Config.TEST_RO_NAME self.ropath = self.roname + "/" # Clean up from previous runs self.rosrs.deleteRO(self.ropath, purge=True) return
class TestEvo(TestROEVOSupport.TestROEVOSupport): TEST_RO_ID = "ro-manager-evo-test-ro" TEST_SNAPHOT_RO_ID = "ro-manager-test-evo-snaphot-ro" TEST_SNAPHOT_ID = "ro-manager-test-evo-snaphot" TEST_CREATED_RO_ID = "" def setUp(self): super(TestEvo, self).setUp() self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN) (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID, "Test RO for ROEVO", "Test Creator", "2012-09-06") self.TEST_CREATED_RO_ID = rouri return def tearDown(self): (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID+"/") (status, reason) = self.rosrs.deleteRO(self.TEST_CREATED_RO_ID) super(TestEvo, self).tearDown() return def testSnapshot(self): rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN) (copy_status, snapshot_uri) = self.createSnapshot(self.TEST_CREATED_RO_ID, self.TEST_SNAPHOT_ID, False) assert copy_status == "DONE" (status, reason, data, evo_type) = rosrs.getROEvolution(snapshot_uri) assert evo_type == 3 self.freeze(snapshot_uri) (status, reason, data, evo_type) = rosrs.getROEvolution(snapshot_uri) assert evo_type == 1 (status, reason) = self.rosrs.deleteRO(self.TEST_CREATED_RO_ID) (status, reason) = self.rosrs.deleteRO(snapshot_uri) def testArchive(self): rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN) (copy_status, archiveUri) = self.createArchive(self.TEST_CREATED_RO_ID, self.TEST_SNAPHOT_ID, False) assert copy_status == "DONE" (status, reason, data, evo_type) = rosrs.getROEvolution(archiveUri) assert evo_type == 3 self.freeze(archiveUri) (status, reason, data, evo_type) = rosrs.getROEvolution(archiveUri) assert evo_type == 2 (status, reason) = self.rosrs.deleteRO(self.TEST_CREATED_RO_ID) (status, reason) = self.rosrs.deleteRO(archiveUri)
def copy_operation(options, args, ro_type): options["rosrs_access_token"] rosrs = ROSRS_Session(options["rosrs_uri"], options["rosrs_access_token"]) service_uri = urljoin(options["rosrs_uri"], "../evo/copy/") body = { 'copyfrom': args[2], 'type': ro_type, 'finalize': ( "%s" % options['freeze']).lower() } body = json.dumps(body) reqheaders = { 'Slug' : args[3] } response = rosrs.doRequest(uripath=service_uri, method="POST", body=body, ctype="application/json", reqheaders=reqheaders) if response[0] != 201: return handle_copy_error(options, rosrs, response, ro_type) if not options["asynchronous"]: return handle_synchronous_copy_operation_with_esc_option(options, rosrs, response, ro_type) if options["asynchronous"]: return handle_asynchronous_copy_operation(options, rosrs, response, ro_type) return 0
def createArchive(self, live_name,sp_name,freeze = True): service_uri = urljoin(ro_test_config.ROSRS_URI, "../evo/copy/") body = { 'copyfrom': live_name, 'target': sp_name, 'type': "ARCHIVE", 'finalize': ( "%s" % freeze).lower() } body = json.dumps(body) reqheaders = { 'token': ro_test_config.ROSRS_ACCESS_TOKEN, 'Slug' : sp_name, } rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN) (status, reason, headers, data) = rosrs.doRequest(uripath=service_uri, method="POST", body=body, ctype="application/json", reqheaders=reqheaders) job_location = get_location(headers) status = "RUNNING" while status == "RUNNING": (status, id) = parse_job(rosrs, job_location) return (status, id)
def __init__(self, roconfig, roref, dummysetupfortest=False): """ Initialize: read manifest from object at given directory into local RDF graph roconfig is the research object manager configuration, supplied as a dictionary roref a URI reference that refers to the Research Object to be accessed, or relative path name (see ro_uriutils.resolveFileAsUri for interpretation) dummysetupfortest is an optional parameter that, if True, suppresses some aspects of the setup (does not attempt to read a RO manifest) for isolated testing. """ self.roconfig = roconfig self.roref = roref self.dummyfortest = dummysetupfortest self.manifestgraph = None self.roannotations = None self.registries = None uri = resolveFileAsUri(roref) if not uri.endswith("/"): uri += "/" self.rouri = rdflib.URIRef(uri) if self._isLocal(): self.rosrs = None else: self.rosrs = ROSRS_Session(self.roconfig["rosrs_uri"], self.roconfig["rosrs_access_token"]) self._loadManifest() # Get RO URI from manifest # May be different from computed value if manifest has absolute URI # Nested URIs may be present; ours is the one described by the manifest URI, # which is determined by the _loadManifest() method. for s in self.manifestgraph.subjects(RDF.type, RO.ResearchObject): if self.manifestgraph.value(s, ORE.isDescribedBy) == self.manifesturi: self.rouri = s # Check that the manifest contained at least one RO URI assert self.rouri is not None return
def testRemoteStatusWithWrongUriGiven(self): self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN) self.rosrs.deleteRO(self.TEST_RO_ID + "/") self.rosrs.deleteRO("some-strange-uri/") args = [ "ro", "status", ro_test_config.ROSRS_URI + "some-strange-uri/", "-r", ro_test_config.ROSRS_URI, "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-v" ] with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) outtxt = self.outstr.getvalue() assert status == -1 self.assertEqual(outtxt.count("Wrong URI was given"), 1) self.rosrs.deleteRO(self.TEST_RO_ID + "/") return
def testRemoteStatusLiveRO(self): self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN) (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID, "Test RO for ROEVO", "Test Creator", "2012-09-06") args = [ "ro", "status", str(rouri), "-r", ro_test_config.ROSRS_URI, "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-v" ] with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == 0 outtxt = self.outstr.getvalue() self.assertEqual(outtxt.count("LIVE"), 1) (status, reason) = self.rosrs.deleteRO(rouri) return
class TestEvoCommands(TestROEVOSupport.TestROEVOSupport): TEST_RO_ID = "ro-manger-evo-test-ro" TEST_SNAPHOT_ID = "ro-manager-evo-test-snaphot" TEST_ARCHIVE_ID = "ro-manager-evo-test-archive-id" TEST_UNDEFINED_ID = "ro-manager-evo-test-undefined-id" CREATED_RO = "" def setUp(self): super(TestEvoCommands, self).setUp() self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN) (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID, "Test RO for ROEVO", "Test Creator", "2012-09-06") self.CREATED_RO = rouri; return def tearDown(self): (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID+"/") (status, reason) = self.rosrs.deleteRO(self.CREATED_RO) super(TestEvoCommands, self).tearDown() return def testSnapshot(self): """ snapshot <live-RO> <snapshot-id> [ --asynchronous ] [ --freeze ] [ -t <access_token> ] [ -t <token> ] """ return def testSnapshotAsynchronous(self): args = [ "ro", "snapshot" , str(self.CREATED_RO), self.TEST_SNAPHOT_ID, "--asynchronous", "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-r", ro_test_config.ROSRS_URI, "-v" ] outLines = "" with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == 0 # simple check if the verbouse mode works well for word in ("ro snapshot --asynchronous "+ro_test_config.ROSRS_URI + self.TEST_RO_ID + " " + self.TEST_SNAPHOT_ID).split(" "): self.assertTrue(self.outstr.getvalue().count(word+ " ") or self.outstr.getvalue().count(" " + word), "snapshot command wasn't parse well") self.assertEqual(self.outstr.getvalue().count("Job Status: "), 1) self.assertEqual(self.outstr.getvalue().count("Job URI: "), 1) self.assertEqual(self.outstr.getvalue().count("Target URI: "), 1) self.assertEqual(self.outstr.getvalue().count("Target Name: "), 1) self.assertEqual(self.outstr.getvalue().count("Response Status: "), 1) self.assertEqual(self.outstr.getvalue().count("Response Reason: "), 1) outLines = self.outstr.getvalue().split("\n") for line in outLines: if "Job URI:" in line: jobLocation = line.split("Job URI:")[1].strip() status = "RUNNING" while status == "RUNNING": (status, id) = parse_job(self.rosrs, jobLocation) assert status == "DONE" self.rosrs.deleteRO(id) return def testSnapshotSynchronous(self): args = [ "ro", "snapshot", self.CREATED_RO, self.TEST_SNAPHOT_ID, "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-r", ro_test_config.ROSRS_URI, "-v" ] outLines = "" with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == 0 # simple check if the verbouse mode works well for word in ("ro snaphot "+ro_test_config.ROSRS_URI + self.TEST_RO_ID + " " + self.TEST_SNAPHOT_ID).split(" "): self.assertTrue(self.outstr.getvalue().count(word+ " ") or self.outstr.getvalue().count(" " + word), "snapshot command wasn't parse well") self.assertEqual(self.outstr.getvalue().count("Target URI: "), 1) outLines = self.outstr.getvalue().split("\n") for line in outLines: if "Target URI:" in line: id = line.split("Target URI:")[1].strip() self.rosrs.deleteRO(id) return def testSnapshotWithEscOption(self): args = [ "ro", "snapshot", self.CREATED_RO, self.TEST_SNAPHOT_ID, "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-r", ro_test_config.ROSRS_URI, "-v" ] outLines = "" with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == 0 # simple check if the verbouse mode works well self.assertEqual(self.outstr.getvalue().count("--asynchronous"), 0, "shouldn't be asynchronous") outLines = self.outstr.getvalue().split("\n") for line in outLines: if "Target URI:" in line: id = line.split("Target URI:")[1].strip() self.rosrs.deleteRO(id) return def testArchive(self): """ archive <live-RO> <snapshot-id> [ --asynchronous ] [ --freeze ] [ -t <access_token> ] [ -t <token> ] """ return def testArchiveAsynchronous(self): args = [ "ro", "archive" , str(self.CREATED_RO), self.TEST_SNAPHOT_ID, "--asynchronous", "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-r", ro_test_config.ROSRS_URI, "-v" ] outLines = "" with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == 0 # simple check if the verbouse mode works well for word in ("ro archive --asynchronous "+ro_test_config.ROSRS_URI + self.TEST_RO_ID + " " + self.TEST_SNAPHOT_ID).split(" "): self.assertTrue(self.outstr.getvalue().count(word+ " ") or self.outstr.getvalue().count(" " + word), "snapshot command wasn't parse well") self.assertEqual(self.outstr.getvalue().count("Job Status: "), 1) self.assertEqual(self.outstr.getvalue().count("Job URI: "), 1) self.assertEqual(self.outstr.getvalue().count("Target URI: "), 1) self.assertEqual(self.outstr.getvalue().count("Target Name: "), 1) self.assertEqual(self.outstr.getvalue().count("Response Status: "), 1) self.assertEqual(self.outstr.getvalue().count("Response Reason: "), 1) outLines = self.outstr.getvalue().split("\n") for line in outLines: if "Job URI:" in line: jobLocation = line.split("Job URI:")[1].strip() status = "RUNNING" while status == "RUNNING": (status, id) = parse_job(self.rosrs, jobLocation) assert status == "DONE" self.rosrs.deleteRO(id) return def testArchiveSynchronous(self): args = [ "ro", "archive", ro_test_config.ROSRS_URI + self.TEST_RO_ID, ro_test_config.ROSRS_URI + self.TEST_SNAPHOT_ID, "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-r", ro_test_config.ROSRS_URI, "-v" ] outLines = "" with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == 0 # simple check if the verbouse mode works well self.assertEqual(self.outstr.getvalue().count("--asynchronous"), 0, "shouldn't be asynchronous") outLines = self.outstr.getvalue().split("\n") for line in outLines: if "Target URI:" in line: id = line.split("Target URI:")[1].strip() self.rosrs.deleteRO(id) return def testFreeze(self): """ freeze <RO-id> """ #preapre snaphot (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID+"/") (status, reason, createdRoUri, manifest) = self.rosrs.createRO(self.TEST_RO_ID, "Test RO for ROEVO", "Test Creator", "2012-09-06") (createdSnapshotStatus, createdSnapshotId) = self.createSnapshot(createdRoUri, self.TEST_SNAPHOT_ID, False) args = [ "ro", "freeze",str(createdSnapshotId), "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-r", ro_test_config.ROSRS_URI, "-v" ] with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == 0 self.assertEqual(self.outstr.getvalue().count("freeze operation finished successfully"), 1) (status, reason) = self.rosrs.deleteRO(createdRoUri) (status, reason) = self.rosrs.deleteRO(createdSnapshotId) return def FreezeNonExistetSnaphot(self): """ freeze <RO-id> """ #preapre snaphot (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID+"/") (status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID, "Test RO for ROEVO", "Test Creator", "2012-09-06") self.createSnapshot(self.TEST_RO_ID+"/", self.TEST_SNAPHOT_ID, True) args = [ "ro", "freeze", self.TEST_SNAPHOT_RO_ID + "non exited", "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-r", ro_test_config.ROSRS_URI, "-v" ] with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == -1 self.assertEqual(self.outstr.getvalue().count("Given URI isn't correct"), 0) (status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID+"/") (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") return def testRemoteStatusSnapshotRO(self): self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN) (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason, createdRoUri, manifest) = self.rosrs.createRO(self.TEST_RO_ID, "Test RO for ROEVO", "Test Creator", "2012-09-06") (status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID + "/") (createdSnapshotStatus, createdSnapshotUri) = self.createSnapshot(createdRoUri, self.TEST_SNAPHOT_ID, True) args = [ "ro", "status", str(createdSnapshotUri), "-r", ro_test_config.ROSRS_URI, "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-v" ] with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == 0 outtxt = self.outstr.getvalue() self.assertEqual(outtxt.count("SNAPSHOT"), 1) (status, reason) = self.rosrs.deleteRO(createdSnapshotUri) (status, reason) = self.rosrs.deleteRO(createdRoUri) return def testRemoteStatusArchiveRO(self): self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN) (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason, createdRoUri, manifest) = self.rosrs.createRO(self.TEST_RO_ID, "Test RO for ROEVO", "Test Creator", "2012-09-06") (status, reason) = self.rosrs.deleteRO(self.TEST_ARCHIVE_ID + "/") (createdArchiveStatus, createdArchiveUri) = self.createArchive(createdRoUri, self.TEST_ARCHIVE_ID, True) args = [ "ro", "status", str(createdArchiveUri), "-r", ro_test_config.ROSRS_URI, "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-v" ] with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == 0 outtxt = self.outstr.getvalue() self.assertEqual(outtxt.count("ARCHIVE"), 1) (status, reason) = self.rosrs.deleteRO(createdArchiveUri) (status, reason) = self.rosrs.deleteRO(createdRoUri) return def testRemoteStatusUndefinedRO(self): self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN) (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason, createdRoUri, manifest) = self.rosrs.createRO(self.TEST_RO_ID, "Test RO for ROEVO", "Test Creator", "2012-09-06") (status, reason) = self.rosrs.deleteRO(self.TEST_ARCHIVE_ID + "/") (createdArchiveStatus, createdArchiveUri) = self.createArchive(createdRoUri, self.TEST_ARCHIVE_ID, False) args = [ "ro", "status", str(createdArchiveUri), "-r", ro_test_config.ROSRS_URI, "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-v" ] with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == 0 outtxt = self.outstr.getvalue() self.assertEqual(outtxt.count("UNDEFINED"), 1) (status, reason) = self.rosrs.deleteRO(createdArchiveUri) (status, reason) = self.rosrs.deleteRO(createdRoUri) return def testRemoteStatusLiveRO(self): self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN) (status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/") (status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID, "Test RO for ROEVO", "Test Creator", "2012-09-06") args = [ "ro", "status", str(rouri), "-r", ro_test_config.ROSRS_URI, "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-v" ] with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) assert status == 0 outtxt = self.outstr.getvalue() self.assertEqual(outtxt.count("LIVE"), 1) (status, reason) = self.rosrs.deleteRO(rouri) return def testRemoteStatusWithWrongUriGiven(self): self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN) self.rosrs.deleteRO(self.TEST_RO_ID + "/") self.rosrs.deleteRO("some-strange-uri/") args = [ "ro", "status", ro_test_config.ROSRS_URI + "some-strange-uri/", "-r", ro_test_config.ROSRS_URI, "-t", ro_test_config.ROSRS_ACCESS_TOKEN, "-v" ] with SwitchStdout(self.outstr): status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args) outtxt = self.outstr.getvalue() assert status == -1 self.assertEqual(outtxt.count("Wrong URI was given"), 1) self.rosrs.deleteRO(self.TEST_RO_ID + "/") return
def remote_status(self, ro_uri): rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN) service_uri = ro_uri (status, reason, headers, data) = rosrs.doRequest(uripath=service_uri, method="POST", body="", ctype="application/json", reqheaders={}) return status, reason, headers, data
<roterms:resource rdf:resource="&BUNDLE;workflow/Create_SNP_Set/in/Entrez_ID" /> </rdf:Description> </roterms:inputValue> <roterms:inputValue> <rdf:Description> <roterms:portName>set_width</roterms:portName> <roterms:resource rdf:resource="&BUNDLE;workflow/Create_SNP_Set/in/set_width" /> </rdf:Description> </roterms:inputValue> <roterms:inputValue> <rdf:Description> <roterms:portName>path_to_output_file</roterms:portName> <roterms:resource rdf:resource="&BUNDLE;workflow/Create_SNP_Set/in/path_to_output_file" /> </rdf:Description> </roterms:inputValue> </rdf:Description> </rdf:RDF> """ # Main program script if __name__ == "__main__": # Set up ROSRS session and key values rosrs = ROSRS_Session(ROSRS_API_URI, accesskey=ROSRS_ACCESS_TOKEN) rouri = getResourceUri(ROSRS_API_URI, RO_PATH) resuri = getResourceUri(rouri, RES_PATH) # Add annotation updateAnnotation(rosrs, rouri, resuri, ANNOTATION_PATH, ANNOTATION_BODY) # Finish up rosrs.close()
class ro_metadata(object): """ Class for accessing RO metadata """ def __init__(self, roconfig, roref, dummysetupfortest=False): """ Initialize: read manifest from object at given directory into local RDF graph roconfig is the research object manager configuration, supplied as a dictionary roref a URI reference that refers to the Research Object to be accessed, or relative path name (see ro_uriutils.resolveFileAsUri for interpretation) dummysetupfortest is an optional parameter that, if True, suppresses some aspects of the setup (does not attempt to read a RO manifest) for isolated testing. """ self.roconfig = roconfig self.roref = roref self.dummyfortest = dummysetupfortest self.manifestgraph = None self.roannotations = None self.registries = None uri = resolveFileAsUri(roref) if not uri.endswith("/"): uri += "/" self.rouri = rdflib.URIRef(uri) if self._isLocal(): self.rosrs = None else: self.rosrs = ROSRS_Session( self.roconfig["rosrs_uri"], self.roconfig["rosrs_access_token"] ) self._loadManifest() # Get RO URI from manifest # May be different from computed value if manifest has absolute URI self.rouri = self.manifestgraph.value(None, RDF.type, RO.ResearchObject) # Check that the manifest contained at least one RO URI assert self.rouri is not None return def _isLocal(self): return isFileUri(self.rouri) def _getManifestUri(self): assert self._isLocal() return self.getComponentUri(ro_settings.MANIFEST_DIR+"/"+ro_settings.MANIFEST_FILE) def _loadManifest(self): if self.manifestgraph: return self.manifestgraph if self.dummyfortest: # Fake minimal manifest graph for testing self.manifestgraph = rdflib.Graph() self.manifestgraph.add( (self.rouri, RDF.type, RO.ResearchObject) ) elif self._isLocal(): # Read manifest graph self.manifestgraph = rdflib.Graph() self.manifestgraph.parse(self._getManifestUri()) else: (status, reason, _h, _u, manifest) = self.rosrs.getROManifest(self.rouri) assert status == 200, ("ro_metadata: Can't access manifest for %s (%03d %s)"% (str(self.rouri), status, reason)) self.manifestgraph = manifest return self.manifestgraph def _updateManifest(self): """ Write updated manifest file for research object """ assert self._isLocal() self._loadManifest().serialize( destination=self.getManifestFilename(), format='xml', base=self.rouri, xml_base="..") return def _iterAnnotations(self, subject=None): """ Return iterator over annotation stubs in the current RO, either for the specified subject resource, or for all annotations in the RO subject is URI of subject whose annotations are returned, or None. """ manifest = self._loadManifest() if self._isLocal(): for (anode, p, subject) in manifest: if p in [RO.annotatesAggregatedResource, AO.annotatesResource]: yield anode else: for anode in self.rosrs.getROAnnotationUris(self.getRoUri(), subject): yield anode return def isAggregatedResource(self, rofile): ''' Returns true if the manifest says that the research object aggregates the resource. Resource URI is resolved against the RO URI unless it's absolute. ''' resuri = self.getComponentUriAbs(rofile) return (self.rouri, ORE.aggregates, resuri) in self.manifestgraph def _loadAnnotations(self): if self.roannotations: return self.roannotations log.debug("_loadannotations") # Assemble annotation graph # NOTE: the manifest itself is included as an annotation by the RO setup if self._isLocal(): manifest = self._loadManifest() self.roannotations = rdflib.Graph() annotation_uris_loaded = set() for anode in self._iterAnnotations(): auri = manifest.value(subject=anode, predicate=AO.body) if auri not in annotation_uris_loaded: self._readAnnotationBody(auri, self.roannotations) annotation_uris_loaded.add(auri) else: self.roannotations = self.rosrs.getROAnnotationGraph(self.rouri) log.debug("roannotations graph:\n"+self.roannotations.serialize()) return self.roannotations def isInternalResource(self, resuri): ''' Check if the resource is internal, i.e. should the resource content be uploaded to the ROSR service. Returns true if the resource URI has the RO URI as a prefix. ''' return resuri.startswith(self.rouri) def isExternalResource(self, resuri): ''' Check if the resource is external, i.e. can be aggregated as a URI reference. Returns true if the URI has 'http' or 'https' scheme. ''' parseduri = urlparse.urlsplit(resuri) return parseduri.scheme in ["http", "https"] def _createAnnotationBody(self, roresource, attrdict, defaultType="string"): """ Create a new annotation body for a single resource in a research object, based on a supplied graph value. Existing annotations for the same resource are not touched; if an annotation is being added or replaced, it is the calling program'sresponsibility to update the manifest to reference the active annotations. A new name is allocated for the created annotation, graph which is returned as the result of this function. roresource is the name of the Research Object component to be annotated, possibly relative to the RO root directory. attrdict is a dictionary of attributes to be saved in the annotation body. Dictionary keys are attribute names that can be resolved via ro_annotation.getAnnotationByName. Returns the name of the annotation body created relative to the RO directory. """ assert self._isLocal() af = ro_annotation.createAnnotationBody( self.roconfig, self.getRoFilename(), roresource, attrdict, defaultType) return os.path.join(ro_settings.MANIFEST_DIR+"/", af) def _createAnnotationGraphBody(self, roresource, anngraph): """ Create a new annotation body for a single resource in a research object, based on a supplied graph value. Existing annotations for the same resource are not touched; if an annotation is being added or replaced, it is the calling program'sresponsibility to update the manifest to reference the active annotations. A new name is allocated for the created annotation, graph which is returned as the result of this function. roresource is the name of the Research Object component to be annotated, possibly relative to the RO root directory. anngraph is an annotation graph that is to be saved. Returns the name of the annotation body created relative to the RO manifest and metadata directory. """ assert self._isLocal() af = ro_annotation.createAnnotationGraphBody( self.roconfig, self.getRoFilename(), roresource, anngraph) return os.path.join(ro_settings.MANIFEST_DIR+"/", af) def _readAnnotationBody(self, annotationref, anngr=None): """ Read annotation body from indicated resource, return RDF Graph of annotation values. annotationref is a URI reference of an annotation, possibly relative to the RO base URI (e.g. as returned by _createAnnotationBody method). anngr if supplied, if an RDF graph to which the annotations are added """ assert self._isLocal() log.debug("_readAnnotationBody %s"%(annotationref)) annotationuri = self.getComponentUri(annotationref) annotationformat = "xml" # Look at file extension to figure format # (rdflib.Graph.parse says; # "used if format can not be determined from the source") if re.search("\.(ttl|n3)$", annotationuri): annotationformat="n3" if anngr == None: log.debug("_readAnnotationBody: new graph") anngr = rdflib.Graph() try: anngr.parse(annotationuri, format=annotationformat) log.debug("_readAnnotationBody parse %s, len %i"%(annotationuri, len(anngr))) except IOError, e: log.debug("_readAnnotationBody "+annotationref+", "+repr(e)) anngr = None return anngr
class TestROSRS_Session(unittest.TestCase): """ This test suite tests the ROSRS_Session client implementation of the ROSRS API """ def setUp(self): super(TestROSRS_Session, self).setUp() self.rosrs = ROSRS_Session(Config.ROSRS_API_URI, accesskey=Config.AUTHORIZATION) # Clean up from previous runs self.rosrs.deleteRO(Config.TEST_RO_PATH, purge=True) self.createdTestRO = None return def tearDown(self): super(TestROSRS_Session, self).tearDown() # Clean up self.rosrs.deleteRO(Config.TEST_RO_PATH) if self.createdTestRO: self.rosrs.deleteRO(self.createdTestRO, purge=True) self.rosrs.close() return def createTestRO(self): (status, reason, rouri, manifest) = self.rosrs.createRO(Config.TEST_RO_NAME, "Test RO for ROSRS_Session", "TestROSRS_Session.py", "2012-09-06") self.assertEqual(status, 201) self.createdTestRO = rouri return (status, reason, rouri, manifest) # Actual tests follow def testHelpers(self): testSplitValues() testParseLinks() return def testListROs(self): ros = self.rosrs.listROs() return def testCreateRO(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) self.assertEqual(reason, "Created") self.assertEqual(str(rouri)[:len(Config.TEST_RO_URI)-1]+"/", Config.TEST_RO_URI) self.assertIn((rouri, RDF.type, RO.ResearchObject), manifest) rolist = self.rosrs.listROs() self.assertIn(str(rouri), [ r["uri"] for r in rolist ]) return def testDeleteRO(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Test that new RO is in collection rolist = self.rosrs.listROs() self.assertIn(str(rouri), [ r["uri"] for r in rolist ]) # Delete RO (status, reason) = self.rosrs.deleteRO(rouri) self.assertEqual(status, 204) self.assertEqual(reason, "No Content") # Test that new RO is not in collection rolist = self.rosrs.listROs() self.assertNotIn(str(rouri), [ r["uri"] for r in rolist ]) # Delete again (status, reason) = self.rosrs.deleteRO(rouri) self.assertEqual(status, 404) self.assertEqual(reason, "Not Found") return def testGetROManifest(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Get manifest (status, reason, headers, manifesturi, manifest) = self.rosrs.getROManifest(rouri) self.assertEqual(status, 200) self.assertEqual(reason, "OK") self.assertEqual(headers["content-type"], "application/rdf+xml") # Check manifest RDF graph self.assertIn((rouri, RDF.type, RO.ResearchObject), manifest) self.assertIn((rouri, DCTERMS.creator, None), manifest) self.assertIn((rouri, DCTERMS.created, None), manifest) self.assertIn((rouri, ORE.isDescribedBy, manifesturi), manifest) return def testGetROPage(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Get landing page (status, reason, headers, pageuri, page) = self.rosrs.getROLandingPage(rouri) self.assertEqual(status, 200) self.assertEqual(reason, "OK") self.assertEqual(headers["content-type"], "text/html;charset=UTF-8") return def testGetROZip(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Get manifest (status, reason, headers, datauri, data) = self.rosrs.getROZip(rouri) self.assertEqual(status, 200) self.assertEqual(reason, "OK") self.assertEqual(headers["content-type"], "application/zip") # @@TODO test content of zip (data)? return def testAggregateResourceInt(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Aggregate internal resource rescontent = "Resource content\n" (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt( rouri, "test/path", ctype="text/plain", body=rescontent) self.assertEqual(status, 201) self.assertEqual(reason, "Created") self.assertEqual(str(resuri), str(rouri)+"test/path") # GET content (status, reason, headers, uri, data) = self.rosrs.getROResource( "test/path", rouri) self.assertEqual(status, 200) self.assertEqual(headers["content-type"], "text/plain") self.assertEqual(data, rescontent) # GET proxy (getproxyuri, manifest) = self.rosrs.getROResourceProxy( "test/path", rouri=rouri) self.assertEqual(getproxyuri, proxyuri) return def testDeleteResourceInt(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Create test resource rescontent = "Resource content\n" (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt( rouri, "test/path", ctype="text/plain", body=rescontent) self.assertEqual(status, 201) # GET content (status, reason, headers, uri, data) = self.rosrs.getROResource( "test/path", rouri) self.assertEqual(status, 200) # Delete resource (status, reason) = self.rosrs.removeResource(rouri, resuri) self.assertEqual(status, 204) self.assertEqual(reason, "No Content") # Check that resource is no longer available (status, reason, headers, uri, data) = self.rosrs.getROResource(resuri) self.assertEqual(status, 404) return def testAggregateResourceExt(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Aggregate external resource externaluri = rdflib.URIRef("http://example.com/external/resource.txt") (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceExt( rouri, externaluri) self.assertEqual(status, 201) self.assertEqual(reason, "Created") self.assertEqual(resuri, externaluri) # GET proxy (note: using rdflib.URIRef value for path) (getproxyuri, manifest) = self.rosrs.getROResourceProxy( externaluri, rouri) self.assertEqual(getproxyuri, proxyuri) return def testDeleteResourceExt(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Create test resource externaluri = rdflib.URIRef("http://example.com/external/resource.txt") (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceExt( rouri, externaluri) self.assertEqual(status, 201) self.assertEqual(resuri, externaluri) # GET proxy (note: using rdflib.URIRef for path) (getproxyuri, manifest) = self.rosrs.getROResourceProxy( externaluri, rouri) self.assertEqual(getproxyuri, proxyuri) # Delete resource (status, reason) = self.rosrs.removeResource(rouri, resuri) self.assertEqual(status, 204) self.assertEqual(reason, "No Content") (getproxyuri, manifest) = self.rosrs.getROResourceProxy( externaluri, rouri) self.assertIsNone(getproxyuri) self.assertIsNotNone(manifest) return def testGetROResource(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Create test resource rescontent = "Resource content\n" (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt( rouri, "test/path", ctype="text/plain", body=rescontent) self.assertEqual(status, 201) # GET content (status, reason, headers, uri, data) = self.rosrs.getROResource( "test/path", rouri) self.assertEqual(status, 200) self.assertEqual(reason, "OK") self.assertEqual(headers["content-type"], "text/plain") self.assertEqual(data, rescontent) return def testGetROResourceRDF(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Create internal test resource rescontent = """<?xml version="1.0" encoding="UTF-8"?> <rdf:RDF xmlns:dct="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" > <rdf:Description rdf:about="http://example.org/file1.txt"> <dct:title>Title for file1.txt</dct:title> </rdf:Description> </rdf:RDF> """ (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt( rouri, "test/file1.rdf", ctype="application/rdf+xml", body=rescontent) self.assertEqual(status, 201) # Get resource content (status, reason, headers, uri, graph)= self.rosrs.getROResourceRDF( "test/file1.rdf", rouri=rouri) self.assertEqual(status, 200) self.assertEqual(reason, "OK") self.assertEqual(headers["content-type"], "application/rdf+xml") s = rdflib.URIRef("http://example.org/file1.txt") self.assertIn((s, DCTERMS.title, rdflib.Literal("Title for file1.txt")), graph) return def testGetROResourceProxy(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Create internal test resource rescontent = "Resource content\n" (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt( rouri, "test/path", ctype="text/plain", body=rescontent) self.assertEqual(status, 201) # Get resource proxy (getproxyuri, manifest) = self.rosrs.getROResourceProxy( "test/path", rouri=rouri) self.assertEqual(getproxyuri, proxyuri) return def testCreateROAnnotationInt(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Create internal test resource rescontent = "Resource content\n" (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt( rouri, "test/file.txt", ctype="text/plain", body=rescontent) self.assertEqual(status, 201) # Create internal annotation annbody = """<?xml version="1.0" encoding="UTF-8"?> <rdf:RDF xmlns:dct="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xml:base="%s" > <rdf:Description rdf:about="test/file.txt"> <dct:title>Title for test/file.txt</dct:title> <rdfs:seeAlso rdf:resource="http://example.org/test" /> </rdf:Description> </rdf:RDF> """%(str(rouri)) agraph = rdflib.graph.Graph() agraph.parse(data=annbody, format="xml") (status, reason, annuri, bodyuri) = self.rosrs.createROAnnotationInt( rouri, resuri, agraph) self.assertEqual(status, 201) self.assertEqual(reason, "Created") # Retrieve annotation URIs auris = list(self.rosrs.getROAnnotationUris(rouri, resuri)) self.assertIn(annuri, auris) buris = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri)) ### self.assertIn(bodyuri, buris) # Retrieve annotation (status, reason, bodyuri, anngr) = self.rosrs.getROAnnotation(annuri) self.assertEqual(status, 200) self.assertEqual(reason, "OK") self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title for test/file.txt")), anngr) self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test")), anngr) return def testGetROAnnotationGraph(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Create internal test resource rescontent = "Resource content\n" (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt( rouri, "test/file.txt", ctype="text/plain", body=rescontent) self.assertEqual(status, 201) # Create internal annotation annbody = """<?xml version="1.0" encoding="UTF-8"?> <rdf:RDF xmlns:dct="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xml:base="%s" > <rdf:Description rdf:about="test/file.txt"> <dct:title>Title for test/file.txt</dct:title> <rdfs:seeAlso rdf:resource="http://example.org/test" /> </rdf:Description> </rdf:RDF> """%(str(rouri)) agraph = rdflib.graph.Graph() agraph.parse(data=annbody, format="xml") (status, reason, annuri, bodyuri) = self.rosrs.createROAnnotationInt( rouri, resuri, agraph) self.assertEqual(status, 201) self.assertEqual(reason, "Created") # Retrieve merged annotations anngr = self.rosrs.getROAnnotationGraph(rouri, resuri) annts = list(anngr.triples((None, None, None))) self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title for test/file.txt")), annts) self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test")), annts) return def testCreateROAnnotationExt(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Create external test resource (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceExt( rouri, rdflib.URIRef("http://example.org/ext")) self.assertEqual(status, 201) # Create annotation using external body reference bodyuri = rdflib.URIRef("http://example.org/ext/ann.rdf") (status, reason, annuri) = self.rosrs.createROAnnotationExt(rouri, resuri, bodyuri) self.assertEqual(status, 201) self.assertEqual(reason, "Created") # Retrieve annotation URIs auris = list(self.rosrs.getROAnnotationUris(rouri, resuri)) self.assertIn(annuri, auris) buris = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri)) ### self.assertIn(bodyuri, buris) return def testUpdateROAnnotationInt(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Create internal test resource rescontent = "Resource content\n" (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt( rouri, "test/file.txt", ctype="text/plain", body=rescontent) self.assertEqual(status, 201) # Create internal annotation annbody1 = """<?xml version="1.0" encoding="UTF-8"?> <rdf:RDF xmlns:dct="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xml:base="%s" > <rdf:Description rdf:about="test/file.txt"> <dct:title>Title 1</dct:title> <rdfs:seeAlso rdf:resource="http://example.org/test1" /> </rdf:Description> </rdf:RDF> """%(str(rouri)) agraph1 = rdflib.graph.Graph() agraph1.parse(data=annbody1, format="xml") (status, reason, annuri, bodyuri1) = self.rosrs.createROAnnotationInt( rouri, resuri, agraph1) self.assertEqual(status, 201) self.assertEqual(reason, "Created") # Retrieve annotation URIs auris1 = list(self.rosrs.getROAnnotationUris(rouri, resuri)) self.assertIn(annuri, auris1) buris1 = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri)) ### self.assertIn(bodyuri1, buris1) # Retrieve annotation (status, reason, auri1, anngr1a) = self.rosrs.getROAnnotation(annuri) self.assertEqual(status, 200) self.assertEqual(reason, "OK") annts1a = list(anngr1a.triples((None, None, None))) self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title 1")), annts1a) self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test1")), annts1a) # Retrieve merged annotations anngr1b = self.rosrs.getROAnnotationGraph(rouri, resuri) annts1b = list(anngr1b.triples((None, None, None))) self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title 1")), annts1b) self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test1")), annts1b) # Update internal annotation annbody2 = """<?xml version="1.0" encoding="UTF-8"?> <rdf:RDF xmlns:dct="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xml:base="%s" > <rdf:Description rdf:about="test/file.txt"> <dct:title>Title 2</dct:title> <rdfs:seeAlso rdf:resource="http://example.org/test2" /> </rdf:Description> </rdf:RDF> """%(str(rouri)) agraph2 = rdflib.graph.Graph() agraph2.parse(data=annbody2, format="xml") (status, reason, bodyuri2) = self.rosrs.updateROAnnotationInt( rouri, annuri, resuri, agraph2) self.assertEqual(status, 200) self.assertEqual(reason, "OK") # Retrieve annotation URIs auris2 = list(self.rosrs.getROAnnotationUris(rouri, resuri)) self.assertIn(annuri, auris2) buris2 = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri)) ### self.assertIn(bodyuri2, buris2) # Retrieve annotation (status, reason, auri2a, anngr2a) = self.rosrs.getROAnnotation(annuri) annts2a = list(anngr2a.triples((None, None, None))) self.assertEqual(status, 200) self.assertEqual(reason, "OK") self.assertNotIn((resuri, DCTERMS.title, rdflib.Literal("Title 1")), annts2a) self.assertNotIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test1")), annts2a) self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title 2")), annts2a) self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test2")), annts2a) # Retrieve merged annotations anngr2b = self.rosrs.getROAnnotationGraph(rouri, resuri) annts2b = list(anngr2b.triples((None, None, None))) self.assertNotIn((resuri, DCTERMS.title, rdflib.Literal("Title 1")), annts2b) self.assertNotIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test1")), annts2b) self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title 2")), annts2b) self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test2")), annts2b) return def testUpdateROAnnotationExt(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Create external test resource (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceExt( rouri, rdflib.URIRef("http://example.org/ext")) self.assertEqual(status, 201) # Create annotation using external body reference bodyuri1 = rdflib.URIRef("http://example.org/ext/ann1.rdf") (status, reason, annuri) = self.rosrs.createROAnnotationExt(rouri, resuri, bodyuri1) self.assertEqual(status, 201) self.assertEqual(reason, "Created") # Retrieve annotation URIs auris1 = list(self.rosrs.getROAnnotationUris(rouri, resuri)) self.assertIn(annuri, auris1) buris1 = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri)) self.assertIn(bodyuri1, buris1) # Update annotation using external body reference # @@TODO - this doesn't check that old annotation is removed. # @@TODO - currently, update is not fully implemented (2013-05). bodyuri2 = rdflib.URIRef("http://example.org/ext/ann2.rdf") (status, reason, annuri) = self.rosrs.createROAnnotationExt(rouri, resuri, bodyuri2) self.assertEqual(status, 201) self.assertEqual(reason, "Created") # Retrieve annotation URIs auris2 = list(self.rosrs.getROAnnotationUris(rouri, resuri)) self.assertIn(annuri, auris2) buris2 = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri)) self.assertIn(bodyuri1, buris2) return def testRemoveROAnnotation(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) # Create internal test resource rescontent = "Resource content\n" (status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt( rouri, "test/file.txt", ctype="text/plain", body=rescontent) self.assertEqual(status, 201) # Create internal annotation annbody = """<?xml version="1.0" encoding="UTF-8"?> <rdf:RDF xmlns:dct="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xml:base="%s" > <rdf:Description rdf:about="test/file.txt"> <dct:title>Title for test/file.txt</dct:title> <rdfs:seeAlso rdf:resource="http://example.org/test" /> </rdf:Description> </rdf:RDF> """%(str(rouri)) agraph = rdflib.graph.Graph() agraph.parse(data=annbody, format="xml") (status, reason, annuri, bodyuri) = self.rosrs.createROAnnotationInt( rouri, resuri, agraph) self.assertEqual(status, 201) self.assertEqual(reason, "Created") # Retrieve annotation URIs auris = list(self.rosrs.getROAnnotationUris(rouri, resuri)) self.assertIn(annuri, auris) buris = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri)) ### self.assertIn(bodyuri, buris) # Remove the annotation (status, reason) = self.rosrs.removeROAnnotation(rouri, annuri) self.assertEqual(status, 204) self.assertEqual(reason, "No Content") # Retrieve annotation URIs auris = list(self.rosrs.getROAnnotationUris(rouri, resuri)) self.assertNotIn(annuri, auris) buris = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri)) ### self.assertNotIn(bodyuri, buris) return # Evolution tests def testCopyRO(self): return def testCancelCopyRO(self): return def testUpdateROStatus(self): return def testGetROEvolution(self): return # Sentinel/placeholder tests def testUnits(self): assert (True) def testComponents(self): assert (True) def testIntegration(self): assert (True) def testPending(self): assert (False), "Pending tests follow"
class ro_metadata(object): """ Class for accessing RO metadata """ def __init__(self, roconfig, roref, dummysetupfortest=False): """ Initialize: read manifest from object at given directory into local RDF graph roconfig is the research object manager configuration, supplied as a dictionary roref a URI reference that refers to the Research Object to be accessed, or relative path name (see ro_uriutils.resolveFileAsUri for interpretation) dummysetupfortest is an optional parameter that, if True, suppresses some aspects of the setup (does not attempt to read a RO manifest) for isolated testing. """ self.roconfig = roconfig self.roref = roref self.dummyfortest = dummysetupfortest self.manifestgraph = None self.roannotations = None self.registries = None uri = resolveFileAsUri(roref) if not uri.endswith("/"): uri += "/" self.rouri = rdflib.URIRef(uri) if self._isLocal(): self.rosrs = None else: self.rosrs = ROSRS_Session(self.roconfig["rosrs_uri"], self.roconfig["rosrs_access_token"]) self._loadManifest() # Get RO URI from manifest # May be different from computed value if manifest has absolute URI # Nested URIs may be present; ours is the one described by the manifest URI, # which is determined by the _loadManifest() method. for s in self.manifestgraph.subjects(RDF.type, RO.ResearchObject): if self.manifestgraph.value(s, ORE.isDescribedBy) == self.manifesturi: self.rouri = s # Check that the manifest contained at least one RO URI assert self.rouri is not None return def _isLocal(self): return isFileUri(self.rouri) def _getLocalManifestUri(self): return self.getComponentUri(ro_settings.MANIFEST_DIR + "/" + ro_settings.MANIFEST_FILE) def _loadManifest(self): if self.manifestgraph != None: return self.manifestgraph if self.dummyfortest: # Fake minimal manifest graph for testing self.manifestgraph = rdflib.Graph() self.manifestgraph.add((self.rouri, RDF.type, RO.ResearchObject)) self.manifesturi = self.rouri elif self._isLocal(): # Read manifest graph self.manifestgraph = rdflib.Graph() for (prefix, uri) in ro_prefixes.prefixes: self.manifestgraph.bind(prefix, rdflib.namespace.Namespace(uri)) self.manifesturi = self._getLocalManifestUri() self.manifestgraph.parse(self.manifesturi) else: (status, reason, _h, manifesturi, manifest) = self.rosrs.getROManifest(self.rouri) if status != 200: msg = ("Can't access RO manifest (%03d %s)" % (status, reason)) raise ROSRS_Error(msg=msg, srsuri=self.rouri) self.manifestgraph = manifest self.manifesturi = manifesturi # log.debug("romanifest graph:\n"+self.manifestgraph.serialize()) return self.manifestgraph def getManifestGraph(self): """ Returns the manifest graph """ return self._loadManifest() def _updateManifest(self): """ Write updated manifest file for research object """ assert self._isLocal() self._loadManifest().serialize(destination=self.getManifestFilename(), format='xml', base=self.rouri, xml_base="..") return def _iterAnnotations(self, subject=None): """ Return iterator over annotation stubs in the current RO, either for the specified subject resource, or for all annotations in the RO subject is URI of subject whose annotations are returned, or None. """ manifest = self._loadManifest() if self._isLocal(): for (anode, p, subject) in manifest: if p in [RO.annotatesAggregatedResource, AO.annotatesResource]: yield anode else: for anode in self.rosrs.getROAnnotationUris( self.getRoUri(), subject): yield anode return def isAggregatedResource(self, rofile): ''' Returns true if the manifest says that the research object aggregates the resource. Resource URI is resolved against the RO URI unless it's absolute. ''' resuri = self.getComponentUriAbs(rofile) return (self.rouri, ORE.aggregates, resuri) in self.manifestgraph def _loadAnnotations(self): if self.roannotations: return self.roannotations log.debug("_loadannotations") # Assemble annotation graph # NOTE: the manifest itself is included as an annotation by the RO setup if self._isLocal(): manifest = self._loadManifest() self.roannotations = rdflib.Graph() annotation_uris_loaded = set() for anode in self._iterAnnotations(): auri = manifest.value(subject=anode, predicate=AO.body) if auri not in annotation_uris_loaded: aref = self.getComponentUriRel(auri) log.debug("_loadAnnotations: aref " + str(aref)) self._readAnnotationBody(aref, self.roannotations) annotation_uris_loaded.add(auri) else: self.roannotations = self.rosrs.getROAnnotationGraph(self.rouri) # log.debug("roannotations graph:\n"+self.roannotations.serialize()) for (prefix, uri) in ro_prefixes.prefixes: self.manifestgraph.bind(prefix, rdflib.namespace.Namespace(uri)) return self.roannotations def isInternalResource(self, resuri): ''' Check if the resource is internal, i.e. should the resource content be uploaded to the ROSR service. Returns true if the resource URI has the RO URI as a prefix. ''' return resuri.startswith(self.rouri) def isExternalResource(self, resuri): ''' Check if the resource is external, i.e. can be aggregated as a URI reference. Returns true if the URI has 'http' or 'https' scheme. ''' parseduri = urlparse.urlsplit(resuri) return parseduri.scheme in ["http", "https"] def _createAnnotationBody(self, roresource, attrdict, defaultType="string"): """ Create a new annotation body for a single resource in a research object, based on a supplied graph value. Existing annotations for the same resource are not touched; if an annotation is being added or replaced, it is the calling program'sresponsibility to update the manifest to reference the active annotations. A new name is allocated for the created annotation, graph which is returned as the result of this function. roresource is the name of the Research Object component to be annotated, possibly relative to the RO root directory. attrdict is a dictionary of attributes to be saved in the annotation body. Dictionary keys are attribute names that can be resolved via ro_annotation.getAnnotationByName. Returns the name of the annotation body created relative to the RO directory. """ assert self._isLocal() af = ro_annotation.createAnnotationBody(self.roconfig, self.getRoFilename(), roresource, attrdict, defaultType) return os.path.join(ro_settings.MANIFEST_DIR + "/", af) def _createAnnotationGraphBody(self, roresource, anngraph): """ Create a new annotation body for a single resource in a research object, based on a supplied graph value. Existing annotations for the same resource are not touched; if an annotation is being added or replaced, it is the calling program'sresponsibility to update the manifest to reference the active annotations. A new name is allocated for the created annotation, graph which is returned as the result of this function. roresource is the name of the Research Object component to be annotated, possibly relative to the RO root directory. anngraph is an annotation graph that is to be saved. Returns the name of the annotation body created relative to the RO manifest and metadata directory. """ assert self._isLocal() af = ro_annotation.createAnnotationGraphBody(self.roconfig, self.getRoFilename(), roresource, anngraph) return os.path.join(ro_settings.MANIFEST_DIR + "/", af) def _readAnnotationBody(self, annotationref, anngr=None): """ Read annotation body from indicated resource, return RDF Graph of annotation values. annotationref is a URI reference of an annotation, possibly relative to the RO base URI (e.g. as returned by _createAnnotationBody method). anngr if supplied, if an RDF graph to which the annotations are added """ assert self._isLocal() log.debug("_readAnnotationBody %s" % (annotationref)) annotationuri = self.getComponentUri(annotationref) annotationformat = "xml" # Look at file extension to figure format # (rdflib.Graph.parse says; # "used if format can not be determined from the source") if re.search("\.(ttl|n3)$", annotationuri): annotationformat = "n3" if anngr == None: log.debug("_readAnnotationBody: new graph") anngr = rdflib.Graph() try: anngr.parse(annotationuri, format=annotationformat) log.debug("_readAnnotationBody parse %s, len %i" % (annotationuri, len(anngr))) except IOError as e: log.debug("_readAnnotationBody %s, %s" % (str(annotationref), repr(e))) anngr = None except Exception as e: log.debug("Failed to load annotation %s as %s" % (annotationuri, annotationformat)) log.debug("Exception %s" % (repr(e))) raise anngr = None return anngr def _addAnnotationToManifest(self, rofile, annfile): """ Add a new annotation body to an RO graph rofile is the research object resource being annotated annfile is the file name of the annotation body to be added, possibly relative to the RO URI, with special characters already URI-escaped. """ assert self._isLocal() # <ore:aggregates> # <ro:AggregatedAnnotation> # <ro:annotatesAggregatedResource rdf:resource="data/UserRequirements-astro.ods" /> # <ao:body rdf:resource=".ro/(annotation).rdf" /> # </ro:AggregatedAnnotation> # </ore:aggregates> log.debug("_addAnnotationToManifest annfile %s" % (annfile)) ann = rdflib.BNode() resuri = self.getComponentUri(rofile) bodyuri = self.getComponentUriAbs(annfile) self.manifestgraph.add((ann, RDF.type, RO.AggregatedAnnotation)) self.manifestgraph.add((ann, RO.annotatesAggregatedResource, resuri)) self.manifestgraph.add((ann, AO.body, bodyuri)) # Aggregate the annotation self.manifestgraph.add((self.getRoUri(), ORE.aggregates, ann)) # Aggregate annotation body if it is RO metadata. # Otherwise aggregation is the caller's responsibility if self.isRoMetadataRef(bodyuri): self.manifestgraph.add((self.getRoUri(), ORE.aggregates, bodyuri)) self.roannotations = None # Flush cached annotation graph return def _removeAnnotationFromManifest(self, ann): """ Remove references to an annotation from an RO graph ann is the the annotation node to be removed """ assert self._isLocal() bodyuri = self.manifestgraph.value(subject=ann, predicate=AO.body) self.manifestgraph.remove((ann, None, None)) # If annotation body is RO Metadata, and there are no other uses as an annotation, # remove it from the RO aggregation. if self.isRoMetadataRef(bodyuri): if not self.manifestgraph.value(subject=ann, predicate=AO.body): self.manifestgraph.remove((None, ORE.aggregates, bodyuri)) self.roannotations = None # Flush cached annotation graph return def addAggregatedResources(self, ro_file, recurse=True, includeDirs=False): """ Scan a local directory and add files found to the RO aggregation """ assert self._isLocal() def notHidden(f): return re.match("\.|.*/\.", f) == None log.debug("addAggregatedResources: roref %s, file %s" % (self.roref, ro_file)) self.getRoFilename() # Check that we have one basedir = os.path.abspath(self.roref) + os.path.sep ### print "- ro_file: %s"%(ro_file) if os.path.isdir(ro_file): ro_file = os.path.abspath(ro_file) + os.path.sep ### print "- ro_file: %s"%(ro_file) ### print "- basedir: %s"%(basedir) #if ro_file.endswith(os.path.sep): # ro_file = ro_file[0:-1] if recurse: rofiles = filter( notHidden, MiscUtils.ScanDirectories.CollectDirectoryContents( ro_file, baseDir=basedir, listDirs=includeDirs, listFiles=True, recursive=recurse, appendSep=True)) log.debug("- rofiles: %s" % (repr(rofiles))) else: rofiles = [ro_file.split(basedir + os.path.sep, 1)[-1]] else: rofiles = [self.getComponentUriRel(ro_file)] s = self.getRoUri() for f in rofiles: ### print "- file %s"%f log.debug("- file %s" % f) stmt = (s, ORE.aggregates, self.getComponentUri(f)) if stmt not in self.manifestgraph: self.manifestgraph.add(stmt) self._updateManifest() return def removeAggregatedResource(self, resuri): """ Remove a specified resource. resref is the URI of a resource to remove """ assert self._isLocal() resuri = rdflib.URIRef(resuri) log.debug("removeAggregatedResource: roref %s, resuri %s" % (self.roref, str(resuri))) manifest = self._loadManifest() for anode in self._iterAnnotations(subject=resuri): self._removeAnnotationFromManifest(anode) manifest.remove((None, ORE.aggregates, resuri)) self._updateManifest() return def getAggregatedResources(self): """ Returns iterator over all resources aggregated by a manifest. Each value returned by the iterator is an aggregated resource URI """ log.debug("getAggregatedResources: uri %s" % (self.rouri)) for r in self._loadManifest().objects(subject=self.rouri, predicate=ORE.aggregates): if not isinstance(r, rdflib.BNode): yield r return def addGraphAnnotation(self, rofile, graph): """ Add an annotation graph for a named resource. Unlike addSimpleAnnotation, this method adds an annotation to the manifest using an existing RDF graph (which is presumably itself in the RO structure). rofile names the file or resource to be annotated, possibly relative to the RO. Note that no checks are performed to ensure that the graph itself actually refers to this resource - that's up the the creator of the graph. This identifies the resourfce with which the annotation body is associated in the RO manifest. graph names the file or resource containing the annotation body. """ assert self._isLocal() ro_graph = self._loadManifest() self._addAnnotationToManifest(rofile, graph) self._updateManifest() return def isAnnotationNode(self, respath): ''' Returns true if the manifest says that the research object aggregates the annotation and it is an ro:AggregatedAnnotation. Resource URI is resolved against the RO URI unless it's absolute. ''' resuri = self.getComponentUriAbs(respath) log.debug("isAnnotationNode: ro uri %s res uri %s" % (self.rouri, resuri)) return (self.rouri, ORE.aggregates, resuri) in self.manifestgraph and \ (resuri, RDF.type, RO.AggregatedAnnotation) in self.manifestgraph def addSimpleAnnotation(self, rofile, attrname, attrvalue, defaultType="string"): """ Add a simple annotation to a file in a research object. rofile names the file or resource to be annotated, possibly relative to the RO. attrname names the attribute in a form recognized by getAnnotationByName attrvalue is a value to be associated with the attribute """ assert self._isLocal() ro_dir = self.getRoFilename() annfile = self._createAnnotationBody(rofile, {attrname: attrvalue}, defaultType) self._addAnnotationToManifest(rofile, annfile) self._updateManifest() return annfile def removeSimpleAnnotation(self, rofile, attrname, attrvalue): """ Remove a simple annotation or multiple matching annotations a research object. rofile names the annotated file or resource, possibly relative to the RO. attrname names the attribute in a form recognized by getAnnotationByName attrvalue is the attribute value to be deleted, or Nomne to delete all vaues """ assert self._isLocal() ro_dir = self.getRoFilename() ro_graph = self._loadManifest() subject = self.getComponentUri(rofile) (predicate, valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname) val = attrvalue and ro_annotation.makeAnnotationValue( self.roconfig, attrvalue, valtype) add_annotations = [] remove_annotations = [] log.debug("removeSimpleAnnotation subject %s, predicate %s, val %s" % (str(subject), str(predicate), val)) # Scan for annotation graph resources containing this annotation for ann_node in self._iterAnnotations(subject=subject): ann_uri = ro_graph.value(subject=ann_node, predicate=AO.body) log.debug("removeSimpleAnnotation ann_uri %s" % (str(ann_uri))) if self.isRoMetadataRef(ann_uri): ann_graph = self._readAnnotationBody( self.getComponentUriRel(ann_uri)) log.debug("removeSimpleAnnotation ann_graph %s" % (ann_graph)) if (subject, predicate, val) in ann_graph: ann_graph.remove((subject, predicate, val)) if (subject, None, None) in ann_graph: # Triples remain in annotation body: write new body and update RO graph ann_name = self._createAnnotationGraphBody( rofile, ann_graph) remove_annotations.append(ann_node) add_annotations.append(ann_name) else: # Remove annotation from RO graph remove_annotations.append(ann_node) # Update RO manifest graph if needed if add_annotations or remove_annotations: for a in remove_annotations: self._removeAnnotationFromManifest(a) for a in add_annotations: self._addAnnotationToManifest(rofile, a) self._updateManifest() return def replaceSimpleAnnotation(self, rofile, attrname, attrvalue): """ Replace a simple annotation in a research object. rofile names the file or resource to be annotated, possibly relative to the RO. attrname names the attribute in a form recognized by getAnnotationByName attrvalue is a new value to be associated with the attribute """ assert self._isLocal() ro_dir = self.getRoFilename() ro_graph = self._loadManifest() subject = self.getComponentUri(rofile) (predicate, valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname) log.debug("Replace annotation: subject %s, predicate %s, value %s" % (repr(subject), repr(predicate), repr(attrvalue))) ro_graph.remove((subject, predicate, None)) ro_graph.add( (subject, predicate, ro_annotation.makeAnnotationValue(self.roconfig, attrvalue, valtype))) self._updateManifest() self.roannotations = None # Flush cached annotation graph return def iterateAnnotations(self, subject=None, property=None): """ Returns an iterator over annotations of the current RO that match the supplied subject and/or property. """ log.debug("iterateAnnotations s:%s, p:%s" % (str(subject), str(property))) ann_graph = self._loadAnnotations() for (s, p, v) in ann_graph.triples((subject, property, None)): if not isinstance(s, rdflib.BNode): if not self.isRoMetadataRef(s): log.debug("Triple: %s %s %s" % (s, p, v)) yield (s, p, v) return def getRoAnnotations(self): """ Returns iterator over annotations applied to the RO as an entity. Each value returned by the iterator is a (subject,predicate,object) triple. """ return self.iterateAnnotations(subject=self.getRoUri()) def getFileAnnotations(self, rofile): """ Returns iterator over annotations applied to a specified component in the RO Each value returned by the iterator is a (subject,predicate,object) triple. """ return self.iterateAnnotations(subject=self.getComponentUri(rofile)) def getAllAnnotations(self): """ Returns iterator over all annotations associated with the RO Each value returned by the iterator is a (subject,predicate,object) triple. """ return self.iterateAnnotations() def getAllAnnotationNodes(self): """ Returns iterator over all annotations aggregated within the RO Each value returned by the iterator is a (annuri, bodyuri, target) triple. """ annotations = self.manifestgraph.subject_objects( predicate=RO.annotatesAggregatedResource) for (ann_node, ann_target) in annotations: ann_body = self.manifestgraph.value(subject=ann_node, predicate=AO.body) yield (ann_node, ann_body, ann_target) return def getAnnotationValues(self, rofile, attrname): """ Returns iterator over annotation values for given subject and attribute """ (predicate, valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname) resuri = self.getComponentUri(rofile) return (v for ( s, p, v) in self.iterateAnnotations(subject=resuri, property=predicate)) def queryAnnotations(self, query, initBindings={}): """ Runs a query over the combined annotation graphs (including the manifest) and returns True or False (for ASK queries) or a list of dictionaries of query results (for SELECT queries). """ log.debug("queryAnnotations: \n----\n%s\n--------\n" % (query)) ann_gr = self._loadAnnotations() # log.debug("queryAnnotations graph: \n----\n%s\n--------\n"%(ann_gr.serialize(format='xml'))) try: resp = ann_gr.query(query, initBindings=initBindings) except: log.info("queryAnnotations failed query: \n----\n%s\n--------\n" % (query)) raise if resp.type == 'ASK': return resp.askAnswer elif resp.type == 'SELECT': return resp.bindings else: assert False, "Unexpected query response type %s" % resp.type return None def getAnnotationGraph(self): """ Returns the combined annotation graphs (including the manifest) """ return self._loadAnnotations() def getAnnotationValue(self, resource, predicate): """ Returns a single annotation value for a resource and the indicated predicate, or None """ return self._loadAnnotations().value(subject=resource, predicate=predicate, object=None) def showAnnotations(self, annotations, outstr): ro_annotation.showAnnotations(self.roconfig, self.getRoFilename(), annotations, outstr) return def replaceUri(self, ann_node, remote_ann_node_uri): for (p, o) in self.manifestgraph.predicate_objects(subject=ann_node): self.manifestgraph.remove((ann_node, p, o)) self.manifestgraph.add((remote_ann_node_uri, p, o)) for (s, p) in self.manifestgraph.subject_predicates(object=ann_node): self.manifestgraph.remove((s, p, ann_node)) self.manifestgraph.add((s, p, remote_ann_node_uri)) self._updateManifest() return # Support methods for accessing the manifest graph def roManifestContains(self, stmt): """ Returns True if the RO manifest contains a statement matching the supplied triple. """ return stmt in self._loadManifest() def getResourceValue(self, resource, predicate): """ Returns value for resource whose URI is supplied assocfiated with indicated predicate """ return self._loadManifest().value(subject=resource, predicate=predicate, object=None) def getResourceType(self, resource): """ Returns type of resource whose URI is supplied """ return self.getResourceValue(resource, RDF.type) def hasResourceType(self, resource, rdfType): """ Check if the resource whose URI is supplied has a provided RDF type. """ return self.roManifestContains((resource, RDF.type, rdfType)) def getRoMetadataDict(self): """ Returns dictionary of metadata about the RO from the manifest graph """ assert self._isLocal() strsubject = "" if isinstance(self.rouri, rdflib.URIRef): strsubject = str(self.rouri) manifestDict = { 'ropath': self.getRoFilename(), 'rouri': strsubject, 'roident': self.getResourceValue(self.rouri, DCTERMS.identifier), 'rotitle': self.getResourceValue(self.rouri, DCTERMS.title), 'rocreator': self.getResourceValue(self.rouri, DCTERMS.creator), 'rocreated': self.getResourceValue(self.rouri, DCTERMS.created), 'rodescription': self.getResourceValue(self.rouri, DCTERMS.description), } return manifestDict # Support methods for accessing RO and component URIs def getRoRef(self): """ Returns RO URI reference supplied (which may be a local file directory string) """ return self.roref def getRoUri(self): return self.rouri def getComponentUri(self, path): """ Return URI for component where relative reference is treated as a file path """ if urlparse.urlsplit(path).scheme == "": path = resolveUri("", str(self.getRoUri()), path) return rdflib.URIRef(path) def getComponentUriAbs(self, path): """ Return absolute URI for component where relative reference is treated as a URI reference """ return rdflib.URIRef(urlparse.urljoin(str(self.getRoUri()), path)) def getComponentUriRel(self, path): """ Return reference relative to RO for a supplied URI """ file_uri = urlparse.urlunsplit( urlparse.urlsplit(str(self.getComponentUriAbs(path)))) ro_uri = urlparse.urlunsplit(urlparse.urlsplit(str(self.getRoUri()))) if ro_uri is not None and file_uri.startswith(ro_uri): file_uri_rel = file_uri.replace(ro_uri, "", 1) else: file_uri_rel = path return rdflib.URIRef(file_uri_rel) def isRoMetadataRef(self, uri): """ Test if supplied URI is a reference to the current RO metadata area """ urirel = self.getComponentUriRel(uri) return str(urirel).startswith(ro_settings.MANIFEST_DIR + "/") def isLocalFileRo(self): """ Test current RO URI to see if it is a local file system reference """ return isFileUri(self.getRoUri()) def getRoFilename(self): assert self._isLocal() return getFilenameFromUri(self.getRoUri()) def getManifestFilename(self): """ Return manifest file name: used for local updates """ assert self._isLocal() return os.path.join(self.getRoFilename(), ro_settings.MANIFEST_DIR + "/", ro_settings.MANIFEST_FILE) def getRegistries(self): ''' Load a dictionary of synchronization data from memory or from a JSON file. ''' log.debug("Get registries") if self.registries: return self.registries try: rf = open( os.path.join(self.getRoFilename(), ro_settings.REGISTRIES_FILE), 'r') self.registries = json.load(rf) except IOError: self.registries = dict() except Exception as e: log.exception(e) self.registries = dict() return self.registries def saveRegistries(self): ''' Save a dictionary of synchronization data to a JSON file. ''' log.debug("Save registries") rf = open( os.path.join(self.getRoFilename(), ro_settings.REGISTRIES_FILE), 'w') if self.registries: json.dump(self.registries, rf) return def calculateChecksum(self, rofile): ''' Calculate a file checksum. ''' m = hashlib.md5() with open(rofile) as f: for line in f: m.update(line) f.close() return m.hexdigest()
class ro_metadata(object): """ Class for accessing RO metadata """ def __init__(self, roconfig, roref, dummysetupfortest=False): """ Initialize: read manifest from object at given directory into local RDF graph roconfig is the research object manager configuration, supplied as a dictionary roref a URI reference that refers to the Research Object to be accessed, or relative path name (see ro_uriutils.resolveFileAsUri for interpretation) dummysetupfortest is an optional parameter that, if True, suppresses some aspects of the setup (does not attempt to read a RO manifest) for isolated testing. """ self.roconfig = roconfig self.roref = roref self.dummyfortest = dummysetupfortest self.manifestgraph = None self.roannotations = None self.registries = None uri = resolveFileAsUri(roref) if not uri.endswith("/"): uri += "/" self.rouri = rdflib.URIRef(uri) if self._isLocal(): self.rosrs = None else: self.rosrs = ROSRS_Session( self.roconfig["rosrs_uri"], self.roconfig["rosrs_access_token"] ) self._loadManifest() # Get RO URI from manifest # May be different from computed value if manifest has absolute URI # Nested URIs may be present; ours is the one described by the manifest URI, # which is determined by the _loadManifest() method. for s in self.manifestgraph.subjects(RDF.type, RO.ResearchObject): if self.manifestgraph.value(s, ORE.isDescribedBy) == self.manifesturi: self.rouri = s # Check that the manifest contained at least one RO URI assert self.rouri is not None return def _isLocal(self): return isFileUri(self.rouri) def _getLocalManifestUri(self): return self.getComponentUri(ro_settings.MANIFEST_DIR+"/"+ro_settings.MANIFEST_FILE) def _loadManifest(self): if self.manifestgraph != None: return self.manifestgraph if self.dummyfortest: # Fake minimal manifest graph for testing self.manifestgraph = rdflib.Graph() self.manifestgraph.add( (self.rouri, RDF.type, RO.ResearchObject) ) self.manifesturi = self.rouri elif self._isLocal(): # Read manifest graph self.manifestgraph = rdflib.Graph() for (prefix, uri) in ro_prefixes.prefixes: self.manifestgraph.bind(prefix, rdflib.namespace.Namespace(uri)) self.manifesturi = self._getLocalManifestUri() self.manifestgraph.parse(self.manifesturi) else: (status, reason, _h, manifesturi, manifest) = self.rosrs.getROManifest(self.rouri) if status != 200: msg = ("Can't access RO manifest (%03d %s)"%(status, reason)) raise ROSRS_Error(msg=msg, srsuri=self.rouri) self.manifestgraph = manifest self.manifesturi = manifesturi # log.debug("romanifest graph:\n"+self.manifestgraph.serialize()) return self.manifestgraph def getManifestGraph(self): """ Returns the manifest graph """ return self._loadManifest() def _updateManifest(self): """ Write updated manifest file for research object """ assert self._isLocal() self._loadManifest().serialize( destination=self.getManifestFilename(), format='xml', base=self.rouri, xml_base="..") return def _iterAnnotations(self, subject=None): """ Return iterator over annotation stubs in the current RO, either for the specified subject resource, or for all annotations in the RO subject is URI of subject whose annotations are returned, or None. """ manifest = self._loadManifest() if self._isLocal(): for (anode, p, subject) in manifest: if p in [RO.annotatesAggregatedResource, AO.annotatesResource]: yield anode else: for anode in self.rosrs.getROAnnotationUris(self.getRoUri(), subject): yield anode return def isAggregatedResource(self, rofile): ''' Returns true if the manifest says that the research object aggregates the resource. Resource URI is resolved against the RO URI unless it's absolute. ''' resuri = self.getComponentUriAbs(rofile) return (self.rouri, ORE.aggregates, resuri) in self.manifestgraph def _loadAnnotations(self): if self.roannotations: return self.roannotations log.debug("_loadannotations") # Assemble annotation graph # NOTE: the manifest itself is included as an annotation by the RO setup if self._isLocal(): manifest = self._loadManifest() self.roannotations = rdflib.Graph() annotation_uris_loaded = set() for anode in self._iterAnnotations(): auri = manifest.value(subject=anode, predicate=AO.body) if auri not in annotation_uris_loaded: aref = self.getComponentUriRel(auri) log.debug("_loadAnnotations: aref "+str(aref)) self._readAnnotationBody(aref, self.roannotations) annotation_uris_loaded.add(auri) else: self.roannotations = self.rosrs.getROAnnotationGraph(self.rouri) # log.debug("roannotations graph:\n"+self.roannotations.serialize()) for (prefix, uri) in ro_prefixes.prefixes: self.manifestgraph.bind(prefix, rdflib.namespace.Namespace(uri)) return self.roannotations def isInternalResource(self, resuri): ''' Check if the resource is internal, i.e. should the resource content be uploaded to the ROSR service. Returns true if the resource URI has the RO URI as a prefix. ''' return resuri.startswith(self.rouri) def isExternalResource(self, resuri): ''' Check if the resource is external, i.e. can be aggregated as a URI reference. Returns true if the URI has 'http' or 'https' scheme. ''' parseduri = urlparse.urlsplit(resuri) return parseduri.scheme in ["http", "https"] def _createAnnotationBody(self, roresource, attrdict, defaultType="string"): """ Create a new annotation body for a single resource in a research object, based on a supplied graph value. Existing annotations for the same resource are not touched; if an annotation is being added or replaced, it is the calling program'sresponsibility to update the manifest to reference the active annotations. A new name is allocated for the created annotation, graph which is returned as the result of this function. roresource is the name of the Research Object component to be annotated, possibly relative to the RO root directory. attrdict is a dictionary of attributes to be saved in the annotation body. Dictionary keys are attribute names that can be resolved via ro_annotation.getAnnotationByName. Returns the name of the annotation body created relative to the RO directory. """ assert self._isLocal() af = ro_annotation.createAnnotationBody( self.roconfig, self.getRoFilename(), roresource, attrdict, defaultType) return os.path.join(ro_settings.MANIFEST_DIR+"/", af) def _createAnnotationGraphBody(self, roresource, anngraph): """ Create a new annotation body for a single resource in a research object, based on a supplied graph value. Existing annotations for the same resource are not touched; if an annotation is being added or replaced, it is the calling program'sresponsibility to update the manifest to reference the active annotations. A new name is allocated for the created annotation, graph which is returned as the result of this function. roresource is the name of the Research Object component to be annotated, possibly relative to the RO root directory. anngraph is an annotation graph that is to be saved. Returns the name of the annotation body created relative to the RO manifest and metadata directory. """ assert self._isLocal() af = ro_annotation.createAnnotationGraphBody( self.roconfig, self.getRoFilename(), roresource, anngraph) return os.path.join(ro_settings.MANIFEST_DIR+"/", af) def _readAnnotationBody(self, annotationref, anngr=None): """ Read annotation body from indicated resource, return RDF Graph of annotation values. annotationref is a URI reference of an annotation, possibly relative to the RO base URI (e.g. as returned by _createAnnotationBody method). anngr if supplied, if an RDF graph to which the annotations are added """ assert self._isLocal() log.debug("_readAnnotationBody %s"%(annotationref)) annotationuri = self.getComponentUri(annotationref) annotationformat = "xml" # Look at file extension to figure format # (rdflib.Graph.parse says; # "used if format can not be determined from the source") if re.search("\.(ttl|n3)$", annotationuri): annotationformat="n3" if anngr == None: log.debug("_readAnnotationBody: new graph") anngr = rdflib.Graph() try: anngr.parse(annotationuri, format=annotationformat) log.debug("_readAnnotationBody parse %s, len %i"%(annotationuri, len(anngr))) except IOError as e: log.debug("_readAnnotationBody %s, %s"%(str(annotationref), repr(e))) anngr = None except Exception as e: log.debug("Failed to load annotation %s as %s"%(annotationuri, annotationformat)) log.debug("Exception %s"%(repr(e))) raise anngr = None return anngr def _addAnnotationToManifest(self, rofile, annfile): """ Add a new annotation body to an RO graph rofile is the research object resource being annotated annfile is the file name of the annotation body to be added, possibly relative to the RO URI, with special characters already URI-escaped. """ assert self._isLocal() # <ore:aggregates> # <ro:AggregatedAnnotation> # <ro:annotatesAggregatedResource rdf:resource="data/UserRequirements-astro.ods" /> # <ao:body rdf:resource=".ro/(annotation).rdf" /> # </ro:AggregatedAnnotation> # </ore:aggregates> log.debug("_addAnnotationToManifest annfile %s"%(annfile)) ann = rdflib.BNode() resuri = self.getComponentUri(rofile) bodyuri = self.getComponentUriAbs(annfile) self.manifestgraph.add((ann, RDF.type, RO.AggregatedAnnotation)) self.manifestgraph.add((ann, RO.annotatesAggregatedResource, resuri)) self.manifestgraph.add((ann, AO.body, bodyuri)) # Aggregate the annotation self.manifestgraph.add((self.getRoUri(), ORE.aggregates, ann)) # Aggregate annotation body if it is RO metadata. # Otherwise aggregation is the caller's responsibility if self.isRoMetadataRef(bodyuri): self.manifestgraph.add((self.getRoUri(), ORE.aggregates, bodyuri)) self.roannotations = None # Flush cached annotation graph return def _removeAnnotationFromManifest(self, ann): """ Remove references to an annotation from an RO graph ann is the the annotation node to be removed """ assert self._isLocal() bodyuri = self.manifestgraph.value(subject=ann, predicate=AO.body) self.manifestgraph.remove((ann, None, None )) # If annotation body is RO Metadata, and there are no other uses as an annotation, # remove it from the RO aggregation. if self.isRoMetadataRef(bodyuri): if not self.manifestgraph.value(subject=ann, predicate=AO.body): self.manifestgraph.remove((None, ORE.aggregates, bodyuri)) self.roannotations = None # Flush cached annotation graph return def addAggregatedResources(self, ro_file, recurse=True, includeDirs=False): """ Scan a local directory and add files found to the RO aggregation """ assert self._isLocal() def notHidden(f): return re.match("\.|.*/\.", f) == None log.debug("addAggregatedResources: roref %s, file %s"%(self.roref, ro_file)) self.getRoFilename() # Check that we have one basedir = os.path.abspath(self.roref)+os.path.sep ### print "- ro_file: %s"%(ro_file) if os.path.isdir(ro_file): ro_file = os.path.abspath(ro_file)+os.path.sep ### print "- ro_file: %s"%(ro_file) ### print "- basedir: %s"%(basedir) #if ro_file.endswith(os.path.sep): # ro_file = ro_file[0:-1] if recurse: rofiles = filter(notHidden, MiscUtils.ScanDirectories.CollectDirectoryContents(ro_file, baseDir=basedir, listDirs=includeDirs, listFiles=True, recursive=recurse, appendSep=True ) ) log.debug("- rofiles: %s"%(repr(rofiles))) else: rofiles = [ro_file.split(basedir+os.path.sep,1)[-1]] else: rofiles = [self.getComponentUriRel(ro_file)] s = self.getRoUri() for f in rofiles: ### print "- file %s"%f log.debug("- file %s"%f) stmt = (s, ORE.aggregates, self.getComponentUri(f)) if stmt not in self.manifestgraph: self.manifestgraph.add(stmt) self._updateManifest() return def removeAggregatedResource(self, resuri): """ Remove a specified resource. resref is the URI of a resource to remove """ assert self._isLocal() resuri = rdflib.URIRef(resuri) log.debug("removeAggregatedResource: roref %s, resuri %s"%(self.roref, str(resuri))) manifest = self._loadManifest() for anode in self._iterAnnotations(subject=resuri): self._removeAnnotationFromManifest(anode) manifest.remove((None, ORE.aggregates, resuri)) self._updateManifest() return def getAggregatedResources(self): """ Returns iterator over all resources aggregated by a manifest. Each value returned by the iterator is an aggregated resource URI """ log.debug("getAggregatedResources: uri %s"%(self.rouri)) for r in self._loadManifest().objects(subject=self.rouri, predicate=ORE.aggregates): if not isinstance(r, rdflib.BNode): yield r return def addGraphAnnotation(self, rofile, graph): """ Add an annotation graph for a named resource. Unlike addSimpleAnnotation, this method adds an annotation to the manifest using an existing RDF graph (which is presumably itself in the RO structure). rofile names the file or resource to be annotated, possibly relative to the RO. Note that no checks are performed to ensure that the graph itself actually refers to this resource - that's up the the creator of the graph. This identifies the resourfce with which the annotation body is associated in the RO manifest. graph names the file or resource containing the annotation body. """ assert self._isLocal() ro_graph = self._loadManifest() self._addAnnotationToManifest(rofile, graph) self._updateManifest() return def isAnnotationNode(self, respath): ''' Returns true if the manifest says that the research object aggregates the annotation and it is an ro:AggregatedAnnotation. Resource URI is resolved against the RO URI unless it's absolute. ''' resuri = self.getComponentUriAbs(respath) log.debug("isAnnotationNode: ro uri %s res uri %s"%(self.rouri, resuri)) return (self.rouri, ORE.aggregates, resuri) in self.manifestgraph and \ (resuri, RDF.type, RO.AggregatedAnnotation) in self.manifestgraph def addSimpleAnnotation(self, rofile, attrname, attrvalue, defaultType="string"): """ Add a simple annotation to a file in a research object. rofile names the file or resource to be annotated, possibly relative to the RO. attrname names the attribute in a form recognized by getAnnotationByName attrvalue is a value to be associated with the attribute """ assert self._isLocal() ro_dir = self.getRoFilename() annfile = self._createAnnotationBody(rofile, {attrname: attrvalue}, defaultType) self._addAnnotationToManifest(rofile, annfile) self._updateManifest() return annfile def removeSimpleAnnotation(self, rofile, attrname, attrvalue): """ Remove a simple annotation or multiple matching annotations a research object. rofile names the annotated file or resource, possibly relative to the RO. attrname names the attribute in a form recognized by getAnnotationByName attrvalue is the attribute value to be deleted, or Nomne to delete all vaues """ assert self._isLocal() ro_dir = self.getRoFilename() ro_graph = self._loadManifest() subject = self.getComponentUri(rofile) (predicate,valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname) val = attrvalue and ro_annotation.makeAnnotationValue(self.roconfig, attrvalue, valtype) add_annotations = [] remove_annotations = [] log.debug("removeSimpleAnnotation subject %s, predicate %s, val %s"% (str(subject), str(predicate), val)) # Scan for annotation graph resources containing this annotation for ann_node in self._iterAnnotations(subject=subject): ann_uri = ro_graph.value(subject=ann_node, predicate=AO.body) log.debug("removeSimpleAnnotation ann_uri %s"%(str(ann_uri))) if self.isRoMetadataRef(ann_uri): ann_graph = self._readAnnotationBody(self.getComponentUriRel(ann_uri)) log.debug("removeSimpleAnnotation ann_graph %s"%(ann_graph)) if (subject, predicate, val) in ann_graph: ann_graph.remove((subject, predicate, val)) if (subject, None, None) in ann_graph: # Triples remain in annotation body: write new body and update RO graph ann_name = self._createAnnotationGraphBody(rofile, ann_graph) remove_annotations.append(ann_node) add_annotations.append(ann_name) else: # Remove annotation from RO graph remove_annotations.append(ann_node) # Update RO manifest graph if needed if add_annotations or remove_annotations: for a in remove_annotations: self._removeAnnotationFromManifest(a) for a in add_annotations: self._addAnnotationToManifest(rofile, a) self._updateManifest() return def replaceSimpleAnnotation(self, rofile, attrname, attrvalue): """ Replace a simple annotation in a research object. rofile names the file or resource to be annotated, possibly relative to the RO. attrname names the attribute in a form recognized by getAnnotationByName attrvalue is a new value to be associated with the attribute """ assert self._isLocal() ro_dir = self.getRoFilename() ro_graph = self._loadManifest() subject = self.getComponentUri(rofile) (predicate,valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname) log.debug("Replace annotation: subject %s, predicate %s, value %s"% (repr(subject), repr(predicate), repr(attrvalue))) ro_graph.remove((subject, predicate, None)) ro_graph.add((subject, predicate, ro_annotation.makeAnnotationValue(self.roconfig, attrvalue, valtype))) self._updateManifest() self.roannotations = None # Flush cached annotation graph return def iterateAnnotations(self, subject=None, property=None): """ Returns an iterator over annotations of the current RO that match the supplied subject and/or property. """ log.debug("iterateAnnotations s:%s, p:%s"%(str(subject),str(property))) ann_graph = self._loadAnnotations() for (s, p, v) in ann_graph.triples((subject, property, None)): if not isinstance(s, rdflib.BNode): if not self.isRoMetadataRef(s): log.debug("Triple: %s %s %s"%(s,p,v)) yield (s, p, v) return def getRoAnnotations(self): """ Returns iterator over annotations applied to the RO as an entity. Each value returned by the iterator is a (subject,predicate,object) triple. """ return self.iterateAnnotations(subject=self.getRoUri()) def getFileAnnotations(self, rofile): """ Returns iterator over annotations applied to a specified component in the RO Each value returned by the iterator is a (subject,predicate,object) triple. """ return self.iterateAnnotations(subject=self.getComponentUri(rofile)) def getAllAnnotations(self): """ Returns iterator over all annotations associated with the RO Each value returned by the iterator is a (subject,predicate,object) triple. """ return self.iterateAnnotations() def getAllAnnotationNodes(self): """ Returns iterator over all annotations aggregated within the RO Each value returned by the iterator is a (annuri, bodyuri, target) triple. """ annotations = self.manifestgraph.subject_objects(predicate=RO.annotatesAggregatedResource) for (ann_node, ann_target) in annotations: ann_body = self.manifestgraph.value(subject=ann_node, predicate=AO.body) yield (ann_node, ann_body, ann_target) return def getAnnotationValues(self, rofile, attrname): """ Returns iterator over annotation values for given subject and attribute """ (predicate,valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname) resuri = self.getComponentUri(rofile) return ( v for (s,p,v) in self.iterateAnnotations(subject=resuri, property=predicate) ) def queryAnnotations(self, query, initBindings={}): """ Runs a query over the combined annotation graphs (including the manifest) and returns True or False (for ASK queries) or a list of dictionaries of query results (for SELECT queries). """ log.debug("queryAnnotations: \n----\n%s\n--------\n"%(query)) ann_gr = self._loadAnnotations() # log.debug("queryAnnotations graph: \n----\n%s\n--------\n"%(ann_gr.serialize(format='xml'))) try: resp = ann_gr.query(query,initBindings=initBindings) except: log.info("queryAnnotations failed query: \n----\n%s\n--------\n"%(query)) raise if resp.type == 'ASK': return resp.askAnswer elif resp.type == 'SELECT': return resp.bindings else: assert False, "Unexpected query response type %s"%resp.type return None def getAnnotationGraph(self): """ Returns the combined annotation graphs (including the manifest) """ return self._loadAnnotations() def getAnnotationValue(self, resource, predicate): """ Returns a single annotation value for a resource and the indicated predicate, or None """ return self._loadAnnotations().value(subject=resource, predicate=predicate, object=None) def showAnnotations(self, annotations, outstr): ro_annotation.showAnnotations(self.roconfig, self.getRoFilename(), annotations, outstr) return def replaceUri(self, ann_node, remote_ann_node_uri): for (p, o) in self.manifestgraph.predicate_objects(subject = ann_node): self.manifestgraph.remove((ann_node, p, o)) self.manifestgraph.add((remote_ann_node_uri, p, o)) for (s, p) in self.manifestgraph.subject_predicates(object = ann_node): self.manifestgraph.remove((s, p, ann_node)) self.manifestgraph.add((s, p, remote_ann_node_uri)) self._updateManifest() return # Support methods for accessing the manifest graph def roManifestContains(self, stmt): """ Returns True if the RO manifest contains a statement matching the supplied triple. """ return stmt in self._loadManifest() def getResourceValue(self, resource, predicate): """ Returns value for resource whose URI is supplied assocfiated with indicated predicate """ return self._loadManifest().value(subject=resource, predicate=predicate, object=None) def getResourceType(self, resource): """ Returns type of resource whose URI is supplied """ return self.getResourceValue(resource, RDF.type) def hasResourceType(self, resource, rdfType): """ Check if the resource whose URI is supplied has a provided RDF type. """ return self.roManifestContains((resource, RDF.type, rdfType)) def getRoMetadataDict(self): """ Returns dictionary of metadata about the RO from the manifest graph """ assert self._isLocal() strsubject = "" if isinstance(self.rouri, rdflib.URIRef): strsubject = str(self.rouri) manifestDict = { 'ropath': self.getRoFilename(), 'rouri': strsubject, 'roident': self.getResourceValue(self.rouri, DCTERMS.identifier ), 'rotitle': self.getResourceValue(self.rouri, DCTERMS.title ), 'rocreator': self.getResourceValue(self.rouri, DCTERMS.creator ), 'rocreated': self.getResourceValue(self.rouri, DCTERMS.created ), 'rodescription': self.getResourceValue(self.rouri, DCTERMS.description ), } return manifestDict # Support methods for accessing RO and component URIs def getRoRef(self): """ Returns RO URI reference supplied (which may be a local file directory string) """ return self.roref def getRoUri(self): return self.rouri def getComponentUri(self, path): """ Return URI for component where relative reference is treated as a file path """ if urlparse.urlsplit(path).scheme == "": path = resolveUri("", str(self.getRoUri()), path) return rdflib.URIRef(path) def getComponentUriAbs(self, path): """ Return absolute URI for component where relative reference is treated as a URI reference """ return rdflib.URIRef(urlparse.urljoin(str(self.getRoUri()), path)) def getComponentUriRel(self, path): """ Return reference relative to RO for a supplied URI """ file_uri = urlparse.urlunsplit(urlparse.urlsplit(str(self.getComponentUriAbs(path)))) ro_uri = urlparse.urlunsplit(urlparse.urlsplit(str(self.getRoUri()))) if ro_uri is not None and file_uri.startswith(ro_uri): file_uri_rel = file_uri.replace(ro_uri, "", 1) else: file_uri_rel = path return rdflib.URIRef(file_uri_rel) def isRoMetadataRef(self, uri): """ Test if supplied URI is a reference to the current RO metadata area """ urirel = self.getComponentUriRel(uri) return str(urirel).startswith(ro_settings.MANIFEST_DIR+"/") def isLocalFileRo(self): """ Test current RO URI to see if it is a local file system reference """ return isFileUri(self.getRoUri()) def getRoFilename(self): assert self._isLocal() return getFilenameFromUri(self.getRoUri()) def getManifestFilename(self): """ Return manifest file name: used for local updates """ assert self._isLocal() return os.path.join(self.getRoFilename(), ro_settings.MANIFEST_DIR+"/", ro_settings.MANIFEST_FILE) def getRegistries(self): ''' Load a dictionary of synchronization data from memory or from a JSON file. ''' log.debug("Get registries") if self.registries: return self.registries try: rf = open(os.path.join(self.getRoFilename(), ro_settings.REGISTRIES_FILE), 'r') self.registries = json.load(rf) except IOError: self.registries = dict() except Exception as e: log.exception(e) self.registries = dict() return self.registries def saveRegistries(self): ''' Save a dictionary of synchronization data to a JSON file. ''' log.debug("Save registries") rf = open(os.path.join(self.getRoFilename(), ro_settings.REGISTRIES_FILE), 'w') if self.registries: json.dump(self.registries, rf) return def calculateChecksum(self, rofile): ''' Calculate a file checksum. ''' m = hashlib.md5() with open(rofile) as f: for line in f: m.update(line) f.close() return m.hexdigest()
class TestROSRSMetadata(TestROSupport.TestROSupport): """ Test ro metadata access via ROSRS API """ def setUp(self): super(TestROSRSMetadata, self).setUp() self.rosrs = ROSRS_Session(Config.ROSRS_API_URI, accesskey=Config.AUTHORIZATION) # Clean up from previous runs self.rosrs.deleteRO(Config.TEST_RO_PATH) return def tearDown(self): super(TestROSRSMetadata, self).tearDown() # Clean up self.rosrs.deleteRO(Config.TEST_RO_PATH) self.rosrs.close() return def createTestRO(self): (status, reason, rouri, manifest) = self.rosrs.createRO(Config.TEST_RO_NAME, "Test RO for ROSRSMetadata", "TestROSRSMetadata.py", "2012-09-11") self.assertEqual(status, 201) # Include manifest as annotation of RO (s1, r1, h1, manifesturi, manifest) = self.rosrs.getROManifest(rouri) self.assertEqual(s1, 200) (s2, r2, annuri) = self.rosrs.createROAnnotationExt( rouri, rouri, manifesturi) self.assertEqual(s2, 201) # Aggregate internal resource rescontent = "Resource content\n" (s3, r3, proxyuri, resuri) = self.rosrs.aggregateResourceInt( rouri, Config.TEST_RESOURCE, ctype="text/plain", body=rescontent) self.assertEqual(s3, 201) self.assertEqual(r3, "Created") self.assertEqual(str(resuri), str(rouri)+Config.TEST_RESOURCE) # Aggregate external resource externaluri = rdflib.URIRef(Config.TEST_EXTERNAL) (s4, r4, proxyuri, resuri) = self.rosrs.aggregateResourceExt( rouri, externaluri) self.assertEqual(s4, 201) self.assertEqual(r4, "Created") self.assertEqual(str(resuri), Config.TEST_EXTERNAL) return (status, reason, rouri, manifest) def createTestAnnotation(self, rouri, resuri, resref): annbody = """<?xml version="1.0" encoding="UTF-8"?> <rdf:RDF xmlns:dct="http://purl.org/dc/terms/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xml:base="%(rouri)s" > <rdf:Description rdf:about="%(resuri)s"> <dct:title>Title for %(resref)s</dct:title> <rdfs:seeAlso rdf:resource="http://example.org/test" /> </rdf:Description> </rdf:RDF> """%{"rouri": str(rouri), "resuri": str(resuri), "resref": resref} agraph = rdflib.graph.Graph() agraph.parse(data=annbody, format="xml") (status, reason, annuri, bodyuri) = self.rosrs.createROAnnotationInt( rouri, resuri, agraph) self.assertEqual(status, 201) return (status, reason, annuri, bodyuri) # Actual tests follow def testNull(self): assert True, 'Null test failed' def testCreateRoMetadata(self): """ Test creation of ro_metadata object, and basic access to manifest content """ (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) self.assertEqual(reason, "Created") self.assertEqual(str(rouri), Config.TEST_RO_URI) self.assertIn((rouri, RDF.type, RO.ResearchObject), manifest) romd = ro_metadata.ro_metadata(ro_config, rouri) resuri = romd.getComponentUriAbs(Config.TEST_RESOURCE) exturi = romd.getComponentUriAbs(Config.TEST_EXTERNAL) resref = Config.TEST_RESOURCE (status, reason, annuri, bodyuri) = self.createTestAnnotation(rouri, resuri, resref) self.assertEqual(status, 201) self.assertEqual(reason, "Created") # self.assertEquals(romd.rouri, rouri) self.assertTrue(romd.roManifestContains((rouri, RDF.type, RO.ResearchObject))) self.assertTrue(romd.roManifestContains((rouri, ORE.aggregates, resuri))) self.assertTrue(romd.roManifestContains((rouri, ORE.aggregates, exturi))) return def testReadRoAnnotationBody(self): """ Test function to create & read a simple annotation body on an RO """ (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) romd = ro_metadata.ro_metadata(ro_config, rouri) resuri = romd.getComponentUriAbs(Config.TEST_RESOURCE) resref = Config.TEST_RESOURCE (status, reason, bodyuri, agraph) = self.createTestAnnotation(rouri, resuri, resref) self.assertEqual(status, 201) # Retrieve annotations anns = list(romd.getFileAnnotations(Config.TEST_RESOURCE)) self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title for "+Config.TEST_RESOURCE)), anns) self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test")), anns) return def testGetInitialRoAnnotations(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) romd = ro_metadata.ro_metadata(ro_config, rouri) # Retrieve the anotations annotations = romd.getRoAnnotations() rouri = romd.getRoUri() expected_annotations = ( [ (rouri, RDF.type, RO.ResearchObject) , (rouri, RDF.type, ROEVO.LiveRO) , (rouri, ORE.isDescribedBy, romd.getComponentUriAbs(ro_test_config.ROMANIFESTPATH)) #, (rouri, DCTERMS.description, rdflib.Literal('Test init RO annotation')) #, (rouri, DCTERMS.title, rdflib.Literal('Test init RO annotation')) #, (rouri, DCTERMS.created, rdflib.Literal('unknown')) #, (rouri, DCTERMS.creator, rdflib.Literal('Test User')) #, (rouri, DCTERMS.identifier, rdflib.Literal('ro-testRoAnnotate')) ]) count = 0 for next in list(annotations): if ( not isinstance(next[2], rdflib.BNode) and not next[1] in [ORE.aggregates, DCTERMS.created, DCTERMS.creator] ): log.debug("- next %s"%(str(next[0])) ) log.debug(" - (%s, %s)"%(str(next[1]),str(next[2])) ) if next in expected_annotations: count += 1 else: self.assertTrue(False, "Not expected (%d) %s"%(count, repr(next))) self.assertEqual(count,3) return def testQueryAnnotations(self): (status, reason, rouri, manifest) = self.createTestRO() self.assertEqual(status, 201) romd = ro_metadata.ro_metadata(ro_config, rouri) resuri = romd.getComponentUriAbs(Config.TEST_RESOURCE) resref = Config.TEST_RESOURCE (status, reason, bodyuri, agraph) = self.createTestAnnotation(rouri, resuri, resref) self.assertEqual(status, 201) # Query the file anotations queryprefixes = """ PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> PREFIX ro: <http://purl.org/wf4ever/ro#> PREFIX ore: <http://www.openarchives.org/ore/terms/> PREFIX ao: <http://purl.org/ao/> PREFIX dcterms: <http://purl.org/dc/terms/> PREFIX roterms: <http://ro.example.org/ro/terms/> """ query = (queryprefixes + """ ASK { ?ro rdf:type ro:ResearchObject ; dcterms:creator ?user . } """) resp = romd.queryAnnotations(query) self.assertTrue(resp, "Expected 'True' result for query: %s"%(query)) query = (queryprefixes + """ ASK { <%(resuri)s> dcterms:title ?title . } """%{"resuri": str(resuri)}) resp = romd.queryAnnotations(query) self.assertTrue(resp, "Expected 'True' result for query: %s"%(query)) query = (queryprefixes + """ ASK { ?ro rdf:type ro:ResearchObject ; dcterms:creator "Not user" . } """) resp = romd.queryAnnotations(query) self.assertFalse(resp, "Expected 'False' result for query: %s"%(query)) query = (queryprefixes + """ SELECT * WHERE { ?ro rdf:type ro:ResearchObject ; ore:aggregates ?file . ?file dcterms:title ?title . } """) rouri = romd.getRoUri() resp = romd.queryAnnotations(query) self.assertEqual(resp[0]['ro'], rouri) self.assertEqual(resp[0]['file'], resuri) self.assertEqual(resp[0]['title'], rdflib.Literal("Title for %s"%(Config.TEST_RESOURCE))) return # Sentinel/placeholder tests def testUnits(self): assert (True) def testComponents(self): assert (True) def testIntegration(self): assert (True) def testPending(self): assert (False), "Pending tests follow"