def freeze(options, args):
rosrs = ROSRS_Session(options["rosrs_uri"], options["rosrs_access_token"])
service_uri = urljoin(options["rosrs_uri"], "../evo/finalize/")
body = {
'target': args[2],
}
body = json.dumps(body)
reqheaders = {}
(status, reason, headers,
data) = response = rosrs.doRequest(uripath=service_uri,
method="POST",
body=body,
ctype="application/json",
reqheaders=reqheaders)
if "location" in headers:
while print_job_status(parse_job(rosrs, headers['location']), options,
True):
time.sleep(1)
print "freeze operation finished successfully"
return 0
else:
print status
print reason
print headers
print data
print "Given URI isn't correct"
return -1
def copy_operation(options, args, ro_type):
options["rosrs_access_token"]
rosrs = ROSRS_Session(options["rosrs_uri"], options["rosrs_access_token"])
service_uri = urljoin(options["rosrs_uri"], "../evo/copy/")
body = {
'copyfrom': args[2],
'type': ro_type,
'finalize': ("%s" % options['freeze']).lower()
}
body = json.dumps(body)
reqheaders = {'Slug': args[3]}
response = rosrs.doRequest(uripath=service_uri,
method="POST",
body=body,
ctype="application/json",
reqheaders=reqheaders)
if response[0] != 201:
return handle_copy_error(options, rosrs, response, ro_type)
if not options["asynchronous"]:
return handle_synchronous_copy_operation_with_esc_option(
options, rosrs, response, ro_type)
if options["asynchronous"]:
return handle_asynchronous_copy_operation(options, rosrs, response,
ro_type)
return 0
def testArchive(self):
rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN)
(copy_status, archiveUri) = self.createArchive(self.TEST_CREATED_RO_ID, self.TEST_SNAPHOT_ID, False)
assert copy_status == "DONE"
(status, reason, data, evo_type) = rosrs.getROEvolution(archiveUri)
assert evo_type == 3
self.freeze(archiveUri)
(status, reason, data, evo_type) = rosrs.getROEvolution(archiveUri)
assert evo_type == 2
(status, reason) = self.rosrs.deleteRO(self.TEST_CREATED_RO_ID)
(status, reason) = self.rosrs.deleteRO(archiveUri)
def freeze(self, ro_uri):
rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN)
service_uri = urljoin(ro_test_config.ROSRS_URI, "../evo/finalize/")
body = {
'target': ro_uri,
}
body = json.dumps(body)
reqheaders = {}
(status, reason, headers, data) = rosrs.doRequest(uripath=service_uri, method="POST", body=body, ctype="application/json", reqheaders=reqheaders)
job_location = get_location(headers)
status = "RUNNING"
while status == "RUNNING":
(status, id) = parse_job(rosrs, job_location)
return status
def setUp(self):
super(TestROSRSMetadata, self).setUp()
self.rosrs = ROSRS_Session(Config.ROSRS_API_URI,
accesskey=Config.AUTHORIZATION)
# Clean up from previous runs
self.rosrs.deleteRO(Config.TEST_RO_PATH)
return
def __init__(self, roconfig, roref, dummysetupfortest=False):
"""
Initialize: read manifest from object at given directory into local RDF graph
roconfig is the research object manager configuration, supplied as a dictionary
roref a URI reference that refers to the Research Object to be accessed, or
relative path name (see ro_uriutils.resolveFileAsUri for interpretation)
dummysetupfortest is an optional parameter that, if True, suppresses some aspects of
the setup (does not attempt to read a RO manifest) for isolated testing.
"""
self.roconfig = roconfig
self.roref = roref
self.dummyfortest = dummysetupfortest
self.manifestgraph = None
self.roannotations = None
self.registries = None
uri = resolveFileAsUri(roref)
if not uri.endswith("/"): uri += "/"
self.rouri = rdflib.URIRef(uri)
if self._isLocal():
self.rosrs = None
else:
self.rosrs = ROSRS_Session(
self.roconfig["rosrs_uri"],
self.roconfig["rosrs_access_token"]
)
self._loadManifest()
# Get RO URI from manifest
# May be different from computed value if manifest has absolute URI
self.rouri = self.manifestgraph.value(None, RDF.type, RO.ResearchObject)
# Check that the manifest contained at least one RO URI
assert self.rouri is not None
return
def setUp(self):
super(TestEvoCommands, self).setUp()
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID,
"Test RO for ROEVO", "Test Creator", "2012-09-06")
self.CREATED_RO = rouri;
return
def setUp(self):
super(TestROSRS_Session, self).setUp()
self.rosrs = ROSRS_Session(Config.ROSRS_API_URI,
accesskey=Config.AUTHORIZATION)
# Clean up from previous runs
self.rosrs.deleteRO(Config.TEST_RO_PATH, purge=True)
self.createdTestRO = None
return
def setUp(self):
super(TestROSRSMetadata, self).setUp()
self.rosrs = ROSRS_Session(Config.ROSRS_API_URI,
accesskey=Config.AUTHORIZATION)
self.roname = Config.TEST_RO_NAME
self.ropath = self.roname + "/"
# Clean up from previous runs
self.rosrs.deleteRO(self.ropath, purge=True)
return
class TestEvo(TestROEVOSupport.TestROEVOSupport):
TEST_RO_ID = "ro-manager-evo-test-ro"
TEST_SNAPHOT_RO_ID = "ro-manager-test-evo-snaphot-ro"
TEST_SNAPHOT_ID = "ro-manager-test-evo-snaphot"
TEST_CREATED_RO_ID = ""
def setUp(self):
super(TestEvo, self).setUp()
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID,
"Test RO for ROEVO", "Test Creator", "2012-09-06")
self.TEST_CREATED_RO_ID = rouri
return
def tearDown(self):
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID+"/")
(status, reason) = self.rosrs.deleteRO(self.TEST_CREATED_RO_ID)
super(TestEvo, self).tearDown()
return
def testSnapshot(self):
rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN)
(copy_status, snapshot_uri) = self.createSnapshot(self.TEST_CREATED_RO_ID, self.TEST_SNAPHOT_ID, False)
assert copy_status == "DONE"
(status, reason, data, evo_type) = rosrs.getROEvolution(snapshot_uri)
assert evo_type == 3
self.freeze(snapshot_uri)
(status, reason, data, evo_type) = rosrs.getROEvolution(snapshot_uri)
assert evo_type == 1
(status, reason) = self.rosrs.deleteRO(self.TEST_CREATED_RO_ID)
(status, reason) = self.rosrs.deleteRO(snapshot_uri)
def testArchive(self):
rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN)
(copy_status, archiveUri) = self.createArchive(self.TEST_CREATED_RO_ID, self.TEST_SNAPHOT_ID, False)
assert copy_status == "DONE"
(status, reason, data, evo_type) = rosrs.getROEvolution(archiveUri)
assert evo_type == 3
self.freeze(archiveUri)
(status, reason, data, evo_type) = rosrs.getROEvolution(archiveUri)
assert evo_type == 2
(status, reason) = self.rosrs.deleteRO(self.TEST_CREATED_RO_ID)
(status, reason) = self.rosrs.deleteRO(archiveUri)
def freeze(options, args):
rosrs = ROSRS_Session(options["rosrs_uri"], options["rosrs_access_token"])
service_uri = urljoin(options["rosrs_uri"], "../evo/finalize/")
body = {
'target': args[2],
}
body = json.dumps(body)
reqheaders = {}
(status, reason, headers, data) = response = rosrs.doRequest(uripath=service_uri, method="POST", body=body, ctype="application/json", reqheaders=reqheaders)
if "location" in headers:
while print_job_status(parse_job(rosrs, headers['location']), options, True):
time.sleep(1)
print "freeze operation finished successfully"
return 0
else:
print status
print reason
print headers
print data
print "Given URI isn't correct"
return -1
def copy_operation(options, args, ro_type):
options["rosrs_access_token"]
rosrs = ROSRS_Session(options["rosrs_uri"], options["rosrs_access_token"])
service_uri = urljoin(options["rosrs_uri"], "../evo/copy/")
body = {
'copyfrom': args[2],
'type': ro_type,
'finalize': ( "%s" % options['freeze']).lower()
}
body = json.dumps(body)
reqheaders = {
'Slug' : args[3]
}
response = rosrs.doRequest(uripath=service_uri, method="POST", body=body, ctype="application/json", reqheaders=reqheaders)
if response[0] != 201:
return handle_copy_error(options, rosrs, response, ro_type)
if not options["asynchronous"]:
return handle_synchronous_copy_operation_with_esc_option(options, rosrs, response, ro_type)
if options["asynchronous"]:
return handle_asynchronous_copy_operation(options, rosrs, response, ro_type)
return 0
def createArchive(self, live_name,sp_name,freeze = True):
service_uri = urljoin(ro_test_config.ROSRS_URI, "../evo/copy/")
body = {
'copyfrom': live_name,
'target': sp_name,
'type': "ARCHIVE",
'finalize': ( "%s" % freeze).lower()
}
body = json.dumps(body)
reqheaders = {
'token': ro_test_config.ROSRS_ACCESS_TOKEN,
'Slug' : sp_name,
}
rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN)
(status, reason, headers, data) = rosrs.doRequest(uripath=service_uri, method="POST", body=body, ctype="application/json", reqheaders=reqheaders)
job_location = get_location(headers)
status = "RUNNING"
while status == "RUNNING":
(status, id) = parse_job(rosrs, job_location)
return (status, id)
def __init__(self, roconfig, roref, dummysetupfortest=False):
"""
Initialize: read manifest from object at given directory into local RDF graph
roconfig is the research object manager configuration, supplied as a dictionary
roref a URI reference that refers to the Research Object to be accessed, or
relative path name (see ro_uriutils.resolveFileAsUri for interpretation)
dummysetupfortest is an optional parameter that, if True, suppresses some aspects of
the setup (does not attempt to read a RO manifest) for isolated testing.
"""
self.roconfig = roconfig
self.roref = roref
self.dummyfortest = dummysetupfortest
self.manifestgraph = None
self.roannotations = None
self.registries = None
uri = resolveFileAsUri(roref)
if not uri.endswith("/"): uri += "/"
self.rouri = rdflib.URIRef(uri)
if self._isLocal():
self.rosrs = None
else:
self.rosrs = ROSRS_Session(self.roconfig["rosrs_uri"],
self.roconfig["rosrs_access_token"])
self._loadManifest()
# Get RO URI from manifest
# May be different from computed value if manifest has absolute URI
# Nested URIs may be present; ours is the one described by the manifest URI,
# which is determined by the _loadManifest() method.
for s in self.manifestgraph.subjects(RDF.type, RO.ResearchObject):
if self.manifestgraph.value(s,
ORE.isDescribedBy) == self.manifesturi:
self.rouri = s
# Check that the manifest contained at least one RO URI
assert self.rouri is not None
return
def testRemoteStatusWithWrongUriGiven(self):
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
self.rosrs.deleteRO(self.TEST_RO_ID + "/")
self.rosrs.deleteRO("some-strange-uri/")
args = [
"ro", "status", ro_test_config.ROSRS_URI + "some-strange-uri/",
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
outtxt = self.outstr.getvalue()
assert status == -1
self.assertEqual(outtxt.count("Wrong URI was given"), 1)
self.rosrs.deleteRO(self.TEST_RO_ID + "/")
return
def testRemoteStatusLiveRO(self):
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID,
"Test RO for ROEVO", "Test Creator", "2012-09-06")
args = [
"ro", "status", str(rouri),
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
outtxt = self.outstr.getvalue()
self.assertEqual(outtxt.count("LIVE"), 1)
(status, reason) = self.rosrs.deleteRO(rouri)
return
class TestEvoCommands(TestROEVOSupport.TestROEVOSupport):
TEST_RO_ID = "ro-manger-evo-test-ro"
TEST_SNAPHOT_ID = "ro-manager-evo-test-snaphot"
TEST_ARCHIVE_ID = "ro-manager-evo-test-archive-id"
TEST_UNDEFINED_ID = "ro-manager-evo-test-undefined-id"
CREATED_RO = ""
def setUp(self):
super(TestEvoCommands, self).setUp()
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID,
"Test RO for ROEVO", "Test Creator", "2012-09-06")
self.CREATED_RO = rouri;
return
def tearDown(self):
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID+"/")
(status, reason) = self.rosrs.deleteRO(self.CREATED_RO)
super(TestEvoCommands, self).tearDown()
return
def testSnapshot(self):
"""
snapshot <live-RO> <snapshot-id> [ --asynchronous ] [ --freeze ] [ -t <access_token> ] [ -t <token> ]
"""
return
def testSnapshotAsynchronous(self):
args = [
"ro", "snapshot" , str(self.CREATED_RO), self.TEST_SNAPHOT_ID,
"--asynchronous",
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-r", ro_test_config.ROSRS_URI,
"-v"
]
outLines = ""
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
# simple check if the verbouse mode works well
for word in ("ro snapshot --asynchronous "+ro_test_config.ROSRS_URI + self.TEST_RO_ID + " " + self.TEST_SNAPHOT_ID).split(" "):
self.assertTrue(self.outstr.getvalue().count(word+ " ") or self.outstr.getvalue().count(" " + word), "snapshot command wasn't parse well")
self.assertEqual(self.outstr.getvalue().count("Job Status: "), 1)
self.assertEqual(self.outstr.getvalue().count("Job URI: "), 1)
self.assertEqual(self.outstr.getvalue().count("Target URI: "), 1)
self.assertEqual(self.outstr.getvalue().count("Target Name: "), 1)
self.assertEqual(self.outstr.getvalue().count("Response Status: "), 1)
self.assertEqual(self.outstr.getvalue().count("Response Reason: "), 1)
outLines = self.outstr.getvalue().split("\n")
for line in outLines:
if "Job URI:" in line:
jobLocation = line.split("Job URI:")[1].strip()
status = "RUNNING"
while status == "RUNNING":
(status, id) = parse_job(self.rosrs, jobLocation)
assert status == "DONE"
self.rosrs.deleteRO(id)
return
def testSnapshotSynchronous(self):
args = [
"ro", "snapshot", self.CREATED_RO, self.TEST_SNAPHOT_ID,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-r", ro_test_config.ROSRS_URI,
"-v"
]
outLines = ""
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
# simple check if the verbouse mode works well
for word in ("ro snaphot "+ro_test_config.ROSRS_URI + self.TEST_RO_ID + " " + self.TEST_SNAPHOT_ID).split(" "):
self.assertTrue(self.outstr.getvalue().count(word+ " ") or self.outstr.getvalue().count(" " + word), "snapshot command wasn't parse well")
self.assertEqual(self.outstr.getvalue().count("Target URI: "), 1)
outLines = self.outstr.getvalue().split("\n")
for line in outLines:
if "Target URI:" in line:
id = line.split("Target URI:")[1].strip()
self.rosrs.deleteRO(id)
return
def testSnapshotWithEscOption(self):
args = [
"ro", "snapshot", self.CREATED_RO, self.TEST_SNAPHOT_ID,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-r", ro_test_config.ROSRS_URI,
"-v"
]
outLines = ""
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
# simple check if the verbouse mode works well
self.assertEqual(self.outstr.getvalue().count("--asynchronous"), 0, "shouldn't be asynchronous")
outLines = self.outstr.getvalue().split("\n")
for line in outLines:
if "Target URI:" in line:
id = line.split("Target URI:")[1].strip()
self.rosrs.deleteRO(id)
return
def testArchive(self):
"""
archive <live-RO> <snapshot-id> [ --asynchronous ] [ --freeze ] [ -t <access_token> ] [ -t <token> ]
"""
return
def testArchiveAsynchronous(self):
args = [
"ro", "archive" , str(self.CREATED_RO), self.TEST_SNAPHOT_ID,
"--asynchronous",
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-r", ro_test_config.ROSRS_URI,
"-v"
]
outLines = ""
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
# simple check if the verbouse mode works well
for word in ("ro archive --asynchronous "+ro_test_config.ROSRS_URI + self.TEST_RO_ID + " " + self.TEST_SNAPHOT_ID).split(" "):
self.assertTrue(self.outstr.getvalue().count(word+ " ") or self.outstr.getvalue().count(" " + word), "snapshot command wasn't parse well")
self.assertEqual(self.outstr.getvalue().count("Job Status: "), 1)
self.assertEqual(self.outstr.getvalue().count("Job URI: "), 1)
self.assertEqual(self.outstr.getvalue().count("Target URI: "), 1)
self.assertEqual(self.outstr.getvalue().count("Target Name: "), 1)
self.assertEqual(self.outstr.getvalue().count("Response Status: "), 1)
self.assertEqual(self.outstr.getvalue().count("Response Reason: "), 1)
outLines = self.outstr.getvalue().split("\n")
for line in outLines:
if "Job URI:" in line:
jobLocation = line.split("Job URI:")[1].strip()
status = "RUNNING"
while status == "RUNNING":
(status, id) = parse_job(self.rosrs, jobLocation)
assert status == "DONE"
self.rosrs.deleteRO(id)
return
def testArchiveSynchronous(self):
args = [
"ro", "archive", ro_test_config.ROSRS_URI + self.TEST_RO_ID, ro_test_config.ROSRS_URI + self.TEST_SNAPHOT_ID,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-r", ro_test_config.ROSRS_URI,
"-v"
]
outLines = ""
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
# simple check if the verbouse mode works well
self.assertEqual(self.outstr.getvalue().count("--asynchronous"), 0, "shouldn't be asynchronous")
outLines = self.outstr.getvalue().split("\n")
for line in outLines:
if "Target URI:" in line:
id = line.split("Target URI:")[1].strip()
self.rosrs.deleteRO(id)
return
def testFreeze(self):
"""
freeze <RO-id>
"""
#preapre snaphot
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID+"/")
(status, reason, createdRoUri, manifest) = self.rosrs.createRO(self.TEST_RO_ID,
"Test RO for ROEVO", "Test Creator", "2012-09-06")
(createdSnapshotStatus, createdSnapshotId) = self.createSnapshot(createdRoUri, self.TEST_SNAPHOT_ID, False)
args = [
"ro", "freeze",str(createdSnapshotId),
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-r", ro_test_config.ROSRS_URI,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
self.assertEqual(self.outstr.getvalue().count("freeze operation finished successfully"), 1)
(status, reason) = self.rosrs.deleteRO(createdRoUri)
(status, reason) = self.rosrs.deleteRO(createdSnapshotId)
return
def FreezeNonExistetSnaphot(self):
"""
freeze <RO-id>
"""
#preapre snaphot
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID+"/")
(status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID,
"Test RO for ROEVO", "Test Creator", "2012-09-06")
self.createSnapshot(self.TEST_RO_ID+"/", self.TEST_SNAPHOT_ID, True)
args = [
"ro", "freeze", self.TEST_SNAPHOT_RO_ID + "non exited",
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-r", ro_test_config.ROSRS_URI,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == -1
self.assertEqual(self.outstr.getvalue().count("Given URI isn't correct"), 0)
(status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID+"/")
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
return
def testRemoteStatusSnapshotRO(self):
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason, createdRoUri, manifest) = self.rosrs.createRO(self.TEST_RO_ID,
"Test RO for ROEVO", "Test Creator", "2012-09-06")
(status, reason) = self.rosrs.deleteRO(self.TEST_SNAPHOT_ID + "/")
(createdSnapshotStatus, createdSnapshotUri) = self.createSnapshot(createdRoUri, self.TEST_SNAPHOT_ID, True)
args = [
"ro", "status", str(createdSnapshotUri),
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
outtxt = self.outstr.getvalue()
self.assertEqual(outtxt.count("SNAPSHOT"), 1)
(status, reason) = self.rosrs.deleteRO(createdSnapshotUri)
(status, reason) = self.rosrs.deleteRO(createdRoUri)
return
def testRemoteStatusArchiveRO(self):
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason, createdRoUri, manifest) = self.rosrs.createRO(self.TEST_RO_ID,
"Test RO for ROEVO", "Test Creator", "2012-09-06")
(status, reason) = self.rosrs.deleteRO(self.TEST_ARCHIVE_ID + "/")
(createdArchiveStatus, createdArchiveUri) = self.createArchive(createdRoUri, self.TEST_ARCHIVE_ID, True)
args = [
"ro", "status", str(createdArchiveUri),
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
outtxt = self.outstr.getvalue()
self.assertEqual(outtxt.count("ARCHIVE"), 1)
(status, reason) = self.rosrs.deleteRO(createdArchiveUri)
(status, reason) = self.rosrs.deleteRO(createdRoUri)
return
def testRemoteStatusUndefinedRO(self):
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason, createdRoUri, manifest) = self.rosrs.createRO(self.TEST_RO_ID,
"Test RO for ROEVO", "Test Creator", "2012-09-06")
(status, reason) = self.rosrs.deleteRO(self.TEST_ARCHIVE_ID + "/")
(createdArchiveStatus, createdArchiveUri) = self.createArchive(createdRoUri, self.TEST_ARCHIVE_ID, False)
args = [
"ro", "status", str(createdArchiveUri),
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
outtxt = self.outstr.getvalue()
self.assertEqual(outtxt.count("UNDEFINED"), 1)
(status, reason) = self.rosrs.deleteRO(createdArchiveUri)
(status, reason) = self.rosrs.deleteRO(createdRoUri)
return
def testRemoteStatusLiveRO(self):
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
(status, reason) = self.rosrs.deleteRO(self.TEST_RO_ID+"/")
(status, reason, rouri, manifest) = self.rosrs.createRO(self.TEST_RO_ID,
"Test RO for ROEVO", "Test Creator", "2012-09-06")
args = [
"ro", "status", str(rouri),
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
outtxt = self.outstr.getvalue()
self.assertEqual(outtxt.count("LIVE"), 1)
(status, reason) = self.rosrs.deleteRO(rouri)
return
def testRemoteStatusWithWrongUriGiven(self):
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
self.rosrs.deleteRO(self.TEST_RO_ID + "/")
self.rosrs.deleteRO("some-strange-uri/")
args = [
"ro", "status", ro_test_config.ROSRS_URI + "some-strange-uri/",
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
outtxt = self.outstr.getvalue()
assert status == -1
self.assertEqual(outtxt.count("Wrong URI was given"), 1)
self.rosrs.deleteRO(self.TEST_RO_ID + "/")
return
def remote_status(self, ro_uri):
rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, ro_test_config.ROSRS_ACCESS_TOKEN)
service_uri = ro_uri
(status, reason, headers, data) = rosrs.doRequest(uripath=service_uri, method="POST", body="", ctype="application/json", reqheaders={})
return status, reason, headers, data
<roterms:resource rdf:resource="&BUNDLE;workflow/Create_SNP_Set/in/Entrez_ID" />
</rdf:Description>
</roterms:inputValue>
<roterms:inputValue>
<rdf:Description>
<roterms:portName>set_width</roterms:portName>
<roterms:resource rdf:resource="&BUNDLE;workflow/Create_SNP_Set/in/set_width" />
</rdf:Description>
</roterms:inputValue>
<roterms:inputValue>
<rdf:Description>
<roterms:portName>path_to_output_file</roterms:portName>
<roterms:resource rdf:resource="&BUNDLE;workflow/Create_SNP_Set/in/path_to_output_file" />
</rdf:Description>
</roterms:inputValue>
</rdf:Description>
</rdf:RDF>
"""
# Main program script
if __name__ == "__main__":
# Set up ROSRS session and key values
rosrs = ROSRS_Session(ROSRS_API_URI, accesskey=ROSRS_ACCESS_TOKEN)
rouri = getResourceUri(ROSRS_API_URI, RO_PATH)
resuri = getResourceUri(rouri, RES_PATH)
# Add annotation
updateAnnotation(rosrs, rouri, resuri, ANNOTATION_PATH, ANNOTATION_BODY)
# Finish up
rosrs.close()
class ro_metadata(object):
"""
Class for accessing RO metadata
"""
def __init__(self, roconfig, roref, dummysetupfortest=False):
"""
Initialize: read manifest from object at given directory into local RDF graph
roconfig is the research object manager configuration, supplied as a dictionary
roref a URI reference that refers to the Research Object to be accessed, or
relative path name (see ro_uriutils.resolveFileAsUri for interpretation)
dummysetupfortest is an optional parameter that, if True, suppresses some aspects of
the setup (does not attempt to read a RO manifest) for isolated testing.
"""
self.roconfig = roconfig
self.roref = roref
self.dummyfortest = dummysetupfortest
self.manifestgraph = None
self.roannotations = None
self.registries = None
uri = resolveFileAsUri(roref)
if not uri.endswith("/"): uri += "/"
self.rouri = rdflib.URIRef(uri)
if self._isLocal():
self.rosrs = None
else:
self.rosrs = ROSRS_Session(
self.roconfig["rosrs_uri"],
self.roconfig["rosrs_access_token"]
)
self._loadManifest()
# Get RO URI from manifest
# May be different from computed value if manifest has absolute URI
self.rouri = self.manifestgraph.value(None, RDF.type, RO.ResearchObject)
# Check that the manifest contained at least one RO URI
assert self.rouri is not None
return
def _isLocal(self):
return isFileUri(self.rouri)
def _getManifestUri(self):
assert self._isLocal()
return self.getComponentUri(ro_settings.MANIFEST_DIR+"/"+ro_settings.MANIFEST_FILE)
def _loadManifest(self):
if self.manifestgraph: return self.manifestgraph
if self.dummyfortest:
# Fake minimal manifest graph for testing
self.manifestgraph = rdflib.Graph()
self.manifestgraph.add( (self.rouri, RDF.type, RO.ResearchObject) )
elif self._isLocal():
# Read manifest graph
self.manifestgraph = rdflib.Graph()
self.manifestgraph.parse(self._getManifestUri())
else:
(status, reason, _h, _u, manifest) = self.rosrs.getROManifest(self.rouri)
assert status == 200, ("ro_metadata: Can't access manifest for %s (%03d %s)"%
(str(self.rouri), status, reason))
self.manifestgraph = manifest
return self.manifestgraph
def _updateManifest(self):
"""
Write updated manifest file for research object
"""
assert self._isLocal()
self._loadManifest().serialize(
destination=self.getManifestFilename(), format='xml',
base=self.rouri, xml_base="..")
return
def _iterAnnotations(self, subject=None):
"""
Return iterator over annotation stubs in the current RO, either for
the specified subject resource, or for all annotations in the RO
subject is URI of subject whose annotations are returned, or None.
"""
manifest = self._loadManifest()
if self._isLocal():
for (anode, p, subject) in manifest:
if p in [RO.annotatesAggregatedResource, AO.annotatesResource]:
yield anode
else:
for anode in self.rosrs.getROAnnotationUris(self.getRoUri(), subject):
yield anode
return
def isAggregatedResource(self, rofile):
'''
Returns true if the manifest says that the research object aggregates the
resource. Resource URI is resolved against the RO URI unless it's absolute.
'''
resuri = self.getComponentUriAbs(rofile)
return (self.rouri, ORE.aggregates, resuri) in self.manifestgraph
def _loadAnnotations(self):
if self.roannotations: return self.roannotations
log.debug("_loadannotations")
# Assemble annotation graph
# NOTE: the manifest itself is included as an annotation by the RO setup
if self._isLocal():
manifest = self._loadManifest()
self.roannotations = rdflib.Graph()
annotation_uris_loaded = set()
for anode in self._iterAnnotations():
auri = manifest.value(subject=anode, predicate=AO.body)
if auri not in annotation_uris_loaded:
self._readAnnotationBody(auri, self.roannotations)
annotation_uris_loaded.add(auri)
else:
self.roannotations = self.rosrs.getROAnnotationGraph(self.rouri)
log.debug("roannotations graph:\n"+self.roannotations.serialize())
return self.roannotations
def isInternalResource(self, resuri):
'''
Check if the resource is internal, i.e. should the resource content be uploaded
to the ROSR service. Returns true if the resource URI has the RO URI as a prefix.
'''
return resuri.startswith(self.rouri)
def isExternalResource(self, resuri):
'''
Check if the resource is external, i.e. can be aggregated as a URI reference.
Returns true if the URI has 'http' or 'https' scheme.
'''
parseduri = urlparse.urlsplit(resuri)
return parseduri.scheme in ["http", "https"]
def _createAnnotationBody(self, roresource, attrdict, defaultType="string"):
"""
Create a new annotation body for a single resource in a research object, based
on a supplied graph value.
Existing annotations for the same resource are not touched; if an annotation is being
added or replaced, it is the calling program'sresponsibility to update the manifest to
reference the active annotations. A new name is allocated for the created annotation,
graph which is returned as the result of this function.
roresource is the name of the Research Object component to be annotated,
possibly relative to the RO root directory.
attrdict is a dictionary of attributes to be saved in the annotation body.
Dictionary keys are attribute names that can be resolved via
ro_annotation.getAnnotationByName.
Returns the name of the annotation body created relative to the RO directory.
"""
assert self._isLocal()
af = ro_annotation.createAnnotationBody(
self.roconfig, self.getRoFilename(), roresource, attrdict, defaultType)
return os.path.join(ro_settings.MANIFEST_DIR+"/", af)
def _createAnnotationGraphBody(self, roresource, anngraph):
"""
Create a new annotation body for a single resource in a research object, based
on a supplied graph value.
Existing annotations for the same resource are not touched; if an annotation is being
added or replaced, it is the calling program'sresponsibility to update the manifest to
reference the active annotations. A new name is allocated for the created annotation,
graph which is returned as the result of this function.
roresource is the name of the Research Object component to be annotated,
possibly relative to the RO root directory.
anngraph is an annotation graph that is to be saved.
Returns the name of the annotation body created relative to the RO
manifest and metadata directory.
"""
assert self._isLocal()
af = ro_annotation.createAnnotationGraphBody(
self.roconfig, self.getRoFilename(), roresource, anngraph)
return os.path.join(ro_settings.MANIFEST_DIR+"/", af)
def _readAnnotationBody(self, annotationref, anngr=None):
"""
Read annotation body from indicated resource, return RDF Graph of annotation values.
annotationref is a URI reference of an annotation, possibly relative to the RO base URI
(e.g. as returned by _createAnnotationBody method).
anngr if supplied, if an RDF graph to which the annotations are added
"""
assert self._isLocal()
log.debug("_readAnnotationBody %s"%(annotationref))
annotationuri = self.getComponentUri(annotationref)
annotationformat = "xml"
# Look at file extension to figure format
# (rdflib.Graph.parse says;
# "used if format can not be determined from the source")
if re.search("\.(ttl|n3)$", annotationuri): annotationformat="n3"
if anngr == None:
log.debug("_readAnnotationBody: new graph")
anngr = rdflib.Graph()
try:
anngr.parse(annotationuri, format=annotationformat)
log.debug("_readAnnotationBody parse %s, len %i"%(annotationuri, len(anngr)))
except IOError, e:
log.debug("_readAnnotationBody "+annotationref+", "+repr(e))
anngr = None
return anngr
class TestROSRS_Session(unittest.TestCase):
"""
This test suite tests the ROSRS_Session client implementation of the ROSRS API
"""
def setUp(self):
super(TestROSRS_Session, self).setUp()
self.rosrs = ROSRS_Session(Config.ROSRS_API_URI,
accesskey=Config.AUTHORIZATION)
# Clean up from previous runs
self.rosrs.deleteRO(Config.TEST_RO_PATH, purge=True)
self.createdTestRO = None
return
def tearDown(self):
super(TestROSRS_Session, self).tearDown()
# Clean up
self.rosrs.deleteRO(Config.TEST_RO_PATH)
if self.createdTestRO:
self.rosrs.deleteRO(self.createdTestRO, purge=True)
self.rosrs.close()
return
def createTestRO(self):
(status, reason, rouri, manifest) = self.rosrs.createRO(Config.TEST_RO_NAME,
"Test RO for ROSRS_Session", "TestROSRS_Session.py", "2012-09-06")
self.assertEqual(status, 201)
self.createdTestRO = rouri
return (status, reason, rouri, manifest)
# Actual tests follow
def testHelpers(self):
testSplitValues()
testParseLinks()
return
def testListROs(self):
ros = self.rosrs.listROs()
return
def testCreateRO(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
self.assertEqual(str(rouri)[:len(Config.TEST_RO_URI)-1]+"/", Config.TEST_RO_URI)
self.assertIn((rouri, RDF.type, RO.ResearchObject), manifest)
rolist = self.rosrs.listROs()
self.assertIn(str(rouri), [ r["uri"] for r in rolist ])
return
def testDeleteRO(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Test that new RO is in collection
rolist = self.rosrs.listROs()
self.assertIn(str(rouri), [ r["uri"] for r in rolist ])
# Delete RO
(status, reason) = self.rosrs.deleteRO(rouri)
self.assertEqual(status, 204)
self.assertEqual(reason, "No Content")
# Test that new RO is not in collection
rolist = self.rosrs.listROs()
self.assertNotIn(str(rouri), [ r["uri"] for r in rolist ])
# Delete again
(status, reason) = self.rosrs.deleteRO(rouri)
self.assertEqual(status, 404)
self.assertEqual(reason, "Not Found")
return
def testGetROManifest(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Get manifest
(status, reason, headers, manifesturi, manifest) = self.rosrs.getROManifest(rouri)
self.assertEqual(status, 200)
self.assertEqual(reason, "OK")
self.assertEqual(headers["content-type"], "application/rdf+xml")
# Check manifest RDF graph
self.assertIn((rouri, RDF.type, RO.ResearchObject), manifest)
self.assertIn((rouri, DCTERMS.creator, None), manifest)
self.assertIn((rouri, DCTERMS.created, None), manifest)
self.assertIn((rouri, ORE.isDescribedBy, manifesturi), manifest)
return
def testGetROPage(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Get landing page
(status, reason, headers, pageuri, page) = self.rosrs.getROLandingPage(rouri)
self.assertEqual(status, 200)
self.assertEqual(reason, "OK")
self.assertEqual(headers["content-type"], "text/html;charset=UTF-8")
return
def testGetROZip(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Get manifest
(status, reason, headers, datauri, data) = self.rosrs.getROZip(rouri)
self.assertEqual(status, 200)
self.assertEqual(reason, "OK")
self.assertEqual(headers["content-type"], "application/zip")
# @@TODO test content of zip (data)?
return
def testAggregateResourceInt(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Aggregate internal resource
rescontent = "Resource content\n"
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt(
rouri, "test/path", ctype="text/plain", body=rescontent)
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
self.assertEqual(str(resuri), str(rouri)+"test/path")
# GET content
(status, reason, headers, uri, data) = self.rosrs.getROResource(
"test/path", rouri)
self.assertEqual(status, 200)
self.assertEqual(headers["content-type"], "text/plain")
self.assertEqual(data, rescontent)
# GET proxy
(getproxyuri, manifest) = self.rosrs.getROResourceProxy(
"test/path", rouri=rouri)
self.assertEqual(getproxyuri, proxyuri)
return
def testDeleteResourceInt(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Create test resource
rescontent = "Resource content\n"
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt(
rouri, "test/path", ctype="text/plain", body=rescontent)
self.assertEqual(status, 201)
# GET content
(status, reason, headers, uri, data) = self.rosrs.getROResource(
"test/path", rouri)
self.assertEqual(status, 200)
# Delete resource
(status, reason) = self.rosrs.removeResource(rouri, resuri)
self.assertEqual(status, 204)
self.assertEqual(reason, "No Content")
# Check that resource is no longer available
(status, reason, headers, uri, data) = self.rosrs.getROResource(resuri)
self.assertEqual(status, 404)
return
def testAggregateResourceExt(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Aggregate external resource
externaluri = rdflib.URIRef("http://example.com/external/resource.txt")
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceExt(
rouri, externaluri)
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
self.assertEqual(resuri, externaluri)
# GET proxy (note: using rdflib.URIRef value for path)
(getproxyuri, manifest) = self.rosrs.getROResourceProxy(
externaluri, rouri)
self.assertEqual(getproxyuri, proxyuri)
return
def testDeleteResourceExt(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Create test resource
externaluri = rdflib.URIRef("http://example.com/external/resource.txt")
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceExt(
rouri, externaluri)
self.assertEqual(status, 201)
self.assertEqual(resuri, externaluri)
# GET proxy (note: using rdflib.URIRef for path)
(getproxyuri, manifest) = self.rosrs.getROResourceProxy(
externaluri, rouri)
self.assertEqual(getproxyuri, proxyuri)
# Delete resource
(status, reason) = self.rosrs.removeResource(rouri, resuri)
self.assertEqual(status, 204)
self.assertEqual(reason, "No Content")
(getproxyuri, manifest) = self.rosrs.getROResourceProxy(
externaluri, rouri)
self.assertIsNone(getproxyuri)
self.assertIsNotNone(manifest)
return
def testGetROResource(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Create test resource
rescontent = "Resource content\n"
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt(
rouri, "test/path", ctype="text/plain", body=rescontent)
self.assertEqual(status, 201)
# GET content
(status, reason, headers, uri, data) = self.rosrs.getROResource(
"test/path", rouri)
self.assertEqual(status, 200)
self.assertEqual(reason, "OK")
self.assertEqual(headers["content-type"], "text/plain")
self.assertEqual(data, rescontent)
return
def testGetROResourceRDF(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Create internal test resource
rescontent = """<?xml version="1.0" encoding="UTF-8"?>
<rdf:RDF
xmlns:dct="http://purl.org/dc/terms/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
>
<rdf:Description rdf:about="http://example.org/file1.txt">
<dct:title>Title for file1.txt</dct:title>
</rdf:Description>
</rdf:RDF>
"""
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt(
rouri, "test/file1.rdf", ctype="application/rdf+xml", body=rescontent)
self.assertEqual(status, 201)
# Get resource content
(status, reason, headers, uri, graph)= self.rosrs.getROResourceRDF(
"test/file1.rdf", rouri=rouri)
self.assertEqual(status, 200)
self.assertEqual(reason, "OK")
self.assertEqual(headers["content-type"], "application/rdf+xml")
s = rdflib.URIRef("http://example.org/file1.txt")
self.assertIn((s, DCTERMS.title, rdflib.Literal("Title for file1.txt")), graph)
return
def testGetROResourceProxy(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Create internal test resource
rescontent = "Resource content\n"
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt(
rouri, "test/path", ctype="text/plain", body=rescontent)
self.assertEqual(status, 201)
# Get resource proxy
(getproxyuri, manifest) = self.rosrs.getROResourceProxy(
"test/path", rouri=rouri)
self.assertEqual(getproxyuri, proxyuri)
return
def testCreateROAnnotationInt(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Create internal test resource
rescontent = "Resource content\n"
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt(
rouri, "test/file.txt", ctype="text/plain", body=rescontent)
self.assertEqual(status, 201)
# Create internal annotation
annbody = """<?xml version="1.0" encoding="UTF-8"?>
<rdf:RDF
xmlns:dct="http://purl.org/dc/terms/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xml:base="%s"
>
<rdf:Description rdf:about="test/file.txt">
<dct:title>Title for test/file.txt</dct:title>
<rdfs:seeAlso rdf:resource="http://example.org/test" />
</rdf:Description>
</rdf:RDF>
"""%(str(rouri))
agraph = rdflib.graph.Graph()
agraph.parse(data=annbody, format="xml")
(status, reason, annuri, bodyuri) = self.rosrs.createROAnnotationInt(
rouri, resuri, agraph)
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
# Retrieve annotation URIs
auris = list(self.rosrs.getROAnnotationUris(rouri, resuri))
self.assertIn(annuri, auris)
buris = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri))
### self.assertIn(bodyuri, buris)
# Retrieve annotation
(status, reason, bodyuri, anngr) = self.rosrs.getROAnnotation(annuri)
self.assertEqual(status, 200)
self.assertEqual(reason, "OK")
self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title for test/file.txt")), anngr)
self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test")), anngr)
return
def testGetROAnnotationGraph(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Create internal test resource
rescontent = "Resource content\n"
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt(
rouri, "test/file.txt", ctype="text/plain", body=rescontent)
self.assertEqual(status, 201)
# Create internal annotation
annbody = """<?xml version="1.0" encoding="UTF-8"?>
<rdf:RDF
xmlns:dct="http://purl.org/dc/terms/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xml:base="%s"
>
<rdf:Description rdf:about="test/file.txt">
<dct:title>Title for test/file.txt</dct:title>
<rdfs:seeAlso rdf:resource="http://example.org/test" />
</rdf:Description>
</rdf:RDF>
"""%(str(rouri))
agraph = rdflib.graph.Graph()
agraph.parse(data=annbody, format="xml")
(status, reason, annuri, bodyuri) = self.rosrs.createROAnnotationInt(
rouri, resuri, agraph)
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
# Retrieve merged annotations
anngr = self.rosrs.getROAnnotationGraph(rouri, resuri)
annts = list(anngr.triples((None, None, None)))
self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title for test/file.txt")), annts)
self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test")), annts)
return
def testCreateROAnnotationExt(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Create external test resource
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceExt(
rouri, rdflib.URIRef("http://example.org/ext"))
self.assertEqual(status, 201)
# Create annotation using external body reference
bodyuri = rdflib.URIRef("http://example.org/ext/ann.rdf")
(status, reason, annuri) = self.rosrs.createROAnnotationExt(rouri, resuri, bodyuri)
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
# Retrieve annotation URIs
auris = list(self.rosrs.getROAnnotationUris(rouri, resuri))
self.assertIn(annuri, auris)
buris = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri))
### self.assertIn(bodyuri, buris)
return
def testUpdateROAnnotationInt(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Create internal test resource
rescontent = "Resource content\n"
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt(
rouri, "test/file.txt", ctype="text/plain", body=rescontent)
self.assertEqual(status, 201)
# Create internal annotation
annbody1 = """<?xml version="1.0" encoding="UTF-8"?>
<rdf:RDF
xmlns:dct="http://purl.org/dc/terms/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xml:base="%s"
>
<rdf:Description rdf:about="test/file.txt">
<dct:title>Title 1</dct:title>
<rdfs:seeAlso rdf:resource="http://example.org/test1" />
</rdf:Description>
</rdf:RDF>
"""%(str(rouri))
agraph1 = rdflib.graph.Graph()
agraph1.parse(data=annbody1, format="xml")
(status, reason, annuri, bodyuri1) = self.rosrs.createROAnnotationInt(
rouri, resuri, agraph1)
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
# Retrieve annotation URIs
auris1 = list(self.rosrs.getROAnnotationUris(rouri, resuri))
self.assertIn(annuri, auris1)
buris1 = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri))
### self.assertIn(bodyuri1, buris1)
# Retrieve annotation
(status, reason, auri1, anngr1a) = self.rosrs.getROAnnotation(annuri)
self.assertEqual(status, 200)
self.assertEqual(reason, "OK")
annts1a = list(anngr1a.triples((None, None, None)))
self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title 1")), annts1a)
self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test1")), annts1a)
# Retrieve merged annotations
anngr1b = self.rosrs.getROAnnotationGraph(rouri, resuri)
annts1b = list(anngr1b.triples((None, None, None)))
self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title 1")), annts1b)
self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test1")), annts1b)
# Update internal annotation
annbody2 = """<?xml version="1.0" encoding="UTF-8"?>
<rdf:RDF
xmlns:dct="http://purl.org/dc/terms/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xml:base="%s"
>
<rdf:Description rdf:about="test/file.txt">
<dct:title>Title 2</dct:title>
<rdfs:seeAlso rdf:resource="http://example.org/test2" />
</rdf:Description>
</rdf:RDF>
"""%(str(rouri))
agraph2 = rdflib.graph.Graph()
agraph2.parse(data=annbody2, format="xml")
(status, reason, bodyuri2) = self.rosrs.updateROAnnotationInt(
rouri, annuri, resuri, agraph2)
self.assertEqual(status, 200)
self.assertEqual(reason, "OK")
# Retrieve annotation URIs
auris2 = list(self.rosrs.getROAnnotationUris(rouri, resuri))
self.assertIn(annuri, auris2)
buris2 = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri))
### self.assertIn(bodyuri2, buris2)
# Retrieve annotation
(status, reason, auri2a, anngr2a) = self.rosrs.getROAnnotation(annuri)
annts2a = list(anngr2a.triples((None, None, None)))
self.assertEqual(status, 200)
self.assertEqual(reason, "OK")
self.assertNotIn((resuri, DCTERMS.title, rdflib.Literal("Title 1")), annts2a)
self.assertNotIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test1")), annts2a)
self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title 2")), annts2a)
self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test2")), annts2a)
# Retrieve merged annotations
anngr2b = self.rosrs.getROAnnotationGraph(rouri, resuri)
annts2b = list(anngr2b.triples((None, None, None)))
self.assertNotIn((resuri, DCTERMS.title, rdflib.Literal("Title 1")), annts2b)
self.assertNotIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test1")), annts2b)
self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title 2")), annts2b)
self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test2")), annts2b)
return
def testUpdateROAnnotationExt(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Create external test resource
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceExt(
rouri, rdflib.URIRef("http://example.org/ext"))
self.assertEqual(status, 201)
# Create annotation using external body reference
bodyuri1 = rdflib.URIRef("http://example.org/ext/ann1.rdf")
(status, reason, annuri) = self.rosrs.createROAnnotationExt(rouri, resuri, bodyuri1)
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
# Retrieve annotation URIs
auris1 = list(self.rosrs.getROAnnotationUris(rouri, resuri))
self.assertIn(annuri, auris1)
buris1 = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri))
self.assertIn(bodyuri1, buris1)
# Update annotation using external body reference
# @@TODO - this doesn't check that old annotation is removed.
# @@TODO - currently, update is not fully implemented (2013-05).
bodyuri2 = rdflib.URIRef("http://example.org/ext/ann2.rdf")
(status, reason, annuri) = self.rosrs.createROAnnotationExt(rouri, resuri, bodyuri2)
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
# Retrieve annotation URIs
auris2 = list(self.rosrs.getROAnnotationUris(rouri, resuri))
self.assertIn(annuri, auris2)
buris2 = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri))
self.assertIn(bodyuri1, buris2)
return
def testRemoveROAnnotation(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
# Create internal test resource
rescontent = "Resource content\n"
(status, reason, proxyuri, resuri) = self.rosrs.aggregateResourceInt(
rouri, "test/file.txt", ctype="text/plain", body=rescontent)
self.assertEqual(status, 201)
# Create internal annotation
annbody = """<?xml version="1.0" encoding="UTF-8"?>
<rdf:RDF
xmlns:dct="http://purl.org/dc/terms/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xml:base="%s"
>
<rdf:Description rdf:about="test/file.txt">
<dct:title>Title for test/file.txt</dct:title>
<rdfs:seeAlso rdf:resource="http://example.org/test" />
</rdf:Description>
</rdf:RDF>
"""%(str(rouri))
agraph = rdflib.graph.Graph()
agraph.parse(data=annbody, format="xml")
(status, reason, annuri, bodyuri) = self.rosrs.createROAnnotationInt(
rouri, resuri, agraph)
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
# Retrieve annotation URIs
auris = list(self.rosrs.getROAnnotationUris(rouri, resuri))
self.assertIn(annuri, auris)
buris = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri))
### self.assertIn(bodyuri, buris)
# Remove the annotation
(status, reason) = self.rosrs.removeROAnnotation(rouri, annuri)
self.assertEqual(status, 204)
self.assertEqual(reason, "No Content")
# Retrieve annotation URIs
auris = list(self.rosrs.getROAnnotationUris(rouri, resuri))
self.assertNotIn(annuri, auris)
buris = list(self.rosrs.getROAnnotationBodyUris(rouri, resuri))
### self.assertNotIn(bodyuri, buris)
return
# Evolution tests
def testCopyRO(self):
return
def testCancelCopyRO(self):
return
def testUpdateROStatus(self):
return
def testGetROEvolution(self):
return
# Sentinel/placeholder tests
def testUnits(self):
assert (True)
def testComponents(self):
assert (True)
def testIntegration(self):
assert (True)
def testPending(self):
assert (False), "Pending tests follow"
class ro_metadata(object):
"""
Class for accessing RO metadata
"""
def __init__(self, roconfig, roref, dummysetupfortest=False):
"""
Initialize: read manifest from object at given directory into local RDF graph
roconfig is the research object manager configuration, supplied as a dictionary
roref a URI reference that refers to the Research Object to be accessed, or
relative path name (see ro_uriutils.resolveFileAsUri for interpretation)
dummysetupfortest is an optional parameter that, if True, suppresses some aspects of
the setup (does not attempt to read a RO manifest) for isolated testing.
"""
self.roconfig = roconfig
self.roref = roref
self.dummyfortest = dummysetupfortest
self.manifestgraph = None
self.roannotations = None
self.registries = None
uri = resolveFileAsUri(roref)
if not uri.endswith("/"): uri += "/"
self.rouri = rdflib.URIRef(uri)
if self._isLocal():
self.rosrs = None
else:
self.rosrs = ROSRS_Session(self.roconfig["rosrs_uri"],
self.roconfig["rosrs_access_token"])
self._loadManifest()
# Get RO URI from manifest
# May be different from computed value if manifest has absolute URI
# Nested URIs may be present; ours is the one described by the manifest URI,
# which is determined by the _loadManifest() method.
for s in self.manifestgraph.subjects(RDF.type, RO.ResearchObject):
if self.manifestgraph.value(s,
ORE.isDescribedBy) == self.manifesturi:
self.rouri = s
# Check that the manifest contained at least one RO URI
assert self.rouri is not None
return
def _isLocal(self):
return isFileUri(self.rouri)
def _getLocalManifestUri(self):
return self.getComponentUri(ro_settings.MANIFEST_DIR + "/" +
ro_settings.MANIFEST_FILE)
def _loadManifest(self):
if self.manifestgraph != None: return self.manifestgraph
if self.dummyfortest:
# Fake minimal manifest graph for testing
self.manifestgraph = rdflib.Graph()
self.manifestgraph.add((self.rouri, RDF.type, RO.ResearchObject))
self.manifesturi = self.rouri
elif self._isLocal():
# Read manifest graph
self.manifestgraph = rdflib.Graph()
for (prefix, uri) in ro_prefixes.prefixes:
self.manifestgraph.bind(prefix,
rdflib.namespace.Namespace(uri))
self.manifesturi = self._getLocalManifestUri()
self.manifestgraph.parse(self.manifesturi)
else:
(status, reason, _h, manifesturi,
manifest) = self.rosrs.getROManifest(self.rouri)
if status != 200:
msg = ("Can't access RO manifest (%03d %s)" % (status, reason))
raise ROSRS_Error(msg=msg, srsuri=self.rouri)
self.manifestgraph = manifest
self.manifesturi = manifesturi
# log.debug("romanifest graph:\n"+self.manifestgraph.serialize())
return self.manifestgraph
def getManifestGraph(self):
"""
Returns the manifest graph
"""
return self._loadManifest()
def _updateManifest(self):
"""
Write updated manifest file for research object
"""
assert self._isLocal()
self._loadManifest().serialize(destination=self.getManifestFilename(),
format='xml',
base=self.rouri,
xml_base="..")
return
def _iterAnnotations(self, subject=None):
"""
Return iterator over annotation stubs in the current RO, either for
the specified subject resource, or for all annotations in the RO
subject is URI of subject whose annotations are returned, or None.
"""
manifest = self._loadManifest()
if self._isLocal():
for (anode, p, subject) in manifest:
if p in [RO.annotatesAggregatedResource, AO.annotatesResource]:
yield anode
else:
for anode in self.rosrs.getROAnnotationUris(
self.getRoUri(), subject):
yield anode
return
def isAggregatedResource(self, rofile):
'''
Returns true if the manifest says that the research object aggregates the
resource. Resource URI is resolved against the RO URI unless it's absolute.
'''
resuri = self.getComponentUriAbs(rofile)
return (self.rouri, ORE.aggregates, resuri) in self.manifestgraph
def _loadAnnotations(self):
if self.roannotations: return self.roannotations
log.debug("_loadannotations")
# Assemble annotation graph
# NOTE: the manifest itself is included as an annotation by the RO setup
if self._isLocal():
manifest = self._loadManifest()
self.roannotations = rdflib.Graph()
annotation_uris_loaded = set()
for anode in self._iterAnnotations():
auri = manifest.value(subject=anode, predicate=AO.body)
if auri not in annotation_uris_loaded:
aref = self.getComponentUriRel(auri)
log.debug("_loadAnnotations: aref " + str(aref))
self._readAnnotationBody(aref, self.roannotations)
annotation_uris_loaded.add(auri)
else:
self.roannotations = self.rosrs.getROAnnotationGraph(self.rouri)
# log.debug("roannotations graph:\n"+self.roannotations.serialize())
for (prefix, uri) in ro_prefixes.prefixes:
self.manifestgraph.bind(prefix, rdflib.namespace.Namespace(uri))
return self.roannotations
def isInternalResource(self, resuri):
'''
Check if the resource is internal, i.e. should the resource content be uploaded
to the ROSR service. Returns true if the resource URI has the RO URI as a prefix.
'''
return resuri.startswith(self.rouri)
def isExternalResource(self, resuri):
'''
Check if the resource is external, i.e. can be aggregated as a URI reference.
Returns true if the URI has 'http' or 'https' scheme.
'''
parseduri = urlparse.urlsplit(resuri)
return parseduri.scheme in ["http", "https"]
def _createAnnotationBody(self,
roresource,
attrdict,
defaultType="string"):
"""
Create a new annotation body for a single resource in a research object, based
on a supplied graph value.
Existing annotations for the same resource are not touched; if an annotation is being
added or replaced, it is the calling program'sresponsibility to update the manifest to
reference the active annotations. A new name is allocated for the created annotation,
graph which is returned as the result of this function.
roresource is the name of the Research Object component to be annotated,
possibly relative to the RO root directory.
attrdict is a dictionary of attributes to be saved in the annotation body.
Dictionary keys are attribute names that can be resolved via
ro_annotation.getAnnotationByName.
Returns the name of the annotation body created relative to the RO directory.
"""
assert self._isLocal()
af = ro_annotation.createAnnotationBody(self.roconfig,
self.getRoFilename(),
roresource, attrdict,
defaultType)
return os.path.join(ro_settings.MANIFEST_DIR + "/", af)
def _createAnnotationGraphBody(self, roresource, anngraph):
"""
Create a new annotation body for a single resource in a research object, based
on a supplied graph value.
Existing annotations for the same resource are not touched; if an annotation is being
added or replaced, it is the calling program'sresponsibility to update the manifest to
reference the active annotations. A new name is allocated for the created annotation,
graph which is returned as the result of this function.
roresource is the name of the Research Object component to be annotated,
possibly relative to the RO root directory.
anngraph is an annotation graph that is to be saved.
Returns the name of the annotation body created relative to the RO
manifest and metadata directory.
"""
assert self._isLocal()
af = ro_annotation.createAnnotationGraphBody(self.roconfig,
self.getRoFilename(),
roresource, anngraph)
return os.path.join(ro_settings.MANIFEST_DIR + "/", af)
def _readAnnotationBody(self, annotationref, anngr=None):
"""
Read annotation body from indicated resource, return RDF Graph of annotation values.
annotationref is a URI reference of an annotation, possibly relative to the RO base URI
(e.g. as returned by _createAnnotationBody method).
anngr if supplied, if an RDF graph to which the annotations are added
"""
assert self._isLocal()
log.debug("_readAnnotationBody %s" % (annotationref))
annotationuri = self.getComponentUri(annotationref)
annotationformat = "xml"
# Look at file extension to figure format
# (rdflib.Graph.parse says;
# "used if format can not be determined from the source")
if re.search("\.(ttl|n3)$", annotationuri): annotationformat = "n3"
if anngr == None:
log.debug("_readAnnotationBody: new graph")
anngr = rdflib.Graph()
try:
anngr.parse(annotationuri, format=annotationformat)
log.debug("_readAnnotationBody parse %s, len %i" %
(annotationuri, len(anngr)))
except IOError as e:
log.debug("_readAnnotationBody %s, %s" %
(str(annotationref), repr(e)))
anngr = None
except Exception as e:
log.debug("Failed to load annotation %s as %s" %
(annotationuri, annotationformat))
log.debug("Exception %s" % (repr(e)))
raise
anngr = None
return anngr
def _addAnnotationToManifest(self, rofile, annfile):
"""
Add a new annotation body to an RO graph
rofile is the research object resource being annotated
annfile is the file name of the annotation body to be added,
possibly relative to the RO URI, with special characters
already URI-escaped.
"""
assert self._isLocal()
# <ore:aggregates>
# <ro:AggregatedAnnotation>
# <ro:annotatesAggregatedResource rdf:resource="data/UserRequirements-astro.ods" />
# <ao:body rdf:resource=".ro/(annotation).rdf" />
# </ro:AggregatedAnnotation>
# </ore:aggregates>
log.debug("_addAnnotationToManifest annfile %s" % (annfile))
ann = rdflib.BNode()
resuri = self.getComponentUri(rofile)
bodyuri = self.getComponentUriAbs(annfile)
self.manifestgraph.add((ann, RDF.type, RO.AggregatedAnnotation))
self.manifestgraph.add((ann, RO.annotatesAggregatedResource, resuri))
self.manifestgraph.add((ann, AO.body, bodyuri))
# Aggregate the annotation
self.manifestgraph.add((self.getRoUri(), ORE.aggregates, ann))
# Aggregate annotation body if it is RO metadata.
# Otherwise aggregation is the caller's responsibility
if self.isRoMetadataRef(bodyuri):
self.manifestgraph.add((self.getRoUri(), ORE.aggregates, bodyuri))
self.roannotations = None # Flush cached annotation graph
return
def _removeAnnotationFromManifest(self, ann):
"""
Remove references to an annotation from an RO graph
ann is the the annotation node to be removed
"""
assert self._isLocal()
bodyuri = self.manifestgraph.value(subject=ann, predicate=AO.body)
self.manifestgraph.remove((ann, None, None))
# If annotation body is RO Metadata, and there are no other uses as an annotation,
# remove it from the RO aggregation.
if self.isRoMetadataRef(bodyuri):
if not self.manifestgraph.value(subject=ann, predicate=AO.body):
self.manifestgraph.remove((None, ORE.aggregates, bodyuri))
self.roannotations = None # Flush cached annotation graph
return
def addAggregatedResources(self, ro_file, recurse=True, includeDirs=False):
"""
Scan a local directory and add files found to the RO aggregation
"""
assert self._isLocal()
def notHidden(f):
return re.match("\.|.*/\.", f) == None
log.debug("addAggregatedResources: roref %s, file %s" %
(self.roref, ro_file))
self.getRoFilename() # Check that we have one
basedir = os.path.abspath(self.roref) + os.path.sep
### print "- ro_file: %s"%(ro_file)
if os.path.isdir(ro_file):
ro_file = os.path.abspath(ro_file) + os.path.sep
### print "- ro_file: %s"%(ro_file)
### print "- basedir: %s"%(basedir)
#if ro_file.endswith(os.path.sep):
# ro_file = ro_file[0:-1]
if recurse:
rofiles = filter(
notHidden,
MiscUtils.ScanDirectories.CollectDirectoryContents(
ro_file,
baseDir=basedir,
listDirs=includeDirs,
listFiles=True,
recursive=recurse,
appendSep=True))
log.debug("- rofiles: %s" % (repr(rofiles)))
else:
rofiles = [ro_file.split(basedir + os.path.sep, 1)[-1]]
else:
rofiles = [self.getComponentUriRel(ro_file)]
s = self.getRoUri()
for f in rofiles:
### print "- file %s"%f
log.debug("- file %s" % f)
stmt = (s, ORE.aggregates, self.getComponentUri(f))
if stmt not in self.manifestgraph: self.manifestgraph.add(stmt)
self._updateManifest()
return
def removeAggregatedResource(self, resuri):
"""
Remove a specified resource.
resref is the URI of a resource to remove
"""
assert self._isLocal()
resuri = rdflib.URIRef(resuri)
log.debug("removeAggregatedResource: roref %s, resuri %s" %
(self.roref, str(resuri)))
manifest = self._loadManifest()
for anode in self._iterAnnotations(subject=resuri):
self._removeAnnotationFromManifest(anode)
manifest.remove((None, ORE.aggregates, resuri))
self._updateManifest()
return
def getAggregatedResources(self):
"""
Returns iterator over all resources aggregated by a manifest.
Each value returned by the iterator is an aggregated resource URI
"""
log.debug("getAggregatedResources: uri %s" % (self.rouri))
for r in self._loadManifest().objects(subject=self.rouri,
predicate=ORE.aggregates):
if not isinstance(r, rdflib.BNode):
yield r
return
def addGraphAnnotation(self, rofile, graph):
"""
Add an annotation graph for a named resource. Unlike addSimpleAnnotation, this
method adds an annotation to the manifest using an existing RDF graph (which is
presumably itself in the RO structure).
rofile names the file or resource to be annotated, possibly relative to the RO.
Note that no checks are performed to ensure that the graph itself actually
refers to this resource - that's up the the creator of the graph. This
identifies the resourfce with which the annotation body is associated in
the RO manifest.
graph names the file or resource containing the annotation body.
"""
assert self._isLocal()
ro_graph = self._loadManifest()
self._addAnnotationToManifest(rofile, graph)
self._updateManifest()
return
def isAnnotationNode(self, respath):
'''
Returns true if the manifest says that the research object aggregates the
annotation and it is an ro:AggregatedAnnotation.
Resource URI is resolved against the RO URI unless it's absolute.
'''
resuri = self.getComponentUriAbs(respath)
log.debug("isAnnotationNode: ro uri %s res uri %s" %
(self.rouri, resuri))
return (self.rouri, ORE.aggregates, resuri) in self.manifestgraph and \
(resuri, RDF.type, RO.AggregatedAnnotation) in self.manifestgraph
def addSimpleAnnotation(self,
rofile,
attrname,
attrvalue,
defaultType="string"):
"""
Add a simple annotation to a file in a research object.
rofile names the file or resource to be annotated, possibly relative to the RO.
attrname names the attribute in a form recognized by getAnnotationByName
attrvalue is a value to be associated with the attribute
"""
assert self._isLocal()
ro_dir = self.getRoFilename()
annfile = self._createAnnotationBody(rofile, {attrname: attrvalue},
defaultType)
self._addAnnotationToManifest(rofile, annfile)
self._updateManifest()
return annfile
def removeSimpleAnnotation(self, rofile, attrname, attrvalue):
"""
Remove a simple annotation or multiple matching annotations a research object.
rofile names the annotated file or resource, possibly relative to the RO.
attrname names the attribute in a form recognized by getAnnotationByName
attrvalue is the attribute value to be deleted, or Nomne to delete all vaues
"""
assert self._isLocal()
ro_dir = self.getRoFilename()
ro_graph = self._loadManifest()
subject = self.getComponentUri(rofile)
(predicate,
valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname)
val = attrvalue and ro_annotation.makeAnnotationValue(
self.roconfig, attrvalue, valtype)
add_annotations = []
remove_annotations = []
log.debug("removeSimpleAnnotation subject %s, predicate %s, val %s" %
(str(subject), str(predicate), val))
# Scan for annotation graph resources containing this annotation
for ann_node in self._iterAnnotations(subject=subject):
ann_uri = ro_graph.value(subject=ann_node, predicate=AO.body)
log.debug("removeSimpleAnnotation ann_uri %s" % (str(ann_uri)))
if self.isRoMetadataRef(ann_uri):
ann_graph = self._readAnnotationBody(
self.getComponentUriRel(ann_uri))
log.debug("removeSimpleAnnotation ann_graph %s" % (ann_graph))
if (subject, predicate, val) in ann_graph:
ann_graph.remove((subject, predicate, val))
if (subject, None, None) in ann_graph:
# Triples remain in annotation body: write new body and update RO graph
ann_name = self._createAnnotationGraphBody(
rofile, ann_graph)
remove_annotations.append(ann_node)
add_annotations.append(ann_name)
else:
# Remove annotation from RO graph
remove_annotations.append(ann_node)
# Update RO manifest graph if needed
if add_annotations or remove_annotations:
for a in remove_annotations:
self._removeAnnotationFromManifest(a)
for a in add_annotations:
self._addAnnotationToManifest(rofile, a)
self._updateManifest()
return
def replaceSimpleAnnotation(self, rofile, attrname, attrvalue):
"""
Replace a simple annotation in a research object.
rofile names the file or resource to be annotated, possibly relative to the RO.
attrname names the attribute in a form recognized by getAnnotationByName
attrvalue is a new value to be associated with the attribute
"""
assert self._isLocal()
ro_dir = self.getRoFilename()
ro_graph = self._loadManifest()
subject = self.getComponentUri(rofile)
(predicate,
valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname)
log.debug("Replace annotation: subject %s, predicate %s, value %s" %
(repr(subject), repr(predicate), repr(attrvalue)))
ro_graph.remove((subject, predicate, None))
ro_graph.add(
(subject, predicate,
ro_annotation.makeAnnotationValue(self.roconfig, attrvalue,
valtype)))
self._updateManifest()
self.roannotations = None # Flush cached annotation graph
return
def iterateAnnotations(self, subject=None, property=None):
"""
Returns an iterator over annotations of the current RO that match the
supplied subject and/or property.
"""
log.debug("iterateAnnotations s:%s, p:%s" %
(str(subject), str(property)))
ann_graph = self._loadAnnotations()
for (s, p, v) in ann_graph.triples((subject, property, None)):
if not isinstance(s, rdflib.BNode):
if not self.isRoMetadataRef(s):
log.debug("Triple: %s %s %s" % (s, p, v))
yield (s, p, v)
return
def getRoAnnotations(self):
"""
Returns iterator over annotations applied to the RO as an entity.
Each value returned by the iterator is a (subject,predicate,object) triple.
"""
return self.iterateAnnotations(subject=self.getRoUri())
def getFileAnnotations(self, rofile):
"""
Returns iterator over annotations applied to a specified component in the RO
Each value returned by the iterator is a (subject,predicate,object) triple.
"""
return self.iterateAnnotations(subject=self.getComponentUri(rofile))
def getAllAnnotations(self):
"""
Returns iterator over all annotations associated with the RO
Each value returned by the iterator is a (subject,predicate,object) triple.
"""
return self.iterateAnnotations()
def getAllAnnotationNodes(self):
"""
Returns iterator over all annotations aggregated within the RO
Each value returned by the iterator is a (annuri, bodyuri, target) triple.
"""
annotations = self.manifestgraph.subject_objects(
predicate=RO.annotatesAggregatedResource)
for (ann_node, ann_target) in annotations:
ann_body = self.manifestgraph.value(subject=ann_node,
predicate=AO.body)
yield (ann_node, ann_body, ann_target)
return
def getAnnotationValues(self, rofile, attrname):
"""
Returns iterator over annotation values for given subject and attribute
"""
(predicate,
valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname)
resuri = self.getComponentUri(rofile)
return (v for (
s, p,
v) in self.iterateAnnotations(subject=resuri, property=predicate))
def queryAnnotations(self, query, initBindings={}):
"""
Runs a query over the combined annotation graphs (including the manifest)
and returns True or False (for ASK queries) or a list of dictionaries of
query results (for SELECT queries).
"""
log.debug("queryAnnotations: \n----\n%s\n--------\n" % (query))
ann_gr = self._loadAnnotations()
# log.debug("queryAnnotations graph: \n----\n%s\n--------\n"%(ann_gr.serialize(format='xml')))
try:
resp = ann_gr.query(query, initBindings=initBindings)
except:
log.info("queryAnnotations failed query: \n----\n%s\n--------\n" %
(query))
raise
if resp.type == 'ASK':
return resp.askAnswer
elif resp.type == 'SELECT':
return resp.bindings
else:
assert False, "Unexpected query response type %s" % resp.type
return None
def getAnnotationGraph(self):
"""
Returns the combined annotation graphs (including the manifest)
"""
return self._loadAnnotations()
def getAnnotationValue(self, resource, predicate):
"""
Returns a single annotation value for a resource and the indicated predicate,
or None
"""
return self._loadAnnotations().value(subject=resource,
predicate=predicate,
object=None)
def showAnnotations(self, annotations, outstr):
ro_annotation.showAnnotations(self.roconfig, self.getRoFilename(),
annotations, outstr)
return
def replaceUri(self, ann_node, remote_ann_node_uri):
for (p, o) in self.manifestgraph.predicate_objects(subject=ann_node):
self.manifestgraph.remove((ann_node, p, o))
self.manifestgraph.add((remote_ann_node_uri, p, o))
for (s, p) in self.manifestgraph.subject_predicates(object=ann_node):
self.manifestgraph.remove((s, p, ann_node))
self.manifestgraph.add((s, p, remote_ann_node_uri))
self._updateManifest()
return
# Support methods for accessing the manifest graph
def roManifestContains(self, stmt):
"""
Returns True if the RO manifest contains a statement matching the supplied triple.
"""
return stmt in self._loadManifest()
def getResourceValue(self, resource, predicate):
"""
Returns value for resource whose URI is supplied assocfiated with indicated predicate
"""
return self._loadManifest().value(subject=resource,
predicate=predicate,
object=None)
def getResourceType(self, resource):
"""
Returns type of resource whose URI is supplied
"""
return self.getResourceValue(resource, RDF.type)
def hasResourceType(self, resource, rdfType):
"""
Check if the resource whose URI is supplied has a provided RDF type.
"""
return self.roManifestContains((resource, RDF.type, rdfType))
def getRoMetadataDict(self):
"""
Returns dictionary of metadata about the RO from the manifest graph
"""
assert self._isLocal()
strsubject = ""
if isinstance(self.rouri, rdflib.URIRef): strsubject = str(self.rouri)
manifestDict = {
'ropath':
self.getRoFilename(),
'rouri':
strsubject,
'roident':
self.getResourceValue(self.rouri, DCTERMS.identifier),
'rotitle':
self.getResourceValue(self.rouri, DCTERMS.title),
'rocreator':
self.getResourceValue(self.rouri, DCTERMS.creator),
'rocreated':
self.getResourceValue(self.rouri, DCTERMS.created),
'rodescription':
self.getResourceValue(self.rouri, DCTERMS.description),
}
return manifestDict
# Support methods for accessing RO and component URIs
def getRoRef(self):
"""
Returns RO URI reference supplied (which may be a local file directory string)
"""
return self.roref
def getRoUri(self):
return self.rouri
def getComponentUri(self, path):
"""
Return URI for component where relative reference is treated as a file path
"""
if urlparse.urlsplit(path).scheme == "":
path = resolveUri("", str(self.getRoUri()), path)
return rdflib.URIRef(path)
def getComponentUriAbs(self, path):
"""
Return absolute URI for component where relative reference is treated as a URI reference
"""
return rdflib.URIRef(urlparse.urljoin(str(self.getRoUri()), path))
def getComponentUriRel(self, path):
"""
Return reference relative to RO for a supplied URI
"""
file_uri = urlparse.urlunsplit(
urlparse.urlsplit(str(self.getComponentUriAbs(path))))
ro_uri = urlparse.urlunsplit(urlparse.urlsplit(str(self.getRoUri())))
if ro_uri is not None and file_uri.startswith(ro_uri):
file_uri_rel = file_uri.replace(ro_uri, "", 1)
else:
file_uri_rel = path
return rdflib.URIRef(file_uri_rel)
def isRoMetadataRef(self, uri):
"""
Test if supplied URI is a reference to the current RO metadata area
"""
urirel = self.getComponentUriRel(uri)
return str(urirel).startswith(ro_settings.MANIFEST_DIR + "/")
def isLocalFileRo(self):
"""
Test current RO URI to see if it is a local file system reference
"""
return isFileUri(self.getRoUri())
def getRoFilename(self):
assert self._isLocal()
return getFilenameFromUri(self.getRoUri())
def getManifestFilename(self):
"""
Return manifest file name: used for local updates
"""
assert self._isLocal()
return os.path.join(self.getRoFilename(),
ro_settings.MANIFEST_DIR + "/",
ro_settings.MANIFEST_FILE)
def getRegistries(self):
'''
Load a dictionary of synchronization data from memory or from a JSON file.
'''
log.debug("Get registries")
if self.registries:
return self.registries
try:
rf = open(
os.path.join(self.getRoFilename(),
ro_settings.REGISTRIES_FILE), 'r')
self.registries = json.load(rf)
except IOError:
self.registries = dict()
except Exception as e:
log.exception(e)
self.registries = dict()
return self.registries
def saveRegistries(self):
'''
Save a dictionary of synchronization data to a JSON file.
'''
log.debug("Save registries")
rf = open(
os.path.join(self.getRoFilename(), ro_settings.REGISTRIES_FILE),
'w')
if self.registries:
json.dump(self.registries, rf)
return
def calculateChecksum(self, rofile):
'''
Calculate a file checksum.
'''
m = hashlib.md5()
with open(rofile) as f:
for line in f:
m.update(line)
f.close()
return m.hexdigest()
class ro_metadata(object):
"""
Class for accessing RO metadata
"""
def __init__(self, roconfig, roref, dummysetupfortest=False):
"""
Initialize: read manifest from object at given directory into local RDF graph
roconfig is the research object manager configuration, supplied as a dictionary
roref a URI reference that refers to the Research Object to be accessed, or
relative path name (see ro_uriutils.resolveFileAsUri for interpretation)
dummysetupfortest is an optional parameter that, if True, suppresses some aspects of
the setup (does not attempt to read a RO manifest) for isolated testing.
"""
self.roconfig = roconfig
self.roref = roref
self.dummyfortest = dummysetupfortest
self.manifestgraph = None
self.roannotations = None
self.registries = None
uri = resolveFileAsUri(roref)
if not uri.endswith("/"): uri += "/"
self.rouri = rdflib.URIRef(uri)
if self._isLocal():
self.rosrs = None
else:
self.rosrs = ROSRS_Session(
self.roconfig["rosrs_uri"],
self.roconfig["rosrs_access_token"]
)
self._loadManifest()
# Get RO URI from manifest
# May be different from computed value if manifest has absolute URI
# Nested URIs may be present; ours is the one described by the manifest URI,
# which is determined by the _loadManifest() method.
for s in self.manifestgraph.subjects(RDF.type, RO.ResearchObject):
if self.manifestgraph.value(s, ORE.isDescribedBy) == self.manifesturi:
self.rouri = s
# Check that the manifest contained at least one RO URI
assert self.rouri is not None
return
def _isLocal(self):
return isFileUri(self.rouri)
def _getLocalManifestUri(self):
return self.getComponentUri(ro_settings.MANIFEST_DIR+"/"+ro_settings.MANIFEST_FILE)
def _loadManifest(self):
if self.manifestgraph != None: return self.manifestgraph
if self.dummyfortest:
# Fake minimal manifest graph for testing
self.manifestgraph = rdflib.Graph()
self.manifestgraph.add( (self.rouri, RDF.type, RO.ResearchObject) )
self.manifesturi = self.rouri
elif self._isLocal():
# Read manifest graph
self.manifestgraph = rdflib.Graph()
for (prefix, uri) in ro_prefixes.prefixes:
self.manifestgraph.bind(prefix, rdflib.namespace.Namespace(uri))
self.manifesturi = self._getLocalManifestUri()
self.manifestgraph.parse(self.manifesturi)
else:
(status, reason, _h, manifesturi, manifest) = self.rosrs.getROManifest(self.rouri)
if status != 200:
msg = ("Can't access RO manifest (%03d %s)"%(status, reason))
raise ROSRS_Error(msg=msg, srsuri=self.rouri)
self.manifestgraph = manifest
self.manifesturi = manifesturi
# log.debug("romanifest graph:\n"+self.manifestgraph.serialize())
return self.manifestgraph
def getManifestGraph(self):
"""
Returns the manifest graph
"""
return self._loadManifest()
def _updateManifest(self):
"""
Write updated manifest file for research object
"""
assert self._isLocal()
self._loadManifest().serialize(
destination=self.getManifestFilename(), format='xml',
base=self.rouri, xml_base="..")
return
def _iterAnnotations(self, subject=None):
"""
Return iterator over annotation stubs in the current RO, either for
the specified subject resource, or for all annotations in the RO
subject is URI of subject whose annotations are returned, or None.
"""
manifest = self._loadManifest()
if self._isLocal():
for (anode, p, subject) in manifest:
if p in [RO.annotatesAggregatedResource, AO.annotatesResource]:
yield anode
else:
for anode in self.rosrs.getROAnnotationUris(self.getRoUri(), subject):
yield anode
return
def isAggregatedResource(self, rofile):
'''
Returns true if the manifest says that the research object aggregates the
resource. Resource URI is resolved against the RO URI unless it's absolute.
'''
resuri = self.getComponentUriAbs(rofile)
return (self.rouri, ORE.aggregates, resuri) in self.manifestgraph
def _loadAnnotations(self):
if self.roannotations: return self.roannotations
log.debug("_loadannotations")
# Assemble annotation graph
# NOTE: the manifest itself is included as an annotation by the RO setup
if self._isLocal():
manifest = self._loadManifest()
self.roannotations = rdflib.Graph()
annotation_uris_loaded = set()
for anode in self._iterAnnotations():
auri = manifest.value(subject=anode, predicate=AO.body)
if auri not in annotation_uris_loaded:
aref = self.getComponentUriRel(auri)
log.debug("_loadAnnotations: aref "+str(aref))
self._readAnnotationBody(aref, self.roannotations)
annotation_uris_loaded.add(auri)
else:
self.roannotations = self.rosrs.getROAnnotationGraph(self.rouri)
# log.debug("roannotations graph:\n"+self.roannotations.serialize())
for (prefix, uri) in ro_prefixes.prefixes:
self.manifestgraph.bind(prefix, rdflib.namespace.Namespace(uri))
return self.roannotations
def isInternalResource(self, resuri):
'''
Check if the resource is internal, i.e. should the resource content be uploaded
to the ROSR service. Returns true if the resource URI has the RO URI as a prefix.
'''
return resuri.startswith(self.rouri)
def isExternalResource(self, resuri):
'''
Check if the resource is external, i.e. can be aggregated as a URI reference.
Returns true if the URI has 'http' or 'https' scheme.
'''
parseduri = urlparse.urlsplit(resuri)
return parseduri.scheme in ["http", "https"]
def _createAnnotationBody(self, roresource, attrdict, defaultType="string"):
"""
Create a new annotation body for a single resource in a research object, based
on a supplied graph value.
Existing annotations for the same resource are not touched; if an annotation is being
added or replaced, it is the calling program'sresponsibility to update the manifest to
reference the active annotations. A new name is allocated for the created annotation,
graph which is returned as the result of this function.
roresource is the name of the Research Object component to be annotated,
possibly relative to the RO root directory.
attrdict is a dictionary of attributes to be saved in the annotation body.
Dictionary keys are attribute names that can be resolved via
ro_annotation.getAnnotationByName.
Returns the name of the annotation body created relative to the RO directory.
"""
assert self._isLocal()
af = ro_annotation.createAnnotationBody(
self.roconfig, self.getRoFilename(), roresource, attrdict, defaultType)
return os.path.join(ro_settings.MANIFEST_DIR+"/", af)
def _createAnnotationGraphBody(self, roresource, anngraph):
"""
Create a new annotation body for a single resource in a research object, based
on a supplied graph value.
Existing annotations for the same resource are not touched; if an annotation is being
added or replaced, it is the calling program'sresponsibility to update the manifest to
reference the active annotations. A new name is allocated for the created annotation,
graph which is returned as the result of this function.
roresource is the name of the Research Object component to be annotated,
possibly relative to the RO root directory.
anngraph is an annotation graph that is to be saved.
Returns the name of the annotation body created relative to the RO
manifest and metadata directory.
"""
assert self._isLocal()
af = ro_annotation.createAnnotationGraphBody(
self.roconfig, self.getRoFilename(), roresource, anngraph)
return os.path.join(ro_settings.MANIFEST_DIR+"/", af)
def _readAnnotationBody(self, annotationref, anngr=None):
"""
Read annotation body from indicated resource, return RDF Graph of annotation values.
annotationref is a URI reference of an annotation, possibly relative to the RO base URI
(e.g. as returned by _createAnnotationBody method).
anngr if supplied, if an RDF graph to which the annotations are added
"""
assert self._isLocal()
log.debug("_readAnnotationBody %s"%(annotationref))
annotationuri = self.getComponentUri(annotationref)
annotationformat = "xml"
# Look at file extension to figure format
# (rdflib.Graph.parse says;
# "used if format can not be determined from the source")
if re.search("\.(ttl|n3)$", annotationuri): annotationformat="n3"
if anngr == None:
log.debug("_readAnnotationBody: new graph")
anngr = rdflib.Graph()
try:
anngr.parse(annotationuri, format=annotationformat)
log.debug("_readAnnotationBody parse %s, len %i"%(annotationuri, len(anngr)))
except IOError as e:
log.debug("_readAnnotationBody %s, %s"%(str(annotationref), repr(e)))
anngr = None
except Exception as e:
log.debug("Failed to load annotation %s as %s"%(annotationuri, annotationformat))
log.debug("Exception %s"%(repr(e)))
raise
anngr = None
return anngr
def _addAnnotationToManifest(self, rofile, annfile):
"""
Add a new annotation body to an RO graph
rofile is the research object resource being annotated
annfile is the file name of the annotation body to be added,
possibly relative to the RO URI, with special characters
already URI-escaped.
"""
assert self._isLocal()
# <ore:aggregates>
# <ro:AggregatedAnnotation>
# <ro:annotatesAggregatedResource rdf:resource="data/UserRequirements-astro.ods" />
# <ao:body rdf:resource=".ro/(annotation).rdf" />
# </ro:AggregatedAnnotation>
# </ore:aggregates>
log.debug("_addAnnotationToManifest annfile %s"%(annfile))
ann = rdflib.BNode()
resuri = self.getComponentUri(rofile)
bodyuri = self.getComponentUriAbs(annfile)
self.manifestgraph.add((ann, RDF.type, RO.AggregatedAnnotation))
self.manifestgraph.add((ann, RO.annotatesAggregatedResource, resuri))
self.manifestgraph.add((ann, AO.body, bodyuri))
# Aggregate the annotation
self.manifestgraph.add((self.getRoUri(), ORE.aggregates, ann))
# Aggregate annotation body if it is RO metadata.
# Otherwise aggregation is the caller's responsibility
if self.isRoMetadataRef(bodyuri):
self.manifestgraph.add((self.getRoUri(), ORE.aggregates, bodyuri))
self.roannotations = None # Flush cached annotation graph
return
def _removeAnnotationFromManifest(self, ann):
"""
Remove references to an annotation from an RO graph
ann is the the annotation node to be removed
"""
assert self._isLocal()
bodyuri = self.manifestgraph.value(subject=ann, predicate=AO.body)
self.manifestgraph.remove((ann, None, None ))
# If annotation body is RO Metadata, and there are no other uses as an annotation,
# remove it from the RO aggregation.
if self.isRoMetadataRef(bodyuri):
if not self.manifestgraph.value(subject=ann, predicate=AO.body):
self.manifestgraph.remove((None, ORE.aggregates, bodyuri))
self.roannotations = None # Flush cached annotation graph
return
def addAggregatedResources(self, ro_file, recurse=True, includeDirs=False):
"""
Scan a local directory and add files found to the RO aggregation
"""
assert self._isLocal()
def notHidden(f):
return re.match("\.|.*/\.", f) == None
log.debug("addAggregatedResources: roref %s, file %s"%(self.roref, ro_file))
self.getRoFilename() # Check that we have one
basedir = os.path.abspath(self.roref)+os.path.sep
### print "- ro_file: %s"%(ro_file)
if os.path.isdir(ro_file):
ro_file = os.path.abspath(ro_file)+os.path.sep
### print "- ro_file: %s"%(ro_file)
### print "- basedir: %s"%(basedir)
#if ro_file.endswith(os.path.sep):
# ro_file = ro_file[0:-1]
if recurse:
rofiles = filter(notHidden,
MiscUtils.ScanDirectories.CollectDirectoryContents(ro_file,
baseDir=basedir,
listDirs=includeDirs,
listFiles=True,
recursive=recurse,
appendSep=True
)
)
log.debug("- rofiles: %s"%(repr(rofiles)))
else:
rofiles = [ro_file.split(basedir+os.path.sep,1)[-1]]
else:
rofiles = [self.getComponentUriRel(ro_file)]
s = self.getRoUri()
for f in rofiles:
### print "- file %s"%f
log.debug("- file %s"%f)
stmt = (s, ORE.aggregates, self.getComponentUri(f))
if stmt not in self.manifestgraph: self.manifestgraph.add(stmt)
self._updateManifest()
return
def removeAggregatedResource(self, resuri):
"""
Remove a specified resource.
resref is the URI of a resource to remove
"""
assert self._isLocal()
resuri = rdflib.URIRef(resuri)
log.debug("removeAggregatedResource: roref %s, resuri %s"%(self.roref, str(resuri)))
manifest = self._loadManifest()
for anode in self._iterAnnotations(subject=resuri):
self._removeAnnotationFromManifest(anode)
manifest.remove((None, ORE.aggregates, resuri))
self._updateManifest()
return
def getAggregatedResources(self):
"""
Returns iterator over all resources aggregated by a manifest.
Each value returned by the iterator is an aggregated resource URI
"""
log.debug("getAggregatedResources: uri %s"%(self.rouri))
for r in self._loadManifest().objects(subject=self.rouri, predicate=ORE.aggregates):
if not isinstance(r, rdflib.BNode):
yield r
return
def addGraphAnnotation(self, rofile, graph):
"""
Add an annotation graph for a named resource. Unlike addSimpleAnnotation, this
method adds an annotation to the manifest using an existing RDF graph (which is
presumably itself in the RO structure).
rofile names the file or resource to be annotated, possibly relative to the RO.
Note that no checks are performed to ensure that the graph itself actually
refers to this resource - that's up the the creator of the graph. This
identifies the resourfce with which the annotation body is associated in
the RO manifest.
graph names the file or resource containing the annotation body.
"""
assert self._isLocal()
ro_graph = self._loadManifest()
self._addAnnotationToManifest(rofile, graph)
self._updateManifest()
return
def isAnnotationNode(self, respath):
'''
Returns true if the manifest says that the research object aggregates the
annotation and it is an ro:AggregatedAnnotation.
Resource URI is resolved against the RO URI unless it's absolute.
'''
resuri = self.getComponentUriAbs(respath)
log.debug("isAnnotationNode: ro uri %s res uri %s"%(self.rouri, resuri))
return (self.rouri, ORE.aggregates, resuri) in self.manifestgraph and \
(resuri, RDF.type, RO.AggregatedAnnotation) in self.manifestgraph
def addSimpleAnnotation(self, rofile, attrname, attrvalue, defaultType="string"):
"""
Add a simple annotation to a file in a research object.
rofile names the file or resource to be annotated, possibly relative to the RO.
attrname names the attribute in a form recognized by getAnnotationByName
attrvalue is a value to be associated with the attribute
"""
assert self._isLocal()
ro_dir = self.getRoFilename()
annfile = self._createAnnotationBody(rofile, {attrname: attrvalue}, defaultType)
self._addAnnotationToManifest(rofile, annfile)
self._updateManifest()
return annfile
def removeSimpleAnnotation(self, rofile, attrname, attrvalue):
"""
Remove a simple annotation or multiple matching annotations a research object.
rofile names the annotated file or resource, possibly relative to the RO.
attrname names the attribute in a form recognized by getAnnotationByName
attrvalue is the attribute value to be deleted, or Nomne to delete all vaues
"""
assert self._isLocal()
ro_dir = self.getRoFilename()
ro_graph = self._loadManifest()
subject = self.getComponentUri(rofile)
(predicate,valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname)
val = attrvalue and ro_annotation.makeAnnotationValue(self.roconfig, attrvalue, valtype)
add_annotations = []
remove_annotations = []
log.debug("removeSimpleAnnotation subject %s, predicate %s, val %s"%
(str(subject), str(predicate), val))
# Scan for annotation graph resources containing this annotation
for ann_node in self._iterAnnotations(subject=subject):
ann_uri = ro_graph.value(subject=ann_node, predicate=AO.body)
log.debug("removeSimpleAnnotation ann_uri %s"%(str(ann_uri)))
if self.isRoMetadataRef(ann_uri):
ann_graph = self._readAnnotationBody(self.getComponentUriRel(ann_uri))
log.debug("removeSimpleAnnotation ann_graph %s"%(ann_graph))
if (subject, predicate, val) in ann_graph:
ann_graph.remove((subject, predicate, val))
if (subject, None, None) in ann_graph:
# Triples remain in annotation body: write new body and update RO graph
ann_name = self._createAnnotationGraphBody(rofile, ann_graph)
remove_annotations.append(ann_node)
add_annotations.append(ann_name)
else:
# Remove annotation from RO graph
remove_annotations.append(ann_node)
# Update RO manifest graph if needed
if add_annotations or remove_annotations:
for a in remove_annotations:
self._removeAnnotationFromManifest(a)
for a in add_annotations:
self._addAnnotationToManifest(rofile, a)
self._updateManifest()
return
def replaceSimpleAnnotation(self, rofile, attrname, attrvalue):
"""
Replace a simple annotation in a research object.
rofile names the file or resource to be annotated, possibly relative to the RO.
attrname names the attribute in a form recognized by getAnnotationByName
attrvalue is a new value to be associated with the attribute
"""
assert self._isLocal()
ro_dir = self.getRoFilename()
ro_graph = self._loadManifest()
subject = self.getComponentUri(rofile)
(predicate,valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname)
log.debug("Replace annotation: subject %s, predicate %s, value %s"%
(repr(subject), repr(predicate), repr(attrvalue)))
ro_graph.remove((subject, predicate, None))
ro_graph.add((subject, predicate,
ro_annotation.makeAnnotationValue(self.roconfig, attrvalue, valtype)))
self._updateManifest()
self.roannotations = None # Flush cached annotation graph
return
def iterateAnnotations(self, subject=None, property=None):
"""
Returns an iterator over annotations of the current RO that match the
supplied subject and/or property.
"""
log.debug("iterateAnnotations s:%s, p:%s"%(str(subject),str(property)))
ann_graph = self._loadAnnotations()
for (s, p, v) in ann_graph.triples((subject, property, None)):
if not isinstance(s, rdflib.BNode):
if not self.isRoMetadataRef(s):
log.debug("Triple: %s %s %s"%(s,p,v))
yield (s, p, v)
return
def getRoAnnotations(self):
"""
Returns iterator over annotations applied to the RO as an entity.
Each value returned by the iterator is a (subject,predicate,object) triple.
"""
return self.iterateAnnotations(subject=self.getRoUri())
def getFileAnnotations(self, rofile):
"""
Returns iterator over annotations applied to a specified component in the RO
Each value returned by the iterator is a (subject,predicate,object) triple.
"""
return self.iterateAnnotations(subject=self.getComponentUri(rofile))
def getAllAnnotations(self):
"""
Returns iterator over all annotations associated with the RO
Each value returned by the iterator is a (subject,predicate,object) triple.
"""
return self.iterateAnnotations()
def getAllAnnotationNodes(self):
"""
Returns iterator over all annotations aggregated within the RO
Each value returned by the iterator is a (annuri, bodyuri, target) triple.
"""
annotations = self.manifestgraph.subject_objects(predicate=RO.annotatesAggregatedResource)
for (ann_node, ann_target) in annotations:
ann_body = self.manifestgraph.value(subject=ann_node, predicate=AO.body)
yield (ann_node, ann_body, ann_target)
return
def getAnnotationValues(self, rofile, attrname):
"""
Returns iterator over annotation values for given subject and attribute
"""
(predicate,valtype) = ro_annotation.getAnnotationByName(self.roconfig, attrname)
resuri = self.getComponentUri(rofile)
return ( v for (s,p,v) in self.iterateAnnotations(subject=resuri, property=predicate) )
def queryAnnotations(self, query, initBindings={}):
"""
Runs a query over the combined annotation graphs (including the manifest)
and returns True or False (for ASK queries) or a list of dictionaries of
query results (for SELECT queries).
"""
log.debug("queryAnnotations: \n----\n%s\n--------\n"%(query))
ann_gr = self._loadAnnotations()
# log.debug("queryAnnotations graph: \n----\n%s\n--------\n"%(ann_gr.serialize(format='xml')))
try:
resp = ann_gr.query(query,initBindings=initBindings)
except:
log.info("queryAnnotations failed query: \n----\n%s\n--------\n"%(query))
raise
if resp.type == 'ASK':
return resp.askAnswer
elif resp.type == 'SELECT':
return resp.bindings
else:
assert False, "Unexpected query response type %s"%resp.type
return None
def getAnnotationGraph(self):
"""
Returns the combined annotation graphs (including the manifest)
"""
return self._loadAnnotations()
def getAnnotationValue(self, resource, predicate):
"""
Returns a single annotation value for a resource and the indicated predicate,
or None
"""
return self._loadAnnotations().value(subject=resource, predicate=predicate, object=None)
def showAnnotations(self, annotations, outstr):
ro_annotation.showAnnotations(self.roconfig, self.getRoFilename(), annotations, outstr)
return
def replaceUri(self, ann_node, remote_ann_node_uri):
for (p, o) in self.manifestgraph.predicate_objects(subject = ann_node):
self.manifestgraph.remove((ann_node, p, o))
self.manifestgraph.add((remote_ann_node_uri, p, o))
for (s, p) in self.manifestgraph.subject_predicates(object = ann_node):
self.manifestgraph.remove((s, p, ann_node))
self.manifestgraph.add((s, p, remote_ann_node_uri))
self._updateManifest()
return
# Support methods for accessing the manifest graph
def roManifestContains(self, stmt):
"""
Returns True if the RO manifest contains a statement matching the supplied triple.
"""
return stmt in self._loadManifest()
def getResourceValue(self, resource, predicate):
"""
Returns value for resource whose URI is supplied assocfiated with indicated predicate
"""
return self._loadManifest().value(subject=resource, predicate=predicate, object=None)
def getResourceType(self, resource):
"""
Returns type of resource whose URI is supplied
"""
return self.getResourceValue(resource, RDF.type)
def hasResourceType(self, resource, rdfType):
"""
Check if the resource whose URI is supplied has a provided RDF type.
"""
return self.roManifestContains((resource, RDF.type, rdfType))
def getRoMetadataDict(self):
"""
Returns dictionary of metadata about the RO from the manifest graph
"""
assert self._isLocal()
strsubject = ""
if isinstance(self.rouri, rdflib.URIRef): strsubject = str(self.rouri)
manifestDict = {
'ropath': self.getRoFilename(),
'rouri': strsubject,
'roident': self.getResourceValue(self.rouri, DCTERMS.identifier ),
'rotitle': self.getResourceValue(self.rouri, DCTERMS.title ),
'rocreator': self.getResourceValue(self.rouri, DCTERMS.creator ),
'rocreated': self.getResourceValue(self.rouri, DCTERMS.created ),
'rodescription': self.getResourceValue(self.rouri, DCTERMS.description ),
}
return manifestDict
# Support methods for accessing RO and component URIs
def getRoRef(self):
"""
Returns RO URI reference supplied (which may be a local file directory string)
"""
return self.roref
def getRoUri(self):
return self.rouri
def getComponentUri(self, path):
"""
Return URI for component where relative reference is treated as a file path
"""
if urlparse.urlsplit(path).scheme == "":
path = resolveUri("", str(self.getRoUri()), path)
return rdflib.URIRef(path)
def getComponentUriAbs(self, path):
"""
Return absolute URI for component where relative reference is treated as a URI reference
"""
return rdflib.URIRef(urlparse.urljoin(str(self.getRoUri()), path))
def getComponentUriRel(self, path):
"""
Return reference relative to RO for a supplied URI
"""
file_uri = urlparse.urlunsplit(urlparse.urlsplit(str(self.getComponentUriAbs(path))))
ro_uri = urlparse.urlunsplit(urlparse.urlsplit(str(self.getRoUri())))
if ro_uri is not None and file_uri.startswith(ro_uri):
file_uri_rel = file_uri.replace(ro_uri, "", 1)
else:
file_uri_rel = path
return rdflib.URIRef(file_uri_rel)
def isRoMetadataRef(self, uri):
"""
Test if supplied URI is a reference to the current RO metadata area
"""
urirel = self.getComponentUriRel(uri)
return str(urirel).startswith(ro_settings.MANIFEST_DIR+"/")
def isLocalFileRo(self):
"""
Test current RO URI to see if it is a local file system reference
"""
return isFileUri(self.getRoUri())
def getRoFilename(self):
assert self._isLocal()
return getFilenameFromUri(self.getRoUri())
def getManifestFilename(self):
"""
Return manifest file name: used for local updates
"""
assert self._isLocal()
return os.path.join(self.getRoFilename(), ro_settings.MANIFEST_DIR+"/",
ro_settings.MANIFEST_FILE)
def getRegistries(self):
'''
Load a dictionary of synchronization data from memory or from a JSON file.
'''
log.debug("Get registries")
if self.registries:
return self.registries
try:
rf = open(os.path.join(self.getRoFilename(), ro_settings.REGISTRIES_FILE), 'r')
self.registries = json.load(rf)
except IOError:
self.registries = dict()
except Exception as e:
log.exception(e)
self.registries = dict()
return self.registries
def saveRegistries(self):
'''
Save a dictionary of synchronization data to a JSON file.
'''
log.debug("Save registries")
rf = open(os.path.join(self.getRoFilename(), ro_settings.REGISTRIES_FILE), 'w')
if self.registries:
json.dump(self.registries, rf)
return
def calculateChecksum(self, rofile):
'''
Calculate a file checksum.
'''
m = hashlib.md5()
with open(rofile) as f:
for line in f:
m.update(line)
f.close()
return m.hexdigest()
class TestROSRSMetadata(TestROSupport.TestROSupport):
"""
Test ro metadata access via ROSRS API
"""
def setUp(self):
super(TestROSRSMetadata, self).setUp()
self.rosrs = ROSRS_Session(Config.ROSRS_API_URI,
accesskey=Config.AUTHORIZATION)
# Clean up from previous runs
self.rosrs.deleteRO(Config.TEST_RO_PATH)
return
def tearDown(self):
super(TestROSRSMetadata, self).tearDown()
# Clean up
self.rosrs.deleteRO(Config.TEST_RO_PATH)
self.rosrs.close()
return
def createTestRO(self):
(status, reason, rouri, manifest) = self.rosrs.createRO(Config.TEST_RO_NAME,
"Test RO for ROSRSMetadata", "TestROSRSMetadata.py", "2012-09-11")
self.assertEqual(status, 201)
# Include manifest as annotation of RO
(s1, r1, h1, manifesturi, manifest) = self.rosrs.getROManifest(rouri)
self.assertEqual(s1, 200)
(s2, r2, annuri) = self.rosrs.createROAnnotationExt(
rouri, rouri, manifesturi)
self.assertEqual(s2, 201)
# Aggregate internal resource
rescontent = "Resource content\n"
(s3, r3, proxyuri, resuri) = self.rosrs.aggregateResourceInt(
rouri, Config.TEST_RESOURCE, ctype="text/plain", body=rescontent)
self.assertEqual(s3, 201)
self.assertEqual(r3, "Created")
self.assertEqual(str(resuri), str(rouri)+Config.TEST_RESOURCE)
# Aggregate external resource
externaluri = rdflib.URIRef(Config.TEST_EXTERNAL)
(s4, r4, proxyuri, resuri) = self.rosrs.aggregateResourceExt(
rouri, externaluri)
self.assertEqual(s4, 201)
self.assertEqual(r4, "Created")
self.assertEqual(str(resuri), Config.TEST_EXTERNAL)
return (status, reason, rouri, manifest)
def createTestAnnotation(self, rouri, resuri, resref):
annbody = """<?xml version="1.0" encoding="UTF-8"?>
<rdf:RDF
xmlns:dct="http://purl.org/dc/terms/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xml:base="%(rouri)s"
>
<rdf:Description rdf:about="%(resuri)s">
<dct:title>Title for %(resref)s</dct:title>
<rdfs:seeAlso rdf:resource="http://example.org/test" />
</rdf:Description>
</rdf:RDF>
"""%{"rouri": str(rouri), "resuri": str(resuri), "resref": resref}
agraph = rdflib.graph.Graph()
agraph.parse(data=annbody, format="xml")
(status, reason, annuri, bodyuri) = self.rosrs.createROAnnotationInt(
rouri, resuri, agraph)
self.assertEqual(status, 201)
return (status, reason, annuri, bodyuri)
# Actual tests follow
def testNull(self):
assert True, 'Null test failed'
def testCreateRoMetadata(self):
"""
Test creation of ro_metadata object, and basic access to manifest content
"""
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
self.assertEqual(str(rouri), Config.TEST_RO_URI)
self.assertIn((rouri, RDF.type, RO.ResearchObject), manifest)
romd = ro_metadata.ro_metadata(ro_config, rouri)
resuri = romd.getComponentUriAbs(Config.TEST_RESOURCE)
exturi = romd.getComponentUriAbs(Config.TEST_EXTERNAL)
resref = Config.TEST_RESOURCE
(status, reason, annuri, bodyuri) = self.createTestAnnotation(rouri, resuri, resref)
self.assertEqual(status, 201)
self.assertEqual(reason, "Created")
#
self.assertEquals(romd.rouri, rouri)
self.assertTrue(romd.roManifestContains((rouri, RDF.type, RO.ResearchObject)))
self.assertTrue(romd.roManifestContains((rouri, ORE.aggregates, resuri)))
self.assertTrue(romd.roManifestContains((rouri, ORE.aggregates, exturi)))
return
def testReadRoAnnotationBody(self):
"""
Test function to create & read a simple annotation body on an RO
"""
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
romd = ro_metadata.ro_metadata(ro_config, rouri)
resuri = romd.getComponentUriAbs(Config.TEST_RESOURCE)
resref = Config.TEST_RESOURCE
(status, reason, bodyuri, agraph) = self.createTestAnnotation(rouri, resuri, resref)
self.assertEqual(status, 201)
# Retrieve annotations
anns = list(romd.getFileAnnotations(Config.TEST_RESOURCE))
self.assertIn((resuri, DCTERMS.title, rdflib.Literal("Title for "+Config.TEST_RESOURCE)), anns)
self.assertIn((resuri, RDFS.seeAlso, rdflib.URIRef("http://example.org/test")), anns)
return
def testGetInitialRoAnnotations(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
romd = ro_metadata.ro_metadata(ro_config, rouri)
# Retrieve the anotations
annotations = romd.getRoAnnotations()
rouri = romd.getRoUri()
expected_annotations = (
[ (rouri, RDF.type, RO.ResearchObject)
, (rouri, RDF.type, ROEVO.LiveRO)
, (rouri, ORE.isDescribedBy, romd.getComponentUriAbs(ro_test_config.ROMANIFESTPATH))
#, (rouri, DCTERMS.description, rdflib.Literal('Test init RO annotation'))
#, (rouri, DCTERMS.title, rdflib.Literal('Test init RO annotation'))
#, (rouri, DCTERMS.created, rdflib.Literal('unknown'))
#, (rouri, DCTERMS.creator, rdflib.Literal('Test User'))
#, (rouri, DCTERMS.identifier, rdflib.Literal('ro-testRoAnnotate'))
])
count = 0
for next in list(annotations):
if ( not isinstance(next[2], rdflib.BNode) and
not next[1] in [ORE.aggregates, DCTERMS.created, DCTERMS.creator] ):
log.debug("- next %s"%(str(next[0])) )
log.debug(" - (%s, %s)"%(str(next[1]),str(next[2])) )
if next in expected_annotations:
count += 1
else:
self.assertTrue(False, "Not expected (%d) %s"%(count, repr(next)))
self.assertEqual(count,3)
return
def testQueryAnnotations(self):
(status, reason, rouri, manifest) = self.createTestRO()
self.assertEqual(status, 201)
romd = ro_metadata.ro_metadata(ro_config, rouri)
resuri = romd.getComponentUriAbs(Config.TEST_RESOURCE)
resref = Config.TEST_RESOURCE
(status, reason, bodyuri, agraph) = self.createTestAnnotation(rouri, resuri, resref)
self.assertEqual(status, 201)
# Query the file anotations
queryprefixes = """
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX ro: <http://purl.org/wf4ever/ro#>
PREFIX ore: <http://www.openarchives.org/ore/terms/>
PREFIX ao: <http://purl.org/ao/>
PREFIX dcterms: <http://purl.org/dc/terms/>
PREFIX roterms: <http://ro.example.org/ro/terms/>
"""
query = (queryprefixes +
"""
ASK
{
?ro rdf:type ro:ResearchObject ;
dcterms:creator ?user .
}
""")
resp = romd.queryAnnotations(query)
self.assertTrue(resp, "Expected 'True' result for query: %s"%(query))
query = (queryprefixes +
"""
ASK
{
<%(resuri)s> dcterms:title ?title .
}
"""%{"resuri": str(resuri)})
resp = romd.queryAnnotations(query)
self.assertTrue(resp, "Expected 'True' result for query: %s"%(query))
query = (queryprefixes +
"""
ASK
{
?ro rdf:type ro:ResearchObject ;
dcterms:creator "Not user" .
}
""")
resp = romd.queryAnnotations(query)
self.assertFalse(resp, "Expected 'False' result for query: %s"%(query))
query = (queryprefixes +
"""
SELECT * WHERE
{
?ro rdf:type ro:ResearchObject ;
ore:aggregates ?file .
?file dcterms:title ?title .
}
""")
rouri = romd.getRoUri()
resp = romd.queryAnnotations(query)
self.assertEqual(resp[0]['ro'], rouri)
self.assertEqual(resp[0]['file'], resuri)
self.assertEqual(resp[0]['title'], rdflib.Literal("Title for %s"%(Config.TEST_RESOURCE)))
return
# Sentinel/placeholder tests
def testUnits(self):
assert (True)
def testComponents(self):
assert (True)
def testIntegration(self):
assert (True)
def testPending(self):
assert (False), "Pending tests follow"
