def testPush(self):
"""
Push a Research Object to ROSRS.
ro push <zip> | -d <dir> [ -f ] [ -r <rosrs_uri> ] [ -t <access_token> ]
"""
rodir = self.createTestRo(testbase, "data/ro-test-1", "RO test ro push", "ro-testRoPush")
localRo = ro_metadata.ro_metadata(ro_config, rodir)
localRo.addAggregatedResources(rodir, recurse=True)
roresource = "subdir1/subdir1-file.txt"
# Add anotations for file
localRo.addSimpleAnnotation(roresource, "type", "Test file")
localRo.addSimpleAnnotation(roresource, "description", "File in test research object")
args = [
"ro", "push",
"-d", rodir,
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
ro_remote_metadata.deleteRO(httpsession, urlparse.urljoin(httpsession.baseuri(), "RO_test_ro_push/"))
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
self.assertEqual(self.outstr.getvalue().count("Resource uploaded:"), 8)
self.assertEqual(self.outstr.getvalue().count("Resource deleted in ROSRS:"), 0)
self.assertEqual(self.outstr.getvalue().count("Annotation pushed:"), 3)
self.assertEqual(self.outstr.getvalue().count("Annotation deleted in ROSRS:"), 0)
# Clean up
self.deleteTestRo(rodir)
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
for line in self.outstr.getvalue().splitlines():
if "Created RO:" in line:
createdRO = line.split("Created RO:")[1].strip()
ro_remote_metadata.deleteRO(httpsession, createdRO)
return
def setUp(self):
super(TestSyncCommands, self).setUp()
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
return
class TestSyncCommands(TestROSupport.TestROSupport):
"""
Test sync ro commands
"""
def setUp(self):
super(TestSyncCommands, self).setUp()
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI, accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
return
def tearDown(self):
super(TestSyncCommands, self).tearDown()
return
# Actual tests follow
def testNull(self):
assert True, 'Null test failed'
def testPushZip(self):
"""
Push a Research Object in zip format to ROSRS.
ro push <zip> | -d <dir> [ -f ] [ -r <rosrs_uri> ] [ -t <access_token> ]
"""
args = [
"ro", "push", "zips/pushro-6.zip",
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
#preparing
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
self.assertEqual(self.outstr.getvalue().count("Job URI"),1)
self.assertEqual(self.outstr.getvalue().count("ENTER"),1)
for line in self.outstr.getvalue().split("\n"):
if "RO URI:" in line:
id = line.split("RO URI:")[1].strip()
self.rosrs.deleteRO(id)
def testPushZipWithNewOption(self):
"""
Push a Research Object in zip format to ROSRS.
ro push <zip> | -d <dir> [ -f ] [ -r <rosrs_uri> ] [ -t <access_token> ]
"""
args = [
"ro", "push", "data/newro.zip", "--new",
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
#preparing
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
self.assertEqual(self.outstr.getvalue().count("new"),1)
self.assertEqual(self.outstr.getvalue().count("Job URI"),1)
self.assertEqual(self.outstr.getvalue().count("ENTER"),1)
for line in self.outstr.getvalue().split("\n"):
if "RO URI:" in line:
id = line.split("RO URI:")[1].strip()
self.rosrs.deleteRO(id)
def testPushZipSynchronous(self):
"""
Push a Research Object in zip format to ROSRS.
ro push <zip> | -d <dir> [ -f ] [ -r <rosrs_uri> ] [ -t <access_token> ]
"""
args = [
"ro", "push", "zips/pushro-6.zip",
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
#preparing
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
#cleaning
ro_remote_metadata.deleteRO(httpsession, ro_test_config.ROSRS_URI + "ro1/")
assert status == 0
self.assertEqual(self.outstr.getvalue().count("Job URI"),1)
for line in self.outstr.getvalue().split("\n"):
if "RO URI:" in line:
id = line.split("RO URI:")[1].strip()
ro_remote_metadata.deleteRO(httpsession,id)
def testPushZipAsynchronous(self):
"""
Push a Research Object in zip format to ROSRS.
ro push <zip> | -d <dir> [ -f ] [ -r <rosrs_uri> ] [ -t <access_token> ]
"""
args = [
"ro", "push", "zips/pushro-6.zip",
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"--asynchronous",
"-v"
]
#preparing
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
ro_remote_metadata.deleteRO(httpsession, ro_test_config.ROSRS_URI + "ro/")
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
self.assertEqual(self.outstr.getvalue().count("Job URI"),1)
#self.assertEqual(self.outstr.getvalue().count("Job Status:"),1)
for line in self.outstr.getvalue().split("\n"):
if "Job URI:" in line:
jobLocation = line.split("Job URI:")[1].strip()
status = "RUNNING"
while status == "RUNNING":
time.sleep(1)
(status, id, processed_resources, submitted_resources) = parse_job(self.rosrs, jobLocation)
assert status == "DONE"
self.rosrs.deleteRO(id)
def testPush(self):
"""
Push a Research Object to ROSRS.
ro push <zip> | -d <dir> [ -f ] [ -r <rosrs_uri> ] [ -t <access_token> ]
"""
rodir = self.createTestRo(testbase, "data/ro-test-1", "RO test ro push", "ro-testRoPush")
localRo = ro_metadata.ro_metadata(ro_config, rodir)
localRo.addAggregatedResources(rodir, recurse=True)
roresource = "subdir1/subdir1-file.txt"
# Add anotations for file
localRo.addSimpleAnnotation(roresource, "type", "Test file")
localRo.addSimpleAnnotation(roresource, "description", "File in test research object")
args = [
"ro", "push",
"-d", rodir,
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
ro_remote_metadata.deleteRO(httpsession, urlparse.urljoin(httpsession.baseuri(), "RO_test_ro_push/"))
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
self.assertEqual(self.outstr.getvalue().count("Resource uploaded:"), 8)
self.assertEqual(self.outstr.getvalue().count("Resource deleted in ROSRS:"), 0)
self.assertEqual(self.outstr.getvalue().count("Annotation pushed:"), 3)
self.assertEqual(self.outstr.getvalue().count("Annotation deleted in ROSRS:"), 0)
# Clean up
self.deleteTestRo(rodir)
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
for line in self.outstr.getvalue().splitlines():
if "Created RO:" in line:
createdRO = line.split("Created RO:")[1].strip()
ro_remote_metadata.deleteRO(httpsession, createdRO)
return
def testCheckout(self):
"""
Checkout a Research Object from ROSRS.
ro checkout [ <RO-identifier> ] [ -d <dir> ] [ -r <rosrs_uri> ] [ -t <access_token> ]
"""
rodir = self.createTestRo(testbase, "data/ro-test-1", "RO test ro checkout", "ro-testRoPush")
localRo = ro_metadata.ro_metadata(ro_config, rodir)
localRo.addAggregatedResources(rodir, recurse=True)
roresource = "subdir1/subdir1-file.txt"
# Add anotations for file
ann1 = localRo.addSimpleAnnotation(roresource, "type", "Test file")
ann2 = localRo.addSimpleAnnotation(roresource, "description", "File in test research object")
# remove previous RO
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
ro_remote_metadata.deleteRO(httpsession, urlparse.urljoin(httpsession.baseuri(), "RO_test_ro_checkout/"))
# push an RO
args = [
"ro", "push",
"-d", rodir,
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
createdUri = self.outstr.getvalue().splitlines()[1].split("Created RO: ")[1].split(ro_test_config.ROSRS_URI)[1]
# check it out as a copy
rodir2 = os.path.join(ro_test_config.ROBASEDIR, createdUri)
rodir_original_path = os.path.join(ro_test_config.ROBASEDIR, "origin")
shutil.rmtree(rodir_original_path, ignore_errors = True)
shutil.rmtree(rodir2, ignore_errors = True)
shutil.move(rodir, rodir_original_path)
args = [
"ro", "checkout", createdUri,
"-d", ro_test_config.ROBASEDIR,
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
files = [ "robase/"+ createdUri +"README-ro-test-1"
, "robase/"+ createdUri + "minim.rdf"
, "robase/"+ createdUri + "subdir1/subdir1-file.txt"
, "robase/"+ createdUri + "subdir2/subdir2-file.txt"
, "robase/"+ createdUri + "filename with spaces.txt"
, "robase/"+ createdUri + "filename#with#hashes.txt"
, "robase/"+ createdUri + ".ro/manifest.rdf"
, "robase/"+ createdUri + ann1
, "robase/"+ createdUri + ann2
, "robase/"+ createdUri + ".ro/evo_info.ttl"
]
self.assertEqual(self.outstr.getvalue().count("ro checkout"), 1)
for f in files:
self.assertEqual(self.outstr.getvalue().count(f), 1, "%s"%f)
self.assertEqual(self.outstr.getvalue().count("%d files checked out" % len(files)), 1)
# compare they're identical, with the exception of registries.pickle
cmpres = filecmp.dircmp(rodir_original_path,rodir2)
self.assertEquals(cmpres.left_only, [ro_settings.REGISTRIES_FILE])
self.assertEquals(cmpres.right_only, [])
self.assertListEqual(cmpres.diff_files, [], "Files should be the same (manifest is ignored)")
# delete the checked out copy
self.deleteTestRo(rodir2)
self.deleteTestRo(rodir_original_path)
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
ro_remote_metadata.deleteRO(httpsession, urlparse.urljoin(httpsession.baseuri(), createdUri))
return
# Sentinel/placeholder tests
def testUnits(self):
assert (True)
def testComponents(self):
assert (True)
def testIntegration(self):
assert (True)
def testPending(self):
assert (False), "Pending tests follow"
def testCheckout(self):
"""
Checkout a Research Object from ROSRS.
ro checkout [ <RO-identifier> ] [ -d <dir> ] [ -r <rosrs_uri> ] [ -t <access_token> ]
"""
rodir = self.createTestRo(testbase, "data/ro-test-1", "RO test ro checkout", "ro-testRoPush")
localRo = ro_metadata.ro_metadata(ro_config, rodir)
localRo.addAggregatedResources(rodir, recurse=True)
roresource = "subdir1/subdir1-file.txt"
# Add anotations for file
ann1 = localRo.addSimpleAnnotation(roresource, "type", "Test file")
ann2 = localRo.addSimpleAnnotation(roresource, "description", "File in test research object")
# remove previous RO
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
ro_remote_metadata.deleteRO(httpsession, urlparse.urljoin(httpsession.baseuri(), "RO_test_ro_checkout/"))
# push an RO
args = [
"ro", "push",
"-d", rodir,
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
createdUri = self.outstr.getvalue().splitlines()[1].split("Created RO: ")[1].split(ro_test_config.ROSRS_URI)[1]
# check it out as a copy
rodir2 = os.path.join(ro_test_config.ROBASEDIR, createdUri)
rodir_original_path = os.path.join(ro_test_config.ROBASEDIR, "origin")
shutil.rmtree(rodir_original_path, ignore_errors = True)
shutil.rmtree(rodir2, ignore_errors = True)
shutil.move(rodir, rodir_original_path)
args = [
"ro", "checkout", createdUri,
"-d", ro_test_config.ROBASEDIR,
"-r", ro_test_config.ROSRS_URI,
"-t", ro_test_config.ROSRS_ACCESS_TOKEN,
"-v"
]
with SwitchStdout(self.outstr):
status = ro.runCommand(ro_test_config.CONFIGDIR, ro_test_config.ROBASEDIR, args)
assert status == 0
files = [ "robase/"+ createdUri +"README-ro-test-1"
, "robase/"+ createdUri + "minim.rdf"
, "robase/"+ createdUri + "subdir1/subdir1-file.txt"
, "robase/"+ createdUri + "subdir2/subdir2-file.txt"
, "robase/"+ createdUri + "filename with spaces.txt"
, "robase/"+ createdUri + "filename#with#hashes.txt"
, "robase/"+ createdUri + ".ro/manifest.rdf"
, "robase/"+ createdUri + ann1
, "robase/"+ createdUri + ann2
, "robase/"+ createdUri + ".ro/evo_info.ttl"
]
self.assertEqual(self.outstr.getvalue().count("ro checkout"), 1)
for f in files:
self.assertEqual(self.outstr.getvalue().count(f), 1, "%s"%f)
self.assertEqual(self.outstr.getvalue().count("%d files checked out" % len(files)), 1)
# compare they're identical, with the exception of registries.pickle
cmpres = filecmp.dircmp(rodir_original_path,rodir2)
self.assertEquals(cmpres.left_only, [ro_settings.REGISTRIES_FILE])
self.assertEquals(cmpres.right_only, [])
self.assertListEqual(cmpres.diff_files, [], "Files should be the same (manifest is ignored)")
# delete the checked out copy
self.deleteTestRo(rodir2)
self.deleteTestRo(rodir_original_path)
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
ro_remote_metadata.deleteRO(httpsession, urlparse.urljoin(httpsession.baseuri(), createdUri))
return
class TestRosrsSync(TestROSupport.TestROSupport):
def setUp(self):
super(TestRosrsSync, self).setUp()
self.rosrs = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
return
def tearDown(self):
super(TestRosrsSync, self).tearDown()
self.rosrs.close()
return
# Setup local config for local tests
def setupConfig(self):
return self.setupTestBaseConfig(testbase)
def testNull(self):
assert True, 'Null test failed'
def testPushZip(self):
httpsession = ROSRS_Session(ro_test_config.ROSRS_URI,
accesskey=ro_test_config.ROSRS_ACCESS_TOKEN)
(status, reason, headers, data) = sendZipRO(httpsession, ro_test_config.ROSRS_URI, "ro1", open("zips/pushro-6.zip", 'rb').read())
status = "RUNNING"
while (status == "RUNNING"):
time.sleep(1)
(status, target_id, processed_resources, submitted_resources) = ro_utils.parse_job(httpsession, headers['location'])
self.assertEqual("DONE", status)
deleteRO(httpsession,target_id)
self.assertEqual(reason, "Created")
def testPush(self):
rodir = self.createTestRo(testbase, "data/ro-test-1", "RO test push", "ro-testRoPush")
localRo = ro_metadata(ro_config, rodir)
localRo.addAggregatedResources(rodir, recurse=True)
# localRo.aggregateResourceExt("http://www.example.org")
roresource = "subdir1/subdir1-file.txt"
# Add anotations for file
ann1 = localRo.addSimpleAnnotation(roresource, "type", "Test file")
ann2 = localRo.addSimpleAnnotation(roresource, "description", "File in test research object")
ann3 = localRo.addSimpleAnnotation(roresource, "rdf:type", ROTERMS.resource)
annotationsCnt = 0
deleteRO(self.rosrs, urlparse.urljoin(self.rosrs.baseuri(), "TestPushRO/"))
(_,_,rouri,_) = createRO(self.rosrs, "TestPushRO")
remoteRo = ro_remote_metadata(ro_test_config, self.rosrs, rouri)
remoteRo.aggregateResourceExt("http://www.anotherexample.org")
resourcesInt = (
[ rdflib.URIRef("README-ro-test-1")
, rdflib.URIRef("minim.rdf")
, rdflib.URIRef("subdir1/subdir1-file.txt")
, rdflib.URIRef("subdir2/subdir2-file.txt")
, rdflib.URIRef("filename%20with%20spaces.txt")
, rdflib.URIRef("filename%23with%23hashes.txt")
, rdflib.URIRef(ann1)
, rdflib.URIRef(ann2)
, rdflib.URIRef(ann3)
])
resourcesIntCnt = 0
for (action, resuri) in ro_rosrs_sync.pushResearchObject(localRo, remoteRo):
log.debug("The following object has been pushed: %s (%s)"%(resuri, action))
# this assumes that the above is the only external resource
if action == ro_rosrs_sync.ACTION_AGGREGATE_EXTERNAL:
self.assertEqual(resuri, rdflib.URIRef("http://www.example.org"), "The external resource is pushed")
self.assertTrue(localRo.isAggregatedResource(resuri), "Resource that is pushed is aggregated locally")
elif action == ro_rosrs_sync.ACTION_AGGREGATE_INTERNAL:
self.assertTrue(localRo.getComponentUriRel(resuri) in resourcesInt, "Resource that is pushed is aggregated locally")
resourcesIntCnt += 1
elif action == ro_rosrs_sync.ACTION_DELETE:
self.assertFalse(localRo.isAggregatedResource(resuri), "Resource that is deaggregated in ROSRS is not aggregated locally")
self.assertEqual(resuri, rdflib.URIRef("http://www.anotherexample.org"), "The external resource is deaggregated (%s)"%resuri)
elif action == ro_rosrs_sync.ACTION_AGGREGATE_ANNOTATION:
self.assertTrue(localRo.isAnnotationNode(resuri), "Annotation that is pushed is aggregated locally (%s)"%(resuri))
annotationsCnt += 1
elif action == ro_rosrs_sync.ACTION_DELETE_ANNOTATION:
self.assertFalse(localRo.isAnnotationNode(resuri), "Annotation that is deaggregated in ROSRS is not aggregated locally")
pass
elif action == ro_rosrs_sync.ACTION_ERROR:
log.warn("Error, not necessarily a fail: %s"%resuri)
else:
self.fail("Unexpected action %s"%action)
self.assertEqual(len(resourcesInt), resourcesIntCnt, "All internal resources were aggregated (should be %d was %d)"%(len(resourcesInt), resourcesIntCnt))
# 3 annotations + manifest which in RO manager also annotates the RO
self.assertEqual(4, annotationsCnt, "All annotations were aggregated (should be %d was %d)"%(4, annotationsCnt))
for (action, resuri) in ro_rosrs_sync.pushResearchObject(localRo, remoteRo):
if action == ro_rosrs_sync.ACTION_UPDATE_ANNOTATION:
self.assertTrue(localRo.isAnnotationNode(resuri), "Annotations that is updated is aggregated locally (%s)"%(resuri))
elif action == ro_rosrs_sync.ACTION_UPDATE_OVERWRITE:
# see https://jira.man.poznan.pl/jira/browse/WFE-671
self.assertTrue(resuri in [rdflib.URIRef(ann1), rdflib.URIRef(ann2), rdflib.URIRef(ann3)], "Annotation bodies can be uploaded twice")
elif action == ro_rosrs_sync.ACTION_ERROR:
log.warn("Error, not necessarily a fail: %s"%resuri)
elif not action == ro_rosrs_sync.ACTION_SKIP:
self.fail("Nothing else should be pushed again (%s, %s)"%(action, resuri))
# Clean up
remoteRo.delete()
self.deleteTestRo(rodir)
return