def testEvalAllPresent(self):
"""
Evaluate complete RO against Minim description
"""
self.setupConfig()
rodir = self.createTestRo(testbase, "test-data-1", "RO test minim", "ro-testMinim")
self.populateTestRo(testbase, rodir)
rometa = ro_metadata(ro_config, rodir)
(g,evalresult) = ro_eval_minim.evaluate(rometa,
"Minim-UserRequirements.rdf", # Minim file
"docs/UserRequirements-astro.csv", # Target resource
"create") # Purpose
self.assertIn(MINIM.fullySatisfies, evalresult['summary'])
self.assertIn(MINIM.nominallySatisfies, evalresult['summary'])
self.assertIn(MINIM.minimallySatisfies, evalresult['summary'])
self.assertEquals(evalresult['missingMust'], [])
self.assertEquals(evalresult['missingShould'], [])
self.assertEquals(evalresult['missingMay'], [])
self.assertEquals(evalresult['rouri'], rometa.getRoUri())
self.assertEquals(evalresult['minimuri'], rometa.getComponentUri("Minim-UserRequirements.rdf"))
self.assertEquals(evalresult['target'], "docs/UserRequirements-astro.csv")
self.assertEquals(evalresult['purpose'], "create")
self.assertEquals(evalresult['constrainturi'],
rometa.getComponentUriAbs("Minim-UserRequirements.rdf#create/docs/UserRequirements-astro.csv"))
self.assertEquals(evalresult['modeluri'],
rometa.getComponentUriAbs("Minim-UserRequirements.rdf#runnableRequirementRO"))
self.deleteTestRo(rodir)
return
def testEvalQueryTestChemboxFail(self):
"""
Test for failing chembox requirement
"""
self.setupConfig()
rodir = self.createTestRo(testbase, "test-chembox", "RO test minim", "ro-testMinim")
rouri = ro_manifest.getRoUri(rodir)
self.populateTestRo(testbase, rodir)
rometa = ro_metadata(ro_config, rodir)
resuri = rometa.getComponentUriAbs("http://purl.org/net/chembox/Ethane")
rometa.addGraphAnnotation(resuri, "Ethane.ttl")
# Now run evaluation against test RO
(g, evalresult) = ro_eval_minim.evaluate(rometa,
"Minim-chembox.ttl", # Minim file
resuri, # Target resource
"fail") # Purpose
log.debug("ro_eval_minim.evaluate result:\n----\n%s"%(repr(evalresult)))
self.assertNotIn(MINIM.fullySatisfies, evalresult['summary'])
self.assertNotIn(MINIM.nominallySatisfies, evalresult['summary'])
self.assertNotIn(MINIM.minimallySatisfies, evalresult['summary'])
self.assertEquals(len(evalresult['missingMust']), 1)
self.assertEquals(evalresult['missingMust'][0][0]['seq'], 'This test should fail')
self.assertEquals(evalresult['missingShould'], [])
self.assertEquals(evalresult['missingMay'], [])
self.assertEquals(evalresult['rouri'], rometa.getRoUri())
self.assertEquals(evalresult['minimuri'], rometa.getComponentUri("Minim-chembox.ttl"))
self.assertEquals(evalresult['target'], resuri)
self.assertEquals(evalresult['purpose'], "fail")
self.deleteTestRo(rodir)
return
def testEvaluateChecklistRemoteFail(self):
# Config remote testing
remotehost = "http://andros.zoo.ox.ac.uk"
remoterobase = "/aleix/ro-catalogue/v0.1/"
remoteroname = "wf74-repeat-fail/"
remoteminim = "simple-requirements-minim.rdf"
#self.setupConfig()
rouri = remotehost+remoterobase+remoteroname
rometa = ro_metadata(ro_config, rouri)
minimuri = rometa.getComponentUri(remoteminim)
# create rometa object
rometa = ro_metadata(ro_config, rouri)
# invoke evaluation service
# ro_eval_minim.evaluate(rometa, minim, target, purpose)
evalresult = ro_eval_minim.evaluate(rometa, minimuri, rouri, "Repeatable")
self.assertEqual(evalresult['rouri'], rdflib.URIRef(rouri))
self.assertEqual(evalresult['minimuri'], rdflib.URIRef(minimuri))
self.assertEqual(evalresult['target'], rouri)
self.assertEqual(evalresult['purpose'], "Repeatable")
self.assertEqual(evalresult['constrainturi'],
rometa.getComponentUri("simple-requirements-minim.rdf#repeatable_RO"))
self.assertEqual(evalresult['modeluri'],
rometa.getComponentUri("simple-requirements-minim.rdf#repeatable_RO_model"))
self.assertNotIn(MINIM.fullySatisfies, evalresult['summary'])
self.assertNotIn(MINIM.nominallySatisfies, evalresult['summary'])
self.assertNotIn(MINIM.minimallySatisfies, evalresult['summary'])
self.assertEqual(len(evalresult['missingMust']), 1)
self.assertEqual(len(evalresult['missingShould']), 0)
self.assertEqual(len(evalresult['missingMay']), 0)
self.assertEqual(len(evalresult['satisfied']), 4)
return
def testEvalQueryTestModelMin(self):
"""
Evaluate RO against minimal Minim description using just QueryTestRules
"""
self.setupConfig()
rodir = self.createTestRo(testbase, "test-data-2", "RO test minim", "ro-testMinim")
rouri = ro_manifest.getRoUri(rodir)
self.populateTestRo(testbase, rodir)
rometa = ro_metadata(ro_config, rodir)
(g, evalresult) = ro_eval_minim.evaluate(rometa,
"Minim-UserRequirements2-min.rdf", # Minim file
"data/UserRequirements-astro.ods", # Target resource
"create") # Purpose
log.debug("ro_eval_minim.evaluate result:\n----\n%s"%(repr(evalresult)))
self.assertIn(MINIM.fullySatisfies, evalresult['summary'])
self.assertIn(MINIM.nominallySatisfies, evalresult['summary'])
self.assertIn(MINIM.minimallySatisfies, evalresult['summary'])
self.assertEquals(evalresult['missingMust'], [])
self.assertEquals(evalresult['missingShould'], [])
self.assertEquals(evalresult['missingMay'], [])
self.assertEquals(evalresult['rouri'], rometa.getRoUri())
self.assertEquals(evalresult['minimuri'], rometa.getComponentUri("Minim-UserRequirements2-min.rdf"))
self.assertEquals(evalresult['target'], "data/UserRequirements-astro.ods")
self.assertEquals(evalresult['purpose'], "create")
self.assertEquals(evalresult['constrainturi'],
rometa.getComponentUriAbs("Minim-UserRequirements2-min.rdf#create/data/UserRequirements-astro.ods"))
self.assertEquals(evalresult['modeluri'],
rometa.getComponentUriAbs("Minim-UserRequirements2-min.rdf#runnableRO"))
self.deleteTestRo(rodir)
return
def real_evaluate(request):
# From: http://tools.ietf.org/html/rfc3986#section-2.1
# gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
# sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
# / "*" / "+" / "," / ";" / "="
quotesafe = ":/?#[]@!$&'()*+,;=" + "%"
# isolate parameters (keep invalid URI characters %-encoded)
RO = urllib.quote(request.params["RO"], quotesafe)
minim = urllib.quote(request.params["minim"], quotesafe)
target = urllib.quote(request.params.get("target","."), quotesafe)
purpose = request.params["purpose"]
log.info("Evaluate RO %s, minim %s, target %s, purpose %s"%(RO,minim,target,purpose))
# create rometa object
# @@TODO: use proper configuration and credentials
ROparse = urlparse.urlparse(RO)
rosrs_uri = (ROparse.scheme or "http") + "://" + (ROparse.netloc or "localhost:8000") + "/"
ro_config = {
"annotationTypes": annotationTypes,
"annotationPrefixes": annotationPrefixes,
"rosrs_uri": rosrs_uri,
#"rosrs_uri": target,
#"rosrs_uri": "http://sandbox.wf4ever-project.org/rodl/ROs/",
#"rosrs_access_token": "ac14dd1a-ab59-40ec-b510-ffdb01a85473",
"rosrs_access_token": None,
}
rometa = ro_metadata(ro_config, RO)
log.info("rometa.rouri: %s"%(rometa.rouri) )
# invoke evaluation service
(graph, evalresult) = ro_eval_minim.evaluate(rometa, minim, target, purpose)
log.debug("evaluate:results: \n"+json.dumps(evalresult, indent=2))
# Assemble graph of results
graph = ro_eval_minim.evalResultGraph(graph, evalresult)
return graph
def testEvalShouldMissing(self):
"""
Evaluate complete RO against Minim description
"""
self.setupConfig()
rodir = self.createTestRo(testbase, "test-data-1", "RO test minim", "ro-testMinim")
self.populateTestRo(testbase, rodir)
rometa = ro_metadata(ro_config, rodir)
minimbase = rometa.getComponentUri("Minim-UserRequirements.rdf")
modeluri = ro_minim.getElementUri(minimbase, "#missingShouldRequirement")
(g,evalresult) = ro_eval_minim.evaluate(rometa,
"Minim-UserRequirements.rdf", # Minim file
"docs/UserRequirements-bio.html", # Target resource
"create") # Purpose
missing_should = (
{ 'level': "SHOULD"
, 'model': modeluri
, 'label': rdflib.Literal("aggregates docs/missing.css")
, 'datarule':
{ 'aggregates': rometa.getComponentUri("docs/missing.css")
, 'show': None
, 'showpass': None
, 'showfail': None
, 'derives': ro_minim.getElementUri(minimbase, "#isPresent/docs/missing.css")
}
, 'uri': ro_minim.getElementUri(minimbase, "#isPresent/docs/missing.css")
})
self.maxDiff=None
self.assertEquals(evalresult['summary'], [MINIM.minimallySatisfies])
self.assertEquals(evalresult['missingMust'], [] )
self.assertEquals(evalresult['missingShould'], [(missing_should,{})] )
self.assertEquals(evalresult['missingMay'], [] )
self.deleteTestRo(rodir)
return
def evaluate(progname, configbase, options, args):
"""
Evaluate RO
ro evaluate checklist [ -d <dir> ] <minim> <purpose> [ <target> ]"
"""
log.debug("evaluate: progname %s, configbase %s, args %s" %
(progname, configbase, repr(args)))
ro_config = getroconfig(configbase, options)
ro_options = (
{ "rodir": options.rodir or ""
, "function": args[2]
})
log.debug("ro_options: " + repr(ro_options))
ro_ref = ro_root_reference(progname + " annotations", ro_config, None, ro_options['rodir'])
if not ro_ref: return 1
# Evaluate...
if ro_options["function"] == "checklist":
if len(args) not in [5, 6]:
print ("%s evaluate checklist: wrong number of arguments provided" % (progname))
print ("Usage: %s evaluate checklist [ -d <dir> ] [ -a | -l <level> ] <minim> <purpose> [ <target> ]" % (progname))
return 1
levels = ["summary", "must", "should", "may", "full"]
if options.level not in ["summary", "must", "should", "may", "full"]:
print ("%s evaluate checklist: invalid reporting level %s, must be one of %s" %
(progname, options.level, repr(levels)))
return 1
ro_options["minim"] = ((len(args) > 3) and args[3]) or "minim.rdf"
ro_options["purpose"] = ((len(args) > 4) and args[4]) or "create"
ro_options["target"] = ((len(args) > 5) and args[5]) or "."
if options.verbose:
print "ro evaluate %(function)s -d \"%(rodir)s\" %(minim)s %(purpose)s %(target)s" % ro_options
rometa = ro_metadata(ro_config, ro_ref)
(minimgraph, evalresult) = ro_eval_minim.evaluate(rometa,
ro_options["minim"], ro_options["target"], ro_options["purpose"])
if options.verbose:
print "== Evaluation result =="
print json.dumps(evalresult, indent=2)
if options.outformat and options.outformat.upper() in RDFTYPSERIALIZERMAP:
# RDF output
graph = ro_eval_minim.evalResultGraph(minimgraph, evalresult)
graph.serialize(destination=sys.stdout,
format=RDFTYPSERIALIZERMAP[options.outformat.upper()])
else:
ro_eval_minim.format(evalresult,
{ "detail" : "full" if options.all else options.level },
sys.stdout)
# elif ... other functions here
else:
print ("%s evaluate: unrecognized function provided (%s)" % (progname, ro_options["function"]))
print ("Usage:")
print (" %s evaluate checklist [ -d <dir> ] [ -a | -l <level> ] <minim> <purpose> [ <target> ]" % (progname))
return 1
return 0
def real_evaluate(request):
# isolate parameters
RO = request.params["RO"]
minim = request.params["minim"]
target = request.params.get("target",".")
purpose = request.params["purpose"]
# create rometa object
# @@TODO: use proper configuration and credentials
ro_config = {
"annotationTypes": annotationTypes,
"annotationPrefixes": annotationPrefixes,
#"rosrs_uri": target,
"rosrs_uri": "http://sandbox.wf4ever-project.org/rodl/ROs/",
"rosrs_access_token": "32801fc0-1df1-4e34-b",
}
rometa = ro_metadata(ro_config, RO)
# invoke evaluation service
# ro_eval_minim.evaluate(rometa, minim, target, purpose)
log.info("Evaluate RO %s, minim %s, target %s, purpose %s"%(RO,minim,target,purpose))
(graph, evalresult) = ro_eval_minim.evaluate(rometa, minim, target, purpose)
log.debug("evaluate:results: \n"+json.dumps(evalresult, indent=2))
# Assemble graph of results
#graph = rdflib.Graph() // start with minim graph
graph.bind("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#")
graph.bind("result", "http://www.w3.org/2001/sw/DataAccess/tests/result-set#")
graph.bind("minim", "http://purl.org/minim/minim#")
rouri = rdflib.URIRef(evalresult['rouri'])
graph.add( (rouri, MINIM.testedConstraint, rdflib.URIRef(evalresult['constrainturi'])) )
graph.add( (rouri, MINIM.testedPurpose, rdflib.Literal(evalresult['purpose'])) )
graph.add( (rouri, MINIM.testedTarget, rdflib.Literal(evalresult['target'])) )
graph.add( (rouri, MINIM.minimUri, rdflib.URIRef(evalresult['minimuri'])) )
graph.add( (rouri, MINIM.modelUri, rdflib.URIRef(evalresult['modeluri'])) )
for level in evalresult['summary']:
log.info("RO %s, level %s, model %s"%(rouri,level,evalresult['modeluri']))
graph.add( (rouri, level, rdflib.URIRef(evalresult['modeluri'])) )
# Add details for all rules tested...
def addRequirementsDetail(results, satlevel):
for (req, binding) in results:
b = rdflib.BNode()
graph.add( (rouri, satlevel, b) )
graph.add( (b, MINIM.tryRequirement, req['uri']) )
for k in binding:
b2 = rdflib.BNode()
graph.add( (b, RESULT.binding, b2) )
graph.add( (b2, RESULT.variable, rdflib.Literal(k)) )
graph.add( (b2, RESULT.value, rdflib.Literal(binding[k])) )
addRequirementsDetail(evalresult['satisfied'], MINIM.satisfied)
addRequirementsDetail(evalresult['missingMay'], MINIM.missingMay)
addRequirementsDetail(evalresult['missingShould'], MINIM.missingShould)
addRequirementsDetail(evalresult['missingMust'], MINIM.missingMust)
return graph
def testEvalMayMissing(self):
"""
Evaluate complete RO against Minim description
"""
self.setupConfig()
rodir = self.createTestRo(testbase, "test-data-1", "RO test minim", "ro-testMinim")
self.populateTestRo(testbase, rodir)
rometa = ro_metadata(ro_config, rodir)
minimbase = rometa.getComponentUri("Minim-UserRequirements.rdf")
modeluri = ro_minim.getElementUri(minimbase, "#missingMayRequirement")
(g,evalresult) = ro_eval_minim.evaluate(rometa,
"Minim-UserRequirements.rdf", # Minim file
"docs/UserRequirements-bio.pdf", # Target resource
"create") # Purpose
missing_may = (
{ 'level': "MAY"
, 'model': modeluri
, 'label': rdflib.Literal("aggregates docs/missing.css")
, 'datarule':
{ 'aggregates': rometa.getComponentUri("docs/missing.css")
, 'show': rdflib.Literal("05 - aggregates docs/missing.css")
, 'showpass': None
, 'showfail': None
, 'showmiss': None
, 'list': []
, 'listpass': []
, 'listfail': []
}
, 'uri': ro_minim.getElementUri(minimbase, "#isPresent/docs/missing.css")
, 'ruleuri': evalresult['missingMay'][0][0]['ruleuri']
, 'seq': "05 - aggregates docs/missing.css"
})
self.maxDiff=None
self.assertEquals(evalresult['summary'], [MINIM.nominallySatisfies, MINIM.minimallySatisfies])
self.assertEquals(evalresult['missingMust'], [] )
self.assertEquals(evalresult['missingShould'], [] )
self.assertEquals(evalresult['missingMay'], [(missing_may,{})] )
self.assertEquals(evalresult['rouri'], rometa.getRoUri())
self.assertEquals(evalresult['minimuri'], rometa.getComponentUri("Minim-UserRequirements.rdf"))
self.assertEquals(evalresult['target'], "docs/UserRequirements-bio.pdf")
self.assertEquals(evalresult['purpose'], "create")
self.assertEquals(evalresult['constrainturi'],
rometa.getComponentUriAbs("Minim-UserRequirements.rdf#create/docs/UserRequirements-bio.pdf"))
self.assertEquals(evalresult['modeluri'],
rometa.getComponentUriAbs("Minim-UserRequirements.rdf#missingMayRequirement"))
self.deleteTestRo(rodir)
return
def testEvaluateRDF(self):
self.setupConfig()
rodir = self.createTestRo(testbase, "test-chembox", "RO test minim", "ro-testMinim")
rouri = ro_manifest.getRoUri(rodir)
self.populateTestRo(testbase, rodir)
rometa = ro_metadata(ro_config, rodir)
resuri = rometa.getComponentUriAbs("http://purl.org/net/chembox/Ethane")
reslabel = "Ethane"
rometa.addGraphAnnotation(resuri, "Ethane.ttl")
# Now run evaluation against test RO
(minimgr, evalresult) = ro_eval_minim.evaluate(rometa,
"Minim-chembox.ttl", # Minim file
resuri, # Target resource
"complete") # Purpose
resultgr = ro_eval_minim.evalResultGraph(minimgr, evalresult)
log.debug("------ resultgr:\n%s\n----"%(resultgr.serialize(format='turtle'))) # pretty-xml
## print "------ resultgr:\n%s\n----"%(resultgr.serialize(format='turtle'))
# Check response returned
modeluri = rdflib.URIRef('http://example.com/chembox-samples/minim_model')
prefixes = make_sparql_prefixes()
probequeries = (
[ '''ASK { _:r minim:testedRO <%s> ; minim:minimUri <%s> }'''%
(rouri, rometa.getComponentUri("Minim-chembox.ttl"))
, '''ASK { _:r minim:testedRO <%s> ; minim:testedModel <%s> }'''%
(rouri, modeluri)
, '''ASK { _:r minim:testedTarget <%s> ; minim:satisfied [ minim:tryMessage "%s" ] }'''%
(resuri, "InChI identifier is present")
, '''ASK { _:r minim:testedTarget <%s> ; minim:satisfied [ minim:tryMessage "%s" ] }'''%
(resuri, "ChemSpider identifier is present")
, '''ASK { _:r minim:testedTarget <%s> ; minim:missingMay [ minim:tryMessage "%s" ] }'''%
(resuri, "No synomym is present")
, '''ASK { _:r minim:testedTarget <%s> ; minim:nominallySatisfies <%s> }'''%
(resuri, modeluri)
, '''ASK { _:r minim:testedTarget <%s> ; minim:minimallySatisfies <%s> }'''%
(resuri, modeluri)
, '''ASK { <%s> rdfs:label "%s" }'''%
(resuri, reslabel)
])
for q in probequeries:
r = resultgr.query(prefixes+q)
self.assertEqual(r.type, 'ASK', "Result type %s for: %s"%(r.type, q))
self.assertTrue(r.askAnswer, "Failed query: %s"%(q))
self.deleteTestRo(rodir)
return
def testEvalQueryTestReportList(self):
"""
Test QueryTestRules reporting list of failed query probes
"""
self.setupConfig()
rodir = self.createTestRo(testbase, "test-data-2", "RO test minim", "ro-testMinim")
rouri = ro_manifest.getRoUri(rodir)
self.populateTestRo(testbase, rodir)
rometa = ro_metadata(ro_config, rodir)
resuri = rometa.getComponentUriAbs("data/NoSuchResource")
rometa.addSimpleAnnotation(resuri, "rdfs:label", "Test label")
# Now run evaluation against test RO
(g, evalresult) = ro_eval_minim.evaluate(rometa,
"Minim-UserRequirements2.rdf", # Minim file
"data/NoSuchResource", # Target resource
"report list") # Purpose
log.debug("ro_eval_minim.evaluate result:\n----\n%s"%(repr(evalresult)))
self.assertNotIn(MINIM.fullySatisfies, evalresult['summary'])
self.assertNotIn(MINIM.nominallySatisfies, evalresult['summary'])
self.assertIn(MINIM.minimallySatisfies, evalresult['summary'])
self.assertEquals(evalresult['missingMust'], [])
self.assertEquals(len(evalresult['missingShould']), 1)
self.assertEquals(evalresult['missingShould'][0][0]['seq'], '04 - aggregates data/NoSuchResource')
self.assertEquals(evalresult['missingMay'], [])
self.assertEquals(evalresult['rouri'], rometa.getRoUri())
self.assertEquals(evalresult['minimuri'], rometa.getComponentUri("Minim-UserRequirements2.rdf"))
self.assertEquals(evalresult['target'], "data/NoSuchResource")
self.assertEquals(evalresult['purpose'], "report list")
self.assertEquals(evalresult['constrainturi'],
rometa.getComponentUriAbs("Minim-UserRequirements2.rdf#report/data/NoSuchResource"))
self.assertEquals(evalresult['modeluri'],
rometa.getComponentUriAbs("Minim-UserRequirements2.rdf#reportList"))
# Check result bindings returned
self.assertEquals(evalresult['missingShould'][0][1]['_count'], 1)
self.assertEquals(evalresult['missingShould'][0][1]['_fileuri'], rometa.getComponentUri("data/NoSuchResource"))
self.assertEquals(evalresult['missingShould'][0][1]['targetres'], rometa.getComponentUri("data/NoSuchResource"))
self.assertEquals(evalresult['missingShould'][0][1]['ro'], str(rometa.getRoUri()))
self.assertEquals(evalresult['missingShould'][0][1]['ro_list'], [str(rometa.getRoUri())])
# Clean up
self.deleteTestRo(rodir)
return
