def testEvalAllPresent(self): """ Evaluate complete RO against Minim description """ self.setupConfig() rodir = self.createTestRo(testbase, "test-data-1", "RO test minim", "ro-testMinim") self.populateTestRo(testbase, rodir) rometa = ro_metadata(ro_config, rodir) (g,evalresult) = ro_eval_minim.evaluate(rometa, "Minim-UserRequirements.rdf", # Minim file "docs/UserRequirements-astro.csv", # Target resource "create") # Purpose self.assertIn(MINIM.fullySatisfies, evalresult['summary']) self.assertIn(MINIM.nominallySatisfies, evalresult['summary']) self.assertIn(MINIM.minimallySatisfies, evalresult['summary']) self.assertEquals(evalresult['missingMust'], []) self.assertEquals(evalresult['missingShould'], []) self.assertEquals(evalresult['missingMay'], []) self.assertEquals(evalresult['rouri'], rometa.getRoUri()) self.assertEquals(evalresult['minimuri'], rometa.getComponentUri("Minim-UserRequirements.rdf")) self.assertEquals(evalresult['target'], "docs/UserRequirements-astro.csv") self.assertEquals(evalresult['purpose'], "create") self.assertEquals(evalresult['constrainturi'], rometa.getComponentUriAbs("Minim-UserRequirements.rdf#create/docs/UserRequirements-astro.csv")) self.assertEquals(evalresult['modeluri'], rometa.getComponentUriAbs("Minim-UserRequirements.rdf#runnableRequirementRO")) self.deleteTestRo(rodir) return
def testEvalQueryTestChemboxFail(self): """ Test for failing chembox requirement """ self.setupConfig() rodir = self.createTestRo(testbase, "test-chembox", "RO test minim", "ro-testMinim") rouri = ro_manifest.getRoUri(rodir) self.populateTestRo(testbase, rodir) rometa = ro_metadata(ro_config, rodir) resuri = rometa.getComponentUriAbs("http://purl.org/net/chembox/Ethane") rometa.addGraphAnnotation(resuri, "Ethane.ttl") # Now run evaluation against test RO (g, evalresult) = ro_eval_minim.evaluate(rometa, "Minim-chembox.ttl", # Minim file resuri, # Target resource "fail") # Purpose log.debug("ro_eval_minim.evaluate result:\n----\n%s"%(repr(evalresult))) self.assertNotIn(MINIM.fullySatisfies, evalresult['summary']) self.assertNotIn(MINIM.nominallySatisfies, evalresult['summary']) self.assertNotIn(MINIM.minimallySatisfies, evalresult['summary']) self.assertEquals(len(evalresult['missingMust']), 1) self.assertEquals(evalresult['missingMust'][0][0]['seq'], 'This test should fail') self.assertEquals(evalresult['missingShould'], []) self.assertEquals(evalresult['missingMay'], []) self.assertEquals(evalresult['rouri'], rometa.getRoUri()) self.assertEquals(evalresult['minimuri'], rometa.getComponentUri("Minim-chembox.ttl")) self.assertEquals(evalresult['target'], resuri) self.assertEquals(evalresult['purpose'], "fail") self.deleteTestRo(rodir) return
def testEvaluateChecklistRemoteFail(self): # Config remote testing remotehost = "http://andros.zoo.ox.ac.uk" remoterobase = "/aleix/ro-catalogue/v0.1/" remoteroname = "wf74-repeat-fail/" remoteminim = "simple-requirements-minim.rdf" #self.setupConfig() rouri = remotehost+remoterobase+remoteroname rometa = ro_metadata(ro_config, rouri) minimuri = rometa.getComponentUri(remoteminim) # create rometa object rometa = ro_metadata(ro_config, rouri) # invoke evaluation service # ro_eval_minim.evaluate(rometa, minim, target, purpose) evalresult = ro_eval_minim.evaluate(rometa, minimuri, rouri, "Repeatable") self.assertEqual(evalresult['rouri'], rdflib.URIRef(rouri)) self.assertEqual(evalresult['minimuri'], rdflib.URIRef(minimuri)) self.assertEqual(evalresult['target'], rouri) self.assertEqual(evalresult['purpose'], "Repeatable") self.assertEqual(evalresult['constrainturi'], rometa.getComponentUri("simple-requirements-minim.rdf#repeatable_RO")) self.assertEqual(evalresult['modeluri'], rometa.getComponentUri("simple-requirements-minim.rdf#repeatable_RO_model")) self.assertNotIn(MINIM.fullySatisfies, evalresult['summary']) self.assertNotIn(MINIM.nominallySatisfies, evalresult['summary']) self.assertNotIn(MINIM.minimallySatisfies, evalresult['summary']) self.assertEqual(len(evalresult['missingMust']), 1) self.assertEqual(len(evalresult['missingShould']), 0) self.assertEqual(len(evalresult['missingMay']), 0) self.assertEqual(len(evalresult['satisfied']), 4) return
def testEvalQueryTestModelMin(self): """ Evaluate RO against minimal Minim description using just QueryTestRules """ self.setupConfig() rodir = self.createTestRo(testbase, "test-data-2", "RO test minim", "ro-testMinim") rouri = ro_manifest.getRoUri(rodir) self.populateTestRo(testbase, rodir) rometa = ro_metadata(ro_config, rodir) (g, evalresult) = ro_eval_minim.evaluate(rometa, "Minim-UserRequirements2-min.rdf", # Minim file "data/UserRequirements-astro.ods", # Target resource "create") # Purpose log.debug("ro_eval_minim.evaluate result:\n----\n%s"%(repr(evalresult))) self.assertIn(MINIM.fullySatisfies, evalresult['summary']) self.assertIn(MINIM.nominallySatisfies, evalresult['summary']) self.assertIn(MINIM.minimallySatisfies, evalresult['summary']) self.assertEquals(evalresult['missingMust'], []) self.assertEquals(evalresult['missingShould'], []) self.assertEquals(evalresult['missingMay'], []) self.assertEquals(evalresult['rouri'], rometa.getRoUri()) self.assertEquals(evalresult['minimuri'], rometa.getComponentUri("Minim-UserRequirements2-min.rdf")) self.assertEquals(evalresult['target'], "data/UserRequirements-astro.ods") self.assertEquals(evalresult['purpose'], "create") self.assertEquals(evalresult['constrainturi'], rometa.getComponentUriAbs("Minim-UserRequirements2-min.rdf#create/data/UserRequirements-astro.ods")) self.assertEquals(evalresult['modeluri'], rometa.getComponentUriAbs("Minim-UserRequirements2-min.rdf#runnableRO")) self.deleteTestRo(rodir) return
def real_evaluate(request): # From: http://tools.ietf.org/html/rfc3986#section-2.1 # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" # sub-delims = "!" / "$" / "&" / "'" / "(" / ")" # / "*" / "+" / "," / ";" / "=" quotesafe = ":/?#[]@!$&'()*+,;=" + "%" # isolate parameters (keep invalid URI characters %-encoded) RO = urllib.quote(request.params["RO"], quotesafe) minim = urllib.quote(request.params["minim"], quotesafe) target = urllib.quote(request.params.get("target","."), quotesafe) purpose = request.params["purpose"] log.info("Evaluate RO %s, minim %s, target %s, purpose %s"%(RO,minim,target,purpose)) # create rometa object # @@TODO: use proper configuration and credentials ROparse = urlparse.urlparse(RO) rosrs_uri = (ROparse.scheme or "http") + "://" + (ROparse.netloc or "localhost:8000") + "/" ro_config = { "annotationTypes": annotationTypes, "annotationPrefixes": annotationPrefixes, "rosrs_uri": rosrs_uri, #"rosrs_uri": target, #"rosrs_uri": "http://sandbox.wf4ever-project.org/rodl/ROs/", #"rosrs_access_token": "ac14dd1a-ab59-40ec-b510-ffdb01a85473", "rosrs_access_token": None, } rometa = ro_metadata(ro_config, RO) log.info("rometa.rouri: %s"%(rometa.rouri) ) # invoke evaluation service (graph, evalresult) = ro_eval_minim.evaluate(rometa, minim, target, purpose) log.debug("evaluate:results: \n"+json.dumps(evalresult, indent=2)) # Assemble graph of results graph = ro_eval_minim.evalResultGraph(graph, evalresult) return graph
def testEvalShouldMissing(self): """ Evaluate complete RO against Minim description """ self.setupConfig() rodir = self.createTestRo(testbase, "test-data-1", "RO test minim", "ro-testMinim") self.populateTestRo(testbase, rodir) rometa = ro_metadata(ro_config, rodir) minimbase = rometa.getComponentUri("Minim-UserRequirements.rdf") modeluri = ro_minim.getElementUri(minimbase, "#missingShouldRequirement") (g,evalresult) = ro_eval_minim.evaluate(rometa, "Minim-UserRequirements.rdf", # Minim file "docs/UserRequirements-bio.html", # Target resource "create") # Purpose missing_should = ( { 'level': "SHOULD" , 'model': modeluri , 'label': rdflib.Literal("aggregates docs/missing.css") , 'datarule': { 'aggregates': rometa.getComponentUri("docs/missing.css") , 'show': None , 'showpass': None , 'showfail': None , 'derives': ro_minim.getElementUri(minimbase, "#isPresent/docs/missing.css") } , 'uri': ro_minim.getElementUri(minimbase, "#isPresent/docs/missing.css") }) self.maxDiff=None self.assertEquals(evalresult['summary'], [MINIM.minimallySatisfies]) self.assertEquals(evalresult['missingMust'], [] ) self.assertEquals(evalresult['missingShould'], [(missing_should,{})] ) self.assertEquals(evalresult['missingMay'], [] ) self.deleteTestRo(rodir) return
def evaluate(progname, configbase, options, args): """ Evaluate RO ro evaluate checklist [ -d <dir> ] <minim> <purpose> [ <target> ]" """ log.debug("evaluate: progname %s, configbase %s, args %s" % (progname, configbase, repr(args))) ro_config = getroconfig(configbase, options) ro_options = ( { "rodir": options.rodir or "" , "function": args[2] }) log.debug("ro_options: " + repr(ro_options)) ro_ref = ro_root_reference(progname + " annotations", ro_config, None, ro_options['rodir']) if not ro_ref: return 1 # Evaluate... if ro_options["function"] == "checklist": if len(args) not in [5, 6]: print ("%s evaluate checklist: wrong number of arguments provided" % (progname)) print ("Usage: %s evaluate checklist [ -d <dir> ] [ -a | -l <level> ] <minim> <purpose> [ <target> ]" % (progname)) return 1 levels = ["summary", "must", "should", "may", "full"] if options.level not in ["summary", "must", "should", "may", "full"]: print ("%s evaluate checklist: invalid reporting level %s, must be one of %s" % (progname, options.level, repr(levels))) return 1 ro_options["minim"] = ((len(args) > 3) and args[3]) or "minim.rdf" ro_options["purpose"] = ((len(args) > 4) and args[4]) or "create" ro_options["target"] = ((len(args) > 5) and args[5]) or "." if options.verbose: print "ro evaluate %(function)s -d \"%(rodir)s\" %(minim)s %(purpose)s %(target)s" % ro_options rometa = ro_metadata(ro_config, ro_ref) (minimgraph, evalresult) = ro_eval_minim.evaluate(rometa, ro_options["minim"], ro_options["target"], ro_options["purpose"]) if options.verbose: print "== Evaluation result ==" print json.dumps(evalresult, indent=2) if options.outformat and options.outformat.upper() in RDFTYPSERIALIZERMAP: # RDF output graph = ro_eval_minim.evalResultGraph(minimgraph, evalresult) graph.serialize(destination=sys.stdout, format=RDFTYPSERIALIZERMAP[options.outformat.upper()]) else: ro_eval_minim.format(evalresult, { "detail" : "full" if options.all else options.level }, sys.stdout) # elif ... other functions here else: print ("%s evaluate: unrecognized function provided (%s)" % (progname, ro_options["function"])) print ("Usage:") print (" %s evaluate checklist [ -d <dir> ] [ -a | -l <level> ] <minim> <purpose> [ <target> ]" % (progname)) return 1 return 0
def real_evaluate(request): # isolate parameters RO = request.params["RO"] minim = request.params["minim"] target = request.params.get("target",".") purpose = request.params["purpose"] # create rometa object # @@TODO: use proper configuration and credentials ro_config = { "annotationTypes": annotationTypes, "annotationPrefixes": annotationPrefixes, #"rosrs_uri": target, "rosrs_uri": "http://sandbox.wf4ever-project.org/rodl/ROs/", "rosrs_access_token": "32801fc0-1df1-4e34-b", } rometa = ro_metadata(ro_config, RO) # invoke evaluation service # ro_eval_minim.evaluate(rometa, minim, target, purpose) log.info("Evaluate RO %s, minim %s, target %s, purpose %s"%(RO,minim,target,purpose)) (graph, evalresult) = ro_eval_minim.evaluate(rometa, minim, target, purpose) log.debug("evaluate:results: \n"+json.dumps(evalresult, indent=2)) # Assemble graph of results #graph = rdflib.Graph() // start with minim graph graph.bind("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#") graph.bind("result", "http://www.w3.org/2001/sw/DataAccess/tests/result-set#") graph.bind("minim", "http://purl.org/minim/minim#") rouri = rdflib.URIRef(evalresult['rouri']) graph.add( (rouri, MINIM.testedConstraint, rdflib.URIRef(evalresult['constrainturi'])) ) graph.add( (rouri, MINIM.testedPurpose, rdflib.Literal(evalresult['purpose'])) ) graph.add( (rouri, MINIM.testedTarget, rdflib.Literal(evalresult['target'])) ) graph.add( (rouri, MINIM.minimUri, rdflib.URIRef(evalresult['minimuri'])) ) graph.add( (rouri, MINIM.modelUri, rdflib.URIRef(evalresult['modeluri'])) ) for level in evalresult['summary']: log.info("RO %s, level %s, model %s"%(rouri,level,evalresult['modeluri'])) graph.add( (rouri, level, rdflib.URIRef(evalresult['modeluri'])) ) # Add details for all rules tested... def addRequirementsDetail(results, satlevel): for (req, binding) in results: b = rdflib.BNode() graph.add( (rouri, satlevel, b) ) graph.add( (b, MINIM.tryRequirement, req['uri']) ) for k in binding: b2 = rdflib.BNode() graph.add( (b, RESULT.binding, b2) ) graph.add( (b2, RESULT.variable, rdflib.Literal(k)) ) graph.add( (b2, RESULT.value, rdflib.Literal(binding[k])) ) addRequirementsDetail(evalresult['satisfied'], MINIM.satisfied) addRequirementsDetail(evalresult['missingMay'], MINIM.missingMay) addRequirementsDetail(evalresult['missingShould'], MINIM.missingShould) addRequirementsDetail(evalresult['missingMust'], MINIM.missingMust) return graph
def testEvalMayMissing(self): """ Evaluate complete RO against Minim description """ self.setupConfig() rodir = self.createTestRo(testbase, "test-data-1", "RO test minim", "ro-testMinim") self.populateTestRo(testbase, rodir) rometa = ro_metadata(ro_config, rodir) minimbase = rometa.getComponentUri("Minim-UserRequirements.rdf") modeluri = ro_minim.getElementUri(minimbase, "#missingMayRequirement") (g,evalresult) = ro_eval_minim.evaluate(rometa, "Minim-UserRequirements.rdf", # Minim file "docs/UserRequirements-bio.pdf", # Target resource "create") # Purpose missing_may = ( { 'level': "MAY" , 'model': modeluri , 'label': rdflib.Literal("aggregates docs/missing.css") , 'datarule': { 'aggregates': rometa.getComponentUri("docs/missing.css") , 'show': rdflib.Literal("05 - aggregates docs/missing.css") , 'showpass': None , 'showfail': None , 'showmiss': None , 'list': [] , 'listpass': [] , 'listfail': [] } , 'uri': ro_minim.getElementUri(minimbase, "#isPresent/docs/missing.css") , 'ruleuri': evalresult['missingMay'][0][0]['ruleuri'] , 'seq': "05 - aggregates docs/missing.css" }) self.maxDiff=None self.assertEquals(evalresult['summary'], [MINIM.nominallySatisfies, MINIM.minimallySatisfies]) self.assertEquals(evalresult['missingMust'], [] ) self.assertEquals(evalresult['missingShould'], [] ) self.assertEquals(evalresult['missingMay'], [(missing_may,{})] ) self.assertEquals(evalresult['rouri'], rometa.getRoUri()) self.assertEquals(evalresult['minimuri'], rometa.getComponentUri("Minim-UserRequirements.rdf")) self.assertEquals(evalresult['target'], "docs/UserRequirements-bio.pdf") self.assertEquals(evalresult['purpose'], "create") self.assertEquals(evalresult['constrainturi'], rometa.getComponentUriAbs("Minim-UserRequirements.rdf#create/docs/UserRequirements-bio.pdf")) self.assertEquals(evalresult['modeluri'], rometa.getComponentUriAbs("Minim-UserRequirements.rdf#missingMayRequirement")) self.deleteTestRo(rodir) return
def testEvaluateRDF(self): self.setupConfig() rodir = self.createTestRo(testbase, "test-chembox", "RO test minim", "ro-testMinim") rouri = ro_manifest.getRoUri(rodir) self.populateTestRo(testbase, rodir) rometa = ro_metadata(ro_config, rodir) resuri = rometa.getComponentUriAbs("http://purl.org/net/chembox/Ethane") reslabel = "Ethane" rometa.addGraphAnnotation(resuri, "Ethane.ttl") # Now run evaluation against test RO (minimgr, evalresult) = ro_eval_minim.evaluate(rometa, "Minim-chembox.ttl", # Minim file resuri, # Target resource "complete") # Purpose resultgr = ro_eval_minim.evalResultGraph(minimgr, evalresult) log.debug("------ resultgr:\n%s\n----"%(resultgr.serialize(format='turtle'))) # pretty-xml ## print "------ resultgr:\n%s\n----"%(resultgr.serialize(format='turtle')) # Check response returned modeluri = rdflib.URIRef('http://example.com/chembox-samples/minim_model') prefixes = make_sparql_prefixes() probequeries = ( [ '''ASK { _:r minim:testedRO <%s> ; minim:minimUri <%s> }'''% (rouri, rometa.getComponentUri("Minim-chembox.ttl")) , '''ASK { _:r minim:testedRO <%s> ; minim:testedModel <%s> }'''% (rouri, modeluri) , '''ASK { _:r minim:testedTarget <%s> ; minim:satisfied [ minim:tryMessage "%s" ] }'''% (resuri, "InChI identifier is present") , '''ASK { _:r minim:testedTarget <%s> ; minim:satisfied [ minim:tryMessage "%s" ] }'''% (resuri, "ChemSpider identifier is present") , '''ASK { _:r minim:testedTarget <%s> ; minim:missingMay [ minim:tryMessage "%s" ] }'''% (resuri, "No synomym is present") , '''ASK { _:r minim:testedTarget <%s> ; minim:nominallySatisfies <%s> }'''% (resuri, modeluri) , '''ASK { _:r minim:testedTarget <%s> ; minim:minimallySatisfies <%s> }'''% (resuri, modeluri) , '''ASK { <%s> rdfs:label "%s" }'''% (resuri, reslabel) ]) for q in probequeries: r = resultgr.query(prefixes+q) self.assertEqual(r.type, 'ASK', "Result type %s for: %s"%(r.type, q)) self.assertTrue(r.askAnswer, "Failed query: %s"%(q)) self.deleteTestRo(rodir) return
def testEvalQueryTestReportList(self): """ Test QueryTestRules reporting list of failed query probes """ self.setupConfig() rodir = self.createTestRo(testbase, "test-data-2", "RO test minim", "ro-testMinim") rouri = ro_manifest.getRoUri(rodir) self.populateTestRo(testbase, rodir) rometa = ro_metadata(ro_config, rodir) resuri = rometa.getComponentUriAbs("data/NoSuchResource") rometa.addSimpleAnnotation(resuri, "rdfs:label", "Test label") # Now run evaluation against test RO (g, evalresult) = ro_eval_minim.evaluate(rometa, "Minim-UserRequirements2.rdf", # Minim file "data/NoSuchResource", # Target resource "report list") # Purpose log.debug("ro_eval_minim.evaluate result:\n----\n%s"%(repr(evalresult))) self.assertNotIn(MINIM.fullySatisfies, evalresult['summary']) self.assertNotIn(MINIM.nominallySatisfies, evalresult['summary']) self.assertIn(MINIM.minimallySatisfies, evalresult['summary']) self.assertEquals(evalresult['missingMust'], []) self.assertEquals(len(evalresult['missingShould']), 1) self.assertEquals(evalresult['missingShould'][0][0]['seq'], '04 - aggregates data/NoSuchResource') self.assertEquals(evalresult['missingMay'], []) self.assertEquals(evalresult['rouri'], rometa.getRoUri()) self.assertEquals(evalresult['minimuri'], rometa.getComponentUri("Minim-UserRequirements2.rdf")) self.assertEquals(evalresult['target'], "data/NoSuchResource") self.assertEquals(evalresult['purpose'], "report list") self.assertEquals(evalresult['constrainturi'], rometa.getComponentUriAbs("Minim-UserRequirements2.rdf#report/data/NoSuchResource")) self.assertEquals(evalresult['modeluri'], rometa.getComponentUriAbs("Minim-UserRequirements2.rdf#reportList")) # Check result bindings returned self.assertEquals(evalresult['missingShould'][0][1]['_count'], 1) self.assertEquals(evalresult['missingShould'][0][1]['_fileuri'], rometa.getComponentUri("data/NoSuchResource")) self.assertEquals(evalresult['missingShould'][0][1]['targetres'], rometa.getComponentUri("data/NoSuchResource")) self.assertEquals(evalresult['missingShould'][0][1]['ro'], str(rometa.getRoUri())) self.assertEquals(evalresult['missingShould'][0][1]['ro_list'], [str(rometa.getRoUri())]) # Clean up self.deleteTestRo(rodir) return