def hybridPredQueryPreparation(self, tp):
lit = BuildUnitermFromTuple(tp, newNss=self.nsBindings)
op = GetOp(lit)
if op in self.hybridPredicates:
lit.setOperator(URIRef(op + u'_derived'))
return lit.toRDFTuple()
else:
return tp
def SetupMetaInterpreter(tBoxGraph, goal, useThingRule=True):
from FuXi.LP.BackwardFixpointProcedure import BackwardFixpointProcedure
from FuXi.Rete.Magic import SetupDDLAndAdornProgram
from FuXi.Horn.PositiveConditions import BuildUnitermFromTuple
from FuXi.Rete.TopDown import PrepareSipCollection
from FuXi.DLP import LloydToporTransformation, makeRule
from FuXi.Rete.SidewaysInformationPassing import GetOp
owlThingAppears = False
if useThingRule and OWL.Thing in tBoxGraph.all_nodes():
owlThingAppears = True
completionRules = HornFromN3(StringIO(RULES))
if owlThingAppears:
completionRules.formulae.extend(
HornFromN3(StringIO(CONDITIONAL_THING_RULE)))
reducedCompletionRules = set()
for rule in completionRules:
for clause in LloydToporTransformation(rule.formula):
rule = makeRule(clause, {})
# log.debug(rule)
# PrettyPrintRule(rule)
reducedCompletionRules.add(rule)
network = SetupRuleStore(makeNetwork=True)[-1]
SetupDDLAndAdornProgram(
tBoxGraph,
reducedCompletionRules,
[goal],
derivedPreds=derivedPredicates,
ignoreUnboundDPreds=True,
hybridPreds2Replace=hybridPredicates)
lit = BuildUnitermFromTuple(goal)
op = GetOp(lit)
lit.setOperator(URIRef(op + u'_derived'))
goal = lit.toRDFTuple()
sipCollection = PrepareSipCollection(reducedCompletionRules)
tBoxGraph.templateMap = {}
bfp = BackwardFixpointProcedure(
tBoxGraph,
network,
derivedPredicates,
goal,
sipCollection,
hybridPredicates=hybridPredicates,
debug=True)
bfp.createTopDownReteNetwork(True)
log.debug(reducedCompletionRules)
rt = bfp.answers(debug=True)
log.debug(rt)
log.debug(bfp.metaInterpNetwork)
bfp.metaInterpNetwork.reportConflictSet(True, sys.stderr)
for query in bfp.edbQueries:
log.debug("Dispatched query against dataset: ", query.asSPARQL())
log.debug(list(bfp.goalSolutions))
def handleInferredTriple(self,
inferredTriple,
tokens,
termNode,
binding,
debug=False,
executeFn=None):
inferredToken=ReteToken(inferredTriple)
self.proofTracers.setdefault(inferredTriple,[]).append(binding)
self.justifications.setdefault(inferredTriple,set()).add(termNode)
if termNode.filter and inferredTriple not in self.filteredFacts:
self.filteredFacts.add(inferredTriple)
if inferredTriple not in self.inferredFacts and inferredToken not in self.workingMemory:
if debug:
lit = BuildUnitermFromTuple(inferredTriple,
newNss=self.nsMap)
print "Inferred triple: ", lit, " from ",termNode.clauseRepresentation()
inferredToken.debug = True
if executeFn:
#The indicated execute action is supposed to be triggered
#when the indicates RHS triple is inferred for the
#first time
executeFn(termNode,inferredTriple,tokens,binding,debug)
if self.goal is not None and self.goal == inferredTriple:#in self.inferredFacts:
for binding in tokens.bindings:
rt=None#self.bfp.extractProof(binding,termNode.ruleNo,inferredTriple,applyBindings=True)
def depth(l):
assert isinstance(l,(tuple))
if isinstance(l,tuple):
depths = [depth(item) for item in l[2]]
return 1 + max(depths) if depths else 1
#self.bfp.proof = -1,rt#depth(rt),rt
if debug:
"Proved goal " + repr(self.goal)
raise InferredGoal("Proved goal " + repr(self.goal))
self.inferredFacts.add(inferredTriple)
self.addWME(inferredToken)
currIdx = self.instanciations.get(termNode,0)
currIdx+=1
self.instanciations[termNode] = currIdx
else:
if debug:
lit = BuildUnitermFromTuple(inferredTriple,
newNss=self.nsMap)
print "Inferred triple skipped: ", lit
if executeFn:
#The indicated execute action is supposed to be triggered
#when the indicates RHS triple is inferred for the
#first time
executeFn(termNode,inferredTriple,tokens,binding,debug)
def getUniterm(self):
from FuXi.Horn.PositiveConditions import BuildUnitermFromTuple
return BuildUnitermFromTuple(
tuple([
val
for var, val in [self.subject, self.predicate, self.object_]
]))
def testQueryMemoization(self):
raise SkipTest(
"SKIPFAIL testQueryMemoization, see test/testBFPQueryMemoization.py"
)
topDownStore = TopDownSPARQLEntailingStore(
self.owlGraph.store,
self.owlGraph,
idb=self.program,
DEBUG=False,
nsBindings=nsMap,
decisionProcedure=BFP_METHOD,
identifyHybridPredicates=True)
targetGraph = Graph(topDownStore)
for pref, nsUri in nsMap.items():
targetGraph.bind(pref, nsUri)
goal = (Variable('SUBJECT'), RDF.type, EX.C)
queryLiteral = EDBQuery([BuildUnitermFromTuple(goal)], self.owlGraph,
[Variable('SUBJECT')])
query = queryLiteral.asSPARQL()
# rt=targetGraph.query(query,initNs=nsMap)
# if len(topDownStore.edbQueries) == len(set(topDownStore.edbQueries)):
# pprint(topDownStore.edbQueries)
print("Queries dispatched against EDB")
for query in self.owlGraph.queriesDispatched:
print(query)
self.failUnlessEqual(len(self.owlGraph.queriesDispatched), 4,
"Duplicate query")
def batch_unify(self, patterns):
"""
Perform RDF triple store-level unification of a list of triple
patterns (4-item tuples which correspond to a SPARQL triple pattern
with an additional constraint for the graph name).
Uses a SW sip-strategy implementation to solve the conjunctive goal
and yield unified bindings
:Parameters:
- `patterns`: a list of 4-item tuples where any of the items can be
one of: Variable, URIRef, BNode, or Literal.
Returns a generator over dictionaries of solutions to the list of
triple patterns that are entailed by the regime.
"""
dPreds = set()
goals = []
for s, p, o, g in patterns:
goals.append((s, p, o))
dPred = o if p == RDF.type else p
if dPred in self.hybridPredicates:
dPreds.add(URIRef(dPred + u'_derived'))
else:
dPreds.add(p == RDF.type and o or p)
if set(dPreds).intersection(self.derivedPredicates):
#Patterns involve derived predicates
self.batch_unification = False
for ansDict in self.conjunctiveSipStrategy(iter(goals), self.edb):
yield ansDict
self.batch_unification = True
else:
#conjunctive query involving EDB predicateso only
vars = []
triples = []
for pat in patterns:
triples.append(BuildUnitermFromTuple(pat[:3]))
vars.extend(
[term for term in pat[:3] if isinstance(term, Variable)])
vars = list(set(vars))
query = RDFTuplesToSPARQL(triples, self.edb, vars=vars)
graphNsMap = dict(self.edb.namespaces())
graphNsMap.update(self.nsBindings)
prefixDefs = '\n'.join([
"PREFIX %s: %s" % (k, v.n3()) for k, v in graphNsMap.items()
if k
])
baseDef = u'' #"BASE %s"%graphNsMap.get(u'').n3() if u'' in graphNsMap else u''
query = '\n'.join([baseDef, prefixDefs, query])
if self.DEBUG:
print "Batch unify resolved against EDB"
print query
rt = self.edb.query(query, initNs=self.nsBindings)
rt = len(vars)>1 and ( dict([(vars[idx],i)
for idx,i in enumerate(v)])
for v in rt ) \
or ( dict([(vars[0],v)]) for v in rt )
for item in rt:
yield item
def MagicOWLProof(self, goals, rules, factGraph, conclusionFile):
progLen = len(rules)
magicRuleNo = 0
dPreds = []
for rule in AdditionalRules(factGraph):
rules.append(rule)
if not GROUND_QUERY:
goalDict = dict([((Variable('SUBJECT'), goalP, goalO), goalS)
for goalS, goalP, goalO in goals])
goals = goalDict.keys()
assert goals
topDownStore = TopDownSPARQLEntailingStore(
factGraph.store,
factGraph,
idb=rules,
DEBUG=DEBUG,
identifyHybridPredicates=True,
nsBindings=nsMap)
targetGraph = Graph(topDownStore)
for pref, nsUri in nsMap.items():
targetGraph.bind(pref, nsUri)
start = time.time()
for goal in goals:
queryLiteral = EDBQuery([BuildUnitermFromTuple(goal)], factGraph,
None if GROUND_QUERY else [goal[0]])
query = queryLiteral.asSPARQL()
print "Goal to solve ", query
rt = targetGraph.query(query, initNs=nsMap)
if GROUND_QUERY:
self.failUnless(rt.askAnswer[0], "Failed top-down problem")
else:
if (goalDict[goal]) not in rt or DEBUG:
for network, _goal in topDownStore.queryNetworks:
print network, _goal
network.reportConflictSet(True)
for query in topDownStore.edbQueries:
print query.asSPARQL()
print "Missing", goalDict[goal]
self.failUnless((goalDict[goal]) in rt,
"Failed top-down problem")
sTime = time.time() - start
if sTime > 1:
sTimeStr = "%s seconds" % sTime
else:
sTime = sTime * 1000
sTimeStr = "%s milli seconds" % sTime
return sTimeStr
def AdornLiteral(rdfTuple, newNss=None, naf=False):
"""
An adornment for an n-ary predicate p is a string a of length n on the
alphabet {b, f}, where b stands for bound and f stands for free. We
assume a fixed order of the arguments of the predicate.
Intuitively, an adorned occurrence of the predicate, p a, corresponds to a
computation of the predicate with some arguments bound to constants, and
the other arguments free, where the bound arguments are those that are
so indicated by the adornment.
>>> EX = Namespace('http://doi.acm.org/10.1145/6012.15399#')
>>> query = RenderSPARQLAlgebra(parse(NON_LINEAR_MS_QUERY)) #doctest: +SKIP
>>> literal = query.patterns[0][:3] #doctest: +SKIP
>>> literal #doctest: +SKIP
(rdflib.URIRef('http://doi.acm.org/10.1145/6012.15399#john'), rdflib.URIRef('http://doi.acm.org/10.1145/6012.15399#sg'), ?X)
>>> aLit = AdornLiteral(literal, query.prologue.prefixBindings) #doctest: +SKIP
>>> aLit #doctest: +SKIP
mst:sg_bf(mst:john ?X)
>>> aLit.adornment #doctest: +SKIP
['b', 'f']
>>> aLit.getBindings(Uniterm(EX.sg, [Variable('X'), EX.jill])) #doctest: +SKIP
{?X: rdflib.URIRef('http://doi.acm.org/10.1145/6012.15399#john')}
"""
args = [rdfTuple[0], rdfTuple[-1]]
newNss = newNss is None and {} or newNss
uTerm = BuildUnitermFromTuple(rdfTuple, newNss)
opArgs = rdfTuple[1] == RDF.type and [args[0]] or args
def isFreeTerm(term):
return isinstance(term, Variable)
adornment = [
isFreeTerm(term) and 'f' or 'b' for idx, term in enumerate(opArgs)
]
return AdornedUniTerm(uTerm, adornment, naf)
def sparql_query(self,
queryString,
queryObj,
graph,
dataSetBase,
extensionFunctions,
initBindings={},
initNs={},
DEBUG=False):
"""
The default 'native' SPARQL implementation is based on sparql-p's expansion trees
layered on top of the read-only RDF APIs of the underlying store
"""
from rdflib.sparql.Algebra import TopEvaluate
from rdflib.QueryResult import QueryResult
from rdflib import plugin
from rdflib.sparql.bison.Query import AskQuery
_expr = self.isaBaseQuery(None, queryObj)
if isinstance(queryObj.query,AskQuery) and \
isinstance(_expr,BasicGraphPattern):
#This is a ground, BGP, involving IDB and can be solved directly
#using top-down decision procedure
#First separate out conjunct into EDB and IDB predicates
#(solving the former first)
from FuXi.SPARQL import EDBQuery
groundConjunct = []
derivedConjunct = []
for s, p, o, func in _expr.patterns:
if self.derivedPredicateFromTriple((s, p, o)) is None:
groundConjunct.append(BuildUnitermFromTuple((s, p, o)))
else:
derivedConjunct.append(BuildUnitermFromTuple((s, p, o)))
if groundConjunct:
baseEDBQuery = EDBQuery(groundConjunct, self.edb)
subQuery, ans = baseEDBQuery.evaluate(DEBUG)
assert isinstance(ans, bool), ans
if groundConjunct and not ans:
askResult = False
else:
askResult = True
for derivedLiteral in derivedConjunct:
goal = derivedLiteral.toRDFTuple()
#Solve ground, derived goal directly
goal = self.hybridPredQueryPreparation(goal)
SetupDDLAndAdornProgram(
self.edb,
self.idb, [goal],
derivedPreds=self.derivedPredicates,
ignoreUnboundDPreds=True)
sipCollection = PrepareSipCollection(
self.edb.adornedProgram)
if self.DEBUG and sipCollection:
for sip in SIPRepresentation(sipCollection):
print >> sys.stderr, sip
pprint(list(self.edb.adornedProgram), sys.stderr)
elif self.DEBUG:
print >> sys.stderr, "No SIP graph!"
rt, node = first(
self.invokeDecisionProcedure(goal, self.edb, {},
self.DEBUG,
sipCollection))
if not rt:
askResult = False
break
return plugin.get('SPARQLQueryResult', QueryResult)(askResult)
else:
rt = TopEvaluate(queryObj,
graph,
initBindings,
DEBUG=self.DEBUG,
dataSetBase=dataSetBase,
extensionFunctions=extensionFunctions)
return plugin.get('SPARQLQueryResult', QueryResult)(rt)
def conjunctiveSipStrategy(self, goalsRemaining, factGraph, bindings=None):
"""
Given a conjunctive set of triples, invoke sip-strategy passing
on intermediate solutions to facilitate 'join' behavior
"""
bindings = bindings if bindings else {}
try:
tp = goalsRemaining.next()
assert isinstance(bindings, dict)
dPred = self.derivedPredicateFromTriple(tp)
if dPred is None:
baseEDBQuery = EDBQuery([BuildUnitermFromTuple(tp)],
self.edb,
bindings=bindings)
if self.DEBUG:
print >>sys.stderr,"Evaluating TP against EDB: ",\
baseEDBQuery.asSPARQL()
query, rt = baseEDBQuery.evaluate()
if isinstance(rt, bool) and rt:
yield bindings
elif not isinstance(rt, bool):
rt = list(rt)
remaining_goals = itertools.tee(goalsRemaining, len(rt))
for idx in range(len(rt)):
item = {}
item.update(rt[idx])
item.update(bindings)
if self.DEBUG:
print >> sys.stderr, "Solution from EDB query: ", item
for ansDict in self.conjunctiveSipStrategy(
remaining_goals[idx], factGraph, item):
yield ansDict
else:
queryLit = BuildUnitermFromTuple(tp)
currentOp = GetOp(queryLit)
queryLit.setOperator(currentOp)
query = EDBQuery([queryLit], self.edb, bindings=bindings)
if bindings:
tp = first(query.formulae).toRDFTuple()
if self.DEBUG:
print >> sys.stderr, "Goal/Query: ", query.asSPARQL()
tp = self.hybridPredQueryPreparation(tp)
SetupDDLAndAdornProgram(self.edb,
self.idb, [tp],
derivedPreds=self.derivedPredicates,
ignoreUnboundDPreds=True,
nsBindings=self.nsBindings)
sipCollection = PrepareSipCollection(self.edb.adornedProgram)
if self.DEBUG and sipCollection:
for sip in SIPRepresentation(sipCollection):
print >> sys.stderr, sip
pprint(list(self.edb.adornedProgram), sys.stderr)
elif self.DEBUG:
print >> sys.stderr, "No SIP graph!"
for nextAnswer, ns in self.invokeDecisionProcedure(
tp, factGraph, bindings, self.DEBUG, sipCollection):
if isinstance(nextAnswer, dict):
#Recieved solutions to 'open' query, merge with given bindings
#and continue
for ansDict in self.conjunctiveSipStrategy(
goalsRemaining, factGraph,
mergeMappings1To2(bindings, nextAnswer)):
yield ansDict
elif nextAnswer:
#we (successfully) proved a ground query, pass on bindings
assert isinstance(nextAnswer, bool)
for ansDict in self.conjunctiveSipStrategy(
goalsRemaining, factGraph, bindings):
yield ansDict
except StopIteration:
yield bindings
def testOwl(self):
log.debug("Running")
options = defaultOptions()
options.debug = True
global REASONING_STRATEGY, GROUND_QUERY, SINGLE_TEST, DEBUG
SINGLE_TEST = options.singleTest
DEBUG = True # options.debug
GROUND_QUERY = options.groundQuery
REASONING_STRATEGY = options.strategy
testData = {}
here = os.getcwd()
log.debug("Here is {}".format(here))
if not here.endswith('/test'):
os.chdir(here + '/test')
for manifest in glob('OWL/*/Manifest*.rdf'):
if manifest in Tests2Skip:
continue
if (REASONING_STRATEGY is not None and manifest in NonNaiveSkip) or \
(REASONING_STRATEGY == 'sld' and manifest in TopDownTests2Skip) or \
(REASONING_STRATEGY == 'bfp' and manifest in BFPTests2SKip) or \
(REASONING_STRATEGY == 'gms' and manifest in MagicTest2Skip):
continue
skip = False
for pattern2Skip in patterns2Skip:
if manifest.find(pattern2Skip) > -1:
skip = True
break
if skip:
continue
manifestStore = plugin.get(RDFLIB_STORE, Store)()
manifestGraph = Graph(manifestStore)
manifestGraph.parse(open(manifest))
rt = manifestGraph.query(
MANIFEST_QUERY,
initNs=nsMap,
DEBUG=False)
log.debug(list(manifestGraph.namespace_manager.namespaces()))
for status, premise, conclusion, feature, description in rt:
if feature in Features2Skip:
continue
premise = manifestGraph.namespace_manager.compute_qname(
premise)[-1]
conclusion = manifestGraph.namespace_manager.compute_qname(
conclusion)[-1]
premiseFile = '/'.join(manifest.split('/')[:2] + [premise])
conclusionFile = '/'.join(manifest.split('/')
[:2] + [conclusion])
log.debug("premiseFile", premiseFile)
log.debug("conclusionFile", conclusionFile)
if str(status) == 'APPROVED':
if SINGLE_TEST and premiseFile != SINGLE_TEST:
continue
assert os.path.exists('.'.join([premiseFile, 'rdf']))
assert os.path.exists('.'.join([conclusionFile, 'rdf']))
log.debug("<%s> :- <%s>" % ('.'.join([conclusionFile, 'rdf']),
'.'.join([premiseFile, 'rdf'])))
store = plugin.get(RDFLIB_STORE, Store)()
store.open(RDFLIB_CONNECTION)
factGraph = Graph(store)
factGraph.parse(open('.'.join([premiseFile, 'rdf'])))
nsMap.update(dict([(k, v)
for k, v in factGraph.namespaces()]))
if DEBUG:
log.debug(
"## Source Graph ##\n", factGraph.serialize(format='n3'))
Individual.factoryGraph = factGraph
for c in AllClasses(factGraph):
if not isinstance(c.identifier, BNode):
log.debug(c.__repr__(True))
if feature in TopDownTests2Skip:
continue
log.debug(premiseFile, feature, description)
program = list(HornFromN3(StringIO(non_DHL_OWL_Semantics)))
program.extend(self.network.setupDescriptionLogicProgramming(
factGraph,
addPDSemantics=False,
constructNetwork=False))
log.debug("Original program")
log.debug(pformat(program))
# timings=[]
if REASONING_STRATEGY is None:
sTimeStr = self.calculateEntailments(factGraph)
expectedFacts = Graph(store)
for triple in expectedFacts.parse('.'.join([conclusionFile, 'rdf'])):
# closureGraph = ReadOnlyGraphAggregate([self.network.inferredFacts,factGraph])
if triple not in self.network.inferredFacts and triple not in factGraph:
log.debug("missing triple %s" %
(pformat(triple)))
log.debug(manifest)
log.debug("feature: ", feature)
log.debug(description)
print(list(self.network.inferredFacts))
raise Exception("Failed test: " + feature)
else:
log.debug("=== Passed! ===")
# print(list(self.network.inferredFacts))
log.debug("\n")
testData[manifest] = sTimeStr
store.rollback()
self.setUp()
# self.network.reset()
# self.network._resetinstantiationStats()
else:
try:
goals = []
for triple in Graph(store).parse('.'.join([conclusionFile, 'rdf'])):
if triple not in factGraph:
goals.append(triple)
testData[manifest] = self.MagicOWLProof(goals,
program,
factGraph,
conclusionFile)
self.setUp()
# self.network._resetinstantiationStats()
# self.network.reset()
# self.network.clear()
except:
# log.debug("missing triple %s"%(pformat(goal)))
log.debug(manifest, premiseFile)
log.debug("feature: ", feature)
log.debug(description)
print([BuildUnitermFromTuple(t)
for t in self.network.inferredFacts])
# from FuXi.Rete.Util import renderNetwork
# dot = renderNetwork(self.network,self.network.nsMap).write_jpeg('test-fail.jpeg')
raise # Exception ("Failed test: "+feature)
print(pformat(testData))
def MagicOWLProof(self, goals, rules, factGraph, conclusionFile):
progLen = len(rules)
magicRuleNo = 0
dPreds = []
for rule in AdditionalRules(factGraph):
rules.append(rule)
if not GROUND_QUERY and REASONING_STRATEGY != 'gms':
goalDict = dict([((Variable('SUBJECT'), goalP, goalO), goalS)
for goalS, goalP, goalO in goals])
goals = goalDict.keys()
assert goals
if REASONING_STRATEGY == 'gms':
for rule in MagicSetTransformation(factGraph,
rules,
goals,
dPreds):
magicRuleNo += 1
self.network.buildNetworkFromClause(rule)
self.network.rules.add(rule)
if DEBUG:
log.debug("\t", rule)
log.debug("rate of reduction in the size of the program: ",
(100 - (float(magicRuleNo) / float(progLen)) * 100))
if REASONING_STRATEGY in ['bfp', 'sld']: # and not GROUND_QUERY:
reasoningAlg = TOP_DOWN_METHOD if REASONING_STRATEGY == 'sld' \
else BFP_METHOD
topDownStore = TopDownSPARQLEntailingStore(
factGraph.store,
factGraph,
idb=rules,
DEBUG=DEBUG,
nsBindings=nsMap,
decisionProcedure=reasoningAlg,
identifyHybridPredicates=REASONING_STRATEGY == 'bfp')
targetGraph = Graph(topDownStore)
for pref, nsUri in nsMap.items():
targetGraph.bind(pref, nsUri)
start = time.time()
for goal in goals:
queryLiteral = EDBQuery([BuildUnitermFromTuple(goal)],
factGraph,
None if GROUND_QUERY else [goal[0]])
query = queryLiteral.asSPARQL()
log.debug("Goal to solve ", query)
rt = targetGraph.query(query, initNs=nsMap)
if GROUND_QUERY:
self.failUnless(rt.askAnswer[0], "Failed top-down problem")
else:
if (goalDict[goal]) not in rt or DEBUG:
for network, _goal in topDownStore.queryNetworks:
log.debug(network, _goal)
network.reportConflictSet(True)
for query in topDownStore.edbQueries:
log.debug(query.asSPARQL())
self.failUnless((goalDict[goal]) in rt,
"Failed top-down problem")
sTime = time.time() - start
if sTime > 1:
sTimeStr = "%s seconds" % sTime
else:
sTime = sTime * 1000
sTimeStr = "%s ms" % sTime
return sTimeStr
elif REASONING_STRATEGY == 'gms':
for goal in goals:
adornedGoalSeed = AdornLiteral(goal).makeMagicPred()
goal = adornedGoalSeed.toRDFTuple()
if DEBUG:
log.debug("Magic seed fact ", adornedGoalSeed)
factGraph.add(goal)
timing = self.calculateEntailments(factGraph)
for goal in goals:
# self.failUnless(goal in self.network.inferredFacts or goal in factGraph,
# "Failed GMS query")
if goal not in self.network.inferredFacts and goal not in factGraph:
log.debug("missing triple %s" % (pformat(goal)))
# print(list(factGraph.adornedProgram))
# from FuXi.Rete.Util import renderNetwork
# dot = renderNetwork(
# self.network,self.network.nsMap).write_jpeg('test-fail.jpeg')
self.network.reportConflictSet(True)
log.debug("=== Failed: %s ====" % pformat(goal))
else:
log.debug("=== Passed! ===")
return timing
def SipStrategy(query,
sipCollection,
factGraph,
derivedPreds,
bindings={},
processedRules=None,
network=None,
debug=False,
buildProof=False,
memoizeMemory=None,
proofLevel=1):
"""
Accordingly, we define a sip-strategy for computing the answers to a query
expressed using a set of Datalog rules, and a set of sips, one for each
adornment of a rule head, as follows...
Each evaluation uses memoization (via Python decorators) but also relies on well-formed
rewrites for using semi-naive bottom up method over large SPARQL data.
"""
memoizeMemory = memoizeMemory and memoizeMemory or {}
queryLiteral = BuildUnitermFromTuple(query)
processedRules = processedRules and processedRules or set()
if bindings:
#There are bindings. Apply them to the terms in the query
queryLiteral.ground(bindings)
if debug:
print("%sSolving" % ('\t' * proofLevel), queryLiteral, bindings)
# Only consider ground triple pattern isomorphism with matching bindings
goalRDFStatement = queryLiteral.toRDFTuple()
if queryLiteral in memoizeMemory:
if debug:
print("%sReturning previously calculated results for " % \
('\t' * proofLevel), queryLiteral)
for answers in memoizeMemory[queryLiteral]:
yield answers
elif AlphaNode(goalRDFStatement).alphaNetworkHash(
True,
skolemTerms=list(bindings.values())) in \
[AlphaNode(r.toRDFTuple()).alphaNetworkHash(True,
skolemTerms=list(bindings.values()))
for r in processedRules
if AdornLiteral(goalRDFStatement).adornment == \
r.adornment]:
if debug:
print("%s Goal already processed..." % \
('\t' * proofLevel))
else:
isGround = literalIsGround(queryLiteral)
if buildProof:
ns = NodeSet(goalRDFStatement, network=network, identifier=BNode())
else:
ns = None
# adornedProgram = factGraph.adornedProgram
queryPred = GetOp(queryLiteral)
if sipCollection is None:
rules = []
else:
#For every rule head matching the query, we invoke the rule,
#thus determining an adornment, and selecting a sip to follow
rules = sipCollection.headToRule.get(queryPred, set())
if None in sipCollection.headToRule:
#If there are second order rules, we add them
#since they are a 'wildcard'
rules.update(sipCollection.headToRule[None])
#maintained list of rules that haven't been processed before and
#match the query
validRules = []
#each subquery contains values for the bound arguments that are passed
#through the sip arcs entering the node corresponding to that literal. For
#each subquery generated, there is a set of answers.
answers = []
# variableMapping = {}
#Some TBox queries can be 'joined' together into SPARQL queries against
#'base' predicates via an RDF dataset
#These atomic concept inclusion axioms can be evaluated together
#using a disjunctive operator at the body of a horn clause
#where each item is a query of the form uniPredicate(?X):
#Or( uniPredicate1(?X1), uniPredicate2(?X), uniPredicate3(?X), ..)
#In this way massive, conjunctive joins can be 'mediated'
#between the stated facts and the top-down solver
@parameterizedPredicate([i for i in derivedPreds])
def IsAtomicInclusionAxiomRHS(rule, dPreds):
"""
This is an atomic inclusion axiom with
a variable (or bound) RHS: uniPred(?ENTITY)
"""
bodyList = list(iterCondition(rule.formula.body))
body = first(bodyList)
return GetOp(body) not in dPreds and \
len(bodyList) == 1 and \
body.op == RDF.type
atomicInclusionAxioms = list(filter(IsAtomicInclusionAxiomRHS, rules))
if atomicInclusionAxioms and len(atomicInclusionAxioms) > 1:
if debug:
print("\tCombining atomic inclusion axioms: ")
pprint(atomicInclusionAxioms, sys.stderr)
if buildProof:
factStep = InferenceStep(ns, source='some RDF graph')
ns.steps.append(factStep)
axioms = [rule.formula.body for rule in atomicInclusionAxioms]
#attempt to exaustively apply any available substitutions
#and determine if query if fully ground
vars = [
v for v in GetArgs(queryLiteral, secondOrder=True)
if isinstance(v, Variable)
]
openVars, axioms, _bindings = \
normalizeBindingsAndQuery(vars,
bindings,
axioms)
if openVars:
# mappings = {}
#See if we need to do any variable mappings from the query literals
#to the literals in the applicable rules
query, rt = EDBQuery(axioms, factGraph, openVars,
_bindings).evaluate(
debug, symmAtomicInclusion=True)
if buildProof:
factStep.groundQuery = subquery
for ans in rt:
if buildProof:
factStep.bindings.update(ans)
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(ans), ns))
yield ans, ns
else:
#All the relevant derivations have been explored and the result
#is a ground query we can directly execute against the facts
if buildProof:
factStep.bindings.update(bindings)
query, rt = EDBQuery(axioms, factGraph, _bindings).evaluate(
debug, symmAtomicInclusion=True)
if buildProof:
factStep.groundQuery = subquery
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(rt), ns))
yield rt, ns
rules = filter(lambda i: not IsAtomicInclusionAxiomRHS(i), rules)
for rule in rules:
#An exception is the special predicate ph; it is treated as a base
#predicate and the tuples in it are those supplied for qb by unification.
headBindings = getBindingsFromLiteral(goalRDFStatement,
rule.formula.head)
# comboBindings = dict([(k, v) for k, v in itertools.chain(
# bindings.items(),
# headBindings.items())])
varMap = rule.formula.head.getVarMapping(queryLiteral)
if headBindings and\
[term for term in rule.formula.head.getDistinguishedVariables(True)
if varMap.get(term, term) not in headBindings]:
continue
# subQueryAnswers = []
# dontStop = True
# projectedBindings = comboBindings.copy()
if debug:
print("%sProcessing rule" % \
('\t' * proofLevel), rule.formula)
if debug and sipCollection:
print("Sideways Information Passing (sip) graph for %s: " %
queryLiteral)
print(sipCollection.serialize(format='n3'))
for sip in SIPRepresentation(sipCollection):
print(sip)
try:
# Invoke the rule
if buildProof:
step = InferenceStep(ns, rule.formula)
else:
step = None
for rt, step in\
invokeRule([headBindings],
iter(iterCondition(rule.formula.body)),
rule.sip,
(proofLevel + 1,
memoizeMemory,
sipCollection,
factGraph,
derivedPreds,
processedRules.union([
AdornLiteral(query)])),
step=step,
debug=debug):
if rt:
if isinstance(rt, dict):
#We received a mapping and must rewrite it via
#correlation between the variables in the rule head
#and the variables in the original query (after applying
#bindings)
varMap = rule.formula.head.getVarMapping(
queryLiteral)
if varMap:
rt = MakeImmutableDict(
refactorMapping(varMap, rt))
if buildProof:
step.bindings = rt
else:
if buildProof:
step.bindings = headBindings
validRules.append(rule)
if buildProof:
ns.steps.append(step)
if isGround:
yield True, ns
else:
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(rt), ns))
yield rt, ns
except RuleFailure:
# Clean up failed antecedents
if buildProof:
if ns in step.antecedents:
step.antecedents.remove(ns)
if not validRules:
#No rules matching, query factGraph for answers
successful = False
if buildProof:
factStep = InferenceStep(ns, source='some RDF graph')
ns.steps.append(factStep)
if not isGround:
subquery, rt = EDBQuery([queryLiteral], factGraph, [
v for v in GetArgs(queryLiteral, secondOrder=True)
if isinstance(v, Variable)
], bindings).evaluate(debug)
if buildProof:
factStep.groundQuery = subquery
for ans in rt:
successful = True
if buildProof:
factStep.bindings.update(ans)
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(ans), ns))
yield ans, ns
if not successful and queryPred not in derivedPreds:
#Open query didn't return any results and the predicate
#is ostensibly marked as derived predicate, so we have failed
memoizeMemory.setdefault(queryLiteral, set()).add(
(False, ns))
yield False, ns
else:
#All the relevant derivations have been explored and the result
#is a ground query we can directly execute against the facts
if buildProof:
factStep.bindings.update(bindings)
subquery, rt = EDBQuery([queryLiteral], factGraph,
bindings).evaluate(debug)
if buildProof:
factStep.groundQuery = subquery
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(rt), ns))
yield rt, ns
def SetupDDLAndAdornProgram(factGraph,
rules,
GOALS,
derivedPreds=None,
strictCheck=DDL_STRICTNESS_FALLBACK_DERIVED,
defaultPredicates=None,
ignoreUnboundDPreds=False,
hybridPreds2Replace=None):
if not defaultPredicates:
defaultPredicates = [], []
# _dPredsProvided = bool(derivedPreds)
if not derivedPreds:
_derivedPreds = DerivedPredicateIterator(
factGraph,
rules,
strict=strictCheck,
defaultPredicates=defaultPredicates)
if not isinstance(derivedPreds, (set, list)):
derivedPreds = list(_derivedPreds)
else:
derivedPreds.extend(_derivedPreds)
hybridPreds2Replace = hybridPreds2Replace or []
adornedProgram = AdornProgram(factGraph,
rules,
GOALS,
derivedPreds,
ignoreUnboundDPreds,
hybridPreds2Replace=hybridPreds2Replace)
if adornedProgram != set([]):
rt = reduce(lambda l, r: l + r, [
list(iterCondition(clause.formula.body))
for clause in adornedProgram
])
else:
rt = set()
for hybridPred, adornment in [
(t, a) for t, a in set([(URIRef(GetOp(term).split('_derived')[0]
) if GetOp(term).find('_derived') +
1 else GetOp(term), ''.join(term.adornment))
for term in rt
if isinstance(term, AdornedUniTerm)])
if t in hybridPreds2Replace
]:
#If there are hybrid predicates, add rules that derived their IDB counterpart
#using information from the adorned queries to determine appropriate arity
#and adornment
hybridPred = URIRef(hybridPred)
hPred = URIRef(hybridPred + u'_derived')
if len(adornment) == 1:
# p_derived^{a}(X) :- p(X)
body = BuildUnitermFromTuple((Variable('X'), RDF.type, hybridPred))
head = BuildUnitermFromTuple((Variable('X'), RDF.type, hPred))
else:
# p_derived^{a}(X, Y) :- p(X, Y)
body = BuildUnitermFromTuple(
(Variable('X'), hybridPred, Variable('Y')))
head = BuildUnitermFromTuple((Variable('X'), hPred, Variable('Y')))
_head = AdornedUniTerm(head, list(adornment))
rule = AdornedRule(Clause(And([body]), _head.clone()))
rule.sip = Graph()
adornedProgram.add(rule)
if factGraph is not None:
factGraph.adornedProgram = adornedProgram
return adornedProgram
def testOwl(self):
testData = {}
for manifest in glob('OWL/*/Manifest*.rdf'):
if manifest in Tests2Skip:
continue
if manifest in NonNaiveSkip or manifest in BFPTests2SKip:
continue
skip = False
for pattern2Skip in patterns2Skip:
if manifest.find(pattern2Skip) > -1:
skip = True
break
if skip:
continue
manifestStore = plugin.get(RDFLIB_STORE, Store)()
manifestGraph = Graph(manifestStore)
manifestGraph.parse(open(manifest))
rt = manifestGraph.query(MANIFEST_QUERY, initNs=nsMap, DEBUG=False)
#print list(manifestGraph.namespace_manager.namespaces())
for status, premise, conclusion, feature, description in rt:
if feature in Features2Skip:
continue
premise = manifestGraph.namespace_manager.compute_qname(
premise)[-1]
conclusion = manifestGraph.namespace_manager.compute_qname(
conclusion)[-1]
premiseFile = '/'.join(manifest.split('/')[:2] + [premise])
conclusionFile = '/'.join(
manifest.split('/')[:2] + [conclusion])
print premiseFile
print conclusionFile
if status == 'APPROVED':
if SINGLE_TEST and premiseFile != SINGLE_TEST:
continue
assert os.path.exists('.'.join([premiseFile, 'rdf']))
assert os.path.exists('.'.join([conclusionFile, 'rdf']))
print "<%s> :- <%s>" % ('.'.join([conclusionFile, 'rdf']),
'.'.join([premiseFile, 'rdf']))
store = plugin.get(RDFLIB_STORE, Store)()
store.open(RDFLIB_CONNECTION)
factGraph = Graph(store)
factGraph.parse(open('.'.join([premiseFile, 'rdf'])))
nsMap.update(
dict([(k, v) for k, v in factGraph.namespaces()]))
if DEBUG:
print "## Source Graph ##\n", factGraph.serialize(
format='n3')
Individual.factoryGraph = factGraph
for c in AllClasses(factGraph):
if not isinstance(c.identifier, BNode):
print c.__repr__(True)
if feature in TopDownTests2Skip:
continue
print premiseFile, feature, description
program = list(HornFromN3(StringIO(non_DHL_OWL_Semantics)))
program.extend(
self.network.setupDescriptionLogicProgramming(
factGraph,
addPDSemantics=False,
constructNetwork=False))
print "Original program"
pprint(program)
timings = []
try:
goals = []
for triple in Graph(store).parse('.'.join(
[conclusionFile, 'rdf'])):
if triple not in factGraph:
goals.append(triple)
testData[manifest] = self.MagicOWLProof(
goals, program, factGraph, conclusionFile)
self.setUp()
# self.network._resetinstanciationStats()
# self.network.reset()
# self.network.clear()
except:
# print "missing triple %s"%(pformat(goal))
print manifest, premiseFile
print "feature: ", feature
print description
from FuXi.Rete.Util import renderNetwork
pprint([
BuildUnitermFromTuple(t)
for t in self.network.inferredFacts
])
# dot=renderNetwork(self.network,self.network.nsMap).write_jpeg('test-fail.jpeg')
raise #Exception ("Failed test: "+feature)
pprint(testData)
def conjunctiveSipStrategy(self, goalsRemaining, factGraph, bindings=None):
"""
Given a conjunctive set of triples, invoke sip-strategy passing
on intermediate solutions to facilitate 'join' behavior
"""
bindings = bindings if bindings else {}
try:
tp = next(goalsRemaining)
assert isinstance(bindings, dict)
dPred = self.derivedPredicateFromTriple(tp)
if dPred is None:
baseEDBQuery = EDBQuery([BuildUnitermFromTuple(tp)],
self.edb,
bindings=bindings)
if self.DEBUG:
print("Evaluating TP against EDB:%s" %
baseEDBQuery.asSPARQL())
query, rt = baseEDBQuery.evaluate()
# _vars = baseEDBQuery.returnVars
for item in rt:
bindings.update(item)
for ansDict in self.conjunctiveSipStrategy(
goalsRemaining, factGraph, bindings):
yield ansDict
else:
queryLit = BuildUnitermFromTuple(tp)
currentOp = GetOp(queryLit)
queryLit.setOperator(currentOp)
query = EDBQuery([queryLit], self.edb, bindings=bindings)
if bindings:
tp = first(query.formulae).toRDFTuple()
if self.DEBUG:
print("Goal/Query: ", query.asSPARQL())
SetupDDLAndAdornProgram(
self.edb,
self.idb, [tp],
derivedPreds=self.derivedPredicates,
ignoreUnboundDPreds=True,
hybridPreds2Replace=self.hybridPredicates)
if self.hybridPredicates:
lit = BuildUnitermFromTuple(tp)
op = GetOp(lit)
if op in self.hybridPredicates:
lit.setOperator(URIRef(op + u'_derived'))
tp = lit.toRDFTuple()
sipCollection = PrepareSipCollection(self.edb.adornedProgram)
if self.DEBUG and sipCollection:
for sip in SIPRepresentation(sipCollection):
print(sip)
pprint(list(self.edb.adornedProgram), sys.stderr)
elif self.DEBUG:
print("No SIP graph.")
for nextAnswer, ns in self.invokeDecisionProcedure(
tp, factGraph, bindings, self.DEBUG, sipCollection):
nonGroundGoal = isinstance(nextAnswer, dict)
if nonGroundGoal or nextAnswer:
#Either we recieved bindings from top-down evaluation
#or we (successfully) proved a ground query
if not nonGroundGoal:
#Attempt to prove a ground query, return the response
rt = nextAnswer
else:
#Recieved solutions to 'open' query, merge with given bindings
#and continue
rt = mergeMappings1To2(bindings, nextAnswer)
#either answers were provided (the goal wasn't grounded) or
#the goal was ground and successfully proved
for ansDict in self.conjunctiveSipStrategy(
goalsRemaining, factGraph, rt):
yield ansDict
except StopIteration:
yield bindings
