def IsHybridPredicateRule(rule, derivedSuffix='derived'):
fullSuffix = '_' + derivedSuffix
basePred = URIRef(GetOp(rule.formula.head).split(fullSuffix)[0])
return GetOp(
rule.formula.head).find(fullSuffix) + 1 and len(
list(iterCondition(rule.formula.body))) == 1 and \
GetOp(list(iterCondition(rule.formula.body))[0]) == basePred
def isSecondOrder(self):
secondOrder = [pred
for pred in itertools.chain(
iterCondition(self.formula.head),
iterCondition(self.formula.body))
if pred.isSecondOrder()]
return bool(secondOrder)
def isSecondOrder(self):
secondOrder = [
pred for pred in itertools.chain(iterCondition(self.formula.head),
iterCondition(self.formula.body))
if pred.isSecondOrder()
]
return bool(secondOrder)
def __init__(self, formulae=None, n3Rules=None, nsMapping=None):
from FuXi.Rete.RuleStore import N3Builtin
self.nsMapping = nsMapping and nsMapping or {}
self.formulae = formulae and formulae or []
if n3Rules:
from FuXi.DLP import breadth_first
# Convert a N3 abstract model (parsed from N3) into a RIF BLD
for lhs, rhs in n3Rules:
allVars = set()
for ruleCondition in [lhs, rhs]:
for stmt in ruleCondition:
if isinstance(stmt, N3Builtin):
ExternalFunction(stmt, newNss=self.nsMapping)
# print(stmt)
# raise
allVars.update(
[term for term in stmt if isinstance(term, (BNode, Variable))])
body = [isinstance(term, N3Builtin) and term or
Uniterm(list(term)[1], [list(term)[0], list(term)[-1]],
newNss=nsMapping) for term in lhs]
body = len(body) == 1 and body[0] or And(body)
head = [Uniterm(p, [s, o], newNss=nsMapping)
for s, p, o in rhs]
head = len(head) == 1 and head[0] or And(head)
# first we identify body variables
bodyVars = set(reduce(lambda x, y: x + y,
[list(extractVariables(i, existential=False))
for i in iterCondition(body)]))
# then we identify head variables
headVars = set(reduce(lambda x, y: x + y,
[list(extractVariables(i, existential=False))
for i in iterCondition(head)]))
# then we identify those variables that should (or should not)
# be converted to skolem terms
updateDict = dict([(var, BNode())
for var in headVars if var not in bodyVars])
for uniTerm in iterCondition(head):
def updateUniterm(uterm):
newArg = [updateDict.get(i, i) for i in uniTerm.arg]
uniTerm.arg = newArg
if isinstance(uniTerm, Uniterm):
updateUniterm(uniTerm)
else:
for u in uniTerm:
updateUniterm(u)
exist = [list(extractVariables(i))
for i in breadth_first(head)]
e = Exists(formula=head,
declare=set(reduce(lambda x, y: x + y, exist, [])))
if reduce(lambda x, y: x + y, exist):
head = e
assert e.declare, exist
self.formulae.append(Rule(Clause(body, head), declare=allVars))
def ReplaceHybridPredcates(rules, hybridPreds2Replace):
for rule in rules:
for headLiteral in iterCondition(rule.formula.head):
headOp = GetOp(headLiteral)
if headOp in hybridPreds2Replace:
headLiteral.setOperator(URIRef(headOp + u'_derived'))
for bodyAtom in iterCondition(rule.formula.body):
bodyPred = GetOp(bodyAtom)
if bodyPred in hybridPreds2Replace:
bodyAtom.setOperator(URIRef(bodyPred + u'_derived'))
def __init__(self, clause, declare=None, nsMapping=None):
decl = set()
self.ruleStr = ''
for pred in itertools.chain(iterCondition(clause.head),
iterCondition(clause.body)):
decl.update(
[term for term in GetArgs(pred) if isinstance(term, Variable)])
if isinstance(pred, AdornedUniTerm):
self.ruleStr += ''.join(pred.adornment)
self.ruleStr += ''.join(pred.toRDFTuple())
super(AdornedRule, self).__init__(clause, decl, nsMapping)
def PrepareHornClauseForRETE(horn_clause):
if isinstance(horn_clause, Rule):
horn_clause = horn_clause.formula
def extractVariables(term, existential=True):
if isinstance(term, existential and BNode or Variable):
yield term
elif isinstance(term, Uniterm):
for t in term.toRDFTuple():
if isinstance(t, existential and BNode or Variable):
yield t
from FuXi.Rete.SidewaysInformationPassing import iterCondition, GetArgs
#first we identify body variables
bodyVars = set(
reduce(lambda x, y: x + y, [
list(extractVariables(i, existential=False))
for i in iterCondition(horn_clause.body)
]))
#then we identify head variables
headVars = set(
reduce(lambda x, y: x + y, [
list(extractVariables(i, existential=False))
for i in iterCondition(horn_clause.head)
]))
#then we identify those variables that should (or should not) be converted to skolem terms
updateDict = dict([(var, BNode()) for var in headVars
if var not in bodyVars])
if set(updateDict.keys()).intersection(GetArgs(horn_clause.head)):
#There are skolem terms in the head
newHead = copy.deepcopy(horn_clause.head)
for uniTerm in iterCondition(newHead):
newArg = [updateDict.get(i, i) for i in uniTerm.arg]
uniTerm.arg = newArg
horn_clause.head = newHead
skolemsInBody = [
list(
itertools.ifilter(lambda term: isinstance(term, BNode),
GetArgs(lit)))
for lit in iterCondition(horn_clause.body)
]
skolemsInBody = reduce(lambda x, y: x + y, skolemsInBody, [])
if skolemsInBody:
newBody = copy.deepcopy(horn_clause.body)
_e = Exists(formula=newBody, declare=set(skolemsInBody))
horn_clause.body = _e
PrepareHeadExistential(horn_clause)
def __init__(self, clause, declare=None, nsMapping=None):
decl = set()
self.ruleStr = ''
for pred in itertools.chain(iterCondition(clause.head),
iterCondition(clause.body)):
decl.update([term
for term in GetArgs(pred)
if isinstance(term, Variable)])
if isinstance(pred, AdornedUniTerm):
self.ruleStr += ''.join(pred.adornment)
self.ruleStr += ''.join(pred.toRDFTuple())
super(AdornedRule, self).__init__(clause, decl, nsMapping)
def PrepareHornClauseForRETE(horn_clause):
if isinstance(horn_clause, Rule):
horn_clause = horn_clause.formula
def extractVariables(term, existential=True):
if isinstance(term, existential and BNode or Variable):
yield term
elif isinstance(term, Uniterm):
for t in term.toRDFTuple():
if isinstance(t, existential and BNode or Variable):
yield t
from FuXi.Rete.SidewaysInformationPassing import iterCondition, GetArgs
# first we identify body variables
bodyVars = set(
reduce(lambda x, y: x + y,
[list(extractVariables(i, existential=False))
for i in iterCondition(horn_clause.body)]))
# then we identify head variables
headVars = set(
reduce(lambda x, y: x + y,
[list(extractVariables(i, existential=False))
for i in iterCondition(horn_clause.head)]))
# then we identify those variables that should (or should not) be
# converted to skolem terms
updateDict = dict(
[(var, BNode()) for var in headVars if var not in bodyVars])
if set(updateDict.keys()).intersection(GetArgs(horn_clause.head)):
# There are skolem terms in the head
newHead = copy.deepcopy(horn_clause.head)
for uniTerm in iterCondition(newHead):
newArg = [updateDict.get(i, i) for i in uniTerm.arg]
uniTerm.arg = newArg
horn_clause.head = newHead
skolemsInBody = [
list(filter(
lambda term:isinstance(term,
BNode),
GetArgs(lit)))
for lit in iterCondition(horn_clause.body)]
skolemsInBody = reduce(lambda x, y: x + y, skolemsInBody,
[])
if skolemsInBody:
newBody = copy.deepcopy(horn_clause.body)
_e = Exists(formula=newBody, declare=set(skolemsInBody))
horn_clause.body = _e
PrepareHeadExistential(horn_clause)
def normalizeQuantification(self, inBody=True):
from FuXi.Rete.SidewaysInformationPassing import GetArgs
if inBody:
bodyExistentials = filter(
lambda i: isinstance(i, BNode),
reduce(lambda l, r: l + r,
map(GetArgs, iterCondition(self.formula.body))))
argMap = dict([(bnode, Variable(bnode))
for bnode in bodyExistentials])
for literal in iterCondition(self.formula.body):
literal.renameVariables(argMap)
else:
raise NotImplementedError(
"normalizing existential in head not supported")
def RDFTuplesToSPARQL(conjunct,
edb,
isGround=False,
vars=[],
symmAtomicInclusion=False):
"""
Takes a conjunction of Horn literals and returns the
corresponding SPARQL query
"""
queryType = isGround and "ASK" or "SELECT %s" % (' '.join(
[v.n3() for v in vars]))
queryShell = len(conjunct) > 1 and "%s {\n%s\n}" or "%s { %s }"
if symmAtomicInclusion:
if vars:
var = vars.pop()
prefix = "%s a ?KIND" % var.n3()
else:
prefix = "%s a ?KIND" % first(
[first(iterCondition(lit)).arg[0].n3() for lit in conjunct])
conjunct = (i.formulae[0] if isinstance(i, And) else i
for i in conjunct)
subquery = queryShell % (
queryType, "%s\nFILTER(%s)" % (prefix, ' ||\n'.join(
['?KIND = %s' % edb.qname(GetOp(lit)) for lit in conjunct])))
else:
subquery = queryShell % (queryType, ' .\n'.join(
['\t' + tripleToTriplePattern(edb, lit) for lit in conjunct]))
return subquery
def collapseMINUS(left,right):
negVars=set()
for pred in iterCondition(right):
negVars.update([term for term in GetArgs(pred)
if isinstance(term,Variable)])
innerCopyPatternNeeded = not negVars.difference(positiveVars)
#A copy pattern is needed if the negative literals don't introduce new vars
if innerCopyPatternNeeded:
innerCopyPatterns,innerVars,innerVarExprs=createCopyPattern([right])
#We use an arbitrary new variable as for the outer FILTER(!BOUND(..))
outerFilterVariable=innerVars.values()[0]
optionalPatterns=[right] +innerCopyPatterns
negatedBGP=optional(*[formula.toRDFTuple()
for formula in optionalPatterns])
negatedBGP.filter(*[k==v for k,v in innerVarExprs.items()])
positiveVars.update([Variable(k.value[0:]) for k in innerVarExprs.keys()])
positiveVars.update(innerVarExprs.values())
else:
#We use an arbitrary, 'independent' variable for the outer FILTER(!BOUND(..))
outerFilterVariable=negVars.difference(positiveVars).pop()
optionalPatterns=[right]
negatedBGP=optional(*[formula.toRDFTuple()
for formula in optionalPatterns])
positiveVars.update(negVars)
left=left.where(*[negatedBGP])
left=left.filter(~op.bound(outerFilterVariable))
return left
def RDFTuplesToSPARQL(conjunct,
edb,
isGround=False,
vars=[],
symmAtomicInclusion=False,
specialBNodeHandling=None):
"""
Takes a conjunction of Horn literals and returns the
corresponding SPARQL query
"""
queryType = isGround and "ASK" or "SELECT %s"%(' '.join([v.n3()
for v in vars]))
queryShell = len(conjunct)>1 and "%s {\n%s\n}" or "%s { %s }"
if symmAtomicInclusion:
if vars:
var = vars.pop()
prefix = "%s a ?KIND"%var.n3()
else:
prefix = "%s a ?KIND"%first([first(iterCondition(lit)).arg[0].n3() for lit in conjunct])
conjunct = ( i.formulae[0] if isinstance(i,And) else i for i in conjunct )
subquery = queryShell%(queryType,
"%s\nFILTER(%s)"%(
prefix,
' ||\n'.join([
'?KIND = %s'%edb.qname(GetOp(lit))
for lit in conjunct])))
else:
subquery = queryShell%(queryType,' .\n'.join(['\t'+tripleToTriplePattern(
edb,
lit,
specialBNodeHandling)
for lit in conjunct ]))
return subquery
def collapseMINUS(left, right):
negVars = set()
for pred in iterCondition(right):
negVars.update(
[term for term in GetArgs(pred) if isinstance(term, Variable)])
innerCopyPatternNeeded = not negVars.difference(positiveVars)
#A copy pattern is needed if the negative literals don't introduce new vars
if innerCopyPatternNeeded:
innerCopyPatterns, innerVars, innerVarExprs = createCopyPattern(
[right])
#We use an arbitrary new variable as for the outer FILTER(!BOUND(..))
outerFilterVariable = list(innerVars.values())[0]
optionalPatterns = [right] + innerCopyPatterns
negatedBGP = optional(
*[formula.toRDFTuple() for formula in optionalPatterns])
negatedBGP.filter(
*[k == v for k, v in list(innerVarExprs.items())])
positiveVars.update(
[Variable(k.value[0:]) for k in list(innerVarExprs.keys())])
positiveVars.update(list(innerVarExprs.values()))
else:
#We use an arbitrary, 'independent' variable for the outer FILTER(!BOUND(..))
outerFilterVariable = negVars.difference(positiveVars).pop()
optionalPatterns = [right]
negatedBGP = optional(
*[formula.toRDFTuple() for formula in optionalPatterns])
positiveVars.update(negVars)
left = left.where(*[negatedBGP])
left = left.filter(~op.bound(outerFilterVariable))
return left
def IsAtomicInclusionAxiomRHS(rule, dPreds):
"""
This is an atomic inclusion axiom with
a variable (or bound) RHS: uniPred(?ENTITY)
"""
bodyList = list(iterCondition(rule.formula.body))
body = first(bodyList)
return GetOp(body) not in dPreds and len(bodyList) == 1 and body.op == RDF.type
def CreateHybridPredicateRule(hybridPred, program, nsMap=None):
hPred = URIRef(hybridPred + u'_derived')
literals = set(reduce(lambda l, r: l + r,
[list(iterCondition(clause.formula.body)) +
list(iterCondition(clause.formula.head))
for clause in program]))
for literal in literals:
if GetOp(literal) == hybridPred:
noArgs = len(GetArgs(literal))
if noArgs == 1:
# p_derived(X) :- p(X)
body = BuildUnitermFromTuple(
(Variable('X'),
RDF.type,
hybridPred),
newNss=nsMap
)
head = BuildUnitermFromTuple(
(Variable('X'),
RDF.type,
hPred),
newNss=nsMap
)
vars = [Variable('X')]
else:
# p_derived(X,Y) :- p(X,Y)
body = BuildUnitermFromTuple(
(Variable('X'),
hybridPred,
Variable('Y')),
newNss=nsMap
)
head = BuildUnitermFromTuple(
(Variable('X'),
hPred,
Variable('Y')),
newNss=nsMap
)
vars = [Variable('Y'), Variable('X')]
return Rule(
Clause(And([body]), head),
nsMapping=nsMap,
declare=vars
)
def IsAtomicInclusionAxiomRHS(rule, dPreds):
"""
This is an atomic inclusion axiom with
a variable (or bound) RHS: uniPred(?ENTITY)
"""
bodyList = list(iterCondition(rule.formula.body))
body = first(bodyList)
return GetOp(body) not in dPreds and \
len(bodyList) == 1 and \
body.op == RDF.type
def PrepareHornClauseForRETE(horn_clause):
if isinstance(horn_clause,Rule):
horn_clause=horn_clause.formula
def extractVariables(term,existential=True):
if isinstance(term,existential and BNode or Variable):
yield term
elif isinstance(term,Uniterm):
for t in term.toRDFTuple():
if isinstance(t,existential and BNode or Variable):
yield t
def iterCondition(condition):
return isinstance(condition,SetOperator) and condition or iter([condition])
#first we identify body variables
bodyVars = set(reduce(lambda x,y:x+y,
[ list(extractVariables(i,existential=False)) for i in iterCondition(horn_clause.body) ]))
#then we identify head variables
headVars = set(reduce(lambda x,y:x+y,
[ list(extractVariables(i,existential=False)) for i in iterCondition(horn_clause.head) ]))
#then we identify those variables that should (or should not) be converted to skolem terms
updateDict = dict([(var,BNode()) for var in headVars if var not in bodyVars])
for uniTerm in iterCondition(horn_clause.head):
newArg = [ updateDict.get(i,i) for i in uniTerm.arg ]
uniTerm.arg = newArg
headExist=[list(extractVariables(i)) for i in breadth_first(horn_clause.body)]
_e=Exists(formula=horn_clause.body,
declare=set(reduce(lambda x,y:x+y,headExist,[])))
if reduce(lambda x,y:x+y,headExist):
horn_clause.body=_e
assert _e.declare,headExist
exist=[list(extractVariables(i)) for i in breadth_first(horn_clause.head)]
e=Exists(formula=horn_clause.head,
declare=set(reduce(lambda x,y:x+y,exist,[])))
if reduce(lambda x,y:x+y,exist):
horn_clause.head=e
assert e.declare,exist
def normalizeQuantification(self,inBody=True):
from FuXi.Rete.SidewaysInformationPassing import GetArgs
if inBody:
bodyExistentials = filter(
lambda i:isinstance(i,BNode),
reduce(
lambda l,r: l+r,
map(
GetArgs,
iterCondition(self.formula.body)
)
)
)
argMap = dict([(bnode,Variable(bnode))
for bnode in bodyExistentials])
for literal in iterCondition(self.formula.body):
literal.renameVariables(argMap)
else:
raise NotImplementedError(
"normalizing existential in head not supported"
)
def PrepareHeadExistential(clause):
from FuXi.Rete.SidewaysInformationPassing import GetArgs
skolemsInHead = [
list(filter(lambda term: isinstance(term, BNode), GetArgs(lit)))
for lit in iterCondition(clause.head)
]
skolemsInHead = reduce(lambda x, y: x + y, skolemsInHead, [])
if skolemsInHead:
newHead = copy.deepcopy(clause.head)
_e = Exists(formula=newHead, declare=set(skolemsInHead))
clause.head = _e
return clause
def isRecursive(self):
def termHash(term):
return GetOp(term), \
reduce(lambda x, y: x + y, term.adornment)
headHash = termHash(self.formula.head)
def recursiveLiteral(term):
return isinstance(term, AdornedUniTerm) and termHash(term) == headHash
if first(filter(recursiveLiteral, iterCondition(self.formula.body))):
return True
else:
return False
def PrepareHeadExistential(clause):
from FuXi.Rete.SidewaysInformationPassing import GetArgs
skolemsInHead=[
list(itertools.ifilter(
lambda term:isinstance(term,
BNode),
GetArgs(lit)))
for lit in iterCondition(clause.head)]
skolemsInHead = reduce(lambda x,y:x+y,skolemsInHead,[])
if skolemsInHead:
newHead = copy.deepcopy(clause.head)
_e=Exists(formula=newHead,declare=set(skolemsInHead))
clause.head=_e
return clause
def isRecursive(self):
def termHash(term):
return GetOp(term), \
reduce(lambda x, y: x + y, term.adornment)
headHash = termHash(self.formula.head)
def recursiveLiteral(term):
return isinstance(term,
AdornedUniTerm) and termHash(term) == headHash
if first(filter(recursiveLiteral, iterCondition(self.formula.body))):
return True
else:
return False
def isSafe(self):
"""
A RIF-Core rule, r is safe if and only if
- r is a rule implication, φ :- ψ, and all the variables that occur
in φ are safe in ψ, and all the variables that occur in ψ are bound
in ψ;
- or r is a universal rule, Forall v1,...,vn (r'), n ≥ 1, and r' is
safe.
>>> clause1 = Clause(And([Uniterm(RDFS.subClassOf,[Variable('C'),Variable('SC')]),
... Uniterm(RDF.type,[Variable('M'),Variable('C')])]),
... Uniterm(RDF.type,[Variable('M'),Variable('SC')]))
>>> r1 = Rule(clause1,[Variable('M'),Variable('SC'),Variable('C')])
>>> clause2 = Clause(And([Uniterm(RDFS.subClassOf,[Variable('C'),Variable('SC')])]),
... Uniterm(RDF.type,[Variable('M'),Variable('SC')]))
>>> r2 = Rule(clause2,[Variable('M'),Variable('SC'),Variable('C')])
>>> r1.isSafe()
True
>>> r2.isSafe()
False
>>> skolemTerm = BNode()
>>> e = Exists(Uniterm(
... RDFS.subClassOf,
... [skolemTerm,Variable('C')]),
... declare=[skolemTerm])
>>> r1.formula.head = e
>>> r1.isSafe()
False
"""
from FuXi.Rete.SidewaysInformationPassing import GetArgs, iterCondition
assert isinstance(self.formula.head, (Exists, Atomic)),\
"Safety can only be checked on rules in normal form"
for var in itertools.ifilter(lambda term: isinstance(term,
(Variable, BNode)),
GetArgs(self.formula.head)):
if not self.formula.body.isSafeForVariable(var):
return False
for var in itertools.ifilter(
lambda term: isinstance(term, (Variable, BNode)),
reduce(lambda l, r: l + r,
[GetArgs(lit)
for lit in iterCondition(self.formula.body)])):
if not self.formula.body.binds(var):
return False
return True
def isSafe(self):
"""
A RIF-Core rule, r is safe if and only if
- r is a rule implication, φ :- ψ, and all the variables that occur
in φ are safe in ψ, and all the variables that occur in ψ are bound in ψ;
- or r is a universal rule, Forall v1,...,vn (r'), n ≥ 1, and r' is safe.
>>> clause1 = Clause(And([Uniterm(RDFS.subClassOf,[Variable('C'),Variable('SC')]),
... Uniterm(RDF.type,[Variable('M'),Variable('C')])]),
... Uniterm(RDF.type,[Variable('M'),Variable('SC')]))
>>> r1 = Rule(clause1,[Variable('M'),Variable('SC'),Variable('C')])
>>> clause2 = Clause(And([Uniterm(RDFS.subClassOf,[Variable('C'),Variable('SC')])]),
... Uniterm(RDF.type,[Variable('M'),Variable('SC')]))
>>> r2 = Rule(clause2,[Variable('M'),Variable('SC'),Variable('C')])
>>> r1.isSafe()
True
>>> r2.isSafe()
False
>>> skolemTerm = BNode()
>>> e = Exists(Uniterm(RDFS.subClassOf,[skolemTerm,Variable('C')]),declare=[skolemTerm])
>>> r1.formula.head = e
>>> r1.isSafe()
False
"""
from FuXi.Rete.SidewaysInformationPassing import GetArgs, iterCondition
assert isinstance(self.formula.head,(Exists,Atomic)),\
"Safety can only be checked on rules in normal form"
for var in itertools.ifilter(
lambda term: isinstance(term, (Variable, BNode)),
GetArgs(self.formula.head)):
if not self.formula.body.isSafeForVariable(var):
return False
for var in itertools.ifilter(
lambda term: isinstance(term, (Variable, BNode)),
reduce(
lambda l, r: l + r,
[GetArgs(lit)
for lit in iterCondition(self.formula.body)])):
if not self.formula.body.binds(var):
return False
return True
def SipStrategy(query,
sipCollection,
factGraph,
derivedPreds,
bindings={},
processedRules=None,
network=None,
debug=False,
buildProof=False,
memoizeMemory=None,
proofLevel=1):
"""
Accordingly, we define a sip-strategy for computing the answers to a query
expressed using a set of Datalog rules, and a set of sips, one for each
adornment of a rule head, as follows...
Each evaluation uses memoization (via Python decorators) but also relies on well-formed
rewrites for using semi-naive bottom up method over large SPARQL data.
"""
memoizeMemory = memoizeMemory and memoizeMemory or {}
queryLiteral = BuildUnitermFromTuple(query)
processedRules = processedRules and processedRules or set()
if bindings:
# There are bindings. Apply them to the terms in the query
queryLiteral.ground(bindings)
if debug:
print("%sSolving" % ('\t' * proofLevel), queryLiteral, bindings)
# Only consider ground triple pattern isomorphism with matching bindings
goalRDFStatement = queryLiteral.toRDFTuple()
if queryLiteral in memoizeMemory:
if debug:
print("%sReturning previously calculated results for " %
('\t' * proofLevel), queryLiteral)
for answers in memoizeMemory[queryLiteral]:
yield answers
elif AlphaNode(goalRDFStatement).alphaNetworkHash(
True,
skolemTerms=list(bindings.values())) in \
[AlphaNode(r.toRDFTuple()).alphaNetworkHash(True,
skolemTerms=list(bindings.values()))
for r in processedRules
if AdornLiteral(goalRDFStatement).adornment ==
r.adornment]:
if debug:
print("%s Goal already processed..." %
('\t' * proofLevel))
else:
isGround = literalIsGround(queryLiteral)
if buildProof:
ns = NodeSet(goalRDFStatement,
network=network, identifier=BNode())
else:
ns = None
# adornedProgram = factGraph.adornedProgram
queryPred = GetOp(queryLiteral)
if sipCollection is None:
rules = []
else:
# For every rule head matching the query, we invoke the rule,
# thus determining an adornment, and selecting a sip to follow
rules = sipCollection.headToRule.get(queryPred, set())
if None in sipCollection.headToRule:
# If there are second order rules, we add them
# since they are a 'wildcard'
rules.update(sipCollection.headToRule[None])
# maintained list of rules that haven't been processed before and
# match the query
validRules = []
# each subquery contains values for the bound arguments that are passed
# through the sip arcs entering the node corresponding to that literal. For
# each subquery generated, there is a set of answers.
answers = []
# variableMapping = {}
# Some TBox queries can be 'joined' together into SPARQL queries against
# 'base' predicates via an RDF dataset
# These atomic concept inclusion axioms can be evaluated together
# using a disjunctive operator at the body of a horn clause
# where each item is a query of the form uniPredicate(?X):
# Or( uniPredicate1(?X1), uniPredicate2(?X), uniPredicate3(?X), ..)
# In this way massive, conjunctive joins can be 'mediated'
# between the stated facts and the top-down solver
@parameterizedPredicate([i for i in derivedPreds])
def IsAtomicInclusionAxiomRHS(rule, dPreds):
"""
This is an atomic inclusion axiom with
a variable (or bound) RHS: uniPred(?ENTITY)
"""
bodyList = list(iterCondition(rule.formula.body))
body = first(bodyList)
return GetOp(body) not in dPreds and \
len(bodyList) == 1 and \
body.op == RDF.type
atomicInclusionAxioms = list(filter(
IsAtomicInclusionAxiomRHS, rules))
if atomicInclusionAxioms and len(atomicInclusionAxioms) > 1:
if debug:
print("\tCombining atomic inclusion axioms: ")
pprint(atomicInclusionAxioms, sys.stderr)
if buildProof:
factStep = InferenceStep(ns, source='some RDF graph')
ns.steps.append(factStep)
axioms = [rule.formula.body
for rule in atomicInclusionAxioms]
# attempt to exaustively apply any available substitutions
# and determine if query if fully ground
vars = [v for v in GetArgs(queryLiteral,
secondOrder=True)
if isinstance(v, Variable)]
openVars, axioms, _bindings = \
normalizeBindingsAndQuery(vars,
bindings,
axioms)
if openVars:
# mappings = {}
# See if we need to do any variable mappings from the query literals
# to the literals in the applicable rules
query, rt = EDBQuery(axioms,
factGraph,
openVars,
_bindings).evaluate(debug,
symmAtomicInclusion=True)
if buildProof:
# FIXME: subquery undefined
factStep.groundQuery = subquery
for ans in rt:
if buildProof:
factStep.bindings.update(ans)
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(ans), ns))
yield ans, ns
else:
# All the relevant derivations have been explored and the result
# is a ground query we can directly execute against the facts
if buildProof:
factStep.bindings.update(bindings)
query, rt = EDBQuery(axioms,
factGraph,
_bindings).evaluate(debug,
symmAtomicInclusion=True)
if buildProof:
# FIXME: subquery undefined
factStep.groundQuery = subquery
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(rt), ns))
yield rt, ns
rules = filter(
lambda i: not IsAtomicInclusionAxiomRHS(i), rules)
for rule in rules:
# An exception is the special predicate ph; it is treated as a base
# predicate and the tuples in it are those supplied for qb by
# unification.
headBindings = getBindingsFromLiteral(
goalRDFStatement, rule.formula.head)
# comboBindings = dict([(k, v) for k, v in itertools.chain(
# bindings.items(),
# headBindings.items())])
varMap = rule.formula.head.getVarMapping(queryLiteral)
if headBindings and\
[term for term in rule.formula.head.getDistinguishedVariables(True)
if varMap.get(term, term) not in headBindings]:
continue
# subQueryAnswers = []
# dontStop = True
# projectedBindings = comboBindings.copy()
if debug:
print("%sProcessing rule" %
('\t' * proofLevel), rule.formula)
if debug and sipCollection:
print(
"Sideways Information Passing (sip) graph for %s: " % queryLiteral)
print(sipCollection.serialize(format='n3'))
for sip in SIPRepresentation(sipCollection):
print(sip)
try:
# Invoke the rule
if buildProof:
step = InferenceStep(ns, rule.formula)
else:
step = None
for rt, step in\
invokeRule([headBindings],
iter(iterCondition(rule.formula.body)),
rule.sip,
(proofLevel + 1,
memoizeMemory,
sipCollection,
factGraph,
derivedPreds,
processedRules.union([
AdornLiteral(query)])),
step=step,
debug=debug):
if rt:
if isinstance(rt, dict):
# We received a mapping and must rewrite it via
# correlation between the variables in the rule head
# and the variables in the original query (after applying
# bindings)
varMap = rule.formula.head.getVarMapping(
queryLiteral)
if varMap:
rt = MakeImmutableDict(
refactorMapping(varMap, rt))
if buildProof:
step.bindings = rt
else:
if buildProof:
step.bindings = headBindings
validRules.append(rule)
if buildProof:
ns.steps.append(step)
if isGround:
yield True, ns
else:
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(rt),
ns))
yield rt, ns
except RuleFailure:
# Clean up failed antecedents
if buildProof:
if ns in step.antecedents:
step.antecedents.remove(ns)
if not validRules:
# No rules matching, query factGraph for answers
successful = False
if buildProof:
factStep = InferenceStep(ns, source='some RDF graph')
ns.steps.append(factStep)
if not isGround:
subquery, rt = EDBQuery([queryLiteral],
factGraph,
[v for v in GetArgs(queryLiteral,
secondOrder=True)
if isinstance(v, Variable)],
bindings).evaluate(debug)
if buildProof:
factStep.groundQuery = subquery
for ans in rt:
successful = True
if buildProof:
factStep.bindings.update(ans)
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(ans),
ns))
yield ans, ns
if not successful and queryPred not in derivedPreds:
# Open query didn't return any results and the predicate
# is ostensibly marked as derived predicate, so we have
# failed
memoizeMemory.setdefault(
queryLiteral, set()).add((False, ns))
yield False, ns
else:
# All the relevant derivations have been explored and the result
# is a ground query we can directly execute against the facts
if buildProof:
factStep.bindings.update(bindings)
subquery, rt = EDBQuery([queryLiteral],
factGraph,
bindings).evaluate(debug)
if buildProof:
factStep.groundQuery = subquery
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(rt),
ns))
yield rt, ns
def SetupDDLAndAdornProgram(factGraph,
rules,
GOALS,
derivedPreds=None,
strictCheck=DDL_STRICTNESS_FALLBACK_DERIVED,
defaultPredicates=None,
ignoreUnboundDPreds=False,
hybridPreds2Replace=None):
if not defaultPredicates:
defaultPredicates = [], []
# _dPredsProvided = bool(derivedPreds)
if not derivedPreds:
_derivedPreds = DerivedPredicateIterator(
factGraph,
rules,
strict=strictCheck,
defaultPredicates=defaultPredicates)
if not isinstance(derivedPreds, (set, list)):
derivedPreds = list(_derivedPreds)
else:
derivedPreds.extend(_derivedPreds)
hybridPreds2Replace = hybridPreds2Replace or []
adornedProgram = AdornProgram(factGraph,
rules,
GOALS,
derivedPreds,
ignoreUnboundDPreds,
hybridPreds2Replace=hybridPreds2Replace)
if adornedProgram != set([]):
rt = reduce(lambda l, r: l + r, [
list(iterCondition(clause.formula.body))
for clause in adornedProgram
])
else:
rt = set()
for hybridPred, adornment in [
(t, a) for t, a in set([(URIRef(GetOp(term).split('_derived')[0]
) if GetOp(term).find('_derived') +
1 else GetOp(term), ''.join(term.adornment))
for term in rt
if isinstance(term, AdornedUniTerm)])
if t in hybridPreds2Replace
]:
#If there are hybrid predicates, add rules that derived their IDB counterpart
#using information from the adorned queries to determine appropriate arity
#and adornment
hybridPred = URIRef(hybridPred)
hPred = URIRef(hybridPred + u'_derived')
if len(adornment) == 1:
# p_derived^{a}(X) :- p(X)
body = BuildUnitermFromTuple((Variable('X'), RDF.type, hybridPred))
head = BuildUnitermFromTuple((Variable('X'), RDF.type, hPred))
else:
# p_derived^{a}(X, Y) :- p(X, Y)
body = BuildUnitermFromTuple(
(Variable('X'), hybridPred, Variable('Y')))
head = BuildUnitermFromTuple((Variable('X'), hPred, Variable('Y')))
_head = AdornedUniTerm(head, list(adornment))
rule = AdornedRule(Clause(And([body]), _head.clone()))
rule.sip = Graph()
adornedProgram.add(rule)
if factGraph is not None:
factGraph.adornedProgram = adornedProgram
return adornedProgram
def AdornProgram(factGraph,
rs,
goals,
derivedPreds=None,
ignoreUnboundDPreds=False,
hybridPreds2Replace=None):
"""
The process starts from the given query. The query determines bindings for q, and we replace
q by an adorned version, in which precisely the positions bound in the query are designated as
bound, say q e . In general, we have a collection of adorned predicates, and as each one is processed,
we will mark it, so that it will not be processed again. If p a is an unmarked adorned
predicate, then for each rule that has p in its head, we generate an adorned version for the rule
and add it to Pad; then p is marked as processed.
The adorned version of a rule contains additional
adorned predicates, and these are added to the collection, unless they already appear
there. The process terminates when no unmarked adorned predicates are left.
"""
from FuXi.DLP import LloydToporTransformation
from collections import deque
goalDict = {}
hybridPreds2Replace = hybridPreds2Replace or []
adornedPredicateCollection = set()
for goal, nsBindings in NormalizeGoals(goals):
adornedPredicateCollection.add(AdornLiteral(goal, nsBindings))
if not derivedPreds:
derivedPreds = list(DerivedPredicateIterator(factGraph, rs))
def unprocessedPreds(aPredCol):
rt = []
for p in aPredCol:
if not p.marked:
rt.append(p)
if p not in goalDict:
goalDict.setdefault(GetOp(p), set()).add(p)
return rt
toDo = deque(unprocessedPreds(adornedPredicateCollection))
adornedProgram = set()
while len(toDo):
term = toDo.popleft()
#check if there is a rule with term as its head
for rule in rs:
for clause in LloydToporTransformation(rule.formula):
head = isinstance(clause.head, Exists) and clause.head.formula or clause.head
# headPredicate = GetOp(head)
if compareAdornedPredToRuleHead(term, head, hybridPreds2Replace):
#for each rule that has p in its head, we generate an adorned version for the rule
adornedRule = AdornRule(derivedPreds,
clause,
term,
ignoreUnboundDPreds=ignoreUnboundDPreds,
hybridPreds2Replace=hybridPreds2Replace)
adornedProgram.add(adornedRule)
#The adorned version of a rule contains additional adorned
#predicates, and these are added
for pred in iterCondition(adornedRule.formula.body):
if isinstance(pred, N3Builtin):
aPred = pred
else:
aPred = not isinstance(pred, AdornedUniTerm) and \
AdornLiteral(pred.toRDFTuple(),
nsBindings,
pred.naf) or pred
op = GetOp(pred)
if (op in derivedPreds or
(op in hybridPreds2Replace if hybridPreds2Replace
else False)
) and aPred not in adornedPredicateCollection:
adornedPredicateCollection.add(aPred)
term.marked = True
toDo.extendleft(unprocessedPreds(adornedPredicateCollection))
factGraph.queryAtoms = goalDict
return adornedProgram
def MagicSetTransformation(factGraph,
rules,
GOALS,
derivedPreds=None,
strictCheck=DDL_STRICTNESS_FALLBACK_DERIVED,
noMagic=None,
defaultPredicates=None):
"""
Takes a goal and a ruleset and returns an iterator
over the rulest that corresponds to the magic set
transformation:
"""
noMagic = noMagic and noMagic or []
magicPredicates = set()
# replacement = {}
adornedProgram = SetupDDLAndAdornProgram(
factGraph,
rules,
GOALS,
derivedPreds=derivedPreds,
strictCheck=strictCheck,
defaultPredicates=defaultPredicates)
newRules = []
for rule in adornedProgram:
if rule.isSecondOrder():
import warnings
warnings.warn(
"Second order rule no supported by GMS: %s" % rule,
RuntimeWarning)
magicPositions = {}
#Generate magic rules
for idx, pred in enumerate(iterCondition(rule.formula.body)):
# magicBody = []
if isinstance(pred, AdornedUniTerm): # and pred not in magicPredicates:
# For each rule r in Pad, and for each occurrence of an adorned
# predicate p a in its body, we generate a magic rule defining magic_p a
prevPreds = [item for _idx, item in enumerate(rule.formula.body)
if _idx < idx]
if 'b' not in pred.adornment:
import warnings
warnings.warn(
"adorned predicate w/out any bound arguments (%s in %s)" % (pred, rule.formula),
RuntimeWarning)
if GetOp(pred) not in noMagic:
magicPred = pred.makeMagicPred()
magicPositions[idx] = (magicPred, pred)
inArcs = [(N, x) for (N, x) in IncomingSIPArcs(rule.sip, getOccurrenceId(pred))
if not set(x).difference(GetArgs(pred))]
if len(inArcs) > 1:
# If there are several arcs entering qi, we define the
# magic rule defining magic_qi in two steps. First,
# for each arc Nj --> qi with label cj , we define a
# rule with head label_qi_j(cj ). The body of the rule
# is the same as the body of the magic rule in the
# case where there is a single arc entering qi
# (described above). Then the magic rule is defined as
# follows. The head is magic_q(0). The body contains
# label_qi_j(cj) for all j (that is, for all arcs
# entering qi ).
#
# We combine all incoming arcs into a single list of
# (body) conditions for the magic set
PrettyPrintRule(rule)
SIPRepresentation(rule.sip)
print(pred, magicPred)
_body = []
additionalRules = []
for idxSip, (N, x) in enumerate(inArcs):
newPred = pred.clone()
SetOp(newPred, URIRef('%s_label_%s' % (newPred.op, idxSip)))
ruleBody = And(buildMagicBody(
N,
prevPreds,
rule.formula.head,
derivedPreds))
additionalRules.append(Rule(Clause(ruleBody, newPred)))
_body.extend(newPred)
# _body.extend(ruleBody)
additionalRules.append(Rule(Clause(And(_body), magicPred)))
newRules.extend(additionalRules)
for i in additionalRules:
print(i)
raise NotImplementedError()
else:
for idxSip, (N, x) in enumerate(inArcs):
ruleBody = And(buildMagicBody(
N,
prevPreds,
rule.formula.head,
derivedPreds,
noMagic))
newRule = Rule(Clause(ruleBody, magicPred))
newRules.append(newRule)
magicPredicates.add(magicPred)
# Modify rules
# we modify the original rule by inserting
# occurrences of the magic predicates corresponding
# to the derived predicates of the body and to the head
# If there are no bound arguments in the head, we don't modify the rule
idxIncrement = 0
newRule = copy.deepcopy(rule)
for idx, (magicPred, origPred) in list(magicPositions.items()):
newRule.formula.body.formulae.insert(idx + idxIncrement, magicPred)
idxIncrement += 1
if 'b' in rule.formula.head.adornment and GetOp(rule.formula.head) not in noMagic:
headMagicPred = rule.formula.head.makeMagicPred()
if isinstance(newRule.formula.body, Uniterm):
newRule.formula.body = And([headMagicPred, newRule.formula.body])
else:
newRule.formula.body.formulae.insert(0, headMagicPred)
newRules.append(newRule)
if not newRules:
newRules.extend(AdditionalRules(factGraph))
for rule in newRules:
if rule.formula.body:
yield rule
def specializeAdornedRuleset(self, debug=False):
"""
Specialization is applied to the BFP meta-interpreter with respect to the
rules of the object program. For each rule of the meta-interpreter
that includes a premise referring to a rule of the object program, one
specialized version is created for each rule of the object program.
"""
rules = set()
for idx, rule in enumerate(self.rules):
label = BFP_RULE[str(idx + 1)]
ruleBodyLen = len(list(iterCondition(rule.formula.body)))
# if rule.isSecondOrder():
# if debug:
# print "Skipping second order rule (%s): %s"%(idx+1,rule)
# continue
if debug:
print "\t%s. %s" % (idx + 1, rule)
for _sip in SIPRepresentation(rule.sip):
print "\t\t", _sip
newRule1 = self.rule1(rule, label, ruleBodyLen)
self.bfpLookup[("a", idx + 1)] = newRule1
rules.add(newRule1)
newRule2 = self.rule2(rule, label, ruleBodyLen)
self.bfpLookup[("b", idx + 1)] = newRule2
rules.add(newRule2)
# indicate no skipping is ongoing
skipMDBQCount = -1
mDBConjFront = None
# _len = len(rule.formula.body)
for bodyIdx, bodyLiteral in enumerate(iterCondition(rule.formula.body)):
bodyPredSymbol = GetOp(bodyLiteral)
if skipMDBQCount > 0:
skipMDBQCount -= 1
continue
elif skipMDBQCount == 0:
# finished skipping maximal db conjuncts, mark end of skipped
# conjuncts and indicate that no skipping is ongoing
self.maxEDBFront2End[mDBConjFront] = (idx + 1, bodyIdx)
mDBConjFront = None
skipMDBQCount = -1
evaluateTerm = Uniterm(BFP_NS.evaluate, [label, Literal(bodyIdx + 1)], newNss=self.namespaces)
priorEvaluateTerm = Uniterm(BFP_NS.evaluate, [label, Literal(bodyIdx)], newNss=self.namespaces)
conj = EDBQueryFromBodyIterator(
self.factGraph, rule.formula.body.formulae[bodyIdx:], self.derivedPredicates, self.hybridPredicates
)
if self.pushDownMDBQ and conj:
# There is a maximal db conjunction, indicate skipping of rules involving
# conjuncts
mDBConjFront = (idx + 1, bodyIdx + 1)
if len(conj) > 1:
skipMDBQCount = len(conj) - 1
self.pushDownQueries[mDBConjFront] = EDBQuery(
[copy.deepcopy(item) for item in conj], self.factGraph
)
else:
self.maxEDBFront2End[mDBConjFront] = (idx + 1, bodyIdx + 1)
if debug and skipMDBQCount > 0:
print "maximal db query: ", self.pushDownQueries[mDBConjFront]
print "skipping %s literals, starting from the %s" % (bodyIdx + 1, skipMDBQCount)
if len(conj) + bodyIdx == len(rule.formula.body):
# maximal db conjunction takes up rest of body
# tokens should go into (k,n) - where n is the body length
self.maxEDBFront2End[mDBConjFront] = (idx + 1, len(rule.formula.body))
if (
not self.pushDownMDBQ
or (
(bodyPredSymbol in FILTERS and len(conj) == 1)
or (
bodyPredSymbol in self.derivedPredicates
or IsHybridPredicate(bodyLiteral, self.hybridPredicates)
) # and
# bodyPredSymbol not in self.hybridPredicates) or (
# bodyPredSymbol not in FILTERS and bodyIdx+1 == _len)
)
) and skipMDBQCount in (1, -1):
# Either not pushing down or:
# 1. It is a lone filter
# 2. It is a derived predicate (need continuation BFP rule)
# evaluate(ruleNo,j+1,X) :- evaluate(ruleNo,j,X) bodyLiteral(..)
newRule = self.makeAdornedRule(And([priorEvaluateTerm, bodyLiteral]), evaluateTerm)
self.bfpLookup[("c", idx + 1, bodyIdx + 1)] = newRule
rules.add(newRule)
elif bodyPredSymbol in FILTERS and len(conj) > 2:
raise NotImplementedError(repr(rule))
if bodyPredSymbol not in FILTERS:
# query_Literal(x0,...,xj) :- evaluate(ruleNo,j,X)
# OpenQuery(query_Literal)
newRule = self.makeAdornedRule(priorEvaluateTerm, self.makeDerivedQueryPredicate(bodyLiteral))
self.bfpLookup[("d", idx + 1, bodyIdx + 1)] = newRule
rules.add(newRule)
return rules
def MagicSetTransformation(factGraph,
rules,
GOALS,
derivedPreds=None,
strictCheck=DDL_STRICTNESS_FALLBACK_DERIVED,
noMagic=None,
defaultPredicates=None):
"""
Takes a goal and a ruleset and returns an iterator
over the rulest that corresponds to the magic set
transformation:
"""
noMagic = noMagic and noMagic or []
magicPredicates = set()
# replacement = {}
adornedProgram = SetupDDLAndAdornProgram(
factGraph,
rules,
GOALS,
derivedPreds=derivedPreds,
strictCheck=strictCheck,
defaultPredicates=defaultPredicates)
newRules = []
for rule in adornedProgram:
if rule.isSecondOrder():
import warnings
warnings.warn("Second order rule no supported by GMS: %s" % rule,
RuntimeWarning)
magicPositions = {}
#Generate magic rules
for idx, pred in enumerate(iterCondition(rule.formula.body)):
# magicBody = []
if isinstance(pred,
AdornedUniTerm): # and pred not in magicPredicates:
# For each rule r in Pad, and for each occurrence of an adorned
# predicate p a in its body, we generate a magic rule defining magic_p a
prevPreds = [
item for _idx, item in enumerate(rule.formula.body)
if _idx < idx
]
if 'b' not in pred.adornment:
import warnings
warnings.warn(
"adorned predicate w/out any bound arguments (%s in %s)"
% (pred, rule.formula), RuntimeWarning)
if GetOp(pred) not in noMagic:
magicPred = pred.makeMagicPred()
magicPositions[idx] = (magicPred, pred)
inArcs = [(N, x) for (
N,
x) in IncomingSIPArcs(rule.sip, getOccurrenceId(pred))
if not set(x).difference(GetArgs(pred))]
if len(inArcs) > 1:
# If there are several arcs entering qi, we define the
# magic rule defining magic_qi in two steps. First,
# for each arc Nj --> qi with label cj , we define a
# rule with head label_qi_j(cj ). The body of the rule
# is the same as the body of the magic rule in the
# case where there is a single arc entering qi
# (described above). Then the magic rule is defined as
# follows. The head is magic_q(0). The body contains
# label_qi_j(cj) for all j (that is, for all arcs
# entering qi ).
#
# We combine all incoming arcs into a single list of
# (body) conditions for the magic set
PrettyPrintRule(rule)
SIPRepresentation(rule.sip)
print(pred, magicPred)
_body = []
additionalRules = []
for idxSip, (N, x) in enumerate(inArcs):
newPred = pred.clone()
SetOp(newPred,
URIRef('%s_label_%s' % (newPred.op, idxSip)))
ruleBody = And(
buildMagicBody(N, prevPreds, rule.formula.head,
derivedPreds))
additionalRules.append(
Rule(Clause(ruleBody, newPred)))
_body.extend(newPred)
# _body.extend(ruleBody)
additionalRules.append(
Rule(Clause(And(_body), magicPred)))
newRules.extend(additionalRules)
for i in additionalRules:
print(i)
raise NotImplementedError()
else:
for idxSip, (N, x) in enumerate(inArcs):
ruleBody = And(
buildMagicBody(N, prevPreds, rule.formula.head,
derivedPreds, noMagic))
newRule = Rule(Clause(ruleBody, magicPred))
newRules.append(newRule)
magicPredicates.add(magicPred)
# Modify rules
# we modify the original rule by inserting
# occurrences of the magic predicates corresponding
# to the derived predicates of the body and to the head
# If there are no bound arguments in the head, we don't modify the rule
idxIncrement = 0
newRule = copy.deepcopy(rule)
for idx, (magicPred, origPred) in list(magicPositions.items()):
newRule.formula.body.formulae.insert(idx + idxIncrement, magicPred)
idxIncrement += 1
if 'b' in rule.formula.head.adornment and GetOp(
rule.formula.head) not in noMagic:
headMagicPred = rule.formula.head.makeMagicPred()
if isinstance(newRule.formula.body, Uniterm):
newRule.formula.body = And(
[headMagicPred, newRule.formula.body])
else:
newRule.formula.body.formulae.insert(0, headMagicPred)
newRules.append(newRule)
if not newRules:
newRules.extend(AdditionalRules(factGraph))
for rule in newRules:
if rule.formula.body:
yield rule
def AdornRule(derivedPreds,
clause,
newHead,
ignoreUnboundDPreds=False,
hybridPreds2Replace=None):
"""
Adorns a horn clause using the given new head and list of
derived predicates
"""
assert len(list(iterCondition(clause.head))) == 1
hybridPreds2Replace = hybridPreds2Replace or []
adornedHead = AdornedUniTerm(clause.head, newHead.adornment)
sip = BuildNaturalSIP(clause,
derivedPreds,
adornedHead,
hybridPreds2Replace=hybridPreds2Replace,
ignoreUnboundDPreds=ignoreUnboundDPreds)
bodyPredReplace = {}
def adornment(arg, headArc, x):
if headArc:
# Sip arc from head
# don't mark bound if query has no bound/distinguished terms
return (arg in x and
arg in adornedHead.getDistinguishedVariables(True)) \
and 'b' or 'f'
else:
return arg in x and 'b' or 'f'
for literal in iterCondition(sip.sipOrder):
op = GetOp(literal)
args = GetArgs(literal)
if op in derivedPreds or (op in hybridPreds2Replace
if hybridPreds2Replace else False):
for N, x in IncomingSIPArcs(sip, getOccurrenceId(literal)):
headArc = len(N) == 1 and N[0] == GetOp(newHead)
if not set(x).difference(args):
# A binding
# for q is useful, however, only if it is a binding for an argument of q.
bodyPredReplace[literal] = AdornedUniTerm(
NormalizeUniterm(literal),
[adornment(arg, headArc, x) for arg in args],
literal.naf)
# For a predicate occurrence with no incoming
# arc, the adornment contains only f. For our purposes here,
# we do not distinguish between a predicate with such an
# adornment and an unadorned predicate (we do in order to support open queries)
if literal not in bodyPredReplace and ignoreUnboundDPreds:
bodyPredReplace[literal] = AdornedUniTerm(
NormalizeUniterm(literal),
['f' for arg in GetArgs(literal)], literal.naf)
if hybridPreds2Replace:
atomPred = GetOp(adornedHead)
if atomPred in hybridPreds2Replace:
adornedHead.setOperator(URIRef(atomPred + u'_derived'))
for bodAtom in [
bodyPredReplace.get(p, p) for p in iterCondition(sip.sipOrder)
]:
bodyPred = GetOp(bodAtom)
if bodyPred in hybridPreds2Replace:
bodAtom.setOperator(URIRef(bodyPred + u'_derived'))
rule = AdornedRule(
Clause(
And([
bodyPredReplace.get(p, p) for p in iterCondition(sip.sipOrder)
]), adornedHead))
rule.sip = sip
return rule
def AdornProgram(factGraph,
rs,
goals,
derivedPreds=None,
ignoreUnboundDPreds=False,
hybridPreds2Replace=None):
"""
The process starts from the given query. The query determines bindings for q, and we replace
q by an adorned version, in which precisely the positions bound in the query are designated as
bound, say q e . In general, we have a collection of adorned predicates, and as each one is processed,
we will mark it, so that it will not be processed again. If p a is an unmarked adorned
predicate, then for each rule that has p in its head, we generate an adorned version for the rule
and add it to Pad; then p is marked as processed.
The adorned version of a rule contains additional
adorned predicates, and these are added to the collection, unless they already appear
there. The process terminates when no unmarked adorned predicates are left.
"""
from FuXi.DLP import LloydToporTransformation
from collections import deque
goalDict = {}
hybridPreds2Replace = hybridPreds2Replace or []
adornedPredicateCollection = set()
for goal, nsBindings in NormalizeGoals(goals):
adornedPredicateCollection.add(AdornLiteral(goal, nsBindings))
if not derivedPreds:
derivedPreds = list(DerivedPredicateIterator(factGraph, rs))
def unprocessedPreds(aPredCol):
rt = []
for p in aPredCol:
if not p.marked:
rt.append(p)
if p not in goalDict:
goalDict.setdefault(GetOp(p), set()).add(p)
return rt
toDo = deque(unprocessedPreds(adornedPredicateCollection))
adornedProgram = set()
while len(toDo):
term = toDo.popleft()
#check if there is a rule with term as its head
for rule in rs:
for clause in LloydToporTransformation(rule.formula):
head = isinstance(
clause.head, Exists) and clause.head.formula or clause.head
# headPredicate = GetOp(head)
if compareAdornedPredToRuleHead(term, head,
hybridPreds2Replace):
#for each rule that has p in its head, we generate an adorned version for the rule
adornedRule = AdornRule(
derivedPreds,
clause,
term,
ignoreUnboundDPreds=ignoreUnboundDPreds,
hybridPreds2Replace=hybridPreds2Replace)
adornedProgram.add(adornedRule)
#The adorned version of a rule contains additional adorned
#predicates, and these are added
for pred in iterCondition(adornedRule.formula.body):
if isinstance(pred, N3Builtin):
aPred = pred
else:
aPred = not isinstance(pred, AdornedUniTerm) and \
AdornLiteral(pred.toRDFTuple(),
nsBindings,
pred.naf) or pred
op = GetOp(pred)
if (op in derivedPreds or
(op in hybridPreds2Replace
if hybridPreds2Replace else False)
) and aPred not in adornedPredicateCollection:
adornedPredicateCollection.add(aPred)
term.marked = True
toDo.extendleft(unprocessedPreds(adornedPredicateCollection))
factGraph.queryAtoms = goalDict
return adornedProgram
def DerivedPredicateIterator(factsOrBasePreds,
ruleset,
strict=DDL_STRICTNESS_FALLBACK_DERIVED,
defaultPredicates=None):
if not defaultPredicates:
defaultPredicates = [], []
defaultBasePreds, defaultDerivedPreds = defaultPredicates
basePreds = [
GetOp(buildUniTerm(fact)) for fact in factsOrBasePreds
if fact[1] != LOG.implies
]
processed = {True: set(), False: set()}
derivedPreds = set()
uncertainPreds = set()
ruleBodyPreds = set()
ruleHeads = set()
for rule in ruleset:
if rule.formula.body:
for idx, term in enumerate(
itertools.chain(iterCondition(rule.formula.head),
iterCondition(rule.formula.body))):
# iterate over terms from head to end of body
op = GetOp(term)
if op not in processed[idx > 0]:
# not processed before
if idx > 0:
# body literal
ruleBodyPreds.add(op)
else:
# head literal
ruleHeads.add(op)
if strict in DDL_MUST_CHECK and \
not (op not in basePreds or idx > 0):
# checking DDL well formedness and
# op is a base predicate *and* a head literal (derived)
if strict in DDL_FALLBACK:
mark = strict == DDL_STRICTNESS_FALLBACK_DERIVED and \
'derived' or 'base'
if strict == DDL_STRICTNESS_FALLBACK_DERIVED and \
op not in defaultBasePreds:
# a clashing predicate is marked as derived due
# to level of strictness
derivedPreds.add(op)
elif strict == DDL_STRICTNESS_FALLBACK_BASE and \
op not in defaultDerivedPreds:
# a clashing predicate is marked as base dur
# to level of strictness
defaultBasePreds.append(op)
import warnings
warnings.warn(
"predicate symbol of %s is in both IDB and EDB. Marking as %s"
% (term, mark))
else:
raise SyntaxError(
"%s is a member of a derived predicate and a base predicate."
% term)
if op in basePreds:
# base predicates are marked for later validation
uncertainPreds.add(op)
else:
if idx == 0 and not isinstance(op, Variable):
# head literal with proper predicate symbol
# identify as a derived predicate
derivedPreds.add(op)
elif not isinstance(op, Variable):
# body literal with proper predicate symbol
# mark for later validation
uncertainPreds.add(op)
processed[idx > 0].add(op)
for pred in uncertainPreds:
# for each predicate marked as 'uncertain'
# do further checking
if (pred not in ruleBodyPreds and not isinstance(pred, Variable)) or\
pred in ruleHeads:
# pred is not in a body literal and is a proper predicate symbol
# or it is a rule head -> mark as a derived predicate
derivedPreds.add(pred)
for pred in derivedPreds:
if not pred in defaultBasePreds:
yield pred
def SetupDDLAndAdornProgram(factGraph,
rules,
GOALS,
derivedPreds=None,
strictCheck=DDL_STRICTNESS_FALLBACK_DERIVED,
defaultPredicates=None,
ignoreUnboundDPreds=False,
hybridPreds2Replace=None):
if not defaultPredicates:
defaultPredicates = [], []
# _dPredsProvided = bool(derivedPreds)
if not derivedPreds:
_derivedPreds = DerivedPredicateIterator(
factGraph, rules,
strict=strictCheck,
defaultPredicates=defaultPredicates)
if not isinstance(derivedPreds, (set, list)):
derivedPreds = list(_derivedPreds)
else:
derivedPreds.extend(_derivedPreds)
hybridPreds2Replace = hybridPreds2Replace or []
adornedProgram = AdornProgram(
factGraph,
rules,
GOALS,
derivedPreds,
ignoreUnboundDPreds,
hybridPreds2Replace=hybridPreds2Replace)
if adornedProgram != set([]):
rt = reduce(lambda l, r: l + r,
[list(iterCondition(clause.formula.body))
for clause in adornedProgram])
else:
rt = set()
for hybridPred, adornment in [(t, a)
for t, a in set(
[(URIRef(GetOp(term).split('_derived')[0]
) if GetOp(term).find('_derived') + 1 else GetOp(term),
''.join(term.adornment)
) for term in rt if isinstance(term, AdornedUniTerm)])
if t in hybridPreds2Replace]:
#If there are hybrid predicates, add rules that derived their IDB counterpart
#using information from the adorned queries to determine appropriate arity
#and adornment
hybridPred = URIRef(hybridPred)
hPred = URIRef(hybridPred + u'_derived')
if len(adornment) == 1:
# p_derived^{a}(X) :- p(X)
body = BuildUnitermFromTuple(
(Variable('X'),
RDF.type,
hybridPred))
head = BuildUnitermFromTuple(
(Variable('X'),
RDF.type,
hPred))
else:
# p_derived^{a}(X, Y) :- p(X, Y)
body = BuildUnitermFromTuple(
(Variable('X'),
hybridPred,
Variable('Y')))
head = BuildUnitermFromTuple(
(Variable('X'),
hPred,
Variable('Y')))
_head = AdornedUniTerm(head, list(adornment))
rule = AdornedRule(Clause(And([body]), _head.clone()))
rule.sip = Graph()
adornedProgram.add(rule)
if factGraph is not None:
factGraph.adornedProgram = adornedProgram
return adornedProgram
def __init__(self, formulae=None, n3Rules=None, nsMapping=None):
from FuXi.Rete.RuleStore import N3Builtin
self.nsMapping = nsMapping and nsMapping or {}
self.formulae = formulae and formulae or []
if n3Rules:
from FuXi.DLP import breadth_first
# Convert a N3 abstract model (parsed from N3) into a RIF BLD
for lhs, rhs in n3Rules:
allVars = set()
for ruleCondition in [lhs, rhs]:
for stmt in ruleCondition:
if isinstance(stmt, N3Builtin):
ExternalFunction(stmt, newNss=self.nsMapping)
# print(stmt)
# raise
allVars.update([
term for term in stmt
if isinstance(term, (BNode, Variable))
])
body = [
isinstance(term, N3Builtin) and term
or Uniterm(list(term)[1],
[list(term)[0], list(term)[-1]],
newNss=nsMapping) for term in lhs
]
body = len(body) == 1 and body[0] or And(body)
head = [
Uniterm(p, [s, o], newNss=nsMapping) for s, p, o in rhs
]
head = len(head) == 1 and head[0] or And(head)
# first we identify body variables
bodyVars = set(
reduce(lambda x, y: x + y, [
list(extractVariables(i, existential=False))
for i in iterCondition(body)
]))
# then we identify head variables
headVars = set(
reduce(lambda x, y: x + y, [
list(extractVariables(i, existential=False))
for i in iterCondition(head)
]))
# then we identify those variables that should (or should not)
# be converted to skolem terms
updateDict = dict([(var, BNode()) for var in headVars
if var not in bodyVars])
for uniTerm in iterCondition(head):
def updateUniterm(uterm):
newArg = [updateDict.get(i, i) for i in uniTerm.arg]
uniTerm.arg = newArg
if isinstance(uniTerm, Uniterm):
updateUniterm(uniTerm)
else:
for u in uniTerm:
updateUniterm(u)
exist = [
list(extractVariables(i)) for i in breadth_first(head)
]
e = Exists(formula=head,
declare=set(reduce(lambda x, y: x + y, exist, [])))
if reduce(lambda x, y: x + y, exist):
head = e
assert e.declare, exist
self.formulae.append(Rule(Clause(body, head), declare=allVars))
def createTopDownReteNetwork(self, debug=False):
"""
Uses the specialized BFP meta-interpretation rules to build a RETE-UL decision
network that is modified to support the propagation of bindings from the evaluate
predicates into a supplimental magic set sip strategy and the generation of subqueries.
The end result is a bottom-up simulation of SLD resolution with complete, sound, and safe
memoization in the face of recursion
"""
for rule in set(self.specializeAdornedRuleset(debug)):
self.metaRule2Network[rule] = self.metaInterpNetwork.buildNetworkFromClause(rule)
if debug:
sortedBFPRules = [
str("%s : %s") % (key, self.bfpLookup[key])
for key in sorted(self.bfpLookup, key=lambda items: str(items[1]) + items[0])
]
for _ruleStr in sortedBFPRules:
print _ruleStr
self.evalHash = {}
self.actionHash = {}
evalVars = {}
self.productions = {}
for idx, rule in enumerate(self.rules):
if rule in self.discardedRules:
continue
# label = BFP_RULE[str(idx+1)]
conjunctLength = len(list(iterCondition(rule.formula.body)))
# Rule a^k
# p(x0,...,xn) :- And(query_p(x0,...,xn) evaluate(ruleNo,n,X))
currentPattern = HashablePatternList([(BFP_RULE[str(idx + 1)], BFP_NS.evaluate, Literal(conjunctLength))])
assert rule.declare
# Find alpha node associated with evaluate condition
node = self.metaInterpNetwork.nodes[currentPattern]
# evaluate(k,n,X) is a condition in only 1 bfp rule
assert len(node.descendentBetaNodes) == 1
bNode = first(node.descendentBetaNodes)
assert bNode.leftNode.aPassThru
assert len(bNode.consequent) == 1
newEvalMemory = SetupEvaluationBetaNode(bNode, rule, self.metaInterpNetwork)
self.evalHash.setdefault((idx + 1, conjunctLength), []).append(newEvalMemory)
if GetOp(rule.formula.head) == GetOp(self.goal):
# This rule matches a goal, add a solution collecting action
goalSolutionAction = GoalSolutionAction(self, rule.formula.head.getVarMapping(self.goal))
bNode.executeActions[rule.formula.head.toRDFTuple()] = (False, goalSolutionAction)
self.productions.setdefault(GetOp(rule.formula.head), []).append((idx, bNode))
# Rule b^k
# evaluate(ruleNo,0,X) :- query_p(x0,...,xn)
_rule = self.bfpLookup[("b", idx + 1)]
# alpha node associated with query predicate for head of original rule
_bodyAlphaNode = self.metaInterpNetwork.nodes[HashablePatternList([_rule.formula.body.toRDFTuple()])]
assert len(_bodyAlphaNode.descendentBetaNodes) == 1
tNode = first(_bodyAlphaNode.descendentBetaNodes)
self.actionHash.setdefault((idx + 1, 0), set()).add(tNode)
# V_{0} = vars(query_p(..))
headQueryPred = list(iterCondition(_rule.formula.body))[0]
try:
evalVars[(idx + 1, 0)] = list(headQueryPred.terms)
except IndexError:
raise
self.discardedRules.add(rule)
continue
self.specializeConjuncts(rule, idx, evalVars)
for (ruleNo, bodyIdx), tNodes in self.actionHash.items():
# Attach evaluate action to p-node that propagates
# token to beta memory associated with evaluate(ruleNo,bodyIdx)
executeAction = EvaluateExecution((ruleNo, bodyIdx), self, tNodes)
evaluationStmt = (BFP_RULE[str(ruleNo)], BFP_NS.evaluate, Literal(bodyIdx))
for tNode in tNodes:
tNode.executeActions[evaluationStmt] = (True, executeAction)
# executeAction = EvaluateExecution(evalHash,(idx+1,bodyIdx+1),self,termNodeCk)
# assert len(termNodeCk.consequent)==1
# termNodeCk.executeAction = (None,executeAction)
# Fix join variables for BetaNodes involving evaluate predicates
for idx, rule in enumerate(self.rules):
if rule in self.discardedRules:
continue
# Rule a^k
# p(x0,...,xn) :- And(query_p(x0,...,xn) evaluate(ruleNo,n,X))
# Join vars = vars(query_p) AND V_{n}
headQueryPred = self.bfpLookup[("b", idx + 1)].formula.body
ruleBodyLen = len(list(iterCondition(rule.formula.body)))
termNode = first(self.evalHash[idx + 1, ruleBodyLen]).successor
termNode.commonVariables = list(
set(evalVars[(idx + 1, ruleBodyLen)]).intersection(GetVariables(headQueryPred, secondOrder=True))
)
skipMDBQCount = 0
for bodyIdx, bodyLiteral in enumerate(iterCondition(rule.formula.body)):
if skipMDBQCount > 0:
skipMDBQCount -= 1
continue
if (idx + 1, bodyIdx + 1) in self.actionHash:
# Rule c^k
# evaluate(ruleNo,j+1,X) :- And(evaluate(ruleNo,j,X) Literal(x0,...,xj))
# Join vars = vars(Literal) AND V_{j}
termNode2 = self.actionHash[(idx + 1, bodyIdx + 1)]
assert len(termNode2) == 1
termNode2 = first(termNode2)
termNode2.commonVariables = list(
set(evalVars[(idx + 1, bodyIdx)]).intersection(GetVariables(bodyLiteral, secondOrder=True))
)
if (idx + 1, bodyIdx + 1) in self.maxEDBFront2End:
endIdx = self.maxEDBFront2End[(idx + 1, bodyIdx + 1)][-1]
skipMDBQCount = endIdx - bodyIdx - 1
def checkNetworkWellformedness(self):
for key, rule in self.bfpLookup.items():
if len(key) == 2:
ruleType, ruleIdx = key
bodyIdx = None
else:
ruleType, ruleIdx, bodyIdx = key
termNode = self.metaRule2Network[rule]
headTuple = rule.formula.head.toRDFTuple()
# p(..) :- q_1(..), q_2(..), ..., q_n(..)
if GetOp(rule.formula.head) == BFP_NS.evaluate:
# evaluate(..,..) :-
override, executeFn = termNode.executeActions.get(headTuple, (None, None))
assert override and isinstance(executeFn, EvaluateExecution), termNode
assert executeFn.ruleNo == ruleIdx
assert executeFn.bodyIdx == headTuple[-1]
assert bodyIdx is None or executeFn.bodyIdx == int(bodyIdx), bodyIdx
if isinstance(rule.formula.body, And):
# c^{ruleIdx}_{bodyIdx}
# evaluate(..,j+1) :- evaluate(..,j), q_{j+1}(..)
# @@ force check builtins or derived predicate c rules
if isinstance(termNode.leftNode, AlphaNode):
# alphaNode = termNode.leftNode
betaNode = termNode.rightNode
assert isinstance(termNode.memories[RIGHT_MEMORY], EvaluateConjunctiveQueryMemory), termNode
assert isinstance(betaNode, BetaNode)
elif not termNode.fedByBuiltin:
# alphaNode = termNode.rightNode
betaNode = termNode.leftNode
assert (
isinstance(termNode.memories[LEFT_MEMORY], EvaluateConjunctiveQueryMemory)
or self.evalHash[(ruleIdx, bodyIdx - 1)][0].successor != termNode
), termNode
assert isinstance(betaNode, BetaNode)
else:
# b^{ruleIdx}
# evaluate(..,j+1) :- query-p(..)
assert termNode.aPassThru
elif isinstance(rule.formula.body, And):
# a^{ruleIdx}
# p(..) :- query-p(..), evaluate(..,n)
assert isinstance(termNode.memories[RIGHT_MEMORY], EvaluateConjunctiveQueryMemory)
else:
# d^{ruleIdx}_{bodyIdx}
# query-q_{j+1}(..) :- evaluate(..,j)
queryLiteral = list(iterCondition(self.rules[ruleIdx - 1].formula.body))[bodyIdx - 1]
if isinstance(queryLiteral, N3Builtin):
headTuple = queryLiteral.toRDFTuple()
executeKind = EvaluateExecution
else:
executeKind = QueryExecution
if GetOp(queryLiteral) not in self.derivedPredicates:
override, executeFn = termNode.executeActions.get(headTuple, (None, None))
assert override and isinstance(executeFn, executeKind), "%s %s %s" % (
termNode.consequent,
termNode.executeActions,
termNode,
)
if not isinstance(queryLiteral, N3Builtin):
assert executeFn.queryLiteral == queryLiteral
# self.bfp.evalHash[self.conjoinedTokenMem]
assert executeFn.conjoinedTokenMem[0] == int(ruleIdx), "%s %s %s" % (termNode, executeFn, key)
def AdornRule(derivedPreds,
clause,
newHead,
ignoreUnboundDPreds=False,
hybridPreds2Replace=None):
"""
Adorns a horn clause using the given new head and list of
derived predicates
"""
assert len(list(iterCondition(clause.head))) == 1
hybridPreds2Replace = hybridPreds2Replace or []
adornedHead = AdornedUniTerm(clause.head,
newHead.adornment)
sip = BuildNaturalSIP(clause,
derivedPreds,
adornedHead,
hybridPreds2Replace=hybridPreds2Replace,
ignoreUnboundDPreds=ignoreUnboundDPreds)
bodyPredReplace = {}
def adornment(arg, headArc, x):
if headArc:
# Sip arc from head
# don't mark bound if query has no bound/distinguished terms
return (arg in x and
arg in adornedHead.getDistinguishedVariables(True)) \
and 'b' or 'f'
else:
return arg in x and 'b' or 'f'
for literal in iterCondition(sip.sipOrder):
op = GetOp(literal)
args = GetArgs(literal)
if op in derivedPreds or (op in hybridPreds2Replace if hybridPreds2Replace else False):
for N, x in IncomingSIPArcs(sip, getOccurrenceId(literal)):
headArc = len(N) == 1 and N[0] == GetOp(newHead)
if not set(x).difference(args):
# A binding
# for q is useful, however, only if it is a binding for an argument of q.
bodyPredReplace[literal] = AdornedUniTerm(NormalizeUniterm(literal),
[adornment(arg, headArc, x) for arg in args], literal.naf)
# For a predicate occurrence with no incoming
# arc, the adornment contains only f. For our purposes here,
# we do not distinguish between a predicate with such an
# adornment and an unadorned predicate (we do in order to support open queries)
if literal not in bodyPredReplace and ignoreUnboundDPreds:
bodyPredReplace[literal] = AdornedUniTerm(NormalizeUniterm(literal),
['f' for arg in GetArgs(literal)], literal.naf)
if hybridPreds2Replace:
atomPred = GetOp(adornedHead)
if atomPred in hybridPreds2Replace:
adornedHead.setOperator(URIRef(atomPred + u'_derived'))
for bodAtom in [bodyPredReplace.get(p, p)
for p in iterCondition(sip.sipOrder)]:
bodyPred = GetOp(bodAtom)
if bodyPred in hybridPreds2Replace:
bodAtom.setOperator(URIRef(bodyPred + u'_derived'))
rule = AdornedRule(Clause(And([bodyPredReplace.get(p, p)
for p in iterCondition(sip.sipOrder)]),
adornedHead))
rule.sip = sip
return rule
def specializeConjuncts(self, rule, idx, evalVars):
"""
Extends the (vanilla) meta-interpreter for magic set and alexander rewriting
sip strategies with capabilities for collapsing chains of extensional
predicate queries (frames whose attributes are in EDB and externally-defined
predicates) into a single SPARQL query
"""
# _len = len(rule.formula.body)
body = list(iterCondition(rule.formula.body))
skipMDBQCount = 0
for bodyIdx, bodyLiteral in enumerate(body):
conjunct = []
# V_{j} = V_{j-1} UNION vars(Literal(..)) where j <> 0
evalVars[(idx + 1, bodyIdx + 1)] = (
list(GetVariables(bodyLiteral, secondOrder=True)) + evalVars[(idx + 1, bodyIdx)]
)
if skipMDBQCount > 0:
skipMDBQCount -= 1
continue
# remainingBodyList = body[bodyIdx+1:] if bodyIdx+1<_len else []
conjunct = EDBQueryFromBodyIterator(
self.factGraph, rule.formula.body.formulae[bodyIdx:], self.derivedPredicates, self.hybridPredicates
)
lazyBaseConjunct = self.pushDownMDBQ and conjunct
pattern = HashablePatternList([(BFP_RULE[str(idx + 1)], BFP_NS.evaluate, Literal(bodyIdx))])
pattern2 = HashablePatternList(
[None, (BFP_RULE[str(idx + 1)], BFP_NS.evaluate, Literal(bodyIdx)), bodyLiteral.toRDFTuple()]
)
aNodeDk = self.metaInterpNetwork.nodes[pattern]
# Rule d^k
# query_Literal(x0,...,xj) :- evaluate(ruleNo,j,X)
# query invokation
tNode = first(aNodeDk.descendentBetaNodes)
assert len(aNodeDk.descendentBetaNodes) == 1
newEvalMemory = SetupEvaluationBetaNode(tNode, rule, self.metaInterpNetwork)
isBase = bodyLiteral.adornment is None if isinstance(bodyLiteral, AdornedUniTerm) else True
if isinstance(bodyLiteral, N3Builtin):
if aNodeDk in self.metaInterpNetwork.alphaNodes:
self.metaInterpNetwork.alphaNodes.remove(aNodeDk)
# evalTerm = (BFP_RULE[str(idx+1)],BFP_NS.evaluate,Literal(bodyIdx))
del aNodeDk
executeAction = EvaluateExecution((idx + 1, bodyIdx), self, [])
builtInNode = self.metaRule2Network[self.bfpLookup[("c", idx + 1, bodyIdx + 1)]]
assert isinstance(builtInNode.rightNode, BuiltInAlphaNode)
builtInNode.executeActions[bodyLiteral.toRDFTuple()] = (True, executeAction)
# We bypass d^k, so when evaluate(ruleNo,j,X) is inferred
# it is added to left memory of pNode associated with c^k
self.evalHash.setdefault((idx + 1, bodyIdx), []).append(builtInNode.memories[LEFT_MEMORY])
self.actionHash.setdefault((idx + 1, bodyIdx + 1), set()).add(builtInNode)
elif conjunct and (isBase or lazyBaseConjunct):
singleConj = EDBQuery([copy.deepcopy(item) for item in conjunct], self.factGraph)
matchingTriple = first(tNode.consequent)
assert len(tNode.consequent) == 1
newAction = QueryExecution(
self,
bodyLiteral,
conjoinedTokenMem=self.maxEDBFront2End[(idx + 1, bodyIdx + 1)] if lazyBaseConjunct else None,
edbConj=conjunct if lazyBaseConjunct else singleConj,
)
self.evalHash.setdefault((idx + 1, bodyIdx), []).append(newEvalMemory)
# If the body predicate is a 2nd order predicate, we don't infer the
# second order query predicate (since it will trigger a query into
# the EDB and thus there is no need to trigger subsequent rules)
tNode.executeActions[matchingTriple] = (True, newAction)
else:
self.evalHash.setdefault((idx + 1, bodyIdx), []).append(newEvalMemory)
if IsHybridPredicate(bodyLiteral, self.hybridPredicates) or (
(GetOp(bodyLiteral) in self.derivedPredicates) and not (isBase and len(conjunct) > 1)
):
# if pattern2 not in self.metaInterpNetwork.nodes: import pdb;pdb.set_trace()
assert pattern2 in self.metaInterpNetwork.nodes
termNodeCk = self.metaInterpNetwork.nodes[pattern2]
# Rule c^k
# evaluate(ruleNo,j+1,X) :- evaluate(ruleNo,j,X), bodyLiteral
self.actionHash.setdefault((idx + 1, bodyIdx + 1), set()).add(termNodeCk)
assert isinstance(termNodeCk.rightNode, AlphaNode)
termNodeCk.leftVariables = set(evalVars[(idx + 1, bodyIdx)])
termNodeCk.rightVariables = set(GetVariables(bodyLiteral, secondOrder=True))
termNodeCk.commonVariables = termNodeCk.rightVariables.intersection(termNodeCk.leftVariables)
if lazyBaseConjunct and len(conjunct) > 1:
endIdx = self.maxEDBFront2End[(idx + 1, bodyIdx + 1)][-1]
skipMDBQCount = endIdx - bodyIdx - 1
def DerivedPredicateIterator(factsOrBasePreds,
ruleset,
strict=DDL_STRICTNESS_FALLBACK_DERIVED,
defaultPredicates=None):
if not defaultPredicates:
defaultPredicates = [], []
defaultBasePreds, defaultDerivedPreds = defaultPredicates
basePreds = [GetOp(buildUniTerm(fact))
for fact in factsOrBasePreds
if fact[1] != LOG.implies]
processed = {True: set(), False: set()}
derivedPreds = set()
uncertainPreds = set()
ruleBodyPreds = set()
ruleHeads = set()
for rule in ruleset:
if rule.formula.body:
for idx, term in enumerate(itertools.chain(iterCondition(rule.formula.head),
iterCondition(rule.formula.body))):
# iterate over terms from head to end of body
op = GetOp(term)
if op not in processed[idx > 0]:
# not processed before
if idx > 0:
# body literal
ruleBodyPreds.add(op)
else:
# head literal
ruleHeads.add(op)
if strict in DDL_MUST_CHECK and \
not (op not in basePreds or idx > 0):
# checking DDL well formedness and
# op is a base predicate *and* a head literal (derived)
if strict in DDL_FALLBACK:
mark = strict == DDL_STRICTNESS_FALLBACK_DERIVED and \
'derived' or 'base'
if strict == DDL_STRICTNESS_FALLBACK_DERIVED and \
op not in defaultBasePreds:
# a clashing predicate is marked as derived due
# to level of strictness
derivedPreds.add(op)
elif strict == DDL_STRICTNESS_FALLBACK_BASE and \
op not in defaultDerivedPreds:
# a clashing predicate is marked as base dur
# to level of strictness
defaultBasePreds.append(op)
import warnings
warnings.warn(
"predicate symbol of %s is in both IDB and EDB. Marking as %s" % (term, mark))
else:
raise SyntaxError(
"%s is a member of a derived predicate and a base predicate." % term)
if op in basePreds:
# base predicates are marked for later validation
uncertainPreds.add(op)
else:
if idx == 0 and not isinstance(op, Variable):
# head literal with proper predicate symbol
# identify as a derived predicate
derivedPreds.add(op)
elif not isinstance(op, Variable):
# body literal with proper predicate symbol
# mark for later validation
uncertainPreds.add(op)
processed[idx > 0].add(op)
for pred in uncertainPreds:
# for each predicate marked as 'uncertain'
# do further checking
if (pred not in ruleBodyPreds and not isinstance(pred, Variable)) or\
pred in ruleHeads:
# pred is not in a body literal and is a proper predicate symbol
# or it is a rule head -> mark as a derived predicate
derivedPreds.add(pred)
for pred in derivedPreds:
if not pred in defaultBasePreds:
yield pred
def SipStrategy(query,
sipCollection,
factGraph,
derivedPreds,
bindings={},
processedRules=None,
network=None,
debug=False,
buildProof=False,
memoizeMemory=None,
proofLevel=1):
"""
Accordingly, we define a sip-strategy for computing the answers to a query
expressed using a set of Datalog rules, and a set of sips, one for each
adornment of a rule head, as follows...
Each evaluation uses memoization (via Python decorators) but also relies on well-formed
rewrites for using semi-naive bottom up method over large SPARQL data.
"""
memoizeMemory = memoizeMemory and memoizeMemory or {}
queryLiteral = BuildUnitermFromTuple(query)
processedRules = processedRules and processedRules or set()
if bindings:
#There are bindings. Apply them to the terms in the query
queryLiteral.ground(bindings)
if debug:
print("%sSolving" % ('\t' * proofLevel), queryLiteral, bindings)
# Only consider ground triple pattern isomorphism with matching bindings
goalRDFStatement = queryLiteral.toRDFTuple()
if queryLiteral in memoizeMemory:
if debug:
print("%sReturning previously calculated results for " % \
('\t' * proofLevel), queryLiteral)
for answers in memoizeMemory[queryLiteral]:
yield answers
elif AlphaNode(goalRDFStatement).alphaNetworkHash(
True,
skolemTerms=list(bindings.values())) in \
[AlphaNode(r.toRDFTuple()).alphaNetworkHash(True,
skolemTerms=list(bindings.values()))
for r in processedRules
if AdornLiteral(goalRDFStatement).adornment == \
r.adornment]:
if debug:
print("%s Goal already processed..." % \
('\t' * proofLevel))
else:
isGround = literalIsGround(queryLiteral)
if buildProof:
ns = NodeSet(goalRDFStatement, network=network, identifier=BNode())
else:
ns = None
# adornedProgram = factGraph.adornedProgram
queryPred = GetOp(queryLiteral)
if sipCollection is None:
rules = []
else:
#For every rule head matching the query, we invoke the rule,
#thus determining an adornment, and selecting a sip to follow
rules = sipCollection.headToRule.get(queryPred, set())
if None in sipCollection.headToRule:
#If there are second order rules, we add them
#since they are a 'wildcard'
rules.update(sipCollection.headToRule[None])
#maintained list of rules that haven't been processed before and
#match the query
validRules = []
#each subquery contains values for the bound arguments that are passed
#through the sip arcs entering the node corresponding to that literal. For
#each subquery generated, there is a set of answers.
answers = []
# variableMapping = {}
#Some TBox queries can be 'joined' together into SPARQL queries against
#'base' predicates via an RDF dataset
#These atomic concept inclusion axioms can be evaluated together
#using a disjunctive operator at the body of a horn clause
#where each item is a query of the form uniPredicate(?X):
#Or( uniPredicate1(?X1), uniPredicate2(?X), uniPredicate3(?X), ..)
#In this way massive, conjunctive joins can be 'mediated'
#between the stated facts and the top-down solver
@parameterizedPredicate([i for i in derivedPreds])
def IsAtomicInclusionAxiomRHS(rule, dPreds):
"""
This is an atomic inclusion axiom with
a variable (or bound) RHS: uniPred(?ENTITY)
"""
bodyList = list(iterCondition(rule.formula.body))
body = first(bodyList)
return GetOp(body) not in dPreds and \
len(bodyList) == 1 and \
body.op == RDF.type
atomicInclusionAxioms = list(filter(IsAtomicInclusionAxiomRHS, rules))
if atomicInclusionAxioms and len(atomicInclusionAxioms) > 1:
if debug:
print("\tCombining atomic inclusion axioms: ")
pprint(atomicInclusionAxioms, sys.stderr)
if buildProof:
factStep = InferenceStep(ns, source='some RDF graph')
ns.steps.append(factStep)
axioms = [rule.formula.body for rule in atomicInclusionAxioms]
#attempt to exaustively apply any available substitutions
#and determine if query if fully ground
vars = [
v for v in GetArgs(queryLiteral, secondOrder=True)
if isinstance(v, Variable)
]
openVars, axioms, _bindings = \
normalizeBindingsAndQuery(vars,
bindings,
axioms)
if openVars:
# mappings = {}
#See if we need to do any variable mappings from the query literals
#to the literals in the applicable rules
query, rt = EDBQuery(axioms, factGraph, openVars,
_bindings).evaluate(
debug, symmAtomicInclusion=True)
if buildProof:
factStep.groundQuery = subquery
for ans in rt:
if buildProof:
factStep.bindings.update(ans)
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(ans), ns))
yield ans, ns
else:
#All the relevant derivations have been explored and the result
#is a ground query we can directly execute against the facts
if buildProof:
factStep.bindings.update(bindings)
query, rt = EDBQuery(axioms, factGraph, _bindings).evaluate(
debug, symmAtomicInclusion=True)
if buildProof:
factStep.groundQuery = subquery
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(rt), ns))
yield rt, ns
rules = filter(lambda i: not IsAtomicInclusionAxiomRHS(i), rules)
for rule in rules:
#An exception is the special predicate ph; it is treated as a base
#predicate and the tuples in it are those supplied for qb by unification.
headBindings = getBindingsFromLiteral(goalRDFStatement,
rule.formula.head)
# comboBindings = dict([(k, v) for k, v in itertools.chain(
# bindings.items(),
# headBindings.items())])
varMap = rule.formula.head.getVarMapping(queryLiteral)
if headBindings and\
[term for term in rule.formula.head.getDistinguishedVariables(True)
if varMap.get(term, term) not in headBindings]:
continue
# subQueryAnswers = []
# dontStop = True
# projectedBindings = comboBindings.copy()
if debug:
print("%sProcessing rule" % \
('\t' * proofLevel), rule.formula)
if debug and sipCollection:
print("Sideways Information Passing (sip) graph for %s: " %
queryLiteral)
print(sipCollection.serialize(format='n3'))
for sip in SIPRepresentation(sipCollection):
print(sip)
try:
# Invoke the rule
if buildProof:
step = InferenceStep(ns, rule.formula)
else:
step = None
for rt, step in\
invokeRule([headBindings],
iter(iterCondition(rule.formula.body)),
rule.sip,
(proofLevel + 1,
memoizeMemory,
sipCollection,
factGraph,
derivedPreds,
processedRules.union([
AdornLiteral(query)])),
step=step,
debug=debug):
if rt:
if isinstance(rt, dict):
#We received a mapping and must rewrite it via
#correlation between the variables in the rule head
#and the variables in the original query (after applying
#bindings)
varMap = rule.formula.head.getVarMapping(
queryLiteral)
if varMap:
rt = MakeImmutableDict(
refactorMapping(varMap, rt))
if buildProof:
step.bindings = rt
else:
if buildProof:
step.bindings = headBindings
validRules.append(rule)
if buildProof:
ns.steps.append(step)
if isGround:
yield True, ns
else:
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(rt), ns))
yield rt, ns
except RuleFailure:
# Clean up failed antecedents
if buildProof:
if ns in step.antecedents:
step.antecedents.remove(ns)
if not validRules:
#No rules matching, query factGraph for answers
successful = False
if buildProof:
factStep = InferenceStep(ns, source='some RDF graph')
ns.steps.append(factStep)
if not isGround:
subquery, rt = EDBQuery([queryLiteral], factGraph, [
v for v in GetArgs(queryLiteral, secondOrder=True)
if isinstance(v, Variable)
], bindings).evaluate(debug)
if buildProof:
factStep.groundQuery = subquery
for ans in rt:
successful = True
if buildProof:
factStep.bindings.update(ans)
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(ans), ns))
yield ans, ns
if not successful and queryPred not in derivedPreds:
#Open query didn't return any results and the predicate
#is ostensibly marked as derived predicate, so we have failed
memoizeMemory.setdefault(queryLiteral, set()).add(
(False, ns))
yield False, ns
else:
#All the relevant derivations have been explored and the result
#is a ground query we can directly execute against the facts
if buildProof:
factStep.bindings.update(bindings)
subquery, rt = EDBQuery([queryLiteral], factGraph,
bindings).evaluate(debug)
if buildProof:
factStep.groundQuery = subquery
memoizeMemory.setdefault(queryLiteral, set()).add(
(prepMemiozedAns(rt), ns))
yield rt, ns
