def query(s):
results = prefix.q( """
SELECT ?a ?p ?o ?t WHERE {
<%s> ?p ?o.
optional {
?o rdf:type ?t.
}
}
""" % s)
d={}
dt={}
for x in results['results']['bindings']:
v = prefix.clean(x['o']['value'])
t = None
if 't' in x:
t = prefix.clean(x['t']['value'])
else:
#pprint.pprint(x)
pass # have no node type
k = x['p']['value']
k = prefix.clean(k)
if k not in d:
if k not in skip:
d[k]=v # the value of the field
dt[k]=t # the domain type of the field object
else:
#d[k]=[d[k],v]
raise Exception("duplicate")
return d, dt
def query(s):
results = prefix.q(
"""
SELECT ?a ?p ?o ?t WHERE {
<%s> ?p ?o.
optional {
?o rdf:type ?t.
}
}
"""
% s
)
d = {"node_id": prefix.clean(s)}
dt = {"node_id": None} # literal has no type...
# pprint.pprint(results)
for x in results["results"]["bindings"]:
v = prefix.clean(x["o"]["value"])
t = None
if "t" in x:
t = prefix.clean(x["t"]["value"])
else:
# pprint.pprint(x)
pass # have no node type
k = x["p"]["value"]
k = prefix.clean(k)
if k not in d:
if k not in skip:
d[k] = v # the value of the field
dt[k] = t # the domain type of the field object
else:
# d[k]=[d[k],v]
raise Exception("duplicate")
pprint.pprint({"query_results": d}, depth=2)
return d, dt
def recurse(s, deep=True):
#print "RECURSE for %s\n" % s
d,dt = query(s)
#pprint.pprint({"Got from db":d})
if 'rdf:type' not in d:
return d
if not deep:
return d
st = d['rdf:type']
#print "st" + str(st)
#pprint.pprint(dt)
found = False
for k in d:
r = None # result of the field
ot = dt[k]
v = d[k]
u = prefix.tg +v
if type(st) is dict:
raise Exception("")
pprint.pprint({
'k' :k,
'ot' :ot,
'st' : st
})
#pprint.pprint(dt)
pass # no type
elif not ot : # no type, a literal
if k.startswith('fld:'):
r = prefix.clean(v) # just a literal
found = True
else:
r = v # we need to store the type field
found = True
elif st in tree['exprs']:
if k in tree['exprs'][st]:
if ot in tree['exprs'][st][k]:
r = recurse(u)
found = True
else:
pass # skip
if not found:
if st in stree['exprs']:
if k in stree['exprs'][st]:
if ot in stree['exprs'][st][k]:
r = recurse(u, False)
#pprint.pprint(r)
found = True
else:
pass
if not found:
r = recurse(u, False) # just get one level of info for types and such
d[k]=r
return (d)
def recurse(s, deep=0):
if s in seen:
#print "Seen"
return "skipthis"
seen[s]=1
if deep > 1000:
#print "Too Deep"
return {"too deep": s}
d,dt = query(s)
#pprint.pprint({"Got from db":d})
if 'rdf:type' not in d:
return d
st = d['rdf:type']
#print "st" + str(st)
#pprint.pprint({"data type": dt})
for k in d:
found = False
if k in (u'fld:qual',
'fld:string',
'fld:chain'):
continue
r = None # result of the field
ot = dt[k]
v = d[k]
u = prefix.tg +v
# pprint.pprint({
# "Check field " : {
# 'k' :k,
# 'u' :u,
# 'ot' :ot,
# 'st' : st
# }})
if type(st) is types.DictType:
raise Exception("")
#pprint.pprint(dt)
pass # no type
elif not ot : # no type, a literal
if k.startswith('fld:'):
r = prefix.clean(v) # just a literal
found = True
else:
r = v # we need to store the type field
found = True
#print "going to recurse to "+ u
if not found:
r = recurse(u, deep + 1)
d[k]=r
# save
return (d)
def recurse(s, deep=True):
# print "RECURSE for %s\n" % s
d, dt = query(s)
# pprint.pprint({"Got from db":d})
if "rdf:type" not in d:
return d
if not deep:
return d
st = d["rdf:type"]
# print "st" + str(st)
# pprint.pprint(dt)
found = False
for k in d:
r = None # result of the field
ot = dt[k]
v = d[k]
u = prefix.tg + v
if type(st) is dict:
raise Exception("")
pprint.pprint({"k": k, "ot": ot, "st": st})
# pprint.pprint(dt)
pass # no type
elif not ot: # no type, a literal
if k.startswith("fld:"):
r = prefix.clean(v) # just a literal
found = True
else:
r = v # we need to store the type field
found = True
elif st in tree["exprs"]:
if k in tree["exprs"][st]:
if ot in tree["exprs"][st][k]:
r = recurse(u)
found = True
else:
pass # skip
if not found:
if st in stree["exprs"]:
if k in stree["exprs"][st]:
if ot in stree["exprs"][st][k]:
r = recurse(u, False)
# pprint.pprint(r)
found = True
else:
pass
if not found:
# just get one level of info for types and such
r = recurse(u, False)
d[k] = r
return d
def recurse(s, deep=0):
if s in seen:
# print "Seen"
return "skipthis"
seen[s] = 1
if deep > 1000:
# print "Too Deep"
return {"too deep": s}
d, dt = query(s)
# pprint.pprint({"Got from db":d})
if "rdf:type" not in d:
return d
st = d["rdf:type"]
# print "st" + str(st)
# pprint.pprint({"data type": dt})
for k in d:
found = False
if k in ("fld:qual", "fld:string", "fld:chain"):
continue
r = None # result of the field
ot = dt[k]
v = d[k]
u = prefix.tg + v
# pprint.pprint({
# "Check field " : {
# 'k' :k,
# 'u' :u,
# 'ot' :ot,
# 'st' : st
# }})
if type(st) is dict:
raise Exception("")
# pprint.pprint(dt)
pass # no type
elif not ot: # no type, a literal
if k.startswith("fld:"):
r = prefix.clean(v) # just a literal
found = True
else:
r = v # we need to store the type field
found = True
# print "going to recurse to "+ u
if not found:
r = recurse(u, deep + 1)
d[k] = r
# save
return d
def recurse_ref(s, subtree):
print(("RECURSE for %s\n" % s))
print(("using subtree : %s" % subtree["name"]))
d, dt = query(s)
pprint.pprint({"Got from db": d})
if "rdf:type" not in d:
return d
st = d["rdf:type"]
# print "st" + str(st)
# pprint.pprint(dt)
found = False
if not "exprs" in subtree:
pprint.pprint({"bad subtree": subtree}, depth=2)
raise Exception()
lookup = subtree["exprs"]
for k in d:
r = None # result of the field
ot = dt[k]
v = d[k]
u = prefix.tg + v
if type(st) is dict:
print(("skip" + st))
pprint.pprint(
{"case": "is type", "k": k, "ot": ot, "st": st}, depth=2
)
# pprint.pprint(dt)
# pass # no type
elif not ot: # no type, a literal
if k.startswith("fld:"):
r = prefix.clean(v) # just a literal
pprint.pprint(
{
"case": "is literal",
"k": k,
"dt": dt,
"ot": ot,
"st": st,
},
depth=2,
)
found = True
else:
pprint.pprint(
{
"case": "is no field",
"k": k,
"ot": ot,
"st": st,
"r": r,
"v": v,
},
depth=2,
)
r = v # we need to store the type field
found = True
elif st in lookup:
if k in lookup[st]:
if ot in lookup[st][k]:
subtree = lookup[st][k]
if type(subtree) is dict:
if "exprs" in subtree:
r = recurse_ref(u, subtree)
pprint.pprint({"Subtree": r}, depth=2)
else:
r = recurse_ref(u, tree)
pprint.pprint({"tree": r}, depth=2)
else:
r = recurse_ref(u, tree)
pprint.pprint({"tree2": r}, depth=2)
found = True
else:
pass # skip
if not found:
r = recurse_ref(
u, just_vals
) # just get one level of info for types and such
pprint.pprint(
{"missing": True, "k": k, "ot": ot, "st": st, "u": u, "r": r},
depth=2,
)
d[k] = r
pprint.pprint({"rec found": d}, depth=2)
return d
def recurse_ref(s, subtree):
print("RECURSE for %s\n" % s)
print("using subtree : %s" % subtree['name'])
d,dt = query(s)
pprint.pprint({"Got from db":d})
if 'rdf:type' not in d:
return d
st = d['rdf:type']
#print "st" + str(st)
#pprint.pprint(dt)
found = False
if not 'exprs' in subtree:
pprint.pprint({"bad subtree": subtree}, depth=2)
raise Exception()
lookup = subtree['exprs']
for k in d:
r = None # result of the field
ot = dt[k]
v = d[k]
u = prefix.tg +v
if type(st) is dict:
print('skip' + st)
pprint.pprint({
'case': 'is type',
'k' :k,
'ot' :ot,
'st' : st
}, depth=2)
#pprint.pprint(dt)
#pass # no type
elif not ot : # no type, a literal
if k.startswith('fld:'):
r = prefix.clean(v) # just a literal
pprint.pprint({
'case': 'is literal',
'k' :k,
'dt': dt,
'ot' :ot,
'st' : st
}, depth=2)
found = True
else:
pprint.pprint({
'case': 'is no field',
'k' :k,
'ot' :ot,
'st' : st,
'r' : r,
'v' : v,
}, depth=2)
r = v # we need to store the type field
found = True
elif st in lookup:
if k in lookup[st]:
if ot in lookup[st][k]:
subtree = lookup[st][k]
if type(subtree) is dict:
if 'exprs' in subtree:
r = recurse_ref(u, subtree)
pprint.pprint({"Subtree":r}, depth=2)
else:
r = recurse_ref(u, tree)
pprint.pprint({"tree":r}, depth=2)
else:
r = recurse_ref(u, tree)
pprint.pprint({"tree2":r}, depth=2)
found = True
else:
pass # skip
if not found:
r = recurse_ref(u, just_vals ) # just get one level of info for types and such
pprint.pprint({
"missing" : True,
'k' :k,
'ot' :ot,
'st' : st,
'u' :u,
'r' :r
}, depth=2)
d[k]=r
pprint.pprint({"rec found":d}, depth=2)
return (d)
def ftypes():
t = {}
results = prefix.q("""
SELECT ?st ?p ?ot (count(?o) as ?count) (min(?o) as ?firsto) (min(?s) as ?firsts)
WHERE {
?s rdf:type ?st.
?s ?p ?o.
?o rdf:type ?ot. } group by ?st ?p ?ot
""")
print(("\t".join(["count", "firsto", "firsts", "st", "p", "ot"])))
d1 = {}
d2 = {}
for x in results["results"]["bindings"]:
s = prefix.clean(x["st"]["value"])
o = prefix.clean(x["ot"]["value"])
p = prefix.clean(x["p"]["value"])
print(("\t".join([
x["count"]["value"],
prefix.clean(x["firsto"]["value"]),
prefix.clean(x["firsts"]["value"]),
prefix.clean(x["st"]["value"]),
prefix.clean(x["p"]["value"]),
prefix.clean(x["ot"]["value"]),
])))
atype = False
if o.endswith("type"):
atype = True
if s.endswith("type"):
atype = True
if p in (
"type",
"domn",
"elts",
"size",
"unql",
"chain",
"scpe",
"min",
"max",
"csts",
"bpos",
"retn",
"argt",
"ptd",
"flds",
"refd",
):
atype = True
if not atype:
d = d1
if s not in d:
d[s] = {}
if p not in d[s]:
d[s][p] = {}
if o not in d[s][p]:
d[s][p][o] = x["count"]["value"]
else:
d[s][p][o] = d[s][p][o] + x["count"]["value"]
else:
d = d2
if s not in d:
d[s] = {}
if p not in d[s]:
d[s][p] = {}
if o not in d[s][p]:
d[s][p][o] = x["count"]["value"]
else:
d[s][p][o] = d[s][p][o] + x["count"]["value"]
pprint.pprint({"exprs": d1, "types": d2})
def ftypes():
t = {}
results = prefix.q( """
SELECT ?st ?p ?ot (count(?o) as ?count) (min(?o) as ?firsto) (min(?s) as ?firsts)
WHERE {
?s rdf:type ?st.
?s ?p ?o.
?o rdf:type ?ot. } group by ?st ?p ?ot
""")
print "\t".join(
[
'count',
'firsto',
'firsts',
'st',
'p',
'ot'])
d1 = {}
d2 = {}
for x in results['results']['bindings']:
s=prefix.clean(x['st']['value'])
o=prefix.clean(x['ot']['value'])
p=prefix.clean(x['p']['value'])
print "\t".join(
[
x['count']['value'],
prefix.clean(x['firsto']['value']),
prefix.clean(x['firsts']['value']),
prefix.clean(x['st']['value']),
prefix.clean(x['p']['value']),
prefix.clean(x['ot']['value'])
])
atype =False
if o.endswith('type'):
atype=True
if s.endswith('type'):
atype=True
if p in ('type','domn','elts',
'size','unql',
'chain',
'scpe',
'min',
'max',
'csts',
'bpos',
'retn',
'argt',
'ptd',
'flds',
'refd'
):
atype=True
if not atype:
d = d1
if s not in d:
d[s] = {}
if p not in d[s]:
d[s][p] = {}
if o not in d[s][p]:
d[s][p][o] =x['count']['value']
else:
d[s][p][o]= d[s][p][o] + x['count']['value']
else:
d = d2
if s not in d:
d[s] = {}
if p not in d[s]:
d[s][p] = {}
if o not in d[s][p]:
d[s][p][o] =x['count']['value']
else:
d[s][p][o]= d[s][p][o] + x['count']['value']
pprint.pprint( { 'exprs' :d1 ,
'types' :d2
})
