def query(s): results = prefix.q( """ SELECT ?a ?p ?o ?t WHERE { <%s> ?p ?o. optional { ?o rdf:type ?t. } } """ % s) d={} dt={} for x in results['results']['bindings']: v = prefix.clean(x['o']['value']) t = None if 't' in x: t = prefix.clean(x['t']['value']) else: #pprint.pprint(x) pass # have no node type k = x['p']['value'] k = prefix.clean(k) if k not in d: if k not in skip: d[k]=v # the value of the field dt[k]=t # the domain type of the field object else: #d[k]=[d[k],v] raise Exception("duplicate") return d, dt
def query(s): results = prefix.q( """ SELECT ?a ?p ?o ?t WHERE { <%s> ?p ?o. optional { ?o rdf:type ?t. } } """ % s ) d = {"node_id": prefix.clean(s)} dt = {"node_id": None} # literal has no type... # pprint.pprint(results) for x in results["results"]["bindings"]: v = prefix.clean(x["o"]["value"]) t = None if "t" in x: t = prefix.clean(x["t"]["value"]) else: # pprint.pprint(x) pass # have no node type k = x["p"]["value"] k = prefix.clean(k) if k not in d: if k not in skip: d[k] = v # the value of the field dt[k] = t # the domain type of the field object else: # d[k]=[d[k],v] raise Exception("duplicate") pprint.pprint({"query_results": d}, depth=2) return d, dt
def recurse(s, deep=True): #print "RECURSE for %s\n" % s d,dt = query(s) #pprint.pprint({"Got from db":d}) if 'rdf:type' not in d: return d if not deep: return d st = d['rdf:type'] #print "st" + str(st) #pprint.pprint(dt) found = False for k in d: r = None # result of the field ot = dt[k] v = d[k] u = prefix.tg +v if type(st) is dict: raise Exception("") pprint.pprint({ 'k' :k, 'ot' :ot, 'st' : st }) #pprint.pprint(dt) pass # no type elif not ot : # no type, a literal if k.startswith('fld:'): r = prefix.clean(v) # just a literal found = True else: r = v # we need to store the type field found = True elif st in tree['exprs']: if k in tree['exprs'][st]: if ot in tree['exprs'][st][k]: r = recurse(u) found = True else: pass # skip if not found: if st in stree['exprs']: if k in stree['exprs'][st]: if ot in stree['exprs'][st][k]: r = recurse(u, False) #pprint.pprint(r) found = True else: pass if not found: r = recurse(u, False) # just get one level of info for types and such d[k]=r return (d)
def recurse(s, deep=0): if s in seen: #print "Seen" return "skipthis" seen[s]=1 if deep > 1000: #print "Too Deep" return {"too deep": s} d,dt = query(s) #pprint.pprint({"Got from db":d}) if 'rdf:type' not in d: return d st = d['rdf:type'] #print "st" + str(st) #pprint.pprint({"data type": dt}) for k in d: found = False if k in (u'fld:qual', 'fld:string', 'fld:chain'): continue r = None # result of the field ot = dt[k] v = d[k] u = prefix.tg +v # pprint.pprint({ # "Check field " : { # 'k' :k, # 'u' :u, # 'ot' :ot, # 'st' : st # }}) if type(st) is types.DictType: raise Exception("") #pprint.pprint(dt) pass # no type elif not ot : # no type, a literal if k.startswith('fld:'): r = prefix.clean(v) # just a literal found = True else: r = v # we need to store the type field found = True #print "going to recurse to "+ u if not found: r = recurse(u, deep + 1) d[k]=r # save return (d)
def recurse(s, deep=True): # print "RECURSE for %s\n" % s d, dt = query(s) # pprint.pprint({"Got from db":d}) if "rdf:type" not in d: return d if not deep: return d st = d["rdf:type"] # print "st" + str(st) # pprint.pprint(dt) found = False for k in d: r = None # result of the field ot = dt[k] v = d[k] u = prefix.tg + v if type(st) is dict: raise Exception("") pprint.pprint({"k": k, "ot": ot, "st": st}) # pprint.pprint(dt) pass # no type elif not ot: # no type, a literal if k.startswith("fld:"): r = prefix.clean(v) # just a literal found = True else: r = v # we need to store the type field found = True elif st in tree["exprs"]: if k in tree["exprs"][st]: if ot in tree["exprs"][st][k]: r = recurse(u) found = True else: pass # skip if not found: if st in stree["exprs"]: if k in stree["exprs"][st]: if ot in stree["exprs"][st][k]: r = recurse(u, False) # pprint.pprint(r) found = True else: pass if not found: # just get one level of info for types and such r = recurse(u, False) d[k] = r return d
def recurse(s, deep=0): if s in seen: # print "Seen" return "skipthis" seen[s] = 1 if deep > 1000: # print "Too Deep" return {"too deep": s} d, dt = query(s) # pprint.pprint({"Got from db":d}) if "rdf:type" not in d: return d st = d["rdf:type"] # print "st" + str(st) # pprint.pprint({"data type": dt}) for k in d: found = False if k in ("fld:qual", "fld:string", "fld:chain"): continue r = None # result of the field ot = dt[k] v = d[k] u = prefix.tg + v # pprint.pprint({ # "Check field " : { # 'k' :k, # 'u' :u, # 'ot' :ot, # 'st' : st # }}) if type(st) is dict: raise Exception("") # pprint.pprint(dt) pass # no type elif not ot: # no type, a literal if k.startswith("fld:"): r = prefix.clean(v) # just a literal found = True else: r = v # we need to store the type field found = True # print "going to recurse to "+ u if not found: r = recurse(u, deep + 1) d[k] = r # save return d
def recurse_ref(s, subtree): print(("RECURSE for %s\n" % s)) print(("using subtree : %s" % subtree["name"])) d, dt = query(s) pprint.pprint({"Got from db": d}) if "rdf:type" not in d: return d st = d["rdf:type"] # print "st" + str(st) # pprint.pprint(dt) found = False if not "exprs" in subtree: pprint.pprint({"bad subtree": subtree}, depth=2) raise Exception() lookup = subtree["exprs"] for k in d: r = None # result of the field ot = dt[k] v = d[k] u = prefix.tg + v if type(st) is dict: print(("skip" + st)) pprint.pprint( {"case": "is type", "k": k, "ot": ot, "st": st}, depth=2 ) # pprint.pprint(dt) # pass # no type elif not ot: # no type, a literal if k.startswith("fld:"): r = prefix.clean(v) # just a literal pprint.pprint( { "case": "is literal", "k": k, "dt": dt, "ot": ot, "st": st, }, depth=2, ) found = True else: pprint.pprint( { "case": "is no field", "k": k, "ot": ot, "st": st, "r": r, "v": v, }, depth=2, ) r = v # we need to store the type field found = True elif st in lookup: if k in lookup[st]: if ot in lookup[st][k]: subtree = lookup[st][k] if type(subtree) is dict: if "exprs" in subtree: r = recurse_ref(u, subtree) pprint.pprint({"Subtree": r}, depth=2) else: r = recurse_ref(u, tree) pprint.pprint({"tree": r}, depth=2) else: r = recurse_ref(u, tree) pprint.pprint({"tree2": r}, depth=2) found = True else: pass # skip if not found: r = recurse_ref( u, just_vals ) # just get one level of info for types and such pprint.pprint( {"missing": True, "k": k, "ot": ot, "st": st, "u": u, "r": r}, depth=2, ) d[k] = r pprint.pprint({"rec found": d}, depth=2) return d
def recurse_ref(s, subtree): print("RECURSE for %s\n" % s) print("using subtree : %s" % subtree['name']) d,dt = query(s) pprint.pprint({"Got from db":d}) if 'rdf:type' not in d: return d st = d['rdf:type'] #print "st" + str(st) #pprint.pprint(dt) found = False if not 'exprs' in subtree: pprint.pprint({"bad subtree": subtree}, depth=2) raise Exception() lookup = subtree['exprs'] for k in d: r = None # result of the field ot = dt[k] v = d[k] u = prefix.tg +v if type(st) is dict: print('skip' + st) pprint.pprint({ 'case': 'is type', 'k' :k, 'ot' :ot, 'st' : st }, depth=2) #pprint.pprint(dt) #pass # no type elif not ot : # no type, a literal if k.startswith('fld:'): r = prefix.clean(v) # just a literal pprint.pprint({ 'case': 'is literal', 'k' :k, 'dt': dt, 'ot' :ot, 'st' : st }, depth=2) found = True else: pprint.pprint({ 'case': 'is no field', 'k' :k, 'ot' :ot, 'st' : st, 'r' : r, 'v' : v, }, depth=2) r = v # we need to store the type field found = True elif st in lookup: if k in lookup[st]: if ot in lookup[st][k]: subtree = lookup[st][k] if type(subtree) is dict: if 'exprs' in subtree: r = recurse_ref(u, subtree) pprint.pprint({"Subtree":r}, depth=2) else: r = recurse_ref(u, tree) pprint.pprint({"tree":r}, depth=2) else: r = recurse_ref(u, tree) pprint.pprint({"tree2":r}, depth=2) found = True else: pass # skip if not found: r = recurse_ref(u, just_vals ) # just get one level of info for types and such pprint.pprint({ "missing" : True, 'k' :k, 'ot' :ot, 'st' : st, 'u' :u, 'r' :r }, depth=2) d[k]=r pprint.pprint({"rec found":d}, depth=2) return (d)
def ftypes(): t = {} results = prefix.q(""" SELECT ?st ?p ?ot (count(?o) as ?count) (min(?o) as ?firsto) (min(?s) as ?firsts) WHERE { ?s rdf:type ?st. ?s ?p ?o. ?o rdf:type ?ot. } group by ?st ?p ?ot """) print(("\t".join(["count", "firsto", "firsts", "st", "p", "ot"]))) d1 = {} d2 = {} for x in results["results"]["bindings"]: s = prefix.clean(x["st"]["value"]) o = prefix.clean(x["ot"]["value"]) p = prefix.clean(x["p"]["value"]) print(("\t".join([ x["count"]["value"], prefix.clean(x["firsto"]["value"]), prefix.clean(x["firsts"]["value"]), prefix.clean(x["st"]["value"]), prefix.clean(x["p"]["value"]), prefix.clean(x["ot"]["value"]), ]))) atype = False if o.endswith("type"): atype = True if s.endswith("type"): atype = True if p in ( "type", "domn", "elts", "size", "unql", "chain", "scpe", "min", "max", "csts", "bpos", "retn", "argt", "ptd", "flds", "refd", ): atype = True if not atype: d = d1 if s not in d: d[s] = {} if p not in d[s]: d[s][p] = {} if o not in d[s][p]: d[s][p][o] = x["count"]["value"] else: d[s][p][o] = d[s][p][o] + x["count"]["value"] else: d = d2 if s not in d: d[s] = {} if p not in d[s]: d[s][p] = {} if o not in d[s][p]: d[s][p][o] = x["count"]["value"] else: d[s][p][o] = d[s][p][o] + x["count"]["value"] pprint.pprint({"exprs": d1, "types": d2})
def ftypes(): t = {} results = prefix.q( """ SELECT ?st ?p ?ot (count(?o) as ?count) (min(?o) as ?firsto) (min(?s) as ?firsts) WHERE { ?s rdf:type ?st. ?s ?p ?o. ?o rdf:type ?ot. } group by ?st ?p ?ot """) print "\t".join( [ 'count', 'firsto', 'firsts', 'st', 'p', 'ot']) d1 = {} d2 = {} for x in results['results']['bindings']: s=prefix.clean(x['st']['value']) o=prefix.clean(x['ot']['value']) p=prefix.clean(x['p']['value']) print "\t".join( [ x['count']['value'], prefix.clean(x['firsto']['value']), prefix.clean(x['firsts']['value']), prefix.clean(x['st']['value']), prefix.clean(x['p']['value']), prefix.clean(x['ot']['value']) ]) atype =False if o.endswith('type'): atype=True if s.endswith('type'): atype=True if p in ('type','domn','elts', 'size','unql', 'chain', 'scpe', 'min', 'max', 'csts', 'bpos', 'retn', 'argt', 'ptd', 'flds', 'refd' ): atype=True if not atype: d = d1 if s not in d: d[s] = {} if p not in d[s]: d[s][p] = {} if o not in d[s][p]: d[s][p][o] =x['count']['value'] else: d[s][p][o]= d[s][p][o] + x['count']['value'] else: d = d2 if s not in d: d[s] = {} if p not in d[s]: d[s][p] = {} if o not in d[s][p]: d[s][p][o] =x['count']['value'] else: d[s][p][o]= d[s][p][o] + x['count']['value'] pprint.pprint( { 'exprs' :d1 , 'types' :d2 })