Example #1
0
def fitData(data):
    """Under the framework developed at: http://arxiv.org/abs/1305.0215"""
    c("plaw fit start")
    fit = powerlaw.Fit(data, discrete=True)
    c("plaw fit end")
    dists = list(fit.supported_distributions.keys())
    dists.remove("power_law")
    for dist in dists:
        c("plaw compare start: " + dist)
        comp = fit.distribution_compare("power_law", dist)
        exec("fit.{}_R=comp[0]".format(dist))
        exec("fit.{}_p=comp[1]".format(dist))
        c("plaw compare end: " + dist)
    del dist, comp
    return locals()
Example #2
0
File: perform.py Project: ttm/music
 def convert(self, notes, reference):
     if 'notes_dict' not in dir(self):
         self.makeDict()
     notes_ = [reference + note for note in notes]
     notes__ = [self.notes_dict[note] for note in notes_]
     c('notes', notes)
     c('notes_', notes_)
     c('notes__', notes__)
     return notes__
Example #3
0
def probeOntology(endpoint_url, graph_urns, final_dir, one_datatype=True):
    if not os.path.isdir(final_dir):
        os.makedirs(final_dir)

    client = P.rdf.sparql.classes.LegacyClient(endpoint_url)
    from_ = ''
    for graph_urn in graph_urns:
        from_ += '\nFROM <%s>' % (graph_urn, )

    def mkQuery(query, plain=True):
        query_ = query.split('WHERE')
        query__ = (query_[0], from_, '\nWHERE ' + query_[1])
        query___ = ''.join(query__)
        result = client.retrieveQuery(query___)
        if plain:
            return pl(result)
        else:
            return result['results']['bindings']

    c('find all classes')
    q = "SELECT DISTINCT ?class WHERE { ?s a ?class . }"
    # classes = pl(client.retrieveQuery(prefix+q))
    classes = mkQuery(q)

    c('antecedents, consequents and restrictions of each class')
    neighbors = {}
    triples = []
    existential_restrictions = {}
    universal_restrictions = {}
    for aclass in classes:
        q = "SELECT DISTINCT ?cs ?p WHERE { ?i a <%s> . ?s ?p ?i . OPTIONAL { ?s a ?cs . } }" % (
            aclass, )
        antecedent_property = mkQuery(q)
        # q = "SELECT DISTINCT ?ap (datatype(?o) as ?do) WHERE { ?i a <%s> . ?i ?ap ?o . filter (datatype(?o) != '') }" % (aclass,)
        # consequent_property = mkQuery(q)
        # q = "SELECT DISTINCT ?ap ?co WHERE { ?i a <%s> . ?i ?ap ?o . ?o a ?co . }" % (aclass,)
        # consequent_property_ = mkQuery(q)
        q = "SELECT DISTINCT ?ap ?co (datatype(?o) as ?do) WHERE { ?i a <%s> . ?i ?ap ?o . OPTIONAL { ?o a ?co . } }" % (
            aclass, )
        consequent_property__ = mkQuery(q, 0)
        consequent_property = [[i['ap']['value'], i['do']['value']]
                               for i in consequent_property__ if 'do' in i]
        consequent_property_ = [[i['ap']['value'], i['co']['value']]
                                for i in consequent_property__ if 'co' in i]
        neighbors[aclass] = (antecedent_property,
                             consequent_property + consequent_property_)
        # neighbors[aclass] = (antecedent_property, dict(consequent_property, **consequent_property_))

        # class restrictions
        q = "SELECT DISTINCT ?p WHERE {?s a <%s>. ?s ?p ?o .}" % (aclass, )
        props_c = mkQuery(q)
        # q = "SELECT DISTINCT ?s WHERE {?s a <%s>}" % (aclass,)
        # inds = mkQuery(q)
        q = "SELECT (COUNT(DISTINCT ?s) as ?cs) WHERE {?s a <%s>}" % (aclass, )
        ninds = pl(client.retrieveQuery(q))[0]
        for pc in props_c:
            if '22-rdf-syntax' in pc:
                continue
            # q = "SELECT DISTINCT ?s ?co  (datatype(?o) as ?do) WHERE {?s a <%s>. ?s <%s> ?o . OPTIONAL {?o a ?co . }}" % (aclass, pc)
            q = "SELECT DISTINCT ?co (datatype(?o) as ?do) WHERE {?s a <%s>. ?s <%s> ?o . OPTIONAL {?o a ?co . }}" % (
                aclass, pc)
            inds2 = mkQuery(q, 0)
            # inds2_ = set([i["s"]["value"] for i in inds2])
            objs = set([i["co"]["value"] for i in inds2 if "co" in i.keys()])
            vals = set([i["do"]["value"] for i in inds2 if "do" in i.keys()])
            q = "SELECT (COUNT(DISTINCT ?s) as ?cs) WHERE {?s a <%s>. ?s <%s> ?o . }" % (
                aclass, pc)
            ninds2 = pl(client.retrieveQuery(q))[0]
            # if len(inds) == len(inds2_):  # existential
            if ninds == ninds2:  # existential
                if len(vals):
                    ob = list(vals)[0]
                else:
                    if len(objs):
                        ob = list(objs)[0]
                    else:
                        ob = 0
                if ob:
                    B = r.BNode()
                    triples += [(aclass, rdfs.subClassOf, B),
                                (B, a, owl.Restriction),
                                (B, owl.onProperty, pc),
                                (B, owl.someValuesFrom, ob)]
                    if aclass in existential_restrictions.keys():
                        existential_restrictions[aclass].append((pc, ob))
                    else:
                        existential_restrictions[aclass] = [(pc, ob)]
            q = "SELECT (COUNT(DISTINCT ?s) as ?cs) WHERE { ?s <%s> ?o . ?s a ?ca . FILTER(str(?ca) != '%s') }" % (
                pc, aclass)
            ninds3 = pl(client.retrieveQuery(q))[0]
            # q = "SELECT DISTINCT ?s WHERE { ?s <%s> ?o .}" % (pc,)
            # inds3 = mkQuery(q)
            # if set(inds) == set(inds3):  # universal
            # if all([i in set(inds) for i in inds3]):  # universal
            # if ninds == ninds3:  # universal
            if ninds3 == 0:  # universal
                if len(vals):
                    ob = list(vals)[0]
                else:
                    if len(objs):
                        ob = list(objs)[0]
                    else:
                        ob = 0
                if ob:
                    B = r.BNode()
                    triples += [(aclass, rdfs.subClassOf, B),
                                (B, a, owl.Restriction),
                                (B, owl.onProperty, pc),
                                (B, owl.allValuesFrom, ob)]
                    if aclass in universal_restrictions.keys():
                        universal_restrictions[aclass].append((pc, ob))
                    else:
                        universal_restrictions[aclass] = [(pc, ob)]
    del q, aclass, antecedent_property, consequent_property
    c('find properties')
    q = "SELECT DISTINCT ?p WHERE {?s ?p ?o}"
    # properties = pl(client.retrieveQuery(prefix+q))
    properties = mkQuery(q)
    # properties_ = [i.split("/")[-1] for i in properties]

    c('check if property is functional and get range and domain')
    functional_properties = set()
    for prop in properties:
        # check if property is functional
        q = 'SELECT DISTINCT (COUNT(?o) as ?co) WHERE { ?s <%s> ?o } GROUP BY ?s' % (
            prop, )
        is_functional = mkQuery(q)
        if len(is_functional) == 1 and is_functional[0] == 1:
            triples.append((prop, a, owl.FunctionalProperty))
            functional_properties.add(prop)

        # datatype or object properties
        suj = mkQuery("SELECT DISTINCT ?cs WHERE { ?s <%s> ?o . ?s a ?cs . }" %
                      (prop, ))
        # obj = mkQuery("SELECT DISTINCT ?co (datatype(?o) as ?do) WHERE { ?s <%s> ?o . OPTIONAL { ?o a ?co . } }" % (prop,))
        obj1 = mkQuery(
            "SELECT DISTINCT ?co WHERE { ?s <%s> ?o . ?o a ?co . }" % (prop, ))
        obj2 = mkQuery(
            "SELECT DISTINCT (datatype(?o) as ?do) WHERE { ?s <%s> ?o . }" %
            (prop, ))
        obj = obj1 + obj2
        if len(obj) and ("XMLS" in obj[0]):
            triples.append((prop, a, owl.DataProperty))
        else:
            triples.append((prop, a, owl.ObjectProperty))
        if len(suj) > 1:
            B = r.BNode()
            triples.append((prop, rdfs.domain, B))
            for ss in suj:
                triples.append((B, owl.unionOf, ss))
        elif suj:
            triples.append((prop, rdfs.domain, suj[0]))
        if len(obj) > 1:
            B = r.BNode()
            triples.append((prop, rdfs.range, B))
            for ss in suj:
                triples.append((B, owl.unionOf, ss))
        elif obj:
            triples.append((prop, rdfs.range, obj[0]))

        # for drawing
        # prop_ = prop.split("/")[-1]
        # suj_ = [i.split('/')[-1] for i in suj]
        # obj_ = [i.split('/')[-1] for i in obj]
    # Drawing
    c('started drawing')
    A = gv.AGraph(directed=True, strict=False)
    q = """PREFIX po: <http://purl.org/socialparticipation/po/>
    SELECT DISTINCT ?snap WHERE { { ?i po:snapshot ?snap } UNION { ?snap po:snapshotID ?idfoo } }"""
    # SELECT DISTINCT ?snap WHERE { ?i po:snapshot ?snap }"""
    snap = mkQuery(q)[0]
    q = """PREFIX po: <http://purl.org/socialparticipation/po/>
    SELECT ?provenance
    WHERE { <%s> po:socialProtocol ?provenance }""" % (snap)
    # WHERE { { <%s> po:socialProtocolTag ?provenance } UNION
    #         { <%s> po:humanizedName ?provenance } }""" % (snap, snap)
    provenance = pl(client.retrieveQuery(q))[0]
    # A.graph_attr["label"] = r"General diagram of ontological structure from %s in the http://purl.org/socialparticipation/participationontology/ namespace.\nGreen edge denotes existential restriction;\ninverted edge nip denotes universal restriction;\nfull edge (non-dashed) denotes functional property." % (provenance,)
    edge_counter = 1
    node_counter = 1
    data_nodes = {}
    for aclass in classes:
        aclass_ = aclass.split('/')[-1]
        if aclass_ not in A.nodes():
            A.add_node(aclass_, style="filled")
            n = A.get_node(aclass_)
            n.attr['color'] = "#A2F3D1"
        neigh = neighbors[aclass]
        # for i in range(len(neigh[0])):  # antecendents
        #     label = neigh[0][i][0].split("/")[-1]
        #     elabel = neigh[0][i][1]
        #     elabel_ = elabel.split("/")[-1]
        #     if label not in A.nodes():
        #         A.add_node(label, style="filled")
        #         n = A.get_node(label)
        #         n.attr['color'] = "#A2F3D1"
        #     ekey = '{}-{}-{}'.format(label, aclass_, edge_counter)
        #     edge_counter += 1
        #     A.add_edge(label, aclass_, ekey)
        #     e = A.get_edge(label, aclass_, key=ekey)
        #     e.attr["label"] = elabel_
        #     e.attr["penwidth"] = 2.
        #     e.attr["arrowsize"] = 2.
        #     if elabel not in functional_properties:
        #         e.attr["style"] = "dashed"
        #     if neigh[0][i][0] in existential_restrictions.keys():
        #         restriction = existential_restrictions[neigh[0][i][0]]
        #         prop = [iii[0] for iii in restriction]
        #         obj = [iii[1] for iii in restriction]
        #         if (elabel in prop) and (obj[prop.index(elabel)] == aclass):
        #             e.attr["color"] = "#A0E0A0"
        #     if neigh[0][i][0] in universal_restrictions.keys():
        #         restriction = universal_restrictions[neigh[0][i][0]]
        #         prop = [iii[0] for iii in restriction]
        #         obj = [iii[1] for iii in restriction]
        #         if (elabel in prop) and (obj[prop.index(elabel)] == aclass):
        #             e.attr["color"] = "inv"
        for i in range(len(neigh[1])):  # consequents
            label = neigh[1][i][1].split("/")[-1]
            elabel = neigh[1][i][0]
            elabel_ = elabel.split('/')[-1]
            if "XMLS" in label:
                color = "#FFE4AA"
                if one_datatype:
                    if label in data_nodes:
                        label_ = data_nodes[label]
                    else:
                        label_ = node_counter
                        node_counter += 1
                        data_nodes[label] = label_
                else:
                    label_ = node_counter
                    node_counter += 1
            else:
                label_ = label
                color = "#A2F3D1"
            if label_ not in A.nodes():
                A.add_node(label_, style="filled")
                n = A.get_node(label_)
                n.attr['label'] = label.split("#")[-1]
                n.attr['color'] = color
            ekey = '{}-{}-{}'.format(aclass_, label_, edge_counter)
            edge_counter += 1
            A.add_edge(aclass_, label_, ekey)
            e = A.get_edge(aclass_, label_, key=ekey)
            e.attr["label"] = elabel_
            e.attr["color"] = color
            e.attr["penwidth"] = 2
            if r.URIRef(elabel) not in functional_properties:
                e.attr["style"] = "dashed"
            if aclass in existential_restrictions.keys():
                restrictions = existential_restrictions[aclass]
                prop = [iii[0] for iii in restrictions]
                if r.URIRef(elabel) in prop:
                    e.attr["color"] = "#A0E0A0"
            if aclass in universal_restrictions.keys():
                restrictions = universal_restrictions[aclass]
                prop = [iii[0] for iii in restrictions]
                if r.URIRef(elabel) in prop:
                    e.attr["arrowhead"] = "inv"
                    e.attr["arrowsize"] = 2.

    # A.draw(os.path.join(final_dir, "{}.png".format(final_dir)), prog="dot")
    # try:
    #     A.draw(os.path.join(final_dir, "{}_circo.png".format(final_dir)), prog="circo")
    # except:
    #     pass
    # A.draw(os.path.join(final_dir, "{}_twopi.png".format(final_dir)), prog="twopi", args="-Granksep=4")
    # A.write(os.path.join(final_dir, "{}.dot".format(final_dir)))
    A.draw(os.path.join(final_dir, "draw.png"), prog="dot")
    try:
        A.draw(os.path.join(final_dir, "draw_circo.png"), prog="circo")
    except:
        pass
    A.draw(os.path.join(final_dir, "draw_twopi.png"),
           prog="twopi",
           args="-Granksep=4")
    A.write(os.path.join(final_dir, "draw.dot"))
    # for triple in triples:
    #     g.add(triple)
    P.start(False)
    P.context('ontology', 'remove')
    P.add(triples, 'ontology')
    g = P.context('ontology')
    g.serialize(os.path.join(final_dir, 'ontology.owl'))
    g.serialize(os.path.join(final_dir, 'ontology.ttl'), 'turtle')
    return locals()
        # os.system('curl --digest --user dba:demo --url "http://192.168.0.11:8890/sparql-graph-crud-auth?graph-uri=urn:{}" -T {}/{}/{}'.format(afile, tdir, adir, afile))
        # os.system('curl --digest --user dba:dba --url "http://192.168.0.11:8890/sparql-graph-crud-auth?graph-uri=urn:{}" -T {}/{}/{}'.format(afile, tdir, adir, afile))
        os.system('curl --digest --user dba:dba --url "http://localhost:8890/sparql-graph-crud-auth?graph-uri=urn:{}" -T {}/{}/{}'.format(afile, tdir, adir, afile))
        # os.system('curl --digest --user dba:dba --url "http://localhost:8890/sparql-graph-crud-auth?graph-uri=urn:percolation" -T {}/{}/{}'.format(tdir, adir, afile))

ans = input('add twitter translates (y/n)')
if ans[0] == 'y':
    tdir = '/home/r/repos/social/tests/twitter_snapshots'
    dirs = os.listdir(tdir)
#    dirs = [i for i in dirs if not ('Snap' in i or 'science' in i or 'Fora' in i)]
    for adir in dirs:
        files = os.listdir('{}/{}'.format(tdir, adir))
        # files_ = [i for i in files if i.endswith('.ttl') and 'Meta' not in i]
        files_ = [i for i in files if i.endswith('.ttl')]
        for afile in files_:
            c(afile, 'pre')
            uploadFile(platform, tdir, adir, afile)
            c(afile, 'pos')
ans = input('add facebook translates (y/n)')
if ans[0] == 'y':
    tdir = '/home/r/repos/social/tests/facebook_snapshots'
    dirs = os.listdir(tdir)
    for adir in dirs:
        files = os.listdir('{}/{}'.format(tdir, adir))
        files_ = [i for i in files if i.endswith('.ttl')]
        # files_ = [i for i in files if i.endswith('.ttl') and 'Meta' not in i]
        for afile in files_:
            c(afile, 'pre')
            uploadFile(platform, tdir, adir, afile)
            c(afile, 'pos')
# irc
Example #5
0
        if "cs" in aa.keys():
            tobj = aa["cs"]["value"]
            ant_.append((tobj, aa["p"]["value"]))
    cons = fazQuery(
        "SELECT DISTINCT ?p ?co (datatype(?o) as ?do) WHERE { ?i a <%s> . ?i ?p ?o . OPTIONAL { ?o a ?co . } }"
        % (classe, ))
    cons_ = []
    for cc in cons:
        if "co" in cc.keys():
            tobj = cc["co"]["value"]
            cons_.append((cc["p"]["value"], tobj))
        elif (("do" in cc.keys()) and ("w3.org" not in cc["p"]["value"])):
            tobj = cc["do"]["value"]
            cons_.append((cc["p"]["value"], tobj))
        elif (("do" in cc.keys()) and ("w3.org" in cc["p"]["value"])):
            c(cc["p"]["value"], tobj)  # to see what triple this is
            sys.exit()
    neighbors[classe] = (ant, cons)
    neighbors_[classe] = (ant_, cons_)
f = open("dumpVV.pickle", "wb")
vv = (neighbors, neighbors_)
pickle.dump(vv, f)
f.close()
fo = open("dumpVV.pickle", "rb")
vv_ = pickle.load(fo)
fo.close()
kk = vv_[1].keys()
for tkey in kk:  # for each class
    cl = tkey
    cl_ = cl.split("/")[-1]
    print(cl_)
            'curl --digest --user dba:dba --url "http://localhost:8890/sparql-graph-crud-auth?graph-uri=urn:{}" -T {}/{}/{}'
            .format(afile, tdir, adir, afile))
        # os.system('curl --digest --user dba:dba --url "http://localhost:8890/sparql-graph-crud-auth?graph-uri=urn:percolation" -T {}/{}/{}'.format(tdir, adir, afile))


ans = input('add twitter translates (y/n)')
if ans[0] == 'y':
    tdir = '/home/r/repos/social/tests/twitter_snapshots'
    dirs = os.listdir(tdir)
    #    dirs = [i for i in dirs if not ('Snap' in i or 'science' in i or 'Fora' in i)]
    for adir in dirs:
        files = os.listdir('{}/{}'.format(tdir, adir))
        # files_ = [i for i in files if i.endswith('.ttl') and 'Meta' not in i]
        files_ = [i for i in files if i.endswith('.ttl')]
        for afile in files_:
            c(afile, 'pre')
            uploadFile(platform, tdir, adir, afile)
            c(afile, 'pos')
ans = input('add facebook translates (y/n)')
if ans[0] == 'y':
    tdir = '/home/r/repos/social/tests/facebook_snapshots'
    dirs = os.listdir(tdir)
    for adir in dirs:
        files = os.listdir('{}/{}'.format(tdir, adir))
        files_ = [i for i in files if i.endswith('.ttl')]
        # files_ = [i for i in files if i.endswith('.ttl') and 'Meta' not in i]
        for afile in files_:
            c(afile, 'pre')
            uploadFile(platform, tdir, adir, afile)
            c(afile, 'pos')
# irc
Example #7
0
    ant_ = []
    for aa in ant:
        if "cs" in aa.keys():
            tobj = aa["cs"]["value"]
            ant_.append((tobj, aa["p"]["value"]))
    cons = fazQuery("SELECT DISTINCT ?p ?co (datatype(?o) as ?do) WHERE { ?i a <%s> . ?i ?p ?o . OPTIONAL { ?o a ?co . } }" % (classe, ))
    cons_ = []
    for cc in cons:
        if "co" in cc.keys():
            tobj = cc["co"]["value"]
            cons_.append((cc["p"]["value"], tobj))
        elif (("do" in cc.keys()) and ("w3.org" not in cc["p"]["value"])):
            tobj = cc["do"]["value"]
            cons_.append((cc["p"]["value"], tobj))
        elif (("do" in cc.keys()) and ("w3.org" in cc["p"]["value"])):
            c(cc["p"]["value"], tobj)  # to see what triple this is
            sys.exit()
    neighbors[classe] = (ant, cons)
    neighbors_[classe] = (ant_, cons_)
f = open("dumpVV.pickle", "wb")
vv = (neighbors, neighbors_)
pickle.dump(vv, f)
f.close()
fo = open("dumpVV.pickle", "rb")
vv_ = pickle.load(fo)
fo.close()
kk = vv_[1].keys()
for tkey in kk:  # for each class
    cl = tkey
    cl_ = cl.split("/")[-1]
    print(cl_)
Example #8
0
import sys
keys=tuple(sys.modules.keys())
for key in keys:
    if "social" in key or "percolation" in key:
        del sys.modules[key]
import social as S, os
from percolation import check as c
snapdirs=os.listdir("./fb/")
snapdirs=[i for i in snapdirs if os.path.isdir("./fb/"+i)]
for snapdir in snapdirs:
    files=os.listdir("./fb/"+snapdir)
    files=[i for i in files if (i.endswith(".rdf") or i.endswith(".ttl"))]
    files.sort()
    for file_ in files:
        c("loading: ", file_, len(S.P.percolation_graph))
        if file_.endswith(".rdf"):
            S.P.percolation_graph.parse("./fb/"+snapdir+"/"+file_)
        elif file_.endswith(".ttl"):
            S.P.percolation_graph.parse("./fb/"+snapdir+"/"+file_,format="turtle")
        else:
            raise ValueError("Only rdf files please")
        c("loaded: ", file_, "ntriples in percolation graph: ", len(S.P.percolation_graph))
Example #9
0
import sys

keys = tuple(sys.modules.keys())
for key in keys:
    if "social" in key or "percolation" in key:
        del sys.modules[key]
import social as S, os
from percolation import check as c

snapdirs = os.listdir("./fb/")
snapdirs = [i for i in snapdirs if os.path.isdir("./fb/" + i)]
for snapdir in snapdirs:
    files = os.listdir("./fb/" + snapdir)
    files = [i for i in files if (i.endswith(".rdf") or i.endswith(".ttl"))]
    files.sort()
    for file_ in files:
        c("loading: ", file_, len(S.P.percolation_graph))
        if file_.endswith(".rdf"):
            S.P.percolation_graph.parse("./fb/" + snapdir + "/" + file_)
        elif file_.endswith(".ttl"):
            S.P.percolation_graph.parse("./fb/" + snapdir + "/" + file_,
                                        format="turtle")
        else:
            raise ValueError("Only rdf files please")
        c("loaded: ", file_, "ntriples in percolation graph: ",
          len(S.P.percolation_graph))