def search_engine(graphdb):
# setup
engine = Engine("search")
engine.search.setup(in_name="request", out_name="graph")
## Search
def Search(query, **kwargs):
query, graph = db_graph(graphdb, query)
gid = query['graph']
q = kwargs.pop("URI")
# field = kwargs.pop("field", None)
#g = query_istex(gid, q, field)
g = query_rdf(gid, q)
graph = merge(gid, graph, g)
nodes = query['nodes']
#g = graph_articles(gid, graph, weighting=["1"], all_articles=True, cut=100, uuids=nodes, **kwargs )
return graph
search = Optionable("RDFSearch")
search._func = Search
search.add_option(
"URI", Text(default=u"http://silene.magistry.fr/data/nan/sinogram/好"))
# search.add_option("field", Text(choices=[ u"*", u"istex", u"auteurs", u"refBibAuteurs", u"keywords" ], default=u"*"))
# search.add_option("results_count", Numeric( vtype=int, min=1, default=10, help="Istex results count"))
engine.search.set(search)
return engine
def explore_engine(graphdb):
""" Prox engine """
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
def subgraph(query, size=50):
gid = query['graph']
uuids = [ q for q in query['units']]
return expand_subgraph(graphdb, gid, uuids, limit= size/len(uuids) if len(uuids) else size )
#return prox_subgraph(graphdb, gid, pzeros, size=size)
graph_search = Optionable("GraphSearch")
graph_search._func = Composable(subgraph)
graph_search.add_option("size", Numeric( vtype=int, default=50))
from cello.graphs.transform import VtxAttr
graph_search |= VtxAttr(color=[(45, 200, 34), ])
graph_search |= VtxAttr(type=1)
engine.graph.set(graph_search)
return engine
def export_calc_engine(graphdb):
def _export_calc(query, calc_id=None, **kwargs):
if calc_id == None:
return {'message': "No calc_id ", 'gid': calc_id, 'url': ""}
query, graph = db_graph(graphdb, query)
url = "http://calc.padagraph.io/_/cillex-%s" % calc_id
print "_export_calc", query, calc_id, url
headers, rows = istex.graph_to_calc(graph)
print("* PUT %s %s " % (url, len(rows)))
r = requests.put(url, data=istex.to_csv(headers, rows))
url = "http://calc.padagraph.io/cillex-%s" % calc_id
return {'message': "Calc exported ", 'gid': calc_id, 'url': url}
export = Optionable("export_calc")
export._func = _export_calc
export.add_option(
"calc_id",
Text(
default=None,
help=
"identifiant du calc, le calc sera sauvegardé vers l’adresse http://calc.padagraph.io/cillex-{calc-id}"
))
engine = Engine("export")
engine.export.setup(in_name="request", out_name="url")
engine.export.set(export)
return engine
def import_calc_engine(graphdb):
def _import_calc(query, calc_id=None, **kwargs):
query, graph = db_graph(graphdb, query)
if calc_id == None:
return None
url = "http://calc.padagraph.io/cillex-%s" % calc_id
graph = istex.pad_to_graph(calc_id, url)
graph['meta']['pedigree'] = pedigree.compute(graph)
graph['properties']['description'] = url
graphdb.graphs[calc_id] = graph
return graph_articles(calc_id, graph, cut=100)
comp = Optionable("import_calc")
comp._func = _import_calc
comp.add_option(
"calc_id",
Text(
default=None,
help=
"identifiant du calc,le calc sera importé depuis l'adresse http://calc.padagraph.io/cillex-{calc-id}"
))
engine = Engine("import_calc")
engine.import_calc.setup(in_name="request", out_name="graph")
engine.import_calc.set(comp)
return engine
def expand_prox_engine(graphdb):
"""
prox with weights and filters on UNodes and UEdges types
input: {
nodes : [ uuid, .. ], //more complex p0 distribution
weights: [float, ..], //list of weight
}
output: {
graph : gid,
scores : [ (uuid_node, score ), .. ]
}
"""
engine = Engine("scores")
engine.scores.setup(in_name="request", out_name="scores")
@Composable
def Expand(query, **kwargs):
query, graph = db_graph(graphdb, query)
gid = query.get("graph")
field = "*"
expand = query['expand']
nodes = query['nodes']
vs = graph.vs.select(uuid_in=expand)
if len(vs) == 0:
raise ValueError('No such node %s' % expand)
v = vs[0]
if (v['nodetype'] == ("_%s_auteurs" % gid)):
field = "auteurs"
q = v['properties']['label']
elif (v['nodetype'] == ("_%s_refBibAuteurs" % gid)):
field = "refBibAuteurs"
q = v['properties']['label']
elif (v['nodetype'] == ("_%s_keywords" % gid)):
field = "keywords"
q = v['properties']['label']
else:
q = v['properties']['label']
g = query_istex(gid, q, field)
graph = merge(gid, graph, g, index=index, vid=vid)
pz = [v.index]
vs = extract(graph, pz, **kwargs)
vs = [(graph.vs[i]['uuid'], v) for i, v in vs]
articles = [(v['uuid'], 1.) for v in graph.vs
if v['nodetype'] == ("_%s_article" % gid)]
return dict(articles + vs)
scores = Optionable("scores")
scores._func = Expand
scores.name = "expand"
engine.scores.set(scores)
return engine
def expand_prox_engine(graphdb):
"""
prox with weights and filters on UNodes and UEdges types
input: {
nodes : [ uuid, .. ], //more complex p0 distribution
weights: [float, ..], //list of weight
}
output: {
graph : gid,
scores : [ (uuid_node, score ), .. ]
}
"""
engine = Engine("scores")
engine.scores.setup(in_name="request", out_name="scores")
@Composable
def Expand(query, **kwargs):
query, graph = db_graph(graphdb, query)
gid = query.get("graph")
field = "*"
nodes = query['nodes']
vs = graph.vs.select(uuid_in=nodes)
if len(vs) == 0:
raise ValueError('No such node %s' % nodes)
v = vs[0]
q = v['properties']['URI']
if (v['nodetype'] == "Entity"):
q = v['properties']['URI']
elif (v['nodetype'] == "Literal"):
q = v['properties']['id']
print(q)
g = query_rdf(gid, q, escape=True)
graph = merge(gid, graph, g)
pz = [v.index]
vs = extract(graph, pz, **kwargs)
print(vs)
vs = [(graph.vs[i]['uuid'], v) for i, v in vs]
# articles = [ (v['uuid'], 1.) for v in graph.vs if v['nodetype'] == ("_%s_article" % gid) ]
return dict(vs)
scores = Optionable("scores")
scores._func = Expand
scores.name = "expand"
engine.scores.set(scores)
return engine
def starred_engine(graphdb):
""" Prox engine """
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
def subgraph(query, limit=200, prune=False):
"""
:param mode:
"""
graph = db_graph(graphdb, query)
return starred(graph, limit=100, prune=True)
graph_search = Optionable("GraphSearch")
graph_search._func = Composable(subgraph)
graph_search.add_option("limit", Numeric(vtype=int, default=200))
graph_search.add_option("prune", Boolean(default=True))
from cello.graphs.transform import VtxAttr
graph_search |= VtxAttr(color=[
(45, 200, 34),
])
graph_search |= VtxAttr(type=1)
engine.graph.set(graph_search)
return engine
def additive_nodes_engine(graphdb):
""" Additive engine
add one or more nodes to the current graph,
needs to know all node actually in the local graph in order to send edges to the client.
POST { request : {
graph : gid,
nodes : [ uuid, uuid, ... ] # current local nodes
add : [ uuid, uuid, ... ] # objects to add to local graph
}
will return a subgraphs with all connections between `nodes` and `add`
"""
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
def subgraph(request):
gid = request['graph']
to_add = set(request['add'])
uuids = set([ q for q in request['nodes']] + list(to_add))
# TODO :: dont get subgraph from N
edge_list = graphdb.get_edge_list(gid, uuids)
edge_list = [ e for e in edge_list if e['source'] in to_add or e['target'] in uuids ]
#nodes = { uuid : graphdb.get_node(uuid) for uuid in uuids }
nodes = { n['uuid']: n for n in graphdb.get_nodes(gid, uuids)}
graph = to_graph(nodes, edge_list)
return graph
graph_search = Optionable("AdditiveEngine")
graph_search._func = Composable(subgraph)
from cello.graphs.transform import VtxAttr
graph_search |= VtxAttr(color=[(45, 200, 34), ])
graph_search |= VtxAttr(type=1)
engine.graph.set(graph_search)
return engine
def expand_prox_engine(graphdb):
"""
prox with weights and filters on UNodes and UEdges types
input: {
nodes : [ uuid, .. ], //more complex p0 distribution
weights: [float, ..], //list of weight
}
output: {
graph : gid,
scores : [ (uuid_node, score ), .. ]
}
"""
engine = Engine("scores")
engine.scores.setup(in_name="request", out_name="scores")
## Search
def expand(query, length=3, cut=300, weightings=None):
graph = db_graph(graphdb, query)
gid = query.get("graph")
nodes = query.get("nodes", [])
expand = query.get("expand", [])
vs = expand_subgraph(graph,
expand,
nodes,
cut=cut,
weightings=weightings)
vs = [(graph.vs[v[0]]['uuid'], v[1]) for v in vs]
return dict(vs)
scores = Optionable("scores")
scores._func = Composable(expand)
scores.add_option("length", Numeric(vtype=int, default=3))
scores.add_option("cut", Numeric(vtype=int, default=50, max=300))
scores.add_option(
"weighting",
Text(choices=[u"0", u"1", u"weight"],
multi=True,
default=u"1",
help="ponderation"))
engine.scores.set(expand)
return engine
def starred_engine(graphdb):
""" Prox engine """
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
def subgraph(query, limit=200, prune=False):
"""
:param mode:
"""
gid = query['graph']
uuids = graphdb.get_starred_node_uuids(gid)
if len(uuids) == 0 :
graph = igraph.Graph(directed=True,
graph_attrs={},
n=0,
vertex_attrs={},
edges=[],
edge_attrs={})
if len(uuids) == 1 :
#FIXME: issue #78
mode = "prox"
graph = expand_subgraph(graphdb, gid, uuids, limit=limit)
elif len(uuids) <= 5:
mode = "expand"
graph = expand_subgraph(graphdb, gid, uuids, limit= limit/len(uuids) if len(uuids) else 0. )
else:
mode = "nodes"
uuids = uuids[:limit]
graph = nodes_subgraph(graphdb, gid, uuids)
if prune :
graph = _prune(graph)
return graph
graph_search = Optionable("GraphSearch")
graph_search._func = Composable(subgraph)
graph_search.add_option("limit", Numeric( vtype=int, default=200))
graph_search.add_option("prune", Boolean(default=True))
from cello.graphs.transform import VtxAttr
graph_search |= VtxAttr(color=[(45, 200, 34), ])
graph_search |= VtxAttr(type=1)
engine.graph.set(graph_search)
return engine
def search_engine(graphdb):
# setup
engine = Engine("search")
engine.search.setup(in_name="request", out_name="graph")
## Search
def Search(query, results_count=10, **kwargs):
query, graph = db_graph(graphdb, query)
gid = query['graph']
q = kwargs.pop("q", "*")
field = kwargs.pop("field", None)
g = query_istex(gid, q, field, results_count)
graph = merge(gid, graph, g, index=index, vid=vid)
nodes = query['nodes']
g = graph_articles(gid,
graph,
weighting=["1"],
all_articles=True,
cut=100,
uuids=nodes,
**kwargs)
return g
search = Optionable("IstexSearch")
search._func = Search
search.add_option("q", Text(default=u"clle erss"))
search.add_option(
"field",
Text(choices=[
u"*", u"istex", u"auteurs", u"refBibAuteurs", u"keywords"
],
default=u"*"))
search.add_option(
"results_count",
Numeric(vtype=int, min=1, default=10, help="Istex results count"))
engine.search.set(search)
return engine
def expand_prox_engine(graphdb):
"""
prox with weights and filters on UNodes and UEdges types
input: {
nodes : [ uuid, .. ], //more complex p0 distribution
weights: [float, ..], //list of weight
}
output: {
graph : gid,
scores : [ (uuid_node, score ), .. ]
}
"""
engine = Engine("scores")
engine.scores.setup(in_name="request", out_name="scores")
## Search
def expand(query, step=3, limit=100, filter_nodes=None, filter_edges=None):
if filter_nodes is None :
filter_nodes = []
if filter_edges is None:
filter_edges = []
gid = query.get("graph")
pzeros = query.get("nodes")
weights = query.get("weights", [])
return graphdb.proxemie( gid, pzeros, weights, filter_edges=filter_edges, filter_nodes=filter_nodes, limit=limit, n_step=step)
scores = Optionable("scores")
scores._func = Composable(expand)
scores.add_option("step", Numeric( vtype=int, default=3))
scores.add_option("limit", Numeric( vtype=int, default=50, max=100))
scores.add_option("filter_nodes", Text( default=set([]), multi=True, uniq=True))
scores.add_option("filter_edges", Text( default=set([]), multi=True, uniq=True))
engine.scores.set(expand)
return engine
def clusters_labels_engine(graphdb):
def _labels(query, weighting=None, count=2, **kwargs):
query, graph = db_graph(graphdb, query)
gid = query['graph']
clusters = []
for clust in query['clusters']:
labels = []
pz = graph.vs.select(uuid_in=clust)
pz = [
v.index for v in pz if v['nodetype'] == ("_%s_article" % gid)
]
if len(pz):
vs = extract(graph, pz, cut=300, weighting=weighting, length=3)
labels = [{
'uuid': graph.vs[i]['uuid'],
'label': graph.vs[i]['properties']['label'],
'score': v
} for i, v in vs
if graph.vs[i]['nodetype'] != ("_%s_article" % gid)
][:count]
clusters.append(labels)
return clusters
comp = Optionable("labels")
comp._func = _labels
comp.add_option(
"weighting",
Text(choices=[
u"0", u"1", u"weight", u"auteurs", u"refBibAuteurs", u"keywords",
u"categories"
],
multi=True,
default=u"1",
help="ponderation"))
comp.add_option("count", Numeric(vtype=int, min=1, default=2))
engine = Engine("labels")
engine.labels.setup(in_name="request", out_name="labels")
engine.labels.set(comp)
return engine
def engine(index):
""" Return a default engine over a lexical graph
"""
# setup
from reliure.engine import Engine
engine = Engine("graph", "clustering", "labelling", "layout")
engine.graph.setup(in_name="query", out_name="graph")
engine.clustering.setup(in_name="graph", out_name="clusters")
engine.labelling.setup(in_name="clusters",
out_name="clusters",
hidden=True)
engine.layout.setup(in_name="graph", out_name="layout")
## Search
def tmuse_subgraph(query, length=50):
return tmuse.subgraph(index, query, length=length)
graph_search = Optionable("GraphSearch")
graph_search._func = Composable(tmuse_subgraph)
graph_search.add_option("length", Numeric(vtype=int, default=50))
from cello.graphs.transform import VtxAttr
graph_search |= VtxAttr(color=[
(45, 200, 34),
])
graph_search |= VtxAttr(type=1)
engine.graph.set(graph_search)
## Clustering
from cello.graphs.transform import EdgeAttr
from cello.clustering.common import Infomap, Walktrap
#RMQ infomap veux un pds, donc on en ajoute un bidon
walktrap = EdgeAttr(weight=1.) | Walktrap()
infomap = EdgeAttr(weight=1.) | Infomap()
engine.clustering.set(infomap, walktrap)
## Labelling
from cello.clustering.labelling.model import Label
from cello.clustering.labelling.basic import VertexAsLabel, TypeFalseLabel, normalize_score_max
def _labelling(graph, cluster, vtx):
score = TypeFalseLabel.scoring_prop_ofclust(graph, cluster, vtx)
return Label(vtx["form"], score=score, role="default")
labelling = VertexAsLabel(_labelling) | normalize_score_max
engine.labelling.set(labelling)
## Layout
from cello.layout.simple import KamadaKawaiLayout
#from cello.layout.proxlayout import ProxLayoutRandomProj
from cello.layout.proxlayout import ProxLayoutPCA
from cello.layout.transform import Shaker
from cello.layout.transform import ByConnectedComponent, normalise
default_layout = ProxLayoutPCA(dim=3,
name="ProxPca3d") | Shaker(kelastic=.9)
default_layout = KamadaKawaiLayout(
dim=3, name="KamadaKawaiLayout") | Shaker(kelastic=.9)
engine.layout.set(
ByConnectedComponent(default_layout) | normalise,
#KamadaKawaiLayout(dim=3, name="KamadaKawai3D"),
#ProxLayoutPCA(dim=2, name="ProxPca2d") | Shaker(kelastic=1.8),
#KamadaKawaiLayout(dim=2, name="KamadaKawai2D")
)
return engine
def explore_engine(graphdb):
""" Prox engine """
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
@Composable
def get_graph(query, **kwargs):
return db_graph(graphdb, query)
@Composable
def subgraph(query,
cut=100,
weighted=True,
length=7,
mode=ALL,
add_loops=False,
**kwargs):
graph = db_graph(graphdb, query)
idx = {v['uuid']: v.index for v in graph.vs}
uuids = [q for q in query.get('units', [])]
uuids = [idx[p] for p in uuids]
return prox_subgraph(graph,
uuids,
cut=cut,
weighted=weighted,
length=length,
mode=mode,
add_loops=add_loops,
**kwargs)
from cello.graphs.transform import VtxAttr
searchs = []
for k, w, l, m, n in [
(u"Search", True, 3, ALL, 100),
]:
search = Optionable("GraphSearch")
search._func = subgraph
search.add_option("weighted", Boolean(default=w))
search.add_option("add_loops",
Boolean(default=True, help="add loops on vertices"))
search.add_option(
"mode",
Numeric(choices=[OUT, IN, ALL], default=m, help="edge directions"))
search.add_option("length", Numeric(vtype=int, min=1, default=l))
search.add_option("cut", Numeric(vtype=int, min=2, default=n))
search |= VtxAttr(color=[
(45, 200, 34),
])
search |= VtxAttr(type=1)
search.name = k
searchs.append(search)
sglobal = get_graph | ProxSubgraph()
sglobal.name = "Global"
sglobal.change_option_default("cut", 200)
searchs.append(sglobal)
engine.graph.set(*searchs)
return engine
def explore_engine(graphdb):
""" Prox engine """
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
@Composable
def get_graph(query, **kwargs):
gid = query['graph']
graph = graphdb.get_graph(gid)
return graph
@Composable
def subgraph(
query,
cut=50,
weighted=True,
length=3,
mode=ALL,
add_loops=False,
):
graph = get_graph(query)
uuids = {v['uuid']: v.index for v in graph.vs}
pz = [q for q in query['units']]
pz = [uuids[p] for p in pz]
extract = ProxExtract()
vs = []
for u in pz:
s = extract(graph,
pzeros=[u],
weighted=weighted,
mode=mode,
cut=cut,
length=length)
vs = pz + vs + list(s.keys())
return graph.subgraph(vs)
from cello.graphs.transform import VtxAttr
searchs = []
for k, w, l, m, n in [(u"Espaces_sémantiques", True, 3, OUT, 30),
(u"Espaces_sémantiques_élargis", True, 4, OUT, 50),
(u"Espaces_lexicaux", False, 3, OUT, 30),
(u"Espaces_lexicaux_élargis", False, 4, OUT, 50)]:
search = Optionable("GraphSearch")
search._func = subgraph
search.add_option("weighted", Boolean(default=w))
search.add_option("add_loops",
Boolean(default=True, help="add loops on vertices"))
search.add_option(
"mode",
Numeric(choices=[IN, OUT, ALL], default=m, help="edge directions"))
search.add_option("length", Numeric(vtype=int, min=1, default=l))
search.add_option("cut", Numeric(vtype=int, min=2, default=n))
search |= VtxAttr(color=[
(45, 200, 34),
])
search |= VtxAttr(type=1)
search.name = k
searchs.append(search)
sglobal = Composable(get_graph) | ProxSubgraph()
sglobal.name = "Global"
#searchs.append(sglobal)
engine.graph.set(*searchs)
return engine
def graph_engine(graphdb):
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
def _global(query, reset=False, all_articles=False, cut=100, **kwargs):
gid = query['graph']
query, graph = db_graph(graphdb, query)
nodes = [] if reset else query['nodes']
g = graph_articles(gid,
graph,
all_articles=all_articles,
cut=cut,
uuids=nodes,
**kwargs)
return g
comp = Optionable("Graph")
comp._func = _global
comp.add_option("reset", Boolean(default=False, help="reset or add"))
comp.add_option("all_articles",
Boolean(default=False, help="includes all articles"))
comp.add_option(
"weighting",
Text(choices=[
u"0", u"1", u"weight", u"auteurs", u"refBibAuteurs", u"keywords",
u"categories"
],
multi=True,
default=u"1",
help="ponderation"))
comp.add_option("length", Numeric(vtype=int, min=1, default=3))
comp.add_option("cut", Numeric(vtype=int, min=2, default=100))
def _reset_global(query, **kwargs):
gid = query['graph']
graph = empty_graph(gid, **kwargs)
graphdb.graphs[gid] = graph
g = graph_articles(gid, graph, all_articles=True, uuids=[], **kwargs)
return g
reset = Optionable('ResetGraph')
reset._func = _reset_global
reset.add_option("reset", Boolean(default=True, help=""), hidden=True)
engine.graph.set(comp, reset)
return engine
