def test_configure(self):
engine = Engine("op1", "op2")
engine.set("op1", self.plus_comp, self.mult_opt, self.max_comp)
engine.set("op2", self.plus_comp)
assert engine.op1["mult_opt"].get_option_value("factor") == 5
engine.configure({
'op1':{
'name': 'mult_opt',
'options': {
'factor': '10'
}
},
'op2':{
'name': 'plus_comp'
}
})
assert engine.op1.selected() == ["mult_opt"]
assert engine.op1["mult_opt"].get_option_value("factor") == 10
assert engine.op1.play(5) == {'op1': 50}
# A new call to configure should reset the default values
engine.configure({}) #this should reset the selected/options to default state
assert engine.op1.selected() == ["plus_comp"]
assert engine.op1["mult_opt"].get_option_value("factor") == 5
def starred_engine(graphdb):
""" Prox engine """
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
def subgraph(query, limit=200, prune=False):
"""
:param mode:
"""
graph = db_graph(graphdb, query)
return starred(graph, limit=100, prune=True)
graph_search = Optionable("GraphSearch")
graph_search._func = Composable(subgraph)
graph_search.add_option("limit", Numeric(vtype=int, default=200))
graph_search.add_option("prune", Boolean(default=True))
from cello.graphs.transform import VtxAttr
graph_search |= VtxAttr(color=[
(45, 200, 34),
])
graph_search |= VtxAttr(type=1)
engine.graph.set(graph_search)
return engine
def search_engine(graphdb):
# setup
engine = Engine("search")
engine.search.setup(in_name="request", out_name="graph")
## Search
def Search(query, **kwargs):
query, graph = db_graph(graphdb, query)
gid = query['graph']
q = kwargs.pop("URI")
# field = kwargs.pop("field", None)
#g = query_istex(gid, q, field)
g = query_rdf(gid, q)
graph = merge(gid, graph, g)
nodes = query['nodes']
#g = graph_articles(gid, graph, weighting=["1"], all_articles=True, cut=100, uuids=nodes, **kwargs )
return graph
search = Optionable("RDFSearch")
search._func = Search
search.add_option(
"URI", Text(default=u"http://silene.magistry.fr/data/nan/sinogram/好"))
# search.add_option("field", Text(choices=[ u"*", u"istex", u"auteurs", u"refBibAuteurs", u"keywords" ], default=u"*"))
# search.add_option("results_count", Numeric( vtype=int, min=1, default=10, help="Istex results count"))
engine.search.set(search)
return engine
def export_calc_engine(graphdb):
def _export_calc(query, calc_id=None, **kwargs):
if calc_id == None:
return {'message': "No calc_id ", 'gid': calc_id, 'url': ""}
query, graph = db_graph(graphdb, query)
url = "http://calc.padagraph.io/_/cillex-%s" % calc_id
print "_export_calc", query, calc_id, url
headers, rows = istex.graph_to_calc(graph)
print("* PUT %s %s " % (url, len(rows)))
r = requests.put(url, data=istex.to_csv(headers, rows))
url = "http://calc.padagraph.io/cillex-%s" % calc_id
return {'message': "Calc exported ", 'gid': calc_id, 'url': url}
export = Optionable("export_calc")
export._func = _export_calc
export.add_option(
"calc_id",
Text(
default=None,
help=
"identifiant du calc, le calc sera sauvegardé vers l’adresse http://calc.padagraph.io/cillex-{calc-id}"
))
engine = Engine("export")
engine.export.setup(in_name="request", out_name="url")
engine.export.set(export)
return engine
def explore_engine(graphdb):
""" Prox engine """
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
def subgraph(query, size=50):
gid = query['graph']
uuids = [ q for q in query['units']]
return expand_subgraph(graphdb, gid, uuids, limit= size/len(uuids) if len(uuids) else size )
#return prox_subgraph(graphdb, gid, pzeros, size=size)
graph_search = Optionable("GraphSearch")
graph_search._func = Composable(subgraph)
graph_search.add_option("size", Numeric( vtype=int, default=50))
from cello.graphs.transform import VtxAttr
graph_search |= VtxAttr(color=[(45, 200, 34), ])
graph_search |= VtxAttr(type=1)
engine.graph.set(graph_search)
return engine
def setUp(self):
self.engine = Engine("op1", "op2")
self.engine.op1.setup(in_name="in", out_name="middle", required=False)
self.engine.op2.setup(in_name="middle", out_name="out")
self.engine.op1.set(OptProductEx())
foisdouze = OptProductEx("foisdouze")
foisdouze.force_option_value("factor", 12)
self.engine.op2.set(foisdouze, OptProductEx())
egn_view = EngineView(self.engine, name="my_egn")
egn_view.add_input("in", Numeric(vtype=int, min=-5, max=5))
egn_view.add_input("middle", Numeric(vtype=int))
print(self.engine.needed_inputs())
egn_view.add_output("in")
egn_view.add_output("middle")
egn_view.add_output("out")
api = ReliureAPI()
api.register_view(egn_view)
app = Flask(__name__)
app.config['TESTING'] = True
app.register_blueprint(api, url_prefix="/api")
self.app = app.test_client()
def test_engine_multi_entry(self):
engine = Engine("op1", "op2")
engine.op1.set(self.mult_opt)
engine.op1.setup(in_name="in1", out_name="middle")
engine.op2.set(MinusTwoInputs())
engine.op2.setup(in_name=["in2", "middle"], out_name="out")
# an input is missing
with self.assertRaises(ReliureError):
res = engine.play(10)
res = engine.play(in1=10, in2=2)
assert len(res) == 4
assert res["middle"] == 50
assert res["out"] == -48
def import_calc_engine(graphdb):
def _import_calc(query, calc_id=None, **kwargs):
query, graph = db_graph(graphdb, query)
if calc_id == None:
return None
url = "http://calc.padagraph.io/cillex-%s" % calc_id
graph = istex.pad_to_graph(calc_id, url)
graph['meta']['pedigree'] = pedigree.compute(graph)
graph['properties']['description'] = url
graphdb.graphs[calc_id] = graph
return graph_articles(calc_id, graph, cut=100)
comp = Optionable("import_calc")
comp._func = _import_calc
comp.add_option(
"calc_id",
Text(
default=None,
help=
"identifiant du calc,le calc sera importé depuis l'adresse http://calc.padagraph.io/cillex-{calc-id}"
))
engine = Engine("import_calc")
engine.import_calc.setup(in_name="request", out_name="graph")
engine.import_calc.set(comp)
return engine
def test_engine_named_inout_pipeline(self):
engine = Engine("op1", "op2", "op3")
engine.op1.set(self.mult_opt)
engine.op1.setup(in_name="in1", out_name="out1")
engine.op2.set(self.plus_comp)
engine.op2.setup(in_name="out1", out_name="out2")
engine.op3.set(self.mult_opt)
engine.op3.setup(in_name="out2", out_name="out3")
res = engine.play(3) # mult * 5 | + 2 | mult
assert res['in1'] == 3
assert res['out1'] == 3*5
assert res['out2'] == 3*5+2
assert res['out3'] == (3*5+2)*5
def expand_prox_engine(graphdb):
"""
prox with weights and filters on UNodes and UEdges types
input: {
nodes : [ uuid, .. ], //more complex p0 distribution
weights: [float, ..], //list of weight
}
output: {
graph : gid,
scores : [ (uuid_node, score ), .. ]
}
"""
engine = Engine("scores")
engine.scores.setup(in_name="request", out_name="scores")
@Composable
def Expand(query, **kwargs):
query, graph = db_graph(graphdb, query)
gid = query.get("graph")
field = "*"
expand = query['expand']
nodes = query['nodes']
vs = graph.vs.select(uuid_in=expand)
if len(vs) == 0:
raise ValueError('No such node %s' % expand)
v = vs[0]
if (v['nodetype'] == ("_%s_auteurs" % gid)):
field = "auteurs"
q = v['properties']['label']
elif (v['nodetype'] == ("_%s_refBibAuteurs" % gid)):
field = "refBibAuteurs"
q = v['properties']['label']
elif (v['nodetype'] == ("_%s_keywords" % gid)):
field = "keywords"
q = v['properties']['label']
else:
q = v['properties']['label']
g = query_istex(gid, q, field)
graph = merge(gid, graph, g, index=index, vid=vid)
pz = [v.index]
vs = extract(graph, pz, **kwargs)
vs = [(graph.vs[i]['uuid'], v) for i, v in vs]
articles = [(v['uuid'], 1.) for v in graph.vs
if v['nodetype'] == ("_%s_article" % gid)]
return dict(articles + vs)
scores = Optionable("scores")
scores._func = Expand
scores.name = "expand"
engine.scores.set(scores)
return engine
def test_engine_named_inout_multiin(self):
# a block should work with multiple inputs
engine = Engine("op1", "op2", "op3")
engine.op1.set(self.mult_opt)
engine.op1.setup(in_name="in1", out_name="out1")
engine.op2.set(self.plus_comp)
engine.op2.setup(in_name="in1", out_name="out2")
engine.op3.set(MinusTwoInputs())
engine.op3.setup(in_name=["out1", "out2"], out_name="out3")
res = engine.play(3) # (mult * 5) - (3 + 2)
assert res['in1'] == 3
assert res['out1'] == 3*5
assert res['out2'] == 3+2
assert res['out3'] == (3*5) - (3+2)
# it should be possible to have a multi in as first component !
engine = Engine("op3")
engine.op3.set(MinusTwoInputs())
engine.op3.setup(in_name=["in1", "in2"], out_name="out")
res = engine.play(3, 40)
assert res['out'] == -37
# it should not validate if multiple data are not present
engine = Engine("op1", "op3")
engine.op1.set(self.mult_opt)
engine.op1.setup(in_name="in1", out_name="out1")
engine.op3.set(MinusTwoInputs())
engine.op3.setup(in_name=["out1", "out2"], out_name="out3")
with self.assertRaises(ReliureError):
engine.validate()
def starred_engine(graphdb):
""" Prox engine """
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
def subgraph(query, limit=200, prune=False):
"""
:param mode:
"""
gid = query['graph']
uuids = graphdb.get_starred_node_uuids(gid)
if len(uuids) == 0 :
graph = igraph.Graph(directed=True,
graph_attrs={},
n=0,
vertex_attrs={},
edges=[],
edge_attrs={})
if len(uuids) == 1 :
#FIXME: issue #78
mode = "prox"
graph = expand_subgraph(graphdb, gid, uuids, limit=limit)
elif len(uuids) <= 5:
mode = "expand"
graph = expand_subgraph(graphdb, gid, uuids, limit= limit/len(uuids) if len(uuids) else 0. )
else:
mode = "nodes"
uuids = uuids[:limit]
graph = nodes_subgraph(graphdb, gid, uuids)
if prune :
graph = _prune(graph)
return graph
graph_search = Optionable("GraphSearch")
graph_search._func = Composable(subgraph)
graph_search.add_option("limit", Numeric( vtype=int, default=200))
graph_search.add_option("prune", Boolean(default=True))
from cello.graphs.transform import VtxAttr
graph_search |= VtxAttr(color=[(45, 200, 34), ])
graph_search |= VtxAttr(type=1)
engine.graph.set(graph_search)
return engine
def test_empty_engine(self):
engine = Engine()
# empty should not validate
with self.assertRaises(ReliureError):
engine.validate()
# should not be possible to require some blocks...
with self.assertRaises(ValueError):
engine.requires()
with self.assertRaises(ValueError):
engine.requires("foo", "foo")
def clustering_engine(optionables):
""" Return a default engine over a lexical graph
"""
# setup
engine = Engine("gbuilder", "clustering")
engine.gbuilder.setup(in_name="request", out_name="graph", hidden=True)
engine.clustering.setup(in_name="graph", out_name="clusters")
engine.gbuilder.set(engines.edge_subgraph)
engine.clustering.set(*optionables)
return engine
def test_engine_multi_entry_point(self):
engine = Engine("op1", "op2")
engine.op1.setup(in_name="in1", out_name="middle", required=False)
engine.op2.setup(in_name="middle", out_name="out")
engine.op1.append(self.mult_opt)
engine.op2.append(self.plus_comp)
# should run all the blocks
engine.op1.select("mult_opt")
res = engine.play(10)
assert res["out"] == 52
assert res["middle"] == 50
# should be possible to named the input of the 1st block
res = engine.play(in1=10)
assert res["out"] == 52
assert res["middle"] == 50
# should need input 'in'
with self.assertRaises(ReliureError):
res = engine.play(middle=10)
# should not be possible to give to many inputs
with self.assertRaises(ReliureError):
res = engine.play(middle=10, in1=10)
# should be possible to not play the op1 block
engine.op1.clear_selections()
res = engine.play(middle=10)
assert len(res) == 2
assert res["middle"] == 10
assert res["out"] == 12
def expand_prox_engine(graphdb):
"""
prox with weights and filters on UNodes and UEdges types
input: {
nodes : [ uuid, .. ], //more complex p0 distribution
weights: [float, ..], //list of weight
}
output: {
graph : gid,
scores : [ (uuid_node, score ), .. ]
}
"""
engine = Engine("scores")
engine.scores.setup(in_name="request", out_name="scores")
@Composable
def Expand(query, **kwargs):
query, graph = db_graph(graphdb, query)
gid = query.get("graph")
field = "*"
nodes = query['nodes']
vs = graph.vs.select(uuid_in=nodes)
if len(vs) == 0:
raise ValueError('No such node %s' % nodes)
v = vs[0]
q = v['properties']['URI']
if (v['nodetype'] == "Entity"):
q = v['properties']['URI']
elif (v['nodetype'] == "Literal"):
q = v['properties']['id']
print(q)
g = query_rdf(gid, q, escape=True)
graph = merge(gid, graph, g)
pz = [v.index]
vs = extract(graph, pz, **kwargs)
print(vs)
vs = [(graph.vs[i]['uuid'], v) for i, v in vs]
# articles = [ (v['uuid'], 1.) for v in graph.vs if v['nodetype'] == ("_%s_article" % gid) ]
return dict(vs)
scores = Optionable("scores")
scores._func = Expand
scores.name = "expand"
engine.scores.set(scores)
return engine
def graph_engine(graphdb):
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
def _global(query, reset=False, all_articles=False, cut=100, **kwargs):
gid = query['graph']
query, graph = db_graph(graphdb, query)
nodes = [] if reset else query['nodes']
g = graph_articles(gid,
graph,
all_articles=all_articles,
cut=cut,
uuids=nodes,
**kwargs)
return g
comp = Optionable("Graph")
comp._func = _global
comp.add_option("reset", Boolean(default=False, help="reset or add"))
comp.add_option("all_articles",
Boolean(default=False, help="includes all articles"))
comp.add_option(
"weighting",
Text(choices=[
u"0", u"1", u"weight", u"auteurs", u"refBibAuteurs", u"keywords",
u"categories"
],
multi=True,
default=u"1",
help="ponderation"))
comp.add_option("length", Numeric(vtype=int, min=1, default=3))
comp.add_option("cut", Numeric(vtype=int, min=2, default=100))
def _reset_global(query, **kwargs):
gid = query['graph']
headers = istex.get_schema()
graph = empty_graph(gid, headers, **kwargs)
graphdb.graphs[gid] = graph
g = graph_articles(gid, graph, all_articles=True, uuids=[], **kwargs)
return g
reset = Optionable('ResetGraph')
reset._func = _reset_global
reset.add_option("reset", Boolean(default=True, help=""), hidden=True)
engine.graph.set(comp, reset)
return engine
def expand_prox_engine(graphdb):
"""
prox with weights and filters on UNodes and UEdges types
input: {
nodes : [ uuid, .. ], //more complex p0 distribution
weights: [float, ..], //list of weight
}
output: {
graph : gid,
scores : [ (uuid_node, score ), .. ]
}
"""
engine = Engine("scores")
engine.scores.setup(in_name="request", out_name="scores")
## Search
def expand(query, length=3, cut=300, weightings=None):
graph = db_graph(graphdb, query)
gid = query.get("graph")
nodes = query.get("nodes", [])
expand = query.get("expand", [])
vs = expand_subgraph(graph,
expand,
nodes,
cut=cut,
weightings=weightings)
vs = [(graph.vs[v[0]]['uuid'], v[1]) for v in vs]
return dict(vs)
scores = Optionable("scores")
scores._func = Composable(expand)
scores.add_option("length", Numeric(vtype=int, default=3))
scores.add_option("cut", Numeric(vtype=int, default=50, max=300))
scores.add_option(
"weighting",
Text(choices=[u"0", u"1", u"weight"],
multi=True,
default=u"1",
help="ponderation"))
engine.scores.set(expand)
return engine
def layout_engine(layouts):
""" Return a default engine over a lexical graph
"""
# setup
engine = Engine("gbuilder", "layout", "export")
engine.gbuilder.setup(in_name="request", out_name="graph", hidden=True)
engine.layout.setup(in_name="graph", out_name="layout")
engine.export.setup(in_name=["graph", "layout"], out_name="layout", hidden=True)
engine.gbuilder.set(engines.edge_subgraph)
for k,v in layouts:
v.name = k
layouts = [ l for n,l in layouts ]
engine.layout.set( *layouts )
engine.export.set( export_layout )
return engine
def search_engine(graphdb):
# setup
engine = Engine("search")
engine.search.setup(in_name="request", out_name="graph")
## Search
def Search(query, results_count=10, **kwargs):
query, graph = db_graph(graphdb, query)
gid = query['graph']
q = kwargs.pop("q", "*")
field = kwargs.pop("field", None)
g = query_istex(gid, q, field, results_count)
graph = merge(gid, graph, g, index=index, vid=vid)
nodes = query['nodes']
g = graph_articles(gid,
graph,
weighting=["1"],
all_articles=True,
cut=100,
uuids=nodes,
**kwargs)
return g
search = Optionable("IstexSearch")
search._func = Search
search.add_option("q", Text(default=u"clle erss"))
search.add_option(
"field",
Text(choices=[
u"*", u"istex", u"auteurs", u"refBibAuteurs", u"keywords"
],
default=u"*"))
search.add_option(
"results_count",
Numeric(vtype=int, min=1, default=10, help="Istex results count"))
engine.search.set(search)
return engine
def additive_nodes_engine(graphdb):
""" Additive engine
add one or more nodes to the current graph,
needs to know all node actually in the local graph in order to send edges to the client.
POST { request : {
graph : gid,
nodes : [ uuid, uuid, ... ] # current local nodes
add : [ uuid, uuid, ... ] # objects to add to local graph
}
will return a subgraphs with all connections between `nodes` and `add`
"""
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
def subgraph(request):
gid = request['graph']
to_add = set(request['add'])
uuids = set([ q for q in request['nodes']] + list(to_add))
# TODO :: dont get subgraph from N
edge_list = graphdb.get_edge_list(gid, uuids)
edge_list = [ e for e in edge_list if e['source'] in to_add or e['target'] in uuids ]
#nodes = { uuid : graphdb.get_node(uuid) for uuid in uuids }
nodes = { n['uuid']: n for n in graphdb.get_nodes(gid, uuids)}
graph = to_graph(nodes, edge_list)
return graph
graph_search = Optionable("AdditiveEngine")
graph_search._func = Composable(subgraph)
from cello.graphs.transform import VtxAttr
graph_search |= VtxAttr(color=[(45, 200, 34), ])
graph_search |= VtxAttr(type=1)
engine.graph.set(graph_search)
return engine
def clusters_labels_engine(graphdb):
def _labels(query, weighting=None, count=2, **kwargs):
query, graph = db_graph(graphdb, query)
gid = query['graph']
clusters = []
for clust in query['clusters']:
labels = []
pz = graph.vs.select(uuid_in=clust)
pz = [
v.index for v in pz if v['nodetype'] == ("_%s_article" % gid)
]
if len(pz):
vs = extract(graph, pz, cut=300, weighting=weighting, length=3)
labels = [{
'uuid': graph.vs[i]['uuid'],
'label': graph.vs[i]['properties']['label'],
'score': v
} for i, v in vs
if graph.vs[i]['nodetype'] != ("_%s_article" % gid)
][:count]
clusters.append(labels)
return clusters
comp = Optionable("labels")
comp._func = _labels
comp.add_option(
"weighting",
Text(choices=[
u"0", u"1", u"weight", u"auteurs", u"refBibAuteurs", u"keywords",
u"categories"
],
multi=True,
default=u"1",
help="ponderation"))
comp.add_option("count", Numeric(vtype=int, min=1, default=2))
engine = Engine("labels")
engine.labels.setup(in_name="request", out_name="labels")
engine.labels.set(comp)
return engine
def expand_prox_engine(graphdb):
"""
prox with weights and filters on UNodes and UEdges types
input: {
nodes : [ uuid, .. ], //more complex p0 distribution
weights: [float, ..], //list of weight
}
output: {
graph : gid,
scores : [ (uuid_node, score ), .. ]
}
"""
engine = Engine("scores")
engine.scores.setup(in_name="request", out_name="scores")
## Search
def expand(query, step=3, limit=100, filter_nodes=None, filter_edges=None):
if filter_nodes is None :
filter_nodes = []
if filter_edges is None:
filter_edges = []
gid = query.get("graph")
pzeros = query.get("nodes")
weights = query.get("weights", [])
return graphdb.proxemie( gid, pzeros, weights, filter_edges=filter_edges, filter_nodes=filter_nodes, limit=limit, n_step=step)
scores = Optionable("scores")
scores._func = Composable(expand)
scores.add_option("step", Numeric( vtype=int, default=3))
scores.add_option("limit", Numeric( vtype=int, default=50, max=100))
scores.add_option("filter_nodes", Text( default=set([]), multi=True, uniq=True))
scores.add_option("filter_edges", Text( default=set([]), multi=True, uniq=True))
engine.scores.set(expand)
return engine
def clustering_engine(optionables):
""" Return a default engine over a lexical graph
"""
# setup
engine = Engine("gbuilder", "clustering", "labelling")
engine.gbuilder.setup(in_name="request", out_name="graph", hidden=True)
engine.clustering.setup(in_name="graph", out_name="clusters")
engine.labelling.setup(in_name="clusters", out_name="clusters", hidden=True)
engine.gbuilder.set(engines.edge_subgraph)
engine.clustering.set(*optionables)
## Labelling
from cello.clustering.labelling.model import Label
from cello.clustering.labelling.basic import VertexAsLabel, TypeFalseLabel, normalize_score_max
def _labelling(graph, cluster, vtx):
return Label(vtx["uuid"], score=1, role="default")
labelling = VertexAsLabel( _labelling ) | normalize_score_max
engine.labelling.set(labelling)
return engine
def test_engine_named_inout_error(self):
engine = Engine("op1", "op2", "op3")
engine.op1.set(self.mult_opt)
engine.op1.setup(in_name="in1", out_name="out1")
engine.op2.set(self.plus_comp)
engine.op2.setup(in_name="out_not_exist", out_name="out2")
engine.op3.set(self.mult_opt)
engine.op3.setup(in_name="out1", out_name="out3")
with self.assertRaises(ReliureError):
engine.validate()
engine.op2.setup(in_name="in1", out_name="out2")
engine.validate()
engine.op2.setup(required=False)
engine.op3.setup(in_name="out2")
with self.assertRaises(ReliureError):
engine.validate()
def test_engine_default_pipeline(self):
engine = Engine("op1", "op2", "op3")
# should have a __len__
assert len(engine) == 3
# should have a __contains__
assert "op1" in engine
assert "op2" in engine
assert "op3" in engine
assert not "op4" in engine
# should not be possible to require a block that do not exist
with self.assertRaises(ReliureError):
engine.requires("op4")
# should be possible to set the component for each block
engine.op1.set(self.mult_opt)
# but not on non existing blocks
with self.assertRaises(ValueError):
engine.op4.set(self.mult_opt)
with self.assertRaises(ValueError):
engine["op4"].set(self.mult_opt)
with self.assertRaises(ValueError):
engine.set("op4", self.mult_opt)
# should not validate if there is no component for all blocks
with self.assertRaises(ReliureError):
engine.validate()
#should be posisble to 'append' a possible component to the block
engine.op2.append(self.mult_opt)
# and to set is as default in the same time
engine.op2.append(self.plus_comp, default=True)
# should be possible to select on component
engine.op1.select("mult_opt")
# should be possible to know if one block is multiple
assert not engine.op2.multiple
# to know wich component will be run
assert engine.op2.selected() == ["plus_comp"]
# to play just one block !
assert engine.op2.play(10) == {'op2': 12}
# and then is should validate is every block has som components
engine.set("op3", self.mult_opt, self.plus_comp, self.max_comp)
engine.validate()
# should play !
res = engine.play(3) # mult * 5 | + 2 | mult
# and all intermediare results should be available
assert res['input'] == 3
assert res['op1'] == 3*5
assert res['op2'] == 3*5+2
assert res['op3'] == (3*5+2)*5
def lexical_graph_engine(graph):
""" Return a default engine over a lexical graph
"""
# setup
engine = Engine()
engine.requires("search", "clustering", "labelling", "layout")
engine.search.setup(in_name="query", out_name="graph")
engine.clustering.setup(in_name="graph", out_name="clusters")
engine.labelling.setup(in_name="clusters", out_name="clusters", hidden=True)
engine.layout.setup(in_name="graph", out_name="layout")
## Search
from cello.graphs.extraction import VtxMatch, ProxMarkovExtractionGlobal, VertexIds
from cello.graphs.builder import Subgraph
#HACK remove the "id" attribute (if any), it enter in conflict when exporting subgraphs to client
if 'id' in graph.vs.attributes():
del graph.vs['id']
# Pour avoir une recherche "_all"
# http://localhost:5000/verb/q/_all
@Composable
def nothing_if_all(query):
""" Make the query be nothing if it is '_all'
"""
if query in ("_all", None):
return "" #Note: p0 to [] make start from all vertices
return query
# real match search
match = VtxMatch(graph, attr_list=[u"label"], default_attr=u"label")
graph_search = nothing_if_all | match
graph_search |= ProxMarkovExtractionGlobal(graph)
graph_search |= Subgraph(graph, score_attr="prox", gdeg_attr="gdeg")
graph_search |= ProxColors(graph, graph['vertices_color'], length=5)
graph_search.name = "ProxSearch"
graph_search.change_option_default("vcount", 50)
engine.search.set(graph_search)
## Clustering
from cello.graphs.transform import EdgeAttr
from cello.clustering.common import Infomap, Walktrap
#RMQ infomap veux un pds, donc on en ajoute un bidon
walktrap = EdgeAttr(weight=1.) |Walktrap()
infomap = EdgeAttr(weight=1.) | Infomap()
engine.clustering.set(walktrap, infomap)
## Labelling
from cello.clustering.labelling.model import Label
from cello.clustering.labelling.basic import VertexAsLabel
vertex_from_vtx = lambda graph, cluster, vtx: Label(vtx["label"], role="default", score=vtx["gdeg"])
engine.labelling.set(VertexAsLabel(vertex_from_vtx))
## Layout
from cello.layout.simple import KamadaKawaiLayout
from cello.layout.proxlayout import ProxLayoutRandomProj
from cello.layout.proxlayout import ProxLayoutPCA
from cello.layout.transform import Shaker
engine.layout.set(
ProxLayoutPCA(dim=3) | Shaker(),
KamadaKawaiLayout(dim=3),
)
return engine
def explore_engine(graphdb):
""" Prox engine """
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
@Composable
def get_graph(query, **kwargs):
return db_graph(graphdb, query)
@Composable
def subgraph(query,
cut=100,
weighted=True,
length=7,
mode=ALL,
add_loops=False,
**kwargs):
graph = db_graph(graphdb, query)
idx = {v['uuid']: v.index for v in graph.vs}
uuids = [q for q in query.get('units', [])]
uuids = [idx[p] for p in uuids]
return prox_subgraph(graph,
uuids,
cut=cut,
weighted=weighted,
length=length,
mode=mode,
add_loops=add_loops,
**kwargs)
from cello.graphs.transform import VtxAttr
searchs = []
for k, w, l, m, n in [
(u"Search", True, 3, ALL, 100),
]:
search = Optionable("GraphSearch")
search._func = subgraph
search.add_option("weighted", Boolean(default=w))
search.add_option("add_loops",
Boolean(default=True, help="add loops on vertices"))
search.add_option(
"mode",
Numeric(choices=[OUT, IN, ALL], default=m, help="edge directions"))
search.add_option("length", Numeric(vtype=int, min=1, default=l))
search.add_option("cut", Numeric(vtype=int, min=2, default=n))
search |= VtxAttr(color=[
(45, 200, 34),
])
search |= VtxAttr(type=1)
search.name = k
searchs.append(search)
sglobal = get_graph | ProxSubgraph()
sglobal.name = "Global"
sglobal.change_option_default("cut", 200)
searchs.append(sglobal)
engine.graph.set(*searchs)
return engine
def explore_engine(graphdb):
""" Prox engine """
# setup
engine = Engine("graph")
engine.graph.setup(in_name="request", out_name="graph")
## Search
@Composable
def get_graph(query, **kwargs):
gid = query['graph']
graph = graphdb.get_graph(gid)
return graph
@Composable
def subgraph(
query,
cut=50,
weighted=True,
length=3,
mode=ALL,
add_loops=False,
):
graph = get_graph(query)
uuids = {v['uuid']: v.index for v in graph.vs}
pz = [q for q in query['units']]
pz = [uuids[p] for p in pz]
extract = ProxExtract()
vs = []
for u in pz:
s = extract(graph,
pzeros=[u],
weighted=weighted,
mode=mode,
cut=cut,
length=length)
vs = pz + vs + list(s.keys())
return graph.subgraph(vs)
from cello.graphs.transform import VtxAttr
searchs = []
for k, w, l, m, n in [(u"Espaces_sémantiques", True, 3, OUT, 30),
(u"Espaces_sémantiques_élargis", True, 4, OUT, 50),
(u"Espaces_lexicaux", False, 3, OUT, 30),
(u"Espaces_lexicaux_élargis", False, 4, OUT, 50)]:
search = Optionable("GraphSearch")
search._func = subgraph
search.add_option("weighted", Boolean(default=w))
search.add_option("add_loops",
Boolean(default=True, help="add loops on vertices"))
search.add_option(
"mode",
Numeric(choices=[IN, OUT, ALL], default=m, help="edge directions"))
search.add_option("length", Numeric(vtype=int, min=1, default=l))
search.add_option("cut", Numeric(vtype=int, min=2, default=n))
search |= VtxAttr(color=[
(45, 200, 34),
])
search |= VtxAttr(type=1)
search.name = k
searchs.append(search)
sglobal = Composable(get_graph) | ProxSubgraph()
sglobal.name = "Global"
#searchs.append(sglobal)
engine.graph.set(*searchs)
return engine
def test_configure_errors(self):
# Should raise valueError when configuration is not valit
engine = Engine("op1", "op2", "op3")
engine.set("op1", self.mult_opt, self.plus_comp, self.max_comp)
engine.set("op2", self.plus_comp, self.mult_opt, self.max_comp)
engine.op2.setup(hidden=True)
engine.set("op3", self.mult_opt, self.plus_comp, self.max_comp)
with self.assertRaises(ValueError): # op2 hidden it can be configured
engine.configure({
'op2':{'name': 'max20'}
})
with self.assertRaises(ValueError): # 'maxmax' doesn't exist
engine.configure({
'op3':{'name': 'maxmax'}
})
with self.assertRaises(ValueError): # error in op1 format
engine.configure({
'op1':{'namss': 'mult'}
})
with self.assertRaises(ValueError): # block doesn't exist
engine.configure({
'op5':{'name': 'max20'}
})
with self.assertRaises(ValueError): # block required !
engine.configure({
'op1':[]
})
with self.assertRaises(ValueError): # block not multi !
engine.configure({
'op1':[{'name': 'max20'}, {'name': 'plus_comp'}]
})
class TestReliureAPIMultiInputs(unittest.TestCase):
maxDiff = None
def setUp(self):
self.engine = Engine("op1", "op2")
self.engine.op1.setup(in_name="in", out_name="middle", required=False)
self.engine.op2.setup(in_name="middle", out_name="out")
self.engine.op1.set(OptProductEx())
foisdouze = OptProductEx("foisdouze")
foisdouze.force_option_value("factor", 12)
self.engine.op2.set(foisdouze, OptProductEx())
egn_view = EngineView(self.engine, name="my_egn")
egn_view.add_input("in", Numeric(vtype=int, min=-5, max=5))
egn_view.add_input("middle", Numeric(vtype=int))
print(self.engine.needed_inputs())
egn_view.add_output("in")
egn_view.add_output("middle")
egn_view.add_output("out")
api = ReliureAPI()
api.register_view(egn_view)
app = Flask(__name__)
app.config['TESTING'] = True
app.register_blueprint(api, url_prefix="/api")
self.app = app.test_client()
def test_routes(self):
resp = self.app.get('api/')
data = json.loads(resp.data.decode("utf-8"))
# test routes
routes = data["routes"]
routes = {route['name']: route for route in routes}
pprint(routes)
# routes GET api/ for this entry point
assert "api.routes" in routes
assert routes["api.routes"]["path"] == u'/api/'
assert sorted(routes["api.routes"]["methods"]) == [u'GET', u'HEAD', u'OPTIONS']
# routes GET api/egn for options
assert "api.my_egn_options" in routes
assert routes["api.my_egn_options"]["path"] == u'/api/my_egn'
assert sorted(routes["api.my_egn_options"]["methods"]) == [u'GET', u'HEAD', u'OPTIONS']
# routes POST api/egn for play
assert "api.my_egn" in routes
assert routes["api.my_egn"]["path"] == u'/api/my_egn'
assert sorted(routes["api.my_egn"]["methods"]) == [u'OPTIONS', u'POST']
def test_options(self):
resp = self.app.get('api/my_egn')
data = json.loads(resp.data.decode("utf-8"))
# check we have the same than in engine
assert data["blocks"] == self.engine.as_dict()["blocks"]
assert data["args"] == ["in", "middle"]
assert data["returns"] == ["in", "middle", "out"]
def test_play_simple(self):
# it should be possible to play the full engine
# prepare query
rdata = {'in': 2}
rdata["options"] = {
'op1': [{
'name': 'mult_opt',
}]
}
json_data = json.dumps(rdata)
resp = self.app.post('api/my_egn', data=json_data, content_type='application/json')
# load the results
resp_data = json.loads(resp.data.decode("utf-8"))
# check it
assert "results" in resp_data
results = resp_data["results"]
assert "out" in results
assert len(results) == 3 # in, middle, out
assert results["out"] == 2*5*12
def test_play_skip_op1(self):
# it should be possible to play the only the block op2
# prepare query
rdata = {'middle': 2}
rdata["options"] = {
'op1': []
}
json_data = json.dumps(rdata)
resp = self.app.post('api/my_egn', data=json_data, content_type='application/json')
# load the results
resp_data = json.loads(resp.data.decode("utf-8"))
# check it
assert "results" in resp_data
results = resp_data["results"]
assert "out" in results
assert len(results) == 2 # middle, out
assert results["out"] == 2*12
def test_as_dict(self):
engine = Engine("op1", "op2")
engine.set("op1", self.plus_comp, self.mult_opt, self.max_comp)
engine.set("op2", self.plus_comp)
assert engine.as_dict() == {
'args': ['input'],
'blocks': [
{
'components': [
{
'default': True, #This is the default NOT the selected value
'name': 'plus_comp',
'options': None
},
{
'default': False,
'name': 'mult_opt',
'options': [
{
'name': 'factor',
'type': 'value',
'value': 5,
'otype': {
'choices': None,
'default': 5,
'help': 'multipliation factor',
'max': None,
'min': None,
'multi': False,
'type': 'Numeric',
'uniq': False,
'vtype': 'int'
}
}
]
},
{
'name': 'max20',
'default': False,
'options': None
}
],
'args': None,
'multiple': False,
'name': 'op1',
'returns': 'op1',
'required': True
},
{
'components': [
{
'name': 'plus_comp',
'default': True,
'options': None
}
],
'args': None,
'multiple': False,
'name': 'op2',
'returns': 'op2',
'required': True
}
]
}
class TestReliureAPISimple(unittest.TestCase):
maxDiff = None
def setUp(self):
self.engine = Engine("op1", "op2")
self.engine.op1.setup(in_name="in")
self.engine.op2.setup(out_name="out")
self.engine.op1.set(OptProductEx())
foisdouze = OptProductEx("foisdouze")
foisdouze.force_option_value("factor", 12)
self.engine.op2.set(foisdouze, OptProductEx())
egn_view = EngineView(self.engine)
egn_view.set_input_type(Numeric(vtype=int, min=-5, max=5))
egn_view.add_output("out")
api = ReliureAPI()
api.register_view(egn_view, url_prefix="egn")
app = Flask(__name__)
self.appp = app
app.config['TESTING'] = True
app.register_blueprint(api, url_prefix="/api")
self.app = app.test_client()
def test_engine_view_init(self):
_egn_view = EngineView(self.engine)
with pytest.raises(ValueError):
_egn_view.add_output("existe_pas")
def test_routes(self):
resp = self.app.get('api/')
data = json.loads(resp.data.decode("utf-8"))
assert data["api"] == "api"
assert data["url_root"] == "http://localhost/"
# test routes
routes = data["routes"]
routes = {route['name']: route for route in routes}
pprint(routes)
# routes GET api/ for this entry point
assert "api.routes" in routes
assert routes["api.routes"]["path"] == u'/api/'
assert sorted(routes["api.routes"]["methods"]) == [u'GET', u'HEAD', u'OPTIONS']
# routes GET api/egn for options
assert "api.egn_options" in routes
assert routes["api.egn_options"]["path"] == u'/api/egn'
assert sorted(routes["api.egn_options"]["methods"]) == [u'GET', u'HEAD', u'OPTIONS']
# routes POST api/egn for play
assert "api.egn" in routes
assert routes["api.egn"]["path"] == u'/api/egn'
assert sorted(routes["api.egn"]["methods"]) == [u'OPTIONS', u'POST']
def test_options(self):
resp = self.app.get('api/egn')
data = json.loads(resp.data.decode("utf-8"))
# check we have the same than in engine
assert data["blocks"] == self.engine.as_dict()["blocks"]
assert data["args"] == ["in"]
assert data["returns"] == ["out"]
def test_play_simple(self):
# prepare query
rdata = {'in': '2'}
json_data = json.dumps(rdata)
resp = self.app.post('api/egn', data=json_data, content_type='application/json')
# load the results
resp_data = json.loads(resp.data.decode("utf-8"))
# check it
assert "results" in resp_data
results = resp_data["results"]
assert "out" in results
assert len(results) == 1
assert results["out"] == 2*5*12
def test_play_nojson(self):
# prepare query
data = {u'in': 3}
resp = self.app.post('api/egn', data=data)
# load the results
resp_data = json.loads(resp.data.decode("utf-8"))
# check it
assert "results" in resp_data
results = resp_data["results"]
assert "out" in results
assert len(results) == 1
assert results["out"] == 3*5*12
def test_play_simple_options(self):
# prepare query
rdata = {'in': '2'}
rdata["options"] = {
'op2': [{
'name': 'mult_opt',
'options': {
'factor': '2'
}
}]
}
json_data = json.dumps(rdata)
resp = self.app.post('api/egn', data=json_data, content_type='application/json')
# load the results
resp_data = json.loads(resp.data.decode("utf-8"))
# check it
assert "results" in resp_data
results = resp_data["results"]
assert "out" in results
assert len(results) == 1
assert results["out"] == 2*2*5
def engine(index):
""" Return a default engine over a lexical graph
"""
# setup
from reliure.engine import Engine
engine = Engine("graph", "clustering", "labelling", "layout")
engine.graph.setup(in_name="query", out_name="graph")
engine.clustering.setup(in_name="graph", out_name="clusters")
engine.labelling.setup(in_name="clusters",
out_name="clusters",
hidden=True)
engine.layout.setup(in_name="graph", out_name="layout")
## Search
def tmuse_subgraph(query, length=50):
return tmuse.subgraph(index, query, length=length)
graph_search = Optionable("GraphSearch")
graph_search._func = Composable(tmuse_subgraph)
graph_search.add_option("length", Numeric(vtype=int, default=50))
from cello.graphs.transform import VtxAttr
graph_search |= VtxAttr(color=[
(45, 200, 34),
])
graph_search |= VtxAttr(type=1)
engine.graph.set(graph_search)
## Clustering
from cello.graphs.transform import EdgeAttr
from cello.clustering.common import Infomap, Walktrap
#RMQ infomap veux un pds, donc on en ajoute un bidon
walktrap = EdgeAttr(weight=1.) | Walktrap()
infomap = EdgeAttr(weight=1.) | Infomap()
engine.clustering.set(infomap, walktrap)
## Labelling
from cello.clustering.labelling.model import Label
from cello.clustering.labelling.basic import VertexAsLabel, TypeFalseLabel, normalize_score_max
def _labelling(graph, cluster, vtx):
score = TypeFalseLabel.scoring_prop_ofclust(graph, cluster, vtx)
return Label(vtx["form"], score=score, role="default")
labelling = VertexAsLabel(_labelling) | normalize_score_max
engine.labelling.set(labelling)
## Layout
from cello.layout.simple import KamadaKawaiLayout
#from cello.layout.proxlayout import ProxLayoutRandomProj
from cello.layout.proxlayout import ProxLayoutPCA
from cello.layout.transform import Shaker
from cello.layout.transform import ByConnectedComponent, normalise
default_layout = ProxLayoutPCA(dim=3,
name="ProxPca3d") | Shaker(kelastic=.9)
default_layout = KamadaKawaiLayout(
dim=3, name="KamadaKawaiLayout") | Shaker(kelastic=.9)
engine.layout.set(
ByConnectedComponent(default_layout) | normalise,
#KamadaKawaiLayout(dim=3, name="KamadaKawai3D"),
#ProxLayoutPCA(dim=2, name="ProxPca2d") | Shaker(kelastic=1.8),
#KamadaKawaiLayout(dim=2, name="KamadaKawai2D")
)
return engine
