def test_configure(self): engine = Engine("op1", "op2") engine.set("op1", self.plus_comp, self.mult_opt, self.max_comp) engine.set("op2", self.plus_comp) assert engine.op1["mult_opt"].get_option_value("factor") == 5 engine.configure({ 'op1':{ 'name': 'mult_opt', 'options': { 'factor': '10' } }, 'op2':{ 'name': 'plus_comp' } }) assert engine.op1.selected() == ["mult_opt"] assert engine.op1["mult_opt"].get_option_value("factor") == 10 assert engine.op1.play(5) == {'op1': 50} # A new call to configure should reset the default values engine.configure({}) #this should reset the selected/options to default state assert engine.op1.selected() == ["plus_comp"] assert engine.op1["mult_opt"].get_option_value("factor") == 5
def starred_engine(graphdb): """ Prox engine """ # setup engine = Engine("graph") engine.graph.setup(in_name="request", out_name="graph") ## Search def subgraph(query, limit=200, prune=False): """ :param mode: """ graph = db_graph(graphdb, query) return starred(graph, limit=100, prune=True) graph_search = Optionable("GraphSearch") graph_search._func = Composable(subgraph) graph_search.add_option("limit", Numeric(vtype=int, default=200)) graph_search.add_option("prune", Boolean(default=True)) from cello.graphs.transform import VtxAttr graph_search |= VtxAttr(color=[ (45, 200, 34), ]) graph_search |= VtxAttr(type=1) engine.graph.set(graph_search) return engine
def search_engine(graphdb): # setup engine = Engine("search") engine.search.setup(in_name="request", out_name="graph") ## Search def Search(query, **kwargs): query, graph = db_graph(graphdb, query) gid = query['graph'] q = kwargs.pop("URI") # field = kwargs.pop("field", None) #g = query_istex(gid, q, field) g = query_rdf(gid, q) graph = merge(gid, graph, g) nodes = query['nodes'] #g = graph_articles(gid, graph, weighting=["1"], all_articles=True, cut=100, uuids=nodes, **kwargs ) return graph search = Optionable("RDFSearch") search._func = Search search.add_option( "URI", Text(default=u"http://silene.magistry.fr/data/nan/sinogram/好")) # search.add_option("field", Text(choices=[ u"*", u"istex", u"auteurs", u"refBibAuteurs", u"keywords" ], default=u"*")) # search.add_option("results_count", Numeric( vtype=int, min=1, default=10, help="Istex results count")) engine.search.set(search) return engine
def export_calc_engine(graphdb): def _export_calc(query, calc_id=None, **kwargs): if calc_id == None: return {'message': "No calc_id ", 'gid': calc_id, 'url': ""} query, graph = db_graph(graphdb, query) url = "http://calc.padagraph.io/_/cillex-%s" % calc_id print "_export_calc", query, calc_id, url headers, rows = istex.graph_to_calc(graph) print("* PUT %s %s " % (url, len(rows))) r = requests.put(url, data=istex.to_csv(headers, rows)) url = "http://calc.padagraph.io/cillex-%s" % calc_id return {'message': "Calc exported ", 'gid': calc_id, 'url': url} export = Optionable("export_calc") export._func = _export_calc export.add_option( "calc_id", Text( default=None, help= "identifiant du calc, le calc sera sauvegardé vers l’adresse http://calc.padagraph.io/cillex-{calc-id}" )) engine = Engine("export") engine.export.setup(in_name="request", out_name="url") engine.export.set(export) return engine
def explore_engine(graphdb): """ Prox engine """ # setup engine = Engine("graph") engine.graph.setup(in_name="request", out_name="graph") ## Search def subgraph(query, size=50): gid = query['graph'] uuids = [ q for q in query['units']] return expand_subgraph(graphdb, gid, uuids, limit= size/len(uuids) if len(uuids) else size ) #return prox_subgraph(graphdb, gid, pzeros, size=size) graph_search = Optionable("GraphSearch") graph_search._func = Composable(subgraph) graph_search.add_option("size", Numeric( vtype=int, default=50)) from cello.graphs.transform import VtxAttr graph_search |= VtxAttr(color=[(45, 200, 34), ]) graph_search |= VtxAttr(type=1) engine.graph.set(graph_search) return engine
def setUp(self): self.engine = Engine("op1", "op2") self.engine.op1.setup(in_name="in", out_name="middle", required=False) self.engine.op2.setup(in_name="middle", out_name="out") self.engine.op1.set(OptProductEx()) foisdouze = OptProductEx("foisdouze") foisdouze.force_option_value("factor", 12) self.engine.op2.set(foisdouze, OptProductEx()) egn_view = EngineView(self.engine, name="my_egn") egn_view.add_input("in", Numeric(vtype=int, min=-5, max=5)) egn_view.add_input("middle", Numeric(vtype=int)) print(self.engine.needed_inputs()) egn_view.add_output("in") egn_view.add_output("middle") egn_view.add_output("out") api = ReliureAPI() api.register_view(egn_view) app = Flask(__name__) app.config['TESTING'] = True app.register_blueprint(api, url_prefix="/api") self.app = app.test_client()
def test_engine_multi_entry(self): engine = Engine("op1", "op2") engine.op1.set(self.mult_opt) engine.op1.setup(in_name="in1", out_name="middle") engine.op2.set(MinusTwoInputs()) engine.op2.setup(in_name=["in2", "middle"], out_name="out") # an input is missing with self.assertRaises(ReliureError): res = engine.play(10) res = engine.play(in1=10, in2=2) assert len(res) == 4 assert res["middle"] == 50 assert res["out"] == -48
def import_calc_engine(graphdb): def _import_calc(query, calc_id=None, **kwargs): query, graph = db_graph(graphdb, query) if calc_id == None: return None url = "http://calc.padagraph.io/cillex-%s" % calc_id graph = istex.pad_to_graph(calc_id, url) graph['meta']['pedigree'] = pedigree.compute(graph) graph['properties']['description'] = url graphdb.graphs[calc_id] = graph return graph_articles(calc_id, graph, cut=100) comp = Optionable("import_calc") comp._func = _import_calc comp.add_option( "calc_id", Text( default=None, help= "identifiant du calc,le calc sera importé depuis l'adresse http://calc.padagraph.io/cillex-{calc-id}" )) engine = Engine("import_calc") engine.import_calc.setup(in_name="request", out_name="graph") engine.import_calc.set(comp) return engine
def test_engine_named_inout_pipeline(self): engine = Engine("op1", "op2", "op3") engine.op1.set(self.mult_opt) engine.op1.setup(in_name="in1", out_name="out1") engine.op2.set(self.plus_comp) engine.op2.setup(in_name="out1", out_name="out2") engine.op3.set(self.mult_opt) engine.op3.setup(in_name="out2", out_name="out3") res = engine.play(3) # mult * 5 | + 2 | mult assert res['in1'] == 3 assert res['out1'] == 3*5 assert res['out2'] == 3*5+2 assert res['out3'] == (3*5+2)*5
def expand_prox_engine(graphdb): """ prox with weights and filters on UNodes and UEdges types input: { nodes : [ uuid, .. ], //more complex p0 distribution weights: [float, ..], //list of weight } output: { graph : gid, scores : [ (uuid_node, score ), .. ] } """ engine = Engine("scores") engine.scores.setup(in_name="request", out_name="scores") @Composable def Expand(query, **kwargs): query, graph = db_graph(graphdb, query) gid = query.get("graph") field = "*" expand = query['expand'] nodes = query['nodes'] vs = graph.vs.select(uuid_in=expand) if len(vs) == 0: raise ValueError('No such node %s' % expand) v = vs[0] if (v['nodetype'] == ("_%s_auteurs" % gid)): field = "auteurs" q = v['properties']['label'] elif (v['nodetype'] == ("_%s_refBibAuteurs" % gid)): field = "refBibAuteurs" q = v['properties']['label'] elif (v['nodetype'] == ("_%s_keywords" % gid)): field = "keywords" q = v['properties']['label'] else: q = v['properties']['label'] g = query_istex(gid, q, field) graph = merge(gid, graph, g, index=index, vid=vid) pz = [v.index] vs = extract(graph, pz, **kwargs) vs = [(graph.vs[i]['uuid'], v) for i, v in vs] articles = [(v['uuid'], 1.) for v in graph.vs if v['nodetype'] == ("_%s_article" % gid)] return dict(articles + vs) scores = Optionable("scores") scores._func = Expand scores.name = "expand" engine.scores.set(scores) return engine
def test_engine_named_inout_multiin(self): # a block should work with multiple inputs engine = Engine("op1", "op2", "op3") engine.op1.set(self.mult_opt) engine.op1.setup(in_name="in1", out_name="out1") engine.op2.set(self.plus_comp) engine.op2.setup(in_name="in1", out_name="out2") engine.op3.set(MinusTwoInputs()) engine.op3.setup(in_name=["out1", "out2"], out_name="out3") res = engine.play(3) # (mult * 5) - (3 + 2) assert res['in1'] == 3 assert res['out1'] == 3*5 assert res['out2'] == 3+2 assert res['out3'] == (3*5) - (3+2) # it should be possible to have a multi in as first component ! engine = Engine("op3") engine.op3.set(MinusTwoInputs()) engine.op3.setup(in_name=["in1", "in2"], out_name="out") res = engine.play(3, 40) assert res['out'] == -37 # it should not validate if multiple data are not present engine = Engine("op1", "op3") engine.op1.set(self.mult_opt) engine.op1.setup(in_name="in1", out_name="out1") engine.op3.set(MinusTwoInputs()) engine.op3.setup(in_name=["out1", "out2"], out_name="out3") with self.assertRaises(ReliureError): engine.validate()
def starred_engine(graphdb): """ Prox engine """ # setup engine = Engine("graph") engine.graph.setup(in_name="request", out_name="graph") ## Search def subgraph(query, limit=200, prune=False): """ :param mode: """ gid = query['graph'] uuids = graphdb.get_starred_node_uuids(gid) if len(uuids) == 0 : graph = igraph.Graph(directed=True, graph_attrs={}, n=0, vertex_attrs={}, edges=[], edge_attrs={}) if len(uuids) == 1 : #FIXME: issue #78 mode = "prox" graph = expand_subgraph(graphdb, gid, uuids, limit=limit) elif len(uuids) <= 5: mode = "expand" graph = expand_subgraph(graphdb, gid, uuids, limit= limit/len(uuids) if len(uuids) else 0. ) else: mode = "nodes" uuids = uuids[:limit] graph = nodes_subgraph(graphdb, gid, uuids) if prune : graph = _prune(graph) return graph graph_search = Optionable("GraphSearch") graph_search._func = Composable(subgraph) graph_search.add_option("limit", Numeric( vtype=int, default=200)) graph_search.add_option("prune", Boolean(default=True)) from cello.graphs.transform import VtxAttr graph_search |= VtxAttr(color=[(45, 200, 34), ]) graph_search |= VtxAttr(type=1) engine.graph.set(graph_search) return engine
def test_empty_engine(self): engine = Engine() # empty should not validate with self.assertRaises(ReliureError): engine.validate() # should not be possible to require some blocks... with self.assertRaises(ValueError): engine.requires() with self.assertRaises(ValueError): engine.requires("foo", "foo")
def clustering_engine(optionables): """ Return a default engine over a lexical graph """ # setup engine = Engine("gbuilder", "clustering") engine.gbuilder.setup(in_name="request", out_name="graph", hidden=True) engine.clustering.setup(in_name="graph", out_name="clusters") engine.gbuilder.set(engines.edge_subgraph) engine.clustering.set(*optionables) return engine
def test_engine_multi_entry_point(self): engine = Engine("op1", "op2") engine.op1.setup(in_name="in1", out_name="middle", required=False) engine.op2.setup(in_name="middle", out_name="out") engine.op1.append(self.mult_opt) engine.op2.append(self.plus_comp) # should run all the blocks engine.op1.select("mult_opt") res = engine.play(10) assert res["out"] == 52 assert res["middle"] == 50 # should be possible to named the input of the 1st block res = engine.play(in1=10) assert res["out"] == 52 assert res["middle"] == 50 # should need input 'in' with self.assertRaises(ReliureError): res = engine.play(middle=10) # should not be possible to give to many inputs with self.assertRaises(ReliureError): res = engine.play(middle=10, in1=10) # should be possible to not play the op1 block engine.op1.clear_selections() res = engine.play(middle=10) assert len(res) == 2 assert res["middle"] == 10 assert res["out"] == 12
def expand_prox_engine(graphdb): """ prox with weights and filters on UNodes and UEdges types input: { nodes : [ uuid, .. ], //more complex p0 distribution weights: [float, ..], //list of weight } output: { graph : gid, scores : [ (uuid_node, score ), .. ] } """ engine = Engine("scores") engine.scores.setup(in_name="request", out_name="scores") @Composable def Expand(query, **kwargs): query, graph = db_graph(graphdb, query) gid = query.get("graph") field = "*" nodes = query['nodes'] vs = graph.vs.select(uuid_in=nodes) if len(vs) == 0: raise ValueError('No such node %s' % nodes) v = vs[0] q = v['properties']['URI'] if (v['nodetype'] == "Entity"): q = v['properties']['URI'] elif (v['nodetype'] == "Literal"): q = v['properties']['id'] print(q) g = query_rdf(gid, q, escape=True) graph = merge(gid, graph, g) pz = [v.index] vs = extract(graph, pz, **kwargs) print(vs) vs = [(graph.vs[i]['uuid'], v) for i, v in vs] # articles = [ (v['uuid'], 1.) for v in graph.vs if v['nodetype'] == ("_%s_article" % gid) ] return dict(vs) scores = Optionable("scores") scores._func = Expand scores.name = "expand" engine.scores.set(scores) return engine
def graph_engine(graphdb): # setup engine = Engine("graph") engine.graph.setup(in_name="request", out_name="graph") def _global(query, reset=False, all_articles=False, cut=100, **kwargs): gid = query['graph'] query, graph = db_graph(graphdb, query) nodes = [] if reset else query['nodes'] g = graph_articles(gid, graph, all_articles=all_articles, cut=cut, uuids=nodes, **kwargs) return g comp = Optionable("Graph") comp._func = _global comp.add_option("reset", Boolean(default=False, help="reset or add")) comp.add_option("all_articles", Boolean(default=False, help="includes all articles")) comp.add_option( "weighting", Text(choices=[ u"0", u"1", u"weight", u"auteurs", u"refBibAuteurs", u"keywords", u"categories" ], multi=True, default=u"1", help="ponderation")) comp.add_option("length", Numeric(vtype=int, min=1, default=3)) comp.add_option("cut", Numeric(vtype=int, min=2, default=100)) def _reset_global(query, **kwargs): gid = query['graph'] headers = istex.get_schema() graph = empty_graph(gid, headers, **kwargs) graphdb.graphs[gid] = graph g = graph_articles(gid, graph, all_articles=True, uuids=[], **kwargs) return g reset = Optionable('ResetGraph') reset._func = _reset_global reset.add_option("reset", Boolean(default=True, help=""), hidden=True) engine.graph.set(comp, reset) return engine
def expand_prox_engine(graphdb): """ prox with weights and filters on UNodes and UEdges types input: { nodes : [ uuid, .. ], //more complex p0 distribution weights: [float, ..], //list of weight } output: { graph : gid, scores : [ (uuid_node, score ), .. ] } """ engine = Engine("scores") engine.scores.setup(in_name="request", out_name="scores") ## Search def expand(query, length=3, cut=300, weightings=None): graph = db_graph(graphdb, query) gid = query.get("graph") nodes = query.get("nodes", []) expand = query.get("expand", []) vs = expand_subgraph(graph, expand, nodes, cut=cut, weightings=weightings) vs = [(graph.vs[v[0]]['uuid'], v[1]) for v in vs] return dict(vs) scores = Optionable("scores") scores._func = Composable(expand) scores.add_option("length", Numeric(vtype=int, default=3)) scores.add_option("cut", Numeric(vtype=int, default=50, max=300)) scores.add_option( "weighting", Text(choices=[u"0", u"1", u"weight"], multi=True, default=u"1", help="ponderation")) engine.scores.set(expand) return engine
def layout_engine(layouts): """ Return a default engine over a lexical graph """ # setup engine = Engine("gbuilder", "layout", "export") engine.gbuilder.setup(in_name="request", out_name="graph", hidden=True) engine.layout.setup(in_name="graph", out_name="layout") engine.export.setup(in_name=["graph", "layout"], out_name="layout", hidden=True) engine.gbuilder.set(engines.edge_subgraph) for k,v in layouts: v.name = k layouts = [ l for n,l in layouts ] engine.layout.set( *layouts ) engine.export.set( export_layout ) return engine
def search_engine(graphdb): # setup engine = Engine("search") engine.search.setup(in_name="request", out_name="graph") ## Search def Search(query, results_count=10, **kwargs): query, graph = db_graph(graphdb, query) gid = query['graph'] q = kwargs.pop("q", "*") field = kwargs.pop("field", None) g = query_istex(gid, q, field, results_count) graph = merge(gid, graph, g, index=index, vid=vid) nodes = query['nodes'] g = graph_articles(gid, graph, weighting=["1"], all_articles=True, cut=100, uuids=nodes, **kwargs) return g search = Optionable("IstexSearch") search._func = Search search.add_option("q", Text(default=u"clle erss")) search.add_option( "field", Text(choices=[ u"*", u"istex", u"auteurs", u"refBibAuteurs", u"keywords" ], default=u"*")) search.add_option( "results_count", Numeric(vtype=int, min=1, default=10, help="Istex results count")) engine.search.set(search) return engine
def additive_nodes_engine(graphdb): """ Additive engine add one or more nodes to the current graph, needs to know all node actually in the local graph in order to send edges to the client. POST { request : { graph : gid, nodes : [ uuid, uuid, ... ] # current local nodes add : [ uuid, uuid, ... ] # objects to add to local graph } will return a subgraphs with all connections between `nodes` and `add` """ # setup engine = Engine("graph") engine.graph.setup(in_name="request", out_name="graph") ## Search def subgraph(request): gid = request['graph'] to_add = set(request['add']) uuids = set([ q for q in request['nodes']] + list(to_add)) # TODO :: dont get subgraph from N edge_list = graphdb.get_edge_list(gid, uuids) edge_list = [ e for e in edge_list if e['source'] in to_add or e['target'] in uuids ] #nodes = { uuid : graphdb.get_node(uuid) for uuid in uuids } nodes = { n['uuid']: n for n in graphdb.get_nodes(gid, uuids)} graph = to_graph(nodes, edge_list) return graph graph_search = Optionable("AdditiveEngine") graph_search._func = Composable(subgraph) from cello.graphs.transform import VtxAttr graph_search |= VtxAttr(color=[(45, 200, 34), ]) graph_search |= VtxAttr(type=1) engine.graph.set(graph_search) return engine
def clusters_labels_engine(graphdb): def _labels(query, weighting=None, count=2, **kwargs): query, graph = db_graph(graphdb, query) gid = query['graph'] clusters = [] for clust in query['clusters']: labels = [] pz = graph.vs.select(uuid_in=clust) pz = [ v.index for v in pz if v['nodetype'] == ("_%s_article" % gid) ] if len(pz): vs = extract(graph, pz, cut=300, weighting=weighting, length=3) labels = [{ 'uuid': graph.vs[i]['uuid'], 'label': graph.vs[i]['properties']['label'], 'score': v } for i, v in vs if graph.vs[i]['nodetype'] != ("_%s_article" % gid) ][:count] clusters.append(labels) return clusters comp = Optionable("labels") comp._func = _labels comp.add_option( "weighting", Text(choices=[ u"0", u"1", u"weight", u"auteurs", u"refBibAuteurs", u"keywords", u"categories" ], multi=True, default=u"1", help="ponderation")) comp.add_option("count", Numeric(vtype=int, min=1, default=2)) engine = Engine("labels") engine.labels.setup(in_name="request", out_name="labels") engine.labels.set(comp) return engine
def expand_prox_engine(graphdb): """ prox with weights and filters on UNodes and UEdges types input: { nodes : [ uuid, .. ], //more complex p0 distribution weights: [float, ..], //list of weight } output: { graph : gid, scores : [ (uuid_node, score ), .. ] } """ engine = Engine("scores") engine.scores.setup(in_name="request", out_name="scores") ## Search def expand(query, step=3, limit=100, filter_nodes=None, filter_edges=None): if filter_nodes is None : filter_nodes = [] if filter_edges is None: filter_edges = [] gid = query.get("graph") pzeros = query.get("nodes") weights = query.get("weights", []) return graphdb.proxemie( gid, pzeros, weights, filter_edges=filter_edges, filter_nodes=filter_nodes, limit=limit, n_step=step) scores = Optionable("scores") scores._func = Composable(expand) scores.add_option("step", Numeric( vtype=int, default=3)) scores.add_option("limit", Numeric( vtype=int, default=50, max=100)) scores.add_option("filter_nodes", Text( default=set([]), multi=True, uniq=True)) scores.add_option("filter_edges", Text( default=set([]), multi=True, uniq=True)) engine.scores.set(expand) return engine
def clustering_engine(optionables): """ Return a default engine over a lexical graph """ # setup engine = Engine("gbuilder", "clustering", "labelling") engine.gbuilder.setup(in_name="request", out_name="graph", hidden=True) engine.clustering.setup(in_name="graph", out_name="clusters") engine.labelling.setup(in_name="clusters", out_name="clusters", hidden=True) engine.gbuilder.set(engines.edge_subgraph) engine.clustering.set(*optionables) ## Labelling from cello.clustering.labelling.model import Label from cello.clustering.labelling.basic import VertexAsLabel, TypeFalseLabel, normalize_score_max def _labelling(graph, cluster, vtx): return Label(vtx["uuid"], score=1, role="default") labelling = VertexAsLabel( _labelling ) | normalize_score_max engine.labelling.set(labelling) return engine
def test_engine_named_inout_error(self): engine = Engine("op1", "op2", "op3") engine.op1.set(self.mult_opt) engine.op1.setup(in_name="in1", out_name="out1") engine.op2.set(self.plus_comp) engine.op2.setup(in_name="out_not_exist", out_name="out2") engine.op3.set(self.mult_opt) engine.op3.setup(in_name="out1", out_name="out3") with self.assertRaises(ReliureError): engine.validate() engine.op2.setup(in_name="in1", out_name="out2") engine.validate() engine.op2.setup(required=False) engine.op3.setup(in_name="out2") with self.assertRaises(ReliureError): engine.validate()
def test_engine_default_pipeline(self): engine = Engine("op1", "op2", "op3") # should have a __len__ assert len(engine) == 3 # should have a __contains__ assert "op1" in engine assert "op2" in engine assert "op3" in engine assert not "op4" in engine # should not be possible to require a block that do not exist with self.assertRaises(ReliureError): engine.requires("op4") # should be possible to set the component for each block engine.op1.set(self.mult_opt) # but not on non existing blocks with self.assertRaises(ValueError): engine.op4.set(self.mult_opt) with self.assertRaises(ValueError): engine["op4"].set(self.mult_opt) with self.assertRaises(ValueError): engine.set("op4", self.mult_opt) # should not validate if there is no component for all blocks with self.assertRaises(ReliureError): engine.validate() #should be posisble to 'append' a possible component to the block engine.op2.append(self.mult_opt) # and to set is as default in the same time engine.op2.append(self.plus_comp, default=True) # should be possible to select on component engine.op1.select("mult_opt") # should be possible to know if one block is multiple assert not engine.op2.multiple # to know wich component will be run assert engine.op2.selected() == ["plus_comp"] # to play just one block ! assert engine.op2.play(10) == {'op2': 12} # and then is should validate is every block has som components engine.set("op3", self.mult_opt, self.plus_comp, self.max_comp) engine.validate() # should play ! res = engine.play(3) # mult * 5 | + 2 | mult # and all intermediare results should be available assert res['input'] == 3 assert res['op1'] == 3*5 assert res['op2'] == 3*5+2 assert res['op3'] == (3*5+2)*5
def lexical_graph_engine(graph): """ Return a default engine over a lexical graph """ # setup engine = Engine() engine.requires("search", "clustering", "labelling", "layout") engine.search.setup(in_name="query", out_name="graph") engine.clustering.setup(in_name="graph", out_name="clusters") engine.labelling.setup(in_name="clusters", out_name="clusters", hidden=True) engine.layout.setup(in_name="graph", out_name="layout") ## Search from cello.graphs.extraction import VtxMatch, ProxMarkovExtractionGlobal, VertexIds from cello.graphs.builder import Subgraph #HACK remove the "id" attribute (if any), it enter in conflict when exporting subgraphs to client if 'id' in graph.vs.attributes(): del graph.vs['id'] # Pour avoir une recherche "_all" # http://localhost:5000/verb/q/_all @Composable def nothing_if_all(query): """ Make the query be nothing if it is '_all' """ if query in ("_all", None): return "" #Note: p0 to [] make start from all vertices return query # real match search match = VtxMatch(graph, attr_list=[u"label"], default_attr=u"label") graph_search = nothing_if_all | match graph_search |= ProxMarkovExtractionGlobal(graph) graph_search |= Subgraph(graph, score_attr="prox", gdeg_attr="gdeg") graph_search |= ProxColors(graph, graph['vertices_color'], length=5) graph_search.name = "ProxSearch" graph_search.change_option_default("vcount", 50) engine.search.set(graph_search) ## Clustering from cello.graphs.transform import EdgeAttr from cello.clustering.common import Infomap, Walktrap #RMQ infomap veux un pds, donc on en ajoute un bidon walktrap = EdgeAttr(weight=1.) |Walktrap() infomap = EdgeAttr(weight=1.) | Infomap() engine.clustering.set(walktrap, infomap) ## Labelling from cello.clustering.labelling.model import Label from cello.clustering.labelling.basic import VertexAsLabel vertex_from_vtx = lambda graph, cluster, vtx: Label(vtx["label"], role="default", score=vtx["gdeg"]) engine.labelling.set(VertexAsLabel(vertex_from_vtx)) ## Layout from cello.layout.simple import KamadaKawaiLayout from cello.layout.proxlayout import ProxLayoutRandomProj from cello.layout.proxlayout import ProxLayoutPCA from cello.layout.transform import Shaker engine.layout.set( ProxLayoutPCA(dim=3) | Shaker(), KamadaKawaiLayout(dim=3), ) return engine
def explore_engine(graphdb): """ Prox engine """ # setup engine = Engine("graph") engine.graph.setup(in_name="request", out_name="graph") ## Search @Composable def get_graph(query, **kwargs): return db_graph(graphdb, query) @Composable def subgraph(query, cut=100, weighted=True, length=7, mode=ALL, add_loops=False, **kwargs): graph = db_graph(graphdb, query) idx = {v['uuid']: v.index for v in graph.vs} uuids = [q for q in query.get('units', [])] uuids = [idx[p] for p in uuids] return prox_subgraph(graph, uuids, cut=cut, weighted=weighted, length=length, mode=mode, add_loops=add_loops, **kwargs) from cello.graphs.transform import VtxAttr searchs = [] for k, w, l, m, n in [ (u"Search", True, 3, ALL, 100), ]: search = Optionable("GraphSearch") search._func = subgraph search.add_option("weighted", Boolean(default=w)) search.add_option("add_loops", Boolean(default=True, help="add loops on vertices")) search.add_option( "mode", Numeric(choices=[OUT, IN, ALL], default=m, help="edge directions")) search.add_option("length", Numeric(vtype=int, min=1, default=l)) search.add_option("cut", Numeric(vtype=int, min=2, default=n)) search |= VtxAttr(color=[ (45, 200, 34), ]) search |= VtxAttr(type=1) search.name = k searchs.append(search) sglobal = get_graph | ProxSubgraph() sglobal.name = "Global" sglobal.change_option_default("cut", 200) searchs.append(sglobal) engine.graph.set(*searchs) return engine
def explore_engine(graphdb): """ Prox engine """ # setup engine = Engine("graph") engine.graph.setup(in_name="request", out_name="graph") ## Search @Composable def get_graph(query, **kwargs): gid = query['graph'] graph = graphdb.get_graph(gid) return graph @Composable def subgraph( query, cut=50, weighted=True, length=3, mode=ALL, add_loops=False, ): graph = get_graph(query) uuids = {v['uuid']: v.index for v in graph.vs} pz = [q for q in query['units']] pz = [uuids[p] for p in pz] extract = ProxExtract() vs = [] for u in pz: s = extract(graph, pzeros=[u], weighted=weighted, mode=mode, cut=cut, length=length) vs = pz + vs + list(s.keys()) return graph.subgraph(vs) from cello.graphs.transform import VtxAttr searchs = [] for k, w, l, m, n in [(u"Espaces_sémantiques", True, 3, OUT, 30), (u"Espaces_sémantiques_élargis", True, 4, OUT, 50), (u"Espaces_lexicaux", False, 3, OUT, 30), (u"Espaces_lexicaux_élargis", False, 4, OUT, 50)]: search = Optionable("GraphSearch") search._func = subgraph search.add_option("weighted", Boolean(default=w)) search.add_option("add_loops", Boolean(default=True, help="add loops on vertices")) search.add_option( "mode", Numeric(choices=[IN, OUT, ALL], default=m, help="edge directions")) search.add_option("length", Numeric(vtype=int, min=1, default=l)) search.add_option("cut", Numeric(vtype=int, min=2, default=n)) search |= VtxAttr(color=[ (45, 200, 34), ]) search |= VtxAttr(type=1) search.name = k searchs.append(search) sglobal = Composable(get_graph) | ProxSubgraph() sglobal.name = "Global" #searchs.append(sglobal) engine.graph.set(*searchs) return engine
def test_configure_errors(self): # Should raise valueError when configuration is not valit engine = Engine("op1", "op2", "op3") engine.set("op1", self.mult_opt, self.plus_comp, self.max_comp) engine.set("op2", self.plus_comp, self.mult_opt, self.max_comp) engine.op2.setup(hidden=True) engine.set("op3", self.mult_opt, self.plus_comp, self.max_comp) with self.assertRaises(ValueError): # op2 hidden it can be configured engine.configure({ 'op2':{'name': 'max20'} }) with self.assertRaises(ValueError): # 'maxmax' doesn't exist engine.configure({ 'op3':{'name': 'maxmax'} }) with self.assertRaises(ValueError): # error in op1 format engine.configure({ 'op1':{'namss': 'mult'} }) with self.assertRaises(ValueError): # block doesn't exist engine.configure({ 'op5':{'name': 'max20'} }) with self.assertRaises(ValueError): # block required ! engine.configure({ 'op1':[] }) with self.assertRaises(ValueError): # block not multi ! engine.configure({ 'op1':[{'name': 'max20'}, {'name': 'plus_comp'}] })
class TestReliureAPIMultiInputs(unittest.TestCase): maxDiff = None def setUp(self): self.engine = Engine("op1", "op2") self.engine.op1.setup(in_name="in", out_name="middle", required=False) self.engine.op2.setup(in_name="middle", out_name="out") self.engine.op1.set(OptProductEx()) foisdouze = OptProductEx("foisdouze") foisdouze.force_option_value("factor", 12) self.engine.op2.set(foisdouze, OptProductEx()) egn_view = EngineView(self.engine, name="my_egn") egn_view.add_input("in", Numeric(vtype=int, min=-5, max=5)) egn_view.add_input("middle", Numeric(vtype=int)) print(self.engine.needed_inputs()) egn_view.add_output("in") egn_view.add_output("middle") egn_view.add_output("out") api = ReliureAPI() api.register_view(egn_view) app = Flask(__name__) app.config['TESTING'] = True app.register_blueprint(api, url_prefix="/api") self.app = app.test_client() def test_routes(self): resp = self.app.get('api/') data = json.loads(resp.data.decode("utf-8")) # test routes routes = data["routes"] routes = {route['name']: route for route in routes} pprint(routes) # routes GET api/ for this entry point assert "api.routes" in routes assert routes["api.routes"]["path"] == u'/api/' assert sorted(routes["api.routes"]["methods"]) == [u'GET', u'HEAD', u'OPTIONS'] # routes GET api/egn for options assert "api.my_egn_options" in routes assert routes["api.my_egn_options"]["path"] == u'/api/my_egn' assert sorted(routes["api.my_egn_options"]["methods"]) == [u'GET', u'HEAD', u'OPTIONS'] # routes POST api/egn for play assert "api.my_egn" in routes assert routes["api.my_egn"]["path"] == u'/api/my_egn' assert sorted(routes["api.my_egn"]["methods"]) == [u'OPTIONS', u'POST'] def test_options(self): resp = self.app.get('api/my_egn') data = json.loads(resp.data.decode("utf-8")) # check we have the same than in engine assert data["blocks"] == self.engine.as_dict()["blocks"] assert data["args"] == ["in", "middle"] assert data["returns"] == ["in", "middle", "out"] def test_play_simple(self): # it should be possible to play the full engine # prepare query rdata = {'in': 2} rdata["options"] = { 'op1': [{ 'name': 'mult_opt', }] } json_data = json.dumps(rdata) resp = self.app.post('api/my_egn', data=json_data, content_type='application/json') # load the results resp_data = json.loads(resp.data.decode("utf-8")) # check it assert "results" in resp_data results = resp_data["results"] assert "out" in results assert len(results) == 3 # in, middle, out assert results["out"] == 2*5*12 def test_play_skip_op1(self): # it should be possible to play the only the block op2 # prepare query rdata = {'middle': 2} rdata["options"] = { 'op1': [] } json_data = json.dumps(rdata) resp = self.app.post('api/my_egn', data=json_data, content_type='application/json') # load the results resp_data = json.loads(resp.data.decode("utf-8")) # check it assert "results" in resp_data results = resp_data["results"] assert "out" in results assert len(results) == 2 # middle, out assert results["out"] == 2*12
def test_as_dict(self): engine = Engine("op1", "op2") engine.set("op1", self.plus_comp, self.mult_opt, self.max_comp) engine.set("op2", self.plus_comp) assert engine.as_dict() == { 'args': ['input'], 'blocks': [ { 'components': [ { 'default': True, #This is the default NOT the selected value 'name': 'plus_comp', 'options': None }, { 'default': False, 'name': 'mult_opt', 'options': [ { 'name': 'factor', 'type': 'value', 'value': 5, 'otype': { 'choices': None, 'default': 5, 'help': 'multipliation factor', 'max': None, 'min': None, 'multi': False, 'type': 'Numeric', 'uniq': False, 'vtype': 'int' } } ] }, { 'name': 'max20', 'default': False, 'options': None } ], 'args': None, 'multiple': False, 'name': 'op1', 'returns': 'op1', 'required': True }, { 'components': [ { 'name': 'plus_comp', 'default': True, 'options': None } ], 'args': None, 'multiple': False, 'name': 'op2', 'returns': 'op2', 'required': True } ] }
class TestReliureAPISimple(unittest.TestCase): maxDiff = None def setUp(self): self.engine = Engine("op1", "op2") self.engine.op1.setup(in_name="in") self.engine.op2.setup(out_name="out") self.engine.op1.set(OptProductEx()) foisdouze = OptProductEx("foisdouze") foisdouze.force_option_value("factor", 12) self.engine.op2.set(foisdouze, OptProductEx()) egn_view = EngineView(self.engine) egn_view.set_input_type(Numeric(vtype=int, min=-5, max=5)) egn_view.add_output("out") api = ReliureAPI() api.register_view(egn_view, url_prefix="egn") app = Flask(__name__) self.appp = app app.config['TESTING'] = True app.register_blueprint(api, url_prefix="/api") self.app = app.test_client() def test_engine_view_init(self): _egn_view = EngineView(self.engine) with pytest.raises(ValueError): _egn_view.add_output("existe_pas") def test_routes(self): resp = self.app.get('api/') data = json.loads(resp.data.decode("utf-8")) assert data["api"] == "api" assert data["url_root"] == "http://localhost/" # test routes routes = data["routes"] routes = {route['name']: route for route in routes} pprint(routes) # routes GET api/ for this entry point assert "api.routes" in routes assert routes["api.routes"]["path"] == u'/api/' assert sorted(routes["api.routes"]["methods"]) == [u'GET', u'HEAD', u'OPTIONS'] # routes GET api/egn for options assert "api.egn_options" in routes assert routes["api.egn_options"]["path"] == u'/api/egn' assert sorted(routes["api.egn_options"]["methods"]) == [u'GET', u'HEAD', u'OPTIONS'] # routes POST api/egn for play assert "api.egn" in routes assert routes["api.egn"]["path"] == u'/api/egn' assert sorted(routes["api.egn"]["methods"]) == [u'OPTIONS', u'POST'] def test_options(self): resp = self.app.get('api/egn') data = json.loads(resp.data.decode("utf-8")) # check we have the same than in engine assert data["blocks"] == self.engine.as_dict()["blocks"] assert data["args"] == ["in"] assert data["returns"] == ["out"] def test_play_simple(self): # prepare query rdata = {'in': '2'} json_data = json.dumps(rdata) resp = self.app.post('api/egn', data=json_data, content_type='application/json') # load the results resp_data = json.loads(resp.data.decode("utf-8")) # check it assert "results" in resp_data results = resp_data["results"] assert "out" in results assert len(results) == 1 assert results["out"] == 2*5*12 def test_play_nojson(self): # prepare query data = {u'in': 3} resp = self.app.post('api/egn', data=data) # load the results resp_data = json.loads(resp.data.decode("utf-8")) # check it assert "results" in resp_data results = resp_data["results"] assert "out" in results assert len(results) == 1 assert results["out"] == 3*5*12 def test_play_simple_options(self): # prepare query rdata = {'in': '2'} rdata["options"] = { 'op2': [{ 'name': 'mult_opt', 'options': { 'factor': '2' } }] } json_data = json.dumps(rdata) resp = self.app.post('api/egn', data=json_data, content_type='application/json') # load the results resp_data = json.loads(resp.data.decode("utf-8")) # check it assert "results" in resp_data results = resp_data["results"] assert "out" in results assert len(results) == 1 assert results["out"] == 2*2*5
def engine(index): """ Return a default engine over a lexical graph """ # setup from reliure.engine import Engine engine = Engine("graph", "clustering", "labelling", "layout") engine.graph.setup(in_name="query", out_name="graph") engine.clustering.setup(in_name="graph", out_name="clusters") engine.labelling.setup(in_name="clusters", out_name="clusters", hidden=True) engine.layout.setup(in_name="graph", out_name="layout") ## Search def tmuse_subgraph(query, length=50): return tmuse.subgraph(index, query, length=length) graph_search = Optionable("GraphSearch") graph_search._func = Composable(tmuse_subgraph) graph_search.add_option("length", Numeric(vtype=int, default=50)) from cello.graphs.transform import VtxAttr graph_search |= VtxAttr(color=[ (45, 200, 34), ]) graph_search |= VtxAttr(type=1) engine.graph.set(graph_search) ## Clustering from cello.graphs.transform import EdgeAttr from cello.clustering.common import Infomap, Walktrap #RMQ infomap veux un pds, donc on en ajoute un bidon walktrap = EdgeAttr(weight=1.) | Walktrap() infomap = EdgeAttr(weight=1.) | Infomap() engine.clustering.set(infomap, walktrap) ## Labelling from cello.clustering.labelling.model import Label from cello.clustering.labelling.basic import VertexAsLabel, TypeFalseLabel, normalize_score_max def _labelling(graph, cluster, vtx): score = TypeFalseLabel.scoring_prop_ofclust(graph, cluster, vtx) return Label(vtx["form"], score=score, role="default") labelling = VertexAsLabel(_labelling) | normalize_score_max engine.labelling.set(labelling) ## Layout from cello.layout.simple import KamadaKawaiLayout #from cello.layout.proxlayout import ProxLayoutRandomProj from cello.layout.proxlayout import ProxLayoutPCA from cello.layout.transform import Shaker from cello.layout.transform import ByConnectedComponent, normalise default_layout = ProxLayoutPCA(dim=3, name="ProxPca3d") | Shaker(kelastic=.9) default_layout = KamadaKawaiLayout( dim=3, name="KamadaKawaiLayout") | Shaker(kelastic=.9) engine.layout.set( ByConnectedComponent(default_layout) | normalise, #KamadaKawaiLayout(dim=3, name="KamadaKawai3D"), #ProxLayoutPCA(dim=2, name="ProxPca2d") | Shaker(kelastic=1.8), #KamadaKawaiLayout(dim=2, name="KamadaKawai2D") ) return engine