def __init__(self, process_id, inputs, outputs, requirements, hints, label, description, steps, context, data_links=None): super(Workflow, self).__init__(process_id, inputs, outputs, requirements, hints, label, description) self.graph = Graph() self.executor = context.executor self.steps = steps self.data_links = data_links or [] self.context = context self.port_step_index = {} for step in steps: node = AppNode(step.app, {}) self.add_node(step.id, node) for inp in step.inputs: self.port_step_index[inp.id] = step.id self.move_connect_to_datalink(inp) if inp.value: node.inputs[inp.id] = inp.value for out in step.outputs: self.port_step_index[out.id] = step.id for inp in self.inputs: self.add_node(inp.id, inp) for out in self.outputs: self.move_connect_to_datalink(out) self.add_node(out.id, out) for dl in self.data_links: dst = dl['destination'].lstrip('#') src = dl['source'].lstrip('#') if src in self.port_step_index and dst in self.port_step_index: rel = Relation(src, dst, dl.get('position', 0)) src = self.port_step_index[src] dst = self.port_step_index[dst] elif src in self._inputs: rel = InputRelation(dst, dl.get('position', 0)) dst = self.port_step_index[dst] elif dst in self._outputs: rel = OutputRelation(src, dl.get('position', 0)) src = self.port_step_index[src] else: raise RabixError("invalid data link %s" % dl) self.graph.add_edge(src, dst, rel) if not self.graph.connected(): pass
def __init__(self, graph=None, debug=0): if graph is None: graph = Graph() self.graphident = self self.graph = graph self.debug = debug self.indent = 0 graph.add_node(self, None)
def __init__(self, process_id, inputs, outputs, requirements, hints, label, description, steps, context, data_links=None): super(Workflow, self).__init__( process_id, inputs, outputs, requirements, hints, label, description ) self.graph = Graph() self.executor = context.executor self.steps = steps self.data_links = data_links or [] self.context = context self.port_step_index = {} for step in steps: node = AppNode(step.app, {}) self.add_node(step.id, node) for inp in step.inputs: self.port_step_index[inp.id] = step.id self.move_connect_to_datalink(inp) if inp.value: node.inputs[inp.id] = inp.value for out in step.outputs: self.port_step_index[out.id] = step.id for inp in self.inputs: self.add_node(inp.id, inp) for out in self.outputs: self.move_connect_to_datalink(out) self.add_node(out.id, out) # dedupe links s = {tuple(dl.items()) for dl in self.data_links} self.data_links = [dict(dl) for dl in s] for dl in self.data_links: dst = dl['destination'].lstrip('#') src = dl['source'].lstrip('#') if src in self.port_step_index and dst in self.port_step_index: rel = Relation(src, dst, dl.get('position', 0)) src = self.port_step_index[src] dst = self.port_step_index[dst] elif src in self._inputs: rel = InputRelation(dst, dl.get('position', 0)) dst = self.port_step_index[dst] elif dst in self._outputs: rel = OutputRelation(src, dl.get('position', 0)) src = self.port_step_index[src] else: raise RabixError("invalid data link %s" % dl) self.graph.add_edge(src, dst, rel) if not self.graph.connected(): pass
def test_constructor(self): graph = Graph(iter([ (1, 2), (2, 3, 'a'), (1, 3), (3, 4), ])) self.assertEqual(graph.number_of_nodes(), 4) self.assertEqual(graph.number_of_edges(), 4) try: graph.edge_by_node(1,2) graph.edge_by_node(2,3) graph.edge_by_node(1,3) graph.edge_by_node(3,4) except GraphError: self.fail("Incorrect graph") self.assertEqual(graph.edge_data(graph.edge_by_node(2, 3)), 'a') self.assertRaises(GraphError, Graph, [(1,2,3,4)])
def test_constructor(self): graph = ObjectGraph() self.assertTrue(isinstance(graph, ObjectGraph)) g = Graph() graph = ObjectGraph(g) self.assertTrue(graph.graph is g) self.assertEqual(graph.debug, 0) self.assertEqual(graph.indent, 0) graph = ObjectGraph(debug=5) self.assertEqual(graph.debug, 5)
def copy_to_graph(session): graph = Graph() count = 0 for artist in session.query(LastFmArtist).all(): if artist.graph_out: graph.add_node(artist.id, artist) edges = list(artist.graph_out) edges.sort(key=lambda e: e.weight, reverse=True) for edge in edges[0:2]: target = edge.to_ graph.add_node(target.id, target) graph.add_edge(artist.id, target.id, edge, create_nodes=False) if not count % 100: LOG.info('Added {0} nodes.'.format(count)) count += 1 else: LOG.warn('Discarding unconnected {0}.'.format(artist.name)) return graph
def davaj_trať(id_relace): ''' zahájí běh programu ''' from altgraph.Graph import Graph graf = Graph() cesty = davaj_cesty_z_relace(id_relace) # vložím cesty do grafu for cesta in cesty: graf.add_edge(cesta.body_v_cestě[0], cesta.body_v_cestě[-1], { 'id': cesta.id_cesty, 'pořadí': cesta.pořadí_cesty }) print('graf má {} vrcholů a {} hran'.format(graf.number_of_nodes(), graf.number_of_edges())) # projdu všechny vrcholy a vyberu ty, které mají pouze jednu hranu jednohrané_vrcholy = [] for bod in graf.node_list(): if graf.all_degree(bod) == 1: # print("vrchol {} vstupuje {} a vystupuje {} hran, celkem {}".format(bod, graf.inc_degree(bod), graf.out_degree(bod), graf.all_degree(bod))) jednohrané_vrcholy.append(bod) # jednohrané vrcholy musí být právě dva if len(jednohrané_vrcholy) != 2: raise ValueError( 'Tož ale graf musí být právě jeden počátek a právě jeden konec') # když má graf právě jeden začátek a právě jeden konec, možeme srovnat hrany, otočit je do jednoho směru else: print('jednohrané vrcholy', jednohrané_vrcholy) # tu práci obsatrá rekurzivně tato funkce, která graf pozmění seřadím_hrany_v_grafu(graf, jednohrané_vrcholy[0]) #včíl projdu už upravený, správně natočený graf a načtu všechny body najdu_body_pro_hrany_v_grafu(graf)
def test_edges_complex(self): g = Graph() g.add_edge(1, 2) e = g.edge_by_node(1,2) g.hide_edge(e) g.hide_node(2) self.assertRaises(GraphError, g.restore_edge, e) g.restore_all_edges() self.assertRaises(GraphError, g.edge_by_id, e)
def test_bfs(self): graph = Graph() graph.add_edge("1", "1.1") graph.add_edge("1.1", "1.1.1") graph.add_edge("1.1", "1.1.2") graph.add_edge("1.1.2", "1.1.2.1") graph.add_edge("1.1.2", "1.1.2.2") graph.add_edge("1", "1.2") graph.add_edge("1", "1.3") graph.add_edge("1.2", "1.2.1") self.assertEqual(graph.forw_bfs("1"), ['1', '1.1', '1.2', '1.3', '1.1.1', '1.1.2', '1.2.1', '1.1.2.1', '1.1.2.2']) self.assertEqual(graph.forw_bfs("1", "1.1.1"), ['1', '1.1', '1.2', '1.3', '1.1.1']) # And the "reverse" graph graph = Graph() graph.add_edge("1.1", "1") graph.add_edge("1.1.1", "1.1") graph.add_edge("1.1.2", "1.1") graph.add_edge("1.1.2.1", "1.1.2") graph.add_edge("1.1.2.2", "1.1.2") graph.add_edge("1.2", "1") graph.add_edge("1.3", "1") graph.add_edge("1.2.1", "1.2") self.assertEqual(graph.back_bfs("1"), ['1', '1.1', '1.2', '1.3', '1.1.1', '1.1.2', '1.2.1', '1.1.2.1', '1.1.2.2']) self.assertEqual(graph.back_bfs("1", "1.1.1"), ['1', '1.1', '1.2', '1.3', '1.1.1']) # check cycle handling graph.add_edge("1", "1.2.1") self.assertEqual(graph.back_bfs("1"), ['1', '1.1', '1.2', '1.3', '1.1.1', '1.1.2', '1.2.1', '1.1.2.1', '1.1.2.2'])
def test_connected(self): graph = Graph() graph.add_node(1) graph.add_node(2) graph.add_node(3) graph.add_node(4) self.assertFalse(graph.connected()) graph.add_edge(1, 2) graph.add_edge(3, 4) self.assertFalse(graph.connected()) graph.add_edge(2, 3) graph.add_edge(4, 1) self.assertTrue(graph.connected())
class WorkflowApp(App): def __init__(self, app_id, steps, context, inputs=None, outputs=None, to=None, app_description=None, annotations=None, platform_features=None): self.graph = Graph() self.inputs = inputs or [] self.outputs = outputs or [] self.executor = context.executor self.steps = steps self.to = to or {} self.context = context for step in steps: self.add_node(step.id, AppNode(step.app, {})) for step in steps: # inputs for input_port, input_val in six.iteritems(step.inputs): inp = wrap_in_list(input_val) for item in inp: self.add_edge_or_input(step, input_port, item) # outputs if step.outputs: for output_port, output_val in six.iteritems(step.outputs): self.to[output_val['$to']] = output_port if isinstance(step.app, WorkflowApp): output_node = step.app.get_output(step.app.to.get(output_port)) else: output_node = step.app.get_output(output_port) output_id = output_val['$to'] self.add_node(output_id, output_node) self.graph.add_edge( step.id, output_id, OutputRelation(output_port) ) # output_node.id = output_val['$to'] self.outputs.append(output_node) if not self.graph.connected(): pass # raise ValidationError('Graph is not connected') schema = { "@type": "JsonSchema", "type": "object", "properties": {}, "required": [] } for inp in self.inputs: schema['properties'][inp.id] = inp.validator.schema if inp.required: schema['required'].append(inp.id) super(WorkflowApp, self).__init__( app_id, JsonSchema(context, schema), self.outputs, app_description=app_description, annotations=annotations, platform_features=platform_features ) def add_edge_or_input(self, step, input_name, input_val): node_id = step.id if isinstance(input_val, dict) and '$from' in input_val: frm = wrap_in_list(input_val['$from']) for inp in frm: if '.' in inp: node, outp = inp.split('.') self.graph.add_edge(node, node_id, Relation(outp, input_name)) else: # TODO: merge input schemas if one input goes to different apps input = step.app.get_input(input_name) if inp not in self.graph.nodes: self.add_node(inp, input) self.graph.add_edge( inp, node_id, InputRelation(input_name) ) wf_input = copy.deepcopy(input) wf_input.id = inp self.inputs.append(wf_input) else: self.graph.node_data(node_id).inputs[input_name] = input_val # Graph.add_node silently fails if node already exists def add_node(self, node_id, node): if node_id in self.graph.nodes: raise ValidationError('Duplicate node ID: %s' % node_id) self.graph.add_node(node_id, node) def hide_nodes(self, type): for node_id in self.graph.node_list(): node = self.graph.node_data(node_id) if isinstance(node, type): self.graph.hide_node(node_id) def run(self, job): eg = ExecutionGraph(self, job) while eg.has_next(): next_id, next = eg.next_job() self.executor.execute(next, eg.job_done, next_id) return eg.outputs def to_dict(self, context): d = super(WorkflowApp, self).to_dict(context) d.update({ "@type": "Workflow", 'steps': [step.to_dict(context) for step in self.steps] }) return d @classmethod def from_dict(cls, context, d): steps = [Step( step['id'], context.from_dict(step['app']), step['inputs'], step.get('outputs') ) for step in d['steps']] return cls( d.get('@id', six.text_type(uuid4())), steps, context )
def test_iterdata(self): graph = Graph() graph.add_node("1", "I") graph.add_node("1.1", "I.I") graph.add_node("1.2", "I.II") graph.add_node("1.3", "I.III") graph.add_node("1.1.1", "I.I.I") graph.add_node("1.1.2", "I.I.II") graph.add_node("1.2.1", "I.II.I") graph.add_node("1.2.2", "I.II.II") graph.add_node("1.2.2.1", "I.II.II.I") graph.add_node("1.2.2.2", "I.II.II.II") graph.add_node("1.2.2.3", "I.II.II.III") graph.add_edge("1", "1.1") graph.add_edge("1", "1.2") graph.add_edge("1", "1.3") graph.add_edge("1.1", "1.1.1") graph.add_edge("1.1", "1.1.2") graph.add_edge("1.2", "1.2.1") graph.add_edge("1.2", "1.2.2") graph.add_edge("1.2.2", "1.2.2.1") graph.add_edge("1.2.2", "1.2.2.2") graph.add_edge("1.2.2", "1.2.2.3") result = list(graph.iterdata("1", forward=True)) self.assertEqual(result, [ 'I', 'I.III', 'I.II', 'I.II.II', 'I.II.II.III', 'I.II.II.II', 'I.II.II.I', 'I.II.I', 'I.I', 'I.I.II', 'I.I.I' ]) result = list(graph.iterdata("1", end="1.2.1", forward=True)) self.assertEqual(result, [ 'I', 'I.III', 'I.II', 'I.II.II', 'I.II.II.III', 'I.II.II.II', 'I.II.II.I', 'I.II.I' ]) result = list(graph.iterdata("1", condition=lambda n: len(n) < 6, forward=True)) self.assertEqual(result, [ 'I', 'I.III', 'I.II', 'I.I', 'I.I.I' ]) # And the revese option: graph = Graph() graph.add_node("1", "I") graph.add_node("1.1", "I.I") graph.add_node("1.2", "I.II") graph.add_node("1.3", "I.III") graph.add_node("1.1.1", "I.I.I") graph.add_node("1.1.2", "I.I.II") graph.add_node("1.2.1", "I.II.I") graph.add_node("1.2.2", "I.II.II") graph.add_node("1.2.2.1", "I.II.II.I") graph.add_node("1.2.2.2", "I.II.II.II") graph.add_node("1.2.2.3", "I.II.II.III") graph.add_edge("1.1", "1") graph.add_edge("1.2", "1") graph.add_edge("1.3", "1") graph.add_edge("1.1.1", "1.1") graph.add_edge("1.1.2", "1.1") graph.add_edge("1.2.1", "1.2") graph.add_edge("1.2.2", "1.2") graph.add_edge("1.2.2.1", "1.2.2") graph.add_edge("1.2.2.2", "1.2.2") graph.add_edge("1.2.2.3", "1.2.2") result = list(graph.iterdata("1", forward=False)) self.assertEqual(result, [ 'I', 'I.III', 'I.II', 'I.II.II', 'I.II.II.III', 'I.II.II.II', 'I.II.II.I', 'I.II.I', 'I.I', 'I.I.II', 'I.I.I' ]) result = list(graph.iterdata("1", end="1.2.1", forward=False)) self.assertEqual(result, [ 'I', 'I.III', 'I.II', 'I.II.II', 'I.II.II.III', 'I.II.II.II', 'I.II.II.I', 'I.II.I' ]) result = list(graph.iterdata("1", condition=lambda n: len(n) < 6, forward=False)) self.assertEqual(result, [ 'I', 'I.III', 'I.II', 'I.I', 'I.I.I' ])
class Workflow(Process): def __init__(self, process_id, inputs, outputs, requirements, hints, label, description, steps, context, data_links=None): super(Workflow, self).__init__(process_id, inputs, outputs, requirements, hints, label, description) self.graph = Graph() self.executor = context.executor self.steps = steps self.data_links = data_links or [] self.context = context self.port_step_index = {} for step in steps: node = AppNode(step.app, {}) self.add_node(step.id, node) for inp in step.inputs: self.port_step_index[inp.id] = step.id self.move_connect_to_datalink(inp) if inp.value: node.inputs[inp.id] = inp.value for out in step.outputs: self.port_step_index[out.id] = step.id for inp in self.inputs: self.add_node(inp.id, inp) for out in self.outputs: self.move_connect_to_datalink(out) self.add_node(out.id, out) # dedupe links s = {tuple(dl.items()) for dl in self.data_links} self.data_links = [dict(dl) for dl in s] for dl in self.data_links: dst = dl['destination'].lstrip('#') src = dl['source'].lstrip('#') if src in self.port_step_index and dst in self.port_step_index: rel = Relation(src, dst, dl.get('position', 0)) src = self.port_step_index[src] dst = self.port_step_index[dst] elif src in self._inputs: rel = InputRelation(dst, dl.get('position', 0)) dst = self.port_step_index[dst] elif dst in self._outputs: rel = OutputRelation(src, dl.get('position', 0)) src = self.port_step_index[src] else: raise RabixError("invalid data link %s" % dl) self.graph.add_edge(src, dst, rel) if not self.graph.connected(): pass # raise ValidationError('Graph is not connected') def move_connect_to_datalink(self, port): for src in port.source: self.data_links.append({ 'source': src, 'destination': '#' + port.id }) del port.source[:] # Graph.add_node silently fails if node already exists def add_node(self, node_id, node): if node_id in self.graph.nodes: raise ValidationError('Duplicate node ID: %s' % node_id) self.graph.add_node(node_id, node) def run(self, job): eg = ExecutionGraph(self, job) while eg.has_next(): next_id, next = eg.next_job() self.executor.execute(next, eg.job_done, next_id) return eg.outputs def to_dict(self, context): d = super(Workflow, self).to_dict(context) d.update({ "class": "Workflow", 'steps': [step.to_dict(context) for step in self.steps] }) return d @classmethod def from_dict(cls, context, d): converted = {} for k, v in six.iteritems(d): if k == 'steps': converted[k] = [Step.from_dict(context, s) for s in v] else: converted[k] = context.from_dict(v) kwargs = Process.kwarg_dict(converted) kwargs.update({ 'steps': converted['steps'], 'data_links': converted.get('dataLinks'), 'context': context, 'inputs': [ InputParameter.from_dict(context, i) for i in converted['inputs'] ], 'outputs': [ WorkflowOutput.from_dict(context, o) for o in converted['outputs'] ] }) return cls(**kwargs)
def test_nodes(self): graph = Graph() self.assertEqual(graph.node_list(), []) o1 = object() o1b = object() o2 = object() graph.add_node(1, o1) graph.add_node(1, o1b) graph.add_node(2, o2) graph.add_node(3) self.assertRaises(TypeError, graph.add_node, []) self.assertTrue(graph.node_data(1) is o1) self.assertTrue(graph.node_data(2) is o2) self.assertTrue(graph.node_data(3) is None) self.assertTrue(1 in graph) self.assertTrue(2 in graph) self.assertTrue(3 in graph) self.assertEqual(graph.number_of_nodes(), 3) self.assertEqual(graph.number_of_hidden_nodes(), 0) self.assertEqual(graph.hidden_node_list(), []) self.assertEqual(list(sorted(graph)), [1, 2, 3]) graph.hide_node(1) graph.hide_node(2) graph.hide_node(3) self.assertEqual(graph.number_of_nodes(), 0) self.assertEqual(graph.number_of_hidden_nodes(), 3) self.assertEqual(list(sorted(graph.hidden_node_list())), [1, 2, 3]) self.assertFalse(1 in graph) self.assertFalse(2 in graph) self.assertFalse(3 in graph) graph.add_node(1) self.assertFalse(1 in graph) graph.restore_node(1) self.assertTrue(1 in graph) self.assertFalse(2 in graph) self.assertFalse(3 in graph) graph.restore_all_nodes() self.assertTrue(1 in graph) self.assertTrue(2 in graph) self.assertTrue(3 in graph) self.assertEqual(list(sorted(graph.node_list())), [1, 2, 3]) v = graph.describe_node(1) self.assertEqual(v, (1, o1, [], []))
def __init__(self, app_id, steps, context, inputs=None, outputs=None, to=None, app_description=None, annotations=None, platform_features=None): self.graph = Graph() self.inputs = inputs or [] self.outputs = outputs or [] self.executor = context.executor self.steps = steps self.to = to or {} self.context = context for step in steps: self.add_node(step.id, AppNode(step.app, {})) for step in steps: # inputs for input_port, input_val in six.iteritems(step.inputs): inp = wrap_in_list(input_val) for item in inp: self.add_edge_or_input(step, input_port, item) # outputs if step.outputs: for output_port, output_val in six.iteritems(step.outputs): self.to[output_val['$to']] = output_port if isinstance(step.app, WorkflowApp): output_node = step.app.get_output(step.app.to.get(output_port)) else: output_node = step.app.get_output(output_port) output_id = output_val['$to'] self.add_node(output_id, output_node) self.graph.add_edge( step.id, output_id, OutputRelation(output_port) ) # output_node.id = output_val['$to'] self.outputs.append(output_node) if not self.graph.connected(): pass # raise ValidationError('Graph is not connected') schema = { "@type": "JsonSchema", "type": "object", "properties": {}, "required": [] } for inp in self.inputs: schema['properties'][inp.id] = inp.validator.schema if inp.required: schema['required'].append(inp.id) super(WorkflowApp, self).__init__( app_id, JsonSchema(context, schema), self.outputs, app_description=app_description, annotations=annotations, platform_features=platform_features )
def test_clust_coef(self): g = Graph() g.add_edge(1, 2) g.add_edge(1, 3) g.add_edge(1, 4) self.assertEqual(g.clust_coef(1), 0) g.add_edge(2, 5) g.add_edge(3, 5) g.add_edge(4, 5) self.assertEqual(g.clust_coef(1), 0) g.add_edge(2, 3) self.assertEqual(g.clust_coef(1), 1./6) g.add_edge(2, 4) self.assertEqual(g.clust_coef(1), 2./6) g.add_edge(4, 2) self.assertEqual(g.clust_coef(1), 3./6) g.add_edge(2, 3) g.add_edge(2, 4) g.add_edge(3, 4) g.add_edge(3, 2) g.add_edge(4, 2) g.add_edge(4, 3) self.assertEqual(g.clust_coef(1), 1)
speedParams = [ 0, 100, 0, 100 ] # highest speed, lowest speed, highest speed of pri, lowest speed of pri planTimeParams = [0, 10000000, 0, 10000000] # latest, earliest, latest of pri, earliest of pri diffSrc = [ {}, {} ] # allCarDict, priCarDict, the key of dict is cross_id, key is unique in dict diffDst = [{}, {}] # allCarDict, priCarDict factor = [0, 0] priority_car_list = [] car_distribution = [0, 0, 0] car_id_list, road_id_list, cross_id_list = [], [], [] cross_dict, car_dict, road_dict = {}, {}, {} graph = Graph() class Car(object): def __init__(self, id, from_, to, speed, planTime, priority, preset): self.id, self.from_, self.to, self.speed, self.planTime, self.priority, self.preset = \ id, from_, to, speed, planTime, priority, preset self.depart_time = None self.state = 0 # state 0-ingarage, 1-wait, 2-end, 3-finishtrip, self.x, self.y = 0, 0 # x is the position of the channel, y is the number of channel self.presentRoad, self.nextCrossId = None, self.from_ self.route, self.routeIndex = None, None self.presetChanged = False def init(self): self.state = 0 # state 0-ingarage, 1-wait, 2-end, 3-finishtrip,
def test_get_hops(self): graph = Graph() graph.add_edge(1, 2) graph.add_edge(1, 3) graph.add_edge(2, 4) graph.add_edge(4, 5) graph.add_edge(5, 7) graph.add_edge(7, 8) self.assertEqual(graph.get_hops(1), [(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)]) self.assertEqual(graph.get_hops(1, 5), [(1, 0), (2, 1), (3, 1), (4, 2), (5, 3)]) graph.add_edge(5, 1) graph.add_edge(7, 1) graph.add_edge(7, 4) self.assertEqual(graph.get_hops(1), [(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)]) # And the reverse graph graph = Graph() graph.add_edge(2, 1) graph.add_edge(3, 1) graph.add_edge(4, 2) graph.add_edge(5, 4) graph.add_edge(7, 5) graph.add_edge(8, 7) self.assertEqual(graph.get_hops(1, forward=False), [(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)]) self.assertEqual(graph.get_hops(1, 5, forward=False), [(1, 0), (2, 1), (3, 1), (4, 2), (5, 3)]) graph.add_edge(1, 5) graph.add_edge(1, 7) graph.add_edge(4, 7) self.assertEqual(graph.get_hops(1, forward=False), [(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)])
def test_toposort(self): graph = Graph() graph.add_node(1) graph.add_node(2) graph.add_node(3) graph.add_node(4) graph.add_node(5) graph.add_edge(1, 2) graph.add_edge(1, 3) graph.add_edge(2, 4) graph.add_edge(3, 5) ok, result = graph.forw_topo_sort() self.assertTrue(ok) for idx in range(1, 6): self.assertTrue(idx in result) self.assertTrue(result.index(1) < result.index(2)) self.assertTrue(result.index(1) < result.index(3)) self.assertTrue(result.index(2) < result.index(4)) self.assertTrue(result.index(3) < result.index(5)) ok, result = graph.back_topo_sort() self.assertTrue(ok) for idx in range(1, 6): self.assertTrue(idx in result) self.assertTrue(result.index(2) < result.index(1)) self.assertTrue(result.index(3) < result.index(1)) self.assertTrue(result.index(4) < result.index(2)) self.assertTrue(result.index(5) < result.index(3)) # Same graph as before, but with edges # reversed, which means we should get # the same results as before if using # back_topo_sort rather than forw_topo_sort # (and v.v.) graph = Graph() graph.add_node(1) graph.add_node(2) graph.add_node(3) graph.add_node(4) graph.add_node(5) graph.add_edge(2, 1) graph.add_edge(3, 1) graph.add_edge(4, 2) graph.add_edge(5, 3) ok, result = graph.back_topo_sort() self.assertTrue(ok) for idx in range(1, 6): self.assertTrue(idx in result) self.assertTrue(result.index(1) < result.index(2)) self.assertTrue(result.index(1) < result.index(3)) self.assertTrue(result.index(2) < result.index(4)) self.assertTrue(result.index(3) < result.index(5)) ok, result = graph.forw_topo_sort() self.assertTrue(ok) for idx in range(1, 6): self.assertTrue(idx in result) self.assertTrue(result.index(2) < result.index(1)) self.assertTrue(result.index(3) < result.index(1)) self.assertTrue(result.index(4) < result.index(2)) self.assertTrue(result.index(5) < result.index(3)) # Create a cycle graph.add_edge(1, 5) ok, result = graph.forw_topo_sort() self.assertFalse(ok) ok, result = graph.back_topo_sort() self.assertFalse(ok)
def test_edges(self): graph = Graph() graph.add_node(1) graph.add_node(2) graph.add_node(3) graph.add_node(4) graph.add_node(5) self.assertTrue(isinstance(graph.edge_list(), list)) graph.add_edge(1, 2) graph.add_edge(4, 5, 'a') self.assertRaises(GraphError, graph.add_edge, 'a', 'b', create_nodes=False) self.assertEqual(graph.number_of_hidden_edges(), 0) self.assertEqual(graph.number_of_edges(), 2) e = graph.edge_by_node(1, 2) self.assertTrue(isinstance(e, int)) graph.hide_edge(e) self.assertEqual(graph.number_of_hidden_edges(), 1) self.assertEqual(graph.number_of_edges(), 1) e2 = graph.edge_by_node(1, 2) self.assertTrue(e2 is None) graph.restore_edge(e) e2 = graph.edge_by_node(1, 2) self.assertEqual(e, e2) self.assertEqual(graph.number_of_hidden_edges(), 0) self.assertEqual(graph.number_of_edges(), 2) e1 = graph.edge_by_node(1, 2) e2 = graph.edge_by_node(4, 5) graph.hide_edge(e1) graph.hide_edge(e2) self.assertEqual(graph.number_of_edges(), 0) graph.restore_all_edges() self.assertEqual(graph.number_of_edges(), 2) self.assertEqual(graph.edge_by_id(e1), (1,2)) self.assertRaises(GraphError, graph.edge_by_id, (e1+1)*(e2+1)+1) self.assertEqual(list(sorted(graph.edge_list())), [e1, e2]) self.assertEqual(graph.describe_edge(e1), (e1, 1, 1, 2)) self.assertEqual(graph.describe_edge(e2), (e2, 'a', 4, 5)) self.assertEqual(graph.edge_data(e1), 1) self.assertEqual(graph.edge_data(e2), 'a') self.assertEqual(graph.head(e2), 4) self.assertEqual(graph.tail(e2), 5) graph.add_edge(1, 3) graph.add_edge(1, 5) graph.add_edge(4, 1) self.assertEqual(list(sorted(graph.out_nbrs(1))), [2, 3, 5]) self.assertEqual(list(sorted(graph.inc_nbrs(1))), [4]) self.assertEqual(list(sorted(graph.inc_nbrs(5))), [1, 4]) self.assertEqual(list(sorted(graph.all_nbrs(1))), [2, 3, 4, 5]) graph.add_edge(5, 1) self.assertEqual(list(sorted(graph.all_nbrs(5))), [1, 4]) self.assertEqual(graph.out_degree(1), 3) self.assertEqual(graph.inc_degree(2), 1) self.assertEqual(graph.inc_degree(5), 2) self.assertEqual(graph.all_degree(5), 3) v = graph.out_edges(4) self.assertTrue(isinstance(v, list)) self.assertEqual(graph.edge_by_id(v[0]), (4, 5)) v = graph.out_edges(1) for e in v: self.assertEqual(graph.edge_by_id(e)[0], 1) v = graph.inc_edges(1) self.assertTrue(isinstance(v, list)) self.assertEqual(graph.edge_by_id(v[0]), (4, 1)) v = graph.inc_edges(5) for e in v: self.assertEqual(graph.edge_by_id(e)[1], 5) v = graph.all_edges(5) for e in v: self.assertTrue(graph.edge_by_id(e)[1] == 5 or graph.edge_by_id(e)[0] == 5) e1 = graph.edge_by_node(1, 2) self.assertTrue(isinstance(e1, int)) graph.hide_node(1) self.assertRaises(GraphError, graph.edge_by_node, 1, 2) graph.restore_node(1) e2 = graph.edge_by_node(1, 2) self.assertEqual(e1, e2)
def test_bfs_subgraph(self): graph = Graph() graph.add_edge(1, 2) graph.add_edge(1, 4) graph.add_edge(2, 4) graph.add_edge(4, 8) graph.add_edge(4, 9) graph.add_edge(4, 10) graph.add_edge(8, 10) subgraph = graph.forw_bfs_subgraph(10) self.assertTrue(isinstance(subgraph, Graph)) self.assertEqual(subgraph.number_of_nodes(), 1) self.assertTrue(10 in subgraph) self.assertEqual(subgraph.number_of_edges(), 0) subgraph = graph.forw_bfs_subgraph(4) self.assertTrue(isinstance(subgraph, Graph)) self.assertEqual(subgraph.number_of_nodes(), 4) self.assertTrue(4 in subgraph) self.assertTrue(8 in subgraph) self.assertTrue(9 in subgraph) self.assertTrue(10 in subgraph) self.assertEqual(subgraph.number_of_edges(), 4) e = subgraph.edge_by_node(4, 8) e = subgraph.edge_by_node(4, 9) e = subgraph.edge_by_node(4, 10) e = subgraph.edge_by_node(8, 10) # same graph as before, but switch around # edges. This results in the same test results # but now for back_bfs_subgraph rather than # forw_bfs_subgraph graph = Graph() graph.add_edge(2, 1) graph.add_edge(4, 1) graph.add_edge(4, 2) graph.add_edge(8, 4) graph.add_edge(9, 4) graph.add_edge(10, 4) graph.add_edge(10, 8) subgraph = graph.back_bfs_subgraph(10) self.assertTrue(isinstance(subgraph, Graph)) self.assertEqual(subgraph.number_of_nodes(), 1) self.assertTrue(10 in subgraph) self.assertEqual(subgraph.number_of_edges(), 0) subgraph = graph.back_bfs_subgraph(4) self.assertTrue(isinstance(subgraph, Graph)) self.assertEqual(subgraph.number_of_nodes(), 4) self.assertTrue(4 in subgraph) self.assertTrue(8 in subgraph) self.assertTrue(9 in subgraph) self.assertTrue(10 in subgraph) self.assertEqual(subgraph.number_of_edges(), 4) e = subgraph.edge_by_node(4, 8) e = subgraph.edge_by_node(4, 9) e = subgraph.edge_by_node(4, 10) e = subgraph.edge_by_node(8, 10)
class Workflow(Process): def __init__(self, process_id, inputs, outputs, requirements, hints, label, description, steps, context, data_links=None): super(Workflow, self).__init__( process_id, inputs, outputs, requirements, hints, label, description ) self.graph = Graph() self.executor = context.executor self.steps = steps self.data_links = data_links or [] self.context = context self.port_step_index = {} for step in steps: node = AppNode(step.app, {}) self.add_node(step.id, node) for inp in step.inputs: self.port_step_index[inp.id] = step.id self.move_connect_to_datalink(inp) if inp.value: node.inputs[inp.id] = inp.value for out in step.outputs: self.port_step_index[out.id] = step.id for inp in self.inputs: self.add_node(inp.id, inp) for out in self.outputs: self.move_connect_to_datalink(out) self.add_node(out.id, out) for dl in self.data_links: dst = dl['destination'].lstrip('#') src = dl['source'].lstrip('#') if src in self.port_step_index and dst in self.port_step_index: rel = Relation(src, dst, dl.get('position', 0)) src = self.port_step_index[src] dst = self.port_step_index[dst] elif src in self._inputs: rel = InputRelation(dst, dl.get('position', 0)) dst = self.port_step_index[dst] elif dst in self._outputs: rel = OutputRelation(src, dl.get('position', 0)) src = self.port_step_index[src] else: raise RabixError("invalid data link %s" % dl) self.graph.add_edge(src, dst, rel) if not self.graph.connected(): pass # raise ValidationError('Graph is not connected') def move_connect_to_datalink(self, port): for dl in port.connect: dl['destination'] = '#'+port.id self.data_links.append(dl) del port.connect[:] # Graph.add_node silently fails if node already exists def add_node(self, node_id, node): if node_id in self.graph.nodes: raise ValidationError('Duplicate node ID: %s' % node_id) self.graph.add_node(node_id, node) def hide_nodes(self, type): for node_id in self.graph.node_list(): node = self.graph.node_data(node_id) if isinstance(node, type): self.graph.hide_node(node_id) def run(self, job): eg = ExecutionGraph(self, job) while eg.has_next(): next_id, next = eg.next_job() self.executor.execute(next, eg.job_done, next_id) return eg.outputs def to_dict(self, context): d = super(Workflow, self).to_dict(context) d.update({ "class": "Workflow", 'steps': [step.to_dict(context) for step in self.steps] }) return d @classmethod def from_dict(cls, context, d): converted = {} for k, v in six.iteritems(d): if k == 'steps': converted[k] = [Step.from_dict(context, s) for s in v] else: converted[k] = context.from_dict(v) kwargs = Process.kwarg_dict(converted) kwargs.update({ 'steps': converted['steps'], 'data_links': converted.get('dataLinks'), 'context': context, 'inputs': [InputParameter.from_dict(context, i) for i in converted['inputs']], 'outputs': [WorkflowOutput.from_dict(context, o) for o in converted['outputs']] }) return cls(**kwargs)
def test_bfs_subgraph_does_not_reverse_egde_direction(self): graph = Graph() graph.add_node('A') graph.add_node('B') graph.add_node('C') graph.add_edge('A', 'B') graph.add_edge('B', 'C') whole_graph = graph.forw_topo_sort() subgraph_backward = graph.back_bfs_subgraph('C') subgraph_backward = subgraph_backward.forw_topo_sort() self.assertEquals(whole_graph, subgraph_backward) subgraph_forward = graph.forw_bfs_subgraph('A') subgraph_forward = subgraph_forward.forw_topo_sort() self.assertEquals(whole_graph, subgraph_forward)
def main(): if len(sys.argv) != 6: logging.info( 'please input args: car_path, road_path, cross_path, answerPath') exit(1) car_path = sys.argv[1] road_path = sys.argv[2] cross_path = sys.argv[3] preset_answer_path = sys.argv[4] answer_path = sys.argv[5] logging.info("car_path is %s" % (car_path)) logging.info("road_path is %s" % (road_path)) logging.info("cross_path is %s" % (cross_path)) logging.info("preset_answer_path is %s" % (preset_answer_path)) logging.info("answer_path is %s" % (answer_path)) car_list, road_list, cross_list, preset_answer_list = readFiles( car_path, road_path, cross_path, preset_answer_path) carInfo = open( car_path, 'r').read().split('\n')[1:] # ['(61838, 1716, 800, 4, 72, 0, 0)', ...] roadInfo = open(road_path, 'r').read().split('\n')[1:] crossInfo = open(cross_path, 'r').read().split('\n')[1:] preset_answer_info = generatePresetAnswer(preset_answer_path) graph = Graph() graph = initMap(graph, road_list, cross_list) queue_length = 100 # car_list = sorted(car_list, key=lambda x: x[4]) # first car scheduled first for non-preset cars preset = [x for x in car_list if x[6] == 1] car_list = replaceDepartTimeForPresetCar(car_list, preset_answer_list) last_preset_depart_time = max([x[4] for x in preset]) split_time = (last_preset_depart_time + 100) // 100 * 100 print("last_preset_depart_time: %s, split_time: %s" % (last_preset_depart_time, split_time)) # priority cars depart first, then non-priority cars depart priority_non_preset = [x for x in car_list if x[5] == 1 and x[6] == 0] non_priority_non_preset = [x for x in car_list if x[5] == 0 and x[6] == 0] priority_non_preset = sorted(priority_non_preset, key=lambda x: x[4]) non_priority_non_preset = sorted(non_priority_non_preset, key=lambda x: x[4]) car_list = priority_non_preset + non_priority_non_preset ori_car_list = copy.deepcopy(car_list) if len(cross_list) > 140: # for big map penaltyFactor = 80 departure_rate = 120 acc_departure_rate = 130 last_d = 100 else: # for small map penaltyFactor = 140 departure_rate = 100 acc_departure_rate = 100 last_d = 100 # TODO 3: to avoid 'program runs too long' # After realtime test, successful scheduling(including route calculation) cannot be over 2 times loop = 3 JudgeTime_list = [999999 for x in range(loop)] answer_info_list = [None for x in range(loop)] last_deadlock = True step = 30 for i in range(loop): print('-----Start calculating route------') print("\ncurrent departure rate: %s, acc departure rate: %s" % (departure_rate, acc_departure_rate)) # car_list = ori_car_list # This is totally wrong! because car_list refer to ori_car_list, not copy the value from ori_car_list !!! car_list = copy.deepcopy(ori_car_list) car_list = chooseDepartTimeForNonPresetCar(car_list, departure_rate, acc_departure_rate, split_time, last_d) # merge non-preset cars and preset cars into car_list car_list.extend(preset) # sort car_list in ascending depart_time, so that dynamic penalty works car_list = sorted(car_list, key=lambda x: x[4]) # TODO 2: static change(pre-decide) preset route, to enable this, not to comment line below # changed_route_dict = changePresetRoute(car_list, road_list, preset_answer_list) # print('change preset car number: %s' % changed_route_dict.__len__()) changed_route_dict = { } # TODO 2: comment this to enable pre-decide preset car route_dict = findRouteForCar(graph, car_list, cross_list, road_list, preset_answer_list, penaltyFactor, queue_length, changed_route_dict) # print('find route number: %s' % route_dict.__len__()) answer_info = generateAnswer(route_dict, car_list) print('-----Finish calculating route------') writeFiles(answer_info, answer_path) print('-----Start scheduling--------------') scheduler = NodeadlockScheduler(carInfo, roadInfo, crossInfo, answer_info, preset_answer_info) isDeadLock, time, JudgeTime = scheduler.schedule() print('-----Finish Scheduling: current JudgeTime: %s, isDeadLock: %s' % (JudgeTime, isDeadLock)) if isDeadLock: if last_deadlock: departure_rate -= step acc_departure_rate -= step else: step = step // 2 departure_rate -= step acc_departure_rate -= step print( 'last time ok, this time deadlock, step half to -%s, now departure rate=%s, acc departure rate=%s' % (step, departure_rate, acc_departure_rate)) last_deadlock = True else: JudgeTime_list[i] = JudgeTime answer_info_list[i] = answer_info if last_deadlock: if i == 0: departure_rate += step acc_departure_rate += step else: step = step // 2 departure_rate += step acc_departure_rate += step print( 'last time deadlock, this time ok, step half to +%s, now departure rate=%s, acc departure rate=%s' % (step, departure_rate, acc_departure_rate)) else: departure_rate += step acc_departure_rate += step last_deadlock = False if i == loop - 2 and answer_info_list.count(None) == 1: # no running time for the last scheduling, get the current optimal answer info print('runs ok 2 times in first %s time, skip the last schedule' % (loop - 1)) break del graph graph = Graph() graph = initMap(graph, road_list, cross_list) # if deadlock for 3 times, get a low result to commit if answer_info_list.count(None) == loop: departure_rate, acc_departure_rate = 40, 50 car_list = copy.deepcopy(ori_car_list) car_list = chooseDepartTimeForNonPresetCar(car_list, departure_rate, acc_departure_rate, split_time, last_d) # merge non-preset cars and preset cars into car_list car_list.extend(preset) # sort car_list in ascending depart_time, so that dynamic penalty works car_list = sorted(car_list, key=lambda x: x[4]) # changed_route_dict = changePresetRoute(car_list, road_list, preset_answer_list) # print('change preset car number: %s' % changed_route_dict.__len__()) changed_route_dict = {} route_dict = findRouteForCar(graph, car_list, cross_list, road_list, preset_answer_list, penaltyFactor, queue_length, changed_route_dict) # print('find route number: %s' % route_dict.__len__()) answer_info = generateAnswer(route_dict, car_list) else: i = JudgeTime_list.index(min(JudgeTime_list)) print('optimal judge time: %d' % min(JudgeTime_list)) answer_info = answer_info_list[i] writeFiles(answer_info, answer_path)
def test_iterdfs(self): graph = Graph() graph.add_edge("1", "1.1") graph.add_edge("1", "1.2") graph.add_edge("1", "1.3") graph.add_edge("1.1", "1.1.1") graph.add_edge("1.1", "1.1.2") graph.add_edge("1.2", "1.2.1") graph.add_edge("1.2", "1.2.2") graph.add_edge("1.2.2", "1.2.2.1") graph.add_edge("1.2.2", "1.2.2.2") graph.add_edge("1.2.2", "1.2.2.3") result = list(graph.iterdfs("1")) self.assertEqual(result, [ '1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2', '1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1' ]) result = list(graph.iterdfs("1", "1.2.1")) self.assertEqual(result, [ '1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2', '1.2.2.1', '1.2.1' ]) result = graph.forw_dfs("1") self.assertEqual(result, [ '1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2', '1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1' ]) result = graph.forw_dfs("1", "1.2.1") self.assertEqual(result, [ '1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2', '1.2.2.1', '1.2.1' ]) graph = Graph() graph.add_edge("1.1", "1") graph.add_edge("1.2", "1") graph.add_edge("1.3", "1") graph.add_edge("1.1.1", "1.1") graph.add_edge("1.1.2", "1.1") graph.add_edge("1.2.1", "1.2") graph.add_edge("1.2.2", "1.2") graph.add_edge("1.2.2.1", "1.2.2") graph.add_edge("1.2.2.2", "1.2.2") graph.add_edge("1.2.2.3", "1.2.2") result = list(graph.iterdfs("1", forward=False)) self.assertEqual(result, [ '1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2', '1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1' ]) result = list(graph.iterdfs("1", "1.2.1", forward=False)) self.assertEqual(result, [ '1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2', '1.2.2.1', '1.2.1' ]) result = graph.back_dfs("1") self.assertEqual(result, [ '1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2', '1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1' ]) result = graph.back_dfs("1", "1.2.1") self.assertEqual(result, [ '1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2', '1.2.2.1', '1.2.1' ]) # Introduce cyle: graph.add_edge("1", "1.2") result = list(graph.iterdfs("1", forward=False)) self.assertEqual(result, [ '1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2', '1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1' ]) result = graph.back_dfs("1") self.assertEqual(result, [ '1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2', '1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1' ])