def __init__(self,
process_id,
inputs,
outputs,
requirements,
hints,
label,
description,
steps,
context,
data_links=None):
super(Workflow, self).__init__(process_id, inputs, outputs,
requirements, hints, label, description)
self.graph = Graph()
self.executor = context.executor
self.steps = steps
self.data_links = data_links or []
self.context = context
self.port_step_index = {}
for step in steps:
node = AppNode(step.app, {})
self.add_node(step.id, node)
for inp in step.inputs:
self.port_step_index[inp.id] = step.id
self.move_connect_to_datalink(inp)
if inp.value:
node.inputs[inp.id] = inp.value
for out in step.outputs:
self.port_step_index[out.id] = step.id
for inp in self.inputs:
self.add_node(inp.id, inp)
for out in self.outputs:
self.move_connect_to_datalink(out)
self.add_node(out.id, out)
for dl in self.data_links:
dst = dl['destination'].lstrip('#')
src = dl['source'].lstrip('#')
if src in self.port_step_index and dst in self.port_step_index:
rel = Relation(src, dst, dl.get('position', 0))
src = self.port_step_index[src]
dst = self.port_step_index[dst]
elif src in self._inputs:
rel = InputRelation(dst, dl.get('position', 0))
dst = self.port_step_index[dst]
elif dst in self._outputs:
rel = OutputRelation(src, dl.get('position', 0))
src = self.port_step_index[src]
else:
raise RabixError("invalid data link %s" % dl)
self.graph.add_edge(src, dst, rel)
if not self.graph.connected():
pass
def __init__(self, graph=None, debug=0):
if graph is None:
graph = Graph()
self.graphident = self
self.graph = graph
self.debug = debug
self.indent = 0
graph.add_node(self, None)
def __init__(self, graph=None, debug=0):
if graph is None:
graph = Graph()
self.graphident = self
self.graph = graph
self.debug = debug
self.indent = 0
graph.add_node(self, None)
def __init__(self, process_id, inputs, outputs, requirements, hints, label,
description, steps, context, data_links=None):
super(Workflow, self).__init__(
process_id, inputs, outputs, requirements,
hints, label, description
)
self.graph = Graph()
self.executor = context.executor
self.steps = steps
self.data_links = data_links or []
self.context = context
self.port_step_index = {}
for step in steps:
node = AppNode(step.app, {})
self.add_node(step.id, node)
for inp in step.inputs:
self.port_step_index[inp.id] = step.id
self.move_connect_to_datalink(inp)
if inp.value:
node.inputs[inp.id] = inp.value
for out in step.outputs:
self.port_step_index[out.id] = step.id
for inp in self.inputs:
self.add_node(inp.id, inp)
for out in self.outputs:
self.move_connect_to_datalink(out)
self.add_node(out.id, out)
# dedupe links
s = {tuple(dl.items()) for dl in self.data_links}
self.data_links = [dict(dl) for dl in s]
for dl in self.data_links:
dst = dl['destination'].lstrip('#')
src = dl['source'].lstrip('#')
if src in self.port_step_index and dst in self.port_step_index:
rel = Relation(src, dst, dl.get('position', 0))
src = self.port_step_index[src]
dst = self.port_step_index[dst]
elif src in self._inputs:
rel = InputRelation(dst, dl.get('position', 0))
dst = self.port_step_index[dst]
elif dst in self._outputs:
rel = OutputRelation(src, dl.get('position', 0))
src = self.port_step_index[src]
else:
raise RabixError("invalid data link %s" % dl)
self.graph.add_edge(src, dst, rel)
if not self.graph.connected():
pass
def test_constructor(self):
graph = Graph(iter([
(1, 2),
(2, 3, 'a'),
(1, 3),
(3, 4),
]))
self.assertEqual(graph.number_of_nodes(), 4)
self.assertEqual(graph.number_of_edges(), 4)
try:
graph.edge_by_node(1,2)
graph.edge_by_node(2,3)
graph.edge_by_node(1,3)
graph.edge_by_node(3,4)
except GraphError:
self.fail("Incorrect graph")
self.assertEqual(graph.edge_data(graph.edge_by_node(2, 3)), 'a')
self.assertRaises(GraphError, Graph, [(1,2,3,4)])
def test_constructor(self):
graph = ObjectGraph()
self.assertTrue(isinstance(graph, ObjectGraph))
g = Graph()
graph = ObjectGraph(g)
self.assertTrue(graph.graph is g)
self.assertEqual(graph.debug, 0)
self.assertEqual(graph.indent, 0)
graph = ObjectGraph(debug=5)
self.assertEqual(graph.debug, 5)
def copy_to_graph(session):
graph = Graph()
count = 0
for artist in session.query(LastFmArtist).all():
if artist.graph_out:
graph.add_node(artist.id, artist)
edges = list(artist.graph_out)
edges.sort(key=lambda e: e.weight, reverse=True)
for edge in edges[0:2]:
target = edge.to_
graph.add_node(target.id, target)
graph.add_edge(artist.id, target.id, edge, create_nodes=False)
if not count % 100:
LOG.info('Added {0} nodes.'.format(count))
count += 1
else:
LOG.warn('Discarding unconnected {0}.'.format(artist.name))
return graph
def davaj_trať(id_relace):
'''
zahájí běh programu
'''
from altgraph.Graph import Graph
graf = Graph()
cesty = davaj_cesty_z_relace(id_relace)
# vložím cesty do grafu
for cesta in cesty:
graf.add_edge(cesta.body_v_cestě[0], cesta.body_v_cestě[-1], {
'id': cesta.id_cesty,
'pořadí': cesta.pořadí_cesty
})
print('graf má {} vrcholů a {} hran'.format(graf.number_of_nodes(),
graf.number_of_edges()))
# projdu všechny vrcholy a vyberu ty, které mají pouze jednu hranu
jednohrané_vrcholy = []
for bod in graf.node_list():
if graf.all_degree(bod) == 1:
# print("vrchol {} vstupuje {} a vystupuje {} hran, celkem {}".format(bod, graf.inc_degree(bod), graf.out_degree(bod), graf.all_degree(bod)))
jednohrané_vrcholy.append(bod)
# jednohrané vrcholy musí být právě dva
if len(jednohrané_vrcholy) != 2:
raise ValueError(
'Tož ale graf musí být právě jeden počátek a právě jeden konec')
# když má graf právě jeden začátek a právě jeden konec, možeme srovnat hrany, otočit je do jednoho směru
else:
print('jednohrané vrcholy', jednohrané_vrcholy)
# tu práci obsatrá rekurzivně tato funkce, která graf pozmění
seřadím_hrany_v_grafu(graf, jednohrané_vrcholy[0])
#včíl projdu už upravený, správně natočený graf a načtu všechny body
najdu_body_pro_hrany_v_grafu(graf)
def test_edges_complex(self):
g = Graph()
g.add_edge(1, 2)
e = g.edge_by_node(1,2)
g.hide_edge(e)
g.hide_node(2)
self.assertRaises(GraphError, g.restore_edge, e)
g.restore_all_edges()
self.assertRaises(GraphError, g.edge_by_id, e)
def test_bfs(self):
graph = Graph()
graph.add_edge("1", "1.1")
graph.add_edge("1.1", "1.1.1")
graph.add_edge("1.1", "1.1.2")
graph.add_edge("1.1.2", "1.1.2.1")
graph.add_edge("1.1.2", "1.1.2.2")
graph.add_edge("1", "1.2")
graph.add_edge("1", "1.3")
graph.add_edge("1.2", "1.2.1")
self.assertEqual(graph.forw_bfs("1"),
['1', '1.1', '1.2', '1.3', '1.1.1', '1.1.2', '1.2.1', '1.1.2.1', '1.1.2.2'])
self.assertEqual(graph.forw_bfs("1", "1.1.1"),
['1', '1.1', '1.2', '1.3', '1.1.1'])
# And the "reverse" graph
graph = Graph()
graph.add_edge("1.1", "1")
graph.add_edge("1.1.1", "1.1")
graph.add_edge("1.1.2", "1.1")
graph.add_edge("1.1.2.1", "1.1.2")
graph.add_edge("1.1.2.2", "1.1.2")
graph.add_edge("1.2", "1")
graph.add_edge("1.3", "1")
graph.add_edge("1.2.1", "1.2")
self.assertEqual(graph.back_bfs("1"),
['1', '1.1', '1.2', '1.3', '1.1.1', '1.1.2', '1.2.1', '1.1.2.1', '1.1.2.2'])
self.assertEqual(graph.back_bfs("1", "1.1.1"),
['1', '1.1', '1.2', '1.3', '1.1.1'])
# check cycle handling
graph.add_edge("1", "1.2.1")
self.assertEqual(graph.back_bfs("1"),
['1', '1.1', '1.2', '1.3', '1.1.1', '1.1.2', '1.2.1', '1.1.2.1', '1.1.2.2'])
def test_connected(self):
graph = Graph()
graph.add_node(1)
graph.add_node(2)
graph.add_node(3)
graph.add_node(4)
self.assertFalse(graph.connected())
graph.add_edge(1, 2)
graph.add_edge(3, 4)
self.assertFalse(graph.connected())
graph.add_edge(2, 3)
graph.add_edge(4, 1)
self.assertTrue(graph.connected())
class WorkflowApp(App):
def __init__(self, app_id, steps, context,
inputs=None, outputs=None, to=None,
app_description=None,
annotations=None,
platform_features=None):
self.graph = Graph()
self.inputs = inputs or []
self.outputs = outputs or []
self.executor = context.executor
self.steps = steps
self.to = to or {}
self.context = context
for step in steps:
self.add_node(step.id, AppNode(step.app, {}))
for step in steps:
# inputs
for input_port, input_val in six.iteritems(step.inputs):
inp = wrap_in_list(input_val)
for item in inp:
self.add_edge_or_input(step, input_port, item)
# outputs
if step.outputs:
for output_port, output_val in six.iteritems(step.outputs):
self.to[output_val['$to']] = output_port
if isinstance(step.app, WorkflowApp):
output_node = step.app.get_output(step.app.to.get(output_port))
else:
output_node = step.app.get_output(output_port)
output_id = output_val['$to']
self.add_node(output_id, output_node)
self.graph.add_edge(
step.id, output_id, OutputRelation(output_port)
)
# output_node.id = output_val['$to']
self.outputs.append(output_node)
if not self.graph.connected():
pass
# raise ValidationError('Graph is not connected')
schema = {
"@type": "JsonSchema",
"type": "object",
"properties": {},
"required": []
}
for inp in self.inputs:
schema['properties'][inp.id] = inp.validator.schema
if inp.required:
schema['required'].append(inp.id)
super(WorkflowApp, self).__init__(
app_id, JsonSchema(context, schema), self.outputs,
app_description=app_description,
annotations=annotations,
platform_features=platform_features
)
def add_edge_or_input(self, step, input_name, input_val):
node_id = step.id
if isinstance(input_val, dict) and '$from' in input_val:
frm = wrap_in_list(input_val['$from'])
for inp in frm:
if '.' in inp:
node, outp = inp.split('.')
self.graph.add_edge(node, node_id, Relation(outp, input_name))
else:
# TODO: merge input schemas if one input goes to different apps
input = step.app.get_input(input_name)
if inp not in self.graph.nodes:
self.add_node(inp, input)
self.graph.add_edge(
inp, node_id, InputRelation(input_name)
)
wf_input = copy.deepcopy(input)
wf_input.id = inp
self.inputs.append(wf_input)
else:
self.graph.node_data(node_id).inputs[input_name] = input_val
# Graph.add_node silently fails if node already exists
def add_node(self, node_id, node):
if node_id in self.graph.nodes:
raise ValidationError('Duplicate node ID: %s' % node_id)
self.graph.add_node(node_id, node)
def hide_nodes(self, type):
for node_id in self.graph.node_list():
node = self.graph.node_data(node_id)
if isinstance(node, type):
self.graph.hide_node(node_id)
def run(self, job):
eg = ExecutionGraph(self, job)
while eg.has_next():
next_id, next = eg.next_job()
self.executor.execute(next, eg.job_done, next_id)
return eg.outputs
def to_dict(self, context):
d = super(WorkflowApp, self).to_dict(context)
d.update({
"@type": "Workflow",
'steps': [step.to_dict(context) for step in self.steps]
})
return d
@classmethod
def from_dict(cls, context, d):
steps = [Step(
step['id'], context.from_dict(step['app']),
step['inputs'], step.get('outputs')
)
for step in d['steps']]
return cls(
d.get('@id', six.text_type(uuid4())),
steps,
context
)
def test_iterdata(self):
graph = Graph()
graph.add_node("1", "I")
graph.add_node("1.1", "I.I")
graph.add_node("1.2", "I.II")
graph.add_node("1.3", "I.III")
graph.add_node("1.1.1", "I.I.I")
graph.add_node("1.1.2", "I.I.II")
graph.add_node("1.2.1", "I.II.I")
graph.add_node("1.2.2", "I.II.II")
graph.add_node("1.2.2.1", "I.II.II.I")
graph.add_node("1.2.2.2", "I.II.II.II")
graph.add_node("1.2.2.3", "I.II.II.III")
graph.add_edge("1", "1.1")
graph.add_edge("1", "1.2")
graph.add_edge("1", "1.3")
graph.add_edge("1.1", "1.1.1")
graph.add_edge("1.1", "1.1.2")
graph.add_edge("1.2", "1.2.1")
graph.add_edge("1.2", "1.2.2")
graph.add_edge("1.2.2", "1.2.2.1")
graph.add_edge("1.2.2", "1.2.2.2")
graph.add_edge("1.2.2", "1.2.2.3")
result = list(graph.iterdata("1", forward=True))
self.assertEqual(result, [
'I', 'I.III', 'I.II', 'I.II.II', 'I.II.II.III', 'I.II.II.II',
'I.II.II.I', 'I.II.I', 'I.I', 'I.I.II', 'I.I.I'
])
result = list(graph.iterdata("1", end="1.2.1", forward=True))
self.assertEqual(result, [
'I', 'I.III', 'I.II', 'I.II.II', 'I.II.II.III', 'I.II.II.II',
'I.II.II.I', 'I.II.I'
])
result = list(graph.iterdata("1", condition=lambda n: len(n) < 6, forward=True))
self.assertEqual(result, [
'I', 'I.III', 'I.II',
'I.I', 'I.I.I'
])
# And the revese option:
graph = Graph()
graph.add_node("1", "I")
graph.add_node("1.1", "I.I")
graph.add_node("1.2", "I.II")
graph.add_node("1.3", "I.III")
graph.add_node("1.1.1", "I.I.I")
graph.add_node("1.1.2", "I.I.II")
graph.add_node("1.2.1", "I.II.I")
graph.add_node("1.2.2", "I.II.II")
graph.add_node("1.2.2.1", "I.II.II.I")
graph.add_node("1.2.2.2", "I.II.II.II")
graph.add_node("1.2.2.3", "I.II.II.III")
graph.add_edge("1.1", "1")
graph.add_edge("1.2", "1")
graph.add_edge("1.3", "1")
graph.add_edge("1.1.1", "1.1")
graph.add_edge("1.1.2", "1.1")
graph.add_edge("1.2.1", "1.2")
graph.add_edge("1.2.2", "1.2")
graph.add_edge("1.2.2.1", "1.2.2")
graph.add_edge("1.2.2.2", "1.2.2")
graph.add_edge("1.2.2.3", "1.2.2")
result = list(graph.iterdata("1", forward=False))
self.assertEqual(result, [
'I', 'I.III', 'I.II', 'I.II.II', 'I.II.II.III', 'I.II.II.II',
'I.II.II.I', 'I.II.I', 'I.I', 'I.I.II', 'I.I.I'
])
result = list(graph.iterdata("1", end="1.2.1", forward=False))
self.assertEqual(result, [
'I', 'I.III', 'I.II', 'I.II.II', 'I.II.II.III', 'I.II.II.II',
'I.II.II.I', 'I.II.I'
])
result = list(graph.iterdata("1", condition=lambda n: len(n) < 6, forward=False))
self.assertEqual(result, [
'I', 'I.III', 'I.II',
'I.I', 'I.I.I'
])
class Workflow(Process):
def __init__(self,
process_id,
inputs,
outputs,
requirements,
hints,
label,
description,
steps,
context,
data_links=None):
super(Workflow, self).__init__(process_id, inputs, outputs,
requirements, hints, label, description)
self.graph = Graph()
self.executor = context.executor
self.steps = steps
self.data_links = data_links or []
self.context = context
self.port_step_index = {}
for step in steps:
node = AppNode(step.app, {})
self.add_node(step.id, node)
for inp in step.inputs:
self.port_step_index[inp.id] = step.id
self.move_connect_to_datalink(inp)
if inp.value:
node.inputs[inp.id] = inp.value
for out in step.outputs:
self.port_step_index[out.id] = step.id
for inp in self.inputs:
self.add_node(inp.id, inp)
for out in self.outputs:
self.move_connect_to_datalink(out)
self.add_node(out.id, out)
# dedupe links
s = {tuple(dl.items()) for dl in self.data_links}
self.data_links = [dict(dl) for dl in s]
for dl in self.data_links:
dst = dl['destination'].lstrip('#')
src = dl['source'].lstrip('#')
if src in self.port_step_index and dst in self.port_step_index:
rel = Relation(src, dst, dl.get('position', 0))
src = self.port_step_index[src]
dst = self.port_step_index[dst]
elif src in self._inputs:
rel = InputRelation(dst, dl.get('position', 0))
dst = self.port_step_index[dst]
elif dst in self._outputs:
rel = OutputRelation(src, dl.get('position', 0))
src = self.port_step_index[src]
else:
raise RabixError("invalid data link %s" % dl)
self.graph.add_edge(src, dst, rel)
if not self.graph.connected():
pass
# raise ValidationError('Graph is not connected')
def move_connect_to_datalink(self, port):
for src in port.source:
self.data_links.append({
'source': src,
'destination': '#' + port.id
})
del port.source[:]
# Graph.add_node silently fails if node already exists
def add_node(self, node_id, node):
if node_id in self.graph.nodes:
raise ValidationError('Duplicate node ID: %s' % node_id)
self.graph.add_node(node_id, node)
def run(self, job):
eg = ExecutionGraph(self, job)
while eg.has_next():
next_id, next = eg.next_job()
self.executor.execute(next, eg.job_done, next_id)
return eg.outputs
def to_dict(self, context):
d = super(Workflow, self).to_dict(context)
d.update({
"class": "Workflow",
'steps': [step.to_dict(context) for step in self.steps]
})
return d
@classmethod
def from_dict(cls, context, d):
converted = {}
for k, v in six.iteritems(d):
if k == 'steps':
converted[k] = [Step.from_dict(context, s) for s in v]
else:
converted[k] = context.from_dict(v)
kwargs = Process.kwarg_dict(converted)
kwargs.update({
'steps':
converted['steps'],
'data_links':
converted.get('dataLinks'),
'context':
context,
'inputs': [
InputParameter.from_dict(context, i)
for i in converted['inputs']
],
'outputs': [
WorkflowOutput.from_dict(context, o)
for o in converted['outputs']
]
})
return cls(**kwargs)
def test_nodes(self):
graph = Graph()
self.assertEqual(graph.node_list(), [])
o1 = object()
o1b = object()
o2 = object()
graph.add_node(1, o1)
graph.add_node(1, o1b)
graph.add_node(2, o2)
graph.add_node(3)
self.assertRaises(TypeError, graph.add_node, [])
self.assertTrue(graph.node_data(1) is o1)
self.assertTrue(graph.node_data(2) is o2)
self.assertTrue(graph.node_data(3) is None)
self.assertTrue(1 in graph)
self.assertTrue(2 in graph)
self.assertTrue(3 in graph)
self.assertEqual(graph.number_of_nodes(), 3)
self.assertEqual(graph.number_of_hidden_nodes(), 0)
self.assertEqual(graph.hidden_node_list(), [])
self.assertEqual(list(sorted(graph)), [1, 2, 3])
graph.hide_node(1)
graph.hide_node(2)
graph.hide_node(3)
self.assertEqual(graph.number_of_nodes(), 0)
self.assertEqual(graph.number_of_hidden_nodes(), 3)
self.assertEqual(list(sorted(graph.hidden_node_list())), [1, 2, 3])
self.assertFalse(1 in graph)
self.assertFalse(2 in graph)
self.assertFalse(3 in graph)
graph.add_node(1)
self.assertFalse(1 in graph)
graph.restore_node(1)
self.assertTrue(1 in graph)
self.assertFalse(2 in graph)
self.assertFalse(3 in graph)
graph.restore_all_nodes()
self.assertTrue(1 in graph)
self.assertTrue(2 in graph)
self.assertTrue(3 in graph)
self.assertEqual(list(sorted(graph.node_list())), [1, 2, 3])
v = graph.describe_node(1)
self.assertEqual(v, (1, o1, [], []))
def __init__(self, app_id, steps, context,
inputs=None, outputs=None, to=None,
app_description=None,
annotations=None,
platform_features=None):
self.graph = Graph()
self.inputs = inputs or []
self.outputs = outputs or []
self.executor = context.executor
self.steps = steps
self.to = to or {}
self.context = context
for step in steps:
self.add_node(step.id, AppNode(step.app, {}))
for step in steps:
# inputs
for input_port, input_val in six.iteritems(step.inputs):
inp = wrap_in_list(input_val)
for item in inp:
self.add_edge_or_input(step, input_port, item)
# outputs
if step.outputs:
for output_port, output_val in six.iteritems(step.outputs):
self.to[output_val['$to']] = output_port
if isinstance(step.app, WorkflowApp):
output_node = step.app.get_output(step.app.to.get(output_port))
else:
output_node = step.app.get_output(output_port)
output_id = output_val['$to']
self.add_node(output_id, output_node)
self.graph.add_edge(
step.id, output_id, OutputRelation(output_port)
)
# output_node.id = output_val['$to']
self.outputs.append(output_node)
if not self.graph.connected():
pass
# raise ValidationError('Graph is not connected')
schema = {
"@type": "JsonSchema",
"type": "object",
"properties": {},
"required": []
}
for inp in self.inputs:
schema['properties'][inp.id] = inp.validator.schema
if inp.required:
schema['required'].append(inp.id)
super(WorkflowApp, self).__init__(
app_id, JsonSchema(context, schema), self.outputs,
app_description=app_description,
annotations=annotations,
platform_features=platform_features
)
def test_clust_coef(self):
g = Graph()
g.add_edge(1, 2)
g.add_edge(1, 3)
g.add_edge(1, 4)
self.assertEqual(g.clust_coef(1), 0)
g.add_edge(2, 5)
g.add_edge(3, 5)
g.add_edge(4, 5)
self.assertEqual(g.clust_coef(1), 0)
g.add_edge(2, 3)
self.assertEqual(g.clust_coef(1), 1./6)
g.add_edge(2, 4)
self.assertEqual(g.clust_coef(1), 2./6)
g.add_edge(4, 2)
self.assertEqual(g.clust_coef(1), 3./6)
g.add_edge(2, 3)
g.add_edge(2, 4)
g.add_edge(3, 4)
g.add_edge(3, 2)
g.add_edge(4, 2)
g.add_edge(4, 3)
self.assertEqual(g.clust_coef(1), 1)
speedParams = [
0, 100, 0, 100
] # highest speed, lowest speed, highest speed of pri, lowest speed of pri
planTimeParams = [0, 10000000, 0,
10000000] # latest, earliest, latest of pri, earliest of pri
diffSrc = [
{}, {}
] # allCarDict, priCarDict, the key of dict is cross_id, key is unique in dict
diffDst = [{}, {}] # allCarDict, priCarDict
factor = [0, 0]
priority_car_list = []
car_distribution = [0, 0, 0]
car_id_list, road_id_list, cross_id_list = [], [], []
cross_dict, car_dict, road_dict = {}, {}, {}
graph = Graph()
class Car(object):
def __init__(self, id, from_, to, speed, planTime, priority, preset):
self.id, self.from_, self.to, self.speed, self.planTime, self.priority, self.preset = \
id, from_, to, speed, planTime, priority, preset
self.depart_time = None
self.state = 0 # state 0-ingarage, 1-wait, 2-end, 3-finishtrip,
self.x, self.y = 0, 0 # x is the position of the channel, y is the number of channel
self.presentRoad, self.nextCrossId = None, self.from_
self.route, self.routeIndex = None, None
self.presetChanged = False
def init(self):
self.state = 0 # state 0-ingarage, 1-wait, 2-end, 3-finishtrip,
def test_get_hops(self):
graph = Graph()
graph.add_edge(1, 2)
graph.add_edge(1, 3)
graph.add_edge(2, 4)
graph.add_edge(4, 5)
graph.add_edge(5, 7)
graph.add_edge(7, 8)
self.assertEqual(graph.get_hops(1),
[(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)])
self.assertEqual(graph.get_hops(1, 5),
[(1, 0), (2, 1), (3, 1), (4, 2), (5, 3)])
graph.add_edge(5, 1)
graph.add_edge(7, 1)
graph.add_edge(7, 4)
self.assertEqual(graph.get_hops(1),
[(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)])
# And the reverse graph
graph = Graph()
graph.add_edge(2, 1)
graph.add_edge(3, 1)
graph.add_edge(4, 2)
graph.add_edge(5, 4)
graph.add_edge(7, 5)
graph.add_edge(8, 7)
self.assertEqual(graph.get_hops(1, forward=False),
[(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)])
self.assertEqual(graph.get_hops(1, 5, forward=False),
[(1, 0), (2, 1), (3, 1), (4, 2), (5, 3)])
graph.add_edge(1, 5)
graph.add_edge(1, 7)
graph.add_edge(4, 7)
self.assertEqual(graph.get_hops(1, forward=False),
[(1, 0), (2, 1), (3, 1), (4, 2), (5, 3), (7, 4), (8, 5)])
def test_toposort(self):
graph = Graph()
graph.add_node(1)
graph.add_node(2)
graph.add_node(3)
graph.add_node(4)
graph.add_node(5)
graph.add_edge(1, 2)
graph.add_edge(1, 3)
graph.add_edge(2, 4)
graph.add_edge(3, 5)
ok, result = graph.forw_topo_sort()
self.assertTrue(ok)
for idx in range(1, 6):
self.assertTrue(idx in result)
self.assertTrue(result.index(1) < result.index(2))
self.assertTrue(result.index(1) < result.index(3))
self.assertTrue(result.index(2) < result.index(4))
self.assertTrue(result.index(3) < result.index(5))
ok, result = graph.back_topo_sort()
self.assertTrue(ok)
for idx in range(1, 6):
self.assertTrue(idx in result)
self.assertTrue(result.index(2) < result.index(1))
self.assertTrue(result.index(3) < result.index(1))
self.assertTrue(result.index(4) < result.index(2))
self.assertTrue(result.index(5) < result.index(3))
# Same graph as before, but with edges
# reversed, which means we should get
# the same results as before if using
# back_topo_sort rather than forw_topo_sort
# (and v.v.)
graph = Graph()
graph.add_node(1)
graph.add_node(2)
graph.add_node(3)
graph.add_node(4)
graph.add_node(5)
graph.add_edge(2, 1)
graph.add_edge(3, 1)
graph.add_edge(4, 2)
graph.add_edge(5, 3)
ok, result = graph.back_topo_sort()
self.assertTrue(ok)
for idx in range(1, 6):
self.assertTrue(idx in result)
self.assertTrue(result.index(1) < result.index(2))
self.assertTrue(result.index(1) < result.index(3))
self.assertTrue(result.index(2) < result.index(4))
self.assertTrue(result.index(3) < result.index(5))
ok, result = graph.forw_topo_sort()
self.assertTrue(ok)
for idx in range(1, 6):
self.assertTrue(idx in result)
self.assertTrue(result.index(2) < result.index(1))
self.assertTrue(result.index(3) < result.index(1))
self.assertTrue(result.index(4) < result.index(2))
self.assertTrue(result.index(5) < result.index(3))
# Create a cycle
graph.add_edge(1, 5)
ok, result = graph.forw_topo_sort()
self.assertFalse(ok)
ok, result = graph.back_topo_sort()
self.assertFalse(ok)
def test_edges(self):
graph = Graph()
graph.add_node(1)
graph.add_node(2)
graph.add_node(3)
graph.add_node(4)
graph.add_node(5)
self.assertTrue(isinstance(graph.edge_list(), list))
graph.add_edge(1, 2)
graph.add_edge(4, 5, 'a')
self.assertRaises(GraphError, graph.add_edge, 'a', 'b', create_nodes=False)
self.assertEqual(graph.number_of_hidden_edges(), 0)
self.assertEqual(graph.number_of_edges(), 2)
e = graph.edge_by_node(1, 2)
self.assertTrue(isinstance(e, int))
graph.hide_edge(e)
self.assertEqual(graph.number_of_hidden_edges(), 1)
self.assertEqual(graph.number_of_edges(), 1)
e2 = graph.edge_by_node(1, 2)
self.assertTrue(e2 is None)
graph.restore_edge(e)
e2 = graph.edge_by_node(1, 2)
self.assertEqual(e, e2)
self.assertEqual(graph.number_of_hidden_edges(), 0)
self.assertEqual(graph.number_of_edges(), 2)
e1 = graph.edge_by_node(1, 2)
e2 = graph.edge_by_node(4, 5)
graph.hide_edge(e1)
graph.hide_edge(e2)
self.assertEqual(graph.number_of_edges(), 0)
graph.restore_all_edges()
self.assertEqual(graph.number_of_edges(), 2)
self.assertEqual(graph.edge_by_id(e1), (1,2))
self.assertRaises(GraphError, graph.edge_by_id, (e1+1)*(e2+1)+1)
self.assertEqual(list(sorted(graph.edge_list())), [e1, e2])
self.assertEqual(graph.describe_edge(e1), (e1, 1, 1, 2))
self.assertEqual(graph.describe_edge(e2), (e2, 'a', 4, 5))
self.assertEqual(graph.edge_data(e1), 1)
self.assertEqual(graph.edge_data(e2), 'a')
self.assertEqual(graph.head(e2), 4)
self.assertEqual(graph.tail(e2), 5)
graph.add_edge(1, 3)
graph.add_edge(1, 5)
graph.add_edge(4, 1)
self.assertEqual(list(sorted(graph.out_nbrs(1))), [2, 3, 5])
self.assertEqual(list(sorted(graph.inc_nbrs(1))), [4])
self.assertEqual(list(sorted(graph.inc_nbrs(5))), [1, 4])
self.assertEqual(list(sorted(graph.all_nbrs(1))), [2, 3, 4, 5])
graph.add_edge(5, 1)
self.assertEqual(list(sorted(graph.all_nbrs(5))), [1, 4])
self.assertEqual(graph.out_degree(1), 3)
self.assertEqual(graph.inc_degree(2), 1)
self.assertEqual(graph.inc_degree(5), 2)
self.assertEqual(graph.all_degree(5), 3)
v = graph.out_edges(4)
self.assertTrue(isinstance(v, list))
self.assertEqual(graph.edge_by_id(v[0]), (4, 5))
v = graph.out_edges(1)
for e in v:
self.assertEqual(graph.edge_by_id(e)[0], 1)
v = graph.inc_edges(1)
self.assertTrue(isinstance(v, list))
self.assertEqual(graph.edge_by_id(v[0]), (4, 1))
v = graph.inc_edges(5)
for e in v:
self.assertEqual(graph.edge_by_id(e)[1], 5)
v = graph.all_edges(5)
for e in v:
self.assertTrue(graph.edge_by_id(e)[1] == 5 or graph.edge_by_id(e)[0] == 5)
e1 = graph.edge_by_node(1, 2)
self.assertTrue(isinstance(e1, int))
graph.hide_node(1)
self.assertRaises(GraphError, graph.edge_by_node, 1, 2)
graph.restore_node(1)
e2 = graph.edge_by_node(1, 2)
self.assertEqual(e1, e2)
def test_bfs_subgraph(self):
graph = Graph()
graph.add_edge(1, 2)
graph.add_edge(1, 4)
graph.add_edge(2, 4)
graph.add_edge(4, 8)
graph.add_edge(4, 9)
graph.add_edge(4, 10)
graph.add_edge(8, 10)
subgraph = graph.forw_bfs_subgraph(10)
self.assertTrue(isinstance(subgraph, Graph))
self.assertEqual(subgraph.number_of_nodes(), 1)
self.assertTrue(10 in subgraph)
self.assertEqual(subgraph.number_of_edges(), 0)
subgraph = graph.forw_bfs_subgraph(4)
self.assertTrue(isinstance(subgraph, Graph))
self.assertEqual(subgraph.number_of_nodes(), 4)
self.assertTrue(4 in subgraph)
self.assertTrue(8 in subgraph)
self.assertTrue(9 in subgraph)
self.assertTrue(10 in subgraph)
self.assertEqual(subgraph.number_of_edges(), 4)
e = subgraph.edge_by_node(4, 8)
e = subgraph.edge_by_node(4, 9)
e = subgraph.edge_by_node(4, 10)
e = subgraph.edge_by_node(8, 10)
# same graph as before, but switch around
# edges. This results in the same test results
# but now for back_bfs_subgraph rather than
# forw_bfs_subgraph
graph = Graph()
graph.add_edge(2, 1)
graph.add_edge(4, 1)
graph.add_edge(4, 2)
graph.add_edge(8, 4)
graph.add_edge(9, 4)
graph.add_edge(10, 4)
graph.add_edge(10, 8)
subgraph = graph.back_bfs_subgraph(10)
self.assertTrue(isinstance(subgraph, Graph))
self.assertEqual(subgraph.number_of_nodes(), 1)
self.assertTrue(10 in subgraph)
self.assertEqual(subgraph.number_of_edges(), 0)
subgraph = graph.back_bfs_subgraph(4)
self.assertTrue(isinstance(subgraph, Graph))
self.assertEqual(subgraph.number_of_nodes(), 4)
self.assertTrue(4 in subgraph)
self.assertTrue(8 in subgraph)
self.assertTrue(9 in subgraph)
self.assertTrue(10 in subgraph)
self.assertEqual(subgraph.number_of_edges(), 4)
e = subgraph.edge_by_node(4, 8)
e = subgraph.edge_by_node(4, 9)
e = subgraph.edge_by_node(4, 10)
e = subgraph.edge_by_node(8, 10)
class Workflow(Process):
def __init__(self, process_id, inputs, outputs, requirements, hints, label,
description, steps, context, data_links=None):
super(Workflow, self).__init__(
process_id, inputs, outputs, requirements,
hints, label, description
)
self.graph = Graph()
self.executor = context.executor
self.steps = steps
self.data_links = data_links or []
self.context = context
self.port_step_index = {}
for step in steps:
node = AppNode(step.app, {})
self.add_node(step.id, node)
for inp in step.inputs:
self.port_step_index[inp.id] = step.id
self.move_connect_to_datalink(inp)
if inp.value:
node.inputs[inp.id] = inp.value
for out in step.outputs:
self.port_step_index[out.id] = step.id
for inp in self.inputs:
self.add_node(inp.id, inp)
for out in self.outputs:
self.move_connect_to_datalink(out)
self.add_node(out.id, out)
for dl in self.data_links:
dst = dl['destination'].lstrip('#')
src = dl['source'].lstrip('#')
if src in self.port_step_index and dst in self.port_step_index:
rel = Relation(src, dst, dl.get('position', 0))
src = self.port_step_index[src]
dst = self.port_step_index[dst]
elif src in self._inputs:
rel = InputRelation(dst, dl.get('position', 0))
dst = self.port_step_index[dst]
elif dst in self._outputs:
rel = OutputRelation(src, dl.get('position', 0))
src = self.port_step_index[src]
else:
raise RabixError("invalid data link %s" % dl)
self.graph.add_edge(src, dst, rel)
if not self.graph.connected():
pass
# raise ValidationError('Graph is not connected')
def move_connect_to_datalink(self, port):
for dl in port.connect:
dl['destination'] = '#'+port.id
self.data_links.append(dl)
del port.connect[:]
# Graph.add_node silently fails if node already exists
def add_node(self, node_id, node):
if node_id in self.graph.nodes:
raise ValidationError('Duplicate node ID: %s' % node_id)
self.graph.add_node(node_id, node)
def hide_nodes(self, type):
for node_id in self.graph.node_list():
node = self.graph.node_data(node_id)
if isinstance(node, type):
self.graph.hide_node(node_id)
def run(self, job):
eg = ExecutionGraph(self, job)
while eg.has_next():
next_id, next = eg.next_job()
self.executor.execute(next, eg.job_done, next_id)
return eg.outputs
def to_dict(self, context):
d = super(Workflow, self).to_dict(context)
d.update({
"class": "Workflow",
'steps': [step.to_dict(context) for step in self.steps]
})
return d
@classmethod
def from_dict(cls, context, d):
converted = {}
for k, v in six.iteritems(d):
if k == 'steps':
converted[k] = [Step.from_dict(context, s) for s in v]
else:
converted[k] = context.from_dict(v)
kwargs = Process.kwarg_dict(converted)
kwargs.update({
'steps': converted['steps'],
'data_links': converted.get('dataLinks'),
'context': context,
'inputs': [InputParameter.from_dict(context, i)
for i in converted['inputs']],
'outputs': [WorkflowOutput.from_dict(context, o)
for o in converted['outputs']]
})
return cls(**kwargs)
def test_bfs_subgraph_does_not_reverse_egde_direction(self):
graph = Graph()
graph.add_node('A')
graph.add_node('B')
graph.add_node('C')
graph.add_edge('A', 'B')
graph.add_edge('B', 'C')
whole_graph = graph.forw_topo_sort()
subgraph_backward = graph.back_bfs_subgraph('C')
subgraph_backward = subgraph_backward.forw_topo_sort()
self.assertEquals(whole_graph, subgraph_backward)
subgraph_forward = graph.forw_bfs_subgraph('A')
subgraph_forward = subgraph_forward.forw_topo_sort()
self.assertEquals(whole_graph, subgraph_forward)
def main():
if len(sys.argv) != 6:
logging.info(
'please input args: car_path, road_path, cross_path, answerPath')
exit(1)
car_path = sys.argv[1]
road_path = sys.argv[2]
cross_path = sys.argv[3]
preset_answer_path = sys.argv[4]
answer_path = sys.argv[5]
logging.info("car_path is %s" % (car_path))
logging.info("road_path is %s" % (road_path))
logging.info("cross_path is %s" % (cross_path))
logging.info("preset_answer_path is %s" % (preset_answer_path))
logging.info("answer_path is %s" % (answer_path))
car_list, road_list, cross_list, preset_answer_list = readFiles(
car_path, road_path, cross_path, preset_answer_path)
carInfo = open(
car_path,
'r').read().split('\n')[1:] # ['(61838, 1716, 800, 4, 72, 0, 0)', ...]
roadInfo = open(road_path, 'r').read().split('\n')[1:]
crossInfo = open(cross_path, 'r').read().split('\n')[1:]
preset_answer_info = generatePresetAnswer(preset_answer_path)
graph = Graph()
graph = initMap(graph, road_list, cross_list)
queue_length = 100
# car_list = sorted(car_list, key=lambda x: x[4]) # first car scheduled first for non-preset cars
preset = [x for x in car_list if x[6] == 1]
car_list = replaceDepartTimeForPresetCar(car_list, preset_answer_list)
last_preset_depart_time = max([x[4] for x in preset])
split_time = (last_preset_depart_time + 100) // 100 * 100
print("last_preset_depart_time: %s, split_time: %s" %
(last_preset_depart_time, split_time))
# priority cars depart first, then non-priority cars depart
priority_non_preset = [x for x in car_list if x[5] == 1 and x[6] == 0]
non_priority_non_preset = [x for x in car_list if x[5] == 0 and x[6] == 0]
priority_non_preset = sorted(priority_non_preset, key=lambda x: x[4])
non_priority_non_preset = sorted(non_priority_non_preset,
key=lambda x: x[4])
car_list = priority_non_preset + non_priority_non_preset
ori_car_list = copy.deepcopy(car_list)
if len(cross_list) > 140:
# for big map
penaltyFactor = 80
departure_rate = 120
acc_departure_rate = 130
last_d = 100
else:
# for small map
penaltyFactor = 140
departure_rate = 100
acc_departure_rate = 100
last_d = 100
# TODO 3: to avoid 'program runs too long'
# After realtime test, successful scheduling(including route calculation) cannot be over 2 times
loop = 3
JudgeTime_list = [999999 for x in range(loop)]
answer_info_list = [None for x in range(loop)]
last_deadlock = True
step = 30
for i in range(loop):
print('-----Start calculating route------')
print("\ncurrent departure rate: %s, acc departure rate: %s" %
(departure_rate, acc_departure_rate))
# car_list = ori_car_list # This is totally wrong! because car_list refer to ori_car_list, not copy the value from ori_car_list !!!
car_list = copy.deepcopy(ori_car_list)
car_list = chooseDepartTimeForNonPresetCar(car_list, departure_rate,
acc_departure_rate,
split_time, last_d)
# merge non-preset cars and preset cars into car_list
car_list.extend(preset)
# sort car_list in ascending depart_time, so that dynamic penalty works
car_list = sorted(car_list, key=lambda x: x[4])
# TODO 2: static change(pre-decide) preset route, to enable this, not to comment line below
# changed_route_dict = changePresetRoute(car_list, road_list, preset_answer_list)
# print('change preset car number: %s' % changed_route_dict.__len__())
changed_route_dict = {
} # TODO 2: comment this to enable pre-decide preset car
route_dict = findRouteForCar(graph, car_list, cross_list, road_list,
preset_answer_list, penaltyFactor,
queue_length, changed_route_dict)
# print('find route number: %s' % route_dict.__len__())
answer_info = generateAnswer(route_dict, car_list)
print('-----Finish calculating route------')
writeFiles(answer_info, answer_path)
print('-----Start scheduling--------------')
scheduler = NodeadlockScheduler(carInfo, roadInfo, crossInfo,
answer_info, preset_answer_info)
isDeadLock, time, JudgeTime = scheduler.schedule()
print('-----Finish Scheduling: current JudgeTime: %s, isDeadLock: %s' %
(JudgeTime, isDeadLock))
if isDeadLock:
if last_deadlock:
departure_rate -= step
acc_departure_rate -= step
else:
step = step // 2
departure_rate -= step
acc_departure_rate -= step
print(
'last time ok, this time deadlock, step half to -%s, now departure rate=%s, acc departure rate=%s'
% (step, departure_rate, acc_departure_rate))
last_deadlock = True
else:
JudgeTime_list[i] = JudgeTime
answer_info_list[i] = answer_info
if last_deadlock:
if i == 0:
departure_rate += step
acc_departure_rate += step
else:
step = step // 2
departure_rate += step
acc_departure_rate += step
print(
'last time deadlock, this time ok, step half to +%s, now departure rate=%s, acc departure rate=%s'
% (step, departure_rate, acc_departure_rate))
else:
departure_rate += step
acc_departure_rate += step
last_deadlock = False
if i == loop - 2 and answer_info_list.count(None) == 1:
# no running time for the last scheduling, get the current optimal answer info
print('runs ok 2 times in first %s time, skip the last schedule' %
(loop - 1))
break
del graph
graph = Graph()
graph = initMap(graph, road_list, cross_list)
# if deadlock for 3 times, get a low result to commit
if answer_info_list.count(None) == loop:
departure_rate, acc_departure_rate = 40, 50
car_list = copy.deepcopy(ori_car_list)
car_list = chooseDepartTimeForNonPresetCar(car_list, departure_rate,
acc_departure_rate,
split_time, last_d)
# merge non-preset cars and preset cars into car_list
car_list.extend(preset)
# sort car_list in ascending depart_time, so that dynamic penalty works
car_list = sorted(car_list, key=lambda x: x[4])
# changed_route_dict = changePresetRoute(car_list, road_list, preset_answer_list)
# print('change preset car number: %s' % changed_route_dict.__len__())
changed_route_dict = {}
route_dict = findRouteForCar(graph, car_list, cross_list, road_list,
preset_answer_list, penaltyFactor,
queue_length, changed_route_dict)
# print('find route number: %s' % route_dict.__len__())
answer_info = generateAnswer(route_dict, car_list)
else:
i = JudgeTime_list.index(min(JudgeTime_list))
print('optimal judge time: %d' % min(JudgeTime_list))
answer_info = answer_info_list[i]
writeFiles(answer_info, answer_path)
def test_iterdfs(self):
graph = Graph()
graph.add_edge("1", "1.1")
graph.add_edge("1", "1.2")
graph.add_edge("1", "1.3")
graph.add_edge("1.1", "1.1.1")
graph.add_edge("1.1", "1.1.2")
graph.add_edge("1.2", "1.2.1")
graph.add_edge("1.2", "1.2.2")
graph.add_edge("1.2.2", "1.2.2.1")
graph.add_edge("1.2.2", "1.2.2.2")
graph.add_edge("1.2.2", "1.2.2.3")
result = list(graph.iterdfs("1"))
self.assertEqual(result, [
'1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2',
'1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1'
])
result = list(graph.iterdfs("1", "1.2.1"))
self.assertEqual(result, [
'1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2',
'1.2.2.1', '1.2.1'
])
result = graph.forw_dfs("1")
self.assertEqual(result, [
'1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2',
'1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1'
])
result = graph.forw_dfs("1", "1.2.1")
self.assertEqual(result, [
'1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2',
'1.2.2.1', '1.2.1'
])
graph = Graph()
graph.add_edge("1.1", "1")
graph.add_edge("1.2", "1")
graph.add_edge("1.3", "1")
graph.add_edge("1.1.1", "1.1")
graph.add_edge("1.1.2", "1.1")
graph.add_edge("1.2.1", "1.2")
graph.add_edge("1.2.2", "1.2")
graph.add_edge("1.2.2.1", "1.2.2")
graph.add_edge("1.2.2.2", "1.2.2")
graph.add_edge("1.2.2.3", "1.2.2")
result = list(graph.iterdfs("1", forward=False))
self.assertEqual(result, [
'1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2',
'1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1'
])
result = list(graph.iterdfs("1", "1.2.1", forward=False))
self.assertEqual(result, [
'1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2',
'1.2.2.1', '1.2.1'
])
result = graph.back_dfs("1")
self.assertEqual(result, [
'1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2',
'1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1'
])
result = graph.back_dfs("1", "1.2.1")
self.assertEqual(result, [
'1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2',
'1.2.2.1', '1.2.1'
])
# Introduce cyle:
graph.add_edge("1", "1.2")
result = list(graph.iterdfs("1", forward=False))
self.assertEqual(result, [
'1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2',
'1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1'
])
result = graph.back_dfs("1")
self.assertEqual(result, [
'1', '1.3', '1.2', '1.2.2', '1.2.2.3', '1.2.2.2',
'1.2.2.1', '1.2.1', '1.1', '1.1.2', '1.1.1'
])
