def test_topological_sort_on_very_deep_graph(self):
gr = pygraph.classes.graph.graph()
gr.add_nodes(list(range(0,20001)))
for i in range(0,20000):
gr.add_edge((i,i+1))
recursionlimit = getrecursionlimit()
topological_sorting(gr)
assert getrecursionlimit() == recursionlimit
def sort_nodes_topologically(graph, nodeLs):
"""
Get a topological sort of a subset of the nodes of a graph
@type graph: graph_wrapper.GraphWrapper
@param graph: a graph in which the nodes reside
@type nodeLs: list [node]
@param nodeLs: a list of nodes from which to generate sorting. nodes must not be mutually accessive!
@rtype: list [node]
@return: topological sort of the nodes
"""
# uid_dic = dict([(node.uid,node) for node in nodeLs])
# helperNodes = uid_dic.keys()
helperGraph = graph.__class__(originalSentence="") # TODO: efficiency - this is done this way to avoid circular dependency
helperGraph.add_nodes(nodeLs)
acc = accessibility(graph)
for node1 in nodeLs:
for node2 in acc[node1]:
if node2 in nodeLs:
if node1.uid != node2.uid: # TODO: efficiency
helperGraph.add_edge((node1, node2))
sorted_nodes = topological_sorting(helperGraph)
return sorted_nodes
def testDigraph(self):
def has_parent(node, list):
for each in list:
if gr.has_edge(each, node):
return True
return (ts == [])
gr = pygraph.digraph()
gr.add_nodes([0,1,2,3,4,5,6,7,8])
gr.add_edge(0,1)
gr.add_edge(0,2)
gr.add_edge(1,3)
gr.add_edge(1,4)
gr.add_edge(2,5)
gr.add_edge(2,6)
gr.add_edge(3,7)
gr.add_edge(8,0)
gr.add_edge(7,5)
gr.add_edge(3,0)
gr.add_edge(4,3)
gr.add_edge(2,7)
gr.add_edge(6,0)
ts = topological_sorting(gr)
while (ts):
x = ts.pop()
assert has_parent(x, ts)
def _invalidate_caches(self): 'invalidate the downstream caches of updated nodes' if len(self.updated) == 0: return # Sort the nodes in worklist and remove duplicates sg = topological_sorting(self.digraph) # sorted graph worklist = [] # insert nodes into worklist in sorted order for node in sg: if node in self.updated: worklist.append(node) self.updated.clear() # iterate through worklist while worklist: node = worklist.pop() # one item at a time downstream = breadth_first_search(self.digraph, root=node)[1] # get all downstream labels for n in downstream: if n in worklist: # remove labels that will already be done worklist.remove(n) # remove cache entries self.cache[n] = None
def resolve_plugin_dependencies(self):
graph = digraph()
problems = defaultdict(list)
def check_plugin_dependencies(plugin_id):
result = True
def add_problem(problem_type, plugin_id, dependency):
problems[plugin_id].append(problem_type(plugin_id, dependency))
result = False
for dependency in self.plugin_dependencies(plugin_id):
if dependency.id not in self.manifests:
add_problem(MissingDependency, plugin_id, dependency)
elif dependency.version:
if manifests[required_id].version not in dependency.version:
add_problem(IncorrectVersion, plugin_id, dependency)
elif dependency.id not in graph:
if dependency.id in self.enabled_plugins:
add_problem(IndirectDependency, plugin_id, dependency)
else:
add_problem(DisabledDependency, plugin_id, dependency)
return result
def remove_dependents(plugin_id):
for node in traversal(graph, plugin_id, 'pre'):
for dependent in graph[node]:
edge = node, dependent
problems[dependent].append(IndirectDependency(dependent,
graph.get_edge_properties(edge)['dependency']))
graph.del_node(node)
graph.add_nodes(self.enabled_plugins)
for plugin_id in self.enabled_plugins:
if check_plugin_dependencies(plugin_id):
for dependency in self.plugin_dependencies(plugin_id):
edge = dependency.id, plugin_id
graph.add_edge(edge)
graph.set_edge_properties(edge, dependency=dependency)
else:
remove_dependents(plugin_id)
transitive_deps = accessibility(graph)
cycle_nodes = [
node
for node in graph
if any(
(node in transitive_deps[dependent])
for dependent in transitive_deps[node]
if dependent != node)]
for node in cycle_nodes:
problems[node].append(CyclicDependency(node))
graph.del_node(node)
self.dependency_graph = graph
self._dependency_problems = problems
self._load_order = topological_sorting(graph)
def allOff(self):
'''Turn all servers off ungracefully
'''
nodeList = topological_sorting(self.graph)
for node in nodeList:
server = self.graph.node_attributes(node)
for outlet in server.getOutlets():
outlet.setState(False)
return
def evaluate_dag(self): 'Force every node to be evaluated' # First invalidate all changed caches self._invalidate_caches() # Then call update on every node sg = topological_sorting(self.digraph) # sorted graph for label in sg: self._update_node(label)
def getSortedNodeList(self):
'''
returns a list of the nodes topologically sorted
:return: a list of the nodes topologically sorted
'''
nodeList = topological_sorting(self.graph)
servers=[]
for node in nodeList:
servers.append(self.graph.node_attributes(node))
return servers
def order_seq_only(graph):
"""
A topological sort is performed on the graph. All actions are
enclosed into a Sequence ISE structure.
"""
_prepare(graph)
nodes = topological_sorting(graph.reverse())
actions = []
for node in nodes:
actions.extend(_create_actions_from(node, graph, create_deps=False))
return _create_final_xml_from(actions, SEQ)
def process(self):
""" Process image processing flow for IPFGraph """
# Invalidate all input ports in __blocks
((iport.invalidate() for iport in block.input_ports.values())
for block in self.__blocks.values())
graph = self._make_flow_graph()
# Apply topological sorting and execute processing blocks in
# topological order
sorted_graph = topological_sorting(graph)
for node in sorted_graph:
node().process()
def topo_up_down(graph):
"""
Yield the nodes of the graph, starting with the leaf-most
(latest topological) node, running up to the root-most
(earliest topological) node, and pushing back down to the leaves,
excepting the leaf-most node.
Undefined behavior if the graph is not a DAG.
graph: A->B->C
yields: (C, B, A, B)
"""
tsort = topological_sorting(graph)
for node in reversed(tsort):
yield node
for node in tsort[1 : -1]:
yield node
def test_topological_sorting_on_tree(self):
gr = testlib.new_graph()
st, pre, post = depth_first_search(gr)
tree = pygraph.classes.digraph.digraph()
for each in st:
if st[each]:
if (each not in tree.nodes()):
tree.add_node(each)
if (st[each] not in tree.nodes()):
tree.add_node(st[each])
tree.add_edge((st[each], each))
ts = topological_sorting(tree)
for each in ts:
if (st[each]):
assert ts.index(each) > ts.index(st[each])
def topo_sort_work(self):
my_work = self.work
work_dict = {}
for w in my_work:
work_dict[w.work_id] = w
graph = digraph()
graph.add_nodes(my_work)
for w in my_work:
for p in w.prereqs:
if not work_dict.has_key(p):
continue
if work_dict[p]:
graph.add_edge((work_dict[p], w))
self.work = topological_sorting(graph)
return
def find_connected_resources(resource, dependency_graph=None):
"""
Collects all resources connected to the given resource and returns a
dictionary mapping member resource classes to new collections containing
the members found.
"""
# Build a resource_graph.
resource_graph = \
build_resource_graph(resource,
dependency_graph=dependency_graph)
entity_map = OrderedDict()
for mb in topological_sorting(resource_graph):
mb_cls = get_member_class(mb)
ents = entity_map.get(mb_cls)
if ents is None:
ents = []
entity_map[mb_cls] = ents
ents.append(mb.get_entity())
return entity_map
def test_topological_sorting_on_digraph(self):
def is_ordered(node, list):
# Has parent on list
for each in list:
if gr.has_edge((each, node)):
return True
# Has no possible ancestors on list
st, pre, post = depth_first_search(gr, node)
for each in list:
if (each in st):
return False
return True
gr = testlib.new_digraph()
ts = topological_sorting(gr)
while (ts):
x = ts.pop()
assert is_ordered(x, ts)
def testTree(self):
gr = pygraph.digraph()
gr.add_nodes([0,1,2,3,4,5,6,7,8])
gr.add_edge(0,1)
gr.add_edge(0,2)
gr.add_edge(1,3)
gr.add_edge(1,4)
gr.add_edge(2,5)
gr.add_edge(2,6)
gr.add_edge(3,7)
gr.add_edge(8,0)
ts = topological_sorting(gr)
assert ts.index(8) < ts.index(0)
assert ts.index(1) > ts.index(0)
assert ts.index(2) > ts.index(0)
assert ts.index(3) > ts.index(1)
assert ts.index(4) > ts.index(1)
assert ts.index(5) > ts.index(2)
assert ts.index(6) > ts.index(2)
assert ts.index(7) > ts.index(3)
def find_connected_resources(resource, dependency_graph=None):
"""
Collects all resources connected to the given resource and returns a
dictionary mapping member resource classes to new collections containing
the members found.
"""
# Build a resource_graph.
resource_graph = \
build_resource_graph(resource,
dependency_graph=dependency_graph)
# Build an ordered dictionary of collections.
collections = OrderedDict()
for mb in topological_sorting(resource_graph):
mb_cls = get_member_class(mb)
coll = collections.get(mb_cls)
if coll is None:
# Create new collection.
coll = create_staging_collection(mb)
collections[mb_cls] = coll
coll.add(mb)
return collections
def transitive_edges(graph):
"""
Return a list of transitive edges.
Example of transitivity within graphs: A -> B, B -> C, A -> C
in this case the transitive edge is: A -> C
@attention: This function is only meaningful for directed acyclic graphs.
@type graph: digraph
@param graph: Digraph
@rtype: List
@return: List containing tuples with transitive edges (or an empty array if the digraph
contains a cycle)
"""
#if the LoopGraph contains a cycle we return an empty array
if not len(find_cycle(graph)) == 0:
return []
tranz_edges = [] # create an empty array that will contain all the tuples
#run trough all the nodes in the LoopGraph
for start in topological_sorting(graph):
#find all the successors on the path for the current node
successors = []
for a in traversal(graph,start,'pre'):
successors.append(a)
del successors[0] #we need all the nodes in it's path except the start node itself
for next in successors:
#look for an intersection between all the neighbors of the
#given node and all the neighbors from the given successor
intersect_array = _intersection(graph.neighbors(next), graph.neighbors(start) )
for a in intersect_array:
if graph.has_edge((start, a)):
##check for the detected edge and append it to the returned array
tranz_edges.append( (start,a) )
return tranz_edges # return the final array
def get_levels(self):
'''arrange the nodes in layers according to degree
level 0 will have no outgoing edges, last level will have no incoming edges'''
# Topological order: any strict order which satisfies the partial order
# as established by the outgoing edges
graph = self._graph
ordered = topological_sorting(graph)
ordered.reverse()
# print 'ordered %s'%ordered
result = []
for elem in ordered:
deps = graph.neighbors(elem)
maximo = -1
for i, level in enumerate(result):
for d in deps:
if d in level and i > maximo:
maximo = i
if maximo + 1 >= len(result):
result.append(set())
result[maximo + 1].add(elem)
return result
def calculate_dependencies(configurers):
"""Given a dictionary of name -> DependencyConfigurer objects, returns a
list with the leaf nodes of the implied dependencies first.
"""
# We do this via graph theory rather than hard-coding a particular startup
# order.
config_graph = digraph()
for configurer in configurers.values():
config_graph.add_node(configurer)
for configurer_name, configurer in configurers.items():
# Add outbound dependencies for every node.
for name in configurer.depends_on_names:
_log.debug("%s depends on %s", configurer_name, name)
config_graph.add_edge(configurer, configurers[name])
# Add inbound dependencies for every node.
for name in configurer.depended_on_names:
_log.debug("%s depends on %s", name, configurer_name)
config_graph.add_edge(configurers[name], configurer)
# Reverse here because in topological sorting, nodes with no inbound are first
# and nodes with no outbound edges are last. We actually want to do things the
# other way around.
return topological_sorting(config_graph.reverse())
def main():
"""Everybody's favorite starting function"""
parser = argparse.ArgumentParser(
description="Run a series of json specified jobs to exercise pipeline and test results."
)
parser.add_argument(
"-r",
"--replace",
help="Replace macro strings in commands with value in CURRENT=REPLACE format.",
action="append",
default=[],
)
parser.add_argument(
"--environment", help="Generic additions to the environment variables (path, etc.)", default="DUMMY=DUMMY"
)
parser.add_argument("--type", help="Types of jobs to run.", default="all")
parser.add_argument("--not_type", help="Types of jobs not to run.", default="all")
parser.add_argument("--jobs", help="json file with jobs to run")
parser.add_argument("--output", help="Directory for results and intermediate files.")
parser.add_argument("--jobnames", help="Name of particular jobs to run (e.g. 'job1|job2|job3').", default=None)
parser.add_argument("--update_file", help="Name of particular job to run.", default=None)
parser.add_argument("--update_decision", help="Name of particular job to run.", default=2)
parser.add_argument("--update_root", help="Path to gold data to update.", default=None)
parser.add_argument(
"--ts_base", help="Base of torrent suite checkout for auto setting of environment", default=None
)
parser.add_argument("--gold", help="Path to base of gold data root", default=None)
parser.add_argument("--data", help="Path to base of data to be used", default=None)
parser.add_argument("--failures", help="Failures log", default=None)
args = parser.parse_args()
args_ok = check_args(args)
# Put the output root in our regular expressions to be replaced
for i in range(len(args.replace)):
if re.match(args.replace[i], "__OUTPUT_ROOT__") != None:
print "__OUTPUT_ROOT_ is specified from --output not manually."
raise
args.replace.append("__OUTPUT_ROOT__=%s" % args.output)
# Put the gold root in our regular expressions to be replaced
if args.gold != None:
args.replace.append("__GOLD_ROOT__=%s" % args.gold)
args.update_root = args.gold
# Put the data root if specified in the regular expressions to be replaced
if args.data != None:
args.replace.append("__DATA__=%s" % args.data)
# If specified, set up the paths based on specified torrent server code root
if args.ts_base != None:
args.environment = (
args.environment
+ " PATH=%s/build/Analysis:%s/build/Analysis/TMAP:%s/pipeline/bin:%s ION_CONFIG=%s/Analysis/config"
% (args.ts_base, args.ts_base, args.ts_base, os.getenv("PATH"), args.ts_base)
)
if args_ok[0] == False:
sys.exit(args_ok[1])
log_file = args.output + "/log.txt"
json_results = args.output + "/results.json"
mkdir_p(args.output)
# Main logger at the debug level
logging.basicConfig(
filename=log_file, filemode="w", level=logging.DEBUG, format="%(asctime)s\t%(levelname)s\t%(message)s"
)
# Add a logger for the console at the info level
console_logger = logging.StreamHandler(sys.stdout)
console_logger.setLevel(logging.INFO)
formatter = logging.Formatter("%(message)s")
console_logger.setFormatter(formatter)
logging.getLogger("").addHandler(console_logger)
if args.failures != None:
fail_log = open(args.failures, "w")
# Read our json
logging.info("Reading json file: " + args.jobs)
jobs_file = open(args.jobs)
jobs = json.loads(jobs_file.read())
jobs_file.close()
logging.info("Got %d jobs from file" % len(jobs))
report_file = os.path.join(args.output, "report.txt")
runner = JobRunner(args, report_file)
job_ids = jobs.keys()
job_values = jobs.values()
# Create our graph for dependency checking
gr = digraph()
# Add all of the nodes
for i in range(len(job_ids)):
logging.debug("Adding: %s" % job_values[i]["job_name"])
if gr.has_node(job_values[i]["job_name"]):
logging.info("Already here?? %s" % job_values[i]["job_name"])
gr.add_node(job_values[i]["job_name"])
# Add all the edges
for i in range(len(job_ids)):
logging.debug("Adding edges for: %s" % job_values[i]["job_name"])
if "dependencies" in job_values[i]:
for j in range(len(job_values[i]["dependencies"])):
logging.debug("Adding edge: %s %s" % (job_values[i]["dependencies"][j], job_values[i]["job_name"]))
gr.add_edge((job_values[i]["dependencies"][j], job_values[i]["job_name"]))
order = topological_sorting(gr)
logging.debug("Order is: %s" % order)
selector = re.compile(args.type)
neg_selector = re.compile(args.not_type)
jobnames = None
if args.jobnames != None:
jobnames = re.compile(args.jobnames)
dorun = True
start_time = time.time()
for i in range(len(order)):
# See if we're running this class of jobs
dorun = args.type == "all"
job = jobs[order[i]]
if dorun == False:
for tag_ix in range(len(job["tags"])):
if selector.match(job["tags"][tag_ix]):
dorun = True
break
for tag_ix in range(len(job["tags"])):
if neg_selector.match(job["tags"][tag_ix]):
dorun = False
break
# Check to see if we're running a particular job which trumps tags
if jobnames != None and jobnames.match(job["job_name"]):
dorun = True
elif jobnames != None:
dorun = False
# Run our job if necessary
if dorun:
logging.info("Running job %d %s" % (i, jobs[order[i]]["job_name"]))
runner.run_job(job["job_name"], job)
if job["ret_val"] == 0:
logging.info("Passed in %.2f seconds" % job["elapsed_time"])
else:
logging.info("Failed in %.2f seconds" % job["elapsed_time"])
if args.failures != None:
fail_log.write("Job %s failed\n" % job["job_name"])
fail_log.write("stdout file: %s\n" % job["stdout"])
sys.stdout.write("stdout file: %s\n" % job["stdout"])
out = open(job["stdout"])
print_head_tail(out, fail_log, alt_out=sys.stdout)
out.close()
fail_log.write("stderr file: %s\n" % job["stderr"])
sys.stdout.write("stderr file: %s\n" % job["stderr"])
out = open(job["stderr"])
print_head_tail(out, fail_log, alt_out=sys.stdout)
out.close()
else:
logging.info("Skipping job %d %s" % (i, job["job_name"]))
elapsed_time = time.time() - start_time
logging.info("All tests took: %.2f seconds" % elapsed_time)
if args.failures != None:
fail_log.write("Ran %d jobs, %d passed and %d failed\n" % (runner.run, runner.passed, runner.failed))
fail_log.close()
json_out = open(json_results, mode="w")
json_out.write(json.dumps(jobs, sort_keys=True, indent=4, separators=(",", ": ")))
logging.info("Ran %d jobs, %d passed and %d failed" % (runner.run, runner.passed, runner.failed))
logging.info("Full run took: %.3f seconds" % (elapsed_time))
# Look to see if we need to update the gold data
if args.update_file != None and args.update_root != None:
ufile = open(os.path.join(args.output, args.update_file))
ustats = json.loads(ufile.read())
if float(ustats["quantile_min"]) > float(args.update_decision) and runner.failed == 0:
gold_backup = "%s_%s" % (args.update_root, datetime.datetime.now().isoformat())
print "Moving %s to %s and %s to new gold" % (args.update_root, gold_backup, args.output)
os.rename(args.update_root, gold_backup)
os.symlink(args.output, args.update_root)
else:
print "Skipping update as metric is: %.2f and threshold is %.2f or numfailed > 0 (%d)" % (
ustats["quantile_min"],
float(args.update_decision),
runner.failed,
)
else:
print "Skipping update as file not specified"
logging.info("Done.")
if runner.failed > 0:
print "Failed."
sys.exit(1)
else:
print "Success."
"zonesystem",
"rawspeed",
]
)
add_edges(gr)
# make sure we don't have cycles:
cycle_list = find_cycle(gr)
if cycle_list:
print "cycles:"
print cycle_list
exit(1)
# get us some sort order!
sorted_nodes = topological_sorting(gr)
length = len(sorted_nodes)
priority = 1000
for n in sorted_nodes:
# now that should be the priority in the c file:
print "%d %s" % (priority, n)
filename = "../src/iop/%s.c" % n
if not os.path.isfile(filename):
filename = "../src/iop/%s.cc" % n
if not os.path.isfile(filename):
if not n == "rawspeed":
print "could not find file `%s', maybe you're not running inside tools/?" % filename
continue
replace_all(
filename,
"( )*?(module->priority)( )*?(=).*?(;).*\n",
def get_bootschedule(self):
""" Calculate a Single Appearance Schedule to boot peeking tokens.
"""
sdfgraph = self.sdfgraph
edges = sdfgraph.edges()
peek_tokens = sdfgraph.peek_tokens
strong_connected = self.strong_connected
repetition = sdfgraph.repetition
boot_schedule = []
if self.boot_schedule != []:
return self.boot_schedule
PEEK_FOUND = False
for edge in edges:
if len(peek_tokens[edge]) > 0:
PEEK_FOUND = True
break
if not PEEK_FOUND:
return []
# calculate repetitions of each to fill up peeks only
rep={}
nodes=topological_sorting(sdfgraph.graph)
nodes.reverse() # reversed
for node in nodes:
if node in rep: # the node is backedge
continue
rep[node] = 0
backedge = [];
for succ in sdfgraph.out_nodes(node):
edge = (node, succ)
if succ not in rep: # the node is backedge
backedge.append(succ) # process later
continue
rep[node]=max(rep[node],int(ceil((rep[succ]*sdfgraph.consumption[edge]+len(peek_tokens[edge]))/float(sdfgraph.production[edge]))))
for u in backedge:
rep[u]=rep[node]
# calculate max level difference
for succ in sdfgraph.out_nodes(node):
edge = (node,succ)
diff = fabs(int(ceil(float(rep[node])/repetition[node]))-int(ceil(float(rep[succ])/repetition[succ])))
if self.max_lev_diff < diff:
self.max_lev_diff = diff
nodes.reverse() # ordered
# initialize number of actor invocations required
schedule_len = 0
for node in nodes:
schedule_len = schedule_len + rep[node]
# initialize fillstate
fillstate = {}
for edge in sdfgraph.edges():
fillstate[edge]=sdfgraph.delay[edge]
while schedule_len > 0:
init_sched_len = schedule_len
for node in nodes:
if rep[node] > 0:
# check whether node is fireable, i.e. enough tokens on the input channel
firable = True
for in_node in sdfgraph.in_nodes(node):
edge=(in_node,node)
pop_rate = sdfgraph.consumption[edge]
peek_rate = len(sdfgraph.peek_tokens[edge])
if fillstate[edge]-(pop_rate+peek_rate) < 0:
firable = False
# If so, add to bootschedule and update fillstate
if firable:
for in_node in sdfgraph.in_nodes(node):
edge=(in_node,node)
fillstate[edge]=fillstate[edge]-sdfgraph.consumption[edge]
for out_node in sdfgraph.out_nodes(node):
edge=(node,out_node)
fillstate[edge]=fillstate[edge]+sdfgraph.production[edge]
rep[node] = rep[node] - 1
boot_schedule.append(node)
schedule_len = schedule_len - 1
if init_sched_len == schedule_len:
print 'ERROR: Boot schedule calculation failed.'
sys.exit(1)
self.boot_schedule = boot_schedule
# test automatically if both of schedules are computed
#self.check_schedules()
return self.boot_schedule
def shortest_greedy_generator(tasks_graph,platform_graph,SE_graph,ALU_graph):
"""
Generator using a randomized greedy algorithm
TODO: clean the code!!!
"""
task_nb = len(tasks_graph.nodes())
topological_order = topological_sorting(tasks_graph)
roots_list = roots_filter(tasks_graph)
routed_platform_graph = copy.deepcopy(platform_graph)
routed_SE_graph = copy.deepcopy(SE_graph)
routed_ALU_graph = copy.deepcopy(ALU_graph)
root_subgraphs = rootgraphs(topological_order,roots_list)
random.shuffle(root_subgraphs)
random_topological_order = [item for sublist in root_subgraphs for item in sublist] # flattening the list
edges_list_sorted = []
for node in random_topological_order:
for edge in tasks_graph.edges():
if edge[0] == node:
edges_list_sorted.append(edge)
mapping = [0 for i in range(task_nb)]
placed = [False for i in range(len(random_topological_order))]
used_nodes = []
for (source_node, target_node) in edges_list_sorted:
if placed[source_node] == False and placed[target_node] == False:
if source_node in roots_list:
# DEPTH = 2
local_depth = DEPTH+1
available_start_nodes = []
while available_start_nodes == []:
for node in routed_platform_graph.nodes():
if 'ALU' in node and node not in used_nodes and int(node.split('_')[1]) < local_depth:
available_start_nodes.append(node)
local_depth += 1
chosen_source_node = available_start_nodes[random.randint(0,len(available_start_nodes)-1)]
used_nodes.append(chosen_source_node)
mapping[source_node] = (int(re.sub('\D', '', chosen_source_node.split('_')[0])),int(chosen_source_node.split('_')[1]))
placed[source_node] = True
# else:
# source_node_location = 'ALU' + mapping[source_node][0] + '_' + mapping[source_node][1]
closest_nodes = shortest_path(routed_platform_graph,chosen_source_node)[1]
shortest_path_list = list(filter(lambda x: 'ALU' in x[0], sorted(closest_nodes.items(), key=lambda x: (x[1],x[0]))))[1:]
# DEPTH = 2
local_depth = DEPTH
chosable_nodes = []
while chosable_nodes == []:
for node, dist in shortest_path_list:
if node not in used_nodes and dist <= local_depth:
chosable_nodes.append(node)
local_depth += 1
chosen_target_node = chosable_nodes[random.randint(0,len(chosable_nodes)-1)]
used_nodes.append(chosen_target_node)
mapping[target_node] = (int(re.sub('\D', '', chosen_target_node.split('_')[0])),int(chosen_target_node.split('_')[1])) # coordinates
placed[target_node] = True
# routed_platform_graph.del_node(chosen_source_node)
# routed_platform_graph.del_node(chosen_target_node)
elif placed[source_node] == True and placed[target_node] == False:
source_node_location = 'ALU' + str(mapping[source_node][0]) + '_' + str(mapping[source_node][1])
closest_nodes = shortest_path(routed_platform_graph,source_node_location)[1]
shortest_path_list = list(filter(lambda x: 'ALU' in x[0], sorted(closest_nodes.items(), key=lambda x: (x[1],x[0]))))[1:]
# DEPTH = 2
local_depth = DEPTH
chosable_nodes = []
while chosable_nodes == []:
for node, dist in shortest_path_list:
if node not in used_nodes and dist <= local_depth:
chosable_nodes.append(node)
local_depth += 1
chosen_target_node = chosable_nodes[random.randint(0,len(chosable_nodes)-1)]
used_nodes.append(chosen_target_node)
mapping[target_node] = (int(re.sub('\D', '', chosen_target_node.split('_')[0])),int(chosen_target_node.split('_')[1])) # coordinates
# routed_platform_graph.del_node()
placed[target_node] = True
elif placed[source_node] == False and placed[target_node] == True:
(target_x, target_y) = (mapping[target_node][0], mapping[target_node][1])
local_depth = DEPTH
chosable_nodes = []
while chosable_nodes == []:
chosable_nodes_coord = []
delta = list(range(-local_depth,local_depth+1))
for i in delta:
for j in delta:
(new_x, new_y) = (target_x+i, target_y+j)
if 0 <= new_x <= PE_col-1 and 0 <= new_y <= PE_row-1:
chosable_nodes_coord.append((new_x,new_y))
chosable_nodes_coord.remove((target_x,target_y))
for x,y in chosable_nodes_coord:
node_location = 'ALU' + str(x) + '_' + str(y)
closest_nodes = shortest_path(routed_platform_graph,node_location)[1]
filtered_closest_nodes = list(filter(lambda x: 'ALU' in x[0] and x[1] <= local_depth, sorted(closest_nodes.items(), key=lambda x: (x[1],x[0]))))[1:]
for node, _ in filtered_closest_nodes:
if node not in used_nodes and node not in chosable_nodes:
chosable_nodes.append(node)
local_depth += 1
chosen_source_node = chosable_nodes[random.randint(0,len(chosable_nodes)-1)]
used_nodes.append(chosen_source_node)
mapping[source_node] = (int(re.sub('\D', '', chosen_target_node.split('_')[0])),int(chosen_target_node.split('_')[1]))
placed[source_node] = True
# shift_mapping(mapping)
# if len(list(set(mapping))) != len(tasks):
# print('WTF')
return mapping
def astar_tasks_routing(tasks_graph,platform_graph,SE_graph,individual):
"""
Routing using a shortest path algorithm (A*) and some BFS
"""
# roots_list, sinks_list = rootsinks_filter(tasks_graph)
topological_order = topological_sorting(tasks_graph)
edges_list_sorted = []
for node in topological_order:
for edge in tasks_graph.edges():
if edge[0] == node:
edges_list_sorted.append(edge)
weighted_tasks_graph = copy.deepcopy(tasks_graph)
routed_platform_graph = copy.deepcopy(platform_graph)
routed_SE_graph = copy.deepcopy(SE_graph)
# SE_search_graph = euclidean() # chow not working!?
# platform_search_graph = euclidean()
# # Optimizing all the original graphs
# SE_search_graph.optimize(routed_SE_graph)
# platform_search_graph.optimize(routed_platform_graph)
# search_graph.optimize(routed_platform_graph)
routable = True
paths_list = []
for edge in edges_list_sorted:
source_node = 'ALU' + str(individual[edge[0]][0]) + '_' + str(individual[edge[0]][1])
target_node = 'ALU' + str(individual[edge[1]][0]) + '_' + str(individual[edge[1]][1])
# TODO: add stupid constraint for SE not routing west if coming from same ALU...
if (source_node,target_node) not in routed_platform_graph.edges():
routable_SE_graph = clean_unroutable_nodes(routed_SE_graph,source_node,target_node)
# search_graph.optimize(routable_SE_graph)
try:
route_path = heuristic_search(routable_SE_graph,source_node,target_node,SE_search_graph)
paths_list.append(route_path)
except:
routable = False
weight = 10000 # high weight penalty to not consider this individual
# TODO: some repair mechanism?
weighted_tasks_graph.set_edge_weight(edge,weight)
return weighted_tasks_graph, edges_list_sorted, paths_list
else:
# search_graph.optimize(routed_platform_graph)
try:
route_path = heuristic_search(routed_platform_graph,source_node,target_node,platform_search_graph)
# routed_platform_graph.del_edge((source_node,target_node))
paths_list.append(route_path)
except:
routable = False
weight = 10000 # high weight penalty to not consider this individual
# TODO: some repair mechanism?
weighted_tasks_graph.set_edge_weight(edge,weight)
return weighted_tasks_graph, edges_list_sorted, paths_list
if routable:
for i in range(len(route_path)-1):
if 'ALU' in route_path[i] and 'ALU' in route_path[i+1]:
routed_platform_graph.del_edge((route_path[i],route_path[i+1]))
else:
routed_platform_graph.del_edge((route_path[i],route_path[i+1]))
routed_SE_graph.del_edge((route_path[i],route_path[i+1]))
weight = path_evaluation(platform_graph,route_path)
weighted_tasks_graph.set_edge_weight(edge,weight)
else:
pass
# weight = 10000 # high weight penalty to not consider this individual
# # TODO: some repair mechanism?
# weighted_tasks_graph.set_edge_weight(edge,weight)
return weighted_tasks_graph, edges_list_sorted, paths_list
def ccm_fast_export(releases, graphs):
global acn_ancestors
global users
users = users()
logger.basicConfig(filename='ccm_fast_export.log',level=logger.DEBUG)
commit_lookup = {}
# Get the initial release
for k, v in releases.iteritems():
if k == 'delimiter':
continue
if k == 'ccm_types':
continue
if v['previous'] is None:
release = k
break
logger.info("Starting at %s as initial release" % release)
if 'created' not in releases[release]:
initial_release_time = 0.0 # epoch for now since releases[release] has no 'created' key :(
else:
initial_release_time = time.mktime(releases[release]['created'].timetuple())
mark = 0
files = []
# Create the initial release
# get all the file objects:
file_objects = [ccm_cache.get_object(o) for o in releases[release]['objects']]
project_obj = ccm_cache.get_object(releases[release]['fourpartname'])
paths = project_obj.get_members()
for o in file_objects:
if o.get_type() != 'dir':
object_mark, mark = create_blob(o, mark)
for p in paths[o.get_object_name()]:
files.append('M ' + releases['ccm_types']['permissions'][o.get_type()] + ' :'+str(object_mark) + ' ' + p)
empty_dirs = releases[release]['empty_dirs']
logger.info("Empty dirs for release %s\n%s" %(release, empty_dirs))
mark = create_blob_for_empty_dir(get_mark(mark))
#file_list = create_file_list(objects, object_lookup, releases['ccm_types'], empty_dirs=empty_dirs, empty_dir_mark=mark)
if empty_dirs:
for d in empty_dirs:
if mark:
path = d + '/.gitignore'
files.append('M 100644 :' + str(mark) + ' ' + path)
mark = get_mark(mark)
commit_info = ['reset refs/tags/' + release, 'commit refs/tags/' + release, 'mark :' + str(mark),
'author Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000",
'committer Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000", 'data 15',
'Initial commit', '\n'.join(files), '']
print '\n'.join(commit_info)
logger.info("git-fast-import:\n%s" %('\n'.join(commit_info)))
tag_msg = 'Release: %s' %release
annotated_tag = ['tag %s' % release,
'from :%s' % str(mark),
'tagger Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000",
'data %s' % len(tag_msg),
tag_msg]
print '\n'.join(annotated_tag)
commit_lookup[release] = mark
# do the following releases (graphs)
release_queue = deque(releases[release]['next'])
while release_queue:
release = release_queue.popleft()
previous_release = releases[release]['previous']
logger.info("Next release: %s" % release)
commit_graph = graphs[release]['commit']
commit_graph = fix_orphan_nodes(commit_graph, previous_release)
commit_graph = ch.spaghettify_digraph(commit_graph, previous_release, release)
#htg.commit_graph_to_image(commit_graph, releases[release], graphs[release]['task'], name=releases[release]['name']+'_after' )
# Find the cutting nodes
logger.info("Finding the cutting nodes")
undirected = graph()
undirected.add_nodes(commit_graph.nodes())
[undirected.add_edge(edge) for edge in commit_graph.edges()]
cutting_nodes = cut_nodes(undirected)
del undirected
# Create the reverse commit graph
logger.info("Building the reverse commit graph")
reverse_commit_graph = commit_graph.reverse()
# Compute the accessibility matrix of the reverse commit graph
logger.info("Compute the ancestors")
ancestors = accessibility(reverse_commit_graph)
del reverse_commit_graph
logger.info("Ancestors of the release: %s" % str(ancestors[release]))
# Clean up the ancestors matrix
for k, v in ancestors.iteritems():
if k in v:
v.remove(k)
# Get the commits order
commits = topological_sorting(commit_graph)
# Fix the commits order list
commits.remove(previous_release)
commits.remove(release)
last_cutting_node = None
# Check if the release (Synergy project has changed name, if it has the
# 'base' directory name needs to be renamed
if releases.has_key('delimiter'):
delim = releases['delimiter']
else:
delim = '-'
previous_name = previous_release.split(delim)[0]
current_name = release.split(delim)[0]
if current_name != previous_name:
logger.info("Name changed: %s -> %s" %(previous_name, current_name))
from_mark = commit_lookup[previous_release]
mark, commit = rename_toplevel_dir(previous_name, current_name, release, releases, mark, from_mark)
print '\n'.join(commit)
# adjust the commit lookup
commit_lookup[previous_release] = mark
for counter, commit in enumerate(commits):
logger.info("Commit %i/%i" % (counter+1, len(commits)))
acn_ancestors = []
if last_cutting_node is not None:
acn_ancestors = ancestors[last_cutting_node]
# Create the references lists. It lists the parents of the commit
#reference = [commit_lookup[parent] for parent in ancestors[commit] if parent not in acn_ancestors]
reference = [commit_lookup[parent] for parent in commit_graph.incidents(commit)]
if len(reference) > 1:
# Merge commit
mark = create_merge_commit(commit, release, releases, mark, reference, graphs, set(ancestors[commit]) - set(acn_ancestors))
else:
# Normal commit
mark = create_commit(commit, release, releases, mark, reference, graphs)
# Update the lookup table
commit_lookup[commit] = mark
# Update the last cutting edge if necessary
if commit in cutting_nodes:
last_cutting_node = commit
if last_cutting_node is not None:
acn_ancestors = ancestors[last_cutting_node]
reference = [commit_lookup[parent] for parent in ancestors[release] if parent not in acn_ancestors]
logger.info("Reference %s" %str([parent for parent in ancestors[release] if parent not in acn_ancestors]))
if not reference:
logger.info("Reference previous %s, mark: %d" % (releases[release]['previous'], commit_lookup[releases[release]['previous']]))
reference = [commit_lookup[ releases[release]['previous'] ] ]
mark, merge_commit = create_release_merge_commit(releases, release, get_mark(mark), reference, graphs, set(ancestors[release]) - set(acn_ancestors))
print '\n'.join(merge_commit)
annotated_tag = create_annotated_tag(releases, release, mark)
print '\n'.join(annotated_tag)
commit_lookup[release] = mark
release_queue.extend(releases[release]['next'])
#release = releases[release]['next']
#release = None
#reset to master
master = get_master_tag()
reset = ['reset refs/heads/master', 'from :' + str(commit_lookup[master])]
logger.info("git-fast-import:\n%s" %('\n'.join(reset)))
print '\n'.join(reset)
def ccm_fast_export(releases, graphs):
global acn_ancestors
global users
users = users()
logger.basicConfig(filename='ccm_fast_export.log',level=logger.DEBUG)
commit_lookup = {}
# Get the initial release
for k, v in releases.iteritems():
if k == 'delimiter':
continue
if k == 'ccm_types':
continue
if v['previous'] is None:
release = k
break
logger.info("Starting at %s as initial release" % release)
if 'created' not in releases[release]:
initial_release_time = 0.0 # epoch for now since releases[release] has no 'created' key :(
else:
initial_release_time = time.mktime(releases[release]['created'].timetuple())
mark = 0
files = []
# Create the initial release
# get all the file objects:
file_objects = (ccm_cache.get_object(o) for o in releases[release]['objects'])
project_obj = ccm_cache.get_object(releases[release]['fourpartname'])
paths = project_obj.get_members()
for o in file_objects:
if o.get_type() != 'dir':
object_mark, mark = create_blob(o, mark)
for p in paths[o.get_object_name()]:
files.append('M ' + releases['ccm_types']['permissions'][o.get_type()] + ' :'+str(object_mark) + ' ' + p)
empty_dirs = releases[release]['empty_dirs']
logger.info("Empty dirs for release %s\n%s" %(release, empty_dirs))
mark = create_blob_for_empty_dir(get_mark(mark))
#file_list = create_file_list(objects, object_lookup, releases['ccm_types'], empty_dirs=empty_dirs, empty_dir_mark=mark)
if empty_dirs:
for d in empty_dirs:
if mark:
path = d + '/.gitignore'
files.append('M 100644 :' + str(mark) + ' ' + path)
mark = get_mark(mark)
commit_info = ['reset refs/tags/' + release, 'commit refs/tags/' + release, 'mark :' + str(mark),
'author Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000",
'committer Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000", 'data 15',
'Initial commit', '\n'.join(files), '']
print '\n'.join(commit_info)
logger.info("git-fast-import:\n%s" %('\n'.join(commit_info)))
tag_msg = 'Release: %s' %release
annotated_tag = ['tag %s' % release,
'from :%s' % str(mark),
'tagger Nokia <*****@*****.**> ' + str(int(initial_release_time)) + " +0000",
'data %s' % len(tag_msg),
tag_msg]
print '\n'.join(annotated_tag)
commit_lookup[release] = mark
# do the following releases (graphs)
release_queue = deque(releases[release]['next'])
while release_queue:
release = release_queue.popleft()
previous_release = releases[release]['previous']
logger.info("Next release: %s" % release)
commit_graph = graphs[release]['commit']
commit_graph = fix_orphan_nodes(commit_graph, previous_release)
commit_graph = ch.spaghettify_digraph(commit_graph, previous_release, release)
#htg.commit_graph_to_image(commit_graph, releases[release], graphs[release]['task'], name=releases[release]['name']+'_after' )
# Find the cutting nodes
logger.info("Finding the cutting nodes")
undirected = graph()
undirected.add_nodes(commit_graph.nodes())
[undirected.add_edge(edge) for edge in commit_graph.edges()]
cutting_nodes = cut_nodes(undirected)
del undirected
# Create the reverse commit graph
logger.info("Building the reverse commit graph")
reverse_commit_graph = commit_graph.reverse()
# Compute the accessibility matrix of the reverse commit graph
logger.info("Compute the ancestors")
ancestors = accessibility(reverse_commit_graph)
del reverse_commit_graph
logger.info("Ancestors of the release: %s" % str(ancestors[release]))
# Clean up the ancestors matrix
for k, v in ancestors.iteritems():
if k in v:
v.remove(k)
# Get the commits order
commits = topological_sorting(commit_graph)
# Fix the commits order list
commits.remove(previous_release)
commits.remove(release)
last_cutting_node = None
# Check if the release (Synergy project has changed name, if it has the
# 'base' directory name needs to be renamed
if releases.has_key('delimiter'):
delim = releases['delimiter']
else:
delim = '-'
previous_name = previous_release.split(delim)[0]
current_name = release.split(delim)[0]
if current_name != previous_name:
logger.info("Name changed: %s -> %s" %(previous_name, current_name))
from_mark = commit_lookup[previous_release]
mark, commit = rename_toplevel_dir(previous_name, current_name, release, releases, mark, from_mark)
print '\n'.join(commit)
# adjust the commit lookup
commit_lookup[previous_release] = mark
for counter, commit in enumerate(commits):
logger.info("Commit %i/%i" % (counter+1, len(commits)))
acn_ancestors = []
if last_cutting_node is not None:
acn_ancestors = ancestors[last_cutting_node]
# Create the references lists. It lists the parents of the commit
#reference = [commit_lookup[parent] for parent in ancestors[commit] if parent not in acn_ancestors]
reference = [commit_lookup[parent] for parent in commit_graph.incidents(commit)]
if len(reference) > 1:
# Merge commit
mark = create_merge_commit(commit, release, releases, mark, reference, graphs, set(ancestors[commit]) - set(acn_ancestors))
else:
# Normal commit
mark = create_commit(commit, release, releases, mark, reference, graphs)
# Update the lookup table
commit_lookup[commit] = mark
# Update the last cutting edge if necessary
if commit in cutting_nodes:
last_cutting_node = commit
if last_cutting_node is not None:
acn_ancestors = ancestors[last_cutting_node]
reference = [commit_lookup[parent] for parent in ancestors[release] if parent not in acn_ancestors]
logger.info("Reference %s" %str([parent for parent in ancestors[release] if parent not in acn_ancestors]))
if not reference:
logger.info("Reference previous %s, mark: %d" % (releases[release]['previous'], commit_lookup[releases[release]['previous']]))
reference = [commit_lookup[ releases[release]['previous'] ] ]
mark, merge_commit = create_release_merge_commit(releases, release, get_mark(mark), reference, graphs, set(ancestors[release]) - set(acn_ancestors))
print '\n'.join(merge_commit)
annotated_tag = create_annotated_tag(releases, release, mark)
print '\n'.join(annotated_tag)
commit_lookup[release] = mark
release_queue.extend(releases[release]['next'])
#release = releases[release]['next']
#release = None
#reset to master
master = get_master_tag()
reset = ['reset refs/heads/master', 'from :' + str(commit_lookup[master])]
logger.info("git-fast-import:\n%s" %('\n'.join(reset)))
print '\n'.join(reset)
def critical_path(graph):
"""
Compute and return the critical path in an acyclic directed weighted LoopGraph.
@attention: This function is only meaningful for directed weighted acyclic graphs
@type graph: digraph
@param graph: Digraph
@rtype: List
@return: List containing all the nodes in the path (or an empty array if the LoopGraph
contains a cycle)
"""
#if the LoopGraph contains a cycle we return an empty array
if not len(find_cycle(graph)) == 0:
return []
#this empty dictionary will contain a tuple for every single node
#the tuple contains the information about the most costly predecessor
#of the given node and the cost of the path to this node
#(predecessor, cost)
node_tuples = {}
topological_nodes = topological_sorting(graph)
#all the tuples must be set to a default value for every node in the LoopGraph
for node in topological_nodes:
node_tuples.update( {node :(None, 0)} )
#run trough all the nodes in a topological order
for node in topological_nodes:
predecessors =[]
#we must check all the predecessors
for pre in graph.incidents(node):
max_pre = node_tuples[pre][1]
predecessors.append( (pre, graph.edge_weight( (pre, node) ) + max_pre ) )
max = 0; max_tuple = (None, 0)
for i in predecessors:#look for the most costly predecessor
if i[1] >= max:
max = i[1]
max_tuple = i
#assign the maximum value to the given node in the node_tuples dictionary
node_tuples[node] = max_tuple
#find the critical node
max = 0; critical_node = None
for k,v in list(node_tuples.items()):
if v[1] >= max:
max= v[1]
critical_node = k
path = []
#find the critical path with backtracking trought the dictionary
def mid_critical_path(end):
if node_tuples[end][0] != None:
path.append(end)
mid_critical_path(node_tuples[end][0])
else:
path.append(end)
#call the recursive function
mid_critical_path(critical_node)
path.reverse()
return path #return the array containing the critical path
'watermark',
'zonesystem',
'rawspeed'
])
add_edges(gr)
# make sure we don't have cycles:
cycle_list = find_cycle(gr)
if cycle_list:
print "cycles:"
print cycle_list
exit(1)
# get us some sort order!
sorted_nodes = topological_sorting(gr)
length = len(sorted_nodes)
priority = 1000
for n in sorted_nodes:
# now that should be the priority in the c file:
print "%d %s" % (priority, n)
filename = "../src/iop/%s.c" % n
if not os.path.isfile(filename):
filename = "../src/iop/%s.cc" % n
if not os.path.isfile(filename):
if not n == "rawspeed":
print "could not find file `%s', maybe you're not running inside tools/?" % filename
continue
replace_all(
filename, "( )*?(module->priority)( )*?(=).*?(;).*\n",
" module->priority = %d; // module order created by iop_dependencies.py, do not edit!\n"
def nodes(self):
""" Return list of nodes in alphabetical order
"""
return topological_sorting(self.graph)
def getSortedNodeListIndex(self):
''' returns a list of the node names topologically sorted
:return: a list of the node names topologically sorted
'''
return topological_sorting(self.graph)
