def partition_raw(self, machine, vertices, no_machine_time_steps,
machine_time_step_us, placer):
'''
entry method for partitioning
'''
placements = list()
subvertices = list()
#sort out vertex's by constraints
sort = lambda vertex: vertex.constraints.placement_cardinality
vertices = sorted(vertices, key=sort, reverse=True)
n_atoms = 0
for vertex in vertices:
n_atoms += vertex.atoms
self.progress = ProgressBar(n_atoms)
self.partitioned = dict()
# Partition one vertex at a time
for vertex in vertices:
#only partition real vertexes. virtual ones are ignored
if vertex.is_virtual():
self.partition_virtual_vertexes(vertex, placements,
subvertices, placer)
elif len(vertex.subvertices) == 0:
self.partition_vertexes(vertex, no_machine_time_steps,
machine_time_step_us, subvertices,
placer, placements)
else:
print "Vertex {} is already partitioned!".format(vertex.label)
self.progress.end()
subedges = self.generate_sub_edges(subvertices)
return subvertices, subedges, placements
def filterSubEdges(self, dao):
"""Go through the newly created list of sub-edges and call a model
specific function on each one, this allows the application to prune
sub-edges that are not really needed.
"""
logger.info("* Running pre-routing sub-edge pruning *")
new_subedges = list()
progress = ProgressBar(len(dao.subedges))
for subedge in dao.subedges:
if subedge.edge.filterSubEdge(subedge):
subedge.pruneable = True
else:
new_subedges.append(subedge)
progress.update()
dao.subedges = new_subedges
progress.end()
def filterSubEdges(self, dao):
"""Go through the newly created list of sub-edges and call a model
specific function on each one, this allows the application to prune
sub-edges that are not really needed.
"""
logger.info("* Running pre-routing sub-edge pruning *")
new_subedges = list()
progress = ProgressBar(len(dao.subedges))
for subedge in dao.subedges:
if subedge.edge.filterSubEdge(subedge):
subedge.pruneable = True
else:
new_subedges.append(subedge)
progress.update()
dao.subedges = new_subedges
progress.end()
def route_raw(machine,
sub_vertexes,
k=1,
l=0,
m=0,
BW_PER_ROUTE_ENTRY=0.01,
MAX_BW=250,
dao=None):
"""
Modified by peter.choy 06.08.13
Generates a list of routings for subedges in a machine...
For the purposes of this algorithm, we are viewing the chips as 'nodes' on a graph.
The weight of the arcs is a function of the number of routing table entries,
the arc length, and the remaining bandwidth along the arc's corresponding
connection.
The inner function of the algorithm is a slightly modified Dijkstra's algorithm.
The outer loop loops over the necessary sub edges, and re-iterates over each
sub edge several times. Repeating Dijkstra's algorithm for each sub edge is greedy,
and so the iterative process may or may not be necessary.
:param `pacman103.lib.lib_machine.Machine` machine:
machine from which to allocate processors.
:param list subedges:
list of :py:class:`pacman103.lib.graph.Subedge` instances to
route across the machine.
:param k:
constant that is added to arc weights to represent path length
:param l:
constant controlling contribution of Q to total arc weights
:param m:
constant controlling contribution of T to total arc weights
:returns:
list of :py:class:`pacman103.lib.lib_map.Routing` instances.
"""
#print("")
#print("Starting stopwatch...")
#start_time = clock()
#print("")
#print("Initialising routing data structures...")
routings = []
nodes_info = DijkstraRouting.initiate_node_info(machine, MAX_BW)
dijkstra_tables = DijkstraRouting.initiate_dijkstra_tables(machine)
#print("")
#print("Starting routing...")
#print("")
DijkstraRouting.update_weights(nodes_info, machine, l, m, k, MAX_BW)
run_num = 0
pruned_sub_edges = 0
edge_considered = 0
#each subsertex represents a core in the board
progress = ProgressBar(len(sub_vertexes))
for subVertex in sub_vertexes:
subedges = subVertex.out_subedges
#locate all destination and soruce coords
dest_processors = []
subedges_to_route = list()
xs, ys, ps = subVertex.placement.processor.get_coordinates()
for subedge in subedges:
if not subedge.pruneable:
dest_processors.append(
subedge.postsubvertex.placement.processor)
subedges_to_route.append(subedge)
else:
pruned_sub_edges += 1
if (len(dest_processors) != 0):
# Update the weights according to the changes in routing
# tables and available bandwidth
#logger.debug("updating weights")
DijkstraRouting.update_weights(nodes_info, machine, l, m, k,
MAX_BW)
# Reset the temporary storage of lowest cost
DijkstraRouting.reset_tables(dijkstra_tables)
# SD Commented this out 20/1/14, to avoid extraneous printing
# AS intergrated this into logger.debug
#logger.debug("***************************************************")
#logger.debug("Source node is ({}, {})".format(xs, ys))
#logger.debug("Destination nodes are(")
#for processor in dest_processors:
# xd, yd, pd = processor.get_coordinates()
# logger.debug("({}, {}, {})".format(xd, yd, pd))
#logger.debug("")
# Set the first activated node in this run as the source
xa, ya = xs, ys
dijkstra_tables["{}:{}".format(xa, ya)]["activated?"] = True
dijkstra_tables["{}:{}".format(xa, ya)]["lowest cost"] = 0
# The cost at the source node is zero for obvious reasons.
# Note that it is NOT 'None'
DijkstraRouting.\
properate_costs_till_reached_destinations(dijkstra_tables,
nodes_info,
xa, ya,
dest_processors,
xs, ys)
#logger.debug("Reached destination from source, retracing.")
#helpful output data
if conf.config.getboolean( "Routing", "generate_graphs" ) and \
dao is not None: # AM
output_folder = dao.get_reports_directory("routing_graphs")
router_utility = util.RouterUtil(
new_output_folder=output_folder)
router_utility.output_routing_weight(
router_utility,
dijkstra_tables,
machine,
graph_label="routing weights",
routing_file_name="routingWeights" +
str(edge_considered))
edge_considered += 1
for subedge in subedges_to_route:
key, mask, = subedge.key, subedge.mask
key_mask_combo = subedge.key_mask_combo
dest = subedge.postsubvertex
xd, yd, pd = dest.placement.processor.get_coordinates()
routing = lib_map.Routing(subedge)
xt, yt = DijkstraRouting.retrace_back_to_source(
xd, yd, machine, nodes_info, dijkstra_tables, key,
mask, key_mask_combo, routing, pd, BW_PER_ROUTE_ENTRY)
# SD Commented out 20/1/14 to remove extraneous printing.
# DijkstraRouting.printRoute(xt, yt, xs, ys, dijkstra_tables, routing)
subedge.routing = routing
routings.append(routing)
run_num += 1
# SD Commented this out 20/1/14, to avoid extraneous printing
# AS modified to debug format
#print ""
#logger.debug("Route number {} completed from ({}, {}) "
# "to ({}, {})".format(run_num, xs, ys, xd, yd))
#logger.debug("route took {}".format(DijkstraRouting.printRoute(xd, yd, xs, ys, dijkstra_tables, routing)))
#logger.debug("*********************************"
# "**********************************")
#if (run_num % 5000) == 0:
# logger.debug("{} routes done, please wait"
# "...".format(run_num))
progress.update()
#finish_time = clock()
#elapsed_time = finish_time - start_time
#print("")
#print("Routing finished! Created %d routes in %s seconds.") % (run_num, elapsed_time)
#print("")
#logger.debug("gained benefit from {} pruned routings".format(pruned_sub_edges))
progress.end()
return routings
struct_file = txrx.checkfile("sark-130.struct")
config_file = txrx.checkfile("spin{}.conf".format(5))
boot.boot(hostname, boot_file, config_file, struct_file)
time.sleep(2.0)
load_targets_path = os.path.join(directory, "pickled_load_targets")
mem_write_targets_path = os.path.join(directory, "pickled_mem_write_targets")
load_targets_file = open(load_targets_path, "rb")
load_targets = pickle.load(load_targets_file)
load_targets_file.close()
mem_write_targets_file = open(mem_write_targets_path, "rb")
mem_write_targets = pickle.load(mem_write_targets_file)
mem_write_targets_file.close()
print("Loading data", file=sys.stderr)
load_progress = ProgressBar(len(load_targets))
for target in load_targets:
txrx.select(target.x, target.y, target.p)
filename = os.path.basename(target.filename)
filename = os.path.join(directory, filename)
txrx.memory_calls.write_mem_from_file(target.address, scamp.TYPE_WORD,
filename)
load_progress.update()
load_progress.end()
print("Writing memory", file=sys.stderr)
mem_progress = ProgressBar(len(mem_write_targets))
for target in mem_write_targets:
txrx.select(target.x, target.y, target.p)
txrx.memory_calls.write_mem(target.address, scamp.TYPE_WORD,
struct.pack("I", target.data))
def generate_output_raw(dao):
"""
The nitty gritty.
Generates load targets and executable targets comprising the simulation, and
for when individual memory locations are to be written, generates memWrites.
This is now largely finished. Data structures are generated for edges
and the data structure generation for vertices is merely a prototype.
*Side effects*:
writes data structures for the load targets to files in the binaries directories
:returns:
Nothing
"""
executable_targets, load_targets, mem_write_targets = list(), list(), list(
)
chipsUsed = set()
progress_bar = ProgressBar(len(dao.placements))
# If we have host-side Spec Execution, execute all Data Specs now:
try:
dao.useHostBasedSpecExecutor = \
conf.config.getboolean( "SpecExecution", "specExecOnHost" )
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
raise Exception("SpecExecutor could not find config information"
" indicating where Spec Execution should occur.")
chips = None
if dao.useHostBasedSpecExecutor == True:
chips = dict()
for placement in dao.placements:
(x, y, p) = placement.processor.get_coordinates()
key = "{}:{}".format(x, y)
if not key in chips:
chips[key] = data_spec_executor.Chip(x, y)
for placement in dao.placements:
if not placement.subvertex.vertex.is_virtual():
start_addr = None
if dao.useHostBasedSpecExecutor == True:
dao.spec_executor = data_spec_executor.SpecExecutor()
(x, y, p) = placement.processor.get_coordinates()
key = "{}:{}".format(x, y)
chip = chips[key]
start_addr = chip.sdram_used + \
data_spec_constants.SDRAM_BASE_ADDR
dao.spec_executor.setup(chip)
subvertex = placement.subvertex
myExecTargets, myLoadTargets, myMemWriteTargets = \
subvertex.generateDataSpec(placement.processor, dao)
# Add this core to the list of targets
if myExecTargets is not None:
executable_targets.append(myExecTargets)
# Append the new dataSpec file to the list of load targets:
if myLoadTargets is not None and len(myLoadTargets) > 0:
load_targets.extend(myLoadTargets)
# Add required memory writes to the list of writes:
if myMemWriteTargets is not None and len(myMemWriteTargets) > 0:
mem_write_targets.extend(myMemWriteTargets)
x, y, p = placement.processor.get_coordinates()
chipsUsed.add((x, y))
hostname = dao.machine.hostname
if dao.useHostBasedSpecExecutor == True:
(x, y, p) = placement.processor.get_coordinates()
f_out = os.path.join(
dao.get_binaries_directory(),
"%s_appData_%d_%d_%d.dat" % (hostname, x, y, p))
dao.spec_executor.finish(f_out)
# TODO: Bring the following in line / neaten
# ----------------------------------------------
# Keep information on the memory region locations
# for later report generation:
index = "%d %d %d" % (x, y, p)
dao.memMaps[index] = [
[i, s.wr_ptr_aligned, s.wr_ptr_offset, s.size, \
s.memory, s.unfilled] \
if s is not None else [i, 0, 0, 0, [], False]
for (i, s) in enumerate(dao.spec_executor.memory_slots)
]
# Add the files produced by the Spec Executor to the
# list of files to load:
load_targets.append(
lib_map.LoadTarget(f_out, x, y, p, start_addr))
mem_write_targets.append(
lib_map.MemWriteTarget(x, y, p, 0xe5007000 + 128 * p + 112,
start_addr))
progress_bar.update()
# populate the DAO with executable, load and memory writing requirements
dao.set_executable_targets(executable_targets)
dao.set_load_targets(load_targets)
dao.set_mem_write_targets(mem_write_targets)
# Generate core map and routing table binaries for each chip
for coord in dao.machine.get_coords_of_all_chips():
x, y = coord['x'], coord['y']
chip = dao.machine.get_chip(x, y)
routeCount = get_route_count(chip)
if (routeCount > 0 or (x, y) in chipsUsed) and not chip.is_virtual():
fileName = generate_routing_table(chip, routeCount, dao)
if (conf.config.getboolean("Reports", "reportsEnabled")
and conf.config.getboolean("Reports", "writeRouterReports")
and conf.config.getboolean("Reports",
"writeRouterDatReport")):
reports.generate_router_report(fileName, chip, dao)
# Place in the list of targets to load at 119.5MB depth in the SDRAM
if not chip.virtual:
load_targets.insert(
0,
lib_map.LoadTarget(
fileName, chip.x, chip.y, 0,
data_spec_constants.ROUTING_TABLE_ADDRESS))
progress_bar.end()
def route_raw(machine, sub_vertexes, k=1, l=0, m=0, BW_PER_ROUTE_ENTRY=0.01, MAX_BW = 250, dao=None):
"""
Modified by peter.choy 06.08.13
Generates a list of routings for subedges in a machine...
For the purposes of this algorithm, we are viewing the chips as 'nodes' on a graph.
The weight of the arcs is a function of the number of routing table entries,
the arc length, and the remaining bandwidth along the arc's corresponding
connection.
The inner function of the algorithm is a slightly modified Dijkstra's algorithm.
The outer loop loops over the necessary sub edges, and re-iterates over each
sub edge several times. Repeating Dijkstra's algorithm for each sub edge is greedy,
and so the iterative process may or may not be necessary.
:param `pacman103.lib.lib_machine.Machine` machine:
machine from which to allocate processors.
:param list subedges:
list of :py:class:`pacman103.lib.graph.Subedge` instances to
route across the machine.
:param k:
constant that is added to arc weights to represent path length
:param l:
constant controlling contribution of Q to total arc weights
:param m:
constant controlling contribution of T to total arc weights
:returns:
list of :py:class:`pacman103.lib.lib_map.Routing` instances.
"""
#print("")
#print("Starting stopwatch...")
#start_time = clock()
#print("")
#print("Initialising routing data structures...")
routings = []
nodes_info = DijkstraRouting.initiate_node_info(machine, MAX_BW)
dijkstra_tables = DijkstraRouting.initiate_dijkstra_tables(machine)
#print("")
#print("Starting routing...")
#print("")
DijkstraRouting.update_weights(nodes_info, machine, l, m, k, MAX_BW)
run_num = 0
pruned_sub_edges = 0
edge_considered = 0
#each subsertex represents a core in the board
progress = ProgressBar(len(sub_vertexes))
for subVertex in sub_vertexes:
subedges = subVertex.out_subedges
#locate all destination and soruce coords
dest_processors = []
subedges_to_route = list()
xs, ys, ps = subVertex.placement.processor.get_coordinates()
for subedge in subedges:
if not subedge.pruneable:
dest_processors.append(
subedge.postsubvertex.placement.processor)
subedges_to_route.append(subedge)
else:
pruned_sub_edges += 1
if(len(dest_processors) != 0):
# Update the weights according to the changes in routing
# tables and available bandwidth
#logger.debug("updating weights")
DijkstraRouting.update_weights(nodes_info, machine,
l, m, k, MAX_BW)
# Reset the temporary storage of lowest cost
DijkstraRouting.reset_tables(dijkstra_tables)
# SD Commented this out 20/1/14, to avoid extraneous printing
# AS intergrated this into logger.debug
#logger.debug("***************************************************")
#logger.debug("Source node is ({}, {})".format(xs, ys))
#logger.debug("Destination nodes are(")
#for processor in dest_processors:
# xd, yd, pd = processor.get_coordinates()
# logger.debug("({}, {}, {})".format(xd, yd, pd))
#logger.debug("")
# Set the first activated node in this run as the source
xa, ya = xs, ys
dijkstra_tables["{}:{}".format(xa, ya)]["activated?"] = True
dijkstra_tables["{}:{}".format(xa, ya)]["lowest cost"] = 0
# The cost at the source node is zero for obvious reasons.
# Note that it is NOT 'None'
DijkstraRouting.\
properate_costs_till_reached_destinations(dijkstra_tables,
nodes_info,
xa, ya,
dest_processors,
xs, ys)
#logger.debug("Reached destination from source, retracing.")
#helpful output data
if conf.config.getboolean( "Routing", "generate_graphs" ) and \
dao is not None: # AM
output_folder = dao.get_reports_directory("routing_graphs")
router_utility= util.RouterUtil(new_output_folder=output_folder)
router_utility.output_routing_weight(router_utility,
dijkstra_tables, machine,
graph_label="routing weights",
routing_file_name="routingWeights" + str(edge_considered))
edge_considered += 1
for subedge in subedges_to_route:
key, mask, = subedge.key, subedge.mask
key_mask_combo = subedge.key_mask_combo
dest = subedge.postsubvertex
xd, yd, pd = dest.placement.processor.get_coordinates()
routing = lib_map.Routing(subedge)
xt, yt = DijkstraRouting.retrace_back_to_source(xd, yd, machine,
nodes_info,
dijkstra_tables,
key, mask,
key_mask_combo,
routing,
pd,
BW_PER_ROUTE_ENTRY)
# SD Commented out 20/1/14 to remove extraneous printing.
# DijkstraRouting.printRoute(xt, yt, xs, ys, dijkstra_tables, routing)
subedge.routing = routing
routings.append(routing)
run_num += 1
# SD Commented this out 20/1/14, to avoid extraneous printing
# AS modified to debug format
#print ""
#logger.debug("Route number {} completed from ({}, {}) "
# "to ({}, {})".format(run_num, xs, ys, xd, yd))
#logger.debug("route took {}".format(DijkstraRouting.printRoute(xd, yd, xs, ys, dijkstra_tables, routing)))
#logger.debug("*********************************"
# "**********************************")
#if (run_num % 5000) == 0:
# logger.debug("{} routes done, please wait"
# "...".format(run_num))
progress.update()
#finish_time = clock()
#elapsed_time = finish_time - start_time
#print("")
#print("Routing finished! Created %d routes in %s seconds.") % (run_num, elapsed_time)
#print("")
#logger.debug("gained benefit from {} pruned routings".format(pruned_sub_edges))
progress.end()
return routings
class PartitionAndPlacePartitioner(AbstractPartitioner):
def __init__(self, dao):
self.dao = dao
#locate the chosen placer
placer_algorithms_list = dict(
map( lambda (name, placer) : (name.replace("Placer",""),placer),
inspect.getmembers(placer_algorithms, inspect.isclass)
))
try:
placer_class = placer_algorithms_list[
conf.config.get("Placer", "algorithm")]
self.placer = placer_class(dao)
except KeyError as e:
raise ValueError("Invalid partitioner algorithm specified. "
" I don't know '%s'." % e)
self.progress = None
def partition(self):
"""
Loads machine and vertex objects from the datastore, calls
:py:func:`pacman103.core.mapper.partition_raw` to generate subvertices
and subedges, and stores them in the datastore.
:param `pacman103.core.dao` dao:
datastore containing machine and vertex objects.
"""
# locate correct placer
logger.info("* Running Partitioner and Placer as one *")
# Load the machine and vertices objects from the dao
machine = self.dao.get_machine()
vertices = self.dao.get_vertices()
#calculate time steps
no_machine_time_steps = None
if self.dao.run_time is not None:
no_machine_time_steps = \
int((self.dao.run_time * 1000.0) / self.dao.machineTimeStep)
machine_time_step_us = self.dao.machineTimeStep
# Partition the vertices into subvertices and consequent subedges
subvertices, subedges, placements = \
self.partition_raw(machine, vertices,no_machine_time_steps,
machine_time_step_us, self.placer)
# Store the results in the dao
self.dao.set_subvertices(subvertices)
self.dao.set_subedges(subedges)
self.dao.set_placements(placements)
#update dao so that controller only calls the next stack aspect
self.dao.done_partitioner = True
self.dao.done_placer = True
def find_max_ratio(self, resources, max_resources):
'''
helper method for finding the max ratio for a resoruces
'''
cpu_ratio = (float(resources.clock_ticks)
/ float(max_resources.clock_ticks))
dtcm_ratio = (float(resources.dtcm) / float(max_resources.dtcm))
sdram_ratio = (float(resources.sdram) / float(max_resources.sdram))
return max((cpu_ratio, dtcm_ratio, sdram_ratio))
def partition_raw(self, machine, vertices, no_machine_time_steps,
machine_time_step_us, placer):
'''
entry method for partitioning
'''
placements = list()
subvertices = list()
#sort out vertex's by constraints
sort = lambda vertex: vertex.constraints.placement_cardinality
vertices = sorted(vertices, key=sort, reverse=True)
n_atoms = 0
for vertex in vertices:
n_atoms += vertex.atoms
self.progress = ProgressBar(n_atoms)
self.partitioned = dict()
# Partition one vertex at a time
for vertex in vertices:
#only partition real vertexes. virtual ones are ignored
if vertex.is_virtual():
self.partition_virtual_vertexes(vertex, placements,
subvertices, placer)
elif len(vertex.subvertices) == 0:
self.partition_vertexes(vertex, no_machine_time_steps,
machine_time_step_us, subvertices,
placer, placements)
else:
print "Vertex {} is already partitioned!".format(vertex.label)
self.progress.end()
subedges = self.generate_sub_edges(subvertices)
return subvertices, subedges, placements
def get_max_atoms_per_core(self, vertices):
max_atoms_per_core = 0
for v in vertices:
max_for_vertex = v.get_maximum_atoms_per_core()
# If there is no maximum, the maximum is the number of atoms
if max_for_vertex is None:
max_for_vertex = v.atoms
# Override the maximum with any custom maximum
if v.custom_max_atoms_per_core is not None:
max_for_vertex = v.custom_max_atoms_per_core
max_atoms_per_core = max(max_atoms_per_core, max_for_vertex)
return max_atoms_per_core
def partition_vertexes(self, vertex, no_machine_time_steps,
machine_time_step_us, subvertices, placer,
placements):
'''
partitions normal vertexes
'''
vertices = list()
vertices.append(vertex)
extra_vertices = vertex.get_partition_dependent_vertices()
if extra_vertices is not None:
for v in extra_vertices:
if v.atoms != vertex.atoms:
raise Exception("A vertex and its partition-dependent"
+ " vertices must have the same number of atoms")
vertices.append(v)
# Prepare for partitioning, getting information
partition_data_objects = [v.get_partition_data_object()
for v in vertices]
max_atoms_per_core = self.get_max_atoms_per_core(vertices)
self.partition_by_atoms(vertices, placer, vertex.atoms,
max_atoms_per_core, no_machine_time_steps, machine_time_step_us,
partition_data_objects, subvertices, placements)
def partition_by_atoms(self, vertices, placer, n_atoms,
max_atoms_per_core, no_machine_time_steps, machine_time_step_us,
partition_data_objects, subvertices, placements):
'''
tries to partition subvertexes on how many atoms it can fit on
each subvert
'''
n_atoms_placed = 0
while n_atoms_placed < n_atoms:
#logger.debug("Maximum available resources for "
# "partitioning: {}".format(resources))
lo_atom = n_atoms_placed
hi_atom = lo_atom + max_atoms_per_core - 1
if hi_atom >= n_atoms:
hi_atom = n_atoms - 1
# Scale down the number of atoms to fit the available resources
used_placements, hi_atom = self.scale_down_resources(
lo_atom, hi_atom, vertices,
no_machine_time_steps, machine_time_step_us,
partition_data_objects, placer,
max_atoms_per_core)
# Update where we are
n_atoms_placed = hi_atom + 1
# Create the subvertices and placements
for (vertex, _, x, y, p, used_resources, _) in used_placements:
subvertex = graph.Subvertex(vertex, lo_atom, hi_atom,
used_resources)
processor = self.dao.machine.get_processor(x, y, p)
placement = lib_map.Placement(subvertex, processor)
subvertices.append(subvertex)
placements.append(placement)
no_atoms_this_placement = (hi_atom - lo_atom) + 1
self.progress.update(no_atoms_this_placement)
def scale_down_resources(self, lo_atom, hi_atom, vertices,
no_machine_time_steps, machine_time_step_us,
partition_data_objects, placer, max_atoms_per_core):
'''
reduces the number of atoms on a core so that it fits within the
resoruces avilable
'''
# Find the number of atoms that will fit in each vertex given the
# resources available
used_placements = list()
min_hi_atom = hi_atom
for i in range(len(vertices)):
vertex = vertices[i]
partition_data_object = partition_data_objects[i]
resources = placer.get_maximum_resources(vertex.constraints)
used_resources = vertex.get_resources_for_atoms(lo_atom, hi_atom,
no_machine_time_steps, machine_time_step_us,
partition_data_object)
ratio = self.find_max_ratio(used_resources, resources)
while ratio > 1.0 and hi_atom >= lo_atom:
# Scale the resources by the ratio
old_n_atoms = (hi_atom - lo_atom) + 1
new_n_atoms = int(float(old_n_atoms) / ratio)
# Avoid looping
if old_n_atoms == new_n_atoms:
new_n_atoms -= 1
else:
# Subtract a tenth of the difference between the old
# and new
new_n_atoms -= int((old_n_atoms - new_n_atoms) / 10.0)
# Find the new resource usage
hi_atom = lo_atom + new_n_atoms - 1
used_resources = \
vertex.get_resources_for_atoms(lo_atom, hi_atom,
no_machine_time_steps,
machine_time_step_us,
partition_data_object)
ratio = self.find_max_ratio(used_resources, resources)
# If we couldn't partition, raise and exception
if hi_atom < lo_atom:
raise Exception("Vertex {} could not be partitioned".format(
vertex.label))
# Try to scale up until just below the resource usage
used_resources, hi_atom = self.scale_up_resource_usage(
used_resources, hi_atom, lo_atom,
max_atoms_per_core, vertex, no_machine_time_steps,
machine_time_step_us, partition_data_object, resources,
ratio)
# If this hi_atom is smaller than the current, minimum update the
# other placements to use (hopefully) less resources
if hi_atom < min_hi_atom:
min_hi_atom = hi_atom
new_used_placements = list()
for (v, part_obj, x, y, p, v_resources, resources) in used_placements:
placer.unplace_subvertex(x, y, p, v_resources)
new_resources = v.get_resources_for_atoms(lo_atom,
min_hi_atom, no_machine_time_steps,
machine_time_step_us, part_obj)
(new_x, new_y, new_p) = placer.place_subvertex(
new_resources, v.constraints)
new_used_placements.append(v, part_obj, new_x, new_y, new_p,
new_resources, resources)
used_placements = new_used_placements
# Place the vertex
x, y, p = placer.place_subvertex(used_resources,
vertex.constraints)
used_placements.append((vertex, partition_data_object, x, y, p,
used_resources, resources))
return used_placements, min_hi_atom
def scale_up_resource_usage(self, used_resources, hi_atom, lo_atom,
max_atoms_per_core, vertex, no_machine_time_steps,
machine_time_step_us, partition_data_object, resources,
ratio):
'''
tries to psuh the number of atoms into a subvertex as it can
with the estimates
'''
previous_used_resources = used_resources
previous_hi_atom = hi_atom
while ((ratio < 1.0) and ((hi_atom + 1) < vertex.atoms)
and ((hi_atom - lo_atom + 2) < max_atoms_per_core)):
#logger.debug("Scaling up - Current subvertex from"
# " %d to %d of %d, ratio = %f, resources = %s" % (lo_atom,
# hi_atom, no_atoms, ratio, used_resources))
previous_hi_atom = hi_atom
hi_atom += 1
# Find the new resource usage
previous_used_resources = used_resources
used_resources = \
vertex.get_resources_for_atoms(lo_atom, hi_atom,
no_machine_time_steps,
machine_time_step_us,
partition_data_object)
ratio = self.find_max_ratio(used_resources, resources)
return previous_used_resources, previous_hi_atom
def partition_virtual_vertexes(self, vertex, placements, subvertices,
placer):
'''
handle the paritioning of virtual vertexes
'''
#ask the vertex how many sub verts to split it into to.
number_of_sub_verts = vertex.split_into_subvertex_count()
number_per_subvert = vertex.atoms / number_of_sub_verts
for subvert_count in range(number_of_sub_verts):
#create a subvert
start = (subvert_count * number_per_subvert)
end = start + number_per_subvert - 1
subvertex = graph.Subvertex(vertex, start, end, 0)
subvertices.append(subvertex)
# Update the constraint to reflect changes if there are
# more than 1 subvert
if vertex.constraints.p is not None:
start_constraint = \
lib_map.VertexConstraints(vertex.constraints.x,
vertex.constraints.y,
vertex.constraints.p + subvert_count)
else:
start_constraint = vertex.constraints
# Place the subvertex
chip = self.dao.machine.get_chip(vertex.constraints.x,
vertex.constraints.y)
x, y, p = placer.place_virtual_subvertex(start_constraint,
chip.get_processors())
processor = self.dao.machine.get_processor(x, y, p)
placement = lib_map.Placement(subvertex, processor)
placements.append(placement)
self.progress.update(vertex.atoms)
#goes though the vertexes and generates subedges for all outcoming edges
def generate_sub_edges(self, subvertices):
'''
Partition edges according to vertex partitioning
'''
subedges = list()
for src_sv in subvertices:
# For each out edge of the parent vertex...
for edge in src_sv.vertex.out_edges:
# ... and create and store a new subedge for each postsubvertex
for dst_sv in edge.postvertex.subvertices:
#logger.debug(
# "Creating subedge between {} ({}-{}) and {} ({}-{})"
# .format(src_sv.vertex.label, src_sv.lo_atom,
# src_sv.hi_atom, dst_sv.vertex.label,
# dst_sv.lo_atom, dst_sv.hi_atom))
subedge = edge.create_subedge(src_sv, dst_sv)
subedges.append(subedge)
return subedges
def generate_output_raw(dao):
"""
The nitty gritty.
Generates load targets and executable targets comprising the simulation, and
for when individual memory locations are to be written, generates memWrites.
This is now largely finished. Data structures are generated for edges
and the data structure generation for vertices is merely a prototype.
*Side effects*:
writes data structures for the load targets to files in the binaries directories
:returns:
Nothing
"""
executable_targets, load_targets, mem_write_targets = list(), list(), list()
chipsUsed = set()
progress_bar = ProgressBar(len(dao.placements))
# If we have host-side Spec Execution, execute all Data Specs now:
try:
dao.useHostBasedSpecExecutor = \
conf.config.getboolean( "SpecExecution", "specExecOnHost" )
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
raise Exception( "SpecExecutor could not find config information"
" indicating where Spec Execution should occur." )
chips = None
if dao.useHostBasedSpecExecutor == True:
chips = dict()
for placement in dao.placements:
(x, y, p) = placement.processor.get_coordinates()
key = "{}:{}".format(x, y)
if not key in chips:
chips[key] = data_spec_executor.Chip(x, y)
for placement in dao.placements:
if not placement.subvertex.vertex.is_virtual():
start_addr = None
if dao.useHostBasedSpecExecutor == True:
dao.spec_executor = data_spec_executor.SpecExecutor()
(x, y, p) = placement.processor.get_coordinates()
key = "{}:{}".format(x, y)
chip = chips[key]
start_addr = chip.sdram_used + \
data_spec_constants.SDRAM_BASE_ADDR
dao.spec_executor.setup(chip)
subvertex = placement.subvertex
myExecTargets, myLoadTargets, myMemWriteTargets = \
subvertex.generateDataSpec(placement.processor, dao)
# Add this core to the list of targets
if myExecTargets is not None:
executable_targets.append(myExecTargets)
# Append the new dataSpec file to the list of load targets:
if myLoadTargets is not None and len(myLoadTargets) > 0:
load_targets.extend(myLoadTargets)
# Add required memory writes to the list of writes:
if myMemWriteTargets is not None and len(myMemWriteTargets) > 0:
mem_write_targets.extend(myMemWriteTargets)
x, y, p = placement.processor.get_coordinates()
chipsUsed.add((x, y))
hostname = dao.machine.hostname
if dao.useHostBasedSpecExecutor == True:
(x, y, p) = placement.processor.get_coordinates()
f_out = os.path.join(
dao.get_binaries_directory(),
"%s_appData_%d_%d_%d.dat" % (hostname, x, y, p)
)
dao.spec_executor.finish(f_out)
# TODO: Bring the following in line / neaten
# ----------------------------------------------
# Keep information on the memory region locations
# for later report generation:
index = "%d %d %d" % (x, y, p)
dao.memMaps[index] = [
[i, s.wr_ptr_aligned, s.wr_ptr_offset, s.size, \
s.memory, s.unfilled] \
if s is not None else [i, 0, 0, 0, [], False]
for (i, s) in enumerate(dao.spec_executor.memory_slots)
]
# Add the files produced by the Spec Executor to the
# list of files to load:
load_targets.append(lib_map.LoadTarget(
f_out, x, y, p, start_addr))
mem_write_targets.append(lib_map.MemWriteTarget(
x, y, p, 0xe5007000 + 128*p + 112, start_addr))
progress_bar.update()
# populate the DAO with executable, load and memory writing requirements
dao.set_executable_targets(executable_targets)
dao.set_load_targets(load_targets)
dao.set_mem_write_targets(mem_write_targets)
# Generate core map and routing table binaries for each chip
for coord in dao.machine.get_coords_of_all_chips():
x, y = coord['x'], coord['y']
chip = dao.machine.get_chip(x, y)
routeCount = get_route_count(chip)
if (routeCount > 0 or (x, y) in chipsUsed) and not chip.is_virtual():
fileName = generate_routing_table(chip, routeCount, dao)
if (conf.config.getboolean("Reports", "reportsEnabled") and
conf.config.getboolean("Reports", "writeRouterReports") and
conf.config.getboolean("Reports", "writeRouterDatReport")):
reports.generate_router_report(fileName, chip, dao)
# Place in the list of targets to load at 119.5MB depth in the SDRAM
if not chip.virtual:
load_targets.insert(
0, lib_map.LoadTarget(
fileName, chip.x, chip.y, 0,
data_spec_constants.ROUTING_TABLE_ADDRESS
)
)
progress_bar.end()