def instanceRouter(self, name, radix, rtr_id):
if self._check_first_build():
sst.addGlobalParams("%s_params"%self._instance_name, self._getGroupParams("params"))
rtr = sst.Component(name, "merlin.hr_router")
self._applyStatisticsSettings(rtr)
rtr.addGlobalParamSet("%s_params"%self._instance_name)
rtr.addParam("num_ports",radix)
rtr.addParam("id",rtr_id)
return rtr
def build(self, nID, num_vNics=1):
if self._check_first_build():
sst.addGlobalParams("params_%s" % self._instance_name,
self._getGroupParams("main"))
sst.addGlobalParam("params_%s" % self._instance_name, "num_vNics",
num_vNics)
nic = sst.Component("nic" + str(nID), "firefly.nic")
self._applyStatisticsSettings(nic)
nic.addGlobalParamSet("params_%s" % self._instance_name)
nic.addParam("nid", nID)
#nic.addParam("num_vNics",num_vNics)
return nic, "rtrLink"
def build(self, engine, nicLink, loopLink, size, nicsPerNode, job_id, pid,
lid, coreId):
if self._check_first_build():
set_name = "params_%s" % self._instance_name
sst.addGlobalParams(set_name, self._getGroupParams("main"))
sst.addGlobalParam(set_name, 'numNodes', size)
sst.addGlobalParam(set_name, 'netMapName', 'Firefly' + str(job_id))
sst.addGlobalParam(set_name, 'netMapSize', size)
sst.addGlobalParams("ctrl_params_%s" % self._instance_name,
self._getGroupParams("ctrl"))
sst.addGlobalParam("ctrl_params_%s" % self._instance_name,
"nicsPerNode", nicsPerNode)
os = engine.setSubComponent("OS", "firefly.hades")
os.addGlobalParamSet("params_%s" % self._instance_name)
os.addParam('netId', pid)
os.addParam('netMapId', lid)
os.addParam('coreId', coreId)
virtNic = os.setSubComponent("virtNic", "firefly.VirtNic")
# For now, just use default parameters
#for key, value in self.driverParams.items():
# if key.startswith(self.driverParams['os.name']+'.virtNic'):
# key = key[key.rfind('.')+1:]
# virtNic.addParam( key,value)
proto = os.setSubComponent("proto", "firefly.CtrlMsgProto")
process = proto.setSubComponent("process", "firefly.ctrlMsg")
proto.addGlobalParamSet("ctrl_params_%s" % self._instance_name)
process.addGlobalParamSet("ctrl_params_%s" % self._instance_name)
# Should figure out how to put this into the main param group
# object
#proto.addParam("nicsPerNode", nicsPerNode)
#process.addParam("nicsPerNode", nicsPerNode)
#nicLink.connect( (virtNic,'nic','1ns' ),(nic,'core'+str(x),'1ns'))
virtNic.addLink(nicLink, 'nic', '1ns')
#loopLink.connect( (process,'loop','1ns' ),(loopBack,'nic'+str(nodeID%self._nicsPerNode)+'core'+str(x),'1ns'))
process.addLink(loopLink, 'loop', '1ns')
def build(self,
comp,
slot,
slot_num,
job_id,
job_size,
logical_nid,
use_nid_remap=False):
if self._check_first_build():
set_name = "params_%s" % self._instance_name
sst.addGlobalParams(set_name, self._getGroupParams("params"))
sst.addGlobalParam(set_name, "job_id", job_id)
sst.addGlobalParam(set_name, "job_size", job_size)
sst.addGlobalParam(set_name, "use_nid_remap", use_nid_remap)
sub = comp.setSubComponent(slot, "merlin.linkcontrol", slot_num)
self._applyStatisticsSettings(sub)
sub.addGlobalParamSet("params_%s" % self._instance_name)
sub.addParam("logical_nid", logical_nid)
return sub, "rtr_port"
def build(self, endpoint):
if self._check_first_build():
sst.addGlobalParams("params_%s" % self._instance_name,
self._getGroupParams("main"))
if self.host_link_latency is None:
self.host_link_latency = self.link_latency
num_peers = self.hosts_per_router * self.routers_per_group * self.num_groups
total_intergroup_links = (self.num_groups - 1) * self.intergroup_links
intergroup_per_router = (total_intergroup_links +
self.routers_per_group -
1) // self.routers_per_group
empty_ports = intergroup_per_router * self.routers_per_group - total_intergroup_links
num_ports = self.routers_per_group - 1 + self.hosts_per_router + intergroup_per_router
links = dict()
#####################
def getLink(name):
if name not in links:
links[name] = sst.Link(name)
return links[name]
#####################
rpg = self.routers_per_group
ng = self.num_groups - 1 # don't count my group
igpr = intergroup_per_router
if self.global_link_map is None:
# Need to define global link map
self.global_link_map = [-1 for i in range(igpr * rpg)]
# Links will be mapped in linear order, but we will
# potentially skip one port per router, depending on the
# parameters. The variable self.empty_ports tells us how
# many routers will have one global port empty.
count = 0
start_skip = rpg - empty_ports
for i in range(0, rpg):
# Determine if we skip last port for this router
end = igpr
if i >= start_skip:
end = end - 1
for j in range(0, end):
self.global_link_map[i * igpr + j] = count
count = count + 1
# End set global link map with default
# g is group number
# r is router number with group
# p is port number relative to start of global ports
def getGlobalLink(g, r, p):
# Look into global link map to get the dest group and link
# number to that group
raw_dest = self.global_link_map[r * igpr + p]
if raw_dest == -1:
return None
# Turn raw_dest into dest_grp and link_num
link_num = raw_dest // ng
dest_grp = raw_dest - link_num * ng
if (self.global_routes == "absolute"):
# Compute dest group ignoring my own group id, for a
# dest_grp >= g, we need to add 1 to get the right group
if dest_grp >= g:
dest_grp = dest_grp + 1
elif (self.global_routes == "relative"):
# For relative, add current group to dest_grp + 1 and
# do modulo of num_groups to get actual group
dest_grp = (dest_grp + g + 1) % (ng + 1)
#else:
# should never happen
src = min(dest_grp, g)
dest = max(dest_grp, g)
#getLink("link:g%dg%dr%d"%(g, src, dst)), "port%d"%port, self.params["link_lat"])
return getLink("%sglobal_link_g%dg%dr%d" %
(self._prefix, src, dest, link_num))
#########################
router_num = 0
nic_num = 0
# GROUPS
for g in range(self.num_groups):
# GROUP ROUTERS
for r in range(self.routers_per_group):
rtr = self._instanceRouter(num_ports, router_num)
# Insert the topology object
sub = rtr.setSubComponent(self.router.getTopologySlotName(),
"merlin.dragonfly", 0)
self._applyStatisticsSettings(sub)
sub.addGlobalParamSet("params_%s" % self._instance_name)
sub.addParam("intergroup_per_router", intergroup_per_router)
if router_num == 0:
# Need to send in the global_port_map
#map_str = str(self.global_link_map).strip('[]')
#rtr.addParam("dragonfly.global_link_map",map_str)
sub.addParam("global_link_map", self.global_link_map)
port = 0
for p in range(self.hosts_per_router):
#(nic, port_name) = endpoint.build(nic_num, {"num_peers":num_peers})
(nic, port_name) = endpoint.build(nic_num, {})
if nic:
link = sst.Link("link_g%dr%dh%d" % (g, r, p))
#network_interface.build(nic,slot,0,link,self.host_link_latency)
link.connect(
(nic, port_name, self.host_link_latency),
(rtr, "port%d" % port, self.host_link_latency))
#link.setNoCut()
#rtr.addLink(link,"port%d"%port,self.host_link_latency)
nic_num = nic_num + 1
port = port + 1
for p in range(self.routers_per_group):
if p != r:
src = min(p, r)
dst = max(p, r)
rtr.addLink(getLink("link_g%dr%dr%d" % (g, src, dst)),
"port%d" % port, self.link_latency)
port = port + 1
for p in range(igpr):
link = getGlobalLink(g, r, p)
if link is not None:
rtr.addLink(link, "port%d" % port, self.link_latency)
port = port + 1
router_num = router_num + 1
def build(self, nodeID, extraKeys):
if self._check_first_build():
sst.addGlobalParams("loopback_params_%s" % self._instance_name, {
"numCores": self._numCores,
"nicsPerNode": self._nicsPerNode
})
sst.addGlobalParams("params_%s" % self._instance_name,
self._getGroupParams("ember"))
sst.addGlobalParam("params_%s" % self._instance_name, 'jobId',
self.job_id)
sst.addGlobalParams("params_%s" % self._instance_name,
self._motifs)
sst.addGlobalParams("params_%s" % self._instance_name, self._apis)
nic, slot_name = self.nic.build(nodeID,
self._numCores // self._nicsPerNode)
#print( nodeID, "nic", self._getGroupParams("nic") )
#print( nodeID, "ember", self._getGroupParams("ember") )
# not needed: nic.addParams( self._getGroupParams("nic") ). This is done already in the nic.build call
# Build NetworkInterface
logical_id = self._nid_map[nodeID]
networkif, port_name = self.network_interface.build(
nic, slot_name, 0, self.job_id, self.size, logical_id, False)
# Store return value for later
retval = (networkif, port_name)
# Set up the loopback for intranode communications. In the
# case of multiple nics, will name things based on the lowest
# nodeID for this node. This will allow different jobs to use
# different numbers of nics per node (thoough the system level
# allocation will have to be a multiple of any of the number
# of nics per node values used.
my_id_name = str((nodeID // self._nicsPerNode) * self._nicsPerNode)
loopBackName = "loopBack" + my_id_name
if nodeID % self._nicsPerNode == 0:
loopBack = sst.Component(loopBackName, "firefly.loopBack")
#loopBack.addParam( "numCores", self._numCores )
#loopBack.addParam( "nicsPerNode", self._nicsPerNode )
loopBack.addGlobalParamSet("loopback_params_%s" %
self._instance_name)
self._loopBackDict[loopBackName] = loopBack
else:
loopBack = self._loopBackDict[loopBackName]
for x in range(self._numCores // self._nicsPerNode):
# Instance the EmberEngine
ep = sst.Component(
"nic" + str(nodeID) + "core" + str(x) + "_EmberEP",
"ember.EmberEngine")
self._applyStatisticsSettings(ep)
ep.addGlobalParamSet("params_%s" % self._instance_name)
# Add the params to the EmberEngine
#ep.addParam('jobId',self.job_id)
# Add the parameters defining the motifs
# Check to see if we need to enable motif logging
if self._logfilePrefix:
if (self._logfileNids is None
or logical_id in self._logfileNids):
ep.addParam("motifLog", self._logfilePrefix)
# Create the links to the OS layer
nicLink = sst.Link("nic" + str(nodeID) + "core" + str(x) + "_Link")
nicLink.setNoCut()
linkName = "loop" + my_id_name + "nic" + str(
nodeID % self._nicsPerNode) + "core" + str(x) + "_Link"
loopLink = sst.Link(linkName)
loopLink.setNoCut()
#nicLink.connect( (virtNic,'nic','1ns' ),(nic,'core'+str(x),'1ns'))
nic.addLink(nicLink, 'core' + str(x), '1ns')
#loopLink.connect( (process,'loop','1ns' ),(loopBack,'nic'+str(nodeID%self._nicsPerNode)+'core'+str(x),'1ns'))
loopBack.addLink(
loopLink,
'nic' + str(nodeID % self._nicsPerNode) + 'core' + str(x),
'1ns')
# Create the OS layer
self.os.build(ep, nicLink, loopLink, self.size, self._nicsPerNode,
self.job_id, nodeID, logical_id, x)
return retval