def reqService(self, offset, eventName, data, rx=None, rxId=None):
#Purpose: Send an event if Simian is running.
engine = self.engine #Get the engine for this entity
if rx != None and offset < engine.minDelay:
if not engine.running:
raise SimianError("Sending event when Simian is idle!")
#If sending to self, then do not check against min-delay
raise SimianError(self.name + "[" + str(self.num) + "]" +
" attempted to send with too little delay")
time = engine.now + offset
if time > engine.endTime: #No need to send this event
return
if rx == None: rx = self.name
if rxId == None: rxId = self.num
e = {
"tx": self.name, #String
"txId": self.num, #Number
"rx": rx, #String
"rxId": rxId, #Number
"name": eventName, #String
"data": data, #Object
"time": time, #Number
}
recvRank = engine.getOffsetRank(rx, rxId)
if recvRank == engine.rank: #Send to self
heapq.heappush(engine.eventQueue, (time, e))
else:
if time < engine.minSent: engine.minSent = time
#engine.MPI.isend(e, recvRank) #Send to others (Problem with MPI buffers getting filled too fast)
engine.MPI.send(e, recvRank) #Send to others
def __init__(self, engine):
self.engine = weakref.ref(engine)
self.event_queue = []
self.current_gpu = None
self.current_stream = (0, 0)
self.process_list = []
#Initialize CUDA
try:
import pycuda.driver as cuda
from pycuda.compiler import SourceModule
self.driver = cuda
self.driver.init()
except:
raise SimianError(
"Please install pyCuda before using Simian for CUDA based simulation"
)
self.num_devices = cuda.Device.count()
print "Found ", self.num_devices, " cuda devices on this node"
if self.num_devices == 0:
raise SimianError("No cuda capable device found on this node")
self.device_queues = []
self.contexts = []
for gpu in range(self.num_devices):
self.contexts.append(cuda.Device(gpu).make_context())
streams = []
num_mp = cuda.Device(gpu).get_attribute(
cuda.device_attribute.MULTIPROCESSOR_COUNT)
for mp in range(num_mp):
streams.append(cuda.Stream())
self.device_queues.append(streams)
print "Device ", gpu, " contains ", num_mp, " multiprocessors"
self.contexts[0].push()
self.current_gpu = 0
def reqService(self, offset, eventName, data, rx=None, rxId=None):
#Purpose: Send an event if Simian is running.
engine = self.engine #Get the engine for this entity
if rx != None and offset < engine.minDelay:
if not engine.running:
raise SimianError("Sending event when Simian is idle!")
#If sending to self, then do not check against min-delay
raise SimianError(self.name + "[" + str(self.num) + "]" +
" attempted to send with too little delay")
time = engine.now + offset
if time > engine.endTime: #No need to send this event
return
if rx == None: rx = self.name
if rxId == None: rxId = self.num
e = {
"tx": self.name, #String
"txId": self.num, #Number
"rx": rx, #String
"rxId": rxId, #Number
"name": eventName, #String
"data": data, #Object
"time": time, #Number
}
# this is a particular mechanism added by Jason Liu for
# allowing different mappings from LPs to ranks
recvRank = engine.getOffsetRank(rx, rxId)
if recvRank == engine.rank: #Send to self
heapq.heappush(engine.eventQueue, (time, e))
else:
engine.MPI.sendAndCount(e, recvRank)
def createProcess(self, name, fun, kind=None): #Creates a named process
global Process
if not Process:
from process import Process
if name == "*":
raise SimianError("Reserved name to represent all child processes: " + name)
proc = Process(name, fun, self, None) #No parent means, entity is parent
if not proc:
raise SimianError("Could not create a valid process named: " + name)
self._procList[name] = proc
if kind != None:
self.categorizeProcess(kind, name) #Categorize
def addEntity(self, name, entityClass, num, *args):
#Purpose: Add an entity to the entity-list if Simian is idle
#This function takes a pointer to a class from which the entities can
#be constructed, a name, and a number for the instance.
if self.running:
raise SimianError("Adding entity when Simian is running!")
if not (name in self.entities):
self.entities[name] = {} #To hold entities of this "name"
entity = self.entities[name]
self.baseRanks[name] = self.getBaseRank(name) #Register base-ranks
computedRank = self.getOffsetRank(name, num)
if computedRank == self.rank: #This entity resides on this engine
#Output log file for this Entity
self.out.write(name + "[" + str(num) + "]: Running on rank " +
str(computedRank) + "\n")
entity[num] = entityClass(
{
"name": name,
"out": self.out,
"engine": self,
"num": num,
}, *args) #Entity is instantiated
def optimisticRollback(self, time, LP):
backup = False
numRolls = 0
if time < self.optimistic_GVT:
raise SimianError(
"rollback before GVT!!!! GVT: %s , Event Queue Dump : %s" %
(self.optimistic_GVT, self.eventQueue))
if len(LP.processedEvents):
while LP.processedEvents[len(LP.processedEvents) -
1][0]["time"] >= time:
if self.statistics:
numRolls += 1
(event, state) = LP.processedEvents.pop(-1)
heap.push(self.eventQueue, (event["time"], dict(event)))
backup = dict(state)
LP.recoverRandoms(state)
LP.recoverAntimessages(state, time)
self.optimisticNumEventsRolledBack += 1
if not len(LP.processedEvents): break
if self.statistics:
self.rollbackLength.append(numRolls)
if backup:
LP.recoverState(backup)
LP.VT = time
def iprobe(self, src=None, tag=None): #Non-blocking asynch
src = src or mpi.MPI_ANY_SOURCE
tag = tag or mpi.MPI_ANY_TAG
if mpi.MPI_Iprobe(src, tag, self.comm, C.byref(self.itemp),
C.byref(self.status)) == mpi.MPI_SUCCESS:
return (self.itemp.value != 0)
raise SimianError("Could not Iprobe in MPI")
def wakeProcess(self, name, *args): #Wake a named process with arguments
if not (name in self._procList):
raise SimianError("Attempted to wake a non existant process: " +
name)
else: #If existing and not been killed asynchronously
proc = self._procList[name]
proc.wake(*args)
def __init__(self, libName):
global mpi
mpi = loadMPI(libName)
if mpi.MPI_Init(None, None) != mpi.MPI_SUCCESS:
raise SimianError("Could not initialize MPI")
self.CBUF_LEN = 32 * 1024 #32kB
self.comm = mpi.MPI_COMM_WORLD
self.BYTE = mpi.MPI_BYTE
self.DOUBLE = mpi.MPI_DOUBLE
self.LONG = mpi.MPI_LONG
self.MIN = mpi.MPI_MIN
self.SUM = mpi.MPI_SUM
self.request = mpi.MPI_Request()
self.status = mpi.MPI_Status()
self.itemp = C.c_int()
self.dtemp0 = C.c_double()
self.dtemp1 = C.c_double()
self.ctemp = C.create_string_buffer(self.CBUF_LEN) #Preallocate
self.numRanks = self.size()
self.sndCounts = (C.c_long * self.numRanks)()
for i in range(len(self.sndCounts)):
self.sndCounts[i] = 0
self.rcvCounts = (C.c_long * self.numRanks)()
def sendAndCount(self, x, dst, tag=None): #Blocking
m = Pack(x)
tag = tag or len(m) #Set to message length if None
if mpi.MPI_Send(m, len(m), self.BYTE, dst, tag,
self.comm) != mpi.MPI_SUCCESS:
raise SimianError("Could not Send in MPI")
self.sndCounts[dst] += 1
def killProcessKind(self, kind): #Kills all @kind processes on entity
if not (kind in self._category):
raise SimianError("killProcessKind: No category of processes on this entity called " + str(kind))
else:
nameSet = self._category[kind]
for name,_ in nameSet.items(): #So we can delete while iterating
proc = self._procList[name]
proc.kill() #Kill itself and all subprocesses
def __init__(self, simName, startTime, endTime, minDelay=1, useMPI=False, mpiLibName=defaultMpichLibName):
self.Entity = Entity #Include in the top Simian namespace
self.name = simName
self.startTime = startTime
self.endTime = endTime
self.minDelay = minDelay
self.now = startTime
#If simulation is running
self.running = False
#Stores the entities available on this LP
self.entities = {}
#Events are stored in a priority-queue or heap, in increasing
#order of time field. Heap top can be accessed using self.eventQueue[0]
#event = {time, name, data, tx, txId, rx, rxId}.
self.eventQueue = []
#Stores the minimum time of any event sent by this process,
#which is used in the global reduce to ensure global time is set to
#the correct minimum.
self.infTime = endTime + 2*minDelay
self.minSent = self.infTime
#[[Base rank is an integer hash of entity's name]]
self.baseRanks = {}
#Make things work correctly with and without MPI
if useMPI:
#Initialize MPI
try:
global MPI
from MPILib import MPI
self.useMPI = True
self.MPI = MPI(mpiLibName)
self.rank = self.MPI.rank()
self.size = self.MPI.size()
except:
raise SimianError("Please ensure libmpich is available to ctypes before using Simian for MPI based simulations.\nTry passing absolute path to libmpich.[dylib/so/dll] to Simian.")
else:
self.useMPI = False
self.MPI = None
self.rank = 0
self.size = 1
#One output file per rank
self.out = open(self.name + "." + str(self.rank) + ".out", "w")
#Write some header information for each output file
self.out.write("===========================================\n")
self.out.write("----------SIMIAN-PIE PDES ENGINE-----------\n")
self.out.write("===========================================\n")
if self.useMPI:
self.out.write("MPI: ON\n\n")
else:
self.out.write("MPI: OFF\n\n")
def categorizeProcess(self, kind, name): #Check for existing process and then categorize
if name in self._procList:
proc = self._procList[name]
#Categorize both ways for easy lookup
proc._kindSet[kind] = True #Indicate to proc that it is of this kind to its entity
#Indicate to entity that proc is of this kind
if not kind in self._category: self._category[kind] = {name: True} #New kind
else: self._category[kind][name] = True #Existing kind
else: raise SimianError("categorize: Expects a proper child to categorize")
def alltoallSum(self):
if (mpi.MPI_Alltoall(self.sndCounts, 1, self.LONG, self.rcvCounts, 1,
self.LONG, self.comm) != mpi.MPI_SUCCESS):
raise SimianError("Could not AllToAll in MPI")
toRcv = 0
for i in range(self.numRanks):
toRcv = toRcv + self.rcvCounts[i]
self.sndCounts[i] = 0
return toRcv
def startProcess(self, name, *args): #Starts a named process
if name in self._procList:
proc = self._procList[name]
if not proc.started:
proc.started = True
#When starting, pass process instance as first arg, which can be accessed inside the "fun"
return proc.wake(proc, *args)
else:
raise SimianError("Starting an already started process: " + proc.name)
def unCategorizeProcess(self, kind, name):
#Check for existing process and then unCategorize
if name in self._procList:
proc = self._procList[name]
#unCategorize both ways for easy lookup
proc._kindSet.pop(kind) #Indicate to proc that it is not of this kind to its entity
#Indicate to entity that proc is not of this kind
if kind in self._category:
self._category[kind].pop(name) #Existing kind deleted
else: raise SimianError("unCategorize: Expects a proper child to un-categorize")
def allreduce(self, partial, op):
self.dtemp0 = C.c_double(partial)
if (mpi.MPI_Allreduce(
C.byref(self.dtemp0),
C.byref(self.dtemp1),
1,
self.DOUBLE, #Single double operand
op,
self.comm) != mpi.MPI_SUCCESS):
raise SimianError("Could not Allreduce in MPI")
return self.dtemp1.value
def wake(thisProcess, *args):
#Arguments "*args" to __call => function-body
#Arguments "*args" to wake => LHS of hibernate
co = thisProcess.co
if co != None and not co.dead:
thisProcess.main = greenlet.getcurrent()
thisProcess.suspended = False
return co.switch(*args)
else:
raise SimianError("Attempted to wake a process: " +
thisProcess.name + " failed")
def recv(self, maxSize, src=None, tag=None): #Blocking
src = src or mpi.MPI_ANY_SOURCE
tag = tag or mpi.MPI_ANY_TAG
m = self.ctemp
if maxSize > self.CBUF_LEN: #Temporary buffer is too small
m = C.create_string_buffer(maxSize)
if mpi.MPI_Recv(m, maxSize, self.BYTE, src, tag, self.comm,
C.byref(self.status)) == mpi.MPI_SUCCESS:
#return Unpack(m.raw)
return Unpack(m[:maxSize])
raise SimianError("Could not Recv in MPI")
def sleep(thisProcess, x, *args):
#Processes which are to implicitly wake at set timeouts
#All return values are passed to __call/wake
if (not isinstance(x, (int, long, float))) or (x < 0):
raise SimianError("Sleep not given non-negative number argument!" +
thisProcess.name)
entity = thisProcess.entity
#Schedule a local alarm event after x timesteps to wakeup
entity.engine.schedService(entity.engine.now + x, "_wakeProcess",
thisProcess.name, entity.name, entity.num)
thisProcess.suspended = True
return thisProcess.main.switch(*args)
def __init__(self, simName, startTime, endTime, minDelay=1, useMPI=False):
self.Entity = Entity #Include in the top Simian namespace
self.name = simName
self.startTime = startTime
self.endTime = endTime
self.minDelay = minDelay
self.now = startTime
#If simulation is running
self.running = False
#Stores the entities available on this LP
self.entities = {}
#Events are stored in a priority-queue or heap, in increasing
#order of time field. Heap top can be accessed using self.eventQueue[0]
#event = {time, name, data, tx, txId, rx, rxId}.
self.eventQueue = []
#Stores the minimum time of any event sent by this process,
#which is used in the global reduce to ensure global time is set to
#the correct minimum.
self.infTime = endTime + 2 * minDelay
self.minSent = self.infTime
#[[Base rank is an integer hash of entity's name]]
self.baseRanks = {}
#Make things work correctly with and without MPI
if useMPI:
#Initialize MPI
global MPI
try:
from mpi4py import MPI
self.useMPI = True
self.comm = MPI.COMM_WORLD
self.rank = self.comm.Get_rank()
self.size = self.comm.Get_size()
except:
raise SimianError(
"Please install mpi4py before using Simian for MPI based simulations"
)
else:
self.useMPI = False
self.comm = None
self.rank = 0
self.size = 1
#One output file per rank
self.out = open(self.name + "." + str(self.rank) + ".out", "w")
def spawn(thisProcess, name, fun, kind=None):
#Create a new named processes as child or @kind
entity = thisProcess.entity
if name in entity._procList:
raise SimianError("spawn: Process by name '" + name +
"' already exists in entity " + entity.name +
"[" + str(entity.num) + "]")
entity.createProcess(name, fun,
kind) #Creates a named process of kind type
#Make this a child of thisProcess
#NOTE: This is the difference between process.spawn and entity.createProcess
entity._procList[name].parent = thisProcess
thisProcess._childList[name] = True
#Author: Nandakishore Santhi
#Date: 23 November, 2014
#Copyright: Open source, must acknowledge original author
#Purpose: PDES Engine in Python, mirroring a subset of the Simian JIT-PDES
# Named process class
from utils import SimianError
from greenlet import greenlet
try:
from greenlet import greenlet
except:
raise SimianError(
"Please install greenlets before using SimianPie to run simulations")
#Making this pythonic - this is a base class that all derived Processes will inherit from
class Process(object):
#ent.createProcess/proc.hibernate <=> proc.wake/ent.wakeProcess
#proc.sleep/proc.compute <=> ent.wakeProcess
def __init__(self, name, fun, thisEntity, thisParent):
self.name = name
self.co = greenlet(run=fun)
self.started = False
self.suspended = False
self.main = greenlet.getcurrent(
) #To hold the main process for to/from context-switching within sleep/wake/hibernate
self.entity = thisEntity
self.parent = thisParent #Parent is None if created by entity
self._kindSet = {} #Set of kinds that it belongs to on its entity
self._childList = {}
def finalize(self):
if mpi.MPI_Finalize() == mpi.MPI_SUCCESS:
return False
raise SimianError("Could not finalize MPI")
def reqService(self, offset, eventName, data, rx=None, rxId=None):
#Purpose: Send an event if Simian is running.
engine = self.engine #Get the engine for this entity
if not engine.optimistic:
if rx != None and offset < engine.minDelay:
if not engine.running: raise SimianError("Sending event when Simian is idle!")
#If sending to self, then do not check against min-delay
raise SimianError(self.name + "[" + str(self.num) + "]"
+ " attempted to send with too little delay")
time = engine.now + offset
if engine.optimistic:
if offset == 0 :
offset = 0.000000001 # add 1 nano unit
#event can not create an event for the same time ( this resolution is debatable)
time = self.VT + offset
#print time
if time > engine.endTime:
return
if engine.optimistic_color == 'white':
color = 'white'
else:
color = 'red'
if rx == None: rx = self.name
if rxId == None: rxId = self.num
e = {
"tx": self.name, #String
"txId": self.num, #Number
"rx": rx, #String
"rxId": rxId, #Number
"name": eventName, #String
"data": data, #Object
"time": time, #Number
"antimessage" : False,
"GVT" : False,
"color" : color,
}
if engine.optimistic:
ae = {
"tx": self.name, #String
"txId": self.num, #Number
"rx": rx, #String
"rxId": rxId, #Number
"name": eventName, #String
"data": data, #Object
"time": time, #Number
"antimessage" : True,
"GVT" : False,
"color" : color,
}
self.sentEvents.append(ae)
# this is a particular mechanism added by Jason Liu for
# allowing different mappings from LPs to ranks
if engine.partfct:
recvRank = engine.partfct(rx, rxId, engine.size, engine.partarg)
else:
recvRank = engine.getOffsetRank(rx, rxId)
if engine.statistics:
try:
self.statisticsWhoSendLP[(rx,rxId)] += 1
self.statisticsWhoSendRank[recvRank] += 1
except KeyError:
self.statisticsWhoSendLP[(rx,rxId)] = 1
self.statisticsWhoSendRank[recvRank] = 1
if recvRank == engine.rank: #Send to self
engine.heap.push(engine.eventQueue, (time, e))
else:
if engine.optimistic:
if engine.optimistic_color == 'white':
engine.optimistic_white += 1 # GVT approx algo
else:
engine.optimistic_t_min = min(engine.optimistic_t_min, time)
engine.MPI.send(e, recvRank)
else:
engine.MPI.sendAndCount(e, recvRank)
def rank(self):
if mpi.MPI_Comm_rank(self.comm,
C.byref(self.itemp)) == mpi.MPI_SUCCESS:
return self.itemp.value
raise SimianError("Could not get rank in MPI")
def size(self):
#size = (C.c_int * 1)()
if mpi.MPI_Comm_size(self.comm,
C.byref(self.itemp)) == mpi.MPI_SUCCESS:
return self.itemp.value
raise SimianError("Could not get size in MPI")
def isend(self, x, dst, tag=None): #Non-Blocking
m = Pack(x)
tag = tag or len(m) #Set to message length if None
if mpi.MPI_Isend(m, len(m), self.BYTE, dst, tag, self.comm,
C.byref(self.request)) != mpi.MPI_SUCCESS:
raise SimianError("Could not Isend in MPI")
def run(self): #Run the simulation
startTime = timeLib.clock()
if self.rank == 0:
print("===========================================")
print("----------SIMIAN-PIE PDES ENGINE-----------")
print("===========================================")
if self.useMPI:
print("MPI: ON")
else:
print("MPI: OFF")
numEvents = 0
self.running = True
globalMinLeft = self.startTime
while globalMinLeft <= self.endTime:
epoch = globalMinLeft + self.minDelay
self.minSent = self.infTime
while len(self.eventQueue) > 0 and self.eventQueue[0][0] < epoch:
(time, event) = heapq.heappop(self.eventQueue) #Next event
if self.now > time:
raise SimianError(
"Out of order event: now=%f, evt=%f" % self.now, time)
self.now = time #Advance time
#Simulate event
entity = self.entities[event["rx"]][event["rxId"]]
service = getattr(entity, event["name"])
service(event["data"], event["tx"], event["txId"]) #Receive
numEvents = numEvents + 1
if self.size > 1:
toRcvCount = self.MPI.alltoallSum()
while toRcvCount > 0:
self.MPI.probe()
remoteEvent = self.MPI.recvAnySize()
heapq.heappush(self.eventQueue,
(remoteEvent["time"], remoteEvent))
toRcvCount -= 1
minLeft = self.infTime
if len(self.eventQueue) > 0: minLeft = self.eventQueue[0][0]
globalMinLeft = self.MPI.allreduce(
minLeft, self.MPI.MIN) #Synchronize minLeft
else:
globalMinLeft = self.infTime
if len(self.eventQueue) > 0:
globalMinLeft = self.eventQueue[0][0]
self.running = False
if self.size > 1:
self.MPI.barrier()
totalEvents = self.MPI.allreduce(numEvents, self.MPI.SUM)
else:
totalEvents = numEvents
if self.rank == 0:
elapsedTime = timeLib.clock() - startTime
print "SIMULATION COMPLETED IN: " + str(elapsedTime) + " SECONDS"
print "SIMULATED EVENTS: " + str(totalEvents)
print "EVENTS PER SECOND: " + str(totalEvents / elapsedTime)
print "==========================================="
sys.stdout.flush()
def barrier(self):
if (mpi.MPI_Barrier(self.comm) != mpi.MPI_SUCCESS):
raise SimianError("Could not Barrier in MPI")
