def reqService(self, offset, eventName, data, rx=None, rxId=None): #Purpose: Send an event if Simian is running. engine = self.engine #Get the engine for this entity if rx != None and offset < engine.minDelay: if not engine.running: raise SimianError("Sending event when Simian is idle!") #If sending to self, then do not check against min-delay raise SimianError(self.name + "[" + str(self.num) + "]" + " attempted to send with too little delay") time = engine.now + offset if time > engine.endTime: #No need to send this event return if rx == None: rx = self.name if rxId == None: rxId = self.num e = { "tx": self.name, #String "txId": self.num, #Number "rx": rx, #String "rxId": rxId, #Number "name": eventName, #String "data": data, #Object "time": time, #Number } recvRank = engine.getOffsetRank(rx, rxId) if recvRank == engine.rank: #Send to self heapq.heappush(engine.eventQueue, (time, e)) else: if time < engine.minSent: engine.minSent = time #engine.MPI.isend(e, recvRank) #Send to others (Problem with MPI buffers getting filled too fast) engine.MPI.send(e, recvRank) #Send to others
def __init__(self, engine): self.engine = weakref.ref(engine) self.event_queue = [] self.current_gpu = None self.current_stream = (0, 0) self.process_list = [] #Initialize CUDA try: import pycuda.driver as cuda from pycuda.compiler import SourceModule self.driver = cuda self.driver.init() except: raise SimianError( "Please install pyCuda before using Simian for CUDA based simulation" ) self.num_devices = cuda.Device.count() print "Found ", self.num_devices, " cuda devices on this node" if self.num_devices == 0: raise SimianError("No cuda capable device found on this node") self.device_queues = [] self.contexts = [] for gpu in range(self.num_devices): self.contexts.append(cuda.Device(gpu).make_context()) streams = [] num_mp = cuda.Device(gpu).get_attribute( cuda.device_attribute.MULTIPROCESSOR_COUNT) for mp in range(num_mp): streams.append(cuda.Stream()) self.device_queues.append(streams) print "Device ", gpu, " contains ", num_mp, " multiprocessors" self.contexts[0].push() self.current_gpu = 0
def reqService(self, offset, eventName, data, rx=None, rxId=None): #Purpose: Send an event if Simian is running. engine = self.engine #Get the engine for this entity if rx != None and offset < engine.minDelay: if not engine.running: raise SimianError("Sending event when Simian is idle!") #If sending to self, then do not check against min-delay raise SimianError(self.name + "[" + str(self.num) + "]" + " attempted to send with too little delay") time = engine.now + offset if time > engine.endTime: #No need to send this event return if rx == None: rx = self.name if rxId == None: rxId = self.num e = { "tx": self.name, #String "txId": self.num, #Number "rx": rx, #String "rxId": rxId, #Number "name": eventName, #String "data": data, #Object "time": time, #Number } # this is a particular mechanism added by Jason Liu for # allowing different mappings from LPs to ranks recvRank = engine.getOffsetRank(rx, rxId) if recvRank == engine.rank: #Send to self heapq.heappush(engine.eventQueue, (time, e)) else: engine.MPI.sendAndCount(e, recvRank)
def createProcess(self, name, fun, kind=None): #Creates a named process global Process if not Process: from process import Process if name == "*": raise SimianError("Reserved name to represent all child processes: " + name) proc = Process(name, fun, self, None) #No parent means, entity is parent if not proc: raise SimianError("Could not create a valid process named: " + name) self._procList[name] = proc if kind != None: self.categorizeProcess(kind, name) #Categorize
def addEntity(self, name, entityClass, num, *args): #Purpose: Add an entity to the entity-list if Simian is idle #This function takes a pointer to a class from which the entities can #be constructed, a name, and a number for the instance. if self.running: raise SimianError("Adding entity when Simian is running!") if not (name in self.entities): self.entities[name] = {} #To hold entities of this "name" entity = self.entities[name] self.baseRanks[name] = self.getBaseRank(name) #Register base-ranks computedRank = self.getOffsetRank(name, num) if computedRank == self.rank: #This entity resides on this engine #Output log file for this Entity self.out.write(name + "[" + str(num) + "]: Running on rank " + str(computedRank) + "\n") entity[num] = entityClass( { "name": name, "out": self.out, "engine": self, "num": num, }, *args) #Entity is instantiated
def optimisticRollback(self, time, LP): backup = False numRolls = 0 if time < self.optimistic_GVT: raise SimianError( "rollback before GVT!!!! GVT: %s , Event Queue Dump : %s" % (self.optimistic_GVT, self.eventQueue)) if len(LP.processedEvents): while LP.processedEvents[len(LP.processedEvents) - 1][0]["time"] >= time: if self.statistics: numRolls += 1 (event, state) = LP.processedEvents.pop(-1) heap.push(self.eventQueue, (event["time"], dict(event))) backup = dict(state) LP.recoverRandoms(state) LP.recoverAntimessages(state, time) self.optimisticNumEventsRolledBack += 1 if not len(LP.processedEvents): break if self.statistics: self.rollbackLength.append(numRolls) if backup: LP.recoverState(backup) LP.VT = time
def iprobe(self, src=None, tag=None): #Non-blocking asynch src = src or mpi.MPI_ANY_SOURCE tag = tag or mpi.MPI_ANY_TAG if mpi.MPI_Iprobe(src, tag, self.comm, C.byref(self.itemp), C.byref(self.status)) == mpi.MPI_SUCCESS: return (self.itemp.value != 0) raise SimianError("Could not Iprobe in MPI")
def wakeProcess(self, name, *args): #Wake a named process with arguments if not (name in self._procList): raise SimianError("Attempted to wake a non existant process: " + name) else: #If existing and not been killed asynchronously proc = self._procList[name] proc.wake(*args)
def __init__(self, libName): global mpi mpi = loadMPI(libName) if mpi.MPI_Init(None, None) != mpi.MPI_SUCCESS: raise SimianError("Could not initialize MPI") self.CBUF_LEN = 32 * 1024 #32kB self.comm = mpi.MPI_COMM_WORLD self.BYTE = mpi.MPI_BYTE self.DOUBLE = mpi.MPI_DOUBLE self.LONG = mpi.MPI_LONG self.MIN = mpi.MPI_MIN self.SUM = mpi.MPI_SUM self.request = mpi.MPI_Request() self.status = mpi.MPI_Status() self.itemp = C.c_int() self.dtemp0 = C.c_double() self.dtemp1 = C.c_double() self.ctemp = C.create_string_buffer(self.CBUF_LEN) #Preallocate self.numRanks = self.size() self.sndCounts = (C.c_long * self.numRanks)() for i in range(len(self.sndCounts)): self.sndCounts[i] = 0 self.rcvCounts = (C.c_long * self.numRanks)()
def sendAndCount(self, x, dst, tag=None): #Blocking m = Pack(x) tag = tag or len(m) #Set to message length if None if mpi.MPI_Send(m, len(m), self.BYTE, dst, tag, self.comm) != mpi.MPI_SUCCESS: raise SimianError("Could not Send in MPI") self.sndCounts[dst] += 1
def killProcessKind(self, kind): #Kills all @kind processes on entity if not (kind in self._category): raise SimianError("killProcessKind: No category of processes on this entity called " + str(kind)) else: nameSet = self._category[kind] for name,_ in nameSet.items(): #So we can delete while iterating proc = self._procList[name] proc.kill() #Kill itself and all subprocesses
def __init__(self, simName, startTime, endTime, minDelay=1, useMPI=False, mpiLibName=defaultMpichLibName): self.Entity = Entity #Include in the top Simian namespace self.name = simName self.startTime = startTime self.endTime = endTime self.minDelay = minDelay self.now = startTime #If simulation is running self.running = False #Stores the entities available on this LP self.entities = {} #Events are stored in a priority-queue or heap, in increasing #order of time field. Heap top can be accessed using self.eventQueue[0] #event = {time, name, data, tx, txId, rx, rxId}. self.eventQueue = [] #Stores the minimum time of any event sent by this process, #which is used in the global reduce to ensure global time is set to #the correct minimum. self.infTime = endTime + 2*minDelay self.minSent = self.infTime #[[Base rank is an integer hash of entity's name]] self.baseRanks = {} #Make things work correctly with and without MPI if useMPI: #Initialize MPI try: global MPI from MPILib import MPI self.useMPI = True self.MPI = MPI(mpiLibName) self.rank = self.MPI.rank() self.size = self.MPI.size() except: raise SimianError("Please ensure libmpich is available to ctypes before using Simian for MPI based simulations.\nTry passing absolute path to libmpich.[dylib/so/dll] to Simian.") else: self.useMPI = False self.MPI = None self.rank = 0 self.size = 1 #One output file per rank self.out = open(self.name + "." + str(self.rank) + ".out", "w") #Write some header information for each output file self.out.write("===========================================\n") self.out.write("----------SIMIAN-PIE PDES ENGINE-----------\n") self.out.write("===========================================\n") if self.useMPI: self.out.write("MPI: ON\n\n") else: self.out.write("MPI: OFF\n\n")
def categorizeProcess(self, kind, name): #Check for existing process and then categorize if name in self._procList: proc = self._procList[name] #Categorize both ways for easy lookup proc._kindSet[kind] = True #Indicate to proc that it is of this kind to its entity #Indicate to entity that proc is of this kind if not kind in self._category: self._category[kind] = {name: True} #New kind else: self._category[kind][name] = True #Existing kind else: raise SimianError("categorize: Expects a proper child to categorize")
def alltoallSum(self): if (mpi.MPI_Alltoall(self.sndCounts, 1, self.LONG, self.rcvCounts, 1, self.LONG, self.comm) != mpi.MPI_SUCCESS): raise SimianError("Could not AllToAll in MPI") toRcv = 0 for i in range(self.numRanks): toRcv = toRcv + self.rcvCounts[i] self.sndCounts[i] = 0 return toRcv
def startProcess(self, name, *args): #Starts a named process if name in self._procList: proc = self._procList[name] if not proc.started: proc.started = True #When starting, pass process instance as first arg, which can be accessed inside the "fun" return proc.wake(proc, *args) else: raise SimianError("Starting an already started process: " + proc.name)
def unCategorizeProcess(self, kind, name): #Check for existing process and then unCategorize if name in self._procList: proc = self._procList[name] #unCategorize both ways for easy lookup proc._kindSet.pop(kind) #Indicate to proc that it is not of this kind to its entity #Indicate to entity that proc is not of this kind if kind in self._category: self._category[kind].pop(name) #Existing kind deleted else: raise SimianError("unCategorize: Expects a proper child to un-categorize")
def allreduce(self, partial, op): self.dtemp0 = C.c_double(partial) if (mpi.MPI_Allreduce( C.byref(self.dtemp0), C.byref(self.dtemp1), 1, self.DOUBLE, #Single double operand op, self.comm) != mpi.MPI_SUCCESS): raise SimianError("Could not Allreduce in MPI") return self.dtemp1.value
def wake(thisProcess, *args): #Arguments "*args" to __call => function-body #Arguments "*args" to wake => LHS of hibernate co = thisProcess.co if co != None and not co.dead: thisProcess.main = greenlet.getcurrent() thisProcess.suspended = False return co.switch(*args) else: raise SimianError("Attempted to wake a process: " + thisProcess.name + " failed")
def recv(self, maxSize, src=None, tag=None): #Blocking src = src or mpi.MPI_ANY_SOURCE tag = tag or mpi.MPI_ANY_TAG m = self.ctemp if maxSize > self.CBUF_LEN: #Temporary buffer is too small m = C.create_string_buffer(maxSize) if mpi.MPI_Recv(m, maxSize, self.BYTE, src, tag, self.comm, C.byref(self.status)) == mpi.MPI_SUCCESS: #return Unpack(m.raw) return Unpack(m[:maxSize]) raise SimianError("Could not Recv in MPI")
def sleep(thisProcess, x, *args): #Processes which are to implicitly wake at set timeouts #All return values are passed to __call/wake if (not isinstance(x, (int, long, float))) or (x < 0): raise SimianError("Sleep not given non-negative number argument!" + thisProcess.name) entity = thisProcess.entity #Schedule a local alarm event after x timesteps to wakeup entity.engine.schedService(entity.engine.now + x, "_wakeProcess", thisProcess.name, entity.name, entity.num) thisProcess.suspended = True return thisProcess.main.switch(*args)
def __init__(self, simName, startTime, endTime, minDelay=1, useMPI=False): self.Entity = Entity #Include in the top Simian namespace self.name = simName self.startTime = startTime self.endTime = endTime self.minDelay = minDelay self.now = startTime #If simulation is running self.running = False #Stores the entities available on this LP self.entities = {} #Events are stored in a priority-queue or heap, in increasing #order of time field. Heap top can be accessed using self.eventQueue[0] #event = {time, name, data, tx, txId, rx, rxId}. self.eventQueue = [] #Stores the minimum time of any event sent by this process, #which is used in the global reduce to ensure global time is set to #the correct minimum. self.infTime = endTime + 2 * minDelay self.minSent = self.infTime #[[Base rank is an integer hash of entity's name]] self.baseRanks = {} #Make things work correctly with and without MPI if useMPI: #Initialize MPI global MPI try: from mpi4py import MPI self.useMPI = True self.comm = MPI.COMM_WORLD self.rank = self.comm.Get_rank() self.size = self.comm.Get_size() except: raise SimianError( "Please install mpi4py before using Simian for MPI based simulations" ) else: self.useMPI = False self.comm = None self.rank = 0 self.size = 1 #One output file per rank self.out = open(self.name + "." + str(self.rank) + ".out", "w")
def spawn(thisProcess, name, fun, kind=None): #Create a new named processes as child or @kind entity = thisProcess.entity if name in entity._procList: raise SimianError("spawn: Process by name '" + name + "' already exists in entity " + entity.name + "[" + str(entity.num) + "]") entity.createProcess(name, fun, kind) #Creates a named process of kind type #Make this a child of thisProcess #NOTE: This is the difference between process.spawn and entity.createProcess entity._procList[name].parent = thisProcess thisProcess._childList[name] = True
#Author: Nandakishore Santhi #Date: 23 November, 2014 #Copyright: Open source, must acknowledge original author #Purpose: PDES Engine in Python, mirroring a subset of the Simian JIT-PDES # Named process class from utils import SimianError from greenlet import greenlet try: from greenlet import greenlet except: raise SimianError( "Please install greenlets before using SimianPie to run simulations") #Making this pythonic - this is a base class that all derived Processes will inherit from class Process(object): #ent.createProcess/proc.hibernate <=> proc.wake/ent.wakeProcess #proc.sleep/proc.compute <=> ent.wakeProcess def __init__(self, name, fun, thisEntity, thisParent): self.name = name self.co = greenlet(run=fun) self.started = False self.suspended = False self.main = greenlet.getcurrent( ) #To hold the main process for to/from context-switching within sleep/wake/hibernate self.entity = thisEntity self.parent = thisParent #Parent is None if created by entity self._kindSet = {} #Set of kinds that it belongs to on its entity self._childList = {}
def finalize(self): if mpi.MPI_Finalize() == mpi.MPI_SUCCESS: return False raise SimianError("Could not finalize MPI")
def reqService(self, offset, eventName, data, rx=None, rxId=None): #Purpose: Send an event if Simian is running. engine = self.engine #Get the engine for this entity if not engine.optimistic: if rx != None and offset < engine.minDelay: if not engine.running: raise SimianError("Sending event when Simian is idle!") #If sending to self, then do not check against min-delay raise SimianError(self.name + "[" + str(self.num) + "]" + " attempted to send with too little delay") time = engine.now + offset if engine.optimistic: if offset == 0 : offset = 0.000000001 # add 1 nano unit #event can not create an event for the same time ( this resolution is debatable) time = self.VT + offset #print time if time > engine.endTime: return if engine.optimistic_color == 'white': color = 'white' else: color = 'red' if rx == None: rx = self.name if rxId == None: rxId = self.num e = { "tx": self.name, #String "txId": self.num, #Number "rx": rx, #String "rxId": rxId, #Number "name": eventName, #String "data": data, #Object "time": time, #Number "antimessage" : False, "GVT" : False, "color" : color, } if engine.optimistic: ae = { "tx": self.name, #String "txId": self.num, #Number "rx": rx, #String "rxId": rxId, #Number "name": eventName, #String "data": data, #Object "time": time, #Number "antimessage" : True, "GVT" : False, "color" : color, } self.sentEvents.append(ae) # this is a particular mechanism added by Jason Liu for # allowing different mappings from LPs to ranks if engine.partfct: recvRank = engine.partfct(rx, rxId, engine.size, engine.partarg) else: recvRank = engine.getOffsetRank(rx, rxId) if engine.statistics: try: self.statisticsWhoSendLP[(rx,rxId)] += 1 self.statisticsWhoSendRank[recvRank] += 1 except KeyError: self.statisticsWhoSendLP[(rx,rxId)] = 1 self.statisticsWhoSendRank[recvRank] = 1 if recvRank == engine.rank: #Send to self engine.heap.push(engine.eventQueue, (time, e)) else: if engine.optimistic: if engine.optimistic_color == 'white': engine.optimistic_white += 1 # GVT approx algo else: engine.optimistic_t_min = min(engine.optimistic_t_min, time) engine.MPI.send(e, recvRank) else: engine.MPI.sendAndCount(e, recvRank)
def rank(self): if mpi.MPI_Comm_rank(self.comm, C.byref(self.itemp)) == mpi.MPI_SUCCESS: return self.itemp.value raise SimianError("Could not get rank in MPI")
def size(self): #size = (C.c_int * 1)() if mpi.MPI_Comm_size(self.comm, C.byref(self.itemp)) == mpi.MPI_SUCCESS: return self.itemp.value raise SimianError("Could not get size in MPI")
def isend(self, x, dst, tag=None): #Non-Blocking m = Pack(x) tag = tag or len(m) #Set to message length if None if mpi.MPI_Isend(m, len(m), self.BYTE, dst, tag, self.comm, C.byref(self.request)) != mpi.MPI_SUCCESS: raise SimianError("Could not Isend in MPI")
def run(self): #Run the simulation startTime = timeLib.clock() if self.rank == 0: print("===========================================") print("----------SIMIAN-PIE PDES ENGINE-----------") print("===========================================") if self.useMPI: print("MPI: ON") else: print("MPI: OFF") numEvents = 0 self.running = True globalMinLeft = self.startTime while globalMinLeft <= self.endTime: epoch = globalMinLeft + self.minDelay self.minSent = self.infTime while len(self.eventQueue) > 0 and self.eventQueue[0][0] < epoch: (time, event) = heapq.heappop(self.eventQueue) #Next event if self.now > time: raise SimianError( "Out of order event: now=%f, evt=%f" % self.now, time) self.now = time #Advance time #Simulate event entity = self.entities[event["rx"]][event["rxId"]] service = getattr(entity, event["name"]) service(event["data"], event["tx"], event["txId"]) #Receive numEvents = numEvents + 1 if self.size > 1: toRcvCount = self.MPI.alltoallSum() while toRcvCount > 0: self.MPI.probe() remoteEvent = self.MPI.recvAnySize() heapq.heappush(self.eventQueue, (remoteEvent["time"], remoteEvent)) toRcvCount -= 1 minLeft = self.infTime if len(self.eventQueue) > 0: minLeft = self.eventQueue[0][0] globalMinLeft = self.MPI.allreduce( minLeft, self.MPI.MIN) #Synchronize minLeft else: globalMinLeft = self.infTime if len(self.eventQueue) > 0: globalMinLeft = self.eventQueue[0][0] self.running = False if self.size > 1: self.MPI.barrier() totalEvents = self.MPI.allreduce(numEvents, self.MPI.SUM) else: totalEvents = numEvents if self.rank == 0: elapsedTime = timeLib.clock() - startTime print "SIMULATION COMPLETED IN: " + str(elapsedTime) + " SECONDS" print "SIMULATED EVENTS: " + str(totalEvents) print "EVENTS PER SECOND: " + str(totalEvents / elapsedTime) print "===========================================" sys.stdout.flush()
def barrier(self): if (mpi.MPI_Barrier(self.comm) != mpi.MPI_SUCCESS): raise SimianError("Could not Barrier in MPI")