def getCircHeaderSize(): align = getAlign() return ( (8 + 4 + 4 + 4 + 2 + 1 + 1 + 4 + 2 * 4 + 4 + 8 + 4 + 4 + 4 + 4 + 4 + utils.pthread_sizeof_mutexcond()[0] + utils.pthread_sizeof_mutexcond()[1] + align - 1) / align ) * align #header contains buffer size (int64), last written to (int32), freq (int32), nstore (int32), forcewriteall(int8),ndim (int8), dtype (int8), forcewrite (int8), shape (6*int32) pid (int32), circhdrsize (int32) circsignal (int8), 3 spare (int24), mutex size(int32), cond size(int32), mutex, cond
def __init__(self, shmname, owner=0, dims=None, dtype=None, nstore=None, raw=0, dirname="/dev/shm"): self.align = getAlign() self.shmname = shmname self.dirname = dirname self.owner = owner self.freqcnt = 0 self.framecnt = 0 self.dtypeSave = None self.ndimSave = None self.shapeArrSave = None self.nstoreSave = None self.raw = raw self.lastReceived = -1 #last frame received... self.lastReceivedFrame = -1 if owner == 1: self.hdrsize = getCircHeaderSize() if dims == None or dtype == None or nstore == None: raise Exception( "Owner of circular buffer must specify dims, dtype and nstore" ) #self.timesize=((8*nstore+self.align-1)/self.align)*self.align#size of the timestamp (float 64 for each entry) #self.frameNoSize=((4*nstore+self.align-1)/self.align)*self.align#size of the frameno (int32 for each entry) #self.dtype=dtype #self.dims=dims self.datasize = reduce(lambda x, y: x * y, dims) self.frameSize = ((self.datasize * numpy.zeros( (1, ), dtype).itemsize + 32 + self.align - 1) / self.align) * self.align self.size = self.hdrsize + self.frameSize * nstore #self.frameNoSize+self.timesize+nstore*self.datasize*numpy.zeros((1,),dtype).itemsize mode = "w+" else: #first open the header to see what size it is, then open the full array. #buf=utils.open((self.hdrsize,),"b",shmname,owner) mode = "r+" self.size = os.stat(self.dirname + shmname).st_size if self.size == 0: time.sleep(0.1) self.size = os.stat(self.dirname + shmname).st_size if self.size == 0: raise Exception("Zero size array %s" % shmname) #utils.unmap(buf) #self.buffer=utils.open((self.size,),"b",shmname,owner) self.buffer = numpy.memmap(self.dirname + shmname, "b", mode, shape=(self.size, )) #now get the hdrsize self.circsignal = self.buffer[56:57] if owner: self.buffer[:] = 0 self.buffer[60:68].view( numpy.int32)[:] = utils.pthread_sizeof_mutexcond() self.buffer[52:56].view(numpy.int32)[0] = self.hdrsize self.buffer[48:52].view(numpy.int32)[0] = os.getpid() else: #if not the owner, wait until the buffer has been initialised properly... one of the last things to be done is that the writer will set ndim... i = 0 while i < 1000 and int(self.buffer[21:22]) == 0: i += 1 time.sleep(0.001) if i == 1000: print "ERROR - buffer ndim not initialised - buffer %s, ndim=%d" % ( shmname, int(self.buffer[21:22])) raise Exception( "ERROR - buffer ndim not initialised - buffer %s, ndim=%d" % (shmname, int(self.buffer[21:22]))) self.hdrsize = int(self.buffer[52:56].view(numpy.int32)[0]) msize = int(self.buffer[60:64].view(numpy.int32)[0]) self.futex = self.buffer[64:64 + msize] self.ownerPid = self.buffer[48:52].view(numpy.int32) #self.semid=utils.newsemid("/dev/shm"+shmname,98,1,1,owner) #get the header arrays self.bufsize = self.buffer[0:8].view(numpy.int64) self.lastWritten = self.buffer[8:12].view("i") self.freq = self.buffer[12:16].view("i") self.nstore = self.buffer[16:20].view("i") self.forcewriteall = self.buffer[20:21] self.ndim = self.buffer[21:22] #.view("h") self.dtype = self.buffer[22:23].view("c") self.forcewrite = self.buffer[23:24] self.shapeArr = self.buffer[24:48].view("i") if owner: self.bufsize[0] = self.size #this should never change now... self.lastWritten[0] = -1 self.freq[0] = 0 self.nstore[0] = nstore self.ndim[0] = len(dims) self.dtype[0] = dtype self.forcewrite[0] = 0 self.circsignal[0] = 0 self.shapeArr[:] = -1 self.shapeArr[:self.ndim[0]] = dims utils.darc_futex_init(self.futex) #utils.initSemaphore(self.semid,0,1)#initialise so that something can block on it waiting for a zero. else: #If is possible that the array isn't initialised yet - which might manifest itself with ndim==0. So wait to see if this is the case. n = 0 while self.ndim[0] == 0 and n < 100: n += 1 time.sleep(0.01) self.makeDataArrays()
def __init__(self, shmname, create=0, size=64 * 1024 * 1024, nhdr=128): self.shmname = shmname self.unlinkOnDel = 0 self.create = 0 #self.nhdr=nhdr if shmname != None: mode = "w+" if create: if os.path.exists("/dev/shm" + shmname): create = 0 else: #create the shm and initialise it. self.arr = numpy.memmap("/dev/shm" + shmname, "c", mode, shape=(size, )) self.arr.view("b")[:] = 0 #write the hdrsize, number of entries etc. self.arr[4:8].view(numpy.int32)[0] = nhdr msize, csize = utils.pthread_sizeof_mutexcond() self.arr[12:16].view(numpy.int32)[0] = msize self.arr[16:20].view(numpy.int32)[0] = csize utils.pthread_cond_init( self.arr[20 + msize:20 + msize + csize], 1) utils.pthread_mutex_init(self.arr[20:20 + msize], 1) hdrsize = 4 + 4 + 4 + 4 + 4 + msize + csize #make it nicely aligned. hdrsize += (16 - ( (self.arr.__array_interface__["data"][0] + hdrsize) & 0xf)) % 16 self.arr[:4].view(numpy.int32)[0] = hdrsize if create == 0: #get the buffer size... mode = "r+" size = os.stat("/dev/shm" + shmname).st_size #[stat.ST_SIZE] print "Opening buffer of size %d bytes" % size self.arr = numpy.memmap("/dev/shm" + shmname, "c", mode, shape=(size, )) #buffer has a info header with hdrsize(4),nhdr(4),flags(4),mutexsize(4),condsize(4),mutex(N),cond(N) hdrsize = int(self.arr[:4].view(numpy.int32)[0]) while hdrsize == 0: print "Waiting to get hdrsize of parambuf" hdrsize = int(self.arr[:4].view(numpy.int32)[0]) time.sleep(0.1) #pointer to the main part of the array. print "Got buffer header size of %d" % hdrsize self.buffer = self.arr[hdrsize:] #get the number of entries. self.nhdr = self.arr[4:8].view(numpy.int32) while self.nhdr[0] == 0: print "Waiting to get number of header entries..." #self.nhdr[0]=int(self.arr[4:8].view(numpy.int32)[0]) time.sleep(0.1) print "Got max number of entries (nhdr) %d" % self.nhdr[0] msize = int(self.arr[12:16].view(numpy.int32)[0]) self.flags = self.arr[8:12].view(numpy.int32) csize = int(self.arr[16:20].view(numpy.int32)[0]) while msize == 0 or csize == 0: msize = int(self.arr[12:16].view(numpy.int32)[0]) csize = int(self.arr[16:20].view(numpy.int32)[0]) print "Waiting to get mutex/cond size" time.sleep(0.1) #get the memory occupied by the condition variable and mutex. self.condmutex = self.arr[20:20 + msize] self.cond = self.arr[20 + msize:20 + msize + csize] #self.semid=utils.newsemid("/dev/shm"+shmname,98,1,1,owner) else: hdrsize = 20 self.arr = numpy.zeros((max(size, hdrsize), ), "c") self.arr[:4].view(numpy.int32)[0] = hdrsize self.arr[4:8].view(numpy.int32)[0] = nhdr self.nhdr = self.arr[4:8].view(numpy.int32) self.flags = self.arr[8:12].view(numpy.int32) self.buffer = self.arr[hdrsize:] self.condmutex = None self.cond = None self.arrhdrsize = self.arr[:4].view(numpy.int32) self.bufferSize = size self.align = 64 #align data to this byte boundaries... self.hdrsize = 57 #the size of an individual entry in the header. See setNhdr() for why... self.setNhdr(self.nhdr[0]) self.tmpname = numpy.zeros((16, ), "c") self.create = create