def cleanupShm (): """ Deletes all shared arrays for this process """ NpShared.DelAll(getShmPrefix()) # above statement don't work for directories and subdirectories os.system("rm -rf /dev/shm/%s"%getShmPrefix())
def __init__(self, MeanResidual, MeanModelImage, PSFServer, DeltaChi2=4., IdSharedMem="", NCPU=6): IdSharedMem += "SmearSM." NpShared.DelAll(IdSharedMem) self.IdSharedMem = IdSharedMem self.NCPU = NCPU self.MeanModelImage = NpShared.ToShared( "%sMeanModelImage" % self.IdSharedMem, MeanModelImage) self.MeanResidual = NpShared.ToShared( "%sMeanResidual" % self.IdSharedMem, MeanResidual) NPixStats = 10000 RandomInd = np.int64(np.random.rand(NPixStats) * (MeanResidual.size)) self.RMS = np.std(np.real(self.MeanResidual.ravel()[RandomInd])) self.FWHMMin = 3. self.PSFServer = PSFServer self.DeltaChi2 = DeltaChi2 self.Var = self.RMS**2 self.NImGauss = 31 self.CubeMeanVariablePSF = NpShared.ToShared( "%sCubeMeanVariablePSF" % self.IdSharedMem, self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF']) self.DicoConvMachine = {} N = self.NImGauss dx, dy = np.mgrid[-(N // 2):N // 2:1j * N, -(N // 2):N // 2:1j * N] ListPixParms = [(int(dx.ravel()[i]), int(dy.ravel()[i])) for i in range(dx.size)] ListPixData = ListPixParms ConvMode = "Matrix" N = self.NImGauss #stop #for #ClassConvMachine(): #def __init__(self,PSF,ListPixParms,ListPixData,ConvMode): d = np.sqrt(dx**2 + dy**2) self.dist = d self.NGauss = 10 GSig = np.linspace(0., 2, self.NGauss) self.GSig = GSig ListGauss = [] One = np.zeros_like(d) One[N // 2, N // 2] = 1. ListGauss.append(One) for sig in GSig[1::]: v = np.exp(-d**2 / (2. * sig**2)) Sv = np.sum(v) v /= Sv ListGauss.append(v) self.ListGauss = ListGauss print("Declare convolution machines", file=log) NJobs = self.PSFServer.NFacets pBAR = ProgressBar(Title=" Declare ") #pBAR.disable() pBAR.render(0, '%4i/%i' % (0, NJobs)) for iFacet in range(self.PSFServer.NFacets): #print iFacet,"/",self.PSFServer.NFacets PSF = self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF'][ iFacet] #[0,0] _, _, NPixPSF, _ = PSF.shape PSF = PSF[:, :, NPixPSF // 2 - N:NPixPSF // 2 + N + 1, NPixPSF // 2 - N:NPixPSF // 2 + N + 1] #print PSF.shape #sig=1 #PSF=(np.exp(-self.dist**2/(2.*sig**2))).reshape(1,1,N,N) self.DicoConvMachine[iFacet] = ClassConvMachine.ClassConvMachine( PSF, ListPixParms, ListPixData, ConvMode) CM = self.DicoConvMachine[iFacet].CM NpShared.ToShared("%sCM_Facet%4.4i" % (self.IdSharedMem, iFacet), CM) #invCM=ModLinAlg.invSVD(np.float64(CM[0,0]))/self.Var #NpShared.ToShared("%sInvCov_Facet%4.4i"%(self.IdSharedMem,iFacet),invCM) NDone = iFacet + 1 intPercent = int(100 * NDone / float(NJobs)) pBAR.render(intPercent, '%4i/%i' % (NDone, NJobs)) PSFMean = np.mean( self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF'], axis=0) self.ConvMachineMeanPSF = ClassConvMachine.ClassConvMachine( PSFMean, ListPixParms, ListPixData, ConvMode) CM = self.ConvMachineMeanPSF.CM invCM = ModLinAlg.invSVD(np.float64(CM[0, 0]), Cut=1e-8) / self.Var NpShared.ToShared("%sInvCov_AllFacet" % (self.IdSharedMem), invCM) self.FindSupport()
def CleanUpSHM(self): NpShared.DelAll(self.IdSharedMem)
group = optparse.OptionGroup(opt, "* SHM") group.add_option( '--ID', help='ID of shared memory to be deleted, default is %default', default=None) opt.add_option_group(group) options, arguments = opt.parse_args() return options if __name__ == "__main__": options = read_options() print >> log, "Clear shared memory" if options.ID is not None: NpShared.DelAll(options.ID) else: NpShared.DelAll() Multiprocessing.cleanupStaleShm() Multiprocessing.cleanupShm() ll = glob.glob("/dev/shm/sem.*") print >> log, "Clear Semaphores" # remove semaphores we don't have access to ll = filter(lambda x: os.access(x, os.W_OK), ll) ListSemaphores = [".".join(l.split(".")[1::]) for l in ll] _pyGridderSmear.pySetSemaphores(ListSemaphores) _pyGridderSmear.pyDeleteSemaphore(ListSemaphores)
def giveDicoInitIndiv(self, ListIslands, ModelImage, DicoDirty, ListDoIsland=None, Parallel=True): NCPU = self.NCPU work_queue = multiprocessing.JoinableQueue() ListIslands = ListIslands #[300:308] DoIsland = True for iIsland in range(len(ListIslands)): if ListDoIsland is not None: DoIsland = ListDoIsland[iIsland] if DoIsland: work_queue.put({"iIsland": iIsland}) result_queue = multiprocessing.JoinableQueue() NJobs = work_queue.qsize() workerlist = [] logger.setSilent(SilentModules) #MyLogger.setLoud(SilentModules) #MyLogger.setLoud("ClassImageDeconvMachineMSMF") print >> log, "Launch MORESANE workers" for ii in range(NCPU): W = WorkerInitMSMF(work_queue, result_queue, self.GD, self.DicoVariablePSF, DicoDirty, self.RefFreq, self.GridFreqs, self.DegridFreqs, self.MainCache, ModelImage, ListIslands, self.IdSharedMem) workerlist.append(W) if Parallel: workerlist[ii].start() timer = ClassTimeIt.ClassTimeIt() pBAR = ProgressBar(Title=" MORESANing islands ") #pBAR.disable() pBAR.render(0, NJobs) iResult = 0 if not Parallel: for ii in range(NCPU): workerlist[ii].run() # just run until all work is completed self.DicoInitIndiv = {} while iResult < NJobs: DicoResult = None if result_queue.qsize() != 0: try: DicoResult = result_queue.get() except: pass if DicoResult == None: time.sleep(0.5) continue if DicoResult["Success"]: iResult += 1 NDone = iResult pBAR.render(NDone, NJobs) iIsland = DicoResult["iIsland"] NameDico = "%sDicoInitIsland_%5.5i" % (self.IdSharedMem, iIsland) Dico = NpShared.SharedToDico(NameDico) self.DicoInitIndiv[iIsland] = copy.deepcopy(Dico) NpShared.DelAll(NameDico) if Parallel: for ii in range(NCPU): workerlist[ii].shutdown() workerlist[ii].terminate() workerlist[ii].join() #MyLogger.setLoud(["pymoresane.main"]) #MyLogger.setLoud(["ClassImageDeconvMachineMSMF","ClassPSFServer","ClassMultiScaleMachine","GiveModelMachine","ClassModelMachineMSMF"]) return self.DicoInitIndiv
def calcDistanceMatrixMinParallel(self, ListIslands, Parallel=True): NIslands = len(ListIslands) self.D = np.zeros((NIslands, NIslands), np.float32) self.dx = np.zeros((NIslands, NIslands), np.int32) self.dy = np.zeros((NIslands, NIslands), np.int32) work_queue = multiprocessing.JoinableQueue() for iIsland in range(NIslands): work_queue.put({"iIsland": (iIsland)}) result_queue = multiprocessing.JoinableQueue() NJobs = work_queue.qsize() workerlist = [] NCPU = self.NCPU ListEdgeIslands = self.giveEdgesIslands(ListIslands) for ii in range(NCPU): W = WorkerDistance(work_queue, result_queue, ListEdgeIslands, self.IdSharedMem) workerlist.append(W) if Parallel: workerlist[ii].start() pBAR = ProgressBar(Title=" Calc. Dist. ") pBAR.render(0, NJobs) iResult = 0 if not Parallel: for ii in range(NCPU): workerlist[ii].run() # just run until all work is completed while iResult < NJobs: DicoResult = None if result_queue.qsize() != 0: try: DicoResult = result_queue.get() except: pass if DicoResult == None: time.sleep(0.5) continue if DicoResult["Success"]: iResult += 1 NDone = iResult pBAR.render(NDone, NJobs) iIsland = DicoResult["iIsland"] Result = NpShared.GiveArray("%sDistances_%6.6i" % (self.IdSharedMem, iIsland)) self.dx[iIsland] = Result[0] self.dy[iIsland] = Result[1] self.D[iIsland] = Result[2] NpShared.DelAll("%sDistances_%6.6i" % (self.IdSharedMem, iIsland)) if Parallel: for ii in range(NCPU): workerlist[ii].shutdown() workerlist[ii].terminate() workerlist[ii].join()