def InitFromCatalog(self): FileCoords = self.FileCoords dtype = [('Name', 'S200'), ("ra", np.float64), ("dec", np.float64), ('Type', 'S200')] #FileCoords="Transient_LOTTS.csv" self.PosArray = np.load(FileCoords) self.PosArray = self.PosArray.view(np.recarray) self.NDirSelected = self.PosArray.shape[0] self.NDir = self.PosArray.shape[0] self.DicoDATA = shared_dict.create("DATA") self.DicoGrids = shared_dict.create("Grids") self.DicoGrids["GridSTD"] = np.zeros((self.na, self.NTimes), np.float64) self.DoJonesCorr_kMS = False self.DicoJones = None if self.SolsName: self.DoJonesCorr_kMS = True self.DicoJones_kMS = shared_dict.create("DicoJones_kMS") self.DoJonesCorr_Beam = False if self.BeamModel: self.DoJonesCorr_Beam = True self.DicoJones_Beam = shared_dict.create("DicoJones_Beam") APP.registerJobHandlers(self) AsyncProcessPool.init(ncpu=self.NCPU, affinity=0) APP.startWorkers()
def Interpol(self): APP.startWorkers() if "TEC" in self.DicoFile0["SmoothMode"]: TECArray = NpShared.ToShared("%sTECArray" % IdSharedMem, self.DicoFile0["SolsTEC"]) CPhaseArray = NpShared.ToShared("%sCPhaseArray" % IdSharedMem, self.DicoFile0["SolsCPhase"]) nt, nd, na = TECArray.shape iJob = 0 for it in range(nt): APP.runJob("InterpolTECTime_%d" % iJob, self.InterpolTECTime, args=(it, )) #,serial=True) iJob += 1 workers_res = APP.awaitJobResults("InterpolTECTime*", progress="Interpol TEC") iJob = 0 for it in range(nt): APP.runJob("InterpolAmpTime_%d" % iJob, self.InterpolAmpTime, args=(it, )) #,serial=True) iJob += 1 workers_res = APP.awaitJobResults("InterpolAmpTime*", progress="Interpol Amp") # APP.terminate() APP.shutdown() Multiprocessing.cleanupShm()
def ConvolveGaussianParallel(shareddict, field_in, field_out, CellSizeRad=None, GaussPars=[(0., 0., 0.)], Normalise=False): """Convolves images held in a dict, using APP. """ Ain0 = shareddict[field_in] nch, npol, _, _ = Ain0.shape Aout = shareddict[field_out] # single channel? Handle serially if nch == 1: return ConvolveGaussian(shareddict, field_in, field_out, 0, CellSizeRad, GaussPars[0], None, Normalise) jobid = "convolve:%s:%s:" % (field_in, field_out) for ch in range(nch): sd_rw = shareddict.readwrite() APP.runJob(jobid + str(ch), _convolveSingleGaussianFFTW_noret, args=(sd_rw, field_in, field_out, ch, CellSizeRad, GaussPars[ch], None, Normalise)) APP.awaitJobResults(jobid + "*") #, progress="Convolving") return Aout
def InitFromCatalog(self): FileCoords = self.FileCoords dtype = [('Name', 'S200'), ("ra", np.float64), ("dec", np.float64), ('Type', 'S200')] #FileCoords="Transient_LOTTS.csv" self.PosArray = np.load(FileCoords) self.PosArray = self.PosArray.view(np.recarray) self.NDirSelected = self.PosArray.shape[0] self.NDir = self.PosArray.shape[0] self.DicoDATA = shared_dict.create("DATA") self.DicoGrids = shared_dict.create("Grids") dChan = np.min([self.StepFreq, self.NChan]) self.DicoGrids["DomainEdges_Freq"] = np.int64( np.linspace(0, self.NChan, int(self.NChan / dChan) + 1)) #self.DicoGrids["DomainEdges_Time"] = np.int64(np.linspace(0,self.NTimes-1,int(self.NTimes/self.StepTime)+1)) DT = self.times.max() - self.times.min() DTSol = np.min([self.StepTime * self.dt, DT]) self.DicoGrids["DomainEdges_Time"] = np.linspace( self.times.min() - 1e-6, self.times.max() + 1e-6, int(DT / DTSol) + 1) self.DicoGrids["GridC2"] = np.zeros( (self.DicoGrids["DomainEdges_Time"].size - 1, self.DicoGrids["DomainEdges_Freq"].size - 1, self.na, self.na), np.complex128) self.DicoGrids["GridC"] = np.zeros( (self.DicoGrids["DomainEdges_Time"].size - 1, self.DicoGrids["DomainEdges_Freq"].size - 1, self.na, self.na), np.complex128) log.print(" DomainEdges_Freq: %s" % (str(self.DicoGrids["DomainEdges_Freq"]))) log.print(" DomainEdges_Time: %s" % (str(self.DicoGrids["DomainEdges_Time"]))) self.DoJonesCorr_kMS = False self.DicoJones = None if self.SolsName: self.DoJonesCorr_kMS = True self.DicoJones_kMS = shared_dict.create("DicoJones_kMS") self.DoJonesCorr_Beam = False if self.BeamModel: self.DoJonesCorr_Beam = True self.DicoJones_Beam = shared_dict.create("DicoJones_Beam") APP.registerJobHandlers(self) AsyncProcessPool.init(ncpu=self.NCPU, affinity=0) APP.startWorkers()
def collectSmearMapping(self, DATA, field): APP.awaitJobCounter(self._job_counter, progress="Mapping %s" % self.name, total=self._nbl, timeout=1) self._outdict.reload() #self._outdict.save("bda.dict") blockdict = self._outdict["blocks"] sizedict = self._outdict["sizes"] # process worker results # for each map (each array returned from worker), BlockSizes[MapName] will # contain a list of BlocksSizesBL entries returned from that worker NTotBlocks = 0 NTotRows = 0 for key in sizedict.iterkeys(): bsz = sizedict[key] NTotBlocks += len(bsz) NTotRows += bsz.sum() mapping = DATA.addSharedArray(field, (2 + NTotBlocks + NTotRows, ), np.int32) mapping[0] = NTotBlocks mapping[1] = NTotBlocks >> 32 FinalMappingSizes = mapping[2:2 + NTotBlocks] FinalMapping = mapping[2 + NTotBlocks:] iii = 0 jjj = 0 # now go through each per-baseline mapping, sorted by baseline for key in sorted(sizedict.iterkeys()): BlocksSizesBL = sizedict[key] BlocksRowsListBL = blockdict[key] FinalMapping[iii:iii + BlocksRowsListBL.size] = BlocksRowsListBL[:] iii += BlocksRowsListBL.size # print "IdWorker,AppendId",IdWorker,AppendId,BlocksSizesBL # MM=np.concatenate((MM,BlocksSizesBL)) FinalMappingSizes[jjj:jjj + BlocksSizesBL.size] = BlocksSizesBL[:] jjj += BlocksSizesBL.size NVis = np.where(DATA["A0"] != DATA["A1"])[0].size * DATA["freqs"].size #print>>log, " Number of blocks: %i"%NTotBlocks #print>>log, " Number of 4-Visibilities: %i"%NVis fact = (100. * (NVis - NTotBlocks) / float(NVis)) # clear temp shared arrays/dicts del sizedict del blockdict self._outdict.delete() return mapping, fact
def giveWeigthParallel(self): self.DATA = shared_dict.attach(self.DictName) self.na = self.DATA["na"] for A0 in range(0, self.na): for A1 in range(A0 + 1, self.na): APP.runJob("giveWeigthChunk:%d:%d" % (A0, A1), self.giveWeigthChunk, args=(A0, A1, self.DATA.readwrite())) #,serial=True) APP.awaitJobResults("giveWeigthChunk:*", progress="CalcWeight")
def __init__(self, GD, NFreqBands, RefFreq, MainCache=None, IdSharedMem=""): self.GD = GD self.InitMachine = ClassInitSSDModel(GD, NFreqBands, RefFreq, MainCache, IdSharedMem) self.NCPU = (self.GD["Parallel"]["NCPU"] or psutil.cpu_count()) APP.registerJobHandlers(self)
def giveWeigthParallel(self): self.DATA = shared_dict.attach(self.DictName) nrow, nch, npol = self.DATA["data"].shape self.na = self.DATA["na"] ntu = self.DATA["times_unique"].size self.DATA["Wa"] = np.zeros((self.na, ntu), np.float32) for A0 in range(self.na): APP.runJob("giveWeigthAnt:%d" % (A0), self.giveWeigthAnt, args=(A0, )) #,serial=True) APP.awaitJobResults("giveWeigthAnt:*", progress="CalcWeight")
def StackAll(self): while self.iCurrentMS<self.nMS: if self.LoadNextMS()=="NotRead": continue print("Making dynamic spectra...", file=log) # for iTime in range(self.NTimes): # APP.runJob("Stack_SingleTime:%d"%(iTime), # self.Stack_SingleTime, # args=(iTime,))#,serial=True) # APP.awaitJobResults("Stack_SingleTime:*", progress="Append MS %i"%self.DicoDATA["iMS"]) FF=self.DicoGrids["DomainEdges_Freq"] TT=self.DicoGrids["DomainEdges_Time"] for iTime in range(TT.size-1): for iFreq in range(FF.size-1): #print(iTime,iFreq) APP.runJob("Stack_SingleTime:%d:%d"%(iTime,iFreq), self.Stack_SingleTime, args=(iTime,iFreq))#,serial=True) APP.awaitJobResults("Stack_SingleTime:*", progress="Append MS %i"%self.DicoDATA["iMS"]) FOut="%s.Weights.npy"%self.DicoDATA["MSName"] self.DicoDATA.reload() # print(self.DicoDATA["WOUT"]) #self.DicoDATA["WOUT"]/=np.median(self.DicoDATA["WOUT"]) w=self.DicoDATA["WOUT"] w/=np.median(w) w[w<=0]=0 w[w>2.]=2 log.print(" saving weights as %s"%FOut) np.save(FOut,self.DicoDATA["WOUT"]) # import pylab # pylab.clf() # pylab.hist(w[:,0].ravel()) # pylab.draw() # pylab.show() #np.save(FOut,self.DicoGrids["GridSTD"]) # for iTime in range(self.NTimes): # self.Stack_SingleTime(iTime) self.Finalise()
def StackAll(self): while self.iCurrentMS < self.nMS: if self.LoadNextMS() == "NotRead": continue print("Making dynamic spectra...", file=log) for iTime in range(self.NTimes): APP.runJob("Stack_SingleTime:%d" % (iTime), self.Stack_SingleTime, args=(iTime, )) #,serial=True) APP.awaitJobResults("Stack_SingleTime:*", progress="Append MS %i" % self.DicoDATA["iMS"]) # for iTime in range(self.NTimes): # self.Stack_SingleTime(iTime) self.Finalise()
def computeSmearMappingInBackground (self, base_job_id, MS, DATA, radiusDeg, Decorr, channel_mapping, mode): l = radiusDeg * np.pi / 180 dPhi = np.sqrt(6. * (1. - Decorr)) # create new empty shared dicts for results self._outdict = shared_dict.create("%s:%s:tmp" %(DATA.path, self.name)) blockdict = self._outdict.addSubdict("blocks") sizedict = self._outdict.addSubdict("sizes") self._nbl = 0 for a0 in xrange(MS.na): for a1 in xrange(MS.na): if a0 != a1: self._nbl += 1 APP.runJob("%s:%s:%d:%d" % (base_job_id, self.name, a0, a1), self._smearmapping_worker, counter=self._job_counter, collect_result=False, args=(DATA.readonly(), blockdict.writeonly(), sizedict.writeonly(), a0, a1, dPhi, l, channel_mapping, mode))
def StackAll(self): while self.iCurrentMS < self.nMS: if self.LoadNextMS() == "NotRead": continue print("Making dynamic spectra...", file=log) for iTime in range(self.NTimes): APP.runJob("Stack_SingleTime:%d" % (iTime), self.Stack_SingleTime, args=(iTime, )) #,serial=True) APP.awaitJobResults("Stack_SingleTime:*", progress="Append MS %i" % self.DicoDATA["iMS"]) FOut = "%s.Weights.npy" % self.DicoDATA["MSName"] log.print(" saving weights as %s" % FOut) self.DicoDATA.reload() # print(self.DicoDATA["WOUT"]) self.DicoDATA["WOUT"] /= np.median(self.DicoDATA["WOUT"]) np.save(FOut, self.DicoDATA["WOUT"]) # for iTime in range(self.NTimes): # self.Stack_SingleTime(iTime) self.Finalise()
def __init__(self, **kwargs): for key, value in kwargs.items(): setattr(self, key, value) LMS = [l.strip() for l in open(self.MSOutFreq).readlines()] self.OutFreqDomains = np.zeros((len(LMS) * self.NFreqPerMS, 2), np.float64) iFTot = 0 for iMS, MS in enumerate(LMS): t = table("%s::SPECTRAL_WINDOW" % MS, ack=False) df = t.getcol("CHAN_WIDTH").flat[0] fs = t.getcol("CHAN_FREQ").ravel() f0, f1 = fs[0] - df / 2., fs[-1] + df / 2. ff = np.linspace(f0, f1, self.NFreqPerMS + 1) for iF in range(self.NFreqPerMS): self.OutFreqDomains[iFTot, 0] = ff[iF] self.OutFreqDomains[iFTot, 1] = ff[iF + 1] iFTot += 1 NFreqsOut = self.OutFreqDomains.shape[0] self.CentralFreqs = np.mean(self.OutFreqDomains, axis=1) log.print("Loading %s" % self.SolsFileIn) self.DicoFile0 = dict(np.load(self.SolsFileIn)) Dico0 = self.DicoFile0 self.Sols0 = self.DicoFile0["Sols"].view(np.recarray) DicoOut = {} DicoOut['ModelName'] = Dico0['ModelName'] DicoOut['StationNames'] = Dico0['StationNames'] DicoOut['BeamTimes'] = Dico0['BeamTimes'] DicoOut['SourceCatSub'] = Dico0['SourceCatSub'] DicoOut['ClusterCat'] = Dico0['ClusterCat'] DicoOut['SkyModel'] = Dico0['SkyModel'] DicoOut['FreqDomains'] = self.OutFreqDomains self.DicoOut = DicoOut self.CentralFreqsIn = np.mean(Dico0['FreqDomains'], axis=1) self.DicoFreqWeights = {} for iChan in range(self.CentralFreqs.size): f = self.CentralFreqs[iChan] i0 = np.where(self.CentralFreqsIn <= f)[0] i1 = np.where(self.CentralFreqsIn > f)[0] if i0.size > 0 and i1.size > 0: i0 = i0[-1] i1 = i1[0] f0 = self.CentralFreqsIn[i0] f1 = self.CentralFreqsIn[i1] df = np.abs(f0 - f1) alpha = 1. - (f - f0) / df c0 = alpha c1 = 1. - alpha self.DicoFreqWeights[iChan] = { "Type": "Dual", "Coefs": (c0, c1), "Index": (i0, i1) } else: i0 = np.argmin(np.abs(self.CentralFreqsIn - f)) self.DicoFreqWeights[iChan] = {"Type": "Single", "Index": i0} # for iChan in range(self.CentralFreqs.size): # print # print self.CentralFreqs[iChan]/1e6 # if self.DicoFreqWeights[iChan]["Type"]=="Dual": # i0,i1=self.DicoFreqWeights[iChan]["Index"] # print self.CentralFreqsIn[i0]/1e6,self.CentralFreqsIn[i1]/1e6,self.DicoFreqWeights[iChan]["Coefs"] # else: # i0=self.DicoFreqWeights[iChan]["Index"] # print self.CentralFreqsIn[i0]/1e6 nt, _, na, nd, _, _ = self.Sols0.G.shape SolsOut = np.zeros( (nt, ), dtype=[("t0", np.float64), ("t1", np.float64), ("G", np.complex64, (NFreqsOut, na, nd, 2, 2)), ("Stats", np.float32, (NFreqsOut, na, 4))]) SolsOut = SolsOut.view(np.recarray) SolsOut.t0 = self.Sols0.t0 SolsOut.t1 = self.Sols0.t1 SolsOut.G[..., 0, 0] = 1. SolsOut.G[..., 1, 1] = 1. self.SolsOut = SolsOut self.GOut = NpShared.ToShared("%sGOut" % IdSharedMem, self.SolsOut.G.copy()) APP.registerJobHandlers(self) AsyncProcessPool.init(ncpu=self.NCPU, affinity=0)
return Aout # wrappers that discard return value for use with APP -- avoids wasteful stuffing of images into result queues def _convolveSingleGaussianFFTW_noret(*args, **kw): _convolveSingleGaussianFFTW(*args, **kw) return None def _convolveSingleGaussianNP_noret(*args, **kw): _convolveSingleGaussianNP(*args, **kw) return None APP.registerJobHandlers(_convolveSingleGaussianFFTW_noret, _convolveSingleGaussianNP_noret) ## FFTW version #def ConvolveGaussianFFTW(Ain0, # CellSizeRad=None, # GaussPars=[(0.,0.,0.)], # Normalise=False, # out=None, # nthreads=1, # min_size_fft=2048): # warnings.warn("deprecated", DeprecationWarning) # assert Ain0.shape == 4, "Expected stack of images: nch, npol, Ny, Nx" # nch,npol,Ny,Nx=Ain0.shape # pady = max(Ny, min_size_fft) # padx = max(Nx, min_size_fft)
def InitMSMF(self, approx=False, cache=True, facetcache=None): """Initializes MSMF basis functions. If approx is True, then uses the central facet's PSF for all facets. Populates the self.facetcache dict, unless facetcache is supplied """ self.DicoMSMachine = {} valid = True if facetcache is not None: print >> log, "HMP basis functions pre-initialized" self.facetcache = facetcache else: cachehash = dict([ (section, self.GD[section]) for section in ("Data", "Beam", "Selection", "Freq", "Image", "Facets", "Weight", "RIME", "Comp", "CF", "HMP") ]) cachepath, valid = self.maincache.checkCache(self.CacheFileName, cachehash, reset=not cache or self.PSFHasChanged) # do not use cache in approx mode if approx or not cache: valid = False if valid: print >> log, "Initialising HMP basis functions from cache %s" % cachepath self.facetcache = shared_dict.create(self.CacheFileName) self.facetcache.restore(cachepath) else: self.facetcache = None centralFacet = self.PSFServer.DicoVariablePSF["CentralFacet"] if approx: print >> log, "HMP approximation mode: using PSF of central facet (%d)" % centralFacet self.PSFServer.setFacet(centralFacet) MSMachine = ClassMultiScaleMachine.ClassMultiScaleMachine( self.GD, self.facetcache.addSubdict(0), self.GainMachine, NFreqBands=self.NFreqBands) MSMachine.setModelMachine(self.ModelMachine) MSMachine.setSideLobeLevel(self.SideLobeLevel, self.OffsetSideLobe) MSMachine.SetFacet(centralFacet) MSMachine.SetPSF(self.PSFServer) # ThisPSF,ThisMeanPSF) MSMachine.FindPSFExtent(verbose=True) MSMachine.MakeMultiScaleCube(verbose=True) MSMachine.MakeBasisMatrix() for iFacet in xrange(self.PSFServer.NFacets): self.DicoMSMachine[iFacet] = MSMachine else: # if no facet cache, init in parallel if self.facetcache is None: self.facetcache = shared_dict.create(self.CacheFileName) # breakout = False for iFacet in xrange(self.PSFServer.NFacets): fcdict = self.facetcache.addSubdict(iFacet) if self.ParallelMode: args = (fcdict.writeonly(), self.DicoVariablePSF.readonly(), iFacet, self.SideLobeLevel, self.OffsetSideLobe, centralFacet) APP.runJob("InitHMP:%d" % iFacet, self._initMSM_handler, args=args) else: args = (fcdict, self.DicoVariablePSF, iFacet, self.SideLobeLevel, self.OffsetSideLobe, centralFacet) self._initMSM_handler(*args) # import pdb; # pdb.set_trace() # if breakout: # raise RuntimeError("exiting") if self.ParallelMode: APP.awaitJobResults("InitHMP:*", progress="Init HMP") self.facetcache.reload() # t = ClassTimeIt.ClassTimeIt() for iFacet in xrange(self.PSFServer.NFacets): self.PSFServer.setFacet(iFacet) MSMachine = ClassMultiScaleMachine.ClassMultiScaleMachine( self.GD, self.facetcache[iFacet], self.GainMachine, NFreqBands=self.NFreqBands) MSMachine.setModelMachine(self.ModelMachine) MSMachine.setSideLobeLevel(self.SideLobeLevel, self.OffsetSideLobe) MSMachine.SetFacet(iFacet) MSMachine.SetPSF(self.PSFServer) # ThisPSF,ThisMeanPSF) MSMachine.FindPSFExtent( verbose=(iFacet == centralFacet )) # only print to log for central facet MSMachine.MakeMultiScaleCube(verbose=(iFacet == centralFacet)) MSMachine.MakeBasisMatrix() self.DicoMSMachine[iFacet] = MSMachine # write cache to disk, unless in a mode where we explicitly don't want it if facetcache is None and not valid and cache and not approx: try: #MyPickle.DicoNPToFile(facetcache,cachepath) #cPickle.dump(facetcache, file(cachepath, 'w'), 2) print >> log, " saving HMP cache to %s" % cachepath self.facetcache.save(cachepath) #self.maincache.saveCache("HMPMachine") self.maincache.saveCache(self.CacheFileName) self.PSFHasChanged = False print >> log, " HMP init done" except: print >> log, traceback.format_exc() print >> log, ModColor.Str( "WARNING: HMP cache could not be written, see error report above. Proceeding anyway." )
elif len(args): print(args) OP.ExitWithError("Incorrect number of arguments. Use -h for help.") sys.exit(1) retcode = report_error = 0 try: main(OP, messages) print(ModColor.Str( "DDFacet ended successfully after %s" % T.timehms(), col="green"), file=log) except KeyboardInterrupt: print(traceback.format_exc(), file=log) print(ModColor.Str("DDFacet interrupted by Ctrl+C", col="red"), file=log) APP.terminate() retcode = 1 #Should at least give the command line an indication of failure except Exceptions.UserInputError: print(ModColor.Str(sys.exc_info()[1], col="red"), file=log) print(ModColor.Str("There was a problem with some user input. See messages above for an indication."), file=log) APP.terminate() retcode = 1 # Should at least give the command line an indication of failure except WorkerProcessError: print(ModColor.Str("A worker process has died on us unexpectedly. This probably indicates a bug:"), file=log) print(ModColor.Str(" the original underlying error may be reported in the log [possibly far] above."), file=log) report_error = True except: if sys.exc_info()[0] is not WorkerProcessError and Exceptions.is_pdb_enabled(): APP.terminate() raise else:
def giveDicoInitIndiv(self, ListIslands, ModelImage, DicoDirty, ListDoIsland=None): DicoInitIndiv = shared_dict.create("DicoInitIsland") ParmDict = shared_dict.create("InitSSDModelHMP") ParmDict["ModelImage"] = ModelImage ParmDict["GridFreqs"] = self.GridFreqs ParmDict["DegridFreqs"] = self.DegridFreqs # ListBigIslands=[] # ListSmallIslands=[] # ListDoBigIsland=[] # ListDoSmallIsland=[] # NParallel=0 # for iIsland,Island in enumerate(ListIslands): # if len(Island)>self.GD["SSDClean"]["ConvFFTSwitch"]: # ListBigIslands.append(Island) # ListDoBigIsland.append(ListDoIsland[iIsland]) # if ListDoIsland or ListDoIsland[iIsland]: # NParallel+=1 # else: # ListSmallIslands.append(Island) # ListDoSmallIsland.append(ListDoIsland[iIsland]) # print>>log,"Initialise big islands (parallelised per island)" # pBAR= ProgressBar(Title="Init islands") # pBAR.render(0, NParallel) # nDone=0 # for iIsland,Island in enumerate(ListBigIslands): # if not ListDoIsland or ListDoBigIsland[iIsland]: # subdict = DicoInitIndiv.addSubdict(iIsland) # # APP.runJob("InitIsland:%d" % iIsland, self._initIsland_worker, # # args=(subdict.writeonly(), iIsland, Island, # # self.DicoVariablePSF.readonly(), DicoDirty.readonly(), # # ParmDict.readonly(), self.InitMachine.DeconvMachine.facetcache.readonly(),self.NCPU),serial=True) # self._initIsland_worker(subdict, iIsland, Island, # self.DicoVariablePSF, DicoDirty, # ParmDict, self.InitMachine.DeconvMachine.facetcache, # self.NCPU) # pBAR.render(nDone+1, NParallel) # nDone+=1 # # APP.awaitJobResults("InitIsland:*", progress="Init islands") # print>>log,"Initialise small islands (parallelised over islands)" # for iIsland,Island in enumerate(ListSmallIslands): # if not ListDoIsland or ListDoSmallIsland[iIsland]: # subdict = DicoInitIndiv.addSubdict(iIsland) # APP.runJob("InitIsland:%d" % iIsland, self._initIsland_worker, # args=(subdict.writeonly(), iIsland, Island, # self.DicoVariablePSF.readonly(), DicoDirty.readonly(), # ParmDict.readonly(), self.InitMachine.DeconvMachine.facetcache.readonly(),1)) # APP.awaitJobResults("InitIsland:*", progress="Init islands") # DicoInitIndiv.reload() print >> log, "Initialise islands (parallelised over islands)" if not self.GD["GAClean"]["ParallelInitHMP"]: pBAR = ProgressBar(Title=" Init islands") for iIsland, Island in enumerate(ListIslands): if not ListDoIsland or ListDoIsland[iIsland]: subdict = DicoInitIndiv.addSubdict(iIsland) self._initIsland_worker( subdict, iIsland, Island, self.DicoVariablePSF, DicoDirty, ParmDict, self.InitMachine.DeconvMachine.facetcache, 1) pBAR.render(iIsland, len(ListIslands)) else: for iIsland, Island in enumerate(ListIslands): if not ListDoIsland or ListDoIsland[iIsland]: subdict = DicoInitIndiv.addSubdict(iIsland) APP.runJob("InitIsland:%d" % iIsland, self._initIsland_worker, args=(subdict.writeonly(), iIsland, Island, self.DicoVariablePSF.readonly(), DicoDirty.readonly(), ParmDict.readonly(), self.InitMachine.DeconvMachine.facetcache. readonly(), 1)) APP.awaitJobResults("InitIsland:*", progress="Init islands") DicoInitIndiv.reload() ParmDict.delete() return DicoInitIndiv
def LoadModel(self): # ClassModelMachine,DicoModel=GiveModelMachine(self.FileDicoModel) # try: # self.GD["GAClean"]["GASolvePars"]=DicoModel["SolveParam"] # except: # self.GD["GAClean"]["GASolvePars"]=["S","Alpha"] # DicoModel["SolveParam"]=self.GD["GAClean"]["GASolvePars"] # self.MM=ClassModelMachine(self.GD) # self.MM.FromDico(DicoModel) # From DicoModel ModConstructor = ClassModModelMachine(self.GD) self.MM = ModConstructor.GiveInitialisedMMFromFile(self.FileDicoModel) #ModelImage0=self.MM.GiveModelImage(np.mean(self.VS.MS.ChanFreq)) if self.GD["GDkMS"]["ImageSkyModel"]["FilterNegComp"]: self.MM.FilterNegComponants(box=15, sig=1) if self.GD["GDkMS"]["ImageSkyModel"]["MaskImage"] != None: self.MM.CleanMaskedComponants( self.GD["GDkMS"]["ImageSkyModel"]["MaskImage"]) #self.ModelImage=self.MM.GiveModelImage(np.mean(self.VS.MS.ChanFreq)) model_freqs = self.VS.FreqBandChannelsDegrid[0] # original_freqs=self.VS.FreqBandChannels[0] # self.MM.setFreqMachine(original_freqs, model_freqs) ModelImage = self.MM.GiveModelImage(model_freqs) log.print("model image @%s MHz (min,max) = (%f, %f)" % (str(model_freqs / 1e6), ModelImage.min(), ModelImage.max())) # # From ModelImage # print "im!!!!!!!!!!!!!!!!!!!!!!!" # im=image("Model.fits") # data=im.getdata() # nch,npol,nx,_=data.shape # for ch in range(nch): # for pol in range(npol): # data[ch,pol]=data[ch,pol].T[::-1] # self.ModelImage=ModelImage=data # # ############################### self.FacetMachine.ToCasaImage(ModelImage, ImageName="%s.Model_kMS" % self.BaseImageName, Fits=True) # #stop #del(data) self.DicoImager = self.FacetMachine.DicoImager NFacets = len(self.FacetMachine.DicoImager) self.NFacets = NFacets #self.NDirs=NFacets #self.Dirs=range(self.NDirs) # SolsFile=self.GD["DDESolutions"]["DDSols"] # if not(".npz" in SolsFile): # ThisMSName=reformat.reformat(os.path.abspath(self.VS.MSName),LastSlash=False) # SolsFile="%s/killMS.%s.sols.npz"%(self.VS.MSName,SolsFile) # #SolsFile="BOOTES24_SB100-109.2ch8s.ms/killMS.KAFCA.Scalar.50Dir.0.1P.BriggsSq.PreCuster4.sols.npz" # DicoSolsFile=np.load(SolsFile) # ClusterCat=DicoSolsFile["ClusterCat"] # ClusterCat=ClusterCat.view(np.recarray) #DicoFacetName="%s.DicoFacet"%self.BaseImageName #DicoFacet=DDFacet.Other.MyPickle.Load(DicoFacetName) NodeFile = "%s.NodesCat.npy" % self.GD["Output"][ "Name"] #BaseImageName NodesCat = np.load(NodeFile) NodesCat = NodesCat.view(np.recarray) self.NDir = NodesCat.shape[0] ClusterCat = np.zeros((self.NDir, ), dtype=[('Name', '|S200'), ('ra', np.float), ('dec', np.float), ('l', np.float), ('m', np.float), ('SumI', np.float), ("Cluster", int)]) ClusterCat = ClusterCat.view(np.recarray) ClusterCat.l = NodesCat.l ClusterCat.m = NodesCat.m ClusterCat.ra = NodesCat.ra ClusterCat.dec = NodesCat.dec NN = ClusterCat.shape[0] Cat=np.zeros((NN,),dtype=[('Name','|S200'),('ra',np.float),('dec',np.float),('Sref',np.float),('I',np.float),('Q',np.float),\ ('U',np.float),('V',np.float),('RefFreq',np.float),('alpha',np.float),('ESref',np.float),\ ('Ealpha',np.float),('kill',np.int),('Cluster',np.int),('Type',np.int),('Gmin',np.float),\ ('Gmaj',np.float),('Gangle',np.float),("Select",np.int),('l',np.float),('m',np.float), ("Exclude",bool)]) Cat = Cat.view(np.recarray) Cat.RefFreq = 1. Cat.ra[:] = ClusterCat.ra Cat.dec[:] = ClusterCat.dec Cat.I[:] = ClusterCat.SumI[:] Cat.Cluster = np.arange(NN) Cat.Sref[:] = ClusterCat.SumI[:] self.SourceCat = Cat self.DicoImager = self.FacetMachine.DicoImager self.ClusterCat = ClusterCat self.ClusterCat.SumI = 0. # ind=np.where(self.ClusterCat.SumI!=0)[0] # self.ClusterCat=self.ClusterCat[ind].copy() # NFacets=self.ClusterCat.shape[0] # log.print( " There are %i non-zero facets"%NFacets) NFacets = len(self.FacetMachine.DicoImager) lFacet = np.zeros((NFacets, ), np.float32) mFacet = np.zeros_like(lFacet) for iFacet in range(NFacets): l, m = self.FacetMachine.DicoImager[iFacet]["lmShift"] lFacet[iFacet] = l mFacet[iFacet] = m NDir = ClusterCat.l.size d = np.sqrt((ClusterCat.l.reshape((NDir, 1)) - lFacet.reshape((1, NFacets)))**2 + (ClusterCat.m.reshape((NDir, 1)) - mFacet.reshape((1, NFacets)))**2) idDir = np.argmin(d, axis=0) for iFacet in range(NFacets): self.FacetMachine.DicoImager[iFacet]["iDirJones"] = idDir[iFacet] # print(iFacet,idDir[iFacet]) self.SM.ClusterCat = self.ClusterCat self.SM.SourceCat = self.SourceCat from DDFacet.Other.AsyncProcessPool import APP APP.startWorkers() #self.VS.CalcWeightsBackground() self.FacetMachine.initCFInBackground() self.FacetMachine.awaitInitCompletion() # for iFacet in range(NFacets): # #self.FacetMachine.SpacialWeigth[iFacet]=NpShared.ToShared("%sSpacialWeight_%3.3i"%(self.IdSharedMem,iFacet),self.FacetMachine.SpacialWeigth[iFacet]) # self.FacetMachine.SpacialWeigth[iFacet]=self.FacetMachine._CF[iFacet]["SW"] # log.print( " Splitting model image") # self.BuildGridsParallel() self.FacetMachine.BuildFacetNormImage() self.FacetMachine.setModelImage(ModelImage) self.FacetMachine.set_model_grid() self.PrepareGridMachinesMapping() #self.BuildGridsSerial() #self.BuildGridsParallel() NFacets = self.ClusterCat.shape[0] self.SM.NDir = self.NDirs self.SM.Dirs = self.Dirs log.print(" There are %i non-zero directions" % self.SM.NDir) self.SM.ClusterCat = self.ClusterCat self.SM.SourceCat = self.SourceCat # self.SM.SourceCat.I[:]=self.ClusterCat.SumI[:] self.SM.DicoJonesDirToFacet = self.DicoJonesDirToFacet self.SM.GD = self.FacetMachine.GD self.SM.DicoImager = self.FacetMachine.DicoImager #self.SM.GD["Comp"]["CompDeGridMode"]=0 CurrentMS = self.VS.ListMS[self.VS.iCurrentMS] self.SM.rac = CurrentMS.rac self.SM.decc = CurrentMS.decc self.SM.AvailableCorrelationProductsIds = self.VS.StokesConverter.AvailableCorrelationProductsIds( ) self.SM.RequiredStokesProductsIds = self.VS.StokesConverter.RequiredStokesProductsIds( ) self.SM.NFreqBands = self.VS.NFreqBands self.SM.Path = {"cf_dict_path": self.FacetMachine._CF.path} #self.SM.ChanMappingDegrid=self.VS.FreqBandChannelsDegrid[0] self.SM.ChanMappingDegrid = self.VS.DicoMSChanMappingDegridding[0] self.SM._model_dict = self.FacetMachine._model_dict self.SM.MapClusterCatOrigToCut = self.MapClusterCatOrigToCut
def __init__(self, Gain=0.3, MaxMinorIter=100, NCPU=1, #psutil.cpu_count() CycleFactor=2.5, FluxThreshold=None, RMSFactor=3, PeakFactor=0, PrevPeakFactor=0, GD=None, SearchMaxAbs=1, ModelMachine=None, NFreqBands=1, RefFreq=None, MainCache=None, IdSharedMem="", ParallelMode=True, CacheFileName="HMPBasis", **kw # absorb any unknown keywords arguments into this ): """ ImageDeconvMachine constructor. Note that this should be called pretty much when setting up the imager, before APP workers are started, because the object registers APP handlers. """ self.IdSharedMem=IdSharedMem self.SearchMaxAbs=SearchMaxAbs self._ModelImage = None self.MaxMinorIter = MaxMinorIter self.NCPU = NCPU self.Chi2Thr = 10000 self._MaskArray = None self.GD = GD self.SubPSF = None self.MultiFreqMode = NFreqBands > 1 self.NFreqBands = NFreqBands self.RefFreq = RefFreq self.FluxThreshold = FluxThreshold self.CycleFactor = CycleFactor self.RMSFactor = RMSFactor self.PeakFactor = PeakFactor self.PrevPeakFactor = PrevPeakFactor self.CacheFileName=CacheFileName self.GainMachine=ClassGainMachine.get_instance() self.ModelMachine = None self.PSFServer = None if ModelMachine is not None: self.updateModelMachine(ModelMachine) self.PSFHasChanged=False self._previous_initial_peak = None self.maincache = MainCache # reset overall iteration counter self._niter = 0 self.facetcache=None self._MaskArray=None self.MaskMachine=None self.ParallelMode=ParallelMode if self.ParallelMode: APP.registerJobHandlers(self) # we are in a worker if not self.ParallelMode: numexpr.set_num_threads(NCPU) # peak finding mode. # "normal" searches for peak in mean dirty image # "sigma" searches for peak in mean_dirty/noise_map (setNoiseMap will have been called) # "weighted" searched for peak in mean_dirty*weight self._peakMode = "normal" self.CurrentNegMask=None self._NoiseMap=None self._PNRStop=None # in _peakMode "sigma", provides addiitonal stopping criterion if self.GD["HMP"]["PeakWeightImage"]: print>> log, " Reading peak weighting image %s" % self.GD["HMP"]["PeakWeightImage"] img = image(self.GD["HMP"]["PeakWeightImage"]).getdata() _, _, nx, ny = img.shape # collapse freq and pol axes img = img.sum(axis=1).sum(axis=0).T[::-1].copy() self._peakWeightImage = img.reshape((1,1,ny,nx)) self._peakMode = "weighted" self._prevPeak = None
def InitMSMF(self, approx=False, cache=True, facetcache=None): """Initializes MSMF basis functions. If approx is True, then uses the central facet's PSF for all facets. Populates the self.facetcache dict, unless facetcache is supplied """ self.DicoMSMachine = {} valid = True if facetcache is not None: print>> log, "HMP basis functions pre-initialized" self.facetcache = facetcache else: cachehash = dict( [(section, self.GD[section]) for section in ( "Data", "Beam", "Selection", "Freq", "Image", "Facets", "Weight", "RIME","DDESolutions", "Comp", "CF", "HMP")]) cachepath, valid = self.maincache.checkCache(self.CacheFileName, cachehash, reset=not cache or self.PSFHasChanged) # do not use cache in approx mode if approx or not cache: valid = False if valid: print>>log,"Initialising HMP basis functions from cache %s"%cachepath self.facetcache = shared_dict.create(self.CacheFileName) self.facetcache.restore(cachepath) else: self.facetcache = None init_cache = self.facetcache is None if init_cache: self.facetcache = shared_dict.create(self.CacheFileName) # in any mode, start by initializing a MS machine for the central facet. This will precompute the scale # functions centralFacet = self.PSFServer.DicoVariablePSF["CentralFacet"] self.DicoMSMachine[centralFacet] = MSM0 = \ self._initMSM_facet(centralFacet, self.facetcache.addSubdict(centralFacet) if init_cache else self.facetcache[centralFacet], None, self.SideLobeLevel, self.OffsetSideLobe, verbose=True) if approx: print>>log, "HMP approximation mode: using PSF of central facet (%d)" % centralFacet for iFacet in xrange(self.PSFServer.NFacets): self.DicoMSMachine[iFacet] = MSM0 elif (self.GD["Facets"]["NFacets"]==1)&(not self.GD["DDESolutions"]["DDSols"]): self.DicoMSMachine[0] = MSM0 else: # if no facet cache, init in parallel if init_cache: for iFacet in xrange(self.PSFServer.NFacets): if iFacet != centralFacet: fcdict = self.facetcache.addSubdict(iFacet) if self.ParallelMode: args=(fcdict.writeonly(), MSM0.ScaleFuncs.readonly(), self.DicoVariablePSF.readonly(), iFacet, self.SideLobeLevel, self.OffsetSideLobe, False) APP.runJob("InitHMP:%d"%iFacet, self._initMSM_handler, args=args) else: self.DicoMSMachine[iFacet] = \ self._initMSM_facet(iFacet, fcdict, None, self.SideLobeLevel, self.OffsetSideLobe, MSM0=MSM0, verbose=False) if self.ParallelMode: APP.awaitJobResults("InitHMP:*", progress="Init HMP") self.facetcache.reload() # t = ClassTimeIt.ClassTimeIt() # now reinit from cache (since cache was computed by subprocesses) for iFacet in xrange(self.PSFServer.NFacets): if iFacet not in self.DicoMSMachine: self.DicoMSMachine[iFacet] = \ self._initMSM_facet(iFacet, self.facetcache[iFacet], None, self.SideLobeLevel, self.OffsetSideLobe, MSM0=MSM0, verbose=False) # write cache to disk, unless in a mode where we explicitly don't want it if facetcache is None and not valid and cache and not approx: try: #MyPickle.DicoNPToFile(facetcache,cachepath) #cPickle.dump(facetcache, file(cachepath, 'w'), 2) print>>log," saving HMP cache to %s"%cachepath self.facetcache.save(cachepath) #self.maincache.saveCache("HMPMachine") self.maincache.saveCache(self.CacheFileName) self.PSFHasChanged=False print>>log," HMP init done" except: print>>log, traceback.format_exc() print >>log, ModColor.Str( "WARNING: HMP cache could not be written, see error report above. Proceeding anyway.")
def __init__(self, InSolsName, OutSolsName, InterpMode="TEC", PolMode="Scalar", Amp_PolyOrder=3, NCPU=0, Amp_GaussKernel=(0, 5), Amp_SmoothType="Poly", CrossMode=1, RemoveAmpBias=0, RemoveMedianAmp=True): if type(InterpMode) == str: InterpMode = InterpMode.split(",") #[InterpMode] self.InSolsName = InSolsName self.OutSolsName = OutSolsName self.RemoveMedianAmp = RemoveMedianAmp log.print("Loading %s" % self.InSolsName) self.DicoFile = dict(np.load(self.InSolsName, allow_pickle=True)) self.Sols = self.DicoFile["Sols"].view(np.recarray) if "MaskedSols" in self.DicoFile.keys(): MaskFreq = np.logical_not( np.all(np.all(np.all(self.DicoFile["MaskedSols"][..., 0, 0], axis=0), axis=1), axis=1)) nt, _, na, nd, _, _ = self.Sols.G.shape self.DicoFile["FreqDomains"] = self.DicoFile["FreqDomains"][ MaskFreq] NFreqsOut = np.count_nonzero(MaskFreq) log.print("There are %i non-zero freq channels" % NFreqsOut) SolsOut = np.zeros( (nt, ), dtype=[("t0", np.float64), ("t1", np.float64), ("G", np.complex64, (NFreqsOut, na, nd, 2, 2)), ("Stats", np.float32, (NFreqsOut, na, 4))]) SolsOut = SolsOut.view(np.recarray) SolsOut.G = self.Sols.G[:, MaskFreq, ...] SolsOut.t0 = self.Sols.t0 SolsOut.t1 = self.Sols.t1 self.Sols = self.DicoFile["Sols"] = SolsOut del (self.DicoFile["MaskedSols"]) #self.Sols=self.Sols[0:10].copy() self.CrossMode = CrossMode self.CentralFreqs = np.mean(self.DicoFile["FreqDomains"], axis=1) self.incrCross = 11 iii = 0 NTEC = 101 NConstPhase = 51 TECGridAmp = 0.1 TECGrid, CPhase = np.mgrid[-TECGridAmp:TECGridAmp:NTEC * 1j, -np.pi:np.pi:NConstPhase * 1j] Z = TECToZ(TECGrid.reshape((-1, 1)), CPhase.reshape((-1, 1)), self.CentralFreqs.reshape((1, -1))) self.Z = Z self.TECGrid, self.CPhase = TECGrid, CPhase self.InterpMode = InterpMode self.Amp_PolyOrder = Amp_PolyOrder self.RemoveAmpBias = RemoveAmpBias if self.RemoveAmpBias: self.CalcFreqAmpSystematics() self.Sols.G /= self.G0 self.GOut = NpShared.ToShared("%sGOut" % IdSharedMem, self.Sols.G.copy()) self.PolMode = PolMode self.Amp_GaussKernel = Amp_GaussKernel if len(self.Amp_GaussKernel) != 2: raise ValueError("GaussKernel should be of size 2") self.Amp_SmoothType = Amp_SmoothType if "TEC" in self.InterpMode: log.print(" Smooth phases using a TEC model") if self.CrossMode: log.print(ModColor.Str("Using CrossMode")) if "Amp" in self.InterpMode: if Amp_SmoothType == "Poly": log.print( " Smooth amplitudes using polynomial model of order %i" % self.Amp_PolyOrder) if Amp_SmoothType == "Gauss": log.print( " Smooth amplitudes using Gaussian kernel of %s (Time/Freq) bins" % str(Amp_GaussKernel)) if self.RemoveAmpBias: self.GOut *= self.G0 APP.registerJobHandlers(self) AsyncProcessPool.init(ncpu=NCPU, affinity=0)
def killWorkers(self): print >> log, "Killing workers" APP.terminate() APP.shutdown() Multiprocessing.cleanupShm()
def __init__(self, ListMSName, ColName="DATA", ModelName="PREDICT_KMS", UVRange=[1., 1000.], SolsName=None, FileCoords=None, Radius=3., NOff=-1, Image=None, SolsDir=None, NCPU=1): self.ListMSName = sorted(ListMSName) #[0:2] self.nMS = len(self.ListMSName) self.ColName = ColName self.ModelName = ModelName self.OutName = self.ListMSName[0].split("/")[-1].split("_")[0] self.UVRange = UVRange self.ReadMSInfos() self.Radius = Radius self.Image = Image self.SolsDir = SolsDir #self.PosArray=np.genfromtxt(FileCoords,dtype=[('Name','S200'),("ra",np.float64),("dec",np.float64),('Type','S200')],delimiter="\t") # identify version in logs print >> log, "DynSpecMS version %s starting up" % version() # should we use the surveys DB? if 'DDF_PIPELINE_DATABASE' in os.environ: print >> log, "Using the surveys database" from surveys_db import SurveysDB with SurveysDB() as sdb: sdb.cur.execute('select * from transients') result = sdb.cur.fetchall() # convert to a list, then to ndarray, then to recarray l = [] for r in result: l.append((r['id'], r['ra'], r['decl'], r['type'])) self.PosArray = np.asarray(l, dtype=[('Name', 'S200'), ("ra", np.float64), ("dec", np.float64), ('Type', 'S200')]) print >> log, "Created an array with %i records" % len(result) else: if FileCoords is None: FileCoords = "Transient_LOTTS.csv" if not os.path.isfile(FileCoords): ssExec = "wget -q --user=anonymous ftp://ftp.strw.leidenuniv.nl/pub/tasse/%s -O %s" % ( FileCoords, FileCoords) print >> log, "Downloading %s" % FileCoords print >> log, " Executing: %s" % ssExec os.system(ssExec) self.PosArray = np.genfromtxt(FileCoords, dtype=[('Name', 'S200'), ("ra", np.float64), ("dec", np.float64), ('Type', 'S200')], delimiter=",")[()] self.PosArray = self.PosArray.view(np.recarray) self.PosArray.ra *= np.pi / 180. self.PosArray.dec *= np.pi / 180. NOrig = self.PosArray.shape[0] Dist = AngDist(self.ra0, self.PosArray.ra, self.dec0, self.PosArray.dec) ind = np.where(Dist < Radius * np.pi / 180)[0] self.PosArray = self.PosArray[ind] self.NDirSelected = self.PosArray.shape[0] print >> log, "Selected %i target [out of the %i in the original list]" % ( self.NDirSelected, NOrig) if self.NDirSelected == 0: print >> log, ModColor.Str(" Have found no sources - returning") self.killWorkers() return if NOff == -1: NOff = self.PosArray.shape[0] * 2 if NOff is not None: print >> log, "Including %i off targets" % (NOff) self.PosArray = np.concatenate( [self.PosArray, self.GiveOffPosArray(NOff)]) self.PosArray = self.PosArray.view(np.recarray) self.NDir = self.PosArray.shape[0] print >> log, "For a total of %i targets" % (self.NDir) self.DicoDATA = shared_dict.create("DATA") self.DicoGrids = shared_dict.create("Grids") self.DicoGrids["GridLinPol"] = np.zeros( (self.NDir, self.NChan, self.NTimes, 4), np.complex128) self.DicoGrids["GridWeight"] = np.zeros( (self.NDir, self.NChan, self.NTimes, 4), np.complex128) self.SolsName = SolsName self.DoJonesCorr = False if self.SolsName: self.DoJonesCorr = True self.DicoJones = shared_dict.create("DicoJones") APP.registerJobHandlers(self) AsyncProcessPool.init(ncpu=NCPU, affinity=0) APP.startWorkers()
def InterpolParallel(self): Sols0 = self.Sols nt, nch, na, nd, _, _ = Sols0.G.shape log.print(" #Times: %i" % nt) log.print(" #Channels: %i" % nch) log.print(" #Antennas: %i" % na) log.print(" #Directions: %i" % nd) # APP.terminate() # APP.shutdown() # Multiprocessing.cleanupShm() APP.startWorkers() iJob = 0 # for iAnt in [49]:#range(na): # for iDir in [0]:#range(nd): if "TEC" in self.InterpMode: #APP.runJob("FitThisTEC_%d"%iJob, self.FitThisTEC, args=(208,)); iJob+=1 self.TECArray = NpShared.ToShared( "%sTECArray" % IdSharedMem, np.zeros((nt, nd, na), np.float32)) self.CPhaseArray = NpShared.ToShared( "%sCPhaseArray" % IdSharedMem, np.zeros((nt, nd, na), np.float32)) for it in range(nt): # for iDir in range(nd): APP.runJob("FitThisTEC_%d" % iJob, self.FitThisTEC, args=(it, )) #,serial=True) iJob += 1 workers_res = APP.awaitJobResults("FitThisTEC*", progress="Fit TEC") if "Amp" in self.InterpMode: for iAnt in range(na): for iDir in range(nd): APP.runJob("FitThisAmp_%d" % iJob, self.FitThisAmp, args=(iAnt, iDir)) #,serial=True) iJob += 1 workers_res = APP.awaitJobResults("FitThisAmp*", progress="Smooth Amp") if "PolyAmp" in self.InterpMode: for iDir in range(nd): APP.runJob("FitThisPolyAmp_%d" % iJob, self.FitThisPolyAmp, args=(iDir, )) iJob += 1 workers_res = APP.awaitJobResults("FitThisPolyAmp*", progress="Smooth Amp") if "Clip" in self.InterpMode: for iDir in range(nd): APP.runJob("ClipThisDir_%d" % iJob, self.ClipThisDir, args=(iDir, ), serial=True) iJob += 1 workers_res = APP.awaitJobResults("ClipThisDir*", progress="Clip Amp") #APP.terminate() APP.shutdown() Multiprocessing.cleanupShm()
def __init__(self, name=None, mode=2): self.name = name or "SMM.%x" % id(self) APP.registerJobHandlers(self) self._job_counter = APP.createJobCounter(self.name) self._data = self._blockdict = self._sizedict = None
def killWorkers(self): print("Killing workers", file=log) APP.terminate() APP.shutdown() Multiprocessing.cleanupShm()
def InitFromCatalog(self): FileCoords = self.FileCoords dtype = [('Name', 'S200'), ("ra", np.float64), ("dec", np.float64), ('Type', 'S200')] # should we use the surveys DB? if 'DDF_PIPELINE_DATABASE' in os.environ: print("Using the surveys database", file=log) from surveys_db import SurveysDB with SurveysDB() as sdb: sdb.cur.execute('select * from transients') result = sdb.cur.fetchall() # convert to a list, then to ndarray, then to recarray l = [] for r in result: l.append((r['id'], r['ra'], r['decl'], r['type'])) if FileCoords is not None: print('Adding data from file ' + FileCoords, file=log) additional = np.genfromtxt(FileCoords, dtype=dtype, delimiter=",")[()] if not additional.shape: # deal with a one-line input file additional = np.array([additional], dtype=dtype) for r in additional: l.append(tuple(r)) self.PosArray = np.asarray(l, dtype=dtype) print("Created an array with %i records" % len(result), file=log) else: #FileCoords="Transient_LOTTS.csv" if FileCoords is None: if not os.path.isfile(FileCoords): ssExec = "wget -q --user=anonymous ftp://ftp.strw.leidenuniv.nl/pub/tasse/%s -O %s" % ( FileCoords, FileCoords) print("Downloading %s" % FileCoords, file=log) print(" Executing: %s" % ssExec, file=log) os.system(ssExec) log.print("Reading cvs file: %s" % FileCoords) #self.PosArray=np.genfromtxt(FileCoords,dtype=dtype,delimiter=",")[()] self.PosArray = np.genfromtxt(FileCoords, dtype=dtype, delimiter=",") self.PosArray = self.PosArray.view(np.recarray) self.PosArray.ra *= np.pi / 180. self.PosArray.dec *= np.pi / 180. Radius = self.Radius NOrig = self.PosArray.Name.shape[0] Dist = AngDist(self.ra0, self.PosArray.ra, self.dec0, self.PosArray.dec) ind = np.where(Dist < (Radius * np.pi / 180))[0] self.PosArray = self.PosArray[ind] self.NDirSelected = self.PosArray.shape[0] print("Selected %i target [out of the %i in the original list]" % (self.NDirSelected, NOrig), file=log) if self.NDirSelected == 0: print(ModColor.Str(" Have found no sources - returning"), file=log) self.killWorkers() return NOff = self.NOff if NOff == -1: NOff = self.PosArray.shape[0] * 2 if NOff is not None: print("Including %i off targets" % (NOff), file=log) self.PosArray = np.concatenate( [self.PosArray, self.GiveOffPosArray(NOff)]) self.PosArray = self.PosArray.view(np.recarray) self.NDir = self.PosArray.shape[0] print("For a total of %i targets" % (self.NDir), file=log) self.DicoDATA = shared_dict.create("DATA") self.DicoGrids = shared_dict.create("Grids") self.DicoGrids["GridLinPol"] = np.zeros( (self.NDir, self.NChan, self.NTimes, 4), np.complex128) self.DicoGrids["GridWeight"] = np.zeros( (self.NDir, self.NChan, self.NTimes, 4), np.complex128) self.DoJonesCorr_kMS = False self.DicoJones = None if self.SolsName: self.DoJonesCorr_kMS = True self.DicoJones_kMS = shared_dict.create("DicoJones_kMS") self.DoJonesCorr_Beam = False if self.BeamModel: self.DoJonesCorr_Beam = True self.DicoJones_Beam = shared_dict.create("DicoJones_Beam") APP.registerJobHandlers(self) AsyncProcessPool.init(ncpu=self.NCPU, affinity=0) APP.startWorkers()