def _initIsland_worker(self, DicoOut, iIsland, Island, DicoVariablePSF, DicoDirty, DicoParm, FacetCache, NCPU): MyLogger.setSilent([ "ClassImageDeconvMachineMSMF", "ClassPSFServer", "ClassMultiScaleMachine", "GiveModelMachine", "ClassModelMachineMSMF" ]) self.InitMachine.Init(DicoVariablePSF, DicoParm["GridFreqs"], DicoParm["DegridFreqs"], facetcache=FacetCache) self.InitMachine.setDirty(DicoDirty) #self.InitMachine.DeconvMachine.setNCPU(NCPU) self.InitMachine.setSSDModelImage(DicoParm["ModelImage"]) try: SModel, AModel = self.InitMachine.giveModel(Island) except: print >> log, traceback.format_exc() FileOut = "errIsland_%6.6i.npy" % iIsland print >> log, ModColor.Str( "...... error on island %i, saving to file %s" % (iIsland, FileOut)) np.save(FileOut, np.array(Island)) self.InitMachine.Reset() return DicoOut["S"] = SModel DicoOut["Alpha"] = AModel self.InitMachine.Reset()
def __init__(self, VS): MyLogger.setSilent(["ClassJones", "ClassLOFARBeam"]) self.VS = VS self.GD = copy.deepcopy(self.VS.GD) self.DoCentralNorm = self.GD["Beam"]["CenterNorm"] self.SmoothBeam = None self.CheckCache() if self.CacheValid: MyLogger.setLoud(["ClassJones", "ClassLOFARBeam"]) return #self.GD["Beam"]["CenterNorm"]=0 self.ListMS = self.VS.ListMS self.MS = self.ListMS[0] rac, decc = self.MS.radec self.CoordMachine = ModCoord.ClassCoordConv(rac, decc) self.CalcGrid() #self.Padding=Padding #self.SumJJsq=np.zeros((self.npix,self.npix,self.MS.Nchan),np.float64) #self.SumWsq=np.zeros((1,self.MS.Nchan),np.float64) self.StackedBeamDict = shared_dict.create("StackedBeamDict") for iDir in range(self.NDir): sd = self.StackedBeamDict.addSubdict(iDir) sd.addSharedArray("SumJJsq", (self.VS.NFreqBands, ), np.float64) sd.addSharedArray("SumWsq", (self.VS.NFreqBands, ), np.float64) self.DicoJonesMachine = {} for iMS, MS in enumerate(self.ListMS): JonesMachine = ClassJones.ClassJones(self.GD, MS, self.VS.FacetMachine) JonesMachine.InitBeamMachine() self.DicoJonesMachine[iMS] = JonesMachine MyLogger.setLoud(["ClassJones", "ClassLOFARBeam"])
import time from DDFacet.Other import MyLogger from DDFacet.Other import ModColor log = MyLogger.getLogger("ClassImageDeconvMachine") from DDFacet.Other import ClassTimeIt from DDFacet.Imager.ClassPSFServer import ClassPSFServer from DDFacet.Other.progressbar import ProgressBar from DDFacet.Imager import ClassGainMachine from SkyModel.PSourceExtract import ClassIncreaseIsland from DDFacet.Imager.SSD.GA.ClassEvolveGA import ClassEvolveGA from DDFacet.Imager.SSD.MCMC.ClassMetropolis import ClassMetropolis from DDFacet.Array import NpParallel import ClassIslandDistanceMachine from DDFacet.Array import shared_dict MyLogger.setSilent("ClassArrayMethodSSD") MyLogger.setSilent("ClassIsland") class ClassImageDeconvMachine(): def __init__( self, Gain=0.3, MaxMinorIter=100, NCPU=6, CycleFactor=2.5, FluxThreshold=None, RMSFactor=3, PeakFactor=0, GD=None, SearchMaxAbs=1,
from DDFacet.Other import reformat import os from DDFacet.ToolsDir.ModToolBox import EstimateNpix from DDFacet.Other import ModColor import tables from DDFacet.Imager.ClassImToGrid import ClassImToGrid from matplotlib.path import Path from SkyModel.Sky import ModVoronoiToReg import Polygon from DDFacet.ToolsDir import rad2hmsdms from DDFacet.Other.ClassTimeIt import ClassTimeIt from DDFacet.Other import MyLogger log = MyLogger.getLogger("ClassFacetMachineTessel") MyLogger.setSilent("MyLogger") #from DDFacet.cbuild.Gridder import _pyGridderSmearPols as _pyGridderSmear class ClassFacetMachineTessel(ClassFacetMachine.ClassFacetMachine): """ This class extends the basic infrastructure set out in ClassFacetMachine to split the sky into a Voronoi tesselated sky. The projection, unprojection and resampling logic remains the same. """ def __init__(self, *args, **kwargs): ClassFacetMachine.ClassFacetMachine.__init__(self, *args, **kwargs) def setFacetsLocs(self): NFacets = self.NFacets Npix = self.GD["Image"]["NPix"]
def StackBeam(self, ThisMSData, iDir): self.StackedBeamDict.reload() MyLogger.setSilent("ClassJones") Dt = self.GD["Beam"]["DtBeamMin"] * 60. JonesMachine = self.DicoJonesMachine[ThisMSData["iMS"]] RAs, DECs = self.radec times = ThisMSData["times"] A0 = ThisMSData["A0"] A1 = ThisMSData["A1"] flags = ThisMSData["flags"] W = ThisMSData["Weights"] ChanToFreqBand = ThisMSData["ChanMapping"] beam_times = np.array( JonesMachine.BeamMachine.getBeamSampleTimes(times, quiet=True)) T2 = ClassTimeIt.ClassTimeIt() T2.disable() CurrentBeamITime = -1 # #print " Estimate beam in %i directions"%(RAs.size) # MS=self.ListMS[ThisMSData["iMS"]] # JonesMachine=ClassJones.ClassJones(self.GD,MS,self.VS.FacetMachine) # JonesMachine.InitBeamMachine() DicoBeam = JonesMachine.EstimateBeam(beam_times, RAs[iDir:iDir + 1], DECs[iDir:iDir + 1], progressBar=False, quiet=True) T2.timeit("GetBeam 1") #DicoBeam=JonesMachine.EstimateBeam(beam_times, RAs[0:10], DECs[0:10],progressBar=False) #T2.timeit("GetBeam 10") #print DicoBeam["Jones"].shape NTRange = DicoBeam["t0"].size #pBAR= ProgressBar(Title=" Mean Beam") #pBAR.render(0, '%4i/%i' % (0,NTRange)) T = ClassTimeIt.ClassTimeIt("Stacking") T.disable() # DicoBeam["Jones"].shape = nt, nd, na, nch, _, _ for iTRange in range(DicoBeam["t0"].size): t0 = DicoBeam["t0"][iTRange] t1 = DicoBeam["t1"][iTRange] J = np.abs(DicoBeam["Jones"][iTRange]) ind = np.where((times >= t0) & (times < t1))[0] T.timeit("0") A0s = A0[ind] A1s = A1[ind] fs = flags[ind] Ws = W[ind] MSnchan = Ws.shape[1] T.timeit("1") nd, na, nch, _, _ = J.shape # ###################### # This call is slow J0 = J[:, A0s, :, :, :] J1 = J[:, A1s, :, :, :] T.timeit("2") # ###################### # J0=np.zeros((nd,A0s.size,nch,2,2),dtype=J.dtype) # #T.timeit("1a") # J0List=[J[:,A0s[i],:,:,:] for i in range(A0s.size)] # #T.timeit("1b") # J1=np.zeros((nd,A0s.size,nch,2,2),dtype=J.dtype) # #T.timeit("1c") # J1List=[J[:,A1s[i],:,:,:] for i in range(A0s.size)] # #T.timeit("1d") # for i in range(A0s.size): # J0[:,i,:,:,:]=J0List[i] # J1[:,i,:,:,:]=J1List[i] # T.timeit("2b") JJ = (J0[:, :, :, 0, 0] * J1[:, :, :, 0, 0] + J0[:, :, :, 1, 1] * J1[:, :, :, 1, 1]) / 2. T.timeit("3") WW = Ws**2 T.timeit("4") WW = WW.reshape((1, ind.size, MSnchan)) T.timeit("5") JJsq = WW * JJ**2 T.timeit("6") SumWsqThisRange = np.sum(JJsq, axis=1) T.timeit("7") #self.SumJJsq+=SumWsqThisRange.reshape((self.npix,self.npix,self.MS.Nchan)) #T.timeit("8") SumWsq = np.sum(WW, axis=1) #self.SumWsq+=SumWsq for iBand in range(self.VS.NFreqBands): indFreqBand, = np.where(ChanToFreqBand == iBand) if indFreqBand.size == 0: continue self.StackedBeamDict[iDir]["SumJJsq"][iBand] += np.sum( SumWsqThisRange.reshape((MSnchan, ))[indFreqBand]) self.StackedBeamDict[iDir]["SumWsq"][iBand] += np.sum( SumWsq.reshape((MSnchan, ))[indFreqBand]) #print SumWsq,self.SumWsq,self.SumJJsq.shape,J0.shape T.timeit("9") #NDone = iTRange+1 #intPercent = int(100 * NDone / float(NTRange)) #pBAR.render(intPercent, '%4i/%i' % (NDone, NTRange)) T2.timeit("Stack") MyLogger.setLoud("ClassJones")
def giveDicoInitIndiv(self,ListIslands,ModelImage,DicoDirty,ListDoIsland=None,Parallel=True): NCPU=self.NCPU work_queue = multiprocessing.JoinableQueue() ListIslands=ListIslands#[300:308] DoIsland=True for iIsland in range(len(ListIslands)): if ListDoIsland is not None: DoIsland=ListDoIsland[iIsland] if DoIsland: work_queue.put({"iIsland":iIsland}) result_queue=multiprocessing.JoinableQueue() NJobs=work_queue.qsize() workerlist=[] MyLogger.setSilent(SilentModules) #MyLogger.setLoud(SilentModules) #MyLogger.setLoud("ClassImageDeconvMachineMSMF") print>>log,"Launch MORESANE workers" for ii in range(NCPU): W = WorkerInitMSMF(work_queue, result_queue, self.GD, self.DicoVariablePSF, DicoDirty, self.RefFreq, self.GridFreqs, self.DegridFreqs, self.MainCache, ModelImage, ListIslands, self.IdSharedMem) workerlist.append(W) if Parallel: workerlist[ii].start() timer = ClassTimeIt.ClassTimeIt() pBAR = ProgressBar(Title=" MORESANing islands ") #pBAR.disable() pBAR.render(0, NJobs) iResult = 0 if not Parallel: for ii in range(NCPU): workerlist[ii].run() # just run until all work is completed self.DicoInitIndiv={} while iResult < NJobs: DicoResult = None if result_queue.qsize() != 0: try: DicoResult = result_queue.get() except: pass if DicoResult == None: time.sleep(0.5) continue if DicoResult["Success"]: iResult+=1 NDone=iResult pBAR.render(NDone,NJobs) iIsland=DicoResult["iIsland"] NameDico="%sDicoInitIsland_%5.5i"%(self.IdSharedMem,iIsland) Dico=NpShared.SharedToDico(NameDico) self.DicoInitIndiv[iIsland]=copy.deepcopy(Dico) NpShared.DelAll(NameDico) if Parallel: for ii in range(NCPU): workerlist[ii].shutdown() workerlist[ii].terminate() workerlist[ii].join() #MyLogger.setLoud(["pymoresane.main"]) #MyLogger.setLoud(["ClassImageDeconvMachineMSMF","ClassPSFServer","ClassMultiScaleMachine","GiveModelMachine","ClassModelMachineMSMF"]) return self.DicoInitIndiv
def __init__(self,GD,DicoVariablePSF,RefFreq,GridFreqs,DegridFreqs, MainCache=None, IdSharedMem="", DoWait=False): self.DicoVariablePSF=DicoVariablePSF GD=copy.deepcopy(GD) self.RefFreq=RefFreq self.GridFreqs=GridFreqs self.DegridFreqs=DegridFreqs self.GD=GD self.GD["Parallel"]["NCPU"]=1 #self.GD["HMP"]["Alpha"]=[0,0,1]#-1.,1.,5] self.GD["HMP"]["Alpha"]=[-1.,1.,5] self.GD["Deconv"]["Mode"]="MORESANE" self.GD["Deconv"]["CycleFactor"]=0 self.GD["Deconv"]["PeakFactor"]=0.0 self.GD["Deconv"]["RMSFactor"]=self.GD["GAClean"]["RMSFactorInitHMP"] self.GD["Deconv"]["Gain"]=self.GD["GAClean"]["GainInitHMP"] self.GD["Deconv"]["AllowNegative"]=self.GD["GAClean"]["AllowNegativeInitHMP"] self.GD["Deconv"]["MaxMinorIter"]=self.GD["GAClean"]["MaxMinorIterInitHMP"] MyLogger.setSilent(SilentModules) self.GD["HMP"]["Scales"]=self.GD["GAClean"]["ScalesInitHMP"] self.GD["HMP"]["Ratios"]=[] #self.GD["MultiScale"]["Ratios"]=[] self.GD["HMP"]["NTheta"]=4 self.GD["HMP"]["SolverMode"]="NNLS" #self.GD["MultiScale"]["SolverMode"]="PI" self.NFreqBands=len(DicoVariablePSF["freqs"]) MinorCycleConfig=dict(self.GD["Deconv"]) MinorCycleConfig["NCPU"]=self.GD["Parallel"]["NCPU"] MinorCycleConfig["NFreqBands"]=self.NFreqBands MinorCycleConfig["GD"] = self.GD MinorCycleConfig["GridFreqs"] = self.GridFreqs MinorCycleConfig["DegridFreqs"] = self.DegridFreqs #MinorCycleConfig["RefFreq"] = self.RefFreq ModConstructor = ClassModModelMachine(self.GD) ModelMachine = ModConstructor.GiveMM(Mode="MORESANE") ModelMachine.setRefFreq(self.RefFreq) MinorCycleConfig["ModelMachine"]=ModelMachine #MinorCycleConfig["CleanMaskImage"]=None self.MinorCycleConfig=MinorCycleConfig self.DeconvMachine=ClassImageDeconvMachineMoresane.ClassImageDeconvMachine(MainCache=MainCache, ParallelMode=False, RefFreq=self.RefFreq, CacheFileName="HMP_Init", IdSharedMem=IdSharedMem, **self.MinorCycleConfig) self.GD["Mask"]["Auto"]=False self.GD["Mask"]["External"]=None self.MaskMachine=ClassMaskMachine.ClassMaskMachine(self.GD) self.DeconvMachine.setMaskMachine(self.MaskMachine) self.Margin=50 #print "Start 3" self.DeconvMachine.Init(PSFVar=self.DicoVariablePSF,PSFAve=self.DicoVariablePSF["PSFSideLobes"], GridFreqs=self.GridFreqs,DegridFreqs=self.DegridFreqs,DoWait=DoWait,RefFreq=self.RefFreq) if DoWait: print "IINit3" time.sleep(10) print "Start 4" self.DeconvMachine.Update(self.DicoDirty,DoSetMask=False) if DoWait: print "IINit4" time.sleep(10)