def _initIsland_worker(self, DicoOut, iIsland, Island, DicoVariablePSF, DicoDirty, DicoParm, FacetCache, NCPU): logger.setSilent([ "ClassImageDeconvMachineMSMF", "ClassPSFServer", "ClassMultiScaleMachine", "GiveModelMachine", "ClassModelMachineMSMF" ]) self.InitMachine.Init(DicoVariablePSF, DicoParm["GridFreqs"], DicoParm["DegridFreqs"], facetcache=FacetCache) self.InitMachine.setDirty(DicoDirty) #self.InitMachine.DeconvMachine.setNCPU(NCPU) self.InitMachine.setSSDModelImage(DicoParm["ModelImage"]) #print ":::::::::::::::::::::::",iIsland try: SModel, AModel = self.InitMachine.giveModel(Island) except: if not self.GD["GAClean"]["ParallelInitHMP"]: raise print >> log, traceback.format_exc() FileOut = "errIsland_%6.6i.npy" % iIsland print >> log, ModColor.Str( "...... error on island %i, saving to file %s" % (iIsland, FileOut)) np.save(FileOut, np.array(Island)) self.InitMachine.Reset() return DicoOut["S"] = SModel DicoOut["Alpha"] = AModel self.InitMachine.Reset()
def __init__(self, VS): logger.setSilent(["ClassJones", "ClassLOFARBeam"]) self.VS = VS self.GD = copy.deepcopy(self.VS.GD) self.DoCentralNorm = self.GD["Beam"]["CenterNorm"] self.SmoothBeam = None self.CheckCache() if self.CacheValid: logger.setLoud(["ClassJones", "ClassLOFARBeam"]) return #self.GD["Beam"]["CenterNorm"]=0 self.ListMS = self.VS.ListMS self.MS = self.ListMS[0] rac, decc = self.MS.radec self.CoordMachine = ModCoord.ClassCoordConv(rac, decc) self.CalcGrid() #self.Padding=Padding #self.SumJJsq=np.zeros((self.npix,self.npix,self.MS.Nchan),np.float64) #self.SumWsq=np.zeros((1,self.MS.Nchan),np.float64) self.StackedBeamDict = shared_dict.create("StackedBeamDict") for iDir in range(self.NDir): sd = self.StackedBeamDict.addSubdict(iDir) sd.addSharedArray("SumJJsq", (self.VS.NFreqBands, ), np.float64) sd.addSharedArray("SumWsq", (self.VS.NFreqBands, ), np.float64) self.DicoJonesMachine = {} for iMS, MS in enumerate(self.ListMS): JonesMachine = ClassJones.ClassJones(self.GD, MS, self.VS.FacetMachine) JonesMachine.InitBeamMachine() self.DicoJonesMachine[iMS] = JonesMachine logger.setLoud(["ClassJones", "ClassLOFARBeam"])
def giveDicoInitIndiv(self, ListIslands, ModelImage, DicoDirty, ListDoIsland=None, Parallel=True): NCPU = self.NCPU work_queue = multiprocessing.JoinableQueue() ListIslands = ListIslands #[300:308] DoIsland = True for iIsland in range(len(ListIslands)): if ListDoIsland is not None: DoIsland = ListDoIsland[iIsland] if DoIsland: work_queue.put({"iIsland": iIsland}) result_queue = multiprocessing.JoinableQueue() NJobs = work_queue.qsize() workerlist = [] logger.setSilent(SilentModules) #MyLogger.setLoud(SilentModules) #MyLogger.setLoud("ClassImageDeconvMachineMSMF") print >> log, "Launch MORESANE workers" for ii in range(NCPU): W = WorkerInitMSMF(work_queue, result_queue, self.GD, self.DicoVariablePSF, DicoDirty, self.RefFreq, self.GridFreqs, self.DegridFreqs, self.MainCache, ModelImage, ListIslands, self.IdSharedMem) workerlist.append(W) if Parallel: workerlist[ii].start() timer = ClassTimeIt.ClassTimeIt() pBAR = ProgressBar(Title=" MORESANing islands ") #pBAR.disable() pBAR.render(0, NJobs) iResult = 0 if not Parallel: for ii in range(NCPU): workerlist[ii].run() # just run until all work is completed self.DicoInitIndiv = {} while iResult < NJobs: DicoResult = None if result_queue.qsize() != 0: try: DicoResult = result_queue.get() except: pass if DicoResult == None: time.sleep(0.5) continue if DicoResult["Success"]: iResult += 1 NDone = iResult pBAR.render(NDone, NJobs) iIsland = DicoResult["iIsland"] NameDico = "%sDicoInitIsland_%5.5i" % (self.IdSharedMem, iIsland) Dico = NpShared.SharedToDico(NameDico) self.DicoInitIndiv[iIsland] = copy.deepcopy(Dico) NpShared.DelAll(NameDico) if Parallel: for ii in range(NCPU): workerlist[ii].shutdown() workerlist[ii].terminate() workerlist[ii].join() #MyLogger.setLoud(["pymoresane.main"]) #MyLogger.setLoud(["ClassImageDeconvMachineMSMF","ClassPSFServer","ClassMultiScaleMachine","GiveModelMachine","ClassModelMachineMSMF"]) return self.DicoInitIndiv
def __init__(self, GD, DicoVariablePSF, RefFreq, GridFreqs, DegridFreqs, MainCache=None, IdSharedMem="", DoWait=False): self.DicoVariablePSF = DicoVariablePSF GD = copy.deepcopy(GD) self.RefFreq = RefFreq self.GridFreqs = GridFreqs self.DegridFreqs = DegridFreqs self.GD = GD self.GD["Parallel"]["NCPU"] = 1 #self.GD["HMP"]["Alpha"]=[0,0,1]#-1.,1.,5] self.GD["HMP"]["Alpha"] = [-1., 1., 5] self.GD["Deconv"]["Mode"] = "MORESANE" self.GD["Deconv"]["CycleFactor"] = 0 self.GD["Deconv"]["PeakFactor"] = 0.0 self.GD["Deconv"]["RMSFactor"] = self.GD["GAClean"]["RMSFactorInitHMP"] self.GD["Deconv"]["Gain"] = self.GD["GAClean"]["GainInitHMP"] self.GD["Deconv"]["AllowNegative"] = self.GD["GAClean"][ "AllowNegativeInitHMP"] self.GD["Deconv"]["MaxMinorIter"] = self.GD["GAClean"][ "MaxMinorIterInitHMP"] logger.setSilent(SilentModules) self.GD["HMP"]["Scales"] = self.GD["GAClean"]["ScalesInitHMP"] self.GD["HMP"]["Ratios"] = [] #self.GD["MultiScale"]["Ratios"]=[] self.GD["HMP"]["NTheta"] = 4 self.GD["HMP"]["SolverMode"] = "NNLS" #self.GD["MultiScale"]["SolverMode"]="PI" self.NFreqBands = len(DicoVariablePSF["freqs"]) MinorCycleConfig = dict(self.GD["Deconv"]) MinorCycleConfig["NCPU"] = self.GD["Parallel"]["NCPU"] MinorCycleConfig["NFreqBands"] = self.NFreqBands MinorCycleConfig["GD"] = self.GD MinorCycleConfig["GridFreqs"] = self.GridFreqs MinorCycleConfig["DegridFreqs"] = self.DegridFreqs #MinorCycleConfig["RefFreq"] = self.RefFreq ModConstructor = ClassModModelMachine(self.GD) ModelMachine = ModConstructor.GiveMM(Mode="MORESANE") ModelMachine.setRefFreq(self.RefFreq) MinorCycleConfig["ModelMachine"] = ModelMachine #MinorCycleConfig["CleanMaskImage"]=None self.MinorCycleConfig = MinorCycleConfig self.DeconvMachine = ClassImageDeconvMachineMoresane.ClassImageDeconvMachine( MainCache=MainCache, ParallelMode=False, RefFreq=self.RefFreq, CacheFileName="HMP_Init", IdSharedMem=IdSharedMem, **self.MinorCycleConfig) self.GD["Mask"]["Auto"] = False self.GD["Mask"]["External"] = None self.MaskMachine = ClassMaskMachine.ClassMaskMachine(self.GD) self.DeconvMachine.setMaskMachine(self.MaskMachine) self.Margin = 50 #print "Start 3" self.DeconvMachine.Init(PSFVar=self.DicoVariablePSF, PSFAve=self.DicoVariablePSF["PSFSideLobes"], GridFreqs=self.GridFreqs, DegridFreqs=self.DegridFreqs, DoWait=DoWait, RefFreq=self.RefFreq) if DoWait: print "IINit3" time.sleep(10) print "Start 4" self.DeconvMachine.Update(self.DicoDirty, DoSetMask=False) if DoWait: print "IINit4" time.sleep(10)
def StackBeam(self, ThisMSData, iDir): self.StackedBeamDict.reload() logger.setSilent("ClassJones") Dt = self.GD["Beam"]["DtBeamMin"] * 60. JonesMachine = self.DicoJonesMachine[ThisMSData["iMS"]] RAs, DECs = self.radec times = ThisMSData["times"] A0 = ThisMSData["A0"] A1 = ThisMSData["A1"] flags = ThisMSData["flags"] W = ThisMSData["Weights"] ChanToFreqBand = ThisMSData["ChanMapping"] beam_times = np.array( JonesMachine.BeamMachine.getBeamSampleTimes(times, quiet=True)) T2 = ClassTimeIt.ClassTimeIt() T2.disable() CurrentBeamITime = -1 # #print " Estimate beam in %i directions"%(RAs.size) # MS=self.ListMS[ThisMSData["iMS"]] # JonesMachine=ClassJones.ClassJones(self.GD,MS,self.VS.FacetMachine) # JonesMachine.InitBeamMachine() DicoBeam = JonesMachine.EstimateBeam(beam_times, RAs[iDir:iDir + 1], DECs[iDir:iDir + 1], progressBar=False, quiet=True) T2.timeit("GetBeam 1") #DicoBeam=JonesMachine.EstimateBeam(beam_times, RAs[0:10], DECs[0:10],progressBar=False) #T2.timeit("GetBeam 10") #print DicoBeam["Jones"].shape NTRange = DicoBeam["t0"].size #pBAR= ProgressBar(Title=" Mean Beam") #pBAR.render(0, '%4i/%i' % (0,NTRange)) T = ClassTimeIt.ClassTimeIt("Stacking") T.disable() # DicoBeam["Jones"].shape = nt, nd, na, nch, _, _ for iTRange in range(DicoBeam["t0"].size): t0 = DicoBeam["t0"][iTRange] t1 = DicoBeam["t1"][iTRange] J = np.abs(DicoBeam["Jones"][iTRange]) ind = np.where((times >= t0) & (times < t1))[0] T.timeit("0") A0s = A0[ind] A1s = A1[ind] fs = flags[ind] Ws = W[ind] MSnchan = Ws.shape[1] T.timeit("1") nd, na, nch, _, _ = J.shape # ###################### # This call is slow J0 = J[:, A0s, :, :, :] J1 = J[:, A1s, :, :, :] T.timeit("2") # ###################### # J0=np.zeros((nd,A0s.size,nch,2,2),dtype=J.dtype) # #T.timeit("1a") # J0List=[J[:,A0s[i],:,:,:] for i in range(A0s.size)] # #T.timeit("1b") # J1=np.zeros((nd,A0s.size,nch,2,2),dtype=J.dtype) # #T.timeit("1c") # J1List=[J[:,A1s[i],:,:,:] for i in range(A0s.size)] # #T.timeit("1d") # for i in range(A0s.size): # J0[:,i,:,:,:]=J0List[i] # J1[:,i,:,:,:]=J1List[i] # T.timeit("2b") JJ = (J0[:, :, :, 0, 0] * J1[:, :, :, 0, 0] + J0[:, :, :, 1, 1] * J1[:, :, :, 1, 1]) / 2. T.timeit("3") WW = Ws**2 T.timeit("4") WW = WW.reshape((1, ind.size, MSnchan)) T.timeit("5") JJsq = WW * JJ**2 T.timeit("6") SumWsqThisRange = np.sum(JJsq, axis=1) T.timeit("7") #self.SumJJsq+=SumWsqThisRange.reshape((self.npix,self.npix,self.MS.Nchan)) #T.timeit("8") SumWsq = np.sum(WW, axis=1) #self.SumWsq+=SumWsq for iBand in range(self.VS.NFreqBands): indFreqBand, = np.where(ChanToFreqBand == iBand) if indFreqBand.size == 0: continue self.StackedBeamDict[iDir]["SumJJsq"][iBand] += np.sum( SumWsqThisRange.reshape((MSnchan, ))[indFreqBand]) self.StackedBeamDict[iDir]["SumWsq"][iBand] += np.sum( SumWsq.reshape((MSnchan, ))[indFreqBand]) #print SumWsq,self.SumWsq,self.SumJJsq.shape,J0.shape T.timeit("9") #NDone = iTRange+1 #intPercent = int(100 * NDone / float(NTRange)) #pBAR.render(intPercent, '%4i/%i' % (NDone, NTRange)) T2.timeit("Stack") logger.setLoud("ClassJones")
from __future__ import absolute_import from __future__ import division from __future__ import print_function import numpy as np from DDFacet.Other import logger log = logger.getLogger("ClassFitAmp") import killMS.Array.ModLinAlg from DDFacet.Other import ClassTimeIt #from skimage.restoration import (denoise_tv_chambolle, denoise_bilateral, # denoise_wavelet, estimate_sigma) logger.setSilent(["ClassFitAmp"]) #from tvd import TotalVariationDenoising def Dot(*args): P = 1. for M in args: #P=np.dot(np.complex128(P),np.complex128(M)) P = np.dot(P, M) return P # iDir=14; S=np.load("L229509_merged.npz"); G=S["Sols"]["G"][:,:,:,iDir,0,0]; f=S["FreqDomains"].mean(axis=1) def Norm(G, iRef=0): nf, na = G.shape for iFreq in range(nf): g0 = G[iFreq, iRef] G[iFreq] *= g0.conj() / np.abs(g0)
from __future__ import absolute_import from __future__ import division from __future__ import print_function import numpy as np from DDFacet.Other import logger log = logger.getLogger("ClassFitTEC") import killMS.Array.ModLinAlg K = 8.4479745e9 import scipy.sparse from DDFacet.Other import ClassTimeIt logger.setSilent(["ClassFitTEC"]) def TECToPhase(TEC, freq): phase = K * TEC * (1. / freq) return phase def TECToZ(TEC, ConstPhase, freq): if ConstPhase is None: ConstPhase = 0 return np.exp(1j * (TECToPhase(TEC, freq) + ConstPhase)) def Dot(*args): P = 1. for M in args: #P=np.dot(np.complex128(P),np.complex128(M)) P = np.dot(P, M) return P
import time from DDFacet.Other import logger from DDFacet.Other import ModColor log=logger.getLogger("ClassImageDeconvMachine") from DDFacet.Other import ClassTimeIt from DDFacet.Imager.ClassPSFServer import ClassPSFServer from DDFacet.Other.progressbar import ProgressBar from DDFacet.Imager import ClassGainMachine from SkyModel.PSourceExtract import ClassIncreaseIsland from DDFacet.Imager.SSD.GA.ClassEvolveGA import ClassEvolveGA from DDFacet.Imager.SSD.MCMC.ClassMetropolis import ClassMetropolis from DDFacet.Array import NpParallel import ClassIslandDistanceMachine from DDFacet.Array import shared_dict logger.setSilent("ClassArrayMethodSSD") logger.setSilent("ClassIsland") class ClassImageDeconvMachine(): def __init__(self,Gain=0.3, MaxMinorIter=100,NCPU=6, CycleFactor=2.5,FluxThreshold=None,RMSFactor=3,PeakFactor=0, GD=None,SearchMaxAbs=1,IdSharedMem=None, ModelMachine=None, NFreqBands=1, RefFreq=None, MainCache=None, **kw # absorb any unknown keywords arguments into this ): #self.im=CasaImage