def doLMIter(self): T = ClassTimeIt.ClassTimeIt() T.disable() #J,H= self.giveJacobianHessian() T.timeit("J, H") z = self.GiveGPredict(self.CurrentX) Y = np.array([(z[iFreq].reshape((-1, 1)) * z[iFreq].conj().reshape( (1, -1))).ravel() for iFreq in range(self.NFreq)]).ravel() r = self.Y - Y[self.Mask][::self.Incr] v = self.JHy(r) H = self.DiagJHJ() T.timeit("diff") Hinv = killMS.Array.ModLinAlg.invSVD(H) T.timeit("inv") X = self.CurrentX + np.real(np.dot(Hinv, v.reshape((-1, 1))).ravel()) xx = self.CurrentX.copy() xx[xx == 0] = 1e-6 self.Diff = np.max(np.abs((X - xx) / xx)) z0 = self.GiveGPredict(self.CurrentX) Norm(z0) self.CurrentX = X z = self.GiveGPredict(self.CurrentX) Norm(z) self.Diff = np.max(np.abs(np.angle(z * z0.conj()))) #print self.Diff return
def giveFitness(Indiv, x=None, y=None, S=None, Polygons=None, PolyCut=None, BigPolygon=None): T = ClassTimeIt.ClassTimeIt("Fitness") T.disable() CMD = ClassMetricDEAP.ClassMetricDEAP(Indiv, x=x, y=y, S=S, Polygons=Polygons, PolyCut=PolyCut, BigPolygon=BigPolygon) fluxPerFacet = CMD.fluxPerFacet() NPerFacet = CMD.NPerFacet() meanDistancePerFacet = CMD.meanDistancePerFacet() overlapPerFacet = CMD.overlapPerFacet() Fitness = 0 Fitness += -np.std(fluxPerFacet) Fitness += -np.std(NPerFacet) Fitness += -1e5 * np.count_nonzero(NPerFacet == 0) # aspectRatioPerFacet=CMD.aspectRatioPerFacet() # A=aspectRatioPerFacet # Fitness+= -np.mean(A[A>0]) Fitness += -np.mean(meanDistancePerFacet) * 10 Fitness += -np.sum(overlapPerFacet) * 1e5 return Fitness,
def testWhereMax(): Npx=1030 Npy=500 nch=1 np.random.seed(8) A=np.zeros((nch,Npx,Npy),dtype=np.float32) A=np.float32(np.random.randn(*(nch,Npx,Npy))) Mask=np.zeros((Npx,Npy),np.bool8) #A[0,100,200]=30. T= ClassTimeIt.ClassTimeIt() #Ans=A_whereMax(A,NCPU=1,Mask=Mask) #print Ans T.timeit("1") f=np.abs #f=lambda x: x ind= np.where(f(A)==np.max(f(A))) T.timeit("2") print(ind,A[ind]) print("===========================") print(A_whereMax(A,NCPU=1,DoAbs=0,Mask=None)) print(A_whereMax(A,NCPU=1,DoAbs=0,Mask=np.zeros(A.shape,np.bool8))) print("===========================") print(A_whereMax(A,NCPU=6,DoAbs=0,Mask=None)) print(A_whereMax(A,NCPU=6,DoAbs=0,Mask=np.zeros(A.shape,np.bool8))) print("===========================") print(A_whereMax(A,NCPU=1,DoAbs=1,Mask=None)) print(A_whereMax(A,NCPU=1,DoAbs=1,Mask=np.zeros(A.shape,np.bool8))) print("===========================") print(A_whereMax(A,NCPU=6,DoAbs=1,Mask=None)) print(A_whereMax(A,NCPU=6,DoAbs=1,Mask=np.zeros(A.shape,np.bool8)))
def fft(self, Ain): axes = (-1, -2) T = ClassTimeIt.ClassTimeIt("ModFFTW") T.disable() sin = Ain.shape if len(Ain.shape) == 2: s = (1, 1, Ain.shape[0], Ain.shape[1]) A = Ain.reshape(s) else: A = Ain nch, npol, _, _ = A.shape for ich in range(nch): for ipol in range(npol): A_2D = iFs(A[ich, ipol].astype(self.ThisType), axes=axes) T.timeit("shift and copy") A_2D[...] = pyfftw.interfaces.numpy_fft.fft2( A_2D, axes=(-1, -2), overwrite_input=True, planner_effort='FFTW_MEASURE', threads=self.ncores) T.timeit("fft") A[ich, ipol] = Fs(A_2D, axes=axes) T.timeit("shift") if self.norm: A /= (A.shape[-1] * A.shape[-2]) return A.reshape(sin)
def testAdd(): Np = 16000 x0 = 1000 x1 = 11000 y0 = 5000 y1 = 15000 nch = 1 _a0 = np.ones((nch, Np, Np), dtype=np.float32) _a1 = np.ones((nch, Np, Np), dtype=np.float32) # b=np.float32(np.arange(Np**2).reshape((Np,Np))) b = np.ones((Np, Np), dtype=np.float32) Aedge = np.array([x0, x1, y0, y1], np.int32) Bedge = np.array([x0 + 1, x1 + 1, y0, y1], np.int32) T = ClassTimeIt.ClassTimeIt() N = 10 NBlocks = 6 factor = -1. for i in xrange(N): # b=np.float32(np.random.randn(Np,Np)) A_add_B_prod_factor(_a0, b, Aedge, Bedge, factor=float(factor), NCPU=NBlocks) # A_add_B_prod_factor(_a0,b,Aedge,Bedge,factor=float(factor),NCPU=NBlocks) # for ch in range(nch): # a0=_a0[ch] # a1=_a1[ch] # a_x0,a_x1,a_y0,a_y1=Aedge # b_x0,b_x1,b_y0,b_y1=Bedge # a1[a_x0:a_x1,a_y0:a_y1]+=b[b_x0:b_x1,b_y0:b_y1]*factor # # print a1[a_x0:a_x1,a_y0:a_y1].shape # # print b[b_x0:b_x1,b_y0:b_y1].shape # # print a_x0,a_x1,a_y0,a_y1 # # print b_x0,b_x1,b_y0,b_y1 # print "done" # print np.max(_a0-_a1) # # print # print _a0 # print _a1 T.timeit("1") for i in xrange(N): for ch in xrange(nch): a0 = _a0[ch] a1 = _a1[ch] a_x0, a_x1, a_y0, a_y1 = Aedge b_x0, b_x1, b_y0, b_y1 = Bedge a1[a_x0:a_x1, a_y0:a_y1] += b[b_x0:b_x1, b_y0:b_y1] * factor # a1=a0+b*factor T.timeit("2")
def Convolve(self, A, Norm=True, OutMode="Data", ConvMode=None): if ConvMode == None: ConvMode = self.ConvMode if ConvMode == "Matrix": return self.ConvolveMatrix(A, OutMode=OutMode) elif ConvMode == "Vector": T = ClassTimeIt.ClassTimeIt("ConvVec") T.disable() C = self.ConvolveVector(A, OutMode=OutMode) T.timeit() elif ConvMode == "FFT": T = ClassTimeIt.ClassTimeIt("ConvFFT") T.disable() C = self.ConvolveFFT(A, OutMode=OutMode) T.timeit() return C
def _dispatch_job(self, jobitem, reraise=False): """Handles job described by jobitem dict. If reraise is True, any eceptions are re-raised. This is useful for debugging.""" timer = ClassTimeIt.ClassTimeIt() event = counter = None try: job_id, event_id, counter_id, args, kwargs = [jobitem.get(attr) for attr in "job_id", "event", "counter", "args", "kwargs"] handler_id, method, handler_desc = jobitem["handler"] handler = self._job_handlers.get(handler_id) if handler is None: raise RuntimeError("Job %s: unknown handler %s. This is a bug." % (job_id, handler_desc)) event, eventname = self._events[event_id] if event_id is not None else (None, None) # find counter object, if specified if counter_id: counter = self._job_counters.get(counter_id) if counter is None: raise RuntimeError("Job %s: unknown counter %s. This is a bug." % (job_id, counter_id)) # instantiate SharedDict arguments # timer.timeit('init '+job_id) args = [ arg.instantiate() if type(arg) is shared_dict.SharedDictRepresentation else arg for arg in args ] for key in kwargs.keys(): if type(kwargs[key]) is shared_dict.SharedDictRepresentation: kwargs[key] = kwargs[key].instantiate() # timer.timeit('instantiated '+job_id) # call the job if self.verbose > 1: print>> log, "job %s: calling %s" % (job_id, handler_desc) if method is None: # call object directly result = handler(*args, **kwargs) else: call = getattr(handler, method, None) if not callable(call): raise KeyError("Job %s: unknown method '%s' for handler %s" % (job_id, method, handler_desc)) result = call(*args, **kwargs) if self.verbose > 3: print>> log, "job %s: %s returns %s" % (job_id, handler_desc, result) # Send result back if jobitem['collect_result']: self._result_queue.put( dict(job_id=job_id, proc_id=self.proc_id, success=True, result=result, time=timer.seconds())) except KeyboardInterrupt: raise except Exception, exc: if reraise: raise print>> log, ModColor.Str("process %s: exception raised processing job %s: %s" % ( AsyncProcessPool.proc_id, job_id, traceback.format_exc())) if jobitem['collect_result']: self._result_queue.put( dict(job_id=job_id, proc_id=self.proc_id, success=False, error=exc, time=timer.seconds()))
def addSubModelToSubDirty(self): T = ClassTimeIt.ClassTimeIt("InitSSD.addSubModelToSubDirty") T.disable() ConvModel = self.giveConvModel(self.SubSSDModelImage) _, _, N0x, N0y = ConvModel.shape MeanConvModel = np.mean(ConvModel, axis=0).reshape((1, 1, N0x, N0y)) self.DicoSubDirty["ImageCube"] += ConvModel self.DicoSubDirty['MeanImage'] += MeanConvModel #print "MAX=",np.max(self.DicoSubDirty['MeanImage']) T.timeit("2")
def DiagJHJ(self): T = ClassTimeIt.ClassTimeIt("JHy") T.disable() H = np.zeros((self.LMode, self.na, self.LMode, self.na), np.complex64) for iAnt in range(self.na): #Jt[self.indA0[iAnt],iAnt]=J_TEC[self.indA0[iAnt]] #Jt[self.indA1[iAnt],iAnt]=-J_TEC[self.indA1[iAnt]] H[0, iAnt, 0, iAnt] += np.sum(self.J_TEC[self.indA0[iAnt]].conj() * self.J_TEC[self.indA0[iAnt]]) H[0, iAnt, 0, iAnt] += np.sum(self.J_TEC[self.indA1[iAnt]].conj() * self.J_TEC[self.indA1[iAnt]]) if "CPhase" in self.Mode: H[0, iAnt, 1, iAnt] += np.sum(self.J_TEC[self.indA0[iAnt]].conj() * self.J_Phase[self.indA0[iAnt]]) H[0, iAnt, 1, iAnt] += np.sum(self.J_TEC[self.indA1[iAnt]].conj() * self.J_Phase[self.indA1[iAnt]]) H[1, iAnt, 0, iAnt] += np.sum(self.J_Phase[self.indA0[iAnt]].conj() * self.J_TEC[self.indA0[iAnt]]) H[1, iAnt, 0, iAnt] += np.sum(self.J_Phase[self.indA1[iAnt]].conj() * self.J_TEC[self.indA1[iAnt]]) H[1, iAnt, 1, iAnt] += np.sum(self.J_Phase[self.indA0[iAnt]].conj() * self.J_Phase[self.indA0[iAnt]]) H[1, iAnt, 1, iAnt] += np.sum(self.J_Phase[self.indA1[iAnt]].conj() * self.J_Phase[self.indA1[iAnt]]) T.timeit("H") H = H.reshape((self.LMode * self.na, self.LMode * self.na)) return H A = np.log10(np.abs(self.H)) B = np.log10(np.abs(H)) vmin, vmax = A.min(), A.max() import pylab pylab.clf() pylab.subplot(1, 2, 1) pylab.imshow(A, interpolation="nearest", vmin=vmin, vmax=vmax) pylab.colorbar() pylab.subplot(1, 2, 2) pylab.imshow(B, interpolation="nearest", vmin=vmin, vmax=vmax) pylab.colorbar() pylab.draw() pylab.show(False) stop return H
def __init__(self, work_queue, result_queue, island_dict=None, iIsland=None, ListPixParms=None, ListPixData=None, GD=None, PSF=None, PauseOnStart=False, PM=None, PixVariance=1e-2, EstimatedStdFromResid=0, MaxFunc=None, WeightMaxFunc=None, DirtyArray=None, ConvMode=None, StopWhenQueueEmpty=False, BestChi2=1., DicoData=None): self.T = ClassTimeIt.ClassTimeIt("WorkerFitness") self.T.disable() multiprocessing.Process.__init__(self) self.work_queue = work_queue self.result_queue = result_queue self.kill_received = False self.exit = multiprocessing.Event() self._pause_on_start = PauseOnStart self._island_dict = island_dict self._chain_dict = island_dict["Chains"] self.GD = GD self.PM = PM self.EstimatedStdFromResid = EstimatedStdFromResid self.ListPixParms = ListPixParms self.ListPixData = ListPixData self.iIsland = iIsland self.PSF = PSF self.PixVariance = PixVariance self.ConvMachine = ClassConvMachine.ClassConvMachine( self.PSF, self.ListPixParms, self.ListPixData, ConvMode) self.ConvMachine.setParamMachine(self.PM) self.DicoData = DicoData self.MutMachine = ClassMutate.ClassMutate(self.PM) self.MutMachine.setData(DicoData) self.MaxFunc = MaxFunc self.WeightMaxFunc = WeightMaxFunc self.DirtyArray = DirtyArray self.T.timeit("init") self.StopWhenQueueEmpty = StopWhenQueueEmpty self.BestChi2 = BestChi2
def giveJacobianHessian(self): T = ClassTimeIt.ClassTimeIt("J") J = np.zeros((self.Y.size, self.na * 2), np.complex64) Jt = J[:, 0:self.na] Jc = J[:, self.na:] TEC = self.CurrentX[0:self.na] dTEC = TEC[self.A0] - TEC[self.A1] if "CPhase" in self.Mode: ConstPhase = self.CurrentX[self.na:] dConstPhase = ConstPhase[self.A0] - ConstPhase[self.A1] else: dConstPhase = 0 Phase = K / self.nu_Y * dTEC + dConstPhase Z = np.exp(1j * Phase) self.J_TEC = J_TEC = 1j * K / self.nu_Y * Z self.J_Phase = J_Phase = 1j * Z return T.timeit("first") for iAnt in range(self.na): Jt[self.indA0[iAnt], iAnt] = J_TEC[self.indA0[iAnt]] Jt[self.indA1[iAnt], iAnt] = -J_TEC[self.indA1[iAnt]] Jc[self.indA0[iAnt], iAnt] = J_Phase[self.indA0[iAnt]] Jc[self.indA1[iAnt], iAnt] = -J_Phase[self.indA1[iAnt]] T.timeit("build") self.J = J self.Jsp = Jsp = scipy.sparse.coo_matrix(J) T.timeit("sp") # import pylab # pylab.clf() # pylab.subplot(1,2,1) # pylab.imshow(Jt.real,interpolation="nearest",aspect="auto") # pylab.subplot(1,2,2) # pylab.imshow(Jc.real,interpolation="nearest",aspect="auto") # pylab.draw() # pylab.show(False) # stop #print np.count_nonzero(J)/float(J.size) T.timeit("prod") H = np.array(np.dot(Jsp.T.conj(), Jsp).todense()) self.H = H T.timeit("Hsp") return J, H
def GiveInstrumentBeam(self,*args,**kwargs): T=ClassTimeIt.ClassTimeIt("GiveInstrumentBeam") T.disable() Beam=self.GiveRawBeam(*args,**kwargs) nd,na,nch,_,_=Beam.shape T.timeit("0") MeanBeam=np.zeros((nd,na,self.NChanJones,2,2),dtype=Beam.dtype) for ich in range(self.NChanJones): indCh=np.where(self.VisToJonesChanMapping==ich)[0] MeanBeam[:,:,ich,:,:]=np.mean(Beam[:,:,indCh,:,:],axis=2) T.timeit("1") return MeanBeam
def MakeSphe(Support, NpixIm): # x,y,CF=Gaussian.Gaussian(3,Support,1) import ClassTimeIt T = ClassTimeIt.ClassTimeIt() SupportSphe = 111 xc = SupportSphe / 2 CF = ModTaper.Sphe2D(SupportSphe) T.timeit("0") CF = np.complex128(CF) # np.array(np.complex128(CF),order="F") fCF = fft2(CF) fCF = fCF[xc - Support / 2:xc + Support / 2 + 1, xc - Support / 2:xc + Support / 2 + 1].copy() zfCF = ZeroPad(fCF, NpixIm) T.timeit("1") ifzfCF = ifft2(zfCF) # ############" # import pylab # pylab.clf() # pylab.subplot(3,2,1) # lpar=list(pylab.imshow.__defaults__) # lpar[3]="nearest" # pylab.imshow.__defaults__=tuple(lpar) # pylab.imshow(CF.real) # pylab.colorbar() # pylab.subplot(3,2,2) # pylab.imshow(CF.imag) # pylab.colorbar() # pylab.subplot(3,2,3) # pylab.imshow(fCF.real) # pylab.colorbar() # pylab.subplot(3,2,4) # pylab.imshow(fCF.imag) # pylab.colorbar() # pylab.subplot(3,2,5) # pylab.imshow(ifzfCF.real) # pylab.colorbar() # pylab.subplot(3,2,6) # pylab.imshow(ifzfCF.imag) # pylab.colorbar() # pylab.draw() # pylab.show(False) # pylab.pause(0.1) # stop return CF, fCF, ifzfCF
def InitCF(self, cf_dict, compute_cf, wmax): T = ClassTimeIt.ClassTimeIt("InitCF_ClassDDEGridMachine") T.disable() self.WTerm = ModCF.ClassWTermModified(Cell=self.Cell, Sup=self.Sup, Npix=self.Npix, Freqs=self.ChanFreq, wmax=wmax, Nw=self.Nw, OverS=self.OverS, lmShift=self.lmShift, cf_dict=cf_dict, compute_cf=compute_cf, IDFacet=self.IDFacet) T.timeit("2") self.ifzfCF = self.WTerm.ifzfCF
def _smearmapping_worker(self, DATA, blockdict, sizedict, a0, a1, dPhi, l, channel_mapping, mode): t = ClassTimeIt.ClassTimeIt() t.disable() if mode == 1: BlocksRowsListBL, BlocksSizesBL, _ = GiveBlocksRowsListBL_old(a0, a1, DATA, dPhi, l, channel_mapping) elif mode == 2: BlocksRowsListBL, BlocksSizesBL, _ = GiveBlocksRowsListBL(a0, a1, DATA, dPhi, l, channel_mapping) else: raise ValueError("unknown BDAMode setting %d"%mode) t.timeit('compute') if BlocksRowsListBL is not None: key = "%d:%d" % (a0,a1) sizedict[key] = np.array(BlocksSizesBL) blockdict[key] = np.array(BlocksRowsListBL) t.timeit('store')
def addSubModelToSubDirty(self): T = ClassTimeIt.ClassTimeIt("InitSSD.addSubModelToSubDirty") T.disable() ConvModel = self.giveConvModel(self.SubSSDModelImage) _, _, N0x, N0y = ConvModel.shape MeanConvModel = np.mean(ConvModel, axis=0).reshape((1, 1, N0x, N0y)) self.DicoSubDirty["ImageCube"] += ConvModel #self.DicoSubDirty['MeanImage']+=MeanConvModel W = np.float32(self.DicoSubDirty["WeightChansImages"]) W = W / np.sum(W) MeanImage = np.sum(self.DicoSubDirty["ImageCube"] * W.reshape( (-1, 1, 1, 1)), axis=0).reshape((1, 1, N0x, N0y)) self.DicoSubDirty['MeanImage'] = MeanImage #print "MAX=",np.max(self.DicoSubDirty['MeanImage']) T.timeit("2")
def GiveConvVector(self, iPix, TypeOut="Data"): T = ClassTimeIt.ClassTimeIt() T.disable() PSF = self.PSF NPixPSF = PSF.shape[-1] xc = yc = NPixPSF / 2 T.timeit("0") x1, y1 = self.ArrayListPixParms[iPix:iPix + 1].T if TypeOut == "Data": M = np.zeros((self.NFreqBands, 1, self.NPixListData, 1), np.float32) x0, y0 = self.ArrayListPixData.T else: M = np.zeros((self.NFreqBands, 1, self.NPixListParms, 1), np.float32) x0, y0 = self.ArrayListPixParms.T N0 = x0.size N1 = x1.size T.timeit("1") dx = (x1.reshape((N1, 1)) - x0.reshape((1, N0)) + xc).T dy = (y1.reshape((N1, 1)) - y0.reshape((1, N0)) + xc).T T.timeit("2") Cx = ((dx >= 0) & (dx < NPixPSF)) Cy = ((dy >= 0) & (dy < NPixPSF)) C = (Cx & Cy) T.timeit("3") indPSF = np.arange(M.shape[-1] * M.shape[-2]) indPSF_sel = indPSF[C.ravel()] indPixPSF = dx.ravel()[C.ravel()] * NPixPSF + dy.ravel()[C.ravel()] T.timeit("4") if indPSF_sel.size != indPSF.size: for iBand in range(self.NFreqBands): PSF_Chan = PSF[iBand, 0] M[iBand, 0].flat[indPSF_sel] = PSF_Chan.flat[indPixPSF.ravel()] return M[:, :, :, 0] else: ListVec = [] for iBand in range(self.NFreqBands): PSF_Chan = PSF[iBand, 0] ListVec.append(PSF_Chan.flat[indPixPSF.ravel()]) return ListVec
def doOverlap(npP0, npP1): T = ClassTimeIt.ClassTimeIt("Overlap") T.disable() if npP0.size == 0: return False if npP1.size == 0: return False P0 = Polygon.Polygon(npP0) P1 = Polygon.Polygon(npP1) T.timeit("declare") P1Cut = (P0 & P1) T.timeit("Cut") aP1 = P1.area() aP1Cut = P1Cut.area() T.timeit("Area") if np.abs(aP1Cut - aP1) < 1e-10: return "Contains" elif aP1Cut == 0: return "Outside" else: return "Cut"
def __init__(self, shape, dtype, norm=True, ncores=1, FromSharedId=None): # if FromSharedId is None: # self.A = pyfftw.n_byte_align_empty( shape[-2::], 16, dtype=dtype) # else: # self.A = NpShared.GiveArray(FromSharedId) #pyfftw.interfaces.cache.enable() #pyfftw.interfaces.cache.set_keepalive_time(3000) self.ncores = ncores or NCPU_global #print "plan" T = ClassTimeIt.ClassTimeIt("ModFFTW") T.disable() #self.A = pyfftw.interfaces.numpy_fft.fft2(self.A, axes=(-1,-2),overwrite_input=True, planner_effort='FFTW_MEASURE', threads=self.ncores) T.timeit("planF") #self.A = pyfftw.interfaces.numpy_fft.ifft2(self.A, axes=(-1,-2),overwrite_input=True, planner_effort='FFTW_MEASURE', threads=self.ncores) T.timeit("planB") #print "done" self.ThisType = dtype self.norm = norm
def JHy(self, y): T = ClassTimeIt.ClassTimeIt("JHy") T.disable() v = np.zeros((self.LMode, self.na), np.complex64) for iAnt in range(self.na): v[0, iAnt] += np.sum(self.J_TEC[self.indA0[iAnt]].conj() * y[self.indA0[iAnt]]) v[0, iAnt] += np.sum(-self.J_TEC[self.indA1[iAnt]].conj() * y[self.indA1[iAnt]]) if "CPhase" in self.Mode: v[1, iAnt] += np.sum(self.J_Phase[self.indA0[iAnt]].conj() * y[self.indA0[iAnt]]) v[1, iAnt] += np.sum(-self.J_Phase[self.indA1[iAnt]].conj() * y[self.indA1[iAnt]]) v = v.ravel() T.timeit("Prod") # PSparse=np.array(np.dot(self.J.T.conj(),scipy.sparse.coo_matrix(y.reshape((-1,1)))).todense()).ravel() # T.timeit("PSparse") return v
def ConvolveVector(self, A, Norm=True, OutMode="Data"): sh = A.shape if OutMode == "Data": OutSize = self.NPixListData elif OutMode == "Parms": OutSize = self.NPixListParms ConvA = np.zeros((self.NFreqBands, 1, OutSize), np.float32) T = ClassTimeIt.ClassTimeIt("Vec") T.disable() for iPix in range(self.NPixListParms): Fch = A[:, iPix] if np.abs(Fch).max() == 0: continue Vec_iPix = self.GiveConvVector(iPix, TypeOut=OutMode) T.timeit("GetVec") for iBand in range(self.NFreqBands): F = Fch[iBand] ConvA[iBand] += F * Vec_iPix[iBand] T.timeit("Sum") return ConvA
def fft(self, A, ChanList=None): axes = (-1, -2) T = ClassTimeIt.ClassTimeIt("ModFFTW") T.disable() nch, npol, n, n = A.shape if ChanList is not None: CSel = ChanList else: CSel = range(nch) for ich in CSel: for ipol in range(npol): B = iFs(A[ich, ipol].astype(A.dtype), axes=axes) T.timeit("shift and copy") B = np.fft.fft2(B, axes=axes) T.timeit("fft") A[ich, ipol] = Fs(B, axes=axes) / (A.shape[-1] * A.shape[-2]) T.timeit("shift") return A
def Deconvolve(self, ch=0, **kwargs): """ Runs minor cycle over image channel 'ch'. initMinor is number of minor iteration (keeps continuous count through major iterations) Nminor is max number of minor iterations Returns tuple of: return_code,continue,updated where return_code is a status string; continue is True if another cycle should be executed (one or more polarizations still need cleaning); update is True if one or more polarization models have been updated """ #No need to set the channel when doing joint deconvolution self.setChannel(ch) exit_msg = "" continue_deconvolution = False update_model = False _, npix, _ = self.Dirty.shape xc = (npix) / 2 npol, _, _ = self.Dirty.shape # Get the PeakMap (first index will always be 0 because we only support I cleaning) PeakMap = self.Dirty[0, :, :] m0, m1 = PeakMap.min(), PeakMap.max() #These options should probably be moved into MinorCycleConfig in parset DoAbs = int(self.GD["Deconv"]["AllowNegative"]) print >> log, " Running minor cycle [MinorIter = %i/%i, SearchMaxAbs = %i]" % ( self._niter, self.MaxMinorIter, DoAbs) ## Determine which stopping criterion to use for flux limit #Get RMS stopping criterion NPixStats = self.GD["Deconv"]["NumRMSSamples"] if NPixStats: RandomInd = np.int64(np.random.rand(NPixStats) * npix**2) RMS = np.std(np.real(PeakMap.ravel()[RandomInd])) else: RMS = np.std(PeakMap) self.RMS = RMS self.GainMachine.SetRMS(RMS) Fluxlimit_RMS = self.RMSFactor * RMS #Find position and intensity of first peak x, y, MaxDirty = NpParallel.A_whereMax(PeakMap, NCPU=self.NCPU, DoAbs=DoAbs, Mask=self.MaskArray) #Get peak factor stopping criterion Fluxlimit_Peak = MaxDirty * self.PeakFactor #Get side lobe stopping criterion Fluxlimit_Sidelobe = ( (self.CycleFactor - 1.) / 4. * (1. - self.SideLobeLevel) + self.SideLobeLevel) * MaxDirty if self.CycleFactor else 0 mm0, mm1 = PeakMap.min(), PeakMap.max() # Choose whichever threshold is highest StopFlux = max(Fluxlimit_Peak, Fluxlimit_RMS, Fluxlimit_Sidelobe, self.FluxThreshold) print >> log, " Dirty image peak flux = %10.6f Jy [(min, max) = (%.3g, %.3g) Jy]" % ( MaxDirty, mm0, mm1) print >> log, " RMS-based threshold = %10.6f Jy [rms = %.3g Jy; RMS factor %.1f]" % ( Fluxlimit_RMS, RMS, self.RMSFactor) print >> log, " Sidelobe-based threshold = %10.6f Jy [sidelobe = %.3f of peak; cycle factor %.1f]" % ( Fluxlimit_Sidelobe, self.SideLobeLevel, self.CycleFactor) print >> log, " Peak-based threshold = %10.6f Jy [%.3f of peak]" % ( Fluxlimit_Peak, self.PeakFactor) print >> log, " Absolute threshold = %10.6f Jy" % ( self.FluxThreshold) print >> log, " Stopping flux = %10.6f Jy [%.3f of peak ]" % ( StopFlux, StopFlux / MaxDirty) T = ClassTimeIt.ClassTimeIt() T.disable() ThisFlux = MaxDirty #print x,y if ThisFlux < StopFlux: print >> log, ModColor.Str( " Initial maximum peak %g Jy below threshold, we're done CLEANing" % (ThisFlux), col="green") exit_msg = exit_msg + " " + "FluxThreshold" continue_deconvolution = False or continue_deconvolution update_model = False or update_model # No need to do anything further if we are already at the stopping flux return exit_msg, continue_deconvolution, update_model # set peak in GainMachine (deprecated?) self.GainMachine.SetFluxMax(ThisFlux) # def GivePercentDone(ThisMaxFlux): # fracDone=1.-(ThisMaxFlux-StopFlux)/(MaxDirty-StopFlux) # return max(int(round(100*fracDone)),100) #Do minor cycle deconvolution loop try: for i in range(self._niter + 1, self.MaxMinorIter + 1): self._niter = i #grab a new peakmap PeakMap = self.Dirty[0, :, :] x, y, ThisFlux = NpParallel.A_whereMax(PeakMap, NCPU=self.NCPU, DoAbs=DoAbs, Mask=self.MaskArray) # deprecated? self.GainMachine.SetFluxMax(ThisFlux) T.timeit("max0") if ThisFlux <= StopFlux: print >> log, ModColor.Str( " CLEANing [iter=%i] peak of %.3g Jy lower than stopping flux" % (i, ThisFlux), col="green") cont = ThisFlux > self.FluxThreshold if not cont: print >> log, ModColor.Str( " CLEANing [iter=%i] absolute flux threshold of %.3g Jy has been reached" % (i, self.FluxThreshold), col="green", Bold=True) exit_msg = exit_msg + " " + "MinFluxRms" continue_deconvolution = cont or continue_deconvolution update_model = True or update_model break # stop cleaning if threshold reached # This is used to track Cleaning progress rounded_iter_step = 1 if i < 10 else (10 if i < 200 else ( 100 if i < 2000 else 1000)) # min(int(10**math.floor(math.log10(i))), 10000) if i >= 10 and i % rounded_iter_step == 0: # if self.GD["Debug"]["PrintMinorCycleRMS"]: #rms = np.std(np.real(self._CubeDirty.ravel()[self.IndStats])) print >> log, " [iter=%i] peak residual %.3g" % ( i, ThisFlux) nch, npol, _, _ = self._Dirty.shape #Fpol contains the intensities at (x,y) per freq and polarisation Fpol = np.zeros([nch, npol, 1, 1], dtype=np.float32) if self.MultiFreqMode: if self.GD["Hogbom"]["FreqMode"] == "Poly": Ncoeffs = self.GD["Hogbom"]["PolyFitOrder"] elif self.GD["Hogbom"]["FreqMode"] == "GPR": Ncoeffs = self.GD["Hogbom"]["NumBasisFuncs"] else: raise NotImplementedError( "FreqMode %s not supported" % self.GD["Hogbom"]["FreqMode"]) Coeffs = np.zeros([npol, Ncoeffs]) else: Coeffs = np.zeros([npol, nch]) # to support per channel cleaning # Get the JonesNorm JonesNorm = (self.DicoDirty["JonesNorm"][:, :, x, y]).reshape( (nch, npol, 1, 1)) # Get the solution Fpol[:, 0, 0, 0] = self._Dirty[:, 0, x, y] / np.sqrt( JonesNorm[:, 0, 0, 0]) # Fit a polynomial to get coeffs # tmp = self.ModelMachine.FreqMachine.Fit(Fpol[:, 0, 0, 0]) # print tmp.shape Coeffs[0, :] = self.ModelMachine.FreqMachine.Fit(Fpol[:, 0, 0, 0]) # Overwrite with polynoimial fit Fpol[:, 0, 0, 0] = self.ModelMachine.FreqMachine.Eval(Coeffs[0, :]) T.timeit("stuff") #Find PSF corresponding to location (x,y) self.PSFServer.setLocation( x, y) #Selects the facet closest to (x,y) PSF, meanPSF = self.PSFServer.GivePSF() #Gives associated PSF _, _, PSFnx, PSFny = PSF.shape # Normalise PSF in each channel PSF /= np.amax(PSF.reshape(nch, npol, PSFnx * PSFny), axis=2, keepdims=True).reshape(nch, npol, 1, 1) T.timeit("FindScale") CurrentGain = self.GainMachine.GiveGain() #Update model self.ModelMachine.AppendComponentToDictStacked((x, y), 1.0, Coeffs[0, :], 0) # Subtract LocalSM*CurrentGain from dirty image self.SubStep((x, y), PSF * Fpol * CurrentGain * np.sqrt(JonesNorm)) T.timeit("SubStep") T.timeit("End") except KeyboardInterrupt: print >> log, ModColor.Str( " CLEANing [iter=%i] minor cycle interrupted with Ctrl+C, peak flux %.3g" % (self._niter, ThisFlux)) exit_msg = exit_msg + " " + "MaxIter" continue_deconvolution = False or continue_deconvolution update_model = True or update_model return exit_msg, continue_deconvolution, update_model if self._niter >= self.MaxMinorIter: #Reached maximum number of iterations: print >> log, ModColor.Str( " CLEANing [iter=%i] Reached maximum number of iterations, peak flux %.3g" % (self._niter, ThisFlux)) exit_msg = exit_msg + " " + "MaxIter" continue_deconvolution = False or continue_deconvolution update_model = True or update_model return exit_msg, continue_deconvolution, update_model
def get(self, times, uvw, visIn, flag, A0A1, ModelImage, PointingID=0, Row0Row1=(0, -1), DicoJonesMatrices=None, freqs=None, ImToGrid=True, TranformModelInput="", ChanMapping=None, sparsification=None): T = ClassTimeIt.ClassTimeIt("get") T.disable() vis = visIn.view() A0, A1 = A0A1 T.timeit("0") if ImToGrid: if np.max(np.abs(ModelImage)) == 0: return vis Grid = self.dtype(self.setModelIm(ModelImage)) else: Grid = ModelImage if ChanMapping is None: ChanMapping = np.zeros((visIn.shape[1], ), np.int32) if TranformModelInput == "FT": if np.max(np.abs(ModelImage)) == 0: return vis if self.GD["RIME"]["Precision"] == "S": Cast = np.complex64 elif self.GD["RIME"]["Precision"] == "D": Cast = np.complex128 Grid = np.complex64(self.getFFTWMachine().fft(Cast(ModelImage))) if freqs.size > 1: df = freqs[1::] - freqs[0:-1] ddf = np.abs(df - np.mean(df)) ChanEquidistant = int(np.max(ddf) < 1.) else: ChanEquidistant = 0 # np.save("Grid",Grid) NVisChan = visIn.shape[1] self.ChanMappingDegrid = np.int32(ChanMapping) self.SumJonesChan = np.zeros((2, NVisChan), np.float64) T.timeit("1") npol = self.npol NChan = self.NChan SumWeigths = self.SumWeigths if vis.shape != flag.shape: raise Exception('vis[%s] and flag[%s] should have the same shape' % (str(vis.shape), str(flag.shape))) l0, m0 = self.lmShift FacetInfos = np.float64( np.array([self.WTerm.Cu, self.WTerm.Cv, l0, m0, self.IDFacet])) Row0, Row1 = Row0Row1 if Row1 == -1: Row1 = uvw.shape[0] RowInfos = np.array([Row0, Row1]).astype(np.int32) T.timeit("2") self.CheckTypes(Grid=Grid, vis=vis, uvw=uvw, flag=flag, ListWTerm=self.WTerm.Wplanes) ParamJonesList = [] if DicoJonesMatrices is not None: ApplyAmp = 0 ApplyPhase = 0 ScaleAmplitude = 0 CalibError = 0. if "A" in self.GD["DDESolutions"]["DDModeDeGrid"]: ApplyAmp = 1 if "P" in self.GD["DDESolutions"]["DDModeDeGrid"]: ApplyPhase = 1 if self.GD["DDESolutions"]["ScaleAmpDeGrid"]: ScaleAmplitude = 1 CalibError = (self.GD["DDESolutions"]["CalibErr"] / 3600.) * np.pi / 180 LApplySol = [ApplyAmp, ApplyPhase, ScaleAmplitude, CalibError] LSumJones = [self.SumJones] LSumJonesChan = [self.SumJonesChan] ParamJonesList = self.GiveParamJonesList(DicoJonesMatrices, times, A0, A1, uvw, degridder=True) ParamJonesList = ParamJonesList+LApplySol+LSumJones+LSumJonesChan + \ [np.float32(self.GD["DDESolutions"]["ReWeightSNR"])] T.timeit("3") #print vis #print "DEGRID:",Grid.shape,ChanMapping if self.GD["RIME"]["ForwardMode"] == "Classic": _ = _pyGridder.pyDeGridderWPol( Grid, vis, uvw, flag, SumWeigths, 0, self.WTerm.WplanesConj, self.WTerm.Wplanes, np.array([ self.WTerm.RefWave, self.WTerm.wmax, len(self.WTerm.Wplanes), self.WTerm.OverS ], dtype=np.float64), self.incr.astype(np.float64), freqs, [self.PolMap, FacetInfos, RowInfos, ChanMapping], ParamJonesList, self.LSmear) elif self.GD["RIME"]["ForwardMode"] == "BDA-degrid": # OptimisationInfos=[self.FullScalarMode,self.ChanEquidistant] OptimisationInfos = [ self.JonesType, ChanEquidistant, self.SkyType, self.PolModeID ] # MapSmear = NpShared.GiveArray( # "%sBDA.DeGrid" % # (self.ChunkDataCache)) vis = _pyGridderSmear.pyDeGridderWPol( Grid, vis, uvw, flag, SumWeigths, 0, self.WTerm.WplanesConj, self.WTerm.Wplanes, np.array([ self.WTerm.RefWave, self.WTerm.wmax, len(self.WTerm.Wplanes), self.WTerm.OverS ], dtype=np.float64), self.incr.astype(np.float64), freqs, [self.PolMap, FacetInfos, RowInfos], ParamJonesList, self._bda_degrid, sparsification if sparsification is not None else np.array([], dtype=np.bool), OptimisationInfos, self.LSmear, np.int32(ChanMapping), np.array(self.DataCorrelationFormat).astype(np.uint16), np.array(self.ExpectedOutputStokes).astype(np.uint16)) elif self.GD["RIME"]["ForwardMode"] == "BDA-degrid-classic": OptimisationInfos = [ self.JonesType, ChanEquidistant, self.SkyType, self.PolModeID ] # MapSmear = NpShared.GiveArray( # "%sBDA.DeGrid" % # (self.ChunkDataCache)) vis = _pyGridderSmearClassic.pyDeGridderWPol( Grid, vis, uvw, flag, SumWeigths, 0, self.WTerm.WplanesConj, self.WTerm.Wplanes, np.array([ self.WTerm.RefWave, self.WTerm.wmax, len(self.WTerm.Wplanes), self.WTerm.OverS ], dtype=np.float64), self.incr.astype(np.float64), freqs, [self.PolMap, FacetInfos, RowInfos], ParamJonesList, self._bda_degrid, sparsification if sparsification is not None else np.array([], dtype=np.bool), OptimisationInfos, self.LSmear, np.int32(ChanMapping), np.array(self.DataCorrelationFormat).astype(np.uint16), np.array(self.ExpectedOutputStokes).astype(np.uint16)) else: raise ValueError("unknown --RIME-ForwardMode %s" % self.GD["RIME"]["ForwardMode"]) T.timeit("4 (degrid)") # print vis # uvw,vis=self.ShiftVis(uvwOrig,vis,reverse=False) # T.timeit("5") return vis
def put(self, times, uvw, visIn, flag, A0A1, W=None, PointingID=0, DoNormWeights=True, DicoJonesMatrices=None, freqs=None, DoPSF=0, ChanMapping=None, ResidueGrid=None, sparsification=None): """ Gridding routine, wraps external python extension C gridder Args: times: uvw: visIn: flag: A0A1: W: PointingID: DoNormWeights: DicoJonesMatrices: freqs: DoPSF: ChanMapping: ResidueGrid: Returns: """ vis = visIn T = ClassTimeIt.ClassTimeIt("put") T.disable() self.DoNormWeights = DoNormWeights if not (self.DoNormWeights): self.reinitGrid() if freqs.size > 1: df = freqs[1::] - freqs[0:-1] ddf = np.abs(df - np.mean(df)) ChanEquidistant = int(np.max(ddf) < 1.) else: ChanEquidistant = 0 if ChanMapping is None: ChanMapping = np.zeros((visIn.shape[1], ), np.int64) self.ChanMappingGrid = ChanMapping Grid = ResidueGrid if Grid.dtype != self.dtype: raise TypeError("Grid must be of type " + str(self.dtype)) A0, A1 = A0A1 npol = self.npol NChan = self.NChan NVisChan = vis.shape[1] self.SumJonesChan = np.zeros((2, NVisChan), np.float64) if isinstance(W, type(None)): W = np.ones((uvw.shape[0], NVisChan), dtype=np.float64) SumWeigths = self.SumWeigths if vis.shape != flag.shape: raise Exception('vis[%s] and flag[%s] should have the same shape' % (str(vis.shape), str(flag.shape))) u, v, w = uvw.T l0, m0 = self.lmShift FacetInfos = np.float64( np.array([self.WTerm.Cu, self.WTerm.Cv, l0, m0, self.IDFacet])) self.CheckTypes(Grid=Grid, vis=vis, uvw=uvw, flag=flag, ListWTerm=self.WTerm.Wplanes, W=W) ParamJonesList = [] if DicoJonesMatrices is not None: ApplyAmp = 0 ApplyPhase = 0 ScaleAmplitude = 0 CalibError = 0. if "A" in self.GD["DDESolutions"]["DDModeGrid"]: ApplyAmp = 1 if "P" in self.GD["DDESolutions"]["DDModeGrid"]: ApplyPhase = 1 if self.GD["DDESolutions"]["ScaleAmpGrid"]: ScaleAmplitude = 1 CalibError = (self.GD["DDESolutions"]["CalibErr"] / 3600.) * np.pi / 180 LApplySol = [ApplyAmp, ApplyPhase, ScaleAmplitude, CalibError] LSumJones = [self.SumJones] LSumJonesChan = [self.SumJonesChan] ParamJonesList = self.GiveParamJonesList(DicoJonesMatrices, times, A0, A1, uvw, gridder=True) ParamJonesList = ParamJonesList + LApplySol + LSumJones + LSumJonesChan + [ np.float32(self.GD["DDESolutions"]["ReWeightSNR"]) ] #T2= ClassTimeIt.ClassTimeIt("Gridder") #T2.disable() T.timeit("stuff") if False: # # self.GD["Comp"]["GridMode"] == 0: # really deprecated for now raise RuntimeError("Deprecated flag. Please use BDA gridder") elif self.GD["RIME"]["BackwardMode"] == "BDA-grid": OptimisationInfos = [ self.JonesType, ChanEquidistant, self.SkyType, self.PolModeID ] _pyGridderSmear.pyGridderWPol( Grid, vis, uvw, flag, W, SumWeigths, DoPSF, self.WTerm.Wplanes, self.WTerm.WplanesConj, np.array([ self.WTerm.RefWave, self.WTerm.wmax, len(self.WTerm.Wplanes), self.WTerm.OverS ], dtype=np.float64), self.incr.astype(np.float64), freqs, [self.PolMap, FacetInfos], ParamJonesList, self._bda_grid, sparsification if sparsification is not None else np.array([], dtype=np.bool), OptimisationInfos, self.LSmear, np.int32(ChanMapping), np.array(self.DataCorrelationFormat).astype(np.uint16), np.array(self.ExpectedOutputStokes).astype(np.uint16)) T.timeit("gridder") T.timeit("grid %d" % self.IDFacet) elif self.GD["RIME"]["BackwardMode"] == "BDA-grid-classic": OptimisationInfos = [ self.JonesType, ChanEquidistant, self.SkyType, self.PolModeID ] _pyGridderSmearClassic.pyGridderWPol( Grid, vis, uvw, flag, W, SumWeigths, DoPSF, self.WTerm.Wplanes, self.WTerm.WplanesConj, np.array([ self.WTerm.RefWave, self.WTerm.wmax, len(self.WTerm.Wplanes), self.WTerm.OverS ], dtype=np.float64), self.incr.astype(np.float64), freqs, [self.PolMap, FacetInfos], ParamJonesList, self._bda_grid, sparsification if sparsification is not None else np.array([], dtype=np.bool), OptimisationInfos, self.LSmear, np.int32(ChanMapping), np.array(self.DataCorrelationFormat).astype(np.uint16), np.array(self.ExpectedOutputStokes).astype(np.uint16)) else: raise ValueError("unknown --RIME-BackwardMode %s" % self.GD["RIME"]["BackwardMode"]) T.timeit("gridder") T.timeit("grid %d" % self.IDFacet)
def testGrid(): import pylab # Parset=ReadCFG.Parset("%s/Parset/DefaultParset.cfg"%os.environ["DDFACET_DIR"]) Parset = ReadCFG.Parset("%s/DDFacet/Parset/DefaultParset.cfg" % os.environ["DDFACET_DIR"]) DC = Parset.DicoPars # 19 (-0.01442078294460315, 0.014406238534169863) 2025 3465 -10.0 #(array([0]), array([0]), array([1015]), array([1201])) #(array([0]), array([0]), array([1050]), array([1398])) # 17 (-0.014391694123736577, 0.01437714971330329) 2025 3465 -10.0 #(array([0]), array([0]), array([1030]), array([1303])) npix = 2025 Cell = 1.5 # Cell=.5 offy, offx = 3465 / 2 - 1030, 3465 / 2 - 1303 offx = offx offy = -offy CellRad = (Cell / 3600.) * np.pi / 180 L = offy * (Cell / 3600.) * np.pi / 180 M = -offx * (Cell / 3600.) * np.pi / 180 l0, m0 = -0.014391694123736577, 0.01437714971330329 #l0,m0=-0.009454, 0. L += l0 M += m0 DC["Image"]["Cell"] = Cell DC["Image"]["NPix"] = npix # DC["Image"]["Padding"]=1 DC["Data"]["MS"] = "Simul.MS.W0.tsel" #/media/6B5E-87D0/DDFacet/Test/TestDegridOleg/TestOlegVLA.MS_p0 DC["CF"]["OverS"] = 81 DC["CF"]["Support"] = 9 DC["CF"]["Nw"] = 2 DC["CF"]["wmax"] = 100000. DC["Stores"]["DeleteDDFProducts"] = False # True IdSharedMem = "123." # DC["Selection"]["UVRangeKm"]=[0.2,2000.e6] DC["Comp"]["CompDeGridDecorr"] = 0.0 DC["Image"]["Robust"] = -1 DC["Image"]["Weighting"] = "Briggs" #DC["Comp"]["CompDeGridMode"] = False #DC["Comp"]["CompGridMode"] = False #DC["Comp"]["DegridMode"] = True VS = ClassVisServer.ClassVisServer( DC["Data"]["MS"], ColName=DC["Data"]["ColName"], TVisSizeMin=DC["Data"]["ChunkHours"] * 60 * 1.1, # DicoSelectOptions=DicoSelectOptions, TChunkSize=DC["Data"]["ChunkHours"], Robust=DC["Image"]["Robust"], Weighting=DC["Image"]["Weighting"], Super=DC["Image"]["SuperUniform"], DicoSelectOptions=dict(DC["Selection"]), NCPU=DC["Parallel"]["NCPU"], GD=DC) Padding = DC["Image"]["Padding"] #_,npix=EstimateNpix(npix,Padding) sh = [1, 1, npix, npix] VS.setFOV(sh, sh, sh, CellRad) VS.CalcWeights() Load = VS.LoadNextVisChunk() DATA = VS.VisChunkToShared() # DicoConfigGM={"NPix":NpixFacet, # "Cell":Cell, # "ChanFreq":ChanFreq, # "DoPSF":False, # "Support":Support, # "OverS":OverS, # "wmax":wmax, # "Nw":Nw, # "WProj":True, # "DoDDE":self.DoDDE, # "Padding":Padding} # GM=ClassDDEGridMachine(Parset.DicoPars,DoDDE=False,WProj=True,lmShift=(0.,0.),JonesDir=3,SpheNorm=True,IdSharedMem="caca") # GM=ClassDDEGridMachine(Parset.DicoPars, # IdSharedMem="caca", # **DicoConfigGMself.DicoImager[iFacet]["DicoConfigGM"]) ChanFreq = VS.CurrentMS.ChanFreq.flatten() GM = ClassDDEGridMachine( DC, ChanFreq, npix, lmShift=(l0, m0), # self.DicoImager[iFacet]["lmShift"], IdSharedMem=IdSharedMem) row0 = 0 row1 = DATA["uvw"].shape[0] # -1 uvw = np.float64(DATA["uvw"]) # [row0:row1] # uvw[:,2]=0 times = np.float64(DATA["times"]) # [row0:row1] data = np.complex64(DATA["data"]) # [row0:row1] # data.fill(1.) # data[:,:,0]=1 # data[:,:,3]=1 A0 = np.int32(DATA["A0"]) # [row0:row1] A1 = np.int32(DATA["A1"]) # [row0:row1] DOrig = data.copy() # uvw.fill(0) flag = np.bool8(DATA["flags"]) # [row0:row1,:,:].copy() # ind=np.where(np.logical_not((A0==12)&(A1==14)))[0] # flag[ind,:,:]=1 # flag.fill(0) # ind=np.where(np.logical_not((A0==0)&(A1==27)))[0] # uvw=uvw[ind].copy() # data=data[ind].copy() # flag[ind,:,:]=1 # A0=A0[ind].copy() # A1=A1[ind].copy() # times=times[ind].copy() # MapSmear=NpShared.GiveArray("%sMappingSmearing"%("caca")) # stop # row=19550 # print A0[row],A1[row],flag[row] # stop # DicoJonesMatrices={} # DicoClusterDirs=NpShared.SharedToDico("%sDicoClusterDirs"%IdSharedMem) # DicoJonesMatrices["DicoClusterDirs"]=DicoClusterDirs # DicoJones_Beam=NpShared.SharedToDico("%sJonesFile_Beam"%IdSharedMem) # DicoJonesMatrices["DicoJones_Beam"]=DicoJones_Beam # DicoJonesMatrices["DicoJones_Beam"]["MapJones"]=NpShared.GiveArray("%sMapJones_Beam"%IdSharedMem) DicoJonesMatrices = None T = ClassTimeIt.ClassTimeIt("main") # print "Start" # Grid=GM.put(times,uvw,data,flag,(A0,A1),W=DATA["Weights"],PointingID=0,DoNormWeights=True, DicoJonesMatrices=DicoJonesMatrices) # print "OK" # pylab.clf() # ax=pylab.subplot(1,3,1) # pylab.imshow(np.real(Grid[0,0]),cmap="gray",interpolation="nearest")#,vmin=-600,vmax=600) # G0=(Grid/np.max(Grid)).copy() # pylab.imshow(np.random.rand(50,50)) # #### # GM=ClassDDEGridMachine(DC, # ChanFreq, # npix, # lmShift=(0.,0.),#self.DicoImager[iFacet]["lmShift"], # IdSharedMem=IdSharedMem) # data.fill(1.) # Grid=GM.put(times,uvw,data,flag,(A0,A1),W=DATA["Weights"],PointingID=0,DoNormWeights=True, DicoJonesMatrices=DicoJonesMatrices) # pylab.subplot(1,3,2,sharex=ax,sharey=ax) # pylab.imshow(np.real(Grid[0,0]),cmap="gray",interpolation="nearest")#,vmin=-600,vmax=600) # pylab.subplot(1,3,3,sharex=ax,sharey=ax) # pylab.imshow(np.real(Grid[0,0])-np.real(G0[0,0]),cmap="gray",interpolation="nearest")#,vmin=-600,vmax=600) # pylab.colorbar() # pylab.draw() # pylab.show(False) # return Grid = np.zeros(sh, np.complex64) T.timeit("grid") # Grid[np.isnan(Grid)]=-1 # Grid[0,0,100,100]=10. # Grid.fill(0) _, _, n, n = Grid.shape Grid[:, :, n / 2 + offx, n / 2 + offy] = 10. data.fill(0) #GM.GD["Comp"]["CompDeGridMode"] = True data = GM.get(times, uvw, data, flag, (A0, A1), Grid, freqs=ChanFreq) # , DicoJonesMatrices=DicoJonesMatrices) data0 = -data.copy() # data.fill(0) # GM.GD["Comp"]["CompDeGridMode"] = False # data1=-GM.get(times,uvw,data,flag,(A0,A1),Grid,freqs=ChanFreq)#, # DicoJonesMatrices=DicoJonesMatrices) # ind=np.where(((A0==12)&(A1==14)))[0] # data0=data0[ind] # data1=data1[ind] # print data0-data1 op0 = np.abs op1 = np.imag # op0=np.abs # op1=np.angle nbl = VS.CurrentMS.nbl U, V, W = uvw.T C = 299792458. N = np.sqrt(1. - L**2 - M**2) U = U.reshape(U.size, 1) V = V.reshape(U.size, 1) W = W.reshape(U.size, 1) #L,M=-0.0194966364621, 0.0112573688 ChanFreq = ChanFreq.reshape(1, ChanFreq.size) K = 10. * np.exp(2. * np.pi * 1j * (ChanFreq[0] / C) * (U * L + V * M + W * (N - 1))) # ind=np.where((d0-d1)[:]!=0) #print -0.0194966364621, 0.0112573688 #-0.0194967821858 0.0112573736754 # print L,M ind = range(U.size) # np.where((A0==49)&(A1==55))[0] d0 = data0[ind, -1, 0].ravel() # d1=data1[ind,-1,0].ravel() k = K[ind, -1] # k=DOrig[ind,-1,0].ravel() # d0=data0[:,:,0].ravel() # d1=data1[:,:,0].ravel() # k=K[:,:] X0 = d0.ravel() # X1=d1.ravel() Y = k.ravel() pylab.clf() pylab.subplot(2, 1, 1) # pylab.plot(op0(d0)) pylab.plot(op0(X0)) # pylab.plot(op0(X1)) pylab.plot(op0(Y)) pylab.plot(op0(X0) - op0(Y)) pylab.subplot(2, 1, 2) pylab.plot(op1(X0)) # pylab.plot(op1(X1)) pylab.plot(op1(Y)) pylab.plot(op1(X0) - op1(Y)) pylab.draw() pylab.show()
def __init__( self, GD, ChanFreq, Npix, lmShift=(0., 0.), IDFacet=0, SpheNorm=True, NFreqBands=1, DataCorrelationFormat=[5, 6, 7, 8], ExpectedOutputStokes=[1], ListSemaphores=None, cf_dict=None, compute_cf=False, wmax=None, # must be supplied if compute_cf=True bda_grid=None, bda_degrid=None, ): """ Args: GD: ChanFreq: Npix: lmShift: IdSharedMem: IdSharedMemData: FacetDataCache: ChunkDataCache: IDFacet: SpheNorm: NFreqBands: DataCorrelationFormat: ExpectedOutputStokes: ListSemaphores: cf_dict: SharedDict from/to which WTerms and Sphes are saved compute_cf: if True, wterm/sphe is recomputed and saved to store_dict """ T = ClassTimeIt.ClassTimeIt("Init_ClassDDEGridMachine") T.disable() self.GD = GD self.IDFacet = IDFacet self.SpheNorm = SpheNorm self.ListSemaphores = ListSemaphores self._bda_grid = bda_grid self._bda_degrid = bda_degrid # self.DoPSF=DoPSF self.DoPSF = False # if DoPSF: # self.DoPSF=True # Npix=Npix*2 Precision = GD["RIME"]["Precision"] PolMode = ExpectedOutputStokes if Precision == "S": self.dtype = np.complex64 elif Precision == "D": self.dtype = np.complex128 self.dtype = np.complex64 T.timeit("0") Padding = GD["Facets"]["Padding"] self.NonPaddedNpix, Npix = EstimateNpix(Npix, Padding) self.Padding = Npix / float(self.NonPaddedNpix) # self.Padding=Padding self.LSmear = [] self.PolMode = PolMode # SkyType & JonesType # 0: scalar # 1: diag # 2: full #if PolMode == "I": # self.npol = 1 # self.PolMap = np.array([0, 5, 5, 0], np.int32) # self.SkyType = 1 # self.PolModeID = 0 #elif PolMode == "IQUV": # self.SkyType = 2 # self.npol = 4 # self.PolMap = np.array([0, 1, 2, 3], np.int32) # self.PolModeID = 1 #DEPRICATION: #These are only to be used in the degridder, they are depricated for the gridder self.npol = len(ExpectedOutputStokes) self.SkyType = 1 self.PolMap = np.array([0, 5, 5, 0], np.int32) self.PolModeID = 0 self.Npix = Npix self.NFreqBands = NFreqBands self.NonPaddedShape = (self.NFreqBands, self.npol, self.NonPaddedNpix, self.NonPaddedNpix) self.GridShape = (self.NFreqBands, self.npol, self.Npix, self.Npix) x0 = (self.Npix - self.NonPaddedNpix) / 2 # +1 self.PaddingInnerCoord = (x0, x0 + self.NonPaddedNpix) T.timeit("1") OverS = GD["CF"]["OverS"] Support = GD["CF"]["Support"] Nw = GD["CF"]["Nw"] Cell = GD["Image"]["Cell"] # T=ClassTimeIt.ClassTimeIt("ClassImager") # T.disable() self.Cell = Cell self.incr = (np.array([-Cell, Cell], dtype=np.float64) / 3600.) * (np.pi / 180) # CF.fill(1.) # print self.ChanEquidistant # self.FullScalarMode=int(GD["DDESolutions"]["FullScalarMode"]) # self.FullScalarMode=0 JonesMode = GD["DDESolutions"]["JonesMode"] if JonesMode == "Scalar": self.JonesType = 0 elif JonesMode == "Diag": self.JonesType = 1 elif JonesMode == "Full": self.JonesType = 2 T.timeit("3") self.ChanFreq = ChanFreq self.Sup = Support self.WProj = True self.Nw = Nw self.OverS = OverS self.lmShift = lmShift T.timeit("4") # if neither is set, then machine is being constructed for ffts only if cf_dict or compute_cf: self.InitCF(cf_dict, compute_cf, wmax) T.timeit("5") self.reinitGrid() self.CasaImage = None self.DicoATerm = None T.timeit("6") self.DataCorrelationFormat = DataCorrelationFormat self.ExpectedOutputStokes = ExpectedOutputStokes self._fftw_machine = None
def _convolveSingleGaussianNP(shareddict, field_in, field_out, ch, CellSizeRad, GaussPars_ch, Normalise=False, return_gaussian=False): """Convolves a single channel in a cube of nchan, npol, Ny, Nx @param shareddict: a dictionary containing an input and output array of size [nchans, npols, Ny, Nx] @param field_in: index of input field in shareddict @param field_out: index of the output field in shareddict (can be the same as field_in @param ch: index of channel to convolve @param CellSizeRad: pixel size in radians of the gaussian in image space @param Normalize: Normalize the Gaussian amplitude @param return_gaussian: return the convolving Gaussian as well """ # The FFT needs to be big enough to avoid spectral leakage in the # transforms, so we pad both sides of the stack of images with the same # number of pixels. This preserves the baseband component correctly: # Even - 4 pixels becomes 6 for instance: # | | | x | | => | | | | x | | | # Odd - 3 pixels becomes 5 for instance: # | | x | | => | | | x | | | # IFFTShift will shift the central location down to 0 (middle + 1 and # middle for even and odd respectively). After FFT the baseband is at # 0 as expected. FFTShift can then recentre the FFT. Going back the # IFF is again applied so baseband is at 0, ifft taken and FFTShift # brings the central location back to middle + 1 and middle for even and # odd respectively. The signal can then safely be unpadded T = ClassTimeIt.ClassTimeIt() T.disable() Ain = shareddict[field_in][ch] Aout = shareddict[field_out][ch] T.timeit("init %d" % ch) npol, npix_y, npix_x = Ain.shape assert npix_y == npix_x, "Only supports square grids at the moment" pad_edge = max( int(np.ceil((ModToolBox.EstimateNpix(npix_x)[1] - npix_x) / 2.0) * 2), 0) PSF = np.pad(GiveGauss(npix_x, CellSizeRad, GaussPars_ch, parallel=True), ((pad_edge // 2, pad_edge // 2), (pad_edge // 2, pad_edge // 2)), mode="constant") # PSF=np.ones((Ain.shape[-1],Ain.shape[-1]),dtype=np.float32) if Normalise: PSF /= np.sum(PSF) T.timeit("givegauss %d" % ch) fPSF = np.fft.rfft2(iFs(PSF)) fPSF = np.abs(fPSF) for pol in range(npol): A = iFs( np.pad(Ain[pol], ((pad_edge // 2, pad_edge // 2), (pad_edge // 2, pad_edge // 2)), mode="constant")) fA = np.fft.rfft2(A) nfA = fA * fPSF Aout[pol, :, :] = Fs(np.fft.irfft2( nfA, s=A.shape))[pad_edge // 2:npix_y + pad_edge // 2, pad_edge // 2:npix_x + pad_edge // 2] T.timeit("convolve %d" % ch) if return_gaussian: return Aout, PSF else: return Aout
def _convolveSingleGaussianFFTW(shareddict, field_in, field_out, ch, CellSizeRad, GaussPars_ch, Gauss=None, Normalise=False, nthreads=1, return_gaussian=False): """Convolves a single channel in a cube of nchan, npol, Ny, Nx @param shareddict: a dictionary containing an input and output array of size [nchans, npols, Ny, Nx] @param field_in: index of input field in shareddict @param field_out: index of the output field in shareddict (can be the same as field_in @param ch: index of channel to convolve @param CellSizeRad: pixel size in radians of the gaussian in image space @param nthreads: number of threads to use in FFTW @param Gauss: if set, Gaussian to use (has been precomputed) @param Normalize: Normalize the gaussian amplitude @param return_gaussian: return the convolving Gaussian as well """ # The FFT needs to be big enough to avoid spectral leakage in the # transforms, so we pad both sides of the stack of images with the same # number of pixels. This preserves the baseband component correctly: # Even - 4 pixels becomes 6 for instance: # | | | x | | => | | | | x | | | # Odd - 3 pixels becomes 5 for instance: # | | x | | => | | | x | | | # IFFTShift will shift the central location down to 0 (middle + 1 and # middle for even and odd respectively). After FFT the baseband is at # 0 as expected. FFTShift can then recentre the FFT. Going back the # IFF is again applied so baseband is at 0, ifft taken and FFTShift # brings the central location back to middle + 1 and middle for even and # odd respectively. The signal can then safely be unpadded T = ClassTimeIt.ClassTimeIt() T.disable() Ain = shareddict[field_in][ch] Aout = shareddict[field_out][ch] T.timeit("init %d" % ch) if Gauss is not None: PSF = Gauss else: PSF = GiveConvolvingGaussian(Ain.shape, CellSizeRad, GaussPars_ch, Normalise=Normalise) npol, npix_y, npix_x = Ain.shape pad_edge = max( int(np.ceil((ModToolBox.EstimateNpix(npix_x)[1] - npix_x) / 2.0) * 2), 0) T.timeit("givegauss %d" % ch) fPSF = pyfftw.interfaces.numpy_fft.rfft2(iFs(PSF), overwrite_input=True, threads=nthreads) fPSF = np.abs(fPSF) for pol in range(npol): A = iFs( np.pad(Ain[pol], ((pad_edge // 2, pad_edge // 2), (pad_edge // 2, pad_edge // 2)), mode="constant")) fA = pyfftw.interfaces.numpy_fft.rfft2(A, overwrite_input=True, threads=nthreads) nfA = fA * fPSF Aout[pol, :, :] = Fs( pyfftw.interfaces.numpy_fft.irfft2( nfA, s=A.shape, overwrite_input=True, threads=nthreads))[pad_edge // 2:pad_edge // 2 + npix_y, pad_edge // 2:pad_edge // 2 + npix_x] T.timeit("convolve %d" % ch) if return_gaussian: return Aout, PSF else: return Aout
def giveDicoInitIndiv(self, ListIslands, ModelImage, DicoDirty, ListDoIsland=None, Parallel=True): NCPU = self.NCPU work_queue = multiprocessing.JoinableQueue() ListIslands = ListIslands #[300:308] DoIsland = True for iIsland in range(len(ListIslands)): if ListDoIsland is not None: DoIsland = ListDoIsland[iIsland] if DoIsland: work_queue.put({"iIsland": iIsland}) result_queue = multiprocessing.JoinableQueue() NJobs = work_queue.qsize() workerlist = [] logger.setSilent(SilentModules) #MyLogger.setLoud(SilentModules) #MyLogger.setLoud("ClassImageDeconvMachineMSMF") print >> log, "Launch MORESANE workers" for ii in range(NCPU): W = WorkerInitMSMF(work_queue, result_queue, self.GD, self.DicoVariablePSF, DicoDirty, self.RefFreq, self.GridFreqs, self.DegridFreqs, self.MainCache, ModelImage, ListIslands, self.IdSharedMem) workerlist.append(W) if Parallel: workerlist[ii].start() timer = ClassTimeIt.ClassTimeIt() pBAR = ProgressBar(Title=" MORESANing islands ") #pBAR.disable() pBAR.render(0, NJobs) iResult = 0 if not Parallel: for ii in range(NCPU): workerlist[ii].run() # just run until all work is completed self.DicoInitIndiv = {} while iResult < NJobs: DicoResult = None if result_queue.qsize() != 0: try: DicoResult = result_queue.get() except: pass if DicoResult == None: time.sleep(0.5) continue if DicoResult["Success"]: iResult += 1 NDone = iResult pBAR.render(NDone, NJobs) iIsland = DicoResult["iIsland"] NameDico = "%sDicoInitIsland_%5.5i" % (self.IdSharedMem, iIsland) Dico = NpShared.SharedToDico(NameDico) self.DicoInitIndiv[iIsland] = copy.deepcopy(Dico) NpShared.DelAll(NameDico) if Parallel: for ii in range(NCPU): workerlist[ii].shutdown() workerlist[ii].terminate() workerlist[ii].join() #MyLogger.setLoud(["pymoresane.main"]) #MyLogger.setLoud(["ClassImageDeconvMachineMSMF","ClassPSFServer","ClassMultiScaleMachine","GiveModelMachine","ClassModelMachineMSMF"]) return self.DicoInitIndiv