def fix_dico_shape(fulldico, outdico, NPixOut): # dico_model = 'image_full_ampphase_di_m.NS.DicoModel' # save_dico = dico_model.replace('.DicoModel', '.restricted.DicoModel') # NPixOut = 10000 dico = MyPickle.Load(fulldico) NPix = dico['ModelShape'][-1] NPix0, _ = EstimateNpix(float(NPix), Padding=1) if NPix != NPix0: raise ValueError("NPix != NPix0") logger.info("Changing image size: %i -> %i pixels" % (NPix, NPixOut)) xc0 = NPix // 2 xc1 = NPixOut // 2 dx = xc0 - xc1 DCompOut = {} for k, v in dico.items(): if k == 'Comp': DCompOut['Comp'] = {} continue DCompOut[k] = v DCompOut["Type"] = "SSD" N, M, _, _ = dico['ModelShape'] DCompOut['ModelShape'] = [N, M, NPixOut, NPixOut] for (x0, y0) in dico['Comp'].keys(): x1 = x0 - dx y1 = y0 - dx c0 = (x1 >= 0) & (x1 < NPixOut) c1 = (y1 >= 0) & (y1 < NPixOut) if c0 & c1: logger.info("Mapping (%i,%i)->(%i,%i)" % (x0, y0, x1, y1)) DCompOut['Comp'][(x1, y1)] = dico['Comp'][(x0, y0)] logger.info("Saving in {}".format(outdico)) MyPickle.Save(DCompOut, outdico)
def read_options(): default_values = Parset.value_dict attrs = Parset.attr_dict desc = """Questions and suggestions: [email protected]""" OP = MyOptParse.MyOptParse(usage='Usage: %prog [parset file] <options>', version='%prog version '+report_version(), description=desc, defaults=default_values, attributes=attrs) # create options based on contents of parset for section in Parset.sections: values = default_values[section] # "_Help" value in each section is its documentation string OP.OptionGroup(values.get("_Help", section), section) for name, value in getattr(default_values[section], "iteritems", default_values[section].items)(): if not attrs[section][name].get("no_cmdline"): OP.add_option(name, value) OP.Finalise() OP.ReadInput() # #optcomplete.autocomplete(opt) # options, arguments = opt.parse_args() MyPickle.Save(OP, SaveFile) return OP
def main(options=None): if options == None: f = open(SaveName, 'rb') options = pickle.load(f) ListDirSpec = [ l.strip() for l in open(options.DirsDynSpecMSProds).readlines() if not ".tgz" in l ] D = {} sdb = surveys_db.SurveysDB() for iFile, f in enumerate(ListDirSpec): OBSID = str(f.split("_L")[1]) Di = sdb.get_observation(OBSID) Field = Di["field"] D[iFile] = {} D[iFile]["OBSID"] = str(OBSID) D[iFile]["Field"] = str(Field) sdb.close() MyPickle.Save(D, "LoTSS_OBSID_vs_Field.Dico") for iFile in list(D.keys()): OBSID = D[iFile]["OBSID"] Field = D[iFile]["Field"] ss = "ms2dynspec.py --imageI /databf/lofar/SURVEYS_KSP/LOTSS/DynSpecMS/IMAGES/%s/image_full_ampphase_di_m.NS_shift.int.facetRestored.fits --imageV /databf/lofar/SURVEYS_KSP/LOTSS/DynSpecMS/IMAGES/%s/image_full_low_stokesV.dirty.fits --BaseDirSpecs /databf/lofar/SURVEYS_KSP/LOTSS/DynSpecMS/DynSpecs_L%s --srclist Transient_LOTTS.csv" % ( Field, Field, OBSID) print(ss) os.system(ss)
def main(args=None, messages=[]): if args is None: args = MyPickle.Load(SaveFile) MSList = expandMSList(args.ms) MSList = [mstuple[0] for mstuple in MSList] D = ClassDynSpecMS(ListMSName=MSList, ColName=args.data, ModelName=args.model, SolsName=args.sols, UVRange=args.uv, FileCoords=args.srclist, Radius=args.rad, NOff=args.noff, Image=args.image, SolsDir=args.SolsDir, NCPU=args.NCPU) if D.NDirSelected == 0: return D.StackAll() SaveMachine = ClassSaveResults.ClassSaveResults(D) SaveMachine.WriteFits() SaveMachine.PlotSpec() SaveMachine.tarDirectory()
def Cluster(self): l, m = self.radec2lm(self.Cat.ra, self.Cat.dec) S = self.Cat.S.copy() PolyList = None self.BigPolygon = [] PolyList = [] if self.AvoidPolygons != "": print("Reading polygon file: %s" % self.AvoidPolygons, file=log) PolyList += MyPickle.Load(self.AvoidPolygons) if len(PolyList) > 0: LPoly = [] inside = np.zeros((l.size, ), np.float32) for iPolygon, Poly in enumerate(PolyList): ra, dec = Poly.T lp, mp = self.radec2lm(ra, dec) Poly[:, 0] = lp Poly[:, 1] = mp P = Polygon.Polygon(Poly) if P.area() > self.BigPolygonSize: self.BigPolygon.append(Poly) for ip in range(l.size): if P.isInside(l[ip], m[ip]): inside[ip] = 1 # l=l[inside==0] # m=m[inside==0] # S=S[inside==0] print("There are %i big polygons" % len(self.BigPolygon), file=log) if self.CentralRadius > 0: print("Create central polygon with radius %f degrees" % self.CentralRadius, file=log) Rad = self.CentralRadius * np.pi / 180 th = np.arange(0, 2. * np.pi, 2. * np.pi / 100) lp = np.cos(th) * Rad mp = np.sin(th) * Rad Poly = np.zeros((lp.size, 2), np.float32) Poly[:, 0] = lp Poly[:, 1] = mp PolyList += [Poly] CC = Sky.ClassClusterDEAP.ClassCluster(l, m, S, nNode=self.NCluster, NGen=self.NGen, NPop=self.NPop, DoPlot=self.DoPlot, PolyCut=self.PolyCut, NCPU=self.NCPU, BigPolygon=self.BigPolygon) CC.setAvoidPolygon(PolyList) xyNodes, self.LPolygon = CC.Cluster() nNodes = xyNodes.size // 2 xc, yc = xyNodes.reshape((2, nNodes)) self.xcyc = xc, yc
def reinitPop2(self, pop): print >> log, "Initialise population" x0, x1 = self.x.min(), self.x.max() y0, y1 = self.y.min(), self.y.max() N = len(pop) # p=np.load("pop.npy") # for i,ii in enumerate(p): pop[i][:]=ii[:] # return pop0 = pop[0:N / 2] pop1 = pop[N / 2:] for iIndiv, Indiv in enumerate(pop0): #print iIndiv,len(pop0) x, y = Indiv.reshape((2, self.nNode)) indSel = DDFacet.ToolsDir.GeneDist.GiveNonRedundantSample( self.S, self.nNode) x[:] = self.x[indSel] y[:] = self.y[indSel] for iIndiv, Indiv in enumerate(pop1): #print iIndiv,len(pop1) x, y = Indiv.reshape((2, self.nNode)) x[:] = np.random.uniform(x0, x1, self.nNode) y[:] = np.random.uniform(y0, y1, self.nNode) MyPickle.Save(pop, "pop.myPickle")
def ToFile(self, FileName, DicoIn=None): print("Saving dico model to %s" % FileName, file=log) if DicoIn is None: D = self.DicoSMStacked else: D = DicoIn D["ListScales"] = self.ListScales D["ModelShape"] = self.ModelShape MyPickle.Save(D, FileName)
def GiveMMFromFile(self,FileName=None): """ Initialise a model machine from a file Input: FileName = The file to read """ if FileName is not None: DicoSMStacked = MyPickle.Load(FileName) return self.GiveMMFromDico(DicoSMStacked) else: return self.GiveMMFromDico()
def ToFile(self, FileName, DicoIn=None): print >> log, "Saving dico model to %s" % FileName if DicoIn is None: D = self.DicoSMStacked else: D = DicoIn D["GD"] = self.GD D["Type"] = "HMP" D["ListScales"] = self.ListScales D["ModelShape"] = self.ModelShape MyPickle.Save(D, FileName)
def ToFile(self, FileName, DicoIn=None): print >> log, "Saving dico model to %s" % FileName if DicoIn is None: D = self.DicoModel else: D = DicoIn D["GD"] = self.GD D["ModelShape"] = self.ModelShape D["Type"] = "MORESANE" MyPickle.Save(D, FileName)
def GiveInitialisedMMFromFile(self, FileName): """ Initialise a model machine from a file Input: FileName = The file to read """ DicoSMStacked = MyPickle.Load(FileName) if self.GD is None: self.GD = DicoSMStacked["GD"] MM = self.GiveMMFromDico(DicoSMStacked) MM.FromDico(DicoSMStacked) return MM
def ToFile(self, FileName, DicoIn=None): print("Saving dico model to %s" % FileName, file=log) if DicoIn is None: D = self.DicoModel else: D = DicoIn D["GD"] = self.GD D["ModelShape"] = self.ModelShape D["FreqsCube"] = self.GridFreqs D["Type"] = "MUFFIN" MyPickle.Save(D, FileName)
def ToFile(self, FileName, DicoIn=None): print >> log, "Saving dico model to %s" % FileName if DicoIn is None: D = self.DicoSMStacked else: D = DicoIn #D["PM"]=self.PM D["GD"] = self.GD D["ModelShape"] = self.ModelShape D["Type"] = "SSD" D["SolveParam"] = self.SolveParam MyPickle.Save(D, FileName)
def ToFile(self, FileName, DicoIn=None): print("Saving dico model to %s" % FileName, file=log) if DicoIn is None: D = self.DicoSMStacked else: D = DicoIn if self.GD is None: print( "Warning - you are haven't initialised GD before writing to the DicoModel" ) D["GD"] = self.GD D["Type"] = "WSCMS" D["ModelShape"] = self.ModelShape MyPickle.Save(D, FileName)
def ToFile(self, FileName, DicoIn=None): print >> log, "Saving dico model to %s" % FileName if DicoIn is None: D = self.DicoSMStacked else: D = DicoIn D["GD"] = self.GD D["Type"] = "WSCMS" # try: # D["ListScales"] = list(self.ScaleMachine.sigmas) # list containing std of Gaussian components # except: # D["ListScales"] = self.ListScales D["ModelShape"] = self.ModelShape MyPickle.Save(D, FileName)
def GiveSpectralIndexMap(self, threshold=0.1, save_dict=True): # Get the model image IM = self.GiveModelImage(self.FreqMachine.Freqsp) nchan, npol, Nx, Ny = IM.shape # Fit the alpha map self.FreqMachine.FitAlphaMap(IM[:, 0, :, :], threshold=threshold) # should set threshold based on SNR of final residual if save_dict: FileName = self.GD['Output']['Name'] + ".Dicoalpha" print>>log, "Saving componentwise SPI map to %s"%FileName MyPickle.Save(self.FreqMachine.alpha_dict, FileName) return self.FreqMachine.weighted_alpha_map.reshape((1, 1, Nx, Ny))
def ToFile(self, FileName, DicoIn=None): print("Saving dico model to %s" % FileName, file=log) if DicoIn is None: D = self.DicoSMStacked else: D = DicoIn if self.GD is None: print("Warning - You have not initialised self.GD in ModelMachine so " \ "we can't write it to the DicoModel", file=log) D["GD"] = self.GD D["Type"] = "Hogbom" D["ListScales"] = "Delta" D["ModelShape"] = self.ModelShape MyPickle.Save(D, FileName)
def testMF_DATA(): Dico = MyPickle.Load("SaveTest") Dirty = Dico["Dirty"] PSF = Dico["PSF"] FreqsInfo = Dico["FreqsInfo"] IslandBestIndiv = Dico["IslandBestIndiv"] ListPixData = Dico["ListPixData"] ListPixParms = Dico["ListPixParms"] GD = Dico["GD"] FacetID = Dico["FacetID"] IdSharedMem = Dico["IdSharedMem"] iIsland = Dico["iIsland"] #GD["GAClean"]["GASolvePars"]=["S","Alpha","GSig"] #GD["GAClean"]["GASolvePars"]=["S","Alpha"] nch = FreqsInfo["MeanJonesBand"][FacetID].size WeightMeanJonesBand = FreqsInfo["MeanJonesBand"][FacetID].reshape( (nch, 1, 1, 1)) WeightMueller = WeightMeanJonesBand.ravel() WeightMuellerSignal = WeightMueller * FreqsInfo["WeightChansImages"].ravel( ) # IncreaseIslandMachine=ClassIncreaseIsland.ClassIncreaseIsland() # ListPixData=IncreaseIslandMachine.IncreaseIsland(ListPixData,dx=20) # IncreaseIslandMachine=ClassIncreaseIsland.ClassIncreaseIsland() # ListPixData=IncreaseIslandMachine.IncreaseIsland(ListPixData,dx=5) #IslandBestIndiv=np.zeros((len(GD["GAClean"]["GASolvePars"])*len( CEv = ClassEvolveGA(Dirty, PSF, FreqsInfo, ListPixParms=ListPixParms, ListPixData=ListPixData, GD=GD, IslandBestIndiv=IslandBestIndiv, WeightFreqBands=WeightMuellerSignal, iIsland=iIsland, IdSharedMem=IdSharedMem) CEv.main()
def Cluster(self): l,m=self.radec2lm(self.Cat.ra,self.Cat.dec) S=self.Cat.S.copy() PolyList=None if self.AvoidPolygons!="": print>>log,"Reading polygon file: %s"%self.AvoidPolygons self.BigPolygon=[] PolyList=MyPickle.Load(self.AvoidPolygons) LPoly=[] inside=np.zeros((l.size,),np.float32) for iPolygon,Poly in enumerate(PolyList): ra,dec=Poly.T lp,mp=self.radec2lm(ra,dec) Poly[:,0]=lp Poly[:,1]=mp P=Polygon.Polygon(Poly) if P.area()>self.BigPolygonSize: self.BigPolygon.append(Poly) for ip in range(l.size): if P.isInside(l[ip],m[ip]): inside[ip]=1 l=l[inside==0] m=m[inside==0] S=S[inside==0] print>>log,"There are %i big polygons"%len(self.BigPolygon) CC=Sky.ClassClusterDEAP.ClassCluster(l,m,S,nNode=self.NCluster, NGen=self.NGen, NPop=self.NPop, DoPlot=self.DoPlot, PolyCut=self.PolyCut, NCPU=self.NCPU, BigPolygon=self.BigPolygon) CC.setAvoidPolygon(PolyList) xyNodes,self.LPolygon=CC.Cluster() nNodes=xyNodes.size/2 xc,yc=xyNodes.reshape((2,nNodes)) self.xcyc=xc,yc
def test(): P = ClassPrint.ClassPrint() Obj, ValObj = MyPickle.Load("test") #return Obj #ValObj,_=Obj.parse_args() #return ValObj LGroups = Obj.option_groups for Group in LGroups: print(Group.title) option_list = Group.option_list for o in option_list: lopt = o._long_opts[0] oname = lopt.split("--")[-1] V = getattr(ValObj, oname) if V != "": P.Print(oname, V) # strName=%s # print " "oname,V print()
def main(args=None, messages=[]): if args is None: args = MyPickle.Load(SaveFile) MSList=None if args.ms: MSList=expandMSList(args.ms) MSList=[mstuple[0] for mstuple in MSList] D = ClassDynSpecMS(ListMSName=MSList, ColName=args.data, ModelName=args.model, SolsName=args.sols, ColWeights=args.WeightCol, UVRange=args.uv, FileCoords=args.srclist, Radius=args.rad, NOff=args.noff, ImageI=args.imageI, ImageV=args.imageV, SolsDir=args.SolsDir,NCPU=args.NCPU, BaseDirSpecs=args.BaseDirSpecs, BeamModel=args.BeamModel, BeamNBand=args.BeamNBand) if D.NDirSelected==0: return if D.Mode=="Spec": D.StackAll() SaveMachine=ClassSaveResults.ClassSaveResults(D) if D.Mode=="Spec": SaveMachine.WriteFits() SaveMachine.PlotSpec() SaveMachine.SaveCatalog() SaveMachine.tarDirectory() else: SaveMachine.SaveCatalog() SaveMachine.PlotSpec(Prefix="_replot")
default=0, help="Boring?", required=False) parser.add_argument("--image", type=str, default=None, help="Survey image to plot", required=False) parser.add_argument("--uv", type=list, default=[1., 1000.], help="UV range in km [UVmin, UVmax]", required=False) parser.add_argument("--SolsDir", type=str, default="", help="Base directory for the DDE solutions", required=False) parser.add_argument("--NCPU", type=int, default=0, help="NCPU", required=False) args = parser.parse_args() MyPickle.Save(args, SaveFile) ModColor.silent = progressbar.ProgressBar.silent = args.LogBoring main(args)
def FromFile(self,FileName): print>>log, "Reading dico model from file %s"%FileName self.DicoModel=MyPickle.Load(FileName) self.FromDico(self.DicoModel)
def main(OP=None, messages=[]): if OP is None: OP = MyPickle.Load(SaveFile) print "Using settings from %s, then command line." % SaveFile DicoConfig = OP.DicoConfig ImageName = DicoConfig["Output"]["Name"] if not ImageName: raise Exceptions.UserInputError( "--Output-Name not specified, can't continue.") if not DicoConfig["Data"]["MS"]: raise Exceptions.UserInputError( "--Data-MS not specified, can't continue.") # create directory if it exists dirname = os.path.dirname(ImageName) if not os.path.exists(dirname) and not dirname == "": os.mkdir(dirname) # setup logging MyLogger.logToFile(ImageName + ".log", append=DicoConfig["Log"]["Append"]) global log log = MyLogger.getLogger("DDFacet") # disable colors and progressbars if requested ModColor.silent = SkyModel.Other.ModColor.silent = \ progressbar.ProgressBar.silent = \ DicoConfig["Log"]["Boring"] if messages: if not DicoConfig["Log"]["Boring"]: #os.system('clear') logo.print_logo() for msg in messages: print >> log, msg if DicoConfig["Debug"]["Pdb"] == "always": print >> log, "--Debug-Pdb=always: unexpected errors will be dropped into pdb" Exceptions.enable_pdb_on_error( ModColor.Str( "DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n" + "(This is because you're running with --Debug-Pdb set to 'always'.)" )) elif DicoConfig["Debug"][ "Pdb"] == "auto" and not DicoConfig["Log"]["Boring"]: print >> log, "--Debug-Pdb=auto and not --Log-Boring: unexpected errors will be dropped into pdb" Exceptions.enable_pdb_on_error( ModColor.Str( "DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n" + "(This is because you're running with --Debug-Pdb set to 'auto' and --Log-Boring is off.)" )) # print current options OP.Print(dest=log) # enable memory logging MyLogger.enableMemoryLogging(DicoConfig["Log"]["Memory"]) # get rid of old shm arrays from previous runs Multiprocessing.cleanupStaleShm() # initialize random seed from config if set, or else from system time if DicoConfig["Misc"]["RandomSeed"] is not None: print >> log, "random seed=%d (explicit)" % DicoConfig["Misc"][ "RandomSeed"] else: DicoConfig["Misc"]["RandomSeed"] = int(time.time()) print >> log, "random seed=%d (automatic)" % DicoConfig["Misc"][ "RandomSeed"] np.random.seed(DicoConfig["Misc"]["RandomSeed"]) # init NCPU for different bits of parallelism ncpu = DicoConfig["Parallel"]["NCPU"] or psutil.cpu_count() DicoConfig["Parallel"]["NCPU"] = ncpu _pyArrays.pySetOMPNumThreads(ncpu) NpParallel.NCPU_global = ModFFTW.NCPU_global = ncpu numexpr.set_num_threads(ncpu) print >> log, "using up to %d CPUs for parallelism" % ncpu # write parset OP.ToParset("%s.parset" % ImageName) Mode = DicoConfig["Output"]["Mode"] # init semaphores, as they're needed for weight calculation too ClassFacetMachine.ClassFacetMachine.setup_semaphores(DicoConfig) # data machine initialized for all cases except PSF-only mode # psf machine initialized for all cases except Predict-only mode Imager = ClassDeconvMachine.ClassImagerDeconv( GD=DicoConfig, BaseName=ImageName, predict_only=(Mode == "Predict" or Mode == "Subtract"), data=(Mode != "PSF"), psf=(Mode != "Predict" and Mode != "Dirty" and Mode != "Subtract"), readcol=(Mode != "Predict" and Mode != "PSF"), deconvolve=("Clean" in Mode)) Imager.Init() # Imager.testDegrid() # stop if "Predict" in Mode or "Subtract" in Mode: Imager.GivePredict() if "Clean" in Mode: Imager.main() elif "Dirty" in Mode: sparsify = DicoConfig["Comp"]["Sparsification"] if sparsify and isinstance(sparsify, list): sparsify = sparsify[0] Imager.GiveDirty(psf="PSF" in Mode, sparsify=sparsify) elif "PSF" in Mode: sparsify = DicoConfig["Comp"]["Sparsification"] if sparsify and isinstance(sparsify, list): sparsify = sparsify[0] Imager.MakePSF(sparsify=sparsify) elif "RestoreAndShift" == Mode: Imager.RestoreAndShift()
def Restore(self): print("Create restored image", file=log) FWHMFact=2.*np.sqrt(2.*np.log(2.)) FWHMFact=2.*np.sqrt(2.*np.log(2.)) BeamPix=self.BeamPix/FWHMFact sigma_x, sigma_y=BeamPix,BeamPix theta=0. bmaj=np.max([sigma_x, sigma_y])*self.CellArcSec*FWHMFact bmin=np.min([sigma_x, sigma_y])*self.CellArcSec*FWHMFact #bmaj=bmin=0.001666666666666667*3600 #sigma_x= self.FWHMBeam=(bmaj/3600./np.sqrt(2.),bmin/3600./np.sqrt(2.),theta) self.PSFGaussPars = (sigma_x*self.CellSizeRad, sigma_y*self.CellSizeRad, theta) #print "!!!!!!!!!!!!!!!!!!!!" #self.PSFGaussPars = (BeamPix,BeamPix,0) RefFreq=self.ModelMachine.RefFreq df=RefFreq*0.5 # ################################" if self.options.PSFCache!="": import os IdSharedMem=str(int(os.getpid()))+"." MeanModelImage=self.ModelMachine.GiveModelImage(RefFreq) # #imNorm=image("6SBc.KAFCA.restoredNew.fits.6SBc.KAFCA.restoredNew.fits.MaskLarge.fits").getdata() # imNorm=image("6SB.KAFCA.GA.BIC_00.AP.dirty.fits.mask.fits").getdata() # MASK=np.zeros_like(imNorm) # nchan,npol,_,_=MASK.shape # for ch in range(nchan): # for pol in range(npol): # MASK[ch,pol,:,:]=imNorm[ch,pol,:,:].T[::-1,:] # MeanModelImage[MASK==0]=0 # MeanModelImage.fill(0) # MeanModelImage[0,0,100,100]=1 from DDFacet.Imager.GA import ClassSmearSM from DDFacet.Imager import ClassPSFServer self.DicoVariablePSF = MyPickle.FileToDicoNP(self.options.PSFCache) self.PSFServer=ClassPSFServer.ClassPSFServer() self.PSFServer.setDicoVariablePSF(self.DicoVariablePSF,NormalisePSF=True) #return self.Residual,MeanModelImage,self.PSFServer # CasaImage=ClassCasaImage.ClassCasaimage("Model.fits",MeanModelImage.shape,self.Cell,self.radec)#Lambda=(Lambda0,dLambda,self.NBands)) # CasaImage.setdata(MeanModelImage,CorrT=True) # CasaImage.ToFits() # #CasaImage.setBeam((SmoothFWHM,SmoothFWHM,0)) # CasaImage.close() SmearMachine=ClassSmearSM.ClassSmearSM(self.Residual, MeanModelImage*self.SqrtNormImage, self.PSFServer, DeltaChi2=4., IdSharedMem=IdSharedMem, NCPU=self.options.NCPU) SmearedModel=SmearMachine.Smear() SmoothFWHM=self.CellArcSec*SmearMachine.RestoreFWHM/3600. ModelSmearImage="%s.RestoredSmear"%self.BaseImageName CasaImage=ClassCasaImage.ClassCasaimage(ModelSmearImage,SmearedModel.shape,self.Cell,self.radec)#Lambda=(Lambda0,dLambda,self.NBands)) CasaImage.setdata(SmearedModel+self.Residual,CorrT=True) #CasaImage.setdata(SmearedModel,CorrT=True) CasaImage.setBeam((SmoothFWHM,SmoothFWHM,0)) CasaImage.ToFits() CasaImage.close() SmearMachine.CleanUpSHM() stop # ################################" #self.ModelMachine.ListScales[0]["Alpha"]=-0.8 # model image #ModelMachine.GiveModelImage(RefFreq) FEdge=np.linspace(RefFreq-df,RefFreq+df,self.NBands+1) FCenter=(FEdge[0:-1]+FEdge[1::])/2. C=299792458. Lambda0=C/FCenter[-1] dLambda=1 if self.NBands>1: dLambda=np.abs(C/FCenter[0]-C/FCenter[1]) ListRestoredIm=[] Lambda=[Lambda0+i*dLambda for i in range(self.NBands)] ListRestoredImCorr=[] ListModelIm=[] #print C/np.array(Lambda) # restored image for l in Lambda: freq=C/l if self.options.RandomCat: print("Create random catalog... ", file=log) ModelImage=self.GiveRandomModelIm() else: print("Get ModelImage... ", file=log) ModelImage=self.ModelMachine.GiveModelImage(freq) if self.options.ZeroNegComp: print("Zeroing negative componants... ", file=log) ModelImage[ModelImage<0]=0 ListModelIm.append(ModelImage) if self.options.Mode=="App": print(" ModelImage to apparent flux... ", file=log) ModelImage=ModelImage*self.SqrtNormImage print("Convolve... ", file=log) print(" MinMax = [%f , %f] @ freq = %f MHz"%(ModelImage.min(),ModelImage.max(),freq/1e6), file=log) #RestoredImage=ModFFTW.ConvolveGaussianScipy(ModelImage,CellSizeRad=self.CellSizeRad,GaussPars=[self.PSFGaussPars]) if self.options.AddNoise>0.: print("Adding Noise... ", file=log) ModelImage+=np.random.randn(*ModelImage.shape)*self.options.AddNoise RestoredImage,_=ModFFTW.ConvolveGaussianWrapper(ModelImage,Sig=BeamPix) # ======= # #RestoredImage,_=ModFFTW.ConvolveGaussianWrapper(ModelImage,Sig=BeamPix) # def GiveGauss(Sig0,Sig1): # npix=20*int(np.sqrt(Sig0**2+Sig1**2)) # if not npix%2: npix+=1 # dx=npix/2 # x,y=np.mgrid[-dx:dx:npix*1j,-dx:dx:npix*1j] # dsq=x**2+y**2 # return Sig0**2/(Sig0**2+Sig1**2)*np.exp(-dsq/(2.*(Sig0**2+Sig1**2))) # R2=np.zeros_like(ModelImage) # Sig0=BeamPix/np.sqrt(2.) # if self.options.RandomCat: # Sig1=(self.options.RandomCat_SigFactor-1.)*Sig0 # else: # Sig1=0. # nch,npol,_,_=ModelImage.shape # for ch in range(nch): # in1=ModelImage[ch,0] # R2[ch,0,:,:]=scipy.signal.fftconvolve(in1,GiveGauss(Sig0,Sig1), mode='same').real # RestoredImage=R2 # self.header_dict["GSIGMA"]=Sig0 # # print np.max(np.abs(R2-RestoredImage)) # # import pylab # # ax=pylab.subplot(1,3,1) # # pylab.imshow(RestoredImage[0,0],interpolation="nearest") # # pylab.colorbar() # # pylab.subplot(1,3,2,sharex=ax,sharey=ax) # # pylab.imshow(R2[0,0],interpolation="nearest") # # pylab.colorbar() # # pylab.subplot(1,3,3,sharex=ax,sharey=ax) # # pylab.imshow((RestoredImage-R2)[0,0],interpolation="nearest") # # pylab.colorbar() # # pylab.show() # # stop # >>>>>>> 0457182a873da89a2758f4be8a18f55cefd88e44 RestoredImageRes=RestoredImage+self.Residual ListRestoredIm.append(RestoredImageRes) RestoredImageResCorr=RestoredImageRes/self.SqrtNormImage ListRestoredImCorr.append(RestoredImageResCorr) #print FEdge,FCenter print("Save... ", file=log) _,_,nx,_=RestoredImageRes.shape RestoredImageRes=np.array(ListRestoredIm).reshape((self.NBands,1,nx,nx)) RestoredImageResCorr=np.array(ListRestoredImCorr).reshape((self.NBands,1,nx,nx)) ModelImage=np.array(ListModelIm).reshape((self.NBands,1,nx,nx)) if self.OutName=="": ImageName="%s.restoredNew"%self.BaseImageName ImageNameCorr="%s.restoredNew.corr"%self.BaseImageName ImageNameModel="%s.model"%self.BaseImageName ImageNameModelConv="%s.modelConv"%self.BaseImageName else: ImageName=self.OutName ImageNameCorr=self.OutName+".corr" ImageNameModel="%s.model"%self.OutName ImageNameModelConv="%s.modelConv"%self.OutName CasaImage=ClassCasaImage.ClassCasaimage(ImageNameModel,RestoredImageRes.shape,self.Cell,self.radec,header_dict=self.header_dict)#Lambda=(Lambda0,dLambda,self.NBands)) CasaImage.setdata(ModelImage,CorrT=True) CasaImage.setBeam(self.FWHMBeam) CasaImage.ToFits() CasaImage.close() CasaImage=ClassCasaImage.ClassCasaimage(ImageName,RestoredImageRes.shape,self.Cell,self.radec,Freqs=C/np.array(Lambda).ravel(),header_dict=self.header_dict)#,Lambda=(Lambda0,dLambda,self.NBands)) CasaImage.setdata(RestoredImageRes,CorrT=True) CasaImage.setBeam(self.FWHMBeam) CasaImage.ToFits() CasaImage.close() CasaImage=ClassCasaImage.ClassCasaimage(ImageNameModelConv,RestoredImage.shape,self.Cell,self.radec,Freqs=C/np.array(Lambda).ravel(),header_dict=self.header_dict)#,Lambda=(Lambda0,dLambda,self.NBands)) CasaImage.setdata(RestoredImage,CorrT=True) CasaImage.setBeam(self.FWHMBeam) CasaImage.ToFits() CasaImage.close() if self.MakeCorrected: CasaImage=ClassCasaImage.ClassCasaimage(ImageNameCorr,RestoredImageResCorr.shape,self.Cell,self.radec,Freqs=C/np.array(Lambda).ravel(),header_dict=self.header_dict)#,Lambda=(Lambda0,dLambda,self.NBands)) CasaImage.setdata(RestoredImageResCorr,CorrT=True) CasaImage.setBeam(self.FWHMBeam) CasaImage.ToFits() CasaImage.close() # ImageName="%s.modelConv"%self.BaseImageName # CasaImage=ClassCasaImage.ClassCasaimage(ImageName,ModelImage.shape,self.Cell,self.radec) # CasaImage.setdata(self.RestoredImage,CorrT=True) # CasaImage.ToFits() # CasaImage.setBeam(self.FWHMBeam) # CasaImage.close() # Alpha image if self.DoAlpha: print("Get Index Map... ", file=log) IndexMap=self.ModelMachine.GiveSpectralIndexMap(CellSizeRad=self.CellSizeRad,GaussPars=[self.PSFGaussPars]) ImageName="%s.alphaNew"%self.BaseImageName print(" Save... ", file=log) CasaImage=ClassCasaImage.ClassCasaimage(ImageName,ModelImage.shape,self.Cell,self.radec) CasaImage.setdata(IndexMap,CorrT=True) CasaImage.ToFits() CasaImage.close() print(" Done. ", file=log)
def FromFile(self, FileName): print >> log, "Reading dico model from %s" % FileName self.DicoSMStacked = MyPickle.Load(FileName) self.FromDico(self.DicoSMStacked)
def setFacetsLocs(self): NFacets = self.NFacets Npix = self.GD["Image"]["NPix"] Padding = self.GD["Facets"]["Padding"] self.Padding = Padding Npix, _ = EstimateNpix(float(Npix), Padding=1) self.Npix = Npix self.OutImShape = (self.nch, self.npol, self.Npix, self.Npix) RadiusTot = self.CellSizeRad * self.Npix / 2 self.RadiusTot = RadiusTot lMainCenter, mMainCenter = 0., 0. self.lmMainCenter = lMainCenter, mMainCenter self.CornersImageTot = np.array( [[lMainCenter - RadiusTot, mMainCenter - RadiusTot], [lMainCenter + RadiusTot, mMainCenter - RadiusTot], [lMainCenter + RadiusTot, mMainCenter + RadiusTot], [lMainCenter - RadiusTot, mMainCenter + RadiusTot]]) # MSName = self.GD["Data"]["MS"] # if ".txt" in MSName: # f = open(MSName) # Ls = f.readlines() # f.close() # MSName = [] # for l in Ls: # ll = l.replace("\n", "") # MSName.append(ll) # MSName = MSName[0] MSName = self.VS.ListMS[0].MSName SolsFile = self.GD["DDESolutions"]["DDSols"] if isinstance(SolsFile, list): SolsFile = self.GD["DDESolutions"]["DDSols"][0] if SolsFile and (not (".npz" in SolsFile)) and (not (".h5" in SolsFile)): Method = SolsFile ThisMSName = reformat.reformat(os.path.abspath(MSName), LastSlash=False) SolsFile = "%s/killMS.%s.sols.npz" % (ThisMSName, Method) # if "CatNodes" in self.GD.keys(): regular_grid = False if self.GD["Facets"]["CatNodes"] is not None: print >> log, "Taking facet directions from Nodes catalog: %s" % self.GD[ "Facets"]["CatNodes"] ClusterNodes = np.load(self.GD["Facets"]["CatNodes"]) ClusterNodes = ClusterNodes.view(np.recarray) raNode = ClusterNodes.ra decNode = ClusterNodes.dec lFacet, mFacet = self.CoordMachine.radec2lm(raNode, decNode) elif ".npz" in SolsFile: print >> log, "Taking facet directions from solutions file: %s" % SolsFile ClusterNodes = np.load(SolsFile)["ClusterCat"] ClusterNodes = ClusterNodes.view(np.recarray) raNode = ClusterNodes.ra decNode = ClusterNodes.dec lFacet, mFacet = self.CoordMachine.radec2lm(raNode, decNode) elif ".h5" in SolsFile: print >> log, "Taking facet directions from HDF5 solutions file: %s" % SolsFile H = tables.open_file(SolsFile) raNode, decNode = H.root.sol000.source[:]["dir"].T lFacet, mFacet = self.CoordMachine.radec2lm(raNode, decNode) H.close() del (H) else: print >> log, "Taking facet directions from regular grid" regular_grid = True CellSizeRad = (self.GD["Image"]["Cell"] / 3600.) * np.pi / 180 lrad = Npix * CellSizeRad * 0.5 NpixFacet = Npix / NFacets lfacet = NpixFacet * CellSizeRad * 0.5 lcenter_max = lrad - lfacet lFacet, mFacet, = np.mgrid[-lcenter_max:lcenter_max:(NFacets) * 1j, -lcenter_max:lcenter_max:(NFacets) * 1j] lFacet = lFacet.flatten() mFacet = mFacet.flatten() print >> log, " There are %i Jones-directions" % lFacet.size self.lmSols = lFacet.copy(), mFacet.copy() raSols, decSols = self.CoordMachine.lm2radec(lFacet.copy(), mFacet.copy()) self.radecSols = raSols, decSols NodesCat = np.zeros((raSols.size, ), dtype=[('ra', np.float), ('dec', np.float), ('l', np.float), ('m', np.float)]) NodesCat = NodesCat.view(np.recarray) NodesCat.ra = raSols NodesCat.dec = decSols # print>>log,"Facet RA %s"%raSols # print>>log,"Facet Dec %s"%decSols NodesCat.l = lFacet NodesCat.m = mFacet ## saving below # NodeFile = "%s.NodesCat.%snpy" % (self.GD["Output"]["Name"], "psf." if self.DoPSF else "") # print>> log, "Saving Nodes catalog in %s" % NodeFile # np.save(NodeFile, NodesCat) self.DicoImager = {} xy = np.zeros((lFacet.size, 2), np.float32) xy[:, 0] = lFacet xy[:, 1] = mFacet regFile = "%s.tessel0.reg" % self.ImageName NFacets = self.NFacets = lFacet.size rac, decc = self.MainRaDec VM = ModVoronoiToReg.VoronoiToReg(rac, decc) if NFacets > 2: vor = Voronoi(xy, furthest_site=False) regions, vertices = ModVoronoi.voronoi_finite_polygons_2d( vor, radius=1.) PP = Polygon.Polygon(self.CornersImageTot) LPolygon = [] ListNode = [] for region, iNode in zip(regions, range(NodesCat.shape[0])): ThisP = np.array(PP & Polygon.Polygon(np.array(vertices[region]))) if ThisP.size > 0: LPolygon.append(ThisP[0]) ListNode.append(iNode) NodesCat = NodesCat[np.array(ListNode)].copy() # ======= # LPolygon = [ # np.array(PP & Polygon.Polygon(np.array(vertices[region])))[0] # for region in regions] # >>>>>>> issue-255 elif NFacets == 1: l0, m0 = lFacet[0], mFacet[0] LPolygon = [self.CornersImageTot] # VM.ToReg(regFile,lFacet,mFacet,radius=.1) NodeFile = "%s.NodesCat.npy" % self.GD["Output"]["Name"] print >> log, "Saving Nodes catalog in %s" % NodeFile np.save(NodeFile, NodesCat) for iFacet, polygon0 in zip(range(len(LPolygon)), LPolygon): # polygon0 = vertices[region] P = polygon0.tolist() # VM.PolygonToReg(regFile,LPolygon,radius=0.1,Col="red") # stop ########################################### # SubDivide def GiveDiam(polygon): lPoly, mPoly = polygon.T l0 = np.max([lMainCenter - RadiusTot, lPoly.min()]) l1 = np.min([lMainCenter + RadiusTot, lPoly.max()]) m0 = np.max([mMainCenter - RadiusTot, mPoly.min()]) m1 = np.min([mMainCenter + RadiusTot, mPoly.max()]) dl = l1 - l0 dm = m1 - m0 diam = np.max([dl, dm]) return diam, (l0, l1, m0, m1) DiamMax = self.GD["Facets"]["DiamMax"] * np.pi / 180 # DiamMax=4.5*np.pi/180 DiamMin = self.GD["Facets"]["DiamMin"] * np.pi / 180 def ClosePolygon(polygon): P = polygon.tolist() polygon = np.array(P + [P[0]]) return polygon def GiveSubDivideRegions(polygonFacet, DMax): polygonFOV = self.CornersImageTot # polygonFOV=ClosePolygon(polygonFOV) PFOV = Polygon.Polygon(polygonFOV) # polygonFacet=ClosePolygon(polygonFacet) P0 = Polygon.Polygon(polygonFacet) P0Cut = Polygon.Polygon(P0 & PFOV) if P0Cut.nPoints() == 0: return [] polygonFacetCut = np.array(P0Cut[0]) # polygonFacetCut=ClosePolygon(polygonFacetCut) diam, (l0, l1, m0, m1) = GiveDiam(polygonFacetCut) if diam < DMax: return [polygonFacetCut] Nl = int((l1 - l0) / DMax) + 1 Nm = int((m1 - m0) / DMax) + 1 dl = (l1 - l0) / Nl dm = (m1 - m0) / Nm lEdge = np.linspace(l0, l1, Nl + 1) mEdge = np.linspace(m0, m1, Nm + 1) lc = (lEdge[0:-1] + lEdge[1::]) / 2 mc = (mEdge[0:-1] + mEdge[1::]) / 2 LPoly = [] Lc, Mc = np.meshgrid(lc, mc) Lc = Lc.ravel().tolist() Mc = Mc.ravel().tolist() DpolySquare = np.array([[-dl, -dm], [dl, -dm], [dl, dm], [-dl, dm] ]) * 0.5 for lc, mc in zip(Lc, Mc): polySquare = DpolySquare.copy( ) # ClosePolygon(DpolySquare.copy()) polySquare[:, 0] += lc polySquare[:, 1] += mc # polySquare=ClosePolygon(polySquare) P1 = Polygon.Polygon(polySquare) POut = (P0Cut & P1) if POut.nPoints() == 0: continue polyOut = np.array(POut[0]) # polyOut=ClosePolygon(polyOut) LPoly.append(polyOut) # pylab.clf() # x,y=polygonFacetCut.T # pylab.plot(x,y,color="blue") # x,y=polygonFacet.T # pylab.plot(x,y,color="blue",ls=":",lw=3) # x,y=np.array(PFOV[0]).T # pylab.plot(x,y,color="black") # x,y=polySquare.T # pylab.plot(x,y,color="green",ls=":",lw=3) # x,y=polyOut.T # pylab.plot(x,y,color="red",ls="--",lw=3) # pylab.xlim(-0.03,0.03) # pylab.ylim(-0.03,0.03) # pylab.draw() # pylab.show(False) # pylab.pause(0.5) return LPoly def PlotPolygon(P, *args, **kwargs): for poly in P: x, y = ClosePolygon(np.array(poly)).T pylab.plot(x, y, *args, **kwargs) LPolygonNew = [] for iFacet in xrange(len(LPolygon)): polygon = LPolygon[iFacet] ThisDiamMax = DiamMax SubReg = GiveSubDivideRegions(polygon, ThisDiamMax) LPolygonNew += SubReg regFile = "%s.FacetMachine.tessel.ReCut.reg" % self.ImageName # VM.PolygonToReg(regFile,LPolygonNew,radius=0.1,Col="green",labels=[str(i) for i in range(len(LPolygonNew))]) DicoPolygon = {} for iFacet in xrange(len(LPolygonNew)): DicoPolygon[iFacet] = {} poly = LPolygonNew[iFacet] DicoPolygon[iFacet]["poly"] = poly diam, (l0, l1, m0, m1) = GiveDiam(poly) DicoPolygon[iFacet]["diam"] = diam DicoPolygon[iFacet]["diamMin"] = np.min([(l1 - l0), (m1 - m0)]) xc, yc = np.mean(poly[:, 0]), np.mean(poly[:, 1]) DicoPolygon[iFacet]["xyc"] = xc, yc dSol = np.sqrt((xc - lFacet)**2 + (yc - mFacet)**2) DicoPolygon[iFacet]["iSol"] = np.where(dSol == np.min(dSol))[0] for iFacet in sorted(DicoPolygon.keys()): diam = DicoPolygon[iFacet]["diamMin"] # print iFacet,diam,DiamMin if diam < DiamMin: dmin = 1e6 xc0, yc0 = DicoPolygon[iFacet]["xyc"] HasClosest = False for iFacetOther in sorted(DicoPolygon.keys()): if iFacetOther == iFacet: continue iSolOther = DicoPolygon[iFacetOther]["iSol"] # print " ",iSolOther,DicoPolygon[iFacet]["iSol"] if iSolOther != DicoPolygon[iFacet]["iSol"]: continue xc, yc = DicoPolygon[iFacetOther]["xyc"] d = np.sqrt((xc - xc0)**2 + (yc - yc0)**2) if d < dmin: dmin = d iFacetClosest = iFacetOther HasClosest = True if (HasClosest): print >> log, "Merging facet #%i to #%i" % (iFacet, iFacetClosest) P0 = Polygon.Polygon(DicoPolygon[iFacet]["poly"]) P1 = Polygon.Polygon(DicoPolygon[iFacetClosest]["poly"]) P2 = (P0 | P1) POut = [] for iP in xrange(len(P2)): POut += P2[iP] poly = np.array(POut) hull = ConvexHull(poly) Contour = np.array([ hull.points[hull.vertices, 0], hull.points[hull.vertices, 1] ]) poly2 = Contour.T del (DicoPolygon[iFacet]) DicoPolygon[iFacetClosest]["poly"] = poly2 DicoPolygon[iFacetClosest]["diam"] = GiveDiam(poly2)[0] DicoPolygon[iFacetClosest]["xyc"] = np.mean( poly2[:, 0]), np.mean(poly2[:, 1]) # stop LPolygonNew = [] for iFacet in sorted(DicoPolygon.keys()): # if DicoPolygon[iFacet]["diam"]<DiamMin: # print>>log, ModColor.Str(" Facet #%i associated to direction #%i is too small, removing it"%(iFacet,DicoPolygon[iFacet]["iSol"])) # continue LPolygonNew.append(DicoPolygon[iFacet]["poly"]) # for iFacet in range(len(regions)): # polygon=LPolygon[iFacet] # ThisDiamMax=DiamMax # while True: # SubReg=GiveSubDivideRegions(polygon,ThisDiamMax) # if SubReg==[]: # break # Diams=[GiveDiam(poly)[0] for poly in SubReg] # if np.min(Diams)>DiamMin: break # ThisDiamMax*=1.1 # LPolygonNew+=SubReg # print regFile = "%s.tessel.%sreg" % (self.GD["Output"]["Name"], "psf." if self.DoPSF else "") # labels=["[F%i.C%i]"%(i,DicoPolygon[i]["iSol"]) for i in range(len(LPolygonNew))] # VM.PolygonToReg(regFile,LPolygonNew,radius=0.1,Col="green",labels=labels) # VM.PolygonToReg(regFile,LPolygonNew,radius=0.1,Col="green") # pylab.clf() # x,y=LPolygonNew[11].T # pylab.plot(x,y) # pylab.draw() # pylab.show() # stop ########################################### NFacets = len(LPolygonNew) NJonesDir = NodesCat.shape[0] self.JonesDirCat = np.zeros( (NodesCat.shape[0], ), dtype=[('Name', '|S200'), ('ra', np.float), ('dec', np.float), ('SumI', np.float), ("Cluster", int), ("l", np.float), ("m", np.float), ("I", np.float)]) self.JonesDirCat = self.JonesDirCat.view(np.recarray) self.JonesDirCat.I = 1 self.JonesDirCat.SumI = 1 self.JonesDirCat.ra = NodesCat.ra self.JonesDirCat.dec = NodesCat.dec self.JonesDirCat.l = NodesCat.l self.JonesDirCat.m = NodesCat.m self.JonesDirCat.Cluster = range(NJonesDir) print >> log, "Sizes (%i facets):" % (self.JonesDirCat.shape[0]) print >> log, " - Main field : [%i x %i] pix" % (self.Npix, self.Npix) l_m_Diam = np.zeros((NFacets, 4), np.float32) l_m_Diam[:, 3] = np.arange(NFacets) Np = 10000 D = {} for iFacet in xrange(NFacets): D[iFacet] = {} polygon = LPolygonNew[iFacet] D[iFacet]["Polygon"] = polygon lPoly, mPoly = polygon.T ThisDiam, (l0, l1, m0, m1) = GiveDiam(polygon) # ############################### # # Find barycenter of polygon # X=(np.random.rand(Np))*ThisDiam+l0 # Y=(np.random.rand(Np))*ThisDiam+m0 # XY = np.dstack((X, Y)) # XY_flat = XY.reshape((-1, 2)) # mpath = Path( polygon ) # XY = np.dstack((X, Y)) # XY_flat = XY.reshape((-1, 2)) # mask_flat = mpath.contains_points(XY_flat) # mask=mask_flat.reshape(X.shape) # ############################### ThisPolygon = Polygon.Polygon(polygon) lc, mc = ThisPolygon.center() dl = np.max(np.abs([l0 - lc, l1 - lc])) dm = np.max(np.abs([m0 - mc, m1 - mc])) ############################### # lc=np.sum(X*mask)/np.sum(mask) # mc=np.sum(Y*mask)/np.sum(mask) # dl=np.max(np.abs(X[mask==1]-lc)) # dm=np.max(np.abs(Y[mask==1]-mc)) diam = 2 * np.max([dl, dm]) ###################### # lc=(l0+l1)/2. # mc=(m0+m1)/2. # dl=l1-l0 # dm=m1-m0 # diam=np.max([dl,dm]) l_m_Diam[iFacet, 0] = lc l_m_Diam[iFacet, 1] = mc l_m_Diam[iFacet, 2] = diam self.SpacialWeigth = {} self.DicoImager = {} # sort facets by size, unless we're in regular grid mode if not regular_grid: indDiam = np.argsort(l_m_Diam[:, 2])[::-1] l_m_Diam = l_m_Diam[indDiam] for iFacet in xrange(l_m_Diam.shape[0]): self.DicoImager[iFacet] = {} self.DicoImager[iFacet]["Polygon"] = D[l_m_Diam[iFacet, 3]]["Polygon"] x0 = round(l_m_Diam[iFacet, 0] / self.CellSizeRad) y0 = round(l_m_Diam[iFacet, 1] / self.CellSizeRad) if x0 % 2 == 0: x0 += 1 if y0 % 2 == 0: y0 += 1 l0 = x0 * self.CellSizeRad m0 = y0 * self.CellSizeRad diam = round( l_m_Diam[iFacet, 2] / self.CellSizeRad) * self.CellSizeRad # self.AppendFacet(iFacet,l0,m0,diam) self.AppendFacet(iFacet, l0, m0, diam) # self.MakeMasksTessel() NpixMax = np.max([ self.DicoImager[iFacet]["NpixFacet"] for iFacet in sorted(self.DicoImager.keys()) ]) NpixMaxPadded = np.max([ self.DicoImager[iFacet]["NpixFacetPadded"] for iFacet in sorted(self.DicoImager.keys()) ]) self.PaddedGridShape = (1, 1, NpixMaxPadded, NpixMaxPadded) self.FacetShape = (1, 1, NpixMax, NpixMax) dmin = 1 for iFacet in xrange(len(self.DicoImager)): l, m = self.DicoImager[iFacet]["l0m0"] d = np.sqrt(l**2 + m**2) if d < dmin: dmin = d iCentralFacet = iFacet self.iCentralFacet = iCentralFacet self.NFacets = len(self.DicoImager) # regFile="%s.tessel.reg"%self.GD["Output"]["Name"] labels = [(self.DicoImager[i]["lmShift"][0], self.DicoImager[i]["lmShift"][1], "[F%i_S%i]" % (i, self.DicoImager[i]["iSol"])) for i in xrange(len(LPolygonNew))] VM.PolygonToReg(regFile, LPolygonNew, radius=0.1, Col="green", labels=labels) self.WriteCoordFacetFile() self.FacetDirections = set([ self.DicoImager[iFacet]["RaDec"] for iFacet in range(len(self.DicoImager)) ]) #DicoName = "%s.DicoFacet" % self.GD["Images"]["ImageName"] DicoName = "%s.%sDicoFacet" % (self.GD["Output"]["Name"], "psf." if self.DoPSF else "") # Find the minimum l,m in the facet (for decorrelation calculation) for iFacet in self.DicoImager.keys(): #Create smoothned facet tessel mask: Npix = self.DicoImager[iFacet]["NpixFacetPadded"] l0, l1, m0, m1 = self.DicoImager[iFacet]["lmExtentPadded"] X, Y = np.mgrid[l0:l1:Npix / 10 * 1j, m0:m1:Npix / 10 * 1j] XY = np.dstack((X, Y)) XY_flat = XY.reshape((-1, 2)) vertices = self.DicoImager[iFacet]["Polygon"] mpath = Path(vertices) # the vertices of the polygon mask_flat = mpath.contains_points(XY_flat) mask = mask_flat.reshape(X.shape) mpath = Path(self.CornersImageTot) mask_flat2 = mpath.contains_points(XY_flat) mask2 = mask_flat2.reshape(X.shape) mask[mask2 == 0] = 0 R = np.sqrt(X**2 + Y**2) R[mask == 0] = 1e6 indx, indy = np.where(R == np.min(R)) lmin, mmin = X[indx[0], indy[0]], Y[indx[0], indy[0]] self.DicoImager[iFacet]["lm_min"] = lmin, mmin print >> log, "Saving DicoImager in %s" % DicoName MyPickle.Save(self.DicoImager, DicoName)
def ComputeNoiseMap(self): print>>log, "Compute noise map..." Boost=self.Boost Acopy=self.Restored[0,0,0::Boost,0::Boost].copy() SBox=(self.box[0]/Boost,self.box[1]/Boost) # MeanAbs=scipy.ndimage.filters.mean_filter(np.abs(Acopy),SBox) # Acopy[Acopy>0]=MeanAbs[Acopy>0] # Noise=np.sqrt(scipy.ndimage.filters.median_filter(np.abs(Acopy)**2,SBox)) x=np.linspace(-10,10,1000) f=0.5*(1.+scipy.special.erf(x/np.sqrt(2.))) n=SBox[0]*SBox[1] F=1.-(1.-f)**n ratio=np.abs(np.interp(0.5,F,x)) Noise=-scipy.ndimage.filters.minimum_filter(Acopy,SBox)/ratio Noise[Noise<0]=1e-10 # indxy=(Acopy>5.*Noise) # Acopy[indxy]=5*Noise[indxy] # Noise=np.sqrt(scipy.ndimage.filters.median_filter(np.abs(Acopy)**2,SBox)) # indxy=(Acopy>5.*Noise) # Acopy[indxy]=5*Noise[indxy] # Noise=np.sqrt(scipy.ndimage.filters.median_filter(np.abs(Acopy)**2,SBox)) #NoiseMed=np.median(Noise) #Noise[Noise<NoiseMed]=NoiseMed self.Noise=np.zeros_like(self.Restored[0,0]) for i in range(Boost): for j in range(Boost): s00,s01=Noise.shape s10,s11=self.Noise[i::Boost,j::Boost].shape s0,s1=min(s00,s10),min(s10,s11) self.Noise[i::Boost,j::Boost][0:s0,0:s1]=Noise[:,:][0:s0,0:s1] ind=np.where(self.Noise==0.) self.Noise[ind]=1e-10 if self.options.OutMaskExtended: GD=None MaskExtended=(self.Noise<0.1*np.median(self.Noise)) OutMaskExtended=self.options.OutMaskExtended nx=MaskExtended.shape[-1] CurrentNegMask=np.logical_not(MaskExtended).reshape((1,1,nx,nx)) PSFServer=None IdSharedMem=None DicoDirty=None IslandDistanceMachine=DDFacet.Imager.SSD.ClassIslandDistanceMachine.ClassIslandDistanceMachine(GD, CurrentNegMask, PSFServer, DicoDirty, IdSharedMem=IdSharedMem) ListIslands=IslandDistanceMachine.SearchIslands(None,Image=self.Restored) ListIslands=IslandDistanceMachine.ConvexifyIsland(ListIslands)#,PolygonFile="%s.pickle"%OutMaskExtended) ListPolygons=IslandDistanceMachine.ListPolygons MaskOut=np.zeros_like(CurrentNegMask) N=0 for Island in ListIslands: x,y=np.array(Island).T #if x.size<=10: continue MaskOut[0,0,x,y]=1 #ff,pol,_,_dec,ra=self.CasaIm.toworld((0,0,0,0)) ListPolygonsRADEC=[] for Polygon in ListPolygons: xx,yy=Polygon.T ThisPolygon=[] for iP in range(xx.shape[0]): xcc,ycc =xx[iP],yy[iP] ff,pol,dec,ra=self.CasaIm.toworld((0,0,xcc,ycc)) ThisPolygon.append((ra,dec)) ListPolygonsRADEC.append(np.array(ThisPolygon)) FName="%s.pickle"%OutMaskExtended print>>log,"Saving %s"%FName MyPickle.Save(ListPolygonsRADEC,FName) REGName="%s.reg"%OutMaskExtended RM=ModRegFile.PolygonNpToReg(ListPolygonsRADEC,REGName) RM.makeRegPolyREG() # TestArray=np.zeros_like(CurrentNegMask) # nx=TestArray.shape[-1] # xx,yy=20,100 # TestArray[0,0,xx,yy]=1 # PutDataInNewImage(self.FitsFile,"TestCoord.fits",np.float32(TestArray)) # ff,pol,dec,ra=self.CasaIm.toworld((0,0,xx,yy)) # pp=[[xx,yy],[yy,xx], # [nx/2-xx,yy],[nx/2+xx,yy], # [nx/2-yy,xx],[nx/2+yy,xx]] # for isol in range(len(pp)): # xx,yy=pp[isol] # ff,pol,dec,ra=self.CasaIm.toworld((0,0,xx,yy)) # sRA =rad2hmsdms(ra,Type="ra").replace(" ",":") # sDEC =rad2hmsdms(dec,Type="dec").replace(" ",":") # stop MaskOut=np.zeros_like(CurrentNegMask) N=0 for Island in ListIslands: x,y=np.array(Island).T if x.size<=10: continue MaskOut[0,0,x,y]=1 N+=1 print>>log,"Number of large enough islands %i"%N MaskExtended=MaskOut os.system("rm -rf %s"%OutMaskExtended) os.system("rm -rf %s.fits"%OutMaskExtended) PutDataInNewImage(self.FitsFile,OutMaskExtended,np.float32(MaskExtended)) NoiseMed=np.median(self.Noise) self.Noise[self.Noise<NoiseMed]=NoiseMed nx=self.Noise.shape[-1] if self.options.ConvNoise: print>>log,"Convolve..." NoiseMap,G=ModFFTW.ConvolveGaussianWrapper(self.Noise.reshape((1,1,nx,nx)),Sig=4*SBox[0]) NoiseMap/=np.sum(G) self.Noise=NoiseMap[0,0] if self.OutNameNoiseMap!="": #print>>log, "Save noise map as %s"%self.OutNameNoiseMap #self.CasaIm.saveas(self.OutNameNoiseMap) #CasaNoise=image(self.OutNameNoiseMap) #CasaNoise.putdata(self.Noise) #CasaNoise.tofits(self.OutNameNoiseMap+".fits") #del(CasaNoise) os.system("rm -rf %s"%self.OutNameNoiseMap) #os.system("rm -rf %s"%self.OutNameNoiseMap+".fits") PutDataInNewImage(self.FitsFile,self.OutNameNoiseMap,np.float32(self.Noise)) os.system("rm -rf %s.mean"%self.OutNameNoiseMap) PutDataInNewImage(self.FitsFile,self.OutNameNoiseMap+".mean",np.float32(np.zeros_like(self.Noise)))
def main(OP=None, messages=[]): if OP is None: OP = MyPickle.Load(SaveFile) print("Using settings from %s, then command line."%SaveFile) DicoConfig = OP.DicoConfig ImageName = DicoConfig["Output"]["Name"] if not ImageName: raise Exceptions.UserInputError("--Output-Name not specified, can't continue.") if not DicoConfig["Data"]["MS"]: raise Exceptions.UserInputError("--Data-MS not specified, can't continue.") # create directory if it exists dirname = os.path.dirname(ImageName) if not os.path.exists(dirname) and not dirname == "": os.mkdir(dirname) # setup logging logger.logToFile(ImageName + ".log", append=DicoConfig["Log"]["Append"]) global log log = logger.getLogger("DDFacet") # disable colors and progressbars if requested ModColor.silent = SkyModel.Other.ModColor.silent = \ progressbar.ProgressBar.silent = \ DicoConfig["Log"]["Boring"] if messages: if not DicoConfig["Log"]["Boring"]: #os.system('clear') logo.print_logo() for msg in messages: print(msg, file=log) print("Checking system configuration:", file=log) # check for SHM size ram_size = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') shm_stats = os.statvfs('/dev/shm') shm_size = shm_stats.f_bsize * shm_stats.f_blocks shm_relsize = shm_size / float(ram_size) shm_avail = shm_stats.f_bsize * shm_stats.f_bavail / float(ram_size) if shm_relsize < 0.6: print(ModColor.Str("""WARNING: max shared memory size is only {:.0%} of total RAM size. This can cause problems for large imaging jobs. A setting of 90% is recommended for DDFacet and killMS. If your processes keep failing with SIGBUS or "bus error" messages, it is most likely for this reason. You can change the memory size by running $ sudo mount -o remount,size=90% /dev/shm To make the change permanent, edit /etc/defaults/tmps, and add a line saying "SHM_SIZE=90%". """.format(shm_relsize)), file=log) else: print(" Max shared memory size is {:.0%} of total RAM size; {:.0%} currently available".format(shm_relsize, shm_avail), file=log) try: output = subprocess.check_output(["/sbin/sysctl", "vm.max_map_count"],universal_newlines=True) except Exception: print(ModColor.Str("""WARNING: /sbin/sysctl vm.max_map_count failed. Unable to check this setting."""), file=log) max_map_count = None else: max_map_count = int(output.strip().rsplit(" ", 1)[-1]) if max_map_count is not None: if max_map_count < 500000: print(ModColor.Str("""WARNING: sysctl vm.max_map_count = {}. This may be too little for large DDFacet and killMS jobs. If you get strange "file exists" errors on /dev/shm, them try to bribe, beg or threaten your friendly local sysadmin into setting vm.max_map_count=1000000 in /etc/sysctl.conf. """.format(max_map_count)), file=log) else: print(" sysctl vm.max_map_count = {}".format(max_map_count), file=log) # check for memory lock limits import resource msoft, mhard = resource.getrlimit(resource.RLIMIT_MEMLOCK) if msoft >=0 or mhard >=0: print(ModColor.Str("""WARNING: your system has a limit on memory locks configured. This may possibly slow down DDFacet performance. You can try removing the limit by running $ ulimit -l unlimited If this gives an "operation not permitted" error, you can try to bribe, beg or threaten your friendly local sysadmin into doing # echo "* - memlock unlimited" >> /etc/security/limits.conf """), file=log) if DicoConfig["Debug"]["Pdb"] == "always": print("--Debug-Pdb=always: unexpected errors will be dropped into pdb", file=log) Exceptions.enable_pdb_on_error(ModColor.Str("DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n" + "(This is because you're running with --Debug-Pdb set to 'always'.)")) elif DicoConfig["Debug"]["Pdb"] == "auto" and not DicoConfig["Log"]["Boring"]: print("--Debug-Pdb=auto and not --Log-Boring: unexpected errors will be dropped into pdb", file=log) Exceptions.enable_pdb_on_error(ModColor.Str("DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n" + "(This is because you're running with --Debug-Pdb set to 'auto' and --Log-Boring is off.)")) # print current options OP.Print(dest=log) # enable memory logging logger.enableMemoryLogging(DicoConfig["Log"]["Memory"]) # get rid of old shm arrays from previous runs Multiprocessing.cleanupStaleShm() # initialize random seed from config if set, or else from system time if DicoConfig["Misc"]["RandomSeed"] is not None: DicoConfig["Misc"]["RandomSeed"]=int(DicoConfig["Misc"]["RandomSeed"]) print("random seed=%d (explicit)" % DicoConfig["Misc"]["RandomSeed"], file=log) else: DicoConfig["Misc"]["RandomSeed"] = int(time.time()) print("random seed=%d (automatic)" % DicoConfig["Misc"]["RandomSeed"], file=log) np.random.seed(DicoConfig["Misc"]["RandomSeed"]) # init NCPU for different bits of parallelism ncpu = int(DicoConfig["Parallel"]["NCPU"] or psutil.cpu_count()) DicoConfig["Parallel"]["NCPU"]=ncpu _pyArrays.pySetOMPNumThreads(ncpu) NpParallel.NCPU_global = ModFFTW.NCPU_global = ncpu numexpr.set_num_threads(ncpu) print("using up to %d CPUs for parallelism" % ncpu, file=log) # write parset OP.ToParset("%s.parset"%ImageName) Mode = DicoConfig["Output"]["Mode"] # init semaphores, as they're needed for weight calculation too ClassFacetMachine.ClassFacetMachine.setup_semaphores(DicoConfig) # data machine initialized for all cases except PSF-only mode # psf machine initialized for all cases except Predict-only mode Imager = ClassDeconvMachine.ClassImagerDeconv(GD=DicoConfig, BaseName=ImageName, predict_only=(Mode == "Predict" or Mode == "Subtract"), data=(Mode != "PSF"), psf=(Mode != "Predict" and Mode != "Dirty" and Mode != "Subtract"), readcol=(Mode != "Predict" and Mode != "PSF"), deconvolve=("Clean" in Mode)) Imager.Init() # Imager.testDegrid() # stop if "Predict" in Mode or "Subtract" in Mode: Imager.GivePredict() if "Clean" in Mode: Imager.main() elif "Dirty" in Mode: sparsify = DicoConfig["Comp"]["Sparsification"] if sparsify and isinstance(sparsify, list): sparsify = sparsify[0] Imager.GiveDirty(psf="PSF" in Mode, sparsify=sparsify) elif "PSF" in Mode: sparsify = DicoConfig["Comp"]["Sparsification"] if sparsify and isinstance(sparsify, list): sparsify = sparsify[0] Imager.MakePSF(sparsify=sparsify) elif "RestoreAndShift" == Mode: Imager.RestoreAndShift()
def testMO_DATA(): Dico= MyPickle.Load("SaveTest") Dirty=Dico["Dirty"] PSF=Dico["PSF"] PSF2=np.squeeze(PSF) ListPixData=Dico["ListPixData"] FreqsInfo=Dico["FreqsInfo"] FreqsInfo=Dico["FreqsInfo"] IslandBestIndiv=Dico["IslandBestIndiv"] ListPixParms=ListPixData ListSquarePix=Dico["ListSquarePix"] ThisPixList=ListPixData ThisSquarePixList=ListSquarePix GD=Dico["GD"] FacetID=Dico["FacetID"] nch=FreqsInfo["MeanJonesBand"][FacetID].size WeightMeanJonesBand=FreqsInfo["MeanJonesBand"][FacetID].reshape((nch,1,1,1)) WeightMueller=WeightMeanJonesBand.ravel() WeightMuellerSignal=WeightMueller*FreqsInfo["WeightChansImages"].ravel() #IncreaseIslandMachine=ClassIncreaseIsland.ClassIncreaseIsland() #ListPixData=IncreaseIslandMachine.IncreaseIsland(ListPixData,dx=5) # 0) Load Island info (center and square data) ListSquarePix_Center=ListSquarePix['IslandCenter'] ListSquarePix_Data=ListSquarePix['IslandSquareData'] ListSquarePix_Mask=ListSquarePix['IslandSquareMask'] orixisland,oriyisland=ListSquarePix_Data.shape # size of the square postage stamp around island ListSquarePix_Data=PSF2 xisland,yisland=ListSquarePix_Data.shape # size of the square postage stamp around island print(xisland) # 1) Shape PSF and Dirty to have even number of pixels (required by Moresane) # DEAL WITH SQUARE DATA OF ISLAND IF UNEVEN # Crop PSF to the island postage stamp #PSFCrop=CropPSF(PSF,xisland) #PSF2=CropPSF(PSF2,71) # MORESANE requires even sized images ==> Padding by one row and one column cropped_square_data_to_even = False if xisland % 2 != 0: # PSFCrop_even = np.zeros((xisland+1, xisland+1)) # PSFCrop_even[:-1, :-1] = np.squeeze(PSFCrop) Dirty_even = np.zeros((xisland - 1, xisland - 1)) Dirty_even[:, :] = ListSquarePix_Data[:-1, :-1] cropped_square_data_to_even = True else: Dirty_even = ListSquarePix_Data # make it even by removing one line and one column (usually outside of the interesting island region) # xbigdirty,ybigdirty=np.squeeze(Dirty).shape # if xbigdirty % 2 != 0: # Dirty_even=np.zeros((xbigdirty-1,xbigdirty-1)) # Dirty_even[:,:]=np.squeeze(Dirty)[:-1,:-1] xbigpsf,ybigpsf=PSF2.shape cropped_square_psf_to_even = False if xbigpsf % 2 != 0: PSF2_even=np.zeros((xbigpsf-1,xbigpsf-1)) PSF2_even[:,:]=PSF2[:-1,:-1] cropped_square_data_to_even = True else: PSF2_even=PSF2 # 2) Run the actual MinorCycle algo DictMoresaneParms=GD['MORESANE'] Moresane=ClassMoresane(Dirty_even,PSF2_even,DictMoresaneParms,GD=GD) Model_Square=Moresane.main() # 3) Apply Island mask to model to get rid of regions outside the island. cropped_square_to_even = False if cropped_square_data_to_even: # then restore the model to its original uneven dimension Model_Square_uneven = np.zeros((xisland, xisland)) Model_Square_uneven[:-1, :-1] = Model_Square Model_Square = Model_Square_uneven if cropped_square_psf_to_even: # restore original PSF size PSF_uneven=PSF2 Model_Square=CropPSF(Model_Square, orixisland) Model_Square *= ListSquarePix_Mask # masking outside the island # 4) Convert back to Island format ( "S" and ThisPixList ) NewModel, NewThisPixList = SquareIslandtoIsland(Model_Square, ThisSquarePixList, ThisPixList) Model = NewModel ThisPixList = NewThisPixList return Model