def CleanFiles(MSName): def EX(ss): #print ss os.system(ss) BaseImageName = os.path.abspath(MSName).split("_")[-3].split(".")[1] MSName = os.path.abspath(MSName).split("/")[-1] if not os.path.isdir("PRODUCTS"): EX("mkdir -p %s" % PDir) KEEP = [ "%s_m.AP_m.app.restored.fits" % BaseImageName, "%s_m.AP_m.tessel.reg" % BaseImageName, "%s_m.AP_m.parset" % BaseImageName, "%s_m.AP_m.log" % BaseImageName, "%s_m.AP_m.DicoModel" % BaseImageName, "%s_m.AP.app.restored.fits.mask.fits" % BaseImageName, "%s_m.app.restored.pybdsm.srl.fits.ClusterCat.npy" % BaseImageName, "SOLS_%s" % MSName ] log.print(ModColor.Str("Moving %s to %s" % (str(KEEP), PDir))) for File in KEEP: EX("mv %s %s" % (File, PDir)) log.print(ModColor.Str("Delete the rest...")) EX("rm -rf %s*" % BaseImageName) EX("rm -rf %s*.ddfcache" % MSName)
def invSVD(A,Cut=1e-6): #print "rand" Ar=A # +np.random.randn(*A.shape)*(1e-6*A.max()) #print "stard",Ar.shape try: u,s,v=np.linalg.svd(Ar) except: Name="errSVDArray_%i"%int(np.random.rand(1)[0]*10000) print ModColor.Str("Problem inverting Matrix, saving as %s"%Name) print ModColor.Str(" will make it svd-able") np.save(Name,Ar) # weird - I found a matrix I cannot do svd on... - that works Cut=1e-20 #Ar=np.complex64(Ar) u,s,v=np.linalg.svd(np.complex64(Ar))#+np.random.randn(*Ar.shape)*(1e-10*np.abs(Ar).max())) #u,s,v=np.linalg.svd() u=np.real(u) s=np.real(s) v=np.real(v) #u,s,v=np.linalg.svd(np.complex128(Ar)) #print "ok" s[s<0.]=Cut s[s<Cut*s.max()]=Cut*s.max() ssq=(1./s) #Asq=np.conj(np.dot(np.dot(v.T,ssq),u.T)) v0=v.T*ssq.reshape(1,ssq.size) Asq=np.conj(np.dot(v0,u.T)) return Asq
def giveOutIm(self, Nout): A = self.A nch, npol, Nin, _ = A.shape if Nin == Nout: return A.copy() elif Nin > Nout: B = np.zeros((nch, npol, Nout, Nout), A.dtype) print >> log, ModColor.Str( " Output image smaller than input image") print >> log, ModColor.Str(" adapting %s --> %s" % (str(A.shape), str(B.shape))) # Input image larger than requested N0 = A.shape[-1] xc0 = yc0 = N0 / 2 x0d, x1d = xc0 - Nout / 2, xc0 - Nout / 2 + Nout s = slice(x0d, x1d) B[:, :, :, :] = A[:, :, s, s] return B else: B = np.zeros((nch, npol, Nout, Nout), A.dtype) # Input image smaller - has to zero pad print >> log, ModColor.Str( " Output image larger than input image") print >> log, ModColor.Str(" adapting %s --> %s" % (str(A.shape), str(B.shape))) Na = A.shape[-1] Nb = B.shape[-1] xa = Na / 2 xb = Nb / 2 x0d, x1d = xb - xa, xb - xa + Na s = slice(x0d, x1d) B[:, :, s, s] = A[:, :, :, :] return B
def RunBDSM(self, filename, rmsmean_map_filename=None, WriteNoise=True, Ratio=None): CatName = filename[:-5] + ".pybdsm.gaul.fits" if os.path.isfile(CatName): print >> log, ModColor.Str("File %s exists" % CatName, col="green") print >> log, ModColor.Str(" Skipping...", col="green") return rmsmean_map_filename = [] if self.rmsmean_map: l = self.rmsmean_map l = l.replace("[", "") l = l.replace("]", "") l = l.split(",") rmsmean_map_filename = l img = bdsf.process_image(filename, thresh_isl=self.bdsm_thresh_isl, thresh_pix=self.bdsm_thresh_pix, rms_box=self.bdsm_rms_box, rmsmean_map_filename=rmsmean_map_filename) img.write_catalog(catalog_type="srl", clobber=True, format="fits") img.write_catalog(catalog_type="gaul", clobber=True, format="fits") if WriteNoise: img.export_image(img_type="rms", clobber=True) img.export_image(img_type="mean", clobber=True)
def Print(self, RejectGroups=[], dest=sys.stdout): P = ClassPrint.ClassPrint(HW=50) print >> dest, ModColor.Str(" Selected Options:") for Group, V in self.DefaultDict.items(): if Group in RejectGroups: continue try: GroupTitle = self.DicoGroupDesc[Group] except: GroupTitle = Group print >> dest, ModColor.Str("[%s] %s" % (Group, GroupTitle), col="green") option_list = self.DefaultDict[Group] for oname in option_list: if oname[0] != "_": # skip "internal" values such as "_Help" V = self.DefaultDict[Group][oname] attrs = self.AttrDict.get(Group).get(oname, {}) if not attrs.get('alias_of') and not attrs.get( "cmdline_only"): # and V!="": if V == "": V = "''" P.Print(oname, V, dest=dest) print
def _run_worker(self, queue): """ Runs worker loop on given queue. Waits on queue, picks off job items, looks them up in context table, calls them, and returns results in the work queue. """ if self.verbose > 0: print >> log, ModColor.Str("started worker pid %d" % os.getpid()) try: pill = True # While no poisoned pill has been given grab items from the queue. while pill: try: # Get queue item, or timeout and check if pill perscribed. #print>>log,"%s: calling queue.get()"%AsyncProcessPool.proc_id jobitem = queue.get(True, 10) #print>>log,"%s: queue.get() returns %s"%(AsyncProcessPool.proc_id, jobitem) except Queue.Empty: continue if jobitem == "POISON-E": if self.verbose: print >> log, "got pill. Qin:{} Qout:{}".format( queue.qsize(), self._result_queue.qsize()) break elif jobitem is not None: self._dispatch_job(jobitem) except KeyboardInterrupt: print >> log, ModColor.Str("Ctrl+C caught, exiting", col="red") return
def __init__(self, dirname, reset=False, cachedir=None, nfswarn=False): """ Initializes cache manager. Args: dirname: directory in which caches are kept reset: if True, cache is reset upon first access cachedir: if set, caches things under cachedir/dirname. Useful for fast local storage. nfswarn: if True and directory is NFS mounted, prints a warning """ # strip trailing slashes while dirname[-1] == "/": dirname = dirname[:-1] if cachedir: dirname = os.path.join(cachedir, os.path.basename(dirname)) self.dirname = dirname self.hashes = {} self.pid = os.getpid() if not os.path.exists(dirname): print("cache directory %s does not exist, creating" % dirname, file=log) os.mkdir(dirname) else: if reset: print( "clearing cache %s, since we were asked to reset the cache" % dirname, file=log) os.system("rm -fr " + dirname) os.mkdir(dirname) # check for NFS system and print warning if nfswarn: try: fstype = subprocess.check_output( ("stat --file-system --format=%T " + dirname).split()).strip() except: print(ModColor.Str( "WARNING: unable to determine filesystem type for %s" % dirname, col="red", Bold=True), file=log) fstype = "unknown" if fstype == "nfs": print(ModColor.Str( "WARNING: cache directory %s is mounted via NFS." % dirname, col="red", Bold=True), file=log) print(ModColor.Str( "This may cause performance issues. Consider using the --Cache-Dir option.", col="red", Bold=True), file=log)
def os_exec(ss, CheckFile=None): print() log.print( ModColor.Str( "====================================================================================================================", col="blue")) if CheckFile is not None: if os.path.isfile(CheckFile): log.print(ModColor.Str("%s exists..." % CheckFile, col="green")) log.print( ModColor.Str(" skipping step: ", col="green") + " %s" % ss) return log.print(ModColor.Str("Executing %s" % ss)) os.system(ss)
def _start_worker(object, proc_id, affinity, worker_queue, pause_on_start=False): """ Helper method for worker process startup. ets up affinity, and calls _run_worker method on object with the specified work queue. Args: object: proc_id: affinity: work_queue: Returns: """ if pause_on_start: os.kill(os.getpid(), signal.SIGSTOP) numexpr.set_num_threads( 1) # no sub-threads in workers, as it messes with everything _pyArrays.pySetOMPNumThreads(1) _pyArrays.pySetOMPDynamicNumThreads(1) AsyncProcessPool.proc_id = proc_id MyLogger.subprocess_id = proc_id if affinity: psutil.Process().cpu_affinity(affinity) object._run_worker(worker_queue) if object.verbose: print >> log, ModColor.Str("exiting worker pid %d" % os.getpid())
def _initIsland_worker(self, DicoOut, iIsland, Island, DicoVariablePSF, DicoDirty, DicoParm, FacetCache, NCPU): logger.setSilent([ "ClassImageDeconvMachineMSMF", "ClassPSFServer", "ClassMultiScaleMachine", "GiveModelMachine", "ClassModelMachineMSMF" ]) self.InitMachine.Init(DicoVariablePSF, DicoParm["GridFreqs"], DicoParm["DegridFreqs"], facetcache=FacetCache) self.InitMachine.setDirty(DicoDirty) #self.InitMachine.DeconvMachine.setNCPU(NCPU) self.InitMachine.setSSDModelImage(DicoParm["ModelImage"]) #print ":::::::::::::::::::::::",iIsland try: SModel, AModel = self.InitMachine.giveModel(Island) except: if not self.GD["GAClean"]["ParallelInitHMP"]: raise print >> log, traceback.format_exc() FileOut = "errIsland_%6.6i.npy" % iIsland print >> log, ModColor.Str( "...... error on island %i, saving to file %s" % (iIsland, FileOut)) np.save(FileOut, np.array(Island)) self.InitMachine.Reset() return DicoOut["S"] = SModel DicoOut["Alpha"] = AModel self.InitMachine.Reset()
def setRefFreq(self, RefFreq, Force=False): if self.RefFreq is not None and not Force: print(ModColor.Str("Reference frequency already set to %f MHz" % (self.RefFreq/1e6)), file=log) return self.RefFreq = RefFreq self.DicoSMStacked["RefFreq"] = RefFreq
def CreateShared(Name, shape, dtype): try: a = SharedArray.create(Name, shape, dtype=dtype) except OSError: print >> log, ModColor.Str("File %s exists, deleting" % Name) DelArray(Name) a = SharedArray.create(Name, shape, dtype=dtype) return a
def setRefFreq(self, RefFreq, Force=False): #,AllFreqs): if self.RefFreq is not None and not Force: print >> log, ModColor.Str( "Reference frequency already set to %f MHz" % (self.RefFreq / 1e6)) return self.RefFreq = RefFreq self.DicoModel["RefFreq"] = RefFreq
def setLoud(Lname): print>>itsLog, ModColor.Str("Set loud: %s" % Lname, col="green") if type(Lname)==str: log=getLogger(Lname) log.logger.setLevel(logging.DEBUG) elif type(Lname)==list: for name in Lname: log=getLogger(name) log.logger.setLevel(logging.DEBUG)
def _dispatch_job(self, jobitem, reraise=False): """Handles job described by jobitem dict. If reraise is True, any eceptions are re-raised. This is useful for debugging.""" timer = ClassTimeIt.ClassTimeIt() event = counter = None try: job_id, event_id, counter_id, args, kwargs = [jobitem.get(attr) for attr in "job_id", "event", "counter", "args", "kwargs"] handler_id, method, handler_desc = jobitem["handler"] handler = self._job_handlers.get(handler_id) if handler is None: raise RuntimeError("Job %s: unknown handler %s. This is a bug." % (job_id, handler_desc)) event, eventname = self._events[event_id] if event_id is not None else (None, None) # find counter object, if specified if counter_id: counter = self._job_counters.get(counter_id) if counter is None: raise RuntimeError("Job %s: unknown counter %s. This is a bug." % (job_id, counter_id)) # instantiate SharedDict arguments # timer.timeit('init '+job_id) args = [ arg.instantiate() if type(arg) is shared_dict.SharedDictRepresentation else arg for arg in args ] for key in kwargs.keys(): if type(kwargs[key]) is shared_dict.SharedDictRepresentation: kwargs[key] = kwargs[key].instantiate() # timer.timeit('instantiated '+job_id) # call the job if self.verbose > 1: print>> log, "job %s: calling %s" % (job_id, handler_desc) if method is None: # call object directly result = handler(*args, **kwargs) else: call = getattr(handler, method, None) if not callable(call): raise KeyError("Job %s: unknown method '%s' for handler %s" % (job_id, method, handler_desc)) result = call(*args, **kwargs) if self.verbose > 3: print>> log, "job %s: %s returns %s" % (job_id, handler_desc, result) # Send result back if jobitem['collect_result']: self._result_queue.put( dict(job_id=job_id, proc_id=self.proc_id, success=True, result=result, time=timer.seconds())) except KeyboardInterrupt: raise except Exception, exc: if reraise: raise print>> log, ModColor.Str("process %s: exception raised processing job %s: %s" % ( AsyncProcessPool.proc_id, job_id, traceback.format_exc())) if jobitem['collect_result']: self._result_queue.put( dict(job_id=job_id, proc_id=self.proc_id, success=False, error=exc, time=timer.seconds()))
def SharedToDico(Prefix): print >> log, ModColor.Str("SharedToDico: start [prefix = %s]" % Prefix) Lnames = ListNames() keys = [Name for Name in Lnames if Prefix in Name] if len(keys) == 0: return None DicoOut = {} for Sharedkey in keys: key = Sharedkey.split(".")[-1] print >> log, ModColor.Str(" %s -> %s" % (Sharedkey, key)) Shared = GiveArray(Sharedkey) if isinstance(Shared, type(None)): print >> log, ModColor.Str(" None existing key %s" % (key)) return None DicoOut[key] = Shared print >> log, ModColor.Str("SharedToDico: done") return DicoOut
def __init__(self, ListMSName=None, ColName="DATA", ModelName="PREDICT_KMS", UVRange=[1., 1000.], ColWeights=None, SolsName=None, FileCoords="Transient_LOTTS.csv", Radius=3., NOff=-1, ImageI=None, ImageV=None, SolsDir=None, NCPU=1, BaseDirSpecs=None, BeamModel=None, BeamNBand=1): self.ColName = ColName self.ModelName = ModelName self.ColWeights = ColWeights self.BeamNBand = BeamNBand self.UVRange = UVRange self.Mode = "Spec" self.BaseDirSpecs = BaseDirSpecs self.NOff = NOff self.SolsName = SolsName self.NCPU = NCPU self.BeamModel = BeamModel if ListMSName is None: print(ModColor.Str("WORKING IN REPLOT MODE"), file=log) self.Mode = "Plot" self.Radius = Radius self.ImageI = ImageI self.ImageV = ImageV self.SolsDir = SolsDir #self.PosArray=np.genfromtxt(FileCoords,dtype=[('Name','S200'),("ra",np.float64),("dec",np.float64),('Type','S200')],delimiter="\t") # identify version in logs print("DynSpecMS version %s starting up" % version(), file=log) self.FileCoords = FileCoords if self.Mode == "Spec": self.ListMSName = sorted(ListMSName) #[0:2] self.nMS = len(self.ListMSName) self.OutName = self.ListMSName[0].split("/")[-1].split("_")[0] self.ReadMSInfos() self.InitFromCatalog() elif self.Mode == "Plot": self.OutName = self.BaseDirSpecs.split("_")[-1] self.InitFromSpecs()
def DicoToShared(Prefix, Dico, DelInput=False): DicoOut = {} print >> log, ModColor.Str("DicoToShared: start [prefix = %s]" % Prefix) for key in Dico.keys(): if not isinstance(Dico[key], np.ndarray): continue # print "%s.%s"%(Prefix,key) ThisKeyPrefix = "%s.%s" % (Prefix, key) print >> log, ModColor.Str(" %s -> %s" % (key, ThisKeyPrefix)) ar = Dico[key] Shared = ToShared(ThisKeyPrefix, ar) DicoOut[key] = Shared if DelInput: del (Dico[key], ar) if DelInput: del (Dico) print >> log, ModColor.Str("DicoToShared: done") #print ModColor.Str("DicoToShared: done") return DicoOut
def __init__(self, **kwargs): for key, value in kwargs.items(): setattr(self, key, value) if self.NCPU == 0: self.NCPU = psutil.cpu_count() - 1 self.bdsm_rms_box = [int(i) for i in self.bdsm_rms_box.split(",")] Files = [self.RestoredIm] for f in Files: if not os.path.isfile(f): raise ValueError("File %s does not exist" % f) print >> log, ModColor.Str("File %s exist" % f, col="green")
def GiveSpectralIndexMap(self, CellSizeRad=1., GaussPars=[(1, 1, 0)], DoConv=True, MaxSpi=100, MaxDR=1e+6, threshold=None): dFreq = 1e6 # f0=self.DicoSMStacked["AllFreqs"].min() # f1=self.DicoSMStacked["AllFreqs"].max() RefFreq = self.DicoSMStacked["RefFreq"] f0 = RefFreq / 1.5 f1 = RefFreq * 1.5 M0 = self.GiveModelImage(f0) M1 = self.GiveModelImage(f1) if DoConv: # M0=ModFFTW.ConvolveGaussian(M0,CellSizeRad=CellSizeRad,GaussPars=GaussPars) # M1=ModFFTW.ConvolveGaussian(M1,CellSizeRad=CellSizeRad,GaussPars=GaussPars) # M0,_=ModFFTW.ConvolveGaussianWrapper(M0,Sig=GaussPars[0][0]/CellSizeRad) # M1,_=ModFFTW.ConvolveGaussianWrapper(M1,Sig=GaussPars[0][0]/CellSizeRad) M0, _ = ModFFTW.ConvolveGaussianScipy(M0, Sig=GaussPars[0][0] / CellSizeRad) M1, _ = ModFFTW.ConvolveGaussianScipy(M1, Sig=GaussPars[0][0] / CellSizeRad) # print M0.shape,M1.shape # compute threshold for alpha computation by rounding DR threshold to .1 digits (i.e. 1.65e-6 rounds to 1.7e-6) if threshold is not None: minmod = threshold elif not np.all(M0 == 0): minmod = float("%.1e" % (np.max(np.abs(M0)) / MaxDR)) else: minmod = 1e-6 # mask out pixels above threshold mask = (M1 < minmod) | (M0 < minmod) print("computing alpha map for model pixels above %.1e Jy (based on max DR setting of %g)" % ( minmod, MaxDR), file=log) M0[mask] = minmod M1[mask] = minmod # with np.errstate(invalid='ignore'): # alpha = (np.log(M0)-np.log(M1))/(np.log(f0/f1)) # print # print np.min(M0),np.min(M1),minmod # print alpha = (np.log(M0) - np.log(M1)) / (np.log(f0 / f1)) alpha[mask] = 0 # mask out |alpha|>MaxSpi. These are not physically meaningful anyway mask = alpha > MaxSpi alpha[mask] = MaxSpi masked = mask.any() mask = alpha < -MaxSpi alpha[mask] = -MaxSpi if masked or mask.any(): print(ModColor.Str("WARNING: some alpha pixels outside +/-%g. Masking them." % MaxSpi, col="red"), file=log) return alpha
def filter(self, event): vss = float(_memory()/(1024**3)) vss_peak = float(_memory_peak()/(1024**3)) rss = float(_resident()/(1024**3)) rss_peak = float(_resident_peak()/(1024**3)) shm = float(_shmem_size()/(1024**3)) setattr(event,"virtual_memory_gb",vss) setattr(event,"resident_memory_gb",rss) setattr(event,"shared_memory_gb",shm) if log_memory and hasattr(event,"msg"): event.msg = "[%.1f/%.1f %.1f/%.1f %.1fGb] "%(rss,rss_peak,vss,vss_peak,shm) + event.msg if subprocess_id: event.msg = ModColor.Str("[%s] "%subprocess_id, col="blue") + event.msg return True
def _handle_exception(type, value, tb): if hasattr(sys, 'ps1') or not sys.stderr.isatty(): # we are in interactive mode or we don't have a tty-like # device, so we call the default hook sys.__excepthook__(type, value, tb) else: print(ModColor.Str(_advise)) import traceback, pdb # we are NOT in interactive mode, print the exception... traceback.print_exception(type, value, tb) print() # ...then start the debugger in post-mortem mode. # pdb.pm() # deprecated pdb.post_mortem(tb) # more "modern"
def run(self): while not self.kill_received and not self.work_queue.empty(): DicoJob = self.work_queue.get() # self.initIsland(DicoJob) try: self.initIsland(DicoJob) except: print traceback.format_exc() iIsland=DicoJob["iIsland"] FileOut="errIsland_%6.6i.npy"%iIsland print ModColor.Str("...... on island %i, saving to file %s"%(iIsland,FileOut)) np.save(FileOut,np.array(self.ListIsland[iIsland])) print
def PrintOptParse(Obj, ValObj, RejectGroup=[]): P = ClassPrint.ClassPrint(HW=30) LGroups = Obj.option_groups print(ModColor.Str(" Selected Options:")) for Group in LGroups: Skip = False for Name in RejectGroup: if Name in Group.title: Skip = True if Skip: continue print(ModColor.Str(Group.title, col="green")) option_list = Group.option_list for o in option_list: lopt = o._long_opts[0] oname = lopt.split("--")[-1] V = getattr(ValObj, oname) # if (V!="")&(V is not None): if True: #V!="": if V == "": V = "''" #P.Print(oname,V) default = o.default H = o.help # if H is not None: # H=o.help.replace("%default",str(default)) # P.Print2(oname,V,H) # else: # P.Print(oname,V) P.Print(oname, V) # strName=%s # print " "oname,V print()
def Print2(self, par, value, helpit, col="white"): WidthTerm = self.getWidth() Lpar = len(str(par)) Lval = len(str(value)) SFill = "." * max(self.LeftW - self.Lproto - Lpar - Lval, 2) WidthHelp = WidthTerm - (self.Lproto + Lpar + Lval + len(SFill)) Spar = "%s" % ModColor.Str(str(par), col=col, Bold=False) Sval = "%s" % ModColor.Str(str(value), col=col, Bold=False) if helpit == "": helpit = "Help yourself" Shelp = "%s" % helpit if WidthHelp < 0: print(self.proto % (Spar, SFill, Sval) + Shelp) return Lhelp = len(str(helpit)) listStrHelp = range(0, Lhelp, WidthHelp) if listStrHelp[-1] != Lhelp: listStrHelp.append(Lhelp) print(self.proto % (Spar, SFill, Sval) + Shelp[0:WidthHelp]) for i in range(1, len(listStrHelp) - 1): parout = "%s: %s" % (" " * (self.LeftW - 2), Shelp[listStrHelp[i]:listStrHelp[i + 1]]) print(parout)
def GiveMMFromDico(self,DicoSMStacked=None): """ Initialise a model machine from a dictionary Input: DicoSMStacked = Dictionary to instantiate ModelMachine with """ if DicoSMStacked["Type"]=="GA": print>>log,ModColor.Str("Model is of deprecated type GA, overwriting with type SSD") DicoSMStacked["Type"]="SSD" if DicoSMStacked is not None: # If the Dict is provided use it to initialise a model machine Type = DicoSMStacked["Type"] # backwards compatibility if Type == "GA": Type = "SSD" elif Type == "MSMF": Type = "HMP" return self.GiveMM(Type) else: # If the dict is not provided use the MinorCycleMode to figure out which model machine to initialise return self.GiveMM(self.GD["Deconv"]["Mode"])
def __init__(self, ListMSName=None, ColName="DATA", ModelName="PREDICT_KMS", UVRange=[1.,1000.], ColWeights=None, SolsName=None, FileCoords="Transient_LOTTS.csv", SolsDir=None, NCPU=1, BeamModel=None, BeamNBand=1): self.ColName = ColName self.ModelName = ModelName self.ColWeights = ColWeights self.BeamNBand = BeamNBand self.UVRange = UVRange self.Mode="Spec" self.SolsName=SolsName self.NCPU=NCPU self.BeamModel=BeamModel self.StepFreq=10#2000 self.StepTime=1 if ListMSName is None: print(ModColor.Str("WORKING IN REPLOT MODE"), file=log) self.Mode="Plot" self.SolsDir=SolsDir self.FileCoords=FileCoords self.ListMSName = sorted(ListMSName)#[0:2] self.nMS = len(self.ListMSName) self.OutName = self.ListMSName[0].split("/")[-1].split("_")[0] self.ReadMSInfos() self.InitFromCatalog()
def GiveMMFromDico(self,DicoSMStacked=None): """ Initialise a model machine from a dictionary Input: DicoSMStacked = Dictionary to instantiate ModelMachine with """ def safe_encode(s): import six return s.decode() if isinstance(s, bytes) and six.PY3 else s Type = safe_encode(DicoSMStacked.get("Type", DicoSMStacked.get(b"Type", None))) if Type=="GA": print(ModColor.Str("Model is of deprecated type GA, overwriting with type SSD"), file=log) DicoSMStacked["Type"]="SSD" if DicoSMStacked is not None: # If the Dict is provided use it to initialise a model machine Type = safe_encode(DicoSMStacked.get("Type", DicoSMStacked.get(b"Type", None))) # backwards compatibility if Type == "GA": Type = "SSD" elif Type == "MSMF": Type = "HMP" return self.GiveMM(Type) else: # If the dict is not provided use the MinorCycleMode to figure out which model machine to initialise return self.GiveMM(self.GD["Deconv"]["Mode"])
def Deconvolve(self, ch=0, **kwargs): """ Runs minor cycle over image channel 'ch'. initMinor is number of minor iteration (keeps continuous count through major iterations) Nminor is max number of minor iterations Returns tuple of: return_code,continue,updated where return_code is a status string; continue is True if another cycle should be executed (one or more polarizations still need cleaning); update is True if one or more polarization models have been updated """ #No need to set the channel when doing joint deconvolution self.setChannel(ch) exit_msg = "" continue_deconvolution = False update_model = False _, npix, _ = self.Dirty.shape xc = (npix) / 2 npol, _, _ = self.Dirty.shape # Get the PeakMap (first index will always be 0 because we only support I cleaning) PeakMap = self.Dirty[0, :, :] m0, m1 = PeakMap.min(), PeakMap.max() #These options should probably be moved into MinorCycleConfig in parset DoAbs = int(self.GD["Deconv"]["AllowNegative"]) print >> log, " Running minor cycle [MinorIter = %i/%i, SearchMaxAbs = %i]" % ( self._niter, self.MaxMinorIter, DoAbs) ## Determine which stopping criterion to use for flux limit #Get RMS stopping criterion NPixStats = self.GD["Deconv"]["NumRMSSamples"] if NPixStats: RandomInd = np.int64(np.random.rand(NPixStats) * npix**2) RMS = np.std(np.real(PeakMap.ravel()[RandomInd])) else: RMS = np.std(PeakMap) self.RMS = RMS self.GainMachine.SetRMS(RMS) Fluxlimit_RMS = self.RMSFactor * RMS #Find position and intensity of first peak x, y, MaxDirty = NpParallel.A_whereMax(PeakMap, NCPU=self.NCPU, DoAbs=DoAbs, Mask=self.MaskArray) #Get peak factor stopping criterion Fluxlimit_Peak = MaxDirty * self.PeakFactor #Get side lobe stopping criterion Fluxlimit_Sidelobe = ( (self.CycleFactor - 1.) / 4. * (1. - self.SideLobeLevel) + self.SideLobeLevel) * MaxDirty if self.CycleFactor else 0 mm0, mm1 = PeakMap.min(), PeakMap.max() # Choose whichever threshold is highest StopFlux = max(Fluxlimit_Peak, Fluxlimit_RMS, Fluxlimit_Sidelobe, self.FluxThreshold) print >> log, " Dirty image peak flux = %10.6f Jy [(min, max) = (%.3g, %.3g) Jy]" % ( MaxDirty, mm0, mm1) print >> log, " RMS-based threshold = %10.6f Jy [rms = %.3g Jy; RMS factor %.1f]" % ( Fluxlimit_RMS, RMS, self.RMSFactor) print >> log, " Sidelobe-based threshold = %10.6f Jy [sidelobe = %.3f of peak; cycle factor %.1f]" % ( Fluxlimit_Sidelobe, self.SideLobeLevel, self.CycleFactor) print >> log, " Peak-based threshold = %10.6f Jy [%.3f of peak]" % ( Fluxlimit_Peak, self.PeakFactor) print >> log, " Absolute threshold = %10.6f Jy" % ( self.FluxThreshold) print >> log, " Stopping flux = %10.6f Jy [%.3f of peak ]" % ( StopFlux, StopFlux / MaxDirty) T = ClassTimeIt.ClassTimeIt() T.disable() ThisFlux = MaxDirty #print x,y if ThisFlux < StopFlux: print >> log, ModColor.Str( " Initial maximum peak %g Jy below threshold, we're done CLEANing" % (ThisFlux), col="green") exit_msg = exit_msg + " " + "FluxThreshold" continue_deconvolution = False or continue_deconvolution update_model = False or update_model # No need to do anything further if we are already at the stopping flux return exit_msg, continue_deconvolution, update_model # set peak in GainMachine (deprecated?) self.GainMachine.SetFluxMax(ThisFlux) # def GivePercentDone(ThisMaxFlux): # fracDone=1.-(ThisMaxFlux-StopFlux)/(MaxDirty-StopFlux) # return max(int(round(100*fracDone)),100) #Do minor cycle deconvolution loop try: for i in range(self._niter + 1, self.MaxMinorIter + 1): self._niter = i #grab a new peakmap PeakMap = self.Dirty[0, :, :] x, y, ThisFlux = NpParallel.A_whereMax(PeakMap, NCPU=self.NCPU, DoAbs=DoAbs, Mask=self.MaskArray) # deprecated? self.GainMachine.SetFluxMax(ThisFlux) T.timeit("max0") if ThisFlux <= StopFlux: print >> log, ModColor.Str( " CLEANing [iter=%i] peak of %.3g Jy lower than stopping flux" % (i, ThisFlux), col="green") cont = ThisFlux > self.FluxThreshold if not cont: print >> log, ModColor.Str( " CLEANing [iter=%i] absolute flux threshold of %.3g Jy has been reached" % (i, self.FluxThreshold), col="green", Bold=True) exit_msg = exit_msg + " " + "MinFluxRms" continue_deconvolution = cont or continue_deconvolution update_model = True or update_model break # stop cleaning if threshold reached # This is used to track Cleaning progress rounded_iter_step = 1 if i < 10 else (10 if i < 200 else ( 100 if i < 2000 else 1000)) # min(int(10**math.floor(math.log10(i))), 10000) if i >= 10 and i % rounded_iter_step == 0: # if self.GD["Debug"]["PrintMinorCycleRMS"]: #rms = np.std(np.real(self._CubeDirty.ravel()[self.IndStats])) print >> log, " [iter=%i] peak residual %.3g" % ( i, ThisFlux) nch, npol, _, _ = self._Dirty.shape #Fpol contains the intensities at (x,y) per freq and polarisation Fpol = np.zeros([nch, npol, 1, 1], dtype=np.float32) if self.MultiFreqMode: if self.GD["Hogbom"]["FreqMode"] == "Poly": Ncoeffs = self.GD["Hogbom"]["PolyFitOrder"] elif self.GD["Hogbom"]["FreqMode"] == "GPR": Ncoeffs = self.GD["Hogbom"]["NumBasisFuncs"] else: raise NotImplementedError( "FreqMode %s not supported" % self.GD["Hogbom"]["FreqMode"]) Coeffs = np.zeros([npol, Ncoeffs]) else: Coeffs = np.zeros([npol, nch]) # to support per channel cleaning # Get the JonesNorm JonesNorm = (self.DicoDirty["JonesNorm"][:, :, x, y]).reshape( (nch, npol, 1, 1)) # Get the solution Fpol[:, 0, 0, 0] = self._Dirty[:, 0, x, y] / np.sqrt( JonesNorm[:, 0, 0, 0]) # Fit a polynomial to get coeffs # tmp = self.ModelMachine.FreqMachine.Fit(Fpol[:, 0, 0, 0]) # print tmp.shape Coeffs[0, :] = self.ModelMachine.FreqMachine.Fit(Fpol[:, 0, 0, 0]) # Overwrite with polynoimial fit Fpol[:, 0, 0, 0] = self.ModelMachine.FreqMachine.Eval(Coeffs[0, :]) T.timeit("stuff") #Find PSF corresponding to location (x,y) self.PSFServer.setLocation( x, y) #Selects the facet closest to (x,y) PSF, meanPSF = self.PSFServer.GivePSF() #Gives associated PSF _, _, PSFnx, PSFny = PSF.shape # Normalise PSF in each channel PSF /= np.amax(PSF.reshape(nch, npol, PSFnx * PSFny), axis=2, keepdims=True).reshape(nch, npol, 1, 1) T.timeit("FindScale") CurrentGain = self.GainMachine.GiveGain() #Update model self.ModelMachine.AppendComponentToDictStacked((x, y), 1.0, Coeffs[0, :], 0) # Subtract LocalSM*CurrentGain from dirty image self.SubStep((x, y), PSF * Fpol * CurrentGain * np.sqrt(JonesNorm)) T.timeit("SubStep") T.timeit("End") except KeyboardInterrupt: print >> log, ModColor.Str( " CLEANing [iter=%i] minor cycle interrupted with Ctrl+C, peak flux %.3g" % (self._niter, ThisFlux)) exit_msg = exit_msg + " " + "MaxIter" continue_deconvolution = False or continue_deconvolution update_model = True or update_model return exit_msg, continue_deconvolution, update_model if self._niter >= self.MaxMinorIter: #Reached maximum number of iterations: print >> log, ModColor.Str( " CLEANing [iter=%i] Reached maximum number of iterations, peak flux %.3g" % (self._niter, ThisFlux)) exit_msg = exit_msg + " " + "MaxIter" continue_deconvolution = False or continue_deconvolution update_model = True or update_model return exit_msg, continue_deconvolution, update_model
def ReadMSInfos(self): DicoMSInfos = {} MSName=self.ListMSName[0] t0 = table(MSName, ack=False) tf0 = table("%s::SPECTRAL_WINDOW"%self.ListMSName[0], ack=False) self.ChanWidth = tf0.getcol("CHAN_WIDTH").ravel()[0] tf0.close() self.times = np.sort(np.unique(t0.getcol("TIME"))) self.dt=(self.times[1::]-self.times[0:-1]).min() t0.close() ta = table("%s::ANTENNA"%self.ListMSName[0], ack=False) self.na=ta.getcol("POSITION").shape[0] ta.close() tField = table("%s::FIELD"%MSName, ack=False) self.ra0, self.dec0 = tField.getcol("PHASE_DIR").ravel() # radians! if self.ra0<0.: self.ra0+=2.*np.pi tField.close() pBAR = ProgressBar(Title="Reading metadata") pBAR.render(0, self.nMS) #for iMS, MSName in enumerate(sorted(self.ListMSName)): for iMS, MSName in enumerate(self.ListMSName): try: t = table(MSName, ack=False) except Exception as e: s = str(e) DicoMSInfos[iMS] = {"Readable": False, "Exception": s} pBAR.render(iMS+1, self.nMS) continue if self.ColName not in t.colnames(): DicoMSInfos[iMS] = {"Readable": False, "Exception": "Missing Data colname %s"%self.ColName} pBAR.render(iMS+1, self.nMS) continue if self.ColWeights and (self.ColWeights not in t.colnames()): DicoMSInfos[iMS] = {"Readable": False, "Exception": "Missing Weights colname %s"%self.ColWeights} pBAR.render(iMS+1, self.nMS) continue if self.ModelName and (self.ModelName not in t.colnames()): DicoMSInfos[iMS] = {"Readable": False, "Exception": "Missing Model colname %s"%self.ModelName} pBAR.render(iMS+1, self.nMS) continue tf = table("%s::SPECTRAL_WINDOW"%MSName, ack=False) ThisTimes = np.unique(t.getcol("TIME")) if not np.allclose(ThisTimes, self.times): raise ValueError("should have the same times") DicoMSInfos[iMS] = {"MSName": MSName, "ChanFreq": tf.getcol("CHAN_FREQ").ravel(), # Hz "ChanWidth": tf.getcol("CHAN_WIDTH").ravel(), # Hz "times": ThisTimes, "startTime": Time(ThisTimes[0]/(24*3600.), format='mjd', scale='utc').isot, "stopTime": Time(ThisTimes[-1]/(24*3600.), format='mjd', scale='utc').isot, "deltaTime": (ThisTimes[-1] - ThisTimes[0])/3600., # h "Readable": True} if DicoMSInfos[iMS]["ChanWidth"][0] != self.ChanWidth: raise ValueError("should have the same chan width") pBAR.render(iMS+1, self.nMS) for iMS in range(self.nMS): if not DicoMSInfos[iMS]["Readable"]: print(ModColor.Str("Problem reading %s"%MSName), file=log) print(ModColor.Str(" %s"%DicoMSInfos[iMS]["Exception"]), file=log) t.close() tf.close() self.DicoMSInfos = DicoMSInfos self.FreqsAll = np.array([DicoMSInfos[iMS]["ChanFreq"] for iMS in list(DicoMSInfos.keys()) if DicoMSInfos[iMS]["Readable"]]) self.Freq_minmax = np.min(self.FreqsAll), np.max(self.FreqsAll) self.NTimes = self.times.size f0, f1 = self.Freq_minmax self.NChan = int((f1 - f0)/self.ChanWidth) + 1 # Fill properties self.tStart = DicoMSInfos[0]["startTime"] self.tStop = DicoMSInfos[0]["stopTime"] self.fMin = self.Freq_minmax[0] self.fMax = self.Freq_minmax[1] self.iCurrentMS=0