def CleanFiles(MSName):
def EX(ss):
#print ss
os.system(ss)
BaseImageName = os.path.abspath(MSName).split("_")[-3].split(".")[1]
MSName = os.path.abspath(MSName).split("/")[-1]
if not os.path.isdir("PRODUCTS"):
EX("mkdir -p %s" % PDir)
KEEP = [
"%s_m.AP_m.app.restored.fits" % BaseImageName,
"%s_m.AP_m.tessel.reg" % BaseImageName,
"%s_m.AP_m.parset" % BaseImageName,
"%s_m.AP_m.log" % BaseImageName,
"%s_m.AP_m.DicoModel" % BaseImageName,
"%s_m.AP.app.restored.fits.mask.fits" % BaseImageName,
"%s_m.app.restored.pybdsm.srl.fits.ClusterCat.npy" % BaseImageName,
"SOLS_%s" % MSName
]
log.print(ModColor.Str("Moving %s to %s" % (str(KEEP), PDir)))
for File in KEEP:
EX("mv %s %s" % (File, PDir))
log.print(ModColor.Str("Delete the rest..."))
EX("rm -rf %s*" % BaseImageName)
EX("rm -rf %s*.ddfcache" % MSName)
def invSVD(A,Cut=1e-6):
#print "rand"
Ar=A # +np.random.randn(*A.shape)*(1e-6*A.max())
#print "stard",Ar.shape
try:
u,s,v=np.linalg.svd(Ar)
except:
Name="errSVDArray_%i"%int(np.random.rand(1)[0]*10000)
print ModColor.Str("Problem inverting Matrix, saving as %s"%Name)
print ModColor.Str(" will make it svd-able")
np.save(Name,Ar)
# weird - I found a matrix I cannot do svd on... - that works
Cut=1e-20
#Ar=np.complex64(Ar)
u,s,v=np.linalg.svd(np.complex64(Ar))#+np.random.randn(*Ar.shape)*(1e-10*np.abs(Ar).max()))
#u,s,v=np.linalg.svd()
u=np.real(u)
s=np.real(s)
v=np.real(v)
#u,s,v=np.linalg.svd(np.complex128(Ar))
#print "ok"
s[s<0.]=Cut
s[s<Cut*s.max()]=Cut*s.max()
ssq=(1./s)
#Asq=np.conj(np.dot(np.dot(v.T,ssq),u.T))
v0=v.T*ssq.reshape(1,ssq.size)
Asq=np.conj(np.dot(v0,u.T))
return Asq
def giveOutIm(self, Nout):
A = self.A
nch, npol, Nin, _ = A.shape
if Nin == Nout:
return A.copy()
elif Nin > Nout:
B = np.zeros((nch, npol, Nout, Nout), A.dtype)
print >> log, ModColor.Str(
" Output image smaller than input image")
print >> log, ModColor.Str(" adapting %s --> %s" %
(str(A.shape), str(B.shape)))
# Input image larger than requested
N0 = A.shape[-1]
xc0 = yc0 = N0 / 2
x0d, x1d = xc0 - Nout / 2, xc0 - Nout / 2 + Nout
s = slice(x0d, x1d)
B[:, :, :, :] = A[:, :, s, s]
return B
else:
B = np.zeros((nch, npol, Nout, Nout), A.dtype)
# Input image smaller - has to zero pad
print >> log, ModColor.Str(
" Output image larger than input image")
print >> log, ModColor.Str(" adapting %s --> %s" %
(str(A.shape), str(B.shape)))
Na = A.shape[-1]
Nb = B.shape[-1]
xa = Na / 2
xb = Nb / 2
x0d, x1d = xb - xa, xb - xa + Na
s = slice(x0d, x1d)
B[:, :, s, s] = A[:, :, :, :]
return B
def RunBDSM(self,
filename,
rmsmean_map_filename=None,
WriteNoise=True,
Ratio=None):
CatName = filename[:-5] + ".pybdsm.gaul.fits"
if os.path.isfile(CatName):
print >> log, ModColor.Str("File %s exists" % CatName, col="green")
print >> log, ModColor.Str(" Skipping...", col="green")
return
rmsmean_map_filename = []
if self.rmsmean_map:
l = self.rmsmean_map
l = l.replace("[", "")
l = l.replace("]", "")
l = l.split(",")
rmsmean_map_filename = l
img = bdsf.process_image(filename,
thresh_isl=self.bdsm_thresh_isl,
thresh_pix=self.bdsm_thresh_pix,
rms_box=self.bdsm_rms_box,
rmsmean_map_filename=rmsmean_map_filename)
img.write_catalog(catalog_type="srl", clobber=True, format="fits")
img.write_catalog(catalog_type="gaul", clobber=True, format="fits")
if WriteNoise:
img.export_image(img_type="rms", clobber=True)
img.export_image(img_type="mean", clobber=True)
def Print(self, RejectGroups=[], dest=sys.stdout):
P = ClassPrint.ClassPrint(HW=50)
print >> dest, ModColor.Str(" Selected Options:")
for Group, V in self.DefaultDict.items():
if Group in RejectGroups:
continue
try:
GroupTitle = self.DicoGroupDesc[Group]
except:
GroupTitle = Group
print >> dest, ModColor.Str("[%s] %s" % (Group, GroupTitle),
col="green")
option_list = self.DefaultDict[Group]
for oname in option_list:
if oname[0] != "_": # skip "internal" values such as "_Help"
V = self.DefaultDict[Group][oname]
attrs = self.AttrDict.get(Group).get(oname, {})
if not attrs.get('alias_of') and not attrs.get(
"cmdline_only"): # and V!="":
if V == "": V = "''"
P.Print(oname, V, dest=dest)
print
def _run_worker(self, queue):
"""
Runs worker loop on given queue. Waits on queue, picks off job items, looks them up in context table,
calls them, and returns results in the work queue.
"""
if self.verbose > 0:
print >> log, ModColor.Str("started worker pid %d" % os.getpid())
try:
pill = True
# While no poisoned pill has been given grab items from the queue.
while pill:
try:
# Get queue item, or timeout and check if pill perscribed.
#print>>log,"%s: calling queue.get()"%AsyncProcessPool.proc_id
jobitem = queue.get(True, 10)
#print>>log,"%s: queue.get() returns %s"%(AsyncProcessPool.proc_id, jobitem)
except Queue.Empty:
continue
if jobitem == "POISON-E":
if self.verbose:
print >> log, "got pill. Qin:{} Qout:{}".format(
queue.qsize(), self._result_queue.qsize())
break
elif jobitem is not None:
self._dispatch_job(jobitem)
except KeyboardInterrupt:
print >> log, ModColor.Str("Ctrl+C caught, exiting", col="red")
return
def __init__(self, dirname, reset=False, cachedir=None, nfswarn=False):
"""
Initializes cache manager.
Args:
dirname: directory in which caches are kept
reset: if True, cache is reset upon first access
cachedir: if set, caches things under cachedir/dirname. Useful for fast local storage.
nfswarn: if True and directory is NFS mounted, prints a warning
"""
# strip trailing slashes
while dirname[-1] == "/":
dirname = dirname[:-1]
if cachedir:
dirname = os.path.join(cachedir, os.path.basename(dirname))
self.dirname = dirname
self.hashes = {}
self.pid = os.getpid()
if not os.path.exists(dirname):
print("cache directory %s does not exist, creating" % dirname,
file=log)
os.mkdir(dirname)
else:
if reset:
print(
"clearing cache %s, since we were asked to reset the cache"
% dirname,
file=log)
os.system("rm -fr " + dirname)
os.mkdir(dirname)
# check for NFS system and print warning
if nfswarn:
try:
fstype = subprocess.check_output(
("stat --file-system --format=%T " +
dirname).split()).strip()
except:
print(ModColor.Str(
"WARNING: unable to determine filesystem type for %s" %
dirname,
col="red",
Bold=True),
file=log)
fstype = "unknown"
if fstype == "nfs":
print(ModColor.Str(
"WARNING: cache directory %s is mounted via NFS." %
dirname,
col="red",
Bold=True),
file=log)
print(ModColor.Str(
"This may cause performance issues. Consider using the --Cache-Dir option.",
col="red",
Bold=True),
file=log)
def os_exec(ss, CheckFile=None):
print()
log.print(
ModColor.Str(
"====================================================================================================================",
col="blue"))
if CheckFile is not None:
if os.path.isfile(CheckFile):
log.print(ModColor.Str("%s exists..." % CheckFile, col="green"))
log.print(
ModColor.Str(" skipping step: ", col="green") + " %s" % ss)
return
log.print(ModColor.Str("Executing %s" % ss))
os.system(ss)
def _start_worker(object,
proc_id,
affinity,
worker_queue,
pause_on_start=False):
"""
Helper method for worker process startup. ets up affinity, and calls _run_worker method on
object with the specified work queue.
Args:
object:
proc_id:
affinity:
work_queue:
Returns:
"""
if pause_on_start:
os.kill(os.getpid(), signal.SIGSTOP)
numexpr.set_num_threads(
1) # no sub-threads in workers, as it messes with everything
_pyArrays.pySetOMPNumThreads(1)
_pyArrays.pySetOMPDynamicNumThreads(1)
AsyncProcessPool.proc_id = proc_id
MyLogger.subprocess_id = proc_id
if affinity:
psutil.Process().cpu_affinity(affinity)
object._run_worker(worker_queue)
if object.verbose:
print >> log, ModColor.Str("exiting worker pid %d" % os.getpid())
def _initIsland_worker(self, DicoOut, iIsland, Island, DicoVariablePSF,
DicoDirty, DicoParm, FacetCache, NCPU):
logger.setSilent([
"ClassImageDeconvMachineMSMF", "ClassPSFServer",
"ClassMultiScaleMachine", "GiveModelMachine",
"ClassModelMachineMSMF"
])
self.InitMachine.Init(DicoVariablePSF,
DicoParm["GridFreqs"],
DicoParm["DegridFreqs"],
facetcache=FacetCache)
self.InitMachine.setDirty(DicoDirty)
#self.InitMachine.DeconvMachine.setNCPU(NCPU)
self.InitMachine.setSSDModelImage(DicoParm["ModelImage"])
#print ":::::::::::::::::::::::",iIsland
try:
SModel, AModel = self.InitMachine.giveModel(Island)
except:
if not self.GD["GAClean"]["ParallelInitHMP"]:
raise
print >> log, traceback.format_exc()
FileOut = "errIsland_%6.6i.npy" % iIsland
print >> log, ModColor.Str(
"...... error on island %i, saving to file %s" %
(iIsland, FileOut))
np.save(FileOut, np.array(Island))
self.InitMachine.Reset()
return
DicoOut["S"] = SModel
DicoOut["Alpha"] = AModel
self.InitMachine.Reset()
def setRefFreq(self, RefFreq, Force=False):
if self.RefFreq is not None and not Force:
print(ModColor.Str("Reference frequency already set to %f MHz" % (self.RefFreq/1e6)), file=log)
return
self.RefFreq = RefFreq
self.DicoSMStacked["RefFreq"] = RefFreq
def CreateShared(Name, shape, dtype):
try:
a = SharedArray.create(Name, shape, dtype=dtype)
except OSError:
print >> log, ModColor.Str("File %s exists, deleting" % Name)
DelArray(Name)
a = SharedArray.create(Name, shape, dtype=dtype)
return a
def setRefFreq(self, RefFreq, Force=False): #,AllFreqs):
if self.RefFreq is not None and not Force:
print >> log, ModColor.Str(
"Reference frequency already set to %f MHz" %
(self.RefFreq / 1e6))
return
self.RefFreq = RefFreq
self.DicoModel["RefFreq"] = RefFreq
def setLoud(Lname):
print>>itsLog, ModColor.Str("Set loud: %s" % Lname, col="green")
if type(Lname)==str:
log=getLogger(Lname)
log.logger.setLevel(logging.DEBUG)
elif type(Lname)==list:
for name in Lname:
log=getLogger(name)
log.logger.setLevel(logging.DEBUG)
def _dispatch_job(self, jobitem, reraise=False):
"""Handles job described by jobitem dict.
If reraise is True, any eceptions are re-raised. This is useful for debugging."""
timer = ClassTimeIt.ClassTimeIt()
event = counter = None
try:
job_id, event_id, counter_id, args, kwargs = [jobitem.get(attr) for attr in
"job_id", "event", "counter", "args", "kwargs"]
handler_id, method, handler_desc = jobitem["handler"]
handler = self._job_handlers.get(handler_id)
if handler is None:
raise RuntimeError("Job %s: unknown handler %s. This is a bug." % (job_id, handler_desc))
event, eventname = self._events[event_id] if event_id is not None else (None, None)
# find counter object, if specified
if counter_id:
counter = self._job_counters.get(counter_id)
if counter is None:
raise RuntimeError("Job %s: unknown counter %s. This is a bug." % (job_id, counter_id))
# instantiate SharedDict arguments
# timer.timeit('init '+job_id)
args = [ arg.instantiate() if type(arg) is shared_dict.SharedDictRepresentation else arg for arg in args ]
for key in kwargs.keys():
if type(kwargs[key]) is shared_dict.SharedDictRepresentation:
kwargs[key] = kwargs[key].instantiate()
# timer.timeit('instantiated '+job_id)
# call the job
if self.verbose > 1:
print>> log, "job %s: calling %s" % (job_id, handler_desc)
if method is None:
# call object directly
result = handler(*args, **kwargs)
else:
call = getattr(handler, method, None)
if not callable(call):
raise KeyError("Job %s: unknown method '%s' for handler %s" % (job_id, method, handler_desc))
result = call(*args, **kwargs)
if self.verbose > 3:
print>> log, "job %s: %s returns %s" % (job_id, handler_desc, result)
# Send result back
if jobitem['collect_result']:
self._result_queue.put(
dict(job_id=job_id, proc_id=self.proc_id, success=True, result=result, time=timer.seconds()))
except KeyboardInterrupt:
raise
except Exception, exc:
if reraise:
raise
print>> log, ModColor.Str("process %s: exception raised processing job %s: %s" % (
AsyncProcessPool.proc_id, job_id, traceback.format_exc()))
if jobitem['collect_result']:
self._result_queue.put(
dict(job_id=job_id, proc_id=self.proc_id, success=False, error=exc, time=timer.seconds()))
def SharedToDico(Prefix):
print >> log, ModColor.Str("SharedToDico: start [prefix = %s]" % Prefix)
Lnames = ListNames()
keys = [Name for Name in Lnames if Prefix in Name]
if len(keys) == 0:
return None
DicoOut = {}
for Sharedkey in keys:
key = Sharedkey.split(".")[-1]
print >> log, ModColor.Str(" %s -> %s" % (Sharedkey, key))
Shared = GiveArray(Sharedkey)
if isinstance(Shared, type(None)):
print >> log, ModColor.Str(" None existing key %s" % (key))
return None
DicoOut[key] = Shared
print >> log, ModColor.Str("SharedToDico: done")
return DicoOut
def __init__(self,
ListMSName=None,
ColName="DATA",
ModelName="PREDICT_KMS",
UVRange=[1., 1000.],
ColWeights=None,
SolsName=None,
FileCoords="Transient_LOTTS.csv",
Radius=3.,
NOff=-1,
ImageI=None,
ImageV=None,
SolsDir=None,
NCPU=1,
BaseDirSpecs=None,
BeamModel=None,
BeamNBand=1):
self.ColName = ColName
self.ModelName = ModelName
self.ColWeights = ColWeights
self.BeamNBand = BeamNBand
self.UVRange = UVRange
self.Mode = "Spec"
self.BaseDirSpecs = BaseDirSpecs
self.NOff = NOff
self.SolsName = SolsName
self.NCPU = NCPU
self.BeamModel = BeamModel
if ListMSName is None:
print(ModColor.Str("WORKING IN REPLOT MODE"), file=log)
self.Mode = "Plot"
self.Radius = Radius
self.ImageI = ImageI
self.ImageV = ImageV
self.SolsDir = SolsDir
#self.PosArray=np.genfromtxt(FileCoords,dtype=[('Name','S200'),("ra",np.float64),("dec",np.float64),('Type','S200')],delimiter="\t")
# identify version in logs
print("DynSpecMS version %s starting up" % version(), file=log)
self.FileCoords = FileCoords
if self.Mode == "Spec":
self.ListMSName = sorted(ListMSName) #[0:2]
self.nMS = len(self.ListMSName)
self.OutName = self.ListMSName[0].split("/")[-1].split("_")[0]
self.ReadMSInfos()
self.InitFromCatalog()
elif self.Mode == "Plot":
self.OutName = self.BaseDirSpecs.split("_")[-1]
self.InitFromSpecs()
def DicoToShared(Prefix, Dico, DelInput=False):
DicoOut = {}
print >> log, ModColor.Str("DicoToShared: start [prefix = %s]" % Prefix)
for key in Dico.keys():
if not isinstance(Dico[key], np.ndarray):
continue
# print "%s.%s"%(Prefix,key)
ThisKeyPrefix = "%s.%s" % (Prefix, key)
print >> log, ModColor.Str(" %s -> %s" % (key, ThisKeyPrefix))
ar = Dico[key]
Shared = ToShared(ThisKeyPrefix, ar)
DicoOut[key] = Shared
if DelInput:
del (Dico[key], ar)
if DelInput:
del (Dico)
print >> log, ModColor.Str("DicoToShared: done")
#print ModColor.Str("DicoToShared: done")
return DicoOut
def __init__(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
if self.NCPU == 0:
self.NCPU = psutil.cpu_count() - 1
self.bdsm_rms_box = [int(i) for i in self.bdsm_rms_box.split(",")]
Files = [self.RestoredIm]
for f in Files:
if not os.path.isfile(f):
raise ValueError("File %s does not exist" % f)
print >> log, ModColor.Str("File %s exist" % f, col="green")
def GiveSpectralIndexMap(self, CellSizeRad=1., GaussPars=[(1, 1, 0)], DoConv=True, MaxSpi=100, MaxDR=1e+6,
threshold=None):
dFreq = 1e6
# f0=self.DicoSMStacked["AllFreqs"].min()
# f1=self.DicoSMStacked["AllFreqs"].max()
RefFreq = self.DicoSMStacked["RefFreq"]
f0 = RefFreq / 1.5
f1 = RefFreq * 1.5
M0 = self.GiveModelImage(f0)
M1 = self.GiveModelImage(f1)
if DoConv:
# M0=ModFFTW.ConvolveGaussian(M0,CellSizeRad=CellSizeRad,GaussPars=GaussPars)
# M1=ModFFTW.ConvolveGaussian(M1,CellSizeRad=CellSizeRad,GaussPars=GaussPars)
# M0,_=ModFFTW.ConvolveGaussianWrapper(M0,Sig=GaussPars[0][0]/CellSizeRad)
# M1,_=ModFFTW.ConvolveGaussianWrapper(M1,Sig=GaussPars[0][0]/CellSizeRad)
M0, _ = ModFFTW.ConvolveGaussianScipy(M0, Sig=GaussPars[0][0] / CellSizeRad)
M1, _ = ModFFTW.ConvolveGaussianScipy(M1, Sig=GaussPars[0][0] / CellSizeRad)
# print M0.shape,M1.shape
# compute threshold for alpha computation by rounding DR threshold to .1 digits (i.e. 1.65e-6 rounds to 1.7e-6)
if threshold is not None:
minmod = threshold
elif not np.all(M0 == 0):
minmod = float("%.1e" % (np.max(np.abs(M0)) / MaxDR))
else:
minmod = 1e-6
# mask out pixels above threshold
mask = (M1 < minmod) | (M0 < minmod)
print("computing alpha map for model pixels above %.1e Jy (based on max DR setting of %g)" % (
minmod, MaxDR), file=log)
M0[mask] = minmod
M1[mask] = minmod
# with np.errstate(invalid='ignore'):
# alpha = (np.log(M0)-np.log(M1))/(np.log(f0/f1))
# print
# print np.min(M0),np.min(M1),minmod
# print
alpha = (np.log(M0) - np.log(M1)) / (np.log(f0 / f1))
alpha[mask] = 0
# mask out |alpha|>MaxSpi. These are not physically meaningful anyway
mask = alpha > MaxSpi
alpha[mask] = MaxSpi
masked = mask.any()
mask = alpha < -MaxSpi
alpha[mask] = -MaxSpi
if masked or mask.any():
print(ModColor.Str("WARNING: some alpha pixels outside +/-%g. Masking them." % MaxSpi, col="red"), file=log)
return alpha
def filter(self, event):
vss = float(_memory()/(1024**3))
vss_peak = float(_memory_peak()/(1024**3))
rss = float(_resident()/(1024**3))
rss_peak = float(_resident_peak()/(1024**3))
shm = float(_shmem_size()/(1024**3))
setattr(event,"virtual_memory_gb",vss)
setattr(event,"resident_memory_gb",rss)
setattr(event,"shared_memory_gb",shm)
if log_memory and hasattr(event,"msg"):
event.msg = "[%.1f/%.1f %.1f/%.1f %.1fGb] "%(rss,rss_peak,vss,vss_peak,shm) + event.msg
if subprocess_id:
event.msg = ModColor.Str("[%s] "%subprocess_id, col="blue") + event.msg
return True
def _handle_exception(type, value, tb):
if hasattr(sys, 'ps1') or not sys.stderr.isatty():
# we are in interactive mode or we don't have a tty-like
# device, so we call the default hook
sys.__excepthook__(type, value, tb)
else:
print(ModColor.Str(_advise))
import traceback, pdb
# we are NOT in interactive mode, print the exception...
traceback.print_exception(type, value, tb)
print()
# ...then start the debugger in post-mortem mode.
# pdb.pm() # deprecated
pdb.post_mortem(tb) # more "modern"
def run(self):
while not self.kill_received and not self.work_queue.empty():
DicoJob = self.work_queue.get()
# self.initIsland(DicoJob)
try:
self.initIsland(DicoJob)
except:
print traceback.format_exc()
iIsland=DicoJob["iIsland"]
FileOut="errIsland_%6.6i.npy"%iIsland
print ModColor.Str("...... on island %i, saving to file %s"%(iIsland,FileOut))
np.save(FileOut,np.array(self.ListIsland[iIsland]))
print
def PrintOptParse(Obj, ValObj, RejectGroup=[]):
P = ClassPrint.ClassPrint(HW=30)
LGroups = Obj.option_groups
print(ModColor.Str(" Selected Options:"))
for Group in LGroups:
Skip = False
for Name in RejectGroup:
if Name in Group.title:
Skip = True
if Skip: continue
print(ModColor.Str(Group.title, col="green"))
option_list = Group.option_list
for o in option_list:
lopt = o._long_opts[0]
oname = lopt.split("--")[-1]
V = getattr(ValObj, oname)
# if (V!="")&(V is not None):
if True: #V!="":
if V == "": V = "''"
#P.Print(oname,V)
default = o.default
H = o.help
# if H is not None:
# H=o.help.replace("%default",str(default))
# P.Print2(oname,V,H)
# else:
# P.Print(oname,V)
P.Print(oname, V)
# strName=%s
# print " "oname,V
print()
def Print2(self, par, value, helpit, col="white"):
WidthTerm = self.getWidth()
Lpar = len(str(par))
Lval = len(str(value))
SFill = "." * max(self.LeftW - self.Lproto - Lpar - Lval, 2)
WidthHelp = WidthTerm - (self.Lproto + Lpar + Lval + len(SFill))
Spar = "%s" % ModColor.Str(str(par), col=col, Bold=False)
Sval = "%s" % ModColor.Str(str(value), col=col, Bold=False)
if helpit == "":
helpit = "Help yourself"
Shelp = "%s" % helpit
if WidthHelp < 0:
print(self.proto % (Spar, SFill, Sval) + Shelp)
return
Lhelp = len(str(helpit))
listStrHelp = range(0, Lhelp, WidthHelp)
if listStrHelp[-1] != Lhelp: listStrHelp.append(Lhelp)
print(self.proto % (Spar, SFill, Sval) + Shelp[0:WidthHelp])
for i in range(1, len(listStrHelp) - 1):
parout = "%s: %s" % (" " * (self.LeftW - 2),
Shelp[listStrHelp[i]:listStrHelp[i + 1]])
print(parout)
def GiveMMFromDico(self,DicoSMStacked=None):
"""
Initialise a model machine from a dictionary
Input:
DicoSMStacked = Dictionary to instantiate ModelMachine with
"""
if DicoSMStacked["Type"]=="GA":
print>>log,ModColor.Str("Model is of deprecated type GA, overwriting with type SSD")
DicoSMStacked["Type"]="SSD"
if DicoSMStacked is not None: # If the Dict is provided use it to initialise a model machine
Type = DicoSMStacked["Type"]
# backwards compatibility
if Type == "GA":
Type = "SSD"
elif Type == "MSMF":
Type = "HMP"
return self.GiveMM(Type)
else: # If the dict is not provided use the MinorCycleMode to figure out which model machine to initialise
return self.GiveMM(self.GD["Deconv"]["Mode"])
def __init__(self,
ListMSName=None,
ColName="DATA",
ModelName="PREDICT_KMS",
UVRange=[1.,1000.],
ColWeights=None,
SolsName=None,
FileCoords="Transient_LOTTS.csv",
SolsDir=None,
NCPU=1,
BeamModel=None,
BeamNBand=1):
self.ColName = ColName
self.ModelName = ModelName
self.ColWeights = ColWeights
self.BeamNBand = BeamNBand
self.UVRange = UVRange
self.Mode="Spec"
self.SolsName=SolsName
self.NCPU=NCPU
self.BeamModel=BeamModel
self.StepFreq=10#2000
self.StepTime=1
if ListMSName is None:
print(ModColor.Str("WORKING IN REPLOT MODE"), file=log)
self.Mode="Plot"
self.SolsDir=SolsDir
self.FileCoords=FileCoords
self.ListMSName = sorted(ListMSName)#[0:2]
self.nMS = len(self.ListMSName)
self.OutName = self.ListMSName[0].split("/")[-1].split("_")[0]
self.ReadMSInfos()
self.InitFromCatalog()
def GiveMMFromDico(self,DicoSMStacked=None):
"""
Initialise a model machine from a dictionary
Input:
DicoSMStacked = Dictionary to instantiate ModelMachine with
"""
def safe_encode(s):
import six
return s.decode() if isinstance(s, bytes) and six.PY3 else s
Type = safe_encode(DicoSMStacked.get("Type", DicoSMStacked.get(b"Type", None)))
if Type=="GA":
print(ModColor.Str("Model is of deprecated type GA, overwriting with type SSD"), file=log)
DicoSMStacked["Type"]="SSD"
if DicoSMStacked is not None: # If the Dict is provided use it to initialise a model machine
Type = safe_encode(DicoSMStacked.get("Type", DicoSMStacked.get(b"Type", None)))
# backwards compatibility
if Type == "GA":
Type = "SSD"
elif Type == "MSMF":
Type = "HMP"
return self.GiveMM(Type)
else: # If the dict is not provided use the MinorCycleMode to figure out which model machine to initialise
return self.GiveMM(self.GD["Deconv"]["Mode"])
def Deconvolve(self, ch=0, **kwargs):
"""
Runs minor cycle over image channel 'ch'.
initMinor is number of minor iteration (keeps continuous count through major iterations)
Nminor is max number of minor iterations
Returns tuple of: return_code,continue,updated
where return_code is a status string;
continue is True if another cycle should be executed (one or more polarizations still need cleaning);
update is True if one or more polarization models have been updated
"""
#No need to set the channel when doing joint deconvolution
self.setChannel(ch)
exit_msg = ""
continue_deconvolution = False
update_model = False
_, npix, _ = self.Dirty.shape
xc = (npix) / 2
npol, _, _ = self.Dirty.shape
# Get the PeakMap (first index will always be 0 because we only support I cleaning)
PeakMap = self.Dirty[0, :, :]
m0, m1 = PeakMap.min(), PeakMap.max()
#These options should probably be moved into MinorCycleConfig in parset
DoAbs = int(self.GD["Deconv"]["AllowNegative"])
print >> log, " Running minor cycle [MinorIter = %i/%i, SearchMaxAbs = %i]" % (
self._niter, self.MaxMinorIter, DoAbs)
## Determine which stopping criterion to use for flux limit
#Get RMS stopping criterion
NPixStats = self.GD["Deconv"]["NumRMSSamples"]
if NPixStats:
RandomInd = np.int64(np.random.rand(NPixStats) * npix**2)
RMS = np.std(np.real(PeakMap.ravel()[RandomInd]))
else:
RMS = np.std(PeakMap)
self.RMS = RMS
self.GainMachine.SetRMS(RMS)
Fluxlimit_RMS = self.RMSFactor * RMS
#Find position and intensity of first peak
x, y, MaxDirty = NpParallel.A_whereMax(PeakMap,
NCPU=self.NCPU,
DoAbs=DoAbs,
Mask=self.MaskArray)
#Get peak factor stopping criterion
Fluxlimit_Peak = MaxDirty * self.PeakFactor
#Get side lobe stopping criterion
Fluxlimit_Sidelobe = (
(self.CycleFactor - 1.) / 4. * (1. - self.SideLobeLevel) +
self.SideLobeLevel) * MaxDirty if self.CycleFactor else 0
mm0, mm1 = PeakMap.min(), PeakMap.max()
# Choose whichever threshold is highest
StopFlux = max(Fluxlimit_Peak, Fluxlimit_RMS, Fluxlimit_Sidelobe,
self.FluxThreshold)
print >> log, " Dirty image peak flux = %10.6f Jy [(min, max) = (%.3g, %.3g) Jy]" % (
MaxDirty, mm0, mm1)
print >> log, " RMS-based threshold = %10.6f Jy [rms = %.3g Jy; RMS factor %.1f]" % (
Fluxlimit_RMS, RMS, self.RMSFactor)
print >> log, " Sidelobe-based threshold = %10.6f Jy [sidelobe = %.3f of peak; cycle factor %.1f]" % (
Fluxlimit_Sidelobe, self.SideLobeLevel, self.CycleFactor)
print >> log, " Peak-based threshold = %10.6f Jy [%.3f of peak]" % (
Fluxlimit_Peak, self.PeakFactor)
print >> log, " Absolute threshold = %10.6f Jy" % (
self.FluxThreshold)
print >> log, " Stopping flux = %10.6f Jy [%.3f of peak ]" % (
StopFlux, StopFlux / MaxDirty)
T = ClassTimeIt.ClassTimeIt()
T.disable()
ThisFlux = MaxDirty
#print x,y
if ThisFlux < StopFlux:
print >> log, ModColor.Str(
" Initial maximum peak %g Jy below threshold, we're done CLEANing"
% (ThisFlux),
col="green")
exit_msg = exit_msg + " " + "FluxThreshold"
continue_deconvolution = False or continue_deconvolution
update_model = False or update_model
# No need to do anything further if we are already at the stopping flux
return exit_msg, continue_deconvolution, update_model
# set peak in GainMachine (deprecated?)
self.GainMachine.SetFluxMax(ThisFlux)
# def GivePercentDone(ThisMaxFlux):
# fracDone=1.-(ThisMaxFlux-StopFlux)/(MaxDirty-StopFlux)
# return max(int(round(100*fracDone)),100)
#Do minor cycle deconvolution loop
try:
for i in range(self._niter + 1, self.MaxMinorIter + 1):
self._niter = i
#grab a new peakmap
PeakMap = self.Dirty[0, :, :]
x, y, ThisFlux = NpParallel.A_whereMax(PeakMap,
NCPU=self.NCPU,
DoAbs=DoAbs,
Mask=self.MaskArray)
# deprecated?
self.GainMachine.SetFluxMax(ThisFlux)
T.timeit("max0")
if ThisFlux <= StopFlux:
print >> log, ModColor.Str(
" CLEANing [iter=%i] peak of %.3g Jy lower than stopping flux"
% (i, ThisFlux),
col="green")
cont = ThisFlux > self.FluxThreshold
if not cont:
print >> log, ModColor.Str(
" CLEANing [iter=%i] absolute flux threshold of %.3g Jy has been reached"
% (i, self.FluxThreshold),
col="green",
Bold=True)
exit_msg = exit_msg + " " + "MinFluxRms"
continue_deconvolution = cont or continue_deconvolution
update_model = True or update_model
break # stop cleaning if threshold reached
# This is used to track Cleaning progress
rounded_iter_step = 1 if i < 10 else (10 if i < 200 else (
100 if i < 2000 else 1000))
# min(int(10**math.floor(math.log10(i))), 10000)
if i >= 10 and i % rounded_iter_step == 0:
# if self.GD["Debug"]["PrintMinorCycleRMS"]:
#rms = np.std(np.real(self._CubeDirty.ravel()[self.IndStats]))
print >> log, " [iter=%i] peak residual %.3g" % (
i, ThisFlux)
nch, npol, _, _ = self._Dirty.shape
#Fpol contains the intensities at (x,y) per freq and polarisation
Fpol = np.zeros([nch, npol, 1, 1], dtype=np.float32)
if self.MultiFreqMode:
if self.GD["Hogbom"]["FreqMode"] == "Poly":
Ncoeffs = self.GD["Hogbom"]["PolyFitOrder"]
elif self.GD["Hogbom"]["FreqMode"] == "GPR":
Ncoeffs = self.GD["Hogbom"]["NumBasisFuncs"]
else:
raise NotImplementedError(
"FreqMode %s not supported" %
self.GD["Hogbom"]["FreqMode"])
Coeffs = np.zeros([npol, Ncoeffs])
else:
Coeffs = np.zeros([npol,
nch]) # to support per channel cleaning
# Get the JonesNorm
JonesNorm = (self.DicoDirty["JonesNorm"][:, :, x, y]).reshape(
(nch, npol, 1, 1))
# Get the solution
Fpol[:, 0, 0, 0] = self._Dirty[:, 0, x, y] / np.sqrt(
JonesNorm[:, 0, 0, 0])
# Fit a polynomial to get coeffs
# tmp = self.ModelMachine.FreqMachine.Fit(Fpol[:, 0, 0, 0])
# print tmp.shape
Coeffs[0, :] = self.ModelMachine.FreqMachine.Fit(Fpol[:, 0, 0,
0])
# Overwrite with polynoimial fit
Fpol[:, 0, 0,
0] = self.ModelMachine.FreqMachine.Eval(Coeffs[0, :])
T.timeit("stuff")
#Find PSF corresponding to location (x,y)
self.PSFServer.setLocation(
x, y) #Selects the facet closest to (x,y)
PSF, meanPSF = self.PSFServer.GivePSF() #Gives associated PSF
_, _, PSFnx, PSFny = PSF.shape
# Normalise PSF in each channel
PSF /= np.amax(PSF.reshape(nch, npol, PSFnx * PSFny),
axis=2,
keepdims=True).reshape(nch, npol, 1, 1)
T.timeit("FindScale")
CurrentGain = self.GainMachine.GiveGain()
#Update model
self.ModelMachine.AppendComponentToDictStacked((x, y), 1.0,
Coeffs[0, :], 0)
# Subtract LocalSM*CurrentGain from dirty image
self.SubStep((x, y),
PSF * Fpol * CurrentGain * np.sqrt(JonesNorm))
T.timeit("SubStep")
T.timeit("End")
except KeyboardInterrupt:
print >> log, ModColor.Str(
" CLEANing [iter=%i] minor cycle interrupted with Ctrl+C, peak flux %.3g"
% (self._niter, ThisFlux))
exit_msg = exit_msg + " " + "MaxIter"
continue_deconvolution = False or continue_deconvolution
update_model = True or update_model
return exit_msg, continue_deconvolution, update_model
if self._niter >= self.MaxMinorIter: #Reached maximum number of iterations:
print >> log, ModColor.Str(
" CLEANing [iter=%i] Reached maximum number of iterations, peak flux %.3g"
% (self._niter, ThisFlux))
exit_msg = exit_msg + " " + "MaxIter"
continue_deconvolution = False or continue_deconvolution
update_model = True or update_model
return exit_msg, continue_deconvolution, update_model
def ReadMSInfos(self):
DicoMSInfos = {}
MSName=self.ListMSName[0]
t0 = table(MSName, ack=False)
tf0 = table("%s::SPECTRAL_WINDOW"%self.ListMSName[0], ack=False)
self.ChanWidth = tf0.getcol("CHAN_WIDTH").ravel()[0]
tf0.close()
self.times = np.sort(np.unique(t0.getcol("TIME")))
self.dt=(self.times[1::]-self.times[0:-1]).min()
t0.close()
ta = table("%s::ANTENNA"%self.ListMSName[0], ack=False)
self.na=ta.getcol("POSITION").shape[0]
ta.close()
tField = table("%s::FIELD"%MSName, ack=False)
self.ra0, self.dec0 = tField.getcol("PHASE_DIR").ravel() # radians!
if self.ra0<0.: self.ra0+=2.*np.pi
tField.close()
pBAR = ProgressBar(Title="Reading metadata")
pBAR.render(0, self.nMS)
#for iMS, MSName in enumerate(sorted(self.ListMSName)):
for iMS, MSName in enumerate(self.ListMSName):
try:
t = table(MSName, ack=False)
except Exception as e:
s = str(e)
DicoMSInfos[iMS] = {"Readable": False,
"Exception": s}
pBAR.render(iMS+1, self.nMS)
continue
if self.ColName not in t.colnames():
DicoMSInfos[iMS] = {"Readable": False,
"Exception": "Missing Data colname %s"%self.ColName}
pBAR.render(iMS+1, self.nMS)
continue
if self.ColWeights and (self.ColWeights not in t.colnames()):
DicoMSInfos[iMS] = {"Readable": False,
"Exception": "Missing Weights colname %s"%self.ColWeights}
pBAR.render(iMS+1, self.nMS)
continue
if self.ModelName and (self.ModelName not in t.colnames()):
DicoMSInfos[iMS] = {"Readable": False,
"Exception": "Missing Model colname %s"%self.ModelName}
pBAR.render(iMS+1, self.nMS)
continue
tf = table("%s::SPECTRAL_WINDOW"%MSName, ack=False)
ThisTimes = np.unique(t.getcol("TIME"))
if not np.allclose(ThisTimes, self.times):
raise ValueError("should have the same times")
DicoMSInfos[iMS] = {"MSName": MSName,
"ChanFreq": tf.getcol("CHAN_FREQ").ravel(), # Hz
"ChanWidth": tf.getcol("CHAN_WIDTH").ravel(), # Hz
"times": ThisTimes,
"startTime": Time(ThisTimes[0]/(24*3600.), format='mjd', scale='utc').isot,
"stopTime": Time(ThisTimes[-1]/(24*3600.), format='mjd', scale='utc').isot,
"deltaTime": (ThisTimes[-1] - ThisTimes[0])/3600., # h
"Readable": True}
if DicoMSInfos[iMS]["ChanWidth"][0] != self.ChanWidth:
raise ValueError("should have the same chan width")
pBAR.render(iMS+1, self.nMS)
for iMS in range(self.nMS):
if not DicoMSInfos[iMS]["Readable"]:
print(ModColor.Str("Problem reading %s"%MSName), file=log)
print(ModColor.Str(" %s"%DicoMSInfos[iMS]["Exception"]), file=log)
t.close()
tf.close()
self.DicoMSInfos = DicoMSInfos
self.FreqsAll = np.array([DicoMSInfos[iMS]["ChanFreq"] for iMS in list(DicoMSInfos.keys()) if DicoMSInfos[iMS]["Readable"]])
self.Freq_minmax = np.min(self.FreqsAll), np.max(self.FreqsAll)
self.NTimes = self.times.size
f0, f1 = self.Freq_minmax
self.NChan = int((f1 - f0)/self.ChanWidth) + 1
# Fill properties
self.tStart = DicoMSInfos[0]["startTime"]
self.tStop = DicoMSInfos[0]["stopTime"]
self.fMin = self.Freq_minmax[0]
self.fMax = self.Freq_minmax[1]
self.iCurrentMS=0
