def do_scale_convolve(self, MeanDirty):
# convolve mean dirty with each scale in parallel
I = slice(self.Npad, self.NpixPadded - self.Npad)
self.FTMachine.xhatim[...] = iFs(np.pad(MeanDirty[0:1],
((0, 0), (0, 0),
(self.Npad, self.Npad),
(self.Npad, self.Npad)),
mode='constant'),
axes=(2, 3))
self.FTMachine.FFTim()
self.FTMachine.Shat[...] = self.FTMachine.xhatim
kernels = self.GaussianSymmetricFT(self.sigmas[:, None, None, None],
mode='Image')
self.FTMachine.Shat *= iFs(kernels, axes=(2, 3))
self.FTMachine.iSFFT()
ConvMeanDirtys = np.ascontiguousarray(
Fs(self.FTMachine.Shat.real, axes=(2, 3))[:, :, I, I])
# find most relevant scale
maxvals = np.zeros(self.Nscales)
for iScale in xrange(self.Nscales):
# get mask for scale (once auto-masking kicks in we use that instead of external mask)
if self.AppendMaskComponents or not self.GD["WSCMS"]["AutoMask"]:
CurrentMask = self.MaskArray
else:
CurrentMask = self.ScaleMaskArray[str(iScale)]
if iScale:
xtmp, ytmp, ConvMaxDirty = NpParallel.A_whereMax(
ConvMeanDirtys[iScale:iScale + 1],
NCPU=self.NCPU,
DoAbs=self.DoAbs,
Mask=CurrentMask)
else:
xtmp, ytmp, ConvMaxDirty = NpParallel.A_whereMax(
MeanDirty,
NCPU=self.NCPU,
DoAbs=self.DoAbs,
Mask=CurrentMask)
maxvals[iScale] = ConvMaxDirty * self.bias[iScale]
if iScale:
# only update if new scale is more significant
if ConvMaxDirty * self.bias[iScale] >= BiasedMaxVal:
x = xtmp
y = ytmp
BiasedMaxVal = ConvMaxDirty * self.bias[iScale]
MaxDirty = ConvMaxDirty
CurrentDirty = ConvMeanDirtys[iScale:iScale +
1] # [None, None, :, :]
CurrentScale = iScale
else:
x = xtmp
y = ytmp
BiasedMaxVal = ConvMaxDirty * self.bias[iScale]
MaxDirty = ConvMaxDirty
CurrentDirty = MeanDirty
CurrentScale = iScale
return x, y, MaxDirty, CurrentDirty, CurrentScale
def do_scale_convolve(self, MeanDirty):
# convolve mean dirty with each scale in parallel
I = slice(self.Npad, self.NpixPadded - self.Npad)
self.FTMachine.xhatim[...] = iFs(np.pad(MeanDirty[0:1],
((0, 0), (0, 0),
(self.Npad, self.Npad),
(self.Npad, self.Npad)),
mode='constant'),
axes=(2, 3))
self.FTMachine.FFTim()
self.FTMachine.Shat[...] = self.FTMachine.xhatim
kernels = self.GaussianSymmetricFT(self.sigmas[:, None, None, None],
mode='Image')
self.FTMachine.Shat *= iFs(kernels, axes=(2, 3))
self.FTMachine.iSFFT()
ConvMeanDirtys = np.ascontiguousarray(
Fs(self.FTMachine.Shat.real, axes=(2, 3))[:, :, I, I])
# reset the zero scale
# LB - for scale 0 we might want to do scale selection based
# on the convolved image instead of MeanDirty
ConvMeanDirtys[0:1] = MeanDirty.copy()
# initialise to zero so we always trigger the
# if statement below at least once
BiasedMaxVal = 0.0
# find most relevant scale
for iScale in range(self.Nscales):
if iScale not in self.retired_scales:
# get mask for scale (once auto-masking kicks in we use that instead of external mask)
if self.AppendMaskComponents or not self.GD["WSCMS"][
"AutoMask"]:
ScaleMask = self.MaskArray
else:
ScaleMask = self.ScaleMaskArray[str(iScale)]
xtmp, ytmp, ConvMaxDirty = NpParallel.A_whereMax(
ConvMeanDirtys[iScale:iScale + 1],
NCPU=self.NCPU,
DoAbs=self.DoAbs,
Mask=ScaleMask)
if ConvMaxDirty * self.bias[iScale] >= BiasedMaxVal:
x = xtmp
y = ytmp
BiasedMaxVal = ConvMaxDirty * self.bias[iScale]
MaxDirty = ConvMaxDirty
CurrentDirty = ConvMeanDirtys[iScale:iScale + 1]
CurrentScale = iScale
CurrentMask = ScaleMask
if BiasedMaxVal == 0:
print("No scale has been selected. This should never happen. Bug!")
print("Forbidden scales = ", self.forbidden_scales)
return x, y, MaxDirty, CurrentDirty, CurrentScale, CurrentMask
def check_stopping_criteria(self, PeakMap, npix, DoAbs):
# Get RMS stopping criterion
NPixStats = self.GD["Deconv"]["NumRMSSamples"]
if NPixStats:
RandomInd = np.int64(np.random.rand(NPixStats) * npix**2)
RMS = np.std(np.real(PeakMap.ravel()[RandomInd]))
else:
RMS = np.std(PeakMap)
self.RMS = RMS
self.GainMachine.SetRMS(RMS)
Fluxlimit_RMS = self.RMSFactor * RMS
# Find position and intensity of first peak
x, y, MaxDirty = NpParallel.A_whereMax(PeakMap,
NCPU=self.NCPU,
DoAbs=DoAbs,
Mask=self.MaskArray)
# Get peak factor stopping criterion
Fluxlimit_Peak = MaxDirty * self.PeakFactor
# Get side lobe stopping criterion
Fluxlimit_Sidelobe = (
(self.CycleFactor - 1.) / 4. * (1. - self.SideLobeLevel) +
self.SideLobeLevel) * MaxDirty if self.CycleFactor else 0
mm0, mm1 = PeakMap.min(), PeakMap.max()
# Choose whichever threshold is highest
StopFlux = max(Fluxlimit_Peak, Fluxlimit_RMS, Fluxlimit_Sidelobe,
self.FluxThreshold)
print >> log, " Dirty image peak flux = %10.6f Jy [(min, max) = (%.3g, %.3g) Jy]" % (
MaxDirty, mm0, mm1)
print >> log, " RMS-based threshold = %10.6f Jy [rms = %.3g Jy; RMS factor %.1f]" % (
Fluxlimit_RMS, RMS, self.RMSFactor)
print >> log, " Sidelobe-based threshold = %10.6f Jy [sidelobe = %.3f of peak; cycle factor %.1f]" % (
Fluxlimit_Sidelobe, self.SideLobeLevel, self.CycleFactor)
print >> log, " Peak-based threshold = %10.6f Jy [%.3f of peak]" % (
Fluxlimit_Peak, self.PeakFactor)
print >> log, " Absolute threshold = %10.6f Jy" % (
self.FluxThreshold)
print >> log, " Stopping flux = %10.6f Jy [%.3f of peak ]" % (
StopFlux, StopFlux / MaxDirty)
return StopFlux, MaxDirty
def check_stopping_criteria(self):
# Get RMS stopping criterion
RMS = np.std(self._MeanDirty)
Fluxlimit_RMS = self.RMSFactor * RMS
# Find position and intensity of first peak
x, y, MaxDirty = NpParallel.A_whereMax(
self._MeanDirty,
NCPU=self.NCPU,
DoAbs=self.GD["Deconv"]["AllowNegative"],
Mask=self.MaskArray)
# Get peak factor stopping criterion
Fluxlimit_Peak = MaxDirty * self.PeakFactor
# Get side lobe stopping criterion
Fluxlimit_Sidelobe = (
(self.CycleFactor - 1.) / 4. * (1. - self.SideLobeLevel) +
self.SideLobeLevel) * MaxDirty if self.CycleFactor else 0
mm0, mm1 = self._MeanDirty.min(), self._MeanDirty.max()
# Choose whichever threshold is highest
StopFlux = max(Fluxlimit_Peak, Fluxlimit_RMS, Fluxlimit_Sidelobe,
self.FluxThreshold)
print(
" Dirty image peak flux = %10.6f Jy [(min, max) = (%.3g, %.3g) Jy]"
% (MaxDirty, mm0, mm1),
file=log)
print(
" RMS-based threshold = %10.6f Jy [rms = %.3g Jy; RMS factor %.1f]"
% (Fluxlimit_RMS, RMS, self.RMSFactor),
file=log)
print(
" Sidelobe-based threshold = %10.6f Jy [sidelobe = %.3f of peak; cycle factor %.1f]"
% (Fluxlimit_Sidelobe, self.SideLobeLevel, self.CycleFactor),
file=log)
print(" Peak-based threshold = %10.6f Jy [%.3f of peak]" %
(Fluxlimit_Peak, self.PeakFactor),
file=log)
print(" Absolute threshold = %10.6f Jy" %
(self.FluxThreshold),
file=log)
print(" Stopping flux = %10.6f Jy [%.3f of peak ]" %
(StopFlux, StopFlux / MaxDirty),
file=log)
return StopFlux, MaxDirty, RMS
def Deconvolve(self, ch=0, **kwargs):
"""
Runs minor cycle over image channel 'ch'.
initMinor is number of minor iteration (keeps continuous count through major iterations)
Nminor is max number of minor iterations
Returns tuple of: return_code,continue,updated
where return_code is a status string;
continue is True if another cycle should be executed (one or more polarizations still need cleaning);
update is True if one or more polarization models have been updated
"""
#No need to set the channel when doing joint deconvolution
self.setChannel(ch)
exit_msg = ""
continue_deconvolution = False
update_model = False
_, npix, _ = self.Dirty.shape
xc = (npix) / 2
npol, _, _ = self.Dirty.shape
# Get the PeakMap (first index will always be 0 because we only support I cleaning)
PeakMap = self.Dirty[0, :, :]
m0, m1 = PeakMap.min(), PeakMap.max()
#These options should probably be moved into MinorCycleConfig in parset
DoAbs = int(self.GD["Deconv"]["AllowNegative"])
print >> log, " Running minor cycle [MinorIter = %i/%i, SearchMaxAbs = %i]" % (
self._niter, self.MaxMinorIter, DoAbs)
## Determine which stopping criterion to use for flux limit
#Get RMS stopping criterion
NPixStats = self.GD["Deconv"]["NumRMSSamples"]
if NPixStats:
RandomInd = np.int64(np.random.rand(NPixStats) * npix**2)
RMS = np.std(np.real(PeakMap.ravel()[RandomInd]))
else:
RMS = np.std(PeakMap)
self.RMS = RMS
self.GainMachine.SetRMS(RMS)
Fluxlimit_RMS = self.RMSFactor * RMS
#Find position and intensity of first peak
x, y, MaxDirty = NpParallel.A_whereMax(PeakMap,
NCPU=self.NCPU,
DoAbs=DoAbs,
Mask=self.MaskArray)
#Get peak factor stopping criterion
Fluxlimit_Peak = MaxDirty * self.PeakFactor
#Get side lobe stopping criterion
Fluxlimit_Sidelobe = (
(self.CycleFactor - 1.) / 4. * (1. - self.SideLobeLevel) +
self.SideLobeLevel) * MaxDirty if self.CycleFactor else 0
mm0, mm1 = PeakMap.min(), PeakMap.max()
# Choose whichever threshold is highest
StopFlux = max(Fluxlimit_Peak, Fluxlimit_RMS, Fluxlimit_Sidelobe,
self.FluxThreshold)
print >> log, " Dirty image peak flux = %10.6f Jy [(min, max) = (%.3g, %.3g) Jy]" % (
MaxDirty, mm0, mm1)
print >> log, " RMS-based threshold = %10.6f Jy [rms = %.3g Jy; RMS factor %.1f]" % (
Fluxlimit_RMS, RMS, self.RMSFactor)
print >> log, " Sidelobe-based threshold = %10.6f Jy [sidelobe = %.3f of peak; cycle factor %.1f]" % (
Fluxlimit_Sidelobe, self.SideLobeLevel, self.CycleFactor)
print >> log, " Peak-based threshold = %10.6f Jy [%.3f of peak]" % (
Fluxlimit_Peak, self.PeakFactor)
print >> log, " Absolute threshold = %10.6f Jy" % (
self.FluxThreshold)
print >> log, " Stopping flux = %10.6f Jy [%.3f of peak ]" % (
StopFlux, StopFlux / MaxDirty)
T = ClassTimeIt.ClassTimeIt()
T.disable()
ThisFlux = MaxDirty
#print x,y
if ThisFlux < StopFlux:
print >> log, ModColor.Str(
" Initial maximum peak %g Jy below threshold, we're done CLEANing"
% (ThisFlux),
col="green")
exit_msg = exit_msg + " " + "FluxThreshold"
continue_deconvolution = False or continue_deconvolution
update_model = False or update_model
# No need to do anything further if we are already at the stopping flux
return exit_msg, continue_deconvolution, update_model
# set peak in GainMachine (deprecated?)
self.GainMachine.SetFluxMax(ThisFlux)
# def GivePercentDone(ThisMaxFlux):
# fracDone=1.-(ThisMaxFlux-StopFlux)/(MaxDirty-StopFlux)
# return max(int(round(100*fracDone)),100)
#Do minor cycle deconvolution loop
try:
for i in range(self._niter + 1, self.MaxMinorIter + 1):
self._niter = i
#grab a new peakmap
PeakMap = self.Dirty[0, :, :]
x, y, ThisFlux = NpParallel.A_whereMax(PeakMap,
NCPU=self.NCPU,
DoAbs=DoAbs,
Mask=self.MaskArray)
# deprecated?
self.GainMachine.SetFluxMax(ThisFlux)
T.timeit("max0")
if ThisFlux <= StopFlux:
print >> log, ModColor.Str(
" CLEANing [iter=%i] peak of %.3g Jy lower than stopping flux"
% (i, ThisFlux),
col="green")
cont = ThisFlux > self.FluxThreshold
if not cont:
print >> log, ModColor.Str(
" CLEANing [iter=%i] absolute flux threshold of %.3g Jy has been reached"
% (i, self.FluxThreshold),
col="green",
Bold=True)
exit_msg = exit_msg + " " + "MinFluxRms"
continue_deconvolution = cont or continue_deconvolution
update_model = True or update_model
break # stop cleaning if threshold reached
# This is used to track Cleaning progress
rounded_iter_step = 1 if i < 10 else (10 if i < 200 else (
100 if i < 2000 else 1000))
# min(int(10**math.floor(math.log10(i))), 10000)
if i >= 10 and i % rounded_iter_step == 0:
# if self.GD["Debug"]["PrintMinorCycleRMS"]:
#rms = np.std(np.real(self._CubeDirty.ravel()[self.IndStats]))
print >> log, " [iter=%i] peak residual %.3g" % (
i, ThisFlux)
nch, npol, _, _ = self._Dirty.shape
#Fpol contains the intensities at (x,y) per freq and polarisation
Fpol = np.zeros([nch, npol, 1, 1], dtype=np.float32)
if self.MultiFreqMode:
if self.GD["Hogbom"]["FreqMode"] == "Poly":
Ncoeffs = self.GD["Hogbom"]["PolyFitOrder"]
elif self.GD["Hogbom"]["FreqMode"] == "GPR":
Ncoeffs = self.GD["Hogbom"]["NumBasisFuncs"]
else:
raise NotImplementedError(
"FreqMode %s not supported" %
self.GD["Hogbom"]["FreqMode"])
Coeffs = np.zeros([npol, Ncoeffs])
else:
Coeffs = np.zeros([npol,
nch]) # to support per channel cleaning
# Get the JonesNorm
JonesNorm = (self.DicoDirty["JonesNorm"][:, :, x, y]).reshape(
(nch, npol, 1, 1))
# Get the solution
Fpol[:, 0, 0, 0] = self._Dirty[:, 0, x, y] / np.sqrt(
JonesNorm[:, 0, 0, 0])
# Fit a polynomial to get coeffs
# tmp = self.ModelMachine.FreqMachine.Fit(Fpol[:, 0, 0, 0])
# print tmp.shape
Coeffs[0, :] = self.ModelMachine.FreqMachine.Fit(Fpol[:, 0, 0,
0])
# Overwrite with polynoimial fit
Fpol[:, 0, 0,
0] = self.ModelMachine.FreqMachine.Eval(Coeffs[0, :])
T.timeit("stuff")
#Find PSF corresponding to location (x,y)
self.PSFServer.setLocation(
x, y) #Selects the facet closest to (x,y)
PSF, meanPSF = self.PSFServer.GivePSF() #Gives associated PSF
_, _, PSFnx, PSFny = PSF.shape
# Normalise PSF in each channel
PSF /= np.amax(PSF.reshape(nch, npol, PSFnx * PSFny),
axis=2,
keepdims=True).reshape(nch, npol, 1, 1)
T.timeit("FindScale")
CurrentGain = self.GainMachine.GiveGain()
#Update model
self.ModelMachine.AppendComponentToDictStacked((x, y), 1.0,
Coeffs[0, :], 0)
# Subtract LocalSM*CurrentGain from dirty image
self.SubStep((x, y),
PSF * Fpol * CurrentGain * np.sqrt(JonesNorm))
T.timeit("SubStep")
T.timeit("End")
except KeyboardInterrupt:
print >> log, ModColor.Str(
" CLEANing [iter=%i] minor cycle interrupted with Ctrl+C, peak flux %.3g"
% (self._niter, ThisFlux))
exit_msg = exit_msg + " " + "MaxIter"
continue_deconvolution = False or continue_deconvolution
update_model = True or update_model
return exit_msg, continue_deconvolution, update_model
if self._niter >= self.MaxMinorIter: #Reached maximum number of iterations:
print >> log, ModColor.Str(
" CLEANing [iter=%i] Reached maximum number of iterations, peak flux %.3g"
% (self._niter, ThisFlux))
exit_msg = exit_msg + " " + "MaxIter"
continue_deconvolution = False or continue_deconvolution
update_model = True or update_model
return exit_msg, continue_deconvolution, update_model
def Deconvolve(self, Nminor=None, ch=0):
if Nminor is None:
Nminor = self.MaxMinorIter
self.setChannel(ch)
_, npix, _ = self.Dirty.shape
xc = (npix) // 2
npol, _, _ = self.Dirty.shape
m0, m1 = self.Dirty[0].min(), self.Dirty[0].max()
# pylab.clf()
# pylab.subplot(1,2,1)
# pylab.imshow(self.Dirty[0],interpolation="nearest",vmin=m0,vmax=m1)
# pylab.draw()
# pylab.show(False)
# pylab.pause(0.1)
print(" Running minor cycle [MaxMinorIter = %i, CycleFactor=%3.1f]" %
(Nminor, self.CycleFactor),
file=log)
NPixStats = 1000
RandomInd = np.int64(np.random.rand(NPixStats) * npix**2)
RMS = np.std(np.real(self.Dirty.ravel()[RandomInd]))
self.RMS = RMS
Threshold_RMS = 5. / (1. - self.SideLobeLevel)
MaxDirty = np.max(np.abs(self.Dirty))
FluxLimit = Threshold_RMS * RMS
#FluxLimit_SideLobe=MaxDirty*(1.-self.SideLobeLevel)
Threshold_SideLobe = self.CycleFactor * MaxDirty * (self.SideLobeLevel)
mm0, mm1 = self.Dirty.min(), self.Dirty.max()
print(
" Dirty image peak flux = %7.3f Jy [(min, max) = (%7.3f, %7.3f) Jy]"
% (MaxDirty, mm0, mm1),
file=log)
print(" RMS threshold flux = %7.3f Jy [rms = %7.3f Jy]" %
(FluxLimit, RMS),
file=log)
print(
" Sidelobe threshold flux = %7.3f Jy [sidelobe = %7.3f of peak]"
% (Threshold_SideLobe, self.SideLobeLevel),
file=log)
MaxModelInit = np.max(np.abs(self.ModelImage))
# Fact=4
# self.BookKeepShape=(npix/Fact,npix/Fact)
# BookKeep=np.zeros(self.BookKeepShape,np.float32)
# NPixBook,_=self.BookKeepShape
# FactorBook=float(NPixBook)/npix
import ClassTimeIt
T = ClassTimeIt.ClassTimeIt()
T.disable()
x, y, ThisFlux = NpParallel.A_whereMax(self.Dirty,
NCPU=self.NCPU,
DoAbs=1)
#print x,y
if ThisFlux < FluxLimit:
print(ModColor.Str(
" Initial maximum peak %f Jy lower that rms-based limit of %f Jy (%i-sigma)"
% (ThisFlux, Threshold_RMS, Threshold_RMS)),
file=log)
return "DoneMinFlux"
for i in range(Nminor):
#x,y,ThisFlux=NpParallel.A_whereMax(self.Dirty,NCPU=self.NCPU,DoAbs=1)
x, y, ThisFlux = NpParallel.A_whereMax(self.Dirty,
NCPU=self.NCPU,
DoAbs=1,
Mask=self.MaskArray)
#x,y=1224, 1994
# print x,y,ThisFlux
# x,y=np.where(self.Dirty[0]==np.max(np.abs(self.Dirty[0])))
# ThisFlux=self.Dirty[0,x,y]
# print x,y,ThisFlux
# stop
T.timeit("max0")
if ThisFlux < FluxLimit:
print(
" [iter=%i] Maximum peak lower that rms-based limit of %f Jy (%i-sigma)"
% (i, FluxLimit, Threshold_RMS),
file=log)
return "MinFlux"
if ThisFlux < Threshold_SideLobe:
print(
" [iter=%i] Peak residual flux %f Jy higher than sidelobe-based limit of %f Jy"
% (i, ThisFlux, Threshold_SideLobe),
file=log)
return "MinFlux"
Fpol = (self.Dirty[:, x, y].reshape(npol, 1, 1)).copy()
#print "Fpol",Fpol
dx = x - xc
dy = y - xc
T.timeit("stuff")
iScale = self.FindBestScale((x, y), np.float32(Fpol))
#print iScale
if iScale == "BadFit": continue
# box=30
# x0,x1=x-box,x+box
# y0,y1=y-box,y+box
# pylab.clf()
# pylab.subplot(1,3,1)
# pylab.imshow(self.Dirty[0][x0:x1,y0:y1],interpolation="nearest")#,vmin=m0,vmax=m1)
# #pylab.subplot(1,3,2)
# #pylab.imshow(self.MaskArray[0],interpolation="nearest",vmin=0,vmax=1,cmap="gray")
# pylab.subplot(1,3,2)
# pylab.imshow(self.ModelImage[0][x0:x1,y0:y1],interpolation="nearest",cmap="gray")
# #pylab.imshow(PSF[0],interpolation="nearest",vmin=0,vmax=1)
# #pylab.colorbar()
self.SubStep((x, y), Fpol, iScale)
T.timeit("add0")
# pylab.subplot(1,3,3)
# pylab.imshow(self.Dirty[0][x0:x1,y0:y1],interpolation="nearest")#,vmin=m0,vmax=m1)
# #pylab.imshow(PSF[0],interpolation="nearest",vmin=0,vmax=1)
# #pylab.colorbar()
# pylab.draw()
# pylab.show(False)
# pylab.pause(0.1)
ThisComp = self.ListScales[iScale]
if ThisComp["ModelType"] == "Delta":
for pol in range(npol):
self.ModelImage[pol, x, y] += Fpol[pol, 0, 0] * self.Gain
elif ThisComp["ModelType"] == "Gaussian":
Gauss = ThisComp["Model"]
Sup, _ = Gauss.shape
x0, x1 = x - Sup // 2, x + Sup // 2 + 1
y0, y1 = y - Sup // 2, y + Sup // 2 + 1
_, N0, _ = self.ModelImage.shape
Aedge, Bedge = self.GiveEdges(x, y, N0, Sup // 2, Sup // 2,
Sup)
x0d, x1d, y0d, y1d = Aedge
x0p, x1p, y0p, y1p = Bedge
for pol in range(npol):
self.ModelImage[pol, x0d:x1d, y0d:y1d] += Gauss[
x0p:x1p, y0p:y1p] * Fpol[pol, 0, 0] * self.Gain
else:
stop
T.timeit("add1")
print(ModColor.Str(
" [iter=%i] Reached maximum number of iterations" % (Nminor)),
file=log)
return "MaxIter"
def Deconvolve(self, ch=0):
if self._niter >= self.MaxMinorIter:
return "MaxIter", False, False
self.setChannel(ch)
_, npix, _ = self.Dirty.shape
xc = (npix) / 2
npol, _, _ = self.Dirty.shape
m0, m1 = self.Dirty[0].min(), self.Dirty[0].max()
DoAbs = int(self.GD["Deconv"]["AllowNegative"])
print >> log, " Running minor cycle [MinorIter = %i/%i, SearchMaxAbs = %i]" % (
self._niter, self.MaxMinorIter, DoAbs)
NPixStats = 1000
#RandomInd=np.int64(np.random.rand(NPixStats)*npix**2)
RandomInd = np.int64(np.linspace(0, self.Dirty.size - 1, NPixStats))
RMS = np.std(np.real(self.Dirty.ravel()[RandomInd]))
#print "::::::::::::::::::::::"
self.RMS = RMS
self.GainMachine.SetRMS(RMS)
Fluxlimit_RMS = self.RMSFactor * RMS
x, y, MaxDirty = NpParallel.A_whereMax(
self.Dirty,
NCPU=self.NCPU,
DoAbs=DoAbs,
Mask=self.MaskMachine.CurrentNegMask)
#MaxDirty=np.max(np.abs(self.Dirty))
#Fluxlimit_SideLobe=MaxDirty*(1.-self.SideLobeLevel)
#Fluxlimit_Sidelobe=self.CycleFactor*MaxDirty*(self.SideLobeLevel)
Fluxlimit_Peak = MaxDirty * self.PeakFactor
Fluxlimit_Sidelobe = self.GiveThreshold(MaxDirty)
mm0, mm1 = self.Dirty.min(), self.Dirty.max()
# work out uper threshold
StopFlux = max(Fluxlimit_Peak, Fluxlimit_RMS, Fluxlimit_Sidelobe,
Fluxlimit_Peak, self.FluxThreshold)
print >> log, " Dirty image peak flux = %10.6f Jy [(min, max) = (%.3g, %.3g) Jy]" % (
MaxDirty, mm0, mm1)
print >> log, " RMS-based threshold = %10.6f Jy [rms = %.3g Jy; RMS factor %.1f]" % (
Fluxlimit_RMS, RMS, self.RMSFactor)
print >> log, " Sidelobe-based threshold = %10.6f Jy [sidelobe = %.3f of peak; cycle factor %.1f]" % (
Fluxlimit_Sidelobe, self.SideLobeLevel, self.CycleFactor)
print >> log, " Peak-based threshold = %10.6f Jy [%.3f of peak]" % (
Fluxlimit_Peak, self.PeakFactor)
print >> log, " Absolute threshold = %10.6f Jy" % (
self.FluxThreshold)
print >> log, " Stopping flux = %10.6f Jy [%.3f of peak ]" % (
StopFlux, StopFlux / MaxDirty)
MaxModelInit = np.max(np.abs(self.ModelImage))
# Fact=4
# self.BookKeepShape=(npix/Fact,npix/Fact)
# BookKeep=np.zeros(self.BookKeepShape,np.float32)
# NPixBook,_=self.BookKeepShape
# FactorBook=float(NPixBook)/npix
T = ClassTimeIt.ClassTimeIt()
T.disable()
x, y, ThisFlux = NpParallel.A_whereMax(
self.Dirty,
NCPU=self.NCPU,
DoAbs=DoAbs,
Mask=self.MaskMachine.CurrentNegMask)
if ThisFlux < StopFlux:
print >> log, ModColor.Str(
" Initial maximum peak %g Jy below threshold, we're done here"
% (ThisFlux),
col="green")
return "FluxThreshold", False, False
self.SearchIslands(StopFlux)
#return None,None,None
self.InitIslands()
if self.DeconvMode == "GAClean":
print >> log, "Evolving %i generations of %i sourcekin" % (
self.GD["GAClean"]["NMaxGen"],
self.GD["GAClean"]["NSourceKin"])
ListBigIslands = []
ListSmallIslands = []
ListInitBigIslands = []
ListInitSmallIslands = []
for iIsland, Island in enumerate(self.ListIslands):
if len(Island) > self.GD["SSDClean"]["ConvFFTSwitch"]:
ListBigIslands.append(Island)
ListInitBigIslands.append(
self.DicoInitIndiv.get(iIsland, None))
else:
ListSmallIslands.append(Island)
ListInitSmallIslands.append(
self.DicoInitIndiv.get(iIsland, None))
if len(ListSmallIslands) > 0:
print >> log, "Deconvolve small islands (<=%i pixels) (parallelised over island)" % (
self.GD["SSDClean"]["ConvFFTSwitch"])
self.DeconvListIsland(ListSmallIslands,
ParallelMode="OverIslands",
ListInitIslands=ListInitSmallIslands)
else:
print >> log, "No small islands"
if len(ListBigIslands) > 0:
print >> log, "Deconvolve large islands (>%i pixels) (parallelised per island)" % (
self.GD["SSDClean"]["ConvFFTSwitch"])
self.DeconvListIsland(ListBigIslands,
ParallelMode="PerIsland",
ListInitIslands=ListInitBigIslands)
else:
print >> log, "No large islands"
elif self.DeconvMode == "MetroClean":
if self.GD["MetroClean"]["MetroNChains"] != "NCPU":
self.NChains = self.GD["MetroClean"]["MetroNChains"]
else:
self.NChains = self.NCPU
print >> log, "Evolving %i chains of %i iterations" % (
self.NChains, self.GD["MetroClean"]["MetroNIter"])
ListBigIslands = []
for ThisPixList in self.ListIslands:
x, y = np.array(ThisPixList, dtype=np.float32).T
dx, dy = x.max() - x.min(), y.max() - y.min()
dd = np.max([dx, dy]) + 1
if dd > self.GD["SSDClean"]["RestoreMetroSwitch"]:
ListBigIslands.append(ThisPixList)
# ListBigIslands=ListBigIslands[1::]
# ListBigIslands=[Island for Island in self.ListIslands if len(Island)>=self.GD["SSDClean"]["RestoreMetroSwitch"]]
print >> log, "Deconvolve %i large islands (>=%i pixels) (parallelised per island)" % (
len(ListBigIslands), self.GD["SSDClean"]["RestoreMetroSwitch"])
self.SelectedIslandsMask = np.zeros_like(
self.DicoDirty["MeanImage"])
for ThisIsland in ListBigIslands:
x, y = np.array(ThisIsland).T
self.SelectedIslandsMask[0, 0, x, y] = 1
self.DeconvListIsland(ListBigIslands, ParallelMode="PerIsland")
return "MaxIter", True, True # stop deconvolution but do update model
def Deconvolve(self, **kwargs):
"""
Runs minor cycle over image channel 'ch'.
initMinor is number of minor iteration (keeps continuous count through major iterations)
Nminor is max number of minor iterations
Returns tuple of: return_code,continue,updated
where return_code is a status string;
continue is True if another cycle should be executed (one or more polarizations still need cleaning);
update is True if one or more polarization models have been updated
"""
exit_msg = ""
continue_deconvolution = False
update_model = False
# # Get the PeakMap (first index will always be 0 because we only support I cleaning)
PeakMap = self._MeanDirty[0, 0, :, :]
#These options should probably be moved into MinorCycleConfig in parset
DoAbs = int(self.GD["Deconv"]["AllowNegative"])
print(" Running minor cycle [MinorIter = %i/%i, SearchMaxAbs = %i]" %
(self._niter, self.MaxMinorIter, DoAbs),
file=log)
## Determine which stopping criterion to use for flux limit
#Get RMS stopping criterion
NPixStats = self.GD["Deconv"]["NumRMSSamples"]
if NPixStats:
RandomInd = np.int64(np.random.rand(NPixStats) * self.Npix**2)
RMS = np.std(np.real(PeakMap.ravel()[RandomInd]))
else:
RMS = np.std(PeakMap)
self.RMS = RMS
Fluxlimit_RMS = self.RMSFactor * RMS
# Find position and intensity of first peak
x, y, MaxDirty = NpParallel.A_whereMax(PeakMap,
NCPU=self.NCPU,
DoAbs=DoAbs,
Mask=self.MaskArray)
# Get peak factor stopping criterion
Fluxlimit_Peak = MaxDirty * self.PeakFactor
# Get side lobe stopping criterion
Fluxlimit_Sidelobe = (
(self.CycleFactor - 1.) / 4. * (1. - self.SideLobeLevel) +
self.SideLobeLevel) * MaxDirty if self.CycleFactor else 0
mm0, mm1 = PeakMap.min(), PeakMap.max()
# Choose whichever threshold is highest
StopFlux = max(Fluxlimit_Peak, Fluxlimit_RMS, Fluxlimit_Sidelobe,
self.FluxThreshold)
print(
" Dirty image peak flux = %10.6f Jy [(min, max) = (%.3g, %.3g) Jy]"
% (MaxDirty, mm0, mm1),
file=log)
print(
" RMS-based threshold = %10.6f Jy [rms = %.3g Jy; RMS factor %.1f]"
% (Fluxlimit_RMS, RMS, self.RMSFactor),
file=log)
print(
" Sidelobe-based threshold = %10.6f Jy [sidelobe = %.3f of peak; cycle factor %.1f]"
% (Fluxlimit_Sidelobe, self.SideLobeLevel, self.CycleFactor),
file=log)
print(" Peak-based threshold = %10.6f Jy [%.3f of peak]" %
(Fluxlimit_Peak, self.PeakFactor),
file=log)
print(" Absolute threshold = %10.6f Jy" %
(self.FluxThreshold),
file=log)
print(" Stopping flux = %10.6f Jy [%.3f of peak ]" %
(StopFlux, StopFlux / MaxDirty),
file=log)
T = ClassTimeIt.ClassTimeIt()
T.disable()
ThisFlux = MaxDirty
if ThisFlux < StopFlux:
print(ModColor.Str(
" Initial maximum peak %g Jy below threshold, we're done CLEANing"
% (ThisFlux),
col="green"),
file=log)
exit_msg = exit_msg + " " + "FluxThreshold"
continue_deconvolution = False or continue_deconvolution
update_model = False or update_model
# No need to do anything further if we are already at the stopping flux
return exit_msg, continue_deconvolution, update_model
#Do minor cycle deconvolution loop
try:
for i in range(self._niter + 1, self.MaxMinorIter + 1):
self._niter = i
#grab a new peakmap
PeakMap = self._MeanDirty[0, 0, :, :]
x, y, ThisFlux = NpParallel.A_whereMax(PeakMap,
NCPU=self.NCPU,
DoAbs=DoAbs,
Mask=self.MaskArray)
T.timeit("max0")
if ThisFlux <= StopFlux:
print(ModColor.Str(
" CLEANing [iter=%i] peak of %.3g Jy lower than stopping flux"
% (i, ThisFlux),
col="green"),
file=log)
cont = ThisFlux > self.FluxThreshold
if not cont:
print(ModColor.Str(
" CLEANing [iter=%i] absolute flux threshold of %.3g Jy has been reached"
% (i, self.FluxThreshold),
col="green",
Bold=True),
file=log)
exit_msg = exit_msg + " " + "MinFluxRms"
continue_deconvolution = cont or continue_deconvolution
update_model = True or update_model
break # stop cleaning if threshold reached
# This is used to track Cleaning progress
rounded_iter_step = 1 if i < 10 else (10 if i < 200 else (
100 if i < 2000 else 1000))
# min(int(10**math.floor(math.log10(i))), 10000)
if i >= 10 and i % rounded_iter_step == 0:
# if self.GD["Debug"]["PrintMinorCycleRMS"]:
#rms = np.std(np.real(self._CubeDirty.ravel()[self.IndStats]))
print(" [iter=%i] peak residual %.3g" % (i, ThisFlux),
file=log)
# Find PSF corresponding to location (x,y)
self.PSFServer.setLocation(
x, y) # Selects the facet closest to (x,y)
# Get the JonesNorm
JonesNorm = self.DicoDirty["JonesNorm"][:, 0, x, y]
# Get the solution (division by JonesNorm handled in fit)
Iapp = self._Dirty[:, 0, x, y]
# Fit a polynomial to get coeffs
Coeffs = self.ModelMachine.FreqMachine.Fit(
Iapp, JonesNorm, self.WeightsChansImages)
# Overwrite with polynoimial fit
Iapp = self.ModelMachine.FreqMachine.Eval(Coeffs)
T.timeit("stuff")
PSF, meanPSF = self.PSFServer.GivePSF() #Gives associated PSF
T.timeit("FindScale")
#Update model
self.ModelMachine.AppendComponentToDictStacked((x, y), Coeffs)
# Subtract LocalSM*CurrentGain from dirty image
self.SubStep(
x, y, PSF * Iapp[:, None, None, None] *
self.GD["Deconv"]["Gain"])
T.timeit("SubStep")
T.timeit("End")
except KeyboardInterrupt:
print(ModColor.Str(
" CLEANing [iter=%i] minor cycle interrupted with Ctrl+C, peak flux %.3g"
% (self._niter, ThisFlux)),
file=log)
exit_msg = exit_msg + " " + "MaxIter"
continue_deconvolution = False or continue_deconvolution
update_model = True or update_model
return exit_msg, continue_deconvolution, update_model
if self._niter >= self.MaxMinorIter: #Reached maximum number of iterations:
print(ModColor.Str(
" CLEANing [iter=%i] Reached maximum number of iterations, peak flux %.3g"
% (self._niter, ThisFlux)),
file=log)
exit_msg = exit_msg + " " + "MaxIter"
continue_deconvolution = False or continue_deconvolution
update_model = True or update_model
return exit_msg, continue_deconvolution, update_model
def Deconvolve(self):
"""
Runs minor cycle over image channel 'ch'.
initMinor is number of minor iteration (keeps continuous count through major iterations)
Nminor is max number of minor iterations
Returns tuple of: return_code,continue,updated
where return_code is a status string;
continue is True if another cycle should be executed (one or more polarizations still need cleaning);
update is True if one or more polarization models have been updated
"""
exit_msg = ""
continue_deconvolution = False
update_model = False
# Get the PeakMap (first index will always be 0 because we only support I cleaning)
PeakMap = self._MeanDirty[0, 0, :, :]
# These options should probably be moved into MinorCycleConfig in parset
DoAbs = int(self.GD["Deconv"]["AllowNegative"])
print >> log, " Running minor cycle [MinorIter = %i/%i, SearchMaxAbs = %i]" % (
self._niter, self.MaxMinorIter, DoAbs)
# Determine which stopping criterion to use for flux limit
StopFlux, MaxDirty = self.check_stopping_criteria(
PeakMap, self.Npix, DoAbs)
T = ClassTimeIt.ClassTimeIt()
T.disable()
ThisFlux = MaxDirty.copy()
if ThisFlux < self.FluxThreshold:
print >> log, ModColor.Str(
" Initial maximum peak %g Jy below threshold, we're done CLEANing"
% (ThisFlux),
col="green")
exit_msg = exit_msg + " " + "FluxThreshold"
continue_deconvolution = False or continue_deconvolution
update_model = False or update_model
# No need to do anything further if we are already at the stopping flux
return exit_msg, continue_deconvolution, update_model
# Do minor cycle deconvolution loop
TrackFlux = MaxDirty.copy()
diverged = False
stalled = False
stall_count = -1 # start here since the first check will always increase the stall count
diverged_count = 0
try:
while self._niter <= self.MaxMinorIter:
# Check if stalling or diverging
if np.abs(ThisFlux) > self.GD["WSCMS"][
"MinorDivergenceFactor"] * np.abs(TrackFlux):
diverged_count += 1
if diverged_count > 5:
diverged = True
# elif np.abs((ThisFlux - TrackFlux)/TrackFlux) < self.GD['WSCMS']['MinorStallThreshold']:
# stall_count += 1
# if stall_count > 50000000000000000000:
# stalled = True
TrackFlux = ThisFlux.copy()
# LB - deprecated?
# self.GainMachine.SetFluxMax(ThisFlux)
T.timeit("max0")
if ThisFlux <= StopFlux or diverged or stalled:
if diverged:
print >> log, ModColor.Str(
" At [iter=%i] minor cycle is diverging so it has been force stopped at a flux of %.3g Jy"
% (self._niter, ThisFlux),
col="green")
elif stalled:
print >> log, ModColor.Str(
" At [iter=%i] minor cycle has stalled so it has been force stopped at a flux of %.3g Jy"
% (self._niter, ThisFlux),
col="green")
else:
print >> log, ModColor.Str(
" CLEANing [iter=%i] peak of %.3g Jy lower than stopping flux"
% (self._niter, ThisFlux),
col="green")
cont = ThisFlux > self.FluxThreshold
if not cont:
print >> log, ModColor.Str(
" CLEANing [iter=%i] absolute flux threshold of %.3g Jy has been reached"
% (self._niter, StopFlux),
col="green",
Bold=True)
exit_msg = exit_msg + " " + "MinFluxRms"
continue_deconvolution = cont or continue_deconvolution
update_model = True or update_model
break # stop cleaning if threshold reached
# self.track_progress(self._niter, ThisFlux)
# Find the relevant scale and do sub-minor loop. Note that the dirty cube is updated during the
# sub-minor loop by subtracting the once convolved PSF's as components are added to the model.
# The model is updated by adding components to the ModelMachine dictionary.
niter, iScale = self.ModelMachine.do_minor_loop(
self._Dirty, self._MeanDirty, self._JonesNorm,
self.WeightsChansImages, ThisFlux, StopFlux, self.RMS)
# compute the new mean image from the weighted sum of over frequency
self._MeanDirty = np.sum(self._Dirty * self.WeightsChansImages,
axis=0,
keepdims=True)
# find peak
x, y, ThisFlux = NpParallel.A_whereMax(self._MeanDirty,
NCPU=self.NCPU,
DoAbs=DoAbs,
Mask=self.MaskArray)
# update counter
self._niter += niter
if iScale != self.LastScale:
print >> log, " [iter=%i] peak residual %.8g, scale = %i" % (
self._niter, ThisFlux, iScale)
self.LastScale = iScale
T.timeit("End")
except KeyboardInterrupt:
print >> log, ModColor.Str(
" CLEANing [iter=%i] minor cycle interrupted with Ctrl+C, peak flux %.3g"
% (self._niter, ThisFlux))
exit_msg = exit_msg + " " + "MaxIter"
continue_deconvolution = False or continue_deconvolution
update_model = True or update_model
return exit_msg, continue_deconvolution, update_model
if self._niter >= self.MaxMinorIter: #Reached maximum number of iterations:
print >> log, ModColor.Str(
" CLEANing [iter=%i] Reached maximum number of iterations, peak flux %.3g"
% (self._niter, ThisFlux))
exit_msg = exit_msg + " " + "MaxIter"
continue_deconvolution = False or continue_deconvolution
update_model = True or update_model
return exit_msg, continue_deconvolution, update_model
def Deconvolve(self):
"""
Runs minor cycle over image channel 'ch'.
initMinor is number of minor iteration (keeps continuous count through major iterations)
Nminor is max number of minor iterations
Returns tuple of: return_code,continue,updated
where return_code is a status string;
continue is True if another cycle should be executed (one or more polarizations still need cleaning);
update is True if one or more polarization models have been updated
"""
exit_msg = ""
continue_deconvolution = False
update_model = False
# These options should probably be moved into MinorCycleConfig in parset
print(" Running minor cycle [MinorIter = %i/%i, SearchMaxAbs = %i]" %
(self._niter, self.MaxMinorIter,
int(self.GD["Deconv"]["AllowNegative"])),
file=log)
# Determine which stopping criterion to use for flux limit
StopFlux, MaxDirty, RMS = self.check_stopping_criteria()
TrackRMS = RMS.copy()
ThisFlux = MaxDirty.copy()
if ThisFlux < self.FluxThreshold:
print(ModColor.Str(
" Initial maximum peak %g Jy below threshold, we're done CLEANing"
% (ThisFlux),
col="green"),
file=log)
exit_msg = exit_msg + " " + "FluxThreshold"
continue_deconvolution = False or continue_deconvolution
update_model = False or update_model
# No need to do anything further if we are already at the stopping flux
return exit_msg, continue_deconvolution, update_model
# Do minor cycle deconvolution loop
TrackFlux = MaxDirty.copy()
diverged = False
diverged_count = 0
stalled = False
scale_stall_count = {}
scales_stalled = np.zeros(self.ModelMachine.ScaleMachine.Nscales,
dtype=np.bool)
# reset retired scales at the start of each major cycle
self.ModelMachine.ScaleMachine.retired_scales = []
for scale in self.ModelMachine.ScaleMachine.forbidden_scales:
self.ModelMachine.ScaleMachine.retired_scales.append(scale)
scales_stalled[scale] = 1
try:
while self._niter <= self.MaxMinorIter:
# Check if diverging
if np.abs(ThisFlux) > self.GD["WSCMS"][
"MinorDivergenceFactor"] * np.abs(TrackFlux):
diverged_count += 1
if diverged_count > 5:
diverged = True
TrackFlux = ThisFlux.copy()
if ThisFlux <= StopFlux or diverged or stalled:
if diverged:
print(ModColor.Str(
" At [iter=%i] minor cycle is diverging so it has been force stopped at a flux of %.3g Jy"
% (self._niter, ThisFlux),
col="green"),
file=log)
elif stalled:
print(ModColor.Str(
" At [iter=%i] minor cycle has stalled so it has been force stopped at a flux of %.3g Jy"
% (self._niter, ThisFlux),
col="green"),
file=log)
else:
print(ModColor.Str(
" CLEANing [iter=%i] peak of %.3g Jy lower than stopping flux"
% (self._niter, ThisFlux),
col="green"),
file=log)
cont = ThisFlux > self.FluxThreshold
if not cont:
print(ModColor.Str(
" CLEANing [iter=%i] absolute flux threshold of %.3g Jy has been reached"
% (self._niter, StopFlux),
col="green",
Bold=True),
file=log)
exit_msg = exit_msg + " " + "MinFluxRms"
continue_deconvolution = cont or continue_deconvolution
update_model = True or update_model
break # stop cleaning if threshold reached
# Find the relevant scale and do sub-minor loop. Note that the dirty cube is updated during the
# sub-minor loop by subtracting the once convolved PSF's as components are added to the model.
# The model is updated by adding components to the ModelMachine dictionary.
niter, iScale = self.ModelMachine.do_minor_loop(
self._Dirty, self._MeanDirty, self._JonesNorm,
self.WeightsChansImages, ThisFlux, StopFlux, RMS)
# compute the new mean image from the weighted sum of over frequency
self._MeanDirty = np.sum(self._Dirty * self.WeightsChansImages,
axis=0,
keepdims=True)
ThisRMS = np.std(self._MeanDirty)
# check for and retire scales that cause stalls
if np.abs((TrackRMS - ThisRMS) /
TrackRMS) < self.GD['WSCMS']['MinorStallThreshold']:
scale_stall_count.setdefault(iScale, 0)
scale_stall_count[iScale] += 1
# retire scale if it causes a stall more than x number of times
if scale_stall_count[iScale] > 10:
self.ModelMachine.ScaleMachine.retired_scales.append(
iScale)
scales_stalled[iScale] = 1
print("Retired scale %i because it was stalling." %
iScale,
file=log)
# if all scales have stalled then we trigger a new major cycle
if np.all(scales_stalled):
stalled = True
TrackRMS = ThisRMS.copy()
# find peak
x, y, ThisFlux = NpParallel.A_whereMax(
self._MeanDirty,
NCPU=self.NCPU,
DoAbs=self.GD["Deconv"]["AllowNegative"],
Mask=self.MaskArray)
# update counter
self._niter += niter
if iScale != self.LastScale:
print(
" [iter=%i] peak residual %.8g, rms = %.8g, scale = %i"
% (self._niter, ThisFlux, TrackRMS, iScale),
file=log)
self.LastScale = iScale
except KeyboardInterrupt:
print(ModColor.Str(
" CLEANing [iter=%i] minor cycle interrupted with Ctrl+C, peak flux %.3g"
% (self._niter, ThisFlux)),
file=log)
exit_msg = exit_msg + " " + "MaxIter"
continue_deconvolution = False or continue_deconvolution
update_model = True or update_model
return exit_msg, continue_deconvolution, update_model
if self._niter >= self.MaxMinorIter: #Reached maximum number of iterations:
print(ModColor.Str(
" CLEANing [iter=%i] Reached maximum number of iterations, peak flux %.3g"
% (self._niter, ThisFlux)),
file=log)
exit_msg = exit_msg + " " + "MaxIter"
continue_deconvolution = False or continue_deconvolution
update_model = True or update_model
return exit_msg, continue_deconvolution, update_model
def Deconvolve(self, ch=0, UpdateRMS=True):
"""
Runs minor cycle over image channel 'ch'.
initMinor is number of minor iteration (keeps continuous count through major iterations)
Nminor is max number of minor iteration
Returns tuple of: return_code,continue,updated
where return_code is a status string;
continue is True if another cycle should be executed;
update is True if model has been updated (note update=False implies continue=False)
"""
if self._niter >= self.MaxMinorIter:
return "MaxIter", False, False
_, npol, npix, _ = self._MeanDirty.shape
xc = (npix)/2
# m0,m1=self._CubeDirty.min(),self._CubeDirty.max()
# pylab.clf()
# pylab.subplot(1,2,1)
# pylab.imshow(self.Dirty[0],interpolation="nearest",vmin=m0,vmax=m1)
# pylab.draw()
# pylab.show(False)
# pylab.pause(0.1)
DoAbs = int(self.GD["Deconv"]["AllowNegative"])
print>>log, " Running minor cycle [MinorIter = %i/%i, SearchMaxAbs = %i]" % (
self._niter, self.MaxMinorIter, DoAbs)
if UpdateRMS: self.updateRMS()
RMS=self.RMS
self.GainMachine.SetRMS(RMS)
Fluxlimit_RMS = self.RMSFactor*RMS
#print "startmax",self._MeanDirty.shape,self._MaskArray.shape
if self.CurrentNegMask is not None:
print>>log," using externally defined Mask (self.CurrentNegMask)"
CurrentNegMask=self.CurrentNegMask
elif self.MaskMachine:
print>>log," using MaskMachine Mask"
CurrentNegMask=self.MaskMachine.CurrentNegMask
elif self._MaskArray is not None:
print>>log," using externally defined Mask (self._MaskArray)"
CurrentNegMask=self._MaskArray
else:
print>>log," not using a mask"
CurrentNegMask=None
x,y,MaxDirty = NpParallel.A_whereMax(self._PeakSearchImage,NCPU=self.NCPU,DoAbs=DoAbs,Mask=CurrentNegMask)
# ThisFlux is evaluated against stopping criteria. In weighted mode, use the true flux. Else use sigma value.
ThisFlux = self._MeanDirty[0,0,x,y] if self._peakMode is "weighted" else MaxDirty
if DoAbs:
ThisFlux = abs(ThisFlux)
# in weighted or noisemap mode, look up the true max as well
trueMaxDirty = MaxDirty if self._peakMode is "normal" else ThisFlux
# return condition indicating cleaning is to be continued
cont = True
CondPeak=(self._previous_initial_peak is not None)
CondDiverge=False
if self._previous_initial_peak is not None:
CondDiverge=(abs(ThisFlux) > self.GD["HMP"]["MajorStallThreshold"]*self._previous_initial_peak)
CondPeakType=(self._peakMode!="sigma")
if CondPeak and CondDiverge and CondPeakType:
print>>log,ModColor.Str("STALL! dirty image peak %10.6g Jy, was %10.6g at previous major cycle."
% (ThisFlux, self._previous_initial_peak), col="red")
print>>log,ModColor.Str("This will be the last major cycle")
cont = False
self._previous_initial_peak = abs(ThisFlux)
#x,y,MaxDirty=NpParallel.A_whereMax(self._MeanDirty.copy(),NCPU=1,DoAbs=DoAbs,Mask=self._MaskArray.copy())
#A=self._MeanDirty.copy()
#A.flat[:]=np.arange(A.size)[:]
#x,y,MaxDirty=NpParallel.A_whereMax(A,NCPU=1,DoAbs=DoAbs)
#print "max",x,y
#stop
# print>>log,"npp: %d %d %g"%(x,y,MaxDirty)
# xy = ma.argmax(ma.masked_array(abs(self._MeanDirty), self._MaskArray))
# x1, y1 = xy/npix, xy%npix
# MaxDirty1 = abs(self._MeanDirty[0,0,x1,y1])
# print>>log,"argmax: %d %d %g"%(x1,y1,MaxDirty1)
Fluxlimit_Peak = ThisFlux*self.PeakFactor
# if previous peak is not set (i.e. first major cycle), use current dirty image peak instead
Fluxlimit_PrevPeak = (self._prevPeak if self._prevPeak is not None else ThisFlux)*self.PrevPeakFactor
Fluxlimit_Sidelobe = ((self.CycleFactor-1.)/4.*(
1.-self.SideLobeLevel)+self.SideLobeLevel)*ThisFlux if self.CycleFactor else 0
mm0, mm1 = self._PeakSearchImage.min(), self._PeakSearchImage.max()
# work out upper peak threshold
StopFlux = max(
Fluxlimit_Peak,
Fluxlimit_RMS,
Fluxlimit_Sidelobe,
Fluxlimit_Peak,
Fluxlimit_PrevPeak,
self.FluxThreshold)
print>>log, " Dirty image peak = %10.6g Jy [(min, max) = (%.3g, %.3g) Jy]" % (
trueMaxDirty, mm0, mm1)
if self._peakMode is "sigma":
print>>log, " in sigma units = %10.6g" % MaxDirty
elif self._peakMode is "weighted":
print>>log, " weighted peak flux is = %10.6g Jy" % MaxDirty
print>>log, " RMS-based threshold = %10.6g Jy [rms = %.3g Jy; RMS factor %.1f]" % (
Fluxlimit_RMS, RMS, self.RMSFactor)
print>>log, " Sidelobe-based threshold = %10.6g Jy [sidelobe = %.3f of peak; cycle factor %.1f]" % (
Fluxlimit_Sidelobe, self.SideLobeLevel, self.CycleFactor)
print>>log, " Peak-based threshold = %10.6g Jy [%.3f of peak]" % (
Fluxlimit_Peak, self.PeakFactor)
print>>log, " Previous peak-based thr = %10.6g Jy [%.3f of previous minor cycle peak]" % (
Fluxlimit_PrevPeak, self.PrevPeakFactor)
print>>log, " Absolute threshold = %10.6g Jy" % (
self.FluxThreshold)
print>>log, " Stopping flux = %10.6g Jy [%.3f of peak ]" % (
StopFlux, StopFlux/ThisFlux)
rms=RMS
# MaxModelInit=np.max(np.abs(self.ModelImage))
# Fact=4
# self.BookKeepShape=(npix/Fact,npix/Fact)
# BookKeep=np.zeros(self.BookKeepShape,np.float32)
# NPixBook,_=self.BookKeepShape
# FactorBook=float(NPixBook)/npix
T = ClassTimeIt.ClassTimeIt()
T.disable()
# #print x,y
# print>>log, "npp: %d %d %g"%(x,y,ThisFlux)
# xy = ma.argmax(ma.masked_array(abs(self._MeanDirty), self._MaskArray))
# x, y = xy/npix, xy%npix
# ThisFlux = abs(self._MeanDirty[0,0,x,y])
# print>> log, "argmax: %d %d %g"%(x, y, ThisFlux)
if ThisFlux < StopFlux:
print>>log, ModColor.Str(
" Initial maximum peak %10.6g Jy below threshold, we're done here" %
ThisFlux, col="green")
return "FluxThreshold", False, False
# self._MaskArray.fill(1)
# self._MaskArray.fill(0)
#self._MaskArray[np.abs(self._MeanDirty) > Fluxlimit_Sidelobe]=0
# DoneScale=np.zeros((self.MSMachine.NScales,),np.float32)
PreviousMaxFlux = 1e30
# pBAR= ProgressBar('white', width=50, block='=', empty=' ',Title="Cleaning ", HeaderSize=20,TitleSize=30)
# pBAR.disable()
self.GainMachine.SetFluxMax(ThisFlux)
# pBAR.render(0,"g=%3.3f"%self.GainMachine.GiveGain())
PreviousFlux=ThisFlux
divergence_factor = 1 + max(self.GD["HMP"]["AllowResidIncrease"],0)
xlast,ylast,Flast=None,None,None
def GivePercentDone(ThisMaxFlux):
fracDone = 1.-(ThisMaxFlux-StopFlux)/(MaxDirty-StopFlux)
return max(int(round(100*fracDone)), 100)
x0 = y0 = None
try:
for i in xrange(self._niter+1, self.MaxMinorIter+1):
self._niter = i
# x,y,ThisFlux=NpParallel.A_whereMax(self.Dirty,NCPU=self.NCPU,DoAbs=1)
x, y, peak = NpParallel.A_whereMax(
self._PeakSearchImage, NCPU=self.NCPU, DoAbs=DoAbs, Mask=CurrentNegMask)
if self.GD["HMP"]["FractionRandomPeak"] is not None:
op=lambda x: x
if DoAbs: op=lambda x: np.abs(x)
_,_,indx,indy=np.where((op(self._PeakSearchImage)>=peak*self.GD["HMP"]["FractionRandomPeak"]) & np.logical_not(CurrentNegMask))
ii=np.int64(np.random.rand(1)[0]*indx.size)
x,y=indx[ii],indy[ii]
peak=op(self._PeakSearchImage[0,0,x,y])
ThisFlux = float(self._MeanDirty[0,0,x,y] if self._peakMode is "weighted" else peak)
if DoAbs:
ThisFlux = abs(ThisFlux)
if xlast is not None:
if x==xlast and y==ylast and np.abs((Flast-peak)/Flast)<1e-6:
print>>log, ModColor.Str(" [iter=%i] peak of %.3g Jy stuck"%(i, ThisFlux), col="red")
return "Stuck", False, True
xlast=x
ylast=y
Flast=peak
#x,y=self.PSFServer.SolveOffsetLM(self._MeanDirty[0,0],x,y); ThisFlux=self._MeanDirty[0,0,x,y]
self.GainMachine.SetFluxMax(ThisFlux)
# #x,y=1224, 1994
# print x,y,ThisFlux
# x,y=np.where(np.abs(self.Dirty[0])==np.max(np.abs(self.Dirty[0])))
# ThisFlux=self.Dirty[0,x,y]
# print x,y,ThisFlux
# stop
T.timeit("max0")
if np.abs(ThisFlux) > divergence_factor*np.abs(PreviousFlux):
print>>log, ModColor.Str(
" [iter=%i] peak of %.3g Jy diverging w.r.t. floor of %.3g Jy " %
(i, ThisFlux, PreviousFlux), col="red")
return "Diverging", False, True
fluxgain = np.abs(ThisFlux-self._prevPeak)/abs(ThisFlux) if self._prevPeak is not None else 1e+99
if x == x0 and y == y0 and fluxgain < 1e-6:
print>>log, ModColor.Str(
" [iter=%i] stalled at peak of %.3g Jy, x=%d y=%d" % (i, ThisFlux, x, y), col="red")
return "Stalled", False, True
if np.abs(ThisFlux) < np.abs(PreviousFlux):
PreviousFlux = ThisFlux
self._prevPeak = ThisFlux
x0, y0 = x, y
ThisPNR=ThisFlux/rms
if ThisFlux <= StopFlux or ThisPNR <= self._PNRStop:
rms = np.std(np.real(self._PeakSearchImage.ravel()[self.IndStats]))
# pBAR.render(100,"peak %.3g"%(ThisFlux,))
if ThisFlux <= StopFlux:
print>>log, ModColor.Str(
" [iter=%i] peak of %.3g Jy lower than stopping flux, PNR %.3g" %
(i, ThisFlux, ThisFlux/rms), col="green")
elif ThisPNR <= self._PNRStop:
print>>log, ModColor.Str(
" [iter=%i] PNR of %.3g lower than stopping PNR, peak of %.3g Jy" %
(i, ThisPNR, ThisFlux), col="green")
cont = cont and ThisFlux > self.FluxThreshold
if not cont:
print>>log, ModColor.Str(
" [iter=%i] absolute flux threshold of %.3g Jy has been reached, PNR %.3g" %
(i, self.FluxThreshold, ThisFlux/rms), col="green", Bold=True)
# DoneScale*=100./np.sum(DoneScale)
# for iScale in range(DoneScale.size):
# print>>log," [Scale %i] %.1f%%"%(iScale,DoneScale[iScale])
# stop deconvolution if hit absolute treshold; update model
return "MinFluxRms", cont, True
# if (i>0)&((i%1000)==0):
# print>>log, " [iter=%i] Peak residual flux %f Jy" % (i,ThisFlux)
# if (i>0)&((i%100)==0):
# PercentDone=GivePercentDone(ThisFlux)
# pBAR.render(PercentDone,"peak %.3g i%d"%(ThisFlux,self._niter))
rounded_iter_step = 1 if i < 10 else (
10 if i<200 else (
100 if i < 2000
else 1000 ))
# min(int(10**math.floor(math.log10(i))), 10000)
if i >= 10 and i % rounded_iter_step == 0:
# if self.GD["Debug"]["PrintMinorCycleRMS"]:
rms = np.std(np.real(self._PeakSearchImage.ravel()[self.IndStats]))
if self._peakMode is "weighted":
print>>log, " [iter=%i] peak residual %.3g, gain %.3g, rms %g, PNR %.3g (weighted peak %.3g at x=%d y=%d)" % (i, ThisFlux, fluxgain, rms, ThisFlux/rms, peak, x, y)
else:
print>>log, " [iter=%i] peak residual %.3g, gain %.3g, rms %g, PNR %.3g (at x=%d y=%d)" % (i, ThisFlux, fluxgain, rms, ThisFlux/rms, x, y)
# else:
# print >>log, " [iter=%i] peak residual %.3g" % (
# i, ThisFlux)
ClassMultiScaleMachine.CleanSolutionsDump.flush()
nch, npol, _, _ = self._CubeDirty.shape
Fpol = np.float32(
(self._CubeDirty[
:, :, x, y].reshape(
(nch, npol, 1, 1))).copy())
#print "Fpol",Fpol
dx = x-xc
dy = y-xc
T.timeit("stuff")
# iScale=self.MSMachine.FindBestScale((x,y),Fpol)
self.PSFServer.setLocation(x, y)
PSF = self.PSFServer.GivePSF()
MSMachine = self.DicoMSMachine[self.PSFServer.iFacet]
LocalSM = MSMachine.GiveLocalSM((x, y), Fpol)
T.timeit("FindScale")
# print iScale
# if iScale=="BadFit": continue
# box=50
# x0,x1=x-box,x+box
# y0,y1=y-box,y+box
# x0,x1=0,-1
# y0,y1=0,-1
# pylab.clf()
# pylab.subplot(1,2,1)
# pylab.imshow(self.Dirty[0][x0:x1,y0:y1],interpolation="nearest",vmin=mm0,vmax=mm1)
# #pylab.subplot(1,3,2)
# #pylab.imshow(self.MaskArray[0],interpolation="nearest",vmin=0,vmax=1,cmap="gray")
# # pylab.subplot(1,2,2)
# # pylab.imshow(self.ModelImage[0][x0:x1,y0:y1],interpolation="nearest",cmap="gray")
# #pylab.imshow(PSF[0],interpolation="nearest",vmin=0,vmax=1)
# #pylab.colorbar()
# CurrentGain=self.GainMachine.GiveGain()
CurrentGain=np.float32(self.GD["Deconv"]["Gain"])
numexpr.evaluate('LocalSM*CurrentGain', out=LocalSM)
self.SubStep((x,y),LocalSM)
T.timeit("SubStep")
# pylab.subplot(1,2,2)
# pylab.imshow(self.Dirty[0][x0:x1,y0:y1],interpolation="nearest",vmin=mm0,vmax=mm1)#,vmin=m0,vmax=m1)
# #pylab.imshow(PSF[0],interpolation="nearest",vmin=0,vmax=1)
# #pylab.colorbar()
# pylab.draw()
# pylab.show(False)
# pylab.pause(0.1)
# ######################################
# ThisComp=self.ListScales[iScale]
# Scale=ThisComp["Scale"]
# DoneScale[Scale]+=1
# if ThisComp["ModelType"]=="Delta":
# for pol in range(npol):
# self.ModelImage[pol,x,y]+=Fpol[pol,0,0]*self.Gain
# elif ThisComp["ModelType"]=="Gaussian":
# Gauss=ThisComp["Model"]
# Sup,_=Gauss.shape
# x0,x1=x-Sup/2,x+Sup/2+1
# y0,y1=y-Sup/2,y+Sup/2+1
# _,N0,_=self.ModelImage.shape
# Aedge,Bedge=self.GiveEdges((x,y),N0,(Sup/2,Sup/2),Sup)
# x0d,x1d,y0d,y1d=Aedge
# x0p,x1p,y0p,y1p=Bedge
# for pol in range(npol):
# self.ModelImage[pol,x0d:x1d,y0d:y1d]+=Gauss[x0p:x1p,y0p:y1p]*pol[pol,0,0]*self.Gain
# else:
# stop
T.timeit("End")
except KeyboardInterrupt:
rms = np.std(np.real(self._PeakSearchImage.ravel()[self.IndStats]))
print>>log, ModColor.Str(
" [iter=%i] minor cycle interrupted with Ctrl+C, peak flux %.3g, PNR %.3g" %
(self._niter, ThisFlux, ThisFlux/rms))
# DoneScale*=100./np.sum(DoneScale)
# for iScale in range(DoneScale.size):
# print>>log," [Scale %i] %.1f%%"%(iScale,DoneScale[iScale])
return "MaxIter", False, True # stop deconvolution but do update model
rms = np.std(np.real(self._PeakSearchImage.ravel()[self.IndStats]))
print>>log, ModColor.Str(
" [iter=%i] Reached maximum number of iterations, peak flux %.3g, PNR %.3g" %
(self._niter, ThisFlux, ThisFlux/rms))
# DoneScale*=100./np.sum(DoneScale)
# for iScale in range(DoneScale.size):
# print>>log," [Scale %i] %.1f%%"%(iScale,DoneScale[iScale])
return "MaxIter", False, True # stop deconvolution but do update model