def test():
nx=17
DataTrue=np.zeros((1,1,nx,nx),np.float32)
DataTrue[0,0,nx/2,nx/2]=1.
DataPSF=DataTrue.copy()
PSF=ModFFTW.ConvolveGaussian(DataPSF,CellSizeRad=1,GaussPars=[(1.,1.,0.)])
DataTrue.fill(0)
DataTrue[0,0,nx/2,nx/2]=1.
DataTrue[0,0,nx/2+5,nx/2+4]=1.
#DataTrue[0,0,nx/2,nx/2]=1.
DataTrue=ModFFTW.ConvolveGaussian(DataTrue,CellSizeRad=1,GaussPars=[(1,.5,0.)])
#DataTrue=np.random.randn(*DataTrue.shape)
DataTrue[0,0,nx/2-5,nx/2+4]+=2.
DataTrue[0,0,nx/2-2,nx/2+2]+=2.
# DataTrue+=np.random.randn(*DataTrue.shape)*0.05
Dirty=ModFFTW.ConvolveGaussian(DataTrue,CellSizeRad=1,GaussPars=[(1.,1.,0.)])
#DataConv=ModFFTW.ConvolveGaussian(DataTrue,CellSizeRad=1,GaussPars=[(1.,1.,0.)])
#DataConv=DataTrue
#IndDataTrue=ArrayToInd(DataConv)
FreqsInfo=None
_,_,indx,indy=np.where(DataTrue>1e-1)
ListPixParms=[ij for ij in zip(indx,indy)]
_,_,indx,indy=np.where(Dirty>1e-1)
ListPixData=[ij for ij in zip(indx,indy)]
CEv=ClassEvolveGA(Dirty,PSF,FreqsInfo,ListPixData=ListPixData,ListPixParms=ListPixParms)
CEv.ArrayMethodsMachine.DataTrue=DataTrue
return CEv
def restoreFittedBeam(rot):
imgSize = 256
cellSize = np.deg2rad(4./3600.)
params = (10, 5, rot) #maj, min, theta
#create input with code borrowed from Tigger:
xx,yy = np.meshgrid(np.arange(0,imgSize),np.arange(0,imgSize))
inp = gauss2d([1,imgSize/2,imgSize/2,params[1],params[0],params[2]],circle=0,rotate=1,vheight=0)(xx,yy)
inp = inp.reshape(1,1,imgSize,imgSize)
#fit
fittedParams = tuple((fitter.FitCleanBeam(inp[0, 0, :, :]) *
np.array([cellSize, cellSize, 1])).tolist())
#restore fitted clean beam with an FFT convolution:
delta = np.zeros([1, 1, imgSize, imgSize])
delta[0, 0, imgSize // 2, imgSize // 2] = 1
rest, _ = fftconvolve.ConvolveGaussianScipy(delta,
Sig=5,
GaussPar=fittedParams)
rest = fftconvolve.ConvolveGaussian(shareddict={"in":delta,
"out":delta},
field_in="in",
field_out="out",
ch=0,
CellSizeRad=cellSize,
GaussPars_ch=fittedParams,
Normalise=False)
assert np.allclose(inp, rest, rtol=1e-2, atol=1e-2)
def testMF():
nx=17
nf=2
nu=np.linspace(100,200,nf)*1e6
FreqsInfo=None
FreqsInfo={"freqs":[nu[0:1],nu[1:2]],"WeightChansImages":np.array([0.5, 0.5])}
# nf=1
# nu=np.linspace(100,200,nf)*1e6
# FreqsInfo=None
PSF=np.zeros((nf,1,nx,nx),np.float32)
PSF[:,0,nx/2,nx/2]=1.
PSF=ModFFTW.ConvolveGaussian(PSF,CellSizeRad=1,GaussPars=[(1.,1.,0.)]*nf)
DataModel=np.zeros((1,1,nx,nx),np.float32)
Alpha=np.zeros((1,1,nx,nx),np.float32)
DataModel[0,0,nx/2,nx/2]=1.
#Alpha[0,0,:,:]=-.8
#DataModel[0,0,nx/2+5,nx/2+4]=1.
DataModel=ModFFTW.ConvolveGaussian(DataModel,CellSizeRad=1,GaussPars=[(1,.5,0.)])
#Alpha[0,0,nx/2,nx/2]=-.8
DataModel[0,0,nx/2,nx/2]=+1.
Alpha[0,0,nx/2,nx/2]=.8
DataModel[0,0,nx/2-5,nx/2+4]+=2.
#DataModel[0,0,nx/2-2,nx/2+2]+=2.
Alpha[0,0,nx/2-5,nx/2+4]=-.8
FreqRef=np.mean(nu)
nur=nu.reshape((2,1,1,1))
DataModelMF=DataModel*(nur/FreqRef)**Alpha
Dirty=ModFFTW.ConvolveGaussian(DataModelMF,CellSizeRad=1,GaussPars=[(1.,1.,0.)]*nf)
#DataConv=ModFFTW.ConvolveGaussian(DataTrue,CellSizeRad=1,GaussPars=[(1.,1.,0.)])
#DataConv=DataTrue
#IndDataTrue=ArrayToInd(DataConv)
indx,indy=np.where(Dirty[0,0]>1e-1)
ListPixParms=[ij for ij in zip(indx,indy)]
indx,indy=np.where(Dirty[0,0]>1e-2)
ListPixParms=[ij for ij in zip(indx,indy)]
ListPixData=ListPixParms
GD={"GAClean":{"GASolvePars":["S","Alpha"]}}
CEv=ClassEvolveGA(Dirty,PSF,FreqsInfo,ListPixParms=ListPixParms,ListPixData=ListPixData,GD=GD)
CEv.ArrayMethodsMachine.DataTrue=DataModelMF
CEv.ArrayMethodsMachine.PM.DicoIParm["S"]["DataModel"]=DataModel
return CEv
def GiveSpectralIndexMap(self, CellSizeRad=1., GaussPars=[(1, 1, 0)], DoConv=True, MaxSpi=100, MaxDR=1e+6,
threshold=None):
dFreq = 1e6
# f0=self.DicoSMStacked["AllFreqs"].min()
# f1=self.DicoSMStacked["AllFreqs"].max()
RefFreq = self.DicoSMStacked["RefFreq"]
f0 = RefFreq / 1.5
f1 = RefFreq * 1.5
M0 = self.GiveModelImage(f0)
M1 = self.GiveModelImage(f1)
if DoConv:
# M0=ModFFTW.ConvolveGaussian(M0,CellSizeRad=CellSizeRad,GaussPars=GaussPars)
# M1=ModFFTW.ConvolveGaussian(M1,CellSizeRad=CellSizeRad,GaussPars=GaussPars)
# M0,_=ModFFTW.ConvolveGaussianWrapper(M0,Sig=GaussPars[0][0]/CellSizeRad)
# M1,_=ModFFTW.ConvolveGaussianWrapper(M1,Sig=GaussPars[0][0]/CellSizeRad)
M0, _ = ModFFTW.ConvolveGaussianScipy(M0, Sig=GaussPars[0][0] / CellSizeRad)
M1, _ = ModFFTW.ConvolveGaussianScipy(M1, Sig=GaussPars[0][0] / CellSizeRad)
# print M0.shape,M1.shape
# compute threshold for alpha computation by rounding DR threshold to .1 digits (i.e. 1.65e-6 rounds to 1.7e-6)
if threshold is not None:
minmod = threshold
elif not np.all(M0 == 0):
minmod = float("%.1e" % (np.max(np.abs(M0)) / MaxDR))
else:
minmod = 1e-6
# mask out pixels above threshold
mask = (M1 < minmod) | (M0 < minmod)
print("computing alpha map for model pixels above %.1e Jy (based on max DR setting of %g)" % (
minmod, MaxDR), file=log)
M0[mask] = minmod
M1[mask] = minmod
# with np.errstate(invalid='ignore'):
# alpha = (np.log(M0)-np.log(M1))/(np.log(f0/f1))
# print
# print np.min(M0),np.min(M1),minmod
# print
alpha = (np.log(M0) - np.log(M1)) / (np.log(f0 / f1))
alpha[mask] = 0
# mask out |alpha|>MaxSpi. These are not physically meaningful anyway
mask = alpha > MaxSpi
alpha[mask] = MaxSpi
masked = mask.any()
mask = alpha < -MaxSpi
alpha[mask] = -MaxSpi
if masked or mask.any():
print(ModColor.Str("WARNING: some alpha pixels outside +/-%g. Masking them." % MaxSpi, col="red"), file=log)
return alpha
def GiveSpectralIndexMap(self,
CellSizeRad=1.,
GaussPars=[(1, 1, 0)],
DoConv=True,
MaxSpi=100,
MaxDR=1e+6,
threshold=None):
dFreq = 1e6
RefFreq = self.DicoSMStacked["RefFreq"]
f0 = RefFreq / 1.5 #self.DicoSMStacked["AllFreqs"].min()
f1 = RefFreq * 1.5 #self.DicoSMStacked["AllFreqs"].max()
M0 = self.GiveModelImage(f0)
M1 = self.GiveModelImage(f1)
if DoConv:
#M0=ModFFTW.ConvolveGaussian(M0,CellSizeRad=CellSizeRad,GaussPars=GaussPars)
#M1=ModFFTW.ConvolveGaussian(M1,CellSizeRad=CellSizeRad,GaussPars=GaussPars)
#M0,_=ModFFTW.ConvolveGaussianWrapper(M0,Sig=GaussPars[0][0]/CellSizeRad)
#M1,_=ModFFTW.ConvolveGaussianWrapper(M1,Sig=GaussPars[0][0]/CellSizeRad)
M0, _ = ModFFTW.ConvolveGaussianScipy(M0,
Sig=GaussPars[0][0] /
CellSizeRad)
M1, _ = ModFFTW.ConvolveGaussianScipy(M1,
Sig=GaussPars[0][0] /
CellSizeRad)
# compute threshold for alpha computation by rounding DR threshold to .1 digits (i.e. 1.65e-6 rounds to 1.7e-6)
if threshold is not None:
minmod = threshold
elif not np.all(M0 == 0):
minmod = float("%.1e" % (np.max(np.abs(M0)) / MaxDR))
else:
minmod = 1e-6
# mask out pixels above threshold
mask = (M1 < minmod) | (M0 < minmod)
print >> log, "computing alpha map for model pixels above %.1e Jy (based on max DR setting of %g)" % (
minmod, MaxDR)
M0[mask] = minmod
M1[mask] = minmod
alpha = (np.log(M0) - np.log(M1)) / (np.log(f0 / f1))
alpha[mask] = 0
# Np=1000
# indx,indy=np.int64(np.random.rand(Np)*M0.shape[0]),np.int64(np.random.rand(Np)*M0.shape[1])
# med=np.median(np.abs(M0[:,:,indx,indy]))
# Mask=((M1>100*med)&(M0>100*med))
# alpha=np.zeros_like(M0)
# alpha[Mask]=(np.log(M0[Mask])-np.log(M1[Mask]))/(np.log(f0/f1))
return alpha
def _getGlobalFFTWMachine(FFTMachineType, GridShape, dtype):
"""Returns an FFTWMachine matching the arguments.
Makes sure an FFTW machine is initialized only once per process, and only as needed.
"""
machine = ClassDDEGridMachine._global_fftw_machines.get(
(GridShape, dtype))
if machine is None:
if FFTMachineType == "FFTW":
# use single-core FFT because we parallelize by facet instead
ClassDDEGridMachine._global_fftw_machines[(
GridShape,
dtype)] = machine = ModFFTW.FFTW_2Donly(GridShape,
dtype,
ncores=1)
elif FFTMachineType == "LAPACK":
ClassDDEGridMachine._global_fftw_machines[(
GridShape, dtype)] = machine = ModFFTW.FFTW_2Donly_np(
GridShape, dtype)
return machine
def __init__(self,
GridShape=None,
PaddingInnerCoord=None,
OverS=None,
Padding=None,
dtype=None,
ifzfCF=None,
Mode="Blender",
GD=None):
self.GridShape = GridShape
self.PaddingInnerCoord = PaddingInnerCoord
self.OverS = OverS
self.Padding = Padding
self.dtype = dtype
self.ifzfCF = ifzfCF
self.GD = GD
self.FFTWMachine = ModFFTW.FFTW_2Donly_np(self.GridShape,
self.dtype,
ncores=1)
self.Mode = Mode
def GiveRandomModelIm(self):
# np.random.seed(0)
SModel,NModel=np.load(self.options.RandomCat_CountsFile).T
ind=np.argsort(SModel)
SModel=SModel[ind]
NModel=NModel[ind]
#SModel/=1e3
NModel/=SModel**(5/2.)
def GiveNPerOmega(s):
xp=np.interp(np.log10(s), np.log10(SModel), np.log10(NModel), left=None, right=None)
# import pylab
# pylab.clf()
# pylab.plot(np.log10(SModel), np.log10(NModel))
# pylab.scatter(np.log10(s),xp)
# pylab.draw()
# pylab.show()
# pylab.pause(0.1)
return 10**xp
std=np.std(self.Residual.flat[np.int64(np.random.rand(1000)*self.Residual.size)])
nbin=10000
smin=2.*std
smax=10.
LogS=np.linspace(np.log10(smin),np.log10(smax),nbin)
Model=np.zeros_like(self.Residual)
Omega=self.Residual.size*self.CellSizeRad**2
nx=Model.shape[-1]
im=image(self.FitsFile)
Lra=[]
Ldec=[]
LS=[]
SRA=[]
SDEC=[]
f,p,_,_=im.toworld((0,0,0,0))
for iBin in range(nbin-1):
ThisS=(10**LogS[iBin]+10**LogS[iBin+1])/2.
dx=10**LogS[iBin+1]-10**LogS[iBin]
n=int(scipy.stats.poisson.rvs(GiveNPerOmega(ThisS)*Omega*dx))#int(round(GiveNPerOmega(ThisS)*Omega*dx))
indx=np.array([np.int64(np.random.rand(n)*nx)]).ravel()
indy=np.array([np.int64(np.random.rand(n)*nx)]).ravel()
s0,s1=10**LogS[iBin],10**LogS[iBin+1]
RandS=np.random.rand(n)*(s1-s0)+s0
Model[0,0,indy,indx]=RandS
for iS in range(indx.size):
_,_,dec,ra=im.toworld((0,0,indy[iS],indx[iS]))
Lra.append(ra)
Ldec.append(dec)
LS.append(RandS[iS])
#SRA.append(rad2hmsdms(ra,Type="ra").replace(" ",":"))
#SDEC.append(rad2hmsdms(dec,Type="dec").replace(" ",":"))
#Cat=np.zeros((len(Lra),),dtype=[("ra",np.float64),("StrRA","S200"),("dec",np.float64),("StrDEC","S200"),("S",np.float64)])
Cat=np.zeros((len(Lra),),dtype=[("ra",np.float64),("dec",np.float64),("S",np.float64)])
Cat=Cat.view(np.recarray)
Cat.ra=np.array(Lra)
Cat.dec=np.array(Ldec)
#Cat.StrRA=np.array(SRA)
#Cat.StrDEC=np.array(SDEC)
Cat.S=np.array(LS)
CatName="%s.cat.npy"%self.OutName
print("Saving simulated catalog as %s"%CatName, file=log)
np.save(CatName,Cat)
ModelOut=np.zeros_like(Model)
ModelOut[0,0]=Model[0,0].T[::-1]
p=self.options.RandomCat_TotalToPeak
if p>1.:
nx=101
x,y=np.mgrid[-nx:nx+1,-nx:nx+1]
r2=x**2+y**2
def G(sig):
C0=1./(2.*np.pi*sig**2)
C=C0*np.exp(-r2/(2.*sig**2))
C/=np.sum(C)
return C
ListSig=np.linspace(0.001,10.,100)
TotToPeak=np.array([1./np.max(G(s)) for s in ListSig])
sig=np.interp(self.options.RandomCat_TotalToPeak,TotToPeak,ListSig)
print("Found a sig of %f"%sig, file=log)
Gaussian=G(sig)
ModelOut[0,0]=scipy.signal.fftconvolve(ModelOut[0,0], G(sig), mode='same')
FWHMFact=2.*np.sqrt(2.*np.log(2.))
BeamPix=self.BeamPix/FWHMFact
Model=G(sig).reshape((1,1,x.shape[0],x.shape[0]))
ConvModel,_=ModFFTW.ConvolveGaussianWrapper(Model,Sig=BeamPix)
self.SimulObsPeak=np.max(ConvModel)
print(" Gaussian Peak: %f"%np.max(Gaussian), file=log)
print(" Gaussian Int : %f"%np.sum(Gaussian), file=log)
print(" Obs peak : %f"%self.SimulObsPeak, file=log)
self.header_dict["OPKRATIO"]=self.SimulObsPeak
self.header_dict["GSIGMA"]=sig
self.header_dict["RTOTPK"]=self.options.RandomCat_TotalToPeak
else:
self.header_dict["OPKRATIO"]=1.
self.header_dict["GSIGMA"]=0.
self.header_dict["RTOTPK"]=1.
return ModelOut
def Restore(self):
print("Create restored image", file=log)
FWHMFact=2.*np.sqrt(2.*np.log(2.))
FWHMFact=2.*np.sqrt(2.*np.log(2.))
BeamPix=self.BeamPix/FWHMFact
sigma_x, sigma_y=BeamPix,BeamPix
theta=0.
bmaj=np.max([sigma_x, sigma_y])*self.CellArcSec*FWHMFact
bmin=np.min([sigma_x, sigma_y])*self.CellArcSec*FWHMFact
#bmaj=bmin=0.001666666666666667*3600
#sigma_x=
self.FWHMBeam=(bmaj/3600./np.sqrt(2.),bmin/3600./np.sqrt(2.),theta)
self.PSFGaussPars = (sigma_x*self.CellSizeRad, sigma_y*self.CellSizeRad, theta)
#print "!!!!!!!!!!!!!!!!!!!!"
#self.PSFGaussPars = (BeamPix,BeamPix,0)
RefFreq=self.ModelMachine.RefFreq
df=RefFreq*0.5
# ################################"
if self.options.PSFCache!="":
import os
IdSharedMem=str(int(os.getpid()))+"."
MeanModelImage=self.ModelMachine.GiveModelImage(RefFreq)
# #imNorm=image("6SBc.KAFCA.restoredNew.fits.6SBc.KAFCA.restoredNew.fits.MaskLarge.fits").getdata()
# imNorm=image("6SB.KAFCA.GA.BIC_00.AP.dirty.fits.mask.fits").getdata()
# MASK=np.zeros_like(imNorm)
# nchan,npol,_,_=MASK.shape
# for ch in range(nchan):
# for pol in range(npol):
# MASK[ch,pol,:,:]=imNorm[ch,pol,:,:].T[::-1,:]
# MeanModelImage[MASK==0]=0
# MeanModelImage.fill(0)
# MeanModelImage[0,0,100,100]=1
from DDFacet.Imager.GA import ClassSmearSM
from DDFacet.Imager import ClassPSFServer
self.DicoVariablePSF = MyPickle.FileToDicoNP(self.options.PSFCache)
self.PSFServer=ClassPSFServer.ClassPSFServer()
self.PSFServer.setDicoVariablePSF(self.DicoVariablePSF,NormalisePSF=True)
#return self.Residual,MeanModelImage,self.PSFServer
# CasaImage=ClassCasaImage.ClassCasaimage("Model.fits",MeanModelImage.shape,self.Cell,self.radec)#Lambda=(Lambda0,dLambda,self.NBands))
# CasaImage.setdata(MeanModelImage,CorrT=True)
# CasaImage.ToFits()
# #CasaImage.setBeam((SmoothFWHM,SmoothFWHM,0))
# CasaImage.close()
SmearMachine=ClassSmearSM.ClassSmearSM(self.Residual,
MeanModelImage*self.SqrtNormImage,
self.PSFServer,
DeltaChi2=4.,
IdSharedMem=IdSharedMem,
NCPU=self.options.NCPU)
SmearedModel=SmearMachine.Smear()
SmoothFWHM=self.CellArcSec*SmearMachine.RestoreFWHM/3600.
ModelSmearImage="%s.RestoredSmear"%self.BaseImageName
CasaImage=ClassCasaImage.ClassCasaimage(ModelSmearImage,SmearedModel.shape,self.Cell,self.radec)#Lambda=(Lambda0,dLambda,self.NBands))
CasaImage.setdata(SmearedModel+self.Residual,CorrT=True)
#CasaImage.setdata(SmearedModel,CorrT=True)
CasaImage.setBeam((SmoothFWHM,SmoothFWHM,0))
CasaImage.ToFits()
CasaImage.close()
SmearMachine.CleanUpSHM()
stop
# ################################"
#self.ModelMachine.ListScales[0]["Alpha"]=-0.8
# model image
#ModelMachine.GiveModelImage(RefFreq)
FEdge=np.linspace(RefFreq-df,RefFreq+df,self.NBands+1)
FCenter=(FEdge[0:-1]+FEdge[1::])/2.
C=299792458.
Lambda0=C/FCenter[-1]
dLambda=1
if self.NBands>1:
dLambda=np.abs(C/FCenter[0]-C/FCenter[1])
ListRestoredIm=[]
Lambda=[Lambda0+i*dLambda for i in range(self.NBands)]
ListRestoredImCorr=[]
ListModelIm=[]
#print C/np.array(Lambda)
# restored image
for l in Lambda:
freq=C/l
if self.options.RandomCat:
print("Create random catalog... ", file=log)
ModelImage=self.GiveRandomModelIm()
else:
print("Get ModelImage... ", file=log)
ModelImage=self.ModelMachine.GiveModelImage(freq)
if self.options.ZeroNegComp:
print("Zeroing negative componants... ", file=log)
ModelImage[ModelImage<0]=0
ListModelIm.append(ModelImage)
if self.options.Mode=="App":
print(" ModelImage to apparent flux... ", file=log)
ModelImage=ModelImage*self.SqrtNormImage
print("Convolve... ", file=log)
print(" MinMax = [%f , %f] @ freq = %f MHz"%(ModelImage.min(),ModelImage.max(),freq/1e6), file=log)
#RestoredImage=ModFFTW.ConvolveGaussianScipy(ModelImage,CellSizeRad=self.CellSizeRad,GaussPars=[self.PSFGaussPars])
if self.options.AddNoise>0.:
print("Adding Noise... ", file=log)
ModelImage+=np.random.randn(*ModelImage.shape)*self.options.AddNoise
RestoredImage,_=ModFFTW.ConvolveGaussianWrapper(ModelImage,Sig=BeamPix)
# =======
# #RestoredImage,_=ModFFTW.ConvolveGaussianWrapper(ModelImage,Sig=BeamPix)
# def GiveGauss(Sig0,Sig1):
# npix=20*int(np.sqrt(Sig0**2+Sig1**2))
# if not npix%2: npix+=1
# dx=npix/2
# x,y=np.mgrid[-dx:dx:npix*1j,-dx:dx:npix*1j]
# dsq=x**2+y**2
# return Sig0**2/(Sig0**2+Sig1**2)*np.exp(-dsq/(2.*(Sig0**2+Sig1**2)))
# R2=np.zeros_like(ModelImage)
# Sig0=BeamPix/np.sqrt(2.)
# if self.options.RandomCat:
# Sig1=(self.options.RandomCat_SigFactor-1.)*Sig0
# else:
# Sig1=0.
# nch,npol,_,_=ModelImage.shape
# for ch in range(nch):
# in1=ModelImage[ch,0]
# R2[ch,0,:,:]=scipy.signal.fftconvolve(in1,GiveGauss(Sig0,Sig1), mode='same').real
# RestoredImage=R2
# self.header_dict["GSIGMA"]=Sig0
# # print np.max(np.abs(R2-RestoredImage))
# # import pylab
# # ax=pylab.subplot(1,3,1)
# # pylab.imshow(RestoredImage[0,0],interpolation="nearest")
# # pylab.colorbar()
# # pylab.subplot(1,3,2,sharex=ax,sharey=ax)
# # pylab.imshow(R2[0,0],interpolation="nearest")
# # pylab.colorbar()
# # pylab.subplot(1,3,3,sharex=ax,sharey=ax)
# # pylab.imshow((RestoredImage-R2)[0,0],interpolation="nearest")
# # pylab.colorbar()
# # pylab.show()
# # stop
# >>>>>>> 0457182a873da89a2758f4be8a18f55cefd88e44
RestoredImageRes=RestoredImage+self.Residual
ListRestoredIm.append(RestoredImageRes)
RestoredImageResCorr=RestoredImageRes/self.SqrtNormImage
ListRestoredImCorr.append(RestoredImageResCorr)
#print FEdge,FCenter
print("Save... ", file=log)
_,_,nx,_=RestoredImageRes.shape
RestoredImageRes=np.array(ListRestoredIm).reshape((self.NBands,1,nx,nx))
RestoredImageResCorr=np.array(ListRestoredImCorr).reshape((self.NBands,1,nx,nx))
ModelImage=np.array(ListModelIm).reshape((self.NBands,1,nx,nx))
if self.OutName=="":
ImageName="%s.restoredNew"%self.BaseImageName
ImageNameCorr="%s.restoredNew.corr"%self.BaseImageName
ImageNameModel="%s.model"%self.BaseImageName
ImageNameModelConv="%s.modelConv"%self.BaseImageName
else:
ImageName=self.OutName
ImageNameCorr=self.OutName+".corr"
ImageNameModel="%s.model"%self.OutName
ImageNameModelConv="%s.modelConv"%self.OutName
CasaImage=ClassCasaImage.ClassCasaimage(ImageNameModel,RestoredImageRes.shape,self.Cell,self.radec,header_dict=self.header_dict)#Lambda=(Lambda0,dLambda,self.NBands))
CasaImage.setdata(ModelImage,CorrT=True)
CasaImage.setBeam(self.FWHMBeam)
CasaImage.ToFits()
CasaImage.close()
CasaImage=ClassCasaImage.ClassCasaimage(ImageName,RestoredImageRes.shape,self.Cell,self.radec,Freqs=C/np.array(Lambda).ravel(),header_dict=self.header_dict)#,Lambda=(Lambda0,dLambda,self.NBands))
CasaImage.setdata(RestoredImageRes,CorrT=True)
CasaImage.setBeam(self.FWHMBeam)
CasaImage.ToFits()
CasaImage.close()
CasaImage=ClassCasaImage.ClassCasaimage(ImageNameModelConv,RestoredImage.shape,self.Cell,self.radec,Freqs=C/np.array(Lambda).ravel(),header_dict=self.header_dict)#,Lambda=(Lambda0,dLambda,self.NBands))
CasaImage.setdata(RestoredImage,CorrT=True)
CasaImage.setBeam(self.FWHMBeam)
CasaImage.ToFits()
CasaImage.close()
if self.MakeCorrected:
CasaImage=ClassCasaImage.ClassCasaimage(ImageNameCorr,RestoredImageResCorr.shape,self.Cell,self.radec,Freqs=C/np.array(Lambda).ravel(),header_dict=self.header_dict)#,Lambda=(Lambda0,dLambda,self.NBands))
CasaImage.setdata(RestoredImageResCorr,CorrT=True)
CasaImage.setBeam(self.FWHMBeam)
CasaImage.ToFits()
CasaImage.close()
# ImageName="%s.modelConv"%self.BaseImageName
# CasaImage=ClassCasaImage.ClassCasaimage(ImageName,ModelImage.shape,self.Cell,self.radec)
# CasaImage.setdata(self.RestoredImage,CorrT=True)
# CasaImage.ToFits()
# CasaImage.setBeam(self.FWHMBeam)
# CasaImage.close()
# Alpha image
if self.DoAlpha:
print("Get Index Map... ", file=log)
IndexMap=self.ModelMachine.GiveSpectralIndexMap(CellSizeRad=self.CellSizeRad,GaussPars=[self.PSFGaussPars])
ImageName="%s.alphaNew"%self.BaseImageName
print(" Save... ", file=log)
CasaImage=ClassCasaImage.ClassCasaimage(ImageName,ModelImage.shape,self.Cell,self.radec)
CasaImage.setdata(IndexMap,CorrT=True)
CasaImage.ToFits()
CasaImage.close()
print(" Done. ", file=log)
def GiveSpectralIndexMap(self, GaussPars=[(1, 1, 0)], ResidCube=None,
GiveComponents=False, ChannelWeights=None):
# convert to radians
ex, ey, pa = GaussPars
ex *= np.pi/180/np.sqrt(2)/2
ey *= np.pi/180/np.sqrt(2)/2
epar = (ex + ey)/2.0
pa = 0.0
# get in terms of number of cells
CellSizeRad = self.GD['Image']['Cell'] * np.pi / 648000
# get Gaussian kernel
GaussKern = ModFFTW.GiveGauss(self.Npix, CellSizeRad=CellSizeRad, GaussPars=(epar, epar, pa), parallel=False)
# take FT
Fs = np.fft.fftshift
iFs = np.fft.ifftshift
# evaluate model
ModelImage = self.GiveModelImage(self.GridFreqs)
# pad GausKern and take FT
GaussKern = np.pad(GaussKern, self.Npad, mode='constant')
FTshape, _ = GaussKern.shape
from scipy import fftpack as FT
GaussKernhat = FT.fft2(iFs(GaussKern))
# pad and FT of ModelImage
ModelImagehat = np.zeros((self.Nchan, FTshape, FTshape), dtype=np.complex128)
ConvModelImage = np.zeros((self.Nchan, self.Npix, self.Npix), dtype=np.float64)
I = slice(self.Npad, -self.Npad)
for i in range(self.Nchan):
tmp_array = np.pad(ModelImage[i, 0], self.Npad, mode='constant')
ModelImagehat[i] = FT.fft2(iFs(tmp_array)) * GaussKernhat
ConvModelImage[i] = Fs(FT.ifft2(ModelImagehat[i]))[I, I].real
if ResidCube is not None:
ConvModelImage += ResidCube.squeeze()
RMS = np.std(ResidCube.flatten())
Threshold = self.GD["SPIMaps"]["AlphaThreshold"] * RMS
else:
AbsModel = np.abs(ModelImage).squeeze()
MinAbsImage = np.amin(AbsModel, axis=0)
RMS = np.min(np.abs(MinAbsImage.flatten())) # base cutoff on smallest value in model
Threshold = self.GD["SPIMaps"]["AlphaThreshold"] * RMS
# get minimum along any freq axis
MinImage = np.amin(ConvModelImage, axis=0)
MaskIndices = np.argwhere(MinImage > Threshold)
FitCube = ConvModelImage[:, MaskIndices[:, 0], MaskIndices[:, 1]]
if ChannelWeights is None:
weights = np.ones(self.Nchan, dtype=np.float32)
else:
weights = ChannelWeights.astype(np.float32)
if ChannelWeights.size != self.Nchan:
import warnings
warnings.warn("The provided channel weights are of incorrect length. Ignoring weights.", RuntimeWarning)
weights = np.ones(self.Nchan, dtype=np.float32)
try:
import traceback
from africanus.model.spi.dask import fit_spi_components
NCPU = self.GD["Parallel"]["NCPU"]
if NCPU:
from multiprocessing.pool import ThreadPool
import dask
dask.config.set(pool=ThreadPool(NCPU))
else:
import multiprocessing
NCPU = multiprocessing.cpu_count()
import dask.array as da
_, ncomps = FitCube.shape
FitCubeDask = da.from_array(FitCube.T.astype(np.float64), chunks=(ncomps//NCPU, self.Nchan))
weightsDask = da.from_array(weights.astype(np.float64), chunks=(self.Nchan))
freqsDask = da.from_array(np.array(self.GridFreqs).astype(np.float64), chunks=(self.Nchan))
alpha, varalpha, Iref, varIref = fit_spi_components(FitCubeDask, weightsDask,
freqsDask, self.RefFreq,
dtype=np.float64).compute()
except Exception as e:
traceback_str = traceback.format_exc(e)
print("Warning - Failed at importing africanus spi fitter. This could be an issue with the dask " \
"version. Falling back to (slow) scipy version", file=log)
print("Original traceback - ", traceback_str, file=log)
alpha, varalpha, Iref, varIref = self.FreqMachine.FitSPIComponents(FitCube, self.GridFreqs, self.RefFreq)
_, _, nx, ny = ModelImage.shape
alphamap = np.zeros([nx, ny])
Irefmap = np.zeros([nx, ny])
alphastdmap = np.zeros([nx, ny])
Irefstdmap = np.zeros([nx, ny])
alphamap[MaskIndices[:, 0], MaskIndices[:, 1]] = alpha
Irefmap[MaskIndices[:, 0], MaskIndices[:, 1]] = Iref
alphastdmap[MaskIndices[:, 0], MaskIndices[:, 1]] = np.sqrt(varalpha)
Irefstdmap[MaskIndices[:, 0], MaskIndices[:, 1]] = np.sqrt(varIref)
if GiveComponents:
return alphamap[None, None], alphastdmap[None, None], alpha
else:
return alphamap[None, None], alphastdmap[None, None]
def MakeMultiScaleCube(self):
if self.CubePSFScales is not None: return
print("Making MultiScale PSFs...", file=log)
LScales = self.GD["HMP"]["Scales"]
if 0 in LScales: LScales.remove(0)
LRatios = self.GD["HMP"]["Ratios"]
NTheta = self.GD["HMP"]["NTheta"]
_, _, nx, ny = self.SubPSF.shape
NScales = len(LScales)
NRatios = len(LRatios)
CubePSFScales = np.zeros(
(NScales + 1 + NRatios * NTheta * (NScales), nx, ny))
Scales = np.array(LScales)
Ratios = np.array(LRatios)
self.ListScales = []
CubePSFScales[0, :, :] = self.SubPSF[0, 0, :, :]
self.ListScales.append({"ModelType": "Delta"})
iSlice = 1
Support = 61
for i in range(NScales):
Minor = Scales[i] / (2. * np.sqrt(2. * np.log(2.)))
Major = Minor
PSFGaussPars = (Major, Minor, 0.)
CubePSFScales[iSlice, :, :] = ModFFTW.ConvolveGaussian(
self.SubPSF, CellSizeRad=1., GaussPars=[PSFGaussPars])[0, 0]
Gauss = ModFFTW.GiveGauss(Support,
CellSizeRad=1.,
GaussPars=PSFGaussPars)
self.ListScales.append({
"ModelType": "Gaussian",
"Model": Gauss,
"ModelParams": PSFGaussPars,
"Scale": i
})
iSlice += 1
Theta = np.arange(0., np.pi - 1e-3, np.pi / NTheta)
for iScale in range(NScales):
for ratio in Ratios:
for th in Theta:
Minor = Scales[iScale] / (2. * np.sqrt(2. * np.log(2.)))
Major = Minor * ratio
PSFGaussPars = (Major, Minor, th)
CubePSFScales[iSlice, :, :] = ModFFTW.ConvolveGaussian(
self.SubPSF, CellSizeRad=1.,
GaussPars=[PSFGaussPars])[0, 0]
Max = np.max(CubePSFScales[iSlice, :, :])
CubePSFScales[iSlice, :, :] /= Max
# pylab.clf()
# pylab.subplot(1,2,1)
# pylab.imshow(CubePSFScales[0,:,:],interpolation="nearest")
# pylab.subplot(1,2,2)
# pylab.imshow(CubePSFScales[iSlice,:,:],interpolation="nearest")
# pylab.title("Scale = %s"%str(PSFGaussPars))
# pylab.draw()
# pylab.show(False)
# pylab.pause(0.1)
iSlice += 1
Gauss = ModFFTW.GiveGauss(
Support, CellSizeRad=1., GaussPars=PSFGaussPars) / Max
self.ListScales.append({
"ModelType": "Gaussian",
"Model": Gauss,
"ModelParams": PSFGaussPars,
"Scale": iScale
})
# Max=np.max(np.max(CubePSFScales,axis=1),axis=1)
# Max=Max.reshape((Max.size,1,1))
# CubePSFScales=CubePSFScales/Max
self.CubePSFScales = np.float32(CubePSFScales)
self.WeightWidth = 6
CellSizeRad = 1.
PSFGaussPars = (self.WeightWidth, self.WeightWidth, 0.)
self.WeightFunction = ModFFTW.GiveGauss(self.SubPSF.shape[-1],
CellSizeRad=1.,
GaussPars=PSFGaussPars)
#self.WeightFunction.fill(1)
self.SupWeightWidth = 3. * self.WeightWidth
print(" ... Done", file=log)
def giveBrutalRestored(self, DicoResidual):
print >> log, " Running Brutal deconvolution..."
ListSilentModules = [
"ClassImageDeconvMachineMSMF", "ClassPSFServer",
"ClassMultiScaleMachine", "GiveModelMachine",
"ClassModelMachineMSMF", "ClassImageDeconvMachineHogbom",
"ClassModelMachineHogbom"
]
# MyLogger.setSilent(ListSilentModules)
self.DicoDirty = DicoResidual
self.Orig_MeanDirty = self.DicoDirty["MeanImage"].copy()
self.Orig_Dirty = self.DicoDirty["ImageCube"].copy()
if self.NoiseMapReShape is not None:
print >> log, "Deconvolving on SNR map"
self.DeconvMachine.RMSFactor = 0
self.DeconvMachine.Init(
PSFVar=self.DicoVariablePSF,
PSFAve=self.DicoVariablePSF["EstimatesAvgPSF"][-1],
GridFreqs=self.GridFreqs,
DegridFreqs=self.DegridFreqs,
RefFreq=self.RefFreq)
if self.NoiseMapReShape is not None:
self.DeconvMachine.setNoiseMap(self.NoiseMapReShape,
PNRStop=self.GD["Mask"]["SigTh"])
# # #########################
# # debug
# MaskImage=image("image_dirin_SSD_test.dirty.fits.mask.fits").getdata()
# nch,npol,_,_=MaskImage.shape
# MaskArray=np.zeros(MaskImage.shape,np.bool8)
# for ch in range(nch):
# for pol in range(npol):
# MaskArray[ch,pol,:,:]=np.bool8(MaskImage[ch,pol].T[::-1].copy())[:,:]
# self.DeconvMachine.setMask(np.bool8(1-MaskArray))
# # #########################
self.DeconvMachine.Update(self.DicoDirty, DoSetMask=False)
self.DeconvMachine.updateRMS()
# ModConstructor = ClassModModelMachine(self.GD)
# ModelMachine = ModConstructor.GiveMM(Mode=self.GD["Deconv"]["Mode"])
# #print "ModelMachine"
# #time.sleep(30)
# self.ModelMachine=ModelMachine
# #self.ModelMachine.DicoSMStacked=self.DicoBasicModelMachine
# self.ModelMachine.setRefFreq(self.RefFreq,Force=True)
# self.MinorCycleConfig["ModelMachine"] = ModelMachine
# #self.ModelMachine.setModelShape(self.SubDirty.shape)
# #self.ModelMachine.setListComponants(self.DeconvMachine.ModelMachine.ListScales)
# #self.DeconvMachine.Update(self.DicoSubDirty,DoSetMask=False)
# #self.DeconvMachine.updateMask(np.logical_not(self.SubMask))
# self.DeconvMachine.updateModelMachine(ModelMachine)
self.DeconvMachine.resetCounter()
self.DeconvMachine.Deconvolve(UpdateRMS=False)
print >> log, " Getting model image..."
Model = self.ModelMachine.GiveModelImage(DoAbs=True)
if "Comp" in self.ExternalModelMachine.DicoSMStacked.keys():
Model += np.abs(self.ExternalModelMachine.GiveModelImage())
ModelImage = Model[0, 0]
print >> log, " Convolving image with beam %s..." % str(
self.DicoVariablePSF["EstimatesAvgPSF"][1])
#from DDFacet.ToolsDir import Gaussian
# Sig_rad=np.max(self.DicoVariablePSF["EstimatesAvgPSF"][1][0:2])
# Sig_pix=Sig_rad/self.DicoDirty["ImageInfo"]["CellSizeRad"]
# Sig_pix=np.max([1,Sig_pix])#*2
# n_pix=int(Sig_pix*4)
# if n_pix%2==0: n_pix+=1
# _,_,G=Gaussian.GaussianSymetric(Sig_pix,n_pix)
# from DDFacet.ToolsDir.GiveEdges import GiveEdgesDissymetric
# N1=G.shape[0]
# N0x,N0y=ModelImage.shape
# indx,indy=np.where(ModelImage!=0)
# ModelConv=np.zeros_like(ModelImage)
# for iComp in range(indx.size):
# xc,yc=indx[iComp],indy[iComp]
# Aedge,Bedge=GiveEdgesDissymetric((xc,yc),(N0x,N0y),(N1/2,N1/2),(N1,N1))
# x0d,x1d,y0d,y1d=Aedge
# x0p,x1p,y0p,y1p=Bedge
# ModelConv[x0d:x1d,y0d:y1d]+=G[x0p:x1p,y0p:y1p]*ModelImage[xc,yc]
# # ModelConv=scipy.signal.convolve2d(ModelImage,G,mode="same")
ModelConv = ModFFTW.ConvolveGaussian(
{0: Model},
0,
0,
0,
CellSizeRad=self.DicoDirty["ImageInfo"]["CellSizeRad"],
GaussPars_ch=self.DicoVariablePSF["EstimatesAvgPSF"][1])
#GaussPar=[i*5 for i in self.DicoVariablePSF["EstimatesAvgPSF"][1]]
#ModelConv+=ModFFTW.ConvolveGaussian(Model, CellSizeRad=self.DicoDirty["ImageInfo"]["CellSizeRad"],
# GaussPars=[GaussPar])
self.ModelConv = ModelConv.reshape(self.DicoDirty["MeanImage"].shape)
self.Restored = self.ModelConv + self.DicoDirty["MeanImage"]
self.DicoDirty["MeanImage"][...] = self.Orig_MeanDirty[...]
self.DicoDirty["ImageCube"][...] = self.Orig_Dirty[...]
self.DeconvMachine.Reset()
#MyLogger.setLoud(ListSilentModules)
return self.Restored
def GiveSpectralIndexMap(self,
GaussPars=[(1, 1, 0)],
ResidCube=None,
GiveComponents=False,
ChannelWeights=None):
# convert to radians
ex, ey, pa = GaussPars
ex *= np.pi / 180
ey *= np.pi / 180
pa *= np.pi / 180
# get in terms of number of cells
CellSizeRad = self.GD['Image']['Cell'] * np.pi / 648000
# ex /= self.GD['Image']['Cell'] * np.pi / 648000
# ey /= self.GD['Image']['Cell'] * np.pi / 648000
# get Gaussian kernel
GaussKern = ModFFTW.GiveGauss(self.Npix,
CellSizeRad=CellSizeRad,
GaussPars=(ex, ey, pa),
parallel=False)
# take FT
Fs = np.fft.fftshift
iFs = np.fft.ifftshift
npad = self.Npad
FTarray = self.ScaleMachine.FTMachine.xhatim.view()
FTarray[...] = iFs(np.pad(GaussKern[None, None],
((0, 0), (0, 0), (npad, npad), (npad, npad)),
mode='constant'),
axes=(2, 3))
# this puts the FT in FTarray
self.ScaleMachine.FTMachine.FFTim()
# need to copy since FTarray and FTcube are views to the same array
FTkernel = FTarray.copy()
# evaluate model
ModelImage = self.GiveModelImage(self.GridFreqs)
# pad and take FT
FTcube = self.ScaleMachine.FTMachine.Chatim.view()
FTcube[...] = iFs(np.pad(ModelImage,
((0, 0), (0, 0), (npad, npad), (npad, npad)),
mode='constant'),
axes=(2, 3))
self.ScaleMachine.FTMachine.CFFTim()
# multiply by kernel
FTcube *= FTkernel
# take iFT
self.ScaleMachine.FTMachine.iCFFTim()
I = slice(npad, -npad)
ConvModelImage = Fs(FTcube, axes=(2, 3))[:, :, I, I].real
ConvModelMean = np.mean(ConvModelImage.squeeze(), axis=0)
ConvModelLow = ConvModelImage[-1, 0]
ConvModelHigh = ConvModelImage[0, 0]
if ResidCube is not None:
ConvModelImage += ResidCube
ConvModelImage = ConvModelImage.squeeze()
RMS = np.std(ResidCube.flatten())
Threshold = self.GD["SPIMaps"]["AlphaThreshold"] * RMS
# get minimum along any freq axis
MinImage = np.amin(ConvModelImage, axis=0)
MaskIndices = np.argwhere(MinImage > Threshold)
FitCube = ConvModelImage[:, MaskIndices[:, 0], MaskIndices[:, 1]]
# Initial guess for I0
I0i = ConvModelMean[MaskIndices[:, 0], MaskIndices[:, 1]]
# initial guess for alphas
Ilow = ConvModelLow[MaskIndices[:, 0], MaskIndices[:, 1]] / I0i
Ihigh = ConvModelHigh[MaskIndices[:, 0], MaskIndices[:, 1]] / I0i
alphai = (np.log(Ihigh) -
np.log(Ilow)) / (np.log(self.GridFreqs[0] / self.RefFreq) -
np.log(self.GridFreqs[-1] / self.RefFreq))
# import matplotlib.pyplot as plt
#
# for i in xrange(self.Nchan):
# plt.imshow(np.where(ConvModelImage[i] > Threshold, ConvModelImage[i], 0.0))
# plt.show()
if ChannelWeights is None:
weights = np.ones(self.Nchan, dtype=np.float32)
else:
weights = ChannelWeights.astype(np.float32)
if ChannelWeights.size != self.Nchan:
import warnings
warnings.warn(
"The provided channel weights are of incorrect length. Ignoring weights.",
RuntimeWarning)
weights = np.ones(self.Nchan, dtype=np.float32)
try:
import traceback
from africanus.model.spi.dask import fit_spi_components
NCPU = self.GD["Parallel"]["NCPU"]
if NCPU:
from multiprocessing.pool import ThreadPool
import dask
dask.config.set(pool=ThreadPool(NCPU))
else:
import multiprocessing
NCPU = multiprocessing.cpu_count()
import dask.array as da
_, ncomps = FitCube.shape
FitCubeDask = da.from_array(FitCube.T.astype(np.float64),
chunks=(ncomps // NCPU, self.Nchan))
weightsDask = da.from_array(weights.astype(np.float64),
chunks=(self.Nchan))
freqsDask = da.from_array(self.GridFreqs.astype(np.float64),
chunks=(self.Nchan))
alpha, varalpha, Iref, varIref = fit_spi_components(
FitCubeDask,
weightsDask,
freqsDask,
self.RefFreq,
dtype=np.float64,
I0i=I0i,
alphai=alphai).compute()
# from africanus.model.spi import fit_spi_components
#
# alpha, varalpha, Iref, varIref = fit_spi_components(FitCube.T.astype(np.float64), weights.astype(np.float64),
# self.GridFreqs.astype(np.float64), self.RefFreq.astype(np.float64),
# dtype=np.float64, I0i=I0i, alphai=alphai)
except Exception as e:
traceback_str = traceback.format_exc(e)
print>>log, "Warning - Failed at importing africanus spi fitter. This could be an issue with the dask " \
"version. Falling back to (slow) scipy version"
print >> log, "Original traceback - ", traceback_str
alpha, varalpha, Iref, varIref = self.FreqMachine.FitSPIComponents(
FitCube, self.GridFreqs, self.RefFreq)
_, _, nx, ny = ModelImage.shape
alphamap = np.zeros([nx, ny])
Irefmap = np.zeros([nx, ny])
alphastdmap = np.zeros([nx, ny])
Irefstdmap = np.zeros([nx, ny])
alphamap[MaskIndices[:, 0], MaskIndices[:, 1]] = alpha
Irefmap[MaskIndices[:, 0], MaskIndices[:, 1]] = Iref
alphastdmap[MaskIndices[:, 0], MaskIndices[:, 1]] = np.sqrt(varalpha)
Irefstdmap[MaskIndices[:, 0], MaskIndices[:, 1]] = np.sqrt(varIref)
if GiveComponents:
return alphamap[None, None], alphastdmap[None, None], alpha
else:
return alphamap[None, None], alphastdmap[None, None]