def predictKernelPolCluster(self,
DicoData,
SM,
iDirection=None,
ApplyJones=None,
ApplyTimeJones=None,
Noise=None):
DicoData = NpShared.DicoToShared("%sDicoMemChunk" % (self.IdMemShared),
DicoData,
DelInput=False)
if ApplyTimeJones != None:
ApplyTimeJones = NpShared.DicoToShared("%sApplyTimeJones" %
(self.IdMemShared),
ApplyTimeJones,
DelInput=False)
nrow, nch, _ = DicoData["data"].shape
RowList = np.int64(np.linspace(0, nrow, self.NCPU + 1))
row0 = RowList[0:-1]
row1 = RowList[1::]
work_queue = multiprocessing.Queue()
result_queue = multiprocessing.Queue()
workerlist = []
NCPU = self.NCPU
NpShared.DelArray("%sPredictData" % (self.IdMemShared))
PredictArray = NpShared.SharedArray.create(
"%sPredictData" % (self.IdMemShared),
DicoData["data"].shape,
dtype=DicoData["data"].dtype)
for ii in range(NCPU):
W = WorkerPredict(work_queue,
result_queue,
self.IdMemShared,
SM=SM,
DoSmearing=self.DoSmearing,
BeamAtFacet=self._BeamAtFacet)
workerlist.append(W)
workerlist[ii].start()
NJobs = row0.size
for iJob in range(NJobs):
ThisJob = row0[iJob], row1[iJob]
work_queue.put(ThisJob)
while int(result_queue.qsize()) < NJobs:
time.sleep(0.1)
continue
for ii in range(NCPU):
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()
return PredictArray
def Interpol(self):
APP.startWorkers()
if "TEC" in self.DicoFile0["SmoothMode"]:
TECArray = NpShared.ToShared("%sTECArray" % IdSharedMem,
self.DicoFile0["SolsTEC"])
CPhaseArray = NpShared.ToShared("%sCPhaseArray" % IdSharedMem,
self.DicoFile0["SolsCPhase"])
nt, nd, na = TECArray.shape
iJob = 0
for it in range(nt):
APP.runJob("InterpolTECTime_%d" % iJob,
self.InterpolTECTime,
args=(it, )) #,serial=True)
iJob += 1
workers_res = APP.awaitJobResults("InterpolTECTime*",
progress="Interpol TEC")
iJob = 0
for it in range(nt):
APP.runJob("InterpolAmpTime_%d" % iJob,
self.InterpolAmpTime,
args=(it, )) #,serial=True)
iJob += 1
workers_res = APP.awaitJobResults("InterpolAmpTime*",
progress="Interpol Amp")
# APP.terminate()
APP.shutdown()
Multiprocessing.cleanupShm()
def GiveCovariance(self,
DicoDataIn,
ApplyTimeJones,
SM,
Mode="DDECovariance"):
DicoData = NpShared.DicoToShared("%sDicoMemChunk" % (self.IdMemShared),
DicoDataIn,
DelInput=False)
if ApplyTimeJones != None:
ApplyTimeJones = NpShared.DicoToShared("%sApplyTimeJones" %
(self.IdMemShared),
ApplyTimeJones,
DelInput=False)
nrow, nch, _ = DicoData["data"].shape
RowList = np.int64(np.linspace(0, nrow, self.NCPU + 1))
row0 = RowList[0:-1]
row1 = RowList[1::]
work_queue = multiprocessing.Queue()
result_queue = multiprocessing.Queue()
workerlist = []
NCPU = self.NCPU
# NpShared.DelArray("%sCorrectedData"%(self.IdMemShared))
# CorrectedData=NpShared.SharedArray.create("%sCorrectedData"%(self.IdMemShared),DicoData["data"].shape,dtype=DicoData["data"].dtype)
# CorrectedData=
for ii in range(NCPU):
W = WorkerPredict(work_queue,
result_queue,
self.IdMemShared,
Mode=Mode,
DoSmearing=self.DoSmearing,
SM=SM,
BeamAtFacet=self._BeamAtFacet)
workerlist.append(W)
workerlist[ii].start()
NJobs = row0.size
for iJob in range(NJobs):
ThisJob = row0[iJob], row1[iJob]
work_queue.put(ThisJob)
while int(result_queue.qsize()) < NJobs:
time.sleep(0.1)
continue
for ii in range(NCPU):
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()
DicoDataIn["W"] = DicoData["W"]
def InterpolTECTime(self, iTime):
GOut = NpShared.GiveArray("%sGOut" % IdSharedMem)
nt, nf, na, nd, _, _ = GOut.shape
TECArray = NpShared.GiveArray("%sTECArray" % IdSharedMem)
CPhaseArray = NpShared.GiveArray("%sCPhaseArray" % IdSharedMem)
for iDir in range(nd):
for iAnt in range(na):
GThis = TECToZ(TECArray[iTime, iDir, iAnt],
CPhaseArray[iTime, iDir, iAnt],
self.CentralFreqs.reshape((1, -1)))
self.GOut[iTime, :, iAnt, iDir, 0, 0] = GThis
self.GOut[iTime, :, iAnt, iDir, 1, 1] = GThis
def run(self):
while not self.kill_received:
try:
Job = self.work_queue.get()
except:
break
# ModelFacet=NpShared.GiveArray("%sModelFacet.%3.3i"%(self.IdSharedMem,iFacet))
# Grid=self.ClassImToGrid.setModelIm(ModelFacet)
# _=NpShared.ToShared("%sModelGrid.%3.3i"%(self.IdSharedMem,iFacet),Grid)
# self.result_queue.put({"Success":True})
iFacet = Job["iFacet"]
FacetDataCache = Job["FacetDataCache"]
Image = NpShared.GiveArray("%sModelImage" % (self.IdSharedMem))
#Grid,SumFlux=self.ClassImToGrid.GiveGridFader(Image,self.DicoImager,iFacet,NormImage)
#SharedMemName="%sSpheroidal.Facet_%3.3i"%(self.IdSharedMem,iFacet)
SPhe = NpShared.GiveArray(Job["SharedMemNameSphe"])
#print SharedMemName
SpacialWeight = NpShared.GiveArray("%sSpacialWeight.Facet_%3.3i" %
(FacetDataCache, iFacet))
NormImage = NpShared.GiveArray("%sNormImage" % self.IdSharedMem)
Im2Grid = ClassImToGrid(OverS=self.GD["ImagerCF"]["OverS"],
GD=self.GD)
Grid, SumFlux = Im2Grid.GiveModelTessel(Image,
self.DicoImager,
iFacet,
NormImage,
SPhe,
SpacialWeight,
ToGrid=True)
#ModelSharedMemName="%sModelImage.Facet_%3.3i"%(self.IdSharedMem,iFacet)
#NpShared.ToShared(ModelSharedMemName,ModelFacet)
_ = NpShared.ToShared(
"%sModelGrid.%3.3i" % (self.IdSharedMem, iFacet), Grid)
self.result_queue.put({
"Success": True,
"iFacet": iFacet,
"SumFlux": SumFlux
})
def EstimateThisTECTime(self, it, iAnt, iDir):
GOut = NpShared.GiveArray("%sGOut" % IdSharedMem)
g = GOut[it, :, iAnt, iDir, 0, 0]
g0 = g / np.abs(g)
W = np.ones(g0.shape, np.float32)
W[g == 1.] = 0
Z = self.Z
for iTry in range(5):
R = (g0.reshape((1, -1)) - Z) * W.reshape((1, -1))
Chi2 = np.sum(np.abs(R)**2, axis=1)
iTec = np.argmin(Chi2)
rBest = R[iTec]
if np.max(np.abs(rBest)) == 0: break
Sig = np.sum(np.abs(rBest * W)) / np.sum(W)
ind = np.where(np.abs(rBest * W) > 5. * Sig)[0]
if ind.size == 0: break
W[ind] = 0
# gz=TECToZ(TECGrid.ravel()[iTec],CPhase.ravel()[iTec],self.CentralFreqs)
# import pylab
# pylab.clf()
# pylab.subplot(2,1,1)
# pylab.scatter(self.CentralFreqs,rBest)
# pylab.scatter(self.CentralFreqs[ind],rBest[ind],color="red")
# pylab.subplot(2,1,2)
# pylab.scatter(self.CentralFreqs,rBest)
# pylab.scatter(self.CentralFreqs[ind],rBest[ind],color="red")
# pylab.draw()
# pylab.show()
# # ###########################
# print iAnt,iDir
# if iAnt==0: return
# f=np.linspace(self.CentralFreqs.min(),self.CentralFreqs.max(),100)
# ztec=TECToZ(TECGrid.ravel()[iTec],CPhase.ravel()[iTec],f)
# import pylab
# pylab.clf()
# pylab.subplot(1,2,1)
# pylab.scatter(self.CentralFreqs,np.abs(g),color="black")
# pylab.plot(self.CentralFreqs,np.abs(gz),ls=":",color="black")
# pylab.plot(self.CentralFreqs,np.abs(gz)-np.abs(g),ls=":",color="red")
# pylab.subplot(1,2,2)
# pylab.scatter(self.CentralFreqs,np.angle(g),color="black")
# pylab.plot(self.CentralFreqs,np.angle(gz),ls=":",color="black")
# pylab.plot(self.CentralFreqs,np.angle(gz)-np.angle(g),ls=":",color="red")
# #pylab.plot(f,np.angle(ztec),ls=":",color="black")
# pylab.ylim(-np.pi,np.pi)
# pylab.draw()
# pylab.show(False)
# pylab.pause(0.1)
# # ###############################
t0 = self.TECGrid.ravel()[iTec]
c0 = self.CPhase.ravel()[iTec]
gz = np.abs(g) * TECToZ(t0, c0, self.CentralFreqs)
return gz, t0, c0
def FitThisTEC(self, it):
nt, nch, na, nd, _, _ = self.Sols.G.shape
TECArray = NpShared.GiveArray("%sTECArray" % IdSharedMem)
CPhaseArray = NpShared.GiveArray("%sCPhaseArray" % IdSharedMem)
for iDir in range(nd):
# for it in range(nt):
Est = None
if it > 0:
E_TEC = TECArray[it - 1, iDir, :]
E_CPhase = CPhaseArray[it - 1, iDir, :]
Est = (E_TEC, E_CPhase)
gz, TEC, CPhase = self.FitThisTECTime(it, iDir, Est=Est)
GOut = NpShared.GiveArray("%sGOut" % IdSharedMem)
GOut[it, :, :, iDir, 0, 0] = gz
GOut[it, :, :, iDir, 1, 1] = gz
TECArray[it, iDir, :] = TEC
CPhaseArray[it, iDir, :] = CPhase
def __init__(self,
work_queue,
result_queue,
argsImToGrid=None,
IdSharedMem=None,
DicoImager=None,
FacetMode="Fader",
GD=None):
multiprocessing.Process.__init__(self)
self.work_queue = work_queue
self.result_queue = result_queue
self.kill_received = False
self.exit = multiprocessing.Event()
self.GD = GD
self.IdSharedMem = IdSharedMem
self.DicoImager = DicoImager
self.FacetMode = FacetMode
self.SharedMemNameSphe = "%sSpheroidal" % (self.IdSharedMem)
self.ifzfCF = NpShared.GiveArray(self.SharedMemNameSphe)
self.ClassImToGrid = ClassImToGrid(*argsImToGrid, ifzfCF=self.ifzfCF)
self.Image = NpShared.GiveArray("%sModelImage" % (self.IdSharedMem))
def GaussSmoothAmp(self, iAnt, iDir):
#print iAnt,iDir
GOut = NpShared.GiveArray("%sGOut" % IdSharedMem)
g = GOut[:, :, iAnt, iDir, 0, 0]
g0 = np.abs(g)
sg0 = scipy.ndimage.filters.gaussian_filter(g0, self.Amp_GaussKernel)
gz = sg0 * g / np.abs(g)
#print iAnt,iDir,GOut.shape,gz.shape
GOut[:, :, iAnt, iDir, 0, 0] = gz[:, :]
GOut[:, :, iAnt, iDir, 1, 1] = gz[:, :]
def FitThisPolyAmp(self, iDir):
nt, nch, na, nd, _, _ = self.Sols.G.shape
GOut = NpShared.GiveArray("%sGOut" % IdSharedMem)
g = GOut[:, :, :, iDir, 0, 0]
AmpMachine = ClassFitAmp.ClassFitAmp(
self.Sols.G[:, :, :, iDir, 0, 0],
self.CentralFreqs,
RemoveMedianAmp=self.RemoveMedianAmp)
gf = AmpMachine.doSmooth()
#print "Done %i"%iDir
gf = gf * g / np.abs(g)
GOut[:, :, :, iDir, 0, 0] = gf[:, :, :]
GOut[:, :, :, iDir, 1, 1] = gf[:, :, :]
def Save(self):
OutFile = self.SolsFileOut
if not ".npz" in OutFile: OutFile += ".npz"
log.print(" Saving interpolated solution file as: %s" % OutFile)
self.DicoOut["Sols"] = self.SolsOut
self.DicoOut["Sols"]["G"][:] = self.GOut[:]
try:
np.savez(OutFile, **(self.DicoOut))
except:
log.print("There was an exception while using savez")
log.print(" you should set the environment variable TMPDIR")
log.print(" to a directory where there is enough space")
NpShared.DelAll("%s" % IdSharedMem)
def ClipThisDir(self, iDir):
nt, nch, na, nd, _, _ = self.Sols.G.shape
GOut = NpShared.GiveArray("%sGOut" % IdSharedMem)
# g=GOut[:,:,:,iDir,0,0]
AmpMachine = ClassClip.ClassClip(self.Sols.G[:, :, :, iDir, 0, 0],
self.CentralFreqs,
RemoveMedianAmp=self.RemoveMedianAmp)
gf = AmpMachine.doClip()
GOut[:, :, :, iDir, 0, 0] = gf[:, :, :]
AmpMachine = ClassClip.ClassClip(self.Sols.G[:, :, :, iDir, 1, 1],
self.CentralFreqs,
RemoveMedianAmp=self.RemoveMedianAmp)
gf = AmpMachine.doClip()
GOut[:, :, :, iDir, 1, 1] = gf[:, :, :]
def InterpolAmpTime(self, iTime):
GOut = NpShared.GiveArray("%sGOut" % IdSharedMem)
nt, nf, na, nd, _, _ = GOut.shape
for iChan in range(nf):
D = self.DicoFreqWeights[iChan]
for iDir in range(nd):
for iAnt in range(na):
if D["Type"] == "Dual":
i0, i1 = D["Index"]
c0, c1 = D["Coefs"]
g = c0 * np.abs(self.Sols0.G[
iTime, i0, iAnt, iDir, 0, 0]) + c1 * np.abs(
self.Sols0.G[iTime, i1, iAnt, iDir, 0, 0])
else:
i0 = D["Index"]
g = np.abs(self.Sols0.G[iTime, i0, iAnt, iDir, 0, 0])
self.GOut[iTime, iChan, iAnt, iDir, 0, 0] *= g
self.GOut[iTime, iChan, iAnt, iDir, 1, 1] *= g
def Save(self):
OutFile = self.OutSolsName
if not ".npz" in OutFile: OutFile += ".npz"
#self.SpacialSmoothTEC()
if "TEC" in self.InterpMode:
# OutFileTEC="%s.TEC_CPhase.npz"%OutFile
# log.print(" Saving TEC/CPhase solution file as: %s"%OutFileTEC)
# np.savez(OutFileTEC,
# TEC=self.TECArray,
# CPhase=self.CPhaseArray)
self.DicoFile["SolsTEC"] = self.TECArray
self.DicoFile["SolsCPhase"] = self.CPhaseArray
log.print(" Saving interpolated solution file as: %s" % OutFile)
self.DicoFile["SmoothMode"] = self.InterpMode
self.DicoFile["SolsOrig"] = copy.deepcopy(self.DicoFile["Sols"])
self.DicoFile["SolsOrig"]["G"][:] = self.DicoFile["Sols"]["G"][:]
self.DicoFile["Sols"]["G"][:] = self.GOut[:]
np.savez(OutFile, **(self.DicoFile))
# import PlotSolsIm
# G=self.DicoFile["Sols"]["G"].view(np.recarray)
# iAnt,iDir=10,0
# import pylab
# pylab.clf()
# A=self.GOut[:,:,iAnt,iDir,0,0]
# B=G[:,:,iAnt,iDir,0,0]
# Gs=np.load(OutFile)["Sols"]["G"].view(np.recarray)
# C=Gs[:,:,iAnt,iDir,0,0]
# pylab.subplot(1,3,1)
# pylab.imshow(np.abs(A).T,interpolation="nearest",aspect="auto")
# pylab.subplot(1,3,2)
# pylab.imshow(np.abs(B).T,interpolation="nearest",aspect="auto")
# pylab.subplot(1,3,3)
# pylab.imshow(np.abs(C).T,interpolation="nearest",aspect="auto")
# pylab.draw()
# pylab.show()
# PlotSolsIm.Plot([self.DicoFile["Sols"].view(np.recarray)])
NpShared.DelAll("%s" % IdSharedMem)
def FitThisAmpTimePoly(self, it, iAnt, iDir):
GOut = NpShared.GiveArray("%sGOut" % IdSharedMem)
g = GOut[it, :, iAnt, iDir, 0, 0]
g0 = np.abs(g)
W = np.ones(g0.shape, np.float32)
W[g0 == 1.] = 0
if np.count_nonzero(W) < self.Amp_PolyOrder * 3: return
for iTry in range(5):
if np.max(W) == 0: return
z = np.polyfit(self.CentralFreqs, g0, self.Amp_PolyOrder, w=W)
p = np.poly1d(z)
gz = p(self.CentralFreqs) * g / np.abs(g)
rBest = (g0 - gz)
if np.max(np.abs(rBest)) == 0: break
Sig = np.sum(np.abs(rBest * W)) / np.sum(W)
ind = np.where(np.abs(rBest * W) > 5. * Sig)[0]
if ind.size == 0: break
W[ind] = 0
GOut[it, :, iAnt, iDir, 0, 0] = gz
GOut[it, :, iAnt, iDir, 1, 1] = gz
def __init__(self,
InSolsName,
OutSolsName,
InterpMode="TEC",
PolMode="Scalar",
Amp_PolyOrder=3,
NCPU=0,
Amp_GaussKernel=(0, 5),
Amp_SmoothType="Poly",
CrossMode=1,
RemoveAmpBias=0,
RemoveMedianAmp=True):
if type(InterpMode) == str:
InterpMode = InterpMode.split(",") #[InterpMode]
self.InSolsName = InSolsName
self.OutSolsName = OutSolsName
self.RemoveMedianAmp = RemoveMedianAmp
log.print("Loading %s" % self.InSolsName)
self.DicoFile = dict(np.load(self.InSolsName, allow_pickle=True))
self.Sols = self.DicoFile["Sols"].view(np.recarray)
if "MaskedSols" in self.DicoFile.keys():
MaskFreq = np.logical_not(
np.all(np.all(np.all(self.DicoFile["MaskedSols"][..., 0, 0],
axis=0),
axis=1),
axis=1))
nt, _, na, nd, _, _ = self.Sols.G.shape
self.DicoFile["FreqDomains"] = self.DicoFile["FreqDomains"][
MaskFreq]
NFreqsOut = np.count_nonzero(MaskFreq)
log.print("There are %i non-zero freq channels" % NFreqsOut)
SolsOut = np.zeros(
(nt, ),
dtype=[("t0", np.float64), ("t1", np.float64),
("G", np.complex64, (NFreqsOut, na, nd, 2, 2)),
("Stats", np.float32, (NFreqsOut, na, 4))])
SolsOut = SolsOut.view(np.recarray)
SolsOut.G = self.Sols.G[:, MaskFreq, ...]
SolsOut.t0 = self.Sols.t0
SolsOut.t1 = self.Sols.t1
self.Sols = self.DicoFile["Sols"] = SolsOut
del (self.DicoFile["MaskedSols"])
#self.Sols=self.Sols[0:10].copy()
self.CrossMode = CrossMode
self.CentralFreqs = np.mean(self.DicoFile["FreqDomains"], axis=1)
self.incrCross = 11
iii = 0
NTEC = 101
NConstPhase = 51
TECGridAmp = 0.1
TECGrid, CPhase = np.mgrid[-TECGridAmp:TECGridAmp:NTEC * 1j,
-np.pi:np.pi:NConstPhase * 1j]
Z = TECToZ(TECGrid.reshape((-1, 1)), CPhase.reshape((-1, 1)),
self.CentralFreqs.reshape((1, -1)))
self.Z = Z
self.TECGrid, self.CPhase = TECGrid, CPhase
self.InterpMode = InterpMode
self.Amp_PolyOrder = Amp_PolyOrder
self.RemoveAmpBias = RemoveAmpBias
if self.RemoveAmpBias:
self.CalcFreqAmpSystematics()
self.Sols.G /= self.G0
self.GOut = NpShared.ToShared("%sGOut" % IdSharedMem,
self.Sols.G.copy())
self.PolMode = PolMode
self.Amp_GaussKernel = Amp_GaussKernel
if len(self.Amp_GaussKernel) != 2:
raise ValueError("GaussKernel should be of size 2")
self.Amp_SmoothType = Amp_SmoothType
if "TEC" in self.InterpMode:
log.print(" Smooth phases using a TEC model")
if self.CrossMode:
log.print(ModColor.Str("Using CrossMode"))
if "Amp" in self.InterpMode:
if Amp_SmoothType == "Poly":
log.print(
" Smooth amplitudes using polynomial model of order %i" %
self.Amp_PolyOrder)
if Amp_SmoothType == "Gauss":
log.print(
" Smooth amplitudes using Gaussian kernel of %s (Time/Freq) bins"
% str(Amp_GaussKernel))
if self.RemoveAmpBias:
self.GOut *= self.G0
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=NCPU, affinity=0)
def FitThisTECTime(self, it, iDir, Est=None):
GOut = NpShared.GiveArray("%sGOut" % IdSharedMem)
nt, nch, na, nd, _, _ = self.Sols.G.shape
T = ClassTimeIt("CrossFit")
T.disable()
Mode = ["TEC", "CPhase"]
Mode = ["TEC"]
TEC0CPhase0 = np.zeros((len(Mode), na), np.float32)
for iAnt in range(na):
_, t0, c0 = self.EstimateThisTECTime(it, iAnt, iDir)
TEC0CPhase0[0, iAnt] = t0
if "CPhase" in Mode:
TEC0CPhase0[1, iAnt] = c0
T.timeit("init")
# ######################################
# Changing method
#print "!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
#print it,iDir
TECMachine = ClassFitTEC.ClassFitTEC(self.Sols.G[it, :, :, iDir, 0, 0],
self.CentralFreqs,
Tol=5.e-2,
Mode=Mode)
TECMachine.setX0(TEC0CPhase0.ravel())
X = TECMachine.doFit()
if "CPhase" in Mode:
TEC, CPhase = X.reshape((len(Mode), na))
else:
TEC, = X.reshape((len(Mode), na))
CPhase = np.zeros((1, na), np.float32)
TEC -= TEC[0]
CPhase -= CPhase[0]
GThis = np.abs(GOut[it, :, :, iDir, 0, 0]).T * TECToZ(
TEC.reshape((-1, 1)), CPhase.reshape(
(-1, 1)), self.CentralFreqs.reshape((1, -1)))
T.timeit("done %i %i %i" % (it, iDir, TECMachine.Current_iIter))
return GThis.T, TEC, CPhase
# ######################################
G = GOut[it, :, :, iDir, 0, 0].T.copy()
if self.CrossMode:
A0, A1 = np.mgrid[0:na, 0:na]
gg_meas = G[A0.ravel(), :] * G[A1.ravel(), :].conj()
gg_meas_reim = np.array([gg_meas.real,
gg_meas.imag]).ravel()[::self.incrCross]
else:
self.incrCross = 1
A0, A1 = np.mgrid[0:na], None
gg_meas = G[A0.ravel(), :]
gg_meas_reim = np.array([gg_meas.real,
gg_meas.imag]).ravel()[::self.incrCross]
# for ibl in range(gg_meas.shape[0])[::-1]:
# import pylab
# pylab.clf()
# pylab.subplot(2,1,1)
# pylab.scatter(self.CentralFreqs,np.abs(gg_meas[ibl]))
# pylab.ylim(0,5)
# pylab.subplot(2,1,2)
# pylab.scatter(self.CentralFreqs,np.angle(gg_meas[ibl]))
# pylab.ylim(-np.pi,np.pi)
# pylab.draw()
# pylab.show(False)
# pylab.pause(0.1)
iIter = np.array([0])
tIter = np.array([0], np.float64)
def _f_resid(TecConst, A0, A1, ggmeas, iIter, tIter):
T2 = ClassTimeIt("resid")
T2.disable()
TEC, CPhase = TecConst.reshape((2, na))
GThis = TECToZ(TEC.reshape((-1, 1)), CPhase.reshape((-1, 1)),
self.CentralFreqs.reshape((1, -1)))
#T2.timeit("1")
if self.CrossMode:
gg_pred = GThis[A0.ravel(), :] * GThis[A1.ravel(), :].conj()
else:
gg_pred = GThis[A0.ravel(), :]
#T2.timeit("2")
gg_pred_reim = np.array([gg_pred.real,
gg_pred.imag]).ravel()[::self.incrCross]
#T2.timeit("3")
r = (ggmeas - gg_pred_reim).ravel()
#print r.shape
#T2.timeit("4")
#return np.angle((ggmeas-gg_pred).ravel())
#print np.mean(np.abs(r))
iIter += 1
#tIter+=T2.timeit("all")
#print iIter[0]
return r
#print _f_resid(TEC0CPhase0,A0,A1,ggmeas)
Sol = least_squares(
_f_resid,
TEC0CPhase0.ravel(),
#method="trf",
method="lm",
args=(A0, A1, gg_meas_reim, iIter, tIter),
ftol=1e-2,
gtol=1e-2,
xtol=1e-2) #,ftol=1,gtol=1,xtol=1,max_nfev=1)
#Sol=leastsq(_f_resid, TEC0CPhase0.ravel(), args=(A0,A1,gg_meas_reim,iIter),ftol=1e-2,gtol=1e-2,xtol=1e-2)
#T.timeit("Done %3i %3i %5i"%(it,iDir,iIter[0]))
#print "total time f=%f"%tIter[0]
TEC, CPhase = Sol.x.reshape((2, na))
TEC -= TEC[0]
CPhase -= CPhase[0]
GThis = np.abs(GOut[it, :, :, iDir, 0, 0]).T * TECToZ(
TEC.reshape((-1, 1)), CPhase.reshape(
(-1, 1)), self.CentralFreqs.reshape((1, -1)))
T.timeit("done")
return GThis.T, TEC, CPhase
def __init__(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
LMS = [l.strip() for l in open(self.MSOutFreq).readlines()]
self.OutFreqDomains = np.zeros((len(LMS) * self.NFreqPerMS, 2),
np.float64)
iFTot = 0
for iMS, MS in enumerate(LMS):
t = table("%s::SPECTRAL_WINDOW" % MS, ack=False)
df = t.getcol("CHAN_WIDTH").flat[0]
fs = t.getcol("CHAN_FREQ").ravel()
f0, f1 = fs[0] - df / 2., fs[-1] + df / 2.
ff = np.linspace(f0, f1, self.NFreqPerMS + 1)
for iF in range(self.NFreqPerMS):
self.OutFreqDomains[iFTot, 0] = ff[iF]
self.OutFreqDomains[iFTot, 1] = ff[iF + 1]
iFTot += 1
NFreqsOut = self.OutFreqDomains.shape[0]
self.CentralFreqs = np.mean(self.OutFreqDomains, axis=1)
log.print("Loading %s" % self.SolsFileIn)
self.DicoFile0 = dict(np.load(self.SolsFileIn))
Dico0 = self.DicoFile0
self.Sols0 = self.DicoFile0["Sols"].view(np.recarray)
DicoOut = {}
DicoOut['ModelName'] = Dico0['ModelName']
DicoOut['StationNames'] = Dico0['StationNames']
DicoOut['BeamTimes'] = Dico0['BeamTimes']
DicoOut['SourceCatSub'] = Dico0['SourceCatSub']
DicoOut['ClusterCat'] = Dico0['ClusterCat']
DicoOut['SkyModel'] = Dico0['SkyModel']
DicoOut['FreqDomains'] = self.OutFreqDomains
self.DicoOut = DicoOut
self.CentralFreqsIn = np.mean(Dico0['FreqDomains'], axis=1)
self.DicoFreqWeights = {}
for iChan in range(self.CentralFreqs.size):
f = self.CentralFreqs[iChan]
i0 = np.where(self.CentralFreqsIn <= f)[0]
i1 = np.where(self.CentralFreqsIn > f)[0]
if i0.size > 0 and i1.size > 0:
i0 = i0[-1]
i1 = i1[0]
f0 = self.CentralFreqsIn[i0]
f1 = self.CentralFreqsIn[i1]
df = np.abs(f0 - f1)
alpha = 1. - (f - f0) / df
c0 = alpha
c1 = 1. - alpha
self.DicoFreqWeights[iChan] = {
"Type": "Dual",
"Coefs": (c0, c1),
"Index": (i0, i1)
}
else:
i0 = np.argmin(np.abs(self.CentralFreqsIn - f))
self.DicoFreqWeights[iChan] = {"Type": "Single", "Index": i0}
# for iChan in range(self.CentralFreqs.size):
# print
# print self.CentralFreqs[iChan]/1e6
# if self.DicoFreqWeights[iChan]["Type"]=="Dual":
# i0,i1=self.DicoFreqWeights[iChan]["Index"]
# print self.CentralFreqsIn[i0]/1e6,self.CentralFreqsIn[i1]/1e6,self.DicoFreqWeights[iChan]["Coefs"]
# else:
# i0=self.DicoFreqWeights[iChan]["Index"]
# print self.CentralFreqsIn[i0]/1e6
nt, _, na, nd, _, _ = self.Sols0.G.shape
SolsOut = np.zeros(
(nt, ),
dtype=[("t0", np.float64), ("t1", np.float64),
("G", np.complex64, (NFreqsOut, na, nd, 2, 2)),
("Stats", np.float32, (NFreqsOut, na, 4))])
SolsOut = SolsOut.view(np.recarray)
SolsOut.t0 = self.Sols0.t0
SolsOut.t1 = self.Sols0.t1
SolsOut.G[..., 0, 0] = 1.
SolsOut.G[..., 1, 1] = 1.
self.SolsOut = SolsOut
self.GOut = NpShared.ToShared("%sGOut" % IdSharedMem,
self.SolsOut.G.copy())
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=self.NCPU, affinity=0)
def Evolve0(self, Gin, Pa, kapa=1.):
done = NpShared.GiveArray("%sSolsArray_done" % self.IdSharedMem)
indDone = np.where(done == 1)[0]
#print kapa
#print type(NpShared.GiveArray("%sSharedCovariance_Q"%self.IdSharedMem))
Q = kapa * NpShared.GiveArray(
"%sSharedCovariance_Q" % self.IdSharedMem)[self.iChanSol,
self.iAnt]
#print indDone.size
#print "mean",np.mean(Q)
##########
# Ptot=Pa+Q
# #nt,_,_,_=Gin.shape
# #print Gin.shape
# g=Gin
# gg=g.ravel()
# #gg+=(np.random.randn(*gg.shape)+1j*np.random.randn(*gg.shape))*np.sqrt(np.diag(Ptot))/np.sqrt(2.)
# # print Pa.shape,Q.shape
# # print np.diag(Pa)
# # print np.diag(Q)
# # print np.diag(Ptot)
# # print
# return Ptot
##############
# return Pa+Q
if indDone.size < 2: return Pa + Q
# #########################
# take scans where no solve has been done into account
G = NpShared.GiveArray("%sSolsArray_G" %
self.IdSharedMem)[indDone][:, self.iChanSol,
self.iAnt, :, 0, 0]
Gm = np.mean(G, axis=-1)
dG = Gm[:-1] - Gm[1:]
done0 = NpShared.GiveArray("%sSolsArray_done" % self.IdSharedMem)
done1 = np.zeros((done0.size, ), int)
done1[1:1 + dG.size] = (dG != 0)
done1[0] = 1
done = (done0 & done1)
indDone = np.where(done == 1)[0]
if indDone.size < 2: return Pa + Q
# #########################
t0 = NpShared.GiveArray("%sSolsArray_t0" % self.IdSharedMem)[indDone]
t1 = NpShared.GiveArray("%sSolsArray_t1" % self.IdSharedMem)[indDone]
tm = NpShared.GiveArray("%sSolsArray_tm" % self.IdSharedMem)[indDone]
G = NpShared.GiveArray("%sSolsArray_G" %
self.IdSharedMem)[indDone][:, self.iChanSol,
self.iAnt, :, :, :]
nt, nd, npolx, npoly = G.shape
#if nt<=self.StepStart: return None
if nt > self.BufferNPoints:
G = G[-self.BufferNPoints::, :, :, :]
tm = tm[-self.BufferNPoints::]
G = G.copy()
nt, _, _, _ = G.shape
NPars = nd * npolx * npoly
G = G.reshape((nt, NPars))
F = np.ones((NPars, ), G.dtype)
PaOut = np.zeros_like(Pa)
tm0 = tm.copy()
tm0 = np.abs(tm - tm[-1])
w = np.exp(-tm0 / self.WeigthScale)
w /= np.sum(w)
w = w[::-1]
for iPar in range(NPars):
#g_t=G[:,iPar][-1]
#ThisG=Gin.ravel()[iPar]
#ratio=1.+(ThisG-g_t)/g_t
g_t = G[:, iPar]
ThisG = Gin.ravel()[iPar]
#ratio=1.+np.std(g_t)
#norm=np.max([np.abs(np.mean(g_t))
#ratio=np.cov(g_t)/Pa[iPar,iPar]
#print np.cov(g_t),Pa[iPar,iPar],ratio
ratio = np.abs(ThisG - np.mean(g_t)) / np.sqrt(Pa[iPar, iPar] +
Q[iPar, iPar])
#ratio=np.abs(g_t[-1]-np.mean(g_t))/np.sqrt(Pa[iPar,iPar])#+Q[iPar,iPar]))
diff = np.sum(w * (ThisG - g_t)) / np.sum(w)
ratio = np.abs(diff) / np.sqrt(Pa[iPar, iPar] + Q[iPar, iPar])
F[iPar] = 1. #ratio#/np.sqrt(2.)
diff = ThisG - g_t[-1] #np.sum(w*(ThisG-g_t))/np.sum(w)
#PaOut[iPar,iPar]=np.abs(diff)**2+Pa[iPar,iPar]+Q[iPar,iPar]
PaOut[iPar,
iPar] = np.abs(diff)**2 + Pa[iPar, iPar] + Q[iPar, iPar]
# Q=np.diag(np.ones((PaOut.shape[0],)))*(self.sigQ**2)
PaOut = F.reshape((NPars, 1)) * Pa * F.reshape((1, NPars)).conj() + Q
#print(F)
#print(Q)
# stop
return PaOut
def Evolve(self, xEst, Pa, CurrentTime):
done = NpShared.GiveArray("%sSolsArray_done" % self.IdSharedMem)
indDone = np.where(done == 1)[0]
Q = NpShared.GiveArray("%sSharedCovariance_Q" %
self.IdSharedMem)[self.iAnt]
t0 = NpShared.GiveArray("%sSolsArray_t0" % self.IdSharedMem)[indDone]
t1 = NpShared.GiveArray("%sSolsArray_t1" % self.IdSharedMem)[indDone]
tm = NpShared.GiveArray("%sSolsArray_tm" % self.IdSharedMem)[indDone]
G = NpShared.GiveArray(
"%sSolsArray_G" % self.IdSharedMem)[indDone][:, self.iAnt, :, :, :]
nt, nd, npol, _ = G.shape
if nt <= self.StepStart: return None, None
if nt > self.BufferNPoints:
G = G[-nt::, :, :, :]
tm = tm[-nt::]
G = G.copy()
tm0 = tm.copy()
tm0 = tm - tm[-1]
ThisTime = CurrentTime - tm[-1]
nt, _, _, _ = G.shape
NPars = nd * npol * npol
G = G.reshape((nt, NPars))
Gout = np.zeros((nd * npol * npol), dtype=G.dtype)
Gout[:] = G[-1]
F = np.ones((NPars, ), G.dtype)
if self.DoEvolve:
if self.WeightType == "exp":
w = np.exp(-tm0 / self.WeigthScale)
w /= np.sum(w)
w = w[::-1]
dx = 1e-6
for iPar in range(NPars):
g_t = G[:, iPar]
g_r = g_t.real.copy()
g_i = g_t.imag.copy()
####
z_r0 = np.polyfit(tm0, g_r, self.order, w=w)
z_i0 = np.polyfit(tm0, g_i, self.order, w=w)
poly_r = np.poly1d(z_r0)
poly_i = np.poly1d(z_i0)
x0_r = poly_r(ThisTime)
x0_i = poly_i(ThisTime)
Gout[iPar] = x0_r + 1j * x0_i
####
g_r[-1] += dx
g_i[-1] += dx
z_r1 = np.polyfit(tm0, g_r, self.order, w=w)
z_i1 = np.polyfit(tm0, g_i, self.order, w=w)
poly_r = np.poly1d(z_r1)
poly_i = np.poly1d(z_i1)
x1_r = poly_r(ThisTime)
x1_i = poly_i(ThisTime)
# dz=((x0_r-x1_r)+1j*(x0_i-x1_i))/dx
xlast = G[-1][iPar]
dz = ((x0_r - xlast.real) + 1j *
(x0_i - xlast.imag)) / np.sqrt(
(Pa[iPar, iPar] + Q[iPar, iPar]))
F[iPar] = dz / np.sqrt(2.)
# if self.iAnt==0:
# xx=np.linspace(tm0.min(),tm0.max(),100)
# pylab.clf()
# pylab.plot(tm0, g_r)
# pylab.plot(xx, poly_r(xx))
# pylab.scatter([ThisTime],[x1_r])
# pylab.draw()
# pylab.show(False)
# pylab.pause(0.1)
# print F
# if self.iAnt==0:
# pylab.clf()
# pylab.imshow(np.diag(F).real,interpolation="nearest")
# pylab.draw()
# pylab.show(False)
# pylab.pause(0.1)
#Pa=P[self.iAnt]
PaOut = np.zeros_like(Pa)
#Q=np.diag(np.ones((PaOut.shape[0],)))*(self.sigQ**2)
PaOut = F.reshape((NPars, 1)) * Pa * F.reshape((1, NPars)).conj() + Q
Gout = Gout.reshape((nd, npol, npol))
print(np.diag(PaOut))
return Gout, PaOut
def run(self):
while not self.kill_received:
try:
Row0, Row1 = self.work_queue.get()
except:
break
D = NpShared.SharedToDico("%sDicoMemChunk" % self.IdSharedMem)
DicoData = {}
DicoData["data"] = D["data"][Row0:Row1]
DicoData["flags"] = D["flags"][Row0:Row1]
DicoData["A0"] = D["A0"][Row0:Row1]
DicoData["A1"] = D["A1"][Row0:Row1]
DicoData["times"] = D["times"][Row0:Row1]
DicoData["uvw"] = D["uvw"][Row0:Row1]
DicoData["freqs"] = D["freqs"]
DicoData["dfreqs"] = D["dfreqs"]
DicoData["freqs_full"] = D["freqs_full"]
DicoData["dfreqs_full"] = D["dfreqs_full"]
# DicoData["UVW_dt"]=D["UVW_dt"]
DicoData["infos"] = D["infos"]
#DicoData["IndRows_All_UVW_dt"]=D["IndRows_All_UVW_dt"]
#DicoData["All_UVW_dt"]=D["All_UVW_dt"]
if self.DoSmearing and "T" in self.DoSmearing:
DicoData["UVW_dt"] = D["UVW_dt"][Row0:Row1]
# DicoData["IndexTimesThisChunk"]=D["IndexTimesThisChunk"][Row0:Row1]
# it0=np.min(DicoData["IndexTimesThisChunk"])
# it1=np.max(DicoData["IndexTimesThisChunk"])+1
# DicoData["UVW_RefAnt"]=D["UVW_RefAnt"][it0:it1,:,:]
if "W" in D.keys():
DicoData["W"] = D["W"][Row0:Row1]
if "resid" in D.keys():
DicoData["resid"] = D["resid"][Row0:Row1]
ApplyTimeJones = NpShared.SharedToDico("%sApplyTimeJones" %
self.IdSharedMem)
#JonesMatrices=ApplyTimeJones["Beam"]
#print ApplyTimeJones["Beam"].flags
ApplyTimeJones["Map_VisToJones_Time"] = ApplyTimeJones[
"Map_VisToJones_Time"][Row0:Row1]
PM = ClassPredict(NCPU=1,
DoSmearing=self.DoSmearing,
BeamAtFacet=self._BeamAtFacet)
#print DicoData.keys()
if self.Mode == "Predict":
PredictData = PM.predictKernelPolCluster(
DicoData, self.SM, ApplyTimeJones=ApplyTimeJones)
PredictArray = NpShared.GiveArray("%sPredictData" %
(self.IdSharedMem))
PredictArray[Row0:Row1] = PredictData[:]
elif self.Mode == "ApplyCal":
PM.ApplyCal(DicoData, ApplyTimeJones, self.iCluster)
elif self.Mode == "DDECovariance":
PM.GiveCovariance(DicoData, ApplyTimeJones, self.SM)
elif self.Mode == "ResidAntCovariance":
PM.GiveResidAntCovariance(DicoData, ApplyTimeJones, self.SM)
self.result_queue.put(True)
def InterpolParallel(self):
Sols0 = self.Sols
nt, nch, na, nd, _, _ = Sols0.G.shape
log.print(" #Times: %i" % nt)
log.print(" #Channels: %i" % nch)
log.print(" #Antennas: %i" % na)
log.print(" #Directions: %i" % nd)
# APP.terminate()
# APP.shutdown()
# Multiprocessing.cleanupShm()
APP.startWorkers()
iJob = 0
# for iAnt in [49]:#range(na):
# for iDir in [0]:#range(nd):
if "TEC" in self.InterpMode:
#APP.runJob("FitThisTEC_%d"%iJob, self.FitThisTEC, args=(208,)); iJob+=1
self.TECArray = NpShared.ToShared(
"%sTECArray" % IdSharedMem, np.zeros((nt, nd, na), np.float32))
self.CPhaseArray = NpShared.ToShared(
"%sCPhaseArray" % IdSharedMem,
np.zeros((nt, nd, na), np.float32))
for it in range(nt):
# for iDir in range(nd):
APP.runJob("FitThisTEC_%d" % iJob,
self.FitThisTEC,
args=(it, )) #,serial=True)
iJob += 1
workers_res = APP.awaitJobResults("FitThisTEC*",
progress="Fit TEC")
if "Amp" in self.InterpMode:
for iAnt in range(na):
for iDir in range(nd):
APP.runJob("FitThisAmp_%d" % iJob,
self.FitThisAmp,
args=(iAnt, iDir)) #,serial=True)
iJob += 1
workers_res = APP.awaitJobResults("FitThisAmp*",
progress="Smooth Amp")
if "PolyAmp" in self.InterpMode:
for iDir in range(nd):
APP.runJob("FitThisPolyAmp_%d" % iJob,
self.FitThisPolyAmp,
args=(iDir, ))
iJob += 1
workers_res = APP.awaitJobResults("FitThisPolyAmp*",
progress="Smooth Amp")
if "Clip" in self.InterpMode:
for iDir in range(nd):
APP.runJob("ClipThisDir_%d" % iJob,
self.ClipThisDir,
args=(iDir, ),
serial=True)
iJob += 1
workers_res = APP.awaitJobResults("ClipThisDir*",
progress="Clip Amp")
#APP.terminate()
APP.shutdown()
Multiprocessing.cleanupShm()
