def SetPSF(self,DicoVariablePSF):
self.PSFServer=ClassPSFServer(self.GD)
DicoVariablePSF=shared_dict.attach(DicoVariablePSF.path)#["CubeVariablePSF"]
self.PSFServer.setDicoVariablePSF(DicoVariablePSF)
self.PSFServer.setRefFreq(self.ModelMachine.RefFreq)
self.DicoVariablePSF=DicoVariablePSF
self.setFreqs(self.PSFServer.DicoMappingDesc)
def giveWeigthParallel(self):
self.DATA = shared_dict.attach(self.DictName)
self.na = self.DATA["na"]
for A0 in range(0, self.na):
for A1 in range(A0 + 1, self.na):
APP.runJob("giveWeigthChunk:%d:%d" % (A0, A1),
self.giveWeigthChunk,
args=(A0, A1,
self.DATA.readwrite())) #,serial=True)
APP.awaitJobResults("giveWeigthChunk:*", progress="CalcWeight")
def giveWeigthParallel(self):
self.DATA = shared_dict.attach(self.DictName)
nrow, nch, npol = self.DATA["data"].shape
self.na = self.DATA["na"]
ntu = self.DATA["times_unique"].size
self.DATA["Wa"] = np.zeros((self.na, ntu), np.float32)
for A0 in range(self.na):
APP.runJob("giveWeigthAnt:%d" % (A0),
self.giveWeigthAnt,
args=(A0, )) #,serial=True)
APP.awaitJobResults("giveWeigthAnt:*", progress="CalcWeight")
def GiveGM(self, iFacet, SM):
"""
Factory: Initializes a gridding machine for this facet
Args:
iFacet: index of facet
Returns:
grid machine instance
"""
# GridMachine=ClassDDEGridMachine.ClassDDEGridMachine(SM.GD,#RaDec=self.DicoImager[iFacet]["RaDec"],
# SM.DicoImager[iFacet]["DicoConfigGM"]["ChanFreq"],
# SM.DicoImager[iFacet]["DicoConfigGM"]["NPix"],
# lmShift=SM.DicoImager[iFacet]["lmShift"],
# IdSharedMem=IdSharedMem,
# IdSharedMemData=IdSharedMemData,
# FacetDataCache=FacetDataCache,
# ChunkDataCache=ChunkDataCache,
# IDFacet=SM.DicoImager[iFacet]["IDFacet"],
# SpheNorm=False)
# #,
# #NFreqBands=self.VS.NFreqBands,
# #DataCorrelationFormat=self.VS.StokesConverter.AvailableCorrelationProductsIds(),
# #ExpectedOutputStokes=self.VS.StokesConverter.RequiredStokesProductsIds(),
# #ListSemaphores=self.ListSemaphores)
SpheNorm = False
FacetInfo = SM.DicoImager[iFacet]
IDFacet = FacetInfo["IDFacet"]
cf_dict = shared_dict.attach(SM.Path["cf_dict_path"])[IDFacet]
#print iFacet,IDFacet
GridMachine = ClassDDEGridMachine.ClassDDEGridMachine(
SM.GD,
FacetInfo["DicoConfigGM"]["ChanFreq"],
FacetInfo["DicoConfigGM"]["NPix"],
lmShift=FacetInfo["lmShift"],
IDFacet=IDFacet,
SpheNorm=SpheNorm,
NFreqBands=SM.NFreqBands,
DataCorrelationFormat=SM.AvailableCorrelationProductsIds,
ExpectedOutputStokes=SM.RequiredStokesProductsIds,
cf_dict=cf_dict)
return GridMachine
def giveWeigthChunk(self, A0, A1, DATA):
self.DATA = shared_dict.attach(self.DictName)
self.DATA.reload()
dA0 = self.DATA["A0"]
dA1 = self.DATA["A1"]
f = self.DATA["flag"]
d = self.DATA["data"]
C0 = (A0 == dA0) & (A1 == dA1)
C1 = (A1 == dA1) & (A0 == dA0)
ind = np.where((C0 | C1))[0]
# C0=(A0==dA0)|(A0==dA1)
# ind=np.where((C0))[0]
if ind.size == 0: return
ds = d[ind]
fs = f[ind]
nt, nch, _ = ds.shape
#nb=100
nb = self.TBinBox
for it in range(nb / 2, nt - nb / 2):
#print it,"/",nt
i0 = np.max([it - nb / 2, 0])
i1 = np.min([it + nb / 2, nt])
nbs = i1 - i0
M = np.zeros((nbs, nbs), np.complex128)
for ic0, it0 in enumerate(range(i0, i1)):
d0 = ds[it0, :, 0::3].ravel()
f0 = fs[it0, :, 0::3].ravel()
df = 1 - f0
nP = np.count_nonzero(df)
if nP == 0: continue
c = np.sum(d0 * d0.conj()) / nP
M[ic0, ic0] = c
# for ic1,it1 in enumerate(range(i0,i1)):
# d0=ds[it0,:,0::3].ravel()
# d1=ds[it1,:,0::3].ravel()
# f0=fs[it0,:,0::3].ravel()
# f1=fs[it1,:,0::3].ravel()
# df=1-(f0|f1)
# nP=np.count_nonzero(df)
# if nP==0: continue
# c=np.sum(d0*d1.conj())/nP
# M[ic0,ic1]=c
# M[ic1,ic0]=c.conj()
indNZ = (np.sum(M, axis=0) != 0)
if np.count_nonzero(indNZ) == 0: continue
M = np.diag(np.diag(M) + 1.)
# # #####################
# M=np.sum(np.abs(ds[:,:])**2,axis=1)
# M=np.diag(M)
# indNZ=(np.sum(M,axis=0)!=0)
# #####################
Minv = Array.ModLinAlg.invSVD(M[indNZ][:, indNZ])
Wc = np.sum(Minv, axis=0)
Wc = Wc.reshape((-1, 1)) * np.ones((1, nch))
W = np.zeros((indNZ.size, nch), np.float64)
W[indNZ, :] = np.abs(Wc[:, :])
#####################
# W[indNZ,:]/=np.sum(W[indNZ,:])
#####################
O = np.zeros_like(W)
O[indNZ, :] = 1
self.DATA["W"][ind[i0:i1], :] += W[:, :]
self.DATA["N"][ind[i0:i1], :] += O[:, :]
def run(self):
while not self.kill_received and self.CondContinue():
#gc.enable()
try:
iIsland, FacetID, JonesNorm, PixVariance, shdict_path = self.work_queue.get(
True, 2)
except Exception, e:
#print "Exception worker: %s"%str(e)
break
# iIsland=DicoOrder["iIsland"]
# FacetID=DicoOrder["FacetID"]
# JonesNorm=DicoOrder["JonesNorm"]
island_dict = shared_dict.attach(shdict_path)
ThisPixList = island_dict["Island"].tolist()
IslandBestIndiv = island_dict["BestIndiv"]
PSF = self.CubeVariablePSF[FacetID]
NGen = self.GD["GAClean"]["NMaxGen"]
NIndiv = self.GD["GAClean"]["NSourceKin"]
ListPixParms = ThisPixList
ListPixData = ThisPixList
dx = self.GD["SSDClean"]["NEnlargeData"]
if dx > 0:
IncreaseIslandMachine = ClassIncreaseIsland.ClassIncreaseIsland(
)
ListPixData = IncreaseIslandMachine.IncreaseIsland(ListPixData,
dx=dx)
# ################################
# DicoSave={"Dirty":self._Dirty,
# "PSF":PSF,
# "FreqsInfo":self.FreqsInfo,
# #"DicoMappingDesc":self.PSFServer.DicoMappingDesc,
# "ListPixData":ListPixData,
# "ListPixParms":ListPixParms,
# "IslandBestIndiv":IslandBestIndiv,
# "GD":self.GD,
# "FacetID":FacetID,
# "iIsland":iIsland,"IdSharedMem":self.IdSharedMem}
# print "saving"
# MyPickle.Save(DicoSave, "SaveTest")
# print "saving ok"
# ################################
if self.DeconvMode == "GAClean":
CEv = ClassEvolveGA(
self._Dirty,
PSF,
self.FreqsInfo,
ListPixParms=ListPixParms,
ListPixData=ListPixData,
iFacet=FacetID,
PixVariance=PixVariance,
IslandBestIndiv=IslandBestIndiv, #*np.sqrt(JonesNorm),
GD=self.GD,
iIsland=iIsland,
island_dict=island_dict,
ParallelFitness=self.ParallelPerIsland,
ListInitIslands=self.ListInitIslands)
Model = CEv.main(NGen=NGen, NIndiv=NIndiv, DoPlot=False)
island_dict["Model"] = np.array(Model)
del (CEv)
self.result_queue.put({
"Success": True,
"iIsland": iIsland,
"HasError": False
})
elif self.DeconvMode == "MetroClean":
CEv = ClassMetropolis(
self._Dirty,
PSF,
self.FreqsInfo,
ListPixParms=ListPixParms,
ListPixData=ListPixData,
iFacet=FacetID,
PixVariance=PixVariance,
IslandBestIndiv=IslandBestIndiv, #*np.sqrt(JonesNorm),
GD=self.GD,
iIsland=iIsland,
island_dict=island_dict,
ParallelFitness=self.ParallelPerIsland,
NChains=self.NChains)
Model, sModel = CEv.main(
NSteps=self.GD["MetroClean"]["MetroNIter"])
island_dict["Model"] = np.array(Model)
island_dict["sModel"] = np.array(sModel)
del (CEv)
self.result_queue.put({
"Success": True,
"iIsland": iIsland,
"HasError": True
})
def giveWeigthAnt(self, A0):
self.DATA = shared_dict.attach(self.DictName)
self.DATA.reload()
dA0 = self.DATA["A0"]
dA1 = self.DATA["A1"]
f = self.DATA["flag"]
d = self.DATA["data"]
times = self.DATA["times"]
ntu = self.DATA["times_unique"].size
indA = np.where((dA0 == A0) | (dA1 == A0))[0]
d = d[indA]
f = f[indA]
dA0 = dA0[indA]
dA1 = dA1[indA]
nrow, nch, npol = d.shape
nbl = nrow / ntu
d = d.reshape((ntu, nbl, nch, npol))
f = f.reshape((ntu, nbl, nch, npol))
dA0 = dA0.reshape((ntu, nbl))
dA1 = dA1.reshape((ntu, nbl))
n = 1 - f
DT = 3
Ntu = ntu / DT
M = np.zeros((Ntu, Ntu), np.complex128)
N = np.zeros((Ntu, Ntu), np.float32)
ds0 = d[0:DT * Ntu, :, :, :].reshape((Ntu, DT, nbl * nch * npol))
fs0 = n[0:DT * Ntu, :, :, :].reshape((Ntu, DT, nbl * nch * npol))
ds0 = np.sum(ds0, axis=1)
fs0 = np.sum(fs0, axis=1)
M[:, :] = np.dot(ds0, ds0.T.conj())
N[:, :] = np.dot(fs0, fs0.T)
# for it0 in range(ntu):
# print it0/float(ntu)
# ds0=d[it0,:,:,:].reshape((1,nbl*nch*npol))
# fs0=n[it0,:,:,:].reshape((1,nbl*nch*npol))
# ds1=d.reshape((ntu,nbl*nch*npol))
# fs1=n.reshape((ntu,nbl*nch*npol))
# M[it0,:]=np.dot(ds0,ds1.T.conj())
# N[it0,:]=np.dot(fs0,fs1.T)
# # for it1 in range(it0,ntu):
# # ds1=d[it1,:,:,:].ravel()
# # pp=ds0*ds1.conj()
# # n=np.count_nonzero(pp)
# # spp=np.sum(pp)
# # M[it0,it1]=spp
# # N[it0,it1]=n
# # M[it1,it0]=spp.conj()
# # N[it1,it0]=n
#print "ok %i"%A0
# nout=ntu/10
# M0=scipy.misc.imresize(np.abs(M), (nout,nout))
# N=scipy.misc.imresize(N, (nout,nout))
# N[N==0]=1
# M/=N
indNZ = (np.sum(N, axis=0) != 0)
if np.count_nonzero(indNZ) == 0:
return
Ms = M[indNZ][:, indNZ]
Ns = N[indNZ][:, indNZ]
Ms[Ns == 0] = np.mean(Ms)
Ns[Ns == 0] = 1.
Ms /= Ns
Minv = killMS.Array.ModLinAlg.invSVD(Ms)
Wc = np.sum(Minv, axis=0)
W = np.zeros((indNZ.size, ), np.float64)
W[indNZ] = np.abs(Wc[:])
W /= np.sum(W)
for it in range(DT):
self.DATA["Wa"][A0, it:DT * Ntu:DT] = W[:]