def Interpol(self):
APP.startWorkers()
if "TEC" in self.DicoFile0["SmoothMode"]:
TECArray = NpShared.ToShared("%sTECArray" % IdSharedMem,
self.DicoFile0["SolsTEC"])
CPhaseArray = NpShared.ToShared("%sCPhaseArray" % IdSharedMem,
self.DicoFile0["SolsCPhase"])
nt, nd, na = TECArray.shape
iJob = 0
for it in range(nt):
APP.runJob("InterpolTECTime_%d" % iJob,
self.InterpolTECTime,
args=(it, )) #,serial=True)
iJob += 1
workers_res = APP.awaitJobResults("InterpolTECTime*",
progress="Interpol TEC")
iJob = 0
for it in range(nt):
APP.runJob("InterpolAmpTime_%d" % iJob,
self.InterpolAmpTime,
args=(it, )) #,serial=True)
iJob += 1
workers_res = APP.awaitJobResults("InterpolAmpTime*",
progress="Interpol Amp")
# APP.terminate()
APP.shutdown()
Multiprocessing.cleanupShm()
def run(self):
while not self.kill_received:
try:
Job = self.work_queue.get()
except:
break
# ModelFacet=NpShared.GiveArray("%sModelFacet.%3.3i"%(self.IdSharedMem,iFacet))
# Grid=self.ClassImToGrid.setModelIm(ModelFacet)
# _=NpShared.ToShared("%sModelGrid.%3.3i"%(self.IdSharedMem,iFacet),Grid)
# self.result_queue.put({"Success":True})
iFacet = Job["iFacet"]
FacetDataCache = Job["FacetDataCache"]
Image = NpShared.GiveArray("%sModelImage" % (self.IdSharedMem))
#Grid,SumFlux=self.ClassImToGrid.GiveGridFader(Image,self.DicoImager,iFacet,NormImage)
#SharedMemName="%sSpheroidal.Facet_%3.3i"%(self.IdSharedMem,iFacet)
SPhe = NpShared.GiveArray(Job["SharedMemNameSphe"])
#print SharedMemName
SpacialWeight = NpShared.GiveArray("%sSpacialWeight.Facet_%3.3i" %
(FacetDataCache, iFacet))
NormImage = NpShared.GiveArray("%sNormImage" % self.IdSharedMem)
Im2Grid = ClassImToGrid(OverS=self.GD["ImagerCF"]["OverS"],
GD=self.GD)
Grid, SumFlux = Im2Grid.GiveModelTessel(Image,
self.DicoImager,
iFacet,
NormImage,
SPhe,
SpacialWeight,
ToGrid=True)
#ModelSharedMemName="%sModelImage.Facet_%3.3i"%(self.IdSharedMem,iFacet)
#NpShared.ToShared(ModelSharedMemName,ModelFacet)
_ = NpShared.ToShared(
"%sModelGrid.%3.3i" % (self.IdSharedMem, iFacet), Grid)
self.result_queue.put({
"Success": True,
"iFacet": iFacet,
"SumFlux": SumFlux
})
def __init__(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
LMS = [l.strip() for l in open(self.MSOutFreq).readlines()]
self.OutFreqDomains = np.zeros((len(LMS) * self.NFreqPerMS, 2),
np.float64)
iFTot = 0
for iMS, MS in enumerate(LMS):
t = table("%s::SPECTRAL_WINDOW" % MS, ack=False)
df = t.getcol("CHAN_WIDTH").flat[0]
fs = t.getcol("CHAN_FREQ").ravel()
f0, f1 = fs[0] - df / 2., fs[-1] + df / 2.
ff = np.linspace(f0, f1, self.NFreqPerMS + 1)
for iF in range(self.NFreqPerMS):
self.OutFreqDomains[iFTot, 0] = ff[iF]
self.OutFreqDomains[iFTot, 1] = ff[iF + 1]
iFTot += 1
NFreqsOut = self.OutFreqDomains.shape[0]
self.CentralFreqs = np.mean(self.OutFreqDomains, axis=1)
log.print("Loading %s" % self.SolsFileIn)
self.DicoFile0 = dict(np.load(self.SolsFileIn))
Dico0 = self.DicoFile0
self.Sols0 = self.DicoFile0["Sols"].view(np.recarray)
DicoOut = {}
DicoOut['ModelName'] = Dico0['ModelName']
DicoOut['StationNames'] = Dico0['StationNames']
DicoOut['BeamTimes'] = Dico0['BeamTimes']
DicoOut['SourceCatSub'] = Dico0['SourceCatSub']
DicoOut['ClusterCat'] = Dico0['ClusterCat']
DicoOut['SkyModel'] = Dico0['SkyModel']
DicoOut['FreqDomains'] = self.OutFreqDomains
self.DicoOut = DicoOut
self.CentralFreqsIn = np.mean(Dico0['FreqDomains'], axis=1)
self.DicoFreqWeights = {}
for iChan in range(self.CentralFreqs.size):
f = self.CentralFreqs[iChan]
i0 = np.where(self.CentralFreqsIn <= f)[0]
i1 = np.where(self.CentralFreqsIn > f)[0]
if i0.size > 0 and i1.size > 0:
i0 = i0[-1]
i1 = i1[0]
f0 = self.CentralFreqsIn[i0]
f1 = self.CentralFreqsIn[i1]
df = np.abs(f0 - f1)
alpha = 1. - (f - f0) / df
c0 = alpha
c1 = 1. - alpha
self.DicoFreqWeights[iChan] = {
"Type": "Dual",
"Coefs": (c0, c1),
"Index": (i0, i1)
}
else:
i0 = np.argmin(np.abs(self.CentralFreqsIn - f))
self.DicoFreqWeights[iChan] = {"Type": "Single", "Index": i0}
# for iChan in range(self.CentralFreqs.size):
# print
# print self.CentralFreqs[iChan]/1e6
# if self.DicoFreqWeights[iChan]["Type"]=="Dual":
# i0,i1=self.DicoFreqWeights[iChan]["Index"]
# print self.CentralFreqsIn[i0]/1e6,self.CentralFreqsIn[i1]/1e6,self.DicoFreqWeights[iChan]["Coefs"]
# else:
# i0=self.DicoFreqWeights[iChan]["Index"]
# print self.CentralFreqsIn[i0]/1e6
nt, _, na, nd, _, _ = self.Sols0.G.shape
SolsOut = np.zeros(
(nt, ),
dtype=[("t0", np.float64), ("t1", np.float64),
("G", np.complex64, (NFreqsOut, na, nd, 2, 2)),
("Stats", np.float32, (NFreqsOut, na, 4))])
SolsOut = SolsOut.view(np.recarray)
SolsOut.t0 = self.Sols0.t0
SolsOut.t1 = self.Sols0.t1
SolsOut.G[..., 0, 0] = 1.
SolsOut.G[..., 1, 1] = 1.
self.SolsOut = SolsOut
self.GOut = NpShared.ToShared("%sGOut" % IdSharedMem,
self.SolsOut.G.copy())
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=self.NCPU, affinity=0)
def InterpolParallel(self):
Sols0 = self.Sols
nt, nch, na, nd, _, _ = Sols0.G.shape
log.print(" #Times: %i" % nt)
log.print(" #Channels: %i" % nch)
log.print(" #Antennas: %i" % na)
log.print(" #Directions: %i" % nd)
# APP.terminate()
# APP.shutdown()
# Multiprocessing.cleanupShm()
APP.startWorkers()
iJob = 0
# for iAnt in [49]:#range(na):
# for iDir in [0]:#range(nd):
if "TEC" in self.InterpMode:
#APP.runJob("FitThisTEC_%d"%iJob, self.FitThisTEC, args=(208,)); iJob+=1
self.TECArray = NpShared.ToShared(
"%sTECArray" % IdSharedMem, np.zeros((nt, nd, na), np.float32))
self.CPhaseArray = NpShared.ToShared(
"%sCPhaseArray" % IdSharedMem,
np.zeros((nt, nd, na), np.float32))
for it in range(nt):
# for iDir in range(nd):
APP.runJob("FitThisTEC_%d" % iJob,
self.FitThisTEC,
args=(it, )) #,serial=True)
iJob += 1
workers_res = APP.awaitJobResults("FitThisTEC*",
progress="Fit TEC")
if "Amp" in self.InterpMode:
for iAnt in range(na):
for iDir in range(nd):
APP.runJob("FitThisAmp_%d" % iJob,
self.FitThisAmp,
args=(iAnt, iDir)) #,serial=True)
iJob += 1
workers_res = APP.awaitJobResults("FitThisAmp*",
progress="Smooth Amp")
if "PolyAmp" in self.InterpMode:
for iDir in range(nd):
APP.runJob("FitThisPolyAmp_%d" % iJob,
self.FitThisPolyAmp,
args=(iDir, ))
iJob += 1
workers_res = APP.awaitJobResults("FitThisPolyAmp*",
progress="Smooth Amp")
if "Clip" in self.InterpMode:
for iDir in range(nd):
APP.runJob("ClipThisDir_%d" % iJob,
self.ClipThisDir,
args=(iDir, ),
serial=True)
iJob += 1
workers_res = APP.awaitJobResults("ClipThisDir*",
progress="Clip Amp")
#APP.terminate()
APP.shutdown()
Multiprocessing.cleanupShm()
def __init__(self,
InSolsName,
OutSolsName,
InterpMode="TEC",
PolMode="Scalar",
Amp_PolyOrder=3,
NCPU=0,
Amp_GaussKernel=(0, 5),
Amp_SmoothType="Poly",
CrossMode=1,
RemoveAmpBias=0,
RemoveMedianAmp=True):
if type(InterpMode) == str:
InterpMode = InterpMode.split(",") #[InterpMode]
self.InSolsName = InSolsName
self.OutSolsName = OutSolsName
self.RemoveMedianAmp = RemoveMedianAmp
log.print("Loading %s" % self.InSolsName)
self.DicoFile = dict(np.load(self.InSolsName, allow_pickle=True))
self.Sols = self.DicoFile["Sols"].view(np.recarray)
if "MaskedSols" in self.DicoFile.keys():
MaskFreq = np.logical_not(
np.all(np.all(np.all(self.DicoFile["MaskedSols"][..., 0, 0],
axis=0),
axis=1),
axis=1))
nt, _, na, nd, _, _ = self.Sols.G.shape
self.DicoFile["FreqDomains"] = self.DicoFile["FreqDomains"][
MaskFreq]
NFreqsOut = np.count_nonzero(MaskFreq)
log.print("There are %i non-zero freq channels" % NFreqsOut)
SolsOut = np.zeros(
(nt, ),
dtype=[("t0", np.float64), ("t1", np.float64),
("G", np.complex64, (NFreqsOut, na, nd, 2, 2)),
("Stats", np.float32, (NFreqsOut, na, 4))])
SolsOut = SolsOut.view(np.recarray)
SolsOut.G = self.Sols.G[:, MaskFreq, ...]
SolsOut.t0 = self.Sols.t0
SolsOut.t1 = self.Sols.t1
self.Sols = self.DicoFile["Sols"] = SolsOut
del (self.DicoFile["MaskedSols"])
#self.Sols=self.Sols[0:10].copy()
self.CrossMode = CrossMode
self.CentralFreqs = np.mean(self.DicoFile["FreqDomains"], axis=1)
self.incrCross = 11
iii = 0
NTEC = 101
NConstPhase = 51
TECGridAmp = 0.1
TECGrid, CPhase = np.mgrid[-TECGridAmp:TECGridAmp:NTEC * 1j,
-np.pi:np.pi:NConstPhase * 1j]
Z = TECToZ(TECGrid.reshape((-1, 1)), CPhase.reshape((-1, 1)),
self.CentralFreqs.reshape((1, -1)))
self.Z = Z
self.TECGrid, self.CPhase = TECGrid, CPhase
self.InterpMode = InterpMode
self.Amp_PolyOrder = Amp_PolyOrder
self.RemoveAmpBias = RemoveAmpBias
if self.RemoveAmpBias:
self.CalcFreqAmpSystematics()
self.Sols.G /= self.G0
self.GOut = NpShared.ToShared("%sGOut" % IdSharedMem,
self.Sols.G.copy())
self.PolMode = PolMode
self.Amp_GaussKernel = Amp_GaussKernel
if len(self.Amp_GaussKernel) != 2:
raise ValueError("GaussKernel should be of size 2")
self.Amp_SmoothType = Amp_SmoothType
if "TEC" in self.InterpMode:
log.print(" Smooth phases using a TEC model")
if self.CrossMode:
log.print(ModColor.Str("Using CrossMode"))
if "Amp" in self.InterpMode:
if Amp_SmoothType == "Poly":
log.print(
" Smooth amplitudes using polynomial model of order %i" %
self.Amp_PolyOrder)
if Amp_SmoothType == "Gauss":
log.print(
" Smooth amplitudes using Gaussian kernel of %s (Time/Freq) bins"
% str(Amp_GaussKernel))
if self.RemoveAmpBias:
self.GOut *= self.G0
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=NCPU, affinity=0)
