def _initIsland_worker(self, DicoOut, iIsland, Island, DicoVariablePSF,
DicoDirty, DicoParm, FacetCache, NCPU):
logger.setSilent([
"ClassImageDeconvMachineMSMF", "ClassPSFServer",
"ClassMultiScaleMachine", "GiveModelMachine",
"ClassModelMachineMSMF"
])
self.InitMachine.Init(DicoVariablePSF,
DicoParm["GridFreqs"],
DicoParm["DegridFreqs"],
facetcache=FacetCache)
self.InitMachine.setDirty(DicoDirty)
#self.InitMachine.DeconvMachine.setNCPU(NCPU)
self.InitMachine.setSSDModelImage(DicoParm["ModelImage"])
#print ":::::::::::::::::::::::",iIsland
try:
SModel, AModel = self.InitMachine.giveModel(Island)
except:
if not self.GD["GAClean"]["ParallelInitHMP"]:
raise
print >> log, traceback.format_exc()
FileOut = "errIsland_%6.6i.npy" % iIsland
print >> log, ModColor.Str(
"...... error on island %i, saving to file %s" %
(iIsland, FileOut))
np.save(FileOut, np.array(Island))
self.InitMachine.Reset()
return
DicoOut["S"] = SModel
DicoOut["Alpha"] = AModel
self.InitMachine.Reset()
def __init__(self, VS):
logger.setSilent(["ClassJones", "ClassLOFARBeam"])
self.VS = VS
self.GD = copy.deepcopy(self.VS.GD)
self.DoCentralNorm = self.GD["Beam"]["CenterNorm"]
self.SmoothBeam = None
self.CheckCache()
if self.CacheValid:
logger.setLoud(["ClassJones", "ClassLOFARBeam"])
return
#self.GD["Beam"]["CenterNorm"]=0
self.ListMS = self.VS.ListMS
self.MS = self.ListMS[0]
rac, decc = self.MS.radec
self.CoordMachine = ModCoord.ClassCoordConv(rac, decc)
self.CalcGrid()
#self.Padding=Padding
#self.SumJJsq=np.zeros((self.npix,self.npix,self.MS.Nchan),np.float64)
#self.SumWsq=np.zeros((1,self.MS.Nchan),np.float64)
self.StackedBeamDict = shared_dict.create("StackedBeamDict")
for iDir in range(self.NDir):
sd = self.StackedBeamDict.addSubdict(iDir)
sd.addSharedArray("SumJJsq", (self.VS.NFreqBands, ), np.float64)
sd.addSharedArray("SumWsq", (self.VS.NFreqBands, ), np.float64)
self.DicoJonesMachine = {}
for iMS, MS in enumerate(self.ListMS):
JonesMachine = ClassJones.ClassJones(self.GD, MS,
self.VS.FacetMachine)
JonesMachine.InitBeamMachine()
self.DicoJonesMachine[iMS] = JonesMachine
logger.setLoud(["ClassJones", "ClassLOFARBeam"])
def giveDicoInitIndiv(self,
ListIslands,
ModelImage,
DicoDirty,
ListDoIsland=None,
Parallel=True):
NCPU = self.NCPU
work_queue = multiprocessing.JoinableQueue()
ListIslands = ListIslands #[300:308]
DoIsland = True
for iIsland in range(len(ListIslands)):
if ListDoIsland is not None:
DoIsland = ListDoIsland[iIsland]
if DoIsland: work_queue.put({"iIsland": iIsland})
result_queue = multiprocessing.JoinableQueue()
NJobs = work_queue.qsize()
workerlist = []
logger.setSilent(SilentModules)
#MyLogger.setLoud(SilentModules)
#MyLogger.setLoud("ClassImageDeconvMachineMSMF")
print >> log, "Launch MORESANE workers"
for ii in range(NCPU):
W = WorkerInitMSMF(work_queue, result_queue, self.GD,
self.DicoVariablePSF, DicoDirty, self.RefFreq,
self.GridFreqs, self.DegridFreqs,
self.MainCache, ModelImage, ListIslands,
self.IdSharedMem)
workerlist.append(W)
if Parallel:
workerlist[ii].start()
timer = ClassTimeIt.ClassTimeIt()
pBAR = ProgressBar(Title=" MORESANing islands ")
#pBAR.disable()
pBAR.render(0, NJobs)
iResult = 0
if not Parallel:
for ii in range(NCPU):
workerlist[ii].run() # just run until all work is completed
self.DicoInitIndiv = {}
while iResult < NJobs:
DicoResult = None
if result_queue.qsize() != 0:
try:
DicoResult = result_queue.get()
except:
pass
if DicoResult == None:
time.sleep(0.5)
continue
if DicoResult["Success"]:
iResult += 1
NDone = iResult
pBAR.render(NDone, NJobs)
iIsland = DicoResult["iIsland"]
NameDico = "%sDicoInitIsland_%5.5i" % (self.IdSharedMem,
iIsland)
Dico = NpShared.SharedToDico(NameDico)
self.DicoInitIndiv[iIsland] = copy.deepcopy(Dico)
NpShared.DelAll(NameDico)
if Parallel:
for ii in range(NCPU):
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()
#MyLogger.setLoud(["pymoresane.main"])
#MyLogger.setLoud(["ClassImageDeconvMachineMSMF","ClassPSFServer","ClassMultiScaleMachine","GiveModelMachine","ClassModelMachineMSMF"])
return self.DicoInitIndiv
def __init__(self,
GD,
DicoVariablePSF,
RefFreq,
GridFreqs,
DegridFreqs,
MainCache=None,
IdSharedMem="",
DoWait=False):
self.DicoVariablePSF = DicoVariablePSF
GD = copy.deepcopy(GD)
self.RefFreq = RefFreq
self.GridFreqs = GridFreqs
self.DegridFreqs = DegridFreqs
self.GD = GD
self.GD["Parallel"]["NCPU"] = 1
#self.GD["HMP"]["Alpha"]=[0,0,1]#-1.,1.,5]
self.GD["HMP"]["Alpha"] = [-1., 1., 5]
self.GD["Deconv"]["Mode"] = "MORESANE"
self.GD["Deconv"]["CycleFactor"] = 0
self.GD["Deconv"]["PeakFactor"] = 0.0
self.GD["Deconv"]["RMSFactor"] = self.GD["GAClean"]["RMSFactorInitHMP"]
self.GD["Deconv"]["Gain"] = self.GD["GAClean"]["GainInitHMP"]
self.GD["Deconv"]["AllowNegative"] = self.GD["GAClean"][
"AllowNegativeInitHMP"]
self.GD["Deconv"]["MaxMinorIter"] = self.GD["GAClean"][
"MaxMinorIterInitHMP"]
logger.setSilent(SilentModules)
self.GD["HMP"]["Scales"] = self.GD["GAClean"]["ScalesInitHMP"]
self.GD["HMP"]["Ratios"] = []
#self.GD["MultiScale"]["Ratios"]=[]
self.GD["HMP"]["NTheta"] = 4
self.GD["HMP"]["SolverMode"] = "NNLS"
#self.GD["MultiScale"]["SolverMode"]="PI"
self.NFreqBands = len(DicoVariablePSF["freqs"])
MinorCycleConfig = dict(self.GD["Deconv"])
MinorCycleConfig["NCPU"] = self.GD["Parallel"]["NCPU"]
MinorCycleConfig["NFreqBands"] = self.NFreqBands
MinorCycleConfig["GD"] = self.GD
MinorCycleConfig["GridFreqs"] = self.GridFreqs
MinorCycleConfig["DegridFreqs"] = self.DegridFreqs
#MinorCycleConfig["RefFreq"] = self.RefFreq
ModConstructor = ClassModModelMachine(self.GD)
ModelMachine = ModConstructor.GiveMM(Mode="MORESANE")
ModelMachine.setRefFreq(self.RefFreq)
MinorCycleConfig["ModelMachine"] = ModelMachine
#MinorCycleConfig["CleanMaskImage"]=None
self.MinorCycleConfig = MinorCycleConfig
self.DeconvMachine = ClassImageDeconvMachineMoresane.ClassImageDeconvMachine(
MainCache=MainCache,
ParallelMode=False,
RefFreq=self.RefFreq,
CacheFileName="HMP_Init",
IdSharedMem=IdSharedMem,
**self.MinorCycleConfig)
self.GD["Mask"]["Auto"] = False
self.GD["Mask"]["External"] = None
self.MaskMachine = ClassMaskMachine.ClassMaskMachine(self.GD)
self.DeconvMachine.setMaskMachine(self.MaskMachine)
self.Margin = 50
#print "Start 3"
self.DeconvMachine.Init(PSFVar=self.DicoVariablePSF,
PSFAve=self.DicoVariablePSF["PSFSideLobes"],
GridFreqs=self.GridFreqs,
DegridFreqs=self.DegridFreqs,
DoWait=DoWait,
RefFreq=self.RefFreq)
if DoWait:
print "IINit3"
time.sleep(10)
print "Start 4"
self.DeconvMachine.Update(self.DicoDirty, DoSetMask=False)
if DoWait:
print "IINit4"
time.sleep(10)
def StackBeam(self, ThisMSData, iDir):
self.StackedBeamDict.reload()
logger.setSilent("ClassJones")
Dt = self.GD["Beam"]["DtBeamMin"] * 60.
JonesMachine = self.DicoJonesMachine[ThisMSData["iMS"]]
RAs, DECs = self.radec
times = ThisMSData["times"]
A0 = ThisMSData["A0"]
A1 = ThisMSData["A1"]
flags = ThisMSData["flags"]
W = ThisMSData["Weights"]
ChanToFreqBand = ThisMSData["ChanMapping"]
beam_times = np.array(
JonesMachine.BeamMachine.getBeamSampleTimes(times, quiet=True))
T2 = ClassTimeIt.ClassTimeIt()
T2.disable()
CurrentBeamITime = -1
# #print " Estimate beam in %i directions"%(RAs.size)
# MS=self.ListMS[ThisMSData["iMS"]]
# JonesMachine=ClassJones.ClassJones(self.GD,MS,self.VS.FacetMachine)
# JonesMachine.InitBeamMachine()
DicoBeam = JonesMachine.EstimateBeam(beam_times,
RAs[iDir:iDir + 1],
DECs[iDir:iDir + 1],
progressBar=False,
quiet=True)
T2.timeit("GetBeam 1")
#DicoBeam=JonesMachine.EstimateBeam(beam_times, RAs[0:10], DECs[0:10],progressBar=False)
#T2.timeit("GetBeam 10")
#print DicoBeam["Jones"].shape
NTRange = DicoBeam["t0"].size
#pBAR= ProgressBar(Title=" Mean Beam")
#pBAR.render(0, '%4i/%i' % (0,NTRange))
T = ClassTimeIt.ClassTimeIt("Stacking")
T.disable()
# DicoBeam["Jones"].shape = nt, nd, na, nch, _, _
for iTRange in range(DicoBeam["t0"].size):
t0 = DicoBeam["t0"][iTRange]
t1 = DicoBeam["t1"][iTRange]
J = np.abs(DicoBeam["Jones"][iTRange])
ind = np.where((times >= t0) & (times < t1))[0]
T.timeit("0")
A0s = A0[ind]
A1s = A1[ind]
fs = flags[ind]
Ws = W[ind]
MSnchan = Ws.shape[1]
T.timeit("1")
nd, na, nch, _, _ = J.shape
# ######################
# This call is slow
J0 = J[:, A0s, :, :, :]
J1 = J[:, A1s, :, :, :]
T.timeit("2")
# ######################
# J0=np.zeros((nd,A0s.size,nch,2,2),dtype=J.dtype)
# #T.timeit("1a")
# J0List=[J[:,A0s[i],:,:,:] for i in range(A0s.size)]
# #T.timeit("1b")
# J1=np.zeros((nd,A0s.size,nch,2,2),dtype=J.dtype)
# #T.timeit("1c")
# J1List=[J[:,A1s[i],:,:,:] for i in range(A0s.size)]
# #T.timeit("1d")
# for i in range(A0s.size):
# J0[:,i,:,:,:]=J0List[i]
# J1[:,i,:,:,:]=J1List[i]
# T.timeit("2b")
JJ = (J0[:, :, :, 0, 0] * J1[:, :, :, 0, 0] +
J0[:, :, :, 1, 1] * J1[:, :, :, 1, 1]) / 2.
T.timeit("3")
WW = Ws**2
T.timeit("4")
WW = WW.reshape((1, ind.size, MSnchan))
T.timeit("5")
JJsq = WW * JJ**2
T.timeit("6")
SumWsqThisRange = np.sum(JJsq, axis=1)
T.timeit("7")
#self.SumJJsq+=SumWsqThisRange.reshape((self.npix,self.npix,self.MS.Nchan))
#T.timeit("8")
SumWsq = np.sum(WW, axis=1)
#self.SumWsq+=SumWsq
for iBand in range(self.VS.NFreqBands):
indFreqBand, = np.where(ChanToFreqBand == iBand)
if indFreqBand.size == 0: continue
self.StackedBeamDict[iDir]["SumJJsq"][iBand] += np.sum(
SumWsqThisRange.reshape((MSnchan, ))[indFreqBand])
self.StackedBeamDict[iDir]["SumWsq"][iBand] += np.sum(
SumWsq.reshape((MSnchan, ))[indFreqBand])
#print SumWsq,self.SumWsq,self.SumJJsq.shape,J0.shape
T.timeit("9")
#NDone = iTRange+1
#intPercent = int(100 * NDone / float(NTRange))
#pBAR.render(intPercent, '%4i/%i' % (NDone, NTRange))
T2.timeit("Stack")
logger.setLoud("ClassJones")
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
from DDFacet.Other import logger
log = logger.getLogger("ClassFitAmp")
import killMS.Array.ModLinAlg
from DDFacet.Other import ClassTimeIt
#from skimage.restoration import (denoise_tv_chambolle, denoise_bilateral,
# denoise_wavelet, estimate_sigma)
logger.setSilent(["ClassFitAmp"])
#from tvd import TotalVariationDenoising
def Dot(*args):
P = 1.
for M in args:
#P=np.dot(np.complex128(P),np.complex128(M))
P = np.dot(P, M)
return P
# iDir=14; S=np.load("L229509_merged.npz"); G=S["Sols"]["G"][:,:,:,iDir,0,0]; f=S["FreqDomains"].mean(axis=1)
def Norm(G, iRef=0):
nf, na = G.shape
for iFreq in range(nf):
g0 = G[iFreq, iRef]
G[iFreq] *= g0.conj() / np.abs(g0)
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
from DDFacet.Other import logger
log = logger.getLogger("ClassFitTEC")
import killMS.Array.ModLinAlg
K = 8.4479745e9
import scipy.sparse
from DDFacet.Other import ClassTimeIt
logger.setSilent(["ClassFitTEC"])
def TECToPhase(TEC, freq):
phase = K * TEC * (1. / freq)
return phase
def TECToZ(TEC, ConstPhase, freq):
if ConstPhase is None: ConstPhase = 0
return np.exp(1j * (TECToPhase(TEC, freq) + ConstPhase))
def Dot(*args):
P = 1.
for M in args:
#P=np.dot(np.complex128(P),np.complex128(M))
P = np.dot(P, M)
return P
import time
from DDFacet.Other import logger
from DDFacet.Other import ModColor
log=logger.getLogger("ClassImageDeconvMachine")
from DDFacet.Other import ClassTimeIt
from DDFacet.Imager.ClassPSFServer import ClassPSFServer
from DDFacet.Other.progressbar import ProgressBar
from DDFacet.Imager import ClassGainMachine
from SkyModel.PSourceExtract import ClassIncreaseIsland
from DDFacet.Imager.SSD.GA.ClassEvolveGA import ClassEvolveGA
from DDFacet.Imager.SSD.MCMC.ClassMetropolis import ClassMetropolis
from DDFacet.Array import NpParallel
import ClassIslandDistanceMachine
from DDFacet.Array import shared_dict
logger.setSilent("ClassArrayMethodSSD")
logger.setSilent("ClassIsland")
class ClassImageDeconvMachine():
def __init__(self,Gain=0.3,
MaxMinorIter=100,NCPU=6,
CycleFactor=2.5,FluxThreshold=None,RMSFactor=3,PeakFactor=0,
GD=None,SearchMaxAbs=1,IdSharedMem=None,
ModelMachine=None,
NFreqBands=1,
RefFreq=None,
MainCache=None,
**kw # absorb any unknown keywords arguments into this
):
#self.im=CasaImage
