def _initIsland_worker(self, DicoOut, iIsland, Island, DicoVariablePSF,
DicoDirty, DicoParm, FacetCache, NCPU):
MyLogger.setSilent([
"ClassImageDeconvMachineMSMF", "ClassPSFServer",
"ClassMultiScaleMachine", "GiveModelMachine",
"ClassModelMachineMSMF"
])
self.InitMachine.Init(DicoVariablePSF,
DicoParm["GridFreqs"],
DicoParm["DegridFreqs"],
facetcache=FacetCache)
self.InitMachine.setDirty(DicoDirty)
#self.InitMachine.DeconvMachine.setNCPU(NCPU)
self.InitMachine.setSSDModelImage(DicoParm["ModelImage"])
try:
SModel, AModel = self.InitMachine.giveModel(Island)
except:
print >> log, traceback.format_exc()
FileOut = "errIsland_%6.6i.npy" % iIsland
print >> log, ModColor.Str(
"...... error on island %i, saving to file %s" %
(iIsland, FileOut))
np.save(FileOut, np.array(Island))
self.InitMachine.Reset()
return
DicoOut["S"] = SModel
DicoOut["Alpha"] = AModel
self.InitMachine.Reset()
def __init__(self, VS):
MyLogger.setSilent(["ClassJones", "ClassLOFARBeam"])
self.VS = VS
self.GD = copy.deepcopy(self.VS.GD)
self.DoCentralNorm = self.GD["Beam"]["CenterNorm"]
self.SmoothBeam = None
self.CheckCache()
if self.CacheValid:
MyLogger.setLoud(["ClassJones", "ClassLOFARBeam"])
return
#self.GD["Beam"]["CenterNorm"]=0
self.ListMS = self.VS.ListMS
self.MS = self.ListMS[0]
rac, decc = self.MS.radec
self.CoordMachine = ModCoord.ClassCoordConv(rac, decc)
self.CalcGrid()
#self.Padding=Padding
#self.SumJJsq=np.zeros((self.npix,self.npix,self.MS.Nchan),np.float64)
#self.SumWsq=np.zeros((1,self.MS.Nchan),np.float64)
self.StackedBeamDict = shared_dict.create("StackedBeamDict")
for iDir in range(self.NDir):
sd = self.StackedBeamDict.addSubdict(iDir)
sd.addSharedArray("SumJJsq", (self.VS.NFreqBands, ), np.float64)
sd.addSharedArray("SumWsq", (self.VS.NFreqBands, ), np.float64)
self.DicoJonesMachine = {}
for iMS, MS in enumerate(self.ListMS):
JonesMachine = ClassJones.ClassJones(self.GD, MS,
self.VS.FacetMachine)
JonesMachine.InitBeamMachine()
self.DicoJonesMachine[iMS] = JonesMachine
MyLogger.setLoud(["ClassJones", "ClassLOFARBeam"])
import time
from DDFacet.Other import MyLogger
from DDFacet.Other import ModColor
log = MyLogger.getLogger("ClassImageDeconvMachine")
from DDFacet.Other import ClassTimeIt
from DDFacet.Imager.ClassPSFServer import ClassPSFServer
from DDFacet.Other.progressbar import ProgressBar
from DDFacet.Imager import ClassGainMachine
from SkyModel.PSourceExtract import ClassIncreaseIsland
from DDFacet.Imager.SSD.GA.ClassEvolveGA import ClassEvolveGA
from DDFacet.Imager.SSD.MCMC.ClassMetropolis import ClassMetropolis
from DDFacet.Array import NpParallel
import ClassIslandDistanceMachine
from DDFacet.Array import shared_dict
MyLogger.setSilent("ClassArrayMethodSSD")
MyLogger.setSilent("ClassIsland")
class ClassImageDeconvMachine():
def __init__(
self,
Gain=0.3,
MaxMinorIter=100,
NCPU=6,
CycleFactor=2.5,
FluxThreshold=None,
RMSFactor=3,
PeakFactor=0,
GD=None,
SearchMaxAbs=1,
from DDFacet.Other import reformat
import os
from DDFacet.ToolsDir.ModToolBox import EstimateNpix
from DDFacet.Other import ModColor
import tables
from DDFacet.Imager.ClassImToGrid import ClassImToGrid
from matplotlib.path import Path
from SkyModel.Sky import ModVoronoiToReg
import Polygon
from DDFacet.ToolsDir import rad2hmsdms
from DDFacet.Other.ClassTimeIt import ClassTimeIt
from DDFacet.Other import MyLogger
log = MyLogger.getLogger("ClassFacetMachineTessel")
MyLogger.setSilent("MyLogger")
#from DDFacet.cbuild.Gridder import _pyGridderSmearPols as _pyGridderSmear
class ClassFacetMachineTessel(ClassFacetMachine.ClassFacetMachine):
"""
This class extends the basic infrastructure set out in ClassFacetMachine to
split the sky into a Voronoi tesselated sky. The projection, unprojection and resampling logic
remains the same.
"""
def __init__(self, *args, **kwargs):
ClassFacetMachine.ClassFacetMachine.__init__(self, *args, **kwargs)
def setFacetsLocs(self):
NFacets = self.NFacets
Npix = self.GD["Image"]["NPix"]
def StackBeam(self, ThisMSData, iDir):
self.StackedBeamDict.reload()
MyLogger.setSilent("ClassJones")
Dt = self.GD["Beam"]["DtBeamMin"] * 60.
JonesMachine = self.DicoJonesMachine[ThisMSData["iMS"]]
RAs, DECs = self.radec
times = ThisMSData["times"]
A0 = ThisMSData["A0"]
A1 = ThisMSData["A1"]
flags = ThisMSData["flags"]
W = ThisMSData["Weights"]
ChanToFreqBand = ThisMSData["ChanMapping"]
beam_times = np.array(
JonesMachine.BeamMachine.getBeamSampleTimes(times, quiet=True))
T2 = ClassTimeIt.ClassTimeIt()
T2.disable()
CurrentBeamITime = -1
# #print " Estimate beam in %i directions"%(RAs.size)
# MS=self.ListMS[ThisMSData["iMS"]]
# JonesMachine=ClassJones.ClassJones(self.GD,MS,self.VS.FacetMachine)
# JonesMachine.InitBeamMachine()
DicoBeam = JonesMachine.EstimateBeam(beam_times,
RAs[iDir:iDir + 1],
DECs[iDir:iDir + 1],
progressBar=False,
quiet=True)
T2.timeit("GetBeam 1")
#DicoBeam=JonesMachine.EstimateBeam(beam_times, RAs[0:10], DECs[0:10],progressBar=False)
#T2.timeit("GetBeam 10")
#print DicoBeam["Jones"].shape
NTRange = DicoBeam["t0"].size
#pBAR= ProgressBar(Title=" Mean Beam")
#pBAR.render(0, '%4i/%i' % (0,NTRange))
T = ClassTimeIt.ClassTimeIt("Stacking")
T.disable()
# DicoBeam["Jones"].shape = nt, nd, na, nch, _, _
for iTRange in range(DicoBeam["t0"].size):
t0 = DicoBeam["t0"][iTRange]
t1 = DicoBeam["t1"][iTRange]
J = np.abs(DicoBeam["Jones"][iTRange])
ind = np.where((times >= t0) & (times < t1))[0]
T.timeit("0")
A0s = A0[ind]
A1s = A1[ind]
fs = flags[ind]
Ws = W[ind]
MSnchan = Ws.shape[1]
T.timeit("1")
nd, na, nch, _, _ = J.shape
# ######################
# This call is slow
J0 = J[:, A0s, :, :, :]
J1 = J[:, A1s, :, :, :]
T.timeit("2")
# ######################
# J0=np.zeros((nd,A0s.size,nch,2,2),dtype=J.dtype)
# #T.timeit("1a")
# J0List=[J[:,A0s[i],:,:,:] for i in range(A0s.size)]
# #T.timeit("1b")
# J1=np.zeros((nd,A0s.size,nch,2,2),dtype=J.dtype)
# #T.timeit("1c")
# J1List=[J[:,A1s[i],:,:,:] for i in range(A0s.size)]
# #T.timeit("1d")
# for i in range(A0s.size):
# J0[:,i,:,:,:]=J0List[i]
# J1[:,i,:,:,:]=J1List[i]
# T.timeit("2b")
JJ = (J0[:, :, :, 0, 0] * J1[:, :, :, 0, 0] +
J0[:, :, :, 1, 1] * J1[:, :, :, 1, 1]) / 2.
T.timeit("3")
WW = Ws**2
T.timeit("4")
WW = WW.reshape((1, ind.size, MSnchan))
T.timeit("5")
JJsq = WW * JJ**2
T.timeit("6")
SumWsqThisRange = np.sum(JJsq, axis=1)
T.timeit("7")
#self.SumJJsq+=SumWsqThisRange.reshape((self.npix,self.npix,self.MS.Nchan))
#T.timeit("8")
SumWsq = np.sum(WW, axis=1)
#self.SumWsq+=SumWsq
for iBand in range(self.VS.NFreqBands):
indFreqBand, = np.where(ChanToFreqBand == iBand)
if indFreqBand.size == 0: continue
self.StackedBeamDict[iDir]["SumJJsq"][iBand] += np.sum(
SumWsqThisRange.reshape((MSnchan, ))[indFreqBand])
self.StackedBeamDict[iDir]["SumWsq"][iBand] += np.sum(
SumWsq.reshape((MSnchan, ))[indFreqBand])
#print SumWsq,self.SumWsq,self.SumJJsq.shape,J0.shape
T.timeit("9")
#NDone = iTRange+1
#intPercent = int(100 * NDone / float(NTRange))
#pBAR.render(intPercent, '%4i/%i' % (NDone, NTRange))
T2.timeit("Stack")
MyLogger.setLoud("ClassJones")
def giveDicoInitIndiv(self,ListIslands,ModelImage,DicoDirty,ListDoIsland=None,Parallel=True):
NCPU=self.NCPU
work_queue = multiprocessing.JoinableQueue()
ListIslands=ListIslands#[300:308]
DoIsland=True
for iIsland in range(len(ListIslands)):
if ListDoIsland is not None:
DoIsland=ListDoIsland[iIsland]
if DoIsland: work_queue.put({"iIsland":iIsland})
result_queue=multiprocessing.JoinableQueue()
NJobs=work_queue.qsize()
workerlist=[]
MyLogger.setSilent(SilentModules)
#MyLogger.setLoud(SilentModules)
#MyLogger.setLoud("ClassImageDeconvMachineMSMF")
print>>log,"Launch MORESANE workers"
for ii in range(NCPU):
W = WorkerInitMSMF(work_queue,
result_queue,
self.GD,
self.DicoVariablePSF,
DicoDirty,
self.RefFreq,
self.GridFreqs,
self.DegridFreqs,
self.MainCache,
ModelImage,
ListIslands,
self.IdSharedMem)
workerlist.append(W)
if Parallel:
workerlist[ii].start()
timer = ClassTimeIt.ClassTimeIt()
pBAR = ProgressBar(Title=" MORESANing islands ")
#pBAR.disable()
pBAR.render(0, NJobs)
iResult = 0
if not Parallel:
for ii in range(NCPU):
workerlist[ii].run() # just run until all work is completed
self.DicoInitIndiv={}
while iResult < NJobs:
DicoResult = None
if result_queue.qsize() != 0:
try:
DicoResult = result_queue.get()
except:
pass
if DicoResult == None:
time.sleep(0.5)
continue
if DicoResult["Success"]:
iResult+=1
NDone=iResult
pBAR.render(NDone,NJobs)
iIsland=DicoResult["iIsland"]
NameDico="%sDicoInitIsland_%5.5i"%(self.IdSharedMem,iIsland)
Dico=NpShared.SharedToDico(NameDico)
self.DicoInitIndiv[iIsland]=copy.deepcopy(Dico)
NpShared.DelAll(NameDico)
if Parallel:
for ii in range(NCPU):
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()
#MyLogger.setLoud(["pymoresane.main"])
#MyLogger.setLoud(["ClassImageDeconvMachineMSMF","ClassPSFServer","ClassMultiScaleMachine","GiveModelMachine","ClassModelMachineMSMF"])
return self.DicoInitIndiv
def __init__(self,GD,DicoVariablePSF,RefFreq,GridFreqs,DegridFreqs,
MainCache=None,
IdSharedMem="",
DoWait=False):
self.DicoVariablePSF=DicoVariablePSF
GD=copy.deepcopy(GD)
self.RefFreq=RefFreq
self.GridFreqs=GridFreqs
self.DegridFreqs=DegridFreqs
self.GD=GD
self.GD["Parallel"]["NCPU"]=1
#self.GD["HMP"]["Alpha"]=[0,0,1]#-1.,1.,5]
self.GD["HMP"]["Alpha"]=[-1.,1.,5]
self.GD["Deconv"]["Mode"]="MORESANE"
self.GD["Deconv"]["CycleFactor"]=0
self.GD["Deconv"]["PeakFactor"]=0.0
self.GD["Deconv"]["RMSFactor"]=self.GD["GAClean"]["RMSFactorInitHMP"]
self.GD["Deconv"]["Gain"]=self.GD["GAClean"]["GainInitHMP"]
self.GD["Deconv"]["AllowNegative"]=self.GD["GAClean"]["AllowNegativeInitHMP"]
self.GD["Deconv"]["MaxMinorIter"]=self.GD["GAClean"]["MaxMinorIterInitHMP"]
MyLogger.setSilent(SilentModules)
self.GD["HMP"]["Scales"]=self.GD["GAClean"]["ScalesInitHMP"]
self.GD["HMP"]["Ratios"]=[]
#self.GD["MultiScale"]["Ratios"]=[]
self.GD["HMP"]["NTheta"]=4
self.GD["HMP"]["SolverMode"]="NNLS"
#self.GD["MultiScale"]["SolverMode"]="PI"
self.NFreqBands=len(DicoVariablePSF["freqs"])
MinorCycleConfig=dict(self.GD["Deconv"])
MinorCycleConfig["NCPU"]=self.GD["Parallel"]["NCPU"]
MinorCycleConfig["NFreqBands"]=self.NFreqBands
MinorCycleConfig["GD"] = self.GD
MinorCycleConfig["GridFreqs"] = self.GridFreqs
MinorCycleConfig["DegridFreqs"] = self.DegridFreqs
#MinorCycleConfig["RefFreq"] = self.RefFreq
ModConstructor = ClassModModelMachine(self.GD)
ModelMachine = ModConstructor.GiveMM(Mode="MORESANE")
ModelMachine.setRefFreq(self.RefFreq)
MinorCycleConfig["ModelMachine"]=ModelMachine
#MinorCycleConfig["CleanMaskImage"]=None
self.MinorCycleConfig=MinorCycleConfig
self.DeconvMachine=ClassImageDeconvMachineMoresane.ClassImageDeconvMachine(MainCache=MainCache,
ParallelMode=False,
RefFreq=self.RefFreq,
CacheFileName="HMP_Init",
IdSharedMem=IdSharedMem,
**self.MinorCycleConfig)
self.GD["Mask"]["Auto"]=False
self.GD["Mask"]["External"]=None
self.MaskMachine=ClassMaskMachine.ClassMaskMachine(self.GD)
self.DeconvMachine.setMaskMachine(self.MaskMachine)
self.Margin=50
#print "Start 3"
self.DeconvMachine.Init(PSFVar=self.DicoVariablePSF,PSFAve=self.DicoVariablePSF["PSFSideLobes"],
GridFreqs=self.GridFreqs,DegridFreqs=self.DegridFreqs,DoWait=DoWait,RefFreq=self.RefFreq)
if DoWait:
print "IINit3"
time.sleep(10)
print "Start 4"
self.DeconvMachine.Update(self.DicoDirty,DoSetMask=False)
if DoWait:
print "IINit4"
time.sleep(10)