def fix_dico_shape(fulldico, outdico, NPixOut):
# dico_model = 'image_full_ampphase_di_m.NS.DicoModel'
# save_dico = dico_model.replace('.DicoModel', '.restricted.DicoModel')
# NPixOut = 10000
dico = MyPickle.Load(fulldico)
NPix = dico['ModelShape'][-1]
NPix0, _ = EstimateNpix(float(NPix), Padding=1)
if NPix != NPix0:
raise ValueError("NPix != NPix0")
logger.info("Changing image size: %i -> %i pixels" % (NPix, NPixOut))
xc0 = NPix // 2
xc1 = NPixOut // 2
dx = xc0 - xc1
DCompOut = {}
for k, v in dico.items():
if k == 'Comp':
DCompOut['Comp'] = {}
continue
DCompOut[k] = v
DCompOut["Type"] = "SSD"
N, M, _, _ = dico['ModelShape']
DCompOut['ModelShape'] = [N, M, NPixOut, NPixOut]
for (x0, y0) in dico['Comp'].keys():
x1 = x0 - dx
y1 = y0 - dx
c0 = (x1 >= 0) & (x1 < NPixOut)
c1 = (y1 >= 0) & (y1 < NPixOut)
if c0 & c1:
logger.info("Mapping (%i,%i)->(%i,%i)" % (x0, y0, x1, y1))
DCompOut['Comp'][(x1, y1)] = dico['Comp'][(x0, y0)]
logger.info("Saving in {}".format(outdico))
MyPickle.Save(DCompOut, outdico)
def read_options():
default_values = Parset.value_dict
attrs = Parset.attr_dict
desc = """Questions and suggestions: [email protected]"""
OP = MyOptParse.MyOptParse(usage='Usage: %prog [parset file] <options>', version='%prog version '+report_version(),
description=desc, defaults=default_values, attributes=attrs)
# create options based on contents of parset
for section in Parset.sections:
values = default_values[section]
# "_Help" value in each section is its documentation string
OP.OptionGroup(values.get("_Help", section), section)
for name, value in getattr(default_values[section], "iteritems", default_values[section].items)():
if not attrs[section][name].get("no_cmdline"):
OP.add_option(name, value)
OP.Finalise()
OP.ReadInput()
# #optcomplete.autocomplete(opt)
# options, arguments = opt.parse_args()
MyPickle.Save(OP, SaveFile)
return OP
def main(options=None):
if options == None:
f = open(SaveName, 'rb')
options = pickle.load(f)
ListDirSpec = [
l.strip() for l in open(options.DirsDynSpecMSProds).readlines()
if not ".tgz" in l
]
D = {}
sdb = surveys_db.SurveysDB()
for iFile, f in enumerate(ListDirSpec):
OBSID = str(f.split("_L")[1])
Di = sdb.get_observation(OBSID)
Field = Di["field"]
D[iFile] = {}
D[iFile]["OBSID"] = str(OBSID)
D[iFile]["Field"] = str(Field)
sdb.close()
MyPickle.Save(D, "LoTSS_OBSID_vs_Field.Dico")
for iFile in list(D.keys()):
OBSID = D[iFile]["OBSID"]
Field = D[iFile]["Field"]
ss = "ms2dynspec.py --imageI /databf/lofar/SURVEYS_KSP/LOTSS/DynSpecMS/IMAGES/%s/image_full_ampphase_di_m.NS_shift.int.facetRestored.fits --imageV /databf/lofar/SURVEYS_KSP/LOTSS/DynSpecMS/IMAGES/%s/image_full_low_stokesV.dirty.fits --BaseDirSpecs /databf/lofar/SURVEYS_KSP/LOTSS/DynSpecMS/DynSpecs_L%s --srclist Transient_LOTTS.csv" % (
Field, Field, OBSID)
print(ss)
os.system(ss)
def main(args=None, messages=[]):
if args is None:
args = MyPickle.Load(SaveFile)
MSList = expandMSList(args.ms)
MSList = [mstuple[0] for mstuple in MSList]
D = ClassDynSpecMS(ListMSName=MSList,
ColName=args.data,
ModelName=args.model,
SolsName=args.sols,
UVRange=args.uv,
FileCoords=args.srclist,
Radius=args.rad,
NOff=args.noff,
Image=args.image,
SolsDir=args.SolsDir,
NCPU=args.NCPU)
if D.NDirSelected == 0:
return
D.StackAll()
SaveMachine = ClassSaveResults.ClassSaveResults(D)
SaveMachine.WriteFits()
SaveMachine.PlotSpec()
SaveMachine.tarDirectory()
def Cluster(self):
l, m = self.radec2lm(self.Cat.ra, self.Cat.dec)
S = self.Cat.S.copy()
PolyList = None
self.BigPolygon = []
PolyList = []
if self.AvoidPolygons != "":
print("Reading polygon file: %s" % self.AvoidPolygons, file=log)
PolyList += MyPickle.Load(self.AvoidPolygons)
if len(PolyList) > 0:
LPoly = []
inside = np.zeros((l.size, ), np.float32)
for iPolygon, Poly in enumerate(PolyList):
ra, dec = Poly.T
lp, mp = self.radec2lm(ra, dec)
Poly[:, 0] = lp
Poly[:, 1] = mp
P = Polygon.Polygon(Poly)
if P.area() > self.BigPolygonSize:
self.BigPolygon.append(Poly)
for ip in range(l.size):
if P.isInside(l[ip], m[ip]):
inside[ip] = 1
# l=l[inside==0]
# m=m[inside==0]
# S=S[inside==0]
print("There are %i big polygons" % len(self.BigPolygon), file=log)
if self.CentralRadius > 0:
print("Create central polygon with radius %f degrees" %
self.CentralRadius,
file=log)
Rad = self.CentralRadius * np.pi / 180
th = np.arange(0, 2. * np.pi, 2. * np.pi / 100)
lp = np.cos(th) * Rad
mp = np.sin(th) * Rad
Poly = np.zeros((lp.size, 2), np.float32)
Poly[:, 0] = lp
Poly[:, 1] = mp
PolyList += [Poly]
CC = Sky.ClassClusterDEAP.ClassCluster(l,
m,
S,
nNode=self.NCluster,
NGen=self.NGen,
NPop=self.NPop,
DoPlot=self.DoPlot,
PolyCut=self.PolyCut,
NCPU=self.NCPU,
BigPolygon=self.BigPolygon)
CC.setAvoidPolygon(PolyList)
xyNodes, self.LPolygon = CC.Cluster()
nNodes = xyNodes.size // 2
xc, yc = xyNodes.reshape((2, nNodes))
self.xcyc = xc, yc
def reinitPop2(self, pop):
print >> log, "Initialise population"
x0, x1 = self.x.min(), self.x.max()
y0, y1 = self.y.min(), self.y.max()
N = len(pop)
# p=np.load("pop.npy")
# for i,ii in enumerate(p): pop[i][:]=ii[:]
# return
pop0 = pop[0:N / 2]
pop1 = pop[N / 2:]
for iIndiv, Indiv in enumerate(pop0):
#print iIndiv,len(pop0)
x, y = Indiv.reshape((2, self.nNode))
indSel = DDFacet.ToolsDir.GeneDist.GiveNonRedundantSample(
self.S, self.nNode)
x[:] = self.x[indSel]
y[:] = self.y[indSel]
for iIndiv, Indiv in enumerate(pop1):
#print iIndiv,len(pop1)
x, y = Indiv.reshape((2, self.nNode))
x[:] = np.random.uniform(x0, x1, self.nNode)
y[:] = np.random.uniform(y0, y1, self.nNode)
MyPickle.Save(pop, "pop.myPickle")
def ToFile(self, FileName, DicoIn=None):
print("Saving dico model to %s" % FileName, file=log)
if DicoIn is None:
D = self.DicoSMStacked
else:
D = DicoIn
D["ListScales"] = self.ListScales
D["ModelShape"] = self.ModelShape
MyPickle.Save(D, FileName)
def GiveMMFromFile(self,FileName=None):
"""
Initialise a model machine from a file
Input:
FileName = The file to read
"""
if FileName is not None:
DicoSMStacked = MyPickle.Load(FileName)
return self.GiveMMFromDico(DicoSMStacked)
else:
return self.GiveMMFromDico()
def ToFile(self, FileName, DicoIn=None):
print >> log, "Saving dico model to %s" % FileName
if DicoIn is None:
D = self.DicoSMStacked
else:
D = DicoIn
D["GD"] = self.GD
D["Type"] = "HMP"
D["ListScales"] = self.ListScales
D["ModelShape"] = self.ModelShape
MyPickle.Save(D, FileName)
def ToFile(self, FileName, DicoIn=None):
print >> log, "Saving dico model to %s" % FileName
if DicoIn is None:
D = self.DicoModel
else:
D = DicoIn
D["GD"] = self.GD
D["ModelShape"] = self.ModelShape
D["Type"] = "MORESANE"
MyPickle.Save(D, FileName)
def GiveInitialisedMMFromFile(self, FileName):
"""
Initialise a model machine from a file
Input:
FileName = The file to read
"""
DicoSMStacked = MyPickle.Load(FileName)
if self.GD is None:
self.GD = DicoSMStacked["GD"]
MM = self.GiveMMFromDico(DicoSMStacked)
MM.FromDico(DicoSMStacked)
return MM
def ToFile(self, FileName, DicoIn=None):
print("Saving dico model to %s" % FileName, file=log)
if DicoIn is None:
D = self.DicoModel
else:
D = DicoIn
D["GD"] = self.GD
D["ModelShape"] = self.ModelShape
D["FreqsCube"] = self.GridFreqs
D["Type"] = "MUFFIN"
MyPickle.Save(D, FileName)
def ToFile(self, FileName, DicoIn=None):
print >> log, "Saving dico model to %s" % FileName
if DicoIn is None:
D = self.DicoSMStacked
else:
D = DicoIn
#D["PM"]=self.PM
D["GD"] = self.GD
D["ModelShape"] = self.ModelShape
D["Type"] = "SSD"
D["SolveParam"] = self.SolveParam
MyPickle.Save(D, FileName)
def ToFile(self, FileName, DicoIn=None):
print("Saving dico model to %s" % FileName, file=log)
if DicoIn is None:
D = self.DicoSMStacked
else:
D = DicoIn
if self.GD is None:
print(
"Warning - you are haven't initialised GD before writing to the DicoModel"
)
D["GD"] = self.GD
D["Type"] = "WSCMS"
D["ModelShape"] = self.ModelShape
MyPickle.Save(D, FileName)
def ToFile(self, FileName, DicoIn=None):
print >> log, "Saving dico model to %s" % FileName
if DicoIn is None:
D = self.DicoSMStacked
else:
D = DicoIn
D["GD"] = self.GD
D["Type"] = "WSCMS"
# try:
# D["ListScales"] = list(self.ScaleMachine.sigmas) # list containing std of Gaussian components
# except:
# D["ListScales"] = self.ListScales
D["ModelShape"] = self.ModelShape
MyPickle.Save(D, FileName)
def GiveSpectralIndexMap(self, threshold=0.1, save_dict=True):
# Get the model image
IM = self.GiveModelImage(self.FreqMachine.Freqsp)
nchan, npol, Nx, Ny = IM.shape
# Fit the alpha map
self.FreqMachine.FitAlphaMap(IM[:, 0, :, :], threshold=threshold) # should set threshold based on SNR of final residual
if save_dict:
FileName = self.GD['Output']['Name'] + ".Dicoalpha"
print>>log, "Saving componentwise SPI map to %s"%FileName
MyPickle.Save(self.FreqMachine.alpha_dict, FileName)
return self.FreqMachine.weighted_alpha_map.reshape((1, 1, Nx, Ny))
def ToFile(self, FileName, DicoIn=None):
print("Saving dico model to %s" % FileName, file=log)
if DicoIn is None:
D = self.DicoSMStacked
else:
D = DicoIn
if self.GD is None:
print("Warning - You have not initialised self.GD in ModelMachine so " \
"we can't write it to the DicoModel", file=log)
D["GD"] = self.GD
D["Type"] = "Hogbom"
D["ListScales"] = "Delta"
D["ModelShape"] = self.ModelShape
MyPickle.Save(D, FileName)
def testMF_DATA():
Dico = MyPickle.Load("SaveTest")
Dirty = Dico["Dirty"]
PSF = Dico["PSF"]
FreqsInfo = Dico["FreqsInfo"]
IslandBestIndiv = Dico["IslandBestIndiv"]
ListPixData = Dico["ListPixData"]
ListPixParms = Dico["ListPixParms"]
GD = Dico["GD"]
FacetID = Dico["FacetID"]
IdSharedMem = Dico["IdSharedMem"]
iIsland = Dico["iIsland"]
#GD["GAClean"]["GASolvePars"]=["S","Alpha","GSig"]
#GD["GAClean"]["GASolvePars"]=["S","Alpha"]
nch = FreqsInfo["MeanJonesBand"][FacetID].size
WeightMeanJonesBand = FreqsInfo["MeanJonesBand"][FacetID].reshape(
(nch, 1, 1, 1))
WeightMueller = WeightMeanJonesBand.ravel()
WeightMuellerSignal = WeightMueller * FreqsInfo["WeightChansImages"].ravel(
)
# IncreaseIslandMachine=ClassIncreaseIsland.ClassIncreaseIsland()
# ListPixData=IncreaseIslandMachine.IncreaseIsland(ListPixData,dx=20)
# IncreaseIslandMachine=ClassIncreaseIsland.ClassIncreaseIsland()
# ListPixData=IncreaseIslandMachine.IncreaseIsland(ListPixData,dx=5)
#IslandBestIndiv=np.zeros((len(GD["GAClean"]["GASolvePars"])*len(
CEv = ClassEvolveGA(Dirty,
PSF,
FreqsInfo,
ListPixParms=ListPixParms,
ListPixData=ListPixData,
GD=GD,
IslandBestIndiv=IslandBestIndiv,
WeightFreqBands=WeightMuellerSignal,
iIsland=iIsland,
IdSharedMem=IdSharedMem)
CEv.main()
def Cluster(self):
l,m=self.radec2lm(self.Cat.ra,self.Cat.dec)
S=self.Cat.S.copy()
PolyList=None
if self.AvoidPolygons!="":
print>>log,"Reading polygon file: %s"%self.AvoidPolygons
self.BigPolygon=[]
PolyList=MyPickle.Load(self.AvoidPolygons)
LPoly=[]
inside=np.zeros((l.size,),np.float32)
for iPolygon,Poly in enumerate(PolyList):
ra,dec=Poly.T
lp,mp=self.radec2lm(ra,dec)
Poly[:,0]=lp
Poly[:,1]=mp
P=Polygon.Polygon(Poly)
if P.area()>self.BigPolygonSize:
self.BigPolygon.append(Poly)
for ip in range(l.size):
if P.isInside(l[ip],m[ip]):
inside[ip]=1
l=l[inside==0]
m=m[inside==0]
S=S[inside==0]
print>>log,"There are %i big polygons"%len(self.BigPolygon)
CC=Sky.ClassClusterDEAP.ClassCluster(l,m,S,nNode=self.NCluster,
NGen=self.NGen,
NPop=self.NPop,
DoPlot=self.DoPlot,
PolyCut=self.PolyCut,
NCPU=self.NCPU,
BigPolygon=self.BigPolygon)
CC.setAvoidPolygon(PolyList)
xyNodes,self.LPolygon=CC.Cluster()
nNodes=xyNodes.size/2
xc,yc=xyNodes.reshape((2,nNodes))
self.xcyc=xc,yc
def test():
P = ClassPrint.ClassPrint()
Obj, ValObj = MyPickle.Load("test")
#return Obj
#ValObj,_=Obj.parse_args()
#return ValObj
LGroups = Obj.option_groups
for Group in LGroups:
print(Group.title)
option_list = Group.option_list
for o in option_list:
lopt = o._long_opts[0]
oname = lopt.split("--")[-1]
V = getattr(ValObj, oname)
if V != "":
P.Print(oname, V)
# strName=%s
# print " "oname,V
print()
def main(args=None, messages=[]):
if args is None:
args = MyPickle.Load(SaveFile)
MSList=None
if args.ms:
MSList=expandMSList(args.ms)
MSList=[mstuple[0] for mstuple in MSList]
D = ClassDynSpecMS(ListMSName=MSList,
ColName=args.data, ModelName=args.model,
SolsName=args.sols,
ColWeights=args.WeightCol,
UVRange=args.uv,
FileCoords=args.srclist,
Radius=args.rad,
NOff=args.noff,
ImageI=args.imageI,
ImageV=args.imageV,
SolsDir=args.SolsDir,NCPU=args.NCPU,
BaseDirSpecs=args.BaseDirSpecs,
BeamModel=args.BeamModel,
BeamNBand=args.BeamNBand)
if D.NDirSelected==0:
return
if D.Mode=="Spec": D.StackAll()
SaveMachine=ClassSaveResults.ClassSaveResults(D)
if D.Mode=="Spec":
SaveMachine.WriteFits()
SaveMachine.PlotSpec()
SaveMachine.SaveCatalog()
SaveMachine.tarDirectory()
else:
SaveMachine.SaveCatalog()
SaveMachine.PlotSpec(Prefix="_replot")
default=0,
help="Boring?",
required=False)
parser.add_argument("--image",
type=str,
default=None,
help="Survey image to plot",
required=False)
parser.add_argument("--uv",
type=list,
default=[1., 1000.],
help="UV range in km [UVmin, UVmax]",
required=False)
parser.add_argument("--SolsDir",
type=str,
default="",
help="Base directory for the DDE solutions",
required=False)
parser.add_argument("--NCPU",
type=int,
default=0,
help="NCPU",
required=False)
args = parser.parse_args()
MyPickle.Save(args, SaveFile)
ModColor.silent = progressbar.ProgressBar.silent = args.LogBoring
main(args)
def FromFile(self,FileName):
print>>log, "Reading dico model from file %s"%FileName
self.DicoModel=MyPickle.Load(FileName)
self.FromDico(self.DicoModel)
def main(OP=None, messages=[]):
if OP is None:
OP = MyPickle.Load(SaveFile)
print "Using settings from %s, then command line." % SaveFile
DicoConfig = OP.DicoConfig
ImageName = DicoConfig["Output"]["Name"]
if not ImageName:
raise Exceptions.UserInputError(
"--Output-Name not specified, can't continue.")
if not DicoConfig["Data"]["MS"]:
raise Exceptions.UserInputError(
"--Data-MS not specified, can't continue.")
# create directory if it exists
dirname = os.path.dirname(ImageName)
if not os.path.exists(dirname) and not dirname == "":
os.mkdir(dirname)
# setup logging
MyLogger.logToFile(ImageName + ".log", append=DicoConfig["Log"]["Append"])
global log
log = MyLogger.getLogger("DDFacet")
# disable colors and progressbars if requested
ModColor.silent = SkyModel.Other.ModColor.silent = \
progressbar.ProgressBar.silent = \
DicoConfig["Log"]["Boring"]
if messages:
if not DicoConfig["Log"]["Boring"]:
#os.system('clear')
logo.print_logo()
for msg in messages:
print >> log, msg
if DicoConfig["Debug"]["Pdb"] == "always":
print >> log, "--Debug-Pdb=always: unexpected errors will be dropped into pdb"
Exceptions.enable_pdb_on_error(
ModColor.Str(
"DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n"
+
"(This is because you're running with --Debug-Pdb set to 'always'.)"
))
elif DicoConfig["Debug"][
"Pdb"] == "auto" and not DicoConfig["Log"]["Boring"]:
print >> log, "--Debug-Pdb=auto and not --Log-Boring: unexpected errors will be dropped into pdb"
Exceptions.enable_pdb_on_error(
ModColor.Str(
"DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n"
+
"(This is because you're running with --Debug-Pdb set to 'auto' and --Log-Boring is off.)"
))
# print current options
OP.Print(dest=log)
# enable memory logging
MyLogger.enableMemoryLogging(DicoConfig["Log"]["Memory"])
# get rid of old shm arrays from previous runs
Multiprocessing.cleanupStaleShm()
# initialize random seed from config if set, or else from system time
if DicoConfig["Misc"]["RandomSeed"] is not None:
print >> log, "random seed=%d (explicit)" % DicoConfig["Misc"][
"RandomSeed"]
else:
DicoConfig["Misc"]["RandomSeed"] = int(time.time())
print >> log, "random seed=%d (automatic)" % DicoConfig["Misc"][
"RandomSeed"]
np.random.seed(DicoConfig["Misc"]["RandomSeed"])
# init NCPU for different bits of parallelism
ncpu = DicoConfig["Parallel"]["NCPU"] or psutil.cpu_count()
DicoConfig["Parallel"]["NCPU"] = ncpu
_pyArrays.pySetOMPNumThreads(ncpu)
NpParallel.NCPU_global = ModFFTW.NCPU_global = ncpu
numexpr.set_num_threads(ncpu)
print >> log, "using up to %d CPUs for parallelism" % ncpu
# write parset
OP.ToParset("%s.parset" % ImageName)
Mode = DicoConfig["Output"]["Mode"]
# init semaphores, as they're needed for weight calculation too
ClassFacetMachine.ClassFacetMachine.setup_semaphores(DicoConfig)
# data machine initialized for all cases except PSF-only mode
# psf machine initialized for all cases except Predict-only mode
Imager = ClassDeconvMachine.ClassImagerDeconv(
GD=DicoConfig,
BaseName=ImageName,
predict_only=(Mode == "Predict" or Mode == "Subtract"),
data=(Mode != "PSF"),
psf=(Mode != "Predict" and Mode != "Dirty" and Mode != "Subtract"),
readcol=(Mode != "Predict" and Mode != "PSF"),
deconvolve=("Clean" in Mode))
Imager.Init()
# Imager.testDegrid()
# stop
if "Predict" in Mode or "Subtract" in Mode:
Imager.GivePredict()
if "Clean" in Mode:
Imager.main()
elif "Dirty" in Mode:
sparsify = DicoConfig["Comp"]["Sparsification"]
if sparsify and isinstance(sparsify, list):
sparsify = sparsify[0]
Imager.GiveDirty(psf="PSF" in Mode, sparsify=sparsify)
elif "PSF" in Mode:
sparsify = DicoConfig["Comp"]["Sparsification"]
if sparsify and isinstance(sparsify, list):
sparsify = sparsify[0]
Imager.MakePSF(sparsify=sparsify)
elif "RestoreAndShift" == Mode:
Imager.RestoreAndShift()
def Restore(self):
print("Create restored image", file=log)
FWHMFact=2.*np.sqrt(2.*np.log(2.))
FWHMFact=2.*np.sqrt(2.*np.log(2.))
BeamPix=self.BeamPix/FWHMFact
sigma_x, sigma_y=BeamPix,BeamPix
theta=0.
bmaj=np.max([sigma_x, sigma_y])*self.CellArcSec*FWHMFact
bmin=np.min([sigma_x, sigma_y])*self.CellArcSec*FWHMFact
#bmaj=bmin=0.001666666666666667*3600
#sigma_x=
self.FWHMBeam=(bmaj/3600./np.sqrt(2.),bmin/3600./np.sqrt(2.),theta)
self.PSFGaussPars = (sigma_x*self.CellSizeRad, sigma_y*self.CellSizeRad, theta)
#print "!!!!!!!!!!!!!!!!!!!!"
#self.PSFGaussPars = (BeamPix,BeamPix,0)
RefFreq=self.ModelMachine.RefFreq
df=RefFreq*0.5
# ################################"
if self.options.PSFCache!="":
import os
IdSharedMem=str(int(os.getpid()))+"."
MeanModelImage=self.ModelMachine.GiveModelImage(RefFreq)
# #imNorm=image("6SBc.KAFCA.restoredNew.fits.6SBc.KAFCA.restoredNew.fits.MaskLarge.fits").getdata()
# imNorm=image("6SB.KAFCA.GA.BIC_00.AP.dirty.fits.mask.fits").getdata()
# MASK=np.zeros_like(imNorm)
# nchan,npol,_,_=MASK.shape
# for ch in range(nchan):
# for pol in range(npol):
# MASK[ch,pol,:,:]=imNorm[ch,pol,:,:].T[::-1,:]
# MeanModelImage[MASK==0]=0
# MeanModelImage.fill(0)
# MeanModelImage[0,0,100,100]=1
from DDFacet.Imager.GA import ClassSmearSM
from DDFacet.Imager import ClassPSFServer
self.DicoVariablePSF = MyPickle.FileToDicoNP(self.options.PSFCache)
self.PSFServer=ClassPSFServer.ClassPSFServer()
self.PSFServer.setDicoVariablePSF(self.DicoVariablePSF,NormalisePSF=True)
#return self.Residual,MeanModelImage,self.PSFServer
# CasaImage=ClassCasaImage.ClassCasaimage("Model.fits",MeanModelImage.shape,self.Cell,self.radec)#Lambda=(Lambda0,dLambda,self.NBands))
# CasaImage.setdata(MeanModelImage,CorrT=True)
# CasaImage.ToFits()
# #CasaImage.setBeam((SmoothFWHM,SmoothFWHM,0))
# CasaImage.close()
SmearMachine=ClassSmearSM.ClassSmearSM(self.Residual,
MeanModelImage*self.SqrtNormImage,
self.PSFServer,
DeltaChi2=4.,
IdSharedMem=IdSharedMem,
NCPU=self.options.NCPU)
SmearedModel=SmearMachine.Smear()
SmoothFWHM=self.CellArcSec*SmearMachine.RestoreFWHM/3600.
ModelSmearImage="%s.RestoredSmear"%self.BaseImageName
CasaImage=ClassCasaImage.ClassCasaimage(ModelSmearImage,SmearedModel.shape,self.Cell,self.radec)#Lambda=(Lambda0,dLambda,self.NBands))
CasaImage.setdata(SmearedModel+self.Residual,CorrT=True)
#CasaImage.setdata(SmearedModel,CorrT=True)
CasaImage.setBeam((SmoothFWHM,SmoothFWHM,0))
CasaImage.ToFits()
CasaImage.close()
SmearMachine.CleanUpSHM()
stop
# ################################"
#self.ModelMachine.ListScales[0]["Alpha"]=-0.8
# model image
#ModelMachine.GiveModelImage(RefFreq)
FEdge=np.linspace(RefFreq-df,RefFreq+df,self.NBands+1)
FCenter=(FEdge[0:-1]+FEdge[1::])/2.
C=299792458.
Lambda0=C/FCenter[-1]
dLambda=1
if self.NBands>1:
dLambda=np.abs(C/FCenter[0]-C/FCenter[1])
ListRestoredIm=[]
Lambda=[Lambda0+i*dLambda for i in range(self.NBands)]
ListRestoredImCorr=[]
ListModelIm=[]
#print C/np.array(Lambda)
# restored image
for l in Lambda:
freq=C/l
if self.options.RandomCat:
print("Create random catalog... ", file=log)
ModelImage=self.GiveRandomModelIm()
else:
print("Get ModelImage... ", file=log)
ModelImage=self.ModelMachine.GiveModelImage(freq)
if self.options.ZeroNegComp:
print("Zeroing negative componants... ", file=log)
ModelImage[ModelImage<0]=0
ListModelIm.append(ModelImage)
if self.options.Mode=="App":
print(" ModelImage to apparent flux... ", file=log)
ModelImage=ModelImage*self.SqrtNormImage
print("Convolve... ", file=log)
print(" MinMax = [%f , %f] @ freq = %f MHz"%(ModelImage.min(),ModelImage.max(),freq/1e6), file=log)
#RestoredImage=ModFFTW.ConvolveGaussianScipy(ModelImage,CellSizeRad=self.CellSizeRad,GaussPars=[self.PSFGaussPars])
if self.options.AddNoise>0.:
print("Adding Noise... ", file=log)
ModelImage+=np.random.randn(*ModelImage.shape)*self.options.AddNoise
RestoredImage,_=ModFFTW.ConvolveGaussianWrapper(ModelImage,Sig=BeamPix)
# =======
# #RestoredImage,_=ModFFTW.ConvolveGaussianWrapper(ModelImage,Sig=BeamPix)
# def GiveGauss(Sig0,Sig1):
# npix=20*int(np.sqrt(Sig0**2+Sig1**2))
# if not npix%2: npix+=1
# dx=npix/2
# x,y=np.mgrid[-dx:dx:npix*1j,-dx:dx:npix*1j]
# dsq=x**2+y**2
# return Sig0**2/(Sig0**2+Sig1**2)*np.exp(-dsq/(2.*(Sig0**2+Sig1**2)))
# R2=np.zeros_like(ModelImage)
# Sig0=BeamPix/np.sqrt(2.)
# if self.options.RandomCat:
# Sig1=(self.options.RandomCat_SigFactor-1.)*Sig0
# else:
# Sig1=0.
# nch,npol,_,_=ModelImage.shape
# for ch in range(nch):
# in1=ModelImage[ch,0]
# R2[ch,0,:,:]=scipy.signal.fftconvolve(in1,GiveGauss(Sig0,Sig1), mode='same').real
# RestoredImage=R2
# self.header_dict["GSIGMA"]=Sig0
# # print np.max(np.abs(R2-RestoredImage))
# # import pylab
# # ax=pylab.subplot(1,3,1)
# # pylab.imshow(RestoredImage[0,0],interpolation="nearest")
# # pylab.colorbar()
# # pylab.subplot(1,3,2,sharex=ax,sharey=ax)
# # pylab.imshow(R2[0,0],interpolation="nearest")
# # pylab.colorbar()
# # pylab.subplot(1,3,3,sharex=ax,sharey=ax)
# # pylab.imshow((RestoredImage-R2)[0,0],interpolation="nearest")
# # pylab.colorbar()
# # pylab.show()
# # stop
# >>>>>>> 0457182a873da89a2758f4be8a18f55cefd88e44
RestoredImageRes=RestoredImage+self.Residual
ListRestoredIm.append(RestoredImageRes)
RestoredImageResCorr=RestoredImageRes/self.SqrtNormImage
ListRestoredImCorr.append(RestoredImageResCorr)
#print FEdge,FCenter
print("Save... ", file=log)
_,_,nx,_=RestoredImageRes.shape
RestoredImageRes=np.array(ListRestoredIm).reshape((self.NBands,1,nx,nx))
RestoredImageResCorr=np.array(ListRestoredImCorr).reshape((self.NBands,1,nx,nx))
ModelImage=np.array(ListModelIm).reshape((self.NBands,1,nx,nx))
if self.OutName=="":
ImageName="%s.restoredNew"%self.BaseImageName
ImageNameCorr="%s.restoredNew.corr"%self.BaseImageName
ImageNameModel="%s.model"%self.BaseImageName
ImageNameModelConv="%s.modelConv"%self.BaseImageName
else:
ImageName=self.OutName
ImageNameCorr=self.OutName+".corr"
ImageNameModel="%s.model"%self.OutName
ImageNameModelConv="%s.modelConv"%self.OutName
CasaImage=ClassCasaImage.ClassCasaimage(ImageNameModel,RestoredImageRes.shape,self.Cell,self.radec,header_dict=self.header_dict)#Lambda=(Lambda0,dLambda,self.NBands))
CasaImage.setdata(ModelImage,CorrT=True)
CasaImage.setBeam(self.FWHMBeam)
CasaImage.ToFits()
CasaImage.close()
CasaImage=ClassCasaImage.ClassCasaimage(ImageName,RestoredImageRes.shape,self.Cell,self.radec,Freqs=C/np.array(Lambda).ravel(),header_dict=self.header_dict)#,Lambda=(Lambda0,dLambda,self.NBands))
CasaImage.setdata(RestoredImageRes,CorrT=True)
CasaImage.setBeam(self.FWHMBeam)
CasaImage.ToFits()
CasaImage.close()
CasaImage=ClassCasaImage.ClassCasaimage(ImageNameModelConv,RestoredImage.shape,self.Cell,self.radec,Freqs=C/np.array(Lambda).ravel(),header_dict=self.header_dict)#,Lambda=(Lambda0,dLambda,self.NBands))
CasaImage.setdata(RestoredImage,CorrT=True)
CasaImage.setBeam(self.FWHMBeam)
CasaImage.ToFits()
CasaImage.close()
if self.MakeCorrected:
CasaImage=ClassCasaImage.ClassCasaimage(ImageNameCorr,RestoredImageResCorr.shape,self.Cell,self.radec,Freqs=C/np.array(Lambda).ravel(),header_dict=self.header_dict)#,Lambda=(Lambda0,dLambda,self.NBands))
CasaImage.setdata(RestoredImageResCorr,CorrT=True)
CasaImage.setBeam(self.FWHMBeam)
CasaImage.ToFits()
CasaImage.close()
# ImageName="%s.modelConv"%self.BaseImageName
# CasaImage=ClassCasaImage.ClassCasaimage(ImageName,ModelImage.shape,self.Cell,self.radec)
# CasaImage.setdata(self.RestoredImage,CorrT=True)
# CasaImage.ToFits()
# CasaImage.setBeam(self.FWHMBeam)
# CasaImage.close()
# Alpha image
if self.DoAlpha:
print("Get Index Map... ", file=log)
IndexMap=self.ModelMachine.GiveSpectralIndexMap(CellSizeRad=self.CellSizeRad,GaussPars=[self.PSFGaussPars])
ImageName="%s.alphaNew"%self.BaseImageName
print(" Save... ", file=log)
CasaImage=ClassCasaImage.ClassCasaimage(ImageName,ModelImage.shape,self.Cell,self.radec)
CasaImage.setdata(IndexMap,CorrT=True)
CasaImage.ToFits()
CasaImage.close()
print(" Done. ", file=log)
def FromFile(self, FileName):
print >> log, "Reading dico model from %s" % FileName
self.DicoSMStacked = MyPickle.Load(FileName)
self.FromDico(self.DicoSMStacked)
def setFacetsLocs(self):
NFacets = self.NFacets
Npix = self.GD["Image"]["NPix"]
Padding = self.GD["Facets"]["Padding"]
self.Padding = Padding
Npix, _ = EstimateNpix(float(Npix), Padding=1)
self.Npix = Npix
self.OutImShape = (self.nch, self.npol, self.Npix, self.Npix)
RadiusTot = self.CellSizeRad * self.Npix / 2
self.RadiusTot = RadiusTot
lMainCenter, mMainCenter = 0., 0.
self.lmMainCenter = lMainCenter, mMainCenter
self.CornersImageTot = np.array(
[[lMainCenter - RadiusTot, mMainCenter - RadiusTot],
[lMainCenter + RadiusTot, mMainCenter - RadiusTot],
[lMainCenter + RadiusTot, mMainCenter + RadiusTot],
[lMainCenter - RadiusTot, mMainCenter + RadiusTot]])
# MSName = self.GD["Data"]["MS"]
# if ".txt" in MSName:
# f = open(MSName)
# Ls = f.readlines()
# f.close()
# MSName = []
# for l in Ls:
# ll = l.replace("\n", "")
# MSName.append(ll)
# MSName = MSName[0]
MSName = self.VS.ListMS[0].MSName
SolsFile = self.GD["DDESolutions"]["DDSols"]
if isinstance(SolsFile, list):
SolsFile = self.GD["DDESolutions"]["DDSols"][0]
if SolsFile and (not (".npz" in SolsFile)) and (not (".h5"
in SolsFile)):
Method = SolsFile
ThisMSName = reformat.reformat(os.path.abspath(MSName),
LastSlash=False)
SolsFile = "%s/killMS.%s.sols.npz" % (ThisMSName, Method)
# if "CatNodes" in self.GD.keys():
regular_grid = False
if self.GD["Facets"]["CatNodes"] is not None:
print >> log, "Taking facet directions from Nodes catalog: %s" % self.GD[
"Facets"]["CatNodes"]
ClusterNodes = np.load(self.GD["Facets"]["CatNodes"])
ClusterNodes = ClusterNodes.view(np.recarray)
raNode = ClusterNodes.ra
decNode = ClusterNodes.dec
lFacet, mFacet = self.CoordMachine.radec2lm(raNode, decNode)
elif ".npz" in SolsFile:
print >> log, "Taking facet directions from solutions file: %s" % SolsFile
ClusterNodes = np.load(SolsFile)["ClusterCat"]
ClusterNodes = ClusterNodes.view(np.recarray)
raNode = ClusterNodes.ra
decNode = ClusterNodes.dec
lFacet, mFacet = self.CoordMachine.radec2lm(raNode, decNode)
elif ".h5" in SolsFile:
print >> log, "Taking facet directions from HDF5 solutions file: %s" % SolsFile
H = tables.open_file(SolsFile)
raNode, decNode = H.root.sol000.source[:]["dir"].T
lFacet, mFacet = self.CoordMachine.radec2lm(raNode, decNode)
H.close()
del (H)
else:
print >> log, "Taking facet directions from regular grid"
regular_grid = True
CellSizeRad = (self.GD["Image"]["Cell"] / 3600.) * np.pi / 180
lrad = Npix * CellSizeRad * 0.5
NpixFacet = Npix / NFacets
lfacet = NpixFacet * CellSizeRad * 0.5
lcenter_max = lrad - lfacet
lFacet, mFacet, = np.mgrid[-lcenter_max:lcenter_max:(NFacets) * 1j,
-lcenter_max:lcenter_max:(NFacets) * 1j]
lFacet = lFacet.flatten()
mFacet = mFacet.flatten()
print >> log, " There are %i Jones-directions" % lFacet.size
self.lmSols = lFacet.copy(), mFacet.copy()
raSols, decSols = self.CoordMachine.lm2radec(lFacet.copy(),
mFacet.copy())
self.radecSols = raSols, decSols
NodesCat = np.zeros((raSols.size, ),
dtype=[('ra', np.float), ('dec', np.float),
('l', np.float), ('m', np.float)])
NodesCat = NodesCat.view(np.recarray)
NodesCat.ra = raSols
NodesCat.dec = decSols
# print>>log,"Facet RA %s"%raSols
# print>>log,"Facet Dec %s"%decSols
NodesCat.l = lFacet
NodesCat.m = mFacet
## saving below
# NodeFile = "%s.NodesCat.%snpy" % (self.GD["Output"]["Name"], "psf." if self.DoPSF else "")
# print>> log, "Saving Nodes catalog in %s" % NodeFile
# np.save(NodeFile, NodesCat)
self.DicoImager = {}
xy = np.zeros((lFacet.size, 2), np.float32)
xy[:, 0] = lFacet
xy[:, 1] = mFacet
regFile = "%s.tessel0.reg" % self.ImageName
NFacets = self.NFacets = lFacet.size
rac, decc = self.MainRaDec
VM = ModVoronoiToReg.VoronoiToReg(rac, decc)
if NFacets > 2:
vor = Voronoi(xy, furthest_site=False)
regions, vertices = ModVoronoi.voronoi_finite_polygons_2d(
vor, radius=1.)
PP = Polygon.Polygon(self.CornersImageTot)
LPolygon = []
ListNode = []
for region, iNode in zip(regions, range(NodesCat.shape[0])):
ThisP = np.array(PP
& Polygon.Polygon(np.array(vertices[region])))
if ThisP.size > 0:
LPolygon.append(ThisP[0])
ListNode.append(iNode)
NodesCat = NodesCat[np.array(ListNode)].copy()
# =======
# LPolygon = [
# np.array(PP & Polygon.Polygon(np.array(vertices[region])))[0]
# for region in regions]
# >>>>>>> issue-255
elif NFacets == 1:
l0, m0 = lFacet[0], mFacet[0]
LPolygon = [self.CornersImageTot]
# VM.ToReg(regFile,lFacet,mFacet,radius=.1)
NodeFile = "%s.NodesCat.npy" % self.GD["Output"]["Name"]
print >> log, "Saving Nodes catalog in %s" % NodeFile
np.save(NodeFile, NodesCat)
for iFacet, polygon0 in zip(range(len(LPolygon)), LPolygon):
# polygon0 = vertices[region]
P = polygon0.tolist()
# VM.PolygonToReg(regFile,LPolygon,radius=0.1,Col="red")
# stop
###########################################
# SubDivide
def GiveDiam(polygon):
lPoly, mPoly = polygon.T
l0 = np.max([lMainCenter - RadiusTot, lPoly.min()])
l1 = np.min([lMainCenter + RadiusTot, lPoly.max()])
m0 = np.max([mMainCenter - RadiusTot, mPoly.min()])
m1 = np.min([mMainCenter + RadiusTot, mPoly.max()])
dl = l1 - l0
dm = m1 - m0
diam = np.max([dl, dm])
return diam, (l0, l1, m0, m1)
DiamMax = self.GD["Facets"]["DiamMax"] * np.pi / 180
# DiamMax=4.5*np.pi/180
DiamMin = self.GD["Facets"]["DiamMin"] * np.pi / 180
def ClosePolygon(polygon):
P = polygon.tolist()
polygon = np.array(P + [P[0]])
return polygon
def GiveSubDivideRegions(polygonFacet, DMax):
polygonFOV = self.CornersImageTot
# polygonFOV=ClosePolygon(polygonFOV)
PFOV = Polygon.Polygon(polygonFOV)
# polygonFacet=ClosePolygon(polygonFacet)
P0 = Polygon.Polygon(polygonFacet)
P0Cut = Polygon.Polygon(P0 & PFOV)
if P0Cut.nPoints() == 0:
return []
polygonFacetCut = np.array(P0Cut[0])
# polygonFacetCut=ClosePolygon(polygonFacetCut)
diam, (l0, l1, m0, m1) = GiveDiam(polygonFacetCut)
if diam < DMax:
return [polygonFacetCut]
Nl = int((l1 - l0) / DMax) + 1
Nm = int((m1 - m0) / DMax) + 1
dl = (l1 - l0) / Nl
dm = (m1 - m0) / Nm
lEdge = np.linspace(l0, l1, Nl + 1)
mEdge = np.linspace(m0, m1, Nm + 1)
lc = (lEdge[0:-1] + lEdge[1::]) / 2
mc = (mEdge[0:-1] + mEdge[1::]) / 2
LPoly = []
Lc, Mc = np.meshgrid(lc, mc)
Lc = Lc.ravel().tolist()
Mc = Mc.ravel().tolist()
DpolySquare = np.array([[-dl, -dm], [dl, -dm], [dl, dm], [-dl, dm]
]) * 0.5
for lc, mc in zip(Lc, Mc):
polySquare = DpolySquare.copy(
) # ClosePolygon(DpolySquare.copy())
polySquare[:, 0] += lc
polySquare[:, 1] += mc
# polySquare=ClosePolygon(polySquare)
P1 = Polygon.Polygon(polySquare)
POut = (P0Cut & P1)
if POut.nPoints() == 0:
continue
polyOut = np.array(POut[0])
# polyOut=ClosePolygon(polyOut)
LPoly.append(polyOut)
# pylab.clf()
# x,y=polygonFacetCut.T
# pylab.plot(x,y,color="blue")
# x,y=polygonFacet.T
# pylab.plot(x,y,color="blue",ls=":",lw=3)
# x,y=np.array(PFOV[0]).T
# pylab.plot(x,y,color="black")
# x,y=polySquare.T
# pylab.plot(x,y,color="green",ls=":",lw=3)
# x,y=polyOut.T
# pylab.plot(x,y,color="red",ls="--",lw=3)
# pylab.xlim(-0.03,0.03)
# pylab.ylim(-0.03,0.03)
# pylab.draw()
# pylab.show(False)
# pylab.pause(0.5)
return LPoly
def PlotPolygon(P, *args, **kwargs):
for poly in P:
x, y = ClosePolygon(np.array(poly)).T
pylab.plot(x, y, *args, **kwargs)
LPolygonNew = []
for iFacet in xrange(len(LPolygon)):
polygon = LPolygon[iFacet]
ThisDiamMax = DiamMax
SubReg = GiveSubDivideRegions(polygon, ThisDiamMax)
LPolygonNew += SubReg
regFile = "%s.FacetMachine.tessel.ReCut.reg" % self.ImageName
# VM.PolygonToReg(regFile,LPolygonNew,radius=0.1,Col="green",labels=[str(i) for i in range(len(LPolygonNew))])
DicoPolygon = {}
for iFacet in xrange(len(LPolygonNew)):
DicoPolygon[iFacet] = {}
poly = LPolygonNew[iFacet]
DicoPolygon[iFacet]["poly"] = poly
diam, (l0, l1, m0, m1) = GiveDiam(poly)
DicoPolygon[iFacet]["diam"] = diam
DicoPolygon[iFacet]["diamMin"] = np.min([(l1 - l0), (m1 - m0)])
xc, yc = np.mean(poly[:, 0]), np.mean(poly[:, 1])
DicoPolygon[iFacet]["xyc"] = xc, yc
dSol = np.sqrt((xc - lFacet)**2 + (yc - mFacet)**2)
DicoPolygon[iFacet]["iSol"] = np.where(dSol == np.min(dSol))[0]
for iFacet in sorted(DicoPolygon.keys()):
diam = DicoPolygon[iFacet]["diamMin"]
# print iFacet,diam,DiamMin
if diam < DiamMin:
dmin = 1e6
xc0, yc0 = DicoPolygon[iFacet]["xyc"]
HasClosest = False
for iFacetOther in sorted(DicoPolygon.keys()):
if iFacetOther == iFacet:
continue
iSolOther = DicoPolygon[iFacetOther]["iSol"]
# print " ",iSolOther,DicoPolygon[iFacet]["iSol"]
if iSolOther != DicoPolygon[iFacet]["iSol"]:
continue
xc, yc = DicoPolygon[iFacetOther]["xyc"]
d = np.sqrt((xc - xc0)**2 + (yc - yc0)**2)
if d < dmin:
dmin = d
iFacetClosest = iFacetOther
HasClosest = True
if (HasClosest):
print >> log, "Merging facet #%i to #%i" % (iFacet,
iFacetClosest)
P0 = Polygon.Polygon(DicoPolygon[iFacet]["poly"])
P1 = Polygon.Polygon(DicoPolygon[iFacetClosest]["poly"])
P2 = (P0 | P1)
POut = []
for iP in xrange(len(P2)):
POut += P2[iP]
poly = np.array(POut)
hull = ConvexHull(poly)
Contour = np.array([
hull.points[hull.vertices, 0],
hull.points[hull.vertices, 1]
])
poly2 = Contour.T
del (DicoPolygon[iFacet])
DicoPolygon[iFacetClosest]["poly"] = poly2
DicoPolygon[iFacetClosest]["diam"] = GiveDiam(poly2)[0]
DicoPolygon[iFacetClosest]["xyc"] = np.mean(
poly2[:, 0]), np.mean(poly2[:, 1])
# stop
LPolygonNew = []
for iFacet in sorted(DicoPolygon.keys()):
# if DicoPolygon[iFacet]["diam"]<DiamMin:
# print>>log, ModColor.Str(" Facet #%i associated to direction #%i is too small, removing it"%(iFacet,DicoPolygon[iFacet]["iSol"]))
# continue
LPolygonNew.append(DicoPolygon[iFacet]["poly"])
# for iFacet in range(len(regions)):
# polygon=LPolygon[iFacet]
# ThisDiamMax=DiamMax
# while True:
# SubReg=GiveSubDivideRegions(polygon,ThisDiamMax)
# if SubReg==[]:
# break
# Diams=[GiveDiam(poly)[0] for poly in SubReg]
# if np.min(Diams)>DiamMin: break
# ThisDiamMax*=1.1
# LPolygonNew+=SubReg
# print
regFile = "%s.tessel.%sreg" % (self.GD["Output"]["Name"],
"psf." if self.DoPSF else "")
# labels=["[F%i.C%i]"%(i,DicoPolygon[i]["iSol"]) for i in range(len(LPolygonNew))]
# VM.PolygonToReg(regFile,LPolygonNew,radius=0.1,Col="green",labels=labels)
# VM.PolygonToReg(regFile,LPolygonNew,radius=0.1,Col="green")
# pylab.clf()
# x,y=LPolygonNew[11].T
# pylab.plot(x,y)
# pylab.draw()
# pylab.show()
# stop
###########################################
NFacets = len(LPolygonNew)
NJonesDir = NodesCat.shape[0]
self.JonesDirCat = np.zeros(
(NodesCat.shape[0], ),
dtype=[('Name', '|S200'), ('ra', np.float), ('dec', np.float),
('SumI', np.float), ("Cluster", int), ("l", np.float),
("m", np.float), ("I", np.float)])
self.JonesDirCat = self.JonesDirCat.view(np.recarray)
self.JonesDirCat.I = 1
self.JonesDirCat.SumI = 1
self.JonesDirCat.ra = NodesCat.ra
self.JonesDirCat.dec = NodesCat.dec
self.JonesDirCat.l = NodesCat.l
self.JonesDirCat.m = NodesCat.m
self.JonesDirCat.Cluster = range(NJonesDir)
print >> log, "Sizes (%i facets):" % (self.JonesDirCat.shape[0])
print >> log, " - Main field : [%i x %i] pix" % (self.Npix,
self.Npix)
l_m_Diam = np.zeros((NFacets, 4), np.float32)
l_m_Diam[:, 3] = np.arange(NFacets)
Np = 10000
D = {}
for iFacet in xrange(NFacets):
D[iFacet] = {}
polygon = LPolygonNew[iFacet]
D[iFacet]["Polygon"] = polygon
lPoly, mPoly = polygon.T
ThisDiam, (l0, l1, m0, m1) = GiveDiam(polygon)
# ###############################
# # Find barycenter of polygon
# X=(np.random.rand(Np))*ThisDiam+l0
# Y=(np.random.rand(Np))*ThisDiam+m0
# XY = np.dstack((X, Y))
# XY_flat = XY.reshape((-1, 2))
# mpath = Path( polygon )
# XY = np.dstack((X, Y))
# XY_flat = XY.reshape((-1, 2))
# mask_flat = mpath.contains_points(XY_flat)
# mask=mask_flat.reshape(X.shape)
# ###############################
ThisPolygon = Polygon.Polygon(polygon)
lc, mc = ThisPolygon.center()
dl = np.max(np.abs([l0 - lc, l1 - lc]))
dm = np.max(np.abs([m0 - mc, m1 - mc]))
###############################
# lc=np.sum(X*mask)/np.sum(mask)
# mc=np.sum(Y*mask)/np.sum(mask)
# dl=np.max(np.abs(X[mask==1]-lc))
# dm=np.max(np.abs(Y[mask==1]-mc))
diam = 2 * np.max([dl, dm])
######################
# lc=(l0+l1)/2.
# mc=(m0+m1)/2.
# dl=l1-l0
# dm=m1-m0
# diam=np.max([dl,dm])
l_m_Diam[iFacet, 0] = lc
l_m_Diam[iFacet, 1] = mc
l_m_Diam[iFacet, 2] = diam
self.SpacialWeigth = {}
self.DicoImager = {}
# sort facets by size, unless we're in regular grid mode
if not regular_grid:
indDiam = np.argsort(l_m_Diam[:, 2])[::-1]
l_m_Diam = l_m_Diam[indDiam]
for iFacet in xrange(l_m_Diam.shape[0]):
self.DicoImager[iFacet] = {}
self.DicoImager[iFacet]["Polygon"] = D[l_m_Diam[iFacet,
3]]["Polygon"]
x0 = round(l_m_Diam[iFacet, 0] / self.CellSizeRad)
y0 = round(l_m_Diam[iFacet, 1] / self.CellSizeRad)
if x0 % 2 == 0:
x0 += 1
if y0 % 2 == 0:
y0 += 1
l0 = x0 * self.CellSizeRad
m0 = y0 * self.CellSizeRad
diam = round(
l_m_Diam[iFacet, 2] / self.CellSizeRad) * self.CellSizeRad
# self.AppendFacet(iFacet,l0,m0,diam)
self.AppendFacet(iFacet, l0, m0, diam)
# self.MakeMasksTessel()
NpixMax = np.max([
self.DicoImager[iFacet]["NpixFacet"]
for iFacet in sorted(self.DicoImager.keys())
])
NpixMaxPadded = np.max([
self.DicoImager[iFacet]["NpixFacetPadded"]
for iFacet in sorted(self.DicoImager.keys())
])
self.PaddedGridShape = (1, 1, NpixMaxPadded, NpixMaxPadded)
self.FacetShape = (1, 1, NpixMax, NpixMax)
dmin = 1
for iFacet in xrange(len(self.DicoImager)):
l, m = self.DicoImager[iFacet]["l0m0"]
d = np.sqrt(l**2 + m**2)
if d < dmin:
dmin = d
iCentralFacet = iFacet
self.iCentralFacet = iCentralFacet
self.NFacets = len(self.DicoImager)
# regFile="%s.tessel.reg"%self.GD["Output"]["Name"]
labels = [(self.DicoImager[i]["lmShift"][0],
self.DicoImager[i]["lmShift"][1],
"[F%i_S%i]" % (i, self.DicoImager[i]["iSol"]))
for i in xrange(len(LPolygonNew))]
VM.PolygonToReg(regFile,
LPolygonNew,
radius=0.1,
Col="green",
labels=labels)
self.WriteCoordFacetFile()
self.FacetDirections = set([
self.DicoImager[iFacet]["RaDec"]
for iFacet in range(len(self.DicoImager))
])
#DicoName = "%s.DicoFacet" % self.GD["Images"]["ImageName"]
DicoName = "%s.%sDicoFacet" % (self.GD["Output"]["Name"],
"psf." if self.DoPSF else "")
# Find the minimum l,m in the facet (for decorrelation calculation)
for iFacet in self.DicoImager.keys():
#Create smoothned facet tessel mask:
Npix = self.DicoImager[iFacet]["NpixFacetPadded"]
l0, l1, m0, m1 = self.DicoImager[iFacet]["lmExtentPadded"]
X, Y = np.mgrid[l0:l1:Npix / 10 * 1j, m0:m1:Npix / 10 * 1j]
XY = np.dstack((X, Y))
XY_flat = XY.reshape((-1, 2))
vertices = self.DicoImager[iFacet]["Polygon"]
mpath = Path(vertices) # the vertices of the polygon
mask_flat = mpath.contains_points(XY_flat)
mask = mask_flat.reshape(X.shape)
mpath = Path(self.CornersImageTot)
mask_flat2 = mpath.contains_points(XY_flat)
mask2 = mask_flat2.reshape(X.shape)
mask[mask2 == 0] = 0
R = np.sqrt(X**2 + Y**2)
R[mask == 0] = 1e6
indx, indy = np.where(R == np.min(R))
lmin, mmin = X[indx[0], indy[0]], Y[indx[0], indy[0]]
self.DicoImager[iFacet]["lm_min"] = lmin, mmin
print >> log, "Saving DicoImager in %s" % DicoName
MyPickle.Save(self.DicoImager, DicoName)
def ComputeNoiseMap(self):
print>>log, "Compute noise map..."
Boost=self.Boost
Acopy=self.Restored[0,0,0::Boost,0::Boost].copy()
SBox=(self.box[0]/Boost,self.box[1]/Boost)
# MeanAbs=scipy.ndimage.filters.mean_filter(np.abs(Acopy),SBox)
# Acopy[Acopy>0]=MeanAbs[Acopy>0]
# Noise=np.sqrt(scipy.ndimage.filters.median_filter(np.abs(Acopy)**2,SBox))
x=np.linspace(-10,10,1000)
f=0.5*(1.+scipy.special.erf(x/np.sqrt(2.)))
n=SBox[0]*SBox[1]
F=1.-(1.-f)**n
ratio=np.abs(np.interp(0.5,F,x))
Noise=-scipy.ndimage.filters.minimum_filter(Acopy,SBox)/ratio
Noise[Noise<0]=1e-10
# indxy=(Acopy>5.*Noise)
# Acopy[indxy]=5*Noise[indxy]
# Noise=np.sqrt(scipy.ndimage.filters.median_filter(np.abs(Acopy)**2,SBox))
# indxy=(Acopy>5.*Noise)
# Acopy[indxy]=5*Noise[indxy]
# Noise=np.sqrt(scipy.ndimage.filters.median_filter(np.abs(Acopy)**2,SBox))
#NoiseMed=np.median(Noise)
#Noise[Noise<NoiseMed]=NoiseMed
self.Noise=np.zeros_like(self.Restored[0,0])
for i in range(Boost):
for j in range(Boost):
s00,s01=Noise.shape
s10,s11=self.Noise[i::Boost,j::Boost].shape
s0,s1=min(s00,s10),min(s10,s11)
self.Noise[i::Boost,j::Boost][0:s0,0:s1]=Noise[:,:][0:s0,0:s1]
ind=np.where(self.Noise==0.)
self.Noise[ind]=1e-10
if self.options.OutMaskExtended:
GD=None
MaskExtended=(self.Noise<0.1*np.median(self.Noise))
OutMaskExtended=self.options.OutMaskExtended
nx=MaskExtended.shape[-1]
CurrentNegMask=np.logical_not(MaskExtended).reshape((1,1,nx,nx))
PSFServer=None
IdSharedMem=None
DicoDirty=None
IslandDistanceMachine=DDFacet.Imager.SSD.ClassIslandDistanceMachine.ClassIslandDistanceMachine(GD,
CurrentNegMask,
PSFServer,
DicoDirty,
IdSharedMem=IdSharedMem)
ListIslands=IslandDistanceMachine.SearchIslands(None,Image=self.Restored)
ListIslands=IslandDistanceMachine.ConvexifyIsland(ListIslands)#,PolygonFile="%s.pickle"%OutMaskExtended)
ListPolygons=IslandDistanceMachine.ListPolygons
MaskOut=np.zeros_like(CurrentNegMask)
N=0
for Island in ListIslands:
x,y=np.array(Island).T
#if x.size<=10: continue
MaskOut[0,0,x,y]=1
#ff,pol,_,_dec,ra=self.CasaIm.toworld((0,0,0,0))
ListPolygonsRADEC=[]
for Polygon in ListPolygons:
xx,yy=Polygon.T
ThisPolygon=[]
for iP in range(xx.shape[0]):
xcc,ycc =xx[iP],yy[iP]
ff,pol,dec,ra=self.CasaIm.toworld((0,0,xcc,ycc))
ThisPolygon.append((ra,dec))
ListPolygonsRADEC.append(np.array(ThisPolygon))
FName="%s.pickle"%OutMaskExtended
print>>log,"Saving %s"%FName
MyPickle.Save(ListPolygonsRADEC,FName)
REGName="%s.reg"%OutMaskExtended
RM=ModRegFile.PolygonNpToReg(ListPolygonsRADEC,REGName)
RM.makeRegPolyREG()
# TestArray=np.zeros_like(CurrentNegMask)
# nx=TestArray.shape[-1]
# xx,yy=20,100
# TestArray[0,0,xx,yy]=1
# PutDataInNewImage(self.FitsFile,"TestCoord.fits",np.float32(TestArray))
# ff,pol,dec,ra=self.CasaIm.toworld((0,0,xx,yy))
# pp=[[xx,yy],[yy,xx],
# [nx/2-xx,yy],[nx/2+xx,yy],
# [nx/2-yy,xx],[nx/2+yy,xx]]
# for isol in range(len(pp)):
# xx,yy=pp[isol]
# ff,pol,dec,ra=self.CasaIm.toworld((0,0,xx,yy))
# sRA =rad2hmsdms(ra,Type="ra").replace(" ",":")
# sDEC =rad2hmsdms(dec,Type="dec").replace(" ",":")
# stop
MaskOut=np.zeros_like(CurrentNegMask)
N=0
for Island in ListIslands:
x,y=np.array(Island).T
if x.size<=10: continue
MaskOut[0,0,x,y]=1
N+=1
print>>log,"Number of large enough islands %i"%N
MaskExtended=MaskOut
os.system("rm -rf %s"%OutMaskExtended)
os.system("rm -rf %s.fits"%OutMaskExtended)
PutDataInNewImage(self.FitsFile,OutMaskExtended,np.float32(MaskExtended))
NoiseMed=np.median(self.Noise)
self.Noise[self.Noise<NoiseMed]=NoiseMed
nx=self.Noise.shape[-1]
if self.options.ConvNoise:
print>>log,"Convolve..."
NoiseMap,G=ModFFTW.ConvolveGaussianWrapper(self.Noise.reshape((1,1,nx,nx)),Sig=4*SBox[0])
NoiseMap/=np.sum(G)
self.Noise=NoiseMap[0,0]
if self.OutNameNoiseMap!="":
#print>>log, "Save noise map as %s"%self.OutNameNoiseMap
#self.CasaIm.saveas(self.OutNameNoiseMap)
#CasaNoise=image(self.OutNameNoiseMap)
#CasaNoise.putdata(self.Noise)
#CasaNoise.tofits(self.OutNameNoiseMap+".fits")
#del(CasaNoise)
os.system("rm -rf %s"%self.OutNameNoiseMap)
#os.system("rm -rf %s"%self.OutNameNoiseMap+".fits")
PutDataInNewImage(self.FitsFile,self.OutNameNoiseMap,np.float32(self.Noise))
os.system("rm -rf %s.mean"%self.OutNameNoiseMap)
PutDataInNewImage(self.FitsFile,self.OutNameNoiseMap+".mean",np.float32(np.zeros_like(self.Noise)))
def main(OP=None, messages=[]):
if OP is None:
OP = MyPickle.Load(SaveFile)
print("Using settings from %s, then command line."%SaveFile)
DicoConfig = OP.DicoConfig
ImageName = DicoConfig["Output"]["Name"]
if not ImageName:
raise Exceptions.UserInputError("--Output-Name not specified, can't continue.")
if not DicoConfig["Data"]["MS"]:
raise Exceptions.UserInputError("--Data-MS not specified, can't continue.")
# create directory if it exists
dirname = os.path.dirname(ImageName)
if not os.path.exists(dirname) and not dirname == "":
os.mkdir(dirname)
# setup logging
logger.logToFile(ImageName + ".log", append=DicoConfig["Log"]["Append"])
global log
log = logger.getLogger("DDFacet")
# disable colors and progressbars if requested
ModColor.silent = SkyModel.Other.ModColor.silent = \
progressbar.ProgressBar.silent = \
DicoConfig["Log"]["Boring"]
if messages:
if not DicoConfig["Log"]["Boring"]:
#os.system('clear')
logo.print_logo()
for msg in messages:
print(msg, file=log)
print("Checking system configuration:", file=log)
# check for SHM size
ram_size = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
shm_stats = os.statvfs('/dev/shm')
shm_size = shm_stats.f_bsize * shm_stats.f_blocks
shm_relsize = shm_size / float(ram_size)
shm_avail = shm_stats.f_bsize * shm_stats.f_bavail / float(ram_size)
if shm_relsize < 0.6:
print(ModColor.Str("""WARNING: max shared memory size is only {:.0%} of total RAM size.
This can cause problems for large imaging jobs. A setting of 90% is recommended for
DDFacet and killMS. If your processes keep failing with SIGBUS or "bus error" messages,
it is most likely for this reason. You can change the memory size by running
$ sudo mount -o remount,size=90% /dev/shm
To make the change permanent, edit /etc/defaults/tmps, and add a line saying "SHM_SIZE=90%".
""".format(shm_relsize)), file=log)
else:
print(" Max shared memory size is {:.0%} of total RAM size; {:.0%} currently available".format(shm_relsize, shm_avail), file=log)
try:
output = subprocess.check_output(["/sbin/sysctl", "vm.max_map_count"],universal_newlines=True)
except Exception:
print(ModColor.Str("""WARNING: /sbin/sysctl vm.max_map_count failed. Unable to check this setting."""), file=log)
max_map_count = None
else:
max_map_count = int(output.strip().rsplit(" ", 1)[-1])
if max_map_count is not None:
if max_map_count < 500000:
print(ModColor.Str("""WARNING: sysctl vm.max_map_count = {}.
This may be too little for large DDFacet and killMS jobs. If you get strange "file exists"
errors on /dev/shm, them try to bribe, beg or threaten your friendly local sysadmin into
setting vm.max_map_count=1000000 in /etc/sysctl.conf.
""".format(max_map_count)), file=log)
else:
print(" sysctl vm.max_map_count = {}".format(max_map_count), file=log)
# check for memory lock limits
import resource
msoft, mhard = resource.getrlimit(resource.RLIMIT_MEMLOCK)
if msoft >=0 or mhard >=0:
print(ModColor.Str("""WARNING: your system has a limit on memory locks configured.
This may possibly slow down DDFacet performance. You can try removing the limit by running
$ ulimit -l unlimited
If this gives an "operation not permitted" error, you can try to bribe, beg or threaten
your friendly local sysadmin into doing
# echo "* - memlock unlimited" >> /etc/security/limits.conf
"""), file=log)
if DicoConfig["Debug"]["Pdb"] == "always":
print("--Debug-Pdb=always: unexpected errors will be dropped into pdb", file=log)
Exceptions.enable_pdb_on_error(ModColor.Str("DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n" +
"(This is because you're running with --Debug-Pdb set to 'always'.)"))
elif DicoConfig["Debug"]["Pdb"] == "auto" and not DicoConfig["Log"]["Boring"]:
print("--Debug-Pdb=auto and not --Log-Boring: unexpected errors will be dropped into pdb", file=log)
Exceptions.enable_pdb_on_error(ModColor.Str("DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n" +
"(This is because you're running with --Debug-Pdb set to 'auto' and --Log-Boring is off.)"))
# print current options
OP.Print(dest=log)
# enable memory logging
logger.enableMemoryLogging(DicoConfig["Log"]["Memory"])
# get rid of old shm arrays from previous runs
Multiprocessing.cleanupStaleShm()
# initialize random seed from config if set, or else from system time
if DicoConfig["Misc"]["RandomSeed"] is not None:
DicoConfig["Misc"]["RandomSeed"]=int(DicoConfig["Misc"]["RandomSeed"])
print("random seed=%d (explicit)" % DicoConfig["Misc"]["RandomSeed"], file=log)
else:
DicoConfig["Misc"]["RandomSeed"] = int(time.time())
print("random seed=%d (automatic)" % DicoConfig["Misc"]["RandomSeed"], file=log)
np.random.seed(DicoConfig["Misc"]["RandomSeed"])
# init NCPU for different bits of parallelism
ncpu = int(DicoConfig["Parallel"]["NCPU"] or psutil.cpu_count())
DicoConfig["Parallel"]["NCPU"]=ncpu
_pyArrays.pySetOMPNumThreads(ncpu)
NpParallel.NCPU_global = ModFFTW.NCPU_global = ncpu
numexpr.set_num_threads(ncpu)
print("using up to %d CPUs for parallelism" % ncpu, file=log)
# write parset
OP.ToParset("%s.parset"%ImageName)
Mode = DicoConfig["Output"]["Mode"]
# init semaphores, as they're needed for weight calculation too
ClassFacetMachine.ClassFacetMachine.setup_semaphores(DicoConfig)
# data machine initialized for all cases except PSF-only mode
# psf machine initialized for all cases except Predict-only mode
Imager = ClassDeconvMachine.ClassImagerDeconv(GD=DicoConfig,
BaseName=ImageName,
predict_only=(Mode == "Predict" or Mode == "Subtract"),
data=(Mode != "PSF"),
psf=(Mode != "Predict" and Mode != "Dirty" and Mode != "Subtract"),
readcol=(Mode != "Predict" and Mode != "PSF"),
deconvolve=("Clean" in Mode))
Imager.Init()
# Imager.testDegrid()
# stop
if "Predict" in Mode or "Subtract" in Mode:
Imager.GivePredict()
if "Clean" in Mode:
Imager.main()
elif "Dirty" in Mode:
sparsify = DicoConfig["Comp"]["Sparsification"]
if sparsify and isinstance(sparsify, list):
sparsify = sparsify[0]
Imager.GiveDirty(psf="PSF" in Mode, sparsify=sparsify)
elif "PSF" in Mode:
sparsify = DicoConfig["Comp"]["Sparsification"]
if sparsify and isinstance(sparsify, list):
sparsify = sparsify[0]
Imager.MakePSF(sparsify=sparsify)
elif "RestoreAndShift" == Mode:
Imager.RestoreAndShift()
def testMO_DATA():
Dico= MyPickle.Load("SaveTest")
Dirty=Dico["Dirty"]
PSF=Dico["PSF"]
PSF2=np.squeeze(PSF)
ListPixData=Dico["ListPixData"]
FreqsInfo=Dico["FreqsInfo"]
FreqsInfo=Dico["FreqsInfo"]
IslandBestIndiv=Dico["IslandBestIndiv"]
ListPixParms=ListPixData
ListSquarePix=Dico["ListSquarePix"]
ThisPixList=ListPixData
ThisSquarePixList=ListSquarePix
GD=Dico["GD"]
FacetID=Dico["FacetID"]
nch=FreqsInfo["MeanJonesBand"][FacetID].size
WeightMeanJonesBand=FreqsInfo["MeanJonesBand"][FacetID].reshape((nch,1,1,1))
WeightMueller=WeightMeanJonesBand.ravel()
WeightMuellerSignal=WeightMueller*FreqsInfo["WeightChansImages"].ravel()
#IncreaseIslandMachine=ClassIncreaseIsland.ClassIncreaseIsland()
#ListPixData=IncreaseIslandMachine.IncreaseIsland(ListPixData,dx=5)
# 0) Load Island info (center and square data)
ListSquarePix_Center=ListSquarePix['IslandCenter']
ListSquarePix_Data=ListSquarePix['IslandSquareData']
ListSquarePix_Mask=ListSquarePix['IslandSquareMask']
orixisland,oriyisland=ListSquarePix_Data.shape # size of the square postage stamp around island
ListSquarePix_Data=PSF2
xisland,yisland=ListSquarePix_Data.shape # size of the square postage stamp around island
print(xisland)
# 1) Shape PSF and Dirty to have even number of pixels (required by Moresane)
# DEAL WITH SQUARE DATA OF ISLAND IF UNEVEN
# Crop PSF to the island postage stamp
#PSFCrop=CropPSF(PSF,xisland)
#PSF2=CropPSF(PSF2,71)
# MORESANE requires even sized images ==> Padding by one row and one column
cropped_square_data_to_even = False
if xisland % 2 != 0:
# PSFCrop_even = np.zeros((xisland+1, xisland+1))
# PSFCrop_even[:-1, :-1] = np.squeeze(PSFCrop)
Dirty_even = np.zeros((xisland - 1, xisland - 1))
Dirty_even[:, :] = ListSquarePix_Data[:-1, :-1]
cropped_square_data_to_even = True
else:
Dirty_even = ListSquarePix_Data
# make it even by removing one line and one column (usually outside of the interesting island region)
# xbigdirty,ybigdirty=np.squeeze(Dirty).shape
# if xbigdirty % 2 != 0:
# Dirty_even=np.zeros((xbigdirty-1,xbigdirty-1))
# Dirty_even[:,:]=np.squeeze(Dirty)[:-1,:-1]
xbigpsf,ybigpsf=PSF2.shape
cropped_square_psf_to_even = False
if xbigpsf % 2 != 0:
PSF2_even=np.zeros((xbigpsf-1,xbigpsf-1))
PSF2_even[:,:]=PSF2[:-1,:-1]
cropped_square_data_to_even = True
else:
PSF2_even=PSF2
# 2) Run the actual MinorCycle algo
DictMoresaneParms=GD['MORESANE']
Moresane=ClassMoresane(Dirty_even,PSF2_even,DictMoresaneParms,GD=GD)
Model_Square=Moresane.main()
# 3) Apply Island mask to model to get rid of regions outside the island.
cropped_square_to_even = False
if cropped_square_data_to_even: # then restore the model to its original uneven dimension
Model_Square_uneven = np.zeros((xisland, xisland))
Model_Square_uneven[:-1, :-1] = Model_Square
Model_Square = Model_Square_uneven
if cropped_square_psf_to_even: # restore original PSF size
PSF_uneven=PSF2
Model_Square=CropPSF(Model_Square, orixisland)
Model_Square *= ListSquarePix_Mask # masking outside the island
# 4) Convert back to Island format ( "S" and ThisPixList )
NewModel, NewThisPixList = SquareIslandtoIsland(Model_Square, ThisSquarePixList, ThisPixList)
Model = NewModel
ThisPixList = NewThisPixList
return Model
