def fix_dico_shape(fulldico, outdico, NPixOut):
# dico_model = 'image_full_ampphase_di_m.NS.DicoModel'
# save_dico = dico_model.replace('.DicoModel', '.restricted.DicoModel')
# NPixOut = 10000
dico = MyPickle.Load(fulldico)
NPix = dico['ModelShape'][-1]
NPix0, _ = EstimateNpix(float(NPix), Padding=1)
if NPix != NPix0:
raise ValueError("NPix != NPix0")
logger.info("Changing image size: %i -> %i pixels" % (NPix, NPixOut))
xc0 = NPix // 2
xc1 = NPixOut // 2
dx = xc0 - xc1
DCompOut = {}
for k, v in dico.items():
if k == 'Comp':
DCompOut['Comp'] = {}
continue
DCompOut[k] = v
DCompOut["Type"] = "SSD"
N, M, _, _ = dico['ModelShape']
DCompOut['ModelShape'] = [N, M, NPixOut, NPixOut]
for (x0, y0) in dico['Comp'].keys():
x1 = x0 - dx
y1 = y0 - dx
c0 = (x1 >= 0) & (x1 < NPixOut)
c1 = (y1 >= 0) & (y1 < NPixOut)
if c0 & c1:
logger.info("Mapping (%i,%i)->(%i,%i)" % (x0, y0, x1, y1))
DCompOut['Comp'][(x1, y1)] = dico['Comp'][(x0, y0)]
logger.info("Saving in {}".format(outdico))
MyPickle.Save(DCompOut, outdico)
def __init__(self,
fn,
phasedir,
facet_padding_factor,
max_facet_size,
min_nfacet_per_axis,
clustercat=None):
self.__dicofn = fn
if fn not in DicoSourceProvider.__cached_model_machines:
self.__dicomodel = MMM.ClassModModelMachine(
).GiveInitialisedMMFromFile(fn)
DicoSourceProvider.__cached_model_machines[fn] = self.__dicomodel
else:
log(2).print(
"Reusing previously initialized DDFacet model machine '{0:s}'".
format(fn))
self.__dicomodel = DicoSourceProvider.__cached_model_machines[fn]
# assume gd is stored in the dico... this is only true for ddf versions 0.4.1 and beyond
self.__pxscale = self.__dicomodel.GD["Image"]["Cell"] # in arcseconds
self.__nchan, self.__npol, self.__ny, self.__nx = self.__dicomodel.ModelShape
assert self.__ny == self.__nx # check cyril's model is square
assert self.__nx % 2 == 1 # must be odd for the logic downstairs to work
# regions relative to the unpadded images
req_npx_img = self.__dicomodel.GD["Image"]["NPix"]
nx_image, nx_image_padded = EstimateNpix(
req_npx_img, Padding=self.__dicomodel.GD["Facets"]["Padding"])
assert nx_image == self.__nx # the predicted image size better correspond to dicomodels shape
self.__padded_nx = self.__padded_ny = nx_image_padded
self.__phasedir = phasedir
cachename = DicoSourceProvider.__cachename_compute(fn, clustercat)
if cachename not in DicoSourceProvider.__cached_providers:
log.info(
"Initializing new source provider for DDFacet model '{0:s}' into regions specified by '{1:s}'."
.format(fn, clustercat if clustercat is not None else 'die'))
self.__clustercat = self.__read_regions_file(
clustercat, facet_padding_factor, max_facet_size,
min_nfacet_per_axis)
DicoSourceProvider.__cached_providers[
cachename] = self.__clustercat
log(2).print(
"initialization sequence of source provider '{0:s}' (regions '{1:s}') completed"
.format(fn, clustercat if clustercat is not None else 'die'))
else:
self.__clustercat = DicoSourceProvider.__cached_providers[
cachename]
log(2).print(
"reused previous initialization of source provider '{0:s}' (regions '{1:s}')"
.format(fn, clustercat if clustercat is not None else 'die'))
self.__current_direction = 0
self.__degridcube = None
self.__degridfreqs = None
self.__padding_factor = facet_padding_factor
def __pad_cluster(c, padding_factor):
npx, _ = c.box_npx # square facet at this point
# this returns an odd npix:
npixunpadded, npixpadded = EstimateNpix(npx,
Padding=padding_factor)
return BoundingBoxFactory.PadBox(c,
npixpadded,
npixpadded,
check_mask_outofbounds=False)
def __init__(
self,
GD,
ChanFreq,
Npix,
lmShift=(0., 0.),
IDFacet=0,
SpheNorm=True,
NFreqBands=1,
DataCorrelationFormat=[5, 6, 7, 8],
ExpectedOutputStokes=[1],
ListSemaphores=None,
cf_dict=None,
compute_cf=False,
wmax=None, # must be supplied if compute_cf=True
bda_grid=None,
bda_degrid=None,
):
"""
Args:
GD:
ChanFreq:
Npix:
lmShift:
IdSharedMem:
IdSharedMemData:
FacetDataCache:
ChunkDataCache:
IDFacet:
SpheNorm:
NFreqBands:
DataCorrelationFormat:
ExpectedOutputStokes:
ListSemaphores:
cf_dict: SharedDict from/to which WTerms and Sphes are saved
compute_cf: if True, wterm/sphe is recomputed and saved to store_dict
"""
T = ClassTimeIt.ClassTimeIt("Init_ClassDDEGridMachine")
T.disable()
self.GD = GD
self.IDFacet = IDFacet
self.SpheNorm = SpheNorm
self.ListSemaphores = ListSemaphores
self._bda_grid = bda_grid
self._bda_degrid = bda_degrid
# self.DoPSF=DoPSF
self.DoPSF = False
# if DoPSF:
# self.DoPSF=True
# Npix=Npix*2
Precision = GD["RIME"]["Precision"]
PolMode = ExpectedOutputStokes
if Precision == "S":
self.dtype = np.complex64
elif Precision == "D":
self.dtype = np.complex128
self.dtype = np.complex64
T.timeit("0")
Padding = GD["Facets"]["Padding"]
self.NonPaddedNpix, Npix = EstimateNpix(Npix, Padding)
self.Padding = Npix / float(self.NonPaddedNpix)
# self.Padding=Padding
self.LSmear = []
self.PolMode = PolMode
# SkyType & JonesType
# 0: scalar
# 1: diag
# 2: full
#if PolMode == "I":
# self.npol = 1
# self.PolMap = np.array([0, 5, 5, 0], np.int32)
# self.SkyType = 1
# self.PolModeID = 0
#elif PolMode == "IQUV":
# self.SkyType = 2
# self.npol = 4
# self.PolMap = np.array([0, 1, 2, 3], np.int32)
# self.PolModeID = 1
#DEPRICATION:
#These are only to be used in the degridder, they are depricated for the gridder
self.npol = len(ExpectedOutputStokes)
self.SkyType = 1
self.PolMap = np.array([0, 5, 5, 0], np.int32)
self.PolModeID = 0
self.Npix = Npix
self.NFreqBands = NFreqBands
self.NonPaddedShape = (self.NFreqBands, self.npol, self.NonPaddedNpix,
self.NonPaddedNpix)
self.GridShape = (self.NFreqBands, self.npol, self.Npix, self.Npix)
x0 = (self.Npix - self.NonPaddedNpix) / 2 # +1
self.PaddingInnerCoord = (x0, x0 + self.NonPaddedNpix)
T.timeit("1")
OverS = GD["CF"]["OverS"]
Support = GD["CF"]["Support"]
Nw = GD["CF"]["Nw"]
Cell = GD["Image"]["Cell"]
# T=ClassTimeIt.ClassTimeIt("ClassImager")
# T.disable()
self.Cell = Cell
self.incr = (np.array([-Cell, Cell], dtype=np.float64) /
3600.) * (np.pi / 180)
# CF.fill(1.)
# print self.ChanEquidistant
# self.FullScalarMode=int(GD["DDESolutions"]["FullScalarMode"])
# self.FullScalarMode=0
JonesMode = GD["DDESolutions"]["JonesMode"]
if JonesMode == "Scalar":
self.JonesType = 0
elif JonesMode == "Diag":
self.JonesType = 1
elif JonesMode == "Full":
self.JonesType = 2
T.timeit("3")
self.ChanFreq = ChanFreq
self.Sup = Support
self.WProj = True
self.Nw = Nw
self.OverS = OverS
self.lmShift = lmShift
T.timeit("4")
# if neither is set, then machine is being constructed for ffts only
if cf_dict or compute_cf:
self.InitCF(cf_dict, compute_cf, wmax)
T.timeit("5")
self.reinitGrid()
self.CasaImage = None
self.DicoATerm = None
T.timeit("6")
self.DataCorrelationFormat = DataCorrelationFormat
self.ExpectedOutputStokes = ExpectedOutputStokes
self._fftw_machine = None
def setFacetsLocs(self):
NFacets = self.NFacets
Npix = self.GD["Image"]["NPix"]
Padding = self.GD["Facets"]["Padding"]
self.Padding = Padding
Npix, _ = EstimateNpix(float(Npix), Padding=1)
self.Npix = Npix
self.OutImShape = (self.nch, self.npol, self.Npix, self.Npix)
RadiusTot = self.CellSizeRad * self.Npix / 2
self.RadiusTot = RadiusTot
lMainCenter, mMainCenter = 0., 0.
self.lmMainCenter = lMainCenter, mMainCenter
self.CornersImageTot = np.array(
[[lMainCenter - RadiusTot, mMainCenter - RadiusTot],
[lMainCenter + RadiusTot, mMainCenter - RadiusTot],
[lMainCenter + RadiusTot, mMainCenter + RadiusTot],
[lMainCenter - RadiusTot, mMainCenter + RadiusTot]])
# MSName = self.GD["Data"]["MS"]
# if ".txt" in MSName:
# f = open(MSName)
# Ls = f.readlines()
# f.close()
# MSName = []
# for l in Ls:
# ll = l.replace("\n", "")
# MSName.append(ll)
# MSName = MSName[0]
MSName = self.VS.ListMS[0].MSName
SolsFile = self.GD["DDESolutions"]["DDSols"]
if isinstance(SolsFile, list):
SolsFile = self.GD["DDESolutions"]["DDSols"][0]
if SolsFile and (not (".npz" in SolsFile)) and (not (".h5"
in SolsFile)):
Method = SolsFile
ThisMSName = reformat.reformat(os.path.abspath(MSName),
LastSlash=False)
SolsFile = "%s/killMS.%s.sols.npz" % (ThisMSName, Method)
# if "CatNodes" in self.GD.keys():
regular_grid = False
if self.GD["Facets"]["CatNodes"] is not None:
print >> log, "Taking facet directions from Nodes catalog: %s" % self.GD[
"Facets"]["CatNodes"]
ClusterNodes = np.load(self.GD["Facets"]["CatNodes"])
ClusterNodes = ClusterNodes.view(np.recarray)
raNode = ClusterNodes.ra
decNode = ClusterNodes.dec
lFacet, mFacet = self.CoordMachine.radec2lm(raNode, decNode)
elif ".npz" in SolsFile:
print >> log, "Taking facet directions from solutions file: %s" % SolsFile
ClusterNodes = np.load(SolsFile)["ClusterCat"]
ClusterNodes = ClusterNodes.view(np.recarray)
raNode = ClusterNodes.ra
decNode = ClusterNodes.dec
lFacet, mFacet = self.CoordMachine.radec2lm(raNode, decNode)
elif ".h5" in SolsFile:
print >> log, "Taking facet directions from HDF5 solutions file: %s" % SolsFile
H = tables.open_file(SolsFile)
raNode, decNode = H.root.sol000.source[:]["dir"].T
lFacet, mFacet = self.CoordMachine.radec2lm(raNode, decNode)
H.close()
del (H)
else:
print >> log, "Taking facet directions from regular grid"
regular_grid = True
CellSizeRad = (self.GD["Image"]["Cell"] / 3600.) * np.pi / 180
lrad = Npix * CellSizeRad * 0.5
NpixFacet = Npix / NFacets
lfacet = NpixFacet * CellSizeRad * 0.5
lcenter_max = lrad - lfacet
lFacet, mFacet, = np.mgrid[-lcenter_max:lcenter_max:(NFacets) * 1j,
-lcenter_max:lcenter_max:(NFacets) * 1j]
lFacet = lFacet.flatten()
mFacet = mFacet.flatten()
print >> log, " There are %i Jones-directions" % lFacet.size
self.lmSols = lFacet.copy(), mFacet.copy()
raSols, decSols = self.CoordMachine.lm2radec(lFacet.copy(),
mFacet.copy())
self.radecSols = raSols, decSols
NodesCat = np.zeros((raSols.size, ),
dtype=[('ra', np.float), ('dec', np.float),
('l', np.float), ('m', np.float)])
NodesCat = NodesCat.view(np.recarray)
NodesCat.ra = raSols
NodesCat.dec = decSols
# print>>log,"Facet RA %s"%raSols
# print>>log,"Facet Dec %s"%decSols
NodesCat.l = lFacet
NodesCat.m = mFacet
## saving below
# NodeFile = "%s.NodesCat.%snpy" % (self.GD["Output"]["Name"], "psf." if self.DoPSF else "")
# print>> log, "Saving Nodes catalog in %s" % NodeFile
# np.save(NodeFile, NodesCat)
self.DicoImager = {}
xy = np.zeros((lFacet.size, 2), np.float32)
xy[:, 0] = lFacet
xy[:, 1] = mFacet
regFile = "%s.tessel0.reg" % self.ImageName
NFacets = self.NFacets = lFacet.size
rac, decc = self.MainRaDec
VM = ModVoronoiToReg.VoronoiToReg(rac, decc)
if NFacets > 2:
vor = Voronoi(xy, furthest_site=False)
regions, vertices = ModVoronoi.voronoi_finite_polygons_2d(
vor, radius=1.)
PP = Polygon.Polygon(self.CornersImageTot)
LPolygon = []
ListNode = []
for region, iNode in zip(regions, range(NodesCat.shape[0])):
ThisP = np.array(PP
& Polygon.Polygon(np.array(vertices[region])))
if ThisP.size > 0:
LPolygon.append(ThisP[0])
ListNode.append(iNode)
NodesCat = NodesCat[np.array(ListNode)].copy()
# =======
# LPolygon = [
# np.array(PP & Polygon.Polygon(np.array(vertices[region])))[0]
# for region in regions]
# >>>>>>> issue-255
elif NFacets == 1:
l0, m0 = lFacet[0], mFacet[0]
LPolygon = [self.CornersImageTot]
# VM.ToReg(regFile,lFacet,mFacet,radius=.1)
NodeFile = "%s.NodesCat.npy" % self.GD["Output"]["Name"]
print >> log, "Saving Nodes catalog in %s" % NodeFile
np.save(NodeFile, NodesCat)
for iFacet, polygon0 in zip(range(len(LPolygon)), LPolygon):
# polygon0 = vertices[region]
P = polygon0.tolist()
# VM.PolygonToReg(regFile,LPolygon,radius=0.1,Col="red")
# stop
###########################################
# SubDivide
def GiveDiam(polygon):
lPoly, mPoly = polygon.T
l0 = np.max([lMainCenter - RadiusTot, lPoly.min()])
l1 = np.min([lMainCenter + RadiusTot, lPoly.max()])
m0 = np.max([mMainCenter - RadiusTot, mPoly.min()])
m1 = np.min([mMainCenter + RadiusTot, mPoly.max()])
dl = l1 - l0
dm = m1 - m0
diam = np.max([dl, dm])
return diam, (l0, l1, m0, m1)
DiamMax = self.GD["Facets"]["DiamMax"] * np.pi / 180
# DiamMax=4.5*np.pi/180
DiamMin = self.GD["Facets"]["DiamMin"] * np.pi / 180
def ClosePolygon(polygon):
P = polygon.tolist()
polygon = np.array(P + [P[0]])
return polygon
def GiveSubDivideRegions(polygonFacet, DMax):
polygonFOV = self.CornersImageTot
# polygonFOV=ClosePolygon(polygonFOV)
PFOV = Polygon.Polygon(polygonFOV)
# polygonFacet=ClosePolygon(polygonFacet)
P0 = Polygon.Polygon(polygonFacet)
P0Cut = Polygon.Polygon(P0 & PFOV)
if P0Cut.nPoints() == 0:
return []
polygonFacetCut = np.array(P0Cut[0])
# polygonFacetCut=ClosePolygon(polygonFacetCut)
diam, (l0, l1, m0, m1) = GiveDiam(polygonFacetCut)
if diam < DMax:
return [polygonFacetCut]
Nl = int((l1 - l0) / DMax) + 1
Nm = int((m1 - m0) / DMax) + 1
dl = (l1 - l0) / Nl
dm = (m1 - m0) / Nm
lEdge = np.linspace(l0, l1, Nl + 1)
mEdge = np.linspace(m0, m1, Nm + 1)
lc = (lEdge[0:-1] + lEdge[1::]) / 2
mc = (mEdge[0:-1] + mEdge[1::]) / 2
LPoly = []
Lc, Mc = np.meshgrid(lc, mc)
Lc = Lc.ravel().tolist()
Mc = Mc.ravel().tolist()
DpolySquare = np.array([[-dl, -dm], [dl, -dm], [dl, dm], [-dl, dm]
]) * 0.5
for lc, mc in zip(Lc, Mc):
polySquare = DpolySquare.copy(
) # ClosePolygon(DpolySquare.copy())
polySquare[:, 0] += lc
polySquare[:, 1] += mc
# polySquare=ClosePolygon(polySquare)
P1 = Polygon.Polygon(polySquare)
POut = (P0Cut & P1)
if POut.nPoints() == 0:
continue
polyOut = np.array(POut[0])
# polyOut=ClosePolygon(polyOut)
LPoly.append(polyOut)
# pylab.clf()
# x,y=polygonFacetCut.T
# pylab.plot(x,y,color="blue")
# x,y=polygonFacet.T
# pylab.plot(x,y,color="blue",ls=":",lw=3)
# x,y=np.array(PFOV[0]).T
# pylab.plot(x,y,color="black")
# x,y=polySquare.T
# pylab.plot(x,y,color="green",ls=":",lw=3)
# x,y=polyOut.T
# pylab.plot(x,y,color="red",ls="--",lw=3)
# pylab.xlim(-0.03,0.03)
# pylab.ylim(-0.03,0.03)
# pylab.draw()
# pylab.show(False)
# pylab.pause(0.5)
return LPoly
def PlotPolygon(P, *args, **kwargs):
for poly in P:
x, y = ClosePolygon(np.array(poly)).T
pylab.plot(x, y, *args, **kwargs)
LPolygonNew = []
for iFacet in xrange(len(LPolygon)):
polygon = LPolygon[iFacet]
ThisDiamMax = DiamMax
SubReg = GiveSubDivideRegions(polygon, ThisDiamMax)
LPolygonNew += SubReg
regFile = "%s.FacetMachine.tessel.ReCut.reg" % self.ImageName
# VM.PolygonToReg(regFile,LPolygonNew,radius=0.1,Col="green",labels=[str(i) for i in range(len(LPolygonNew))])
DicoPolygon = {}
for iFacet in xrange(len(LPolygonNew)):
DicoPolygon[iFacet] = {}
poly = LPolygonNew[iFacet]
DicoPolygon[iFacet]["poly"] = poly
diam, (l0, l1, m0, m1) = GiveDiam(poly)
DicoPolygon[iFacet]["diam"] = diam
DicoPolygon[iFacet]["diamMin"] = np.min([(l1 - l0), (m1 - m0)])
xc, yc = np.mean(poly[:, 0]), np.mean(poly[:, 1])
DicoPolygon[iFacet]["xyc"] = xc, yc
dSol = np.sqrt((xc - lFacet)**2 + (yc - mFacet)**2)
DicoPolygon[iFacet]["iSol"] = np.where(dSol == np.min(dSol))[0]
for iFacet in sorted(DicoPolygon.keys()):
diam = DicoPolygon[iFacet]["diamMin"]
# print iFacet,diam,DiamMin
if diam < DiamMin:
dmin = 1e6
xc0, yc0 = DicoPolygon[iFacet]["xyc"]
HasClosest = False
for iFacetOther in sorted(DicoPolygon.keys()):
if iFacetOther == iFacet:
continue
iSolOther = DicoPolygon[iFacetOther]["iSol"]
# print " ",iSolOther,DicoPolygon[iFacet]["iSol"]
if iSolOther != DicoPolygon[iFacet]["iSol"]:
continue
xc, yc = DicoPolygon[iFacetOther]["xyc"]
d = np.sqrt((xc - xc0)**2 + (yc - yc0)**2)
if d < dmin:
dmin = d
iFacetClosest = iFacetOther
HasClosest = True
if (HasClosest):
print >> log, "Merging facet #%i to #%i" % (iFacet,
iFacetClosest)
P0 = Polygon.Polygon(DicoPolygon[iFacet]["poly"])
P1 = Polygon.Polygon(DicoPolygon[iFacetClosest]["poly"])
P2 = (P0 | P1)
POut = []
for iP in xrange(len(P2)):
POut += P2[iP]
poly = np.array(POut)
hull = ConvexHull(poly)
Contour = np.array([
hull.points[hull.vertices, 0],
hull.points[hull.vertices, 1]
])
poly2 = Contour.T
del (DicoPolygon[iFacet])
DicoPolygon[iFacetClosest]["poly"] = poly2
DicoPolygon[iFacetClosest]["diam"] = GiveDiam(poly2)[0]
DicoPolygon[iFacetClosest]["xyc"] = np.mean(
poly2[:, 0]), np.mean(poly2[:, 1])
# stop
LPolygonNew = []
for iFacet in sorted(DicoPolygon.keys()):
# if DicoPolygon[iFacet]["diam"]<DiamMin:
# print>>log, ModColor.Str(" Facet #%i associated to direction #%i is too small, removing it"%(iFacet,DicoPolygon[iFacet]["iSol"]))
# continue
LPolygonNew.append(DicoPolygon[iFacet]["poly"])
# for iFacet in range(len(regions)):
# polygon=LPolygon[iFacet]
# ThisDiamMax=DiamMax
# while True:
# SubReg=GiveSubDivideRegions(polygon,ThisDiamMax)
# if SubReg==[]:
# break
# Diams=[GiveDiam(poly)[0] for poly in SubReg]
# if np.min(Diams)>DiamMin: break
# ThisDiamMax*=1.1
# LPolygonNew+=SubReg
# print
regFile = "%s.tessel.%sreg" % (self.GD["Output"]["Name"],
"psf." if self.DoPSF else "")
# labels=["[F%i.C%i]"%(i,DicoPolygon[i]["iSol"]) for i in range(len(LPolygonNew))]
# VM.PolygonToReg(regFile,LPolygonNew,radius=0.1,Col="green",labels=labels)
# VM.PolygonToReg(regFile,LPolygonNew,radius=0.1,Col="green")
# pylab.clf()
# x,y=LPolygonNew[11].T
# pylab.plot(x,y)
# pylab.draw()
# pylab.show()
# stop
###########################################
NFacets = len(LPolygonNew)
NJonesDir = NodesCat.shape[0]
self.JonesDirCat = np.zeros(
(NodesCat.shape[0], ),
dtype=[('Name', '|S200'), ('ra', np.float), ('dec', np.float),
('SumI', np.float), ("Cluster", int), ("l", np.float),
("m", np.float), ("I", np.float)])
self.JonesDirCat = self.JonesDirCat.view(np.recarray)
self.JonesDirCat.I = 1
self.JonesDirCat.SumI = 1
self.JonesDirCat.ra = NodesCat.ra
self.JonesDirCat.dec = NodesCat.dec
self.JonesDirCat.l = NodesCat.l
self.JonesDirCat.m = NodesCat.m
self.JonesDirCat.Cluster = range(NJonesDir)
print >> log, "Sizes (%i facets):" % (self.JonesDirCat.shape[0])
print >> log, " - Main field : [%i x %i] pix" % (self.Npix,
self.Npix)
l_m_Diam = np.zeros((NFacets, 4), np.float32)
l_m_Diam[:, 3] = np.arange(NFacets)
Np = 10000
D = {}
for iFacet in xrange(NFacets):
D[iFacet] = {}
polygon = LPolygonNew[iFacet]
D[iFacet]["Polygon"] = polygon
lPoly, mPoly = polygon.T
ThisDiam, (l0, l1, m0, m1) = GiveDiam(polygon)
# ###############################
# # Find barycenter of polygon
# X=(np.random.rand(Np))*ThisDiam+l0
# Y=(np.random.rand(Np))*ThisDiam+m0
# XY = np.dstack((X, Y))
# XY_flat = XY.reshape((-1, 2))
# mpath = Path( polygon )
# XY = np.dstack((X, Y))
# XY_flat = XY.reshape((-1, 2))
# mask_flat = mpath.contains_points(XY_flat)
# mask=mask_flat.reshape(X.shape)
# ###############################
ThisPolygon = Polygon.Polygon(polygon)
lc, mc = ThisPolygon.center()
dl = np.max(np.abs([l0 - lc, l1 - lc]))
dm = np.max(np.abs([m0 - mc, m1 - mc]))
###############################
# lc=np.sum(X*mask)/np.sum(mask)
# mc=np.sum(Y*mask)/np.sum(mask)
# dl=np.max(np.abs(X[mask==1]-lc))
# dm=np.max(np.abs(Y[mask==1]-mc))
diam = 2 * np.max([dl, dm])
######################
# lc=(l0+l1)/2.
# mc=(m0+m1)/2.
# dl=l1-l0
# dm=m1-m0
# diam=np.max([dl,dm])
l_m_Diam[iFacet, 0] = lc
l_m_Diam[iFacet, 1] = mc
l_m_Diam[iFacet, 2] = diam
self.SpacialWeigth = {}
self.DicoImager = {}
# sort facets by size, unless we're in regular grid mode
if not regular_grid:
indDiam = np.argsort(l_m_Diam[:, 2])[::-1]
l_m_Diam = l_m_Diam[indDiam]
for iFacet in xrange(l_m_Diam.shape[0]):
self.DicoImager[iFacet] = {}
self.DicoImager[iFacet]["Polygon"] = D[l_m_Diam[iFacet,
3]]["Polygon"]
x0 = round(l_m_Diam[iFacet, 0] / self.CellSizeRad)
y0 = round(l_m_Diam[iFacet, 1] / self.CellSizeRad)
if x0 % 2 == 0:
x0 += 1
if y0 % 2 == 0:
y0 += 1
l0 = x0 * self.CellSizeRad
m0 = y0 * self.CellSizeRad
diam = round(
l_m_Diam[iFacet, 2] / self.CellSizeRad) * self.CellSizeRad
# self.AppendFacet(iFacet,l0,m0,diam)
self.AppendFacet(iFacet, l0, m0, diam)
# self.MakeMasksTessel()
NpixMax = np.max([
self.DicoImager[iFacet]["NpixFacet"]
for iFacet in sorted(self.DicoImager.keys())
])
NpixMaxPadded = np.max([
self.DicoImager[iFacet]["NpixFacetPadded"]
for iFacet in sorted(self.DicoImager.keys())
])
self.PaddedGridShape = (1, 1, NpixMaxPadded, NpixMaxPadded)
self.FacetShape = (1, 1, NpixMax, NpixMax)
dmin = 1
for iFacet in xrange(len(self.DicoImager)):
l, m = self.DicoImager[iFacet]["l0m0"]
d = np.sqrt(l**2 + m**2)
if d < dmin:
dmin = d
iCentralFacet = iFacet
self.iCentralFacet = iCentralFacet
self.NFacets = len(self.DicoImager)
# regFile="%s.tessel.reg"%self.GD["Output"]["Name"]
labels = [(self.DicoImager[i]["lmShift"][0],
self.DicoImager[i]["lmShift"][1],
"[F%i_S%i]" % (i, self.DicoImager[i]["iSol"]))
for i in xrange(len(LPolygonNew))]
VM.PolygonToReg(regFile,
LPolygonNew,
radius=0.1,
Col="green",
labels=labels)
self.WriteCoordFacetFile()
self.FacetDirections = set([
self.DicoImager[iFacet]["RaDec"]
for iFacet in range(len(self.DicoImager))
])
#DicoName = "%s.DicoFacet" % self.GD["Images"]["ImageName"]
DicoName = "%s.%sDicoFacet" % (self.GD["Output"]["Name"],
"psf." if self.DoPSF else "")
# Find the minimum l,m in the facet (for decorrelation calculation)
for iFacet in self.DicoImager.keys():
#Create smoothned facet tessel mask:
Npix = self.DicoImager[iFacet]["NpixFacetPadded"]
l0, l1, m0, m1 = self.DicoImager[iFacet]["lmExtentPadded"]
X, Y = np.mgrid[l0:l1:Npix / 10 * 1j, m0:m1:Npix / 10 * 1j]
XY = np.dstack((X, Y))
XY_flat = XY.reshape((-1, 2))
vertices = self.DicoImager[iFacet]["Polygon"]
mpath = Path(vertices) # the vertices of the polygon
mask_flat = mpath.contains_points(XY_flat)
mask = mask_flat.reshape(X.shape)
mpath = Path(self.CornersImageTot)
mask_flat2 = mpath.contains_points(XY_flat)
mask2 = mask_flat2.reshape(X.shape)
mask[mask2 == 0] = 0
R = np.sqrt(X**2 + Y**2)
R[mask == 0] = 1e6
indx, indy = np.where(R == np.min(R))
lmin, mmin = X[indx[0], indy[0]], Y[indx[0], indy[0]]
self.DicoImager[iFacet]["lm_min"] = lmin, mmin
print >> log, "Saving DicoImager in %s" % DicoName
MyPickle.Save(self.DicoImager, DicoName)
def main(data_dir, working_dir, ncpu, keeplongbaselines, chunkhours, predict_column, sub_column, npix_out):
data_dir = os.path.abspath(data_dir)
working_dir = os.path.abspath(working_dir)
try:
os.makedirs(working_dir)
except:
pass
try:
os.makedirs(os.path.join(working_dir, 'SOLSDIR'))
except:
pass
os.chdir(working_dir)
solsdir = os.path.join(data_dir, 'SOLSDIR')
mslist_file, mslist, fullmask, indico, clustercat = get_filenames(data_dir)
predict_dico = os.path.join(data_dir,
'image_full_ampphase_di_m.NS.not_{sub_column}.DicoModel'.format(sub_column=sub_column))
filtered_dico = os.path.join(data_dir,
'image_full_ampphase_di_m.NS.{sub_column}.DicoModel'.format(sub_column=sub_column))
predict_mask = os.path.join(data_dir, 'predict_mask_{sub_column}.fits'.format(
sub_column=sub_column)) # just a name, can be anything
region_file = os.path.join(working_dir, 'box_subtract_region.reg')
if keeplongbaselines:
uvsel = "[0.100000,5000.000000]"
else:
uvsel = "[0.100000,1000.000000]"
robust = -0.5
imagecell = 1.5
data_colname = 'DATA'
columnchecker(mslist, data_colname)
imagenpix = getimsize(fullmask)
npix_out, _ = EstimateNpix(float(npix_out), Padding=1)
# predict
if os.path.isfile(predict_dico):
os.unlink(predict_dico)
if os.path.isfile(predict_mask):
os.unlink(predict_mask)
if os.path.isfile(filtered_dico):
os.unlink(filtered_dico)
make_predict_mask(fullmask, region_file, predict_mask, npix_out=npix_out)
make_predict_dico(indico, predict_dico, predict_mask, npix_out=npix_out)
make_filtered_dico(fullmask, predict_mask, indico, filtered_dico, npix_out=npix_out)
args = dict(chunkhours=chunkhours, mslist_file=mslist_file, data_colname=data_colname, ncpu=ncpu,
clustercat=clustercat,
robust=robust, imagenpix=imagenpix, imagecell=imagecell, predict_mask=predict_mask,
predict_dico=predict_dico, uvsel=uvsel,
solsdir=solsdir, predict_column=predict_column)
logger.info("Predicting...")
cmd_call("DDF.py --Output-Name=image_full_ampphase_di_m.NS_SUB --Data-ChunkHours={chunkhours} --Data-MS={mslist_file} \
--Deconv-PeakFactor=0.001000 --Data-ColName={data_colname} --Parallel-NCPU={ncpu} --Facets-CatNodes={clustercat} \
--Beam-CenterNorm=1 --Deconv-Mode=SSD --Beam-Model=LOFAR --Beam-LOFARBeamMode=A --Weight-Robust={robust} \
--Image-NPix={imagenpix} --CF-wmax=50000 --CF-Nw=100 --Output-Also=onNeds --Image-Cell={imagecell} \
--Facets-NFacets=11 --SSDClean-NEnlargeData=0 --Freq-NDegridBand=1 --Beam-NBand=1 --Facets-DiamMax=1.5 \
--Facets-DiamMin=0.1 --Deconv-RMSFactor=3.000000 --SSDClean-ConvFFTSwitch 10000 --Data-Sort=1 --Cache-Dir=. \
--Log-Memory=1 --Cache-Weight=reset --Output-Mode=Predict --Output-RestoringBeam=6.000000 --Freq-NBand=2 \
--RIME-DecorrMode=FT --SSDClean-SSDSolvePars=[S,Alpha] --SSDClean-BICFactor=0 --Mask-Auto=1 --Mask-SigTh=5.00 \
--Mask-External={predict_mask} --DDESolutions-GlobalNorm=None --DDESolutions-DDModeGrid=AP \
--DDESolutions-DDModeDeGrid=AP --DDESolutions-DDSols=[DDS3_full_smoothed,DDS3_full_slow] \
--Predict-InitDicoModel={predict_dico} --Selection-UVRangeKm={uvsel} --GAClean-MinSizeInit=10 --Cache-Reset=1 \
--Beam-Smooth=1 --Predict-ColName={predict_column} --DDESolutions-SolsDir={solsdir}".format(**args))
# subtract
logger.info("Subtracting...")
for ms in mslist:
with pt.table(ms, readonly=False, ack=True) as t:
colnames = t.colnames()
if sub_column not in colnames:
# Append new column containing all sources
desc = t.getcoldesc(data_colname)
newdesc = pt.makecoldesc(sub_column, desc)
newdmi = t.getdminfo(data_colname)
newdmi['NAME'] = 'Dysco' + sub_column
t.addcols(newdesc, newdmi)
for row in range(0, t.nrows(), 3000000):
logger.info('Reading {}'.format(predict_column))
f = t.getcol(predict_column, startrow=row, nrow=3000000, rowincr=1)
logger.info('Reading', data_colname)
d = t.getcol(data_colname, startrow=row, nrow=3000000, rowincr=1)
logger.info('Writing', sub_column)
t.putcol(sub_column, d - f, startrow=row, nrow=3000000, rowincr=1)
addextraweights(mslist)
cleanup_working_dir(working_dir)
def ConvolveFFT(self, A, OutMode="Data", AddNoise=None):
shin = A.shape
T = ClassTimeIt.ClassTimeIt("ConvolveFFT")
T.disable()
Asq = self.PM.ModelToSquareArray(A, TypeInOut=("Parms", OutMode))
T.timeit("0")
NFreqBand, npol, N, _ = Asq.shape
zN = 2 * N + 1
zN, _ = EstimateNpix(float(zN))
zAsq = np.zeros((NFreqBand, npol, zN, zN), dtype=Asq.dtype)
zAsq[:, :, zN / 2 - N / 2:zN / 2 + N / 2 + 1,
zN / 2 - N / 2:zN / 2 + N / 2 + 1] = Asq[:, :, :, :]
T.timeit("1")
if AddNoise is not None:
zAsq += np.random.randn(*zAsq.shape) * AddNoise
N0x = zAsq.shape[-1]
xc0 = N0x / 2
N1 = self.PSF.shape[-1]
Aedge, Bedge = GiveEdgesDissymetric((xc0, xc0), (N0x, N0x),
(N1 / 2, N1 / 2), (N1, N1))
x0d, x1d, y0d, y1d = Aedge
x0s, x1s, y0s, y1s = Bedge
SubPSF = self.PSF[:, :, x0s:x1s, y0s:y1s]
#xc=self.PSF.shape[-1]/2
#SubPSF=self.PSF[:,:,xc-N:xc+N+1,xc-N:xc+N+1]
Conv = np.zeros_like(zAsq)
T.timeit("2")
for ich in range(NFreqBand):
for ipol in range(npol):
Conv[ich, ipol] = scipy.signal.fftconvolve(zAsq[ich, ipol],
SubPSF[ich, ipol],
mode='same')
#Conv[ich,ipol]=ModFFTW.ConvolveFFTW2D((zAsq[ich,ipol]), (SubPSF[ich,ipol]))
# import pylab
# pylab.subplot(1,3,1)
# pylab.imshow(Conv[ich,ipol][zN/2-N/2:zN/2+N/2+1,zN/2-N/2:zN/2+N/2+1],interpolation="nearest")
# pylab.subplot(1,3,2)
# pylab.imshow(Conv1[ich,ipol][zN/2-N/2:zN/2+N/2+1,zN/2-N/2:zN/2+N/2+1],interpolation="nearest")
# pylab.subplot(1,3,3)
# pylab.imshow((Conv-Conv1)[ich,ipol][zN/2-N/2:zN/2+N/2+1,zN/2-N/2:zN/2+N/2+1],interpolation="nearest")
# pylab.colorbar()
# pylab.draw()
# pylab.show()
T.timeit("3 [%s]" % str(zAsq.shape))
A = self.PM.SquareArrayToModel(Conv[:, :,
zN / 2 - N / 2:zN / 2 + N / 2 + 1,
zN / 2 - N / 2:zN / 2 + N / 2 + 1],
TypeInOut=(OutMode, OutMode))
T.timeit("4")
if OutMode == "Data":
NPixOut = self.NPixListData
else:
NPixOut = self.NPixListParms
# import pylab
# pylab.clf()
# pylab.subplot(1,3,1)
# pylab.imshow(zAsq[0,0],interpolation="nearest")
# pylab.subplot(1,3,2)
# pylab.imshow(SubPSF[0,0],interpolation="nearest")
# pylab.subplot(1,3,3)
# pylab.imshow(Conv[0,0],interpolation="nearest")
# pylab.draw()
# pylab.show(False)
# stop
return A.reshape((NFreqBand, npol, NPixOut))
