def invSVD(A):
try:
u,s,v=np.linalg.svd(np.complex128(A))#+np.random.randn(*A.shape)*(1e-6*A.max()))
except:
Name="errSVDArray_%i"%int(np.random.rand(1)[0]*10000)
print(ModColor.Str("Problem inverting Matrix, saving as %s"%Name))
print(ModColor.Str(" will make it svd-able"))
np.save(Name,A)
# weird - I found a matrix I cannot do svd on... - that works
Cut=1e-20
#Ar=np.complex64(Ar)
u,s,v=np.linalg.svd(np.complex128(A)+np.random.randn(*A.shape)*(1e-10*np.abs(A).max()))
#s[s<0.]=1.e-6
s0=s.copy()
Th=1e-10
s[s<Th*s.max()]=Th*s.max()
ssq=(1./s)
#Asq=np.conj(np.dot(np.dot(v.T,ssq),u.T))
v0=v.T*ssq.reshape(1,ssq.size)
Asq=np.conj(np.dot(v0,u.T))
#PlotMatSVD(A,s0.flatten(),Asq)
return Asq
def __str__(self):
ll = []
ll.append(ModColor.Str(" MS PROPERTIES: "))
ll.append(" - File Name: %s" %
ModColor.Str(self.MSName, col="green"))
ll.append(" - Column Name: %s" %
ModColor.Str(str(self.ColName), col="green"))
ll.append(" - Selection: %s" %
(ModColor.Str(str(self.TaQL), col="green")))
ll.append(" - Pointing center: (ra, dec)=(%s, %s) "%(rad2hmsdms(self.rarad,Type="ra").replace(" ",":")\
,rad2hmsdms(self.decrad,Type="dec").replace(" ",".")))
ll.append(" - Frequency = %s MHz" % str(self.reffreq / 1e6))
ll.append(" - Wavelength = %5.2f meters" %
(np.mean(self.wavelength_chan)))
ll.append(" - Time bin = %4.1f seconds" % (self.dt))
ll.append(" - Total Integration time = %6.2f hours" % self.DTh)
ll.append(" - Number of antenna = %i" % self.na)
ll.append(" - Number of baseline = %i" % self.nbl)
ll.append(" - Number of SPW = %i" % self.NSPW)
ll.append(" - Number of channels = %i" % self.Nchan)
s = " ".join(["%.2f" % (x / 1e6) for x in self.ChanFreq.flatten()])
#ll.append(" - Chan freqs = %s"%(ListToStr(s.split(" "),Unit="MHz")))
ss = "\n".join(ll) + "\n"
return ss
def Print2(self,par,value,helpit,col="white"):
WidthTerm=self.getWidth()
Lpar=len(str(par))
Lval=len(str(value))
SFill="."*max(self.LeftW-self.Lproto-Lpar-Lval,2)
WidthHelp=WidthTerm-(self.Lproto+Lpar+Lval+len(SFill))
Spar="%s"%ModColor.Str(str(par),col=col,Bold=False)
Sval="%s"%ModColor.Str(str(value),col=col,Bold=False)
if helpit=="":
helpit="Help yourself"
Shelp="%s"%helpit
if WidthHelp<0:
print(self.proto%(Spar,SFill,Sval)+Shelp)
return
Lhelp=len(str(helpit))
listStrHelp=range(0,Lhelp,WidthHelp)
if listStrHelp[-1]!=Lhelp: listStrHelp.append(Lhelp)
print(self.proto%(Spar,SFill,Sval)+Shelp[0:WidthHelp])
for i in range(1,len(listStrHelp)-1):
parout="%s: %s"%(" "*(self.LeftW-2),Shelp[listStrHelp[i]:listStrHelp[i+1]])
print(parout)
def SaveVis(self, vis=None, Col="CORRECTED_DATA", spw=0, DoPrint=True):
if vis == None:
vis = self.data
if DoPrint:
log.print("Writing data in column %s" %
ModColor.Str(Col, col="green"))
print("Givemain")
table_all = self.GiveMainTable(readonly=False)
if self.swapped:
visout = np.swapaxes(vis[spw * self.Nchan:(spw + 1) * self.Nchan],
0, 1)
flag_all = np.swapaxes(
self.flag_all[spw * self.Nchan:(spw + 1) * self.Nchan], 0, 1)
else:
visout = vis
flag_all = self.flag_all
print("Col")
table_all.putcol(Col, visout.astype(self.data.dtype), self.ROW0,
self.nRowRead)
print("Flag")
table_all.putcol("FLAG", flag_all, self.ROW0, self.nRowRead)
print("Weight")
if self.HasWeights:
table_all.putcol("WEIGHT", self.Weights, self.ROW0, self.nRowRead)
#print("ok w")
print("Close")
table_all.close()
def SharedToDico(Prefix):
log.print(ModColor.Str("SharedToDico: start [prefix = %s]" % Prefix))
T = ClassTimeIt.ClassTimeIt(" SharedToDico")
T.disable()
Lnames = ListNames()
T.timeit("0: ListNames")
keys = [Name for Name in Lnames if Prefix in Name]
if len(keys) == 0: return None
DicoOut = {}
T.timeit("1")
for Sharedkey in keys:
key = Sharedkey.split(".")[-1]
log.print(ModColor.Str(" %s -> %s" % (Sharedkey, key)))
Shared = GiveArray(Sharedkey)
DicoOut[key] = Shared
T.timeit("2a")
log.print(ModColor.Str("SharedToDico: done"))
return DicoOut
def SharedObjectToDico(SObject):
if SObject == None: return None
Prefix = SObject.prefixName
Fields = SObject.DicoKeys
log.print(ModColor.Str("SharedToDico: start [prefix = %s]" % Prefix))
T = ClassTimeIt.ClassTimeIt(" SharedToDico")
T.disable()
DicoOut = {}
T.timeit("1")
for field in Fields:
Sharedkey = "%s.%s" % (Prefix, field)
#log.print( ModColor.Str(" %s -> %s"%(Sharedkey,key)))
Shared = GiveArray(Sharedkey)
DicoOut[field] = Shared
T.timeit("2a")
log.print(ModColor.Str("SharedToDico: done"))
return DicoOut
def DicoToShared(Prefix, Dico, DelInput=False):
DicoOut = {}
log.print(ModColor.Str("DicoToShared: start [prefix = %s]" % Prefix))
for key in Dico.keys():
if type(Dico[key]) != np.ndarray: continue
#print "%s.%s"%(Prefix,key)
ThisKeyPrefix = "%s.%s" % (Prefix, key)
log.print(ModColor.Str(" %s -> %s" % (key, ThisKeyPrefix)))
ar = Dico[key]
Shared = ToShared(ThisKeyPrefix, ar)
#T.timeit("getarray %s"%ThisKeyPrefix)
DicoOut[key] = Shared
if DelInput:
del (Dico[key], ar)
if DelInput:
del (Dico)
log.print(ModColor.Str("DicoToShared: done"))
return DicoOut
def ToShared(Name, A):
try:
a = SharedArray.create(Name, A.shape, dtype=A.dtype)
except:
log.print(ModColor.Str("File %s exists, delete it..." % Name))
#DelArray(Name.decode("byte"))
DelArray(Name)
a = SharedArray.create(Name, A.shape, dtype=A.dtype)
a[:] = A[:]
return a
def __init__(self, BaseImageName, VS, IdSharedMem, *args, **kwargs):
ClassImagerDeconv.__init__(self, **kwargs)
self.BaseImageName = BaseImageName
self.FileDicoModel = "%s.DicoModel" % self.BaseImageName
if self.GD["GDkMS"]["ImageSkyModel"]["DicoModel"] is not None:
log.print(
ModColor.Str("Using %s instead of %s" %
(self.GD["GDkMS"]["ImageSkyModel"]["DicoModel"],
self.FileDicoModel)))
self.FileDicoModel = self.GD["GDkMS"]["ImageSkyModel"]["DicoModel"]
self.ModelImageName = "%s.model.fits" % self.BaseImageName
self.VS = VS
if self.GD["CF"]["wmax"] == 0:
log.print("Computing wmax from UVW column")
t = table(self.VS.ListMS[0].MSName, ack=False)
u, v, w = t.getcol("UVW").T
t.close()
self.GD["CF"]["wmax"] = np.max(np.abs(w))
log.print(" found a wmax=%f meters" % self.GD["CF"]["wmax"])
# DC=self.GD
# MSName=DC["VisData"]["MSName"]
# self.VS=ClassVisServer.ClassVisServer(MSName,
# ColName=DC["VisData"]["ColName"],
# TVisSizeMin=DC["VisData"]["TChunkSize"]*60,
# #DicoSelectOptions=DicoSelectOptions,
# TChunkSize=DC["VisData"]["TChunkSize"],
# IdSharedMem=self.IdSharedMem,
# Robust=DC["ImagerGlobal"]["Robust"],
# Weighting=DC["ImagerGlobal"]["Weighting"],
# Super=DC["ImagerGlobal"]["Super"],
# DicoSelectOptions=dict(DC["DataSelection"]),
# NCPU=self.GD["Parallel"]["NCPU"],
# GD=self.GD)
#self.GD["GAClean"]["GASolvePars"]=["S","Alpha"]
self.IdSharedMem = IdSharedMem #kwargs["IdSharedMem"]
self.SM = ClassImageSM()
if self.GD["GDkMS"]["ImageSkyModel"]["NodesFile"] is not None:
self.GD["Facets"]["CatNodes"] = self.GD["GDkMS"]["ImageSkyModel"][
"NodesFile"]
self.GD["DDESolutions"]["DDSols"] = None
# self.InitFacetMachine()
self.CreateFacetMachines()
self.LoadModel()
def print_logo():
#os.system('clear')
version=report_version()
print(""" __ _ __ __ ____ ____ ______ """)
print(""" [ | _ (_) [ | [ | |_ \ / _|.' ____ \ """)
print(""" | | / ] __ | | | | | \/ | | (___ \_| """)
print(""" | '' < [ | | | | | | |\ /| | _.____`. """)
print(""" | |`\ \ | | | | | | _| |_\/_| |_ | \____) | """)
print(""" [__| \_][___][___][___]|_____||_____| \______.' """)
print(""" This is version : %s""" %ModColor.Str(version))
print(""" """)
def PrintOptParse(Obj, ValObj, RejectGroup=[]):
P = ClassPrint.ClassPrint(HW=30)
LGroups = Obj.option_groups
print(ModColor.Str(" Selected Options:"))
for Group in LGroups:
Skip = False
for Name in RejectGroup:
if Name in Group.title:
Skip = True
if Skip: continue
print(ModColor.Str(Group.title, col="green"))
option_list = Group.option_list
for o in option_list:
lopt = o._long_opts[0]
oname = lopt.split("--")[-1]
V = getattr(ValObj, oname)
# if (V!="")&(V!=None):
if True: #V!="":
if V == "": V = "''"
#P.Print(oname,V)
default = o.default
H = o.help
# if H!=None:
# H=o.help.replace("%default",str(default))
# P.Print2(oname,V,H)
# else:
# P.Print(oname,V)
P.Print(oname, V)
# strName=%s
# print " "oname,V
print()
def Print(self, RejectGroup=[]):
P = ClassPrint.ClassPrint(HW=50)
print(ModColor.Str(" Selected Options:"))
for Group, V in self.DefaultDict.items():
Skip = False
for Name in RejectGroup:
if Name in Group:
Skip = True
if Skip: continue
try:
GroupTitle = self.DicoGroupDesc[Group]
except:
GroupTitle = Group
print(ModColor.Str(GroupTitle, col="green"))
option_list = self.DefaultDict[Group]
for oname in option_list:
V = self.DefaultDict[Group][oname]
if True: #V!="":
if V == "": V = "''"
P.Print(oname, V)
print()
def ReClusterSkyModel(self, SM, MSName):
SolRefFile = self.GD["PreApply"]["PreApplySols"][0]
if (SolRefFile != "") & (not (".npz" in SolRefFile)):
Method = SolRefFile
ThisMSName = reformat.reformat(os.path.abspath(MSName),
LastSlash=False)
SolRefFile = "%s/killMS.%s.sols.npz" % (ThisMSName, Method)
log.print(
ModColor.Str(
"Re-clustering input SkyModel to match %s clustering" %
SolRefFile))
ClusterCat0 = np.load(SolRefFile)["ClusterCat"]
ClusterCat0 = ClusterCat0.view(np.recarray)
lc = ClusterCat0.l
mc = ClusterCat0.m
lc = lc.reshape((1, lc.size))
mc = mc.reshape((1, mc.size))
l = SM.SourceCat.l
m = SM.SourceCat.m
l = l.reshape((l.size, 1))
m = m.reshape((m.size, 1))
d = np.sqrt((l - lc)**2 + (m - mc)**2)
Cluster = np.argmin(d, axis=1)
#print SM.SourceCat.Cluster
SM.SourceCat.Cluster[:] = Cluster[:]
SM.ClusterCat = ClusterCat0
#print SM.SourceCat.Cluster
SM.Dirs = sorted(list(set(SM.SourceCat.Cluster.tolist())))
SM.NDir = len(SM.Dirs)
log.print(" There are %i clusters in the re-clustered skymodel" %
SM.NDir)
NDir = lc.size
for iDir in range(NDir):
ind = (SM.SourceCat.Cluster == iDir)
SM.ClusterCat.SumI[iDir] = np.sum(SM.SourceCat.I[ind])
# test
#import numpy
#print numpy.__file__
#import pyrap
#print pyrap.__file__
#stop
if "nocol" in sys.argv:
print("nocol")
ModColor.silent = 1
if "nox" in sys.argv:
import matplotlib
matplotlib.use('agg')
print(ModColor.Str(" == !NOX! =="))
import time
import os
import numpy as np
import pickle
from itertools import product as ItP
NameSave = "last_plotSols.obj"
def read_options():
desc = """killMS Questions and suggestions: [email protected]"""
opt = optparse.OptionParser(
def ReadMSInfo(self, MSname, DoPrint=True):
T = ClassTimeIt.ClassTimeIt()
T.enableIncr()
T.disable()
#print(MSname+'/ANTENNA')
# open main table
table_all = table(MSname, ack=False)
#print(MSname+'/ANTENNA')
ta = table(table_all.getkeyword('ANTENNA'), ack=False)
#ta=table(MSname+'::ANTENNA',ack=False)
StationNames = ta.getcol('NAME')
na = ta.getcol('POSITION').shape[0]
self.StationPos = ta.getcol('POSITION')
#nbl=(na*(na-1))/2+na
A0, A1 = table_all.getcol("ANTENNA1"), table_all.getcol("ANTENNA2")
ind = np.where(A0 == A1)[0]
self.HasAutoCorr = (ind.size > 0)
A = np.concatenate([A0, A1])
nas = np.unique(A).size
self.nbl = (nas**2 - nas) // 2
if self.HasAutoCorr:
self.nbl += nas
if A0.size % self.nbl != 0:
log.print(ModColor.Str("MS is non conformant!"))
raise
#nbl=(na*(na-1))/2
ta.close()
T.timeit()
#table_all=table(MSname,ack=False)
self.ColNames = table_all.colnames()
TimeIntervals = table_all.getcol("INTERVAL")
SPW = table_all.getcol('DATA_DESC_ID')
if self.SelectSPW != None:
self.ListSPW = self.SelectSPW
#print("dosel")
else:
self.ListSPW = sorted(list(set(SPW.tolist())))
T.timeit()
self.F_nrows = table_all.getcol("TIME").shape[0]
F_time_all = table_all.getcol("TIME")[SPW == self.ListSPW[0]]
self.F_A0 = table_all.getcol("ANTENNA1")[SPW == self.ListSPW[0]]
self.F_A1 = table_all.getcol("ANTENNA2")[SPW == self.ListSPW[0]]
#nbl=(np.where(F_time_all==F_time_all[0])[0]).shape[0]
T.timeit()
F_time_slots_all = np.array(sorted(list(set(F_time_all.tolist()))))
F_ntimes = F_time_slots_all.shape[0]
T.timeit()
ta_spectral = table(table_all.getkeyword('SPECTRAL_WINDOW'), ack=False)
reffreq = ta_spectral.getcol('REF_FREQUENCY')
chan_freq = ta_spectral.getcol('CHAN_FREQ')
self.NChanOrig = chan_freq.size
chan_freq = chan_freq[:, self.ChanSlice]
self.dFreq = ta_spectral.getcol("CHAN_WIDTH").flatten()[self.ChanSlice]
self.ChanWidth = ta_spectral.getcol('CHAN_WIDTH')[:, self.ChanSlice]
if chan_freq.shape[0] > len(self.ListSPW):
print(
ModColor.Str(
" ====================== >> More SPW in headers, modifying that error...."
))
chan_freq = chan_freq[np.array(self.ListSPW), :]
reffreq = reffreq[np.array(self.ListSPW)]
T.timeit()
wavelength = 299792456. / reffreq
NSPW = chan_freq.shape[0]
self.ChanFreq = chan_freq
self.Freq_Mean = np.mean(chan_freq)
wavelength_chan = 299792456. / chan_freq
if NSPW > 1:
print("Don't deal with multiple SPW yet")
Nchan = wavelength_chan.shape[1]
NSPWChan = NSPW * Nchan
ta = table(table_all.getkeyword('FIELD'), ack=False)
rarad, decrad = ta.getcol('PHASE_DIR')[self.Field][0]
if rarad < 0.: rarad += 2. * np.pi
T.timeit()
radeg = rarad * 180. / np.pi
decdeg = decrad * 180. / np.pi
ta.close()
self.DoRevertChans = False
if Nchan > 1:
self.DoRevertChans = (self.ChanFreq.flatten()[0] >
self.ChanFreq.flatten()[-1])
if self.DoRevertChans:
log.print(
ModColor.Str(
" ====================== >> Revert Channel order!"))
wavelength_chan = wavelength_chan[0, ::-1]
self.ChanFreq = self.ChanFreq[0, ::-1]
self.ChanWidth = -self.ChanWidth[0, ::-1]
self.dFreq = np.abs(self.dFreq)
T.timeit()
MS_STOKES_ENUMS = [
"Undefined", "I", "Q", "U", "V", "RR", "RL", "LR", "LL", "XX",
"XY", "YX", "YY", "RX", "RY", "LX", "LY", "XR", "XL", "YR", "YL",
"PP", "PQ", "QP", "QQ", "RCircular", "LCircular", "Linear",
"Ptotal", "Plinear", "PFtotal", "PFlinear", "Pangle"
]
tp = table(table_all.getkeyword('POLARIZATION'), ack=False)
# get list of corrype enums for first row of polarization table, and convert to strings via MS_STOKES_ENUMS.
# self.CorrelationNames will be a list of strings
self.CorrelationIds = tp.getcol('CORR_TYPE', 0, 1)[0]
self.CorrelationNames = [(ctype >= 0 and ctype < len(MS_STOKES_ENUMS)
and MS_STOKES_ENUMS[ctype]) or None
for ctype in self.CorrelationIds]
self.Ncorr = len(self.CorrelationNames)
# NB: it is possible for the MS to have different polarization
table_all.close()
self.na = na
self.Nchan = Nchan
self.NSPW = NSPW
self.NSPWChan = NSPWChan
self.F_tstart = F_time_all[0]
self.F_times_all = F_time_all
self.F_times = F_time_slots_all
self.F_ntimes = F_time_slots_all.shape[0]
self.dt = TimeIntervals[0]
self.DTs = F_time_all[-1] - F_time_all[0] + self.dt
self.DTh = self.DTs / 3600.
self.radec = self.OriginalRadec = (rarad, decrad)
self.rarad = rarad
self.decrad = decrad
self.reffreq = reffreq
self.StationNames = StationNames
self.wavelength_chan = wavelength_chan
self.rac = rarad
self.decc = decrad
#self.nbl=nbl
self.StrRA = rad2hmsdms(self.rarad, Type="ra").replace(" ", ":")
self.StrDEC = rad2hmsdms(self.decrad, Type="dec").replace(" ", ".")
if self.ToRADEC is not None:
ranew, decnew = rarad, decrad
# get RA/Dec from first MS, or else parse as coordinate string
if self.ToRADEC == "align":
stop
if first_ms is not None:
ranew, decnew = first_ms.rarad, first_ms.decrad
which = "the common phase centre"
else:
which = "%s %s" % tuple(self.ToRADEC)
SRa, SDec = self.ToRADEC
srah, sram, sras = SRa.split(":")
sdecd, sdecm, sdecs = SDec.split(":")
ranew = (np.pi / 180) * 15. * (
float(srah) + float(sram) / 60. + float(sras) / 3600.)
decnew = (np.pi / 180) * np.sign(float(sdecd)) * (abs(
float(sdecd)) + float(sdecm) / 60. + float(sdecs) / 3600.)
# only enable rotation if coordinates actually change
if ranew != rarad or decnew != decrad:
print(ModColor.Str("MS %s will be rephased to %s" %
(self.MSName, which)),
file=log)
self.OldRadec = rarad, decrad
self.NewRadec = ranew, decnew
rarad, decrad = ranew, decnew
else:
self.ToRADEC = None
T.timeit()
def ReadData(self, t0=0, t1=-1, DoPrint=False, ReadWeight=False):
if DoPrint == True:
print(" ... Reading MS")
# TODO: read this from MS properly, as in DDFacet
self.CorrelationNames = "xx", "xy", "yx", "yy"
row0 = 0
row1 = self.F_nrows
DATA_CHUNK = {}
if t1 > t0:
t0 = t0 * 3600.
t1 = t1 * 3600.
self.CurrentChunkTimeRange_SinceT0_sec = (t0, t1)
t0 = t0 + self.F_tstart
t1 = t1 + self.F_tstart
#ind0=np.argmin(np.abs(t0-self.F_times))
#ind1=np.argmin(np.abs(t1-self.F_times))
# ind0=np.where((t0-self.F_times)<=0)[0][0]
# row0=ind0*self.nbl
# ind1=np.where((t1-self.F_times)<0)[0]
# if ind1.size==0:
# row1=self.F_nrows
# else:
# ind1=ind1[0]
# row1=ind1*self.nbl
ind = np.where((self.F_times_all >= t0)
& (self.F_times_all < t1))[0]
if ind.size == 0:
return None
# row0=self.ROW1
# row1=row0
else:
row0 = ind[0]
ind1 = row0 + ind.size
row1 = ind1
# print("!!!!!!!=======")
# row0,row1=1207458, 1589742
self.ROW0 = row0
self.ROW1 = row1
self.nRowRead = row1 - row0
# if chunk is empty, return None
if self.nRowRead <= 0:
return None
log.print(" Reading rows [%i -> %i]" % (self.ROW0, self.ROW1))
DATA_CHUNK["ROW0"] = row0
DATA_CHUNK["ROW1"] = row1
DATA_CHUNK["nRowRead"] = self.nRowRead
nRowRead = self.nRowRead
#table_all=table(self.MSName,ack=False)
table_all = self.GiveMainTable()
try:
SPW = table_all.getcol('DATA_DESC_ID', row0, nRowRead)
except Exception as e:
log.print(
ModColor.Str("There was a problem reading DATA_DESC_ID:" +
str(e)))
DATA_DESC_ID = np.unique(table_all.getcol('DATA_DESC_ID'))
if DATA_DESC_ID.size == 1:
log.print(
ModColor.Str(
" All DATA_DESC_ID are the same, can proceed"))
SPW = np.zeros((nRowRead, ), )
SPW.fill(DATA_DESC_ID[0])
else:
raise
A0 = table_all.getcol('ANTENNA1', row0,
nRowRead)[SPW == self.ListSPW[0]]
A1 = table_all.getcol('ANTENNA2', row0,
nRowRead)[SPW == self.ListSPW[0]]
#print(self.ListSPW[0])
time_all = table_all.getcol("TIME", row0,
nRowRead)[SPW == self.ListSPW[0]]
self.Time0 = table_all.getcol("TIME", 0, 1)[0]
#print(np.max(time_all)-np.min(time_all))
time_slots_all = np.array(sorted(list(set(time_all))))
ntimes = time_all.shape[0] / self.nbl
flag_all = table_all.getcol(
"FLAG", row0, nRowRead)[SPW == self.ListSPW[0]][:,
self.ChanSlice, :]
self.HasWeights = False
if ReadWeight == True:
self.Weights = table_all.getcol("WEIGHT", row0, nRowRead)
self.HasWeights = True
if self.EqualizeFlag:
for i in range(self.Nchan):
fcol = flag_all[:, i,
0] | flag_all[:, i,
1] | flag_all[:, i,
2] | flag_all[:, i,
3]
for pol in range(4):
flag_all[:, i, pol] = fcol
self.multidata = (type(self.ColName) == list)
self.ReverseAntOrder = (np.where((A0 == 0)
& (A1 == 1))[0]).shape[0] > 0
self.swapped = False
uvw = table_all.getcol('UVW', row0, nRowRead)[SPW == self.ListSPW[0]]
self.TimeInterVal = table_all.getcol("INTERVAL")
if self.ReOrder:
vis_all = table_all.getcol(self.ColName, row0,
nRowRead)[:, self.ChanSlice, :]
if self.zero_flag: vis_all[flag_all == 1] = 0.
if self.zero_flag:
noise=(np.random.randn(vis_all.shape[0],vis_all.shape[1],vis_all.shape[2])\
+1j*np.random.randn(vis_all.shape[0],vis_all.shape[1],vis_all.shape[2]))*1e-6
vis_all[flag_all == 1] = noise[flag_all == 1]
vis_all[np.isnan(vis_all)] = 0.
listDataSPW = [
np.swapaxes(vis_all[SPW == i, :, :], 0, 1)
for i in self.ListSPW
]
self.data = np.concatenate(
listDataSPW) #np.swapaxes(np.concatenate(listDataSPW),0,1)
listFlagSPW = [
np.swapaxes(flag_all[SPW == i, :, :], 0, 1)
for i in self.ListSPW
]
flag_all = np.concatenate(
listFlagSPW) #np.swapaxes(np.concatenate(listDataSPW),0,1)
self.uvw = uvw
self.swapped = True
else:
self.uvw = uvw
if self.multidata:
self.data = []
for colin in self.ColName:
print("... read %s" % colin)
vis_all = table_all.getcol(
colin, row0,
nRowRead)[SPW == self.ListSPW[0]][:, self.ChanSlice, :]
print(" shape: %s" % str(vis_all.shape))
if self.zero_flag: vis_all[flag_all == 1] = 0.
vis_all[np.isnan(vis_all)] = 0.
self.data.append(vis_all)
else:
vis_all = table_all.getcol(self.ColName, row0,
nRowRead)[:, self.ChanSlice, :]
#if self.zero_flag: vis_all[flag_all==1]=0.
#vis_all[np.isnan(vis_all)]=0.
self.data = vis_all
# import pylab
# pylab.plot(time_all[::111],vis[::111,512,0].real)
# pylab.show()
fnan = np.isnan(vis_all)
vis_all[fnan] = 0.
flag_all[fnan] = 1
#flag_all[vis_all==0]=1
self.flag_all = flag_all
self.uvw_dt = None
if self.ReadUVWDT:
tu = table(self.MSName, ack=False)
ColNames = tu.colnames()
tu.close()
del (tu)
if 'UVWDT' not in ColNames:
self.AddUVW_dt()
log.print("Reading uvw_dt column")
tu = table(self.MSName, ack=False)
self.uvw_dt = np.float64(tu.getcol('UVWDT', row0, nRowRead))
tu.close()
if self.RejectAutoCorr:
indGetCorrelation = np.where(A0 != A1)[0]
A0 = A0[indGetCorrelation]
A1 = A1[indGetCorrelation]
self.uvw = self.uvw[indGetCorrelation, :]
time_all = time_all[indGetCorrelation]
if self.swapped:
self.data = self.data[:, indGetCorrelation, :]
self.flag_all = self.flag_all[:, indGetCorrelation, :]
else:
self.data = self.data[indGetCorrelation, :, :]
self.flag_all = self.flag_all[indGetCorrelation, :, :]
self.nbl = (self.na * (self.na - 1)) // 2
if self.DoRevertChans:
self.flag_all = self.flag_all[:, ::-1, :]
if not (type(self.data) == list):
self.data = self.data[:, ::-1, :]
else:
for icol in range(len(self.data)):
self.data[icol] = self.data[icol][:, ::-1, :]
if self.ToRADEC is not None:
DATA = {"uvw": uvw, "data": self.data}
self.Rotate(DATA, RotateType=["uvw", "vis"])
self.data = DATA["data"]
self.NPolOrig = self.data.shape[-1]
if self.data.shape[-1] != 4:
log.print(
ModColor.Str("Data has only two polarisation, adapting shape"))
nrow, nch, _ = self.data.shape
flag_all = np.zeros((nrow, nch, 4), self.flag_all.dtype)
data = np.zeros((nrow, nch, 4), self.data.dtype)
flag_all[:, :, 0] = self.flag_all[:, :, 0]
flag_all[:, :, -1] = self.flag_all[:, :, -1]
data[:, :, 0] = self.data[:, :, 0]
data[:, :, -1] = self.data[:, :, -1]
self.data = data
self.flag_all = flag_all
if "IMAGING_WEIGHT" in table_all.colnames():
log.print("Flagging the zeros-weighted visibilities")
fw = table_all.getcol(
"IMAGING_WEIGHT", row0,
nRowRead)[SPW == self.ListSPW[0]][:, self.ChanSlice]
nrr, nchr = fw.shape
fw = fw.reshape((nrr, nchr, 1)) * np.ones((1, 1, 4))
MedW = np.median(fw)
fflagged0 = np.count_nonzero(flag_all)
flag_all[fw < MedW * 1e-6] = 1
fflagged1 = np.count_nonzero(flag_all)
if fflagged1 > 0 and fflagged0 != 0:
log.print(" Increase in flag fraction: %f" %
(fflagged1 / float(fflagged0) - 1))
table_all.close()
self.times_all = time_all
self.times = time_slots_all
self.ntimes = time_slots_all.shape[0]
self.nrows = time_all.shape[0]
self.IndFlag = np.where(flag_all == True)
#self.NPol=vis_all.shape[2]
self.A0 = A0
self.A1 = A1
return True
def GiveMappingAnt(self,
ListStrSel,
row0row1=(None, None),
FlagAutoCorr=True,
WriteAttribute=True):
row0, row1 = row0row1
if type(ListStrSel) != list:
assert (False)
#ListStrSel=["RT9-RTA", "RTA-RTB", "RTC-RTD", "RT6-RT7", "RT5"]
log.print(
ModColor.Str(" ... Building BL-mapping for %s" % str(ListStrSel)))
if row1 == None:
row0 = 0
row1 = self.nbl
A0 = self.F_A0[row0:row1]
A1 = self.F_A1[row0:row1]
MapOut = np.ones((self.nbl, ), dtype=np.bool)
if FlagAutoCorr:
ind = np.where(A0 == A1)[0]
MapOut[ind] = False
def GiveStrAntToNum(self, StrAnt):
ind = []
for i in range(len(self.StationNames)):
if StrAnt in self.StationNames[i]:
ind.append(i)
#print ind
return ind
#MapOut=np.ones(A0.shape,bool)###)
LFlaggedStations = []
LNumFlaggedStations = []
for blsel in ListStrSel:
if blsel == "": continue
if "-" in blsel:
StrA0, StrA1 = blsel.split("-")
LNumStrA0 = GiveStrAntToNum(self, StrA0)
LNumStrA1 = GiveStrAntToNum(self, StrA1)
for NumStrA0 in LNumStrA0:
for NumStrA1 in LNumStrA1:
NumA0 = np.where(
np.array(self.StationNames) == NumStrA0)[0]
NumA1 = np.where(
np.array(self.StationNames) == NumStrA1)[0]
C0 = ((A0 == NumA0) & (A1 == NumA1))
C1 = ((A1 == NumA0) & (A0 == NumA1))
ind = np.where(C1 | C0)[0]
MapOut[ind] = False
else:
#NumA0=np.where(np.array(self.StationNames)==blsel)[0]
StrA0 = blsel
LNumStrA0 = GiveStrAntToNum(self, StrA0)
LNumFlaggedStations.append(LNumStrA0)
for NumStrA0 in LNumStrA0:
LFlaggedStations.append(self.StationNames[NumStrA0])
# NumA0=np.where(np.array(self.StationNames)==NumStrA0)[0]
# stop
# print NumStrA0,NumA0
C0 = (A0 == NumStrA0)
C1 = (A1 == NumStrA0)
ind = np.where(C1 | C0)[0]
MapOut[ind] = False
if WriteAttribute:
self.MapSelBLs = MapOut
self.LFlaggedStations = list(set(LFlaggedStations))
return self.MapSelBLs
else:
LNumFlaggedStations = sorted(
list(
set(range(self.na)) -
set(np.array(LNumFlaggedStations).flatten().tolist())))
return LNumFlaggedStations
def PrepareGridMachinesMapping(self):
log.print(" Make the solution-directions to gridmachine mapping")
ListGrid = []
NFacets = len(self.DicoImager.keys())
ClusterCat = self.ClusterCat
DicoJonesDirToFacet = {}
for iDir in range(self.ClusterCat.shape[0]):
DicoJonesDirToFacet[iDir] = {}
DicoJonesDirToFacet[iDir]["FacetsIDs"] = []
DicoJonesDirToFacet[iDir]["SumFlux"] = 0.
model_dict = self.FacetMachine._model_dict
model_dict.reload()
for iFacet in sorted(self.FacetMachine.DicoImager.keys()):
iDirJones = self.FacetMachine.DicoImager[iFacet]["iDirJones"]
ThisFacetSumFlux = model_dict[iFacet]["SumFlux"]
ClusterCat.SumI[iDirJones] += np.real(ThisFacetSumFlux)
self.FacetMachine.DicoImager[iFacet]["SumFlux"] = np.real(
ThisFacetSumFlux)
if self.FacetMachine.DicoImager[iFacet]["SumFlux"] != 0.:
DicoJonesDirToFacet[iDirJones]["FacetsIDs"].append(iFacet)
DicoJonesDirToFacet[iDirJones]["SumFlux"] += np.real(
ThisFacetSumFlux)
for iFacet in sorted(self.FacetMachine.DicoImager.keys()):
Grid = model_dict[iFacet]["FacetGrid"]
ListGrid.append(Grid)
self.DicoImager = self.FacetMachine.DicoImager
self.DicoJonesDirToFacet = DicoJonesDirToFacet
D = {}
iDirNew = 0
self.ClusterCatOrig = self.ClusterCat.copy()
self.NDirsOrig = self.ClusterCat.shape[0]
self.NDirs = self.ClusterCat.shape[0]
from killMS.Data import ClassBeam
#self.GD["Beam"]["BeamModel"]=self.GD["Beam"]["Model"]
Th = float(self.GD["GDkMS"]["ImageSkyModel"]["ThSolve"])
Keep = np.zeros((self.NDirs, ), bool)
if Th > 0:
log.print("Compute mean beam for direction removal...")
BeamMachine = ClassBeam.ClassBeam(
self.VS.ListMS[0].MSName, self.GD["GDkMS"],
self.SM) #,ColName=self.GD["Data"]["ColName"])
AbsMeanBeam = BeamMachine.GiveMeanBeam(NTimes=10)
AbsMeanBeamAnt = np.mean(AbsMeanBeam[:, :, 0, 0, 0], axis=1)
AppFlux = np.array([
self.DicoJonesDirToFacet[iDirJones]["SumFlux"] *
(AbsMeanBeamAnt[iDirJones])**2
for iDirJones in sorted(DicoJonesDirToFacet.keys())
])
else:
AppFlux = np.array([
self.DicoJonesDirToFacet[iDirJones]["SumFlux"]
for iDirJones in sorted(DicoJonesDirToFacet.keys())
])
MaxAppFlux = AppFlux.max()
HasRemoved = 0
for iDirJones in sorted(DicoJonesDirToFacet.keys()):
#print(self.DicoJonesDirToFacet[iDirJones]["SumFlux"])
#if self.DicoJonesDirToFacet[iDirJones]["SumFlux"]==0:
if AppFlux[iDirJones] <= MaxAppFlux * Th:
log.print(
ModColor.Str(
" Remove Jones direction %i [%f < %f Jy [Th = %f of Max %f Jy]]"
% (iDirJones, AppFlux[iDirJones], MaxAppFlux * Th, Th,
MaxAppFlux)))
HasRemoved = 1
#print(" !!!!!!!!!!!!!!!!!!!!!!!")
else:
D[iDirNew] = self.DicoJonesDirToFacet[iDirJones]
iDirNew += 1
Keep[iDirJones] = 1
if not HasRemoved:
log.print(
ModColor.Str("All directions have been kept in the solve"))
#Keep.fill(0)
#Keep[1:5]=1
self.MapClusterCatOrigToCut = Keep
self.DicoJonesDirToFacet = D
self.ClusterCat = self.ClusterCat[Keep].copy()
# self.SourceCat=self.SourceCat[Keep].copy()
self.Dirs = self.DicoJonesDirToFacet.keys()
self.NDirs = len(self.Dirs)
# NpShared.PackListArray("%sGrids"%(self.IdSharedMem),ListGrid)
return True
def DoSimul(self):
Noise = .01
MS = self.MS
SM = self.SM
VS = self.VS
ApplyBeam = self.ApplyBeam
na = MS.na
nd = SM.NDir
NCPU = 6
#PM=ClassPredict(NCPU=NCPU,DoSmearing="F")
PM = ClassPredict(NCPU=NCPU)
PM5 = ClassPredict5(NCPU=NCPU)
na = MS.na
nd = SM.NDir
Load = VS.LoadNextVisChunk()
Jones = self.GiveJones()
log.print(ModColor.Str("Substract sources ... ", col="green"))
#SM.SelectSubCat(SM.SourceCat.kill==0)
t0 = np.mean(VS.MS.times_all)
dt = VS.MS.dt
def VariableFunc(t, nu):
f = np.exp(-(t - t0) / (3 * dt))
if f > 1:
return 0.
else:
return f
# PredictData=PM.predictKernelPolCluster(VS.ThisDataChunk,SM,ApplyTimeJones=Jones,Noise=Noise,
# VariableFunc=VariableFunc)
PredictData = PM.predictKernelPolCluster(VS.ThisDataChunk,
SM,
ApplyTimeJones=Jones,
Noise=Noise,
VariableFunc=None)
print(Jones["Beam"].ravel()[0])
# PredictData=PM5.predictKernelPolCluster(VS.ThisDataChunk,SM,ApplyTimeJones=Jones)
# import pylab
# ind=np.where((VS.ThisDataChunk["A0"]==7)&(VS.ThisDataChunk["A1"]==17))[0]
# op0=np.real
# op1=np.abs
# pylab.clf()
# pylab.subplot(2,1,1)
# pylab.plot(op0(PredictData[ind,0,0]))
# pylab.plot(op0(PredictData5[ind,0,0]))
# pylab.plot(op0(PredictData[ind,0,0])-op0(PredictData5[ind,0,0]))
# pylab.subplot(2,1,2)
# pylab.plot(op1(PredictData[ind,0,0]))
# pylab.plot(op1(PredictData5[ind,0,0]))
# pylab.plot(op1(PredictData[ind,0,0])-op1(PredictData5[ind,0,0]))
# pylab.draw()
# pylab.show(False)
# stop
# #PredictData=PM.predictKernelPolCluster(VS.ThisDataChunk,SM)
#SM.RestoreCat()
MS.data = PredictData
#VS.MS.SaveVis(Col="DATA")
#VS.MS.SaveVis(Col="CORRECTED_DATA")
VS.MS.SaveVis(Col="CORRECTED_DATA")
#VS.MS.SaveVis(Col="CORRECTED_DATA")
t = table(self.MSName, readonly=False)
# dp=t.getcol("DDF_PREDICT")
# dp.fill(0)
# dp[:,:,0]=1.
# dp[:,:,-1]=1.
# t.putcol("DDF_PREDICT",dp)
# t.putcol("DATA",MS.data-dp)
f = t.getcol("FLAG")
f.fill(0)
#r=np.random.rand(*(f.shape[0:2]))
#ff=(r<0.3)
# indr,indf=np.where(ff)
# f[indr,indf,:]=True
# # MS.flag_all=f
# # MS.data[f]=1.e10
t.putcol("FLAG", f)
#t.putcol("FLAG_BACKUP",f)
w = t.getcol("IMAGING_WEIGHT")
w.fill(1)
t.putcol("IMAGING_WEIGHT", w)
# # ###############################
# w.fill(0)
# A0s = t.getcol("ANTENNA1")
# A1s = t.getcol("ANTENNA2")
# C2=self.Cr#np.dot(self.Cr.T.conj(),self.Cr)
# V=C2[A0s,A1s]
# ind=(V==0)
# V[ind]=1e10
# w0=1./(V+Noise**2)
# w0[ind]=0
# for ich in np.arange(w.shape[1]):
# w[:,ich]=np.abs(w0)
# np.save("WSim.npy",w)
t.close()
Sols = self.Sols
FileName = "%s/killMS.%s.sols.npz" % (self.VS.MS.MSName, "Simul")
print("Saving %s" % FileName)
print(self.FreqDomains)
if not self.DoClusterJones:
np.savez(FileName,
Sols=Sols,
StationNames=MS.StationNames,
SkyModel=SM.ClusterCat,
ClusterCat=SM.ClusterCat,
BeamTimes=np.array([], np.float64),
FreqDomains=self.FreqDomains)
else:
np.savez(FileName,
Sols=self.SolsCluster,
StationNames=MS.StationNames,
SkyModel=self.kMS_ClusterDirCat,
ClusterCat=self.kMS_ClusterDirCat,
BeamTimes=np.array([], np.float64),
FreqDomains=self.FreqDomains)
def GiveJones(self):
if self.Sols is None:
Sols = self.GiveSols()
else:
Sols = self.Sols
MS = self.MS
SM = self.SM
VS = self.VS
ApplyBeam = self.ApplyBeam
na = MS.na
nd = SM.NDir
if self.BeamAt == "facet":
NDir = SM.SourceCat.shape[0]
SM.SourceCat.Cluster = np.arange(NDir)
SM.Dirs = SM.SourceCat.Cluster
SM.NDir = NDir
SM.ClusterCat = np.zeros((NDir, ), SM.ClusterCat.dtype)
SM.ClusterCat = SM.ClusterCat.view(np.recarray)
SM.ClusterCat.ra = SM.SourceCat.ra
SM.ClusterCat.dec = SM.SourceCat.dec
Jones = {}
Jones["t0"] = Sols.t0
Jones["t1"] = Sols.t1
nt, nch, na, nd, _, _ = Sols.G.shape
G = np.swapaxes(Sols.G, 1, 3).reshape((nt, nd, na, nch, 2, 2))
# G[:,:,:,:,0,0]/=np.abs(G[:,:,:,:,0,0])
# G[:,:,:,:,1,1]=G[:,:,:,:,0,0]
# G.fill(0)
# G[:,:,:,:,0,0]=1
# G[:,:,:,:,1,1]=1
nt, nd, na, nch, _, _ = G.shape
# G=np.random.randn(*G.shape)+1j*np.random.randn(*G.shape)
useArrayFactor = True
useElementBeam = False
if ApplyBeam:
print(ModColor.Str("Apply Beam"))
MS.LoadSR(useElementBeam=useElementBeam,
useArrayFactor=useArrayFactor)
RA = SM.ClusterCat.ra
DEC = SM.ClusterCat.dec
NDir = RA.size
Tm = Sols.tm
T0s = Sols.t0
T1s = Sols.t1
DicoBeam = {}
DicoBeam["Jones"] = np.zeros(
(Tm.size, NDir, MS.na, MS.NSPWChan, 2, 2), dtype=np.complex64)
DicoBeam["t0"] = np.zeros((Tm.size, ), np.float64)
DicoBeam["t1"] = np.zeros((Tm.size, ), np.float64)
DicoBeam["tm"] = np.zeros((Tm.size, ), np.float64)
rac, decc = MS.OriginalRadec
def GB(time, ra, dec):
Beam = np.zeros(
(ra.shape[0], self.MS.na, self.MS.NSPWChan, 2, 2),
dtype=np.complex)
# Beam[...,0,0]=1
# Beam[...,1,1]=1
# return Beam
for i in range(ra.shape[0]):
self.MS.SR.setDirection(ra[i], dec[i])
Beam[i] = self.MS.SR.evaluate(time)
return Beam
for itime in range(Tm.size):
print(itime)
DicoBeam["t0"][itime] = T0s[itime]
DicoBeam["t1"][itime] = T1s[itime]
DicoBeam["tm"][itime] = Tm[itime]
ThisTime = Tm[itime]
Beam = GB(ThisTime, RA, DEC)
###### Normalise
Beam0 = GB(ThisTime, np.array([rac]), np.array([decc]))
Beam0inv = ModLinAlg.BatchInverse(Beam0)
nd, _, _, _, _ = Beam.shape
Ones = np.ones((nd, 1, 1, 1, 1), np.float32)
Beam0inv = Beam0inv * Ones
nd_, na_, nf_, _, _ = Beam.shape
# Beam_=np.ones((nd_,na_,nf_),np.float32)*(1+np.arange(nd_).reshape((-1,1,1)))
# Beam.fill(0)
# Beam[:,:,:,0,0]=Beam_[:,:,:]
# Beam[:,:,:,1,1]=Beam_[:,:,:]
Beam = ModLinAlg.BatchDot(Beam0inv, Beam)
######
DicoBeam["Jones"][itime] = Beam
nt, nd, na, nch, _, _ = DicoBeam["Jones"].shape
#m=np.mean(np.abs(DicoBeam["Jones"][:,1,:,:,:,:]))
# m=np.mean(np.abs(DicoBeam["Jones"][:,1,:,:,:,:]))
# DicoBeam["Jones"][:,1,0:6,:,:,:]*=2
# DicoBeam["Jones"][:,1,:,:,:,:]/=np.mean(np.abs(DicoBeam["Jones"][:,1,:,:,:,:]))
# DicoBeam["Jones"][:,1,:,:,:,:]*=m
# #################"
# # Single Channel
# DicoBeam["Jones"]=np.mean(DicoBeam["Jones"],axis=3).reshape((nt,nd,na,1,2,2))
# G=ModLinAlg.BatchDot(G,DicoBeam["Jones"])
# #################"
# Multiple Channel
Ones = np.ones((1, 1, 1, nch, 1, 1), np.float32)
G = G * Ones
G = ModLinAlg.BatchDot(G, DicoBeam["Jones"])
# #################"
# G[:,:,:,:,0,0]=1
# G[:,:,:,:,0,1]=0.5
# G[:,:,:,:,1,0]=2.
# G[:,:,:,:,1,1]=1
print("Done")
# #################
# Multiple Channel
self.ChanMap = range(nch)
# #################
Jones["Beam"] = G
Jones["BeamH"] = ModLinAlg.BatchH(G)
if self.ChanMap is None:
self.ChanMap = np.zeros((VS.MS.NSPWChan, ), np.int32).tolist()
Jones["ChanMap"] = self.ChanMap
# ###### for PM5
# Jones["Map_VisToJones_Freq"]=self.ChanMap
# Jones["Jones"]=Jones["Beam"]
# nt=VS.MS.times_all.size
# ntJones=DicoBeam["tm"].size
# d=VS.MS.times_all.reshape((nt,1))-DicoBeam["tm"].reshape((1,ntJones))
# Jones["Map_VisToJones_Time"]=np.argmin(np.abs(d),axis=1)
return Jones