def InitFromCatalog(self):
FileCoords = self.FileCoords
dtype = [('Name', 'S200'), ("ra", np.float64), ("dec", np.float64),
('Type', 'S200')]
#FileCoords="Transient_LOTTS.csv"
self.PosArray = np.load(FileCoords)
self.PosArray = self.PosArray.view(np.recarray)
self.NDirSelected = self.PosArray.shape[0]
self.NDir = self.PosArray.shape[0]
self.DicoDATA = shared_dict.create("DATA")
self.DicoGrids = shared_dict.create("Grids")
self.DicoGrids["GridSTD"] = np.zeros((self.na, self.NTimes),
np.float64)
self.DoJonesCorr_kMS = False
self.DicoJones = None
if self.SolsName:
self.DoJonesCorr_kMS = True
self.DicoJones_kMS = shared_dict.create("DicoJones_kMS")
self.DoJonesCorr_Beam = False
if self.BeamModel:
self.DoJonesCorr_Beam = True
self.DicoJones_Beam = shared_dict.create("DicoJones_Beam")
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=self.NCPU, affinity=0)
APP.startWorkers()
def InitFromCatalog(self):
FileCoords = self.FileCoords
dtype = [('Name', 'S200'), ("ra", np.float64), ("dec", np.float64),
('Type', 'S200')]
#FileCoords="Transient_LOTTS.csv"
self.PosArray = np.load(FileCoords)
self.PosArray = self.PosArray.view(np.recarray)
self.NDirSelected = self.PosArray.shape[0]
self.NDir = self.PosArray.shape[0]
self.DicoDATA = shared_dict.create("DATA")
self.DicoGrids = shared_dict.create("Grids")
dChan = np.min([self.StepFreq, self.NChan])
self.DicoGrids["DomainEdges_Freq"] = np.int64(
np.linspace(0, self.NChan,
int(self.NChan / dChan) + 1))
#self.DicoGrids["DomainEdges_Time"] = np.int64(np.linspace(0,self.NTimes-1,int(self.NTimes/self.StepTime)+1))
DT = self.times.max() - self.times.min()
DTSol = np.min([self.StepTime * self.dt, DT])
self.DicoGrids["DomainEdges_Time"] = np.linspace(
self.times.min() - 1e-6,
self.times.max() + 1e-6,
int(DT / DTSol) + 1)
self.DicoGrids["GridC2"] = np.zeros(
(self.DicoGrids["DomainEdges_Time"].size - 1,
self.DicoGrids["DomainEdges_Freq"].size - 1, self.na, self.na),
np.complex128)
self.DicoGrids["GridC"] = np.zeros(
(self.DicoGrids["DomainEdges_Time"].size - 1,
self.DicoGrids["DomainEdges_Freq"].size - 1, self.na, self.na),
np.complex128)
log.print(" DomainEdges_Freq: %s" %
(str(self.DicoGrids["DomainEdges_Freq"])))
log.print(" DomainEdges_Time: %s" %
(str(self.DicoGrids["DomainEdges_Time"])))
self.DoJonesCorr_kMS = False
self.DicoJones = None
if self.SolsName:
self.DoJonesCorr_kMS = True
self.DicoJones_kMS = shared_dict.create("DicoJones_kMS")
self.DoJonesCorr_Beam = False
if self.BeamModel:
self.DoJonesCorr_Beam = True
self.DicoJones_Beam = shared_dict.create("DicoJones_Beam")
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=self.NCPU, affinity=0)
APP.startWorkers()
def __init__(self,
GD,
NFreqBands,
RefFreq,
MainCache=None,
IdSharedMem=""):
self.GD = GD
self.InitMachine = ClassInitSSDModel(GD, NFreqBands, RefFreq,
MainCache, IdSharedMem)
self.NCPU = (self.GD["Parallel"]["NCPU"] or psutil.cpu_count())
APP.registerJobHandlers(self)
return Aout
# wrappers that discard return value for use with APP -- avoids wasteful stuffing of images into result queues
def _convolveSingleGaussianFFTW_noret(*args, **kw):
_convolveSingleGaussianFFTW(*args, **kw)
return None
def _convolveSingleGaussianNP_noret(*args, **kw):
_convolveSingleGaussianNP(*args, **kw)
return None
APP.registerJobHandlers(_convolveSingleGaussianFFTW_noret,
_convolveSingleGaussianNP_noret)
## FFTW version
#def ConvolveGaussianFFTW(Ain0,
# CellSizeRad=None,
# GaussPars=[(0.,0.,0.)],
# Normalise=False,
# out=None,
# nthreads=1,
# min_size_fft=2048):
# warnings.warn("deprecated", DeprecationWarning)
# assert Ain0.shape == 4, "Expected stack of images: nch, npol, Ny, Nx"
# nch,npol,Ny,Nx=Ain0.shape
# pady = max(Ny, min_size_fft)
# padx = max(Nx, min_size_fft)
def __init__(self, Gain=0.3,
MaxMinorIter=100,
NCPU=1, #psutil.cpu_count()
CycleFactor=2.5,
FluxThreshold=None,
RMSFactor=3,
PeakFactor=0,
PrevPeakFactor=0,
GD=None,
SearchMaxAbs=1,
ModelMachine=None,
NFreqBands=1,
RefFreq=None,
MainCache=None,
IdSharedMem="",
ParallelMode=True,
CacheFileName="HMPBasis",
**kw # absorb any unknown keywords arguments into this
):
"""
ImageDeconvMachine constructor. Note that this should be called pretty much when setting up the imager,
before APP workers are started, because the object registers APP handlers.
"""
self.IdSharedMem=IdSharedMem
self.SearchMaxAbs=SearchMaxAbs
self._ModelImage = None
self.MaxMinorIter = MaxMinorIter
self.NCPU = NCPU
self.Chi2Thr = 10000
self._MaskArray = None
self.GD = GD
self.SubPSF = None
self.MultiFreqMode = NFreqBands > 1
self.NFreqBands = NFreqBands
self.RefFreq = RefFreq
self.FluxThreshold = FluxThreshold
self.CycleFactor = CycleFactor
self.RMSFactor = RMSFactor
self.PeakFactor = PeakFactor
self.PrevPeakFactor = PrevPeakFactor
self.CacheFileName=CacheFileName
self.GainMachine=ClassGainMachine.get_instance()
self.ModelMachine = None
self.PSFServer = None
if ModelMachine is not None:
self.updateModelMachine(ModelMachine)
self.PSFHasChanged=False
self._previous_initial_peak = None
self.maincache = MainCache
# reset overall iteration counter
self._niter = 0
self.facetcache=None
self._MaskArray=None
self.MaskMachine=None
self.ParallelMode=ParallelMode
if self.ParallelMode:
APP.registerJobHandlers(self)
# we are in a worker
if not self.ParallelMode:
numexpr.set_num_threads(NCPU)
# peak finding mode.
# "normal" searches for peak in mean dirty image
# "sigma" searches for peak in mean_dirty/noise_map (setNoiseMap will have been called)
# "weighted" searched for peak in mean_dirty*weight
self._peakMode = "normal"
self.CurrentNegMask=None
self._NoiseMap=None
self._PNRStop=None # in _peakMode "sigma", provides addiitonal stopping criterion
if self.GD["HMP"]["PeakWeightImage"]:
print>> log, " Reading peak weighting image %s" % self.GD["HMP"]["PeakWeightImage"]
img = image(self.GD["HMP"]["PeakWeightImage"]).getdata()
_, _, nx, ny = img.shape
# collapse freq and pol axes
img = img.sum(axis=1).sum(axis=0).T[::-1].copy()
self._peakWeightImage = img.reshape((1,1,ny,nx))
self._peakMode = "weighted"
self._prevPeak = None
def __init__(self, **kwargs):
for key, value in kwargs.items():
setattr(self, key, value)
LMS = [l.strip() for l in open(self.MSOutFreq).readlines()]
self.OutFreqDomains = np.zeros((len(LMS) * self.NFreqPerMS, 2),
np.float64)
iFTot = 0
for iMS, MS in enumerate(LMS):
t = table("%s::SPECTRAL_WINDOW" % MS, ack=False)
df = t.getcol("CHAN_WIDTH").flat[0]
fs = t.getcol("CHAN_FREQ").ravel()
f0, f1 = fs[0] - df / 2., fs[-1] + df / 2.
ff = np.linspace(f0, f1, self.NFreqPerMS + 1)
for iF in range(self.NFreqPerMS):
self.OutFreqDomains[iFTot, 0] = ff[iF]
self.OutFreqDomains[iFTot, 1] = ff[iF + 1]
iFTot += 1
NFreqsOut = self.OutFreqDomains.shape[0]
self.CentralFreqs = np.mean(self.OutFreqDomains, axis=1)
log.print("Loading %s" % self.SolsFileIn)
self.DicoFile0 = dict(np.load(self.SolsFileIn))
Dico0 = self.DicoFile0
self.Sols0 = self.DicoFile0["Sols"].view(np.recarray)
DicoOut = {}
DicoOut['ModelName'] = Dico0['ModelName']
DicoOut['StationNames'] = Dico0['StationNames']
DicoOut['BeamTimes'] = Dico0['BeamTimes']
DicoOut['SourceCatSub'] = Dico0['SourceCatSub']
DicoOut['ClusterCat'] = Dico0['ClusterCat']
DicoOut['SkyModel'] = Dico0['SkyModel']
DicoOut['FreqDomains'] = self.OutFreqDomains
self.DicoOut = DicoOut
self.CentralFreqsIn = np.mean(Dico0['FreqDomains'], axis=1)
self.DicoFreqWeights = {}
for iChan in range(self.CentralFreqs.size):
f = self.CentralFreqs[iChan]
i0 = np.where(self.CentralFreqsIn <= f)[0]
i1 = np.where(self.CentralFreqsIn > f)[0]
if i0.size > 0 and i1.size > 0:
i0 = i0[-1]
i1 = i1[0]
f0 = self.CentralFreqsIn[i0]
f1 = self.CentralFreqsIn[i1]
df = np.abs(f0 - f1)
alpha = 1. - (f - f0) / df
c0 = alpha
c1 = 1. - alpha
self.DicoFreqWeights[iChan] = {
"Type": "Dual",
"Coefs": (c0, c1),
"Index": (i0, i1)
}
else:
i0 = np.argmin(np.abs(self.CentralFreqsIn - f))
self.DicoFreqWeights[iChan] = {"Type": "Single", "Index": i0}
# for iChan in range(self.CentralFreqs.size):
# print
# print self.CentralFreqs[iChan]/1e6
# if self.DicoFreqWeights[iChan]["Type"]=="Dual":
# i0,i1=self.DicoFreqWeights[iChan]["Index"]
# print self.CentralFreqsIn[i0]/1e6,self.CentralFreqsIn[i1]/1e6,self.DicoFreqWeights[iChan]["Coefs"]
# else:
# i0=self.DicoFreqWeights[iChan]["Index"]
# print self.CentralFreqsIn[i0]/1e6
nt, _, na, nd, _, _ = self.Sols0.G.shape
SolsOut = np.zeros(
(nt, ),
dtype=[("t0", np.float64), ("t1", np.float64),
("G", np.complex64, (NFreqsOut, na, nd, 2, 2)),
("Stats", np.float32, (NFreqsOut, na, 4))])
SolsOut = SolsOut.view(np.recarray)
SolsOut.t0 = self.Sols0.t0
SolsOut.t1 = self.Sols0.t1
SolsOut.G[..., 0, 0] = 1.
SolsOut.G[..., 1, 1] = 1.
self.SolsOut = SolsOut
self.GOut = NpShared.ToShared("%sGOut" % IdSharedMem,
self.SolsOut.G.copy())
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=self.NCPU, affinity=0)
def InitFromCatalog(self):
FileCoords = self.FileCoords
dtype = [('Name', 'S200'), ("ra", np.float64), ("dec", np.float64),
('Type', 'S200')]
# should we use the surveys DB?
if 'DDF_PIPELINE_DATABASE' in os.environ:
print("Using the surveys database", file=log)
from surveys_db import SurveysDB
with SurveysDB() as sdb:
sdb.cur.execute('select * from transients')
result = sdb.cur.fetchall()
# convert to a list, then to ndarray, then to recarray
l = []
for r in result:
l.append((r['id'], r['ra'], r['decl'], r['type']))
if FileCoords is not None:
print('Adding data from file ' + FileCoords, file=log)
additional = np.genfromtxt(FileCoords,
dtype=dtype,
delimiter=",")[()]
if not additional.shape:
# deal with a one-line input file
additional = np.array([additional], dtype=dtype)
for r in additional:
l.append(tuple(r))
self.PosArray = np.asarray(l, dtype=dtype)
print("Created an array with %i records" % len(result), file=log)
else:
#FileCoords="Transient_LOTTS.csv"
if FileCoords is None:
if not os.path.isfile(FileCoords):
ssExec = "wget -q --user=anonymous ftp://ftp.strw.leidenuniv.nl/pub/tasse/%s -O %s" % (
FileCoords, FileCoords)
print("Downloading %s" % FileCoords, file=log)
print(" Executing: %s" % ssExec, file=log)
os.system(ssExec)
log.print("Reading cvs file: %s" % FileCoords)
#self.PosArray=np.genfromtxt(FileCoords,dtype=dtype,delimiter=",")[()]
self.PosArray = np.genfromtxt(FileCoords,
dtype=dtype,
delimiter=",")
self.PosArray = self.PosArray.view(np.recarray)
self.PosArray.ra *= np.pi / 180.
self.PosArray.dec *= np.pi / 180.
Radius = self.Radius
NOrig = self.PosArray.Name.shape[0]
Dist = AngDist(self.ra0, self.PosArray.ra, self.dec0,
self.PosArray.dec)
ind = np.where(Dist < (Radius * np.pi / 180))[0]
self.PosArray = self.PosArray[ind]
self.NDirSelected = self.PosArray.shape[0]
print("Selected %i target [out of the %i in the original list]" %
(self.NDirSelected, NOrig),
file=log)
if self.NDirSelected == 0:
print(ModColor.Str(" Have found no sources - returning"),
file=log)
self.killWorkers()
return
NOff = self.NOff
if NOff == -1:
NOff = self.PosArray.shape[0] * 2
if NOff is not None:
print("Including %i off targets" % (NOff), file=log)
self.PosArray = np.concatenate(
[self.PosArray, self.GiveOffPosArray(NOff)])
self.PosArray = self.PosArray.view(np.recarray)
self.NDir = self.PosArray.shape[0]
print("For a total of %i targets" % (self.NDir), file=log)
self.DicoDATA = shared_dict.create("DATA")
self.DicoGrids = shared_dict.create("Grids")
self.DicoGrids["GridLinPol"] = np.zeros(
(self.NDir, self.NChan, self.NTimes, 4), np.complex128)
self.DicoGrids["GridWeight"] = np.zeros(
(self.NDir, self.NChan, self.NTimes, 4), np.complex128)
self.DoJonesCorr_kMS = False
self.DicoJones = None
if self.SolsName:
self.DoJonesCorr_kMS = True
self.DicoJones_kMS = shared_dict.create("DicoJones_kMS")
self.DoJonesCorr_Beam = False
if self.BeamModel:
self.DoJonesCorr_Beam = True
self.DicoJones_Beam = shared_dict.create("DicoJones_Beam")
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=self.NCPU, affinity=0)
APP.startWorkers()
def __init__(self, name=None, mode=2):
self.name = name or "SMM.%x" % id(self)
APP.registerJobHandlers(self)
self._job_counter = APP.createJobCounter(self.name)
self._data = self._blockdict = self._sizedict = None
def __init__(self,
InSolsName,
OutSolsName,
InterpMode="TEC",
PolMode="Scalar",
Amp_PolyOrder=3,
NCPU=0,
Amp_GaussKernel=(0, 5),
Amp_SmoothType="Poly",
CrossMode=1,
RemoveAmpBias=0,
RemoveMedianAmp=True):
if type(InterpMode) == str:
InterpMode = InterpMode.split(",") #[InterpMode]
self.InSolsName = InSolsName
self.OutSolsName = OutSolsName
self.RemoveMedianAmp = RemoveMedianAmp
log.print("Loading %s" % self.InSolsName)
self.DicoFile = dict(np.load(self.InSolsName, allow_pickle=True))
self.Sols = self.DicoFile["Sols"].view(np.recarray)
if "MaskedSols" in self.DicoFile.keys():
MaskFreq = np.logical_not(
np.all(np.all(np.all(self.DicoFile["MaskedSols"][..., 0, 0],
axis=0),
axis=1),
axis=1))
nt, _, na, nd, _, _ = self.Sols.G.shape
self.DicoFile["FreqDomains"] = self.DicoFile["FreqDomains"][
MaskFreq]
NFreqsOut = np.count_nonzero(MaskFreq)
log.print("There are %i non-zero freq channels" % NFreqsOut)
SolsOut = np.zeros(
(nt, ),
dtype=[("t0", np.float64), ("t1", np.float64),
("G", np.complex64, (NFreqsOut, na, nd, 2, 2)),
("Stats", np.float32, (NFreqsOut, na, 4))])
SolsOut = SolsOut.view(np.recarray)
SolsOut.G = self.Sols.G[:, MaskFreq, ...]
SolsOut.t0 = self.Sols.t0
SolsOut.t1 = self.Sols.t1
self.Sols = self.DicoFile["Sols"] = SolsOut
del (self.DicoFile["MaskedSols"])
#self.Sols=self.Sols[0:10].copy()
self.CrossMode = CrossMode
self.CentralFreqs = np.mean(self.DicoFile["FreqDomains"], axis=1)
self.incrCross = 11
iii = 0
NTEC = 101
NConstPhase = 51
TECGridAmp = 0.1
TECGrid, CPhase = np.mgrid[-TECGridAmp:TECGridAmp:NTEC * 1j,
-np.pi:np.pi:NConstPhase * 1j]
Z = TECToZ(TECGrid.reshape((-1, 1)), CPhase.reshape((-1, 1)),
self.CentralFreqs.reshape((1, -1)))
self.Z = Z
self.TECGrid, self.CPhase = TECGrid, CPhase
self.InterpMode = InterpMode
self.Amp_PolyOrder = Amp_PolyOrder
self.RemoveAmpBias = RemoveAmpBias
if self.RemoveAmpBias:
self.CalcFreqAmpSystematics()
self.Sols.G /= self.G0
self.GOut = NpShared.ToShared("%sGOut" % IdSharedMem,
self.Sols.G.copy())
self.PolMode = PolMode
self.Amp_GaussKernel = Amp_GaussKernel
if len(self.Amp_GaussKernel) != 2:
raise ValueError("GaussKernel should be of size 2")
self.Amp_SmoothType = Amp_SmoothType
if "TEC" in self.InterpMode:
log.print(" Smooth phases using a TEC model")
if self.CrossMode:
log.print(ModColor.Str("Using CrossMode"))
if "Amp" in self.InterpMode:
if Amp_SmoothType == "Poly":
log.print(
" Smooth amplitudes using polynomial model of order %i" %
self.Amp_PolyOrder)
if Amp_SmoothType == "Gauss":
log.print(
" Smooth amplitudes using Gaussian kernel of %s (Time/Freq) bins"
% str(Amp_GaussKernel))
if self.RemoveAmpBias:
self.GOut *= self.G0
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=NCPU, affinity=0)
def __init__(self,
ListMSName,
ColName="DATA",
ModelName="PREDICT_KMS",
UVRange=[1., 1000.],
SolsName=None,
FileCoords=None,
Radius=3.,
NOff=-1,
Image=None,
SolsDir=None,
NCPU=1):
self.ListMSName = sorted(ListMSName) #[0:2]
self.nMS = len(self.ListMSName)
self.ColName = ColName
self.ModelName = ModelName
self.OutName = self.ListMSName[0].split("/")[-1].split("_")[0]
self.UVRange = UVRange
self.ReadMSInfos()
self.Radius = Radius
self.Image = Image
self.SolsDir = SolsDir
#self.PosArray=np.genfromtxt(FileCoords,dtype=[('Name','S200'),("ra",np.float64),("dec",np.float64),('Type','S200')],delimiter="\t")
# identify version in logs
print >> log, "DynSpecMS version %s starting up" % version()
# should we use the surveys DB?
if 'DDF_PIPELINE_DATABASE' in os.environ:
print >> log, "Using the surveys database"
from surveys_db import SurveysDB
with SurveysDB() as sdb:
sdb.cur.execute('select * from transients')
result = sdb.cur.fetchall()
# convert to a list, then to ndarray, then to recarray
l = []
for r in result:
l.append((r['id'], r['ra'], r['decl'], r['type']))
self.PosArray = np.asarray(l,
dtype=[('Name', 'S200'),
("ra", np.float64),
("dec", np.float64),
('Type', 'S200')])
print >> log, "Created an array with %i records" % len(result)
else:
if FileCoords is None:
FileCoords = "Transient_LOTTS.csv"
if not os.path.isfile(FileCoords):
ssExec = "wget -q --user=anonymous ftp://ftp.strw.leidenuniv.nl/pub/tasse/%s -O %s" % (
FileCoords, FileCoords)
print >> log, "Downloading %s" % FileCoords
print >> log, " Executing: %s" % ssExec
os.system(ssExec)
self.PosArray = np.genfromtxt(FileCoords,
dtype=[('Name', 'S200'),
("ra", np.float64),
("dec", np.float64),
('Type', 'S200')],
delimiter=",")[()]
self.PosArray = self.PosArray.view(np.recarray)
self.PosArray.ra *= np.pi / 180.
self.PosArray.dec *= np.pi / 180.
NOrig = self.PosArray.shape[0]
Dist = AngDist(self.ra0, self.PosArray.ra, self.dec0,
self.PosArray.dec)
ind = np.where(Dist < Radius * np.pi / 180)[0]
self.PosArray = self.PosArray[ind]
self.NDirSelected = self.PosArray.shape[0]
print >> log, "Selected %i target [out of the %i in the original list]" % (
self.NDirSelected, NOrig)
if self.NDirSelected == 0:
print >> log, ModColor.Str(" Have found no sources - returning")
self.killWorkers()
return
if NOff == -1:
NOff = self.PosArray.shape[0] * 2
if NOff is not None:
print >> log, "Including %i off targets" % (NOff)
self.PosArray = np.concatenate(
[self.PosArray, self.GiveOffPosArray(NOff)])
self.PosArray = self.PosArray.view(np.recarray)
self.NDir = self.PosArray.shape[0]
print >> log, "For a total of %i targets" % (self.NDir)
self.DicoDATA = shared_dict.create("DATA")
self.DicoGrids = shared_dict.create("Grids")
self.DicoGrids["GridLinPol"] = np.zeros(
(self.NDir, self.NChan, self.NTimes, 4), np.complex128)
self.DicoGrids["GridWeight"] = np.zeros(
(self.NDir, self.NChan, self.NTimes, 4), np.complex128)
self.SolsName = SolsName
self.DoJonesCorr = False
if self.SolsName:
self.DoJonesCorr = True
self.DicoJones = shared_dict.create("DicoJones")
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=NCPU, affinity=0)
APP.startWorkers()
