def InitFromCatalog(self):
FileCoords = self.FileCoords
dtype = [('Name', 'S200'), ("ra", np.float64), ("dec", np.float64),
('Type', 'S200')]
#FileCoords="Transient_LOTTS.csv"
self.PosArray = np.load(FileCoords)
self.PosArray = self.PosArray.view(np.recarray)
self.NDirSelected = self.PosArray.shape[0]
self.NDir = self.PosArray.shape[0]
self.DicoDATA = shared_dict.create("DATA")
self.DicoGrids = shared_dict.create("Grids")
self.DicoGrids["GridSTD"] = np.zeros((self.na, self.NTimes),
np.float64)
self.DoJonesCorr_kMS = False
self.DicoJones = None
if self.SolsName:
self.DoJonesCorr_kMS = True
self.DicoJones_kMS = shared_dict.create("DicoJones_kMS")
self.DoJonesCorr_Beam = False
if self.BeamModel:
self.DoJonesCorr_Beam = True
self.DicoJones_Beam = shared_dict.create("DicoJones_Beam")
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=self.NCPU, affinity=0)
APP.startWorkers()
def InitFromCatalog(self):
FileCoords = self.FileCoords
dtype = [('Name', 'S200'), ("ra", np.float64), ("dec", np.float64),
('Type', 'S200')]
#FileCoords="Transient_LOTTS.csv"
self.PosArray = np.load(FileCoords)
self.PosArray = self.PosArray.view(np.recarray)
self.NDirSelected = self.PosArray.shape[0]
self.NDir = self.PosArray.shape[0]
self.DicoDATA = shared_dict.create("DATA")
self.DicoGrids = shared_dict.create("Grids")
dChan = np.min([self.StepFreq, self.NChan])
self.DicoGrids["DomainEdges_Freq"] = np.int64(
np.linspace(0, self.NChan,
int(self.NChan / dChan) + 1))
#self.DicoGrids["DomainEdges_Time"] = np.int64(np.linspace(0,self.NTimes-1,int(self.NTimes/self.StepTime)+1))
DT = self.times.max() - self.times.min()
DTSol = np.min([self.StepTime * self.dt, DT])
self.DicoGrids["DomainEdges_Time"] = np.linspace(
self.times.min() - 1e-6,
self.times.max() + 1e-6,
int(DT / DTSol) + 1)
self.DicoGrids["GridC2"] = np.zeros(
(self.DicoGrids["DomainEdges_Time"].size - 1,
self.DicoGrids["DomainEdges_Freq"].size - 1, self.na, self.na),
np.complex128)
self.DicoGrids["GridC"] = np.zeros(
(self.DicoGrids["DomainEdges_Time"].size - 1,
self.DicoGrids["DomainEdges_Freq"].size - 1, self.na, self.na),
np.complex128)
log.print(" DomainEdges_Freq: %s" %
(str(self.DicoGrids["DomainEdges_Freq"])))
log.print(" DomainEdges_Time: %s" %
(str(self.DicoGrids["DomainEdges_Time"])))
self.DoJonesCorr_kMS = False
self.DicoJones = None
if self.SolsName:
self.DoJonesCorr_kMS = True
self.DicoJones_kMS = shared_dict.create("DicoJones_kMS")
self.DoJonesCorr_Beam = False
if self.BeamModel:
self.DoJonesCorr_Beam = True
self.DicoJones_Beam = shared_dict.create("DicoJones_Beam")
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=self.NCPU, affinity=0)
APP.startWorkers()
def startWorkers(self):
"""Starts worker threads. All job handlers and events must be registered *BEFORE*"""
self._shared_state = shared_dict.create("APP")
self._job_counters.finalize(self._shared_state)
if self.ncpu > 1:
self._taras_bulba.start()
self._started = True
def computeSmearMappingInBackground (self, base_job_id, MS, DATA, radiusDeg, Decorr, channel_mapping, mode):
l = radiusDeg * np.pi / 180
dPhi = np.sqrt(6. * (1. - Decorr))
# create new empty shared dicts for results
self._outdict = shared_dict.create("%s:%s:tmp" %(DATA.path, self.name))
blockdict = self._outdict.addSubdict("blocks")
sizedict = self._outdict.addSubdict("sizes")
self._nbl = 0
for a0 in xrange(MS.na):
for a1 in xrange(MS.na):
if a0 != a1:
self._nbl += 1
APP.runJob("%s:%s:%d:%d" % (base_job_id, self.name, a0, a1), self._smearmapping_worker,
counter=self._job_counter, collect_result=False,
args=(DATA.readonly(), blockdict.writeonly(), sizedict.writeonly(), a0, a1, dPhi, l,
channel_mapping, mode))
def __init__(self, VS):
logger.setSilent(["ClassJones", "ClassLOFARBeam"])
self.VS = VS
self.GD = copy.deepcopy(self.VS.GD)
self.DoCentralNorm = self.GD["Beam"]["CenterNorm"]
self.SmoothBeam = None
self.CheckCache()
if self.CacheValid:
logger.setLoud(["ClassJones", "ClassLOFARBeam"])
return
#self.GD["Beam"]["CenterNorm"]=0
self.ListMS = self.VS.ListMS
self.MS = self.ListMS[0]
rac, decc = self.MS.radec
self.CoordMachine = ModCoord.ClassCoordConv(rac, decc)
self.CalcGrid()
#self.Padding=Padding
#self.SumJJsq=np.zeros((self.npix,self.npix,self.MS.Nchan),np.float64)
#self.SumWsq=np.zeros((1,self.MS.Nchan),np.float64)
self.StackedBeamDict = shared_dict.create("StackedBeamDict")
for iDir in range(self.NDir):
sd = self.StackedBeamDict.addSubdict(iDir)
sd.addSharedArray("SumJJsq", (self.VS.NFreqBands, ), np.float64)
sd.addSharedArray("SumWsq", (self.VS.NFreqBands, ), np.float64)
self.DicoJonesMachine = {}
for iMS, MS in enumerate(self.ListMS):
JonesMachine = ClassJones.ClassJones(self.GD, MS,
self.VS.FacetMachine)
JonesMachine.InitBeamMachine()
self.DicoJonesMachine[iMS] = JonesMachine
logger.setLoud(["ClassJones", "ClassLOFARBeam"])
def DeconvListIsland(self,
ListIslands,
ParallelMode="OverIsland",
ListInitIslands=None):
# ================== Parallel part
NIslands = len(ListIslands)
if NIslands == 0: return
if ParallelMode == "OverIslands":
NCPU = self.NCPU
NCPU = np.min([NCPU, NIslands])
Parallel = True
ParallelPerIsland = False
StopWhenQueueEmpty = True
elif ParallelMode == "PerIsland":
NCPU = 1 #self.NCPU
Parallel = True
ParallelPerIsland = True
StopWhenQueueEmpty = True
# ######### Debug
# ParallelPerIsland=False
# Parallel=False
# NCPU=1
# StopWhenQueueEmpty=True
# ##################
work_queue = multiprocessing.Queue()
# shared dict to hold inputs and outputs to workers (each island number is a key)
deconv_dict = shared_dict.create("DeconvListIslands")
NJobs = NIslands
T = ClassTimeIt.ClassTimeIt(" ")
T.disable()
for iIsland, ThisPixList in enumerate(ListIslands):
island_dict = deconv_dict.addSubdict(iIsland)
# print "%i/%i"%(iIsland,self.NIslands)
island_dict["Island"] = np.array(ThisPixList)
XY = np.array(ThisPixList, dtype=np.float32)
xm, ym = np.mean(np.float32(XY), axis=0).astype(int)
T.timeit("xm,ym")
nchan, npol, _, _ = self._Dirty.shape
JonesNorm = (self.DicoDirty["JonesNorm"][:, :, xm, ym]).reshape(
(nchan, npol, 1, 1))
W = self.DicoDirty["WeightChansImages"]
JonesNorm = np.sum(JonesNorm * W.reshape((nchan, 1, 1, 1)),
axis=0).reshape((1, npol, 1, 1))
T.timeit("JonesNorm")
IslandBestIndiv = self.ModelMachine.GiveIndividual(ThisPixList)
T.timeit("GiveIndividual")
FacetID = self.PSFServer.giveFacetID2(xm, ym)
T.timeit("FacetID")
island_dict["BestIndiv"] = IslandBestIndiv
ListOrder = [
iIsland, FacetID, JonesNorm.flat[0], self.RMS**2,
island_dict.path
]
work_queue.put(ListOrder)
T.timeit("Put")
# ListArrayIslands=[np.array(ListIslands[iIsland]) for iIsland in range(NIslands)]
# NpShared.PackListArray(SharedListIsland,ListArrayIslands)
# T.timeit("Pack0")
# SharedBestIndiv="%s.ListBestIndiv"%(self.IdSharedMem)
# NpShared.PackListArray(SharedBestIndiv,ListBestIndiv)
# T.timeit("Pack1")
workerlist = []
# List_Result_queue=[]
# for ii in range(NCPU):
# List_Result_queue.append(multiprocessing.JoinableQueue())
result_queue = multiprocessing.Queue()
Title = " Evolve pop."
if self.DeconvMode == "MetroClean":
Title = " Running chain"
pBAR = ProgressBar(Title=Title)
#pBAR.disable()
pBAR.render(0, NJobs)
for ii in range(NCPU):
W = WorkerDeconvIsland(work_queue,
result_queue,
self.GD,
self._Dirty,
self.DicoVariablePSF["CubeVariablePSF"],
IdSharedMem=self.IdSharedMem,
FreqsInfo=self.PSFServer.DicoMappingDesc,
ParallelPerIsland=ParallelPerIsland,
StopWhenQueueEmpty=StopWhenQueueEmpty,
DeconvMode=self.DeconvMode,
NChains=self.NChains,
ListInitIslands=ListInitIslands)
workerlist.append(W)
if Parallel:
workerlist[ii].start()
else:
workerlist[ii].run()
iResult = 0
#print "!!!!!!!!!!!!!!!!!!!!!!!!",iResult,NJobs
while iResult < NJobs:
DicoResult = None
# for result_queue in List_Result_queue:
# if result_queue.qsize()!=0:
# try:
# DicoResult=result_queue.get_nowait()
# break
# except:
# pass
# #DicoResult=result_queue.get()
#print "!!!!!!!!!!!!!!!!!!!!!!!!! Qsize",result_queue.qsize()
if result_queue.qsize() != 0:
try:
DicoResult = result_queue.get_nowait()
except:
pass
#DicoResult=result_queue.get()
if DicoResult is None:
time.sleep(0.05)
continue
iResult += 1
NDone = iResult
intPercent = int(100 * NDone / float(NJobs))
pBAR.render(NDone, NJobs)
if DicoResult["Success"]:
iIsland = DicoResult["iIsland"]
island_dict = deconv_dict[iIsland]
island_dict.reload()
self.ModelMachine.AppendIsland(ListIslands[iIsland],
island_dict["Model"].copy())
if DicoResult["HasError"]:
self.ErrorModelMachine.AppendIsland(
ThisPixList, ListIslands[iIsland],
island_dict["sModel"].copy())
deconv_dict.delete()
for ii in range(NCPU):
try:
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()
except:
pass
def giveDicoInitIndiv(self,
ListIslands,
ModelImage,
DicoDirty,
ListDoIsland=None):
DicoInitIndiv = shared_dict.create("DicoInitIsland")
ParmDict = shared_dict.create("InitSSDModelHMP")
ParmDict["ModelImage"] = ModelImage
ParmDict["GridFreqs"] = self.GridFreqs
ParmDict["DegridFreqs"] = self.DegridFreqs
# ListBigIslands=[]
# ListSmallIslands=[]
# ListDoBigIsland=[]
# ListDoSmallIsland=[]
# NParallel=0
# for iIsland,Island in enumerate(ListIslands):
# if len(Island)>self.GD["SSDClean"]["ConvFFTSwitch"]:
# ListBigIslands.append(Island)
# ListDoBigIsland.append(ListDoIsland[iIsland])
# if ListDoIsland or ListDoIsland[iIsland]:
# NParallel+=1
# else:
# ListSmallIslands.append(Island)
# ListDoSmallIsland.append(ListDoIsland[iIsland])
# print>>log,"Initialise big islands (parallelised per island)"
# pBAR= ProgressBar(Title="Init islands")
# pBAR.render(0, NParallel)
# nDone=0
# for iIsland,Island in enumerate(ListBigIslands):
# if not ListDoIsland or ListDoBigIsland[iIsland]:
# subdict = DicoInitIndiv.addSubdict(iIsland)
# # APP.runJob("InitIsland:%d" % iIsland, self._initIsland_worker,
# # args=(subdict.writeonly(), iIsland, Island,
# # self.DicoVariablePSF.readonly(), DicoDirty.readonly(),
# # ParmDict.readonly(), self.InitMachine.DeconvMachine.facetcache.readonly(),self.NCPU),serial=True)
# self._initIsland_worker(subdict, iIsland, Island,
# self.DicoVariablePSF, DicoDirty,
# ParmDict, self.InitMachine.DeconvMachine.facetcache,
# self.NCPU)
# pBAR.render(nDone+1, NParallel)
# nDone+=1
# # APP.awaitJobResults("InitIsland:*", progress="Init islands")
# print>>log,"Initialise small islands (parallelised over islands)"
# for iIsland,Island in enumerate(ListSmallIslands):
# if not ListDoIsland or ListDoSmallIsland[iIsland]:
# subdict = DicoInitIndiv.addSubdict(iIsland)
# APP.runJob("InitIsland:%d" % iIsland, self._initIsland_worker,
# args=(subdict.writeonly(), iIsland, Island,
# self.DicoVariablePSF.readonly(), DicoDirty.readonly(),
# ParmDict.readonly(), self.InitMachine.DeconvMachine.facetcache.readonly(),1))
# APP.awaitJobResults("InitIsland:*", progress="Init islands")
# DicoInitIndiv.reload()
print >> log, "Initialise islands (parallelised over islands)"
if not self.GD["GAClean"]["ParallelInitHMP"]:
pBAR = ProgressBar(Title=" Init islands")
for iIsland, Island in enumerate(ListIslands):
if not ListDoIsland or ListDoIsland[iIsland]:
subdict = DicoInitIndiv.addSubdict(iIsland)
self._initIsland_worker(
subdict, iIsland, Island, self.DicoVariablePSF,
DicoDirty, ParmDict,
self.InitMachine.DeconvMachine.facetcache, 1)
pBAR.render(iIsland, len(ListIslands))
else:
for iIsland, Island in enumerate(ListIslands):
if not ListDoIsland or ListDoIsland[iIsland]:
subdict = DicoInitIndiv.addSubdict(iIsland)
APP.runJob("InitIsland:%d" % iIsland,
self._initIsland_worker,
args=(subdict.writeonly(), iIsland, Island,
self.DicoVariablePSF.readonly(),
DicoDirty.readonly(), ParmDict.readonly(),
self.InitMachine.DeconvMachine.facetcache.
readonly(), 1))
APP.awaitJobResults("InitIsland:*", progress="Init islands")
DicoInitIndiv.reload()
ParmDict.delete()
return DicoInitIndiv
def InitMSMF(self, approx=False, cache=True, facetcache=None):
"""Initializes MSMF basis functions. If approx is True, then uses the central facet's PSF for
all facets.
Populates the self.facetcache dict, unless facetcache is supplied
"""
self.DicoMSMachine = {}
valid = True
if facetcache is not None:
print>> log, "HMP basis functions pre-initialized"
self.facetcache = facetcache
else:
cachehash = dict(
[(section, self.GD[section]) for section in (
"Data", "Beam", "Selection", "Freq",
"Image", "Facets", "Weight", "RIME","DDESolutions",
"Comp", "CF",
"HMP")])
cachepath, valid = self.maincache.checkCache(self.CacheFileName, cachehash, reset=not cache or self.PSFHasChanged)
# do not use cache in approx mode
if approx or not cache:
valid = False
if valid:
print>>log,"Initialising HMP basis functions from cache %s"%cachepath
self.facetcache = shared_dict.create(self.CacheFileName)
self.facetcache.restore(cachepath)
else:
self.facetcache = None
init_cache = self.facetcache is None
if init_cache:
self.facetcache = shared_dict.create(self.CacheFileName)
# in any mode, start by initializing a MS machine for the central facet. This will precompute the scale
# functions
centralFacet = self.PSFServer.DicoVariablePSF["CentralFacet"]
self.DicoMSMachine[centralFacet] = MSM0 = \
self._initMSM_facet(centralFacet,
self.facetcache.addSubdict(centralFacet) if init_cache else self.facetcache[centralFacet],
None, self.SideLobeLevel, self.OffsetSideLobe, verbose=True)
if approx:
print>>log, "HMP approximation mode: using PSF of central facet (%d)" % centralFacet
for iFacet in xrange(self.PSFServer.NFacets):
self.DicoMSMachine[iFacet] = MSM0
elif (self.GD["Facets"]["NFacets"]==1)&(not self.GD["DDESolutions"]["DDSols"]):
self.DicoMSMachine[0] = MSM0
else:
# if no facet cache, init in parallel
if init_cache:
for iFacet in xrange(self.PSFServer.NFacets):
if iFacet != centralFacet:
fcdict = self.facetcache.addSubdict(iFacet)
if self.ParallelMode:
args=(fcdict.writeonly(), MSM0.ScaleFuncs.readonly(), self.DicoVariablePSF.readonly(),
iFacet, self.SideLobeLevel, self.OffsetSideLobe, False)
APP.runJob("InitHMP:%d"%iFacet, self._initMSM_handler,
args=args)
else:
self.DicoMSMachine[iFacet] = \
self._initMSM_facet(iFacet, fcdict, None,
self.SideLobeLevel, self.OffsetSideLobe, MSM0=MSM0, verbose=False)
if self.ParallelMode:
APP.awaitJobResults("InitHMP:*", progress="Init HMP")
self.facetcache.reload()
# t = ClassTimeIt.ClassTimeIt()
# now reinit from cache (since cache was computed by subprocesses)
for iFacet in xrange(self.PSFServer.NFacets):
if iFacet not in self.DicoMSMachine:
self.DicoMSMachine[iFacet] = \
self._initMSM_facet(iFacet, self.facetcache[iFacet], None,
self.SideLobeLevel, self.OffsetSideLobe, MSM0=MSM0, verbose=False)
# write cache to disk, unless in a mode where we explicitly don't want it
if facetcache is None and not valid and cache and not approx:
try:
#MyPickle.DicoNPToFile(facetcache,cachepath)
#cPickle.dump(facetcache, file(cachepath, 'w'), 2)
print>>log," saving HMP cache to %s"%cachepath
self.facetcache.save(cachepath)
#self.maincache.saveCache("HMPMachine")
self.maincache.saveCache(self.CacheFileName)
self.PSFHasChanged=False
print>>log," HMP init done"
except:
print>>log, traceback.format_exc()
print >>log, ModColor.Str(
"WARNING: HMP cache could not be written, see error report above. Proceeding anyway.")
def InitFromCatalog(self):
FileCoords = self.FileCoords
dtype = [('Name', 'S200'), ("ra", np.float64), ("dec", np.float64),
('Type', 'S200')]
# should we use the surveys DB?
if 'DDF_PIPELINE_DATABASE' in os.environ:
print("Using the surveys database", file=log)
from surveys_db import SurveysDB
with SurveysDB() as sdb:
sdb.cur.execute('select * from transients')
result = sdb.cur.fetchall()
# convert to a list, then to ndarray, then to recarray
l = []
for r in result:
l.append((r['id'], r['ra'], r['decl'], r['type']))
if FileCoords is not None:
print('Adding data from file ' + FileCoords, file=log)
additional = np.genfromtxt(FileCoords,
dtype=dtype,
delimiter=",")[()]
if not additional.shape:
# deal with a one-line input file
additional = np.array([additional], dtype=dtype)
for r in additional:
l.append(tuple(r))
self.PosArray = np.asarray(l, dtype=dtype)
print("Created an array with %i records" % len(result), file=log)
else:
#FileCoords="Transient_LOTTS.csv"
if FileCoords is None:
if not os.path.isfile(FileCoords):
ssExec = "wget -q --user=anonymous ftp://ftp.strw.leidenuniv.nl/pub/tasse/%s -O %s" % (
FileCoords, FileCoords)
print("Downloading %s" % FileCoords, file=log)
print(" Executing: %s" % ssExec, file=log)
os.system(ssExec)
log.print("Reading cvs file: %s" % FileCoords)
#self.PosArray=np.genfromtxt(FileCoords,dtype=dtype,delimiter=",")[()]
self.PosArray = np.genfromtxt(FileCoords,
dtype=dtype,
delimiter=",")
self.PosArray = self.PosArray.view(np.recarray)
self.PosArray.ra *= np.pi / 180.
self.PosArray.dec *= np.pi / 180.
Radius = self.Radius
NOrig = self.PosArray.Name.shape[0]
Dist = AngDist(self.ra0, self.PosArray.ra, self.dec0,
self.PosArray.dec)
ind = np.where(Dist < (Radius * np.pi / 180))[0]
self.PosArray = self.PosArray[ind]
self.NDirSelected = self.PosArray.shape[0]
print("Selected %i target [out of the %i in the original list]" %
(self.NDirSelected, NOrig),
file=log)
if self.NDirSelected == 0:
print(ModColor.Str(" Have found no sources - returning"),
file=log)
self.killWorkers()
return
NOff = self.NOff
if NOff == -1:
NOff = self.PosArray.shape[0] * 2
if NOff is not None:
print("Including %i off targets" % (NOff), file=log)
self.PosArray = np.concatenate(
[self.PosArray, self.GiveOffPosArray(NOff)])
self.PosArray = self.PosArray.view(np.recarray)
self.NDir = self.PosArray.shape[0]
print("For a total of %i targets" % (self.NDir), file=log)
self.DicoDATA = shared_dict.create("DATA")
self.DicoGrids = shared_dict.create("Grids")
self.DicoGrids["GridLinPol"] = np.zeros(
(self.NDir, self.NChan, self.NTimes, 4), np.complex128)
self.DicoGrids["GridWeight"] = np.zeros(
(self.NDir, self.NChan, self.NTimes, 4), np.complex128)
self.DoJonesCorr_kMS = False
self.DicoJones = None
if self.SolsName:
self.DoJonesCorr_kMS = True
self.DicoJones_kMS = shared_dict.create("DicoJones_kMS")
self.DoJonesCorr_Beam = False
if self.BeamModel:
self.DoJonesCorr_Beam = True
self.DicoJones_Beam = shared_dict.create("DicoJones_Beam")
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=self.NCPU, affinity=0)
APP.startWorkers()
def InitMSMF(self, approx=False, cache=True, facetcache=None):
"""Initializes MSMF basis functions. If approx is True, then uses the central facet's PSF for
all facets.
Populates the self.facetcache dict, unless facetcache is supplied
"""
self.DicoMSMachine = {}
valid = True
if facetcache is not None:
print >> log, "HMP basis functions pre-initialized"
self.facetcache = facetcache
else:
cachehash = dict([
(section, self.GD[section])
for section in ("Data", "Beam", "Selection", "Freq", "Image",
"Facets", "Weight", "RIME", "Comp", "CF",
"HMP")
])
cachepath, valid = self.maincache.checkCache(self.CacheFileName,
cachehash,
reset=not cache
or self.PSFHasChanged)
# do not use cache in approx mode
if approx or not cache:
valid = False
if valid:
print >> log, "Initialising HMP basis functions from cache %s" % cachepath
self.facetcache = shared_dict.create(self.CacheFileName)
self.facetcache.restore(cachepath)
else:
self.facetcache = None
centralFacet = self.PSFServer.DicoVariablePSF["CentralFacet"]
if approx:
print >> log, "HMP approximation mode: using PSF of central facet (%d)" % centralFacet
self.PSFServer.setFacet(centralFacet)
MSMachine = ClassMultiScaleMachine.ClassMultiScaleMachine(
self.GD,
self.facetcache.addSubdict(0),
self.GainMachine,
NFreqBands=self.NFreqBands)
MSMachine.setModelMachine(self.ModelMachine)
MSMachine.setSideLobeLevel(self.SideLobeLevel, self.OffsetSideLobe)
MSMachine.SetFacet(centralFacet)
MSMachine.SetPSF(self.PSFServer) # ThisPSF,ThisMeanPSF)
MSMachine.FindPSFExtent(verbose=True)
MSMachine.MakeMultiScaleCube(verbose=True)
MSMachine.MakeBasisMatrix()
for iFacet in xrange(self.PSFServer.NFacets):
self.DicoMSMachine[iFacet] = MSMachine
else:
# if no facet cache, init in parallel
if self.facetcache is None:
self.facetcache = shared_dict.create(self.CacheFileName)
# breakout = False
for iFacet in xrange(self.PSFServer.NFacets):
fcdict = self.facetcache.addSubdict(iFacet)
if self.ParallelMode:
args = (fcdict.writeonly(),
self.DicoVariablePSF.readonly(), iFacet,
self.SideLobeLevel, self.OffsetSideLobe,
centralFacet)
APP.runJob("InitHMP:%d" % iFacet,
self._initMSM_handler,
args=args)
else:
args = (fcdict, self.DicoVariablePSF, iFacet,
self.SideLobeLevel, self.OffsetSideLobe,
centralFacet)
self._initMSM_handler(*args)
# import pdb;
# pdb.set_trace()
# if breakout:
# raise RuntimeError("exiting")
if self.ParallelMode:
APP.awaitJobResults("InitHMP:*", progress="Init HMP")
self.facetcache.reload()
# t = ClassTimeIt.ClassTimeIt()
for iFacet in xrange(self.PSFServer.NFacets):
self.PSFServer.setFacet(iFacet)
MSMachine = ClassMultiScaleMachine.ClassMultiScaleMachine(
self.GD,
self.facetcache[iFacet],
self.GainMachine,
NFreqBands=self.NFreqBands)
MSMachine.setModelMachine(self.ModelMachine)
MSMachine.setSideLobeLevel(self.SideLobeLevel,
self.OffsetSideLobe)
MSMachine.SetFacet(iFacet)
MSMachine.SetPSF(self.PSFServer) # ThisPSF,ThisMeanPSF)
MSMachine.FindPSFExtent(
verbose=(iFacet == centralFacet
)) # only print to log for central facet
MSMachine.MakeMultiScaleCube(verbose=(iFacet == centralFacet))
MSMachine.MakeBasisMatrix()
self.DicoMSMachine[iFacet] = MSMachine
# write cache to disk, unless in a mode where we explicitly don't want it
if facetcache is None and not valid and cache and not approx:
try:
#MyPickle.DicoNPToFile(facetcache,cachepath)
#cPickle.dump(facetcache, file(cachepath, 'w'), 2)
print >> log, " saving HMP cache to %s" % cachepath
self.facetcache.save(cachepath)
#self.maincache.saveCache("HMPMachine")
self.maincache.saveCache(self.CacheFileName)
self.PSFHasChanged = False
print >> log, " HMP init done"
except:
print >> log, traceback.format_exc()
print >> log, ModColor.Str(
"WARNING: HMP cache could not be written, see error report above. Proceeding anyway."
)
def __init__(self,
ListMSName,
ColName="DATA",
ModelName="PREDICT_KMS",
UVRange=[1., 1000.],
SolsName=None,
FileCoords=None,
Radius=3.,
NOff=-1,
Image=None,
SolsDir=None,
NCPU=1):
self.ListMSName = sorted(ListMSName) #[0:2]
self.nMS = len(self.ListMSName)
self.ColName = ColName
self.ModelName = ModelName
self.OutName = self.ListMSName[0].split("/")[-1].split("_")[0]
self.UVRange = UVRange
self.ReadMSInfos()
self.Radius = Radius
self.Image = Image
self.SolsDir = SolsDir
#self.PosArray=np.genfromtxt(FileCoords,dtype=[('Name','S200'),("ra",np.float64),("dec",np.float64),('Type','S200')],delimiter="\t")
# identify version in logs
print >> log, "DynSpecMS version %s starting up" % version()
# should we use the surveys DB?
if 'DDF_PIPELINE_DATABASE' in os.environ:
print >> log, "Using the surveys database"
from surveys_db import SurveysDB
with SurveysDB() as sdb:
sdb.cur.execute('select * from transients')
result = sdb.cur.fetchall()
# convert to a list, then to ndarray, then to recarray
l = []
for r in result:
l.append((r['id'], r['ra'], r['decl'], r['type']))
self.PosArray = np.asarray(l,
dtype=[('Name', 'S200'),
("ra", np.float64),
("dec", np.float64),
('Type', 'S200')])
print >> log, "Created an array with %i records" % len(result)
else:
if FileCoords is None:
FileCoords = "Transient_LOTTS.csv"
if not os.path.isfile(FileCoords):
ssExec = "wget -q --user=anonymous ftp://ftp.strw.leidenuniv.nl/pub/tasse/%s -O %s" % (
FileCoords, FileCoords)
print >> log, "Downloading %s" % FileCoords
print >> log, " Executing: %s" % ssExec
os.system(ssExec)
self.PosArray = np.genfromtxt(FileCoords,
dtype=[('Name', 'S200'),
("ra", np.float64),
("dec", np.float64),
('Type', 'S200')],
delimiter=",")[()]
self.PosArray = self.PosArray.view(np.recarray)
self.PosArray.ra *= np.pi / 180.
self.PosArray.dec *= np.pi / 180.
NOrig = self.PosArray.shape[0]
Dist = AngDist(self.ra0, self.PosArray.ra, self.dec0,
self.PosArray.dec)
ind = np.where(Dist < Radius * np.pi / 180)[0]
self.PosArray = self.PosArray[ind]
self.NDirSelected = self.PosArray.shape[0]
print >> log, "Selected %i target [out of the %i in the original list]" % (
self.NDirSelected, NOrig)
if self.NDirSelected == 0:
print >> log, ModColor.Str(" Have found no sources - returning")
self.killWorkers()
return
if NOff == -1:
NOff = self.PosArray.shape[0] * 2
if NOff is not None:
print >> log, "Including %i off targets" % (NOff)
self.PosArray = np.concatenate(
[self.PosArray, self.GiveOffPosArray(NOff)])
self.PosArray = self.PosArray.view(np.recarray)
self.NDir = self.PosArray.shape[0]
print >> log, "For a total of %i targets" % (self.NDir)
self.DicoDATA = shared_dict.create("DATA")
self.DicoGrids = shared_dict.create("Grids")
self.DicoGrids["GridLinPol"] = np.zeros(
(self.NDir, self.NChan, self.NTimes, 4), np.complex128)
self.DicoGrids["GridWeight"] = np.zeros(
(self.NDir, self.NChan, self.NTimes, 4), np.complex128)
self.SolsName = SolsName
self.DoJonesCorr = False
if self.SolsName:
self.DoJonesCorr = True
self.DicoJones = shared_dict.create("DicoJones")
APP.registerJobHandlers(self)
AsyncProcessPool.init(ncpu=NCPU, affinity=0)
APP.startWorkers()
