def __init__(self, work_queue, result_queue, IdSharedMem=None, StopWhenQueueEmpty=False, NImGauss=31, DeltaChi2=4., ListGauss=None, GSig=None, SigMin=None, Var=None): multiprocessing.Process.__init__(self) self.work_queue = work_queue self.result_queue = result_queue self.kill_received = False self.exit = multiprocessing.Event() self.IdSharedMem = IdSharedMem self.StopWhenQueueEmpty = StopWhenQueueEmpty self.CubeMeanVariablePSF = NpShared.GiveArray("%sCubeMeanVariablePSF" % self.IdSharedMem) self.MeanModelImage = NpShared.GiveArray("%sMeanModelImage" % self.IdSharedMem) self.MeanResidual = NpShared.GiveArray("%sMeanResidual" % self.IdSharedMem) self.NImGauss = NImGauss self.DeltaChi2 = DeltaChi2 self.ListGauss = ListGauss self.NGauss = len(ListGauss) self.GSig = GSig self.SigMin = SigMin self.Var = Var
def run(self): while not self.kill_received and self.CondContinue(): #gc.enable() try: iQueue = self.work_queue.get_nowait() #(True,2) except Exception, e: #print "Exception worker: %s"%str(e) break #print "Start %i"%iQueue Queue = NpShared.GiveArray("%sQueue_%3.3i" % (self.IdSharedMem, iQueue)) self.CurrentInvCov = NpShared.GiveArray("%sInvCov_AllFacet" % (self.IdSharedMem)) for iJob in range(Queue.shape[0]): x0, y0, FacetID = Queue[iJob] iFacet = FacetID self.CurrentFacetID = FacetID #self.CurrentCF=NpShared.GiveArray("%sConvMatrix_Facet_%4.4i"%(self.IdSharedMem,iFacet)) self.CurrentCM = NpShared.GiveArray("%sCM_Facet%4.4i" % (self.IdSharedMem, iFacet)) #self.CurrentInvCov=NpShared.GiveArray("%sInvCov_Facet%4.4i"%(self.IdSharedMem,iFacet)) # if self.CurrentInvCov is None: # invCM=ModLinAlg.invSVD(np.float64(self.CurrentCM[0,0]))/self.Var # self.CurrentInvCov=NpShared.ToShared("%sInvCov_Facet%4.4i"%(self.IdSharedMem,iFacet),invCM) iGauss = self.SmearThisComp(x0, y0) Queue[iJob, 2] = iGauss self.result_queue.put({"Success": True, "iQueue": iQueue})
def GiveDicoJonesMatrices(self): print >> log, " Getting Jones matrices from Shared Memory" DicoJonesMatrices = {} GD = self.GD SolsFile = GD["DDESolutions"]["DDSols"] if SolsFile != "": DicoJones_killMS = NpShared.SharedToDico("%sJonesFile_killMS" % self.IdSharedMem) DicoJonesMatrices["DicoJones_killMS"] = DicoJones_killMS DicoJonesMatrices["DicoJones_killMS"][ "MapJones"] = NpShared.GiveArray("%sMapJones_killMS" % self.IdSharedMem) DicoClusterDirs_killMS = NpShared.SharedToDico( "%sDicoClusterDirs_killMS" % self.IdSharedMem) DicoJonesMatrices["DicoJones_killMS"][ "DicoClusterDirs"] = DicoClusterDirs_killMS ApplyBeam = (GD["Beam"]["Model"] is not None) if ApplyBeam: DicoJones_Beam = NpShared.SharedToDico("%sJonesFile_Beam" % self.IdSharedMem) DicoJonesMatrices["DicoJones_Beam"] = DicoJones_Beam DicoJonesMatrices["DicoJones_Beam"][ "MapJones"] = NpShared.GiveArray("%sMapJones_Beam" % self.IdSharedMem) DicoClusterDirs_Beam = NpShared.SharedToDico( "%sDicoClusterDirs_Beam" % self.IdSharedMem) DicoJonesMatrices["DicoJones_Beam"][ "DicoClusterDirs"] = DicoClusterDirs_Beam return DicoJonesMatrices
def UnPackMapping(self): Map = NpShared.GiveArray("%sMappingSmearing" % (self.IdSharedMem)) Nb = Map[0] NRowInBlocks = Map[1:Nb + 1] StartRow = Map[Nb + 1:2 * Nb + 1] # print # print NRowInBlocks.tolist() # print StartRow.tolist() MaxRow = 0 for i in [3507]: # range(Nb): ii = StartRow[i] MaxRow = np.max([MaxRow, np.max(Map[ii:ii + NRowInBlocks[i]])]) print "(iblock= %i , istart= %i), Nrow=%i" % \ (i, StartRow[i], NRowInBlocks[i]), Map[ii:ii+NRowInBlocks[i]] #if MaxRow>=1080: stop print MaxRow
def Smear(self, Parallel=True): if Parallel: NCPU = self.NCPU else: NCPU = 1 StopWhenQueueEmpty = True print("Building queue", file=log) self.ModelOut = np.zeros_like(self.MeanModelImage) indx, indy = np.where(self.MeanModelImage[0, 0] != 0) #indx,indy=np.where(self.MeanModelImage==np.max(self.MeanModelImage)) work_queue = multiprocessing.Queue() result_queue = multiprocessing.Queue() SizeMax = int(indx.size / float(NCPU) / 100.) SizeMax = np.max([SizeMax, 1]) iPix = 0 iQueue = 0 Queue = [] while iPix < indx.size: xc, yc = indx[iPix], indy[iPix] FacetID = self.PSFServer.giveFacetID2(xc, yc) #DicoOrder={"xy":(xc,yc), # "FacetID":FacetID} Queue.append([xc, yc, FacetID]) iPix += 1 if (len(Queue) == SizeMax) | (iPix == indx.size): NpShared.ToShared("%sQueue_%3.3i" % (self.IdSharedMem, iQueue), np.array(Queue)) work_queue.put(iQueue) Queue = [] iQueue += 1 NJobs = indx.size workerlist = [] pBAR = ProgressBar(Title=" Find gaussian") #pBAR.disable() pBAR.render(0, '%4i/%i' % (0, NJobs)) for ii in range(NCPU): W = WorkerSmear(work_queue, result_queue, IdSharedMem=self.IdSharedMem, StopWhenQueueEmpty=StopWhenQueueEmpty, NImGauss=self.NImGauss, DeltaChi2=self.DeltaChi2, ListGauss=self.ListGauss, GSig=self.GSig, Var=self.Var, SigMin=self.SigMin) workerlist.append(W) if Parallel: workerlist[ii].start() else: workerlist[ii].run() N = self.NImGauss iResult = 0 #print "!!!!!!!!!!!!!!!!!!!!!!!!",iResult,NJobs while iResult < NJobs: DicoResult = None # for result_queue in List_Result_queue: # if result_queue.qsize()!=0: # try: # DicoResult=result_queue.get_nowait() # break # except: # pass # #DicoResult=result_queue.get() #print "!!!!!!!!!!!!!!!!!!!!!!!!! Qsize",result_queue.qsize() #print work_queue.qsize(),result_queue.qsize() if result_queue.qsize() != 0: try: DicoResult = result_queue.get_nowait() except: pass #DicoResult=result_queue.get() if DicoResult is None: time.sleep(0.001) continue if DicoResult["Success"]: iQueue = DicoResult["iQueue"] Queue = NpShared.GiveArray("%sQueue_%3.3i" % (self.IdSharedMem, iQueue)) for iJob in range(Queue.shape[0]): x0, y0, iGauss = Queue[iJob] SMax = self.MeanModelImage[0, 0, x0, y0] SubModelOut = self.ModelOut[0, 0][x0 - N // 2:x0 + N // 2 + 1, y0 - N // 2:y0 + N // 2 + 1] SubModelOut += self.ListRestoredGauss[iGauss] * SMax #iGauss=0 #print #print SMax #print np.sum(self.ListGauss[iGauss]) #print SubModelOut += self.ListGauss[iGauss] * SMax iResult += 1 NDone = iResult intPercent = int(100 * NDone / float(NJobs)) pBAR.render(intPercent, '%4i/%i' % (NDone, NJobs)) for ii in range(NCPU): try: workerlist[ii].shutdown() workerlist[ii].terminate() workerlist[ii].join() except: pass return self.ModelOut
def BuildSmearMappingParallel(self, DATA, GridChanMapping): print >> log, "Build decorrelation mapping ..." na = self.MS.na l = self.radiusRad dPhi = np.sqrt(6. * (1. - self.Decorr)) NChan = self.MS.ChanFreq.size self.BlocksRowsList = [] InfoSmearMapping = {} InfoSmearMapping["freqs"] = self.MS.ChanFreq InfoSmearMapping["dfreqs"] = self.MS.dFreq InfoSmearMapping["dPhi"] = dPhi InfoSmearMapping["l"] = l BlocksRowsList = [] joblist = [(a0, a1) for a0 in xrange(na) for a1 in xrange(na) if a0 != a1] WorkerMapName = Multiprocessing.getShmURL("SmearWorker.%d") results = Multiprocessing.runjobs( joblist, title="Smear mapping", target=_smearmapping_worker, kwargs=dict(DATA=DATA, InfoSmearMapping=InfoSmearMapping, WorkerMapName=WorkerMapName, GridChanMapping=GridChanMapping)) # process worker results # for each map (each array returned from worker), BlockSizes[MapName] will # contain a list of BlocksSizesBL entries returned from that worker RowsBlockSizes = {} NTotBlocks = 0 NTotRows = 0 worker_maps = {} for DicoResult in results: if not DicoResult["Empty"]: MapName = DicoResult["MapName"] map = worker_maps.get(MapName) if map is None: map = worker_maps[MapName] = NpShared.GiveArray(MapName) bl = DicoResult["bl"] rowslice = DicoResult["Slice"] bsz = np.array(DicoResult["BlocksSizesBL"]) RowsBlockSizes[bl] = map[rowslice], bsz NTotBlocks += DicoResult["NBlocksTotBL"] NTotRows += bsz.sum() # output mapping has 2 words for the total size, plus 2*NTotBlocks header, plus NTotRows blocklists OutputMapping = np.zeros((2 + 2 * NTotBlocks + NTotRows, ), np.int32) # just in case NTotBlocks is over 2^31... # (don't want to use np.int32 for the whole mapping as that just wastes space, we may assume # that we have substantially fewer rows, so int32 is perfectly good as a row index etc.) OutputMapping[0] = NTotBlocks OutputMapping[1] = NTotBlocks >> 32 BlockListSizes = OutputMapping[2:2 + NTotBlocks] BlockLists = OutputMapping[2 + NTotBlocks:] iii = 0 jjj = 0 # now go through each per-baseline mapping, sorted by baseline for _, (BlocksRowsListBL, BlocksSizesBL) in sorted(RowsBlockSizes.items()): #print>>log, " Worker: %i"%(IdWorker) BlockLists[iii:iii + BlocksRowsListBL.size] = BlocksRowsListBL[:] iii += BlocksRowsListBL.size # print "IdWorker,AppendId",IdWorker,AppendId,BlocksSizesBL # MM=np.concatenate((MM,BlocksSizesBL)) BlockListSizes[jjj:jjj + BlocksSizesBL.size] = BlocksSizesBL[:] jjj += BlocksSizesBL.size for MapName in worker_maps.iterkeys(): NpShared.DelArray(MapName) #print>>log, " Put in shared mem" NVis = np.where(DATA["A0"] != DATA["A1"])[0].size * NChan #print>>log, " Number of blocks: %i"%NTotBlocks #print>>log, " Number of 4-Visibilities: %i"%NVis fact = (100. * (NVis - NTotBlocks) / float(NVis)) # self.UnPackMapping() # print FinalMapping return OutputMapping, fact
def load_impl(self): return NpShared.GiveArray(_to_shm(self.path))
def CalcWeights(self, uvw_weights_flags_freqs, Robust=0, Weighting="Briggs", Super=1, nbands=1, band_mapping=None, weightnorm=1, force_unity_weight=False): """ Computes imaging weights in "MFS mode", when all uv-points are binned onto a single grid. Args: uvw_weights_flags_freqs: list of (uv, weights, flags, freqs) tuples, one per each MS if weights is a string, it is treated as the filename for a shared array Robust: robustness Weighting: natural, uniform, briggs Super: !=1 for superuniform or superrobust: uv bin size is 1/(super*FoV) nbands: number of frequency bands to compute weights on (if band_mapping is not None) band_mapping: band_mapping[iMS][ichan] gives the band number of channel #ichan of MS #iMS if None, the "MFS weighting" is used, with all frequency points weighted on a single grid weightnorm: multiply weights by this factor force_unity_weight: force all weights to 1 Returns: list of imaging weights arrays, one per MS, same shape as original data weights """ Weighting = Weighting.lower() if Weighting == "natural": print>> log, "Weighting in natural mode" if force_unity_weight: for uv, weights, flags, freqs in uvw_weights_flags_freqs: if flags is not None: if type(weights) is str: NpShared.GiveArray(weights).fill(1) else: weights.fill(1) return [x[1] for x in uvw_weights_flags_freqs] nch, npol, npixIm, _ = self.ImShape FOV = self.CellSizeRad * npixIm cell = 1. / (Super * FOV) if band_mapping is None: nbands = 1 print>> log, "initializing weighting grid for single band (or MFS weighting)" else: print>> log, "initializing weighting grids for %d bands"%nbands # find max grid extent by considering _unflagged_ UVs xymax = 0 for uv, weights, flags, freqs in uvw_weights_flags_freqs: # max |u|,|v| in lambda uvsel=abs(uv)[~flags, :] if uvsel.size==0: print>> log, ModColor.Str(" A dataset is fully flagged") continue uvmax = uvsel.max() * freqs.max() / _cc xymax = max(xymax, int(math.floor(uvmax / cell))) if flags is not None: # max |u|,|v| in lambda uvmax = abs(uv)[~flags, :].max() * freqs.max() / _cc xymax = max(xymax, int(math.floor(uvmax / cell))) if xymax == 0: raise Exception('All datasets are fully flagged') xymax += 1 # grid will be from [-xymax,xymax] in U and [0,xymax] in V npixx = xymax * 2 + 1 npixy = xymax + 1 npix = npixx * npixy print>> log, "Calculating imaging weights on an [%i,%i]x%i grid with cellsize %g" % (npixx, npixy, nbands, cell) grid0 = np.zeros((nbands, npix), np.float64) grid = grid0.reshape((nbands*npix,)) # this will ve a per-MS list of weights and an index array, or None if an MS is all flagged weights_index = [(None, None)] * len(uvw_weights_flags_freqs) for iMS, (uv, weights_or_path, flags, freqs) in enumerate(uvw_weights_flags_freqs): if flags is None: # entire chunk flagged continue weights = NpShared.GiveArray(weights_or_path) if type(weights_or_path) is str else weights_or_path if force_unity_weight: weights.fill(1) weights[flags,...]=0 elif weightnorm != 1: weights *= weightnorm # flip sign of negative v values -- we'll only grid the top half of the plane uv[uv[:, 1] < 0] *= -1 # convert u/v to lambda, and then to pixel offset uv = uv[..., np.newaxis] * freqs[np.newaxis, np.newaxis, :] / _cc uv = np.floor(uv / cell).astype(int) # u is offset, v isn't since it's the top half x = uv[:, 0, :] y = uv[:, 1, :] x += xymax # offset, since X grid starts at -xymax # convert to index array -- this gives the number of the uv-bin on the grid index = y * npixx + x # if we're in per-band weighting mode, then adjust the index to refer to each band's grid if band_mapping is not None: bandmap = band_mapping[iMS] # uv has shape nvis,nfreq; bandmap has shape nfreq index += bandmap[np.newaxis,:]*npix # zero weight refers to zero cell (otherwise it may end up outside the grid, since grid is # only big enough to accommodate the *unflagged* uv-points) index[weights==0] = 0 weights_index[iMS] = weights_or_path, index del uv print>> log, "Accumulating weights (%d/%d)" % (iMS + 1, len(uvw_weights_flags_freqs)) # accumulate onto grid # print>>log,weights,index _pyGridderSmearPols.pyAccumulateWeightsOntoGrid(grid, weights.ravel(), index.ravel()) if Weighting == "uniform": # print>>log,"adjusting grid to uniform weight" # grid[grid!=0] = 1/grid[grid!=0] print>> log, ("applying uniform weighting (super=%.2f)" % Super) for weights_or_path, index in weights_index: if index is not None: weights = NpShared.GiveArray(weights_or_path) if type(weights_or_path) is str else weights_or_path weights /= grid[index] elif Weighting == "briggs" or Weighting == "robust": numeratorSqrt = 5.0 * 10 ** (-Robust) for band in range(nbands): print>> log, ("applying Briggs weighting (robust=%.2f, super=%.2f, band %d)" % (Robust, Super, band)) grid1 = grid0[band,:] avgW = (grid1 ** 2).sum() / grid1.sum() sSq = numeratorSqrt ** 2 / avgW grid1[...] = 1 / (1 + grid1 * sSq) for weights_or_path, index in weights_index: if index is not None: weights = NpShared.GiveArray(weights_or_path) if type(weights_or_path) is str else weights_or_path weights *= grid[index] else: raise ValueError("unknown weighting \"%s\"" % Weighting) print>> log, "weights computed"
def calcDistanceMatrixMinParallel(self, ListIslands, Parallel=True): NIslands = len(ListIslands) self.D = np.zeros((NIslands, NIslands), np.float32) self.dx = np.zeros((NIslands, NIslands), np.int32) self.dy = np.zeros((NIslands, NIslands), np.int32) work_queue = multiprocessing.JoinableQueue() for iIsland in range(NIslands): work_queue.put({"iIsland": (iIsland)}) result_queue = multiprocessing.JoinableQueue() NJobs = work_queue.qsize() workerlist = [] NCPU = self.NCPU ListEdgeIslands = self.giveEdgesIslands(ListIslands) for ii in range(NCPU): W = WorkerDistance(work_queue, result_queue, ListEdgeIslands, self.IdSharedMem) workerlist.append(W) if Parallel: workerlist[ii].start() pBAR = ProgressBar(Title=" Calc. Dist. ") pBAR.render(0, NJobs) iResult = 0 if not Parallel: for ii in range(NCPU): workerlist[ii].run() # just run until all work is completed while iResult < NJobs: DicoResult = None if result_queue.qsize() != 0: try: DicoResult = result_queue.get() except: pass if DicoResult == None: time.sleep(0.5) continue if DicoResult["Success"]: iResult += 1 NDone = iResult pBAR.render(NDone, NJobs) iIsland = DicoResult["iIsland"] Result = NpShared.GiveArray("%sDistances_%6.6i" % (self.IdSharedMem, iIsland)) self.dx[iIsland] = Result[0] self.dy[iIsland] = Result[1] self.D[iIsland] = Result[2] NpShared.DelAll("%sDistances_%6.6i" % (self.IdSharedMem, iIsland)) if Parallel: for ii in range(NCPU): workerlist[ii].shutdown() workerlist[ii].terminate() workerlist[ii].join()