def EstimateBeam(self, TimesBeam, RA, DEC,progressBar=True, quiet=False): TimesBeam = np.float64(np.array(TimesBeam)) T0s = TimesBeam[:-1].copy() T1s = TimesBeam[1:].copy() Tm = (T0s+T1s)/2. # RA,DEC=self.BeamRAs,self.BeamDECs NDir=RA.size DicoBeam={} FreqDomains=self.BeamMachine.getFreqDomains() DicoBeam["VisToJonesChanMapping"]=self.GiveVisToJonesChanMapping(FreqDomains) if not quiet: print>>log,"VisToJonesChanMapping: %s"%DicoBeam["VisToJonesChanMapping"] DicoBeam["Jones"]=np.zeros((Tm.size,NDir,self.MS.na,FreqDomains.shape[0],2,2),dtype=np.complex64) DicoBeam["t0"]=np.zeros((Tm.size,),np.float64) DicoBeam["t1"]=np.zeros((Tm.size,),np.float64) DicoBeam["tm"]=np.zeros((Tm.size,),np.float64) rac,decc=self.MS.OriginalRadec pBAR= ProgressBar(Title=" Init E-Jones ")#, HeaderSize=10,TitleSize=13) if not progressBar: pBAR.disable() pBAR.render(0, Tm.size) for itime in range(Tm.size): DicoBeam["t0"][itime]=T0s[itime] DicoBeam["t1"][itime]=T1s[itime] DicoBeam["tm"][itime]=Tm[itime] ThisTime=Tm[itime] Beam=self.GiveInstrumentBeam(ThisTime,RA,DEC) # if self.GD["Beam"]["CenterNorm"]==1: Beam0=self.GiveInstrumentBeam(ThisTime,np.array([rac]),np.array([decc])) Beam0inv= ModLinAlg.BatchInverse(Beam0) nd,_,_,_,_=Beam.shape Ones=np.ones((nd, 1, 1, 1, 1),np.float32) Beam0inv=Beam0inv*Ones Beam= ModLinAlg.BatchDot(Beam0inv, Beam) DicoBeam["Jones"][itime]=Beam NDone=itime+1 pBAR.render(NDone,Tm.size) DicoBeam["Jones"][itime] = Beam nt, nd, na, nch, _, _ = DicoBeam["Jones"].shape # DicoBeam["Jones"]=np.mean(DicoBeam["Jones"],axis=3).reshape((nt,nd,na,1,2,2)) # print TimesBeam-TimesBeam[0] # print t0-t1 # print DicoBeam["t1"][-1]-DicoBeam["t0"][0] return DicoBeam
def CalcCrossIslandFlux(self, ListIslands): if self.PSFCross is None: self.CalcCrossIslandPSF(ListIslands) NIslands = len(ListIslands) print >> log, " grouping cross contaminating islands..." MaxIslandFlux = np.zeros((NIslands, ), np.float32) DicoIsland = {} Dirty = self.DicoDirty["MeanImage"] for iIsland in range(NIslands): x0, y0 = np.array(ListIslands[iIsland]).T PixVals0 = Dirty[0, 0, x0, y0] MaxIslandFlux[iIsland] = np.max(PixVals0) DicoIsland[iIsland] = ListIslands[iIsland] self.CrossFluxContrib = self.PSFCross * MaxIslandFlux.reshape( (1, NIslands)) self.DicoIsland = DicoIsland NDone = 0 NJobs = NIslands pBAR = ProgressBar(Title=" Group islands") pBAR.disable() pBAR.render(0, NJobs) Th = 0.05 ListIslandMerged = [] self.setCheckedIslands = set([]) for iIsland in range(NIslands): x0, y0 = np.array(ListIslands[iIsland]).T #print "Main %i (%f, %f)"%(iIsland,np.mean(x0),np.mean(y0)) NDone += 1 intPercent = int(100 * NDone / float(NJobs)) pBAR.render(NDone, NJobs) ListIslandMerged.append( list(self.GiveNearbyIsland(iIsland, set([])))) ListIslands = [] for indIsland in ListIslandMerged: if len(indIsland) == 0: continue ThisIsland = DicoIsland[indIsland[0]] for iIsland in indIsland[1::]: ThisIsland += DicoIsland[iIsland] ListIslands.append(ThisIsland) print >> log, " have grouped %i --> %i islands" % (NIslands, len(ListIslands)) return ListIslands
def calcDistanceMatrixMin(self, ListIslands): NIslands = len(ListIslands) self.D = np.zeros((NIslands, NIslands), np.float32) self.dx = np.zeros((NIslands, NIslands), np.int32) self.dy = np.zeros((NIslands, NIslands), np.int32) pBAR = ProgressBar(Title=" Calc Dist") #pBAR.disable() NDone = 0 NJobs = NIslands pBAR.render(0, NJobs) for iIsland in range(NIslands): x0, y0 = np.array(ListIslands[iIsland]).T for jIsland in range(iIsland + 1, NIslands): x1, y1 = np.array(ListIslands[jIsland]).T dx = x0.reshape((-1, 1)) - x1.reshape((1, -1)) dy = y0.reshape((-1, 1)) - y1.reshape((1, -1)) d = np.sqrt(dx**2 + dy**2) dmin = np.min(d) self.D[jIsland, iIsland] = self.D[iIsland, jIsland] = dmin indx, indy = np.where(d == dmin) #print dx[indx[0],indy[0]],dy[indx[0],indy[0]],dmin self.dx[jIsland, iIsland] = self.dx[iIsland, jIsland] = dx[indx[0], indy[0]] self.dy[jIsland, iIsland] = self.dy[iIsland, jIsland] = dy[indx[0], indy[0]] NDone += 1 pBAR.render(NDone, NJobs)
def AddUVW_dt(self): log.print("Adding uvw speed info to main table: %s" % self.MSName) log.print("Compute UVW speed column") MSName = self.MSName MS = self t = table(MSName, readonly=False, ack=False) times = t.getcol("TIME") A0 = t.getcol("ANTENNA1") A1 = t.getcol("ANTENNA2") UVW = t.getcol("UVW") UVW_dt = np.zeros_like(UVW) if "UVWDT" not in t.colnames(): log.print("Adding column UVWDT in %s" % self.MSName) desc = t.getcoldesc("UVW") desc["name"] = "UVWDT" desc['comment'] = desc['comment'].replace(" ", "_") t.addcols(desc) # # ####################### # LTimes=np.sort(np.unique(times)) # for iTime,ThisTime in enumerate(LTimes): # print(iTime,LTimes.size) # ind=np.where(times==ThisTime)[0] # UVW_dt[ind]=MS.Give_dUVW_dt(times[ind],A0[ind],A1[ind]) # # ####################### na = MS.na pBAR = ProgressBar(Title=" Calc dUVW/dt ") pBAR.render(0, na) for ant0 in range(na): for ant1 in range(ant0, MS.na): if ant0 == ant1: continue C0 = ((A0 == ant0) & (A1 == ant1)) C1 = ((A1 == ant0) & (A0 == ant1)) ind = np.where(C0 | C1)[0] if len(ind) == 0: continue # e.g. if antenna missing UVWs = UVW[ind] timess = times[ind] dtimess = timess[1::] - timess[0:-1] UVWs_dt0 = (UVWs[1::] - UVWs[0:-1]) / dtimess.reshape((-1, 1)) UVW_dt[ind[0:-1]] = UVWs_dt0 UVW_dt[ind[-1]] = UVWs_dt0[-1] intPercent = int(100 * (ant0 + 1) / float(na)) pBAR.render(ant0 + 1, na) log.print("Writing in column UVWDT") t.putcol("UVWDT", UVW_dt) t.close()
def awaitJobCounter(self, counter, progress=None, total=None, timeout=10): if self.verbose > 2: print >> log, " %s is complete" % counter.name if progress: current = counter.getValue() total = total or current or 1 pBAR = ProgressBar(Title=" " + progress) #pBAR.disable() pBAR.render(total - current, total) while current: current = counter.awaitZeroWithTimeout(timeout) pBAR.render(total - current, total) if self._termination_event.is_set(): if self.verbose > 1: print >> log, " termination event spotted, exiting" raise WorkerProcessError() else: counter.awaitZero() if self._termination_event.is_set(): if self.verbose > 1: print >> log, " termination event spotted, exiting" raise WorkerProcessError() if self.verbose > 2: print >> log, " %s is complete" % counter.name
class DataDictionaryManager(object): """ Given a dictionary of MS arrays, infers the number of: - timesteps - baselines - antenna - channels ASSUMPTIONS: 1. We're dealing with a single band 2. We're dealing with a single field 3. We're dealing with a single observation Arguments: MS: DDFacet.Data.ClassMS instance pointing_sols: DDFacet.Data.PointingProvider associated to MS solver_polarization_type: initiate montblanc for linear or circular feeds """ def __init__(self, MS, pointing_sols): self._data_feed_labels = ClassStokes(MS.CorrelationIds, ["I"]).AvailableCorrelationProducts() # Extract the required prediction feeds from the MS if set(self._data_feed_labels) <= set(["XX", "XY", "YX", "YY"]): self._montblanc_feed_labels = ["XX", "XY", "YX", "YY"] self._solver_polarization_type = "linear" elif set(self._data_feed_labels) <= set(["RR", "RL", "LR", "LL"]): self._montblanc_feed_labels = ["RR", "RL", "LR", "LL"] self._solver_polarization_type = "circular" else: raise RuntimeError("Montblanc only supports linear or circular feed measurements.") # MS correlation to montblanc correlation map self._predict_feed_map = [self._montblanc_feed_labels.index(x) for x in self._data_feed_labels] montblanc.log.info("Montblanc to MS feed mapping: %s" % ",".join([str(x) for x in self._predict_feed_map])) # Extract the antenna positions and # phase direction from the measurement set self._station_names = MS.StationNames self._antenna_positions = MS.StationPos self._phase_dir = np.array((MS.rarad, MS.decrad)) montblanc.log.info("Phase centre of {pc}".format(pc=np.rad2deg(self._phase_dir))) # RIME frequencies from ::SPECTRAL_WINDOW self._frequency = MS.ChanFreq self._ref_frequency = MS.reffreq # this montblanc instance is associated to the meta data of a single MS (or selection) if not isinstance(MS, ClassMS): raise TypeError("MS argument must be of type ClassMS") self._MS = MS # set preinstantiated pointing solutions provider if not isinstance(pointing_sols, PointingProvider): raise TypeError("pointing_sols argument must be of type PointingProvider") self._pointing_solutions = pointing_sols def _transform(self, c): """ Computes the lm coordinates from pixel coordinates assuming a SIN projection in the WCS """ coord = np.deg2rad(self._wcs.wcs_pix2world(np.asarray([c]), 1)) delta_ang = coord[0] - self._phase_dir if DEBUG: montblanc.log.debug("WCS src coordinates [deg] '{c}'".format(c=np.rad2deg(coord))) # assume SIN Projection l = np.cos(coord[0, 1])*np.sin(delta_ang[0]) m = np.sin(coord[0, 1])*np.cos(self._phase_dir[1]) - \ np.cos(coord[0, 1])*np.sin(self._phase_dir[1])*np.cos(delta_ang[0]) return (l, m) def _radec(self, c): """ Computes the radec image plane coordinates from pixel coordinates assuming a SIN projection in the WCS """ coord = np.deg2rad(self._wcs.wcs_pix2world(np.asarray([c]), 1)) if DEBUG: montblanc.log.debug("WCS src coordinates [deg] '{c}'".format(c=np.rad2deg(coord))) return (coord[0, 0], coord[0, 1]) @classmethod def _synthesize_uvw(cls, station_ECEF, time, a1, a2, phase_ref): """ Synthesizes new UVW coordinates based on time according to NRAO CASA convention (same as in fixvis) User should check these UVW coordinates carefully - if time centroid was used to compute original uvw coordinates the centroids of these new coordinates may be wrong, depending on whether data timesteps were heavily flagged. station_ECEF: ITRF station coordinates read from MS::ANTENNA time: time column, preferably time centroid (padded to nrow = unique time * unique bl) a1: ANTENNA_1 index (padded to nrow = unique time * unique bl) a2: ANTENNA_2 index (padded to nrow = unique time * unique bl) phase_ref: phase reference centre in radians """ assert time.size == a1.size assert a1.size == a2.size dm = measures() epoch = dm.epoch("UT1", quantity(time[0], "s")) refdir = dm.direction("j2000", quantity(phase_ref[0], "rad"), quantity(phase_ref[1], "rad")) obs = dm.position("ITRF", quantity(station_ECEF[0, 0], "m"), quantity(station_ECEF[0, 1], "m"), quantity(station_ECEF[0, 2], "m")) #setup local horizon coordinate frame with antenna 0 as reference position dm.do_frame(obs) dm.do_frame(refdir) dm.do_frame(epoch) ants = np.concatenate((a1, a2)) unique_ants = np.unique(ants) unique_time = np.unique(time) na = unique_ants.size nbl = na * (na - 1) // 2 + na ntime = unique_time.size assert time.size == nbl * ntime, "Input arrays must be padded to include autocorrelations, all baselines and all time" antenna_indicies = DataDictionaryManager.antenna_indicies(na, auto_correlations=True) new_uvw = np.zeros((ntime*nbl, 3)) for ti, t in enumerate(unique_time): epoch = dm.epoch("UT1", quantity(t, "s")) dm.do_frame(epoch) station_uv = np.zeros_like(station_ECEF) for iapos, apos in enumerate(station_ECEF): station_uv[iapos] = dm.to_uvw(dm.baseline("ITRF", quantity([apos[0], station_ECEF[0, 0]], "m"), quantity([apos[1], station_ECEF[0, 1]], "m"), quantity([apos[2], station_ECEF[0, 2]], "m")))["xyz"].get_value()[0:3] for bl in range(nbl): blants = antenna_indicies[bl] bla1 = blants[0] bla2 = blants[1] new_uvw[ti*nbl + bl, :] = station_uv[bla1] - station_uv[bla2] # same as in CASA convention (Convention for UVW calculations in CASA, Rau 2013) return new_uvw def cfg_from_data_dict(self, data, models, residuals, npix, cell_size_rad): time, uvw, antenna1, antenna2, flag = (data[i] for i in ('times', 'uvw', 'A0', 'A1', 'flags')) self._npix = npix self._cell_size_rad = cell_size_rad # Get point and gaussian sources self._point_sources = [] self._gaussian_sources = [] for model in models: if model[0] == 'Delta': self._point_sources.append(model) elif model[0] == 'Gaussian': self._gaussian_sources.append(model) else: raise ValueError("Montblanc does not predict " "'{mt}' model types.".format(mt=model[0])) self._npsrc = npsrc = len(self._point_sources) self._ngsrc = ngsrc = len(self._gaussian_sources) montblanc.log.info("{n} point sources".format(n=npsrc)) montblanc.log.info("{n} gaussian sources".format(n=ngsrc)) self._nchan = nchan = self._frequency.size # Merge antenna ID's and count their frequencies ants = np.concatenate((antenna1, antenna2)) unique_ants = np.unique(ants) ant_counts = np.bincount(ants) self._na = na = unique_ants.size assert self._na <= self._antenna_positions.shape[0], "ANTENNA_1 and ANTENNA_2 contains more indicies than antennae specified through MS.StationPos" self._na = na = self._antenna_positions.shape[0] # going to pad to the maximum number of antennas # can compute the time indexes using a scan operator # assuming the dataset is ordered and the time column # contains the integration centroid of the correlator dumps # note: time first, then antenna1 slow varying then antenna2 # fast varying self._sort_index = np.lexsort(np.array((time, antenna1, antenna2))[::-1]) tfilter = np.zeros(time[self._sort_index].shape, dtype=np.bool) tfilter[1:] = time[self._sort_index][1:] != time[self._sort_index][:-1] self._tindx = np.cumsum(tfilter) self._ntime = self._tindx[-1] + 1 # include the autocorrelations to safely pad the array to a maximum possible size self._nbl = self._na * (self._na - 1) // 2 + self._na self._blindx = self.baseline_index(antenna1[self._sort_index], antenna2[self._sort_index], self._na) # Sanity check antenna and row dimensions if self._antenna_positions.shape[0] < na: raise ValueError("Number of antenna positions {aps} " "is less than the number of antenna '{na}' " "found in antenna1/antenna2".format( aps=self._antenna_positions.shape, na=na)) self._nrow = self._nbl * self._ntime if self._nrow < uvw.shape[0]: raise ValueError("Number of rows in input chunk exceeds " "nbl * ntime. Please ensure that only one field, " "one observation and one data descriptor is present in the chunk you're " "trying to predict.") # construct a mask to indicate values sparse matrix self._datamask = np.zeros((self._ntime, self._nbl), dtype=np.bool) self._datamask[self._tindx, self._blindx] = True self._datamask = self._datamask.reshape((self._nrow)) print("Padding data matrix for missing baselines (%.2f %% data missing from MS)" % \ (100.0 - 100.0 * float(np.sum(self._datamask)) / self._datamask.size), file=log) # padded row numbers of the sparce data matrix self._sparseindx = np.cumsum(self._datamask) - 1 # pad the antenna array self._padded_a1 = np.empty((self._nbl), dtype=np.int32) self._padded_a2 = np.empty((self._nbl), dtype=np.int32) antenna_indicies = DataDictionaryManager.antenna_indicies(na, auto_correlations=True) for bl in range(self._nbl): blants = antenna_indicies[bl] self._padded_a1[bl] = blants[0] self._padded_a2[bl] = blants[1] self._padded_a1 = np.tile(self._padded_a1, self._ntime) self._padded_a2 = np.tile(self._padded_a2, self._ntime) assert np.all(self._padded_a1[self._datamask] == antenna1[self._sort_index]) assert np.all(self._padded_a2[self._datamask] == antenna2[self._sort_index]) # pad the time array self._unique_time = np.unique(time) # sorted unique times self._padded_time = self._unique_time.repeat(self._nbl) assert np.all(self._padded_time[self._datamask] == time[self._sort_index]) # Pad the uvw array to contain nbl * ntime entries (including the autocorrs) # with zeros where baselines may be missing self._residuals = residuals.view() self._padded_uvw = np.zeros((self._ntime, self._nbl, 3), dtype=uvw.dtype) self._padded_uvw[self._tindx, self._blindx, :] = uvw[self._sort_index, :] self._padded_uvw = self._padded_uvw.reshape((self._nrow, 3)) assert np.all(self._padded_uvw[self._datamask] == uvw[self._sort_index]) # CASA split may have removed completely flagged baselines so resynthesize # uv coordinates as best as possible print("Synthesizing new UVW coordinates from TIME column to fill gaps in measurement set", file=log) self._synth_uvw = DataDictionaryManager._synthesize_uvw(self._antenna_positions, self._padded_time, self._padded_a1, self._padded_a2, self._phase_dir) # for safety only fill missing slots with new UVW coordinates self._padded_uvw[np.logical_not(self._datamask)] = self._synth_uvw[np.logical_not(self._datamask)] q1 = np.percentile(np.abs(self._synth_uvw[self._datamask] - uvw[self._sort_index]), 1.0) q2 = np.percentile(np.abs(self._synth_uvw[self._datamask] - uvw[self._sort_index]), 50.0) q3 = np.percentile(np.abs(self._synth_uvw[self._datamask] - uvw[self._sort_index]), 99.0) print(ModColor.Str("WARNING: The 99th percentile error on newly synthesized UVW coordinates may be as large as %.5f" % q3), file=log) self._flag = flag # progress... self._numtotal = None self._numcomplete = 0 self._pBAR = ProgressBar(Title=" montblanc predict") self.render_progressbar() # Initialize WCS frame wcs = pywcs.WCS(naxis=2) # note half a pixel will correspond to even sized image projection poles montblanc.log.debug("NPIX: %d" % self._npix) assert self._npix % 2 == 1, "Currently only supports odd-sized maps" l0m0 = [self._npix // 2, self._npix // 2] wcs.wcs.crpix = l0m0 # remember that the WCS frame uses degrees wcs.wcs.cdelt = [np.rad2deg(self._cell_size_rad), np.rad2deg(self._cell_size_rad)] # assume SIN image projection wcs.wcs.ctype = ["RA---SIN","DEC--SIN"] wcs.wcs.crval = [np.rad2deg(self._phase_dir[0]), np.rad2deg(self._phase_dir[1])] self._wcs = wcs # Finally output some info montblanc.log.info('Chunk contains:' % unique_ants) montblanc.log.info('\tntime: %s' % self._ntime) montblanc.log.info('\tnbl: %s' % self._nbl) montblanc.log.info('\tnchan: %s' % self._nchan) montblanc.log.info('\tna: %s' % self._na) @classmethod def antenna_indicies(cls, na, auto_correlations=True): """ Compute base antenna pairs from baseline index """ k = 0 if auto_correlations == True else 1 ant1, ant2 = np.triu_indices(na, k) return np.stack([ant1, ant2], axis=1) @classmethod def baseline_index(cls, a1, a2, no_antennae): """ Computes unique index of a baseline given antenna 1 and antenna 2 (zero indexed) as input. The arrays may or may not contain auto-correlations. There is a quadratic series expression relating a1 and a2 to a unique baseline index(can be found by the double difference method) Let slow_varying_index be S = min(a1, a2). The goal is to find the number of fast varying terms. As the slow varying terms increase these get fewer and fewer, because we only consider unique baselines and not the conjugate baselines) B = (-S ^ 2 + 2 * S * # Ant + S) / 2 + diff between the slowest and fastest varying antenna :param a1: array of ANTENNA_1 ids :param a2: array of ANTENNA_2 ids :param no_antennae: number of antennae in the array :return: array of baseline ids """ if a1.shape != a2.shape: raise ValueError("a1 and a2 must have the same shape!") slow_index = np.min(np.array([a1, a2]), axis=0) return (slow_index * (-slow_index + (2 * no_antennae + 1))) // 2 + \ np.abs(a1 - a2) def updated_dimensions(self): return [('ntime', self._ntime), ('nbl', self._nbl), ('na', self._na), ('nbands', 1), ('nchan', self._nchan), ('npsrc', self._npsrc), ('ngsrc', self._ngsrc)] def render_progressbar(self): if self._numtotal is not None: self._pBAR.render(self._numcomplete, self._numtotal)
def __init__(self, MeanResidual, MeanModelImage, PSFServer, DeltaChi2=4., IdSharedMem="", NCPU=6): IdSharedMem += "SmearSM." NpShared.DelAll(IdSharedMem) self.IdSharedMem = IdSharedMem self.NCPU = NCPU self.MeanModelImage = NpShared.ToShared( "%sMeanModelImage" % self.IdSharedMem, MeanModelImage) self.MeanResidual = NpShared.ToShared( "%sMeanResidual" % self.IdSharedMem, MeanResidual) NPixStats = 10000 RandomInd = np.int64(np.random.rand(NPixStats) * (MeanResidual.size)) self.RMS = np.std(np.real(self.MeanResidual.ravel()[RandomInd])) self.FWHMMin = 3. self.PSFServer = PSFServer self.DeltaChi2 = DeltaChi2 self.Var = self.RMS**2 self.NImGauss = 31 self.CubeMeanVariablePSF = NpShared.ToShared( "%sCubeMeanVariablePSF" % self.IdSharedMem, self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF']) self.DicoConvMachine = {} N = self.NImGauss dx, dy = np.mgrid[-(N // 2):N // 2:1j * N, -(N // 2):N // 2:1j * N] ListPixParms = [(int(dx.ravel()[i]), int(dy.ravel()[i])) for i in range(dx.size)] ListPixData = ListPixParms ConvMode = "Matrix" N = self.NImGauss #stop #for #ClassConvMachine(): #def __init__(self,PSF,ListPixParms,ListPixData,ConvMode): d = np.sqrt(dx**2 + dy**2) self.dist = d self.NGauss = 10 GSig = np.linspace(0., 2, self.NGauss) self.GSig = GSig ListGauss = [] One = np.zeros_like(d) One[N // 2, N // 2] = 1. ListGauss.append(One) for sig in GSig[1::]: v = np.exp(-d**2 / (2. * sig**2)) Sv = np.sum(v) v /= Sv ListGauss.append(v) self.ListGauss = ListGauss print("Declare convolution machines", file=log) NJobs = self.PSFServer.NFacets pBAR = ProgressBar(Title=" Declare ") #pBAR.disable() pBAR.render(0, '%4i/%i' % (0, NJobs)) for iFacet in range(self.PSFServer.NFacets): #print iFacet,"/",self.PSFServer.NFacets PSF = self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF'][ iFacet] #[0,0] _, _, NPixPSF, _ = PSF.shape PSF = PSF[:, :, NPixPSF // 2 - N:NPixPSF // 2 + N + 1, NPixPSF // 2 - N:NPixPSF // 2 + N + 1] #print PSF.shape #sig=1 #PSF=(np.exp(-self.dist**2/(2.*sig**2))).reshape(1,1,N,N) self.DicoConvMachine[iFacet] = ClassConvMachine.ClassConvMachine( PSF, ListPixParms, ListPixData, ConvMode) CM = self.DicoConvMachine[iFacet].CM NpShared.ToShared("%sCM_Facet%4.4i" % (self.IdSharedMem, iFacet), CM) #invCM=ModLinAlg.invSVD(np.float64(CM[0,0]))/self.Var #NpShared.ToShared("%sInvCov_Facet%4.4i"%(self.IdSharedMem,iFacet),invCM) NDone = iFacet + 1 intPercent = int(100 * NDone / float(NJobs)) pBAR.render(intPercent, '%4i/%i' % (NDone, NJobs)) PSFMean = np.mean( self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF'], axis=0) self.ConvMachineMeanPSF = ClassConvMachine.ClassConvMachine( PSFMean, ListPixParms, ListPixData, ConvMode) CM = self.ConvMachineMeanPSF.CM invCM = ModLinAlg.invSVD(np.float64(CM[0, 0]), Cut=1e-8) / self.Var NpShared.ToShared("%sInvCov_AllFacet" % (self.IdSharedMem), invCM) self.FindSupport()
def Smear(self, Parallel=True): if Parallel: NCPU = self.NCPU else: NCPU = 1 StopWhenQueueEmpty = True print("Building queue", file=log) self.ModelOut = np.zeros_like(self.MeanModelImage) indx, indy = np.where(self.MeanModelImage[0, 0] != 0) #indx,indy=np.where(self.MeanModelImage==np.max(self.MeanModelImage)) work_queue = multiprocessing.Queue() result_queue = multiprocessing.Queue() SizeMax = int(indx.size / float(NCPU) / 100.) SizeMax = np.max([SizeMax, 1]) iPix = 0 iQueue = 0 Queue = [] while iPix < indx.size: xc, yc = indx[iPix], indy[iPix] FacetID = self.PSFServer.giveFacetID2(xc, yc) #DicoOrder={"xy":(xc,yc), # "FacetID":FacetID} Queue.append([xc, yc, FacetID]) iPix += 1 if (len(Queue) == SizeMax) | (iPix == indx.size): NpShared.ToShared("%sQueue_%3.3i" % (self.IdSharedMem, iQueue), np.array(Queue)) work_queue.put(iQueue) Queue = [] iQueue += 1 NJobs = indx.size workerlist = [] pBAR = ProgressBar(Title=" Find gaussian") #pBAR.disable() pBAR.render(0, '%4i/%i' % (0, NJobs)) for ii in range(NCPU): W = WorkerSmear(work_queue, result_queue, IdSharedMem=self.IdSharedMem, StopWhenQueueEmpty=StopWhenQueueEmpty, NImGauss=self.NImGauss, DeltaChi2=self.DeltaChi2, ListGauss=self.ListGauss, GSig=self.GSig, Var=self.Var, SigMin=self.SigMin) workerlist.append(W) if Parallel: workerlist[ii].start() else: workerlist[ii].run() N = self.NImGauss iResult = 0 #print "!!!!!!!!!!!!!!!!!!!!!!!!",iResult,NJobs while iResult < NJobs: DicoResult = None # for result_queue in List_Result_queue: # if result_queue.qsize()!=0: # try: # DicoResult=result_queue.get_nowait() # break # except: # pass # #DicoResult=result_queue.get() #print "!!!!!!!!!!!!!!!!!!!!!!!!! Qsize",result_queue.qsize() #print work_queue.qsize(),result_queue.qsize() if result_queue.qsize() != 0: try: DicoResult = result_queue.get_nowait() except: pass #DicoResult=result_queue.get() if DicoResult is None: time.sleep(0.001) continue if DicoResult["Success"]: iQueue = DicoResult["iQueue"] Queue = NpShared.GiveArray("%sQueue_%3.3i" % (self.IdSharedMem, iQueue)) for iJob in range(Queue.shape[0]): x0, y0, iGauss = Queue[iJob] SMax = self.MeanModelImage[0, 0, x0, y0] SubModelOut = self.ModelOut[0, 0][x0 - N // 2:x0 + N // 2 + 1, y0 - N // 2:y0 + N // 2 + 1] SubModelOut += self.ListRestoredGauss[iGauss] * SMax #iGauss=0 #print #print SMax #print np.sum(self.ListGauss[iGauss]) #print SubModelOut += self.ListGauss[iGauss] * SMax iResult += 1 NDone = iResult intPercent = int(100 * NDone / float(NJobs)) pBAR.render(intPercent, '%4i/%i' % (NDone, NJobs)) for ii in range(NCPU): try: workerlist[ii].shutdown() workerlist[ii].terminate() workerlist[ii].join() except: pass return self.ModelOut
def awaitJobResults(self, jobspecs, progress=None, timing=None): """ Waits for job(s) given by arguments to complete, and returns their results. Note that this only works for jobs scheduled by the same process, since each process has its own results map. A process will block indefinitely is asked to await on jobs scheduled by another process. Args: jobspec: a job spec, or a list of job specs. Each spec can contain a wildcard e.g. "job*", to wait for multiple jobs. progress: if True, a progress bar with that title will be rendered timing: if True, a timing report with that title will be printed (note that progress implies timing) Returns: a list of results. Each entry is the result returned by the job (if no wildcard), or a list of results from each job (if has a wildcard) """ if os.getpid() != parent_pid: raise RuntimeError( "This method can only be called in the parent process. This is a bug." ) if type(jobspecs) is str: jobspecs = [jobspecs] # make a dict of all jobs still outstanding awaiting_jobs = { } # this maps job_id to a set of jobspecs (if multiple) that it matches job_results = OrderedDict() # this maps jobspec to a list of results total_jobs = complete_jobs = 0 for jobspec in jobspecs: matching_jobs = [ job_id for job_id in self._results_map.iterkeys() if fnmatch.fnmatch(job_id, jobspec) ] for job_id in matching_jobs: awaiting_jobs.setdefault(job_id, set()).add(jobspec) if not matching_jobs: raise RuntimeError( "no pending jobs matching '%s'. This is probably a bug." % jobspec) total_jobs += len(matching_jobs) job_results[jobspec] = len(matching_jobs), [] # check dict of already returned results (perhaps from previous calls to awaitJobs). Remove # matching results, and assign them to appropriate jobspec lists for job_id, job in self._results_map.items(): if job_id in awaiting_jobs and job.complete: for jobspec in awaiting_jobs[job_id]: job_results[jobspec][1].append(job.result) complete_jobs += 1 if not job.singleton: del self._results_map[job_id] del awaiting_jobs[job_id] if progress: pBAR = ProgressBar(Title=" " + progress) pBAR.render(complete_jobs, (total_jobs or 1)) if self.verbose > 1: print >> log, "checking job results: %s (%d still pending)" % ( ", ".join([ "%s %d/%d" % (jobspec, len(results), njobs) for jobspec, (njobs, results) in job_results.iteritems() ]), len(awaiting_jobs)) # sit here while any pending jobs remain while awaiting_jobs and not self._termination_event.is_set(): try: result = self._result_queue.get(True, 10) except Queue.Empty: # print>> log, "checking for dead workers" # shoot the zombie process, if any multiprocessing.active_children() continue # ok, dispatch the result job_id = result["job_id"] job = self._results_map.get(job_id) if job is None: raise KeyError( "Job '%s' was not enqueued. This is a logic error." % job_id) job.setResult(result) # if being awaited, dispatch appropriately if job_id in awaiting_jobs: for jobspec in awaiting_jobs[job_id]: job_results[jobspec][1].append(result) complete_jobs += 1 if not job.singleton: del self._results_map[job_id] del awaiting_jobs[job_id] if progress: pBAR.render(complete_jobs, (total_jobs or 1)) # print status update if self.verbose > 1: print >> log, "received job results %s" % " ".join([ "%s:%d" % (jobspec, len(results)) for jobspec, (_, results) in job_results.iteritems() ]) # render complete if progress: pBAR.render(complete_jobs, (total_jobs or 1)) if self._termination_event.is_set(): if self.verbose > 1: print >> log, " termination event spotted, exiting" raise WorkerProcessError() # process list of results for each jobspec to check for errors for jobspec, (njobs, results) in job_results.iteritems(): times = np.array([res['time'] for res in results]) num_errors = len([res for res in results if not res['success']]) if timing or progress: print >> log, "%s: %d jobs complete, average single-core time %.2fs per job" % ( timing or progress, len(results), times.mean()) elif self.verbose > 0: print >> log, "%s: %d jobs complete, average single-core time %.2fs per job" % ( jobspec, len(results), times.mean()) if num_errors: print >> log, ModColor.Str( "%s: %d jobs returned an error. Aborting." % (jobspec, num_errors), col="red") raise RuntimeError("some distributed jobs have failed") # return list of results result_values = [] for jobspec, (_, results) in job_results.iteritems(): resvals = [ resitem["result"] if resitem["success"] else resitem["error"] for resitem in results ] if '*' not in jobspec: resvals = resvals[0] result_values.append(resvals) return result_values[0] if len(result_values) == 1 else result_values
def giveDicoInitIndiv(self, ListIslands, ModelImage, DicoDirty, ListDoIsland=None, Parallel=True): NCPU = self.NCPU work_queue = multiprocessing.JoinableQueue() ListIslands = ListIslands #[300:308] DoIsland = True for iIsland in range(len(ListIslands)): if ListDoIsland is not None: DoIsland = ListDoIsland[iIsland] if DoIsland: work_queue.put({"iIsland": iIsland}) result_queue = multiprocessing.JoinableQueue() NJobs = work_queue.qsize() workerlist = [] logger.setSilent(SilentModules) #MyLogger.setLoud(SilentModules) #MyLogger.setLoud("ClassImageDeconvMachineMSMF") print >> log, "Launch MORESANE workers" for ii in range(NCPU): W = WorkerInitMSMF(work_queue, result_queue, self.GD, self.DicoVariablePSF, DicoDirty, self.RefFreq, self.GridFreqs, self.DegridFreqs, self.MainCache, ModelImage, ListIslands, self.IdSharedMem) workerlist.append(W) if Parallel: workerlist[ii].start() timer = ClassTimeIt.ClassTimeIt() pBAR = ProgressBar(Title=" MORESANing islands ") #pBAR.disable() pBAR.render(0, NJobs) iResult = 0 if not Parallel: for ii in range(NCPU): workerlist[ii].run() # just run until all work is completed self.DicoInitIndiv = {} while iResult < NJobs: DicoResult = None if result_queue.qsize() != 0: try: DicoResult = result_queue.get() except: pass if DicoResult == None: time.sleep(0.5) continue if DicoResult["Success"]: iResult += 1 NDone = iResult pBAR.render(NDone, NJobs) iIsland = DicoResult["iIsland"] NameDico = "%sDicoInitIsland_%5.5i" % (self.IdSharedMem, iIsland) Dico = NpShared.SharedToDico(NameDico) self.DicoInitIndiv[iIsland] = copy.deepcopy(Dico) NpShared.DelAll(NameDico) if Parallel: for ii in range(NCPU): workerlist[ii].shutdown() workerlist[ii].terminate() workerlist[ii].join() #MyLogger.setLoud(["pymoresane.main"]) #MyLogger.setLoud(["ClassImageDeconvMachineMSMF","ClassPSFServer","ClassMultiScaleMachine","GiveModelMachine","ClassModelMachineMSMF"]) return self.DicoInitIndiv
def giveDicoInitIndiv(self, ListIslands, ModelImage, DicoDirty, ListDoIsland=None): DicoInitIndiv = shared_dict.create("DicoInitIsland") ParmDict = shared_dict.create("InitSSDModelHMP") ParmDict["ModelImage"] = ModelImage ParmDict["GridFreqs"] = self.GridFreqs ParmDict["DegridFreqs"] = self.DegridFreqs # ListBigIslands=[] # ListSmallIslands=[] # ListDoBigIsland=[] # ListDoSmallIsland=[] # NParallel=0 # for iIsland,Island in enumerate(ListIslands): # if len(Island)>self.GD["SSDClean"]["ConvFFTSwitch"]: # ListBigIslands.append(Island) # ListDoBigIsland.append(ListDoIsland[iIsland]) # if ListDoIsland or ListDoIsland[iIsland]: # NParallel+=1 # else: # ListSmallIslands.append(Island) # ListDoSmallIsland.append(ListDoIsland[iIsland]) # print>>log,"Initialise big islands (parallelised per island)" # pBAR= ProgressBar(Title="Init islands") # pBAR.render(0, NParallel) # nDone=0 # for iIsland,Island in enumerate(ListBigIslands): # if not ListDoIsland or ListDoBigIsland[iIsland]: # subdict = DicoInitIndiv.addSubdict(iIsland) # # APP.runJob("InitIsland:%d" % iIsland, self._initIsland_worker, # # args=(subdict.writeonly(), iIsland, Island, # # self.DicoVariablePSF.readonly(), DicoDirty.readonly(), # # ParmDict.readonly(), self.InitMachine.DeconvMachine.facetcache.readonly(),self.NCPU),serial=True) # self._initIsland_worker(subdict, iIsland, Island, # self.DicoVariablePSF, DicoDirty, # ParmDict, self.InitMachine.DeconvMachine.facetcache, # self.NCPU) # pBAR.render(nDone+1, NParallel) # nDone+=1 # # APP.awaitJobResults("InitIsland:*", progress="Init islands") # print>>log,"Initialise small islands (parallelised over islands)" # for iIsland,Island in enumerate(ListSmallIslands): # if not ListDoIsland or ListDoSmallIsland[iIsland]: # subdict = DicoInitIndiv.addSubdict(iIsland) # APP.runJob("InitIsland:%d" % iIsland, self._initIsland_worker, # args=(subdict.writeonly(), iIsland, Island, # self.DicoVariablePSF.readonly(), DicoDirty.readonly(), # ParmDict.readonly(), self.InitMachine.DeconvMachine.facetcache.readonly(),1)) # APP.awaitJobResults("InitIsland:*", progress="Init islands") # DicoInitIndiv.reload() print >> log, "Initialise islands (parallelised over islands)" if not self.GD["GAClean"]["ParallelInitHMP"]: pBAR = ProgressBar(Title=" Init islands") for iIsland, Island in enumerate(ListIslands): if not ListDoIsland or ListDoIsland[iIsland]: subdict = DicoInitIndiv.addSubdict(iIsland) self._initIsland_worker( subdict, iIsland, Island, self.DicoVariablePSF, DicoDirty, ParmDict, self.InitMachine.DeconvMachine.facetcache, 1) pBAR.render(iIsland, len(ListIslands)) else: for iIsland, Island in enumerate(ListIslands): if not ListDoIsland or ListDoIsland[iIsland]: subdict = DicoInitIndiv.addSubdict(iIsland) APP.runJob("InitIsland:%d" % iIsland, self._initIsland_worker, args=(subdict.writeonly(), iIsland, Island, self.DicoVariablePSF.readonly(), DicoDirty.readonly(), ParmDict.readonly(), self.InitMachine.DeconvMachine.facetcache. readonly(), 1)) APP.awaitJobResults("InitIsland:*", progress="Init islands") DicoInitIndiv.reload() ParmDict.delete() return DicoInitIndiv
def PlotSpec(self,Prefix=""): # Pdf file of target positions pdfname = "%s/%s_TARGET%s.pdf"%(self.DIRNAME,self.DynSpecMS.OutName,Prefix) print("Making pdf overview: %s"%pdfname, file=log) pBAR = ProgressBar(Title="Making pages") NPages=self.DynSpecMS.NDirSelected #Selected iDone=0 pBAR.render(0, NPages) with PdfPages(pdfname) as pdf: for iDir in range(self.DynSpecMS.NDir): self.fig = pylab.figure(1,figsize=(15, 15)) if self.DynSpecMS.PosArray.Type[iDir] == b"Off": continue self.PlotSpecSingleDir(iDir) pdf.savefig(bbox_inches='tight') pylab.close() iDone+=1 pBAR.render(iDone, NPages) # Pdf file of off positions NPages=self.DynSpecMS.NDir-self.DynSpecMS.NDirSelected #Off pix if NPages==0: return pdfname = "%s/%s_OFF%s.pdf"%(self.DIRNAME,self.DynSpecMS.OutName,Prefix) print("Making pdf overview: %s"%pdfname, file=log) pBAR = ProgressBar(Title="Making pages") pBAR.render(0, NPages) iDone=0 with PdfPages(pdfname) as pdf: for iDir in range(self.DynSpecMS.NDir): self.fig = pylab.figure(1, figsize=(15, 15)) if self.DynSpecMS.PosArray.Type[iDir]!=b"Off": continue self.PlotSpecSingleDir(iDir) pdf.savefig(bbox_inches='tight') pylab.close() iDone+=1 pBAR.render(iDone, NPages)
def calcDistanceMatrixMinParallel(self, ListIslands, Parallel=True): NIslands = len(ListIslands) self.D = np.zeros((NIslands, NIslands), np.float32) self.dx = np.zeros((NIslands, NIslands), np.int32) self.dy = np.zeros((NIslands, NIslands), np.int32) work_queue = multiprocessing.JoinableQueue() for iIsland in range(NIslands): work_queue.put({"iIsland": (iIsland)}) result_queue = multiprocessing.JoinableQueue() NJobs = work_queue.qsize() workerlist = [] NCPU = self.NCPU ListEdgeIslands = self.giveEdgesIslands(ListIslands) for ii in range(NCPU): W = WorkerDistance(work_queue, result_queue, ListEdgeIslands, self.IdSharedMem) workerlist.append(W) if Parallel: workerlist[ii].start() pBAR = ProgressBar(Title=" Calc. Dist. ") pBAR.render(0, NJobs) iResult = 0 if not Parallel: for ii in range(NCPU): workerlist[ii].run() # just run until all work is completed while iResult < NJobs: DicoResult = None if result_queue.qsize() != 0: try: DicoResult = result_queue.get() except: pass if DicoResult == None: time.sleep(0.5) continue if DicoResult["Success"]: iResult += 1 NDone = iResult pBAR.render(NDone, NJobs) iIsland = DicoResult["iIsland"] Result = NpShared.GiveArray("%sDistances_%6.6i" % (self.IdSharedMem, iIsland)) self.dx[iIsland] = Result[0] self.dy[iIsland] = Result[1] self.D[iIsland] = Result[2] NpShared.DelAll("%sDistances_%6.6i" % (self.IdSharedMem, iIsland)) if Parallel: for ii in range(NCPU): workerlist[ii].shutdown() workerlist[ii].terminate() workerlist[ii].join()
def cfg_from_data_dict(self, data, models, residuals, npix, cell_size_rad): time, uvw, antenna1, antenna2, flag = (data[i] for i in ('times', 'uvw', 'A0', 'A1', 'flags')) self._npix = npix self._cell_size_rad = cell_size_rad # Get point and gaussian sources self._point_sources = [] self._gaussian_sources = [] for model in models: if model[0] == 'Delta': self._point_sources.append(model) elif model[0] == 'Gaussian': self._gaussian_sources.append(model) else: raise ValueError("Montblanc does not predict " "'{mt}' model types.".format(mt=model[0])) self._npsrc = npsrc = len(self._point_sources) self._ngsrc = ngsrc = len(self._gaussian_sources) montblanc.log.info("{n} point sources".format(n=npsrc)) montblanc.log.info("{n} gaussian sources".format(n=ngsrc)) self._nchan = nchan = self._frequency.size # Merge antenna ID's and count their frequencies ants = np.concatenate((antenna1, antenna2)) unique_ants = np.unique(ants) ant_counts = np.bincount(ants) self._na = na = unique_ants.size assert self._na <= self._antenna_positions.shape[0], "ANTENNA_1 and ANTENNA_2 contains more indicies than antennae specified through MS.StationPos" self._na = na = self._antenna_positions.shape[0] # going to pad to the maximum number of antennas # can compute the time indexes using a scan operator # assuming the dataset is ordered and the time column # contains the integration centroid of the correlator dumps # note: time first, then antenna1 slow varying then antenna2 # fast varying self._sort_index = np.lexsort(np.array((time, antenna1, antenna2))[::-1]) tfilter = np.zeros(time[self._sort_index].shape, dtype=np.bool) tfilter[1:] = time[self._sort_index][1:] != time[self._sort_index][:-1] self._tindx = np.cumsum(tfilter) self._ntime = self._tindx[-1] + 1 # include the autocorrelations to safely pad the array to a maximum possible size self._nbl = self._na * (self._na - 1) // 2 + self._na self._blindx = self.baseline_index(antenna1[self._sort_index], antenna2[self._sort_index], self._na) # Sanity check antenna and row dimensions if self._antenna_positions.shape[0] < na: raise ValueError("Number of antenna positions {aps} " "is less than the number of antenna '{na}' " "found in antenna1/antenna2".format( aps=self._antenna_positions.shape, na=na)) self._nrow = self._nbl * self._ntime if self._nrow < uvw.shape[0]: raise ValueError("Number of rows in input chunk exceeds " "nbl * ntime. Please ensure that only one field, " "one observation and one data descriptor is present in the chunk you're " "trying to predict.") # construct a mask to indicate values sparse matrix self._datamask = np.zeros((self._ntime, self._nbl), dtype=np.bool) self._datamask[self._tindx, self._blindx] = True self._datamask = self._datamask.reshape((self._nrow)) print("Padding data matrix for missing baselines (%.2f %% data missing from MS)" % \ (100.0 - 100.0 * float(np.sum(self._datamask)) / self._datamask.size), file=log) # padded row numbers of the sparce data matrix self._sparseindx = np.cumsum(self._datamask) - 1 # pad the antenna array self._padded_a1 = np.empty((self._nbl), dtype=np.int32) self._padded_a2 = np.empty((self._nbl), dtype=np.int32) antenna_indicies = DataDictionaryManager.antenna_indicies(na, auto_correlations=True) for bl in range(self._nbl): blants = antenna_indicies[bl] self._padded_a1[bl] = blants[0] self._padded_a2[bl] = blants[1] self._padded_a1 = np.tile(self._padded_a1, self._ntime) self._padded_a2 = np.tile(self._padded_a2, self._ntime) assert np.all(self._padded_a1[self._datamask] == antenna1[self._sort_index]) assert np.all(self._padded_a2[self._datamask] == antenna2[self._sort_index]) # pad the time array self._unique_time = np.unique(time) # sorted unique times self._padded_time = self._unique_time.repeat(self._nbl) assert np.all(self._padded_time[self._datamask] == time[self._sort_index]) # Pad the uvw array to contain nbl * ntime entries (including the autocorrs) # with zeros where baselines may be missing self._residuals = residuals.view() self._padded_uvw = np.zeros((self._ntime, self._nbl, 3), dtype=uvw.dtype) self._padded_uvw[self._tindx, self._blindx, :] = uvw[self._sort_index, :] self._padded_uvw = self._padded_uvw.reshape((self._nrow, 3)) assert np.all(self._padded_uvw[self._datamask] == uvw[self._sort_index]) # CASA split may have removed completely flagged baselines so resynthesize # uv coordinates as best as possible print("Synthesizing new UVW coordinates from TIME column to fill gaps in measurement set", file=log) self._synth_uvw = DataDictionaryManager._synthesize_uvw(self._antenna_positions, self._padded_time, self._padded_a1, self._padded_a2, self._phase_dir) # for safety only fill missing slots with new UVW coordinates self._padded_uvw[np.logical_not(self._datamask)] = self._synth_uvw[np.logical_not(self._datamask)] q1 = np.percentile(np.abs(self._synth_uvw[self._datamask] - uvw[self._sort_index]), 1.0) q2 = np.percentile(np.abs(self._synth_uvw[self._datamask] - uvw[self._sort_index]), 50.0) q3 = np.percentile(np.abs(self._synth_uvw[self._datamask] - uvw[self._sort_index]), 99.0) print(ModColor.Str("WARNING: The 99th percentile error on newly synthesized UVW coordinates may be as large as %.5f" % q3), file=log) self._flag = flag # progress... self._numtotal = None self._numcomplete = 0 self._pBAR = ProgressBar(Title=" montblanc predict") self.render_progressbar() # Initialize WCS frame wcs = pywcs.WCS(naxis=2) # note half a pixel will correspond to even sized image projection poles montblanc.log.debug("NPIX: %d" % self._npix) assert self._npix % 2 == 1, "Currently only supports odd-sized maps" l0m0 = [self._npix // 2, self._npix // 2] wcs.wcs.crpix = l0m0 # remember that the WCS frame uses degrees wcs.wcs.cdelt = [np.rad2deg(self._cell_size_rad), np.rad2deg(self._cell_size_rad)] # assume SIN image projection wcs.wcs.ctype = ["RA---SIN","DEC--SIN"] wcs.wcs.crval = [np.rad2deg(self._phase_dir[0]), np.rad2deg(self._phase_dir[1])] self._wcs = wcs # Finally output some info montblanc.log.info('Chunk contains:' % unique_ants) montblanc.log.info('\tntime: %s' % self._ntime) montblanc.log.info('\tnbl: %s' % self._nbl) montblanc.log.info('\tnchan: %s' % self._nchan) montblanc.log.info('\tna: %s' % self._na)
def DeconvListIsland(self, ListIslands, ParallelMode="OverIsland", ListInitIslands=None): # ================== Parallel part NIslands = len(ListIslands) if NIslands == 0: return if ParallelMode == "OverIslands": NCPU = self.NCPU NCPU = np.min([NCPU, NIslands]) Parallel = True ParallelPerIsland = False StopWhenQueueEmpty = True elif ParallelMode == "PerIsland": NCPU = 1 #self.NCPU Parallel = True ParallelPerIsland = True StopWhenQueueEmpty = True # ######### Debug # ParallelPerIsland=False # Parallel=False # NCPU=1 # StopWhenQueueEmpty=True # ################## work_queue = multiprocessing.Queue() # shared dict to hold inputs and outputs to workers (each island number is a key) deconv_dict = shared_dict.create("DeconvListIslands") NJobs = NIslands T = ClassTimeIt.ClassTimeIt(" ") T.disable() for iIsland, ThisPixList in enumerate(ListIslands): island_dict = deconv_dict.addSubdict(iIsland) # print "%i/%i"%(iIsland,self.NIslands) island_dict["Island"] = np.array(ThisPixList) XY = np.array(ThisPixList, dtype=np.float32) xm, ym = np.mean(np.float32(XY), axis=0).astype(int) T.timeit("xm,ym") nchan, npol, _, _ = self._Dirty.shape JonesNorm = (self.DicoDirty["JonesNorm"][:, :, xm, ym]).reshape( (nchan, npol, 1, 1)) W = self.DicoDirty["WeightChansImages"] JonesNorm = np.sum(JonesNorm * W.reshape((nchan, 1, 1, 1)), axis=0).reshape((1, npol, 1, 1)) T.timeit("JonesNorm") IslandBestIndiv = self.ModelMachine.GiveIndividual(ThisPixList) T.timeit("GiveIndividual") FacetID = self.PSFServer.giveFacetID2(xm, ym) T.timeit("FacetID") island_dict["BestIndiv"] = IslandBestIndiv ListOrder = [ iIsland, FacetID, JonesNorm.flat[0], self.RMS**2, island_dict.path ] work_queue.put(ListOrder) T.timeit("Put") # ListArrayIslands=[np.array(ListIslands[iIsland]) for iIsland in range(NIslands)] # NpShared.PackListArray(SharedListIsland,ListArrayIslands) # T.timeit("Pack0") # SharedBestIndiv="%s.ListBestIndiv"%(self.IdSharedMem) # NpShared.PackListArray(SharedBestIndiv,ListBestIndiv) # T.timeit("Pack1") workerlist = [] # List_Result_queue=[] # for ii in range(NCPU): # List_Result_queue.append(multiprocessing.JoinableQueue()) result_queue = multiprocessing.Queue() Title = " Evolve pop." if self.DeconvMode == "MetroClean": Title = " Running chain" pBAR = ProgressBar(Title=Title) #pBAR.disable() pBAR.render(0, NJobs) for ii in range(NCPU): W = WorkerDeconvIsland(work_queue, result_queue, self.GD, self._Dirty, self.DicoVariablePSF["CubeVariablePSF"], IdSharedMem=self.IdSharedMem, FreqsInfo=self.PSFServer.DicoMappingDesc, ParallelPerIsland=ParallelPerIsland, StopWhenQueueEmpty=StopWhenQueueEmpty, DeconvMode=self.DeconvMode, NChains=self.NChains, ListInitIslands=ListInitIslands) workerlist.append(W) if Parallel: workerlist[ii].start() else: workerlist[ii].run() iResult = 0 #print "!!!!!!!!!!!!!!!!!!!!!!!!",iResult,NJobs while iResult < NJobs: DicoResult = None # for result_queue in List_Result_queue: # if result_queue.qsize()!=0: # try: # DicoResult=result_queue.get_nowait() # break # except: # pass # #DicoResult=result_queue.get() #print "!!!!!!!!!!!!!!!!!!!!!!!!! Qsize",result_queue.qsize() if result_queue.qsize() != 0: try: DicoResult = result_queue.get_nowait() except: pass #DicoResult=result_queue.get() if DicoResult is None: time.sleep(0.05) continue iResult += 1 NDone = iResult intPercent = int(100 * NDone / float(NJobs)) pBAR.render(NDone, NJobs) if DicoResult["Success"]: iIsland = DicoResult["iIsland"] island_dict = deconv_dict[iIsland] island_dict.reload() self.ModelMachine.AppendIsland(ListIslands[iIsland], island_dict["Model"].copy()) if DicoResult["HasError"]: self.ErrorModelMachine.AppendIsland( ThisPixList, ListIslands[iIsland], island_dict["sModel"].copy()) deconv_dict.delete() for ii in range(NCPU): try: workerlist[ii].shutdown() workerlist[ii].terminate() workerlist[ii].join() except: pass
def ReadMSInfos(self): DicoMSInfos = {} MSName=self.ListMSName[0] t0 = table(MSName, ack=False) tf0 = table("%s::SPECTRAL_WINDOW"%self.ListMSName[0], ack=False) self.ChanWidth = tf0.getcol("CHAN_WIDTH").ravel()[0] tf0.close() self.times = np.sort(np.unique(t0.getcol("TIME"))) self.dt=(self.times[1::]-self.times[0:-1]).min() t0.close() ta = table("%s::ANTENNA"%self.ListMSName[0], ack=False) self.na=ta.getcol("POSITION").shape[0] ta.close() tField = table("%s::FIELD"%MSName, ack=False) self.ra0, self.dec0 = tField.getcol("PHASE_DIR").ravel() # radians! if self.ra0<0.: self.ra0+=2.*np.pi tField.close() pBAR = ProgressBar(Title="Reading metadata") pBAR.render(0, self.nMS) #for iMS, MSName in enumerate(sorted(self.ListMSName)): for iMS, MSName in enumerate(self.ListMSName): try: t = table(MSName, ack=False) except Exception as e: s = str(e) DicoMSInfos[iMS] = {"Readable": False, "Exception": s} pBAR.render(iMS+1, self.nMS) continue if self.ColName not in t.colnames(): DicoMSInfos[iMS] = {"Readable": False, "Exception": "Missing Data colname %s"%self.ColName} pBAR.render(iMS+1, self.nMS) continue if self.ColWeights and (self.ColWeights not in t.colnames()): DicoMSInfos[iMS] = {"Readable": False, "Exception": "Missing Weights colname %s"%self.ColWeights} pBAR.render(iMS+1, self.nMS) continue if self.ModelName and (self.ModelName not in t.colnames()): DicoMSInfos[iMS] = {"Readable": False, "Exception": "Missing Model colname %s"%self.ModelName} pBAR.render(iMS+1, self.nMS) continue tf = table("%s::SPECTRAL_WINDOW"%MSName, ack=False) ThisTimes = np.unique(t.getcol("TIME")) if not np.allclose(ThisTimes, self.times): raise ValueError("should have the same times") DicoMSInfos[iMS] = {"MSName": MSName, "ChanFreq": tf.getcol("CHAN_FREQ").ravel(), # Hz "ChanWidth": tf.getcol("CHAN_WIDTH").ravel(), # Hz "times": ThisTimes, "startTime": Time(ThisTimes[0]/(24*3600.), format='mjd', scale='utc').isot, "stopTime": Time(ThisTimes[-1]/(24*3600.), format='mjd', scale='utc').isot, "deltaTime": (ThisTimes[-1] - ThisTimes[0])/3600., # h "Readable": True} if DicoMSInfos[iMS]["ChanWidth"][0] != self.ChanWidth: raise ValueError("should have the same chan width") pBAR.render(iMS+1, self.nMS) for iMS in range(self.nMS): if not DicoMSInfos[iMS]["Readable"]: print(ModColor.Str("Problem reading %s"%MSName), file=log) print(ModColor.Str(" %s"%DicoMSInfos[iMS]["Exception"]), file=log) t.close() tf.close() self.DicoMSInfos = DicoMSInfos self.FreqsAll = np.array([DicoMSInfos[iMS]["ChanFreq"] for iMS in list(DicoMSInfos.keys()) if DicoMSInfos[iMS]["Readable"]]) self.Freq_minmax = np.min(self.FreqsAll), np.max(self.FreqsAll) self.NTimes = self.times.size f0, f1 = self.Freq_minmax self.NChan = int((f1 - f0)/self.ChanWidth) + 1 # Fill properties self.tStart = DicoMSInfos[0]["startTime"] self.tStop = DicoMSInfos[0]["stopTime"] self.fMin = self.Freq_minmax[0] self.fMax = self.Freq_minmax[1] self.iCurrentMS=0
def eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None, halloffame=None, verbose=__debug__, PlotMachine=None): """This algorithm reproduce the simplest evolutionary algorithm as presented in chapter 7 of [Back2000]_. :param population: A list of individuals. :param toolbox: A :class:`~deap.base.Toolbox` that contains the evolution operators. :param cxpb: The probability of mating two individuals. :param mutpb: The probability of mutating an individual. :param ngen: The number of generation. :param stats: A :class:`~deap.tools.Statistics` object that is updated inplace, optional. :param halloffame: A :class:`~deap.tools.HallOfFame` object that will contain the best individuals, optional. :param verbose: Whether or not to log the statistics. :returns: The final population and a :class:`~deap.tools.Logbook` with the statistics of the evolution. The algorithm takes in a population and evolves it in place using the :meth:`varAnd` method. It returns the optimized population and a :class:`~deap.tools.Logbook` with the statistics of the evolution (if any). The logbook will contain the generation number, the number of evalutions for each generation and the statistics if a :class:`~deap.tools.Statistics` if any. The *cxpb* and *mutpb* arguments are passed to the :func:`varAnd` function. The pseudocode goes as follow :: evaluate(population) for g in range(ngen): population = select(population, len(population)) offspring = varAnd(population, toolbox, cxpb, mutpb) evaluate(offspring) population = offspring As stated in the pseudocode above, the algorithm goes as follow. First, it evaluates the individuals with an invalid fitness. Second, it enters the generational loop where the selection procedure is applied to entirely replace the parental population. The 1:1 replacement ratio of this algorithm **requires** the selection procedure to be stochastic and to select multiple times the same individual, for example, :func:`~deap.tools.selTournament` and :func:`~deap.tools.selRoulette`. Third, it applies the :func:`varAnd` function to produce the next generation population. Fourth, it evaluates the new individuals and compute the statistics on this population. Finally, when *ngen* generations are done, the algorithm returns a tuple with the final population and a :class:`~deap.tools.Logbook` of the evolution. .. note:: Using a non-stochastic selection method will result in no selection as the operator selects *n* individuals from a pool of *n*. This function expects the :meth:`toolbox.mate`, :meth:`toolbox.mutate`, :meth:`toolbox.select` and :meth:`toolbox.evaluate` aliases to be registered in the toolbox. .. [Back2000] Back, Fogel and Michalewicz, "Evolutionary Computation 1 : Basic Algorithms and Operators", 2000. """ T = ClassTimeIt.ClassTimeIt("VarAnd") T.disable() iT = 0 logbook = tools.Logbook() T.timeit(iT) iT += 1 logbook.header = ['gen', 'nevals'] + (stats.fields if stats else []) T.timeit(iT) iT += 1 # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in population if not ind.fitness.valid] T.timeit(iT) iT += 1 fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) T.timeit(iT) iT += 1 for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit T.timeit(iT) iT += 1 if halloffame is not None: halloffame.update(population) T.timeit(iT) iT += 1 record = stats.compile(population) if stats else {} # logbook.record(gen=0, nevals=len(invalid_ind), **record) # if verbose: # print logbook.stream T.timeit(iT) iT += 1 pBAR = ProgressBar('white', width=50, block='=', empty=' ', Title="Solving ", HeaderSize=10, TitleSize=13) NDone = 0 NJob = ngen intPercent = int(100 * NDone / float(NJob)) pBAR.render(intPercent, '%4i/%i' % (NDone, NJob)) # Begin the generational process for gen in range(1, ngen + 1): # Select the next generation individuals offspring = toolbox.select(population, len(population)) T.timeit(iT) iT += 1 # Vary the pool of individuals offspring = varAnd(offspring, toolbox, cxpb, mutpb) T.timeit(iT) iT += 1 # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in offspring if not ind.fitness.valid] #print len(invalid_ind) fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit T.timeit(iT) iT += 1 # Update the hall of fame with the generated individuals if halloffame is not None: halloffame.update(offspring) T.timeit(iT) iT += 1 # Replace the current population by the offspring population[:] = offspring T.timeit(iT) iT += 1 # Append the current generation statistics to the logbook record = stats.compile(population) if stats else {} # logbook.record(gen=gen, nevals=len(invalid_ind), **record) # if verbose: # print logbook.stream T.timeit(iT) iT += 1 #print halloffame[-1].fitness if PlotMachine is not None and PlotMachine is not False: PlotMachine.Plot(halloffame) T.timeit(iT) iT += 1 NDone = gen NJob = ngen intPercent = int(100 * NDone / float(NJob)) pBAR.render(intPercent, '%4i/%i' % (NDone, NJob)) return population, logbook