def calcDistanceMatrixMin(self, ListIslands):
NIslands = len(ListIslands)
self.D = np.zeros((NIslands, NIslands), np.float32)
self.dx = np.zeros((NIslands, NIslands), np.int32)
self.dy = np.zeros((NIslands, NIslands), np.int32)
pBAR = ProgressBar(Title=" Calc Dist")
#pBAR.disable()
NDone = 0
NJobs = NIslands
pBAR.render(0, NJobs)
for iIsland in range(NIslands):
x0, y0 = np.array(ListIslands[iIsland]).T
for jIsland in range(iIsland + 1, NIslands):
x1, y1 = np.array(ListIslands[jIsland]).T
dx = x0.reshape((-1, 1)) - x1.reshape((1, -1))
dy = y0.reshape((-1, 1)) - y1.reshape((1, -1))
d = np.sqrt(dx**2 + dy**2)
dmin = np.min(d)
self.D[jIsland, iIsland] = self.D[iIsland, jIsland] = dmin
indx, indy = np.where(d == dmin)
#print dx[indx[0],indy[0]],dy[indx[0],indy[0]],dmin
self.dx[jIsland, iIsland] = self.dx[iIsland,
jIsland] = dx[indx[0],
indy[0]]
self.dy[jIsland, iIsland] = self.dy[iIsland,
jIsland] = dy[indx[0],
indy[0]]
NDone += 1
pBAR.render(NDone, NJobs)
def EstimateBeam(self, TimesBeam, RA, DEC,progressBar=True, quiet=False):
TimesBeam = np.float64(np.array(TimesBeam))
T0s = TimesBeam[:-1].copy()
T1s = TimesBeam[1:].copy()
Tm = (T0s+T1s)/2.
# RA,DEC=self.BeamRAs,self.BeamDECs
NDir=RA.size
DicoBeam={}
FreqDomains=self.BeamMachine.getFreqDomains()
DicoBeam["VisToJonesChanMapping"]=self.GiveVisToJonesChanMapping(FreqDomains)
if not quiet:
print>>log,"VisToJonesChanMapping: %s"%DicoBeam["VisToJonesChanMapping"]
DicoBeam["Jones"]=np.zeros((Tm.size,NDir,self.MS.na,FreqDomains.shape[0],2,2),dtype=np.complex64)
DicoBeam["t0"]=np.zeros((Tm.size,),np.float64)
DicoBeam["t1"]=np.zeros((Tm.size,),np.float64)
DicoBeam["tm"]=np.zeros((Tm.size,),np.float64)
rac,decc=self.MS.OriginalRadec
pBAR= ProgressBar(Title=" Init E-Jones ")#, HeaderSize=10,TitleSize=13)
if not progressBar: pBAR.disable()
pBAR.render(0, Tm.size)
for itime in range(Tm.size):
DicoBeam["t0"][itime]=T0s[itime]
DicoBeam["t1"][itime]=T1s[itime]
DicoBeam["tm"][itime]=Tm[itime]
ThisTime=Tm[itime]
Beam=self.GiveInstrumentBeam(ThisTime,RA,DEC)
#
if self.GD["Beam"]["CenterNorm"]==1:
Beam0=self.GiveInstrumentBeam(ThisTime,np.array([rac]),np.array([decc]))
Beam0inv= ModLinAlg.BatchInverse(Beam0)
nd,_,_,_,_=Beam.shape
Ones=np.ones((nd, 1, 1, 1, 1),np.float32)
Beam0inv=Beam0inv*Ones
Beam= ModLinAlg.BatchDot(Beam0inv, Beam)
DicoBeam["Jones"][itime]=Beam
NDone=itime+1
pBAR.render(NDone,Tm.size)
DicoBeam["Jones"][itime] = Beam
nt, nd, na, nch, _, _ = DicoBeam["Jones"].shape
# DicoBeam["Jones"]=np.mean(DicoBeam["Jones"],axis=3).reshape((nt,nd,na,1,2,2))
# print TimesBeam-TimesBeam[0]
# print t0-t1
# print DicoBeam["t1"][-1]-DicoBeam["t0"][0]
return DicoBeam
def CalcCrossIslandFlux(self, ListIslands):
if self.PSFCross is None:
self.CalcCrossIslandPSF(ListIslands)
NIslands = len(ListIslands)
print >> log, " grouping cross contaminating islands..."
MaxIslandFlux = np.zeros((NIslands, ), np.float32)
DicoIsland = {}
Dirty = self.DicoDirty["MeanImage"]
for iIsland in range(NIslands):
x0, y0 = np.array(ListIslands[iIsland]).T
PixVals0 = Dirty[0, 0, x0, y0]
MaxIslandFlux[iIsland] = np.max(PixVals0)
DicoIsland[iIsland] = ListIslands[iIsland]
self.CrossFluxContrib = self.PSFCross * MaxIslandFlux.reshape(
(1, NIslands))
self.DicoIsland = DicoIsland
NDone = 0
NJobs = NIslands
pBAR = ProgressBar(Title=" Group islands")
pBAR.disable()
pBAR.render(0, NJobs)
Th = 0.05
ListIslandMerged = []
self.setCheckedIslands = set([])
for iIsland in range(NIslands):
x0, y0 = np.array(ListIslands[iIsland]).T
#print "Main %i (%f, %f)"%(iIsland,np.mean(x0),np.mean(y0))
NDone += 1
intPercent = int(100 * NDone / float(NJobs))
pBAR.render(NDone, NJobs)
ListIslandMerged.append(
list(self.GiveNearbyIsland(iIsland, set([]))))
ListIslands = []
for indIsland in ListIslandMerged:
if len(indIsland) == 0: continue
ThisIsland = DicoIsland[indIsland[0]]
for iIsland in indIsland[1::]:
ThisIsland += DicoIsland[iIsland]
ListIslands.append(ThisIsland)
print >> log, " have grouped %i --> %i islands" % (NIslands,
len(ListIslands))
return ListIslands
def PlotSpec(self,Prefix=""):
# Pdf file of target positions
pdfname = "%s/%s_TARGET%s.pdf"%(self.DIRNAME,self.DynSpecMS.OutName,Prefix)
print("Making pdf overview: %s"%pdfname, file=log)
pBAR = ProgressBar(Title="Making pages")
NPages=self.DynSpecMS.NDirSelected #Selected
iDone=0
pBAR.render(0, NPages)
with PdfPages(pdfname) as pdf:
for iDir in range(self.DynSpecMS.NDir):
self.fig = pylab.figure(1,figsize=(15, 15))
if self.DynSpecMS.PosArray.Type[iDir] == b"Off": continue
self.PlotSpecSingleDir(iDir)
pdf.savefig(bbox_inches='tight')
pylab.close()
iDone+=1
pBAR.render(iDone, NPages)
# Pdf file of off positions
NPages=self.DynSpecMS.NDir-self.DynSpecMS.NDirSelected #Off pix
if NPages==0: return
pdfname = "%s/%s_OFF%s.pdf"%(self.DIRNAME,self.DynSpecMS.OutName,Prefix)
print("Making pdf overview: %s"%pdfname, file=log)
pBAR = ProgressBar(Title="Making pages")
pBAR.render(0, NPages)
iDone=0
with PdfPages(pdfname) as pdf:
for iDir in range(self.DynSpecMS.NDir):
self.fig = pylab.figure(1, figsize=(15, 15))
if self.DynSpecMS.PosArray.Type[iDir]!=b"Off": continue
self.PlotSpecSingleDir(iDir)
pdf.savefig(bbox_inches='tight')
pylab.close()
iDone+=1
pBAR.render(iDone, NPages)
def AddUVW_dt(self):
log.print("Adding uvw speed info to main table: %s" % self.MSName)
log.print("Compute UVW speed column")
MSName = self.MSName
MS = self
t = table(MSName, readonly=False, ack=False)
times = t.getcol("TIME")
A0 = t.getcol("ANTENNA1")
A1 = t.getcol("ANTENNA2")
UVW = t.getcol("UVW")
UVW_dt = np.zeros_like(UVW)
if "UVWDT" not in t.colnames():
log.print("Adding column UVWDT in %s" % self.MSName)
desc = t.getcoldesc("UVW")
desc["name"] = "UVWDT"
desc['comment'] = desc['comment'].replace(" ", "_")
t.addcols(desc)
# # #######################
# LTimes=np.sort(np.unique(times))
# for iTime,ThisTime in enumerate(LTimes):
# print(iTime,LTimes.size)
# ind=np.where(times==ThisTime)[0]
# UVW_dt[ind]=MS.Give_dUVW_dt(times[ind],A0[ind],A1[ind])
# # #######################
na = MS.na
pBAR = ProgressBar(Title=" Calc dUVW/dt ")
pBAR.render(0, na)
for ant0 in range(na):
for ant1 in range(ant0, MS.na):
if ant0 == ant1: continue
C0 = ((A0 == ant0) & (A1 == ant1))
C1 = ((A1 == ant0) & (A0 == ant1))
ind = np.where(C0 | C1)[0]
if len(ind) == 0: continue # e.g. if antenna missing
UVWs = UVW[ind]
timess = times[ind]
dtimess = timess[1::] - timess[0:-1]
UVWs_dt0 = (UVWs[1::] - UVWs[0:-1]) / dtimess.reshape((-1, 1))
UVW_dt[ind[0:-1]] = UVWs_dt0
UVW_dt[ind[-1]] = UVWs_dt0[-1]
intPercent = int(100 * (ant0 + 1) / float(na))
pBAR.render(ant0 + 1, na)
log.print("Writing in column UVWDT")
t.putcol("UVWDT", UVW_dt)
t.close()
def awaitJobCounter(self, counter, progress=None, total=None, timeout=10):
if self.verbose > 2:
print >> log, " %s is complete" % counter.name
if progress:
current = counter.getValue()
total = total or current or 1
pBAR = ProgressBar(Title=" " + progress)
#pBAR.disable()
pBAR.render(total - current, total)
while current:
current = counter.awaitZeroWithTimeout(timeout)
pBAR.render(total - current, total)
if self._termination_event.is_set():
if self.verbose > 1:
print >> log, " termination event spotted, exiting"
raise WorkerProcessError()
else:
counter.awaitZero()
if self._termination_event.is_set():
if self.verbose > 1:
print >> log, " termination event spotted, exiting"
raise WorkerProcessError()
if self.verbose > 2:
print >> log, " %s is complete" % counter.name
def ReadMSInfos(self):
DicoMSInfos = {}
MSName=self.ListMSName[0]
t0 = table(MSName, ack=False)
tf0 = table("%s::SPECTRAL_WINDOW"%self.ListMSName[0], ack=False)
self.ChanWidth = tf0.getcol("CHAN_WIDTH").ravel()[0]
tf0.close()
self.times = np.sort(np.unique(t0.getcol("TIME")))
self.dt=(self.times[1::]-self.times[0:-1]).min()
t0.close()
ta = table("%s::ANTENNA"%self.ListMSName[0], ack=False)
self.na=ta.getcol("POSITION").shape[0]
ta.close()
tField = table("%s::FIELD"%MSName, ack=False)
self.ra0, self.dec0 = tField.getcol("PHASE_DIR").ravel() # radians!
if self.ra0<0.: self.ra0+=2.*np.pi
tField.close()
pBAR = ProgressBar(Title="Reading metadata")
pBAR.render(0, self.nMS)
#for iMS, MSName in enumerate(sorted(self.ListMSName)):
for iMS, MSName in enumerate(self.ListMSName):
try:
t = table(MSName, ack=False)
except Exception as e:
s = str(e)
DicoMSInfos[iMS] = {"Readable": False,
"Exception": s}
pBAR.render(iMS+1, self.nMS)
continue
if self.ColName not in t.colnames():
DicoMSInfos[iMS] = {"Readable": False,
"Exception": "Missing Data colname %s"%self.ColName}
pBAR.render(iMS+1, self.nMS)
continue
if self.ColWeights and (self.ColWeights not in t.colnames()):
DicoMSInfos[iMS] = {"Readable": False,
"Exception": "Missing Weights colname %s"%self.ColWeights}
pBAR.render(iMS+1, self.nMS)
continue
if self.ModelName and (self.ModelName not in t.colnames()):
DicoMSInfos[iMS] = {"Readable": False,
"Exception": "Missing Model colname %s"%self.ModelName}
pBAR.render(iMS+1, self.nMS)
continue
tf = table("%s::SPECTRAL_WINDOW"%MSName, ack=False)
ThisTimes = np.unique(t.getcol("TIME"))
if not np.allclose(ThisTimes, self.times):
raise ValueError("should have the same times")
DicoMSInfos[iMS] = {"MSName": MSName,
"ChanFreq": tf.getcol("CHAN_FREQ").ravel(), # Hz
"ChanWidth": tf.getcol("CHAN_WIDTH").ravel(), # Hz
"times": ThisTimes,
"startTime": Time(ThisTimes[0]/(24*3600.), format='mjd', scale='utc').isot,
"stopTime": Time(ThisTimes[-1]/(24*3600.), format='mjd', scale='utc').isot,
"deltaTime": (ThisTimes[-1] - ThisTimes[0])/3600., # h
"Readable": True}
if DicoMSInfos[iMS]["ChanWidth"][0] != self.ChanWidth:
raise ValueError("should have the same chan width")
pBAR.render(iMS+1, self.nMS)
for iMS in range(self.nMS):
if not DicoMSInfos[iMS]["Readable"]:
print(ModColor.Str("Problem reading %s"%MSName), file=log)
print(ModColor.Str(" %s"%DicoMSInfos[iMS]["Exception"]), file=log)
t.close()
tf.close()
self.DicoMSInfos = DicoMSInfos
self.FreqsAll = np.array([DicoMSInfos[iMS]["ChanFreq"] for iMS in list(DicoMSInfos.keys()) if DicoMSInfos[iMS]["Readable"]])
self.Freq_minmax = np.min(self.FreqsAll), np.max(self.FreqsAll)
self.NTimes = self.times.size
f0, f1 = self.Freq_minmax
self.NChan = int((f1 - f0)/self.ChanWidth) + 1
# Fill properties
self.tStart = DicoMSInfos[0]["startTime"]
self.tStop = DicoMSInfos[0]["stopTime"]
self.fMin = self.Freq_minmax[0]
self.fMax = self.Freq_minmax[1]
self.iCurrentMS=0
class DataDictionaryManager(object):
"""
Given a dictionary of MS arrays, infers the number of:
- timesteps
- baselines
- antenna
- channels
ASSUMPTIONS:
1. We're dealing with a single band
2. We're dealing with a single field
3. We're dealing with a single observation
Arguments:
MS: DDFacet.Data.ClassMS instance
pointing_sols: DDFacet.Data.PointingProvider associated to MS
solver_polarization_type: initiate montblanc for linear or circular feeds
"""
def __init__(self, MS, pointing_sols):
self._data_feed_labels = ClassStokes(MS.CorrelationIds, ["I"]).AvailableCorrelationProducts()
# Extract the required prediction feeds from the MS
if set(self._data_feed_labels) <= set(["XX", "XY", "YX", "YY"]):
self._montblanc_feed_labels = ["XX", "XY", "YX", "YY"]
self._solver_polarization_type = "linear"
elif set(self._data_feed_labels) <= set(["RR", "RL", "LR", "LL"]):
self._montblanc_feed_labels = ["RR", "RL", "LR", "LL"]
self._solver_polarization_type = "circular"
else:
raise RuntimeError("Montblanc only supports linear or circular feed measurements.")
# MS correlation to montblanc correlation map
self._predict_feed_map = [self._montblanc_feed_labels.index(x) for x in self._data_feed_labels]
montblanc.log.info("Montblanc to MS feed mapping: %s" % ",".join([str(x) for x in self._predict_feed_map]))
# Extract the antenna positions and
# phase direction from the measurement set
self._station_names = MS.StationNames
self._antenna_positions = MS.StationPos
self._phase_dir = np.array((MS.rarad, MS.decrad))
montblanc.log.info("Phase centre of {pc}".format(pc=np.rad2deg(self._phase_dir)))
# RIME frequencies from ::SPECTRAL_WINDOW
self._frequency = MS.ChanFreq
self._ref_frequency = MS.reffreq
# this montblanc instance is associated to the meta data of a single MS (or selection)
if not isinstance(MS, ClassMS):
raise TypeError("MS argument must be of type ClassMS")
self._MS = MS
# set preinstantiated pointing solutions provider
if not isinstance(pointing_sols, PointingProvider):
raise TypeError("pointing_sols argument must be of type PointingProvider")
self._pointing_solutions = pointing_sols
def _transform(self, c):
"""
Computes the lm coordinates from pixel coordinates assuming
a SIN projection in the WCS
"""
coord = np.deg2rad(self._wcs.wcs_pix2world(np.asarray([c]), 1))
delta_ang = coord[0] - self._phase_dir
if DEBUG:
montblanc.log.debug("WCS src coordinates [deg] '{c}'".format(c=np.rad2deg(coord)))
# assume SIN Projection
l = np.cos(coord[0, 1])*np.sin(delta_ang[0])
m = np.sin(coord[0, 1])*np.cos(self._phase_dir[1]) - \
np.cos(coord[0, 1])*np.sin(self._phase_dir[1])*np.cos(delta_ang[0])
return (l, m)
def _radec(self, c):
"""
Computes the radec image plane coordinates from pixel coordinates assuming
a SIN projection in the WCS
"""
coord = np.deg2rad(self._wcs.wcs_pix2world(np.asarray([c]), 1))
if DEBUG:
montblanc.log.debug("WCS src coordinates [deg] '{c}'".format(c=np.rad2deg(coord)))
return (coord[0, 0], coord[0, 1])
@classmethod
def _synthesize_uvw(cls, station_ECEF, time, a1, a2, phase_ref):
"""
Synthesizes new UVW coordinates based on time according to NRAO CASA convention (same as in fixvis)
User should check these UVW coordinates carefully - if time centroid was used to compute
original uvw coordinates the centroids of these new coordinates may be wrong, depending on whether
data timesteps were heavily flagged.
station_ECEF: ITRF station coordinates read from MS::ANTENNA
time: time column, preferably time centroid (padded to nrow = unique time * unique bl)
a1: ANTENNA_1 index (padded to nrow = unique time * unique bl)
a2: ANTENNA_2 index (padded to nrow = unique time * unique bl)
phase_ref: phase reference centre in radians
"""
assert time.size == a1.size
assert a1.size == a2.size
dm = measures()
epoch = dm.epoch("UT1", quantity(time[0], "s"))
refdir = dm.direction("j2000", quantity(phase_ref[0], "rad"), quantity(phase_ref[1], "rad"))
obs = dm.position("ITRF", quantity(station_ECEF[0, 0], "m"), quantity(station_ECEF[0, 1], "m"), quantity(station_ECEF[0, 2], "m"))
#setup local horizon coordinate frame with antenna 0 as reference position
dm.do_frame(obs)
dm.do_frame(refdir)
dm.do_frame(epoch)
ants = np.concatenate((a1, a2))
unique_ants = np.unique(ants)
unique_time = np.unique(time)
na = unique_ants.size
nbl = na * (na - 1) // 2 + na
ntime = unique_time.size
assert time.size == nbl * ntime, "Input arrays must be padded to include autocorrelations, all baselines and all time"
antenna_indicies = DataDictionaryManager.antenna_indicies(na, auto_correlations=True)
new_uvw = np.zeros((ntime*nbl, 3))
for ti, t in enumerate(unique_time):
epoch = dm.epoch("UT1", quantity(t, "s"))
dm.do_frame(epoch)
station_uv = np.zeros_like(station_ECEF)
for iapos, apos in enumerate(station_ECEF):
station_uv[iapos] = dm.to_uvw(dm.baseline("ITRF", quantity([apos[0], station_ECEF[0, 0]], "m"),
quantity([apos[1], station_ECEF[0, 1]], "m"),
quantity([apos[2], station_ECEF[0, 2]], "m")))["xyz"].get_value()[0:3]
for bl in range(nbl):
blants = antenna_indicies[bl]
bla1 = blants[0]
bla2 = blants[1]
new_uvw[ti*nbl + bl, :] = station_uv[bla1] - station_uv[bla2] # same as in CASA convention (Convention for UVW calculations in CASA, Rau 2013)
return new_uvw
def cfg_from_data_dict(self, data, models, residuals, npix, cell_size_rad):
time, uvw, antenna1, antenna2, flag = (data[i] for i in ('times', 'uvw', 'A0', 'A1', 'flags'))
self._npix = npix
self._cell_size_rad = cell_size_rad
# Get point and gaussian sources
self._point_sources = []
self._gaussian_sources = []
for model in models:
if model[0] == 'Delta':
self._point_sources.append(model)
elif model[0] == 'Gaussian':
self._gaussian_sources.append(model)
else:
raise ValueError("Montblanc does not predict "
"'{mt}' model types.".format(mt=model[0]))
self._npsrc = npsrc = len(self._point_sources)
self._ngsrc = ngsrc = len(self._gaussian_sources)
montblanc.log.info("{n} point sources".format(n=npsrc))
montblanc.log.info("{n} gaussian sources".format(n=ngsrc))
self._nchan = nchan = self._frequency.size
# Merge antenna ID's and count their frequencies
ants = np.concatenate((antenna1, antenna2))
unique_ants = np.unique(ants)
ant_counts = np.bincount(ants)
self._na = na = unique_ants.size
assert self._na <= self._antenna_positions.shape[0], "ANTENNA_1 and ANTENNA_2 contains more indicies than antennae specified through MS.StationPos"
self._na = na = self._antenna_positions.shape[0] # going to pad to the maximum number of antennas
# can compute the time indexes using a scan operator
# assuming the dataset is ordered and the time column
# contains the integration centroid of the correlator dumps
# note: time first, then antenna1 slow varying then antenna2
# fast varying
self._sort_index = np.lexsort(np.array((time, antenna1, antenna2))[::-1])
tfilter = np.zeros(time[self._sort_index].shape, dtype=np.bool)
tfilter[1:] = time[self._sort_index][1:] != time[self._sort_index][:-1]
self._tindx = np.cumsum(tfilter)
self._ntime = self._tindx[-1] + 1
# include the autocorrelations to safely pad the array to a maximum possible size
self._nbl = self._na * (self._na - 1) // 2 + self._na
self._blindx = self.baseline_index(antenna1[self._sort_index],
antenna2[self._sort_index],
self._na)
# Sanity check antenna and row dimensions
if self._antenna_positions.shape[0] < na:
raise ValueError("Number of antenna positions {aps} "
"is less than the number of antenna '{na}' "
"found in antenna1/antenna2".format(
aps=self._antenna_positions.shape, na=na))
self._nrow = self._nbl * self._ntime
if self._nrow < uvw.shape[0]:
raise ValueError("Number of rows in input chunk exceeds "
"nbl * ntime. Please ensure that only one field, "
"one observation and one data descriptor is present in the chunk you're "
"trying to predict.")
# construct a mask to indicate values sparse matrix
self._datamask = np.zeros((self._ntime, self._nbl), dtype=np.bool)
self._datamask[self._tindx, self._blindx] = True
self._datamask = self._datamask.reshape((self._nrow))
print("Padding data matrix for missing baselines (%.2f %% data missing from MS)" % \
(100.0 - 100.0 * float(np.sum(self._datamask)) / self._datamask.size), file=log)
# padded row numbers of the sparce data matrix
self._sparseindx = np.cumsum(self._datamask) - 1
# pad the antenna array
self._padded_a1 = np.empty((self._nbl), dtype=np.int32)
self._padded_a2 = np.empty((self._nbl), dtype=np.int32)
antenna_indicies = DataDictionaryManager.antenna_indicies(na, auto_correlations=True)
for bl in range(self._nbl):
blants = antenna_indicies[bl]
self._padded_a1[bl] = blants[0]
self._padded_a2[bl] = blants[1]
self._padded_a1 = np.tile(self._padded_a1, self._ntime)
self._padded_a2 = np.tile(self._padded_a2, self._ntime)
assert np.all(self._padded_a1[self._datamask] == antenna1[self._sort_index])
assert np.all(self._padded_a2[self._datamask] == antenna2[self._sort_index])
# pad the time array
self._unique_time = np.unique(time) # sorted unique times
self._padded_time = self._unique_time.repeat(self._nbl)
assert np.all(self._padded_time[self._datamask] == time[self._sort_index])
# Pad the uvw array to contain nbl * ntime entries (including the autocorrs)
# with zeros where baselines may be missing
self._residuals = residuals.view()
self._padded_uvw = np.zeros((self._ntime, self._nbl, 3), dtype=uvw.dtype)
self._padded_uvw[self._tindx, self._blindx, :] = uvw[self._sort_index, :]
self._padded_uvw = self._padded_uvw.reshape((self._nrow, 3))
assert np.all(self._padded_uvw[self._datamask] == uvw[self._sort_index])
# CASA split may have removed completely flagged baselines so resynthesize
# uv coordinates as best as possible
print("Synthesizing new UVW coordinates from TIME column to fill gaps in measurement set", file=log)
self._synth_uvw = DataDictionaryManager._synthesize_uvw(self._antenna_positions,
self._padded_time,
self._padded_a1,
self._padded_a2,
self._phase_dir)
# for safety only fill missing slots with new UVW coordinates
self._padded_uvw[np.logical_not(self._datamask)] = self._synth_uvw[np.logical_not(self._datamask)]
q1 = np.percentile(np.abs(self._synth_uvw[self._datamask] - uvw[self._sort_index]), 1.0)
q2 = np.percentile(np.abs(self._synth_uvw[self._datamask] - uvw[self._sort_index]), 50.0)
q3 = np.percentile(np.abs(self._synth_uvw[self._datamask] - uvw[self._sort_index]), 99.0)
print(ModColor.Str("WARNING: The 99th percentile error on newly synthesized UVW coordinates may be as large as %.5f" % q3), file=log)
self._flag = flag
# progress...
self._numtotal = None
self._numcomplete = 0
self._pBAR = ProgressBar(Title=" montblanc predict")
self.render_progressbar()
# Initialize WCS frame
wcs = pywcs.WCS(naxis=2)
# note half a pixel will correspond to even sized image projection poles
montblanc.log.debug("NPIX: %d" % self._npix)
assert self._npix % 2 == 1, "Currently only supports odd-sized maps"
l0m0 = [self._npix // 2, self._npix // 2]
wcs.wcs.crpix = l0m0
# remember that the WCS frame uses degrees
wcs.wcs.cdelt = [np.rad2deg(self._cell_size_rad),
np.rad2deg(self._cell_size_rad)]
# assume SIN image projection
wcs.wcs.ctype = ["RA---SIN","DEC--SIN"]
wcs.wcs.crval = [np.rad2deg(self._phase_dir[0]),
np.rad2deg(self._phase_dir[1])]
self._wcs = wcs
# Finally output some info
montblanc.log.info('Chunk contains:' % unique_ants)
montblanc.log.info('\tntime: %s' % self._ntime)
montblanc.log.info('\tnbl: %s' % self._nbl)
montblanc.log.info('\tnchan: %s' % self._nchan)
montblanc.log.info('\tna: %s' % self._na)
@classmethod
def antenna_indicies(cls, na, auto_correlations=True):
""" Compute base antenna pairs from baseline index """
k = 0 if auto_correlations == True else 1
ant1, ant2 = np.triu_indices(na, k)
return np.stack([ant1, ant2], axis=1)
@classmethod
def baseline_index(cls, a1, a2, no_antennae):
"""
Computes unique index of a baseline given antenna 1 and antenna 2
(zero indexed) as input. The arrays may or may not contain
auto-correlations.
There is a quadratic series expression relating a1 and a2
to a unique baseline index(can be found by the double difference
method)
Let slow_varying_index be S = min(a1, a2). The goal is to find
the number of fast varying terms. As the slow
varying terms increase these get fewer and fewer, because
we only consider unique baselines and not the conjugate
baselines)
B = (-S ^ 2 + 2 * S * # Ant + S) / 2 + diff between the
slowest and fastest varying antenna
:param a1: array of ANTENNA_1 ids
:param a2: array of ANTENNA_2 ids
:param no_antennae: number of antennae in the array
:return: array of baseline ids
"""
if a1.shape != a2.shape:
raise ValueError("a1 and a2 must have the same shape!")
slow_index = np.min(np.array([a1, a2]), axis=0)
return (slow_index * (-slow_index + (2 * no_antennae + 1))) // 2 + \
np.abs(a1 - a2)
def updated_dimensions(self):
return [('ntime', self._ntime), ('nbl', self._nbl),
('na', self._na), ('nbands', 1), ('nchan', self._nchan),
('npsrc', self._npsrc), ('ngsrc', self._ngsrc)]
def render_progressbar(self):
if self._numtotal is not None:
self._pBAR.render(self._numcomplete, self._numtotal)
def __init__(self,
MeanResidual,
MeanModelImage,
PSFServer,
DeltaChi2=4.,
IdSharedMem="",
NCPU=6):
IdSharedMem += "SmearSM."
NpShared.DelAll(IdSharedMem)
self.IdSharedMem = IdSharedMem
self.NCPU = NCPU
self.MeanModelImage = NpShared.ToShared(
"%sMeanModelImage" % self.IdSharedMem, MeanModelImage)
self.MeanResidual = NpShared.ToShared(
"%sMeanResidual" % self.IdSharedMem, MeanResidual)
NPixStats = 10000
RandomInd = np.int64(np.random.rand(NPixStats) * (MeanResidual.size))
self.RMS = np.std(np.real(self.MeanResidual.ravel()[RandomInd]))
self.FWHMMin = 3.
self.PSFServer = PSFServer
self.DeltaChi2 = DeltaChi2
self.Var = self.RMS**2
self.NImGauss = 31
self.CubeMeanVariablePSF = NpShared.ToShared(
"%sCubeMeanVariablePSF" % self.IdSharedMem,
self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF'])
self.DicoConvMachine = {}
N = self.NImGauss
dx, dy = np.mgrid[-(N // 2):N // 2:1j * N, -(N // 2):N // 2:1j * N]
ListPixParms = [(int(dx.ravel()[i]), int(dy.ravel()[i]))
for i in range(dx.size)]
ListPixData = ListPixParms
ConvMode = "Matrix"
N = self.NImGauss
#stop
#for
#ClassConvMachine():
#def __init__(self,PSF,ListPixParms,ListPixData,ConvMode):
d = np.sqrt(dx**2 + dy**2)
self.dist = d
self.NGauss = 10
GSig = np.linspace(0., 2, self.NGauss)
self.GSig = GSig
ListGauss = []
One = np.zeros_like(d)
One[N // 2, N // 2] = 1.
ListGauss.append(One)
for sig in GSig[1::]:
v = np.exp(-d**2 / (2. * sig**2))
Sv = np.sum(v)
v /= Sv
ListGauss.append(v)
self.ListGauss = ListGauss
print("Declare convolution machines", file=log)
NJobs = self.PSFServer.NFacets
pBAR = ProgressBar(Title=" Declare ")
#pBAR.disable()
pBAR.render(0, '%4i/%i' % (0, NJobs))
for iFacet in range(self.PSFServer.NFacets):
#print iFacet,"/",self.PSFServer.NFacets
PSF = self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF'][
iFacet] #[0,0]
_, _, NPixPSF, _ = PSF.shape
PSF = PSF[:, :, NPixPSF // 2 - N:NPixPSF // 2 + N + 1,
NPixPSF // 2 - N:NPixPSF // 2 + N + 1]
#print PSF.shape
#sig=1
#PSF=(np.exp(-self.dist**2/(2.*sig**2))).reshape(1,1,N,N)
self.DicoConvMachine[iFacet] = ClassConvMachine.ClassConvMachine(
PSF, ListPixParms, ListPixData, ConvMode)
CM = self.DicoConvMachine[iFacet].CM
NpShared.ToShared("%sCM_Facet%4.4i" % (self.IdSharedMem, iFacet),
CM)
#invCM=ModLinAlg.invSVD(np.float64(CM[0,0]))/self.Var
#NpShared.ToShared("%sInvCov_Facet%4.4i"%(self.IdSharedMem,iFacet),invCM)
NDone = iFacet + 1
intPercent = int(100 * NDone / float(NJobs))
pBAR.render(intPercent, '%4i/%i' % (NDone, NJobs))
PSFMean = np.mean(
self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF'], axis=0)
self.ConvMachineMeanPSF = ClassConvMachine.ClassConvMachine(
PSFMean, ListPixParms, ListPixData, ConvMode)
CM = self.ConvMachineMeanPSF.CM
invCM = ModLinAlg.invSVD(np.float64(CM[0, 0]), Cut=1e-8) / self.Var
NpShared.ToShared("%sInvCov_AllFacet" % (self.IdSharedMem), invCM)
self.FindSupport()
def Smear(self, Parallel=True):
if Parallel:
NCPU = self.NCPU
else:
NCPU = 1
StopWhenQueueEmpty = True
print("Building queue", file=log)
self.ModelOut = np.zeros_like(self.MeanModelImage)
indx, indy = np.where(self.MeanModelImage[0, 0] != 0)
#indx,indy=np.where(self.MeanModelImage==np.max(self.MeanModelImage))
work_queue = multiprocessing.Queue()
result_queue = multiprocessing.Queue()
SizeMax = int(indx.size / float(NCPU) / 100.)
SizeMax = np.max([SizeMax, 1])
iPix = 0
iQueue = 0
Queue = []
while iPix < indx.size:
xc, yc = indx[iPix], indy[iPix]
FacetID = self.PSFServer.giveFacetID2(xc, yc)
#DicoOrder={"xy":(xc,yc),
# "FacetID":FacetID}
Queue.append([xc, yc, FacetID])
iPix += 1
if (len(Queue) == SizeMax) | (iPix == indx.size):
NpShared.ToShared("%sQueue_%3.3i" % (self.IdSharedMem, iQueue),
np.array(Queue))
work_queue.put(iQueue)
Queue = []
iQueue += 1
NJobs = indx.size
workerlist = []
pBAR = ProgressBar(Title=" Find gaussian")
#pBAR.disable()
pBAR.render(0, '%4i/%i' % (0, NJobs))
for ii in range(NCPU):
W = WorkerSmear(work_queue,
result_queue,
IdSharedMem=self.IdSharedMem,
StopWhenQueueEmpty=StopWhenQueueEmpty,
NImGauss=self.NImGauss,
DeltaChi2=self.DeltaChi2,
ListGauss=self.ListGauss,
GSig=self.GSig,
Var=self.Var,
SigMin=self.SigMin)
workerlist.append(W)
if Parallel:
workerlist[ii].start()
else:
workerlist[ii].run()
N = self.NImGauss
iResult = 0
#print "!!!!!!!!!!!!!!!!!!!!!!!!",iResult,NJobs
while iResult < NJobs:
DicoResult = None
# for result_queue in List_Result_queue:
# if result_queue.qsize()!=0:
# try:
# DicoResult=result_queue.get_nowait()
# break
# except:
# pass
# #DicoResult=result_queue.get()
#print "!!!!!!!!!!!!!!!!!!!!!!!!! Qsize",result_queue.qsize()
#print work_queue.qsize(),result_queue.qsize()
if result_queue.qsize() != 0:
try:
DicoResult = result_queue.get_nowait()
except:
pass
#DicoResult=result_queue.get()
if DicoResult is None:
time.sleep(0.001)
continue
if DicoResult["Success"]:
iQueue = DicoResult["iQueue"]
Queue = NpShared.GiveArray("%sQueue_%3.3i" %
(self.IdSharedMem, iQueue))
for iJob in range(Queue.shape[0]):
x0, y0, iGauss = Queue[iJob]
SMax = self.MeanModelImage[0, 0, x0, y0]
SubModelOut = self.ModelOut[0,
0][x0 - N // 2:x0 + N // 2 + 1,
y0 - N // 2:y0 + N // 2 + 1]
SubModelOut += self.ListRestoredGauss[iGauss] * SMax
#iGauss=0
#print
#print SMax
#print np.sum(self.ListGauss[iGauss])
#print
SubModelOut += self.ListGauss[iGauss] * SMax
iResult += 1
NDone = iResult
intPercent = int(100 * NDone / float(NJobs))
pBAR.render(intPercent, '%4i/%i' % (NDone, NJobs))
for ii in range(NCPU):
try:
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()
except:
pass
return self.ModelOut
def awaitJobResults(self, jobspecs, progress=None, timing=None):
"""
Waits for job(s) given by arguments to complete, and returns their results.
Note that this only works for jobs scheduled by the same process, since each process has its own results map.
A process will block indefinitely is asked to await on jobs scheduled by another process.
Args:
jobspec: a job spec, or a list of job specs. Each spec can contain a wildcard e.g. "job*", to wait for
multiple jobs.
progress: if True, a progress bar with that title will be rendered
timing: if True, a timing report with that title will be printed (note that progress implies timing)
Returns:
a list of results. Each entry is the result returned by the job (if no wildcard), or a list
of results from each job (if has a wildcard)
"""
if os.getpid() != parent_pid:
raise RuntimeError(
"This method can only be called in the parent process. This is a bug."
)
if type(jobspecs) is str:
jobspecs = [jobspecs]
# make a dict of all jobs still outstanding
awaiting_jobs = {
} # this maps job_id to a set of jobspecs (if multiple) that it matches
job_results = OrderedDict() # this maps jobspec to a list of results
total_jobs = complete_jobs = 0
for jobspec in jobspecs:
matching_jobs = [
job_id for job_id in self._results_map.iterkeys()
if fnmatch.fnmatch(job_id, jobspec)
]
for job_id in matching_jobs:
awaiting_jobs.setdefault(job_id, set()).add(jobspec)
if not matching_jobs:
raise RuntimeError(
"no pending jobs matching '%s'. This is probably a bug." %
jobspec)
total_jobs += len(matching_jobs)
job_results[jobspec] = len(matching_jobs), []
# check dict of already returned results (perhaps from previous calls to awaitJobs). Remove
# matching results, and assign them to appropriate jobspec lists
for job_id, job in self._results_map.items():
if job_id in awaiting_jobs and job.complete:
for jobspec in awaiting_jobs[job_id]:
job_results[jobspec][1].append(job.result)
complete_jobs += 1
if not job.singleton:
del self._results_map[job_id]
del awaiting_jobs[job_id]
if progress:
pBAR = ProgressBar(Title=" " + progress)
pBAR.render(complete_jobs, (total_jobs or 1))
if self.verbose > 1:
print >> log, "checking job results: %s (%d still pending)" % (
", ".join([
"%s %d/%d" % (jobspec, len(results), njobs)
for jobspec, (njobs, results) in job_results.iteritems()
]), len(awaiting_jobs))
# sit here while any pending jobs remain
while awaiting_jobs and not self._termination_event.is_set():
try:
result = self._result_queue.get(True, 10)
except Queue.Empty:
# print>> log, "checking for dead workers"
# shoot the zombie process, if any
multiprocessing.active_children()
continue
# ok, dispatch the result
job_id = result["job_id"]
job = self._results_map.get(job_id)
if job is None:
raise KeyError(
"Job '%s' was not enqueued. This is a logic error." %
job_id)
job.setResult(result)
# if being awaited, dispatch appropriately
if job_id in awaiting_jobs:
for jobspec in awaiting_jobs[job_id]:
job_results[jobspec][1].append(result)
complete_jobs += 1
if not job.singleton:
del self._results_map[job_id]
del awaiting_jobs[job_id]
if progress:
pBAR.render(complete_jobs, (total_jobs or 1))
# print status update
if self.verbose > 1:
print >> log, "received job results %s" % " ".join([
"%s:%d" % (jobspec, len(results))
for jobspec, (_, results) in job_results.iteritems()
])
# render complete
if progress:
pBAR.render(complete_jobs, (total_jobs or 1))
if self._termination_event.is_set():
if self.verbose > 1:
print >> log, " termination event spotted, exiting"
raise WorkerProcessError()
# process list of results for each jobspec to check for errors
for jobspec, (njobs, results) in job_results.iteritems():
times = np.array([res['time'] for res in results])
num_errors = len([res for res in results if not res['success']])
if timing or progress:
print >> log, "%s: %d jobs complete, average single-core time %.2fs per job" % (
timing or progress, len(results), times.mean())
elif self.verbose > 0:
print >> log, "%s: %d jobs complete, average single-core time %.2fs per job" % (
jobspec, len(results), times.mean())
if num_errors:
print >> log, ModColor.Str(
"%s: %d jobs returned an error. Aborting." %
(jobspec, num_errors),
col="red")
raise RuntimeError("some distributed jobs have failed")
# return list of results
result_values = []
for jobspec, (_, results) in job_results.iteritems():
resvals = [
resitem["result"] if resitem["success"] else resitem["error"]
for resitem in results
]
if '*' not in jobspec:
resvals = resvals[0]
result_values.append(resvals)
return result_values[0] if len(result_values) == 1 else result_values
def giveDicoInitIndiv(self,
ListIslands,
ModelImage,
DicoDirty,
ListDoIsland=None,
Parallel=True):
NCPU = self.NCPU
work_queue = multiprocessing.JoinableQueue()
ListIslands = ListIslands #[300:308]
DoIsland = True
for iIsland in range(len(ListIslands)):
if ListDoIsland is not None:
DoIsland = ListDoIsland[iIsland]
if DoIsland: work_queue.put({"iIsland": iIsland})
result_queue = multiprocessing.JoinableQueue()
NJobs = work_queue.qsize()
workerlist = []
logger.setSilent(SilentModules)
#MyLogger.setLoud(SilentModules)
#MyLogger.setLoud("ClassImageDeconvMachineMSMF")
print >> log, "Launch MORESANE workers"
for ii in range(NCPU):
W = WorkerInitMSMF(work_queue, result_queue, self.GD,
self.DicoVariablePSF, DicoDirty, self.RefFreq,
self.GridFreqs, self.DegridFreqs,
self.MainCache, ModelImage, ListIslands,
self.IdSharedMem)
workerlist.append(W)
if Parallel:
workerlist[ii].start()
timer = ClassTimeIt.ClassTimeIt()
pBAR = ProgressBar(Title=" MORESANing islands ")
#pBAR.disable()
pBAR.render(0, NJobs)
iResult = 0
if not Parallel:
for ii in range(NCPU):
workerlist[ii].run() # just run until all work is completed
self.DicoInitIndiv = {}
while iResult < NJobs:
DicoResult = None
if result_queue.qsize() != 0:
try:
DicoResult = result_queue.get()
except:
pass
if DicoResult == None:
time.sleep(0.5)
continue
if DicoResult["Success"]:
iResult += 1
NDone = iResult
pBAR.render(NDone, NJobs)
iIsland = DicoResult["iIsland"]
NameDico = "%sDicoInitIsland_%5.5i" % (self.IdSharedMem,
iIsland)
Dico = NpShared.SharedToDico(NameDico)
self.DicoInitIndiv[iIsland] = copy.deepcopy(Dico)
NpShared.DelAll(NameDico)
if Parallel:
for ii in range(NCPU):
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()
#MyLogger.setLoud(["pymoresane.main"])
#MyLogger.setLoud(["ClassImageDeconvMachineMSMF","ClassPSFServer","ClassMultiScaleMachine","GiveModelMachine","ClassModelMachineMSMF"])
return self.DicoInitIndiv
def DeconvListIsland(self,
ListIslands,
ParallelMode="OverIsland",
ListInitIslands=None):
# ================== Parallel part
NIslands = len(ListIslands)
if NIslands == 0: return
if ParallelMode == "OverIslands":
NCPU = self.NCPU
NCPU = np.min([NCPU, NIslands])
Parallel = True
ParallelPerIsland = False
StopWhenQueueEmpty = True
elif ParallelMode == "PerIsland":
NCPU = 1 #self.NCPU
Parallel = True
ParallelPerIsland = True
StopWhenQueueEmpty = True
# ######### Debug
# ParallelPerIsland=False
# Parallel=False
# NCPU=1
# StopWhenQueueEmpty=True
# ##################
work_queue = multiprocessing.Queue()
# shared dict to hold inputs and outputs to workers (each island number is a key)
deconv_dict = shared_dict.create("DeconvListIslands")
NJobs = NIslands
T = ClassTimeIt.ClassTimeIt(" ")
T.disable()
for iIsland, ThisPixList in enumerate(ListIslands):
island_dict = deconv_dict.addSubdict(iIsland)
# print "%i/%i"%(iIsland,self.NIslands)
island_dict["Island"] = np.array(ThisPixList)
XY = np.array(ThisPixList, dtype=np.float32)
xm, ym = np.mean(np.float32(XY), axis=0).astype(int)
T.timeit("xm,ym")
nchan, npol, _, _ = self._Dirty.shape
JonesNorm = (self.DicoDirty["JonesNorm"][:, :, xm, ym]).reshape(
(nchan, npol, 1, 1))
W = self.DicoDirty["WeightChansImages"]
JonesNorm = np.sum(JonesNorm * W.reshape((nchan, 1, 1, 1)),
axis=0).reshape((1, npol, 1, 1))
T.timeit("JonesNorm")
IslandBestIndiv = self.ModelMachine.GiveIndividual(ThisPixList)
T.timeit("GiveIndividual")
FacetID = self.PSFServer.giveFacetID2(xm, ym)
T.timeit("FacetID")
island_dict["BestIndiv"] = IslandBestIndiv
ListOrder = [
iIsland, FacetID, JonesNorm.flat[0], self.RMS**2,
island_dict.path
]
work_queue.put(ListOrder)
T.timeit("Put")
# ListArrayIslands=[np.array(ListIslands[iIsland]) for iIsland in range(NIslands)]
# NpShared.PackListArray(SharedListIsland,ListArrayIslands)
# T.timeit("Pack0")
# SharedBestIndiv="%s.ListBestIndiv"%(self.IdSharedMem)
# NpShared.PackListArray(SharedBestIndiv,ListBestIndiv)
# T.timeit("Pack1")
workerlist = []
# List_Result_queue=[]
# for ii in range(NCPU):
# List_Result_queue.append(multiprocessing.JoinableQueue())
result_queue = multiprocessing.Queue()
Title = " Evolve pop."
if self.DeconvMode == "MetroClean":
Title = " Running chain"
pBAR = ProgressBar(Title=Title)
#pBAR.disable()
pBAR.render(0, NJobs)
for ii in range(NCPU):
W = WorkerDeconvIsland(work_queue,
result_queue,
self.GD,
self._Dirty,
self.DicoVariablePSF["CubeVariablePSF"],
IdSharedMem=self.IdSharedMem,
FreqsInfo=self.PSFServer.DicoMappingDesc,
ParallelPerIsland=ParallelPerIsland,
StopWhenQueueEmpty=StopWhenQueueEmpty,
DeconvMode=self.DeconvMode,
NChains=self.NChains,
ListInitIslands=ListInitIslands)
workerlist.append(W)
if Parallel:
workerlist[ii].start()
else:
workerlist[ii].run()
iResult = 0
#print "!!!!!!!!!!!!!!!!!!!!!!!!",iResult,NJobs
while iResult < NJobs:
DicoResult = None
# for result_queue in List_Result_queue:
# if result_queue.qsize()!=0:
# try:
# DicoResult=result_queue.get_nowait()
# break
# except:
# pass
# #DicoResult=result_queue.get()
#print "!!!!!!!!!!!!!!!!!!!!!!!!! Qsize",result_queue.qsize()
if result_queue.qsize() != 0:
try:
DicoResult = result_queue.get_nowait()
except:
pass
#DicoResult=result_queue.get()
if DicoResult is None:
time.sleep(0.05)
continue
iResult += 1
NDone = iResult
intPercent = int(100 * NDone / float(NJobs))
pBAR.render(NDone, NJobs)
if DicoResult["Success"]:
iIsland = DicoResult["iIsland"]
island_dict = deconv_dict[iIsland]
island_dict.reload()
self.ModelMachine.AppendIsland(ListIslands[iIsland],
island_dict["Model"].copy())
if DicoResult["HasError"]:
self.ErrorModelMachine.AppendIsland(
ThisPixList, ListIslands[iIsland],
island_dict["sModel"].copy())
deconv_dict.delete()
for ii in range(NCPU):
try:
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()
except:
pass
def giveDicoInitIndiv(self,
ListIslands,
ModelImage,
DicoDirty,
ListDoIsland=None):
DicoInitIndiv = shared_dict.create("DicoInitIsland")
ParmDict = shared_dict.create("InitSSDModelHMP")
ParmDict["ModelImage"] = ModelImage
ParmDict["GridFreqs"] = self.GridFreqs
ParmDict["DegridFreqs"] = self.DegridFreqs
# ListBigIslands=[]
# ListSmallIslands=[]
# ListDoBigIsland=[]
# ListDoSmallIsland=[]
# NParallel=0
# for iIsland,Island in enumerate(ListIslands):
# if len(Island)>self.GD["SSDClean"]["ConvFFTSwitch"]:
# ListBigIslands.append(Island)
# ListDoBigIsland.append(ListDoIsland[iIsland])
# if ListDoIsland or ListDoIsland[iIsland]:
# NParallel+=1
# else:
# ListSmallIslands.append(Island)
# ListDoSmallIsland.append(ListDoIsland[iIsland])
# print>>log,"Initialise big islands (parallelised per island)"
# pBAR= ProgressBar(Title="Init islands")
# pBAR.render(0, NParallel)
# nDone=0
# for iIsland,Island in enumerate(ListBigIslands):
# if not ListDoIsland or ListDoBigIsland[iIsland]:
# subdict = DicoInitIndiv.addSubdict(iIsland)
# # APP.runJob("InitIsland:%d" % iIsland, self._initIsland_worker,
# # args=(subdict.writeonly(), iIsland, Island,
# # self.DicoVariablePSF.readonly(), DicoDirty.readonly(),
# # ParmDict.readonly(), self.InitMachine.DeconvMachine.facetcache.readonly(),self.NCPU),serial=True)
# self._initIsland_worker(subdict, iIsland, Island,
# self.DicoVariablePSF, DicoDirty,
# ParmDict, self.InitMachine.DeconvMachine.facetcache,
# self.NCPU)
# pBAR.render(nDone+1, NParallel)
# nDone+=1
# # APP.awaitJobResults("InitIsland:*", progress="Init islands")
# print>>log,"Initialise small islands (parallelised over islands)"
# for iIsland,Island in enumerate(ListSmallIslands):
# if not ListDoIsland or ListDoSmallIsland[iIsland]:
# subdict = DicoInitIndiv.addSubdict(iIsland)
# APP.runJob("InitIsland:%d" % iIsland, self._initIsland_worker,
# args=(subdict.writeonly(), iIsland, Island,
# self.DicoVariablePSF.readonly(), DicoDirty.readonly(),
# ParmDict.readonly(), self.InitMachine.DeconvMachine.facetcache.readonly(),1))
# APP.awaitJobResults("InitIsland:*", progress="Init islands")
# DicoInitIndiv.reload()
print >> log, "Initialise islands (parallelised over islands)"
if not self.GD["GAClean"]["ParallelInitHMP"]:
pBAR = ProgressBar(Title=" Init islands")
for iIsland, Island in enumerate(ListIslands):
if not ListDoIsland or ListDoIsland[iIsland]:
subdict = DicoInitIndiv.addSubdict(iIsland)
self._initIsland_worker(
subdict, iIsland, Island, self.DicoVariablePSF,
DicoDirty, ParmDict,
self.InitMachine.DeconvMachine.facetcache, 1)
pBAR.render(iIsland, len(ListIslands))
else:
for iIsland, Island in enumerate(ListIslands):
if not ListDoIsland or ListDoIsland[iIsland]:
subdict = DicoInitIndiv.addSubdict(iIsland)
APP.runJob("InitIsland:%d" % iIsland,
self._initIsland_worker,
args=(subdict.writeonly(), iIsland, Island,
self.DicoVariablePSF.readonly(),
DicoDirty.readonly(), ParmDict.readonly(),
self.InitMachine.DeconvMachine.facetcache.
readonly(), 1))
APP.awaitJobResults("InitIsland:*", progress="Init islands")
DicoInitIndiv.reload()
ParmDict.delete()
return DicoInitIndiv
def calcDistanceMatrixMinParallel(self, ListIslands, Parallel=True):
NIslands = len(ListIslands)
self.D = np.zeros((NIslands, NIslands), np.float32)
self.dx = np.zeros((NIslands, NIslands), np.int32)
self.dy = np.zeros((NIslands, NIslands), np.int32)
work_queue = multiprocessing.JoinableQueue()
for iIsland in range(NIslands):
work_queue.put({"iIsland": (iIsland)})
result_queue = multiprocessing.JoinableQueue()
NJobs = work_queue.qsize()
workerlist = []
NCPU = self.NCPU
ListEdgeIslands = self.giveEdgesIslands(ListIslands)
for ii in range(NCPU):
W = WorkerDistance(work_queue, result_queue, ListEdgeIslands,
self.IdSharedMem)
workerlist.append(W)
if Parallel:
workerlist[ii].start()
pBAR = ProgressBar(Title=" Calc. Dist. ")
pBAR.render(0, NJobs)
iResult = 0
if not Parallel:
for ii in range(NCPU):
workerlist[ii].run() # just run until all work is completed
while iResult < NJobs:
DicoResult = None
if result_queue.qsize() != 0:
try:
DicoResult = result_queue.get()
except:
pass
if DicoResult == None:
time.sleep(0.5)
continue
if DicoResult["Success"]:
iResult += 1
NDone = iResult
pBAR.render(NDone, NJobs)
iIsland = DicoResult["iIsland"]
Result = NpShared.GiveArray("%sDistances_%6.6i" %
(self.IdSharedMem, iIsland))
self.dx[iIsland] = Result[0]
self.dy[iIsland] = Result[1]
self.D[iIsland] = Result[2]
NpShared.DelAll("%sDistances_%6.6i" %
(self.IdSharedMem, iIsland))
if Parallel:
for ii in range(NCPU):
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()
def eaSimple(population,
toolbox,
cxpb,
mutpb,
ngen,
stats=None,
halloffame=None,
verbose=__debug__,
PlotMachine=None):
"""This algorithm reproduce the simplest evolutionary algorithm as
presented in chapter 7 of [Back2000]_.
:param population: A list of individuals.
:param toolbox: A :class:`~deap.base.Toolbox` that contains the evolution
operators.
:param cxpb: The probability of mating two individuals.
:param mutpb: The probability of mutating an individual.
:param ngen: The number of generation.
:param stats: A :class:`~deap.tools.Statistics` object that is updated
inplace, optional.
:param halloffame: A :class:`~deap.tools.HallOfFame` object that will
contain the best individuals, optional.
:param verbose: Whether or not to log the statistics.
:returns: The final population and a :class:`~deap.tools.Logbook`
with the statistics of the evolution.
The algorithm takes in a population and evolves it in place using the
:meth:`varAnd` method. It returns the optimized population and a
:class:`~deap.tools.Logbook` with the statistics of the evolution (if
any). The logbook will contain the generation number, the number of
evalutions for each generation and the statistics if a
:class:`~deap.tools.Statistics` if any. The *cxpb* and *mutpb* arguments
are passed to the :func:`varAnd` function. The pseudocode goes as follow
::
evaluate(population)
for g in range(ngen):
population = select(population, len(population))
offspring = varAnd(population, toolbox, cxpb, mutpb)
evaluate(offspring)
population = offspring
As stated in the pseudocode above, the algorithm goes as follow. First, it
evaluates the individuals with an invalid fitness. Second, it enters the
generational loop where the selection procedure is applied to entirely
replace the parental population. The 1:1 replacement ratio of this
algorithm **requires** the selection procedure to be stochastic and to
select multiple times the same individual, for example,
:func:`~deap.tools.selTournament` and :func:`~deap.tools.selRoulette`.
Third, it applies the :func:`varAnd` function to produce the next
generation population. Fourth, it evaluates the new individuals and
compute the statistics on this population. Finally, when *ngen*
generations are done, the algorithm returns a tuple with the final
population and a :class:`~deap.tools.Logbook` of the evolution.
.. note::
Using a non-stochastic selection method will result in no selection as
the operator selects *n* individuals from a pool of *n*.
This function expects the :meth:`toolbox.mate`, :meth:`toolbox.mutate`,
:meth:`toolbox.select` and :meth:`toolbox.evaluate` aliases to be
registered in the toolbox.
.. [Back2000] Back, Fogel and Michalewicz, "Evolutionary Computation 1 :
Basic Algorithms and Operators", 2000.
"""
T = ClassTimeIt.ClassTimeIt("VarAnd")
T.disable()
iT = 0
logbook = tools.Logbook()
T.timeit(iT)
iT += 1
logbook.header = ['gen', 'nevals'] + (stats.fields if stats else [])
T.timeit(iT)
iT += 1
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in population if not ind.fitness.valid]
T.timeit(iT)
iT += 1
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
T.timeit(iT)
iT += 1
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
T.timeit(iT)
iT += 1
if halloffame is not None:
halloffame.update(population)
T.timeit(iT)
iT += 1
record = stats.compile(population) if stats else {}
# logbook.record(gen=0, nevals=len(invalid_ind), **record)
# if verbose:
# print logbook.stream
T.timeit(iT)
iT += 1
pBAR = ProgressBar('white',
width=50,
block='=',
empty=' ',
Title="Solving ",
HeaderSize=10,
TitleSize=13)
NDone = 0
NJob = ngen
intPercent = int(100 * NDone / float(NJob))
pBAR.render(intPercent, '%4i/%i' % (NDone, NJob))
# Begin the generational process
for gen in range(1, ngen + 1):
# Select the next generation individuals
offspring = toolbox.select(population, len(population))
T.timeit(iT)
iT += 1
# Vary the pool of individuals
offspring = varAnd(offspring, toolbox, cxpb, mutpb)
T.timeit(iT)
iT += 1
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
#print len(invalid_ind)
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
T.timeit(iT)
iT += 1
# Update the hall of fame with the generated individuals
if halloffame is not None:
halloffame.update(offspring)
T.timeit(iT)
iT += 1
# Replace the current population by the offspring
population[:] = offspring
T.timeit(iT)
iT += 1
# Append the current generation statistics to the logbook
record = stats.compile(population) if stats else {}
# logbook.record(gen=gen, nevals=len(invalid_ind), **record)
# if verbose:
# print logbook.stream
T.timeit(iT)
iT += 1
#print halloffame[-1].fitness
if PlotMachine is not None and PlotMachine is not False:
PlotMachine.Plot(halloffame)
T.timeit(iT)
iT += 1
NDone = gen
NJob = ngen
intPercent = int(100 * NDone / float(NJob))
pBAR.render(intPercent, '%4i/%i' % (NDone, NJob))
return population, logbook
