def FitSerial(self, psf, incr, StdResidual):
PMin, PMaj, PPA = psf
Islands = self.Islands
ImOut = np.zeros(Islands.MaskImage.shape, np.float32)
pBAR = ProgressBar('white', block='=', empty=' ', Title="Fit islands")
sourceList = []
for i in range(len(Islands.ListX)):
comment = 'Isl %i/%i' % (i + 1, len(Islands.ListX))
pBAR.render(int(100 * float(i + 1) / len(Islands.ListX)), comment)
xin, yin, zin = np.array(Islands.ListX[i]), np.array(
Islands.ListY[i]), np.array(Islands.ListS[i])
#xm=int(np.sum(xin*zin)/np.sum(zin))
#ym=int(np.sum(yin*zin)/np.sum(zin))
# Fit=ClassFit(xin,yin,zin,psf=(PMaj/incr,PMin/incr,PPA),noise=Islands.Noise[xm,ym])
Fit = ClassFit(xin,
yin,
zin,
psf=(PMaj / incr, PMin / incr, PPA + np.pi / 2),
noise=StdResidual) #,FreePars=["l", "m","s"])
sourceList.append(Fit.DoAllFit())
Fit.PutFittedArray(ImOut)
Islands.FitIm = ImOut
return sourceList
def BuildIslandList(self):
import scipy.ndimage
print >> log, " Labeling islands"
self.ImIsland, NIslands = scipy.ndimage.label(self.ImMask)
ImIsland = self.ImIsland
NIslands += 1
nx, _ = ImIsland.shape
print >> log, " Found %i islands" % NIslands
NMaxPix = 100000
Island = np.zeros((NIslands, NMaxPix, 2), np.int32)
NIslandNonZero = np.zeros((NIslands, ), np.int32)
print >> log, " Extracting pixels in islands"
pBAR = ProgressBar('white',
width=50,
block='=',
empty=' ',
Title=" Extracting ",
HeaderSize=10,
TitleSize=13)
comment = ''
for ipix in range(nx):
pBAR.render(int(100 * ipix / (nx - 1)), comment)
for jpix in range(nx):
iIsland = self.ImIsland[ipix, jpix]
if iIsland:
NThis = NIslandNonZero[iIsland]
Island[iIsland, NThis, 0] = ipix
Island[iIsland, NThis, 1] = jpix
NIslandNonZero[iIsland] += 1
print >> log, " Listing pixels in islands"
NMinPixIsland = 5
DicoIslands = collections.OrderedDict()
for iIsland in range(1, NIslands):
ind = np.where(Island[iIsland, :, 0] != 0)[0]
if ind.size < NMinPixIsland: continue
Npix = ind.size
Comps = np.zeros((Npix, 3), np.float32)
for ipix in range(Npix):
x, y = Island[iIsland, ipix, 0], Island[iIsland, ipix, 1]
s = self.Restored[0, 0, x, y]
Comps[ipix, 0] = x
Comps[ipix, 1] = y
Comps[ipix, 2] = s
DicoIslands[iIsland] = Comps
print >> log, " Final number of islands: %i" % len(DicoIslands)
self.DicoIslands = DicoIslands
def FitParallel(self, psf, incr, StdResidual):
NCPU = self.NCPU
PMin, PMaj, PPA = psf
Islands = self.Islands
ImOut = np.zeros(Islands.MaskImage.shape, np.float32)
sourceList = []
work_queue = multiprocessing.Queue()
result_queue = multiprocessing.Queue()
NJobs = len(Islands.ListX)
for iJob in range(NJobs):
work_queue.put([
iJob,
np.array(Islands.ListX[iJob]),
np.array(Islands.ListY[iJob]),
np.array(Islands.ListS[iJob])
])
workerlist = []
for ii in range(NCPU):
W = Worker(work_queue, result_queue, psf, incr, StdResidual)
workerlist.append(W)
workerlist[ii].start()
pBAR = ProgressBar('white',
width=50,
block='=',
empty=' ',
Title=" Init W ",
HeaderSize=10,
TitleSize=13)
pBAR.render(0, '%4i/%i' % (0, NJobs))
iResult = 0
SourceList = []
while iResult < NJobs:
DicoResult = result_queue.get()
if DicoResult["Success"]:
iResult += 1
NDone = iResult
intPercent = int(100 * NDone / float(NJobs))
pBAR.render(intPercent, '%4i/%i' % (NDone, NJobs))
SourceList.append(DicoResult["FitPars"])
for ii in range(NCPU):
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()
return SourceList
def eaSimple(population,
toolbox,
cxpb,
mutpb,
ngen,
stats=None,
halloffame=None,
verbose=__debug__,
PlotMachine=None):
"""This algorithm reproduce the simplest evolutionary algorithm as
presented in chapter 7 of [Back2000]_.
:param population: A list of individuals.
:param toolbox: A :class:`~deap.base.Toolbox` that contains the evolution
operators.
:param cxpb: The probability of mating two individuals.
:param mutpb: The probability of mutating an individual.
:param ngen: The number of generation.
:param stats: A :class:`~deap.tools.Statistics` object that is updated
inplace, optional.
:param halloffame: A :class:`~deap.tools.HallOfFame` object that will
contain the best individuals, optional.
:param verbose: Whether or not to log the statistics.
:returns: The final population and a :class:`~deap.tools.Logbook`
with the statistics of the evolution.
The algorithm takes in a population and evolves it in place using the
:meth:`varAnd` method. It returns the optimized population and a
:class:`~deap.tools.Logbook` with the statistics of the evolution (if
any). The logbook will contain the generation number, the number of
evalutions for each generation and the statistics if a
:class:`~deap.tools.Statistics` if any. The *cxpb* and *mutpb* arguments
are passed to the :func:`varAnd` function. The pseudocode goes as follow
::
evaluate(population)
for g in range(ngen):
population = select(population, len(population))
offspring = varAnd(population, toolbox, cxpb, mutpb)
evaluate(offspring)
population = offspring
As stated in the pseudocode above, the algorithm goes as follow. First, it
evaluates the individuals with an invalid fitness. Second, it enters the
generational loop where the selection procedure is applied to entirely
replace the parental population. The 1:1 replacement ratio of this
algorithm **requires** the selection procedure to be stochastic and to
select multiple times the same individual, for example,
:func:`~deap.tools.selTournament` and :func:`~deap.tools.selRoulette`.
Third, it applies the :func:`varAnd` function to produce the next
generation population. Fourth, it evaluates the new individuals and
compute the statistics on this population. Finally, when *ngen*
generations are done, the algorithm returns a tuple with the final
population and a :class:`~deap.tools.Logbook` of the evolution.
.. note::
Using a non-stochastic selection method will result in no selection as
the operator selects *n* individuals from a pool of *n*.
This function expects the :meth:`toolbox.mate`, :meth:`toolbox.mutate`,
:meth:`toolbox.select` and :meth:`toolbox.evaluate` aliases to be
registered in the toolbox.
.. [Back2000] Back, Fogel and Michalewicz, "Evolutionary Computation 1 :
Basic Algorithms and Operators", 2000.
"""
T = ClassTimeIt.ClassTimeIt("VarAnd")
T.disable()
iT = 0
logbook = tools.Logbook()
T.timeit(iT)
iT += 1
logbook.header = ['gen', 'nevals'] + (stats.fields if stats else [])
T.timeit(iT)
iT += 1
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in population if not ind.fitness.valid]
T.timeit(iT)
iT += 1
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
T.timeit(iT)
iT += 1
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
T.timeit(iT)
iT += 1
if halloffame is not None:
halloffame.update(population)
T.timeit(iT)
iT += 1
record = stats.compile(population) if stats else {}
# logbook.record(gen=0, nevals=len(invalid_ind), **record)
# if verbose:
# print logbook.stream
T.timeit(iT)
iT += 1
pBAR = ProgressBar('white',
width=50,
block='=',
empty=' ',
Title="Solving ",
HeaderSize=10,
TitleSize=13)
NDone = 0
NJob = ngen
intPercent = int(100 * NDone / float(NJob))
pBAR.render(intPercent, '%4i/%i' % (NDone, NJob))
# Begin the generational process
for gen in range(1, ngen + 1):
# Select the next generation individuals
offspring = toolbox.select(population, len(population))
T.timeit(iT)
iT += 1
# Vary the pool of individuals
offspring = varAnd(offspring, toolbox, cxpb, mutpb)
T.timeit(iT)
iT += 1
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
#print len(invalid_ind)
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
T.timeit(iT)
iT += 1
# Update the hall of fame with the generated individuals
if halloffame is not None:
halloffame.update(offspring)
T.timeit(iT)
iT += 1
# Replace the current population by the offspring
population[:] = offspring
T.timeit(iT)
iT += 1
# Append the current generation statistics to the logbook
record = stats.compile(population) if stats else {}
# logbook.record(gen=gen, nevals=len(invalid_ind), **record)
# if verbose:
# print logbook.stream
T.timeit(iT)
iT += 1
#print halloffame[-1].fitness
if PlotMachine is not None and PlotMachine is not False:
PlotMachine.Plot(halloffame)
T.timeit(iT)
iT += 1
NDone = gen
NJob = ngen
intPercent = int(100 * NDone / float(NJob))
pBAR.render(intPercent, '%4i/%i' % (NDone, NJob))
return population, logbook
def FindAllIslands(self):
A = self.A
if (self.Noise == None) & (self.MaskImage == None):
self.ComputeNoiseMap()
# N=self.Noise
# T=self.T
# if T!=None:
# #self.plot()
# indx,indy=np.where((A/N)>T)
# Abool=((A/N)>T)
# else:
# indx,indy=np.where(self.MaskImage!=0)
# Abool=self.MaskImage
# Lpos=[(indx[i],indy[i]) for i in range(indx.shape[0])]
# LIslands=[]
# # import pylab
# # pylab.imshow(Abool)
# # pylab.draw()
# # pylab.show()
# pBAR = ProgressBar('white', block='=', empty=' ',Title="Find islands")
# Lini=len(Lpos)
# while True:
# l=[]
# #print Lpos
# self.FindIsland(Abool,l,Lpos[0][0],Lpos[0][1])
# #print l
# LIslands.append(l)
# Lpos=list(set(Lpos)-set(l))
# comment=''
# pBAR.render(int(100* float(Lini-len(Lpos)) / Lini), comment)
# if len(Lpos)==0: break
LIslandsOut = []
#ImIsland=np.zeros_like(self.A)
inum = 1
self.ListX = []
self.ListY = []
self.ListS = []
import scipy.ndimage
print >> log, " Labeling islands"
self.ImIsland, NIslands = scipy.ndimage.label(self.MaskImage)
ImIsland = self.ImIsland
#NIslands+=1
nx, _ = ImIsland.shape
print >> log, " Found %i islands" % NIslands
NMaxPix = 300**2
Island = np.zeros((NIslands, NMaxPix, 2), np.int32)
NIslandNonZero = np.zeros((NIslands, ), np.int32)
print >> log, " Extractinng pixels in islands"
pBAR = ProgressBar('white',
width=50,
block='=',
empty=' ',
Title=" Extracting ",
HeaderSize=10,
TitleSize=13)
comment = ''
# for ipix in range(nx):
# pBAR.render(int(100*ipix / (nx-1)), comment)
# for jpix in range(nx):
# iIsland=self.ImIsland[ipix,jpix]
# if iIsland:
# NThis=NIslandNonZero[iIsland-1]
# Island[iIsland-1,NThis,0]=ipix
# Island[iIsland-1,NThis,1]=jpix
# NIslandNonZero[iIsland-1]+=1
indx, indy = np.where(self.ImIsland)
nx = indx.size
for iPix in range(nx):
#pBAR.render(int(100*iPix / (nx-1)), comment)
x, y = indx[iPix], indy[iPix]
iIsland = self.ImIsland[x, y]
if iIsland:
NThis = NIslandNonZero[iIsland - 1]
Island[iIsland - 1, NThis, 0] = x
Island[iIsland - 1, NThis, 1] = y
NIslandNonZero[iIsland - 1] += 1
print >> log, " Listing pixels in islands"
LIslands = []
for iIsland in range(NIslands):
ind = np.where(Island[iIsland, :, 0] != 0)[0]
ThisIsland = []
Npix = ind.size
for ipix in range(Npix):
ThisIsland.append([
Island[iIsland, ipix, 0].tolist(), Island[iIsland, ipix,
1].tolist()
])
LIslands.append(ThisIsland)
print >> log, " Selecting pixels in islands"
for i in LIslands:
condMinPix = (len(i) > self.MinPerIsland)
xpos = [i[ii][0] for ii in range(len(i))]
ypos = [i[ii][1] for ii in range(len(i))]
if len(xpos) == 0: continue
dx = np.max(xpos) - np.min(xpos)
dy = np.max(ypos) - np.min(ypos)
condDelta = (dx >= self.DeltaXYMin) & (dy >= self.DeltaXYMin)
if condMinPix & condDelta:
LIslandsOut.append(i)
X = []
Y = []
S = []
for ii in i:
X.append(ii[0])
Y.append(ii[1])
S.append(A[ii[0], ii[1]])
# print
# print X,Y,S
for iii in range(self.ExtendIsland):
self.ExtIsland(X, Y, S)
# print
# print X,Y,S
for ii in range(len(X)):
ImIsland[X[ii], Y[ii]] = 1 #inum
# stop
self.ListX.append(X)
self.ListY.append(Y)
self.ListS.append(S)
inum += 1
print >> log, " Final number of islands: %i" % len(self.ListX)
self.ImIsland = ImIsland
self.LIslands = LIslandsOut
def main(options=None):
if options == None:
f = open("last_MakePModel.obj", 'rb')
options = pickle.load(f)
Boost = int(options.Boost)
#CMethod=int(options.CMethod)
#NCluster=int(options.NCluster)
Osm = options.Osm
Pfact = float(options.Pfact)
DoPlot = (options.DoPlot == "1")
imname = options.im
snr = float(options.snr)
if Osm == "":
Osm = reformat.reformat(imname, LastSlash=False)
im = image(imname)
PMaj = None
try:
PMaj = (im.imageinfo()["restoringbeam"]["major"]["value"])
PMin = (im.imageinfo()["restoringbeam"]["minor"]["value"])
PPA = (im.imageinfo()["restoringbeam"]["positionangle"]["value"])
PMaj *= Pfact
PMin *= Pfact
except:
print(ModColor.Str(" No psf seen in header"))
pass
if options.PSF != "":
m0, m1, pa = options.PSF.split(',')
PMaj, PMin, PPA = float(m0), float(m1), float(pa)
PMaj *= Pfact
PMin *= Pfact
if PMaj != None:
print(
ModColor.Str(
" - Using psf (maj,min,pa)=(%6.2f, %6.2f, %6.2f) (mult. fact.=%6.2f)"
% (PMaj, PMin, PPA, Pfact),
col='green',
Bold=False))
else:
print(ModColor.Str(" - No psf info could be gotten from anywhere"))
print(
ModColor.Str(
" use PSF keyword to tell what the psf is or is not"))
exit()
ToSig = (1. / 3600.) * (np.pi / 180.) / (2. * np.sqrt(2. * np.log(2)))
PMaj *= ToSig
PMin *= ToSig
PPA *= np.pi / 180
b = im.getdata()[0, 0, :, :]
#b=b[3000:4000,3000:4000]#[120:170,300:370]
c = im.coordinates()
incr = np.abs(c.dict()["direction0"]["cdelt"][0])
print(
ModColor.Str(" - Psf Size Sigma_(Maj,Min) = (%5.1f,%5.1f) pixels" %
(PMaj / incr, PMin / incr),
col="green",
Bold=False))
Islands = ClassIslands.ClassIslands(b, snr, Boost=Boost, DoPlot=DoPlot)
Islands.FindAllIslands()
ImOut = np.zeros_like(b)
pBAR = ProgressBar('white', block='=', empty=' ', Title="Fit islands")
#print "ion"
#import pylab
#pylab.ion()
sourceList = []
for i in range(len(Islands.ListX)):
comment = 'Isl %i/%i' % (i + 1, len(Islands.ListX))
pBAR.render(int(100 * float(i + 1) / len(Islands.ListX)), comment)
xin, yin, zin = np.array(Islands.ListX[i]), np.array(
Islands.ListY[i]), np.array(Islands.ListS[i])
xm = int(np.sum(xin * zin) / np.sum(zin))
ym = int(np.sum(yin * zin) / np.sum(zin))
#Fit=ClassFit(xin,yin,zin,psf=(PMaj/incr,PMin/incr,PPA),noise=Islands.Noise[xm,ym])
Fit = ClassFit(xin,
yin,
zin,
psf=(PMaj / incr, PMin / incr, PPA),
noise=Islands.Noise[xm, ym],
FreePars=["l", "m", "s"])
sourceList.append(Fit.DoAllFit())
Fit.PutFittedArray(ImOut)
Islands.FitIm = ImOut
xlist = []
ylist = []
slist = []
Cat = np.zeros((50000, ),
dtype=[('ra', np.float), ('dec', np.float), ('s', np.float),
('Gmaj', np.float), ('Gmin', np.float),
('PA', np.float)])
Cat = Cat.view(np.recarray)
isource = 0
for Dico in sourceList:
for iCompDico in sorted(Dico.keys()):
CompDico = Dico[iCompDico]
i = CompDico["l"]
j = CompDico["m"]
s = CompDico["s"]
xlist.append(i)
ylist.append(j)
slist.append(s)
f, d, dec, ra = im.toworld((0, 0, i, j))
Cat.ra[isource] = ra
Cat.dec[isource] = dec
Cat.s[isource] = s
Cat.Gmin[isource] = CompDico["Sm"] * incr / ToSig
Cat.Gmaj[isource] = CompDico["SM"] * incr / ToSig
Cat.PA[isource] = CompDico["PA"]
isource += 1
Cat = Cat[Cat.ra != 0].copy()
Islands.FittedComps = (xlist, ylist, slist)
Islands.plot()
SM = ClassSM.ClassSM(
Osm, ReName=True, DoREG=True, SaveNp=True,
FromExt=Cat) #,NCluster=NCluster,DoPlot=DoPlot,ClusterMethod=CMethod)
#SM=ClassSM.ClassSM(Osm,ReName=True,SaveNp=True,DoPlot=DoPlot,FromExt=Cat)
SM.MakeREG()
SM.Save()