def cleanupShm ():
"""
Deletes all shared arrays for this process
"""
NpShared.DelAll(getShmPrefix())
# above statement don't work for directories and subdirectories
os.system("rm -rf /dev/shm/%s"%getShmPrefix())
def __init__(self,
MeanResidual,
MeanModelImage,
PSFServer,
DeltaChi2=4.,
IdSharedMem="",
NCPU=6):
IdSharedMem += "SmearSM."
NpShared.DelAll(IdSharedMem)
self.IdSharedMem = IdSharedMem
self.NCPU = NCPU
self.MeanModelImage = NpShared.ToShared(
"%sMeanModelImage" % self.IdSharedMem, MeanModelImage)
self.MeanResidual = NpShared.ToShared(
"%sMeanResidual" % self.IdSharedMem, MeanResidual)
NPixStats = 10000
RandomInd = np.int64(np.random.rand(NPixStats) * (MeanResidual.size))
self.RMS = np.std(np.real(self.MeanResidual.ravel()[RandomInd]))
self.FWHMMin = 3.
self.PSFServer = PSFServer
self.DeltaChi2 = DeltaChi2
self.Var = self.RMS**2
self.NImGauss = 31
self.CubeMeanVariablePSF = NpShared.ToShared(
"%sCubeMeanVariablePSF" % self.IdSharedMem,
self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF'])
self.DicoConvMachine = {}
N = self.NImGauss
dx, dy = np.mgrid[-(N // 2):N // 2:1j * N, -(N // 2):N // 2:1j * N]
ListPixParms = [(int(dx.ravel()[i]), int(dy.ravel()[i]))
for i in range(dx.size)]
ListPixData = ListPixParms
ConvMode = "Matrix"
N = self.NImGauss
#stop
#for
#ClassConvMachine():
#def __init__(self,PSF,ListPixParms,ListPixData,ConvMode):
d = np.sqrt(dx**2 + dy**2)
self.dist = d
self.NGauss = 10
GSig = np.linspace(0., 2, self.NGauss)
self.GSig = GSig
ListGauss = []
One = np.zeros_like(d)
One[N // 2, N // 2] = 1.
ListGauss.append(One)
for sig in GSig[1::]:
v = np.exp(-d**2 / (2. * sig**2))
Sv = np.sum(v)
v /= Sv
ListGauss.append(v)
self.ListGauss = ListGauss
print("Declare convolution machines", file=log)
NJobs = self.PSFServer.NFacets
pBAR = ProgressBar(Title=" Declare ")
#pBAR.disable()
pBAR.render(0, '%4i/%i' % (0, NJobs))
for iFacet in range(self.PSFServer.NFacets):
#print iFacet,"/",self.PSFServer.NFacets
PSF = self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF'][
iFacet] #[0,0]
_, _, NPixPSF, _ = PSF.shape
PSF = PSF[:, :, NPixPSF // 2 - N:NPixPSF // 2 + N + 1,
NPixPSF // 2 - N:NPixPSF // 2 + N + 1]
#print PSF.shape
#sig=1
#PSF=(np.exp(-self.dist**2/(2.*sig**2))).reshape(1,1,N,N)
self.DicoConvMachine[iFacet] = ClassConvMachine.ClassConvMachine(
PSF, ListPixParms, ListPixData, ConvMode)
CM = self.DicoConvMachine[iFacet].CM
NpShared.ToShared("%sCM_Facet%4.4i" % (self.IdSharedMem, iFacet),
CM)
#invCM=ModLinAlg.invSVD(np.float64(CM[0,0]))/self.Var
#NpShared.ToShared("%sInvCov_Facet%4.4i"%(self.IdSharedMem,iFacet),invCM)
NDone = iFacet + 1
intPercent = int(100 * NDone / float(NJobs))
pBAR.render(intPercent, '%4i/%i' % (NDone, NJobs))
PSFMean = np.mean(
self.PSFServer.DicoVariablePSF['CubeMeanVariablePSF'], axis=0)
self.ConvMachineMeanPSF = ClassConvMachine.ClassConvMachine(
PSFMean, ListPixParms, ListPixData, ConvMode)
CM = self.ConvMachineMeanPSF.CM
invCM = ModLinAlg.invSVD(np.float64(CM[0, 0]), Cut=1e-8) / self.Var
NpShared.ToShared("%sInvCov_AllFacet" % (self.IdSharedMem), invCM)
self.FindSupport()
def CleanUpSHM(self):
NpShared.DelAll(self.IdSharedMem)
group = optparse.OptionGroup(opt, "* SHM")
group.add_option(
'--ID',
help='ID of shared memory to be deleted, default is %default',
default=None)
opt.add_option_group(group)
options, arguments = opt.parse_args()
return options
if __name__ == "__main__":
options = read_options()
print >> log, "Clear shared memory"
if options.ID is not None:
NpShared.DelAll(options.ID)
else:
NpShared.DelAll()
Multiprocessing.cleanupStaleShm()
Multiprocessing.cleanupShm()
ll = glob.glob("/dev/shm/sem.*")
print >> log, "Clear Semaphores"
# remove semaphores we don't have access to
ll = filter(lambda x: os.access(x, os.W_OK), ll)
ListSemaphores = [".".join(l.split(".")[1::]) for l in ll]
_pyGridderSmear.pySetSemaphores(ListSemaphores)
_pyGridderSmear.pyDeleteSemaphore(ListSemaphores)
def giveDicoInitIndiv(self,
ListIslands,
ModelImage,
DicoDirty,
ListDoIsland=None,
Parallel=True):
NCPU = self.NCPU
work_queue = multiprocessing.JoinableQueue()
ListIslands = ListIslands #[300:308]
DoIsland = True
for iIsland in range(len(ListIslands)):
if ListDoIsland is not None:
DoIsland = ListDoIsland[iIsland]
if DoIsland: work_queue.put({"iIsland": iIsland})
result_queue = multiprocessing.JoinableQueue()
NJobs = work_queue.qsize()
workerlist = []
logger.setSilent(SilentModules)
#MyLogger.setLoud(SilentModules)
#MyLogger.setLoud("ClassImageDeconvMachineMSMF")
print >> log, "Launch MORESANE workers"
for ii in range(NCPU):
W = WorkerInitMSMF(work_queue, result_queue, self.GD,
self.DicoVariablePSF, DicoDirty, self.RefFreq,
self.GridFreqs, self.DegridFreqs,
self.MainCache, ModelImage, ListIslands,
self.IdSharedMem)
workerlist.append(W)
if Parallel:
workerlist[ii].start()
timer = ClassTimeIt.ClassTimeIt()
pBAR = ProgressBar(Title=" MORESANing islands ")
#pBAR.disable()
pBAR.render(0, NJobs)
iResult = 0
if not Parallel:
for ii in range(NCPU):
workerlist[ii].run() # just run until all work is completed
self.DicoInitIndiv = {}
while iResult < NJobs:
DicoResult = None
if result_queue.qsize() != 0:
try:
DicoResult = result_queue.get()
except:
pass
if DicoResult == None:
time.sleep(0.5)
continue
if DicoResult["Success"]:
iResult += 1
NDone = iResult
pBAR.render(NDone, NJobs)
iIsland = DicoResult["iIsland"]
NameDico = "%sDicoInitIsland_%5.5i" % (self.IdSharedMem,
iIsland)
Dico = NpShared.SharedToDico(NameDico)
self.DicoInitIndiv[iIsland] = copy.deepcopy(Dico)
NpShared.DelAll(NameDico)
if Parallel:
for ii in range(NCPU):
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()
#MyLogger.setLoud(["pymoresane.main"])
#MyLogger.setLoud(["ClassImageDeconvMachineMSMF","ClassPSFServer","ClassMultiScaleMachine","GiveModelMachine","ClassModelMachineMSMF"])
return self.DicoInitIndiv
def calcDistanceMatrixMinParallel(self, ListIslands, Parallel=True):
NIslands = len(ListIslands)
self.D = np.zeros((NIslands, NIslands), np.float32)
self.dx = np.zeros((NIslands, NIslands), np.int32)
self.dy = np.zeros((NIslands, NIslands), np.int32)
work_queue = multiprocessing.JoinableQueue()
for iIsland in range(NIslands):
work_queue.put({"iIsland": (iIsland)})
result_queue = multiprocessing.JoinableQueue()
NJobs = work_queue.qsize()
workerlist = []
NCPU = self.NCPU
ListEdgeIslands = self.giveEdgesIslands(ListIslands)
for ii in range(NCPU):
W = WorkerDistance(work_queue, result_queue, ListEdgeIslands,
self.IdSharedMem)
workerlist.append(W)
if Parallel:
workerlist[ii].start()
pBAR = ProgressBar(Title=" Calc. Dist. ")
pBAR.render(0, NJobs)
iResult = 0
if not Parallel:
for ii in range(NCPU):
workerlist[ii].run() # just run until all work is completed
while iResult < NJobs:
DicoResult = None
if result_queue.qsize() != 0:
try:
DicoResult = result_queue.get()
except:
pass
if DicoResult == None:
time.sleep(0.5)
continue
if DicoResult["Success"]:
iResult += 1
NDone = iResult
pBAR.render(NDone, NJobs)
iIsland = DicoResult["iIsland"]
Result = NpShared.GiveArray("%sDistances_%6.6i" %
(self.IdSharedMem, iIsland))
self.dx[iIsland] = Result[0]
self.dy[iIsland] = Result[1]
self.D[iIsland] = Result[2]
NpShared.DelAll("%sDistances_%6.6i" %
(self.IdSharedMem, iIsland))
if Parallel:
for ii in range(NCPU):
workerlist[ii].shutdown()
workerlist[ii].terminate()
workerlist[ii].join()