def Interpol(self): APP.startWorkers() if "TEC" in self.DicoFile0["SmoothMode"]: TECArray = NpShared.ToShared("%sTECArray" % IdSharedMem, self.DicoFile0["SolsTEC"]) CPhaseArray = NpShared.ToShared("%sCPhaseArray" % IdSharedMem, self.DicoFile0["SolsCPhase"]) nt, nd, na = TECArray.shape iJob = 0 for it in range(nt): APP.runJob("InterpolTECTime_%d" % iJob, self.InterpolTECTime, args=(it, )) #,serial=True) iJob += 1 workers_res = APP.awaitJobResults("InterpolTECTime*", progress="Interpol TEC") iJob = 0 for it in range(nt): APP.runJob("InterpolAmpTime_%d" % iJob, self.InterpolAmpTime, args=(it, )) #,serial=True) iJob += 1 workers_res = APP.awaitJobResults("InterpolAmpTime*", progress="Interpol Amp") # APP.terminate() APP.shutdown() Multiprocessing.cleanupShm()
def killWorkers(self): print("Killing workers", file=log) APP.terminate() APP.shutdown() Multiprocessing.cleanupShm()
default=None) opt.add_option_group(group) options, arguments = opt.parse_args() return options if __name__ == "__main__": options = read_options() print >> log, "Clear shared memory" if options.ID is not None: NpShared.DelAll(options.ID) else: NpShared.DelAll() Multiprocessing.cleanupStaleShm() Multiprocessing.cleanupShm() ll = glob.glob("/dev/shm/sem.*") print >> log, "Clear Semaphores" # remove semaphores we don't have access to ll = filter(lambda x: os.access(x, os.W_OK), ll) ListSemaphores = [".".join(l.split(".")[1::]) for l in ll] _pyGridderSmear.pySetSemaphores(ListSemaphores) _pyGridderSmear.pyDeleteSemaphore(ListSemaphores) print >> log, "Clear shared dictionaries" ll = glob.glob("/dev/shm/shared_dict:*") ll = filter(lambda x: os.access(x, os.W_OK), ll)
def main(OP=None, messages=[]): if OP is None: OP = MyPickle.Load(SaveFile) print("Using settings from %s, then command line."%SaveFile) DicoConfig = OP.DicoConfig ImageName = DicoConfig["Output"]["Name"] if not ImageName: raise Exceptions.UserInputError("--Output-Name not specified, can't continue.") if not DicoConfig["Data"]["MS"]: raise Exceptions.UserInputError("--Data-MS not specified, can't continue.") # create directory if it exists dirname = os.path.dirname(ImageName) if not os.path.exists(dirname) and not dirname == "": os.mkdir(dirname) # setup logging logger.logToFile(ImageName + ".log", append=DicoConfig["Log"]["Append"]) global log log = logger.getLogger("DDFacet") # disable colors and progressbars if requested ModColor.silent = SkyModel.Other.ModColor.silent = \ progressbar.ProgressBar.silent = \ DicoConfig["Log"]["Boring"] if messages: if not DicoConfig["Log"]["Boring"]: #os.system('clear') logo.print_logo() for msg in messages: print(msg, file=log) print("Checking system configuration:", file=log) # check for SHM size ram_size = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') shm_stats = os.statvfs('/dev/shm') shm_size = shm_stats.f_bsize * shm_stats.f_blocks shm_relsize = shm_size / float(ram_size) shm_avail = shm_stats.f_bsize * shm_stats.f_bavail / float(ram_size) if shm_relsize < 0.6: print(ModColor.Str("""WARNING: max shared memory size is only {:.0%} of total RAM size. This can cause problems for large imaging jobs. A setting of 90% is recommended for DDFacet and killMS. If your processes keep failing with SIGBUS or "bus error" messages, it is most likely for this reason. You can change the memory size by running $ sudo mount -o remount,size=90% /dev/shm To make the change permanent, edit /etc/defaults/tmps, and add a line saying "SHM_SIZE=90%". """.format(shm_relsize)), file=log) else: print(" Max shared memory size is {:.0%} of total RAM size; {:.0%} currently available".format(shm_relsize, shm_avail), file=log) try: output = subprocess.check_output(["/sbin/sysctl", "vm.max_map_count"],universal_newlines=True) except Exception: print(ModColor.Str("""WARNING: /sbin/sysctl vm.max_map_count failed. Unable to check this setting."""), file=log) max_map_count = None else: max_map_count = int(output.strip().rsplit(" ", 1)[-1]) if max_map_count is not None: if max_map_count < 500000: print(ModColor.Str("""WARNING: sysctl vm.max_map_count = {}. This may be too little for large DDFacet and killMS jobs. If you get strange "file exists" errors on /dev/shm, them try to bribe, beg or threaten your friendly local sysadmin into setting vm.max_map_count=1000000 in /etc/sysctl.conf. """.format(max_map_count)), file=log) else: print(" sysctl vm.max_map_count = {}".format(max_map_count), file=log) # check for memory lock limits import resource msoft, mhard = resource.getrlimit(resource.RLIMIT_MEMLOCK) if msoft >=0 or mhard >=0: print(ModColor.Str("""WARNING: your system has a limit on memory locks configured. This may possibly slow down DDFacet performance. You can try removing the limit by running $ ulimit -l unlimited If this gives an "operation not permitted" error, you can try to bribe, beg or threaten your friendly local sysadmin into doing # echo "* - memlock unlimited" >> /etc/security/limits.conf """), file=log) if DicoConfig["Debug"]["Pdb"] == "always": print("--Debug-Pdb=always: unexpected errors will be dropped into pdb", file=log) Exceptions.enable_pdb_on_error(ModColor.Str("DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n" + "(This is because you're running with --Debug-Pdb set to 'always'.)")) elif DicoConfig["Debug"]["Pdb"] == "auto" and not DicoConfig["Log"]["Boring"]: print("--Debug-Pdb=auto and not --Log-Boring: unexpected errors will be dropped into pdb", file=log) Exceptions.enable_pdb_on_error(ModColor.Str("DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n" + "(This is because you're running with --Debug-Pdb set to 'auto' and --Log-Boring is off.)")) # print current options OP.Print(dest=log) # enable memory logging logger.enableMemoryLogging(DicoConfig["Log"]["Memory"]) # get rid of old shm arrays from previous runs Multiprocessing.cleanupStaleShm() # initialize random seed from config if set, or else from system time if DicoConfig["Misc"]["RandomSeed"] is not None: DicoConfig["Misc"]["RandomSeed"]=int(DicoConfig["Misc"]["RandomSeed"]) print("random seed=%d (explicit)" % DicoConfig["Misc"]["RandomSeed"], file=log) else: DicoConfig["Misc"]["RandomSeed"] = int(time.time()) print("random seed=%d (automatic)" % DicoConfig["Misc"]["RandomSeed"], file=log) np.random.seed(DicoConfig["Misc"]["RandomSeed"]) # init NCPU for different bits of parallelism ncpu = int(DicoConfig["Parallel"]["NCPU"] or psutil.cpu_count()) DicoConfig["Parallel"]["NCPU"]=ncpu _pyArrays.pySetOMPNumThreads(ncpu) NpParallel.NCPU_global = ModFFTW.NCPU_global = ncpu numexpr.set_num_threads(ncpu) print("using up to %d CPUs for parallelism" % ncpu, file=log) # write parset OP.ToParset("%s.parset"%ImageName) Mode = DicoConfig["Output"]["Mode"] # init semaphores, as they're needed for weight calculation too ClassFacetMachine.ClassFacetMachine.setup_semaphores(DicoConfig) # data machine initialized for all cases except PSF-only mode # psf machine initialized for all cases except Predict-only mode Imager = ClassDeconvMachine.ClassImagerDeconv(GD=DicoConfig, BaseName=ImageName, predict_only=(Mode == "Predict" or Mode == "Subtract"), data=(Mode != "PSF"), psf=(Mode != "Predict" and Mode != "Dirty" and Mode != "Subtract"), readcol=(Mode != "Predict" and Mode != "PSF"), deconvolve=("Clean" in Mode)) Imager.Init() # Imager.testDegrid() # stop if "Predict" in Mode or "Subtract" in Mode: Imager.GivePredict() if "Clean" in Mode: Imager.main() elif "Dirty" in Mode: sparsify = DicoConfig["Comp"]["Sparsification"] if sparsify and isinstance(sparsify, list): sparsify = sparsify[0] Imager.GiveDirty(psf="PSF" in Mode, sparsify=sparsify) elif "PSF" in Mode: sparsify = DicoConfig["Comp"]["Sparsification"] if sparsify and isinstance(sparsify, list): sparsify = sparsify[0] Imager.MakePSF(sparsify=sparsify) elif "RestoreAndShift" == Mode: Imager.RestoreAndShift()
def BuildSmearMappingParallel(self, DATA, GridChanMapping): print >> log, "Build decorrelation mapping ..." na = self.MS.na l = self.radiusRad dPhi = np.sqrt(6. * (1. - self.Decorr)) NChan = self.MS.ChanFreq.size self.BlocksRowsList = [] InfoSmearMapping = {} InfoSmearMapping["freqs"] = self.MS.ChanFreq InfoSmearMapping["dfreqs"] = self.MS.dFreq InfoSmearMapping["dPhi"] = dPhi InfoSmearMapping["l"] = l BlocksRowsList = [] joblist = [(a0, a1) for a0 in xrange(na) for a1 in xrange(na) if a0 != a1] WorkerMapName = Multiprocessing.getShmURL("SmearWorker.%d") results = Multiprocessing.runjobs( joblist, title="Smear mapping", target=_smearmapping_worker, kwargs=dict(DATA=DATA, InfoSmearMapping=InfoSmearMapping, WorkerMapName=WorkerMapName, GridChanMapping=GridChanMapping)) # process worker results # for each map (each array returned from worker), BlockSizes[MapName] will # contain a list of BlocksSizesBL entries returned from that worker RowsBlockSizes = {} NTotBlocks = 0 NTotRows = 0 worker_maps = {} for DicoResult in results: if not DicoResult["Empty"]: MapName = DicoResult["MapName"] map = worker_maps.get(MapName) if map is None: map = worker_maps[MapName] = NpShared.GiveArray(MapName) bl = DicoResult["bl"] rowslice = DicoResult["Slice"] bsz = np.array(DicoResult["BlocksSizesBL"]) RowsBlockSizes[bl] = map[rowslice], bsz NTotBlocks += DicoResult["NBlocksTotBL"] NTotRows += bsz.sum() # output mapping has 2 words for the total size, plus 2*NTotBlocks header, plus NTotRows blocklists OutputMapping = np.zeros((2 + 2 * NTotBlocks + NTotRows, ), np.int32) # just in case NTotBlocks is over 2^31... # (don't want to use np.int32 for the whole mapping as that just wastes space, we may assume # that we have substantially fewer rows, so int32 is perfectly good as a row index etc.) OutputMapping[0] = NTotBlocks OutputMapping[1] = NTotBlocks >> 32 BlockListSizes = OutputMapping[2:2 + NTotBlocks] BlockLists = OutputMapping[2 + NTotBlocks:] iii = 0 jjj = 0 # now go through each per-baseline mapping, sorted by baseline for _, (BlocksRowsListBL, BlocksSizesBL) in sorted(RowsBlockSizes.items()): #print>>log, " Worker: %i"%(IdWorker) BlockLists[iii:iii + BlocksRowsListBL.size] = BlocksRowsListBL[:] iii += BlocksRowsListBL.size # print "IdWorker,AppendId",IdWorker,AppendId,BlocksSizesBL # MM=np.concatenate((MM,BlocksSizesBL)) BlockListSizes[jjj:jjj + BlocksSizesBL.size] = BlocksSizesBL[:] jjj += BlocksSizesBL.size for MapName in worker_maps.iterkeys(): NpShared.DelArray(MapName) #print>>log, " Put in shared mem" NVis = np.where(DATA["A0"] != DATA["A1"])[0].size * NChan #print>>log, " Number of blocks: %i"%NTotBlocks #print>>log, " Number of 4-Visibilities: %i"%NVis fact = (100. * (NVis - NTotBlocks) / float(NVis)) # self.UnPackMapping() # print FinalMapping return OutputMapping, fact
def killWorkers(self): print >> log, "Killing workers" APP.terminate() APP.shutdown() Multiprocessing.cleanupShm()
def main(OP=None, messages=[]): if OP is None: OP = MyPickle.Load(SaveFile) print "Using settings from %s, then command line." % SaveFile DicoConfig = OP.DicoConfig ImageName = DicoConfig["Output"]["Name"] if not ImageName: raise Exceptions.UserInputError( "--Output-Name not specified, can't continue.") if not DicoConfig["Data"]["MS"]: raise Exceptions.UserInputError( "--Data-MS not specified, can't continue.") # create directory if it exists dirname = os.path.dirname(ImageName) if not os.path.exists(dirname) and not dirname == "": os.mkdir(dirname) # setup logging MyLogger.logToFile(ImageName + ".log", append=DicoConfig["Log"]["Append"]) global log log = MyLogger.getLogger("DDFacet") # disable colors and progressbars if requested ModColor.silent = SkyModel.Other.ModColor.silent = \ progressbar.ProgressBar.silent = \ DicoConfig["Log"]["Boring"] if messages: if not DicoConfig["Log"]["Boring"]: #os.system('clear') logo.print_logo() for msg in messages: print >> log, msg if DicoConfig["Debug"]["Pdb"] == "always": print >> log, "--Debug-Pdb=always: unexpected errors will be dropped into pdb" Exceptions.enable_pdb_on_error( ModColor.Str( "DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n" + "(This is because you're running with --Debug-Pdb set to 'always'.)" )) elif DicoConfig["Debug"][ "Pdb"] == "auto" and not DicoConfig["Log"]["Boring"]: print >> log, "--Debug-Pdb=auto and not --Log-Boring: unexpected errors will be dropped into pdb" Exceptions.enable_pdb_on_error( ModColor.Str( "DDFacet has encountered an unexpected error. Dropping you into pdb for a post-mortem.\n" + "(This is because you're running with --Debug-Pdb set to 'auto' and --Log-Boring is off.)" )) # print current options OP.Print(dest=log) # enable memory logging MyLogger.enableMemoryLogging(DicoConfig["Log"]["Memory"]) # get rid of old shm arrays from previous runs Multiprocessing.cleanupStaleShm() # initialize random seed from config if set, or else from system time if DicoConfig["Misc"]["RandomSeed"] is not None: print >> log, "random seed=%d (explicit)" % DicoConfig["Misc"][ "RandomSeed"] else: DicoConfig["Misc"]["RandomSeed"] = int(time.time()) print >> log, "random seed=%d (automatic)" % DicoConfig["Misc"][ "RandomSeed"] np.random.seed(DicoConfig["Misc"]["RandomSeed"]) # init NCPU for different bits of parallelism ncpu = DicoConfig["Parallel"]["NCPU"] or psutil.cpu_count() DicoConfig["Parallel"]["NCPU"] = ncpu _pyArrays.pySetOMPNumThreads(ncpu) NpParallel.NCPU_global = ModFFTW.NCPU_global = ncpu numexpr.set_num_threads(ncpu) print >> log, "using up to %d CPUs for parallelism" % ncpu # write parset OP.ToParset("%s.parset" % ImageName) Mode = DicoConfig["Output"]["Mode"] # init semaphores, as they're needed for weight calculation too ClassFacetMachine.ClassFacetMachine.setup_semaphores(DicoConfig) # data machine initialized for all cases except PSF-only mode # psf machine initialized for all cases except Predict-only mode Imager = ClassDeconvMachine.ClassImagerDeconv( GD=DicoConfig, BaseName=ImageName, predict_only=(Mode == "Predict" or Mode == "Subtract"), data=(Mode != "PSF"), psf=(Mode != "Predict" and Mode != "Dirty" and Mode != "Subtract"), readcol=(Mode != "Predict" and Mode != "PSF"), deconvolve=("Clean" in Mode)) Imager.Init() # Imager.testDegrid() # stop if "Predict" in Mode or "Subtract" in Mode: Imager.GivePredict() if "Clean" in Mode: Imager.main() elif "Dirty" in Mode: sparsify = DicoConfig["Comp"]["Sparsification"] if sparsify and isinstance(sparsify, list): sparsify = sparsify[0] Imager.GiveDirty(psf="PSF" in Mode, sparsify=sparsify) elif "PSF" in Mode: sparsify = DicoConfig["Comp"]["Sparsification"] if sparsify and isinstance(sparsify, list): sparsify = sparsify[0] Imager.MakePSF(sparsify=sparsify) elif "RestoreAndShift" == Mode: Imager.RestoreAndShift()
def InterpolParallel(self): Sols0 = self.Sols nt, nch, na, nd, _, _ = Sols0.G.shape log.print(" #Times: %i" % nt) log.print(" #Channels: %i" % nch) log.print(" #Antennas: %i" % na) log.print(" #Directions: %i" % nd) # APP.terminate() # APP.shutdown() # Multiprocessing.cleanupShm() APP.startWorkers() iJob = 0 # for iAnt in [49]:#range(na): # for iDir in [0]:#range(nd): if "TEC" in self.InterpMode: #APP.runJob("FitThisTEC_%d"%iJob, self.FitThisTEC, args=(208,)); iJob+=1 self.TECArray = NpShared.ToShared( "%sTECArray" % IdSharedMem, np.zeros((nt, nd, na), np.float32)) self.CPhaseArray = NpShared.ToShared( "%sCPhaseArray" % IdSharedMem, np.zeros((nt, nd, na), np.float32)) for it in range(nt): # for iDir in range(nd): APP.runJob("FitThisTEC_%d" % iJob, self.FitThisTEC, args=(it, )) #,serial=True) iJob += 1 workers_res = APP.awaitJobResults("FitThisTEC*", progress="Fit TEC") if "Amp" in self.InterpMode: for iAnt in range(na): for iDir in range(nd): APP.runJob("FitThisAmp_%d" % iJob, self.FitThisAmp, args=(iAnt, iDir)) #,serial=True) iJob += 1 workers_res = APP.awaitJobResults("FitThisAmp*", progress="Smooth Amp") if "PolyAmp" in self.InterpMode: for iDir in range(nd): APP.runJob("FitThisPolyAmp_%d" % iJob, self.FitThisPolyAmp, args=(iDir, )) iJob += 1 workers_res = APP.awaitJobResults("FitThisPolyAmp*", progress="Smooth Amp") if "Clip" in self.InterpMode: for iDir in range(nd): APP.runJob("ClipThisDir_%d" % iJob, self.ClipThisDir, args=(iDir, ), serial=True) iJob += 1 workers_res = APP.awaitJobResults("ClipThisDir*", progress="Clip Amp") #APP.terminate() APP.shutdown() Multiprocessing.cleanupShm()