def run(self, expRef, butler): """ sets up the pool and scatters the processing of the individual CCDs, in processCcd, apparently all nodes (master and slaves) run this method, but I don't really get that on the gather, we just check that any of the sources ran """ pool = Pool(self._DefaultName) pool.cacheClear() pool.storeSet(butler=butler) dataIdList = dict([(ccdRef.get("ccdExposureId"), ccdRef.dataId) for ccdRef in expRef.subItems("ccd") if ccdRef.datasetExists("raw")]) dataIdList = collections.OrderedDict(sorted(dataIdList.items())) # Scatter: process CCDs independently outList = pool.map(self.process, dataIdList.values()) numGood = sum(1 for s in outList if s == 0) if numGood == 0: self.log.warn("All CCDs in exposure failed") return
def run(self, expRef, butler): """ sets up the pool and scatters the processing of the individual CCDs, in processCcd, apparently all nodes (master and slaves) run this method, but I don't really get that on the gather, we just check that any of the sources ran """ pool = Pool(self._DefaultName) pool.cacheClear() pool.storeSet(butler=butler) dataIdList = dict([(ccdRef.get("ccdExposureId"), ccdRef.dataId) for ccdRef in expRef.subItems("ccd") if ccdRef.datasetExists("raw")]) dataIdList = collections.OrderedDict(sorted(dataIdList.items())) # Scatter: process CCDs independently outList = pool.map(self.process, dataIdList.values()) numGood = sum(1 for s in outList if s==0) if numGood == 0: self.log.warn("All CCDs in exposure failed") return
def run(self, patchRefList, butler): """ Set up the pool and scatters the processing of the individual Patches. """ for patchRef in patchRefList: if patchRef: butler = patchRef.getButler() break else: raise RuntimeError("No valid patch") pool = Pool(self._DefaultName) pool.cacheClear() pool.storeSet(butler=butler) patchRefList = [ patchRef for patchRef in patchRefList if patchRef.datasetExists(self.config.coaddName + "Coadd_calexp") and patchRef.datasetExists(self.config.coaddName + "Coadd_det") ] dataIdList = [patchRef.dataId for patchRef in patchRefList] # Group by patch patches = {} tract = None for patchRef in patchRefList: dataId = patchRef.dataId if tract is None: tract = dataId['tract'] else: assert tract == dataId['tract'] patch = dataId["patch"] if patch not in patches: patches[patch] = [] patches[patch].append(dataId) # Scatter: process CCDs independently outList = pool.map(self.process, dataIdList) numGood = sum(1 for s in outList if s == 0) if numGood == 0: self.log.warn("Failed on ALL Patches!") return
def run(self, patchRefList, butler): """ Set up the pool and scatters the processing of the individual Patches. """ for patchRef in patchRefList: if patchRef: butler = patchRef.getButler() break else: raise RuntimeError("No valid patch") pool = Pool(self._DefaultName) pool.cacheClear() pool.storeSet(butler=butler) patchRefList = [patchRef for patchRef in patchRefList if patchRef.datasetExists(self.config.coaddName + "Coadd_calexp") and patchRef.datasetExists(self.config.coaddName + "Coadd_det")] dataIdList = [patchRef.dataId for patchRef in patchRefList] # Group by patch patches = {} tract = None for patchRef in patchRefList: dataId = patchRef.dataId if tract is None: tract = dataId['tract'] else: assert tract == dataId['tract'] patch = dataId["patch"] if patch not in patches: patches[patch] = [] patches[patch].append(dataId) # Scatter: process CCDs independently outList = pool.map(self.process, dataIdList) numGood = sum(1 for s in outList if s == 0) if numGood == 0: self.log.warn("Failed on ALL Patches!") return