Esempio n. 1
0
    def run(self, expRef, butler):
        """
        sets up the pool and scatters the processing of the individual CCDs,

        in processCcd, apparently all nodes (master and slaves) run this
        method, but I don't really get that

        on the gather, we just check that any of the sources ran
        """

        pool = Pool(self._DefaultName)
        pool.cacheClear()
        pool.storeSet(butler=butler)

        dataIdList = dict([(ccdRef.get("ccdExposureId"), ccdRef.dataId)
                           for ccdRef in expRef.subItems("ccd")
                           if ccdRef.datasetExists("raw")])
        dataIdList = collections.OrderedDict(sorted(dataIdList.items()))

        # Scatter: process CCDs independently
        outList = pool.map(self.process, dataIdList.values())
        numGood = sum(1 for s in outList if s == 0)
        if numGood == 0:
            self.log.warn("All CCDs in exposure failed")
            return
Esempio n. 2
0
    def run(self, expRef, butler):
        """
        sets up the pool and scatters the processing of the individual CCDs,

        in processCcd, apparently all nodes (master and slaves) run this
        method, but I don't really get that

        on the gather, we just check that any of the sources ran
        """

        pool = Pool(self._DefaultName)
        pool.cacheClear()
        pool.storeSet(butler=butler)

        dataIdList = dict([(ccdRef.get("ccdExposureId"), ccdRef.dataId)
                           for ccdRef in expRef.subItems("ccd") if
                           ccdRef.datasetExists("raw")])
        dataIdList = collections.OrderedDict(sorted(dataIdList.items()))

        # Scatter: process CCDs independently
        outList = pool.map(self.process, dataIdList.values())
        numGood = sum(1 for s in outList if s==0)
        if numGood == 0:
            self.log.warn("All CCDs in exposure failed")
            return
Esempio n. 3
0
    def run(self, patchRefList, butler):
        """
        Set up the pool and scatters the processing of the individual Patches.

        """
        for patchRef in patchRefList:
            if patchRef:
                butler = patchRef.getButler()
                break
        else:
            raise RuntimeError("No valid patch")

        pool = Pool(self._DefaultName)
        pool.cacheClear()
        pool.storeSet(butler=butler)

        patchRefList = [
            patchRef for patchRef in patchRefList
            if patchRef.datasetExists(self.config.coaddName + "Coadd_calexp")
            and patchRef.datasetExists(self.config.coaddName + "Coadd_det")
        ]
        dataIdList = [patchRef.dataId for patchRef in patchRefList]

        # Group by patch
        patches = {}
        tract = None
        for patchRef in patchRefList:
            dataId = patchRef.dataId
            if tract is None:
                tract = dataId['tract']
            else:
                assert tract == dataId['tract']

            patch = dataId["patch"]
            if patch not in patches:
                patches[patch] = []
            patches[patch].append(dataId)

        # Scatter: process CCDs independently
        outList = pool.map(self.process, dataIdList)

        numGood = sum(1 for s in outList if s == 0)
        if numGood == 0:
            self.log.warn("Failed on ALL Patches!")
            return
Esempio n. 4
0
    def run(self, patchRefList, butler):
        """
        Set up the pool and scatters the processing of the individual Patches.

        """
        for patchRef in patchRefList:
            if patchRef:
                butler = patchRef.getButler()
                break
        else:
            raise RuntimeError("No valid patch")

        pool = Pool(self._DefaultName)
        pool.cacheClear()
        pool.storeSet(butler=butler)

        patchRefList = [patchRef for patchRef in patchRefList if
                        patchRef.datasetExists(self.config.coaddName +
                                               "Coadd_calexp") and
                        patchRef.datasetExists(self.config.coaddName +
                                               "Coadd_det")]
        dataIdList = [patchRef.dataId for patchRef in patchRefList]

        # Group by patch
        patches = {}
        tract = None
        for patchRef in patchRefList:
            dataId = patchRef.dataId
            if tract is None:
                tract = dataId['tract']
            else:
                assert tract == dataId['tract']

            patch = dataId["patch"]
            if patch not in patches:
                patches[patch] = []
            patches[patch].append(dataId)

        # Scatter: process CCDs independently
        outList = pool.map(self.process, dataIdList)

        numGood = sum(1 for s in outList if s == 0)
        if numGood == 0:
            self.log.warn("Failed on ALL Patches!")
            return