class FakeOutput(harnessStage.ParallelProcessing): """ this stage simulates work by sleeping """ def setup(self): if not self.log: self.log = Log.getDefaultLog() self.mylog = Log(self.log, "output") self.outputDatasetsKey = \ self.policy.getString("inputKeys.outputDatasets") self.possibleDatasetsKey = \ self.policy.getString("inputKeys.possibleDatasets") def process(self, clipboard): expected = clipboard.get(self.possibleDatasetsKey) outputds = clipboard.get(self.outputDatasetsKey) # this implementation will pretend to write out all of the # expected datasets. It will also put each dataset written # out into the outputDatasets list. if expected: for ds in expected: self.mylog.log(Log.INFO, "Writing out " + ds.toString()) outputds.append(ds) else: self.log.log(Log.WARN, "No expected datasets on clipboard") clipboard.put(self.outputDatasetsKey, outputds)
def checkExitByVisit(self): log = Log(self.log, "checkExitByVisit") if((self._stop.isSet()) and (self.exitLevel == 4)): log.log(Log.INFO, "Pipeline stop is set at exitLevel of 4") log.log(Log.INFO, "Exit here at the end of the Visit") self.forceShutdown = 1
def _link(self, clipboard): """ Link one or more sourcePaths (from policy) to destPaths after formatting each with additionalData derived from the clipboard and stage information. """ if not self.policy.exists('Links'): mylog = Log(Log.defaultLog(), "pex.harness.SymLinkStage.SymLinkStageSerial") mylog.log(Log.WARN, "No Links found") return additionalData = lsst.pex.harness.Utils.createAdditionalData(self, \ self.policy, clipboard) linkPolicyList = self.policy.getPolicyArray('Links') for linkPolicy in linkPolicyList: sourcePath = lsst.daf.persistence.LogicalLocation( linkPolicy.getString('sourcePath'), additionalData).locString() destPath = lsst.daf.persistence.LogicalLocation( linkPolicy.getString('destPath'), additionalData).locString() lsst.pex.logging.Trace("pex.harness.SymLinkStage.SymLinkStageSerial", 3, \ "linking %s to %s" % (sourcePath, destPath)) parentDir = os.path.dirname(destPath) if parentDir and not os.path.exists(parentDir): os.makedirs(parentDir) try: os.symlink(sourcePath, destPath) except OSError, e: # ignore "file exists" but re-raise anything else if e.errno != 17: raise e
def main(): """execute the watchLogs script""" try: (cl.opts, cl.args) = cl.parse_args() Log.getDefaultLog().setThreshold( run.verbosity2threshold(cl.opts.verbosity, 0)) if cl.opts.inclhosts: cl.opts.inclhosts = cl.opts.inclhosts.split(',') if cl.opts.exclhosts: cl.opts.exclhosts = cl.opts.exclhosts.split(',') hosts = cl.opts.inclhosts if not hosts: hosts = cl.opts.exclhosts if cl.opts.minimport and cl.opts.maximport and \ cl.opts.minimport > cl.opts.maximport: raise run.UsageError( "-m value (%i) should be less than -M value (%i)" % (cl.opts.minimport, cl.opts.maximport)) watchLogs(cl.args[0], cl.args[1:], cl.opts.sleep, cl.opts.slice, hosts, not bool(cl.opts.inclhosts), cl.opts.minimport, cl.opts.maximport) except run.UsageError, e: print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e) sys.exit(1)
def main(): """execute the testEventLogger script""" try: (cl.opts, cl.args) = cl.parse_args() Log.getDefaultLog().setThreshold( run.verbosity2threshold(cl.opts.verbosity, 0)) props = {} if cl.opts.stage: props["stageId"] = cl.opts.stage if cl.opts.pipeline: props["pipeline"] = cl.opts.pipeline broker = None if len(cl.args) > 0: broker = cl.args[0] input = None if cl.opts.stdin: input = sys.stdin testEventLogger(broker, cl.opts.runid, cl.opts.slice, props, input, cl.opts.logname, cl.opts.logtopic) except run.UsageError, e: print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e) sys.exit(1)
def main(): "execute the launchMospipe script" logger = Log(Log.getDefaultLog(), "launchMos") try: (cl.opts, cl.args) = cl.parse_args() Log.getDefaultLog().setThreshold(-10 * cl.opts.verbosity) if cl.opts.pipeverb is None: cl.opts.pipeverb = "trace" t = filter(lambda x: x.startswith(cl.opts.datatype.lower()), datatypes.keys()) if len(t) > 1: raise ValueError("Ambiguous data type name: " + cl.opts.datatype) if len(t) == 0: raise ValueError("Unrecognized data type name: "+ cl.opts.datatype) cl.opts.datatype = datatypes[t[0]] colls = [] # parse the collection names if cl.opts.colls is not None: colls = cl.opts.colls.split(',') launchMos(cl.args[0], cl.args[1], cl.args[2:], colls, cl.opts, logger) except run.UsageError, e: print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e) sys.exit(1)
def showAllLogProperties(self, show): """control whether log properties are displayed to the screen. These include, for example, the DATE (and time) of the message. @param show if true, show all the properties when a log message is printed. If false, don't show them. """ Log.getDefaultLog().setShowAll(show)
class IsrCcdSdqaStageParallel(harnessStage.ParallelProcessing): """ Description: Policy Dictionary: Clipboard Input: ClipboardOutput: """ def setup(self): self.log = Log(self.log, "CcdSdqaStage -- Parallel") policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "IsrCcdSdqaStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath()) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy) def process(self, clipboard): """ """ self.log.log(Log.INFO, "Calculate SDQA metrics based on the assembled ccd.") #grab exposure from clipboard exposure = clipboard.get(self.policy.getString("inputKeys.ccdExposure")) ipIsr.calculateSdqaCcdRatings(exposure) #output products clipboard.put(self.policy.get("outputKeys.sdqaCcdExposure"), exposure)
class IsrCcdSdqaStageParallel(harnessStage.ParallelProcessing): """ Description: Policy Dictionary: Clipboard Input: ClipboardOutput: """ def setup(self): self.log = Log(self.log, "CcdSdqaStage -- Parallel") policyFile = pexPolicy.DefaultPolicyFile( "ip_pipeline", "IsrCcdSdqaStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy( policyFile, policyFile.getRepositoryPath()) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy) def process(self, clipboard): """ """ self.log.log(Log.INFO, "Calculate SDQA metrics based on the assembled ccd.") #grab exposure from clipboard exposure = clipboard.get( self.policy.getString("inputKeys.ccdExposure")) ipIsr.calculateSdqaCcdRatings(exposure) #output products clipboard.put(self.policy.get("outputKeys.sdqaCcdExposure"), exposure)
def checkExitBySyncPoint(self): log = Log(self.log, "checkExitBySyncPoint") if((self._stop.isSet()) and (self.exitLevel == 2)): log.log(Log.INFO, "Pipeline stop is set at exitLevel of 2") log.log(Log.INFO, "Exit here at a Synchronization point") self.forceShutdown = 1
class IsrVarianceStageParallel(harnessStage.ParallelProcessing): """ Description: Policy Dictionary: Clipboard Input: ClipboardOutput: """ def setup(self): self.log = Log(self.log, "IsrVarianceStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "IsrVarianceStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath()) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy) def process(self, clipboard): """ """ self.log.log(Log.INFO, "Calculating variance from image counts.") #grab exposure from clipboard exposure = clipboard.get(self.policy.getString("inputKeys.exposure")) ipIsr.updateVariance(exposure) #output products clipboard.put(self.policy.get("outputKeys.varianceAddedExposure"), exposure)
def shutdown(self): """ Shutdown the Slice execution """ shutlog = Log(self.log, "shutdown", Log.INFO) shutlog.log(Log.INFO, "Shutting down Slice") self.cppSlice.shutdown()
def main(): "execute the launchMospipe script" logger = Log(Log.getDefaultLog(), "launchMos") try: (cl.opts, cl.args) = cl.parse_args() Log.getDefaultLog().setThreshold(-10 * cl.opts.verbosity) if cl.opts.pipeverb is None: cl.opts.pipeverb = "trace" t = filter(lambda x: x.startswith(cl.opts.datatype.lower()), datatypes.keys()) if len(t) > 1: raise ValueError("Ambiguous data type name: " + cl.opts.datatype) if len(t) == 0: raise ValueError("Unrecognized data type name: " + cl.opts.datatype) cl.opts.datatype = datatypes[t[0]] colls = [] # parse the collection names if cl.opts.colls is not None: colls = cl.opts.colls.split(',') launchMos(cl.args[0], cl.args[1], cl.args[2:], colls, cl.opts, logger) except run.UsageError, e: print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e) sys.exit(1)
class IsrDarkStageParallel(harnessStage.ParallelProcessing): """ Description: Policy Dictionary: Clipboard Input: ClipboardOutput: """ def setup(self): self.log = Log(self.log, "IsrDarkStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "IsrDarkStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath()) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy) def process(self, clipboard): """ """ self.log.log(Log.INFO, "Doing dark subtraction.") #grab exposure and dark from clipboard darkexposure = clipboard.get(self.policy.getString("inputKeys.darkexposure")) exposure = clipboard.get(self.policy.getString("inputKeys.exposure")) darkscaling = darkexposure.getCalib().getExptime() expscaling = exposure.getCalib().getExptime() ipIsr.darkCorrection(exposure, darkexposure, float(expscaling), float(darkscaling)) #output products clipboard.put(self.policy.get("outputKeys.darkSubtractedExposure"), exposure)
class IsrVarianceStageParallel(harnessStage.ParallelProcessing): """ Description: Policy Dictionary: Clipboard Input: ClipboardOutput: """ def setup(self): self.log = Log(self.log, "IsrVarianceStage - parallel") policyFile = pexPolicy.DefaultPolicyFile( "ip_pipeline", "IsrVarianceStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy( policyFile, policyFile.getRepositoryPath()) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy) def process(self, clipboard): """ """ self.log.log(Log.INFO, "Calculating variance from image counts.") #grab exposure from clipboard exposure = clipboard.get(self.policy.getString("inputKeys.exposure")) ipIsr.updateVariance(exposure) #output products clipboard.put(self.policy.get("outputKeys.varianceAddedExposure"), exposure)
def __init__(self, policy, log=None, stageId=-1, eventBroker=None, sysdata=None): """ initialize this stage with the policy that defines the stage and some contextual system data. Applications normally do not directly call this constructor. Instead they either construct a Stage subclass or create a Stage instance using makeStage() or makeStageFromPolicy(). @param policy the policy that will configure the SerialProcessing and ParallelProcessing @param log the log object the stage instance should should use. If not provided, a default will be used. @param eventBroker the name of the host where the event broker is running. If not provided, an eventBroker will not be available to the stage. @param sysdata a dictionary of data describing the execution context. The stage uses this information to set some of its internal data. See StageProcessing documentation for datails. The name provided in the policy will override the name in this dictionary """ if sysdata is None: sysdata = {} self.sysdata = sysdata self.stagePolicy = policy self.eventBroker = eventBroker if log is None: log = Log(Log.getDefaultLog(), "stage") self.log = log
def process(self, clipboard): """ Execute the needed processing code for this Stage """ print 'Python apps.SyncSetupStageParallel process : stageId %i' % self.stageId print 'Python apps.SyncSetupStageParallel process : _rank %i' % self.rank root = Log.getDefaultLog() log = Log(root, "lsst.pexhexamples.apps.SyncSetupStageParallel.process") lr = LogRec(log, Log.INFO) lr << "Retrieving Clipboard" lr << LogRec.endr propertySet = dafBase.PropertySet() propertySet.setInt("sliceRank", self.rank) propertySet.setString("Level", "Debug") clipboard.put("rankKey", propertySet) clipboard.setShared("rankKey", True) lr = LogRec(log, Log.INFO) lr << "Posted data to be Shared on Clipboard" lr << LogRec.endr
class IsrFringeStageParallel(harnessStage.ParallelProcessing): """ Description: Policy Dictionary: Clipboard Input: ClipboardOutput: """ def setup(self): self.log = Log(self.log, "IsrFringeStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "IsrFringeStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath()) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy) def process(self, clipboard): """ """ self.log.log(Log.INFO, "Doing Fringe correction.") #grab exposure from clipboard exposure = clipboard.get(self.policy.getString("inputKeys.exposure")) fringeexposure = clipboard.get(self.policy.getString("inputKeys.fringeexposure")) #fringeCorrection is not implemented self.log.log(Log.INFO, "ipIsr.fringeCorrection is not implemented -- \n\tOriginal exposure is returned") #ipIsr.fringeCorrection(exposure, fringeexposure) #output products clipboard.put(self.policy.get("outputKeys.fringeCorrectedExposure"), exposure)
def shutdown(self): """ Shutdown the Slice execution """ shutlog = Log(self.log, "shutdown", Log.INFO); shutlog.log(Log.INFO, "Shutting down Slice") self.cppSlice.shutdown()
def checkExitByStage(self): log = Log(self.log, "checkExitByStage") if((self._stop.isSet()) and (self.exitLevel == 3)): log.log(Log.INFO, "Pipeline stop is set at exitLevel of 3") log.log(Log.INFO, "Exit here at the end of the Stage") self.forceShutdown = 1
class _StopThread(threading.Thread): def __init__(self, joboffice, stopTopic, runId, brokerHost, brokerPort=None, waittime=60): threading.Thread.__init__(self, name=joboffice.getName()+".stop") self.setDaemon(True) self.jo = joboffice self.timeout = waittime self.log = Log(self.jo.log, "stop") selector = "" if runId: selector = "RUNID='%s'" % runId if brokerPort: self.rcvr = EventReceiver(brokerHost, brokerPort, stopTopic, selector) else: self.rcvr = EventReceiver(brokerHost, stopTopic, selector) def run(self): while True: event = self.rcvr.receiveEvent(self.timeout) if event: self.log.log(Log.INFO-1, "received stop event; " + "shutting down JobOffice thread") self.jo.stop() if self.jo.halt: return
class IsrBiasStageParallel(harnessStage.ParallelProcessing): """ Description: Policy Dictionary: Clipboard Input: ClipboardOutput: """ def setup(self): self.log = Log(self.log, "IsrBiasStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "IsrBiasStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath()) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy) def process(self, clipboard): """ """ self.log.log(Log.INFO, "Doing bias subtraction.") #grab exposure and bias from clipboard biasexposure = clipboard.get(self.policy.getString("inputKeys.biasexposure")) exposure = clipboard.get(self.policy.getString("inputKeys.exposure")) ipIsr.biasCorrection(exposure, biasexposure) #output products clipboard.put(self.policy.get("outputKeys.biasSubtractedExposure"), exposure)
def setup(self): if not self.log: self.log = Log.getDefaultLog() self.mylog = Log(self.log, "output") self.outputDatasetsKey = \ self.policy.getString("inputKeys.outputDatasets") self.possibleDatasetsKey = \ self.policy.getString("inputKeys.possibleDatasets")
def run(): Log.getDefaultLog() memId0 = dafBase.Citizen_getNextMemId() main() # check for memory leaks if dafBase.Citizen_census(0, memId0) != 0: print dafBase.Citizen_census(0, memId0), 'Objects leaked:' print dafBase.Citizen_census(dafBase.cout, memId0)
def shutdown(self): """ Shutdown the Slice execution """ shutlog = Log(self.log, "shutdown", Log.INFO); pid = os.getpid() shutlog.log(Log.INFO, "Shutting down Slice: pid " + str(pid)) os.kill(pid, signal.SIGKILL)
def shutdown(self): """ Shutdown the Slice execution """ shutlog = Log(self.log, "shutdown", Log.INFO) pid = os.getpid() shutlog.log(Log.INFO, "Shutting down Slice: pid " + str(pid)) os.kill(pid, signal.SIGKILL)
class DiffImStageParallel(harnessStage.ParallelProcessing): """ Description: This stage wraps image subtraction Policy Dictionary: lsst/ip/pipeline/policy/DiffImStageDictionary.paf Clipboard Input: - Template Exposure : to be convolved - Science Exposure : to be matched to Clipboard Output: - Difference Exposure : resulting difference image - Psf Matching Kernel : the spatial model of the Psf matching Kernel - Background Function : differential background model """ def setup(self): self.log = Log(self.log, "DiffImStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "DiffImStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy( policyFile, policyFile.getRepositoryPath(), # repos True) # validate if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy.getDictionary()) self.diffImPolicy = ipDiffim.makeDefaultPolicy() def process(self, clipboard): """ Run image subtraction """ self.log.log(Log.INFO, "Running image subtraction") # grab exposures from clipboard templateExposure = clipboard.get( self.policy.getString("inputKeys.templateExposureKey")) scienceExposure = clipboard.get( self.policy.getString("inputKeys.scienceExposureKey")) # run image subtraction psfMatch = ipDiffim.ImagePsfMatch(self.diffImPolicy) results = psfMatch.subtractExposures(templateExposure, scienceExposure) # parse results differenceExposure, spatialKernel, backgroundModel, kernelCellSet = results #output products clipboard.put(self.policy.get("outputKeys.differenceExposureKey"), differenceExposure) clipboard.put(self.policy.get("outputKeys.psfMatchingKernelKey"), spatialKernel) clipboard.put(self.policy.get("outputKeys.backgroundFunctionKey"), backgroundModel)
def preprocess(self, clipboard): """ Processing code for this Stage to be executed by the main Pipeline prior to invoking Slice process """ log = Log(self.log, "lsst.pexhexamples.pipeline.SampleStageSerial.preprocess") log.log(Log.INFO, 'Executing SampleStageSerial preprocess')
def setup(self): if not self.log: self.log = Log.getDefaultLog() self.mylog = Log(self.log, "fakeProcess") self.jobIdentityItem = \ self.policy.getString("inputKeys.jobIdentity") self.sleeptime = self.policy.getInt("sleep") self.visitCount = 0 self.failOnVisitN = self.policy.getInt("failIteration")
def __init__(self, stageId=-1, policy=None): """ Standard Stage initializer. """ lsst.pex.harness.Stage.Stage.__init__(self, stageId, policy) self.mopsLog = Log(Log.getDefaultLog(), 'mops.stage') if isinstance(self.mopsLog, log.ScreenLog): self.mopsLog.setScreenVerbose(True) return
def __init__(self, fullPath=None): if fullPath is None: pDir = os.environ["CAT_DIR"] if pDir is None: raise RuntimeError('CAT_DIR env var required') fullPath = os.path.join(pDir, 'policy/defaultProdCatPolicy.paf') self.policyObj = pexPolicy.Policy.createPolicy(fullPath) log = Log(Log.getDefaultLog(), "cat") log.log(Log.DEBUG, 'Reading policy from %s' % fullPath)
class SourceMeasurementStageParallel(harnessStage.ParallelProcessing): """ Description: This stage wraps the measurement of sources on an exposure. The exposures to measure on should be in the clipboard along with the FootprintSet(s) to measure on those exposures. The keys for the exposures, and the FootprintSet(s) can be specified in the policy file. If not specified, default keys will be used """ def setup(self): self.log = Log(self.log, "SourceMeasurementStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("meas_pipeline", "SourceMeasurementStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True) if self.policy is None: self.policy = defPolicy else: self.policy.mergeDefaults(defPolicy.getDictionary()) def process(self, clipboard): """ Measure sources in the worker process """ self.log.log(Log.INFO, "Measuring Sources in process") #this may raise exceptions try: measurePolicy, exposure, psf, positiveDetection, negativeDetection = \ self.getClipboardData(clipboard) except pexExcept.LsstException, e: self.log.log(Log.FATAL, str(e)) # # Need to do something smart about merging positive and negative # detection sets. # # For now, assume they are disjoint sets, so merge is trivial # footprintLists = [] if positiveDetection: self.log.log(Log.DEBUG, "Positive FootprintSet found") isNegative = False footprintLists.append([positiveDetection.getFootprints(), isNegative]) if negativeDetection: self.log.log(Log.DEBUG, "Negative FootprintSet found") isNegative = True footprintLists.append([negativeDetection.getFootprints(), isNegative]) sourceSet = srcMeas.sourceMeasurement(exposure, psf, footprintLists, measurePolicy) # place SourceSet on the clipboard sourceKey = self.policy.get("outputKeys.sources") clipboard.put(sourceKey, sourceSet) clipboard.put(sourceKey + "_persistable", afwDet.PersistableSourceVector(sourceSet))
def setup(self): self.log = Log(self.log, "simpleDiffImStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "SimpleDiffImStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy.getDictionary())
def setDebugVerbosity(self, verbLimit): """set the verbosity of the default log. This and setLogThreshold() are different APIs that affect the same underlying limit that controls how many messages get logged. @param verbLimit debug messages with a verbosity level larger than this will not be printed. If positive INFO, WARN, and FATAL messages will also be printed. """ Log.getDefaultLog().setThreshold(-1 * verbLimit)
def setLogThreshold(self, threshold): """set the importance threshold for the default log. This and setDebugVerbosity are different APIs that affect the same underlying limit that controls how many messages get logged. Normally one uses one of the predefined values--Log.DEBUG, Log.INFO, Log.WARN, and Log.FATAL--as input. @param threshold the minimum importance of the message that will get printed. """ Log.getDefaultLog().setThreshold(threshold)
def setDebugVerbosity(self, verbLimit): """set the verbosity of the default log. This and setLogThreshold() are different APIs that affect the same underlying limit that controls how many messages get logged. @param verbLimit debug messages with a verbosity level larger than this will not be printed. If positive INFO, WARN, and FATAL messages will also be printed. """ Log.getDefaultLog().setThreshold(-1*verbLimit)
def setup(self): self.log = Log(self.log, "CcdAssembly -- Parallel") policyFile = pexPolicy.DefaultPolicyFile( "ip_pipeline", "IsrCcdAssemblyStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy( policyFile, policyFile.getRepositoryPath()) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy)
def setup(self): self.log = Log(self.log, "SourceMeasurementStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("meas_pipeline", "SourceMeasurementStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True) if self.policy is None: self.policy = defPolicy else: self.policy.mergeDefaults(defPolicy.getDictionary())
def setLogThreshold(self, level): """ set the default message importance threshold to be used for recording messages. This will value be applied to the default root (system-wide) logger (or what it will be after logging is initialized) so that all software components are affected. """ if self.log is not None: Log.getDefaultLog().setThreshold(level) self.log.log(Log.INFO, "Upating Root Log Message Threshold to %i" % level) self.logthresh = level
def setup(self): self.log = Log(self.log, "BackgroundEstimationStage - parallel") policyFile = pexPolicy.DefaultPolicyFile( "meas_pipeline", "BackgroundEstimationStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy( policyFile, policyFile.getRepositoryPath(), True) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy.getDictionary())
def setup(self): self.log = Log(self.log, "ApertureCorrectionApplyStage - parallel") # aperture correction policy apCorrPolicyFile = pexPolicy.DefaultPolicyFile("meas_pipeline", "ApertureCorrectionApplyStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(apCorrPolicyFile, apCorrPolicyFile.getRepositoryPath(), True) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy.getDictionary())
def threadBarrier(self, iStage): """ Create an approximate barrier where all Slices intercommunicate with the Pipeline """ log = Log(self.log, "threadBarrier") self.checkExitBySyncPoint() # if((self._stop.isSet()) and (self.exitLevel == 2)): # log.log(Log.INFO, "Pipeline stop is set at exitLevel of 2; exit here at a synchronization point") # print "Pipeline stop is set at exitLevel of 2; exit here at a synchronization point" # os._exit() # sys.exit() # log.log(Log.INFO, "Pipeline Ever reach here ?? ") entryTime = time.time() log.log(Log.DEBUG, "Entry time %f" % (entryTime)) for i in range(self.nSlices): k = 2 * i loopEventA = self.loopEventList[k] loopEventB = self.loopEventList[k + 1] signalTime1 = time.time() log.log(Log.DEBUG, "Signal to Slice %d %f" % (i, signalTime1)) loopEventA.set() log.log(Log.DEBUG, "Wait for signal from Slice %d" % (i)) # Wait for the B event to be set by the Slice # Excute time sleep in between checks to free the GIL periodically useDelay = self.barrierDelay if (iStage == 1): useDelay = 0.1 if (iStage == 290): useDelay = 0.1 while (not (loopEventB.isSet())): time.sleep(useDelay) signalTime2 = time.time() log.log( Log.DEBUG, "Done waiting for signal from Slice %d %f" % (i, signalTime2)) if (loopEventB.isSet()): loopEventB.clear() self.checkExitBySyncPoint()
class CrRejectStageParallel(harnessStage.ParallelProcessing): """ Description: This stage wraps estimating and possibly subtracting cosmic rays from an exposure on the clipboard. Policy Dictionary: lsst/ip/pipeline/policy/CrRejectStageDictionary.paf Clipboard Input: - Calibrated science Exposure(s) (without background) - a PSF may be specified by policy attribute inputPsfKey. Alternatively, the stage's policy may request that a psf be constructed, by providing the psfPolicy attribute. ClipboardOutput: - Exposure with CRs removed. Key specified by policy attribute 'crSubtractedExposureKey' - nCR The number of CRs detected - PSF: the psf used to smooth the exposure before detection Key specified by policy attribute 'psfKey' """ def setup(self): self.log = Log(self.log, "CrRejectStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "CrRejectStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy.getDictionary()) self.crRejectPolicy = self.policy.get("crRejectPolicy") def process(self, clipboard): """ Detect CRs in the worker process """ self.log.log(Log.INFO, "Detecting CRs in process") #grab exposure from clipboard exposure = clipboard.get(self.policy.get("inputKeys.exposure")) defaultFwhm = self.policy.get('parameters.defaultFwhm') # in arcsec keepCRs = self.policy.get('parameters.keepCRs') crs = ipUtils.cosmicRays.findCosmicRays(exposure, self.crRejectPolicy, defaultFwhm, keepCRs) nCR = len(crs) #output products clipboard.put("nCR", nCR) clipboard.put(self.policy.get("outputKeys.exposure"), exposure)
def main(): """execute the showEvents script""" try: (cl.opts, cl.args) = cl.parse_args() Log.getDefaultLog().setThreshold( run.verbosity2threshold(cl.opts.verbosity, 0)) showEvents(cl.args[0], cl.args[1:], cl.opts.sleep) except run.UsageError, e: print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e) sys.exit(1)
class DiffImStageParallel(harnessStage.ParallelProcessing): """ Description: This stage wraps image subtraction Policy Dictionary: lsst/ip/pipeline/policy/DiffImStageDictionary.paf Clipboard Input: - Template Exposure : to be convolved - Science Exposure : to be matched to Clipboard Output: - Difference Exposure : resulting difference image - Psf Matching Kernel : the spatial model of the Psf matching Kernel - Background Function : differential background model """ def setup(self): self.log = Log(self.log, "DiffImStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("ip_pipeline", "DiffImStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), # repos True) # validate if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy.getDictionary()) self.diffImPolicy = ipDiffim.makeDefaultPolicy() def process(self, clipboard): """ Run image subtraction """ self.log.log(Log.INFO, "Running image subtraction") # grab exposures from clipboard templateExposure = clipboard.get(self.policy.getString("inputKeys.templateExposureKey")) scienceExposure = clipboard.get(self.policy.getString("inputKeys.scienceExposureKey")) # run image subtraction psfMatch = ipDiffim.ImagePsfMatch(self.diffImPolicy) results = psfMatch.subtractExposures(templateExposure, scienceExposure) # parse results differenceExposure, spatialKernel, backgroundModel, kernelCellSet = results #output products clipboard.put(self.policy.get("outputKeys.differenceExposureKey"), differenceExposure) clipboard.put(self.policy.get("outputKeys.psfMatchingKernelKey"), spatialKernel) clipboard.put(self.policy.get("outputKeys.backgroundFunctionKey"), backgroundModel)
def testEventLogger(broker, runid, sliceid, props, input=None, logname=None, logtopic="LSSTLogging"): """ test logging through a broker. This will send a single message to a the logger, and, if input is a list of string or and file input object, it will also send all input message in order. If broker is None, the log messages will not got to any event broker, only to the screen. @param broker the host running the event broker @param runid the runid to assume for the log (ignored if broker is None) @param sliceid the sliceid to assume for the log (ignored if broker is None) @param props a set of properties to attach to all messages @param input if not None, messages to send, either in the form of of a list of strings or a file object to read from @param logname the log name to send messages to @param logtopic the event topic to use (def: "LSSTLogging") """ if broker: thresh = Log.getDefaultLog().getThreshold() setEventLogger(broker, runid, sliceid, thresh <= Log.DEBUG, logtopic) Log.getDefaultLog().setThreshold(thresh) logger.log(VERB, "Created event logger") else: logger.log(VERB, "Messages only going to screen") logger.setShowAll(Log.getDefaultLog().getThreshold() <= Log.DEBUG) if logname is None: logname = "showEvents" uselog = Log(Log.getDefaultLog(), logname) for key in props.keys(): uselog.addPreamblePropertyString(key, props[key]) testLogger(uselog, input)
def populateClipboard(self, inputParamPropertySetPtr, iStage, eventTopic): """ Place the event payload onto the Clipboard """ log = Log(self.log, "populateClipboard"); log.log(Log.DEBUG,'Python Pipeline populateClipboard'); queue = self.queueList[iStage-1] clipboard = queue.element() # Slice does not disassemble the payload of the event. # It knows nothing of the contents. # It simply places the payload on the clipboard with key of the eventTopic clipboard.put(eventTopic, inputParamPropertySetPtr)
class BackgroundEstimationStageParallel(harnessStage.ParallelProcessing): """ Description: This stage wraps estimating and possibly subtracting the background from an exposure on the clipboard. Policy Dictionary: lsst/meas/pipeline/policy/BackgroundEstimationStageDictionary.paf Clipboard Input: - Calibrated science Exposure(s) (including background) ClipboardOutput: - background subtracted Exposure used in the detection. Key specified by policy attribute 'backgroundSubtractedExposure' - the measured background object itself. Key specified by policy attribute 'background' """ def setup(self): self.log = Log(self.log, "BackgroundEstimationStage - parallel") policyFile = pexPolicy.DefaultPolicyFile("meas_pipeline", "BackgroundEstimationStageDictionary.paf", "policy") defPolicy = pexPolicy.Policy.createPolicy(policyFile, policyFile.getRepositoryPath(), True) if self.policy is None: self.policy = pexPolicy.Policy() self.policy.mergeDefaults(defPolicy.getDictionary()) def process(self, clipboard): """ Detect sources in the worker process """ self.log.log(Log.INFO, "Subtracting background in process") #grab exposure from clipboard exposure = clipboard.get(self.policy.get("inputKeys.exposure")) #estimate and maybe subtract the background background, backgroundSubtractedExposure = sourceDetection.estimateBackground( exposure, self.policy.get("parameters.backgroundPolicy"), self.policy.get("parameters.subtractBackground")) #output products clipboard.put(self.policy.get("outputKeys.background"), background) if backgroundSubtractedExposure: clipboard.put(self.policy.get("outputKeys.backgroundSubtractedExposure"), backgroundSubtractedExposure)
def __init__(self, persistDir, logger=None, runId=None, fromSubclass=False): """ create the JobOffice @param persistDir the directory where the blackboard should be persisted @param logger the logger to use. If not given, the default will be used. @param runId the run ID to restrict our attention to. If not None, incoming events will be restricted to the given runId. """ self._checkAbstract(fromSubclass, "JobOffice") threading.Thread.__init__(self, name="JobOffice") self.setDaemon(False) self.bb = Blackboard(persistDir) self.esys = EventSystem.getDefaultEventSystem() self.runId = runId self.brokerHost = None self.brokerPort = None self.originatorId = self.esys.createOriginatorId() if logger is None: logger = Log.getDefaultLog() self.log = logger # override this in sub-class self.halt = False self.finalDatasetSent = False self.jobOfficeCompletedSent = False self.stopTopic = "JobOfficeStop" self.stopThread = None self.exc = None