def testVerbOpt(self):
cl = optparse.OptionParser()
run.addVerbosityOption(cl, "P")
self.assert_(cl.has_option("--log-verbosity"),
"Failed to set verbosity option")
self.assert_(cl.has_option("-P"),
"Failed to set verbosity option as -P")
(opts, args) = cl.parse_args(["testrun.py"])
self.assert_(opts.verbosity is None,
"unexpected default for verbosity")
for name in "silent quiet info trace verb1 verb2 verb3 debug".split():
self.assert_(run.msglev.has_key(name),
"Message level name not defined: " + name)
(opts, args) = cl.parse_args("testrun.py -P".split() + [name])
self.assert_(
run.verbosity2threshold(
opts.verbosity) == -1 * run.msglev[name],
"failed to translate verbosity level: " + opts.verbosity)
self.assert_(
run.verbosity2threshold(3) == -3,
"failed to translate integer verbosity level: 3")
self.assert_(
run.verbosity2threshold('-4') == 4,
"failed to translate string verbosity level: '-4'")
self.assertRaises(run.UsageError, run.verbosity2threshold, "goob")
self.assertRaises(run.UsageError, run.verbosity2threshold, 43.0)
def testVerbOpt(self):
cl = optparse.OptionParser()
run.addVerbosityOption(cl, "P")
self.assert_(cl.has_option("--log-verbosity"),
"Failed to set verbosity option")
self.assert_(cl.has_option("-P"),
"Failed to set verbosity option as -P")
(opts, args) = cl.parse_args(["testrun.py"])
self.assert_(opts.verbosity is None, "unexpected default for verbosity")
for name in "silent quiet info trace verb1 verb2 verb3 debug".split():
self.assert_(run.msglev.has_key(name),
"Message level name not defined: " + name)
(opts, args) = cl.parse_args("testrun.py -P".split() + [name])
self.assert_(
run.verbosity2threshold(opts.verbosity) == -1 * run.msglev[name],
"failed to translate verbosity level: " + opts.verbosity)
self.assert_(run.verbosity2threshold(3) == -3,
"failed to translate integer verbosity level: 3")
self.assert_(run.verbosity2threshold('-4') == 4,
"failed to translate string verbosity level: '-4'")
self.assertRaises(run.UsageError, run.verbosity2threshold, "goob")
self.assertRaises(run.UsageError, run.verbosity2threshold, 43.0)
def main():
"""execute the watchLogs script"""
try:
(cl.opts, cl.args) = cl.parse_args()
Log.getDefaultLog().setThreshold(
run.verbosity2threshold(cl.opts.verbosity, 0))
if cl.opts.inclhosts:
cl.opts.inclhosts = cl.opts.inclhosts.split(',')
if cl.opts.exclhosts:
cl.opts.exclhosts = cl.opts.exclhosts.split(',')
hosts = cl.opts.inclhosts
if not hosts:
hosts = cl.opts.exclhosts
if cl.opts.minimport and cl.opts.maximport and \
cl.opts.minimport > cl.opts.maximport:
raise run.UsageError(
"-m value (%i) should be less than -M value (%i)" %
(cl.opts.minimport, cl.opts.maximport))
watchLogs(cl.args[0], cl.args[1:], cl.opts.sleep,
cl.opts.slice, hosts, not bool(cl.opts.inclhosts),
cl.opts.minimport, cl.opts.maximport)
except run.UsageError, e:
print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e)
sys.exit(1)
def main():
"""execute the testEventLogger script"""
try:
(cl.opts, cl.args) = cl.parse_args()
Log.getDefaultLog().setThreshold(
run.verbosity2threshold(cl.opts.verbosity, 0))
props = {}
if cl.opts.stage:
props["stageId"] = cl.opts.stage
if cl.opts.pipeline:
props["pipeline"] = cl.opts.pipeline
broker = None
if len(cl.args) > 0:
broker = cl.args[0]
input = None
if cl.opts.stdin:
input = sys.stdin
testEventLogger(broker, cl.opts.runid, cl.opts.slice, props, input,
cl.opts.logname, cl.opts.logtopic)
except run.UsageError, e:
print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e)
sys.exit(1)
def main(cmdline):
"""
run the script with the given command line
@param cmdline an OptionParser instance with command-line options defined
"""
cl = cmdline
(cl.opts, cl.args) = cl.parse_args()
pexLog.Log.getDefaultLog().setThreshold( \
run.verbosity2threshold(cl.opts.verb, 0))
if len(sys.argv) < 3:
raise run.UsageError("Missing arguments")
inputDirectoryList = cl.args[0]
datatypePolicy = pexPolicy.Policy.createPolicy(cl.args[1])
expTime = EXP_TIME
slewTime = SLEW_TIME
if len(cl.args) > 2:
expTime = float(cl.args[2])
if len(cl.args) > 3:
slewTime = float(cl.args[3])
metadataPolicy = None
if cl.opts.mdpolicy is not None:
metadataPolicy = pexPolicy.Policy.createPolicy(metadataPolicy)
EventFromInputFileList(inputDirectoryList, datatypePolicy, expTime,
slewTime, cl.opts.maxvisits, cl.opts.topic,
cl.opts.broker, metadataPolicy)
def main():
"""execute the testEventLogger script"""
try:
(cl.opts, cl.args) = cl.parse_args()
Log.getDefaultLog().setThreshold(
run.verbosity2threshold(cl.opts.verbosity, 0))
props = {}
if cl.opts.stage:
props["stageId"] = cl.opts.stage
if cl.opts.pipeline:
props["pipeline"] = cl.opts.pipeline
broker = None
if len(cl.args) > 0:
broker = cl.args[0]
input = None
if cl.opts.stdin:
input = sys.stdin
testEventLogger(broker, cl.opts.runid, cl.opts.slice, props,
input, cl.opts.logname, cl.opts.logtopic)
except run.UsageError, e:
print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e)
sys.exit(1)
def main(cmdline):
"""
run the script with the given command line
@param cmdline an OptionParser instance with command-line options defined
"""
cl = cmdline
(cl.opts, cl.args) = cl.parse_args()
pexLog.Log.getDefaultLog().setThreshold( \
run.verbosity2threshold(cl.opts.verb, 0))
mdPolicyFileName = cl.opts.mdpolicy
if mdPolicyFileName is None:
mpf = pexPolicy.DefaultPolicyFile("ctrl_mospipe",
"mosEventMetadataPolicy.paf",
"pipeline")
metadataPolicy = pexPolicy.Policy.createPolicy(mpf,
mpf.getRepositoryPath())
else:
metadataPolicy = pexPolicy.Policy.createPolicy(mdPolicyFileName)
dataPolicy = pexPolicy.Policy.createPolicy(cl.args[1])
if not EventFromInputfile(cl.args[0], dataPolicy, metadataPolicy,
cl.opts.topic, cl.opts.broker):
# EventFromInputfile will print error message
sys.exit(3)
def main():
"""parse the input arguments and execute the pipeline
"""
(cl.opts, cl.args) = cl.parse_args()
logthresh = run.verbosity2threshold(cl.opts.verbosity)
if(len(cl.args) < 2):
print >> sys.stderr, \
"%s: missing required argument(s)." % cl.get_prog_name()
print cl.get_usage()
sys.exit(1)
pipelinePolicyName = cl.args[0]
runId = cl.args[1]
logger.log(Log.INFO, "pipelinePolicyName " + pipelinePolicyName)
logger.log(Log.INFO, "runId " + runId)
if (logthresh == None):
logger.log(Log.INFO, "logthresh is None")
else:
logger.log(Log.INFO, "logthresh " + str(logthresh))
if (cl.opts.name == None):
logger.log(Log.INFO, "cl.opts.name is None")
else:
logger.log(Log.INFO, "name " + cl.opts.name)
if (cl.opts.workerid == None):
logger.log(Log.INFO, "cl.opts.workerid is None")
else:
logger.log(Log.INFO, "workerid " + cl.opts.workerid)
runPipeline(pipelinePolicyName, runId, logthresh, cl.opts.name, cl.opts.logdir, cl.opts.workerid)
def main(cmdline):
"""
run the script with the given command line
@param cmdline an OptionParser instance with command-line options defined
"""
cl = cmdline
(cl.opts, cl.args) = cl.parse_args()
pexLog.Log.getDefaultLog().setThreshold( \
run.verbosity2threshold(cl.opts.verb, 0))
mdPolicyFileName = cl.opts.mdpolicy
if mdPolicyFileName is None:
mpf = pexPolicy.DefaultPolicyFile("ctrl_mospipe",
"mosEventMetadataPolicy.paf",
"pipeline")
metadataPolicy = pexPolicy.Policy.createPolicy(mpf,
mpf.getRepositoryPath())
else:
metadataPolicy = pexPolicy.Policy.createPolicy(mdPolicyFileName)
dataPolicy = pexPolicy.Policy.createPolicy(cl.args[1])
if not EventFromInputfile(cl.args[0], dataPolicy, metadataPolicy,
cl.opts.topic, cl.opts.broker):
# EventFromInputfile will print error message
sys.exit(3)
def main():
"""
run the job office
"""
(cl.opts, cl.args) = cl.parse_args()
if len(cl.args) == 0:
fail("Missing policy file")
if not os.path.exists(cl.args[0]):
fail("%s: policy file not found" % cl.args[0])
if not os.path.exists(cl.opts.rootdir):
fail("%s: root directory not found" % cl.opts.rootdir)
if not cl.opts.runid:
logger.log(Log.WARN, "No RunID given (via -r)")
defpolf = DefaultPolicyFile("ctrl_sched", "JobOffice_dict.paf", "policies")
policy = Policy.createPolicy(cl.args[0])
policy.mergeDefaults(Policy.createPolicy(defpolf,
defpolf.getRepositoryPath()))
name = policy.getString("name")
# set job office root directory
if not os.path.isabs(cl.opts.rootdir):
cl.opts.rootdir = os.path.abspath(cl.opts.rootdir)
persistdir = os.path.join(cl.opts.rootdir, name)
if policy.exists("persist.dir"):
persistdir = policy.get("persist.dir") % \
{"schedroot": cl.opts.rootdir, "name": name }
# create the logger(s)
logfile = cl.opts.logfile
if not logfile:
logfile = os.path.join(persistdir, "joboffice.log")
if not os.path.exists(logfile):
if not os.path.exists(os.path.dirname(logfile)):
os.makedirs(os.path.dirname(logfile))
if not cl.opts.asdaemon or cl.opts.toscreen:
ofclogger = DualLog(logfile, Log.DEBUG, Log.DEBUG, False)
# logging bug workaround
ofclogger.setScreenVerbose(False)
else:
ofclogger = Log()
ofclogger.addDestination(logfile)
ofclogger.setThreshold(run.verbosity2threshold(cl.opts.logverb, 0))
ofclogger.log(-2,"office threshold: %i" % ofclogger.getThreshold())
try:
# create the JobOffice instance
office = createJobOffice(cl.opts.rootdir, policy, ofclogger,
cl.opts.runid, cl.opts.brokerhost,
cl.opts.brokerport)
except Exception, ex:
logger.log(Log.FATAL, "Failed to create job office: " + str(ex))
raise
sys.exit(1)
def main():
"""execute the showEvents script"""
try:
(cl.opts, cl.args) = cl.parse_args()
Log.getDefaultLog().setThreshold(
run.verbosity2threshold(cl.opts.verbosity, 0))
showEvents(cl.args[0], cl.args[1:], cl.opts.sleep)
except run.UsageError, e:
print >> sys.stderr, "%s: %s" % (cl.get_prog_name(), e)
sys.exit(1)
def main():
"""parse the input arguments and execute the pipeline
"""
(cl.opts, cl.args) = cl.parse_args()
logthresh = run.verbosity2threshold(cl.opts.verbosity)
if (len(cl.args) < 2):
print >> sys.stderr, \
"%s: missing required argument(s)." % cl.get_prog_name()
print cl.get_usage()
sys.exit(1)
pipelinePolicyName = cl.args[0]
runId = cl.args[1]
runPipeline(pipelinePolicyName, runId, logthresh, cl.opts.name)
def main():
"""parse the input arguments and execute the pipeline
"""
(cl.opts, cl.args) = cl.parse_args()
logthresh = run.verbosity2threshold(cl.opts.verbosity)
if(len(cl.args) < 2):
print >> sys.stderr, \
"%s: missing required argument(s)." % cl.get_prog_name()
print cl.get_usage()
sys.exit(1)
pipelinePolicyName = cl.args[0]
runId = cl.args[1]
runPipeline(pipelinePolicyName, runId, logthresh, cl.opts.name)
def main(cmdline):
import sets
"""
run the script with the given command line
@param cmdline an OptionParser instance with command-line options defined
"""
cl = cmdline
(cl.opts, cl.args) = cl.parse_args()
pexLog.Log.getDefaultLog().setThreshold( \
run.verbosity2threshold(cl.opts.verb, 0))
if len(cl.args) < 3:
raise run.UsageError("Missing arguments")
subsets = [x.upper() for x in cl.args[0].split()]
# Make sure thet the subsets make sense.
if(not sets.Set(subsets).issubset(sets.Set(SUBSETS))):
raise run.UsageError('subsets must be a combination of %s'
% ', '.join(SUBSETS) )
inputDirectoryList = cl.args[0]
datatypePolicy = pexPolicy.Policy.createPolicy(cl.args[1])
expTime = EXP_TIME
slewTime = SLEW_TIME
if len(cl.args) > 2:
expTime = float(cl.args[2])
if len(cl.args) > 3:
slewTime = float(cl.args[3])
metadataPolicy = None
if cl.opts.mdpolicy is not None:
metadataPolicy = pexPolicy.Policy.createPolicy(metadataPolicy)
EventFromInputSubsets(subsets, datatypePolicy, expTime,
slewTime, cl.opts.maxvisits, cl.opts.topic,
cl.opts.broker, metadataPolicy)
def main(cmdline):
import sets
"""
run the script with the given command line
@param cmdline an OptionParser instance with command-line options defined
"""
cl = cmdline
(cl.opts, cl.args) = cl.parse_args()
pexLog.Log.getDefaultLog().setThreshold( \
run.verbosity2threshold(cl.opts.verb, 0))
if len(cl.args) < 3:
raise run.UsageError("Missing arguments")
subsets = [x.upper() for x in cl.args[0].split()]
# Make sure thet the subsets make sense.
if (not sets.Set(subsets).issubset(sets.Set(SUBSETS))):
raise run.UsageError('subsets must be a combination of %s' %
', '.join(SUBSETS))
inputDirectoryList = cl.args[0]
datatypePolicy = pexPolicy.Policy.createPolicy(cl.args[1])
expTime = EXP_TIME
slewTime = SLEW_TIME
if len(cl.args) > 2:
expTime = float(cl.args[2])
if len(cl.args) > 3:
slewTime = float(cl.args[3])
metadataPolicy = None
if cl.opts.mdpolicy is not None:
metadataPolicy = pexPolicy.Policy.createPolicy(metadataPolicy)
EventFromInputSubsets(subsets, datatypePolicy, expTime, slewTime,
cl.opts.maxvisits, cl.opts.topic, cl.opts.broker,
metadataPolicy)
def main():
"""parse the input arguments and execute the pipeline
"""
(cl.opts, cl.args) = cl.parse_args()
logthresh = run.verbosity2threshold(cl.opts.verbosity)
if (len(cl.args) < 2):
print >> sys.stderr, \
"%s: missing required argument(s)." % cl.get_prog_name()
print cl.get_usage()
sys.exit(1)
pipelinePolicyName = cl.args[0]
runId = cl.args[1]
logger.log(Log.INFO, "pipelinePolicyName " + pipelinePolicyName)
logger.log(Log.INFO, "runId " + runId)
if (logthresh == None):
logger.log(Log.INFO, "logthresh is None")
else:
logger.log(Log.INFO, "logthresh " + str(logthresh))
if (cl.opts.name == None):
logger.log(Log.INFO, "cl.opts.name is None")
else:
logger.log(Log.INFO, "name " + cl.opts.name)
if (cl.opts.workerid == None):
logger.log(Log.INFO, "cl.opts.workerid is None")
else:
logger.log(Log.INFO, "workerid " + cl.opts.workerid)
runPipeline(pipelinePolicyName, runId, logthresh, cl.opts.name,
cl.opts.logdir, cl.opts.workerid)
def setVerbosity(verbosity):
logger.setThreshold(run.verbosity2threshold(verbosity, -1))
import lsst.ctrl.events as ctrlEvents
from lsst.pex.harness import run
from lsst.ctrl.mospipe.MetadataStages import transformMetadata, validateMetadata
usage = """Usage: %prog [-dvqs] [-V lev] [-b host] [-t topic] FITSfile policyfile"""
desc = """Send an incoming visit event to instruct the alert production to process
a given FITS file. The data for the event is extracted from the FITS file.
The given policy file controls the transformation of the FITS metadata into
the event metadata where different input data collections (CFHT, simulated,
etc.) will require different policies. See the
$CTRL_DC3PIPE/pipeline/datatypePolicy directory for samples.
"""
logger = pexLog.Log(pexLog.Log.getDefaultLog(), "mospipe.eventFromFitsfile")
exposureCount = 0
VERB3 = run.verbosity2threshold("verb3", logger.INFO - 3)
def defineCmdLine(usage=usage, description=desc):
cl = optparse.OptionParser(usage=usage, description=description)
run.addAllVerbosityOptions(cl, "V", "verb")
cl.add_option("-b",
"--broker",
action="store",
dest="broker",
default="newfield.as.arizona.edu",
help="event broker host")
cl.add_option("-t",
"--topic",
action="store",
dest="topic",
def setVerbosity(verbosity):
logger.setThreshold(run.verbosity2threshold(verbosity, -1))
def waitForReady(policy, runid, eventrcvr, logverb, logger):
"""
attempt to wait until all pipelines are configured and running before
sending event data.
"""
# This implimentation tries to wait until all pipelines have reached
# the point of waiting for their first event
# determine whether the pipeline verbosity is enough to get the
# particular "ready" signals we will be looking for
prodthresh = None
if logverb is not None:
prodthresh = run.verbosity2threshold(logverb)
if prodthresh is None and policy.exists("logThreshold"):
prodthresh = policy.get("logThreshold")
timeout = setuptime
pldescs = policy.get("pipelines")
names = pldescs.policyNames(True)
pipelines = []
for pl in names:
plpol = pldescs.getPolicy(pl)
if not plpol.getBool("launch"):
continue
if prodthresh is None or prodthresh > -1:
config = plpol.getPolicy("configuration")
if config.exists("execute"):
config = config.getPolicy("execute")
if config.exists("logThreshold") and \
config.getInt("logThreshold") > -1:
logger.log(Log.WARN, "%s pipeline's logging not verbose enough to track its readiness" % pl)
continue
pipelines.append(pl)
logger.log(Log.DEBUG,
"Waiting for the %s pipeline to be ready..." % pl)
if "IPSD" not in pipelines:
timeout = shortsetuptime # seconds
if len(pipelines) > 0:
logger.log(Log.INFO,
"Waiting for pipelines to setup (this can take a while)...")
tick = time.time()
while len(pipelines) > 0:
waittime = 1000 * (timeout - int(round(time.time()-tick)))
if waittime > 0:
# waitprops = eventrcvr.receive(waittime)
waitprops = eventrcvr.matchingReceive("LOG", waitLogName,
waittime)
else:
waitprops = None
if waitprops is None:
LogRec(logger, Log.WARN) \
<< "Have yet to hear back from the following pipelines: " +\
", ".join(pipelines) \
<< "Proceeding to send visit events" << LogRec.endr
break
if waitprops.getString("STATUS", "") == "start" and \
waitprops.getString("runId", "") == runid:
pipename = waitprops.getString("pipeline", "unknown")
if pipename in pipelines:
pipelines.remove(pipename)
# pipelines.pop(0)
logger.log(Log.DEBUG, "%s is ready" % pipename)
else:
LogRec(logger, Log.WARN) \
<< "Unable to detect when pipelines are ready" \
<< "Proceeding to send visit events in %d seconds" % \
shortsetuptime \
<< LogRec.endr
time.sleep(shortsetuptime)
return
import lsst.ctrl.events as ctrlEvents
from lsst.pex.harness import run
from lsst.ctrl.mospipe.MetadataStages import transformMetadata, validateMetadata
usage = """Usage: %prog [-dvqs] [-V lev] [-b host] [-t topic] FITSfile policyfile"""
desc = """Send an incoming visit event to instruct the alert production to process
a given FITS file. The data for the event is extracted from the FITS file.
The given policy file controls the transformation of the FITS metadata into
the event metadata where different input data collections (CFHT, simulated,
etc.) will require different policies. See the
$CTRL_DC3PIPE/pipeline/datatypePolicy directory for samples.
"""
logger = pexLog.Log(pexLog.Log.getDefaultLog(), "mospipe.eventFromFitsfile")
exposureCount = 0
VERB3 = run.verbosity2threshold("verb3", logger.INFO-3)
def defineCmdLine(usage=usage, description=desc):
cl = optparse.OptionParser(usage=usage, description=description)
run.addAllVerbosityOptions(cl, "V", "verb")
cl.add_option("-b", "--broker", action="store", dest="broker",
default="newfield.as.arizona.edu", help="event broker host")
cl.add_option("-t", "--topic", action="store", dest="topic",
default="triggerImageprocEvent", help="event topic name")
cl.add_option("-M", "--metadata-policy", action="store", dest="mdpolicy",
default=None,
help="policy file defining the event metadata types")
return cl
def main(cmdline):
"""
def waitForReady(policy, runid, eventrcvr, logverb, logger):
"""
attempt to wait until all pipelines are configured and running before
sending event data.
"""
# This implimentation tries to wait until all pipelines have reached
# the point of waiting for their first event
# determine whether the pipeline verbosity is enough to get the
# particular "ready" signals we will be looking for
prodthresh = None
if logverb is not None:
prodthresh = run.verbosity2threshold(logverb)
if prodthresh is None and policy.exists("logThreshold"):
prodthresh = policy.get("logThreshold")
timeout = setuptime
pldescs = policy.get("pipelines")
names = pldescs.policyNames(True)
pipelines = []
for pl in names:
plpol = pldescs.getPolicy(pl)
if not plpol.getBool("launch"):
continue
if prodthresh is None or prodthresh > -1:
config = plpol.getPolicy("configuration")
if config.exists("execute"):
config = config.getPolicy("execute")
if config.exists("logThreshold") and \
config.getInt("logThreshold") > -1:
logger.log(
Log.WARN,
"%s pipeline's logging not verbose enough to track its readiness"
% pl)
continue
pipelines.append(pl)
logger.log(Log.DEBUG,
"Waiting for the %s pipeline to be ready..." % pl)
if "IPSD" not in pipelines:
timeout = shortsetuptime # seconds
if len(pipelines) > 0:
logger.log(
Log.INFO,
"Waiting for pipelines to setup (this can take a while)...")
tick = time.time()
while len(pipelines) > 0:
waittime = 1000 * (timeout - int(round(time.time() - tick)))
if waittime > 0:
# waitprops = eventrcvr.receive(waittime)
waitprops = eventrcvr.matchingReceive("LOG", waitLogName,
waittime)
else:
waitprops = None
if waitprops is None:
LogRec(logger, Log.WARN) \
<< "Have yet to hear back from the following pipelines: " +\
", ".join(pipelines) \
<< "Proceeding to send visit events" << LogRec.endr
break
if waitprops.getString("STATUS", "") == "start" and \
waitprops.getString("runId", "") == runid:
pipename = waitprops.getString("pipeline", "unknown")
if pipename in pipelines:
pipelines.remove(pipename)
# pipelines.pop(0)
logger.log(Log.DEBUG, "%s is ready" % pipename)
else:
LogRec(logger, Log.WARN) \
<< "Unable to detect when pipelines are ready" \
<< "Proceeding to send visit events in %d seconds" % \
shortsetuptime \
<< LogRec.endr
time.sleep(shortsetuptime)
return