def __init__(self,name,inptype,config,logger=None):
if logger is None:
self.m_log=rlogger.rotseLogger().getlog(name)
else:
self.m_log=logger
self.__inpType__=type(inptype)
self.name=name
self.config=config
self.__deviation = None
self.m_log.debug("initializing Monitoring alg {}".format(name))
def setup_pipeline(config):
"""
Given a configuration, this sets up a pipeline [pa,qa] and also returns a
conversion dictionary from the configuration dictionary so that Pipeline steps (PA) can
take them. This is required for runpipeline.
"""
rlog = rlogger.rotseLogger()
log = rlog.getlog()
if config is None:
return None
log.debug("Reading Configuration")
night = config["Night"]
telescope = config["Telescope"]
flavor = config["Flavor"]
program = config["Program"]
hbeat = HB.Heartbeat(log, config["Timeout"])
if config["Timeout"] > 600.0:
log.warning("Heartbeat timeout exceeding 200.0 seconds")
if "basePath" in config:
basePath = config["basePath"]
convdict = {}
images = None
pipeline = []
for step in config["Pipeline"]:
pa = getobject(step["PA"], log)
if len(pipeline) == 0:
if not pa.is_compatible(type(images)):
log.critical(
"Pipeline configuration is incorrect! check configuration {} {}"
.format(images, pa.is_compatible(images)))
sys.exit("Wrong pipeline configuration")
else:
if not pa.is_compatible(pipeline[-1][0].get_output_type()):
log.critical(
"Pipeline configuration is incorrect! check configuration")
log.critical(
"Can't connect input of {} to output of {}. Incompatible types"
.format(pa.name, pipeline[-1][0].name))
sys.exit("Wrong pipeline configuration")
qas = []
for q in step["QAs"]:
qa = getobject(q, log)
if not qa.is_compatible(pa.get_output_type()):
log.warning(
"QA {} can not be used for output of {}. Skipping expecting {} got {} {}"
.format(qa.name, pa.name, qa.__inpType__,
pa.get_output_type(),
qa.is_compatible(pa.get_output_type())))
else:
qas.append(qa)
pipeline.append([pa, qas])
return pipeline, convdict
def getobject(conf, log):
rlog = rlogger.rotseLogger("ROTSE-III", 20)
log = rlog.getlog()
log.debug("Running for {} {} {}".format(conf["ModuleName"],
conf["ClassName"], conf))
try:
mod = __import__(conf["ModuleName"], fromlist=[conf["ClassName"]])
klass = getattr(mod, conf["ClassName"])
if "Name" in conf.keys():
return klass(conf["Name"], conf)
else:
return klass(conf["ClassName"], conf)
except Exception as e:
log.error("Failed to import {} from {}. Error was '{}'".format(
conf["ClassName"], conf["ModuleName"], e))
return None
def rotse_main(args=None):
import os, sys
from rotseproc import rotse, rlogger, rotse_config
if args is None:
args = parse()
rlog = rlogger.rotseLogger(name="ROTSE-III", loglevel=args.loglvl)
log = rlog.getlog()
if args.config is not None:
if args.datadir:
datadir = args.datadir
else:
if 'ROTSE_DATA' not in os.environ:
log.critical(
"Must set $ROTSE_DATA environment variable or provide datadir"
)
sys.exit()
datadir = os.getenv('ROTSE_DATA')
if args.reduxdir:
reduxdir = args.reduxdir
else:
if 'ROTSE_REDUX' not in os.environ:
log.critical(
"Must set $ROTSE_REDUX environment variable or provide reduxdir"
)
sys.exit()
reduxdir = os.getenv('ROTSE_REDUX')
outdir = os.path.join(reduxdir, args.outdir)
tempdir = None
if args.tempdir:
tempdir = args.tempdir
else:
if 'ROTSE_TEMPLATE' in os.environ:
tempdir = os.getenv('ROTSE_TEMPLATE')
log.info(
"Running ROTSE-III pipeline using configuration file {}".format(
args.config))
if os.path.exists(args.config):
if "yaml" in args.config:
config = rotse_config.Config(args.config,
args.night,
args.telescope,
args.field,
args.ra,
args.dec,
datadir=datadir,
outdir=outdir,
tempdir=tempdir,
plots=args.plots)
configdict = config.expand_config()
else:
log.critical("Can't open configuration file {}".format(
args.config))
sys.exit("Can't open configuration file")
else:
sys.exit("File does not exist: {}".format(args.config))
else:
sys.exit(
"Must provide a valid configuration file. See rotseproc/config for an example"
)
pipeline, convdict = rotse.setup_pipeline(configdict)
res = rotse.runpipeline(pipeline, convdict, configdict)
log.info("ROTSE-III Pipeline completed")
"""
Monitoring algorithms for the ROTSE-III pipeline
"""
import os, sys
import numpy as np
import datetime
from astropy.io import fits
from rotseproc.io.qa import write_qa_file
from rotseproc.qa import qaplots
from rotseproc.qa.qas import check_QA_status, MonitoringAlg, QASeverity
from rotseproc import exceptions, rlogger
from astropy.time import Time
from rotseproc.qa import qalib
rlog = rlogger.rotseLogger("ROTSE-III",0)
log = rlog.getlog()
def get_inputs(*args,**kwargs):
"""
Get inputs required for each QA
"""
inputs={}
if "paname" in kwargs: inputs["paname"] = kwargs["paname"]
else: inputs["paname"] = None
if "flavor" in kwargs: inputs["flavor"] = kwargs["flavor"]
else: inputs["flavor"] = None
if "program" in kwargs: inputs["program"] = kwargs["program"]
def runpipeline(pl, convdict, conf):
"""
Runs the rotse pipeline as configured
Args:
pl: is a list of [pa,qas] where pa is a pipeline step and qas the corresponding
qas for that pa
convdict: converted dictionary, details in setup_pipeline method below for examples.
conf: a configured dictionary, read from the configuration yaml file.
e.g: conf=configdict=yaml.safe_load(open('configfile.yaml','rb'))
"""
rlog = rlogger.rotseLogger()
log = rlog.getlog()
hb = HB.Heartbeat(log, conf["Timeout"])
inp = None
paconf = conf["Pipeline"]
passqadict = None #- pass this dict to QAs downstream
schemaMerger = QAMerger(convdict)
QAresults = []
import numpy as np
qa = None
qas = [[], ['Count_Pixels'], [], [], [], [], []]
for s, step in enumerate(pl):
log.info("Starting to run step {}".format(paconf[s]["StepName"]))
pa = step[0]
pargs = mapkeywords(step[0].config["kwargs"], convdict)
schemaStep = schemaMerger.addPipelineStep(paconf[s]["StepName"])
try:
hb.start("Running {}".format(step[0].name))
oldinp = inp #- copy for QAs that need to see earlier input
inp = pa(inp, **pargs)
except Exception as e:
log.critical("Failed to run PA {} error was {}".format(
step[0].name, e),
exc_info=True)
sys.exit("Failed to run PA {}".format(step[0].name))
qaresult = {}
for qa in step[1]:
try:
qargs = mapkeywords(qa.config["kwargs"], convdict)
hb.start("Running {}".format(qa.name))
qargs[
"dict_countbins"] = passqadict #- pass this to all QA downstream
if isinstance(inp, tuple):
res = qa(inp[0], **qargs)
else:
res = qa(inp, **qargs)
# if "qafile" in qargs:
# qawriter.write_qa_file(qargs["qafile"],res)
log.debug("{} {}".format(qa.name, inp))
qaresult[qa.name] = res
schemaStep.addParams(res['PARAMS'])
schemaStep.addMetrics(res['METRICS'])
except Exception as e:
log.warning("Failed to run QA {}. Got Exception {}".format(
qa.name, e),
exc_info=True)
hb.stop("Step {} finished.".format(paconf[s]["StepName"]))
QAresults.append([pa.name, qaresult])
hb.stop("Pipeline processing finished. Serializing result")
def __init__(self,
configfile,
night,
telescope,
field,
ra,
dec,
datadir=None,
outdir=None,
tempdir=None,
plots=False):
"""
configfile : ROTSE-III configuration file (e.g. rotseproc/config/config_science.yaml)
night : night for the data to process (e.g. 20130101)
telescope : instrument to process (e.g. 3b)
field : observed field on the sky (e.g. sks0246+3652)
ra : target RA
dec : target DEC
datadir : directory containing data
outdir : output directory
"""
rlog = rlogger.rotseLogger(name="RotseConfig")
self.log = rlog.getlog()
with open(configfile, 'r') as f:
self.conf = yaml.safe_load(f)
f.close()
self.program = self.conf["Program"]
self.log.info("Expanding configuration for {} program".format(
self.program))
self.night = night
self.telescope = telescope
self.field = field
self.flavor = self.conf["Flavor"]
self.datadir = datadir
self.outdir = outdir
self.tempdir = tempdir
# Convert RA and DEC to floating point numbers
if ra is None:
self.ra = ra
self.dec = dec
elif ':' in ra:
ra_split = ra.split(':')
dec_split = dec.split(':')
self.ra = float(ra_split[0]) * 15. + float(
ra_split[1]) / 4. + float(ra_split[2]) / 240.
if float(dec_split[0]) >= 0.:
self.dec = float(dec_split[0]) + float(
dec_split[1]) / 60. + float(dec_split[2]) / 3600.
else:
self.dec = float(dec_split[0]) - float(
dec_split[1]) / 60. - float(dec_split[2]) / 3600.
elif float(ra) > 0. and float(ra) < 360.:
self.ra = float(ra)
self.dec = float(dec)
else:
self.log.warning(
"RA and DEC are not in the right format, this could cause downstream issues."
)
self.plotconf = None
self.hardplots = False
#- Load plotting configuration file
if plots != 'noplots' and plots is not None:
with open(plots, 'r') as pf:
self.plotconf = yaml.safe_load(pf)
pf.close()
#- Use hard coded plotting algorithms
elif plots is None:
self.hardplots = True
self.pipeline = self.conf["Pipeline"]
self.algorithms = self.conf["Algorithms"]
self._palist = Palist(self.pipeline, self.algorithms)
self.pamodule = self._palist.pamodule
self.qamodule = self._palist.qamodule
algokeys = self.algorithms.keys()
# Extract mapping of scalar/refence key names for each QA
qaRefKeys = {}
for i in algokeys:
for k in self.algorithms[i]["QA"].keys():
qaparams = self.algorithms[i]["QA"][k]["PARAMS"]
for par in qaparams.keys():
if "NORMAL_RANGE" in par:
scalar = par.replace("_NORMAL_RANGE", "")
qaRefKeys[k] = scalar
# Get pixel radius for subimages
self.pixrad = None
if "Make_Subimages" in self.algorithms.keys():
self.pixrad = self.algorithms["Make_Subimages"]["PixelRadius"]
self.t_before = None
self.t_after = None
if "Find_Data" in self.algorithms.keys():
self.t_before = self.algorithms["Find_Data"]["TimeBeforeDiscovery"]
self.t_after = self.algorithms["Find_Data"]["TimeAfterDiscovery"]
self._qaRefKeys = qaRefKeys