def showCals(self, rc):
# Instantiate the log
log = logutils.get_logger(__name__)
if str(rc["showcals"]).lower() == "all":
num = 0
# print "pG256: showcals=all", repr (rc.calibrations)
for calkey in rc.calibrations:
num += 1
log.stdinfo(rc.calibrations[calkey], category="calibrations")
if (num == 0):
log.stdinfo("There are no calibrations in the cache.")
else:
for adr in rc.inputs:
sid = IDFactory.generate_astro_data_id(adr.ad)
num = 0
for calkey in rc.calibrations:
if sid in calkey :
num += 1
log.stdinfo(rc.calibrations[calkey],
category="calibrations")
if (num == 0):
log.stdinfo("There are no calibrations in the cache.")
yield rc
def getList(self, rc):
"""
This primitive will check the files in the stack lists are on disk,
and then update the inputs list to include all members of the stack
for stacking.
:param purpose:
:type purpose: string
"""
# Instantiate the log
log = gemLog.getGeminiLog(logType=rc["logType"],
logLevel=rc["logLevel"])
# Get purpose of list
sidset = set()
purpose=rc["purpose"]
if purpose is None:
purpose = ""
# Get ID for all inputs
for inp in rc.inputs:
sidset.add(purpose+IDFactory.generate_stackable_id(inp.ad))
# Import inputs from all lists
for sid in sidset:
stacklist = rc.get_list(sid) #.filelist
log.stdinfo("List for stack id %s(...):" % sid[0:35])
for f in stacklist:
rc.report_output(f, stream=rc["to_stream"])
log.stdinfo(" %s" % os.path.basename(f))
yield rc
def getDisplayID(self, ad):
'''
'''
#@@TODO:
#=======================================================================
# Here or IDFactory is where I believe the majority of ADType / Stacking
# intelligence will be done. Because ids can be passed in, the intelligence
# can also be done externally (in reduce, for example).
#=======================================================================
if 'avgcomb_' in ad.filename:
fid = 'STACKED_' + IDFactory.generate_stackable_id( ad )
else:
fid = IDFactory.generate_stackable_id( ad )
return fid
def showList(self, rc):
"""
This primitive will log the list of files in the stacking list matching
the current inputs and 'purpose' value.
:param purpose:
:type purpose: string
"""
# Instantiate the log
log = gemLog.getGeminiLog(logType=rc["logType"],
logLevel=rc["logLevel"])
sidset = set()
purpose = rc["purpose"]
if purpose is None:
purpose = ""
# print "pG710"
if purpose == "all":
allsids = rc.get_stack_ids()
# print "pG713:", repr(allsids)
for sid in allsids:
sidset.add(sid)
else:
for inp in rc.inputs:
sidset.add(purpose+IDFactory.generate_stackable_id(inp.ad))
for sid in sidset:
stacklist = rc.get_list(sid) #.filelist
log.status("List for stack id=%s" % sid)
if len(stacklist) > 0:
for f in stacklist:
log.status(" %s" % os.path.basename(f))
else:
log.status("No datasets in list")
yield rc
def command_clause(ro, coi):
global prs
global log
if log==None:
log = gemLog.getGeminiLog()
coi.process_cmd_req()
while (coi.paused):
time.sleep(.100)
if coi.finished:
return
ml = coi.get_metric_list(clear=True)
prs = Proxies.PRSProxy.get_adcc(check_once=True)
if prs is not None:
prs.report_qametrics(ml)
#process reques
for rq in coi.rorqs:
rqTyp = type(rq)
msg = 'REDUCE:\n'
msg += '-'*30+'\n'
if rqTyp == CalibrationRequest:
#print "RO490:", repr(rq.as_dict())
#print "RO491:", coi["calurl_dict"]
sci_ad = rq.ad
if sci_ad:
fn=sci_ad.filename
else:
fn = rq.filename
typ = rq.caltype
calname = None
## THIS IS THE CACHE CHECK, DISABLED NOW: calname = coi.get_cal(fn, typ)
# print "r399:", "handling calibrations"
calmd5 = None
if calname == None:
# Do the calibration search
calurl = None
if usePRS and prs == None:
# print "r454: getting prs"
prs = Proxies.PRSProxy.get_adcc()
if usePRS:
#log.debug("RO484:", pprint.pformat(rq.as_dict()), user_cal_service)
try:
if user_cal_service:
calurl = user_cal_service.get_calibration(caltype = rq.caltype)
#print "488: calurl", repr(calurl)
#if calname:
# return calname
if calurl and len(calurl) and calurl[0] == None:
log.warning(calurl[1])
calurl = None
if calurl == None:
# print "RO492", repr(rq)
calurl = prs.calibration_search( rq )
if calurl is not None:
if len(calurl) and calurl[0] == None:
adcc_msg = calurl[1]
calurl = None
log.error("CALIBRATION SERVICE REPORT:\n"*2)
log.error(adcc_msg)
log.error("END CAL SERVICE REPORT:\n"*2)
calurl = None
else:
calurl,calmd5 = calurl
# print "RO475:", calurl, calmd5
except:
calurl = None
raise
if calurl == None:
log.warning('No '+str(typ)+' calibration file found for '+\
str(fn))
# this is not fatal because perhaps there isn't a calibration
# the system checks both the local and central source
# raise RecipeExcept("CALIBRATION for %s NOT FOUND, FATAL" % fn)
#break
continue
log.info("found calibration (url): " + calurl)
if calurl.startswith("file://"):
calfile = calurl[7:]
calurl = calfile
else:
calfile = None
#print "RO393:", calurl, calfile
msg += 'A suitable %s found:\n' %(str(typ))
useCached = False
storenames = {"bias":"retrievedbiases",
"flat":"retrievedflats"
}
if calfile:
calfname = os.path.basename(calfile)
caldname = os.path.dirname(calfile)
# print "ro517:[%s]\n%s (%s)" % (calfile, calfname, caldname)
elif os.path.exists(calurl):
calfname = calurl
caldname = None
else:
calfname = os.path.join(coi["retrievedcals"], typ, os.path.basename(calurl))
caldname = os.path.dirname(calfname)
if caldname and not os.path.exists(caldname):
os.mkdir(caldname)
# print "RO400:",calfname
if os.path.exists(calfname) and caldname:
#coi.add_cal(fn, typ, calfname)
# check md5
ondiskmd5 = IDFactory.generate_md5_file( calfname)
if calmd5 == ondiskmd5:
log.stdinfo("File %s exists at calibration location, " \
"md5 checksums match, using cached copy." % os.path.basename(calfname))
useCached = True
else:
log.stdinfo("File %s exists at calibration location, " \
"but md5 checksums DO NOT MATCH, retrieving." % os.path.basename(calfname))
try:
if useCached:
ad = AstroData(calfname)
else:
ad = AstroData(calurl, store=caldname)
except urllib2.HTTPError, error:
ad = None
errstr = "Could not retrieve %s" % calurl
log.error(errstr)
#@@TODO: should this raise? raise ReductionExcept(errstr)
if ad:
coi.add_cal(sci_ad, typ, ad.filename)
# adcc handles this now: coi.persist_cal_index()
calname = calurl
else:
msg += '%s already stored.\n' %(str(typ))
msg += 'Using:\n'
msg += '%s%s%s' %( os.path.dirname(calname), os.path.sep, os.path.basename(calname))
#print msg
#print '-'*30
elif rqTyp == UpdateStackableRequest:
coi.stack_append(rq.stk_id, rq.stk_list, stkindfile)
coi.persist_stk_index( stkindfile )
