def showCals(self, rc): # Instantiate the log log = logutils.get_logger(__name__) if str(rc["showcals"]).lower() == "all": num = 0 # print "pG256: showcals=all", repr (rc.calibrations) for calkey in rc.calibrations: num += 1 log.stdinfo(rc.calibrations[calkey], category="calibrations") if (num == 0): log.stdinfo("There are no calibrations in the cache.") else: for adr in rc.inputs: sid = IDFactory.generate_astro_data_id(adr.ad) num = 0 for calkey in rc.calibrations: if sid in calkey : num += 1 log.stdinfo(rc.calibrations[calkey], category="calibrations") if (num == 0): log.stdinfo("There are no calibrations in the cache.") yield rc
def getList(self, rc): """ This primitive will check the files in the stack lists are on disk, and then update the inputs list to include all members of the stack for stacking. :param purpose: :type purpose: string """ # Instantiate the log log = gemLog.getGeminiLog(logType=rc["logType"], logLevel=rc["logLevel"]) # Get purpose of list sidset = set() purpose=rc["purpose"] if purpose is None: purpose = "" # Get ID for all inputs for inp in rc.inputs: sidset.add(purpose+IDFactory.generate_stackable_id(inp.ad)) # Import inputs from all lists for sid in sidset: stacklist = rc.get_list(sid) #.filelist log.stdinfo("List for stack id %s(...):" % sid[0:35]) for f in stacklist: rc.report_output(f, stream=rc["to_stream"]) log.stdinfo(" %s" % os.path.basename(f)) yield rc
def getDisplayID(self, ad): ''' ''' #@@TODO: #======================================================================= # Here or IDFactory is where I believe the majority of ADType / Stacking # intelligence will be done. Because ids can be passed in, the intelligence # can also be done externally (in reduce, for example). #======================================================================= if 'avgcomb_' in ad.filename: fid = 'STACKED_' + IDFactory.generate_stackable_id( ad ) else: fid = IDFactory.generate_stackable_id( ad ) return fid
def showList(self, rc): """ This primitive will log the list of files in the stacking list matching the current inputs and 'purpose' value. :param purpose: :type purpose: string """ # Instantiate the log log = gemLog.getGeminiLog(logType=rc["logType"], logLevel=rc["logLevel"]) sidset = set() purpose = rc["purpose"] if purpose is None: purpose = "" # print "pG710" if purpose == "all": allsids = rc.get_stack_ids() # print "pG713:", repr(allsids) for sid in allsids: sidset.add(sid) else: for inp in rc.inputs: sidset.add(purpose+IDFactory.generate_stackable_id(inp.ad)) for sid in sidset: stacklist = rc.get_list(sid) #.filelist log.status("List for stack id=%s" % sid) if len(stacklist) > 0: for f in stacklist: log.status(" %s" % os.path.basename(f)) else: log.status("No datasets in list") yield rc
def command_clause(ro, coi): global prs global log if log==None: log = gemLog.getGeminiLog() coi.process_cmd_req() while (coi.paused): time.sleep(.100) if coi.finished: return ml = coi.get_metric_list(clear=True) prs = Proxies.PRSProxy.get_adcc(check_once=True) if prs is not None: prs.report_qametrics(ml) #process reques for rq in coi.rorqs: rqTyp = type(rq) msg = 'REDUCE:\n' msg += '-'*30+'\n' if rqTyp == CalibrationRequest: #print "RO490:", repr(rq.as_dict()) #print "RO491:", coi["calurl_dict"] sci_ad = rq.ad if sci_ad: fn=sci_ad.filename else: fn = rq.filename typ = rq.caltype calname = None ## THIS IS THE CACHE CHECK, DISABLED NOW: calname = coi.get_cal(fn, typ) # print "r399:", "handling calibrations" calmd5 = None if calname == None: # Do the calibration search calurl = None if usePRS and prs == None: # print "r454: getting prs" prs = Proxies.PRSProxy.get_adcc() if usePRS: #log.debug("RO484:", pprint.pformat(rq.as_dict()), user_cal_service) try: if user_cal_service: calurl = user_cal_service.get_calibration(caltype = rq.caltype) #print "488: calurl", repr(calurl) #if calname: # return calname if calurl and len(calurl) and calurl[0] == None: log.warning(calurl[1]) calurl = None if calurl == None: # print "RO492", repr(rq) calurl = prs.calibration_search( rq ) if calurl is not None: if len(calurl) and calurl[0] == None: adcc_msg = calurl[1] calurl = None log.error("CALIBRATION SERVICE REPORT:\n"*2) log.error(adcc_msg) log.error("END CAL SERVICE REPORT:\n"*2) calurl = None else: calurl,calmd5 = calurl # print "RO475:", calurl, calmd5 except: calurl = None raise if calurl == None: log.warning('No '+str(typ)+' calibration file found for '+\ str(fn)) # this is not fatal because perhaps there isn't a calibration # the system checks both the local and central source # raise RecipeExcept("CALIBRATION for %s NOT FOUND, FATAL" % fn) #break continue log.info("found calibration (url): " + calurl) if calurl.startswith("file://"): calfile = calurl[7:] calurl = calfile else: calfile = None #print "RO393:", calurl, calfile msg += 'A suitable %s found:\n' %(str(typ)) useCached = False storenames = {"bias":"retrievedbiases", "flat":"retrievedflats" } if calfile: calfname = os.path.basename(calfile) caldname = os.path.dirname(calfile) # print "ro517:[%s]\n%s (%s)" % (calfile, calfname, caldname) elif os.path.exists(calurl): calfname = calurl caldname = None else: calfname = os.path.join(coi["retrievedcals"], typ, os.path.basename(calurl)) caldname = os.path.dirname(calfname) if caldname and not os.path.exists(caldname): os.mkdir(caldname) # print "RO400:",calfname if os.path.exists(calfname) and caldname: #coi.add_cal(fn, typ, calfname) # check md5 ondiskmd5 = IDFactory.generate_md5_file( calfname) if calmd5 == ondiskmd5: log.stdinfo("File %s exists at calibration location, " \ "md5 checksums match, using cached copy." % os.path.basename(calfname)) useCached = True else: log.stdinfo("File %s exists at calibration location, " \ "but md5 checksums DO NOT MATCH, retrieving." % os.path.basename(calfname)) try: if useCached: ad = AstroData(calfname) else: ad = AstroData(calurl, store=caldname) except urllib2.HTTPError, error: ad = None errstr = "Could not retrieve %s" % calurl log.error(errstr) #@@TODO: should this raise? raise ReductionExcept(errstr) if ad: coi.add_cal(sci_ad, typ, ad.filename) # adcc handles this now: coi.persist_cal_index() calname = calurl else: msg += '%s already stored.\n' %(str(typ)) msg += 'Using:\n' msg += '%s%s%s' %( os.path.dirname(calname), os.path.sep, os.path.basename(calname)) #print msg #print '-'*30 elif rqTyp == UpdateStackableRequest: coi.stack_append(rq.stk_id, rq.stk_list, stkindfile) coi.persist_stk_index( stkindfile )