Ejemplo n.º 1
0
 def getProcessedFringe(self, rc):
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     caltype = "processed_fringe"
     source = rc["source"]
     if source == None:
         rc.run("getCalibration(caltype=%s)" % caltype)
     else:
         rc.run("getCalibration(caltype=%s, source=%s)" % (caltype,source))
         
     # List calibrations found
     # Fringe correction is always optional, so don't raise errors if fringe
     # not found
     first = True
     for ad in rc.get_inputs_as_astrodata():
         calurl = rc.get_cal(ad, caltype) #get from cache
         if calurl:
             cal = AstroData(calurl)
             if cal.filename is not None:
                 if first:
                     log.stdinfo("getCalibration: Results")
                     first = False
                 log.stdinfo("   %s\n      for %s" % (cal.filename,
                                                  ad.filename))
     
     yield rc
Ejemplo n.º 2
0
 def showCals(self, rc):
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     if str(rc["showcals"]).lower() == "all":
         num = 0
         # print "pG256: showcals=all", repr (rc.calibrations)
         for calkey in rc.calibrations:
             num += 1
             log.stdinfo(rc.calibrations[calkey], category="calibrations")
         if (num == 0):
             log.stdinfo("There are no calibrations in the cache.")
     else:
         for adr in rc.inputs:
             sid = IDFactory.generate_astro_data_id(adr.ad)
             num = 0
             for calkey in rc.calibrations:
                 if sid in calkey :
                     num += 1
                     log.stdinfo(rc.calibrations[calkey], 
                                  category="calibrations")
         if (num == 0):
             log.stdinfo("There are no calibrations in the cache.")
     
     yield rc
Ejemplo n.º 3
0
    def separateLampOff(self, rc):
        """
        This primitive is intended to run on gcal imaging flats. 
        It goes through the input list and figures out which ones are lamp-on
        and which ones are lamp-off
        """
        # Instantiate the log
        log = logutils.get_logger(__name__)

        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "separateLampOff", "starting"))

        # Initialize the list of output AstroData objects
        lampon_list = []
        lampoff_list = []

        # Loop over the input frames
        for ad in rc.get_inputs_as_astrodata():
            if('GCAL_IR_ON' in ad.types):
                log.stdinfo("%s is a lamp-on flat" % ad.data_label())
                #rc.run("addToList(purpose=lampOn)")
                lampon_list.append(ad)
            elif('GCAL_IR_OFF' in ad.types):
                log.stdinfo("%s is a lamp-off flat" % ad.data_label())
                #rc.run("addToList(purpose=lampOff)")
                lampoff_list.append(ad)
            else:
                log.warning("Not a GCAL flatfield? Cannot tell if it is lamp-on or lamp-off for %s" % ad.data_label())

        rc.report_output(lampon_list, stream="lampOn")
        rc.report_output(lampoff_list, stream="lampOff")

        yield rc
Ejemplo n.º 4
0
 def failCalibration(self,rc):
     # Mark a given calibration "fail" and upload it 
     # to the system. This is intended to be used to mark a 
     # calibration file that has already been uploaded, so that
     # it will not be returned as a valid match for future data.
     
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     # Initialize the list of output AstroData objects 
     adoutput_list = []
     
     # Loop over each input AstroData object in the input list
     for ad in rc.get_inputs_as_astrodata():
         
         # Change the two keywords -- BAD and NO = Fail
         ad.phu_set_key_value("RAWGEMQA","BAD",
                              comment=self.keyword_comments["RAWGEMQA"])
         ad.phu_set_key_value("RAWPIREQ","NO",
                              comment=self.keyword_comments["RAWPIREQ"])
         log.fullinfo("%s has been marked %s" % (ad.filename,ad.qa_state()))
         
         # Append the output AstroData object to the list
         # of output AstroData objects
         adoutput_list.append(ad)
     
     # Report the list of output AstroData objects to the 
     # reduction context
     rc.report_output(adoutput_list)
     
     # Run the storeCalibration primitive, so that the 
     # failed file gets re-uploaded
     rc.run("storeCalibration")
     
     yield rc
Ejemplo n.º 5
0
 def storeProcessedFlat(self, rc):
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     # Log the standard "starting primitive" debug message
     log.debug(gt.log_message("primitive", "storeProcessedFlat",
                              "starting"))
     
     # Loop over each input AstroData object in the input list
     for ad in rc.get_inputs_as_astrodata():
         
         # Updating the file name with the suffix for this primitive and
         # then report the new file to the reduction context
         ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
                                           strip=True)
         
         # Adding a PROCFLAT time stamp to the PHU
         gt.mark_history(adinput=ad, keyword="PROCFLAT")
         
         # Refresh the AD types to reflect new processed status
         ad.refresh_types()
     
     # Upload to cal system
     rc.run("storeCalibration")
     
     yield rc
Ejemplo n.º 6
0
    def subtractLampOnLampOff(self, rc):
        """
        This primitive subtracts the lamp off stack from the lampon stack. It expects there to be only
        one file (the stack) on each stream - call stackLampOnLampOff to do the stacking before calling this
        """

        # Instantiate the log
        log = logutils.get_logger(__name__)

        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "subtractLampOnLampOff", "starting"))

        # Initialize the list of output AstroData objects
        adoutput_list = []

        lampon = rc.get_stream(stream="lampOn", style="AD")[0]
        lampoff = rc.get_stream(stream="lampOff", style="AD")[0]

        log.stdinfo("Lamp ON is: %s %s" % (lampon.data_label(), lampon.filename))
        log.stdinfo("Lamp OFF is: %s %s" % (lampoff.data_label(), lampoff.filename))
        lampon.sub(lampoff)
        lampon.filanme = gt.filename_updater(adinput=lampon, suffix="lampOnOff")

        adoutput_list.append(lampon)
        rc.report_output(adoutput_list)

        yield rc
Ejemplo n.º 7
0
 def getProcessedArc(self, rc):
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     caltype = "processed_arc"
     source = rc["source"]
     if source == None:
         rc.run("getCalibration(caltype=%s)" % caltype)
     else:
         rc.run("getCalibration(caltype=%s, source=%s)" % (caltype,source))
     
     # List calibrations found
     first = True
     for ad in rc.get_inputs_as_astrodata():
         calurl = rc.get_cal(ad, caltype) #get from cache
         if calurl:
             cal = AstroData(calurl)
             if cal.filename is None:
                 if "qa" not in rc.context:
                     raise Errors.InputError("Calibration not found for " \
                                             "%s" % ad.filename)
             else:
                 if first:
                     log.stdinfo("getCalibration: Results")
                     first = False
                 log.stdinfo("   %s\n      for %s" % (cal.filename,
                                                      ad.filename))
         else: 
             if "qa" not in rc.context:
                 raise Errors.InputError("Calibration not found for %s" % 
                                         ad.filename)
     
     yield rc
Ejemplo n.º 8
0
 def storeCalibration(self, rc):
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     # Log the standard "starting primitive" debug message
     log.debug(gt.log_message("primitive", "storeCalibration", "starting"))
     
     # Determine the path where the calibration will be stored
     storedcals = rc["cachedict"]["storedcals"]
     
     # Loop over each input AstroData object in the input list
     for ad in rc.get_inputs_as_astrodata():
         
         # Construct the filename of the calibration, including the path
         fname = os.path.join(storedcals, os.path.basename(ad.filename))
         
         # Write the calibration to disk. Use rename=False so that
         # ad.filename does not change (i.e., does not include the
         # calibration path)
         ad.write(filename=fname, rename=False, clobber=True)
         log.stdinfo("Calibration stored as %s" % fname)
         
         if "upload" in rc.context:
             try:
                 upload_calibration(fname)
             except:
                 log.warning("Unable to upload file to calibration system")
             else:
                 log.stdinfo("File %s uploaded to fitsstore." % 
                             os.path.basename(ad.filename))
         yield rc
     
     yield rc
Ejemplo n.º 9
0
    def normalizeFlat(self, rc):
        """
        This primitive normalizes each science extension of the input
        AstroData object by its mean
        """
        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "normalizeFlat", "starting"))
        
        # Define the keyword to be used for the time stamp for this primitive
        timestamp_key = self.timestamp_keys["normalizeFlat"]

        # Initialize the list of output AstroData objects
        adoutput_list = []
        
        # Loop over each input AstroData object in the input list
        for ad in rc.get_inputs_as_astrodata():
            
            # Check whether the normalizeFlat primitive has been run previously
            if ad.phu_get_key_value(timestamp_key):
                log.warning("No changes will be made to %s, since it has " \
                            "already been processed by normalizeFlat" \
                            % (ad.filename))
                # Append the input AstroData object to the list of output
                # AstroData objects without further processing
                adoutput_list.append(ad)
                continue
            
            # Loop over each science extension in each input AstroData object
            for ext in ad[SCI]:
                
                # Normalise the input AstroData object. Calculate the mean
                # value of the science extension
                mean = np.mean(ext.data, dtype=np.float64)
                # Divide the science extension by the mean value of the science
                # extension
                log.fullinfo("Normalizing %s[%s,%d] by dividing by the mean " \
                             "= %f" % (ad.filename, ext.extname(),
                                       ext.extver(), mean))
                ext = ext.div(mean)

            # Add the appropriate time stamps to the PHU
            gt.mark_history(adinput=ad, keyword=timestamp_key)

            # Change the filename
            ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"], 
                                              strip=True)

            # Append the output AstroData object to the list
            # of output AstroData objects
            adoutput_list.append(ad)
        
        # Report the list of output AstroData objects to the reduction
        # context
        rc.report_output(adoutput_list)
        
        yield rc
Ejemplo n.º 10
0
    def cutFootprints(self, rc):
 
        """
        This primitive will create and append multiple HDU to the output
        AD object. Each HDU correspond to a rectangular cut containing a
        slit from a MOS Flat exposure or a XD flat exposure as in the
        Gnirs case.

        :param logLevel: Verbosity setting for log messages to the screen.
        :type logLevel: integer from 0-6, 0=nothing to screen, 6=everything to
                        screen. OR the message level as a string (i.e.,
                        'critical', 'status', 'fullinfo'...)
        """

        # Instantiate the log
        log = logutils.get_logger(__name__)

        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "cutFootprints", "starting"))

        # Initialize the list of output AstroData objects
        adoutput_list = []

        # Loop over each input AstroData object in the input list
        for ad in rc.get_inputs_as_astrodata():
            # Call the  user level function
      
            # Check that the input ad has the TRACEFP extension,
            # otherwise, create it.
            if ad['TRACEFP'] == None:
                ad = trace_footprints(ad)

            log.stdinfo("Cutting_footprints for: %s"%ad.filename)
            try:
                adout = cut_footprints(ad)
            except:
                log.error("Error in cut_slits with file: %s"%ad.filename)
                # DO NOT add this input ad to the adoutput_lis
                continue
               
               
            # Change the filename
            adout.filename = gt.filename_updater(adinput=ad, 
                                                 suffix=rc["suffix"],
                                                 strip=True)

            # Append the output AstroData object to the list of output 
            # AstroData objects.
            adoutput_list.append(adout)

        # Report the list of output AstroData objects to the reduction
        # context
        rc.report_output(adoutput_list)

        yield rc
Ejemplo n.º 11
0
    def traceFootprints(self, rc):
 
        """
        This primitive will create and append a 'TRACEFP' Bintable HDU to the
        AD object. The content of this HDU is the footprints information 
        from the espectroscopic flat in the SCI array.

        :param logLevel: Verbosity setting for log messages to the screen.
        :type logLevel: integer from 0-6, 0=nothing to screen, 6=everything to
                        screen. OR the message level as a string (i.e.,
                        'critical', 'status', 'fullinfo'...)
        """
        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "", "starting"))

        # Initialize the list of output AstroData objects
        adoutput_list = []

        # Loop over each input AstroData object in the input list
        for ad in rc.get_inputs_as_astrodata():
            # Check whether this primitive has been run previously
            if ad.phu_get_key_value("TRACEFP"):
                log.warning("%s has already been processed by traceSlits" \
                            % (ad.filename))
                # Append the input AstroData object to the list of output
                # AstroData objects without further processing
                adoutput_list.append(ad)
                continue
            # Call the  user level function
            try:
               adout = trace_footprints(ad,function=rc["function"],
                                  order=rc["order"],
                                  trace_threshold=rc["trace_threshold"])
            except:
               log.warning("Error in traceFootprints with file: %s"%ad.filename)
               
            # Change the filename
            adout.filename = gt.filename_updater(adinput=ad, 
                                                 suffix=rc["suffix"],
                                                 strip=True)

            # Append the output AstroData object to the list of output 
            # AstroData objects.
            adoutput_list.append(adout)

        # Report the list of output AstroData objects to the reduction
        # context
        rc.report_output(adoutput_list)

        yield rc
Ejemplo n.º 12
0
    def wcalResampleToLinearCoords(self,rc):

        """ Uses the Wavecal fit_image solution
        """
        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Define the keyword to be used for the time stamp
        timestamp_key = self.timestamp_keys["wcalResampleToLinearCoords"]

        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "wcalResampleToLinearCoords", 
                                 "starting"))
                
        # Initialize the list of output AstroData objects
        adoutput_list = []

        # Loop over each input AstroData object in the input list
        for ad in rc.get_inputs_as_astrodata():

            # Check for a wavelength solution
            if ad["WAVECAL"] is None:
                if "qa" in rc.context:
                    log.warning("No wavelength solution found for %s" %
                                ad.filename)

                    adout=ad   # Don't do anything 
                else:
                    raise Errors.InputError("No wavelength solution found "\
                                            "for %s" % ad.filename)
            else:
                # Wavelength solution found. 
                wc = Wavecal(ad)
                wc.read_wavecal_table()
                adout = wc.resample_image_asAstrodata()

            # Add the appropriate time stamps to the PHU
            gt.mark_history(adinput=adout, keyword=timestamp_key)

            # Change the filename
            adout.filename = gt.filename_updater(
                adinput=adout, suffix=rc["suffix"], strip=True)
            
            # Append the output AstroData object to the list
            # of output AstroData objects
            adoutput_list.append(adout)
        
        # Report the list of output AstroData objects to the reduction
        # context
        rc.report_output(adoutput_list)
        
        yield rc
Ejemplo n.º 13
0
 def validateData(self, rc):
     """
     This primitive is used to validate NIRI data, specifically.
     
     :param repair: Set to True to repair the data, if necessary. Note: this
                    feature does not work yet. 
     :type repair: Python boolean
     """
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     # Log the standard "starting primitive" debug message
     log.debug(gt.log_message("primitive", "validateData", "starting"))
     
     # Define the keyword to be used for the time stamp for this primitive
     timestamp_key = self.timestamp_keys["validateData"]
     
     # Initialize the list of output AstroData objects
     adoutput_list = []
     
     # Loop over each input AstroData object in the input list
     for ad in rc.get_inputs_as_astrodata():
         
         # Check whether the validateData primitive has been run previously
         if ad.phu_get_key_value(timestamp_key):
             log.warning("No changes will be made to %s, since it has "
                         "already been processed by validateData"
                         % ad.filename)
             
             # Append the input AstroData object to the list of output
             # AstroData objects without further processing
             adoutput_list.append(ad)
             continue
         
         # Validate the input AstroData object.
         log.status("No validation required for %s" % ad.filename)
         
         # Add the appropriate time stamps to the PHU
         gt.mark_history(adinput=ad, keyword=timestamp_key)
         
         # Change the filename
         ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
                                           strip=True)
         
         # Append the output AstroData object to the list of output
         # AstroData objects
         adoutput_list.append(ad)
     
     # Report the list of output AstroData objects to the reduction context
     rc.report_output(adoutput_list)
     
     yield rc
Ejemplo n.º 14
0
    def skyCorrectFromSlit(self,rc):

        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Define the keyword to be used for the time stamp
        timestamp_key = self.timestamp_keys["skyCorrectFromSlit"]

        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "skyCorrectFromSlit", "starting"))
                
        # Initialize the list of output AstroData objects
        adoutput_list = []

        # Loop over each input AstroData object in the input list
        for ad in rc.get_inputs_as_astrodata():

            try:
                xbin = ad.detector_x_bin().as_pytype()
                ybin = ad.detector_y_bin().as_pytype()
                bin_factor = xbin*ybin
                roi = ad.detector_roi_setting().as_pytype()
            except:
                bin_factor = 1
                roi = "unknown"

            if bin_factor<=2 and roi=="Full Frame" and "qa" in rc.context:
                log.warning("Frame is too large to subtract sky efficiently; not "\
                            "subtracting sky for %s" % ad.filename)
                adoutput_list.append(ad)
                continue

            # Instantiate ETI and then run the task 
            gsskysub_task = eti.gsskysubeti.GsskysubETI(rc,ad)
            adout = gsskysub_task.run()

            # Add the appropriate time stamps to the PHU
            gt.mark_history(adinput=adout, keyword=timestamp_key)

            # Change the filename
            adout.filename = gt.filename_updater(
                adinput=adout, suffix=rc["suffix"], strip=True)
            
            # Append the output AstroData object to the list
            # of output AstroData objects
            adoutput_list.append(adout)
        
        # Report the list of output AstroData objects to the reduction
        # context
        rc.report_output(adoutput_list)
        
        yield rc
Ejemplo n.º 15
0
    def determineWavelengthSolution(self,rc):

        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Define the keyword to be used for the time stamp
        timestamp_key = self.timestamp_keys["determineWavelengthSolution"]

        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "determineWavelengthSolution",
                                 "starting"))
                
        # Initialize the list of output AstroData objects
        adoutput_list = []

        # Loop over each input AstroData object in the input list
        for ad in rc.get_inputs_as_astrodata():

            # Instantiate ETI and then run the task 
            # Run in a try/except because gswavelength sometimes fails
            # badly, and we want to be able to continue without
            # wavelength calibration in the QA case
            gswavelength_task = eti.gswavelengtheti.GswavelengthETI(rc,ad)
            try:
                adout = gswavelength_task.run()
            except Errors.OutputError:
                gswavelength_task.clean()
                if "qa" in rc.context:
                    log.warning("gswavelength failed for input " + ad.filename)
                    adoutput_list.append(ad)
                    continue
                else:
                    raise Errors.ScienceError("gswavelength failed for input "+
                                              ad.filename + ". Try interactive"+
                                              "=True")

            # Add the appropriate time stamps to the PHU
            gt.mark_history(adinput=adout, keyword=timestamp_key)

            # Change the filename
            adout.filename = gt.filename_updater(
                adinput=adout, suffix=rc["suffix"], strip=True)
            
            # Append the output AstroData object to the list
            # of output AstroData objects
            adoutput_list.append(adout)
        
        # Report the list of output AstroData objects to the reduction
        # context
        rc.report_output(adoutput_list)

        yield rc
Ejemplo n.º 16
0
    def alignAndStack(self, rc):
        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "alignAndStack", "starting"))
         
        # Add the input frame to the forStack list and 
        # get other available frames from the same list
        rc.run("addToList(purpose=forStack)")
        rc.run("getList(purpose=forStack)")

        # Check whether two or more input AstroData objects were provided
        adinput = rc.get_inputs_as_astrodata()
        if len(adinput) <= 1:
            log.stdinfo("No alignment or correction will be performed, since "
                        "at least two input AstroData objects are required "
                        "for alignAndStack")
            rc.report_output(adinput)
        else:
            recipe_list = []

            # Check to see if detectSources needs to be run
            run_ds = False
            for ad in adinput:
                objcat = ad["OBJCAT"]
                if objcat is None:
                    run_ds = True
                    break
            if run_ds:
                recipe_list.append("detectSources")
            
            # Register all images to the first one
            recipe_list.append("correctWCSToReferenceFrame")
            
            # Align all images to the first one
            recipe_list.append("alignToReferenceFrame")
            
            # Correct background level in all images to the first one
            recipe_list.append("correctBackgroundToReferenceImage")

            # Stack all frames
            recipe_list.append("stackFrames") 
            
            # Run all the needed primitives
            rc.run("\n".join(recipe_list))
        
        yield rc
Ejemplo n.º 17
0
def mkRO(dataset="", astrotype="", copy_input=False, args = None, argv = None):

    log = logutils.get_logger(__name__)
    
    rl = RecipeLibrary()
    if dataset != "":
        ad = AstroData(dataset)
        ro = rl.retrieve_reduction_object(ad)
    elif astrotype != "":
        ad = None
        ro = rl.retrieve_reduction_object(astrotype = astrotype)

    # using standard command clause supplied in RecipeLibrary module
    ro.register_command_clause(command_clause)
    rc = ReductionContext(adcc_mode="start_lazy")
    rc.ro = ro
    ro.context = rc
    reductionObject = ro
    
    # Override copy_input argument if passed in argv
    if argv is not None:
        if argv.has_key("copy_input"):
            copy_input = argv["copy_input"]

    # Add input passed in args
    if args:
        arglist = []
        for arg in args:
            if isinstance(arg,list):
                for subarg in arg:
                    if copy_input:
                        subarg = deepcopy(subarg)
                    arglist.append(subarg)
            else:
                if copy_input:
                    arg = deepcopy(arg)
                arglist.append(arg)
        rc.populate_stream(arglist)

    rc.initialize_inputs()
    rc.set_context("pif")
    rc.update({'logindent':logutils.SW})
    rc.update(argv)
    ro.init(rc)
    
    return reductionObject
Ejemplo n.º 18
0
 def markAsPrepared(self, rc):
     """
     This primitive is used to add a time stamp keyword to the PHU of the
     AstroData object and update the AstroData type, allowing the output
     AstroData object to be recognised as PREPARED.
     
     """
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     # Log the standard "starting primitive" debug message
     log.debug(gt.log_message("primitive", "markAsPrepared", "starting"))
     
     # Define the keyword to be used for the time stamp for this primitive
     timestamp_key = self.timestamp_keys["prepare"]
     
     # Initialize the list of output AstroData objects
     adoutput_list = []
     
     # Loop over each input AstroData object in the input list
     for ad in rc.get_inputs_as_astrodata():
         
         # Add the appropriate time stamps to the PHU
         gt.mark_history(adinput=ad, keyword=timestamp_key)
         
         # Update the AstroData type so that the AstroData object is
         # recognised as being prepared
         ad.refresh_types()
         
         # Change the filename
         ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
                                           strip=True)
         
         # Append the output AstroData object to the list of output
         # AstroData objects
         adoutput_list.append(ad)
     
     # Report the list of output AstroData objects to the reduction context
     rc.report_output(adoutput_list)
     
     yield rc
Ejemplo n.º 19
0
    def makeFlat(self,rc):

        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Define the keyword to be used for the time stamp
        timestamp_key = self.timestamp_keys["makeFlat"]

        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "makeFlat", "starting"))
                
        # Initialize the list of output AstroData objects
        adoutput_list = []

        # Check if inputs prepared
        for ad in rc.get_inputs_as_astrodata():
            if "PREPARED" not in ad.types:
                raise Errors.InputError("%s must be prepared" % ad.filename)

        # Instantiate ETI and then run the task 
        gsflat_task = eti.gsflateti.GsflatETI(rc)
        adout = gsflat_task.run()

        # Set any zero-values to 1 (to avoid dividing by zero)
        for sciext in adout["SCI"]:
            sciext.data[sciext.data==0] = 1.0

        # Blank out any position or program information in the
        # header (spectroscopy flats are often taken with science data)
        adout = gt.convert_to_cal_header(adinput=adout,caltype="flat")[0]

        # Add the appropriate time stamps to the PHU
        gt.mark_history(adinput=adout, keyword=timestamp_key)

        adoutput_list.append(adout)

        # Report the list of output AstroData objects to the reduction
        # context
        rc.report_output(adoutput_list)
        
        yield rc
Ejemplo n.º 20
0
    def getCalibration(self, rc):
        
        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Retrieve type of calibration requested
        caltype = rc["caltype"]
        if caltype == None:
            log.error("getCalibration: caltype not set")
            raise Errors.PrimitiveError("getCalibration: caltype not set")
        
        # Retrieve source of calibration
        source = rc["source"]
        if source == None:
            source = "all"
            
        # Check whether calibrations are already available
        calibrationless_adlist = []
        adinput = rc.get_inputs_as_astrodata()
        
        
        #print "70: WRITE ALL CALIBRATION SOURCES\n"*10
        #for ad in adinput:
        #    ad.write(clobber=True)

        #for ad in adinput:
        #    ad.mode = "update"
        #    calurl = rc.get_cal(ad,caltype)
        #    if not calurl:
        #        calibrationless_adlist.append(ad)
        calibrationless_adlist = adinput
        # Request any needed calibrations
        if len(calibrationless_adlist) ==0:
            # print "pG603: calibrations for all files already present"
            pass
        else:
            rc.rq_cal(caltype, calibrationless_adlist, source=source)
        
        yield rc
Ejemplo n.º 21
0
    def stackLampOnLampOff(self, rc):
        """
        This primitive stacks the Lamp On flats and the LampOff flats, then subtracts the two stacks
        """
        # Instantiate the log
        log = logutils.get_logger(__name__)

        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "stackLampOnLampOff", "starting"))

        # Initialize the list of output AstroData objects
        adoutput_list = []

        # Get the lamp on list, stack it, and add the stack to the lampOnStack stream
        rc.run("showInputs(stream=lampOn)")
        rc.run("stackFrames(stream=lampOn)")

        # Get the lamp off list, stack it, and add the stack to the lampOnStack stream
        rc.run("showInputs(stream=lampOff)")
        rc.run("stackFrames(stream=lampOff)")

        yield rc
Ejemplo n.º 22
0
 def storeProcessedFringe(self, rc):
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     # Log the standard "starting primitive" debug message
     log.debug(gt.log_message("primitive", "storeProcessedFringe",
                              "starting"))
     
     # Initialize the list of output AstroData objects
     adoutput_list = []
     
     # Loop over each input AstroData object in the input list
     for ad in rc.get_inputs_as_astrodata():
         
         # Updating the file name with the suffix for this primitive and
         # then report the new file to the reduction context
         ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
                                           strip=True)
         
         # Sanitize the headers of the file so that it looks like
         # a public calibration file rather than a science file
         ad = gt.convert_to_cal_header(adinput=ad, caltype="fringe")[0]
         
         # Adding a PROCFRNG time stamp to the PHU
         gt.mark_history(adinput=ad, keyword="PROCFRNG")
         
         # Refresh the AD types to reflect new processed status
         ad.refresh_types()
         
         adoutput_list.append(ad)
     
     # Upload to cal system
     rc.run("storeCalibration")
     
     # Report the list of output AstroData objects to the reduction
     # context
     rc.report_output(adoutput_list)
     
     yield rc
Ejemplo n.º 23
0
    def approximateWaveCal(self,rc):

        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Define the keyword to be used for the time stamp
        timestamp_key = self.timestamp_keys["appwave"]

        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "appwave", "starting"))
                
        # Initialize the list of output AstroData objects
        adoutput_list = []

        # Loop over each input AstroData object in the input list
        for ad in rc.get_inputs_as_astrodata():

            applyApproxWaveCal(ad)

            adout = ad

            # Add the appropriate time stamps to the PHU
            gt.mark_history(adinput=adout, keyword=timestamp_key)

            # Change the filename
            adout.filename = gt.filename_updater(
                adinput=adout, suffix=rc["suffix"], strip=True)
            
            # Append the output AstroData object to the list
            # of output AstroData objects
            adoutput_list.append(adout)
        
        # Report the list of output AstroData objects to the reduction
        # context
        rc.report_output(adoutput_list)

        yield rc
Ejemplo n.º 24
0
import os
from astrodata.eti.pyrafetiparam import PyrafETIParam, IrafStdout
from pyraf import iraf
from astrodata.adutils import logutils
from gempy.gemini.gemini_tools import calc_nbiascontam

log = logutils.get_logger(__name__)

class GireduceParam(PyrafETIParam):
    """This class coordinates the ETI parameters as it pertains to the IRAF
    task gireduce directly.
    """
    rc = None
    key = None
    value = None
    def __init__(self, rc=None, key=None, value=None):
        """
        :param rc: Used to store reduction information
        :type rc: ReductionContext

        :param key: A parameter name that is added as a dict key in prepare
        :type key: any

        :param value: A parameter value that is added as a dict value
                      in prepare
        :type value: any
        """
        log.debug("GireduceParam __init__")
        PyrafETIParam.__init__(self, rc)
        self.key = key
        self.value = value
Ejemplo n.º 25
0
from astrodata.ReductionObjects import PrimitiveSet
from astrodata.adutils import logutils, ksutil
from astrodata.AstroDataType import globalClassificationLibrary as gCL
from primitives_SETREF import SetRefPrimitives
import pandas as pd

from astrodata.generaldata import GeneralData

try:
    from astrodata.adutils import termcolor
    COLORSTR = termcolor.line_color
except:
    COLORSTR = lambda arg: arg

log = logutils.get_logger(__name__)


class PandasPrimitives(SetRefPrimitives):
    astrotype = "TABLE"

    def loadTables(self, rc):
        for inp in rc.get_inputs():
            inp.load()
        yield rc

    ## COLUMN_RELATE ##########
    def columnRelate(self, rc):
        for inp in rc.get_inputs():
            log.stdinfo("rows=%s" %
                        " | ".join(inp.dataframe.columns.values.tolist()))
Ejemplo n.º 26
0
    def _calculate_var(self, adinput=None, add_read_noise=False,
                       add_poisson_noise=False):
        """
        The _calculate_var helper function is used to calculate the variance
        and add a variance extension to the single input AstroData object.
        
        """
        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Get the gain and the read noise using the appropriate descriptors.
        gain_dv = adinput.gain()
        read_noise_dv = adinput.read_noise()

        # Only check read_noise here as gain descriptor is only used if units
        # are in ADU
        if read_noise_dv.is_none() and add_read_noise:
            # The descriptor functions return None if a value cannot be found
            # and stores the exception info. Re-raise the exception.
            if hasattr(adinput, "exception_info"):
                raise adinput.exception_info
            else:
                raise Errors.InputError("read_noise descriptor "
                                        "returned None...\n%s"
                                        % (read_noise_dv.info()))
            
        # Set the data type of the final variance array
        var_dtype = np.dtype(np.float32)
        
        # Loop over the science extensions in the dataset
        for ext in adinput[SCI]:
            extver = ext.extver()
            bunit  = ext.get_key_value("BUNIT")

            if bunit == "adu":
                # Get the gain value using the appropriate descriptor. The gain
                # is only used if the units are in ADU. Raise if gain is None
                gain = gain_dv.get_value(extver=extver)
                if gain is not None:
                    log.fullinfo("Gain for %s[%s,%d] = %f"
                                 % (adinput.filename, SCI, extver, gain))
                elif add_read_noise or add_poisson_noise:
                    err_msg = ("Gain for %s[%s,%d] is None. Cannot calculate "
                                "variance properly. Setting to zero."
                                % (adinput.filename, SCI, extver))
                    raise Errors.InputError(err_msg)
                
                units = "ADU"
            elif bunit == "electron" or bunit == "electrons":
                units = "electrons"
            else:
                # Perhaps something more sensible should be done here?
                raise Errors.InputError("No units found. Not calculating "
                                        "variance.")
            
            if add_read_noise:
                # Get the read noise value (in units of electrons) using the
                # appropriate descriptor. The read noise is only used if
                # add_read_noise is True
                read_noise = read_noise_dv.get_value(extver=extver)
                if read_noise is not None:
                    log.fullinfo("Read noise for %s[%s,%d] = %f"
                                 % (adinput.filename, SCI, extver, read_noise))
                    
                    # Determine the variance value to use when calculating the
                    # read noise component of the variance.
                    read_noise_var_value = read_noise
                    if units == "ADU":
                        read_noise_var_value = read_noise / gain
                    
                    # Add the read noise component of the variance to a zeros
                    # array that is the same size as the pixel data in the
                    # science extension
                    log.fullinfo("Calculating the read noise component of the "
                                 "variance in %s" % units)
                    var_array_rn = np.add(
                      np.zeros(ext.data.shape), (read_noise_var_value)**2)
                else:
                    logwarning("Read noise for %s[%s,%d] is None. Setting to "
                               "zero" % (adinput.filename, SCI, extver))
                    var_array_rn = np.zeros(ext.data.shape)
                    
            if add_poisson_noise:
                # Determine the variance value to use when calculating the
                # poisson noise component of the variance
                poisson_noise_var_value = ext.data
                if units == "ADU":
                    poisson_noise_var_value = ext.data / gain
                
                # Calculate the poisson noise component of the variance. Set
                # pixels that are less than or equal to zero to zero.
                log.fullinfo("Calculating the poisson noise component of "
                             "the variance in %s" % units)
                var_array_pn = np.where(
                  ext.data > 0, poisson_noise_var_value, 0)
            
            # Create the final variance array
            if add_read_noise and add_poisson_noise:
                var_array_final = np.add(var_array_rn, var_array_pn)
            
            if add_read_noise and not add_poisson_noise:
                var_array_final = var_array_rn
            
            if not add_read_noise and add_poisson_noise:
                var_array_final = var_array_pn
            
            var_array_final = var_array_final.astype(var_dtype)
            
            # If the read noise component and the poisson noise component are
            # calculated and added separately, then a variance extension will
            # already exist in the input AstroData object. In this case, just
            # add this new array to the current variance extension
            if adinput[VAR, extver]:
                
                # If both the read noise component and the poisson noise
                # component have been calculated, don't add to the variance
                # extension
                if add_read_noise and add_poisson_noise:
                    raise Errors.InputError(
                        "Cannot add read noise component and poisson noise "
                        "component to variance extension as the variance "
                        "extension already exists")
                else:
                    log.fullinfo("Combining the newly calculated variance "
                                 "with the current variance extension "
                                 "%s[%s,%d]" % (adinput.filename, VAR, extver))
                    adinput[VAR, extver].data = np.add(
                      adinput[VAR, extver].data,
                      var_array_final).astype(var_dtype)
            else:
                # Create the variance AstroData object
                var = AstroData(data=var_array_final)
                var.rename_ext(VAR, ver=extver)
                var.filename = adinput.filename
                
                # Call the _update_var_header helper function to update the
                # header of the variance extension with some useful keywords
                var = self._update_var_header(sci=ext, var=var, bunit=bunit)
                
                # Append the variance AstroData object to the input AstroData
                # object. 
                log.fullinfo("Adding the [%s,%d] extension to the input "
                             "AstroData object %s" % (VAR, extver,
                                                      adinput.filename))
                adinput.append(moredata=var)
        
        return adinput
Ejemplo n.º 27
0
 def addVAR(self, rc):
     """
     This primitive calculates the variance of each science extension in the
     input AstroData object and adds the variance as an additional
     extension. This primitive will determine the units of the pixel data in
     the input science extension and calculate the variance in the same
     units. The two main components of the variance can be calculated and
     added separately, if desired, using the following formula:
     
     variance(read_noise) [electrons] = (read_noise [electrons])^2 
     variance(read_noise) [ADU] = ((read_noise [electrons]) / gain)^2
     
     variance(poisson_noise) [electrons] =
         (number of electrons in that pixel)
     variance(poisson_noise) [ADU] =
         ((number of electrons in that pixel) / gain)
     
     The pixel data in the variance extensions will be the same size as the
     pixel data in the science extension.
     
     The read noise component of the variance can be calculated and added to
     the variance extension at any time, but should be done before
     performing operations with other datasets.
     
     The Poisson noise component of the variance can be calculated and added
     to the variance extension only after any bias levels have been
     subtracted from the pixel data in the science extension. 
     
     The variance of a raw bias frame contains only a read noise component
     (which represents the uncertainty in the bias level of each pixel),
     since the Poisson noise component of a bias frame is meaningless.
     
     :param read_noise: set to True to add the read noise component of the
                        variance to the variance extension
     :type read_noise: Python boolean
     
     :param poisson_noise: set to True to add the Poisson noise component
                           of the variance to the variance extension
     :type poisson_noise: Python boolean
     
     """
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     # Log the standard "starting primitive" debug message
     log.debug(gt.log_message("primitive", "addVAR", "starting"))
     
     # Define the keyword to be used for the time stamp for this primitive
     timestamp_key = self.timestamp_keys["addVAR"]
     
     # Initialize the list of output AstroData objects
     adoutput_list = []
     
     # Check to see what component of variance will be added and whether it
     # is sensible to do so
     read_noise = rc["read_noise"]
     poisson_noise = rc["poisson_noise"]
     
     if read_noise and poisson_noise:
         log.stdinfo("Adding the read noise component and the poisson "
                     "noise component of the variance")
     if read_noise and not poisson_noise:
         log.stdinfo("Adding the read noise component of the variance")
     if not read_noise and poisson_noise:
         log.stdinfo("Adding the poisson noise component of the variance")
     if not read_noise and not poisson_noise:
         log.warning("Cannot add a variance extension since no variance "
                     "component has been selected")
     
     # Loop over each input AstroData object in the input list
     for ad in rc.get_inputs_as_astrodata():
         
         if poisson_noise and "BIAS" in ad.types:
             log.warning("It is not recommended to add a poisson noise "
                         "component to the variance of a bias frame")
         if (poisson_noise and "GMOS" in ad.types and not
             ad.phu_get_key_value(self.timestamp_keys["subtractBias"])):
             
             log.warning("It is not recommended to calculate a poisson "
                         "noise component of the variance using data that "
                         "still contains a bias level")
         
         # Call the _calculate_var helper function to calculate and add the
         # variance extension to the input AstroData object
         ad = self._calculate_var(adinput=ad, add_read_noise=read_noise,
                                  add_poisson_noise=poisson_noise)
         
         # Add the appropriate time stamps to the PHU
         gt.mark_history(adinput=ad, keyword=timestamp_key)
         
         # Change the filename
         ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
                                           strip=True)
         
         # Append the output AstroData object to the list of output
         # AstroData objects
         adoutput_list.append(ad)
     
     # Report the list of output AstroData objects to the reduction context
     rc.report_output(adoutput_list)
     
     yield rc
Ejemplo n.º 28
0
    def addDQ(self, rc):
        """
        This primitive is used to add a DQ extension to the input AstroData
        object. The value of a pixel in the DQ extension will be the sum of the
        following: (0=good, 1=bad pixel (found in bad pixel mask), 2=pixel is
        in the non-linear regime, 4=pixel is saturated). This primitive will
        trim the BPM to match the input AstroData object(s).
        
        :param bpm: The file name, including the full path, of the BPM(s) to be
                    used to flag bad pixels in the DQ extension. If only one
                    BPM is provided, that BPM will be used to flag bad pixels
                    in the DQ extension for all input AstroData object(s). If
                    more than one BPM is provided, the number of BPMs must
                    match the number of input AstroData objects. If no BPM is
                    provided, the primitive will attempt to determine an
                    appropriate BPM.
        :type bpm: string or list of strings
        """
        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "addDQ", "starting"))
        
        # Define the keyword to be used for the time stamp for this primitive
        timestamp_key = self.timestamp_keys["addDQ"]
        
        # Initialize the list of output AstroData objects
        adoutput_list = []
        
        # Set the data type of the data quality array
        # It can be uint8 for now, it will get converted up as we assign higher bit values
        # shouldn't need to force it up to 16bpp yet.
        dq_dtype = np.dtype(np.uint8)
        #dq_dtype = np.dtype(np.uint16)
        
        # Get the input AstroData objects
        adinput = rc.get_inputs_as_astrodata()
        
        # Loop over each input AstroData object in the input list
        for ad in adinput:
            
            # Check whether the addDQ primitive has been run previously
            if ad.phu_get_key_value(timestamp_key):
                log.warning("No changes will be made to %s, since it has "
                            "already been processed by addDQ" % ad.filename)
                
                # Append the input AstroData object to the list of output
                # AstroData objects without further processing
                adoutput_list.append(ad)
                continue
            
            # Parameters specified on the command line to reduce are converted
            # to strings, including None
            ##M What about if a user doesn't want to add a BPM at all?
            ##M Are None's not converted to Nonetype from the command line?
            if rc["bpm"] and rc["bpm"] != "None":
                # The user supplied an input to the bpm parameter
                bpm = rc["bpm"]
            else:
                # The user did not supply an input to the bpm parameter, so try
                # to find an appropriate one. Get the dictionary containing the
                # list of BPMs for all instruments and modes.
                all_bpm_dict = Lookups.get_lookup_table("Gemini/BPMDict",
                                                        "bpm_dict")
                
                # Call the _get_bpm_key helper function to get the key for the
                # lookup table 
                key = self._get_bpm_key(ad)
                
                # Get the appropriate BPM from the look up table
                if key in all_bpm_dict:
                    bpm = lookup_path(all_bpm_dict[key])
                else:
                    bpm = None
                    log.warning("No BPM found for %s, no BPM will be "
                                "included" % ad.filename)

            # Ensure that the BPMs are AstroData objects
            bpm_ad = None
            if bpm is not None:
                log.fullinfo("Using %s as BPM" % str(bpm))
                if isinstance(bpm, AstroData):
                    bpm_ad = bpm
                else:
                    bpm_ad = AstroData(bpm)
                    ##M Do we want to fail here depending on context?
                    if bpm_ad is None:
                        log.warning("Cannot convert %s into an AstroData "
                                    "object, no BPM will be added" % bpm)

            final_bpm = None
            if bpm_ad is not None:
                # Clip the BPM data to match the size of the input AstroData
                # object science and pad with overscan region, if necessary
                final_bpm = gt.clip_auxiliary_data(adinput=ad, aux=bpm_ad,
                                                   aux_type="bpm")[0]

            # Get the non-linear level and the saturation level using the
            # appropriate descriptors - Individual values get checked in the
            # next loop 
            non_linear_level_dv = ad.non_linear_level()
            saturation_level_dv = ad.saturation_level()

            # Loop over each science extension in each input AstroData object
            for ext in ad[SCI]:
                
                # Retrieve the extension number for this extension
                extver = ext.extver()
                
                # Check whether an extension with the same name as the DQ
                # AstroData object already exists in the input AstroData object
                if ad[DQ, extver]:
                    log.warning("A [%s,%d] extension already exists in %s"
                                % (DQ, extver, ad.filename))
                    continue
                
                # Get the non-linear level and the saturation level for this
                # extension
                non_linear_level = non_linear_level_dv.get_value(extver=extver)
                saturation_level = saturation_level_dv.get_value(extver=extver)

                # To store individual arrays created for each of the DQ bit
                # types
                dq_bit_arrays = []

                # Create an array that contains pixels that have a value of 2
                # when that pixel is in the non-linear regime in the input
                # science extension
                if non_linear_level is not None:
                    non_linear_array = None
                    if saturation_level is not None:
                        # Test the saturation level against non_linear level
                        # They can be the same or the saturation level can be
                        # greater than but not less than the non-linear level.
                        # If they are the same then only flag saturated pixels
                        # below. This just means not creating an unneccessary
                        # intermediate array.
                        if saturation_level > non_linear_level:
                            log.fullinfo("Flagging pixels in the DQ extension "
                                         "corresponding to non linear pixels "
                                         "in %s[%s,%d] using non linear "
                                         "level = %.2f" % (ad.filename, SCI,
                                                           extver,
                                                           non_linear_level))

                            non_linear_array = np.where(
                                ((ext.data >= non_linear_level) &
                                (ext.data < saturation_level)), 2, 0)
                            
                        elif saturation_level < non_linear_level:
                            log.warning("%s[%s,%d] saturation_level value is"
                                        "less than the non_linear_level not"
                                        "flagging non linear pixels" %
                                        (ad.filname, SCI, extver))
                        else:
                            log.fullinfo("Saturation and non-linear values "
                                         "for %s[%s,%d] are the same. Only "
                                         "flagging saturated pixels."
                                         % (ad.filename, SCI, extver))
                            
                    else:
                        log.fullinfo("Flagging pixels in the DQ extension "
                                     "corresponding to non linear pixels "
                                     "in %s[%s,%d] using non linear "
                                     "level = %.2f" % (ad.filename, SCI, extver,
                                                       non_linear_level))

                        non_linear_array = np.where(
                            (ext.data >= non_linear_level), 2, 0)
                    
                    dq_bit_arrays.append(non_linear_array)

                # Create an array that contains pixels that have a value of 4
                # when that pixel is saturated in the input science extension
                if saturation_level is not None:
                    saturation_array = None
                    log.fullinfo("Flagging pixels in the DQ extension "
                                 "corresponding to saturated pixels in "
                                 "%s[%s,%d] using saturation level = %.2f" %
                                 (ad.filename, SCI, extver, saturation_level))
                    saturation_array = np.where(
                        ext.data >= saturation_level, 4, 0)
                    dq_bit_arrays.append(saturation_array)
                
                # BPMs have an EXTNAME equal to DQ
                bpmname = None
                if final_bpm is not None:
                    bpm_array = None
                    bpmname = os.path.basename(final_bpm.filename)
                    log.fullinfo("Flagging pixels in the DQ extension "
                                 "corresponding to bad pixels in %s[%s,%d] "
                                 "using the BPM %s[%s,%d]" %
                                 (ad.filename, SCI, extver, bpmname, DQ, extver))
                    bpm_array = final_bpm[DQ, extver].data
                    dq_bit_arrays.append(bpm_array)
                
                # Create a single DQ extension from the three arrays (BPM,
                # non-linear and saturated)
                if not dq_bit_arrays:
                    # The BPM, non-linear and saturated arrays were not
                    # created. Create a single DQ array with all pixels set
                    # equal to 0 
                    log.fullinfo("The BPM, non-linear and saturated arrays "
                                 "were not created. Creating a single DQ "
                                 "array with all the pixels set equal to zero")
                    final_dq_array = np.zeros(ext.data.shape).astype(dq_dtype)

                else:
                    final_dq_array = self._bitwise_OR_list(dq_bit_arrays)
                    final_dq_array = final_dq_array.astype(dq_dtype)
                
                # Create a data quality AstroData object
                dq = AstroData(data=final_dq_array)
                dq.rename_ext(DQ, ver=extver)
                dq.filename = ad.filename
                
                # Call the _update_dq_header helper function to update the
                # header of the data quality extension with some useful
                # keywords
                dq = self._update_dq_header(sci=ext, dq=dq, bpmname=bpmname)
                
                # Append the DQ AstroData object to the input AstroData object
                log.fullinfo("Adding extension [%s,%d] to %s"
                             % (DQ, extver, ad.filename))
                ad.append(moredata=dq)
            
            # Add the appropriate time stamps to the PHU
            gt.mark_history(adinput=ad, keyword=timestamp_key)
            
            # Change the filename
            ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
                                              strip=True)
            
            # Append the output AstroData object to the list of output
            # AstroData objects
            adoutput_list.append(ad)

        # Report the list of output AstroData objects to the reduction context
        rc.report_output(adoutput_list)
        
        yield rc
Ejemplo n.º 29
0
 def addMDF(self, rc):
     """
     This primitive is used to add an MDF extension to the input AstroData
     object. If only one MDF is provided, that MDF will be add to all input
     AstroData object(s). If more than one MDF is provided, the number of
     MDF AstroData objects must match the number of input AstroData objects.
     If no MDF is provided, the primitive will attempt to determine an
     appropriate MDF.
     
     :param mdf: The file name of the MDF(s) to be added to the input(s)
     :type mdf: string
     
     """
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     # Log the standard "starting primitive" debug message
     log.debug(gt.log_message("primitive", "addMDF", "starting"))
     
     # Define the keyword to be used for the time stamp for this primitive
     timestamp_key = self.timestamp_keys["addMDF"]
     
     # Initialize the list of output AstroData objects
     adoutput_list = []
     
     # Get the input AstroData objects
     adinput = rc.get_inputs_as_astrodata()
     
     # Loop over each input AstroData object in the input list
     for ad in adinput:
         
         # Check whether the addMDF primitive has been run previously
         if ad.phu_get_key_value(timestamp_key):
             log.warning("No changes will be made to %s, since it has "
                         "already been processed by addMDF" % ad.filename)
             
             # Append the input AstroData object to the list of output
             # AstroData objects without further processing
             adoutput_list.append(ad)
             continue
         
         # Check whether the input is spectroscopic data
         if "SPECT" not in ad.types:
             log.stdinfo("%s is not spectroscopic data, so no MDF will be "
                         "added" % ad.filename)
             
             # Append the input AstroData object to the list of output
             # AstroData objects without further processing
             adoutput_list.append(ad)
             continue
         
         # Check whether an MDF extension already exists in the input
         # AstroData object
         if ad["MDF"]:
             log.warning("An MDF extension already exists in %s, so no MDF "
                         "will be added" % ad.filename)
             
             # Append the input AstroData object to the list of output
             # AstroData objects without further processing
             adoutput_list.append(ad)
             continue
         
         # Parameters specified on the command line to reduce are converted
         # to strings, including None
         if rc["mdf"] and rc["mdf"] != "None":
             # The user supplied an input to the mdf parameter
             mdf = rc["mdf"]
         else:
             # The user did not supply an input to the mdf parameter, so try
             # to find an appropriate one. Get the dictionary containing the
             # list of MDFs for all instruments and modes.
             all_mdf_dict = Lookups.get_lookup_table("Gemini/MDFDict",
                                                     "mdf_dict")
             
             # The MDFs are keyed by the instrument and the MASKNAME. Get
             # the instrument and the MASKNAME values using the appropriate
             # descriptors 
             instrument = ad.instrument()
             mask_name = ad.phu_get_key_value("MASKNAME")
             
             # Create the key for the lookup table
             if instrument is None or mask_name is None:
                 log.warning("Unable to create the key for the lookup "
                             "table (%s), so no MDF will be added"
                             % ad.exception_info)
                 
                 # Append the input AstroData object to the list of output
                 # AstroData objects without further processing
                 adoutput_list.append(ad)
                 continue
             
             key = "%s_%s" % (instrument, mask_name)
             
             # Get the appropriate MDF from the look up table
             if key in all_mdf_dict:
                 mdf = lookup_path(all_mdf_dict[key])
             else:
                 # The MASKNAME keyword defines the actual name of an MDF
                 if not mask_name.endswith(".fits"):
                     mdf = "%s.fits" % mask_name
                 else:
                     mdf = str(mask_name)
                 
                 # Check if the MDF exists in the current working directory
                 if not os.path.exists(mdf):
                     log.warning("The MDF %s was not found in the current "
                                 "working directory, so no MDF will be "
                                 "added" % mdf)
                 
                 # Append the input AstroData object to the list of output
                 # AstroData objects without further processing
                 adoutput_list.append(ad)
                 continue
         
         # Ensure that the MDFs are AstroData objects
         if not isinstance(mdf, AstroData):
             mdf_ad = AstroData(mdf)
         
         if mdf_ad is None:
             log.warning("Cannot convert %s into an AstroData object, so "
                         "no MDF will be added" % mdf)
             
             # Append the input AstroData object to the list of output
             # AstroData objects without further processing
             adoutput_list.append(ad)
             continue
         
         # Check if the MDF is a single extension fits file
         if len(mdf_ad) > 1:
             log.warning("The MDF %s is not a single extension fits file, "
                         "so no MDF will be added" % mdf)
             
             # Append the input AstroData object to the list of output
             # AstroData objects without further processing
             adoutput_list.append(ad)
             continue
             
         # Name the extension appropriately
         mdf_ad.rename_ext("MDF", 1)
         
         # Append the MDF AstroData object to the input AstroData object
         log.fullinfo("Adding the MDF %s to the input AstroData object "
                      "%s" % (mdf_ad.filename, ad.filename))
         ad.append(moredata=mdf_ad)
         
         # Add the appropriate time stamps to the PHU
         gt.mark_history(adinput=ad, keyword=timestamp_key)
         
         # Change the filename
         ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"],
                                           strip=True)
         
         # Append the output AstroData object to the list of output
         # AstroData objects
         adoutput_list.append(ad)
     
     # Report the list of output AstroData objects to the reduction context
     rc.report_output(adoutput_list)
     
     yield rc
Ejemplo n.º 30
0
    def attachWavelengthSolution(self,rc):

        # Instantiate the log
        log = logutils.get_logger(__name__)
        
        # Define the keyword to be used for the time stamp
        timestamp_key = self.timestamp_keys["attachWavelengthSolution"]

        # Log the standard "starting primitive" debug message
        log.debug(gt.log_message("primitive", "attachWavelengthSolution",
                                 "starting"))

        # Initialize the list of output AstroData objects
        adoutput_list = []

        # Check for a user-supplied arc
        adinput = rc.get_inputs_as_astrodata()
        arc_param = rc["arc"]
        arc_dict = None
        if arc_param is not None:
            # The user supplied an input to the arc parameter
            if not isinstance(arc_param, list):
                arc_list = [arc_param]
            else:
                arc_list = arc_param

            # Convert filenames to AD instances if necessary
            tmp_list = []
            for arc in arc_list:
                if type(arc) is not AstroData:
                    arc = AstroData(arc)
                tmp_list.append(arc)
            arc_list = tmp_list
            
            arc_dict = gt.make_dict(key_list=adinput, value_list=arc_list)

        for ad in adinput:
            if arc_dict is not None:
                arc = arc_dict[ad]
            else:
                arc = rc.get_cal(ad, "processed_arc")
            
                # Take care of the case where there was no arc 
                if arc is None:
                    log.warning("Could not find an appropriate arc for %s" \
                                % (ad.filename))
                    adoutput_list.append(ad)
                    continue
                else:
                    arc = AstroData(arc)

            wavecal = arc["WAVECAL"]
            if wavecal is not None:
                # Remove old versions
                if ad["WAVECAL"] is not None:
                    for wc in ad["WAVECAL"]:
                        ad.remove((wc.extname(),wc.extver()))
                # Append new solution
                ad.append(wavecal)

                # Add the appropriate time stamps to the PHU
                gt.mark_history(adinput=ad, keyword=timestamp_key)

                # Change the filename
                ad.filename = gt.filename_updater(adinput=ad, 
                                                  suffix=rc["suffix"], 
                                                  strip=True)
                adoutput_list.append(ad)
            else:
                log.warning("No wavelength solution found for %s" % ad.filename)
                adoutput_list.append(ad)

        # Report the list of output AstroData objects to the reduction
        # context
        rc.report_output(adoutput_list)
      
        yield rc
Ejemplo n.º 31
0
 def standardizeGeminiHeaders(self, rc):
     """
     This primitive is used to make the changes and additions to the
     keywords in the headers of Gemini data.
     """
     # Instantiate the log
     log = logutils.get_logger(__name__)
     
     # Log the standard "starting primitive" debug message
     log.debug(gt.log_message("primitive", "standardizeGeminiHeaders",
                              "starting"))
     
     # Define the keyword to be used for the time stamp for this primitive
     timestamp_key = self.timestamp_keys["standardizeGeminiHeaders"]
     
     # Initialize the list of output AstroData objects
     adoutput_list = []
     
     # Loop over each input AstroData object in the input list
     for ad in rc.get_inputs_as_astrodata():
         
         # Check whether the standardizeGeminiHeaders primitive has been run
         # previously
         if ad.phu_get_key_value(timestamp_key):
             log.warning("No changes will be made to %s, since it has "
                         "already been processed by "
                         "standardizeGeminiHeaders" % ad.filename)
             
             # Append the input AstroData object to the list of output
             # AstroData objects without further processing
             adoutput_list.append(ad)
             continue
         
         # Standardize the headers of the input AstroData object. Update the
         # keywords in the headers that are common to all Gemini data.
         log.status("Updating keywords that are common to all Gemini data")
         
         # Original name
         ad.store_original_name()
         
         # Number of science extensions
         gt.update_key(adinput=ad, keyword="NSCIEXT",
                       value=ad.count_exts("SCI"), comment=None,
                       extname="PHU") 
         
         # Number of extensions
         gt.update_key(adinput=ad, keyword="NEXTEND", value=len(ad),
                       comment=None, extname="PHU")
         
         # Physical units (assuming raw data has units of ADU)
         gt.update_key(adinput=ad, keyword="BUNIT", value="adu",
                       comment=None, extname="SCI")
         
         # Add the appropriate time stamps to the PHU
         gt.mark_history(adinput=ad, keyword=timestamp_key)
         
         # Change the filename
         ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"], 
                                           strip=True)
         
         # Append the output AstroData object to the list of output
         # AstroData objects 
         adoutput_list.append(ad)
     
     # Report the list of output AstroData objects to the reduction context
     rc.report_output(adoutput_list)
     
     yield rc