def test_clip_auxiliary_data(self): ad = astrodata.open(os.path.join(TESTDATAPATH, 'NIRI', 'N20160620S0035.fits')) bpm_ad = astrodata.open('geminidr/niri/lookups/BPM/NIRI_bpm.fits') ret = gt.clip_auxiliary_data(ad, bpm_ad, 'bpm', np.int16) assert ret[0].data.shape == ad[0].data.shape assert np.all(ret[0].data == bpm_ad[0].data[256:768,256:768])
def addLatencyToDQ(self, adinputs=None, **params): """ Flags pixels in the DQ plane of an image based on whether the same pixel has been flagged as saturated in a previous image. Parameters ---------- suffix: str suffix to be added to output files non_linear : bool flag non-linear pixels (as well as saturated ones)? time: float time (in seconds) for which latency is an issue """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) flags = DQ.saturated | (DQ.non_linear if params["non_linear"] else 0) # Create a timedelta object using the value of the "time" parameter seconds = datetime.timedelta(seconds=params["time"]) # Avoids n^2 calls to the descriptor times = [ad.ut_datetime() for ad in adinputs] for i, ad in enumerate(adinputs): # Find which frames have their bright pixels propagated propagated = [ x for x in zip(adinputs, times) if (x[1] < times[i] and times[i] - x[1] < seconds) ] if propagated: log.stdinfo('{} affected by {}'.format( ad.filename, ','.join([x[0].filename for x in propagated]))) for ad_latent in list(zip(*propagated))[0]: # AD extensions might not be in the same order # Set aux_type to 'bpm' which means hot pixels in a subarray # can still be propagated to a subsequent full-array image ad_latent = gt.clip_auxiliary_data(ad, aux=ad_latent, aux_type='bpm') for ext, ext_latent in zip(ad, ad_latent): if ext_latent.mask is not None: latency = np.where(ext_latent.mask & flags, DQ.cosmic_ray, 0).astype(DQ.datatype) ext.mask = latency if ext.mask is None \ else ext.mask | latency else: log.stdinfo('{} is not affected by latency'.format( ad.filename)) ad.update_filename(suffix=params["suffix"], strip=True) return adinputs
def addIllumMaskToDQ(self, adinputs=None, suffix=None, illum_mask=None): """ Adds an illumination mask to each AD object Parameters ---------- suffix: str suffix to be added to output files mask: str/None name of illumination mask mask (None -> use default) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] # Getting all the filenames first prevents reopening the same file # for each science AD if illum_mask is None: illum_mask = [self._get_illum_mask_filename(ad) for ad in adinputs] for ad, illum in zip( *gt.make_lists(adinputs, illum_mask, force_ad=True)): if ad.phu.get(timestamp_key): log.warning( 'No changes will be made to {}, since it has ' 'already been processed by addIllumMaskToDQ'.format( ad.filename)) continue if illum is None: # So it can be zipped with the AD final_illum = [None] * len(ad) else: log.fullinfo("Using {} as illumination mask".format( illum.filename)) final_illum = gt.clip_auxiliary_data(ad, aux=illum, aux_type='bpm', return_dtype=DQ.datatype) for ext, illum_ext in zip(ad, final_illum): if illum_ext is not None: # Ensure we're only adding the unilluminated bit iext = np.where(illum_ext.data > 0, DQ.unilluminated, 0).astype(DQ.datatype) ext.mask = iext if ext.mask is None else ext.mask | iext # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def addIllumMaskToDQ(self, adinputs=None, suffix=None, illum_mask=None): """ Adds an illumination mask to each AD object Parameters ---------- suffix: str suffix to be added to output files mask: str/None name of illumination mask mask (None -> use default) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] # Getting all the filenames first prevents reopening the same file # for each science AD if illum_mask is None: illum_mask = [self._get_illum_mask_filename(ad) for ad in adinputs] for ad, illum in zip(*gt.make_lists(adinputs, illum_mask, force_ad=True)): if ad.phu.get(timestamp_key): log.warning('No changes will be made to {}, since it has ' 'already been processed by addIllumMaskToDQ'. format(ad.filename)) continue if illum is None: # So it can be zipped with the AD final_illum = [None] * len(ad) else: log.fullinfo("Using {} as illumination mask".format(illum.filename)) final_illum = gt.clip_auxiliary_data(ad, aux=illum, aux_type='bpm', return_dtype=DQ.datatype) for ext, illum_ext in zip(ad, final_illum): if illum_ext is not None: # Ensure we're only adding the unilluminated bit iext = np.where(illum_ext.data > 0, DQ.unilluminated, 0).astype(DQ.datatype) ext.mask = iext if ext.mask is None else ext.mask | iext # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def addLatencyToDQ(self, adinputs=None, **params): """ Flags pixels in the DQ plane of an image based on whether the same pixel has been flagged as saturated in a previous image. Parameters ---------- suffix: str suffix to be added to output files non_linear : bool flag non-linear pixels (as well as saturated ones)? time: float time (in seconds) for which latency is an issue """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) flags = DQ.saturated | (DQ.non_linear if params["non_linear"] else 0) # Create a timedelta object using the value of the "time" parameter seconds = datetime.timedelta(seconds=params["time"]) # Avoids n^2 calls to the descriptor times = [ad.ut_datetime() for ad in adinputs] for i, ad in enumerate(adinputs): # Find which frames have their bright pixels propagated propagated = [x for x in zip(adinputs, times) if (x[1]<times[i] and times[i]-x[1]<seconds)] if propagated: log.stdinfo('{} affected by {}'.format(ad.filename, ','.join([x[0].filename for x in propagated]))) for ad_latent in list(zip(*propagated))[0]: # AD extensions might not be in the same order # Set aux_type to 'bpm' which means hot pixels in a subarray # can still be propagated to a subsequent full-array image ad_latent = gt.clip_auxiliary_data(ad, aux=ad_latent, aux_type='bpm') for ext, ext_latent in zip(ad, ad_latent): if ext_latent.mask is not None: latency = np.where(ext_latent.mask & flags, DQ.cosmic_ray, 0).astype(DQ.datatype) ext.mask = latency if ext.mask is None \ else ext.mask | latency else: log.stdinfo('{} is not affected by latency'.format(ad.filename)) ad.update_filename(suffix=params["suffix"], strip=True) return adinputs
def fringeCorrect(self, adinputs=None, **params): """ Correct science frames for the effects of fringing, using a fringe frame. The fringe frame is obtained either from a specified parameter, or the "fringe" stream, or the calibration database. This is basically a bookkeeping wrapper for subtractFringe(), which does all the work. Parameters ---------- suffix: str suffix to be added to output files fringe: list/str/AstroData/None fringe frame(s) to subtract """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] # Exit now if nothing needs a correction, to avoid an error when the # calibration search fails. If images with different exposure times # are used, some frames may not require a correction (but the calibration # search will succeed), so still need to check individual inputs later. if not any(self._needs_fringe_correction(ad) for ad in adinputs): log.stdinfo("No input images require a fringe correction.") return adinputs fringe = params["fringe"] scale = params["scale"] if fringe is None: try: fringe_list = self.streams['fringe'] assert len(fringe_list) == 1 scale = False log.stdinfo("Using fringe frame in 'fringe' stream. " "Setting scale=False") except (KeyError, AssertionError): self.getProcessedFringe(adinputs) fringe_list = self._get_cal(adinputs, "processed_fringe") else: fringe_list = fringe # Usual stuff to ensure that we have an iterable of the correct length # for the scale factors regardless of what the input is scale_factor = params["scale_factor"] try: factors = iter(scale_factor) except TypeError: factors = iter([scale_factor] * len(adinputs)) else: # In case a single-element list was passed if len(scale_factor) == 1: factors = iter(scale_factor * len(adinputs)) # Get a fringe AD object for every science frame for ad, fringe in zip(*gt.make_lists(adinputs, fringe_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by subtractFringe". format(ad.filename)) continue # Check the inputs have matching filters, binning, and shapes try: gt.check_inputs_match(ad, fringe) except ValueError: fringe = gt.clip_auxiliary_data(adinput=ad, aux=fringe, aux_type="cal") gt.check_inputs_match(ad, fringe) if scale: factor = next(factors) if factor is None: factor = self._calculate_fringe_scaling(ad, fringe) log.stdinfo("Scaling fringe frame by factor {:.3f} before " "subtracting from {}".format(factor, ad.filename)) # Since all elements of fringe_list might be references to the # same AD, need to make a copy before multiplying fringe_copy = deepcopy(fringe) fringe_copy.multiply(factor) ad.subtract(fringe_copy) else: ad.subtract(fringe) # Timestamp and update header and filename ad.phu.set("FRINGEIM", fringe.filename, self.keyword_comments["FRINGEIM"]) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=params["suffix"], strip=True) return adinputs
def biasCorrect(self, adinputs=None, suffix=None, bias=None, do_bias=True): """ The biasCorrect primitive will subtract the science extension of the input bias frames from the science extension of the input science frames. The variance and data quality extension will be updated, if they exist. If no bias is provided, getProcessedBias will be called to ensure a bias exists for every adinput. Parameters ---------- suffix: str suffix to be added to output files bias: str/list of str bias(es) to subtract do_bias: bool perform bias subtraction? """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] if not do_bias: log.warning("Bias correction has been turned off.") return adinputs if bias is None: self.getProcessedBias(adinputs, refresh=False) bias_list = self._get_cal(adinputs, 'processed_bias') else: bias_list = bias # Provide a bias AD object for every science frame for ad, bias in zip(*gt.make_lists(adinputs, bias_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by biasCorrect". format(ad.filename)) continue if bias is None: if 'qa' in self.mode: log.warning("No changes will be made to {}, since no " "bias was specified".format(ad.filename)) continue else: raise IOError('No processed bias listed for {}'. format(ad.filename)) try: gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) except ValueError: bias = gt.clip_auxiliary_data(ad, aux=bias, aux_type='cal') # An Error will be raised if they don't match now gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) log.fullinfo('Subtracting this bias from {}:\n{}'. format(ad.filename, bias.filename)) ad.subtract(bias) # Record bias used, timestamp, and update filename ad.phu.set('BIASIM', bias.filename, self.keyword_comments['BIASIM']) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def pointing_in_field(pos, refpos, frac_FOV=1.0, frac_slit=1.0): """ See gemini_tools.pointing_in_field() for the API. This is an instrument-specific back end that you shouldn't be calling directly. No inputs are validated at this level; that's the responsibility of the calling function, for reasons of efficiency. The GNIRS FOV is determined by whether the calculated center point (according to the center of mass of the illumination mask) of the image falls within the illumination mask of the reference image. :param pos: AstroData instance to be checked for whether it belongs in the same sky grouping as refpos :type pos: AstroData instance :param refpos: This is the POFFSET and QOFFSET of the reference image :type refpos: tuple of floats :param frac_FOV: For use with spectroscopy data :type frac_FOV: float :param frac_slit: For use with spectroscopy data :type frac_slit: float """ # Since this function gets looked up and evaluated, we have to do any # essential imports in-line (but Python caches them) import math # Extract pointing info in terms of the x and y offsets xshift = refpos[1] - pos.phu['QOFFSET'] yshift = refpos[0] - pos.phu['POFFSET'] ad = pos # Imaging: if 'IMAGE' in pos.tags: illum = get_illum_mask_filename(ad) if illum: illum_ad = gt.clip_auxiliary_data(adinput=pos, aux=astrodata.open(illum), aux_type="bpm") illum_data = illum_ad[0].data else: raise OSError("Cannot find illumination mask for {}". format(ad.filename)) # Finding the center of the illumination mask center_illum = (illum_ad.phu['CENMASSX'], illum_ad.phu['CENMASSY']) checkpos = (int(center_illum[0] + xshift), int(center_illum[1] + yshift)) # If the position to check is going to fall outside the illumination # mask, return straight away to avoid an error if ((abs(xshift) >= abs(center_illum[0])) or (abs(yshift) >= abs(center_illum[1]))): return False # Note that numpy data arrays are reversed in x and y return illum_data[checkpos[1], checkpos[0]] == 0 # Spectroscopy: elif 'SPECT' in ad.tags: raise NotImplementedError("FOV lookup not yet supported for GNIRS " "Spectroscopy") # Some engineering observation or bad mask value etc.: else: raise ValueError("Can't determine FOV for unrecognized GNIRS config " "({}, {})".format(ad.focal_plane_mask(), ad.disperser()))
def addIllumMaskToDQ(self, adinputs=None, **params): """ This primitive combines the illumination mask from the lookup directory into the DQ plane Parameters ---------- suffix: str suffix to be added to output files """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) # Since this primitive needs a reference, it must no-op without any if not adinputs: return adinputs # Get list of input and identify a suitable reference frame. # In most case it will be the first image, but for lamp-on, lamp-off # flats, one wants the reference frame to be a lamp-on since there's # next to no signal in the lamp-off. # # BEWARE: See note on the only GCAL_IR_OFF case below this block. lampons = self.selectFromInputs(adinputs, 'GCAL_IR_ON') reference = lampons[0] if lampons else adinputs[0] # When only a GCAL_IR_OFF is available: # To cover the one-at-a-time mode check for a compatible list # if list found, try to find a lamp-on in there to use as # reference for the mask and the shifts. # The mask's name and the shifts should stored in the headers # of the reference to simplify this and speed things up. # # NOT NEEDED NOW because we calling addIllumMask after the # lamp-offs have been subtracted. But kept the idea here # in case we needed. # Fetching a corrected illumination mask with a keyhole that aligns # with the science data illum = self._get_illum_mask_filename(reference) if illum is None: log.warning("No illumination mask found for {}, no mask can " "be added to the DQ planes of the inputs". format(reference.filename)) return adinputs illum_ad = astrodata.open(illum) corr_illum_ad = _position_illum_mask(reference, illum_ad, log) for ad in adinputs: final_illum = gt.clip_auxiliary_data(ad, aux=corr_illum_ad, aux_type="bpm") # binary_OR the illumination mask or create a DQ plane from it. if ad[0].mask is None: ad[0].mask = final_illum[0].data else: ad[0].mask |= final_illum[0].data # Update the filename ad.update_filename(suffix=params["suffix"], strip=True) return adinputs
def addDQ(self, adinputs=None, **params): """ This primitive is used to add a DQ extension to the input AstroData object. The value of a pixel in the DQ extension will be the sum of the following: (0=good, 1=bad pixel (found in bad pixel mask), 2=pixel is in the non-linear regime, 4=pixel is saturated). This primitive will trim the BPM to match the input AstroData object(s). Parameters ---------- suffix: str suffix to be added to output files static_bpm: str Name of bad pixel mask ("default" -> use default from look-up table) If set to None, no static_bpm will be added. user_bpm: str Name of the bad pixel mask created by the user from flats and darks. It is an optional BPM that can be added to the static one. illum_mask: bool add illumination mask? """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys["addDQ"] sfx = params["suffix"] # Getting all the filenames first prevents reopening the same file # for each science AD static_bpm_list = params['static_bpm'] user_bpm_list = params['user_bpm'] if static_bpm_list == "default": static_bpm_list = [self._get_bpm_filename(ad) for ad in adinputs] for ad, static, user in zip(*gt.make_lists( adinputs, static_bpm_list, user_bpm_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning('No changes will be made to {}, since it has ' 'already been processed by addDQ'.format( ad.filename)) continue if static is None: # So it can be zipped with the AD final_static = [None] * len(ad) else: log.fullinfo("Using {} as static BPM".format(static.filename)) final_static = gt.clip_auxiliary_data(ad, aux=static, aux_type='bpm', return_dtype=DQ.datatype) if user is None: final_user = [None] * len(ad) else: log.fullinfo("Using {} as user BPM".format(user.filename)) final_user = gt.clip_auxiliary_data(ad, aux=user, aux_type='bpm', return_dtype=DQ.datatype) for ext, static_ext, user_ext in zip(ad, final_static, final_user): extver = ext.hdr['EXTVER'] if ext.mask is not None: log.warning( 'A mask already exists in extver {}'.format(extver)) continue non_linear_level = ext.non_linear_level() saturation_level = ext.saturation_level() # Need to create the array first for 3D raw F2 data, with 2D BPM ext.mask = np.zeros_like(ext.data, dtype=DQ.datatype) if static_ext is not None: ext.mask |= static_ext.data if user_ext is not None: ext.mask |= user_ext.data if saturation_level: log.fullinfo('Flagging saturated pixels in {}:{} ' 'above level {:.2f}'.format( ad.filename, extver, saturation_level)) ext.mask |= np.where(ext.data >= saturation_level, DQ.saturated, 0).astype(DQ.datatype) if non_linear_level: if saturation_level: if saturation_level > non_linear_level: log.fullinfo('Flagging non-linear pixels in {}:{} ' 'above level {:.2f}'.format( ad.filename, extver, non_linear_level)) ext.mask |= np.where( (ext.data >= non_linear_level) & (ext.data < saturation_level), DQ.non_linear, 0).astype(DQ.datatype) # Readout modes of IR detectors can result in # saturated pixels having values below the # saturation level. Flag those. Assume we have an # IR detector here because both non-linear and # saturation levels are defined and nonlin<sat regions, nregions = measurements.label( ext.data < non_linear_level) # In all my tests, region 1 has been the majority # of the image; however, I cannot guarantee that # this is always the case and therefore we should # check the size of each region region_sizes = measurements.labeled_comprehension( ext.data, regions, np.arange(1, nregions + 1), len, int, 0) # First, assume all regions are saturated, and # remove any very large ones. This is much faster # than progressively adding each region to DQ hidden_saturation_array = np.where( regions > 0, 4, 0).astype(DQ.datatype) for region in range(1, nregions + 1): # Limit of 10000 pixels for a hole is a bit arbitrary if region_sizes[region - 1] > 10000: hidden_saturation_array[regions == region] = 0 ext.mask |= hidden_saturation_array elif saturation_level < non_linear_level: log.warning('{}:{} has saturation level less than ' 'non-linear level'.format( ad.filename, extver)) else: log.fullinfo('Saturation and non-linear levels ' 'are the same for {}:{}. Only ' 'flagging saturated pixels'.format( ad.filename, extver)) else: log.fullinfo('Flagging non-linear pixels in {}:{} ' 'above level {:.2f}'.format( ad.filename, extver, non_linear_level)) ext.mask |= np.where(ext.data >= non_linear_level, DQ.non_linear, 0).astype(DQ.datatype) # Handle latency if reqested if params.get("latency", False): try: adinputs = self.addLatencyToDQ(adinputs, time=params["time"], non_linear=params["non_linear"]) except AttributeError: log.warning( "addLatencyToDQ() not defined in primitivesClass " + self.__class__.__name__) # Add the illumination mask if requested if params['add_illum_mask']: adinputs = self.addIllumMaskToDQ(adinputs, illum_mask=params["illum_mask"]) # Timestamp and update filenames for ad in adinputs: gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=sfx, strip=True) return adinputs
def divideByFlat(self, rc): """ This primitive will divide each SCI extension of the inputs by those of the corresponding flat. If the inputs contain VAR or DQ frames, those will also be updated accordingly due to the division on the data. """ # Instantiate the log log = logutils.get_logger(__name__) # Log the standard "starting primitive" debug message log.debug(gt.log_message("primitive", "divideByFlat", "starting")) # Define the keyword to be used for the time stamp for this primitive timestamp_key = self.timestamp_keys["divideByFlat"] # Initialize the list of output AstroData objects adoutput_list = [] # Check for a user-supplied flat adinput = rc.get_inputs_as_astrodata() flat_param = rc["flat"] flat_dict = None if flat_param is not None: # The user supplied an input to the flat parameter if not isinstance(flat_param, list): flat_list = [flat_param] else: flat_list = flat_param # Convert filenames to AD instances if necessary tmp_list = [] for flat in flat_list: if type(flat) is not AstroData: flat = AstroData(flat) tmp_list.append(flat) flat_list = tmp_list flat_dict = gt.make_dict(key_list=adinput, value_list=flat_list) # Loop over each input AstroData object in the input list for ad in adinput: # Check whether the divideByFlat primitive has been run previously if ad.phu_get_key_value(timestamp_key): log.warning("No changes will be made to %s, since it has " \ "already been processed by divideByFlat" \ % (ad.filename)) # Append the input AstroData object to the list of output # AstroData objects without further processing adoutput_list.append(ad) continue # Retrieve the appropriate flat if flat_dict is not None: flat = flat_dict[ad] else: flat = rc.get_cal(ad, "processed_flat") # If there is no appropriate flat, there is no need to divide by # the flat in QA context; in SQ context, raise an error if flat is None: if "qa" in rc.context: log.warning("No changes will be made to %s, since no " \ "appropriate flat could be retrieved" \ % (ad.filename)) # Append the input AstroData object to the list of output # AstroData objects without further processing adoutput_list.append(ad) continue else: raise Errors.PrimitiveError("No processed flat found "\ "for %s" % ad.filename) else: flat = AstroData(flat) # Check the inputs have matching filters, binning, and SCI shapes. try: gt.check_inputs_match(ad1=ad, ad2=flat) except Errors.ToolboxError: # If not, try to clip the flat frame to the size # of the science data # For a GMOS example, this allows a full frame flat to # be used for a CCD2-only science frame. flat = gt.clip_auxiliary_data( adinput=ad,aux=flat,aux_type="cal")[0] # Check again, but allow it to fail if they still don't match gt.check_inputs_match(ad1=ad, ad2=flat) # Divide the adinput by the flat log.fullinfo("Dividing the input AstroData object (%s) " \ "by this flat:\n%s" % (ad.filename, flat.filename)) ad = ad.div(flat) # Record the flat file used ad.phu_set_key_value("FLATIM", os.path.basename(flat.filename), comment=self.keyword_comments["FLATIM"]) # Add the appropriate time stamps to the PHU gt.mark_history(adinput=ad, keyword=timestamp_key) # Change the filename ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"], strip=True) # Append the output AstroData object to the list # of output AstroData objects adoutput_list.append(ad) # Report the list of output AstroData objects to the reduction # context rc.report_output(adoutput_list) yield rc
def addDQ(self, rc): """ This primitive is used to add a DQ extension to the input AstroData object. The value of a pixel in the DQ extension will be the sum of the following: (0=good, 1=bad pixel (found in bad pixel mask), 2=pixel is in the non-linear regime, 4=pixel is saturated). This primitive will trim the BPM to match the input AstroData object(s). :param bpm: The file name, including the full path, of the BPM(s) to be used to flag bad pixels in the DQ extension. If only one BPM is provided, that BPM will be used to flag bad pixels in the DQ extension for all input AstroData object(s). If more than one BPM is provided, the number of BPMs must match the number of input AstroData objects. If no BPM is provided, the primitive will attempt to determine an appropriate BPM. :type bpm: string or list of strings """ # Instantiate the log log = logutils.get_logger(__name__) # Log the standard "starting primitive" debug message log.debug(gt.log_message("primitive", "addDQ", "starting")) # Define the keyword to be used for the time stamp for this primitive timestamp_key = self.timestamp_keys["addDQ"] # Initialize the list of output AstroData objects adoutput_list = [] # Set the data type of the data quality array # It can be uint8 for now, it will get converted up as we assign higher bit values # shouldn't need to force it up to 16bpp yet. dq_dtype = np.dtype(np.uint8) #dq_dtype = np.dtype(np.uint16) # Get the input AstroData objects adinput = rc.get_inputs_as_astrodata() # Loop over each input AstroData object in the input list for ad in adinput: # Check whether the addDQ primitive has been run previously if ad.phu_get_key_value(timestamp_key): log.warning("No changes will be made to %s, since it has " "already been processed by addDQ" % ad.filename) # Append the input AstroData object to the list of output # AstroData objects without further processing adoutput_list.append(ad) continue # Parameters specified on the command line to reduce are converted # to strings, including None ##M What about if a user doesn't want to add a BPM at all? ##M Are None's not converted to Nonetype from the command line? if rc["bpm"] and rc["bpm"] != "None": # The user supplied an input to the bpm parameter bpm = rc["bpm"] else: # The user did not supply an input to the bpm parameter, so try # to find an appropriate one. Get the dictionary containing the # list of BPMs for all instruments and modes. all_bpm_dict = Lookups.get_lookup_table("Gemini/BPMDict", "bpm_dict") # Call the _get_bpm_key helper function to get the key for the # lookup table key = self._get_bpm_key(ad) # Get the appropriate BPM from the look up table if key in all_bpm_dict: bpm = lookup_path(all_bpm_dict[key]) else: bpm = None log.warning("No BPM found for %s, no BPM will be " "included" % ad.filename) # Ensure that the BPMs are AstroData objects bpm_ad = None if bpm is not None: log.fullinfo("Using %s as BPM" % str(bpm)) if isinstance(bpm, AstroData): bpm_ad = bpm else: bpm_ad = AstroData(bpm) ##M Do we want to fail here depending on context? if bpm_ad is None: log.warning("Cannot convert %s into an AstroData " "object, no BPM will be added" % bpm) final_bpm = None if bpm_ad is not None: # Clip the BPM data to match the size of the input AstroData # object science and pad with overscan region, if necessary final_bpm = gt.clip_auxiliary_data(adinput=ad, aux=bpm_ad, aux_type="bpm")[0] # Get the non-linear level and the saturation level using the # appropriate descriptors - Individual values get checked in the # next loop non_linear_level_dv = ad.non_linear_level() saturation_level_dv = ad.saturation_level() # Loop over each science extension in each input AstroData object for ext in ad[SCI]: # Retrieve the extension number for this extension extver = ext.extver() # Check whether an extension with the same name as the DQ # AstroData object already exists in the input AstroData object if ad[DQ, extver]: log.warning("A [%s,%d] extension already exists in %s" % (DQ, extver, ad.filename)) continue # Get the non-linear level and the saturation level for this # extension non_linear_level = non_linear_level_dv.get_value(extver=extver) saturation_level = saturation_level_dv.get_value(extver=extver) # To store individual arrays created for each of the DQ bit # types dq_bit_arrays = [] # Create an array that contains pixels that have a value of 2 # when that pixel is in the non-linear regime in the input # science extension if non_linear_level is not None: non_linear_array = None if saturation_level is not None: # Test the saturation level against non_linear level # They can be the same or the saturation level can be # greater than but not less than the non-linear level. # If they are the same then only flag saturated pixels # below. This just means not creating an unneccessary # intermediate array. if saturation_level > non_linear_level: log.fullinfo("Flagging pixels in the DQ extension " "corresponding to non linear pixels " "in %s[%s,%d] using non linear " "level = %.2f" % (ad.filename, SCI, extver, non_linear_level)) non_linear_array = np.where( ((ext.data >= non_linear_level) & (ext.data < saturation_level)), 2, 0) elif saturation_level < non_linear_level: log.warning("%s[%s,%d] saturation_level value is" "less than the non_linear_level not" "flagging non linear pixels" % (ad.filname, SCI, extver)) else: log.fullinfo("Saturation and non-linear values " "for %s[%s,%d] are the same. Only " "flagging saturated pixels." % (ad.filename, SCI, extver)) else: log.fullinfo("Flagging pixels in the DQ extension " "corresponding to non linear pixels " "in %s[%s,%d] using non linear " "level = %.2f" % (ad.filename, SCI, extver, non_linear_level)) non_linear_array = np.where( (ext.data >= non_linear_level), 2, 0) dq_bit_arrays.append(non_linear_array) # Create an array that contains pixels that have a value of 4 # when that pixel is saturated in the input science extension if saturation_level is not None: saturation_array = None log.fullinfo("Flagging pixels in the DQ extension " "corresponding to saturated pixels in " "%s[%s,%d] using saturation level = %.2f" % (ad.filename, SCI, extver, saturation_level)) saturation_array = np.where( ext.data >= saturation_level, 4, 0) dq_bit_arrays.append(saturation_array) # BPMs have an EXTNAME equal to DQ bpmname = None if final_bpm is not None: bpm_array = None bpmname = os.path.basename(final_bpm.filename) log.fullinfo("Flagging pixels in the DQ extension " "corresponding to bad pixels in %s[%s,%d] " "using the BPM %s[%s,%d]" % (ad.filename, SCI, extver, bpmname, DQ, extver)) bpm_array = final_bpm[DQ, extver].data dq_bit_arrays.append(bpm_array) # Create a single DQ extension from the three arrays (BPM, # non-linear and saturated) if not dq_bit_arrays: # The BPM, non-linear and saturated arrays were not # created. Create a single DQ array with all pixels set # equal to 0 log.fullinfo("The BPM, non-linear and saturated arrays " "were not created. Creating a single DQ " "array with all the pixels set equal to zero") final_dq_array = np.zeros(ext.data.shape).astype(dq_dtype) else: final_dq_array = self._bitwise_OR_list(dq_bit_arrays) final_dq_array = final_dq_array.astype(dq_dtype) # Create a data quality AstroData object dq = AstroData(data=final_dq_array) dq.rename_ext(DQ, ver=extver) dq.filename = ad.filename # Call the _update_dq_header helper function to update the # header of the data quality extension with some useful # keywords dq = self._update_dq_header(sci=ext, dq=dq, bpmname=bpmname) # Append the DQ AstroData object to the input AstroData object log.fullinfo("Adding extension [%s,%d] to %s" % (DQ, extver, ad.filename)) ad.append(moredata=dq) # Add the appropriate time stamps to the PHU gt.mark_history(adinput=ad, keyword=timestamp_key) # Change the filename ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"], strip=True) # Append the output AstroData object to the list of output # AstroData objects adoutput_list.append(ad) # Report the list of output AstroData objects to the reduction context rc.report_output(adoutput_list) yield rc
def subtractFringe(self, rc): # Instantiate the log log = gemLog.getGeminiLog(logType=rc["logType"], logLevel=rc["logLevel"]) # Log the standard "starting primitive" debug message log.debug(gt.log_message("primitive", "subtractFringe", "starting")) # Define the keyword to be used for the time stamp for this primitive timestamp_key = self.timestamp_keys["subtractFringe"] # Initialize the list of output AstroData objects adoutput_list = [] # Check for a user-supplied fringe adinput = rc.get_inputs_as_astrodata() fringe_param = rc["fringe"] fringe_dict = None if fringe_param is not None: # The user supplied an input to the fringe parameter if not isinstance(fringe_param, list): fringe_list = [fringe_param] else: fringe_list = fringe_param # Convert filenames to AD instances if necessary tmp_list = [] for fringe in fringe_list: if type(fringe) is not AstroData: fringe = AstroData(fringe) tmp_list.append(fringe) fringe_list = tmp_list fringe_dict = gt.make_dict(key_list=adinput, value_list=fringe_list) # Loop over each input AstroData object in the input list for ad in adinput: # Check whether the subtractFringe primitive has been run # previously if ad.phu_get_key_value(timestamp_key): log.warning("No changes will be made to %s, since it has " \ "already been processed by subtractFringe" \ % (ad.filename)) # Append the input AstroData object to the list of output # AstroData objects without further processing adoutput_list.append(ad) continue # Retrieve the appropriate fringe if fringe_dict is not None: fringe = fringe_dict[ad] else: fringe = rc.get_cal(ad, "processed_fringe") # Take care of the case where there was no fringe if fringe is None: log.warning("Could not find an appropriate fringe for %s" \ % (ad.filename)) # Append the input to the output without further processing adoutput_list.append(ad) continue else: fringe = AstroData(fringe) # Check the inputs have matching filters, binning and SCI shapes. try: gt.check_inputs_match(ad1=ad, ad2=fringe) except Errors.ToolboxError: # If not, try to clip the fringe frame to the size of the # science data # For a GMOS example, this allows a full frame fringe to # be used for a CCD2-only science frame. fringe = gt.clip_auxiliary_data( adinput=ad, aux=fringe, aux_type="cal")[0] # Check again, but allow it to fail if they still don't match gt.check_inputs_match(ad1=ad, ad2=fringe) # Subtract the fringe from the science ad = ad.sub(fringe) # Record the fringe file used ad.phu_set_key_value("FRINGEIM", os.path.basename(fringe.filename), comment=self.keyword_comments["FRINGEIM"]) # Add the appropriate time stamps to the PHU gt.mark_history(adinput=ad, keyword=timestamp_key) # Change the filename ad.filename = gt.filename_updater(adinput=ad, suffix=rc["suffix"], strip=True) # Append the output AstroData object to the list # of output AstroData objects adoutput_list.append(ad) # Report the list of output AstroData objects to the reduction context rc.report_output(adoutput_list) yield rc
def slitIllumCorrect(self, adinputs=None, slit_illum=None, do_illum=True, suffix="_illumCorrected"): """ This primitive will divide each SCI extension of the inputs by those of the corresponding slit illumination image. If the inputs contain VAR or DQ frames, those will also be updated accordingly due to the division on the data. Parameters ---------- adinputs : list of AstroData Data to be corrected. slit_illum : str or AstroData Slit illumination path or AstroData object. do_illum: bool, optional Perform slit illumination correction? (Default: True) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] qecorr_key = self.timestamp_keys['QECorrect'] if not do_illum: log.warning("Slit Illumination correction has been turned off.") return adinputs if slit_illum is None: raise NotImplementedError else: slit_illum_list = slit_illum # Provide a Slit Illum Ad object for every science frame ad_outputs = [] for ad, slit_illum_ad in zip( *gt.make_lists(adinputs, slit_illum_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by flatCorrect".format( ad.filename)) continue if slit_illum_ad is None: if self.mode in ['sq']: raise OSError( "No processed slit illumination listed for {}".format( ad.filename)) else: log.warning("No changes will be made to {}, since no slit " "illumination has been specified".format( ad.filename)) continue gt.check_inputs_match(ad, slit_illum_ad, check_shape=False) if not all( [e1.shape == e2.shape for (e1, e2) in zip(ad, slit_illum_ad)]): slit_illum_ad = gt.clip_auxiliary_data(adinput=[ad], aux=[slit_illum_ad])[0] log.info("Dividing the input AstroData object {} by this \n" "slit illumination file: \n{}".format( ad.filename, slit_illum_ad.filename)) ad_out = deepcopy(ad) ad_out.divide(slit_illum_ad) # Update the header and filename, copying QECORR keyword from flat ad_out.phu.set("SLTILLIM", slit_illum_ad.filename, self.keyword_comments["SLTILLIM"]) try: qecorr_value = slit_illum_ad.phu[qecorr_key] except KeyError: pass else: log.fullinfo( "Copying {} keyword from slit illumination".format( qecorr_key)) ad_out.phu.set(qecorr_key, qecorr_value, slit_illum_ad.phu.comments[qecorr_key]) gt.mark_history(ad_out, primname=self.myself(), keyword=timestamp_key) ad_out.update_filename(suffix=suffix, strip=True) if slit_illum_ad.path: add_provenance(ad_out, slit_illum_ad.filename, md5sum(slit_illum_ad.path) or "", self.myself()) ad_outputs.append(ad_out) return ad_outputs
def fringeCorrect(self, adinputs=None, **params): """ Correct science frames for the effects of fringing, using a fringe frame. The fringe frame is obtained either from a specified parameter, or the "fringe" stream, or the calibration database. This is basically a bookkeeping wrapper for subtractFringe(), which does all the work. Parameters ---------- suffix: str suffix to be added to output files fringe: list/str/AstroData/None fringe frame(s) to subtract do_fringe: bool/None apply fringe correction? (None => use pipeline default for data) scale: bool/None scale fringe frame? (None => False if fringe frame has same group_id() as data scale_factor: float/sequence/None factor(s) to scale fringe """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] fringe = params["fringe"] scale = params["scale"] do_cal = params["do_cal"] # Exit now if nothing needs a correction, to avoid an error when the # calibration search fails. If images with different exposure times # are used, some frames may not require a correction (but the calibration # search will succeed), so still need to check individual inputs later. needs_correction = [self._needs_fringe_correction(ad) for ad in adinputs] if any(needs_correction): if do_cal == 'skip': log.warning("Fringe correction has been turned off but is " "recommended.") return adinputs else: if do_cal == 'procmode' or do_cal == 'skip': log.stdinfo("No input images require a fringe correction.") return adinputs else: # do_cal == 'force': log.warning("Fringe correction has been forced on but may not " "be required.") if fringe is None: # This logic is for QAP try: fringe_list = self.streams['fringe'] assert len(fringe_list) == 1 scale = False log.stdinfo("Using fringe frame in 'fringe' stream. " "Setting scale=False") fringe_list = (fringe_list[0], "stream") except (KeyError, AssertionError): fringe_list = self.caldb.get_processed_fringe(adinputs) else: fringe_list = (fringe, None) # Usual stuff to ensure that we have an iterable of the correct length # for the scale factors regardless of what the input is scale_factor = params["scale_factor"] try: factors = iter(scale_factor) except TypeError: factors = iter([scale_factor] * len(adinputs)) else: # In case a single-element list was passed if len(scale_factor) == 1: factors = iter(scale_factor * len(adinputs)) # Get a fringe AD object for every science frame for ad, fringe, origin, correct in zip(*gt.make_lists( adinputs, *fringe_list, needs_correction, force_ad=(1,))): if ad.phu.get(timestamp_key): log.warning(f"{ad.filename}: already processed by " "fringeCorrect. Continuing.") continue # Logic to deal with different exposure times where only # some inputs might require fringe correction # KL: for now, I'm not allowing the "force" to do anything when # the correction is not needed. if (do_cal == 'procmode' or do_cal == 'force') and not correct: log.stdinfo("{} does not require a fringe correction". format(ad.filename)) ad.update_filename(suffix=params["suffix"], strip=True) continue # At this point, we definitely want to do a fringe correction # so we'd better have a fringe frame! if fringe is None: if 'sq' not in self.mode and do_cal != 'force': log.warning("No changes will be made to {}, since no " "fringe frame has been specified". format(ad.filename)) continue else: log.warning(f"{ad.filename}: no fringe was specified. " "Continuing.") continue # Check the inputs have matching filters, binning, and shapes try: gt.check_inputs_match(ad, fringe) except ValueError: fringe = gt.clip_auxiliary_data(adinput=ad, aux=fringe, aux_type="cal") gt.check_inputs_match(ad, fringe) # origin_str = f" (obtained from {origin})" if origin else "" log.stdinfo(f"{ad.filename}: using the fringe frame " f"{fringe.filename}{origin_str}") matched_groups = (ad.group_id() == fringe.group_id()) if scale or (scale is None and not matched_groups): factor = next(factors) if factor is None: factor = self._calculate_fringe_scaling(ad, fringe) log.stdinfo("Scaling fringe frame by factor {:.3f} before " "subtracting from {}".format(factor, ad.filename)) # Since all elements of fringe_list might be references to the # same AD, need to make a copy before multiplying fringe_copy = deepcopy(fringe) fringe_copy.multiply(factor) ad.subtract(fringe_copy) else: if scale is None: log.stdinfo("Not scaling fringe frame with same group ID " "as {}".format(ad.filename)) ad.subtract(fringe) # Timestamp and update header and filename ad.phu.set("FRINGEIM", fringe.filename, self.keyword_comments["FRINGEIM"]) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=params["suffix"], strip=True) if fringe.path: add_provenance(ad, fringe.filename, md5sum(fringe.path) or "", self.myself()) return adinputs
def pointing_in_field(ad, refpos, frac_FOV=1.0, frac_slit=1.0): """ See gemini_tools.pointing_in_field() for the API. This is an instrument-specific back end that you shouldn't be calling directly. No inputs are validated at this level; that's the responsibility of the calling function, for reasons of efficiency. The GNIRS FOV is determined by whether the calculated center point (according to the center of mass of the illumination mask) of the image falls within the illumination mask of the reference image. :param pos: AstroData instance to be checked for whether it belongs in the same sky grouping as refpos :type pos: AstroData instance :param refpos: This is the POFFSET and QOFFSET of the reference image :type refpos: tuple of floats :param frac_FOV: For use with spectroscopy data :type frac_FOV: float :param frac_slit: For use with spectroscopy data :type frac_slit: float """ # Object location in AD = refpos + shift xshift = ad.detector_x_offset() - refpos[0] yshift = ad.detector_y_offset() - refpos[1] # We inverse-scale by frac_FOV fov_xshift = -int(xshift / frac_FOV) fov_yshift = -int(yshift / frac_FOV) # Imaging: if 'IMAGE' in ad.tags: if (abs(fov_yshift) >= ad[0].shape[1] or abs(fov_xshift) >= ad[0].shape[1]): return False illum = get_illum_mask_filename(ad) if illum: illum_ad = gt.clip_auxiliary_data(adinput=ad, aux=astrodata.open(illum), aux_type="bpm") illum_data = illum_ad[0].data else: raise OSError("Cannot find illumination mask for {}".format( ad.filename)) # Shift the illumination mask and see if the shifted keyhole # overlaps with the original keyhole shifted_data = shift(illum_data, (fov_yshift, fov_xshift), order=0, cval=DQ.unilluminated) return (illum_data | shifted_data == 0).any() # Spectroscopy: elif 'SPECT' in ad.tags: raise NotImplementedError("FOV lookup not yet supported for GNIRS " "Spectroscopy") # Some engineering observation or bad mask value etc.: else: raise ValueError("Can't determine FOV for unrecognized GNIRS config " "({}, {})".format(ad.focal_plane_mask(), ad.disperser()))
def processSlits(self, adinputs=None, **params): """ Compute and record the mean exposure epoch for a slit viewer image The 'slit viewer image' for each observation will almost certainly be a sequence of short exposures of the slit viewer camera, collected together for convenience. However, it cannot be guaranteed that slit viewer exposures will be taken throughout an entire science exposure; therefore, it is necessary to be able to compute the mean exposure epoch (i.e. the effective time that the combined slit viewer exposures were taken at). This allows a single science observation to be calibrated using multiple packets of slit viewer exposures, with appropriate weighting for the time delay between them. ``processSlits`` effectively computes a weighted average of the exposure epoch of all constituent slit viewer exposures, taking into account: - Length of each exposure; - Whether there is any overlap between the start/end of the exposure and the start/end of the overall 'image'; - Time of each exposure, relative to the start of the 'image'. Parameters ---------- suffix: str suffix to be added to output files slitflat: str/None name of the slitflat to use (if None, use the calibration system) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] flat_list = params["flat"] if flat_list is None: self.getProcessedSlitFlat(adinputs) flat_list = [ self._get_cal(ad, 'processed_slitflat') for ad in adinputs ] for ad, slitflat in zip( *gt.make_lists(adinputs, flat_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by processSlits".format( ad.filename)) continue if slitflat is None: log.warning("Unable to find slitflat calibration for {}; " "skipping".format(ad.filename)) continue else: sv_flat = slitflat[0].data # accumulators for computing the mean epoch sum_of_weights = 0.0 accum_weighted_time = 0.0 # Check the inputs have matching binning and SCI shapes. try: gt.check_inputs_match(adinput1=ad, adinput2=slitflat, check_filter=False) except ValueError: # This is most likely because the science frame has multiple # extensions and the slitflat needs to be copied slitflat = gt.clip_auxiliary_data(ad, slitflat, aux_type='cal') # An Error will be raised if they don't match now gt.check_inputs_match(ad, slitflat, check_filter=False) # get science start/end times sc_start = parse_timestr(ad.phu['UTSTART']) sc_end = parse_timestr(ad.phu['UTEND']) res = ad.res_mode() for ext in ad: sv_start = parse_timestr(ext.hdr['EXPUTST']) sv_end = parse_timestr(ext.hdr['EXPUTEND']) # compute overlap percentage and slit view image duration latest_start = max(sc_start, sv_start) earliest_end = min(sc_end, sv_end) overlap = (earliest_end - latest_start).seconds overlap = 0.0 if overlap < 0.0 else overlap # no overlap edge case sv_duration = (sv_end - sv_start).seconds overlap /= sv_duration # convert into a percentage # compute the offset (the value to be weighted), in seconds, # from the start of the science exposure offset = 42.0 # init value: overridden if overlap, else 0-scaled if sc_start <= sv_start and sv_end <= sc_end: offset = (sv_start - sc_start).seconds + sv_duration / 2.0 elif sv_start < sc_start: offset = overlap * sv_duration / 2.0 elif sv_end > sc_end: offset = overlap * sv_duration / 2.0 offset += (sv_start - sc_start).seconds # add flux-weighted offset (plus weight itself) to accumulators flux = _total_obj_flux(res, ext.data, sv_flat) weight = flux * overlap sum_of_weights += weight accum_weighted_time += weight * offset # final mean exposure epoch computation if sum_of_weights > 0.0: mean_offset = accum_weighted_time / sum_of_weights mean_offset = timedelta(seconds=mean_offset) # write the mean exposure epoch into the PHU sc_start = parse_timestr(ad.phu['UTSTART']) mean_epoch = sc_start + mean_offset ad.phu['AVGEPOCH'] = ( # hope this keyword string is ok mean_epoch.strftime("%H:%M:%S.%f")[:-3], 'Mean Exposure Epoch') # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=params["suffix"], strip=True) return adinputs
def addIllumMaskToDQ(self, adinputs=None, suffix=None, illum_mask=None, shift=None, max_shift=20): """ Adds an illumination mask to each AD object. This is only done for full-frame (not Central Spectrum) GMOS spectra, and is calculated by making a model illumination patter from the attached MDF and cross- correlating it with the spatial profile of the data. Parameters ---------- suffix : str suffix to be added to output files illum_mask : str/None name of illumination mask mask (None -> use default) shift : int/None user-defined shift to apply to illumination mask max_shift : int maximum shift (in unbinned pixels) allowable for the cross- correlation """ offset_dict = { ("GMOS-N", "Hamamatsu-N"): 1.5, ("GMOS-N", "e2vDD"): -0.2, ("GMOS-N", "EEV"): 0.7, ("GMOS-S", "Hamamatsu-S"): 5.5, ("GMOS-S", "EEV"): 3.8 } edges = 50 # try to eliminate issues at the very edges log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] # Do this now for memory management reasons. We'll be creating large # arrays temporarily and don't want the permanent mask arrays to # fragment the free memory. for ad in adinputs: for ext in ad: if ext.mask is None: ext.mask = np.zeros_like(ext.data).astype(DQ.datatype) for ad, illum in zip( *gt.make_lists(adinputs, illum_mask, force_ad=True)): if ad.phu.get(timestamp_key): log.warning( 'No changes will be made to {}, since it has ' 'already been processed by addIllumMaskToDQ'.format( ad.filename)) continue ybin = ad.detector_y_bin() ad_detsec = ad.detector_section() no_bridges = all(detsec.y1 > 1600 and detsec.y2 < 2900 for detsec in ad_detsec) has_48rows = (all(detsec.y2 == 4224 for detsec in ad_detsec) and 'Hamamatsu' in ad.detector_name(pretty=True)) if illum: log.fullinfo("Using {} as illumination mask".format( illum.filename)) final_illum = gt.clip_auxiliary_data(ad, aux=illum, aux_type='bpm', return_dtype=DQ.datatype) for ext, illum_ext in zip(ad, final_illum): if illum_ext is not None: # Ensure we're only adding the unilluminated bit iext = np.where(illum_ext.data > 0, DQ.unilluminated, 0).astype(DQ.datatype) ext.mask |= iext elif not no_bridges: # i.e. there are bridges. try: mdf = ad.MDF except AttributeError: log.warning(f"MDF not found for {ad.filename} - cannot " "add illumination mask.") continue # Default operation for GMOS full-frame LS # Sadly, we cannot do this reliably without concatenating the # arrays and using a big chunk of memory. row_medians = np.percentile(np.concatenate( [ext.data for ext in ad], axis=1), 95, axis=1) row_medians -= at.boxcar(row_medians, size=50 // ybin) # Construct a model of the slit illumination from the MDF # coefficients are from G-IRAF except c0, approx. from data model = np.zeros_like(row_medians, dtype=int) for ypos, ysize in mdf['slitpos_my', 'slitsize_my']: y = ypos + np.array([-0.5, 0.5]) * ysize c0 = offset_dict[ad.instrument(), ad.detector_name(pretty=True)] if ad.instrument() == "GMOS-S": c1, c2, c3 = (0.99911, -1.7465e-5, 3.0494e-7) else: c1, c2, c3 = (0.99591859227, 5.3042211333437e-8, 1.7447902551997e-7) yccd = ((c0 + y * (c1 + y * (c2 + y * c3))) * 1.611444 / ad.pixel_scale() + 0.5 * model.size).astype(int) model[yccd[0]:yccd[1] + 1] = 1 log.stdinfo("Expected slit location from pixels " f"{yccd[0]+1} to {yccd[1]+1}") if shift is None: max_shift = 50 mshift = max_shift // ybin + 2 mshift2 = mshift + edges # model[] indexing avoids reduction in signal as slit # is shifted off the top of the image cntr = model.size - edges - mshift2 - 1 xcorr = correlate(row_medians[edges:-edges], model[mshift2:-mshift2], mode='full')[cntr - mshift:cntr + mshift] # This line avoids numerical errors in the spline fit xcorr -= np.median(xcorr) # This calculates the offsets of each point from the # straight line between its neighbours std = (xcorr[1:-1] - 0.5 * (xcorr + np.roll(xcorr, 2))[2:]).std() xspline = fit_1D(xcorr, function="spline3", order=None, weights=np.full(len(xcorr), 1. / std)).evaluate() yshift = xspline.argmax() - mshift maxima = xspline[1:-1][np.logical_and( np.diff(xspline[:-1]) > 0, np.diff(xspline[1:]) < 0)] significant_maxima = (maxima > xspline.max() - 3 * std).sum() if significant_maxima > 1 or abs( yshift // ybin) > max_shift: log.warning( f"{ad.filename}: cross-correlation peak is" " untrustworthy so not adding illumination " "mask. Please re-run with a specified shift.") yshift = None else: yshift = shift if yshift is not None: log.stdinfo( f"{ad.filename}: Shifting mask by {yshift} pixels") row_mask = np.ones_like(model, dtype=int) if yshift < 0: row_mask[:yshift] = 1 - model[-yshift:] elif yshift > 0: row_mask[yshift:] = 1 - model[:-yshift] else: row_mask[:] = 1 - model for ext in ad: ext.mask |= (row_mask * DQ.unilluminated).astype( DQ.datatype)[:, np.newaxis] if has_48rows: actual_rows = 48 // ybin for ext in ad: ext.mask[:actual_rows] |= DQ.unilluminated # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def pointing_in_field(pos, refpos, frac_FOV=1.0, frac_slit=1.0): """ See gemini_tools.pointing_in_field() for the API. This is an instrument-specific back end that you shouldn't be calling directly. No inputs are validated at this level; that's the responsibility of the calling function, for reasons of efficiency. The GNIRS FOV is determined by whether the calculated center point (according to the center of mass of the illumination mask) of the image falls within the illumination mask of the reference image. :param pos: AstroData instance to be checked for whether it belongs in the same sky grouping as refpos :type pos: AstroData instance :param refpos: This is the POFFSET and QOFFSET of the reference image :type refpos: tuple of floats :param frac_FOV: For use with spectroscopy data :type frac_FOV: float :param frac_slit: For use with spectroscopy data :type frac_slit: float """ # Since this function gets looked up and evaluated, we have to do any # essential imports in-line (but Python caches them) import math # Extract pointing info in terms of the x and y offsets xshift = refpos[1] - pos.phu['QOFFSET'] yshift = refpos[0] - pos.phu['POFFSET'] ad = pos # Imaging: if 'IMAGE' in pos.tags: illum = get_illum_mask_filename(ad) if illum: illum_ad = gt.clip_auxiliary_data(adinput=pos, aux=astrodata.open(illum), aux_type="bpm") illum_data = illum_ad[0].data else: raise IOError("Cannot find illumination mask for {}". format(ad.filename)) # Finding the center of the illumination mask center_illum = (illum_ad.phu['CENMASSX'], illum_ad.phu['CENMASSY']) checkpos = (int(center_illum[0] + xshift), int(center_illum[1] + yshift)) # If the position to check is going to fall outside the illumination # mask, return straight away to avoid an error if ((abs(xshift) >= abs(center_illum[0])) or (abs(yshift) >= abs(center_illum[1]))): return False # Note that numpy data arrays are reversed in x and y return illum_data[checkpos[1], checkpos[0]] == 0 # Spectroscopy: elif 'SPECT' in ad.tags: raise NotImplementedError("FOV lookup not yet supported for GNIRS " "Spectroscopy") # Some engineering observation or bad mask value etc.: else: raise ValueError("Can't determine FOV for unrecognized GNIRS config " "({}, {})".format(ad.focal_plane_mask(), ad.disperser()))
def biasCorrect(self, adinputs=None, suffix=None, bias=None, do_cal=None): """ The biasCorrect primitive will subtract the science extension of the input bias frames from the science extension of the input science frames. The variance and data quality extension will be updated, if they exist. If no bias is provided, the calibration database(s) will be queried. Parameters ---------- suffix: str suffix to be added to output files bias: str/list of str bias(es) to subtract do_cal: str perform bias subtraction? """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] if do_cal == 'skip': log.warning("Bias correction has been turned off.") return adinputs if bias is None: bias_list = self.caldb.get_processed_bias(adinputs) else: bias_list = (bias, None) # Provide a bias AD object for every science frame, and an origin for ad, bias, origin in zip(*gt.make_lists(adinputs, *bias_list, force_ad=(1,))): if ad.phu.get(timestamp_key): log.warning(f"{ad.filename}: already processed by " "biasCorrect. Continuing.") continue if bias is None: if 'sq' not in self.mode and do_cal != 'force': log.warning("No changes will be made to {}, since no " "bias was specified".format(ad.filename)) continue else: log.warning(f"{ad.filename}: no bias was specified. " "Continuing.") continue try: gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) except ValueError: bias = gt.clip_auxiliary_data(ad, aux=bias, aux_type='cal') # An Error will be raised if they don't match now gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) origin_str = f" (obtained from {origin})" if origin else "" log.stdinfo(f"{ad.filename}: subtracting the bias " f"{bias.filename}{origin_str}") ad.subtract(bias) # Record bias used, timestamp, and update filename ad.phu.set('BIASIM', bias.filename, self.keyword_comments['BIASIM']) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) if bias.path: add_provenance(ad, bias.filename, md5sum(bias.path) or "", self.myself()) timestamp = datetime.now() return adinputs
def addIllumMaskToDQ(self, adinputs=None, suffix=None, illum_mask=None): """ Adds an illumination mask to each AD object Parameters ---------- suffix: str suffix to be added to output files illum_mask: str/None name of illumination mask mask (None -> use default) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] for ad, illum in zip( *gt.make_lists(adinputs, illum_mask, force_ad=True)): if ad.phu.get(timestamp_key): log.warning( 'No changes will be made to {}, since it has ' 'already been processed by addIllumMaskToDQ'.format( ad.filename)) continue ad_detsec = ad.detector_section() no_bridges = all(detsec.y1 > 1600 and detsec.y2 < 2900 for detsec in ad_detsec) has_48rows = (all(detsec.y2 == 4224 for detsec in ad_detsec) and 'Hamamatsu' in ad.detector_name(pretty=True)) if illum: log.fullinfo("Using {} as illumination mask".format( illum.filename)) final_illum = gt.clip_auxiliary_data(ad, aux=illum, aux_type='bpm', return_dtype=DQ.datatype) for ext, illum_ext in zip(ad, final_illum): if illum_ext is not None: # Ensure we're only adding the unilluminated bit iext = np.where(illum_ext.data > 0, DQ.unilluminated, 0).astype(DQ.datatype) ext.mask = iext if ext.mask is None else ext.mask | iext elif not no_bridges: # i.e. there are bridges. # Default operation for GMOS full-frame LS # The 95% cut should ensure that we're sampling something # bright (even for an arc) # The max is intended to handle R150 data, where many of # the extensions are unilluminated row_medians = np.max(np.array( [np.percentile(ext.data, 95, axis=1) for ext in ad]), axis=0) rows = np.arange(len(row_medians)) m_init = models.Polynomial1D(degree=3) fit_it = fitting.FittingWithOutlierRemoval( fitting.LinearLSQFitter(), outlier_func=sigma_clip, sigma_upper=1, sigma_lower=3) m_final, _ = fit_it(m_init, rows, row_medians) model_fit = m_final(rows) # Find points which are significantly below the smooth illumination fit # First ensure we don't worry about single rows row_mask = at.boxcar(model_fit - row_medians > 0.1 * model_fit, operation=np.logical_and, size=1) row_mask = at.boxcar(row_mask, operation=np.logical_or, size=3) for ext in ad: ext.mask |= (row_mask * DQ.unilluminated).astype( DQ.datatype)[:, np.newaxis] if has_48rows: actual_rows = 48 // ad.detector_y_bin() for ext in ad: ext.mask[:actual_rows] |= DQ.unilluminated # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def addDQ(self, adinputs=None, **params): """ This primitive is used to add a DQ extension to the input AstroData object. The value of a pixel in the DQ extension will be the sum of the following: (0=good, 1=bad pixel (found in bad pixel mask), 2=pixel is in the non-linear regime, 4=pixel is saturated). This primitive will trim the BPM to match the input AstroData object(s). Parameters ---------- suffix: str suffix to be added to output files static_bpm: str Name of bad pixel mask ("default" -> use default from look-up table) If set to None, no static_bpm will be added. user_bpm: str Name of the bad pixel mask created by the user from flats and darks. It is an optional BPM that can be added to the static one. illum_mask: bool add illumination mask? """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys["addDQ"] sfx = params["suffix"] # Getting all the filenames first prevents reopening the same file # for each science AD static_bpm_list = params['static_bpm'] user_bpm_list = params['user_bpm'] if static_bpm_list == "default": static_bpm_list = [self._get_bpm_filename(ad) for ad in adinputs] for ad, static, user in zip(*gt.make_lists(adinputs, static_bpm_list, user_bpm_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning('No changes will be made to {}, since it has ' 'already been processed by addDQ'.format(ad.filename)) continue if static is None: # So it can be zipped with the AD final_static = [None] * len(ad) else: log.fullinfo("Using {} as static BPM".format(static.filename)) final_static = gt.clip_auxiliary_data(ad, aux=static, aux_type='bpm', return_dtype=DQ.datatype) if user is None: final_user = [None] * len(ad) else: log.fullinfo("Using {} as user BPM".format(user.filename)) final_user = gt.clip_auxiliary_data(ad, aux=user, aux_type='bpm', return_dtype=DQ.datatype) for ext, static_ext, user_ext in zip(ad, final_static, final_user): extver = ext.hdr['EXTVER'] if ext.mask is not None: log.warning('A mask already exists in extver {}'. format(extver)) continue non_linear_level = ext.non_linear_level() saturation_level = ext.saturation_level() # Need to create the array first for 3D raw F2 data, with 2D BPM ext.mask = np.zeros_like(ext.data, dtype=DQ.datatype) if static_ext is not None: ext.mask |= static_ext.data if user_ext is not None: ext.mask |= user_ext.data if saturation_level: log.fullinfo('Flagging saturated pixels in {}:{} ' 'above level {:.2f}'. format(ad.filename, extver, saturation_level)) ext.mask |= np.where(ext.data >= saturation_level, DQ.saturated, 0).astype(DQ.datatype) if non_linear_level: if saturation_level: if saturation_level > non_linear_level: log.fullinfo('Flagging non-linear pixels in {}:{} ' 'above level {:.2f}'. format(ad.filename, extver, non_linear_level)) ext.mask |= np.where((ext.data >= non_linear_level) & (ext.data < saturation_level), DQ.non_linear, 0).astype(DQ.datatype) # Readout modes of IR detectors can result in # saturated pixels having values below the # saturation level. Flag those. Assume we have an # IR detector here because both non-linear and # saturation levels are defined and nonlin<sat regions, nregions = measurements.label( ext.data < non_linear_level) # In all my tests, region 1 has been the majority # of the image; however, I cannot guarantee that # this is always the case and therefore we should # check the size of each region region_sizes = measurements.labeled_comprehension( ext.data, regions, np.arange(1, nregions+1), len, int, 0) # First, assume all regions are saturated, and # remove any very large ones. This is much faster # than progressively adding each region to DQ hidden_saturation_array = np.where(regions > 0, 4, 0).astype(DQ.datatype) for region in range(1, nregions+1): # Limit of 10000 pixels for a hole is a bit arbitrary if region_sizes[region-1] > 10000: hidden_saturation_array[regions==region] = 0 ext.mask |= hidden_saturation_array elif saturation_level < non_linear_level: log.warning('{}:{} has saturation level less than ' 'non-linear level'.format(ad.filename, extver)) else: log.fullinfo('Saturation and non-linear levels ' 'are the same for {}:{}. Only ' 'flagging saturated pixels'. format(ad.filename, extver)) else: log.fullinfo('Flagging non-linear pixels in {}:{} ' 'above level {:.2f}'. format(ad.filename, extver, non_linear_level)) ext.mask |= np.where(ext.data >= non_linear_level, DQ.non_linear, 0).astype(DQ.datatype) # Handle latency if reqested if params.get("latency", False): try: adinputs = self.addLatencyToDQ(adinputs, time=params["time"], non_linear=params["non_linear"]) except AttributeError: log.warning("addLatencyToDQ() not defined in primitivesClass " + self.__class__.__name__) # Add the illumination mask if requested if params['add_illum_mask']: adinputs = self.addIllumMaskToDQ(adinputs, illum_mask=params["illum_mask"]) # Timestamp and update filenames for ad in adinputs: gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=sfx, strip=True) return adinputs
def biasCorrect(self, adinputs=None, suffix=None, bias=None, do_bias=True): """ The biasCorrect primitive will subtract the science extension of the input bias frames from the science extension of the input science frames. The variance and data quality extension will be updated, if they exist. If no bias is provided, getProcessedBias will be called to ensure a bias exists for every adinput. Parameters ---------- suffix: str suffix to be added to output files bias: str/list of str bias(es) to subtract do_bias: bool perform bias subtraction? """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] if not do_bias: log.warning("Bias correction has been turned off.") return adinputs if bias is None: self.getProcessedBias(adinputs, refresh=False) bias_list = self._get_cal(adinputs, 'processed_bias') else: bias_list = bias # Provide a bias AD object for every science frame for ad, bias in zip( *gt.make_lists(adinputs, bias_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by biasCorrect".format( ad.filename)) continue if bias is None: if 'qa' in self.mode: log.warning("No changes will be made to {}, since no " "bias was specified".format(ad.filename)) continue else: raise OSError('No processed bias listed for {}'.format( ad.filename)) try: gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) except ValueError: bias = gt.clip_auxiliary_data(ad, aux=bias, aux_type='cal') # An Error will be raised if they don't match now gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) log.fullinfo('Subtracting this bias from {}:\n{}'.format( ad.filename, bias.filename)) ad.subtract(bias) # Record bias used, timestamp, and update filename ad.phu.set('BIASIM', bias.filename, self.keyword_comments['BIASIM']) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) if bias.path: add_provenance(ad, bias.filename, md5sum(bias.path) or "", self.myself()) timestamp = datetime.now() return adinputs
def scaleFringeToScience(self, rc): """ This primitive will scale the fringes to their matching science data The fringes should be in the stream this primitive is called on, and the reference science frames should be loaded into the RC, as, eg. rc["science"] = adinput. There are two ways to find the value to scale fringes by: 1. If stats_scale is set to True, the equation: (letting science data = b (or B), and fringe = a (or A)) arrayB = where({where[SCIb < (SCIb.median+2.5*SCIb.std)]} > [SCIb.median-3*SCIb.std]) scale = arrayB.std / SCIa.std The section of the SCI arrays to use for calculating these statistics is the CCD2 SCI data excluding the outer 5% pixels on all 4 sides. Future enhancement: allow user to choose section 2. If stats_scale=False, then scale will be calculated using: exposure time of science / exposure time of fringe :param stats_scale: Use statistics to calculate the scale values, rather than exposure time :type stats_scale: Python boolean (True/False) """ # Instantiate the log log = gemLog.getGeminiLog(logType=rc["logType"], logLevel=rc["logLevel"]) # Log the standard "starting primitive" debug message log.debug(gt.log_message("primitive", "scaleFringeToScience", "starting")) # Define the keyword to be used for the time stamp for this primitive timestamp_key = self.timestamp_keys["scaleFringeToScience"] # Check for user-supplied science frames fringe = rc.get_inputs_as_astrodata() science_param = rc["science"] fringe_dict = None if science_param is not None: # The user supplied an input to the science parameter if not isinstance(science_param, list): science_list = [science_param] else: science_list = science_param # If there is one fringe and multiple science frames, # the fringe must be deepcopied to allow it to be # scaled separately for each frame if len(fringe)==1 and len(science_list)>1: fringe = [deepcopy(fringe[0]) for img in science_list] # Convert filenames to AD instances if necessary tmp_list = [] for science in science_list: if type(science) is not AstroData: science = AstroData(science) tmp_list.append(science) science_list = tmp_list fringe_dict = gt.make_dict(key_list=science_list, value_list=fringe) fringe_output = [] else: log.warning("No science frames specified; no scaling will be done") science_list = [] fringe_output = fringe # Loop over each AstroData object in the science list for ad in science_list: # Retrieve the appropriate fringe fringe = fringe_dict[ad] # Check the inputs have matching filters, binning and SCI shapes. try: gt.check_inputs_match(ad1=ad, ad2=fringe) except Errors.ToolboxError: # If not, try to clip the fringe frame to the size of the # science data # For a GMOS example, this allows a full frame fringe to # be used for a CCD2-only science frame. fringe = gt.clip_auxiliary_data( adinput=ad, aux=fringe, aux_type="cal")[0] # Check again, but allow it to fail if they still don't match gt.check_inputs_match(ad1=ad, ad2=fringe) # Check whether statistics should be used stats_scale = rc["stats_scale"] # Calculate the scale value scale = 1.0 if not stats_scale: # Use the exposure times to calculate the scale log.fullinfo("Using exposure times to calculate the scaling"+ " factor") try: scale = ad.exposure_time() / fringe.exposure_time() except: raise Errors.InputError("Could not get exposure times " + "for %s, %s. Try stats_scale=True" % (ad.filename,fringe.filename)) else: # Use statistics to calculate the scaling factor log.fullinfo("Using statistics to calculate the " + "scaling factor") # Deepcopy the input so it can be manipulated without # affecting the original statsad = deepcopy(ad) statsfringe = deepcopy(fringe) # Trim off any overscan region still present statsad,statsfringe = gt.trim_to_data_section([statsad, statsfringe]) # Check the number of science extensions; if more than # one, use CCD2 data only nsciext = statsad.count_exts("SCI") if nsciext>1: # Get the CCD numbers and ordering information # corresponding to each extension log.fullinfo("Trimming data to data section to remove "\ "overscan region") sci_info,frng_info = gt.array_information([statsad, statsfringe]) # Pull out CCD2 data scidata = [] frngdata = [] dqdata = [] for i in range(nsciext): # Get the next extension in physical order sciext = statsad["SCI",sci_info["amps_order"][i]] frngext = statsfringe["SCI",frng_info["amps_order"][i]] # Check to see if it is on CCD2; if so, keep it if sci_info[ "array_number"][("SCI",sciext.extver())]==2: scidata.append(sciext.data) dqext = statsad["DQ",sci_info["amps_order"][i]] maskext = statsad["OBJMASK", sci_info["amps_order"][i]] if dqext is not None and maskext is not None: dqdata.append(dqext.data | maskext.data) elif dqext is not None: dqdata.append(dqext.data) elif maskext is not None: dqdata.append(maskext.data) if frng_info[ "array_number"][("SCI",frngext.extver())]==2: frngdata.append(frngext.data) # Stack data if necessary if len(scidata)>1: scidata = np.hstack(scidata) frngdata = np.hstack(frngdata) else: scidata = scidata[0] frngdata = frngdata[0] if len(dqdata)>0: if len(dqdata)>1: dqdata = np.hstack(dqdata) else: dqdata = dqdata[0] else: dqdata = None else: scidata = statsad["SCI"].data frngdata = statsfringe["SCI"].data dqext = statsad["DQ"] maskext = statsad["OBJMASK"] if dqext is not None and maskext is not None: dqdata = dqext.data | maskext.data elif dqext is not None: dqdata = dqext.data elif maskext is not None: dqdata = maskext.data else: dqdata = None if dqdata is not None: # Replace any DQ-flagged data with the median value smed = np.median(scidata[dqdata==0]) scidata = np.where(dqdata!=0,smed,scidata) # Calculate the maximum and minimum in a box centered on # each data point. The local depth of the fringe is # max - min. The overall fringe strength is the median # of the local fringe depths. # Width of the box is binning and # filter dependent, determined by experimentation # Results don't seem to depend heavily on the box size if ad.filter_name(pretty=True).as_pytype=="i": size = 20 else: size = 40 size /= ad.detector_x_bin().as_pytype() # Use ndimage maximum_filter and minimum_filter to # get the local maxima and minima import scipy.ndimage as ndimage sci_max = ndimage.filters.maximum_filter(scidata,size) sci_min = ndimage.filters.minimum_filter(scidata,size) # Take off 5% of the width as a border xborder = int(0.05 * scidata.shape[1]) yborder = int(0.05 * scidata.shape[0]) if xborder<20: xborder = 20 if yborder<20: yborder = 20 sci_max = sci_max[yborder:-yborder,xborder:-xborder] sci_min = sci_min[yborder:-yborder,xborder:-xborder] # Take the median difference sci_df = np.median(sci_max - sci_min) # Do the same for the fringe frn_max = ndimage.filters.maximum_filter(frngdata,size) frn_min = ndimage.filters.minimum_filter(frngdata,size) frn_max = frn_max[yborder:-yborder,xborder:-xborder] frn_min = frn_min[yborder:-yborder,xborder:-xborder] frn_df = np.median(frn_max - frn_min) # Scale factor # This tends to overestimate the factor, but it is # at least in the right ballpark, unlike the estimation # used in girmfringe (masked_sci.std/fringe.std) scale = sci_df / frn_df log.fullinfo("Scale factor found = "+str(scale)) # Use mult from the arith toolbox to perform the scaling of # the fringe frame scaled_fringe = fringe.mult(scale) # Add the appropriate time stamps to the PHU gt.mark_history(adinput=scaled_fringe, keyword=timestamp_key) # Change the filename scaled_fringe.filename = gt.filename_updater( adinput=ad, suffix=rc["suffix"], strip=True) fringe_output.append(scaled_fringe) # Report the list of output AstroData objects to the reduction context rc.report_output(fringe_output) yield rc