def transferAttribute(self, adinputs=None, source=None, attribute=None): """ This primitive takes an attribute (e.g., "mask", or "OBJCAT") from the AD(s) in another ("source") stream and applies it to the ADs in this stream. There must be either the same number of ADs in each stream, or only 1 in the source stream. Parameters ---------- source: str name of stream containing ADs whose attributes you want attribute: str attribute to transfer from ADs in other stream """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) if source not in self.streams.keys(): log.info("Stream {} does not exist so nothing to transfer".format(source)) return adinputs source_length = len(self.streams[source]) if not (source_length == 1 or source_length == len(adinputs)): log.warning("Incompatible stream lengths: {} and {}". format(len(adinputs), source_length)) return adinputs log.stdinfo("Transferring attribute {} from stream {}".format(attribute, source)) # Keep track of whether we find anything to transfer, as failing to # do so might indicate a problem and we should warn the user found = False for ad1, ad2 in zip(*gt.make_lists(adinputs, self.streams[source])): # Attribute could be top-level or extension-level # Use deepcopy so references to original object don't remain if hasattr(ad2, attribute): try: setattr(ad1, attribute, copy.deepcopy(getattr(ad2, attribute))) except ValueError: # data, mask, are gettable not settable pass else: found = True continue for ext1, ext2 in zip(ad1, ad2): if hasattr(ext2, attribute): setattr(ext1, attribute, copy.deepcopy(getattr(ext2, attribute))) found = True if not found: log.warning("Did not find any {} attributes to transfer".format(attribute)) return adinputs
def addIllumMaskToDQ(self, adinputs=None, suffix=None, illum_mask=None): """ Adds an illumination mask to each AD object Parameters ---------- suffix: str suffix to be added to output files mask: str/None name of illumination mask mask (None -> use default) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] # Getting all the filenames first prevents reopening the same file # for each science AD if illum_mask is None: illum_mask = [self._get_illum_mask_filename(ad) for ad in adinputs] for ad, illum in zip( *gt.make_lists(adinputs, illum_mask, force_ad=True)): if ad.phu.get(timestamp_key): log.warning( 'No changes will be made to {}, since it has ' 'already been processed by addIllumMaskToDQ'.format( ad.filename)) continue if illum is None: # So it can be zipped with the AD final_illum = [None] * len(ad) else: log.fullinfo("Using {} as illumination mask".format( illum.filename)) final_illum = gt.clip_auxiliary_data(ad, aux=illum, aux_type='bpm', return_dtype=DQ.datatype) for ext, illum_ext in zip(ad, final_illum): if illum_ext is not None: # Ensure we're only adding the unilluminated bit iext = np.where(illum_ext.data > 0, DQ.unilluminated, 0).astype(DQ.datatype) ext.mask = iext if ext.mask is None else ext.mask | iext # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def addIllumMaskToDQ(self, adinputs=None, suffix=None, illum_mask=None): """ Adds an illumination mask to each AD object Parameters ---------- suffix: str suffix to be added to output files mask: str/None name of illumination mask mask (None -> use default) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] # Getting all the filenames first prevents reopening the same file # for each science AD if illum_mask is None: illum_mask = [self._get_illum_mask_filename(ad) for ad in adinputs] for ad, illum in zip(*gt.make_lists(adinputs, illum_mask, force_ad=True)): if ad.phu.get(timestamp_key): log.warning('No changes will be made to {}, since it has ' 'already been processed by addIllumMaskToDQ'. format(ad.filename)) continue if illum is None: # So it can be zipped with the AD final_illum = [None] * len(ad) else: log.fullinfo("Using {} as illumination mask".format(illum.filename)) final_illum = gt.clip_auxiliary_data(ad, aux=illum, aux_type='bpm', return_dtype=DQ.datatype) for ext, illum_ext in zip(ad, final_illum): if illum_ext is not None: # Ensure we're only adding the unilluminated bit iext = np.where(illum_ext.data > 0, DQ.unilluminated, 0).astype(DQ.datatype) ext.mask = iext if ext.mask is None else ext.mask | iext # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def standardizeStructure(self, adinputs=None, **params): """ This primitive is used to standardize the structure of F2 data, specifically. Parameters ---------- suffix: str suffix to be added to output files attach_mdf: bool attach an MDF to the AD objects? (ignored if not tagged as SPECT) mdf: str full path of the MDF to attach """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] adoutputs = [] for ad, mdf in zip(*gt.make_lists(adinputs, params['mdf'])): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by standardizeStructure". format(ad.filename)) adoutputs.append(ad) continue # Attach an MDF to each input AstroData object if params["attach_mdf"] and 'SPECT' in ad.tags: ad = self.addMDF([ad], mdf=mdf)[0] # Raw FLAMINGOS-2 pixel data have three dimensions (2048x2048x1). # Remove the single length dimension from the pixel data. # CD3_3 keyword must also be removed or alignAndStack complains. ad = remove_single_length_dimension(ad) # Need to change dtype from int32 to float32, or else numpy will # promote to float64. There's no VAR or DQ at this stage. ad[0].data = ad[0].data.astype(np.float32) # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=params["suffix"], strip=True) adoutputs.append(ad) return adoutputs
def standardizeStructure(self, adinputs=None, **params): """ This primitive is used to standardize the structure of GMOS data, specifically. Parameters ---------- suffix: str suffix to be added to output files attach_mdf: bool attach an MDF to the AD objects? (ignored if not tagged as SPECT) mdf: str full path of the MDF to attach """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] adoutputs = [] # If attach_mdf=False, this just zips up the ADs with a list of Nones, # which has no side-effects. for ad, mdf in zip(*gt.make_lists(adinputs, params['mdf'])): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by standardizeStructure". format(ad.filename)) adoutputs.append(ad) continue # Attach an MDF to each input AstroData object if params["attach_mdf"] and 'SPECT' in ad.tags: ad = self.addMDF([ad], mdf=mdf)[0] # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=params["suffix"], strip=True) adoutputs.append(ad) return adoutputs
def addDQ(self, adinputs=None, **params): """ This primitive is used to add a DQ extension to the input AstroData object. The value of a pixel in the DQ extension will be the sum of the following: (0=good, 1=bad pixel (found in bad pixel mask), 2=pixel is in the non-linear regime, 4=pixel is saturated). This primitive will trim the BPM to match the input AstroData object(s). Parameters ---------- suffix: str suffix to be added to output files static_bpm: str Name of bad pixel mask ("default" -> use default from look-up table) If set to None, no static_bpm will be added. user_bpm: str Name of the bad pixel mask created by the user from flats and darks. It is an optional BPM that can be added to the static one. illum_mask: bool add illumination mask? """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys["addDQ"] sfx = params["suffix"] # Getting all the filenames first prevents reopening the same file # for each science AD static_bpm_list = params['static_bpm'] user_bpm_list = params['user_bpm'] if static_bpm_list == "default": static_bpm_list = [self._get_bpm_filename(ad) for ad in adinputs] for ad, static, user in zip(*gt.make_lists( adinputs, static_bpm_list, user_bpm_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning('No changes will be made to {}, since it has ' 'already been processed by addDQ'.format( ad.filename)) continue if static is None: # So it can be zipped with the AD final_static = [None] * len(ad) else: log.fullinfo("Using {} as static BPM".format(static.filename)) final_static = gt.clip_auxiliary_data(ad, aux=static, aux_type='bpm', return_dtype=DQ.datatype) if user is None: final_user = [None] * len(ad) else: log.fullinfo("Using {} as user BPM".format(user.filename)) final_user = gt.clip_auxiliary_data(ad, aux=user, aux_type='bpm', return_dtype=DQ.datatype) for ext, static_ext, user_ext in zip(ad, final_static, final_user): extver = ext.hdr['EXTVER'] if ext.mask is not None: log.warning( 'A mask already exists in extver {}'.format(extver)) continue non_linear_level = ext.non_linear_level() saturation_level = ext.saturation_level() # Need to create the array first for 3D raw F2 data, with 2D BPM ext.mask = np.zeros_like(ext.data, dtype=DQ.datatype) if static_ext is not None: ext.mask |= static_ext.data if user_ext is not None: ext.mask |= user_ext.data if saturation_level: log.fullinfo('Flagging saturated pixels in {}:{} ' 'above level {:.2f}'.format( ad.filename, extver, saturation_level)) ext.mask |= np.where(ext.data >= saturation_level, DQ.saturated, 0).astype(DQ.datatype) if non_linear_level: if saturation_level: if saturation_level > non_linear_level: log.fullinfo('Flagging non-linear pixels in {}:{} ' 'above level {:.2f}'.format( ad.filename, extver, non_linear_level)) ext.mask |= np.where( (ext.data >= non_linear_level) & (ext.data < saturation_level), DQ.non_linear, 0).astype(DQ.datatype) # Readout modes of IR detectors can result in # saturated pixels having values below the # saturation level. Flag those. Assume we have an # IR detector here because both non-linear and # saturation levels are defined and nonlin<sat regions, nregions = measurements.label( ext.data < non_linear_level) # In all my tests, region 1 has been the majority # of the image; however, I cannot guarantee that # this is always the case and therefore we should # check the size of each region region_sizes = measurements.labeled_comprehension( ext.data, regions, np.arange(1, nregions + 1), len, int, 0) # First, assume all regions are saturated, and # remove any very large ones. This is much faster # than progressively adding each region to DQ hidden_saturation_array = np.where( regions > 0, 4, 0).astype(DQ.datatype) for region in range(1, nregions + 1): # Limit of 10000 pixels for a hole is a bit arbitrary if region_sizes[region - 1] > 10000: hidden_saturation_array[regions == region] = 0 ext.mask |= hidden_saturation_array elif saturation_level < non_linear_level: log.warning('{}:{} has saturation level less than ' 'non-linear level'.format( ad.filename, extver)) else: log.fullinfo('Saturation and non-linear levels ' 'are the same for {}:{}. Only ' 'flagging saturated pixels'.format( ad.filename, extver)) else: log.fullinfo('Flagging non-linear pixels in {}:{} ' 'above level {:.2f}'.format( ad.filename, extver, non_linear_level)) ext.mask |= np.where(ext.data >= non_linear_level, DQ.non_linear, 0).astype(DQ.datatype) # Handle latency if reqested if params.get("latency", False): try: adinputs = self.addLatencyToDQ(adinputs, time=params["time"], non_linear=params["non_linear"]) except AttributeError: log.warning( "addLatencyToDQ() not defined in primitivesClass " + self.__class__.__name__) # Add the illumination mask if requested if params['add_illum_mask']: adinputs = self.addIllumMaskToDQ(adinputs, illum_mask=params["illum_mask"]) # Timestamp and update filenames for ad in adinputs: gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=sfx, strip=True) return adinputs
def QECorrect(self, adinputs=None, **params): """ This primitive applies a wavelength-dependent QE correction to a 2D spectral image, based on the wavelength solution of an associated processed_arc. It is only designed to work on FLATs, and therefore unmosaicked data. Parameters ---------- suffix: str suffix to be added to output files arc : {None, AstroData, str} Arc(s) with distortion map. """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] sfx = params["suffix"] arc = params["arc"] use_iraf = params["use_iraf"] do_cal = params["do_cal"] if do_cal == 'skip': log.warning("QE correction has been turned off.") return adinputs # Get a suitable arc frame (with distortion map) for every science AD if arc is None: arc_list = self.caldb.get_processed_arc(adinputs) else: arc_list = (arc, None) # Provide an arc AD object for every science frame, and an origin for ad, arc, origin in zip( *gt.make_lists(adinputs, *arc_list, force_ad=(1, ))): if ad.phu.get(timestamp_key): log.warning(f"{ad.filename}: already processed by QECorrect. " "Continuing.") continue if 'e2v' in ad.detector_name(pretty=True): log.stdinfo(f"{ad.filename} has the e2v CCDs, so no QE " "correction is necessary") continue if self.timestamp_keys['mosaicDetectors'] in ad.phu: log.warning(f"{ad.filename} has been processed by mosaic" "Detectors so QECorrect cannot be run") continue # Determines whether to multiply or divide by QE correction is_flat = 'FLAT' in ad.tags # If the arc's binning doesn't match, we may still be able to # fall back to the approximate solution xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin() if arc is not None and (arc.detector_x_bin() != xbin or arc.detector_y_bin() != ybin): log.warning("Science frame and arc have different binnings.") arc = None # The plan here is to attach the mosaic gWCS to the science frame, # apply an origin shift to put it in the frame of the arc, and # then use the arc's WCS to get the wavelength. If there's no arc, # we just use the science frame's WCS. # Since we're going to change that WCS, store it for restoration. original_wcs = [ext.wcs for ext in ad] try: transform.add_mosaic_wcs(ad, geotable) except ValueError: log.warning(f"{ad.filename} already has a 'mosaic' coordinate" "frame. This is unexpected but I'll continue.") if arc is None: if 'sq' in self.mode or do_cal == 'force': raise OSError(f"No processed arc listed for {ad.filename}") else: log.warning(f"{ad.filename}: no arc was specified. Using " "wavelength solution in science frame.") else: # OK, we definitely want to try to do this, get a wavelength solution origin_str = f" (obtained from {origin})" if origin else "" log.stdinfo(f"{ad.filename}: using the arc {arc.filename}" f"{origin_str}") if self.timestamp_keys[ 'determineWavelengthSolution'] not in arc.phu: msg = f"Arc {arc.filename} (for {ad.filename} has not been wavelength calibrated." if 'sq' in self.mode or do_cal == 'force': raise IOError(msg) else: log.warning(msg) # We'll be modifying this arc_wcs = deepcopy(arc[0].wcs) if 'distortion_corrected' not in arc_wcs.available_frames: msg = f"Arc {arc.filename} (for {ad.filename}) has no distortion model." if 'sq' in self.mode or do_cal == 'force': raise OSError(msg) else: log.warning(msg) # NB. At this point, we could have an arc that has no good # wavelength solution nor distortion correction. But we will # use its WCS rather than the science frame's because it must # have been supplied by the user. # This is GMOS so no need to be as generic as distortionCorrect ad_detsec = ad.detector_section() arc_detsec = arc.detector_section()[0] if (ad_detsec[0].x1, ad_detsec[-1].x2) != (arc_detsec.x1, arc_detsec.x2): raise ValueError("Cannot process the offsets between " f"{ad.filename} and {arc.filename}") yoff1 = arc_detsec.y1 - ad_detsec[0].y1 yoff2 = arc_detsec.y2 - ad_detsec[0].y2 arc_ext_shapes = [(ext.shape[0] - yoff1 + yoff2, ext.shape[1]) for ext in ad] arc_corners = np.concatenate([ transform.get_output_corners(ext.wcs.get_transform( ext.wcs.input_frame, 'mosaic'), input_shape=arc_shape, origin=(yoff1, 0)) for ext, arc_shape in zip(ad, arc_ext_shapes) ], axis=1) arc_origin = tuple( np.ceil(min(corners)) for corners in arc_corners) # So this is what was applied to the ARC to get the # mosaic frame to its pixel frame, in which the distortion # correction model was calculated. Convert coordinates # from python order to Model order. origin_shift = reduce( Model.__and__, [models.Shift(-origin) for origin in arc_origin[::-1]]) arc_wcs.insert_transform(arc_wcs.input_frame, origin_shift, after=True) array_info = gt.array_information(ad) if array_info.detector_shape == (1, 3): ccd2_indices = array_info.extensions[1] else: raise ValueError( f"{ad.filename} does not have 3 separate detectors") for index, ext in enumerate(ad): if index in ccd2_indices: continue # Use the WCS in the extension if we don't have an arc, # otherwise use the arc's mosaic->world transformation if arc is None: trans = ext.wcs.forward_transform else: trans = (ext.wcs.get_transform(ext.wcs.input_frame, 'mosaic') | arc_wcs.forward_transform) ygrid, xgrid = np.indices(ext.shape) # TODO: want with_units waves = trans(xgrid, ygrid)[0] * u.nm # Wavelength always axis 0 # Tapering required to prevent QE correction from blowing up # at the extremes (remember, this is a ratio, not the actual QE) # We use half-Gaussians to taper taper = np.ones_like(ext.data) taper_locut, taper_losig = 350 * u.nm, 25 * u.nm taper_hicut, taper_hisig = 1200 * u.nm, 200 * u.nm taper[waves < taper_locut] = np.exp(-( (waves[waves < taper_locut] - taper_locut) / taper_losig)**2) taper[waves > taper_hicut] = np.exp(-( (waves[waves > taper_hicut] - taper_hicut) / taper_hisig)**2) try: qe_correction = (qeModel(ext, use_iraf=use_iraf)( (waves / u.nm).to(u.dimensionless_unscaled).value). astype(np.float32) - 1) * taper + 1 except TypeError: # qeModel() returns None msg = f"No QE correction found for {ad.filename} extension {ext.id}" if 'sq' in self.mode: raise ValueError(msg) else: log.warning(msg) continue log.stdinfo(f"Mean relative QE of extension {ext.id} is " f"{qe_correction.mean():.5f}") if not is_flat: qe_correction = 1. / qe_correction ext.multiply(qe_correction) for ext, orig_wcs in zip(ad, original_wcs): ext.wcs = orig_wcs # Timestamp and update the filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=sfx, strip=True) return adinputs
def slitIllumCorrect(self, adinputs=None, slit_illum=None, do_illum=True, suffix="_illumCorrected"): """ This primitive will divide each SCI extension of the inputs by those of the corresponding slit illumination image. If the inputs contain VAR or DQ frames, those will also be updated accordingly due to the division on the data. Parameters ---------- adinputs : list of AstroData Data to be corrected. slit_illum : str or AstroData Slit illumination path or AstroData object. do_illum: bool, optional Perform slit illumination correction? (Default: True) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] qecorr_key = self.timestamp_keys['QECorrect'] if not do_illum: log.warning("Slit Illumination correction has been turned off.") return adinputs if slit_illum is None: raise NotImplementedError else: slit_illum_list = slit_illum # Provide a Slit Illum Ad object for every science frame ad_outputs = [] for ad, slit_illum_ad in zip( *gt.make_lists(adinputs, slit_illum_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by flatCorrect".format( ad.filename)) continue if slit_illum_ad is None: if self.mode in ['sq']: raise OSError( "No processed slit illumination listed for {}".format( ad.filename)) else: log.warning("No changes will be made to {}, since no slit " "illumination has been specified".format( ad.filename)) continue gt.check_inputs_match(ad, slit_illum_ad, check_shape=False) if not all( [e1.shape == e2.shape for (e1, e2) in zip(ad, slit_illum_ad)]): slit_illum_ad = gt.clip_auxiliary_data(adinput=[ad], aux=[slit_illum_ad])[0] log.info("Dividing the input AstroData object {} by this \n" "slit illumination file: \n{}".format( ad.filename, slit_illum_ad.filename)) ad_out = deepcopy(ad) ad_out.divide(slit_illum_ad) # Update the header and filename, copying QECORR keyword from flat ad_out.phu.set("SLTILLIM", slit_illum_ad.filename, self.keyword_comments["SLTILLIM"]) try: qecorr_value = slit_illum_ad.phu[qecorr_key] except KeyError: pass else: log.fullinfo( "Copying {} keyword from slit illumination".format( qecorr_key)) ad_out.phu.set(qecorr_key, qecorr_value, slit_illum_ad.phu.comments[qecorr_key]) gt.mark_history(ad_out, primname=self.myself(), keyword=timestamp_key) ad_out.update_filename(suffix=suffix, strip=True) if slit_illum_ad.path: add_provenance(ad_out, slit_illum_ad.filename, md5sum(slit_illum_ad.path) or "", self.myself()) ad_outputs.append(ad_out) return ad_outputs
def transferAttribute(self, adinputs=None, source=None, attribute=None): """ This primitive takes an attribute (e.g., "mask", or "OBJCAT") from the AD(s) in another ("source") stream and applies it to the ADs in this stream. There must be either the same number of ADs in each stream, or only 1 in the source stream. Parameters ---------- source: str name of stream containing ADs whose attributes you want attribute: str attribute to transfer from ADs in other stream """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) if source not in self.streams.keys(): log.info("Stream {} does not exist so nothing to transfer".format( source)) return adinputs source_length = len(self.streams[source]) if not (source_length == 1 or source_length == len(adinputs)): log.warning("Incompatible stream lengths: {} and {}".format( len(adinputs), source_length)) return adinputs log.stdinfo("Transferring attribute {} from stream {}".format( attribute, source)) # Keep track of whether we find anything to transfer, as failing to # do so might indicate a problem and we should warn the user found = False for ad1, ad2 in zip(*gt.make_lists(adinputs, self.streams[source])): # Attribute could be top-level or extension-level # Use deepcopy so references to original object don't remain if hasattr(ad2, attribute): try: setattr(ad1, attribute, copy.deepcopy(getattr(ad2, attribute))) except ValueError: # data, mask, are gettable not settable pass else: found = True continue for ext1, ext2 in zip(ad1, ad2): if hasattr(ext2, attribute): setattr(ext1, attribute, copy.deepcopy(getattr(ext2, attribute))) found = True if not found: log.warning( "Did not find any {} attributes to transfer".format(attribute)) return adinputs
def fringeCorrect(self, adinputs=None, **params): """ Correct science frames for the effects of fringing, using a fringe frame. The fringe frame is obtained either from a specified parameter, or the "fringe" stream, or the calibration database. This is basically a bookkeeping wrapper for subtractFringe(), which does all the work. Parameters ---------- suffix: str suffix to be added to output files fringe: list/str/AstroData/None fringe frame(s) to subtract """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] # Exit now if nothing needs a correction, to avoid an error when the # calibration search fails. If images with different exposure times # are used, some frames may not require a correction (but the calibration # search will succeed), so still need to check individual inputs later. if not any(self._needs_fringe_correction(ad) for ad in adinputs): log.stdinfo("No input images require a fringe correction.") return adinputs fringe = params["fringe"] scale = params["scale"] if fringe is None: try: fringe_list = self.streams['fringe'] assert len(fringe_list) == 1 scale = False log.stdinfo("Using fringe frame in 'fringe' stream. " "Setting scale=False") except (KeyError, AssertionError): self.getProcessedFringe(adinputs) fringe_list = self._get_cal(adinputs, "processed_fringe") else: fringe_list = fringe # Usual stuff to ensure that we have an iterable of the correct length # for the scale factors regardless of what the input is scale_factor = params["scale_factor"] try: factors = iter(scale_factor) except TypeError: factors = iter([scale_factor] * len(adinputs)) else: # In case a single-element list was passed if len(scale_factor) == 1: factors = iter(scale_factor * len(adinputs)) # Get a fringe AD object for every science frame for ad, fringe in zip(*gt.make_lists(adinputs, fringe_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by subtractFringe". format(ad.filename)) continue # Check the inputs have matching filters, binning, and shapes try: gt.check_inputs_match(ad, fringe) except ValueError: fringe = gt.clip_auxiliary_data(adinput=ad, aux=fringe, aux_type="cal") gt.check_inputs_match(ad, fringe) if scale: factor = next(factors) if factor is None: factor = self._calculate_fringe_scaling(ad, fringe) log.stdinfo("Scaling fringe frame by factor {:.3f} before " "subtracting from {}".format(factor, ad.filename)) # Since all elements of fringe_list might be references to the # same AD, need to make a copy before multiplying fringe_copy = deepcopy(fringe) fringe_copy.multiply(factor) ad.subtract(fringe_copy) else: ad.subtract(fringe) # Timestamp and update header and filename ad.phu.set("FRINGEIM", fringe.filename, self.keyword_comments["FRINGEIM"]) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=params["suffix"], strip=True) return adinputs
def processSlits(self, adinputs=None, **params): """ Compute and record the mean exposure epoch for a slit viewer image The 'slit viewer image' for each observation will almost certainly be a sequence of short exposures of the slit viewer camera, collected together for convenience. However, it cannot be guaranteed that slit viewer exposures will be taken throughout an entire science exposure; therefore, it is necessary to be able to compute the mean exposure epoch (i.e. the effective time that the combined slit viewer exposures were taken at). This allows a single science observation to be calibrated using multiple packets of slit viewer exposures, with appropriate weighting for the time delay between them. ``processSlits`` effectively computes a weighted average of the exposure epoch of all constituent slit viewer exposures, taking into account: - Length of each exposure; - Whether there is any overlap between the start/end of the exposure and the start/end of the overall 'image'; - Time of each exposure, relative to the start of the 'image'. Parameters ---------- suffix: str suffix to be added to output files slitflat: str/None name of the slitflat to use (if None, use the calibration system) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] flat_list = params["flat"] if flat_list is None: self.getProcessedSlitFlat(adinputs) flat_list = [ self._get_cal(ad, 'processed_slitflat') for ad in adinputs ] for ad, slitflat in zip( *gt.make_lists(adinputs, flat_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by processSlits".format( ad.filename)) continue if slitflat is None: log.warning("Unable to find slitflat calibration for {}; " "skipping".format(ad.filename)) continue else: sv_flat = slitflat[0].data # accumulators for computing the mean epoch sum_of_weights = 0.0 accum_weighted_time = 0.0 # Check the inputs have matching binning and SCI shapes. try: gt.check_inputs_match(adinput1=ad, adinput2=slitflat, check_filter=False) except ValueError: # This is most likely because the science frame has multiple # extensions and the slitflat needs to be copied slitflat = gt.clip_auxiliary_data(ad, slitflat, aux_type='cal') # An Error will be raised if they don't match now gt.check_inputs_match(ad, slitflat, check_filter=False) # get science start/end times sc_start = parse_timestr(ad.phu['UTSTART']) sc_end = parse_timestr(ad.phu['UTEND']) res = ad.res_mode() for ext in ad: sv_start = parse_timestr(ext.hdr['EXPUTST']) sv_end = parse_timestr(ext.hdr['EXPUTEND']) # compute overlap percentage and slit view image duration latest_start = max(sc_start, sv_start) earliest_end = min(sc_end, sv_end) overlap = (earliest_end - latest_start).seconds overlap = 0.0 if overlap < 0.0 else overlap # no overlap edge case sv_duration = (sv_end - sv_start).seconds overlap /= sv_duration # convert into a percentage # compute the offset (the value to be weighted), in seconds, # from the start of the science exposure offset = 42.0 # init value: overridden if overlap, else 0-scaled if sc_start <= sv_start and sv_end <= sc_end: offset = (sv_start - sc_start).seconds + sv_duration / 2.0 elif sv_start < sc_start: offset = overlap * sv_duration / 2.0 elif sv_end > sc_end: offset = overlap * sv_duration / 2.0 offset += (sv_start - sc_start).seconds # add flux-weighted offset (plus weight itself) to accumulators flux = _total_obj_flux(res, ext.data, sv_flat) weight = flux * overlap sum_of_weights += weight accum_weighted_time += weight * offset # final mean exposure epoch computation if sum_of_weights > 0.0: mean_offset = accum_weighted_time / sum_of_weights mean_offset = timedelta(seconds=mean_offset) # write the mean exposure epoch into the PHU sc_start = parse_timestr(ad.phu['UTSTART']) mean_epoch = sc_start + mean_offset ad.phu['AVGEPOCH'] = ( # hope this keyword string is ok mean_epoch.strftime("%H:%M:%S.%f")[:-3], 'Mean Exposure Epoch') # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=params["suffix"], strip=True) return adinputs
def addIllumMaskToDQ(self, adinputs=None, suffix=None, illum_mask=None, shift=None, max_shift=20): """ Adds an illumination mask to each AD object. This is only done for full-frame (not Central Spectrum) GMOS spectra, and is calculated by making a model illumination patter from the attached MDF and cross- correlating it with the spatial profile of the data. Parameters ---------- suffix : str suffix to be added to output files illum_mask : str/None name of illumination mask mask (None -> use default) shift : int/None user-defined shift to apply to illumination mask max_shift : int maximum shift (in unbinned pixels) allowable for the cross- correlation """ offset_dict = { ("GMOS-N", "Hamamatsu-N"): 1.5, ("GMOS-N", "e2vDD"): -0.2, ("GMOS-N", "EEV"): 0.7, ("GMOS-S", "Hamamatsu-S"): 5.5, ("GMOS-S", "EEV"): 3.8 } edges = 50 # try to eliminate issues at the very edges log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] # Do this now for memory management reasons. We'll be creating large # arrays temporarily and don't want the permanent mask arrays to # fragment the free memory. for ad in adinputs: for ext in ad: if ext.mask is None: ext.mask = np.zeros_like(ext.data).astype(DQ.datatype) for ad, illum in zip( *gt.make_lists(adinputs, illum_mask, force_ad=True)): if ad.phu.get(timestamp_key): log.warning( 'No changes will be made to {}, since it has ' 'already been processed by addIllumMaskToDQ'.format( ad.filename)) continue ybin = ad.detector_y_bin() ad_detsec = ad.detector_section() no_bridges = all(detsec.y1 > 1600 and detsec.y2 < 2900 for detsec in ad_detsec) has_48rows = (all(detsec.y2 == 4224 for detsec in ad_detsec) and 'Hamamatsu' in ad.detector_name(pretty=True)) if illum: log.fullinfo("Using {} as illumination mask".format( illum.filename)) final_illum = gt.clip_auxiliary_data(ad, aux=illum, aux_type='bpm', return_dtype=DQ.datatype) for ext, illum_ext in zip(ad, final_illum): if illum_ext is not None: # Ensure we're only adding the unilluminated bit iext = np.where(illum_ext.data > 0, DQ.unilluminated, 0).astype(DQ.datatype) ext.mask |= iext elif not no_bridges: # i.e. there are bridges. try: mdf = ad.MDF except AttributeError: log.warning(f"MDF not found for {ad.filename} - cannot " "add illumination mask.") continue # Default operation for GMOS full-frame LS # Sadly, we cannot do this reliably without concatenating the # arrays and using a big chunk of memory. row_medians = np.percentile(np.concatenate( [ext.data for ext in ad], axis=1), 95, axis=1) row_medians -= at.boxcar(row_medians, size=50 // ybin) # Construct a model of the slit illumination from the MDF # coefficients are from G-IRAF except c0, approx. from data model = np.zeros_like(row_medians, dtype=int) for ypos, ysize in mdf['slitpos_my', 'slitsize_my']: y = ypos + np.array([-0.5, 0.5]) * ysize c0 = offset_dict[ad.instrument(), ad.detector_name(pretty=True)] if ad.instrument() == "GMOS-S": c1, c2, c3 = (0.99911, -1.7465e-5, 3.0494e-7) else: c1, c2, c3 = (0.99591859227, 5.3042211333437e-8, 1.7447902551997e-7) yccd = ((c0 + y * (c1 + y * (c2 + y * c3))) * 1.611444 / ad.pixel_scale() + 0.5 * model.size).astype(int) model[yccd[0]:yccd[1] + 1] = 1 log.stdinfo("Expected slit location from pixels " f"{yccd[0]+1} to {yccd[1]+1}") if shift is None: max_shift = 50 mshift = max_shift // ybin + 2 mshift2 = mshift + edges # model[] indexing avoids reduction in signal as slit # is shifted off the top of the image cntr = model.size - edges - mshift2 - 1 xcorr = correlate(row_medians[edges:-edges], model[mshift2:-mshift2], mode='full')[cntr - mshift:cntr + mshift] # This line avoids numerical errors in the spline fit xcorr -= np.median(xcorr) # This calculates the offsets of each point from the # straight line between its neighbours std = (xcorr[1:-1] - 0.5 * (xcorr + np.roll(xcorr, 2))[2:]).std() xspline = fit_1D(xcorr, function="spline3", order=None, weights=np.full(len(xcorr), 1. / std)).evaluate() yshift = xspline.argmax() - mshift maxima = xspline[1:-1][np.logical_and( np.diff(xspline[:-1]) > 0, np.diff(xspline[1:]) < 0)] significant_maxima = (maxima > xspline.max() - 3 * std).sum() if significant_maxima > 1 or abs( yshift // ybin) > max_shift: log.warning( f"{ad.filename}: cross-correlation peak is" " untrustworthy so not adding illumination " "mask. Please re-run with a specified shift.") yshift = None else: yshift = shift if yshift is not None: log.stdinfo( f"{ad.filename}: Shifting mask by {yshift} pixels") row_mask = np.ones_like(model, dtype=int) if yshift < 0: row_mask[:yshift] = 1 - model[-yshift:] elif yshift > 0: row_mask[yshift:] = 1 - model[:-yshift] else: row_mask[:] = 1 - model for ext in ad: ext.mask |= (row_mask * DQ.unilluminated).astype( DQ.datatype)[:, np.newaxis] if has_48rows: actual_rows = 48 // ybin for ext in ad: ext.mask[:actual_rows] |= DQ.unilluminated # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def biasCorrect(self, adinputs=None, suffix=None, bias=None, do_cal=None): """ The biasCorrect primitive will subtract the science extension of the input bias frames from the science extension of the input science frames. The variance and data quality extension will be updated, if they exist. If no bias is provided, the calibration database(s) will be queried. Parameters ---------- suffix: str suffix to be added to output files bias: str/list of str bias(es) to subtract do_cal: str perform bias subtraction? """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] if do_cal == 'skip': log.warning("Bias correction has been turned off.") return adinputs if bias is None: bias_list = self.caldb.get_processed_bias(adinputs) else: bias_list = (bias, None) # Provide a bias AD object for every science frame, and an origin for ad, bias, origin in zip(*gt.make_lists(adinputs, *bias_list, force_ad=(1,))): if ad.phu.get(timestamp_key): log.warning(f"{ad.filename}: already processed by " "biasCorrect. Continuing.") continue if bias is None: if 'sq' not in self.mode and do_cal != 'force': log.warning("No changes will be made to {}, since no " "bias was specified".format(ad.filename)) continue else: log.warning(f"{ad.filename}: no bias was specified. " "Continuing.") continue try: gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) except ValueError: bias = gt.clip_auxiliary_data(ad, aux=bias, aux_type='cal') # An Error will be raised if they don't match now gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) origin_str = f" (obtained from {origin})" if origin else "" log.stdinfo(f"{ad.filename}: subtracting the bias " f"{bias.filename}{origin_str}") ad.subtract(bias) # Record bias used, timestamp, and update filename ad.phu.set('BIASIM', bias.filename, self.keyword_comments['BIASIM']) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) if bias.path: add_provenance(ad, bias.filename, md5sum(bias.path) or "", self.myself()) timestamp = datetime.now() return adinputs
def biasCorrect(self, adinputs=None, suffix=None, bias=None, do_bias=True): """ The biasCorrect primitive will subtract the science extension of the input bias frames from the science extension of the input science frames. The variance and data quality extension will be updated, if they exist. If no bias is provided, getProcessedBias will be called to ensure a bias exists for every adinput. Parameters ---------- suffix: str suffix to be added to output files bias: str/list of str bias(es) to subtract do_bias: bool perform bias subtraction? """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] if not do_bias: log.warning("Bias correction has been turned off.") return adinputs if bias is None: self.getProcessedBias(adinputs, refresh=False) bias_list = self._get_cal(adinputs, 'processed_bias') else: bias_list = bias # Provide a bias AD object for every science frame for ad, bias in zip(*gt.make_lists(adinputs, bias_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by biasCorrect". format(ad.filename)) continue if bias is None: if 'qa' in self.mode: log.warning("No changes will be made to {}, since no " "bias was specified".format(ad.filename)) continue else: raise IOError('No processed bias listed for {}'. format(ad.filename)) try: gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) except ValueError: bias = gt.clip_auxiliary_data(ad, aux=bias, aux_type='cal') # An Error will be raised if they don't match now gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) log.fullinfo('Subtracting this bias from {}:\n{}'. format(ad.filename, bias.filename)) ad.subtract(bias) # Record bias used, timestamp, and update filename ad.phu.set('BIASIM', bias.filename, self.keyword_comments['BIASIM']) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def addMDF(self, adinputs=None, suffix=None, mdf=None): """ This primitive is used to add an Mask Definition File (MDF) extension to the input AstroData object. This MDF extension consists of a FITS binary table with information about where the spectroscopy slits are in the focal plane mask. In IFU, it is the position of the fibers. In Multi-Object Spectroscopy, it is the position of the multiple slits. In longslit is it the position of the single slit. If only one MDF is provided, that MDF will be add to all input AstroData object(s). If more than one MDF is provided, the number of MDF AstroData objects must match the number of input AstroData objects. If no MDF is provided, the primitive will attempt to determine an appropriate MDF. Parameters ---------- suffix: str suffix to be added to output files mdf: str/None name of MDF to add (None => use default) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] mdf_list = mdf or self.caldb.get_calibrations(adinputs, caltype="mask").files for ad, mdf in zip(*gt.make_lists(adinputs, mdf_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning('No changes will be made to {}, since it has ' 'already been processed by addMDF'.format( ad.filename)) continue if hasattr(ad, 'MDF'): log.warning('An MDF extension already exists in {}, so no ' 'MDF will be added'.format(ad.filename)) continue if mdf is None: log.stdinfo('No MDF could be retrieved for {}'.format( ad.filename)) continue try: # This will raise some sort of exception unless the MDF file # has a single MDF Table extension ad.MDF = mdf.MDF except: if len(mdf.tables) == 1: ad.MDF = getattr(mdf, mdf.tables.pop()) else: log.warning('Cannot find MDF in {}, so no MDF will be ' 'added'.format(mdf.filename)) continue log.fullinfo('Attaching the MDF {} to {}'.format( mdf.filename, ad.filename)) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def fringeCorrect(self, adinputs=None, **params): """ Correct science frames for the effects of fringing, using a fringe frame. The fringe frame is obtained either from a specified parameter, or the "fringe" stream, or the calibration database. This is basically a bookkeeping wrapper for subtractFringe(), which does all the work. Parameters ---------- suffix: str suffix to be added to output files fringe: list/str/AstroData/None fringe frame(s) to subtract do_fringe: bool/None apply fringe correction? (None => use pipeline default for data) scale: bool/None scale fringe frame? (None => False if fringe frame has same group_id() as data scale_factor: float/sequence/None factor(s) to scale fringe """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] fringe = params["fringe"] scale = params["scale"] do_cal = params["do_cal"] # Exit now if nothing needs a correction, to avoid an error when the # calibration search fails. If images with different exposure times # are used, some frames may not require a correction (but the calibration # search will succeed), so still need to check individual inputs later. needs_correction = [self._needs_fringe_correction(ad) for ad in adinputs] if any(needs_correction): if do_cal == 'skip': log.warning("Fringe correction has been turned off but is " "recommended.") return adinputs else: if do_cal == 'procmode' or do_cal == 'skip': log.stdinfo("No input images require a fringe correction.") return adinputs else: # do_cal == 'force': log.warning("Fringe correction has been forced on but may not " "be required.") if fringe is None: # This logic is for QAP try: fringe_list = self.streams['fringe'] assert len(fringe_list) == 1 scale = False log.stdinfo("Using fringe frame in 'fringe' stream. " "Setting scale=False") fringe_list = (fringe_list[0], "stream") except (KeyError, AssertionError): fringe_list = self.caldb.get_processed_fringe(adinputs) else: fringe_list = (fringe, None) # Usual stuff to ensure that we have an iterable of the correct length # for the scale factors regardless of what the input is scale_factor = params["scale_factor"] try: factors = iter(scale_factor) except TypeError: factors = iter([scale_factor] * len(adinputs)) else: # In case a single-element list was passed if len(scale_factor) == 1: factors = iter(scale_factor * len(adinputs)) # Get a fringe AD object for every science frame for ad, fringe, origin, correct in zip(*gt.make_lists( adinputs, *fringe_list, needs_correction, force_ad=(1,))): if ad.phu.get(timestamp_key): log.warning(f"{ad.filename}: already processed by " "fringeCorrect. Continuing.") continue # Logic to deal with different exposure times where only # some inputs might require fringe correction # KL: for now, I'm not allowing the "force" to do anything when # the correction is not needed. if (do_cal == 'procmode' or do_cal == 'force') and not correct: log.stdinfo("{} does not require a fringe correction". format(ad.filename)) ad.update_filename(suffix=params["suffix"], strip=True) continue # At this point, we definitely want to do a fringe correction # so we'd better have a fringe frame! if fringe is None: if 'sq' not in self.mode and do_cal != 'force': log.warning("No changes will be made to {}, since no " "fringe frame has been specified". format(ad.filename)) continue else: log.warning(f"{ad.filename}: no fringe was specified. " "Continuing.") continue # Check the inputs have matching filters, binning, and shapes try: gt.check_inputs_match(ad, fringe) except ValueError: fringe = gt.clip_auxiliary_data(adinput=ad, aux=fringe, aux_type="cal") gt.check_inputs_match(ad, fringe) # origin_str = f" (obtained from {origin})" if origin else "" log.stdinfo(f"{ad.filename}: using the fringe frame " f"{fringe.filename}{origin_str}") matched_groups = (ad.group_id() == fringe.group_id()) if scale or (scale is None and not matched_groups): factor = next(factors) if factor is None: factor = self._calculate_fringe_scaling(ad, fringe) log.stdinfo("Scaling fringe frame by factor {:.3f} before " "subtracting from {}".format(factor, ad.filename)) # Since all elements of fringe_list might be references to the # same AD, need to make a copy before multiplying fringe_copy = deepcopy(fringe) fringe_copy.multiply(factor) ad.subtract(fringe_copy) else: if scale is None: log.stdinfo("Not scaling fringe frame with same group ID " "as {}".format(ad.filename)) ad.subtract(fringe) # Timestamp and update header and filename ad.phu.set("FRINGEIM", fringe.filename, self.keyword_comments["FRINGEIM"]) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=params["suffix"], strip=True) if fringe.path: add_provenance(ad, fringe.filename, md5sum(fringe.path) or "", self.myself()) return adinputs
def QECorrect(self, adinputs=None, **params): """ This primitive applies a wavelength-dependent QE correction to a 2D spectral image, based on the wavelength solution of an associated processed_arc. It is only designed to work on FLATs, and therefore unmosaicked data. Parameters ---------- suffix: str suffix to be added to output files arc : {None, AstroData, str} Arc(s) with distortion map. """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] sfx = params["suffix"] arc = params["arc"] # Get a suitable arc frame (with distortion map) for every science AD if arc is None: self.getProcessedArc(adinputs, refresh=False) arc_list = self._get_cal(adinputs, 'processed_arc') else: arc_list = arc for ad, arc in zip(*gt.make_lists(adinputs, arc_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by QECorrect". format(ad.filename)) continue if 'e2v' in ad.detector_name(pretty=True): log.stdinfo(f"{ad.filename} has the e2v CCDs, so no QE " "correction is necessary") continue if self.timestamp_keys['mosaicDetectors'] in ad.phu: log.warning(f"{ad.filename} has been processed by mosaic" "Detectors so QECorrect cannot be run") continue # Determines whether to multiply or divide by QE correction is_flat = 'FLAT' in ad.tags # If the arc's binning doesn't match, we may still be able to # fall back to the approximate solution xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin() if arc is not None and (arc.detector_x_bin() != xbin or arc.detector_y_bin() != ybin): log.warning("Science frame {} and arc {} have different binnings," "so cannot use arc".format(ad.filename, arc.filename)) arc = None # The plan here is to attach the mosaic gWCS to the science frame, # apply an origin shift to put it in the frame of the arc, and # then use the arc's WCS to get the wavelength. If there's no arc, # we just use the science frame's WCS. # Since we're going to change that WCS, store it for restoration. original_wcs = [ext.wcs for ext in ad] try: transform.add_mosaic_wcs(ad, geotable) except ValueError: log.warning(f"{ad.filename} already has a 'mosaic' coordinate" "frame. This is unexpected but I'll continue.") if arc is None: if 'sq' in self.mode: raise OSError(f"No processed arc listed for {ad.filename}") else: log.warning(f"No arc supplied for {ad.filename}") else: # OK, we definitely want to try to do this, get a wavelength solution if self.timestamp_keys['determineWavelengthSolution'] not in arc.phu: msg = f"Arc {arc.filename} (for {ad.filename} has not been wavelength calibrated." if 'sq' in self.mode: raise IOError(msg) else: log.warning(msg) # We'll be modifying this arc_wcs = deepcopy(arc[0].wcs) if 'distortion_corrected' not in arc_wcs.available_frames: msg = f"Arc {arc.filename} (for {ad.filename}) has no distortion model." if 'sq' in self.mode: raise OSError(msg) else: log.warning(msg) # NB. At this point, we could have an arc that has no good # wavelength solution nor distortion correction. But we will # use its WCS rather than the science frame's because it must # have been supplied by the user. # This is GMOS so no need to be as generic as distortionCorrect ad_detsec = ad.detector_section() arc_detsec = arc.detector_section()[0] if (ad_detsec[0].x1, ad_detsec[-1].x2) != (arc_detsec.x1, arc_detsec.x2): raise ValueError("I don't know how to process the " f"offsets between {ad.filename} " f"and {arc.filename}") yoff1 = arc_detsec.y1 - ad_detsec[0].y1 yoff2 = arc_detsec.y2 - ad_detsec[0].y2 arc_ext_shapes = [(ext.shape[0] - yoff1 + yoff2, ext.shape[1]) for ext in ad] arc_corners = np.concatenate([transform.get_output_corners( ext.wcs.get_transform(ext.wcs.input_frame, 'mosaic'), input_shape=arc_shape, origin=(yoff1, 0)) for ext, arc_shape in zip(ad, arc_ext_shapes)], axis=1) arc_origin = tuple(np.ceil(min(corners)) for corners in arc_corners) # So this is what was applied to the ARC to get the # mosaic frame to its pixel frame, in which the distortion # correction model was calculated. Convert coordinates # from python order to Model order. origin_shift = reduce(Model.__and__, [models.Shift(-origin) for origin in arc_origin[::-1]]) arc_wcs.insert_transform(arc_wcs.input_frame, origin_shift, after=True) array_info = gt.array_information(ad) if array_info.detector_shape == (1, 3): ccd2_indices = array_info.extensions[1] else: raise ValueError(f"{ad.filename} does not have 3 separate detectors") for index, ext in enumerate(ad): if index in ccd2_indices: continue # Use the WCS in the extension if we don't have an arc, # otherwise use the arc's mosaic->world transformation if arc is None: trans = ext.wcs.forward_transform else: trans = (ext.wcs.get_transform(ext.wcs.input_frame, 'mosaic') | arc_wcs.forward_transform) ygrid, xgrid = np.indices(ext.shape) # TODO: want with_units waves = trans(xgrid, ygrid)[0] * u.nm # Wavelength always axis 0 try: qe_correction = qeModel(ext)((waves / u.nm).to(u.dimensionless_unscaled).value).astype(np.float32) except TypeError: # qeModel() returns None msg = "No QE correction found for {}:{}".format(ad.filename, ext.hdr['EXTVER']) if 'sq' in self.mode: raise ValueError(msg) else: log.warning(msg) log.stdinfo("Mean relative QE of EXTVER {} is {:.5f}". format(ext.hdr['EXTVER'], qe_correction.mean())) if not is_flat: qe_correction = 1. / qe_correction qe_correction[qe_correction < 0] = 0 qe_correction[qe_correction > 10] = 0 ext.multiply(qe_correction) for ext, orig_wcs in zip(ad, original_wcs): ext.wcs = orig_wcs # Timestamp and update the filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=sfx, strip=True) return adinputs
def addMDF(self, adinputs=None, suffix=None, mdf=None): """ This primitive is used to add an MDF extension to the input AstroData object. If only one MDF is provided, that MDF will be add to all input AstroData object(s). If more than one MDF is provided, the number of MDF AstroData objects must match the number of input AstroData objects. If no MDF is provided, the primitive will attempt to determine an appropriate MDF. Parameters ---------- suffix: str suffix to be added to output files mdf: str/None name of MDF to add (None => use default) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] if mdf is None: self.getMDF(adinputs) mdf_list = [self._get_cal(ad, 'mask') for ad in adinputs] else: mdf_list = mdf for ad, mdf in zip(*gt.make_lists(adinputs, mdf_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning('No changes will be made to {}, since it has ' 'already been processed by addMDF'. format(ad.filename)) continue if hasattr(ad, 'MDF'): log.warning('An MDF extension already exists in {}, so no ' 'MDF will be added'.format(ad.filename)) continue if 'SPECT' not in ad.tags: log.stdinfo('{} is not spectroscopic data, so no MDF will ' 'be added'.format(ad.filename)) continue if mdf is None: log.stdinfo('No MDF could be retrieved for {}'. format(ad.filename)) continue try: # This will raise some sort of exception unless the MDF file # has a single MDF Table extension ad.MDF = mdf.MDF except: if len(mdf.tables) == 1: ad.MDF = getattr(mdf, mdf.tables.pop()) else: log.warning('Cannot find MDF in {}, so no MDF will be ' 'added'.format(mdf.filename)) continue log.fullinfo('Attaching the MDF {} to {}'.format(mdf.filename, ad.filename)) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def addIllumMaskToDQ(self, adinputs=None, suffix=None, illum_mask=None): """ Adds an illumination mask to each AD object Parameters ---------- suffix: str suffix to be added to output files illum_mask: str/None name of illumination mask mask (None -> use default) """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] for ad, illum in zip( *gt.make_lists(adinputs, illum_mask, force_ad=True)): if ad.phu.get(timestamp_key): log.warning( 'No changes will be made to {}, since it has ' 'already been processed by addIllumMaskToDQ'.format( ad.filename)) continue ad_detsec = ad.detector_section() no_bridges = all(detsec.y1 > 1600 and detsec.y2 < 2900 for detsec in ad_detsec) has_48rows = (all(detsec.y2 == 4224 for detsec in ad_detsec) and 'Hamamatsu' in ad.detector_name(pretty=True)) if illum: log.fullinfo("Using {} as illumination mask".format( illum.filename)) final_illum = gt.clip_auxiliary_data(ad, aux=illum, aux_type='bpm', return_dtype=DQ.datatype) for ext, illum_ext in zip(ad, final_illum): if illum_ext is not None: # Ensure we're only adding the unilluminated bit iext = np.where(illum_ext.data > 0, DQ.unilluminated, 0).astype(DQ.datatype) ext.mask = iext if ext.mask is None else ext.mask | iext elif not no_bridges: # i.e. there are bridges. # Default operation for GMOS full-frame LS # The 95% cut should ensure that we're sampling something # bright (even for an arc) # The max is intended to handle R150 data, where many of # the extensions are unilluminated row_medians = np.max(np.array( [np.percentile(ext.data, 95, axis=1) for ext in ad]), axis=0) rows = np.arange(len(row_medians)) m_init = models.Polynomial1D(degree=3) fit_it = fitting.FittingWithOutlierRemoval( fitting.LinearLSQFitter(), outlier_func=sigma_clip, sigma_upper=1, sigma_lower=3) m_final, _ = fit_it(m_init, rows, row_medians) model_fit = m_final(rows) # Find points which are significantly below the smooth illumination fit # First ensure we don't worry about single rows row_mask = at.boxcar(model_fit - row_medians > 0.1 * model_fit, operation=np.logical_and, size=1) row_mask = at.boxcar(row_mask, operation=np.logical_or, size=3) for ext in ad: ext.mask |= (row_mask * DQ.unilluminated).astype( DQ.datatype)[:, np.newaxis] if has_48rows: actual_rows = 48 // ad.detector_y_bin() for ext in ad: ext.mask[:actual_rows] |= DQ.unilluminated # Timestamp and update filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def addDQ(self, adinputs=None, **params): """ This primitive is used to add a DQ extension to the input AstroData object. The value of a pixel in the DQ extension will be the sum of the following: (0=good, 1=bad pixel (found in bad pixel mask), 2=pixel is in the non-linear regime, 4=pixel is saturated). This primitive will trim the BPM to match the input AstroData object(s). Parameters ---------- suffix: str suffix to be added to output files static_bpm: str Name of bad pixel mask ("default" -> use default from look-up table) If set to None, no static_bpm will be added. user_bpm: str Name of the bad pixel mask created by the user from flats and darks. It is an optional BPM that can be added to the static one. illum_mask: bool add illumination mask? """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys["addDQ"] sfx = params["suffix"] # Getting all the filenames first prevents reopening the same file # for each science AD static_bpm_list = params['static_bpm'] user_bpm_list = params['user_bpm'] if static_bpm_list == "default": static_bpm_list = [self._get_bpm_filename(ad) for ad in adinputs] for ad, static, user in zip(*gt.make_lists(adinputs, static_bpm_list, user_bpm_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning('No changes will be made to {}, since it has ' 'already been processed by addDQ'.format(ad.filename)) continue if static is None: # So it can be zipped with the AD final_static = [None] * len(ad) else: log.fullinfo("Using {} as static BPM".format(static.filename)) final_static = gt.clip_auxiliary_data(ad, aux=static, aux_type='bpm', return_dtype=DQ.datatype) if user is None: final_user = [None] * len(ad) else: log.fullinfo("Using {} as user BPM".format(user.filename)) final_user = gt.clip_auxiliary_data(ad, aux=user, aux_type='bpm', return_dtype=DQ.datatype) for ext, static_ext, user_ext in zip(ad, final_static, final_user): extver = ext.hdr['EXTVER'] if ext.mask is not None: log.warning('A mask already exists in extver {}'. format(extver)) continue non_linear_level = ext.non_linear_level() saturation_level = ext.saturation_level() # Need to create the array first for 3D raw F2 data, with 2D BPM ext.mask = np.zeros_like(ext.data, dtype=DQ.datatype) if static_ext is not None: ext.mask |= static_ext.data if user_ext is not None: ext.mask |= user_ext.data if saturation_level: log.fullinfo('Flagging saturated pixels in {}:{} ' 'above level {:.2f}'. format(ad.filename, extver, saturation_level)) ext.mask |= np.where(ext.data >= saturation_level, DQ.saturated, 0).astype(DQ.datatype) if non_linear_level: if saturation_level: if saturation_level > non_linear_level: log.fullinfo('Flagging non-linear pixels in {}:{} ' 'above level {:.2f}'. format(ad.filename, extver, non_linear_level)) ext.mask |= np.where((ext.data >= non_linear_level) & (ext.data < saturation_level), DQ.non_linear, 0).astype(DQ.datatype) # Readout modes of IR detectors can result in # saturated pixels having values below the # saturation level. Flag those. Assume we have an # IR detector here because both non-linear and # saturation levels are defined and nonlin<sat regions, nregions = measurements.label( ext.data < non_linear_level) # In all my tests, region 1 has been the majority # of the image; however, I cannot guarantee that # this is always the case and therefore we should # check the size of each region region_sizes = measurements.labeled_comprehension( ext.data, regions, np.arange(1, nregions+1), len, int, 0) # First, assume all regions are saturated, and # remove any very large ones. This is much faster # than progressively adding each region to DQ hidden_saturation_array = np.where(regions > 0, 4, 0).astype(DQ.datatype) for region in range(1, nregions+1): # Limit of 10000 pixels for a hole is a bit arbitrary if region_sizes[region-1] > 10000: hidden_saturation_array[regions==region] = 0 ext.mask |= hidden_saturation_array elif saturation_level < non_linear_level: log.warning('{}:{} has saturation level less than ' 'non-linear level'.format(ad.filename, extver)) else: log.fullinfo('Saturation and non-linear levels ' 'are the same for {}:{}. Only ' 'flagging saturated pixels'. format(ad.filename, extver)) else: log.fullinfo('Flagging non-linear pixels in {}:{} ' 'above level {:.2f}'. format(ad.filename, extver, non_linear_level)) ext.mask |= np.where(ext.data >= non_linear_level, DQ.non_linear, 0).astype(DQ.datatype) # Handle latency if reqested if params.get("latency", False): try: adinputs = self.addLatencyToDQ(adinputs, time=params["time"], non_linear=params["non_linear"]) except AttributeError: log.warning("addLatencyToDQ() not defined in primitivesClass " + self.__class__.__name__) # Add the illumination mask if requested if params['add_illum_mask']: adinputs = self.addIllumMaskToDQ(adinputs, illum_mask=params["illum_mask"]) # Timestamp and update filenames for ad in adinputs: gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=sfx, strip=True) return adinputs
def biasCorrect(self, adinputs=None, suffix=None, bias=None, do_bias=True): """ The biasCorrect primitive will subtract the science extension of the input bias frames from the science extension of the input science frames. The variance and data quality extension will be updated, if they exist. If no bias is provided, getProcessedBias will be called to ensure a bias exists for every adinput. Parameters ---------- suffix: str suffix to be added to output files bias: str/list of str bias(es) to subtract do_bias: bool perform bias subtraction? """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] if not do_bias: log.warning("Bias correction has been turned off.") return adinputs if bias is None: self.getProcessedBias(adinputs, refresh=False) bias_list = self._get_cal(adinputs, 'processed_bias') else: bias_list = bias # Provide a bias AD object for every science frame for ad, bias in zip( *gt.make_lists(adinputs, bias_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by biasCorrect".format( ad.filename)) continue if bias is None: if 'qa' in self.mode: log.warning("No changes will be made to {}, since no " "bias was specified".format(ad.filename)) continue else: raise OSError('No processed bias listed for {}'.format( ad.filename)) try: gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) except ValueError: bias = gt.clip_auxiliary_data(ad, aux=bias, aux_type='cal') # An Error will be raised if they don't match now gt.check_inputs_match(ad, bias, check_filter=False, check_units=True) log.fullinfo('Subtracting this bias from {}:\n{}'.format( ad.filename, bias.filename)) ad.subtract(bias) # Record bias used, timestamp, and update filename ad.phu.set('BIASIM', bias.filename, self.keyword_comments['BIASIM']) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) if bias.path: add_provenance(ad, bias.filename, md5sum(bias.path) or "", self.myself()) timestamp = datetime.now() return adinputs
def applyQECorrection(self, adinputs=None, **params): """ This primitive applies a wavelength-dependent QE correction to a 2D spectral image, based on the wavelength solution of an associated processed_arc. It is only designed to work on FLATs, and therefore unmosaicked data. Parameters ---------- suffix: str suffix to be added to output files """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] sfx = params["suffix"] arc = params["arc"] # Get a suitable arc frame (with distortion map) for every science AD if arc is None: self.getProcessedArc(adinputs, refresh=False) arc_list = self._get_cal(adinputs, 'processed_arc') else: arc_list = arc distort_model = models.Identity(2) for ad, arc in zip(*gt.make_lists(adinputs, arc_list, force_ad=True)): if ad.phu.get(timestamp_key): log.warning( "No changes will be made to {}, since it has " "already been processed by applyQECorrection".format( ad.filename)) continue if 'e2v' in ad.detector_name(pretty=True): log.warning("{} has the e2v CCDs, so no QE correction " "is necessary".format(ad.filename)) continue # Determines whether to multiply or divide by QE correction is_flat = 'FLAT' in ad.tags # If the arc's binning doesn't match, we may still be able to # fall back to the approximate solution xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin() if arc is not None and (arc.detector_x_bin() != xbin or arc.detector_y_bin() != ybin): log.warning( "Science frame {} and arc {} have different binnings," "so cannot use arc".format(ad.filename, arc.filename)) arc = None # OK, we definitely want to try to do this, get a wavelength solution try: wavecal = arc[0].WAVECAL except (TypeError, AttributeError): wave_model = None else: model_dict = dict(zip(wavecal['name'], wavecal['coefficients'])) wave_model = astromodels.dict_to_chebyshev(model_dict) if not isinstance(wave_model, models.Chebyshev1D): log.warning("Problem reading wavelength solution from arc " "{}".format(arc.filename)) if wave_model is None: if 'sq' in self.mode: raise OSError("No wavelength solution for {}".format( ad.filename)) else: log.warning("Using approximate wavelength solution for " "{}".format(ad.filename)) try: fitcoord = arc[0].FITCOORD except (TypeError, AttributeError): # distort_model already has Identity inverse so nothing required pass else: # TODO: This is copied from determineDistortion() and will need # to be refactored out. Or we might be able to simply replace it # with a gWCS.pixel_to_world() call model_dict = dict( zip(fitcoord['inv_name'], fitcoord['inv_coefficients'])) m_inverse = astromodels.dict_to_chebyshev(model_dict) if not isinstance(m_inverse, models.Chebyshev2D): log.warning("Problem reading distortion model from arc " "{}".format(arc.filename)) else: distort_model.inverse = models.Mapping( (0, 1, 1)) | (m_inverse & models.Identity(1)) if distort_model.inverse == distort_model: # Identity(2) if 'sq' in self.mode: raise OSError("No distortion model for {}".format( ad.filename)) else: log.warning( "Proceeding without a disortion correction for " "{}".format(ad.filename)) ad_detsec = ad.detector_section() adg = transform.create_mosaic_transform(ad, geotable) if arc is not None: arc_detsec = arc.detector_section()[0] shifts = [ c1 - c2 for c1, c2 in zip( np.array(ad_detsec).min(axis=0), arc_detsec) ] xshift, yshift = shifts[0] / xbin, shifts[2] / ybin # x1, y1 if xshift or yshift: log.stdinfo("Found a shift of ({},{}) pixels between " "{} and the calibration.".format( xshift, yshift, ad.filename)) add_shapes, add_transforms = [], [] for (arr, trans) in adg: # Try to work out shape of this Block in the unmosaicked # arc, and then apply a shift to align it with the # science Block before applying the same transform. if xshift == 0: add_shapes.append( ((arc_detsec.y2 - arc_detsec.y1) // ybin, arr.shape[1])) else: add_shapes.append( (arr.shape[0], (arc_detsec.x2 - arc_detsec.x1) // xbin)) t = transform.Transform( models.Shift(-xshift) & models.Shift(-yshift)) t.append(trans) add_transforms.append(t) adg.calculate_output_shape( additional_array_shapes=add_shapes, additional_transforms=add_transforms) origin_shift = models.Shift(-adg.origin[1]) & models.Shift( -adg.origin[0]) for t in adg.transforms: t.append(origin_shift) # Irrespective of arc or not, apply the distortion model (it may # be Identity), recalculate output_shape and reset the origin for t in adg.transforms: t.append(distort_model.copy()) adg.calculate_output_shape() adg.reset_origin() # Now we know the shape of the output, we can construct the # approximate wavelength solution; ad.dispersion() returns a list! if wave_model is None: wave_model = ( models.Shift(-0.5 * adg.output_shape[1]) | models.Scale(ad.dispersion(asNanometers=True)[0]) | models.Shift(ad.central_wavelength(asNanometers=True))) for ccd, (block, trans) in enumerate(adg, start=1): if ccd == 2: continue for ext, corner in zip(block, block.corners): ygrid, xgrid = np.indices(ext.shape) xgrid += corner[1] # No need for ygrid xnew = trans(xgrid, ygrid)[0] # Some unit-based stuff here to prepare for gWCS waves = wave_model(xnew) * u.nm try: qe_correction = qeModel(ext)( (waves / u.nm).to(u.dimensionless_unscaled).value) except TypeError: # qeModel() returns None msg = "No QE correction found for {}:{}".format( ad.filename, ext.hdr['EXTVER']) if 'sq' in self.mode: raise ValueError(msg) else: log.warning(msg) log.fullinfo( "Mean relative QE of EXTVER {} is {:.5f}".format( ext.hdr['EXTVER'], qe_correction.mean())) if not is_flat: qe_correction = 1. / qe_correction qe_correction[qe_correction < 0] = 0 qe_correction[qe_correction > 10] = 0 ext.multiply(qe_correction) # Timestamp and update the filename gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=sfx, strip=True) return adinputs