def trimOverscan(self, adinputs=None, suffix=None): """ The trimOverscan primitive trims the overscan region from the input AstroData object and updates the headers. Parameters ---------- suffix: str suffix to be added to output files """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] for ad in adinputs: if ad.phu.get(timestamp_key) is not None: log.warning('No changes will be made to {}, since it has ' 'already been processed by trimOverscan'. format(ad.filename)) continue ad = gt.trim_to_data_section(ad, keyword_comments=self.keyword_comments) # Set keyword, timestamp, and update filename ad.phu.set('TRIMMED', 'yes', self.keyword_comments['TRIMMED']) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def trimOverscan(self, adinputs=None, suffix=None): """ The trimOverscan primitive trims the overscan region from the input AstroData object and updates the headers. Parameters ---------- suffix: str suffix to be added to output files """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] for ad in adinputs: if ad.phu.get(timestamp_key) is not None: log.warning('No changes will be made to {}, since it has ' 'already been processed by trimOverscan'.format( ad.filename)) continue ad = gt.trim_to_data_section( ad, keyword_comments=self.keyword_comments) # Set keyword, timestamp, and update filename ad.phu.set('TRIMMED', 'yes', self.keyword_comments['TRIMMED']) gt.mark_history(ad, primname=self.myself(), keyword=timestamp_key) ad.update_filename(suffix=suffix, strip=True) return adinputs
def mosaicDetectors(self, adinputs=None, **params): """ This primitive does a full mosaic of all the arrays in an AD object. An appropriate geometry_conf.py module containing geometric information is required. Parameters ---------- suffix: str suffix to be added to output files. sci_only: bool mosaic only SCI image data. Default is False order: int (1-5) order of spline interpolation """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] suffix = params['suffix'] order = params['order'] attributes = ['data'] if params['sci_only'] else None geotable = import_module('.geometry_conf', self.inst_lookups) adoutputs = [] for ad in adinputs: if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by mosaicDetectors". format(ad.filename)) adoutputs.append(ad) continue if len(ad) == 1: log.warning("{} has only one extension, so there's nothing " "to mosaic".format(ad.filename)) adoutputs.append(ad) continue # If there's an overscan section, we must trim it before mosaicking try: overscan_kw = ad._keyword_for('overscan_section') except AttributeError: # doesn't exist for this AD, so carry on pass else: if overscan_kw in ad.hdr: ad = gt.trim_to_data_section(ad, self.keyword_comments) adg = transform.create_mosaic_transform(ad, geotable) ad_out = adg.transform(attributes=attributes, order=order, process_objcat=False) ad_out.orig_filename = ad.filename gt.mark_history(ad_out, primname=self.myself(), keyword=timestamp_key) ad_out.update_filename(suffix=suffix, strip=True) adoutputs.append(ad_out) return adoutputs
def test_trim_to_data_section(self): ad = astrodata.open(os.path.join(TESTDATAPATH, 'GMOS', 'S20160914S0274.fits')) new_crpix1 = [ext.hdr['CRPIX1'] - (32 if ext.hdr['EXTVER'] % 2 == 0 else 0) for ext in ad] ret = gt.trim_to_data_section(ad, keyword_comments) assert all([ext.data.shape == (2112,256) for ext in ret]) for rv, cv in zip(ret.hdr['CRPIX1'], new_crpix1): assert abs(rv -cv) < 0.001
def mosaicDetectors(self, adinputs=None, **params): """ This primitive does a full mosaic of all the arrays in an AD object. An appropriate geometry_conf.py module containing geometric information is required. Parameters ---------- suffix: str suffix to be added to output files. sci_only: bool mosaic only SCI image data. Default is False order: int (1-5) order of spline interpolation """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] suffix = params['suffix'] order = params['order'] attributes = ['data'] if params['sci_only'] else None geotable = import_module('.geometry_conf', self.inst_lookups) adoutputs = [] for ad in adinputs: if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by mosaicDetectors".format( ad.filename)) adoutputs.append(ad) continue if len(ad) == 1: log.warning("{} has only one extension, so there's nothing " "to mosaic".format(ad.filename)) adoutputs.append(ad) continue # If there's an overscan section, we must trim it before mosaicking try: overscan_kw = ad._keyword_for('overscan_section') except AttributeError: # doesn't exist for this AD, so carry on pass else: if overscan_kw in ad.hdr: ad = gt.trim_to_data_section(ad, self.keyword_comments) # Create the blocks (individual physical detectors) array_info = gt.array_information(ad) blocks = [ Block(ad[arrays], shape=shape) for arrays, shape in zip( array_info.extensions, array_info.array_shapes) ] offsets = [ ad[exts[0]].array_section() for exts in array_info.extensions ] detname = ad.detector_name() xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin() geometry = geotable.geometry[detname] default_shape = geometry.get('default_shape') adg = AstroDataGroup() for block, origin, offset in zip(blocks, array_info.origins, offsets): # Origins are in (x, y) order in LUT block_geom = geometry[origin[::-1]] nx, ny = block_geom.get('shape', default_shape) nx /= xbin ny /= ybin shift = block_geom.get('shift', (0, 0)) rot = block_geom.get('rotation', 0.) mag = block_geom.get('magnification', (1, 1)) transform = Transform() # Shift the Block's coordinates based on its location within # the full array, to ensure any rotation takes place around # the true centre. if offset.x1 != 0 or offset.y1 != 0: transform.append( models.Shift(float(offset.x1) / xbin) & models.Shift(float(offset.y1) / ybin)) if rot != 0 or mag != (1, 1): # Shift to centre, do whatever, and then shift back transform.append( models.Shift(-0.5 * (nx - 1)) & models.Shift(-0.5 * (ny - 1))) if rot != 0: # Cope with non-square pixels by scaling in one # direction to make them square before applying the # rotation, and then reversing that. if xbin != ybin: transform.append( models.Identity(1) & models.Scale(ybin / xbin)) transform.append(models.Rotation2D(rot)) if xbin != ybin: transform.append( models.Identity(1) & models.Scale(xbin / ybin)) if mag != (1, 1): transform.append( models.Scale(mag[0]) & models.Scale(mag[1])) transform.append( models.Shift(0.5 * (nx - 1)) & models.Shift(0.5 * (ny - 1))) transform.append( models.Shift(float(shift[0]) / xbin) & models.Shift(float(shift[1]) / ybin)) adg.append(block, transform) adg.set_reference() ad_out = adg.transform(attributes=attributes, order=order, process_objcat=False) ad_out.orig_filename = ad.filename gt.mark_history(ad_out, primname=self.myself(), keyword=timestamp_key) ad_out.update_filename(suffix=suffix, strip=True) adoutputs.append(ad_out) return adoutputs
def test_split_mosaic_into_extensions(request): """ Tests helper function that split a mosaicked data into several extensions based on another multi-extension file that contains gWCS. """ astrofaker = pytest.importorskip("astrofaker") ad = astrofaker.create('GMOS-S') ad.init_default_extensions(binning=2) ad = transform.add_mosaic_wcs(ad, geotable) ad = gt.trim_to_data_section( ad, keyword_comments={'NAXIS1': "", 'NAXIS2': "", 'DATASEC': "", 'TRIMSEC': "", 'CRPIX1': "", 'CRPIX2': ""}) for i, ext in enumerate(ad): x1 = ext.detector_section().x1 x2 = ext.detector_section().x2 xb = ext.detector_x_bin() data = np.arange(x1 // xb, x2 // xb)[np.newaxis, :] data = np.repeat(data, ext.data.shape[0], axis=0) data = data + 0.1 * (0.5 - np.random.random(data.shape)) ext.data = data mosaic_ad = transform.resample_from_wcs( ad, "mosaic", attributes=None, order=1, process_objcat=False) mosaic_ad[0].data = np.pad(mosaic_ad[0].data, 10, mode='edge') mosaic_ad[0].hdr[mosaic_ad._keyword_for('data_section')] = \ '[1:{},1:{}]'.format(*mosaic_ad[0].shape[::-1]) ad2 = primitives_gmos_longslit._split_mosaic_into_extensions( ad, mosaic_ad, border_size=10) if request.config.getoption("--do-plots"): palette = copy(plt.cm.viridis) palette.set_bad('r', 1) fig = plt.figure(num="Test: Split Mosaic Into Extensions", figsize=(8, 6.5), dpi=120) fig.suptitle("Test Split Mosaic Into Extensions\n Difference between" " input and mosaicked/demosaicked data") gs = fig.add_gridspec(nrows=4, ncols=len(ad) // 3, wspace=0.1, height_ratios=[1, 1, 1, 0.1]) for i, (ext, ext2) in enumerate(zip(ad, ad2)): data1 = ext.data data2 = ext2.data diff = np.ma.masked_array(data1 - data2, mask=np.abs(data1 - data2) > 1) height, width = data1.shape row = i // 4 col = i % 4 ax = fig.add_subplot(gs[row, col]) ax.set_title("Ext {}".format(i + 1)) ax.set_xticks([]) ax.set_xticklabels([]) ax.set_yticks([]) ax.set_yticklabels([]) _ = [ax.spines[s].set_visible(False) for s in ax.spines] if col == 0: ax.set_ylabel("Det {}".format(row + 1)) sub_gs = gridspec.GridSpecFromSubplotSpec(2, 2, ax, wspace=0.05, hspace=0.05) for j in range(4): sx = fig.add_subplot(sub_gs[j]) im = sx.imshow(diff, origin='lower', cmap=palette, vmin=-0.1, vmax=0.1) sx.set_xticks([]) sx.set_yticks([]) sx.set_xticklabels([]) sx.set_yticklabels([]) _ = [sx.spines[s].set_visible(False) for s in sx.spines] if j == 0: sx.set_xlim(0, 25) sx.set_ylim(height - 25, height) if j == 1: sx.set_xlim(width - 25, width) sx.set_ylim(height - 25, height) if j == 2: sx.set_xlim(0, 25) sx.set_ylim(0, 25) if j == 3: sx.set_xlim(width - 25, width) sx.set_ylim(0, 25) cax = fig.add_subplot(gs[3, :]) cbar = plt.colorbar(im, cax=cax, orientation="horizontal") cbar.set_label("Difference levels") os.makedirs(PLOT_PATH, exist_ok=True) fig.savefig( os.path.join(PLOT_PATH, "test_split_mosaic_into_extensions.png")) # Actual test ---- for i, (ext, ext2) in enumerate(zip(ad, ad2)): data1 = np.ma.masked_array(ext.data[1:-1, 1:-1], mask=ext.mask) data2 = np.ma.masked_array(ext2.data[1:-1, 1:-1], mask=ext2.mask) np.testing.assert_almost_equal(data1, data2, decimal=1)
def mosaicDetectors(self, adinputs=None, **params): """ This primitive does a full mosaic of all the arrays in an AD object. An appropriate geometry_conf.py module containing geometric information is required. Parameters ---------- suffix: str suffix to be added to output files. sci_only: bool mosaic only SCI image data. Default is False order: int (1-5) order of spline interpolation """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] suffix = params['suffix'] order = params['order'] attributes = ['data'] if params['sci_only'] else None geotable = import_module('.geometry_conf', self.inst_lookups) adoutputs = [] for ad in adinputs: if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by mosaicDetectors". format(ad.filename)) adoutputs.append(ad) continue if len(ad) == 1: log.warning("{} has only one extension, so there's nothing " "to mosaic".format(ad.filename)) adoutputs.append(ad) continue # If there's an overscan section, we must trim it before mosaicking try: overscan_kw = ad._keyword_for('overscan_section') except AttributeError: # doesn't exist for this AD, so carry on pass else: if overscan_kw in ad.hdr: ad = gt.trim_to_data_section(ad, self.keyword_comments) # Create the blocks (individual physical detectors) array_info = gt.array_information(ad) blocks = [Block(ad[arrays], shape=shape) for arrays, shape in zip(array_info.extensions, array_info.array_shapes)] offsets = [ad[exts[0]].array_section() for exts in array_info.extensions] detname = ad.detector_name() xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin() geometry = geotable.geometry[detname] default_shape = geometry.get('default_shape') adg = AstroDataGroup() for block, origin, offset in zip(blocks, array_info.origins, offsets): # Origins are in (x, y) order in LUT block_geom = geometry[origin[::-1]] nx, ny = block_geom.get('shape', default_shape) nx /= xbin ny /= ybin shift = block_geom.get('shift', (0, 0)) rot = block_geom.get('rotation', 0.) mag = block_geom.get('magnification', (1, 1)) transform = Transform() # Shift the Block's coordinates based on its location within # the full array, to ensure any rotation takes place around # the true centre. if offset.x1 != 0 or offset.y1 != 0: transform.append(models.Shift(float(offset.x1) / xbin) & models.Shift(float(offset.y1) / ybin)) if rot != 0 or mag != (1, 1): # Shift to centre, do whatever, and then shift back transform.append(models.Shift(-0.5*(nx-1)) & models.Shift(-0.5*(ny-1))) if rot != 0: # Cope with non-square pixels by scaling in one # direction to make them square before applying the # rotation, and then reversing that. if xbin != ybin: transform.append(models.Identity(1) & models.Scale(ybin / xbin)) transform.append(models.Rotation2D(rot)) if xbin != ybin: transform.append(models.Identity(1) & models.Scale(xbin / ybin)) if mag != (1, 1): transform.append(models.Scale(mag[0]) & models.Scale(mag[1])) transform.append(models.Shift(0.5*(nx-1)) & models.Shift(0.5*(ny-1))) transform.append(models.Shift(float(shift[0]) / xbin) & models.Shift(float(shift[1]) / ybin)) adg.append(block, transform) adg.set_reference() ad_out = adg.transform(attributes=attributes, order=order, process_objcat=False) ad_out.orig_filename = ad.filename gt.mark_history(ad_out, primname=self.myself(), keyword=timestamp_key) ad_out.update_filename(suffix=suffix, strip=True) adoutputs.append(ad_out) return adoutputs
def mosaicDetectors(self, adinputs=None, **params): """ This primitive does a full mosaic of all the arrays in an AD object. An appropriate geometry_conf.py module containing geometric information is required. Parameters ---------- suffix: str suffix to be added to output files. sci_only: bool mosaic only SCI image data. Default is False order: int (1-5) order of spline interpolation """ log = self.log log.debug(gt.log_message("primitive", self.myself(), "starting")) timestamp_key = self.timestamp_keys[self.myself()] suffix = params['suffix'] order = params['order'] attributes = ['data'] if params['sci_only'] else None geotable = import_module('.geometry_conf', self.inst_lookups) adoutputs = [] for ad in adinputs: if ad.phu.get(timestamp_key): log.warning("No changes will be made to {}, since it has " "already been processed by mosaicDetectors".format( ad.filename)) adoutputs.append(ad) continue if len(ad) == 1: log.warning("{} has only one extension, so there's nothing " "to mosaic".format(ad.filename)) adoutputs.append(ad) continue if not all( np.issubdtype(ext.data.dtype, np.floating) for ext in ad): log.warning("Cannot mosaic {} with non-floating point data. " "Use tileArrays instead".format(ad.filename)) adoutputs.append(ad) continue transform.add_mosaic_wcs(ad, geotable) # If there's an overscan section in the data, this will crash, but # we can catch that, trim, and try again. Don't catch anything else try: ad_out = transform.resample_from_wcs(ad, "mosaic", attributes=attributes, order=order, process_objcat=False) except ValueError as e: if 'data sections' in repr(e): ad = gt.trim_to_data_section(ad, self.keyword_comments) ad_out = transform.resample_from_wcs(ad, "mosaic", attributes=attributes, order=order, process_objcat=False) else: raise e ad_out.orig_filename = ad.filename gt.mark_history(ad_out, primname=self.myself(), keyword=timestamp_key) ad_out.update_filename(suffix=suffix, strip=True) adoutputs.append(ad_out) return adoutputs
def scaleFringeToScience(self, rc): """ This primitive will scale the fringes to their matching science data The fringes should be in the stream this primitive is called on, and the reference science frames should be loaded into the RC, as, eg. rc["science"] = adinput. There are two ways to find the value to scale fringes by: 1. If stats_scale is set to True, the equation: (letting science data = b (or B), and fringe = a (or A)) arrayB = where({where[SCIb < (SCIb.median+2.5*SCIb.std)]} > [SCIb.median-3*SCIb.std]) scale = arrayB.std / SCIa.std The section of the SCI arrays to use for calculating these statistics is the CCD2 SCI data excluding the outer 5% pixels on all 4 sides. Future enhancement: allow user to choose section 2. If stats_scale=False, then scale will be calculated using: exposure time of science / exposure time of fringe :param stats_scale: Use statistics to calculate the scale values, rather than exposure time :type stats_scale: Python boolean (True/False) """ # Instantiate the log log = gemLog.getGeminiLog(logType=rc["logType"], logLevel=rc["logLevel"]) # Log the standard "starting primitive" debug message log.debug(gt.log_message("primitive", "scaleFringeToScience", "starting")) # Define the keyword to be used for the time stamp for this primitive timestamp_key = self.timestamp_keys["scaleFringeToScience"] # Check for user-supplied science frames fringe = rc.get_inputs_as_astrodata() science_param = rc["science"] fringe_dict = None if science_param is not None: # The user supplied an input to the science parameter if not isinstance(science_param, list): science_list = [science_param] else: science_list = science_param # If there is one fringe and multiple science frames, # the fringe must be deepcopied to allow it to be # scaled separately for each frame if len(fringe)==1 and len(science_list)>1: fringe = [deepcopy(fringe[0]) for img in science_list] # Convert filenames to AD instances if necessary tmp_list = [] for science in science_list: if type(science) is not AstroData: science = AstroData(science) tmp_list.append(science) science_list = tmp_list fringe_dict = gt.make_dict(key_list=science_list, value_list=fringe) fringe_output = [] else: log.warning("No science frames specified; no scaling will be done") science_list = [] fringe_output = fringe # Loop over each AstroData object in the science list for ad in science_list: # Retrieve the appropriate fringe fringe = fringe_dict[ad] # Check the inputs have matching filters, binning and SCI shapes. try: gt.check_inputs_match(ad1=ad, ad2=fringe) except Errors.ToolboxError: # If not, try to clip the fringe frame to the size of the # science data # For a GMOS example, this allows a full frame fringe to # be used for a CCD2-only science frame. fringe = gt.clip_auxiliary_data( adinput=ad, aux=fringe, aux_type="cal")[0] # Check again, but allow it to fail if they still don't match gt.check_inputs_match(ad1=ad, ad2=fringe) # Check whether statistics should be used stats_scale = rc["stats_scale"] # Calculate the scale value scale = 1.0 if not stats_scale: # Use the exposure times to calculate the scale log.fullinfo("Using exposure times to calculate the scaling"+ " factor") try: scale = ad.exposure_time() / fringe.exposure_time() except: raise Errors.InputError("Could not get exposure times " + "for %s, %s. Try stats_scale=True" % (ad.filename,fringe.filename)) else: # Use statistics to calculate the scaling factor log.fullinfo("Using statistics to calculate the " + "scaling factor") # Deepcopy the input so it can be manipulated without # affecting the original statsad = deepcopy(ad) statsfringe = deepcopy(fringe) # Trim off any overscan region still present statsad,statsfringe = gt.trim_to_data_section([statsad, statsfringe]) # Check the number of science extensions; if more than # one, use CCD2 data only nsciext = statsad.count_exts("SCI") if nsciext>1: # Get the CCD numbers and ordering information # corresponding to each extension log.fullinfo("Trimming data to data section to remove "\ "overscan region") sci_info,frng_info = gt.array_information([statsad, statsfringe]) # Pull out CCD2 data scidata = [] frngdata = [] dqdata = [] for i in range(nsciext): # Get the next extension in physical order sciext = statsad["SCI",sci_info["amps_order"][i]] frngext = statsfringe["SCI",frng_info["amps_order"][i]] # Check to see if it is on CCD2; if so, keep it if sci_info[ "array_number"][("SCI",sciext.extver())]==2: scidata.append(sciext.data) dqext = statsad["DQ",sci_info["amps_order"][i]] maskext = statsad["OBJMASK", sci_info["amps_order"][i]] if dqext is not None and maskext is not None: dqdata.append(dqext.data | maskext.data) elif dqext is not None: dqdata.append(dqext.data) elif maskext is not None: dqdata.append(maskext.data) if frng_info[ "array_number"][("SCI",frngext.extver())]==2: frngdata.append(frngext.data) # Stack data if necessary if len(scidata)>1: scidata = np.hstack(scidata) frngdata = np.hstack(frngdata) else: scidata = scidata[0] frngdata = frngdata[0] if len(dqdata)>0: if len(dqdata)>1: dqdata = np.hstack(dqdata) else: dqdata = dqdata[0] else: dqdata = None else: scidata = statsad["SCI"].data frngdata = statsfringe["SCI"].data dqext = statsad["DQ"] maskext = statsad["OBJMASK"] if dqext is not None and maskext is not None: dqdata = dqext.data | maskext.data elif dqext is not None: dqdata = dqext.data elif maskext is not None: dqdata = maskext.data else: dqdata = None if dqdata is not None: # Replace any DQ-flagged data with the median value smed = np.median(scidata[dqdata==0]) scidata = np.where(dqdata!=0,smed,scidata) # Calculate the maximum and minimum in a box centered on # each data point. The local depth of the fringe is # max - min. The overall fringe strength is the median # of the local fringe depths. # Width of the box is binning and # filter dependent, determined by experimentation # Results don't seem to depend heavily on the box size if ad.filter_name(pretty=True).as_pytype=="i": size = 20 else: size = 40 size /= ad.detector_x_bin().as_pytype() # Use ndimage maximum_filter and minimum_filter to # get the local maxima and minima import scipy.ndimage as ndimage sci_max = ndimage.filters.maximum_filter(scidata,size) sci_min = ndimage.filters.minimum_filter(scidata,size) # Take off 5% of the width as a border xborder = int(0.05 * scidata.shape[1]) yborder = int(0.05 * scidata.shape[0]) if xborder<20: xborder = 20 if yborder<20: yborder = 20 sci_max = sci_max[yborder:-yborder,xborder:-xborder] sci_min = sci_min[yborder:-yborder,xborder:-xborder] # Take the median difference sci_df = np.median(sci_max - sci_min) # Do the same for the fringe frn_max = ndimage.filters.maximum_filter(frngdata,size) frn_min = ndimage.filters.minimum_filter(frngdata,size) frn_max = frn_max[yborder:-yborder,xborder:-xborder] frn_min = frn_min[yborder:-yborder,xborder:-xborder] frn_df = np.median(frn_max - frn_min) # Scale factor # This tends to overestimate the factor, but it is # at least in the right ballpark, unlike the estimation # used in girmfringe (masked_sci.std/fringe.std) scale = sci_df / frn_df log.fullinfo("Scale factor found = "+str(scale)) # Use mult from the arith toolbox to perform the scaling of # the fringe frame scaled_fringe = fringe.mult(scale) # Add the appropriate time stamps to the PHU gt.mark_history(adinput=scaled_fringe, keyword=timestamp_key) # Change the filename scaled_fringe.filename = gt.filename_updater( adinput=ad, suffix=rc["suffix"], strip=True) fringe_output.append(scaled_fringe) # Report the list of output AstroData objects to the reduction context rc.report_output(fringe_output) yield rc