def test_fake_pedestals(darkcases, rates, pedestals): '''Test ramp-fit step with fake data.''' # open ramp to get data shape and headers m = RampModel(darkcases) tgroup = m.meta.exposure.group_time rates = np.float(rates) pedestals = np.float(pedestals) nrows = int(m.meta.subarray.xsize) ncols = int(m.meta.subarray.ysize) ngroups = int(m.meta.exposure.ngroups) nints = int(m.meta.exposure.nints) # create fake ramps with known slope and pedestal new_data = np.zeros((nints, ngroups, nrows, ncols), dtype=np.float32) for i in np.arange(0, ngroups): for j in np.arange(4, 2044): for k in np.arange(4, 2044): new_data[0, i, j, k] = pedestals + rates * ((i + 1) * tgroup) # save it m.data = new_data m.err = np.zeros((nints, ngroups, nrows, ncols), dtype=np.float32) fake_data_outname = darkcases[:-5] + "_rate" + str( rates) + "_pedestal" + str( pedestals) + "_test_fake_pedestals_uncal.fits" m.save(fake_data_outname, overwrite=True) output, outint = RampFitStep.call( m, output_file=fake_data_outname[:-5] + "_rate.fits", save_opt=True, opt_name=fake_data_outname[:-5] + "_rate_opt.fits") optoutput = fits.open(fake_data_outname[:-5] + "rate_opt.fits") # check pedestal clip = sigma_clip(optoutput['PEDESTAL'].data) clip.data[clip.mask] = np.nan meanped = np.nanmean(clip.data) assert np.allclose(pedestals, meanped, rtol=2, atol=2) == True optoutput.close()
def run(self): #check the proposed output name. If it exists, remove it. if self.outfile == None: dot = self.infile.rfind('.') self.outfile = self.infile[0:dot] + '_refpixg0.fits' if os.path.isfile(self.outfile): os.remove(self.outfile) #read in data ramp = RampModel(self.infile) data = ramp.data #get data shape nint, ngroup, ny, nx = data.shape #make a copy of the 0th read zero_read = copy.deepcopy(ramp.data[:, 0, :, :]) #subtract the zeroth read from all subsequent reads for integration in range(nint): data[integration, :, :, :] -= zero_read[integration, :, :] ramp.data = data #run the SSB pipeline's refpix step ramp = RefPixStep.call(ramp, use_side_ref_pixels=True, odd_even_columns=True, odd_even_rows=False, config_file='refpix.cfg') #now add the original 0th read back in data = ramp.data for integration in range(nint): data[integration, :, :, :] += zero_read[integration, :, :] #dd = data[0,0,:,:] - zero_read[0,:,:] ramp.data = data #save the result ramp.save(self.outfile)
def test_1overf(cases, sigmas, smoothing_lengths, tolerances, side_gains): '''Test amp average and 1/f noise subtraction.''' test_name = 'only_1overf' # pipeline refpix correction results # ---------------- refq = RefPixStep.call(cases, odd_even_columns=False, use_side_ref_pixels=True, side_smoothing_length=smoothing_lengths, side_gain=side_gains, odd_even_rows=False) outname = cases[:-5] + '_refpix_only1overf_pipeline.fits' refq.save(outname) # manual refpix correction results # -------------- # read in input file ramp = RampModel(cases) rampdata = np.copy(ramp.data) pixeldq = ramp.pixeldq goodpix = pixeldq == 0 #extract subarray if necessary xs, xe, ys, ye = get_coords_rampmodel(ramp) # get bounds num_left = np.max([4 - xs, 0]) num_right = np.max([xe - 2044, 0]) num_bottom = np.max([4 - ys, 0]) num_top = np.max([ye - 2044, 0]) # do the manual subtraction refq_manual, table = amp_only_refpix_corr(rampdata, sigmas, num_left, num_right, num_bottom, num_top, goodpix) refq_manual, outtable = include_1overf(refq_manual, sigmas, num_left, num_right, num_bottom, num_top, pixeldq, smoothing_lengths, side_gains) # save table to compare groups and amps save_df_table(outtable, cases[:-5] + '_include_1overf_amponly.dat') # compare manual to pipeline diff = refq_manual - refq.data # save an image of the differences between manual and pipeline subtraction images = RampModel() images.data = diff images.save(cases[:-5] + '_refpix_only1overf_differences.fits', overwrite=True) # check some values print("Group, Diffs: Amp1 through Amp4") for i in np.arange(0, diff.shape[1]): print('') print('Pipeline: ' + str(i) + ',' + str(refq.data[0, i, 12, 12]) + ',' + str(refq.data[0, i, 12, 600]) + ',' + str(refq.data[0, i, 12, 1030]) + ',' + str(refq.data[0, i, 12, 1600])) print('Manual: ' + str(i) + ',' + str(refq_manual[0, i, 12, 12]) + ',' + str(refq_manual[0, i, 12, 600]) + ',' + str(refq_manual[0, i, 12, 1030]) + ',' + str(refq_manual[0, i, 12, 1600])) # save out data outramp = RampModel() outramp.data = refq_manual outramp.save(cases[:-5] + '_refpix_only1overf_manual.fits', overwrite=True) # pytest to make sure pipeline ~= manual if np.allclose(refq_manual, refq.data, rtol=tolerances[1], atol=tolerances[0], equal_nan=True) == False: # if test fails, get image of (manual - pipeline) for each group display_multi_image(diff, np.shape(diff)[1], tolerances[1], test_name) print('') print("Group, Max Difference") for i in np.arange(0, diff.shape[1]): print('Max difference between pipeline and manual: ' + str(i) + ',' + str(np.max(np.absolute(diff[0, i, :, :])))) print('') assert np.allclose(refq_manual, refq.data, rtol=tolerances[1], atol=tolerances[0], equal_nan=True) == True
def run(self): '''main function''' #check for the existance of the output file if self.outfile == None: self.outfile = self.infile[ 0:-5] + '_REGROUP_' + self.readpatt + '_ngroup' + str( self.ngroup) + '.fits' if (os.path.isfile(self.outfile)): # & self.clobber == False): print("WARNING: Proposed output file {} already exists. Removing.". format(self.outfile)) os.remove(self.outfile) #read in the exposure to use. Read in with RampModel exposure = RampModel(self.infile) #assume that the readpattern of the input file is 'RAPID'. If not, throw an error. rp = exposure.meta.exposure.readpatt if rp != 'RAPID': print( 'WARNING! INPUT DATA WERE NOT COLLECTED USING THE RAPID READPATTERN. QUITTING.' ) sys.exit(0) #extract data data = exposure.data err = exposure.err groupdq = exposure.groupdq #sizes integrations = data.shape[0] ingroups = data.shape[1] ydim = data.shape[2] xdim = data.shape[3] #if the number of groups was not requested, use the maximum for the given readpattern if self.ngroup == None: self.ngroup = readpatts[self.readpatt.lower()]['ngroup'] #group the input groups into collections of frames which will be averaged into the output groups #Only group as many input groups as you need to make the requested number of output groups frames_per_group = readpatts[self.readpatt.lower()]['nframe'] frames_to_skip = readpatts[self.readpatt.lower()]['nskip'] total_frames = (frames_per_group * self.ngroup) + (frames_to_skip * (self.ngroup - 1)) total_exposure_time = total_frames * readpatts['rapid']['tgroup'] #if the total number of frames needed to make the requested integration don't exist #throw an error if total_frames > ingroups: print( "WARNING: Requested regrouping requires more groups than are contained in the input file {}. Quitting." .format(self.infile)) sys.exit(0) #starting and ending indexes of the input groups to be averaged to create the new groups groupstart_index = np.arange(0, total_frames, frames_per_group + frames_to_skip) groupend_index = groupstart_index + frames_per_group #prepare for averaging newdata = np.zeros((integrations, self.ngroup, ydim, xdim)) newerrs = np.zeros((integrations, self.ngroup, ydim, xdim)) newgroupdq = np.zeros((integrations, self.ngroup, ydim, xdim)) #average the input data to create the output data for integration in xrange(integrations): newgp = 0 for gs, ge in izip(groupstart_index, groupend_index): #average the data frames print("Averaging groups {} to {}.".format(gs, ge - 1)) newframe = self.avg_frame(data[integration, gs:ge, :, :]) newdata[integration, newgp, :, :] = newframe #reduce the error in the new frames by sqrt(number of frames) for now newerrs[integration, newgp, :, :] = err[integration, gs + frames_per_group / 2, :, :] / np.sqrt(frames_per_group) #just keep the DQ array from the final frame of the group newgroupdq[integration, newgp, :, :] = groupdq[integration, ge, :, :] #increment the counter for the new group number newgp += 1 #place the updated data back into the model instance exposure.data = newdata exposure.err = newerrs exposure.groupdq = newgroupdq #update header exposure.meta.exposure.ngroups = self.ngroup exposure.meta.exposure.nframes = frames_per_group exposure.meta.exposure.groupgap = frames_to_skip exposure.meta.exposure.group_time = readpatts[ self.readpatt.lower()]['tgroup'] exposure.meta.exposure.exptime = total_exposure_time exposure.meta.exposure.readpatt = self.readpatt.upper() #write the regrouped file out to a new file exposure.save(self.outfile)
def export(self, outfile, all_data=False): """ Export the simulated data to a JWST pipeline ingestible FITS file Parameters ---------- outfile: str The path of the output file """ # Make a RampModel data = self.tso mod = RampModel(data=data, groupdq=np.zeros_like(data), pixeldq=np.zeros((self.nrows, self.ncols)), err=np.zeros_like(data)) pix = utils.subarray_specs(self.subarray) # Set meta data values for header keywords mod.meta.telescope = 'JWST' mod.meta.instrument.name = 'NIRISS' mod.meta.instrument.detector = 'NIS' mod.meta.instrument.filter = self.filter mod.meta.instrument.pupil = 'GR700XD' mod.meta.exposure.type = 'NIS_SOSS' mod.meta.exposure.nints = self.nints mod.meta.exposure.ngroups = self.ngrps mod.meta.exposure.nframes = self.nframes mod.meta.exposure.readpatt = 'NISRAPID' mod.meta.exposure.groupgap = 0 mod.meta.exposure.frame_time = self.frame_time mod.meta.exposure.group_time = self.group_time mod.meta.exposure.duration = self.time[-1] - self.time[0] mod.meta.subarray.name = self.subarray mod.meta.subarray.xsize = data.shape[3] mod.meta.subarray.ysize = data.shape[2] mod.meta.subarray.xstart = pix.get('xloc', 1) mod.meta.subarray.ystart = pix.get('yloc', 1) mod.meta.subarray.fastaxis = -2 mod.meta.subarray.slowaxis = -1 mod.meta.observation.date = self.obs_date mod.meta.observation.time = self.obs_time mod.meta.target.ra = self.ra mod.meta.target.dec = self.dec mod.meta.target.source_type = 'POINT' # Save the file mod.save(outfile, overwrite=True) # Save input data with fits.open(outfile) as hdul: # Save input star data hdul.append( fits.ImageHDU(data=np.array([i.value for i in self.star], dtype=np.float64), name='STAR')) hdul['STAR'].header.set('FUNITS', str(self.star[1].unit)) hdul['STAR'].header.set('WUNITS', str(self.star[0].unit)) # Save input planet data if self.planet is not None: hdul.append( fits.ImageHDU(data=np.asarray(self.planet, dtype=np.float64), name='PLANET')) for param, val in self.tmodel.__dict__.items(): if isinstance(val, (float, int, str)): hdul['PLANET'].header.set(param.upper()[:8], val) elif isinstance(val, np.ndarray) and len(val) == 1: hdul['PLANET'].header.set(param.upper(), val[0]) elif isinstance(val, type(None)): hdul['PLANET'].header.set(param.upper(), '') elif param == 'u': for n, v in enumerate(val): hdul['PLANET'].header.set('U{}'.format(n + 1), v) else: print(param, val, type(val)) # Write to file hdul.writeto(outfile, overwrite=True) print('File saved as', outfile)