def get_optical_psf(expid, aos=False): # set up objects. make sure I get the right mesh digestor = Digestor() PSF_Evaluator = Moment_Evaluator() mesh_name = "Science-20121120s1-v20i2_All" PSF_Interpolator = Mesh_Interpolator(mesh_name=mesh_name, directory=mesh_directory) # This will be our main wavefront WF = DECAM_Model_Wavefront(PSF_Interpolator=PSF_Interpolator) # load up data expid_path = "{0:08d}/{1:08d}".format(expid - expid % 1000, expid) data_directory = base_directory + expid_path files = sorted(glob(data_directory + "/*{0}".format("_selpsfcat.fits"))) # load up all the data from an exposure. Unfortunately, pandas is stupid and # can't handle the vignet format, so we don't load those up # note that you CAN load them up by passing "do_exclude=True", which then # returns a second variable containing the vignets and aperture fluxes and # errors # data has certain columns removed, needed for processing. # unfortunately I need full_data's vignettes and other info for later steps # TODO optimize this cuz this is clearly wasteful # I'm loading an HDUlist in 2 places, but overhauling the digestor to load it once would be a challenge data = digestor.digest_fits(files[0], do_exclude=False) metaHDUList = [fits.open(files[0])] # list of HDULists #META for file in files[1:]: tmpData = digestor.digest_fits(file, do_exclude=False) data = data.append(tmpData) metaHDUList.append(fits.open(file)) fit_i = jamierod_results.loc[expid] misalignment = { "z04d": fit_i["z04d"], "z04x": fit_i["z04x"], "z04y": fit_i["z04y"], "z05d": fit_i["z05d"], "z05x": fit_i["z05x"], "z05y": fit_i["z05y"], "z06d": fit_i["z06d"], "z06x": fit_i["z06x"], "z06y": fit_i["z06y"], "z07d": fit_i["z07d"], "z07x": fit_i["z07x"], "z07y": fit_i["z07y"], "z08d": fit_i["z08d"], "z08x": fit_i["z08x"], "z08y": fit_i["z08y"], "z09d": fit_i["z09d"], "z09x": fit_i["z09x"], "z09y": fit_i["z09y"], "z10d": fit_i["z10d"], "z10x": fit_i["z10x"], "z10y": fit_i["z10y"], "rzero": fit_i["rzero"], } # print(misalignment['rzero']) # rzero needs to be adjusted to be smaller than the stars! x = 0.3 / 0.14 # 4 misalignment["rzero"] = 1 / (1 / misalignment["rzero"] - x) # print(misalignment['rzero']) # print(.14*x ) data["rzero"] = misalignment["rzero"] optPSFStamps, model = WF.draw_psf(data, misalignment=misalignment) # optPSFStamps is a numpy data cube # full_data is data frame including vignettes of the stars return optPSFStamps, metaHDUList
expid_fails = [] for expid in expids[every * args['expid']: every * (args['expid'] + 1)]: try: print(expid) base_directory = '/nfs/slac/g/ki/ki18/des/cpd/psfex_catalogs/SVA1_FINALCUT/psfcat/' # load up data expid_path = '{0:08d}/{1:08d}'.format(expid - expid % 1000, expid) data_directory = base_directory + expid_path # load up all the data from an exposure. Unfortunately, pandas is stupid and # can't handle the vignet format, so we don't load those up # luckily we don't need to because we already have moments measured fits_files = glob(data_directory + '/*{1}_{0}psfcat.fits'.format(args['type'], TEMP_CHIP_STR)) model, data_stamps = digestor.digest_fits(fits_files[0], do_exclude=True) data_stamps = data_stamps['VIGNET'] sexhdu = fits.open(fits_files[0])[1].data[0][0] data_stamps_raw = np.copy(clean_stamps(data_stamps, sexhdu)) for fits_file in fits_files[1:]: model_i, data_stamps_i = digestor.digest_fits(fits_file, do_exclude=True) model = model.append(model_i, ignore_index=True) data_stamps_i = data_stamps_i['VIGNET'] sexhdu = fits.open(fits_file)[1].data[0][0] data_stamps_raw_i = np.copy(clean_stamps(data_stamps_i, sexhdu)) data_stamps = np.vstack((data_stamps, data_stamps_i)) data_stamps_raw = np.vstack((data_stamps_raw, data_stamps_raw_i)) # try this cleaning instead data_stamps[data_stamps < 0] = 0 data_stamps = data_stamps.astype(np.float64) # evaluate regular data stamps
def stamp_collector(expid, Nmax=0, rzero=None, snr_max=400, snr_min=90): results = np.load(jamierod_dir + '/params/params_{0:08d}.npy'.format(expid)).item() keys = sorted([key for key in results.keys() if ('error' not in key and 'fix' not in key and 'limit' not in key)]) misalignment = {key: results[key] for key in keys} params = {'expid': expid, 'misalignment': misalignment, # jamie's params: 'mesh_directory': '/nfs/slac/g/ki/ki22/roodman/ComboMeshesv20', 'mesh_name': 'Science-20121120s1-v20i2_All', } params_default = {'snr_key': 'SNR_WIN', 'snr_max': snr_max, 'snr_min': snr_min, 'data_name': '_selpsfcat.fits', 'num_bins': 4 } params_default.update(param_default_kils(expid)) params_default.update(params) params = params_default fits_files = glob(params['data_directory'] + '/*{0}'.format(params['data_name'])) dig = Digestor() model, data_stamps = dig.digest_fits(fits_files[0], do_exclude=True) data_stamps = data_stamps['VIGNET'] for fits_file in fits_files[1:]: model_i, data_stamps_i = dig.digest_fits(fits_file, do_exclude=True) model = model.append(model_i, ignore_index=True) data_stamps = np.vstack((data_stamps, data_stamps_i['VIGNET'])) if params['snr_key'] in model.columns: conds = ((model[params['snr_key']] > params['snr_min']) & (model[params['snr_key']] < params['snr_max'])) data_stamps = data_stamps[conds.values] model = model[conds] if Nmax > 0: # cut out Nmax objects rows = np.random.choice(len(model), Nmax, replace=False) model = model.iloc[rows] data_stamps = data_stamps[rows] # set data_stamps to be float64 like the model stamps data_stamps = data_stamps.astype(np.float64) if type(rzero) == type(None): model['rzero'] = results['rzero'] else: model['rzero'] = rzero # get the PSF_Interpolator PSF_Interpolator = Mesh_Interpolator(mesh_name=params['mesh_name'], directory=params['mesh_directory']) WF = DECAM_Model_Wavefront(PSF_Interpolator=PSF_Interpolator, num_bins=params['num_bins'], model=model) def plane(rzero, z04d, z04x, z04y, z05d, z05x, z05y, z06d, z06x, z06y, z07d, z07x, z07y, z08d, z08x, z08y, z09d, z09x, z09y, z10d, z10x, z10y, z11d, z11x, z11y, dz, dx, dy, xt, yt, e0, e1, e2, delta1, delta2, zeta1, zeta2, **kwargs): wf_misalignment = {'z04d': z04d, 'z04x': z04x, 'z04y': z04y, 'z05d': z05d, 'z05x': z05x, 'z05y': z05y, 'z06d': z06d, 'z06x': z06x, 'z06y': z06y, 'z07d': z07d, 'z07x': z07x, 'z07y': z07y, 'z08d': z08d, 'z08x': z08x, 'z08y': z08y, 'z09d': z09d, 'z09x': z09x, 'z09y': z09y, 'z10d': z10d, 'z10x': z10x, 'z10y': z10y, 'z11d': z11d, 'z11x': z11x, 'z11y': z11y, 'dz': dz, 'dx': dx, 'dy': dy, 'xt': xt, 'yt': yt} return wf_misalignment wf_misalignment = plane(**misalignment) # get evaluated PSFs stamps, eval_data = WF.draw_psf(WF.data, force_interpolation=True, misalignment=wf_misalignment) # apply shear to stamps directly! evaluated_psfs = WF.evaluate_psf(stamps, misalignment=wf_misalignment) # make sure evaluated_psfs has the right index evaluated_psfs.index = eval_data.index # combine the results from PSF_Evaluator with your input data combined_df = evaluated_psfs.combine_first(eval_data) # TODO: Deal with constant jitter terms! Currently the results are off and seem to be off related to the overall magnitude... ## # add dc factors ## # combined_df['e0'] += results['e0'] ## # combined_df['e1'] += results['e1'] ## # combined_df['e2'] += results['e2'] ## # combined_df['delta1'] += results['delta1'] ## # combined_df['delta2'] += results['delta2'] ## # combined_df['zeta1'] += results['zeta1'] ## # combined_df['zeta2'] += results['zeta2'] ## e1 = results['e1'] * 5 # empirical correction? ## e2 = results['e2'] * 5 ## Mx = combined_df['Mx'].values ## My = combined_df['My'].values ## esq = e1 * e1 + e2 * e2 ## if esq < 1e-8: ## A = 1 + esq / 8 + e1 * (0.5 + esq * 3 / 16) ## B = 1 + esq / 8 - e1 * (0.5 + esq * 3 / 16) ## C = e2 * (0.5 + esq * 3 / 16) ## else: ## temp = np.sqrt(1 - esq) ## cc = np.sqrt(0.5 * (1 + 1 / temp)) ## temp = cc * (1 - temp) / esq ## C = temp * e2 ## temp *= e1 ## A = cc + temp ## B = cc - temp ## matrix = np.array([[A, C], [C, B]]) ## matrix /= A * B - C * C ## offsetsx = -Mx * (A + C - 1) ## offsetsy = -My * (C + B - 1) ## # apply transformation ## stamps_transformed = [] ## for stamp, offsetx, offsety in zip(stamps, offsetsx, offsetsy): ## stamps_transformed_i = affine_transform(stamp, matrix, (offsetx, offsety)) ## stamps_transformed.append(stamps_transformed_i) ## stamps_transformed = np.array(stamps_transformed) ## evaluated_psfs = WF.evaluate_psf(stamps_transformed, misalignment=wf_misalignment) ## # make sure evaluated_psfs has the right index ## evaluated_psfs.index = eval_data.index ## # combine the results from PSF_Evaluator with your input data ## combined_df_trans = evaluated_psfs.combine_first(eval_data) ## e1 = combined_df['e1'] ## e2 = combined_df['e2'] ## e1_t = combined_df_trans['e1'] ## e2_t = combined_df_trans['e2'] ## print((e1 - e1_t) / (results['e1'])) ## print((e2 - e2_t) / (results['e2'])) return combined_df, stamps, data_stamps
def make_wavefront(expid, output_dir, optpsf=None, atmpsf=None, starminusopt=None, model=None): """ Make a wavefront, useful for diagnostic plots :param expid: the id of the exposure being studied :param output_dir: the directory to store the output and temp files :param optpsf: (Optional) the optical psf in a datacube :param atmpsf: (Optional) the atmopsheric psf in a datacube :param starminusopt: (Optional) the residual when the optpsf is deconvolved :param model: (model) the convolution of optpsf and atmpsf :return: None """ # these give the deconvolved stars # Wish I knew how to loop this if optpsf is None: deconvopt_loc = output_dir + '{0:08d}/{0}_opt.npy'.format(expid) optpsf = np.load(deconvopt_loc) if atmpsf is None: deconvatm_loc = output_dir + '{0:08d}/{0}_atm.npy'.format(expid) atmpsf = np.load(deconvatm_loc) if starminusopt is None: deconvstarsminusopt_loc = output_dir + '{0:08d}/{0}_stars_minus_opt.npy'.format(expid) # set the shape to be right starminusopt = np.load(deconvstarsminusopt_loc)[:, 15:47, 15:47] if model is None: deconvmodel_loc = output_dir + '{0:08d}/{0}_stars.npy'.format(expid) model = np.load(deconvmodel_loc) mesh_directory = '/nfs/slac/g/ki/ki22/roodman/ComboMeshesv20' # directory containing the input data files base_directory = '/nfs/slac/g/ki/ki18/des/cpd/psfex_catalogs/SVA1_FINALCUT/psfcat/' # set up objects. make sure I get the right mesh digestor = Digestor() mesh_name = 'Science-20121120s1-v20i2_All' PSF_Interpolator = Mesh_Interpolator(mesh_name=mesh_name, directory=mesh_directory) # This will be our main wavefront WF = DECAM_Model_Wavefront(PSF_Interpolator=PSF_Interpolator) # load up data expid_path = '/{0:08d}/{1:08d}'.format(expid - expid % 1000, expid) data_directory = base_directory + expid_path files = sorted(glob(data_directory + '/*{0}'.format('_selpsfcat.fits'))) data_df = digestor.digest_fits(files[0], do_exclude=False) # Can't use the new one above, because we're calling on different data. meta_hdulist = [fits.open(files[0])] for file in files[1:]: tmpData = digestor.digest_fits(file, do_exclude=False) data_df = data_df.append(tmpData) meta_hdulist.append(fits.open(file)) hdu_idxs = get_hdu_idxs(meta_hdulist) NObj = hdu_idxs[-1] # make the psfex models for both portions psf_files = sorted(glob(data_directory + '/*{0}'.format('psfcat_validation_subtracted.psf'))) psfexpsf = load_psfex(psf_files, NObj, meta_hdulist) stars = get_vignettes(NObj, meta_hdulist, hdu_idxs) stars_df = evaluate_stamps_and_combine_with_data(WF, stars, data_df) psfexpsf_df = evaluate_stamps_and_combine_with_data(WF, psfexpsf, data_df) atmpsf_df = evaluate_stamps_and_combine_with_data(WF, atmpsf, data_df) optpsf_df = evaluate_stamps_and_combine_with_data(WF, optpsf, data_df) starminusopt_df = evaluate_stamps_and_combine_with_data(WF, starminusopt, data_df) model_df = evaluate_stamps_and_combine_with_data(WF, model, data_df) combinekeys = ['e0', 'e1', 'e2', 'E1norm', 'E2norm', 'delta1', 'delta2', 'zeta1', 'zeta2'] # make a big df with all the above columns combined df = stars_df.copy() names = ['model', 'psfex', 'starminusopt', 'opt', 'atm', 'psfex_flip'] df_list = [model_df, psfexpsf_df, starminusopt_df, optpsf_df, atmpsf_df] # names += ['opt_load'] # df_list += [optpsf_load_df] # names += ['atm_make'] # df_list += [atmpsf_make_df] for key in combinekeys: # add the other medsub if key == 'E1norm': df[key] = df['e1'] / df['e0'] elif key == 'E2norm': df[key] = df['e2'] / df['e0'] df['{0}_medsub'.format(key)] = df[key] - df[key].median() for name, psf in zip(names, df_list): if key == 'E1norm': psf[key] = psf['e1'] / psf['e0'] elif key == 'E2norm': psf[key] = psf['e2'] / psf['e0'] df['{0}_{1}'.format(name, key)] = psf[key] # add medsub df['{0}_{1}_medsub'.format(name, key)] = df['{0}_{1}'.format(name, key)] - df[ '{0}_{1}'.format(name, key)].median() df['{0}_{1}_diff'.format(name, key)] = df['{0}_{1}'.format(name, key)] - df[key] df['{0}_{1}_medsub_diff'.format(name, key)] = df['{0}_{1}_medsub'.format(name, key)]\ - df['{0}_medsub'.format(key)] np.save(output_dir + '{0:08d}/{0}_psfexalone.npy'.format(expid), psfexpsf) np.save(output_dir + '{0:08d}/{0}_data.npy'.format(expid), stars) df.to_hdf(output_dir + '{0:08d}/results.h5'.format(expid), key='table_{0:08d}'.format(expid), mode='a', format='table', append=False)