def write_bad_pixel_mask(): # bad pixel mask is currently based only on flat field, and flat field # can only be constructed from forDK.tar.gz data for CIN, so # for now all extensions will be the same par = common.ci_misc_params() outname = par['static_mask_filename'] outname = os.path.join(os.environ[par['etc_env_var']], outname) assert(not os.path.exists(outname)) mask = bad_pixels_master_flat('CIN') hdus = [] for ci_extnum in range(par['n_cameras']): ci_extname = common.ci_extnum_to_extname(ci_extnum, fz=False) print('Assembling HDU for: ' + ci_extname) if len(hdus) == 0: hdu = fits.PrimaryHDU(mask) else: hdu = fits.ImageHDU(mask) hdu.header = static_mask_header_cards(hdu, ci_extname) hdus.append(hdu) hdul = fits.HDUList(hdus) hdul.writeto(outname)
def write_master_dark(outname=None): par = common.ci_misc_params() if outname is None: outname = os.path.join( '/project/projectdirs/desi/users/ameisner/CI/post_install_calibs/CI_master_dark-20190330.fits' ) assert (not os.path.exists(outname)) ci_extnames = common.valid_image_extname_list() hdus = [] for ci_extname in ci_extnames: print('Working on master dark for: ' + ci_extname) dark_image = master_dark_1camera(ci_extname) dark_image = dark_image.astype('float32') # convert to counts per second !!! dark_image = dark_image / dark_exptime dark_image = dark_image.astype('float32') if len(hdus) == 0: hdu = fits.PrimaryHDU(dark_image) else: hdu = fits.ImageHDU(dark_image) hdu.header = master_dark_header_cards(hdu, ci_extname) hdus.append(hdu) hdul = fits.HDUList(hdus) hdul.writeto(outname)
def estimate_flatfield(setnum): assert ((setnum == 1) or (setnum == 2)) flist = get_flat_frames_names(setnum) par = common.ci_misc_params() countrate_tot = np.zeros( (par['height_pix_native'], par['width_pix_native'])) countrate_wt = 0.0 for f in flist: countrate, countrate_ivar = countrate_1flat(f) countrate_tot += countrate * countrate_ivar countrate_wt += countrate_ivar countrate_est = countrate_tot / countrate_wt # normalize flat field to a median value of 1 countrate_med = np.median(countrate_est) countrate_norm = countrate_est / countrate_med countrate_norm_ivar = countrate_wt * (countrate_med**2) # note that countrate_norm is an image whereas countrate_norm_ivar is # a scalar value return countrate_norm, countrate_norm_ivar
def write_master_bias(outname=None): par = common.ci_misc_params() if outname is None: outname = par['master_bias_filename'] outname = os.path.join(os.environ[par['etc_env_var']], outname) assert (not os.path.exists(outname)) ci_extnames = common.valid_image_extname_list() hdus = [] for ci_extname in ci_extnames: print('Working on master bias for: ' + ci_extname) bias_image = master_bias_1camera(ci_extname) bias_image = bias_image.astype('float32') if len(hdus) == 0: hdu = fits.PrimaryHDU(bias_image) else: hdu = fits.ImageHDU(bias_image) hdu.header = master_bias_header_cards(hdu, ci_extname) hdus.append(hdu) hdul = fits.HDUList(hdus) hdul.writeto(outname)
def write_master_flat(): par = common.ci_misc_params() outname = par['master_flat_filename'] outname = os.path.join(os.environ[par['etc_env_var']], outname) assert (not os.path.exists(outname)) flat, ivar = create_master_flat() flat = flat.astype('float32') ci_extnames = common.valid_image_extname_list() hdus = [] for ci_extnum in range(len(ci_extnames)): ci_extname = common.ci_extnum_to_extname(ci_extnum, fz=False) print('Assembling HDU for: ' + ci_extname) if len(hdus) == 0: hdu = fits.PrimaryHDU(flat) else: hdu = fits.ImageHDU(flat) hdu.header = master_flat_header_cards(hdu, ci_extname, ivar) hdus.append(hdu) hdul = fits.HDUList(hdus) hdul.writeto(outname)
def write_image_level_outputs(exp, outdir, fname_in, gzip=True, cube_index=None): # exp is a CI_exposure object # outdir is the output directory (string) # fname_in is the input filename (string) par = common.ci_misc_params() for flavor in par['reduced_image_flavors']: _gzip = (gzip if (flavor != 'REDUCED') else False) outname = reduced_image_fname(outdir, fname_in, flavor, gzip=_gzip, cube_index=cube_index) hdulist = exp.to_hdulist(flavor=flavor) print('Attempting to write ' + flavor + ' image output to ' + outname) hdulist.writeto(outname) print('Successfully wrote ' + flavor + ' image output to ' + outname)
def segmentation_map(image, extname, get_kernel=False): # in this context image means a 2D numpy array rather than a CI_image # object par = common.ci_misc_params() fwhm_pix = par['nominal_fwhm_asec'] / \ util.nominal_pixel_sidelen_arith(extname) threshold = detect_threshold(image, snr=2.0) sigma = fwhm_pix * gaussian_fwhm_to_sigma kernel = Gaussian2DKernel(sigma, x_size=int(np.round(fwhm_pix)), y_size=int(np.round(fwhm_pix))) kernel.normalize() segm = detect_sources(image, threshold, npixels=5, filter_kernel=kernel) # add my own dilation of segm.array ? # incorporate masking based on master flat/bias in this analysis ? if not get_kernel: return segm else: return segm, kernel
def __init__(self, image_list, dummy_fz_header=None): # images is a dictionary of CI_image objects par = common.ci_misc_params() self.images = dict(zip(common.valid_image_extname_list(), par['n_cameras']*[None])) self.assign_image_list(image_list) self.dummy_fz_header = dummy_fz_header self.pixels_calibrated = None
def create_satmask(im, extname): # im is just a 2D array of pixels, not a CI_image object par = common.ci_misc_params() gain = common.ci_camera_gain(extname) sat_thresh = par['full_well_electrons'] / gain satmask = (im >= sat_thresh) return satmask
def check_image_level_outputs_exist(outdir, fname_in, gzip=True, cube_index=None): par = common.ci_misc_params() for flavor in par['reduced_image_flavors']: _ = reduced_image_fname(outdir, fname_in, flavor, gzip=gzip, cube_index=cube_index)
def load_exposure(fname, verbose=True, realtime=False, cube_index=None): assert (os.path.exists(fname)) print('Attempting to load exposure : ' + fname) par = common.ci_misc_params() if not realtime: hdul = fits.open(fname) else: hdul = realtime_raw_read(fname) dummy_fz_header = None is_image_hdu = np.zeros(len(hdul), dtype=bool) for i, hdu in enumerate(hdul): # real data has another dummy extension added with no EXTNAME keywords = [c[0] for c in hdu.header.cards] if not ('EXTNAME' in keywords): continue if hdu.header['EXTNAME'] not in common.valid_extname_list(): continue if (hdu.header['EXTNAME']).strip() == par['fz_dummy_extname']: dummy_fz_header = hdu.header continue is_image_hdu[i] = True w_im = np.where(is_image_hdu)[0] is_cube = (len(hdul[w_im[0]].data.shape) == 3) assert ((is_cube and (cube_index is None)) == False) assert (((not is_cube) and (cube_index is not None)) == False) try: imlist = [ load_image_from_hdu(hdul[ind], verbose=verbose, cube_index=cube_index) for ind in w_im ] except: print('failed to load exposure at image list creation stage') return None exp = CI_exposure(imlist, dummy_fz_header=dummy_fz_header) print('Successfully loaded exposure : ' + fname) print('Exposure has ' + str(exp.num_images_populated()) + ' image extensions populated') print('Populated image extension names are : ' + str(exp.populated_extnames())) return exp
def read_dark_image(ci_extname): assert (common.is_valid_extname(ci_extname)) par = common.ci_misc_params() dark_fname = os.path.join(os.environ[par['etc_env_var']], \ par['master_dark_filename']) print('Attempting to read master dark : ' + dark_fname + ', extension name : ' + ci_extname) assert (os.path.exists(dark_fname)) dark, hdark = fits.getdata(dark_fname, extname=ci_extname, header=True) dark = remove_overscan(dark) return dark, hdark
def read_bias_image(ci_extname): assert (common.is_valid_extname(ci_extname)) par = common.ci_misc_params() bias_fname = os.path.join(os.environ[par['etc_env_var']], \ par['master_bias_filename']) print('Attempting to read master bias : ' + bias_fname + ', extension name : ' + ci_extname) assert (os.path.exists(bias_fname)) bias = fits.getdata(bias_fname, extname=ci_extname) bias = remove_overscan(bias) return bias
def read_static_mask_image(ci_extname): assert (common.is_valid_extname(ci_extname)) par = common.ci_misc_params() mask_fname = os.path.join(os.environ[par['etc_env_var']], \ par['static_mask_filename']) print('Attempting to read static bad pixel mask : ' + mask_fname + ', extension name : ' + ci_extname) assert (os.path.exists(mask_fname)) mask = fits.getdata(mask_fname, extname=ci_extname) mask = remove_overscan(mask) return mask
def gaia_chunknames(ipix, ps1=False): # could add checks to make sure that all ipix values are # sane HEALPix pixel indices # RIGHT NOW THIS ASSUMES IPIX IS AN ARRAY !! # should eventually make this also work for scalar ipix par = common.ci_misc_params() env_var = par['ps1_env_var'] if ps1 else par['gaia_env_var'] gaia_dir = os.environ[env_var] flist = [ os.path.join(gaia_dir, 'chunk-' + str(i).zfill(5) + '.fits') for i in ipix ] return flist
def nominal_tan_wcs(telra, teldec, extname): # Create a new WCS object. The number of axes must be set # from the start par = common.ci_misc_params() fname = os.path.join(os.environ[par['etc_env_var']], par['headers_dummy_filename']) h = fits.getheader(fname, extname=extname) h['CRVAL1'] = telra h['CRVAL2'] = teldec w = wcs.WCS(h) return w
def read_flat_image(ci_extname): # at some point should add option to return master flat's # inverse variance as well assert (common.is_valid_extname(ci_extname)) par = common.ci_misc_params() flat_fname = os.path.join(os.environ[par['etc_env_var']], \ par['master_flat_filename']) print('Attempting to read master flat : ' + flat_fname + ', extension name : ' + ci_extname) assert (os.path.exists(flat_fname)) flat = fits.getdata(flat_fname, extname=ci_extname) flat = remove_overscan(flat) return flat
def adu_to_surface_brightness(sky_adu_1pixel, acttime, extname): """ convert from ADU (per pixel) to mag per square asec (AB) note that this is meant to be applied to an average sky value across an entire CI camera; this function does not take into account platescale variations within a camera """ if (sky_adu_1pixel <= 0) or (acttime <= 0): return np.nan par = common.ci_misc_params() pixel_area_sq_asec = util.nominal_pixel_area_sq_asec(extname) sky_adu_per_sq_asec = sky_adu_1pixel / pixel_area_sq_asec sky_adu_per_sec_sq_asec = sky_adu_per_sq_asec / acttime sky_e_per_sec_sq_asec = sky_adu_per_sec_sq_asec * common.ci_camera_gain( extname) return (par['nominal_zeropoint'] - 2.5 * np.log10(sky_e_per_sec_sq_asec))
def _proc(fname_in, outdir=None, careful_sky=False, no_cataloging=False, no_gaia_xmatch=False, no_ps1_xmatch=False, cube_index=None, skip_image_outputs=False, realtime=False): print('Starting GFA reduction pipeline at: ' + str(datetime.utcnow()) + ' UTC') t0 = time.time() try: print('Running on host: ' + str(os.environ.get('HOSTNAME'))) except: print('Could not retrieve hostname!') write_outputs = (outdir is not None) assert (os.path.exists(fname_in)) gitrev = io.retrieve_git_rev() if write_outputs: if not os.path.exists(outdir): os.mkdir(outdir) # fail if ANY of expected outputs already exist io.check_image_level_outputs_exist(outdir, fname_in, gzip=True, cube_index=cube_index) exp = io.load_exposure(fname_in, cube_index=cube_index, realtime=realtime) # check for simulated data if util.has_wrong_dimensions(exp): # point is to not crash, for sake of real time reductions print('EXITING: exposure may be a simulation?!') return print('Attempting to compute basic statistics of raw pixel data') imstats = io.gather_pixel_stats(exp) # create data quality bitmasks exp.create_all_bitmasks() # go from "raw" images to "reduced" images exp.calibrate_pixels() # calculate sky brightness in mag per sq asec exp.estimate_all_sky_mags(careful_sky=careful_sky) exp.estimate_all_sky_sigmas(careful_sky=careful_sky) par = common.ci_misc_params() if not no_cataloging: catalogs = exp.all_source_catalogs() for extname, cat in catalogs.items(): if cat is not None: util.create_det_ids(cat, extname, fname_in, cube_index=cube_index) # reformat the output catalogs into a single merged astropy Table catalog = io.combine_per_camera_catalogs(catalogs) # run astrometric recalibration print('Attempting astrometric recalibration relative to Gaia DR2') astr = wcs.recalib_astrom(catalog, fname_in) exp.update_wcs(astr) exp.recompute_catalog_radec(catalog) if (not no_ps1_xmatch) and (par['ps1_env_var'] in os.environ): # probably should look into dec < -30 handling more at some point print('Attempting to perform PS1 cross-matching...') io.write_ps1_matches(catalog, outdir, fname_in, cube_index=cube_index) if (not no_gaia_xmatch) and (par['gaia_env_var'] in os.environ): print('Attempting to identify Gaia cross-matches') catalog = io.append_gaia_crossmatches(catalog) # try to write image-level outputs if outdir is specified if write_outputs: if not skip_image_outputs: print('Attempting to write image-level outputs to directory : ' + outdir) # could add command line arg for turning off gzip compression io.write_image_level_outputs(exp, outdir, fname_in, gzip=True, cube_index=cube_index) if not no_cataloging: io.write_exposure_source_catalog(catalog, outdir, fname_in, cube_index=cube_index) # make this work correctly in the case that --no_cataloging is set io.write_ccds_table(imstats, catalog, exp, outdir, fname_in, cube_index=cube_index) print('Successfully finished reducing ' + fname_in) dt = time.time() - t0 print('GFA reduction pipeline took ' + '{:.2f}'.format(dt) + ' seconds') print('GFA reduction pipeline completed at: ' + str(datetime.utcnow()) + ' UTC')
def kentools_center(catalog, skyra, skydec, extname='GUIDE0', arcmin_max=3.5, gaia=None): # cat needs to have fields xcentroid and ycentroid # skyra, skydec are the initial guesses of the # actual center of the field of view; these need to be within # arcmin_max of the true FOV center for this routine to succeed #assert(os.path.exists(fname_cat)) # output needs to include, at a minimum: # xshift_best # yshift_best # contrast # extname # probably also want # expid retrieved from the catalog table #fname_reduced = fname_cat.replace('_catalog', '_reduced') #assert(os.path.exists(fname_reduced)) #h = fits.getheader(fname_reduced, extname=extname) #expid = h['EXPID'] #cat = fits.getdata(fname_cat) cat = copy.deepcopy(catalog) # should just entirely rename racen, deccen to skyr, skydec ... racen = skyra deccen = skydec # this apparently happened for the CI in some cases... if isinstance(racen, str) or isinstance(deccen, str): return None if gaia is None: gaia = gaia_cat_for_exp(racen, deccen) g = SkyCoord(gaia['ra'] * u.deg, gaia['dec'] * u.deg) c = SkyCoord(racen * u.deg, deccen * u.deg) dangle = g.separation(c) gaia = gaia[dangle.degree < 2] g = SkyCoord(gaia['ra'] * u.deg, gaia['dec'] * u.deg) cat = cat[cat['camera'] == extname] assert (len(cat) > 0) n_desi_max = 150 if len(cat) > n_desi_max: cat = downselected_star_sample(cat, n_desi_max) par = common.ci_misc_params() fname_wcs_templates = os.environ['GFA_REDUCE_ETC'] + '/' + par[ 'headers_dummy_filename'] assert (os.path.exists(fname_wcs_templates)) h = fits.getheader(fname_wcs_templates, extname=extname) assert (h['EXTNAME'] == extname) astrom = wcs.WCS(h) astrom.wcs.crval = [racen, deccen] x_gaia_guess, y_gaia_guess = astrom.wcs_world2pix(gaia['ra'], gaia['dec'], 0) dx_all = np.array([], dtype='float64') dy_all = np.array([], dtype='float64') cat = cat[np.argsort(cat['dec'])] # not really necessary for i in range(len(cat)): print(i + 1, ' of ', len(cat)) c = SkyCoord(cat[i]['ra'] * u.deg, cat[i]['dec'] * u.deg) dangle = c.separation(g) w = (np.where(dangle.arcminute < arcmin_max))[0] print(len(w), len(g), len(gaia)) if len(w) == 0: continue dx = cat[i]['xcentroid'] - x_gaia_guess[w] dy = cat[i]['ycentroid'] - y_gaia_guess[w] dx_all = np.concatenate((dx_all, dx)) dy_all = np.concatenate((dy_all, dy)) assert (len(dx_all) == len(dy_all)) #print(np.min(dy_all), np.max(dy_all)) axlim = max(np.round(arcmin_max * 60.0 / 0.214), 1000.0) #print(axlim) dx = 1.0 dy = 1.0 nx = ny = 2 * axlim counts, x_edges_left, y_edges_left = amm_2dhist(-1.0 * axlim, -1.0 * axlim, nx, ny, dx, dy, dx_all, dy_all) counts = counts.astype(float) # 7.5 value is tailored to the CI -- revisit for GFAs !! fwhm_pix = 4.7 sigma_pix = fwhm_pix / (2 * np.sqrt(2 * np.log(2))) smth = gaussian_filter(counts, sigma_pix, mode='constant') counts_shape = counts.shape sidelen = counts_shape[0] #plt.imshow(smth, cmap='gray_r') #plt.show() indmax = np.argmax(smth) indx = (indmax // sidelen).astype(int) indy = (indmax % sidelen).astype(int) #print(indmax, indx, indy, counts_shape, x_edges_left[indx], # y_edges_left[indy]) #print(x_edges_left[indx], y_edges_left[indy], ' ~~~~~~~~~~~~~~~~') xshift_best = x_edges_left[indx] + 0.5 * dx yshift_best = y_edges_left[indy] + 0.5 * dy # assumes dx = dy = 1 !! ycen_grid, xcen_grid = np.meshgrid(x_edges_left[0:(len(x_edges_left) - 1)], y_edges_left[0:(len(y_edges_left) - 1)]) d = np.sqrt( np.power(xcen_grid - xshift_best, 2) + np.power(ycen_grid - yshift_best, 2)) #fitsio.write('/global/cscratch1/sd/ameisner/smth.fits', smth) #fitsio.write('/global/cscratch1/sd/ameisner/d.fits', d) r_max = 4 * 4.7 # roughly 4 asec radius for the CI sind = np.argsort(np.ravel(smth)) val_90 = np.ravel(smth)[sind[int(np.round(0.925 * len(sind)))]] high = [(d < r_max) & ((smth == np.max(smth)) | (smth > val_90))] assert (np.sum(high) > 0) print(xshift_best, yshift_best) wt = np.sum(high * smth) xshift_best = np.sum(high * xcen_grid * smth) / wt yshift_best = np.sum(high * ycen_grid * smth) / wt print(xshift_best, yshift_best) contrast = center_contrast(smth) #print(xcen_grid.shape) #print(counts.shape, smth.shape) result = { 'xshift_best': xshift_best, 'yshift_best': yshift_best, 'contrast': contrast, 'extname': extname, 'astr_guess': astrom } return result
def do_aper_phot(data, catalog, extname, ivar_adu): # catalog should be the catalog with refined centroids # for **one CI camera** print('Attempting to do aperture photometry') positions = list(zip(catalog['xcentroid'], catalog['ycentroid'])) radii = aper_rad_pix(extname) apertures = [CircularAperture(positions, r=r) for r in radii] annulus_apertures = CircularAnnulus(positions, r_in=60.0, r_out=65.0) annulus_masks = annulus_apertures.to_mask(method='center') par = common.ci_misc_params() b_over_a = (1.0 if (extname == 'CIC') else par['nominal_mer_cd'] / par['nominal_sag_cd']) # the long axis of elliptical aperture (in terms of pixels) needs to # be in the CI pixel Y direction apertures_ell = [ EllipticalAperture(positions, a, a * b_over_a, theta=np.pi / 2) for a in radii ] # 107 um fiber diam, 9 um on a side for a pixel # fiber diam from Table 4.1 of https://arxiv.org/abs/1611.00037 rad_fiber_pix_sag = (107.0 / 9.0) / 2.0 deg_to_normal = 5.43 # [desi-commiss 522] if extname != 'CIC': rad_fiber_pix_mer = rad_fiber_pix_sag * np.sin(deg_to_normal / (180.0 / np.pi)) else: rad_fiber_pix_mer = rad_fiber_pix_sag aper_fib = EllipticalAperture(positions, rad_fiber_pix_sag, rad_fiber_pix_mer, theta=np.pi / 2) bkg_median = [] for mask in annulus_masks: annulus_data = mask.multiply(data) annulus_data_1d = annulus_data[mask.data > 0] # this sigma_clipped_stats call is actually the slow part !! _, median_sigclip, std_bg = sigma_clipped_stats(annulus_data_1d) bkg_median.append(median_sigclip) bkg_median = np.array(bkg_median) phot = aperture_photometry(data, apertures, error=aper_phot_unc_map(ivar_adu)) for i, aperture in enumerate(apertures): aper_bkg_tot = bkg_median * _get_area_from_ap(aperture) catalog['aper_sum_bkgsub_' + str(i)] = phot['aperture_sum_' + str(i)] - aper_bkg_tot catalog['aper_bkg_' + str(i)] = aper_bkg_tot catalog['aperture_sum_err_' + str(i)] = phot['aperture_sum_err_' + str(i)] ### del phot phot = aperture_photometry(data, apertures_ell, error=aper_phot_unc_map(ivar_adu)) for i, aperture in enumerate(apertures_ell): aper_bkg_tot = bkg_median * _get_area_from_ap(aperture) catalog['aper_ell_sum_bkgsub_' + str(i)] = phot['aperture_sum_' + str(i)] - aper_bkg_tot catalog['aper_ell_bkg_' + str(i)] = aper_bkg_tot catalog['aperture_ell_sum_err_' + str(i)] = phot['aperture_sum_err_' + str(i)] ### ### del phot phot = aperture_photometry(data, aper_fib, error=aper_phot_unc_map(ivar_adu)) aper_bkg_tot = bkg_median * _get_area_from_ap(aper_fib) catalog['aper_sum_bkgsub_fib'] = phot['aperture_sum'] - aper_bkg_tot catalog['aper_bkg_fib'] = aper_bkg_tot catalog['aperture_sum_err_fib'] = phot['aperture_sum_err'] #### # is .area() result a vector or scalar ?? catalog['sky_annulus_area_pix'] = _get_area_from_ap(annulus_apertures) catalog['sky_annulus_median'] = bkg_median