def write_bad_pixel_mask():
# bad pixel mask is currently based only on flat field, and flat field
# can only be constructed from forDK.tar.gz data for CIN, so
# for now all extensions will be the same
par = common.ci_misc_params()
outname = par['static_mask_filename']
outname = os.path.join(os.environ[par['etc_env_var']], outname)
assert(not os.path.exists(outname))
mask = bad_pixels_master_flat('CIN')
hdus = []
for ci_extnum in range(par['n_cameras']):
ci_extname = common.ci_extnum_to_extname(ci_extnum, fz=False)
print('Assembling HDU for: ' + ci_extname)
if len(hdus) == 0:
hdu = fits.PrimaryHDU(mask)
else:
hdu = fits.ImageHDU(mask)
hdu.header = static_mask_header_cards(hdu, ci_extname)
hdus.append(hdu)
hdul = fits.HDUList(hdus)
hdul.writeto(outname)
def write_master_dark(outname=None):
par = common.ci_misc_params()
if outname is None:
outname = os.path.join(
'/project/projectdirs/desi/users/ameisner/CI/post_install_calibs/CI_master_dark-20190330.fits'
)
assert (not os.path.exists(outname))
ci_extnames = common.valid_image_extname_list()
hdus = []
for ci_extname in ci_extnames:
print('Working on master dark for: ' + ci_extname)
dark_image = master_dark_1camera(ci_extname)
dark_image = dark_image.astype('float32')
# convert to counts per second !!!
dark_image = dark_image / dark_exptime
dark_image = dark_image.astype('float32')
if len(hdus) == 0:
hdu = fits.PrimaryHDU(dark_image)
else:
hdu = fits.ImageHDU(dark_image)
hdu.header = master_dark_header_cards(hdu, ci_extname)
hdus.append(hdu)
hdul = fits.HDUList(hdus)
hdul.writeto(outname)
def estimate_flatfield(setnum):
assert ((setnum == 1) or (setnum == 2))
flist = get_flat_frames_names(setnum)
par = common.ci_misc_params()
countrate_tot = np.zeros(
(par['height_pix_native'], par['width_pix_native']))
countrate_wt = 0.0
for f in flist:
countrate, countrate_ivar = countrate_1flat(f)
countrate_tot += countrate * countrate_ivar
countrate_wt += countrate_ivar
countrate_est = countrate_tot / countrate_wt
# normalize flat field to a median value of 1
countrate_med = np.median(countrate_est)
countrate_norm = countrate_est / countrate_med
countrate_norm_ivar = countrate_wt * (countrate_med**2)
# note that countrate_norm is an image whereas countrate_norm_ivar is
# a scalar value
return countrate_norm, countrate_norm_ivar
def write_master_bias(outname=None):
par = common.ci_misc_params()
if outname is None:
outname = par['master_bias_filename']
outname = os.path.join(os.environ[par['etc_env_var']], outname)
assert (not os.path.exists(outname))
ci_extnames = common.valid_image_extname_list()
hdus = []
for ci_extname in ci_extnames:
print('Working on master bias for: ' + ci_extname)
bias_image = master_bias_1camera(ci_extname)
bias_image = bias_image.astype('float32')
if len(hdus) == 0:
hdu = fits.PrimaryHDU(bias_image)
else:
hdu = fits.ImageHDU(bias_image)
hdu.header = master_bias_header_cards(hdu, ci_extname)
hdus.append(hdu)
hdul = fits.HDUList(hdus)
hdul.writeto(outname)
def write_master_flat():
par = common.ci_misc_params()
outname = par['master_flat_filename']
outname = os.path.join(os.environ[par['etc_env_var']], outname)
assert (not os.path.exists(outname))
flat, ivar = create_master_flat()
flat = flat.astype('float32')
ci_extnames = common.valid_image_extname_list()
hdus = []
for ci_extnum in range(len(ci_extnames)):
ci_extname = common.ci_extnum_to_extname(ci_extnum, fz=False)
print('Assembling HDU for: ' + ci_extname)
if len(hdus) == 0:
hdu = fits.PrimaryHDU(flat)
else:
hdu = fits.ImageHDU(flat)
hdu.header = master_flat_header_cards(hdu, ci_extname, ivar)
hdus.append(hdu)
hdul = fits.HDUList(hdus)
hdul.writeto(outname)
def write_image_level_outputs(exp,
outdir,
fname_in,
gzip=True,
cube_index=None):
# exp is a CI_exposure object
# outdir is the output directory (string)
# fname_in is the input filename (string)
par = common.ci_misc_params()
for flavor in par['reduced_image_flavors']:
_gzip = (gzip if (flavor != 'REDUCED') else False)
outname = reduced_image_fname(outdir,
fname_in,
flavor,
gzip=_gzip,
cube_index=cube_index)
hdulist = exp.to_hdulist(flavor=flavor)
print('Attempting to write ' + flavor + ' image output to ' + outname)
hdulist.writeto(outname)
print('Successfully wrote ' + flavor + ' image output to ' + outname)
def segmentation_map(image, extname, get_kernel=False):
# in this context image means a 2D numpy array rather than a CI_image
# object
par = common.ci_misc_params()
fwhm_pix = par['nominal_fwhm_asec'] / \
util.nominal_pixel_sidelen_arith(extname)
threshold = detect_threshold(image, snr=2.0)
sigma = fwhm_pix * gaussian_fwhm_to_sigma
kernel = Gaussian2DKernel(sigma,
x_size=int(np.round(fwhm_pix)),
y_size=int(np.round(fwhm_pix)))
kernel.normalize()
segm = detect_sources(image, threshold, npixels=5, filter_kernel=kernel)
# add my own dilation of segm.array ?
# incorporate masking based on master flat/bias in this analysis ?
if not get_kernel:
return segm
else:
return segm, kernel
def __init__(self, image_list, dummy_fz_header=None):
# images is a dictionary of CI_image objects
par = common.ci_misc_params()
self.images = dict(zip(common.valid_image_extname_list(),
par['n_cameras']*[None]))
self.assign_image_list(image_list)
self.dummy_fz_header = dummy_fz_header
self.pixels_calibrated = None
def create_satmask(im, extname):
# im is just a 2D array of pixels, not a CI_image object
par = common.ci_misc_params()
gain = common.ci_camera_gain(extname)
sat_thresh = par['full_well_electrons'] / gain
satmask = (im >= sat_thresh)
return satmask
def check_image_level_outputs_exist(outdir,
fname_in,
gzip=True,
cube_index=None):
par = common.ci_misc_params()
for flavor in par['reduced_image_flavors']:
_ = reduced_image_fname(outdir,
fname_in,
flavor,
gzip=gzip,
cube_index=cube_index)
def load_exposure(fname, verbose=True, realtime=False, cube_index=None):
assert (os.path.exists(fname))
print('Attempting to load exposure : ' + fname)
par = common.ci_misc_params()
if not realtime:
hdul = fits.open(fname)
else:
hdul = realtime_raw_read(fname)
dummy_fz_header = None
is_image_hdu = np.zeros(len(hdul), dtype=bool)
for i, hdu in enumerate(hdul):
# real data has another dummy extension added with no EXTNAME
keywords = [c[0] for c in hdu.header.cards]
if not ('EXTNAME' in keywords):
continue
if hdu.header['EXTNAME'] not in common.valid_extname_list():
continue
if (hdu.header['EXTNAME']).strip() == par['fz_dummy_extname']:
dummy_fz_header = hdu.header
continue
is_image_hdu[i] = True
w_im = np.where(is_image_hdu)[0]
is_cube = (len(hdul[w_im[0]].data.shape) == 3)
assert ((is_cube and (cube_index is None)) == False)
assert (((not is_cube) and (cube_index is not None)) == False)
try:
imlist = [
load_image_from_hdu(hdul[ind],
verbose=verbose,
cube_index=cube_index) for ind in w_im
]
except:
print('failed to load exposure at image list creation stage')
return None
exp = CI_exposure(imlist, dummy_fz_header=dummy_fz_header)
print('Successfully loaded exposure : ' + fname)
print('Exposure has ' + str(exp.num_images_populated()) +
' image extensions populated')
print('Populated image extension names are : ' +
str(exp.populated_extnames()))
return exp
def read_dark_image(ci_extname):
assert (common.is_valid_extname(ci_extname))
par = common.ci_misc_params()
dark_fname = os.path.join(os.environ[par['etc_env_var']], \
par['master_dark_filename'])
print('Attempting to read master dark : ' + dark_fname +
', extension name : ' + ci_extname)
assert (os.path.exists(dark_fname))
dark, hdark = fits.getdata(dark_fname, extname=ci_extname, header=True)
dark = remove_overscan(dark)
return dark, hdark
def read_bias_image(ci_extname):
assert (common.is_valid_extname(ci_extname))
par = common.ci_misc_params()
bias_fname = os.path.join(os.environ[par['etc_env_var']], \
par['master_bias_filename'])
print('Attempting to read master bias : ' + bias_fname +
', extension name : ' + ci_extname)
assert (os.path.exists(bias_fname))
bias = fits.getdata(bias_fname, extname=ci_extname)
bias = remove_overscan(bias)
return bias
def read_static_mask_image(ci_extname):
assert (common.is_valid_extname(ci_extname))
par = common.ci_misc_params()
mask_fname = os.path.join(os.environ[par['etc_env_var']], \
par['static_mask_filename'])
print('Attempting to read static bad pixel mask : ' + mask_fname +
', extension name : ' + ci_extname)
assert (os.path.exists(mask_fname))
mask = fits.getdata(mask_fname, extname=ci_extname)
mask = remove_overscan(mask)
return mask
def gaia_chunknames(ipix, ps1=False):
# could add checks to make sure that all ipix values are
# sane HEALPix pixel indices
# RIGHT NOW THIS ASSUMES IPIX IS AN ARRAY !!
# should eventually make this also work for scalar ipix
par = common.ci_misc_params()
env_var = par['ps1_env_var'] if ps1 else par['gaia_env_var']
gaia_dir = os.environ[env_var]
flist = [
os.path.join(gaia_dir, 'chunk-' + str(i).zfill(5) + '.fits')
for i in ipix
]
return flist
def nominal_tan_wcs(telra, teldec, extname):
# Create a new WCS object. The number of axes must be set
# from the start
par = common.ci_misc_params()
fname = os.path.join(os.environ[par['etc_env_var']],
par['headers_dummy_filename'])
h = fits.getheader(fname, extname=extname)
h['CRVAL1'] = telra
h['CRVAL2'] = teldec
w = wcs.WCS(h)
return w
def read_flat_image(ci_extname):
# at some point should add option to return master flat's
# inverse variance as well
assert (common.is_valid_extname(ci_extname))
par = common.ci_misc_params()
flat_fname = os.path.join(os.environ[par['etc_env_var']], \
par['master_flat_filename'])
print('Attempting to read master flat : ' + flat_fname +
', extension name : ' + ci_extname)
assert (os.path.exists(flat_fname))
flat = fits.getdata(flat_fname, extname=ci_extname)
flat = remove_overscan(flat)
return flat
def adu_to_surface_brightness(sky_adu_1pixel, acttime, extname):
"""
convert from ADU (per pixel) to mag per square asec (AB)
note that this is meant to be applied to an average sky value across
an entire CI camera; this function does not take into account
platescale variations within a camera
"""
if (sky_adu_1pixel <= 0) or (acttime <= 0):
return np.nan
par = common.ci_misc_params()
pixel_area_sq_asec = util.nominal_pixel_area_sq_asec(extname)
sky_adu_per_sq_asec = sky_adu_1pixel / pixel_area_sq_asec
sky_adu_per_sec_sq_asec = sky_adu_per_sq_asec / acttime
sky_e_per_sec_sq_asec = sky_adu_per_sec_sq_asec * common.ci_camera_gain(
extname)
return (par['nominal_zeropoint'] - 2.5 * np.log10(sky_e_per_sec_sq_asec))
def _proc(fname_in,
outdir=None,
careful_sky=False,
no_cataloging=False,
no_gaia_xmatch=False,
no_ps1_xmatch=False,
cube_index=None,
skip_image_outputs=False,
realtime=False):
print('Starting GFA reduction pipeline at: ' + str(datetime.utcnow()) +
' UTC')
t0 = time.time()
try:
print('Running on host: ' + str(os.environ.get('HOSTNAME')))
except:
print('Could not retrieve hostname!')
write_outputs = (outdir is not None)
assert (os.path.exists(fname_in))
gitrev = io.retrieve_git_rev()
if write_outputs:
if not os.path.exists(outdir):
os.mkdir(outdir)
# fail if ANY of expected outputs already exist
io.check_image_level_outputs_exist(outdir,
fname_in,
gzip=True,
cube_index=cube_index)
exp = io.load_exposure(fname_in, cube_index=cube_index, realtime=realtime)
# check for simulated data
if util.has_wrong_dimensions(exp):
# point is to not crash, for sake of real time reductions
print('EXITING: exposure may be a simulation?!')
return
print('Attempting to compute basic statistics of raw pixel data')
imstats = io.gather_pixel_stats(exp)
# create data quality bitmasks
exp.create_all_bitmasks()
# go from "raw" images to "reduced" images
exp.calibrate_pixels()
# calculate sky brightness in mag per sq asec
exp.estimate_all_sky_mags(careful_sky=careful_sky)
exp.estimate_all_sky_sigmas(careful_sky=careful_sky)
par = common.ci_misc_params()
if not no_cataloging:
catalogs = exp.all_source_catalogs()
for extname, cat in catalogs.items():
if cat is not None:
util.create_det_ids(cat,
extname,
fname_in,
cube_index=cube_index)
# reformat the output catalogs into a single merged astropy Table
catalog = io.combine_per_camera_catalogs(catalogs)
# run astrometric recalibration
print('Attempting astrometric recalibration relative to Gaia DR2')
astr = wcs.recalib_astrom(catalog, fname_in)
exp.update_wcs(astr)
exp.recompute_catalog_radec(catalog)
if (not no_ps1_xmatch) and (par['ps1_env_var'] in os.environ):
# probably should look into dec < -30 handling more at some point
print('Attempting to perform PS1 cross-matching...')
io.write_ps1_matches(catalog,
outdir,
fname_in,
cube_index=cube_index)
if (not no_gaia_xmatch) and (par['gaia_env_var'] in os.environ):
print('Attempting to identify Gaia cross-matches')
catalog = io.append_gaia_crossmatches(catalog)
# try to write image-level outputs if outdir is specified
if write_outputs:
if not skip_image_outputs:
print('Attempting to write image-level outputs to directory : ' +
outdir)
# could add command line arg for turning off gzip compression
io.write_image_level_outputs(exp,
outdir,
fname_in,
gzip=True,
cube_index=cube_index)
if not no_cataloging:
io.write_exposure_source_catalog(catalog,
outdir,
fname_in,
cube_index=cube_index)
# make this work correctly in the case that --no_cataloging is set
io.write_ccds_table(imstats,
catalog,
exp,
outdir,
fname_in,
cube_index=cube_index)
print('Successfully finished reducing ' + fname_in)
dt = time.time() - t0
print('GFA reduction pipeline took ' + '{:.2f}'.format(dt) + ' seconds')
print('GFA reduction pipeline completed at: ' + str(datetime.utcnow()) +
' UTC')
def kentools_center(catalog,
skyra,
skydec,
extname='GUIDE0',
arcmin_max=3.5,
gaia=None):
# cat needs to have fields xcentroid and ycentroid
# skyra, skydec are the initial guesses of the
# actual center of the field of view; these need to be within
# arcmin_max of the true FOV center for this routine to succeed
#assert(os.path.exists(fname_cat))
# output needs to include, at a minimum:
# xshift_best
# yshift_best
# contrast
# extname
# probably also want
# expid retrieved from the catalog table
#fname_reduced = fname_cat.replace('_catalog', '_reduced')
#assert(os.path.exists(fname_reduced))
#h = fits.getheader(fname_reduced, extname=extname)
#expid = h['EXPID']
#cat = fits.getdata(fname_cat)
cat = copy.deepcopy(catalog)
# should just entirely rename racen, deccen to skyr, skydec ...
racen = skyra
deccen = skydec
# this apparently happened for the CI in some cases...
if isinstance(racen, str) or isinstance(deccen, str):
return None
if gaia is None:
gaia = gaia_cat_for_exp(racen, deccen)
g = SkyCoord(gaia['ra'] * u.deg, gaia['dec'] * u.deg)
c = SkyCoord(racen * u.deg, deccen * u.deg)
dangle = g.separation(c)
gaia = gaia[dangle.degree < 2]
g = SkyCoord(gaia['ra'] * u.deg, gaia['dec'] * u.deg)
cat = cat[cat['camera'] == extname]
assert (len(cat) > 0)
n_desi_max = 150
if len(cat) > n_desi_max:
cat = downselected_star_sample(cat, n_desi_max)
par = common.ci_misc_params()
fname_wcs_templates = os.environ['GFA_REDUCE_ETC'] + '/' + par[
'headers_dummy_filename']
assert (os.path.exists(fname_wcs_templates))
h = fits.getheader(fname_wcs_templates, extname=extname)
assert (h['EXTNAME'] == extname)
astrom = wcs.WCS(h)
astrom.wcs.crval = [racen, deccen]
x_gaia_guess, y_gaia_guess = astrom.wcs_world2pix(gaia['ra'], gaia['dec'],
0)
dx_all = np.array([], dtype='float64')
dy_all = np.array([], dtype='float64')
cat = cat[np.argsort(cat['dec'])] # not really necessary
for i in range(len(cat)):
print(i + 1, ' of ', len(cat))
c = SkyCoord(cat[i]['ra'] * u.deg, cat[i]['dec'] * u.deg)
dangle = c.separation(g)
w = (np.where(dangle.arcminute < arcmin_max))[0]
print(len(w), len(g), len(gaia))
if len(w) == 0:
continue
dx = cat[i]['xcentroid'] - x_gaia_guess[w]
dy = cat[i]['ycentroid'] - y_gaia_guess[w]
dx_all = np.concatenate((dx_all, dx))
dy_all = np.concatenate((dy_all, dy))
assert (len(dx_all) == len(dy_all))
#print(np.min(dy_all), np.max(dy_all))
axlim = max(np.round(arcmin_max * 60.0 / 0.214), 1000.0)
#print(axlim)
dx = 1.0
dy = 1.0
nx = ny = 2 * axlim
counts, x_edges_left, y_edges_left = amm_2dhist(-1.0 * axlim, -1.0 * axlim,
nx, ny, dx, dy, dx_all,
dy_all)
counts = counts.astype(float)
# 7.5 value is tailored to the CI -- revisit for GFAs !!
fwhm_pix = 4.7
sigma_pix = fwhm_pix / (2 * np.sqrt(2 * np.log(2)))
smth = gaussian_filter(counts, sigma_pix, mode='constant')
counts_shape = counts.shape
sidelen = counts_shape[0]
#plt.imshow(smth, cmap='gray_r')
#plt.show()
indmax = np.argmax(smth)
indx = (indmax // sidelen).astype(int)
indy = (indmax % sidelen).astype(int)
#print(indmax, indx, indy, counts_shape, x_edges_left[indx],
# y_edges_left[indy])
#print(x_edges_left[indx], y_edges_left[indy], ' ~~~~~~~~~~~~~~~~')
xshift_best = x_edges_left[indx] + 0.5 * dx
yshift_best = y_edges_left[indy] + 0.5 * dy
# assumes dx = dy = 1 !!
ycen_grid, xcen_grid = np.meshgrid(x_edges_left[0:(len(x_edges_left) - 1)],
y_edges_left[0:(len(y_edges_left) - 1)])
d = np.sqrt(
np.power(xcen_grid - xshift_best, 2) +
np.power(ycen_grid - yshift_best, 2))
#fitsio.write('/global/cscratch1/sd/ameisner/smth.fits', smth)
#fitsio.write('/global/cscratch1/sd/ameisner/d.fits', d)
r_max = 4 * 4.7 # roughly 4 asec radius for the CI
sind = np.argsort(np.ravel(smth))
val_90 = np.ravel(smth)[sind[int(np.round(0.925 * len(sind)))]]
high = [(d < r_max) & ((smth == np.max(smth)) | (smth > val_90))]
assert (np.sum(high) > 0)
print(xshift_best, yshift_best)
wt = np.sum(high * smth)
xshift_best = np.sum(high * xcen_grid * smth) / wt
yshift_best = np.sum(high * ycen_grid * smth) / wt
print(xshift_best, yshift_best)
contrast = center_contrast(smth)
#print(xcen_grid.shape)
#print(counts.shape, smth.shape)
result = {
'xshift_best': xshift_best,
'yshift_best': yshift_best,
'contrast': contrast,
'extname': extname,
'astr_guess': astrom
}
return result
def do_aper_phot(data, catalog, extname, ivar_adu):
# catalog should be the catalog with refined centroids
# for **one CI camera**
print('Attempting to do aperture photometry')
positions = list(zip(catalog['xcentroid'], catalog['ycentroid']))
radii = aper_rad_pix(extname)
apertures = [CircularAperture(positions, r=r) for r in radii]
annulus_apertures = CircularAnnulus(positions, r_in=60.0, r_out=65.0)
annulus_masks = annulus_apertures.to_mask(method='center')
par = common.ci_misc_params()
b_over_a = (1.0 if (extname == 'CIC') else par['nominal_mer_cd'] /
par['nominal_sag_cd'])
# the long axis of elliptical aperture (in terms of pixels) needs to
# be in the CI pixel Y direction
apertures_ell = [
EllipticalAperture(positions, a, a * b_over_a, theta=np.pi / 2)
for a in radii
]
# 107 um fiber diam, 9 um on a side for a pixel
# fiber diam from Table 4.1 of https://arxiv.org/abs/1611.00037
rad_fiber_pix_sag = (107.0 / 9.0) / 2.0
deg_to_normal = 5.43 # [desi-commiss 522]
if extname != 'CIC':
rad_fiber_pix_mer = rad_fiber_pix_sag * np.sin(deg_to_normal /
(180.0 / np.pi))
else:
rad_fiber_pix_mer = rad_fiber_pix_sag
aper_fib = EllipticalAperture(positions,
rad_fiber_pix_sag,
rad_fiber_pix_mer,
theta=np.pi / 2)
bkg_median = []
for mask in annulus_masks:
annulus_data = mask.multiply(data)
annulus_data_1d = annulus_data[mask.data > 0]
# this sigma_clipped_stats call is actually the slow part !!
_, median_sigclip, std_bg = sigma_clipped_stats(annulus_data_1d)
bkg_median.append(median_sigclip)
bkg_median = np.array(bkg_median)
phot = aperture_photometry(data,
apertures,
error=aper_phot_unc_map(ivar_adu))
for i, aperture in enumerate(apertures):
aper_bkg_tot = bkg_median * _get_area_from_ap(aperture)
catalog['aper_sum_bkgsub_' +
str(i)] = phot['aperture_sum_' + str(i)] - aper_bkg_tot
catalog['aper_bkg_' + str(i)] = aper_bkg_tot
catalog['aperture_sum_err_' + str(i)] = phot['aperture_sum_err_' +
str(i)]
###
del phot
phot = aperture_photometry(data,
apertures_ell,
error=aper_phot_unc_map(ivar_adu))
for i, aperture in enumerate(apertures_ell):
aper_bkg_tot = bkg_median * _get_area_from_ap(aperture)
catalog['aper_ell_sum_bkgsub_' +
str(i)] = phot['aperture_sum_' + str(i)] - aper_bkg_tot
catalog['aper_ell_bkg_' + str(i)] = aper_bkg_tot
catalog['aperture_ell_sum_err_' + str(i)] = phot['aperture_sum_err_' +
str(i)]
###
###
del phot
phot = aperture_photometry(data,
aper_fib,
error=aper_phot_unc_map(ivar_adu))
aper_bkg_tot = bkg_median * _get_area_from_ap(aper_fib)
catalog['aper_sum_bkgsub_fib'] = phot['aperture_sum'] - aper_bkg_tot
catalog['aper_bkg_fib'] = aper_bkg_tot
catalog['aperture_sum_err_fib'] = phot['aperture_sum_err']
####
# is .area() result a vector or scalar ??
catalog['sky_annulus_area_pix'] = _get_area_from_ap(annulus_apertures)
catalog['sky_annulus_median'] = bkg_median
