def quad_pix_limits(quad):
# q is an integer quadrant number, one of [1, 2, 3, 4]
assert(quad in [1, 2, 3, 4])
par = common.pc_params()
half_x = par['nx'] // 2
half_y = par['ny'] // 2
if (quad == 1) or (quad == 4):
xmin = half_x
xmax = par['nx']
else:
xmin = 0
xmax = half_x
if (quad == 1) or (quad == 2):
ymin = half_y
ymax = par['ny']
else:
ymin = 0
ymax = half_y
result = {'xmin' : xmin, 'xmax' : xmax, 'ymin' : ymin, 'ymax' : ymax}
return result
def calc_many_zps(cat, exp, one_aper=False, checkplot=True, nmp=None):
print('Attempting to calculate zeropoints')
par = common.pc_params()
aper_radii = par['aper_phot_objrad'] if not one_aper else [par['aper_phot_objrad_best']]
args = []
for q in [0, 1, 2, 3, 4]:
for aper_ind in range(len(aper_radii)):
args.append((cat, aper_ind, exp.time_seconds, exp.fname_im, q, checkplot, one_aper))
if nmp is not None:
p = Pool(min(nmp, len(args)))
results = p.starmap(calc_zp, args)
p.close()
p.join()
else:
results = [calc_zp(*_arg) for _arg in args]
results = vstack(results)
results['mjd_obs'] = exp.header['MJD-OBS']
results['obs_night'] = exp.obs_night
sind = np.argsort(1000*results['quadrant'] + results['aper_ind'])
results = results[sind]
return results
def get_g_prime(G, BP_RP):
# right now this is pretty much trivial but in the future it
# could become more complex
par = common.pc_params()
g_prime = G + par['bp_rp_coeff']*BP_RP
return g_prime
def pc_gaia_cat(exp, mag_thresh=None, edge_pad_pix=0, nmp=None,
max_n_stars=3000, pm_corr=False):
print('Reading Gaia DR2 catalogs...')
xgrid, ygrid = xy_subsamp_grid()
wcs = exp.wcs
# last arg is 0 rather than 1 because that's what agrees with IDL
ra, dec = wcs.all_pix2world(xgrid, ygrid, 0)
cat = read_gaia_cat(ra, dec, nmp=nmp)
if mag_thresh is not None:
keep = (cat['PHOT_G_MEAN_MAG'] <= mag_thresh)
assert(np.sum(keep) > 0)
cat = cat[keep]
if pm_corr:
gaia_pm_corr(cat, exp.header['MJD-OBS'])
x_gaia_guess, y_gaia_guess = wcs.all_world2pix(cat['RA'], cat['DEC'], 0)
par = common.pc_params()
keep = (x_gaia_guess > edge_pad_pix) & (y_gaia_guess > edge_pad_pix) & (x_gaia_guess < (par['nx'] - 1 - edge_pad_pix)) & (y_gaia_guess < (par['ny'] - 1 - edge_pad_pix))
assert(np.sum(keep) > 0)
cat = cat[keep]
x_gaia_guess = x_gaia_guess[keep]
y_gaia_guess = y_gaia_guess[keep]
cat = Table(cat)
cat['x_gaia_guess'] = x_gaia_guess
cat['y_gaia_guess'] = y_gaia_guess
if len(cat) > max_n_stars:
# retain brightest max_n_stars
# it'd be better to do this cut based on
# 'G_PRIME', the color-corrected pointing
# camera Gaia-based mag
# in the future can evaluate trying to spread the
# selected max_n_stars evenly across quadrants
# (could imagine a pathological case with e.g., a
# globular cluster in the FOV)
print('Restricting to the brightest ' + str(max_n_stars) + \
' of ' + str(len(cat)) + ' stars')
sind = np.argsort(cat['PHOT_G_MEAN_MAG'])
cat = cat[sind[0:max_n_stars]]
return cat
def load_static_badpix():
par = common.pc_params()
fname = os.path.join(os.environ[par['meta_env_var']],
par['static_mask_filename'])
assert (os.path.exists(fname))
mask = fits.getdata(fname)
return mask
def get_quadrant(im, q):
assert(q in [1, 2, 3, 4])
check_image_dimensions(im)
par = common.pc_params()
p = quad_pix_limits(q)
quad = im[p['ymin']:p['ymax'], p['xmin']:p['xmax']]
return quad
def min_edge_dist_pix(x, y):
# minimum distance to any image edge
# works for array-valued x, y
min_edge_dist = 20000
par = common.pc_params()
min_edge_dist = np.minimum(x + 0.5, y + 0.5)
min_edge_dist = np.minimum(min_edge_dist, par['nx'] - 0.5 - x)
min_edge_dist = np.minimum(min_edge_dist, par['ny'] - 0.5 - y)
return min_edge_dist
def check_image_dimensions(image):
# check that image dimensions make sense
# of particular relevance is not getting fooled by downbinned data
print('Checking raw pointing camera image dimensions...')
par = common.pc_params()
sh = image.shape
assert(sh[0] == par['ny'])
assert(sh[1] == par['nx'])
print('Raw pointing camera image has correct dimensions')
def subtract_quad_offs(im):
par = common.pc_params()
# just in case this hasn't already been taken care of...
if im.dtype.name != 'float32':
im = im.astype('float32')
check_image_dimensions(im)
for q in [1, 2, 3, 4]:
p = quad_pix_limits(q)
im[p['ymin']:p['ymax'], p['xmin']:p['xmax']] -= par['bias_med'][q-1]
return im
def gaia_chunknames(ipix):
# could add checks to make sure that all ipix values are
# sane HEALPix pixel indices
# RIGHT NOW THIS ASSUMES IPIX IS AN ARRAY !!
# should eventually make this also work for scalar ipix
par = common.pc_params()
gaia_dir = os.environ[par['gaia_env_var']]
flist = [
os.path.join(gaia_dir, 'chunk-' + str(i).zfill(5) + '.fits')
for i in ipix
]
return flist
def pc_recentroid(im, cat):
par = common.pc_params()
im = im.astype(float)
x_center = par['nx']/2 + 0.5
y_center = par['ny']/2 + 0.5
dist_pix = np.sqrt(np.power(cat['x_gaia_guess'] - x_center, 2) + \
np.power(cat['y_gaia_guess'] - y_center, 2))
dist_max = np.sqrt(x_center**2 + y_center**2)
slope = 2.0/dist_max
cmaxshift = np.minimum(np.maximum(dist_pix*slope + 2.5, 2.5), 4.5)
assert(np.sum(cmaxshift < 2.5) == 0)
assert(np.sum(cmaxshift > 4.5) == 0)
xcen = np.zeros(len(cat), dtype=float)
ycen = np.zeros(len(cat), dtype=float)
qmaxshift = np.zeros(len(cat), dtype=int)
for i in range(len(cat)):
_xcen, _ycen, q = djs_photcen(cat['x_gaia_guess'][i],
cat['y_gaia_guess'][i], im, cbox=8,
cmaxiter=10, cmaxshift=cmaxshift[i])
xcen[i] = _xcen
ycen[i] = _ycen
qmaxshift[i] = q
result = Table()
result['xcentroid'] = xcen
result['ycentroid'] = ycen
result['x_shift'] = xcen - cat['x_gaia_guess']
result['y_shift'] = ycen - cat['y_gaia_guess']
result['cmaxshift'] = cmaxshift
result['qmaxshift'] = qmaxshift
result['centroid_shift_flag'] = (np.abs(result['x_shift']) > cmaxshift) | (np.abs(result['y_shift']) > cmaxshift) | (qmaxshift != 0)
assert(len(result) == len(cat))
cat = hstack([cat, result])
return cat
def pc_aper_phot(im, cat, one_aper=False, bg_sigclip=False):
im = im.astype(float)
par = common.pc_params()
positions = list(zip(cat['xcentroid'], cat['ycentroid']))
radii = par['aper_phot_objrad'] if not one_aper else [par['aper_phot_objrad_best']]
ann_radii = par['annulus_radii'] # should have 2 elements - inner and outer
apertures = [CircularAperture(positions, r=r) for r in radii]
annulus_apertures = CircularAnnulus(positions, r_in=ann_radii[0],
r_out=ann_radii[1])
annulus_masks = annulus_apertures.to_mask(method='center')
bkg_median = []
for mask in annulus_masks:
annulus_data = mask.multiply(im)
annulus_data_1d = annulus_data[mask.data > 0]
if bg_sigclip:
# this sigma_clipped_stats call is actually the slow part !!
_, median_sigclip, std_bg = sigma_clipped_stats(annulus_data_1d)
bkg_median.append(median_sigclip)
else:
bkg_median.append(np.median(annulus_data_1d))
bkg_median = np.array(bkg_median)
phot = aperture_photometry(im, apertures)
for i, aperture in enumerate(apertures):
aper_bkg_tot = bkg_median*_get_area_from_ap(aperture)
cat['aper_sum_bkgsub_' + str(i)] = phot['aperture_sum_' + str(i)] - aper_bkg_tot
cat['aper_bkg_' + str(i)] = aper_bkg_tot
cat['sky_annulus_area_pix'] = _get_area_from_ap(annulus_apertures)
cat['sky_annulus_median'] = bkg_median
flux_adu = np.zeros((len(cat), len(radii)), dtype=float)
for i in range(len(radii)):
flux_adu[:, i] = cat['aper_sum_bkgsub_' + str(i)]
cat['flux_adu'] = flux_adu
return cat
def source_raw_pixel_metrics(cat, raw):
par = common.pc_params()
ixcen = np.round(cat['xcentroid']).astype(int)
iycen = np.round(cat['ycentroid']).astype(int)
ixcen = np.minimum(np.maximum(ixcen, 0), par['nx']-1)
iycen = np.minimum(np.maximum(iycen, 0), par['ny']-1)
centroid_pixel_vals = raw[iycen, ixcen] # ordering of indices !
centroid_pixel_saturated = (centroid_pixel_vals == par['raw_satur_val']).astype(int)
# modify the input catalog
cat['centroid_raw_pixel_val'] = centroid_pixel_vals
cat['centroid_pixel_saturated'] = centroid_pixel_saturated
def quadrant_from_xy(x, y):
# works for array-valued x, y
par = common.pc_params()
half_x = par['nx']/2 - 0.5
half_y = par['ny']/2 - 0.5
quadrant = np.zeros(len(x), dtype=int)
quadrant[(x <= half_x) & (y <= half_y)] = 3
quadrant[(x <= half_x) & (y > half_y)] = 2
quadrant[(x > half_x) & (y <= half_y)] = 4
quadrant[(x > half_x) & (y > half_y)] = 1
return quadrant
def subtract_dark_current(im, time_seconds):
print('Subtracting dark current')
assert(time_seconds < 30)
par = common.pc_params()
result = im.astype('float32')
for q in [1, 2, 3, 4]:
dark_adu = par['dark_adu_per_s_quad'][q-1]*time_seconds
print('quadrant : ', q, ', dark counts/pix : ', '{:.3f}'.format(dark_adu))
p = quad_pix_limits(q)
result[p['ymin']:p['ymax'], p['xmin']:p['xmax']] -= dark_adu
return result
def calc_zp(_cat, aper_ind, time_seconds, fname_im, quadrant=0,
one_aper=False, checkplot=True):
# quadrant = 0 means whole image (all quadrants combined)
print('Computing zeropoint for quadrant : ', quadrant, ' , aper ', aper_ind)
assert(time_seconds > 0)
assert(quadrant in [0, 1, 2, 3, 4]) # note the 0 option here...
par = common.pc_params()
n_aper = len(par['aper_phot_objrad'])
aper_ind = int(aper_ind)
assert(aper_ind in np.arange(n_aper))
cat = copy.deepcopy(_cat)
good = np.logical_not(cat['centroid_pixel_saturated']) & \
(cat['centroid_raw_pixel_val'] < 15300) & \
np.logical_not(cat['centroid_shift_flag']) & \
np.logical_not(cat['wrong_source_centroid']) & \
np.isfinite(cat['PHOT_BP_MEAN_MAG']) & \
np.isfinite(cat['PHOT_RP_MEAN_MAG']) & \
np.isfinite(cat['PHOT_G_MEAN_MAG'])
if quadrant != 0:
good = good & (cat['quadrant'] == quadrant)
if np.sum(good) == 0:
return None
cat = cat[good]
n = len(cat)
m_inst = cat['m_inst'][:, aper_ind]
diff = cat['G_PRIME'] - m_inst
nf = np.sum(np.isfinite(diff))
zp = np.nanmedian(diff)
resid = diff - zp
# now calculate robust sigma about the median zeropoint offset
# at first glance argsort appears to put NaN's at end of sorted array
sind = np.argsort(resid) # should look into what exactly happens with NaN's
ind_l = max(int(round(0.16*nf)), 0)
ind_u = min(int(round(0.84*nf)), n-1)
resid_l = resid[sind[ind_l]]
resid_u = resid[sind[ind_u]]
sig_robust = (np.abs(resid_l) + np.abs(resid_u))/2.0
result = Table()
result['quadrant'] = [quadrant]
result['aper_ind'] = [aper_ind]
result['zp_adu_per_s'] = [zp]
result['n_sources_for_zp'] = [n]
result['time_seconds'] = [time_seconds]
result['bp_rp_median'] = [np.nanmedian(cat['BP_RP'])]
result['gaia_g_median'] = [np.nanmedian(cat['PHOT_G_MEAN_MAG'])]
result['robust_sigma_mag'] = [sig_robust]
result['fname_raw'] = [fname_im]
# checkplot (eventually make this optional)
best_aper_ind = 1 if not one_aper else 0
if checkplot and (quadrant == 0) and (aper_ind == best_aper_ind):
plt.cla()
plt.figure(1)
xtitle = 'G + 0.25*(BP-RP)'
ytitle = '-2.5' + r'$\times$' + 'log' + r'$_{10}$' + '(ADU/sec)'
title = fname_im.split('/')[-1]
title = title.replace('.fits', '')
title += '; aper' + str(best_aper_ind) + '; all quads'
plt.scatter(cat['G_PRIME'], m_inst, s=20, edgecolor='none',
facecolor='k')
xmin = np.nanmin(cat['G_PRIME'])
xmax = np.nanmax(cat['G_PRIME'])
ymin = np.nanmin(m_inst)
ymax = np.nanmax(m_inst)
xsamp = np.array([xmin, xmax])
ysamp = xsamp - zp
plt.plot(xsamp, ysamp, linewidth=2, c='r')
ax = plt.gca()
xlim = ax.get_xlim()
ylim = ax.get_ylim()
xtext = xlim[0] + (xlim[1] - xlim[0])*0.125
ytext = ylim[0] + (ylim[1] - ylim[0])*0.875
# this shouldn't crash for NaN zeropoint value...
plt.text(xtext, ytext, 'ZP = ' + '{:.2f}'.format(zp), color='r')
plt.title(title)
plt.xlabel(xtitle)
plt.ylabel(ytitle)
return result
def _check_bitpix(h_im):
par = common.pc_params()
print('Checking raw image BITPIX value')
assert(h_im['BITPIX'] == par['bitpix'])
'--max_n_stars',
default=3000,
type=int,
help="limit analysis to brightest max_n_stars Gaia stars")
parser.add_argument(
'--pm_corr',
default=False,
action='store_true',
help="make Gaia proper motion corrections based on MJD")
args = parser.parse_args()
# basic checks on requested number of multiprocessing threads
if args.multiproc is not None:
par = common.pc_params()
assert (args.multiproc > 1)
assert (args.multiproc <= par['ncpus'])
pc_proc(args.fname_in[0],
outdir=args.outdir,
dont_write_detrended=args.dont_write_detrended,
skip_checkplot=args.skip_checkplot,
nightly_subdir=args.nightly_subdir,
send_redis=args.send_redis,
one_aper=args.one_aper,
bg_sigclip=args.bg_sigclip,
nmp=args.multiproc,
max_n_stars=args.max_n_stars,
pm_corr=args.pm_corr)