def read_data(exps, work, keys, limit_bands=None, prefix='piff', use_reserved=False, frac=1.): RESERVED = 64 NOT_STAR = 128 BAD_CCDS = [2, 31, 61] MAX_TILING = 10 all_keys = keys if 'x' in keys: # Note: use this syntax rather than += to avoid changing input keys all_keys = all_keys + ['fov_x', 'fov_y'] all_keys = all_keys + ['exp', 'ccd', 'band', 'tiling'] all_data = { key : [] for key in all_keys } bands = set() # This is the set of all bands being used tilings = set() # This is the set of all tilings being used n_reject_mean_dt = 0 n_reject_mean_de1 = 0 n_reject_mean_de2 = 0 n_reject_mean_e1 = 0 n_reject_mean_e2 = 0 n_reject_std_dt = 0 n_reject_std_de1 = 0 n_reject_std_de2 = 0 n_reject_rho2 = 0 nrows = 0 n_good_obj = 0 n_bad_obj = 0 for exp in sorted(exps): expnum = int(exp) old = False expname = os.path.join(work, exp, 'exp_psf_cat_%d.fits'%expnum) if not os.path.exists(expname): # Old file name (v21 and earlier) old = True expname = os.path.join(work, exp, 'exp_info_%d.fits'%expnum) if not os.path.exists(expname): print('Neither kind of exposure catalog exists.') print(os.path.join(work, exp, 'exp_psf_cat_%d.fits'%expnum)) print(expname) print('Skip this exposure') continue try: if old: expcat = fitsio.read(expname) else: print(expname) expcat = fitsio.read(expname, ext='info') except Exception as e: print('Error reading exposure catalog') print('Caught ',e) print('Skip this exposure') continue if len(expcat) == 0: print('expcat for exp=%d has no entries!',expnum) continue band = expcat['band'][0] if (limit_bands is not None) and (band not in limit_bands): #print('Not doing band = %s.'%band) continue if expcat['expnum'][0] != expnum: print('%s != %s'%(expcat['expnum'][0], expnum)) print('expnum in catalog does not match expected value') sys.exit() print('Start work on exp = ',exp) print('band = ',band) if 'tiling' in expcat.dtype.names: tiling = int(expcat['tiling'][0]) if tiling == 0: # This shouldn't happen, but it did for a few exposures. Just skip them, since this # might indicate some kind of problem. print('tiling == 0. Skip this exposure.') continue if tiling > MAX_TILING: print('tiling is > %d. Skip this exposure.'%MAX_TILING) continue print('tiling = ',tiling) else: tiling = 0 if old: mask = ~np.in1d(expcat['ccdnum'], BAD_CCDS) mask &= expcat['flag'] == 0 data, ccdnums = old_read_ccd_data(expcat[mask], work, expnum) else: data = fitsio.read(expname, ext='stars') ccdnums = data['ccdnum'].astype(int) flag = data[prefix+'_flag'].astype(int) ntot = len(data) nused = np.sum((flag & 1) != 0) nreserved = np.sum((flag & RESERVED) != 0) ngood = np.sum(flag == 0) print('ntot = ',ntot) print('nused = ',nused) print('nreserved = ',nreserved) print('ngood = ',ngood) mask = (flag & NOT_STAR) == 0 mask &= ~np.in1d(ccdnums, BAD_CCDS) if use_reserved: mask &= (flag & RESERVED) != 0 used = (flag & ~RESERVED) == 0 #print('flag = ',flag) #print('mask = ',mask) #print('used = ',used) #print('flag where flag == RESERVED: ',flag[flag==RESERVED+1]) #print('mask where flag == RESERVED: ',mask[flag==RESERVED+1]) #print('used where flag == RESERVED: ',mask[flag==RESERVED+1]) print('nmask = ',np.sum(mask)) print('nused = ',np.sum(used)) T = data['obs_T'] e1 = data['obs_e1'] e2 = data['obs_e2'] dT = data['obs_T'] - data[prefix + '_T'] de1 = data['obs_e1'] - data[prefix + '_e1'] de2 = data['obs_e2'] - data[prefix + '_e2'] print(expnum, len(dT), band) #print('T = ',np.mean(T[used]),np.std(T[used])) #print('e1 = ',np.mean(e1[used]),np.std(e1[used])) #print('e2 = ',np.mean(e2[used]),np.std(e2[used])) #print('dT/T = ',np.mean(dT[used]/T[used]),np.std(dT[used]/T[used])) #print('de1 = ',np.mean(de1[used]),np.std(de1[used])) #print('de2 = ',np.mean(de2[used]),np.std(de2[used])) rho1 = (de1 - 1j*de2) * (de1 + 1j*de2) #print('mean rho1 = ',np.mean(rho1[used])) rho2 = (e1 - 1j*e2) * (de1 + 1j*de2) #print('mean rho2 = ',np.mean(rho2[used])) if abs(np.mean(dT[used]/T[used])) > 0.01: print('mean dT/T = %f.'%(np.mean(dT[used]/T[used]))) n_reject_mean_dt += 1 #continue if abs(np.mean(de1[used])) > 0.01: print('mean de1 = %f.'%(np.mean(de1[used]))) n_reject_mean_de1 += 1 #continue if abs(np.mean(de2[used])) > 0.01: print('mean de2 = %f.'%(np.mean(de2[used]))) n_reject_mean_de2 += 1 #continue if abs(np.std(dT[used]/T[used])) > 0.1: print('std dT/T = %f.'%(np.std(dT[used]/T[used]))) n_reject_std_dt += 1 #continue if abs(np.std(de1[used])) > 0.1: print('std de1 = %f.'%(np.std(de1[used]))) n_reject_std_de1 += 1 #continue if abs(np.std(de2[used])) > 0.1: print('std de2 = %f.'%(np.std(de2[used]))) n_reject_std_de2 += 1 #continue if abs(np.mean(rho1[used])) > 5.e-4: print('mean rho1 = %s.'%(np.mean(rho1[used]))) n_reject_rho2 += 1 #continue if abs(np.mean(rho2[used])) > 5.e-4: print('mean rho2 = %s.'%(np.mean(rho2[used]))) n_reject_rho2 += 1 #continue if abs(np.mean(e1[used])) > 0.03: print('mean e1 = %f.'%(np.mean(e1[used]))) n_reject_mean_e1 += 1 #continue if abs(np.mean(e2[used])) > 0.03: print('mean e2 = %f.'%(np.mean(e2[used]))) n_reject_mean_e2 += 1 #continue # Filter out egregiously bad values. Just in case. good = (abs(dT/T) < 0.1) & (abs(de1) < 0.1) & (abs(de2) < 0.1) n1 = np.sum(mask) mask = mask & good n2 = np.sum(mask) print('"good" filter removed %d/%d objects'%(n1-n2,n1)) n_good_obj += n2 n_bad_obj += n1-n2 #print('mask = ',len(mask),np.sum(mask),mask) mask = np.where(mask)[0] #print('mask = ',len(mask),mask) if frac != 1.: mask = np.random.choice(mask, int(frac * len(mask)), replace=False) #print('mask = ',len(mask),mask) ngood = len(mask) print('ngood = ',ngood,'/',len(data)) assert ngood == len(data[mask]) if ngood == 0: print('All objects in exp %d are flagged.'%expnum) continue # Start with just the input keys, which should be columns in data. for key in keys: all_data[key].append(data[key][mask]) # Now add the extra ones we added to all_keys if 'x' in keys: # Convert to focal position. x,y = toFocal(ccdnums[mask], data['x'][mask], data['y'][mask]) # This comes back in units of mm. Convert to arcsec. # 1 pixel = 15e-3 mm = 0.263 arcsec x *= 0.263/15e-3 y *= 0.263/15e-3 all_data['fov_x'].append(x) all_data['fov_y'].append(y) all_data['ccd'].append(ccdnums[mask]) all_data['exp'].append([expnum] * ngood) all_data['band'].append([band] * ngood) all_data['tiling'].append([tiling] * ngood) bands.add(band) tilings.add(tiling) nrows += ngood print('\nFinished processing %d exposures'%len(exps)) print('bands = ',bands) print('tilings = ',tilings) print('total "good" stars = ',n_good_obj) print('total "bad" stars = ',n_bad_obj) print('total good stars selected = ',nrows) print('Potential rejections: (not enabled):') print('n_reject_mean_dt = ',n_reject_mean_dt) print('n_reject_mean_de1 = ',n_reject_mean_de1) print('n_reject_mean_de2 = ',n_reject_mean_de2) print('n_reject_mean_e1 = ',n_reject_mean_de1) print('n_reject_mean_e2 = ',n_reject_mean_de2) print('n_reject_std_dt = ',n_reject_std_dt) print('n_reject_std_de1 = ',n_reject_std_de1) print('n_reject_std_de2 = ',n_reject_std_de2) print('n_reject_rho2 = ',n_reject_rho2) # Turn the data into a recarray # Pick appropriate formats for each kind of data formats = [] for key in all_keys: if key == 'ccd' or key == 'tiling': formats.append('i2') elif key == 'exp' or 'flag' in key: formats.append('i4') elif key == 'band': formats.append('a1') else: formats.append('f8') data = np.recarray(shape=(nrows,), formats=formats, names=all_keys) #print('data.dtype = ',data.dtype) for key in all_keys: data[key] = np.concatenate(all_data[key]) print('made recarray') return data, bands, tilings
def bin_by_fov(ccd, x, y, e1, e2, s, w=None, nwhisk=5): from toFocal import toFocal all_x = np.array([]) all_y = np.array([]) all_e1 = np.array([]) all_e2 = np.array([]) all_s = np.array([]) if w is None: w = np.ones(len(x)) x_bins = np.linspace(0,2048,nwhisk+1) y_bins = np.linspace(0,4096,2*nwhisk+1) print('x_bins = ',x_bins) print('y_bins = ',y_bins) ccdnums = np.unique(ccd) print('ccdnums = ',ccdnums) for ccdnum in ccdnums: mask = np.where(ccd == ccdnum)[0] print('ccdnum = ',ccdnum,', nstar = ',len(mask)) if mask.sum() < 100: continue if ccdnum in [31, 61]: continue x_index = np.digitize(x[mask], x_bins) y_index = np.digitize(y[mask], y_bins) nbins = (len(x_bins)-1) * (len(y_bins)-1) bin_e1 = np.empty(nbins) bin_e2 = np.empty(nbins) bin_s = np.empty(nbins) bin_x = np.empty(nbins) bin_y = np.empty(nbins) bin_nz = np.empty(nbins, dtype=bool) sumesq = 0. for i in range(1, len(x_bins)): for j in range(1, len(y_bins)): k = (i-1)*(len(y_bins)-1) + (j-1) mask2 = np.where( (x_index == i) & (y_index == j) )[0] ww = w[mask][mask2] ws = ww.sum() bin_e1[k] = (ww * e1[mask][mask2]).sum() / ws bin_e2[k] = (ww * e2[mask][mask2]).sum() / ws bin_s[k] = (ww * s[mask][mask2]).sum() / ws bin_x[k] = (ww * x[mask][mask2]).sum() / ws bin_y[k] = (ww * y[mask][mask2]).sum() / ws bin_nz[k] = len(mask2) > 0 print(i,j,k,len(mask2), bin_e1[k], bin_e2[k]) sumesq += bin_e1[k]**2 + bin_e2[k]**2 print('rms e = ',np.sqrt(sumesq / nbins)) print('ccdnum = ',ccdnum) print('bin_x,y = ',bin_x,bin_y) focal_x, focal_y = toFocal(int(ccdnum), bin_x, bin_y) print('x,y = ',focal_x, focal_y) print('num with count > 0 = ',bin_nz.sum()) all_x = np.append(all_x, focal_x[bin_nz]) all_y = np.append(all_y, focal_y[bin_nz]) all_e1 = np.append(all_e1, bin_e1[bin_nz]) all_e2 = np.append(all_e2, bin_e2[bin_nz]) all_s = np.append(all_s, bin_s[bin_nz]) return all_x, all_y, all_e1, all_e2, all_s
def read_data(args, work, limit_filters=None, subtract_mean=True, reserved=False): import astropy.io.fits as pyfits datadir = '/astro/u/astrodat/data/DES' RESERVED = 64 if args.file != '': print 'Read file ',args.file with open(args.file) as fin: data = [ line.split() for line in fin ] runs, exps = zip(*data) else: runs = args.runs exps = args.exps expinfo_file = '/astro/u/mjarvis/work/exposure_info_' + args.tag + '.fits' #expinfo_file = '/astro/u/mjarvis/work/exposure_info_y1a1-v01.fits' #expinfo_file = 'exposure_info_y1spte-v02.fits' print 'reading exposure_info file: ',expinfo_file with pyfits.open(expinfo_file) as pyf: expinfo = pyf[1].data keys = ['ra', 'dec', 'x', 'y', 'e1', 'e2', 'size', 'psf_e1', 'psf_e2', 'psf_size'] all_data = { key : [] for key in keys } all_keys = keys all_data['exp'] = [] all_data['ccd'] = [] all_keys = all_keys + ['exp', 'ccd' ] if subtract_mean: all_data['alt_e1'] = [] all_data['alt_e2'] = [] all_data['alt_size'] = [] all_keys = all_keys + ['alt_e1', 'alt_e2', 'alt_size'] if 'x' in keys: all_data['fov_x'] = [] all_data['fov_y'] = [] all_keys = all_keys + ['fov_x', 'fov_y'] if args.oldkeys: inkeys = [ k.replace('psf_','psfex_') for k in keys ] else: inkeys = keys all_filters = [] # This keeps track of the filter for each record all_tilings = [] # This keeps track of the tiling for each record filters = set() # This is the set of all filters being used tilings = set() # This is the set of all tilings being used ntot = 0 nused = 0 nreserved = 0 ngood = 0 cat_dir = os.path.join(work,'psf_cats') for run,exp in zip(runs,exps): print 'Start work on run, exp = ',run,exp expnum = int(exp[6:]) print 'expnum = ',expnum if expnum not in expinfo['expnum']: print 'expnum is not in expinfo!' print 'expinfo[expnum] = ',expinfo['expnum'] print 'Could not find information about this expnum. Skipping ',run,exp continue k = numpy.nonzero(expinfo['expnum'] == expnum)[0][0] print 'k = ',k filter = expinfo['filter'][k] print 'filter[k] = ',filter if (limit_filters is not None) and (filter not in limit_filters): print 'Not doing this filter.' continue tiling = int(expinfo['tiling'][k]) print 'tiling[k] = ',tiling if tiling == 0: # This shouldn't happen, but it did for a few exposures. Just skip them, since this # might indicate some kind of problem. print 'tiling == 0. Skip this exposure.' continue if tiling > args.max_tiling: print 'tiling is > %d. Skip this exposure.'%args.max_tiling continue cat_file = os.path.join(cat_dir, exp + "_exppsf.fits") if not os.path.exists(cat_file): cat_file = os.path.join(cat_dir, exp + "_psf.fits") print 'cat_file = ',cat_file try: with pyfits.open(cat_file) as pyf: data = pyf[1].data except: print 'Unable to open cat_file %s. Skipping this file.'%cat_file continue #ccdnums = numpy.unique(data['ccdnum']) #print 'ccdnums = ',ccdnums print 'n = ',len(data) print 'nused = ',numpy.sum((data['flag'] & 1) != 0) print 'nreserved = ',numpy.sum((data['flag'] & 64) != 0) print 'ngood = ',numpy.sum(data['flag'] == 0) ntot += len(data) nused += numpy.sum((data['flag'] & 1) != 0) nreserved += numpy.sum((data['flag'] & 64) != 0) ngood += numpy.sum(data['flag'] == 0) if args.use_reserved: mask = (data['flag'] == RESERVED) else: mask = (data['flag'] == 0) ngood = numpy.sum(mask) assert ngood == len(data[mask]) if ngood == 0: print 'All objects in this exposure are flagged.' print 'Probably due to astrometry flags. Skip this exposure.' continue for key, inkey in zip(keys, inkeys): all_data[key].append(data[inkey][mask]) all_data['exp'].append([expnum] * ngood) all_data['ccd'].append(data['ccdnum'][mask]) if subtract_mean: e1 = data['e1'][mask] e2 = data['e2'][mask] s = data['size'][mask] p_e1 = data['psf_e1'][mask] p_e2 = data['psf_e2'][mask] p_s = data['psf_size'][mask] de1 = numpy.mean(e1-p_e1) de2 = numpy.mean(e2-p_e2) # Really want <(s^2 - p_s^2)/s^2> => 0 after subtracting ds # <1 - p_s^2/(s-ds)^2> = 0 # <1 - p_s^2/s^2 (1 + 2ds/s + 3ds^2/s^2 + ...) > = 0 # 1 - <p_s^2/s^2> - 2ds<p_s^2/s^3> - 3ds^2<p_s^2/s^4> = 0 a1 = numpy.mean(p_s**2/s**2) a2 = numpy.mean(p_s**2/s**3) a3 = numpy.mean(p_s**2/s**4) ds = (1. - a1) / (2.*a2) # Iterate once to refine ds = (1. - a1 - 3.*ds**2*a3) / (2.*a2) print 'de = ',de1,de2, 'mean e = ',numpy.mean(e1),numpy.mean(e2), print ' -> ',numpy.mean(e1-de1), numpy.mean(e2-de2) print 'ds = ',ds, 'mean s = ',numpy.mean(s),' -> ',numpy.mean(s-ds) print 'mean dt = ',numpy.mean( (s**2 - p_s**2) / s**2 ), print ' -> ',numpy.mean( ((s-ds)**2 - p_s**2) / (s-ds)**2 ) all_data['alt_e1'].append(e1 - de1) all_data['alt_e2'].append(e2 - de2) all_data['alt_size'].append(s - ds) if 'x' in keys: # Convert to focal position. x,y = toFocal(data['ccdnum'][mask], data['x'][mask], data['y'][mask]) # This comes back in units of mm. Convert to arcsec. # 1 pixel = 15e-3 mm = 0.263 arcsec x *= 0.263/15e-3 y *= 0.263/15e-3 all_data['fov_x'].append(x) all_data['fov_y'].append(y) all_filters.extend( ([filter] * ngood) ) all_tilings.extend( ([tiling] * ngood) ) filters.add(filter) tilings.add(tiling) print '\nFinished processing all exposures' print 'filters = ',filters print 'tilings = ',tilings # Turn the data into a recarray print 'all_data.keys = ',all_data.keys() formats = ['f8'] * len(all_keys) + ['a1', 'i2'] names = all_keys + ['filter', 'tiling'] data = numpy.recarray(shape = (len(all_filters),), formats = formats, names = names) print 'data.dtype = ',data.dtype for key in all_keys: data[key] = numpy.concatenate(all_data[key]) data['filter'] = all_filters data['tiling'] = all_tilings print 'made recarray' print 'ntot = ',ntot print 'nused = ',nused print 'nreserved = ',nreserved print 'ngood = ',ngood return data, filters, tilings
def bin_by_fov(ccd, x, y, dT, de1, de2, bands): import numpy as np import matplotlib.pyplot as plt from toFocal import toFocal all_x = np.array([]) all_y = np.array([]) all_e1 = np.array([]) all_e2 = np.array([]) all_T = np.array([]) nwhisk = 5 x_bins = np.linspace(0,2048,nwhisk+1) y_bins = np.linspace(0,4096,2*nwhisk+1) ccdnums = np.unique(ccd) for ccdnum in ccdnums: mask = np.where(ccd == ccdnum)[0] print('ccdnum = ',ccdnum,', nstar = ',mask.sum()) if mask.sum() < 100: continue x_index = np.digitize(x[mask], x_bins) y_index = np.digitize(y[mask], y_bins) bin_de1 = np.array([ de1[mask][(x_index == i) & (y_index == j)].mean() for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ]) print('bin_de1 = ',bin_de1) bin_de2 = np.array([ de2[mask][(x_index == i) & (y_index == j)].mean() for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ]) print('bin_de2 = ',bin_de2) bin_dT = np.array([ dT[mask][(x_index == i) & (y_index == j)].mean() for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ]) print('bin_dT = ',bin_dT) bin_x = np.array([ x[mask][(x_index == i) & (y_index == j)].mean() for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ]) print('bin_x = ',bin_x) bin_y = np.array([ y[mask][(x_index == i) & (y_index == j)].mean() for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ]) print('bin_y = ',bin_y) bin_count = np.array([ ((x_index == i) & (y_index == j)).sum() for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ]) print('bin_count = ',bin_count) focal_x, focal_y = toFocal(ccdnum, bin_x, bin_y) mask2 = np.where(bin_count > 0)[0] print('num with count > 0 = ',mask2.sum()) all_x = np.append(all_x, focal_x[mask2]) all_y = np.append(all_y, focal_y[mask2]) all_e1 = np.append(all_e1, bin_de1[mask2]) all_e2 = np.append(all_e2, bin_de2[mask2]) all_T = np.append(all_T, bin_dT[mask2]) plt.clf() #plt.title('PSF Ellipticity residuals in DES focal plane') theta = np.arctan2(all_e2,all_e1)/2. r = np.sqrt(all_e1**2 + all_e2**2) u = r*np.cos(theta) v = r*np.sin(theta) plt.xlim(-250,250) plt.ylim(-250,250) print('all_x = ',all_x) print('len(all_x) = ',len(all_x)) qv = plt.quiver(all_x,all_y,u,v, pivot='middle', scale_units='xy', headwidth=0., headlength=0., headaxislength=0., width=0.001, scale=1.e-3, color='blue') ref_scale = 0.01 if 'raw' in bands: ref_label = 'e = ' + str(ref_scale) else: ref_label = 'de = ' + str(ref_scale) plt.quiverkey(qv, 0.10, 0.08, ref_scale, ref_label, coordinates='axes', color='darkred', labelcolor='darkred', labelpos='E', fontproperties={'size':'x-small'}) plt.axis('off') plt.savefig('de_fov_' + bands + '.png') plt.savefig('de_fov_' + bands + '.pdf') plt.savefig('de_fov_' + bands + '.eps')
def read_data(exps, work, keys, limit_bands=None, prefix='piff', use_reserved=False, frac=1.): RESERVED = 64 NOT_STAR = 128 BAD_CCDS = [2, 31, 61] MAX_TILING = 10 all_keys = keys if 'x' in keys: # Note: use this syntax rather than += to avoid changing input keys all_keys = all_keys + ['fov_x', 'fov_y'] all_keys = all_keys + ['exp', 'ccd', 'band', 'tiling'] all_data = {key: [] for key in all_keys} bands = set() # This is the set of all bands being used tilings = set() # This is the set of all tilings being used n_reject_mean_dt = 0 n_reject_mean_de1 = 0 n_reject_mean_de2 = 0 n_reject_mean_e1 = 0 n_reject_mean_e2 = 0 n_reject_std_dt = 0 n_reject_std_de1 = 0 n_reject_std_de2 = 0 n_reject_rho2 = 0 nrows = 0 n_good_obj = 0 n_bad_obj = 0 for exp in sorted(exps): expnum = int(exp) old = False expname = os.path.join(work, exp, 'exp_psf_cat_%d.fits' % expnum) if not os.path.exists(expname): # Old file name (v21 and earlier) old = True expname = os.path.join(work, exp, 'exp_info_%d.fits' % expnum) if not os.path.exists(expname): print('Neither kind of exposure catalog exists.') print(os.path.join(work, exp, 'exp_psf_cat_%d.fits' % expnum)) print(expname) print('Skip this exposure') continue try: if old: expcat = fitsio.read(expname) else: print(expname) expcat = fitsio.read(expname, ext='info') except Exception as e: print('Error reading exposure catalog') print('Caught ', e) print('Skip this exposure') continue if len(expcat) == 0: print('expcat for exp=%d has no entries!', expnum) continue band = expcat['band'][0] if (limit_bands is not None) and (band not in limit_bands): #print('Not doing band = %s.'%band) continue if expcat['expnum'][0] != expnum: print('%s != %s' % (expcat['expnum'][0], expnum)) print('expnum in catalog does not match expected value') sys.exit() print('Start work on exp = ', exp) print('band = ', band) if 'tiling' in expcat.dtype.names: tiling = int(expcat['tiling'][0]) if tiling == 0: # This shouldn't happen, but it did for a few exposures. Just skip them, since this # might indicate some kind of problem. print('tiling == 0. Skip this exposure.') continue if tiling > MAX_TILING: print('tiling is > %d. Skip this exposure.' % MAX_TILING) continue print('tiling = ', tiling) else: tiling = 0 if old: mask = ~np.in1d(expcat['ccdnum'], BAD_CCDS) mask &= expcat['flag'] == 0 data, ccdnums = old_read_ccd_data(expcat[mask], work, expnum) else: data = fitsio.read(expname, ext='stars') ccdnums = data['ccdnum'].astype(int) flag = data[prefix + '_flag'].astype(int) ntot = len(data) nused = np.sum((flag & 1) != 0) nreserved = np.sum((flag & RESERVED) != 0) ngood = np.sum(flag == 0) print('ntot = ', ntot) print('nused = ', nused) print('nreserved = ', nreserved) print('ngood = ', ngood) mask = (flag & NOT_STAR) == 0 mask &= ~np.in1d(ccdnums, BAD_CCDS) if use_reserved: mask &= (flag & RESERVED) != 0 used = (flag & ~RESERVED) == 0 #print('flag = ',flag) #print('mask = ',mask) #print('used = ',used) #print('flag where flag == RESERVED: ',flag[flag==RESERVED+1]) #print('mask where flag == RESERVED: ',mask[flag==RESERVED+1]) #print('used where flag == RESERVED: ',mask[flag==RESERVED+1]) print('nmask = ', np.sum(mask)) print('nused = ', np.sum(used)) T = data['obs_T'] e1 = data['obs_e1'] e2 = data['obs_e2'] dT = data['obs_T'] - data[prefix + '_T'] de1 = data['obs_e1'] - data[prefix + '_e1'] de2 = data['obs_e2'] - data[prefix + '_e2'] print(expnum, len(dT), band) #print('T = ',np.mean(T[used]),np.std(T[used])) #print('e1 = ',np.mean(e1[used]),np.std(e1[used])) #print('e2 = ',np.mean(e2[used]),np.std(e2[used])) #print('dT/T = ',np.mean(dT[used]/T[used]),np.std(dT[used]/T[used])) #print('de1 = ',np.mean(de1[used]),np.std(de1[used])) #print('de2 = ',np.mean(de2[used]),np.std(de2[used])) rho1 = (de1 - 1j * de2) * (de1 + 1j * de2) #print('mean rho1 = ',np.mean(rho1[used])) rho2 = (e1 - 1j * e2) * (de1 + 1j * de2) #print('mean rho2 = ',np.mean(rho2[used])) if abs(np.mean(dT[used] / T[used])) > 0.01: print('mean dT/T = %f.' % (np.mean(dT[used] / T[used]))) n_reject_mean_dt += 1 #continue if abs(np.mean(de1[used])) > 0.01: print('mean de1 = %f.' % (np.mean(de1[used]))) n_reject_mean_de1 += 1 #continue if abs(np.mean(de2[used])) > 0.01: print('mean de2 = %f.' % (np.mean(de2[used]))) n_reject_mean_de2 += 1 #continue if abs(np.std(dT[used] / T[used])) > 0.1: print('std dT/T = %f.' % (np.std(dT[used] / T[used]))) n_reject_std_dt += 1 #continue if abs(np.std(de1[used])) > 0.1: print('std de1 = %f.' % (np.std(de1[used]))) n_reject_std_de1 += 1 #continue if abs(np.std(de2[used])) > 0.1: print('std de2 = %f.' % (np.std(de2[used]))) n_reject_std_de2 += 1 #continue if abs(np.mean(rho1[used])) > 5.e-4: print('mean rho1 = %s.' % (np.mean(rho1[used]))) n_reject_rho2 += 1 #continue if abs(np.mean(rho2[used])) > 5.e-4: print('mean rho2 = %s.' % (np.mean(rho2[used]))) n_reject_rho2 += 1 #continue if abs(np.mean(e1[used])) > 0.03: print('mean e1 = %f.' % (np.mean(e1[used]))) n_reject_mean_e1 += 1 #continue if abs(np.mean(e2[used])) > 0.03: print('mean e2 = %f.' % (np.mean(e2[used]))) n_reject_mean_e2 += 1 #continue # Filter out egregiously bad values. Just in case. good = (abs(dT / T) < 0.1) & (abs(de1) < 0.1) & (abs(de2) < 0.1) n1 = np.sum(mask) mask = mask & good n2 = np.sum(mask) print('"good" filter removed %d/%d objects' % (n1 - n2, n1)) n_good_obj += n2 n_bad_obj += n1 - n2 #print('mask = ',len(mask),np.sum(mask),mask) mask = np.where(mask)[0] #print('mask = ',len(mask),mask) if frac != 1.: mask = np.random.choice(mask, int(frac * len(mask)), replace=False) #print('mask = ',len(mask),mask) ngood = len(mask) print('ngood = ', ngood, '/', len(data)) assert ngood == len(data[mask]) if ngood == 0: print('All objects in exp %d are flagged.' % expnum) continue # Start with just the input keys, which should be columns in data. for key in keys: all_data[key].append(data[key][mask]) # Now add the extra ones we added to all_keys if 'x' in keys: # Convert to focal position. x, y = toFocal(ccdnums[mask], data['x'][mask], data['y'][mask]) # This comes back in units of mm. Convert to arcsec. # 1 pixel = 15e-3 mm = 0.263 arcsec x *= 0.263 / 15e-3 y *= 0.263 / 15e-3 all_data['fov_x'].append(x) all_data['fov_y'].append(y) all_data['ccd'].append(ccdnums[mask]) all_data['exp'].append([expnum] * ngood) all_data['band'].append([band] * ngood) all_data['tiling'].append([tiling] * ngood) bands.add(band) tilings.add(tiling) nrows += ngood print('\nFinished processing %d exposures' % len(exps)) print('bands = ', bands) print('tilings = ', tilings) print('total "good" stars = ', n_good_obj) print('total "bad" stars = ', n_bad_obj) print('total good stars selected = ', nrows) print('Potential rejections: (not enabled):') print('n_reject_mean_dt = ', n_reject_mean_dt) print('n_reject_mean_de1 = ', n_reject_mean_de1) print('n_reject_mean_de2 = ', n_reject_mean_de2) print('n_reject_mean_e1 = ', n_reject_mean_de1) print('n_reject_mean_e2 = ', n_reject_mean_de2) print('n_reject_std_dt = ', n_reject_std_dt) print('n_reject_std_de1 = ', n_reject_std_de1) print('n_reject_std_de2 = ', n_reject_std_de2) print('n_reject_rho2 = ', n_reject_rho2) # Turn the data into a recarray # Pick appropriate formats for each kind of data formats = [] for key in all_keys: if key == 'ccd' or key == 'tiling': formats.append('i2') elif key == 'exp' or 'flag' in key: formats.append('i4') elif key == 'band': formats.append('a1') else: formats.append('f8') data = np.recarray(shape=(nrows, ), formats=formats, names=all_keys) #print('data.dtype = ',data.dtype) for key in all_keys: data[key] = np.concatenate(all_data[key]) print('made recarray') return data, bands, tilings
def bin_by_fov(ccd, x, y, dT, de1, de2, bands): import numpy as np import matplotlib.pyplot as plt from toFocal import toFocal all_x = np.array([]) all_y = np.array([]) all_e1 = np.array([]) all_e2 = np.array([]) all_T = np.array([]) nwhisk = 5 x_bins = np.linspace(0, 2048, nwhisk + 1) y_bins = np.linspace(0, 4096, 2 * nwhisk + 1) ccdnums = np.unique(ccd) for ccdnum in ccdnums: mask = np.where(ccd == ccdnum)[0] print('ccdnum = ', ccdnum, ', nstar = ', mask.sum()) if mask.sum() < 100: continue x_index = np.digitize(x[mask], x_bins) y_index = np.digitize(y[mask], y_bins) bin_de1 = np.array([ de1[mask][(x_index == i) & (y_index == j)].mean() for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ]) print('bin_de1 = ', bin_de1) bin_de2 = np.array([ de2[mask][(x_index == i) & (y_index == j)].mean() for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ]) print('bin_de2 = ', bin_de2) bin_dT = np.array([ dT[mask][(x_index == i) & (y_index == j)].mean() for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ]) print('bin_dT = ', bin_dT) bin_x = np.array([ x[mask][(x_index == i) & (y_index == j)].mean() for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ]) print('bin_x = ', bin_x) bin_y = np.array([ y[mask][(x_index == i) & (y_index == j)].mean() for i in range(1, len(x_bins)) for j in range(1, len(y_bins)) ]) print('bin_y = ', bin_y) bin_count = np.array([((x_index == i) & (y_index == j)).sum() for i in range(1, len(x_bins)) for j in range(1, len(y_bins))]) print('bin_count = ', bin_count) focal_x, focal_y = toFocal(ccdnum, bin_x, bin_y) mask2 = np.where(bin_count > 0)[0] print('num with count > 0 = ', mask2.sum()) all_x = np.append(all_x, focal_x[mask2]) all_y = np.append(all_y, focal_y[mask2]) all_e1 = np.append(all_e1, bin_de1[mask2]) all_e2 = np.append(all_e2, bin_de2[mask2]) all_T = np.append(all_T, bin_dT[mask2]) plt.clf() #plt.title('PSF Ellipticity residuals in DES focal plane') theta = np.arctan2(all_e2, all_e1) / 2. r = np.sqrt(all_e1**2 + all_e2**2) u = r * np.cos(theta) v = r * np.sin(theta) plt.xlim(-250, 250) plt.ylim(-250, 250) print('all_x = ', all_x) print('len(all_x) = ', len(all_x)) qv = plt.quiver(all_x, all_y, u, v, pivot='middle', scale_units='xy', headwidth=0., headlength=0., headaxislength=0., width=0.001, scale=1.e-3, color='blue') ref_scale = 0.01 if 'raw' in bands: ref_label = 'e = ' + str(ref_scale) else: ref_label = 'de = ' + str(ref_scale) plt.quiverkey(qv, 0.10, 0.08, ref_scale, ref_label, coordinates='axes', color='darkred', labelcolor='darkred', labelpos='E', fontproperties={'size': 'x-small'}) plt.axis('off') plt.savefig('de_fov_' + bands + '.png') plt.savefig('de_fov_' + bands + '.pdf') plt.savefig('de_fov_' + bands + '.eps')
def read_data(args, work, limit_bands=None, prefix='piff'): import fitsio RESERVED = 64 BAD_CCDS = [2, 31, 61] if args.file != '': print('Read file ',args.file) with open(args.file) as fin: exps = [ line.strip() for line in fin if line[0] != '#' ] print('File included %d exposures'%len(exps)) else: exps = args.exps print('Explicit listing of %d exposures'%len(exps)) exps = sorted(exps) keys = ['ra', 'dec', 'x', 'y', 'mag', 'obs_e1', 'obs_e2', 'obs_T', prefix+'_e1', prefix+'_e2', prefix+'_T'] all_data = { key : [] for key in keys } all_keys = keys all_data['exp'] = [] all_data['ccd'] = [] all_keys = all_keys + ['exp', 'ccd' ] if 'x' in keys: all_data['fov_x'] = [] all_data['fov_y'] = [] all_keys = all_keys + ['fov_x', 'fov_y'] inkeys = keys all_bands = [] # This keeps track of the band for each record #all_tilings = [] # This keeps track of the tiling for each record bands = set() # This is the set of all bands being used #tilings = set() # This is the set of all tilings being used for exp in exps: print('Start work on exp = ',exp) expnum = int(exp) print('expnum = ',expnum) expinfo = fitsio.read(os.path.join(work, exp, 'exp_info_%d.fits'%expnum)) if expnum not in expinfo['expnum']: print('expnum is not in expinfo!') print('expinfo[expnum] = ',expinfo['expnum']) print('Could not find information about this expnum. Skipping ',run,exp) continue i = np.nonzero(expinfo['expnum'] == expnum)[0][0] #print('i = ',i) band = expinfo['band'][i] #print('band[k] = ',band) if (limit_bands is not None) and (band not in limit_bands): print('Not doing band = %s.'%band) continue #tiling = int(expinfo['tiling'][k]) #print('tiling[k] = ',tiling) #if tiling == 0: # This shouldn't happen, but it did for a few exposures. Just skip them, since this # might indicate some kind of problem. #print('tiling == 0. Skip this exposure.') #continue #if tiling > args.max_tiling: #print('tiling is > %d. Skip this exposure.'%args.max_tiling) #continue for k in range(len(expinfo)): ccdnum = expinfo[k]['ccdnum'] if expinfo[k]['flag'] != 0: print('Skipping ccd %d because it is blacklisted: '%ccdnum, expinfo[k]['flag']) continue if ccdnum in BAD_CCDS: print('Skipping ccd %d because it is BAD'%ccdnum) continue cat_file = os.path.join(work, exp, "psf_cat_%d_%d.fits"%(expnum,ccdnum)) #print('cat_file = ',cat_file) try: data = fitsio.read(cat_file) flag = data[prefix+'_flag'] except (OSError, IOError): print('Unable to open cat_file %s. Skipping this file.'%cat_file) continue ntot = len(data) nused = np.sum((flag & 1) != 0) nreserved = np.sum((flag & RESERVED) != 0) ngood = np.sum(flag == 0) #print('nused = ',nused) #print('nreserved = ',nreserved) #print('ngood = ',ngood) if args.use_reserved: mask = (flag == RESERVED) | (flag == RESERVED+1) else: mask = (flag == 0) #print('mask = ',mask) T = data['obs_T'] dT = (data[prefix + '_T'] - data['obs_T']) de1 = (data[prefix + '_e1'] - data['obs_e1']) de2 = (data[prefix + '_e2'] - data['obs_e2']) used = (flag == 0) #if np.std(dT[used]/T[used]) > 0.03: #continue #if np.std(de1[used]) > 0.02: #continue #if np.std(de2[used]) > 0.02: #continue good = (abs(dT/data['obs_T']) < 0.1) & (abs(de1) < 0.1) & (abs(de2) < 0.1) mask = mask & good ngood = np.sum(mask) #print('ngood = ',ngood,'/',len(data)) assert ngood == len(data[mask]) if ngood == 0: print('All objects in ccd %d are flagged.'%ccdnum) print('Probably due to astrometry flags. Skip this exposure.') continue for key, inkey in zip(keys, inkeys): all_data[key].append(data[inkey][mask]) all_data['exp'].append([expnum] * ngood) all_data['ccd'].append([ccdnum] * ngood) if 'x' in keys: # Convert to focal position. x,y = toFocal(ccdnum, data['x'][mask], data['y'][mask]) # This comes back in units of mm. Convert to arcsec. # 1 pixel = 15e-3 mm = 0.263 arcsec x *= 0.263/15e-3 y *= 0.263/15e-3 all_data['fov_x'].append(x) all_data['fov_y'].append(y) all_bands.extend( ([band] * ngood) ) #all_tilings.extend( ([tiling] * ngood) ) bands.add(band) #tilings.add(tiling) print('\nFinished processing all exposures') print('bands = ',bands) #print('tilings = ',tilings) # Turn the data into a recarray print('all_data.keys = ',all_data.keys()) formats = ['f8'] * len(all_keys) + ['a1', 'i2'] #names = all_keys + ['band', 'tiling'] names = all_keys + ['band'] data = np.recarray(shape = (len(all_bands),), formats = formats, names = names) print('data.dtype = ',data.dtype) for key in all_keys: data[key] = np.concatenate(all_data[key]) data['band'] = all_bands #data['tiling'] = all_tilings print('made recarray') tilings = None return data, bands, tilings