def finalPatch(solver): files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE, shell=True).stdout.read().split() #run the solver sb.Popen('%s patch_obs.dat patch 1e-12 40000 > /dev/null'%solver, shell=True).wait() sb.Popen('cp patch_bestfit_FpPatch.dat test_bestfit_FpPatch.dat', shell=True).wait() npatch = np.size(files) patchdat={} final_keys = ('patchid', 'patchmag', 'healid') for key in final_keys: patchdat[key]=[] keys=('patchid','patchmag') for i in np.arange(npatch): temp = ui.readDatafile(files[i], keys, keytypes=('u8','float')) heal_num = np.float(files[i][1:-18]) #insert if statement to deal with stupid patches that are not continuous patchdat['patchid'].append(np.ravel(temp['patchid'])) patchdat['healid'].append(np.ravel(temp['patchid']*0+heal_num)) patchdat['patchmag'].append(np.ravel(temp['patchmag'])) for key in list(patchdat.keys()): patchdat[key] = np.concatenate(patchdat[key]) ord = np.argsort(patchdat['healid']) for key in list(patchdat.keys()): patchdat[key] = patchdat[key][ord] keys=('hpid','hpzp') healdat = ui.readDatafile('patch_bestfit_Patch.dat',keys, keytypes=('u8','float')) keys=('patchid','patchmag') fit_patchdat = ui.readDatafile('patch_bestfit_Star.dat',keys, keytypes=('u8','float')) left = np.searchsorted(patchdat['healid'],healdat['hpid']) right = np.searchsorted(patchdat['healid'],healdat['hpid'], side='right') for i in np.arange(np.size(right)): patchdat['patchmag'][left[i]:right[i]] = patchdat['patchmag'][left[i]:right[i]]-healdat['hpzp'][i] patchid = np.unique(patchdat['patchid']) patchmag = patchid*0 ord = np.argsort(patchdat['patchid']) patchdat['patchmag'] = patchdat['patchmag'][ord] patchdat['patchid'] = patchdat['patchid'][ord] left = np.searchsorted(patchdat['patchid'],patchid) patchmag = patchdat['patchmag'][left] #now plug in the best fit mags where they exist left = np.searchsorted(patchid, fit_patchdat['patchid']) patchmag[left]=fit_patchdat['patchmag'] file = open('test_bestfit_Patch.dat','w') for i in np.arange(np.size(patchmag)): print("%d %f"%(patchid[i], patchmag[i]), file=file) file.close()
def finalStar(infile='star_obs.dat', outfile='test_bestfit_Star.dat'): """read in the best fit patches, and the raw observations. Apply patch zps to all the star magnitudes, then take a weighted mean of each star to make a bestfit star magnitude. Also calc an stdev for each star""" patchzp = ui.readDatafile('test_bestfit_Patch.dat', ('patchid','patchzp'), keytypes=('u8','float') ) star_obs = ui.readDatafile(infile, ('patchid','starid','starmag','starerr'), keytypes=('u8','u8','float','float')) order = np.argsort(star_obs['patchid']) for key in list(star_obs.keys()): star_obs[key] = star_obs[key][order] left = np.searchsorted(star_obs['patchid'], patchzp['patchid']) right = np.searchsorted(star_obs['patchid'], patchzp['patchid'], side='right') for i in np.arange(left.size): star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][left[i]:right[i]] - patchzp['patchzp'][i] order = np.argsort(star_obs['starid']) for key in list(star_obs.keys()): star_obs[key] = star_obs[key][order] ustar = np.unique(star_obs['starid']) mag = np.zeros(ustar.size, dtype='float') magerr = np.zeros(ustar.size, dtype='float') magstd_raw = np.zeros(ustar.size, dtype='float') left = np.searchsorted(star_obs['starid'], ustar) right = np.searchsorted(star_obs['starid'], ustar, side='right') for i in np.arange(left.size): mag[i] = np.sum(star_obs['starmag'][left[i]:right[i]]/star_obs['starerr'][left[i]:right[i]]**2)/np.sum(1./star_obs['starerr'][left[i]:right[i]]**2) magerr[i] = 1./np.sqrt(np.sum(1./star_obs['starerr'][left[i]:right[i]]**2)) magstd_raw[i] = star_obs['starmag'][left[i]:right[i]].std() file = open(outfile, 'w') print('#StarID Mag Err STDEV', file=file) for i in np.arange(ustar.size): print("%i %f %f %f"%(ustar[i], mag[i], magerr[i], magstd_raw[i]), file=file) file.close()
def patch_combine(): files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE, shell=True).stdout.read().split() npatch=np.size(files) patchdat={} final_keys = ('patchid', 'patchmag', 'healid') for key in final_keys: patchdat[key]=[] keys=('patchid','patchmag') for i in np.arange(npatch): temp = ui.readDatafile(files[i], keys, keytypes=('u8','float')) heal_num = np.float(files[i][1:-18]) patchdat['patchid'].append(np.ravel(temp['patchid'])) patchdat['healid'].append(np.ravel(temp['patchid']*0+heal_num)) patchdat['patchmag'].append(np.ravel(temp['patchmag'])) for key in list(patchdat.keys()): patchdat[key] = np.concatenate(patchdat[key]) ord = np.argsort(patchdat['healid']) for key in list(patchdat.keys()): patchdat[key] = patchdat[key][ord] file = open('patch_obs.dat', 'w') print("%s %s %s %s %s" %("#HealID", "PatchID", "PatchMag", "PatchMagErr", "FpPatchID"), file=file) for obj in range(0,len(patchdat['patchid'])): print("%d %d %f %f %d"%(patchdat['healid'][obj],patchdat['patchid'][obj], patchdat['patchmag'][obj], 0.001, 0), file=file) file.close()
def read_true_patchfile(patchfile='patchdata.dat'): """Read the patch file information from simSelfCalib.py""" keys = ('visitid', 'patchid', 'subpatch', 'nstars', 'ra', 'dec', 'magtrue', 'color', 'dmag', 'dmag_rms', 'dmag_rand', 'dmag_rand_rms', 'dmag_zp', 'dmag_zp_rms', 'dmag_color', 'dmag_color_rms') patchdat = ui.readDatafile(patchfile, keys) # dmag = stars' (true magnitude - 'observed') magnitude, average #patchdat['dmag'] = -1*patchdat['dmag'] return patchdat
def read_true_patchfile(patchfile='patchdata.dat'): """Read the patch file information from simSelfCalib.py""" keys = ('visitid', 'patchid', 'subpatch', 'nstars', 'ra', 'dec', 'magtrue', 'color', 'dmag', 'dmag_rms', 'dmag_rand', 'dmag_rand_rms', 'dmag_zp', 'dmag_zp_rms', 'dmag_color', 'dmag_color_rms') patchdat = ui.readDatafile(patchfile, keys) # dmag = stars' (true magnitude - 'observed') magnitude, average #patchdat['dmag'] = -1*patchdat['dmag'] return patchdat
def finalStar_bad(): """apply the healpix zeropoints to each block and collect all the bestfit star mags. I need to check that the multiple solutions per star are reasonable... is this an FpPatch issue?""" stardat = {} keys = ('starid', 'starmag') for key in keys: stardat[key] = [] healzp = ui.readDatafile('patch_bestfit_Patch.dat', ('healid', 'healmag'), keytypes=('u8', 'float')) for i in np.arange(np.size(healzp['healid'])): temp = ui.readDatafile('h%i_bestfit_Star.dat' % healzp['healid'][i], keys, keytypes=('u8', 'float')) fp = ui.readDatafile('h%i_bestfit_FpPatch.dat' % healzp['healid'][i], ('id', 'mag')) temp['starmag'] = temp['starmag'] - healzp['healmag'][i] - fp['mag'] stardat['starid'].append(np.ravel(temp['starid'])) stardat['starmag'].append(np.ravel(temp['starmag'])) for key in list(stardat.keys()): stardat[key] = np.concatenate(stardat[key]) #Crap, looks like there's more jitter solution-to-solution than I expected. might need to go back through the star_obs.dat file after all #convert stardat from dict of np arrays to a numpy rec array #sd = np.array(zip(stardat['starid'], stardat['starmag']), dtype=[('starid', 'u8'), ('starmag','float')] ) #sd = np.core.records.fromarrays(stardat['starid'], stardat['starmag'],names='starid,starmag' ) #sd.sort(order='starid') order = np.argsort(stardat['starid']) for key in list(stardat.keys()): stardat[key] = stardat[key][order] uid = np.unique(stardat['starid']) left = np.searchsorted(stardat['starid'], uid) right = np.searchsorted(stardat['starid'], uid, side='right') mags = np.zeros(left.size, dtype='float') stds = mags.copy() for i in np.arange(left.size): mags[i] = stardat['starmag'][left[i]:right[i]].mean() stds[i] = stardat['starmag'][left[i]:right[i]].std() file = open('test_bestfit_Star.dat', 'w') for i in np.arange(uid.size): print("%i %f" % (uid[i], mags[i]), file=file) file.close()
def finalStar(infile='star_obs.dat', outfile='test_bestfit_Star.dat'): """read in the best fit patches, and the raw observations. Apply patch zps to all the star magnitudes, then take a weighted mean of each star to make a bestfit star magnitude. Also calc an stdev for each star""" patchzp = ui.readDatafile('test_bestfit_Patch.dat', ('patchid', 'patchzp'), keytypes=('u8', 'float')) star_obs = ui.readDatafile(infile, ('patchid', 'starid', 'starmag', 'starerr'), keytypes=('u8', 'u8', 'float', 'float')) order = np.argsort(star_obs['patchid']) for key in list(star_obs.keys()): star_obs[key] = star_obs[key][order] left = np.searchsorted(star_obs['patchid'], patchzp['patchid']) right = np.searchsorted(star_obs['patchid'], patchzp['patchid'], side='right') for i in np.arange(left.size): star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][ left[i]:right[i]] - patchzp['patchzp'][i] order = np.argsort(star_obs['starid']) for key in list(star_obs.keys()): star_obs[key] = star_obs[key][order] ustar = np.unique(star_obs['starid']) mag = np.zeros(ustar.size, dtype='float') magerr = np.zeros(ustar.size, dtype='float') magstd_raw = np.zeros(ustar.size, dtype='float') left = np.searchsorted(star_obs['starid'], ustar) right = np.searchsorted(star_obs['starid'], ustar, side='right') for i in np.arange(left.size): mag[i] = np.sum(star_obs['starmag'][left[i]:right[i]] / star_obs['starerr'][left[i]:right[i]]**2) / np.sum( 1. / star_obs['starerr'][left[i]:right[i]]**2) magerr[i] = 1. / np.sqrt( np.sum(1. / star_obs['starerr'][left[i]:right[i]]**2)) magstd_raw[i] = star_obs['starmag'][left[i]:right[i]].std() file = open(outfile, 'w') print('#StarID Mag Err STDEV', file=file) for i in np.arange(ustar.size): print("%i %f %f %f" % (ustar[i], mag[i], magerr[i], magstd_raw[i]), file=file) file.close()
def finalStar_bad(): """apply the healpix zeropoints to each block and collect all the bestfit star mags. I need to check that the multiple solutions per star are reasonable... is this an FpPatch issue?""" stardat={} keys = ('starid','starmag') for key in keys: stardat[key] = [] healzp = ui.readDatafile('patch_bestfit_Patch.dat',('healid','healmag'), keytypes=('u8','float')) for i in np.arange(np.size(healzp['healid'])): temp = ui.readDatafile('h%i_bestfit_Star.dat'%healzp['healid'][i], keys, keytypes=('u8','float')) fp = ui.readDatafile('h%i_bestfit_FpPatch.dat'%healzp['healid'][i], ('id','mag')) temp['starmag'] = temp['starmag'] - healzp['healmag'][i] - fp['mag'] stardat['starid'].append(np.ravel(temp['starid'])) stardat['starmag'].append(np.ravel(temp['starmag'])) for key in list(stardat.keys()): stardat[key] = np.concatenate(stardat[key]) #Crap, looks like there's more jitter solution-to-solution than I expected. might need to go back through the star_obs.dat file after all #convert stardat from dict of np arrays to a numpy rec array #sd = np.array(zip(stardat['starid'], stardat['starmag']), dtype=[('starid', 'u8'), ('starmag','float')] ) #sd = np.core.records.fromarrays(stardat['starid'], stardat['starmag'],names='starid,starmag' ) #sd.sort(order='starid') order = np.argsort(stardat['starid']) for key in list(stardat.keys()): stardat[key] = stardat[key][order] uid = np.unique(stardat['starid']) left = np.searchsorted(stardat['starid'],uid) right = np.searchsorted(stardat['starid'],uid, side='right') mags = np.zeros(left.size, dtype='float') stds = mags.copy() for i in np.arange(left.size): mags[i] = stardat['starmag'][left[i]:right[i]].mean() stds[i] = stardat['starmag'][left[i]:right[i]].std() file = open('test_bestfit_Star.dat', 'w') for i in np.arange(uid.size): print("%i %f"%(uid[i], mags[i]), file=file) file.close()
def patch_combine(): files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE, shell=True).stdout.read().split() npatch = np.size(files) patchdat = {} final_keys = ('patchid', 'patchmag', 'healid') for key in final_keys: patchdat[key] = [] keys = ('patchid', 'patchmag') for i in np.arange(npatch): temp = ui.readDatafile(files[i], keys, keytypes=('u8', 'float')) heal_num = np.float(files[i][1:-18]) patchdat['patchid'].append(np.ravel(temp['patchid'])) patchdat['healid'].append(np.ravel(temp['patchid'] * 0 + heal_num)) patchdat['patchmag'].append(np.ravel(temp['patchmag'])) for key in list(patchdat.keys()): patchdat[key] = np.concatenate(patchdat[key]) ord = np.argsort(patchdat['healid']) for key in list(patchdat.keys()): patchdat[key] = patchdat[key][ord] file = open('patch_obs.dat', 'w') print("%s %s %s %s %s" % ("#HealID", "PatchID", "PatchMag", "PatchMagErr", "FpPatchID"), file=file) for obj in range(0, len(patchdat['patchid'])): print("%d %d %f %f %d" % (patchdat['healid'][obj], patchdat['patchid'][obj], patchdat['patchmag'][obj], 0.001, 0), file=file) file.close()
def illum_combine(): """combine the patch AND illumination corrections """ files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE, shell=True).stdout.read().split() #illums = sb.Popen('ls h*bestfit_FpPatch.dat', stdout=sb.PIPE, shell=True).stdout.read().split() #so_files = sb.Popen('ls h*bestfit_FpPatch.dat', stdout=sb.PIPE, shell=True).stdout.read().split() npatch = np.size(files) patchdat = {} final_keys = ('patchid', 'patchmag', 'patchmagerr', 'illumid', 'healid') illum_keys = ('illumid', 'illummag') so_keys = ('patchid', 'starid', 'starmag', 'starmagerr', 'illumid', 'hpid') for key in final_keys: patchdat[key] = [] keys = ('patchid', 'patchmag') for i in np.arange(npatch): temp = ui.readDatafile(files[i], keys, keytypes=('u8', 'float')) heal_num = np.int(files[i][1:-18]) temp_so = ui.readDatafile('h%u' % heal_num + 'star_obs.dat', so_keys, keytypes=('u8', 'u8', 'float', 'float', 'u8', 'u8')) temp_illum = ui.readDatafile('h%u' % heal_num + '_bestfit_FpPatch.dat', illum_keys, keytypes=('u8', 'float')) #now I need to construct a line for each unique patchid+illumid pair temp_so['patch_plus_illum'] = np.core.defchararray.add( temp_so['patchid'].astype('|S10'), np.core.defchararray.add( np.core.defchararray.replace( np.zeros(np.size(temp_so['patchid']), dtype='|S1'), '', ','), temp_so['illumid'].astype('|S10'))) temp_so = dictsort(temp_so, 'patch_plus_illum') upatch_plus_illum, unique_index = np.unique( temp_so['patch_plus_illum'], return_index=True) left = np.searchsorted(temp_so['patch_plus_illum'], upatch_plus_illum) right = np.searchsorted(temp_so['patch_plus_illum'], upatch_plus_illum, side='right') for key in so_keys: temp_so[key] = temp_so[key][unique_index] temp_so['nstars'] = right - left temp_so = np.core.records.fromarrays([ temp_so['patchid'], temp_so['illumid'], temp_so['patchid'] * 0 + heal_num, temp_so['starmag'] * 0, 1. / np.sqrt(temp_so['nstars']), temp_so['starmag'] * 0 ], names=('patchid', 'illumid', 'healid', 'patchmag', 'patchmagerr', 'illummag')) ord = np.argsort(temp['patchid']) for key in keys: temp[key] = temp[key][ord] temp_so.sort(order='patchid') left = np.searchsorted(temp_so['patchid'], temp['patchid']) right = np.searchsorted(temp_so['patchid'], temp['patchid'], side='right') for i in np.arange(np.size(left)): temp_so['patchmag'][left[i]:right[i]] = temp['patchmag'][i] temp_so.sort(order='illumid') ord = np.argsort(temp_illum['illumid']) for key in illum_keys: temp_illum[key] = temp_illum[key][ord] left = np.searchsorted(temp_so['illumid'], temp_illum['illumid']) right = np.searchsorted(temp_so['illumid'], temp_illum['illumid'], side='right') for i in np.arange(np.size(left)): temp_so['patchmag'][left[i]:right[i]] = temp_so['patchmag'][ left[i]:right[i]] + temp_illum['illummag'][i] patchdat['patchid'].append(np.ravel(temp_so['patchid'])) patchdat['healid'].append(np.ravel(temp_so['healid'])) patchdat['patchmag'].append(np.ravel(temp_so['patchmag'])) patchdat['illumid'].append(np.ravel(temp_so['illumid'])) patchdat['patchmagerr'].append(np.ravel(temp_so['patchmagerr'])) for key in list(patchdat.keys()): patchdat[key] = np.concatenate(patchdat[key]) ord = np.argsort(patchdat['healid']) for key in list(patchdat.keys()): patchdat[key] = patchdat[key][ord] file = open('patch_obs.dat', 'w') print("%s %s %s %s %s" % ("#HealID", "PatchID", "PatchMag", "PatchMagErr", "FpPatchID"), file=file) for obj in range(0, len(patchdat['patchid'])): print("%d %d %f %f %d" % (patchdat['healid'][obj], patchdat['patchid'][obj], patchdat['patchmag'][obj], patchdat['patchmagerr'][obj], patchdat['illumid'][obj]), file=file) #print >>file, "%d %d %f %f %d"%(patchdat['healid'][obj],patchdat['patchid'][obj], patchdat['patchmag'][obj], .01, patchdat['illumid'][obj]) file.close()
def finalPatch(solver): files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE, shell=True).stdout.read().split() #run the solver sb.Popen('%s patch_obs.dat patch 1e-12 40000 > /dev/null' % solver, shell=True).wait() sb.Popen('cp patch_bestfit_FpPatch.dat test_bestfit_FpPatch.dat', shell=True).wait() npatch = np.size(files) patchdat = {} final_keys = ('patchid', 'patchmag', 'healid') for key in final_keys: patchdat[key] = [] keys = ('patchid', 'patchmag') for i in np.arange(npatch): temp = ui.readDatafile(files[i], keys, keytypes=('u8', 'float')) heal_num = np.float(files[i][1:-18]) #insert if statement to deal with stupid patches that are not continuous patchdat['patchid'].append(np.ravel(temp['patchid'])) patchdat['healid'].append(np.ravel(temp['patchid'] * 0 + heal_num)) patchdat['patchmag'].append(np.ravel(temp['patchmag'])) for key in list(patchdat.keys()): patchdat[key] = np.concatenate(patchdat[key]) ord = np.argsort(patchdat['healid']) for key in list(patchdat.keys()): patchdat[key] = patchdat[key][ord] keys = ('hpid', 'hpzp') healdat = ui.readDatafile('patch_bestfit_Patch.dat', keys, keytypes=('u8', 'float')) keys = ('patchid', 'patchmag') fit_patchdat = ui.readDatafile('patch_bestfit_Star.dat', keys, keytypes=('u8', 'float')) left = np.searchsorted(patchdat['healid'], healdat['hpid']) right = np.searchsorted(patchdat['healid'], healdat['hpid'], side='right') for i in np.arange(np.size(right)): patchdat['patchmag'][left[i]:right[i]] = patchdat['patchmag'][ left[i]:right[i]] - healdat['hpzp'][i] patchid = np.unique(patchdat['patchid']) patchmag = patchid * 0 ord = np.argsort(patchdat['patchid']) patchdat['patchmag'] = patchdat['patchmag'][ord] patchdat['patchid'] = patchdat['patchid'][ord] left = np.searchsorted(patchdat['patchid'], patchid) patchmag = patchdat['patchmag'][left] #now plug in the best fit mags where they exist left = np.searchsorted(patchid, fit_patchdat['patchid']) patchmag[left] = fit_patchdat['patchmag'] file = open('test_bestfit_Patch.dat', 'w') for i in np.arange(np.size(patchmag)): print("%d %f" % (patchid[i], patchmag[i]), file=file) file.close()
def read_bestfit_stars(starcalfile='test_bestfit_Star.dat'): """Read a bestfit_Star.dat file from simple.x""" keys = ('id', 'magcal') starcal = ui.readDatafile(starcalfile, keys) return starcal
def read_bestfit_patchfile(bestfitpatchfile='test_bestfit_Patch.dat'): """Read the best fit patch file information from simple.x""" keys = ('patchid', 'dmagcal') patchcal = ui.readDatafile(bestfitpatchfile, keys) # dmagcal = zeropoint calculated by calsim return patchcal
def read_visitfile(visitfile='visit.dat'): """Read the visit information from simSelfCalib.py""" keys = ('visitid', 'ra', 'dec', 'skyrot', 'time', 'zpOff', 'zpGrad', 'zpGradDir', 'colorterm', 'colortermRad') visits = ui.readDatafile(visitfile, keys) return visits
def finalStarP(infiles='h*star_obs.dat', outfile='test_bestfit_Star.dat', outstats='stats_stars'): """ read in all the healpix files and apply patch zeropoints. Compute statistics for each star as well as final magnitude """ patchzp = ui.readDatafile('test_bestfit_Patch.dat', ('patchid', 'patchzp'), keytypes=('u8', 'float')) illumzp = ui.readDatafile('test_bestfit_FpPatch.dat', ('illumid', 'illumzp'), keytypes=('u8', 'float')) files = np.array( sb.Popen('ls %s' % infiles, stdout=sb.PIPE, shell=True).stdout.read().split()) hps = files.copy() for i in np.arange(np.size(hps)): hps[i] = hps[i].replace('h', '').replace('star_obs.dat', '') hps = hps.astype('int') #true_stars = ui.readData('stardata.dat', ('starid','stardbid','ra','dec','mag','color'), keytypes=('u8','u8','float','float','float','float')) results = {} result_keys = ('starid', 'nobs', 'mag', 'stdev', 'stdev_IQR') for key in result_keys: results[key] = [] for i in np.arange(np.size(files)): #read in file star_obs = ui.readDatafile( files[i], ('patchid', 'starid', 'starmag', 'starerr', 'fppatch', 'hp'), keytypes=('u8', 'u8', 'float', 'float', 'u8', 'u8')) in_hp = np.where( star_obs['hp'] == hps[i]) #crop down to just those stars in the center HP for key in list(star_obs.keys()): star_obs[key] = star_obs[key][in_hp] order = np.argsort(star_obs['patchid']) for key in list(star_obs.keys()): star_obs[key] = star_obs[key][order] left = np.searchsorted(star_obs['patchid'], patchzp['patchid']) right = np.searchsorted(star_obs['patchid'], patchzp['patchid'], side='right') for i in np.arange(left.size): #apply patch zeropoints star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][ left[i]:right[i]] - patchzp['patchzp'][i] order = np.argsort(star_obs['fppatch']) for key in list(star_obs.keys()): star_obs[key] = star_obs[key][order] left = np.searchsorted(star_obs['fppatch'], illumzp['illumid']) right = np.searchsorted(star_obs['fppatch'], illumzp['illumid'], side='right') for i in np.arange(left.size): star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][ left[i]:right[i]] - illumzp['illumzp'][i] order = np.argsort(star_obs['starid']) for key in list(star_obs.keys()): star_obs[key] = star_obs[key][order] ustar = np.unique(star_obs['starid']) left = np.searchsorted(star_obs['starid'], ustar) right = np.searchsorted(star_obs['starid'], ustar, side='right') temp_results = {} for key in result_keys: temp_results[key] = np.zeros(left.size) temp_results['starid'] = ustar.copy() for i in np.arange(left.size): temp_results['nobs'][i] = np.size( star_obs['starmag'][left[i]:right[i]]) temp_results['mag'][i] = np.sum( star_obs['starmag'][left[i]:right[i]] / star_obs['starerr'][left[i]:right[i]]**2) / np.sum( 1. / star_obs['starerr'][left[i]:right[i]]**2) temp_results['stdev'][i] = np.std( star_obs['starmag'][left[i]:right[i]]) temp_results['stdev_IQR'][i] = robustSigma( star_obs['starmag'][left[i]:right[i]]) for key in result_keys: results[key].append(temp_results[key].copy()) for key in result_keys: results[key] = np.concatenate(results[key]) ord = np.argsort(results['starid']) for key in result_keys: results[key] = results[key][ord] #output test_bestfit_Star w/ID, mag and then the rest. #I guess I could put the print inside the loop if I was worried about memory file = open('test_bestfit_Star.dat', 'w') for i in np.arange(np.size(results['starid'])): print('%d %f %f %f %d' % (results['starid'][i], results['mag'][i], results['stdev'][i], results['stdev_IQR'][i], results['nobs'][i]), file=file) file.close()
def illum_combine(): """combine the patch AND illumination corrections """ files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE, shell=True).stdout.read().split() #illums = sb.Popen('ls h*bestfit_FpPatch.dat', stdout=sb.PIPE, shell=True).stdout.read().split() #so_files = sb.Popen('ls h*bestfit_FpPatch.dat', stdout=sb.PIPE, shell=True).stdout.read().split() npatch=np.size(files) patchdat={} final_keys = ('patchid', 'patchmag', 'patchmagerr', 'illumid', 'healid') illum_keys = ('illumid','illummag') so_keys = ('patchid','starid','starmag','starmagerr','illumid','hpid') for key in final_keys: patchdat[key]=[] keys=('patchid','patchmag') for i in np.arange(npatch): temp = ui.readDatafile(files[i], keys, keytypes=('u8','float')) heal_num = np.int(files[i][1:-18]) temp_so = ui.readDatafile('h%u'%heal_num+'star_obs.dat', so_keys, keytypes=('u8','u8','float','float','u8','u8')) temp_illum = ui.readDatafile('h%u'%heal_num+'_bestfit_FpPatch.dat', illum_keys, keytypes=('u8','float')) #now I need to construct a line for each unique patchid+illumid pair temp_so['patch_plus_illum'] = np.core.defchararray.add(temp_so['patchid'].astype('|S10'),np.core.defchararray.add(np.core.defchararray.replace(np.zeros(np.size(temp_so['patchid']), dtype='|S1'), '',','), temp_so['illumid'].astype('|S10'))) temp_so = dictsort(temp_so,'patch_plus_illum') upatch_plus_illum, unique_index = np.unique(temp_so['patch_plus_illum'],return_index=True ) left = np.searchsorted(temp_so['patch_plus_illum'], upatch_plus_illum) right = np.searchsorted(temp_so['patch_plus_illum'], upatch_plus_illum, side='right') for key in so_keys: temp_so[key] = temp_so[key][unique_index] temp_so['nstars'] = right-left temp_so = np.core.records.fromarrays([temp_so['patchid'], temp_so['illumid'], temp_so['patchid']*0+heal_num, temp_so['starmag']*0,1./np.sqrt(temp_so['nstars']),temp_so['starmag']*0 ], names=('patchid','illumid','healid','patchmag', 'patchmagerr','illummag')) ord = np.argsort(temp['patchid']) for key in keys: temp[key] = temp[key][ord] temp_so.sort(order='patchid') left = np.searchsorted(temp_so['patchid'], temp['patchid']) right = np.searchsorted(temp_so['patchid'], temp['patchid'], side='right') for i in np.arange(np.size(left)): temp_so['patchmag'][left[i]:right[i]] = temp['patchmag'][i] temp_so.sort(order='illumid') ord = np.argsort(temp_illum['illumid']) for key in illum_keys: temp_illum[key] = temp_illum[key][ord] left = np.searchsorted(temp_so['illumid'], temp_illum['illumid']) right = np.searchsorted(temp_so['illumid'], temp_illum['illumid'], side='right') for i in np.arange(np.size(left)): temp_so['patchmag'][left[i]:right[i]] = temp_so['patchmag'][left[i]:right[i]] + temp_illum['illummag'][i] patchdat['patchid'].append(np.ravel(temp_so['patchid'])) patchdat['healid'].append(np.ravel(temp_so['healid'])) patchdat['patchmag'].append(np.ravel(temp_so['patchmag'])) patchdat['illumid'].append(np.ravel(temp_so['illumid'])) patchdat['patchmagerr'].append(np.ravel(temp_so['patchmagerr'])) for key in list(patchdat.keys()): patchdat[key] = np.concatenate(patchdat[key]) ord = np.argsort(patchdat['healid']) for key in list(patchdat.keys()): patchdat[key] = patchdat[key][ord] file = open('patch_obs.dat', 'w') print("%s %s %s %s %s" %("#HealID", "PatchID", "PatchMag", "PatchMagErr", "FpPatchID"), file=file) for obj in range(0,len(patchdat['patchid'])): print("%d %d %f %f %d"%(patchdat['healid'][obj],patchdat['patchid'][obj], patchdat['patchmag'][obj], patchdat['patchmagerr'][obj], patchdat['illumid'][obj]), file=file) #print >>file, "%d %d %f %f %d"%(patchdat['healid'][obj],patchdat['patchid'][obj], patchdat['patchmag'][obj], .01, patchdat['illumid'][obj]) file.close()
def read_bestfit_patchfile(bestfitpatchfile='test_bestfit_Patch.dat'): """Read the best fit patch file information from simple.x""" keys = ('patchid', 'dmagcal') patchcal = ui.readDatafile(bestfitpatchfile, keys) # dmagcal = zeropoint calculated by calsim return patchcal
def read_true_stars(stardatfile='stardata.dat'): """Read a stardata.dat file from simSelfCalib.py""" keys = ('id', 'dbid', 'ra', 'dec', 'magtrue', 'color') stardat = ui.readDatafile(stardatfile, keys) return stardat
def read_starsobs(starobsfile='star_obs.dat'): """Read the star_obs.dat file from simSelfCalib.py that is input to simple.x""" keys = ('patchid', 'starid', 'rmagobs', 'magerr') starobs = ui.readDatafile(starobsfile, keys) return starobs
def read_bestfit_stars(starcalfile='test_bestfit_Star.dat'): """Read a bestfit_Star.dat file from simple.x""" keys = ('id', 'magcal') starcal = ui.readDatafile(starcalfile, keys) return starcal
def starPlots(truestarfile = "stardata.dat", bestfitstarfile = "test_bestfit_Star.dat"): #assume both the files are sorted by star id. stardat = cp.read_true_stars(truestarfile) #keys are ('id', 'dbid', 'ra', 'dec', 'magtrue', 'color') starcal = ui.readDatafile(bestfitstarfile, keys=('id','magcal','rms','iqr_sigma','nobs')) HP = True if np.size(starcal['id']) == 0: starcal = ui.readDatafile(bestfitstarfile, keys=('id','magcal')) HP = False fitted_stars = np.in1d(stardat['id'], starcal['id']) for key in stardat.keys(): stardat[key] = stardat[key][fitted_stars] for key in starcal.keys(): stardat[key] = starcal[key] stardat['magdiff'] = stardat['magcal'] - stardat['magtrue'] stardat = cp.adjust_stars_calvstrue(stardat) starResults = {} HPResults = {} starResults['N stars Fit'] = np.size(stardat['magdiff']) outlier_clip=np.where(np.abs(stardat['magdiff']) < .05) rms = stardat['magdiff'][outlier_clip].std() print 'true-bestfit RMS = %f, true-bestfit (clipped) = %f'%(stardat['magdiff'].std(), rms) lim = rms*2.5 if lim < 0.001: lim = round(lim * 10000) / 10000.0 elif lim < 0.01: lim = round(lim * 1000) / 1000.0 print "Using magnitude limits for plots of %f, %f" %(-lim, lim) maglim= [-lim, lim] starResults['Residuals RMS (mmag)'] = stardat['magdiff'].std()*1e3 starResults['IQR_sigma (mmag)'] = so.robustSigma(stardat['magdiff'])*1e3 #star residual map stardat['x'], stardat['y'] = cpu.hammer_project_toxy(stardat['ra']*deg2rad, stardat['dec']*deg2rad) gridsize = cpu.calc_gridsize(stardat, binsize='patch') etitle = " RMS = %4.1f mmag"%(rms*1000) cpu.plot_density(stardat['x'], stardat['y'], stardat['magdiff']*1000, z_method=np.mean, zlimits=np.array(maglim)*1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag') figtitle = "Stars dMag(true-bestfit)" pyl.title(figtitle, fontsize='medium') pyl.savefig('Sdmag.png',type='png') maglim2 = [0, maglim[1]/2.] cp.plot_stars_dmagcaltrue(stardat, maglim=maglim2, etitle=" RMS "+etitle, z_method=np.std) pyl.savefig('Sdmag_rms.png', format='png') histrange = [maglim[0]*2.5, maglim[1]*2.5] nbins = 100 cp.histogram_stars_dmagcaltrue(stardat, nbins=nbins, range=histrange, etitle=etitle) pyl.savefig('Sdamg_hist.png', format='png') #color distribution of stars pyl.figure() pyl.hist(stardat['color'], bins=100) pyl.xlabel(r'$g-i$') pyl.ylabel('# of stars') pyl.title('Color Distribution') pyl.savefig('Scolordist.png',type='png') #accuracy v color pyl.figure() #pyl.scatter(stardat['color'], stardat['magdiff']*1000., c=stardat['magtrue'], alpha=0.1,edgecolors=None) pyl.hexbin(stardat['color'], stardat['magdiff']*1000., bins='log') cb=pyl.colorbar() cb.ax.set_ylabel(r'$\log{\rm{N}}$') pyl.xlabel(r'$g-i$') pyl.ylabel('Bestfit-True (mmag)') pyl.savefig('Saccuracyvcolor.png',type='png') #accuracy v mag pyl.figure() #pyl.scatter(stardat['magtrue'], stardat['magdiff']*1000., c=stardat['color'], alpha=0.1,edgecolors=None) pyl.hexbin(stardat['magtrue'], stardat['magdiff']*1000., bins='log') cb=pyl.colorbar() cb.ax.set_ylabel(r'$\log{\rm{N}}$') pyl.xlabel('True Mag') pyl.ylabel('Bestfit-True (mmag)') pyl.savefig('Saccuracyvmag.png',type='png') if HP: zlim = [np.min(stardat['nobs']), (np.median(stardat['nobs'])-np.min(stardat['nobs']))+np.median(stardat['nobs'])] cpu.plot_density(stardat['x'], stardat['y'],stardat['nobs'], z_method=np.mean, zlimits=zlim, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='N observations') pyl.title('Visit Density') pyl.savefig('Snobs.png', format='png') pyl.figure() #I should just figure out from the data what nsides is! hpid = hp.ang2pix(16, (stardat['dec']+90.)*deg2rad, stardat['ra']*deg2rad) cpu.plot_density(stardat['x'], stardat['y'],np.mod(hpid,250), z_method=np.median, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='HEALpix ID mod 250') pyl.title("HEALpix Map") pyl.savefig('HPid.png') HPResults['number of HEALpix'] = np.size(np.unique(hpid)) pyl.figure() num, b, p = pyl.hist(stardat['nobs'], bins=100, range=zlim, histtype='bar') pyl.xlabel("Number of Observations per Star") pyl.title('Median = %d'%np.median(stardat['nobs'])) pyl.savefig('Snobs_hist.png',type='png') starResults['median repeat obs'] = np.median(stardat['nobs']) pyl.figure() x = np.sort(stardat['iqr_sigma'])*1e3 y = np.arange(np.size(x), dtype='float') y = y/np.max(y) per50 = np.max(np.where(y <= .5)) per90 = np.max(np.where(y <= .9)) pyl.plot(x,y, 'k--', label='IQR, 50th,90th %2.1f, %2.1f'%(x[per50],x[per90])) pyl.plot([0,x[per50]],[y[per50],y[per50]],'b--') pyl.plot([x[per50],x[per50]],[0,y[per50]],'b--') pyl.plot([0,x[per90]],[y[per90],y[per90]],'b--') pyl.plot([x[per90],x[per90]],[0,y[per90]],'b--') #pyl.title('Cumulative Distribution of Stellar Repeatability') #pyl.xlabel('Repeatability IQR RMS (mmag)') #pyl.ylabel('Cumulative Fraction') #pyl.savefig('Srepeat_IQR_cumulative.png',type='png') pyl.legend() pyl.savefig('Srepeat_cumulative.png',type='png') #repeatability from IQR rs = so.robustSigma(stardat['iqr_sigma']) med = np.median(stardat['iqr_sigma']) maglim = np.around(np.array([med-3*rs, med+3.*rs])*1000, decimals=2)/1000 cpu.plot_density(stardat['x'], stardat['y'], stardat['iqr_sigma']*1000, z_method=np.mean, zlimits=maglim*1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag') pyl.title('Repeatability (IQR)') pyl.savefig('Srepeat_IQR.png',type='png') pyl.figure() num, b, p = pyl.hist(stardat['iqr_sigma']*1000, bins=100, range=maglim*1000, histtype='bar') pyl.xlabel('RMS per star (mmag)') pyl.title('Repeatability IQR, Median = %4.2f mmag'%np.median(stardat['iqr_sigma']*1000)) pyl.savefig('Srepeat_IQR_hist.png',type='png') starResults['Median Repeatability (IQR) (mmag)'] = np.median(med)*1e3 pyl.figure() good = np.where(stardat['magtrue'] < 18) num, b, p = pyl.hist(stardat['iqr_sigma'][good]*1000, bins=100, range=maglim*1000, histtype='bar') pyl.xlabel('RMS per star, m < 18 (mmag)') pyl.title('Repeatability IQR, Median = %4.2f mmag'%np.median(stardat['iqr_sigma'][good]*1000)) pyl.savefig('Srepeat_IQR_bright_hist.png',type='png') starResults['Median Repeatability Bright (IQR) (mmag)'] = np.median(stardat['iqr_sigma'][good])*1e3 #repeatability v color pyl.figure() #pyl.plot(stardat['color'], stardat['rms']*1000., 'ko', alpha=.1) pyl.hexbin(stardat['color'], stardat['iqr_sigma']*1000., bins='log') cb = pyl.colorbar() cb.set_label(r'$\log{\rm{N}}$') pyl.ylim([0,40]) pyl.xlabel(r'$g-i$') pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)') pyl.savefig('Srepeatvcolor.png',type='png') #repeatability v mag pyl.figure() #pyl.plot(stardat['magtrue'], stardat['rms']*1000., 'ko', alpha=.1) pyl.hexbin(stardat['magtrue'], stardat['iqr_sigma']*1000.,bins='log') cb = pyl.colorbar() cb.set_label(r'$\log{\rm{N}}$') pyl.ylim([0,40]) pyl.xlabel('True Mag') pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)') pyl.savefig('Srepeatvmag.png',type='png') #repeatability rs = so.robustSigma(stardat['rms']) med = np.median(stardat['rms']) maglim = np.around(np.array([med-3*rs, med+3.*rs])*1000, decimals=2)/1000 cpu.plot_density(stardat['x'], stardat['y'], stardat['rms']*1000, z_method=np.mean, zlimits=maglim*1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag') pyl.title('Repeatability') pyl.savefig('Srepeat.png',type='png') pyl.figure() num, b, p = pyl.hist(stardat['rms']*1000, bins=100, range=maglim*1000, histtype='bar') pyl.xlabel('RMS per star (mmag)') pyl.title('Repeatability, Median = %4.2f mmag'%np.median(stardat['rms']*1000)) pyl.savefig('Srepeat_hist.png',type='png') starResults['Median Repeatability (mmag)'] = np.median(med)*1e3 pyl.figure() x = np.sort(stardat['rms'])*1e3 y = np.arange(np.size(x), dtype='float') y = y/np.max(y) per50 = np.max(np.where(y <= .5)) per90 = np.max(np.where(y <= .9)) pyl.plot(x,y, 'k', label='RMS, %2.1f, %2.1f'%(x[per50],x[per90])) pyl.plot([0,x[per50]],[y[per50],y[per50]],'k') pyl.plot([x[per50],x[per50]],[0,y[per50]],'k') pyl.plot([0,x[per90]],[y[per90],y[per90]],'k') pyl.plot([x[per90],x[per90]],[0,y[per90]],'k') pyl.title('Cumulative Distribution of Stellar Repeatability') pyl.xlabel('Repeatability RMS (mmag)') pyl.ylabel('Cumulative Fraction') pyl.savefig('Srepeat_cumulative.png',type='png') #need to make a spatial uniformity plot since we can now have varying densities of stars. nside = 64 stardat['hpid'] = hp.ang2pix(nside, (stardat['dec']+90.)*deg2rad, stardat['ra']*deg2rad) uhpid = np.unique(stardat['hpid']) hp_mean = uhpid*0. hp_std = uhpid*0. hp_dec, hp_ra = hp.pix2ang(nside,uhpid) #hp_ra = hp_ra hp_dec = hp_dec-90.*deg2rad hp_x, hp_y = cpu.hammer_project_toxy(hp_ra, hp_dec) stardat = dictsort(stardat,'hpid') left = np.searchsorted(stardat['hpid'], uhpid) right = np.searchsorted(stardat['hpid'], uhpid,side='right') for i in np.arange(left.size): hp_mean[i] = np.mean(stardat['magdiff'][left[i]:right[i]]) hp_std[i] = np.std(stardat['magdiff'][left[i]:right[i]]) cpu.plot_density(hp_x, hp_y, hp_mean*1000, z_method=np.mean, zlimits=None, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag') HPResults['Spatial Uniformity RMS (mmag)'] = np.std(hp_mean)*1e3 HPResults['Spatial Uniformity IQR RMS (mmag)'] = so.robustSigma(hp_mean)*1e3 pyl.title('mean(True-Bestfit), binned by HEALpix') pyl.savefig('HPresid.png',type='png') cpu.plot_density(hp_x, hp_y, hp_std*1000, z_method=np.mean, zlimits=None, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag') pyl.title('std(True-Bestfit), binned by HEALpix') pyl.savefig('HPstd.png',type='png') pyl.figure() num, b, p = pyl.hist( hp_mean*1000, bins=100, range=None, histtype='bar') pyl.xlabel("HEALpix(True-Bestfit) (mmag)") pyl.title('Spatial Uniformity, RMS=%4.2f'%HPResults['Spatial Uniformity RMS (mmag)']) pyl.savefig('HPmean_hist.png',type='png') pyl.figure() num, b, p = pyl.hist( hp_std*1000, bins=100, range=None, histtype='bar') pyl.xlabel("std(True-Bestfit) per HEALpix (mmag)") pyl.title('Variation within HEALpix, median=%4.2f mmag'%np.median(hp_std*1000)) pyl.savefig('HPstd_hist.png',type='png') return starResults,HPResults
def finalStarP(infiles='h*star_obs.dat', outfile='test_bestfit_Star.dat', outstats='stats_stars'): """ read in all the healpix files and apply patch zeropoints. Compute statistics for each star as well as final magnitude """ patchzp = ui.readDatafile('test_bestfit_Patch.dat', ('patchid','patchzp'), keytypes=('u8','float') ) illumzp = ui.readDatafile('test_bestfit_FpPatch.dat', ('illumid', 'illumzp'), keytypes=('u8','float')) files = np.array(sb.Popen('ls %s'%infiles, stdout=sb.PIPE, shell=True).stdout.read().split()) hps = files.copy() for i in np.arange(np.size(hps)): hps[i]=hps[i].replace('h','').replace('star_obs.dat','') hps=hps.astype('int') #true_stars = ui.readData('stardata.dat', ('starid','stardbid','ra','dec','mag','color'), keytypes=('u8','u8','float','float','float','float')) results = {} result_keys = ('starid','nobs','mag','stdev','stdev_IQR') for key in result_keys: results[key]=[] for i in np.arange(np.size(files)): #read in file star_obs = ui.readDatafile(files[i], ('patchid','starid','starmag','starerr','fppatch','hp'), keytypes=('u8','u8','float','float','u8','u8')) in_hp = np.where(star_obs['hp'] == hps[i])#crop down to just those stars in the center HP for key in list(star_obs.keys()): star_obs[key] = star_obs[key][in_hp] order = np.argsort(star_obs['patchid']) for key in list(star_obs.keys()): star_obs[key] = star_obs[key][order] left = np.searchsorted(star_obs['patchid'], patchzp['patchid']) right = np.searchsorted(star_obs['patchid'], patchzp['patchid'], side='right') for i in np.arange(left.size): #apply patch zeropoints star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][left[i]:right[i]] - patchzp['patchzp'][i] order = np.argsort(star_obs['fppatch']) for key in list(star_obs.keys()): star_obs[key] = star_obs[key][order] left = np.searchsorted(star_obs['fppatch'], illumzp['illumid']) right = np.searchsorted(star_obs['fppatch'], illumzp['illumid'], side='right') for i in np.arange(left.size): star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][left[i]:right[i]] - illumzp['illumzp'][i] order = np.argsort(star_obs['starid']) for key in list(star_obs.keys()): star_obs[key] = star_obs[key][order] ustar = np.unique(star_obs['starid']) left = np.searchsorted( star_obs['starid'], ustar) right = np.searchsorted(star_obs['starid'], ustar,side='right') temp_results = {} for key in result_keys: temp_results[key] = np.zeros(left.size) temp_results['starid'] = ustar.copy() for i in np.arange(left.size): temp_results['nobs'][i] = np.size(star_obs['starmag'][left[i]:right[i]]) temp_results['mag'][i] = np.sum(star_obs['starmag'][left[i]:right[i]]/star_obs['starerr'][left[i]:right[i]]**2)/np.sum(1./star_obs['starerr'][left[i]:right[i]]**2) temp_results['stdev'][i] = np.std(star_obs['starmag'][left[i]:right[i]]) temp_results['stdev_IQR'][i] = robustSigma(star_obs['starmag'][left[i]:right[i]]) for key in result_keys: results[key].append(temp_results[key].copy()) for key in result_keys: results[key] = np.concatenate(results[key]) ord = np.argsort(results['starid']) for key in result_keys: results[key] = results[key][ord] #output test_bestfit_Star w/ID, mag and then the rest. #I guess I could put the print inside the loop if I was worried about memory file = open('test_bestfit_Star.dat','w') for i in np.arange(np.size(results['starid'])): print('%d %f %f %f %d'%(results['starid'][i], results['mag'][i], results['stdev'][i], results['stdev_IQR'][i],results['nobs'][i]), file=file) file.close()
def split_obs(nside,shift_size=5., splitNS=False, NorthOnly=False, SouthOnly=False, blocksize=1e6): """ XXX kill this function XXX replaced by just including neighboring healpixels""" """nside gives the number of sides for healpixels. shift_size is in degrees and sets how """ #XXX-prob should put in a check so I don't sort repeatedly #if not os.path.isfile('star_obs.dat.s'): waitpop('sort -nk 1 star_obs.dat > star_obs.dat.s') npix = hp.nside2npix(nside) keys = ('visitid', 'patchid', 'subpatch', 'nstars', 'ra', 'dec') patchfile='patchdata.dat' patchdat = ui.readDatafile(patchfile, keys) ord = np.argsort(patchdat['patchid']) for key in list(patchdat.keys()): patchdat[key]=patchdat[key][ord] shift_size = shift_size*np.pi/180. #1.*np.pi/180. #compute galactic coords for healpixels and patches heal={} heal['ra'], heal['dec'] = hp.pix2ang(nside, np.arange(npix)) heal['dec'] = heal['dec']-90.*np.pi/180. heal['b'] = galac_b(heal['ra'], heal['dec']) #np.arcsin(np.sin(heal['dec'])*np.cos(62.6*np.pi/180.)-np.cos(heal['dec'])*np.sin(heal['ra']-282.25*np.pi/180.)*np.sin(62.6*np.pi/180.)) patchdat['dec'] = patchdat['dec']*np.pi/180. patchdat['ra'] = patchdat['ra']*np.pi/180. patchdat['b'] = galac_b(patchdat['ra'], patchdat['dec']) patchdat['l'] = galac_l(patchdat['ra'], patchdat['dec']) patchdat['dec']=patchdat['dec']+90.*np.pi/180. #stupid healpix wants 0-180 I think... patch_healpix = hp.ang2pix(nside, patchdat['dec'], patchdat['ra']) pyl.figure() cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix) pyl.savefig('heal_check.png', type='png') patch_healpix_shifted1 = hp.ang2pix(nside, patchdat['dec']+shift_size, patchdat['ra']+shift_size*2) patch_healpix_shifted2 = hp.ang2pix(nside, patchdat['dec']-shift_size, patchdat['ra']+shift_size*2) patch_healpix_shifted3 = hp.ang2pix(nside, patchdat['dec']+shift_size, patchdat['ra']-shift_size*2) patch_healpix_shifted4 = hp.ang2pix(nside, patchdat['dec']-shift_size, patchdat['ra']-shift_size*2) pyl.figure() cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix_shifted1) pyl.savefig('heal_check1.png', type='png') #ok, just go through and set duplicates to -1. Also set any of the shifted ones that are not in patch_healpix to -1 uhp = np.unique(patch_healpix) patch_healpix_shifted1[np.invert(np.in1d(patch_healpix_shifted1, uhp))] = -1 patch_healpix_shifted2[np.invert(np.in1d(patch_healpix_shifted2, uhp))] = -1 patch_healpix_shifted3[np.invert(np.in1d(patch_healpix_shifted3, uhp))] = -1 patch_healpix_shifted4[np.invert(np.in1d(patch_healpix_shifted4, uhp))] = -1 diff = patch_healpix_shifted1-patch_healpix good = np.where(diff == 0) patch_healpix_shifted1[good] = -1 diff1 = patch_healpix_shifted2-patch_healpix diff2 = patch_healpix_shifted2-patch_healpix_shifted1 good = np.where( (diff1 == 0) | (diff2 == 0)) patch_healpix_shifted2[good] = -1 diff1 = patch_healpix_shifted3-patch_healpix diff2 = patch_healpix_shifted3-patch_healpix_shifted1 diff3 = patch_healpix_shifted3-patch_healpix_shifted2 good = np.where( (diff1 == 0) | (diff2 == 0) | (diff3 == 0)) patch_healpix_shifted3[good] = -1 diff1 = patch_healpix_shifted4-patch_healpix diff2 = patch_healpix_shifted4-patch_healpix_shifted1 diff3 = patch_healpix_shifted4-patch_healpix_shifted2 diff4 = patch_healpix_shifted4-patch_healpix_shifted3 good = np.where( (diff1 == 0) | (diff2 == 0) | (diff3 == 0) | (diff4 == 0)) patch_healpix_shifted4[good] = -1 pyl.figure() cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix_shifted1) pyl.savefig('heal_check1_post.png', type='png') heal_pixels = np.unique(patch_healpix) keys = [patch_healpix, patch_healpix_shifted1, patch_healpix_shifted2,patch_healpix_shifted3,patch_healpix_shifted4] filenames = ['hp1.dat','hp2.dat','hp3.dat','hp4.dat','hp5.dat'] #ok, if a healpix stradles b =0, I want to split it in two # I could also just loop through each HP. if the HP has patches w/ b> 0 and b< 0 and min l < xxx or max l < xxx, then break it apart. if splitNS: break_up = np.where( (patchdat['b'] < 0) & ( (patchdat['l'] < 120.*np.pi/180.) | (patchdat['l'] > 240)*np.pi/180.) & (np.abs(patchdat['b']) < 5*np.pi/180.)) for key in keys: mask=np.where(key == -1) key[break_up] = key[break_up]+max(heal_pixels)+1 key[mask] = -1 print('reading star_obs.dat.s') star_obs = open('star_obs.dat.s', 'r') #read in just the patch id's from star_obs all_patch_ids = [] for line in star_obs: if line.startswith('#'): continue linevalues = line.split() all_patch_ids.append(linevalues[0]) all_patch_ids = np.array(all_patch_ids, dtype=np.int) star_obs.close() for j in np.arange(4): filename = filenames[j] patch_healpix = keys[j] hpid = all_patch_ids*0 left = np.searchsorted(all_patch_ids, patchdat['patchid']) right = np.searchsorted(all_patch_ids, patchdat['patchid'], side='right') for i in np.arange(np.size(left)): hpid[left[i]:right[i]] = patch_healpix[i] print('left %i, hpid %i, all_patch_ids %i, patch_healpix %i '%(np.size(left), np.size(hpid), np.size(all_patch_ids), np.size( patch_healpix))) file = open(filename,'w') print("#", file=file) for value in hpid: print(value, file=file) file.close() #print j, np.size(hpid), np.size(all_patch_ids) waitpop('paste hp1.dat star_obs.dat.s > hp1_star_obs.dat') waitpop('paste hp2.dat star_obs.dat.s > hp2_star_obs.dat') waitpop("sed -i '/^-1/d' hp2_star_obs.dat" ) waitpop('paste hp3.dat star_obs.dat.s > hp3_star_obs.dat') waitpop("sed -i '/^-1/d' hp3_star_obs.dat" ) waitpop('paste hp4.dat star_obs.dat.s > hp4_star_obs.dat') waitpop("sed -i '/^-1/d' hp4_star_obs.dat" ) waitpop('paste hp5.dat star_obs.dat.s > hp5_star_obs.dat') waitpop("sed -i '/^-1/d' hp5_star_obs.dat" ) #waitpop("mv temp.dat hp2_star_obs.dat") waitpop('cat hp2_star_obs.dat >> hp1_star_obs.dat') waitpop('cat hp3_star_obs.dat >> hp1_star_obs.dat') waitpop('cat hp4_star_obs.dat >> hp1_star_obs.dat') waitpop('cat hp5_star_obs.dat >> hp1_star_obs.dat') waitpop(''' awk '{print $2" "$3" "$4" "$5 " "$6> ("h"$1"star_obs.dat")}' hp1_star_obs.dat''') waitpop('rm h#star_obs.dat') waitpop('mv hp1_star_obs.dat star_obs_hp1.dat') waitpop('mv hp2_star_obs.dat star_obs_hp2.dat') waitpop('mv hp3_star_obs.dat star_obs_hp3.dat') waitpop('mv hp4_star_obs.dat star_obs_hp4.dat') waitpop('mv hp5_star_obs.dat star_obs_hp5.dat')
def starPlots(truestarfile="stardata.dat", bestfitstarfile="test_bestfit_Star.dat"): #assume both the files are sorted by star id. stardat = cp.read_true_stars( truestarfile ) #keys are ('id', 'dbid', 'ra', 'dec', 'magtrue', 'color') starcal = ui.readDatafile(bestfitstarfile, keys=('id', 'magcal', 'rms', 'iqr_sigma', 'nobs')) HP = True if np.size(starcal['id']) == 0: starcal = ui.readDatafile(bestfitstarfile, keys=('id', 'magcal')) HP = False fitted_stars = np.in1d(stardat['id'], starcal['id']) for key in list(stardat.keys()): stardat[key] = stardat[key][fitted_stars] for key in list(starcal.keys()): stardat[key] = starcal[key] stardat['magdiff'] = stardat['magcal'] - stardat['magtrue'] stardat = cp.adjust_stars_calvstrue(stardat) starResults = {} HPResults = {} starResults['N stars Fit'] = np.size(stardat['magdiff']) outlier_clip = np.where(np.abs(stardat['magdiff']) < .05) rms = stardat['magdiff'][outlier_clip].std() print('true-bestfit RMS = %f, true-bestfit (clipped) = %f' % (stardat['magdiff'].std(), rms)) lim = rms * 2.5 if lim < 0.001: lim = round(lim * 10000) / 10000.0 elif lim < 0.01: lim = round(lim * 1000) / 1000.0 print("Using magnitude limits for plots of %f, %f" % (-lim, lim)) maglim = [-lim, lim] starResults['Residuals RMS (mmag)'] = stardat['magdiff'].std() * 1e3 starResults['IQR_sigma (mmag)'] = so.robustSigma(stardat['magdiff']) * 1e3 #star residual map stardat['x'], stardat['y'] = cpu.hammer_project_toxy( stardat['ra'] * deg2rad, stardat['dec'] * deg2rad) gridsize = cpu.calc_gridsize(stardat, binsize='patch') etitle = " RMS = %4.1f mmag" % (rms * 1000) cpu.plot_density(stardat['x'], stardat['y'], stardat['magdiff'] * 1000, z_method=np.mean, zlimits=np.array(maglim) * 1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag') figtitle = "Stars dMag(true-bestfit)" pyl.title(figtitle, fontsize='medium') pyl.savefig('Sdmag.png', type='png') maglim2 = [0, maglim[1] / 2.] cp.plot_stars_dmagcaltrue(stardat, maglim=maglim2, etitle=" RMS " + etitle, z_method=np.std) pyl.savefig('Sdmag_rms.png', format='png') histrange = [maglim[0] * 2.5, maglim[1] * 2.5] nbins = 100 cp.histogram_stars_dmagcaltrue(stardat, nbins=nbins, range=histrange, etitle=etitle) pyl.savefig('Sdamg_hist.png', format='png') #color distribution of stars pyl.figure() pyl.hist(stardat['color'], bins=100) pyl.xlabel(r'$g-i$') pyl.ylabel('# of stars') pyl.title('Color Distribution') pyl.savefig('Scolordist.png', type='png') #accuracy v color pyl.figure() #pyl.scatter(stardat['color'], stardat['magdiff']*1000., c=stardat['magtrue'], alpha=0.1,edgecolors=None) pyl.hexbin(stardat['color'], stardat['magdiff'] * 1000., bins='log') cb = pyl.colorbar() cb.ax.set_ylabel(r'$\log{\rm{N}}$') pyl.xlabel(r'$g-i$') pyl.ylabel('Bestfit-True (mmag)') pyl.savefig('Saccuracyvcolor.png', type='png') #accuracy v mag pyl.figure() #pyl.scatter(stardat['magtrue'], stardat['magdiff']*1000., c=stardat['color'], alpha=0.1,edgecolors=None) pyl.hexbin(stardat['magtrue'], stardat['magdiff'] * 1000., bins='log') cb = pyl.colorbar() cb.ax.set_ylabel(r'$\log{\rm{N}}$') pyl.xlabel('True Mag') pyl.ylabel('Bestfit-True (mmag)') pyl.savefig('Saccuracyvmag.png', type='png') if HP: zlim = [ np.min(stardat['nobs']), (np.median(stardat['nobs']) - np.min(stardat['nobs'])) + np.median(stardat['nobs']) ] cpu.plot_density(stardat['x'], stardat['y'], stardat['nobs'], z_method=np.mean, zlimits=zlim, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='N observations') pyl.title('Visit Density') pyl.savefig('Snobs.png', format='png') pyl.figure() #I should just figure out from the data what nsides is! hpid = hp.ang2pix(16, (stardat['dec'] + 90.) * deg2rad, stardat['ra'] * deg2rad) cpu.plot_density(stardat['x'], stardat['y'], np.mod(hpid, 250), z_method=np.median, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='HEALpix ID mod 250') pyl.title("HEALpix Map") pyl.savefig('HPid.png') HPResults['number of HEALpix'] = np.size(np.unique(hpid)) pyl.figure() num, b, p = pyl.hist(stardat['nobs'], bins=100, range=zlim, histtype='bar') pyl.xlabel("Number of Observations per Star") pyl.title('Median = %d' % np.median(stardat['nobs'])) pyl.savefig('Snobs_hist.png', type='png') starResults['median repeat obs'] = np.median(stardat['nobs']) pyl.figure() x = np.sort(stardat['iqr_sigma']) * 1e3 y = np.arange(np.size(x), dtype='float') y = y / np.max(y) per50 = np.max(np.where(y <= .5)) per90 = np.max(np.where(y <= .9)) pyl.plot(x, y, 'k--', label='IQR, 50th,90th %2.1f, %2.1f' % (x[per50], x[per90])) pyl.plot([0, x[per50]], [y[per50], y[per50]], 'b--') pyl.plot([x[per50], x[per50]], [0, y[per50]], 'b--') pyl.plot([0, x[per90]], [y[per90], y[per90]], 'b--') pyl.plot([x[per90], x[per90]], [0, y[per90]], 'b--') #pyl.title('Cumulative Distribution of Stellar Repeatability') #pyl.xlabel('Repeatability IQR RMS (mmag)') #pyl.ylabel('Cumulative Fraction') #pyl.savefig('Srepeat_IQR_cumulative.png',type='png') pyl.legend() pyl.savefig('Srepeat_cumulative.png', type='png') #repeatability from IQR rs = so.robustSigma(stardat['iqr_sigma']) med = np.median(stardat['iqr_sigma']) maglim = np.around(np.array([med - 3 * rs, med + 3. * rs]) * 1000, decimals=2) / 1000 cpu.plot_density(stardat['x'], stardat['y'], stardat['iqr_sigma'] * 1000, z_method=np.mean, zlimits=maglim * 1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag') pyl.title('Repeatability (IQR)') pyl.savefig('Srepeat_IQR.png', type='png') pyl.figure() num, b, p = pyl.hist(stardat['iqr_sigma'] * 1000, bins=100, range=maglim * 1000, histtype='bar') pyl.xlabel('RMS per star (mmag)') pyl.title('Repeatability IQR, Median = %4.2f mmag' % np.median(stardat['iqr_sigma'] * 1000)) pyl.savefig('Srepeat_IQR_hist.png', type='png') starResults['Median Repeatability (IQR) (mmag)'] = np.median(med) * 1e3 pyl.figure() good = np.where(stardat['magtrue'] < 18) num, b, p = pyl.hist(stardat['iqr_sigma'][good] * 1000, bins=100, range=maglim * 1000, histtype='bar') pyl.xlabel('RMS per star, m < 18 (mmag)') pyl.title('Repeatability IQR, Median = %4.2f mmag' % np.median(stardat['iqr_sigma'][good] * 1000)) pyl.savefig('Srepeat_IQR_bright_hist.png', type='png') starResults['Median Repeatability Bright (IQR) (mmag)'] = np.median( stardat['iqr_sigma'][good]) * 1e3 #repeatability v color pyl.figure() #pyl.plot(stardat['color'], stardat['rms']*1000., 'ko', alpha=.1) pyl.hexbin(stardat['color'], stardat['iqr_sigma'] * 1000., bins='log') cb = pyl.colorbar() cb.set_label(r'$\log{\rm{N}}$') pyl.ylim([0, 40]) pyl.xlabel(r'$g-i$') pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)') pyl.savefig('Srepeatvcolor.png', type='png') #repeatability v mag pyl.figure() #pyl.plot(stardat['magtrue'], stardat['rms']*1000., 'ko', alpha=.1) pyl.hexbin(stardat['magtrue'], stardat['iqr_sigma'] * 1000., bins='log') cb = pyl.colorbar() cb.set_label(r'$\log{\rm{N}}$') pyl.ylim([0, 40]) pyl.xlabel('True Mag') pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)') pyl.savefig('Srepeatvmag.png', type='png') #repeatability rs = so.robustSigma(stardat['rms']) med = np.median(stardat['rms']) maglim = np.around(np.array([med - 3 * rs, med + 3. * rs]) * 1000, decimals=2) / 1000 cpu.plot_density(stardat['x'], stardat['y'], stardat['rms'] * 1000, z_method=np.mean, zlimits=maglim * 1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag') pyl.title('Repeatability') pyl.savefig('Srepeat.png', type='png') pyl.figure() num, b, p = pyl.hist(stardat['rms'] * 1000, bins=100, range=maglim * 1000, histtype='bar') pyl.xlabel('RMS per star (mmag)') pyl.title('Repeatability, Median = %4.2f mmag' % np.median(stardat['rms'] * 1000)) pyl.savefig('Srepeat_hist.png', type='png') starResults['Median Repeatability (mmag)'] = np.median(med) * 1e3 pyl.figure() x = np.sort(stardat['rms']) * 1e3 y = np.arange(np.size(x), dtype='float') y = y / np.max(y) per50 = np.max(np.where(y <= .5)) per90 = np.max(np.where(y <= .9)) pyl.plot(x, y, 'k', label='RMS, %2.1f, %2.1f' % (x[per50], x[per90])) pyl.plot([0, x[per50]], [y[per50], y[per50]], 'k') pyl.plot([x[per50], x[per50]], [0, y[per50]], 'k') pyl.plot([0, x[per90]], [y[per90], y[per90]], 'k') pyl.plot([x[per90], x[per90]], [0, y[per90]], 'k') pyl.title('Cumulative Distribution of Stellar Repeatability') pyl.xlabel('Repeatability RMS (mmag)') pyl.ylabel('Cumulative Fraction') pyl.savefig('Srepeat_cumulative.png', type='png') #need to make a spatial uniformity plot since we can now have varying densities of stars. nside = 64 stardat['hpid'] = hp.ang2pix(nside, (stardat['dec'] + 90.) * deg2rad, stardat['ra'] * deg2rad) uhpid = np.unique(stardat['hpid']) hp_mean = uhpid * 0. hp_std = uhpid * 0. hp_dec, hp_ra = hp.pix2ang(nside, uhpid) #hp_ra = hp_ra hp_dec = hp_dec - 90. * deg2rad hp_x, hp_y = cpu.hammer_project_toxy(hp_ra, hp_dec) stardat = dictsort(stardat, 'hpid') left = np.searchsorted(stardat['hpid'], uhpid) right = np.searchsorted(stardat['hpid'], uhpid, side='right') for i in np.arange(left.size): hp_mean[i] = np.mean(stardat['magdiff'][left[i]:right[i]]) hp_std[i] = np.std(stardat['magdiff'][left[i]:right[i]]) cpu.plot_density(hp_x, hp_y, hp_mean * 1000, z_method=np.mean, zlimits=None, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag') HPResults['Spatial Uniformity RMS (mmag)'] = np.std(hp_mean) * 1e3 HPResults['Spatial Uniformity IQR RMS (mmag)'] = so.robustSigma( hp_mean) * 1e3 pyl.title('mean(True-Bestfit), binned by HEALpix') pyl.savefig('HPresid.png', type='png') cpu.plot_density(hp_x, hp_y, hp_std * 1000, z_method=np.mean, zlimits=None, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag') pyl.title('std(True-Bestfit), binned by HEALpix') pyl.savefig('HPstd.png', type='png') pyl.figure() num, b, p = pyl.hist(hp_mean * 1000, bins=100, range=None, histtype='bar') pyl.xlabel("HEALpix(True-Bestfit) (mmag)") pyl.title('Spatial Uniformity, RMS=%4.2f' % HPResults['Spatial Uniformity RMS (mmag)']) pyl.savefig('HPmean_hist.png', type='png') pyl.figure() num, b, p = pyl.hist(hp_std * 1000, bins=100, range=None, histtype='bar') pyl.xlabel("std(True-Bestfit) per HEALpix (mmag)") pyl.title('Variation within HEALpix, median=%4.2f mmag' % np.median(hp_std * 1000)) pyl.savefig('HPstd_hist.png', type='png') return starResults, HPResults
def read_true_stars(stardatfile='stardata.dat'): """Read a stardata.dat file from simSelfCalib.py""" keys = ('id', 'dbid', 'ra', 'dec', 'magtrue', 'color') stardat = ui.readDatafile(stardatfile, keys) return stardat
def split_obs(nside, shift_size=5., splitNS=False, NorthOnly=False, SouthOnly=False, blocksize=1e6): """ XXX kill this function XXX replaced by just including neighboring healpixels""" """nside gives the number of sides for healpixels. shift_size is in degrees and sets how """ #XXX-prob should put in a check so I don't sort repeatedly #if not os.path.isfile('star_obs.dat.s'): waitpop('sort -nk 1 star_obs.dat > star_obs.dat.s') npix = hp.nside2npix(nside) keys = ('visitid', 'patchid', 'subpatch', 'nstars', 'ra', 'dec') patchfile = 'patchdata.dat' patchdat = ui.readDatafile(patchfile, keys) ord = np.argsort(patchdat['patchid']) for key in list(patchdat.keys()): patchdat[key] = patchdat[key][ord] shift_size = shift_size * np.pi / 180. #1.*np.pi/180. #compute galactic coords for healpixels and patches heal = {} heal['ra'], heal['dec'] = hp.pix2ang(nside, np.arange(npix)) heal['dec'] = heal['dec'] - 90. * np.pi / 180. heal['b'] = galac_b( heal['ra'], heal['dec'] ) #np.arcsin(np.sin(heal['dec'])*np.cos(62.6*np.pi/180.)-np.cos(heal['dec'])*np.sin(heal['ra']-282.25*np.pi/180.)*np.sin(62.6*np.pi/180.)) patchdat['dec'] = patchdat['dec'] * np.pi / 180. patchdat['ra'] = patchdat['ra'] * np.pi / 180. patchdat['b'] = galac_b(patchdat['ra'], patchdat['dec']) patchdat['l'] = galac_l(patchdat['ra'], patchdat['dec']) patchdat['dec'] = patchdat[ 'dec'] + 90. * np.pi / 180. #stupid healpix wants 0-180 I think... patch_healpix = hp.ang2pix(nside, patchdat['dec'], patchdat['ra']) pyl.figure() cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix) pyl.savefig('heal_check.png', type='png') patch_healpix_shifted1 = hp.ang2pix(nside, patchdat['dec'] + shift_size, patchdat['ra'] + shift_size * 2) patch_healpix_shifted2 = hp.ang2pix(nside, patchdat['dec'] - shift_size, patchdat['ra'] + shift_size * 2) patch_healpix_shifted3 = hp.ang2pix(nside, patchdat['dec'] + shift_size, patchdat['ra'] - shift_size * 2) patch_healpix_shifted4 = hp.ang2pix(nside, patchdat['dec'] - shift_size, patchdat['ra'] - shift_size * 2) pyl.figure() cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix_shifted1) pyl.savefig('heal_check1.png', type='png') #ok, just go through and set duplicates to -1. Also set any of the shifted ones that are not in patch_healpix to -1 uhp = np.unique(patch_healpix) patch_healpix_shifted1[np.invert(np.in1d(patch_healpix_shifted1, uhp))] = -1 patch_healpix_shifted2[np.invert(np.in1d(patch_healpix_shifted2, uhp))] = -1 patch_healpix_shifted3[np.invert(np.in1d(patch_healpix_shifted3, uhp))] = -1 patch_healpix_shifted4[np.invert(np.in1d(patch_healpix_shifted4, uhp))] = -1 diff = patch_healpix_shifted1 - patch_healpix good = np.where(diff == 0) patch_healpix_shifted1[good] = -1 diff1 = patch_healpix_shifted2 - patch_healpix diff2 = patch_healpix_shifted2 - patch_healpix_shifted1 good = np.where((diff1 == 0) | (diff2 == 0)) patch_healpix_shifted2[good] = -1 diff1 = patch_healpix_shifted3 - patch_healpix diff2 = patch_healpix_shifted3 - patch_healpix_shifted1 diff3 = patch_healpix_shifted3 - patch_healpix_shifted2 good = np.where((diff1 == 0) | (diff2 == 0) | (diff3 == 0)) patch_healpix_shifted3[good] = -1 diff1 = patch_healpix_shifted4 - patch_healpix diff2 = patch_healpix_shifted4 - patch_healpix_shifted1 diff3 = patch_healpix_shifted4 - patch_healpix_shifted2 diff4 = patch_healpix_shifted4 - patch_healpix_shifted3 good = np.where((diff1 == 0) | (diff2 == 0) | (diff3 == 0) | (diff4 == 0)) patch_healpix_shifted4[good] = -1 pyl.figure() cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix_shifted1) pyl.savefig('heal_check1_post.png', type='png') heal_pixels = np.unique(patch_healpix) keys = [ patch_healpix, patch_healpix_shifted1, patch_healpix_shifted2, patch_healpix_shifted3, patch_healpix_shifted4 ] filenames = ['hp1.dat', 'hp2.dat', 'hp3.dat', 'hp4.dat', 'hp5.dat'] #ok, if a healpix stradles b =0, I want to split it in two # I could also just loop through each HP. if the HP has patches w/ b> 0 and b< 0 and min l < xxx or max l < xxx, then break it apart. if splitNS: break_up = np.where((patchdat['b'] < 0) & ((patchdat['l'] < 120. * np.pi / 180.) | (patchdat['l'] > 240) * np.pi / 180.) & (np.abs(patchdat['b']) < 5 * np.pi / 180.)) for key in keys: mask = np.where(key == -1) key[break_up] = key[break_up] + max(heal_pixels) + 1 key[mask] = -1 print('reading star_obs.dat.s') star_obs = open('star_obs.dat.s', 'r') #read in just the patch id's from star_obs all_patch_ids = [] for line in star_obs: if line.startswith('#'): continue linevalues = line.split() all_patch_ids.append(linevalues[0]) all_patch_ids = np.array(all_patch_ids, dtype=np.int) star_obs.close() for j in np.arange(4): filename = filenames[j] patch_healpix = keys[j] hpid = all_patch_ids * 0 left = np.searchsorted(all_patch_ids, patchdat['patchid']) right = np.searchsorted(all_patch_ids, patchdat['patchid'], side='right') for i in np.arange(np.size(left)): hpid[left[i]:right[i]] = patch_healpix[i] print('left %i, hpid %i, all_patch_ids %i, patch_healpix %i ' % (np.size(left), np.size(hpid), np.size(all_patch_ids), np.size(patch_healpix))) file = open(filename, 'w') print("#", file=file) for value in hpid: print(value, file=file) file.close() #print j, np.size(hpid), np.size(all_patch_ids) waitpop('paste hp1.dat star_obs.dat.s > hp1_star_obs.dat') waitpop('paste hp2.dat star_obs.dat.s > hp2_star_obs.dat') waitpop("sed -i '/^-1/d' hp2_star_obs.dat") waitpop('paste hp3.dat star_obs.dat.s > hp3_star_obs.dat') waitpop("sed -i '/^-1/d' hp3_star_obs.dat") waitpop('paste hp4.dat star_obs.dat.s > hp4_star_obs.dat') waitpop("sed -i '/^-1/d' hp4_star_obs.dat") waitpop('paste hp5.dat star_obs.dat.s > hp5_star_obs.dat') waitpop("sed -i '/^-1/d' hp5_star_obs.dat") #waitpop("mv temp.dat hp2_star_obs.dat") waitpop('cat hp2_star_obs.dat >> hp1_star_obs.dat') waitpop('cat hp3_star_obs.dat >> hp1_star_obs.dat') waitpop('cat hp4_star_obs.dat >> hp1_star_obs.dat') waitpop('cat hp5_star_obs.dat >> hp1_star_obs.dat') waitpop( ''' awk '{print $2" "$3" "$4" "$5 " "$6> ("h"$1"star_obs.dat")}' hp1_star_obs.dat''' ) waitpop('rm h#star_obs.dat') waitpop('mv hp1_star_obs.dat star_obs_hp1.dat') waitpop('mv hp2_star_obs.dat star_obs_hp2.dat') waitpop('mv hp3_star_obs.dat star_obs_hp3.dat') waitpop('mv hp4_star_obs.dat star_obs_hp4.dat') waitpop('mv hp5_star_obs.dat star_obs_hp5.dat')
def read_starsobs(starobsfile='star_obs.dat'): """Read the star_obs.dat file from simSelfCalib.py that is input to simple.x""" keys = ('patchid', 'starid', 'rmagobs', 'magerr') starobs = ui.readDatafile(starobsfile, keys) return starobs
def read_visitfile(visitfile='visit.dat'): """Read the visit information from simSelfCalib.py""" keys = ('visitid', 'ra', 'dec', 'skyrot', 'time', 'zpOff', 'zpGrad', 'zpGradDir', 'colorterm', 'colortermRad') visits = ui.readDatafile(visitfile, keys) return visits