def finalPatch(solver):
files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE, shell=True).stdout.read().split()
#run the solver
sb.Popen('%s patch_obs.dat patch 1e-12 40000 > /dev/null'%solver, shell=True).wait()
sb.Popen('cp patch_bestfit_FpPatch.dat test_bestfit_FpPatch.dat', shell=True).wait()
npatch = np.size(files)
patchdat={}
final_keys = ('patchid', 'patchmag', 'healid')
for key in final_keys:
patchdat[key]=[]
keys=('patchid','patchmag')
for i in np.arange(npatch):
temp = ui.readDatafile(files[i], keys, keytypes=('u8','float'))
heal_num = np.float(files[i][1:-18])
#insert if statement to deal with stupid patches that are not continuous
patchdat['patchid'].append(np.ravel(temp['patchid']))
patchdat['healid'].append(np.ravel(temp['patchid']*0+heal_num))
patchdat['patchmag'].append(np.ravel(temp['patchmag']))
for key in list(patchdat.keys()):
patchdat[key] = np.concatenate(patchdat[key])
ord = np.argsort(patchdat['healid'])
for key in list(patchdat.keys()):
patchdat[key] = patchdat[key][ord]
keys=('hpid','hpzp')
healdat = ui.readDatafile('patch_bestfit_Patch.dat',keys, keytypes=('u8','float'))
keys=('patchid','patchmag')
fit_patchdat = ui.readDatafile('patch_bestfit_Star.dat',keys, keytypes=('u8','float'))
left = np.searchsorted(patchdat['healid'],healdat['hpid'])
right = np.searchsorted(patchdat['healid'],healdat['hpid'], side='right')
for i in np.arange(np.size(right)):
patchdat['patchmag'][left[i]:right[i]] = patchdat['patchmag'][left[i]:right[i]]-healdat['hpzp'][i]
patchid = np.unique(patchdat['patchid'])
patchmag = patchid*0
ord = np.argsort(patchdat['patchid'])
patchdat['patchmag'] = patchdat['patchmag'][ord]
patchdat['patchid'] = patchdat['patchid'][ord]
left = np.searchsorted(patchdat['patchid'],patchid)
patchmag = patchdat['patchmag'][left]
#now plug in the best fit mags where they exist
left = np.searchsorted(patchid, fit_patchdat['patchid'])
patchmag[left]=fit_patchdat['patchmag']
file = open('test_bestfit_Patch.dat','w')
for i in np.arange(np.size(patchmag)):
print("%d %f"%(patchid[i], patchmag[i]), file=file)
file.close()
def finalStar(infile='star_obs.dat', outfile='test_bestfit_Star.dat'):
"""read in the best fit patches, and the raw observations. Apply patch zps to all the star magnitudes, then take a weighted mean of each star to make a bestfit star magnitude. Also calc an stdev for each star"""
patchzp = ui.readDatafile('test_bestfit_Patch.dat', ('patchid','patchzp'), keytypes=('u8','float') )
star_obs = ui.readDatafile(infile, ('patchid','starid','starmag','starerr'),
keytypes=('u8','u8','float','float'))
order = np.argsort(star_obs['patchid'])
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][order]
left = np.searchsorted(star_obs['patchid'], patchzp['patchid'])
right = np.searchsorted(star_obs['patchid'], patchzp['patchid'], side='right')
for i in np.arange(left.size):
star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][left[i]:right[i]] - patchzp['patchzp'][i]
order = np.argsort(star_obs['starid'])
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][order]
ustar = np.unique(star_obs['starid'])
mag = np.zeros(ustar.size, dtype='float')
magerr = np.zeros(ustar.size, dtype='float')
magstd_raw = np.zeros(ustar.size, dtype='float')
left = np.searchsorted(star_obs['starid'], ustar)
right = np.searchsorted(star_obs['starid'], ustar, side='right')
for i in np.arange(left.size):
mag[i] = np.sum(star_obs['starmag'][left[i]:right[i]]/star_obs['starerr'][left[i]:right[i]]**2)/np.sum(1./star_obs['starerr'][left[i]:right[i]]**2)
magerr[i] = 1./np.sqrt(np.sum(1./star_obs['starerr'][left[i]:right[i]]**2))
magstd_raw[i] = star_obs['starmag'][left[i]:right[i]].std()
file = open(outfile, 'w')
print('#StarID Mag Err STDEV', file=file)
for i in np.arange(ustar.size):
print("%i %f %f %f"%(ustar[i], mag[i], magerr[i], magstd_raw[i]), file=file)
file.close()
def patch_combine():
files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE, shell=True).stdout.read().split()
npatch=np.size(files)
patchdat={}
final_keys = ('patchid', 'patchmag', 'healid')
for key in final_keys:
patchdat[key]=[]
keys=('patchid','patchmag')
for i in np.arange(npatch):
temp = ui.readDatafile(files[i], keys, keytypes=('u8','float'))
heal_num = np.float(files[i][1:-18])
patchdat['patchid'].append(np.ravel(temp['patchid']))
patchdat['healid'].append(np.ravel(temp['patchid']*0+heal_num))
patchdat['patchmag'].append(np.ravel(temp['patchmag']))
for key in list(patchdat.keys()):
patchdat[key] = np.concatenate(patchdat[key])
ord = np.argsort(patchdat['healid'])
for key in list(patchdat.keys()):
patchdat[key] = patchdat[key][ord]
file = open('patch_obs.dat', 'w')
print("%s %s %s %s %s" %("#HealID", "PatchID", "PatchMag", "PatchMagErr", "FpPatchID"), file=file)
for obj in range(0,len(patchdat['patchid'])):
print("%d %d %f %f %d"%(patchdat['healid'][obj],patchdat['patchid'][obj], patchdat['patchmag'][obj], 0.001, 0), file=file)
file.close()
def read_true_patchfile(patchfile='patchdata.dat'):
"""Read the patch file information from simSelfCalib.py"""
keys = ('visitid', 'patchid', 'subpatch', 'nstars', 'ra', 'dec', 'magtrue', 'color',
'dmag', 'dmag_rms', 'dmag_rand', 'dmag_rand_rms', 'dmag_zp', 'dmag_zp_rms',
'dmag_color', 'dmag_color_rms')
patchdat = ui.readDatafile(patchfile, keys)
# dmag = stars' (true magnitude - 'observed') magnitude, average
#patchdat['dmag'] = -1*patchdat['dmag']
return patchdat
def read_true_patchfile(patchfile='patchdata.dat'):
"""Read the patch file information from simSelfCalib.py"""
keys = ('visitid', 'patchid', 'subpatch', 'nstars', 'ra', 'dec', 'magtrue',
'color', 'dmag', 'dmag_rms', 'dmag_rand', 'dmag_rand_rms',
'dmag_zp', 'dmag_zp_rms', 'dmag_color', 'dmag_color_rms')
patchdat = ui.readDatafile(patchfile, keys)
# dmag = stars' (true magnitude - 'observed') magnitude, average
#patchdat['dmag'] = -1*patchdat['dmag']
return patchdat
def finalStar_bad():
"""apply the healpix zeropoints to each block and collect all the bestfit star mags.
I need to check that the multiple solutions per star are reasonable... is this an FpPatch issue?"""
stardat = {}
keys = ('starid', 'starmag')
for key in keys:
stardat[key] = []
healzp = ui.readDatafile('patch_bestfit_Patch.dat', ('healid', 'healmag'),
keytypes=('u8', 'float'))
for i in np.arange(np.size(healzp['healid'])):
temp = ui.readDatafile('h%i_bestfit_Star.dat' % healzp['healid'][i],
keys,
keytypes=('u8', 'float'))
fp = ui.readDatafile('h%i_bestfit_FpPatch.dat' % healzp['healid'][i],
('id', 'mag'))
temp['starmag'] = temp['starmag'] - healzp['healmag'][i] - fp['mag']
stardat['starid'].append(np.ravel(temp['starid']))
stardat['starmag'].append(np.ravel(temp['starmag']))
for key in list(stardat.keys()):
stardat[key] = np.concatenate(stardat[key])
#Crap, looks like there's more jitter solution-to-solution than I expected. might need to go back through the star_obs.dat file after all
#convert stardat from dict of np arrays to a numpy rec array
#sd = np.array(zip(stardat['starid'], stardat['starmag']), dtype=[('starid', 'u8'), ('starmag','float')] )
#sd = np.core.records.fromarrays(stardat['starid'], stardat['starmag'],names='starid,starmag' )
#sd.sort(order='starid')
order = np.argsort(stardat['starid'])
for key in list(stardat.keys()):
stardat[key] = stardat[key][order]
uid = np.unique(stardat['starid'])
left = np.searchsorted(stardat['starid'], uid)
right = np.searchsorted(stardat['starid'], uid, side='right')
mags = np.zeros(left.size, dtype='float')
stds = mags.copy()
for i in np.arange(left.size):
mags[i] = stardat['starmag'][left[i]:right[i]].mean()
stds[i] = stardat['starmag'][left[i]:right[i]].std()
file = open('test_bestfit_Star.dat', 'w')
for i in np.arange(uid.size):
print("%i %f" % (uid[i], mags[i]), file=file)
file.close()
def finalStar(infile='star_obs.dat', outfile='test_bestfit_Star.dat'):
"""read in the best fit patches, and the raw observations. Apply patch zps to all the star magnitudes, then take a weighted mean of each star to make a bestfit star magnitude. Also calc an stdev for each star"""
patchzp = ui.readDatafile('test_bestfit_Patch.dat', ('patchid', 'patchzp'),
keytypes=('u8', 'float'))
star_obs = ui.readDatafile(infile,
('patchid', 'starid', 'starmag', 'starerr'),
keytypes=('u8', 'u8', 'float', 'float'))
order = np.argsort(star_obs['patchid'])
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][order]
left = np.searchsorted(star_obs['patchid'], patchzp['patchid'])
right = np.searchsorted(star_obs['patchid'],
patchzp['patchid'],
side='right')
for i in np.arange(left.size):
star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][
left[i]:right[i]] - patchzp['patchzp'][i]
order = np.argsort(star_obs['starid'])
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][order]
ustar = np.unique(star_obs['starid'])
mag = np.zeros(ustar.size, dtype='float')
magerr = np.zeros(ustar.size, dtype='float')
magstd_raw = np.zeros(ustar.size, dtype='float')
left = np.searchsorted(star_obs['starid'], ustar)
right = np.searchsorted(star_obs['starid'], ustar, side='right')
for i in np.arange(left.size):
mag[i] = np.sum(star_obs['starmag'][left[i]:right[i]] /
star_obs['starerr'][left[i]:right[i]]**2) / np.sum(
1. / star_obs['starerr'][left[i]:right[i]]**2)
magerr[i] = 1. / np.sqrt(
np.sum(1. / star_obs['starerr'][left[i]:right[i]]**2))
magstd_raw[i] = star_obs['starmag'][left[i]:right[i]].std()
file = open(outfile, 'w')
print('#StarID Mag Err STDEV', file=file)
for i in np.arange(ustar.size):
print("%i %f %f %f" % (ustar[i], mag[i], magerr[i], magstd_raw[i]),
file=file)
file.close()
def finalStar_bad():
"""apply the healpix zeropoints to each block and collect all the bestfit star mags.
I need to check that the multiple solutions per star are reasonable... is this an FpPatch issue?"""
stardat={}
keys = ('starid','starmag')
for key in keys:
stardat[key] = []
healzp = ui.readDatafile('patch_bestfit_Patch.dat',('healid','healmag'), keytypes=('u8','float'))
for i in np.arange(np.size(healzp['healid'])):
temp = ui.readDatafile('h%i_bestfit_Star.dat'%healzp['healid'][i], keys, keytypes=('u8','float'))
fp = ui.readDatafile('h%i_bestfit_FpPatch.dat'%healzp['healid'][i], ('id','mag'))
temp['starmag'] = temp['starmag'] - healzp['healmag'][i] - fp['mag']
stardat['starid'].append(np.ravel(temp['starid']))
stardat['starmag'].append(np.ravel(temp['starmag']))
for key in list(stardat.keys()):
stardat[key] = np.concatenate(stardat[key])
#Crap, looks like there's more jitter solution-to-solution than I expected. might need to go back through the star_obs.dat file after all
#convert stardat from dict of np arrays to a numpy rec array
#sd = np.array(zip(stardat['starid'], stardat['starmag']), dtype=[('starid', 'u8'), ('starmag','float')] )
#sd = np.core.records.fromarrays(stardat['starid'], stardat['starmag'],names='starid,starmag' )
#sd.sort(order='starid')
order = np.argsort(stardat['starid'])
for key in list(stardat.keys()):
stardat[key] = stardat[key][order]
uid = np.unique(stardat['starid'])
left = np.searchsorted(stardat['starid'],uid)
right = np.searchsorted(stardat['starid'],uid, side='right')
mags = np.zeros(left.size, dtype='float')
stds = mags.copy()
for i in np.arange(left.size):
mags[i] = stardat['starmag'][left[i]:right[i]].mean()
stds[i] = stardat['starmag'][left[i]:right[i]].std()
file = open('test_bestfit_Star.dat', 'w')
for i in np.arange(uid.size):
print("%i %f"%(uid[i], mags[i]), file=file)
file.close()
def patch_combine():
files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE,
shell=True).stdout.read().split()
npatch = np.size(files)
patchdat = {}
final_keys = ('patchid', 'patchmag', 'healid')
for key in final_keys:
patchdat[key] = []
keys = ('patchid', 'patchmag')
for i in np.arange(npatch):
temp = ui.readDatafile(files[i], keys, keytypes=('u8', 'float'))
heal_num = np.float(files[i][1:-18])
patchdat['patchid'].append(np.ravel(temp['patchid']))
patchdat['healid'].append(np.ravel(temp['patchid'] * 0 + heal_num))
patchdat['patchmag'].append(np.ravel(temp['patchmag']))
for key in list(patchdat.keys()):
patchdat[key] = np.concatenate(patchdat[key])
ord = np.argsort(patchdat['healid'])
for key in list(patchdat.keys()):
patchdat[key] = patchdat[key][ord]
file = open('patch_obs.dat', 'w')
print("%s %s %s %s %s" %
("#HealID", "PatchID", "PatchMag", "PatchMagErr", "FpPatchID"),
file=file)
for obj in range(0, len(patchdat['patchid'])):
print("%d %d %f %f %d" %
(patchdat['healid'][obj], patchdat['patchid'][obj],
patchdat['patchmag'][obj], 0.001, 0),
file=file)
file.close()
def illum_combine():
"""combine the patch AND illumination corrections """
files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE,
shell=True).stdout.read().split()
#illums = sb.Popen('ls h*bestfit_FpPatch.dat', stdout=sb.PIPE, shell=True).stdout.read().split()
#so_files = sb.Popen('ls h*bestfit_FpPatch.dat', stdout=sb.PIPE, shell=True).stdout.read().split()
npatch = np.size(files)
patchdat = {}
final_keys = ('patchid', 'patchmag', 'patchmagerr', 'illumid', 'healid')
illum_keys = ('illumid', 'illummag')
so_keys = ('patchid', 'starid', 'starmag', 'starmagerr', 'illumid', 'hpid')
for key in final_keys:
patchdat[key] = []
keys = ('patchid', 'patchmag')
for i in np.arange(npatch):
temp = ui.readDatafile(files[i], keys, keytypes=('u8', 'float'))
heal_num = np.int(files[i][1:-18])
temp_so = ui.readDatafile('h%u' % heal_num + 'star_obs.dat',
so_keys,
keytypes=('u8', 'u8', 'float', 'float', 'u8',
'u8'))
temp_illum = ui.readDatafile('h%u' % heal_num + '_bestfit_FpPatch.dat',
illum_keys,
keytypes=('u8', 'float'))
#now I need to construct a line for each unique patchid+illumid pair
temp_so['patch_plus_illum'] = np.core.defchararray.add(
temp_so['patchid'].astype('|S10'),
np.core.defchararray.add(
np.core.defchararray.replace(
np.zeros(np.size(temp_so['patchid']), dtype='|S1'), '',
','), temp_so['illumid'].astype('|S10')))
temp_so = dictsort(temp_so, 'patch_plus_illum')
upatch_plus_illum, unique_index = np.unique(
temp_so['patch_plus_illum'], return_index=True)
left = np.searchsorted(temp_so['patch_plus_illum'], upatch_plus_illum)
right = np.searchsorted(temp_so['patch_plus_illum'],
upatch_plus_illum,
side='right')
for key in so_keys:
temp_so[key] = temp_so[key][unique_index]
temp_so['nstars'] = right - left
temp_so = np.core.records.fromarrays([
temp_so['patchid'], temp_so['illumid'],
temp_so['patchid'] * 0 + heal_num, temp_so['starmag'] * 0,
1. / np.sqrt(temp_so['nstars']), temp_so['starmag'] * 0
],
names=('patchid', 'illumid',
'healid', 'patchmag',
'patchmagerr', 'illummag'))
ord = np.argsort(temp['patchid'])
for key in keys:
temp[key] = temp[key][ord]
temp_so.sort(order='patchid')
left = np.searchsorted(temp_so['patchid'], temp['patchid'])
right = np.searchsorted(temp_so['patchid'],
temp['patchid'],
side='right')
for i in np.arange(np.size(left)):
temp_so['patchmag'][left[i]:right[i]] = temp['patchmag'][i]
temp_so.sort(order='illumid')
ord = np.argsort(temp_illum['illumid'])
for key in illum_keys:
temp_illum[key] = temp_illum[key][ord]
left = np.searchsorted(temp_so['illumid'], temp_illum['illumid'])
right = np.searchsorted(temp_so['illumid'],
temp_illum['illumid'],
side='right')
for i in np.arange(np.size(left)):
temp_so['patchmag'][left[i]:right[i]] = temp_so['patchmag'][
left[i]:right[i]] + temp_illum['illummag'][i]
patchdat['patchid'].append(np.ravel(temp_so['patchid']))
patchdat['healid'].append(np.ravel(temp_so['healid']))
patchdat['patchmag'].append(np.ravel(temp_so['patchmag']))
patchdat['illumid'].append(np.ravel(temp_so['illumid']))
patchdat['patchmagerr'].append(np.ravel(temp_so['patchmagerr']))
for key in list(patchdat.keys()):
patchdat[key] = np.concatenate(patchdat[key])
ord = np.argsort(patchdat['healid'])
for key in list(patchdat.keys()):
patchdat[key] = patchdat[key][ord]
file = open('patch_obs.dat', 'w')
print("%s %s %s %s %s" %
("#HealID", "PatchID", "PatchMag", "PatchMagErr", "FpPatchID"),
file=file)
for obj in range(0, len(patchdat['patchid'])):
print("%d %d %f %f %d" %
(patchdat['healid'][obj], patchdat['patchid'][obj],
patchdat['patchmag'][obj], patchdat['patchmagerr'][obj],
patchdat['illumid'][obj]),
file=file)
#print >>file, "%d %d %f %f %d"%(patchdat['healid'][obj],patchdat['patchid'][obj], patchdat['patchmag'][obj], .01, patchdat['illumid'][obj])
file.close()
def finalPatch(solver):
files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE,
shell=True).stdout.read().split()
#run the solver
sb.Popen('%s patch_obs.dat patch 1e-12 40000 > /dev/null' % solver,
shell=True).wait()
sb.Popen('cp patch_bestfit_FpPatch.dat test_bestfit_FpPatch.dat',
shell=True).wait()
npatch = np.size(files)
patchdat = {}
final_keys = ('patchid', 'patchmag', 'healid')
for key in final_keys:
patchdat[key] = []
keys = ('patchid', 'patchmag')
for i in np.arange(npatch):
temp = ui.readDatafile(files[i], keys, keytypes=('u8', 'float'))
heal_num = np.float(files[i][1:-18])
#insert if statement to deal with stupid patches that are not continuous
patchdat['patchid'].append(np.ravel(temp['patchid']))
patchdat['healid'].append(np.ravel(temp['patchid'] * 0 + heal_num))
patchdat['patchmag'].append(np.ravel(temp['patchmag']))
for key in list(patchdat.keys()):
patchdat[key] = np.concatenate(patchdat[key])
ord = np.argsort(patchdat['healid'])
for key in list(patchdat.keys()):
patchdat[key] = patchdat[key][ord]
keys = ('hpid', 'hpzp')
healdat = ui.readDatafile('patch_bestfit_Patch.dat',
keys,
keytypes=('u8', 'float'))
keys = ('patchid', 'patchmag')
fit_patchdat = ui.readDatafile('patch_bestfit_Star.dat',
keys,
keytypes=('u8', 'float'))
left = np.searchsorted(patchdat['healid'], healdat['hpid'])
right = np.searchsorted(patchdat['healid'], healdat['hpid'], side='right')
for i in np.arange(np.size(right)):
patchdat['patchmag'][left[i]:right[i]] = patchdat['patchmag'][
left[i]:right[i]] - healdat['hpzp'][i]
patchid = np.unique(patchdat['patchid'])
patchmag = patchid * 0
ord = np.argsort(patchdat['patchid'])
patchdat['patchmag'] = patchdat['patchmag'][ord]
patchdat['patchid'] = patchdat['patchid'][ord]
left = np.searchsorted(patchdat['patchid'], patchid)
patchmag = patchdat['patchmag'][left]
#now plug in the best fit mags where they exist
left = np.searchsorted(patchid, fit_patchdat['patchid'])
patchmag[left] = fit_patchdat['patchmag']
file = open('test_bestfit_Patch.dat', 'w')
for i in np.arange(np.size(patchmag)):
print("%d %f" % (patchid[i], patchmag[i]), file=file)
file.close()
def read_bestfit_stars(starcalfile='test_bestfit_Star.dat'):
"""Read a bestfit_Star.dat file from simple.x"""
keys = ('id', 'magcal')
starcal = ui.readDatafile(starcalfile, keys)
return starcal
def read_bestfit_patchfile(bestfitpatchfile='test_bestfit_Patch.dat'):
"""Read the best fit patch file information from simple.x"""
keys = ('patchid', 'dmagcal')
patchcal = ui.readDatafile(bestfitpatchfile, keys)
# dmagcal = zeropoint calculated by calsim
return patchcal
def read_visitfile(visitfile='visit.dat'):
"""Read the visit information from simSelfCalib.py"""
keys = ('visitid', 'ra', 'dec', 'skyrot', 'time', 'zpOff', 'zpGrad', 'zpGradDir',
'colorterm', 'colortermRad')
visits = ui.readDatafile(visitfile, keys)
return visits
def finalStarP(infiles='h*star_obs.dat',
outfile='test_bestfit_Star.dat',
outstats='stats_stars'):
""" read in all the healpix files and apply patch zeropoints. Compute statistics for each star as well as final magnitude """
patchzp = ui.readDatafile('test_bestfit_Patch.dat', ('patchid', 'patchzp'),
keytypes=('u8', 'float'))
illumzp = ui.readDatafile('test_bestfit_FpPatch.dat',
('illumid', 'illumzp'),
keytypes=('u8', 'float'))
files = np.array(
sb.Popen('ls %s' % infiles, stdout=sb.PIPE,
shell=True).stdout.read().split())
hps = files.copy()
for i in np.arange(np.size(hps)):
hps[i] = hps[i].replace('h', '').replace('star_obs.dat', '')
hps = hps.astype('int')
#true_stars = ui.readData('stardata.dat', ('starid','stardbid','ra','dec','mag','color'), keytypes=('u8','u8','float','float','float','float'))
results = {}
result_keys = ('starid', 'nobs', 'mag', 'stdev', 'stdev_IQR')
for key in result_keys:
results[key] = []
for i in np.arange(np.size(files)):
#read in file
star_obs = ui.readDatafile(
files[i],
('patchid', 'starid', 'starmag', 'starerr', 'fppatch', 'hp'),
keytypes=('u8', 'u8', 'float', 'float', 'u8', 'u8'))
in_hp = np.where(
star_obs['hp'] ==
hps[i]) #crop down to just those stars in the center HP
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][in_hp]
order = np.argsort(star_obs['patchid'])
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][order]
left = np.searchsorted(star_obs['patchid'], patchzp['patchid'])
right = np.searchsorted(star_obs['patchid'],
patchzp['patchid'],
side='right')
for i in np.arange(left.size): #apply patch zeropoints
star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][
left[i]:right[i]] - patchzp['patchzp'][i]
order = np.argsort(star_obs['fppatch'])
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][order]
left = np.searchsorted(star_obs['fppatch'], illumzp['illumid'])
right = np.searchsorted(star_obs['fppatch'],
illumzp['illumid'],
side='right')
for i in np.arange(left.size):
star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][
left[i]:right[i]] - illumzp['illumzp'][i]
order = np.argsort(star_obs['starid'])
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][order]
ustar = np.unique(star_obs['starid'])
left = np.searchsorted(star_obs['starid'], ustar)
right = np.searchsorted(star_obs['starid'], ustar, side='right')
temp_results = {}
for key in result_keys:
temp_results[key] = np.zeros(left.size)
temp_results['starid'] = ustar.copy()
for i in np.arange(left.size):
temp_results['nobs'][i] = np.size(
star_obs['starmag'][left[i]:right[i]])
temp_results['mag'][i] = np.sum(
star_obs['starmag'][left[i]:right[i]] /
star_obs['starerr'][left[i]:right[i]]**2) / np.sum(
1. / star_obs['starerr'][left[i]:right[i]]**2)
temp_results['stdev'][i] = np.std(
star_obs['starmag'][left[i]:right[i]])
temp_results['stdev_IQR'][i] = robustSigma(
star_obs['starmag'][left[i]:right[i]])
for key in result_keys:
results[key].append(temp_results[key].copy())
for key in result_keys:
results[key] = np.concatenate(results[key])
ord = np.argsort(results['starid'])
for key in result_keys:
results[key] = results[key][ord]
#output test_bestfit_Star w/ID, mag and then the rest.
#I guess I could put the print inside the loop if I was worried about memory
file = open('test_bestfit_Star.dat', 'w')
for i in np.arange(np.size(results['starid'])):
print('%d %f %f %f %d' %
(results['starid'][i], results['mag'][i], results['stdev'][i],
results['stdev_IQR'][i], results['nobs'][i]),
file=file)
file.close()
def illum_combine():
"""combine the patch AND illumination corrections """
files = sb.Popen('ls h*bestfit_Patch.dat', stdout=sb.PIPE, shell=True).stdout.read().split()
#illums = sb.Popen('ls h*bestfit_FpPatch.dat', stdout=sb.PIPE, shell=True).stdout.read().split()
#so_files = sb.Popen('ls h*bestfit_FpPatch.dat', stdout=sb.PIPE, shell=True).stdout.read().split()
npatch=np.size(files)
patchdat={}
final_keys = ('patchid', 'patchmag', 'patchmagerr', 'illumid', 'healid')
illum_keys = ('illumid','illummag')
so_keys = ('patchid','starid','starmag','starmagerr','illumid','hpid')
for key in final_keys:
patchdat[key]=[]
keys=('patchid','patchmag')
for i in np.arange(npatch):
temp = ui.readDatafile(files[i], keys, keytypes=('u8','float'))
heal_num = np.int(files[i][1:-18])
temp_so = ui.readDatafile('h%u'%heal_num+'star_obs.dat', so_keys, keytypes=('u8','u8','float','float','u8','u8'))
temp_illum = ui.readDatafile('h%u'%heal_num+'_bestfit_FpPatch.dat', illum_keys, keytypes=('u8','float'))
#now I need to construct a line for each unique patchid+illumid pair
temp_so['patch_plus_illum'] = np.core.defchararray.add(temp_so['patchid'].astype('|S10'),np.core.defchararray.add(np.core.defchararray.replace(np.zeros(np.size(temp_so['patchid']), dtype='|S1'), '',','), temp_so['illumid'].astype('|S10')))
temp_so = dictsort(temp_so,'patch_plus_illum')
upatch_plus_illum, unique_index = np.unique(temp_so['patch_plus_illum'],return_index=True )
left = np.searchsorted(temp_so['patch_plus_illum'], upatch_plus_illum)
right = np.searchsorted(temp_so['patch_plus_illum'], upatch_plus_illum, side='right')
for key in so_keys:
temp_so[key] = temp_so[key][unique_index]
temp_so['nstars'] = right-left
temp_so = np.core.records.fromarrays([temp_so['patchid'], temp_so['illumid'], temp_so['patchid']*0+heal_num, temp_so['starmag']*0,1./np.sqrt(temp_so['nstars']),temp_so['starmag']*0 ], names=('patchid','illumid','healid','patchmag', 'patchmagerr','illummag'))
ord = np.argsort(temp['patchid'])
for key in keys:
temp[key] = temp[key][ord]
temp_so.sort(order='patchid')
left = np.searchsorted(temp_so['patchid'], temp['patchid'])
right = np.searchsorted(temp_so['patchid'], temp['patchid'], side='right')
for i in np.arange(np.size(left)):
temp_so['patchmag'][left[i]:right[i]] = temp['patchmag'][i]
temp_so.sort(order='illumid')
ord = np.argsort(temp_illum['illumid'])
for key in illum_keys:
temp_illum[key] = temp_illum[key][ord]
left = np.searchsorted(temp_so['illumid'], temp_illum['illumid'])
right = np.searchsorted(temp_so['illumid'], temp_illum['illumid'], side='right')
for i in np.arange(np.size(left)):
temp_so['patchmag'][left[i]:right[i]] = temp_so['patchmag'][left[i]:right[i]] + temp_illum['illummag'][i]
patchdat['patchid'].append(np.ravel(temp_so['patchid']))
patchdat['healid'].append(np.ravel(temp_so['healid']))
patchdat['patchmag'].append(np.ravel(temp_so['patchmag']))
patchdat['illumid'].append(np.ravel(temp_so['illumid']))
patchdat['patchmagerr'].append(np.ravel(temp_so['patchmagerr']))
for key in list(patchdat.keys()):
patchdat[key] = np.concatenate(patchdat[key])
ord = np.argsort(patchdat['healid'])
for key in list(patchdat.keys()):
patchdat[key] = patchdat[key][ord]
file = open('patch_obs.dat', 'w')
print("%s %s %s %s %s" %("#HealID", "PatchID", "PatchMag", "PatchMagErr", "FpPatchID"), file=file)
for obj in range(0,len(patchdat['patchid'])):
print("%d %d %f %f %d"%(patchdat['healid'][obj],patchdat['patchid'][obj], patchdat['patchmag'][obj], patchdat['patchmagerr'][obj], patchdat['illumid'][obj]), file=file)
#print >>file, "%d %d %f %f %d"%(patchdat['healid'][obj],patchdat['patchid'][obj], patchdat['patchmag'][obj], .01, patchdat['illumid'][obj])
file.close()
def read_bestfit_patchfile(bestfitpatchfile='test_bestfit_Patch.dat'):
"""Read the best fit patch file information from simple.x"""
keys = ('patchid', 'dmagcal')
patchcal = ui.readDatafile(bestfitpatchfile, keys)
# dmagcal = zeropoint calculated by calsim
return patchcal
def read_true_stars(stardatfile='stardata.dat'):
"""Read a stardata.dat file from simSelfCalib.py"""
keys = ('id', 'dbid', 'ra', 'dec', 'magtrue', 'color')
stardat = ui.readDatafile(stardatfile, keys)
return stardat
def read_starsobs(starobsfile='star_obs.dat'):
"""Read the star_obs.dat file from simSelfCalib.py that is input to simple.x"""
keys = ('patchid', 'starid', 'rmagobs', 'magerr')
starobs = ui.readDatafile(starobsfile, keys)
return starobs
def read_bestfit_stars(starcalfile='test_bestfit_Star.dat'):
"""Read a bestfit_Star.dat file from simple.x"""
keys = ('id', 'magcal')
starcal = ui.readDatafile(starcalfile, keys)
return starcal
def starPlots(truestarfile = "stardata.dat", bestfitstarfile = "test_bestfit_Star.dat"):
#assume both the files are sorted by star id.
stardat = cp.read_true_stars(truestarfile) #keys are ('id', 'dbid', 'ra', 'dec', 'magtrue', 'color')
starcal = ui.readDatafile(bestfitstarfile, keys=('id','magcal','rms','iqr_sigma','nobs'))
HP = True
if np.size(starcal['id']) == 0:
starcal = ui.readDatafile(bestfitstarfile, keys=('id','magcal'))
HP = False
fitted_stars = np.in1d(stardat['id'], starcal['id'])
for key in stardat.keys():
stardat[key] = stardat[key][fitted_stars]
for key in starcal.keys():
stardat[key] = starcal[key]
stardat['magdiff'] = stardat['magcal'] - stardat['magtrue']
stardat = cp.adjust_stars_calvstrue(stardat)
starResults = {}
HPResults = {}
starResults['N stars Fit'] = np.size(stardat['magdiff'])
outlier_clip=np.where(np.abs(stardat['magdiff']) < .05)
rms = stardat['magdiff'][outlier_clip].std()
print 'true-bestfit RMS = %f, true-bestfit (clipped) = %f'%(stardat['magdiff'].std(), rms)
lim = rms*2.5
if lim < 0.001:
lim = round(lim * 10000) / 10000.0
elif lim < 0.01:
lim = round(lim * 1000) / 1000.0
print "Using magnitude limits for plots of %f, %f" %(-lim, lim)
maglim= [-lim, lim]
starResults['Residuals RMS (mmag)'] = stardat['magdiff'].std()*1e3
starResults['IQR_sigma (mmag)'] = so.robustSigma(stardat['magdiff'])*1e3
#star residual map
stardat['x'], stardat['y'] = cpu.hammer_project_toxy(stardat['ra']*deg2rad,
stardat['dec']*deg2rad)
gridsize = cpu.calc_gridsize(stardat, binsize='patch')
etitle = " RMS = %4.1f mmag"%(rms*1000)
cpu.plot_density(stardat['x'], stardat['y'], stardat['magdiff']*1000, z_method=np.mean,
zlimits=np.array(maglim)*1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag')
figtitle = "Stars dMag(true-bestfit)"
pyl.title(figtitle, fontsize='medium')
pyl.savefig('Sdmag.png',type='png')
maglim2 = [0, maglim[1]/2.]
cp.plot_stars_dmagcaltrue(stardat, maglim=maglim2, etitle=" RMS "+etitle, z_method=np.std)
pyl.savefig('Sdmag_rms.png', format='png')
histrange = [maglim[0]*2.5, maglim[1]*2.5]
nbins = 100
cp.histogram_stars_dmagcaltrue(stardat, nbins=nbins, range=histrange, etitle=etitle)
pyl.savefig('Sdamg_hist.png', format='png')
#color distribution of stars
pyl.figure()
pyl.hist(stardat['color'], bins=100)
pyl.xlabel(r'$g-i$')
pyl.ylabel('# of stars')
pyl.title('Color Distribution')
pyl.savefig('Scolordist.png',type='png')
#accuracy v color
pyl.figure()
#pyl.scatter(stardat['color'], stardat['magdiff']*1000., c=stardat['magtrue'], alpha=0.1,edgecolors=None)
pyl.hexbin(stardat['color'], stardat['magdiff']*1000., bins='log')
cb=pyl.colorbar()
cb.ax.set_ylabel(r'$\log{\rm{N}}$')
pyl.xlabel(r'$g-i$')
pyl.ylabel('Bestfit-True (mmag)')
pyl.savefig('Saccuracyvcolor.png',type='png')
#accuracy v mag
pyl.figure()
#pyl.scatter(stardat['magtrue'], stardat['magdiff']*1000., c=stardat['color'], alpha=0.1,edgecolors=None)
pyl.hexbin(stardat['magtrue'], stardat['magdiff']*1000., bins='log')
cb=pyl.colorbar()
cb.ax.set_ylabel(r'$\log{\rm{N}}$')
pyl.xlabel('True Mag')
pyl.ylabel('Bestfit-True (mmag)')
pyl.savefig('Saccuracyvmag.png',type='png')
if HP:
zlim = [np.min(stardat['nobs']), (np.median(stardat['nobs'])-np.min(stardat['nobs']))+np.median(stardat['nobs'])]
cpu.plot_density(stardat['x'], stardat['y'],stardat['nobs'], z_method=np.mean,
zlimits=zlim, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='N observations')
pyl.title('Visit Density')
pyl.savefig('Snobs.png', format='png')
pyl.figure()
#I should just figure out from the data what nsides is!
hpid = hp.ang2pix(16, (stardat['dec']+90.)*deg2rad, stardat['ra']*deg2rad)
cpu.plot_density(stardat['x'], stardat['y'],np.mod(hpid,250), z_method=np.median,
gridsize=gridsize, radecgrid=True, newfig=True, cb_label='HEALpix ID mod 250')
pyl.title("HEALpix Map")
pyl.savefig('HPid.png')
HPResults['number of HEALpix'] = np.size(np.unique(hpid))
pyl.figure()
num, b, p = pyl.hist(stardat['nobs'], bins=100, range=zlim, histtype='bar')
pyl.xlabel("Number of Observations per Star")
pyl.title('Median = %d'%np.median(stardat['nobs']))
pyl.savefig('Snobs_hist.png',type='png')
starResults['median repeat obs'] = np.median(stardat['nobs'])
pyl.figure()
x = np.sort(stardat['iqr_sigma'])*1e3
y = np.arange(np.size(x), dtype='float')
y = y/np.max(y)
per50 = np.max(np.where(y <= .5))
per90 = np.max(np.where(y <= .9))
pyl.plot(x,y, 'k--', label='IQR, 50th,90th %2.1f, %2.1f'%(x[per50],x[per90]))
pyl.plot([0,x[per50]],[y[per50],y[per50]],'b--')
pyl.plot([x[per50],x[per50]],[0,y[per50]],'b--')
pyl.plot([0,x[per90]],[y[per90],y[per90]],'b--')
pyl.plot([x[per90],x[per90]],[0,y[per90]],'b--')
#pyl.title('Cumulative Distribution of Stellar Repeatability')
#pyl.xlabel('Repeatability IQR RMS (mmag)')
#pyl.ylabel('Cumulative Fraction')
#pyl.savefig('Srepeat_IQR_cumulative.png',type='png')
pyl.legend()
pyl.savefig('Srepeat_cumulative.png',type='png')
#repeatability from IQR
rs = so.robustSigma(stardat['iqr_sigma'])
med = np.median(stardat['iqr_sigma'])
maglim = np.around(np.array([med-3*rs, med+3.*rs])*1000, decimals=2)/1000
cpu.plot_density(stardat['x'], stardat['y'], stardat['iqr_sigma']*1000, z_method=np.mean,
zlimits=maglim*1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag')
pyl.title('Repeatability (IQR)')
pyl.savefig('Srepeat_IQR.png',type='png')
pyl.figure()
num, b, p = pyl.hist(stardat['iqr_sigma']*1000, bins=100, range=maglim*1000, histtype='bar')
pyl.xlabel('RMS per star (mmag)')
pyl.title('Repeatability IQR, Median = %4.2f mmag'%np.median(stardat['iqr_sigma']*1000))
pyl.savefig('Srepeat_IQR_hist.png',type='png')
starResults['Median Repeatability (IQR) (mmag)'] = np.median(med)*1e3
pyl.figure()
good = np.where(stardat['magtrue'] < 18)
num, b, p = pyl.hist(stardat['iqr_sigma'][good]*1000, bins=100, range=maglim*1000, histtype='bar')
pyl.xlabel('RMS per star, m < 18 (mmag)')
pyl.title('Repeatability IQR, Median = %4.2f mmag'%np.median(stardat['iqr_sigma'][good]*1000))
pyl.savefig('Srepeat_IQR_bright_hist.png',type='png')
starResults['Median Repeatability Bright (IQR) (mmag)'] = np.median(stardat['iqr_sigma'][good])*1e3
#repeatability v color
pyl.figure()
#pyl.plot(stardat['color'], stardat['rms']*1000., 'ko', alpha=.1)
pyl.hexbin(stardat['color'], stardat['iqr_sigma']*1000., bins='log')
cb = pyl.colorbar()
cb.set_label(r'$\log{\rm{N}}$')
pyl.ylim([0,40])
pyl.xlabel(r'$g-i$')
pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)')
pyl.savefig('Srepeatvcolor.png',type='png')
#repeatability v mag
pyl.figure()
#pyl.plot(stardat['magtrue'], stardat['rms']*1000., 'ko', alpha=.1)
pyl.hexbin(stardat['magtrue'], stardat['iqr_sigma']*1000.,bins='log')
cb = pyl.colorbar()
cb.set_label(r'$\log{\rm{N}}$')
pyl.ylim([0,40])
pyl.xlabel('True Mag')
pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)')
pyl.savefig('Srepeatvmag.png',type='png')
#repeatability
rs = so.robustSigma(stardat['rms'])
med = np.median(stardat['rms'])
maglim = np.around(np.array([med-3*rs, med+3.*rs])*1000, decimals=2)/1000
cpu.plot_density(stardat['x'], stardat['y'], stardat['rms']*1000, z_method=np.mean,
zlimits=maglim*1000, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag')
pyl.title('Repeatability')
pyl.savefig('Srepeat.png',type='png')
pyl.figure()
num, b, p = pyl.hist(stardat['rms']*1000, bins=100, range=maglim*1000, histtype='bar')
pyl.xlabel('RMS per star (mmag)')
pyl.title('Repeatability, Median = %4.2f mmag'%np.median(stardat['rms']*1000))
pyl.savefig('Srepeat_hist.png',type='png')
starResults['Median Repeatability (mmag)'] = np.median(med)*1e3
pyl.figure()
x = np.sort(stardat['rms'])*1e3
y = np.arange(np.size(x), dtype='float')
y = y/np.max(y)
per50 = np.max(np.where(y <= .5))
per90 = np.max(np.where(y <= .9))
pyl.plot(x,y, 'k', label='RMS, %2.1f, %2.1f'%(x[per50],x[per90]))
pyl.plot([0,x[per50]],[y[per50],y[per50]],'k')
pyl.plot([x[per50],x[per50]],[0,y[per50]],'k')
pyl.plot([0,x[per90]],[y[per90],y[per90]],'k')
pyl.plot([x[per90],x[per90]],[0,y[per90]],'k')
pyl.title('Cumulative Distribution of Stellar Repeatability')
pyl.xlabel('Repeatability RMS (mmag)')
pyl.ylabel('Cumulative Fraction')
pyl.savefig('Srepeat_cumulative.png',type='png')
#need to make a spatial uniformity plot since we can now have varying densities of stars.
nside = 64
stardat['hpid'] = hp.ang2pix(nside, (stardat['dec']+90.)*deg2rad, stardat['ra']*deg2rad)
uhpid = np.unique(stardat['hpid'])
hp_mean = uhpid*0.
hp_std = uhpid*0.
hp_dec, hp_ra = hp.pix2ang(nside,uhpid)
#hp_ra = hp_ra
hp_dec = hp_dec-90.*deg2rad
hp_x, hp_y = cpu.hammer_project_toxy(hp_ra, hp_dec)
stardat = dictsort(stardat,'hpid')
left = np.searchsorted(stardat['hpid'], uhpid)
right = np.searchsorted(stardat['hpid'], uhpid,side='right')
for i in np.arange(left.size):
hp_mean[i] = np.mean(stardat['magdiff'][left[i]:right[i]])
hp_std[i] = np.std(stardat['magdiff'][left[i]:right[i]])
cpu.plot_density(hp_x, hp_y, hp_mean*1000, z_method=np.mean,
zlimits=None, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag')
HPResults['Spatial Uniformity RMS (mmag)'] = np.std(hp_mean)*1e3
HPResults['Spatial Uniformity IQR RMS (mmag)'] = so.robustSigma(hp_mean)*1e3
pyl.title('mean(True-Bestfit), binned by HEALpix')
pyl.savefig('HPresid.png',type='png')
cpu.plot_density(hp_x, hp_y, hp_std*1000, z_method=np.mean,
zlimits=None, gridsize=gridsize, radecgrid=True, newfig=True, cb_label='mmag')
pyl.title('std(True-Bestfit), binned by HEALpix')
pyl.savefig('HPstd.png',type='png')
pyl.figure()
num, b, p = pyl.hist( hp_mean*1000, bins=100, range=None, histtype='bar')
pyl.xlabel("HEALpix(True-Bestfit) (mmag)")
pyl.title('Spatial Uniformity, RMS=%4.2f'%HPResults['Spatial Uniformity RMS (mmag)'])
pyl.savefig('HPmean_hist.png',type='png')
pyl.figure()
num, b, p = pyl.hist( hp_std*1000, bins=100, range=None, histtype='bar')
pyl.xlabel("std(True-Bestfit) per HEALpix (mmag)")
pyl.title('Variation within HEALpix, median=%4.2f mmag'%np.median(hp_std*1000))
pyl.savefig('HPstd_hist.png',type='png')
return starResults,HPResults
def finalStarP(infiles='h*star_obs.dat', outfile='test_bestfit_Star.dat', outstats='stats_stars'):
""" read in all the healpix files and apply patch zeropoints. Compute statistics for each star as well as final magnitude """
patchzp = ui.readDatafile('test_bestfit_Patch.dat', ('patchid','patchzp'), keytypes=('u8','float') )
illumzp = ui.readDatafile('test_bestfit_FpPatch.dat', ('illumid', 'illumzp'), keytypes=('u8','float'))
files = np.array(sb.Popen('ls %s'%infiles, stdout=sb.PIPE, shell=True).stdout.read().split())
hps = files.copy()
for i in np.arange(np.size(hps)):
hps[i]=hps[i].replace('h','').replace('star_obs.dat','')
hps=hps.astype('int')
#true_stars = ui.readData('stardata.dat', ('starid','stardbid','ra','dec','mag','color'), keytypes=('u8','u8','float','float','float','float'))
results = {}
result_keys = ('starid','nobs','mag','stdev','stdev_IQR')
for key in result_keys:
results[key]=[]
for i in np.arange(np.size(files)):
#read in file
star_obs = ui.readDatafile(files[i], ('patchid','starid','starmag','starerr','fppatch','hp'),
keytypes=('u8','u8','float','float','u8','u8'))
in_hp = np.where(star_obs['hp'] == hps[i])#crop down to just those stars in the center HP
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][in_hp]
order = np.argsort(star_obs['patchid'])
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][order]
left = np.searchsorted(star_obs['patchid'], patchzp['patchid'])
right = np.searchsorted(star_obs['patchid'], patchzp['patchid'], side='right')
for i in np.arange(left.size): #apply patch zeropoints
star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][left[i]:right[i]] - patchzp['patchzp'][i]
order = np.argsort(star_obs['fppatch'])
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][order]
left = np.searchsorted(star_obs['fppatch'], illumzp['illumid'])
right = np.searchsorted(star_obs['fppatch'], illumzp['illumid'], side='right')
for i in np.arange(left.size):
star_obs['starmag'][left[i]:right[i]] = star_obs['starmag'][left[i]:right[i]] - illumzp['illumzp'][i]
order = np.argsort(star_obs['starid'])
for key in list(star_obs.keys()):
star_obs[key] = star_obs[key][order]
ustar = np.unique(star_obs['starid'])
left = np.searchsorted( star_obs['starid'], ustar)
right = np.searchsorted(star_obs['starid'], ustar,side='right')
temp_results = {}
for key in result_keys:
temp_results[key] = np.zeros(left.size)
temp_results['starid'] = ustar.copy()
for i in np.arange(left.size):
temp_results['nobs'][i] = np.size(star_obs['starmag'][left[i]:right[i]])
temp_results['mag'][i] = np.sum(star_obs['starmag'][left[i]:right[i]]/star_obs['starerr'][left[i]:right[i]]**2)/np.sum(1./star_obs['starerr'][left[i]:right[i]]**2)
temp_results['stdev'][i] = np.std(star_obs['starmag'][left[i]:right[i]])
temp_results['stdev_IQR'][i] = robustSigma(star_obs['starmag'][left[i]:right[i]])
for key in result_keys:
results[key].append(temp_results[key].copy())
for key in result_keys:
results[key] = np.concatenate(results[key])
ord = np.argsort(results['starid'])
for key in result_keys:
results[key] = results[key][ord]
#output test_bestfit_Star w/ID, mag and then the rest.
#I guess I could put the print inside the loop if I was worried about memory
file = open('test_bestfit_Star.dat','w')
for i in np.arange(np.size(results['starid'])):
print('%d %f %f %f %d'%(results['starid'][i], results['mag'][i],
results['stdev'][i], results['stdev_IQR'][i],results['nobs'][i]), file=file)
file.close()
def split_obs(nside,shift_size=5., splitNS=False, NorthOnly=False, SouthOnly=False, blocksize=1e6):
""" XXX kill this function XXX replaced by just including neighboring healpixels"""
"""nside gives the number of sides for healpixels. shift_size is in degrees and sets how """
#XXX-prob should put in a check so I don't sort repeatedly
#if not os.path.isfile('star_obs.dat.s'):
waitpop('sort -nk 1 star_obs.dat > star_obs.dat.s')
npix = hp.nside2npix(nside)
keys = ('visitid', 'patchid', 'subpatch', 'nstars', 'ra', 'dec')
patchfile='patchdata.dat'
patchdat = ui.readDatafile(patchfile, keys)
ord = np.argsort(patchdat['patchid'])
for key in list(patchdat.keys()): patchdat[key]=patchdat[key][ord]
shift_size = shift_size*np.pi/180. #1.*np.pi/180.
#compute galactic coords for healpixels and patches
heal={}
heal['ra'], heal['dec'] = hp.pix2ang(nside, np.arange(npix))
heal['dec'] = heal['dec']-90.*np.pi/180.
heal['b'] = galac_b(heal['ra'], heal['dec']) #np.arcsin(np.sin(heal['dec'])*np.cos(62.6*np.pi/180.)-np.cos(heal['dec'])*np.sin(heal['ra']-282.25*np.pi/180.)*np.sin(62.6*np.pi/180.))
patchdat['dec'] = patchdat['dec']*np.pi/180.
patchdat['ra'] = patchdat['ra']*np.pi/180.
patchdat['b'] = galac_b(patchdat['ra'], patchdat['dec'])
patchdat['l'] = galac_l(patchdat['ra'], patchdat['dec'])
patchdat['dec']=patchdat['dec']+90.*np.pi/180. #stupid healpix wants 0-180 I think...
patch_healpix = hp.ang2pix(nside, patchdat['dec'], patchdat['ra'])
pyl.figure()
cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix)
pyl.savefig('heal_check.png', type='png')
patch_healpix_shifted1 = hp.ang2pix(nside, patchdat['dec']+shift_size, patchdat['ra']+shift_size*2)
patch_healpix_shifted2 = hp.ang2pix(nside, patchdat['dec']-shift_size, patchdat['ra']+shift_size*2)
patch_healpix_shifted3 = hp.ang2pix(nside, patchdat['dec']+shift_size, patchdat['ra']-shift_size*2)
patch_healpix_shifted4 = hp.ang2pix(nside, patchdat['dec']-shift_size, patchdat['ra']-shift_size*2)
pyl.figure()
cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix_shifted1)
pyl.savefig('heal_check1.png', type='png')
#ok, just go through and set duplicates to -1. Also set any of the shifted ones that are not in patch_healpix to -1
uhp = np.unique(patch_healpix)
patch_healpix_shifted1[np.invert(np.in1d(patch_healpix_shifted1, uhp))] = -1
patch_healpix_shifted2[np.invert(np.in1d(patch_healpix_shifted2, uhp))] = -1
patch_healpix_shifted3[np.invert(np.in1d(patch_healpix_shifted3, uhp))] = -1
patch_healpix_shifted4[np.invert(np.in1d(patch_healpix_shifted4, uhp))] = -1
diff = patch_healpix_shifted1-patch_healpix
good = np.where(diff == 0)
patch_healpix_shifted1[good] = -1
diff1 = patch_healpix_shifted2-patch_healpix
diff2 = patch_healpix_shifted2-patch_healpix_shifted1
good = np.where( (diff1 == 0) | (diff2 == 0))
patch_healpix_shifted2[good] = -1
diff1 = patch_healpix_shifted3-patch_healpix
diff2 = patch_healpix_shifted3-patch_healpix_shifted1
diff3 = patch_healpix_shifted3-patch_healpix_shifted2
good = np.where( (diff1 == 0) | (diff2 == 0) | (diff3 == 0))
patch_healpix_shifted3[good] = -1
diff1 = patch_healpix_shifted4-patch_healpix
diff2 = patch_healpix_shifted4-patch_healpix_shifted1
diff3 = patch_healpix_shifted4-patch_healpix_shifted2
diff4 = patch_healpix_shifted4-patch_healpix_shifted3
good = np.where( (diff1 == 0) | (diff2 == 0) | (diff3 == 0) | (diff4 == 0))
patch_healpix_shifted4[good] = -1
pyl.figure()
cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix_shifted1)
pyl.savefig('heal_check1_post.png', type='png')
heal_pixels = np.unique(patch_healpix)
keys = [patch_healpix, patch_healpix_shifted1, patch_healpix_shifted2,patch_healpix_shifted3,patch_healpix_shifted4]
filenames = ['hp1.dat','hp2.dat','hp3.dat','hp4.dat','hp5.dat']
#ok, if a healpix stradles b =0, I want to split it in two
# I could also just loop through each HP. if the HP has patches w/ b> 0 and b< 0 and min l < xxx or max l < xxx, then break it apart.
if splitNS:
break_up = np.where( (patchdat['b'] < 0) & ( (patchdat['l'] < 120.*np.pi/180.) | (patchdat['l'] > 240)*np.pi/180.) & (np.abs(patchdat['b']) < 5*np.pi/180.))
for key in keys:
mask=np.where(key == -1)
key[break_up] = key[break_up]+max(heal_pixels)+1
key[mask] = -1
print('reading star_obs.dat.s')
star_obs = open('star_obs.dat.s', 'r')
#read in just the patch id's from star_obs
all_patch_ids = []
for line in star_obs:
if line.startswith('#'):
continue
linevalues = line.split()
all_patch_ids.append(linevalues[0])
all_patch_ids = np.array(all_patch_ids, dtype=np.int)
star_obs.close()
for j in np.arange(4):
filename = filenames[j]
patch_healpix = keys[j]
hpid = all_patch_ids*0
left = np.searchsorted(all_patch_ids, patchdat['patchid'])
right = np.searchsorted(all_patch_ids, patchdat['patchid'], side='right')
for i in np.arange(np.size(left)):
hpid[left[i]:right[i]] = patch_healpix[i]
print('left %i, hpid %i, all_patch_ids %i, patch_healpix %i '%(np.size(left), np.size(hpid), np.size(all_patch_ids), np.size( patch_healpix)))
file = open(filename,'w')
print("#", file=file)
for value in hpid: print(value, file=file)
file.close()
#print j, np.size(hpid), np.size(all_patch_ids)
waitpop('paste hp1.dat star_obs.dat.s > hp1_star_obs.dat')
waitpop('paste hp2.dat star_obs.dat.s > hp2_star_obs.dat')
waitpop("sed -i '/^-1/d' hp2_star_obs.dat" )
waitpop('paste hp3.dat star_obs.dat.s > hp3_star_obs.dat')
waitpop("sed -i '/^-1/d' hp3_star_obs.dat" )
waitpop('paste hp4.dat star_obs.dat.s > hp4_star_obs.dat')
waitpop("sed -i '/^-1/d' hp4_star_obs.dat" )
waitpop('paste hp5.dat star_obs.dat.s > hp5_star_obs.dat')
waitpop("sed -i '/^-1/d' hp5_star_obs.dat" )
#waitpop("mv temp.dat hp2_star_obs.dat")
waitpop('cat hp2_star_obs.dat >> hp1_star_obs.dat')
waitpop('cat hp3_star_obs.dat >> hp1_star_obs.dat')
waitpop('cat hp4_star_obs.dat >> hp1_star_obs.dat')
waitpop('cat hp5_star_obs.dat >> hp1_star_obs.dat')
waitpop(''' awk '{print $2" "$3" "$4" "$5 " "$6> ("h"$1"star_obs.dat")}' hp1_star_obs.dat''')
waitpop('rm h#star_obs.dat')
waitpop('mv hp1_star_obs.dat star_obs_hp1.dat')
waitpop('mv hp2_star_obs.dat star_obs_hp2.dat')
waitpop('mv hp3_star_obs.dat star_obs_hp3.dat')
waitpop('mv hp4_star_obs.dat star_obs_hp4.dat')
waitpop('mv hp5_star_obs.dat star_obs_hp5.dat')
def starPlots(truestarfile="stardata.dat",
bestfitstarfile="test_bestfit_Star.dat"):
#assume both the files are sorted by star id.
stardat = cp.read_true_stars(
truestarfile
) #keys are ('id', 'dbid', 'ra', 'dec', 'magtrue', 'color')
starcal = ui.readDatafile(bestfitstarfile,
keys=('id', 'magcal', 'rms', 'iqr_sigma',
'nobs'))
HP = True
if np.size(starcal['id']) == 0:
starcal = ui.readDatafile(bestfitstarfile, keys=('id', 'magcal'))
HP = False
fitted_stars = np.in1d(stardat['id'], starcal['id'])
for key in list(stardat.keys()):
stardat[key] = stardat[key][fitted_stars]
for key in list(starcal.keys()):
stardat[key] = starcal[key]
stardat['magdiff'] = stardat['magcal'] - stardat['magtrue']
stardat = cp.adjust_stars_calvstrue(stardat)
starResults = {}
HPResults = {}
starResults['N stars Fit'] = np.size(stardat['magdiff'])
outlier_clip = np.where(np.abs(stardat['magdiff']) < .05)
rms = stardat['magdiff'][outlier_clip].std()
print('true-bestfit RMS = %f, true-bestfit (clipped) = %f' %
(stardat['magdiff'].std(), rms))
lim = rms * 2.5
if lim < 0.001:
lim = round(lim * 10000) / 10000.0
elif lim < 0.01:
lim = round(lim * 1000) / 1000.0
print("Using magnitude limits for plots of %f, %f" % (-lim, lim))
maglim = [-lim, lim]
starResults['Residuals RMS (mmag)'] = stardat['magdiff'].std() * 1e3
starResults['IQR_sigma (mmag)'] = so.robustSigma(stardat['magdiff']) * 1e3
#star residual map
stardat['x'], stardat['y'] = cpu.hammer_project_toxy(
stardat['ra'] * deg2rad, stardat['dec'] * deg2rad)
gridsize = cpu.calc_gridsize(stardat, binsize='patch')
etitle = " RMS = %4.1f mmag" % (rms * 1000)
cpu.plot_density(stardat['x'],
stardat['y'],
stardat['magdiff'] * 1000,
z_method=np.mean,
zlimits=np.array(maglim) * 1000,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='mmag')
figtitle = "Stars dMag(true-bestfit)"
pyl.title(figtitle, fontsize='medium')
pyl.savefig('Sdmag.png', type='png')
maglim2 = [0, maglim[1] / 2.]
cp.plot_stars_dmagcaltrue(stardat,
maglim=maglim2,
etitle=" RMS " + etitle,
z_method=np.std)
pyl.savefig('Sdmag_rms.png', format='png')
histrange = [maglim[0] * 2.5, maglim[1] * 2.5]
nbins = 100
cp.histogram_stars_dmagcaltrue(stardat,
nbins=nbins,
range=histrange,
etitle=etitle)
pyl.savefig('Sdamg_hist.png', format='png')
#color distribution of stars
pyl.figure()
pyl.hist(stardat['color'], bins=100)
pyl.xlabel(r'$g-i$')
pyl.ylabel('# of stars')
pyl.title('Color Distribution')
pyl.savefig('Scolordist.png', type='png')
#accuracy v color
pyl.figure()
#pyl.scatter(stardat['color'], stardat['magdiff']*1000., c=stardat['magtrue'], alpha=0.1,edgecolors=None)
pyl.hexbin(stardat['color'], stardat['magdiff'] * 1000., bins='log')
cb = pyl.colorbar()
cb.ax.set_ylabel(r'$\log{\rm{N}}$')
pyl.xlabel(r'$g-i$')
pyl.ylabel('Bestfit-True (mmag)')
pyl.savefig('Saccuracyvcolor.png', type='png')
#accuracy v mag
pyl.figure()
#pyl.scatter(stardat['magtrue'], stardat['magdiff']*1000., c=stardat['color'], alpha=0.1,edgecolors=None)
pyl.hexbin(stardat['magtrue'], stardat['magdiff'] * 1000., bins='log')
cb = pyl.colorbar()
cb.ax.set_ylabel(r'$\log{\rm{N}}$')
pyl.xlabel('True Mag')
pyl.ylabel('Bestfit-True (mmag)')
pyl.savefig('Saccuracyvmag.png', type='png')
if HP:
zlim = [
np.min(stardat['nobs']),
(np.median(stardat['nobs']) - np.min(stardat['nobs'])) +
np.median(stardat['nobs'])
]
cpu.plot_density(stardat['x'],
stardat['y'],
stardat['nobs'],
z_method=np.mean,
zlimits=zlim,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='N observations')
pyl.title('Visit Density')
pyl.savefig('Snobs.png', format='png')
pyl.figure()
#I should just figure out from the data what nsides is!
hpid = hp.ang2pix(16, (stardat['dec'] + 90.) * deg2rad,
stardat['ra'] * deg2rad)
cpu.plot_density(stardat['x'],
stardat['y'],
np.mod(hpid, 250),
z_method=np.median,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='HEALpix ID mod 250')
pyl.title("HEALpix Map")
pyl.savefig('HPid.png')
HPResults['number of HEALpix'] = np.size(np.unique(hpid))
pyl.figure()
num, b, p = pyl.hist(stardat['nobs'],
bins=100,
range=zlim,
histtype='bar')
pyl.xlabel("Number of Observations per Star")
pyl.title('Median = %d' % np.median(stardat['nobs']))
pyl.savefig('Snobs_hist.png', type='png')
starResults['median repeat obs'] = np.median(stardat['nobs'])
pyl.figure()
x = np.sort(stardat['iqr_sigma']) * 1e3
y = np.arange(np.size(x), dtype='float')
y = y / np.max(y)
per50 = np.max(np.where(y <= .5))
per90 = np.max(np.where(y <= .9))
pyl.plot(x,
y,
'k--',
label='IQR, 50th,90th %2.1f, %2.1f' % (x[per50], x[per90]))
pyl.plot([0, x[per50]], [y[per50], y[per50]], 'b--')
pyl.plot([x[per50], x[per50]], [0, y[per50]], 'b--')
pyl.plot([0, x[per90]], [y[per90], y[per90]], 'b--')
pyl.plot([x[per90], x[per90]], [0, y[per90]], 'b--')
#pyl.title('Cumulative Distribution of Stellar Repeatability')
#pyl.xlabel('Repeatability IQR RMS (mmag)')
#pyl.ylabel('Cumulative Fraction')
#pyl.savefig('Srepeat_IQR_cumulative.png',type='png')
pyl.legend()
pyl.savefig('Srepeat_cumulative.png', type='png')
#repeatability from IQR
rs = so.robustSigma(stardat['iqr_sigma'])
med = np.median(stardat['iqr_sigma'])
maglim = np.around(np.array([med - 3 * rs, med + 3. * rs]) * 1000,
decimals=2) / 1000
cpu.plot_density(stardat['x'],
stardat['y'],
stardat['iqr_sigma'] * 1000,
z_method=np.mean,
zlimits=maglim * 1000,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='mmag')
pyl.title('Repeatability (IQR)')
pyl.savefig('Srepeat_IQR.png', type='png')
pyl.figure()
num, b, p = pyl.hist(stardat['iqr_sigma'] * 1000,
bins=100,
range=maglim * 1000,
histtype='bar')
pyl.xlabel('RMS per star (mmag)')
pyl.title('Repeatability IQR, Median = %4.2f mmag' %
np.median(stardat['iqr_sigma'] * 1000))
pyl.savefig('Srepeat_IQR_hist.png', type='png')
starResults['Median Repeatability (IQR) (mmag)'] = np.median(med) * 1e3
pyl.figure()
good = np.where(stardat['magtrue'] < 18)
num, b, p = pyl.hist(stardat['iqr_sigma'][good] * 1000,
bins=100,
range=maglim * 1000,
histtype='bar')
pyl.xlabel('RMS per star, m < 18 (mmag)')
pyl.title('Repeatability IQR, Median = %4.2f mmag' %
np.median(stardat['iqr_sigma'][good] * 1000))
pyl.savefig('Srepeat_IQR_bright_hist.png', type='png')
starResults['Median Repeatability Bright (IQR) (mmag)'] = np.median(
stardat['iqr_sigma'][good]) * 1e3
#repeatability v color
pyl.figure()
#pyl.plot(stardat['color'], stardat['rms']*1000., 'ko', alpha=.1)
pyl.hexbin(stardat['color'], stardat['iqr_sigma'] * 1000., bins='log')
cb = pyl.colorbar()
cb.set_label(r'$\log{\rm{N}}$')
pyl.ylim([0, 40])
pyl.xlabel(r'$g-i$')
pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)')
pyl.savefig('Srepeatvcolor.png', type='png')
#repeatability v mag
pyl.figure()
#pyl.plot(stardat['magtrue'], stardat['rms']*1000., 'ko', alpha=.1)
pyl.hexbin(stardat['magtrue'],
stardat['iqr_sigma'] * 1000.,
bins='log')
cb = pyl.colorbar()
cb.set_label(r'$\log{\rm{N}}$')
pyl.ylim([0, 40])
pyl.xlabel('True Mag')
pyl.ylabel(r'Repeatability $\sigma_{\rm{IQR}}$ (mmag)')
pyl.savefig('Srepeatvmag.png', type='png')
#repeatability
rs = so.robustSigma(stardat['rms'])
med = np.median(stardat['rms'])
maglim = np.around(np.array([med - 3 * rs, med + 3. * rs]) * 1000,
decimals=2) / 1000
cpu.plot_density(stardat['x'],
stardat['y'],
stardat['rms'] * 1000,
z_method=np.mean,
zlimits=maglim * 1000,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='mmag')
pyl.title('Repeatability')
pyl.savefig('Srepeat.png', type='png')
pyl.figure()
num, b, p = pyl.hist(stardat['rms'] * 1000,
bins=100,
range=maglim * 1000,
histtype='bar')
pyl.xlabel('RMS per star (mmag)')
pyl.title('Repeatability, Median = %4.2f mmag' %
np.median(stardat['rms'] * 1000))
pyl.savefig('Srepeat_hist.png', type='png')
starResults['Median Repeatability (mmag)'] = np.median(med) * 1e3
pyl.figure()
x = np.sort(stardat['rms']) * 1e3
y = np.arange(np.size(x), dtype='float')
y = y / np.max(y)
per50 = np.max(np.where(y <= .5))
per90 = np.max(np.where(y <= .9))
pyl.plot(x, y, 'k', label='RMS, %2.1f, %2.1f' % (x[per50], x[per90]))
pyl.plot([0, x[per50]], [y[per50], y[per50]], 'k')
pyl.plot([x[per50], x[per50]], [0, y[per50]], 'k')
pyl.plot([0, x[per90]], [y[per90], y[per90]], 'k')
pyl.plot([x[per90], x[per90]], [0, y[per90]], 'k')
pyl.title('Cumulative Distribution of Stellar Repeatability')
pyl.xlabel('Repeatability RMS (mmag)')
pyl.ylabel('Cumulative Fraction')
pyl.savefig('Srepeat_cumulative.png', type='png')
#need to make a spatial uniformity plot since we can now have varying densities of stars.
nside = 64
stardat['hpid'] = hp.ang2pix(nside, (stardat['dec'] + 90.) * deg2rad,
stardat['ra'] * deg2rad)
uhpid = np.unique(stardat['hpid'])
hp_mean = uhpid * 0.
hp_std = uhpid * 0.
hp_dec, hp_ra = hp.pix2ang(nside, uhpid)
#hp_ra = hp_ra
hp_dec = hp_dec - 90. * deg2rad
hp_x, hp_y = cpu.hammer_project_toxy(hp_ra, hp_dec)
stardat = dictsort(stardat, 'hpid')
left = np.searchsorted(stardat['hpid'], uhpid)
right = np.searchsorted(stardat['hpid'], uhpid, side='right')
for i in np.arange(left.size):
hp_mean[i] = np.mean(stardat['magdiff'][left[i]:right[i]])
hp_std[i] = np.std(stardat['magdiff'][left[i]:right[i]])
cpu.plot_density(hp_x,
hp_y,
hp_mean * 1000,
z_method=np.mean,
zlimits=None,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='mmag')
HPResults['Spatial Uniformity RMS (mmag)'] = np.std(hp_mean) * 1e3
HPResults['Spatial Uniformity IQR RMS (mmag)'] = so.robustSigma(
hp_mean) * 1e3
pyl.title('mean(True-Bestfit), binned by HEALpix')
pyl.savefig('HPresid.png', type='png')
cpu.plot_density(hp_x,
hp_y,
hp_std * 1000,
z_method=np.mean,
zlimits=None,
gridsize=gridsize,
radecgrid=True,
newfig=True,
cb_label='mmag')
pyl.title('std(True-Bestfit), binned by HEALpix')
pyl.savefig('HPstd.png', type='png')
pyl.figure()
num, b, p = pyl.hist(hp_mean * 1000,
bins=100,
range=None,
histtype='bar')
pyl.xlabel("HEALpix(True-Bestfit) (mmag)")
pyl.title('Spatial Uniformity, RMS=%4.2f' %
HPResults['Spatial Uniformity RMS (mmag)'])
pyl.savefig('HPmean_hist.png', type='png')
pyl.figure()
num, b, p = pyl.hist(hp_std * 1000,
bins=100,
range=None,
histtype='bar')
pyl.xlabel("std(True-Bestfit) per HEALpix (mmag)")
pyl.title('Variation within HEALpix, median=%4.2f mmag' %
np.median(hp_std * 1000))
pyl.savefig('HPstd_hist.png', type='png')
return starResults, HPResults
def read_true_stars(stardatfile='stardata.dat'):
"""Read a stardata.dat file from simSelfCalib.py"""
keys = ('id', 'dbid', 'ra', 'dec', 'magtrue', 'color')
stardat = ui.readDatafile(stardatfile, keys)
return stardat
def split_obs(nside,
shift_size=5.,
splitNS=False,
NorthOnly=False,
SouthOnly=False,
blocksize=1e6):
""" XXX kill this function XXX replaced by just including neighboring healpixels"""
"""nside gives the number of sides for healpixels. shift_size is in degrees and sets how """
#XXX-prob should put in a check so I don't sort repeatedly
#if not os.path.isfile('star_obs.dat.s'):
waitpop('sort -nk 1 star_obs.dat > star_obs.dat.s')
npix = hp.nside2npix(nside)
keys = ('visitid', 'patchid', 'subpatch', 'nstars', 'ra', 'dec')
patchfile = 'patchdata.dat'
patchdat = ui.readDatafile(patchfile, keys)
ord = np.argsort(patchdat['patchid'])
for key in list(patchdat.keys()):
patchdat[key] = patchdat[key][ord]
shift_size = shift_size * np.pi / 180. #1.*np.pi/180.
#compute galactic coords for healpixels and patches
heal = {}
heal['ra'], heal['dec'] = hp.pix2ang(nside, np.arange(npix))
heal['dec'] = heal['dec'] - 90. * np.pi / 180.
heal['b'] = galac_b(
heal['ra'], heal['dec']
) #np.arcsin(np.sin(heal['dec'])*np.cos(62.6*np.pi/180.)-np.cos(heal['dec'])*np.sin(heal['ra']-282.25*np.pi/180.)*np.sin(62.6*np.pi/180.))
patchdat['dec'] = patchdat['dec'] * np.pi / 180.
patchdat['ra'] = patchdat['ra'] * np.pi / 180.
patchdat['b'] = galac_b(patchdat['ra'], patchdat['dec'])
patchdat['l'] = galac_l(patchdat['ra'], patchdat['dec'])
patchdat['dec'] = patchdat[
'dec'] + 90. * np.pi / 180. #stupid healpix wants 0-180 I think...
patch_healpix = hp.ang2pix(nside, patchdat['dec'], patchdat['ra'])
pyl.figure()
cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix)
pyl.savefig('heal_check.png', type='png')
patch_healpix_shifted1 = hp.ang2pix(nside, patchdat['dec'] + shift_size,
patchdat['ra'] + shift_size * 2)
patch_healpix_shifted2 = hp.ang2pix(nside, patchdat['dec'] - shift_size,
patchdat['ra'] + shift_size * 2)
patch_healpix_shifted3 = hp.ang2pix(nside, patchdat['dec'] + shift_size,
patchdat['ra'] - shift_size * 2)
patch_healpix_shifted4 = hp.ang2pix(nside, patchdat['dec'] - shift_size,
patchdat['ra'] - shift_size * 2)
pyl.figure()
cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix_shifted1)
pyl.savefig('heal_check1.png', type='png')
#ok, just go through and set duplicates to -1. Also set any of the shifted ones that are not in patch_healpix to -1
uhp = np.unique(patch_healpix)
patch_healpix_shifted1[np.invert(np.in1d(patch_healpix_shifted1,
uhp))] = -1
patch_healpix_shifted2[np.invert(np.in1d(patch_healpix_shifted2,
uhp))] = -1
patch_healpix_shifted3[np.invert(np.in1d(patch_healpix_shifted3,
uhp))] = -1
patch_healpix_shifted4[np.invert(np.in1d(patch_healpix_shifted4,
uhp))] = -1
diff = patch_healpix_shifted1 - patch_healpix
good = np.where(diff == 0)
patch_healpix_shifted1[good] = -1
diff1 = patch_healpix_shifted2 - patch_healpix
diff2 = patch_healpix_shifted2 - patch_healpix_shifted1
good = np.where((diff1 == 0) | (diff2 == 0))
patch_healpix_shifted2[good] = -1
diff1 = patch_healpix_shifted3 - patch_healpix
diff2 = patch_healpix_shifted3 - patch_healpix_shifted1
diff3 = patch_healpix_shifted3 - patch_healpix_shifted2
good = np.where((diff1 == 0) | (diff2 == 0) | (diff3 == 0))
patch_healpix_shifted3[good] = -1
diff1 = patch_healpix_shifted4 - patch_healpix
diff2 = patch_healpix_shifted4 - patch_healpix_shifted1
diff3 = patch_healpix_shifted4 - patch_healpix_shifted2
diff4 = patch_healpix_shifted4 - patch_healpix_shifted3
good = np.where((diff1 == 0) | (diff2 == 0) | (diff3 == 0) | (diff4 == 0))
patch_healpix_shifted4[good] = -1
pyl.figure()
cpu.plot_density(patchdat['ra'], patchdat['dec'], z=patch_healpix_shifted1)
pyl.savefig('heal_check1_post.png', type='png')
heal_pixels = np.unique(patch_healpix)
keys = [
patch_healpix, patch_healpix_shifted1, patch_healpix_shifted2,
patch_healpix_shifted3, patch_healpix_shifted4
]
filenames = ['hp1.dat', 'hp2.dat', 'hp3.dat', 'hp4.dat', 'hp5.dat']
#ok, if a healpix stradles b =0, I want to split it in two
# I could also just loop through each HP. if the HP has patches w/ b> 0 and b< 0 and min l < xxx or max l < xxx, then break it apart.
if splitNS:
break_up = np.where((patchdat['b'] < 0)
& ((patchdat['l'] < 120. * np.pi / 180.)
| (patchdat['l'] > 240) * np.pi / 180.)
& (np.abs(patchdat['b']) < 5 * np.pi / 180.))
for key in keys:
mask = np.where(key == -1)
key[break_up] = key[break_up] + max(heal_pixels) + 1
key[mask] = -1
print('reading star_obs.dat.s')
star_obs = open('star_obs.dat.s', 'r')
#read in just the patch id's from star_obs
all_patch_ids = []
for line in star_obs:
if line.startswith('#'):
continue
linevalues = line.split()
all_patch_ids.append(linevalues[0])
all_patch_ids = np.array(all_patch_ids, dtype=np.int)
star_obs.close()
for j in np.arange(4):
filename = filenames[j]
patch_healpix = keys[j]
hpid = all_patch_ids * 0
left = np.searchsorted(all_patch_ids, patchdat['patchid'])
right = np.searchsorted(all_patch_ids,
patchdat['patchid'],
side='right')
for i in np.arange(np.size(left)):
hpid[left[i]:right[i]] = patch_healpix[i]
print('left %i, hpid %i, all_patch_ids %i, patch_healpix %i ' %
(np.size(left), np.size(hpid), np.size(all_patch_ids),
np.size(patch_healpix)))
file = open(filename, 'w')
print("#", file=file)
for value in hpid:
print(value, file=file)
file.close()
#print j, np.size(hpid), np.size(all_patch_ids)
waitpop('paste hp1.dat star_obs.dat.s > hp1_star_obs.dat')
waitpop('paste hp2.dat star_obs.dat.s > hp2_star_obs.dat')
waitpop("sed -i '/^-1/d' hp2_star_obs.dat")
waitpop('paste hp3.dat star_obs.dat.s > hp3_star_obs.dat')
waitpop("sed -i '/^-1/d' hp3_star_obs.dat")
waitpop('paste hp4.dat star_obs.dat.s > hp4_star_obs.dat')
waitpop("sed -i '/^-1/d' hp4_star_obs.dat")
waitpop('paste hp5.dat star_obs.dat.s > hp5_star_obs.dat')
waitpop("sed -i '/^-1/d' hp5_star_obs.dat")
#waitpop("mv temp.dat hp2_star_obs.dat")
waitpop('cat hp2_star_obs.dat >> hp1_star_obs.dat')
waitpop('cat hp3_star_obs.dat >> hp1_star_obs.dat')
waitpop('cat hp4_star_obs.dat >> hp1_star_obs.dat')
waitpop('cat hp5_star_obs.dat >> hp1_star_obs.dat')
waitpop(
''' awk '{print $2" "$3" "$4" "$5 " "$6> ("h"$1"star_obs.dat")}' hp1_star_obs.dat'''
)
waitpop('rm h#star_obs.dat')
waitpop('mv hp1_star_obs.dat star_obs_hp1.dat')
waitpop('mv hp2_star_obs.dat star_obs_hp2.dat')
waitpop('mv hp3_star_obs.dat star_obs_hp3.dat')
waitpop('mv hp4_star_obs.dat star_obs_hp4.dat')
waitpop('mv hp5_star_obs.dat star_obs_hp5.dat')
def read_starsobs(starobsfile='star_obs.dat'):
"""Read the star_obs.dat file from simSelfCalib.py that is input to simple.x"""
keys = ('patchid', 'starid', 'rmagobs', 'magerr')
starobs = ui.readDatafile(starobsfile, keys)
return starobs
def read_visitfile(visitfile='visit.dat'):
"""Read the visit information from simSelfCalib.py"""
keys = ('visitid', 'ra', 'dec', 'skyrot', 'time', 'zpOff', 'zpGrad',
'zpGradDir', 'colorterm', 'colortermRad')
visits = ui.readDatafile(visitfile, keys)
return visits