def lacosmic_on_filters(args):
obj_list=set_paths_and_params()['objects']
filter=args[0]
create_pngs=args[1]
temp_folder=args[2]
data_dir=set_paths_and_params()['data']+'/data'
#filters=glob.glob(data_dir+'/'+object+'/*')
#filters_list = [path.replace(data_dir+'/'+object+'/','') for path in filters]
dirs_list=[data_dir+'/'+obj+'/'+filter for obj in obj_list]
if temp_folder:
dirs_list=[d + '/temp_for_photometry' for d in dirs_list]
param_dict = lacosmic_param_dictionary()
for dir in dirs_list:
if 'GRW70' in dir:
object = 'GRW70'
if 'GD153' in dir:
object = 'GD153'
fits_list = glob.glob(str(dir)+'/'+'*flt.fits')
if len(fits_list) == 0 :
print 'no temp directory for / empty temp directory' + dir
continue
print 'entering filter directory ' + dir
if filter not in param_dict.keys():
print "Filter {0} not in Param Dictionary. Using default values.".format(filter)
sigclip, objlim = 5.0,2
sigfrac,niter,sigclip = 0.3, 3, 9.5
sigclip_pf=9.5
else:
sigclip = param_dict[filter][0]
sigfrac = param_dict[filter][1]
objlim = param_dict[filter][2]
niter = param_dict[filter][3]
sigclip_pf = param_dict[filter][4]
for fits in fits_list:
if temp_folder:
origin=data_dir+'/'+str(object)+'/'+str(filter)+'/temp_for_photometry/'
cleanfile = fits.replace('_flt.fits','_flt.clean.fits')
folder_cleanfile=cleanfile.replace('/temp_for_photometry/','/temp_for_photometry/flt_cleans/')
if os.path.isfile(cleanfile):
print cleanfile + ' this already exists in working dir! moving on'
continue
if os.path.isfile(folder_cleanfile):
print folder_cleanfile + ' this already exists in flt_cleans, moving on'
continue
else:
origin=data_dir+'/'+str(object)+'/'+str(filter)
print 'working on ' , fits
run_lacosmic(fits, sigclip, sigfrac, objlim, niter, sigclip_pf)
#now, sort the files!
sort_files(origin,temp_folder,create_pngs)
def read_montecarlo(obj):
""" Reads in the montecarlo data table for either GD153 and GRW70.
make sure the header for that file is:
Filter , Amp , slope_yr , mean_mc , stdev.
The existing script that generates the table outputs a different header to be pasted into
latex so swap it with that.
Returns:
pandas dataframe with filters, chips, and the MC standard deviations
"""
plot_output_dir = set_paths_and_params()['plot_output_dir']
datfile=plot_output_dir+'/monte_output/'+obj+'_monteCarloTable.txt' #path to where montecarlo table lives
df = pd.read_csv(datfile, delimiter = ' & ')
print df
filters=np.array(df['Filter'].tolist())
amps=np.array(df['Amp'].tolist())
stdevs=np.array(df['stdev'].tolist())
dat = {'filters':filters,'amps':amps,'stdevs':stdevs}
df = pd.DataFrame(data=dat,index=None)
return df
def make_plots(df, obj): df = df[df['objs']==obj] slopes=df['slopes'].values amps=df['amps'].values filters = df['filters'].values filter_indicies=df['filter_indicies'].values slopes_chip_A = slopes[amps=='A'] filters_chip_A = filters[amps=='A'] print filters_chip_A filter_indicies_chipA=filter_indicies[amps=='A'] print 'chip A' for i, filt in enumerate(filters_chip_A): print filt, ' ' , slopes_chip_A[i] slopes_chip_C = slopes[amps=='C'] filters_chip_C =filters[amps=='C'] print filters_chip_C filter_indicies_chipC=filter_indicies[amps=='C'] print 'chip C' for i, filt in enumerate(filters_chip_C): print filt, ' ' ,slopes_chip_C[i] #now find errorbars errs_a=[] errs_c=[] mc_data=read_montecarlo(obj) for val in filters_chip_A: print val print return_matching_stdev(mc_data,'A',val) errs_a.append(return_matching_stdev(mc_data,'A',val)) for i, val in enumerate(filters_chip_A): errs_c.append(return_matching_stdev(mc_data,'A',val)) #plt.scatter(filter_indicies_chipA,slopes_chip_A, c='r',marker='^', s=40,label = 'Quad. A',yerr=errs_a) #plt.scatter(filter_indicies_chipC,slopes_chip_C,marker='^', s=40, c='b',label = 'Quad. C',yerr_errs_c) plt.errorbar(filter_indicies_chipA,slopes_chip_A,yerr=errs_a,fmt='r^',label = 'Quad. A') plt.errorbar(filter_indicies_chipC,slopes_chip_C,yerr=errs_c,fmt='b^',label = 'Quad. C') selected_filters=choose_filters() plt.xticks(np.arange(len(selected_filters)),selected_filters) plt.ylabel(r'$\Delta$ Flux [\% per year]',fontsize=20) plt.ylim(-0.7,0.2) plt.xlim(-0.5,7) plt.xlabel(r'Filter',fontsize=20) plt.text(0.02,0.07,obj,fontsize=20) plt.legend(loc='best') plt.savefig(set_paths_and_params()['plot_output_dir']+'/'+obj+'_slope_vs_filter.png') plt.show()
def main_run_lacosmic_on_filters(): data_dir=set_paths_and_params()['data'] #make sure to set the correct object! objects = set_paths_and_params()['objects'] filters_list=set_paths_and_params()['filters'] create_pngs=[False]*len(filters_list) temp_for_photometry=[True]*len(filters_list) args=zip(filters_list,create_pngs,temp_for_photometry) p = Pool(8) p.map(lacosmic_on_filters,args) for filt in filters_list: lacosmic_on_filters((filt,False,True))
plt.errorbar(filter_indicies_chipA,slopes_chip_A,yerr=errs_a,fmt='r^',label = 'Quad. A') plt.errorbar(filter_indicies_chipC,slopes_chip_C,yerr=errs_c,fmt='b^',label = 'Quad. C') selected_filters=choose_filters() plt.xticks(np.arange(len(selected_filters)),selected_filters) plt.ylabel(r'$\Delta$ Flux [\% per year]',fontsize=20) plt.ylim(-0.7,0.2) plt.xlim(-0.5,7) plt.xlabel(r'Filter',fontsize=20) plt.text(0.02,0.07,obj,fontsize=20) plt.legend(loc='best') plt.savefig(set_paths_and_params()['plot_output_dir']+'/'+obj+'_slope_vs_filter.png') plt.show() def main_make_plot_slope_vs_filter(ifile): df = read_in_stats_file(ifile) make_plots(df, 'GRW70') make_plots(df, 'GD153') if __name__=='__main__': plot_output_dir = set_paths_and_params()['plot_output_dir'] #where the stats.dat file is main_make_plot_slope_vs_filter(plot_output_dir+'/stats.dat')
filters = paths['filters']
data_dir = paths['data']+'/data'
if temp_dir:
strr = 'temp_for_photometry'
else:
strr = ''
for ob in objects:
for filterr in filters:
dir = data_dir+'/{0}/{1}/{2}/flt_cleans'.format(ob,filterr,strr)
phot_file_path = data_dir+'/{0}/{1}/flt_cleans/{1}_photcat.dat'.format(ob,filterr,strr)
ifiles = glob.glob(dir+'/*clean.fits')
if len(ifiles) > 0:
print 'beginning photometry on {0}, {1}'.format(ob,filterr)
main_ptsrc_photometry(ifiles,phot_file_path, paths, PAMcorr)
else:
print 'no files. moving on to next filter.'
continue
if __name__ == '__main__':
paths = set_paths_and_params()
wrapper_ptsrc_photometry_photutils(paths,temp_dir = True)
stuff_in_flt_cleans = glob.glob(temp_dir + 'flt_cleans/*.fits')
#make a permanent flt_cleans in the filter directory if it doesn't already exist
if not isdir(str(dir) + '/flt_cleans'):
os.makedirs(dir + '/flt_cleans')
print 'making directory ' + dir + '/flt_cleans'
for thing in stuff_in_flt_cleans:
print 'moving ' + thing + ' to ' + dir + '/flt_cleans'
if '.dat' not in thing: #don't overwrite photcat just in case !
shutil.move(thing, dir + '/flt_cleans')
else:
os.remove(thing)
print 'removing temp catalog in temp /flt_cleans'
###same process for flt_masks
stuff_in_flt_masks = glob.glob(temp_dir + 'flt_masks/*')
if not isdir(str(dir) + '/flt_masks'):
os.makedirs(dir + '/flt_masks')
print 'making directory ' + dir + '/flt_masks'
for thing in stuff_in_flt_masks:
print 'moving ' + thing + ' to ' + dir + '/flt_masks'
shutil.move(thing, dir + '/flt_masks')
if __name__ == '__main__':
clean_temp_directories(set_paths_and_params())
def main_uvis_contam_monitor(paths, calibrate_raws=True):
#######################################################
# query the QL database for new data #
#######################################################
#unpack paths & parameters - from set_paths_and_params.py
data_dir = paths['data'] + '/data'
new_data_dir = paths['data'] + '/new_flts'
raw_dir = paths['data'] + '/new_raws'
filters = paths['filters']
objects = paths['objects']
proposal_ids = paths['prop_ids']
cal_vers = paths['cal_ver']
#Iterate over proposal IDS to search for unprocessed files in the QL database
print 'Querying the QL database for FLTs from contam monitor programs.'
for proposal in proposal_ids:
flts = query_ql(proposal, object_names())
print len(flts), ' total observations in proposal ', proposal
if len(flts) == 0:
continue
#check all files in QL directories for proposal against files in data directory
new_flts = check_new_files_against_existing_files(
flts, data_dir, new_data_dir, raw_dir)
if len(new_flts) == 0:
print 'no new files found for proposal {0}'.format(proposal)
continue
else:
print 'found ' + str(
len(new_flts)) + ' new files for proposal {0}'.format(proposal)
cal_version_flts = check_cal_version(new_flts, cal_vers)
new_flts = cal_version_flts[
0] #files calibrated with correct version of calwf3
print len(
new_flts
), ' of the new files were calibrated with the correct version of the pipeline. adding files to new data directory.'
for new_flt in new_flts:
shutil.copy(new_flt, new_data_dir)
wrong_version_flts = cal_version_flts[1]
wrong_version_raws = [
f.replace('flt', 'raw') for f in wrong_version_flts
]
print len(
wrong_version_flts
), ' of the new flts found were not calibrated with the correct version of calwf3 specified in set_paths_and_params.py. Copying the raw files to new_raws for calibration.'
for raw in wrong_version_raws:
shutil.copy(raw, raw_dir)
new_flts = glob.glob(new_data_dir + '/*flt.fits')[0:10]
new_raws = glob.glob(raw_dir + '/*flt.fits')
#######################################################
# calibrate the raw files if there are any #
#######################################################
"""if calibrate_raws:
print 'Calibrating RAW files.'
cwd = os.getcwd()
os.chdir(raw_dir)
print os.getcwd()
os.system('python ' + 'run_calwf3.py')
os.chdir(cwd)"""
#######################################################
# now organize the newly fetched data #
#######################################################
if len(new_flts) > 0:
#put into temp holding areas for lacosmic / photometry
print 'organizing new FLTs into proper star and filter subdirectories.'
organize_flts(new_data_dir, data_dir, photometry_temp_directory=True)
if len(new_raws) > 0:
print 'organizing new FLTs into proper star and filter subdirectories.'
organize_flts(raw_dir, data_dir, photometry_temp_directory=True)
print "done organizing new FLTs."
else:
print 'no new FLTs to organize.'
#######################################################
# Run LAcosmic on the new data #
#######################################################
print 'Cleaning cosmic rays from new images.'
main_run_lacosmic_on_filters()
print 'Done cleaning cosmic rays from new images.'
#######################################################
# Run the photometry #
#######################################################
print 'Beginning photometry on new images.'
wrapper_ptsrc_photometry_photutils(set_paths_and_params())
#main_photom()
print 'Photometry complete.'
#######################################################
# Clean up temp directories #
#######################################################
print 'Cleaning up temporary directories.'
clean_temp_directories(paths)
#######################################################
# Generate plots #
#######################################################
#print 'Generating plots and stats file...'
main_wrapper_make_uvis_contam_plots(paths)
"""Reads in all the stats files from the monte-carlo simulations and outputs the info to
a text file. This is to make generating a latex table for the ISR easier so I don't
have to open each file!"""
import glob
import numpy as np
from set_paths_and_params import *
plot_output_dir = set_paths_and_params()['plot_output_dir'] + '/'
#read in my stats file
#obj,amp,filter,slope_withWeights,b_withWeights,slope_noWeights,b_noWeights,stdev
my_objs = []
my_amps = []
my_filters = []
my_slopes = []
with open(set_paths_and_params()['plot_output_dir'] + '/stats.dat', 'r') as f:
lines = f.readlines()[1:]
for line in lines:
line = line.split(',')
my_objs.append(line[0])
my_amps.append(line[1])
my_filters.append(line[2])
my_slopes.append(line[3])
stars = ['GD153', 'GRW70']
objects = []
filters = []
amps = []
with open(plot_output_dir + '/stats.dat', 'w') as f:
f.write('#obj,amp,filter,slope_mjd,b,stdev_mjd,slope_yr,stdev_yr\n')
for obj in objects:
print obj
paths = [
data_dir + '/data/{0}/{1}/flt_cleans'.format(obj, f)
for f in filters
]
for path in paths:
ifile = glob.glob(path + '/*.dat')
if len(ifile) > 0:
print ifile[0]
filename = os.path.basename(ifile[0])
#filtername = filename.replace('_photcat.dat','')
filtername = filename.replace('_photcat.dat', '')
#make plots
main_fluxDif_MJD(ifile[0],
filtername,
obj,
plot_output_dir,
wrapper=True)
else:
print 'no photcat. moving on'
if __name__ == '__main__':
main_wrapper_make_uvis_contam_plots(set_paths_and_params())
def run_ptsrc_photometry(ifile, PAMcorr=True):
#for i, ifile in enumerate(ifiles):
paths = set_paths_and_params()
try:
hdu_list = fits.open(ifile)
except IOError:
print 'fits file corrupted. removing file'
os.remove(ifile)
return 0
header = hdu_list[0].header
sci = hdu_list[0].data
filename = os.path.basename(ifile)
fits_data = (hdu_list, filename)
filename = os.path.basename(ifile)[0:9]
aperture_sizes = set_paths_and_params()['aperture_sizes']
obs_info = fetch_header_data(ifile, hdu_list, paths)
if not obs_info:
return 0
amp = obs_info[1]
coo_tab = run_dao_star_finder(ifile, filename, hdu_list)
if coo_tab: #if a single source was found
xc, yc = coo_tab['xcentroid'][0], coo_tab['ycentroid'][0]
obs_info.append(xc)
obs_info.append(yc)
back, backrms = calculate_bkgrnd(hdu_list, filename, coo_tab)
obs_info.append(back)
obs_info.append(backrms)
if PAMcorr:
hdu_list.close()
make_PAMcorr_image(ifile, outfile=ifile + '.pamcorr', extension=0)
ifile = ifile + '.pamcorr'
hdu_list = fits.open(ifile)
header = hdu_list[0].header
sci = hdu_list[0].data
fluxes, flux_errs, mags, mag_errs = [], [], [], []
for ap_size in aperture_sizes:
flux, flux_err,mag, merr = run_aperture_photometry(ifile, hdu_list, \
coo_tab, back, backrms, ap_size)
fluxes.append(flux)
flux_errs.append(flux_err)
mags.append(mag)
mag_errs.append(merr)
#print mag, merr
for i, val in enumerate(fluxes):
obs_info.append(fluxes[i])
obs_info.append(flux_errs[i])
for i, val in enumerate(mags):
obs_info.append(mags[i])
obs_info.append(mag_errs[i])
else:
hdu_list.close()
return 0
hdu_list.close()
#if PAMcorr:
#os.remove(ifile)
print 'done processing ', ifile
return obs_info
