def do_calibration(files, date, use_existing=False, no_db=False):
'''
Calibrate the given files, using a passed in dictionary of files and their
calibrations.
Params:
- files: Dictionary containing raw file names, bp, hp, dark, flat:
[{'RAW':'YYYYMMDD/image0123.fits', 'DARK':'filepath', 'FLAT':'filepath',
'BP':'filepath', 'HP':'filepath'}, {...}, ...]
- date: The night for which images are being calibrated
- use_existing: Command line argument indicating whether or not to generate
calibrations (used for skies)
- no_db: Command line argument indicating whether or not to store in database
(used for skies)
Return:
- calib_objs: A list of calibrated wirc_data objects
'''
if len(files) == 0:
return []
# Create the appropriate directory for Calibrated Files
file_path = 'Calibrated Files/' + files[0]['RAW'][:8] + '/Auto_Reduced/'
if not os.path.exists(file_path):
os.makedirs(file_path)
calib_file_dicts = []
calib_objs = []
calib_files = []
for file in files:
# Create a wirc object
data = wo.wirc_data(raw_filename=file['RAW'], dark_fn=file['DARK'], \
flat_fn=file['FLAT'], bp_fn=file['BP'], hp_fn=file['HP'])
data.calibrate()
# Save the calibrated wirc object
filename = file_path + file['RAW'].split('/')[1][:-5] + '_cal.fits'
data.header['CAL_FN'] = filename
data.save_wirc_object(filename)
calib_files.append(filename)
calib_objs.append(data)
masterSkies = []
master_sci_files = []
# If user wants new calibrations to be created, generate master skies
if not use_existing:
# Create appropriate filepath for auto reduced master skies
path = 'calibrations/Auto_Reduced/master_skies/'
if not os.path.exists(path):
os.makedirs(path)
masterSkies, master_sci_files = generate_sky(calib_files, date)
# If user is allowing database uploads, upload skies to database
if not no_db:
for sky_list in masterSkies:
load_db.new_skies(sky_list)
# Match each calibrated object with a master sky
for obj in calib_objs:
closest_sky(obj, masterSkies, master_sci_files)
obj.save_wirc_object(obj.header['CAL_FN'])
return calib_objs
#blank cube for alignment
align_cube = []
first_time = True
#now the actual loop
while True:
all_files = sorted(glob.glob('*_auto_extracted.fits')) #get all files in the directory
# print(all_files)
time.sleep(0.1)
all_spec_cube = []
BD_all_spec_cube = []
nfiles = np.size(all_files)
hwp_ang = np.zeros(nfiles)
for im in np.arange(nfiles):
wirc_object = wo.wirc_data(wirc_object_filename=all_files[im],verbose=False)
hwp_ang[im] = wirc_object.header['HWP_ANG']
BD_all_spec_cube.append(wirc_object.source_list[0].trace_spectra)
fn_string = all_files[im].split('_auto_extracted.fits')[0]
#Three images at each position
n_ang = np.size(hwp_ang)
BD_up_all_spec_cube = np.array(BD_all_spec_cube)
from wirc_drp.utils import calibration, spec_utils as su
BD_up_all_spec_cube = su.align_spectral_cube(BD_up_all_spec_cube,ref_trace=cal_spec[0,1])
spectra = BD_up_all_spec_cube[:,:,1,:]
#Shift things
for i in range(spectra.shape[0]):
BD_up_all_spec_cube[i,1,1,:] = shift(BD_up_all_spec_cube[i,1,1,:],-1)
BD_up_all_spec_cube[i,2,1,:] = shift(BD_up_all_spec_cube[i,2,1,:],1)
matplotlib.interactive(True)
#First we'll set up all the directories and filenames:
wircpol_dir = os.environ[
'WIRC_DRP'] # Your WIRCPOL_DRP directory (with a "/" at the end!)
tutorial_dir = wircpol_dir + "Tutorial/sample_data/Single_File_Tutorial/"
raw_fn = tutorial_dir + "wirc1586.fits"
flat_fn = tutorial_dir + "wirc2012_master_flat.fits"
dark_fn = tutorial_dir + "wirc0141_master_dark.fits"
bp_fn = tutorial_dir + "wirc2012_bp_map.fits"
#Now we'll create the wirc_data object, passing in the filenames for the master dark, flat and bad pixel maps
raw_data = wo.wirc_data(raw_filename=raw_fn,
flat_fn=flat_fn,
dark_fn=dark_fn,
bp_fn=bp_fn,
verbose=True,
update_bjd=False)
#calibrate the data object
raw_data.calibrate(mask_bad_pixels=False, verbose=True)
#save newly calibrated file
raw_data.save_wirc_object(tutorial_dir + "calibrated.fits", verbose=True)
#First we'll create a new data object, mostly just to demonstrate how to read in an existing wirc_data object.
calibrated_data = wo.wirc_data(wirc_object_filename=tutorial_dir +
"calibrated.fits",
verbose=True)
#generate background image using one of several different options
def process_sources(calib_objs, date='', sources=False):
'''
Find the sources in the given calibrated images. Then, find cutouts, extract spectra,
plot Q and U, and save various images to appropriate files.
Params:
calib_objs: Either a list of wirc_data objects or a blank list to be populated
date: The date (YYYYMMDD), if needed to populate calib_objs
sources: A boolean representing whether or not the pipeline is only processing sources
Return:
The list of calibrated objects, now modified with source information, if any
'''
# If the sources command line argument is active
if sources:
# If there are no calib_objs, then the user only gave a date
if len(calib_objs) == 0:
calib_objs = list(
glob.glob('Calibrated Files/' + date + '/Auto_Reduced/*'))
else:
# Else, the user entered files with the -f argument
for i in range(len(calib_objs)):
calib_objs[
i] = 'Calibrated Files/' + date + '/Auto_Reduced/' + calib_objs[
i].split('/')[-1]
# Load wirc objects for the necessary files (passed or from directory)
calib_names = calib_objs
calib_objs = []
for name in calib_names:
try:
object = wo.wirc_data(wirc_object_filename=name)
object.sky_fn = object.header['SKY_FN']
object.raw_filename = object.header['RAW_FN']
calib_objs.append(object)
except KeyError:
pass
print('STARTING SOURCE PROCESSING')
# Initialize the paths for the reduced data output
plots_root = 'Reduced Data/Plots/' + date + '/'
traces_root = 'Reduced Data/Trace Spectra/' + date + '/'
cutouts_root = 'Reduced Data/Cutouts/' + date + '/'
ex_cutouts_root = 'Reduced Data/Extracted Cutouts/' + date + '/'
for root_path in [plots_root, traces_root, cutouts_root, ex_cutouts_root]:
if not os.path.exists(root_path):
os.makedirs(root_path)
for obj in calib_objs:
# Differentiate the process for sky files(skip),
# science images with no sky file(perform no subtraction), and ones with
# sky files (perform a subtraction)
try:
test = obj.header['SKY_FN']
if test == 'None':
data = obj.full_image
sky_fn = None
else:
sky_data = fits.open(obj.sky_fn)
sky = sky_data[0].data
sky_data.close()
data = obj.full_image - sky
sky_fn = obj.sky_fn
except KeyError:
continue
# Find sources in the image
obj.find_sources(data,
sky=sky_fn,
threshold_sigma=5,
plot=False,
im_package='scipy',
mode='pol',
verbose=False)
# print(obj.source_list)
# Process each source for the given object
for i, source in enumerate(obj.source_list):
# Create specific file_paths for the data which will be gathered
plots_path = plots_root + obj.raw_filename.split(
'/')[1][:-5] + '_Q_U_Plot{}.png'.format(i)
trace_path = traces_root + obj.raw_filename.split(
'/')[1][:-5] + '_Spectra{}.png'.format(i)
cutouts_path = cutouts_root + obj.raw_filename.split(
'/')[1][:-5] + '_Cutouts{}.png'.format(i)
ex_cutouts_path = ex_cutouts_root + obj.raw_filename.split(
'/')[1][:-5] + '_ExCutouts{}.png'.format(i)
# Get cutouts
source.get_cutouts(obj.full_image,
obj.DQ_image,
obj.filter_name,
replace_bad_pixels=True,
method='interpolate',
verbose=True)
# Use the plot_cutouts() function to save cutout plots to disk
source.plot_cutouts(output_name=cutouts_path, show=False)
# Extract spectra
source.extract_spectra(plot=False)
# Use the plot_trace_spectra() function to save trace spectra plots to disk
source.plot_trace_spectra(output_name=trace_path, show=False)
# Use the plot_extracted_cutouts() function to save extracted cutout plots to disk
source.plot_extracted_cutouts(output_name=ex_cutouts_path,
show=False)
# Calibrate and generate the Stokes Q and U plot for this source
source.rough_lambda_calibration(method=2)
source.compute_polarization(cutmin=20, cutmax=150)
source.plot_Q_and_U(output_name=plots_path, show=False)
#save_cutouts(obj, date)
obj.save_wirc_object(obj.header['CAL_FN'])
print('FINISHED PROCESSING SOURCES')
return calib_objs
import wirc_drp.constants as constants
from wirc_drp.utils import calibration, spec_utils as su, image_utils as iu
from wirc_drp.masks import *
#First we'll set up all the directories and filenames:
wircpol_dir = os.environ['WIRC_DRP'] # Your WIRCPOL_DRP directory (with a "/" at the end!)
tutorial_dir = wircpol_dir + "Tutorial/sample_data/"
#background sky frames
sky_frames = [tutorial_dir+'wirc0141.fits',
tutorial_dir+'wirc0142.fits',
tutorial_dir+'wirc0158.fits',
tutorial_dir+'wirc0159.fits',
tutorial_dir+'wirc0160.fits']
image = wo.wirc_data(tutorial_dir+'wirc0140.fits')
#source finding
print('testing source finding')
image.find_sources_v2(sigma_threshold=0, show_plots=False)
#background subtraction
print('testing background subtraction methods')
image.generate_bkg(method='shift_and_subtract', bkg_fns=sky_frames, shift_dir='diagonal', bkg_sub_shift_size=31,
filter_bkg_size=None, same_HWP=False)
image = wo.wirc_data(tutorial_dir+'wirc0140.fits')
# image.generate_bkg(method='PCA', num_PCA_modes=3, ref_lib=sky_frames, same_HWP=False)
image.generate_bkg(method='PCA', num_PCA_modes=3, bkg_fns=sky_frames, same_HWP=False)
image = wo.wirc_data(tutorial_dir+'wirc0140.fits')
#flat_fn = base_cal+"cleaned_PG_flat_%s.fits"%filter_name
#bp_fn = base_cal+"bad_pix_map.fits"
#dark_fn = base_cal+"archive_dark_%.1fs.fits"%exp_time
#flat_fn = base_cal+"image0233_master_PG_flat.fits"
flat_fn = base_cal + "image0143_master_flat.fits"
bp_fn = base_cal + "image0143_bp_map.fits"
#dark_fn = base_cal+"image0021_master_dark.fits" #1s
#dark_fn = base_cal+"image0093_master_dark.fits" #5s
#dark_fn = base_cal+"image0063_master_dark.fits" #15s
dark_fn = base_cal + "image0123_master_dark.fits" #30s
#load in data as wirc object, and calibrate
bkg_fn = "/scr/data/quicklook/auto_reduction/20190421/J1501_60.0s_auto/wirc0381_auto_extracted.fits"
#bkg_fn = None
data = wo.wirc_data(raw_filename=file_name,
flat_fn=flat_fn,
dark_fn=dark_fn,
bp_fn=bp_fn,
bkg_fn=bkg_fn)
data.calibrate(mask_bad_pixels=False,
verbose=False,
sub_bkg_now=True,
bkg_by_quadrants=True)
#after calibration, get thumbnails and extract spectra!
#add source at the given location x,y
data.add_source(int(x_coord), int(y_coord))
#data.source_list.append(wo.wircpol_source([int(y_coord),int(x_coord)], 'slitless', data.n_sources + 1))
#data.n_sources += 1
#get cutouts