def basic_2d_proc(rawFile,imgType=None,CLOBBER=False):
# set up file names based on our convention
oScanFile = 'pre_reduced/o{}'.format(rawFile)
toScanFile = 'pre_reduced/to{}'.format(rawFile)
toScanFlat = 'pre_reduced/to{}'.format(rawFile.replace('.fits','_norm.fits'))
# run the basic 2D stuff on this image if necessary
if (not os.path.isfile(oScanFile)) or CLOBBER:
# get the instrument configuration
inst = instruments.blue_or_red(rawFile)[1]
iraf.specred.dispaxi = inst.get('dispaxis')
iraf.longslit.dispaxi = inst.get('dispaxis')
_biassec0 = inst.get('biassec')
_trimsec0 = inst.get('trimsec')
_flatsec0 = inst.get('flatsec')
# remove destination files
for file in [oScanFile,toScanFile]:
if os.path.isfile(file) and CLOBBER:
os.remove(file)
# check the ultimate desination file, since intermediates get deleted
if not os.path.isfile(toScanFile):
if inst.get('instrument')=='kast':
# do Lick specific bias operations
util.kastbias(rawFile,oScanFile)
elif inst.get('instrument')=='binospec':
util.binospecbias(rawFile,oScanFile,inst)
# do general (IRAF is in such a sorry state I'm not even sure if this works)
else:
iraf.ccdproc(rawFile, output=oScanFile,
overscan='yes', trim='yes',
zerocor="no", flatcor="no",readaxi='line',
trimsec=str(_trimsec0), biassec=str(_biassec0),
Stdout=1)
# trim (same trimming operation for all telescopes)
iraf.ccdproc(oScanFile, output=toScanFile,
overscan='no', trim='yes', zerocor="no", flatcor="no",
readaxi='line',trimsec=str(_trimsec0), Stdout=1)
#create trimmed flats for norm region
if imgType == 'CAL_FLAT' and inst.get('instrument')=='kast':
iraf.ccdproc(oScanFile, output=toScanFlat,
overscan='no', trim='yes', zerocor="no", flatcor="no",
readaxi='line',trimsec=str(_flatsec0), Stdout=1)
if imgType == 'CAL_FLAT' and inst.get('instrument')=='binospec':
iraf.ccdproc(oScanFile, output=toScanFlat,
overscan='no', trim='yes', zerocor="no", flatcor="no",
readaxi='line',trimsec=str(_flatsec0), Stdout=1)
os.remove(oScanFile)
return 0
def needs_to_be_reduced(file):
hdu = fits.open(file, mode='readonly')
header = hdu[0].header
arm, inst_dict = instruments.blue_or_red(file)
inst = inst_dict['name']
# Select against imaging mode data for LRIS
if 'LRIS' in inst.upper():
if 'GRISTRAN' in header.keys() and header['GRISTRAN'].lower(
) == 'stowed':
return (False)
return (True)
def reduce(imglist, files_arc, files_flat, _cosmic, _interactive_extraction,
_arc):
import string
import os
import re
import sys
import pdb
os.environ["PYRAF_BETA_STATUS"] = "1"
try:
from astropy.io import fits as pyfits
except:
import pyfits
import numpy as np
import util
import instruments
import combine_sides as cs
import cosmics
from pyraf import iraf
dv = util.dvex()
scal = np.pi / 180.
if not _interactive_extraction:
_interactive = False
else:
_interactive = True
if not _arc:
_arc_identify = False
else:
_arc_identify = True
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.specred(_doprint=0)
iraf.disp(inlist='1', reference='1')
toforget = [
'ccdproc', 'imcopy', 'specred.apall', 'longslit.identify',
'longslit.reidentify', 'specred.standard', 'longslit.fitcoords',
'onedspec.wspectext'
]
for t in toforget:
iraf.unlearn(t)
iraf.ccdred.verbose = 'no'
iraf.specred.verbose = 'no'
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.ccdtype = ''
iraf.longslit.mode = 'h'
iraf.specred.mode = 'h'
iraf.noao.mode = 'h'
iraf.ccdred.instrument = "ccddb$kpno/camera.dat"
list_arc_b = []
list_arc_r = []
for arcs in files_arc:
hdr = util.readhdr(arcs)
br, inst = instruments.blue_or_red(arcs)
if br == 'blue':
list_arc_b.append(arcs)
elif br == 'red':
list_arc_r.append(arcs)
else:
errStr = '{} '.format(str(util.readkey3(hdr, 'VERSION')))
errStr += 'not in database'
print(errStr)
sys.exit()
asci_files = []
newlist = [[], []]
print('\n### images to reduce :', imglist)
#raise TypeError
for img in imglist:
if 'b' in img:
newlist[0].append(img)
elif 'r' in img:
newlist[1].append(img)
if len(newlist[1]) < 1:
newlist = newlist[:-1]
elif len(newlist[0]) < 1:
newlist = newlist[1:]
else:
sides = raw_input("Reduce which side? ([both]/b/r): ")
if sides == 'b':
newlist = newlist[:-1]
elif sides == 'r':
newlist = newlist[1:]
for imgs in newlist:
hdr = util.readhdr(imgs[0])
br, inst = instruments.blue_or_red(imgs[0])
if br == 'blue':
flat_file = '../RESP_blue'
elif br == 'red':
flat_file = '../RESP_red'
else:
errStr = 'Not in intrument list'
print(errStr)
sys.exit()
iraf.specred.dispaxi = inst.get('dispaxis')
iraf.longslit.dispaxi = inst.get('dispaxis')
_gain = inst.get('gain')
_ron = inst.get('read_noise')
iraf.specred.apall.readnoi = _ron
iraf.specred.apall.gain = _gain
_object0 = util.readkey3(hdr, 'OBJECT')
_date0 = util.readkey3(hdr, 'DATE-OBS')
_object0 = re.sub(' ', '', _object0)
_object0 = re.sub('/', '_', _object0)
nameout0 = str(_object0) + '_' + inst.get('name') + '_' + str(_date0)
nameout0 = util.name_duplicate(imgs[0], nameout0, '')
timg = nameout0
print('\n### now processing :', timg, ' for -> ', inst.get('name'))
if len(imgs) > 1:
img_str = ''
for i in imgs:
img_str = img_str + i + ','
iraf.imcombine(img_str, output=timg)
else:
img = imgs[0]
if os.path.isfile(timg):
os.system('rm -rf ' + timg)
iraf.imcopy(img, output=timg)
# should just do this by hand
iraf.ccdproc(timg,
output='',
overscan='no',
trim='no',
zerocor="no",
flatcor="yes",
readaxi='line',
flat=flat_file,
Stdout=1)
img = timg
#raw_input("Press Enter to continue...")
if _cosmic:
print('\n### starting cosmic removal')
array, header = cosmics.fromfits(img)
c = cosmics.cosmicsimage(array,
gain=inst.get('gain'),
readnoise=inst.get('read_noise'),
sigclip=5,
sigfrac=0.5,
objlim=2.0)
c.run(maxiter=5)
cosmics.tofits('cosmic_' + img, c.cleanarray, header)
img = 'cosmic_' + img
print('\n### cosmic removal finished')
else:
print(
'\n### No cosmic removal, saving normalized image for inspection???'
)
if inst.get('arm') == 'blue' and len(list_arc_b) > 0:
arcfile = list_arc_b[0]
elif inst.get('arm') == 'red' and len(list_arc_r) > 0:
arcfile = list_arc_r[0]
else:
arcfile = None
if arcfile is not None and not arcfile.endswith(".fits"):
arcfile = arcfile + '.fits'
if not os.path.isdir('database/'):
os.mkdir('database/')
if _arc_identify:
os.system('cp ' + arcfile + ' .')
arcfile = string.split(arcfile, '/')[-1]
arc_ex = re.sub('.fits', '.ms.fits', arcfile)
arcref = inst.get('archive_arc_extracted')
arcref_img = string.split(arcref, '/')[-1]
arcref_img = arcref_img.replace('.ms.fits', '')
arcrefid = inst.get('archive_arc_extracted_id')
os.system('cp ' + arcref + ' .')
arcref = string.split(arcref, '/')[-1]
os.system('cp ' + arcrefid + ' ./database')
aperture = inst.get('archive_arc_aperture')
os.system('cp ' + aperture + ' ./database')
print('\n### arcfile : ', arcfile)
print('\n### arcfile extraction : ', arc_ex)
print('\n### arc reference : ', arcref)
# read for some meta data to get the row right
tmpHDU = pyfits.open(arcfile)
header = tmpHDU[0].header
try:
spatialBin = int(header['binning'].split(',')[0])
except KeyError:
spatialBin = 1
apLine = 700 // spatialBin
iraf.specred.apall(arcfile,
output=arc_ex,
ref=arcref_img,
line=apLine,
nsum=10,
interactive='no',
extract='yes',
find='yes',
nfind=1,
format='multispec',
trace='no',
back='no',
recen='no')
iraf.longslit.reidentify(referenc=arcref,
images=arc_ex,
interac='NO',
section=inst.get('section'),
coordli=inst.get('line_list'),
shift='INDEF',
search='INDEF',
mode='h',
verbose='YES',
step=0,
nsum=5,
nlost=2,
cradius=10,
refit='yes',
overrid='yes',
newaps='no')
print('\n### extraction using apall')
result = []
hdr_image = util.readhdr(img)
_type = util.readkey3(hdr_image, 'object')
if (_type.startswith("arc") or _type.startswith("dflat")
or _type.startswith("Dflat") or _type.startswith("Dbias")
or _type.startswith("Bias")):
print('\n### warning problem \n exit ')
sys.exit()
else:
imgex = util.extractspectrum(img, dv, inst, _interactive, 'obj')
print('\n### applying wavelength solution')
print(arc_ex)
iraf.disp(inlist=imgex, reference=arc_ex)
result = result + [imgex] + [timg]
# asci_files.append(imgasci)
if not os.path.isdir(_object0 + '_ex/'):
os.mkdir(_object0 + '_ex/')
if not _arc_identify:
util.delete(arcref)
else:
util.delete(arcfile)
util.delete(arc_ex)
util.delete(img)
util.delete(imgex)
util.delete(arcref)
util.delete('logfile')
#if _cosmic:
#util.delete(img[7:])
#util.delete("cosmic_*")
os.system('mv ' + 'd' + imgex + ' ' + _object0 + '_ex/')
use_sens = raw_input('Use archival flux calibration? [y]/n ')
if use_sens != 'no':
sensfile = inst.get('archive_sens')
os.system('cp ' + sensfile + ' ' + _object0 + '_ex/')
bstarfile = inst.get('archive_bstar')
os.system('cp ' + bstarfile + ' ' + _object0 + '_ex/')
return result
def main():
description = "> Performs pre-reduction steps"
usage = "%prog \t [option] \n Recommended syntax: %prog -i -c"
parser = OptionParser(usage=usage, description=description, version="0.1")
option, args = parser.parse_args()
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.specred(_doprint=0)
iraf.ccdred.verbose = 'no'
iraf.specred.verbose = 'no'
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.ccdtype = ''
iraf.longslit.mode = 'h'
iraf.specred.mode = 'h'
iraf.noao.mode = 'h'
iraf.ccdred.instrument = "ccddb$kpno/camera.dat"
mkarc = raw_input("Make arc? ([y]/n): ")
mkflat = raw_input("Make flat? ([y]/n): ")
if len(args) > 1:
files = []
sys.argv.append('--help')
option, args = parser.parse_args()
sys.exit()
elif len(args) == 1:
files = util.readlist(args[0])
sys.exit()
else:
listfile = glob.glob('*.fits')
files_science = []
files_arc = []
files_dflat = []
#print 'checking your files ...'
for img in listfile:
_type = ''
hdr0 = util.readhdr(img)
_type = util.readkey3(hdr0, 'object')
if 'flat' in _type.lower():
files_dflat.append(img)
elif 'arc' not in _type.lower() and 'arc' not in img.lower():
files_science.append(img)
if mkarc != 'n':
mkarc_b = raw_input(
"List blue arc files to combine (.fits will be added): "
).split()
mkarc_r = raw_input(
"List red arc files to combine (.fits will be added): ").split(
)
for arc in mkarc_b:
files_arc.append(arc + '.fits')
for arc in mkarc_r:
files_arc.append(arc + '.fits')
if mkarc != 'n':
list_arc_b = []
list_arc_r = []
for arcs in files_arc:
if instruments.blue_or_red(arcs)[0] == 'blue':
list_arc_b.append(arcs)
elif instruments.blue_or_red(arcs)[0] == 'red':
list_arc_r.append(arcs)
else:
sys.exit()
if mkflat != 'n':
list_flat_b = []
list_flat_r = []
for dflats in files_dflat:
if instruments.blue_or_red(dflats)[0] == 'blue':
list_flat_b.append(dflats)
elif instruments.blue_or_red(dflats)[0] == 'red':
list_flat_r.append(dflats)
else:
sys.exit()
# make pre_reduced if it doesn't exist
if not os.path.isdir('pre_reduced/'):
os.mkdir('pre_reduced/')
# log the existing processed files (need to verify this works if pre_reduced is empty...)
pfiles = []
new_files = []
for root, dirnames, filenames in os.walk('pre_reduced'):
for file in filenames:
if file.startswith('to'):
pfiles.append(file)
print(pfiles)
# loop over each image in pre_reduced
for img in listfile:
hdr = util.readhdr(img)
targ = util.readkey3(hdr, 'object')
# if file is not not a processed file, run the overscan+trim code
if 'to' + img not in pfiles:
# if the file is a science file, grab the name for later
if 'arc' not in targ.lower() and 'flat' not in targ.lower():
new_files.append(img)
print('Adding data for: ' + targ)
inst = instruments.blue_or_red(img)[1]
iraf.specred.dispaxi = inst.get('dispaxis')
iraf.longslit.dispaxi = inst.get('dispaxis')
_biassec0 = inst.get('biassec')
_trimsec0 = inst.get('trimsec')
######################################################################
#
# JB: this chunk of code needs attention
# It seems incredibly hacky for anything but Kast...
#
# overscan
if not img.startswith('o') and inst.get('observatory') == 'lick':
if os.path.isfile('pre_reduced/o' + img):
os.remove('pre_reduced/o' + img)
util.kastbias(img, 'pre_reduced/o' + img)
elif not img.startswith('o') and inst.get('observatory') != 'lick':
if os.path.isfile('pre_reduced/o' + img):
os.remove('pre_reduced/o' + img)
os.system('cp ' + img + ' ' + 'pre_reduced/' + img)
# trim
if not img.startswith('t') and inst.get('observatory') == 'lick':
if os.path.isfile('pre_reduced/to' + img):
os.remove('pre_reduced/to' + img)
iraf.ccdproc('pre_reduced/o' + img,
output='pre_reduced/to' + img,
overscan='no',
trim='yes',
zerocor="no",
flatcor="no",
readaxi='line',
trimsec=str(_trimsec0),
Stdout=1)
elif not img.startswith('t') and inst.get('observatory') != 'lick':
if os.path.isfile('pre_reduced/to' + img):
os.remove('pre_reduced/to' + img)
iraf.ccdproc('pre_reduced/' + img,
output='pre_reduced/to' + img,
overscan='yes',
trim='yes',
zerocor="no",
flatcor="no",
readaxi='line',
trimsec=str(_trimsec0),
biassec=str(_biassec0),
Stdout=1)
# combine the arcs
if mkarc != 'n':
# blue arcs
if len(list_arc_b) > 0:
if len(list_arc_b) == 1:
arc_blue = list_arc_b[0]
os.system('cp ' + 'pre_reduced/to' + arc_blue + ' ' +
'pre_reduced/ARC_blue.fits')
else:
arc_str = ''
for arc in list_arc_b:
arc_str = arc_str + 'pre_reduced/to' + arc + ','
if os.path.isfile('pre_reduced/ARC_blue.fits'):
os.remove('pre_reduced/ARC_blue.fits')
iraf.imcombine(arc_str, output='pre_reduced/ARC_blue.fits')
# red arcs
if len(list_arc_r) > 0:
if len(list_arc_r) == 1:
arc_red = list_arc_r[0]
os.system('cp ' + 'pre_reduced/to' + arc_red + ' ' +
'pre_reduced/ARC_red.fits')
else:
arc_str = ''
for arc in list_arc_r:
arc_str = arc_str + 'pre_reduced/to' + arc + ','
if os.path.isfile('pre_reduced/ARC_red.fits'):
os.remove('pre_reduced/ARC_red.fits')
iraf.imcombine(arc_str, output='pre_reduced/ARC_red.fits')
# combine the flats
if mkflat != 'n':
inter = 'yes'
# blue flats
if len(list_flat_b) > 0:
br, inst = instruments.blue_or_red(list_flat_b[0])
iraf.specred.dispaxi = inst.get('dispaxis')
if len(list_flat_b) == 1:
# Flat_blue = 'pre_reduced/to'+ list_flat_b[0]
Flat_blue = list_flat_b[0]
else:
flat_str = ''
for flat in list_flat_b:
flat_str = flat_str + 'pre_reduced/to' + flat + ','
#subsets = 'no'
if os.path.isfile('pre_reduced/toFlat_blue'):
os.remove('pre_reduced/toFlat_blue')
iraf.flatcombine(flat_str,
output='pre_reduced/toFlat_blue',
ccdtype='',
rdnoise=3.7,
subsets='no',
process='no')
Flat_blue = 'Flat_blue.fits'
#What is the output here? Check for overwrite
iraf.specred.response('pre_reduced/to' + Flat_blue,
normaliz='pre_reduced/to' + Flat_blue,
response='pre_reduced/RESP_blue',
interac=inter,
thresho='INDEF',
sample='*',
naverage=2,
function='legendre',
low_rej=3,
high_rej=3,
order=60,
niterat=20,
grow=0,
graphic='stdgraph')
# red flats
if len(list_flat_r) > 0:
br, inst = instruments.blue_or_red(list_flat_r[0])
iraf.specred.dispaxi = inst.get('dispaxis')
if len(list_flat_r) == 1:
# Flat_red = 'pre_reduced/to' + list_flat_r[0]
Flat_red = list_flat_r[0]
else:
flat_str = ''
for flat in list_flat_r:
flat_str = flat_str + 'pre_reduced/to' + flat + ','
if os.path.isfile('pre_reduced/toFlat_red'):
os.remove('pre_reduced/toFlat_red')
iraf.flatcombine(flat_str,
output='pre_reduced/toFlat_red',
ccdtype='',
rdnoise=3.8,
subsets='yes',
process='no')
Flat_red = 'Flat_red.fits'
#What is the output here? Check for overwrite
iraf.specred.response('pre_reduced/to' + Flat_red,
normaliz='pre_reduced/to' + Flat_red,
response='pre_reduced/RESP_red',
interac=inter,
thresho='INDEF',
sample='*',
naverage=2,
function='legendre',
low_rej=3,
high_rej=3,
order=80,
niterat=20,
grow=0,
graphic='stdgraph')
# science files should have 't' in front now
# this just gets the base name, to prefix assumed below
if new_files is not None:
files_science = new_files
# get all the science objects for the night
science_targets = []
for obj in files_science:
hdr = util.readhdr(obj)
_type = util.readkey3(hdr, 'object')
science_targets.append(_type)
# make a dir for each sci object
science_targets = set(science_targets)
for targ in science_targets:
if not os.path.isdir('pre_reduced/' + targ + '/'):
os.mkdir('pre_reduced/' + targ + '/')
# copy the files into the obj dir
for obj in files_science:
hdr = util.readhdr(obj)
targ = util.readkey3(hdr, 'object')
if not obj.startswith('to'):
os.system('cp ' + 'pre_reduced/to' + obj + ' ' + 'pre_reduced/' +
targ + '/')
else:
os.system('cp ' + 'pre_reduced/' + obj + ' ' + 'pre_reduced/' +
targ + '/')
rawfiles = glob.glob('*.fits')
ofiles = glob.glob('pre_reduced/o' + '*.fits')
tfiles = glob.glob('pre_reduced/to' + '*.fits')
# delete raw files from the pre_reduced dir
# there shouldn't be any there though?
# maybe if the overscan isn't implemented for that detector
for img in rawfiles:
util.delete('pre_reduced/' + img)
# delete the ofiles from pre_reduced dir
for img in ofiles:
util.delete(img)
def pre_reduction_dev(*args,**kwargs):
# parse kwargs
VERBOSE = kwargs.get('VERBOSE')
CLOBBER = kwargs.get('CLOBBER')
FAKE_BASIC_2D = kwargs.get('FAKE_BASIC_2D')
FULL_CLEAN = kwargs.get('FULL_CLEAN')
FAST = kwargs.get('FAST')
CONFIG_FILE = kwargs.get('CONFIG_FILE')
MAKE_ARCS = kwargs.get('MAKE_ARCS')
MAKE_FLATS = kwargs.get('MAKE_FLATS')
QUICK = kwargs.get('QUICK')
RED_AMP_BAD = kwargs.get('RED_AMP_BAD')
BLUE_AMP_BAD = kwargs.get('BLUE_AMP_BAD')
HOST = kwargs.get('HOST')
TRIM = kwargs.get('TRIM')
FIX_AMP_OFFSET = kwargs.get('FIX_AMP_OFFSET')
RED_DIR = kwargs.get('RED_DIR')
# init iraf stuff
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.specred(_doprint=0)
iraf.ccdred.verbose = 'no'
iraf.specred.verbose = 'no'
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.ccdtype = ''
iraf.longslit.mode = 'h'
iraf.specred.mode = 'h'
iraf.noao.mode = 'h'
iraf.ccdred.instrument = "ccddb$kpno/camera.dat"
prereddir= os.path.join(RED_DIR, 'pre_reduced/')
# set up config
if CONFIG_FILE:
with open(CONFIG_FILE,'r') as fin:
configDict = json.load(fin)
else:
STANDARD_STAR_LIBRARY = mu.construct_standard_star_library()
observations = sorted(glob.glob('*.fits'))
#TODO: Better first pass at config file, std have exptime < 250s(?)
# CDK - updated this and placed it after arm/inst_name definition to
# better generalize configDict
configDict = {}
for obsfile in observations:
arm, inst_dict = instruments.blue_or_red(obsfile)
inst_name = inst_dict.get('instrument').upper()
for key in ['SCI','STD','CAL_ARC','CAL_FLAT','CAL']:
if key not in configDict.keys():
configDict[key]={}
if arm.upper() not in configDict[key].keys():
configDict[key][arm.upper()]={}
if 'CAL_' in key:
full_key = key.replace('CAL_','CALIBRATION_')
configDict[key][arm.upper()]={full_key: []}
# CDK - added this to explicitly skip files that dont need to be
# reduced.
if not mu.needs_to_be_reduced(obsfile): continue
hdu = fits.open(obsfile)
if 'lris' in inst_dict.get('name'):
hdu[0].header['OBJECT']=hdu[0].header['TARGNAME']
hdu.writeto(obsfile, overwrite=True)
use_ext = inst_dict['use_ext']
header = hdu[use_ext].header
imageType = mu.determine_image_type(header, inst_name, STANDARD_STAR_LIBRARY)
channel, inst_dict = instruments.blue_or_red(obsfile)
obj = header.get('OBJECT').strip()
if imageType == 'SCI' or imageType == 'STD':
if obj in configDict[imageType][channel.upper()].keys():
configDict[imageType][channel.upper()][obj].append(obsfile)
else:
configDict[imageType][channel.upper()][obj] = [obsfile]
if imageType == 'CAL_ARC' and 'foc' not in obsfile:
configDict[imageType][channel.upper()]['CALIBRATION_ARC'].append(obsfile)
if imageType == 'CAL_FLAT':
configDict[imageType][channel.upper()]['CALIBRATION_FLAT'].append(obsfile)
with open('custom_config.json','w') as fout:
fout.write(json.dumps(configDict,indent=4))
outStr = '\n\nOk, not config supplied, so I wrote a first pass custom_config.json\n'
outStr += 'Use at your own risk! Manually edit if needed and run again with -c custom_config.json\n'
outStr += 'You can manually add files to the appropriate lists and rerun.\n'
outStr += 'WARNING: make sure you rename your config file, or it could get overwritten!\n\n'
print(outStr)
sys.exit(1)
if not FAST:
# do visual inspection of frames via ds9 windows
usrResp = ''
while usrResp != 'C':
promptStr = '\nYou\'ve opted to display images before kicking off the reduction.\n'
promptStr += 'At this point you may:\n'
promptStr += ' (D)isplay the current state of the reduction config\n'
promptStr += ' (C)ontinue with these files as is\n'
promptStr += ' (R)emove a file from the current config\n'
promptStr += ' (A)dd a file to the current config\n'
promptStr += ' (Q)uit the whole thing. \n'
promptStr += 'I recommend you (D)isplay and remove unwanted frames from your config file,\n'
promptStr += '(Q)uit, and then rerun with the updated config file.\nCommand: '
usrRespOrig = raw_input(promptStr)
try:
usrResp = usrRespOrig.strip().upper()
except Exception as e:
usrResp = 'nothing'
# (D)isplay all the images in the lists
if usrResp == 'D':
blueArcList = configDict['CAL_ARC']['BLUE']['CALIBRATION_ARC']
redArcList = configDict['CAL_ARC']['RED']['CALIBRATION_ARC']
blueFlatList = configDict['CAL_FLAT']['BLUE']['CALIBRATION_FLAT']
redFlatList = configDict['CAL_FLAT']['RED']['CALIBRATION_FLAT']
blueStdList = []
redStdList = []
blueSciList = []
redSciList = []
for targ,imgList in configDict['STD']['BLUE'].items():
for file in imgList:
blueStdList.append(file)
for targ,imgList in configDict['STD']['RED'].items():
for file in imgList:
redStdList.append(file)
for targ,imgList in configDict['SCI']['BLUE'].items():
for file in imgList:
blueSciList.append(file)
for targ,imgList in configDict['SCI']['RED'].items():
for file in imgList:
redSciList.append(file)
blueArcDS9 = show_ds9_list(blueArcList,instanceName='BlueArcs')
redArcDS9 = show_ds9_list(redArcList,instanceName='RedArcs')
blueFlatDS9 = show_ds9_list(blueFlatList,instanceName='BlueFlats')
redFlatDS9 = show_ds9_list(redFlatList,instanceName='RedFlats')
blueStdDS9 = show_ds9_list(blueStdList,instanceName='BlueStandards')
redStdDS9 = show_ds9_list(redStdList,instanceName='RedStandards')
blueSciDS9 = show_ds9_list(blueSciList,instanceName='BlueScience')
redSciDS9 = show_ds9_list(redSciList,instanceName='RedScience')
if usrResp == 'R':
configDict = user_adjust_config(configDict,operation='REMOVE')
if usrResp == 'A':
configDict = user_adjust_config(configDict,operation='ADD')
if usrResp == 'Q':
print('Okay, quitting pre_reduction...')
sys.exit(1)
# pre_reduced does not exist, needs to be made
if not os.path.isdir(prereddir):
os.mkdir(prereddir)
if QUICK:
file =glob.glob('*.fits')[0]
inst = instruments.blue_or_red(file)[1]
if 'kast' in inst['name']:
b_inst = instruments.kast_blue
r_inst = instruments.kast_red
if 'lris' in inst['name']:
b_inst = instruments.lris_blue
r_inst = instruments.lris_red
if 'goodman' in inst['name']:
b_inst = instruments.goodman_blue
r_inst = instruments.goodman_red
if not os.path.isdir(prereddir+'master_files/'):
os.mkdir(prereddir+'master_files/')
b_arcsol = b_inst.get('archive_arc_extracted_id')
b_resp = b_inst.get('archive_flat_file')
r_arcsol = r_inst.get('archive_arc_extracted_id')
r_resp = r_inst.get('archive_flat_file')
if os.path.isdir(prereddir+'master_files/'):
os.system('cp ' + b_arcsol + ' ' + 'pre_reduced/master_files/')
os.system('cp ' + b_resp + ' ' + 'pre_reduced/')
os.system('cp ' + r_arcsol + ' ' + 'pre_reduced/master_files/')
os.system('cp ' + r_resp + ' ' + 'pre_reduced/')
# pre_reduced exists, but we want to clobber/do a clean reduction
elif FULL_CLEAN:
promptStr = 'Do you really want to wipe pre_reduced? [y/n]: '
usrRespOrig = raw_input(promptStr)
if usrRespOrig and usrRespOrig[0].strip().upper() == 'Y':
# remove all pre_reduced files
shutil.rmtree(prereddir)
os.mkdir(prereddir)
# pre_reduced exists, need to document what is there
else:
# get existing pre_reduced files
preRedFiles = glob.glob(prereddir+'*.fits')
# loop over raw files in configDict, if the destination exists, do nothing
# # otherwise, do the bias/reorient/trim/output/etc
for imgType,typeDict in configDict.items():
for chan,objDict in typeDict.items():
for obj,fileList in objDict.items():
for rawFile in fileList:
# try:
# res = basic_2d_proc(rawFile,CLOBBER=CLOBBER)
# if res != 0:
# raise ValueError('Something bad happened in basic_2d_proc on {}'.format(rawFile))
# except (Exception,ValueError) as e:
# print('Exception (basic_2d): {}'.format(e))
if not FAKE_BASIC_2D:
inst = instruments.blue_or_red(rawFile)[1]
if inst['name'] == 'lris_blue' or inst['name'] == 'lris_red':
fix_lris_header(rawFile)
# res = keck_basic_2d.main([rawFile])
if imgType != 'CAL_FLAT':
print (imgType)
res = keck_basic_2d.main([rawFile], TRIM=TRIM, ISDFLAT=False, RED_AMP_BAD=RED_AMP_BAD, MASK_MIDDLE_RED=False, MASK_MIDDLE_BLUE=False, FIX_AMP_OFFSET=FIX_AMP_OFFSET, BLUE_AMP_BAD=BLUE_AMP_BAD)
else:
print (imgType)
res = keck_basic_2d.main([rawFile], TRIM=TRIM, ISDFLAT = True, RED_AMP_BAD=RED_AMP_BAD, MASK_MIDDLE_RED=False, MASK_MIDDLE_BLUE=False, FIX_AMP_OFFSET=FIX_AMP_OFFSET, BLUE_AMP_BAD=BLUE_AMP_BAD)
else:
res = basic_2d_proc(rawFile,imgType=imgType,CLOBBER=CLOBBER)
else:
# here we're faking the basic 2D reduction because we've done
# specialized 2D reduction (e.g., keck_basic_2d)
res = 0
if res != 0:
raise ValueError('Something bad happened in basic_2d_proc on {}'.format(rawFile))
# move the std and sci files into their appropriate directories
try:
res = reorg_files(configDict,CLOBBER=CLOBBER)
if res != 0:
raise ValueError('Something bad happened in reorg_files')
except (Exception,ValueError) as e:
print('Exception (reorg): {}'.format(e))
### some blocks of code from the original pre_reduction ###
# combine the arcs
if MAKE_ARCS:
# CDK - generalized arc creation method
for key in configDict['CAL_ARC'].keys():
list_arc = configDict['CAL_ARC'][key]['CALIBRATION_ARC']
if len(list_arc)>0:
first = prereddir+'to{}'.format(list_arc[0])
br, inst = instruments.blue_or_red(first)
destFile = prereddir+'ARC_{0}.fits'.format(key.lower())
util.make_arc(list_arc, destFile, inst, iraf)
# combine the flats
if MAKE_FLATS and 'lris' in inst['name']:
list_flat_b = configDict['CAL_FLAT']['BLUE']['CALIBRATION_FLAT']
list_flat_r = configDict['CAL_FLAT']['RED']['CALIBRATION_FLAT']
inter = 'yes'
b_amp1_list = []
b_amp2_list = []
r_amp1_list = []
r_amp2_list = []
predir = prereddir+'to'
for flat in list_flat_b:
suffix = '.'.join(flat.split('.')[1:])
b_amp1_file = flat.split('.')[0]+'_amp1.'+suffix
b_amp2_file = flat.split('.')[0]+'_amp2.'+suffix
if os.path.exists(predir+b_amp1_file): b_amp1_list.append(b_amp1_file)
if os.path.exists(predir+b_amp2_file): b_amp2_list.append(b_amp2_file)
for flat in list_flat_r:
suffix = '.'.join(flat.split('.')[1:])
r_amp1_file = flat.split('.')[0]+'_amp1.'+suffix
r_amp2_file = flat.split('.')[0]+'_amp2.'+suffix
print(r_amp1_file)
if os.path.exists(predir+r_amp1_file): r_amp1_list.append(r_amp1_file)
if os.path.exists(predir+r_amp2_file): r_amp2_list.append(r_amp2_file)
# blue flats
if len(list_flat_b) > 0:
# br, inst = instruments.blue_or_red(list_flat_b[0])
br, inst = instruments.blue_or_red(prereddir+'to{}'.format(b_amp1_list[0]))
dispaxis = inst.get('dispaxis')
iraf.specred.dispaxi = dispaxis
Flat_blue_amp1 = prereddir+'toFlat_blue_amp1.fits'
Flat_blue_amp2 = prereddir+'toFlat_blue_amp2.fits'
flat_list_amp1 = []
for flat in b_amp1_list:
flat_list_amp1.append(prereddir+'to'+ flat)
if os.path.isfile(Flat_blue_amp1):
os.remove(Flat_blue_amp1)
# first, combine all the flat files into a master flat
res = combine_flats(flat_list_amp1,OUTFILE=Flat_blue_amp1,MEDIAN_COMBINE=True)
# run iraf response
iraf.specred.response(Flat_blue_amp1,
normaliz=Flat_blue_amp1,
response=prereddir+'RESP_blue_amp1',
interac=inter, thresho='INDEF',
sample='*', naverage=2, function='legendre',
low_rej=5,high_rej=5, order=60, niterat=20,
grow=0, graphic='stdgraph')
# finally, inspect the flat and mask bad regions
res = inspect_flat([prereddir+'RESP_blue_amp1.fits'],
OUTFILE=prereddir+'RESP_blue_amp1.fits', DISPAXIS=dispaxis)
hdu_amp1 = fits.open(prereddir+'RESP_blue_amp1.fits')
amp1_flatten = np.asarray(hdu_amp1[0].data).flatten()
shape1=hdu_amp1[0].data.shape
dim1 = shape1[0]
dim2 = shape1[1]
concat_amps = amp1_flatten
if not BLUE_AMP_BAD:
flat_list_amp2 = []
for flat in b_amp2_list:
flat_list_amp2.append(prereddir+'to'+ flat)
if os.path.isfile(Flat_blue_amp2):
os.remove(Flat_blue_amp2)
res = combine_flats(flat_list_amp2,OUTFILE=Flat_blue_amp2,MEDIAN_COMBINE=True)
iraf.specred.response(Flat_blue_amp2,
normaliz=Flat_blue_amp2,
response=prereddir+'RESP_blue_amp2',
interac=inter, thresho='INDEF',
sample='*', naverage=2, function='legendre',
low_rej=5,high_rej=5, order=60, niterat=20,
grow=0, graphic='stdgraph')
res = inspect_flat([prereddir+'RESP_blue_amp2.fits'],
OUTFILE=prereddir+'RESP_blue_amp2.fits', DISPAXIS=dispaxis)
hdu_amp2 = fits.open(prereddir+'RESP_blue_amp2.fits')
amp2_flatten = np.asarray(hdu_amp2[0].data).flatten()
shape2=hdu_amp2[0].data.shape
dim1+=shape2[0]
concat_amps = np.concatenate([concat_amps, amp2_flatten])
print('Output blue response dimensions:',dim1,dim2)
resp_blue_data = np.reshape(concat_amps, (dim1, dim2))
header = hdu_amp1[0].header
if os.path.exists(prereddir+'RESP_blue.fits'):
os.remove(prereddir+'RESP_blue.fits')
hdu = fits.PrimaryHDU(resp_blue_data,header)
hdu.writeto(prereddir+'RESP_blue.fits',output_verify='ignore')
resp_files = ['RESP_blue_amp1.fits','RESP_blue_amp2.fits']
for file in resp_files:
if os.path.exists(prereddir+file):
os.remove(prereddir+file)
# red flats
if len(list_flat_r) > 0:
# br, inst = instruments.blue_or_red(list_flat_r[0])
br, inst = instruments.blue_or_red(prereddir+'to{}'.format(r_amp1_list[0]))
dispaxis = inst.get('dispaxis')
iraf.specred.dispaxi = dispaxis
Flat_red_amp1 = prereddir+'toFlat_red_amp1.fits'
Flat_red_amp2 = prereddir+'toFlat_red_amp2.fits'
flat_list_amp1 = []
for flat in r_amp1_list:
flat_list_amp1.append(prereddir+'to'+ flat)
if os.path.isfile(Flat_red_amp1):
os.remove(Flat_red_amp1)
flat_list_amp2 = []
for flat in r_amp2_list:
flat_list_amp2.append(prereddir+'to'+ flat)
if os.path.isfile(Flat_red_amp2):
os.remove(Flat_red_amp2)
amp2flag = len(flat_list_amp2)>0
# first, combine all the flat files into a master flat
if amp2flag:
res = combine_flats(flat_list_amp1,OUTFILE=Flat_red_amp1,MEDIAN_COMBINE=True)
res = combine_flats(flat_list_amp2,OUTFILE=Flat_red_amp2,MEDIAN_COMBINE=True)
else:
res = combine_flats(flat_list_amp1,OUTFILE=Flat_red_amp1,MEDIAN_COMBINE=True)
#What is the output here? Check for overwrite
iraf.specred.response(Flat_red_amp1,
normaliz=Flat_red_amp1,
response='pre_reduced/RESP_red_amp1',
interac=inter, thresho='INDEF',
sample='*', naverage=2, function='legendre',
low_rej=5,high_rej=5, order=80, niterat=20,
grow=0, graphic='stdgraph')
if amp2flag:
iraf.specred.response(Flat_red_amp2,
normaliz=Flat_red_amp2,
response='pre_reduced/RESP_red_amp2',
interac=inter, thresho='INDEF',
sample='*', naverage=2, function='legendre',
low_rej=5,high_rej=5, order=80, niterat=20,
grow=0, graphic='stdgraph')
# finally, inspect the flat and mask bad regions
if amp2flag:
res = inspect_flat([prereddir+'RESP_red_amp1.fits'],
OUTFILE=prereddir+'RESP_red_amp1.fits', DISPAXIS=dispaxis)
res = inspect_flat([prereddir+'RESP_red_amp2.fits'],
OUTFILE=prereddir+'RESP_red_amp2.fits', DISPAXIS=dispaxis)
else:
res = inspect_flat([prereddir+'RESP_red_amp1.fits'],
OUTFILE=prereddir+'RESP_red.fits', DISPAXIS=dispaxis)
if amp2flag:
hdu_amp1 = fits.open(prereddir+'RESP_red_amp1.fits')
hdu_amp2 = fits.open(prereddir+'RESP_red_amp2.fits')
head = hdu_amp1[0].header
shape1 = hdu_amp1[0].data.shape
shape2 = hdu_amp2[0].data.shape
amp1_flatten = np.asarray(hdu_amp1[0].data).flatten()
amp2_flatten = np.asarray(hdu_amp2[0].data).flatten()
concat_amps = np.concatenate([amp2_flatten, amp1_flatten])
xbin, ybin = [int(ibin) for ibin in head['BINNING'].split(',')]
resp_red_data = np.reshape(concat_amps, (shape1[0]+shape2[0],shape1[1]))
resp_red_data[278:294,:] = 1.
header = hdu_amp1[0].header
if os.path.isfile(prereddir+'RESP_red.fits'):
os.remove(prereddir+'RESP_red.fits')
hdu = fits.PrimaryHDU(resp_red_data,header)
hdu.writeto(prereddir+'RESP_red.fits',output_verify='ignore')
os.remove(prereddir+'RESP_red_amp1.fits')
os.remove(prereddir+'RESP_red_amp2.fits')
else:
os.remove(prereddir+'RESP_red_amp1.fits')
elif MAKE_FLATS:
# CDK - generalized arc creation method
for key in configDict['CAL_FLAT'].keys():
list_flat = configDict['CAL_FLAT'][key]['CALIBRATION_FLAT']
if len(list_flat)>0:
first = prereddir+'to{}'.format(list_flat[0])
br, inst = instruments.blue_or_red(first)
dispaxis = inst.get('dispaxis')
iraf.specred.dispaxi = dispaxis
inter = True
Flat_out = prereddir+'toFlat_{0}.fits'.format(br.lower())
dum = prereddir+'dummy_{0}.fits'.format(br.lower())
resp = prereddir+'RESP_{0}.fits'.format(br.lower())
flat_list = []
norm_list = []
for flat in list_flat:
flat_list.append(prereddir+'to'+flat)
norm = flat.replace('.fits','_norm.fits')
norm_list.append(prereddir+'to'+norm)
for file in [Flat_out, dum, resp]:
if os.path.exists(file):
os.remove(Flat_out)
# first, combine all the flat files into a master flat
res = combine_flats(flat_list,OUTFILE=Flat_out,
MEDIAN_COMBINE=True)
# combine all the flat files for norm region
res = combine_flats(norm_list,OUTFILE=dum, MEDIAN_COMBINE=True)
iraf.specred.response(Flat_out, normaliz=dum, response=resp,
interac=inter, thresho='INDEF', sample='*', naverage=2,
function='legendre', low_rej=5,high_rej=5, order=60,
niterat=20, grow=0, graphic='stdgraph')
# finally, inspect the flat and mask bad regions
res = inspect_flat([resp], OUTFILE=resp, DISPAXIS=dispaxis)
for flat in norm_list:
os.remove(flat)
if HOST:
host_gals.make_host_metadata(configDict)
return 0
def inspect_flat(flat_list,*args,**kwargs):
'''
Model a flat field from slitflat data
Parameters
----------
flat_list : list
List of flat field image filenames
OUTFILE : string, optional
If specified, this will automatically be the file location of
the output flat image
DISPAXIS : int, optional (Default = 1)
Specifies the dispersion axis of the image. DISPAXIS=1
corresponds to (spectral, spatial), DISPAXIS=2 corresponds
to (spatial, spectral)
REMOVE_COLOR: bool, optional (Default = False)
If true, remove the color term from the supplied images.
Returns
-------
int : 0, and writes files to disk
'''
# unpack
outFile = kwargs.get('OUTFILE',None)
dispaxis = kwargs.get('DISPAXIS',1)
remove_color = kwargs.get('REMOVE_COLOR',False)
read_from_file = kwargs.get('READ_FROM_FILE',False)
# if user specified a file containing a list, read into flat_list
if read_from_file:
flat_list_tmp = []
with open(flat_list,'r') as fin:
for line in fin:
if len(line.split()) > 0 and line.split()[0] != '#':
flat_list_tmp.append(line.split()[0].strip())
# assign to flat_list, then proceed as usual
flat_list = flat_list_tmp
# if user passed multiple files, combine them, otherwise, just read in the single file
if len(flat_list) > 1:
flat_comb_image, header, inst = combine_flats(flat_list,**kwargs)
else:
br, inst = instruments.blue_or_red(flat_list[0])
hdu = fits.open(flat_list[0])
flat_comb_image = hdu[0].data
header = hdu[0].header
# transpose if we're dealing with cols x rows
if dispaxis == 2:
flat_comb_image = flat_comb_image.T
# this is a hack and inferior to running IRAF response
if remove_color:
# for each column, divide by the median
for i in range(len(flat_comb_image[0,:])):
flat_comb_image[:,i] /= np.median(flat_comb_image[:,i])
# set up plotting window
plt.ion()
fig=plt.figure(figsize=(16,8))
axMain = plt.subplot2grid((36,36), (0,0), rowspan=36, colspan=36)
ax1 = plt.subplot2grid((36,36), (0,0), rowspan=11, colspan=12)
ax2 = plt.subplot2grid((36,36), (0,12), rowspan=11, colspan=12)
ax3 = plt.subplot2grid((36,36), (0,24), rowspan=11, colspan=12)
ax4 = plt.subplot2grid((36,36), (12,0), rowspan=12, colspan=36)
ax5 = plt.subplot2grid((36,36), (24,0), rowspan=12, colspan=36)
# ax1 data
bsd_col_lo = 0
bsd_col_up = bsd_col_lo + 500
blueSkyData = np.median(flat_comb_image[:,bsd_col_lo:bsd_col_up],axis=0)
blueSkyX = np.arange(bsd_col_lo,bsd_col_up,1)
# ax2 data
# msd_col_lo = flat_comb_image.shape[1] // 2
msd_col_lo = 1800
msd_col_up = msd_col_lo + 300
midSkyData = np.median(flat_comb_image[:,msd_col_lo:msd_col_up],axis=0)
midSkyX = np.arange(msd_col_lo,msd_col_up,1)
# ax3 data
# rsd_col_lo = flat_comb_image.shape[1] - 500
rsd_col_lo = 3200
rsd_col_up = rsd_col_lo + 400
redSkyData = np.median(flat_comb_image[:,rsd_col_lo:rsd_col_up],axis=0)
redSkyX = np.arange(rsd_col_lo,rsd_col_up,1)
# ax1.plot(blueSkyX,blueSkyData,c='k',ls='-',lw=3.)
# ax2.plot(midSkyX, midSkyData,c='k',ls='-',lw=3.)
# ax3.plot(redSkyX,redSkyData,c='k',ls='-',lw=3.)
# grab the values of the 10 and 90 percentile pixels
flat_comb_image_ravel = np.ravel(flat_comb_image)
sortedIndexes = np.argsort(flat_comb_image_ravel)
# vmin = 1.*flat_comb_image_ravel[sortedIndexes[int(0.2*len(sortedIndexes))]]
# vmax = 2.*flat_comb_image_ravel[sortedIndexes[int(0.8*len(sortedIndexes))]]
vmin = 0.9
vmax = 1.1
# image
ax4.imshow(flat_comb_image,aspect=1.,origin='lower',
norm=colors.Normalize(vmin=vmin, vmax=vmax))
# residuals
ax5.imshow(flat_comb_image,aspect=1.,origin='lower',
norm=colors.Normalize(vmin=vmin, vmax=vmax))
# ranges on image plots
ax4.set_xlim([0,flat_comb_image.shape[1]])
ax4.set_ylim([0,flat_comb_image.shape[0]])
ax5.set_xlim([0,flat_comb_image.shape[1]])
ax5.set_ylim([0,flat_comb_image.shape[0]])
# sparse axis labels
ax1.set_xlabel('column')
ax1.set_ylabel('counts')
ax2.set_xlabel('column')
ax3.set_xlabel('column')
ax1.set_yticklabels([])
ax2.set_yticklabels([])
ax3.set_yticklabels([])
flatFitObj = fitFlatClass(flat_comb_image,fig,inst)
while True:
# this really should be a dict of key/value pairs
# and then the prompt is dynamically generated
validResps = ['A','R','F','S','U','H', # standard options
'AHARD','RHARD','REFINE', # poweruser/hidden options
'W','D','Q','Q!'] # stardard ends
promptStr = 'Enter (a) to add an exclusion region.\n'
promptStr += 'Enter (r) to remove a region.\n'
promptStr += 'Enter (f) to fit the exclusion regions.\n'
promptStr += 'Enter (s) to substitute model in exclusion regions.\n'
promptStr += 'Enter (h) to substitute the median profile in a region\n'
promptStr += 'Enter (u) to undo everything and restart.\n'
promptStr += 'Enter (w) to write the improved flat to disk.\n'
promptStr += 'Enter (d) to enter the debugger.\n'
promptStr += 'Enter (q) to quit and write the current flat to disk.\n'
promptStr += 'Enter (q!) to quit and do nothing.'
promptStr += 'Answer: '
usrResp = raw_input(promptStr).strip().upper()
if usrResp in validResps:
# add region by marking it
if usrResp == 'A':
promptStr = 'Enter the name of the sky region (e.g. c1): '
name = raw_input(promptStr).strip().upper()
flatFitObj.add_fit_region(name)
# add hardcoded region
if usrResp == 'AHARD':
promptStr = 'Enter name colLo colUp (e.g. c1 113 171): '
usrResp = raw_input(promptStr).upper().strip()
try:
name = usrResp.split()[0]
colLo = int(usrResp.split()[1])
colUp = int(usrResp.split()[2])
flatFitObj.add_fit_region(name,colLo=colLo,colUp=colUp)
except Exception as e:
print(e)
# remove
if usrResp == 'R' or usrResp == 'RHARD':
flatFitObj.remove_fit_region()
# fit sky
if usrResp == 'F':
flatFitObj.fit_sky_background()
# substitute model in sketchy regions
if usrResp == 'S':
flatFitObj.subsitute_model_flat()
if usrResp == 'U':
flatFitObj = fitFlatClass(flat_comb_image,fig,inst)
flatFitObj.refresh_plot()
if usrResp == 'H':
flatFitObj.hard_mask()
if usrResp == 'REFINE':
# this substitutes the corrected
# flat for the input data...
# basically an experiment, not intended for use
flatFitObj.refine()
# write file
if usrResp == 'W':
# if cols x rows, transpose before writting
if dispaxis == 2:
flatFitObj.flatCorrData = flatFitObj.flatCorrData.T
if outFile is None:
promptStr = 'Enter name of save file (e.g. RESP_blue): '
outFile = raw_input(promptStr).strip()
promptStr = 'Write to file {} [y/n]: '.format(outFile)
usrResp = raw_input(promptStr).upper().strip()
if usrResp == 'Y':
flatFitObj.save_flat('pre_reduced/'+outFile,header=header)
break
else:
outFile = None
print('Ok, aborting save...')
else:
if os.path.isfile(outFile):
os.remove(outFile)
flatFitObj.save_flat(outFile,header=header)
# debug
if usrResp == 'D':
pdb.set_trace()
# quit
if usrResp == 'Q':
#mask problematic pixels
print (flatFitObj.flatCorrData[flatFitObj.flatCorrData <= 0.001])
flatFitObj.flatCorrData[flatFitObj.flatCorrData <= 0.001] = 1.
print('Saving and quitting.')
# if cols x rows, transpose before writting
if dispaxis == 2:
flatFitObj.flatCorrData = flatFitObj.flatCorrData.T
if outFile is None:
promptStr = 'Enter name of save file '
promptStr += '(e.g. RESP_blue.fits): '
outFile = raw_input(promptStr).strip()
promptStr = 'Write to file {} [y/n]: '.format(outFile)
usrResp = raw_input(promptStr).upper().strip()
if usrResp == 'Y':
flatFitObj.save_flat('pre_reduced/'+outFile,header=header)
break
else:
outFile = None
print('Ok, aborting save...')
else:
if os.path.isfile(outFile):
os.remove(outFile)
flatFitObj.save_flat(outFile,header=header)
break
# quit
if usrResp == 'Q!':
print('Quitting.')
break
else:
errStr = 'I don\'t understand, try again...'
print(errStr)
return 0
def combine_flats(flat_list,MEDIAN_COMBINE=False,**kwargs):
''' Stacks images in flat_list, returns a master flat and header '''
# unpack
outFile = kwargs.get('OUTFILE')
clobber = kwargs.get('CLOBBER')
# read data
flat_comb_image = np.array([])
median_image_stack = np.array([])
nImages = 0
expTime = 0
nFlatLimit = 15
# calculate the stack batch size
if nFlatLimit < len(flat_list):
batchSize = len(flat_list) // 2 + 1
while batchSize > len(flat_list):
batchSize = batchSize // 2 + 1
else:
batchSize = len(flat_list)
# loop over the flat files
for file in flat_list:
hdu = fits.open(file)
br, inst = instruments.blue_or_red(file)
data = hdu[0].data
header = hdu[0].header
nImages += 1
expTime += header.get('EXPTIME')
# scale to expTime
data /= expTime
if len(flat_comb_image) == 0:
flat_comb_image = np.copy(data)
else:
# if median combining, stack the data
if MEDIAN_COMBINE:
# stack images in the z direction
flat_comb_image = np.dstack((flat_comb_image,data))
# if flat_comb_image is getting too big, or if we're at the end of flat_list,
# then squash it along the z axis with a median
if ((flat_comb_image.shape[2] == batchSize) or
(flat_comb_image.shape[2] == len(flat_list)-1)):
# stack the intermediate squashed frames
if len(median_image_stack) == 0:
median_image_stack = np.median(flat_comb_image,axis=2)
else:
median_image_stack = np.dstack((median_image_stack,
np.median(flat_comb_image,axis=2)))
# reset the stack
flat_comb_image = np.array([])
# otherwise just sum them
else:
flat_comb_image += np.copy(data)
# if median combining, squash the stack of median
if MEDIAN_COMBINE:
# if there are multiple median images in the stack, median them
if len(median_image_stack.shape) > 2:
flat_comb_image = np.median(median_image_stack,axis=2)
# otherwise just return the single frame
else:
flat_comb_image = np.copy(median_image_stack)
header.add_history('combine_flats: median combined {} files'.format(nImages))
else:
header.add_history('combine_flats: summed {} files'.format(nImages))
# counts / sec * expTime
flat_comb_image *= expTime
if outFile:
# clear space
if os.path.isfile(outFile) and clobber:
os.remove(outFile)
# write correct flat data
hdu = fits.PrimaryHDU(flat_comb_image,header)
hdu.writeto(outFile,output_verify='ignore')
return (flat_comb_image,header,inst)
def main(rawFiles,*args,**kwargs):
'''
Run basic 2D CCD reduction on Keck LRIS data
Parameters
----------
CLOBBER : bool, optional (default=False)
Overwrite the individual files in pre_reduced, but
do not wipe subdirectories
FULL_CLEAN : bool, optional (default=False)
Completely wipe pre_reduced and all subdirectories
BPM : bool, optional (default=False)
Mask bad pixels (not currently implemented)
PIXEL_FLOOR : bool, optional (default=False)
Removes negative pixel values
REORIENT : bool, optional (default=True)
Transposes to wavelength increasing rightward. LRIS images
are by default (spatial, spectral), and in general should
be transposed to (spectral, spatial).
TRIM : bool, optional (default=True)
Trim to some hard coded section of the detector
MASK_MIDDLE_BLUE : bool (default=False)
Mask the middle section of the blue images
MASK_MIDDLE_RED : bool (default=False)
Mask the middle section of the red images (useful if
there's wildly disparate values that make the iraf
windowing tedious)
Returns
-------
int : 0, and writes files to disk
'''
# unpack supported kwargs
CLOBBER = kwargs.get('CLOBBER',False) #
FULL_CLEAN = kwargs.get('FULL_CLEAN',False) # this will completely wipe pre_reduced.
BPM = kwargs.get('BPM',False) # no bad pixel mask available (at least for our binning)
PIXEL_FLOOR = kwargs.get('PIXEL_FLOOR',False)
REORIENT = kwargs.get('REORIENT',True)
TRIM = kwargs.get('TRIM',False)
ISDFLAT = kwargs.get('ISDFLAT',False)
RED_AMP_BAD = kwargs.get('RED_AMP_BAD',False)
BLUE_AMP_BAD = kwargs.get('BLUE_AMP_BAD',False)
MASK_MIDDLE_RED = kwargs.get('MASK_MIDDLE_RED',False)
MASK_MIDDLE_BLUE = kwargs.get('MASK_MIDDLE_BLUE',False)
FIX_AMP_OFFSET = kwargs.get('FIX_AMP_OFFSET',False)
# pre_reduced does not exist, needs to be made
if not os.path.isdir('pre_reduced/'):
os.mkdir('pre_reduced/')
# pre_reduced exists, but we want to clobber/do a clean reduction
elif FULL_CLEAN:
promptStr = 'Do you really want to wipe pre_reduced? [y/n]: '
usrRespOrig = raw_input(promptStr)
if usrRespOrig and usrRespOrig[0].strip().upper() == 'Y':
# remove all pre_reduced files
shutil.rmtree('pre_reduced')
os.mkdir('pre_reduced/')
# pre_reduced exists, need to document what is there
else:
# get files
preRedFiles = glob.glob('pre_reduced/*.fits')
# loop over raw files, if the destination exists, do nothing
# otherwise, do the bias/reorient/trim/output
for i in xrange(len(rawFiles)):
rawFile = rawFiles[i]
inst = instruments.blue_or_red(rawFile)[1]['name']
print('Working on:',rawFile,inst)
# Get list of output file names and their index ranges
outfiles, indices = get_file_names_and_indices(rawFile, **kwargs)
if any([not os.path.exists(f) for f in outfiles]) or CLOBBER:
# read file
img,gain_img,head,secs = read_lris(rawFile)
dsec,osec = np.array(secs[0]),np.array(secs[1])
xbin, ybin = [int(ibin) for ibin in head['BINNING'].split(',')]
# fix header
head['EXPTIME'] = head['ELAPTIME']
head['DATE-OBS'] = head['DATE_BEG']
head['BASIC-2D'] = 'DONE'
# get number of extensions for nAmps
tmpHDU = fits.open(rawFile)
nAmps = len(tmpHDU) - 1
# perform oscan/bias subtraction
noBiasImg = subtract_overscan(img,nAmps,dsec,osec,
gain_image=gain_img,method='polynomial',params=[5,65])
# mask bad pixels
if BPM:
print('Bad pixel masking not yet implemented...')
# apply floor to pixel values
if PIXEL_FLOOR:
noBiasImg[noBiasImg < 0] = 0.
# rotate/flip/transpose (wavelength increasing w/ increasing row)
if REORIENT:
outImg = noBiasImg.T
else:
outImg = noBiasImg
if FIX_AMP_OFFSET:
if inst=='lris_blue':
ampsplit = 2260//xbin
amp1 = np.median(outImg[ampsplit-10:ampsplit])
amp2 = np.median(outImg[ampsplit:ampsplit+10])
outImg[ampsplit:] = outImg[ampsplit:] + (amp1-amp2)
for outfile,index in zip(outfiles, indices):
outdata = outImg[index]
if 'amp' not in outfile:
# This method is more robust for getting "background" level
# than np.median
data = outdata.flatten()
n, bins = np.histogram(data, 5000)
back_level = bins[np.argmax(n)]
if MASK_MIDDLE_RED and inst=='lris_red':
if not RED_AMP_BAD:
#outdata[480//xbin:520//xbin,:] = back_level
# Instead of changing pixel values, remove pixels
outdata=np.delete(outdata, np.s_[468//xbin:520//xbin], 0)
else:
outdata[190//xbin:250//xbin,:] = back_level
if MASK_MIDDLE_BLUE and inst=='lris_blue':
outdata[380//xbin:480//xbin,:] = back_level
write_out_file(outdata, head, outfile, clobber=CLOBBER)
else:
outStr = 'Files exist: {}'.format(outfiles)
print(outStr)
return 0
def get_file_names_and_indices(rawFile, **kwargs):
CLOBBER = kwargs.get('CLOBBER',False) #
FULL_CLEAN = kwargs.get('FULL_CLEAN',False) # this will completely wipe pre_reduced.
BPM = kwargs.get('BPM',False) # no bad pixel mask available (at least for our binning)
PIXEL_FLOOR = kwargs.get('PIXEL_FLOOR',False)
REORIENT = kwargs.get('REORIENT',True)
TRIM = kwargs.get('TRIM',True)
ISDFLAT = kwargs.get('ISDFLAT',False)
RED_AMP_BAD = kwargs.get('RED_AMP_BAD',False)
BLUE_AMP_BAD = kwargs.get('BLUE_AMP_BAD',False)
MASK_MIDDLE_BLUE = kwargs.get('MASK_MIDDLE_BLUE',False)
MASK_MIDDLE_RED = kwargs.get('MASK_MIDDLE_RED',False)
TRIM=True
outfiles = []
indices = []
hdu = fits.open(rawFile, mode='readonly')
head = hdu[0].header
xbin, ybin = [int(ibin) for ibin in head['BINNING'].split(',')]
inst = instruments.blue_or_red(rawFile)[1]['name']
nAmps = len(hdu)-1
oScanFile = 'pre_reduced/to{}'.format(rawFile)
suffix = '.'.join(oScanFile.split('.')[1:])
oScanFile_amp1 = oScanFile.split('.')[0]+'_amp1.'+suffix
oScanFile_amp2 = oScanFile.split('.')[0]+'_amp2.'+suffix
allidx = slice(None, None, None)
if not TRIM:
outfiles.append(oScanFile)
indices.append((allidx, allidx))
if ISDFLAT and inst == 'lris_blue' and not BLUE_AMP_BAD:
# amp1
outfiles.append(oScanFile_amp1)
indices.append((slice(2260//xbin, 2800//xbin, None), allidx))
# amp2
outfiles.append(oScanFile_amp2)
indices.append((slice(1800//xbin, 2260//xbin, None), allidx))
elif ISDFLAT and inst == 'lris_blue':
# amp1
outfiles.append(oScanFile_amp1)
indices.append((slice(2260//xbin, 2800//xbin, None), allidx))
elif ISDFLAT and inst=='lris_red':
outfiles.append(oScanFile_amp1)
indices.append((slice(None, 290, None), allidx))
outfiles.append(oScanFile_amp2)
indices.append((slice(290, None, None), allidx))
return(outfiles, indices)
if ISDFLAT:
if inst == 'lris_blue' and not BLUE_AMP_BAD:
# amp1
outfiles.append(oScanFile_amp1)
indices.append((slice(2260//xbin, 2800//xbin, None), allidx))
# amp2
outfiles.append(oScanFile_amp2)
indices.append((slice(1800//xbin, 2260//xbin, None), allidx))
elif inst=='lris_blue':
# amp1
outfiles.append(oScanFile_amp1)
indices.append((slice(2260//xbin, 2800//xbin, None), allidx))
elif nAmps==2 and not RED_AMP_BAD:
outfiles.append(oScanFile_amp1)
indices.append((slice(26, 290, None), slice(None, -55, None)))
outfiles.append(oScanFile_amp2)
indices.append((slice(290, 500, None), slice(None, -55, None)))
elif nAmps==2 and RED_AMP_BAD:
outfiles.append(oScanFile_amp1)
indices.append((slice(290, 500, None), slice(None, -55, None)))
else:
outfiles.append(oScanFile_amp1)
indices.append((slice(1600//xbin, 2600//xbin, None), allidx))
outfiles.append(oScanFile)
if inst=='lris_blue' and not BLUE_AMP_BAD:
indices.append((slice(1800//xbin, 2800//xbin, None), allidx))
elif inst=='lris_blue':
indices.append((slice(2260//xbin, 2800//xbin, None), allidx))
elif nAmps==2 and RED_AMP_BAD:
indices.append((slice(290, 500, None), slice(None, -55, None)))
elif nAmps==2:
indices.append((slice(26, 500, None), slice(None, -55, None)))
else:
indices.append((slice(1600//xbin, 2600//xbin, None), allidx))
return(outfiles, indices)