def gmos_ls_proc2(
sciTargets,
stdTarget,
dbFile='./raw/obsLog.sqlite3',
qd_full={
'use_me': 1,
'Instrument': 'GMOS-S',
'CcdBin': '2 4',
'RoI': 'Full',
'Disperser': 'B600+_%',
'CentWave': 485.0,
'AperMask': '1.0arcsec',
'Object': 'AM2306-72%',
'DateObs': '2007-06-05:2007-07-07'
},
qd_censp={
'use_me': 1,
'Instrument': 'GMOS-S',
'CcdBin': '2 4',
'RoI': 'CenSp',
'Disperser': 'B600+_%',
'CentWave': 485.0,
'AperMask': '1.0arcsec',
'Object': 'LTT9239',
'DateObs': '2007-06-05:2007-07-07'
},
biasFlags={
'logfile': 'biasLog.txt',
'rawpath': './raw/',
'fl_vardq': 'yes',
'verbose': 'no'
},
flatFlags={
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_dark': 'no',
'fl_fixpix': 'no',
'fl_oversize': 'no',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'rawpath': './raw',
'fl_inter': 'no',
'fl_detec': 'yes',
'function': 'spline3',
'order': '13,11,28',
'logfile': 'gsflatLog.txt',
'verbose': 'no'
},
sciFlags={
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_gscrrej': 'no',
'fl_dark': 'no',
'fl_flat': 'yes',
'fl_gmosaic': 'yes',
'fl_fixpix': 'no',
'fl_gsappwave': 'yes',
'fl_oversize': 'no',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'rawpath': './raw',
'fl_inter': 'no',
'logfile': 'gsreduceLog.txt',
'verbose': 'no'
},
waveFlags={
'coordlist': 'gmos$data/CuAr_GMOS.dat',
'fwidth': 6,
'nsum': 50,
'function': 'chebyshev',
'order': 5,
'fl_inter': 'no',
'logfile': 'gswaveLog.txt',
'verbose': 'no'
},
sciCombFlags={
'combine': 'average',
'reject': 'ccdclip',
'fl_vardq': 'yes',
'fl_dqprop': 'yes',
'logfile': 'gemcombineLog.txt',
'verbose': 'no'
},
transFlags={
'fl_vardq': 'yes',
'interptype': 'linear',
'fl_flux': 'yes',
'logfile': 'gstransLog.txt'
},
skyFlags={
'fl_oversize': 'no',
'fl_vardq': 'yes',
'logfile': 'gsskysubLog.txt'
},
extrFlags={
'apwidth': 3.,
'fl_inter': 'yes',
'find': 'yes',
'trace': 'yes',
'tfunction': 'chebyshev',
'torder': '6',
'tnsum': 20,
'background': 'fit',
'bfunction': 'chebyshev',
'border': 2,
'fl_vardq': 'no',
'logfile': 'gsextrLog.txt'
},
calibFlags={
'extinction': 'onedstds$ctioextinct.dat',
'fl_ext': 'yes',
'fl_scale': 'no',
'sfunction': 'sens',
'fl_vardq': 'yes',
'logfile': 'gscalibrateLog.txt'
},
skip_wavecal=True,
clean_files=False):
"""
Parameters
----------
dbFile : str
Filename containing the SQL sqlite3 database created by obslog.py
It must be placed in the ./raw/ directory
Default is `./raw/obsLog.sqlite3`
sciTargets : dict
Dictionary with the associations of science targets and its associated ARC for wavelength calibration
as well as the regions defining the sky along the slit.
e.g. sciTargetd = {'AM2306-721_a': {'arc': 'gsS20070623S0071', 'sky': '520:720'},
'AM2306-72_b': {'arc': 'gsS20070623S0081', 'sky': '670:760,920:1020'}}
Note that there could be more than one target defined this way.
stdTarget : dict
Dictionary with the associations of standard star targets and its associated ARC for wavelength calibration
as well as the regions defining the sky along the slit.
e.g. stdTarget = {'LTT1788': {'arc': 'S20180711S0281', 'sky': '170:380,920:1080'}}
qd_full : dictionary
Query Dictionary of essential parameter=value pairs for Full RoI. Meant for science object.
qd_censp : dictionary
Query Dictionary of essential parameter=value pairs for CenSp RoI. Meant for standard star.
biasFlags : dict
Dictionary for the keyword flags of gmos.gbias() function
flatFlags : dict
Dictionary for the keyword flags of gmos.gsflat() function
sciFlags : dict
Dictionary for the keyword flags of gmos.gsreduce() function
Based on these flags a set of arcFlags and stdFlags dictionaries will be created
for basic processing.
waveFlags : dict
Dictionary for the keyword flags of gmos.gswavelength() function
sciCombFlags : dict
Dictionary for the keyword flags of gemtools.gemcombine() function
Based on these flags a set of stdCombFlags dictionary will be created for the standard advanced processing.
transFlags : dict
Dictionary for the keyword flags of gmos.gstransform() function.
xxx
skyFlags : dict
Dictionary for the keyword flags of gmos.gsskysub() function
extrFlags : dict
Dictionary for the keywords flags of gmos.gsextract() function
calibFlags : dict
XXX
skip_wavecal : bool
Whether to skip interactive wavelength calibration.
Useful when this is already done.
Returns
-------
"""
print("### Begin Processing GMOS/Longslit Images ###")
print("###")
print("=== Creating MasterCals ===")
# From the work_directory:
# Create the query dictionary of essential parameter=value pairs for Full and CenSp RoIs
qd = {'Full': qd_full, 'CenSp': qd_censp}
print(" --Creating Bias MasterCal--")
# Set the task parameters.
gemtools.gemextn.unlearn() # Disarm a bug in gbias
gmos.gbias.unlearn()
regions = ['Full', 'CenSp']
for r in regions:
# The following SQL generates the list of full-frame files to process.
SQL = fs.createQuery('bias', qd[r])
biasFiles = fs.fileListQuery(dbFile, SQL, qd[r])
# The str.join() funciton is needed to transform a python list into a
# comma-separated string of file names that IRAF can understand.
if len(biasFiles) > 1:
# NT comment: sometimes if there are too many files, gmos.gbias() raises an error.
# import pdb; pdb.set_trace()
gmos.gbias(','.join(str(x) for x in biasFiles), 'MCbias' + r,
**biasFlags)
# Clean up
year_obs = qd_full['DateObs'].split('-')[0]
if clean_files:
iraf.imdel("gS{}*.fits".format(year_obs))
ask_user(
"MC Bias done. Would you like to continue to proceed with GCAL Spectral Master Flats? (y/n): ",
['y', 'yes'])
print(" -- Creating GCAL Spectral Flat-Field MasterCals --")
# Set the task parameters.
qd['Full'].update({'DateObs': '*'})
qd['CenSp'].update({'DateObs': '*'})
gmos.gireduce.unlearn()
gmos.gsflat.unlearn()
# Normalize the spectral flats per CCD.
# The response fitting should be done interactively.
if flatFlags['fl_inter'] != 'yes':
print(
"The response fitting should be done interactively. Please set flatFlags['fl_inter'] = 'yes'."
)
ask_user(
"Do you still want to proceed despite this important warning? (y/n): ",
['yes', 'y'])
for r in regions:
qr = qd[r]
flatFiles = fs.fileListQuery(dbFile, fs.createQuery('gcalFlat', qr),
qr)
if len(flatFiles) > 0:
gmos.gsflat(','.join(str(x) for x in flatFiles),
'MCflat' + r,
bias='MCbias' + r,
**flatFlags)
if clean_files:
iraf.imdel('gS{}*.fits,gsS{}*.fits'.format(year_obs, year_obs))
ask_user(
"GCAL Spectral Flat-Field MasterCals done. Would you like to continue to proceed with Basic Processing? (y/n): ",
['y', 'yes'])
print("=== Processing Science Files ===")
print(" -- Performing Basic Processing --")
# Set task parameters.
gmos.gsreduce.unlearn()
sciFlags = sciFlags # redundant but put here because NT likes it
arcFlags = copy.deepcopy(sciFlags)
arcFlags.update({'fl_flat': 'no', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
stdFlags = copy.deepcopy(sciFlags)
stdFlags.update({'fl_fixpix': 'yes', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
# Perform basic reductions on all exposures for science targets.
print(" - Arc exposures -")
for r in regions:
qr = qd[r]
arcFiles = fs.fileListQuery(dbFile, fs.createQuery('arc', qr), qr)
if len(arcFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in arcFiles),
bias='MCbias' + r,
**arcFlags)
print(" - Std star exposures -")
r = 'CenSp'
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qd[r]), qd[r])
if len(stdFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in stdFiles),
bias='MCbias' + r,
flatim='MCflat' + r,
**stdFlags)
print(" - Science exposures -")
r = 'Full'
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qd[r]),
qd[r])
if len(sciFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in sciFiles),
bias='MCbias' + r,
flatim='MCflat' + r,
**sciFlags)
# Clean up
if clean_files:
iraf.imdel('gS{}*.fits'.format(year_obs))
ask_user(
"Basic processing done. Would you like to continue to determine wavelength calibration? (y/n): ",
['y', 'yes'])
print(" -- Determine wavelength calibration --")
# Set task parameters
gmos.gswavelength.unlearn()
# The fit to the dispersion relation should be performed interactively.
# Here we will use a previously determined result.
if waveFlags['fl_inter'] != 'yes':
print(
"The fit to the dispersion relation should be performed interactively. Please set waveFlags['fl_inter'] = 'yes'."
)
ask_user(
"Do you still want to proceed despite this important warning? (y/n): ",
['yes', 'y'])
# Need to select specific wavecals to match science exposures.
# NT: we do this now from the sciTargets + stdTarget input dictionaries
# e.g.
'''
sciTargets = {
'AM2306-721_a': {'arc': 'gsS20070623S0071', 'sky': '520:720'},
'AM2306-72_b': {'arc': 'gsS20070623S0081', 'sky': '670:760,920:1020'},
'AM2306-721_c': {'arc': 'gsS20070623S0091', 'sky': '170:380,920:1080'}
}
'''
#prefix = 'gsS20070623S0'
#for arc in ['071', '081', '091', '109']:
# gmos.gswavelength(prefix + arc, **waveFlags)
prefix = 'gs'
arc_files = []
for key in sciTargets.keys():
arc_files += [sciTargets[key]['arc']]
for key in stdTarget.keys():
arc_files += [stdTarget[key]['arc']]
# import pdb; pdb.set_trace()
if skip_wavecal is not True:
for arc in arc_files:
gmos.gswavelength(prefix + arc, **waveFlags)
### End of basic processing. Continue with advanced processing.
ask_user(
"Wavelength solution done. Would you like to continue with advanced processing? (y/n): ",
['y', 'yes'])
print(" -- Performing Advanced Processing --")
print(" -- Combine exposures, apply dispersion, subtract sky --")
# Set task parameters.
gemtools.gemcombine.unlearn()
sciCombFlags = sciCombFlags
stdCombFlags = copy.deepcopy(sciCombFlags)
stdCombFlags.update({'fl_vardq': 'no', 'fl_dqprop': 'no'})
gmos.gstransform.unlearn()
# apply gtransform to standard
# Process the Standard Star
prefix = "gs"
qs = qd['CenSp']
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qs), qs)
std_name = stdTarget.keys()[0]
if len(stdFiles) == 0:
ValueError(
"No standard star associated. Please check parameters of search (e.g. RoI=CentSp)"
)
# import pdb; pdb.set_trace()
if len(stdFiles) > 1:
# import pdb; pdb.set_trace()
gemtools.gemcombine(','.join(prefix + str(x) for x in stdFiles),
std_name, **stdCombFlags)
else:
os.system("cp {}.fits {}.fits".format(prefix + stdFiles[0], std_name))
gmos.gstransform(std_name,
wavtraname=prefix + stdTarget[std_name]['arc'],
**transFlags)
# The sky regions should be selected with care, using e.g. prows/pcols:
# pcols ("tAM2306b.fits[SCI]", 1100, 2040, wy1=40, wy2=320)
print(
"The sky regions should be selected with care, using e.g. with prows/pcols (see tutorial)."
)
'''
answer = raw_input("Please provide the long_sample string to apply to gmos.gsskysub() for the standard star."
"e.g. '20:70,190:230' (say 'no' for using the example as the default values): ")
if answer in ['n', 'no']:
print("Using default long_sample set by stdTarget values {}.".format(stdTarget[std_name]['sky']))
long_sample_std = stdTarget[std_name]['sky']
else:
long_sample_std = answer
'''
long_sample_std = stdTarget[std_name]['sky']
ask_user(
"Before proceeding it is important that you have set a good sky region for the standard.\n"
"Thus far you have selected: {}\n Would you like to proceed with the current one? (y/n): "
.format(long_sample_std), ['yes', 'y'])
# apply sky substraction
skyFlags = skyFlags
gmos.gsskysub.unlearn()
gmos.gsskysub('t{}'.format(std_name), long_sample=long_sample_std)
# NT: make sure the process works ok until here before proceeding further. i.e. setting the sky region manually and correctly.
# NT: seems to be working.
print(" -- Extract Std spectrum --")
# Extract the std spectrum using a large aperture.
# It's important to trace the spectra interactively.
gmos.gsextract.unlearn()
gmos.gsextract("st" + std_name, **extrFlags)
print(" -- Derive the Flux calibration --")
gmos.gsstandard.unlearn()
sensFlags = {
'fl_inter': 'no',
'starname': 'XXX',
'caldir': 'onedstds$ctionewcal/',
'observatory': 'Gemini-South',
'extinction': 'onedstds$ctioextinct.dat',
'function': 'chebyshev',
'order': 9,
'verbose': 'no',
'logfile': 'gsstdLog.txt'
}
sensFlags['starname'] = stdTarget[std_name][
'iraf_name'] # replace corresponding starname
gmos.gsstandard('est' + std_name,
sfile='std.txt',
sfunction='sens',
**sensFlags)
ask_user(
"Sensitivity function from standard star done. Would you like to continue with reduction of science"
" exposures? (y/n): ", ['yes', 'y'])
# Process the science targets.
# Use a dictionary to associate science targets with Arcs and sky regions.
prefix = 'gs'
extract_individuals = True
for targ, p in sciTargets.iteritems():
qs = qd['Full']
qs['Object'] = p['name']
# Fix up the target name for the output file
sciOut = p['name_out']
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qs), qs)
all_files = ','.join(prefix + str(x) for x in sciFiles)
gemtools.gemcombine(all_files, sciOut, **sciCombFlags)
gmos.gstransform(sciOut, wavtraname=prefix + p['arc'], **transFlags)
ask_user(
"It is important to select a good sky region for substraction. Thus far you have selected {}"
" based on the sciTargets input dictionary. Would you like to continue? (y/n): "
.format(p['sky']), ['y', 'yes'])
gmos.gsskysub('t' + sciOut, long_sample=p['sky'], **skyFlags)
if extract_individuals:
import pdb
pdb.set_trace()
for fname in sciFiles:
gmos.gstransform(prefix + fname,
wavtraname=prefix + p['arc'],
**transFlags)
gmos.gsskysub('t' + prefix + fname,
long_sample=p['sky'],
**skyFlags)
gmos.gscalibrate.unlearn()
gmos.gscalibrate('st' + prefix + fname, **calibFlags)
# Clean up
if clean_files:
iraf.imdel("gsS{}*.fits".format(year_obs))
ask_user(
"Sky substraction done. Would you like to continue to apply sensitivity function? (y/n): ",
['y'])
## Apply the sensitivity function.
gmos.gscalibrate.unlearn()
gmos.gscalibrate('st' + sciOut + '*', **calibFlags)
calibFlags.update({'fl_vardq': 'no'})
gmos.gscalibrate('est' + std_name, **calibFlags)
print(" -- Extract Target Spectra --")
method = 'gsextract'
if method == 'gsextract':
gmos.gsextract.unlearn()
# import pdb;pdb.set_trace()
gmos.gsextract("cst" + sciOut, **extrFlags)
elif method == 'sarith':
# not implemented yet
onedspec.nsum = 4
onedspec.sarith('cst{}.fits[SCI]'.format(sciOut),
'copy',
'',
'ecst{}.ms'.format(sciOut),
apertures='222-346x4')
print("=== Finished Calibration Processing ===")
def gmos_mos_proc():
'''
Modified version of the GMOS Data Reduction Cookbook companion script to the chapter:
"Reduction of Multi-Object Spectra with IRAF"
PyRAF script to:
Process MOS exposures for Sculptor Dwarf field 1, in program GN-2008-B-Q-025
The names for the relevant header keywords and their expected values are
described in the DRC chapter entitled "Supplementary Material"
Perform the following starting in the parent work directory:
cd /path/to/work_directory
Place the fileSelect.py module in your work directory. Now execute this
script from the unix prompt:
python gmos_img_proc.py
'''
print ("### Begin Processing GMOS/MOS Spectra ###")
print (' ')
print ("---> You must have the MDF files:")
print ("---> GS2008BQ025-01.fits and GS2008BQ025-02.fits")
print ("---> in your work directory. ")
print (' ')
print ("=== Creating MasterCals ===")
dbFile='raw/obsLog.sqlite3'
#Create query dictionaries for the science observations at each CentWave
# Select bias exposures within ~2 months of the target observations:
qdf = {'use_me':1,
'CcdBin':'4 2',
'DateObs':'2008-09-10:2008-12-12',
#'DateObs':'2008-10-20:2008-11-21',
'Instrument':'GMOS-S',
'Disperser':'B600+_%',
'AperMask':'GS2008BQ025-01',
'CentWave':520.0,
'Object':'Sculptor-field1',
'RoI':'Full'
}
#Create query dictionaries for the standard star observation
qd_std = copy.deepcopy(qdf)
qd_std['AperMask'] = '1.0arcsec'
qd_std['Object'] = 'LTT1020'
print (" --Creating Bias MasterCal-- ")
#Use primarily the default task parameters
gemtools.gemextn.unlearn() # Disarm a bug in gbias
gmos.gbias.unlearn()
#gmos.gbias.logfile = 'biasLog.txt'
#gmos.gbias.rawpath = './raw/'
#gmos.gbias.fl_vardq = 'yes'
#gmos.gbias.verbose = 'no'
biasFlags = {
'logfile':'biasLog.txt','rawpath':'./raw/','fl_vardq':'yes',
'verbose':'no'
}
#This SQL query generates the list of full-frame files to process. Note that since the std star has the
# same RoI, CCD binning, and CCD gain/read-out speed, we only need to make one bias file.
SQL = fs.createQuery('bias', qdf)
biasFull = fs.fileListQuery(dbFile, SQL, qdf)
# The join function originally used runs into problems - use this f.write
# to make a string of comma-separated files that IRAF can understand.
print (" --Generating MasterCal for Full-- ")
with open('biases.lis', 'w') as f:
[f.write(x+'\n') for x in biasFull]
#Create the bias MasterCal
gmos.gbias('@biases.lis', 'MCbiasFull.fits', **biasFlags)
# Clean up
#iraf.imdel('gS2008*.fits')
print (" --Creating GCAL Spectral Flat-Field MasterCals--")
# Set the task parameters.
gmos.gireduce.unlearn()
gmos.gsflat.unlearn()
gmos.gsflat.fl_vardq = 'yes'
gmos.gsflat.fl_fulldq = 'yes'
gmos.gsflat.fl_oversize = 'no'
gmos.gsflat.fl_inter = 'no'
gmos.gsflat.logfile = 'gsflatLog.txt'
gmos.gsflat.rawpath = './raw'
gmos.gsflat.verbose = 'no'
call("ls Sculptor*.fits",shell=True)
#Perform flat-field normalization for the science images
print (" -Full Flat (GCAL & Twi) normalization for science images, non-interactive-")
qdf['DateObs'] = '*'
qdf['Filter2'] = 'open2-8'
cwf = {'B6-520':520.0, 'B6-525':525.0, 'B6-522':522.5}
#flatType = ['gcalFlat', 'twiFlat']
#No twilight flats were available for this observation - calibrating using only the gcal flats
flatType = ['gcalFlat']
for ft in flatType:
for tag,w in cwf.iteritems():
qdf['Disperser'] = tag[0:2] + '00+_%'
qdf['CentWave'] = w
flatName = 'MC' + ft + '-M01_' + tag
combName = 'MC' + ft + 'Comb-M01_' + tag
flatFull = fs.fileListQuery(dbFile, fs.createQuery(ft, qdf), qdf)
with open('flats_sci.lis', 'w') as f:
[f.write(x+'\n') for x in flatFull]
print "Flatfielding for " + str(ft) + " and " + str(w)
gmos.gsflat ('@flats_sci.lis', flatName, bias='MCbiasFull',
fl_keep='yes', combflat=combName, fl_usegrad='yes',
fl_seprows='no', order='53')
os.remove('flats_sci.lis')
call("ls Sculptor*.fits",shell=True)
#Perform flat-field normalization for the standard star. Standard star was taken at centw 415,520,625
print (" -Full Flat (GCAL & Twi) normalization for the standard star, non-interactive-")
qd_std['DateObs'] = '*'
qd_std['Filter2'] = 'open2-8'
cws = {'B6-415':415.0, 'B6-520':520.0, 'B6-625':625.0}
#flatType = ['gcalFlat', 'twiFlat']
#No twilight flats were available for this observation - calibrating using only the gcal flats
flatType = ['gcalFlat']
for ft in flatType:
for tag,w in cws.iteritems():
qd_std['Disperser'] = tag[0:2] + '00+_%'
qd_std['CentWave'] = w
flatName = 'MC' + ft + '-M01_' + tag
combName = 'MC' + ft + 'Comb-M01_' + tag
flatFull = fs.fileListQuery(dbFile, fs.createQuery(ft, qd_std), qd_std)
with open('flats_std.lis', 'w') as f:
[f.write(x+'\n') for x in flatFull]
gmos.gsflat ('@flats_std.lis', flatName, bias='MCbiasFull',
fl_keep='yes', combflat=combName, fl_usegrad='yes',
fl_seprows='no', order='53')
os.remove('flats_std.lis')
call("ls Sculptor*.fits",shell=True)
print ("=== Processing Science Files ===")
print (" -- Performing Basic Processing --")
# Use primarily the default task parameters.
gmos.gsreduce.unlearn()
gmos.gsreduce.logfile = 'gsreduceLog.txt'
gmos.gsreduce.rawpath = './raw'
gmos.gsreduce.verbose = 'no'
gmos.gsreduce.fl_fixpix = 'no'
gmos.gsreduce.fl_oversize = 'no'
#Perform single-frame CR rejection
#gmos.gsreduce.fl_gscr = 'yes'
print (" - GSReducing MOS Science and Arc exposures -")
for tag,w in cwf.iteritems():
qdf['Disperser'] = tag[0:2] + '00+_%'
qdf['CentWave'] = w
flatName = 'MCgcalFlat-M01_' + tag
gradName = 'MCgcalFlatComb-M01_' + tag
arcFull = fs.fileListQuery(dbFile, fs.createQuery('arc', qdf), qdf)
gmos.gsreduce (','.join(str(x) for x in arcFull), bias='MCbiasFull',
gradimage=gradName, fl_flat='no')
sciFull = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qdf), qdf)
gmos.gsreduce (','.join(str(x) for x in sciFull), bias='MCbiasFull',
flatim=flatName, gradimage=gradName,
fl_vardq='yes', fl_fulldq='yes')
call("ls Sculptor*.fits",shell=True)
print (" - GSReducing Longslit Std-star and Arc exposures -")
for tag,w in cws.iteritems():
qd_std['Disperser'] = tag[0:2] + '00+_%'
qd_std['CentWave'] = w
flatName = 'MCgcalFlat-M01_' + tag
arc_std = fs.fileListQuery(dbFile, fs.createQuery('arc', qd_std), qd_std)
gmos.gsreduce (','.join(str(x) for x in arc_std), bias='MCbiasFull',
fl_flat='no')
std_files = fs.fileListQuery(dbFile, fs.createQuery('std', qd_std), qd_std)
gmos.gsreduce (','.join(str(x) for x in std_files), bias='MCbiasFull',
flatim=flatName, fl_fixpix='yes')
call("ls Sculptor*.fits",shell=True)
# Clean up - uncomment this eventually
#iraf.imdel('gS2008*.fits')
print ("=== Finished Basic Calibration Processing ===")
print ("\n")
print ("=== Performing cosmic-ray rejection using gemcrspec ===")
#note that this construction works because there's only one exposure per position/grating/cenwave combo
#if you have multiple exposures, comment this block out and use gemcombine to do outlier rejection when combining images instead
gemtools.gemcrspec.unlearn()
gemtools.gemcrspec.xorder = '9'
gemtools.gemcrspec.yorder = '-1'
gemtools.gemcrspec.sigclip = '4.5'
gemtools.gemcrspec.sigfrac= '0.5'
gemtools.gemcrspec.objlim = '1.0'
gemtools.gemcrspec.verbose = 'no'
prefix = 'gs'
for tag,w in cwf.iteritems():
qdf['Disperser'] = tag[0:2] + '00+_%'
qdf['CentWave'] = w
outFile = qdf['Object'] + '-M01_' + tag
sciFull = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qdf), qdf)
print sciFull
gemtools.gemcrspec(','.join(prefix+str(x) for x in sciFull), outFile)
# Do the same for the standard star
for tag,w in cws.iteritems():
qd_std['Disperser'] = tag[0:2] + '00+_%'
qd_std['CentWave'] = w
outFile = qd_std['Object'] + '-M01_' + tag
stdFull = fs.fileListQuery(dbFile, fs.createQuery('std', qd_std), qd_std)
print stdFull
gemtools.gemcrspec(','.join(prefix+str(x) for x in stdFull), outFile)
call("ls Sculptor*.fits",shell=True)
#Unused block for doing outlier rejection with multiple exposures per position/grating/cenwave combo
'''
# Use primarily the default task parameters.
gemtools.gemcombine.unlearn()
gemtools.gemcombine.logfile = 'gemcombineLog.txt'
gemtools.gemcombine.reject = 'ccdclip'
gemtools.gemcombine.fl_vardq = 'yes'
gemtools.gemcombine.fl_dqprop = 'yes'
gemtools.gemcombine.verbose = 'no'
prefix = 'gs'
for tag,w in cwf.iteritems():
qdf['Disperser'] = tag[0:2] + '00+_%'
qdf['CentWave'] = w
outFile = qdf['Object'] + tag
sciFull = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qdf), qdf)
gemtools.gemcombine (','.join(prefix+str(x) for x in sciFull), outFile)
# Do the same for the standard star
for tag,w in cws.iteritems():
qdf['Disperser'] = tag[0:2] + '00+_%'
qdf['CentWave'] = w
outFile = qd_std['Object'] + tag
stdFull = fs.fileListQuery(dbFile, fs.createQuery('std', qd_std), qd_std)
gemtools.gemcombine (','.join(prefix+str(x) for x in stdFull), outFile)
'''
print ("=== Beginning wavelength calibration ===")
print (" -- Deriving wavelength calibration --")
# Begin with longslit Arcs.
# The fit to the dispersion relation should be performed interactively;
# here we will us a previously determined result.
# There are many arcs to choose from: we only need one for each setting.
gmos.gswavelength.unlearn()
waveFlags = {
'coordlist':'gmos$data/CuAr_GMOS.dat','fwidth':6,'nsum':50,
'function':'chebyshev','order':5,
'fl_inter':'no','logfile':'gswaveLog.txt','verbose':'no'
}
for seq in ['091','092','093']:
inFile = prefix + 'S20081129S0' + seq
gmos.gswavelength(inFile,**waveFlags)
# Now for the MOS arcs
waveFlags.update({'order':7,'nsum':20,'step':2})
for seq in ['249','250','251']:
inFile = prefix + 'S20081120S0' + seq
gmos.gswavelength(inFile,**waveFlags)
call("ls Sculptor*.fits",shell=True)
#This block is in the tutorial but it seems incorrect - it applies the calibration to non gsreduced arcs!!
'''
for tag,w in cwf.iteritems():
qdf['Disperser'] = tag[0:2] + '00+_%'
qdf['CentWave'] = w
outFile = qdf['Object'] + tag
arcFull = fs.fileListQuery(dbFile, fs.createQuery('arcP', qdf), qdf)
gmos.gswavelength (','.join(prefix+str(x) for x in arcFull),**waveFlags)
'''
print (" -- Applying wavelength calibration -- ")
gmos.gstransform.unlearn()
transFlags = {
'fl_vardq':'no','interptype':'linear','fl_flux':'yes',
'logfile':'gstransformLog.txt','verbose':'no'
}
#Construct a mapping for the wavelength calibration. Format (arc id,sci/std id,target):'filter/disperser'
print (" -- Calibrating standard star exposures -- ")
gmos.gstransform ('LTT1020-M01_B6-415', wavtraname='gsS20081129S0091',
**transFlags)
gmos.gstransform ('LTT1020-M01_B6-520', wavtraname='gsS20081129S0092',
**transFlags)
gmos.gstransform ('LTT1020-M01_B6-625', wavtraname='gsS20081129S0093',
**transFlags)
call("ls Sculptor*.fits",shell=True)
#This block seems to operate on the non CR-cleaned images!
'''
transMap = {
('091','036','LTT1020'):'B6-415',
('092','039','LTT1020'):'B6-520',
('093','040','LTT1020'):'B6-625'
}
print (" -- Calibrating standard star exposures -- ")
for id,tag in transMap.iteritems():
inFile = 'gsS20081129S0' + id[1]
wavFile = 'gsS20081129S0' + id[0]
outFile = 't' + id[2] + '_' + tag
gmos.gstransform(inFile,outimages=outFile,wavtraname=wavFile)
'''
print (" -- Calibrating MOS science exposures -- ")
transFlags.update({'fl_vardq':'yes'})
gmos.gstransform ('Sculptor-field1-M01_B6-520', wavtraname='gsS20081120S0249',
**transFlags)
gmos.gstransform ('Sculptor-field1-M01_B6-522', wavtraname='gsS20081120S0250',
**transFlags)
gmos.gstransform ('Sculptor-field1-M01_B6-525', wavtraname='gsS20081120S0251',
**transFlags)
call("ls Sculptor*.fits",shell=True)
#Note that our standard star is extremely bright so the signal is swamped by light from the star - this step is probably optional
print (" == Beginning flux calibration == " )
print (" -- Performing sky subtraction on standard star -- " )
# This will require summing the spectra along columns, e.g.:
# iraf.pcols("tLTT1020-M01_B6-415.fits[SCI]",400,1400,wy1=0,wy2=1000)
# The sky regions should be selected with care, using e.g. prows/pcols.
gmos.gsskysub.unlearn()
gmos.gsskysub.logfile='gsskysubLog.txt'
gmos.gsskysub.fl_oversize = 'no'
gmos.gsskysub.verbose = 'no'
gmos.gsskysub ('tLTT1020-M01_*', long_sample='850:1000,1350:1500')
call("ls Sculptor*.fits",shell=True)
print(" -- Extracting longslit 1-D spectra of standard stars -- ")
gmos.gsextract.unlearn()
extrFlags = {
'apwidth':3.,'fl_inter':'no','find':'yes',
'trace':'yes','tfunction':'spline3','tnsum':20,'tstep':50,
'weights':'none','background':'none',
'fl_vardq':'no','verbose':'no','logfile':'gsextractLog.txt'
}
ordc = {'B6-415':7,'B6-520':8,'B6-625':8}
#Perform extraction on the standard star:
for tag,o in ordc.iteritems():
infile = 'stLTT1020-M01_' + tag
gmos.gsextract(infile, torder=o,**extrFlags)
print(" -- Flux calibrating the standard star -- ")
gmos.gsstandard.unlearn()
sensFlags = {
'fl_inter':'yes','starname':'l1020','caldir':'onedstds$ctionewcal/',
'observatory':'Gemini-South','extinction':'onedstds$ctioextinct.dat',
'function':'spline3','order':7,'verbose':'no','logfile':'gsstdLog.txt'
}
gmos.gsstandard ('estLTT1020-M01_B6*', sfile='std_B6', sfunction='sens_B6',
**sensFlags)
call("ls Sculptor*.fits",shell=True)
print (" --Processing done-- ")
'logfile': 'gsreduceLog.txt',
'verbose': 'no'
}
arcFlags = copy.deepcopy(sciFlags)
arcFlags.update({'fl_flat': 'no', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
stdFlags = copy.deepcopy(sciFlags)
stdFlags.update({'fl_fixpix': 'yes', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
# Perform basic reductions on all exposures for science targets.
print(" - Arc exposures -")
for r in regions:
qr = qd[r]
arcFiles = fs.fileListQuery(dbFile, fs.createQuery('arc', qr), qr)
if len(arcFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in arcFiles),
bias='MCbias' + r,
**arcFlags)
print(" - Std star exposures -")
r = 'CenSp'
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qd[r]), qd[r])
if len(stdFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in stdFiles),
bias='MCbias' + r,
flatim='MCflat' + r,
**stdFlags)
print(" - Science exposures -")
r = 'Full'
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qd[r]),
qd[r])
def reduce_science(summary, cent_waves):
"""
Reduce the science and arc images and apply wavelength transformation
this produces non sky subtracted, non-combined, transformed 2D slits to then
be loaded into PYPIT.
:param summary: astroypy Table object with header data in it for each file
:param cent_waves: list of floats, central wavelengths from summary
:return: None
"""
print("=== Processing Science Files ===")
print(" -- Performing Basic Processing --")
# Use primarily the default task parameters.
gmos.gsreduce.unlearn()
sciFlags = {
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_gscrrej': 'no',
'fl_dark': 'no',
'fl_flat': 'yes',
'fl_gmosaic': 'yes',
'fl_fixpix': 'no',
'fl_gsappwave': 'yes',
'fl_oversize': 'no',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'fl_inter': 'no',
'logfile': 'gsreduceLog.txt',
'verbose': 'no'
}
arcFlags = copy.deepcopy(sciFlags)
arcFlags.update({'fl_flat': 'no', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
gmos.gswavelength.unlearn()
# USE JESS' PARAMS
if use_jess:
waveFlags = {
'function': 'chebyshev',
'order': 4,
'fl_inter': 'no',
'logfile': 'gswaveLog.txt',
'verbose': 'yes',
'step': 2,
'nlost': 10
}
else:
waveFlags = {
'coordlist': 'gmos$data/CuAr_GMOS.dat',
'fwidth': 6,
'nsum': 50,
'function': 'chebyshev',
'order': 7,
'fl_inter': 'no',
'logfile': 'gswaveLog.txt',
'verbose': 'no'
}
waveFlags.update({'order': 7, 'nsum': 20, 'step': 2})
gmos.gstransform.unlearn()
transFlags = {
'fl_vardq': 'yes',
'interptype': 'linear',
'fl_flux': 'yes',
'logfile': 'gstransformLog.txt',
'verbose': 'no'
}
print(' - MOS Science and Arc exposures -')
prefix = 'gs' #
ft = 'gcalFlat' # flat type
for i, cw in enumerate(cent_waves):
flatName = 'MC' + ft + str(cw)[:-2]
gradName = 'MC' + ft + 'Comb' + str(cw)[:-2]
################### ARCS: Reduce and find wavelengths ###############################
# Arcs
# arcFull = fs.fileListQuery(dbFile, fs.createQuery('arc', qdf), qdf)
arcs = summary[(summary['OBSTYPE'] == 'ARC')
& (summary['CENTWAVE'] == cw)]['FILENAME']
arcFull = [os.path.basename(f) for f in arcs]
reduced_arcs = [prefix + str(x) for x in arcFull]
if not os.path.exists(reduced_arcs[0]):
gmos.gsreduce(','.join(str(x) for x in arcFull),
bias=biasName,
gradimage=gradName,
**arcFlags)
# find max extenstion number:
max_ext = get_max_extension(reduced_arcs[0])
# import pdb; pdb.set_trace()
# IF YOU WANT TO SKIP A SLIT USE THESE LINES!
if skip_slit:
if num_slits == 1:
if slits_to_skip[0] == 1:
iraf.mgswavelength(','.join(prefix + str(x)
for x in arcFull),
firstsciext=2,
lastsciext=max_ext,
**waveFlags)
else:
iraf.mgswavelength(','.join(prefix + str(x)
for x in arcFull),
firstsciext=1,
lastsciext=slits_to_skip[0] - 1,
**waveFlags)
iraf.mgswavelength(','.join(prefix + str(x)
for x in arcFull),
firstsciext=slits_to_skip[0] + 1,
lastsciext=max_ext,
**waveFlags)
elif num_slits == 2:
# check they arent sequential
if slits_to_skip[1] - slits_to_skip[0] != 1:
if slits_to_skip[0] == 1:
iraf.mgswavelength(','.join(prefix + str(x)
for x in arcFull),
firstsciext=2,
lastsciext=slits_to_skip[1] - 1,
**waveFlags)
iraf.mgswavelength(
','.join(prefix + str(x) for x in arcFull),
firstsciext=slits_to_skip[1] + 1,
lastsciext=max_ext,
**waveFlags)
else:
iraf.mgswavelength(','.join(prefix + str(x)
for x in arcFull),
firstsciext=1,
lastsciext=slits_to_skip[0] - 1,
**waveFlags)
iraf.mgswavelength(
','.join(prefix + str(x) for x in arcFull),
firstsciext=slits_to_skip[0] + 1,
lastsciext=slits_to_skip[1] - 1,
**waveFlags)
iraf.mgswavelength(
','.join(prefix + str(x) for x in arcFull),
firstsciext=slits_to_skip[1] + 1,
lastsciext=max_ext,
**waveFlags)
# if the slits ARE sequential
else:
if slits_to_skip[0] == 1:
iraf.mgswavelength(
','.join(prefix + str(x) for x in arcFull),
firstsciext=slits_to_skip[1] + 1,
lastsciext=ax_ext,
**waveFlags)
else:
iraf.mgswavelength(','.join(prefix + str(x)
for x in arcFull),
firstsciext=1,
lastsciext=slits_to_skip[0] - 1,
**waveFlags)
iraf.mgswavelength(
','.join(prefix + str(x) for x in arcFull),
firstsciext=slits_to_skip[1] + 1,
lastsciext=max_ext,
**waveFlags)
else:
print("you input too many slits")
else:
gmos.gswavelength(','.join(prefix + str(x) for x in arcFull),
**waveFlags)
################### SCI: Reduce and find wavelengths ###############################
# Science Images
# sciFull = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qdf), qdf)
sci = summary[(summary['OBSTYPE'] == 'OBJECT')
& (summary['CENTWAVE'] == cw)]['FILENAME']
sciFull = [os.path.basename(f) for f in sci]
reduced_imgs = [prefix + str(x) for x in sciFull]
# get the quasar name convention right: quasar+mask+centwave
quasar_name = summary[(summary['OBSTYPE'] == 'OBJECT')
& (summary['CENTWAVE'] == cw)]['OBJECT'][0]
maskname = summary[(summary['OBSTYPE'] == 'OBJECT')
& (summary['CENTWAVE'] == cw)]['MASKNAME'][0][-2:]
outFile = quasar_name + '_m' + maskname + '_' + str(cw)[:-2]
outFile = outFile.replace('+', '_')
gmos.gsreduce.unlearn()
# Rollup: 2 central waves and 2 exposures at each.
# This means we can combine the exposure at the same central wavelength to perform
# cosmic ray reduction
if len(cent_waves) < 3:
# reduce the SCIENCE images
if not os.path.exists(reduced_imgs[0]):
gmos.gsreduce(','.join(str(x) for x in sciFull),
bias=biasName,
flatim=flatName,
gradimage=gradName,
**sciFlags)
# Set CR task parameters.
# NOTE: Turned on crreject. Does a good job of getting rid of cosmic rays
gemtools.gemcombine.unlearn()
sciCombFlags = {
'combine': 'average',
'reject': 'crreject',
'fl_vardq': 'yes',
'fl_dqprop': 'yes',
'logfile': 'gemcombineLog.txt',
'verbose': 'no'
}
# Rollup data with 2 science exposures and 2 centwaves
i = i + 1
combined_outFile = "cr_sci" + str(i) + "w_auto_slit.fits"
# combine the 2 exposures at the same central wavelength
gemtools.gemcombine(','.join(prefix + str(x) for x in sciFull),
combined_outFile, **sciCombFlags)
# transform the sci images
gmos.gstransform(combined_outFile,
wavtraname='gs' + arcFull[0],
outimages=outFile,
**transFlags)
else:
# This is the Long Program data with 3 central wavelength with 1 exposure at each.
# Cosmic ray reduction needs to be performed
# can be done with fl_crspec=yes in gs reduce...
sciFlags = {
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_gscrrej': 'yes',
'fl_dark': 'no',
'fl_flat': 'yes',
'fl_gmosaic': 'yes',
'fl_fixpix': 'no',
'fl_gsappwave': 'yes',
'fl_oversize': 'no',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'fl_inter': 'no',
'logfile': 'gsreduceLog.txt',
'verbose': 'no'
}
# reduce the SCIENCE images
if not os.path.exists(reduced_imgs[0]):
gmos.gsreduce(','.join(str(x) for x in sciFull),
bias=biasName,
flatim=flatName,
gradimage=gradName,
**sciFlags)
# LP with 3 central wavelengths and 1 science exposure
else:
print("skipping gsreduce: file already exists " +
reduced_imgs[0] + ".fits")
# get the quasar name convention right: quasar+mask+centwave
quasar_name = summary[(summary['OBSTYPE'] == 'OBJECT')
& (summary['CENTWAVE'] == cw)]['OBJECT'][0]
maskname = summary[(summary['OBSTYPE'] == 'OBJECT') &
(summary['CENTWAVE'] == cw)]['MASKNAME'][0][-2:]
outFile = quasar_name + '_m' + maskname + '_' + str(cw)[:-2]
outFile = outFile.replace('+', '_')
if not os.path.exists(outFile + ".fits"):
# transform the sci images
gmos.gstransform(reduced_imgs[0],
wavtraname='gs' + arcFull[0],
outimages=outFile,
**transFlags)
else:
print("skipping gstransform: file already exists" + outFile +
".fits")
return None
'fl_dark': 'no',
'fl_qecorr': 'no',
'fl_flat': 'no',
'fl_gmosaic': 'no',
'fl_fixpix': 'no',
'fl_gsappwave': 'no',
'fl_scatsub': 'no',
'fl_cut': 'no',
'fl_title': 'no',
'fl_oversize': 'yes',
'fl_vardq': 'no',
'fl_fulldq': 'no',
'fl_inter': 'no',
'verbose': 'no'
}
gmos.gsreduce(flat, bias='../Zero.fits', **Flags)
flat = 'gs' + flat
print("=== gsreduce finished ===")
print("=== Pre-processing Arcs ===")
print("WARNING! Don't forget to implot your arcs to check instrument flexure.")
# Prompts for arc lamp file; format 'gFILE.fits'
arc = input('Arc file: ')
print("Running gsreduce on the arc images in order to:")
print("- perform overscan subtraction;")
print("- trim the frames;")
print("- perform bias subtraction;")
print("- mosaic the science extensions.")
def gmos_ls_proc2(
sciTargets,
stdTarget,
dbFile='./raw/obsLog.sqlite3',
qd_full={'use_me': 1, 'Instrument': 'GMOS-S', 'CcdBin': '2 4', 'RoI': 'Full', 'Disperser': 'B600+_%', 'CentWave': 485.0, 'AperMask': '1.0arcsec', 'Object': 'AM2306-72%','DateObs': '2007-06-05:2007-07-07'},
qd_censp={'use_me': 1, 'Instrument': 'GMOS-S', 'CcdBin': '2 4', 'RoI': 'CenSp', 'Disperser': 'B600+_%', 'CentWave': 485.0, 'AperMask': '1.0arcsec', 'Object': 'LTT9239','DateObs': '2007-06-05:2007-07-07'},
biasFlags={'logfile': 'biasLog.txt', 'rawpath': './raw/', 'fl_vardq': 'yes', 'verbose': 'no'},
flatFlags = {'fl_over': 'yes', 'fl_trim': 'yes', 'fl_bias': 'yes', 'fl_dark': 'no', 'fl_fixpix': 'no', 'fl_oversize': 'no', 'fl_vardq': 'yes', 'fl_fulldq': 'yes','rawpath': './raw', 'fl_inter': 'no', 'fl_detec': 'yes', 'function': 'spline3', 'order': '13,11,28', 'logfile': 'gsflatLog.txt', 'verbose': 'no'},
sciFlags = {'fl_over': 'yes', 'fl_trim': 'yes', 'fl_bias': 'yes', 'fl_gscrrej': 'no','fl_dark': 'no', 'fl_flat': 'yes', 'fl_gmosaic': 'yes', 'fl_fixpix': 'no', 'fl_gsappwave': 'yes', 'fl_oversize': 'no', 'fl_vardq': 'yes', 'fl_fulldq': 'yes', 'rawpath': './raw', 'fl_inter': 'no', 'logfile': 'gsreduceLog.txt', 'verbose': 'no'},
waveFlags = {'coordlist': 'gmos$data/CuAr_GMOS.dat', 'fwidth': 6, 'nsum': 50, 'function': 'chebyshev', 'order': 5, 'fl_inter': 'no', 'logfile': 'gswaveLog.txt', 'verbose': 'no'},
sciCombFlags = {'combine': 'average', 'reject': 'ccdclip', 'fl_vardq': 'yes', 'fl_dqprop': 'yes', 'logfile': 'gemcombineLog.txt', 'verbose': 'no'},
transFlags={'fl_vardq': 'yes', 'interptype': 'linear', 'fl_flux': 'yes', 'logfile': 'gstransLog.txt'},
skyFlags={'fl_oversize': 'no', 'fl_vardq': 'yes', 'logfile': 'gsskysubLog.txt'},
extrFlags = {'apwidth': 3., 'fl_inter': 'yes', 'find': 'yes','trace': 'yes', 'tfunction': 'chebyshev', 'torder': '6', 'tnsum': 20, 'background': 'fit', 'bfunction': 'chebyshev', 'border': 2, 'fl_vardq': 'no', 'logfile': 'gsextrLog.txt'},
calibFlags = {'extinction': 'onedstds$ctioextinct.dat', 'fl_ext': 'yes', 'fl_scale': 'no','sfunction': 'sens', 'fl_vardq': 'yes', 'logfile': 'gscalibrateLog.txt'},
skip_wavecal=True,
clean_files=False):
"""
Parameters
----------
dbFile : str
Filename containing the SQL sqlite3 database created by obslog.py
It must be placed in the ./raw/ directory
Default is `./raw/obsLog.sqlite3`
sciTargets : dict
Dictionary with the associations of science targets and its associated ARC for wavelength calibration
as well as the regions defining the sky along the slit.
e.g. sciTargetd = {'AM2306-721_a': {'arc': 'gsS20070623S0071', 'sky': '520:720'},
'AM2306-72_b': {'arc': 'gsS20070623S0081', 'sky': '670:760,920:1020'}}
Note that there could be more than one target defined this way.
stdTarget : dict
Dictionary with the associations of standard star targets and its associated ARC for wavelength calibration
as well as the regions defining the sky along the slit.
e.g. stdTarget = {'LTT1788': {'arc': 'S20180711S0281', 'sky': '170:380,920:1080'}}
qd_full : dictionary
Query Dictionary of essential parameter=value pairs for Full RoI. Meant for science object.
qd_censp : dictionary
Query Dictionary of essential parameter=value pairs for CenSp RoI. Meant for standard star.
biasFlags : dict
Dictionary for the keyword flags of gmos.gbias() function
flatFlags : dict
Dictionary for the keyword flags of gmos.gsflat() function
sciFlags : dict
Dictionary for the keyword flags of gmos.gsreduce() function
Based on these flags a set of arcFlags and stdFlags dictionaries will be created
for basic processing.
waveFlags : dict
Dictionary for the keyword flags of gmos.gswavelength() function
sciCombFlags : dict
Dictionary for the keyword flags of gemtools.gemcombine() function
Based on these flags a set of stdCombFlags dictionary will be created for the standard advanced processing.
transFlags : dict
Dictionary for the keyword flags of gmos.gstransform() function.
xxx
skyFlags : dict
Dictionary for the keyword flags of gmos.gsskysub() function
extrFlags : dict
Dictionary for the keywords flags of gmos.gsextract() function
calibFlags : dict
XXX
skip_wavecal : bool
Whether to skip interactive wavelength calibration.
Useful when this is already done.
Returns
-------
"""
print ("### Begin Processing GMOS/Longslit Images ###")
print ("###")
print ("=== Creating MasterCals ===")
# From the work_directory:
# Create the query dictionary of essential parameter=value pairs for Full and CenSp RoIs
qd = {'Full': qd_full, 'CenSp': qd_censp}
print (" --Creating Bias MasterCal--")
# Set the task parameters.
gemtools.gemextn.unlearn() # Disarm a bug in gbias
gmos.gbias.unlearn()
regions = ['Full', 'CenSp']
for r in regions:
# The following SQL generates the list of full-frame files to process.
SQL = fs.createQuery('bias', qd[r])
biasFiles = fs.fileListQuery(dbFile, SQL, qd[r])
# The str.join() funciton is needed to transform a python list into a
# comma-separated string of file names that IRAF can understand.
if len(biasFiles) > 1:
# NT comment: sometimes if there are too many files, gmos.gbias() raises an error.
# import pdb; pdb.set_trace()
gmos.gbias(','.join(str(x) for x in biasFiles), 'MCbias' + r,
**biasFlags)
# Clean up
year_obs = qd_full['DateObs'].split('-')[0]
if clean_files:
iraf.imdel("gS{}*.fits".format(year_obs))
ask_user("MC Bias done. Would you like to continue to proceed with GCAL Spectral Master Flats? (y/n): ",['y','yes'])
print (" -- Creating GCAL Spectral Flat-Field MasterCals --")
# Set the task parameters.
qd['Full'].update({'DateObs': '*'})
qd['CenSp'].update({'DateObs': '*'})
gmos.gireduce.unlearn()
gmos.gsflat.unlearn()
# Normalize the spectral flats per CCD.
# The response fitting should be done interactively.
if flatFlags['fl_inter'] != 'yes':
print("The response fitting should be done interactively. Please set flatFlags['fl_inter'] = 'yes'.")
ask_user("Do you still want to proceed despite this important warning? (y/n): ", ['yes','y'])
for r in regions:
qr = qd[r]
flatFiles = fs.fileListQuery(dbFile, fs.createQuery('gcalFlat', qr), qr)
if len(flatFiles) > 0:
gmos.gsflat(','.join(str(x) for x in flatFiles), 'MCflat' + r,
bias='MCbias' + r, **flatFlags)
if clean_files:
iraf.imdel('gS{}*.fits,gsS{}*.fits'.format(year_obs, year_obs))
ask_user("GCAL Spectral Flat-Field MasterCals done. Would you like to continue to proceed with Basic Processing? (y/n): ",['y','yes'])
print ("=== Processing Science Files ===")
print (" -- Performing Basic Processing --")
# Set task parameters.
gmos.gsreduce.unlearn()
sciFlags = sciFlags # redundant but put here because NT likes it
arcFlags = copy.deepcopy(sciFlags)
arcFlags.update({'fl_flat': 'no', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
stdFlags = copy.deepcopy(sciFlags)
stdFlags.update({'fl_fixpix': 'yes', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
# Perform basic reductions on all exposures for science targets.
print (" - Arc exposures -")
for r in regions:
qr = qd[r]
arcFiles = fs.fileListQuery(dbFile, fs.createQuery('arc', qr), qr)
if len(arcFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in arcFiles),
bias='MCbias' + r, **arcFlags)
print (" - Std star exposures -")
r = 'CenSp'
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qd[r]), qd[r])
if len(stdFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in stdFiles), bias='MCbias' + r,
flatim='MCflat' + r, **stdFlags)
print (" - Science exposures -")
r = 'Full'
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qd[r]), qd[r])
if len(sciFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in sciFiles), bias='MCbias' + r,
flatim='MCflat' + r, **sciFlags)
# Clean up
if clean_files:
iraf.imdel('gS{}*.fits'.format(year_obs))
ask_user("Basic processing done. Would you like to continue to determine wavelength calibration? (y/n): ",['y','yes'])
print (" -- Determine wavelength calibration --")
# Set task parameters
gmos.gswavelength.unlearn()
# The fit to the dispersion relation should be performed interactively.
# Here we will use a previously determined result.
if waveFlags['fl_inter'] != 'yes':
print("The fit to the dispersion relation should be performed interactively. Please set waveFlags['fl_inter'] = 'yes'.")
ask_user("Do you still want to proceed despite this important warning? (y/n): ", ['yes','y'])
# Need to select specific wavecals to match science exposures.
# NT: we do this now from the sciTargets + stdTarget input dictionaries
# e.g.
'''
sciTargets = {
'AM2306-721_a': {'arc': 'gsS20070623S0071', 'sky': '520:720'},
'AM2306-72_b': {'arc': 'gsS20070623S0081', 'sky': '670:760,920:1020'},
'AM2306-721_c': {'arc': 'gsS20070623S0091', 'sky': '170:380,920:1080'}
}
'''
#prefix = 'gsS20070623S0'
#for arc in ['071', '081', '091', '109']:
# gmos.gswavelength(prefix + arc, **waveFlags)
prefix = 'gs'
arc_files = []
for key in sciTargets.keys():
arc_files += [sciTargets[key]['arc']]
for key in stdTarget.keys():
arc_files += [stdTarget[key]['arc']]
# import pdb; pdb.set_trace()
if skip_wavecal is not True:
for arc in arc_files:
gmos.gswavelength(prefix + arc, **waveFlags)
### End of basic processing. Continue with advanced processing.
ask_user("Wavelength solution done. Would you like to continue with advanced processing? (y/n): ",['y','yes'])
print (" -- Performing Advanced Processing --")
print (" -- Combine exposures, apply dispersion, subtract sky --")
# Set task parameters.
gemtools.gemcombine.unlearn()
sciCombFlags = sciCombFlags
stdCombFlags = copy.deepcopy(sciCombFlags)
stdCombFlags.update({'fl_vardq': 'no', 'fl_dqprop': 'no'})
gmos.gstransform.unlearn()
# apply gtransform to standard
# Process the Standard Star
prefix = "gs"
qs = qd['CenSp']
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qs), qs)
std_name = stdTarget.keys()[0]
if len(stdFiles) == 0:
ValueError("No standard star associated. Please check parameters of search (e.g. RoI=CentSp)")
# import pdb; pdb.set_trace()
if len(stdFiles) > 1:
# import pdb; pdb.set_trace()
gemtools.gemcombine(','.join(prefix + str(x) for x in stdFiles),
std_name, **stdCombFlags)
else:
os.system("cp {}.fits {}.fits".format(prefix + stdFiles[0], std_name))
gmos.gstransform(std_name, wavtraname=prefix + stdTarget[std_name]['arc'], **transFlags)
# The sky regions should be selected with care, using e.g. prows/pcols:
# pcols ("tAM2306b.fits[SCI]", 1100, 2040, wy1=40, wy2=320)
print("The sky regions should be selected with care, using e.g. with prows/pcols (see tutorial).")
'''
answer = raw_input("Please provide the long_sample string to apply to gmos.gsskysub() for the standard star."
"e.g. '20:70,190:230' (say 'no' for using the example as the default values): ")
if answer in ['n', 'no']:
print("Using default long_sample set by stdTarget values {}.".format(stdTarget[std_name]['sky']))
long_sample_std = stdTarget[std_name]['sky']
else:
long_sample_std = answer
'''
long_sample_std = stdTarget[std_name]['sky']
ask_user("Before proceeding it is important that you have set a good sky region for the standard.\n"
"Thus far you have selected: {}\n Would you like to proceed with the current one? (y/n): ".format(long_sample_std), ['yes','y'])
# apply sky substraction
skyFlags = skyFlags
gmos.gsskysub.unlearn()
gmos.gsskysub('t{}'.format(std_name), long_sample=long_sample_std)
# NT: make sure the process works ok until here before proceeding further. i.e. setting the sky region manually and correctly.
# NT: seems to be working.
print (" -- Extract Std spectrum --")
# Extract the std spectrum using a large aperture.
# It's important to trace the spectra interactively.
gmos.gsextract.unlearn()
gmos.gsextract("st" + std_name, **extrFlags)
print (" -- Derive the Flux calibration --")
gmos.gsstandard.unlearn()
sensFlags = {
'fl_inter': 'no', 'starname': 'XXX', 'caldir': 'onedstds$ctionewcal/',
'observatory': 'Gemini-South', 'extinction': 'onedstds$ctioextinct.dat',
'function': 'chebyshev', 'order': 9, 'verbose': 'no', 'logfile': 'gsstdLog.txt'
}
sensFlags['starname'] = stdTarget[std_name]['iraf_name'] # replace corresponding starname
gmos.gsstandard('est'+std_name, sfile='std.txt', sfunction='sens', **sensFlags)
ask_user("Sensitivity function from standard star done. Would you like to continue with reduction of science"
" exposures? (y/n): ",['yes','y'])
# Process the science targets.
# Use a dictionary to associate science targets with Arcs and sky regions.
prefix = 'gs'
extract_individuals = True
for targ, p in sciTargets.iteritems():
qs = qd['Full']
qs['Object'] = p['name']
# Fix up the target name for the output file
sciOut = p['name_out']
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qs), qs)
all_files = ','.join(prefix + str(x) for x in sciFiles)
gemtools.gemcombine(all_files, sciOut, **sciCombFlags)
gmos.gstransform(sciOut, wavtraname=prefix + p['arc'], **transFlags)
ask_user("It is important to select a good sky region for substraction. Thus far you have selected {}"
" based on the sciTargets input dictionary. Would you like to continue? (y/n): ".format(p['sky']),['y','yes'])
gmos.gsskysub('t' + sciOut, long_sample=p['sky'], **skyFlags)
if extract_individuals:
import pdb; pdb.set_trace()
for fname in sciFiles:
gmos.gstransform(prefix + fname, wavtraname=prefix + p['arc'], **transFlags)
gmos.gsskysub('t' + prefix + fname, long_sample=p['sky'], **skyFlags)
gmos.gscalibrate.unlearn()
gmos.gscalibrate('st'+prefix+fname, **calibFlags)
# Clean up
if clean_files:
iraf.imdel("gsS{}*.fits".format(year_obs))
ask_user("Sky substraction done. Would you like to continue to apply sensitivity function? (y/n): ",['y'])
## Apply the sensitivity function.
gmos.gscalibrate.unlearn()
gmos.gscalibrate('st'+sciOut+'*', **calibFlags)
calibFlags.update({'fl_vardq': 'no'})
gmos.gscalibrate('est'+std_name, **calibFlags)
print (" -- Extract Target Spectra --")
method = 'gsextract'
if method == 'gsextract':
gmos.gsextract.unlearn()
# import pdb;pdb.set_trace()
gmos.gsextract("cst" + sciOut, **extrFlags)
elif method == 'sarith':
# not implemented yet
onedspec.nsum = 4
onedspec.sarith('cst{}.fits[SCI]'.format(sciOut), 'copy', '', 'ecst{}.ms'.format(sciOut),
apertures='222-346x4')
print ("=== Finished Calibration Processing ===")
'fl_gsappwave':'yes','fl_oversize':'no',
'fl_vardq':'yes','fl_fulldq':'yes','rawpath':'./raw',
'fl_inter':'no','logfile':'gsreduceLog.txt','verbose':'no'
}
arcFlags = copy.deepcopy(sciFlags)
arcFlags.update({'fl_flat':'no','fl_vardq':'no','fl_fulldq':'no'})
stdFlags = copy.deepcopy(sciFlags)
stdFlags.update({'fl_fixpix':'yes','fl_vardq':'no','fl_fulldq':'no'})
# Perform basic reductions on all exposures for science targets.
print (" - Arc exposures -")
for r in regions:
qr = qd[r]
arcFiles = fs.fileListQuery(dbFile, fs.createQuery('arc', qr), qr)
if len(arcFiles) > 0:
gmos.gsreduce (','.join(str(x) for x in arcFiles),
bias='MCbias'+r, **arcFlags)
print (" - Std star exposures -")
r = 'CenSp'
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qd[r]), qd[r])
if len(stdFiles) > 0:
gmos.gsreduce (','.join(str(x) for x in stdFiles), bias='MCbias'+r,
flatim='MCflat'+r, **stdFlags)
print (" - Science exposures -")
r = 'Full'
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qd[r]), qd[r])
if len(sciFiles) > 0:
gmos.gsreduce (','.join(str(x) for x in sciFiles), bias='MCbias'+r,
flatim='MCflat'+r, **sciFlags)
def gmos_ls_proc():
'''
GMOS Data Reduction Cookbook companion script to the chapter:
"Reduction of Longslit Spectra with PyRAF"
PyRAF script to:
Process GMOS spectra for AM2306-721, in program GS-2007A-Q-76.
The names for the relevant header keywords and their expected values are
described in the DRC chapter entitled "Supplementary Material"
Perform the following starting in the parent work directory:
cd /path/to/work_directory
Place the fileSelect.py module in your work directory. Now execute this
script from the unix prompt:
python gmos_ls_proc.py
'''
print("### Begin Processing GMOS/Longslit Images ###")
print("###")
print("=== Creating MasterCals ===")
# This whole example depends upon first having built an sqlite3 database of metadata:
# cd ./raw
# python obslog.py obsLog.sqlite3
dbFile = './raw/obsLog.sqlite3'
# From the work_directory:
# Create the query dictionary of essential parameter=value pairs.
# Select bias exposures within ~2 months of the target observations:
qd = {
'Full': {
'use_me': 1,
'Instrument': 'GMOS-S',
'CcdBin': '2 4',
'RoI': 'Full',
'Disperser': 'B600+_%',
'CentWave': 485.0,
'AperMask': '1.0arcsec',
'Object': 'AM2306-72%',
'DateObs': '2007-06-05:2007-07-07'
}
}
# Make another copy for the CenterSpec RoI:
qd['CenSp'] = copy.deepcopy(qd['Full'])
qd['CenSp'].update({'RoI': 'CentSp', 'Object': 'LTT9239'})
print(" --Creating Bias MasterCal--")
# Set the task parameters.
gemtools.gemextn.unlearn() # Disarm a bug in gbias
gmos.gbias.unlearn()
biasFlags = {
'logfile': 'biasLog.txt',
'rawpath': './raw/',
'fl_vardq': 'yes',
'verbose': 'no'
}
regions = ['Full', 'CenSp']
for r in regions:
# The following SQL generates the list of full-frame files to process.
SQL = fs.createQuery('bias', qd[r])
biasFiles = fs.fileListQuery(dbFile, SQL, qd[r])
# The str.join() funciton is needed to transform a python list into a
# comma-separated string of file names that IRAF can understand.
if len(biasFiles) > 1:
gmos.gbias(','.join(str(x) for x in biasFiles), 'MCbias' + r,
**biasFlags)
# Clean up
iraf.imdel("gS2007*.fits")
print(" -- Creating GCAL Spectral Flat-Field MasterCals --")
# Set the task parameters.
qd['Full'].update({'DateObs': '*'})
qd['CenSp'].update({'DateObs': '*'})
gmos.gireduce.unlearn()
gmos.gsflat.unlearn()
# Normalize the spectral flats per CCD.
# The response fitting should be done interactively.
flatFlags = {
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_dark': 'no',
'fl_fixpix': 'no',
'fl_oversize': 'no',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'rawpath': './raw',
'fl_inter': 'no',
'fl_detec': 'yes',
'function': 'spline3',
'order': '13,11,28',
'logfile': 'gsflatLog.txt',
'verbose': 'no'
}
for r in regions:
qr = qd[r]
flatFiles = fs.fileListQuery(dbFile, fs.createQuery('gcalFlat', qr),
qr)
if len(flatFiles) > 0:
gmos.gsflat(','.join(str(x) for x in flatFiles),
'MCflat' + r,
bias='MCbias' + r,
**flatFlags)
iraf.imdel('gS2007*.fits,gsS2007*.fits')
print("=== Processing Science Files ===")
print(" -- Performing Basic Processing --")
# Set task parameters.
gmos.gsreduce.unlearn()
sciFlags = {
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_gscrrej': 'no',
'fl_dark': 'no',
'fl_flat': 'yes',
'fl_gmosaic': 'yes',
'fl_fixpix': 'no',
'fl_gsappwave': 'yes',
'fl_oversize': 'no',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'rawpath': './raw',
'fl_inter': 'no',
'logfile': 'gsreduceLog.txt',
'verbose': 'no'
}
arcFlags = copy.deepcopy(sciFlags)
arcFlags.update({'fl_flat': 'no', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
stdFlags = copy.deepcopy(sciFlags)
stdFlags.update({'fl_fixpix': 'yes', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
# Perform basic reductions on all exposures for science targets.
print(" - Arc exposures -")
for r in regions:
qr = qd[r]
arcFiles = fs.fileListQuery(dbFile, fs.createQuery('arc', qr), qr)
if len(arcFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in arcFiles),
bias='MCbias' + r,
**arcFlags)
print(" - Std star exposures -")
r = 'CenSp'
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qd[r]), qd[r])
if len(stdFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in stdFiles),
bias='MCbias' + r,
flatim='MCflat' + r,
**stdFlags)
print(" - Science exposures -")
r = 'Full'
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qd[r]),
qd[r])
if len(sciFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in sciFiles),
bias='MCbias' + r,
flatim='MCflat' + r,
**sciFlags)
# Clean up
iraf.imdel('gS2007*.fits')
print(" -- Determine wavelength calibration --")
# Set task parameters
gmos.gswavelength.unlearn()
waveFlags = {
'coordlist': 'gmos$data/CuAr_GMOS.dat',
'fwidth': 6,
'nsum': 50,
'function': 'chebyshev',
'order': 5,
'fl_inter': 'no',
'logfile': 'gswaveLog.txt',
'verbose': 'no'
}
# The fit to the dispersion relation should be performed interactively.
# Here we will use a previously determined result.
# Need to select specific wavecals to match science exposures.
prefix = 'gsS20070623S0'
for arc in ['071', '081', '091', '109']:
gmos.gswavelength(prefix + arc, **waveFlags)
### End of basic processing. Continue with advanced processing.
print(" -- Performing Advanced Processing --")
print(" -- Combine exposures, apply dispersion, subtract sky --")
# Set task parameters.
gemtools.gemcombine.unlearn()
sciCombFlags = {
'combine': 'average',
'reject': 'ccdclip',
'fl_vardq': 'yes',
'fl_dqprop': 'yes',
'logfile': 'gemcombineLog.txt.txt',
'verbose': 'no'
}
stdCombFlags = copy.deepcopy(sciCombFlags)
stdCombFlags.update({'fl_vardq': 'no', 'fl_dqprop': 'no'})
gmos.gstransform.unlearn()
transFlags = {
'fl_vardq': 'yes',
'interptype': 'linear',
'fl_flux': 'yes',
'logfile': 'gstransLog.txt'
}
# The sky regions should be selected with care, using e.g. prows/pcols:
# pcols ("tAM2306b.fits[SCI]", 1100, 2040, wy1=40, wy2=320)
gmos.gsskysub.unlearn()
skyFlags = {
'fl_oversize': 'no',
'fl_vardq': 'yes',
'logfile': 'gsskysubLog.txt'
}
# Process the Standard Star
prefix = "gs"
qs = qd['CenSp']
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qs), qs)
gemtools.gemcombine(','.join(prefix + str(x) for x in stdFiles), 'LTT9239',
**stdCombFlags)
gmos.gstransform('LTT9239', wavtraname='gsS20070623S0109', **transFlags)
gmos.gsskysub('tLTT9239', long_sample='20:70,190:230')
print(" -- Extract Std spectrum --")
# Extract the std spectruma using a large aperture.
# It's important to trace the spectra interactively.
gmos.gsextract.unlearn()
extrFlags = {
'apwidth': 3.,
'fl_inter': 'no',
'find': 'yes',
'trace': 'yes',
'tfunction': 'chebyshev',
'torder': '6',
'tnsum': 20,
'background': 'fit',
'bfunction': 'chebyshev',
'border': 2,
'fl_vardq': 'no',
'logfile': 'gsextrLog.txt'
}
gmos.gsextract("stLTT9239", **extrFlags)
print(" -- Derive the Flux calibration --")
gmos.gsstandard.unlearn()
sensFlags = {
'fl_inter': 'yes',
'starname': 'l9239',
'caldir': 'onedstds$ctionewcal/',
'observatory': 'Gemini-South',
'extinction': 'onedstds$ctioextinct.dat',
'function': 'chebyshev',
'order': 9,
'verbose': 'no',
'logfile': 'gsstdLog.txt'
}
gmos.gsstandard('estLTT9239',
sfile='std.txt',
sfunction='sens',
**sensFlags)
# Process the science targets.
# Use a dictionary to associate science targets with Arcs and sky regions.
sciTargets = {
'AM2306-721_a': {
'arc': 'gsS20070623S0071',
'sky': '520:720'
},
'AM2306-72_b': {
'arc': 'gsS20070623S0081',
'sky': '670:760,920:1020'
},
'AM2306-721_c': {
'arc': 'gsS20070623S0091',
'sky': '170:380,920:1080'
}
}
for targ, p in sciTargets.iteritems():
qs = qd['Full']
qs['Object'] = targ
# Fix up the target name for the output file
sciOut = targ.split('-')[0] + targ[-1]
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qs), qs)
gemtools.gemcombine(','.join(prefix + str(x) for x in sciFiles),
sciOut, **sciCombFlags)
gmos.gstransform(sciOut, wavtraname=p['arc'], **transFlags)
gmos.gsskysub('t' + sciOut, long_sample=p['sky'], **skyFlags)
# Clean up
iraf.imdel("gsS2007*.fits")
## Apply the sensitivity function.
gmos.gscalibrate.unlearn()
calibFlags = {
'extinction': 'onedstds$ctioextinct.dat',
'fl_ext': 'yes',
'fl_scale': 'no',
'sfunction': 'sens',
'fl_vardq': 'yes',
'logfile': 'gscalibrateLog.txt'
}
gmos.gscalibrate('stAM2306*', **calibFlags)
calibFlags.update({'fl_vardq': 'no'})
gmos.gscalibrate('estLTT9239', **calibFlags)
print(" -- Extract Target Spectra --")
onedspec.nsum = 4
onedspec.sarith('cstAM2306b.fits[SCI]',
'copy',
'',
'ecstAM2306b.ms',
apertures='222-346x4')
print("=== Finished Calibration Processing ===")