def calibrate_standard_images(qd,
dbFile,
std_name,
biasfilename='MCbias',
overwrite=True):
print("=== Processing Science Images ===")
prefix = 'rg'
cur_dir = os.getcwd()
iraf.chdir(data_dir)
# Set task parameters.
# Employ the imaging Static BPM for this set of detectors.
gmos.gireduce.unlearn()
sciFlags = {
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_dark': 'no',
'fl_flat': 'yes',
'logfile': 'gireduceLog.txt',
'rawpath': '',
'fl_vardq': 'yes',
#####'bpm':'bpm_gmos-s_EEV_v1_2x2_img_MEF.fits',
'verbose': 'no'
}
gemtools.gemextn.unlearn() # disarms a bug in gmosaic
gmos.gmosaic.unlearn()
mosaicFlags = {
'fl_paste': 'no',
'fl_fixpix': 'no',
'fl_clean': 'yes',
'geointer': 'nearest',
'logfile': 'gmosaicLog.txt',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'verbose': 'no'
}
# Reduce the science images, then mosaic the extensions in a loop
filters = ['Ha', 'HaC', 'SII', 'r', 'i']
for f in filters:
print(" Processing science images for: %s" % (f))
qd['Filter2'] = f + '_G%'
flatFile = 'MCflat_' + f
sql_query = '''SELECT file FROM obslog WHERE Object='{}' AND Filter2 LIKE '{}%' '''.format(
std_name, f)
sciFiles = run_query(sql_query, dbFile)
sciFiles = [x[0] for x in sciFiles]
if len(sciFiles) > 0:
gmos.gireduce(','.join(str(x) for x in sciFiles),
bias=biasfilename,
flat1=flatFile,
**sciFlags)
for file in sciFiles:
gmos.gmosaic(prefix + file, **mosaicFlags)
iraf.chdir(cur_dir)
def gmos_img_proc2(
dbFile="./raw/obsLog.sqlite3",
qd={
'use_me': 1,
'Instrument': 'GMOS-S',
'CcdBin': '2 2',
'RoI': 'Full',
'Object': 'M8-%',
'DateObs': '2006-09-01:2006-10-30'
},
bias_dateobs="2006-09-01:2006-10-30",
biasFlags={
'logfile': 'biasLog.txt',
'rawpath': './raw/',
'fl_vardq': 'yes',
'verbose': 'yes'
},
flat_dateobs='2006-09-10:2006-10-10',
flatFlags={
'fl_scale': 'yes',
'sctype': 'mean',
'fl_vardq': 'yes',
'rawpath': './raw/',
'logfile': 'giflatLog.txt',
'verbose': 'yes'
},
filters=['Ha', 'HaC', 'SII', 'r', 'i'],
sciFlags={
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_dark': 'no',
'fl_flat': 'yes',
'logfile': 'gireduceLog.txt',
'rawpath': './raw/',
'fl_vardq': 'yes',
'bpm': bpm_gmos,
'verbose': 'yes'
},
mosaicFlags={
'fl_paste': 'no',
'fl_fixpix': 'no',
'fl_clean': 'yes',
'geointer': 'nearest',
'logfile': 'gmosaicLog.txt',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'verbose': 'yes'
},
coaddFlags={
'fwhm': 3,
'datamax': 6.e4,
'geointer': 'nearest',
'logfile': 'imcoaddLog.txt'
},
targets=['M8-1', 'M8-2', 'M8-3'],
clean_files=False):
"""
Parameters
----------
dbFile : str
Filename containing the SQL sqlite3 database created by obslog.py
It must be placed in the ./raw/ directory
Default is `./raw/obsLog.sqlite3`
qd : dictionary
Query Dictionary of essential parameter=value pairs.
Select bias exposures within ~2 months of the target observations
e.g.
qd= {'use_me': 1,
'Instrument': 'GMOS-S', 'CcdBin': '2 2', 'RoI': 'Full', 'Object': 'M8-%',
'DateObs': '2006-09-01:2006-10-30'
}
bias_dateobs : str
String representing the bias search Obsdate
e.g. bias_dateobs = `2006-09-01:2006-10-30`
biasFlags : dict
Dictionary for the keyword flags of gmos.gbias() function
flat_dateobs : str
String representing the flat search Obsdate
e.g. flat_dateobs = `2006-09-10:2006-10-10`
flatFlags : dict
Dictionary for the keyword flags of gmos.giflat() function
e.g. flatFlags = {'fl_scale': 'yes', 'sctype': 'mean', 'fl_vardq': 'yes','rawpath': './raw/',
'logfile': 'giflatLog.txt', 'verbose': 'yes'}
filters : list
List of filter names to perform reduction
e.g. filters=['Ha', 'HaC', 'SII', 'r', 'i']
sciFlags : dict
Dictionary for the keyword flags of gmos.gireduce() function
mosaicFlags : dict
Dictionary for the keyword flags of gmos.gimosaic() function
coaddFlags : dict
Dictionary for the keyword flags of gemtools.imcoadd() function
targets : list
List of names of target observations for the co-addition
e.g. targets = ['M8-1', 'M8-2', 'M8-3']
clean_files : bool
Whether to clean intermediate files from reduction process
Returns
-------
Reduce GMOS imaging based on tutorial example.
"""
print("### Begin Processing GMOS/MOS Images ###")
print("###")
print("=== Creating MasterCals ===")
# From the work_directory:
# Create the query dictionary of essential parameter=value pairs.
# Select bias exposures within ~2 months of the target observations:
print(" --Creating Bias MasterCal--")
qd.update({'DateObs': bias_dateobs})
# Set the task parameters.
gmos.gbias.unlearn()
# The following SQL generates the list of files to process.
SQL = fs.createQuery('bias', qd)
biasFiles = fs.fileListQuery(dbFile, SQL, qd)
# The str.join() function is needed to transform a python list into a string
# filelist that IRAF can understand.
if len(biasFiles) > 1:
files_all = ','.join(str(x) for x in biasFiles)
# import pdb; pdb.set_trace()
gmos.gbias(files_all, 'MCbias.fits', **biasFlags)
# Clean up
year_obs = qd['DateObs'].split('-')[0]
if clean_files:
iraf.imdel('gS{}*.fits'.format(year_obs))
ask_user(
"MC Bias done. Would you like to continue to proceed with Master Flats? (y/n): ",
['y', 'yes'])
print(" --Creating Twilight Imaging Flat-Field MasterCal--")
# Select flats obtained contemporaneously with the observations.
qd.update({'DateObs': flat_dateobs})
# Set the task parameters.
gmos.giflat.unlearn()
#filters = ['Ha', 'HaC', 'SII', 'r', 'i']
for f in filters:
print " Building twilight flat MasterCal for filter: %s" % (f)
# Select filter name using a substring of the official designation.
qd['Filter2'] = f + '_G%'
mcName = 'MCflat_%s.fits' % (f)
flatFiles = fs.fileListQuery(dbFile, fs.createQuery('twiFlat', qd), qd)
if len(flatFiles) > 0:
files_all = ','.join(str(x) for x in flatFiles)
# import pdb; pdb.set_trace()
gmos.giflat(files_all, mcName, bias='MCbias', **flatFlags)
if clean_files:
iraf.imdel('gS{}*.fits,rgS{}*.fits'.format(year_obs, year_obs))
ask_user(
"MC Flats done. Would you like to continue to proceed with processing Science Images? (y/n): ",
['yes', 'y'])
print("=== Processing Science Images ===")
# Remove restriction on date range
qd['DateObs'] = '*'
prefix = 'rg'
gmos.gireduce.unlearn()
gemtools.gemextn.unlearn() # disarms a bug in gmosaic
gmos.gmosaic.unlearn()
# Reduce the science images, then mosaic the extensions in a loop
for f in filters:
print " Processing science images for filter: %s" % (f)
qd['Filter2'] = f + '_G%'
flatFile = 'MCflat_' + f + '.fits'
SQL = fs.createQuery('sciImg', qd)
sciFiles = fs.fileListQuery(dbFile, SQL, qd)
if len(sciFiles) > 0:
# Make sure BPM table is in sciFlags for employing the imaging Static BPM for this set of detectors.
# import pdb; pdb.set_trace()
all_files = ','.join(str(x) for x in sciFiles)
gmos.gireduce(all_files, bias='MCbias', flat1=flatFile, **sciFlags)
for file in sciFiles:
gmos.gmosaic(prefix + file, **mosaicFlags)
else:
print("No Science images found for filter {}. Check database.".
format(f))
import pdb
pdb.set_trace()
if clean_files:
iraf.imdelete('gS{}*.fits,rgS{}*.fits'.format(year_obs, year_obs))
ask_user(
"Science Images done. Would you like to continue to proceed with image co-addition? (y/n): ",
['y', 'yes'])
## Co-add the images, per position and filter.
print(" -- Begin image co-addition --")
# Use primarily the default task parameters.
gemtools.imcoadd.unlearn()
prefix = 'mrg'
for f in filters:
print " - Co-addding science images in filter: %s" % (f)
qd['Filter2'] = f + '_G%'
for t in targets:
qd['Object'] = t + '%'
print " - Co-addding science images for position: %s" % (t)
outImage = t + '_' + f + '.fits'
coAddFiles = fs.fileListQuery(dbFile, fs.createQuery('sciImg', qd),
qd)
all_files = ','.join(prefix + str(x) for x in coAddFiles)
if all_files == '':
print(
'No files available for co-addition. Check that the target names are written correctly.'
)
import pdb
pdb.set_trace()
gemtools.imcoadd(all_files, outimage=outImage, **coaddFlags)
ask_user(
"Co-addition done. Would you like to clean the latest intermediate reduction files? (y/n): ",
['y', 'yes'])
if clean_files:
iraf.delete("*_trn*,*_pos,*_cen")
iraf.imdelete("*badpix.pl,*_med.fits,*_mag.fits")
# iraf.imdelete ("mrgS*.fits")
print("=== Finished Calibration Processing ===")
'fl_clean': 'yes',
'geointer': 'nearest',
'logfile': 'gmosaicLog.txt',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'verbose': 'no'
}
# Reduce the science images, then mosaic the extensions in a loop
for f in filters:
print " Processing science images for: %s" % (f)
qd['Filter2'] = f + '_G%'
flatFile = 'MCflat_' + f + '.fits'
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciImg', qd), qd)
if len(sciFiles) > 0:
gmos.gireduce(','.join(str(x) for x in sciFiles),
bias='MCbias',
flat1=flatFile,
**sciFlags)
for file in sciFiles:
gmos.gmosaic(prefix + file, **mosaicFlags)
iraf.imdelete('gS2006*.fits,rgS2006*.fits')
## Co-add the images, per position and filter.
print(" -- Begin image co-addition --")
# Use primarily the default task parameters.
gemtools.imcoadd.unlearn()
coaddFlags = {
'fwhm': 3,
'datamax': 6.e4,
'geointer': 'nearest',
else:
T = Table.read(geminiArchiveFileSummaryFile_calibAssoc,
format='ascii.basic')
# reduce the science images
for j, fname in enumerate(T['filename']):
pl.close('all')
f = fname[0:-4]
inFileName = os.path.join(rawDir, f)
outFileName = os.path.join(reducedDir, f.replace('.fits', '_red.fits'))
mosaicFileName = os.path.join(mosaicDir,
'm' + f.replace('.fits', '_red.fits'))
if ((not os.path.isfile(mosaicFileName)) or (1)):
if 1:
gmos.gireduce(inimages=inFileName,
outimages=outFileName,
bias=T['masterbiasFitsFile'][j],
flat1=T['masterflatFitsFile'][j])
# clean up
homeDir = os.environ['HOME']
os.remove(os.path.join(homeDir, 'g' + f))
os.remove(os.path.join(homeDir, 'gmos.log'))
gred.make_gemini_frame_mosaic(outFileName,
overwrite=0,
outputDir=mosaicDir,
makePDF=makePDF,
outputDirPDF=thumbnail_dir)
# clean up
os.remove(outFileName)
else:
def gmos_img_proc2(dbFile="./raw/obsLog.sqlite3", qd={'use_me': 1,'Instrument': 'GMOS-S', 'CcdBin': '2 2', 'RoI': 'Full', 'Object': 'M8-%', 'DateObs': '2006-09-01:2006-10-30'},
bias_dateobs="2006-09-01:2006-10-30",
biasFlags={'logfile': 'biasLog.txt', 'rawpath': './raw/', 'fl_vardq': 'yes', 'verbose': 'yes'},
flat_dateobs='2006-09-10:2006-10-10',
flatFlags = {'fl_scale': 'yes', 'sctype': 'mean', 'fl_vardq': 'yes','rawpath': './raw/', 'logfile': 'giflatLog.txt', 'verbose': 'yes'},
filters = ['Ha', 'HaC', 'SII', 'r', 'i'],
sciFlags={'fl_over': 'yes', 'fl_trim': 'yes', 'fl_bias':'yes', 'fl_dark': 'no','fl_flat': 'yes', 'logfile':'gireduceLog.txt', 'rawpath': './raw/','fl_vardq': 'yes','bpm':bpm_gmos, 'verbose': 'yes'},
mosaicFlags = {'fl_paste': 'no', 'fl_fixpix': 'no', 'fl_clean': 'yes', 'geointer': 'nearest', 'logfile': 'gmosaicLog.txt', 'fl_vardq': 'yes', 'fl_fulldq': 'yes', 'verbose': 'yes'},
coaddFlags = {'fwhm': 3, 'datamax': 6.e4, 'geointer': 'nearest', 'logfile': 'imcoaddLog.txt'},
targets = ['M8-1', 'M8-2', 'M8-3'],
clean_files = False
):
"""
Parameters
----------
dbFile : str
Filename containing the SQL sqlite3 database created by obslog.py
It must be placed in the ./raw/ directory
Default is `./raw/obsLog.sqlite3`
qd : dictionary
Query Dictionary of essential parameter=value pairs.
Select bias exposures within ~2 months of the target observations
e.g.
qd= {'use_me': 1,
'Instrument': 'GMOS-S', 'CcdBin': '2 2', 'RoI': 'Full', 'Object': 'M8-%',
'DateObs': '2006-09-01:2006-10-30'
}
bias_dateobs : str
String representing the bias search Obsdate
e.g. bias_dateobs = `2006-09-01:2006-10-30`
biasFlags : dict
Dictionary for the keyword flags of gmos.gbias() function
flat_dateobs : str
String representing the flat search Obsdate
e.g. flat_dateobs = `2006-09-10:2006-10-10`
flatFlags : dict
Dictionary for the keyword flags of gmos.giflat() function
e.g. flatFlags = {'fl_scale': 'yes', 'sctype': 'mean', 'fl_vardq': 'yes','rawpath': './raw/',
'logfile': 'giflatLog.txt', 'verbose': 'yes'}
filters : list
List of filter names to perform reduction
e.g. filters=['Ha', 'HaC', 'SII', 'r', 'i']
sciFlags : dict
Dictionary for the keyword flags of gmos.gireduce() function
mosaicFlags : dict
Dictionary for the keyword flags of gmos.gimosaic() function
coaddFlags : dict
Dictionary for the keyword flags of gemtools.imcoadd() function
targets : list
List of names of target observations for the co-addition
e.g. targets = ['M8-1', 'M8-2', 'M8-3']
clean_files : bool
Whether to clean intermediate files from reduction process
Returns
-------
Reduce GMOS imaging based on tutorial example.
"""
print ("### Begin Processing GMOS/MOS Images ###")
print ("###")
print ("=== Creating MasterCals ===")
# From the work_directory:
# Create the query dictionary of essential parameter=value pairs.
# Select bias exposures within ~2 months of the target observations:
print (" --Creating Bias MasterCal--")
qd.update({'DateObs': bias_dateobs})
# Set the task parameters.
gmos.gbias.unlearn()
# The following SQL generates the list of files to process.
SQL = fs.createQuery('bias', qd)
biasFiles = fs.fileListQuery(dbFile, SQL, qd)
# The str.join() function is needed to transform a python list into a string
# filelist that IRAF can understand.
if len(biasFiles) > 1:
files_all = ','.join(str(x) for x in biasFiles)
# import pdb; pdb.set_trace()
gmos.gbias(files_all, 'MCbias.fits', **biasFlags)
# Clean up
year_obs = qd['DateObs'].split('-')[0]
if clean_files:
iraf.imdel('gS{}*.fits'.format(year_obs))
ask_user("MC Bias done. Would you like to continue to proceed with Master Flats? (y/n): ",['y','yes'])
print (" --Creating Twilight Imaging Flat-Field MasterCal--")
# Select flats obtained contemporaneously with the observations.
qd.update({'DateObs': flat_dateobs})
# Set the task parameters.
gmos.giflat.unlearn()
#filters = ['Ha', 'HaC', 'SII', 'r', 'i']
for f in filters:
print " Building twilight flat MasterCal for filter: %s" % (f)
# Select filter name using a substring of the official designation.
qd['Filter2'] = f + '_G%'
mcName = 'MCflat_%s.fits' % (f)
flatFiles = fs.fileListQuery(dbFile, fs.createQuery('twiFlat', qd), qd)
if len(flatFiles) > 0:
files_all = ','.join(str(x) for x in flatFiles)
# import pdb; pdb.set_trace()
gmos.giflat(files_all, mcName, bias='MCbias', **flatFlags)
if clean_files:
iraf.imdel('gS{}*.fits,rgS{}*.fits'.format(year_obs, year_obs))
ask_user("MC Flats done. Would you like to continue to proceed with processing Science Images? (y/n): ", ['yes','y'])
print ("=== Processing Science Images ===")
# Remove restriction on date range
qd['DateObs'] = '*'
prefix = 'rg'
gmos.gireduce.unlearn()
gemtools.gemextn.unlearn() # disarms a bug in gmosaic
gmos.gmosaic.unlearn()
# Reduce the science images, then mosaic the extensions in a loop
for f in filters:
print " Processing science images for filter: %s" % (f)
qd['Filter2'] = f + '_G%'
flatFile = 'MCflat_' + f + '.fits'
SQL = fs.createQuery('sciImg', qd)
sciFiles = fs.fileListQuery(dbFile, SQL, qd)
if len(sciFiles) > 0:
# Make sure BPM table is in sciFlags for employing the imaging Static BPM for this set of detectors.
# import pdb; pdb.set_trace()
all_files = ','.join(str(x) for x in sciFiles)
gmos.gireduce(all_files, bias='MCbias', flat1=flatFile, **sciFlags)
for file in sciFiles:
gmos.gmosaic(prefix + file, **mosaicFlags)
else:
print("No Science images found for filter {}. Check database.".format(f))
import pdb; pdb.set_trace()
if clean_files:
iraf.imdelete('gS{}*.fits,rgS{}*.fits'.format(year_obs,year_obs))
ask_user("Science Images done. Would you like to continue to proceed with image co-addition? (y/n): ", ['y','yes'])
## Co-add the images, per position and filter.
print (" -- Begin image co-addition --")
# Use primarily the default task parameters.
gemtools.imcoadd.unlearn()
prefix = 'mrg'
for f in filters:
print " - Co-addding science images in filter: %s" % (f)
qd['Filter2'] = f + '_G%'
for t in targets:
qd['Object'] = t + '%'
print " - Co-addding science images for position: %s" % (t)
outImage = t + '_' + f + '.fits'
coAddFiles = fs.fileListQuery(dbFile, fs.createQuery('sciImg', qd), qd)
all_files = ','.join(prefix + str(x) for x in coAddFiles)
if all_files == '':
print('No files available for co-addition. Check that the target names are written correctly.')
import pdb; pdb.set_trace()
gemtools.imcoadd(all_files, outimage=outImage, **coaddFlags)
ask_user("Co-addition done. Would you like to clean the latest intermediate reduction files? (y/n): ", ['y','yes'])
if clean_files:
iraf.delete("*_trn*,*_pos,*_cen")
iraf.imdelete("*badpix.pl,*_med.fits,*_mag.fits")
# iraf.imdelete ("mrgS*.fits")
print ("=== Finished Calibration Processing ===")
'fl_vardq':'yes','bpm':'bpm_gmos-s_EEV_v1_2x2_img_MEF.fits','verbose':'no'
}
gemtools.gemextn.unlearn() # disarms a bug in gmosaic
gmos.gmosaic.unlearn()
mosaicFlags = {
'fl_paste':'no','fl_fixpix':'no','fl_clean':'yes','geointer':'nearest',
'logfile':'gmosaicLog.txt','fl_vardq':'yes','fl_fulldq':'yes','verbose':'no'
}
# Reduce the science images, then mosaic the extensions in a loop
for f in filters:
print " Processing science images for: %s" % (f)
qd['Filter2'] = f + '_G%'
flatFile = 'MCflat_' + f + '.fits'
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciImg', qd), qd)
if len(sciFiles) > 0:
gmos.gireduce (','.join(str(x) for x in sciFiles), bias='MCbias',
flat1=flatFile, **sciFlags)
for file in sciFiles:
gmos.gmosaic (prefix+file, **mosaicFlags)
iraf.imdelete('gS2006*.fits,rgS2006*.fits')
## Co-add the images, per position and filter.
print (" -- Begin image co-addition --")
# Use primarily the default task parameters.
gemtools.imcoadd.unlearn()
coaddFlags = {
'fwhm':3,'datamax':6.e4,'geointer':'nearest','logfile':'imcoaddLog.txt'
}
targets = ['M8-1', 'M8-2', 'M8-3']
prefix = 'mrg'
def calibrate_science_images(qd,
dbFile,
data_dir,
biasfilename='MCbias',
overwrite=True):
'''
despite the fact that it looks like this can be run from outside the
raw directory, it can't be.
#Bad pixel maps live in /Users/bostroem/anaconda/envs/geminiconda/iraf_extern/gemini/gmos/data
#You can find this directory with pyraf; cd gmos; cd data; pwd
'''
print("=== Processing Science Images ===")
cur_dir = os.getcwd()
iraf.chdir(data_dir)
prefix = 'rg'
# Set task parameters.
# Employ the imaging Static BPM for this set of detectors.
gmos.gireduce.unlearn()
sciFlags = {
'fl_over': 'yes', #Overscan subtraction
'fl_trim': 'yes', #Overscan region trimmed
'fl_bias': 'yes', #Subtract Bias residual
'fl_dark': 'no', #Subtract Dark
'fl_flat': 'yes', #Subtract flat
'logfile': 'gireduceLog.txt',
'rawpath': '',
'fl_vardq': 'yes', #Propagate VAR and DQ extensions
'verbose': 'no'
}
if qd['Instrument'] == 'GMOS-N':
sciFlags['bpm'] = 'gmos$data/gmos-n_bpm_HAM_22_12amp_v1.fits'
else:
sciFlags['bpm'] = 'gmos$data/gmos-s_bpm_HAM_22_12amp_v1.fits'
gemtools.gemextn.unlearn() # disarms a bug in gmosaic
gmos.gmosaic.unlearn()
mosaicFlags = {
'fl_paste': 'no',
'fl_fixpix': 'no',
'fl_clean': 'yes',
'geointer': 'nearest',
'logfile': 'gmosaicLog.txt',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'verbose': 'no'
}
# Reduce the science images, then mosaic the extensions in a loop
filters = ['Ha', 'HaC', 'SII', 'r', 'i']
for f in filters:
print(" Processing science images for: %s" % (f))
qd['Filter2'] = f + '_G%'
flatFile = 'MCflat_' + f
sciFiles = fileSelect.fileListQuery(
dbFile, fileSelect.createQuery('sciImg', qd), qd)
if len(sciFiles) > 0:
gmos.gireduce(','.join(str(x) for x in sciFiles),
bias=biasfilename,
flat1=flatFile,
**sciFlags)
#Combine multi-extension images into one image
for file in sciFiles:
gmos.gmosaic(prefix + file, **mosaicFlags)
iraf.chdir(cur_dir)