def run_ell(file_in, file_tab, file_dat, mag, **ellipse_par):
# prochaine etape : mettre en argument sous une certaine forme les parametres associes epar des taches 1) strict minimum sous forme de dico fixe 2) tout pouvoir changer sous forme de dictionnaire et tester les cles ...
# import iraf packages
iraf.stsdas()
iraf.analysis()
iraf.isophote()
p = Popen("ds9", shell = True)
time.sleep(5)
iraf.display(file_in, 1)
set_iraf_param(ellipse_par)
# iraf.lpar(iraf.geompar)
iraf.ellipse(file_in, file_tab, mag0 = str(mag), verbose = "no")
# iraf.lpar(iraf.isoimap)
iraf.isoimap(file_in, file_tab)
iraf.tproject(file_tab, "temp.fits", "SMA,INTENS,INT_ERR,ELLIP,ELLIP_ERR, PA, PA_ERR, X0, Y0, TFLUX_C, TMAG_E,NPIX_E")
iraf.tprint("temp.fits", pwidth = 200, Stdout = file_dat)
iraf.imdel("temp.fits")
p.kill()
def reduce(self, image, pref="gs", bias="no", gscrrej="no", crspec="no", \
qecorr="no", flat="no", fixpix="yes", vardq="no", biasPath="", \
flatPath="", refimPath="", imPath="", bpmPath="gmos$data/chipgaps.dat", \
logPath="", verbose="yes", frameNo=1):
print "RUNNING GSREDUCE"
# stash task settings
self.stashSettings('gsreduce', outpref=pref, fl_bias=bias, \
bias=biasPath, fl_flat=flat, flatim=flatPath, fl_qecorr=qecorr, \
qe_refim=refimPath, fl_gscrrej=gscrrej, fl_crspec=crspec, \
fl_fixpix=fixpix, fl_vardq=vardq, bpm=bpmPath, logfile=logPath)
# next delete previous copies of the output (should they exist)
iraf.imdel("g" + image)
iraf.imdel(pref + image)
# run gsreduce and view output
iraf.gsreduce(image, outpref=pref, fl_bias=bias, fl_gscrrej=gscrrej, \
fl_crspec=crspec, fl_qecorr=qecorr, fl_flat=flat, fl_fixpix=fixpix, \
fl_vardq=vardq, bias=biasPath, flatim=flatPath, qe_refim=refimPath, \
rawpath=imPath, bpm=bpmPath, logfile=logPath, verbose=verbose)
image = pref + image
self.viewIm(image, frameNo=frameNo)
return
def create_master_bias(qd,
dbFile,
data_dir,
master_bias='MCbias.fits',
overwrite=True):
cur_dir = os.getcwd()
#os.chdir(data_dir)
iraf.chdir(data_dir)
if os.path.exists(master_bias):
if overwrite is True:
remove = raw_input('Remove bias file? (y), n ')
if remove != 'n':
os.remove(master_bias)
else:
return None
else:
print('Master bias, {} already exists and overwrite={}'.format(
master_bias, overwrite))
return None
print(" --Creating Bias MasterCal--")
SQL = fileSelect.createQuery('bias', qd)
bias_files = fileSelect.fileListQuery(dbFile, SQL, qd)
start_date = Time(qd['DateObs'].split(':')[0], out_subfmt='date')
end_date = qd['DateObs'].split(':')[1]
while len(bias_files) < 7:
start_date = start_date - 1.0 * u.day
qd['DateObs'] = '{}:{}'.format(start_date, end_date)
SQL = fileSelect.createQuery('bias', qd)
bias_files = fileSelect.fileListQuery(dbFile, SQL, qd)
gmos.gbias.unlearn()
bias_flags = {
'logfile': 'biasLog.txt',
'rawpath': '',
'fl_vardq': 'yes',
'verbose': 'no'
}
print('{} bias frames used in Master Bias over date range {}'.format(
len(bias_files), qd['DateObs']))
if len(bias_files) < 10:
print('******WARNING less than 10 bias files********')
if len(bias_files) > 1:
gmos.gbias(','.join(str(x) for x in bias_files), master_bias,
**bias_flags)
# Clean up
if not os.path.exists(master_bias): #Check that IRAF didn't error
sys.exit('ERROR creating Master Bias: {}'.format(master_bias))
#Remove intermediate files
if qd['Instrument'] == 'GMOS-N':
image_str = 'gN'
else:
image_str = 'gS'
image_str = '{}{}*.fits'.format(image_str, qd['DateObs'][0:4])
iraf.imdel(image_str)
flist = glob.glob('tmplist*')
for ifile in flist:
os.remove(ifile)
iraf.chdir(cur_dir)
def reflesh(list_str, str_unvalu, flag):
j = 0
while j < len(list_str):
if list_str[j][flag] == str_unvalu:
iraf.imdel(list_str[j][0], veri=0) #delete the term of list
del list_str[j] #delete associated image
else:
j += 1
return list_str
def custom1(filename): # for NACO timing mode cubes - removes horizontal banding
#iraf.imarith(filename,'-','dark','temp')
iraf.imarith(filename,'/','flatK','temp')
im = pyfits.getdata('temp.fits')
med = median(im.transpose())
out = ((im).transpose()-med).transpose()
(pyfits.ImageHDU(out)).writeto("temp2.fits",clobber=True)
iraf.imdel('temp')
iraf.imcopy('temp2[1]','temp')
def nuke(fileparm):
'''Delete a list of images/files, but only try it if they exist. USE WITH CAUTION.
With FITS files, this is a bit over-cautious, but with imh files, you reallly need
to use imdel to get rid of the .pix files properly.'''
files = get_files(fileparm)
if files is None: return
for file in files:
if os.path.isfile(file):
if re.search(r'(fits|imh)$', file):
iraf.imdel(file)
else:
os.unlink(file)
def domask999():
files = glob.glob('fm*.fits')
mask = 'mask2007.pl' #bad values = 1, good values=0
iraf.imarith(1, "-", mask, "tempmask") #good values=1, badvalues=0
mode = -9999.
for current in files:
mfile = "m" + current
iraf.imarith(current, "*", "tempmask", mfile)
iraf.imarith(mask, "*", mode, "tempmask2")
print "masking ", current, " with mode = ", mode
iraf.imarith(mfile, "+", "tempmask2", mfile)
iraf.imdel("tempmask2")
iraf.imdel("tempmask")
def psfconv(df_image,psf):
print "\n************ Running the psf convolution steps ************\n"
iraf.imdel('_model.fits')
iraf.imdel('_model_4.fits')
iraf.imdel('_res*.fits')
iraf.imdel('_psf*.fits')
iraf.imdel('_df_sub')
'subtract the sky value from the dragonfly image header'
try:
df_backval = fits.getheader(df_image)['BACKVAL']
iraf.imarith('%s'%df_image,'-','%s'%df_backval,'_df_sub')
except:
print "WARNING: No BACKVAL to subtract! Skipping the background subtraction..."
iraf.imcopy('%s'%df_image,'_df_sub.fits')
##### subtract the background from the cfht image?
'convolve the model with the Dragonfly PSF'
if usemodelpsf:
makeallisonspsf()
psf = './psf/psf_static_fullframe.fits'
if verbose:
print 'VERBOSE: Using %s for the psf convolution.'%psf
'resample the PSF by a factor of 4'
iraf.magnify('%s'%psf,'_psf_4',4,4,interp="spline3")
'this is just to retain the same total flux in the psf'
iraf.imarith('_psf_4','*',16.,'_psf_4')
'Convolve with the psf'
# from scipy import signal
# fluxmoddata,fluxmodheader = fits.getdata('_fluxmod_dragonfly.fits',header=True)
# psfdata = fits.getdata('_psf_4.fits')
# fluxmodheader['COMMENT']='convolved with '+'_psf_4.fits'
# modeldata = signal.fftconvolve(fluxmoddata, psfdata)
# print ""
# print fluxmoddata.shape
# print modeldata.shape
# print ""
# writeFITS(modeldata,fluxmodheader,'_model_4.fits')
iraf.stsdas.analysis.fourier.fconvolve('_fluxmod_dragonfly','_psf_4','_model_4')
'now after the convolution we can go back to the Dragonfly resolution'
iraf.blkavg('_model_4','_model',4,4,option="average")
return None
def domask(files, invmask):
current = files
#for current in files:
mfile = "m" + current
#iraf.mask(current,mask=invmask,stat="[400:600,400:600]")
mode = iraf.imstatistics(current,
fields="mode",
lower=1,
format=0,
Stdout=1)
mode = float(mode[0])
print "masking ", current, " with mode = ", mode
iraf.imarith(1, "-", invmask, "tempmask")
iraf.imarith(current, "*", "tempmask", mfile)
iraf.imarith(invmask, "*", mode, "tempmask2")
iraf.imarith(mfile, "+", "tempmask2", mfile)
iraf.imdel("tempmask")
iraf.imdel("tempmask2")
def transform(self, image, lamRef, pref="t", vardq="no", logPath="", \
verbose="yes", frameNo=2):
print "RUNNING GSTRANSFORM"
# stash task settings
self.stashSettings('gstransform', outpref=pref, wavtraname=lamRef, \
fl_vardq=vardq, logfile=logPath)
# next delete previous copy of the output (should it exist)
iraf.imdel(pref + image)
# run gstransform and view output
iraf.gstransform(image, outpref=pref, wavtraname=lamRef, \
fl_vardq=vardq, logfile=logPath, verbose=verbose)
image = pref + image
self.viewIm(image, frameNo=frameNo)
return
def skysub(self, image, pref="s", vardq="no", sample="*", func="chebyshev", \
order=1, low=2.5, high=2.5, inter="no", logPath="", verbose="yes", \
frameNo=3):
print "RUNNING GSSKYSUB"
# stash task settings
self.stashSettings('gsskysub', outpref=pref, fl_vardq=vardq, \
long_sample=sample, function=func, order=order, low_rej=low, \
high_rej=high, fl_inter=inter, logfile=logPath)
# next delete previous copy of the output (should it exit)
iraf.imdel(pref + image)
# run gsskysub and view output
iraf.gsskysub(image, outpref=pref, fl_vardq=vardq, long_sample=sample, \
function=func, order=order, low_rej=low, high_rej=high, \
fl_inter=inter, logfile=logPath, verbose=verbose)
image = pref + image
self.viewIm(image, frameNo=frameNo)
return
def extract(self, image, pref="e", apRef="", width=1., inter="no", \
center="yes", trace="yes", weights="none", vardq="no", logPath="", \
verbose="yes", view=False):
print "RUNNING GSEXTRACT"
# stash task settings
self.stashSettings('gsextract', outpref=pref, refimages=apRef, \
apwidth=width, fl_inter=inter, recenter=center, trace=trace, \
weights=weights, fl_vardq=vardq, logfile=logPath)
# next delete previous copy of the output (should it exit)
iraf.imdel(pref + image)
# run gsextract and view output (if directed)
iraf.gsextract(image, outpref=pref, refimages=apRef, apwidth=width, \
fl_inter=inter, recenter=center, trace=trace, weights=weights, \
fl_vardq=vardq, logfile=logPath, verbose=verbose)
if view:
image = pref + image + "[sci]"
self.viewSpec(image)
return
def mask(res_org,upperlim=0.04):
print "\n**** Masking the residual, using upperlim of %s ****\n" % upperlim
upperlim = float(upperlim)
iraf.imdel('_res_final.fits')
iraf.imdel('_model_mask')
iraf.imdel('_model_maskb')
iraf.imcopy('_model_sc.fits','_model_mask.fits')
iraf.imreplace('_model_mask.fits',0,upper=upperlim)
iraf.imreplace('_model_mask.fits',1,lower=upperlim/2.)
iraf.boxcar('_model_mask','_model_maskb',5,5)
iraf.imreplace('_model_maskb.fits',1,lower=0.1)
iraf.imreplace('_model_maskb.fits',0,upper=0.9)
iraf.imarith(1,'-','_model_maskb','_model_maskb')
iraf.imarith('_model_maskb','*',res_org,'_res_final')
return None
def gmos_ls_proc2(
sciTargets,
stdTarget,
dbFile='./raw/obsLog.sqlite3',
qd_full={
'use_me': 1,
'Instrument': 'GMOS-S',
'CcdBin': '2 4',
'RoI': 'Full',
'Disperser': 'B600+_%',
'CentWave': 485.0,
'AperMask': '1.0arcsec',
'Object': 'AM2306-72%',
'DateObs': '2007-06-05:2007-07-07'
},
qd_censp={
'use_me': 1,
'Instrument': 'GMOS-S',
'CcdBin': '2 4',
'RoI': 'CenSp',
'Disperser': 'B600+_%',
'CentWave': 485.0,
'AperMask': '1.0arcsec',
'Object': 'LTT9239',
'DateObs': '2007-06-05:2007-07-07'
},
biasFlags={
'logfile': 'biasLog.txt',
'rawpath': './raw/',
'fl_vardq': 'yes',
'verbose': 'no'
},
flatFlags={
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_dark': 'no',
'fl_fixpix': 'no',
'fl_oversize': 'no',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'rawpath': './raw',
'fl_inter': 'no',
'fl_detec': 'yes',
'function': 'spline3',
'order': '13,11,28',
'logfile': 'gsflatLog.txt',
'verbose': 'no'
},
sciFlags={
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_gscrrej': 'no',
'fl_dark': 'no',
'fl_flat': 'yes',
'fl_gmosaic': 'yes',
'fl_fixpix': 'no',
'fl_gsappwave': 'yes',
'fl_oversize': 'no',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'rawpath': './raw',
'fl_inter': 'no',
'logfile': 'gsreduceLog.txt',
'verbose': 'no'
},
waveFlags={
'coordlist': 'gmos$data/CuAr_GMOS.dat',
'fwidth': 6,
'nsum': 50,
'function': 'chebyshev',
'order': 5,
'fl_inter': 'no',
'logfile': 'gswaveLog.txt',
'verbose': 'no'
},
sciCombFlags={
'combine': 'average',
'reject': 'ccdclip',
'fl_vardq': 'yes',
'fl_dqprop': 'yes',
'logfile': 'gemcombineLog.txt',
'verbose': 'no'
},
transFlags={
'fl_vardq': 'yes',
'interptype': 'linear',
'fl_flux': 'yes',
'logfile': 'gstransLog.txt'
},
skyFlags={
'fl_oversize': 'no',
'fl_vardq': 'yes',
'logfile': 'gsskysubLog.txt'
},
extrFlags={
'apwidth': 3.,
'fl_inter': 'yes',
'find': 'yes',
'trace': 'yes',
'tfunction': 'chebyshev',
'torder': '6',
'tnsum': 20,
'background': 'fit',
'bfunction': 'chebyshev',
'border': 2,
'fl_vardq': 'no',
'logfile': 'gsextrLog.txt'
},
calibFlags={
'extinction': 'onedstds$ctioextinct.dat',
'fl_ext': 'yes',
'fl_scale': 'no',
'sfunction': 'sens',
'fl_vardq': 'yes',
'logfile': 'gscalibrateLog.txt'
},
skip_wavecal=True,
clean_files=False):
"""
Parameters
----------
dbFile : str
Filename containing the SQL sqlite3 database created by obslog.py
It must be placed in the ./raw/ directory
Default is `./raw/obsLog.sqlite3`
sciTargets : dict
Dictionary with the associations of science targets and its associated ARC for wavelength calibration
as well as the regions defining the sky along the slit.
e.g. sciTargetd = {'AM2306-721_a': {'arc': 'gsS20070623S0071', 'sky': '520:720'},
'AM2306-72_b': {'arc': 'gsS20070623S0081', 'sky': '670:760,920:1020'}}
Note that there could be more than one target defined this way.
stdTarget : dict
Dictionary with the associations of standard star targets and its associated ARC for wavelength calibration
as well as the regions defining the sky along the slit.
e.g. stdTarget = {'LTT1788': {'arc': 'S20180711S0281', 'sky': '170:380,920:1080'}}
qd_full : dictionary
Query Dictionary of essential parameter=value pairs for Full RoI. Meant for science object.
qd_censp : dictionary
Query Dictionary of essential parameter=value pairs for CenSp RoI. Meant for standard star.
biasFlags : dict
Dictionary for the keyword flags of gmos.gbias() function
flatFlags : dict
Dictionary for the keyword flags of gmos.gsflat() function
sciFlags : dict
Dictionary for the keyword flags of gmos.gsreduce() function
Based on these flags a set of arcFlags and stdFlags dictionaries will be created
for basic processing.
waveFlags : dict
Dictionary for the keyword flags of gmos.gswavelength() function
sciCombFlags : dict
Dictionary for the keyword flags of gemtools.gemcombine() function
Based on these flags a set of stdCombFlags dictionary will be created for the standard advanced processing.
transFlags : dict
Dictionary for the keyword flags of gmos.gstransform() function.
xxx
skyFlags : dict
Dictionary for the keyword flags of gmos.gsskysub() function
extrFlags : dict
Dictionary for the keywords flags of gmos.gsextract() function
calibFlags : dict
XXX
skip_wavecal : bool
Whether to skip interactive wavelength calibration.
Useful when this is already done.
Returns
-------
"""
print("### Begin Processing GMOS/Longslit Images ###")
print("###")
print("=== Creating MasterCals ===")
# From the work_directory:
# Create the query dictionary of essential parameter=value pairs for Full and CenSp RoIs
qd = {'Full': qd_full, 'CenSp': qd_censp}
print(" --Creating Bias MasterCal--")
# Set the task parameters.
gemtools.gemextn.unlearn() # Disarm a bug in gbias
gmos.gbias.unlearn()
regions = ['Full', 'CenSp']
for r in regions:
# The following SQL generates the list of full-frame files to process.
SQL = fs.createQuery('bias', qd[r])
biasFiles = fs.fileListQuery(dbFile, SQL, qd[r])
# The str.join() funciton is needed to transform a python list into a
# comma-separated string of file names that IRAF can understand.
if len(biasFiles) > 1:
# NT comment: sometimes if there are too many files, gmos.gbias() raises an error.
# import pdb; pdb.set_trace()
gmos.gbias(','.join(str(x) for x in biasFiles), 'MCbias' + r,
**biasFlags)
# Clean up
year_obs = qd_full['DateObs'].split('-')[0]
if clean_files:
iraf.imdel("gS{}*.fits".format(year_obs))
ask_user(
"MC Bias done. Would you like to continue to proceed with GCAL Spectral Master Flats? (y/n): ",
['y', 'yes'])
print(" -- Creating GCAL Spectral Flat-Field MasterCals --")
# Set the task parameters.
qd['Full'].update({'DateObs': '*'})
qd['CenSp'].update({'DateObs': '*'})
gmos.gireduce.unlearn()
gmos.gsflat.unlearn()
# Normalize the spectral flats per CCD.
# The response fitting should be done interactively.
if flatFlags['fl_inter'] != 'yes':
print(
"The response fitting should be done interactively. Please set flatFlags['fl_inter'] = 'yes'."
)
ask_user(
"Do you still want to proceed despite this important warning? (y/n): ",
['yes', 'y'])
for r in regions:
qr = qd[r]
flatFiles = fs.fileListQuery(dbFile, fs.createQuery('gcalFlat', qr),
qr)
if len(flatFiles) > 0:
gmos.gsflat(','.join(str(x) for x in flatFiles),
'MCflat' + r,
bias='MCbias' + r,
**flatFlags)
if clean_files:
iraf.imdel('gS{}*.fits,gsS{}*.fits'.format(year_obs, year_obs))
ask_user(
"GCAL Spectral Flat-Field MasterCals done. Would you like to continue to proceed with Basic Processing? (y/n): ",
['y', 'yes'])
print("=== Processing Science Files ===")
print(" -- Performing Basic Processing --")
# Set task parameters.
gmos.gsreduce.unlearn()
sciFlags = sciFlags # redundant but put here because NT likes it
arcFlags = copy.deepcopy(sciFlags)
arcFlags.update({'fl_flat': 'no', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
stdFlags = copy.deepcopy(sciFlags)
stdFlags.update({'fl_fixpix': 'yes', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
# Perform basic reductions on all exposures for science targets.
print(" - Arc exposures -")
for r in regions:
qr = qd[r]
arcFiles = fs.fileListQuery(dbFile, fs.createQuery('arc', qr), qr)
if len(arcFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in arcFiles),
bias='MCbias' + r,
**arcFlags)
print(" - Std star exposures -")
r = 'CenSp'
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qd[r]), qd[r])
if len(stdFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in stdFiles),
bias='MCbias' + r,
flatim='MCflat' + r,
**stdFlags)
print(" - Science exposures -")
r = 'Full'
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qd[r]),
qd[r])
if len(sciFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in sciFiles),
bias='MCbias' + r,
flatim='MCflat' + r,
**sciFlags)
# Clean up
if clean_files:
iraf.imdel('gS{}*.fits'.format(year_obs))
ask_user(
"Basic processing done. Would you like to continue to determine wavelength calibration? (y/n): ",
['y', 'yes'])
print(" -- Determine wavelength calibration --")
# Set task parameters
gmos.gswavelength.unlearn()
# The fit to the dispersion relation should be performed interactively.
# Here we will use a previously determined result.
if waveFlags['fl_inter'] != 'yes':
print(
"The fit to the dispersion relation should be performed interactively. Please set waveFlags['fl_inter'] = 'yes'."
)
ask_user(
"Do you still want to proceed despite this important warning? (y/n): ",
['yes', 'y'])
# Need to select specific wavecals to match science exposures.
# NT: we do this now from the sciTargets + stdTarget input dictionaries
# e.g.
'''
sciTargets = {
'AM2306-721_a': {'arc': 'gsS20070623S0071', 'sky': '520:720'},
'AM2306-72_b': {'arc': 'gsS20070623S0081', 'sky': '670:760,920:1020'},
'AM2306-721_c': {'arc': 'gsS20070623S0091', 'sky': '170:380,920:1080'}
}
'''
#prefix = 'gsS20070623S0'
#for arc in ['071', '081', '091', '109']:
# gmos.gswavelength(prefix + arc, **waveFlags)
prefix = 'gs'
arc_files = []
for key in sciTargets.keys():
arc_files += [sciTargets[key]['arc']]
for key in stdTarget.keys():
arc_files += [stdTarget[key]['arc']]
# import pdb; pdb.set_trace()
if skip_wavecal is not True:
for arc in arc_files:
gmos.gswavelength(prefix + arc, **waveFlags)
### End of basic processing. Continue with advanced processing.
ask_user(
"Wavelength solution done. Would you like to continue with advanced processing? (y/n): ",
['y', 'yes'])
print(" -- Performing Advanced Processing --")
print(" -- Combine exposures, apply dispersion, subtract sky --")
# Set task parameters.
gemtools.gemcombine.unlearn()
sciCombFlags = sciCombFlags
stdCombFlags = copy.deepcopy(sciCombFlags)
stdCombFlags.update({'fl_vardq': 'no', 'fl_dqprop': 'no'})
gmos.gstransform.unlearn()
# apply gtransform to standard
# Process the Standard Star
prefix = "gs"
qs = qd['CenSp']
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qs), qs)
std_name = stdTarget.keys()[0]
if len(stdFiles) == 0:
ValueError(
"No standard star associated. Please check parameters of search (e.g. RoI=CentSp)"
)
# import pdb; pdb.set_trace()
if len(stdFiles) > 1:
# import pdb; pdb.set_trace()
gemtools.gemcombine(','.join(prefix + str(x) for x in stdFiles),
std_name, **stdCombFlags)
else:
os.system("cp {}.fits {}.fits".format(prefix + stdFiles[0], std_name))
gmos.gstransform(std_name,
wavtraname=prefix + stdTarget[std_name]['arc'],
**transFlags)
# The sky regions should be selected with care, using e.g. prows/pcols:
# pcols ("tAM2306b.fits[SCI]", 1100, 2040, wy1=40, wy2=320)
print(
"The sky regions should be selected with care, using e.g. with prows/pcols (see tutorial)."
)
'''
answer = raw_input("Please provide the long_sample string to apply to gmos.gsskysub() for the standard star."
"e.g. '20:70,190:230' (say 'no' for using the example as the default values): ")
if answer in ['n', 'no']:
print("Using default long_sample set by stdTarget values {}.".format(stdTarget[std_name]['sky']))
long_sample_std = stdTarget[std_name]['sky']
else:
long_sample_std = answer
'''
long_sample_std = stdTarget[std_name]['sky']
ask_user(
"Before proceeding it is important that you have set a good sky region for the standard.\n"
"Thus far you have selected: {}\n Would you like to proceed with the current one? (y/n): "
.format(long_sample_std), ['yes', 'y'])
# apply sky substraction
skyFlags = skyFlags
gmos.gsskysub.unlearn()
gmos.gsskysub('t{}'.format(std_name), long_sample=long_sample_std)
# NT: make sure the process works ok until here before proceeding further. i.e. setting the sky region manually and correctly.
# NT: seems to be working.
print(" -- Extract Std spectrum --")
# Extract the std spectrum using a large aperture.
# It's important to trace the spectra interactively.
gmos.gsextract.unlearn()
gmos.gsextract("st" + std_name, **extrFlags)
print(" -- Derive the Flux calibration --")
gmos.gsstandard.unlearn()
sensFlags = {
'fl_inter': 'no',
'starname': 'XXX',
'caldir': 'onedstds$ctionewcal/',
'observatory': 'Gemini-South',
'extinction': 'onedstds$ctioextinct.dat',
'function': 'chebyshev',
'order': 9,
'verbose': 'no',
'logfile': 'gsstdLog.txt'
}
sensFlags['starname'] = stdTarget[std_name][
'iraf_name'] # replace corresponding starname
gmos.gsstandard('est' + std_name,
sfile='std.txt',
sfunction='sens',
**sensFlags)
ask_user(
"Sensitivity function from standard star done. Would you like to continue with reduction of science"
" exposures? (y/n): ", ['yes', 'y'])
# Process the science targets.
# Use a dictionary to associate science targets with Arcs and sky regions.
prefix = 'gs'
extract_individuals = True
for targ, p in sciTargets.iteritems():
qs = qd['Full']
qs['Object'] = p['name']
# Fix up the target name for the output file
sciOut = p['name_out']
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qs), qs)
all_files = ','.join(prefix + str(x) for x in sciFiles)
gemtools.gemcombine(all_files, sciOut, **sciCombFlags)
gmos.gstransform(sciOut, wavtraname=prefix + p['arc'], **transFlags)
ask_user(
"It is important to select a good sky region for substraction. Thus far you have selected {}"
" based on the sciTargets input dictionary. Would you like to continue? (y/n): "
.format(p['sky']), ['y', 'yes'])
gmos.gsskysub('t' + sciOut, long_sample=p['sky'], **skyFlags)
if extract_individuals:
import pdb
pdb.set_trace()
for fname in sciFiles:
gmos.gstransform(prefix + fname,
wavtraname=prefix + p['arc'],
**transFlags)
gmos.gsskysub('t' + prefix + fname,
long_sample=p['sky'],
**skyFlags)
gmos.gscalibrate.unlearn()
gmos.gscalibrate('st' + prefix + fname, **calibFlags)
# Clean up
if clean_files:
iraf.imdel("gsS{}*.fits".format(year_obs))
ask_user(
"Sky substraction done. Would you like to continue to apply sensitivity function? (y/n): ",
['y'])
## Apply the sensitivity function.
gmos.gscalibrate.unlearn()
gmos.gscalibrate('st' + sciOut + '*', **calibFlags)
calibFlags.update({'fl_vardq': 'no'})
gmos.gscalibrate('est' + std_name, **calibFlags)
print(" -- Extract Target Spectra --")
method = 'gsextract'
if method == 'gsextract':
gmos.gsextract.unlearn()
# import pdb;pdb.set_trace()
gmos.gsextract("cst" + sciOut, **extrFlags)
elif method == 'sarith':
# not implemented yet
onedspec.nsum = 4
onedspec.sarith('cst{}.fits[SCI]'.format(sciOut),
'copy',
'',
'ecst{}.ms'.format(sciOut),
apertures='222-346x4')
print("=== Finished Calibration Processing ===")
'verbose': 'yes'
}
regions = ['Full', 'CenSp']
for r in regions:
# The following SQL generates the list of full-frame files to process.
SQL = fs.createQuery('bias', qd[r])
biasFiles = fs.fileListQuery(dbFile, SQL, qd[r])
# The str.join() funciton is needed to transform a python list into a
# comma-separated string of file names that IRAF can understand.
if len(biasFiles) > 1:
gmos.gbias(','.join(str(x) for x in biasFiles), 'MCbias' + r,
**biasFlags)
# Clean up
iraf.imdel("gS2007*.fits")
print(" -- Creating GCAL Spectral Flat-Field MasterCals --")
# Set the task parameters.
qd['Full'].update({'DateObs': '*'})
qd['CenSp'].update({'DateObs': '*'})
gmos.gireduce.unlearn()
gmos.gsflat.unlearn()
# Normalize the spectral flats per CCD.
# The response fitting should be done interactively.
flatFlags = {
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_dark': 'no',
'fl_fixpix': 'no',
def gmos_img_proc2(
dbFile="./raw/obsLog.sqlite3",
qd={
'use_me': 1,
'Instrument': 'GMOS-S',
'CcdBin': '2 2',
'RoI': 'Full',
'Object': 'M8-%',
'DateObs': '2006-09-01:2006-10-30'
},
bias_dateobs="2006-09-01:2006-10-30",
biasFlags={
'logfile': 'biasLog.txt',
'rawpath': './raw/',
'fl_vardq': 'yes',
'verbose': 'yes'
},
flat_dateobs='2006-09-10:2006-10-10',
flatFlags={
'fl_scale': 'yes',
'sctype': 'mean',
'fl_vardq': 'yes',
'rawpath': './raw/',
'logfile': 'giflatLog.txt',
'verbose': 'yes'
},
filters=['Ha', 'HaC', 'SII', 'r', 'i'],
sciFlags={
'fl_over': 'yes',
'fl_trim': 'yes',
'fl_bias': 'yes',
'fl_dark': 'no',
'fl_flat': 'yes',
'logfile': 'gireduceLog.txt',
'rawpath': './raw/',
'fl_vardq': 'yes',
'bpm': bpm_gmos,
'verbose': 'yes'
},
mosaicFlags={
'fl_paste': 'no',
'fl_fixpix': 'no',
'fl_clean': 'yes',
'geointer': 'nearest',
'logfile': 'gmosaicLog.txt',
'fl_vardq': 'yes',
'fl_fulldq': 'yes',
'verbose': 'yes'
},
coaddFlags={
'fwhm': 3,
'datamax': 6.e4,
'geointer': 'nearest',
'logfile': 'imcoaddLog.txt'
},
targets=['M8-1', 'M8-2', 'M8-3'],
clean_files=False):
"""
Parameters
----------
dbFile : str
Filename containing the SQL sqlite3 database created by obslog.py
It must be placed in the ./raw/ directory
Default is `./raw/obsLog.sqlite3`
qd : dictionary
Query Dictionary of essential parameter=value pairs.
Select bias exposures within ~2 months of the target observations
e.g.
qd= {'use_me': 1,
'Instrument': 'GMOS-S', 'CcdBin': '2 2', 'RoI': 'Full', 'Object': 'M8-%',
'DateObs': '2006-09-01:2006-10-30'
}
bias_dateobs : str
String representing the bias search Obsdate
e.g. bias_dateobs = `2006-09-01:2006-10-30`
biasFlags : dict
Dictionary for the keyword flags of gmos.gbias() function
flat_dateobs : str
String representing the flat search Obsdate
e.g. flat_dateobs = `2006-09-10:2006-10-10`
flatFlags : dict
Dictionary for the keyword flags of gmos.giflat() function
e.g. flatFlags = {'fl_scale': 'yes', 'sctype': 'mean', 'fl_vardq': 'yes','rawpath': './raw/',
'logfile': 'giflatLog.txt', 'verbose': 'yes'}
filters : list
List of filter names to perform reduction
e.g. filters=['Ha', 'HaC', 'SII', 'r', 'i']
sciFlags : dict
Dictionary for the keyword flags of gmos.gireduce() function
mosaicFlags : dict
Dictionary for the keyword flags of gmos.gimosaic() function
coaddFlags : dict
Dictionary for the keyword flags of gemtools.imcoadd() function
targets : list
List of names of target observations for the co-addition
e.g. targets = ['M8-1', 'M8-2', 'M8-3']
clean_files : bool
Whether to clean intermediate files from reduction process
Returns
-------
Reduce GMOS imaging based on tutorial example.
"""
print("### Begin Processing GMOS/MOS Images ###")
print("###")
print("=== Creating MasterCals ===")
# From the work_directory:
# Create the query dictionary of essential parameter=value pairs.
# Select bias exposures within ~2 months of the target observations:
print(" --Creating Bias MasterCal--")
qd.update({'DateObs': bias_dateobs})
# Set the task parameters.
gmos.gbias.unlearn()
# The following SQL generates the list of files to process.
SQL = fs.createQuery('bias', qd)
biasFiles = fs.fileListQuery(dbFile, SQL, qd)
# The str.join() function is needed to transform a python list into a string
# filelist that IRAF can understand.
if len(biasFiles) > 1:
files_all = ','.join(str(x) for x in biasFiles)
# import pdb; pdb.set_trace()
gmos.gbias(files_all, 'MCbias.fits', **biasFlags)
# Clean up
year_obs = qd['DateObs'].split('-')[0]
if clean_files:
iraf.imdel('gS{}*.fits'.format(year_obs))
ask_user(
"MC Bias done. Would you like to continue to proceed with Master Flats? (y/n): ",
['y', 'yes'])
print(" --Creating Twilight Imaging Flat-Field MasterCal--")
# Select flats obtained contemporaneously with the observations.
qd.update({'DateObs': flat_dateobs})
# Set the task parameters.
gmos.giflat.unlearn()
#filters = ['Ha', 'HaC', 'SII', 'r', 'i']
for f in filters:
print " Building twilight flat MasterCal for filter: %s" % (f)
# Select filter name using a substring of the official designation.
qd['Filter2'] = f + '_G%'
mcName = 'MCflat_%s.fits' % (f)
flatFiles = fs.fileListQuery(dbFile, fs.createQuery('twiFlat', qd), qd)
if len(flatFiles) > 0:
files_all = ','.join(str(x) for x in flatFiles)
# import pdb; pdb.set_trace()
gmos.giflat(files_all, mcName, bias='MCbias', **flatFlags)
if clean_files:
iraf.imdel('gS{}*.fits,rgS{}*.fits'.format(year_obs, year_obs))
ask_user(
"MC Flats done. Would you like to continue to proceed with processing Science Images? (y/n): ",
['yes', 'y'])
print("=== Processing Science Images ===")
# Remove restriction on date range
qd['DateObs'] = '*'
prefix = 'rg'
gmos.gireduce.unlearn()
gemtools.gemextn.unlearn() # disarms a bug in gmosaic
gmos.gmosaic.unlearn()
# Reduce the science images, then mosaic the extensions in a loop
for f in filters:
print " Processing science images for filter: %s" % (f)
qd['Filter2'] = f + '_G%'
flatFile = 'MCflat_' + f + '.fits'
SQL = fs.createQuery('sciImg', qd)
sciFiles = fs.fileListQuery(dbFile, SQL, qd)
if len(sciFiles) > 0:
# Make sure BPM table is in sciFlags for employing the imaging Static BPM for this set of detectors.
# import pdb; pdb.set_trace()
all_files = ','.join(str(x) for x in sciFiles)
gmos.gireduce(all_files, bias='MCbias', flat1=flatFile, **sciFlags)
for file in sciFiles:
gmos.gmosaic(prefix + file, **mosaicFlags)
else:
print("No Science images found for filter {}. Check database.".
format(f))
import pdb
pdb.set_trace()
if clean_files:
iraf.imdelete('gS{}*.fits,rgS{}*.fits'.format(year_obs, year_obs))
ask_user(
"Science Images done. Would you like to continue to proceed with image co-addition? (y/n): ",
['y', 'yes'])
## Co-add the images, per position and filter.
print(" -- Begin image co-addition --")
# Use primarily the default task parameters.
gemtools.imcoadd.unlearn()
prefix = 'mrg'
for f in filters:
print " - Co-addding science images in filter: %s" % (f)
qd['Filter2'] = f + '_G%'
for t in targets:
qd['Object'] = t + '%'
print " - Co-addding science images for position: %s" % (t)
outImage = t + '_' + f + '.fits'
coAddFiles = fs.fileListQuery(dbFile, fs.createQuery('sciImg', qd),
qd)
all_files = ','.join(prefix + str(x) for x in coAddFiles)
if all_files == '':
print(
'No files available for co-addition. Check that the target names are written correctly.'
)
import pdb
pdb.set_trace()
gemtools.imcoadd(all_files, outimage=outImage, **coaddFlags)
ask_user(
"Co-addition done. Would you like to clean the latest intermediate reduction files? (y/n): ",
['y', 'yes'])
if clean_files:
iraf.delete("*_trn*,*_pos,*_cen")
iraf.imdelete("*badpix.pl,*_med.fits,*_mag.fits")
# iraf.imdelete ("mrgS*.fits")
print("=== Finished Calibration Processing ===")
}
regions = ['Full','CenSp']
for r in regions:
# The following SQL generates the list of full-frame files to process.
SQL = fs.createQuery('bias', qd[r])
biasFiles = fs.fileListQuery(dbFile, SQL, qd[r])
# The str.join() funciton is needed to transform a python list into a
# comma-separated string of file names that IRAF can understand.
if len(biasFiles) > 1:
gmos.gbias(','.join(str(x) for x in biasFiles), 'MCbias'+r,
**biasFlags)
# Clean up
iraf.imdel("gS2007*.fits")
print (" -- Creating GCAL Spectral Flat-Field MasterCals --")
# Set the task parameters.
qd['Full'].update({'DateObs':'*'})
qd['CenSp'].update({'DateObs':'*'})
gmos.gireduce.unlearn()
gmos.gsflat.unlearn()
# Normalize the spectral flats per CCD.
# The response fitting should be done interactively.
flatFlags = {
'fl_over':'yes','fl_trim':'yes','fl_bias':'yes','fl_dark':'no',
'fl_fixpix':'no','fl_oversize':'no','fl_vardq':'yes','fl_fulldq':'yes',
'rawpath':'./raw','fl_inter':'no','fl_detec':'yes',
'function':'spline3','order':'13,11,28',
'logfile':'gsflatLog.txt','verbose':'yes'
print('flat image not found!')
sys.exit()
for filename in file_list:
#flatfielding
iraf.imar(filename,'/',flat,filename[:-5]+'f.fits')
#geometric distortion correction
iraf.distcor(filename,filename[:-5]+'fg.fits',geomap)
a1=filename[:-5]+ch1
b1=filename[:-5]+'fg_ch1'
a2=filename[:-5]+ch2
b2=filename[:-5]+'fg_ch2'
iraf.imcopy(a1,b1)
iraf.imcopy(a2,b2)
#remove intermediary files
iraf.imdel(filename[:-5]+'f.fits')
iraf.imdel(filename[:-5]+'fg.fits')
############################################
'''
skysubtraction
'''
#!/usr/bin/env python
import glob
try:
from astropy.io import fits as pf
except:
import pyfits as pf
line = fd.readline()
# Loop through the calspec list file...
while line != "":
# This is the name of the calspec star file we are working on now...
calspecFileName = string.strip(line)
# Create an output file based on the original filter file name...
#outFileName='des_'+calspecFileName
index6 = calspecFileName.find("uk_")
index7 = calspecFileName.find(".fits")
picklesnum = str(int(calspecFileName[index6 + 3:index7]))
outFileName = 'des_pickles_' + picklesnum + '.fits'
print outFileName
# If a file with this name already exists, delete the old one...
iraf.imdel(outFileName)
#This loop cycles through each of the ccds on the focal plane
for i in range(len(filterFileNameList)):
# This is the particular filter file we will work on now...
filterFileName = filterFileNameList[i]
# Open the filter list file...
fd1 = open(filterFileName, 'r')
line1 = fd1.readline()
while line1 != "":
# This is the name of the calspec star file we are working on now...
filterBandFileName = string.strip(line1)
def fastfocus(inpat,
prepfile=yes,
endhow="never",
endwhen="",
clobber=globclob,
verbose=globver):
""" derive best focus from an SAOFOCUS run on P60 """
# Defaults
twait = 30
reduced = {}
bpmname = "BPM.pl"
saokey = "IMGTYPE"
saore = "SAOFOCUS"
fseqkey = "SAOFVALS"
sigma = 10.0
satval = 50000.0
masksfx = "mask"
pix = 0.3787 # arcsec per pixel
mindiff = 2 # minimum tolerable diff b/ best and worst seeing (pix)
# Parse end-condition "time"
if endhow == "time":
re1 = re.search("^(\d+):(\d+)", endwhen)
if re1:
tnow = time.gmtime()
# Force UT calculation w/ ignoring of DST issues
reftime = time.mktime([
tnow[0], tnow[1], tnow[2],
int(re1.group(1)),
int(re1.group(2)), 0, tnow[6], tnow[7], 0
]) - time.timezone
if reftime < time.time():
reftime += 86400
if verbose:
print "Running until %s" % \
time.strftime("%d %b %H:%M",time.gmtime(reftime))
else:
print "Failed to parse %s as UT time" % endwhen
print "Running until stopped..."
endhow = "never"
# Open fake OCS
fakeocs = ocs_ooriented(TEST=1)
fakeocs.read_foctable()
##################################################
# Big Loop
done = no
while not done:
# Parse inputs
allfiles = glob.glob(inpat)
newfiles = []
for image in allfiles:
if not reduced.has_key(image):
# Interested only in SAOFOCUS images
saoval = get_head(image, saokey)
if re.search(saore, saoval, re.I):
reduced[image] = no
newfiles.append(image)
else:
# Skip
reduced[image] = yes
for image in newfiles:
if verbose:
print "Processing new SAOFOCUS image %s" % image
# Only attempt reduction once for each image
reduced[image] = yes
# Demosaic
ccdproc = iraf.ccdproc
ccdproc.overscan = yes
ccdproc.trim = yes
ccdproc.zerocor = no
iraf.iqmosaic(image,
outpfx="",
biassec="!BIASSEC",
trimsec="!TRIMSEC",
joinkeys="DETSEC",
splitkeys="AMPSEC,CCDSEC,TRIM,OVERSCAN",
clobber=yes,
verbose=verbose)
# Fix the WCS that IRAF has now clobbered
iraf.add_wcs(image, instrument="p60new")
# Get some important keywords
[raoff,decoff,saonseq,saoshift] = \
get_head(image,['RAOFF','DECOFF','NUMFOC',
'SAOSHIFT'])
# Type conversions
saonseq = int(saonseq)
saoshift = float(saoshift)
# No bias-subtraction or flat-fielding
# Subtract stair-step background if requested
if prepfile:
qtmp = iraf.mktemp("saofoc") + ".fits"
iraf.imcopy(image, qtmp, verbose=no)
for iamp in [-1, 1]:
for istep in range(saonseq):
qsec = iraf.mktemp("saofsc") + ".fits"
y1 = 1024 + iamp * (1 + istep * saoshift) + max(
0, iamp)
y2 = y1 + iamp * (saoshift - 1)
if istep == 0:
y1 -= iamp
elif istep == saonseq - 1:
y2 = 1 + max(0, iamp) * 2047
ymin = min(y1, y2)
ymax = max(y1, y2)
statsec = '[*,%d:%d]' % (ymin, ymax)
iraf.iterstat(image + statsec,
nsigrej=5,
maxiter=10,
prin=no,
verbose=no)
medreg = float(iraf.iterstat.median)
iraf.imarith(image + statsec,
'-',
medreg,
qsec,
verbose=no,
noact=no)
iraf.imcopy(qsec, qtmp + statsec, verbose=no)
iraf.imdel(qsec, verify=no, go_ahead=yes)
iraf.imdel(image, verify=no, go_ahead=yes)
iraf.imrename(qtmp, image, verbose=no)
# No renaming of the file
# No Bad Pixel Masking or rudimentary WCS
# Object-detection
doobj = 1
if check_head(image, 'STARFILE'):
if os.path.exists(get_head(image, 'STARFILE')):
doobj = 0
if doobj:
iraf.iqobjs(image,
sigma,
satval,
skyval="0.0",
masksfx=masksfx,
wtimage="none",
minlim=no,
clobber=yes,
verbose=verbose)
# Establish our "region of interest" for pixels
pixrough = 800
pixminx = 1024 - float(decoff) / pix - pixrough / 2
pixmaxx = pixminx + pixrough
pixmaxy = 1023
pixminy = pixmaxy - float(raoff) / pix - (
1 + saonseq) * saoshift - pixrough / 2
# The array of focus positions (from low Y to high Y)
farrkeys = []
for i in range(saonseq):
farrkeys.append("SAOFOC%02d" % (i + 1))
farr = get_head(image, farrkeys)
# Starlist from Sextractor
starfile = get_head(image, 'STARFILE')
stars = Starlist(starfile)
stars.mag_sort()
# Find some of the stars in the sequence
xvals = []
yvals = []
good1 = []
for star in stars:
# We're going in order from bright to faint
if star.xval>pixminx and star.xval<pixmaxx and \
star.yval<pixmaxy and star.yval>pixminy and \
star.elong<3:
good1.append(star)
xvals.append(star.xval)
yvals.append(star.yval)
if len(good1) >= 1.2 * saonseq:
break
# Sanity checking
if len(xvals) < 1:
print "No stars found in search region"
update_head(image, 'SAOFOCUS', 0,
"saofocus processing failed for this image")
continue
# Guess for x is median of the xvals of the goodstars
xctr = median(xvals, pick=1)
# First guess for y is median of the goodstars that fall
# within +/- (pixfine) pix of this X value
#pixfine=10
pixfine = 20
yval2 = []
good2 = []
for star in good1:
if abs(star.xval - xctr) < pixfine:
good2.append(star)
yval2.append(star.yval)
# Sanity checking
if len(yval2) < 1:
print "No stars found in refined search region"
update_head(image, 'SAOFOCUS', 0,
"saofocus processing failed for this image")
continue
yctr = median(yval2, pick=1)
# This will be our reference star
usestar = good2[yval2.index(yctr)]
[refx, refy] = [usestar.xval, usestar.yval]
# Identify additional stars, working out from the
# reference star
usestars = [usestar]
# Search increasing & decreasing in Y
# (we use a bigger box in Y, and correct for the last star
# position, because of potential for tracking errors)
for isgn in [+1, -1]:
shifty = refy
for i in range(1, saonseq):
shifty += isgn * saoshift
tgtstars = stars.starsinbox(
[[refx - pixfine, refx + pixfine],
[shifty - pixfine, shifty + pixfine]])
#[shifty-1.5*pixfine,
#shifty+1.5*pixfine]])
if len(tgtstars) > 0:
minmag = 99.99
# Pick the brightest in the box
for tstar in tgtstars:
if tstar.mag < minmag:
svstar = tstar
minmag = tstar.mag
usestars.append(svstar)
shifty = svstar.yval
else:
# Quit searching when there are no stars in box
break
# Exclude faint stars if we have too many
# (could also drop stars until we reach the number, but
# then we would want to check that all the y-values are in
# strict succession...)
magrough = 2.0
if len(usestars) > saonseq:
usemags = []
for star in usestars:
usemags.append(star.mag)
minmag = min(usemags)
use2 = []
for star in usestars:
if star.mag < minmag + magrough:
use2.append(star)
if len(use2) == saonseq:
usestars = use2
# Check for the right number
if len(usestars) != saonseq:
print "Failed to get right number of target stars"
update_head(image, 'SAOFOCUS', 0,
"saofocus processing failed for this image")
continue
# Construct the array of x- and y-values
xval3 = []
yval3 = []
for star in usestars:
xval3.append(star.xval)
yval3.append(star.yval)
# Choose a single x-value for the run
xctr2 = "%.3f" % median(xval3)
# Increasing y-value means increasing index in farr
yval3.sort()
ylis = ','.join(map(str, yval3))
flis = ','.join(map(str, farr))
# Run iqfocus
iraf.iqfocus(image,
xctr2,
ylis,
flis,
method="acorr",
boxsize=saoshift,
focuskey="BESTFOC",
seekey="SEEPIX",
fseekey="SAOSEE",
update=yes,
clobber=yes,
verbose=verbose)
# Check for successful iqfocus run
if not check_head(image, 'BESTFOC'):
print "iqfocus run failed to find good focus"
update_head(image, 'SAOFOCUS', 0,
"saofocus processing failed for this image")
continue
# Grab focus positions and seeing values
bestfoc, saosee = get_head(image, ['BESTFOC', 'SAOSEE'])
# Some information
if verbose:
print "Focus settings: " + flis
print "Seeing values: " + saosee
print "Best focus: %.3f" % bestfoc
# Check for best-focus "near edge" condition
for i in range(saonseq):
if ("%.3f" % bestfoc) == ("%.3f" % float(farr[i])):
ibest = i
break
if (saonseq>3 and (ibest==0 or ibest==saonseq-1)) or \
(saonseq>5 and (ibest==1 or ibest==saonseq-2)):
update_head(image, 'SAOEDGE', 1,
"Best seeing is near edge of focus run")
# Check for insufficient variation across the focus run
maxsee = 0
minsee = 0.5 * saoshift
saoseevals = eval(saosee)
for seeval in saoseevals:
minsee = min(minsee, seeval)
maxsee = max(maxsee, seeval)
if (maxsee - minsee) < mindiff:
print "Failed to observe significant variation "+\
"across focus run"
update_head(image, 'SAOFOCUS', 0,
"saofocus processing failed for this image")
continue
# Convert seeing value to arcsec
seepix = get_head(image, 'SEEPIX')
update_head(image, 'BESTSEE', seepix,
"Best seeing in pixels for this series")
try:
seeasec = float(seepix) * pix
update_head(image, 'SEEING', seeasec,
"Best seeing in arcsec for this series")
except:
print "Failed to convert seeing to arcsec for %s" % image
# Successful processing
update_head(image, 'SAOFOCUS', 1,
"saofocus processing succeeded for this image")
# Get other necessary keywords
[filter, btubtemp, airmass] = get_head(image, \
['FILTER','BTUBTEMP','AIRMASS'])
# Update focus file
fakeocs.focus = [filter, bestfoc, float(btubtemp), float(airmass)]
fakeocs.write_foctable()
##############################
# Test end conditions
if endhow == "never":
done = no
elif endhow == "once":
done = yes
elif endhow == "time":
if time.time() > reftime:
done = yes
# Wait a little while
if not done:
time.sleep(twait)
def gmos_img_proc2(dbFile="./raw/obsLog.sqlite3", qd={'use_me': 1,'Instrument': 'GMOS-S', 'CcdBin': '2 2', 'RoI': 'Full', 'Object': 'M8-%', 'DateObs': '2006-09-01:2006-10-30'},
bias_dateobs="2006-09-01:2006-10-30",
biasFlags={'logfile': 'biasLog.txt', 'rawpath': './raw/', 'fl_vardq': 'yes', 'verbose': 'yes'},
flat_dateobs='2006-09-10:2006-10-10',
flatFlags = {'fl_scale': 'yes', 'sctype': 'mean', 'fl_vardq': 'yes','rawpath': './raw/', 'logfile': 'giflatLog.txt', 'verbose': 'yes'},
filters = ['Ha', 'HaC', 'SII', 'r', 'i'],
sciFlags={'fl_over': 'yes', 'fl_trim': 'yes', 'fl_bias':'yes', 'fl_dark': 'no','fl_flat': 'yes', 'logfile':'gireduceLog.txt', 'rawpath': './raw/','fl_vardq': 'yes','bpm':bpm_gmos, 'verbose': 'yes'},
mosaicFlags = {'fl_paste': 'no', 'fl_fixpix': 'no', 'fl_clean': 'yes', 'geointer': 'nearest', 'logfile': 'gmosaicLog.txt', 'fl_vardq': 'yes', 'fl_fulldq': 'yes', 'verbose': 'yes'},
coaddFlags = {'fwhm': 3, 'datamax': 6.e4, 'geointer': 'nearest', 'logfile': 'imcoaddLog.txt'},
targets = ['M8-1', 'M8-2', 'M8-3'],
clean_files = False
):
"""
Parameters
----------
dbFile : str
Filename containing the SQL sqlite3 database created by obslog.py
It must be placed in the ./raw/ directory
Default is `./raw/obsLog.sqlite3`
qd : dictionary
Query Dictionary of essential parameter=value pairs.
Select bias exposures within ~2 months of the target observations
e.g.
qd= {'use_me': 1,
'Instrument': 'GMOS-S', 'CcdBin': '2 2', 'RoI': 'Full', 'Object': 'M8-%',
'DateObs': '2006-09-01:2006-10-30'
}
bias_dateobs : str
String representing the bias search Obsdate
e.g. bias_dateobs = `2006-09-01:2006-10-30`
biasFlags : dict
Dictionary for the keyword flags of gmos.gbias() function
flat_dateobs : str
String representing the flat search Obsdate
e.g. flat_dateobs = `2006-09-10:2006-10-10`
flatFlags : dict
Dictionary for the keyword flags of gmos.giflat() function
e.g. flatFlags = {'fl_scale': 'yes', 'sctype': 'mean', 'fl_vardq': 'yes','rawpath': './raw/',
'logfile': 'giflatLog.txt', 'verbose': 'yes'}
filters : list
List of filter names to perform reduction
e.g. filters=['Ha', 'HaC', 'SII', 'r', 'i']
sciFlags : dict
Dictionary for the keyword flags of gmos.gireduce() function
mosaicFlags : dict
Dictionary for the keyword flags of gmos.gimosaic() function
coaddFlags : dict
Dictionary for the keyword flags of gemtools.imcoadd() function
targets : list
List of names of target observations for the co-addition
e.g. targets = ['M8-1', 'M8-2', 'M8-3']
clean_files : bool
Whether to clean intermediate files from reduction process
Returns
-------
Reduce GMOS imaging based on tutorial example.
"""
print ("### Begin Processing GMOS/MOS Images ###")
print ("###")
print ("=== Creating MasterCals ===")
# From the work_directory:
# Create the query dictionary of essential parameter=value pairs.
# Select bias exposures within ~2 months of the target observations:
print (" --Creating Bias MasterCal--")
qd.update({'DateObs': bias_dateobs})
# Set the task parameters.
gmos.gbias.unlearn()
# The following SQL generates the list of files to process.
SQL = fs.createQuery('bias', qd)
biasFiles = fs.fileListQuery(dbFile, SQL, qd)
# The str.join() function is needed to transform a python list into a string
# filelist that IRAF can understand.
if len(biasFiles) > 1:
files_all = ','.join(str(x) for x in biasFiles)
# import pdb; pdb.set_trace()
gmos.gbias(files_all, 'MCbias.fits', **biasFlags)
# Clean up
year_obs = qd['DateObs'].split('-')[0]
if clean_files:
iraf.imdel('gS{}*.fits'.format(year_obs))
ask_user("MC Bias done. Would you like to continue to proceed with Master Flats? (y/n): ",['y','yes'])
print (" --Creating Twilight Imaging Flat-Field MasterCal--")
# Select flats obtained contemporaneously with the observations.
qd.update({'DateObs': flat_dateobs})
# Set the task parameters.
gmos.giflat.unlearn()
#filters = ['Ha', 'HaC', 'SII', 'r', 'i']
for f in filters:
print " Building twilight flat MasterCal for filter: %s" % (f)
# Select filter name using a substring of the official designation.
qd['Filter2'] = f + '_G%'
mcName = 'MCflat_%s.fits' % (f)
flatFiles = fs.fileListQuery(dbFile, fs.createQuery('twiFlat', qd), qd)
if len(flatFiles) > 0:
files_all = ','.join(str(x) for x in flatFiles)
# import pdb; pdb.set_trace()
gmos.giflat(files_all, mcName, bias='MCbias', **flatFlags)
if clean_files:
iraf.imdel('gS{}*.fits,rgS{}*.fits'.format(year_obs, year_obs))
ask_user("MC Flats done. Would you like to continue to proceed with processing Science Images? (y/n): ", ['yes','y'])
print ("=== Processing Science Images ===")
# Remove restriction on date range
qd['DateObs'] = '*'
prefix = 'rg'
gmos.gireduce.unlearn()
gemtools.gemextn.unlearn() # disarms a bug in gmosaic
gmos.gmosaic.unlearn()
# Reduce the science images, then mosaic the extensions in a loop
for f in filters:
print " Processing science images for filter: %s" % (f)
qd['Filter2'] = f + '_G%'
flatFile = 'MCflat_' + f + '.fits'
SQL = fs.createQuery('sciImg', qd)
sciFiles = fs.fileListQuery(dbFile, SQL, qd)
if len(sciFiles) > 0:
# Make sure BPM table is in sciFlags for employing the imaging Static BPM for this set of detectors.
# import pdb; pdb.set_trace()
all_files = ','.join(str(x) for x in sciFiles)
gmos.gireduce(all_files, bias='MCbias', flat1=flatFile, **sciFlags)
for file in sciFiles:
gmos.gmosaic(prefix + file, **mosaicFlags)
else:
print("No Science images found for filter {}. Check database.".format(f))
import pdb; pdb.set_trace()
if clean_files:
iraf.imdelete('gS{}*.fits,rgS{}*.fits'.format(year_obs,year_obs))
ask_user("Science Images done. Would you like to continue to proceed with image co-addition? (y/n): ", ['y','yes'])
## Co-add the images, per position and filter.
print (" -- Begin image co-addition --")
# Use primarily the default task parameters.
gemtools.imcoadd.unlearn()
prefix = 'mrg'
for f in filters:
print " - Co-addding science images in filter: %s" % (f)
qd['Filter2'] = f + '_G%'
for t in targets:
qd['Object'] = t + '%'
print " - Co-addding science images for position: %s" % (t)
outImage = t + '_' + f + '.fits'
coAddFiles = fs.fileListQuery(dbFile, fs.createQuery('sciImg', qd), qd)
all_files = ','.join(prefix + str(x) for x in coAddFiles)
if all_files == '':
print('No files available for co-addition. Check that the target names are written correctly.')
import pdb; pdb.set_trace()
gemtools.imcoadd(all_files, outimage=outImage, **coaddFlags)
ask_user("Co-addition done. Would you like to clean the latest intermediate reduction files? (y/n): ", ['y','yes'])
if clean_files:
iraf.delete("*_trn*,*_pos,*_cen")
iraf.imdelete("*badpix.pl,*_med.fits,*_mag.fits")
# iraf.imdelete ("mrgS*.fits")
print ("=== Finished Calibration Processing ===")
'rawpath': './raw/',
'fl_vardq': 'yes',
'verbose': 'no'
}
# The following SQL generates the list of files to process.
SQL = fs.createQuery('bias', qd)
biasFiles = fs.fileListQuery(dbFile, SQL, qd)
# The str.join() function is needed to transform a python list into a string
# filelist that IRAF can understand.
if len(biasFiles) > 1:
gmos.gbias(','.join(str(x) for x in biasFiles), 'MCbias.fits',
**biasFlags)
# Clean up
iraf.imdel('gS2006*.fits')
print(" --Creating Twilight Imaging Flat-Field MasterCal--")
# Select flats obtained contemporaneously with the observations.
qd.update({'DateObs': '2006-09-10:2006-10-10'})
# Set the task parameters.
gmos.giflat.unlearn()
flatFlags = {
'fl_scale': 'yes',
'sctype': 'mean',
'fl_vardq': 'yes',
'rawpath': './raw/',
'logfile': 'giflatLog.txt',
'verbose': 'yes'
}
def iqp60(inpat,endhow="never",endwhen="",focmode="all",
logfile="",clobber=globclob,verbose=globver):
""" intelligent processing of P60 data files """
# Defaults
twait=30
reduced={}
filtkey="FILTER"
biaskey="IMGTYPE"
biasre="BIAS"
biasroot="Bias"
biaspfx="b"
flatkey="IMGTYPE"
flatre="DOMEFLAT"
flatpre="Flat-"
flatpfx="f"
bpm0root="BPM0"
bpmroot="BPM"
binkey="CCDSUM"
detseckey="DETSEC"
binroikey="BINROI"
binroidef="A"
nfocus=10
focuskey="IMGTYPE"
focusre="^FOCUS"
fseqkey="FOCUSSEQ"
skipkey="IMGTYPE"
skipre="SAOFOCUS"
statsec=""
reqkeys=['IMGTYPE','OBJECT','FILTER','BINROI',
'CCDSUM','DETSEC','ROISEC']
sigma=3.0
minraw=0.0
maxraw=65535.0
satval=50000.0
masksfx="mask"
rawpix=0.3787
zptkey="ZEROPT"
zpukey="ZEROPTU"
# For now:
#catmags={'B':'BMAG','V':'BMAG','g':'BMAG',
# 'R':'RMAG','r':'RMAG','I':'IMAG',
# 'i':'IMAG','ip':'IMAG','z':'IMAG',
# 'zp':'IMAG','zl':'IMAG','zs':'IMAG'}
## In the future, once Sloan mags are written to the ubcone output:
catmags={'B':'BMAG','V':'VMAG','g':'GPMAG',
'R':'RMAG','r':'RPMAG','I':'IMAG',
'i':'IMAG','ip':'IPMAG','z':'ZPMAG',
'zp':'ZPMAG','zl':'ZPMAG','zs':'ZPMAG',
'upr':'UPMAG','gpr':'GPMAG','rpr':'RPMAG',
'ipr':'IPMAG','zpr':'ZPMAG'}
exptmkey="EXPTIME"
exptmstd=60.0
minzpts={}
minskys={}
fwhm0=1.5
gain=2.3
readn=7.5
crsfx="crm"
statfile="Status_Science.txt"
# Parse end-condition "time"
if endhow=="time":
re1=re.search("^(\d+):(\d+)",endwhen)
if re1:
tnow=time.gmtime()
# Force UT calculation w/ ignoring of DST issues
reftime=time.mktime([tnow[0],tnow[1],tnow[2],
int(re1.group(1)),int(re1.group(2)),0,
tnow[6],tnow[7],0]) - time.timezone
if reftime<time.time():
reftime+=86400
if verbose:
print "Running until %s" % \
time.strftime("%d %b %H:%M",time.gmtime(reftime))
else:
print "Failed to parse %s as UT time" % endwhen
print "Running until stopped..."
endhow="never"
# Parse focmode setting
fonly=0
if focmode=="nofocus":
skipre="FOCUS"
elif focmode=="focus":
fonly=1
skipre="SCIENCE"
# Parse logfile setting
if len(logfile)>0:
check_exist(logfile,'w',yes)
try:
log=open(logfile,'w')
sys.stdout=log
except:
print "Failed to open logfile %s for writing" % logfile
# Setup ccdproc options
ccdproc=iraf.ccdred.ccdproc
ccdproc.overscan=yes
ccdproc.trim=yes
ccdproc.zerocor=no
ccdproc.interactive=no
ccdproc.function="chebyshev"
ccdproc.order=3
ccdproc.sample="*"
ccdproc.naverage=3
ccdproc.niterate=2
ccdproc.low_reject=3.0
ccdproc.high_reject=3.0
ccdproc.grow=0.0
# Demosaic all the files
allfiles=glob.glob(inpat)
alllist=','.join(allfiles)
iraf.iqmosaic(alllist,outpfx="",biassec="!BIASSEC",
trimsec="!TRIMSEC",joinkeys="CCDSIZE,DETSEC,ROISEC",
splitkeys="AMPSEC,CCDSEC,TRIM,OVERSCAN,CCDPROC",
clobber=yes,verbose=verbose)
iraf.add_wcs(alllist,instrument="p60new",binkey=binkey)
# Set ccdproc params for bias-subtraction only
ccdproc.overscan=no
ccdproc.trim=no
ccdproc.zerocor=yes
# Determine if we need to make calibration files
dobias=no
doflats=no
for image in allfiles:
# Criteria for bias image
biasval,flatval=get_head(image,[biaskey,flatkey])
if re.search(biasre,biasval,re.I):
dobias=yes
reduced[image]=yes
elif re.search(flatre,flatval,re.I):
doflats=yes
reduced[image]=yes
# Calibrations
if (dobias or doflats) and not fonly:
iraf.iqcals(alllist,dobias=dobias,biasproc=no,
biaskey=biaskey,biasre=biasre,biasroot=biasroot,
doflats=doflats,flatproc=yes,flatkey=flatkey,
flatre=flatre,filtkey=filtkey,flatpre=flatpre,
flatscale="mode",statsec="",normflat=yes,
dobpm=no,bpmroot=bpmroot,classkey=binroikey,
classdef=binroidef,mosaic=no,
clobber=clobber,verbose=verbose)
# Delete binroi-specific BPMs and flatfields
oldbpms=glob.glob(bpmroot+'-'+'*.pl')
for oldbpm in oldbpms:
iraf.imdel(oldbpm,verify=no,go_ahead=yes)
oldflats=glob.glob(flatpre+'*-*.fits')
for oldflat in oldflats:
iraf.imdel(oldflat,verify=no,go_ahead=yes)
# Clip all flatfields and update BPM as we go
bpm0name=bpm0root+'.pl'
bpmname=bpmroot+'.pl'
if os.path.exists(bpmname):
check_exist(bpm0name,'w',clobber=yes)
iraf.imrename(bpmname,bpm0name,verbose=no)
allflats=glob.glob(flatpre+'*.fits')
for flatimg in allflats:
iraf.iqclip(flatimg,lthresh=0.75,hthresh=1.25,bookend=no,
replace=1.0,maskin=bpm0name,maskout=bpmname,
maskval=1,clobber=yes,verbose=no)
iraf.imdel(bpm0name,verify=no,go_ahead=yes)
iraf.imrename(bpmname,bpm0name,verbose=no)
update_head(flatimg,"CLIPFLAT",1,"Has flatfield been clipped?")
update_head(flatimg,"CLIPREPL",1.0,
"Replacement value for clipped pixels")
iraf.imrename(bpm0name,bpmname,verbose=no)
##################################################
# Lists of focusfiles
allfocus=[]
focusfiles=[]
# Should additional calibrations be taken later on...
biasfiles=[]
flatfiles=[]
# Big Loop
done=no
while not done:
# Parse inputs
allfiles=glob.glob(inpat)
newfiles=[]
for image in allfiles:
if not reduced.has_key(image):
# Exclude Bias & Flats
if re.search(biasroot,image,re.I) or \
re.search(flatpre,image,re.I):
reduced[image]=yes
continue
# Exclude calibration files (for now)
biasval,flatval=get_head(image,[biaskey,flatkey])
if re.search(biasre,biasval,re.I):
biasfiles.append(image)
reduced[image]=yes
elif re.search(flatre,flatval,re.I):
flatfiles.append(image)
reduced[image]=yes
# Exclude "skippable" files
skipval=get_head(image,skipkey)
if re.search(skipre,skipval,re.I):
reduced[image]=yes
# Queue file for processing
else:
reduced[image]=no
newfiles.append(image)
elif not reduced[image]:
# Attempted reduction before, but failed?
newfiles.append(image)
if newfiles:
# Brief pause to avoid file conflicts (?)
time.sleep(3.0)
# Demosaic
newlist=','.join(newfiles)
ccdproc.overscan=yes
ccdproc.trim=yes
ccdproc.zerocor=no
iraf.iqmosaic(newlist,outpfx="",biassec="!BIASSEC",
trimsec="!TRIMSEC",joinkeys="DETSEC",
splitkeys="AMPSEC,CCDSEC,TRIM,OVERSCAN",
clobber=yes,verbose=verbose)
iraf.add_wcs(newlist,instrument="p60new",binkey=binkey)
# Reset ccdproc params for bias-subtraction only
ccdproc.overscan=no
ccdproc.biassec=""
ccdproc.trim=no
ccdproc.trimsec=""
ccdproc.zerocor=yes
ccdproc.flatcor=no
##############################
# Process new images
for image in newfiles:
# Required keyword check
if not check_head(image,reqkeys):
print "Image %s is missing required keywords" % image
continue
# Attempt processing only once per image
reduced[image]=yes
if verbose:
print "Reducing new image %s" % image
# Track useful status information
scistat={}
# Prepare for processing
pix=rawpix
binroi=binroidef
biasname=biasroot+'.fits'
bpmname=bpmroot+'.pl'
filt=get_head(image,filtkey)
flatname=flatpre+filt+".fits"
# Calibration files for BINROI setting
if check_head(image,binroikey):
binroi=get_head(image,binroikey)
if binroi != binroidef:
# Get the right calibration files
fullbias=biasname
biasname=biasroot+'-'+binroi+'.fits'
fullbpm =bpmroot+'.pl'
bpmname =bpmroot+'-'+binroi+'.pl'
fullflat=flatname
flatname=flatpre+filt+'-'+binroi+".fits"
# Details of this binning/roi combo
[binval,detsec]=get_head(image,[binkey,detseckey])
binning=p60parsebin(binval)
roi=p60parseroi(detsec)
# Pixel scale (no anisotropic binning, sorry)
pix=binning[0]*rawpix
# Check for existance of bias; make it if we have to
if not os.path.exists(biasname):
print ("Failed to find bias for %s=%s; "+ \
"constructing one from full-frame bias") % \
(binroikey,binroi)
p60roiflat(fullbias,biasname,binning,roi)
# Produce the flatfield if necessary
if not os.path.exists(flatname):
p60roiflat(fullflat,flatname,binning,roi)
# Produce the bad-pixel mask if necessary
if not os.path.exists(bpmname):
# Get BPM into FITS format
bpm1=iraf.mktemp("iqbpm")+".fits"
iraf.imcopy(fullbpm,bpm1,verbose=no)
# Adjust BPM for binning/ROI
bpm2=iraf.mktemp("iqbpm")+".fits"
p60roibpm(bpm1,bpm2,binning,roi)
# Convert BPM to PL format
iraf.imcopy(bpm2,bpmname,verbose=no)
# Delete temporary files
iraf.imdel(bpm1,verify=no,go_ahead=yes)
iraf.imdel(bpm2,verify=no,go_ahead=yes)
# Bias subtraction
check_exist(biasname,"r")
ccdproc.zero=biasname
image1=biaspfx+image
check_exist(image1,'w',clobber)
iraf.ccdproc(image,output=image1,Stdout=1)
# Flatfielding
check_exist(flatname,"r")
iraf.iqflatten(image1,flatname,outpfx=flatpfx,
normflat=no,clipflat=no,
statsec=statsec,subsky=yes,
vignflat=no,clobber=yes,verbose=no)
imagef=flatpfx+image1
# Clip (bookend) flattened & sky-subtracted image
skybkg=float(get_head(image1,'SKYBKG'))
iraf.iqclip(image1,lthresh=minraw-skybkg,hthresh=maxraw-skybkg,
bookend=yes,maskin="",maskout="",clobber=yes,
verbose=no)
# Rename the file, if everything is well-behaved
reim=re.search('^%s%s(.+[^r])r?\.fits' % (flatpfx,biaspfx),
imagef,re.I)
if reim:
image2=reim.group(1)+'p.fits'
check_exist(image2,"w",yes)
iraf.imrename(imagef,image2,verbose=verbose)
else:
image2=imagef
# Set bad pixels to zero
update_head(image2,'BPM',bpmname,"Bad Pixel Mask")
iraf.iqmask(image2,mask='!BPM',method='constant',value='0.0',
clobber=yes,verbose=no)
# Defringing happens later, in bunches
# Cosmic Ray Rejection
tmpimg=iraf.mktemp("iqcr")+".fits"
check_exist("%s_%s.fits" % (image2[:15],crsfx),"w",clobber)
iraf.lacos_im(image2,tmpimg,"%s_%s.fits" % (image2[:15],crsfx),
gain=gain,readn=readn,skyval=skybkg,sigclip=4.5,
sigfrac=0.5,objlim=1.0,niter=1)
iraf.imdel(image2,verify=no,go_ahead=yes)
iraf.imcopy(tmpimg,image2,verbose=no)
iraf.imdel(tmpimg,verify=no,go_ahead=yes)
# Object-detection
if scistat.has_key("SEEING_%s" % file):
try:
fwhm=float(scistat["SEEING_%s" % file])
except:
fwhm=fwhm0
else:
fwhm=fwhm0
iraf.iqobjs(image2,sigma,satval,skyval="!SKYBKG",masksfx=masksfx,
wtimage="",fwhm=fwhm,pix=rawpix,aperture=2*fwhm/rawpix,
gain=gain,minlim=no,clobber=yes,verbose=no)
# Track some status information
nstars=get_head(image2,'NSTARS')
scistat['NSTARS']=nstars
if minskys.has_key(filt):
minsky=minskys[filt]
scistat["SKYBKG_%s" % filt]=skybkg/minsky
# Is it a focus frame?
focusval=get_head(image2,focuskey)
# Focus frame actions
if re.search(focusre,focusval,re.I):
focusfiles.append(image2)
# Find its place in the sequence
focusseq=get_head(image2,fseqkey)
ref=re.search("(\d+)of(\d+)",focusseq,re.I)
if ref:
ifocus=int(ref.group(1))
nfocus=int(ref.group(2))
# Non-focus frame actions
else:
# Make a new keyword for mosaic objects
objkey='OBJECT'
if check_head(image2,'MSCSIZE'):
mscsize=get_head(image2,'MSCSIZE')
mscdims=mscsize.split(',')
if int(mscdims[0])>1 or int(mscdims[1])>1:
object=get_head(image2,'OBJECT')
mscposn=get_head(image2,'MSCPOSN')
objmsc=object+'-'+mscposn
update_head(image2,'OBJMSC',objmsc,
'Object Name/Mosaic Position')
objkey='OBJMSC'
# Refine WCS (also calculates seeing in arcsec)
iraf.iqwcs(image2,objkey=objkey,rakey='RA',
deckey='DEC',pixtol=0.05,starfile='!STARFILE',
nstar=40,catalog='web',ubhost="localhost",
diffuse=yes,clobber=yes,verbose=verbose,
nstarmax=40)
# Retry if unsuccessful
if not check_head(image2,'IQWCS'):
iraf.iqwcs(image2,objkey=objkey,rakey='RA',
deckey='DEC',pixtol=0.05,starfile="!STARFILE",
nstar=40,catalog='web',ubhost="localhost",
diffuse=yes,clobber=yes,verbose=verbose,
nstarmax=1000)
# Calculate further quantities if WCS was successful
extinct='INDEF'
if check_head(image2,'IQWCS'):
scistat['WCSGOOD'] = 1
scistat["SEEING_%s" % filt]=get_head(image2,'SEEING')
[rapt,dcpt,lenx,leny]=get_head(image2,
['RA','DEC','NAXIS1','NAXIS2'])
[[ractr,dcctr]]=impix2wcs(image2,0.5*(1+float(lenx)),
0.5*(1+float(leny)))
coopt=astrocoords(rapt,dcpt)
cooctr=astrocoords(ractr,dcctr)
[offra,offdc]=radecdiff(coopt.radeg(),coopt.dcdeg(),
cooctr.radeg(),cooctr.dcdeg())
update_head(image2,'PTOFFSET',"%.2f %.2f" % \
(offra,offdc),
"Offset from nominal pointing (arcsec)")
scistat['PTOFFSET']="%.2f,%.2f" % (offra,offdc)
# Get zero-point against SDSS/NOMAD/USNO-B
if catmags.has_key(filt):
if check_head(image2,'OBJMSC'):
object=get_head(image2,'OBJMSC')
else:
object=get_head(image2,'OBJECT')
zpcat=""
# See if attempt already made for SDSS
if not (os.path.exists("%s.sdss" % object)):
getsdss(image2, "%s.sdss" % object)
# If successful, set to SDSS
sdssfile=open("%s.sdss" % object)
if (len(sdssfile.readlines())>2):
catalog="%s.sdss" % object
zpcat="SDSS"
# If SDSS unsuccessful, update USNO-B
if not zpcat:
update_usnob("%s.cat" % object,
outfile="%s.reg" % object)
catalog="%s.reg" % object
zpcat="USNO-B"
# Update image header
update_head(image2,"ZPCAT",zpcat)
# Run the zeropoint calculation
iraf.iqzeropt(image2,catmags[filt],
starfile="!STARFILE",catalog=catalog,
pixtol=3.0,useflags=yes,maxnum=50,
method="mean",rejout=1,fencelim=0.50,
sigma=1.5,maxfrac=0.25,
zptkey=zptkey,zpukey=zpukey,
clobber=yes,verbose=verbose)
# Use that to get an extinction estimate
zeropt,zeroptu,exptime=get_head(image2,[zptkey,
zpukey,exptmkey])
zeropt=float(zeropt)
zeroptu=float(zeroptu)
exptime=float(exptime)
scistat["ZEROPT_%s" % filt] = \
zeropt - 2.5*math.log10(exptime/exptmstd)
if minzpts.has_key(filt):
extinct = zeropt - \
2.5*math.log10(exptime/exptmstd) - \
minzpts[filt]
scistat["EXTINCT_%s" % filt]=extinct
# Fix the SEEING keyword if WCS fit was not successful
else:
scistat['WCSGOOD'] = 0
try:
seepix=float(get_head(image2,'SEEPIX'))
seeing= "%.3f" % (pix*seepix)
except:
seeing='INDEF'
update_head(image2,'SEEING',seeing,
"Estimated seeing in arcsec or INDEF")
# Write Archive Keywords
update_head(image2,'PROCESSD',1,
"Image has been processed by iqp60")
update_head(image2,'PROCVER',version,
"Version number of iqp60 used")
update_head(image2,'EXTINCT',extinct,
"Estimated mags extinction or INDEF")
update_head(image2,'PROCPROB','None',
"Problems encountered in iqp60 processing")
# Update status file
p60status(statfile,image2,scistat)
# Clean up
check_exist(image1,"w",yes)
##############################
# If a set of focus images have been processed, find best focus
if len(focusfiles)>=nfocus:
print "%d focus images have been processed" % len(focusfiles)
# Pick a starfile
focusfiles.sort()
medfile=focusfiles[len(focusfiles)/2]
starfile=get_head(medfile,'STARFILE')
# Update seeing values in all images based on good
# stars in the starfile.
ffiles=','.join(focusfiles)
iraf.iqseeing(ffiles,stars=starfile,
seekey="SEEING",method="acorr",useflags=yes,
skipstars=0,usestars=10,boxsize=64,
strictbox=no,imgsec="[940:1980,64:1980]",
update=yes,clobber=yes,verbose=no)
# Grab focus positions and seeing values
focx=[]
focy=[]
for image in focusfiles:
[fpos,seep]=list2float(get_head(image,['FOCUSPOS','SEEING']))
fpos=float(fpos)
seep=float(seep)
focx.append(fpos)
pix=rawpix
if check_head(image,binkey):
binval=get_head(image,binkey)
binels=binval.split()
binning=[int(binels[0]),int(binels[1])]
# Let's hope we don't ever do asymmetric binning
pix=binning[0]*rawpix
focy.append(pix*seep)
update_head(image,'SEEING',pix*seep,
"Estimated seeing in arcsec")
# Choose best-seeing as minimum of these
bestsee=min(focy)
bestfoc=focx[focy.index(bestsee)]
# This was too much verbosity for Brad
if verbose and 0:
print "Focus settings: "+liststr(focx,"%.2f")
print "Seeing values: "+liststr(focy,"%.1f")
print
print "Best focus: %.2f" % bestfoc
# Expunge the focusfiles list
allfocus.append(focusfiles)
focusfiles=[]
##############################
# Test end conditions
if endhow=="never":
done=no
elif endhow=="once":
done=yes
elif endhow=="time":
if time.time()>reftime:
done=yes
# Wait a little while
if not done:
time.sleep(twait)
sys.exit()
logging.info('(1) Flat-fielding: {}'.format(flat))
#geometric distortion correction
geom_out = filename[:-5] + 'fg.fits'
try:
iraf.distcor(flat_out, geom_out, geomap) #should save header
except Exception as e:
print(e)
logging.warning(e)
sys.exit()
logging.info('(2) Distortion correction:\n{0} & {1}'.format(
distcorr_file, geomap))
#cropping
a1 = geom_out[:-5] + ch1
b1 = filename[:-5] + 'fg_ch1'
a2 = geom_out[:-5] + ch2
b2 = filename[:-5] + 'fg_ch2'
try:
iraf.imcopy(a1, b1) #should save header
iraf.imcopy(a2, b2) #should save header
except Exception as e:
print(e)
logging.warning(e)
sys.exit()
logging.info('(3) Cropped into o ({0}) and e channgels ({1})'.format(
ch1, ch2))
#remove intermediate files
#iraf.imdel(dark_out)
iraf.imdel(flat_out)
iraf.imdel(geom_out)
def subract(df_image,shifts=99.99,width_cfhtsm=0.,upperlim=0.04,medphotosc=True,numsources=100,sigmaclip=False,cutout=False):
print "\n************ Running the subtraction steps ************\n"
#iraf.imdel('_psf*.fits')
#iraf.imdel('_model*.fits')
#iraf.imdel('_df_sub')
iraf.imdel('_res*.fits')
iraf.imdel('cc_images')
iraf.imdel('_model_sh')
iraf.imdel('_model_sc')
iraf.imdel('_model_mask')
iraf.imdel('_model_maskb')
'shift the images so they have the same physical coordinates'
if shifts[0]==99.99:
print('\n**** Calculating shifts... ****\n')
iraf.stsdas.analysis.dither.crossdriz('_df_sub.fits','_model.fits','cc_images',dinp='no',dref='no')
iraf.stsdas.analysis.dither.shiftfind('cc_images.fits','shift_values')
x_shift=0
y_shift=0
with open('shift_values','r') as datafile:
line = datafile.read().split()
x_shift = float(line[2])
y_shift = float(line[4])
print('Shifting model image by x and y: '+str(x_shift)+','+str(y_shift))
iraf.imshift('_model','_model_sh',0-x_shift,0-y_shift)
else:
print('Shifting model image by x and y: '+str(shifts[0])+','+str(shifts[1]))
iraf.imshift('_model','_model_sh',shifts[0],shifts[1])
'scale the model so that roughly the same as the _df_sub image'
# if usemodelpsf:
# iraf.imarith('_model_sh','/',16.,'_model_sc')
# else:
'the photometric step, matching the images to each other. '
'in principle this comes from the headers - both datasets are calibrated,'
'so this multiplication should be something like 10^((ZP_DF - ZP_CFHT)/-2.5)'
'(perhaps with a correction for the difference in pixel size - depending'
'on what wregister does - so another factor (PIX_SIZE_DF)^2/(PIX_SIZE_CFHT)^2'
avgphotosc,medianphotosc = getphotosc('_model_sh.fits','_df_sub.fits',numsources=numsources,sigmaclip=sigmaclip,cutout=cutout)
if medphotosc:
photosc = medianphotosc
print 'Using median calculated photosc: %s'%photosc
else:
photosc = avgphotosc
print 'Using average calculated photosc: %s'%photosc
iraf.imarith('_model_sh','*',photosc,'_model_sc')
iraf.imdel('_df_ga.fits')
'correction for the smoothing that was applied to the CFHT'
if width_cfhtsm != 0:
iraf.gauss('_df_sub','_df_ga',width_cfhtsm) # iraf.gauss('%s'%df_image,'_df_ga',width_cfhtsm)
else:
print 'WARNING: No smoothing applied to the dragonfly image before subtracting the model.'
iraf.imcopy('_df_sub.fits','_df_ga.fits')
'subtract the model from the dragonfly cutout'
iraf.imarith('_df_ga','-','_model_sc','_res')
'duplicate the residual so that we can redo the masking step later'
iraf.imcopy('_res.fits','_res_org.fits')
'mask the image to get block out stuff that didnt subtract nicely'
mask('_res',upperlim=upperlim)
return None
def combineARCSpectra(args):
import sys
import glob
import os
from pyraf import iraf
from shutil import copyfile
from operator import itemgetter
import pandas as pd
# Setup relavent IRAF tasks...
iraf.image()
iraf.imutil()
iraf.immatch()
iraf.onedspec()
# Grab list of file names from argument list
# and convert it into a python list...
#fnameList = 'WDC0442-0536.0086b.ms.fits,WDC0442-0536.0087b.ms.fits,WDC0442-0536.0088b.ms.fits'
fnameList = args.fileList
fnameList = fnameList.split(',')
# Clean up file name list to exclude any excess blank spaces...
fileNameList = []
for fname in fnameList:
fileName = fname.strip()
fileNameList.append(fileName)
# Start a new file name list, for the imsliced
# files to be imcombined...
newFileNameList = []
# Loop through the input file name list...
for fileName in fileNameList:
print fileName
# Create an output file name...
baseFileName = os.path.splitext(fileName)[0] + '.tmp'
outputFileName = baseFileName + '.fits'
# Delete the any temporary files called tempA00?.fits,
# and then run imslice on fileName...
iraf.imdel('tempA00?.fits')
iraf.imslice(fileName, 'tempA', 2)
# Delete the any temporary files called tempB00?.fits,
# and then run imslice on tempA001.fits (an output
# from the first imslice)...
iraf.imdel('tempB00?.fits')
iraf.imslice('tempA001.fits', 'tempB', 2)
# Rename the output file tempB001.fits from the second
# imslice procedure; tempB001.fits contains the 1D
# version of the target spectrum. (The other file,
# tempB002.fits, which we ignore, contains the 1D
# version of the sky background spectrum.)
os.rename('tempB001.fits', outputFileName)
# Append the output file name to the new file name list...
newFileNameList.append(outputFileName)
# Clean up unneeded temporary files...
iraf.imdel('tempA00?.fits')
iraf.imdel('tempB00?.fits')
# Print the list of file names that will be fed to imcombine...
print newFileNameList
# Here, we convert the python list into an IRAF list...
inlist = ''
for newFileName in newFileNameList:
inlist = inlist + ',' + newFileName
inlist = inlist[1:]
print inlist
# Run imcombine to create a median-combined spectrum
# and its sigma image...
medianFile = newFileNameList[0].split('.')[0] + '.median.b.ms.fits'
sigmaFile = 'sigma-' + newFileNameList[0].split(
'.')[0] + '.median.b.ms.fits'
iraf.imdel(medianFile)
iraf.imdel(sigmaFile)
iraf.imcombine(inlist,
medianFile,
combine='median',
scale='median',
sigma=sigmaFile)
# Create ASCII text file equivalents...
medianFileTxt = newFileNameList[0].split('.')[0] + '.median.b.ms.txt'
sigmaFileTxt = 'sigma-' + newFileNameList[0].split(
'.')[0] + '.median.b.ms.txt'
iraf.onedspec.wspectext(medianFile, medianFileTxt)
iraf.onedspec.wspectext(sigmaFile, sigmaFileTxt)
# Combine median and sigma text files...
df1 = pd.read_csv(medianFileTxt,
header=None,
names=['wave', 'flux'],
delim_whitespace=True)
df2 = pd.read_csv(sigmaFileTxt,
header=None,
names=['wave', 'flux_err'],
delim_whitespace=True)
df12 = df1.merge(df2, on='wave')
outputFile = newFileNameList[0].split('.')[0] + '.median.flm'
df12.to_csv(outputFile, index=False, sep=' ')
# Run imcombine to create a mean-combined spectrum
# and its sigma image...
meanFile = newFileNameList[0].split('.')[0] + '.mean.b.ms.fits'
sigmaFile = 'sigma-' + newFileNameList[0].split('.')[0] + '.mean.b.ms.fits'
iraf.imdel(meanFile)
iraf.imdel(sigmaFile)
iraf.imcombine(inlist,
meanFile,
combine='average',
scale='median',
sigma=sigmaFile)
# Create ASCII text file equivalents...
meanFileTxt = newFileNameList[0].split('.')[0] + '.mean.b.ms.txt'
sigmaFileTxt = 'sigma-' + newFileNameList[0].split(
'.')[0] + '.mean.b.ms.txt'
iraf.onedspec.wspectext(meanFile, meanFileTxt)
iraf.onedspec.wspectext(sigmaFile, sigmaFileTxt)
# Combine mean and sigma text files...
df1 = pd.read_csv(meanFileTxt,
header=None,
names=['wave', 'flux'],
delim_whitespace=True)
df2 = pd.read_csv(sigmaFileTxt,
header=None,
names=['wave', 'flux_err'],
delim_whitespace=True)
df12 = df1.merge(df2, on='wave')
outputFile = newFileNameList[0].split('.')[0] + '.mean.flm'
df12.to_csv(outputFile, index=False, sep=' ')
return 0
biasFlags = {
'logfile':'biasLog.txt','rawpath':'./raw/','fl_vardq':'yes',
'verbose':'no'
}
# The following SQL generates the list of files to process.
SQL = fs.createQuery('bias', qd)
biasFiles = fs.fileListQuery(dbFile, SQL, qd)
# The str.join() function is needed to transform a python list into a string
# filelist that IRAF can understand.
if len(biasFiles) > 1:
gmos.gbias(','.join(str(x) for x in biasFiles), 'MCbias.fits',
**biasFlags)
# Clean up
iraf.imdel('gS2006*.fits')
print (" --Creating Twilight Imaging Flat-Field MasterCal--")
# Select flats obtained contemporaneously with the observations.
qd.update({'DateObs':'2006-09-10:2006-10-10'})
# Set the task parameters.
gmos.giflat.unlearn()
flatFlags = {
'fl_scale':'yes','sctype':'mean','fl_vardq':'yes',
'rawpath':'./raw/','logfile':'giflatLog.txt','verbose':'yes'
}
filters = ['Ha', 'HaC', 'SII', 'r', 'i']
for f in filters:
print " Building twilight flat MasterCal for: %s" % (f)
def subract(df_image, psf):
iraf.imdel('_model*.fits')
iraf.imdel('_res*.fits')
iraf.imdel('_df_sub')
'subtract the sky value from the dragonfly image header'
df_backval = fits.getheader(df_image)['BACKVAL']
iraf.imarith('%s' % df_image, '-', '%s' % df_backval, '_df_sub')
'convolve the model with the Dragonfly PSF'
if usemodelpsf:
makeallisonspsf()
psf = './psf/psf_static_fullframe.fits'
### XXX resample the PSF by a factor of 4
if verbose:
print 'VERBOSE: Using %s for the psf convolution.' % psf
iraf.stsdas.analysis.fourier.fconvolve('_fluxmod_dragonfly', '%s' % psf,
'_model')
'shift the images so they have the same physical coordinates'
iraf.stsdas.analysis.dither.crossdriz('_df_sub.fits',
'_model.fits',
'cc_images',
dinp='no',
dref='no')
iraf.stsdas.analysis.dither.shiftfind('cc_images.fits', 'shift_values')
x_shift = 0
y_shift = 0
with open('shift_values', 'r') as datafile:
line = datafile.read().split()
x_shift = float(line[2])
y_shift = float(line[4])
print('The shift in x and y are: ' + str(x_shift) + ',' + str(y_shift))
iraf.imshift('_model', '_model_sh', 0 - x_shift, 0 - y_shift)
'scale the model so that roughly the same as the _df_sub image'
if usemodelpsf:
iraf.imarith(
'_model_sh', '/', 16., '_model_sc'
) ## just trying to match it to the original (below) approximately
else:
iraf.imarith(
'_model_sh', '/', 2422535.2, '_model_sc'
) ## difference between the flux of the star used to make the psf in both frames
iraf.imarith('_model_sc', '*', 1.5, '_model_sc')
'subtract the model frm the dragonfly cutout'
iraf.imarith('_df_sub', '-', '_model_sc', '_res')
'????'
iraf.imcopy('_model_sc', '_model_mask')
iraf.imreplace('_model_mask.fits', 0, upper=50)
iraf.imreplace('_model_mask.fits', 1, lower=0.01)
iraf.boxcar('_model_mask', '_model_maskb', 5, 5)
iraf.imreplace('_model_maskb.fits', 1, lower=0.1)
iraf.imreplace('_model_maskb.fits', 0, upper=0.9)
iraf.imarith(1, '-', '_model_maskb', '_model_maskb')
iraf.imarith('_model_maskb', '*', '_res', '_res_final')
return None
def rectify_image(image,
reference,
output,
refout=None,
fitgeometry='general',
function='polynomial',
xxorder=3,
yyorder=3,
xyorder=3,
yxorder=3,
clobber=1,
objects=None,
interactive=1,
maxiter=5,
reject=3,
xrange=None,
yrange=None,
thresh=2.0,
scale=0.125,
fwhm=4,
image_sigma=None,
reference_sigma=None,
debug=0,
aoffset=0):
'''This task will take an image and a reference image and apply geomap/
geotran to aligned them. image is registered and output to output.
You can specify the fit geometry (shift, xyscale, rotate, rscale,
rxyscale, general). You can also choose the geenral function
(legendre, chebyshev, polynomial). You can specfy the order of all
the terms (xxorder, yyorder, xyorder, yxorder). If you choose
clobber=1 (the default), new images overwrite older ones. Lastly, if
you want to use a pre-existing list of objects, specify them as the
objects keyword.'''
# First, if we already have objects to use, save time by not finding them
# again!
if objects is None:
sex(image,
output='image.cat',
thresh=thresh,
scale=scale,
fwhm=fwhm,
sigma_map=image_sigma)
sex(reference,
output='reference.cat',
thresh=thresh,
scale=scale,
fwhm=fwhm,
sigma_map=reference_sigma)
im_masks1 = get_mask(image)
im_masks2 = get_mask(reference)
if aoffset == 0:
# Check the ROTANG header field to see if we need to give a rotation
# offset to the object matcher.
rotang1 = get_header(reference, "ROTANG", float)
rotang2 = get_header(image, "ROTANG", float)
if rotang1 is not None and rotang2 is not None:
if abs(rotang1 - rotang2) > 0.1:
aoffset = rotang1 - rotang2
# Run the matchum routine to get corresponding objects:
(x2, y2, x1, y1, o1, o2) = matchum('image.cat',
'reference.cat',
im_masks1=im_masks1,
im_masks2=im_masks2,
xrange=xrange,
yrange=yrange,
debug=debug,
aoffset=aoffset)
f = open('geomap.objects', 'w')
for i in range(len(x1)):
f.write('%.2f %.2f %.2f %.2f\n' % (x1[i], y1[i], x2[i], y2[i]))
f.close()
objects = "geomap.objects"
else:
data = pygplot.columns(objects)
x1 = data[0]
y1 = data[1]
x2 = data[2]
y2 = data[3]
xshift = int(stats.bwt(x2 - x1)[0])
yshift = int(stats.bwt(y2 - y1)[0])
# We now have some selected objects to use with geomap. Let's figure out
# the area of validity for the data using ths shifts computed above.
(r_xmax, r_ymax) = get_image_size(reference)
if xshift < 0:
xmin = -xshift
xmax = r_xmax
else:
xmin = 1
xmax = r_xmax - xshift
if yshift < 0:
ymin = -yshift
ymax = r_ymax
else:
ymin = 1
ymax = r_ymax - yshift
#xmin = min(x1); xmax = max(x1)
#ymin = min(y1); ymax = max(y1)
print xshift, yshift, xmin, xmax, ymin, ymax
iraf.images()
if os.path.isfile('geomap.db'): os.unlink('geomap.db')
# We may not have enough points to do a general fit, so we do them in order of
# complexity. Hopefully, we have at least ONE point (for the shift)
fitgeometries = [fitgeometry, 'rxyscale', 'rotate', 'shift']
success = 0
for fitgeo in fitgeometries:
print "using fitgeometry = ", fitgeo
res = iraf.geomap(input=objects,
database='geomap.db',
xmin=1,
ymin=1,
xmax=r_xmax,
ymax=r_ymax,
fitgeometry=fitgeo,
function=function,
xxorder=xxorder,
yyorder=yyorder,
xyorder=xyorder,
yxorder=yxorder,
results='',
transforms=objects,
interactive=interactive,
maxiter=maxiter,
reject=reject,
Stdout=1)
# assume all went well
success = 1
for line in res:
print line
if re.search('Too few points', line):
success = 0
if success:
break
if clobber and os.path.isfile(output): iraf.imdel(output)
iraf.geotran(input=image,
output='temp.rectify',
database='geomap.db',
transforms=objects,
geometry='geometric',
interpolant="linear")
section = "[%d:%d,%d:%d]" % (xmin, xmax, ymin, ymax)
print 'Valid section of data:', section
iraf.imcopy(input='temp.rectify', output=output)
iraf.imdel('temp.rectify')
return (xmin, xmax, ymin, ymax)
def wirc_flatscale(infiles,inflat,outflat,clipfrac=0.03,
clobber=globclob):
# "infiles" is a list of filenames
inlist=','.join(infiles)
bpmfile=get_head(infiles[0],'BPM')
# Make a median combination of input files
imcmb1=iraf.mktemp("iqwircc")+".fits"
iraf.imcombine(inlist,imcmb1,combine="median",reject="none",
project=no,outtype="real",outlimits="",offsets="",
masktype="none",blank=0.0,scale="!SKYBKG",zero="none",
weight="none",statsec="",lthreshold="INDEF",
hthreshold="INDEF",nkeep=1)
[naxis1,naxis2]=get_head(imcmb1,['NAXIS1','NAXIS2'])
npixall=float(naxis1)*float(naxis2)
# Calculate sky background & divide, subtract 1
skybkg=wirc_sky(imcmb1,bpmfile)
imcmb2=iraf.mktemp("iqwircc")+".fits"
iraf.imarith(imcmb1,'/',skybkg,imcmb2,verbose=no,noact=no)
iraf.imdel(imcmb1,verify=no,go_ahead=yes)
iraf.imarith(imcmb2,'-',1.0,imcmb1,verbose=no,noact=no)
# Surface fit to median image
imsurf1=iraf.mktemp("iqwircs")+".fits"
iraf.imsurfit(imcmb1,imsurf1,xorder=6,yorder=6,type_output="fit",
function="chebyshev",cross_terms=yes,
xmedian=21,ymedian=21,median_percent=50.0,
lower=0.0,upper=0.0,ngrow=0,niter=0,
regions="all",rows="*",columns="*",border=50,
sections="",circle="",div_min="INDEF")
# Corresponding bad pixel mask
imbpm1=iraf.mktemp("iqwircb")+".pl"
iraf.imexpr("abs(x)>%.3f ? 0 : 1" % clipfrac,imbpm1,x=imsurf1)
# Subtract 1.0 from flatfield
imflat1=iraf.mktemp("iqwircf")+".fits"
iraf.imarith(inflat,'-',1.0,imflat1,verbose=no,noact=no)
# Surface fit to the flatfield
imsurf2=iraf.mktemp("iqwircs")+".fits"
iraf.imsurfit(imflat1,imsurf2,xorder=6,yorder=6,type_output="fit",
function="chebyshev",cross_terms=yes,
xmedian=21,ymedian=21,median_percent=50.0,
lower=0.0,upper=0.0,ngrow=0,niter=0,
regions="all",rows="*",columns="*",border=50,
sections="",circle="",div_min="INDEF")
# Corresponding bad pixel mask
imbpm2=iraf.mktemp("iqwircb")+".pl"
iraf.imexpr("abs(x)>%.3f ? 0 : 1" % clipfrac,imbpm2,x=imsurf2)
# Combine bad pixel masks for median + flat
imbpm3=iraf.mktemp("iqwircb")+".pl"
iraf.imexpr("(x>0 || y>0) ? 1 : 0",imbpm3,x=imbpm1,y=imbpm2)
# Calculate the ratio image
imratio=iraf.mktemp("iqwircr")+".fits"
iraf.imexpr("z>0 ? 0 : x/y",imratio,x=imsurf1,y=imsurf2,z=imbpm3)
# Mimstat on the ratio image
mimstat=iraf.mimstatistics(imratio,imasks=imbpm3,omasks="",
fields='image,npix,mean,stddev,min,max,mode',
lower='INDEF',upper='INDEF',nclip=4,
lsigma=5.0,usigma=5.0,binwidth=0.1,
format=no,Stdout=1,Stderr=1)
mimels=mimstat[0].split()
npix=float(mimels[1])
xmult=float(mimels[2])
# Check that a reasonable number of pixels have made the grade
check_exist(outflat,'w',clobber=clobber)
if npix<0.05*npixall:
print "Less than 5% of pixels passed the cut... preserving flatfield"
iraf.imcopy(inflat,outflat,verbose=no)
xmult=1.0
else:
# Create the final flatfield image
iraf.imexpr("x*%.3f + 1" % xmult,outflat,x=imflat1)
# Update header keywords
update_head(outflat,'RESCALE',1,"Flatfield has been rescaled")
update_head(outflat,'ORIGFLAT',inflat,
"Input flatfield name (before rescaling)")
update_head(outflat,'XMULT',xmult,
"Multiplied ORIGFLAT by this factor to rescale")
# Clean up
iraf.imdel(imcmb1,verify=no,go_ahead=yes)
iraf.imdel(imcmb2,verify=no,go_ahead=yes)
iraf.imdel(imsurf1,verify=no,go_ahead=yes)
iraf.imdel(imsurf2,verify=no,go_ahead=yes)
iraf.imdel(imbpm1,verify=no,go_ahead=yes)
iraf.imdel(imbpm2,verify=no,go_ahead=yes)
iraf.imdel(imbpm3,verify=no,go_ahead=yes)
iraf.imdel(imflat1,verify=no,go_ahead=yes)
iraf.imdel(imratio,verify=no,go_ahead=yes)
def gen_ar():
#cargo resultado de f2c
temps, pasos, intensidad = f2c(res_tem="S", step_tem=100)
x = pasos
y = temps
z = intensidad
print(x[z.index(max(z))], y[z.index(max(z))])
os.chdir("../")
# obtengo parametros para para fxcor
obfx = "aB" + str(x[z.index(max(z))])
temp = str(int(y[z.index(max(z))]))
if len(temp) == 4:
temp = "0" + temp
iraf.imdel("ss")
iraf.imdel("tem")
iraf.sarit("T" + temp, "-", 1, "ss")
iraf.sarit("ss", "*", "amortigua", "tem")
fig = plt.figure(figsize=(12, 9.0))
fig.subplots_adjust(hspace=0.4,
bottom=0.06,
top=0.94,
left=0.12,
right=0.94)
#ejecuto y ploteo fxcor
ax2 = plt.subplot2grid((4, 1), (0, 0)) # grafico la func, de correlacion
program_dir = os.getcwd() + "/"
ccf_command = program_dir + "fxcor_cursor_command.txt" #comando para extraeer los datos del grafico
interact = 1
iraf.imdel("new_ccf.txt")
iraf.fxcor(object=obfx,
templates="tem",
cursor=ccf_command,
ccftype="text",
interactive=interact) #fxcor de irad
data = asciitable.read("new_ccf.txt") #archivo de salida del fxcor
ax2.set_xlim(-2000, 2000)
plt.axhline(linewidth=2, color='r')
plt.plot(data["col1"], data["col2"], "-", color="black")
plt.axvspan(-100, 100, facecolor='grey', alpha=0.5)
#plt.axhspan(0.9, 1.00, facecolor='grey', alpha=0.5)
ax2.yaxis.set_major_locator(plt.MaxNLocator(5))
ax2.set_xlabel(r'Velocity', fontsize=18)
ax2.set_ylabel(r'Correlation', fontsize=18)
#datos para scale
iraf.imdel("scale")
iraf.imdel("tplsca")
obscale = "B" + str(x[z.index(max(z))])
temp = "T" + temp
w1 = 5900
w2 = 6490
wi = 300
iraf.imdel("scale.txt")
iraf.imdel("tplsca.txt")
iraf.scale(spec=obscale, tpl=temp, w1=w1, w2=w2, width=wi)
iraf.wspectext("scale.fits", "scale.txt", header="no")
iraf.wspectext("tplsca.fits", "tplsca.txt", header="no")
iraf.wspectext(obscale, "obscale.txt", header="no")
#ploteo resultados de f2c
ax1 = plt.subplot2grid((4, 1), (1, 0), rowspan=3)
# define grid.
xi = np.linspace(min(x), max(x), 1000)
yi = np.linspace(min(y), max(y), 1000)
# grid the data.
zi = griddata((x, y), z, (xi[None, :], yi[:, None]), method='linear')
# contour the gridded data, plotting dots at the randomly spaced data points.
CS = plt.contour(xi, yi, zi, 15, linewidths=0.5, colors='k')
CS = plt.contourf(xi, yi, zi, 15, cmap=plt.cm.jet)
cb = plt.colorbar(orientation='horizontal', pad=0.1,
aspect=30) # draw colorbar
cb.set_label('Intensity', fontsize=18)
# plot data points.
plt.scatter(x, y, marker='o', c='b', s=5)
plt.scatter(x[z.index(max(z))],
y[z.index(max(z))],
marker='o',
color="w",
s=20)
plt.xlabel('Mass ratio')
plt.ylabel('Secondary Temperature (K)')
ax1.set_xlabel(r'Mass ratio', fontsize=18)
ax1.set_ylabel(r'Secondary Temperature (K)', fontsize=18)
ax1.set_xlim(min(pasos), max(pasos))
ax1.set_ylim(min(temps), max(temps))
ax1.yaxis.set_major_locator(plt.MaxNLocator(6))
ax1.xaxis.set_major_locator(plt.MaxNLocator(8))
ax2.text(0.03,
0.96,
"T=%s K" % y[z.index(max(z))],
ha='left',
va='top',
transform=ax2.transAxes,
fontsize=20)
ax2.text(0.04,
0.5,
"q=%s" % x[z.index(max(z))],
ha='left',
va='top',
transform=ax2.transAxes,
fontsize=20)
#plot scale
fig = plt.figure(figsize=(12, 9.0))
fig.subplots_adjust(hspace=0.4,
bottom=0.06,
top=0.94,
left=0.12,
right=0.94)
ax2 = plt.subplot2grid((4, 1), (0, 0))
data = asciitable.read("tplsca.txt") #archivo de salida del scale
#ax2.set_xlim(min(data["col1"]),max(data["col1"]))
plt.plot(data["col1"], data["col2"], "-", color="red")
data = asciitable.read("obscale.txt")
plt.plot(data["col1"], data["col2"], "-", color="black")
ax2.yaxis.set_major_locator(plt.MaxNLocator(5))
ax2.set_xlabel(r'Wavelength ($\AA$)', fontsize=18)
ax2.set_ylabel(r'Intensity', fontsize=18)
data = asciitable.read("scale.txt") #archivo de salida del scale
ax2.set_xlim(min(data["col1"]), max(data["col1"]) - 1)
plt.plot(data["col1"], data["col2"] * 100, "-", color="black")
ax2.yaxis.set_major_locator(plt.MaxNLocator(5))
ax2.set_xlabel(r'Wavelength ($\AA$)', fontsize=18)
ax2.set_ylabel(r'$I_2/I_T$ (%)', fontsize=18)
#plt.savefig('mass_temp.pdf',format="pdf", bbox_inches='tight',dpi = 300)
plt.show()
iraf.sarit(input1=specf, op='/', input2=atmofile13, output=spect13)
spect14 = re.sub('.fits', '_e14.fits', specf)
iraf.sarit(input1=specf, op='/', input2=atmofile14, output=spect14)
spect15 = re.sub('.fits', '_e15.fits', specf)
iraf.sarit(input1=specf, op='/', input2=atmofile15, output=spect15)
ax = subplot(111)
xl = array([6500, 8000])
listatmo = [
spect05, spect06, spect07, spect08, spect09, spect10, spect11, spect12,
spect13, spect14, spect15
]
i = 0
while i != len(listatmo):
iraf.imdel('tmp.fits', verify='no')
sp = listatmo[i] + "[*,1,1]"
iraf.imcopy(sp, 'tmp.fits', verbose='no')
xx = iraf.listpixel('tmp.fits', wcs='world', Stdout=1)
lam, fl = [], []
for x in xx:
lam.append(float(string.split(x)[0]))
fl.append(float(string.split(x)[1]))
fl2 = []
for z in fl:
fl2.append(float(z))
cr = [
'#ee560b', 'orange', '#e6d114', 'g', 'c', 'k', 'b', 'purple', 'm',
'pink', 'r'
def psf_construct(img,
x,
y,
halosize,
norm=None,
magnify=1,
pixel_scale=0.168,
sub_indiv_bkg=False,
show_figure=False,
verbose=False):
from pyraf import iraf
from iraf import stsdas
from iraf import analysis
from iraf import isophote
from iraf import bmodel
from sep import Background
from .image import save_to_fits, extract_obj, seg_remove_cen_obj
if not os.path.isdir('./temp/'):
os.mkdir('./temp/')
# Delete `fits` file first
iraf.imdel('./temp/_halo*.fits')
x_int = x.astype(np.int)
y_int = y.astype(np.int)
dx = -1.0 * (x - x_int)
dy = -1.0 * (y - y_int)
halosize = int(halosize)
# Note that stars near the boundary will have problems when making cutout. So we pad the image here.
padsize = 40
ny, nx = img.shape
im_padded = np.zeros((ny + 2 * padsize, nx + 2 * padsize))
# Making the left edge empty
im_padded[padsize:ny + padsize, padsize:nx + padsize] = img
halo_i = (im_padded[y_int + padsize - halosize:y_int + padsize + halosize +
1, x_int + padsize - halosize:x_int + padsize +
halosize + 1])
if verbose:
print('### Cuting out the star and masking out contaminations ###')
# Build mask to move out contaminations
psf_raw = halo_i.byteswap().newbyteorder()
from astropy.convolution import convolve, Box2DKernel
psf_blur = convolve(abs(psf_raw), Box2DKernel(1.5))
psf_objects, psf_segmap = extract_obj(abs(psf_blur),
b=5,
f=4,
sigma=5.5,
minarea=2,
pixel_scale=pixel_scale,
deblend_nthresh=32,
deblend_cont=0.0001,
sky_subtract=False,
show_fig=show_figure,
verbose=verbose)
# remove central object
psf_segmap = seg_remove_cen_obj(psf_segmap)
psf_mask = (psf_segmap != 0).astype(float)
save_to_fits(psf_mask, './temp/_halo_mask.fits')
# Here I subtract local sky, so that the final image will not be too negative.
if sub_indiv_bkg is True:
# Evaluate local sky backgroud within `halo_i`
# Actually this should be estimated in larger cutuouts.
# So make another cutout (larger)!
psf_raw = psf_raw.byteswap().newbyteorder()
bk = Background(psf_raw, psf_segmap != 0)
glbbck = bk.globalback
if verbose:
print('# Global background: ', glbbck)
save_to_fits(halo_i - glbbck, './temp/_halo_img.fits') #
else:
save_to_fits(halo_i, './temp/_halo_img.fits') #
# Shift halo_i to integer grid
iraf.imshift('./temp/_halo_img.fits',
'./temp/_halo_img_shift.fits',
dx,
dy,
interp_type='poly3',
boundary='nearest')
iraf.imshift('./temp/_halo_mask.fits',
'./temp/_halo_mask_shift.fits',
dx,
dy,
interp_type='poly3',
boundary='nearest')
if not isinstance(magnify, int):
raise TypeError('# "magnify" must be positive integer less than 10!')
elif magnify != 1 and magnify > 10:
raise TypeError('# "magnify" must be positive integer less than 10!')
elif magnify == 1:
iraf.imcopy('./temp/_halo_img_shift.fits',
'./temp/_halo_i_shift.fits',
verbose=verbose)
iraf.imcopy('./temp/_halo_mask_shift.fits',
'./temp/_halo_m_shift.fits',
verbose=verbose)
else:
iraf.magnify('./temp/_halo_img_shift.fits',
'./temp/_halo_i_shift.fits',
magnify,
magnify,
interpo='poly3',
bound='refl',
const=0.)
iraf.magnify('./temp/_halo_mask_shift.fits',
'./temp/_halo_m_shift.fits',
magnify,
magnify,
interpo='poly3',
bound='refl',
const=0.)
return psf_raw, psf_mask #, x_int, y_int, dx, dy
def gmos_ls_proc2(
sciTargets,
stdTarget,
dbFile='./raw/obsLog.sqlite3',
qd_full={'use_me': 1, 'Instrument': 'GMOS-S', 'CcdBin': '2 4', 'RoI': 'Full', 'Disperser': 'B600+_%', 'CentWave': 485.0, 'AperMask': '1.0arcsec', 'Object': 'AM2306-72%','DateObs': '2007-06-05:2007-07-07'},
qd_censp={'use_me': 1, 'Instrument': 'GMOS-S', 'CcdBin': '2 4', 'RoI': 'CenSp', 'Disperser': 'B600+_%', 'CentWave': 485.0, 'AperMask': '1.0arcsec', 'Object': 'LTT9239','DateObs': '2007-06-05:2007-07-07'},
biasFlags={'logfile': 'biasLog.txt', 'rawpath': './raw/', 'fl_vardq': 'yes', 'verbose': 'no'},
flatFlags = {'fl_over': 'yes', 'fl_trim': 'yes', 'fl_bias': 'yes', 'fl_dark': 'no', 'fl_fixpix': 'no', 'fl_oversize': 'no', 'fl_vardq': 'yes', 'fl_fulldq': 'yes','rawpath': './raw', 'fl_inter': 'no', 'fl_detec': 'yes', 'function': 'spline3', 'order': '13,11,28', 'logfile': 'gsflatLog.txt', 'verbose': 'no'},
sciFlags = {'fl_over': 'yes', 'fl_trim': 'yes', 'fl_bias': 'yes', 'fl_gscrrej': 'no','fl_dark': 'no', 'fl_flat': 'yes', 'fl_gmosaic': 'yes', 'fl_fixpix': 'no', 'fl_gsappwave': 'yes', 'fl_oversize': 'no', 'fl_vardq': 'yes', 'fl_fulldq': 'yes', 'rawpath': './raw', 'fl_inter': 'no', 'logfile': 'gsreduceLog.txt', 'verbose': 'no'},
waveFlags = {'coordlist': 'gmos$data/CuAr_GMOS.dat', 'fwidth': 6, 'nsum': 50, 'function': 'chebyshev', 'order': 5, 'fl_inter': 'no', 'logfile': 'gswaveLog.txt', 'verbose': 'no'},
sciCombFlags = {'combine': 'average', 'reject': 'ccdclip', 'fl_vardq': 'yes', 'fl_dqprop': 'yes', 'logfile': 'gemcombineLog.txt', 'verbose': 'no'},
transFlags={'fl_vardq': 'yes', 'interptype': 'linear', 'fl_flux': 'yes', 'logfile': 'gstransLog.txt'},
skyFlags={'fl_oversize': 'no', 'fl_vardq': 'yes', 'logfile': 'gsskysubLog.txt'},
extrFlags = {'apwidth': 3., 'fl_inter': 'yes', 'find': 'yes','trace': 'yes', 'tfunction': 'chebyshev', 'torder': '6', 'tnsum': 20, 'background': 'fit', 'bfunction': 'chebyshev', 'border': 2, 'fl_vardq': 'no', 'logfile': 'gsextrLog.txt'},
calibFlags = {'extinction': 'onedstds$ctioextinct.dat', 'fl_ext': 'yes', 'fl_scale': 'no','sfunction': 'sens', 'fl_vardq': 'yes', 'logfile': 'gscalibrateLog.txt'},
skip_wavecal=True,
clean_files=False):
"""
Parameters
----------
dbFile : str
Filename containing the SQL sqlite3 database created by obslog.py
It must be placed in the ./raw/ directory
Default is `./raw/obsLog.sqlite3`
sciTargets : dict
Dictionary with the associations of science targets and its associated ARC for wavelength calibration
as well as the regions defining the sky along the slit.
e.g. sciTargetd = {'AM2306-721_a': {'arc': 'gsS20070623S0071', 'sky': '520:720'},
'AM2306-72_b': {'arc': 'gsS20070623S0081', 'sky': '670:760,920:1020'}}
Note that there could be more than one target defined this way.
stdTarget : dict
Dictionary with the associations of standard star targets and its associated ARC for wavelength calibration
as well as the regions defining the sky along the slit.
e.g. stdTarget = {'LTT1788': {'arc': 'S20180711S0281', 'sky': '170:380,920:1080'}}
qd_full : dictionary
Query Dictionary of essential parameter=value pairs for Full RoI. Meant for science object.
qd_censp : dictionary
Query Dictionary of essential parameter=value pairs for CenSp RoI. Meant for standard star.
biasFlags : dict
Dictionary for the keyword flags of gmos.gbias() function
flatFlags : dict
Dictionary for the keyword flags of gmos.gsflat() function
sciFlags : dict
Dictionary for the keyword flags of gmos.gsreduce() function
Based on these flags a set of arcFlags and stdFlags dictionaries will be created
for basic processing.
waveFlags : dict
Dictionary for the keyword flags of gmos.gswavelength() function
sciCombFlags : dict
Dictionary for the keyword flags of gemtools.gemcombine() function
Based on these flags a set of stdCombFlags dictionary will be created for the standard advanced processing.
transFlags : dict
Dictionary for the keyword flags of gmos.gstransform() function.
xxx
skyFlags : dict
Dictionary for the keyword flags of gmos.gsskysub() function
extrFlags : dict
Dictionary for the keywords flags of gmos.gsextract() function
calibFlags : dict
XXX
skip_wavecal : bool
Whether to skip interactive wavelength calibration.
Useful when this is already done.
Returns
-------
"""
print ("### Begin Processing GMOS/Longslit Images ###")
print ("###")
print ("=== Creating MasterCals ===")
# From the work_directory:
# Create the query dictionary of essential parameter=value pairs for Full and CenSp RoIs
qd = {'Full': qd_full, 'CenSp': qd_censp}
print (" --Creating Bias MasterCal--")
# Set the task parameters.
gemtools.gemextn.unlearn() # Disarm a bug in gbias
gmos.gbias.unlearn()
regions = ['Full', 'CenSp']
for r in regions:
# The following SQL generates the list of full-frame files to process.
SQL = fs.createQuery('bias', qd[r])
biasFiles = fs.fileListQuery(dbFile, SQL, qd[r])
# The str.join() funciton is needed to transform a python list into a
# comma-separated string of file names that IRAF can understand.
if len(biasFiles) > 1:
# NT comment: sometimes if there are too many files, gmos.gbias() raises an error.
# import pdb; pdb.set_trace()
gmos.gbias(','.join(str(x) for x in biasFiles), 'MCbias' + r,
**biasFlags)
# Clean up
year_obs = qd_full['DateObs'].split('-')[0]
if clean_files:
iraf.imdel("gS{}*.fits".format(year_obs))
ask_user("MC Bias done. Would you like to continue to proceed with GCAL Spectral Master Flats? (y/n): ",['y','yes'])
print (" -- Creating GCAL Spectral Flat-Field MasterCals --")
# Set the task parameters.
qd['Full'].update({'DateObs': '*'})
qd['CenSp'].update({'DateObs': '*'})
gmos.gireduce.unlearn()
gmos.gsflat.unlearn()
# Normalize the spectral flats per CCD.
# The response fitting should be done interactively.
if flatFlags['fl_inter'] != 'yes':
print("The response fitting should be done interactively. Please set flatFlags['fl_inter'] = 'yes'.")
ask_user("Do you still want to proceed despite this important warning? (y/n): ", ['yes','y'])
for r in regions:
qr = qd[r]
flatFiles = fs.fileListQuery(dbFile, fs.createQuery('gcalFlat', qr), qr)
if len(flatFiles) > 0:
gmos.gsflat(','.join(str(x) for x in flatFiles), 'MCflat' + r,
bias='MCbias' + r, **flatFlags)
if clean_files:
iraf.imdel('gS{}*.fits,gsS{}*.fits'.format(year_obs, year_obs))
ask_user("GCAL Spectral Flat-Field MasterCals done. Would you like to continue to proceed with Basic Processing? (y/n): ",['y','yes'])
print ("=== Processing Science Files ===")
print (" -- Performing Basic Processing --")
# Set task parameters.
gmos.gsreduce.unlearn()
sciFlags = sciFlags # redundant but put here because NT likes it
arcFlags = copy.deepcopy(sciFlags)
arcFlags.update({'fl_flat': 'no', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
stdFlags = copy.deepcopy(sciFlags)
stdFlags.update({'fl_fixpix': 'yes', 'fl_vardq': 'no', 'fl_fulldq': 'no'})
# Perform basic reductions on all exposures for science targets.
print (" - Arc exposures -")
for r in regions:
qr = qd[r]
arcFiles = fs.fileListQuery(dbFile, fs.createQuery('arc', qr), qr)
if len(arcFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in arcFiles),
bias='MCbias' + r, **arcFlags)
print (" - Std star exposures -")
r = 'CenSp'
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qd[r]), qd[r])
if len(stdFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in stdFiles), bias='MCbias' + r,
flatim='MCflat' + r, **stdFlags)
print (" - Science exposures -")
r = 'Full'
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qd[r]), qd[r])
if len(sciFiles) > 0:
gmos.gsreduce(','.join(str(x) for x in sciFiles), bias='MCbias' + r,
flatim='MCflat' + r, **sciFlags)
# Clean up
if clean_files:
iraf.imdel('gS{}*.fits'.format(year_obs))
ask_user("Basic processing done. Would you like to continue to determine wavelength calibration? (y/n): ",['y','yes'])
print (" -- Determine wavelength calibration --")
# Set task parameters
gmos.gswavelength.unlearn()
# The fit to the dispersion relation should be performed interactively.
# Here we will use a previously determined result.
if waveFlags['fl_inter'] != 'yes':
print("The fit to the dispersion relation should be performed interactively. Please set waveFlags['fl_inter'] = 'yes'.")
ask_user("Do you still want to proceed despite this important warning? (y/n): ", ['yes','y'])
# Need to select specific wavecals to match science exposures.
# NT: we do this now from the sciTargets + stdTarget input dictionaries
# e.g.
'''
sciTargets = {
'AM2306-721_a': {'arc': 'gsS20070623S0071', 'sky': '520:720'},
'AM2306-72_b': {'arc': 'gsS20070623S0081', 'sky': '670:760,920:1020'},
'AM2306-721_c': {'arc': 'gsS20070623S0091', 'sky': '170:380,920:1080'}
}
'''
#prefix = 'gsS20070623S0'
#for arc in ['071', '081', '091', '109']:
# gmos.gswavelength(prefix + arc, **waveFlags)
prefix = 'gs'
arc_files = []
for key in sciTargets.keys():
arc_files += [sciTargets[key]['arc']]
for key in stdTarget.keys():
arc_files += [stdTarget[key]['arc']]
# import pdb; pdb.set_trace()
if skip_wavecal is not True:
for arc in arc_files:
gmos.gswavelength(prefix + arc, **waveFlags)
### End of basic processing. Continue with advanced processing.
ask_user("Wavelength solution done. Would you like to continue with advanced processing? (y/n): ",['y','yes'])
print (" -- Performing Advanced Processing --")
print (" -- Combine exposures, apply dispersion, subtract sky --")
# Set task parameters.
gemtools.gemcombine.unlearn()
sciCombFlags = sciCombFlags
stdCombFlags = copy.deepcopy(sciCombFlags)
stdCombFlags.update({'fl_vardq': 'no', 'fl_dqprop': 'no'})
gmos.gstransform.unlearn()
# apply gtransform to standard
# Process the Standard Star
prefix = "gs"
qs = qd['CenSp']
stdFiles = fs.fileListQuery(dbFile, fs.createQuery('std', qs), qs)
std_name = stdTarget.keys()[0]
if len(stdFiles) == 0:
ValueError("No standard star associated. Please check parameters of search (e.g. RoI=CentSp)")
# import pdb; pdb.set_trace()
if len(stdFiles) > 1:
# import pdb; pdb.set_trace()
gemtools.gemcombine(','.join(prefix + str(x) for x in stdFiles),
std_name, **stdCombFlags)
else:
os.system("cp {}.fits {}.fits".format(prefix + stdFiles[0], std_name))
gmos.gstransform(std_name, wavtraname=prefix + stdTarget[std_name]['arc'], **transFlags)
# The sky regions should be selected with care, using e.g. prows/pcols:
# pcols ("tAM2306b.fits[SCI]", 1100, 2040, wy1=40, wy2=320)
print("The sky regions should be selected with care, using e.g. with prows/pcols (see tutorial).")
'''
answer = raw_input("Please provide the long_sample string to apply to gmos.gsskysub() for the standard star."
"e.g. '20:70,190:230' (say 'no' for using the example as the default values): ")
if answer in ['n', 'no']:
print("Using default long_sample set by stdTarget values {}.".format(stdTarget[std_name]['sky']))
long_sample_std = stdTarget[std_name]['sky']
else:
long_sample_std = answer
'''
long_sample_std = stdTarget[std_name]['sky']
ask_user("Before proceeding it is important that you have set a good sky region for the standard.\n"
"Thus far you have selected: {}\n Would you like to proceed with the current one? (y/n): ".format(long_sample_std), ['yes','y'])
# apply sky substraction
skyFlags = skyFlags
gmos.gsskysub.unlearn()
gmos.gsskysub('t{}'.format(std_name), long_sample=long_sample_std)
# NT: make sure the process works ok until here before proceeding further. i.e. setting the sky region manually and correctly.
# NT: seems to be working.
print (" -- Extract Std spectrum --")
# Extract the std spectrum using a large aperture.
# It's important to trace the spectra interactively.
gmos.gsextract.unlearn()
gmos.gsextract("st" + std_name, **extrFlags)
print (" -- Derive the Flux calibration --")
gmos.gsstandard.unlearn()
sensFlags = {
'fl_inter': 'no', 'starname': 'XXX', 'caldir': 'onedstds$ctionewcal/',
'observatory': 'Gemini-South', 'extinction': 'onedstds$ctioextinct.dat',
'function': 'chebyshev', 'order': 9, 'verbose': 'no', 'logfile': 'gsstdLog.txt'
}
sensFlags['starname'] = stdTarget[std_name]['iraf_name'] # replace corresponding starname
gmos.gsstandard('est'+std_name, sfile='std.txt', sfunction='sens', **sensFlags)
ask_user("Sensitivity function from standard star done. Would you like to continue with reduction of science"
" exposures? (y/n): ",['yes','y'])
# Process the science targets.
# Use a dictionary to associate science targets with Arcs and sky regions.
prefix = 'gs'
extract_individuals = True
for targ, p in sciTargets.iteritems():
qs = qd['Full']
qs['Object'] = p['name']
# Fix up the target name for the output file
sciOut = p['name_out']
sciFiles = fs.fileListQuery(dbFile, fs.createQuery('sciSpec', qs), qs)
all_files = ','.join(prefix + str(x) for x in sciFiles)
gemtools.gemcombine(all_files, sciOut, **sciCombFlags)
gmos.gstransform(sciOut, wavtraname=prefix + p['arc'], **transFlags)
ask_user("It is important to select a good sky region for substraction. Thus far you have selected {}"
" based on the sciTargets input dictionary. Would you like to continue? (y/n): ".format(p['sky']),['y','yes'])
gmos.gsskysub('t' + sciOut, long_sample=p['sky'], **skyFlags)
if extract_individuals:
import pdb; pdb.set_trace()
for fname in sciFiles:
gmos.gstransform(prefix + fname, wavtraname=prefix + p['arc'], **transFlags)
gmos.gsskysub('t' + prefix + fname, long_sample=p['sky'], **skyFlags)
gmos.gscalibrate.unlearn()
gmos.gscalibrate('st'+prefix+fname, **calibFlags)
# Clean up
if clean_files:
iraf.imdel("gsS{}*.fits".format(year_obs))
ask_user("Sky substraction done. Would you like to continue to apply sensitivity function? (y/n): ",['y'])
## Apply the sensitivity function.
gmos.gscalibrate.unlearn()
gmos.gscalibrate('st'+sciOut+'*', **calibFlags)
calibFlags.update({'fl_vardq': 'no'})
gmos.gscalibrate('est'+std_name, **calibFlags)
print (" -- Extract Target Spectra --")
method = 'gsextract'
if method == 'gsextract':
gmos.gsextract.unlearn()
# import pdb;pdb.set_trace()
gmos.gsextract("cst" + sciOut, **extrFlags)
elif method == 'sarith':
# not implemented yet
onedspec.nsum = 4
onedspec.sarith('cst{}.fits[SCI]'.format(sciOut), 'copy', '', 'ecst{}.ms'.format(sciOut),
apertures='222-346x4')
print ("=== Finished Calibration Processing ===")
def saofocus(inpat,prepfile=yes,endhow="never",endwhen="",
clobber=globclob,verbose=globver):
""" derive best focus from an SAOFOCUS run on P60 """
# Defaults
twait=30
reduced={}
bpmname="BPM.pl"
saokey="IMGTYPE"
saore="SAOFOCUS"
fseqkey="SAOFVALS"
sigma=2.0
satval=50000.0
masksfx="mask"
pix=0.3787 # arcsec per pixel
mindiff=2 # minimum tolerable diff b/ best and worst seeing (pix)
# Parse end-condition "time"
if endhow=="time":
re1=re.search("^(\d+):(\d+)",endwhen)
if re1:
tnow=time.gmtime()
# Force UT calculation w/ ignoring of DST issues
reftime=time.mktime([tnow[0],tnow[1],tnow[2],
int(re1.group(1)),int(re1.group(2)),0,
tnow[6],tnow[7],0]) - time.timezone
if reftime<time.time():
reftime+=86400
if verbose:
print "Running until %s" % \
time.strftime("%d %b %H:%M",time.gmtime(reftime))
else:
print "Failed to parse %s as UT time" % endwhen
print "Running until stopped..."
endhow="never"
##################################################
# Big Loop
done=no
while not done:
# Parse inputs
allfiles=glob.glob(inpat)
newfiles=[]
for image in allfiles:
if not reduced.has_key(image):
# Interested only in SAOFOCUS images
saoval=get_head(image,saokey)
if re.search(saore,saoval,re.I):
reduced[image]=no
newfiles.append(image)
else:
# Skip
reduced[image]=yes
for image in newfiles:
if verbose:
print "Processing new SAOFOCUS image %s" % image
# Only attempt reduction once for each image
reduced[image]=yes
# Demosaic
ccdproc=iraf.ccdproc
ccdproc.overscan=yes
ccdproc.trim=yes
ccdproc.zerocor=no
iraf.iqmosaic(image,outpfx="",biassec="!BIASSEC",
trimsec="!TRIMSEC",joinkeys="DETSEC",
splitkeys="AMPSEC,CCDSEC,TRIM,OVERSCAN",
clobber=yes,verbose=verbose)
# Fix the WCS that IRAF has now clobbered
iraf.add_wcs(image,instrument="p60new")
# Get some important keywords
[raoff,decoff,saonseq,saoshift] = \
get_head(image,['RAOFF','DECOFF','NUMFOC',
'SAOSHIFT'])
# Type conversions
saonseq=int(saonseq)
saoshift=float(saoshift)
# No bias-subtraction or flat-fielding
# Subtract stair-step background if requested
if prepfile:
qtmp=iraf.mktemp("saofoc")+".fits"
iraf.imcopy(image,qtmp,verbose=no)
for iamp in [-1,1]:
for istep in range(saonseq):
qsec=iraf.mktemp("saofsc")+".fits"
y1=1024+iamp*(1+istep*saoshift)+max(0,iamp)
y2=y1+iamp*(saoshift-1)
if istep==0:
y1-=iamp
elif istep==saonseq-1:
y2=1+max(0,iamp)*2047
ymin=min(y1,y2)
ymax=max(y1,y2)
statsec='[*,%d:%d]' % (ymin,ymax)
iraf.iterstat(image+statsec,nsigrej=5,
maxiter=10,prin=no,verbose=no)
medreg=float(iraf.iterstat.median)
iraf.imarith(image+statsec,'-',medreg,
qsec,verbose=no,noact=no)
iraf.imcopy(qsec,qtmp+statsec,verbose=no)
iraf.imdel(qsec,verify=no,go_ahead=yes)
iraf.imdel(image,verify=no,go_ahead=yes)
iraf.imrename(qtmp,image,verbose=no)
# No renaming of the file
# No Bad Pixel Masking or rudimentary WCS
# Object-detection
doobj=1
if check_head(image,'STARFILE'):
if os.path.exists(get_head(image,'STARFILE')):
doobj=0
if doobj:
iraf.iqobjs(image,sigma,satval,skyval="0.0",
masksfx=masksfx,wtimage="none",minlim=no,
clobber=yes,verbose=verbose)
# Establish our "region of interest" for pixels
pixrough=800
pixminx=1024-float(decoff)/pix-pixrough/2
pixmaxx=pixminx+pixrough
pixmaxy=1023
pixminy=pixmaxy-float(raoff)/pix-(1+saonseq)*saoshift-pixrough/2
# The array of focus positions (from low Y to high Y)
farrkeys=[]
for i in range(saonseq):
farrkeys.append("SAOFOC%02d" % (i+1))
farr=get_head(image,farrkeys)
# Starlist from Sextractor
starfile=get_head(image,'STARFILE')
stars=Starlist(starfile)
stars.mag_sort()
# Find some of the stars in the sequence
xvals=[]
yvals=[]
good1=[]
for star in stars:
# We're going in order from bright to faint
if star.xval>pixminx and star.xval<pixmaxx and \
star.yval<pixmaxy and star.yval>pixminy and \
star.elong<3:
good1.append(star)
xvals.append(star.xval)
yvals.append(star.yval)
if len(good1)>=1.2*saonseq:
break
# Sanity checking
if len(xvals)<1:
print "No stars found in search region"
update_head(image,'SAOFOCUS',0,
"saofocus processing failed for this image")
continue
# Guess for x is median of the xvals of the goodstars
xctr=median(xvals,pick=1)
# First guess for y is median of the goodstars that fall
# within +/- (pixfine) pix of this X value
pixfine=8
yval2=[]
good2=[]
for star in good1:
if abs(star.xval-xctr)<pixfine:
good2.append(star)
yval2.append(star.yval)
# Sanity checking
if len(yval2)<1:
print "No stars found in refined search region"
update_head(image,'SAOFOCUS',0,
"saofocus processing failed for this image")
continue
yctr=median(yval2,pick=1)
# This will be our reference star
usestar=good2[yval2.index(yctr)]
[refx,refy]=[usestar.xval,usestar.yval]
# Identify additional stars, working out from the
# reference star
usestars=[usestar]
# Search increasing & decreasing in Y
# (we use a bigger box in Y, and correct for the last star
# position, because of potential for tracking errors)
for isgn in [+1,-1]:
shifty=refy
for i in range(1,saonseq):
shifty += isgn*saoshift
tgtstars=stars.starsinbox([[refx-pixfine,refx+pixfine],
[shifty-1.5*pixfine,
shifty+1.5*pixfine]])
if len(tgtstars)>0:
minmag=99.99
# Pick the brightest in the box
for tstar in tgtstars:
if tstar.mag<minmag:
svstar=tstar
minmag=tstar.mag
usestars.append(svstar)
shifty=svstar.yval
else:
# Quit searching when there are no stars in box
break
# Exclude faint stars if we have too many
# (could also drop stars until we reach the number, but
# then we would want to check that all the y-values are in
# strict succession...)
magrough=2.0
if len(usestars)>saonseq:
usemags=[]
for star in usestars:
usemags.append(star.mag)
minmag=min(usemags)
use2=[]
for star in usestars:
if star.mag<minmag+magrough:
use2.append(star)
if len(use2)==saonseq:
usestars=use2
# Check for the right number
if len(usestars)!=saonseq:
print "Failed to get right number of target stars"
update_head(image,'SAOFOCUS',0,
"saofocus processing failed for this image")
continue
# Construct the array of x- and y-values
xval3=[]
yval3=[]
for star in usestars:
xval3.append(star.xval)
yval3.append(star.yval)
# Choose a single x-value for the run
xctr2="%.3f" % median(xval3)
# Increasing y-value means increasing index in farr
yval3.sort()
ylis=','.join(map(str,yval3))
flis=','.join(map(str,farr))
# Run iqfocus
iraf.iqfocus(image,xctr2,ylis,flis,method="acorr",
boxsize=saoshift,focuskey="BESTFOC",
seekey="SEEING",fseekey="SAOSEE",
update=yes,clobber=yes,verbose=verbose)
# Check for successful iqfocus run
if not check_head(image,'BESTFOC'):
print "iqfocus run failed to find good focus"
update_head(image,'SAOFOCUS',0,
"saofocus processing failed for this image")
continue
# Grab focus positions and seeing values
bestfoc,saosee=get_head(image,['BESTFOC','SAOSEE'])
# Some information
if verbose:
print "Focus settings: "+flis
print "Seeing values: "+saosee
print "Best focus: %.3f" % bestfoc
# Check for best-focus "near edge" condition
for i in range(saonseq):
if ("%.3f" % bestfoc)==("%.3f" % float(farr[i])):
ibest=i
break
if (saonseq>3 and (ibest==0 or ibest==saonseq-1)) or \
(saonseq>5 and (ibest==1 or ibest==saonseq-2)):
update_head(image,'SAOEDGE',1,
"Best seeing is near edge of focus run")
# Check for insufficient variation across the focus run
maxsee=0
minsee=0.5*saoshift
saoseevals=eval(saosee)
for seeval in saoseevals:
minsee=min(minsee,seeval)
maxsee=max(maxsee,seeval)
if (maxsee-minsee) < mindiff:
print "Failed to observe significant variation "+\
"across focus run"
update_head(image,'SAOFOCUS',0,
"saofocus processing failed for this image")
continue
# Convert seeing value to arcsec
seepix=get_head(image,'SEEPIX')
update_head(image,'BESTSEE',seepix,
"Best seeing in pixels for this series")
try:
seeasec=float(seepix)*pix
update_head(image,'SEEING',seeasec,
"Best seeing in arcsec for this series")
except:
print "Failed to convert seeing to arcsec for %s" % image
# Successful processing
update_head(image,'SAOFOCUS',1,
"saofocus processing succeeded for this image")
##############################
# Test end conditions
if endhow=="never":
done=no
elif endhow=="once":
done=yes
elif endhow=="time":
if time.time()>reftime:
done=yes
# Wait a little while
if not done:
time.sleep(twait)
radius=18.0,
ylevel=0.05,
background='fit',
b_function='chebyshev',
b_order=2,
b_sample='-22:-15,15:22',
b_naverage=-3,
weights='variance',
pfit='fit2d',
readnoise='RDNOISE',
gain='GAIN',
apertures='6-96')
print "Removing scattered inter-order light from the spectrum..."
s2 = 'noscat' + s1
iraf.imcopy(s1, s2)
iraf.imdel(s1, verify=no)
print "Removing scattered light with apscatter..."
iraf.apscatter(input=s2,
output=s1,
ref=s1,
interac=no,
find=no,
recent=no,
resize=no,
edit=no,
trace=no,
fittrace=no,
subtrac=yes,
smooth=yes,
fitscat=no,
fitsmoo=no,
def prep(df_image,hi_res_image,width_mask=1.5,unmaskgal=False,galvalues = None):
print "\n************ Running the preparation steps ************\n"
iraf.imdel('_mask.fits')
iraf.imdel('_fluxmod_cfht*.fits')
iraf.imdel('_df_4*.fits')
'run SExtractor to get bright sources that are easily detected in the high resolution data'
subprocess.call('sex %s'%hi_res_image,shell=True) ##### Add in option to change sextractor threshold detect_thresh and analysis_thresh 2/3
'copy the segmentation map to a mask'
iraf.imcopy('seg.fits','_mask.fits')
##### Add step to get rid of diffraction spikes
if unmaskgal:
'Run SExtractor to get values for the central galaxy'
print '\nDoing a second sextractor run to unmask central galaxy. \n'
analysis_thresh_lg=2;back_size_lg=128;detect_thresh_lg=2;detect_minarea_lg=60
segname = run_SExtractor(hi_res_image,detect_thresh=detect_thresh_lg,analysis_thresh=analysis_thresh_lg,back_size=back_size_lg,detect_minarea=detect_minarea_lg)
print '\nSegmentation map is named: '+segname
'Open up the data'
seg2data = fits.getdata(segname)
segdata,segheader = fits.getdata('_mask.fits',header=True)
'Detect sources from the segmentation map'
from photutils import detect_sources
segrefdata = detect_sources(seg2data, 3, npixels=5)#, filter_kernel=kernel)
segrefdata = segrefdata.data
segrefname = re.sub('.fits','_ds.fits',segname)
writeFITS(segrefdata,segheader,segrefname)
print '\nSource separated segmentation map is named: '+segrefname
'Use detected source seg map to mask galaxies' #galvalues = [3531,5444,5496]
print 'Unmask the galaxies in the mask from the original segmap. \n'
segdatanew = unmaskgalaxy(segdata,'_mask.fits',segrefname=segrefname,segref=segrefdata,galvalues=galvalues)
writeFITS(segdatanew,segheader,'_mask.fits')
if verbose:
print_verbose_string('Carrying on')
'replace the values in the segments in the segmentation map (i.e. stars) all to 1, all the background is still 0'
iraf.imreplace('_mask.fits',1,lower=0.5)
'multiply the mask (with 1s at the stars) by the high res image to get the star flux back - now have the flux model'
iraf.imarith('_mask.fits','*','%s'%hi_res_image,'_fluxmod_cfht')
'smooth the flux model'
'increase size of mask, so more is subtracted'
'the "1.5" in the next line should be a user-defined parameter that is given'
'to the script; it controls how much of the low surface brightness'
'emission in the outskirts of galaxies is subtracted. This choice'
'depends on the science application'
iraf.gauss('_fluxmod_cfht','_fluxmod_cfht_smoothed',width_mask,nsigma=4.)
iraf.imreplace('_fluxmod_cfht_smoothed', -1, lower=0, upper=0)
iraf.imreplace('_fluxmod_cfht_smoothed', 1, lower=-0.5)
iraf.imreplace('_fluxmod_cfht_smoothed', 0, upper=-0.5)
iraf.imarith('%s'%hi_res_image, '*','_fluxmod_cfht_smoothed', '_fluxmod_cfht_new')
' this is the key new step! we"re registering the CFHT image'
' to a frame that is 4x finer sampled than the Dragonfly image.'
' this avoids all the pixelation effects we had before.'
' (4x seems enough; but we could have it as a free parameter - need'
' to be careful as it occurs elsewhere in the scripts too) '
iraf.blkrep('%s'%df_image,'_df_4',4,4)
'register the flux model onto the same pixel scale as the dragonfly image'
iraf.wregister('_fluxmod_cfht_new','_df_4','_fluxmod_dragonfly',interpo='linear',fluxcon='yes')
return None
