def listpix(inim):
"""
Extract calibrated (wavelength, flux) 1d spectrum into data file.
Header information of object, exptime, date-obs(UT) are required to change file name terminally.
"""
import glob
import os, sys
from pyraf import iraf
from astropy.io import fits
from astropy.time import Time
hdr = fits.getheader(inim)
obj = hdr['object']
exptime = hdr['exptime']
obsdate = hdr['date-obs']
t = Time(obsdate)
ccd = 'SAO_KL400'
output_data = 'Calib-' + ccd + '-' + obj + '-' + str(
t.datetime.year).zfill(2) + str(t.datetime.month).zfill(2) + str(
t.datetime.day).zfill(2) + '-' + str(
t.datetime.hour).zfill(2) + str(
t.datetime.minute).zfill(2) + str(
t.datetime.second).zfill(2) + '-' + str(
int(exptime)) + '.data'
iraf.imutil()
org_stdout = sys.stdout
f = open(output_data, "w+")
sys.stdout = f
iraf.listpixels(images=inim)
sys.stdout = org_stdout
f.close()
os.system('pluma ' + output_data + ' &')
def get_ao_performance_data():
# Retrieve information from image headers
outfile = workdir + 'data/ao_perform.txt'
ir.images()
ir.imutil()
ir.hselect('*_img.fits', 'DATAFILE,DATE-OBS,UTC,AOLBFWHM,LGRMSWF', 'yes', Stdout=outfile)
# Retrieve MASS/DIMM data files
# get_mass_dimm.go('20081021') # this one doesn't exist
get_mass_dimm.go('20100815')
get_mass_dimm.go('20100828')
get_mass_dimm.go('20100829')
def sub_bias(image, combined_bias, image_b):
# Import IRAF modules:
iraf.images(_doprint=0)
iraf.imutil(_doprint=0)
parList = "bias_subtraction_imarith.par"
# Check input file and combined_bias frame exists before proceeding:
if os.path.isfile(image) == True:
if os.path.isfile(combined_bias) == True:
if os.path.isfile(parList) == True:
# Subtract combined bias frame from input frame (object or flat)
# using IRAF task imarithmetic:
iraf.imarith.setParList(ParList="bias_subtraction_imarith.par")
iraf.imarith(operand1=image, operand2=combined_bias, result=image_b)
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'Bias frame ' + str(combined_bias)
print 'subtracted from input ' + str(image)
print 'to create ' + str(image_b)
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
else:
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'Bias frame subtraction IRAF .par file '
print str(parList)
print 'does not exist. Exiting script. '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
print ' '
sys.exit()
else:
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'Combined bias frame '
print str(combined_bias)
print 'does not exist. Exiting script. '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
print ' '
sys.exit()
else:
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'Input frame '
print str(image)
print 'does not exist. Exiting script. '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
print ' '
sys.exit()
def flat_field(object_b, combined_flat_b_n, object_b_fn):
# Import IRAF modules:
iraf.images(_doprint=0)
iraf.imutil(_doprint=0)
# Check input file and combined_flat frame exist before proceeding:
if os.path.isfile(object_b) == True:
if os.path.isfile(combined_flat_b_n) == True:
# Divide bias-subtracted object frame by normalized, bias
# subtracted combined flat frame using IRAF task imarithmetic:
iraf.imarith(operand1=object_b, op="/", operand2=combined_flat_b_n, result=object_b_fn)
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'Bias-subtracted object frame '
print str(object_b)
print 'successfully flat-fielded using division by '
print str(combined_flat_b_n)
print 'to create bias-subtracted, normalized flat-fielded '
print str(object_b_fn) + ' frame.'
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
else:
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'Combined flat frame '
print str(combined_flat_b_n)
print 'does not exist. Exiting script. '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
print ' '
sys.exit()
else:
print ' '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'Input frame '
print str(object_b)
print 'does not exist. Exiting script. '
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print ' '
print ' '
sys.exit()
def makeflat(lista):
# print "LOGX:: Entering `makeflat` method/function in %(__file__)s" %
# globals()
flat = ''
import datetime
import glob
import os
import ntt
from ntt.util import readhdr, readkey3, delete, name_duplicate, updateheader, correctcard
from pyraf import iraf
iraf.images(_doprint=0)
iraf.imutil(_doprint=0)
iraf.imgeom(_doprint=0)
# iraf.blkavg(_doprint=0)
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.generic(_doprint=0)
toforget = ['imgeom.blkavg', 'imutil.imarith',
'immatch.imcombine', 'noao.imred']
for t in toforget:
iraf.unlearn(t)
import datetime
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
_date = readkey3(readhdr(lista[0]), 'date-night')
_filter = readkey3(readhdr(lista[0]), 'filter')
output = name_duplicate(
lista[3], 'flat_' + str(_date) + '_' + str(_filter) + '_' + str(MJDtoday), '')
if os.path.isfile(output):
answ = raw_input('file already prooduced, do again [y/[n]] ? ')
if not answ:
answ = 'n'
else:
answ = 'y'
if answ in ['yes', 'y', 'YES', 'Y', 'Yes']:
delete("temp_off.fits,temp_off_mask.fits,temp_on_mask.fits,temp_on.fits")
iraf.image.immatch.imcombine(
lista[0] + ',' + lista[7], output="temp_off.fits")
iraf.image.immatch.imcombine(
lista[1] + ',' + lista[6], output="temp_off_mask.fits")
iraf.image.immatch.imcombine(
lista[2] + ',' + lista[5], output="temp_on_mask.fits")
iraf.image.immatch.imcombine(
lista[3] + ',' + lista[4], output="temp_on.fits")
# create the bias correction for the flat-on according to the
# Lidman technique0
delete("temp_onA.fits,temp_onC.fits,temp_onB.fits,temp_onAC.fits,temp_onACB.fits,temp_onACB_2D.fits")
delete("temp_on_bias.fits")
iraf.imgeom.blkavg(
input="temp_on.fits[500:600,*]", output="temp_onA.fits", option="average", b1=101, b2=1)
iraf.imgeom.blkavg(
input="temp_on_mask.fits[500:600,*]", output="temp_onC.fits", option="average", b1=101, b2=1)
iraf.imgeom.blkavg(
input="temp_on_mask.fits[50:150,*]", output="temp_onB.fits", option="average", b1=101, b2=1)
iraf.imutil.imarith("temp_onA.fits", "-",
"temp_onC.fits", "temp_onAC.fits")
iraf.imutil.imarith("temp_onAC.fits", "+",
"temp_onB.fits", "temp_onACB.fits")
iraf.imgeom.blkrep(input="temp_onACB.fits",
output="temp_onACB_2D.fits", b1=1024, b2=1)
iraf.imutil.imarith("temp_on.fits", "-",
"temp_onACB_2D.fits", "temp_on_bias.fits")
# same as above for the flat-off
delete("temp_offA.fits,temp_offC.fits,temp_offB.fits,temp_offAC.fits,temp_offACB.fits,temp_offACB_2D.fits")
delete("temp_off_bias.fits")
iraf.imgeom.blkavg(
input="temp_off.fits[500:600,*]", output="temp_offA.fits", option="average", b1=101, b2=1)
iraf.imgeom.blkavg(
input="temp_off_mask.fits[500:600,*]", output="temp_offC.fits", option="average", b1=101, b2=1)
iraf.imgeom.blkavg(
input="temp_off_mask.fits[50:150,*]", output="temp_offB.fits", option="average", b1=101, b2=1)
iraf.imutil.imarith("temp_offA.fits", "-",
"temp_offC.fits", "temp_offAC.fits")
iraf.imutil.imarith("temp_offAC.fits", "+",
"temp_offB.fits", "temp_offACB.fits")
iraf.imgeom.blkrep(input="temp_offACB.fits",
output="temp_offACB_2D.fits", b1=1024, b2=1)
iraf.imutil.imarith("temp_off.fits", "-",
"temp_offACB_2D.fits", "temp_off_bias.fits")
# create the corrected flat-field
# output=name_duplicate("temp_on_bias.fits",'flat_'+str(_date)+'_'+str(_filter)+'_'+str(MJDtoday),'')
output = name_duplicate(
lista[3], 'flat_' + str(_date) + '_' + str(_filter) + '_' + str(MJDtoday), '')
# print lista[0],'flat_'+str(_date)+'_'+str(_filter)+'_'+str(MJDtoday)
delete(output)
iraf.imutil.imarith("temp_on_bias.fits", "-",
"temp_off_bias.fits", output)
iraf.noao.imred.generic.normalize(output) # normalize the flat-field
correctcard(output)
delete("temp_on*.fits") # delete the temporary images
delete("temp_off*.fits")
print 'flat -> ' + str(output)
else:
print 'skip redoing the flat'
return output
def main(argv):
home_root = os.environ['HOME']
iraf.images(_doprint=0)
iraf.tv(_doprint=0)
iraf.ptools(_doprint=0)
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.photcal(_doprint=0)
iraf.apphot(_doprint=0)
iraf.imutil(_doprint=0)
for file_ in os.listdir("./"):
if file_.endswith("_i.fits"):
fits_file_i = file_
title_string = file_[0:9]
if file_.endswith("_g.fits"):
fits_file_g = file_
fits_h_i = fits.open(fits_file_i)
fits_h_g = fits.open(fits_file_g)
fwhm_i = fits_h_i[0].header['FWHMPSF']
fwhm_g = fits_h_g[0].header['FWHMPSF']
print 'Image FWHM :: g = {0:5.3f} : i = {1:5.3f}'.format(fwhm_g,fwhm_i)
# if not os.path.isdir('psf'): # make a psf subfolder if it doesn't exist
# os.mkdir('psf')
# # and then copy in the data files if they don't already exist as files
# if not os.path.isfile('psf/'+fits_file_i):
# iraf.images.imcopy(fits_file_i,'psf/'+fits_file_i,verbose="yes")
#
# if not os.path.isfile('psf/'+fits_file_g) :
# iraf.images.imcopy(fits_file_g,'psf/'+fits_file_g,verbose="yes")
#
# # also copy in the apcor star lists
# if not (os.path.isfile('psf/apcor_stars_i.txt') or os.path.islink('psf/apcor_stars_i.txt')) :
# shutil.copyfile('apcor_stars_i.txt','psf/apcor_stars_i.txt')
#
# if not (os.path.isfile('psf/apcor_stars_g.txt') or os.path.islink('psf/apcor_stars_g.txt')) :
# shutil.copyfile('apcor_stars_g.txt','psf/apcor_stars_g.txt')
#
# # and change to the psf folder
# os.chdir('psf')
iraf.unlearn(iraf.phot,iraf.datapars,iraf.photpars,iraf.centerpars,iraf.fitskypars)
iraf.apphot.phot.setParam('interactive',"no")
iraf.apphot.phot.setParam('verify',"no")
iraf.datapars.setParam('datamax',"INDEF")
iraf.datapars.setParam('gain',"gain")
iraf.datapars.setParam('ccdread',"rdnoise")
iraf.datapars.setParam('exposure',"exptime")
iraf.datapars.setParam('airmass',"airmass")
iraf.datapars.setParam('filter',"filter")
iraf.datapars.setParam('obstime',"time-obs")
iraf.datapars.setParam('sigma',"INDEF")
iraf.photpars.setParam('zmag',0.)
iraf.centerpars.setParam('cbox',9.)
iraf.centerpars.setParam('maxshift',3.)
iraf.fitskypars.setParam('salgorithm',"median")
iraf.fitskypars.setParam('dannulus',10.)
iraf.datapars.setParam('fwhmpsf',fwhm_g)
iraf.photpars.setParam('apertures',iraf.nint(4.*fwhm_g))
iraf.fitskypars.setParam('annulus',(6.*fwhm_g))
iraf.apphot.phot(image=fits_file_g, coords='apcor_stars_g.txt', output="g_psfstars.mag.1")
iraf.datapars.setParam('fwhmpsf',fwhm_i)
iraf.photpars.setParam('apertures',iraf.nint(4.*fwhm_i))
iraf.fitskypars.setParam('annulus',(6.*fwhm_i))
iraf.apphot.phot(image=fits_file_i, coords='apcor_stars_i.txt', output="i_psfstars.mag.1")
with open("apcor_stars_i.txt") as foo:
lines = len(foo.readlines())
iraf.daophot(_doprint=0)
iraf.pstselect.setParam('image',fits_file_i)
iraf.pstselect.setParam('photfile',"i_psfstars.mag.1")
iraf.pstselect.setParam('pstfile',"default")
iraf.pstselect.setParam('maxnpsf',lines)
iraf.pstselect.setParam('plottype',"mesh")
iraf.daophot.pstselect(verify='no', mode='h')
with open("apcor_stars_g.txt") as foo:
lines = len(foo.readlines())
iraf.pstselect.setParam('image',fits_file_g)
iraf.pstselect.setParam('photfile',"g_psfstars.mag.1")
iraf.pstselect.setParam('pstfile',"default")
iraf.pstselect.setParam('maxnpsf',lines)
iraf.pstselect.setParam('plottype',"mesh")
iraf.daophot.pstselect()
iraf.datapars.setParam('datamax',"INDEF")
iraf.datapars.setParam('gain',"gain")
iraf.datapars.setParam('ccdread',"rdnoise")
iraf.datapars.setParam('exposure',"exptime")
iraf.datapars.setParam('airmass',"airmass")
iraf.datapars.setParam('filter',"filter")
iraf.datapars.setParam('obstime',"time-obs")
iraf.datapars.setParam('sigma',"INDEF")
iraf.daopars.setParam('psfrad',iraf.nint(3.*fwhm_i))
iraf.daopars.setParam('fitrad',fwhm_i)
iraf.psf.setParam('image',fits_file_i)
iraf.psf.setParam('photfile',"i_psfstars.mag.1")
iraf.psf.setParam('pstfile',fits_file_i+".pst.1")
iraf.psf.setParam('interactive', 'no')
iraf.daophot.psf()
iraf.psf.setParam('image',fits_file_g)
iraf.psf.setParam('photfile',"g_psfstars.mag.1")
iraf.psf.setParam('pstfile',fits_file_g+".pst.1")
iraf.psf.setParam('interactive', 'no')
iraf.daophot.psf()
def load_modules():
# Define a function to load all of the modules so that they don't' import
# unless we need them
global iraf
from pyraf import iraf
iraf.pysalt()
iraf.saltspec()
iraf.saltred()
iraf.set(clobber='YES')
global sys
import sys
global os
import os
global shutil
import shutil
global glob
from glob import glob
global pyfits
import pyfits
global np
import numpy as np
global lacosmicx
import lacosmicx
global interp
from scipy import interp
global signal
from scipy import signal
global ndimage
from scipy import ndimage
global interpolate
from scipy import interpolate
global WCS
from astropy.wcs import WCS
global optimize
from scipy import optimize
global ds9
import pyds9 as ds9
global GaussianProcess
from sklearn.gaussian_process import GaussianProcess
global pandas
import pandas
iraf.onedspec()
iraf.twodspec()
iraf.longslit()
iraf.apextract()
iraf.imutil()
iraf.rvsao(motd='no')
def efoscspec1Dredu(files,
_interactive,
_ext_trace,
_dispersionline,
liststandard,
listatmo0,
_automaticex,
_verbose=False):
# print "LOGX:: Entering `efoscspec1Dredu` method/function in
# %(__file__)s" % globals()
import ntt
try:
import pyfits
except:
from astropy.io import fits as pyfits
import re
import string
import sys
import os
import numpy as np
os.environ["PYRAF_BETA_STATUS"] = "1"
_extinctdir = 'direc$standard/extinction/'
_extinction = 'lasilla2.txt'
_observatory = 'lasilla'
import datetime
now = datetime.datetime.now()
datenow = now.strftime('20%y%m%d%H%M')
MJDtoday = 55927 + (datetime.date.today() -
datetime.date(2012, 01, 01)).days
dv = ntt.dvex()
scal = np.pi / 180.
_gain = ntt.util.readkey3(ntt.util.readhdr(re.sub('\n', '', files[0])),
'gain')
_rdnoise = ntt.util.readkey3(ntt.util.readhdr(re.sub('\n', '', files[0])),
'ron')
std, rastd, decstd, magstd = ntt.util.readstandard(
'standard_efosc_mab.txt')
objectlist = {}
for img in files:
hdr = ntt.util.readhdr(img)
img = re.sub('\n', '', img)
ntt.util.correctcard(img)
_ra = ntt.util.readkey3(hdr, 'RA')
_dec = ntt.util.readkey3(hdr, 'DEC')
_object = ntt.util.readkey3(hdr, 'object')
_grism = ntt.util.readkey3(hdr, 'grism')
_filter = ntt.util.readkey3(hdr, 'filter')
_slit = ntt.util.readkey3(hdr, 'slit')
dd = np.arccos(
np.sin(_dec * scal) * np.sin(decstd * scal) +
np.cos(_dec * scal) * np.cos(decstd * scal) * np.cos(
(_ra - rastd) * scal)) * ((180 / np.pi) * 3600)
if min(dd) < 100:
_type = 'stdsens'
else:
_type = 'obj'
if min(dd) < 100:
ntt.util.updateheader(img, 0,
{'stdname': [std[np.argmin(dd)], '']})
ntt.util.updateheader(
img, 0, {'magstd': [float(magstd[np.argmin(dd)]), '']})
if _type not in objectlist:
objectlist[_type] = {}
if (_grism, _filter, _slit) not in objectlist[_type]:
objectlist[_type][_grism, _filter, _slit] = [img]
else:
objectlist[_type][_grism, _filter, _slit].append(img)
from pyraf import iraf
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.specred(_doprint=0)
iraf.imutil(_doprint=0)
toforget = ['imutil.imcopy', 'specred.sarith', 'specred.standard']
for t in toforget:
iraf.unlearn(t)
iraf.specred.verbose = 'no'
iraf.specred.dispaxi = 2
iraf.set(direc=ntt.__path__[0] + '/')
sens = {}
print objectlist
outputfile = []
if 'obj' in objectlist.keys():
tpe = 'obj'
elif 'stdsens' in objectlist.keys():
tpe = 'stdsens'
else:
sys.exit('error: no objects and no standards in the list')
for setup in objectlist[tpe]:
extracted = []
listatmo = []
if setup not in sens:
sens[setup] = []
if tpe == 'obj':
print '\n### setup= ', setup, '\n### objects= ', objectlist['obj'][
setup], '\n'
for img in objectlist['obj'][setup]:
# hdr=readhdr(img)
print '\n\n### next object= ', img, ' ', ntt.util.readkey3(
ntt.util.readhdr(img), 'object'), '\n'
if os.path.isfile(re.sub('.fits', '_ex.fits', img)):
if ntt.util.readkey3(
ntt.util.readhdr(re.sub('.fits', '_ex.fits', img)),
'quality') == 'Rapid':
ntt.util.delete(re.sub('.fits', '_ex.fits', img))
imgex = ntt.util.extractspectrum(img,
dv,
_ext_trace,
_dispersionline,
_interactive,
'obj',
automaticex=_automaticex)
if not os.path.isfile(imgex):
sys.exit('### error, extraction not computed')
if not ntt.util.readkey3(ntt.util.readhdr(imgex), 'shift') and \
ntt.util.readkey3(ntt.util.readhdr(imgex), 'shift') != 0.0:
# if not readkey3(readhdr(imgex),'shift'):
ntt.efoscspec1Ddef.checkwavestd(imgex, _interactive)
extracted.append(imgex)
if imgex not in outputfile:
outputfile.append(imgex)
ntt.util.updateheader(
imgex, 0, {'FILETYPE': [22107, 'extracted 1D spectrum ']})
ntt.util.updateheader(
imgex, 0, {
'PRODCATG': [
'SCIENCE.' + ntt.util.readkey3(
ntt.util.readhdr(imgex), 'tech').upper(),
'Data product category'
]
})
ntt.util.updateheader(imgex, 0,
{'TRACE1': [img, 'Originating file']})
if os.path.isfile('database/ap' +
re.sub('_ex.fits', '', imgex)):
if 'database/ap' + re.sub('_ex.fits', '',
imgex) not in outputfile:
outputfile.append('database/ap' +
re.sub('_ex.fits', '', imgex))
print '\n### all object with this setup extracted\n'
if liststandard:
standardlist = liststandard
_type = 'stdfromdreducer'
else:
try:
standardlist = objectlist['stdsens'][setup]
_type = 'stdsens'
except:
standardlist = ''
_type = ''
if _type == 'stdfromdreducer' and len(extracted) >= 1:
_outputsens2 = ntt.util.searchsens(extracted[0], standardlist)[0]
print '\n### using standard from reducer ' + str(_outputsens2)
elif _type not in ['stdsens', 'stdfromdreducer'
] and len(extracted) >= 1:
_outputsens2 = ntt.util.searchsens(extracted[0], '')[0]
os.system('cp ' + _outputsens2 + ' .')
_outputsens2 = string.split(_outputsens2, '/')[-1]
print '\n### no standard in the list, using standard from archive'
else:
for simg in standardlist:
print '\n### standard for setup ' + \
str(setup) + ' = ', simg, ' ', ntt.util.readkey3(
ntt.util.readhdr(simg), 'object'), '\n'
simgex = ntt.util.extractspectrum(simg,
dv,
False,
False,
_interactive,
'std',
automaticex=_automaticex)
ntt.util.updateheader(
simgex, 0, {'FILETYPE': [22107, 'extracted 1D spectrum']})
ntt.util.updateheader(
simgex, 0, {
'PRODCATG': [
'SCIENCE.' + ntt.util.readkey3(
ntt.util.readhdr(simgex), 'tech').upper(),
'Data product category'
]
})
ntt.util.updateheader(simgex, 0,
{'TRACE1': [simg, 'Originating file']})
if not ntt.util.readkey3(ntt.util.readhdr(simgex), 'shift') and \
ntt.util.readkey3(ntt.util.readhdr(simgex), 'shift') != 0.0:
# if not readkey3(readhdr(simgex),'shift'):
ntt.efoscspec1Ddef.checkwavestd(simgex, _interactive)
atmofile = ntt.efoscspec1Ddef.telluric_atmo(
simgex) # atmo file2
ntt.util.updateheader(atmofile, 0,
{'TRACE1': [simgex, 'Originating file']})
ntt.util.updateheader(
atmofile, 0,
{'FILETYPE': [21211, 'telluric correction 1D spectrum ']})
if tpe != 'obj' and atmofile not in outputfile:
outputfile.append(atmofile)
if not listatmo0:
listatmo.append(atmofile)
sens[setup].append(simgex)
if simgex not in outputfile:
outputfile.append(simgex)
if setup[0] == 'Gr13' and setup[1] == 'Free':
if os.path.isfile(re.sub('Free', 'GG495', simg)):
print '\n### extract standard frame with blocking filter to correct for second order contamination\n'
simg2 = re.sub('Free', 'GG495', simg)
simgex2 = ntt.util.extractspectrum(
simg2,
dv,
False,
False,
_interactive,
'std',
automaticex=_automaticex)
ntt.util.updateheader(
simgex2, 0,
{'FILETYPE': [22107, 'extracted 1D spectrum']})
ntt.util.updateheader(
simgex2, 0, {
'PRODCATG': [
'SCIENCE.' + ntt.util.readkey3(
ntt.util.readhdr(simgex2),
'tech').upper(),
'Data product category'
]
})
if not ntt.util.readkey3(ntt.util.readhdr(simgex2), 'shift') and \
ntt.util.readkey3(ntt.util.readhdr(simgex2), 'shift') != 0.0:
# if not readkey3(readhdr(simgex2),'shift'):
ntt.efoscspec1Ddef.checkwavestd(
simgex2, _interactive)
ntt.util.updateheader(
simgex2, 0,
{'TRACE1': [simg2, 'Originating file']})
print '\n### standard available: ', sens[setup]
if tpe == 'obj':
if len(sens[setup]) > 1:
goon = 'no'
while goon != 'yes':
stdused = raw_input(
'\n### more than one standard for this setup, which one do you want to use ['
+ sens[setup][0] + '] ?')
if not stdused:
stdused = sens[setup][0]
if os.path.isfile(stdused):
goon = 'yes'
else:
stdused = sens[setup][0]
stdvec = [stdused]
else:
stdvec = sens[setup]
for stdused in stdvec:
stdusedclean = re.sub('_ex', '_clean', stdused)
ntt.util.delete(stdusedclean)
iraf.specred.sarith(input1=stdused,
op='/',
input2=atmofile,
output=stdusedclean,
format='multispec')
_outputsens2 = ntt.efoscspec1Ddef.sensfunction(
stdusedclean, 'spline3', 16, _interactive)
ntt.util.updateheader(
_outputsens2, 0,
{'FILETYPE': [21212, 'sensitivity function']})
ntt.util.updateheader(
_outputsens2, 0, {'TRACE1': [stdused, 'Originating file']})
if setup[0] == 'Gr13' and setup[1] == 'Free':
if os.path.isfile(re.sub('Free', 'GG495', stdused)):
print '\n### compute sensitivity function of grim 13 with blocking filter ' \
'to correct for second order contamination \n'
stdused2 = re.sub('Free', 'GG495', stdused)
if not ntt.util.readkey3(ntt.util.readhdr(stdused2),
'STDNAME'):
ntt.util.updateheader(
stdused2, 0, {
'STDNAME': [
ntt.util.readkey3(
ntt.util.readhdr(stdused),
'STDNAME'), ''
]
})
atmofile2 = ntt.efoscspec1Ddef.telluric_atmo(
stdused2) # atmo file2
stdusedclean2 = re.sub('_ex', '_clean', stdused2)
ntt.util.delete(stdusedclean2)
iraf.specred.sarith(input1=stdused2,
op='/',
input2=atmofile2,
output=stdusedclean2,
format='multispec')
_outputsens3 = ntt.efoscspec1Ddef.sensfunction(
stdusedclean2, 'spline3', 16, _interactive)
ntt.util.updateheader(
_outputsens3, 0,
{'FILETYPE': [21212, 'sensitivity function']})
ntt.util.updateheader(
_outputsens3, 0,
{'TRACE1': [stdused2, 'Originating file']})
_outputsens2 = correctsens(_outputsens2, _outputsens3)
if _outputsens2 not in outputfile:
outputfile.append(_outputsens2)
if _outputsens2 and tpe == 'obj':
####################################################
for img in objectlist['obj'][setup]: # flux calibrate 2d images
imgd = fluxcalib2d(img, _outputsens2)
ntt.util.updateheader(
imgd, 0, {
'FILETYPE':
[22209, '2D wavelength and flux calibrated spectrum ']
})
ntt.util.updateheader(imgd, 0,
{'TRACE1': [img, 'Originating files']})
iraf.hedit(imgd,
'PRODCATG',
delete='yes',
update='yes',
verify='no')
if imgd not in outputfile:
outputfile.append(imgd)
####################################################
# flux calib in the standard way
if not listatmo and listatmo0:
listatmo = listatmo0[:]
for _imgex in extracted:
_airmass = ntt.util.readkey3(ntt.util.readhdr(_imgex),
'airmass')
_exptime = ntt.util.readkey3(ntt.util.readhdr(_imgex),
'exptime')
_imgf = re.sub('_ex.fits', '_f.fits', _imgex)
ntt.util.delete(_imgf)
qqq = iraf.specred.calibrate(input=_imgex,
output=_imgf,
sensiti=_outputsens2,
extinct='yes',
flux='yes',
extinction=_extinctdir +
_extinction,
observatory=_observatory,
airmass=_airmass,
ignorea='yes',
exptime=_exptime,
fnu='no')
hedvec = {
'SENSFUN': [_outputsens2, ''],
'FILETYPE':
[22208, '1D wavelength and flux calibrated spectrum', ''],
# 'SNR':[ntt.util.StoN(_imgf,50),'Average signal to noise ratio per pixel'],
'SNR': [
ntt.util.StoN2(_imgf, False),
'Average signal to noise ratio per pixel'
],
'BUNIT':
['erg/cm2/s/Angstrom', 'Physical unit of array values'],
'TRACE1': [_imgex, 'Originating file'],
'ASSON1': [
re.sub('_f.fits', '_2df.fits', _imgf),
'Name of associated file'
],
'ASSOC1':
['ANCILLARY.2DSPECTRUM', 'Category of associated file']
}
ntt.util.updateheader(_imgf, 0, hedvec)
if _imgf not in outputfile:
outputfile.append(_imgf)
if listatmo:
atmofile = ntt.util.searcharc(_imgex, listatmo)[0]
if atmofile:
_imge = re.sub('_f.fits', '_e.fits', _imgf)
ntt.util.delete(_imge)
iraf.specred.sarith(input1=_imgf,
op='/',
input2=atmofile,
output=_imge,
w1='INDEF',
w2='INDEF',
format='multispec')
try:
iraf.imutil.imcopy(input=_imgf + '[*,1,2]',
output=_imge + '[*,1,2]',
verbose='no')
except:
pass
try:
iraf.imutil.imcopy(input=_imgf + '[*,1,3]',
output=_imge + '[*,1,3]',
verbose='no')
except:
pass
try:
iraf.imutil.imcopy(input=_imgf + '[*,1,4]',
output=_imge + '[*,1,4]',
verbose='no')
except:
pass
if _imge not in outputfile:
outputfile.append(_imge)
ntt.util.updateheader(
_imge, 0, {
'FILETYPE': [
22210,
'1D, wave, flux calib, telluric corr.'
]
})
if atmofile not in outputfile:
outputfile.append(atmofile)
ntt.util.updateheader(_imge, 0,
{'ATMOFILE': [atmofile, '']})
ntt.util.updateheader(
_imge, 0, {'TRACE1': [_imgf, 'Originating file']})
imgin = _imge
else:
imgin = _imgf
else:
imgin = _imgf
imgasci = re.sub('.fits', '.asci', imgin)
ntt.util.delete(imgasci)
iraf.onedspec(_doprint=0)
iraf.onedspec.wspectext(imgin + '[*,1,1]',
imgasci,
header='no')
if imgasci not in outputfile:
outputfile.append(imgasci)
print '\n### adding keywords for phase 3 ....... '
for img in outputfile:
if str(img)[-5:] == '.fits':
try:
ntt.util.phase3header(img) # phase 3 definitions
ntt.util.updateheader(img, 0, {'quality': ['Final', '']})
except:
print 'Warning: ' + img + ' is not a fits file'
try:
if int(re.sub('\.', '', str(pyfits.__version__))[:2]) <= 30:
aa = 'HIERARCH '
else:
aa = ''
except:
aa = ''
imm = pyfits.open(img, mode='update')
hdr = imm[0].header
if aa + 'ESO DPR CATG' in hdr:
hdr.pop(aa + 'ESO DPR CATG')
if aa + 'ESO DPR TECH' in hdr:
hdr.pop(aa + 'ESO DPR TECH')
if aa + 'ESO DPR TYPE' in hdr:
hdr.pop(aa + 'ESO DPR TYPE')
imm.flush()
imm.close()
print outputfile
reduceddata = ntt.rangedata(outputfile)
f = open(
'logfile_spec1d_' + str(reduceddata) + '_' + str(datenow) +
'.raw.list', 'w')
for img in outputfile:
try:
f.write(ntt.util.readkey3(ntt.util.readhdr(img), 'arcfile') + '\n')
except:
pass
f.close()
return outputfile, 'logfile_spec1d_' + str(reduceddata) + '_' + str(
datenow) + '.raw.list'
def efoscspec1Dredu(files, _interactive, _ext_trace, _dispersionline, liststandard, listatmo0, _automaticex,
_verbose=False):
# print "LOGX:: Entering `efoscspec1Dredu` method/function in
# %(__file__)s" % globals()
import ntt
try: import pyfits
except: from astropy.io import fits as pyfits
import re
import string
import sys
import os
import numpy as np
os.environ["PYRAF_BETA_STATUS"] = "1"
_extinctdir = 'direc$standard/extinction/'
_extinction = 'lasilla2.txt'
_observatory = 'lasilla'
import datetime
now = datetime.datetime.now()
datenow = now.strftime('20%y%m%d%H%M')
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
dv = ntt.dvex()
scal = np.pi / 180.
_gain = ntt.util.readkey3(ntt.util.readhdr(
re.sub('\n', '', files[0])), 'gain')
_rdnoise = ntt.util.readkey3(
ntt.util.readhdr(re.sub('\n', '', files[0])), 'ron')
std, rastd, decstd, magstd = ntt.util.readstandard(
'standard_efosc_mab.txt')
objectlist = {}
for img in files:
hdr = ntt.util.readhdr(img)
img = re.sub('\n', '', img)
ntt.util.correctcard(img)
_ra = ntt.util.readkey3(hdr, 'RA')
_dec = ntt.util.readkey3(hdr, 'DEC')
_object = ntt.util.readkey3(hdr, 'object')
_grism = ntt.util.readkey3(hdr, 'grism')
_filter = ntt.util.readkey3(hdr, 'filter')
_slit = ntt.util.readkey3(hdr, 'slit')
dd = np.arccos(np.sin(_dec * scal) * np.sin(decstd * scal) + np.cos(_dec * scal) *
np.cos(decstd * scal) * np.cos((_ra - rastd) * scal)) * ((180 / np.pi) * 3600)
if min(dd) < 100:
_type = 'stdsens'
else:
_type = 'obj'
if min(dd) < 100:
ntt.util.updateheader(
img, 0, {'stdname': [std[np.argmin(dd)], '']})
ntt.util.updateheader(
img, 0, {'magstd': [float(magstd[np.argmin(dd)]), '']})
if _type not in objectlist:
objectlist[_type] = {}
if (_grism, _filter, _slit) not in objectlist[_type]:
objectlist[_type][_grism, _filter, _slit] = [img]
else:
objectlist[_type][_grism, _filter, _slit].append(img)
from pyraf import iraf
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.specred(_doprint=0)
iraf.imutil(_doprint=0)
toforget = ['imutil.imcopy', 'specred.sarith', 'specred.standard']
for t in toforget:
iraf.unlearn(t)
iraf.specred.verbose = 'no'
iraf.specred.dispaxi = 2
iraf.set(direc=ntt.__path__[0] + '/')
sens = {}
print objectlist
outputfile = []
if 'obj' in objectlist.keys():
tpe = 'obj'
elif 'stdsens' in objectlist.keys():
tpe = 'stdsens'
else:
sys.exit('error: no objects and no standards in the list')
for setup in objectlist[tpe]:
extracted = []
listatmo = []
if setup not in sens:
sens[setup] = []
if tpe == 'obj':
print '\n### setup= ', setup, '\n### objects= ', objectlist['obj'][setup], '\n'
for img in objectlist['obj'][setup]:
# hdr=readhdr(img)
print '\n\n### next object= ', img, ' ', ntt.util.readkey3(ntt.util.readhdr(img), 'object'), '\n'
if os.path.isfile(re.sub('.fits', '_ex.fits', img)):
if ntt.util.readkey3(ntt.util.readhdr(re.sub('.fits', '_ex.fits', img)), 'quality') == 'Rapid':
ntt.util.delete(re.sub('.fits', '_ex.fits', img))
imgex = ntt.util.extractspectrum(img, dv, _ext_trace, _dispersionline, _interactive, 'obj',
automaticex=_automaticex)
if not os.path.isfile(imgex):
sys.exit('### error, extraction not computed')
if not ntt.util.readkey3(ntt.util.readhdr(imgex), 'shift') and \
ntt.util.readkey3(ntt.util.readhdr(imgex), 'shift') != 0.0:
# if not readkey3(readhdr(imgex),'shift'):
ntt.efoscspec1Ddef.checkwavestd(imgex, _interactive)
extracted.append(imgex)
if imgex not in outputfile:
outputfile.append(imgex)
ntt.util.updateheader(
imgex, 0, {'FILETYPE': [22107, 'extracted 1D spectrum ']})
ntt.util.updateheader(imgex, 0, {
'PRODCATG': ['SCIENCE.' +
ntt.util.readkey3(ntt.util.readhdr(imgex), 'tech').upper(), 'Data product category']})
ntt.util.updateheader(
imgex, 0, {'TRACE1': [img, 'Originating file']})
if os.path.isfile('database/ap' + re.sub('_ex.fits', '', imgex)):
if 'database/ap' + re.sub('_ex.fits', '', imgex) not in outputfile:
outputfile.append(
'database/ap' + re.sub('_ex.fits', '', imgex))
print '\n### all object with this setup extracted\n'
if liststandard:
standardlist = liststandard
_type = 'stdfromdreducer'
else:
try:
standardlist = objectlist['stdsens'][setup]
_type = 'stdsens'
except:
standardlist = ''
_type = ''
if _type == 'stdfromdreducer' and len(extracted) >= 1:
_outputsens2 = ntt.util.searchsens(extracted[0], standardlist)[0]
print '\n### using standard from reducer ' + str(_outputsens2)
elif _type not in ['stdsens', 'stdfromdreducer'] and len(extracted) >= 1:
_outputsens2 = ntt.util.searchsens(extracted[0], '')[0]
os.system('cp ' + _outputsens2 + ' .')
_outputsens2 = string.split(_outputsens2, '/')[-1]
print '\n### no standard in the list, using standard from archive'
else:
for simg in standardlist:
print '\n### standard for setup ' + \
str(setup) + ' = ', simg, ' ', ntt.util.readkey3(
ntt.util.readhdr(simg), 'object'), '\n'
simgex = ntt.util.extractspectrum(
simg, dv, False, False, _interactive, 'std', automaticex=_automaticex)
ntt.util.updateheader(
simgex, 0, {'FILETYPE': [22107, 'extracted 1D spectrum']})
ntt.util.updateheader(simgex, 0, {
'PRODCATG': [
'SCIENCE.' + ntt.util.readkey3(ntt.util.readhdr(simgex), 'tech').upper(), 'Data product category']})
ntt.util.updateheader(
simgex, 0, {'TRACE1': [simg, 'Originating file']})
if not ntt.util.readkey3(ntt.util.readhdr(simgex), 'shift') and \
ntt.util.readkey3(ntt.util.readhdr(simgex), 'shift') != 0.0:
# if not readkey3(readhdr(simgex),'shift'):
ntt.efoscspec1Ddef.checkwavestd(simgex, _interactive)
atmofile = ntt.efoscspec1Ddef.telluric_atmo(
simgex) # atmo file2
ntt.util.updateheader(
atmofile, 0, {'TRACE1': [simgex, 'Originating file']})
ntt.util.updateheader(
atmofile, 0, {'FILETYPE': [21211, 'telluric correction 1D spectrum ']})
if tpe != 'obj' and atmofile not in outputfile:
outputfile.append(atmofile)
if not listatmo0:
listatmo.append(atmofile)
sens[setup].append(simgex)
if simgex not in outputfile:
outputfile.append(simgex)
if setup[0] == 'Gr13' and setup[1] == 'Free':
if os.path.isfile(re.sub('Free', 'GG495', simg)):
print '\n### extract standard frame with blocking filter to correct for second order contamination\n'
simg2 = re.sub('Free', 'GG495', simg)
simgex2 = ntt.util.extractspectrum(simg2, dv, False, False, _interactive, 'std',
automaticex=_automaticex)
ntt.util.updateheader(
simgex2, 0, {'FILETYPE': [22107, 'extracted 1D spectrum']})
ntt.util.updateheader(simgex2, 0, {
'PRODCATG': ['SCIENCE.' +
ntt.util.readkey3(
ntt.util.readhdr(simgex2), 'tech').upper(), 'Data product category']})
if not ntt.util.readkey3(ntt.util.readhdr(simgex2), 'shift') and \
ntt.util.readkey3(ntt.util.readhdr(simgex2), 'shift') != 0.0:
# if not readkey3(readhdr(simgex2),'shift'):
ntt.efoscspec1Ddef.checkwavestd(
simgex2, _interactive)
ntt.util.updateheader(
simgex2, 0, {'TRACE1': [simg2, 'Originating file']})
print '\n### standard available: ', sens[setup]
if tpe == 'obj':
if len(sens[setup]) > 1:
goon = 'no'
while goon != 'yes':
stdused = raw_input(
'\n### more than one standard for this setup, which one do you want to use [' + sens[setup][
0] + '] ?')
if not stdused:
stdused = sens[setup][0]
if os.path.isfile(stdused):
goon = 'yes'
else:
stdused = sens[setup][0]
stdvec = [stdused]
else:
stdvec = sens[setup]
for stdused in stdvec:
stdusedclean = re.sub('_ex', '_clean', stdused)
ntt.util.delete(stdusedclean)
iraf.specred.sarith(
input1=stdused, op='/', input2=atmofile, output=stdusedclean, format='multispec')
_outputsens2 = ntt.efoscspec1Ddef.sensfunction(
stdusedclean, 'spline3', 16, _interactive)
ntt.util.updateheader(_outputsens2, 0, {'FILETYPE': [
21212, 'sensitivity function']})
ntt.util.updateheader(
_outputsens2, 0, {'TRACE1': [stdused, 'Originating file']})
if setup[0] == 'Gr13' and setup[1] == 'Free':
if os.path.isfile(re.sub('Free', 'GG495', stdused)):
print '\n### compute sensitivity function of grim 13 with blocking filter ' \
'to correct for second order contamination \n'
stdused2 = re.sub('Free', 'GG495', stdused)
if not ntt.util.readkey3(ntt.util.readhdr(stdused2), 'STDNAME'):
ntt.util.updateheader(stdused2, 0, {
'STDNAME': [ntt.util.readkey3(ntt.util.readhdr(stdused), 'STDNAME'), '']})
atmofile2 = ntt.efoscspec1Ddef.telluric_atmo(
stdused2) # atmo file2
stdusedclean2 = re.sub('_ex', '_clean', stdused2)
ntt.util.delete(stdusedclean2)
iraf.specred.sarith(input1=stdused2, op='/', input2=atmofile2, output=stdusedclean2,
format='multispec')
_outputsens3 = ntt.efoscspec1Ddef.sensfunction(
stdusedclean2, 'spline3', 16, _interactive)
ntt.util.updateheader(_outputsens3, 0, {'FILETYPE': [
21212, 'sensitivity function']})
ntt.util.updateheader(
_outputsens3, 0, {'TRACE1': [stdused2, 'Originating file']})
_outputsens2 = correctsens(_outputsens2, _outputsens3)
if _outputsens2 not in outputfile:
outputfile.append(_outputsens2)
if _outputsens2 and tpe == 'obj':
####################################################
for img in objectlist['obj'][setup]: # flux calibrate 2d images
imgd = fluxcalib2d(img, _outputsens2)
ntt.util.updateheader(
imgd, 0, {'FILETYPE': [22209, '2D wavelength and flux calibrated spectrum ']})
ntt.util.updateheader(
imgd, 0, {'TRACE1': [img, 'Originating files']})
iraf.hedit(imgd, 'PRODCATG', delete='yes',
update='yes', verify='no')
if imgd not in outputfile:
outputfile.append(imgd)
####################################################
# flux calib in the standard way
if not listatmo and listatmo0:
listatmo = listatmo0[:]
for _imgex in extracted:
_airmass = ntt.util.readkey3(
ntt.util.readhdr(_imgex), 'airmass')
_exptime = ntt.util.readkey3(
ntt.util.readhdr(_imgex), 'exptime')
_imgf = re.sub('_ex.fits', '_f.fits', _imgex)
ntt.util.delete(_imgf)
qqq = iraf.specred.calibrate(input=_imgex, output=_imgf, sensiti=_outputsens2, extinct='yes',
flux='yes',
extinction=_extinctdir + _extinction, observatory=_observatory,
airmass=_airmass, ignorea='yes', exptime=_exptime, fnu='no')
hedvec = {'SENSFUN': [_outputsens2, ''],
'FILETYPE': [22208, '1D wavelength and flux calibrated spectrum', ''],
# 'SNR':[ntt.util.StoN(_imgf,50),'Average signal to noise ratio per pixel'],
'SNR': [ntt.util.StoN2(_imgf, False), 'Average signal to noise ratio per pixel'],
'BUNIT': ['erg/cm2/s/Angstrom', 'Physical unit of array values'],
'TRACE1': [_imgex, 'Originating file'],
'ASSON1': [re.sub('_f.fits', '_2df.fits', _imgf), 'Name of associated file'],
'ASSOC1': ['ANCILLARY.2DSPECTRUM', 'Category of associated file']}
ntt.util.updateheader(_imgf, 0, hedvec)
if _imgf not in outputfile:
outputfile.append(_imgf)
if listatmo:
atmofile = ntt.util.searcharc(_imgex, listatmo)[0]
if atmofile:
_imge = re.sub('_f.fits', '_e.fits', _imgf)
ntt.util.delete(_imge)
iraf.specred.sarith(input1=_imgf, op='/', input2=atmofile, output=_imge, w1='INDEF', w2='INDEF',
format='multispec')
try:
iraf.imutil.imcopy(
input=_imgf + '[*,1,2]', output=_imge + '[*,1,2]', verbose='no')
except:
pass
try:
iraf.imutil.imcopy(
input=_imgf + '[*,1,3]', output=_imge + '[*,1,3]', verbose='no')
except:
pass
try:
iraf.imutil.imcopy(
input=_imgf + '[*,1,4]', output=_imge + '[*,1,4]', verbose='no')
except:
pass
if _imge not in outputfile:
outputfile.append(_imge)
ntt.util.updateheader(
_imge, 0, {'FILETYPE': [22210, '1D, wave, flux calib, telluric corr.']})
if atmofile not in outputfile:
outputfile.append(atmofile)
ntt.util.updateheader(
_imge, 0, {'ATMOFILE': [atmofile, '']})
ntt.util.updateheader(
_imge, 0, {'TRACE1': [_imgf, 'Originating file']})
imgin = _imge
else:
imgin = _imgf
else:
imgin = _imgf
imgasci = re.sub('.fits', '.asci', imgin)
ntt.util.delete(imgasci)
iraf.onedspec(_doprint=0)
iraf.onedspec.wspectext(
imgin + '[*,1,1]', imgasci, header='no')
if imgasci not in outputfile:
outputfile.append(imgasci)
print '\n### adding keywords for phase 3 ....... '
for img in outputfile:
if str(img)[-5:] == '.fits':
try:
ntt.util.phase3header(img) # phase 3 definitions
ntt.util.updateheader(img, 0, {'quality': ['Final', '']})
except:
print 'Warning: ' + img + ' is not a fits file'
try:
if int(re.sub('\.', '', str(pyfits.__version__))[:2]) <= 30:
aa = 'HIERARCH '
else:
aa = ''
except:
aa = ''
imm = pyfits.open(img, mode='update')
hdr = imm[0].header
if aa + 'ESO DPR CATG' in hdr:
hdr.pop(aa + 'ESO DPR CATG')
if aa + 'ESO DPR TECH' in hdr:
hdr.pop(aa + 'ESO DPR TECH')
if aa + 'ESO DPR TYPE' in hdr:
hdr.pop(aa + 'ESO DPR TYPE')
imm.flush()
imm.close()
print outputfile
reduceddata = ntt.rangedata(outputfile)
f = open('logfile_spec1d_' + str(reduceddata) +
'_' + str(datenow) + '.raw.list', 'w')
for img in outputfile:
try:
f.write(ntt.util.readkey3(ntt.util.readhdr(img), 'arcfile') + '\n')
except:
pass
f.close()
return outputfile, 'logfile_spec1d_' + str(reduceddata) + '_' + str(datenow) + '.raw.list'
def psfphot(inlist, ra, dec, reffilt, interact, fwhm, readnoise, gain,
threshold,refimage=None,starfile=None,maxnpsf=5,
clobber=globclob,verbose=globver,skykey='SKYBKG',
filtkey='FILTER',pixtol=3.0):
""" perform PSF-based photometry on a single target star (SN?) at RA, Dec and
also on a set of comparison stars, using daophot. simultaneously
perform aperture photometry on all the comparison stars (after
subtracting off contributions from neighbors) to enable absolute
photometry by comparison to aperture photometry of standard stars
observed in other fields """
# Defaults / constants
psfmult=5.0 #standard factor (multiplied by fwhm to get psfradius)
psfmultsmall=3.0 #similar to psfmult, adjusted for nstar and substar
# Necessary package
iraf.imutil()
# Parse inputs
infiles=iraffiles(inlist)
# Which file is reffilt? call it refimage
if refimage==None:
for image in infiles:
if check_head(image, filtkey):
try:
imgfilt = get_head(image, filtkey)
if imgfilt == reffilt:
refimage = image
break
except:
pass
if not refimage:
print "BAD USER! No image corresponds to the filter: %s" % reffilt
return
else:
refroot='s'+refimage.split('.')[0]
#first make sure to add back in background of sky
iraf.iqsubsky(inlist, sub=no, skykey=skykey)
#put reference image first on list
infiles.remove(refimage)
infiles.insert(0,refimage)
#setup for keywords
if gain == "!GAIN":
try: gainval = float(get_head(image, gain))
except:
print "Bad header keyword for gain."
else:
gainval = float(gain)
if readnoise == "!READNOISE":
try: readval = float(get_head(image, readnoise))
except:
print "Bad header keyword for readnoise."
else:
readval = float(readnoise)
# Process each file in turn
for image in infiles:
# Check that the image is there
check_exist(image,"r")
# Grab image root name
root=image.split('.')[0]
# Map image to reference image
if not (image==refimage):
[nx,ny]=get_head(image,['NAXIS1','NAXIS2'])
stars=Starlist(get_head(image,'STARFILE'))
refstars=Starlist(get_head(refimage,'STARFILE'))
refstars.pix2wcs(refimage)
refstars.wcs2pix(image)
match,refmatch=stars.match(refstars,useflags=yes,tol=10.0)
nstars=len(match)
if not (nstars>2):
print 'Could not find star matches between reference and %s' % image
infiles.remove(image)
continue
refmatch.pix2wcs(image)
refmatch.wcs2pix(refimage)
matchfile=open('%s.match' % root, 'w')
for i in range(len(match)):
matchfile.write('%10.3f%10.3f%10.3f%10.3f\n' %
(refmatch[i].xval,refmatch[i].yval,
match[i].xval,match[i].yval))
matchfile.close()
check_exist('%s.geodb' % root, 'w', clobber=clobber)
iraf.geomap('%s.match' % root,'%s.geodb' % root,1.0,nx,1.0,ny,
verbose=no,interactive=no)
check_exist('s%s.fits' % root, 'w', clobber=clobber)
iraf.geotran(image,'s%s' % root,'%s.geodb' % root,
'%s.match' % root,geometry="geometric",
boundary="constant",verbose=no)
else:
iraf.imcopy(image,'s%s' % root)
root='s%s' % root
#get sky level and calculate sigma
#if check_head(image, skykey):
# try:
# sky=float(get_head(image, skykey))
# except:
# print "No sky levels in header."
#sigma= (((sky * gainval) + readval**2)**.5) / gainval
iraf.iterstat(image)
# Saturation level
if not check_head(image, "SATURATE"):
saturate = 60000.0
else:
saturate = get_head(image, "SATURATE")
# Update datapars and daopars
iraf.datapars.fwhmpsf=fwhm
iraf.datapars.sigma=iraf.iterstat.sigma
iraf.datapars.datamin=iraf.iterstat.median-10*iraf.iterstat.sigma
iraf.datapars.datamax=0.90*saturate
iraf.datapars.readnoise=readval
iraf.datapars.epadu=gainval
iraf.datapars.filter=filtkey
iraf.daopars.psfrad=psfmult*fwhm
iraf.daopars.fitrad=fwhm
iraf.daopars.function="gauss,moffat15,moffat25,lorentz,penny1"
#find stars in image unless a starlist is given
if image==refimage and starfile==None:
iraf.daophot.daofind(root,'refimage.coo.1',threshold=threshold,verify=no,
verbose=verbose)
elif image==refimage:
shutil.copy(starfile,'refimage.coo.1')
#initial photometry
iraf.daophot.phot(root,'refimage.coo.1','default',aperture=fwhm,verify=no,
verbose=verbose)
#select stars for psf the first time
refstarsfile = "refimage.pst.1"
if image == refimage:
iraf.pstselect(root,'default',refstarsfile,maxnpsf,
interactive=yes,verify=no,verbose=verbose)
#fit the psf
iraf.psf(root,'default',refstarsfile,'default','default','default',
interactive=interact,verify=no,verbose=verbose)
#identify neighboring/interfering stars to selected stars
groupingfile = root+".psg.1"
iraf.nstar(root,groupingfile,'default','default','default',
psfrad= psfmultsmall * fwhm,verify=no,verbose=verbose)
#subtract out neighboring stars from image
iraf.substar(root,'default',refstarsfile,'default','default',
psfrad=psfmultsmall*fwhm,verify=no,verbose=verbose)
#repeat psf to get better psf model
#IRAF's interactive version usually crashes
subtractedimage = root+".sub.1"
iraf.psf(subtractedimage,root+".nst.1",refstarsfile,'%s.psf.2' % root,
'%s.pst.2' % root,'%s.psg.2' % root,interactive=interact,
verify=no,verbose=verbose)
#Need to make sure SN was detected by daofind
stars=Starlist('%s.mag.1' % root)
SN=Star(name='SN',radeg=ra,dcdeg=dec,fwhm=2.0,fwhmw=2.0)
SNlis=Starlist(stars=[SN])
SNlis.wcs2pix(image)
if (len(stars.match(SNlis)[0])==0):
#No match - need to add to daofind file
print "No match!"
coofile=open('refimage.coo.1', 'a+')
coofile.write('%10.3f%10.3f%9.3f%8.3f%13.3f%12.3f%8i\n' % (SNlis[0].xval, SNlis[0].yval,99.999,0.500,0.000,0.000,999))
coofile.close()
#repeat aperture photometry to get good comparisons to standard fields
iraf.daophot.phot(root,'refimage.coo.1','default',aperture=psfmult*fwhm,
verify=no,verbose=verbose)
# allstar run
iraf.allstar(root,'default','default','default','default','default',
verify=no,verbose=verbose)
def sofispec1Dredu(files, _interactive, _ext_trace, _dispersionline, _automaticex, _verbose=False):
# print "LOGX:: Entering `sofispec1Dredu` method/function in %(__file__)s"
# % globals()
import re
import string
import sys
import os
os.environ["PYRAF_BETA_STATUS"] = "1"
import ntt
try: import pyfits
except: from astropy.io import fits as pyfits
import numpy as np
import datetime
import pylab as pl
from pyraf import iraf
dv = ntt.dvex()
now = datetime.datetime.now()
datenow = now.strftime('20%y%m%d%H%M')
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
scal = np.pi / 180.
hdr0 = ntt.util.readhdr(re.sub('\n', '', files[0]))
_gain = ntt.util.readkey3(hdr0, 'gain')
_rdnoise = ntt.util.readkey3(hdr0, 'ron')
std_sun, rastd_sun, decstd_sun, magstd_sun = ntt.util.readstandard(
'standard_sofi_sun.txt')
std_vega, rastd_vega, decstd_vega, magstd_vega = ntt.util.readstandard(
'standard_sofi_vega.txt')
std_phot, rastd_phot, decstd_phot, magstd_phot = ntt.util.readstandard(
'standard_sofi_phot.txt')
outputfile = []
objectlist, RA, DEC = {}, {}, {}
for img in files:
img = re.sub('\n', '', img)
hdr = ntt.util.readhdr(img)
_ra = ntt.util.readkey3(hdr, 'RA')
_dec = ntt.util.readkey3(hdr, 'DEC')
_grism = ntt.util.readkey3(hdr, 'grism')
_filter = ntt.util.readkey3(hdr, 'filter')
_slit = ntt.util.readkey3(hdr, 'slit')
cc_sun = np.arccos(np.sin(_dec * scal) * np.sin(decstd_sun * scal) + np.cos(_dec * scal) *
np.cos(decstd_sun * scal) * np.cos((_ra - rastd_sun) * scal)) * ((180 / np.pi) * 3600)
cc_vega = np.arccos(np.sin(_dec * scal) * np.sin(decstd_vega * scal) + np.cos(_dec * scal) *
np.cos(decstd_vega * scal) * np.cos((_ra - rastd_vega) * scal)) * ((180 / np.pi) * 3600)
cc_phot = np.arccos(np.sin(_dec * scal) * np.sin(decstd_phot * scal) + np.cos(_dec * scal) *
np.cos(decstd_phot * scal) * np.cos((_ra - rastd_phot) * scal)) * ((180 / np.pi) * 3600)
if min(cc_sun) < 100:
_type = 'sun'
elif min(cc_phot) < 100:
_type = 'stdp'
elif min(cc_vega) < 100:
_type = 'vega'
else:
_type = 'obj'
if min(cc_phot) < 100:
if _verbose:
print img, 'phot', str(min(cc_phot)), str(std_phot[np.argmin(cc_phot)])
ntt.util.updateheader(img, 0, {'stdname': [std_phot[np.argmin(cc_phot)], ''],
'magstd': [float(magstd_phot[np.argmin(cc_phot)]), '']})
# ntt.util.updateheader(img,0,{'magstd':[float(magstd_phot[argmin(cc_phot)]),'']})
elif min(cc_sun) < 100:
if _verbose:
print img, 'sun', str(min(cc_sun)), str(std_sun[np.argmin(cc_sun)])
ntt.util.updateheader(img, 0, {'stdname': [std_sun[np.argmin(cc_sun)], ''],
'magstd': [float(magstd_sun[np.argmin(cc_sun)]), '']})
# ntt.util.updateheader(img,0,{'magstd':[float(magstd_sun[argmin(cc_sun)]),'']})
elif min(cc_vega) < 100:
if _verbose:
print img, 'vega', str(min(cc_vega)), str(std_vega[np.argmin(cc_vega)])
ntt.util.updateheader(img, 0, {'stdname': [std_vega[np.argmin(cc_vega)], ''],
'magstd': [float(magstd_vega[np.argmin(cc_vega)]), '']})
# ntt.util.updateheader(img,0,{'magstd':[float(magstd_vega[argmin(cc_vega)]),'']})
else:
if _verbose:
print img, 'object'
_OBID = (ntt.util.readkey3(hdr, 'esoid'))
if _type not in objectlist:
objectlist[_type] = {}
if _grism not in objectlist[_type]:
objectlist[_type][_grism] = {}
if _OBID not in objectlist[_type][_grism]:
objectlist[_type][_grism][_OBID] = []
objectlist[_type][_grism][_OBID].append(img)
if 'stdp' not in objectlist:
print '### warning: not photometric standard'
else:
print '### photometric standard in the list of object'
if 'sun' not in objectlist:
print '### warning: not telluric G standard (sun type)'
else:
print '### telluric G standard (sun type) in the list of object'
if 'vega' not in objectlist:
print '### warning: not telluric A standard (vega type)'
else:
print '### telluric A standard (vega type) in the list of object'
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.specred(_doprint=0)
iraf.immatch(_doprint=0)
iraf.imutil(_doprint=0)
toforget = ['specred.apall', 'specred.transform']
for t in toforget:
iraf.unlearn(t)
iraf.specred.apall.readnoi = _rdnoise
iraf.specred.apall.gain = _gain
iraf.specred.dispaxi = 2
for _type in objectlist:
for setup in objectlist[_type]:
for _ID in objectlist[_type][setup]:
listmerge = objectlist[_type][setup][_ID]
listmerge = ntt.sortbyJD(listmerge)
_object = ntt.util.readkey3(
ntt.util.readhdr(listmerge[0]), 'object')
if string.count(_object, '/') or string.count(_object, '.') or string.count(_object, ' '):
nameobj = string.split(_object, '/')[0]
nameobj = string.split(nameobj, ' ')[0]
nameobj = string.split(nameobj, '.')[0]
else:
nameobj = _object
_date = ntt.util.readkey3(
ntt.util.readhdr(listmerge[0]), 'date-night')
outputimage = nameobj + '_' + _date + \
'_' + setup + '_merge_' + str(MJDtoday)
outputimage = ntt.util.name_duplicate(
listmerge[0], outputimage, '')
print '### setup= ', setup, ' name field= ', nameobj, ' merge image= ', outputimage, '\n'
#################
# added to avoid crashing with a single frame
# header will not be updated with all info
#################
if len(listmerge)==1:
ntt.util.delete(outputimage)
iraf.imutil.imcopy(listmerge[0], output=outputimage, verbose='no')
answ= 'n'
else:
if os.path.isfile(outputimage) and _interactive:
answ = raw_input(
'combine frame of dithered spectra already created. Do you want to make it again [[y]/n] ? ')
if not answ:
answ = 'y'
else:
answ = 'y'
#################
if answ in ['Yes', 'y', 'Y', 'yes']:
if _interactive:
automaticmerge = raw_input(
'\n### Do you want to try to find the dither bethween frames automatically [[y]/n]')
if not automaticmerge:
automaticmerge = 'yes'
elif automaticmerge.lower() in ['y', 'yes']:
automaticmerge = 'yes'
else:
automaticmerge = 'no'
else:
automaticmerge = 'yes'
if automaticmerge == 'yes':
offset = 0
offsetvec = []
_center0 = ntt.sofispec1Ddef.findaperture(
listmerge[0], False)
_offset0 = ntt.util.readkey3(
ntt.util.readhdr(listmerge[0]), 'xcum')
print '\n### Try to merge spectra considering their offset along x axes .......'
f = open('_offset', 'w')
for img in listmerge:
_center = ntt.sofispec1Ddef.findaperture(
img, False)
_center2 = (
float(_center) + (float(_offset0) - float(_center0))) * (-1)
_offset = (-1) * \
ntt.util.readkey3(
ntt.util.readhdr(img), 'xcum')
if abs(_center2 - _offset) >= 20:
automaticmerge = 'no'
break
else:
offset3 = _center2
offsetvec.append(offset3)
line = str(offset3) + ' 0\n'
f.write(line)
f.close()
if automaticmerge == 'yes':
print '### automatic merge .......... done'
else:
print '\n### warning: try identification of spectra position in interactive way '
offset = 0
offsetvec = []
_z1, _z2, goon = ntt.util.display_image(
listmerge[0], 1, '', '', False)
print '\n### find aperture on first frame and use it as reference position of ' \
'the spectra (mark with ' + '"' + 'm' + '"' + ')'
_center0 = ntt.sofispec1Ddef.findaperture(
listmerge[0], True)
_offset0 = ntt.util.readkey3(
ntt.util.readhdr(listmerge[0]), 'xcum')
print '\n### find the aperture on all the spectra frames (mark with ' + '"' + 'm' + '"' + ')'
f = open('_offset', 'w')
for img in listmerge:
print '\n### ', img
_z1, _z2, goon = ntt.util.display_image(
img, 1, '', '', False)
_center = ntt.sofispec1Ddef.findaperture(img, True)
_center2 = (
float(_center) + (float(_offset0) - float(_center0))) * (-1)
_offset = (-1) * \
ntt.util.readkey3(
ntt.util.readhdr(img), 'xcum')
print '\n### position from dither header: ' + str(_offset)
print '### position identified interactively: ' + str(_center2)
offset3 = raw_input(
'\n### which is the right position [' + str(_center2) + '] ?')
if not offset3:
offset3 = _center2
offsetvec.append(offset3)
line = str(offset3) + ' 0\n'
f.write(line)
f.close()
print offsetvec
start = int(max(offsetvec) - min(offsetvec))
print start
f = open('_goodlist', 'w')
print listmerge
for img in listmerge:
f.write(img + '\n')
f.close()
ntt.util.delete(outputimage)
ntt.util.delete('_output.fits')
yy1 = pyfits.open(listmerge[0])[0].data[:, 10]
iraf.immatch.imcombine('@_goodlist', '_output', combine='sum', reject='none', offset='_offset',
masktyp='', rdnoise=_rdnoise, gain=_gain, zero='mode', Stdout=1)
_head = pyfits.open('_output.fits')[0].header
if _head['NAXIS1'] < 1024:
stop = str(_head['NAXIS1'])
else:
stop = '1024'
iraf.imutil.imcopy(
'_output[' + str(start) + ':'+stop+',*]', output=outputimage, verbose='no')
print outputimage
print len(listmerge)
hdr1 = ntt.util.readhdr(outputimage)
ntt.util.updateheader(outputimage, 0,
{'SINGLEXP': [False, 'TRUE if resulting from single exposure'],
'M_EPOCH': [False, 'TRUE if resulting from multiple epochs'],
'EXPTIME': [ntt.util.readkey3(hdr1, 'EXPTIME') * len(listmerge),
'Total integration time per pixel (s)'],
'TEXPTIME': [float(ntt.util.readkey3(hdr1, 'TEXPTIME')) * len(listmerge),
'Total integration time of all exposures (s)'],
'APERTURE': [2.778e-4 * float(re.sub('long_slit_', '',
ntt.util.readkey3(hdr1, 'slit'))),
'[deg] Aperture diameter'],
'NOFFSETS': [2, 'Number of offset positions'],
'NUSTEP': [0, 'Number of microstep positions'],
'NJITTER': [int(ntt.util.readkey3(hdr1, 'NCOMBINE') / 2),
'Number of jitter positions']})
hdr = ntt.util.readhdr(outputimage)
matching = [s for s in hdr.keys() if "IMCMB" in s]
for imcmb in matching:
aaa = iraf.hedit(outputimage, imcmb, delete='yes', update='yes',
verify='no', Stdout=1)
if 'SKYSUB' in hdr.keys():
aaa = iraf.hedit(outputimage, 'SKYSUB', delete='yes', update='yes',
verify='no', Stdout=1)
mjdend = []
mjdstart = []
num = 0
for img in listmerge:
num = num + 1
hdrm = ntt.util.readhdr(img)
ntt.util.updateheader(outputimage, 0,
{'PROV' + str(num):
[ntt.util.readkey3(
hdrm, 'ARCFILE'), 'Originating file'],
'TRACE' + str(num): [img, 'Originating file']})
mjdend.append(ntt.util.readkey3(hdrm, 'MJD-END'))
mjdstart.append(ntt.util.readkey3(hdrm, 'MJD-OBS'))
_dateobs = ntt.util.readkey3(ntt.util.readhdr(
listmerge[np.argmin(mjdstart)]), 'DATE-OBS')
_telapse = (max(mjdend) - min(mjdstart)) * \
60. * 60 * 24. # *86400
_tmid = (max(mjdend) + min(mjdstart)) / 2
_title = str(_tmid)[0:9] + ' ' + str(ntt.util.readkey3(hdr, 'object')) + ' ' + str(
ntt.util.readkey3(hdr, 'grism')) + ' ' + \
str(ntt.util.readkey3(hdr, 'filter')) + \
' ' + str(ntt.util.readkey3(hdr, 'slit'))
ntt.util.updateheader(outputimage, 0,
{'MJD-OBS': [min(mjdstart), 'MJD start'],
'MJD-END': [max(mjdend), 'MJD end'],
'TELAPSE': [_telapse, 'Total elapsed time [days]'],
'TMID': [_tmid, '[d] MJD mid exposure'],
'TITLE': [_title, 'Dataset title'],
'DATE-OBS': [_dateobs, 'Date of observation']})
# missing: merge airmass
else:
print '\n### skip making again combined spectrum'
objectlist[_type][setup][_ID] = [outputimage]
print '\n### setup= ', setup, ' name field= ', nameobj, ' merge image= ', outputimage, '\n'
if outputimage not in outputfile:
outputfile.append(outputimage)
ntt.util.updateheader(outputimage, 0, {'FILETYPE': [
42116, 'combine 2D spectra frame']})
if _verbose:
if 'obj' in objectlist:
print objectlist['obj']
if 'stdp' in objectlist:
print objectlist['stdp']
if 'sun' in objectlist:
print objectlist['sun']
if 'vega' in objectlist:
print objectlist['vega']
if 'obj' not in objectlist.keys():
sys.exit('\n### error: no objects in the list')
sens = {}
print '\n############################################\n### extract the spectra '
# print objectlist
for setup in objectlist['obj']:
reduced = []
for _ID in objectlist['obj'][setup]:
for img in objectlist['obj'][setup][_ID]:
hdr = ntt.util.readhdr(img)
print '\n### next object\n ', img, ntt.util.readkey3(hdr, 'object')
_grism = ntt.util.readkey3(hdr, 'grism')
_exptimeimg = ntt.util.readkey3(hdr, 'exptime')
_JDimg = ntt.util.readkey3(hdr, 'JD')
imgex = ntt.util.extractspectrum(img, dv, _ext_trace, _dispersionline, _interactive, 'obj',
automaticex=_automaticex)
if imgex not in outputfile:
outputfile.append(imgex)
ntt.util.updateheader(imgex, 0, {'FILETYPE': [42107, 'extracted 1D wave calib'],
'PRODCATG': ['SCIENCE.' + ntt.util.readkey3(hdr, 'tech').upper(),
'Data product category']})
hdr = ntt.util.readhdr(imgex)
matching = [s for s in hdr.keys() if "TRACE" in s]
for imcmb in matching:
aaa = iraf.hedit(imgex, imcmb, delete='yes',
update='yes', verify='no', Stdout=1)
ntt.util.updateheader(
imgex, 0, {'TRACE1': [img, 'Originating file']})
if os.path.isfile('database/ap' + re.sub('_ex.fits', '', imgex)):
if 'database/ap' + re.sub('_ex.fits', '', imgex) not in outputfile:
outputfile.append(
'database/ap' + re.sub('_ex.fits', '', imgex))
########################### telluric standard #############
if 'sun' in objectlist and setup in objectlist['sun']:
_type = 'sun'
elif 'vega' in objectlist and setup in objectlist['vega']:
_type = 'vega'
else:
_type = 'none'
if _type in ['sun', 'vega']:
stdref = ntt.__path__[
0] + '/standard/fits/' + str(_type) + '.fits'
stdvec, airmassvec, JDvec = [], [], []
for _ID in objectlist[_type][setup]:
for std in objectlist[_type][setup][_ID]:
_airmassstd = ntt.util.readkey3(
ntt.util.readhdr(std), 'airmass')
_JDstd = ntt.util.readkey3(
ntt.util.readhdr(std), 'JD')
JDvec.append(abs(_JDstd - _JDimg))
stdvec.append(std)
airmassvec.append(_airmassstd)
stdtelluric = stdvec[np.argmin(JDvec)]
_exptimestd = ntt.util.readkey3(
ntt.util.readhdr(stdtelluric), 'exptime')
_magstd = ntt.util.readkey3(
ntt.util.readhdr(stdtelluric), 'magstd')
print '\n\n ##### closer standard for telluric corrections #### \n\n'
print stdtelluric, airmassvec[np.argmin(JDvec)]
stdtelluric_ex = ntt.util.extractspectrum(stdtelluric, dv, False, False, _interactive, 'std',
automaticex=_automaticex)
if stdtelluric_ex not in outputfile:
outputfile.append(stdtelluric_ex)
ntt.util.updateheader(stdtelluric_ex, 0, {'FILETYPE': [
42107, 'extracted 1D wave calib ']})
ntt.util.updateheader(stdtelluric_ex, 0, {'PRODCATG': [
'SCIENCE.' + ntt.util.readkey3(
ntt.util.readhdr(stdtelluric_ex), 'tech').upper(), 'Data product category']})
hdr = ntt.util.readhdr(stdtelluric_ex)
matching = [s for s in hdr.keys() if "TRACE" in s]
for imcmb in matching:
aaa = iraf.hedit(
stdtelluric_ex, imcmb, delete='yes', update='yes', verify='no', Stdout=1)
ntt.util.updateheader(stdtelluric_ex, 0, {'TRACE1': [
stdtelluric, 'Originating file']})
###########################################################
# SN tellurich calibration
imgf = re.sub('_ex.fits', '_f.fits', imgex)
imgf, senstelluric = ntt.sofispec1Ddef.calibrationsofi(imgex, stdtelluric_ex, stdref, imgf,
_interactive)
if imgf not in outputfile:
outputfile.append(imgf)
if senstelluric not in outputfile:
outputfile.append(senstelluric)
ntt.util.updateheader(imgf, 0, {'FILETYPE': [42208, '1D wave calib, tell cor.'],
# 'SNR': [ntt.util.StoN(imgf, 50),
'SNR': [ntt.util.StoN2(imgf, False),
'Average signal to noise ratio per pixel'],
'TRACE1': [imgex, 'Originating file'],
'ASSON1': [re.sub('_f.fits', '_2df.fits', imgf),
'Name of associated file'],
'ASSOC1': ['ANCILLARY.2DSPECTRUM', 'Category of associated file']})
###########################################################
imgd = ntt.efoscspec1Ddef.fluxcalib2d(
img, senstelluric) # flux calibration 2d images
ntt.util.updateheader(
imgd, 0, {'FILETYPE': [42209, '2D wavelength and flux calibrated spectrum']})
iraf.hedit(imgd, 'PRODCATG', delete='yes',
update='yes', verify='no')
hdrd = ntt.util.readhdr(imgd)
matching = [s for s in hdrd.keys() if "TRACE" in s]
for imcmb in matching:
aaa = iraf.hedit(
imgd, imcmb, delete='yes', update='yes', verify='no', Stdout=1)
ntt.util.updateheader(
imgd, 0, {'TRACE1': [img, 'Originating file']})
if imgd not in outputfile:
outputfile.append(imgd)
###############################################################
if 'stdp' in objectlist and setup in objectlist['stdp']:
print '\n ##### photometric calibration ######\n '
standardfile = []
for _ID in objectlist['stdp'][setup]:
for stdp in objectlist['stdp'][setup][_ID]:
stdp_ex = ntt.util.extractspectrum(stdp, dv, False, _dispersionline, _interactive, 'std',
automaticex=_automaticex)
standardfile.append(stdp_ex)
if stdp_ex not in outputfile:
outputfile.append(stdp_ex)
ntt.util.updateheader(stdp_ex, 0, {
'FILETYPE': [42107, 'extracted 1D wave calib'],
'TRACE1': [stdp_ex, 'Originating file'],
'PRODCATG': ['SCIENCE.' + ntt.util.readkey3(ntt.util.readhdr(stdp_ex), 'tech').upper(),
'Data product category']})
print '\n### ', standardfile, ' \n'
if len(standardfile) >= 2:
standardfile0 = raw_input(
'which one do you want to use [' + str(standardfile[0]) + '] ? ')
if not standardfile0:
standardfile0 = standardfile[0]
else:
standardfile0 = standardfile[0]
print standardfile0
stdpf = re.sub('_ex.fits', '_f.fits', standardfile0)
stdpf, senstelluric2 = ntt.sofispec1Ddef.calibrationsofi(standardfile0, stdtelluric_ex, stdref,
stdpf, _interactive)
if stdpf not in outputfile:
outputfile.append(stdpf)
ntt.util.updateheader(stdpf, 0, {'FILETYPE': [42208, '1D wave calib, tell cor'],
'TRACE1': [stdp, 'Originating file']})
stdname = ntt.util.readkey3(
ntt.util.readhdr(standardfile0), 'stdname')
standardfile = ntt.__path__[
0] + '/standard/flux/' + stdname
xx, yy = ntt.util.ReadAscii2(standardfile)
crval1 = pyfits.open(stdpf)[0].header.get('CRVAL1')
cd1 = pyfits.open(stdpf)[0].header.get('CD1_1')
datastdpf, hdrstdpf = pyfits.getdata(stdpf, 0, header=True)
xx1 = np.arange(len(datastdpf[0][0]))
aa1 = crval1 + (xx1) * cd1
yystd = np.interp(aa1, xx, yy)
rcut = np.compress(
((aa1 < 13000) | (aa1 > 15150)) & ((11700 < aa1) | (aa1 < 11000)) & (aa1 > 10000) &
((aa1 < 17800) | (aa1 > 19600)) & (aa1 < 24000), datastdpf[0][0] / yystd)
aa11 = np.compress(
((aa1 < 13000) | (aa1 > 15150)) & ((11700 < aa1) | (aa1 < 11000)) & (aa1 > 10000) &
((aa1 < 17800) | (aa1 > 19600)) & (aa1 < 24000), aa1)
yy1clean = np.interp(aa1, aa11, rcut)
aa1 = np.array(aa1)
yy1clean = np.array(yy1clean)
A = np.ones((len(rcut), 2), dtype=float)
A[:, 0] = aa11
result = np.linalg.lstsq(A, rcut) # result=[zero,slope]
p = [result[0][1], result[0][0]]
yfit = ntt.util.pval(aa1, p)
pl.clf()
pl.ion()
pl.plot(aa1, datastdpf[0][0] / yystd,
color='red', label='std')
pl.plot(aa1, yfit, color='blue', label='fit')
pl.legend(numpoints=1, markerscale=1.5)
# sens function sofi spectra
outputsens = 'sens_' + stdpf
ntt.util.delete(outputsens)
datastdpf[0][0] = yfit
pyfits.writeto(outputsens, np.float32(datastdpf), hdrstdpf)
#################
imgsc = re.sub('_ex.fits', '_sc.fits', imgex)
ntt.util.delete(imgsc)
crval2 = pyfits.open(imgf)[0].header.get('CRVAL1')
cd2 = pyfits.open(imgf)[0].header.get('CD1_1')
dataf, hdrf = pyfits.getdata(imgf, 0, header=True)
xx2 = np.arange(len(dataf[0][0]))
aa2 = crval2 + (xx2) * cd2
yyscale = np.interp(aa2, aa1, yfit)
dataf[0][0] = dataf[0][0] / yyscale
dataf[1][0] = dataf[1][0] / yyscale
dataf[2][0] = dataf[2][0] / yyscale
dataf[3][0] = dataf[3][0] / yyscale
pyfits.writeto(imgsc, np.float32(dataf), hdrf)
ntt.util.updateheader(imgsc, 0, {'SENSPHOT': [outputsens, 'sens used to flux cal'],
'FILETYPE': [42208, '1D wave,flux calib, tell cor'],
'TRACE1': [imgf, 'Originating file']})
# ntt.util.updateheader(imgsc,0,{'FILETYPE':[42208,'1D wave,flux calib, tell cor']})
# ntt.util.updateheader(imgsc,0,{'TRACE1':[imgf,'']})
print '\n### flux calibrated spectrum= ', imgf, ' with the standard= ', stdpf
if imgsc not in outputfile:
outputfile.append(imgsc)
else:
print '\n### photometric calibrated not performed \n'
print '\n### adding keywords for phase 3 ....... '
reduceddata = ntt.util.rangedata(outputfile)
f = open('logfile_spec1d_' + str(reduceddata) +
'_' + str(datenow) + '.raw.list', 'w')
for img in outputfile:
if str(img)[-5:] == '.fits':
hdr = ntt.util.readhdr(img)
# added for DR2
if 'NCOMBINE' in hdr:
_ncomb = ntt.util.readkey3(hdr, 'NCOMBINE')
else:
_ncomb = 1.0
_effron = 12. * \
(1 / np.sqrt(ntt.util.readkey3(hdr, 'ndit') * _ncomb)) * \
np.sqrt(np.pi / 2)
try:
ntt.util.phase3header(img) # phase 3 definitions
ntt.util.updateheader(img, 0, {'quality': ['Final', ''],
'EFFRON': [_effron, 'Effective readout noise per output (e-)']})
f.write(ntt.util.readkey3(
ntt.util.readhdr(img), 'arcfile') + '\n')
except:
print 'Warning: ' + img + ' is not a fits file'
f.close()
return outputfile, 'logfile_spec1d_' + str(reduceddata) + '_' + str(datenow) + '.raw.list'
def makefringingmask(listimg,
_output,
_interactive,
_combine='average',
_rejection='avsigclip'):
# print "LOGX:: Entering `makefringingmask` method/function in
# %(__file__)s" % globals()
import ntt
from ntt.util import readhdr, readkey3, delete, updateheader
import glob
import os
import sys
import re
import string
from pyraf import iraf
iraf.noao(_doprint=0)
iraf.immatch(_doprint=0)
iraf.imutil(_doprint=0)
iraf.nproto(_doprint=0)
iraf.proto(_doprint=0)
toforget = ['nproto.objmasks', 'proto.fixpix']
for t in toforget:
iraf.unlearn(t)
if _interactive == True:
listimg2 = []
for img in listimg:
_exptime = readkey3(readhdr(img), 'exptime')
if float(_exptime) >= 10:
answ = 'xxx'
while answ.lower() not in ['y', 'n', 's', 'a']:
iraf.display(img, frame=1, fill='yes')
answ = raw_input(
'use this image (yes,no,stop (not more images),all) [[y]/n/s/a] ? '
)
if not answ:
answ = 'y'
if answ.lower() == 'y':
listimg2.append(img)
elif answ.lower() == 'a':
listimg2 = listimg[:]
if answ.lower() in ['a', 's']:
break
listimg = listimg2[:]
iraf.nproto.objmasks1.fitxord = 1
iraf.nproto.objmasks1.fityord = 1
hdr0 = readhdr(listimg[0])
_date = readkey3(hdr0, 'date-obs')
_filter = readkey3(hdr0, 'filter')
_exptime = readkey3(hdr0, 'exptime')
_instrume = readkey3(hdr0, 'instrume')
_ron = readkey3(hdr0, 'ron')
_gain = readkey3(hdr0, 'gain')
badpixelmask = 'bad_pixel_mask.pl'
if not os.path.isfile(badpixelmask):
os.system('cp ' + ntt.__path__[0] + '/archive/' + _instrume +
'/badpixels/badpixel_20100210.pl ' + badpixelmask)
ff = open('_listmask', 'w')
hh = open('_listobgz', 'w')
for img in listimg:
_exptime = readkey3(readhdr(img), 'exptime')
hh.write('z_' + img + '\n')
ff.write('mask_' + img + '\n')
delete('mask_' + img)
aaa = iraf.hedit(img,
delete='yes',
field='OBJMASK',
up='yes',
verify='no',
Stdout=1)
aaa = iraf.hedit(img,
delete='yes',
field='BPM',
up='yes',
verify='no',
Stdout=1)
delete('z_' + img)
iraf.imutil.imexpr(expr='(a - median(a))/' + str(_exptime),
a=img,
output='z_' + img,
verbose='no')
ntt.util.updateheader('z_' + img, 0, {'EXPTIME': [1, '']})
ff.close()
hh.close()
if not _output:
_output = 'fringing_' + str(_date) + '_' + str(_filter) + '.fits'
delete(_output)
print ' making mask for each frame .......'
ccc = iraf.nproto.objmasks(images='@_listobgz',
objmasks='@_listmask',
omtype='boolean',
blksize=-16,
convolv='block 3 3',
hsigma=5,
lsigma=3,
minpix=10,
ngrow=2,
agrow=4.,
Stdout=1)
print 'combining all frames, masking the objects .....'
iraf.imcombine('@_listobgz',
output=_output,
masktyp='!OBJMASK',
maskval=0,
combine=_combine,
reject=_rejection,
scale='none',
statsec='[100:800,100:800]',
rdnoise='',
gain='',
nlow=1,
nhigh=1,
logfile='imcombinelog')
ntt.util.phase3header(_output)
ntt.util.updateheader(_output, 0,
{'BUNIT': ['ADU', 'pixel units(ADU,electrons)']})
ntt.util.updateheader(_output, 0, {'FILETYPE': [11231, 'fringing frame']})
return _output
""" This file is meant to be run by itself, demonstrating the Python-to-IRAF
API for use in testing, starting with the very simplest of function calls.
This test is useful in special installations, such as one without IRAF. """
import sys
print("Simple #1 - about to: from pyraf import iraf ...")
from pyraf import iraf
print("Simple #2 - about to import pyraf ...") # should be essentially a no-op
import pyraf
print("Simple #3 - ver is ...")
print(pyraf.__version__)
print("Simple #4 - pwd is ...")
iraf.pwd() # will print
print("Simple #5 - has IRAF is ...")
from pyraf import irafinst
print(str(irafinst.EXISTS))
if not irafinst.EXISTS:
sys.exit(0)
print("Simple #6 - files output is ... (the rest require IRAF)")
iraf.files('file_a.txt,file-b.txt,file.c.txt,,filed.txt')
print("Simple #7 - loading imutil")
iraf.images(_doprint=0) # load images
iraf.imutil(_doprint=0) # load imutil
print("Simple #8 - imaccess to dev$pix is ...")
print(iraf.imaccess("dev$pix"))
print("Simple #9 - imheader of dev$pix is ...")
iraf.imheader("dev$pix")
sys.exit(0)
""" This file is meant to be run by itself, demonstrating the Python-to-IRAF
API for use in testing, starting with the very simplest of function calls.
This test is useful in special installations, such as one without IRAF. """
import sys
print("Simple #1 - about to: from pyraf import iraf ...")
from pyraf import iraf
print("Simple #2 - about to import pyraf ...") # should be essentially a no-op
import pyraf
print("Simple #3 - ver is ...")
print(pyraf.__version__)
print("Simple #4 - pwd is ...")
iraf.pwd() # will print
print("Simple #5 - has IRAF is ...")
from pyraf import irafinst
print(str(irafinst.EXISTS))
if not irafinst.EXISTS:
sys.exit(0)
print("Simple #6 - files output is ... (the rest require IRAF)")
iraf.files('file_a.txt,file-b.txt,file.c.txt,,filed.txt')
print("Simple #7 - loading imutil")
iraf.images(_doprint=0) # load images
iraf.imutil(_doprint=0) # load imutil
print("Simple #8 - imaccess to dev$pix is ...")
print(iraf.imaccess("dev$pix"))
print("Simple #9 - imheader of dev$pix is ...")
iraf.imheader("dev$pix")
sys.exit(0)
import pyfits #import numarray import numpy import time from types import * # Local packages import add_wcs import iqpkg from iqutils import * # Necessary IRAF packages iraf.imred() iraf.ccdred() iraf.images() iraf.imutil() iraf.imfit() iraf.proto() # Shortcuts yes=iraf.yes no=iraf.no INDEF=iraf.INDEF hedit=iraf.hedit imgets=iraf.imgets # Pyraf parameters: Where to find them pyrafdir="python/pyraf/" pyrafdir_key='PYRAFPARS' if os.environ.has_key(pyrafdir_key):
def psfphot(inlist,
ra,
dec,
reffilt,
interact,
fwhm,
readnoise,
gain,
threshold,
refimage=None,
starfile=None,
maxnpsf=5,
clobber=globclob,
verbose=globver,
skykey='SKYBKG',
filtkey='FILTER',
pixtol=3.0):
""" perform PSF-based photometry on a single target star (SN?) at RA, Dec and
also on a set of comparison stars, using daophot. simultaneously
perform aperture photometry on all the comparison stars (after
subtracting off contributions from neighbors) to enable absolute
photometry by comparison to aperture photometry of standard stars
observed in other fields """
# Defaults / constants
psfmult = 5.0 #standard factor (multiplied by fwhm to get psfradius)
psfmultsmall = 3.0 #similar to psfmult, adjusted for nstar and substar
# Necessary package
iraf.imutil()
# Parse inputs
infiles = iraffiles(inlist)
# Which file is reffilt? call it refimage
if refimage == None:
for image in infiles:
if check_head(image, filtkey):
try:
imgfilt = get_head(image, filtkey)
if imgfilt == reffilt:
refimage = image
break
except:
pass
if not refimage:
print "BAD USER! No image corresponds to the filter: %s" % reffilt
return
else:
refroot = 's' + refimage.split('.')[0]
#first make sure to add back in background of sky
iraf.iqsubsky(inlist, sub=no, skykey=skykey)
#put reference image first on list
infiles.remove(refimage)
infiles.insert(0, refimage)
#setup for keywords
if gain == "!GAIN":
try:
gainval = float(get_head(image, gain))
except:
print "Bad header keyword for gain."
else:
gainval = float(gain)
if readnoise == "!READNOISE":
try:
readval = float(get_head(image, readnoise))
except:
print "Bad header keyword for readnoise."
else:
readval = float(readnoise)
# Process each file in turn
for image in infiles:
# Check that the image is there
check_exist(image, "r")
# Grab image root name
root = image.split('.')[0]
# Map image to reference image
if not (image == refimage):
[nx, ny] = get_head(image, ['NAXIS1', 'NAXIS2'])
stars = Starlist(get_head(image, 'STARFILE'))
refstars = Starlist(get_head(refimage, 'STARFILE'))
refstars.pix2wcs(refimage)
refstars.wcs2pix(image)
match, refmatch = stars.match(refstars, useflags=yes, tol=10.0)
nstars = len(match)
if not (nstars > 2):
print 'Could not find star matches between reference and %s' % image
infiles.remove(image)
continue
refmatch.pix2wcs(image)
refmatch.wcs2pix(refimage)
matchfile = open('%s.match' % root, 'w')
for i in range(len(match)):
matchfile.write('%10.3f%10.3f%10.3f%10.3f\n' %
(refmatch[i].xval, refmatch[i].yval,
match[i].xval, match[i].yval))
matchfile.close()
check_exist('%s.geodb' % root, 'w', clobber=clobber)
iraf.geomap('%s.match' % root,
'%s.geodb' % root,
1.0,
nx,
1.0,
ny,
verbose=no,
interactive=no)
check_exist('s%s.fits' % root, 'w', clobber=clobber)
iraf.geotran(image,
's%s' % root,
'%s.geodb' % root,
'%s.match' % root,
geometry="geometric",
boundary="constant",
verbose=no)
else:
iraf.imcopy(image, 's%s' % root)
root = 's%s' % root
#get sky level and calculate sigma
#if check_head(image, skykey):
# try:
# sky=float(get_head(image, skykey))
# except:
# print "No sky levels in header."
#sigma= (((sky * gainval) + readval**2)**.5) / gainval
iraf.iterstat(image)
# Saturation level
if not check_head(image, "SATURATE"):
saturate = 60000.0
else:
saturate = get_head(image, "SATURATE")
# Update datapars and daopars
iraf.datapars.fwhmpsf = fwhm
iraf.datapars.sigma = iraf.iterstat.sigma
iraf.datapars.datamin = iraf.iterstat.median - 10 * iraf.iterstat.sigma
iraf.datapars.datamax = 0.90 * saturate
iraf.datapars.readnoise = readval
iraf.datapars.epadu = gainval
iraf.datapars.filter = filtkey
iraf.daopars.psfrad = psfmult * fwhm
iraf.daopars.fitrad = fwhm
iraf.daopars.function = "gauss,moffat15,moffat25,lorentz,penny1"
#find stars in image unless a starlist is given
if image == refimage and starfile == None:
iraf.daophot.daofind(root,
'refimage.coo.1',
threshold=threshold,
verify=no,
verbose=verbose)
elif image == refimage:
shutil.copy(starfile, 'refimage.coo.1')
#initial photometry
iraf.daophot.phot(root,
'refimage.coo.1',
'default',
aperture=fwhm,
verify=no,
verbose=verbose)
#select stars for psf the first time
refstarsfile = "refimage.pst.1"
if image == refimage:
iraf.pstselect(root,
'default',
refstarsfile,
maxnpsf,
interactive=yes,
verify=no,
verbose=verbose)
#fit the psf
iraf.psf(root,
'default',
refstarsfile,
'default',
'default',
'default',
interactive=interact,
verify=no,
verbose=verbose)
#identify neighboring/interfering stars to selected stars
groupingfile = root + ".psg.1"
iraf.nstar(root,
groupingfile,
'default',
'default',
'default',
psfrad=psfmultsmall * fwhm,
verify=no,
verbose=verbose)
#subtract out neighboring stars from image
iraf.substar(root,
'default',
refstarsfile,
'default',
'default',
psfrad=psfmultsmall * fwhm,
verify=no,
verbose=verbose)
#repeat psf to get better psf model
#IRAF's interactive version usually crashes
subtractedimage = root + ".sub.1"
iraf.psf(subtractedimage,
root + ".nst.1",
refstarsfile,
'%s.psf.2' % root,
'%s.pst.2' % root,
'%s.psg.2' % root,
interactive=interact,
verify=no,
verbose=verbose)
#Need to make sure SN was detected by daofind
stars = Starlist('%s.mag.1' % root)
SN = Star(name='SN', radeg=ra, dcdeg=dec, fwhm=2.0, fwhmw=2.0)
SNlis = Starlist(stars=[SN])
SNlis.wcs2pix(image)
if (len(stars.match(SNlis)[0]) == 0):
#No match - need to add to daofind file
print "No match!"
coofile = open('refimage.coo.1', 'a+')
coofile.write('%10.3f%10.3f%9.3f%8.3f%13.3f%12.3f%8i\n' %
(SNlis[0].xval, SNlis[0].yval, 99.999, 0.500, 0.000,
0.000, 999))
coofile.close()
#repeat aperture photometry to get good comparisons to standard fields
iraf.daophot.phot(root,
'refimage.coo.1',
'default',
aperture=psfmult * fwhm,
verify=no,
verbose=verbose)
# allstar run
iraf.allstar(root,
'default',
'default',
'default',
'default',
'default',
verify=no,
verbose=verbose)
db = KirbyBase()
images = db.select(
imgdb, [deckeyfilenum], ['\d\d*'],
returnType='dict',
useRegExp=True,
sortFields=[
'setname', 'mjd'
]) # the \d\d* is a trick to select both 0000-like and 000-like
# we select all images that have a deckeyfilenum
print "I've selected", len(images), "images."
proquest(askquestions)
iraf.imutil()
iraf.unlearn(iraf.imutil.imarith)
for image in images:
print image[deckeyfilenum], image['imgname']
if decnormfieldname == "None":
mycoeff = 1.0 # in this case the user *wants* to make a deconvolution without normalization.
else:
mycoeff = image[decnormfieldname]
if mycoeff == None:
print "WARNING : no coeff available, using 1.0"
mycoeff = 1.0
print mycoeff
def makefringingmask(listimg, _output, _interactive, _combine='average', _rejection='avsigclip'):
# print "LOGX:: Entering `makefringingmask` method/function in
# %(__file__)s" % globals()
import ntt
from ntt.util import readhdr, readkey3, delete, updateheader
import glob
import os
import sys
import re
import string
from pyraf import iraf
iraf.noao(_doprint=0)
iraf.immatch(_doprint=0)
iraf.imutil(_doprint=0)
iraf.nproto(_doprint=0)
iraf.proto(_doprint=0)
toforget = ['nproto.objmasks', 'proto.fixpix']
for t in toforget:
iraf.unlearn(t)
if _interactive == True:
listimg2 = []
for img in listimg:
_exptime = readkey3(readhdr(img), 'exptime')
if float(_exptime) >= 10:
answ = 'xxx'
while answ.lower() not in ['y', 'n', 's', 'a']:
iraf.display(img, frame=1, fill='yes')
answ = raw_input(
'use this image (yes,no,stop (not more images),all) [[y]/n/s/a] ? ')
if not answ:
answ = 'y'
if answ.lower() == 'y':
listimg2.append(img)
elif answ.lower() == 'a':
listimg2 = listimg[:]
if answ.lower() in ['a', 's']:
break
listimg = listimg2[:]
iraf.nproto.objmasks1.fitxord = 1
iraf.nproto.objmasks1.fityord = 1
hdr0 = readhdr(listimg[0])
_date = readkey3(hdr0, 'date-obs')
_filter = readkey3(hdr0, 'filter')
_exptime = readkey3(hdr0, 'exptime')
_instrume = readkey3(hdr0, 'instrume')
_ron = readkey3(hdr0, 'ron')
_gain = readkey3(hdr0, 'gain')
badpixelmask = 'bad_pixel_mask.pl'
if not os.path.isfile(badpixelmask):
os.system('cp ' + ntt.__path__[0] + '/archive/' + _instrume +
'/badpixels/badpixel_20100210.pl ' + badpixelmask)
ff = open('_listmask', 'w')
hh = open('_listobgz', 'w')
for img in listimg:
_exptime = readkey3(readhdr(img), 'exptime')
hh.write('z_' + img + '\n')
ff.write('mask_' + img + '\n')
delete('mask_' + img)
aaa = iraf.hedit(img, delete='yes', field='OBJMASK',
up='yes', verify='no', Stdout=1)
aaa = iraf.hedit(img, delete='yes', field='BPM',
up='yes', verify='no', Stdout=1)
delete('z_' + img)
iraf.imutil.imexpr(expr='(a - median(a))/' + str(_exptime),
a=img, output='z_' + img, verbose='no')
ntt.util.updateheader('z_' + img, 0, {'EXPTIME': [1, '']})
ff.close()
hh.close()
if not _output:
_output = 'fringing_' + str(_date) + '_' + str(_filter) + '.fits'
delete(_output)
print ' making mask for each frame .......'
ccc = iraf.nproto.objmasks(images='@_listobgz', objmasks='@_listmask', omtype='boolean',
blksize=-16, convolv='block 3 3', hsigma=5, lsigma=3, minpix=10, ngrow=2, agrow=4., Stdout=1)
print 'combining all frames, masking the objects .....'
iraf.imcombine('@_listobgz', output=_output, masktyp='!OBJMASK', maskval=0, combine=_combine, reject=_rejection,
scale='none', statsec='[100:800,100:800]', rdnoise='', gain='', nlow=1, nhigh=1, logfile='imcombinelog')
ntt.util.phase3header(_output)
ntt.util.updateheader(
_output, 0, {'BUNIT': ['ADU', 'pixel units(ADU,electrons)']})
ntt.util.updateheader(_output, 0, {'FILETYPE': [11231, 'fringing frame']})
return _output
def load_modules():
# Define a function to load all of the modules so that they don't' import
# unless we need them
global iraf
from pyraf import iraf
iraf.pysalt()
iraf.saltspec()
iraf.saltred()
iraf.set(clobber='YES')
global sys
import sys
global os
import os
global shutil
import shutil
global glob
from glob import glob
global pyfits
import pyfits
global np
import numpy as np
global lacosmicx
import lacosmicx
global interp
from scipy import interp
global signal
from scipy import signal
global ndimage
from scipy import ndimage
global interpolate
from scipy import interpolate
global WCS
from astropy.wcs import WCS
global optimize
from scipy import optimize
global ds9
import ds9
global GaussianProcess
from sklearn.gaussian_process import GaussianProcess
global pandas
import pandas
iraf.onedspec()
iraf.twodspec()
iraf.longslit()
iraf.apextract()
iraf.imutil()
def psfphot(image,
clobber=globclob,
verbose=globver,
pixtol=3.0,
maxnpsf=5,
interact=yes):
""" perform PSF-based photometry on a single target star (SN?) at RA, Dec and
also on a set of comparison stars, using daophot. simultaneously
perform aperture photometry on all the comparison stars (after
subtracting off contributions from neighbors) to enable absolute
photometry by comparison to aperture photometry of standard stars
observed in other fields """
# Defaults / constants
psfmult = 5.0 #standard factor (multiplied by fwhm to get psfradius)
psfmultsmall = 3.0 #similar to psfmult, adjusted for nstar and substar
# Necessary package
iraf.imutil()
# Detect stars
iqpkg.iqobjs(image, 3.0, 50000.0, wtimage="", skyval="!MEDSKY")
root = image[:-5]
[gain, rnoise, fwhm] = get_head(image, ["GAIN", "READNOI", "SEEPIX"])
fwhm = float(fwhm)
rnoise = float(rnoise)
iraf.iterstat(image)
# Saturation level
if not check_head(image, "SATURATE"):
saturate = 60000.0
else:
saturate = get_head(image, "SATURATE")
# Update datapars and daopars
iraf.datapars.fwhmpsf = fwhm
iraf.datapars.sigma = iraf.iterstat.sigma
iraf.datapars.datamin = iraf.iterstat.median - 10 * iraf.iterstat.sigma
iraf.datapars.datamax = 70000.0
iraf.datapars.readnoise = rnoise
iraf.datapars.epadu = gain
iraf.daopars.psfrad = psfmult * fwhm
iraf.daopars.fitrad = fwhm
iraf.daopars.function = "gauss,moffat15,moffat25,lorentz,penny1"
# coo file
stars = Starlist("%s.stars" % image)
outf = open("%s.coo.1" % image[:-5], "w")
for star in stars:
outf.write("%10.3f%10.3f\n" % (star.xval, star.yval))
outf.close()
#initial photometry
iraf.daophot.phot(root,
'default',
'default',
aperture=fwhm,
verify=no,
verbose=verbose)
iraf.datapars.datamax = 30000.0
iraf.pstselect(root,
'default',
'default',
maxnpsf,
interactive=yes,
verify=no,
verbose=verbose)
iraf.psf(root,
'default',
'default',
'default',
'default',
'default',
interactive=interact,
verify=no,
verbose=verbose)
iraf.allstar(root,
'default',
'default',
'default',
'default',
'default',
verify=no,
verbose=verbose)
iraf.iterstat("%s.sub.fits" % root)
iraf.datapars.sigma = iraf.iterstat.sigma
iraf.datapars.datamin = iraf.iterstat.median - 10 * iraf.iterstat.sigma
iraf.datapars.datamax = 70000.0
iraf.daophot.phot("%s.sub.fits" % root,
"SN.coo",
'default',
'default',
aperture=fwhm,
verify=no,
verbose=verbose)
iraf.datapars.datamax = 30000.0
iraf.daopars.fitrad = fwhm * 2.0
iraf.allstar("%s.sub.fits" % root,
'default',
"%s.psf.1.fits" % root,
'default',
'default',
'default',
verify=no,
verbose=no)
def calibphot(inlist, fwhm,
rdnoise='2.6', gain='2.5',
interact=yes, config='',
objkey="OBJECT", airkey="AIRMASS",
filtkey='FILTER', skykey='SKYBKG', phosfx="pho",
possfx="pos",
clobber=globclob, verbose=globver):
""" perform aperture photometry on a standard field to fit the
transformation equations. these transformation equations will
then be applied to comparison stars' aperture photometries
outputted by psfphot.py, in order to find the true magnitudes of
the comparison stars. this will then provide and apply the
corrections to the magnitude of a single target star (SN?). """
# Defaults / constants
alpha=list('abcdefghijklmnopqrstuvwxyz')
psfmult=5 #standard factor (multiplied by fwhm to get psfradius)
imgprop={}
imglist={}
allobjs={}
allfilt={}
# Necessary package
iraf.imutil()
# Parse gain/readnoise inputs
gainkey=None
re1=re.search('^!(.+)',gain)
re2=re.search('^[\d\.]+',gain)
if re1:
gainkey=re1.group(1)
elif re2:
try:
gainval=float(gain)
except:
print "Error converting gain"
return
else:
print "Error finding gain"
return
rdnskey=None
re1=re.search('^!(.+)',rdnoise)
re2=re.search('^[\d\.]+',rdnoise)
if re1:
rdnskey=re1.group(1)
elif re2:
try:
readval=float(rdnoise)
except:
print "Error converting readnoise"
return
else:
print "Error finding readnoise"
return
# Parse inputs
infiles=iraffiles(inlist)
#first make sure to add back in background of sky
iraf.iqsubsky(inlist, sub=no, skykey=skykey)
# Process each file in turn
for image in infiles:
# Check that the image is there
check_exist(image,"r")
# Grab all the useful keywords
[sky,objname,filter,airmass] = \
get_head(image,[skykey,objkey,filtkey,airkey])
# Get gain & readnoise, if necessary
if gainkey:
gainval=get_head(image,gainkey)
if rdnskey:
readval=get_head(image,rdnskey)
# Calculate theoretical sigma of sky
if len(str(sky))>0:
sigma= (((sky * gainval) + readval**2)**.5) / gainval
else:
print "Failed to retrieve sky value from image header"
return
# check standard field name
if len(objname)==0:
print "Failed to retrieve object name from image header"
return
# Generate posfile and phofile names
posfile=objname+'.'+possfx
phofile=objname+'.'+phosfx
# check filter name
if len(filter)==0:
print "Failed to retrieve filter name from image header"
return
# Check airmass value
if len(airmass)>0:
try:
airval=float(airmass)
except:
print "Failed to convert airmass value to float"
return
else:
print "Failed to retrieve airmass value from image header"
return
# Save keyword values for this image
imgprop[image]=[gainval,readval,sky,objname,filter,airmass]
# Add dictionary entries for the current image
listkey=objname+'-'+filter
if imglist.has_key(listkey):
imglist[listkey].append(image)
else:
imglist[listkey]=[image]
# Keep track of all filters/object fields
allobjs[objname]=1
allfilt[filter]=1
#add in extra range of aperture radii, leave in background?
fwhmpsf=float(fwhm)
ap1 = fwhmpsf
ap15 = 1.5 * fwhmpsf
ap2 = 2 * fwhmpsf
ap25 = 2.5 * fwhmpsf
ap3 = 3 * fwhmpsf
innersky = 3.5 * fwhmpsf
#do photometry on all stars in field
check_exist(image+".mag.std.1", "w", clobber=yes)
iraf.phot(image, output = image+".mag.std.1", coords=posfile,
sigma=sigma, datamin = sky - 3.5*sigma, apertures = ap25,
dannulus="10", annulus=innersky, verify="no")
##############################
#make imsets file
imsetsfile="stdfield.dat"
imsetstuff = open(imsetsfile,"w")
for objname in allobjs.keys():
maxnum=0
for key in imglist.keys():
if re.search(objname,key):
maxnum=max(maxnum,len(imglist[key]))
for i in range(maxnum):
imset=objname+alpha[i]+" :"
for key in imglist.keys():
if re.search(objname,key):
if i<len(imglist[key]):
imset+=" "+imglist[key][i]
imsetstuff.write(imset+"\n")
imsetstuff.close()
#make observations file (specify aperture number?)
obsfile = "standobs"
check_exist(obsfile, "w", clobber=yes)
iraf.mknobsfile(photfiles = "*.mag.std.1",
idfilters = " ".join(allfilt.keys()),
imsets = imsetsfile, observations = obsfile,
tolerance="500")
#rename stars in catalog file so they agree, print good stars to new file
catpholines = getlines(phofile)
s=1
newcatpholines = [catpholines[0]]
lastline = catpholines[-1]
lastlinedata = lastline.split()
digits = len(lastlinedata[0])+1
for line in catpholines:
if re.search("vary?", line):
continue
ess=str(s)
idlength=len(objname)+len(ess)+1
numspace= digits - idlength + 1
newline = " "*numspace+objname+"-"+ess+line[digits:]
s=s+1
if not re.search("99.999",newline):
newcatpholines.append(newline)
newcatdata = "\n".join(newcatpholines)
newphofile = phofile+".2"
newphostuff = open(phofile+".2", "w")
newphostuff.write(newcatdata)
newphostuff.close()
#rename stars in standobs to be field-# instead of fielda-#, fieldb-#, etc
#what is field?
obsdatalines=getlines(obsfile)
for i in range(len(obsdatalines)):
line=obsdatalines[i]
for objname in allobjs.keys():
if re.search(objname,line):
index=line.index(objname)
lline=list(line)
lline[index:index+len(objname)+1]=list(" "+objname)
line="".join(lline)
obsdatalines[i]=line
break
newobsdata = "\n".join(obsdatalines)
fixobsfile = "fix"+obsfile
fixobsstuff = open(fixobsfile, "w")
fixobsstuff.write(newobsdata)
fixobsstuff.close()
#if no config file exists, make one
#first, make a file that has stetson catalog formats in it
if not config:
newconfig="fcat.dat"
check_exist(newconfig, "w", clobber=yes)
catformat=open(newconfig, "w")
catinfo = ["#Declare the catalog variables", " ", "catalog", " ",
" B\t2", "Berror\t3", "V\t6", "Verror\t7",
"R\t10", "Rerror\t11", "I\t14", "Ierror\t15"]
catdata = "\n".join(catinfo)
catformat.write(catdata)
catformat.close()
config="cat.cfg"
transfile="ftrans.dat"
check_exist(transfile, "w", clobber=yes)
transstuff=open(transfile, "w")
transinfo = ["transformation", " ", "fit v1 = 0, v2 = 0.17, v3= 0", "fit b1 = 0, b2 = 0.35, b3= 0", "fit r1 = 0, r2 = 0.08, r3= 0", " ", "const v4 = 0", "const b4 = 0", "const r4 = 0", " ", "Vfit : V = mV + v1 + v2*XV + v3*(mB-mV) + v4*(mB-mV)*XV", "Bfit : B = mB + b1 + b2*XB + b3*(mB-mV) + b4*(mB-mV)*XB", "Rfit : R = mR + r1 + r2*XR + r3*(mV-mR) + r4*(mV-mR)*XR"]
transdata = "\n".join(transinfo)+"\n"
transstuff.write(transdata)
transstuff.close()
check_exist(config, "w", clobber=yes)
iraf.mkconfig(config, catalog = newconfig,
observations = "f"+obsfile+".dat",
transform=transfile, check=no, edit=no)
#fit parameters of transformation equations (specify aperture in ansfile?)
ansfile = objname+".ans.1"
check_exist(ansfile, "w", clobber=yes)
iraf.fitparams(observations = fixobsfile, catalogs = newphofile,
config=config, parameters = ansfile, interactive = interact)
#apply to standard stars to check to make sure fits are okay?
calibfile = objname+".calib.1"
check_exist(calibfile, "w", clobber=yes)
iraf.evalfit(observations = fixobsfile, config = config,
parameters = ansfile, calib = calibfile)
def makeflat(lista):
# print "LOGX:: Entering `makeflat` method/function in %(__file__)s" %
# globals()
flat = ''
import datetime
import glob
import os
import ntt
from ntt.util import readhdr, readkey3, delete, name_duplicate, updateheader, correctcard
from pyraf import iraf
iraf.images(_doprint=0, Stdout=0)
iraf.imutil(_doprint=0, Stdout=0)
iraf.imgeom(_doprint=0, Stdout=0)
# iraf.blkavg(_doprint=0, Stdout=0)
iraf.noao(_doprint=0, Stdout=0)
iraf.imred(_doprint=0, Stdout=0)
iraf.generic(_doprint=0, Stdout=0)
toforget = [
'imgeom.blkavg', 'imutil.imarith', 'immatch.imcombine', 'noao.imred'
]
for t in toforget:
iraf.unlearn(t)
import datetime
MJDtoday = 55927 + (datetime.date.today() -
datetime.date(2012, 01, 01)).days
_date = readkey3(readhdr(lista[0]), 'date-night')
_filter = readkey3(readhdr(lista[0]), 'filter')
output = name_duplicate(
lista[3],
'flat_' + str(_date) + '_' + str(_filter) + '_' + str(MJDtoday), '')
if os.path.isfile(output):
answ = raw_input('file already prooduced, do again [y/[n]] ? ')
if not answ:
answ = 'n'
else:
answ = 'y'
if answ in ['yes', 'y', 'YES', 'Y', 'Yes']:
delete(
"temp_off.fits,temp_off_mask.fits,temp_on_mask.fits,temp_on.fits")
iraf.image.immatch.imcombine(lista[0] + ',' + lista[7],
output="temp_off.fits")
iraf.image.immatch.imcombine(lista[1] + ',' + lista[6],
output="temp_off_mask.fits")
iraf.image.immatch.imcombine(lista[2] + ',' + lista[5],
output="temp_on_mask.fits")
iraf.image.immatch.imcombine(lista[3] + ',' + lista[4],
output="temp_on.fits")
# create the bias correction for the flat-on according to the
# Lidman technique0
delete(
"temp_onA.fits,temp_onC.fits,temp_onB.fits,temp_onAC.fits,temp_onACB.fits,temp_onACB_2D.fits"
)
delete("temp_on_bias.fits")
iraf.imgeom.blkavg(input="temp_on.fits[500:600,*]",
output="temp_onA.fits",
option="average",
b1=101,
b2=1)
iraf.imgeom.blkavg(input="temp_on_mask.fits[500:600,*]",
output="temp_onC.fits",
option="average",
b1=101,
b2=1)
iraf.imgeom.blkavg(input="temp_on_mask.fits[50:150,*]",
output="temp_onB.fits",
option="average",
b1=101,
b2=1)
iraf.imutil.imarith("temp_onA.fits", "-", "temp_onC.fits",
"temp_onAC.fits")
iraf.imutil.imarith("temp_onAC.fits", "+", "temp_onB.fits",
"temp_onACB.fits")
iraf.imgeom.blkrep(input="temp_onACB.fits",
output="temp_onACB_2D.fits",
b1=1024,
b2=1)
iraf.imutil.imarith("temp_on.fits", "-", "temp_onACB_2D.fits",
"temp_on_bias.fits")
# same as above for the flat-off
delete(
"temp_offA.fits,temp_offC.fits,temp_offB.fits,temp_offAC.fits,temp_offACB.fits,temp_offACB_2D.fits"
)
delete("temp_off_bias.fits")
iraf.imgeom.blkavg(input="temp_off.fits[500:600,*]",
output="temp_offA.fits",
option="average",
b1=101,
b2=1)
iraf.imgeom.blkavg(input="temp_off_mask.fits[500:600,*]",
output="temp_offC.fits",
option="average",
b1=101,
b2=1)
iraf.imgeom.blkavg(input="temp_off_mask.fits[50:150,*]",
output="temp_offB.fits",
option="average",
b1=101,
b2=1)
iraf.imutil.imarith("temp_offA.fits", "-", "temp_offC.fits",
"temp_offAC.fits")
iraf.imutil.imarith("temp_offAC.fits", "+", "temp_offB.fits",
"temp_offACB.fits")
iraf.imgeom.blkrep(input="temp_offACB.fits",
output="temp_offACB_2D.fits",
b1=1024,
b2=1)
iraf.imutil.imarith("temp_off.fits", "-", "temp_offACB_2D.fits",
"temp_off_bias.fits")
# create the corrected flat-field
# output=name_duplicate("temp_on_bias.fits",'flat_'+str(_date)+'_'+str(_filter)+'_'+str(MJDtoday),'')
output = name_duplicate(
lista[3],
'flat_' + str(_date) + '_' + str(_filter) + '_' + str(MJDtoday),
'')
# print lista[0],'flat_'+str(_date)+'_'+str(_filter)+'_'+str(MJDtoday)
delete(output)
iraf.imutil.imarith("temp_on_bias.fits", "-", "temp_off_bias.fits",
output)
iraf.noao.imred.generic.normalize(output) # normalize the flat-field
correctcard(output)
delete("temp_on*.fits") # delete the temporary images
delete("temp_off*.fits")
print 'flat -> ' + str(output)
else:
print 'skip redoing the flat'
return output
#! /usr/local/bin/python
import os, sys
import numpy as np
from pyraf import iraf
from odi_calibrate import download_sdss, js_calibrate
from sdss_fit import getVabs
from collections import OrderedDict
iraf.images(_doprint=0)
iraf.tv(_doprint=0)
iraf.ptools(_doprint=0)
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.photcal(_doprint=0)
iraf.apphot(_doprint=0)
iraf.imutil(_doprint=0)
def getHImass(object, dm):
# print object, mpc
uchvcdb = os.path.dirname(os.path.abspath(__file__))+'/predblist.sort.csv'
name, mass = np.loadtxt(uchvcdb, usecols=(1,6), dtype=str, delimiter=',', unpack=True)
# find the right row
coord = [i for i,this in enumerate(mass) if object.upper() in name[i]][0]
# print 'the HI mass of', name[coord], 'is', mass[coord], 'at 1 Mpc'
# mpc = mpc/1000.
mpc = pow(10,((dm + 5.)/5.))/1000000.
logm = float(mass[coord])
mass = mpc*mpc*10**logm # make sure to scale by the distance in Mpc^2
print '{:3.1f}'.format(np.log10(mass))
def makeillumination(lista, flatfield): #,outputfile,illum_frame):
import os, glob, string, re
from astropy.io import fits as pyfits
import ntt
from ntt.util import readhdr, readkey3, delete, display_image, defsex, name_duplicate, correctcard
from numpy import compress, array, argmax, argmin, min, argsort, float32
import datetime
MJDtoday = 55927 + (datetime.date.today() -
datetime.date(2012, 01, 01)).days
_date = readkey3(readhdr(lista[0]), 'date-night')
_filter = readkey3(readhdr(lista[0]), 'filter')
illum_frame = name_duplicate(
lista[0], 'illum_' + _date + '_' + _filter + '_' + str(MJDtoday), '')
from pyraf import iraf
iraf.images(_doprint=0, Stdout=0)
iraf.imutil(_doprint=0, Stdout=0)
iraf.utilities(_doprint=0, Stdout=0)
iraf.noao(_doprint=0, Stdout=0)
iraf.imred(_doprint=0, Stdout=0)
iraf.ccdred(_doprint=0, Stdout=0)
iraf.digiphot(_doprint=0, Stdout=0)
iraf.daophot(_doprint=0, Stdout=0)
iraf.generic(_doprint=0, Stdout=0)
toforget = [
'digiphot.daophot', 'imutil.imarith', 'image', 'utilities.surfit'
]
for t in toforget:
iraf.unlearn(t)
n = len(lista)
# start loop to read image names from the input file
lista1 = []
iraf.ccdred.verbose = 'no'
ff = open('templist.lst', 'w')
for i in range(0, len(lista)):
ff.write('C' + lista[i] + '\n')
delete('C' + lista[i])
delete('C' + re.sub('.fits', '_sub.fits', lista[i]))
ntt.sofiphotredudef.crosstalk(lista[i], 'C' + lista[i])
iraf.noao.imred.ccdred.ccdproc('C' + lista[i],
output='',
overscan="no",
trim="yes",
ccdtype='',
darkcor='no',
fixpix='no',
zerocor="no",
flatcor='yes',
illumco='no',
trimsec='[1:1024,1:1007]',
biassec='',
flat=flatfield,
illum='')
correctcard('C' + lista[i])
lista1.append('C' + lista[i])
ff.close()
print '\n### prereducing STD frames to compute illumination correction ........'
lista2, skyfile = ntt.sofiphotredudef.skysub(
lista1, readkey3(readhdr(lista1[0]), 'ron'),
readkey3(readhdr(lista1[0]), 'gain'), True)
lista2 = ntt.sofiphotredudef.sortbyJD(lista2)
print '\n### use x on the star and q to continue....'
display_image(lista2[0], 2, '', '', False)
delete('tmpone.coo')
iraf.image.tv.imexamine(lista2[0],
2,
logfile='tmpone.coo',
keeplog='yes',
xformat='',
yformat='',
wcs='logical')
iraf.tvmark(2,
'tmpone.coo',
mark="circle",
number='yes',
label='no',
radii=8,
nxoffse=5,
nyoffse=5,
color=204,
txsize=2)
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x0, y0 = string.split(xycoo[0])
x0 = float(x0)
y0 = float(y0)
xcum0 = readkey3(readhdr(lista2[0]), 'xcum')
ycum0 = readkey3(readhdr(lista2[0]), 'ycum')
iraf.digiphot(_doprint=0, Stdout=0)
iraf.daophot(_doprint=0, Stdout=0)
iraf.noao.digiphot.daophot.datapars.datamin = -1000
iraf.noao.digiphot.daophot.datapars.datamax = 60000
iraf.noao.digiphot.daophot.daopars.function = 'gauss'
iraf.noao.digiphot.daophot.photpars.zmag = 0
namesex = defsex('default.sex')
for i in range(0, len(lista2)):
j = i + 1
xcum = readkey3(readhdr(lista2[i]), 'xcum')
ycum = readkey3(readhdr(lista2[i]), 'ycum')
xx = x0 - xcum0 + xcum
yy = y0 - ycum0 + ycum
# sex objects
os.system('sex ' + lista2[i] + ' -c ' + namesex + '> _logsex')
delete('_logsex')
xpix = iraf.proto.fields('detections.cat', fields='2', Stdout=1)
ypix = iraf.proto.fields('detections.cat', fields='3', Stdout=1)
cm = iraf.proto.fields('detections.cat', fields='4', Stdout=1)
cm = compress((array(xpix) != ''), array(cm, float))
ypix = compress((array(xpix) != ''), array(ypix, float))
xpix = compress((array(xpix) != ''), array(xpix, float))
if len(xpix) > 300:
num = 300
else:
num = len(xpix) - 1
xpix = xpix[argsort(cm)][0:num]
ypix = ypix[argsort(cm)][0:num]
distance = (ypix - yy)**2 + (xpix - xx)**2
xx1, yy1 = xpix[argmin(distance)], ypix[argmin(distance)]
f = open('tmpone.coo', 'w')
f.write(str(xx1) + ' ' + str(yy1) + '\n')
f.close()
display_image(lista2[i], 1, '', '', False)
iraf.tvmark(1,
'tmpone.coo',
mark="circle",
number='yes',
label='no',
radii=8,
nxoffse=5,
nyoffse=5,
color=204,
txsize=2)
answ = 'n'
while answ != 'y':
answ = raw_input('selected the right one [[y]/n] ?')
if not answ:
answ = 'y'
if answ in ['y', 'YES', 'yes', 'Y']:
print lista2[i]
delete('pippo.' + str(j) + '.mag')
gggg = iraf.digiphot.daophot.phot(lista2[i],
"tmpone.coo",
output="pippo." + str(j) +
".mag",
verify='no',
interac='no',
Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
else:
print '\n### select the std star'
display_image(lista2[i], 1, '', '', False)
iraf.image.tv.imexamine(lista2[i],
1,
logfile='tmpone.coo',
keeplog='yes',
xformat='',
yformat='',
wcs='logical')
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x2, y2 = string.split(xycoo[0])
f = open('tmpone.coo', 'w')
f.write(str(x2) + ' ' + str(y2) + '\n')
f.close()
delete('pippo.' + str(j) + '.mag')
print '###### new selection ' + str(x2), str(y2)
gggg = iraf.digiphot.daophot.phot(lista2[i],
"tmpone.coo",
output='pippo.' + str(j) +
'.mag',
verify='no',
interac='no',
Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
os.system('ls pippo.*.mag > tempmag.lst')
tmptbl0 = iraf.txdump(textfile="@tempmag.lst",
fields="XCENTER,YCENTER,FLUX",
expr='yes',
Stdout=1)
ff = open('magnitudini', 'w')
for i in tmptbl0:
ff.write(i + '\n')
ff.close()
# delete the temporary images and files
delete("temp*.fits")
delete('temp*.lst')
delete(illum_frame)
print '\n### fitting the illumination surface...'
aaa = iraf.utilities.surfit('magnitudini',
image=illum_frame,
function="polynomial",
xorder=2,
yorder=2,
xterms="full",
ncols=1024,
nlines=1024,
Stdout=1)
iraf.noao.imred.generic.normalize(illum_frame)
correctcard(lista[0])
data, hdr = pyfits.getdata(illum_frame, 0, header=True)
data0, hdr0 = pyfits.getdata(lista[0], 0, header=True)
delete(illum_frame)
pyfits.writeto(illum_frame, float32(data), hdr0)
flatfield0 = string.split(flatfield, '/')[-1]
ntt.util.updateheader(illum_frame, 0,
{'MKILLUM': [flatfield0, 'flat field']})
display_image(illum_frame, 1, '', '', False)
for i in range(0, len(lista)): # in lista:
img = lista[i]
delete('pippo.' + str(i) + '.mag')
delete('C' + img)
delete('C' + re.sub('.fits', '_sky.fits', img))
# delete('C*.fits.mag.1')
# iraf.hedit(illum_frame,'MKILLUM','Illum. corr. created '+flatfield,add='yes',update='yes',verify='no')
return illum_frame
fwhm = float(coordLines[9].split()[3])
sigma = float(coordLines[19].split()[3])
threshold = float(coordLines[27].split()[3])
coordFile.close()
print(objname, filter_, fwhm, sigma, threshold)
print('creating psf...')
iraf.images(_doprint=0)
iraf.tv(_doprint=0)
iraf.ptools(_doprint=0)
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.photcal(_doprint=0)
iraf.apphot(_doprint=0)
iraf.imutil(_doprint=0)
iraf.daophot(_doprint=0)
iraf.tables(_doprint=0)
iraf.unlearn(iraf.ptools.pselect, iraf.tables.tcreate, iraf.tables.tmatch,
iraf.tables.tinfo, iraf.tables.tdump)
# first measure the psf for the image
iraf.unlearn(iraf.phot, iraf.datapars, iraf.photpars, iraf.centerpars,
iraf.fitskypars)
iraf.apphot.phot.setParam('interactive', "no")
iraf.apphot.phot.setParam('verify', "no")
iraf.datapars.setParam('datamax', "INDEF")
iraf.datapars.setParam('gain', "gain")
iraf.datapars.setParam('ccdread', "rdnoise")
def run_hedit(images, fields, value, result):
iraf.imutil(_doprint=0)
iraf.hedit(images=images, fields=fields, value=value, add="Yes", addonly="No",\
delete="No", verify="Yes", show="Yes", update="Yes", mode="al")
def combineARCSpectra(args):
import sys
import glob
import os
from pyraf import iraf
from shutil import copyfile
from operator import itemgetter
import pandas as pd
# Setup relavent IRAF tasks...
iraf.image()
iraf.imutil()
iraf.immatch()
iraf.onedspec()
# Grab list of file names from argument list
# and convert it into a python list...
#fnameList = 'WDC0442-0536.0086b.ms.fits,WDC0442-0536.0087b.ms.fits,WDC0442-0536.0088b.ms.fits'
fnameList = args.fileList
fnameList = fnameList.split(',')
# Clean up file name list to exclude any excess blank spaces...
fileNameList = []
for fname in fnameList:
fileName = fname.strip()
fileNameList.append(fileName)
# Start a new file name list, for the imsliced
# files to be imcombined...
newFileNameList = []
# Loop through the input file name list...
for fileName in fileNameList:
print fileName
# Create an output file name...
baseFileName = os.path.splitext(fileName)[0] + '.tmp'
outputFileName = baseFileName + '.fits'
# Delete the any temporary files called tempA00?.fits,
# and then run imslice on fileName...
iraf.imdel('tempA00?.fits')
iraf.imslice(fileName, 'tempA', 2)
# Delete the any temporary files called tempB00?.fits,
# and then run imslice on tempA001.fits (an output
# from the first imslice)...
iraf.imdel('tempB00?.fits')
iraf.imslice('tempA001.fits', 'tempB', 2)
# Rename the output file tempB001.fits from the second
# imslice procedure; tempB001.fits contains the 1D
# version of the target spectrum. (The other file,
# tempB002.fits, which we ignore, contains the 1D
# version of the sky background spectrum.)
os.rename('tempB001.fits', outputFileName)
# Append the output file name to the new file name list...
newFileNameList.append(outputFileName)
# Clean up unneeded temporary files...
iraf.imdel('tempA00?.fits')
iraf.imdel('tempB00?.fits')
# Print the list of file names that will be fed to imcombine...
print newFileNameList
# Here, we convert the python list into an IRAF list...
inlist = ''
for newFileName in newFileNameList:
inlist = inlist + ',' + newFileName
inlist = inlist[1:]
print inlist
# Run imcombine to create a median-combined spectrum
# and its sigma image...
medianFile = newFileNameList[0].split('.')[0] + '.median.b.ms.fits'
sigmaFile = 'sigma-' + newFileNameList[0].split(
'.')[0] + '.median.b.ms.fits'
iraf.imdel(medianFile)
iraf.imdel(sigmaFile)
iraf.imcombine(inlist,
medianFile,
combine='median',
scale='median',
sigma=sigmaFile)
# Create ASCII text file equivalents...
medianFileTxt = newFileNameList[0].split('.')[0] + '.median.b.ms.txt'
sigmaFileTxt = 'sigma-' + newFileNameList[0].split(
'.')[0] + '.median.b.ms.txt'
iraf.onedspec.wspectext(medianFile, medianFileTxt)
iraf.onedspec.wspectext(sigmaFile, sigmaFileTxt)
# Combine median and sigma text files...
df1 = pd.read_csv(medianFileTxt,
header=None,
names=['wave', 'flux'],
delim_whitespace=True)
df2 = pd.read_csv(sigmaFileTxt,
header=None,
names=['wave', 'flux_err'],
delim_whitespace=True)
df12 = df1.merge(df2, on='wave')
outputFile = newFileNameList[0].split('.')[0] + '.median.flm'
df12.to_csv(outputFile, index=False, sep=' ')
# Run imcombine to create a mean-combined spectrum
# and its sigma image...
meanFile = newFileNameList[0].split('.')[0] + '.mean.b.ms.fits'
sigmaFile = 'sigma-' + newFileNameList[0].split('.')[0] + '.mean.b.ms.fits'
iraf.imdel(meanFile)
iraf.imdel(sigmaFile)
iraf.imcombine(inlist,
meanFile,
combine='average',
scale='median',
sigma=sigmaFile)
# Create ASCII text file equivalents...
meanFileTxt = newFileNameList[0].split('.')[0] + '.mean.b.ms.txt'
sigmaFileTxt = 'sigma-' + newFileNameList[0].split(
'.')[0] + '.mean.b.ms.txt'
iraf.onedspec.wspectext(meanFile, meanFileTxt)
iraf.onedspec.wspectext(sigmaFile, sigmaFileTxt)
# Combine mean and sigma text files...
df1 = pd.read_csv(meanFileTxt,
header=None,
names=['wave', 'flux'],
delim_whitespace=True)
df2 = pd.read_csv(sigmaFileTxt,
header=None,
names=['wave', 'flux_err'],
delim_whitespace=True)
df12 = df1.merge(df2, on='wave')
outputFile = newFileNameList[0].split('.')[0] + '.mean.flm'
df12.to_csv(outputFile, index=False, sep=' ')
return 0
def run_imarith(input_1, operator, input_2, result):
iraf.imutil(_doprint=0)
iraf.imarith(operand1=input_1 , op=operator, operand2=input_2, result=result,\
title='', divzero=0.0, hparams='', pixtype='', calctype='',\
verbose="No", noact="No", mode="al")
import time
import traceback
from lmfit import minimize, Parameters, report_fit
#import cosmics
iraf.noao(_doprint=0,Stdout="/dev/null")
iraf.rv(_doprint=0,Stdout="/dev/null")
iraf.imred(_doprint=0,Stdout="/dev/null")
iraf.ccdred(_doprint=0,Stdout="/dev/null")
iraf.images(_doprint=0,Stdout="/dev/null")
iraf.immatch(_doprint=0,Stdout="/dev/null")
iraf.onedspec(_doprint=0,Stdout="/dev/null")
iraf.twodspec(_doprint=0,Stdout="/dev/null")
iraf.apextract(_doprint=0,Stdout="/dev/null")
iraf.imutil(_doprint=0,Stdout="/dev/null")
iraf.echelle(_doprint=0,Stdout="/dev/null")
iraf.astutil(_doprint=0,Stdout="/dev/null")
iraf.apextract.dispaxi=1
iraf.echelle.dispaxi=1
#fixes a bug with latest versions of iraf
iraf.ccdred.instrum='blank.txt'
os.environ['PYRAF_BETA_STATUS'] = '1'
# REFS NEW: ecEC59550 ecEC59758 ecEC59795 ecEC59842 ecEC59844 ecEC59864 ecEC59866 ecEC59881 ecEC59883 ecEC59885
observations = {
'cemp_cand': {'ORIG_DIR':'/home/nandir/IRAF_Echelle_Asiago/CEMP_obs/201712/',
'biases':['EC60040', 'EC60041', 'EC60042', 'EC60043', 'EC60044'],
def psfphot(image,
coofile,
ot,
wtimage="",
varorder=1,
clobber=globclob,
verbose=globver,
pixtol=3.0,
maxnpsf=25):
""" perform PSF-based photometry on a single target star (SN?) at RA, Dec and
also on a set of comparison stars, using daophot. simultaneously
perform aperture photometry on all the comparison stars (after
subtracting off contributions from neighbors) to enable absolute
photometry by comparison to aperture photometry of standard stars
observed in other fields """
# Defaults / constants
psfmult = 5.0 #standard factor (multiplied by fwhm to get psfradius)
psfmultsmall = 3.0 #similar to psfmult, adjusted for nstar and substar
# Necessary package
iraf.imutil()
iraf.digiphot()
iraf.daophot()
# Detect stars
iqobjs("%s.sub.fits" % image[:-5],
1.5,
12000.0,
wtimage=wtimage,
skyval="0.0")
root = image[:-5]
[gain, rnoise, fwhm] = get_head(image, ["GAIN", "READN", "SEEPIX"])
fwhm = float(fwhm)
rnoise = float(rnoise)
iraf.iterstat(image)
# Saturation level
if not check_head(image, "SATURATE"):
saturate = 60000.0
else:
saturate = get_head(image, "SATURATE")
# Update datapars and daopars
iraf.datapars.fwhmpsf = fwhm
iraf.datapars.sigma = iraf.iterstat.sigma
iraf.datapars.datamin = iraf.iterstat.median - 10 * iraf.iterstat.sigma
iraf.datapars.datamax = 0.50 * saturate
iraf.datapars.readnoise = rnoise
iraf.datapars.epadu = gain
iraf.daopars.psfrad = psfmult * fwhm
iraf.daopars.fitrad = fwhm
iraf.daopars.function = "gauss,moffat15,moffat25,lorentz,penny1"
iraf.daopars.varorder = varorder
# Reference stars file
stars = Starlist(coofile)
stars.wcs2pix(image)
outf = open("%s.coo.1" % image[:-5], "w")
for star in stars:
outf.write("%10.3f%10.3f\n" % (star.xval, star.yval))
outf.close()
#Aperture photometry
iraf.daophot.phot(root,
'default',
'default',
apertures=fwhm,
verify=no,
interac=no,
verbose=verbose)
iraf.datapars.datamax = 0.50 * saturate
iraf.pstselect(root,
'default',
'default',
maxnpsf,
interactive=no,
verify=no,
verbose=verbose)
iraf.psf(root,
'default',
'default',
'default',
'default',
'default',
interactive=no,
showplots=no,
verify=no,
verbose=verbose)
iraf.allstar(root,
'default',
'default',
'default',
'default',
'default',
verify=no,
verbose=verbose)
# Prep for subtracted image
iraf.iterstat("%s.sub.fits" % root)
iraf.datapars.sigma = iraf.iterstat.sigma
iraf.datapars.datamin = iraf.iterstat.median - 10 * iraf.iterstat.sigma
iraf.datapars.datamax = saturate
# Look for source at OT location
substars = Starlist("%s.sub.fits.stars" % image[:-5])
otstars = Starlist(ot)
otstars.wcs2pix("%s.sub.fits" % image[:-5])
smatch, omatch = substars.match(otstars, tol=pixtol, useflags=no)
# Generate coo file
otcoo = open("%s.sub.coo.1" % image[:-5], "w")
if len(smatch) == 0:
otcoo.write("%10.3f%10.3f\n" % (otstars[0].xval, otstars[0].yval))
else:
otcoo.write("%10.3f%10.3f\n" % (smatch[0].xval, smatch[0].yval))
otcoo.close()
iraf.daophot.phot("%s.sub.fits" % root,
"%s.sub.coo.1" % image[:-5],
'default',
'default',
apertures=fwhm,
calgorithm="none",
interac=no,
verify=no,
verbose=verbose)
if len(smatch) == 0:
print "No match in subtracted image: %s.sub.fits" % root
else:
iraf.allstar("%s.sub.fits" % root,
'default',
"%s.psf.1.fits" % root,
'default',
'default',
'default',
verify=no,
verbose=no)
return
def makeillumination(lista,flatfield):#,outputfile,illum_frame):
import os,glob,string,re
from astropy.io import fits as pyfits
import ntt
from ntt.util import readhdr, readkey3, delete, display_image, defsex, name_duplicate, correctcard
from numpy import compress, array, argmax, argmin, min, argsort, float32
import datetime
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
_date = readkey3(readhdr(lista[0]), 'date-night')
_filter = readkey3(readhdr(lista[0]), 'filter')
illum_frame = name_duplicate(
lista[0], 'illum_' + _date + '_' + _filter + '_' + str(MJDtoday), '')
from pyraf import iraf
iraf.images(_doprint=0)
iraf.imutil(_doprint=0)
iraf.utilities(_doprint=0)
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.daophot(_doprint=0)
iraf.generic(_doprint=0)
toforget = ['digiphot.daophot', 'imutil.imarith',
'image', 'utilities.surfit']
for t in toforget:
iraf.unlearn(t)
n = len(lista)
# start loop to read image names from the input file
lista1 = []
iraf.ccdred.verbose = 'no'
ff = open('templist.lst', 'w')
for i in range(0, len(lista)):
ff.write('C' + lista[i] + '\n')
delete('C' + lista[i])
delete('C' + re.sub('.fits', '_sub.fits', lista[i]))
ntt.sofiphotredudef.crosstalk(lista[i], 'C' + lista[i])
iraf.noao.imred.ccdred.ccdproc('C' + lista[i], output='', overscan="no", trim="yes", ccdtype='', darkcor='no', fixpix='no', zerocor="no", flatcor='yes',
illumco='no', trimsec='[1:1024,1:1007]', biassec='', flat=flatfield, illum='')
correctcard('C' + lista[i])
lista1.append('C' + lista[i])
ff.close()
print '\n### prereducing STD frames to compute illumination correction ........'
lista2, skyfile = ntt.sofiphotredudef.skysub(lista1, readkey3(
readhdr(lista1[0]), 'ron'), readkey3(readhdr(lista1[0]), 'gain'), True)
lista2 = ntt.sofiphotredudef.sortbyJD(lista2)
print '\n### use x on the star and q to continue....'
display_image(lista2[0], 2, '', '', False)
delete('tmpone.coo')
iraf.image.tv.imexamine(lista2[0], 2, logfile='tmpone.coo',
keeplog='yes', xformat='', yformat='', wcs='logical')
iraf.tvmark(2, 'tmpone.coo', mark="circle", number='yes',
label='no', radii=8, nxoffse=5, nyoffse=5, color=204, txsize=2)
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x0, y0 = string.split(xycoo[0])
x0 = float(x0)
y0 = float(y0)
xcum0 = readkey3(readhdr(lista2[0]), 'xcum')
ycum0 = readkey3(readhdr(lista2[0]), 'ycum')
iraf.digiphot(_doprint=0)
iraf.daophot(_doprint=0)
iraf.noao.digiphot.daophot.datapars.datamin = -1000
iraf.noao.digiphot.daophot.datapars.datamax = 60000
iraf.noao.digiphot.daophot.daopars.function = 'gauss'
iraf.noao.digiphot.daophot.photpars.zmag = 0
namesex = defsex('default.sex')
for i in range(0, len(lista2)):
j = i + 1
xcum = readkey3(readhdr(lista2[i]), 'xcum')
ycum = readkey3(readhdr(lista2[i]), 'ycum')
xx = x0 - xcum0 + xcum
yy = y0 - ycum0 + ycum
# sex objects
os.system('sex ' + lista2[i] + ' -c ' + namesex + '> _logsex')
delete('_logsex')
xpix = iraf.proto.fields('detections.cat', fields='2', Stdout=1)
ypix = iraf.proto.fields('detections.cat', fields='3', Stdout=1)
cm = iraf.proto.fields('detections.cat', fields='4', Stdout=1)
cm = compress((array(xpix) != ''), array(cm, float))
ypix = compress((array(xpix) != ''), array(ypix, float))
xpix = compress((array(xpix) != ''), array(xpix, float))
if len(xpix) > 300:
num = 300
else:
num = len(xpix) - 1
xpix = xpix[argsort(cm)][0:num]
ypix = ypix[argsort(cm)][0:num]
distance = (ypix - yy)**2 + (xpix - xx)**2
xx1, yy1 = xpix[argmin(distance)], ypix[argmin(distance)]
f = open('tmpone.coo', 'w')
f.write(str(xx1) + ' ' + str(yy1) + '\n')
f.close()
display_image(lista2[i], 1, '', '', False)
iraf.tvmark(1, 'tmpone.coo', mark="circle", number='yes',
label='no', radii=8, nxoffse=5, nyoffse=5, color=204, txsize=2)
answ = 'n'
while answ != 'y':
answ = raw_input('selected the right one [[y]/n] ?')
if not answ:
answ = 'y'
if answ in ['y', 'YES', 'yes', 'Y']:
print lista2[i]
delete('pippo.' + str(j) + '.mag')
gggg = iraf.digiphot.daophot.phot(
lista2[i], "tmpone.coo", output="pippo." + str(j) + ".mag", verify='no', interac='no', Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
else:
print '\n### select the std star'
display_image(lista2[i], 1, '', '', False)
iraf.image.tv.imexamine(lista2[
i], 1, logfile='tmpone.coo', keeplog='yes', xformat='', yformat='', wcs='logical')
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x2, y2 = string.split(xycoo[0])
f = open('tmpone.coo', 'w')
f.write(str(x2) + ' ' + str(y2) + '\n')
f.close()
delete('pippo.' + str(j) + '.mag')
print '###### new selection ' + str(x2), str(y2)
gggg = iraf.digiphot.daophot.phot(
lista2[i], "tmpone.coo", output='pippo.' + str(j) + '.mag', verify='no', interac='no', Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
os.system('ls pippo.*.mag > tempmag.lst')
tmptbl0 = iraf.txdump(textfile="@tempmag.lst",
fields="XCENTER,YCENTER,FLUX", expr='yes', Stdout=1)
ff = open('magnitudini', 'w')
for i in tmptbl0:
ff.write(i + '\n')
ff.close()
# delete the temporary images and files
delete("temp*.fits")
delete('temp*.lst')
delete(illum_frame)
print '\n### fitting the illumination surface...'
aaa = iraf.utilities.surfit('magnitudini', image=illum_frame, function="polynomial",
xorder=2, yorder=2, xterms="full", ncols=1024, nlines=1024, Stdout=1)
iraf.noao.imred.generic.normalize(illum_frame)
correctcard(lista[0])
data, hdr = pyfits.getdata(illum_frame, 0, header=True)
data0, hdr0 = pyfits.getdata(lista[0], 0, header=True)
delete(illum_frame)
pyfits.writeto(illum_frame, float32(data), hdr0)
flatfield0 = string.split(flatfield, '/')[-1]
ntt.util.updateheader(
illum_frame, 0, {'MKILLUM': [flatfield0, 'flat field']})
display_image(illum_frame, 1, '', '', False)
for i in range(0, len(lista)): # in lista:
img = lista[i]
delete('pippo.' + str(i) + '.mag')
delete('C' + img)
delete('C' + re.sub('.fits', '_sky.fits', img))
# delete('C*.fits.mag.1')
# iraf.hedit(illum_frame,'MKILLUM','Illum. corr. created '+flatfield,add='yes',update='yes',verify='no')
return illum_frame
def calibphot(inlist,
fwhm,
rdnoise='2.6',
gain='2.5',
interact=yes,
config='',
objkey="OBJECT",
airkey="AIRMASS",
filtkey='FILTER',
skykey='SKYBKG',
phosfx="pho",
possfx="pos",
clobber=globclob,
verbose=globver):
""" perform aperture photometry on a standard field to fit the
transformation equations. these transformation equations will
then be applied to comparison stars' aperture photometries
outputted by psfphot.py, in order to find the true magnitudes of
the comparison stars. this will then provide and apply the
corrections to the magnitude of a single target star (SN?). """
# Defaults / constants
alpha = list('abcdefghijklmnopqrstuvwxyz')
psfmult = 5 #standard factor (multiplied by fwhm to get psfradius)
imgprop = {}
imglist = {}
allobjs = {}
allfilt = {}
# Necessary package
iraf.imutil()
# Parse gain/readnoise inputs
gainkey = None
re1 = re.search('^!(.+)', gain)
re2 = re.search('^[\d\.]+', gain)
if re1:
gainkey = re1.group(1)
elif re2:
try:
gainval = float(gain)
except:
print "Error converting gain"
return
else:
print "Error finding gain"
return
rdnskey = None
re1 = re.search('^!(.+)', rdnoise)
re2 = re.search('^[\d\.]+', rdnoise)
if re1:
rdnskey = re1.group(1)
elif re2:
try:
readval = float(rdnoise)
except:
print "Error converting readnoise"
return
else:
print "Error finding readnoise"
return
# Parse inputs
infiles = iraffiles(inlist)
#first make sure to add back in background of sky
iraf.iqsubsky(inlist, sub=no, skykey=skykey)
# Process each file in turn
for image in infiles:
# Check that the image is there
check_exist(image, "r")
# Grab all the useful keywords
[sky,objname,filter,airmass] = \
get_head(image,[skykey,objkey,filtkey,airkey])
# Get gain & readnoise, if necessary
if gainkey:
gainval = get_head(image, gainkey)
if rdnskey:
readval = get_head(image, rdnskey)
# Calculate theoretical sigma of sky
if len(str(sky)) > 0:
sigma = (((sky * gainval) + readval**2)**.5) / gainval
else:
print "Failed to retrieve sky value from image header"
return
# check standard field name
if len(objname) == 0:
print "Failed to retrieve object name from image header"
return
# Generate posfile and phofile names
posfile = objname + '.' + possfx
phofile = objname + '.' + phosfx
# check filter name
if len(filter) == 0:
print "Failed to retrieve filter name from image header"
return
# Check airmass value
if len(airmass) > 0:
try:
airval = float(airmass)
except:
print "Failed to convert airmass value to float"
return
else:
print "Failed to retrieve airmass value from image header"
return
# Save keyword values for this image
imgprop[image] = [gainval, readval, sky, objname, filter, airmass]
# Add dictionary entries for the current image
listkey = objname + '-' + filter
if imglist.has_key(listkey):
imglist[listkey].append(image)
else:
imglist[listkey] = [image]
# Keep track of all filters/object fields
allobjs[objname] = 1
allfilt[filter] = 1
#add in extra range of aperture radii, leave in background?
fwhmpsf = float(fwhm)
ap1 = fwhmpsf
ap15 = 1.5 * fwhmpsf
ap2 = 2 * fwhmpsf
ap25 = 2.5 * fwhmpsf
ap3 = 3 * fwhmpsf
innersky = 3.5 * fwhmpsf
#do photometry on all stars in field
check_exist(image + ".mag.std.1", "w", clobber=yes)
iraf.phot(image,
output=image + ".mag.std.1",
coords=posfile,
sigma=sigma,
datamin=sky - 3.5 * sigma,
apertures=ap25,
dannulus="10",
annulus=innersky,
verify="no")
##############################
#make imsets file
imsetsfile = "stdfield.dat"
imsetstuff = open(imsetsfile, "w")
for objname in allobjs.keys():
maxnum = 0
for key in imglist.keys():
if re.search(objname, key):
maxnum = max(maxnum, len(imglist[key]))
for i in range(maxnum):
imset = objname + alpha[i] + " :"
for key in imglist.keys():
if re.search(objname, key):
if i < len(imglist[key]):
imset += " " + imglist[key][i]
imsetstuff.write(imset + "\n")
imsetstuff.close()
#make observations file (specify aperture number?)
obsfile = "standobs"
check_exist(obsfile, "w", clobber=yes)
iraf.mknobsfile(photfiles="*.mag.std.1",
idfilters=" ".join(allfilt.keys()),
imsets=imsetsfile,
observations=obsfile,
tolerance="500")
#rename stars in catalog file so they agree, print good stars to new file
catpholines = getlines(phofile)
s = 1
newcatpholines = [catpholines[0]]
lastline = catpholines[-1]
lastlinedata = lastline.split()
digits = len(lastlinedata[0]) + 1
for line in catpholines:
if re.search("vary?", line):
continue
ess = str(s)
idlength = len(objname) + len(ess) + 1
numspace = digits - idlength + 1
newline = " " * numspace + objname + "-" + ess + line[digits:]
s = s + 1
if not re.search("99.999", newline):
newcatpholines.append(newline)
newcatdata = "\n".join(newcatpholines)
newphofile = phofile + ".2"
newphostuff = open(phofile + ".2", "w")
newphostuff.write(newcatdata)
newphostuff.close()
#rename stars in standobs to be field-# instead of fielda-#, fieldb-#, etc
#what is field?
obsdatalines = getlines(obsfile)
for i in range(len(obsdatalines)):
line = obsdatalines[i]
for objname in allobjs.keys():
if re.search(objname, line):
index = line.index(objname)
lline = list(line)
lline[index:index + len(objname) + 1] = list(" " + objname)
line = "".join(lline)
obsdatalines[i] = line
break
newobsdata = "\n".join(obsdatalines)
fixobsfile = "fix" + obsfile
fixobsstuff = open(fixobsfile, "w")
fixobsstuff.write(newobsdata)
fixobsstuff.close()
#if no config file exists, make one
#first, make a file that has stetson catalog formats in it
if not config:
newconfig = "fcat.dat"
check_exist(newconfig, "w", clobber=yes)
catformat = open(newconfig, "w")
catinfo = [
"#Declare the catalog variables", " ", "catalog", " ", " B\t2",
"Berror\t3", "V\t6", "Verror\t7", "R\t10", "Rerror\t11", "I\t14",
"Ierror\t15"
]
catdata = "\n".join(catinfo)
catformat.write(catdata)
catformat.close()
config = "cat.cfg"
transfile = "ftrans.dat"
check_exist(transfile, "w", clobber=yes)
transstuff = open(transfile, "w")
transinfo = [
"transformation", " ", "fit v1 = 0, v2 = 0.17, v3= 0",
"fit b1 = 0, b2 = 0.35, b3= 0", "fit r1 = 0, r2 = 0.08, r3= 0",
" ", "const v4 = 0", "const b4 = 0", "const r4 = 0", " ",
"Vfit : V = mV + v1 + v2*XV + v3*(mB-mV) + v4*(mB-mV)*XV",
"Bfit : B = mB + b1 + b2*XB + b3*(mB-mV) + b4*(mB-mV)*XB",
"Rfit : R = mR + r1 + r2*XR + r3*(mV-mR) + r4*(mV-mR)*XR"
]
transdata = "\n".join(transinfo) + "\n"
transstuff.write(transdata)
transstuff.close()
check_exist(config, "w", clobber=yes)
iraf.mkconfig(config,
catalog=newconfig,
observations="f" + obsfile + ".dat",
transform=transfile,
check=no,
edit=no)
#fit parameters of transformation equations (specify aperture in ansfile?)
ansfile = objname + ".ans.1"
check_exist(ansfile, "w", clobber=yes)
iraf.fitparams(observations=fixobsfile,
catalogs=newphofile,
config=config,
parameters=ansfile,
interactive=interact)
#apply to standard stars to check to make sure fits are okay?
calibfile = objname + ".calib.1"
check_exist(calibfile, "w", clobber=yes)
iraf.evalfit(observations=fixobsfile,
config=config,
parameters=ansfile,
calib=calibfile)
def escut(image, pos_file, fwhm, peak):
# input image file name, file name with matched source positions, **np.array of fwhm measurements for each source
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats
from pyraf import iraf
# import sewpy
import os
from matplotlib.path import Path
iraf.images(_doprint=0)
iraf.tv(_doprint=0)
iraf.ptools(_doprint=0)
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.photcal(_doprint=0)
iraf.apphot(_doprint=0)
iraf.imutil(_doprint=0)
iraf.unlearn(iraf.phot, iraf.datapars, iraf.photpars, iraf.centerpars,
iraf.fitskypars)
iraf.apphot.phot.setParam('interactive', "no")
iraf.apphot.phot.setParam('verify', "no")
iraf.datapars.setParam('datamax', 50000.)
iraf.datapars.setParam('gain', "gain")
iraf.datapars.setParam('ccdread', "rdnoise")
iraf.datapars.setParam('exposure', "exptime")
iraf.datapars.setParam('airmass', "airmass")
iraf.datapars.setParam('filter', "filter")
iraf.datapars.setParam('obstime', "time-obs")
# iraf.datapars.setParam('obstime',"date-obs")
iraf.datapars.setParam('sigma', "INDEF")
iraf.photpars.setParam('zmag', 0.)
iraf.centerpars.setParam('cbox', 9.)
iraf.centerpars.setParam('maxshift', 3.)
iraf.fitskypars.setParam('salgorithm', "median")
iraf.fitskypars.setParam('dannulus', 10.)
# clean up the indefs so we can actually do stats, but reassign them to 99999 so we don't lose track of things
# keep a separate list without them to do the median (we need floats)
indefs = np.where(fwhm == 'INDEF')
good = np.where(fwhm != 'INDEF')
fwhm[indefs] = 99.999
fwhm = fwhm.astype(float)
fwhm_good = fwhm[good].astype(float)
indefs = np.where(peak == 'INDEF')
peak[indefs] = -999.999
peak = peak.astype(float)
peak_good = peak[good].astype(float)
if not os.path.isfile(image[0:-5] + '.txdump'):
# findavgfwhm = sewpy.SEW(
# params = ["X_IMAGE", "Y_IMAGE", "FWHM_IMAGE", "FLAGS"],
# config = {"DETECT_THRESH":200.0},
# sexpath = "sex"
# )
#
# out = findavgfwhm(image)["table"]
#
# fwhms = out['FWHM_IMAGE'] # This is an astropy table.
# flags = out['FLAGS']
# get a really rough estimate of the stellar FWHM in the image to set apertures
# use the input fwhm measurement
# ap1x = fwhm_est
# xpos = datatable['X_IMAGE']
# ypos = datatable['Y_IMAGE']
# fwhm = datatable['FWHM_IMAGE']
# flags = datatable['FLAGS']
# idno = datatable['NUMBER']
ap1x = np.median(
fwhm_good
) # only use isolated detections of stars, this is the 1x aperture
# print ap1x
ap2x = 2.0 * ap1x
# these = [ i for i,id in enumerate(idno) if (flags[i] == 0)]
# with open(image[0:-5]+'.escut.pos','w+') as f:
# for j in range(len(xpos)):
# print >> f, xpos[j], ypos[j], fwhm[j], idno[j]
iraf.datapars.setParam('fwhmpsf', ap1x)
iraf.photpars.setParam('apertures', repr(ap1x) + ', ' + repr(ap2x))
iraf.fitskypars.setParam('annulus', 4. * ap1x)
iraf.apphot.phot(image=image,
coords=pos_file,
output=image[0:-5] + '.phot')
with open(image[0:-5] + '.txdump', 'w+') as txdump_out:
iraf.ptools.txdump(
textfiles=image[0:-5] + '.phot',
fields=
"id,mag,merr,msky,stdev,rapert,xcen,ycen,ifilter,xairmass,image",
expr=
'MAG[1] != INDEF && MERR[1] != INDEF && MAG[2] != INDEF && MERR[2] != INDEF',
headers='no',
Stdout=txdump_out)
mag1x, mag2x = np.loadtxt(image[0:-5] + '.txdump',
usecols=(1, 2),
unpack=True)
iraf_id = np.loadtxt(image[0:-5] + '.txdump',
usecols=(0, ),
dtype=int,
unpack=True)
# idno = np.loadtxt(image[0:-5]+'.escut.pos', usecols=(3,), dtype=int, unpack=True)
xpos, ypos = np.loadtxt(pos_file, usecols=(0, 1), unpack=True)
keepIndex = iraf_id - 1
xpos, ypos, fwhm, peak = xpos[keepIndex], ypos[keepIndex], fwhm[
keepIndex], peak[keepIndex]
# print idno.size, iraf_id.size, xpos.size
diff = mag2x - mag1x
diffCut = diff
magCut = mag2x
xCut = xpos #[good]
yCut = ypos #[good]
idCut = iraf_id
fwhmCut = fwhm #_good
peakCut = peak
print(peakCut.size, magCut.size, diffCut.size)
print(diffCut.size, 0, np.median(diffCut), diffCut.std())
nRemoved = 1
# plt.clf()
# plt.scatter(peakCut, magCut, edgecolor='none')
# plt.savefig('peaktest.pdf')
plt.clf()
# plt.hlines(bin_edges, -2, 1, colors='red', linestyle='dashed')
plt.scatter(diff, mag2x, edgecolor='none', facecolor='black', s=4)
# plt.scatter(diffCut, magCut, edgecolor='none', facecolor='blue', s=4)
magdiff = list(
zip(magCut.tolist(), diffCut.tolist(), peakCut.tolist(),
idCut.tolist()))
dtype = [('mag', float), ('diff', float), ('peak', float), ('id', int)]
magdiff = np.array(magdiff, dtype=dtype)
magSort = np.sort(magdiff, order='peak')
peakRange = (magSort['peak'] > 20000.0) & (magSort['peak'] < 40000.0)
peakVal = np.median((magSort['diff'])[np.where(peakRange)])
# peakVal = np.median(diffCut)
print(peakVal)
plt.scatter((magSort['diff'])[np.where(peakRange)],
(magSort['mag'])[np.where(peakRange)],
edgecolor='none',
facecolor='blue',
s=4)
while nRemoved != 0:
nBefore = diffCut.size
diffCheck = np.where(
abs(peakVal - diffCut) < 2.0 *
diffCut.std()) #[i for i,d in enumerate(diff) if (-0.5 < d < 0.0)]
#
diffCut = diffCut[diffCheck]
nRemoved = nBefore - diffCut.size
magCut = magCut[diffCheck]
xCut = xCut[diffCheck]
yCut = yCut[diffCheck]
idCut = idCut[diffCheck]
fwhmCut = fwhmCut[diffCheck]
print(diffCut.size, nRemoved, np.median(diffCut), diffCut.std())
if 0.05 < diffCut.std() < 0.06:
nRemoved = 0
# plt.fill_betweenx(bin_centers, bin_meds+3.0*bin_stds, bin_meds-3.0*bin_stds, facecolor='red', edgecolor='none', alpha=0.4, label='2x RMS sigma clipping region')
# with open('escutSTD_i.pos','w+') as f:
# for i,blah in enumerate(xCut):
# print >> f, xCut[i], yCut[i], diffCut[i]
bin_meds, bin_edges, binnumber = stats.binned_statistic(magCut,
diffCut,
statistic='median',
bins=24,
range=(-12, 0))
bin_stds, bin_edges, binnumber = stats.binned_statistic(magCut,
diffCut,
statistic=np.std,
bins=24,
range=(-12, 0))
bin_width = (bin_edges[1] - bin_edges[0])
bin_centers = bin_edges[1:] - bin_width / 2
# print bin_meds, bin_stds
bin_hw = np.zeros_like(bin_stds)
for i, bin_std in enumerate(bin_stds):
if bin_std > 0.025:
bin_hw[i] = 3.0 * bin_std
else:
bin_hw[i] = 0.075
# print len(binnumber)
# for i,bin_hwi in enumerate(bin_hw):
left_edge = np.array(list(zip(peakVal - bin_hw, bin_centers)))
right_edge = np.flipud(np.array(list(zip(peakVal + bin_hw, bin_centers))))
# print left_edge, right_edge
verts = np.vstack((left_edge, right_edge))
# print verts
# verts = np.delete(verts, np.array([0,1,2,22,23,24,25,45,46,47]), axis=0)
# DON'T USE A PATH BECAUSE APPARENTLY IT CAN SELECT THE INVERSE SET!! WTF
# print verts
esRegion = Path(verts)
sources = esRegion.contains_points(list(zip(diff, mag2x)))
# print sources
with open('escutREG_i.pos', 'w+') as f:
for i, blah in enumerate(xpos[sources]):
print((xpos[sources])[i], (ypos[sources])[i], (diff[sources])[i],
file=f)
magCut2 = mag2x[sources]
magCut1 = mag1x[sources]
fwhmCut = fwhm[sources]
xCut = xpos[sources]
yCut = ypos[sources]
diffCut = diff[sources]
# find the sources that are in the std method but not the region method
# print idCut, idno[sources]
# extrasSTD = np.setdiff1d(idno[sources], idCut)
# print extrasSTD.size
# print extrasSTD
# with open('escutUNIQUE.pos','w+') as f:
# for i,blah in enumerate(extrasSTD):
# print >> f, xpos[blah-1], ypos[blah-1]
# fwhmcheck = np.loadtxt('testfwhmREG.log', usecols=(10,), unpack=True)
fwhmchk2 = np.where((magCut2 < -4) & (fwhmCut < 90.0))
print(np.median(fwhmCut[fwhmchk2]), np.std(fwhmCut[fwhmchk2]))
fwchk = np.where(
np.abs(fwhmCut - np.median(fwhmCut[fwhmchk2])) > 10.0 *
np.std(fwhmCut[fwhmchk2]))
drop = np.abs(fwhmCut - np.median(fwhmCut[fwhmchk2])) > 10.0 * np.std(
fwhmCut[fwhmchk2])
keep = np.abs(fwhmCut - np.median(fwhmCut[fwhmchk2])) <= 10.0 * np.std(
fwhmCut[fwhmchk2])
with open('escutVBAD_i.pos', 'w+') as f:
for i, blah in enumerate(xCut[fwchk]):
print((xCut[fwchk])[i], (yCut[fwchk])[i], file=f)
with open('escut_i.pos', 'w+') as f:
for i, blah in enumerate(xCut):
if not drop[i]:
print(xCut[i],
yCut[i],
magCut2[i],
fwhmCut[i],
magCut1[i],
file=f)
with open('escut_g.pos', 'w+') as f:
for i, blah in enumerate(xCut):
if not drop[i]:
print(xCut[i],
yCut[i],
magCut2[i],
fwhmCut[i],
magCut1[i],
file=f)
plt.fill_betweenx(bin_centers,
peakVal + bin_hw,
peakVal - bin_hw,
facecolor='red',
edgecolor='none',
alpha=0.4,
label='2x RMS sigma clipping region')
plt.scatter(diffCut[fwchk],
magCut2[fwchk],
edgecolor='none',
facecolor='red',
s=4)
plt.ylim(0, -12)
plt.xlabel('$m_{2x} - m_{1x}$')
plt.ylabel('$m_{2x}$')
plt.xlim(-2, 1)
plt.savefig('testmagiraf.pdf')
plt.clf()
plt.scatter(magCut2, fwhmCut, edgecolor='none', facecolor='black')
plt.scatter(magCut2[fwchk],
fwhmCut[fwchk],
edgecolor='none',
facecolor='red')
plt.hlines([np.median(fwhmCut)], -12, 0, colors='red', linestyle='dashed')
plt.hlines([
np.median(fwhmCut) + fwhmCut.std(),
np.median(fwhmCut) - fwhmCut.std()
],
-12,
0,
colors='red',
linestyle='dotted')
plt.ylim(0, 20)
plt.xlim(-12, 0)
plt.ylabel('fwhm')
plt.xlabel('$m_{2x}$')
plt.savefig('fwhmcheck.pdf')
return fwhmCut[keep]
def escut(image, pos_file, fwhm, peak):
# input image file name, file name with matched source positions, **np.array of fwhm measurements for each source
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats
from pyraf import iraf
# import sewpy
import os
from matplotlib.path import Path
iraf.images(_doprint=0)
iraf.tv(_doprint=0)
iraf.ptools(_doprint=0)
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.photcal(_doprint=0)
iraf.apphot(_doprint=0)
iraf.imutil(_doprint=0)
iraf.unlearn(iraf.phot,iraf.datapars,iraf.photpars,iraf.centerpars,iraf.fitskypars)
iraf.apphot.phot.setParam('interactive',"no")
iraf.apphot.phot.setParam('verify',"no")
iraf.datapars.setParam('datamax',50000.)
iraf.datapars.setParam('gain',"gain")
iraf.datapars.setParam('ccdread',"rdnoise")
iraf.datapars.setParam('exposure',"exptime")
iraf.datapars.setParam('airmass',"airmass")
iraf.datapars.setParam('filter',"filter")
iraf.datapars.setParam('obstime',"time-obs")
# iraf.datapars.setParam('obstime',"date-obs")
iraf.datapars.setParam('sigma',"INDEF")
iraf.photpars.setParam('zmag',0.)
iraf.centerpars.setParam('cbox',9.)
iraf.centerpars.setParam('maxshift',3.)
iraf.fitskypars.setParam('salgorithm',"median")
iraf.fitskypars.setParam('dannulus',10.)
# clean up the indefs so we can actually do stats, but reassign them to 99999 so we don't lose track of things
# keep a separate list without them to do the median (we need floats)
indefs = np.where(fwhm=='INDEF')
good = np.where(fwhm!='INDEF')
fwhm[indefs] = 99.999
fwhm = fwhm.astype(float)
fwhm_good = fwhm[good].astype(float)
indefs = np.where(peak=='INDEF')
peak[indefs] = -999.999
peak = peak.astype(float)
peak_good = peak[good].astype(float)
if not os.path.isfile(image[0:-5]+'.txdump'):
# findavgfwhm = sewpy.SEW(
# params = ["X_IMAGE", "Y_IMAGE", "FWHM_IMAGE", "FLAGS"],
# config = {"DETECT_THRESH":200.0},
# sexpath = "sex"
# )
#
# out = findavgfwhm(image)["table"]
#
# fwhms = out['FWHM_IMAGE'] # This is an astropy table.
# flags = out['FLAGS']
# get a really rough estimate of the stellar FWHM in the image to set apertures
# use the input fwhm measurement
# ap1x = fwhm_est
# xpos = datatable['X_IMAGE']
# ypos = datatable['Y_IMAGE']
# fwhm = datatable['FWHM_IMAGE']
# flags = datatable['FLAGS']
# idno = datatable['NUMBER']
ap1x = np.median(fwhm_good) # only use isolated detections of stars, this is the 1x aperture
# print ap1x
ap2x = 2.0*ap1x
# these = [ i for i,id in enumerate(idno) if (flags[i] == 0)]
# with open(image[0:-5]+'.escut.pos','w+') as f:
# for j in range(len(xpos)):
# print >> f, xpos[j], ypos[j], fwhm[j], idno[j]
iraf.datapars.setParam('fwhmpsf',ap1x)
iraf.photpars.setParam('apertures',repr(ap1x)+', '+repr(ap2x))
iraf.fitskypars.setParam('annulus',4.*ap1x)
iraf.apphot.phot(image=image, coords=pos_file, output=image[0:-5]+'.phot')
with open(image[0:-5]+'.txdump','w+') as txdump_out :
iraf.ptools.txdump(textfiles=image[0:-5]+'.phot', fields="id,mag,merr,msky,stdev,rapert,xcen,ycen,ifilter,xairmass,image", expr='MAG[1] != INDEF && MERR[1] != INDEF && MAG[2] != INDEF && MERR[2] != INDEF', headers='no', Stdout=txdump_out)
mag1x, mag2x = np.loadtxt(image[0:-5]+'.txdump', usecols=(1,2), unpack=True)
iraf_id = np.loadtxt(image[0:-5]+'.txdump', usecols=(0,), dtype=int, unpack=True)
# idno = np.loadtxt(image[0:-5]+'.escut.pos', usecols=(3,), dtype=int, unpack=True)
xpos, ypos = np.loadtxt(pos_file, usecols=(0,1), unpack=True)
keepIndex = iraf_id - 1
xpos, ypos, fwhm, peak = xpos[keepIndex], ypos[keepIndex], fwhm[keepIndex], peak[keepIndex]
# print idno.size, iraf_id.size, xpos.size
diff = mag2x - mag1x
diffCut = diff
magCut = mag2x
xCut = xpos#[good]
yCut = ypos#[good]
idCut = iraf_id
fwhmCut = fwhm#_good
peakCut = peak
print peakCut.size, magCut.size, diffCut.size
print diffCut.size, 0, np.median(diffCut), diffCut.std()
nRemoved = 1
# plt.clf()
# plt.scatter(peakCut, magCut, edgecolor='none')
# plt.savefig('peaktest.pdf')
plt.clf()
# plt.hlines(bin_edges, -2, 1, colors='red', linestyle='dashed')
plt.scatter(diff, mag2x, edgecolor='none', facecolor='black', s=4)
# plt.scatter(diffCut, magCut, edgecolor='none', facecolor='blue', s=4)
magdiff = zip(magCut.tolist(), diffCut.tolist(), peakCut.tolist(), idCut.tolist())
dtype = [('mag',float), ('diff', float), ('peak', float), ('id', int)]
magdiff = np.array(magdiff, dtype=dtype)
magSort = np.sort(magdiff, order='peak')
peakRange = (magSort['peak'] > 20000.0) & (magSort['peak'] < 40000.0)
peakVal = np.median((magSort['diff'])[np.where(peakRange)])
# peakVal = np.median(diffCut)
print peakVal
plt.scatter((magSort['diff'])[np.where(peakRange)], (magSort['mag'])[np.where(peakRange)], edgecolor='none', facecolor='blue', s=4)
while nRemoved != 0:
nBefore = diffCut.size
diffCheck = np.where(abs(peakVal-diffCut) < 2.5*diffCut.std())#[i for i,d in enumerate(diff) if (-0.5 < d < 0.0)]
#
diffCut = diffCut[diffCheck]
nRemoved = nBefore - diffCut.size
magCut = magCut[diffCheck]
xCut = xCut[diffCheck]
yCut = yCut[diffCheck]
idCut = idCut[diffCheck]
fwhmCut = fwhmCut[diffCheck]
print diffCut.size, nRemoved, np.median(diffCut), diffCut.std()
if 0.05 < diffCut.std() <0.06:
nRemoved=0
# plt.fill_betweenx(bin_centers, bin_meds+3.0*bin_stds, bin_meds-3.0*bin_stds, facecolor='red', edgecolor='none', alpha=0.4, label='2x RMS sigma clipping region')
# with open('escutSTD_i.pos','w+') as f:
# for i,blah in enumerate(xCut):
# print >> f, xCut[i], yCut[i], diffCut[i]
bin_meds, bin_edges, binnumber = stats.binned_statistic(magCut, diffCut, statistic='median', bins=24, range=(magCut.min(),magCut.max()))
bin_stds, bin_edges, binnumber = stats.binned_statistic(magCut, diffCut, statistic=np.std, bins=24, range=(magCut.min(),magCut.max()))
bin_width = (bin_edges[1] - bin_edges[0])
bin_centers = bin_edges[1:] - bin_width/2
# print bin_meds, bin_stds
left_edge = np.array(zip(peakVal-2.5*bin_stds, bin_centers))
right_edge = np.flipud(np.array(zip(peakVal+2.5*bin_stds, bin_centers)))
# print left_edge, right_edge
verts = np.vstack((left_edge, right_edge))
# print verts
# verts = np.delete(verts, np.array([0,1,2,22,23,24,25,45,46,47]), axis=0)
# print verts
esRegion = Path(verts)
sources = esRegion.contains_points(zip(diff,mag2x))
# print sources
with open('escutREG_i.pos','w+') as f:
for i,blah in enumerate(xpos[sources]):
print >> f, (xpos[sources])[i], (ypos[sources])[i], (diff[sources])[i]
magCut = mag2x[sources]
fwhmCut = fwhm[sources]
xCut = xpos[sources]
yCut = ypos[sources]
diffCut = diff[sources]
# find the sources that are in the std method but not the region method
# print idCut, idno[sources]
# extrasSTD = np.setdiff1d(idno[sources], idCut)
# print extrasSTD.size
# print extrasSTD
# with open('escutUNIQUE.pos','w+') as f:
# for i,blah in enumerate(extrasSTD):
# print >> f, xpos[blah-1], ypos[blah-1]
# fwhmcheck = np.loadtxt('testfwhmREG.log', usecols=(10,), unpack=True)
fwhmchk2 = np.where((magCut<-4) & (fwhmCut<90.0))
print np.median(fwhmCut[fwhmchk2]), np.std(fwhmCut[fwhmchk2])
fwchk = np.where(np.abs(fwhmCut-np.median(fwhmCut[fwhmchk2])) > 10.0*np.std(fwhmCut[fwhmchk2]))
drop = np.abs(fwhmCut-np.median(fwhmCut[fwhmchk2])) > 10.0*np.std(fwhmCut[fwhmchk2])
# fwmag = mag2x[sources]
with open('escutVBAD_i.pos','w+') as f:
for i,blah in enumerate(xCut[fwchk]):
print >> f, (xCut[fwchk])[i], (yCut[fwchk])[i]
with open('escut_r.pos','w+') as f:
for i,blah in enumerate(xCut):
if not drop[i]:
print >> f, xCut[i], yCut[i]
with open('escut_g.pos','w+') as f:
for i,blah in enumerate(xCut):
if not drop[i]:
print >> f, xCut[i], yCut[i]
plt.fill_betweenx(bin_centers, peakVal+2.5*bin_stds, peakVal-2.5*bin_stds, facecolor='red', edgecolor='none', alpha=0.4, label='2x RMS sigma clipping region')
plt.scatter(diffCut[fwchk], magCut[fwchk], edgecolor='none', facecolor='red', s=4)
plt.ylim(0,-12)
plt.xlabel('$m_{2x} - m_{1x}$')
plt.ylabel('$m_{2x}$')
plt.xlim(-2,1)
plt.savefig('testmagiraf.pdf')
plt.clf()
plt.scatter(magCut, fwhmCut, edgecolor='none', facecolor='black')
plt.scatter(magCut[fwchk], fwhmCut[fwchk], edgecolor='none', facecolor='red')
plt.hlines([np.median(fwhmCut)], -12, 0, colors='red', linestyle='dashed')
plt.hlines([np.median(fwhmCut)+fwhmCut.std(), np.median(fwhmCut)-fwhmCut.std()], -12, 0, colors='red', linestyle='dotted')
plt.ylim(0,20)
plt.xlim(-12,0)
plt.ylabel('fwhm')
plt.xlabel('$m_{2x}$')
plt.savefig('fwhmcheck.pdf')
return True
##########
##########
# I am essentially for looping easy_phot.
# This only uses iraf and no astropy so it can run on UW astro computers.
import numpy as np # for vectorized arrays
import os,sys
# Import IRAF and packages to use
from pyraf import iraf
iraf.imutil() # For hselect to grab from fits headers
import easy_phot as ep # for running methods in easy_phot.py specifically "execute()"
parameterFile = sys.argv[-1]
param_file = open(parameterFile, 'r')
param_lines = [line.rstrip('\n') for line in param_file]
param_file.close()
if(len(param_lines) is not 16):
print "Error...You have %d unique lines in the parameter file there should be 16."%len(param_lines)
print "Exiting program..."
sys.exit(1)
############
############ Global Parameters (You only make a parameter file!!)
def lickshane1Dredu(files,
_interactive,
_ext_trace,
_dispersionline,
_automaticex,
_verbose=False):
import lickshane
import datetime
import os
import re
import string
import sys
liststandard = ''
listatmo0 = ''
# os.environ["PYRAF_BETA_STATUS"] = "1"
_extinctdir = 'direc$standard/extinction/'
_extinction = 'lick.dat'
_observatory = 'lick'
now = datetime.datetime.now()
datenow = now.strftime('20%y%m%d%H%M')
MJDtoday = 55927 + (datetime.date.today() -
datetime.date(2012, 01, 01)).days
scal = np.pi / 180.
dv = lickshane.util.dvex()
std, rastd, decstd, magstd = lickshane.util.readstandard(
'standard_lick_mab.txt')
objectlist = {}
for img in files:
hdr = lickshane.util.readhdr(img)
img = re.sub('\n', '', img)
lickshane.util.correctcard(img)
_ra = lickshane.util.readkey3(hdr, 'RA')
_dec = lickshane.util.readkey3(hdr, 'DEC')
_object = lickshane.util.readkey3(hdr, 'object')
_grism = lickshane.util.readkey3(hdr, 'grism')
_slit = lickshane.util.readkey3(hdr, 'slit')
dd = np.arccos(
np.sin(_dec * scal) * np.sin(decstd * scal) +
np.cos(_dec * scal) * np.cos(decstd * scal) * np.cos(
(_ra - rastd) * scal)) * ((180 / np.pi) * 3600)
if min(dd) < 100:
_type = 'stdsens'
else:
_type = 'obj'
print img, _type
if min(dd) < 100:
lickshane.util.updateheader(img, 0,
{'stdname': [std[np.argmin(dd)], '']})
lickshane.util.updateheader(
img, 0, {'magstd': [float(magstd[np.argmin(dd)]), '']})
if _type not in objectlist:
objectlist[_type] = {}
if (_grism, _slit) not in objectlist[_type]:
objectlist[_type][_grism, _slit] = [img]
else:
objectlist[_type][_grism, _slit].append(img)
from pyraf import iraf
iraf.set(stdimage='imt2048')
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.specred(_doprint=0)
iraf.imutil(_doprint=0)
toforget = ['imutil.imcopy', 'specred.sarith', 'specred.standard']
for t in toforget:
iraf.unlearn(t)
iraf.specred.verbose = 'no'
iraf.specred.dispaxi = 2
iraf.set(direc=lickshane.__path__[0] + '/')
sens = {}
print objectlist
outputfile = []
if 'obj' in objectlist.keys():
tpe = 'obj'
elif 'stdsens' in objectlist.keys():
tpe = 'stdsens'
else:
sys.exit('error: no objects and no standards in the list')
for setup in objectlist[tpe]:
extracted = []
listatmo = []
if setup not in sens:
sens[setup] = []
if tpe == 'obj':
print '\n### setup= ', setup, '\n### objects= ', objectlist['obj'][
setup], '\n'
for img in objectlist['obj'][setup]:
# hdr=readhdr(img)
print '\n\n### next object= ', img, ' ', lickshane.util.readkey3(
lickshane.util.readhdr(img), 'object'), '\n'
#_automaticex = ''
imgex = lickshane.lickshane1Ddef.extractspectrum(
img,
dv,
_ext_trace,
_dispersionline,
_interactive,
'obj',
automaticex=_automaticex)
if not os.path.isfile(imgex):
sys.exit('### error, extraction not computed')
if not lickshane.util.readkey3(lickshane.util.readhdr(imgex), 'shift') and \
lickshane.util.readkey3(lickshane.util.readhdr(imgex), 'shift') != 0.0:
if setup in ['300_7500']:
lickshane.lickshane1Ddef.checkwavestd(
imgex, _interactive)
else:
print 'wave check using teluric not possible'
extracted.append(imgex)
if imgex not in outputfile:
outputfile.append(imgex)
lickshane.util.updateheader(
imgex, 0, {'TRACE1': [img, 'Originating file']})
if os.path.isfile('database/ap' +
re.sub('_ex.fits', '', imgex)):
if 'database/ap' + re.sub('_ex.fits', '',
imgex) not in outputfile:
outputfile.append('database/ap' +
re.sub('_ex.fits', '', imgex))
print '\n### all object with this setup extracted\n'
if liststandard:
standardlist = liststandard
_type = 'stdfromdreducer'
else:
try:
standardlist = objectlist['stdsens'][setup]
_type = 'stdsens'
except:
standardlist = ''
_type = ''
if _type == 'stdfromdreducer' and len(extracted) >= 1:
_outputsens2 = lichshane.util.searchsens(extracted[0],
standardlist)[0]
print '\n### using standard from reducer ' + str(_outputsens2)
elif _type not in ['stdsens', 'stdfromdreducer'
] and len(extracted) >= 1:
_outputsens2 = lickshane.util.searchsens(extracted[0], '')[0]
os.system('cp ' + _outputsens2 + ' .')
_outputsens2 = string.split(_outputsens2, '/')[-1]
print '\n### no standard in the list, using standard from archive'
else:
for simg in standardlist:
print '\n### standard for setup ' + \
str(setup) + ' = ', simg, ' ', lickshane.util.readkey3(
lickshane.util.readhdr(simg), 'object'), '\n'
simgex = lickshane.lickshane1Ddef.extractspectrum(
simg,
dv,
False,
False,
_interactive,
'std',
automaticex=_automaticex)
lickshane.util.updateheader(
simgex, 0, {'TRACE1': [simg, 'Originating file']})
if not lickshane.util.readkey3(
lickshane.util.readhdr(simgex),
'shift') and lickshane.util.readkey3(
lickshane.util.readhdr(simgex), 'shift') != 0.0:
lickshane.lickshane1Ddef.checkwavestd(simgex, _interactive)
print simgex
atmofile = lickshane.lickshane1Ddef.telluric_atmo(
simgex) # atmo file2
print atmofile
lickshane.util.updateheader(
atmofile, 0, {'TRACE1': [simgex, 'Originating file']})
if tpe != 'obj' and atmofile not in outputfile:
outputfile.append(atmofile)
if not listatmo0:
listatmo.append(atmofile)
sens[setup].append(simgex)
if simgex not in outputfile:
outputfile.append(simgex)
print '\n### standard available: ', sens[setup]
if tpe == 'obj':
if len(sens[setup]) > 1:
goon = 'no'
while goon != 'yes':
stdused = raw_input(
'\n### more than one standard for this setup, which one do you want to use ['
+ sens[setup][0] + '] ?')
if not stdused:
stdused = sens[setup][0]
if os.path.isfile(stdused):
goon = 'yes'
else:
stdused = sens[setup][0]
stdvec = [stdused]
else:
stdvec = sens[setup]
for stdused in stdvec:
stdusedclean = re.sub('_ex', '_clean', stdused)
lickshane.util.delete(stdusedclean)
iraf.specred.sarith(input1=stdused,
op='/',
input2=atmofile,
output=stdusedclean,
format='multispec')
_outputsens2 = lickshane.lickshane1Ddef.sensfunction(
stdusedclean, 'spline3', 16, _interactive)
lickshane.util.updateheader(
_outputsens2, 0, {'TRACE1': [stdused, 'Originating file']})
if _outputsens2 not in outputfile:
outputfile.append(_outputsens2)
if _outputsens2 and tpe == 'obj':
####################################################
for img in objectlist['obj'][setup]: # flux calibrate 2d images
imgd = fluxcalib2d(img, _outputsens2)
lickshane.util.updateheader(
imgd, 0, {'TRACE1': [img, 'Originating files']})
if imgd not in outputfile:
outputfile.append(imgd)
####################################################
# flux calib in the standard way
if not listatmo and listatmo0:
listatmo = listatmo0[:]
for _imgex in extracted:
_airmass = lickshane.util.readkey3(
lickshane.util.readhdr(_imgex), 'airmass')
_exptime = lickshane.util.readkey3(
lickshane.util.readhdr(_imgex), 'exptime')
_imgf = re.sub('_ex.fits', '_f.fits', _imgex)
lickshane.util.delete(_imgf)
qqq = iraf.specred.calibrate(input=_imgex,
output=_imgf,
sensiti=_outputsens2,
extinct='yes',
flux='yes',
extinction=_extinctdir +
_extinction,
observatory=_observatory,
airmass=_airmass,
ignorea='yes',
exptime=_exptime,
fnu='no')
hedvec = {
'SENSFUN': [_outputsens2, ''],
# 'SNR': [lickshane.util.StoN2(_imgf, False), 'Average signal to noise ratio per pixel'],
'BUNIT':
['erg/cm2/s/Angstrom', 'Physical unit of array values'],
'TRACE1': [_imgex, 'Originating file'],
'ASSON1': [
re.sub('_f.fits', '_2df.fits', _imgf),
'Name of associated file'
],
'ASSOC1':
['ANCILLARY.2DSPECTRUM', 'Category of associated file']
}
lickshane.util.updateheader(_imgf, 0, hedvec)
if _imgf not in outputfile:
outputfile.append(_imgf)
if listatmo:
atmofile = lickshane.util.searcharc(_imgex, listatmo)[0]
if atmofile:
_imge = re.sub('_f.fits', '_e.fits', _imgf)
lickshane.util.delete(_imge)
iraf.specred.sarith(input1=_imgf,
op='/',
input2=atmofile,
output=_imge,
w1='INDEF',
w2='INDEF',
format='multispec')
try:
iraf.imutil.imcopy(input=_imgf + '[*,1,2]',
output=_imge + '[*,1,2]',
verbose='no')
except:
pass
try:
iraf.imutil.imcopy(input=_imgf + '[*,1,3]',
output=_imge + '[*,1,3]',
verbose='no')
except:
pass
try:
iraf.imutil.imcopy(input=_imgf + '[*,1,4]',
output=_imge + '[*,1,4]',
verbose='no')
except:
pass
if _imge not in outputfile:
outputfile.append(_imge)
if atmofile not in outputfile:
outputfile.append(atmofile)
lickshane.util.updateheader(
_imge, 0, {'ATMOFILE': [atmofile, '']})
lickshane.util.updateheader(
_imge, 0, {'TRACE1': [_imgf, 'Originating file']})
imgin = _imge
else:
imgin = _imgf
else:
imgin = _imgf
imgasci = re.sub('.fits', '.asci', imgin)
lickshane.util.delete(imgasci)
iraf.onedspec(_doprint=0)
iraf.onedspec.wspectext(imgin + '[*,1,1]',
imgasci,
header='no')
if imgasci not in outputfile:
outputfile.append(imgasci)
return objectlist, 'ddd'
