def __init__(self,
rawnight,
procdir=None,
verb=None,
skyfilters=('i', 'z'),
Namps=4):
self.modName = string.split(string.split(str(self))[0],
'.')[0][1:] # this is the module name
self.verb = verb
self.cwd = os.getcwd()
self.tstart = time.time()
self.Namps = Namps
# Proc and Raw dirs
self.rawdir = rawnight
self.procdir = procdir
# The filters for which we will generate Sflats
self.skyfilters = skyfilters
# Check for environ vars
if not os.getenv('BCSPIPE'):
os.environ['BCSPIPE'] = '/Users/felipe/BCSPIPE.coligue'
self.BCSPIPE = os.getenv('BCSPIPE')
print(self.BCSPIPE)
# Add the the iraf env bcspipe
iraf.set(bcspipe=self.BCSPIPE)
return
def __init__(self, input, **kwargs):
"""
Init.
:param input: FITS association
:type input: string
:param kwargs: additional keyword arguments
:type kwargs: dictionary
"""
self.input = input
self.settings = dict(asndir='asn',
rawdir='raw',
jref='/grp/hst/cdbs/jref/',
mtab='/grp/hst/cdbs/mtab/',
sourceImage='POL0V_drz.fits',
sigma=5.0)
self.settings.update(kwargs)
#add env variables to both system and IRAF
os.putenv('jref', self.settings['jref'])
os.putenv('mtab', self.settings['mtab'])
iraf.set(jref=self.settings['jref'])
iraf.set(mtab=self.settings['mtab'])
#IRAF has funny True and False
self.yes = iraf.yes
self.no = iraf.no
def get_seeing(file, scale, ref):
coords = file.replace('fits','dao')
log = file.replace('fits','psf')
out = file.replace('fits','seeing')
iraf.noao()
iraf.noao.obsutil()
iraf.set(stdgraph="uepsf")
data = iraf.psfmeasure(file, coords='markall', wcs='logical', display='no', frame=1, level=0.5, size='FWHM', radius=10.0, sbuffer=1.0, swidth=3.0, iterations=1, logfile=log, imagecur=coords, Stdout=1)
fwhm = data.pop().split().pop()
fwhm_pix = float(fwhm)
fwhm = fwhm_pix*scale
seeing = dimm_seeing(fwhm_pix, ref, scale)
print "Seeing: ", seeing
fp = open(out, 'w')
fp.write("%f %f\n" % (fwhm_pix, seeing))
fp.close()
os.system("echo \"image;text 85 500 # text={Spot FWHM = %5.2f pixels}\" | xpaset WFS regions" % fwhm_pix)
os.system("echo \'image;text 460 500 # text={Seeing = %4.2f\"}\' | xpaset WFS regions" % seeing)
os.system("echo \"set wfs_seeing %4.2f\" | nc hacksaw 7666" % seeing)
# New function call to insert seeing value into MySQL database.
fits2mysql(file, mode, fwhm, seeing)
def ApplyBadPixelMask(images, cal_path, outbase = '_bpm'):
"""
#####################################################################################
# Fix a list of images with a bad pixel mask
#
# PARAMETERS:
# images - list of images to fix
# cal_path - path to calibration data
# outbase - addition to the name of the image after mask has been applied
#
# RETURNS:
# list of fixed images
#####################################################################################
"""
print '########################################'
print '# Applying Bad Pixel Masks'
print '########################################'
# finds the bad pixel mask
badPixelMask = [FindPixelMask(i, cal_path) for i in images]
# Build the list of output image names
out = [os.path.splitext(i) for i in images] # Split off the extension
out = [i[0] + outbase + '.fit' for i in out] # Paste the outbase at the end of the filename
# and put the extension back on
# Set up to call the task
iraf.set(clobber='yes')
iraf.proto()
iraf.fixpix.unlearn() # Forget everything
iraf.fixpix.mode = 'h' # Don't ask for user input
for (i,o,m) in zip(images, out, badPixelMask):
# gets image data and copies it to the corresponding output file name
shutil.copy(i,o)
#runs fixpix on the output file (fixpix overwrites file data)
if not os.path.isfile(i):
print i + ": Input file does not exist."
elif not os.path.isfile(m):
print i + ": No pixel mask applied."
else:
try:
#set parameters
iraf.fixpix.images = o
iraf.fixpix.masks = m
print i + ": applying mask at " + m
iraf.fixpix()
except iraf.IrafError, e:
print "fixpix failed"
print "error #" + str(e.errno)
print "Msg: " + e.errmsg
print "Task: " + e.errtask
raise e
def __init__(self, input, **kwargs):
"""
Init.
:param input: FITS association
:type input: string
:param kwargs: additional keyword arguments
:type kwargs: dictionary
"""
self.input = input
self.settings = dict(asndir='asn', rawdir='raw',
jref='/grp/hst/cdbs/jref/',
mtab='/grp/hst/cdbs/mtab/',
sourceImage='POL0V_drz.fits',
sigma=5.0)
self.settings.update(kwargs)
#add env variables to both system and IRAF
os.putenv('jref', self.settings['jref'])
os.putenv('mtab', self.settings['mtab'])
iraf.set(jref=self.settings['jref'])
iraf.set(mtab=self.settings['mtab'])
#IRAF has funny True and False
self.yes = iraf.yes
self.no = iraf.no
def initialize_instrument(self, output_dir):
self.logger.info("Initializing instrument.")
inst_file_name = output_dir + '/tmp/cp.dat'
if self.debug:
iraf.set(debug=1)
iraf.set(use_new_imt='no')
iraf.noao.imred(_doprint=0)
iraf.noao.imred.ccdred(_doprint=0)
if not os.path.exists(inst_file_name):
with open(inst_file_name, 'w+') as inst_file:
inst_file.write('subset FILTER\n\n')
inst_file.write('darktime EXPTIME\n\n')
inst_file.write('\'Dark Frame\' dark\n')
inst_file.write('\'Bias Frame\' zero\n')
inst_file.write('\'Light Frame\' object\n')
inst_file.write('\'Flat Field\' flat\n')
iraf.noao.imred.ccdred.setinst(instrument='cp',
review='no',
mode='h',
dir=output_dir + '/tmp/',
site='')
iraf.noao.digiphot(_doprint=0)
iraf.noao.digiphot.apphot(_doprint=0)
iraf.dataio(_doprint=0)
iraf.noao.digiphot.ptools(_doprint=0)
def credit(frameid):
if os.path.exists(frameid + '.cr_org.fits'):
print(frameid + '.cr_org.fits exists. ' +
'No need to backup the original file.')
else:
print('Copying ' + frameid + '.cr.fits to ' + frameid + '.cr_org.fits')
shutil.copy(frameid + '.cr.fits', frameid + '.cr_org.fits')
if os.path.exists(frameid + '.mask_org.fits'):
print(frameid + '.mask_org.fits exists. ' +
'No need to backup the original file.')
else:
print('Copying ' + frameid + '.mask.fits to ' + frameid +
'.mask_org.fits')
shutil.copy(frameid + '.mask.fits', frameid + '.mask_org.fits')
iraf.set(stdimage='imt8192')
iraf.imedit(
frameid + '.cr.fits',
command='display $image 1 erase=$erase fill=no order=0 >& dev$null')
data_ov = fits.getdata(frameid + '.ov.fits')
data_cr = fits.getdata(frameid + '.cr.fits')
hdl_mask = fits.open(frameid + '.mask.fits', mode='update')
data_temp = data_ov - data_cr
np.seterr(invalid='ignore') # ignore RuntimeWarning in dividing 0 by 0.
hdl_mask[0].data = np.nan_to_num(data_temp / data_temp)
hdl_mask.close()
return
def initialize_instrument(self, output_dir):
self.logger.info("Initializing instrument.")
inst_file_name = output_dir + '/tmp/cp.dat'
if self.debug:
iraf.set(debug=1)
iraf.set(use_new_imt='no')
iraf.noao.imred(_doprint=0)
iraf.noao.imred.ccdred(_doprint=0)
if not os.path.exists(inst_file_name):
with open(inst_file_name, 'w+') as inst_file:
inst_file.write('subset FILTER\n\n')
inst_file.write('darktime EXPTIME\n\n')
inst_file.write('\'Dark Frame\' dark\n')
inst_file.write('\'Bias Frame\' zero\n')
inst_file.write('\'Light Frame\' object\n')
inst_file.write('\'Flat Field\' flat\n')
iraf.noao.imred.ccdred.setinst(
instrument = 'cp',
review = 'no',
mode = 'h',
dir = output_dir + '/tmp/',
site = ''
)
iraf.noao.digiphot(_doprint=0)
iraf.noao.digiphot.apphot(_doprint=0)
iraf.dataio(_doprint=0)
iraf.noao.digiphot.ptools(_doprint=0)
def display(self,extn=None):
"""Display SCI extensions from the Astrodata object
and mark objects and reference stars found in the
input file tables OBJCAT and REFCAT.
DS9 must be running.
"""
try:
from stsci.numdisplay import display as ndisplay
except ImportError:
from numdisplay import display as ndisplay
#file = self.file
#ad = AstroData(file)
ad = self.adout
nscext = ad.count_exts('SCI')
if extn != None:
si,ei = extn,extn+1
else:
si,ei = 1,nscext+1
fr = 1
for xt in range(si,ei):
#ff = '%s[%d]' % (file,xt)
ff = ad['SCI',xt].data
iraf.set (stdimage='imt47')
ndisplay(ff, frame=fr)
xtnad = ad['OBJCAT',xt]
if xtnad:
x = xtnad.data.field('x')
y = xtnad.data.field('y')
print 'Marking %d objects with red.' % len(x)
np.savetxt('/tmp/xy.txt', zip(x,y), fmt='%.2f', delimiter=' ')
tvmark(frame=fr,coords='/tmp/xy.txt',color=204)
else:
print "No stars found in extension %d."%xt
xtnad = ad['REFCAT',xt]
if xtnad:
x = xtnad.data.field('x')
y = xtnad.data.field('y')
print 'Marking %d refstars with green.' % len(x)
np.savetxt('/tmp/xy.txt', zip(x,y), fmt='%.2f', delimiter=' ')
tvmark(frame=fr,coords='/tmp/xy.txt',color=205)
else:
print "No reference stars found in extension %d."%xt
fr += 1
def build(self, fwhm=None, maxlin=40000, zeropoint=None):
"""
@param fwhm: initial guess of the FWHM, used for selecting sources that might be stars
@param thresh: the detection threshold for stars (in sky sigma)
@param maxlin: maximum linearity of the CCD, brighter sources are saturated.
@param zeropoint: the cts/exptime mag zeropoint of the image
@return:
"""
if not zeropoint:
zeropoint = storage.get_zeropoint(self.expnum,
self.ccd,
self.prefix,
self.version)
if not fwhm:
fwhm = storage.get_fwhm(self.expnum,self.ccd,prefix=self.prefix, version=self.version)
filename = storage.get_image(self.expnum,
self.ccd,
version=self.version,
prefix=self.prefix)
basename = os.path.splitext(filename)[0]
iraf.set(writepars=0)
iraf.kinky
iraf.pipeline
phot_filename = "{}.{}".format(basename, PHOT_EXT)
psf_filename = "{}.{}".format(basename, PSF_EXT)
zeropt_filename = "{}.{}".format(basename, ZERO_PT_USED)
with open(zeropt_filename, 'w') as z:
z.write("{}\n".format(zeropoint))
apmax = AP_OUT_SCALE * fwhm
swidth = fwhm
# Turn off verbosity and parameter checking in DAOPHOT
iraf.daophot.verify=iraf.no
iraf.daophot.verbose=iraf.no
iraf.daophot.update=iraf.no
iraf.psf.showplots=iraf.no
iraf.psf.interactive=iraf.no
iraf.centerpars.calgori="centroid"
iraf.centerpars.maxshift=3
# set the basic 'datapars' for the images.
iraf.datapars.exposure=""
# Note the itime is set to 1 as the SGwyn ZP includes the exposure time.
iraf.datapars.itime=1
def marksn2(img,fitstab,frame=1,fitstab2='',verbose=False):
from pyraf import iraf
from numpy import array #,log10
import lsc
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.daophot(_doprint=0)
iraf.images(_doprint=0)
iraf.imcoords(_doprint=0)
iraf.proto(_doprint=0)
iraf.set(stdimage='imt1024')
hdr=lsc.util.readhdr(fitstab)
_filter=lsc.util.readkey3(hdr,'filter')
column=lsc.lscabsphotdef.makecatalogue([fitstab])[_filter][fitstab]
rasex=array(column['ra0'],float)
decsex=array(column['dec0'],float)
if fitstab2:
hdr=lsc.util.readhdr(fitstab2)
_filter=lsc.util.readkey3(hdr,'filter')
_exptime=lsc.util.readkey3(hdr,'exptime')
column=lsc.lscabsphotdef.makecatalogue([fitstab2])[_filter][fitstab2]
rasex2=array(column['ra0'],float)
decsex2=array(column['dec0'],float)
iraf.set(stdimage='imt1024')
iraf.display(img + '[0]',frame,fill=True,Stdout=1)
vector=[]
for i in range(0,len(rasex)):
vector.append(str(rasex[i])+' '+str(decsex[i]))
xy = iraf.wcsctran('STDIN',output="STDOUT",Stdin=vector,Stdout=1,image=img+'[0]',inwcs='world',units='degrees degrees',outwcs='logical',\
formats='%10.1f %10.1f',verbose='yes')[3:]
iraf.tvmark(frame,'STDIN',Stdin=list(xy),mark="circle",number='yes',label='no',radii=10,nxoffse=5,nyoffse=5,color=207,txsize=2)
if verbose:
# print 2.5*log10(_exptime)
for i in range(0,len(column['ra0'])):
print xy[i],column['ra0'][i],column['dec0'][i],column['magp3'][i],column['magp4'][i],column['smagf'][i],column['magp2'][i]
if fitstab2:
vector2=[]
for i in range(0,len(rasex2)):
vector2.append(str(rasex2[i])+' '+str(decsex2[i]))
xy1 = iraf.wcsctran('STDIN',output="STDOUT",Stdin=vector2,Stdout=1,image=img+'[0]',inwcs='world',units='degrees degrees',outwcs='logical',\
formats='%10.1f %10.1f',verbose='yes')[3:]
iraf.tvmark(frame,'STDIN',Stdin=list(xy1),mark="cross",number='yes',label='no',radii=10,nxoffse=5,nyoffse=5,color=205,txsize=2)
def buildtmc(tmcname):
from pyraf import iraf
from iraf import stsdas, hst_calib, synphot
out = open('buildtmc.log', 'w')
f = pyfits.open(tmcname)
flist = f[1].data.field('filename')
iraf.set(crrefer='./') #work locally
for k in range(len(flist)):
oldname = iraf.osfn(flist[k]).split('[')[0]
newname = fincre(oldname)
if os.path.exists(newname):
flist[k] = fincre(flist[k])
else:
out.write("%s: no change necessary\n" % oldname)
f.writeto(tmcname.replace(".fits", "_new.fits"))
out.close()
def buildtmc(tmcname):
from pyraf import iraf
from iraf import stsdas,hst_calib,synphot
out=open('buildtmc.log','w')
f=pyfits.open(tmcname)
flist=f[1].data.field('filename')
iraf.set(crrefer='./') #work locally
for k in range(len(flist)):
oldname=iraf.osfn(flist[k]).split('[')[0]
newname=fincre(oldname)
if os.path.exists(newname):
flist[k]=fincre(flist[k])
else:
out.write("%s: no change necessary\n"%oldname)
f.writeto(tmcname.replace(".fits","_new.fits"))
out.close()
def atmofile(imgstd, imgout=''):
# print "LOGX:: Entering `atmofile` method/function in %(__file__)s" %
# globals()
from pyraf import iraf
import os
import ntt
iraf.noao(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.set(direc=ntt.__path__[0] + '/')
_cursor = 'direc$standard/ident/cursor_sky_0'
if not imgout:
imgout = 'atmo_' + imgstd
os.system('rm -rf ' + imgout)
iraf.noao.onedspec.bplot(imgstd, cursor=_cursor,
spec2=imgstd, new_ima=imgout, overwri='yes')
return imgout
def __init__(self,
shortparlists,
parlists,
FitsDir,
logfile,
verbose=1,
clean_up=1,
skyKey='ALIGNSKY',
hdrGain=0,
crlower=None,
imNsci=1):
self.modName = string.split(string.split(str(self))[0], '.')[0][1:]
self.shortparlists = shortparlists
self.parlists = parlists
self.Fits = FitsDir
self.verbose = verbose
self.crmasks = {} # cosmic ray masks names
self.removeList = []
self.clean_up = clean_up
self.skyKey = skyKey
self.hdrGain = hdrGain
self.crlower = crlower
if imNsci < 1:
raise ValueError, 'Error: pyblot got imNsci = ' + imNsci
self.imNsci = imNsci
self.logfile = logfile
print self.modName, 'version', __version__
self.logfile.write('Instantiating ' + self.modName + ' version ' +
__version__)
# make sure these packages are loaded
iraf.stsdas()
iraf.toolbox()
iraf.imgtool()
iraf.fourier()
iraf.fitting()
iraf.ttools()
iraf.analysis()
iraf.dither()
# flush the cash! twice!
iraf.flpr()
iraf.flpr()
iraf.reset(imtype='fits') # seems to make deriv task a bit happier
iraf.set(tmp='./')
def atmofile(imgstd, imgout=''):
from pyraf import iraf
import os
import ntt
iraf.noao(_doprint=0, Stdout=0)
iraf.onedspec(_doprint=0, Stdout=0)
iraf.set(direc=ntt.__path__[0] + '/')
_cursor = 'direc$standard/ident/cursor_sky_0'
if not imgout:
imgout = 'atmo_' + imgstd
os.system('rm -rf ' + imgout)
iraf.noao.onedspec.bplot(imgstd,
cursor=_cursor,
spec2=imgstd,
new_ima=imgout,
overwri='yes')
return imgout
def doPhot():
print 'Running doPhot'
try:
mapFile = open('map.dat')
except:
print 'map.dat not found. Exiting.'
sys.exit()
try:
changeFile = open('change.dat')
except:
print 'change.dat not found. Exiting.'
sys.exit()
coordList = list()
for line in mapFile:
coords = map(float,line.split())
coordList.append(coords)
coordList = np.array(coordList)
iraf.noao()
iraf.digiphot()
iraf.daophot()
iraf.ptools()
iraf.set(clobber='yes')
photFile = open('phot.dat', 'w')
for line in changeFile.readlines():
elms = line.split()
imageName = elms[0]
changeCoords = np.array([float(elms[1]),float(elms[2])])
newCoords = coordList + changeCoords
print 'Image: '+imageName
coordFile = makeCoordFile(newCoords)
iraf.noao.digiphot.daophot.phot(image=imagesDir+imageName, coords=coordFile, output='.temp-phot', skyfile='', verify='no', fwhmpsf=fwhmpsf, sigma=sigma, readnoise=readnoise, epadu=epadu, exposure=exposureHeader, obstime=obstimeHeader, calgorithm=calgorithm, cbox=cbox, apertures=apertures, annulus=annulus, dannulus=dannulus)
result = iraf.noao.digiphot.ptools.txdump(Stdout=1, textfiles='.temp-phot', fields='mag, merr, otime', expr='yes')
writeString = result[0].split()[-1] +' '+ ' '.join([' '.join(x.split()[:2]) for x in result])
photFile.write(writeString+"\n")
photFile.close()
raw_input('doPhot Done. Hit return key to continue.')
def atmofile(imgstd, imgout=''):
# print "LOGX:: Entering `atmofile` method/function in %(__file__)s" %
# globals()
from pyraf import iraf
import os
import ntt
iraf.noao(_doprint=0)
iraf.onedspec(_doprint=0)
iraf.set(direc=ntt.__path__[0] + '/')
_cursor = 'direc$standard/ident/cursor_sky_0'
if not imgout:
imgout = 'atmo_' + imgstd
os.system('rm -rf ' + imgout)
iraf.noao.onedspec.bplot(imgstd,
cursor=_cursor,
spec2=imgstd,
new_ima=imgout,
overwri='yes')
return imgout
def setiraf(self):
"""
Set IRAF global parameters and load DAOPHOT package for aperture
photometry.
Parameters
----------
Returns
-------
None
"""
iraf.prcacheOff()
iraf.set(writepars=0)
# Load IRAF packages
iraf.noao(_doprint = 0)
iraf.noao.digiphot(_doprint = 0)
iraf.noao.digiphot.daophot(_doprint = 0)
return
def setiraf(self):
"""
Set IRAF global parameters and load DAOPHOT package for aperture
photometry.
Parameters
----------
Returns
-------
None
"""
iraf.prcacheOff()
iraf.set(writepars=0)
# Load IRAF packages
iraf.noao(_doprint = 0)
iraf.noao.digiphot(_doprint = 0)
iraf.noao.digiphot.daophot(_doprint = 0)
return
def get_seeing(file, scale, ref):
# log = file.replace('fits','psf') # never used
# Gets called with full path to the fits file
# print "daofind get_seeing: ", file
iraf.noao()
iraf.noao.obsutil()
iraf.set(stdgraph="uepsf")
# name of file with coordinates of star images.
coords = file.replace('fits','dao')
data = iraf.psfmeasure(file, coords='markall', wcs='logical', display='no', frame=1, level=0.5, size='FWHM', radius=10.0, sbuffer=1.0, swidth=3.0, iterations=1, logfile=log, imagecur=coords, Stdout=1)
# print "daofind psfmeasure data: ", data
# last = data.pop()
# data.append(last)
# print "daofind psfmeasure last line of data: ", last
# psfmeasure returns lots of information (data for each spot), but we just
# want the last line (the average fwhm), which looks like:
# Average full width at half maximum (FWHM) of 6.0163
fwhm = data.pop().split().pop()
print "daofind psfmeasure fwhm: ", fwhm
fwhm_pix = float(fwhm)
fwhm = fwhm_pix*scale
seeing = dimm_seeing(fwhm_pix, ref, scale)
print "Seeing: ", seeing
seeingfile = file.replace('fits','seeing')
fp = open(seeingfile, 'w')
fp.write("%f %f\n" % (fwhm_pix, seeing))
fp.close()
os.system("echo \"image;text 85 500 # text={Spot FWHM = %5.2f pixels}\" | xpaset WFS regions" % fwhm_pix)
os.system("echo \'image;text 460 500 # text={Seeing = %4.2f\"}\' | xpaset WFS regions" % seeing)
os.system("echo \"set wfs_seeing %4.2f\" | nc hacksaw 7666" % seeing)
def get_seeing(file, scale, ref):
coords = file.replace("fits", "dao")
log = file.replace("fits", "psf")
out = file.replace("fits", "seeing")
iraf.noao()
iraf.noao.obsutil()
iraf.set(stdgraph="uepsf")
data = iraf.psfmeasure(
file,
coords="markall",
wcs="logical",
display="no",
frame=1,
level=0.5,
size="FWHM",
radius=10.0,
sbuffer=1.0,
swidth=3.0,
iterations=1,
logfile=log,
imagecur=coords,
Stdout=1,
)
fwhm = data.pop().split().pop()
fwhm_pix = float(fwhm)
fwhm = fwhm_pix * scale
seeing = dimm_seeing(fwhm_pix, ref, scale)
print seeing
fp = open(out, "w")
fp.write("%f %f\n" % (fwhm_pix, seeing))
fp.close()
os.system('echo "image;text 85 500 # text={Spot FWHM = %5.2f pixels}" | xpaset WFS regions' % fwhm_pix)
os.system("echo 'image;text 460 500 # text={Seeing = %4.2f\"}' | xpaset WFS regions" % seeing)
os.system('echo "set wfs_seeing %4.2f" | nc hacksaw 7666' % seeing)
def dark_sky_flat(filter):
med_otalist = []
print 'making dark sky flats for',filter
for key in tqdm(odi.OTA_dictionary):
image_list = odi.OTA_dictionary[key]+'.'+filter+'.lis'
med_out = image_list.replace('.lis','.med.fits')
med_otalist.append(med_out)
iraf.unlearn(iraf.immatch.imcombine)
iraf.immatch.imcombine.setParam('input','@'+str(image_list))
iraf.immatch.imcombine.setParam('output',odi.skyflatpath+med_out)
iraf.immatch.imcombine.setParam('combine','median')
iraf.immatch.imcombine.setParam('masktype','goodvalue')
iraf.immatch.imcombine.setParam('maskvalue',0)
iraf.immatch.imcombine.setParam('scale','median')
# iraf.immatch.imcombine.setParam('zero','none')
iraf.immatch.imcombine.setParam('zero','median')
iraf.immatch.imcombine(logfile='imcombine.log.txt', mode='h')
if key == 1:
data,header = odi.fits.getdata(odi.skyflatpath+med_out,header=True)
mean, median, std = odi.sigma_clipped_stats(data, sigma=3.0)
normalization_factor = median
iraf.set(clobber = 'yes')
print 'smoothing dark sky flats for',filter
for i in tqdm(range(len(med_otalist))):
iraf.unlearn(iraf.imutil.imarith,iraf.imfilter.median)
iraf.imutil.imarith.setParam('operand1',odi.skyflatpath+med_otalist[i])
iraf.imutil.imarith.setParam('op','/')
iraf.imutil.imarith.setParam('operand2',normalization_factor)
iraf.imutil.imarith.setParam('result',odi.skyflatpath+med_otalist[i])
iraf.imutil.imarith(verbose='no', mode='h')
iraf.imfilter.median.setParam('input',odi.skyflatpath+med_otalist[i])
iraf.imfilter.median.setParam('output',odi.skyflatpath+med_otalist[i])
iraf.imfilter.median.setParam('xwindow',3)
iraf.imfilter.median.setParam('ywindow',3)
iraf.imfilter.median(verbose='no', mode='h')
iraf.set(clobber = 'no')
return normalization_factor
def get_star_data(asteroid_id, mag, expnum, header):
"""
From ossos psf fitted image, calculate mean of the flux of each row of the rotated PSF
"""
# calculate mean psf
uri = storage.get_uri(expnum.strip('p'), header[_CCD].split('d')[1])
ossos_psf = '{}.psf.fits'.format(uri.strip('.fits'))
local_psf = '{}{}.psf.fits'.format(expnum, header[_CCD].split('d')[1])
local_file_path = '{}/{}'.format(_STAMPS_DIR, local_psf)
storage.copy(ossos_psf, local_file_path)
# pvwcs = wcs.WCS(header)
# x, y = pvwcs.sky2xy(asteroid_id['ra'].values, asteroid_id['dec'].values)
x = asteroid_id[_XMID_HEADER].values[0]
y = asteroid_id[_YMID_HEADER].values[0]
# run seepsf on the mean psf image
iraf.set(uparm="./")
iraf.digiphot(_doprint=0)
iraf.apphot(_doprint=0)
iraf.daophot(_doprint=0)
iraf.seepsf(local_file_path, local_psf, xpsf=x, ypsf=y, magnitude=mag)
with fits.open(local_psf) as hdulist:
data = hdulist[0].data
th = math.degrees(asteroid_id[_THETA_HEADER].values[0])
data_rot = rotate(data, th)
data_rot = np.ma.masked_where(data_rot == 0, data_rot)
data_mean = np.ma.mean(data_rot, axis=1)
os.unlink(local_psf)
os.unlink(local_file_path)
return data_mean[np.nonzero(np.ma.fix_invalid(data_mean, fill_value=0))[0]]
def mask_create(images, no_unlearn=False):
output_files = []
# If the output file list doesn't exsist, create it.
if len(output_files) == 0:
output_files = [os.path.join(os.path.dirname(i), "mask_" + os.path.basename(i)) for i in images]
# Create a temporary file to hold the input image list
tfile = tempfile.mkstemp()
os.write(tfile[0], "\n".join(images) + "\n")
os.close(tfile[0])
# Create a temporary file to hold the input image list
tofile = tempfile.mkstemp()
os.write(tfile[0], "\n".join(output_files) + "\n")
os.close(tfile[0])
try:
# Set up to call the task
iraf.set(clobber="yes")
iraf.nproto()
if no_unlearn == False:
print "Objmasks is unlearning!!"
iraf.objmasks.unlearn()
# set parameters
iraf.objmasks.images = "@" + tfile[1]
iraf.objmasks.objmasks = "@" + tofile[1]
iraf.objmasks.omtype = "boolean"
iraf.objmasks()
except iraf.IrafError, e:
print "objmasks failed"
print "error #" + str(e.errno)
print "Msg: " + e.errmsg
print "Task: " + e.errtask
raise e
def dark_sky_flat():
med_otalist = []
for key in OTA_dictionary:
image_list = OTA_dictionary[key]+'.lis'
med_out = image_list.replace('.lis','.med.fits')
med_otalist.append(med_out)
iraf.unlearn(iraf.immatch.imcombine)
iraf.immatch.imcombine.setParam('input','@'+str(image_list))
iraf.immatch.imcombine.setParam('output',med_out)
iraf.immatch.imcombine.setParam('combine','median')
iraf.immatch.imcombine.setParam('masktype','goodvalue')
iraf.immatch.imcombine.setParam('maskvalue',0)
iraf.immatch.imcombine.setParam('scale','median')
iraf.immatch.imcombine.setParam('zero','median')
iraf.immatch.imcombine(mode='h')
if key == 1:
data,header = fits.getdata(med_out,header=True)
mean, median, std = sigma_clipped_stats(data, sigma=3.0)
normalization_factor = median
iraf.set(clobber = 'yes')
for i in range(len(med_otalist)):
iraf.unlearn(iraf.imutil.imarith,iraf.imfilter.median)
iraf.imutil.imarith.setParam('operand1',med_otalist[i])
iraf.imutil.imarith.setParam('op','/')
iraf.imutil.imarith.setParam('operand2',normalization_factor)
iraf.imutil.imarith.setParam('result',med_otalist[i])
iraf.imutil.imarith(mode='h')
iraf.imfilter.median.setParam('input',med_otalist[i])
iraf.imfilter.median.setParam('output',med_otalist[i])
iraf.imfilter.median.setParam('xwindow',3)
iraf.imfilter.median.setParam('ywindow',3)
iraf.imfilter.median()
iraf.set(clobber = 'no')
return normalization_factor
def setIraf(obs, verb=None):
# Set vars
extras.cl_bye(verb)
iraf.set(imclobber="yes")
iraf.set(clobber="yes")
iraf.set(Verbose='yes')
# Initialize some tasks
print(" setting up IRAF's initial params", file=sys.stderr)
#set_instrument(verb)
set_mscred(verb)
set_ccdproc(obs, verb)
return
def plant_kbos(filename, psf, kbos, shifts, prefix):
"""
Add KBOs to an image
:param filename: name of the image to add KBOs to
:param psf: Point Spread Function in IRAF/DAOPHOT format, used by ADDSTAR
:param kbos: list of KBOs to add, has format as returned by KBOGenerator
:param shifts: dictionary to transform coordinates to reference frame.
:param prefix: an estimate FWHM of the image, used to determine trailing.
:return: None
"""
iraf.set(uparm="./")
iraf.digiphot()
iraf.apphot()
iraf.daophot(_doprint=0)
iraf.images()
if shifts['nmag'] < 4:
logging.warning("Mag shift based on fewer than 4 common stars.")
fd = open("plant.WARNING", 'a')
fd.write("Mag shift based on fewer than 4 common stars.")
fd.close()
if shifts['emag'] > 0.05:
logging.warning("Mag shift has large uncertainty.")
fd = open("plant.WARNING", 'a')
fd.write("Mag shift hsa large uncertainty.")
fd.close()
addstar = tempfile.NamedTemporaryFile(suffix=".add", mode='w')
# transform KBO locations to this frame using the shifts provided.
w = get_wcs(shifts)
header = fits.open(filename)[0].header
# set the rate of motion in units of pixels/hour instead of ''/hour
scale = header['PIXSCAL1']
rate = kbos['sky_rate'] / scale
# compute the location of the KBOs in the current frame.
# offset magnitudes from the reference frame to the current one.
mag = kbos['mag'] - shifts['dmag']
angle = radians(kbos['angle'])
# Move the x/y locations to account for the sky motion of the source.
x = kbos['x'] - rate * 24.0 * shifts['dmjd'] * cos(angle)
y = kbos['y'] - rate * 24.0 * shifts['dmjd'] * sin(angle)
x, y = w.wcs_world2pix(x, y, 1)
# Each source will be added as a series of PSFs so that a new PSF is
# added for each pixel the source moves.
itime = float(header['EXPTIME']) / 3600.0
npsf = fabs(rint(rate * itime)) + 1
mag += 2.5 * log10(npsf)
dt_per_psf = itime / npsf
# Build an addstar file to be used in the planting of source.
idx = 0
for record in transpose([x, y, mag, npsf, dt_per_psf, rate, angle]):
x = record[0]
y = record[1]
mag = record[2]
npsf = record[3]
dt = record[4]
rate = record[5]
angle = record[6]
for i in range(int(npsf)):
idx += 1
x += dt * rate * math.cos(angle)
y += dt * rate * math.sin(angle)
addstar.write("{} {} {} {}\n".format(x, y, mag, idx))
addstar.flush()
fk_image = prefix + filename
try:
os.unlink(fk_image)
except OSError as err:
if err.errno == errno.ENOENT:
pass
else:
raise
# add the sources to the image.
if os.access(f'{fk_image}.art', os.R_OK):
os.unlink(f'{fk_image}.art')
iraf.daophot.addstar(filename,
addstar.name,
psf,
fk_image,
simple=True,
verify=False,
verbose=False)
# convert the image to short integers.
iraf.images.chpix(fk_image, fk_image, 'ushort')
def load_modules():
# Define a function to load all of the modules so that they don't' import
# unless we need them
global iraf
from pyraf import iraf
iraf.pysalt()
iraf.saltspec()
iraf.saltred()
iraf.set(clobber='YES')
global sys
import sys
global os
import os
global shutil
import shutil
global glob
from glob import glob
global pyfits
import pyfits
global np
import numpy as np
global lacosmicx
import lacosmicx
global interp
from scipy import interp
global signal
from scipy import signal
global ndimage
from scipy import ndimage
global interpolate
from scipy import interpolate
global WCS
from astropy.wcs import WCS
global optimize
from scipy import optimize
global ds9
import pyds9 as ds9
global GaussianProcess
from sklearn.gaussian_process import GaussianProcess
global pandas
import pandas
iraf.onedspec()
iraf.twodspec()
iraf.longslit()
iraf.apextract()
iraf.imutil()
iraf.rvsao(motd='no')
def efoscspec1Dredu(files, _interactive, _ext_trace, _dispersionline, liststandard, listatmo0, _automaticex,
_verbose=False):
# print "LOGX:: Entering `efoscspec1Dredu` method/function in
# %(__file__)s" % globals()
import ntt
try: import pyfits
except: from astropy.io import fits as pyfits
import re
import string
import sys
import os
import numpy as np
os.environ["PYRAF_BETA_STATUS"] = "1"
_extinctdir = 'direc$standard/extinction/'
_extinction = 'lasilla2.txt'
_observatory = 'lasilla'
import datetime
now = datetime.datetime.now()
datenow = now.strftime('20%y%m%d%H%M')
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
dv = ntt.dvex()
scal = np.pi / 180.
_gain = ntt.util.readkey3(ntt.util.readhdr(
re.sub('\n', '', files[0])), 'gain')
_rdnoise = ntt.util.readkey3(
ntt.util.readhdr(re.sub('\n', '', files[0])), 'ron')
std, rastd, decstd, magstd = ntt.util.readstandard(
'standard_efosc_mab.txt')
objectlist = {}
for img in files:
hdr = ntt.util.readhdr(img)
img = re.sub('\n', '', img)
ntt.util.correctcard(img)
_ra = ntt.util.readkey3(hdr, 'RA')
_dec = ntt.util.readkey3(hdr, 'DEC')
_object = ntt.util.readkey3(hdr, 'object')
_grism = ntt.util.readkey3(hdr, 'grism')
_filter = ntt.util.readkey3(hdr, 'filter')
_slit = ntt.util.readkey3(hdr, 'slit')
dd = np.arccos(np.sin(_dec * scal) * np.sin(decstd * scal) + np.cos(_dec * scal) *
np.cos(decstd * scal) * np.cos((_ra - rastd) * scal)) * ((180 / np.pi) * 3600)
if min(dd) < 100:
_type = 'stdsens'
else:
_type = 'obj'
if min(dd) < 100:
ntt.util.updateheader(
img, 0, {'stdname': [std[np.argmin(dd)], '']})
ntt.util.updateheader(
img, 0, {'magstd': [float(magstd[np.argmin(dd)]), '']})
if _type not in objectlist:
objectlist[_type] = {}
if (_grism, _filter, _slit) not in objectlist[_type]:
objectlist[_type][_grism, _filter, _slit] = [img]
else:
objectlist[_type][_grism, _filter, _slit].append(img)
from pyraf import iraf
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.specred(_doprint=0)
iraf.imutil(_doprint=0)
toforget = ['imutil.imcopy', 'specred.sarith', 'specred.standard']
for t in toforget:
iraf.unlearn(t)
iraf.specred.verbose = 'no'
iraf.specred.dispaxi = 2
iraf.set(direc=ntt.__path__[0] + '/')
sens = {}
print objectlist
outputfile = []
if 'obj' in objectlist.keys():
tpe = 'obj'
elif 'stdsens' in objectlist.keys():
tpe = 'stdsens'
else:
sys.exit('error: no objects and no standards in the list')
for setup in objectlist[tpe]:
extracted = []
listatmo = []
if setup not in sens:
sens[setup] = []
if tpe == 'obj':
print '\n### setup= ', setup, '\n### objects= ', objectlist['obj'][setup], '\n'
for img in objectlist['obj'][setup]:
# hdr=readhdr(img)
print '\n\n### next object= ', img, ' ', ntt.util.readkey3(ntt.util.readhdr(img), 'object'), '\n'
if os.path.isfile(re.sub('.fits', '_ex.fits', img)):
if ntt.util.readkey3(ntt.util.readhdr(re.sub('.fits', '_ex.fits', img)), 'quality') == 'Rapid':
ntt.util.delete(re.sub('.fits', '_ex.fits', img))
imgex = ntt.util.extractspectrum(img, dv, _ext_trace, _dispersionline, _interactive, 'obj',
automaticex=_automaticex)
if not os.path.isfile(imgex):
sys.exit('### error, extraction not computed')
if not ntt.util.readkey3(ntt.util.readhdr(imgex), 'shift') and \
ntt.util.readkey3(ntt.util.readhdr(imgex), 'shift') != 0.0:
# if not readkey3(readhdr(imgex),'shift'):
ntt.efoscspec1Ddef.checkwavestd(imgex, _interactive)
extracted.append(imgex)
if imgex not in outputfile:
outputfile.append(imgex)
ntt.util.updateheader(
imgex, 0, {'FILETYPE': [22107, 'extracted 1D spectrum ']})
ntt.util.updateheader(imgex, 0, {
'PRODCATG': ['SCIENCE.' +
ntt.util.readkey3(ntt.util.readhdr(imgex), 'tech').upper(), 'Data product category']})
ntt.util.updateheader(
imgex, 0, {'TRACE1': [img, 'Originating file']})
if os.path.isfile('database/ap' + re.sub('_ex.fits', '', imgex)):
if 'database/ap' + re.sub('_ex.fits', '', imgex) not in outputfile:
outputfile.append(
'database/ap' + re.sub('_ex.fits', '', imgex))
print '\n### all object with this setup extracted\n'
if liststandard:
standardlist = liststandard
_type = 'stdfromdreducer'
else:
try:
standardlist = objectlist['stdsens'][setup]
_type = 'stdsens'
except:
standardlist = ''
_type = ''
if _type == 'stdfromdreducer' and len(extracted) >= 1:
_outputsens2 = ntt.util.searchsens(extracted[0], standardlist)[0]
print '\n### using standard from reducer ' + str(_outputsens2)
elif _type not in ['stdsens', 'stdfromdreducer'] and len(extracted) >= 1:
_outputsens2 = ntt.util.searchsens(extracted[0], '')[0]
os.system('cp ' + _outputsens2 + ' .')
_outputsens2 = string.split(_outputsens2, '/')[-1]
print '\n### no standard in the list, using standard from archive'
else:
for simg in standardlist:
print '\n### standard for setup ' + \
str(setup) + ' = ', simg, ' ', ntt.util.readkey3(
ntt.util.readhdr(simg), 'object'), '\n'
simgex = ntt.util.extractspectrum(
simg, dv, False, False, _interactive, 'std', automaticex=_automaticex)
ntt.util.updateheader(
simgex, 0, {'FILETYPE': [22107, 'extracted 1D spectrum']})
ntt.util.updateheader(simgex, 0, {
'PRODCATG': [
'SCIENCE.' + ntt.util.readkey3(ntt.util.readhdr(simgex), 'tech').upper(), 'Data product category']})
ntt.util.updateheader(
simgex, 0, {'TRACE1': [simg, 'Originating file']})
if not ntt.util.readkey3(ntt.util.readhdr(simgex), 'shift') and \
ntt.util.readkey3(ntt.util.readhdr(simgex), 'shift') != 0.0:
# if not readkey3(readhdr(simgex),'shift'):
ntt.efoscspec1Ddef.checkwavestd(simgex, _interactive)
atmofile = ntt.efoscspec1Ddef.telluric_atmo(
simgex) # atmo file2
ntt.util.updateheader(
atmofile, 0, {'TRACE1': [simgex, 'Originating file']})
ntt.util.updateheader(
atmofile, 0, {'FILETYPE': [21211, 'telluric correction 1D spectrum ']})
if tpe != 'obj' and atmofile not in outputfile:
outputfile.append(atmofile)
if not listatmo0:
listatmo.append(atmofile)
sens[setup].append(simgex)
if simgex not in outputfile:
outputfile.append(simgex)
if setup[0] == 'Gr13' and setup[1] == 'Free':
if os.path.isfile(re.sub('Free', 'GG495', simg)):
print '\n### extract standard frame with blocking filter to correct for second order contamination\n'
simg2 = re.sub('Free', 'GG495', simg)
simgex2 = ntt.util.extractspectrum(simg2, dv, False, False, _interactive, 'std',
automaticex=_automaticex)
ntt.util.updateheader(
simgex2, 0, {'FILETYPE': [22107, 'extracted 1D spectrum']})
ntt.util.updateheader(simgex2, 0, {
'PRODCATG': ['SCIENCE.' +
ntt.util.readkey3(
ntt.util.readhdr(simgex2), 'tech').upper(), 'Data product category']})
if not ntt.util.readkey3(ntt.util.readhdr(simgex2), 'shift') and \
ntt.util.readkey3(ntt.util.readhdr(simgex2), 'shift') != 0.0:
# if not readkey3(readhdr(simgex2),'shift'):
ntt.efoscspec1Ddef.checkwavestd(
simgex2, _interactive)
ntt.util.updateheader(
simgex2, 0, {'TRACE1': [simg2, 'Originating file']})
print '\n### standard available: ', sens[setup]
if tpe == 'obj':
if len(sens[setup]) > 1:
goon = 'no'
while goon != 'yes':
stdused = raw_input(
'\n### more than one standard for this setup, which one do you want to use [' + sens[setup][
0] + '] ?')
if not stdused:
stdused = sens[setup][0]
if os.path.isfile(stdused):
goon = 'yes'
else:
stdused = sens[setup][0]
stdvec = [stdused]
else:
stdvec = sens[setup]
for stdused in stdvec:
stdusedclean = re.sub('_ex', '_clean', stdused)
ntt.util.delete(stdusedclean)
iraf.specred.sarith(
input1=stdused, op='/', input2=atmofile, output=stdusedclean, format='multispec')
_outputsens2 = ntt.efoscspec1Ddef.sensfunction(
stdusedclean, 'spline3', 16, _interactive)
ntt.util.updateheader(_outputsens2, 0, {'FILETYPE': [
21212, 'sensitivity function']})
ntt.util.updateheader(
_outputsens2, 0, {'TRACE1': [stdused, 'Originating file']})
if setup[0] == 'Gr13' and setup[1] == 'Free':
if os.path.isfile(re.sub('Free', 'GG495', stdused)):
print '\n### compute sensitivity function of grim 13 with blocking filter ' \
'to correct for second order contamination \n'
stdused2 = re.sub('Free', 'GG495', stdused)
if not ntt.util.readkey3(ntt.util.readhdr(stdused2), 'STDNAME'):
ntt.util.updateheader(stdused2, 0, {
'STDNAME': [ntt.util.readkey3(ntt.util.readhdr(stdused), 'STDNAME'), '']})
atmofile2 = ntt.efoscspec1Ddef.telluric_atmo(
stdused2) # atmo file2
stdusedclean2 = re.sub('_ex', '_clean', stdused2)
ntt.util.delete(stdusedclean2)
iraf.specred.sarith(input1=stdused2, op='/', input2=atmofile2, output=stdusedclean2,
format='multispec')
_outputsens3 = ntt.efoscspec1Ddef.sensfunction(
stdusedclean2, 'spline3', 16, _interactive)
ntt.util.updateheader(_outputsens3, 0, {'FILETYPE': [
21212, 'sensitivity function']})
ntt.util.updateheader(
_outputsens3, 0, {'TRACE1': [stdused2, 'Originating file']})
_outputsens2 = correctsens(_outputsens2, _outputsens3)
if _outputsens2 not in outputfile:
outputfile.append(_outputsens2)
if _outputsens2 and tpe == 'obj':
####################################################
for img in objectlist['obj'][setup]: # flux calibrate 2d images
imgd = fluxcalib2d(img, _outputsens2)
ntt.util.updateheader(
imgd, 0, {'FILETYPE': [22209, '2D wavelength and flux calibrated spectrum ']})
ntt.util.updateheader(
imgd, 0, {'TRACE1': [img, 'Originating files']})
iraf.hedit(imgd, 'PRODCATG', delete='yes',
update='yes', verify='no')
if imgd not in outputfile:
outputfile.append(imgd)
####################################################
# flux calib in the standard way
if not listatmo and listatmo0:
listatmo = listatmo0[:]
for _imgex in extracted:
_airmass = ntt.util.readkey3(
ntt.util.readhdr(_imgex), 'airmass')
_exptime = ntt.util.readkey3(
ntt.util.readhdr(_imgex), 'exptime')
_imgf = re.sub('_ex.fits', '_f.fits', _imgex)
ntt.util.delete(_imgf)
qqq = iraf.specred.calibrate(input=_imgex, output=_imgf, sensiti=_outputsens2, extinct='yes',
flux='yes',
extinction=_extinctdir + _extinction, observatory=_observatory,
airmass=_airmass, ignorea='yes', exptime=_exptime, fnu='no')
hedvec = {'SENSFUN': [_outputsens2, ''],
'FILETYPE': [22208, '1D wavelength and flux calibrated spectrum', ''],
# 'SNR':[ntt.util.StoN(_imgf,50),'Average signal to noise ratio per pixel'],
'SNR': [ntt.util.StoN2(_imgf, False), 'Average signal to noise ratio per pixel'],
'BUNIT': ['erg/cm2/s/Angstrom', 'Physical unit of array values'],
'TRACE1': [_imgex, 'Originating file'],
'ASSON1': [re.sub('_f.fits', '_2df.fits', _imgf), 'Name of associated file'],
'ASSOC1': ['ANCILLARY.2DSPECTRUM', 'Category of associated file']}
ntt.util.updateheader(_imgf, 0, hedvec)
if _imgf not in outputfile:
outputfile.append(_imgf)
if listatmo:
atmofile = ntt.util.searcharc(_imgex, listatmo)[0]
if atmofile:
_imge = re.sub('_f.fits', '_e.fits', _imgf)
ntt.util.delete(_imge)
iraf.specred.sarith(input1=_imgf, op='/', input2=atmofile, output=_imge, w1='INDEF', w2='INDEF',
format='multispec')
try:
iraf.imutil.imcopy(
input=_imgf + '[*,1,2]', output=_imge + '[*,1,2]', verbose='no')
except:
pass
try:
iraf.imutil.imcopy(
input=_imgf + '[*,1,3]', output=_imge + '[*,1,3]', verbose='no')
except:
pass
try:
iraf.imutil.imcopy(
input=_imgf + '[*,1,4]', output=_imge + '[*,1,4]', verbose='no')
except:
pass
if _imge not in outputfile:
outputfile.append(_imge)
ntt.util.updateheader(
_imge, 0, {'FILETYPE': [22210, '1D, wave, flux calib, telluric corr.']})
if atmofile not in outputfile:
outputfile.append(atmofile)
ntt.util.updateheader(
_imge, 0, {'ATMOFILE': [atmofile, '']})
ntt.util.updateheader(
_imge, 0, {'TRACE1': [_imgf, 'Originating file']})
imgin = _imge
else:
imgin = _imgf
else:
imgin = _imgf
imgasci = re.sub('.fits', '.asci', imgin)
ntt.util.delete(imgasci)
iraf.onedspec(_doprint=0)
iraf.onedspec.wspectext(
imgin + '[*,1,1]', imgasci, header='no')
if imgasci not in outputfile:
outputfile.append(imgasci)
print '\n### adding keywords for phase 3 ....... '
for img in outputfile:
if str(img)[-5:] == '.fits':
try:
ntt.util.phase3header(img) # phase 3 definitions
ntt.util.updateheader(img, 0, {'quality': ['Final', '']})
except:
print 'Warning: ' + img + ' is not a fits file'
try:
if int(re.sub('\.', '', str(pyfits.__version__))[:2]) <= 30:
aa = 'HIERARCH '
else:
aa = ''
except:
aa = ''
imm = pyfits.open(img, mode='update')
hdr = imm[0].header
if aa + 'ESO DPR CATG' in hdr:
hdr.pop(aa + 'ESO DPR CATG')
if aa + 'ESO DPR TECH' in hdr:
hdr.pop(aa + 'ESO DPR TECH')
if aa + 'ESO DPR TYPE' in hdr:
hdr.pop(aa + 'ESO DPR TYPE')
imm.flush()
imm.close()
print outputfile
reduceddata = ntt.rangedata(outputfile)
f = open('logfile_spec1d_' + str(reduceddata) +
'_' + str(datenow) + '.raw.list', 'w')
for img in outputfile:
try:
f.write(ntt.util.readkey3(ntt.util.readhdr(img), 'arcfile') + '\n')
except:
pass
f.close()
return outputfile, 'logfile_spec1d_' + str(reduceddata) + '_' + str(datenow) + '.raw.list'
def ecpsf(img, ofwhm, threshold, psfstars, distance, interactive, ds9, psffun='gauss', fixaperture=False, _catalog='', _datamax=None, show=False):
try:
import lsc, string
hdr = lsc.util.readhdr(img + '.fits')
instrument = lsc.util.readkey3(hdr, 'instrume')
print 'INSTRUMENT:', instrument
if 'PIXSCALE' in hdr:
scale = lsc.util.readkey3(hdr, 'PIXSCALE')
elif 'CCDSCALE' in hdr:
if 'CCDXBIN' in hdr:
scale = lsc.util.readkey3(hdr, 'CCDSCALE') * lsc.util.readkey3(hdr, 'CCDXBIN')
elif 'CCDSUM' in hdr:
scale = lsc.util.readkey3(hdr, 'CCDSCALE') * int(string.split(lsc.util.readkey3(hdr, 'CCDSUM'))[0])
if _datamax is None and 'kb' in instrument:
_datamax = 45000
elif _datamax is None:
_datamax = 65000
_wcserr = lsc.util.readkey3(hdr, 'wcserr')
print _wcserr
if float(_wcserr) == 0:
if 'L1FWHM' in hdr:
seeing = float(lsc.util.readkey3(hdr, 'L1FWHM'))
elif 'L1SEEING' in hdr:
seeing = float(lsc.util.readkey3(hdr, 'L1SEEING')) * scale
else:
seeing = 3
else:
seeing = 3
if 'PSF_FWHM' in hdr:
seeing = float(lsc.util.readkey3(hdr, 'PSF_FWHM'))
else:
sys.exit('astrometry not good')
#except: sys.exit('astrometry not good')
fwhm = seeing / scale
print 'FWHM[header] ', fwhm, ' in pixel'
if ofwhm: fwhm = float(ofwhm)
print ' FWHM[input] ', fwhm, ' in pixel'
xdim, ydim = iraf.hselect(img+'[0]', 'i_naxis1,i_naxis2', 'yes', Stdout=1)[0].split()
print img, fwhm, threshold, scale,xdim
#################################################################################
################### write file to compute psf _psf.coo ############
#################################################################################
if interactive:
iraf.display(img, 1, fill=True)
iraf.delete('tmp.lo?', verify=False)
print '_' * 80
print '>>> Mark reference stars with "a". Then "q"'
print '-' * 80
iraf.imexamine(img, 1, wcs='logical', logfile='tmp.log', keeplog=True)
xyrefer = iraf.fields('tmp.log', '1,2,6,15', Stdout=1)
xns, yns, _fws = [], [], []
############# write file for PSF #########################
ff = open('_psf.coo', 'w')
for i in range(len(xyrefer)):
xns.append(float(xyrefer[i].split()[0]))
yns.append(float(xyrefer[i].split()[1]))
_fws.append(float(xyrefer[i].split()[3]))
ff.write('%10.3f %10.3f %7.2f \n' % (xns[i], yns[i], float(_fws[i])))
ff.close()
fwhm = np.median(_fws)
elif _catalog:
print '\n#### use catalog to measure the psf'
ddd=iraf.wcsctran(input=_catalog,output='STDOUT',Stdout=1,image=img + '[0]',inwcs='world',outwcs='logical',
units='degrees degrees',columns='1 2',formats='%10.1f %10.1f',verbose='no')
ddd=[i for i in ddd if i[0]!='#']
ddd=[' '.join(i.split()[0:3]) for i in ddd]
line=''
for i in ddd:
a,b,c = string.split(i)
if float(a) < float(xdim) and float(b) < float(ydim) and float(b) > 0:
line = line + '%10s %10s %10s \n' % (a, b, c)
if line:
ff = open('_psf.coo', 'w')
ff.write(line)
ff.close()
else:
sys.exit('error: no catalog objects are in the field')
else:
############ run sextractor #####################################
xs, ys, ran, decn, magbest, classstar, fluxrad, bkg = runsex(img, fwhm, threshold, scale)
tot = np.compress(abs(np.array(fluxrad) * 1.6 - fwhm) / fwhm < .5, fluxrad)
if len(tot) < 5:
print 'warning: fwhm from sexractor different from fwhm computed during pre-reduction'
print 'try using option --fwhm xxx'
ff = open('tmp.cursor', 'w')
image_hdu = fits.open(img + '.fits')
for i in range(len(xs)):
_xs = np.delete(xs, i)
_ys = np.delete(ys, i)
dist2 = np.sqrt((_xs - xs[i]) ** 2 + (_ys - ys[i]) ** 2)
########### cut star, not near other object ##########################
if abs(fluxrad[i] * 1.6 - fwhm) / fwhm < .5 and min(dist2) > distance * fwhm:
x1, x2 = int(xs[i] - fwhm * 3), int(xs[i] + fwhm * 3)
y1, y2 = int(ys[i] - fwhm * 3), int(ys[i] + fwhm * 3)
if x1 < 1: x1 = 1
if y1 < 1: y1 = 1
if x2 > int(xdim):
x2 = int(xdim)
if y2 > int(ydim):
y2 = int(ydim)
fmax = np.max(image_hdu[0].data[y1-1:y2, x1-1:x2])
########## cut saturated object ########################
if float(fmax) < _datamax: # not saturated
ff.write('%10.3f %10.3f 1 m \n' % (xs[i], ys[i]))
ff.close()
image_hdu.close()
iraf.delete('tmp.lo?,tmp.sta?,tmp.gk?', verify=False)
iraf.psfmeasure(img+'[0]', imagecur='tmp.cursor', logfile='tmp.log', radius=int(fwhm), iter=3,
display=False, StdoutG='tmp.gki')
ff = open('tmp.log')
righe = ff.readlines()
xn = [float(righe[3].split()[1])]
yn = [float(righe[3].split()[2])]
_fw = [float(righe[3].split()[4])]
for r in righe[4:-2]:
if len(r) > 0:
xn.append(float(r.split()[0]))
yn.append(float(r.split()[1]))
_fw.append(float(r.split()[3]))
print 'FWHM: ', righe[-1].split()[-1]
print 80 * "#"
######
############## eliminate double object identification ###########################
xns, yns, _fws = [xn[0]], [yn[0]], [_fw[0]]
for i in range(1, len(xn)):
if abs(xn[i] - xn[i - 1]) > .2 and abs(yn[i] - yn[i - 1]) > .2:
xns.append(xn[i])
yns.append(yn[i])
_fws.append(_fw[i])
######### write clean file for PSF #########################
fw = []
ff = open('_psf.coo', 'w')
for i in range(len(xns)):
if abs(_fws[i] - fwhm) / fwhm < .3:
ff.write('%10.3f %10.3f %7.2f \n' % (xns[i], yns[i], float(_fws[i])))
fw.append(_fws[i])
ff.close() ## End automatic selection
######################################################################################
################### write file of object to store in fits table #############
######################################################################################
if interactive:
xs, ys, ran, decn, magbest, classstar, fluxrad, bkg = runsex(img, fwhm, threshold, scale)
ff = open('_psf2.coo', 'w')
for i in range(len(xs)):
ff.write('%10s %10s %10s \n' % (xs[i], ys[i], fluxrad[i]))
ff.close()
elif _catalog:
ddd=iraf.wcsctran(input=_catalog,output='STDOUT',Stdout=1,image=img + '[0]',inwcs='world',outwcs='logical',
units='degrees degrees',columns='1 2',formats='%10.1f %10.1f',verbose='no')
ddd=[i for i in ddd if i[0]!='#']
ddd=[' '.join(i.split()[0:3]) for i in ddd]
ff = open('_psf2.coo', 'w')
for i in ddd:
a,b,c = string.split(i)
ff.write('%10s %10s %10s \n' % (a, b, c))
ff.close()
else:
os.system('cp _psf.coo _psf2.coo')
# dflux = fluxrad - np.median(fluxrad)
# fstar = np.compress(dflux < np.std(fluxrad), fluxrad)
#################################################################################################################
print 80 * "#"
photmag, pst, fitmag = psffit(img, fwhm, psfstars, hdr, interactive, _datamax, psffun, fixaperture)
photmag2, fitmag2 = psffit2(img, fwhm, psfstars, hdr, _datamax, psffun, fixaperture)
radec = iraf.wcsctran(input='STDIN', output='STDOUT', Stdin=photmag,
Stdout=1, image=img + '[0]', inwcs='logical', outwcs='world', columns="1 2",
format='%13.3H %12.2h', min_sig=9, mode='h')[3:]
radec2 = iraf.wcsctran(input='STDIN', output='STDOUT', Stdin=photmag2,
Stdout=1, image=img + '[0]', inwcs='logical', outwcs='world', columns="1 2",
format='%13.3H %12.2h', min_sig=9, mode='h')[3:]
if ds9 == 0 and (interactive or show):
iraf.set(stdimage='imt1024')
iraf.display(img, 1, fill=True, Stdout=1)
iraf.tvmark(1, coords='STDIN', mark='circle', radii=15, label=True, Stdin=photmag, nxoffset=5, nyoffset=5, txsize=2)
iraf.tvmark(1, coords='STDIN', mark='circle', radii=35, label=False, Stdin=pst, color=208)
# iraf.tvmark(1, coords='STDIN', mark='cross', length=35, label=False, Stdin=fitmag2, color=204)
idpsf = []
for i in range(len(pst)):
idpsf.append(pst[i].split()[2])
dmag = []
for i in range(len(radec)):
ra, dec, idph, magp2, magp3, magp4, merrp2, merrp3, merrp4 = radec[i].split()
dmag.append(9.99)
for j in range(len(fitmag)):
raf, decf, idf, magf, magerrf = fitmag[j].split()
if idph == idf and idph in idpsf and \
magp3 != 'INDEF' and magf != 'INDEF':
dmag[i] = float(magp3) - float(magf)
break
_dmag = np.compress(np.array(dmag) < 9.99, np.array(dmag))
print '>>> Aperture correction (phot) %6.3f +/- %5.3f %3d ' % \
(np.mean(_dmag), np.std(_dmag), len(_dmag))
if len(_dmag) > 3:
_dmag = np.compress(np.abs(_dmag - np.median(_dmag)) < 2 * np.std(_dmag), _dmag)
print '>>> 2 sigma rejection) %6.3f +/- %5.3f %3d [default]' \
% (np.mean(_dmag), np.std(_dmag), len(_dmag))
print '>>> fwhm %s ' % (str(fwhm))
for i in range(len(dmag)):
if dmag[i] == 9.99:
dmag[i] = ''
else:
dmag[i] = '%6.3f' % (dmag[i])
exptime = lsc.util.readkey3(hdr, 'exptime')
object = lsc.util.readkey3(hdr, 'object').replace(' ', '')
filtro = lsc.util.readkey3(hdr, 'filter')
#######################################
rap, decp, magp2, magp3, magp4, smagf, merrp3, smagerrf = [], [], [], [], [], [], [], []
rap0, decp0 = [], []
for i in range(len(radec2)):
aa = radec2[i].split()
rap.append(aa[0])
decp.append(aa[1])
rap0.append(lsc.lscabsphotdef.deg2HMS(ra=aa[0]))
decp0.append(lsc.lscabsphotdef.deg2HMS(dec=aa[1]))
idp = aa[2]
magp2.append(aa[3])
magp3.append(aa[4])
magp4.append(aa[5])
merrp3.append(aa[7])
_smagf, _smagerrf = 9999, 9999
for j in range(len(fitmag2)):
raf, decf, idf, magf, magerrf = fitmag2[j].split()
if idf == idp:
_smagf = magf
_smagerrf = magerrf
break
smagf.append(_smagf)
smagerrf.append(_smagerrf)
tbhdu = fits.BinTableHDU.from_columns(fits.ColDefs([fits.Column(name='ra', format='20A', array=np.array(rap)),
fits.Column(name='dec', format='20A', array=np.array(decp)),
fits.Column(name='ra0', format='E', array=np.array(rap0)),
fits.Column(name='dec0', format='E', array=np.array(decp0)),
fits.Column(name='magp2', format='E',
array=np.array(np.where((np.array(magp2) != 'INDEF'),
np.array(magp2), 9999), float)),
fits.Column(name='magp3', format='E',
array=np.array(np.where((np.array(magp3) != 'INDEF'),
np.array(magp3), 9999), float)),
fits.Column(name='merrp3', format='E',
array=np.array(np.where((np.array(merrp3) != 'INDEF'),
np.array(merrp3), 9999), float)),
fits.Column(name='magp4', format='E',
array=np.array(np.where((np.array(magp4) != 'INDEF'),
np.array(magp4), 9999), float)),
fits.Column(name='smagf', format='E',
array=np.array(np.where((np.array(smagf) != 'INDEF'),
np.array(smagf), 9999), float)),
fits.Column(name='smagerrf', format='E',
array=np.array(np.where((np.array(smagerrf) != 'INDEF'),
np.array(smagerrf), 9999),
float)),
]))
hdu = fits.PrimaryHDU(header=hdr)
thdulist = fits.HDUList([hdu, tbhdu])
lsc.util.delete(img + '.sn2.fits')
thdulist.writeto(img + '.sn2.fits')
lsc.util.updateheader(img + '.sn2.fits', 0, {'APCO': [np.mean(_dmag), 'Aperture correction']})
lsc.util.updateheader(img + '.sn2.fits', 0, {'APCOERR': [np.std(_dmag), 'Aperture correction error']})
lsc.util.updateheader(img + '.sn2.fits', 0, {'XDIM': [lsc.util.readkey3(hdr, 'naxis1'), 'x number of pixels']})
lsc.util.updateheader(img + '.sn2.fits', 0, {'YDIM': [lsc.util.readkey3(hdr, 'naxis2'), 'y number of pixels']})
lsc.util.updateheader(img + '.sn2.fits', 0,
{'PSF_FWHM': [fwhm * scale, 'FWHM (arcsec) - computed with daophot']})
os.chmod(img + '.sn2.fits', 0664)
os.chmod(img + '.psf.fits', 0664)
result = 1
except:
result = 0
fwhm = 0.0
traceback.print_exc()
return result, fwhm * scale
ir.aptrace.step = trace_step
ir.aptrace.naverage = 1
ir.aptrace.nlost = 999
ir.aptrace.recenter = "yes"
# Set detector properties:
gain = 4.0 # photons (i.e., electrons) per data unit
readnoise = 10.0 # photons (i.e., electrons)
ir.imcombine.gain = gain
ir.imcombine.rdnoise = readnoise
ir.apall.gain = gain
ir.apall.readnoise = readnoise
ir.apnormalize.gain = gain
ir.apnormalize.readnoise = readnoise
ir.set(observatory=observ)
# Combine dark frames into a single dark frame:
if makeDark:
ir.imdelete(_sdark)
ir.imdelete(_sdarks)
print "rawdark file list>>" + rawdark
ir.imcombine("@" + rawdark,
output=_sdark,
combine="average",
reject="avsigclip",
sigmas=_sdarks,
scale="none",
weight="none",
bpmasks="")
from astropy import table from astropy.table import Table from functools import partial from pyexcel_ods import get_data #get the daophot tasks iraf.digiphot.daophot() print 'Iraf loaded' print # Allow overwriting files iraf.set(clobber="yes") #takes an uncollimated table and converts into recarray def make_recarray(tab, title_list): dtype_list = ['|S20' for item in title_list] name_dtype = [tuple(line) for line in zip(title_list, dtype_list)] data_array = [] for i in range(len(title_list)): col = [line[i] for line in tab] data_array.append(col)
def reduce_science(rawdir, rundir, flat, arc, twilight, twilight_flat, sciimg,
starimg, bias, overscan, vardq, observatory, lacos,
apply_lacos, lacos_xorder, lacos_yorder, lacos_sigclip,
lacos_objlim, bpm, instrument, slits, fl_gscrrej,
wltrim_frac, grow_gap, cube_bit_mask):
"""
Reduction pipeline for standard star.
Parameters
----------
rawdir: string
Directory containing raw images.
rundir: string
Directory where processed files are saved.
flat: string
Names of the files containing flat field images.
arc: string
Arc images.
twilight: string
Twilight flat images.
twilight_flat: string
Flat field for twilight image.
starimg: string
Name of the file containing the image to be reduced.
bias: string
Bias images.
grow_gap: number
Number of pixels by which to grow the bad pixel mask around
the chip gaps.
"""
iraf.set(stdimage='imtgmos')
iraf.gemini()
iraf.unlearn('gemini')
iraf.gmos()
iraf.unlearn('gmos')
iraf.gemtools()
iraf.unlearn('gemtools')
# os.path.isfile('arquivo')
iraf.unlearn('gemini')
iraf.unlearn('gmos')
iraf.task(lacos_spec=lacos)
# set directories
iraf.set(caldir=rawdir) #
iraf.set(rawdir=rawdir) # raw files
iraf.set(procdir=rundir) # processed files
iraf.gmos.logfile = 'logfile.log'
iraf.gfextract.verbose = 'no'
iraf.cd('procdir')
flat = flat.replace('.fits', '')
twilight = twilight.replace('.fits', '')
twilight_flat = twilight_flat.replace('.fits', '')
arc = arc.replace('.fits', '')
starimg = starimg.replace('.fits', '')
sciimg = sciimg.replace('.fits', '')
mdffile = 'mdf' + flat + '.fits'
iraf.gfreduce.bias = 'caldir$' + bias
iraf.gfreduce.fl_fulldq = 'yes'
iraf.gfreduce.fl_fixgaps = 'yes'
iraf.gfreduce.grow = grow_gap
iraf.gireduce.bpm = 'rawdir$' + bpm
iraf.gfextract.verbose = 'no'
cal_reduction(rawdir=rawdir,
rundir=rundir,
flat=flat,
arc=arc,
twilight=twilight,
bias=bias,
bpm=bpm,
overscan=overscan,
vardq=vardq,
instrument=instrument,
slits=slits,
twilight_flat=twilight_flat,
grow_gap=grow_gap)
#
# Actually reduce science
#
image_name = 'rg' + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gfreduce(sciimg,
slits='header',
rawpath='rawdir$',
fl_inter='no',
fl_addmdf='yes',
key_mdf='MDF',
mdffile=mdffile,
weights='no',
fl_over=overscan,
fl_trim='yes',
fl_bias='yes',
trace='no',
recenter='no',
fl_fulldq='yes',
fl_flux='no',
fl_gscrrej='no',
fl_extract='no',
fl_gsappwave='no',
fl_wavtran='no',
fl_novl='yes',
fl_skysub='no',
fl_vardq=vardq,
mdfdir='procdir$')
prefix = 'rg'
# Gemfix
image_name = 'p' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gemfix(prefix + sciimg,
out='p' + prefix + sciimg,
method='fit1d',
bitmask=65535,
axis=1)
prefix = 'p' + prefix
# LA Cosmic - cosmic ray removal
if apply_lacos:
image_name = 'l' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gemcrspec(prefix + sciimg,
out='l' + prefix + sciimg,
sigfrac=0.5,
niter=4,
fl_vardq=vardq,
xorder=lacos_xorder,
yorder=lacos_yorder,
sigclip=lacos_sigclip,
objlim=lacos_objlim)
prefix = 'l' + prefix
if fl_gscrrej:
image_name = 'ex' + prefix + sciimg + '.fits'
else:
image_name = 'e' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gfreduce(prefix + sciimg,
slits='header',
rawpath='./',
fl_inter='no',
fl_addmdf='no',
key_mdf='MDF',
mdffile=mdffile,
fl_over='no',
fl_trim='no',
fl_bias='no',
trace='no',
recenter='no',
fl_flux='no',
fl_gscrrej=fl_gscrrej,
fl_extract='yes',
fl_gsappwave='yes',
fl_wavtran='no',
fl_novl='no',
fl_skysub='no',
grow=grow_gap,
reference='eprg' + flat,
weights='no',
wavtraname='eprg' + arc,
response='eprg' + flat + '_response.fits',
fl_vardq=vardq,
fl_fulldq='yes',
fl_fixgaps='yes')
if fl_gscrrej:
prefix = 'ex' + prefix
else:
prefix = 'e' + prefix
# if wl2 > 7550.0:
# wl2 = 7550.0
#
# Apply wavelength transformation
#
wl1, wl2 = wl_lims(prefix + sciimg + '.fits', wltrim_frac)
image_name = 't' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gftransform(
prefix + sciimg,
wavtraname='eprg' + arc,
fl_vardq=vardq,
w1=wl1,
w2=wl2,
)
prefix = 't' + prefix
#
# Sky subtraction
#
image_name = 's' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gfskysub(
prefix + sciimg,
expr='default',
combine='median',
reject='avsigclip',
scale='none',
zero='none',
weight='none',
sepslits='yes',
fl_inter='no',
lsigma=1,
hsigma=1,
)
prefix = 's' + prefix
#
# Apply flux calibration to galaxy
#
image_name = 'c' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gscalibrate(prefix + sciimg,
sfuncti=starimg,
extinct='onedstds$ctioextinct.dat',
observatory=observatory,
fluxsca=1,
fl_vardq=vardq)
prefix = 'c' + prefix
#
# Remove spurious data with PCA
#
image_name = 'x' + prefix + sciimg + '.fits'
print(os.getcwd())
print(image_name)
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
t = pca.Tomography(prefix + sciimg + '.fits')
t.decompose()
t.remove_cosmic_rays(sigma_threshold=10.0)
t.write(image_name)
prefix = 'x' + prefix
#
# Create data cubes
#
image_name = 'd' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
data_cube = CubeBuilder(prefix + sciimg + '.fits')
data_cube.build_cube()
data_cube.fit_refraction_function()
data_cube.fix_atmospheric_refraction()
data_cube.write(image_name)
def start():
"""
nifsBaselineCalibration
This module contains all the functions needed to reduce
NIFS GENERAL BASELINE CALIBRATIONS
INPUT FILES FOR EACH BASELINE CALIBRATION:
Raw files:
- Flat frames (lamps on)
- Flat frames (lamps off)
- Arc frames
- Arc dark frames
- Ronchi mask flat frames
OUTPUT FILES:
- Shift file. Eg: sCALFLAT.fits
- Bad Pixel Mask. Eg: rgnCALFLAT_sflat_bmp.pl
- Flat field. Eg: rgnCALFLAT_flat.fits
- Reduced arc frame. Eg: wrgnARC.fits
- Reduced ronchi mask. Eg: rgnRONCHI.fits
- Reduced dark frame. Eg: rgnARCDARK.fits
Args:
# Loaded from runtimeData/config.cfg
calDirList: list of paths to calibrations. ['path/obj/date/Calibrations_grating']
over (boolean): overwrite old files. Default: False.
start (int): starting step of daycal reduction. Specified at command line with -a. Default: 1.
stop (int): stopping step of daycal reduction. Specified at command line with -z. Default: 6.
debug (boolean): enable optional debugging pauses. Default: False.
"""
# TODO(nat): stop using first frame from list as name for combined frames. Find better names and implement
# them in pipeline and docs.
# TODO(nat): Finish converting the print statements to logging.info() statements.
# Store current working directory for later use.
path = os.getcwd()
# Set up the logging file.
log = os.getcwd()+'/Nifty.log'
logging.info('#################################################')
logging.info('# #')
logging.info('# Start the NIFS Baseline Calibration Reduction #')
logging.info('# #')
logging.info('#################################################')
# Set up/prepare IRAF.
iraf.gemini()
iraf.nifs()
iraf.gnirs()
iraf.gemtools()
# Reset to default parameters the used IRAF tasks.
iraf.unlearn(iraf.gemini,iraf.gemtools,iraf.gnirs,iraf.nifs)
# From http://bishop.astro.pomona.edu/Penprase/webdocuments/iraf/beg/beg-image.html:
# Before doing anything involving image display the environment variable
# stdimage must be set to the correct frame buffer size for the display
# servers (as described in the dev$graphcap file under the section "STDIMAGE
# devices") or to the correct image display device. The task GDEVICES is
# helpful for determining this information for the display servers.
iraf.set(stdimage='imt2048')
# Prepare the IRAF package for NIFS.
# NSHEADERS lists the header parameters used by the various tasks in the
# NIFS package (excluding headers values which have values fixed by IRAF or
# FITS conventions).
iraf.nsheaders("nifs",logfile=log)
# Set clobber to 'yes' for the script. This still does not make the gemini
# tasks overwrite files, so:
# YOU WILL LIKELY HAVE TO REMOVE FILES IF YOU RE_RUN THE SCRIPT.
user_clobber=iraf.envget("clobber")
iraf.reset(clobber='yes')
# Load reduction parameters from ./config.cfg.
with open('./config.cfg') as config_file:
options = ConfigObj(config_file, unrepr=True)
calDirList = options['calibrationDirectoryList']
over = options['over']
start = options['rstart']
stop = options['rstop']
debug = options['debug']
################################################################################
# Define Variables, Reduction Lists AND identify/run number of reduction steps #
################################################################################
# Loop over the Calibrations directories and reduce the day calibrations in each one.
for calpath in calDirList:
os.chdir(calpath)
pwdDir = os.getcwd()+"/"
iraffunctions.chdir(pwdDir)
# However, don't do the reduction for a Calibration_"grating" directory without associated telluric or science data.
# Check that a "grating" directory exists at the same level as the Calibrations_"grating" directory.
# If not, skip the reduction of calibrations in that Calibrations_grating directory.
# "grating" should be the last letter of calpath.
grating = calpath[-1]
if not os.path.exists("../"+grating):
print "\n##############################################################################"
print ""
print " No grating directory (including science or telluric data) found for "
print " ", calpath
print " Skipping reduction of calibrations in that directory."
print ""
print "##############################################################################\n"
continue
# Create lists of each type of calibration from textfiles in Calibrations directory.
flatlist = open('flatlist', "r").readlines()
flatdarklist = open("flatdarklist", "r").readlines()
arcdarklist = open("arcdarklist", "r").readlines()
arclist = open("arclist", "r").readlines()
ronchilist = open("ronchilist", "r").readlines()
# Store the name of the first image of each calibration-type-list in
# a variable for later use (Eg: calflat). This is because gemcombine will
# merge a list of files (Eg: "n"+flatlist) and the output file will have the same
# name as the first file in the list (Eg: calflat). These first file names are used
# later in the pipeline.
calflat = (flatlist[0].strip()).rstrip('.fits')
flatdark = (flatdarklist[0].strip()).rstrip('.fits')
arcdark = (arcdarklist[0].strip()).rstrip('.fits')
arc = (arclist[0].strip()).rstrip('.fits')
ronchiflat = (ronchilist[0].strip()).rstrip('.fits')
# Check start and stop values for reduction steps. Ask user for a correction if
# input is not valid.
valindex = start
while valindex > stop or valindex < 1 or stop > 4:
print "\n#####################################################################"
print "#####################################################################"
print ""
print " WARNING in calibrate: invalid start/stop values of calibration "
print " reduction steps."
print ""
print "#####################################################################"
print "#####################################################################\n"
valindex = int(raw_input("\nPlease enter a valid start value (1 to 4, default 1): "))
stop = int(raw_input("\nPlease enter a valid stop value (1 to 4, default 4): "))
# Print the current directory of calibrations being processed.
print "\n#################################################################################"
print " "
print " Currently working on calibrations "
print " in ", calpath
print " "
print "#################################################################################\n"
while valindex <= stop:
#############################################################################
## STEP 1: Determine the shift to the MDF (mask definition file) ##
## using nfprepare (nsoffset). Ie: locate the spectra. ##
## Output: First image in flatlist with "s" prefix. ##
#############################################################################
if valindex == 1:
if debug:
a = raw_input("About to enter step 1: locate the spectrum.")
getShift(calflat, over, log)
print "\n###################################################################"
print ""
print " STEP 1: Locate the Spectrum (Determine the shift to the MDF) - COMPLETED"
print ""
print "###################################################################\n"
#############################################################################
## STEP 2: Create Flat Field frame and BPM (Bad Pixel Mask) ##
## Output: Flat Field image with spatial and spectral information. ##
## First image in flatlist with "rgn" prefix and "_flat" suffix. ##
#############################################################################
elif valindex == 2:
if debug:
a = raw_input("About to enter step 2: flat field.")
makeFlat(flatlist, flatdarklist, calflat, flatdark, over, log)
print "\n###################################################################"
print ""
print " STEP 2: Flat Field (Create Flat Field image and BPM image) - COMPLETED "
print ""
print "###################################################################\n"
############################################################################
## STEP 3: NFPREPARE and Combine arc darks. ##
## NFPREPARE, Combine and flat field arcs. ##
## Determine the wavelength solution and create the wavelength ##
## referenced arc. ##
############################################################################
elif valindex == 3:
if debug:
a = raw_input("About to enter step 3: wavelength solution.")
reduceArc(arclist, arc, arcdarklist, arcdark, log, over)
wavecal(arc, log, over, path)
print "\n###################################################################"
print ""
print " STEP 3: Wavelength Solution (NFPREPARE and Combine arc darks. "
print " NFPREPARE, Combine and flat field arcs."
print " Determine the wavelength solution and create the"
print " wavelength referenced arc) - COMPLETED"
print ""
print "###################################################################\n"
######################################################################################
## Step 4: Trace the spatial curvature and spectral distortion in the Ronchi flat. ##
######################################################################################
elif valindex == 4:
if debug:
a = raw_input("About to enter step 4: spatial distortion.")
ronchi(ronchilist, ronchiflat, calflat, over, flatdark, log)
print "\n###################################################################"
print ""
print " Step 4: Spatial Distortion (Trace the spatial curvature and spectral distortion "
print " in the Ronchi flat) - COMPLETED"
print ""
print "###################################################################\n"
else:
print "\nERROR in nifs_baseline_calibration: step ", valindex, " is not valid.\n"
raise SystemExit
valindex += 1
print "\n##############################################################################"
print ""
print " COMPLETE - Calibration reductions completed for "
print " ", calpath
print ""
print "##############################################################################\n"
# Return to directory script was begun from.
os.chdir(path)
return
def ecpsf(img, ofwhm, threshold, psfstars, distance, interactive, psffun='gauss', fixaperture=False, _catalog='', _datamax=None, show=False):
try:
import lsc, string
hdr = lsc.util.readhdr(img + '.fits')
instrument = lsc.util.readkey3(hdr, 'instrume')
print 'INSTRUMENT:', instrument
if 'PIXSCALE' in hdr:
scale = lsc.util.readkey3(hdr, 'PIXSCALE')
elif 'CCDSCALE' in hdr:
if 'CCDXBIN' in hdr:
scale = lsc.util.readkey3(hdr, 'CCDSCALE') * lsc.util.readkey3(hdr, 'CCDXBIN')
elif 'CCDSUM' in hdr:
scale = lsc.util.readkey3(hdr, 'CCDSCALE') * int(string.split(lsc.util.readkey3(hdr, 'CCDSUM'))[0])
if _datamax is None:
_datamax = lsc.util.readkey3(hdr, 'datamax')
_wcserr = lsc.util.readkey3(hdr, 'wcserr')
print _wcserr
if float(_wcserr) == 0:
if 'L1FWHM' in hdr:
seeing = float(lsc.util.readkey3(hdr, 'L1FWHM'))
elif 'L1SEEING' in hdr:
seeing = float(lsc.util.readkey3(hdr, 'L1SEEING')) * scale
else:
seeing = 3
else:
seeing = 3
if 'PSF_FWHM' in hdr:
seeing = float(lsc.util.readkey3(hdr, 'PSF_FWHM'))
else:
sys.exit('astrometry not good')
#except: sys.exit('astrometry not good')
fwhm = seeing / scale
print 'FWHM[header] ', fwhm, ' in pixel'
if ofwhm: fwhm = float(ofwhm)
print ' FWHM[input] ', fwhm, ' in pixel'
xdim, ydim = iraf.hselect(img+'[0]', 'i_naxis1,i_naxis2', 'yes', Stdout=1)[0].split()
print img, fwhm, threshold, scale,xdim
#################################################################################
################### write file to compute psf _psf.coo ############
#################################################################################
if _catalog:
print '\n#### use catalog to measure the psf'
ddd=iraf.wcsctran(input=_catalog,output='STDOUT',Stdout=1,image=img + '[0]',inwcs='world',outwcs='logical',
units='degrees degrees',columns='1 2',formats='%10.1f %10.1f',verbose='no')
ddd=[i for i in ddd if i[0]!='#']
ddd=[' '.join(i.split()[0:3]) for i in ddd]
line=''
for i in ddd:
a,b,c = string.split(i)
if float(a) < float(xdim) and float(b) < float(ydim) and float(b) > 0:
line = line + '%10s %10s %10s \n' % (a, b, c)
if line:
ff = open('_psf.coo', 'w')
ff.write(line)
ff.close()
else:
sys.exit('error: no catalog objects are in the field')
elif interactive:
iraf.display(img + '[0]', 1, fill=True)
iraf.delete('tmp.lo?', verify=False)
print '_' * 80
print '>>> Mark reference stars with "a". Then "q"'
print '-' * 80
iraf.imexamine(img, 1, wcs='logical', logfile='tmp.log', keeplog=True)
xyrefer = iraf.fields('tmp.log', '1,2,6,15', Stdout=1)
xns, yns, _fws = [], [], []
############# write file for PSF #########################
ff = open('_psf.coo', 'w')
for i in range(len(xyrefer)):
xns.append(float(xyrefer[i].split()[0]))
yns.append(float(xyrefer[i].split()[1]))
_fws.append(float(xyrefer[i].split()[3]))
ff.write('%10.3f %10.3f %7.2f \n' % (xns[i], yns[i], float(_fws[i])))
ff.close()
fwhm = np.median(_fws)
else:
############ run sextractor #####################################
xs, ys, ran, decn, magbest, classstar, fluxrad, bkg = runsex(img, fwhm, threshold, scale)
tot = np.compress(abs(np.array(fluxrad) * 1.6 - fwhm) / fwhm < .5, fluxrad)
if len(tot) < 5:
print 'warning: fwhm from sexractor different from fwhm computed during pre-reduction'
print 'try using option --fwhm xxx'
ff = open('tmp.cursor', 'w')
image_hdu = fits.open(img + '.fits')
for i in range(len(xs)):
_xs = np.delete(xs, i)
_ys = np.delete(ys, i)
dist2 = np.sqrt((_xs - xs[i]) ** 2 + (_ys - ys[i]) ** 2)
########### cut star, not near other object ##########################
if abs(fluxrad[i] * 1.6 - fwhm) / fwhm < .5 and min(dist2) > distance * fwhm:
x1, x2 = int(xs[i] - fwhm * 3), int(xs[i] + fwhm * 3)
y1, y2 = int(ys[i] - fwhm * 3), int(ys[i] + fwhm * 3)
if x1 < 1: x1 = 1
if y1 < 1: y1 = 1
if x2 > int(xdim):
x2 = int(xdim)
if y2 > int(ydim):
y2 = int(ydim)
fmax = np.max(image_hdu[0].data[y1-1:y2, x1-1:x2])
########## cut saturated object ########################
if float(fmax) < _datamax: # not saturated
ff.write('%10.3f %10.3f 1 m \n' % (xs[i], ys[i]))
ff.close()
image_hdu.close()
iraf.delete('tmp.lo?,tmp.sta?,tmp.gk?', verify=False)
iraf.psfmeasure(img+'[0]', imagecur='tmp.cursor', logfile='tmp.log', radius=int(fwhm), iter=3,
display=False, StdoutG='tmp.gki')
ff = open('tmp.log')
righe = ff.readlines()
xn = [float(righe[3].split()[1])]
yn = [float(righe[3].split()[2])]
_fw = [float(righe[3].split()[4])]
for r in righe[4:-2]:
if len(r) > 0:
xn.append(float(r.split()[0]))
yn.append(float(r.split()[1]))
_fw.append(float(r.split()[3]))
print 'FWHM: ', righe[-1].split()[-1]
print 80 * "#"
######
############## eliminate double object identification ###########################
xns, yns, _fws = [xn[0]], [yn[0]], [_fw[0]]
for i in range(1, len(xn)):
if abs(xn[i] - xn[i - 1]) > .2 and abs(yn[i] - yn[i - 1]) > .2:
xns.append(xn[i])
yns.append(yn[i])
_fws.append(_fw[i])
######### write clean file for PSF #########################
fw = []
ff = open('_psf.coo', 'w')
for i in range(len(xns)):
if abs(_fws[i] - fwhm) / fwhm < .3:
ff.write('%10.3f %10.3f %7.2f \n' % (xns[i], yns[i], float(_fws[i])))
fw.append(_fws[i])
ff.close() ## End automatic selection
######################################################################################
################### write file of object to store in fits table #############
######################################################################################
if _catalog:
ddd=iraf.wcsctran(input=_catalog,output='STDOUT',Stdout=1,image=img + '[0]',inwcs='world',outwcs='logical',
units='degrees degrees',columns='1 2',formats='%10.1f %10.1f',verbose='no')
ddd=[i for i in ddd if i[0]!='#']
ddd=[' '.join(i.split()[0:3]) for i in ddd]
ff = open('_psf2.coo', 'w')
for i in ddd:
a,b,c = string.split(i)
ff.write('%10s %10s %10s \n' % (a, b, c))
ff.close()
elif interactive:
xs, ys, ran, decn, magbest, classstar, fluxrad, bkg = runsex(img, fwhm, threshold, scale)
ff = open('_psf2.coo', 'w')
for i in range(len(xs)):
ff.write('%10s %10s %10s \n' % (xs[i], ys[i], fluxrad[i]))
ff.close()
else:
os.system('cp _psf.coo _psf2.coo')
# dflux = fluxrad - np.median(fluxrad)
# fstar = np.compress(dflux < np.std(fluxrad), fluxrad)
#################################################################################################################
print 80 * "#"
photmag, pst, fitmag = psffit(img, fwhm, psfstars, hdr, interactive, _datamax, psffun, fixaperture)
photmag2, fitmag2 = psffit2(img, fwhm, psfstars, hdr, _datamax, psffun, fixaperture)
radec = iraf.wcsctran(input='STDIN', output='STDOUT', Stdin=photmag,
Stdout=1, image=img + '[0]', inwcs='logical', outwcs='world', columns="1 2",
format='%13.3H %12.2h', min_sig=9, mode='h')[3:]
radec2 = iraf.wcsctran(input='STDIN', output='STDOUT', Stdin=photmag2,
Stdout=1, image=img + '[0]', inwcs='logical', outwcs='world', columns="1 2",
format='%13.3H %12.2h', min_sig=9, mode='h')[3:]
if interactive or show:
iraf.set(stdimage='imt1024')
iraf.display(img + '[0]', 1, fill=True, Stdout=1)
iraf.tvmark(1, coords='STDIN', mark='circle', radii=15, label=True, Stdin=photmag, nxoffset=5, nyoffset=5, txsize=2)
iraf.tvmark(1, coords='STDIN', mark='circle', radii=35, label=False, Stdin=pst, color=208)
# iraf.tvmark(1, coords='STDIN', mark='cross', length=35, label=False, Stdin=fitmag2, color=204)
idpsf = []
for i in range(len(pst)):
idpsf.append(pst[i].split()[2])
dmag = []
for i in range(len(radec)):
ra, dec, idph, magp2, magp3, magp4, merrp2, merrp3, merrp4 = radec[i].split()
dmag.append(9.99)
for j in range(len(fitmag)):
raf, decf, idf, magf, magerrf = fitmag[j].split()
if idph == idf and idph in idpsf and \
magp3 != 'INDEF' and magf != 'INDEF':
dmag[i] = float(magp3) - float(magf)
break
_dmag = np.compress(np.array(dmag) < 9.99, np.array(dmag))
print '>>> Aperture correction (phot) %6.3f +/- %5.3f %3d ' % \
(np.mean(_dmag), np.std(_dmag), len(_dmag))
if len(_dmag) > 3:
_dmag = np.compress(np.abs(_dmag - np.median(_dmag)) < 2 * np.std(_dmag), _dmag)
print '>>> 2 sigma rejection) %6.3f +/- %5.3f %3d [default]' \
% (np.mean(_dmag), np.std(_dmag), len(_dmag))
print '>>> fwhm %s ' % (str(fwhm))
for i in range(len(dmag)):
if dmag[i] == 9.99:
dmag[i] = ''
else:
dmag[i] = '%6.3f' % (dmag[i])
exptime = lsc.util.readkey3(hdr, 'exptime')
object = lsc.util.readkey3(hdr, 'object').replace(' ', '')
filtro = lsc.util.readkey3(hdr, 'filter')
#######################################
rap, decp, magp2, magp3, magp4, smagf, merrp3, smagerrf = [], [], [], [], [], [], [], []
rap0, decp0 = [], []
for i in range(len(radec2)):
aa = radec2[i].split()
rap.append(aa[0])
decp.append(aa[1])
rap0.append(lsc.lscabsphotdef.deg2HMS(ra=aa[0]))
decp0.append(lsc.lscabsphotdef.deg2HMS(dec=aa[1]))
idp = aa[2]
magp2.append(aa[3])
magp3.append(aa[4])
magp4.append(aa[5])
merrp3.append(aa[7])
_smagf, _smagerrf = 9999, 9999
for j in range(len(fitmag2)):
raf, decf, idf, magf, magerrf = fitmag2[j].split()
if idf == idp:
_smagf = magf
_smagerrf = magerrf
break
smagf.append(_smagf)
smagerrf.append(_smagerrf)
tbhdu = fits.BinTableHDU.from_columns(fits.ColDefs([fits.Column(name='ra', format='20A', array=np.array(rap)),
fits.Column(name='dec', format='20A', array=np.array(decp)),
fits.Column(name='ra0', format='E', array=np.array(rap0)),
fits.Column(name='dec0', format='E', array=np.array(decp0)),
fits.Column(name='magp2', format='E',
array=np.array(np.where((np.array(magp2) != 'INDEF'),
np.array(magp2), 9999), float)),
fits.Column(name='magp3', format='E',
array=np.array(np.where((np.array(magp3) != 'INDEF'),
np.array(magp3), 9999), float)),
fits.Column(name='merrp3', format='E',
array=np.array(np.where((np.array(merrp3) != 'INDEF'),
np.array(merrp3), 9999), float)),
fits.Column(name='magp4', format='E',
array=np.array(np.where((np.array(magp4) != 'INDEF'),
np.array(magp4), 9999), float)),
fits.Column(name='smagf', format='E',
array=np.array(np.where((np.array(smagf) != 'INDEF'),
np.array(smagf), 9999), float)),
fits.Column(name='smagerrf', format='E',
array=np.array(np.where((np.array(smagerrf) != 'INDEF'),
np.array(smagerrf), 9999),
float)),
]))
hdu = fits.PrimaryHDU(header=hdr)
thdulist = fits.HDUList([hdu, tbhdu])
lsc.util.delete(img + '.sn2.fits')
thdulist.writeto(img + '.sn2.fits')
lsc.util.updateheader(img + '.sn2.fits', 0, {'APCO': [np.mean(_dmag), 'Aperture correction']})
lsc.util.updateheader(img + '.sn2.fits', 0, {'APCOERR': [np.std(_dmag), 'Aperture correction error']})
lsc.util.updateheader(img + '.sn2.fits', 0, {'XDIM': [lsc.util.readkey3(hdr, 'naxis1'), 'x number of pixels']})
lsc.util.updateheader(img + '.sn2.fits', 0, {'YDIM': [lsc.util.readkey3(hdr, 'naxis2'), 'y number of pixels']})
lsc.util.updateheader(img + '.sn2.fits', 0,
{'PSF_FWHM': [fwhm * scale, 'FWHM (arcsec) - computed with daophot']})
os.chmod(img + '.sn2.fits', 0664)
os.chmod(img + '.psf.fits', 0664)
result = 1
except:
result = 0
fwhm = 0.0
traceback.print_exc()
return result, fwhm * scale
def doit(fn,dp,mt):
""" JHT: this is the module that analyses an ALFOSC image of
a Vertical Slit, and recommends an alignment offset to the
current APERTURE WHEEL stepper motor units."""
## First check if data file exists
if not os.access(dp+fn+".fits",os.F_OK):
messageOut(mt,"File not found: "+dp+fn+".fits\n")
return "File not found: "+dp+fn+".fits"
messageOut(mt,"\nVertical-Slit analysis of file: "+dp+fn+".fits\n")
from pyraf import iraf
from pyraf import gwm
## Read current grism wheel position from image FITS header
iraf.images.imutil.imgets(dp+fn,"GRISM")
grismid=iraf.images.imutil.imgets.value
if grismid.find("pen")==-1:
messageOut(mt,"Vertical-Slit mode: grism wheel should be Open\n")
return "File %s: Vertical-Slit mode: grism wheel should be Open" % fn
## Read current aperture wheel position from image FITS header
iraf.images.imutil.imgets(dp+fn,"APERTUR")
slitid=iraf.images.imutil.imgets.value
if slitid.find("Vert")==-1:
messageOut(mt,"Vertical-Slit mode: no vertical slit in aperture wheel\n")
return "File %s: Vertical-Slit mode: no vertical slit in aperture wheel" % fn
slitwidth=float(slitid.split('_',9)[1].strip('\"'))
messageOut(mt,"Slit width "+str(slitwidth)+" "+slitid+"\n")
iraf.noao(_doprint=0)
iraf.noao.imred(_doprint=0)
iraf.noao.imred.specred(_doprint=0)
if not os.access("/tmp/tiasgat/",os.F_OK):
os.mkdir("/tmp/tiasgat/")
os.chmod("/tmp/tiasgat/",0777)
if os.access("/tmp/tiasgat/plot",os.F_OK): os.remove("/tmp/tiasgat/plot")
if os.access("/tmp/tiasgat/plot2",os.F_OK): os.remove("/tmp/tiasgat/plot2")
if os.access("/tmp/tiasgat/aplast",os.F_OK): os.remove("/tmp/tiasgat/aplast")
if os.access("/tmp/tiasgat/tvmarks",os.F_OK): os.remove("/tmp/tiasgat/tvmarks")
if os.access("/tmp/tiasgat/logfile",os.F_OK): os.remove("/tmp/tiasgat/logfile")
## Note that this will *not* update any uparm files !! (see pyraf documentation)
iraf.noao.imred.specred.dispaxis=2
iraf.noao.imred.specred.database="/tmp/tiasgat/"
iraf.noao.imred.specred.plotfile="/tmp/tiasgat/plot"
iraf.noao.imred.specred.logfile="/tmp/tiasgat/logfile"
width=6*slitwidth # x6 or else the widest slits go wrong
if width<4: width=4 # smaller than 4 and apfind will not find it
messageOut(mt,"Using width of "+str(width)+" pixels \n")
iraf.noao.imred.specred.apedit.width=width
## Display image on ds9
iraf.set(stdimage="imt512")
iraf.display(dp+fn,1,fill="no",Stdout="/dev/null")
# Suppress IRAF query for number of apertures to find
# This is only necesary for the widest slits: then the call to
# apfind results in an empty database file, as it cannot find an aperture.
# But aptrace works fine anyway (except for the annoying query) !?
iraf.noao.imred.specred.apfind.setParam('nfind.p_value', 1)
iraf.noao.imred.specred.apfind.setParam('nfind.p_mode','h')
## 'find' and trace spectrum; this will dump a plot to /tmp/tiasgat/plot
lines = iraf.noao.imred.specred.apfind(dp+fn,nfind=1,interactive="no", edit="no", nsum=20, Stdout=1)
for i in range (0,len(lines)): messageOut(mt,lines[i]+"\n")
lines = iraf.noao.imred.specred.aptrace(dp+fn,interactive="no",step=5,low_reject=2.5,
high_reject=2.5,function="leg",order=2,niterate=5,naverage=1, Stdout=1)
for i in range (0,len(lines)): messageOut(mt,lines[i]+"\n")
## Start graphics window; select the correct plot; show plot
gwm.window("Tiasgat! graphics")
iraf.plot.gkiextract("/tmp/tiasgat/plot",2,Stdout="/tmp/tiasgat/plot2")
gwm.getActiveGraphicsWindow().load("/tmp/tiasgat/plot2")
### how to read the aperture file, as output by aptrace ####
###
### center line 6 gives zero point
### max,min lines 24-25 n = (2 * x - (max + min)) / (max - min)
### c1,c2 lines 26-27
###
### The polynomial can be expressed as the sum
###
### poly = sum from i=1 to order {c_i * z_i}
###
### where the the c_i are the coefficients and the z_i are defined
### interatively as:
###
### z_1 = 1
### z_2 = n
### z_i = ((2*i-3) * n * z_{i-1} - (i-2) * z_{i-2}) / (i - 1)
###
### So for order=2 and for vertical slit/grism: X=center+c1+c2*n
### X=center + c1 + c2*(2 * Y - (max + min)) / (max - min)
### translated to X=a + bY
### a=center + c1 - c2*(max+min)/(max-min)
### b=2*C2/(max-min)
## Read the aperture definition file
apfile=open("/tmp/tiasgat/ap"+dp.replace('/','_')+fn,'r')
lines=apfile.readlines()
apfile.close()
#print lines[5], lines[23:]
c0 = float(lines[5].split(None,9)[1].strip())
lower = float(lines[23].strip())
upper = float(lines[24].strip())
c1 = float(lines[25].strip())
c2 = float(lines[26].strip())
a = c0 + c1 - c2*(upper+lower)/(upper-lower)
b = 2*c2/(upper-lower)
#print "zeropoint ", a, " slope ",b
## Remove aperture definition file
if os.access("/tmp/tiasgat/ap"+dp.replace('/','_')+fn,os.F_OK):
os.remove("/tmp/tiasgat/ap"+dp.replace('/','_')+fn)
## Mark the fit on the image display
if os.access("/tmp/tiasgat/tvmarks",os.F_OK): os.remove("/tmp/tiasgat/tvmarks")
tvmarkfile=open("/tmp/tiasgat/tvmarks",'w')
for i in range(int(lower),int(upper)+1,3):
tvmarkfile.write(str(a+b*i)+" "+str(i)+" 100 s \n")
tvmarkfile.close()
iraf.tv.tvmark(1,"",commands="/tmp/tiasgat/tvmarks",interactive="no")
## Read current grism wheel position from image FITS header
iraf.images.imutil.imgets(dp+fn,"ALAPRSTP")
oldwheelunits=float(iraf.images.imutil.imgets.value)
#print "APERSTEP", oldwheelunits
## Read binning FITS headers
iraf.images.imutil.imgets(dp+fn,"CDELT1")
xbin=float(iraf.images.imutil.imgets.value)
iraf.images.imutil.imgets(dp+fn,"CDELT2")
ybin=float(iraf.images.imutil.imgets.value)
iraf.images.imutil.imgets(dp+fn,"CRVAL1")
xstart=float(iraf.images.imutil.imgets.value)
messageOut(mt,"\nBinning factors "+str(int(xbin))+" x "+str(int(ybin))+"\n")
#messageOut(mt,"Xstart "+str(int(xstart))+"\n")
#messageOut(mt,"half "+str(int((lower+upper)/2))+"\n")
if xbin==1:
messageOut(mt,"\nSlitpos: X-position at CCD center "+str((xstart-1+a+b*(lower+upper)/2))+" according to fit\n")
else:
messageOut(mt,"\nSlitpos: X-binning is not 1: set wrongly in vsalign.run ?!\n\n")
## Correct the angle for the binning factors.
## A full wheel turn corresponds to 320000 units
offsetangle=320000 * math.atan(b*xbin/ybin) / (2*math.pi)
messageOut(mt,"Alignment: Offset to motor units "+str(offsetangle)+"\n")
newwheelunits=offsetangle + oldwheelunits
if newwheelunits < 0: newwheelunits+=320000
return "Result for %s : current APERTURE wheel units %d, suggested new value %d" % \
(fn, (0.5+oldwheelunits), (0.5+newwheelunits))
def telluric_atmo(imgstd):
# print "LOGX:: Entering `telluric_atmo` method/function in %(__file__)s"
# % globals()
import numpy as np
import ntt
from pyraf import iraf
try: import pyfits
except: from astropy.io import fits as pyfits
iraf.images(_doprint=0)
iraf.noao(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.onedspec(_doprint=0)
toforget = ['imfilter.gauss', 'specred.apall', 'longslit.identify', 'longslit.reidentify', 'specred.standard',
'onedspec.wspectext']
for t in toforget:
iraf.unlearn(t)
_grism = ntt.util.readkey3(ntt.util.readhdr(imgstd), 'grism')
imgout = 'invers_atmo_' + imgstd
ntt.util.delete(imgout)
iraf.set(direc=ntt.__path__[0] + '/')
_cursor = 'direc$standard/ident/cursor_sky_0'
iraf.noao.onedspec.bplot(imgstd, cursor=_cursor,
spec2=imgstd, new_ima=imgout, overwri='yes')
xxstd, ffstd = ntt.util.readspectrum(imgout)
if _grism in ['Gr13', 'Gr16']:
llo2 = np.compress((np.array(xxstd) >= 7550) & (
np.array(xxstd) <= 7750), np.array(xxstd))
llh2o = np.compress((np.array(xxstd) >= 7100) & (
np.array(xxstd) <= 7500), np.array(xxstd))
ffo2 = np.compress((np.array(xxstd) >= 7550) & (
np.array(xxstd) <= 7750), np.array(ffstd))
ffh2o = np.compress((np.array(xxstd) >= 7100) & (
np.array(xxstd) <= 7500), np.array(ffstd))
elif _grism in ['Gr11']:
llo2 = np.compress((np.array(xxstd) >= 6830) & (
np.array(xxstd) <= 7100), np.array(xxstd))
llh2o = np.compress((np.array(xxstd) >= 7100) & (
np.array(xxstd) <= 7500), np.array(xxstd))
ffo2 = np.compress((np.array(xxstd) >= 6830) & (
np.array(xxstd) <= 7100), np.array(ffstd))
ffh2o = np.compress((np.array(xxstd) >= 7100) & (
np.array(xxstd) <= 7500), np.array(ffstd))
if _grism in ['Gr13', 'Gr16', 'Gr11']:
_skyfileh2o = 'direc$standard/ident/ATLAS_H2O.fits'
_skyfileo2 = 'direc$standard/ident/ATLAS_O2.fits'
atlas_smooto2 = '_atlas_smoot_o2.fits'
atlas_smooth2o = '_atlas_smoot_h2o.fits'
_sigma = 200
ntt.util.delete(atlas_smooto2)
ntt.util.delete(atlas_smooth2o)
iraf.imfilter.gauss(_skyfileh2o, output=atlas_smooth2o, sigma=_sigma)
iraf.imfilter.gauss(_skyfileo2, output=atlas_smooto2, sigma=_sigma)
llskyh2o, ffskyh2o = ntt.util.readspectrum(atlas_smooth2o)
llskyo2, ffskyo2 = ntt.util.readspectrum(atlas_smooto2)
ffskyo2cut = np.interp(llo2, llskyo2, ffskyo2)
ffskyh2ocut = np.interp(llh2o, llskyh2o, ffskyh2o)
_scaleh2o = []
integral_h2o = []
for i in range(1, 21):
j = 0.6 + i * 0.04
_ffskyh2ocut = list((np.array(ffskyh2ocut) * j) + 1 - j)
diff_h2o = abs(_ffskyh2ocut - ffh2o)
integraleh2o = np.trapz(diff_h2o, llh2o)
integral_h2o.append(integraleh2o)
_scaleh2o.append(j)
_scaleo2 = []
integral_o2 = []
for i in range(1, 21):
j = 0.6 + i * 0.04
_ffskyo2cut = list((np.array(ffskyo2cut) * j) + 1 - j)
diff_o2 = abs(_ffskyo2cut - ffo2)
integraleo2 = np.trapz(diff_o2, llo2)
integral_o2.append(integraleo2)
_scaleo2.append(j)
sh2o = _scaleh2o[np.argmin(integral_h2o)]
so2 = _scaleo2[np.argmin(integral_o2)]
telluric_features = ((np.array(ffskyh2o) * sh2o) +
1 - sh2o) + ((np.array(ffskyo2) * so2) + 1 - so2) - 1
telluric_features = np.array([1] + list(telluric_features) + [1])
llskyo2 = np.array([1000] + list(llskyo2) + [15000])
telluric_features_cut = np.interp(xxstd, llskyo2, telluric_features)
_imgout = 'atmo_' + imgstd
data1, hdr = pyfits.getdata(imgstd, 0, header=True)
data1[0] = np.array(telluric_features_cut)
data1[1] = data1[1] / data1[1]
data1[2] = data1[2] / data1[2]
data1[3] = data1[3] / data1[3]
ntt.util.delete(_imgout)
pyfits.writeto(_imgout, np.float32(data1), hdr)
ntt.util.delete(atlas_smooto2)
ntt.util.delete(atlas_smooth2o)
ntt.util.delete(imgout)
else:
_imgout = ''
print '### telluric correction with model not possible '
return _imgout
def start(kind, telluricDirectoryList="", scienceDirectoryList=""):
"""
start(kind): Do a full reduction of either Science or Telluric data.
nifsReduce- for the telluric and science data reduction.
Reduces NIFS telluric and science frames and attempts a flux calibration.
Parameters are loaded from runtimeData/config.cfg. This script will
automatically detect if it is being run on telluric data or science data.
There are 6 steps.
INPUT:
+ Raw files
- Science frames
- Sky frames
+ Calibration files
- MDF shift file
- Bad Pixel Mask (BPM)
- Flat field frame
- Reduced arc frame
- Reduced ronchi mask frame
- arc and ronchi database/ files
OUTPUT:
- If telluric reduction an efficiency spectrum used to telluric correct and absolute flux
calibrate science frames
- If science reduction a reduced science data cube.
Args:
kind (string): either 'Telluric' or 'Science'.
telluricDirectoryList (string): Used by low memory pipeline.
scienceDirectoryList (string): Used by low memory pipeline.
"""
# TODO(nat): Right now the pipeline will crash if you decide to skip, say, doing a bad
# pixel correction. This is because each step adds a prefix to the frame name, and most following
# steps depend on that prefix being there.
# One way to fix this is if a step is to be skipped, iraf.copy() is called instead to copy the frame and
# add the needed prefix. Messy but it might work for now.
###########################################################################
## ##
## BEGIN - GENERAL REDUCTION SETUP ##
## ##
###########################################################################
# Store current working directory for later use.
path = os.getcwd()
# Set up the logging file.
log = os.getcwd() + '/Nifty.log'
logging.info('\n#################################################')
logging.info('# #')
logging.info('# Start the NIFS Science and Telluric Reduction #')
logging.info('# #')
logging.info('#################################################\n')
# Set up/prepare IRAF.
iraf.gemini()
iraf.gemtools()
iraf.gnirs()
iraf.nifs()
# Reset to default parameters the used IRAF tasks.
iraf.unlearn(iraf.gemini, iraf.gemtools, iraf.gnirs, iraf.nifs,
iraf.imcopy)
# From http://bishop.astro.pomona.edu/Penprase/webdocuments/iraf/beg/beg-image.html:
# Before doing anything involving image display the environment variable
# stdimage must be set to the correct frame buffer size for the display
# servers (as described in the dev$graphcap file under the section "STDIMAGE
# devices") or to the correct image display device. The task GDEVICES is
# helpful for determining this information for the display servers.
iraf.set(stdimage='imt2048')
# Prepare the IRAF package for NIFS.
# NSHEADERS lists the header parameters used by the various tasks in the
# NIFS package (excluding headers values which have values fixed by IRAF or
# FITS conventions).
iraf.nsheaders("nifs", logfile=log)
# Set clobber to 'yes' for the script. This still does not make the gemini
# tasks overwrite files, so:
# YOU WILL LIKELY HAVE TO REMOVE FILES IF YOU RE_RUN THE SCRIPT.
user_clobber = iraf.envget("clobber")
iraf.reset(clobber='yes')
# This helps make sure all variables are initialized to prevent bugs.
scienceSkySubtraction = None
scienceOneDExtraction = None
extractionXC = None
extractionYC = None
extractionRadius = None
telluricSkySubtraction = None
# Load reduction parameters from runtimeData/config.cfg.
with open('./config.cfg') as config_file:
config = ConfigObj(config_file, unrepr=True)
# Read general pipeline config.
over = config['over']
manualMode = config['manualMode']
calDirList = config['calibrationDirectoryList']
scienceOneDExtraction = config['scienceOneDExtraction']
extractionXC = config['extractionXC']
extractionYC = config['extractionYC']
extractionRadius = config['extractionRadius']
if kind == 'Telluric':
# Telluric reduction specific config.
telluricReductionConfig = config['telluricReductionConfig']
if telluricDirectoryList:
observationDirectoryList = telluricDirectoryList
elif not telluricDirectoryList:
observationDirectoryList = config['telluricDirectoryList']
start = telluricReductionConfig['telStart']
stop = telluricReductionConfig['telStop']
telluricSkySubtraction = telluricReductionConfig[
'telluricSkySubtraction']
if kind == 'Science':
# Science reduction specific config.
scienceReductionConfig = config['scienceReductionConfig']
if scienceDirectoryList:
observationDirectoryList = scienceDirectoryList
elif not scienceDirectoryList:
observationDirectoryList = config['scienceDirectoryList']
start = scienceReductionConfig['sciStart']
stop = scienceReductionConfig['sciStop']
scienceSkySubtraction = scienceReductionConfig[
'scienceSkySubtraction']
###########################################################################
## ##
## COMPLETE - GENERAL REDUCTION SETUP ##
## ##
###########################################################################
# nifsReduce has two nested loops that reduced data.
# It loops through each science (or telluric) directory, and
# runs through a series of calibrations steps on the data in that directory.
# Loop through all the observation (telluric or science) directories to perform a reduction on each one.
for observationDirectory in observationDirectoryList:
###########################################################################
## ##
## BEGIN - OBSERVATION SPECIFIC SETUP ##
## ##
###########################################################################
# Print the current directory of data being reduced.
logging.info(
"\n#################################################################################"
)
logging.info(" ")
logging.info(" Currently working on reductions in")
logging.info(" in " + str(observationDirectory))
logging.info(" ")
logging.info(
"#################################################################################\n"
)
os.chdir(observationDirectory)
tempObs = observationDirectory.split(os.sep)
obsid = tempObs[-1]
# Change the iraf directory to the current directory.
pwd = os.getcwd()
iraffunctions.chdir(pwd)
# Copy relevant calibrations over to the science directory.
# Open and store the name of the MDF shift reference file from shiftfile into shift.
shift = 'calibrations/shiftFile'
# Open and store the name of the flat frame
flat = 'calibrations/finalFlat'
# Open and store the bad pixel mask
finalBadPixelMask = 'calibrations/finalBadPixelMask'
# Ronchi, arc and database must all be in local calibrations directory
# Open and store the name of the reduced spatial correction ronchi flat frame name from ronchifile in ronchi.
ronchi = 'finalRonchi'
# Open and store the name of the reduced wavelength calibration arc frame from arclist in arc.
arc = 'finalArc'
if os.path.exists(os.getcwd() + '/' + ronchi + ".fits"):
if over:
iraf.delete(os.getcwd() + '/calibrations/finalRonchi.fits')
# Copy the spatial calibration ronchi flat frame from Calibrations_grating to the observation directory.
shutil.copy(os.getcwd() + '/calibrations/finalRonchi.fits',
ronchi + '.fits')
else:
print "\nOutput exists and -over not set - skipping copy of reduced ronchi"
else:
shutil.copy(os.getcwd() + '/calibrations/finalRonchi.fits',
ronchi + '.fits')
if os.path.exists(os.getcwd() + '/' + arc + ".fits"):
if over:
iraf.delete(os.getcwd() + '/calibrations/finalArc.fits')
# Copy the spatial calibration arc flat frame from Calibrations_grating to the observation directory.
shutil.copy(os.getcwd() + '/calibrations/finalArc.fits',
arc + '.fits')
else:
print "\nOutput exists and -over not set - skipping copy of reduced arc"
else:
shutil.copy(os.getcwd() + '/calibrations/finalArc.fits',
arc + '.fits')
# Make sure the database files are in place. Current understanding is that
# these should be local to the reduction directory, so need to be copied from
# the calDir.
if os.path.isdir("./database"):
if over:
shutil.rmtree("./database")
os.mkdir("./database")
for item in glob.glob("calibrations/database/*"):
shutil.copy(item, "./database/")
else:
print "\nOutput exists and -over not set - skipping copy of database directory"
else:
os.mkdir('./database/')
for item in glob.glob("calibrations/database/*"):
shutil.copy(item, "./database/")
if telluricSkySubtraction or scienceSkySubtraction:
# Read the list of sky frames in the observation directory.
try:
skyFrameList = open("skyFrameList", "r").readlines()
skyFrameList = [frame.strip() for frame in skyFrameList]
except:
logging.info(
"\n#####################################################################"
)
logging.info(
"#####################################################################"
)
logging.info("")
logging.info(
" WARNING in reduce: No sky frames were found in a directory."
)
logging.info(" Please make a skyFrameList in: " +
str(os.getcwd()))
logging.info("")
logging.info(
"#####################################################################"
)
logging.info(
"#####################################################################\n"
)
raise SystemExit
sky = skyFrameList[0]
# If we are doing a telluric reduction, open the list of telluric frames in the observation directory.
# If we are doing a science reduction, open the list of science frames in the observation directory.
if kind == 'Telluric':
tellist = open('tellist', 'r').readlines()
tellist = [frame.strip() for frame in tellist]
elif kind == 'Science':
scienceFrameList = open("scienceFrameList", "r").readlines()
scienceFrameList = [frame.strip() for frame in scienceFrameList]
# For science frames, check to see if the number of sky frames matches the number of science frames.
# IF NOT duplicate the sky frames and rewrite the sky file and skyFrameList.
if scienceSkySubtraction:
if not len(skyFrameList) == len(scienceFrameList):
skyFrameList = makeSkyList(skyFrameList, scienceFrameList,
observationDirectory)
###########################################################################
## ##
## COMPLETE - OBSERVATION SPECIFIC SETUP ##
## BEGIN DATA REDUCTION FOR AN OBSERVATION ##
## ##
###########################################################################
# Check start and stop values for reduction steps. Ask user for a correction if
# input is not valid.
valindex = start
while valindex > stop or valindex < 1 or stop > 6:
logging.info(
"\n#####################################################################"
)
logging.info(
"#####################################################################"
)
logging.info("")
logging.info(
" WARNING in reduce: invalid start/stop values of observation"
)
logging.info(" reduction steps.")
logging.info("")
logging.info(
"#####################################################################"
)
logging.info(
"#####################################################################\n"
)
valindex = int(
raw_input(
"\nPlease enter a valid start value (1 to 7, default 1): ")
)
stop = int(
raw_input(
"\nPlease enter a valid stop value (1 to 7, default 7): "))
while valindex <= stop:
###########################################################################
## STEP 1: Prepare raw data; science, telluric and sky frames ->n ##
###########################################################################
if valindex == 1:
if manualMode:
a = raw_input(
"About to enter step 1: locate the spectrum.")
if kind == 'Telluric':
tellist = prepare(tellist, shift, finalBadPixelMask, log,
over)
elif kind == 'Science':
scienceFrameList = prepare(scienceFrameList, shift,
finalBadPixelMask, log, over)
if telluricSkySubtraction or scienceSkySubtraction:
skyFrameList = prepare(skyFrameList, shift,
finalBadPixelMask, log, over)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 1: Locate the Spectrum (and prepare raw data) ->n - COMPLETED "
)
logging.info("")
logging.info(
"##############################################################################\n"
)
###########################################################################
## STEP 2: Sky Subtraction ->sn ##
###########################################################################
elif valindex == 2:
if manualMode:
a = raw_input("About to enter step 2: sky subtraction.")
# Combine telluric sky frames.
if kind == 'Telluric':
if telluricSkySubtraction:
if len(skyFrameList) > 1:
combineImages(skyFrameList, "gn" + sky, log, over)
else:
copyImage(skyFrameList, 'gn' + sky + '.fits', over)
skySubtractTel(tellist, "gn" + sky, log, over)
else:
for image in tellist:
iraf.copy('n' + image + '.fits',
'sn' + image + '.fits')
if kind == 'Science':
if scienceSkySubtraction:
skySubtractObj(scienceFrameList, skyFrameList, log,
over)
else:
for image in scienceFrameList:
iraf.copy('n' + image + '.fits',
'sn' + image + '.fits')
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(" STEP 2: Sky Subtraction ->sn - COMPLETED ")
logging.info("")
logging.info(
"##############################################################################\n"
)
##############################################################################
## STEP 3: Flat field, slice, subtract dark and correct bad pixels ->brsn ##
##############################################################################
elif valindex == 3:
if manualMode:
a = raw_input(
"About to enter step 3: flat fielding and bad pixels correction."
)
if kind == 'Telluric':
applyFlat(tellist, flat, log, over, kind)
fixBad(tellist, log, over)
elif kind == 'Science':
applyFlat(scienceFrameList, flat, log, over, kind)
fixBad(scienceFrameList, log, over)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 3: Flat fielding and Bad Pixels Correction ->brsn - COMPLETED "
)
logging.info("")
logging.info(
"##############################################################################\n"
)
###########################################################################
## STEP 4: Derive and apply 2D to 3D transformation ->tfbrsn ##
###########################################################################
elif valindex == 4:
if manualMode:
a = raw_input(
"About to enter step 4: 2D to 3D transformation and Wavelength Calibration."
)
if kind == 'Telluric':
fitCoords(tellist, arc, ronchi, log, over, kind)
transform(tellist, log, over)
elif kind == 'Science':
fitCoords(scienceFrameList, arc, ronchi, log, over, kind)
transform(scienceFrameList, log, over)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 4: 2D to 3D transformation and Wavelength Calibration ->tfbrsn - COMPLETED "
)
logging.info("")
logging.info(
"##############################################################################\n"
)
############################################################################
## STEP 5 (tellurics): For telluric data derive a telluric ##
## correction ->gxtfbrsn ##
## STEP 5 (science): For science apply an efficiency correction and make ##
## a data cube (not necessarily in that order). ##
## (i) Python method applies correction to nftransformed cube. ##
## Good for faint objects. ->cptfbrsn ##
## (ii) iraf.telluric method applies correction to ##
## nftransformed result (not quite a data cube) then ##
## nftransforms cube. ->catfbrsn ##
## (iii) If no telluric correction/flux calibration to be ##
## applied make a plain data cube. ->ctfbrsn ##
############################################################################
elif valindex == 5:
if manualMode:
a = raw_input("About to enter step 5.")
# For telluric data:
# Make a combined extracted 1D standard star spectrum.
if kind == 'Telluric':
extractOneD(tellist, kind, log, over, extractionXC,
extractionYC, extractionRadius)
# TODO(nat): add this as a parameter; encapsulate this.
copyToScience = True
if copyToScience:
# Copy final extracted results to science directory.
try:
with open("scienceMatchedTellsList", "r") as f:
lines = f.readlines()
lines = [x.strip() for x in lines]
for i in range(len(lines)):
if "obs" in lines[i]:
k = 1
while i + k != len(
lines) and "obs" not in lines[i +
k]:
copyResultsToScience(
"gxtfbrsn" + tellist[0] + ".fits",
"0_tel" + lines[i + k] + ".fits",
over)
k += 1
except IOError:
logging.info(
"\nNo scienceMatchedTellsList found in " +
os.getcwd() +
" . Skipping copy of extracted spectra to science directory."
)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 5a: Extract 1D Spectra and Make Combined 1D Standard Star Spectrum"
)
logging.info(" ->gxtfbrsn - COMPLETED")
logging.info("")
logging.info(
"##############################################################################\n"
)
#TODO(nat): add this as a parameter.
makeTelluricCube = True
if makeTelluricCube:
makeCube('tfbrsn', tellist, log, over)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 5b: Make uncorrected standard star data cubes, ->ctfbrsn - COMPLETED"
)
logging.info("")
logging.info(
"##############################################################################\n"
)
# For Science data:
# Possibly extract 1D spectra, and make uncorrected cubes.
elif kind == 'Science':
if scienceOneDExtraction:
extractOneD(scienceFrameList, kind, log, over,
extractionXC, extractionYC,
extractionRadius)
copyExtracted(scienceFrameList, over)
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 5a: Make extracted 1D Science spectra, ->ctgbrsn - COMPLETED"
)
logging.info("")
logging.info(
"##############################################################################\n"
)
makeCube('tfbrsn', scienceFrameList, log, over)
# TODO(nat): encapsulate this inside a function.
if os.path.exists('products_uncorrected'):
if over:
shutil.rmtree('products_uncorrected')
os.mkdir('products_uncorrected')
else:
logging.info(
"\nOutput exists and -over not set - skipping creating of products_uncorrected directory"
)
else:
os.mkdir('products_uncorrected')
for item in scienceFrameList:
if os.path.exists('products_uncorrected/ctfbrsn' +
item + '.fits'):
if over:
os.remove('products_uncorrected/ctfbrsn' +
item + '.fits')
shutil.copy(
'ctfbrsn' + item + '.fits',
'products_uncorrected/ctfbrsn' + item +
'.fits')
else:
logging.info(
"\nOutput exists and -over not set - skipping copy of uncorrected cube"
)
else:
shutil.copy(
'ctfbrsn' + item + '.fits',
'products_uncorrected/ctfbrsn' + item +
'.fits')
if os.path.exists('products_telluric_corrected'):
if over:
shutil.rmtree('products_telluric_corrected')
os.mkdir('products_telluric_corrected')
else:
logging.info(
"\nOutput exists and -over not set - skipping creating of products_telluric_corrected directory"
)
else:
os.mkdir('products_telluric_corrected')
for item in scienceFrameList:
if os.path.exists(
'products_telluric_corrected/ctfbrsn' + item +
'.fits'):
if over:
os.remove(
'products_telluric_corrected/ctfbrsn' +
item + '.fits')
shutil.copy(
'ctfbrsn' + item + '.fits',
'products_telluric_corrected/ctfbrsn' +
item + '.fits')
else:
logging.info(
"\nOutput exists and -over not set - skipping copy of uncorrected cube"
)
else:
shutil.copy(
'ctfbrsn' + item + '.fits',
'products_telluric_corrected/ctfbrsn' + item +
'.fits')
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(
" STEP 5b: Make uncorrected science data cubes, ->ctfbrsn - COMPLETED"
)
logging.info("")
logging.info(
"##############################################################################\n"
)
valindex += 1
logging.info(
"\n##############################################################################"
)
logging.info("")
logging.info(" COMPLETE - Reductions completed for " +
str(observationDirectory))
logging.info("")
logging.info(
"##############################################################################\n"
)
# Return to directory script was begun from.
os.chdir(path)
def start(obsDirList, calDirList, start, stop, tel, telinter, over):
path = os.getcwd()
# set up log
FORMAT = '%(asctime)s %(message)s'
DATEFMT = datefmt()
logging.basicConfig(filename='main.log',
format=FORMAT,
datefmt=DATEFMT,
level=logging.DEBUG)
log = os.getcwd() + '/main.log'
# loops through all the observation directories to perform the science reduction on each one
for obsDir in obsDirList:
os.chdir(obsDir)
tempObs = obsDir.split(os.sep)
# finds the Calibrations directory that corresponds to the science observation date
for calDir in calDirList:
tempCal = calDir.split(os.sep)
if tempObs[-3] == tempCal[-2]:
calDir = calDir + '/'
break
elif tempObs[-4] == tempCal[-2]:
calDir = calDir + '/'
break
obsid = tempObs[-1]
# reset iraf tasks
iraf.unlearn(iraf.gemini, iraf.gemtools, iraf.gnirs, iraf.nifs,
iraf.imcopy)
iraf.set(stdimage='imt2048')
# change the iraf directory to the current directory
pwd = os.getcwd()
iraffunctions.chdir(pwd)
iraf.nsheaders("nifs", logfile=log)
# define all the necessary variables and lists for the calibration and science images
shift = calDir + str(open(calDir + "shiftfile",
"r").readlines()[0]).strip()
flat = calDir + str(open(calDir + "flatfile",
"r").readlines()[0]).strip()
ronchi = open(calDir + "ronchifile", "r").readlines()[0].strip()
iraf.copy(calDir + ronchi + ".fits", output="./")
sflat_bpm = calDir + str(
open(calDir + "sflat_bpmfile", "r").readlines()[0]).strip()
arcdark = calDir + str(
open(calDir + "arcdarkfile", "r").readlines()[0]).strip()
# copy wavelength calibrated arc to obsDir
arc = "wrgn" + str(open(calDir + "arclist",
"r").readlines()[0]).strip()
iraf.copy(calDir + arc + ".fits", output="./")
# determines whether the data is science or telluric
if tempObs[-2] == 'Tellurics':
kind = 'Telluric'
objlist = open('tellist', 'r').readlines()
objlist = [image.strip() for image in objlist]
try:
skylist = open("skylist", "r").readlines()
skylist = [image.strip() for image in skylist]
except:
print "\nNo sky images were found for standard star. Please make a skylist in the telluric directory\n"
raise SystemExit
sky = skylist[0]
else:
kind = 'Object'
objlist = open("objlist", "r").readlines()
objlist = [image.strip() for image in objlist]
skylist = open("skylist", "r").readlines()
skylist = [image.strip() for image in skylist]
sky = skylist[0]
# check to see if the number of sky images matches the number of science images and if not duplicates sky images and rewrites the sky file and skylist
if not len(skylist) == len(objlist):
skylist = makeSkyList(skylist, objlist, obsDir)
centers = writeCenters(objlist)
# Make sure the database files are in place. Current understanding is that
# these should be local to the reduction directory, so need to be copied from
# the calDir.
if os.path.isdir("./database"):
if over:
shutil.rmtree("./database")
os.mkdir("./database")
elif not os.path.isdir("./database"):
os.mkdir('./database/')
iraf.copy(input=calDir + 'database/*', output="./database/")
#=========================================================
# Start main processing steps. Do this within a while loop
# to allow the use of start and stop positions
#=========================================================
logging.info('############################')
logging.info(' ')
logging.info(' Reducing Observations ')
logging.info(' ')
logging.info('############################')
print '############################'
print ' '
print ' Reducing Observations '
print ' '
print '############################'
valindex = start
if valindex > stop or valindex < 1 or stop > 9:
print "problem with start/stop values"
while valindex <= stop:
####################
## Prepare raw data ->n
logging.info('Prepare raw data ->n')
if valindex == 1:
objlist = prepare(objlist, shift, sflat_bpm, log, over)
skylist = prepare(skylist, shift, sflat_bpm, log, over)
#####################
## Combine multiple frames ->gn
elif valindex == 2:
if kind == 'Object':
logging.info('Combine multiple frames ->gn')
if len(skylist) > 1:
combineImages(skylist, "gn" + sky, log, over)
else:
copyImage(skylist, 'gn' + sky + '.fits', over)
else:
pass
##################
## Sky Subtraction ->gn
elif valindex == 3:
skySubtractObj(objlist, skylist, log, over)
logging.info('Sky Subtraction ->gn')
#################
## Flat field ->rgn
elif valindex == 4:
applyFlat(objlist, flat, log, over, kind)
logging.info('Flat field ->rgn')
#################
## Correct bad pixels ->brgn
elif valindex == 5:
fixBad(objlist, log, over)
logging.info('Correct bad pixels ->brgn')
#################
## Derive 2D->3D transformation ->fbrgn
elif valindex == 6:
fitCoords(objlist, arc, ronchi, log, over, kind)
logging.info('Derive 2D->3D transformation ->fbrgn')
#################
## Apply transformation ->tfbrgn
elif valindex == 7:
transform(objlist, log, over)
logging.info('Apply transformation ->tfbrgn')
#################
## Derive or apply telluric correction ->atfbrgn
elif valindex == 8:
logging.info('Derive or apply telluric correction ->atfbrgn')
if kind == 'Telluric':
makeTelluric(objlist, log, over)
elif kind == 'Object' and tel and telinter == 'no':
makeCube('tfbrgn', objlist, False, obsDir, log, over)
applyTelluric(objlist, obsid, skylist, telinter, log, over)
elif kind == 'Object' and tel and telinter == 'yes':
applyTelluric(objlist, obsid, skylist, telinter, log, over)
#################
## Create a 3D cube -> catfbrgn
elif valindex == 9:
if kind == "Telluric":
print "No cube being made for tellurics"
elif telinter == 'yes' and kind == 'Object' and tel:
logging.info('Create a 3D cube -> catfbrgn')
makeCube('atfbrgn', objlist, tel, obsDir, log, over)
elif kind == 'Object' and not tel and telinter == 'yes':
logging.info('Create a 3D cube -> ctfbrgn')
makeCube('tfbrgn', objlist, tel, obsDir, log, over)
valindex += 1
os.chdir(path)
return
def performPhotometry(task, logger):
#iraf.prcacheOff()
[iraf.unlearn(t) for t in ('phot','pstselect','psf','allstar')]
iraf.set(imtype="fits,noinherit") # set image output format
iraf.set(clobber="yes")
hdu=pyfits.open(task['images'])[0]
hdr = hdu.header
imdata = hdu.data
for key,value in task['fits'].iteritems():
task[key] = hdr.get(value,1)
#Sextractor to find stars; add an object for the force detect
logger.info('Running SExtractor on [%s]' % os.path.basename(task['images']))
sex = sextractor.SExtractor()
makeSexConfig(sex,task)
sex.run(task['images'])
catalog = sex.catalog()
#Set up image parameters
MIN_X = max(1,int(task['numpixx']*task['filtfactor']))
MIN_Y = max(1,int(task['numpixy']*task['filtfactor']))
MAX_X = int(task['numpixx']*(1-task['filtfactor']))
MAX_Y = int(task['numpixy']*(1-task['filtfactor']))
AREAXY = '[%s:%s,%s:%s]' % (MIN_X, MAX_X, MIN_Y, MAX_Y)
AREANO = '[%s:%s,%s:%s]' % (MIN_X, MAX_X-2*MIN_X, MIN_Y, MAX_Y-2*MIN_Y)
try:
task['pixscale'] = abs(hdr.get('CD1_1'))*3600.
except TypeError:
task['pixscale'] = abs(hdr.get('CDELT1'))*3600.
task['seeing'] = np.median( sorted([i['FWHM_IMAGE'] for i in catalog])[:int(-len(catalog)*0.5)] ) #Take the median of the "bottom" 50% of objects
logger.info('--> %s SExtractor detected bright objects in the field' % (len(catalog),) )
logger.info('--> %0.2f median FWHM of bright objects in the field, in arcsec' % (task['seeing']*task['pixscale'],))
task['objects'] = [(i['ALPHA_J2000'],i['DELTA_J2000']) for i in catalog]
task['objects'].append(task['objwcs'])
task['objectlist'] = open(os.path.join(task['output_directory'],'objectlist'),'w')
task['objectlist'].write('\n'.join([' %s %s' % (i[0],i[1]) for i in task['objects']]))
task['objectlist'].close()
logger.info('Running iraf.imstat')
irafoutput = iraf.imstat(images=task['images']+AREANO,fields='midpt,min,max,stddev', format=0, Stdout=1)
task['nimgs'] = hdr.get('NIMGS',1)
task['gain'] *= task['nimgs']*2/3.
task['ron'] *= np.sqrt(task['nimgs'])/task['nimgs']*constants.INTERPSM[task['band']]
task['datamean'], task['datamin'], task['datamax'], task['datastdev'] = map(float, irafoutput[0].split())
irafoutput = iraf.imstat(images=task['images'],fields='stddev,midpt',nclip=25,format=0,cache='yes',Stdout=1)
task['skynoise'], task['datamean'] = map(float, irafoutput[0].split() )
task['skynoise'] *= constants.INTERPSM[task['band']]
task['airmass'] = hdr.get('AIRMASS',1)
task['zmag'] -= (float(task['airmass'])-1.0)*constants.extinction_coefficients[task['band']]
task['match_proximity'] = 2.5 * task['seeing']
logger.info('--> %5.2f counts: Sky noise, corrected for drizzle imcombine' % task['skynoise'])
logger.info('--> %5.2f Median count value, after background subtraction' % task['datamean'])
logger.info('--> %5.2f Airmass' % task['airmass'])
#prepare temp files that iraf will use
for filename in ('photfile','pstfile','psfimg','opstfile','groupfile','allstarfile','rejfile','subimage'):
task[filename] = open(os.path.join(task['output_directory'],filename),'w')
task[filename].close()
#iraf.phot to get APP magnitudes
logger.info('Running iraf.apphot.phot')
#apsizes = [i*task['faperture']*task['seeing'] for i in (0.4,0.5,0.6,0.8,1.0,1.2,1.5,2.0,2.5,3.0)]
#irafapsizes = ','.join(['%.2f' % i for i in apsizes])
irafapsizes = '%0.2f' % (task['faperture']*task['seeing'])
kwargs = dict(image=task['images'],coords=task['objectlist'].name,
output=task['photfile'].name,
interac='no',scale=1,
fwhmpsf=task['seeing'],
wcsin='world', wcsout='physical',
sigma=task['skynoise'],
datamin=task['datamin'],
datamax=task['datamax'],
readnoi=task['ron'],
epadu=task['gain'],
itime=task['exposure'],
xairmass=task['airmass'],
ifilter=task['band'],
otime=task['dateobs'],
aperture= irafapsizes,
zmag=task['zmag'],
annulus=task['fannulus']*task['seeing'],
dannulus=task['fdannulus']*task['seeing'],
calgorithm='gauss',
cbox = 1.5*task['seeing'],
maxshift=2.0*task['seeing'],
mode="h",Stdout=1,verify=0)
iraf.phot(**kwargs)
if task['band'] not in constants.infrared:
#iraf.pstselect to choose objects for PSF modelling
logger.info('Running iraf.daophot.pstselect')
kwargs = dict(image=task['images'],
photfile=task['photfile'].name,pstfile=task['pstfile'].name,
maxnpsf=task['pstnumber'],
wcsin='physical',
wcsout='physical',
interac="no",verify='no',scale=1,
fwhmpsf=task['seeing'],
datamin=0,
datamax=task['datamax'],
psfrad=3.0*task['seeing'],
fitrad=1.0*task['seeing'],
recente='yes',
nclean=task['nclean'],
mode="h",Stdout=1)
iraf.pstselect(**kwargs)
#iraf.psf to model PSF
logger.info('Running iraf.daophot.psf')
kwargs = dict( image=task['images'],
photfile=task['photfile'].name,
pstfile=task['pstfile'].name,
psfimage=task['psfimg'].name,
opstfile=task['opstfile'].name,
groupfile=task['groupfile'].name,
wcsin='physical',wcsout='physical',
interac="no",verify="no",scale=1,
fwhmpsf=task['seeing'],
sigma=task['skynoise'],
datamin=task['datamin'],
datamax=task['datamax'],
readnoi=task['ron'],
epadu=task['gain'],
itime=task['exposure'],
xairmass=task['airmass'],
ifilter=task['band'],
otime=task['dateobs'],
function=task['func'],
varorder=task['varorder'],
saturat='no',
psfrad=3.0*task['seeing'],
fitrad=1.*task['faperture']*task['seeing'],
nclean=task['nclean'],
mergerad=1.5*task['seeing'],
mode='h',Stdout=1)
iraf.psf(**kwargs)
logger.info('Running iraf.daophot.allstar')
#iraf.allstars to compute PSF photometry; recenter with recenter='yes', mergerad=<value> to avoid duplicate detection
kwargs = dict(image=task['images'],
photfile=task['photfile'].name,
wcsin='physical',
wcsout='physical',
psfimage=task['psfimg'].name,
allstarf=task['allstarfile'].name,
rejfile=task['rejfile'].name,
subimage=task['subimage'].name,
verbose=1,verify='no',scale=1,
fwhmpsf=task['seeing'],
sigma=task['skynoise'],
datamin=task['datamin'],
datamax=task['datamax'],
readnoi=task['ron'],
epadu=task['gain'],
itime=task['exposure'],
xairmass=task['airmass'],
ifilter=task['band'],
otime=task['dateobs'],
function=task['func'],
varorder=task['varorder'],
psfrad=3.*task['seeing'],
fitrad=1.*task['faperture']*task['seeing'],
recenter='yes',
mergerad=1.5*task['seeing'],
mode='h',Stdout=1)
iraf.allstar(**kwargs)
#Parse both photometry, convert to RA,DEC,MAG,MAGERR
logger.info('iraf tasks complete. Parsing results and calibrating')
photometry = {}
photometry['APP'] = iraf.txdump(textfiles=task['photfile'].name,
fields='XCENTER,YCENTER,MAG,MERR',expr='yes',
headers='no',Stdout=1)
if task['band'] not in constants.infrared:
photometry['PSF'] = iraf.txdump(textfiles=task['allstarfile'].name,
fields='XCENTER,YCENTER,MAG,MERR',expr='yes',
headers='no',Stdout=1)
for phototype in photometry:
kwargs = dict(input='STDIN',
output='STDOUT',
insystem='%s physical' % task['images'],
outsystem='%s world' % task['images'],
ilatuni='physical',
ilnguni='physical',
olnguni='degrees',
olatuni='degrees',
ilngfor='%10.7f',
ilatfor='%10.7f',
olngfor='%10.5f',
olatfor='%10.5f',
Stdin=photometry[phototype],Stdout=1)
photometry[phototype] = [i.split() for i in iraf.skyctran(**kwargs) if i and not i.startswith('#') and 'INDEF' not in i]
photometry[phototype] = [map(float,(i[4],i[5],i[2],i[3])) for i in photometry[phototype] ] #Now we have [(ra,dec,'mag','mageerr'),...]
results = calibrate((task['objwcs'][0],task['objwcs'][1]),task,photometry,logger)
# if 'PSF' not in results:
return results
def sensfunction(standardfile, _function, _order, _interactive):
# print "LOGX:: Entering `sensfunction` method/function in %(__file__)s" %
# globals()
import re
import os
import sys
import ntt
import datetime
try: import pyfits # added later
except: from astropy.io import fits as pyfits
from pyraf import iraf
import numpy as np
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.specred(_doprint=0)
toforget = ['specred.scopy', 'specred.sensfunc', 'specred.standard']
for t in toforget:
iraf.unlearn(t)
iraf.specred.scopy.format = 'multispec'
iraf.specred.verbose = 'no'
hdrs = ntt.util.readhdr(standardfile)
try:
_outputsens = 'sens_' + str(ntt.util.readkey3(hdrs, 'date-night')) + '_' + \
str(ntt.util.readkey3(hdrs, 'grism')) + '_' + str(ntt.util.readkey3(hdrs, 'filter')) + '_' + \
re.sub('.dat', '', ntt.util.readkey3(
hdrs, 'stdname')) + '_' + str(MJDtoday)
except:
sys.exit('Error: missing header -stdname- in standard ' +
str(standardfile) + ' ')
_outputsens = ntt.util.name_duplicate(standardfile, _outputsens, '')
if os.path.isfile(_outputsens):
if _interactive.lower() != 'yes':
ntt.util.delete(_outputsens)
else:
answ = raw_input(
'sensitivity function already computed, do you want to do it again [[y]/n] ? ')
if not answ:
answ = 'y'
if answ.lower() in ['y', 'yes']:
ntt.util.delete(_outputsens)
if not os.path.isfile(_outputsens):
iraf.set(direc=ntt.__path__[0] + '/')
_caldir = 'direc$standard/MAB/'
_extinctdir = 'direc$standard/extinction/'
_observatory = 'lasilla'
_extinction = 'lasilla2.txt'
refstar = 'm' + \
re.sub('.dat', '', pyfits.open(standardfile)
[0].header.get('stdname'))
_airmass = ntt.util.readkey3(hdrs, 'airmass')
_exptime = ntt.util.readkey3(hdrs, 'exptime')
_outputstd = 'std_' + str(ntt.util.readkey3(hdrs, 'grism')) + '_' + \
str(ntt.util.readkey3(hdrs, 'filter')) + '.fits'
ntt.util.delete(_outputstd)
ntt.util.delete(_outputsens)
iraf.specred.standard(input=standardfile, output=_outputstd, extinct=_extinctdir + _extinction,
caldir=_caldir, observa=_observatory, star_nam=refstar, airmass=_airmass,
exptime=_exptime, interac=_interactive)
iraf.specred.sensfunc(standard=_outputstd, sensitiv=_outputsens, extinct=_extinctdir + _extinction,
ignorea='yes', observa=_observatory, functio=_function, order=_order,
interac=_interactive)
data, hdr = pyfits.getdata(standardfile, 0, header=True) # added later
data1, hdr1 = pyfits.getdata(
_outputsens, 0, header=True) # added later
ntt.util.delete(_outputsens) # added later
pyfits.writeto(_outputsens, np.float32(data1), hdr) # added later
return _outputsens
def phot(fits_filename,
x_in,
y_in,
aperture=15,
sky=20,
swidth=10,
apcor=0.3,
maxcount=30000.0,
exptime=1.0,
zmag=None,
extno=0,
centroid=True):
"""
Compute the centroids and magnitudes of a bunch sources on fits image.
:rtype : astropy.table.Table
:param fits_filename: Name of fits image to measure source photometry on.
:type fits_filename: str
:param x_in: x location of source to measure
:type x_in: float, numpy.array
:param y_in: y location of source to measure
:type y_in: float, numpy.array
:param aperture: radius of circular aperture to use.
:type aperture: float
:param sky: radius of inner sky annulus
:type sky: float
:param swidth: width of the sky annulus
:type swidth: float
:param apcor: Aperture correction to take aperture flux to full flux.
:type apcor: float
:param maxcount: maximum linearity in the image.
:type maxcount: float
:param exptime: exposure time, relative to zmag supplied
:type exptime: float
:param zmag: zeropoint magnitude
:param extno: extension of fits_filename the x/y location refers to.
"""
if not hasattr(x_in, '__iter__'):
x_in = [
x_in,
]
if not hasattr(y_in, '__iter__'):
y_in = [
y_in,
]
if (not os.path.exists(fits_filename)
and not fits_filename.endswith(".fits")):
# For convenience, see if we just forgot to provide the extension
fits_filename += ".fits"
try:
input_hdulist = fits.open(fits_filename)
except Exception as err:
logger.debug(str(err))
raise TaskError("Failed to open input image: %s" % err.message)
# get the filter for this image
filter_name = input_hdulist[extno].header.get('FILTER', 'DEFAULT')
# Some nominal CFHT zeropoints that might be useful
zeropoints = {
"I": 25.77,
"R": 26.07,
"V": 26.07,
"B": 25.92,
"DEFAULT": 26.0,
"g.MP9401": 32.0,
'r.MP9601': 31.9,
'gri.MP9603': 33.520
}
if zmag is None:
logger.warning(
"No zmag supplied to daophot, looking for header or default values."
)
zmag = input_hdulist[extno].header.get('PHOTZP',
zeropoints[filter_name])
logger.warning("Setting zmag to: {}".format(zmag))
# check for magic 'zeropoint.used' files
for zpu_file in [
"{}.zeropoint.used".format(os.path.splitext(fits_filename)[0]),
"zeropoint.used"
]:
if os.access(zpu_file, os.R_OK):
with open(zpu_file) as zpu_fh:
zmag = float(zpu_fh.read())
logger.warning("Using file {} to set zmag to: {}".format(
zpu_file, zmag))
break
photzp = input_hdulist[extno].header.get(
'PHOTZP', zeropoints.get(filter_name, zeropoints["DEFAULT"]))
if zmag != photzp:
logger.warning(
("zmag sent to daophot: ({}) "
"doesn't match PHOTZP value in image header: ({})".format(
zmag, photzp)))
# setup IRAF to do the magnitude/centroid measurements
iraf.set(uparm="./")
iraf.digiphot()
iraf.apphot()
iraf.daophot(_doprint=0)
iraf.photpars.apertures = aperture
iraf.photpars.zmag = zmag
iraf.datapars.datamin = 0
iraf.datapars.datamax = maxcount
iraf.datapars.exposur = ""
iraf.datapars.itime = exptime
iraf.fitskypars.annulus = sky
iraf.fitskypars.dannulus = swidth
iraf.fitskypars.salgorithm = "mode"
iraf.fitskypars.sloclip = 5.0
iraf.fitskypars.shiclip = 5.0
if centroid:
iraf.centerpars.calgori = "centroid"
iraf.centerpars.cbox = 5.
iraf.centerpars.cthreshold = 0.
iraf.centerpars.maxshift = 2.
iraf.centerpars.clean = 'no'
else:
iraf.centerpars.calgori = "none"
iraf.phot.update = 'no'
iraf.phot.verbose = 'no'
iraf.phot.verify = 'no'
iraf.phot.interactive = 'no'
# Used for passing the input coordinates
coofile = tempfile.NamedTemporaryFile(suffix=".coo", delete=False)
for i in range(len(x_in)):
coofile.write("%f %f \n" % (x_in[i], y_in[i]))
coofile.flush()
# Used for receiving the results of the task
# mag_fd, mag_path = tempfile.mkstemp(suffix=".mag")
magfile = tempfile.NamedTemporaryFile(suffix=".mag", delete=False)
# Close the temp files before sending to IRAF due to docstring:
# "Whether the target_name can be used to open the file a second time, while
# the named temporary file is still open, varies across platforms"
coofile.close()
magfile.close()
os.remove(magfile.name)
iraf.phot(fits_filename + "[{}]".format(extno), coofile.name, magfile.name)
pdump_out = ascii.read(magfile.name, format='daophot')
logging.debug("PHOT FILE:\n" + str(pdump_out))
if not len(pdump_out) > 0:
mag_content = open(magfile.name).read()
raise TaskError("photometry failed. {}".format(mag_content))
# apply the aperture correction
pdump_out['MAG'] -= apcor
# if pdump_out['PIER'][0] != 0 or pdump_out['SIER'][0] != 0 or pdump_out['CIER'][0] != 0:
# raise ValueError("Photometry failed:\n {}".format(pdump_out))
# Clean up temporary files generated by IRAF
os.remove(coofile.name)
os.remove(magfile.name)
logger.debug("Computed aperture photometry on {} objects in {}".format(
len(pdump_out), fits_filename))
del input_hdulist
return pdump_out
# The user should provide us with information regarding wavelength, pixel scale, # extension of the cube, instrument, physical size of the target, and WCS header # information. # NOTETOSELF: make sure that the conventions at http://docs.astropy.org/en/latest/development/codeguide.html#standard-output-warnings-and-errors # are being followed. Maybe we can have a --verbose mode where extra # information gets printed to stdout. from __future__ import print_function, division #things to import regarding pyraf & iraf import pyraf from pyraf import iraf #the following line is to override the login.cl requirement of IRAF iraf.set(uparm='.') from iraf import noao, images from iraf import artdata, immatch, imcoords import sys import getopt import glob import math import os from astropy.io import fits from astropy.nddata import make_kernel, convolve
zeropoints={"I": 25.77,
"R": 26.07,
"V": 26.07,
"B": 25.92,
"DEFAULT": 26.0,
"g.MP9401": 26.4
}
### load the
if not zeropoints.has_key(filter):
filter="DEFAULT"
zmag=f[0].header.get('PHOT_C',zeropoints[filter])
f.close()
### setup IRAF to do the magnitude/centroid measurements
iraf.set(uparm="./")
iraf.digiphot()
iraf.apphot()
iraf.daophot()
### check for the majical 'zeropoint.used' file
import os
if os.access('zeropoint.used',os.R_OK):
f=file('zeropoint.used')
zmag=float(f.read())
if os.access(opt.output,os.R_OK):
os.unlink(opt.output)
iraf.photpars.apertures=int(opt.aperture)
iraf.photpars.zmag=zmag
def start(obsDirList, calDirList, over, start, stop):
# Set up the logging file
FORMAT = '%(asctime)s %(message)s'
DATEFMT = datefmt()
logging.basicConfig(filename='main.log',
format=FORMAT,
datefmt=DATEFMT,
level=logging.DEBUG)
log = os.getcwd() + '/main.log'
logging.info('###############################')
logging.info('# #')
logging.info('# Start Calibration Reduction #')
logging.info('# #')
logging.info('###############################')
print '###############################'
print '# #'
print '# Start Calibration Reduction #'
print '# #'
print '###############################'
# Unlearn the used tasks
iraf.unlearn(iraf.gemini, iraf.gemtools, iraf.gnirs, iraf.nifs)
# Prepare the package for NIFS
iraf.nsheaders("nifs", logfile=log)
iraf.set(stdimage='imt2048')
user_clobber = iraf.envget("clobber")
iraf.reset(clobber='yes')
path = os.getcwd()
# loop over the Calibrations directories and reduce the day cals in each one
for calpath in calDirList:
os.chdir(calpath)
pwdDir = os.getcwd() + "/"
iraffunctions.chdir(pwdDir)
# define the cals lists and images
flatlist = open('flatlist', "r").readlines()
flatdarklist = open("flatdarklist", "r").readlines()
arcdarklist = open("arcdarklist", "r").readlines()
arclist = open("arclist", "r").readlines()
ronchilist = open("ronchilist", "r").readlines()
calflat = (flatlist[0].strip()).rstrip('.fits')
flatdark = (flatdarklist[0].strip()).rstrip('.fits')
arcdark = (arcdarklist[0].strip()).rstrip('.fits')
arc = (arclist[0].strip()).rstrip('.fits')
ronchiflat = (ronchilist[0].strip()).rstrip('.fits')
# check start and stop values for reduction steps
valindex = start
if valindex > stop or valindex < 1 or stop > 6:
print "problem with start/stop values"
print(valindex, start, stop)
while valindex <= stop:
####################
## Prepare raw data
if valindex == 1:
getShift(calflat, over, log)
####################
## Make flat
elif valindex == 2:
makeFlat(flatlist, flatdarklist, calflat, flatdark, over, log)
####################
## Combine arc darks
elif valindex == 3:
makeArcDark(arcdarklist, arcdark, calflat, over, log)
####################
## Combine and flat field arcs
elif valindex == 4:
reduceArc(arclist, arc, log, over)
####################
## Determine the wavelength of the observation and set the arc coordinate file
elif valindex == 5:
wavecal("rgn" + arc, log, over)
####################
## Combine arc darks
elif valindex == 6:
ronchi(ronchilist, ronchiflat, calflat, over, flatdark, log)
else:
print "No step associated to this value"
valindex += 1
os.chdir(path)
return
def marksn2(img, fitstab, frame=1, fitstab2='', verbose=False):
from pyraf import iraf
from numpy import array #,log10
import lsc
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.daophot(_doprint=0)
iraf.images(_doprint=0)
iraf.imcoords(_doprint=0)
iraf.proto(_doprint=0)
iraf.set(stdimage='imt1024')
hdr = lsc.util.readhdr(fitstab)
_filter = lsc.util.readkey3(hdr, 'filter')
column = lsc.lscabsphotdef.makecatalogue([fitstab])[_filter][fitstab]
rasex = array(column['ra0'], float)
decsex = array(column['dec0'], float)
if fitstab2:
hdr = lsc.util.readhdr(fitstab2)
_filter = lsc.util.readkey3(hdr, 'filter')
_exptime = lsc.util.readkey3(hdr, 'exptime')
column = lsc.lscabsphotdef.makecatalogue([fitstab2])[_filter][fitstab2]
rasex2 = array(column['ra0'], float)
decsex2 = array(column['dec0'], float)
iraf.set(stdimage='imt1024')
iraf.display(img + '[0]', frame, fill=True, Stdout=1)
vector = []
for i in range(0, len(rasex)):
vector.append(str(rasex[i]) + ' ' + str(decsex[i]))
xy = iraf.wcsctran('STDIN',output="STDOUT",Stdin=vector,Stdout=1,image=img+'[0]',inwcs='world',units='degrees degrees',outwcs='logical',\
formats='%10.1f %10.1f',verbose='yes')[3:]
iraf.tvmark(frame,
'STDIN',
Stdin=list(xy),
mark="circle",
number='yes',
label='no',
radii=10,
nxoffse=5,
nyoffse=5,
color=207,
txsize=2)
if verbose:
# print 2.5*log10(_exptime)
for i in range(0, len(column['ra0'])):
print xy[i], column['ra0'][i], column['dec0'][i], column['magp3'][
i], column['magp4'][i], column['smagf'][i], column['magp2'][i]
if fitstab2:
vector2 = []
for i in range(0, len(rasex2)):
vector2.append(str(rasex2[i]) + ' ' + str(decsex2[i]))
xy1 = iraf.wcsctran('STDIN',output="STDOUT",Stdin=vector2,Stdout=1,image=img+'[0]',inwcs='world',units='degrees degrees',outwcs='logical',\
formats='%10.1f %10.1f',verbose='yes')[3:]
iraf.tvmark(frame,
'STDIN',
Stdin=list(xy1),
mark="cross",
number='yes',
label='no',
radii=10,
nxoffse=5,
nyoffse=5,
color=205,
txsize=2)
# Importing pyraf -- note that this requires IRAF to be installed
# and the "iraf" and "IRAFARCH" environment variables to be set
with warnings.catch_warnings():
# Ignore the ImportWarning we often get when importing pyraf, because pyraf
# likes to create working dirs with its name in the local dir.
warnings.filterwarnings("ignore", message="(.*)Not importing dir(.*)")
# Disable pyraf graphics:
from stsci.tools import capable
capable.OF_GRAPHICS = False
from pyraf import iraf
from iraf import noao, digiphot, daophot
# Turn cache off to enable multiprocessing to be used;
# cf. http://www.stsci.edu/institute/software_hardware/pyraf/pyraf_faq#3.5
iraf.prcacheOff()
iraf.set(writepars=0)
from .utils import timed
class DaophotError(Exception):
"""Raised if Daophot failed to perform an operation."""
pass
class Daophot(object):
"""DAOPHOT wrapper class.
The constructor of this class expects to receive
ALL the non-default configuration parameters because IRAF is very stateful.
from scipy import interpolate, ndimage, signal, optimize
import pf_model as pfm
import statsmodels as sm
from astropy.modeling import models, fitting
import astropy
iraf.cd(os.getcwd())
iraf.gemini()
iraf.gmos()
iraf.onedspec()
bluecut = 3450
iraf.gmos.logfile = "log.txt"
iraf.gmos.mode = 'h'
iraf.set(clobber='yes')
iraf.set(stdimage='imtgmos')
dooverscan = False
is_GS = False
def normalize_fitting_coordinate(x):
xrange = x.max() - x.min()
return (x - x.min()) / xrange
class offset_left_model(astropy.modeling.Fittable1DModel):
cutoff = astropy.modeling.Parameter(default=0)
scale = astropy.modeling.Parameter(default=1)
def _setup_iraf(self, datamin=0, datamax=60000, epadu=1., fitrad_fwhm=1.,
fitsky='no', function='moffat25', fwhmpsf=3., itime=10.,
maxiter=50, maxnpsf=60, mergerad_fwhm=2., nclean=10,
psfrad_fwhm=10., ratio=1., readnoi=0, recenter='yes',
roundlo=-1.0, roundhi=1.0,
saturated='no', sharplo=0.2, sharphi=1.0,
sigma=5., theta=0., threshold=3., varorder=1, zmag=20.,
sannulus_fwhm=2., wsannulus_fwhm=2.,
annulus_fwhm=10., dannulus_fwhm=2.):
"""Sets the IRAF/DAOPHOT configuration parameters.
Parameters
----------
fitrad_fwhm : float
The PSF fitting radius in units `fwhmpsf`. Only pixels within the
fitting radius of the center of a star will contribute to the
fits computed by the `allstar` task. For most images the fitting
radius should be approximately equal to the FWHM of the PSF
(i.e. `fitrad_fwhm = 1`). Under severely crowded conditions a
somewhat smaller value may be used in order to improve the fit.
If the PSF is variable, the FWHM is very small, or sky fitting
is enabled on the other hand, it may be necessary to increase the
fitting radius to achieve a good fit. (default: 1.)
fitsky : str (optional)
Compute new sky values for the stars in the input list (allstar)?
If fitsky = "no", the `allstar` task compute a group sky value by
averaging the sky values of the stars in the group.
If fitsky = "yes", the `allstar` task computes new sky values for
each star every third iteration by subtracting off the best
current fit for the star and and estimating the median of the
pixels in the annulus defined by sannulus and wsannulus.
The new group sky value is the average of the new individual
values. (default: 'no')
function : str (optional)
PSF model function. One of "auto", "gauss", "moffat15", "moffat25",
"lorentz", "penny1", or "penny2". (default: 'moffat25')
maxiter : int (optional)
The maximum number of times that the `allstar` task will iterate
on the PSF fit before giving up. (default: 50)
maxnpsf : int (optional)
The maximum number of candidate psf stars to be selected. (default: 60)
mergerad_fwhm : float (optional)
The critical separation in units `psffwhm` between two objects
for an object merger to be considered by the `allstar` task.
Objects with separations > mergerad will not be merged; faint
objects with separations <= mergerad will be considered for merging.
The default value of mergerad is sqrt (2 *(PAR1**2 + PAR2**2)),
where PAR1 and PAR2 are the half-width at half-maximum along the
major and minor axes of the psf model. Merging can be turned off
altogether by setting mergerad to 0.0. (default: 2.0)
nclean: int
The number of additional iterations the `psf` task performs to
compute the PSF look-up tables. If nclean is > 0, stars which
contribute deviant residuals to the PSF look-up tables in the
first iteration, will be down-weighted in succeeding iterations.
The DAOPHOT manual recommends 5 passes. (default: 10)
psfrad_fwhm : float
The radius of the circle in units `fwhmpsf` within which the PSF
model is defined (i.e. the PSF model radius in pixels is obtained
by multiplying `psfrad_fwhm` with `fwhmpsf`).
Psfrad_fwhm should be slightly larger than the radius at
which the intensity of the brightest star of interest fades into
the noise. Large values are computationally expensive, however.
Must be larger than `fitrad_fwhm` in any case. (default: 10.)
ratio : float (optional)
Ratio of minor to major axis of Gaussian kernel for object
detection (default: 1.0)
recenter : str (optional)
One of 'yes' or 'no'.
roundlo : float (optional)
Lower bound on roundness for object detection (default: -1.0)
roundhi : float (optional)
Upper bound on roundness for object detection (default: 1.0)
sannulus_fwhm : float (optional)
The inner radius of the sky annulus in units `fwhmpsf` used
by `allstar` to recompute the sky values. (default: 2.)
saturated : str (optional)
Use saturated stars to improve the signal-to-noise in the wings
of the PSF model computed by the PSF task? This parameter should
only be set to "yes" where there are too few high signal-to-noise
unsaturated stars in the image to compute a reasonable model
for the stellar profile wings. (default: "no")
sharplo : float (optional)
Lower bound on sharpness for object detection (default: 0.2)
sharphi : float (optional)
Upper bound on sharpness for object detection (default: 1.0)
theta : float (optional)
Position angle of major axis of Gaussian kernel for object
detection (default: 0.0)
threshold: float (optional)
Threshold in sigma for object detection (default: 3.0)
varorder : int (optional)
Variation of psf model: 0=constant, 1=linear, 2=cubic
varorder = -1 (analytic) gives very poor results
(though it may be more robust in crowded fields, in principle).
varorder 1 or 2 is possible marginally better than 0,
though it is not obvious in VPHAS data.
wsannulus_fwhm : float (optional)
The width of the sky annulus in units `fwhmpsf` used by `allstar`
to recompute the sky values. (default: 2.)
"""
# Ensure we start from the iraf defaults
for module in ['datapars', 'findpars', 'centerpars', 'fitskypars',
'photpars', 'daopars', 'daofind', 'phot']:
iraf.unlearn(module)
# Avoid the "Out of space in image header" exception
iraf.set(min_lenuserarea=640000)
# Allow overwriting files
iraf.set(clobber="yes")
# Set data-dependent IRAF/DAOPHOT configuration parameters
iraf.datapars.fwhmpsf = fwhmpsf # [pixels]
iraf.datapars.sigma = sigma # sigma(background) [ADU]
iraf.datapars.datamin = datamin # min good pix value [ADU]
iraf.datapars.datamax = datamax # max good pix value [ADU]
iraf.datapars.readnoi = readnoi # [electrons]
iraf.datapars.epadu = epadu # [electrons per ADU]
iraf.datapars.itime = itime # exposure time [seconds]
iraf.daofind.ratio = ratio # 2D Gaussian PSF fit ratio
iraf.daofind.theta = theta # 2D Gaussian PSF fit angle
iraf.photpars.aperture = fwhmpsf # Aperture radius
iraf.photpars.zmag = zmag # Magnitude zero point
# PSF parameters
iraf.daopars.psfrad = psfrad_fwhm * fwhmpsf # PSF radius
iraf.daopars.fitrad = fitrad_fwhm * fwhmpsf # PSF fitting radius
iraf.daopars.mergerad = mergerad_fwhm * fwhmpsf
# Setting a good sky annulus is important; a large annulus will ignore
# background changes on small scales, and will cause aperture photom
# in the wings of bright stars to be overestimated.
# => Inner radius of sky fitting annulus in scale units, default: 0.0
iraf.daopars.sannulus = sannulus_fwhm * fwhmpsf
# => Width of sky fitting annulus in scale units, default: 11.0
iraf.daopars.wsannulus = wsannulus_fwhm * fwhmpsf
# Allstar will re-measure the background in the PSF-subtracted image
# using this sky annulus -- change this if you see haloes around stars.
# The annulus should lie outside the psfrad to get the prettiest
# subtracted image, but putting the annulus far from the star might
# cause trouble in case of a spatially variable background!
iraf.fitskypars.annulus = annulus_fwhm * fwhmpsf # iraf.daopars.psfrad
iraf.fitskypars.dannulus = dannulus_fwhm * fwhmpsf # 2 * fwhmpsf
# Non data-dependent parameters
iraf.daopars.recenter = recenter
iraf.daopars.nclean = nclean
iraf.daopars.varorder = varorder
iraf.daopars.maxnstar = 5e4
iraf.daopars.fitsky = fitsky
iraf.daopars.saturated = saturated
iraf.daopars.maxiter = maxiter
iraf.daopars.function = function
# Object detection (daofind) parameters
iraf.findpars.threshold = threshold
iraf.findpars.sharplo = sharplo
iraf.findpars.sharphi = sharphi
iraf.findpars.roundlo = roundlo
iraf.findpars.roundhi = roundhi
# PSF fitting star selection
iraf.pstselect.maxnpsf = maxnpsf
def efoscfastredu(imglist, _listsens, _listarc, _ext_trace, _dispersionline,
_cosmic, _interactive):
# print "LOGX:: Entering `efoscfastredu` method/function in %(__file__)s"
# % globals()
import string
import os
import re
import sys
os.environ["PYRAF_BETA_STATUS"] = "1"
try:
from astropy.io import fits as pyfits
except:
import pyfits
from ntt.util import readhdr, readkey3
import ntt
import numpy as np
dv = ntt.dvex()
scal = np.pi / 180.
if not _interactive:
_interactive = False
_inter = 'NO'
else:
_inter = 'YES'
from pyraf import iraf
iraf.noao(_doprint=0, Stdout=0)
iraf.imred(_doprint=0, Stdout=0)
iraf.ccdred(_doprint=0, Stdout=0)
iraf.twodspec(_doprint=0, Stdout=0)
iraf.longslit(_doprint=0, Stdout=0)
iraf.onedspec(_doprint=0, Stdout=0)
iraf.specred(_doprint=0, Stdout=0)
toforget = [
'ccdproc', 'imcopy', 'specred.apall', 'longslit.identify',
'longslit.reidentify', 'specred.standard', 'longslit.fitcoords',
'onedspec.wspectext'
]
for t in toforget:
iraf.unlearn(t)
iraf.ccdred.verbose = 'no' # not print steps
iraf.specred.verbose = 'no' # not print steps
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.ccdtype = ''
_gain = ntt.util.readkey3(ntt.util.readhdr(imglist[0]), 'gain')
_ron = ntt.util.readkey3(ntt.util.readhdr(imglist[0]), 'ron')
iraf.specred.apall.readnoi = _ron
iraf.specred.apall.gain = _gain
iraf.specred.dispaxi = 2
iraf.longslit.dispaxi = 2
iraf.longslit.mode = 'h'
iraf.specred.mode = 'h'
iraf.noao.mode = 'h'
iraf.ccdred.instrument = "ccddb$kpno/camera.dat"
iraf.set(direc=ntt.__path__[0] + '/')
for img in imglist:
hdr = ntt.util.readhdr(img)
_tech = ntt.util.readkey3(hdr, 'tech')
if _tech != 'SPECTRUM':
sys.exit('error: ' + str(img) + ' is not a spectrum ')
print '\n#### image name = ' + img + '\n'
_grism0 = readkey3(hdr, 'grism')
_filter0 = readkey3(hdr, 'filter')
_slit0 = readkey3(hdr, 'slit')
_object0 = readkey3(hdr, 'object')
_date0 = readkey3(hdr, 'date-night')
setup = (_grism0, _filter0, _slit0)
_biassec0 = '[3:1010,1026:1029]'
if _grism0 == 'Gr16':
_trimsec0 = '[100:950,1:950]'
elif _grism0 == 'Gr13':
if _filter0 == 'Free':
_trimsec0 = '[100:950,1:1015]'
elif _filter0 == 'GG495':
_trimsec0 = '[100:950,208:1015]'
elif _filter0 == 'OG530':
_trimsec0 = '[100:950,300:1015]'
elif _grism0 == 'Gr11':
_trimsec0 = '[100:950,5:1015]'
else:
_trimsec0 = '[100:950,5:1015]'
_object0 = re.sub(' ', '', _object0)
_object0 = re.sub('/', '_', _object0)
nameout0 = 't' + str(_object0) + '_' + str(_date0)
for _set in setup:
nameout0 = nameout0 + '_' + _set
nameout0 = ntt.util.name_duplicate(img, nameout0, '')
timg = nameout0
if os.path.isfile(timg):
os.system('rm -rf ' + timg)
iraf.imcopy(img, output=timg)
iraf.ccdproc(timg,
output='',
overscan='no',
trim='yes',
zerocor="no",
flatcor="no",
readaxi='column',
trimsec=str(_trimsec0),
biassec=_biassec0,
Stdout=1)
img = timg
if _listarc:
arcfile = ntt.util.searcharc(img, _listarc)[0]
else:
arcfile = ''
if not arcfile:
arcfile = ntt.util.searcharc(img, '')[0]
else:
iraf.ccdproc(arcfile,
output='t' + arcfile,
overscan='no',
trim='yes',
zerocor="no",
flatcor="no",
readaxi='column',
trimsec=str(_trimsec0),
biassec=str(_biassec0),
Stdout=1)
arcfile = 't' + arcfile
if _cosmic:
# print cosmic rays rejection
ntt.cosmics.lacos(img,
output='',
gain=_gain,
readn=_ron,
xorder=9,
yorder=9,
sigclip=4.5,
sigfrac=0.5,
objlim=1,
verbose=True,
interactive=False)
print '\n### cosmic rays rejections ........ done '
if not arcfile:
print '\n### warning no arcfile \n exit '
else:
arcref = ntt.util.searcharc(img, '')[0]
if arcfile[0] == '/':
os.system('cp ' + arcfile + ' ' +
string.split(arcfile, '/')[-1])
arcfile = string.split(arcfile, '/')[-1]
arcref = string.split(arcref, '/')[-1]
if arcref:
os.system('cp ' + arcref + ' .')
arcref = string.split(arcref, '/')[-1]
if not os.path.isdir('database/'):
os.mkdir('database/')
if os.path.isfile(
ntt.util.searcharc(img, '')[1] + '/database/id' +
re.sub('.fits', '', arcref)):
os.system('cp ' + ntt.util.searcharc(img, '')[1] +
'/database/id' + re.sub('.fits', '', arcref) +
' database/')
iraf.longslit.reidentify(
referenc=arcref,
images=arcfile,
interac=_inter,
section='column 10',
coordli='direc$standard/ident/Lines_HgCdHeNeAr600.dat',
overrid='yes',
step=0,
newaps='no',
nsum=5,
nlost=2,
mode='h',
verbose='no')
else:
iraf.longslit.identify(
images=arcfile,
section='column 10',
coordli='direc$standard/ident/Lines_HgCdHeNeAr600.dat',
nsum=10,
fwidth=7,
order=3,
mode='h')
iraf.longslit.reident(
referenc=arcfile,
images=arcfile,
interac='NO',
section='column 10',
coordli='direc$standard/ident/Lines_HgCdHeNeAr600.dat',
overrid='yes',
step=10,
newaps='yes',
nsum=5,
nlost=2,
mode='h',
verbose='no')
qqq = iraf.longslit.fitcoords(images=re.sub('.fits', '', arcfile),
fitname=re.sub('.fits', '', arcfile),
interac='no',
combine='yes',
databas='database',
function='legendre',
yorder=4,
logfile='logfile',
plotfil='',
mode='h')
iraf.specred.transform(input=img,
output=img,
minput='',
fitnames=re.sub('.fits', '', arcfile),
databas='database',
x1='INDEF',
x2='INDEF',
y1='INDEF',
y2='INDEF',
flux='yes',
mode='h',
logfile='logfile')
# ###################### check wavelength calibration ############
_skyfile = ntt.__path__[0] + '/standard/ident/sky_' + setup[
0] + '_' + setup[1] + '.fits'
shift = ntt.efoscspec2Ddef.skyfrom2d(img, _skyfile)
print '\n### check in wavelengh performed ...... spectrum shifted of ' + str(
shift) + ' Angstrom \n'
zro = pyfits.open(img)[0].header.get('CRVAL2')
ntt.util.updateheader(img, 0, {'CRVAL2': [zro + int(shift), '']})
std, rastd, decstd, magstd = ntt.util.readstandard(
'standard_efosc_mab.txt')
hdrt = readhdr(img)
_ra = readkey3(hdrt, 'RA')
_dec = readkey3(hdrt, 'DEC')
_object = readkey3(hdrt, 'object')
dd = np.arccos(
np.sin(_dec * scal) * np.sin(decstd * scal) +
np.cos(_dec * scal) * np.cos(decstd * scal) * np.cos(
(_ra - rastd) * scal)) * ((180 / np.pi) * 3600)
if min(dd) < 100:
_type = 'stdsens'
ntt.util.updateheader(img, 0,
{'stdname': [std[np.argmin(dd)], '']})
ntt.util.updateheader(
img, 0, {'magstd': [float(magstd[np.argmin(dd)]), '']})
else:
_type = 'obj'
print '\n### EXTRACTION USING IRAF TASK APALL \n'
result = []
if _type == 'obj':
imgex = ntt.util.extractspectrum(img, dv, _ext_trace,
_dispersionline, _interactive,
_type)
ntt.util.updateheader(
imgex, 0, {'FILETYPE': [22107, 'extracted 1D spectrum ']})
ntt.util.updateheader(
imgex, 0, {
'PRODCATG': [
'SCIENCE.' +
readkey3(readhdr(imgex), 'tech').upper(),
'Data product category'
]
})
ntt.util.updateheader(imgex, 0, {'TRACE1': [img, '']})
result.append(imgex)
if _listsens:
sensfile = ntt.util.searchsens(img, _listsens)[0]
else:
sensfile = ''
if not sensfile:
sensfile = ntt.util.searchsens(img, '')[0]
if sensfile:
imgf = re.sub('.fits', '_f.fits', img)
_extinctdir = 'direc$standard/extinction/'
_extinction = 'extinction_lasilla.dat'
_observatory = 'lasilla'
_exptime = readkey3(hdrt, 'exptime')
_airmass = readkey3(hdrt, 'airmass')
ntt.util.delete(imgf)
iraf.specred.calibrate(input=imgex,
output=imgf,
sensiti=sensfile,
extinct='yes',
flux='yes',
ignorea='yes',
extinction=_extinctdir +
_extinction,
observatory=_observatory,
airmass=_airmass,
exptime=_exptime,
fnu='no')
hedvec = {
'SENSFUN': [
string.split(sensfile, '/')[-1],
'sensitivity function'
],
'FILETYPE':
[22208, '1D wavelength and flux calibrated spectrum '],
'SNR':
[ntt.util.StoN2(imgf, False), 'Average S/N ratio'],
'BUNIT':
['erg/cm2/s/Angstrom', 'Flux Calibration Units'],
'TRACE1': [imgex, '']
}
ntt.util.updateheader(imgf, 0, hedvec)
imgout = imgf
imgd = ntt.efoscspec1Ddef.fluxcalib2d(img, sensfile)
ntt.util.updateheader(
imgd, 0, {
'FILETYPE': [
22209,
'2D wavelength and flux calibrated spectrum '
]
})
ntt.util.updateheader(imgd, 0, {'TRACE1': [img, '']})
imgasci = re.sub('.fits', '.asci', imgout)
ntt.util.delete(imgasci)
iraf.onedspec.wspectext(imgout + '[*,1,1]',
imgasci,
header='no')
result = result + [imgout, imgd, imgasci]
else:
imgex = ntt.util.extractspectrum(img, dv, _ext_trace,
_dispersionline, _interactive,
'std')
imgout = ntt.efoscspec1Ddef.sensfunction(
imgex, 'spline3', 6, _inter)
result = result + [imgout]
for img in result:
if img[-5:] == '.fits':
ntt.util.phase3header(img) # phase 3 definitions
ntt.util.airmass(img) # phase 3 definitions
ntt.util.updateheader(
img, 0, {'quality': ['Rapid', 'Final or Rapid reduction']})
return result
from glob import glob
from pyraf import iraf
iraf.stsdas()
iraf.hst_calib()
iraf.stis()
import pyfits
from numpy import isnan, min, logical_and, bitwise_and, linspace, median, arange, correlate, exp, average
from scipy.interpolate import interp1d
from scipy.integrate import trapz
import os
from scipy.optimize import curve_fit
import stistools
iraf.twodspec()
iraf.apextract()
iraf.set(clobber='yes')
def gauss(x, a, x0, sigma, sky):
return a * exp(-(x - x0)**2 / (2 * sigma**2) + sky)
def tofits(filename, data, hdr=None, clobber=False):
"""simple pyfits wrapper to make saving fits files easier."""
from pyfits import PrimaryHDU, HDUList
hdu = PrimaryHDU(data)
if hdr is not None:
hdu.header = hdr
hdulist = HDUList([hdu])
hdulist.writeto(filename, clobber=clobber, output_verify='ignore')
global iraf
from pyraf import iraf
import numpy as np
import pyfits
from glob import glob
import os
iraf.pysalt()
iraf.saltspec()
iraf.saltred()
iraf.set(clobber='YES')
iraf.noao()
iraf.twodspec()
iraf.longslit()
def tofits(filename, data, hdr=None, clobber=False):
"""simple pyfits wrapper to make saving fits files easier."""
from pyfits import PrimaryHDU, HDUList
hdu = PrimaryHDU(data)
if hdr is not None:
hdu.header = hdr
hdulist = HDUList([hdu])
hdulist.writeto(filename, clobber=clobber, output_verify='ignore')
def get_ims(fs, imtype):
imtypekeys = {'sci': 'OBJECT', 'arc': 'ARC', 'flat': 'FLAT'}
ims = []
grangles = []
for f in fs:
# within ds9. They show the background image with masked regions superposed. # These are useful for testing whether stellar outskirts are sufficiently masked #Author: Ryan Muther, Union College, June-July 2013 #This program was written as a replacement for the IDL program of the same #name, originally written by Cullen Blake 1999. # Updated May 8, 2014 by Rebecca Koopmann: # - added documentation from pyraf import iraf from astropy.io import fits import numpy as np import sys # set stdimage to 2048 iraf.set(stdimage='imt2048') # Load the bad pixel mask and reverse its values so the mask exists in a sensible manner. # Currently, the zeroes represent good pixels and the ones represent masked pixels. # This is the IRAF default, but less useful than the reverse when manipulating the masks # and images. This reversal process allows for the creation of a masked image by simple # elementwise multiplication of the mask and the image to be masked def reverseMask(filename): mask = fits.getdata(filename+"mask.fits") manyZeros = np.zeros_like(mask) manyOnes = np.ones_like(mask) maskReversed = np.where((mask!=0),manyZeros,manyOnes)
iraf.imred(_doprint=0, Stdout="/dev/null")
iraf.ccdred(_doprint=0, Stdout="/dev/null")
iraf.images(_doprint=0, Stdout="/dev/null")
iraf.immatch(_doprint=0, Stdout="/dev/null")
iraf.onedspec(_doprint=0, Stdout="/dev/null")
iraf.twodspec(_doprint=0, Stdout="/dev/null")
iraf.apextract(_doprint=0, Stdout="/dev/null")
iraf.imutil(_doprint=0, Stdout="/dev/null")
iraf.echelle(_doprint=0, Stdout="/dev/null")
iraf.astutil(_doprint=0, Stdout="/dev/null")
iraf.apextract.dispaxi = 1
iraf.echelle.dispaxi = 1
iraf.ccdred.instrum = 'blank.txt'
os.environ['PYRAF_BETA_STATUS'] = '1'
os.system('mkdir uparm')
iraf.set(uparm=os.getcwd() + '/uparm')
data_dir = '/data4/travegre/Projects/Asiago_binaries/'
reduc_dir = '/data4/travegre/Projects/Echelle_Asiago_Reduction/delo/observations/'
obs_metadata = Table.read(data_dir + 'star_data_all.csv')
obs_metadata = obs_metadata[obs_metadata['odkdaj'] == 'nova']
_go_to_dir('Binaries_spectra')
copyfile(data_dir + 'star_data_all.csv', 'star_data_all.csv')
stars = [
'TV_LMi', 'GZ_Dra', 'V455_Aur', 'GK_Dra', 'DT_Cam', 'V394_Vul', 'CI_CVn',
'V1898_Cyg', 'V417_Aur', 'EQ_Boo', 'V994_Her', 'CN_Lyn', 'DV_Cam'
][6:15]
def phot(fits_filename, x_in, y_in, aperture=15, sky=20, swidth=10, apcor=0.3,
maxcount=30000.0, exptime=1.0):
"""
Compute the centroids and magnitudes of a bunch sources detected on
CFHT-MEGAPRIME images.
Args:
fits_filename: str
The name of the file containing the image to be processed.
Returns a MOPfiles data structure.
"""
if (not os.path.exists(fits_filename) and
not fits_filename.endswith(".fits")):
# For convenience, see if we just forgot to provide the extension
fits_filename += ".fits"
try:
input_hdulist = fits.open(fits_filename)
except Exception as err:
raise TaskError("Failed to open input image: %s" % err.message)
## get the filter for this image
filter = input_hdulist[0].header.get('FILTER', 'DEFAULT')
### Some CFHT zeropoints that might be useful
zeropoints = {"I": 25.77,
"R": 26.07,
"V": 26.07,
"B": 25.92,
"DEFAULT": 26.0,
"g.MP9401": 26.4
}
### load the
if not filter in zeropoints:
filter = "DEFAULT"
zmag = input_hdulist[0].header.get('PHOTZP', zeropoints[filter])
### setup IRAF to do the magnitude/centroid measurements
iraf.set(uparm="./")
iraf.digiphot()
iraf.apphot()
iraf.daophot(_doprint=0)
### check for the magical 'zeropoint.used' file
zpu_file = "zeropoint.used"
if os.access(zpu_file, os.R_OK):
with open(zpu_file) as zpu_fh:
zmag = float(zpu_fh.read())
iraf.photpars.apertures = int(aperture)
iraf.photpars.zmag = zmag
iraf.datapars.datamin = 0
iraf.datapars.datamax = maxcount
#iraf.datapars.exposur="EXPTIME"
iraf.datapars.exposur = ""
iraf.datapars.itime = exptime
iraf.fitskypars.annulus = sky
iraf.fitskypars.dannulus = swidth
iraf.centerpars.calgori = "centroid"
iraf.centerpars.cbox = 5.
iraf.centerpars.cthreshold = 0.
iraf.centerpars.maxshift = 2.
iraf.centerpars.clean = 'no'
iraf.phot.update = 'no'
iraf.phot.verbose = 'no'
iraf.phot.verify = 'no'
iraf.phot.interactive = 'no'
# Used for passing the input coordinates
coofile = tempfile.NamedTemporaryFile(suffix=".coo", delete=False)
coofile.write("%f %f \n" % (x_in, y_in))
# Used for receiving the results of the task
# mag_fd, mag_path = tempfile.mkstemp(suffix=".mag")
magfile = tempfile.NamedTemporaryFile(suffix=".mag", delete=False)
# Close the temp files before sending to IRAF due to docstring:
# "Whether the name can be used to open the file a second time, while
# the named temporary file is still open, varies across platforms"
coofile.close()
magfile.close()
os.remove(magfile.name)
iraf.phot(fits_filename, coofile.name, magfile.name)
pdump_out = iraf.pdump(magfile.name, "XCENTER,YCENTER,MAG,MERR,ID,XSHIFT,YSHIFT,LID",
"MERR < 0.4 && MERR != INDEF && MAG != INDEF && PIER==0", header='no', parameters='yes',
Stdout=1)
os.remove(coofile.name)
os.remove(magfile.name)
### setup the mop output file structure
hdu = {}
hdu['header'] = {'image': input_hdulist,
'aper': aperture,
's_aper': sky,
'd_s_aper': swidth,
'aper_cor': apcor,
'zeropoint': zmag}
hdu['order'] = ['X', 'Y', 'MAG', 'MERR', 'ID', 'XSHIFT', 'YSHIFT', 'LID']
hdu['format'] = {'X': '%10.2f',
'Y': '%10.2f',
'MAG': '%10.2f',
'MERR': '%10.2f',
'ID': '%8d',
'XSHIFT': '%10.2f',
'YSHIFT': '%10.2f',
'LID': '%8d'}
hdu['data'] = {}
for col in hdu['order']:
hdu['data'][col] = []
for line in pdump_out:
values = line.split()
for col in hdu['order']:
if re.match('\%.*f', hdu['format'][col]):
if col == 'MAG':
values[0] = float(values[0]) - float(apcor)
hdu['data'][col].append(float(values.pop(0)))
elif re.match('\%.*d', hdu['format'][col]):
hdu['data'][col].append(int(values.pop(0)))
else:
hdu['data'][col].append(values.pop(0))
# Clean up temporary files generated by IRAF
os.remove("datistabe.par")
os.remove("datpdump.par")
return hdu
if os.path.exists(img + '.sn2.fits'):
hdr2 = agnkey.util.readhdr(img + '.sn2.fits')
snmag1 = agnkey.util.readkey3(hdr2, 'PSFMAG1')
if snmag1 and not redo: skip = 1
else:
sys.exit('psf not computed')
print psfimage
if not os.path.exists(psfimage + '.fits'):
sys.exit('missing psf file')
if redo:
skip = 0
####################################################################################
if skip == 0:
iraf.set(stdimage='imt2048')
if _interactive:
_z1, _z2, goon = agnkey.util.display_image(img + '.fits',
1,
'',
'',
True,
_xsize=1,
_ysize=1)
elif _show:
_z1, _z2, goon = agnkey.util.display_image(
img + '.fits', 1, '', '', False)
apco0 = agnkey.util.readkey3(hdr2, 'APCO')
if 'PIXSCALE' in hdr2:
pixelscale = agnkey.util.readkey3(hdr2, 'PIXSCALE')
def dark_sky_flat(filter):
# from cv2 import medianBlur
med_otalist = []
print 'making dark sky flats for', filter
for key in tqdm(odi.OTA_dictionary):
image_list = odi.OTA_dictionary[key] + '.' + filter + '.lis'
med_out = image_list.replace('.lis', '.med.fits')
med_otalist.append(med_out)
iraf.unlearn(iraf.immatch.imcombine)
iraf.immatch.imcombine.setParam('input', '@' + str(image_list))
iraf.immatch.imcombine.setParam('output', odi.skyflatpath + med_out)
iraf.immatch.imcombine.setParam('combine', 'median')
iraf.immatch.imcombine.setParam('masktype', 'goodvalue')
iraf.immatch.imcombine.setParam('maskvalue', 0)
iraf.immatch.imcombine.setParam('scale', 'median')
iraf.immatch.imcombine.setParam('zero', 'none')
# iraf.immatch.imcombine.setParam('zero','median')
iraf.immatch.imcombine(logfile='imcombine.log.txt', mode='h')
iraf.set(clobber='yes')
print 'smoothing dark sky flats for', filter
for key in tqdm(odi.OTA_dictionary):
image_list = odi.OTA_dictionary[key] + '.' + filter + '.lis'
med_out = image_list.replace('.lis', '.med.fits')
# don't overwrite the original flat
med_smooth = image_list.replace('.lis', '.med.smooth.fits')
# smooth the flat with a 50 x 50 median box
iraf.unlearn(iraf.imutil.imarith, iraf.imfilter.median)
iraf.imfilter.fmedian.setParam('input', odi.skyflatpath + med_out)
iraf.imfilter.fmedian.setParam('output',
repr(key) + 'temp_smooth.fits')
iraf.imfilter.fmedian.setParam('xwindow', 51)
iraf.imfilter.fmedian.setParam('ywindow', 51)
iraf.imfilter.fmedian.setParam(
'zloreject',
1.0) # ignore 0.0s in the smoothing, they'll cause image artifacts
iraf.imfilter.fmedian(verbose='no', mode='h')
# smoothing is not leaving zeros but instead some very small number--replace them with 0.0s
iraf.imutil.imexpr('(a < 1.51) ? 0 : a',
odi.skyflatpath + med_smooth,
repr(key) + 'temp_smooth.fits',
verbose='no')
iraf.imutil.imdelete(repr(key) + 'temp_smooth.fits')
# determine the normalization factor from the _smoothed_ image
if key == 1:
data, header = odi.fits.getdata(odi.skyflatpath + med_smooth,
header=True)
mean, median, std = odi.sigma_clipped_stats(
data, sigma=3.0,
mask_value=0.0) # be sure to ignore 0.0s in the flat
normalization_factor = median
print normalization_factor
# smoothing using numpy, this method is much slower
# data, header = odi.fits.getdata(odi.skyflatpath+med_out, header=True)
# # data_gaps = data.astype(bool)
# data_smoothed = medianBlur(data, ksize=51)
# plt.imshow(data_smoothed, interpolation='nearest')
# plt.show()
# divide the smoothed image by the normalization factor
iraf.imutil.imarith.setParam('operand1', odi.skyflatpath + med_smooth)
iraf.imutil.imarith.setParam('op', '/')
iraf.imutil.imarith.setParam('operand2', normalization_factor)
iraf.imutil.imarith.setParam('result', odi.skyflatpath + med_smooth)
iraf.imutil.imarith(verbose='no', mode='h')
# make an image of 0s and 1s to restore the cell gaps
# iraf.imutil.imexpr('(a == 0) ? 0 : 1', odi.skyflatpath+'temp_gaps', odi.skyflatpath+med_smooth, verbose='no')
# iraf.imutil.imarith.setParam('operand1',odi.skyflatpath+med_smooth)
# iraf.imutil.imarith.setParam('op','*')
# iraf.imutil.imarith.setParam('operand2',odi.skyflatpath+'temp_gaps')
# iraf.imutil.imarith.setParam('result',odi.skyflatpath+med_smooth)
# iraf.imutil.imarith(verbose='no', mode='h')
iraf.imutil.imdelete('temp_gaps', mode='h')
iraf.set(clobber='no')
return normalization_factor
def absphot(img,_field,_catalogue,_fix,_color,rejection,_interactive,_type='fit',redo=False,show=False,cutmag=-1,database='dataredulco',_calib='sloan'):
from astropy.io import fits
import lsc
import math
import sys,re,string,os
from lsc.util import readkey3, readhdr
from numpy import array, compress, zeros, median, std, asarray, isfinite,mean
from pyraf import iraf
if show:
from pylab import ion,plot,draw,clf
import time
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.daophot(_doprint=0)
iraf.images(_doprint=0)
iraf.imcoords(_doprint=0)
iraf.proto(_doprint=0)
t = fits.open(img)
tbdata = t[1].data
hdr2=t[1].header
hdr=lsc.util.readhdr(img)
_cat=readkey3(hdr,'catalog')
_telescope=lsc.util.readkey3(hdr,'telescop')
_instrume=lsc.util.readkey3(hdr,'instrume')
_filter=lsc.util.readkey3(hdr,'filter')
_airmass=lsc.util.readkey3(hdr,'airmass')
_exptime=lsc.util.readkey3(hdr,'exptime')
_date=lsc.util.readkey3(hdr,'date-obs')
_object=lsc.util.readkey3(hdr,'object')
_ra=lsc.util.readkey3(hdr,'RA')
_dec=lsc.util.readkey3(hdr,'DEC')
print _filter
if _telescope in ['lsc','1m0-04','1m0-05','1m0-09']: kk=lsc.sites.extintion('ctio')
elif _telescope in ['elp','1m0-08']: kk=lsc.sites.extintion('mcdonald')
elif _telescope in ['cpt','1m0-12','1m0-10','1m0-13']: kk=lsc.sites.extintion('southafrica')
elif _telescope in ['ftn','Faulkes Telescope North']: kk=lsc.sites.extintion('mauna')
elif _telescope in ['1m0-03','1m0-11','coj','fts','Faulkes Telescope South']: kk=lsc.sites.extintion('siding')
if _calib not in ['sloan','sloanprime','natural','apass','']: colorefisso=lsc.sites.colfix(_instrume)
else: colorefisso=lsc.sites.colfix(_instrume,_calib)
print redo
print _cat
if _cat and not redo:
print 'already calibrated'
else:
try:
lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]):
lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
except: print 'module mysqldef not found'
column=makecatalogue([img])[_filter][img]
rasex=array(column['ra0'],float)
decsex=array(column['dec0'],float)
if _type=='fit':
magsex=array(column['smagf'],float)
magerrsex=array(column['smagerrf'],float)
elif _type=='ph':
magsex=array(column['magp3'],float)
magerrsex=array(column['merrp3'],float)
else: sys.exit(_type+' not valid (ph or fit)')
print len(rasex)
if not cutmag: cutmag=99
#else: cutmag= cutmag-2.5*math.log10(float(_exptime))
if len(compress( array(magsex) < float(cutmag) , magsex)) < 5 : cutmag=99 # not cut if only few object
rasex = compress(array(magsex,float)<=cutmag,rasex)
decsex = compress(array(magsex,float)<=cutmag,decsex)
magerrsex = compress(array(magsex,float)<=cutmag,magerrsex)
magsex = compress(array(magsex,float)<=cutmag,array(magsex))
print len(rasex)
if _interactive:
iraf.set(stdimage='imt1024')
iraf.display(re.sub('.sn2','',img),1,fill=True,Stdout=1)
vector=[]
for i in range(0,len(rasex)):
vector.append(str(rasex[i])+' '+str(decsex[i]))
xy = iraf.wcsctran('STDIN',output="STDOUT",Stdin=vector,Stdout=1,image=img,inwcs='world',units='degrees degrees',outwcs='logical',\
formats='%10.1f %10.1f',verbose='yes')[3:]
iraf.tvmark(1,'STDIN',Stdin=list(xy),mark="circle",number='yes',label='no',radii=10,nxoffse=5,nyoffse=5,color=207,txsize=2)
# raw_input('here')
if _catalogue:
######## use external catalogue
if _catalogue[0]=='/': stdcooC=lsc.lscastrodef.readtxt(_catalogue)
else: stdcooC=lsc.lscastrodef.readtxt(lsc.__path__[0]+'/standard/cat/'+_catalogue)
rastdC,decstdL=array(stdcooC['ra'],float),array(stdcooC['dec'],float)
lsc.util.delete('tmp.stdL.pix')
colonne=str(stdcooC['rapos'])+' '+str(stdcooC['decpos'])
if _catalogue[0]=='/':
iraf.wcsctran(_catalogue,'tmp.stdL.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',\
columns=colonne,formats='%10.1f %10.1f',verbose='no')
else:
iraf.wcsctran(lsc.__path__[0]+'/standard/cat/'+_catalogue,'tmp.stdL.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',\
columns=colonne,formats='%10.1f %10.1f',verbose='no')
standardpixC=lsc.lscastrodef.readtxt('tmp.stdL.pix')
if _interactive:
iraf.tvmark(1,'tmp.stdL.pix',mark="circle",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=204,txsize=2)
xstdC=standardpixC['ra']
ystdC=standardpixC['dec']
idstdC=standardpixC['id']
xstdC=compress((array(xstdC,float)<readkey3(hdr,'XDIM'))&(array(xstdC,float)>0)&(array(ystdC,float)>0)&(array(ystdC,float)<readkey3(hdr,'YDIM')),xstdC)
xstdL=xstdLL=xstdS=xstdC
standardpixL=standardpixLL=standardpixS=standardpixC
stdcooL=stdcooLL=stdcooS=stdcooC
else:
######## check if it is landolt field
stdcooL=lsc.lscastrodef.readtxt(lsc.__path__[0]+'/standard/cat/landolt.cat')
rastdL,decstdL=array(stdcooL['ra'],float),array(stdcooL['dec'],float)
lsc.util.delete('tmp.stdL.pix')
iraf.wcsctran(lsc.__path__[0]+'/standard/cat/landolt.cat','tmp.stdL.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',\
columns='1 2',formats='%10.1f %10.1f',verbose='no')
standardpixL=lsc.lscastrodef.readtxt('tmp.stdL.pix')
if _interactive:
iraf.tvmark(1,'tmp.stdL.pix',mark="circle",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=204,txsize=2)
print 'yelow circles sextractor'
xstdL=standardpixL['ra']
ystdL=standardpixL['dec']
idstdL=standardpixL['id']
xstdL=compress((array(xstdL,float)<readkey3(hdr,'XDIM'))&(array(xstdL,float)>0)&(array(ystdL,float)>0)&(array(ystdL,float)<readkey3(hdr,'YDIM')),xstdL)
######## check if it is Stetson field
stdcooLL=lsc.lscastrodef.readtxt(lsc.__path__[0]+'/standard/cat/StetsonCat.dat')
ww=asarray([i for i in range(len(stdcooLL['ra'])) if ( abs(float(stdcooLL['ra'][i])-float(_ra))<.2 and abs(float(stdcooLL['dec'][i])-_dec)<.2 )])
if len(ww)>0:
for hh in stdcooLL.keys():
if type(stdcooLL[hh])!=int:
if hh not in ['id','ra','dec']:
stdcooLL[hh]=array(array(stdcooLL[hh])[ww],float)
else:
stdcooLL[hh]=array(stdcooLL[hh])[ww]
lll=[]
for i in range(0,len(stdcooLL['ra'])): lll.append(stdcooLL['ra'][i]+' '+stdcooLL['dec'][i])
rastdLL,decstdLL=array(stdcooLL['ra'],float),array(stdcooLL['dec'],float)
lsc.util.delete('tmp.stdLL.pix')
iraf.wcsctran('STDIN','tmp.stdLL.pix',img,inwcs='world',Stdin=lll,units='degrees degrees',outwcs='logical',\
columns='1 2',formats='%10.1f %10.1f',verbose='no')
if _interactive:
iraf.tvmark(1,'tmp.stdLL.pix',mark="cross",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=204,txsize=2)
print 'red crosses Stetson'
standardpixLL={}
for ii in stdcooLL.keys(): standardpixLL[ii]=stdcooLL[ii]
standardpixLL['ra']=array(iraf.proto.fields('tmp.stdLL.pix',fields='1',Stdout=1),float) #standardpixLL['ra']
standardpixLL['dec']=array(iraf.proto.fields('tmp.stdLL.pix',fields='2',Stdout=1),float) #standardpixLL['dec']
xstdLL=array(iraf.proto.fields('tmp.stdLL.pix',fields='1',Stdout=1),float) #standardpixLL['ra']
ystdLL=array(iraf.proto.fields('tmp.stdLL.pix',fields='2',Stdout=1),float) #standardpixLL['dec']
idstdLL=standardpixLL['id']
xstdLL=compress((array(xstdLL,float)<readkey3(hdr,'XDIM'))&(array(xstdLL,float)>0)&(array(ystdLL,float)>0)&(array(ystdLL,float)<readkey3(hdr,'YDIM')),xstdLL)
######## check if it is sloan field
magsel0,magsel1=12,18
_ids=lsc.lscastrodef.sloan2file(_ra,_dec,20,float(magsel0),float(magsel1),'_tmpsloan.cat')
ascifile='_tmpsloan.cat'
stdcooS=lsc.lscastrodef.readtxt(ascifile)
rastdS,decstdS=array(stdcooS['ra'],float),array(stdcooS['dec'],float)
lsc.util.delete('tmp.stdS.pix')
iraf.wcsctran(ascifile,'tmp.stdS.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',columns='1 2',formats='%10.1f %10.1f',verbose='no')
standardpixS=lsc.lscastrodef.readtxt('tmp.stdS.pix')
if _interactive:
iraf.tvmark(1,'tmp.stdS.pix',mark="cross",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=205,txsize=2)
print 'green cross sloan'
xstdS=standardpixS['ra']
ystdS=standardpixS['dec']
idstdS=standardpixS['id']
xstdS=compress((array(xstdS,float)<readkey3(hdr,'XDIM'))&(array(xstdS,float)>0)&(array(ystdS,float)>0)&(array(ystdS,float)<readkey3(hdr,'YDIM')),xstdS)
##############################################3
if not _catalogue and len(xstdLL)>0:
xstdL=xstdLL
standardpixL=standardpixLL
stdcooL=stdcooLL
if _filter in ['U', 'B', 'V', 'R','I','Bessell-B','Bessell-V','Bessell-R','Bessell-I']:
filters={'U':'U', 'B':'B', 'V':'V', 'R':'R', 'I':'I','Bessell-B':'B','Bessell-V':'V','Bessell-R':'R','Bessell-I':'I'}
if _color:
colors=lsc.myloopdef.chosecolor(_color,False)
if not colors: colors={'U':['UB'],'B':['UB','BV'],'V':['BV','VR'],'R':['VR','RI'],'I':['RI']}
else:
colors={'U':['UB'],'B':['UB','BV'],'V':['BV','VR'],'R':['VR','RI'],'I':['RI']}
if _field=='sloan':
standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]}
print 'filters and field selected do not match'
else:
_field='landolt'
if len(xstdL)>=1:
standardpix=standardpixL
stdcoo=stdcooL
if not _catalogue:
if len(xstdLL)>0: _catalogue='StetsonCat.dat'
else: _catalogue='landolt.dat'
elif len(xstdS)>=1:
if not _catalogue: _catalogue='sdss8'
standardpix=standardpixS
stdcoo=stdcooS
stdcoo=lsc.lscastrodef.transformsloanlandolt(stdcoo)
if not _catalogue: _catalogue='sdss8'
print '\n### transform sloan in landolt'
else:
print 'landolt, but catalogue not found'
standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]}
elif _filter in ['up','gp','rp','ip','zs','SDSS-G','SDSS-R','SDSS-I','Pan-Starrs-Z']:
filters={'up':'u','ip':'i','gp':'g','rp':'r','zs':'z','SDSS-G':'g','SDSS-R':'r','SDSS-I':'i','Pan-Starrs-Z':'z'}
if _color:
colors=lsc.myloopdef.chosecolor(_color,False)
if not colors: colors={'i':['ri','iz'],'r':['gr','ri'],'g':['ug','gr'],'z':['iz'],'u':['ug']}
else:
colors={'i':['ri','iz'],'r':['gr','ri'],'g':['ug','gr'],'z':['iz'],'u':['ug']}
if _field=='landolt':
standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]}
print 'filters and field selected do not match'
else:
_field='sloan'
if len(xstdS)>=1:
if not _catalogue: _catalogue='sdss8'
standardpix=standardpixS
stdcoo=stdcooS
elif len(xstdL)>=1:
standardpix=standardpixL
stdcoo=stdcooL
stdcoo=lsc.lscastrodef.transformlandoltsloan(stdcoo)
if not _catalogue: _catalogue='landolt.dat'
print '\n### transform landolt to sloan'
else:
print 'sloan, but not in the sdss footprint'
standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]}
xstd=standardpix['ra']
ystd=standardpix['dec']
idstd=standardpix['id']
rastd,decstd=array(stdcoo['ra'],float),array(stdcoo['dec'],float)
xstd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)&(array(ystd,float)<readkey3(hdr,'YDIM')),xstd)
if len(xstd0)>1: ######## go only if standard stars are in the field ##########
magstd0={}
airmass0={}
result={}
fileph={}
print '\n### standard field: '+str(_field)
ystd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\
&(array(ystd,float)<readkey3(hdr,'YDIM')),ystd)
rastd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\
&(array(ystd,float)<readkey3(hdr,'YDIM')),rastd)
decstd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\
&(array(ystd,float)<readkey3(hdr,'YDIM')),decstd)
idstd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\
&(array(ystd,float)<readkey3(hdr,'YDIM')),idstd)
stdcoo0={}
for key in stdcoo.keys():
if key in 'ugrizUBVRI':
stdcoo0[key]=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\
&(array(ystd,float)<readkey3(hdr,'YDIM')),stdcoo[key])
###############################################################
# pos0 = standard pos1 = sextractor
distvec,pos0,pos1=lsc.lscastrodef.crossmatch(array(rastd0),array(decstd0),array(rasex),array(decsex),10)
for key in stdcoo0.keys(): stdcoo0[key]=stdcoo0[key][pos0]
rastd0=rastd0[pos0]
decstd0=decstd0[pos0]
idstd0=idstd0[pos0]
rasex=rasex[pos1]
decsex=decsex[pos1]
# after change in may 2013 mag in sn2.fits file are already at 1s
magsex=magsex[pos1]-kk[filters[_filter]]*float(_airmass) # - K x airmass
# magsex=magsex[pos1]+2.5*math.log10(float(_exptime))-kk[filters[_filter]]*float(_airmass) # mag exptime - K x airmass
#################################################################################
if _field=='landolt':
print '\n### landolt system'
for _filtlandolt in 'UBVRI':
if _filtlandolt==filters[_filter]: airmass0[_filtlandolt]= 0 #_airmass
else: airmass0[_filtlandolt]= 0
magstd0[_filtlandolt]=stdcoo0[_filtlandolt]
fileph['mU']=zeros(len(rastd0))+999
fileph['mB']=zeros(len(rastd0))+999
fileph['mV']=zeros(len(rastd0))+999
fileph['mR']=zeros(len(rastd0))+999
fileph['mI']=zeros(len(rastd0))+999
fileph['V']=magstd0['V']
fileph['BV']=array(array(magstd0['B'],float)-array(magstd0['V'],float),str)
fileph['UB']=array(array(magstd0['U'],float)-array(magstd0['B'],float),str)
fileph['VR']=array(array(magstd0['V'],float)-array(magstd0['R'],float),str)
fileph['RI']=array(array(magstd0['R'],float)-array(magstd0['I'],float),str)
elif _field=='sloan':
for _filtsloan in 'ugriz':
if _filtsloan==filters[_filter]: airmass0[_filtsloan]= 0 # _airmass
else: airmass0[_filtsloan]=0
magstd0[_filtsloan]=stdcoo0[_filtsloan]
fileph['mu']=zeros(len(rastd0))+999
fileph['mg']=zeros(len(rastd0))+999
fileph['mr']=zeros(len(rastd0))+999
fileph['mi']=zeros(len(rastd0))+999
fileph['mz']=zeros(len(rastd0))+999
fileph['r']=magstd0['r']
fileph['gr']=array(array(magstd0['g'],float)-array(magstd0['r'],float),str)
fileph['ri']=array(array(magstd0['r'],float)-array(magstd0['i'],float),str)
fileph['ug']=array(array(magstd0['u'],float)-array(magstd0['g'],float),str)
fileph['iz']=array(array(magstd0['i'],float)-array(magstd0['z'],float),str)
########################################################################################
zero=[]
magcor=[]
fil = open(re.sub('.fits','.ph',img),'w')
fil.write(str(_instrume)+' '+str(_date)+'\n')
fil.write('*** '+_object+' '+str(len(magsex))+'\n')
if _field=='landolt':
fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(1),str(1),str(1),str(1),str(1))) # exptime
fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(airmass0['U']),str(airmass0['B']),str(airmass0['V']),str(airmass0['R']),str(airmass0['I'])))
elif _field=='sloan':
fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(1),str(1),str(1),str(1),str(1))) # exptime
fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(airmass0['u']),str(airmass0['g']),str(airmass0['r']),str(airmass0['i']),str(airmass0['z'])))
for i in range(0,len(magsex)):
fileph['m'+filters[_filter]][i]=magsex[i] # instrumental mangitude of std in pos0[i]
if _field=='landolt':
stringastandard='%12.12s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s' % (idstd0[i],fileph['V'][i],fileph['BV'][i],fileph['UB'][i],\
fileph['VR'][i],fileph['RI'][i])
fil.write('%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%60.60s\n' \
% (str(fileph['mU'][i]),str(fileph['mB'][i]),str(fileph['mV'][i]),str(fileph['mR'][i]),str(fileph['mI'][i]),str(stringastandard)))
elif _field=='sloan':
stringastandard='%12.12s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s' % (idstd0[i],fileph['r'][i],fileph['gr'][i],fileph['ug'][i],\
fileph['ri'][i],fileph['iz'][i])
fil.write('%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%60.60s\n' \
% (str(fileph['mu'][i]),str(fileph['mg'][i]),str(fileph['mr'][i]),str(fileph['mi'][i]),str(fileph['mz'][i]),str(stringastandard)))
zero.append(float(float(magstd0[filters[_filter]][i]))-float(magsex[i]))
magcor.append(magsex[i])
fil.close()
if show:
import time
from pylab import ion,plot,draw
ion()
aa=mean(compress(abs(array(zero))<99,zero))
xxx=compress((abs(array(magcor))<99)&(abs(array(zero))<99),magcor)
yyy=compress((abs(array(zero))<99)&(abs(array(magcor))<99),zero)
plot(xxx,yyy,'or')
plot([min(compress(abs(array(magcor))<99,magcor)),max(compress(abs(array(magcor))<99,magcor))],[aa,aa],'-b')
draw()
print std(compress(abs(array(zero))<99,zero))
time.sleep(5)
colorvec=colors[filters[_filter]]
for col in colorvec:
col0=magstd0[col[0]]
col1=magstd0[col[1]]
colstd0=array(col0,float)-array(col1,float)
################## sex ######################
colore=[]
for i in range(0,len(pos1)): colore.append(colstd0[i])
colore1=compress(abs(array(zero))<50,array(colore))
zero1=compress(abs(array(zero))<50,array(zero))
zero2=compress(abs(array(colore1))<2,array(zero1))
colore2=compress(abs(array(colore1))<2,array(colore1))
if _fix: fisso=colorefisso[filters[_filter]+col]
else: fisso=''
if len(colore2)==0:
print 'no calibration, '+_filter+' '+_field
b,a,sa,sb=9999,9999,0,0
else:
if _interactive: a,sa,b,sb=fitcol(colore2,zero2,_filter,col,fisso)
else: a,sa,b,sb=fitcol2(colore2,zero2,_filter,col,fisso,show,rejection)
print a,sa,b,sb
result[filters[_filter]+col]=[a,sa,b,sb]
if result:
print '\n### zeropoint ..... done at airmass 0'
if _catalogue:
lsc.util.updateheader(img,0,{'CATALOG':[str(string.split(_catalogue,'/')[-1]),'catalogue source']})
stringa=''
for ll in result:
for kk in range(0,len(result[ll])):
if not isfinite(result[ll][kk]): result[ll][kk]=0.0
valore='%3.3s %6.6s %6.6s %6.6s %6.6s' % (str(ll),str(result[ll][0]),str(result[ll][2]),str(result[ll][1]),str(result[ll][3]))
print '### ',valore
lsc.util.updateheader(img,0,{'zp'+ll:[str(valore),'a b sa sb in y=a+bx']})
if ll[0]==ll[2]: num=2
elif ll[0]==ll[1]: num=1
else: sys.exit('somthing wrong with color '+ll)
print ll,num
try:
print 'zcol'+str(num),ll[1:],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1]
lsc.mysqldef.updatevalue(database,'zcol'+str(num),ll[1:],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'z'+str(num),result[ll][0],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'c'+str(num),result[ll][2],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'dz'+str(num),result[ll][1],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'dc'+str(num),result[ll][3],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]):
lsc.mysqldef.updatevalue(database,'zcol'+str(num),ll[1:],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'z'+str(num),result[ll][0],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'c'+str(num),result[ll][2],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'dz'+str(num),result[ll][1],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'dc'+str(num),result[ll][3],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
if result[ll][0]!=9999:
print _catalogue
lsc.mysqldef.updatevalue(database,'zcat',string.split(_catalogue,'/')[-1],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]):
lsc.mysqldef.updatevalue(database,'zcat',string.split(_catalogue,'/')[-1],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
else:
lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]):
lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
except: print 'module mysqldef not found'
# Module: do/config.py
#
# Usage: Contains configuration parameters for all the scripts.
#
# As a standalone, sets the instrument.
#
###############################################################################
from pyraf import iraf
###############################################################################
# Configure folders
root = iraf.show('home', Stdout=1)[0]
iraf.set(list=root + 'list/') # The lists (bias, flat, ... )
iraf.set(master=root + 'master/') # The master files (bias, dark, ...)
iraf.set(align=root + 'align/') # The aligned files go here
iraf.set(T=root + 'temp/') # Temporary files
resultsDir = 'results/' # The results go here
###############################################################################
# Reduction parameters
noReduce = True # do not subtract bias/darks
normScience = False # normalize the science files by exposure
rotateScience = True # rotate the science
zero_comb = 'median' # average or median