def clean_image(img, cleanimg):
# print "LOGX:: Entering `clean_image` method/function in %(__file__)s" %
# globals()
import ntt
from ntt.util import readkey3, readhdr, delete
array, header = ntt.cosmics.fromfits(img, verbose=False)
import warnings
def fxn():
# print "LOGX:: Entering `fxn` method/function in %(__file__)s" %
# globals()
warnings.warn(" ", DeprecationWarning)
original_filters = warnings.filters[:]
# Ignore warnings.
warnings.simplefilter("ignore")
try:
c = ntt.cosmics.cosmicsimage(array, gain=readkey3(header, 'gain'), readnoise=readkey3(
header, 'ron'), sigclip=5.0, sigfrac=0.3, objlim=5.0, verbose=False)
c.run(maxiter=4, verbose=False)
fxn()
finally:
warnings.filters = original_filters
if not cleanimg:
delete(img)
cleanimg = img
ntt.cosmics.tofits(cleanimg, c.cleanarray, header, verbose=False)
return cleanimg
def skysofifrom2d(fitsfile, skyfile):
# print "LOGX:: Entering `skysofifrom2d` method/function in %(__file__)s"
# % globals()
import ntt
from ntt.util import readhdr, readkey3, delete
from numpy import mean, arange, compress
try:
from astropy.io import fits as pyfits
except:
import pyfits
from numpy import interp as ninterp
hdr = readhdr(fitsfile)
_grism = readkey3(hdr, 'grism')
if _grism == 'GR':
_order1 = 10
else:
_order1 = 6
yy1 = pyfits.open(fitsfile)[0].data[:, :].mean(1)
crval2 = readkey3(hdr, 'CRVAL2')
cd2 = readkey3(hdr, 'CD2_2')
xx1 = arange(len(yy1))
aa1 = crval2 + (xx1) * cd2
yy1cut = compress((aa1 < 18400) | (aa1 > 18650), yy1)
aa1cut = compress((aa1 < 18400) | (aa1 > 18650), aa1)
yy1cut1 = compress((aa1cut < 11600) | (aa1cut > 11800), yy1cut)
aa1cut1 = compress((aa1cut < 11600) | (aa1cut > 11800), aa1cut)
yy1interp = ninterp(aa1, aa1cut1, yy1cut1)
delete('_new3.fits')
hdu = pyfits.PrimaryHDU(yy1interp)
hdulist = pyfits.HDUList([hdu])
hdulist.writeto('_new3.fits')
hdulist.close()
ntt.util.updateheader('_new3.fits',0,{'CRVAL1':[crval2,''],'CD1_1':[cd2,'']})
# hdulist[0].header.update('CRVAL1', crval2)
# hdulist[0].header.update('CD1_1', cd2)
fitsfile = ntt.efoscspec2Ddef.continumsub('_new3.fits', _order1, 1)
yy1 = pyfits.open(fitsfile)[0].data
crval2 = pyfits.open(fitsfile)[0].header.get('CRVAL1')
cd2 = pyfits.open(fitsfile)[0].header.get('CD1_1')
xx1 = arange(len(yy1))
aa1 = crval2 + (xx1) * cd2
skyff = pyfits.open(skyfile)[0].data
crval1 = pyfits.open(skyfile)[0].header.get('CRVAL1')
cd1 = pyfits.open(skyfile)[0].header.get('CD1_1')
skyxx = arange(len(skyff))
skyaa = crval1 + (skyxx) * cd1
shift = ntt.efoscspec2Ddef.checkwavelength_arc(
aa1, yy1, skyaa, skyff, '', '')
delete('_new3.fits')
return shift
def makeillumination(lista,flatfield):#,outputfile,illum_frame):
import os,glob,string,re
from astropy.io import fits as pyfits
import ntt
from ntt.util import readhdr, readkey3, delete, display_image, defsex, name_duplicate, correctcard
from numpy import compress, array, argmax, argmin, min, argsort, float32
import datetime
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
_date = readkey3(readhdr(lista[0]), 'date-night')
_filter = readkey3(readhdr(lista[0]), 'filter')
illum_frame = name_duplicate(
lista[0], 'illum_' + _date + '_' + _filter + '_' + str(MJDtoday), '')
from pyraf import iraf
iraf.images(_doprint=0)
iraf.imutil(_doprint=0)
iraf.utilities(_doprint=0)
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.daophot(_doprint=0)
iraf.generic(_doprint=0)
toforget = ['digiphot.daophot', 'imutil.imarith',
'image', 'utilities.surfit']
for t in toforget:
iraf.unlearn(t)
n = len(lista)
# start loop to read image names from the input file
lista1 = []
iraf.ccdred.verbose = 'no'
ff = open('templist.lst', 'w')
for i in range(0, len(lista)):
ff.write('C' + lista[i] + '\n')
delete('C' + lista[i])
delete('C' + re.sub('.fits', '_sub.fits', lista[i]))
ntt.sofiphotredudef.crosstalk(lista[i], 'C' + lista[i])
iraf.noao.imred.ccdred.ccdproc('C' + lista[i], output='', overscan="no", trim="yes", ccdtype='', darkcor='no', fixpix='no', zerocor="no", flatcor='yes',
illumco='no', trimsec='[1:1024,1:1007]', biassec='', flat=flatfield, illum='')
correctcard('C' + lista[i])
lista1.append('C' + lista[i])
ff.close()
print '\n### prereducing STD frames to compute illumination correction ........'
lista2, skyfile = ntt.sofiphotredudef.skysub(lista1, readkey3(
readhdr(lista1[0]), 'ron'), readkey3(readhdr(lista1[0]), 'gain'), True)
lista2 = ntt.sofiphotredudef.sortbyJD(lista2)
print '\n### use x on the star and q to continue....'
display_image(lista2[0], 2, '', '', False)
delete('tmpone.coo')
iraf.image.tv.imexamine(lista2[0], 2, logfile='tmpone.coo',
keeplog='yes', xformat='', yformat='', wcs='logical')
iraf.tvmark(2, 'tmpone.coo', mark="circle", number='yes',
label='no', radii=8, nxoffse=5, nyoffse=5, color=204, txsize=2)
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x0, y0 = string.split(xycoo[0])
x0 = float(x0)
y0 = float(y0)
xcum0 = readkey3(readhdr(lista2[0]), 'xcum')
ycum0 = readkey3(readhdr(lista2[0]), 'ycum')
iraf.digiphot(_doprint=0)
iraf.daophot(_doprint=0)
iraf.noao.digiphot.daophot.datapars.datamin = -1000
iraf.noao.digiphot.daophot.datapars.datamax = 60000
iraf.noao.digiphot.daophot.daopars.function = 'gauss'
iraf.noao.digiphot.daophot.photpars.zmag = 0
namesex = defsex('default.sex')
for i in range(0, len(lista2)):
j = i + 1
xcum = readkey3(readhdr(lista2[i]), 'xcum')
ycum = readkey3(readhdr(lista2[i]), 'ycum')
xx = x0 - xcum0 + xcum
yy = y0 - ycum0 + ycum
# sex objects
os.system('sex ' + lista2[i] + ' -c ' + namesex + '> _logsex')
delete('_logsex')
xpix = iraf.proto.fields('detections.cat', fields='2', Stdout=1)
ypix = iraf.proto.fields('detections.cat', fields='3', Stdout=1)
cm = iraf.proto.fields('detections.cat', fields='4', Stdout=1)
cm = compress((array(xpix) != ''), array(cm, float))
ypix = compress((array(xpix) != ''), array(ypix, float))
xpix = compress((array(xpix) != ''), array(xpix, float))
if len(xpix) > 300:
num = 300
else:
num = len(xpix) - 1
xpix = xpix[argsort(cm)][0:num]
ypix = ypix[argsort(cm)][0:num]
distance = (ypix - yy)**2 + (xpix - xx)**2
xx1, yy1 = xpix[argmin(distance)], ypix[argmin(distance)]
f = open('tmpone.coo', 'w')
f.write(str(xx1) + ' ' + str(yy1) + '\n')
f.close()
display_image(lista2[i], 1, '', '', False)
iraf.tvmark(1, 'tmpone.coo', mark="circle", number='yes',
label='no', radii=8, nxoffse=5, nyoffse=5, color=204, txsize=2)
answ = 'n'
while answ != 'y':
answ = raw_input('selected the right one [[y]/n] ?')
if not answ:
answ = 'y'
if answ in ['y', 'YES', 'yes', 'Y']:
print lista2[i]
delete('pippo.' + str(j) + '.mag')
gggg = iraf.digiphot.daophot.phot(
lista2[i], "tmpone.coo", output="pippo." + str(j) + ".mag", verify='no', interac='no', Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
else:
print '\n### select the std star'
display_image(lista2[i], 1, '', '', False)
iraf.image.tv.imexamine(lista2[
i], 1, logfile='tmpone.coo', keeplog='yes', xformat='', yformat='', wcs='logical')
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x2, y2 = string.split(xycoo[0])
f = open('tmpone.coo', 'w')
f.write(str(x2) + ' ' + str(y2) + '\n')
f.close()
delete('pippo.' + str(j) + '.mag')
print '###### new selection ' + str(x2), str(y2)
gggg = iraf.digiphot.daophot.phot(
lista2[i], "tmpone.coo", output='pippo.' + str(j) + '.mag', verify='no', interac='no', Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
os.system('ls pippo.*.mag > tempmag.lst')
tmptbl0 = iraf.txdump(textfile="@tempmag.lst",
fields="XCENTER,YCENTER,FLUX", expr='yes', Stdout=1)
ff = open('magnitudini', 'w')
for i in tmptbl0:
ff.write(i + '\n')
ff.close()
# delete the temporary images and files
delete("temp*.fits")
delete('temp*.lst')
delete(illum_frame)
print '\n### fitting the illumination surface...'
aaa = iraf.utilities.surfit('magnitudini', image=illum_frame, function="polynomial",
xorder=2, yorder=2, xterms="full", ncols=1024, nlines=1024, Stdout=1)
iraf.noao.imred.generic.normalize(illum_frame)
correctcard(lista[0])
data, hdr = pyfits.getdata(illum_frame, 0, header=True)
data0, hdr0 = pyfits.getdata(lista[0], 0, header=True)
delete(illum_frame)
pyfits.writeto(illum_frame, float32(data), hdr0)
flatfield0 = string.split(flatfield, '/')[-1]
ntt.util.updateheader(
illum_frame, 0, {'MKILLUM': [flatfield0, 'flat field']})
display_image(illum_frame, 1, '', '', False)
for i in range(0, len(lista)): # in lista:
img = lista[i]
delete('pippo.' + str(i) + '.mag')
delete('C' + img)
delete('C' + re.sub('.fits', '_sky.fits', img))
# delete('C*.fits.mag.1')
# iraf.hedit(illum_frame,'MKILLUM','Illum. corr. created '+flatfield,add='yes',update='yes',verify='no')
return illum_frame
def makeflat(lista):
# print "LOGX:: Entering `makeflat` method/function in %(__file__)s" %
# globals()
flat = ''
import datetime
import glob
import os
import ntt
from ntt.util import readhdr, readkey3, delete, name_duplicate, updateheader, correctcard
from pyraf import iraf
iraf.images(_doprint=0)
iraf.imutil(_doprint=0)
iraf.imgeom(_doprint=0)
# iraf.blkavg(_doprint=0)
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.generic(_doprint=0)
toforget = ['imgeom.blkavg', 'imutil.imarith',
'immatch.imcombine', 'noao.imred']
for t in toforget:
iraf.unlearn(t)
import datetime
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
_date = readkey3(readhdr(lista[0]), 'date-night')
_filter = readkey3(readhdr(lista[0]), 'filter')
output = name_duplicate(
lista[3], 'flat_' + str(_date) + '_' + str(_filter) + '_' + str(MJDtoday), '')
if os.path.isfile(output):
answ = raw_input('file already prooduced, do again [y/[n]] ? ')
if not answ:
answ = 'n'
else:
answ = 'y'
if answ in ['yes', 'y', 'YES', 'Y', 'Yes']:
delete("temp_off.fits,temp_off_mask.fits,temp_on_mask.fits,temp_on.fits")
iraf.image.immatch.imcombine(
lista[0] + ',' + lista[7], output="temp_off.fits")
iraf.image.immatch.imcombine(
lista[1] + ',' + lista[6], output="temp_off_mask.fits")
iraf.image.immatch.imcombine(
lista[2] + ',' + lista[5], output="temp_on_mask.fits")
iraf.image.immatch.imcombine(
lista[3] + ',' + lista[4], output="temp_on.fits")
# create the bias correction for the flat-on according to the
# Lidman technique0
delete("temp_onA.fits,temp_onC.fits,temp_onB.fits,temp_onAC.fits,temp_onACB.fits,temp_onACB_2D.fits")
delete("temp_on_bias.fits")
iraf.imgeom.blkavg(
input="temp_on.fits[500:600,*]", output="temp_onA.fits", option="average", b1=101, b2=1)
iraf.imgeom.blkavg(
input="temp_on_mask.fits[500:600,*]", output="temp_onC.fits", option="average", b1=101, b2=1)
iraf.imgeom.blkavg(
input="temp_on_mask.fits[50:150,*]", output="temp_onB.fits", option="average", b1=101, b2=1)
iraf.imutil.imarith("temp_onA.fits", "-",
"temp_onC.fits", "temp_onAC.fits")
iraf.imutil.imarith("temp_onAC.fits", "+",
"temp_onB.fits", "temp_onACB.fits")
iraf.imgeom.blkrep(input="temp_onACB.fits",
output="temp_onACB_2D.fits", b1=1024, b2=1)
iraf.imutil.imarith("temp_on.fits", "-",
"temp_onACB_2D.fits", "temp_on_bias.fits")
# same as above for the flat-off
delete("temp_offA.fits,temp_offC.fits,temp_offB.fits,temp_offAC.fits,temp_offACB.fits,temp_offACB_2D.fits")
delete("temp_off_bias.fits")
iraf.imgeom.blkavg(
input="temp_off.fits[500:600,*]", output="temp_offA.fits", option="average", b1=101, b2=1)
iraf.imgeom.blkavg(
input="temp_off_mask.fits[500:600,*]", output="temp_offC.fits", option="average", b1=101, b2=1)
iraf.imgeom.blkavg(
input="temp_off_mask.fits[50:150,*]", output="temp_offB.fits", option="average", b1=101, b2=1)
iraf.imutil.imarith("temp_offA.fits", "-",
"temp_offC.fits", "temp_offAC.fits")
iraf.imutil.imarith("temp_offAC.fits", "+",
"temp_offB.fits", "temp_offACB.fits")
iraf.imgeom.blkrep(input="temp_offACB.fits",
output="temp_offACB_2D.fits", b1=1024, b2=1)
iraf.imutil.imarith("temp_off.fits", "-",
"temp_offACB_2D.fits", "temp_off_bias.fits")
# create the corrected flat-field
# output=name_duplicate("temp_on_bias.fits",'flat_'+str(_date)+'_'+str(_filter)+'_'+str(MJDtoday),'')
output = name_duplicate(
lista[3], 'flat_' + str(_date) + '_' + str(_filter) + '_' + str(MJDtoday), '')
# print lista[0],'flat_'+str(_date)+'_'+str(_filter)+'_'+str(MJDtoday)
delete(output)
iraf.imutil.imarith("temp_on_bias.fits", "-",
"temp_off_bias.fits", output)
iraf.noao.imred.generic.normalize(output) # normalize the flat-field
correctcard(output)
delete("temp_on*.fits") # delete the temporary images
delete("temp_off*.fits")
print 'flat -> ' + str(output)
else:
print 'skip redoing the flat'
return output
def lacos(_input0, output='clean.fits', outmask='mask.fits', gain=1.3, readn=9, xorder=9, yorder=9, sigclip=4.5, sigfrac=0.5, objlim=1, verbose=True, interactive=False):
# print "LOGX:: Entering `lacos` method/function in %(__file__)s" %
# globals()
import ntt
from ntt.util import delete
import sys
import re
import os
import string
from pyraf import iraf
import numpy as np
oldoutput, galaxy, skymod, med5 = 'oldoutput.fits', 'galaxy.fits', 'skymod.fits', 'med5.fits'
blk, lapla, med3, med7, sub5, sigima, finalsel = 'blk.fits', 'lapla.fits', 'med3.fits', 'med7.fits', 'sub5.fits', 'sigima.fits', 'finalsel.fits'
deriv2, noise, sigmap, firstsel, starreject = 'deriv2.fits', 'noise.fits', 'sigmap.fits', 'firstsel.fits', 'starreject.fits'
inputmask = 'inputmask.fits'
# set some parameters in standard IRAF tasks
iraf.convolve.bilinear = 'no'
iraf.convolve.radsym = 'no'
# create Laplacian kernel
# laplkernel = np.array([[0.0, -1.0, 0.0], [-1.0, 4.0, -1.0], [0.0, -1.0, 0.0]])
f = open('_kernel', 'w')
f.write('0 -1 0;\n-1 4 -1;\n0 -1 0')
f.close()
# create growth kernel
f = open('_gkernel', 'w')
f.write('1 1 1;\n1 1 1;\n1 1 1')
f.close()
gkernel = np.array([[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]])
delete(galaxy)
delete(skymod)
delete(oldoutput)
if not output:
output = _input0
else:
delete(output)
iraf.imcopy(_input0, output, verbose='no')
delete('_xxx.fits,_yyy.fits')
iraf.imcopy(_input0 + '[350:550,*]', '_xxx.fits', verbose='no')
_input = '_xxx.fits'
arrayinput, headerinput = ntt.cosmics.fromfits(_input, verbose=False)
ntt.cosmics.tofits(outmask, np.float32(
arrayinput - arrayinput), headerinput, verbose=False)
# subtract object spectra if desired
iraf.fit1d(_input, galaxy, "fit", axis=2, order=9, func="leg", low=4.,
high=4., nav=1, inter='no', sample="*", niter=3, grow=0, cursor="")
iraf.imarith(_input, "-", galaxy, oldoutput)
# Subtract sky lines
iraf.fit1d(oldoutput, skymod, "fit", axis=1, order=5, func="leg", low=4., high=4.,
inter='no', sample="*", nav=1, niter=3, grow=0, cursor="")
iraf.imarith(oldoutput, "-", skymod, oldoutput)
arrayoldoutput, headeroldoutput = ntt.cosmics.fromfits(
oldoutput, verbose=False)
# add object spectra to sky model
iraf.imarith(skymod, "+", galaxy, skymod)
delete(med5)
# add median of residuals to sky model
iraf.median(oldoutput, med5, 5, 5, zlor='INDEF',
zhir='INDEF', verbose='no')
# m5 = ndimage.filters.median_filter(_inputarray, size=5, mode='mirror')
iraf.imarith(skymod, "+", med5, med5)
# take second-order derivative (Laplacian) of input image
# kernel is convolved with subsampled image, in order to remove negative
# pattern around high pixels
delete(blk)
delete(lapla)
iraf.blkrep(oldoutput, blk, 2, 2)
iraf.convolve(blk, lapla, '_kernel')
iraf.imreplace(lapla, 0, upper=0, lower='INDEF')
delete(deriv2)
delete(noise)
iraf.blkavg(lapla, deriv2, 2, 2, option="average")
# create noise model
iraf.imutil.imexpr(expr='sqrt(a*' + str(gain) + '+' + str(readn) +
'**2)/' + str(gain), a=med5, output=noise, verbose='no')
iraf.imreplace(med5, 0.00001, upper=0, lower='INDEF')
# divide Laplacian by noise model
delete(sigmap)
iraf.imutil.imexpr(expr='(a/b)/2', a=deriv2, b=noise,
output=sigmap, verbose='no')
# removal of large structure (bright, extended objects)
delete(med5)
iraf.median(sigmap, med5, 5, 5, zlo='INDEF', zhi='INDEF', verbose='no')
iraf.imarith(sigmap, "-", med5, sigmap)
# find all candidate cosmic rays
# this selection includes sharp features such as stars and HII regions
arraysigmap, headersigmap = ntt.cosmics.fromfits(sigmap, verbose=False)
arrayf = np.where(arraysigmap < sigclip, 0, arraysigmap)
arrayf = np.where(arrayf > 0.1, 1, arrayf)
ntt.cosmics.tofits(firstsel, np.float32(
arrayf), headersigmap, verbose=False)
# compare candidate CRs to median filtered image
# this step rejects bright, compact sources from the initial CR list
# subtract background and smooth component of objects
delete(med3)
iraf.median(oldoutput, med3, 3, 3, zlo='INDEF', zhi='INDEF', verbose='no')
delete(med7)
delete('_' + med3)
iraf.median(med3, med7, 7, 7, zlo='INDEF', zhi='INDEF', verbose='no')
iraf.imutil.imexpr(expr='(a-b)/c', a=med3, b=med7,
c=noise, output='_' + med3, verbose='no')
iraf.imreplace('_' + med3, 0.01, upper=0.01, lower='INDEF')
# compare CR flux to object flux
delete(starreject)
iraf.imutil.imexpr(expr='a+b+c', a=firstsel, b=sigmap,
c="_" + med3, output=starreject, verbose='no')
# discard if CR flux <= objlim * object flux
iraf.imreplace(starreject, 0, upper=objlim, lower='INDEF')
iraf.imreplace(starreject, 1, lower=0.5, upper='INDEF')
iraf.imarith(firstsel, "*", starreject, firstsel)
# grow CRs by one pixel and check in original sigma map
arrayfirst, headerfirst = ntt.cosmics.fromfits(firstsel, verbose=False)
arraygfirst = ntt.cosmics.my_convolve_with_FFT2(arrayfirst, gkernel)
arraygfirst = np.where(arraygfirst > 0.5, 1, arraygfirst)
arraygfirst = arraygfirst * arraysigmap
arraygfirst = np.where(arraygfirst < sigclip, 0, arraygfirst)
arraygfirst = np.where(arraygfirst > 0.1, 1, arraygfirst)
# grow CRs by one pixel and lower detection limit
sigcliplow = sigfrac * sigclip
# Finding neighbouring pixels affected by cosmic rays
arrayfinal = ntt.cosmics.my_convolve_with_FFT2(arraygfirst, gkernel)
arrayfinal = np.where(arrayfinal > 0.5, 1, arrayfinal)
arrayfinal = arrayfinal * arraysigmap
arrayfinal = np.where(arrayfinal < sigcliplow, 0, arrayfinal)
arrayfinal = np.where(arrayfinal > 0.1, 1, arrayfinal)
# determine number of CRs found in this iteration
arraygfirst = (1 - (arrayfinal - arrayfinal)) * arrayfinal
npix = [str(int(np.size(np.where(arraygfirst > 0.5)) / 2.))]
# create cleaned output image; use 3x3 median with CRs excluded
arrayoutmask = np.where(arrayfinal > 1, 1, arrayfinal)
ntt.cosmics.tofits(outmask, np.float32(
arrayoutmask), headerfirst, verbose=False)
delete(inputmask)
arrayinputmask = (1 - (10000 * arrayoutmask)) * arrayoldoutput
ntt.cosmics.tofits(inputmask, np.float32(
arrayinputmask), headerfirst, verbose=False)
delete(med5)
iraf.median(inputmask, med5, 5, 5, zloreject=-
9999, zhi='INDEF', verbose='no')
iraf.imarith(outmask, "*", med5, med5)
delete('_yyy.fits')
iraf.imutil.imexpr(expr='(1-a)*b+c', a=outmask, b=oldoutput,
c=med5, output='_yyy.fits', verbose='no')
# add sky and object spectra back in
iraf.imarith('_yyy.fits', "+", skymod, '_yyy.fits')
# cleanup and get ready for next iteration
if npix == 0:
stop = yes
# delete temp files
iraf.imcopy('_yyy.fits', output + '[350:550,*]', verbose='no')
delete(blk + "," + lapla + "," + deriv2 + "," + med5)
delete(med3 + "," + med7 + "," + noise + "," + sigmap)
delete(firstsel + "," + starreject)
delete(finalsel + "," + inputmask)
delete(oldoutput + "," + skymod + "," + galaxy)
delete("_" + med3 + ",_" + sigmap)
delete('_kernel' + "," + '_gkernel')
delete(outmask)
delete('_xxx.fits,_yyy.fits')
def lacos_im(_input, _output='clean.fits', outmask='mask.fits', gain=1.3, readn=9, xorder=9, yorder=9, sigclip=4.5, sigfrac=0.5, objlim=1, skyval=0, niter=2, verbose=True, interactive=False):
# print "LOGX:: Entering `lacos_im` method/function in %(__file__)s" %
# globals()
import ntt
from ntt.util import delete
import sys
import re
import os
import string
from pyraf import iraf
import numpy as np
iraf.convolve.bilinear = 'no'
iraf.convolve.radsym = 'no'
# make temporary files
oldoutput, galaxy, skymod, med5 = 'oldoutput.fits', 'galaxy.fits', 'skymod.fits', 'med5.fits'
blk, lapla, med3, med7, sub5, sigima, finalsel = 'blk.fits', 'lapla.fits', 'med3.fits', 'med7.fits', 'sub5.fits', 'sigima.fits', 'finalsel.fits'
deriv2, noise, sigmap, firstsel, starreject = 'deriv2.fits', 'noise.fits', 'sigmap.fits', 'firstsel.fits', 'starreject.fits'
inputmask, gfirstsel = 'inputmask.fits', 'gfirstsel.fits'
f = open('_kernel', 'w')
f.write('0 -1 0;\n-1 4 -1;\n0 -1 0')
f.close()
# create growth kernel
f = open('_gkernel', 'w')
f.write('1 1 1;\n1 1 1;\n1 1 1')
f.close()
gkernel = np.array([[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]])
# initialize loop
usegain = gain
i = 1
stop = 'no'
previous = 0
if not _output:
_output = _input
arrayinput, headerinput = ntt.cosmics.fromfits(_input, verbose=False)
ntt.cosmics.tofits(outmask, np.float32(
arrayinput - arrayinput), headerinput, verbose=False)
delete(oldoutput)
if skyval > 0:
arrayoldoutput = arrayinput + skyval
else:
arrayoldoutput = arrayinput
ntt.cosmics.tofits(oldoutput, np.float32(
arrayoldoutput), headerinput, verbose=False)
# start iterations
while stop == 'no':
# take second-order derivative (Laplacian) of input image
# kernel is convolved with subsampled image, in order to remove negative
# pattern around high pixels
delete(blk)
delete(lapla)
delete(deriv2)
iraf.blkrep(oldoutput, blk, 2, 2)
iraf.convolve(blk, lapla, '_kernel')
iraf.imreplace(lapla, 0, upper=0, lower='INDEF', radius=0)
iraf.blkavg(lapla, deriv2, 2, 2, option="average")
delete(med5)
# create model of background flux - 5x5 box should exclude all CRs
iraf.median(oldoutput, med5, 5, 5, zlo='INDEF',
zhi='INDEF', verbose='no')
iraf.imreplace(med5, 0.0001, upper=0, lower='INDEF', radius=0)
# create noise model
delete(noise)
iraf.imutil.imexpr(expr='sqrt(a*' + str(usegain) + '+' + str(readn) +
'**2)/' + str(usegain), a=med5, output=noise, verbose='no')
# divide Laplacian by noise model
delete(sigmap)
iraf.imarith(deriv2, "/", noise, sigmap)
# Laplacian of blkreplicated image counts edges twice:
iraf.imarith(sigmap, "/", 2., sigmap)
# removal of large structure (bright, extended objects)
delete(med5)
iraf.median(sigmap, med5, 5, 5, zlo='INDEF', zhi='INDEF', verbose='no')
arraysigmap, headersigmap = ntt.cosmics.fromfits(sigmap, verbose=False)
arraymed5, headermed5 = ntt.cosmics.fromfits(med5, verbose=False)
arraysigmap = arraysigmap - arraymed5
iraf.imarith(sigmap, "-", med5, sigmap)
# find all candidate cosmic rays
# this selection includes sharp features such as stars and HII regions
delete(firstsel)
iraf.imcopy(sigmap, firstsel, verbose='no')
iraf.imreplace(firstsel, 0, upper=sigclip, lower='INDEF', radius=0)
iraf.imreplace(firstsel, 1, lower=0.1, upper='INDEF', radius=0)
# arraygfirst=arraysigmap
# arraygfirst = np.where(arraygfirst<sigclip,0,arraygfirst)
# arraygfirst = np.where(arraygfirst>0.1,1,arraygfirst)
# compare candidate CRs to median filtered image
# this step rejects bright, compact sources from the initial CR list
# subtract background and smooth component of objects
delete(med3)
delete(med7)
iraf.median(oldoutput, med3, 3, 3, zlo='INDEF',
zhi='INDEF', verbose='no')
iraf.median(med3, med7, 7, 7, zlo='INDEF', zhi='INDEF', verbose='no')
iraf.imarith(med3, "-", med7, med3)
iraf.imarith(med3, "/", noise, med3)
iraf.imreplace(med3, 0.01, upper=0.01, lower='INDEF', radius=0)
# compare CR flux to object flux
delete(starreject)
iraf.imutil.imexpr(expr="(a*b)/c", a=firstsel, b=sigmap,
c=med3, output=starreject, verbose='no')
# discard if CR flux <= objlim * object flux
iraf.imreplace(starreject, 0, upper=objlim, lower='INDEF', radius=0)
iraf.imreplace(starreject, 1, lower=0.5, upper='INDEF', radius=0)
iraf.imarith(firstsel, "*", starreject, firstsel)
# grow CRs by one pixel and check in original sigma map
delete(gfirstsel)
iraf.convolve(firstsel, gfirstsel, '_gkernel')
iraf.imreplace(gfirstsel, 1, lower=0.5, upper='INDEF', radius=0)
iraf.imarith(gfirstsel, "*", sigmap, gfirstsel)
iraf.imreplace(gfirstsel, 0, upper=sigclip, lower='INDEF', radius=0)
iraf.imreplace(gfirstsel, 1, lower=0.1, upper='INDEF', radius=0)
# grow CRs by one pixel and lower detection limit
sigcliplow = sigfrac * sigclip
delete(finalsel)
iraf.convolve(gfirstsel, finalsel, '_gkernel')
iraf.imreplace(finalsel, 1, lower=0.5, upper='INDEF', radius=0)
iraf.imarith(finalsel, "*", sigmap, finalsel)
iraf.imreplace(finalsel, 0, upper=sigcliplow, lower='INDEF', radius=0)
iraf.imreplace(finalsel, 1, lower=0.1, upper='INDEF', radius=0)
# determine number of CRs found in this iteration
delete(gfirstsel)
iraf.imutil.imexpr(expr="(1-b)*a", a=outmask,
b=finalsel, output=gfirstsel, verbose='no')
npix = iraf.imstat(gfirstsel, fields="npix",
lower=0.5, upper='INDEF', Stdout=1)
# create cleaned output image; use 3x3 median with CRs excluded
delete(med5)
iraf.imarith(outmask, "+", finalsel, outmask)
iraf.imreplace(outmask, 1, lower=1, upper='INDEF', radius=0)
delete(inputmask)
iraf.imutil.imexpr(expr="(1-10000*a)", a=outmask,
output=inputmask, verbose='no')
iraf.imarith(oldoutput, "*", inputmask, inputmask)
delete(med5)
iraf.median(inputmask, med5, 5, 5, zloreject=-
9999, zhi='INDEF', verbose='no')
iraf.imarith(outmask, "*", med5, med5)
if i > 1:
delete(_output)
delete(_output)
iraf.imutil.imexpr(expr="(1.-b)*a+c", a=oldoutput,
b=outmask, c=med5, output=_output, verbose='no')
# cleanup and get ready for next iteration
delete(oldoutput)
iraf.imcopy(_output, oldoutput, verbose='no')
if npix == 0:
stop = 'yes'
i = i + 1
if i > niter:
stop = 'yes'
# delete temp files
delete(blk + "," + lapla + "," + deriv2 + "," + med5)
delete(med3 + "," + med7 + "," + noise + "," + sigmap)
delete(firstsel + "," + starreject + "," + gfirstsel)
delete(finalsel + "," + inputmask)
if skyval > 0:
iraf.imarith(_output, "-", skyval, _output)
delete('_kernel' + "," + '_gkernel')
delete(oldoutput)
def makeillumination(lista, flatfield): #,outputfile,illum_frame):
import os, glob, string, re
from astropy.io import fits as pyfits
import ntt
from ntt.util import readhdr, readkey3, delete, display_image, defsex, name_duplicate, correctcard
from numpy import compress, array, argmax, argmin, min, argsort, float32
import datetime
MJDtoday = 55927 + (datetime.date.today() -
datetime.date(2012, 01, 01)).days
_date = readkey3(readhdr(lista[0]), 'date-night')
_filter = readkey3(readhdr(lista[0]), 'filter')
illum_frame = name_duplicate(
lista[0], 'illum_' + _date + '_' + _filter + '_' + str(MJDtoday), '')
from pyraf import iraf
iraf.images(_doprint=0, Stdout=0)
iraf.imutil(_doprint=0, Stdout=0)
iraf.utilities(_doprint=0, Stdout=0)
iraf.noao(_doprint=0, Stdout=0)
iraf.imred(_doprint=0, Stdout=0)
iraf.ccdred(_doprint=0, Stdout=0)
iraf.digiphot(_doprint=0, Stdout=0)
iraf.daophot(_doprint=0, Stdout=0)
iraf.generic(_doprint=0, Stdout=0)
toforget = [
'digiphot.daophot', 'imutil.imarith', 'image', 'utilities.surfit'
]
for t in toforget:
iraf.unlearn(t)
n = len(lista)
# start loop to read image names from the input file
lista1 = []
iraf.ccdred.verbose = 'no'
ff = open('templist.lst', 'w')
for i in range(0, len(lista)):
ff.write('C' + lista[i] + '\n')
delete('C' + lista[i])
delete('C' + re.sub('.fits', '_sub.fits', lista[i]))
ntt.sofiphotredudef.crosstalk(lista[i], 'C' + lista[i])
iraf.noao.imred.ccdred.ccdproc('C' + lista[i],
output='',
overscan="no",
trim="yes",
ccdtype='',
darkcor='no',
fixpix='no',
zerocor="no",
flatcor='yes',
illumco='no',
trimsec='[1:1024,1:1007]',
biassec='',
flat=flatfield,
illum='')
correctcard('C' + lista[i])
lista1.append('C' + lista[i])
ff.close()
print '\n### prereducing STD frames to compute illumination correction ........'
lista2, skyfile = ntt.sofiphotredudef.skysub(
lista1, readkey3(readhdr(lista1[0]), 'ron'),
readkey3(readhdr(lista1[0]), 'gain'), True)
lista2 = ntt.sofiphotredudef.sortbyJD(lista2)
print '\n### use x on the star and q to continue....'
display_image(lista2[0], 2, '', '', False)
delete('tmpone.coo')
iraf.image.tv.imexamine(lista2[0],
2,
logfile='tmpone.coo',
keeplog='yes',
xformat='',
yformat='',
wcs='logical')
iraf.tvmark(2,
'tmpone.coo',
mark="circle",
number='yes',
label='no',
radii=8,
nxoffse=5,
nyoffse=5,
color=204,
txsize=2)
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x0, y0 = string.split(xycoo[0])
x0 = float(x0)
y0 = float(y0)
xcum0 = readkey3(readhdr(lista2[0]), 'xcum')
ycum0 = readkey3(readhdr(lista2[0]), 'ycum')
iraf.digiphot(_doprint=0, Stdout=0)
iraf.daophot(_doprint=0, Stdout=0)
iraf.noao.digiphot.daophot.datapars.datamin = -1000
iraf.noao.digiphot.daophot.datapars.datamax = 60000
iraf.noao.digiphot.daophot.daopars.function = 'gauss'
iraf.noao.digiphot.daophot.photpars.zmag = 0
namesex = defsex('default.sex')
for i in range(0, len(lista2)):
j = i + 1
xcum = readkey3(readhdr(lista2[i]), 'xcum')
ycum = readkey3(readhdr(lista2[i]), 'ycum')
xx = x0 - xcum0 + xcum
yy = y0 - ycum0 + ycum
# sex objects
os.system('sex ' + lista2[i] + ' -c ' + namesex + '> _logsex')
delete('_logsex')
xpix = iraf.proto.fields('detections.cat', fields='2', Stdout=1)
ypix = iraf.proto.fields('detections.cat', fields='3', Stdout=1)
cm = iraf.proto.fields('detections.cat', fields='4', Stdout=1)
cm = compress((array(xpix) != ''), array(cm, float))
ypix = compress((array(xpix) != ''), array(ypix, float))
xpix = compress((array(xpix) != ''), array(xpix, float))
if len(xpix) > 300:
num = 300
else:
num = len(xpix) - 1
xpix = xpix[argsort(cm)][0:num]
ypix = ypix[argsort(cm)][0:num]
distance = (ypix - yy)**2 + (xpix - xx)**2
xx1, yy1 = xpix[argmin(distance)], ypix[argmin(distance)]
f = open('tmpone.coo', 'w')
f.write(str(xx1) + ' ' + str(yy1) + '\n')
f.close()
display_image(lista2[i], 1, '', '', False)
iraf.tvmark(1,
'tmpone.coo',
mark="circle",
number='yes',
label='no',
radii=8,
nxoffse=5,
nyoffse=5,
color=204,
txsize=2)
answ = 'n'
while answ != 'y':
answ = raw_input('selected the right one [[y]/n] ?')
if not answ:
answ = 'y'
if answ in ['y', 'YES', 'yes', 'Y']:
print lista2[i]
delete('pippo.' + str(j) + '.mag')
gggg = iraf.digiphot.daophot.phot(lista2[i],
"tmpone.coo",
output="pippo." + str(j) +
".mag",
verify='no',
interac='no',
Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
else:
print '\n### select the std star'
display_image(lista2[i], 1, '', '', False)
iraf.image.tv.imexamine(lista2[i],
1,
logfile='tmpone.coo',
keeplog='yes',
xformat='',
yformat='',
wcs='logical')
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x2, y2 = string.split(xycoo[0])
f = open('tmpone.coo', 'w')
f.write(str(x2) + ' ' + str(y2) + '\n')
f.close()
delete('pippo.' + str(j) + '.mag')
print '###### new selection ' + str(x2), str(y2)
gggg = iraf.digiphot.daophot.phot(lista2[i],
"tmpone.coo",
output='pippo.' + str(j) +
'.mag',
verify='no',
interac='no',
Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
os.system('ls pippo.*.mag > tempmag.lst')
tmptbl0 = iraf.txdump(textfile="@tempmag.lst",
fields="XCENTER,YCENTER,FLUX",
expr='yes',
Stdout=1)
ff = open('magnitudini', 'w')
for i in tmptbl0:
ff.write(i + '\n')
ff.close()
# delete the temporary images and files
delete("temp*.fits")
delete('temp*.lst')
delete(illum_frame)
print '\n### fitting the illumination surface...'
aaa = iraf.utilities.surfit('magnitudini',
image=illum_frame,
function="polynomial",
xorder=2,
yorder=2,
xterms="full",
ncols=1024,
nlines=1024,
Stdout=1)
iraf.noao.imred.generic.normalize(illum_frame)
correctcard(lista[0])
data, hdr = pyfits.getdata(illum_frame, 0, header=True)
data0, hdr0 = pyfits.getdata(lista[0], 0, header=True)
delete(illum_frame)
pyfits.writeto(illum_frame, float32(data), hdr0)
flatfield0 = string.split(flatfield, '/')[-1]
ntt.util.updateheader(illum_frame, 0,
{'MKILLUM': [flatfield0, 'flat field']})
display_image(illum_frame, 1, '', '', False)
for i in range(0, len(lista)): # in lista:
img = lista[i]
delete('pippo.' + str(i) + '.mag')
delete('C' + img)
delete('C' + re.sub('.fits', '_sky.fits', img))
# delete('C*.fits.mag.1')
# iraf.hedit(illum_frame,'MKILLUM','Illum. corr. created '+flatfield,add='yes',update='yes',verify='no')
return illum_frame
def makeflat(lista):
# print "LOGX:: Entering `makeflat` method/function in %(__file__)s" %
# globals()
flat = ''
import datetime
import glob
import os
import ntt
from ntt.util import readhdr, readkey3, delete, name_duplicate, updateheader, correctcard
from pyraf import iraf
iraf.images(_doprint=0, Stdout=0)
iraf.imutil(_doprint=0, Stdout=0)
iraf.imgeom(_doprint=0, Stdout=0)
# iraf.blkavg(_doprint=0, Stdout=0)
iraf.noao(_doprint=0, Stdout=0)
iraf.imred(_doprint=0, Stdout=0)
iraf.generic(_doprint=0, Stdout=0)
toforget = [
'imgeom.blkavg', 'imutil.imarith', 'immatch.imcombine', 'noao.imred'
]
for t in toforget:
iraf.unlearn(t)
import datetime
MJDtoday = 55927 + (datetime.date.today() -
datetime.date(2012, 01, 01)).days
_date = readkey3(readhdr(lista[0]), 'date-night')
_filter = readkey3(readhdr(lista[0]), 'filter')
output = name_duplicate(
lista[3],
'flat_' + str(_date) + '_' + str(_filter) + '_' + str(MJDtoday), '')
if os.path.isfile(output):
answ = raw_input('file already prooduced, do again [y/[n]] ? ')
if not answ:
answ = 'n'
else:
answ = 'y'
if answ in ['yes', 'y', 'YES', 'Y', 'Yes']:
delete(
"temp_off.fits,temp_off_mask.fits,temp_on_mask.fits,temp_on.fits")
iraf.image.immatch.imcombine(lista[0] + ',' + lista[7],
output="temp_off.fits")
iraf.image.immatch.imcombine(lista[1] + ',' + lista[6],
output="temp_off_mask.fits")
iraf.image.immatch.imcombine(lista[2] + ',' + lista[5],
output="temp_on_mask.fits")
iraf.image.immatch.imcombine(lista[3] + ',' + lista[4],
output="temp_on.fits")
# create the bias correction for the flat-on according to the
# Lidman technique0
delete(
"temp_onA.fits,temp_onC.fits,temp_onB.fits,temp_onAC.fits,temp_onACB.fits,temp_onACB_2D.fits"
)
delete("temp_on_bias.fits")
iraf.imgeom.blkavg(input="temp_on.fits[500:600,*]",
output="temp_onA.fits",
option="average",
b1=101,
b2=1)
iraf.imgeom.blkavg(input="temp_on_mask.fits[500:600,*]",
output="temp_onC.fits",
option="average",
b1=101,
b2=1)
iraf.imgeom.blkavg(input="temp_on_mask.fits[50:150,*]",
output="temp_onB.fits",
option="average",
b1=101,
b2=1)
iraf.imutil.imarith("temp_onA.fits", "-", "temp_onC.fits",
"temp_onAC.fits")
iraf.imutil.imarith("temp_onAC.fits", "+", "temp_onB.fits",
"temp_onACB.fits")
iraf.imgeom.blkrep(input="temp_onACB.fits",
output="temp_onACB_2D.fits",
b1=1024,
b2=1)
iraf.imutil.imarith("temp_on.fits", "-", "temp_onACB_2D.fits",
"temp_on_bias.fits")
# same as above for the flat-off
delete(
"temp_offA.fits,temp_offC.fits,temp_offB.fits,temp_offAC.fits,temp_offACB.fits,temp_offACB_2D.fits"
)
delete("temp_off_bias.fits")
iraf.imgeom.blkavg(input="temp_off.fits[500:600,*]",
output="temp_offA.fits",
option="average",
b1=101,
b2=1)
iraf.imgeom.blkavg(input="temp_off_mask.fits[500:600,*]",
output="temp_offC.fits",
option="average",
b1=101,
b2=1)
iraf.imgeom.blkavg(input="temp_off_mask.fits[50:150,*]",
output="temp_offB.fits",
option="average",
b1=101,
b2=1)
iraf.imutil.imarith("temp_offA.fits", "-", "temp_offC.fits",
"temp_offAC.fits")
iraf.imutil.imarith("temp_offAC.fits", "+", "temp_offB.fits",
"temp_offACB.fits")
iraf.imgeom.blkrep(input="temp_offACB.fits",
output="temp_offACB_2D.fits",
b1=1024,
b2=1)
iraf.imutil.imarith("temp_off.fits", "-", "temp_offACB_2D.fits",
"temp_off_bias.fits")
# create the corrected flat-field
# output=name_duplicate("temp_on_bias.fits",'flat_'+str(_date)+'_'+str(_filter)+'_'+str(MJDtoday),'')
output = name_duplicate(
lista[3],
'flat_' + str(_date) + '_' + str(_filter) + '_' + str(MJDtoday),
'')
# print lista[0],'flat_'+str(_date)+'_'+str(_filter)+'_'+str(MJDtoday)
delete(output)
iraf.imutil.imarith("temp_on_bias.fits", "-", "temp_off_bias.fits",
output)
iraf.noao.imred.generic.normalize(output) # normalize the flat-field
correctcard(output)
delete("temp_on*.fits") # delete the temporary images
delete("temp_off*.fits")
print 'flat -> ' + str(output)
else:
print 'skip redoing the flat'
return output
def makefringingmask(listimg, _output, _interactive, _combine='average', _rejection='avsigclip'):
# print "LOGX:: Entering `makefringingmask` method/function in
# %(__file__)s" % globals()
import ntt
from ntt.util import readhdr, readkey3, delete, updateheader
import glob
import os
import sys
import re
import string
from pyraf import iraf
iraf.noao(_doprint=0)
iraf.immatch(_doprint=0)
iraf.imutil(_doprint=0)
iraf.nproto(_doprint=0)
iraf.proto(_doprint=0)
toforget = ['nproto.objmasks', 'proto.fixpix']
for t in toforget:
iraf.unlearn(t)
if _interactive == True:
listimg2 = []
for img in listimg:
_exptime = readkey3(readhdr(img), 'exptime')
if float(_exptime) >= 10:
answ = 'xxx'
while answ.lower() not in ['y', 'n', 's', 'a']:
iraf.display(img, frame=1, fill='yes')
answ = raw_input(
'use this image (yes,no,stop (not more images),all) [[y]/n/s/a] ? ')
if not answ:
answ = 'y'
if answ.lower() == 'y':
listimg2.append(img)
elif answ.lower() == 'a':
listimg2 = listimg[:]
if answ.lower() in ['a', 's']:
break
listimg = listimg2[:]
iraf.nproto.objmasks1.fitxord = 1
iraf.nproto.objmasks1.fityord = 1
hdr0 = readhdr(listimg[0])
_date = readkey3(hdr0, 'date-obs')
_filter = readkey3(hdr0, 'filter')
_exptime = readkey3(hdr0, 'exptime')
_instrume = readkey3(hdr0, 'instrume')
_ron = readkey3(hdr0, 'ron')
_gain = readkey3(hdr0, 'gain')
badpixelmask = 'bad_pixel_mask.pl'
if not os.path.isfile(badpixelmask):
os.system('cp ' + ntt.__path__[0] + '/archive/' + _instrume +
'/badpixels/badpixel_20100210.pl ' + badpixelmask)
ff = open('_listmask', 'w')
hh = open('_listobgz', 'w')
for img in listimg:
_exptime = readkey3(readhdr(img), 'exptime')
hh.write('z_' + img + '\n')
ff.write('mask_' + img + '\n')
delete('mask_' + img)
aaa = iraf.hedit(img, delete='yes', field='OBJMASK',
up='yes', verify='no', Stdout=1)
aaa = iraf.hedit(img, delete='yes', field='BPM',
up='yes', verify='no', Stdout=1)
delete('z_' + img)
iraf.imutil.imexpr(expr='(a - median(a))/' + str(_exptime),
a=img, output='z_' + img, verbose='no')
ntt.util.updateheader('z_' + img, 0, {'EXPTIME': [1, '']})
ff.close()
hh.close()
if not _output:
_output = 'fringing_' + str(_date) + '_' + str(_filter) + '.fits'
delete(_output)
print ' making mask for each frame .......'
ccc = iraf.nproto.objmasks(images='@_listobgz', objmasks='@_listmask', omtype='boolean',
blksize=-16, convolv='block 3 3', hsigma=5, lsigma=3, minpix=10, ngrow=2, agrow=4., Stdout=1)
print 'combining all frames, masking the objects .....'
iraf.imcombine('@_listobgz', output=_output, masktyp='!OBJMASK', maskval=0, combine=_combine, reject=_rejection,
scale='none', statsec='[100:800,100:800]', rdnoise='', gain='', nlow=1, nhigh=1, logfile='imcombinelog')
ntt.util.phase3header(_output)
ntt.util.updateheader(
_output, 0, {'BUNIT': ['ADU', 'pixel units(ADU,electrons)']})
ntt.util.updateheader(_output, 0, {'FILETYPE': [11231, 'fringing frame']})
return _output
def sofispecreduction(files, _interactive, _doflat, listflat, _docross, _verbose=False):
# print "LOGX:: Entering `sofispecreduction` method/function in
# %(__file__)s" % globals()
import ntt
from ntt.util import delete, readhdr, readkey3, correctcard, rangedata
import string, re, sys, os, glob
try:
from astropy.io import fits as pyfits
except:
import pyfits
from pyraf import iraf
from numpy import argmin, array, min, isnan, arange, mean, sum
from numpy import sqrt, pi
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.twodspec(_doprint=0)
iraf.longslit(_doprint=0)
iraf.specred(_doprint=0)
toforget = ['ccdred.flatcombine', 'ccdproc', 'specred.apall', 'longslit.identify', 'longslit.reidentify',
'longslit.fitcoords', 'specred.transform', 'specred.response', 'imutil.hedit']
for t in toforget:
iraf.unlearn(t)
iraf.longslit.dispaxi = 2
iraf.longslit.mode = 'h'
iraf.specred.dispaxi = 2
iraf.specred.mode = 'h'
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.overscan = 'no'
iraf.ccdproc.ccdtype = ''
iraf.ccdred.instrument = "/dev/null"
iraf.set(direc=ntt.__path__[0] + '/')
if _interactive:
_interact = 'yes'
else:
_interact = 'no'
if _verbose:
iraf.ccdred.verbose = 'yes'
iraf.specred.verbose = 'yes'
else:
iraf.specred.verbose = 'no'
iraf.ccdred.verbose = 'no'
import datetime
import time
now = datetime.datetime.now()
datenow = now.strftime('20%y%m%d%H%M')
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
# if they are not sorted the fieldlist dict could crash
files = ntt.sofiphotredudef.sortbyJD(files)
outputlist = []
setup = []
fieldlist = {}
OBID = {}
RA = {}
DEC = {}
objects = {}
flats = {}
lamps1 = {}
_rdnoise = readkey3(readhdr(re.sub('\n', '', files[0])), 'ron')
_gain = readkey3(readhdr(re.sub('\n', '', files[0])), 'gain')
for img in files:
img = re.sub('\n', '', img)
hdr = readhdr(img)
_object = readkey3(hdr, 'object')
_filter = readkey3(hdr, 'filter')
_date = readkey3(hdr, 'date-night')
_exptime = readkey3(hdr, 'exptime')
_grism = readkey3(hdr, 'grism')
_obsmode = readkey3(hdr, 'obsmode')
_type = ''
if _grism.lower() not in ['gr', 'gb']:
_type = 'image'
if not _type:
if _object.lower() == 'flat':
_type = 'flat'
if _date not in flats:
flats[_date] = {}
if _grism not in flats[_date]:
flats[_date][_grism] = [img]
else:
flats[_date][_grism].append(img)
elif _object.lower() == 'lamp':
_lampid = (readkey3(hdr, 'esoid'), readkey3(hdr, 'grism'))
if _lampid not in lamps1:
lamps1[_lampid] = [None, None]
if readkey3(hdr, 'lamp1') == 'Xenon':
lamps1[_lampid][0] = img
else:
lamps1[_lampid][1] = img
_type = 'lamp'
# if readkey3(hdr,'lamp1')=='Xenon':
# _type='lamp'
# if _grism not in lamps:
# lamps[_grism]=[img]
# else:
# lamps[_grism].append(img)
# else:
# _type='notgood'
if not _type:
_ra = readkey3(hdr, 'RA')
_dec = readkey3(hdr, 'DEC')
_object_name = readkey3(hdr, 'object')
_OBID = (readkey3(hdr, 'esoid'), _grism)
if string.count(_object_name, '/') or string.count(_object_name, '.') or string.count(_object_name, ' '):
nameobj = string.split(_object_name, '/')[0]
nameobj = string.split(nameobj, ' ')[0]
nameobj = string.split(nameobj, '.')[0]
else:
nameobj = _object_name
if _grism not in fieldlist:
fieldlist[_grism] = {}
if _OBID not in OBID:
count = 1
nameobj0 = nameobj + '_' + str(count)
answ = 'yes'
while answ == 'yes':
if nameobj0 in fieldlist[_grism]:
count = count + 1
nameobj0 = nameobj + '_' + str(count)
else:
answ = 'no'
fieldlist[_grism][nameobj0] = []
OBID[readkey3(hdr, 'esoid'), _grism] = nameobj0
fieldlist[_grism][nameobj0].append(img)
if _verbose:
print img
print _type, _object, _filter
print 'lamps', lamps1
lamps = {}
for _lampid in lamps1:
lamp = ''
output = 'arc_' + str(_lampid[0]) + '_' + str(_lampid[1]) + '.fits'
if lamps1[_lampid][0] and lamps1[_lampid][1]:
print lamps1[_lampid][0], lamps1[_lampid][1]
# try:
ntt.util.delete(output)
iraf.imarith(lamps1[_lampid][0], '-', lamps1[_lampid]
[1], result=output, verbose='yes')
# except:
# print 'warning, lamp file not ON/OFF'
# os.system('cp '+lamps1[_lampid][0]+' '+output)
lamp = output
elif lamps1[_lampid][0] and not lamps1[_lampid][1]:
os.system('cp ' + lamps1[_lampid][0] + ' ' + output)
lamp = output
if lamp:
if _lampid[1] not in lamps:
lamps[_lampid[1]] = [lamp]
else:
lamps[_lampid[1]].append(lamp)
if _verbose:
print '\n### FIELDS\n', fieldlist
print '\n### OBID\n', OBID
print '\n### FLATS\n', flats
print '\n### LAMPS\n', lamps
# if not flats:
# sys.exit('\n### error: spectroscopic flat not available, add flats in the directory and try again')
# if not lamps:
# sys.exit('\n### error: spectroscopic lamp not available, add lamps in
# the directory and try again')
if not listflat:
print '\n### list of available spectroscopic flats (ON,OFF):'
for _date in flats:
for _grism in flats[_date]:
for img in flats[_date][_grism]:
if pyfits.open(img)[0].data.mean() >= 2000:
print img, _grism, _date, 'ON ? '
else:
print img, _grism, _date, 'OFF ? '
for _date in flats:
for _grism in flats[_date]:
flat = {'ON': [], 'OFF': []}
for img in flats[_date][_grism]:
_type = ''
if readkey3(hdr, 'lamp3'):
print '\n### header lamp3 found: flat ON ', str(img)
_type = 'ON'
else:
if pyfits.open(img)[0].data.mean() >= 2000:
_type = 'ON'
else:
_type = 'OFF'
aa, bb, cc = ntt.util.display_image(img, 1, '', '', False)
print '\n### number of flat already selected (ON,OFF): \n ### please select same number ' \
'of ON and OFF flats \n' + \
str(len(flat['ON'])) + ' ' + str(len(flat['OFF']))
print '\n### image ' + str(img)
answ = raw_input(
'ON/OFF/REJECT/STOP [' + str(_type) + '] ok (ON[n]/OFF[f]/r/s) [' + _type + '] ? ')
if not answ:
answ = _type
if answ in ['ON', 'on', 'n']:
_type = 'ON'
if answ in ['OFF', 'off', 'f']:
_type = 'OFF'
if answ in ['s', 'S', 'STOP', 'stop', 'Stop']:
_type = 'stop'
if answ in ['r', 'R', 'reject']:
_type = 'r'
if _type in ['ON', 'OFF']:
flat[_type].append(img)
elif _type == 'stop':
if len(flat['ON']) == len(flat['OFF']) and len(flat['OFF']) >= 2:
break
elif len(flat['ON']) == len(flat['OFF']) and len(flat['OFF']) == 0:
break
else:
print '\n### Warning: you can stop only if the numbers of ON and OFF are the same'
print len(flat['ON']), len(flat['OFF'])
if len(flat['ON']) == len(flat['OFF']) and len(flat['OFF']) >= 2:
ff = open('_flatlist', 'w')
for ii in range(0, len(flat['OFF'])):
delete('flat_' + str(_date) + '_' + str(_grism) +
'_' + str(MJDtoday) + '_' + str(ii) + '.fits')
iraf.imarith(flat['ON'][ii], '-', flat['OFF'][ii],
result='flat_' + str(_date) + '_' + str(_grism) + '_' + str(MJDtoday) + '_' + str(
ii) + '.fits', verbose='no')
ff.write(
'flat_' + str(_date) + '_' + str(_grism) + '_' + str(MJDtoday) + '_' + str(ii) + '.fits\n')
ff.close()
masterflat = 'flat_' + \
str(_date) + '_' + str(_grism) + \
'_' + str(MJDtoday) + '.fits'
delete(masterflat)
_order = '80'
iraf.ccdred.flatcombine(input='@_flatlist', output=masterflat, combine='median', rdnoise=_rdnoise,
gain=_gain, ccdtype='')
hdr = readhdr(masterflat)
matching = [s for s in hdr.keys() if "IMCMB" in s]
for imcmb in matching:
aaa = iraf.hedit(masterflat, imcmb, delete='yes', update='yes',
verify='no', Stdout=1)
delete('_flatlist')
print masterflat
correctcard(masterflat)
if masterflat not in outputlist:
outputlist.append(masterflat)
ntt.util.updateheader(masterflat, 0, {'FILETYPE': [41102, 'flat field'],
'SINGLEXP': [False, 'TRUE if resulting from single exposure'],
'M_EPOCH': [False, 'TRUE if resulting from multiple epochs']})
print '\n### master flat ........... done '
delete('n' + masterflat)
iraf.specred.response(masterflat, normaliz=masterflat + '[100:900,*]',
response='n' + masterflat, interac=_interact, thresho='INDEF', sample='*',
naverage=2,
function='spline3', low_rej=3, high_rej=3, order=_order, niterat=20, grow=0,
graphic='stdgraph', mode='q')
listflat.append('n' + masterflat)
if 'n' + masterflat not in outputlist:
outputlist.append('n' + masterflat)
ntt.util.updateheader('n' + masterflat, 0, {'FILETYPE': [41203, 'normalized flat field'],
'TRACE1': [masterflat, 'Originating file']})
# ntt.util.updateheader('n'+masterflat,0,{'TRACE1':[masterflat,'']})
flattot = flat['ON'] + flat['OFF']
num = 0
for img in flattot:
num = num + 1
ntt.util.updateheader(masterflat, 0, {
'PROV' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file'],
'TRACE' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
ntt.util.updateheader('n' + masterflat, 0, {
'PROV' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
if listflat:
print '\n### flat available:\n### ' + str(listflat), '\n'
elif len(flat['ON']) == len(flat['OFF']) and len(flat['OFF']) == 0:
print '\n### no good flats in this set ......'
else:
sys.exit('\n### Error: number of ON and OFF not the same')
for _grism in fieldlist:
obj0 = fieldlist[_grism][fieldlist[_grism].keys()[0]][0]
# ############# arc #########################
if _grism not in lamps:
print '\n### take arc from archive '
arcfile = ntt.util.searcharc(obj0, '')[0]
if arcfile[0] == '/':
os.system('cp ' + arcfile + ' ' +
string.split(arcfile, '/')[-1])
arcfile = string.split(arcfile, '/')[-1]
lamps[_grism] = [arcfile]
if _grism in lamps:
arclist = lamps[_grism]
if arclist:
arcfile = ntt.util.searcharc(obj0, arclist)[0]
else:
arcfile = ntt.util.searcharc(obj0, '')[0]
print arcfile
if arcfile:
print arcfile
datea = readkey3(readhdr(arcfile), 'date-night')
if arcfile[0] == '/':
os.system('cp ' + arcfile + ' ' +
string.split(arcfile, '/')[-1])
arcfile = string.split(arcfile, '/')[-1]
if _doflat:
if listflat:
flat0 = ntt.util.searchflat(arcfile, listflat)[0]
else:
flat0 = ''
else:
flat0 = ''
if flat0:
_flatcor = 'yes'
else:
_flatcor = 'no'
_doflat = False
ntt.util.delete('arc_' + datea + '_' + _grism +
'_' + str(MJDtoday) + '.fits')
print arcfile, flat0, _flatcor, _doflat
if _doflat:
iraf.noao.imred.ccdred.ccdproc(arcfile,
output='arc_' + datea + '_' + _grism +
'_' +
str(MJDtoday) + '.fits',
overscan='no', trim='no', zerocor='no', flatcor=_flatcor, flat=flat0)
else:
os.system('cp ' + arcfile + ' ' + 'arc_' + datea +
'_' + _grism + '_' + str(MJDtoday) + '.fits')
iraf.noao.imred.ccdred.ccdproc('arc_' + datea + '_' + _grism + '_' + str(MJDtoday) + '.fits', output='',
overscan='no', trim='yes', zerocor='no', flatcor='no', flat='',
trimsec='[30:1000,1:1024]')
arcfile = 'arc_' + datea + '_' + \
_grism + '_' + str(MJDtoday) + '.fits'
ntt.util.correctcard(arcfile)
print arcfile
if arcfile not in outputlist:
outputlist.append(arcfile)
ntt.util.updateheader(arcfile, 0, {'FILETYPE': [41104, 'pre-reduced 2D arc'],
'SINGLEXP': [True, 'TRUE if resulting from single exposure'],
'M_EPOCH': [False, 'TRUE if resulting from multiple epochs'],
'PROV1': [readkey3(readhdr(arcfile), 'ARCFILE'), 'Originating file'],
'TRACE1': [readkey3(readhdr(arcfile), 'ARCFILE'),
'Originating file']})
arcref = ntt.util.searcharc(obj0, '')[0]
if not arcref:
identific = iraf.longslit.identify(images=arcfile, section='column 10',
coordli='direc$standard/ident/Lines_XeAr_SOFI.dat', nsum=10,
fwidth=7, order=3, mode='h', Stdout=1, verbose='yes')
else:
print arcref
os.system('cp ' + arcref + ' .')
arcref = string.split(arcref, '/')[-1]
if not os.path.isdir('database/'):
os.mkdir('database/')
if os.path.isfile(ntt.util.searcharc(obj0, '')[1] + '/database/id' + re.sub('.fits', '', arcref)):
os.system('cp ' + ntt.util.searcharc(obj0, '')[1] + '/database/id' + re.sub('.fits', '',
arcref) + ' database/')
print arcref, arcfile
# time.sleep(5)
# os.system('rm -rf database/idarc_20130417_GR_56975')
# raw_input('ddd')
identific = iraf.longslit.reidentify(referenc=arcref, images=arcfile, interac='NO', # _interact,
section='column 10', shift=0.0,
coordli='direc$standard/ident/Lines_XeAr_SOFI.dat',
overrid='yes', step=0, newaps='no', nsum=5, nlost=2,
mode='h', verbose='yes', Stdout=1)
# print identific
# raw_input('ddd')
identific = iraf.longslit.reidentify(referenc=arcref, images=arcfile, interac=_interact,
section='column 10', shift=1.0,
coordli='direc$standard/ident/Lines_XeAr_SOFI.dat',
overrid='yes', step=0, newaps='no', nsum=5, nlost=2,
mode='h', verbose='yes', Stdout=1)
# fitsfile = ntt.efoscspec2Ddef.continumsub('new3.fits', 6, 1)
# I need to run twice I don't know why
# print identific
# raw_input('ddd')
if _interactive:
answ = raw_input(
'\n### do you like the identification [[y]/n]')
if not answ:
answ = 'y'
else:
answ = 'y'
if answ in ['n', 'N', 'no', 'NO', 'No']:
yy1 = pyfits.open(arcref)[0].data[:, 10:20].mean(1)
xx1 = arange(len(yy1))
yy2 = pyfits.open(arcfile)[0].data[:, 10:20].mean(1)
xx2 = arange(len(yy2))
ntt.util.delete('_new3.fits')
hdu = pyfits.PrimaryHDU(yy1)
hdulist = pyfits.HDUList([hdu])
hdulist.writeto('_new3.fits')
fitsfile = ntt.efoscspec2Ddef.continumsub('_new3.fits', 4, 1)
yy1 = pyfits.open(fitsfile)[0].data
ntt.util.delete('_new3.fits')
hdu = pyfits.PrimaryHDU(yy2)
hdulist = pyfits.HDUList([hdu])
hdulist.writeto('_new3.fits')
fitsfile = ntt.efoscspec2Ddef.continumsub('_new3.fits', 4, 1)
yy2 = pyfits.open(fitsfile)[0].data
_shift = ntt.efoscspec2Ddef.checkwavelength_arc(
xx1, yy1, xx2, yy2, '', '') * (-1)
print arcref, arcfile, _shift
identific = iraf.longslit.reidentify(referenc=arcref, images=arcfile, interac='YES',
section='column 10', shift=_shift,
coordli='direc$standard/ident/Lines_XeAr_SOFI.dat',
overrid='yes', step=0, newaps='no', nsum=5, nlost=2,
mode='h', verbose='yes', Stdout=1)
answ = raw_input('\n### is it ok now ? [[y]/n] ')
if not answ:
answ = 'y'
if answ in ['n', 'N', 'no', 'NO', 'No']:
sys.exit(
'\n### Warning: line identification with some problems')
iraf.longslit.reidentify(referenc=arcfile, images=arcfile, interac='NO', section='column 10',
coordli='direc$standard/ident/Lines_XeAr_SOFI.dat', overrid='yes', step=10,
newaps='yes', nsum=5, nlost=2, mode='h', verbose='no')
iraf.longslit.fitcoords(images=re.sub('.fits', '', arcfile), fitname=re.sub('.fits', '', arcfile),
interac='no', combine='yes', databas='database',
function='legendre', yorder=4, logfile='', plotfil='', mode='h')
if identific:
_rms = float(identific[-1].split()[-1])
_num = float(identific[-1].split()[2].split('/')[0])
hdr = ntt.util.readhdr(arcfile)
hedvec = {'LAMRMS': [_rms * .1, 'residual RMS [nm]'],
'LAMNLIN': [_num, 'Nb of arc lines used in the fit of the wavel. solution'],
'SPEC_ERR': [(_rms * .1) / sqrt(float(_num)), 'statistical uncertainty'],
'SPEC_SYE': [0.1, 'systematic error']}
ntt.util.updateheader(arcfile, 0, hedvec)
else:
sys.exit('Warning: arcfile not found')
else:
print 'here'
# ########################################################################################################
for field in fieldlist[_grism]:
listaobj = fieldlist[_grism][field]
listaobj = ntt.sofiphotredudef.sortbyJD(listaobj)
listatemp = listaobj[:]
# ############## flat ######################
if listflat and _doflat:
flat0 = ntt.util.searchflat(listaobj[0], listflat)[0]
else:
flat0 = ''
if flat0:
_flatcor = 'yes'
else:
_flatcor = 'no'
########## crosstalk ###########################
listatemp2 = []
_date = readkey3(readhdr(listatemp[0]), 'date-night')
for img in listatemp:
# num2=listatemp.index(listasub[j])
imgout = field + '_' + str(_date) + '_' + str(_grism) + '_' + str(MJDtoday) + '_' + str(
listatemp.index(img)) + '.fits'
print '\n### input image: ' + str(img)
delete(imgout)
listatemp2.append(imgout)
if _docross:
print '### correct for cross talk ..... done'
ntt.sofiphotredudef.crosstalk(img, imgout)
correctcard(imgout)
ntt.util.updateheader(
imgout, 0, {'CROSSTAL': ['True', '']})
else:
os.system('cp ' + img + ' ' + imgout)
correctcard(imgout)
if _flatcor == 'yes':
print '### correct for flat field ..... done'
try:
iraf.noao.imred.ccdred.ccdproc(imgout, output='', overscan='no', trim='no', zerocor='no',
flatcor=_flatcor, flat=flat0)
except:
iraf.imutil.imreplace(
images=flat0, value=0.01, lower='INDEF', upper=0.01, radius=0)
iraf.noao.imred.ccdred.ccdproc(imgout, output='', overscan='no', trim='no', zerocor='no',
flatcor=_flatcor, flat=flat0)
iraf.noao.imred.ccdred.ccdproc(imgout, output='', overscan='no', trim='yes', zerocor='no',
flatcor='no', flat='', trimsec='[30:1000,1:1024]')
ntt.util.updateheader(
imgout, 0, {'FLATCOR': [flat0, 'flat correction']})
if imgout not in outputlist:
outputlist.append(imgout)
ntt.util.updateheader(imgout, 0, {'FILETYPE': [42104, 'pre-reduced frame'],
'SINGLEXP': [True, 'TRUE if resulting from single exposure'],
'M_EPOCH': [False, 'TRUE if resulting from multiple epochs'],
'PROV1': [readkey3(readhdr(imgout), 'ARCFILE'), 'Originating file'],
'TRACE1': [readkey3(readhdr(imgout), 'ARCFILE'), 'Originating file']})
print '### output image: ' + str(imgout)
listatemp = listatemp2[:]
######### differences object images #####################
listasub = ntt.sofispec2Ddef.findsubimage(listatemp)
reduced = []
print '\n### Select Frames to be subtracted (eg A-B, B-A, C-D, D-C, ....) '
print '### frame1 \t frame2 \t offset1 \t offset2 \t JD1 \t JD2\n'
if len(listatemp) >= 2 and len(listasub) >= 2:
for j in range(0, len(listatemp)):
print '### ', listatemp[j], listasub[j], str(readkey3(readhdr(listatemp[j]), 'xcum')), str(
readkey3(readhdr(listasub[j]), 'xcum')), \
str(readkey3(readhdr(listatemp[j]), 'JD')), str(
readkey3(readhdr(listatemp[j]), 'JD'))
if _interactive:
answ = raw_input('\n### ok [[y]/n] ? ')
if not answ:
answ = 'y'
else:
answ = 'y'
num1 = j
image1 = listatemp[j]
_date = readkey3(readhdr(image1), 'date-night')
if answ == 'y':
num2 = listatemp.index(listasub[j])
image2 = listasub[j]
else:
image2 = raw_input(
'which image do you want to subtract')
num2 = listatemp.index(image2)
imgoutsub = field + '_' + str(_date) + '_' + str(_grism) + '_' + str(MJDtoday) + '_' + str(
num1) + '_' + str(num2) + '.fits'
delete(imgoutsub)
iraf.images.imutil.imarith(
operand1=image1, op='-', operand2=image2, result=imgoutsub, verbose='no')
ntt.util.updateheader(imgoutsub, 0, {'skysub': [image2, 'sky image subtracted'],
'FILETYPE': [42115, 'pre-reduced frame sky subtracted'],
'TRACE1': [image1, 'Originating file'],
'PROV2': [readkey3(readhdr(image2), 'ARCFILE'),
'Originating file'],
'TRACE2': [image2, 'Originating file']})
reduced.append(imgoutsub)
if imgoutsub not in outputlist:
outputlist.append(imgoutsub)
######################## 2D wavelengh calibration ########
for img in reduced:
if arcfile:
hdra = ntt.util.readhdr(arcfile)
delete('t' + img)
iraf.specred.transform(input=img, output='t' + img, minput='',
fitnames=re.sub('.fits', '', arcfile), databas='database',
x1='INDEF', x2='INDEF', y1='INDEF', y2='INDEF', flux='yes', mode='h',
logfile='logfile')
ntt.util.updateheader('t' + img, 0,
{'ARC': [arcfile, ''], 'FILETYPE': [42106, 'wavelength calibrate 2D frames'],
'TRACE1': [img, 'Originating file']})
ntt.util.updateheader(
't' + img, 0, {'TRACE1': [img, 'Originating file']})
ntt.util.updateheader('t' + img, 0,
{'LAMRMS': [ntt.util.readkey3(hdra, 'LAMRMS'), 'residual RMS [nm]'],
'LAMNLIN': [ntt.util.readkey3(hdra, 'LAMNLIN'), 'number of arc lines'],
'SPEC_ERR': [ntt.util.readkey3(hdra, 'SPEC_ERR'), 'statistical uncertainty'],
'SPEC_SYE': [ntt.util.readkey3(hdra, 'SPEC_SYE'), 'systematic error']})
###########################
delete('t' + arcfile)
iraf.specred.transform(input=arcfile, output='t' + arcfile, minput='',
fitnames=re.sub('.fits', '', arcfile), databas='database',
x1='INDEF', x2='INDEF', y1='INDEF', y2='INDEF', flux='yes', mode='h',
logfile='logfile')
specred = ntt.util.spectraresolution2(arcfile, 50)
if specred:
ntt.util.updateheader(
't' + img, 0, {'SPEC_RES': [specred, 'Spectral resolving power']})
delete('t' + arcfile)
###########################
iraf.hedit('t' + img, 'TRACE2', delete='yes',
update='yes', verify='no', Stdout=1)
if 't' + img not in outputlist:
outputlist.append('t' + img)
print '\n### 2D frame t' + str(img) + ' wavelengh calibrated ............ done'
_skyfile = ntt.__path__[
0] + '/standard/ident/sky_' + _grism + '.fits' # check in wavelengh #########
hdr = ntt.util.readhdr(img)
if glob.glob(_skyfile) and readkey3(hdr, 'exptime') > 20.:
_original = readkey3(hdr, 'ORIGFILE')
_archive = readkey3(hdr, 'ARCFILE')
if os.path.isfile(_archive):
imgstart = _archive
elif os.path.isfile(_original):
imgstart = _original
else:
imgstart = ''
if imgstart:
delete('_tmp.fits')
print imgstart, arcfile
iraf.specred.transform(input=imgstart, output='_tmp.fits', minput='',
fitnames=re.sub('.fits', '', arcfile), databas='database',
x1='INDEF', x2='INDEF', y1='INDEF', y2='INDEF', flux='yes', mode='h',
logfile='logfile')
shift = ntt.sofispec2Ddef.skysofifrom2d('_tmp.fits', _skyfile)
zro = pyfits.open('_tmp.fits')[0].header.get('CRVAL2')
delete('_tmp.fits')
if _interactive:
answ = raw_input(
'do you want to correct the wavelengh calibration with this shift: ' + str(
shift) + ' [[y]/n] ? ')
if not answ:
answ = 'y'
else:
answ = 'y'
if answ.lower() in ['y', 'yes']:
ntt.util.updateheader('t' + img, 0,
{'CRVAL2': [zro + int(shift), ''], 'shift': [float(shift), '']})
# ntt.util.updateheader('t'+img,0,{'shift':[float(shift),'']})
print '\n### check wavelengh calibration with sky lines ..... done'
try:
hdrt = ntt.util.readhdr('t' + img)
wavelmin = float(readkey3(hdrt, 'CRVAL2')) + (0.5 - float(readkey3(hdrt, 'CRPIX2'))) * float(
readkey3(hdrt, 'CDELT2'))
wavelmax = float(readkey3(hdrt, 'CRVAL2')) + (
(float(readkey3(hdrt, 'NAXIS2')) + 0.5 - float(readkey3(hdrt, 'CRPIX2'))) * float(
readkey3(hdrt, 'CDELT2')))
hedvec = {}
hedvec['WAVELMIN'] = [
wavelmin * .1, '[nm] minimum wavelength']
hedvec['WAVELMAX'] = [
wavelmax * .1, ' [nm] maximum wavelength']
hedvec['XMIN'] = [wavelmin, '[A] minimum wavelength']
hedvec['XMAX'] = [wavelmax, '[A] maximum wavelength']
hedvec['SPEC_BW'] = [
(wavelmax * .1) - (wavelmin * .1), '[nm] Bandpass Width Wmax - Wmin']
hedvec['SPEC_VAL'] = [
((wavelmax * .1) + (wavelmin * .1)) / 2., '[nm] Mean Wavelength']
hedvec['SPEC_BIN'] = [
((wavelmax * .1) - (wavelmin * .1)) /
(float(readkey3(hdr, 'NAXIS2')) - 1),
'Wavelength bin size [nm/pix]']
hedvec['VOCLASS'] = ['SPECTRUM V1.0', 'VO Data Model']
hedvec['VOPUB'] = ['ESO/SAF',
'VO Publishing Authority']
# hedvec['APERTURE']=[float(re.sub('slit','',readkey3(hdrt,'slit'))),'aperture width']
ntt.util.updateheader('t' + img, 0, hedvec)
except:
pass
else:
print '\n### Warning: arc not found for the image ' + str(img) + ' with setup ' + str(_grism)
reduceddata = rangedata(outputlist)
print '\n### adding keywords for phase 3 ....... '
f = open('logfile_spec2d_' + str(reduceddata) +
'_' + str(datenow) + '.raw.list', 'w')
for img in outputlist:
if img[-4:] == 'fits':
hdr = readhdr(img)
# ###############################################
# cancel pc matrix
if 'PC1_1' in hdr.keys():
aaa = iraf.hedit(img, 'PC1_1', delete='yes',
update='yes', verify='no', Stdout=1)
if 'PC2_2' in hdr.keys():
aaa = iraf.hedit(img, 'PC2_2', delete='yes',
update='yes', verify='no', Stdout=1)
if 'PC1_2' in hdr.keys():
aaa = iraf.hedit(img, 'PC1_2', delete='yes',
update='yes', verify='no', Stdout=1)
if 'PC2_1' in hdr.keys():
aaa = iraf.hedit(img, 'PC2_1', delete='yes',
update='yes', verify='no', Stdout=1)
#################
# added for DR2
print img
if 'NCOMBINE' in hdr:
_ncomb = readkey3(hdr, 'NCOMBINE')
else:
_ncomb = 1.0
ntt.util.updateheader(
img, 0, {'DETRON ': [12, 'Readout noise per output (e-)']})
ntt.util.updateheader(img, 0, {'EFFRON': [12. * (1 / sqrt(readkey3(hdr, 'ndit') * _ncomb)) * sqrt(pi / 2),
'Effective readout noise per output (e-)']})
ntt.util.phase3header(img) # phase 3 definitions
############################
# change for DR2
############################
texp = float(readkey3(hdr, 'dit')) * float(readkey3(hdr, 'ndit'))
mjdend = float(readkey3(hdr, 'MJD-OBS')) + (float(readkey3(hdr, 'ndit')) * (
float(readkey3(hdr, 'dit')) + 1.8)) / (60. * 60. * 24.)
strtexp = time.strftime('%H:%M:%S', time.gmtime(texp))
_telapse = (mjdend - float(readkey3(hdr, 'MJD-OBS'))) * \
60. * 60 * 24.
# tmid=_telapse/2.
tmid = (mjdend + float(readkey3(hdr, 'MJD-OBS'))) / 2
ntt.util.updateheader(img, 0, {'quality': ['Final', 'fast or rapid reduction'],
'BUNIT': ['ADU', 'Physical unit of array values'],
'DIT': [readkey3(hdr, 'dit'), 'Detector Integration Time'],
'NDIT': [readkey3(hdr, 'ndit'), 'Number of sub-integrations'],
'TEXPTIME': [texp, 'Total integration time of all exposures (s)'],
'EXPTIME': [texp, 'Total integration time. ' + strtexp],
'MJD-END': [mjdend, 'End of observations (days)'],
'TELAPSE': [_telapse, 'Total elapsed time [days]'],
'TMID': [tmid, '[d] MJD mid exposure'],
'TITLE': [readkey3(hdr, 'object'), 'Dataset title'],
#'TITLE':[str(tmid)[0:9]+' '+str(readkey3(hdr,'object'))+' '+str(readkey3(hdr,'grism'))+' '+\
# str(readkey3(hdr,'filter'))+'
# '+str(readkey3(hdr,'slit')),'Dataset
# title'],\
'EXT_OBJ': [False, 'TRUE if extended'],
'CONTNORM': [False, 'spectrum normalized to the continuum'],
'TOT_FLUX': [False, 'TRUE if phot cond and all src flux is captured'],
'SPECSYS': ['TOPOCENT', 'Reference frame for spectral coordinate'],
'FLUXCAL': ['ABSOLUTE', 'type of flux calibration'],
'FLUXERR': [34.7, 'Fractional uncertainty of the flux [%]'],
'DISPELEM': ['Gr#' + re.sub('Gr', '', readkey3(hdr, 'grism')),
'Dispersive element name']})
if readkey3(hdr, 'tech'):
ntt.util.updateheader(
img, 0, {'PRODCATG': ['SCIENCE.IMAGE', 'Data product category']})
aaa = str(readkey3(hdr, 'arcfiles')) + '\n'
f.write(aaa)
try:
ntt.util.airmass(img) # phase 3 definitions
except:
print '\n### airmass not computed for image: ', img
else:
print img + ' is not a fits image'
f.close()
return outputlist, 'logfile_spec2d_' + str(reduceddata) + '_' + str(datenow) + '.raw.list'
def makefringingmask(listimg,
_output,
_interactive,
_combine='average',
_rejection='avsigclip'):
# print "LOGX:: Entering `makefringingmask` method/function in
# %(__file__)s" % globals()
import ntt
from ntt.util import readhdr, readkey3, delete, updateheader
import glob
import os
import sys
import re
import string
from pyraf import iraf
iraf.noao(_doprint=0)
iraf.immatch(_doprint=0)
iraf.imutil(_doprint=0)
iraf.nproto(_doprint=0)
iraf.proto(_doprint=0)
toforget = ['nproto.objmasks', 'proto.fixpix']
for t in toforget:
iraf.unlearn(t)
if _interactive == True:
listimg2 = []
for img in listimg:
_exptime = readkey3(readhdr(img), 'exptime')
if float(_exptime) >= 10:
answ = 'xxx'
while answ.lower() not in ['y', 'n', 's', 'a']:
iraf.display(img, frame=1, fill='yes')
answ = raw_input(
'use this image (yes,no,stop (not more images),all) [[y]/n/s/a] ? '
)
if not answ:
answ = 'y'
if answ.lower() == 'y':
listimg2.append(img)
elif answ.lower() == 'a':
listimg2 = listimg[:]
if answ.lower() in ['a', 's']:
break
listimg = listimg2[:]
iraf.nproto.objmasks1.fitxord = 1
iraf.nproto.objmasks1.fityord = 1
hdr0 = readhdr(listimg[0])
_date = readkey3(hdr0, 'date-obs')
_filter = readkey3(hdr0, 'filter')
_exptime = readkey3(hdr0, 'exptime')
_instrume = readkey3(hdr0, 'instrume')
_ron = readkey3(hdr0, 'ron')
_gain = readkey3(hdr0, 'gain')
badpixelmask = 'bad_pixel_mask.pl'
if not os.path.isfile(badpixelmask):
os.system('cp ' + ntt.__path__[0] + '/archive/' + _instrume +
'/badpixels/badpixel_20100210.pl ' + badpixelmask)
ff = open('_listmask', 'w')
hh = open('_listobgz', 'w')
for img in listimg:
_exptime = readkey3(readhdr(img), 'exptime')
hh.write('z_' + img + '\n')
ff.write('mask_' + img + '\n')
delete('mask_' + img)
aaa = iraf.hedit(img,
delete='yes',
field='OBJMASK',
up='yes',
verify='no',
Stdout=1)
aaa = iraf.hedit(img,
delete='yes',
field='BPM',
up='yes',
verify='no',
Stdout=1)
delete('z_' + img)
iraf.imutil.imexpr(expr='(a - median(a))/' + str(_exptime),
a=img,
output='z_' + img,
verbose='no')
ntt.util.updateheader('z_' + img, 0, {'EXPTIME': [1, '']})
ff.close()
hh.close()
if not _output:
_output = 'fringing_' + str(_date) + '_' + str(_filter) + '.fits'
delete(_output)
print ' making mask for each frame .......'
ccc = iraf.nproto.objmasks(images='@_listobgz',
objmasks='@_listmask',
omtype='boolean',
blksize=-16,
convolv='block 3 3',
hsigma=5,
lsigma=3,
minpix=10,
ngrow=2,
agrow=4.,
Stdout=1)
print 'combining all frames, masking the objects .....'
iraf.imcombine('@_listobgz',
output=_output,
masktyp='!OBJMASK',
maskval=0,
combine=_combine,
reject=_rejection,
scale='none',
statsec='[100:800,100:800]',
rdnoise='',
gain='',
nlow=1,
nhigh=1,
logfile='imcombinelog')
ntt.util.phase3header(_output)
ntt.util.updateheader(_output, 0,
{'BUNIT': ['ADU', 'pixel units(ADU,electrons)']})
ntt.util.updateheader(_output, 0, {'FILETYPE': [11231, 'fringing frame']})
return _output
def fringing2(img, fmask, _interactive, _verbose=False):
# print "LOGX:: Entering `fringing2` method/function in %(__file__)s" %
# globals()
from ntt.util import delete, display_image, updateheader, readhdr, readkey3
from ntt.efoscphotredudef import searchfringe
import datetime
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
import ntt
import os
import string
import re
from numpy import median, where
from pyraf import iraf
iraf.nproto(_doprint=0)
iraf.unlearn('nproto.objmasks')
if _verbose:
ver = 'yes'
else:
ver = 'no'
hdr = readhdr(img)
_filter = readkey3(hdr, 'filter')
_exptime = readkey3(hdr, 'exptime')
_date = readkey3(hdr, 'date-night')
imgout = img
maskname = ''
if not readkey3(hdr, 'FRICOR'):
if fmask:
fmask = searchfringe(img, fmask)[0]
if fmask:
print '###### use fringing mask from user ' + fmask
else:
fmask = searchfringe(img, '')[0]
if fmask:
print '###### use fringing mask from archive ' + fmask
if fmask:
imgout = re.sub('.fits', '_fr.fits', img)
_trim = readkey3(hdr, 'TRIM')
_trimmask = readkey3(readhdr(fmask), 'TRIM')
maskname = 'fmask_' + str(_date) + '_' + \
_filter + '_' + str(MJDtoday) + '.fits'
delete(maskname)
if _trim and not _trimmask:
_trim = '[' + string.split(_trim, '[')[1]
iraf.ccdred.ccdproc(fmask, output=maskname, overscan="no", trim="yes", zerocor="no",
trimsec=_trim, flatcor="no", zero="", Stdout=1)
elif _trim and _trimmask:
if _trim == _trimmask:
os.system('cp ' + str(fmask) + ' ' + str(maskname))
else:
sys.exit('ERROR: fringing correction can be applied only to UNTRIMMED images or '
'images with this trim ' + str(_trimmask))
elif not _trim and _trimmask:
sys.exit(
'ERROR: image is not trimmed while selected fringing mask is trimmed: ' + str(_trimmask))
else:
os.system('cp ' + str(fmask) + ' ' + str(maskname))
iraf.nproto.objmasks1.fitxord = 1
iraf.nproto.objmasks1.fityord = 1
delete('mask_' + img)
imgcut = re.sub('.fits', '', img)
xxx = iraf.nproto.objmasks(images=imgcut, objmasks='mask_' + img, omtype='boolean',
blksize=-16, convolv='block 3 3', hsigma=5, lsigma=3, minpix=10, ngrow=2,
agrow=4., Stdout=1)
matimg = pyfits.open(img)[0].data
matmask = pyfits.open('mask_' + img)[1].data
matfrin = pyfits.open(maskname)[0].data
indices = where(matmask < 1)
scalevalue = median(
(matimg[indices] - median(matimg[indices])) / (matfrin[indices] - median(matfrin)))
delete(imgout)
iraf.imutil.imexpr(expr='a - (' + str(scalevalue) + '* (b - ' + str(median(matfrin)) + ') )', a=img,
b=maskname, output=imgout, verbose='no')
print '\n### fringing correction ..... done'
iraf.hedit(images=imgout, fields='OBJMASK', value='',
delete='yes', update='yes', verify='no')
ntt.util.updateheader(imgout, 0, {
'FRICOR': [str(scalevalue) + ' * (' + str(maskname) + ' - ' + str(median(matfrin)) + ')', '']})
ntt.util.updateheader(
imgout, 0, {'FILETYPE': [12205, 'pre-reduced image fringing corrected']})
ntt.util.updateheader(imgout, 0, {'PROV1': [readkey3(
readhdr(imgout), 'ARCFILE'), 'Originating file']})
ntt.util.updateheader(
imgout, 0, {'TRACE1': [img, 'Originating file']})
stringa = '%7.7s * (fmask_%8s.fits - %5.5s )' % (
str(scalevalue), _date, str(median(matfrin)))
else:
print '\n### fringing mask not available for this filter'
else:
print '\n### fringing correction already applyed to this image'
return imgout, maskname
def efoscreduction(imglist, _interactive, _doflat, _dobias, listflat, listbias, _dobadpixel, badpixelmask,
fringingmask, _archive, typefile, filenameobjects, _system, _cosmic, _verbose=False, method='iraf'):
# print "LOGX:: Entering `efoscreduction` method/function in %(__file__)s"
# % globals()
import ntt
from ntt.efoscphotredudef import searchbias
from ntt.util import delete, readhdr, readkey3, display_image, searchflat, rangedata, correctcard
from numpy import argmin, min, abs, sqrt
import string
import os
import re
import math
import sys
from pyraf import iraf
# ## Call and set parameters for useful iraf tasks
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.proto(_doprint=0)
toforget = ['ccdproc', 'zerocombine',
'flatcombine', 'imreplace', 'proto.fixpix']
for t in toforget:
iraf.unlearn(t)
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.overscan = 'no'
iraf.ccdproc.ccdtype = ''
iraf.ccdproc.biassec = ''
iraf.ccdred.instrument = "/dev/null"
if _verbose:
iraf.ccdred.verbose = 'yes'
else:
iraf.ccdred.verbose = 'no'
import datetime
import time
# starttime=time.time()
now = datetime.datetime.now()
datenow = now.strftime('20%y%m%d%H%M')
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
outputfile = []
reduceddata = rangedata(imglist)
img = re.sub('\n', '', imglist[0])
hdr = readhdr(img)
_gain = readkey3(hdr, 'gain')
_rdnoise = readkey3(hdr, 'ron')
_instrume = readkey3(hdr, 'instrume')
_trimsec = '[3:1010,1:1015]'
biaslist = {}
flatlist1 = {}
flatlist2 = {}
objectlist = {}
filterlist1 = []
filterlist2 = []
for img in imglist:
_type = ''
img = re.sub('\n', '', img)
hdr = readhdr(img)
_naxis1 = readkey3(hdr, 'NAXIS1')
_naxis2 = readkey3(hdr, 'NAXIS2')
if _naxis1 != 1030 or _naxis2 != 1030:
ntt.util.writeinthelog(
'image ' + str(img) + ' different dimension =\n', './logNTT.txt')
_type = 'not good'
if not _type and readkey3(hdr, 'speed') != 'fastL':
_type = 'not good'
if not _type and readkey3(hdr, 'instrume') != 'efosc':
_type = 'not good'
_imagetype = readkey3(hdr, 'tech')
if not _type and _imagetype == 'SPECTRUM':
_type = 'spectroscopic data'
if not _type:
_exptime = readkey3(hdr, 'exptime')
_date = readkey3(hdr, 'date-night')
_filter = readkey3(hdr, 'filter')
if float(_exptime) == 0.0:
if _date not in biaslist:
biaslist[_date] = []
biaslist[_date].append(img)
_type = 'bias'
if not _type:
_object = readkey3(hdr, 'object')
if _filter.lower() in ['g782', 'r784', 'z623', 'u640', 'b639', 'v641', 'r642',
'i705'] and _imagetype == 'IMAGE':
if 'sky,flat' in _object.lower():
_type = 'flat'
elif 'dome' in _object.lower() or 'flat' in _object.lower():
_type = 'flat dome'
if _type == 'flat':
if _filter not in filterlist1:
filterlist1.append(_filter)
flatlist1[_filter] = []
flatlist1[_filter].append(img)
if _type == 'flat dome':
if _filter not in filterlist2:
filterlist2.append(_filter)
flatlist2[_filter] = []
flatlist2[_filter].append(img)
if not _type:
_catg = readkey3(hdr, 'catg')
if 'science' in _catg.lower() or 'acquisition' in _catg.lower():
_type = 'object'
if _filter not in objectlist:
objectlist[_filter] = []
objectlist[_filter].append(img)
if 'acquisition' in _catg.lower():
try:
correctcard(img)
_ra1, _dec1, _name = ntt.util.correctobject(
img, 'standard_efosc_mab.txt')
_ra1, _dec1, _name = ntt.util.correctobject(
img, filenameobjects)
except:
pass
elif 'focus' in _object.lower():
_type = 'not good'
if not _type:
print '\n### warning: object not recognized '
_object = readkey3(hdr, 'object')
print img, _object, _imagetype
answ = raw_input(
'what is it: bias [1], flat [3], object[4], test [5] ? [5] ')
if not answ:
answ = '5'
if answ == '1':
if _date not in biaslist:
biaslist[_date] = ()
biaslist[_date].append(img)
elif answ == '4':
if _filter not in objectlist:
objectlist[_filter] = []
objectlist[_filter].append(img)
elif answ == '3':
tt = raw_input('dome or sky [d/[s]] ? ')
if tt == 's':
_type = 'flat'
_filter = readkey3(hdr, 'filter')
if _filter not in filterlist1:
filterlist1.append(_filter)
flatlist1[_filter] = []
flatlist1[_filter].append(img)
elif tt == 'd':
_type = 'flat dome'
_filter = readkey3(hdr, 'filter')
if _filter not in filterlist2:
filterlist2.append(_filter)
flatlist2[_filter] = []
flatlist2[_filter].append(img)
elif answ == '5':
_type = 'not good'
filterlist = list(set(filterlist1 + filterlist2))
if _verbose:
print filterlist1
print filterlist2
print flatlist1
print flatlist2
flatlist = {}
for _filt in filterlist:
if _filt not in flatlist1.keys():
if _filt in flatlist2.keys():
if len(flatlist2[_filt]) >= 3:
flatlist[_filt] = flatlist2[_filt]
elif len(flatlist1[_filt]) < 3:
if _filt in flatlist2.keys():
if len(flatlist2[_filt]) >= 3:
flatlist[_filt] = flatlist2[_filt]
elif _filt in flatlist1.keys():
if len(flatlist1[_filt]) >= 3:
flatlist[_filt] = flatlist1[_filt]
listaout = []
if _verbose:
print '\n### flat ', str(flatlist), '\n'
print '\n### bias ', str(biaslist), '\n'
print '\n### object ', str(objectlist), '\n'
###### masterbias #################
if _dobias:
if not _archive:
if listbias:
masterbiaslist = listbias
else:
masterbiaslist = []
if biaslist:
for _date in biaslist:
print '\n do bias ' + str(_date) + '\n'
biaslist[_date] = rejectbias(
biaslist[_date], False, 10)
if len(biaslist[_date]) >= 3:
masterbiasfile = 'bias_' + \
str(_date) + '_' + str(MJDtoday) + '.fits'
delete(masterbiasfile)
f = open('biaslist', 'w')
h = open('obiaslist', 'w')
for img in biaslist[_date]:
f.write(img + '\n')
h.write('o' + img + '\n')
delete('o' + img)
f.close()
h.close()
try:
print 'processing bias .....'
iraf.ccdproc('@biaslist', output='@obiaslist', overscan="no", trim="yes", zerocor='no',
fixpix='no', ccdtype='', flatcor='no', darkcor='no', biassec='',
trimsec=str(_trimsec), readaxi='column', Stdout=1)
iraf.zerocombine('@obiaslist', output=masterbiasfile, combine='median',
reject='ccdclip', ccdtype='', process='no',
rdnoise=_rdnoise, gain=_gain, Stdout=1)
correctcard(masterbiasfile)
num = 0
for img in biaslist[_date]:
num = num + 1
ntt.util.updateheader(masterbiasfile, 0, {
'PROV' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
ntt.util.updateheader(masterbiasfile, 0, {
'TRACE' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
delete('o' + img)
ntt.util.updateheader(masterbiasfile, 0,
{'M_EPOCH': [False, 'TRUE if resulting from multiple epochs']})
ntt.util.updateheader(masterbiasfile, 0,
{'SINGLEXP': [False, 'TRUE if resulting from single exposure']})
ntt.util.updateheader(masterbiasfile, 0, {
'FILETYPE': [11201, 'bias']})
masterbiaslist.append(masterbiasfile)
if masterbiasfile not in outputfile:
outputfile.append(masterbiasfile)
except:
ntt.util.writeinthelog(
'Warning ' +
str(biaslist[_date]) +
' problem with this list of bias \n',
'./logNTT.txt')
if masterbiasfile and _interactive:
aa, bb, cc = display_image(
masterbiasfile, 1, '', '', False)
answ = raw_input(
'is the masterbias ok [[y]/n] ?')
if not answ:
answ = 'y'
if answ in ['n', 'no']:
sys.exit(
'remove bad bias from input list and restart')
else:
masterbiaslist = []
########## masterflat #########################
if _doflat:
if not _archive:
if listflat:
masterflatlist = listflat
else:
masterflatlist = []
if flatlist:
for _filter in flatlist:
print '\n do flat ' + str(_filter) + '\n'
flatlist[_filter] = rejectflat(
flatlist[_filter], False)
if len(flatlist[_filter]) >= 3:
_date = readkey3(
readhdr(flatlist[_filter][0]), 'date-night')
masterflat = 'flat_' + \
str(_date) + '_' + str(_filter) + \
'_' + str(MJDtoday) + '.fits'
listaflat = 'flatlist_' + \
str(_date) + '_' + str(_filter)
_bias = ''
if masterbiaslist:
_bias = searchbias(flatlist[_filter][
0], masterbiaslist)[0]
if not _bias:
_bias = searchbias(flatlist[_filter][0], '')[0]
if _bias:
if _bias[0] == '/':
os.system('cp ' + _bias + ' .')
_bias = string.split(_bias, '/')[-1]
_zerocor = 'yes'
else:
_zerocor = 'yes'
else:
_zerocor = 'no'
answ0 = 'n'
while answ0 != 'y':
f = open(listaflat, 'w')
h = open('o' + listaflat, 'w')
for img in flatlist[_filter]:
f.write(img + '\n')
h.write('o' + img + '\n')
delete('o' + img)
f.close()
h.close()
try:
print 'processing flat .....'
iraf.ccdproc('@' + listaflat, output='@o' + listaflat, overscan='no', trim='yes',
darkcor='no', fixpix='no',
zerocor=_zerocor, flatcor='no', trimsec=str(_trimsec), biassec='',
zero=_bias, readaxi='column', ccdtype='', Stdout=1)
delete(masterflat)
iraf.flatcombine('@o' + listaflat, output=masterflat, combine='average',
reject='avsigclip', ccdtype='', process='no',
rdnoise=_rdnoise, gain=_gain, statsec='[100:800,100:800]',
lsigma=3, hsigma=2, Stdout=1)
masterflatlist.append(masterflat)
correctcard(masterflat)
num = 0
for img in flatlist[_filter]:
num = num + 1
ntt.util.updateheader(masterflat, 0, {
'PROV' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
ntt.util.updateheader(masterflat, 0, {
'TRACE' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
delete('o' + img)
ntt.util.updateheader(
masterflat, 0, {'ZEROCOR': [_bias, '']})
ntt.util.updateheader(masterflat, 0, {
'M_EPOCH': [False, 'TRUE if resulting from multiple epochs']})
ntt.util.updateheader(masterflat, 0, {
'SINGLEXP': [False, 'TRUE if resulting from single exposure']})
ntt.util.updateheader(
masterflat, 0, {'FILETYPE': [11202, 'flat field']})
if masterflat not in outputfile:
outputfile.append(masterflat)
except:
ntt.util.writeinthelog(
'Warning ' +
str(flatlist[
_filter]) + ' problem with this list of flat \n',
'./logNTT.txt')
aa, bb, cc = display_image(
masterflat, 1, '', '', False)
if masterflat and _interactive:
answ = raw_input(
'is the masterflat ok [[y]/n] ?')
if not answ:
answ = 'y'
if answ.lower() in ['n', 'no']:
answ1 = raw_input(
'try again [[y]/n] ?')
if not answ1:
answ1 = 'y'
if answ1.lower() in ['y', 'yes']:
flatlist[_filter] = ntt.efoscphotredudef.rejectflat(
flatlist[_filter], True)
else:
sys.exit(
'error: problem with flat .... exit')
else:
answ0 = 'y'
else:
answ0 = 'y'
else:
masterflatlist = []
##########################################################################
if len(masterbiaslist) == 0:
masterbiaslist = ''
if len(masterflatlist) == 0:
masterflatlist = ''
######################################
if _verbose:
print ''
print '#############################'
print masterflatlist
print masterbiaslist
print '#############################'
print ''
if masterflatlist:
listaout = listaout + masterflatlist
if masterbiaslist:
listaout = listaout + masterbiaslist
if typefile == 'calib':
objectlist = {}
for _filter in objectlist:
for img in objectlist[_filter]:
hdr = readhdr(img)
print '\n#####################################################################\n'
_object = readkey3(hdr, 'object')
_object = re.sub(' ', '', _object)
_object = re.sub('/', '_', _object)
_object = re.sub('\n', '', _object)
_exptime = readkey3(hdr, 'exptime')
_date = readkey3(hdr, 'date-night')
nameout = ntt.util.name_duplicate(img, str(_object) + '_' + str(_date) + '_' + str(_filter) + '_' + str(
MJDtoday), '')
_bias = ''
if _dobias:
if masterbiaslist:
_bias = searchbias(img, masterbiaslist)[0]
if not _bias:
_bias = searchbias(img, '')[0]
_flat = ''
if _doflat:
if masterflatlist:
_flat = searchflat(img, masterflatlist)[0]
if not _flat:
_flat = searchflat(img, '')[0]
if _bias: # bias ###
if _bias[0] == '/':
os.system('cp ' + _bias + ' .')
_bias = string.split(_bias, '/')[-1]
_zerocor = 'yes'
else:
_zerocor = 'no'
else:
_zerocor = 'no'
if _flat: # flat ###
if _flat[0] == '/':
os.system('cp ' + _flat + ' .')
_flat = string.split(_flat, '/')[-1]
_flatcor = 'yes'
else:
_flatcor = 'no'
sss = str(_object) + '_' + str(_date) + '_' + str(_filter)
print '### input', img, sss
print '### bias ', _zerocor, _bias
print '### flat ', _flatcor, _flat
print '### name ', nameout
delete(nameout)
try:
iraf.ccdproc(img, output=nameout, overscan="no", trim="yes", zerocor=_zerocor, flatcor='no',
darkcor='no', trimsec=str(_trimsec), zero=_bias, biassec='', readaxi='column', Stdout=1)
try:
iraf.ccdproc(nameout, output='', overscan="no", trim="no", zerocor='no', flatcor=_flatcor,
darkcor='no', flat=_flat, readaxi='column', ccdtype='', Stdout=1)
except:
iraf.imrepla(images=_flat, value=0.01,
lower='INDEF', upper=0.01, radius=0)
iraf.ccdproc(nameout, output='', overscan="no", trim="no", zerocor='no', flatcor=_flatcor,
darkcor='no', flat=_flat, readaxi='column', ccdtype='', Stdout=1)
correctcard(nameout)
ntt.util.updateheader(nameout, 0, {'FILTER': [readkey3(readhdr(nameout), 'filter'), 'Filter name'],
'SINGLEXP': [True, 'TRUE if resulting from single exposure'],
'M_EPOCH': [False, 'TRUE if resulting from multiple epochs'],
'FLATCOR': [_flat, ''],
'ZEROCOR': [_bias, ''], 'FILETYPE': [12204, 'pre-reduced image'],
'PROV1': [readkey3(readhdr(nameout), 'ARCFILE'), 'Originating file'],
'NCOMBINE': [1, 'Number of raw science data'],
'TRACE1': [readkey3(readhdr(nameout), 'ARCFILE'),
'Originating file']})
ntt.util.airmass(nameout) # phase 3 definitions
ntt.util.writeinthelog('\n', './logNTT.txt')
ntt.util.writeinthelog(
'image= ' + str(img) + ' output= ' + str(nameout) + '\n', './logNTT.txt')
ntt.util.writeinthelog(
'bias= ' + str(_bias) + ', flat= ' + str(_flat) + '\n', './logNTT.txt')
ntt.util.writeinthelog('\n', './logNTT.txt')
if nameout not in outputfile:
outputfile.append(nameout)
except:
ntt.util.writeinthelog(
'image ' + str(img) + ' probably corrupted\n', './logNTT.txt')
if _dobadpixel:
if not badpixelmask:
badpixelmask = 'bad_pixel_mask.fits'
delete(badpixelmask)
os.system('cp ' + ntt.__path__[0] + '/archive/' + str(
_instrume) + '/badpixels/badpixel.fits ' + badpixelmask)
iraf.proto.fixpix(images=nameout, masks=badpixelmask,
linterp='INDEF', cinterp='INDEF', verbose='no')
ntt.util.updateheader(
nameout, 0, {'FIXPIX': [badpixelmask, '']})
ntt.util.writeinthelog('image ' + str(nameout) + ' bad pixel corrected with ' + badpixelmask + '\n',
'./logNTT.txt')
print '\n### bad pixel mask correction ..... done'
else:
ntt.util.writeinthelog(
'image ' + str(nameout) + ' bad pixel NOT corrected\n', './logNTT.txt')
if _cosmic:
try:
print '\n### cosmic ..... '
ntt.cosmics.lacos_im(nameout, _output='', gain=_gain, readn=_rdnoise, xorder=9, yorder=9,
sigclip=4.5, sigfrac=0.5, objlim=1, skyval=0, niter=0, verbose=True,
interactive=False)
ntt.util.updateheader(nameout, 0, {
'LACOSMIC': [True, 'TRUE if Laplacian cosmic ray rejection has been applied to the image']})
print '\n### cosmic ..... removed '
except Exception, e:
print e
else:
ntt.util.updateheader(nameout, 0, {
'LACOSMIC': [False, 'TRUE if Laplacian cosmic ray rejection has been applied to the image']})
try:
##########################
sexvec = ntt.efoscastrodef.sextractor(nameout)
for cat in ['2mass', 'usnoa2', 'usnob1']:
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = ntt.efoscastrodef.efoscastroloop(
[nameout], cat, False, 40, 40, 100, 'rxyscale', 100, 30, sexvec, True, 10, method)
if rmsx3 <= 2 and rmsy3 <= 2:
break
if rmsx3 > 2 and rmsy3 > 2:
for cat in ['2mass', 'usnoa2', 'usnob1']:
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = ntt.efoscastrodef.efoscastroloop(
[nameout], cat, False, 20, int(20), int(50), 'rxyscale', 100, 30, sexvec, True, 5, method)
if rmsx3 <= 2 and rmsy3 <= 2:
break
if rmsx3 > 2 and rmsy3 > 2:
for cat in ['2mass', 'usnoa2', 'usnob1']:
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = ntt.efoscastrodef.efoscastroloop(
[nameout], cat, False, int(10), int(10),
int(25), 'rxyscale', 100, 30, sexvec, True, int(3), method)
##########################
astrostring = str(rmsx3) + ' ' + str(rmsy3) + ' ' + str(num3)
ntt.util.updateheader(
nameout, 0, {'ASTROMET': [astrostring, 'rmsx rmsy nstars']})
print '\n### check astrometry: fine \n### rmsx rmsy nstars: ' + astrostring
except Exception, e:
print e
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = '', '', '', '', '', '', '', '', ''
print '\n### problem with astrometry, do you have network ? '
if fwhmgess and fwhmgess < 99:
ntt.util.updateheader(nameout, 0, {'PSF_FWHM': [fwhmgess, 'Spatial resolution (arcsec)'],
'ELLIPTIC': [ellgess, 'Average ellipticity of point sources'],
'CRDER1': [(1 / sqrt(2.)) * float(rmsx3) * (1. / 3600.),
'Random error (degree)'],
'CRDER2': [(1 / sqrt(2.)) * float(rmsy3) * (1. / 3600.),
'Random error (degree)'],
'CUNIT1': ['deg', 'unit of the coord. trans.'],
'CUNIT2': ['deg', 'unit of the coord. trans.'],
'CSYER1': [rasys3, 'Systematic error (RA_m - Ra_ref)'],
'CSYER2': [decsys3, 'Systematic error (DEC_m - DEC_ref)']})
else:
ntt.util.updateheader(nameout, 0, {'PSF_FWHM': [9999., 'FHWM (arcsec) - computed with sectractor'],
'ELLIPTIC': [9999., 'ellipticity of point sources (1-b/a)'],
'CRDER1': [9999., 'Random error in axis 1'],
'CRDER2': [9999., 'Random error in axis 2'],
'CUNIT1': ['deg', 'unit of the coord. trans.'],
'CUNIT2': ['deg', 'unit of the coord. trans.'],
'CSYER1': [9999., 'Systematic error (RA_m - Ra_ref)'],
'CSYER2': [9999., 'Systematic error (DEC_m - DEC_ref)']})
try:
result = ntt.efoscastrodef.zeropoint(
nameout, _system, method, False, False)
except:
result = ''
if result:
if os.path.isfile(re.sub('.fits', '.ph', nameout)):
if re.sub('.fits', '.ph', nameout) not in outputfile:
outputfile.append(
re.sub('.fits', '.ph', nameout))
print '\n### zeropoint ..... done'
for ll in result:
valore = '%3.3s %6.6s %6.6s' % (
str(ll), str(result[ll][1]), str(result[ll][0]))
print '### ', valore
ntt.util.updateheader(
nameout, 0, {'zp' + ll: [str(valore), '']})
if magsat3:
if readkey3(readhdr(nameout), 'FLUXCAL') == 'ABSOLUTE':
try:
ntt.util.updateheader(nameout, 0, {
'ABMAGSAT': [float(magsat3) + float(readkey3(readhdr(nameout)), 'PHOTZP'),
'Saturation limit for point sources (AB mags)']})
except:
ntt.util.updateheader(nameout, 0, {
'ABMAGSAT': [float(magsat3), 'Saturation limit for point sources (AB mags)']})
else:
ntt.util.updateheader(nameout, 0, {
'ABMAGSAT': [float(magsat3), 'Saturation limit for point sources (AB mags)']})
else:
ntt.util.updateheader(nameout, 0, {'ABMAGSAT': [
9999., 'Saturation limit for point sources (AB mags)']})
maglim = ntt.util.limmag(nameout)
if maglim:
ntt.util.updateheader(nameout, 0,
{'ABMAGLIM': [maglim, '5-sigma limiting AB magnitude for point sources']})
else:
ntt.util.updateheader(nameout, 0,
{'ABMAGLIM': [9999., '5-sigma limiting AB magnitude for point sources']})
if readkey3(readhdr(nameout), 'filter') in ['i705']:
try:
nameout, maskname = ntt.efoscphotredudef.fringing2(
nameout, fringingmask, _interactive, False)
if nameout not in outputfile:
outputfile.append(nameout)
if maskname not in outputfile:
outputfile.append(maskname)
except:
ntt.util.writeinthelog(
'image ' + str(nameout) + ' probably corrupted\n', './logNTT.txt')
print '\n### problem with fringing correction'