def rejectflat(lista, _interactive):
from numpy import where, size
from ntt.util import display_image
from pyraf import iraf
iraf.images(_doprint=0)
import os
import string
listone = []
for img in lista:
dataimg = pyfits.open(img)[0].data
numberpixels = size(dataimg)
indices = where(dataimg > 60000)
numbersaturated = size(dataimg[indices])
indices2 = where(dataimg < 1000)
numberlow = size(dataimg[indices2])
if 100 * float(numbersaturated) / float(numberpixels) <= 5 and \
100 * float(numberlow) / float(numberpixels) <= 10:
listone.append(img)
listgood = []
if _interactive:
for img in listone:
aa, bb, cc = display_image(img, 1, '', '', False)
titolo, result = iraf.imstat(img, Stdout=1)
print titolo[1:]
print result
answ = 'nn'
while answ not in ['g', 'G', 'b', 's']:
answ = raw_input('good/bad [[g]/b/G(all good)/s(stop)]? ')
if not answ:
answ = 'g'
if answ not in ['g', 'G', 'b', 's']:
print 'warning: value not valid, try again'
if answ == 'g':
listgood.append(img)
elif answ == 'G':
listgood = listone
break
elif answ == 's':
break
else:
listgood = listone
return listgood
def makeillumination(lista,flatfield):#,outputfile,illum_frame):
import os,glob,string,re
from astropy.io import fits as pyfits
import ntt
from ntt.util import readhdr, readkey3, delete, display_image, defsex, name_duplicate, correctcard
from numpy import compress, array, argmax, argmin, min, argsort, float32
import datetime
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
_date = readkey3(readhdr(lista[0]), 'date-night')
_filter = readkey3(readhdr(lista[0]), 'filter')
illum_frame = name_duplicate(
lista[0], 'illum_' + _date + '_' + _filter + '_' + str(MJDtoday), '')
from pyraf import iraf
iraf.images(_doprint=0)
iraf.imutil(_doprint=0)
iraf.utilities(_doprint=0)
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.daophot(_doprint=0)
iraf.generic(_doprint=0)
toforget = ['digiphot.daophot', 'imutil.imarith',
'image', 'utilities.surfit']
for t in toforget:
iraf.unlearn(t)
n = len(lista)
# start loop to read image names from the input file
lista1 = []
iraf.ccdred.verbose = 'no'
ff = open('templist.lst', 'w')
for i in range(0, len(lista)):
ff.write('C' + lista[i] + '\n')
delete('C' + lista[i])
delete('C' + re.sub('.fits', '_sub.fits', lista[i]))
ntt.sofiphotredudef.crosstalk(lista[i], 'C' + lista[i])
iraf.noao.imred.ccdred.ccdproc('C' + lista[i], output='', overscan="no", trim="yes", ccdtype='', darkcor='no', fixpix='no', zerocor="no", flatcor='yes',
illumco='no', trimsec='[1:1024,1:1007]', biassec='', flat=flatfield, illum='')
correctcard('C' + lista[i])
lista1.append('C' + lista[i])
ff.close()
print '\n### prereducing STD frames to compute illumination correction ........'
lista2, skyfile = ntt.sofiphotredudef.skysub(lista1, readkey3(
readhdr(lista1[0]), 'ron'), readkey3(readhdr(lista1[0]), 'gain'), True)
lista2 = ntt.sofiphotredudef.sortbyJD(lista2)
print '\n### use x on the star and q to continue....'
display_image(lista2[0], 2, '', '', False)
delete('tmpone.coo')
iraf.image.tv.imexamine(lista2[0], 2, logfile='tmpone.coo',
keeplog='yes', xformat='', yformat='', wcs='logical')
iraf.tvmark(2, 'tmpone.coo', mark="circle", number='yes',
label='no', radii=8, nxoffse=5, nyoffse=5, color=204, txsize=2)
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x0, y0 = string.split(xycoo[0])
x0 = float(x0)
y0 = float(y0)
xcum0 = readkey3(readhdr(lista2[0]), 'xcum')
ycum0 = readkey3(readhdr(lista2[0]), 'ycum')
iraf.digiphot(_doprint=0)
iraf.daophot(_doprint=0)
iraf.noao.digiphot.daophot.datapars.datamin = -1000
iraf.noao.digiphot.daophot.datapars.datamax = 60000
iraf.noao.digiphot.daophot.daopars.function = 'gauss'
iraf.noao.digiphot.daophot.photpars.zmag = 0
namesex = defsex('default.sex')
for i in range(0, len(lista2)):
j = i + 1
xcum = readkey3(readhdr(lista2[i]), 'xcum')
ycum = readkey3(readhdr(lista2[i]), 'ycum')
xx = x0 - xcum0 + xcum
yy = y0 - ycum0 + ycum
# sex objects
os.system('sex ' + lista2[i] + ' -c ' + namesex + '> _logsex')
delete('_logsex')
xpix = iraf.proto.fields('detections.cat', fields='2', Stdout=1)
ypix = iraf.proto.fields('detections.cat', fields='3', Stdout=1)
cm = iraf.proto.fields('detections.cat', fields='4', Stdout=1)
cm = compress((array(xpix) != ''), array(cm, float))
ypix = compress((array(xpix) != ''), array(ypix, float))
xpix = compress((array(xpix) != ''), array(xpix, float))
if len(xpix) > 300:
num = 300
else:
num = len(xpix) - 1
xpix = xpix[argsort(cm)][0:num]
ypix = ypix[argsort(cm)][0:num]
distance = (ypix - yy)**2 + (xpix - xx)**2
xx1, yy1 = xpix[argmin(distance)], ypix[argmin(distance)]
f = open('tmpone.coo', 'w')
f.write(str(xx1) + ' ' + str(yy1) + '\n')
f.close()
display_image(lista2[i], 1, '', '', False)
iraf.tvmark(1, 'tmpone.coo', mark="circle", number='yes',
label='no', radii=8, nxoffse=5, nyoffse=5, color=204, txsize=2)
answ = 'n'
while answ != 'y':
answ = raw_input('selected the right one [[y]/n] ?')
if not answ:
answ = 'y'
if answ in ['y', 'YES', 'yes', 'Y']:
print lista2[i]
delete('pippo.' + str(j) + '.mag')
gggg = iraf.digiphot.daophot.phot(
lista2[i], "tmpone.coo", output="pippo." + str(j) + ".mag", verify='no', interac='no', Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
else:
print '\n### select the std star'
display_image(lista2[i], 1, '', '', False)
iraf.image.tv.imexamine(lista2[
i], 1, logfile='tmpone.coo', keeplog='yes', xformat='', yformat='', wcs='logical')
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x2, y2 = string.split(xycoo[0])
f = open('tmpone.coo', 'w')
f.write(str(x2) + ' ' + str(y2) + '\n')
f.close()
delete('pippo.' + str(j) + '.mag')
print '###### new selection ' + str(x2), str(y2)
gggg = iraf.digiphot.daophot.phot(
lista2[i], "tmpone.coo", output='pippo.' + str(j) + '.mag', verify='no', interac='no', Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
os.system('ls pippo.*.mag > tempmag.lst')
tmptbl0 = iraf.txdump(textfile="@tempmag.lst",
fields="XCENTER,YCENTER,FLUX", expr='yes', Stdout=1)
ff = open('magnitudini', 'w')
for i in tmptbl0:
ff.write(i + '\n')
ff.close()
# delete the temporary images and files
delete("temp*.fits")
delete('temp*.lst')
delete(illum_frame)
print '\n### fitting the illumination surface...'
aaa = iraf.utilities.surfit('magnitudini', image=illum_frame, function="polynomial",
xorder=2, yorder=2, xterms="full", ncols=1024, nlines=1024, Stdout=1)
iraf.noao.imred.generic.normalize(illum_frame)
correctcard(lista[0])
data, hdr = pyfits.getdata(illum_frame, 0, header=True)
data0, hdr0 = pyfits.getdata(lista[0], 0, header=True)
delete(illum_frame)
pyfits.writeto(illum_frame, float32(data), hdr0)
flatfield0 = string.split(flatfield, '/')[-1]
ntt.util.updateheader(
illum_frame, 0, {'MKILLUM': [flatfield0, 'flat field']})
display_image(illum_frame, 1, '', '', False)
for i in range(0, len(lista)): # in lista:
img = lista[i]
delete('pippo.' + str(i) + '.mag')
delete('C' + img)
delete('C' + re.sub('.fits', '_sky.fits', img))
# delete('C*.fits.mag.1')
# iraf.hedit(illum_frame,'MKILLUM','Illum. corr. created '+flatfield,add='yes',update='yes',verify='no')
return illum_frame
def doflatsofi(flats, _doflat, illum, _output):
# print "LOGX:: Entering `doflatsofi` method/function in %(__file__)s" %
# globals()
import ntt
from ntt.util import display_image, delete, searchflat, name_duplicate
from pyraf import iraf
import glob
import string
onofflimit = {'J': 1000, 'H': 1000, 'Ks': 5000}
masklimit = {'J': {'ON': 1000, 'OFF': 30}, 'H': {
'ON': 1000, 'OFF': 40}, 'Ks': {'ON': 1000, 'OFF': 1000}}
if flats and _doflat:
listaflat = []
for _filter in flats:
for ID in flats[_filter]:
images = flats[_filter][ID]
if len(images) == 8:
mflat = makeflat(images)
listaflat.append(mflat)
display_image(mflat, 1, '', '', False)
raw_input('go on ')
elif len(images) != 8: # % 8 == 0:
print '\n### to compute a flat field you need a sequence of 8 calibration files in the following orders:'
print 'OFF OFFMASK ONMASK ON ON ONMASK OFFMASK OFF\n'
print len(images), _filter, ID
tipo = ['OFF', 'OFFMASK', 'ONMASK', 'ON',
'ON', 'ONMASK', 'OFFMASK', 'OFF']
listtmp = []
ii = 0
nn = 0
for img in images:
onoffvalue = float(string.split(iraf.imstat(
img + '[500:600,900:1000]', Stdout=1)[1])[2])
maskvalue = float(string.split(iraf.imstat(
img + '[100:200,900:1000]', Stdout=1)[1])[2])
if onoffvalue >= onofflimit[_filter]:
onoff = 'ON'
else:
onoff = 'OFF'
if maskvalue >= masklimit[_filter][onoff]:
mask = 'none'
else:
mask = 'MASK'
# display_image(img,1,'','',False)
print onoff, mask, onoffvalue, maskvalue, img, tipo[nn]
for img in images:
onoffvalue = float(string.split(iraf.imstat(
img + '[500:600,900:1000]', Stdout=1)[1])[2])
maskvalue = float(string.split(iraf.imstat(
img + '[100:200,900:1000]', Stdout=1)[1])[2])
if onoffvalue >= onofflimit[_filter]:
onoff = 'ON'
else:
onoff = 'OFF'
if maskvalue >= masklimit[_filter][onoff]:
mask = 'none'
else:
mask = 'MASK'
display_image(img, 1, '', '', False)
print onoff, mask, onoffvalue, maskvalue, img, tipo[nn]
answ = raw_input('ok [[y]/n/r/s] ? ')
if not answ:
answ = 'y'
if answ == 'y':
listtmp.append(img)
ii = ii + 1
nn = nn + 1
if len(listtmp) == 8:
print '### number images selected: ', str(len(listtmp))
mflat = ntt.soficalibdef.makeflat(listtmp)
listaflat.append(mflat)
display_image(mflat, 1, '', '', False)
nn = 0
listtmp = []
elif answ == 'r':
listtmp = []
ii = 0
nn = 0
elif answ == 's':
break
else:
print len(images), _filter, ID
print '### number images selected: ', str(len(listtmp))
else:
listaflat = flats
listaillum = []
if illum:
for _filter in illum:
for ID in illum[_filter]:
images = illum[_filter][ID]
flatfield = searchflat(images[0], listaflat)[0]
if flatfield:
illum_frame = ntt.soficalibdef.makeillumination(
images, flatfield)
listaillum.append(illum_frame)
else:
print 'flat field not found'
for img in listaflat:
try:
ntt.util.phase3header(img) # phase 3 definitions
ntt.util.updateheader(
img, 0, {'BUNIT': ['ADU', 'pixel units(ADU,electrons)']})
ntt.util.updateheader(
img, 0, {'FILETYPE': [31202, 'flat field']})
except:
print '\n### problems with phase 3 definitions'
for img in listaillum:
try:
ntt.util.phase3header(img) # phase 3 definitions
ntt.util.updateheader(
img, 0, {'BUNIT': ['ADU', 'pixel units(ADU,electrons)']})
ntt.util.updateheader(
img, 0, {'FILETYPE': [31213, 'illum corr frames']})
except:
print '\n### problems with phase 3 definitions'
return listaflat, listaillum
def doflatsofi(flats, _doflat, illum, _output):
# print "LOGX:: Entering `doflatsofi` method/function in %(__file__)s" %
# globals()
import ntt
from ntt.util import display_image, delete, searchflat, name_duplicate
from pyraf import iraf
import glob
import string
onofflimit = {'J': 1000, 'H': 1000, 'Ks': 5000}
masklimit = {
'J': {
'ON': 1000,
'OFF': 30
},
'H': {
'ON': 1000,
'OFF': 40
},
'Ks': {
'ON': 1000,
'OFF': 1000
}
}
if flats and _doflat:
listaflat = []
for _filter in flats:
for ID in flats[_filter]:
images = flats[_filter][ID]
if len(images) == 8:
mflat = makeflat(images)
listaflat.append(mflat)
display_image(mflat, 1, '', '', False)
raw_input('go on ')
elif len(images) != 8: # % 8 == 0:
print '\n### to compute a flat field you need a sequence of 8 calibration files in the following orders:'
print 'OFF OFFMASK ONMASK ON ON ONMASK OFFMASK OFF\n'
print len(images), _filter, ID
tipo = [
'OFF', 'OFFMASK', 'ONMASK', 'ON', 'ON', 'ONMASK',
'OFFMASK', 'OFF'
]
listtmp = []
ii = 0
nn = 0
for img in images:
onoffvalue = float(
string.split(
iraf.imstat(img + '[500:600,900:1000]',
Stdout=1)[1])[2])
maskvalue = float(
string.split(
iraf.imstat(img + '[100:200,900:1000]',
Stdout=1)[1])[2])
if onoffvalue >= onofflimit[_filter]:
onoff = 'ON'
else:
onoff = 'OFF'
if maskvalue >= masklimit[_filter][onoff]:
mask = 'none'
else:
mask = 'MASK'
# display_image(img,1,'','',False)
print onoff, mask, onoffvalue, maskvalue, img, tipo[nn]
for img in images:
onoffvalue = float(
string.split(
iraf.imstat(img + '[500:600,900:1000]',
Stdout=1)[1])[2])
maskvalue = float(
string.split(
iraf.imstat(img + '[100:200,900:1000]',
Stdout=1)[1])[2])
if onoffvalue >= onofflimit[_filter]:
onoff = 'ON'
else:
onoff = 'OFF'
if maskvalue >= masklimit[_filter][onoff]:
mask = 'none'
else:
mask = 'MASK'
display_image(img, 1, '', '', False)
print onoff, mask, onoffvalue, maskvalue, img, tipo[nn]
answ = raw_input('ok [[y]/n/r/s] ? ')
if not answ:
answ = 'y'
if answ == 'y':
listtmp.append(img)
ii = ii + 1
nn = nn + 1
if len(listtmp) == 8:
print '### number images selected: ', str(
len(listtmp))
mflat = ntt.soficalibdef.makeflat(listtmp)
listaflat.append(mflat)
display_image(mflat, 1, '', '', False)
nn = 0
listtmp = []
elif answ == 'r':
listtmp = []
ii = 0
nn = 0
elif answ == 's':
break
else:
print len(images), _filter, ID
print '### number images selected: ', str(len(listtmp))
else:
listaflat = flats
listaillum = []
if illum:
for _filter in illum:
for ID in illum[_filter]:
images = illum[_filter][ID]
flatfield = searchflat(images[0], listaflat)[0]
if flatfield:
illum_frame = ntt.soficalibdef.makeillumination(
images, flatfield)
listaillum.append(illum_frame)
else:
print 'flat field not found'
for img in listaflat:
try:
ntt.util.phase3header(img) # phase 3 definitions
ntt.util.updateheader(
img, 0, {'BUNIT': ['ADU', 'pixel units(ADU,electrons)']})
ntt.util.updateheader(img, 0,
{'FILETYPE': [31202, 'flat field']})
except:
print '\n### problems with phase 3 definitions'
for img in listaillum:
try:
ntt.util.phase3header(img) # phase 3 definitions
ntt.util.updateheader(
img, 0, {'BUNIT': ['ADU', 'pixel units(ADU,electrons)']})
ntt.util.updateheader(
img, 0, {'FILETYPE': [31213, 'illum corr frames']})
except:
print '\n### problems with phase 3 definitions'
return listaflat, listaillum
def makeillumination(lista, flatfield): #,outputfile,illum_frame):
import os, glob, string, re
from astropy.io import fits as pyfits
import ntt
from ntt.util import readhdr, readkey3, delete, display_image, defsex, name_duplicate, correctcard
from numpy import compress, array, argmax, argmin, min, argsort, float32
import datetime
MJDtoday = 55927 + (datetime.date.today() -
datetime.date(2012, 01, 01)).days
_date = readkey3(readhdr(lista[0]), 'date-night')
_filter = readkey3(readhdr(lista[0]), 'filter')
illum_frame = name_duplicate(
lista[0], 'illum_' + _date + '_' + _filter + '_' + str(MJDtoday), '')
from pyraf import iraf
iraf.images(_doprint=0, Stdout=0)
iraf.imutil(_doprint=0, Stdout=0)
iraf.utilities(_doprint=0, Stdout=0)
iraf.noao(_doprint=0, Stdout=0)
iraf.imred(_doprint=0, Stdout=0)
iraf.ccdred(_doprint=0, Stdout=0)
iraf.digiphot(_doprint=0, Stdout=0)
iraf.daophot(_doprint=0, Stdout=0)
iraf.generic(_doprint=0, Stdout=0)
toforget = [
'digiphot.daophot', 'imutil.imarith', 'image', 'utilities.surfit'
]
for t in toforget:
iraf.unlearn(t)
n = len(lista)
# start loop to read image names from the input file
lista1 = []
iraf.ccdred.verbose = 'no'
ff = open('templist.lst', 'w')
for i in range(0, len(lista)):
ff.write('C' + lista[i] + '\n')
delete('C' + lista[i])
delete('C' + re.sub('.fits', '_sub.fits', lista[i]))
ntt.sofiphotredudef.crosstalk(lista[i], 'C' + lista[i])
iraf.noao.imred.ccdred.ccdproc('C' + lista[i],
output='',
overscan="no",
trim="yes",
ccdtype='',
darkcor='no',
fixpix='no',
zerocor="no",
flatcor='yes',
illumco='no',
trimsec='[1:1024,1:1007]',
biassec='',
flat=flatfield,
illum='')
correctcard('C' + lista[i])
lista1.append('C' + lista[i])
ff.close()
print '\n### prereducing STD frames to compute illumination correction ........'
lista2, skyfile = ntt.sofiphotredudef.skysub(
lista1, readkey3(readhdr(lista1[0]), 'ron'),
readkey3(readhdr(lista1[0]), 'gain'), True)
lista2 = ntt.sofiphotredudef.sortbyJD(lista2)
print '\n### use x on the star and q to continue....'
display_image(lista2[0], 2, '', '', False)
delete('tmpone.coo')
iraf.image.tv.imexamine(lista2[0],
2,
logfile='tmpone.coo',
keeplog='yes',
xformat='',
yformat='',
wcs='logical')
iraf.tvmark(2,
'tmpone.coo',
mark="circle",
number='yes',
label='no',
radii=8,
nxoffse=5,
nyoffse=5,
color=204,
txsize=2)
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x0, y0 = string.split(xycoo[0])
x0 = float(x0)
y0 = float(y0)
xcum0 = readkey3(readhdr(lista2[0]), 'xcum')
ycum0 = readkey3(readhdr(lista2[0]), 'ycum')
iraf.digiphot(_doprint=0, Stdout=0)
iraf.daophot(_doprint=0, Stdout=0)
iraf.noao.digiphot.daophot.datapars.datamin = -1000
iraf.noao.digiphot.daophot.datapars.datamax = 60000
iraf.noao.digiphot.daophot.daopars.function = 'gauss'
iraf.noao.digiphot.daophot.photpars.zmag = 0
namesex = defsex('default.sex')
for i in range(0, len(lista2)):
j = i + 1
xcum = readkey3(readhdr(lista2[i]), 'xcum')
ycum = readkey3(readhdr(lista2[i]), 'ycum')
xx = x0 - xcum0 + xcum
yy = y0 - ycum0 + ycum
# sex objects
os.system('sex ' + lista2[i] + ' -c ' + namesex + '> _logsex')
delete('_logsex')
xpix = iraf.proto.fields('detections.cat', fields='2', Stdout=1)
ypix = iraf.proto.fields('detections.cat', fields='3', Stdout=1)
cm = iraf.proto.fields('detections.cat', fields='4', Stdout=1)
cm = compress((array(xpix) != ''), array(cm, float))
ypix = compress((array(xpix) != ''), array(ypix, float))
xpix = compress((array(xpix) != ''), array(xpix, float))
if len(xpix) > 300:
num = 300
else:
num = len(xpix) - 1
xpix = xpix[argsort(cm)][0:num]
ypix = ypix[argsort(cm)][0:num]
distance = (ypix - yy)**2 + (xpix - xx)**2
xx1, yy1 = xpix[argmin(distance)], ypix[argmin(distance)]
f = open('tmpone.coo', 'w')
f.write(str(xx1) + ' ' + str(yy1) + '\n')
f.close()
display_image(lista2[i], 1, '', '', False)
iraf.tvmark(1,
'tmpone.coo',
mark="circle",
number='yes',
label='no',
radii=8,
nxoffse=5,
nyoffse=5,
color=204,
txsize=2)
answ = 'n'
while answ != 'y':
answ = raw_input('selected the right one [[y]/n] ?')
if not answ:
answ = 'y'
if answ in ['y', 'YES', 'yes', 'Y']:
print lista2[i]
delete('pippo.' + str(j) + '.mag')
gggg = iraf.digiphot.daophot.phot(lista2[i],
"tmpone.coo",
output="pippo." + str(j) +
".mag",
verify='no',
interac='no',
Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
else:
print '\n### select the std star'
display_image(lista2[i], 1, '', '', False)
iraf.image.tv.imexamine(lista2[i],
1,
logfile='tmpone.coo',
keeplog='yes',
xformat='',
yformat='',
wcs='logical')
xycoo = iraf.proto.fields('tmpone.coo', '1,2', Stdout=1)
x2, y2 = string.split(xycoo[0])
f = open('tmpone.coo', 'w')
f.write(str(x2) + ' ' + str(y2) + '\n')
f.close()
delete('pippo.' + str(j) + '.mag')
print '###### new selection ' + str(x2), str(y2)
gggg = iraf.digiphot.daophot.phot(lista2[i],
"tmpone.coo",
output='pippo.' + str(j) +
'.mag',
verify='no',
interac='no',
Stdout=1)
try:
float(string.split(gggg[0])[3])
answ = 'y'
except:
print '\n### warning'
answ = 'n'
os.system('ls pippo.*.mag > tempmag.lst')
tmptbl0 = iraf.txdump(textfile="@tempmag.lst",
fields="XCENTER,YCENTER,FLUX",
expr='yes',
Stdout=1)
ff = open('magnitudini', 'w')
for i in tmptbl0:
ff.write(i + '\n')
ff.close()
# delete the temporary images and files
delete("temp*.fits")
delete('temp*.lst')
delete(illum_frame)
print '\n### fitting the illumination surface...'
aaa = iraf.utilities.surfit('magnitudini',
image=illum_frame,
function="polynomial",
xorder=2,
yorder=2,
xterms="full",
ncols=1024,
nlines=1024,
Stdout=1)
iraf.noao.imred.generic.normalize(illum_frame)
correctcard(lista[0])
data, hdr = pyfits.getdata(illum_frame, 0, header=True)
data0, hdr0 = pyfits.getdata(lista[0], 0, header=True)
delete(illum_frame)
pyfits.writeto(illum_frame, float32(data), hdr0)
flatfield0 = string.split(flatfield, '/')[-1]
ntt.util.updateheader(illum_frame, 0,
{'MKILLUM': [flatfield0, 'flat field']})
display_image(illum_frame, 1, '', '', False)
for i in range(0, len(lista)): # in lista:
img = lista[i]
delete('pippo.' + str(i) + '.mag')
delete('C' + img)
delete('C' + re.sub('.fits', '_sky.fits', img))
# delete('C*.fits.mag.1')
# iraf.hedit(illum_frame,'MKILLUM','Illum. corr. created '+flatfield,add='yes',update='yes',verify='no')
return illum_frame
def efoscreduction(imglist, _interactive, _doflat, _dobias, listflat, listbias, _dobadpixel, badpixelmask,
fringingmask, _archive, typefile, filenameobjects, _system, _cosmic, _verbose=False, method='iraf'):
# print "LOGX:: Entering `efoscreduction` method/function in %(__file__)s"
# % globals()
import ntt
from ntt.efoscphotredudef import searchbias
from ntt.util import delete, readhdr, readkey3, display_image, searchflat, rangedata, correctcard
from numpy import argmin, min, abs, sqrt
import string
import os
import re
import math
import sys
from pyraf import iraf
# ## Call and set parameters for useful iraf tasks
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.proto(_doprint=0)
toforget = ['ccdproc', 'zerocombine',
'flatcombine', 'imreplace', 'proto.fixpix']
for t in toforget:
iraf.unlearn(t)
iraf.ccdproc.darkcor = 'no'
iraf.ccdproc.fixpix = 'no'
iraf.ccdproc.flatcor = 'no'
iraf.ccdproc.zerocor = 'no'
iraf.ccdproc.overscan = 'no'
iraf.ccdproc.ccdtype = ''
iraf.ccdproc.biassec = ''
iraf.ccdred.instrument = "/dev/null"
if _verbose:
iraf.ccdred.verbose = 'yes'
else:
iraf.ccdred.verbose = 'no'
import datetime
import time
# starttime=time.time()
now = datetime.datetime.now()
datenow = now.strftime('20%y%m%d%H%M')
MJDtoday = 55927 + (datetime.date.today() - datetime.date(2012, 01, 01)).days
outputfile = []
reduceddata = rangedata(imglist)
img = re.sub('\n', '', imglist[0])
hdr = readhdr(img)
_gain = readkey3(hdr, 'gain')
_rdnoise = readkey3(hdr, 'ron')
_instrume = readkey3(hdr, 'instrume')
_trimsec = '[3:1010,1:1015]'
biaslist = {}
flatlist1 = {}
flatlist2 = {}
objectlist = {}
filterlist1 = []
filterlist2 = []
for img in imglist:
_type = ''
img = re.sub('\n', '', img)
hdr = readhdr(img)
_naxis1 = readkey3(hdr, 'NAXIS1')
_naxis2 = readkey3(hdr, 'NAXIS2')
if _naxis1 != 1030 or _naxis2 != 1030:
ntt.util.writeinthelog(
'image ' + str(img) + ' different dimension =\n', './logNTT.txt')
_type = 'not good'
if not _type and readkey3(hdr, 'speed') != 'fastL':
_type = 'not good'
if not _type and readkey3(hdr, 'instrume') != 'efosc':
_type = 'not good'
_imagetype = readkey3(hdr, 'tech')
if not _type and _imagetype == 'SPECTRUM':
_type = 'spectroscopic data'
if not _type:
_exptime = readkey3(hdr, 'exptime')
_date = readkey3(hdr, 'date-night')
_filter = readkey3(hdr, 'filter')
if float(_exptime) == 0.0:
if _date not in biaslist:
biaslist[_date] = []
biaslist[_date].append(img)
_type = 'bias'
if not _type:
_object = readkey3(hdr, 'object')
if _filter.lower() in ['g782', 'r784', 'z623', 'u640', 'b639', 'v641', 'r642',
'i705'] and _imagetype == 'IMAGE':
if 'sky,flat' in _object.lower():
_type = 'flat'
elif 'dome' in _object.lower() or 'flat' in _object.lower():
_type = 'flat dome'
if _type == 'flat':
if _filter not in filterlist1:
filterlist1.append(_filter)
flatlist1[_filter] = []
flatlist1[_filter].append(img)
if _type == 'flat dome':
if _filter not in filterlist2:
filterlist2.append(_filter)
flatlist2[_filter] = []
flatlist2[_filter].append(img)
if not _type:
_catg = readkey3(hdr, 'catg')
if 'science' in _catg.lower() or 'acquisition' in _catg.lower():
_type = 'object'
if _filter not in objectlist:
objectlist[_filter] = []
objectlist[_filter].append(img)
if 'acquisition' in _catg.lower():
try:
correctcard(img)
_ra1, _dec1, _name = ntt.util.correctobject(
img, 'standard_efosc_mab.txt')
_ra1, _dec1, _name = ntt.util.correctobject(
img, filenameobjects)
except:
pass
elif 'focus' in _object.lower():
_type = 'not good'
if not _type:
print '\n### warning: object not recognized '
_object = readkey3(hdr, 'object')
print img, _object, _imagetype
answ = raw_input(
'what is it: bias [1], flat [3], object[4], test [5] ? [5] ')
if not answ:
answ = '5'
if answ == '1':
if _date not in biaslist:
biaslist[_date] = ()
biaslist[_date].append(img)
elif answ == '4':
if _filter not in objectlist:
objectlist[_filter] = []
objectlist[_filter].append(img)
elif answ == '3':
tt = raw_input('dome or sky [d/[s]] ? ')
if tt == 's':
_type = 'flat'
_filter = readkey3(hdr, 'filter')
if _filter not in filterlist1:
filterlist1.append(_filter)
flatlist1[_filter] = []
flatlist1[_filter].append(img)
elif tt == 'd':
_type = 'flat dome'
_filter = readkey3(hdr, 'filter')
if _filter not in filterlist2:
filterlist2.append(_filter)
flatlist2[_filter] = []
flatlist2[_filter].append(img)
elif answ == '5':
_type = 'not good'
filterlist = list(set(filterlist1 + filterlist2))
if _verbose:
print filterlist1
print filterlist2
print flatlist1
print flatlist2
flatlist = {}
for _filt in filterlist:
if _filt not in flatlist1.keys():
if _filt in flatlist2.keys():
if len(flatlist2[_filt]) >= 3:
flatlist[_filt] = flatlist2[_filt]
elif len(flatlist1[_filt]) < 3:
if _filt in flatlist2.keys():
if len(flatlist2[_filt]) >= 3:
flatlist[_filt] = flatlist2[_filt]
elif _filt in flatlist1.keys():
if len(flatlist1[_filt]) >= 3:
flatlist[_filt] = flatlist1[_filt]
listaout = []
if _verbose:
print '\n### flat ', str(flatlist), '\n'
print '\n### bias ', str(biaslist), '\n'
print '\n### object ', str(objectlist), '\n'
###### masterbias #################
if _dobias:
if not _archive:
if listbias:
masterbiaslist = listbias
else:
masterbiaslist = []
if biaslist:
for _date in biaslist:
print '\n do bias ' + str(_date) + '\n'
biaslist[_date] = rejectbias(
biaslist[_date], False, 10)
if len(biaslist[_date]) >= 3:
masterbiasfile = 'bias_' + \
str(_date) + '_' + str(MJDtoday) + '.fits'
delete(masterbiasfile)
f = open('biaslist', 'w')
h = open('obiaslist', 'w')
for img in biaslist[_date]:
f.write(img + '\n')
h.write('o' + img + '\n')
delete('o' + img)
f.close()
h.close()
try:
print 'processing bias .....'
iraf.ccdproc('@biaslist', output='@obiaslist', overscan="no", trim="yes", zerocor='no',
fixpix='no', ccdtype='', flatcor='no', darkcor='no', biassec='',
trimsec=str(_trimsec), readaxi='column', Stdout=1)
iraf.zerocombine('@obiaslist', output=masterbiasfile, combine='median',
reject='ccdclip', ccdtype='', process='no',
rdnoise=_rdnoise, gain=_gain, Stdout=1)
correctcard(masterbiasfile)
num = 0
for img in biaslist[_date]:
num = num + 1
ntt.util.updateheader(masterbiasfile, 0, {
'PROV' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
ntt.util.updateheader(masterbiasfile, 0, {
'TRACE' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
delete('o' + img)
ntt.util.updateheader(masterbiasfile, 0,
{'M_EPOCH': [False, 'TRUE if resulting from multiple epochs']})
ntt.util.updateheader(masterbiasfile, 0,
{'SINGLEXP': [False, 'TRUE if resulting from single exposure']})
ntt.util.updateheader(masterbiasfile, 0, {
'FILETYPE': [11201, 'bias']})
masterbiaslist.append(masterbiasfile)
if masterbiasfile not in outputfile:
outputfile.append(masterbiasfile)
except:
ntt.util.writeinthelog(
'Warning ' +
str(biaslist[_date]) +
' problem with this list of bias \n',
'./logNTT.txt')
if masterbiasfile and _interactive:
aa, bb, cc = display_image(
masterbiasfile, 1, '', '', False)
answ = raw_input(
'is the masterbias ok [[y]/n] ?')
if not answ:
answ = 'y'
if answ in ['n', 'no']:
sys.exit(
'remove bad bias from input list and restart')
else:
masterbiaslist = []
########## masterflat #########################
if _doflat:
if not _archive:
if listflat:
masterflatlist = listflat
else:
masterflatlist = []
if flatlist:
for _filter in flatlist:
print '\n do flat ' + str(_filter) + '\n'
flatlist[_filter] = rejectflat(
flatlist[_filter], False)
if len(flatlist[_filter]) >= 3:
_date = readkey3(
readhdr(flatlist[_filter][0]), 'date-night')
masterflat = 'flat_' + \
str(_date) + '_' + str(_filter) + \
'_' + str(MJDtoday) + '.fits'
listaflat = 'flatlist_' + \
str(_date) + '_' + str(_filter)
_bias = ''
if masterbiaslist:
_bias = searchbias(flatlist[_filter][
0], masterbiaslist)[0]
if not _bias:
_bias = searchbias(flatlist[_filter][0], '')[0]
if _bias:
if _bias[0] == '/':
os.system('cp ' + _bias + ' .')
_bias = string.split(_bias, '/')[-1]
_zerocor = 'yes'
else:
_zerocor = 'yes'
else:
_zerocor = 'no'
answ0 = 'n'
while answ0 != 'y':
f = open(listaflat, 'w')
h = open('o' + listaflat, 'w')
for img in flatlist[_filter]:
f.write(img + '\n')
h.write('o' + img + '\n')
delete('o' + img)
f.close()
h.close()
try:
print 'processing flat .....'
iraf.ccdproc('@' + listaflat, output='@o' + listaflat, overscan='no', trim='yes',
darkcor='no', fixpix='no',
zerocor=_zerocor, flatcor='no', trimsec=str(_trimsec), biassec='',
zero=_bias, readaxi='column', ccdtype='', Stdout=1)
delete(masterflat)
iraf.flatcombine('@o' + listaflat, output=masterflat, combine='average',
reject='avsigclip', ccdtype='', process='no',
rdnoise=_rdnoise, gain=_gain, statsec='[100:800,100:800]',
lsigma=3, hsigma=2, Stdout=1)
masterflatlist.append(masterflat)
correctcard(masterflat)
num = 0
for img in flatlist[_filter]:
num = num + 1
ntt.util.updateheader(masterflat, 0, {
'PROV' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
ntt.util.updateheader(masterflat, 0, {
'TRACE' + str(num): [readkey3(readhdr(img), 'ARCFILE'), 'Originating file']})
delete('o' + img)
ntt.util.updateheader(
masterflat, 0, {'ZEROCOR': [_bias, '']})
ntt.util.updateheader(masterflat, 0, {
'M_EPOCH': [False, 'TRUE if resulting from multiple epochs']})
ntt.util.updateheader(masterflat, 0, {
'SINGLEXP': [False, 'TRUE if resulting from single exposure']})
ntt.util.updateheader(
masterflat, 0, {'FILETYPE': [11202, 'flat field']})
if masterflat not in outputfile:
outputfile.append(masterflat)
except:
ntt.util.writeinthelog(
'Warning ' +
str(flatlist[
_filter]) + ' problem with this list of flat \n',
'./logNTT.txt')
aa, bb, cc = display_image(
masterflat, 1, '', '', False)
if masterflat and _interactive:
answ = raw_input(
'is the masterflat ok [[y]/n] ?')
if not answ:
answ = 'y'
if answ.lower() in ['n', 'no']:
answ1 = raw_input(
'try again [[y]/n] ?')
if not answ1:
answ1 = 'y'
if answ1.lower() in ['y', 'yes']:
flatlist[_filter] = ntt.efoscphotredudef.rejectflat(
flatlist[_filter], True)
else:
sys.exit(
'error: problem with flat .... exit')
else:
answ0 = 'y'
else:
answ0 = 'y'
else:
masterflatlist = []
##########################################################################
if len(masterbiaslist) == 0:
masterbiaslist = ''
if len(masterflatlist) == 0:
masterflatlist = ''
######################################
if _verbose:
print ''
print '#############################'
print masterflatlist
print masterbiaslist
print '#############################'
print ''
if masterflatlist:
listaout = listaout + masterflatlist
if masterbiaslist:
listaout = listaout + masterbiaslist
if typefile == 'calib':
objectlist = {}
for _filter in objectlist:
for img in objectlist[_filter]:
hdr = readhdr(img)
print '\n#####################################################################\n'
_object = readkey3(hdr, 'object')
_object = re.sub(' ', '', _object)
_object = re.sub('/', '_', _object)
_object = re.sub('\n', '', _object)
_exptime = readkey3(hdr, 'exptime')
_date = readkey3(hdr, 'date-night')
nameout = ntt.util.name_duplicate(img, str(_object) + '_' + str(_date) + '_' + str(_filter) + '_' + str(
MJDtoday), '')
_bias = ''
if _dobias:
if masterbiaslist:
_bias = searchbias(img, masterbiaslist)[0]
if not _bias:
_bias = searchbias(img, '')[0]
_flat = ''
if _doflat:
if masterflatlist:
_flat = searchflat(img, masterflatlist)[0]
if not _flat:
_flat = searchflat(img, '')[0]
if _bias: # bias ###
if _bias[0] == '/':
os.system('cp ' + _bias + ' .')
_bias = string.split(_bias, '/')[-1]
_zerocor = 'yes'
else:
_zerocor = 'no'
else:
_zerocor = 'no'
if _flat: # flat ###
if _flat[0] == '/':
os.system('cp ' + _flat + ' .')
_flat = string.split(_flat, '/')[-1]
_flatcor = 'yes'
else:
_flatcor = 'no'
sss = str(_object) + '_' + str(_date) + '_' + str(_filter)
print '### input', img, sss
print '### bias ', _zerocor, _bias
print '### flat ', _flatcor, _flat
print '### name ', nameout
delete(nameout)
try:
iraf.ccdproc(img, output=nameout, overscan="no", trim="yes", zerocor=_zerocor, flatcor='no',
darkcor='no', trimsec=str(_trimsec), zero=_bias, biassec='', readaxi='column', Stdout=1)
try:
iraf.ccdproc(nameout, output='', overscan="no", trim="no", zerocor='no', flatcor=_flatcor,
darkcor='no', flat=_flat, readaxi='column', ccdtype='', Stdout=1)
except:
iraf.imrepla(images=_flat, value=0.01,
lower='INDEF', upper=0.01, radius=0)
iraf.ccdproc(nameout, output='', overscan="no", trim="no", zerocor='no', flatcor=_flatcor,
darkcor='no', flat=_flat, readaxi='column', ccdtype='', Stdout=1)
correctcard(nameout)
ntt.util.updateheader(nameout, 0, {'FILTER': [readkey3(readhdr(nameout), 'filter'), 'Filter name'],
'SINGLEXP': [True, 'TRUE if resulting from single exposure'],
'M_EPOCH': [False, 'TRUE if resulting from multiple epochs'],
'FLATCOR': [_flat, ''],
'ZEROCOR': [_bias, ''], 'FILETYPE': [12204, 'pre-reduced image'],
'PROV1': [readkey3(readhdr(nameout), 'ARCFILE'), 'Originating file'],
'NCOMBINE': [1, 'Number of raw science data'],
'TRACE1': [readkey3(readhdr(nameout), 'ARCFILE'),
'Originating file']})
ntt.util.airmass(nameout) # phase 3 definitions
ntt.util.writeinthelog('\n', './logNTT.txt')
ntt.util.writeinthelog(
'image= ' + str(img) + ' output= ' + str(nameout) + '\n', './logNTT.txt')
ntt.util.writeinthelog(
'bias= ' + str(_bias) + ', flat= ' + str(_flat) + '\n', './logNTT.txt')
ntt.util.writeinthelog('\n', './logNTT.txt')
if nameout not in outputfile:
outputfile.append(nameout)
except:
ntt.util.writeinthelog(
'image ' + str(img) + ' probably corrupted\n', './logNTT.txt')
if _dobadpixel:
if not badpixelmask:
badpixelmask = 'bad_pixel_mask.fits'
delete(badpixelmask)
os.system('cp ' + ntt.__path__[0] + '/archive/' + str(
_instrume) + '/badpixels/badpixel.fits ' + badpixelmask)
iraf.proto.fixpix(images=nameout, masks=badpixelmask,
linterp='INDEF', cinterp='INDEF', verbose='no')
ntt.util.updateheader(
nameout, 0, {'FIXPIX': [badpixelmask, '']})
ntt.util.writeinthelog('image ' + str(nameout) + ' bad pixel corrected with ' + badpixelmask + '\n',
'./logNTT.txt')
print '\n### bad pixel mask correction ..... done'
else:
ntt.util.writeinthelog(
'image ' + str(nameout) + ' bad pixel NOT corrected\n', './logNTT.txt')
if _cosmic:
try:
print '\n### cosmic ..... '
ntt.cosmics.lacos_im(nameout, _output='', gain=_gain, readn=_rdnoise, xorder=9, yorder=9,
sigclip=4.5, sigfrac=0.5, objlim=1, skyval=0, niter=0, verbose=True,
interactive=False)
ntt.util.updateheader(nameout, 0, {
'LACOSMIC': [True, 'TRUE if Laplacian cosmic ray rejection has been applied to the image']})
print '\n### cosmic ..... removed '
except Exception, e:
print e
else:
ntt.util.updateheader(nameout, 0, {
'LACOSMIC': [False, 'TRUE if Laplacian cosmic ray rejection has been applied to the image']})
try:
##########################
sexvec = ntt.efoscastrodef.sextractor(nameout)
for cat in ['2mass', 'usnoa2', 'usnob1']:
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = ntt.efoscastrodef.efoscastroloop(
[nameout], cat, False, 40, 40, 100, 'rxyscale', 100, 30, sexvec, True, 10, method)
if rmsx3 <= 2 and rmsy3 <= 2:
break
if rmsx3 > 2 and rmsy3 > 2:
for cat in ['2mass', 'usnoa2', 'usnob1']:
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = ntt.efoscastrodef.efoscastroloop(
[nameout], cat, False, 20, int(20), int(50), 'rxyscale', 100, 30, sexvec, True, 5, method)
if rmsx3 <= 2 and rmsy3 <= 2:
break
if rmsx3 > 2 and rmsy3 > 2:
for cat in ['2mass', 'usnoa2', 'usnob1']:
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = ntt.efoscastrodef.efoscastroloop(
[nameout], cat, False, int(10), int(10),
int(25), 'rxyscale', 100, 30, sexvec, True, int(3), method)
##########################
astrostring = str(rmsx3) + ' ' + str(rmsy3) + ' ' + str(num3)
ntt.util.updateheader(
nameout, 0, {'ASTROMET': [astrostring, 'rmsx rmsy nstars']})
print '\n### check astrometry: fine \n### rmsx rmsy nstars: ' + astrostring
except Exception, e:
print e
rmsx3, rmsy3, num3, fwhmgess, ellgess, ccc, rasys3, decsys3, magsat3 = '', '', '', '', '', '', '', '', ''
print '\n### problem with astrometry, do you have network ? '
if fwhmgess and fwhmgess < 99:
ntt.util.updateheader(nameout, 0, {'PSF_FWHM': [fwhmgess, 'Spatial resolution (arcsec)'],
'ELLIPTIC': [ellgess, 'Average ellipticity of point sources'],
'CRDER1': [(1 / sqrt(2.)) * float(rmsx3) * (1. / 3600.),
'Random error (degree)'],
'CRDER2': [(1 / sqrt(2.)) * float(rmsy3) * (1. / 3600.),
'Random error (degree)'],
'CUNIT1': ['deg', 'unit of the coord. trans.'],
'CUNIT2': ['deg', 'unit of the coord. trans.'],
'CSYER1': [rasys3, 'Systematic error (RA_m - Ra_ref)'],
'CSYER2': [decsys3, 'Systematic error (DEC_m - DEC_ref)']})
else:
ntt.util.updateheader(nameout, 0, {'PSF_FWHM': [9999., 'FHWM (arcsec) - computed with sectractor'],
'ELLIPTIC': [9999., 'ellipticity of point sources (1-b/a)'],
'CRDER1': [9999., 'Random error in axis 1'],
'CRDER2': [9999., 'Random error in axis 2'],
'CUNIT1': ['deg', 'unit of the coord. trans.'],
'CUNIT2': ['deg', 'unit of the coord. trans.'],
'CSYER1': [9999., 'Systematic error (RA_m - Ra_ref)'],
'CSYER2': [9999., 'Systematic error (DEC_m - DEC_ref)']})
try:
result = ntt.efoscastrodef.zeropoint(
nameout, _system, method, False, False)
except:
result = ''
if result:
if os.path.isfile(re.sub('.fits', '.ph', nameout)):
if re.sub('.fits', '.ph', nameout) not in outputfile:
outputfile.append(
re.sub('.fits', '.ph', nameout))
print '\n### zeropoint ..... done'
for ll in result:
valore = '%3.3s %6.6s %6.6s' % (
str(ll), str(result[ll][1]), str(result[ll][0]))
print '### ', valore
ntt.util.updateheader(
nameout, 0, {'zp' + ll: [str(valore), '']})
if magsat3:
if readkey3(readhdr(nameout), 'FLUXCAL') == 'ABSOLUTE':
try:
ntt.util.updateheader(nameout, 0, {
'ABMAGSAT': [float(magsat3) + float(readkey3(readhdr(nameout)), 'PHOTZP'),
'Saturation limit for point sources (AB mags)']})
except:
ntt.util.updateheader(nameout, 0, {
'ABMAGSAT': [float(magsat3), 'Saturation limit for point sources (AB mags)']})
else:
ntt.util.updateheader(nameout, 0, {
'ABMAGSAT': [float(magsat3), 'Saturation limit for point sources (AB mags)']})
else:
ntt.util.updateheader(nameout, 0, {'ABMAGSAT': [
9999., 'Saturation limit for point sources (AB mags)']})
maglim = ntt.util.limmag(nameout)
if maglim:
ntt.util.updateheader(nameout, 0,
{'ABMAGLIM': [maglim, '5-sigma limiting AB magnitude for point sources']})
else:
ntt.util.updateheader(nameout, 0,
{'ABMAGLIM': [9999., '5-sigma limiting AB magnitude for point sources']})
if readkey3(readhdr(nameout), 'filter') in ['i705']:
try:
nameout, maskname = ntt.efoscphotredudef.fringing2(
nameout, fringingmask, _interactive, False)
if nameout not in outputfile:
outputfile.append(nameout)
if maskname not in outputfile:
outputfile.append(maskname)
except:
ntt.util.writeinthelog(
'image ' + str(nameout) + ' probably corrupted\n', './logNTT.txt')
print '\n### problem with fringing correction'