def correctobject(img,coordinatefile):
import os,re,sys,string
from numpy import arccos, sin,cos,pi,argmin
import lsc
from lsc.util import readstandard, readhdr,readkey3, updateheader
scal=pi/180.
std,rastd,decstd,magstd=readstandard(coordinatefile)
img=re.sub('\n','',img)
hdr=readhdr(img)
_ra=readkey3(hdr,'RA')
_dec=readkey3(hdr,'DEC')
dd=arccos(sin(_dec*scal)*sin(decstd*scal)+cos(_dec*scal)*cos(decstd*scal)*cos((_ra-rastd)*scal))*((180/pi)*3600)
if min(dd)<200:
updateheader(img,0,{'OBJECT':(std[argmin(dd)],'Original target.')})
aa,bb,cc=rastd[argmin(dd)],decstd[argmin(dd)],std[argmin(dd)]
else: aa,bb,cc='','',''
return aa,bb,cc
def limmag(img):
import lsc
from lsc.util import readhdr
hdr = readhdr(img)
_ZP = readkey3(hdr, 'PHOTZP')
_gain = readkey3(hdr, 'gain')
_exptime = readkey3(hdr, 'exptime')
_fwhm = readkey3(hdr, 'PSF_FWHM')
_mbkg = readkey3(hdr, 'MBKG') # background from sextractor
_instrume = readkey3(hdr, 'instrume')
check = 1
if not _ZP: check = 0
if not _gain: check = 0
if not _fwhm: check = 0
if not _mbkg: check = 0
else:
if _mbkg <= 0: _mbkg = 0
if check == 1:
# formula from McLean 1997)
from numpy import pi, log10
if _instrume == 'efosc': ps = readkey3(hdr, 'binx') * .12
else: ps = 0.288
n = pi * ((_fwhm / ps)**2)
sn = 5 # signal to noise
maglim = _ZP - 2.5 * log10(sn * (1 / _gain) *
((n * _mbkg / _exptime)**(.5)))
return maglim
else:
return ''
def checksnlist(img,listfile):
import lsc
import string
from lsc.util import readkey3,readhdr
from numpy import cos,sin,arccos,pi, argmin
scal=pi/180.
std,rastd,decstd,magstd=lsc.util.readstandard(listfile)
hdrt=readhdr(img)
_ra=readkey3(hdrt,'RA')
_dec=readkey3(hdrt,'DEC')
_object=readkey3(hdrt,'object')
_xdimen=readkey3(hdrt,'XDIM')
_ydimen=readkey3(hdrt,'YDIM')
if not _xdimen: _xdimen=readkey3(hdrt,'NAXIS1')
if not _ydimen: _ydimen=readkey3(hdrt,'NAXIS2')
dd=arccos(sin(_dec*scal)*sin(decstd*scal)+cos(_dec*scal)*cos(decstd*scal)*cos((_ra-rastd)*scal))*((180/pi)*3600)
lll=[str(rastd[argmin(dd)])+' '+str(decstd[argmin(dd)])]
from pyraf import iraf
bbb=iraf.wcsctran('STDIN','STDOUT',img+'[0]',Stdin=lll,inwcs='world',units='degrees degrees',outwcs='logical',columns='1 2',formats='%10.5f %10.5f',Stdout=1)[3]
if 'INDEF' not in bbb and float(string.split(bbb)[0])<=_xdimen and float(string.split(bbb)[1])<=_ydimen and float(string.split(bbb)[0])>=0 and float(string.split(bbb)[1])>=0:
#print str(std[argmin(dd)])+' in the field '+str(bbb)
_RA=rastd[argmin(dd)]
_DEC=decstd[argmin(dd)]
_SN=std[argmin(dd)]
else:
#print 'out '+str(bbb)
_RA,_DEC,_SN='','',''
return _RA,_DEC,_SN
def limmag(img):
import lsc
from lsc.util import readhdr
hdr=readhdr(img)
_ZP=readkey3(hdr,'PHOTZP')
_gain=readkey3(hdr,'gain')
_exptime=readkey3(hdr,'exptime')
_fwhm=readkey3(hdr,'PSF_FWHM')
_mbkg=readkey3(hdr,'MBKG') # background from sextractor
_instrume=readkey3(hdr,'instrume')
check=1
if not _ZP: check=0
if not _gain: check=0
if not _fwhm: check=0
if not _mbkg: check=0
else:
if _mbkg<=0: _mbkg=0
if check==1:
# formula from McLean 1997)
from numpy import pi,log10
if _instrume=='efosc': ps=readkey3(hdr,'binx')*.12
else: ps=0.288
n=pi*((_fwhm/ps)**2)
sn=5 # signal to noise
maglim=_ZP -2.5 * log10(sn * (1/_gain) * ((n*_mbkg/_exptime)**(.5)) )
return maglim
else: return ''
def correctobject(img, coordinatefile):
import os, re, sys, string
from numpy import arccos, sin, cos, pi, argmin
import lsc
from lsc.util import readstandard, readhdr, readkey3, updateheader
scal = pi / 180.
std, rastd, decstd, magstd = readstandard(coordinatefile)
img = re.sub('\n', '', img)
hdr = readhdr(img)
_ra = readkey3(hdr, 'RA')
_dec = readkey3(hdr, 'DEC')
dd = arccos(
sin(_dec * scal) * sin(decstd * scal) +
cos(_dec * scal) * cos(decstd * scal) * cos((_ra - rastd) * scal)) * (
(180 / pi) * 3600)
if min(dd) < 200:
updateheader(img, 0, {'OBJECT': (std[argmin(dd)], 'Original target.')})
aa, bb, cc = rastd[argmin(dd)], decstd[argmin(dd)], std[argmin(dd)]
else:
aa, bb, cc = '', '', ''
return aa, bb, cc
def name_duplicate(img,nome,ext): ###########################
import re,string,os,glob
import lsc
from lsc.util import readhdr,readkey3, delete
dimg=readkey3(readhdr(img),'DATE-OBS')
listafile=glob.glob(nome+'_?'+ext+'.fits')+glob.glob(nome+'_??'+ext+'.fits')
if len(listafile) == 0: nome = nome+"_1"+ext+'.fits'
else:
date=[]
for l in listafile:
date.append(readkey3(readhdr(l),'DATE-OBS'))
if dimg in date:
nome=listafile[date.index(dimg)]
# if overwrite:
# delete(nome)
else:
n=1
while nome+'_'+str(n)+str(ext)+'.fits' in listafile:
n=n+1
nome=nome+'_'+str(n)+str(ext)+'.fits'
return nome
def airmass(img, overwrite=True, _observatory='lasilla'):
import lsc
from lsc.util import readhdr, readkey3, delete, updateheader
from pyraf import iraf
iraf.astutil(_doprint=0)
hdr = readhdr(img)
if readkey3(hdr, 'UTC'):
_UT = (readkey3(hdr, 'UTC') + (readkey3(hdr, 'exptime') / 2)) / 3600
_date = readkey3(hdr, 'date-obs')
_date = _date[0:4] + '-' + _date[4:6] + '-' + _date[6:8]
_RA = readkey3(hdr, 'RA') / 15
_DEC = readkey3(hdr, 'DEC')
f = file('airmass.txt', 'w')
f.write('mst = mst ("' + str(_date) + '",' + str(_UT) + ', obsdb ("' +
str(_observatory) + '", "longitude"))\n')
f.write('air = airmass (' + str(_RA) + ',' + str(_DEC) +
',mst, obsdb ("' + str(_observatory) + '", "latitude"))\n')
f.write('print(air)\n')
f.close()
_air = iraf.astcalc(image=img, command="airmass.txt", Stdout=1)[0]
try:
_air = float(_air)
except:
_air = 999
delete('airmass.txt')
if overwrite and _air < 99.:
updateheader(
img, 0,
{'AIRMASS': (_air, 'mean airmass computed with astcalc')})
else:
_air = ''
return _air
def airmass(img,overwrite=True,_observatory='lasilla'):
import lsc
from lsc.util import readhdr,readkey3, delete, updateheader
from pyraf import iraf
iraf.astutil(_doprint=0)
hdr=readhdr(img)
if readkey3(hdr,'UTC'):
_UT=(readkey3(hdr,'UTC')+(readkey3(hdr,'exptime')/2))/3600
_date=readkey3(hdr,'date-obs')
_date=_date[0:4]+'-'+_date[4:6]+'-'+_date[6:8]
_RA=readkey3(hdr,'RA')/15
_DEC=readkey3(hdr,'DEC')
f = file('airmass.txt','w')
f.write('mst = mst ("'+str(_date)+'",'+str(_UT)+', obsdb ("'+str(_observatory)+'", "longitude"))\n')
f.write('air = airmass ('+str(_RA)+','+str(_DEC)+',mst, obsdb ("'+str(_observatory)+'", "latitude"))\n')
f.write('print(air)\n')
f.close()
_air=iraf.astcalc(image=img, command="airmass.txt",Stdout=1)[0]
try: _air=float(_air)
except: _air=999
delete('airmass.txt')
if overwrite and _air<99.:
updateheader(img,0,{'AIRMASS':(_air,'mean airmass computed with astcalc')})
else: _air=''
return _air
def name_duplicate(img, nome, ext): ###########################
import re, string, os, glob
import lsc
from lsc.util import readhdr, readkey3, delete
dimg = readkey3(readhdr(img), 'DATE-OBS')
listafile = glob.glob(nome + '_?' + ext +
'.fits') + glob.glob(nome + '_??' + ext + '.fits')
if len(listafile) == 0: nome = nome + "_1" + ext + '.fits'
else:
date = []
for l in listafile:
date.append(readkey3(readhdr(l), 'DATE-OBS'))
if dimg in date:
nome = listafile[date.index(dimg)]
# if overwrite:
# delete(nome)
else:
n = 1
while nome + '_' + str(n) + str(ext) + '.fits' in listafile:
n = n + 1
nome = nome + '_' + str(n) + str(ext) + '.fits'
return nome
def checksnlist(img, listfile):
import lsc
import string
from lsc.util import readkey3, readhdr
from numpy import cos, sin, arccos, pi, argmin
scal = pi / 180.
std, rastd, decstd, magstd = lsc.util.readstandard(listfile)
hdrt = readhdr(img)
_ra = readkey3(hdrt, 'RA')
_dec = readkey3(hdrt, 'DEC')
_object = readkey3(hdrt, 'object')
_xdimen = readkey3(hdrt, 'XDIM')
_ydimen = readkey3(hdrt, 'YDIM')
if not _xdimen: _xdimen = readkey3(hdrt, 'NAXIS1')
if not _ydimen: _ydimen = readkey3(hdrt, 'NAXIS2')
dd = arccos(
sin(_dec * scal) * sin(decstd * scal) +
cos(_dec * scal) * cos(decstd * scal) * cos((_ra - rastd) * scal)) * (
(180 / pi) * 3600)
lll = [str(rastd[argmin(dd)]) + ' ' + str(decstd[argmin(dd)])]
from pyraf import iraf
bbb = iraf.wcsctran('STDIN',
'STDOUT',
img + '[0]',
Stdin=lll,
inwcs='world',
units='degrees degrees',
outwcs='logical',
columns='1 2',
formats='%10.5f %10.5f',
Stdout=1)[3]
if 'INDEF' not in bbb and float(string.split(bbb)[0]) <= _xdimen and float(
string.split(bbb)[1]) <= _ydimen and float(
string.split(bbb)[0]) >= 0 and float(
string.split(bbb)[1]) >= 0:
#print str(std[argmin(dd)])+' in the field '+str(bbb)
_RA = rastd[argmin(dd)]
_DEC = decstd[argmin(dd)]
_SN = std[argmin(dd)]
else:
#print 'out '+str(bbb)
_RA, _DEC, _SN = '', '', ''
return _RA, _DEC, _SN
def Docosmic(img, _sigclip=5.5, _sigfrac=0.2, _objlim=4.5):
import time
start = time.time()
import lsc
import re, os, string
import numpy as np
import tempfile
ar, hd = fits.getdata(img, header=True)
if 'TELID' in hd:
_tel = hd['TELID']
elif 'telescop' in hd:
_tel = hd['telescop']
else:
_tel = 'extdata'
if _tel in ['fts', 'ftn']:
temp_file0 = next(tempfile._get_candidate_names())
lsc.delete(temp_file0)
out_fits = fits.PrimaryHDU(header=hd, data=ar)
try:
out_fits.scale('float32', bzero=0, bscale=1)
except TypeError as e:
print "FITS rescaling failed (but it probably doesn't matter). See Astropy Issue #5955."
print e
out_fits.writeto(temp_file0, overwrite=True, output_verify='fix')
ar = fits.getdata(temp_file0)
lsc.delete(temp_file0)
gain = hd['GAIN']
sat = 35000
rdnoise = hd['RDNOISE']
else:
if 'gain' in hd:
gain = hd['GAIN']
else:
print 'warning GAIN not found'
gain = 1
if 'saturate' in hd:
sat = hd['SATURATE']
else:
print 'warning SATURATE not found'
sat = 60000
if 'RDNOISE' in hd:
rdnoise = hd['RDNOISE']
else:
print 'warning RDNOISE not found'
rdnoise = 1
if '-e91.' in img:
ar[ar < readkey3(hd, 'datamin')] = sat
_pssl = 0.
else:
# need to trick LACosmic into using the right sigma for a sky-subtracted image
med = np.median(ar) # median pixel of image (in ADU)
noise = 1.4826 * np.median(
np.abs(ar -
med)) # using median absolute deviation instead of sigma
_pssl = gain * noise**2 - rdnoise**2 / gain - med # previously subtracted sky level
ar[ar <
-_pssl] = sat # change (what will be) negative values to saturated
print 'gain sat noise sigclip objlim sigfrac pssl'
print '{:<7.1f} {:<7.0f} {:<7.1f} {:<7.1f} {:<7.0f} {:<7.1f} {:<7.2f}'.format(
gain, sat, rdnoise, _sigclip, _objlim, _sigfrac, _pssl)
niter = 1
c = lsc.cosmics.cosmicsimage(ar,
pssl=_pssl,
gain=gain,
readnoise=rdnoise,
sigclip=5,
sigfrac=0.3,
objlim=5,
satlevel=sat)
c.run(maxiter=niter)
out = re.sub('.fits', '.clean.fits', string.split(img, '/')[-1])
outmask = re.sub('.fits', '.mask.fits', string.split(img, '/')[-1])
outsat = re.sub('.fits', '.sat.fits', string.split(img, '/')[-1])
out1 = c.cleanarray
out2 = c.cleanarray - c.rawarray
out3 = c.getsatstars()
out_fits = fits.PrimaryHDU(header=hd, data=out1)
out_fits.writeto(out, overwrite=True, output_verify='fix')
# we are going to register the mask for the template image,
# so it makes sense to save it as a float instead of an int
if 'temp' in img: pixtype = 'float32'
else: pixtype = 'uint8'
out_fits = fits.PrimaryHDU(header=hd, data=(out2 != 0).astype(pixtype))
out_fits.writeto(outmask, overwrite=True, output_verify='fix')
out_fits = fits.PrimaryHDU(header=hd, data=(out3 != 0).astype('uint8'))
out_fits.writeto(outsat, overwrite=True, output_verify='fix')
print 'time to do cosmic ray rejection:', time.time() - start
return out, outmask, outsat
def updateDatabase(tarfile):
import string, os, re, math, sys, shutil, glob, socket, pickle
import MySQLdb
host = socket.gethostname()
import ntt
from ntt.util import readkey3, readhdr
from mysqldef import updatevalue, getvaluefromarchive, dbConnect, getfromdataraw, ingestredu
os.system('tar -zxvf ' + str(tarfile))
pklfile = re.sub('tar.gz', 'pkl', tarfile)
if glob.glob(pklfile):
pkl_file = open(pklfile, 'rb')
mydict2 = pickle.load(pkl_file)
pkl_file.close()
fitsfile = []
for i in mydict2:
try:
command = mydict2[i]['command']
if command == 'ingestredu':
lista = mydict2[i]['lista']
instrument = mydict2[i]['instument']
output = mydict2[i]['output']
ingestredu(lista)
if output not in fitsfile: fitsfile.append(output)
elif command == 'getvaluefromarchive':
table = mydict2[i]['table']
column = mydict2[i]['column']
value = mydict2[i]['value']
_input = mydict2[i]['input']
hh = getvaluefromarchive(table, column, _input, value)
elif command == 'updatevalue':
table = mydict2[i]['table']
column = mydict2[i]['column']
value = mydict2[i]['value']
_input = mydict2[i]['input']
if value != 'voce':
updatevalue(table, column, value, _input)
else:
if column == 'voce':
for voce in hh:
if voce != 'id' and voce != 'filename':
updatevalue(table, voce, hh[voce], _input)
else:
updatevalue(table, column, hh[column], _input)
else:
print 'warning: command not recognise'
except:
print '#####################'
print mydict2[i]
print 'problems'
print fitsfile
dir1 = '/data/obsdata/y20' + string.split(tarfile, '_')[2][-2:] + '/'
dir2 = '/data/obsdata/y20' + string.split(
tarfile, '_')[2][-2:] + '/' + string.split(tarfile, '_')[2] + '/'
if not os.path.isdir(dir1): os.mkdir(dir1)
if not os.path.isdir(dir2): os.mkdir(dir2)
for img in fitsfile:
arcfile = readkey3(readhdr(img), 'ARCFILE')
print arcfile
try:
bbbb = getvaluefromarchive('datarawNTT', 'filename', arcfile,
'*')
except:
bbbb = ''
if bbbb:
directoryraw = bbbb['filepath']
directoryreduced = re.sub('raw/', '',
directoryraw) + 'reduced/'
else:
dir3 = dir2 + string.split(tarfile,
'_')[3] + '_' + string.split(
tarfile, '_')[4] + '/'
if not os.path.isdir(dir3): os.mkdir(dir3)
directoryreduced = dir3 + 'reduced/'
if not os.path.isdir(directoryreduced): os.mkdir(directoryreduced)
os.system('cp ' + img + ' ' + directoryreduced)
updatevalue('redulogNTT', 'filepath', directoryreduced, img)
updatevalue('datareduNTT', 'filepath', directoryreduced, img)
def updateDatabase(tarfile):
import string,os,re,math,sys,shutil,glob,socket,pickle
import MySQLdb
host=socket.gethostname()
import ntt
from ntt.util import readkey3,readhdr
from mysqldef import updatevalue, getvaluefromarchive, dbConnect,getfromdataraw,ingestredu
os.system('tar -zxvf '+str(tarfile))
pklfile=re.sub('tar.gz','pkl',tarfile)
if glob.glob(pklfile):
pkl_file = open(pklfile, 'rb')
mydict2 = pickle.load(pkl_file)
pkl_file.close()
fitsfile=[]
for i in mydict2:
try:
command=mydict2[i]['command']
if command=='ingestredu':
lista=mydict2[i]['lista']
instrument=mydict2[i]['instument']
output=mydict2[i]['output']
ingestredu(lista)
if output not in fitsfile: fitsfile.append(output)
elif command=='getvaluefromarchive':
table=mydict2[i]['table']
column=mydict2[i]['column']
value=mydict2[i]['value']
_input=mydict2[i]['input']
hh=getvaluefromarchive(table,column,_input,value)
elif command=='updatevalue':
table=mydict2[i]['table']
column=mydict2[i]['column']
value=mydict2[i]['value']
_input=mydict2[i]['input']
if value!='voce':
updatevalue(table,column,value,_input)
else:
if column=='voce':
for voce in hh:
if voce!='id' and voce!='filename':
updatevalue(table,voce,hh[voce],_input)
else:
updatevalue(table,column,hh[column],_input)
else:
print 'warning: command not recognise'
except:
print '#####################'
print mydict2[i]
print 'problems'
print fitsfile
dir1='/data/obsdata/y20'+string.split(tarfile,'_')[2][-2:]+'/'
dir2='/data/obsdata/y20'+string.split(tarfile,'_')[2][-2:]+'/'+string.split(tarfile,'_')[2]+'/'
if not os.path.isdir(dir1): os.mkdir(dir1)
if not os.path.isdir(dir2): os.mkdir(dir2)
for img in fitsfile:
arcfile=readkey3(readhdr(img),'ARCFILE')
print arcfile
try:
bbbb=getvaluefromarchive('datarawNTT','filename',arcfile,'*')
except: bbbb=''
if bbbb:
directoryraw=bbbb['filepath']
directoryreduced=re.sub('raw/','',directoryraw)+'reduced/'
else:
dir3=dir2+string.split(tarfile,'_')[3]+'_'+string.split(tarfile,'_')[4]+'/'
if not os.path.isdir(dir3): os.mkdir(dir3)
directoryreduced=dir3+'reduced/'
if not os.path.isdir(directoryreduced): os.mkdir(directoryreduced)
os.system('cp '+img+' '+directoryreduced)
updatevalue('redulogNTT','filepath',directoryreduced,img)
updatevalue('datareduNTT','filepath',directoryreduced,img)
def Docosmic(img,_sigclip=5.5,_sigfrac=0.2,_objlim=4.5):
import time
start=time.time()
import lsc
import re,os,string
import numpy as np
import tempfile
ar, hd = fits.getdata(img, header=True)
if 'TELID' in hd:
_tel=hd['TELID']
elif 'telescop' in hd:
_tel = hd['telescop']
else:
_tel='extdata'
if _tel in ['fts','ftn']:
temp_file0 = next(tempfile._get_candidate_names())
lsc.delete(temp_file0)
out_fits = fits.PrimaryHDU(header=hd,data=ar)
try:
out_fits.scale('float32',bzero=0,bscale=1)
except TypeError as e:
print "FITS rescaling failed (but it probably doesn't matter). See Astropy Issue #5955."
print e
out_fits.writeto(temp_file0, overwrite=True, output_verify='fix')
ar = fits.getdata(temp_file0)
lsc.delete(temp_file0)
gain = hd['GAIN']
sat = 35000
rdnoise = hd['RDNOISE']
else:
if 'gain' in hd:
gain = hd['GAIN']
else:
print 'warning GAIN not found'
gain = 1
if 'saturate' in hd:
sat = hd['SATURATE']
else:
print 'warning SATURATE not found'
sat = 60000
if 'RDNOISE' in hd:
rdnoise = hd['RDNOISE']
else:
print 'warning RDNOISE not found'
rdnoise = 1
if '-e91.' in img:
ar[ar < readkey3(hd, 'datamin')] = sat
_pssl = 0.
else:
# need to trick LACosmic into using the right sigma for a sky-subtracted image
med = np.median(ar) # median pixel of image (in ADU)
noise = 1.4826*np.median(np.abs(ar - med)) # using median absolute deviation instead of sigma
_pssl = gain*noise**2 - rdnoise**2/gain - med # previously subtracted sky level
ar[ar < -_pssl] = sat # change (what will be) negative values to saturated
print 'gain sat noise sigclip objlim sigfrac pssl'
print '{:<7.1f} {:<7.0f} {:<7.1f} {:<7.1f} {:<7.0f} {:<7.1f} {:<7.2f}'.format(gain, sat, rdnoise, _sigclip, _objlim, _sigfrac, _pssl)
niter = 1
c = lsc.cosmics.cosmicsimage(ar, pssl=_pssl, gain=gain, readnoise=rdnoise, sigclip=5, sigfrac=0.3 , objlim=5, satlevel=sat)
c.run(maxiter = niter)
out=re.sub('.fits','.clean.fits',string.split(img,'/')[-1])
outmask=re.sub('.fits','.mask.fits',string.split(img,'/')[-1])
outsat=re.sub('.fits','.sat.fits',string.split(img,'/')[-1])
out1=c.cleanarray
out2=c.cleanarray-c.rawarray
out3=c.getsatstars()
out_fits = fits.PrimaryHDU(header=hd,data=out1)
out_fits.writeto(out, overwrite=True, output_verify='fix')
# we are going to register the mask for the template image,
# so it makes sense to save it as a float instead of an int
if 'temp' in img: pixtype = 'float32'
else: pixtype = 'uint8'
out_fits = fits.PrimaryHDU(header=hd,data=(out2!=0).astype(pixtype))
out_fits.writeto(outmask, overwrite=True, output_verify='fix')
out_fits = fits.PrimaryHDU(header=hd,data=(out3!=0).astype('uint8'))
out_fits.writeto(outsat, overwrite=True, output_verify='fix')
print 'time to do cosmic ray rejection:', time.time()-start
return out,outmask,outsat
def ingestredu(imglist, force='no', dataredutable='photlco', filetype=1):
import string, re, os, sys
import lsc
from lsc.util import readkey3, readhdr
from datetime import datetime
hostname, username, passwd, database = lsc.mysqldef.getconnection('lcogt2')
conn = lsc.mysqldef.dbConnect(hostname, username, passwd, database)
for fullpath in imglist:
path, img = os.path.split(fullpath)
path += '/'
exist = lsc.mysqldef.getfromdataraw(conn,
dataredutable,
'filename',
string.split(img, '/')[-1],
column2='filename')
exist2 = lsc.mysqldef.getfromdataraw(conn,
'photlcoraw',
'filename',
string.split(img, '/')[-1],
column2='filename, groupidcode')
if exist2:
print exist2
_groupidcode = exist2[0]['groupidcode']
else:
_groupidcode = ''
if exist:
if force == 'yes':
print img, database
lsc.mysqldef.deleteredufromarchive(
string.split(img, '/')[-1], dataredutable)
print 'delete line from ' + str(database)
exist = lsc.mysqldef.getfromdataraw(conn,
dataredutable,
'filename',
string.split(img, '/')[-1],
column2='filename')
if not exist or force == 'update':
hdr = readhdr(fullpath)
_targetid = lsc.mysqldef.targimg(fullpath)
try:
_tracknumber = int(readkey3(hdr, 'TRACKNUM'))
except:
_tracknumber = 0
if hdr.get('TELESCOP'):
_tel = hdr.get('TELESCOP')
else:
_tel = ''
if _tel in ['Faulkes Telescope South', 'fts']:
_tel = '2m0-02'
elif _tel in ['Faulkes Telescope North', 'ftn']:
_tel = '2m0-01'
_inst = hdr.get('instrume')
dictionary={'dateobs':readkey3(hdr,'date-obs'),'dayobs':readkey3(hdr,'DAY-OBS'),'filename':img,'filepath':path,'filetype':filetype,'targetid':_targetid,\
'exptime':readkey3(hdr,'exptime'), 'filter':readkey3(hdr,'filter'),'mjd':readkey3(hdr,'mjd'),'tracknumber':_tracknumber,'groupidcode':_groupidcode,\
'telescope':_tel,'airmass':readkey3(hdr,'airmass'),'objname':readkey3(hdr,'object'),'ut':readkey3(hdr,'ut'),\
'wcs':readkey3(hdr,'wcserr'),'instrument':_inst,'ra0':readkey3(hdr,'RA'),'dec0':readkey3(hdr,'DEC')}
_telid = lsc.mysqldef.getfromdataraw(conn,
'telescopes',
'name',
_tel,
column2='id')
if not _telid:
print 'Telescope ', _tel, ' not recognized. Adding to telescopes table.'
# the short name is needed to calibrate the magnitude with extinction
lsc.mysqldef.insert_values(conn, 'telescopes', {
'name': _tel,
'shortname': readkey3(hdr, 'SITEID')
})
_telid = lsc.mysqldef.getfromdataraw(conn,
'telescopes',
'name',
_tel,
column2='id')
telid = _telid[0]['id']
dictionary['telescopeid'] = str(telid)
_instid = lsc.mysqldef.getfromdataraw(conn,
'instruments',
'name',
_inst,
column2='id')
if not _instid:
print 'Instrument ', _inst, ' not recognized. Adding to instruments table.'
lsc.mysqldef.insert_values(conn, 'instruments', {
'name': _inst,
'type': guess_instrument_type(_inst)
})
_instid = lsc.mysqldef.getfromdataraw(conn,
'instruments',
'name',
_inst,
column2='id')
instid = _instid[0]['id']
dictionary['instrumentid'] = str(instid)
dictionary['lastunpacked'] = str(datetime.utcnow())
print dictionary
print 'insert reduced'
ggg = lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename',
str(img), '*')
if not ggg:
lsc.mysqldef.insert_values(conn, dataredutable, dictionary)
else:
for voce in dictionary:
lsc.mysqldef.updatevalue(dataredutable, voce,
dictionary[voce],
string.split(img, '/')[-1])
else:
print 'already ingested'
def absphot(img,_field,_catalogue,_fix,_color,rejection,_interactive,_type='fit',redo=False,show=False,cutmag=-1,database='dataredulco',_calib='sloan'):
from astropy.io import fits
import lsc
import math
import sys,re,string,os
from lsc.util import readkey3, readhdr
from numpy import array, compress, zeros, median, std, asarray, isfinite,mean
from pyraf import iraf
if show:
from pylab import ion,plot,draw,clf
import time
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.daophot(_doprint=0)
iraf.images(_doprint=0)
iraf.imcoords(_doprint=0)
iraf.proto(_doprint=0)
t = fits.open(img)
tbdata = t[1].data
hdr2=t[1].header
hdr=lsc.util.readhdr(img)
_cat=readkey3(hdr,'catalog')
_telescope=lsc.util.readkey3(hdr,'telescop')
_instrume=lsc.util.readkey3(hdr,'instrume')
_filter=lsc.util.readkey3(hdr,'filter')
_airmass=lsc.util.readkey3(hdr,'airmass')
_exptime=lsc.util.readkey3(hdr,'exptime')
_date=lsc.util.readkey3(hdr,'date-obs')
_object=lsc.util.readkey3(hdr,'object')
_ra=lsc.util.readkey3(hdr,'RA')
_dec=lsc.util.readkey3(hdr,'DEC')
print _filter
if _telescope in ['lsc','1m0-04','1m0-05','1m0-09']: kk=lsc.sites.extintion('ctio')
elif _telescope in ['elp','1m0-08']: kk=lsc.sites.extintion('mcdonald')
elif _telescope in ['cpt','1m0-12','1m0-10','1m0-13']: kk=lsc.sites.extintion('southafrica')
elif _telescope in ['ftn','Faulkes Telescope North']: kk=lsc.sites.extintion('mauna')
elif _telescope in ['1m0-03','1m0-11','coj','fts','Faulkes Telescope South']: kk=lsc.sites.extintion('siding')
if _calib not in ['sloan','sloanprime','natural','apass','']: colorefisso=lsc.sites.colfix(_instrume)
else: colorefisso=lsc.sites.colfix(_instrume,_calib)
print redo
print _cat
if _cat and not redo:
print 'already calibrated'
else:
try:
lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]):
lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
except: print 'module mysqldef not found'
column=makecatalogue([img])[_filter][img]
rasex=array(column['ra0'],float)
decsex=array(column['dec0'],float)
if _type=='fit':
magsex=array(column['smagf'],float)
magerrsex=array(column['smagerrf'],float)
elif _type=='ph':
magsex=array(column['magp3'],float)
magerrsex=array(column['merrp3'],float)
else: sys.exit(_type+' not valid (ph or fit)')
print len(rasex)
if not cutmag: cutmag=99
#else: cutmag= cutmag-2.5*math.log10(float(_exptime))
if len(compress( array(magsex) < float(cutmag) , magsex)) < 5 : cutmag=99 # not cut if only few object
rasex = compress(array(magsex,float)<=cutmag,rasex)
decsex = compress(array(magsex,float)<=cutmag,decsex)
magerrsex = compress(array(magsex,float)<=cutmag,magerrsex)
magsex = compress(array(magsex,float)<=cutmag,array(magsex))
print len(rasex)
if _interactive:
iraf.set(stdimage='imt1024')
iraf.display(re.sub('.sn2','',img),1,fill=True,Stdout=1)
vector=[]
for i in range(0,len(rasex)):
vector.append(str(rasex[i])+' '+str(decsex[i]))
xy = iraf.wcsctran('STDIN',output="STDOUT",Stdin=vector,Stdout=1,image=img,inwcs='world',units='degrees degrees',outwcs='logical',\
formats='%10.1f %10.1f',verbose='yes')[3:]
iraf.tvmark(1,'STDIN',Stdin=list(xy),mark="circle",number='yes',label='no',radii=10,nxoffse=5,nyoffse=5,color=207,txsize=2)
# raw_input('here')
if _catalogue:
######## use external catalogue
if _catalogue[0]=='/': stdcooC=lsc.lscastrodef.readtxt(_catalogue)
else: stdcooC=lsc.lscastrodef.readtxt(lsc.__path__[0]+'/standard/cat/'+_catalogue)
rastdC,decstdL=array(stdcooC['ra'],float),array(stdcooC['dec'],float)
lsc.util.delete('tmp.stdL.pix')
colonne=str(stdcooC['rapos'])+' '+str(stdcooC['decpos'])
if _catalogue[0]=='/':
iraf.wcsctran(_catalogue,'tmp.stdL.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',\
columns=colonne,formats='%10.1f %10.1f',verbose='no')
else:
iraf.wcsctran(lsc.__path__[0]+'/standard/cat/'+_catalogue,'tmp.stdL.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',\
columns=colonne,formats='%10.1f %10.1f',verbose='no')
standardpixC=lsc.lscastrodef.readtxt('tmp.stdL.pix')
if _interactive:
iraf.tvmark(1,'tmp.stdL.pix',mark="circle",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=204,txsize=2)
xstdC=standardpixC['ra']
ystdC=standardpixC['dec']
idstdC=standardpixC['id']
xstdC=compress((array(xstdC,float)<readkey3(hdr,'XDIM'))&(array(xstdC,float)>0)&(array(ystdC,float)>0)&(array(ystdC,float)<readkey3(hdr,'YDIM')),xstdC)
xstdL=xstdLL=xstdS=xstdC
standardpixL=standardpixLL=standardpixS=standardpixC
stdcooL=stdcooLL=stdcooS=stdcooC
else:
######## check if it is landolt field
stdcooL=lsc.lscastrodef.readtxt(lsc.__path__[0]+'/standard/cat/landolt.cat')
rastdL,decstdL=array(stdcooL['ra'],float),array(stdcooL['dec'],float)
lsc.util.delete('tmp.stdL.pix')
iraf.wcsctran(lsc.__path__[0]+'/standard/cat/landolt.cat','tmp.stdL.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',\
columns='1 2',formats='%10.1f %10.1f',verbose='no')
standardpixL=lsc.lscastrodef.readtxt('tmp.stdL.pix')
if _interactive:
iraf.tvmark(1,'tmp.stdL.pix',mark="circle",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=204,txsize=2)
print 'yelow circles sextractor'
xstdL=standardpixL['ra']
ystdL=standardpixL['dec']
idstdL=standardpixL['id']
xstdL=compress((array(xstdL,float)<readkey3(hdr,'XDIM'))&(array(xstdL,float)>0)&(array(ystdL,float)>0)&(array(ystdL,float)<readkey3(hdr,'YDIM')),xstdL)
######## check if it is Stetson field
stdcooLL=lsc.lscastrodef.readtxt(lsc.__path__[0]+'/standard/cat/StetsonCat.dat')
ww=asarray([i for i in range(len(stdcooLL['ra'])) if ( abs(float(stdcooLL['ra'][i])-float(_ra))<.2 and abs(float(stdcooLL['dec'][i])-_dec)<.2 )])
if len(ww)>0:
for hh in stdcooLL.keys():
if type(stdcooLL[hh])!=int:
if hh not in ['id','ra','dec']:
stdcooLL[hh]=array(array(stdcooLL[hh])[ww],float)
else:
stdcooLL[hh]=array(stdcooLL[hh])[ww]
lll=[]
for i in range(0,len(stdcooLL['ra'])): lll.append(stdcooLL['ra'][i]+' '+stdcooLL['dec'][i])
rastdLL,decstdLL=array(stdcooLL['ra'],float),array(stdcooLL['dec'],float)
lsc.util.delete('tmp.stdLL.pix')
iraf.wcsctran('STDIN','tmp.stdLL.pix',img,inwcs='world',Stdin=lll,units='degrees degrees',outwcs='logical',\
columns='1 2',formats='%10.1f %10.1f',verbose='no')
if _interactive:
iraf.tvmark(1,'tmp.stdLL.pix',mark="cross",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=204,txsize=2)
print 'red crosses Stetson'
standardpixLL={}
for ii in stdcooLL.keys(): standardpixLL[ii]=stdcooLL[ii]
standardpixLL['ra']=array(iraf.proto.fields('tmp.stdLL.pix',fields='1',Stdout=1),float) #standardpixLL['ra']
standardpixLL['dec']=array(iraf.proto.fields('tmp.stdLL.pix',fields='2',Stdout=1),float) #standardpixLL['dec']
xstdLL=array(iraf.proto.fields('tmp.stdLL.pix',fields='1',Stdout=1),float) #standardpixLL['ra']
ystdLL=array(iraf.proto.fields('tmp.stdLL.pix',fields='2',Stdout=1),float) #standardpixLL['dec']
idstdLL=standardpixLL['id']
xstdLL=compress((array(xstdLL,float)<readkey3(hdr,'XDIM'))&(array(xstdLL,float)>0)&(array(ystdLL,float)>0)&(array(ystdLL,float)<readkey3(hdr,'YDIM')),xstdLL)
######## check if it is sloan field
magsel0,magsel1=12,18
_ids=lsc.lscastrodef.sloan2file(_ra,_dec,20,float(magsel0),float(magsel1),'_tmpsloan.cat')
ascifile='_tmpsloan.cat'
stdcooS=lsc.lscastrodef.readtxt(ascifile)
rastdS,decstdS=array(stdcooS['ra'],float),array(stdcooS['dec'],float)
lsc.util.delete('tmp.stdS.pix')
iraf.wcsctran(ascifile,'tmp.stdS.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',columns='1 2',formats='%10.1f %10.1f',verbose='no')
standardpixS=lsc.lscastrodef.readtxt('tmp.stdS.pix')
if _interactive:
iraf.tvmark(1,'tmp.stdS.pix',mark="cross",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=205,txsize=2)
print 'green cross sloan'
xstdS=standardpixS['ra']
ystdS=standardpixS['dec']
idstdS=standardpixS['id']
xstdS=compress((array(xstdS,float)<readkey3(hdr,'XDIM'))&(array(xstdS,float)>0)&(array(ystdS,float)>0)&(array(ystdS,float)<readkey3(hdr,'YDIM')),xstdS)
##############################################3
if not _catalogue and len(xstdLL)>0:
xstdL=xstdLL
standardpixL=standardpixLL
stdcooL=stdcooLL
if _filter in ['U', 'B', 'V', 'R','I','Bessell-B','Bessell-V','Bessell-R','Bessell-I']:
filters={'U':'U', 'B':'B', 'V':'V', 'R':'R', 'I':'I','Bessell-B':'B','Bessell-V':'V','Bessell-R':'R','Bessell-I':'I'}
if _color:
colors=lsc.myloopdef.chosecolor(_color,False)
if not colors: colors={'U':['UB'],'B':['UB','BV'],'V':['BV','VR'],'R':['VR','RI'],'I':['RI']}
else:
colors={'U':['UB'],'B':['UB','BV'],'V':['BV','VR'],'R':['VR','RI'],'I':['RI']}
if _field=='sloan':
standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]}
print 'filters and field selected do not match'
else:
_field='landolt'
if len(xstdL)>=1:
standardpix=standardpixL
stdcoo=stdcooL
if not _catalogue:
if len(xstdLL)>0: _catalogue='StetsonCat.dat'
else: _catalogue='landolt.dat'
elif len(xstdS)>=1:
if not _catalogue: _catalogue='sdss8'
standardpix=standardpixS
stdcoo=stdcooS
stdcoo=lsc.lscastrodef.transformsloanlandolt(stdcoo)
if not _catalogue: _catalogue='sdss8'
print '\n### transform sloan in landolt'
else:
print 'landolt, but catalogue not found'
standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]}
elif _filter in ['up','gp','rp','ip','zs','SDSS-G','SDSS-R','SDSS-I','Pan-Starrs-Z']:
filters={'up':'u','ip':'i','gp':'g','rp':'r','zs':'z','SDSS-G':'g','SDSS-R':'r','SDSS-I':'i','Pan-Starrs-Z':'z'}
if _color:
colors=lsc.myloopdef.chosecolor(_color,False)
if not colors: colors={'i':['ri','iz'],'r':['gr','ri'],'g':['ug','gr'],'z':['iz'],'u':['ug']}
else:
colors={'i':['ri','iz'],'r':['gr','ri'],'g':['ug','gr'],'z':['iz'],'u':['ug']}
if _field=='landolt':
standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]}
print 'filters and field selected do not match'
else:
_field='sloan'
if len(xstdS)>=1:
if not _catalogue: _catalogue='sdss8'
standardpix=standardpixS
stdcoo=stdcooS
elif len(xstdL)>=1:
standardpix=standardpixL
stdcoo=stdcooL
stdcoo=lsc.lscastrodef.transformlandoltsloan(stdcoo)
if not _catalogue: _catalogue='landolt.dat'
print '\n### transform landolt to sloan'
else:
print 'sloan, but not in the sdss footprint'
standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]}
xstd=standardpix['ra']
ystd=standardpix['dec']
idstd=standardpix['id']
rastd,decstd=array(stdcoo['ra'],float),array(stdcoo['dec'],float)
xstd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)&(array(ystd,float)<readkey3(hdr,'YDIM')),xstd)
if len(xstd0)>1: ######## go only if standard stars are in the field ##########
magstd0={}
airmass0={}
result={}
fileph={}
print '\n### standard field: '+str(_field)
ystd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\
&(array(ystd,float)<readkey3(hdr,'YDIM')),ystd)
rastd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\
&(array(ystd,float)<readkey3(hdr,'YDIM')),rastd)
decstd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\
&(array(ystd,float)<readkey3(hdr,'YDIM')),decstd)
idstd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\
&(array(ystd,float)<readkey3(hdr,'YDIM')),idstd)
stdcoo0={}
for key in stdcoo.keys():
if key in 'ugrizUBVRI':
stdcoo0[key]=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\
&(array(ystd,float)<readkey3(hdr,'YDIM')),stdcoo[key])
###############################################################
# pos0 = standard pos1 = sextractor
distvec,pos0,pos1=lsc.lscastrodef.crossmatch(array(rastd0),array(decstd0),array(rasex),array(decsex),10)
for key in stdcoo0.keys(): stdcoo0[key]=stdcoo0[key][pos0]
rastd0=rastd0[pos0]
decstd0=decstd0[pos0]
idstd0=idstd0[pos0]
rasex=rasex[pos1]
decsex=decsex[pos1]
# after change in may 2013 mag in sn2.fits file are already at 1s
magsex=magsex[pos1]-kk[filters[_filter]]*float(_airmass) # - K x airmass
# magsex=magsex[pos1]+2.5*math.log10(float(_exptime))-kk[filters[_filter]]*float(_airmass) # mag exptime - K x airmass
#################################################################################
if _field=='landolt':
print '\n### landolt system'
for _filtlandolt in 'UBVRI':
if _filtlandolt==filters[_filter]: airmass0[_filtlandolt]= 0 #_airmass
else: airmass0[_filtlandolt]= 0
magstd0[_filtlandolt]=stdcoo0[_filtlandolt]
fileph['mU']=zeros(len(rastd0))+999
fileph['mB']=zeros(len(rastd0))+999
fileph['mV']=zeros(len(rastd0))+999
fileph['mR']=zeros(len(rastd0))+999
fileph['mI']=zeros(len(rastd0))+999
fileph['V']=magstd0['V']
fileph['BV']=array(array(magstd0['B'],float)-array(magstd0['V'],float),str)
fileph['UB']=array(array(magstd0['U'],float)-array(magstd0['B'],float),str)
fileph['VR']=array(array(magstd0['V'],float)-array(magstd0['R'],float),str)
fileph['RI']=array(array(magstd0['R'],float)-array(magstd0['I'],float),str)
elif _field=='sloan':
for _filtsloan in 'ugriz':
if _filtsloan==filters[_filter]: airmass0[_filtsloan]= 0 # _airmass
else: airmass0[_filtsloan]=0
magstd0[_filtsloan]=stdcoo0[_filtsloan]
fileph['mu']=zeros(len(rastd0))+999
fileph['mg']=zeros(len(rastd0))+999
fileph['mr']=zeros(len(rastd0))+999
fileph['mi']=zeros(len(rastd0))+999
fileph['mz']=zeros(len(rastd0))+999
fileph['r']=magstd0['r']
fileph['gr']=array(array(magstd0['g'],float)-array(magstd0['r'],float),str)
fileph['ri']=array(array(magstd0['r'],float)-array(magstd0['i'],float),str)
fileph['ug']=array(array(magstd0['u'],float)-array(magstd0['g'],float),str)
fileph['iz']=array(array(magstd0['i'],float)-array(magstd0['z'],float),str)
########################################################################################
zero=[]
magcor=[]
fil = open(re.sub('.fits','.ph',img),'w')
fil.write(str(_instrume)+' '+str(_date)+'\n')
fil.write('*** '+_object+' '+str(len(magsex))+'\n')
if _field=='landolt':
fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(1),str(1),str(1),str(1),str(1))) # exptime
fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(airmass0['U']),str(airmass0['B']),str(airmass0['V']),str(airmass0['R']),str(airmass0['I'])))
elif _field=='sloan':
fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(1),str(1),str(1),str(1),str(1))) # exptime
fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(airmass0['u']),str(airmass0['g']),str(airmass0['r']),str(airmass0['i']),str(airmass0['z'])))
for i in range(0,len(magsex)):
fileph['m'+filters[_filter]][i]=magsex[i] # instrumental mangitude of std in pos0[i]
if _field=='landolt':
stringastandard='%12.12s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s' % (idstd0[i],fileph['V'][i],fileph['BV'][i],fileph['UB'][i],\
fileph['VR'][i],fileph['RI'][i])
fil.write('%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%60.60s\n' \
% (str(fileph['mU'][i]),str(fileph['mB'][i]),str(fileph['mV'][i]),str(fileph['mR'][i]),str(fileph['mI'][i]),str(stringastandard)))
elif _field=='sloan':
stringastandard='%12.12s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s' % (idstd0[i],fileph['r'][i],fileph['gr'][i],fileph['ug'][i],\
fileph['ri'][i],fileph['iz'][i])
fil.write('%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%60.60s\n' \
% (str(fileph['mu'][i]),str(fileph['mg'][i]),str(fileph['mr'][i]),str(fileph['mi'][i]),str(fileph['mz'][i]),str(stringastandard)))
zero.append(float(float(magstd0[filters[_filter]][i]))-float(magsex[i]))
magcor.append(magsex[i])
fil.close()
if show:
import time
from pylab import ion,plot,draw
ion()
aa=mean(compress(abs(array(zero))<99,zero))
xxx=compress((abs(array(magcor))<99)&(abs(array(zero))<99),magcor)
yyy=compress((abs(array(zero))<99)&(abs(array(magcor))<99),zero)
plot(xxx,yyy,'or')
plot([min(compress(abs(array(magcor))<99,magcor)),max(compress(abs(array(magcor))<99,magcor))],[aa,aa],'-b')
draw()
print std(compress(abs(array(zero))<99,zero))
time.sleep(5)
colorvec=colors[filters[_filter]]
for col in colorvec:
col0=magstd0[col[0]]
col1=magstd0[col[1]]
colstd0=array(col0,float)-array(col1,float)
################## sex ######################
colore=[]
for i in range(0,len(pos1)): colore.append(colstd0[i])
colore1=compress(abs(array(zero))<50,array(colore))
zero1=compress(abs(array(zero))<50,array(zero))
zero2=compress(abs(array(colore1))<2,array(zero1))
colore2=compress(abs(array(colore1))<2,array(colore1))
if _fix: fisso=colorefisso[filters[_filter]+col]
else: fisso=''
if len(colore2)==0:
print 'no calibration, '+_filter+' '+_field
b,a,sa,sb=9999,9999,0,0
else:
if _interactive: a,sa,b,sb=fitcol(colore2,zero2,_filter,col,fisso)
else: a,sa,b,sb=fitcol2(colore2,zero2,_filter,col,fisso,show,rejection)
print a,sa,b,sb
result[filters[_filter]+col]=[a,sa,b,sb]
if result:
print '\n### zeropoint ..... done at airmass 0'
if _catalogue:
lsc.util.updateheader(img,0,{'CATALOG':[str(string.split(_catalogue,'/')[-1]),'catalogue source']})
stringa=''
for ll in result:
for kk in range(0,len(result[ll])):
if not isfinite(result[ll][kk]): result[ll][kk]=0.0
valore='%3.3s %6.6s %6.6s %6.6s %6.6s' % (str(ll),str(result[ll][0]),str(result[ll][2]),str(result[ll][1]),str(result[ll][3]))
print '### ',valore
lsc.util.updateheader(img,0,{'zp'+ll:[str(valore),'a b sa sb in y=a+bx']})
if ll[0]==ll[2]: num=2
elif ll[0]==ll[1]: num=1
else: sys.exit('somthing wrong with color '+ll)
print ll,num
try:
print 'zcol'+str(num),ll[1:],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1]
lsc.mysqldef.updatevalue(database,'zcol'+str(num),ll[1:],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'z'+str(num),result[ll][0],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'c'+str(num),result[ll][2],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'dz'+str(num),result[ll][1],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'dc'+str(num),result[ll][3],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]):
lsc.mysqldef.updatevalue(database,'zcol'+str(num),ll[1:],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'z'+str(num),result[ll][0],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'c'+str(num),result[ll][2],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'dz'+str(num),result[ll][1],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
lsc.mysqldef.updatevalue(database,'dc'+str(num),result[ll][3],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
if result[ll][0]!=9999:
print _catalogue
lsc.mysqldef.updatevalue(database,'zcat',string.split(_catalogue,'/')[-1],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]):
lsc.mysqldef.updatevalue(database,'zcat',string.split(_catalogue,'/')[-1],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
else:
lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.sn2.fits','.fits',img),'/')[-1])
if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]):
lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1])
except: print 'module mysqldef not found'
def ingestredu(imglist,force='no',dataredutable='photlco',filetype=1):
import string,re,os,sys
import lsc
from lsc.util import readkey3, readhdr
from datetime import datetime
hostname, username, passwd, database=lsc.mysqldef.getconnection('lcogt2')
conn = lsc.mysqldef.dbConnect(hostname, username, passwd, database)
for fullpath in imglist:
path, img = os.path.split(fullpath)
path += '/'
exist=lsc.mysqldef.getfromdataraw(conn,dataredutable,'filename', string.split(img,'/')[-1],column2='filename')
exist2=lsc.mysqldef.getfromdataraw(conn,'photlcoraw','filename', string.split(img,'/')[-1],column2='filename, groupidcode')
if exist2:
print exist2
_groupidcode=exist2[0]['groupidcode']
else:
_groupidcode=''
if exist:
if force=='yes':
print img,database
lsc.mysqldef.deleteredufromarchive(string.split(img,'/')[-1],dataredutable)
print 'delete line from '+str(database)
exist=lsc.mysqldef.getfromdataraw(conn,dataredutable,'filename', string.split(img,'/')[-1],column2='filename')
if not exist or force =='update':
hdr=readhdr(fullpath)
_targetid=lsc.mysqldef.targimg(fullpath)
try:
_tracknumber=int(readkey3(hdr,'TRACKNUM'))
except:
_tracknumber=0
if hdr.get('TELESCOP'):
_tel=hdr.get('TELESCOP')
else:
_tel=''
if _tel in ['Faulkes Telescope South','fts']:
_tel='2m0-02'
elif _tel in ['Faulkes Telescope North','ftn']:
_tel='2m0-01'
_inst=hdr.get('instrume')
dictionary={'dateobs':readkey3(hdr,'date-obs'),'dayobs':readkey3(hdr,'DAY-OBS'),'filename':img,'filepath':path,'filetype':filetype,'targetid':_targetid,\
'exptime':readkey3(hdr,'exptime'), 'filter':readkey3(hdr,'filter'),'mjd':readkey3(hdr,'mjd'),'tracknumber':_tracknumber,'groupidcode':_groupidcode,\
'telescope':_tel,'airmass':readkey3(hdr,'airmass'),'objname':readkey3(hdr,'object'),'ut':readkey3(hdr,'ut'),\
'wcs':readkey3(hdr,'wcserr'),'instrument':_inst,'ra0':readkey3(hdr,'RA'),'dec0':readkey3(hdr,'DEC')}
_telid=lsc.mysqldef.getfromdataraw(conn,'telescopes','name',_tel,column2='id')
if not _telid:
print 'Telescope ',_tel,' not recognized. Adding to telescopes table.'
lsc.mysqldef.insert_values(conn,'telescopes',{'name':_tel})
_telid=lsc.mysqldef.getfromdataraw(conn,'telescopes','name',_tel,column2='id')
telid=_telid[0]['id']
dictionary['telescopeid']=str(telid)
_instid=lsc.mysqldef.getfromdataraw(conn,'instruments','name',_inst,column2='id')
if not _instid:
print 'Instrument ',_inst,' not recognized. Adding to instruments table.'
lsc.mysqldef.insert_values(conn,'instruments',{'name':_inst})
_instid=lsc.mysqldef.getfromdataraw(conn,'instruments','name',_inst,column2='id')
instid=_instid[0]['id']
dictionary['instrumentid']=str(instid)
dictionary['lastunpacked'] = str(datetime.utcnow())
print dictionary
print 'insert reduced'
ggg=lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename',str(img), '*')
if not ggg:
lsc.mysqldef.insert_values(conn,dataredutable,dictionary)
else:
for voce in dictionary:
lsc.mysqldef.updatevalue(dataredutable,voce,dictionary[voce],string.split(img,'/')[-1])
else:
print 'already ingested'
