def limmag(img):
import lsc
from lsc.util import readhdr
hdr=readhdr(img)
_ZP=readkey3(hdr,'PHOTZP')
_gain=readkey3(hdr,'gain')
_exptime=readkey3(hdr,'exptime')
_fwhm=readkey3(hdr,'PSF_FWHM')
_mbkg=readkey3(hdr,'MBKG') # background from sextractor
_instrume=readkey3(hdr,'instrume')
check=1
if not _ZP: check=0
if not _gain: check=0
if not _fwhm: check=0
if not _mbkg: check=0
else:
if _mbkg<=0: _mbkg=0
if check==1:
# formula from McLean 1997)
from numpy import pi,log10
if _instrume=='efosc': ps=readkey3(hdr,'binx')*.12
else: ps=0.288
n=pi*((_fwhm/ps)**2)
sn=5 # signal to noise
maglim=_ZP -2.5 * log10(sn * (1/_gain) * ((n*_mbkg/_exptime)**(.5)) )
return maglim
else: return ''
def checksnlist(img,listfile):
import lsc
import string
from lsc.util import readkey3,readhdr
from numpy import cos,sin,arccos,pi, argmin
scal=pi/180.
std,rastd,decstd,magstd=lsc.util.readstandard(listfile)
hdrt=readhdr(img)
_ra=readkey3(hdrt,'RA')
_dec=readkey3(hdrt,'DEC')
_object=readkey3(hdrt,'object')
_xdimen=readkey3(hdrt,'XDIM')
_ydimen=readkey3(hdrt,'YDIM')
if not _xdimen: _xdimen=readkey3(hdrt,'NAXIS1')
if not _ydimen: _ydimen=readkey3(hdrt,'NAXIS2')
dd=arccos(sin(_dec*scal)*sin(decstd*scal)+cos(_dec*scal)*cos(decstd*scal)*cos((_ra-rastd)*scal))*((180/pi)*3600)
lll=[str(rastd[argmin(dd)])+' '+str(decstd[argmin(dd)])]
from pyraf import iraf
bbb=iraf.wcsctran('STDIN','STDOUT',img+'[0]',Stdin=lll,inwcs='world',units='degrees degrees',outwcs='logical',columns='1 2',formats='%10.5f %10.5f',Stdout=1)[3]
if 'INDEF' not in bbb and float(string.split(bbb)[0])<=_xdimen and float(string.split(bbb)[1])<=_ydimen and float(string.split(bbb)[0])>=0 and float(string.split(bbb)[1])>=0:
#print str(std[argmin(dd)])+' in the field '+str(bbb)
_RA=rastd[argmin(dd)]
_DEC=decstd[argmin(dd)]
_SN=std[argmin(dd)]
else:
#print 'out '+str(bbb)
_RA,_DEC,_SN='','',''
return _RA,_DEC,_SN
def airmass(img,overwrite=True,_observatory='lasilla'):
import lsc
from lsc.util import readhdr,readkey3, delete, updateheader
from pyraf import iraf
iraf.astutil(_doprint=0)
hdr=readhdr(img)
if readkey3(hdr,'UTC'):
_UT=(readkey3(hdr,'UTC')+(readkey3(hdr,'exptime')/2))/3600
_date=readkey3(hdr,'date-obs')
_date=_date[0:4]+'-'+_date[4:6]+'-'+_date[6:8]
_RA=readkey3(hdr,'RA')/15
_DEC=readkey3(hdr,'DEC')
f = file('airmass.txt','w')
f.write('mst = mst ("'+str(_date)+'",'+str(_UT)+', obsdb ("'+str(_observatory)+'", "longitude"))\n')
f.write('air = airmass ('+str(_RA)+','+str(_DEC)+',mst, obsdb ("'+str(_observatory)+'", "latitude"))\n')
f.write('print(air)\n')
f.close()
_air=iraf.astcalc(image=img, command="airmass.txt",Stdout=1)[0]
try: _air=float(_air)
except: _air=999
delete('airmass.txt')
if overwrite and _air<99.:
updateheader(img,0,{'AIRMASS':(_air,'mean airmass computed with astcalc')})
else: _air=''
return _air
def name_duplicate(img,nome,ext): ###########################
import re,string,os,glob
import lsc
from lsc.util import readhdr,readkey3, delete
dimg=readkey3(readhdr(img),'DATE-OBS')
listafile=glob.glob(nome+'_?'+ext+'.fits')+glob.glob(nome+'_??'+ext+'.fits')
if len(listafile) == 0: nome = nome+"_1"+ext+'.fits'
else:
date=[]
for l in listafile:
date.append(readkey3(readhdr(l),'DATE-OBS'))
if dimg in date:
nome=listafile[date.index(dimg)]
# if overwrite:
# delete(nome)
else:
n=1
while nome+'_'+str(n)+str(ext)+'.fits' in listafile:
n=n+1
nome=nome+'_'+str(n)+str(ext)+'.fits'
return nome
def correctobject(img,coordinatefile):
import os,re,sys,string
from numpy import arccos, sin,cos,pi,argmin
import lsc
from lsc.util import readstandard, readhdr,readkey3, updateheader
scal=pi/180.
std,rastd,decstd,magstd=readstandard(coordinatefile)
img=re.sub('\n','',img)
hdr=readhdr(img)
_ra=readkey3(hdr,'RA')
_dec=readkey3(hdr,'DEC')
dd=arccos(sin(_dec*scal)*sin(decstd*scal)+cos(_dec*scal)*cos(decstd*scal)*cos((_ra-rastd)*scal))*((180/pi)*3600)
if min(dd)<200:
updateheader(img,0,{'OBJECT':(std[argmin(dd)],'Original target.')})
aa,bb,cc=rastd[argmin(dd)],decstd[argmin(dd)],std[argmin(dd)]
else: aa,bb,cc='','',''
return aa,bb,cc
def updateDatabase(tarfile):
import string, os, re, math, sys, shutil, glob, socket, pickle
import MySQLdb
host = socket.gethostname()
import ntt
from ntt.util import readkey3, readhdr
from mysqldef import updatevalue, getvaluefromarchive, dbConnect, getfromdataraw, ingestredu
os.system('tar -zxvf ' + str(tarfile))
pklfile = re.sub('tar.gz', 'pkl', tarfile)
if glob.glob(pklfile):
pkl_file = open(pklfile, 'rb')
mydict2 = pickle.load(pkl_file)
pkl_file.close()
fitsfile = []
for i in mydict2:
try:
command = mydict2[i]['command']
if command == 'ingestredu':
lista = mydict2[i]['lista']
instrument = mydict2[i]['instument']
output = mydict2[i]['output']
ingestredu(lista)
if output not in fitsfile: fitsfile.append(output)
elif command == 'getvaluefromarchive':
table = mydict2[i]['table']
column = mydict2[i]['column']
value = mydict2[i]['value']
_input = mydict2[i]['input']
hh = getvaluefromarchive(table, column, _input, value)
elif command == 'updatevalue':
table = mydict2[i]['table']
column = mydict2[i]['column']
value = mydict2[i]['value']
_input = mydict2[i]['input']
if value != 'voce':
updatevalue(table, column, value, _input)
else:
if column == 'voce':
for voce in hh:
if voce != 'id' and voce != 'filename':
updatevalue(table, voce, hh[voce], _input)
else:
updatevalue(table, column, hh[column], _input)
else:
print 'warning: command not recognise'
except:
print '#####################'
print mydict2[i]
print 'problems'
print fitsfile
dir1 = '/data/obsdata/y20' + string.split(tarfile, '_')[2][-2:] + '/'
dir2 = '/data/obsdata/y20' + string.split(
tarfile, '_')[2][-2:] + '/' + string.split(tarfile, '_')[2] + '/'
if not os.path.isdir(dir1): os.mkdir(dir1)
if not os.path.isdir(dir2): os.mkdir(dir2)
for img in fitsfile:
arcfile = readkey3(readhdr(img), 'ARCFILE')
print arcfile
try:
bbbb = getvaluefromarchive('datarawNTT', 'filename', arcfile,
'*')
except:
bbbb = ''
if bbbb:
directoryraw = bbbb['filepath']
directoryreduced = re.sub('raw/', '',
directoryraw) + 'reduced/'
else:
dir3 = dir2 + string.split(tarfile,
'_')[3] + '_' + string.split(
tarfile, '_')[4] + '/'
if not os.path.isdir(dir3): os.mkdir(dir3)
directoryreduced = dir3 + 'reduced/'
if not os.path.isdir(directoryreduced): os.mkdir(directoryreduced)
os.system('cp ' + img + ' ' + directoryreduced)
updatevalue('redulogNTT', 'filepath', directoryreduced, img)
updatevalue('datareduNTT', 'filepath', directoryreduced, img)
def ingestredu(imglist, force='no', dataredutable='photlco', filetype=1):
import string, re, os, sys
import lsc
from lsc.util import readkey3, readhdr
from datetime import datetime
hostname, username, passwd, database = lsc.mysqldef.getconnection('lcogt2')
conn = lsc.mysqldef.dbConnect(hostname, username, passwd, database)
for fullpath in imglist:
path, img = os.path.split(fullpath)
path += '/'
exist = lsc.mysqldef.getfromdataraw(conn,
dataredutable,
'filename',
string.split(img, '/')[-1],
column2='filename')
exist2 = lsc.mysqldef.getfromdataraw(conn,
'photlcoraw',
'filename',
string.split(img, '/')[-1],
column2='filename, groupidcode')
if exist2:
print exist2
_groupidcode = exist2[0]['groupidcode']
else:
_groupidcode = ''
if exist:
if force == 'yes':
print img, database
lsc.mysqldef.deleteredufromarchive(
string.split(img, '/')[-1], dataredutable)
print 'delete line from ' + str(database)
exist = lsc.mysqldef.getfromdataraw(conn,
dataredutable,
'filename',
string.split(img, '/')[-1],
column2='filename')
if not exist or force == 'update':
hdr = readhdr(fullpath)
_targetid = lsc.mysqldef.targimg(fullpath)
try:
_tracknumber = int(readkey3(hdr, 'TRACKNUM'))
except:
_tracknumber = 0
if hdr.get('TELESCOP'):
_tel = hdr.get('TELESCOP')
else:
_tel = ''
if _tel in ['Faulkes Telescope South', 'fts']:
_tel = '2m0-02'
elif _tel in ['Faulkes Telescope North', 'ftn']:
_tel = '2m0-01'
_inst = hdr.get('instrume')
dictionary={'dateobs':readkey3(hdr,'date-obs'),'dayobs':readkey3(hdr,'DAY-OBS'),'filename':img,'filepath':path,'filetype':filetype,'targetid':_targetid,\
'exptime':readkey3(hdr,'exptime'), 'filter':readkey3(hdr,'filter'),'mjd':readkey3(hdr,'mjd'),'tracknumber':_tracknumber,'groupidcode':_groupidcode,\
'telescope':_tel,'airmass':readkey3(hdr,'airmass'),'objname':readkey3(hdr,'object'),'ut':readkey3(hdr,'ut'),\
'wcs':readkey3(hdr,'wcserr'),'instrument':_inst,'ra0':readkey3(hdr,'RA'),'dec0':readkey3(hdr,'DEC')}
_telid = lsc.mysqldef.getfromdataraw(conn,
'telescopes',
'name',
_tel,
column2='id')
if not _telid:
print 'Telescope ', _tel, ' not recognized. Adding to telescopes table.'
# the short name is needed to calibrate the magnitude with extinction
lsc.mysqldef.insert_values(conn, 'telescopes', {
'name': _tel,
'shortname': readkey3(hdr, 'SITEID')
})
_telid = lsc.mysqldef.getfromdataraw(conn,
'telescopes',
'name',
_tel,
column2='id')
telid = _telid[0]['id']
dictionary['telescopeid'] = str(telid)
_instid = lsc.mysqldef.getfromdataraw(conn,
'instruments',
'name',
_inst,
column2='id')
if not _instid:
print 'Instrument ', _inst, ' not recognized. Adding to instruments table.'
lsc.mysqldef.insert_values(conn, 'instruments', {
'name': _inst,
'type': guess_instrument_type(_inst)
})
_instid = lsc.mysqldef.getfromdataraw(conn,
'instruments',
'name',
_inst,
column2='id')
instid = _instid[0]['id']
dictionary['instrumentid'] = str(instid)
dictionary['lastunpacked'] = str(datetime.utcnow())
print dictionary
print 'insert reduced'
ggg = lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename',
str(img), '*')
if not ggg:
lsc.mysqldef.insert_values(conn, dataredutable, dictionary)
else:
for voce in dictionary:
lsc.mysqldef.updatevalue(dataredutable, voce,
dictionary[voce],
string.split(img, '/')[-1])
else:
print 'already ingested'
def updateDatabase(tarfile):
import string,os,re,math,sys,shutil,glob,socket,pickle
import MySQLdb
host=socket.gethostname()
import ntt
from ntt.util import readkey3,readhdr
from mysqldef import updatevalue, getvaluefromarchive, dbConnect,getfromdataraw,ingestredu
os.system('tar -zxvf '+str(tarfile))
pklfile=re.sub('tar.gz','pkl',tarfile)
if glob.glob(pklfile):
pkl_file = open(pklfile, 'rb')
mydict2 = pickle.load(pkl_file)
pkl_file.close()
fitsfile=[]
for i in mydict2:
try:
command=mydict2[i]['command']
if command=='ingestredu':
lista=mydict2[i]['lista']
instrument=mydict2[i]['instument']
output=mydict2[i]['output']
ingestredu(lista)
if output not in fitsfile: fitsfile.append(output)
elif command=='getvaluefromarchive':
table=mydict2[i]['table']
column=mydict2[i]['column']
value=mydict2[i]['value']
_input=mydict2[i]['input']
hh=getvaluefromarchive(table,column,_input,value)
elif command=='updatevalue':
table=mydict2[i]['table']
column=mydict2[i]['column']
value=mydict2[i]['value']
_input=mydict2[i]['input']
if value!='voce':
updatevalue(table,column,value,_input)
else:
if column=='voce':
for voce in hh:
if voce!='id' and voce!='filename':
updatevalue(table,voce,hh[voce],_input)
else:
updatevalue(table,column,hh[column],_input)
else:
print 'warning: command not recognise'
except:
print '#####################'
print mydict2[i]
print 'problems'
print fitsfile
dir1='/data/obsdata/y20'+string.split(tarfile,'_')[2][-2:]+'/'
dir2='/data/obsdata/y20'+string.split(tarfile,'_')[2][-2:]+'/'+string.split(tarfile,'_')[2]+'/'
if not os.path.isdir(dir1): os.mkdir(dir1)
if not os.path.isdir(dir2): os.mkdir(dir2)
for img in fitsfile:
arcfile=readkey3(readhdr(img),'ARCFILE')
print arcfile
try:
bbbb=getvaluefromarchive('datarawNTT','filename',arcfile,'*')
except: bbbb=''
if bbbb:
directoryraw=bbbb['filepath']
directoryreduced=re.sub('raw/','',directoryraw)+'reduced/'
else:
dir3=dir2+string.split(tarfile,'_')[3]+'_'+string.split(tarfile,'_')[4]+'/'
if not os.path.isdir(dir3): os.mkdir(dir3)
directoryreduced=dir3+'reduced/'
if not os.path.isdir(directoryreduced): os.mkdir(directoryreduced)
os.system('cp '+img+' '+directoryreduced)
updatevalue('redulogNTT','filepath',directoryreduced,img)
updatevalue('datareduNTT','filepath',directoryreduced,img)
def ingestredu(imglist,force='no',dataredutable='photlco',filetype=1):
import string,re,os,sys
import lsc
from lsc.util import readkey3, readhdr
from datetime import datetime
hostname, username, passwd, database=lsc.mysqldef.getconnection('lcogt2')
conn = lsc.mysqldef.dbConnect(hostname, username, passwd, database)
for fullpath in imglist:
path, img = os.path.split(fullpath)
path += '/'
exist=lsc.mysqldef.getfromdataraw(conn,dataredutable,'filename', string.split(img,'/')[-1],column2='filename')
exist2=lsc.mysqldef.getfromdataraw(conn,'photlcoraw','filename', string.split(img,'/')[-1],column2='filename, groupidcode')
if exist2:
print exist2
_groupidcode=exist2[0]['groupidcode']
else:
_groupidcode=''
if exist:
if force=='yes':
print img,database
lsc.mysqldef.deleteredufromarchive(string.split(img,'/')[-1],dataredutable)
print 'delete line from '+str(database)
exist=lsc.mysqldef.getfromdataraw(conn,dataredutable,'filename', string.split(img,'/')[-1],column2='filename')
if not exist or force =='update':
hdr=readhdr(fullpath)
_targetid=lsc.mysqldef.targimg(fullpath)
try:
_tracknumber=int(readkey3(hdr,'TRACKNUM'))
except:
_tracknumber=0
if hdr.get('TELESCOP'):
_tel=hdr.get('TELESCOP')
else:
_tel=''
if _tel in ['Faulkes Telescope South','fts']:
_tel='2m0-02'
elif _tel in ['Faulkes Telescope North','ftn']:
_tel='2m0-01'
_inst=hdr.get('instrume')
dictionary={'dateobs':readkey3(hdr,'date-obs'),'dayobs':readkey3(hdr,'DAY-OBS'),'filename':img,'filepath':path,'filetype':filetype,'targetid':_targetid,\
'exptime':readkey3(hdr,'exptime'), 'filter':readkey3(hdr,'filter'),'mjd':readkey3(hdr,'mjd'),'tracknumber':_tracknumber,'groupidcode':_groupidcode,\
'telescope':_tel,'airmass':readkey3(hdr,'airmass'),'objname':readkey3(hdr,'object'),'ut':readkey3(hdr,'ut'),\
'wcs':readkey3(hdr,'wcserr'),'instrument':_inst,'ra0':readkey3(hdr,'RA'),'dec0':readkey3(hdr,'DEC')}
_telid=lsc.mysqldef.getfromdataraw(conn,'telescopes','name',_tel,column2='id')
if not _telid:
print 'Telescope ',_tel,' not recognized. Adding to telescopes table.'
lsc.mysqldef.insert_values(conn,'telescopes',{'name':_tel})
_telid=lsc.mysqldef.getfromdataraw(conn,'telescopes','name',_tel,column2='id')
telid=_telid[0]['id']
dictionary['telescopeid']=str(telid)
_instid=lsc.mysqldef.getfromdataraw(conn,'instruments','name',_inst,column2='id')
if not _instid:
print 'Instrument ',_inst,' not recognized. Adding to instruments table.'
lsc.mysqldef.insert_values(conn,'instruments',{'name':_inst})
_instid=lsc.mysqldef.getfromdataraw(conn,'instruments','name',_inst,column2='id')
instid=_instid[0]['id']
dictionary['instrumentid']=str(instid)
dictionary['lastunpacked'] = str(datetime.utcnow())
print dictionary
print 'insert reduced'
ggg=lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename',str(img), '*')
if not ggg:
lsc.mysqldef.insert_values(conn,dataredutable,dictionary)
else:
for voce in dictionary:
lsc.mysqldef.updatevalue(dataredutable,voce,dictionary[voce],string.split(img,'/')[-1])
else:
print 'already ingested'
