def limmag(img): import lsc from lsc.util import readhdr hdr=readhdr(img) _ZP=readkey3(hdr,'PHOTZP') _gain=readkey3(hdr,'gain') _exptime=readkey3(hdr,'exptime') _fwhm=readkey3(hdr,'PSF_FWHM') _mbkg=readkey3(hdr,'MBKG') # background from sextractor _instrume=readkey3(hdr,'instrume') check=1 if not _ZP: check=0 if not _gain: check=0 if not _fwhm: check=0 if not _mbkg: check=0 else: if _mbkg<=0: _mbkg=0 if check==1: # formula from McLean 1997) from numpy import pi,log10 if _instrume=='efosc': ps=readkey3(hdr,'binx')*.12 else: ps=0.288 n=pi*((_fwhm/ps)**2) sn=5 # signal to noise maglim=_ZP -2.5 * log10(sn * (1/_gain) * ((n*_mbkg/_exptime)**(.5)) ) return maglim else: return ''
def checksnlist(img,listfile): import lsc import string from lsc.util import readkey3,readhdr from numpy import cos,sin,arccos,pi, argmin scal=pi/180. std,rastd,decstd,magstd=lsc.util.readstandard(listfile) hdrt=readhdr(img) _ra=readkey3(hdrt,'RA') _dec=readkey3(hdrt,'DEC') _object=readkey3(hdrt,'object') _xdimen=readkey3(hdrt,'XDIM') _ydimen=readkey3(hdrt,'YDIM') if not _xdimen: _xdimen=readkey3(hdrt,'NAXIS1') if not _ydimen: _ydimen=readkey3(hdrt,'NAXIS2') dd=arccos(sin(_dec*scal)*sin(decstd*scal)+cos(_dec*scal)*cos(decstd*scal)*cos((_ra-rastd)*scal))*((180/pi)*3600) lll=[str(rastd[argmin(dd)])+' '+str(decstd[argmin(dd)])] from pyraf import iraf bbb=iraf.wcsctran('STDIN','STDOUT',img+'[0]',Stdin=lll,inwcs='world',units='degrees degrees',outwcs='logical',columns='1 2',formats='%10.5f %10.5f',Stdout=1)[3] if 'INDEF' not in bbb and float(string.split(bbb)[0])<=_xdimen and float(string.split(bbb)[1])<=_ydimen and float(string.split(bbb)[0])>=0 and float(string.split(bbb)[1])>=0: #print str(std[argmin(dd)])+' in the field '+str(bbb) _RA=rastd[argmin(dd)] _DEC=decstd[argmin(dd)] _SN=std[argmin(dd)] else: #print 'out '+str(bbb) _RA,_DEC,_SN='','','' return _RA,_DEC,_SN
def airmass(img,overwrite=True,_observatory='lasilla'): import lsc from lsc.util import readhdr,readkey3, delete, updateheader from pyraf import iraf iraf.astutil(_doprint=0) hdr=readhdr(img) if readkey3(hdr,'UTC'): _UT=(readkey3(hdr,'UTC')+(readkey3(hdr,'exptime')/2))/3600 _date=readkey3(hdr,'date-obs') _date=_date[0:4]+'-'+_date[4:6]+'-'+_date[6:8] _RA=readkey3(hdr,'RA')/15 _DEC=readkey3(hdr,'DEC') f = file('airmass.txt','w') f.write('mst = mst ("'+str(_date)+'",'+str(_UT)+', obsdb ("'+str(_observatory)+'", "longitude"))\n') f.write('air = airmass ('+str(_RA)+','+str(_DEC)+',mst, obsdb ("'+str(_observatory)+'", "latitude"))\n') f.write('print(air)\n') f.close() _air=iraf.astcalc(image=img, command="airmass.txt",Stdout=1)[0] try: _air=float(_air) except: _air=999 delete('airmass.txt') if overwrite and _air<99.: updateheader(img,0,{'AIRMASS':(_air,'mean airmass computed with astcalc')}) else: _air='' return _air
def name_duplicate(img,nome,ext): ########################### import re,string,os,glob import lsc from lsc.util import readhdr,readkey3, delete dimg=readkey3(readhdr(img),'DATE-OBS') listafile=glob.glob(nome+'_?'+ext+'.fits')+glob.glob(nome+'_??'+ext+'.fits') if len(listafile) == 0: nome = nome+"_1"+ext+'.fits' else: date=[] for l in listafile: date.append(readkey3(readhdr(l),'DATE-OBS')) if dimg in date: nome=listafile[date.index(dimg)] # if overwrite: # delete(nome) else: n=1 while nome+'_'+str(n)+str(ext)+'.fits' in listafile: n=n+1 nome=nome+'_'+str(n)+str(ext)+'.fits' return nome
def correctobject(img,coordinatefile): import os,re,sys,string from numpy import arccos, sin,cos,pi,argmin import lsc from lsc.util import readstandard, readhdr,readkey3, updateheader scal=pi/180. std,rastd,decstd,magstd=readstandard(coordinatefile) img=re.sub('\n','',img) hdr=readhdr(img) _ra=readkey3(hdr,'RA') _dec=readkey3(hdr,'DEC') dd=arccos(sin(_dec*scal)*sin(decstd*scal)+cos(_dec*scal)*cos(decstd*scal)*cos((_ra-rastd)*scal))*((180/pi)*3600) if min(dd)<200: updateheader(img,0,{'OBJECT':(std[argmin(dd)],'Original target.')}) aa,bb,cc=rastd[argmin(dd)],decstd[argmin(dd)],std[argmin(dd)] else: aa,bb,cc='','','' return aa,bb,cc
def updateDatabase(tarfile): import string, os, re, math, sys, shutil, glob, socket, pickle import MySQLdb host = socket.gethostname() import ntt from ntt.util import readkey3, readhdr from mysqldef import updatevalue, getvaluefromarchive, dbConnect, getfromdataraw, ingestredu os.system('tar -zxvf ' + str(tarfile)) pklfile = re.sub('tar.gz', 'pkl', tarfile) if glob.glob(pklfile): pkl_file = open(pklfile, 'rb') mydict2 = pickle.load(pkl_file) pkl_file.close() fitsfile = [] for i in mydict2: try: command = mydict2[i]['command'] if command == 'ingestredu': lista = mydict2[i]['lista'] instrument = mydict2[i]['instument'] output = mydict2[i]['output'] ingestredu(lista) if output not in fitsfile: fitsfile.append(output) elif command == 'getvaluefromarchive': table = mydict2[i]['table'] column = mydict2[i]['column'] value = mydict2[i]['value'] _input = mydict2[i]['input'] hh = getvaluefromarchive(table, column, _input, value) elif command == 'updatevalue': table = mydict2[i]['table'] column = mydict2[i]['column'] value = mydict2[i]['value'] _input = mydict2[i]['input'] if value != 'voce': updatevalue(table, column, value, _input) else: if column == 'voce': for voce in hh: if voce != 'id' and voce != 'filename': updatevalue(table, voce, hh[voce], _input) else: updatevalue(table, column, hh[column], _input) else: print 'warning: command not recognise' except: print '#####################' print mydict2[i] print 'problems' print fitsfile dir1 = '/data/obsdata/y20' + string.split(tarfile, '_')[2][-2:] + '/' dir2 = '/data/obsdata/y20' + string.split( tarfile, '_')[2][-2:] + '/' + string.split(tarfile, '_')[2] + '/' if not os.path.isdir(dir1): os.mkdir(dir1) if not os.path.isdir(dir2): os.mkdir(dir2) for img in fitsfile: arcfile = readkey3(readhdr(img), 'ARCFILE') print arcfile try: bbbb = getvaluefromarchive('datarawNTT', 'filename', arcfile, '*') except: bbbb = '' if bbbb: directoryraw = bbbb['filepath'] directoryreduced = re.sub('raw/', '', directoryraw) + 'reduced/' else: dir3 = dir2 + string.split(tarfile, '_')[3] + '_' + string.split( tarfile, '_')[4] + '/' if not os.path.isdir(dir3): os.mkdir(dir3) directoryreduced = dir3 + 'reduced/' if not os.path.isdir(directoryreduced): os.mkdir(directoryreduced) os.system('cp ' + img + ' ' + directoryreduced) updatevalue('redulogNTT', 'filepath', directoryreduced, img) updatevalue('datareduNTT', 'filepath', directoryreduced, img)
def ingestredu(imglist, force='no', dataredutable='photlco', filetype=1): import string, re, os, sys import lsc from lsc.util import readkey3, readhdr from datetime import datetime hostname, username, passwd, database = lsc.mysqldef.getconnection('lcogt2') conn = lsc.mysqldef.dbConnect(hostname, username, passwd, database) for fullpath in imglist: path, img = os.path.split(fullpath) path += '/' exist = lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename', string.split(img, '/')[-1], column2='filename') exist2 = lsc.mysqldef.getfromdataraw(conn, 'photlcoraw', 'filename', string.split(img, '/')[-1], column2='filename, groupidcode') if exist2: print exist2 _groupidcode = exist2[0]['groupidcode'] else: _groupidcode = '' if exist: if force == 'yes': print img, database lsc.mysqldef.deleteredufromarchive( string.split(img, '/')[-1], dataredutable) print 'delete line from ' + str(database) exist = lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename', string.split(img, '/')[-1], column2='filename') if not exist or force == 'update': hdr = readhdr(fullpath) _targetid = lsc.mysqldef.targimg(fullpath) try: _tracknumber = int(readkey3(hdr, 'TRACKNUM')) except: _tracknumber = 0 if hdr.get('TELESCOP'): _tel = hdr.get('TELESCOP') else: _tel = '' if _tel in ['Faulkes Telescope South', 'fts']: _tel = '2m0-02' elif _tel in ['Faulkes Telescope North', 'ftn']: _tel = '2m0-01' _inst = hdr.get('instrume') dictionary={'dateobs':readkey3(hdr,'date-obs'),'dayobs':readkey3(hdr,'DAY-OBS'),'filename':img,'filepath':path,'filetype':filetype,'targetid':_targetid,\ 'exptime':readkey3(hdr,'exptime'), 'filter':readkey3(hdr,'filter'),'mjd':readkey3(hdr,'mjd'),'tracknumber':_tracknumber,'groupidcode':_groupidcode,\ 'telescope':_tel,'airmass':readkey3(hdr,'airmass'),'objname':readkey3(hdr,'object'),'ut':readkey3(hdr,'ut'),\ 'wcs':readkey3(hdr,'wcserr'),'instrument':_inst,'ra0':readkey3(hdr,'RA'),'dec0':readkey3(hdr,'DEC')} _telid = lsc.mysqldef.getfromdataraw(conn, 'telescopes', 'name', _tel, column2='id') if not _telid: print 'Telescope ', _tel, ' not recognized. Adding to telescopes table.' # the short name is needed to calibrate the magnitude with extinction lsc.mysqldef.insert_values(conn, 'telescopes', { 'name': _tel, 'shortname': readkey3(hdr, 'SITEID') }) _telid = lsc.mysqldef.getfromdataraw(conn, 'telescopes', 'name', _tel, column2='id') telid = _telid[0]['id'] dictionary['telescopeid'] = str(telid) _instid = lsc.mysqldef.getfromdataraw(conn, 'instruments', 'name', _inst, column2='id') if not _instid: print 'Instrument ', _inst, ' not recognized. Adding to instruments table.' lsc.mysqldef.insert_values(conn, 'instruments', { 'name': _inst, 'type': guess_instrument_type(_inst) }) _instid = lsc.mysqldef.getfromdataraw(conn, 'instruments', 'name', _inst, column2='id') instid = _instid[0]['id'] dictionary['instrumentid'] = str(instid) dictionary['lastunpacked'] = str(datetime.utcnow()) print dictionary print 'insert reduced' ggg = lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename', str(img), '*') if not ggg: lsc.mysqldef.insert_values(conn, dataredutable, dictionary) else: for voce in dictionary: lsc.mysqldef.updatevalue(dataredutable, voce, dictionary[voce], string.split(img, '/')[-1]) else: print 'already ingested'
def updateDatabase(tarfile): import string,os,re,math,sys,shutil,glob,socket,pickle import MySQLdb host=socket.gethostname() import ntt from ntt.util import readkey3,readhdr from mysqldef import updatevalue, getvaluefromarchive, dbConnect,getfromdataraw,ingestredu os.system('tar -zxvf '+str(tarfile)) pklfile=re.sub('tar.gz','pkl',tarfile) if glob.glob(pklfile): pkl_file = open(pklfile, 'rb') mydict2 = pickle.load(pkl_file) pkl_file.close() fitsfile=[] for i in mydict2: try: command=mydict2[i]['command'] if command=='ingestredu': lista=mydict2[i]['lista'] instrument=mydict2[i]['instument'] output=mydict2[i]['output'] ingestredu(lista) if output not in fitsfile: fitsfile.append(output) elif command=='getvaluefromarchive': table=mydict2[i]['table'] column=mydict2[i]['column'] value=mydict2[i]['value'] _input=mydict2[i]['input'] hh=getvaluefromarchive(table,column,_input,value) elif command=='updatevalue': table=mydict2[i]['table'] column=mydict2[i]['column'] value=mydict2[i]['value'] _input=mydict2[i]['input'] if value!='voce': updatevalue(table,column,value,_input) else: if column=='voce': for voce in hh: if voce!='id' and voce!='filename': updatevalue(table,voce,hh[voce],_input) else: updatevalue(table,column,hh[column],_input) else: print 'warning: command not recognise' except: print '#####################' print mydict2[i] print 'problems' print fitsfile dir1='/data/obsdata/y20'+string.split(tarfile,'_')[2][-2:]+'/' dir2='/data/obsdata/y20'+string.split(tarfile,'_')[2][-2:]+'/'+string.split(tarfile,'_')[2]+'/' if not os.path.isdir(dir1): os.mkdir(dir1) if not os.path.isdir(dir2): os.mkdir(dir2) for img in fitsfile: arcfile=readkey3(readhdr(img),'ARCFILE') print arcfile try: bbbb=getvaluefromarchive('datarawNTT','filename',arcfile,'*') except: bbbb='' if bbbb: directoryraw=bbbb['filepath'] directoryreduced=re.sub('raw/','',directoryraw)+'reduced/' else: dir3=dir2+string.split(tarfile,'_')[3]+'_'+string.split(tarfile,'_')[4]+'/' if not os.path.isdir(dir3): os.mkdir(dir3) directoryreduced=dir3+'reduced/' if not os.path.isdir(directoryreduced): os.mkdir(directoryreduced) os.system('cp '+img+' '+directoryreduced) updatevalue('redulogNTT','filepath',directoryreduced,img) updatevalue('datareduNTT','filepath',directoryreduced,img)
def ingestredu(imglist,force='no',dataredutable='photlco',filetype=1): import string,re,os,sys import lsc from lsc.util import readkey3, readhdr from datetime import datetime hostname, username, passwd, database=lsc.mysqldef.getconnection('lcogt2') conn = lsc.mysqldef.dbConnect(hostname, username, passwd, database) for fullpath in imglist: path, img = os.path.split(fullpath) path += '/' exist=lsc.mysqldef.getfromdataraw(conn,dataredutable,'filename', string.split(img,'/')[-1],column2='filename') exist2=lsc.mysqldef.getfromdataraw(conn,'photlcoraw','filename', string.split(img,'/')[-1],column2='filename, groupidcode') if exist2: print exist2 _groupidcode=exist2[0]['groupidcode'] else: _groupidcode='' if exist: if force=='yes': print img,database lsc.mysqldef.deleteredufromarchive(string.split(img,'/')[-1],dataredutable) print 'delete line from '+str(database) exist=lsc.mysqldef.getfromdataraw(conn,dataredutable,'filename', string.split(img,'/')[-1],column2='filename') if not exist or force =='update': hdr=readhdr(fullpath) _targetid=lsc.mysqldef.targimg(fullpath) try: _tracknumber=int(readkey3(hdr,'TRACKNUM')) except: _tracknumber=0 if hdr.get('TELESCOP'): _tel=hdr.get('TELESCOP') else: _tel='' if _tel in ['Faulkes Telescope South','fts']: _tel='2m0-02' elif _tel in ['Faulkes Telescope North','ftn']: _tel='2m0-01' _inst=hdr.get('instrume') dictionary={'dateobs':readkey3(hdr,'date-obs'),'dayobs':readkey3(hdr,'DAY-OBS'),'filename':img,'filepath':path,'filetype':filetype,'targetid':_targetid,\ 'exptime':readkey3(hdr,'exptime'), 'filter':readkey3(hdr,'filter'),'mjd':readkey3(hdr,'mjd'),'tracknumber':_tracknumber,'groupidcode':_groupidcode,\ 'telescope':_tel,'airmass':readkey3(hdr,'airmass'),'objname':readkey3(hdr,'object'),'ut':readkey3(hdr,'ut'),\ 'wcs':readkey3(hdr,'wcserr'),'instrument':_inst,'ra0':readkey3(hdr,'RA'),'dec0':readkey3(hdr,'DEC')} _telid=lsc.mysqldef.getfromdataraw(conn,'telescopes','name',_tel,column2='id') if not _telid: print 'Telescope ',_tel,' not recognized. Adding to telescopes table.' lsc.mysqldef.insert_values(conn,'telescopes',{'name':_tel}) _telid=lsc.mysqldef.getfromdataraw(conn,'telescopes','name',_tel,column2='id') telid=_telid[0]['id'] dictionary['telescopeid']=str(telid) _instid=lsc.mysqldef.getfromdataraw(conn,'instruments','name',_inst,column2='id') if not _instid: print 'Instrument ',_inst,' not recognized. Adding to instruments table.' lsc.mysqldef.insert_values(conn,'instruments',{'name':_inst}) _instid=lsc.mysqldef.getfromdataraw(conn,'instruments','name',_inst,column2='id') instid=_instid[0]['id'] dictionary['instrumentid']=str(instid) dictionary['lastunpacked'] = str(datetime.utcnow()) print dictionary print 'insert reduced' ggg=lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename',str(img), '*') if not ggg: lsc.mysqldef.insert_values(conn,dataredutable,dictionary) else: for voce in dictionary: lsc.mysqldef.updatevalue(dataredutable,voce,dictionary[voce],string.split(img,'/')[-1]) else: print 'already ingested'