def correctobject(img,coordinatefile): import os,re,sys,string from numpy import arccos, sin,cos,pi,argmin import lsc from lsc.util import readstandard, readhdr,readkey3, updateheader scal=pi/180. std,rastd,decstd,magstd=readstandard(coordinatefile) img=re.sub('\n','',img) hdr=readhdr(img) _ra=readkey3(hdr,'RA') _dec=readkey3(hdr,'DEC') dd=arccos(sin(_dec*scal)*sin(decstd*scal)+cos(_dec*scal)*cos(decstd*scal)*cos((_ra-rastd)*scal))*((180/pi)*3600) if min(dd)<200: updateheader(img,0,{'OBJECT':(std[argmin(dd)],'Original target.')}) aa,bb,cc=rastd[argmin(dd)],decstd[argmin(dd)],std[argmin(dd)] else: aa,bb,cc='','','' return aa,bb,cc
def limmag(img): import lsc from lsc.util import readhdr hdr = readhdr(img) _ZP = readkey3(hdr, 'PHOTZP') _gain = readkey3(hdr, 'gain') _exptime = readkey3(hdr, 'exptime') _fwhm = readkey3(hdr, 'PSF_FWHM') _mbkg = readkey3(hdr, 'MBKG') # background from sextractor _instrume = readkey3(hdr, 'instrume') check = 1 if not _ZP: check = 0 if not _gain: check = 0 if not _fwhm: check = 0 if not _mbkg: check = 0 else: if _mbkg <= 0: _mbkg = 0 if check == 1: # formula from McLean 1997) from numpy import pi, log10 if _instrume == 'efosc': ps = readkey3(hdr, 'binx') * .12 else: ps = 0.288 n = pi * ((_fwhm / ps)**2) sn = 5 # signal to noise maglim = _ZP - 2.5 * log10(sn * (1 / _gain) * ((n * _mbkg / _exptime)**(.5))) return maglim else: return ''
def checksnlist(img,listfile): import lsc import string from lsc.util import readkey3,readhdr from numpy import cos,sin,arccos,pi, argmin scal=pi/180. std,rastd,decstd,magstd=lsc.util.readstandard(listfile) hdrt=readhdr(img) _ra=readkey3(hdrt,'RA') _dec=readkey3(hdrt,'DEC') _object=readkey3(hdrt,'object') _xdimen=readkey3(hdrt,'XDIM') _ydimen=readkey3(hdrt,'YDIM') if not _xdimen: _xdimen=readkey3(hdrt,'NAXIS1') if not _ydimen: _ydimen=readkey3(hdrt,'NAXIS2') dd=arccos(sin(_dec*scal)*sin(decstd*scal)+cos(_dec*scal)*cos(decstd*scal)*cos((_ra-rastd)*scal))*((180/pi)*3600) lll=[str(rastd[argmin(dd)])+' '+str(decstd[argmin(dd)])] from pyraf import iraf bbb=iraf.wcsctran('STDIN','STDOUT',img+'[0]',Stdin=lll,inwcs='world',units='degrees degrees',outwcs='logical',columns='1 2',formats='%10.5f %10.5f',Stdout=1)[3] if 'INDEF' not in bbb and float(string.split(bbb)[0])<=_xdimen and float(string.split(bbb)[1])<=_ydimen and float(string.split(bbb)[0])>=0 and float(string.split(bbb)[1])>=0: #print str(std[argmin(dd)])+' in the field '+str(bbb) _RA=rastd[argmin(dd)] _DEC=decstd[argmin(dd)] _SN=std[argmin(dd)] else: #print 'out '+str(bbb) _RA,_DEC,_SN='','','' return _RA,_DEC,_SN
def limmag(img): import lsc from lsc.util import readhdr hdr=readhdr(img) _ZP=readkey3(hdr,'PHOTZP') _gain=readkey3(hdr,'gain') _exptime=readkey3(hdr,'exptime') _fwhm=readkey3(hdr,'PSF_FWHM') _mbkg=readkey3(hdr,'MBKG') # background from sextractor _instrume=readkey3(hdr,'instrume') check=1 if not _ZP: check=0 if not _gain: check=0 if not _fwhm: check=0 if not _mbkg: check=0 else: if _mbkg<=0: _mbkg=0 if check==1: # formula from McLean 1997) from numpy import pi,log10 if _instrume=='efosc': ps=readkey3(hdr,'binx')*.12 else: ps=0.288 n=pi*((_fwhm/ps)**2) sn=5 # signal to noise maglim=_ZP -2.5 * log10(sn * (1/_gain) * ((n*_mbkg/_exptime)**(.5)) ) return maglim else: return ''
def correctobject(img, coordinatefile): import os, re, sys, string from numpy import arccos, sin, cos, pi, argmin import lsc from lsc.util import readstandard, readhdr, readkey3, updateheader scal = pi / 180. std, rastd, decstd, magstd = readstandard(coordinatefile) img = re.sub('\n', '', img) hdr = readhdr(img) _ra = readkey3(hdr, 'RA') _dec = readkey3(hdr, 'DEC') dd = arccos( sin(_dec * scal) * sin(decstd * scal) + cos(_dec * scal) * cos(decstd * scal) * cos((_ra - rastd) * scal)) * ( (180 / pi) * 3600) if min(dd) < 200: updateheader(img, 0, {'OBJECT': (std[argmin(dd)], 'Original target.')}) aa, bb, cc = rastd[argmin(dd)], decstd[argmin(dd)], std[argmin(dd)] else: aa, bb, cc = '', '', '' return aa, bb, cc
def name_duplicate(img,nome,ext): ########################### import re,string,os,glob import lsc from lsc.util import readhdr,readkey3, delete dimg=readkey3(readhdr(img),'DATE-OBS') listafile=glob.glob(nome+'_?'+ext+'.fits')+glob.glob(nome+'_??'+ext+'.fits') if len(listafile) == 0: nome = nome+"_1"+ext+'.fits' else: date=[] for l in listafile: date.append(readkey3(readhdr(l),'DATE-OBS')) if dimg in date: nome=listafile[date.index(dimg)] # if overwrite: # delete(nome) else: n=1 while nome+'_'+str(n)+str(ext)+'.fits' in listafile: n=n+1 nome=nome+'_'+str(n)+str(ext)+'.fits' return nome
def airmass(img, overwrite=True, _observatory='lasilla'): import lsc from lsc.util import readhdr, readkey3, delete, updateheader from pyraf import iraf iraf.astutil(_doprint=0) hdr = readhdr(img) if readkey3(hdr, 'UTC'): _UT = (readkey3(hdr, 'UTC') + (readkey3(hdr, 'exptime') / 2)) / 3600 _date = readkey3(hdr, 'date-obs') _date = _date[0:4] + '-' + _date[4:6] + '-' + _date[6:8] _RA = readkey3(hdr, 'RA') / 15 _DEC = readkey3(hdr, 'DEC') f = file('airmass.txt', 'w') f.write('mst = mst ("' + str(_date) + '",' + str(_UT) + ', obsdb ("' + str(_observatory) + '", "longitude"))\n') f.write('air = airmass (' + str(_RA) + ',' + str(_DEC) + ',mst, obsdb ("' + str(_observatory) + '", "latitude"))\n') f.write('print(air)\n') f.close() _air = iraf.astcalc(image=img, command="airmass.txt", Stdout=1)[0] try: _air = float(_air) except: _air = 999 delete('airmass.txt') if overwrite and _air < 99.: updateheader( img, 0, {'AIRMASS': (_air, 'mean airmass computed with astcalc')}) else: _air = '' return _air
def airmass(img,overwrite=True,_observatory='lasilla'): import lsc from lsc.util import readhdr,readkey3, delete, updateheader from pyraf import iraf iraf.astutil(_doprint=0) hdr=readhdr(img) if readkey3(hdr,'UTC'): _UT=(readkey3(hdr,'UTC')+(readkey3(hdr,'exptime')/2))/3600 _date=readkey3(hdr,'date-obs') _date=_date[0:4]+'-'+_date[4:6]+'-'+_date[6:8] _RA=readkey3(hdr,'RA')/15 _DEC=readkey3(hdr,'DEC') f = file('airmass.txt','w') f.write('mst = mst ("'+str(_date)+'",'+str(_UT)+', obsdb ("'+str(_observatory)+'", "longitude"))\n') f.write('air = airmass ('+str(_RA)+','+str(_DEC)+',mst, obsdb ("'+str(_observatory)+'", "latitude"))\n') f.write('print(air)\n') f.close() _air=iraf.astcalc(image=img, command="airmass.txt",Stdout=1)[0] try: _air=float(_air) except: _air=999 delete('airmass.txt') if overwrite and _air<99.: updateheader(img,0,{'AIRMASS':(_air,'mean airmass computed with astcalc')}) else: _air='' return _air
def name_duplicate(img, nome, ext): ########################### import re, string, os, glob import lsc from lsc.util import readhdr, readkey3, delete dimg = readkey3(readhdr(img), 'DATE-OBS') listafile = glob.glob(nome + '_?' + ext + '.fits') + glob.glob(nome + '_??' + ext + '.fits') if len(listafile) == 0: nome = nome + "_1" + ext + '.fits' else: date = [] for l in listafile: date.append(readkey3(readhdr(l), 'DATE-OBS')) if dimg in date: nome = listafile[date.index(dimg)] # if overwrite: # delete(nome) else: n = 1 while nome + '_' + str(n) + str(ext) + '.fits' in listafile: n = n + 1 nome = nome + '_' + str(n) + str(ext) + '.fits' return nome
def checksnlist(img, listfile): import lsc import string from lsc.util import readkey3, readhdr from numpy import cos, sin, arccos, pi, argmin scal = pi / 180. std, rastd, decstd, magstd = lsc.util.readstandard(listfile) hdrt = readhdr(img) _ra = readkey3(hdrt, 'RA') _dec = readkey3(hdrt, 'DEC') _object = readkey3(hdrt, 'object') _xdimen = readkey3(hdrt, 'XDIM') _ydimen = readkey3(hdrt, 'YDIM') if not _xdimen: _xdimen = readkey3(hdrt, 'NAXIS1') if not _ydimen: _ydimen = readkey3(hdrt, 'NAXIS2') dd = arccos( sin(_dec * scal) * sin(decstd * scal) + cos(_dec * scal) * cos(decstd * scal) * cos((_ra - rastd) * scal)) * ( (180 / pi) * 3600) lll = [str(rastd[argmin(dd)]) + ' ' + str(decstd[argmin(dd)])] from pyraf import iraf bbb = iraf.wcsctran('STDIN', 'STDOUT', img + '[0]', Stdin=lll, inwcs='world', units='degrees degrees', outwcs='logical', columns='1 2', formats='%10.5f %10.5f', Stdout=1)[3] if 'INDEF' not in bbb and float(string.split(bbb)[0]) <= _xdimen and float( string.split(bbb)[1]) <= _ydimen and float( string.split(bbb)[0]) >= 0 and float( string.split(bbb)[1]) >= 0: #print str(std[argmin(dd)])+' in the field '+str(bbb) _RA = rastd[argmin(dd)] _DEC = decstd[argmin(dd)] _SN = std[argmin(dd)] else: #print 'out '+str(bbb) _RA, _DEC, _SN = '', '', '' return _RA, _DEC, _SN
def Docosmic(img, _sigclip=5.5, _sigfrac=0.2, _objlim=4.5): import time start = time.time() import lsc import re, os, string import numpy as np import tempfile ar, hd = fits.getdata(img, header=True) if 'TELID' in hd: _tel = hd['TELID'] elif 'telescop' in hd: _tel = hd['telescop'] else: _tel = 'extdata' if _tel in ['fts', 'ftn']: temp_file0 = next(tempfile._get_candidate_names()) lsc.delete(temp_file0) out_fits = fits.PrimaryHDU(header=hd, data=ar) try: out_fits.scale('float32', bzero=0, bscale=1) except TypeError as e: print "FITS rescaling failed (but it probably doesn't matter). See Astropy Issue #5955." print e out_fits.writeto(temp_file0, overwrite=True, output_verify='fix') ar = fits.getdata(temp_file0) lsc.delete(temp_file0) gain = hd['GAIN'] sat = 35000 rdnoise = hd['RDNOISE'] else: if 'gain' in hd: gain = hd['GAIN'] else: print 'warning GAIN not found' gain = 1 if 'saturate' in hd: sat = hd['SATURATE'] else: print 'warning SATURATE not found' sat = 60000 if 'RDNOISE' in hd: rdnoise = hd['RDNOISE'] else: print 'warning RDNOISE not found' rdnoise = 1 if '-e91.' in img: ar[ar < readkey3(hd, 'datamin')] = sat _pssl = 0. else: # need to trick LACosmic into using the right sigma for a sky-subtracted image med = np.median(ar) # median pixel of image (in ADU) noise = 1.4826 * np.median( np.abs(ar - med)) # using median absolute deviation instead of sigma _pssl = gain * noise**2 - rdnoise**2 / gain - med # previously subtracted sky level ar[ar < -_pssl] = sat # change (what will be) negative values to saturated print 'gain sat noise sigclip objlim sigfrac pssl' print '{:<7.1f} {:<7.0f} {:<7.1f} {:<7.1f} {:<7.0f} {:<7.1f} {:<7.2f}'.format( gain, sat, rdnoise, _sigclip, _objlim, _sigfrac, _pssl) niter = 1 c = lsc.cosmics.cosmicsimage(ar, pssl=_pssl, gain=gain, readnoise=rdnoise, sigclip=5, sigfrac=0.3, objlim=5, satlevel=sat) c.run(maxiter=niter) out = re.sub('.fits', '.clean.fits', string.split(img, '/')[-1]) outmask = re.sub('.fits', '.mask.fits', string.split(img, '/')[-1]) outsat = re.sub('.fits', '.sat.fits', string.split(img, '/')[-1]) out1 = c.cleanarray out2 = c.cleanarray - c.rawarray out3 = c.getsatstars() out_fits = fits.PrimaryHDU(header=hd, data=out1) out_fits.writeto(out, overwrite=True, output_verify='fix') # we are going to register the mask for the template image, # so it makes sense to save it as a float instead of an int if 'temp' in img: pixtype = 'float32' else: pixtype = 'uint8' out_fits = fits.PrimaryHDU(header=hd, data=(out2 != 0).astype(pixtype)) out_fits.writeto(outmask, overwrite=True, output_verify='fix') out_fits = fits.PrimaryHDU(header=hd, data=(out3 != 0).astype('uint8')) out_fits.writeto(outsat, overwrite=True, output_verify='fix') print 'time to do cosmic ray rejection:', time.time() - start return out, outmask, outsat
def updateDatabase(tarfile): import string, os, re, math, sys, shutil, glob, socket, pickle import MySQLdb host = socket.gethostname() import ntt from ntt.util import readkey3, readhdr from mysqldef import updatevalue, getvaluefromarchive, dbConnect, getfromdataraw, ingestredu os.system('tar -zxvf ' + str(tarfile)) pklfile = re.sub('tar.gz', 'pkl', tarfile) if glob.glob(pklfile): pkl_file = open(pklfile, 'rb') mydict2 = pickle.load(pkl_file) pkl_file.close() fitsfile = [] for i in mydict2: try: command = mydict2[i]['command'] if command == 'ingestredu': lista = mydict2[i]['lista'] instrument = mydict2[i]['instument'] output = mydict2[i]['output'] ingestredu(lista) if output not in fitsfile: fitsfile.append(output) elif command == 'getvaluefromarchive': table = mydict2[i]['table'] column = mydict2[i]['column'] value = mydict2[i]['value'] _input = mydict2[i]['input'] hh = getvaluefromarchive(table, column, _input, value) elif command == 'updatevalue': table = mydict2[i]['table'] column = mydict2[i]['column'] value = mydict2[i]['value'] _input = mydict2[i]['input'] if value != 'voce': updatevalue(table, column, value, _input) else: if column == 'voce': for voce in hh: if voce != 'id' and voce != 'filename': updatevalue(table, voce, hh[voce], _input) else: updatevalue(table, column, hh[column], _input) else: print 'warning: command not recognise' except: print '#####################' print mydict2[i] print 'problems' print fitsfile dir1 = '/data/obsdata/y20' + string.split(tarfile, '_')[2][-2:] + '/' dir2 = '/data/obsdata/y20' + string.split( tarfile, '_')[2][-2:] + '/' + string.split(tarfile, '_')[2] + '/' if not os.path.isdir(dir1): os.mkdir(dir1) if not os.path.isdir(dir2): os.mkdir(dir2) for img in fitsfile: arcfile = readkey3(readhdr(img), 'ARCFILE') print arcfile try: bbbb = getvaluefromarchive('datarawNTT', 'filename', arcfile, '*') except: bbbb = '' if bbbb: directoryraw = bbbb['filepath'] directoryreduced = re.sub('raw/', '', directoryraw) + 'reduced/' else: dir3 = dir2 + string.split(tarfile, '_')[3] + '_' + string.split( tarfile, '_')[4] + '/' if not os.path.isdir(dir3): os.mkdir(dir3) directoryreduced = dir3 + 'reduced/' if not os.path.isdir(directoryreduced): os.mkdir(directoryreduced) os.system('cp ' + img + ' ' + directoryreduced) updatevalue('redulogNTT', 'filepath', directoryreduced, img) updatevalue('datareduNTT', 'filepath', directoryreduced, img)
def updateDatabase(tarfile): import string,os,re,math,sys,shutil,glob,socket,pickle import MySQLdb host=socket.gethostname() import ntt from ntt.util import readkey3,readhdr from mysqldef import updatevalue, getvaluefromarchive, dbConnect,getfromdataraw,ingestredu os.system('tar -zxvf '+str(tarfile)) pklfile=re.sub('tar.gz','pkl',tarfile) if glob.glob(pklfile): pkl_file = open(pklfile, 'rb') mydict2 = pickle.load(pkl_file) pkl_file.close() fitsfile=[] for i in mydict2: try: command=mydict2[i]['command'] if command=='ingestredu': lista=mydict2[i]['lista'] instrument=mydict2[i]['instument'] output=mydict2[i]['output'] ingestredu(lista) if output not in fitsfile: fitsfile.append(output) elif command=='getvaluefromarchive': table=mydict2[i]['table'] column=mydict2[i]['column'] value=mydict2[i]['value'] _input=mydict2[i]['input'] hh=getvaluefromarchive(table,column,_input,value) elif command=='updatevalue': table=mydict2[i]['table'] column=mydict2[i]['column'] value=mydict2[i]['value'] _input=mydict2[i]['input'] if value!='voce': updatevalue(table,column,value,_input) else: if column=='voce': for voce in hh: if voce!='id' and voce!='filename': updatevalue(table,voce,hh[voce],_input) else: updatevalue(table,column,hh[column],_input) else: print 'warning: command not recognise' except: print '#####################' print mydict2[i] print 'problems' print fitsfile dir1='/data/obsdata/y20'+string.split(tarfile,'_')[2][-2:]+'/' dir2='/data/obsdata/y20'+string.split(tarfile,'_')[2][-2:]+'/'+string.split(tarfile,'_')[2]+'/' if not os.path.isdir(dir1): os.mkdir(dir1) if not os.path.isdir(dir2): os.mkdir(dir2) for img in fitsfile: arcfile=readkey3(readhdr(img),'ARCFILE') print arcfile try: bbbb=getvaluefromarchive('datarawNTT','filename',arcfile,'*') except: bbbb='' if bbbb: directoryraw=bbbb['filepath'] directoryreduced=re.sub('raw/','',directoryraw)+'reduced/' else: dir3=dir2+string.split(tarfile,'_')[3]+'_'+string.split(tarfile,'_')[4]+'/' if not os.path.isdir(dir3): os.mkdir(dir3) directoryreduced=dir3+'reduced/' if not os.path.isdir(directoryreduced): os.mkdir(directoryreduced) os.system('cp '+img+' '+directoryreduced) updatevalue('redulogNTT','filepath',directoryreduced,img) updatevalue('datareduNTT','filepath',directoryreduced,img)
def Docosmic(img,_sigclip=5.5,_sigfrac=0.2,_objlim=4.5): import time start=time.time() import lsc import re,os,string import numpy as np import tempfile ar, hd = fits.getdata(img, header=True) if 'TELID' in hd: _tel=hd['TELID'] elif 'telescop' in hd: _tel = hd['telescop'] else: _tel='extdata' if _tel in ['fts','ftn']: temp_file0 = next(tempfile._get_candidate_names()) lsc.delete(temp_file0) out_fits = fits.PrimaryHDU(header=hd,data=ar) try: out_fits.scale('float32',bzero=0,bscale=1) except TypeError as e: print "FITS rescaling failed (but it probably doesn't matter). See Astropy Issue #5955." print e out_fits.writeto(temp_file0, overwrite=True, output_verify='fix') ar = fits.getdata(temp_file0) lsc.delete(temp_file0) gain = hd['GAIN'] sat = 35000 rdnoise = hd['RDNOISE'] else: if 'gain' in hd: gain = hd['GAIN'] else: print 'warning GAIN not found' gain = 1 if 'saturate' in hd: sat = hd['SATURATE'] else: print 'warning SATURATE not found' sat = 60000 if 'RDNOISE' in hd: rdnoise = hd['RDNOISE'] else: print 'warning RDNOISE not found' rdnoise = 1 if '-e91.' in img: ar[ar < readkey3(hd, 'datamin')] = sat _pssl = 0. else: # need to trick LACosmic into using the right sigma for a sky-subtracted image med = np.median(ar) # median pixel of image (in ADU) noise = 1.4826*np.median(np.abs(ar - med)) # using median absolute deviation instead of sigma _pssl = gain*noise**2 - rdnoise**2/gain - med # previously subtracted sky level ar[ar < -_pssl] = sat # change (what will be) negative values to saturated print 'gain sat noise sigclip objlim sigfrac pssl' print '{:<7.1f} {:<7.0f} {:<7.1f} {:<7.1f} {:<7.0f} {:<7.1f} {:<7.2f}'.format(gain, sat, rdnoise, _sigclip, _objlim, _sigfrac, _pssl) niter = 1 c = lsc.cosmics.cosmicsimage(ar, pssl=_pssl, gain=gain, readnoise=rdnoise, sigclip=5, sigfrac=0.3 , objlim=5, satlevel=sat) c.run(maxiter = niter) out=re.sub('.fits','.clean.fits',string.split(img,'/')[-1]) outmask=re.sub('.fits','.mask.fits',string.split(img,'/')[-1]) outsat=re.sub('.fits','.sat.fits',string.split(img,'/')[-1]) out1=c.cleanarray out2=c.cleanarray-c.rawarray out3=c.getsatstars() out_fits = fits.PrimaryHDU(header=hd,data=out1) out_fits.writeto(out, overwrite=True, output_verify='fix') # we are going to register the mask for the template image, # so it makes sense to save it as a float instead of an int if 'temp' in img: pixtype = 'float32' else: pixtype = 'uint8' out_fits = fits.PrimaryHDU(header=hd,data=(out2!=0).astype(pixtype)) out_fits.writeto(outmask, overwrite=True, output_verify='fix') out_fits = fits.PrimaryHDU(header=hd,data=(out3!=0).astype('uint8')) out_fits.writeto(outsat, overwrite=True, output_verify='fix') print 'time to do cosmic ray rejection:', time.time()-start return out,outmask,outsat
def ingestredu(imglist, force='no', dataredutable='photlco', filetype=1): import string, re, os, sys import lsc from lsc.util import readkey3, readhdr from datetime import datetime hostname, username, passwd, database = lsc.mysqldef.getconnection('lcogt2') conn = lsc.mysqldef.dbConnect(hostname, username, passwd, database) for fullpath in imglist: path, img = os.path.split(fullpath) path += '/' exist = lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename', string.split(img, '/')[-1], column2='filename') exist2 = lsc.mysqldef.getfromdataraw(conn, 'photlcoraw', 'filename', string.split(img, '/')[-1], column2='filename, groupidcode') if exist2: print exist2 _groupidcode = exist2[0]['groupidcode'] else: _groupidcode = '' if exist: if force == 'yes': print img, database lsc.mysqldef.deleteredufromarchive( string.split(img, '/')[-1], dataredutable) print 'delete line from ' + str(database) exist = lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename', string.split(img, '/')[-1], column2='filename') if not exist or force == 'update': hdr = readhdr(fullpath) _targetid = lsc.mysqldef.targimg(fullpath) try: _tracknumber = int(readkey3(hdr, 'TRACKNUM')) except: _tracknumber = 0 if hdr.get('TELESCOP'): _tel = hdr.get('TELESCOP') else: _tel = '' if _tel in ['Faulkes Telescope South', 'fts']: _tel = '2m0-02' elif _tel in ['Faulkes Telescope North', 'ftn']: _tel = '2m0-01' _inst = hdr.get('instrume') dictionary={'dateobs':readkey3(hdr,'date-obs'),'dayobs':readkey3(hdr,'DAY-OBS'),'filename':img,'filepath':path,'filetype':filetype,'targetid':_targetid,\ 'exptime':readkey3(hdr,'exptime'), 'filter':readkey3(hdr,'filter'),'mjd':readkey3(hdr,'mjd'),'tracknumber':_tracknumber,'groupidcode':_groupidcode,\ 'telescope':_tel,'airmass':readkey3(hdr,'airmass'),'objname':readkey3(hdr,'object'),'ut':readkey3(hdr,'ut'),\ 'wcs':readkey3(hdr,'wcserr'),'instrument':_inst,'ra0':readkey3(hdr,'RA'),'dec0':readkey3(hdr,'DEC')} _telid = lsc.mysqldef.getfromdataraw(conn, 'telescopes', 'name', _tel, column2='id') if not _telid: print 'Telescope ', _tel, ' not recognized. Adding to telescopes table.' # the short name is needed to calibrate the magnitude with extinction lsc.mysqldef.insert_values(conn, 'telescopes', { 'name': _tel, 'shortname': readkey3(hdr, 'SITEID') }) _telid = lsc.mysqldef.getfromdataraw(conn, 'telescopes', 'name', _tel, column2='id') telid = _telid[0]['id'] dictionary['telescopeid'] = str(telid) _instid = lsc.mysqldef.getfromdataraw(conn, 'instruments', 'name', _inst, column2='id') if not _instid: print 'Instrument ', _inst, ' not recognized. Adding to instruments table.' lsc.mysqldef.insert_values(conn, 'instruments', { 'name': _inst, 'type': guess_instrument_type(_inst) }) _instid = lsc.mysqldef.getfromdataraw(conn, 'instruments', 'name', _inst, column2='id') instid = _instid[0]['id'] dictionary['instrumentid'] = str(instid) dictionary['lastunpacked'] = str(datetime.utcnow()) print dictionary print 'insert reduced' ggg = lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename', str(img), '*') if not ggg: lsc.mysqldef.insert_values(conn, dataredutable, dictionary) else: for voce in dictionary: lsc.mysqldef.updatevalue(dataredutable, voce, dictionary[voce], string.split(img, '/')[-1]) else: print 'already ingested'
def absphot(img,_field,_catalogue,_fix,_color,rejection,_interactive,_type='fit',redo=False,show=False,cutmag=-1,database='dataredulco',_calib='sloan'): from astropy.io import fits import lsc import math import sys,re,string,os from lsc.util import readkey3, readhdr from numpy import array, compress, zeros, median, std, asarray, isfinite,mean from pyraf import iraf if show: from pylab import ion,plot,draw,clf import time iraf.noao(_doprint=0) iraf.digiphot(_doprint=0) iraf.daophot(_doprint=0) iraf.images(_doprint=0) iraf.imcoords(_doprint=0) iraf.proto(_doprint=0) t = fits.open(img) tbdata = t[1].data hdr2=t[1].header hdr=lsc.util.readhdr(img) _cat=readkey3(hdr,'catalog') _telescope=lsc.util.readkey3(hdr,'telescop') _instrume=lsc.util.readkey3(hdr,'instrume') _filter=lsc.util.readkey3(hdr,'filter') _airmass=lsc.util.readkey3(hdr,'airmass') _exptime=lsc.util.readkey3(hdr,'exptime') _date=lsc.util.readkey3(hdr,'date-obs') _object=lsc.util.readkey3(hdr,'object') _ra=lsc.util.readkey3(hdr,'RA') _dec=lsc.util.readkey3(hdr,'DEC') print _filter if _telescope in ['lsc','1m0-04','1m0-05','1m0-09']: kk=lsc.sites.extintion('ctio') elif _telescope in ['elp','1m0-08']: kk=lsc.sites.extintion('mcdonald') elif _telescope in ['cpt','1m0-12','1m0-10','1m0-13']: kk=lsc.sites.extintion('southafrica') elif _telescope in ['ftn','Faulkes Telescope North']: kk=lsc.sites.extintion('mauna') elif _telescope in ['1m0-03','1m0-11','coj','fts','Faulkes Telescope South']: kk=lsc.sites.extintion('siding') if _calib not in ['sloan','sloanprime','natural','apass','']: colorefisso=lsc.sites.colfix(_instrume) else: colorefisso=lsc.sites.colfix(_instrume,_calib) print redo print _cat if _cat and not redo: print 'already calibrated' else: try: lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.sn2.fits','.fits',img),'/')[-1]) if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]): lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]) except: print 'module mysqldef not found' column=makecatalogue([img])[_filter][img] rasex=array(column['ra0'],float) decsex=array(column['dec0'],float) if _type=='fit': magsex=array(column['smagf'],float) magerrsex=array(column['smagerrf'],float) elif _type=='ph': magsex=array(column['magp3'],float) magerrsex=array(column['merrp3'],float) else: sys.exit(_type+' not valid (ph or fit)') print len(rasex) if not cutmag: cutmag=99 #else: cutmag= cutmag-2.5*math.log10(float(_exptime)) if len(compress( array(magsex) < float(cutmag) , magsex)) < 5 : cutmag=99 # not cut if only few object rasex = compress(array(magsex,float)<=cutmag,rasex) decsex = compress(array(magsex,float)<=cutmag,decsex) magerrsex = compress(array(magsex,float)<=cutmag,magerrsex) magsex = compress(array(magsex,float)<=cutmag,array(magsex)) print len(rasex) if _interactive: iraf.set(stdimage='imt1024') iraf.display(re.sub('.sn2','',img),1,fill=True,Stdout=1) vector=[] for i in range(0,len(rasex)): vector.append(str(rasex[i])+' '+str(decsex[i])) xy = iraf.wcsctran('STDIN',output="STDOUT",Stdin=vector,Stdout=1,image=img,inwcs='world',units='degrees degrees',outwcs='logical',\ formats='%10.1f %10.1f',verbose='yes')[3:] iraf.tvmark(1,'STDIN',Stdin=list(xy),mark="circle",number='yes',label='no',radii=10,nxoffse=5,nyoffse=5,color=207,txsize=2) # raw_input('here') if _catalogue: ######## use external catalogue if _catalogue[0]=='/': stdcooC=lsc.lscastrodef.readtxt(_catalogue) else: stdcooC=lsc.lscastrodef.readtxt(lsc.__path__[0]+'/standard/cat/'+_catalogue) rastdC,decstdL=array(stdcooC['ra'],float),array(stdcooC['dec'],float) lsc.util.delete('tmp.stdL.pix') colonne=str(stdcooC['rapos'])+' '+str(stdcooC['decpos']) if _catalogue[0]=='/': iraf.wcsctran(_catalogue,'tmp.stdL.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',\ columns=colonne,formats='%10.1f %10.1f',verbose='no') else: iraf.wcsctran(lsc.__path__[0]+'/standard/cat/'+_catalogue,'tmp.stdL.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',\ columns=colonne,formats='%10.1f %10.1f',verbose='no') standardpixC=lsc.lscastrodef.readtxt('tmp.stdL.pix') if _interactive: iraf.tvmark(1,'tmp.stdL.pix',mark="circle",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=204,txsize=2) xstdC=standardpixC['ra'] ystdC=standardpixC['dec'] idstdC=standardpixC['id'] xstdC=compress((array(xstdC,float)<readkey3(hdr,'XDIM'))&(array(xstdC,float)>0)&(array(ystdC,float)>0)&(array(ystdC,float)<readkey3(hdr,'YDIM')),xstdC) xstdL=xstdLL=xstdS=xstdC standardpixL=standardpixLL=standardpixS=standardpixC stdcooL=stdcooLL=stdcooS=stdcooC else: ######## check if it is landolt field stdcooL=lsc.lscastrodef.readtxt(lsc.__path__[0]+'/standard/cat/landolt.cat') rastdL,decstdL=array(stdcooL['ra'],float),array(stdcooL['dec'],float) lsc.util.delete('tmp.stdL.pix') iraf.wcsctran(lsc.__path__[0]+'/standard/cat/landolt.cat','tmp.stdL.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',\ columns='1 2',formats='%10.1f %10.1f',verbose='no') standardpixL=lsc.lscastrodef.readtxt('tmp.stdL.pix') if _interactive: iraf.tvmark(1,'tmp.stdL.pix',mark="circle",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=204,txsize=2) print 'yelow circles sextractor' xstdL=standardpixL['ra'] ystdL=standardpixL['dec'] idstdL=standardpixL['id'] xstdL=compress((array(xstdL,float)<readkey3(hdr,'XDIM'))&(array(xstdL,float)>0)&(array(ystdL,float)>0)&(array(ystdL,float)<readkey3(hdr,'YDIM')),xstdL) ######## check if it is Stetson field stdcooLL=lsc.lscastrodef.readtxt(lsc.__path__[0]+'/standard/cat/StetsonCat.dat') ww=asarray([i for i in range(len(stdcooLL['ra'])) if ( abs(float(stdcooLL['ra'][i])-float(_ra))<.2 and abs(float(stdcooLL['dec'][i])-_dec)<.2 )]) if len(ww)>0: for hh in stdcooLL.keys(): if type(stdcooLL[hh])!=int: if hh not in ['id','ra','dec']: stdcooLL[hh]=array(array(stdcooLL[hh])[ww],float) else: stdcooLL[hh]=array(stdcooLL[hh])[ww] lll=[] for i in range(0,len(stdcooLL['ra'])): lll.append(stdcooLL['ra'][i]+' '+stdcooLL['dec'][i]) rastdLL,decstdLL=array(stdcooLL['ra'],float),array(stdcooLL['dec'],float) lsc.util.delete('tmp.stdLL.pix') iraf.wcsctran('STDIN','tmp.stdLL.pix',img,inwcs='world',Stdin=lll,units='degrees degrees',outwcs='logical',\ columns='1 2',formats='%10.1f %10.1f',verbose='no') if _interactive: iraf.tvmark(1,'tmp.stdLL.pix',mark="cross",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=204,txsize=2) print 'red crosses Stetson' standardpixLL={} for ii in stdcooLL.keys(): standardpixLL[ii]=stdcooLL[ii] standardpixLL['ra']=array(iraf.proto.fields('tmp.stdLL.pix',fields='1',Stdout=1),float) #standardpixLL['ra'] standardpixLL['dec']=array(iraf.proto.fields('tmp.stdLL.pix',fields='2',Stdout=1),float) #standardpixLL['dec'] xstdLL=array(iraf.proto.fields('tmp.stdLL.pix',fields='1',Stdout=1),float) #standardpixLL['ra'] ystdLL=array(iraf.proto.fields('tmp.stdLL.pix',fields='2',Stdout=1),float) #standardpixLL['dec'] idstdLL=standardpixLL['id'] xstdLL=compress((array(xstdLL,float)<readkey3(hdr,'XDIM'))&(array(xstdLL,float)>0)&(array(ystdLL,float)>0)&(array(ystdLL,float)<readkey3(hdr,'YDIM')),xstdLL) ######## check if it is sloan field magsel0,magsel1=12,18 _ids=lsc.lscastrodef.sloan2file(_ra,_dec,20,float(magsel0),float(magsel1),'_tmpsloan.cat') ascifile='_tmpsloan.cat' stdcooS=lsc.lscastrodef.readtxt(ascifile) rastdS,decstdS=array(stdcooS['ra'],float),array(stdcooS['dec'],float) lsc.util.delete('tmp.stdS.pix') iraf.wcsctran(ascifile,'tmp.stdS.pix',img,inwcs='world',units='degrees degrees',outwcs='logical',columns='1 2',formats='%10.1f %10.1f',verbose='no') standardpixS=lsc.lscastrodef.readtxt('tmp.stdS.pix') if _interactive: iraf.tvmark(1,'tmp.stdS.pix',mark="cross",number='yes',label='no',radii=8,nxoffse=5,nyoffse=5,color=205,txsize=2) print 'green cross sloan' xstdS=standardpixS['ra'] ystdS=standardpixS['dec'] idstdS=standardpixS['id'] xstdS=compress((array(xstdS,float)<readkey3(hdr,'XDIM'))&(array(xstdS,float)>0)&(array(ystdS,float)>0)&(array(ystdS,float)<readkey3(hdr,'YDIM')),xstdS) ##############################################3 if not _catalogue and len(xstdLL)>0: xstdL=xstdLL standardpixL=standardpixLL stdcooL=stdcooLL if _filter in ['U', 'B', 'V', 'R','I','Bessell-B','Bessell-V','Bessell-R','Bessell-I']: filters={'U':'U', 'B':'B', 'V':'V', 'R':'R', 'I':'I','Bessell-B':'B','Bessell-V':'V','Bessell-R':'R','Bessell-I':'I'} if _color: colors=lsc.myloopdef.chosecolor(_color,False) if not colors: colors={'U':['UB'],'B':['UB','BV'],'V':['BV','VR'],'R':['VR','RI'],'I':['RI']} else: colors={'U':['UB'],'B':['UB','BV'],'V':['BV','VR'],'R':['VR','RI'],'I':['RI']} if _field=='sloan': standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]} print 'filters and field selected do not match' else: _field='landolt' if len(xstdL)>=1: standardpix=standardpixL stdcoo=stdcooL if not _catalogue: if len(xstdLL)>0: _catalogue='StetsonCat.dat' else: _catalogue='landolt.dat' elif len(xstdS)>=1: if not _catalogue: _catalogue='sdss8' standardpix=standardpixS stdcoo=stdcooS stdcoo=lsc.lscastrodef.transformsloanlandolt(stdcoo) if not _catalogue: _catalogue='sdss8' print '\n### transform sloan in landolt' else: print 'landolt, but catalogue not found' standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]} elif _filter in ['up','gp','rp','ip','zs','SDSS-G','SDSS-R','SDSS-I','Pan-Starrs-Z']: filters={'up':'u','ip':'i','gp':'g','rp':'r','zs':'z','SDSS-G':'g','SDSS-R':'r','SDSS-I':'i','Pan-Starrs-Z':'z'} if _color: colors=lsc.myloopdef.chosecolor(_color,False) if not colors: colors={'i':['ri','iz'],'r':['gr','ri'],'g':['ug','gr'],'z':['iz'],'u':['ug']} else: colors={'i':['ri','iz'],'r':['gr','ri'],'g':['ug','gr'],'z':['iz'],'u':['ug']} if _field=='landolt': standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]} print 'filters and field selected do not match' else: _field='sloan' if len(xstdS)>=1: if not _catalogue: _catalogue='sdss8' standardpix=standardpixS stdcoo=stdcooS elif len(xstdL)>=1: standardpix=standardpixL stdcoo=stdcooL stdcoo=lsc.lscastrodef.transformlandoltsloan(stdcoo) if not _catalogue: _catalogue='landolt.dat' print '\n### transform landolt to sloan' else: print 'sloan, but not in the sdss footprint' standardpix,stdcoo={'ra':[9999],'dec':[9999],'id':[1]},{'ra':[9999],'dec':[9999]} xstd=standardpix['ra'] ystd=standardpix['dec'] idstd=standardpix['id'] rastd,decstd=array(stdcoo['ra'],float),array(stdcoo['dec'],float) xstd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)&(array(ystd,float)<readkey3(hdr,'YDIM')),xstd) if len(xstd0)>1: ######## go only if standard stars are in the field ########## magstd0={} airmass0={} result={} fileph={} print '\n### standard field: '+str(_field) ystd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\ &(array(ystd,float)<readkey3(hdr,'YDIM')),ystd) rastd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\ &(array(ystd,float)<readkey3(hdr,'YDIM')),rastd) decstd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\ &(array(ystd,float)<readkey3(hdr,'YDIM')),decstd) idstd0=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\ &(array(ystd,float)<readkey3(hdr,'YDIM')),idstd) stdcoo0={} for key in stdcoo.keys(): if key in 'ugrizUBVRI': stdcoo0[key]=compress((array(xstd,float)<readkey3(hdr,'XDIM'))&(array(xstd,float)>0)&(array(ystd,float)>0)\ &(array(ystd,float)<readkey3(hdr,'YDIM')),stdcoo[key]) ############################################################### # pos0 = standard pos1 = sextractor distvec,pos0,pos1=lsc.lscastrodef.crossmatch(array(rastd0),array(decstd0),array(rasex),array(decsex),10) for key in stdcoo0.keys(): stdcoo0[key]=stdcoo0[key][pos0] rastd0=rastd0[pos0] decstd0=decstd0[pos0] idstd0=idstd0[pos0] rasex=rasex[pos1] decsex=decsex[pos1] # after change in may 2013 mag in sn2.fits file are already at 1s magsex=magsex[pos1]-kk[filters[_filter]]*float(_airmass) # - K x airmass # magsex=magsex[pos1]+2.5*math.log10(float(_exptime))-kk[filters[_filter]]*float(_airmass) # mag exptime - K x airmass ################################################################################# if _field=='landolt': print '\n### landolt system' for _filtlandolt in 'UBVRI': if _filtlandolt==filters[_filter]: airmass0[_filtlandolt]= 0 #_airmass else: airmass0[_filtlandolt]= 0 magstd0[_filtlandolt]=stdcoo0[_filtlandolt] fileph['mU']=zeros(len(rastd0))+999 fileph['mB']=zeros(len(rastd0))+999 fileph['mV']=zeros(len(rastd0))+999 fileph['mR']=zeros(len(rastd0))+999 fileph['mI']=zeros(len(rastd0))+999 fileph['V']=magstd0['V'] fileph['BV']=array(array(magstd0['B'],float)-array(magstd0['V'],float),str) fileph['UB']=array(array(magstd0['U'],float)-array(magstd0['B'],float),str) fileph['VR']=array(array(magstd0['V'],float)-array(magstd0['R'],float),str) fileph['RI']=array(array(magstd0['R'],float)-array(magstd0['I'],float),str) elif _field=='sloan': for _filtsloan in 'ugriz': if _filtsloan==filters[_filter]: airmass0[_filtsloan]= 0 # _airmass else: airmass0[_filtsloan]=0 magstd0[_filtsloan]=stdcoo0[_filtsloan] fileph['mu']=zeros(len(rastd0))+999 fileph['mg']=zeros(len(rastd0))+999 fileph['mr']=zeros(len(rastd0))+999 fileph['mi']=zeros(len(rastd0))+999 fileph['mz']=zeros(len(rastd0))+999 fileph['r']=magstd0['r'] fileph['gr']=array(array(magstd0['g'],float)-array(magstd0['r'],float),str) fileph['ri']=array(array(magstd0['r'],float)-array(magstd0['i'],float),str) fileph['ug']=array(array(magstd0['u'],float)-array(magstd0['g'],float),str) fileph['iz']=array(array(magstd0['i'],float)-array(magstd0['z'],float),str) ######################################################################################## zero=[] magcor=[] fil = open(re.sub('.fits','.ph',img),'w') fil.write(str(_instrume)+' '+str(_date)+'\n') fil.write('*** '+_object+' '+str(len(magsex))+'\n') if _field=='landolt': fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(1),str(1),str(1),str(1),str(1))) # exptime fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(airmass0['U']),str(airmass0['B']),str(airmass0['V']),str(airmass0['R']),str(airmass0['I']))) elif _field=='sloan': fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(1),str(1),str(1),str(1),str(1))) # exptime fil.write('%6.6s\t%6.6s\t%6.6s\t%6.6s\t%6.6s\n' % (str(airmass0['u']),str(airmass0['g']),str(airmass0['r']),str(airmass0['i']),str(airmass0['z']))) for i in range(0,len(magsex)): fileph['m'+filters[_filter]][i]=magsex[i] # instrumental mangitude of std in pos0[i] if _field=='landolt': stringastandard='%12.12s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s' % (idstd0[i],fileph['V'][i],fileph['BV'][i],fileph['UB'][i],\ fileph['VR'][i],fileph['RI'][i]) fil.write('%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%60.60s\n' \ % (str(fileph['mU'][i]),str(fileph['mB'][i]),str(fileph['mV'][i]),str(fileph['mR'][i]),str(fileph['mI'][i]),str(stringastandard))) elif _field=='sloan': stringastandard='%12.12s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s' % (idstd0[i],fileph['r'][i],fileph['gr'][i],fileph['ug'][i],\ fileph['ri'][i],fileph['iz'][i]) fil.write('%7.7s\t%7.7s\t%7.7s\t%7.7s\t%7.7s\t%60.60s\n' \ % (str(fileph['mu'][i]),str(fileph['mg'][i]),str(fileph['mr'][i]),str(fileph['mi'][i]),str(fileph['mz'][i]),str(stringastandard))) zero.append(float(float(magstd0[filters[_filter]][i]))-float(magsex[i])) magcor.append(magsex[i]) fil.close() if show: import time from pylab import ion,plot,draw ion() aa=mean(compress(abs(array(zero))<99,zero)) xxx=compress((abs(array(magcor))<99)&(abs(array(zero))<99),magcor) yyy=compress((abs(array(zero))<99)&(abs(array(magcor))<99),zero) plot(xxx,yyy,'or') plot([min(compress(abs(array(magcor))<99,magcor)),max(compress(abs(array(magcor))<99,magcor))],[aa,aa],'-b') draw() print std(compress(abs(array(zero))<99,zero)) time.sleep(5) colorvec=colors[filters[_filter]] for col in colorvec: col0=magstd0[col[0]] col1=magstd0[col[1]] colstd0=array(col0,float)-array(col1,float) ################## sex ###################### colore=[] for i in range(0,len(pos1)): colore.append(colstd0[i]) colore1=compress(abs(array(zero))<50,array(colore)) zero1=compress(abs(array(zero))<50,array(zero)) zero2=compress(abs(array(colore1))<2,array(zero1)) colore2=compress(abs(array(colore1))<2,array(colore1)) if _fix: fisso=colorefisso[filters[_filter]+col] else: fisso='' if len(colore2)==0: print 'no calibration, '+_filter+' '+_field b,a,sa,sb=9999,9999,0,0 else: if _interactive: a,sa,b,sb=fitcol(colore2,zero2,_filter,col,fisso) else: a,sa,b,sb=fitcol2(colore2,zero2,_filter,col,fisso,show,rejection) print a,sa,b,sb result[filters[_filter]+col]=[a,sa,b,sb] if result: print '\n### zeropoint ..... done at airmass 0' if _catalogue: lsc.util.updateheader(img,0,{'CATALOG':[str(string.split(_catalogue,'/')[-1]),'catalogue source']}) stringa='' for ll in result: for kk in range(0,len(result[ll])): if not isfinite(result[ll][kk]): result[ll][kk]=0.0 valore='%3.3s %6.6s %6.6s %6.6s %6.6s' % (str(ll),str(result[ll][0]),str(result[ll][2]),str(result[ll][1]),str(result[ll][3])) print '### ',valore lsc.util.updateheader(img,0,{'zp'+ll:[str(valore),'a b sa sb in y=a+bx']}) if ll[0]==ll[2]: num=2 elif ll[0]==ll[1]: num=1 else: sys.exit('somthing wrong with color '+ll) print ll,num try: print 'zcol'+str(num),ll[1:],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1] lsc.mysqldef.updatevalue(database,'zcol'+str(num),ll[1:],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1]) lsc.mysqldef.updatevalue(database,'z'+str(num),result[ll][0],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1]) lsc.mysqldef.updatevalue(database,'c'+str(num),result[ll][2],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1]) lsc.mysqldef.updatevalue(database,'dz'+str(num),result[ll][1],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1]) lsc.mysqldef.updatevalue(database,'dc'+str(num),result[ll][3],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1]) if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]): lsc.mysqldef.updatevalue(database,'zcol'+str(num),ll[1:],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]) lsc.mysqldef.updatevalue(database,'z'+str(num),result[ll][0],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]) lsc.mysqldef.updatevalue(database,'c'+str(num),result[ll][2],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]) lsc.mysqldef.updatevalue(database,'dz'+str(num),result[ll][1],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]) lsc.mysqldef.updatevalue(database,'dc'+str(num),result[ll][3],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]) if result[ll][0]!=9999: print _catalogue lsc.mysqldef.updatevalue(database,'zcat',string.split(_catalogue,'/')[-1],string.split(re.sub('.sn2.fits','.fits',img),'/')[-1]) if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]): lsc.mysqldef.updatevalue(database,'zcat',string.split(_catalogue,'/')[-1],string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]) else: lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.sn2.fits','.fits',img),'/')[-1]) if os.path.isfile(string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]): lsc.mysqldef.updatevalue(database,'zcat','X',string.split(re.sub('.diff.sn2.fits','.fits',img),'/')[-1]) except: print 'module mysqldef not found'
def ingestredu(imglist,force='no',dataredutable='photlco',filetype=1): import string,re,os,sys import lsc from lsc.util import readkey3, readhdr from datetime import datetime hostname, username, passwd, database=lsc.mysqldef.getconnection('lcogt2') conn = lsc.mysqldef.dbConnect(hostname, username, passwd, database) for fullpath in imglist: path, img = os.path.split(fullpath) path += '/' exist=lsc.mysqldef.getfromdataraw(conn,dataredutable,'filename', string.split(img,'/')[-1],column2='filename') exist2=lsc.mysqldef.getfromdataraw(conn,'photlcoraw','filename', string.split(img,'/')[-1],column2='filename, groupidcode') if exist2: print exist2 _groupidcode=exist2[0]['groupidcode'] else: _groupidcode='' if exist: if force=='yes': print img,database lsc.mysqldef.deleteredufromarchive(string.split(img,'/')[-1],dataredutable) print 'delete line from '+str(database) exist=lsc.mysqldef.getfromdataraw(conn,dataredutable,'filename', string.split(img,'/')[-1],column2='filename') if not exist or force =='update': hdr=readhdr(fullpath) _targetid=lsc.mysqldef.targimg(fullpath) try: _tracknumber=int(readkey3(hdr,'TRACKNUM')) except: _tracknumber=0 if hdr.get('TELESCOP'): _tel=hdr.get('TELESCOP') else: _tel='' if _tel in ['Faulkes Telescope South','fts']: _tel='2m0-02' elif _tel in ['Faulkes Telescope North','ftn']: _tel='2m0-01' _inst=hdr.get('instrume') dictionary={'dateobs':readkey3(hdr,'date-obs'),'dayobs':readkey3(hdr,'DAY-OBS'),'filename':img,'filepath':path,'filetype':filetype,'targetid':_targetid,\ 'exptime':readkey3(hdr,'exptime'), 'filter':readkey3(hdr,'filter'),'mjd':readkey3(hdr,'mjd'),'tracknumber':_tracknumber,'groupidcode':_groupidcode,\ 'telescope':_tel,'airmass':readkey3(hdr,'airmass'),'objname':readkey3(hdr,'object'),'ut':readkey3(hdr,'ut'),\ 'wcs':readkey3(hdr,'wcserr'),'instrument':_inst,'ra0':readkey3(hdr,'RA'),'dec0':readkey3(hdr,'DEC')} _telid=lsc.mysqldef.getfromdataraw(conn,'telescopes','name',_tel,column2='id') if not _telid: print 'Telescope ',_tel,' not recognized. Adding to telescopes table.' lsc.mysqldef.insert_values(conn,'telescopes',{'name':_tel}) _telid=lsc.mysqldef.getfromdataraw(conn,'telescopes','name',_tel,column2='id') telid=_telid[0]['id'] dictionary['telescopeid']=str(telid) _instid=lsc.mysqldef.getfromdataraw(conn,'instruments','name',_inst,column2='id') if not _instid: print 'Instrument ',_inst,' not recognized. Adding to instruments table.' lsc.mysqldef.insert_values(conn,'instruments',{'name':_inst}) _instid=lsc.mysqldef.getfromdataraw(conn,'instruments','name',_inst,column2='id') instid=_instid[0]['id'] dictionary['instrumentid']=str(instid) dictionary['lastunpacked'] = str(datetime.utcnow()) print dictionary print 'insert reduced' ggg=lsc.mysqldef.getfromdataraw(conn, dataredutable, 'filename',str(img), '*') if not ggg: lsc.mysqldef.insert_values(conn,dataredutable,dictionary) else: for voce in dictionary: lsc.mysqldef.updatevalue(dataredutable,voce,dictionary[voce],string.split(img,'/')[-1]) else: print 'already ingested'