def update_phot_blagorodnova(image):
'''
Updates the DB with the aperture photometry extrated from the fits and the interpolated zeropoint.
'''
racen, deccen = cc.hour2deg(fitsutils.get_par(image, "OBJRA"),
fitsutils.get_par(image, "OBJRA"))
issub = 'f'
refsys = 'SDSSinterpolated'
filt = fitsutils.get_par(image, "FILTER")
exptime = fitsutils.get_par(image, "EXPTIME")
utdate = time_utils.jd2utc(fitsutils.get_par(image, "JD"), string=True)
mag = np.round(fitsutils.get_par(image, "APPMAG"), 3)
magerr = np.round(fitsutils.get_par(image, "APPMAGER"), 3)
limmag = np.round(fitsutils.get_par(image, "ZEROPT"), 3)
name = fitsutils.get_par(image, "NAME").replace("PTF", "")
observer = 'SEDMachine'
reducedby = 'Blagorodnova Automated Pipeline SEDM'
if mag == 0 or mag > limmag or mag > 22:
mag = 99
magerr = 99
db = pg.DB(dbname='ptftransient',
user='******',
passwd='followup',
host='yupana.caltech.edu')
getsrcidquery = "SELECT id from sources where name='%s'" % name
result = db.query(getsrcidquery)
for row in result.dictresult():
srcid = int(row['id'])
query3 = "SELECT id from phot WHERE sourceid=%d and instrumentid=64 and obsdate='%s' and filter='%s' and reducedby='Blagorodnova Automated Pipeline SEDM';" % (
srcid, utdate, filter)
result = db.query(query3)
print query3
if len(result.dictresult()) == 0:
print srcid, racen, deccen, utdate, exptime, filt, mag, magerr, limmag, issub, refsys, observer, reducedby
query2 = "INSERT INTO phot (sourceid,programid,instrumentid,ra,dec,obsdate,exptime,filter,mag,emag,limmag,issub,refsys,observer,reducedby) VALUES "
query2 += "(%d,1,64,%f,%f,'%s',%f,'%s',%f,%f,%f,'%s','%s','%s','%s');" % (
srcid, racen, deccen, utdate, exptime, filt, mag, magerr, limmag,
issub, refsys, observer, reducedby)
print query2
db.query(query2)
else:
for row in result.dictresult():
photid = int(row['id'])
query2 = "DELETE from phot WHERE id=%d;" % (photid)
print query2
db.query(query2)
query2 = "INSERT INTO phot (sourceid,programid,instrumentid,ra,dec,obsdate,exptime,filter,mag,emag,limmag,issub,refsys,observer,reducedby) VALUES "
query2 += "(%d,1,64,%f,%f,'%s',%f,'%s',%f,%f,%f,'%s','%s','%s','%s');" % (
srcid, racen, deccen, utdate, exptime, filt, mag, magerr,
limmag, issub, refsys, observer, reducedby)
print query2
db.query(query2)
def update_phot_blagorodnova(image):
'''
Updates the DB with the aperture photometry extrated from the fits and the interpolated zeropoint.
'''
racen, deccen = cc.hour2deg(fitsutils.get_par(image, "OBJRA"), fitsutils.get_par(image, "OBJRA"))
issub='f'
refsys='SDSSinterpolated'
filt=fitsutils.get_par(image, "FILTER")
exptime=fitsutils.get_par(image, "EXPTIME")
utdate=time_utils.jd2utc(fitsutils.get_par(image, "JD"), string=True)
mag = np.round(fitsutils.get_par(image, "APPMAG"), 3)
magerr = np.round(fitsutils.get_par(image, "APPMAGER"), 3)
limmag = np.round(fitsutils.get_par(image, "ZEROPT"), 3)
name = fitsutils.get_par(image, "NAME").replace("PTF", "")
observer = 'SEDMachine'
reducedby = 'Blagorodnova Automated Pipeline SEDM'
if mag==0 or mag > limmag or mag>22:
mag = 99
magerr = 99
db = pg.DB(dbname='ptftransient',user='******',passwd='followup',host='yupana.caltech.edu')
getsrcidquery = "SELECT id from sources where name='%s'"%name
result = db.query(getsrcidquery)
for row in result.dictresult():
srcid = int(row['id'])
query3 = "SELECT id from phot WHERE sourceid=%d and instrumentid=64 and obsdate='%s' and filter='%s' and reducedby='Blagorodnova Automated Pipeline SEDM';" % (srcid,utdate,filter)
result=db.query(query3)
print query3
if len(result.dictresult()) == 0:
print srcid,racen,deccen,utdate,exptime,filt,mag,magerr,limmag,issub,refsys,observer,reducedby
query2 = "INSERT INTO phot (sourceid,programid,instrumentid,ra,dec,obsdate,exptime,filter,mag,emag,limmag,issub,refsys,observer,reducedby) VALUES "
query2 += "(%d,1,64,%f,%f,'%s',%f,'%s',%f,%f,%f,'%s','%s','%s','%s');" % (srcid,racen,deccen,utdate,exptime,filt,mag,magerr,limmag,issub,refsys,observer,reducedby)
print query2
db.query(query2)
else:
for row in result.dictresult():
photid = int(row['id'])
query2 = "DELETE from phot WHERE id=%d;" % (photid)
print query2
db.query(query2)
query2 = "INSERT INTO phot (sourceid,programid,instrumentid,ra,dec,obsdate,exptime,filter,mag,emag,limmag,issub,refsys,observer,reducedby) VALUES "
query2 += "(%d,1,64,%f,%f,'%s',%f,'%s',%f,%f,%f,'%s','%s','%s','%s');" % (srcid,racen,deccen,utdate,exptime,filt,mag,magerr,limmag,issub,refsys,observer,reducedby)
print query2
db.query(query2)
def get_flats_counts(directory):
flatlist = []
corners = {
"g" : [1, 910, 1, 900],
"i" : [1, 910, 1060, 2045],
"r" : [1040, 2045, 1015, 2045],
"u" : [1030, 2045, 1, 900]
}
for f in glob.glob(directory + "/rc*fits"):
if fitsutils.has_par(f, "IMGTYPE") and fitsutils.get_par(f, "IMGTYPE") == "TWILIGHT":
flatlist.append(f)
if (len(flatlist)==0):
print "No suitable twilight flats found in directory: %s"%directory
return
counts = {"u":[], "g":[], "r":[], "i":[]}
sun_decs = {"u":[], "g":[], "r":[], "i":[]}
for f in flatlist:
print f
print fitsutils.get_par(f, "JD")
bias = rcred.get_overscan_bias_rc(f)
exptime = fitsutils.get_par(f, "EXPTIME")
sunsettime = fitsutils.get_par(f, "SUNSET")
utc = datetime.datetime.strptime(time_utils.jd2utc(fitsutils.get_par(f, "JD")), "%Y-%m-%d %H:%M:%S.%f")
st = datetime.datetime.strptime(sunsettime, "%H:%M")
elapsed = 3600*(utc.hour - st.hour) + 60*(utc.minute - st.minute) + (utc.second - st.second)
if (elapsed > 3000):
continue
print elapsed, utc, st
data = pf.open(f)[0].data
for band in corners.keys():
c = corners[band]
sf = data[c[0]:c[1], c[2]:c[3]]
if (np.percentile(sf, 90) < 55000):
counts[band].append( (np.percentile(sf, 90)-bias)/exptime)
sun_decs[band].append(elapsed)
for band in corners.keys():
coefs = np.polyfit(sun_decs[band], np.log10(counts[band]), deg=1, w=1./np.sqrt(np.log10(counts[band])))
p = np.poly1d(coefs)
x = np.linspace(np.min(sun_decs[band]), np.max(sun_decs[band]), 1000)
plt.plot(sun_decs[band], np.log10(counts[band]), "o", label=band)
plt.plot(x, p(x), label="Model "+band)
plt.xlabel("Elapsed second since Sunset")
plt.ylabel("Counts/s")
plt.legend()
plt.show()
def get_flats_counts(directory):
'''
Reads all the images in a directory marked as "twilight" and uses them to infer the average count rate for a given
number of seconds after the sunset.
It plots the number of counts and fits a 2D polynomial used to interpolate in the future.
'''
flatlist = []
corners = {
"g" : [1, 910, 1, 900],
"i" : [1, 910, 1060, 2045],
"r" : [1040, 2045, 1015, 2045],
"u" : [1030, 2045, 1, 900]
}
for f in glob.glob(directory + "/rc*fits"):
if fitsutils.has_par(f, "IMGTYPE") and fitsutils.get_par(f, "IMGTYPE") == "TWILIGHT":
flatlist.append(f)
if (len(flatlist)==0):
print "No suitable twilight flats found in directory: %s"%directory
return
counts = {"u":[], "g":[], "r":[], "i":[]}
sun_decs = {"u":[], "g":[], "r":[], "i":[]}
colors = {"u":"purple", "g":"green", "r":"r", "i":"orange"}
for f in flatlist:
print f
print fitsutils.get_par(f, "JD")
bias = rcred.get_overscan_bias_rc(f)
exptime = fitsutils.get_par(f, "EXPTIME")
sunsettime = fitsutils.get_par(f, "SUNSET")
utc = time_utils.jd2utc(fitsutils.get_par(f, "JD"))
st = datetime.datetime.strptime(sunsettime, "%H:%M")
elapsed = 3600*(utc.hour - st.hour) + 60*(utc.minute - st.minute) + (utc.second - st.second)
if (elapsed > 5000):
continue
print elapsed, utc, st
data = pf.open(f)[0].data
for band in corners.keys():
c = corners[band]
sf = data.T[c[0]:c[1], c[2]:c[3]]
if (np.percentile(sf, 90) < 55000):
counts[band].append( (np.percentile(sf, 90)-bias)/exptime)
sun_decs[band].append(elapsed)
t = Table(names=('filter', 'c2', 'c1', 'c0'), dtype=('S1', 'f8', 'f8', 'f8'))
for band in corners.keys():
print sun_decs[band], np.log10(counts[band])
coefs = np.polyfit(sun_decs[band], np.log10(counts[band]), deg=2)#, w=1./np.sqrt(np.log10(counts[band])))
p = np.poly1d(coefs)
x = np.linspace(np.min(sun_decs[band]), np.max(sun_decs[band]), 1000)
t.add_row([band, coefs[0], coefs[1], coefs[2]])
plt.plot(sun_decs[band], np.log10(counts[band]), "o", label=band, color=colors[band])
plt.plot(x, p(x), label="Model "+band, color=colors[band])
t.write("/tmp/test_flat", format='csv')
plt.xlabel("Elapsed second since Sunset")
plt.ylabel("log Counts/s")
plt.legend()
plt.show()
def plot_stats(statfile):
colors = {"ACQUISITION": "b", "SCIENCE": "r", "FOCUS": "g", "GUIDER": "k"}
s = np.genfromtxt(statfile, delimiter=",", dtype=None)
s.sort(order="f2")
s = s[s["f3"] > 1]
day_frac_diff = datetime.timedelta(
np.ceil((datetime.datetime.now() -
datetime.datetime.utcnow()).total_seconds()) / 3600 / 24)
datestat = np.array([time_utils.jd2utc(jd) for jd in s["f2"]])
datestat = datestat + day_frac_diff
#We add 5h to the UTC date, so it alwasy keeps the date of the end of the night.
day = ("%s" % (datestat[-1] + datetime.timedelta(5. / 24))).split()[0]
xfmt = md.DateFormatter('%H:%M')
f, ((ax1, ax2), (ax3, ax4), (ax5, ax6)) = plt.subplots(3, 2)
plt.suptitle("Statistics %s" % day)
f.set_figwidth(16)
f.set_figheight(12)
ax1.plot(datestat, s["f3"], ".-")
ax1.set_title('Number of bright sources extracted')
for im in set(s["f9"]):
mask = s["f9"] == im
ax2.plot(datestat[mask],
s["f4"][mask],
".",
color=colors[im],
label=im)
ax2.set_title('FWHM [arcsec]')
ax3.plot(datestat, s["f6"], ".-")
ax3.set_title('Background')
ax4.plot(datestat, s["f7"], ".-")
ax4.set_title('Airmass')
ax5.plot(datestat, s["f8"], ".-", label="Inside")
ax5.plot(datestat, s["f10"], ".-", label="Outside")
#ax5.plot(datestat, s["f11"], ".-")
ax5.set_title('Temperature')
ax6.plot(datestat, s["f5"], ".-")
ax6.set_title('Ellipticity')
ax1.xaxis.set_major_formatter(xfmt)
ax2.xaxis.set_major_formatter(xfmt)
ax3.xaxis.set_major_formatter(xfmt)
ax4.xaxis.set_major_formatter(xfmt)
ax5.xaxis.set_major_formatter(xfmt)
ax6.xaxis.set_major_formatter(xfmt)
labels = ax1.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
labels = ax2.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
labels = ax3.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
labels = ax4.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
labels = ax5.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
labels = ax6.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
ax2.legend(labelspacing=0.3,
loc="upper right",
fontsize=11,
numpoints=1,
frameon=False,
ncol=1,
fancybox=False,
shadow=True,
bbox_to_anchor=(1., 1.))
ax5.legend(labelspacing=0.3,
loc="upper left",
fontsize=11,
numpoints=1,
frameon=False,
ncol=1,
fancybox=False,
shadow=True,
bbox_to_anchor=(0., 1.))
plt.savefig(statfile.replace(".log", "%s.png" % (day)), bbox="tight")
def plot_stats(statfile):
colors = {"ACQUISITION":"b", "SCIENCE":"r", "FOCUS":"g", "GUIDER":"k"}
s = np.genfromtxt(statfile, delimiter=",", dtype=None)
s.sort(order="f2")
s = s[s["f3"]>1]
day_frac_diff = datetime.timedelta(np.ceil((datetime.datetime.now() - datetime.datetime.utcnow() ).total_seconds())/3600/24)
datestat = np.array([ time_utils.jd2utc(jd) for jd in s["f2"]])
datestat = datestat + day_frac_diff
#We add 5h to the UTC date, so it alwasy keeps the date of the end of the night.
day = ("%s"%(datestat[-1]+datetime.timedelta(5./24))).split()[0]
xfmt = md.DateFormatter('%H:%M')
f, ((ax1, ax2), (ax3, ax4), (ax5, ax6)) = plt.subplots(3, 2)
plt.suptitle("Statistics %s"%day)
f.set_figwidth(16)
f.set_figheight(12)
ax1.plot(datestat, s["f3"], ".-")
ax1.set_title('Number of bright sources extracted')
for im in set(s["f9"]):
mask = s["f9"]==im
ax2.plot(datestat[mask], s["f4"][mask], ".", color=colors[im], label=im)
ax2.set_title('FWHM [arcsec]')
ax3.plot(datestat, s["f6"], ".-")
ax3.set_title('Background')
ax4.plot(datestat, s["f7"], ".-")
ax4.set_title('Airmass')
ax5.plot(datestat, s["f8"], ".-", label="Inside")
ax5.plot(datestat, s["f10"], ".-", label="Outside")
#ax5.plot(datestat, s["f11"], ".-")
ax5.set_title('Temperature')
ax6.plot(datestat, s["f5"], ".-")
ax6.set_title('Ellipticity')
ax1.xaxis.set_major_formatter(xfmt)
ax2.xaxis.set_major_formatter(xfmt)
ax3.xaxis.set_major_formatter(xfmt)
ax4.xaxis.set_major_formatter(xfmt)
ax5.xaxis.set_major_formatter(xfmt)
ax6.xaxis.set_major_formatter(xfmt)
labels = ax1.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
labels = ax2.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
labels = ax3.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
labels = ax4.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
labels = ax5.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
labels = ax6.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
ax2.legend(labelspacing=0.3, loc="upper right", fontsize=11, numpoints=1, frameon=False, ncol=1, fancybox=False, shadow=True, bbox_to_anchor=(1., 1.))
ax5.legend(labelspacing=0.3, loc="upper left", fontsize=11, numpoints=1, frameon=False, ncol=1, fancybox=False, shadow=True, bbox_to_anchor=(0., 1.))
plt.savefig(statfile.replace(".log", "%s.png"%(day)), bbox="tight")