def is_on_target(image):
'''
Add as a parameter whether the image is on target or not.
'''
import coordinates_conversor as cc
ra, dec = cc.hour2deg(fitsutils.get_par(image, 'OBJRA'), fitsutils.get_par(image, 'OBJDEC'))
impf = fits.open(image)
w = wcs.WCS(impf[0].header)
filt = fitsutils.get_par(image, "FILTER")
#pra, pdec = wcs.wcs_sky2pix(np.array([ra, dec], ndmin=2), 1)[0]
pra, pdec = get_xy_coords(image, ra,dec)
shape = impf[0].data.shape
if (pra > 0) and (pra < shape[0]) and (pdec > 0) and (pdec < shape[1]):
fitsutils.update_par(image, "ONTARGET", 1)
return True
else:
fitsutils.update_par(image, "ONTARGET", 0)
return False
def clean_cosmic(f):
'''
From lacosmic.
'''
import cosmics
out = f.replace('.fits', '_clean.fits')
#If it does already exist, just return the name.
if (os.path.isfile(out)):
return out
#Otherwise, run the cosmic ray rejection based on LA Cosmic.
g = fitsutils.get_par(f, "GAIN")
if (fitsutils.has_par(f, "RDNOISE")):
rn = fitsutils.get_par(f, "RDNOISE")
else:
rn = 20
array, header = cosmics.fromfits(f)
try:
c = cosmics.cosmicsimage(array, gain=g, readnoise=rn, sigclip = 8.0, sigfrac = 0.3, satlevel = 64000.0)
c.run(maxiter = 3)
out = f.replace('.fits', '_clean.fits')
cosmics.tofits(out, c.cleanarray, header)
fitsutils.update_par(out, "CRREJ", 1)
#os.remove(f)
except:
pass
return out
def solve_astrometry(img, radius=6.5, with_pix=True):
'''
img: fits image where astrometry should be solved.
radius: radius of uncertainty on astrometric position in image.
'''
ra = fitsutils.get_par(img, 'RA')
dec = fitsutils.get_par(img, 'DEC')
print "Solving astrometry on field with (ra,dec)=", ra, dec
astro = "a_" + img
cmd = "solve-field --ra %s --dec %s --radius %.4f -p --new-fits %s \
-W none -B none -P none -M none -R none -S none --overwrite %s"%(ra, dec, radius, astro, img)
if (with_pix):
cmd = cmd + "--scale-units arcsecperpix --scale-low 0.375 --scale-high 0.425"
print cmd
subprocess.call(cmd, shell=True)
#Cleaning after astrometry.net
if (os.path.isfile(img.replace(".fits", ".axy"))):
os.remove(img.replace(".fits", ".axy"))
if (os.path.isfile(img.replace(".fits", "-indx.xyls"))):
os.remove(img.replace(".fits", "-indx.xyls"))
if (os.path.isfile("none")):
os.remove("none")
return astro
def update_phot_folder(reduced):
for image in glob.glob(os.path.join(reduced, "rc*fits")):
cat=fitsutils.get_par(image, "ZPCAT")
mag=fitsutils.get_par(image, "APPMAG")
ontarget = fitsutils.get_par(image, "ONTARGET")
if (ontarget ==1 and cat == 'SDSSinterpolated' and mag !=0 and fitsutils.has_par(f, "IMGTYPE") and fitsutils.get_par(f, "IMGTYPE") == "SCIENCE"):
update_phot_blagorodnova(image)
def finder(myfile,searchrad=0.2/60.):
ra, dec = coordinates_conversor.hour2deg(fitsutils.get_par(myfile, "RA"), fitsutils.get_par(myfile, "DEC"))
hdulist = pf.open(myfile)[0]
img = hdulist.data * 1.
wcs = pywcs.WCS(hdulist.header)
target_pix = wcs.wcs_sky2pix([(np.array([ra,dec], np.float_))], 1)[0]
corner_pix = wcs.wcs_sky2pix([(np.array([ra,dec+searchrad], np.float_))], 1)[0]
dx = int(np.abs(np.ceil(corner_pix[1] - target_pix[1])))
imgslice = img[int(target_pix[0])-2*dx:int(target_pix[0])+2*dx, int(target_pix[1])-2*dx:int(target_pix[1])+2*dx]
#zmin, zmax = zscale.zscale()
zmin = np.percentile(imgslice.flatten(), 5)
zmax = np.percentile(imgslice.flatten(), 99)
gc = aplpy.FITSFigure(myfile, figsize=(10,9), north=True)
gc.show_grayscale(vmin=zmin, vmax=zmax)
gc.show_scalebar(0.1/60.)
gc.scalebar.set_label('10 arcsec')
gc.scalebar.set_color('white')
gc.recenter(ra, dec, searchrad)
gc.show_markers(ra,dec+searchrad/20.,edgecolor='red',facecolor='none',marker="|",s=250, lw=10)
gc.show_markers(ra-(searchrad/20.)/np.cos(np.deg2rad(dec)),dec,edgecolor='red',facecolor='none',marker="_",s=250, lw=10)
ras = np.array([ra , ra])
decs = np.array([dec, dec])
dxs = np.array([0, searchrad/10 / np.cos(np.deg2rad(dec))])
dys = np.array([searchrad/10, 0])
gc.show_arrows(ras, decs, dxs, dys, edgecolor="red", facecolor="red", head_width=0)
ras = np.array([ra+searchrad*0.7/ np.cos(np.deg2rad(dec)), ra+searchrad*0.7/ np.cos(np.deg2rad(dec))])
decs = np.array([dec-searchrad*0.9, dec-searchrad*0.9])
dxs = np.array([0, searchrad/5 / np.cos(np.deg2rad(dec))])
dys = np.array([searchrad/5, 0])
gc.show_arrows(ras, decs, dxs, dys, edgecolor="k", facecolor="k")
gc.add_label(ras[0]+dxs[0]*1.1, decs[0]+dys[0]*1.1, 'N', relative=False, color="k", horizontalalignment="center")
gc.add_label(ras[1]+dxs[1]*1.1, decs[1]+dys[1]*1.1, 'E', relative=False, color="k", horizontalalignment="center")
name = fitsutils.get_par(myfile, "OBJECT")
gc.add_label(0.05, 0.95, 'Object: %s'%(name), relative=True, color="white", horizontalalignment="left")
gc.add_label(0.05, 0.9, 'Coordinates: RA=%s DEC=%s'%(coordinates_conversor.deg2hour(ra, dec)), relative=True, color="white", horizontalalignment="left")
gc.add_label(0.05, 0.84, 'Filter: SDSS r', relative=True, color="white", horizontalalignment="left")
findername = '%s_finder.jpg'%(name)
gc.save(findername)
return findername
def solve_astrometry(img, radius=3.0, with_pix=True, first_call=True, tweak=3):
'''
img: fits image where astrometry should be solved.
radius: radius of uncertainty on astrometric position in image.
'''
try:
ra = fitsutils.get_par(img, 'OBJRA')
dec = fitsutils.get_par(img, 'OBJDEC')
except:
ra = fitsutils.get_par(img, 'RA')
dec = fitsutils.get_par(img, 'DEC')
mydir = os.path.dirname(img)
if mydir=="":
mydir = "."
os.chdir(mydir)
astro = os.path.join( os.path.dirname(img), "a_" + os.path.basename(img))
print "Solving astrometry on field with (ra,dec)=", ra, dec, "Image",img, "New image", astro
cmd = " solve-field --ra %s --dec %s --radius %.4f -p --new-fits %s -W none -B none -P none -M none -R none -S none -t %d --overwrite %s "%(ra, dec, radius, astro, tweak, img)
if (with_pix):
cmd = cmd + " --scale-units arcsecperpix --scale-low 0.375 --scale-high 0.425"
print cmd
logger.info(cmd)
cmd = cmd + " > /tmp/astrometry_fail 2>/tmp/astrometry_fail"
subprocess.call(cmd, shell=True)
#Cleaning after astrometry.net
if (os.path.isfile(img.replace(".fits", ".axy"))):
os.remove(img.replace(".fits", ".axy"))
if (os.path.isfile(img.replace(".fits", "-indx.xyls"))):
os.remove(img.replace(".fits", "-indx.xyls"))
if (os.path.isfile("none")):
os.remove("none")
try:
os.remove("/tmp/tmp.*")
except:
pass
if(not os.path.isfile(astro) and first_call):
print "Astrometry failed on file %s! Trying with a larger radius..."%astro
solve_astrometry(img, radius=8, with_pix=True, first_call=False)
if(not os.path.isfile(astro)):
print "Astrometry FAILED!"
logger.error("Astrometry FAILED!")
return astro
def get_sequential_name(target_dir, name, i=0):
'''
Gets a sequential name if we have a file with the same object name imaged several times.
'''
newname = os.path.join(target_dir, name).replace(".fits", "_%d.fits"%i)
#If the destination file does not exist...
if (os.path.isfile(name) and not os.path.isfile(newname)):
return newname
#If it does exist, but it is another exposure
elif(os.path.isfile(name) and os.path.isfile(newname)):
#If it is not the same exposure, add with different name. Otherwise, replace.
if fitsutils.get_par(name, "JD") != fitsutils.get_par(newname, "JD"):
newname = get_sequential_name(target_dir, name, i=i+1)
return newname
def get_sextractor_stats(files):
files.sort()
sexfiles = [os.path.join(os.path.join(os.path.dirname(f), "sextractor"), os.path.basename(f).replace(".fits", ".sex")) for f in files]
sexfiles.sort()
if not os.path.isdir(os.path.join( os.path.dirname(files[0]), "stats")):
os.makedirs(os.path.join(os.path.dirname(files[0]), "stats"))
with open(os.path.join( os.path.dirname(files[0]), "stats/stats.log"), "w") as out:
for i, f in enumerate(files):
if (fitsutils.has_par(f, "IMGTYPE")):
imtype = fitsutils.get_par(f, "IMGTYPE")
else:
imtype = "NONE"
if not (imtype == "ACQUISITION" or imtype == "SCIENCE" or imtype=="FOCUS" or imtype=="GUIDER"):
continue
if not os.path.isfile(sexfiles[i]):
sf = sextractor.run_sex([f])
else:
sf = sexfiles[i]
print f
hd = pf.open(files[i])[0].header
try:
jd = hd["JD"]
obj = hd["OBJECT"]
airmass = hd["AIRMASS"]
in_temp = hd["IN_AIR"]
out_temp = hd["OUT_AIR"]
in_hum = hd["IN_HUM"]
ns, fwhm, ellipticity, bkg = sextractor.analyse_image(sf)
out.write("%s,%s,%.3f,%d,%.2f,%.3f,%.3f,%.2f,%.1f,%s,%.2f,%.2f\n"%(os.path.abspath(f),obj,jd,ns,fwhm,ellipticity,bkg,airmass,in_temp,imtype,out_temp,in_hum))
except:
pass
def compute_offsets(fitsdir):
ra_prev = "00:00:00"
dec_prev = "00:00:00"
lfiles = glob.glob(fitsdir + "/rc*fits")
lfiles.sort()
for f in lfiles:
f = os.path.basename(f)
try:
if ("[r]" in fitsutils.get_par(f, "OBJECT") or fitsutils.get_par(f, "OBJTYPE")=="ACQUISITION"):
ra = fitsutils.get_par(f, "OBJRA")
dec = fitsutils.get_par(f, "OBJDEC")
obj = fitsutils.get_par(f, "OBJECT")
isAB = False
if (fitsutils.has_par(f, "ABPAIR") and fitsutils.get_par(f, "ABPAIR")=="True"):
isAB = True
if ((ra!="" and dec !="" and (ra != ra_prev or dec != dec_prev)) or (fitsutils.get_par(f, "OBJTYPE")=="ACQUISITION")):
#print "Found image %s as first acquisition image after the slew. Computing offset for IFU..."%f
recenter_ifu.main(os.path.abspath(f), isAB, astro=True, plot=True)
ra_prev = ra
dec_prev = dec
except KeyError:
print "File %s does not have a good header."%f
def compute_offsets(fitsdir):
ra_prev = "00:00:00"
dec_prev = "00:00:00"
lfiles = glob.glob(fitsdir + "/rc*fits")
lfiles.sort()
for f in lfiles:
f = os.path.basename(f)
try:
if ("[r]" in fitsutils.get_par(f, "OBJECT")):
ra = fitsutils.get_par(f, "OBRA")
dec = fitsutils.get_par(f, "OBDEC")
if (ra!="" and dec !="" and (ra != ra_prev or dec != dec_prev)):
print "Found image %s as first acquisition image after the slew. Computing offset for IFU..."%f
recenter_ifu.main(f)
ra_prev = ra
dec_prev = dec
except KeyError:
print "File %s does not have a good header."%f
def update_phot_blagorodnova(image):
'''
Updates the DB with the aperture photometry extrated from the fits and the interpolated zeropoint.
'''
racen, deccen = cc.hour2deg(fitsutils.get_par(image, "OBJRA"), fitsutils.get_par(image, "OBJRA"))
issub='f'
refsys='SDSSinterpolated'
filt=fitsutils.get_par(image, "FILTER")
exptime=fitsutils.get_par(image, "EXPTIME")
utdate=time_utils.jd2utc(fitsutils.get_par(image, "JD"), string=True)
mag = np.round(fitsutils.get_par(image, "APPMAG"), 3)
magerr = np.round(fitsutils.get_par(image, "APPMAGER"), 3)
limmag = np.round(fitsutils.get_par(image, "ZEROPT"), 3)
name = fitsutils.get_par(image, "NAME").replace("PTF", "")
observer = 'SEDMachine'
reducedby = 'Blagorodnova Automated Pipeline SEDM'
if mag==0 or mag > limmag or mag>22:
mag = 99
magerr = 99
db = pg.DB(dbname='ptftransient',user='******',passwd='followup',host='yupana.caltech.edu')
getsrcidquery = "SELECT id from sources where name='%s'"%name
result = db.query(getsrcidquery)
for row in result.dictresult():
srcid = int(row['id'])
query3 = "SELECT id from phot WHERE sourceid=%d and instrumentid=64 and obsdate='%s' and filter='%s' and reducedby='Blagorodnova Automated Pipeline SEDM';" % (srcid,utdate,filter)
result=db.query(query3)
print query3
if len(result.dictresult()) == 0:
print srcid,racen,deccen,utdate,exptime,filt,mag,magerr,limmag,issub,refsys,observer,reducedby
query2 = "INSERT INTO phot (sourceid,programid,instrumentid,ra,dec,obsdate,exptime,filter,mag,emag,limmag,issub,refsys,observer,reducedby) VALUES "
query2 += "(%d,1,64,%f,%f,'%s',%f,'%s',%f,%f,%f,'%s','%s','%s','%s');" % (srcid,racen,deccen,utdate,exptime,filt,mag,magerr,limmag,issub,refsys,observer,reducedby)
print query2
db.query(query2)
else:
for row in result.dictresult():
photid = int(row['id'])
query2 = "DELETE from phot WHERE id=%d;" % (photid)
print query2
db.query(query2)
query2 = "INSERT INTO phot (sourceid,programid,instrumentid,ra,dec,obsdate,exptime,filter,mag,emag,limmag,issub,refsys,observer,reducedby) VALUES "
query2 += "(%d,1,64,%f,%f,'%s',%f,'%s',%f,%f,%f,'%s','%s','%s','%s');" % (srcid,racen,deccen,utdate,exptime,filt,mag,magerr,limmag,issub,refsys,observer,reducedby)
print query2
db.query(query2)
def clean_cosmic(f, name):
'''
From lacosmic.
'''
import cosmics
g = fitsutils.get_par(f, "GAIN")
rn = fitsutils.get_par(f, "RDNOISE")
array, header = cosmics.fromfits(f)
try:
c = cosmics.cosmicsimage(array, gain=g, readnoise=rn, sigclip = 8.0, sigfrac = 0.3, satlevel = 64000.0)
c.run(maxiter = 10)
out = f.replace('.fits', '_clean.fits')
cosmics.tofits(out, c.cleanarray, header)
os.remove(f)
except:
pass
def get_offset_center(f, plot=False, interactive=False):
'''
Given a fits image, returns the offset in Ra, DEC, that needs to be applied for the telescope tp go
from the current pointing position, to the coodinates of the object specified in the fits file.
'''
if(not os.path.isfile(f)):
print "File %s does not exist! Returning Zero offsets..."%f
return -1, 0,0
else:
image = pf.open(f)
wcs = pywcs.WCS(image[0].header)
rra, rdec = cc.hour2deg(image[0].header['OBJRA'],image[0].header['OBJDEC'] )
x, y = np.round(wcs.wcs_sky2pix(rra, rdec, 0), 0)
pra, pdec = wcs.wcs_pix2sky(np.array([[1293., 1280.]] , np.float_), 0)[0]
dra, ddec = cc.get_offset(pra, pdec, rra, rdec)
xl, yu = np.round(wcs.wcs_sky2pix(rra+90./3600, rdec-90./3600, 0), 0)
xu, yl = np.round(wcs.wcs_sky2pix(rra-90./3600, rdec+90./3600, 0), 0)
imageloc = image[0].data.T[xl:xu,yl:yu]
if imageloc.shape[0]==0 or imageloc.shape[1]==0:
logger.warn( "Astrometry has FAILED on this! The object is outside the frame! Resending to the numb astrometric solution")
logger.error("Astrometry has FAILED on this! The object is outside the frame! Resending to the numb astrometric solution")
print "Pixels are", xl, xu, yl, yu
try:
code, dra, ddec = get_offset_center_failed_astro(f, plot=plot, interactive=interactive)
return 2, dra, ddec
except:
return -1,0,0
if(plot):
plt.figure(figsize=(8,8))
zmin, zmax = zscale.zscale(imageloc)
#print zmin, zmax, imageloc, (xl,xu,yl,yu)
obj = fitsutils.get_par(f, "OBJECT")
plt.suptitle(obj, fontsize=20)
plt.imshow(imageloc.T, extent=(xl[0],xu[0],yl[0],yu[0]), aspect="equal", interpolation="none", origin="lower", vmin=zmin, vmax=zmax)
plt.plot(1293., 1280., "ws", ms=7, label="Current pointing")
plt.plot(x, y, "b*", ms=10, label="Target pointing")
plt.gca().invert_xaxis()
plt.legend()
if (interactive):
plt.show()
else:
plt.savefig(os.path.join(os.path.dirname(f).replace("raw", "phot"), os.path.basename(f).replace(".fits", "_a.png")))
plt.clf()
return 0, dra, ddec
def correct_files(dirname):
cmd = "gethead OBJECT *fits | grep -v -i calib | grep PTF | grep -v -E \"\[A|\[B|findi\" | grep ifu | awk '{print $1}' > update_science"
subprocess.all(cmd, shell=True)
filelist = os.path.join(dirname, "update_science")
for f in np.genfromtxt(filelist, dtype=None):
fitsutils.update_par(f, "OBJDEC", fitsutils.get_par(f, "DEC"))
for f in np.genfromtxt(filelist, dtype=None):
fitsutils.update_par(f, "OBJRA", fitsutils.get_par(f, "RA"))
for f in np.genfromtxt(filelist, dtype=None):
fitsutils.update_par(f, "FILTER", fitsutils.get_par(f, "OBJECT").split()[0].replace("]","").replace("[", ""))
for f in np.genfromtxt(filelist, dtype=None):
fitsutils.update_par(f, "NAME", fitsutils.get_par(f, "OBJECT").split()[1])
for f in np.genfromtxt(filelist, dtype=None):
fitsutils.update_par(f, "IMGTYPE", "SCIENCE")
def reset_zp(directory):
"""
Resets the zeropoint keyword to if the result is interpolated.
"""
files = glob.glob(directory + "/*fits")
for f in files:
if fitsutils.get_par(f, "ZPCAT") == "SDSSinterpolated":
fitsutils.update_par(f, "IQZEROPT", 0)
fitsutils.update_par(f, "ZEROPT", 0)
fitsutils.update_par(f, "ZEROPTU", 0)
def compile_stats_pointing():
ra = 0
dec = 0
out = open("/tmp/pointing", "w")
out.write("#f, imtype, obj, jd, filter, radeg, decdeg, dra, ddec\n")
myfiles = glob.glob("/scr2/sedm/phot/20160616/a_*[0-9].fits")
myfiles.sort()
for f in myfiles:
#try:
imtype = fitsutils.get_par(f, "IMGTYPE")
newra = fitsutils.get_par(f, "OBJRA")
newdec = fitsutils.get_par(f, "OBJDEC")
newra, newdec = cc.hour2deg(newra, newdec)
myfilter = fitsutils.get_par(f, "FILTER")
if (imtype == "ACQUISITION" or imtype == "SCIENCE"):#: and np.round(ra, 2) != np.round(newra, 2) and np.round(dec, 2) != np.round(newdec, 2):
obj = fitsutils.get_par(f, "OBJECT")
jd = fitsutils.get_par(f, "JD")
status, dra, ddec = recenter_ifu.get_offset_center(f, plot=False, interactive=False)
print f, ra, newra, imtype, jd, dra, ddec
out.write("%s,%s,%s,%.2f,%s,%.5f,%.5f,%.2f,%.2f\n"%(f, imtype, obj, jd, myfilter, newra, newdec, dra, ddec))
ra = newra
dec = newdec
#except:
# pass
out.close()
def solve_astrometry(img, radius=3, with_pix=True, overwrite=False, tweak=3):
'''
img: fits image where astrometry should be solved.
radius: radius of uncertainty on astrometric position in image.
'''
img = os.path.abspath(img)
ra = fitsutils.get_par(img, 'OBJRA')
dec = fitsutils.get_par(img, 'OBJDEC')
#logger.info( "Solving astrometry on field with (ra,dec)=%s %s"%(ra, dec))
astro = os.path.join( os.path.dirname(img), "a_" + os.path.basename(img))
#If astrometry exists, we don't run it again.
if (os.path.isfile(astro) and not overwrite):
return astro
#Store a temporary file with the multiplication of the mask
mask = "mask.fits"
cmd = "solve-field --ra %s --dec %s --radius %.4f -p --new-fits %s \
-W none -B none -P none -M none -R none -S none -t %d --overwrite %s "%(ra, dec, radius, astro, tweak, img)
if (with_pix):
cmd = cmd + " --scale-units arcsecperpix --scale-low 0.375 --scale-high 0.4"
#logger.info( cmd)
subprocess.call(cmd, shell=True)
#Cleaning after astrometry.net
if (os.path.isfile(img.replace(".fits", ".axy"))):
os.remove(img.replace(".fits", ".axy"))
if (os.path.isfile(img.replace(".fits", "-indx.xyls"))):
os.remove(img.replace(".fits", "-indx.xyls"))
if (os.path.isfile("none")):
os.remove("none")
is_on_target(img)
return astro
def add_to_zp_cal(ref_stars, image, logname):
'''
Records the parameters and instrumental and standard star magnitudes needed for zeropoint calibration.
minst = M + c0 + c1(AIRMASS) + c2(color) + c3(UT)
'''
coldic = {'u':'g', 'g':'r', 'r':'i', 'i':'z', 'z':'i'}
r = np.genfromtxt(ref_stars, delimiter=" ", dtype=None, names=True)
my = np.genfromtxt(image+".app.mag", comments="#", dtype=[("id","<f4"), ("X","<f4"), ("Y","<f4"),("Xshift","<f4"), ("Yshift","<f4"),("fwhm","<f4"), ("ph_mag","<f4"), ("stdev","<f4"), ("fit_mag","<f4"), ("fiterr","<f4")])
if (my.size < 2):
my = np.array([my])
if (r.size < 2):
r = np.array([r])
coldic = {'u':'g', 'g':'r', 'r':'i', 'i':'z', 'z':'i'}
band = fitsutils.get_par(image, 'filter')
mask_valid1 = np.array(my["fwhm"]<9000) * np.array(my["ph_mag"]<9000) * np.array(~ np.isnan(r[band])) * np.array(~ np.isnan(my["fit_mag"]))
r = r[mask_valid1]
my = my[mask_valid1]
N = len(r)
my["fiterr"][np.isnan( my["fiterr"])] = 100
col_band = coldic[band]
exptime = fitsutils.get_par(image, "exptime")
date = fitsutils.get_par(image, "JD")
airmass = fitsutils.get_par(image, "AIRMASS")
if (not os.path.isfile(logname)):
with open( logname, "a") as f:
f.write("#filename,filter,std,stderr,inst,insterr,jd,airmass,color,exptime\n")
with open( logname, "a") as f:
for i in range(N):
f.write("%s,%s,%.3f,%.3f,%3f,%.3f,%.3f,%.3f,%.3f,%.3f\n"%
(image, band, r[i][band], r[i]["d"+band], my[i]['fit_mag'], my[i]['fiterr'], date, airmass, r[i][band]-r[i][col_band],exptime))
def calibrate_zeropoint(image, plot=True, debug=False, refstars=None):
filt = fitsutils.get_par(image, 'filter')
exptime = fitsutils.get_par(image, "exptime")
if fitsutils.has_par(image, "JD"):
date = fitsutils.get_par(image, "JD")
elif fitsutils.has_par(image, "MJD"):
date = fitsutils.get_par(image, "MJD")
objname = fitsutils.get_par(image, "OBJECT")
airmass = fitsutils.get_par(image, "AIRMASS")
print "Starting calibration of ZP for image", image,"for object", objname
if (exptime < 10):
print "ERROR. Exposure time too short for this image to see anything..."
return
extracted = extract_star_sequence(image, filt, plot=plot, survey='sdss', debug=debug, refstars=refstars)
if (not extracted):
print "Field not in SDSS or error when retrieving the catalogue... Skipping."
return
#If extraction worked, we can get the FWHM
fwhm = fitsutils.get_par(image, "fwhm")
fwhm_as = fwhm * 0.394
app_phot.get_app_phot("/tmp/sdss_cat_det.txt", image, wcsin='logic')
z, c, err = find_zeropoint_noid("/tmp/sdss_cat_det.txt", image, plot=plot)
#Log the current zeropoint for this image
logname = os.path.join(os.path.dirname(image), "zeropoint.log")
#Add the data to a later stage zeropoint calibrtion with all-sky data.
zplogname = os.path.join(os.path.dirname(image), "allstars_zp.log")
add_to_zp_cal("/tmp/sdss_cat_det.txt", image, zplogname)
if (not os.path.isfile(logname)):
with open( logname, "a") as f:
f.write("#filename,exptime,filter,date,airmass,fwhm_pix,fwhm_as,zeropoint,color,err\n")
with open( logname, "a") as f:
f.write("%s,%.1f,%s,%3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f\n"%(image,exptime,filt,date,airmass,fwhm,fwhm_as,z,c,err))
def reduce_on_the_fly(photdir):
'''
Waits for new images to appear in the directory to trigger their incremental reduction as well.
'''
#Get the current the number of files
nfiles = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
dayname = os.path.basename(photdir)
time_ini = datetime.datetime.now()
time_curr = datetime.datetime.now()
#Run this loop for 12h since the start.
while (time_curr-time_ini).total_seconds() < 12.*3600:
nfilesnew = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
if len(nfilesnew) == len(nfiles):
time.sleep(10)
else:
new = [f for f in nfilesnew if f not in nfiles]
for n in new:
if (not fitsutils.has_par(n, "IMGTYPE")):
print "Image",n,"Does not have an IMGTYPE"
time.sleep(0.5)
if (not fitsutils.has_par(n, "IMGTYPE")):
print "Image",n,"STILL Does not have an IMGTYPE"
continue
if (fitsutils.get_par(n, "IMGTYPE")=="SCIENCE"):
reduced = rcred.reduce_image(n)
try:
zeropoint.calibrate_zeropoint(reduced)
except:
logger.error("Could not calibrate zeropoint for image %s"%reduced)
#Copy them to transient
for r in reduced:
cmd = "rcp %s [email protected]:/scr3/mansi/ptf/p60phot/fremling_pipeline/sedm/reduced/%s/."%(r, dayname)
subprocess.call(cmd, shell=True)
logger.info(cmd)
logger.info( "Successfully copied the image: %s"% cmd)
time_curr = datetime.datetime.now()
nfiles = nfilesnew
def run_sex(flist, mask=False, cosmics=False, overwrite=True):
d = os.path.dirname(flist[0])
if d == "":
d = "."
os.chdir(d)
#Create the directory where the sextracted images are going to go.
sexdir = os.path.join(d, "sextractor")
if (not os.path.isdir(sexdir)):
os.makedirs(sexdir)
newlist = []
for f in flist:
newimage = os.path.join(sexdir, os.path.basename(f).replace(".fits", ".sex"))
if (os.path.isfile(newimage) and not overwrite):
newlist.append(newimage)
print "Sextracted image %s already exists."%newimage
else:
try:
f = os.path.abspath(f)
if (mask):
out = rcred.get_masked_image(f)
else:
out = f
if (cosmics and (not fitsutils.has_par(out, "CRREJ") or fitsutils.get_par(out, "CRREJ") ==0)):
out = rcred.clean_cosmic(out)
cmd="sex -c %s/config/daofind.sex %s"%(os.environ["SEDMPH"], out)
subprocess.call(cmd, shell=True)
print cmd
shutil.move("image.sex", newimage)
newlist.append(newimage)
except IOError:
print "IOError detected reading file",f
pass
return newlist
def main(reduced):
"""
Performs the main zeropoint calculations for the folder and plots the results.
"""
os.chdir(reduced)
plotdir = "zeropoint"
if not os.path.isdir(plotdir):
os.makedirs(plotdir)
for f in glob.glob("*.fits|*.new"):
logger.info("Starting calibration of zeropoint for %s" % f)
if not fitsutils.has_par(f, "IMGTYPE") or fitsutils.get_par(f, "IMGTYPE") == "SCIENCE":
calibrate_zeropoint(f, plotdir=os.path.abspath(plotdir))
if os.path.isfile("zeropoint.log"):
plot_zp("zeropoint.log", plotdir)
if os.path.isfile("allstars_zp.log"):
lsq_zeropoint("allstars_zp.log", plotdir)
interpolate_zp(reduced, "allstars_zp.log")
clean_tmp_files()
def create_masterflat(flatdir=None, biasdir=None, channel='rc', plot=True):
'''
Creates a masterflat from both dome flats and sky flats if the number of counts in the given filter
is not saturated and not too low (between 3000 and 40000).
'''
if (flatdir == None or flatdir == ""): flatdir = "."
if (biasdir == None or biasdir == ""): biasdir = flatdir
os.chdir(flatdir)
if (plot and not os.path.isdir("reduced/flats")):
os.makedirs("reduced/flats")
if (len(glob.glob("Flat_%s*norm.fits" % channel)) == 4):
logger.info("Master Flat exists!")
return
if (len(glob.glob("Flat_%s*norm.fits" % channel)) > 0):
logger.info("Some Master Flat exist!")
else:
logger.info("Starting the Master Flat creation!")
bias_slow = "Bias_%s_slow.fits" % channel
bias_fast = "Bias_%s_fast.fits" % channel
if (not os.path.isfile(bias_slow) and not os.path.isfile(bias_fast)):
create_masterbias(biasdir)
lsflat = []
lfflat = []
obj = ""
imtype = ""
#Select all filts that are Flats with same instrument
for f in glob.glob(channel + "*fits"):
try:
if fitsutils.has_par(f, "OBJECT"):
obj = str.upper(fitsutils.get_par(f, "OBJECT"))
else:
continue
if fitsutils.has_par(f, "IMGTYPE"):
imtype = str.upper(fitsutils.get_par(f, "IMGTYPE"))
else:
continue
#if ("RAINBOW CAM" in str.upper(fitsutils.get_par(f, "CAM_NAME")) and ("DOME" in obj or "FLAT" in obj or "Twilight" in obj or "TWILIGHT" in imtype or "DOME" in imtype)):
if ("twilight" in imtype.lower()):
if (fitsutils.get_par(f, "ADCSPEED") == 2):
lfflat.append(f)
else:
lsflat.append(f)
except:
logger.error("Error with retrieving parameters for file %s" % f)
pass
logger.info("Files for slow flat %s" % lsflat)
logger.info("Files for fast flat %s" % lfflat)
fsfile = "lflat_slow_" + channel
np.savetxt(fsfile, np.array(lsflat), fmt="%s")
fffile = "lflat_fast_" + channel
np.savetxt(fffile, np.array(lfflat), fmt="%s")
# Running IRAF
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Remove bias from the flat
if len(lsflat) > 0:
iraf.imarith("@" + fsfile, "-", bias_slow, "b_@" + fsfile)
if len(lfflat) > 0:
iraf.imarith("@" + fffile, "-", bias_fast, "b_@" + fffile)
#Remove the list files
os.remove(fsfile)
os.remove(fffile)
#Slices the flats.
debiased_flats = glob.glob("b_*.fits")
for f in debiased_flats:
logger.info("Slicing file %s" % f)
try:
slice_rc(f)
except:
logger.error(
"Error when slicing file... deleting the unsliced one...")
#Remove the un-sliced file
os.remove(f)
#Selects the ones that are suitable given the number of counts and combines them.
bands = ['u', 'g', 'r', 'i']
for b in bands:
out = "Flat_%s_%s.fits" % (channel, b)
out_norm = out.replace(".fits", "_norm.fits")
if (os.path.isfile(out_norm)):
logger.error("Master Flat for filter %s exists. Skipping..." % b)
continue
lfiles = []
for f in glob.glob('b_*_%s.fits' % b):
fi = fits.open(f)
d = fi[0].data
status = "rejected"
if np.percentile(d, 90) > 4000 and np.percentile(d, 90) < 45000:
lfiles.append(f)
mymode = 1. * np.median(d.flatten())
d[d > 45000] = mymode
fi[0].data = d
fi.writeto(f, clobber=True)
status = "accepted"
if (plot):
plt.title("Flat filter %s. %s" % (b, status))
plt.imshow(d.T, cmap=plt.get_cmap("nipy_spectral"))
plt.colorbar()
plt.savefig("reduced/flats/%s" % (f.replace(".fits", ".png")))
plt.close()
#Make sure that the optimum number of counts is not too low and not saturated.
if len(lfiles) == 0:
logger.error(
"WARNING!!! Could not find suitable flats for band %s" % b)
continue
if len(lfiles) < 3:
logger.error(
"WARNING!!! Could find less than 3 flats for band %s. Skipping, as it is not reliable..."
% b)
continue
ffile = "lflat_" + b
np.savetxt(ffile, np.array(lfiles), fmt="%s")
#Cleaning of old files
if (os.path.isfile(out)): os.remove(out)
if (os.path.isfile(out_norm)): os.remove(out_norm)
if (os.path.isfile("Flat_stats")): os.remove("Flat_stats")
#Combine flats
iraf.imcombine(input = "@"+ffile, \
output = out, \
combine = "median",\
scale = "mode",
reject = "sigclip", lsigma = 2., hsigma = 2, gain=1.7, rdnoise=4.)
iraf.imstat(out,
fields="image,npix,mean,stddev,min,max,mode",
Stdout="Flat_stats")
st = np.genfromtxt("Flat_stats", names=True, dtype=None)
#Normalize flats
iraf.imarith(out, "/", st["MODE"], out_norm)
#Do some cleaning
logger.info('Removing from lfiles')
for f in glob.glob('b_*_%s.fits' % b):
os.remove(f)
os.remove(ffile)
if os.path.isfile(fsfile):
os.remove(fsfile)
if os.path.isfile(fffile):
os.remove(fffile)
#copy into the reference folder with current date
newdir = os.path.join("../../refphot/",
os.path.basename(os.path.abspath(flatdir)))
if (not os.path.isdir(newdir)):
os.makedirs(newdir)
shutil.copy(out_norm, os.path.join(newdir, os.path.basename(out_norm)))
copy_ref_calib(flatdir, "Flat")
def create_masterbias(biasdir=None, channel='rc'):
'''
Combines slow and fast readout mode biases for the specified channel.
'''
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
if (biasdir == None) or biasdir=="": biasdir = "."
outs = "Bias_%s_slow.fits"%channel
outf = "Bias_%s_fast.fits"%channel
if (os.path.isfile(os.path.join(biasdir,outs)) and os.path.isfile(os.path.join(biasdir,outf))):
print "Master Bias exists!"
return
else:
print "Starting the Master Bias creation!"
os.chdir(biasdir)
lfastbias = []
lslowbias = []
#Select all filts that are Bias with same instrument
for f in glob.glob("*fits"):
try:
if (channel == fitsutils.get_par(f, "CHANNEL") and "BIAS" in str.upper(fitsutils.get_par(f, "OBJECT")) ):
if (fitsutils.get_par(f, "ADCSPEED")==2):
lfastbias.append(f)
else:
lslowbias.append(f)
except:
pass
print "Files for bias SLOW mode: ", lslowbias
print "Files for bias FAST mode: ", lfastbias
if len(lfastbias) > 0:
bfile_fast ="lbias_fast_"+channel
np.savetxt(bfile_fast, np.array(lfastbias), fmt="%s")
if (os.path.isfile("Bias_stats_fast")): os.remove("Bias_stats_fast")
iraf.imstat("@"+bfile_fast, Stdout="Bias_stats_fast")
st = np.genfromtxt("Bias_stats_fast", names=True, dtype=None)
print st
iraf.imcombine(input = "@"+bfile_fast, \
output = outf, \
combine = "median",\
scale = "mode")
os.remove(bfile_fast)
if len(lslowbias) > 0:
bfile_slow ="lbias_slow_"+channel
np.savetxt(bfile_slow, np.array(lslowbias), fmt="%s")
if (os.path.isfile("Bias_stats_slow")): os.remove("Bias_stats_slow")
iraf.imstat("@"+bfile_slow, Stdout="Bias_stats_slow")
st = np.genfromtxt("Bias_stats_slow", names=True, dtype=None)
print st
iraf.imcombine(input = "@"+bfile_slow, \
output = outs, \
combine = "median",\
scale = "mode")
os.remove(bfile_slow)
def get_app_phot(coords, image, plot_only=False, store=True, wcsin="world", fwhm=2, plotdir=".", box=15):
'''
coords: files:
wcsin: can be "world", "logic"
'''
# Load packages; splot is in the onedspec package, which is in noao.
# The special keyword _doprint=0 turns off displaying the tasks
# when loading a package.
if (not plot_only):
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.apphot(_doprint=0)
iraf.unlearn("apphot")
imdir = os.path.dirname(image)
imname = os.path.basename(image)
plotdir = os.path.join(imdir, "photometry")
if not os.path.isdir(plotdir):
os.makedirs(plotdir)
out_name = os.path.join(plotdir, imname + ".seq.mag")
clean_name = os.path.join(plotdir, imname + ".app.mag")
# Read values from .ec file
ecfile= image+".ec"
filter_value=''.join(ecfile).split('.',1)[0]
fwhm_value = fwhm
if (fitsutils.has_par(image, 'FWHM')):
fwhm_value = fitsutils.get_par(image, 'FWHM')
if (fitsutils.has_par(image, 'AIRMASS')):
airmass_value = fitsutils.get_par(image, 'AIRMASS')
else:
airmass_value = 1.3
exptime = fitsutils.get_par(image, 'EXPTIME')
gain = fitsutils.get_par(image, 'GAIN')
try:
with open(''.join(ecfile),'r') as f:
for line in f:
if "airmass" in line:
airmass_value = line.split('=',1)[1]
else:
airmass_value = 1
if "FWHM" in line:
print line
fwhm_value = line.split('FWHM=',1)[1]
fwhm_value = fwhm_value.rsplit("aperture")[0]
except:
pass
print "FWHM", fwhm_value
aperture_rad = math.ceil(float(fwhm_value)*2) # Set aperture radius to three times the PSF radius
sky_rad= math.ceil(aperture_rad)*5
print aperture_rad, sky_rad
if (not plot_only):
if os.path.isfile(out_name): os.remove(out_name)
if os.path.isfile(clean_name): os.remove(clean_name)
# Check if files in list, otherwise exit
if not ecfile:
print "No .ec files in directory, exiting"
sys.exit()
iraf.noao.digiphot.apphot.qphot(image = image,\
cbox = box ,\
annulus = sky_rad ,\
dannulus = 15. ,\
aperture = str(aperture_rad),\
coords = coords ,\
output = out_name ,\
plotfile = "" ,\
zmag = 0. ,\
exposure = "exptime" ,\
airmass = "airmass" ,\
filter = "filters" ,\
obstime = "DATE" ,\
epadu = gain ,\
interactive = "no" ,\
radplots = "yes" ,\
verbose = "no" ,\
graphics = "stdgraph" ,\
display = "stdimage" ,\
icommands = "" ,\
wcsin = wcsin,
wcsout = "logical",
gcommands = "")
#iraf.noao.digiphot.apphot.phot(image=image, cbox=5., annulus=12.4, dannulus=10., salgori = "centroid", aperture=9.3,wcsin="world",wcsout="tv", interac = "no", coords=coords, output=out_name)
iraf.txdump(out_name, "id,image,xcenter,ycenter,xshift,yshift,fwhm,msky,stdev,mag,merr", "yes", Stdout=clean_name)
ma = np.genfromtxt(clean_name, comments="#", dtype=[("id","<f4"), ("image","|S20"), ("X","<f4"), ("Y","<f4"), ("Xshift","<f4"), ("Yshift","<f4"),("fwhm","<f4"), ("ph_mag","<f4"), ("stdev","<f4"), ("fit_mag","<f4"), ("fiterr","<f4")])
if (ma.size > 0):
m = ma[~np.isnan(ma["fit_mag"])]
else:
print "Only one object found!"
m = np.array([ma])
hdulist = pf.open(image)
prihdr = hdulist[0].header
img = hdulist[0].data * 1.
nx, ny = img.shape
dimX = int(4)
dimY = int(np.ceil(len(m)*1./4))
outerrad = sky_rad+10
cutrad = outerrad + 15
plt.suptitle("FWHM="+str(fwhm_value))
k = 0
for i in np.arange(dimX):
for j in np.arange(dimY):
if ( k < len(m)):
ax = plt.subplot2grid((dimX,dimY),(i, j))
y1, y2, x1, x2 = m[k]["X"]-cutrad, m[k]["X"]+cutrad, m[k]["Y"]-cutrad, m[k]["Y"]+cutrad
y1, y2, x1, x2 = int(y1), int(y2), int(x1), int(x2)
try:
zmin, zmax = zscale.zscale(img[x1:x2,y1:y2], nsamples=1000, contrast=0.25)
except:
sh= img[x1:x2,y1:y2].shape
if sh[0]>0 and sh[1]>0:
zmin = np.nanmin(img[x1:x2,y1:y2])
zmax = np.nanmax(img[x1:x2,y1:y2])
continue
else:
continue
ax.imshow(img[x1:x2,y1:y2], aspect="equal", extent=(-cutrad, cutrad, -cutrad, cutrad), origin="lower", cmap=matplotlib.cm.gray_r, interpolation="none", vmin=zmin, vmax=zmax)
c1 = plt.Circle( (0, 0), edgecolor="r", facecolor="none", radius=5.)
c2 = plt.Circle( (0, 0), edgecolor="orange", facecolor="none", radius=sky_rad)
c3 = plt.Circle( (0, 0), edgecolor="yellow", facecolor="none", radius=sky_rad+10)
plt.gca().add_artist(c1)
plt.gca().add_artist(c2)
plt.gca().add_artist(c3)
ax.set_xticks([])
ax.set_yticks([])
plt.text(+5, +5, "%d"%m[k]["id"])
plt.text(-cutrad, -cutrad, "%.2f$\pm$%.2f"%(m[k]["fit_mag"], m[k]["fiterr"]), color="b")
k = k+1
plt.savefig(os.path.join(plotdir, imname + "plot.png"))
plt.clf()
def get_app_phot_target(image, plot=False, store=True, wcsin="logical", fwhm=2, box=4, ra=None, dec=None):
'''
coords: files:
wcsin: can be "world", "logic"
'''
# Load packages; splot is in the onedspec package, which is in noao.
# The special keyword _doprint=0 turns off displaying the tasks
# when loading a package.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
fxn()
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.apphot(_doprint=0)
iraf.unlearn("apphot")
impf = pf.open(image)
wcs = pywcs.WCS(impf[0].header)
#Check that actually the object is within this frame.
if (ra is None or dec is None):
if (fitsutils.has_par(image, "OBJRA") and fitsutils.has_par(image, "OBJRA")):
ra, dec = cc.hour2deg(fitsutils.get_par(image, 'OBJRA'), fitsutils.get_par(image, 'OBJDEC'))
else:
ra, dec = cc.hour2deg(fitsutils.get_par(image, 'RA'), fitsutils.get_par(image, 'DEC'))
pra, pdec = get_xy_coords(image, ra, dec)
else:
if(wcsin == "logical"):
pra, pdec = ra, dec
else:
#Using new method to derive the X, Y pixel coordinates, as pywcs does not seem to be working well.
pra, pdec = get_xy_coords(image, ra, dec)
#pra, pdec = wcs.wcs_sky2pix(ra, dec, 1)
#pra, pdec = wcs.wcs_sky2pix(np.array([ra, dec], ndmin=2), 1)[0]
shape = impf[0].data.shape
if (pra > 0) and (pra < shape[0]) and (pdec > 0) and (pdec < shape[1]):
pass
else:
print image, "ERROR! Object coordinates are outside this frame. Skipping any aperture photometry!!"
print pra, pdec, shape
return
imdir = os.path.dirname(image)
imname = os.path.basename(image)
plotdir = os.path.join(imdir, "photometry")
if not os.path.isdir(plotdir):
os.makedirs(plotdir)
out_name = os.path.join(plotdir, imname + ".seq.mag")
clean_name = os.path.join(plotdir, imname + ".objapp.mag")
fwhm_value = fwhm
nsrc, fwhm_value, ellip = sextractor.get_image_pars(image)
if np.isnan(fwhm_value):
fwhm_value=99
fitsutils.update_par(image, 'FWHM', fwhm_value)
if (fitsutils.has_par(image, 'AIRMASS')):
airmass_value = fitsutils.get_par(image, 'AIRMASS')
else:
airmass_value = 1.3
exptime = fitsutils.get_par(image, 'EXPTIME')
gain = fitsutils.get_par(image, 'GAIN')
#print "FWHM", fwhm_value
aperture_rad = math.ceil(float(fwhm_value)*3) # Set aperture radius to three times the PSF radius
sky_rad= math.ceil(aperture_rad*4)
#print aperture_rad, sky_rad
print "Saving coodinates for the object in pixels",pra,pdec
coords = "/tmp/coords.dat"
np.savetxt("/tmp/coords.dat", np.array([[pra, pdec]]), fmt="%.4f %.4f")
if (plot):
zmin, zmax = zscale.zscale(impf[0].data)
im = plt.imshow(impf[0].data, vmin=zmin, vmax=zmax, origin="bottom")
plt.scatter(pra, pdec, marker="o", s=100, facecolor="none")
plt.savefig(os.path.join(plotdir, imname+".png"))
plt.clf()
if os.path.isfile(out_name): os.remove(out_name)
if os.path.isfile(clean_name): os.remove(clean_name)
iraf.noao.digiphot.apphot.qphot(image = image,\
cbox = box ,\
annulus = sky_rad ,\
dannulus = 15. ,\
aperture = str(aperture_rad),\
coords = coords ,\
output = out_name ,\
plotfile = "" ,\
zmag = 0. ,\
exposure = "exptime" ,\
airmass = "airmass" ,\
filter = "filter" ,\
obstime = "DATE" ,\
epadu = gain ,\
interactive = "no" ,\
radplots = "yes" ,\
verbose = "no" ,\
graphics = "stdgraph" ,\
display = "stdimage" ,\
icommands = "" ,\
wcsin = "logical",
wcsout = "logical",
gcommands = "")
#iraf.noao.digiphot.apphot.phot(image=image, cbox=5., annulus=12.4, dannulus=10., salgori = "centroid", aperture=9.3,wcsin="world",wcsout="tv", interac = "no", coords=coords, output=out_name)
iraf.txdump(out_name, "id,image,xcenter,ycenter,xshift,yshift,fwhm,msky,stdev,mag,merr", "yes", Stdout=clean_name)
ma = np.genfromtxt(clean_name, comments="#", dtype=[("id","<f4"), ("image","|S20"), ("X","<f4"), ("Y","<f4"), ("Xshift","<f4"), ("Yshift","<f4"),("fwhm","<f4"), ("ph_mag","<f4"), ("stdev","<f4"), ("fit_mag","<f4"), ("fiterr","<f4")])
if (ma.size > 0):
if (ma.size==1):
ma = np.array([ma])
m = ma[~np.isnan(ma["fit_mag"])]
else:
print "Only one object found!"
m = np.array([ma])
insmag = np.round(ma['fit_mag'][0] , 3)
insmagerr = np.round(ma['fiterr'][0], 3)
if (fitsutils.has_par(image, "ZEROPT")):
mag = insmag + float(fitsutils.get_par(image, "ZEROPT"))
magerr = np.sqrt(insmagerr**2+ float(fitsutils.get_par(image, "ZEROPTU"))**2)
if np.isnan(mag):
mag, magerr = 0, 0
insmag, insmagerr = 0,0
fitsutils.update_par(image, "INSMAG", "%.3f"%insmag )
fitsutils.update_par(image, "INSMAGER", "%.3f"%insmagerr)
fitsutils.update_par(image, "APPMAG", np.round(mag, 3) )
fitsutils.update_par(image, "APPMAGER", np.round(magerr, 3))
if (plot):
X = int(ma["X"][0])
Y = int(ma["Y"][0])
pra = int(pra)
pdec = int(pdec)
plt.scatter(X, Y, marker="o", s=100, facecolor="none", edgecolor="red")
plt.colorbar(im)
plt.savefig(os.path.join(plotdir, imname+".png"))
plt.clf()
zmin, zmax = zscale.zscale(impf[0].data.T[X-50:X+50,Y-50:Y+50].T)
im = plt.imshow(impf[0].data.T[pra-50:pra+50,pdec-50:pdec+50].T, vmin=zmin, vmax=zmax, interpolation="none", origin="bottom", extent=(-50,50,-50,50))
c1 = plt.Circle( (pra-X, pdec-Y), edgecolor="k", facecolor="none", radius=aperture_rad, label="Initial position")
c11 = plt.Circle( (pra-X, pdec-Y), edgecolor="k", facecolor="none", radius=sky_rad)
c2 = plt.Circle( (0, 0), edgecolor="orange", facecolor="none", radius=aperture_rad, label="Adjusted centroid")
c22 = plt.Circle( (0, 0), edgecolor="orange", facecolor="none", radius=sky_rad)
plt.gca().add_artist(c1)
plt.gca().add_artist(c11)
plt.gca().add_artist(c2)
plt.gca().add_artist(c22)
plt.colorbar(im)
myhandles = []
markers = ["o", "o"]
labels = ["Initial position", "Adjusted centroid"]
cols = ["k", "orange"]
for i in np.arange(len(markers)):
myhandles.append(mlines.Line2D([], [], mec=cols[i], mfc="none", marker=markers[i], ls="None", markersize=10, label=labels[i]))
plt.legend(handles=myhandles, loc="lower left", labelspacing=0.3, fontsize=11, numpoints=1, frameon=False, ncol=5, bbox_to_anchor=(0.0, 0.00), fancybox=False, shadow=True)
plt.title("MIN: %.0f MAX: %.0f"%(np.nanmin(impf[0].data.T[X-50:X+50,Y-50:Y+50]), np.nanmax(impf[0].data.T[X-50:X+50,Y-50:Y+50])))
plt.savefig(os.path.join(plotdir, imname+"_zoom.png"))
plt.clf()
Runs astrometry.net on the image specified as a parameter and returns
the offset needed to be applied in order to center the object coordinates
in the reference pixel.
''', formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-d', '--reddir', type=str, dest="reduced", help='Fits directory file with reduced images.', default=None)
args = parser.parse_args()
reduced = args.reduced
if (reduced is None):
timestamp=datetime.datetime.isoformat(datetime.datetime.utcnow())
timestamp = timestamp.split("T")[0].replace("-","")
reduced = os.path.join("/scr2/sedm/phot/", timestamp, "reduced")
os.chdir(reduced)
for f in glob.glob("*.fits"):
if(fitsutils.has_par(f, "IMGTYPE") and fitsutils.get_par(f, "IMGTYPE") == "SCIENCE" or fitsutils.get_par(f, "IMGTYPE") == "ACQUISITION"):
#print f
get_app_phot_target(f, box=5)
cmd = 'echo "FILE ONTARGET NAME FILTER JD LST APPMAG APPMAGER INSMAG INSMAGER ZEROPT ZEROPTU" > photometry/magnitudes.dat; gethead ONTARGET NAME FILTER JD LST APPMAG APPMAGER INSMAG INSMAGER ZEROPT ZEROPTU *fits | grep -E "fits[\ ]+1" >> photometry/magnitudes.dat'
subprocess.call(cmd, shell=True)
def create_masterbias(biasdir=None, channel='rc'):
'''
Combines slow and fast readout mode biases for the specified channel.
'''
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
if (biasdir is None) or biasdir=="": biasdir = "."
outs = "Bias_%s_slow.fits"%channel
outf = "Bias_%s_fast.fits"%channel
doslow = True
dofast = True
if (os.path.isfile(os.path.join(biasdir,outs))):
logger.warn( "%s master Bias exists!"%outs)
doslow = False
if ( os.path.isfile(os.path.join(biasdir,outf))):
logger.warn("%s master Bias exists!"%outs)
dofast = False
if(doslow or dofast):
logger.info("Starting the Master Bias creation!")
else:
return
os.chdir(biasdir)
lfastbias = []
lslowbias = []
#Select all filts that are Bias with same instrument
for f in glob.glob("rc*fits"):
try:
if ( "BIAS" in str.upper(fitsutils.get_par(f, "IMGTYPE")) ):
if (fitsutils.get_par(f, "ADCSPEED")==2):
lfastbias.append(f)
else:
lslowbias.append(f)
except:
pass
logger.info("Files for bias SLOW mode: %s"% lslowbias)
logger.info( "Files for bias FAST mode: %s"% lfastbias)
if len(lfastbias) > 0 and dofast:
bfile_fast ="lbias_fast_"+channel
np.savetxt(bfile_fast, np.array(lfastbias), fmt="%s")
if (os.path.isfile("Bias_stats_fast")): os.remove("Bias_stats_fast")
iraf.imstat("@"+bfile_fast, Stdout="Bias_stats_fast")
st = np.genfromtxt("Bias_stats_fast", names=True, dtype=None)
logger.info("%s"%st)
iraf.imcombine(input = "@"+bfile_fast, \
output = outf, \
combine = "median",\
scale = "mode")
os.remove(bfile_fast)
#copy into the reference folder with current date
newdir = os.path.join("../../refphot/", os.path.basename(os.path.abspath(biasdir)))
if (not os.path.isdir(newdir)):
os.makedirs(newdir)
shutil.copy(outf, os.path.join(newdir, os.path.basename(outf)) )
else:
copy_ref_calib(biasdir, outf)
if len(lslowbias) > 0 and doslow:
bfile_slow ="lbias_slow_"+channel
np.savetxt(bfile_slow, np.array(lslowbias), fmt="%s")
if (os.path.isfile("Bias_stats_slow")): os.remove("Bias_stats_slow")
iraf.imstat("@"+bfile_slow, Stdout="Bias_stats_slow")
st = np.genfromtxt("Bias_stats_slow", names=True, dtype=None)
logger.info("%s"%st)
iraf.imcombine(input = "@"+bfile_slow, \
output = outs, \
combine = "median",\
scale = "mode")
os.remove(bfile_slow)
#copy into the reference folder with current date
newdir = os.path.join("../../refphot/", os.path.basename(os.path.abspath(biasdir)))
if (not os.path.isdir(newdir)):
os.makedirs(newdir)
shutil.copy(outs, os.path.join(newdir, os.path.basename(outs)) )
else:
copy_ref_calib(biasdir, outs)
def create_masterbias(biasdir=None, channel='rc'):
'''
Combines slow and fast readout mode biases for the specified channel.
'''
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
if (biasdir is None) or biasdir == "": biasdir = "."
outs = "Bias_%s_slow.fits" % channel
outf = "Bias_%s_fast.fits" % channel
doslow = True
dofast = True
if (os.path.isfile(os.path.join(biasdir, outs))):
logger.warn("%s master Bias exists!" % outs)
doslow = False
if (os.path.isfile(os.path.join(biasdir, outf))):
logger.warn("%s master Bias exists!" % outs)
dofast = False
if (doslow or dofast):
logger.info("Starting the Master Bias creation!")
else:
return
os.chdir(biasdir)
lfastbias = []
lslowbias = []
#Select all filts that are Bias with same instrument
for f in glob.glob("rc*fits"):
try:
if ("BIAS" in str.upper(fitsutils.get_par(f, "IMGTYPE").upper())):
if (fitsutils.get_par(f, "ADCSPEED") == 2):
lfastbias.append(f)
else:
lslowbias.append(f)
except:
pass
logger.info("Files for bias SLOW mode: %s" % lslowbias)
logger.info("Files for bias FAST mode: %s" % lfastbias)
if len(lfastbias) > 0 and dofast:
bfile_fast = "lbias_fast_" + channel
np.savetxt(bfile_fast, np.array(lfastbias), fmt="%s")
if (os.path.isfile("Bias_stats_fast")): os.remove("Bias_stats_fast")
iraf.imstat("@" + bfile_fast, Stdout="Bias_stats_fast")
st = np.genfromtxt("Bias_stats_fast", names=True, dtype=None)
logger.info("%s" % st)
iraf.imcombine(input = "@"+bfile_fast, \
output = outf, \
combine = "median",\
scale = "mode")
os.remove(bfile_fast)
#copy into the reference folder with current date
newdir = os.path.join("../../refphot/",
os.path.basename(os.path.abspath(biasdir)))
if (not os.path.isdir(newdir)):
os.makedirs(newdir)
shutil.copy(outf, os.path.join(newdir, os.path.basename(outf)))
else:
copy_ref_calib(biasdir, "Bias")
if len(lslowbias) > 0 and doslow:
bfile_slow = "lbias_slow_" + channel
np.savetxt(bfile_slow, np.array(lslowbias), fmt="%s")
if (os.path.isfile("Bias_stats_slow")): os.remove("Bias_stats_slow")
iraf.imstat("@" + bfile_slow, Stdout="Bias_stats_slow")
st = np.genfromtxt("Bias_stats_slow", names=True, dtype=None)
logger.info("%s" % st)
iraf.imcombine(input = "@"+bfile_slow, \
output = outs, \
combine = "median",\
scale = "mode")
os.remove(bfile_slow)
#copy into the reference folder with current date
newdir = os.path.join("../../refphot/",
os.path.basename(os.path.abspath(biasdir)))
if (not os.path.isdir(newdir)):
os.makedirs(newdir)
shutil.copy(outs, os.path.join(newdir, os.path.basename(outs)))
else:
copy_ref_calib(biasdir, outs)
def create_superflat(imdir, filters=["u", "g", "r", "i"]):
#Locate images for each filter
imlist = glob.glob("rc*fits")
#Run sextractor to locate bright sources
sexfiles = sextractor.run_sex(imlist, overwrite=False)
maskfiles = []
for i, im in enumerate(imlist):
#Create a mask and store it int he mask directory
maskfile = mask_stars(im, sexfiles[i])
maskfiles.append(maskfile)
fitsutils.update_par(im, "BPM", os.path.relpath(maskfile))
for filt in filters:
fimlist = [
im for im in imlist if fitsutils.get_par(im, "FILTER") == filt
]
fmasklist = [
im for im in maskfiles if fitsutils.get_par(im, "FILTER") == filt
]
if len(fimlist) == 0:
continue
fsfile = "lflat_%s" % filt
msfile = "lmask_%s" % filt
np.savetxt(fsfile, np.array(fimlist), fmt="%s")
np.savetxt(msfile, np.array(fmasklist), fmt="%s")
'''masklist = []
for m in fmasklist:
hdulist = fits.open(m)
data = hdulist[0].data
masklist.append(data)
masklist = np.array(masklist)
hdu = fits.PrimaryHDU(masklist)
hdulist = fits.HDUList([hdu])
hdulist.writeto("mastermask_%s.fits"%filt)'''
# Running IRAF
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.imarith("@" + fsfile, "*", "@" + msfile, "m_@" + fsfile)
#Combine flats
iraf.imcombine(input = "m_@"+fsfile, \
output = "superflat_%s.fits"%filt, \
combine = "median",\
scale = "mode", \
masktype="badvalue",\
maskvalue = 0)
iraf.imstat("superflat_%s.fits" % filt,
fields="image,npix,mean,stddev,min,max,mode",
Stdout="Flat_stats")
time.sleep(0.1)
st = np.genfromtxt("Flat_stats", names=True, dtype=None)
#Normalize flats
iraf.imarith("superflat_%s.fits" % filt, "/", st["MODE"],
"superflat_%s_norm.fits" % filt)
def solve_astrometry(img,
outimage=None,
radius=3,
with_pix=True,
overwrite=False,
tweak=3):
'''
img: fits image where astrometry should be solved.
outimage: name for the astrometry solved image. If none is provided, the name will be "a_"img.
radius: radius of uncertainty on astrometric position in image.
with_pix: if we want to include the constraint on the pixel size for the RCCam.
overwrite: wether the astrometrically solved image should go on top of the old one.
tewak: parameter for astrometry.net
'''
from astropy.wcs import InconsistentAxisTypesError
img = os.path.abspath(img)
ra = fitsutils.get_par(img, 'RA')
dec = fitsutils.get_par(img, 'DEC')
#logger.info( "Solving astrometry on field with (ra,dec)=%s %s"%(ra, dec))
astro = os.path.join(os.path.dirname(img), "a_" + os.path.basename(img))
#If astrometry exists, we don't run it again.
if (os.path.isfile(astro) and not overwrite):
return astro
#Store a temporary file with the multiplication of the mask
mask = "mask.fits"
cmd = "solve-field --ra %s --dec %s --radius %.4f -p --new-fits %s \
-W none -B none -P none -M none -R none -S none -t %d --overwrite %s " % (
ra, dec, radius, astro, tweak, img)
if (with_pix):
cmd = cmd + " --scale-units arcsecperpix --scale-low 0.375 --scale-high 0.4"
#logger.info( cmd)
subprocess.call(cmd, shell=True)
#Cleaning after astrometry.net
if (os.path.isfile(img.replace(".fits", ".axy"))):
os.remove(img.replace(".fits", ".axy"))
if (os.path.isfile(img.replace(".fits", "-indx.xyls"))):
os.remove(img.replace(".fits", "-indx.xyls"))
if (os.path.isfile("none")):
os.remove("none")
try:
is_on_target(img)
except InconsistentAxisTypesError as e:
fitsutils.update_par(img, "ONTARGET", 0)
print "Error detected with WCS when reading file %s. \n %s" % (img, e)
if (not outimage is None and overwrite and os.path.isfile(astro)):
shutil.move(astro, outimage)
return outimage
elif (outimage is None and overwrite and os.path.isfile(astro)):
shutil.move(astro, img)
return img
else:
return astro
#Get the files from the same day directory.
files = glob.glob(os.path.join(raw, "ifu*fits"))
#files_hg = [f for f in files if "Calib: Hg" in get_par(f, "OBJECT")]
#Get the files from one day before directory to see the differences between days.
daybefore = datetime.datetime.isoformat(datetime.datetime.utcnow() -
datetime.timedelta(1))
daybefore = daybefore.split("T")[0].replace("-", "")
daybefore = os.path.join("/scr2/sedm/phot/", daybefore)
if (os.path.isdir(daybefore)):
filesold = glob.glob(os.path.join(daybefore, "ifu*fits"))
files.extend(filesold)
files_hg = [
f for f in files if fitsutils.has_par(f, "OBJECT")
and "Calib: Hg" in fitsutils.get_par(f, "OBJECT")
]
logger.info("Found the following Hg files: %s" % files_hg)
if (len(files_hg) > 1):
files_hg.sort()
sexfiles = sextractor.run_sex(files_hg, mask=False)
plotdir = os.path.join(raw, "stats")
if (not os.path.isdir(plotdir)):
os.makedirs(plotdir)
run_flexure_test(sexfiles, plotdir=plotdir)
def reduce_image(image, flatdir=None, biasdir=None, cosmic=False, astrometry=True, channel='rc', target_dir='reduced', overwrite=False):
'''
Applies Flat field and bias calibrations to the image.
Steps:
1. - Solve astrometry on the entire image.
2. - Computes cosmic ray rejectionon the entire image.
3. - Compute master bias (if it does not exist) and de-bias the image.
4. - Separate the image into 4 filters.
5. - Compute flat field for each filter (if it does not exist) and apply flat fielding on the image.
6. - Compute the image zeropoint.
'''
logger.info("Reducing image %s"% image)
print "Reducing image ", image
image = os.path.abspath(image)
imname = os.path.basename(image).replace(".fits", "")
try:
objectname = fitsutils.get_par(image, "NAME").replace(" ","")+"_"+fitsutils.get_par(image, "FILTER")
except:
logger.error( "ERROR, image "+ image + " does not have a NAME or a FILTER!!!")
return
print "For object", objectname
logger.info( "For object %s"% objectname)
#Change to image directory
mydir = os.path.dirname(image)
if mydir=="": mydir = "."
mydir = os.path.abspath(mydir)
os.chdir(mydir)
#Create destination directory
if (not os.path.isdir(target_dir)):
os.makedirs(target_dir)
#If we don't want to overwrite the already extracted images, we check wether they exist.
if (not overwrite):
existing = True
for band in ['u', 'g', 'r', 'i']:
destfile = os.path.join(target_dir, imname + "_f_b_a_%s_%s_0.fits"%(objectname, band))
logger.info( "Looking if file %s exists: %s"%( destfile, \
(os.path.isfile(destfile) ) ) )
existing = existing and (os.path.isfile( destfile ) )
if existing:
return []
#Initialize the basic parameters.
init_header_reduced(image)
astro = ""
if (astrometry):
logger.info( "Solving astometry for the whole image...")
img = solve_astrometry(image)
if (os.path.isfile(img)):
astro="a_"
fitsutils.update_par(img, "IQWCS", 1)
else:
logger.error( "ASTROMETRY DID NOT SOLVE ON IMAGE %s"% image)
img = image
#Update noise parameters needed for cosmic reection
if (fitsutils.get_par(img, "ADCSPEED")==2):
fitsutils.update_par(img, "RDNOISE", 20.)
else:
fitsutils.update_par(img, "RDNOISE", 4.)
if (cosmic):
logger.info( "Correcting for cosmic rays...")
# Correct for cosmics each filter
cleanimg = clean_cosmic(os.path.join(os.path.abspath(mydir), img))
img = cleanimg
#Get basic statistics for the image
nsrc, fwhm, ellip, bkg = sextractor.get_image_pars(img)
logger.info( "Sextractor statistics: nscr %d, fwhm (pixel) %.2f, ellipticity %.2f"% (nsrc, fwhm, ellip))
print "Sextractor statistics: nscr %d, fwhm (pixel) %.2f, ellipticity %.2f"% (nsrc, fwhm, ellip)
dic = {"SEEPIX": fwhm/0.394, "NSRC":nsrc, "ELLIP":ellip}
#Update the seeing information from sextractor
fitsutils.update_pars(img, dic)
#Compute BIAS
if (biasdir is None or biasdir==""): biasdir = "."
create_masterbias(biasdir)
bias_slow = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'slow'))
bias_fast = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'fast'))
# Running IRAF to DE-BIAS
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Compute flat field
if (flatdir is None or flatdir==""): flatdir = "."
create_masterflat(flatdir, biasdir)
#New names for the object.
debiased = os.path.join(os.path.dirname(img), "b_" + os.path.basename(img))
logger.info( "Creating debiased file, %s"%debiased)
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.warn( "Master bias not found! Tryting to copy from reference folder...")
copy_ref_calib(mydir, "Bias")
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.error( "Bias not found in reference folder")
return
#Clean first
if (os.path.isfile(debiased)):
os.remove(debiased)
#Debias
if (fitsutils.get_par(img, "ADCSPEED")==2):
iraf.imarith(img, "-", bias_fast, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_fast)
fitsutils.update_par(debiased, "RDNOISE", 20.)
else:
iraf.imarith(img, "-", bias_slow, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_slow)
fitsutils.update_par(debiased, "RDNOISE", 4.)
#Set negative counts to zero
hdu = fits.open(debiased)
header = hdu[0].header
hdu[0].data[hdu[0].data<0] = 0
hdu.writeto(debiased, clobber=True)
#Slicing the image for flats
slice_names = slice_rc(debiased)
print "Creating sliced files, ", slice_names
#Remove un-sliced image
os.remove(debiased)
# DE-flat each filter and store under object name
for i, debiased_f in enumerate(slice_names):
b = fitsutils.get_par(debiased_f, 'filter')
deflatted = os.path.join(os.path.dirname(image), target_dir, imname + "_f_b_" + astro + objectname + "_%s.fits"%b)
#Flat to be used for that filter
flat = os.path.join(flatdir, "Flat_%s_%s_norm.fits"%(channel, b))
if (not os.path.isfile(flat)):
logger.warn( "Master flat not found in %s"% flat)
copy_ref_calib(mydir, "Flat_%s_%s_norm"%(channel, b))
continue
else:
logger.info( "Using flat %s"%flat)
#Cleans the deflatted file if exists
if (os.path.isfile(deflatted)):
os.remove(deflatted)
if (os.path.isfile(debiased_f) and os.path.isfile(flat)):
logger.info( "Storing de-flatted %s as %s"%(debiased_f, deflatted))
time.sleep(1)
iraf.imarith(debiased_f, "/", flat, deflatted)
else:
logger.error( "SOMETHING IS WRONG. Error when dividing %s by the flat field %s!"%(debiased_f, flat))
#Removes the de-biased file
os.remove(debiased_f)
logger.info( "Updating header with original filename and flat field used.")
fitsutils.update_par(deflatted, "ORIGFILE", os.path.basename(image))
fitsutils.update_par(deflatted, "FLATFILE", flat)
slice_names[i] = deflatted
#Moving files to the target directory
for image in slice_names:
bkg = get_median_bkg(image)
fitsutils.update_par(image, "SKYBKG", bkg)
#shutil.move(name, newname)
#Compute the zeropoints
for image in slice_names:
zeropoint.calibrate_zeropoint(image)
return slice_names
else:
timestamp = os.path.basename(os.path.abspath(photdir))
os.chdir(photdir)
print("Changed to directory where the data is. %s" % photdir)
if not (os.path.isdir("finders")):
os.makedirs("finders")
#We gather all RC images to locate the Acquisition ones.
files = glob.glob("rc*fits")
files.sort()
filesacq = []
for f in files:
if ((fitsutils.get_par(f, "IMGTYPE").upper() == "ACQUISITION"
or "ACQ" in fitsutils.get_par(f, "IMGTYPE").upper())
and ("TEST" not in fitsutils.get_par(f, "IMGTYPE").upper())):
filesacq.append(f)
print("Found %d files for finders: %s" % (len(filesacq), filesacq))
for f in filesacq:
try:
object = fitsutils.get_par(f, "OBJECT").upper()
except:
print(
'There is no object in this file %s. Skipping the finder and moving to the next file.'
% f)
continue
def finder(myfile, findername, searchrad=0.2 / 60.):
ra, dec = coordinates_conversor.hour2deg(
fitsutils.get_par(myfile, "OBJRA"),
fitsutils.get_par(myfile, "OBJDEC"))
hdulist = pf.open(myfile)[0]
img = hdulist.data * 1.
img = img.T
wcs = WCS(hdulist.header)
target_pix = wcs.wcs_world2pix([(np.array([ra, dec], np.float_))], 1)[0]
corner_pix = wcs.wcs_world2pix(
[(np.array([ra, dec + searchrad], np.float_))], 1)[0]
dx = int(np.abs(np.ceil(corner_pix[1] - target_pix[1])))
imgslice = img[int(target_pix[0]) - 2 * dx:int(target_pix[0]) + 2 * dx,
int(target_pix[1]) - 2 * dx:int(target_pix[1]) + 2 * dx]
imgslice_target = img[int(target_pix[0]) - dx:int(target_pix[0]) + dx,
int(target_pix[1]) - dx:int(target_pix[1]) + dx]
#Maybe the object has moved out of this frame. In this case, make the finder larger.
x, y = imgslice_target.shape
print(img.shape, target_pix, corner_pix, dx,
int(target_pix[0]) - dx,
int(target_pix[0]) + dx,
int(target_pix[1]) - dx,
int(target_pix[1]) + dx)
if (x < 2 * dx - 1) or (y < 2 * dx - 1):
imgslice_target = img[int(target_pix[0]) - 2 * dx:int(target_pix[0]) +
2 * dx,
int(target_pix[1]) - 2 * dx:int(target_pix[1]) +
2 * dx]
#zmin, zmax = zscale.zscale()
zmin = np.percentile(imgslice_target.flatten(), 5)
zmax = np.percentile(imgslice_target.flatten(), 98.5)
print("Min: %.1f, max: %.1f" % (zmin, zmax))
gc = aplpy.FITSFigure(myfile, figsize=(10, 9), north=True)
gc.show_grayscale(vmin=zmin, vmax=zmax, smooth=1, kernel="gauss")
gc.show_scalebar(0.1 / 60.)
gc.scalebar.set_label('10 arcsec')
gc.scalebar.set_color('white')
gc.recenter(ra, dec, searchrad)
#gc.show_markers(ra,dec+searchrad/20.,edgecolor='red',facecolor='none',marker="|",s=250, lw=10)
#gc.show_markers(ra-(searchrad/20.)/np.cos(np.deg2rad(dec)),dec,edgecolor='red',facecolor='none',marker="_",s=250, lw=10)
ras = np.array([ra, ra])
decs = np.array([dec, dec])
dxs = np.array([0, searchrad / 10 / np.cos(np.deg2rad(dec))])
dys = np.array([searchrad / 10, 0])
gc.show_arrows(ras,
decs,
dxs,
dys,
edgecolor="red",
facecolor="red",
head_width=0)
ras = np.array([
ra + searchrad * 0.7 / np.cos(np.deg2rad(dec)),
ra + searchrad * 0.7 / np.cos(np.deg2rad(dec))
])
decs = np.array([dec - searchrad * 0.9, dec - searchrad * 0.9])
dxs = np.array([0, searchrad / 5 / np.cos(np.deg2rad(dec))])
dys = np.array([searchrad / 5, 0])
gc.show_arrows(ras, decs, dxs, dys, edgecolor="k", facecolor="k")
gc.add_label(ras[0] + dxs[0] * 1.1,
decs[0] + dys[0] * 1.1,
'N',
relative=False,
color="k",
horizontalalignment="center")
gc.add_label(ras[1] + dxs[1] * 1.1,
decs[1] + dys[1] * 1.1,
'E',
relative=False,
color="k",
horizontalalignment="center")
name = fitsutils.get_par(myfile, "NAME").strip()
filter = fitsutils.get_par(myfile, "FILTER")
gc.add_label(0.05,
0.95,
'Object: %s' % (name),
relative=True,
color="white",
horizontalalignment="left")
gc.add_label(0.05,
0.9,
'Coordinates: RA=%s DEC=%s' %
(coordinates_conversor.deg2hour(ra, dec)),
relative=True,
color="white",
horizontalalignment="left")
gc.add_label(0.05,
0.84,
'Filter: SDSS %s' % filter,
relative=True,
color="white",
horizontalalignment="left")
gc.save(findername)
def create_masterflat(flatdir=None, biasdir=None, channel='rc'):
'''
Creates a masterflat from both dome flats and sky flats if the number of counts in the given filter
is not saturated and not too low (between 1500 and 40000).
'''
if (flatdir == None or flatdir==""): flatdir = "."
if (biasdir == None or biasdir==""): biasdir = "."
os.chdir(flatdir)
if (len(glob.glob("Flat_%s*norm.fits"%channel)) == 4):
print "Master Flat exists!"
return
else:
print "Starting the Master Flat creation!"
bias_slow = "Bias_%s_fast.fits"%channel
bias_fast = "Bias_%s_fast.fits"%channel
if (not os.path.isfile(bias_slow) and not os.path.isfile(bias_fast) ):
create_masterbias(biasdir)
lsflat = []
lfflat = []
#Select all filts that are Flats with same instrument
for f in glob.glob("*fits"):
#try:
if fitsutils.has_par(f, "OBJECT"):
obj = str.upper(fitsutils.get_par(f, "OBJECT"))
else:
continue
if ( ("DOME" in obj or "FLAT" in obj) and (channel == fitsutils.get_par(f, "CHANNEL"))):
if (fitsutils.get_par(f, "ADCSPEED")==2):
lfflat.append(f)
else:
lsflat.append(f)
#except:
# print "Error with retrieving parameters for file", f
# pass
print "Files for slow flat", lsflat
print "Files for fast flat", lfflat
fsfile ="lflat_slow_"+channel
np.savetxt(fsfile, np.array(lsflat), fmt="%s")
fffile ="lflat_fast_"+channel
np.savetxt(fffile, np.array(lfflat), fmt="%s")
# Running IRAF
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Remove bias from the flat
if len(lsflat) >0:
iraf.imarith("@"+fsfile, "-", bias_slow, "b_@"+fsfile)
if len(lfflat) >0:
iraf.imarith("@"+fffile, "-", bias_fast, "b_@"+fffile)
#Slices the flats.
debiased_flats = glob.glob("b_*.fits")
for f in debiased_flats:
print "Slicing file", f
slice_rc(f)
#Remove the un-sliced file
os.remove(f)
#Selects the ones that are suitable given the number of counts and combines them.
bands = ['u', 'g', 'r', 'i']
for b in bands:
out = "Flat_%s_%s.fits"%(channel, b)
out_norm = out.replace(".fits","_norm.fits")
if (os.path.isfile(out_norm)):
print "Master Flat for filter %s exists. Skipping..."%b
continue
lfiles = []
for f in glob.glob('b_*_%s.fits'%b):
d = pf.open(f)[0].data
if np.percentile(d, 90)>1500 and np.percentile(d, 90)<40000:
lfiles.append(f)
if len(lfiles) == 0:
print "WARNING!!! Could not find suitable flats for band %s"%b
continue
ffile ="lflat_"+b
np.savetxt(ffile, np.array(lfiles), fmt="%s")
#Cleaning of old files
if(os.path.isfile(out)): os.remove(out)
if(os.path.isfile(out_norm)): os.remove(out_norm)
if(os.path.isfile("Flat_stats")): os.remove("Flat_stats")
#Combine flats
iraf.imcombine(input = "@"+ffile, \
output = out, \
combine = "median",\
scale = "mode",
weight = "exposure")
iraf.imstat(out, fields="image,npix,mean,stddev,min,max,mode", Stdout="Flat_stats")
st = np.genfromtxt("Flat_stats", names=True, dtype=None)
#Normalize flats
iraf.imarith(out, "/", st["MODE"], out_norm)
#Do some cleaning
print 'Removing from lfiles'
for f in glob.glob('b_*_%s.fits'%b):
os.remove(f)
os.remove(ffile)
if os.path.isfile(fsfile):
os.remove(fsfile)
if os.path.isfile(fffile):
os.remove(fffile)
def __fill_ifu_dic(ifu_img, ifu_dic={}):
'''
Fills some of the parameters for the IFU image that will be used to gather its guider images later.
'''
imgtype = fitsutils.get_par(ifu_img, "IMGTYPE")
if not imgtype is None:
imgtype = imgtype.upper()
#In Richard's pipeline, the JD is the beginning of the exposure,
#in Nick's one is the end.
pipeline_jd_end = fitsutils.get_par(ifu_img, "TELESCOP") == '60'
if imgtype == "SCIENCE" or imgtype == "STANDARD":
if pipeline_jd_end:
jd_ini = fitsutils.get_par(ifu_img, "JD") - fitsutils.get_par(
ifu_img, "EXPTIME") / (24 * 3600.)
jd_end = fitsutils.get_par(ifu_img, "JD")
else:
jd_ini = fitsutils.get_par(ifu_img, "JD")
jd_end = fitsutils.get_par(ifu_img, "JD") + fitsutils.get_par(
ifu_img, "EXPTIME") / (24 * 3600.)
#We only fill the dictionary if the exposure is a valid science or standard image.
name = fitsutils.get_par(ifu_img, "OBJECT")
ra = fitsutils.get_par(ifu_img, "RA")
dec = fitsutils.get_par(ifu_img, "DEC")
rad, decd = cc.hour2deg(ra, dec)
exptime = fitsutils.get_par(ifu_img, "EXPTIME")
ifu_dic[ifu_img] = (name, jd_ini, jd_end, rad, decd, exptime)
else:
logger.warn("Image %s is not SCIENCE or STANDARD." % ifu_img)
def reduce_image(img, flatdir=None, biasdir=None, cosmic=True, astrometry=True, channel='rc', target_dir='reduced'):
'''
Applies Flat field and bias calibrations to the image.
Steps:
1. - Solve astrometry on the entire image.
2. - Compute master bias and de-bias the image.
3. - Separate the image into 4 filters.
4. - Compute flat field for each filter and apply flat fielding on the image.
5. - Computes cosmic ray rejectionon the entire image.
6. - Compute zeropoint for each image and store in a log file.
7. - Plot zeropoint for the night.
'''
print "Reducing image ", img
objectname = fitsutils.get_par(img, "OBJECT").replace(" ", "").replace("]","").replace("[", "")
print "For object", objectname
#Change to image directory
mydir = os.path.dirname(img)
if mydir=="": mydir = "."
mydir = os.path.abspath(mydir)
os.chdir(mydir)
#Create destination directory
if (not os.path.isdir(target_dir)):
os.makedirs(target_dir)
#Rename to the image name only
img = os.path.basename(img)
if (astrometry):
print "Solving astometry for the whole image..."
img = solve_astrometry(img)
astro = "a_"
else:
astro = ""
#Compute BIAS
if (biasdir == None or biasdir==""): biasdir = "."
create_masterbias(biasdir)
bias_slow = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'slow'))
bias_fast = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'fast'))
# Running IRAF to DE-BIAS
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Compute flat field
if (flatdir == None or flatdir==""): flatdir = "."
create_masterflat(flatdir, biasdir)
#New names for the object.
debiased = "b_" + astro + img
print "Creating debiased file, ",debiased
if (not os.path.isfile(bias_slow) or not os.path.isfile(bias_fast)):
print "Master bias not found!"
return
#Debias
if (fitsutils.get_par(img, "ADCSPEED")==2):
iraf.imarith(img, "-", bias_fast, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_fast)
fitsutils.update_par(debiased, "RDNOISE", 20.)
else:
iraf.imarith(img, "-", bias_slow, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_slow)
fitsutils.update_par(debiased, "RDNOISE", 4.)
#Set negative counts to zero
hdu = pf.open(debiased)
header = hdu[0].header
hdu[0].data[hdu[0].data<0] = 0
hdu.writeto(debiased, clobber=True)
#Slicing the image for flats
slice_names = slice_rc(debiased)
#Remove un-sliced image
os.remove(debiased)
# DE-flat each filter and store under object name
for i, debiased_f in enumerate(slice_names):
b = fitsutils.get_par(debiased_f, 'filter')
deflatted = "f_b_" + astro + objectname + "_%s.fits"%b
#Flat to be used for that filter
flat = os.path.join(flatdir, "Flat_%s_%s_norm.fits"%(channel, b))
if (not os.path.isfile(flat)):
print "Master flat not found in", flat
return
#Cleans the deflatted file if exists
if (os.path.isfile(deflatted)):
os.remove(deflatted)
iraf.imarith(debiased_f, "/", flat, deflatted)
#Removes the de-biased file
os.remove(debiased_f)
print "Updating header with original filename and flat field used."
fitsutils.update_par(deflatted, "ORIGFILE", img)
fitsutils.update_par(deflatted, "FLATFILE", flat)
slice_names[i] = deflatted
if (cosmic):
print "Correcting for cosmic rays..."
# Correct for cosmics each filter
for i, deflatted in enumerate(slice_names):
cclean = "c_" +name
clean_cosmic(os.path.join(os.path.abspath(mydir), deflatted), cclean)
slice_names[i] = cclean
#Moving files to the target directory
for name in slice_names:
if (os.path.isfile(name)):
shutil.move(name, os.path.join(target_dir, name))
def create_masterguide(lfiles, out=None):
'''
Receives a list of guider images for the same object.
It will remove the bias from it, combine them using the median, and comput the astrometry for the image.
'''
curdir = os.getcwd()
if len(lfiles) == 0:
return
else:
os.chdir(os.path.abspath(os.path.dirname(lfiles[0])))
fffile = "/tmp/l_guide"
np.savetxt(fffile, np.array(lfiles), fmt="%s")
#If the bias file exists in the directory, we use it, otherwise we pass
bias_fast = "Bias_rc_fast.fits"
debias = os.path.isfile(bias_fast)
if debias:
debiased = ["b_" + os.path.basename(img) for img in lfiles]
bffile = "/tmp/lb_guider"
np.savetxt(bffile, np.array(debiased), fmt="%s")
if (out is None):
obj = fitsutils.get_par(img, "OBJECT")
out = os.path.join(os.path.dirname(img),
obj.replace(" ", "").replace(":", "") + ".fits")
# Running IRAF
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Remove bias from the guider images
if debias:
try:
iraf.imarith("@" + fffile, "-", bias_fast, "@" + bffile)
except IrafError:
iraf.imarith("@" + fffile, "-", bias_fast, "@" + bffile)
else:
bffile = fffile
#Combine flats
iraf.imcombine(input = "@"+bffile, \
output = out, \
combine = "median",\
scale = "mode",
reject = "sigclip", lsigma = 2., hsigma = 2, gain=1.7, rdnoise=4.)
iraf.imstat(out,
fields="image,npix,mean,stddev,min,max,mode",
Stdout="guide_stats")
#st = np.genfromtxt("guide_stats", names=True, dtype=None)
#Do some cleaning
if debias:
logger.info('Removing from lfiles')
for f in debiased:
if os.path.isfile(f): os.remove(f)
if os.path.isfile(fffile):
os.remove(fffile)
if os.path.isfile(bffile):
os.remove(bffile)
solve_astrometry(out, overwrite=True)
os.chdir(curdir)
photdir = os.path.join(_photpath, timestamp)
print '''WARNING! You did not specify the directory or the list:\n
- A filelist name with the images you want to reduce [-l]
OR
- The name of the directory which you want to reduce [-d].
A default name for the directory will be assumed on today's date: %s
''' % photdir
mydir = os.path.abspath(photdir)
#Gather all RC fits files in the folder with the keyword IMGTYPE=SCIENCE
for f in glob.glob(os.path.join(mydir, "rc*fits")):
try:
if (fitsutils.has_par(f, "IMGTYPE") and
((fitsutils.get_par(f, "IMGTYPE").upper() == "SCIENCE") or
("ACQ" in fitsutils.get_par(f, "IMGTYPE").upper()))):
myfiles.append(f)
except:
print "problems opening file %s" % f
create_masterbias(mydir)
print "Create masterflat", mydir
create_masterflat(mydir)
if (len(myfiles) == 0):
print "Found no files to process"
sys.exit()
else:
print "Found %d files to process" % len(myfiles)
def reduce_image(image,
flatdir=None,
biasdir=None,
cosmic=False,
astrometry=True,
channel='rc',
target_dir='reduced',
overwrite=False):
'''
Applies Flat field and bias calibrations to the image.
Steps:
1. - Solve astrometry on the entire image.
2. - Computes cosmic ray rejectionon the entire image.
3. - Compute master bias (if it does not exist) and de-bias the image.
4. - Separate the image into 4 filters.
5. - Compute flat field for each filter (if it does not exist) and apply flat fielding on the image.
6. - Compute the image zeropoint.
'''
logger.info("Reducing image %s" % image)
print "Reducing image ", image
image = os.path.abspath(image)
imname = os.path.basename(image).replace(".fits", "")
try:
objectname = fitsutils.get_par(image, "NAME").replace(
" ", "") + "_" + fitsutils.get_par(image, "FILTER")
except:
logger.error("ERROR, image " + image +
" does not have a NAME or a FILTER!!!")
return
print "For object", objectname
logger.info("For object %s" % objectname)
#Change to image directory
mydir = os.path.dirname(image)
if mydir == "": mydir = "."
mydir = os.path.abspath(mydir)
os.chdir(mydir)
#Create destination directory
if (not os.path.isdir(target_dir)):
os.makedirs(target_dir)
#If we don't want to overwrite the already extracted images, we check wether they exist.
if (not overwrite):
existing = True
for band in ['u', 'g', 'r', 'i']:
destfile = os.path.join(
target_dir,
imname + "_f_b_a_%s_%s_0.fits" % (objectname, band))
logger.info( "Looking if file %s exists: %s"%( destfile, \
(os.path.isfile(destfile) ) ) )
existing = existing and (os.path.isfile(destfile))
if existing:
return []
#Initialize the basic parameters.
init_header_reduced(image)
astro = ""
if (astrometry):
logger.info("Solving astometry for the whole image...")
img = solve_astrometry(image)
if (os.path.isfile(img)):
astro = "a_"
fitsutils.update_par(img, "IQWCS", 1)
else:
logger.error("ASTROMETRY DID NOT SOLVE ON IMAGE %s" % image)
img = image
#Update noise parameters needed for cosmic reection
if (fitsutils.get_par(img, "ADCSPEED") == 2):
fitsutils.update_par(img, "RDNOISE", 20.)
else:
fitsutils.update_par(img, "RDNOISE", 4.)
if (cosmic):
logger.info("Correcting for cosmic rays...")
# Correct for cosmics each filter
cleanimg = clean_cosmic(os.path.join(os.path.abspath(mydir), img))
img = cleanimg
#Compute BIAS
if (biasdir is None or biasdir == ""): biasdir = "."
create_masterbias(biasdir)
bias_slow = os.path.join(biasdir, "Bias_%s_%s.fits" % (channel, 'slow'))
bias_fast = os.path.join(biasdir, "Bias_%s_%s.fits" % (channel, 'fast'))
# Running IRAF to DE-BIAS
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Compute flat field
if (flatdir is None or flatdir == ""): flatdir = "."
create_masterflat(flatdir, biasdir)
#New names for the object.
debiased = os.path.join(os.path.dirname(img), "b_" + os.path.basename(img))
logger.info("Creating debiased file, %s" % debiased)
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.warn(
"Master bias not found! Tryting to copy from reference folder...")
copy_ref_calib(mydir, "Bias")
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.error("Bias not found in reference folder")
return
#Clean first
if (os.path.isfile(debiased)):
os.remove(debiased)
#Debias
if (fitsutils.get_par(img, "ADCSPEED") == 2):
iraf.imarith(img, "-", bias_fast, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_fast)
fitsutils.update_par(debiased, "RDNOISE", 20.)
else:
iraf.imarith(img, "-", bias_slow, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_slow)
fitsutils.update_par(debiased, "RDNOISE", 4.)
#Set negative counts to zero
hdu = fits.open(debiased)
header = hdu[0].header
hdu[0].data[hdu[0].data < 0] = 0
hdu.writeto(debiased, clobber=True)
#Slicing the image for flats
slice_names = slice_rc(debiased)
print "Creating sliced files, ", slice_names
#Remove un-sliced image
os.remove(debiased)
# DE-flat each filter and store under object name
for i, debiased_f in enumerate(slice_names):
b = fitsutils.get_par(debiased_f, 'filter')
deflatted = os.path.join(
os.path.dirname(image), target_dir,
imname + "_f_b_" + astro + objectname + "_%s.fits" % b)
#Flat to be used for that filter
flat = os.path.join(flatdir, "Flat_%s_%s_norm.fits" % (channel, b))
if (not os.path.isfile(flat)):
logger.warn("Master flat not found in %s" % flat)
copy_ref_calib(mydir, "Flat")
continue
else:
logger.info("Using flat %s" % flat)
#Cleans the deflatted file if exists
if (os.path.isfile(deflatted)):
os.remove(deflatted)
if (os.path.isfile(debiased_f) and os.path.isfile(flat)):
logger.info("Storing de-flatted %s as %s" %
(debiased_f, deflatted))
time.sleep(1)
iraf.imarith(debiased_f, "/", flat, deflatted)
else:
logger.error(
"SOMETHING IS WRONG. Error when dividing %s by the flat field %s!"
% (debiased_f, flat))
#Removes the de-biased file
os.remove(debiased_f)
logger.info(
"Updating header with original filename and flat field used.")
fitsutils.update_par(deflatted, "ORIGFILE", os.path.basename(image))
fitsutils.update_par(deflatted, "FLATFILE", flat)
slice_names[i] = deflatted
#Moving files to the target directory
for image in slice_names:
bkg = get_median_bkg(image)
fitsutils.update_par(image, "SKYBKG", bkg)
#Get basic statistics for the image
nsrc, fwhm, ellip, bkg = sextractor.get_image_pars(image)
logger.info(
"Sextractor statistics: nscr %d, fwhm (arcsec) %.2f, ellipticity %.2f"
% (nsrc, fwhm, ellip))
print "Sextractor statistics: nscr %d, fwhm (arcsec) %.2f, ellipticity %.2f" % (
nsrc, fwhm, ellip)
dic = {
"FWHM": np.round(fwhm, 3),
"FWHMPIX": np.round(fwhm / 0.394, 3),
"NSRC": nsrc,
"ELLIP": np.round(ellip, 3)
}
#Update the seeing information from sextractor
fitsutils.update_pars(image, dic)
#Compute the zeropoints
for image in slice_names:
zeropoint.calibrate_zeropoint(image)
return slice_names
