def reduce_on_the_fly(photdir):
'''
Waits for new images to appear in the directory to trigger their incremental reduction as well.
'''
#Get the current the number of files
nfiles = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
logger.info(
"Starting the on-the-fly reduction for directory %s. Found %d files to process."
% (photdir, len(nfiles)))
dayname = os.path.basename(photdir)
time_ini = datetime.datetime.now()
time_curr = datetime.datetime.now()
#Run this loop for 12h since the start.
while (time_curr - time_ini).total_seconds() < 12. * 3600:
nfilesnew = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
if len(nfilesnew) == len(nfiles):
time.sleep(10)
else:
new = [f for f in nfilesnew if f not in nfiles]
new.sort()
logger.info("Detected new %d incoming files in the last 10s." %
len(new))
for n in new:
if (not fitsutils.has_par(n, "IMGTYPE")):
print "Image", n, "Does not have an IMGTYPE"
time.sleep(0.5)
if (not fitsutils.has_par(n, "IMGTYPE")):
print "Image", n, "STILL Does not have an IMGTYPE"
continue
plot_image(n)
imtype = fitsutils.get_par(n, "IMGTYPE")
if (imtype.upper() == "SCIENCE"):
reduced = rcred.reduce_image(n)
try:
zeropoint.calibrate_zeropoint(reduced)
except:
logger.error(
"Could not calibrate zeropoint for image %s" %
reduced)
#Copy them to transient
for r in reduced:
cmd = "rcp %s [email protected]:/scr3/mansi/ptf/p60phot/fremling_pipeline/sedm/reduced/%s/." % (
r, dayname)
subprocess.call(cmd, shell=True)
logger.info(cmd)
logger.info("Successfully copied the image: %s" % cmd)
time_curr = datetime.datetime.now()
nfiles = nfilesnew
def clean_cosmic(f):
'''
From lacosmic.
'''
import cosmics
out = f.replace('.fits', '_clean.fits')
#If it does already exist, just return the name.
if (os.path.isfile(out)):
return out
#Otherwise, run the cosmic ray rejection based on LA Cosmic.
g = fitsutils.get_par(f, "GAIN")
if (fitsutils.has_par(f, "RDNOISE")):
rn = fitsutils.get_par(f, "RDNOISE")
else:
rn = 20
array, header = cosmics.fromfits(f)
try:
c = cosmics.cosmicsimage(array, gain=g, readnoise=rn, sigclip = 8.0, sigfrac = 0.3, satlevel = 64000.0)
c.run(maxiter = 3)
out = f.replace('.fits', '_clean.fits')
cosmics.tofits(out, c.cleanarray, header)
fitsutils.update_par(out, "CRREJ", 1)
#os.remove(f)
except:
pass
return out
def get_sextractor_stats(files):
files.sort()
sexfiles = [os.path.join(os.path.join(os.path.dirname(f), "sextractor"), os.path.basename(f).replace(".fits", ".sex")) for f in files]
sexfiles.sort()
if not os.path.isdir(os.path.join( os.path.dirname(files[0]), "stats")):
os.makedirs(os.path.join(os.path.dirname(files[0]), "stats"))
with open(os.path.join( os.path.dirname(files[0]), "stats/stats.log"), "w") as out:
for i, f in enumerate(files):
if (fitsutils.has_par(f, "IMGTYPE")):
imtype = fitsutils.get_par(f, "IMGTYPE")
else:
imtype = "NONE"
if not (imtype == "ACQUISITION" or imtype == "SCIENCE" or imtype=="FOCUS" or imtype=="GUIDER"):
continue
if not os.path.isfile(sexfiles[i]):
sf = sextractor.run_sex([f])
else:
sf = sexfiles[i]
print f
hd = pf.open(files[i])[0].header
try:
jd = hd["JD"]
obj = hd["OBJECT"]
airmass = hd["AIRMASS"]
in_temp = hd["IN_AIR"]
out_temp = hd["OUT_AIR"]
in_hum = hd["IN_HUM"]
ns, fwhm, ellipticity, bkg = sextractor.analyse_image(sf)
out.write("%s,%s,%.3f,%d,%.2f,%.3f,%.3f,%.2f,%.1f,%s,%.2f,%.2f\n"%(os.path.abspath(f),obj,jd,ns,fwhm,ellipticity,bkg,airmass,in_temp,imtype,out_temp,in_hum))
except:
pass
def compute_offsets(fitsdir):
ra_prev = "00:00:00"
dec_prev = "00:00:00"
lfiles = glob.glob(fitsdir + "/rc*fits")
lfiles.sort()
for f in lfiles:
f = os.path.basename(f)
try:
if ("[r]" in fitsutils.get_par(f, "OBJECT")
or fitsutils.get_par(f, "OBJTYPE") == "ACQUISITION"):
ra = fitsutils.get_par(f, "OBJRA")
dec = fitsutils.get_par(f, "OBJDEC")
obj = fitsutils.get_par(f, "OBJECT")
isAB = False
if (fitsutils.has_par(f, "ABPAIR")
and fitsutils.get_par(f, "ABPAIR") == "True"):
isAB = True
if ((ra != "" and dec != "" and
(ra != ra_prev or dec != dec_prev))
or (fitsutils.get_par(f, "OBJTYPE") == "ACQUISITION")):
#print "Found image %s as first acquisition image after the slew. Computing offset for IFU..."%f
recenter_ifu.main(os.path.abspath(f),
isAB,
astro=True,
plot=True)
ra_prev = ra
dec_prev = dec
except KeyError:
print "File %s does not have a good header." % f
def compute_offsets(fitsdir):
ra_prev = "00:00:00"
dec_prev = "00:00:00"
lfiles = glob.glob(fitsdir + "/rc*fits")
lfiles.sort()
for f in lfiles:
f = os.path.basename(f)
try:
if ("[r]" in fitsutils.get_par(f, "OBJECT") or fitsutils.get_par(f, "OBJTYPE")=="ACQUISITION"):
ra = fitsutils.get_par(f, "OBJRA")
dec = fitsutils.get_par(f, "OBJDEC")
obj = fitsutils.get_par(f, "OBJECT")
isAB = False
if (fitsutils.has_par(f, "ABPAIR") and fitsutils.get_par(f, "ABPAIR")=="True"):
isAB = True
if ((ra!="" and dec !="" and (ra != ra_prev or dec != dec_prev)) or (fitsutils.get_par(f, "OBJTYPE")=="ACQUISITION")):
#print "Found image %s as first acquisition image after the slew. Computing offset for IFU..."%f
recenter_ifu.main(os.path.abspath(f), isAB, astro=True, plot=True)
ra_prev = ra
dec_prev = dec
except KeyError:
print "File %s does not have a good header."%f
def update_phot_folder(reduced):
for image in glob.glob(os.path.join(reduced, "rc*fits")):
cat=fitsutils.get_par(image, "ZPCAT")
mag=fitsutils.get_par(image, "APPMAG")
ontarget = fitsutils.get_par(image, "ONTARGET")
if (ontarget ==1 and cat == 'SDSSinterpolated' and mag !=0 and fitsutils.has_par(f, "IMGTYPE") and fitsutils.get_par(f, "IMGTYPE") == "SCIENCE"):
update_phot_blagorodnova(image)
def calibrate_zeropoint(image, plot=True, debug=False, refstars=None):
filt = fitsutils.get_par(image, 'filter')
exptime = fitsutils.get_par(image, "exptime")
if fitsutils.has_par(image, "JD"):
date = fitsutils.get_par(image, "JD")
elif fitsutils.has_par(image, "MJD"):
date = fitsutils.get_par(image, "MJD")
objname = fitsutils.get_par(image, "OBJECT")
airmass = fitsutils.get_par(image, "AIRMASS")
print "Starting calibration of ZP for image", image,"for object", objname
if (exptime < 10):
print "ERROR. Exposure time too short for this image to see anything..."
return
extracted = extract_star_sequence(image, filt, plot=plot, survey='sdss', debug=debug, refstars=refstars)
if (not extracted):
print "Field not in SDSS or error when retrieving the catalogue... Skipping."
return
#If extraction worked, we can get the FWHM
fwhm = fitsutils.get_par(image, "fwhm")
fwhm_as = fwhm * 0.394
app_phot.get_app_phot("/tmp/sdss_cat_det.txt", image, wcsin='logic')
z, c, err = find_zeropoint_noid("/tmp/sdss_cat_det.txt", image, plot=plot)
#Log the current zeropoint for this image
logname = os.path.join(os.path.dirname(image), "zeropoint.log")
#Add the data to a later stage zeropoint calibrtion with all-sky data.
zplogname = os.path.join(os.path.dirname(image), "allstars_zp.log")
add_to_zp_cal("/tmp/sdss_cat_det.txt", image, zplogname)
if (not os.path.isfile(logname)):
with open( logname, "a") as f:
f.write("#filename,exptime,filter,date,airmass,fwhm_pix,fwhm_as,zeropoint,color,err\n")
with open( logname, "a") as f:
f.write("%s,%.1f,%s,%3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f\n"%(image,exptime,filt,date,airmass,fwhm,fwhm_as,z,c,err))
def update_phot_folder(reduced):
for image in glob.glob(os.path.join(reduced, "rc*fits")):
cat = fitsutils.get_par(image, "ZPCAT")
mag = fitsutils.get_par(image, "APPMAG")
ontarget = fitsutils.get_par(image, "ONTARGET")
if (ontarget == 1 and cat == 'SDSSinterpolated' and mag != 0
and fitsutils.has_par(f, "IMGTYPE")
and fitsutils.get_par(f, "IMGTYPE") == "SCIENCE"):
update_phot_blagorodnova(image)
def reduce_on_the_fly(photdir):
'''
Waits for new images to appear in the directory to trigger their incremental reduction as well.
'''
#Get the current the number of files
nfiles = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
dayname = os.path.basename(photdir)
time_ini = datetime.datetime.now()
time_curr = datetime.datetime.now()
#Run this loop for 12h since the start.
while (time_curr-time_ini).total_seconds() < 12.*3600:
nfilesnew = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
if len(nfilesnew) == len(nfiles):
time.sleep(10)
else:
new = [f for f in nfilesnew if f not in nfiles]
for n in new:
if (not fitsutils.has_par(n, "IMGTYPE")):
print "Image",n,"Does not have an IMGTYPE"
time.sleep(0.5)
if (not fitsutils.has_par(n, "IMGTYPE")):
print "Image",n,"STILL Does not have an IMGTYPE"
continue
if (fitsutils.get_par(n, "IMGTYPE")=="SCIENCE"):
reduced = rcred.reduce_image(n)
try:
zeropoint.calibrate_zeropoint(reduced)
except:
logger.error("Could not calibrate zeropoint for image %s"%reduced)
#Copy them to transient
for r in reduced:
cmd = "rcp %s [email protected]:/scr3/mansi/ptf/p60phot/fremling_pipeline/sedm/reduced/%s/."%(r, dayname)
subprocess.call(cmd, shell=True)
logger.info(cmd)
logger.info( "Successfully copied the image: %s"% cmd)
time_curr = datetime.datetime.now()
nfiles = nfilesnew
def get_sextractor_stats(files):
files.sort()
sexfiles = [
os.path.join(os.path.join(os.path.dirname(f), "sextractor"),
os.path.basename(f).replace(".fits", ".sex"))
for f in files
]
sexfiles.sort()
if not os.path.isdir(os.path.join(os.path.dirname(files[0]), "stats")):
os.makedirs(os.path.join(os.path.dirname(files[0]), "stats"))
with open(os.path.join(os.path.dirname(files[0]), "stats/stats.log"),
"w") as out:
for i, f in enumerate(files):
try:
if (fitsutils.has_par(f, "IMGTYPE")):
imtype = fitsutils.get_par(f, "IMGTYPE")
imtype = imtype.upper()
else:
imtype = "NONE"
if not ("ACQ" in imtype or imtype == "SCIENCE"
or imtype == "FOCUS" or imtype == "GUIDER"):
continue
if not os.path.isfile(sexfiles[i]):
sflist = sextractor.run_sex([f])
if (not sflist is None and len(sflist) > 0):
sf = sflist[0]
else:
sf = sexfiles[i]
hd = pf.open(f)[0].header
try:
jd = hd["JD"]
obj = hd["OBJECT"]
airmass = hd["AIRMASS"]
in_temp = hd["IN_AIR"]
out_temp = hd["OUT_AIR"]
in_hum = hd["IN_HUM"]
ns, fwhm, ellipticity, bkg = sextractor.analyse_image(sf)
out.write(
"%s,%s,%.3f,%d,%.2f,%.3f,%.3f,%.2f,%.1f,%s,%.2f,%.2f\n"
% (os.path.abspath(f), obj, jd, ns, fwhm, ellipticity,
bkg, airmass, in_temp, imtype, out_temp, in_hum))
except Exception as e:
print "Error when retrieving the stats parameters from the header of file %s.\n Error %s" % (
f, e)
except IOError:
print "Error when opening file %s" % f
def run_sex(flist, mask=False, cosmics=False, overwrite=True):
d = os.path.dirname(flist[0])
if d == "":
d = "."
os.chdir(d)
#Create the directory where the sextracted images are going to go.
sexdir = os.path.join(d, "sextractor")
if (not os.path.isdir(sexdir)):
os.makedirs(sexdir)
newlist = []
for f in flist:
newimage = os.path.join(sexdir, os.path.basename(f).replace(".fits", ".sex"))
if (os.path.isfile(newimage) and not overwrite):
newlist.append(newimage)
print "Sextracted image %s already exists."%newimage
else:
try:
f = os.path.abspath(f)
if (mask):
out = rcred.get_masked_image(f)
else:
out = f
if (cosmics and (not fitsutils.has_par(out, "CRREJ") or fitsutils.get_par(out, "CRREJ") ==0)):
out = rcred.clean_cosmic(out)
cmd="sex -c %s/config/daofind.sex %s"%(os.environ["SEDMPH"], out)
subprocess.call(cmd, shell=True)
print cmd
shutil.move("image.sex", newimage)
newlist.append(newimage)
except IOError:
print "IOError detected reading file",f
pass
return newlist
def main(reduced):
"""
Performs the main zeropoint calculations for the folder and plots the results.
"""
os.chdir(reduced)
plotdir = "zeropoint"
if not os.path.isdir(plotdir):
os.makedirs(plotdir)
for f in glob.glob("*.fits|*.new"):
logger.info("Starting calibration of zeropoint for %s" % f)
if not fitsutils.has_par(f, "IMGTYPE") or fitsutils.get_par(f, "IMGTYPE") == "SCIENCE":
calibrate_zeropoint(f, plotdir=os.path.abspath(plotdir))
if os.path.isfile("zeropoint.log"):
plot_zp("zeropoint.log", plotdir)
if os.path.isfile("allstars_zp.log"):
lsq_zeropoint("allstars_zp.log", plotdir)
interpolate_zp(reduced, "allstars_zp.log")
clean_tmp_files()
def clean_cosmic(f):
'''
From lacosmic.
'''
import cosmics
out = f.replace('.fits', '_clean.fits')
#If it does already exist, just return the name.
if (os.path.isfile(out)):
return out
#Otherwise, run the cosmic ray rejection based on LA Cosmic.
g = fitsutils.get_par(f, "GAIN")
if (fitsutils.has_par(f, "RDNOISE")):
rn = fitsutils.get_par(f, "RDNOISE")
else:
rn = 20
array, header = cosmics.fromfits(f)
try:
c = cosmics.cosmicsimage(array,
gain=g,
readnoise=rn,
sigclip=8.0,
sigfrac=0.3,
satlevel=64000.0)
c.run(maxiter=3)
out = f.replace('.fits', '_clean.fits')
cosmics.tofits(out, c.cleanarray, header)
fitsutils.update_par(out, "CRREJ", 1)
#os.remove(f)
except:
pass
return out
def get_app_phot(coords, image, plot_only=False, store=True, wcsin="world", fwhm=2.5, plotdir=None, box=15, arcsecpix=0.394):
'''
coords: files:
wcsin: can be "world", "logic"
'''
# Load packages; splot is in the onedspec package, which is in noao.
# The special keyword _doprint=0 turns off displaying the tasks
# when loading a package.
from pyraf import iraf
if (not plot_only):
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.apphot(_doprint=0)
iraf.unlearn("apphot")
imdir = os.path.dirname(image)
imname = os.path.basename(image)
if (plotdir is None):
plotdir = os.path.join(imdir, "photometry")
if not os.path.isdir(plotdir):
os.makedirs(plotdir)
out_name = os.path.join(plotdir, imname + ".seq.mag")
clean_name = os.path.join(plotdir, imname + ".app.mag")
print "Will create output files", out_name, clean_name
# Read values from .ec file
fwhm_value = fwhm/arcsecpix
if (fitsutils.has_par(image, 'FWHM')):
fwhm_value = fitsutils.get_par(image, 'FWHM')/arcsecpix
elif (fwhm is None):
fwhm_value=3.5/arcsecpix
if (fitsutils.has_par(image, 'AIRMASS')):
airmass_value = fitsutils.get_par(image, 'AIRMASS')
else:
airmass_value = 1.3
exptime = fitsutils.get_par(image, 'EXPTIME')
gain = fitsutils.get_par(image, 'GAIN')
noise = fitsutils.get_par(image, 'RDNOISE')
print "FWHM: %.1f pixels, %.1f arcsec"%(fwhm_value, fwhm_value*arcsecpix)
aperture_rad = math.ceil(float(fwhm_value)*1.5) # Set aperture radius to three times the PSF radius
sky_rad= math.ceil(aperture_rad)*4
if (not plot_only):
iraf.noao.digiphot.apphot.qphot(image = image,\
cbox = box ,\
annulus = sky_rad ,\
dannulus = 20. ,\
aperture = str(aperture_rad),\
coords = coords ,\
output = out_name ,\
plotfile = "" ,\
zmag = 0. ,\
exposure = "exptime",\
airmass = "airmass" ,\
filter = "filter" ,\
obstime = "DATE" ,\
#fwhm = fwhm_value,\
epadu = gain ,\
interactive = "no" ,\
radplots = "yes" ,\
verbose = "no" ,\
graphics = "stdgraph" ,\
display = "stdimage" ,\
icommands = "" ,\
wcsin = wcsin,
wcsout = "logical",
gcommands = "")
#iraf.noao.digiphot.apphot.phot(image=image, cbox=5., annulus=12.4, dannulus=10., salgori = "centroid", aperture=9.3,wcsin="world",wcsout="tv", interac = "no", coords=coords, output=out_name)
iraf.txdump(out_name, "id,image,xcenter,ycenter,xshift,yshift,fwhm,msky,stdev,cier,rapert,sum,area,nsky,flux,itime,mag,merr", "yes", Stdout=clean_name)
ma = np.genfromtxt(clean_name, comments="#", dtype=[("id","<f4"), ("image","|S20"), ("X","<f4"), ("Y","<f4"), ("Xshift","<f4"), ("Yshift","<f4"),("fwhm","<f4"), ("msky","<f4"), ("stdev","<f4"),\
("flags", np.int), ("rapert", "<f4"), ("sum", "<f4"), ("area", "<f4"), ("nsky","<f4") , ("flux", "<f4"), ("itime", "<f4"), ("fit_mag","<f4"), ("fiterr","<f4")])
if (ma.size > 0):
m = ma[~np.isnan(ma["fit_mag"])]
else:
print "Only one object found!"
m = np.array([ma])
hdulist = pf.open(image)
prihdr = hdulist[0].header
img = hdulist[0].data * 1.
nx, ny = img.shape
dimX = int(np.floor(np.sqrt(len(m))))
dimY = int(np.ceil(len(m)*1./dimX))
outerrad = sky_rad+10
cutrad = outerrad + 15
print "Cutrad %.1f"%cutrad
plt.suptitle("FWHM=%.2f arcsec. %d stars"%(fwhm_value*arcsecpix, len(m)))
for i in np.arange(dimX):
for j in np.arange(dimY):
if ( i*dimY + j < len(m)):
k = i*dimY + j
#print dimX, dimY, i, j, k
ax = plt.subplot2grid((dimX,dimY),(i, j))
y1, y2, x1, x2 = m[k]["X"]-cutrad, m[k]["X"]+cutrad, m[k]["Y"]-cutrad, m[k]["Y"]+cutrad
y1, y2, x1, x2 = int(y1), int(y2), int(x1), int(x2)
y1 = np.maximum(y1, 0); y2=np.maximum(y2, 0); x1=np.maximum(x1, 0); x2 = np.maximum(x2, 0)
try:
zmin, zmax = zscale.zscale(img[x1:x2,y1:y2], nsamples=1000, contrast=0.25)
except ValueError:
print y1, y2, x1, x2
print img[x1:x2,y1:y2]
sh= img[x1:x2,y1:y2].shape
if sh[0]>0 and sh[1]>0:
zmin = np.nanmin(img[x1:x2,y1:y2])
zmax = np.nanmax(img[x1:x2,y1:y2])
ax.imshow(img[x1:x2,y1:y2], aspect="equal", extent=(-cutrad, cutrad, -cutrad, cutrad), origin="lower", cmap=matplotlib.cm.gray_r, interpolation="none", vmin=zmin, vmax=zmax)
c1 = plt.Circle( (0, 0), edgecolor="r", facecolor="none", radius=aperture_rad)
c2 = plt.Circle( (0, 0), edgecolor="orange", facecolor="none", radius=sky_rad)
c3 = plt.Circle( (0, 0), edgecolor="yellow", facecolor="none", radius=sky_rad+20)
plt.gca().add_artist(c1)
plt.gca().add_artist(c2)
plt.gca().add_artist(c3)
ax.set_xticks([])
ax.set_yticks([])
plt.text(+5, +5, "%d"%m[k]["id"])
plt.text(-cutrad, -cutrad, "%.2f$\pm$%.2f"%(m[k]["fit_mag"], m[k]["fiterr"]), color="b")
plt.tight_layout()
plt.savefig(os.path.join(plotdir, imname + "plot.png"))
plt.clf()
def calibrate_zeropoint(image, plot=True, plotdir=None, debug=False, refstars=None):
"""
Calibrates the zeropoint using SDSS catalogue.
"""
if plot and plotdir is None:
plotdir = os.path.join(os.path.dirname(image), "photometry")
if not os.path.isdir(plotdir):
os.makedirs(plotdir)
filt = fitsutils.get_par(image, "filter")
exptime = fitsutils.get_par(image, "exptime")
if fitsutils.has_par(image, "JD"):
date = fitsutils.get_par(image, "JD")
elif fitsutils.has_par(image, "MJD"):
date = fitsutils.get_par(image, "MJD")
elif fitsutils.has_par(image, "MJD-OBS"):
date = fitsutils.get_par(image, "MJD-OBS")
else:
date = 0
if fitsutils.has_par(image, "AIRMASS"):
airmass = fitsutils.get_par(image, "AIRMASS")
else:
airmass = 1.3
objname = fitsutils.get_par(image, "OBJECT")
band = fitsutils.get_par(image, "FILTER")
logger.info("Starting calibration of ZP for image %s for object %s with filter %s." % (image, objname, band))
if exptime < 10:
logger.error("ERROR. Exposure time too short for image (%s) to see anything..." % image)
return
extracted = extract_star_sequence(
os.path.abspath(image), filt, plot=plot, survey="sdss", debug=debug, refstars=refstars, plotdir=plotdir
)
if not extracted:
logger.warn(
"Field not in SDSS or error when retrieving the catalogue... Skipping. Image %s not zeropoint calibrated."
% image
)
# Add these values to the header.
pardic = {"IQZEROPT": 0, "ZPCAT": "None", "ZEROPTU": 0, "ZEROPT": 0, "ZP": 0, "ZPERR": 0}
fitsutils.update_pars(image, pardic)
return
# If extraction worked, we can get the FWHM
fwhm = fitsutils.get_par(image, "fwhm")
fwhm_as = fwhm * 0.394
app_phot.get_app_phot("/tmp/sdss_cat_det_%s.txt" % creationdate, image, wcsin="logic", plotdir=plotdir, box=20)
# Compute the zeropoint for the specific image.
z, c, err = find_zeropoint_noid("/tmp/sdss_cat_det_%s.txt" % creationdate, image, plot=plot, plotdir=plotdir)
# Add these values to the header.
pardic = {"IQZEROPT": 1, "ZPCAT": "SDSS", "ZEROPTU": np.round(err, 3), "ZEROPT": np.round(z, 3)}
fitsutils.update_pars(image, pardic)
# Log the current zeropoint for this image
logname = os.path.join(os.path.dirname(image), "zeropoint.log")
# Add the data to a later stage zeropoint calibrtion with all-sky data.
zplogname = os.path.join(os.path.dirname(image), "allstars_zp.log")
add_to_zp_cal("/tmp/sdss_cat_det_%s.txt" % creationdate, image, zplogname)
if not os.path.isfile(logname):
with open(logname, "a") as f:
f.write("#filename,exptime,filter,date,airmass,fwhm_pix,fwhm_as,zeropoint,color,err\n")
with open(logname, "a") as f:
f.write(
"%s,%.1f,%s,%3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f\n"
% (image, exptime, filt, date, airmass, fwhm, fwhm_as, z, c, err)
)
clean_tmp_files()
def create_masterflat(flatdir=None, biasdir=None, channel='rc'):
'''
Creates a masterflat from both dome flats and sky flats if the number of counts in the given filter
is not saturated and not too low (between 1500 and 40000).
'''
if (flatdir == None or flatdir==""): flatdir = "."
if (biasdir == None or biasdir==""): biasdir = "."
os.chdir(flatdir)
if (len(glob.glob("Flat_%s*norm.fits"%channel)) == 4):
print "Master Flat exists!"
return
else:
print "Starting the Master Flat creation!"
bias_slow = "Bias_%s_fast.fits"%channel
bias_fast = "Bias_%s_fast.fits"%channel
if (not os.path.isfile(bias_slow) and not os.path.isfile(bias_fast) ):
create_masterbias(biasdir)
lsflat = []
lfflat = []
#Select all filts that are Flats with same instrument
for f in glob.glob("*fits"):
#try:
if fitsutils.has_par(f, "OBJECT"):
obj = str.upper(fitsutils.get_par(f, "OBJECT"))
else:
continue
if ( ("DOME" in obj or "FLAT" in obj) and (channel == fitsutils.get_par(f, "CHANNEL"))):
if (fitsutils.get_par(f, "ADCSPEED")==2):
lfflat.append(f)
else:
lsflat.append(f)
#except:
# print "Error with retrieving parameters for file", f
# pass
print "Files for slow flat", lsflat
print "Files for fast flat", lfflat
fsfile ="lflat_slow_"+channel
np.savetxt(fsfile, np.array(lsflat), fmt="%s")
fffile ="lflat_fast_"+channel
np.savetxt(fffile, np.array(lfflat), fmt="%s")
# Running IRAF
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Remove bias from the flat
if len(lsflat) >0:
iraf.imarith("@"+fsfile, "-", bias_slow, "b_@"+fsfile)
if len(lfflat) >0:
iraf.imarith("@"+fffile, "-", bias_fast, "b_@"+fffile)
#Slices the flats.
debiased_flats = glob.glob("b_*.fits")
for f in debiased_flats:
print "Slicing file", f
slice_rc(f)
#Remove the un-sliced file
os.remove(f)
#Selects the ones that are suitable given the number of counts and combines them.
bands = ['u', 'g', 'r', 'i']
for b in bands:
out = "Flat_%s_%s.fits"%(channel, b)
out_norm = out.replace(".fits","_norm.fits")
if (os.path.isfile(out_norm)):
print "Master Flat for filter %s exists. Skipping..."%b
continue
lfiles = []
for f in glob.glob('b_*_%s.fits'%b):
d = pf.open(f)[0].data
if np.percentile(d, 90)>1500 and np.percentile(d, 90)<40000:
lfiles.append(f)
if len(lfiles) == 0:
print "WARNING!!! Could not find suitable flats for band %s"%b
continue
ffile ="lflat_"+b
np.savetxt(ffile, np.array(lfiles), fmt="%s")
#Cleaning of old files
if(os.path.isfile(out)): os.remove(out)
if(os.path.isfile(out_norm)): os.remove(out_norm)
if(os.path.isfile("Flat_stats")): os.remove("Flat_stats")
#Combine flats
iraf.imcombine(input = "@"+ffile, \
output = out, \
combine = "median",\
scale = "mode",
weight = "exposure")
iraf.imstat(out, fields="image,npix,mean,stddev,min,max,mode", Stdout="Flat_stats")
st = np.genfromtxt("Flat_stats", names=True, dtype=None)
#Normalize flats
iraf.imarith(out, "/", st["MODE"], out_norm)
#Do some cleaning
print 'Removing from lfiles'
for f in glob.glob('b_*_%s.fits'%b):
os.remove(f)
os.remove(ffile)
if os.path.isfile(fsfile):
os.remove(fsfile)
if os.path.isfile(fffile):
os.remove(fffile)
def get_app_phot(coords, image, plot_only=False, store=True, wcsin="world", fwhm=2, plotdir=".", box=15):
'''
coords: files:
wcsin: can be "world", "logic"
'''
# Load packages; splot is in the onedspec package, which is in noao.
# The special keyword _doprint=0 turns off displaying the tasks
# when loading a package.
if (not plot_only):
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.apphot(_doprint=0)
iraf.unlearn("apphot")
imdir = os.path.dirname(image)
imname = os.path.basename(image)
plotdir = os.path.join(imdir, "photometry")
if not os.path.isdir(plotdir):
os.makedirs(plotdir)
out_name = os.path.join(plotdir, imname + ".seq.mag")
clean_name = os.path.join(plotdir, imname + ".app.mag")
# Read values from .ec file
ecfile= image+".ec"
filter_value=''.join(ecfile).split('.',1)[0]
fwhm_value = fwhm
if (fitsutils.has_par(image, 'FWHM')):
fwhm_value = fitsutils.get_par(image, 'FWHM')
if (fitsutils.has_par(image, 'AIRMASS')):
airmass_value = fitsutils.get_par(image, 'AIRMASS')
else:
airmass_value = 1.3
exptime = fitsutils.get_par(image, 'EXPTIME')
gain = fitsutils.get_par(image, 'GAIN')
try:
with open(''.join(ecfile),'r') as f:
for line in f:
if "airmass" in line:
airmass_value = line.split('=',1)[1]
else:
airmass_value = 1
if "FWHM" in line:
print line
fwhm_value = line.split('FWHM=',1)[1]
fwhm_value = fwhm_value.rsplit("aperture")[0]
except:
pass
print "FWHM", fwhm_value
aperture_rad = math.ceil(float(fwhm_value)*2) # Set aperture radius to three times the PSF radius
sky_rad= math.ceil(aperture_rad)*5
print aperture_rad, sky_rad
if (not plot_only):
if os.path.isfile(out_name): os.remove(out_name)
if os.path.isfile(clean_name): os.remove(clean_name)
# Check if files in list, otherwise exit
if not ecfile:
print "No .ec files in directory, exiting"
sys.exit()
iraf.noao.digiphot.apphot.qphot(image = image,\
cbox = box ,\
annulus = sky_rad ,\
dannulus = 15. ,\
aperture = str(aperture_rad),\
coords = coords ,\
output = out_name ,\
plotfile = "" ,\
zmag = 0. ,\
exposure = "exptime" ,\
airmass = "airmass" ,\
filter = "filters" ,\
obstime = "DATE" ,\
epadu = gain ,\
interactive = "no" ,\
radplots = "yes" ,\
verbose = "no" ,\
graphics = "stdgraph" ,\
display = "stdimage" ,\
icommands = "" ,\
wcsin = wcsin,
wcsout = "logical",
gcommands = "")
#iraf.noao.digiphot.apphot.phot(image=image, cbox=5., annulus=12.4, dannulus=10., salgori = "centroid", aperture=9.3,wcsin="world",wcsout="tv", interac = "no", coords=coords, output=out_name)
iraf.txdump(out_name, "id,image,xcenter,ycenter,xshift,yshift,fwhm,msky,stdev,mag,merr", "yes", Stdout=clean_name)
ma = np.genfromtxt(clean_name, comments="#", dtype=[("id","<f4"), ("image","|S20"), ("X","<f4"), ("Y","<f4"), ("Xshift","<f4"), ("Yshift","<f4"),("fwhm","<f4"), ("ph_mag","<f4"), ("stdev","<f4"), ("fit_mag","<f4"), ("fiterr","<f4")])
if (ma.size > 0):
m = ma[~np.isnan(ma["fit_mag"])]
else:
print "Only one object found!"
m = np.array([ma])
hdulist = pf.open(image)
prihdr = hdulist[0].header
img = hdulist[0].data * 1.
nx, ny = img.shape
dimX = int(4)
dimY = int(np.ceil(len(m)*1./4))
outerrad = sky_rad+10
cutrad = outerrad + 15
plt.suptitle("FWHM="+str(fwhm_value))
k = 0
for i in np.arange(dimX):
for j in np.arange(dimY):
if ( k < len(m)):
ax = plt.subplot2grid((dimX,dimY),(i, j))
y1, y2, x1, x2 = m[k]["X"]-cutrad, m[k]["X"]+cutrad, m[k]["Y"]-cutrad, m[k]["Y"]+cutrad
y1, y2, x1, x2 = int(y1), int(y2), int(x1), int(x2)
try:
zmin, zmax = zscale.zscale(img[x1:x2,y1:y2], nsamples=1000, contrast=0.25)
except:
sh= img[x1:x2,y1:y2].shape
if sh[0]>0 and sh[1]>0:
zmin = np.nanmin(img[x1:x2,y1:y2])
zmax = np.nanmax(img[x1:x2,y1:y2])
continue
else:
continue
ax.imshow(img[x1:x2,y1:y2], aspect="equal", extent=(-cutrad, cutrad, -cutrad, cutrad), origin="lower", cmap=matplotlib.cm.gray_r, interpolation="none", vmin=zmin, vmax=zmax)
c1 = plt.Circle( (0, 0), edgecolor="r", facecolor="none", radius=5.)
c2 = plt.Circle( (0, 0), edgecolor="orange", facecolor="none", radius=sky_rad)
c3 = plt.Circle( (0, 0), edgecolor="yellow", facecolor="none", radius=sky_rad+10)
plt.gca().add_artist(c1)
plt.gca().add_artist(c2)
plt.gca().add_artist(c3)
ax.set_xticks([])
ax.set_yticks([])
plt.text(+5, +5, "%d"%m[k]["id"])
plt.text(-cutrad, -cutrad, "%.2f$\pm$%.2f"%(m[k]["fit_mag"], m[k]["fiterr"]), color="b")
k = k+1
plt.savefig(os.path.join(plotdir, imname + "plot.png"))
plt.clf()
def get_flats_counts(directory):
flatlist = []
corners = {
"g" : [1, 910, 1, 900],
"i" : [1, 910, 1060, 2045],
"r" : [1040, 2045, 1015, 2045],
"u" : [1030, 2045, 1, 900]
}
for f in glob.glob(directory + "/rc*fits"):
if fitsutils.has_par(f, "IMGTYPE") and fitsutils.get_par(f, "IMGTYPE") == "TWILIGHT":
flatlist.append(f)
if (len(flatlist)==0):
print "No suitable twilight flats found in directory: %s"%directory
return
counts = {"u":[], "g":[], "r":[], "i":[]}
sun_decs = {"u":[], "g":[], "r":[], "i":[]}
for f in flatlist:
print f
print fitsutils.get_par(f, "JD")
bias = rcred.get_overscan_bias_rc(f)
exptime = fitsutils.get_par(f, "EXPTIME")
sunsettime = fitsutils.get_par(f, "SUNSET")
utc = datetime.datetime.strptime(time_utils.jd2utc(fitsutils.get_par(f, "JD")), "%Y-%m-%d %H:%M:%S.%f")
st = datetime.datetime.strptime(sunsettime, "%H:%M")
elapsed = 3600*(utc.hour - st.hour) + 60*(utc.minute - st.minute) + (utc.second - st.second)
if (elapsed > 3000):
continue
print elapsed, utc, st
data = pf.open(f)[0].data
for band in corners.keys():
c = corners[band]
sf = data[c[0]:c[1], c[2]:c[3]]
if (np.percentile(sf, 90) < 55000):
counts[band].append( (np.percentile(sf, 90)-bias)/exptime)
sun_decs[band].append(elapsed)
for band in corners.keys():
coefs = np.polyfit(sun_decs[band], np.log10(counts[band]), deg=1, w=1./np.sqrt(np.log10(counts[band])))
p = np.poly1d(coefs)
x = np.linspace(np.min(sun_decs[band]), np.max(sun_decs[band]), 1000)
plt.plot(sun_decs[band], np.log10(counts[band]), "o", label=band)
plt.plot(x, p(x), label="Model "+band)
plt.xlabel("Elapsed second since Sunset")
plt.ylabel("Counts/s")
plt.legend()
plt.show()
Runs astrometry.net on the image specified as a parameter and returns
the offset needed to be applied in order to center the object coordinates
in the reference pixel.
''', formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-d', '--reddir', type=str, dest="reduced", help='Fits directory file with reduced images.', default=None)
args = parser.parse_args()
reduced = args.reduced
if (reduced is None):
timestamp=datetime.datetime.isoformat(datetime.datetime.utcnow())
timestamp = timestamp.split("T")[0].replace("-","")
reduced = os.path.join("/scr2/sedm/phot/", timestamp, "reduced")
os.chdir(reduced)
for f in glob.glob("*.fits"):
if(fitsutils.has_par(f, "IMGTYPE") and fitsutils.get_par(f, "IMGTYPE") == "SCIENCE" or fitsutils.get_par(f, "IMGTYPE") == "ACQUISITION"):
#print f
get_app_phot_target(f, box=5)
cmd = 'echo "FILE ONTARGET NAME FILTER JD LST APPMAG APPMAGER INSMAG INSMAGER ZEROPT ZEROPTU" > photometry/magnitudes.dat; gethead ONTARGET NAME FILTER JD LST APPMAG APPMAGER INSMAG INSMAGER ZEROPT ZEROPTU *fits | grep -E "fits[\ ]+1" >> photometry/magnitudes.dat'
subprocess.call(cmd, shell=True)
def get_app_phot_target(image, plot=False, store=True, wcsin="logical", fwhm=2, box=4, ra=None, dec=None):
'''
coords: files:
wcsin: can be "world", "logic"
'''
# Load packages; splot is in the onedspec package, which is in noao.
# The special keyword _doprint=0 turns off displaying the tasks
# when loading a package.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
fxn()
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.apphot(_doprint=0)
iraf.unlearn("apphot")
impf = pf.open(image)
wcs = pywcs.WCS(impf[0].header)
#Check that actually the object is within this frame.
if (ra is None or dec is None):
if (fitsutils.has_par(image, "OBJRA") and fitsutils.has_par(image, "OBJRA")):
ra, dec = cc.hour2deg(fitsutils.get_par(image, 'OBJRA'), fitsutils.get_par(image, 'OBJDEC'))
else:
ra, dec = cc.hour2deg(fitsutils.get_par(image, 'RA'), fitsutils.get_par(image, 'DEC'))
pra, pdec = get_xy_coords(image, ra, dec)
else:
if(wcsin == "logical"):
pra, pdec = ra, dec
else:
#Using new method to derive the X, Y pixel coordinates, as pywcs does not seem to be working well.
pra, pdec = get_xy_coords(image, ra, dec)
#pra, pdec = wcs.wcs_sky2pix(ra, dec, 1)
#pra, pdec = wcs.wcs_sky2pix(np.array([ra, dec], ndmin=2), 1)[0]
shape = impf[0].data.shape
if (pra > 0) and (pra < shape[0]) and (pdec > 0) and (pdec < shape[1]):
pass
else:
print image, "ERROR! Object coordinates are outside this frame. Skipping any aperture photometry!!"
print pra, pdec, shape
return
imdir = os.path.dirname(image)
imname = os.path.basename(image)
plotdir = os.path.join(imdir, "photometry")
if not os.path.isdir(plotdir):
os.makedirs(plotdir)
out_name = os.path.join(plotdir, imname + ".seq.mag")
clean_name = os.path.join(plotdir, imname + ".objapp.mag")
fwhm_value = fwhm
nsrc, fwhm_value, ellip = sextractor.get_image_pars(image)
if np.isnan(fwhm_value):
fwhm_value=99
fitsutils.update_par(image, 'FWHM', fwhm_value)
if (fitsutils.has_par(image, 'AIRMASS')):
airmass_value = fitsutils.get_par(image, 'AIRMASS')
else:
airmass_value = 1.3
exptime = fitsutils.get_par(image, 'EXPTIME')
gain = fitsutils.get_par(image, 'GAIN')
#print "FWHM", fwhm_value
aperture_rad = math.ceil(float(fwhm_value)*3) # Set aperture radius to three times the PSF radius
sky_rad= math.ceil(aperture_rad*4)
#print aperture_rad, sky_rad
print "Saving coodinates for the object in pixels",pra,pdec
coords = "/tmp/coords.dat"
np.savetxt("/tmp/coords.dat", np.array([[pra, pdec]]), fmt="%.4f %.4f")
if (plot):
zmin, zmax = zscale.zscale(impf[0].data)
im = plt.imshow(impf[0].data, vmin=zmin, vmax=zmax, origin="bottom")
plt.scatter(pra, pdec, marker="o", s=100, facecolor="none")
plt.savefig(os.path.join(plotdir, imname+".png"))
plt.clf()
if os.path.isfile(out_name): os.remove(out_name)
if os.path.isfile(clean_name): os.remove(clean_name)
iraf.noao.digiphot.apphot.qphot(image = image,\
cbox = box ,\
annulus = sky_rad ,\
dannulus = 15. ,\
aperture = str(aperture_rad),\
coords = coords ,\
output = out_name ,\
plotfile = "" ,\
zmag = 0. ,\
exposure = "exptime" ,\
airmass = "airmass" ,\
filter = "filter" ,\
obstime = "DATE" ,\
epadu = gain ,\
interactive = "no" ,\
radplots = "yes" ,\
verbose = "no" ,\
graphics = "stdgraph" ,\
display = "stdimage" ,\
icommands = "" ,\
wcsin = "logical",
wcsout = "logical",
gcommands = "")
#iraf.noao.digiphot.apphot.phot(image=image, cbox=5., annulus=12.4, dannulus=10., salgori = "centroid", aperture=9.3,wcsin="world",wcsout="tv", interac = "no", coords=coords, output=out_name)
iraf.txdump(out_name, "id,image,xcenter,ycenter,xshift,yshift,fwhm,msky,stdev,mag,merr", "yes", Stdout=clean_name)
ma = np.genfromtxt(clean_name, comments="#", dtype=[("id","<f4"), ("image","|S20"), ("X","<f4"), ("Y","<f4"), ("Xshift","<f4"), ("Yshift","<f4"),("fwhm","<f4"), ("ph_mag","<f4"), ("stdev","<f4"), ("fit_mag","<f4"), ("fiterr","<f4")])
if (ma.size > 0):
if (ma.size==1):
ma = np.array([ma])
m = ma[~np.isnan(ma["fit_mag"])]
else:
print "Only one object found!"
m = np.array([ma])
insmag = np.round(ma['fit_mag'][0] , 3)
insmagerr = np.round(ma['fiterr'][0], 3)
if (fitsutils.has_par(image, "ZEROPT")):
mag = insmag + float(fitsutils.get_par(image, "ZEROPT"))
magerr = np.sqrt(insmagerr**2+ float(fitsutils.get_par(image, "ZEROPTU"))**2)
if np.isnan(mag):
mag, magerr = 0, 0
insmag, insmagerr = 0,0
fitsutils.update_par(image, "INSMAG", "%.3f"%insmag )
fitsutils.update_par(image, "INSMAGER", "%.3f"%insmagerr)
fitsutils.update_par(image, "APPMAG", np.round(mag, 3) )
fitsutils.update_par(image, "APPMAGER", np.round(magerr, 3))
if (plot):
X = int(ma["X"][0])
Y = int(ma["Y"][0])
pra = int(pra)
pdec = int(pdec)
plt.scatter(X, Y, marker="o", s=100, facecolor="none", edgecolor="red")
plt.colorbar(im)
plt.savefig(os.path.join(plotdir, imname+".png"))
plt.clf()
zmin, zmax = zscale.zscale(impf[0].data.T[X-50:X+50,Y-50:Y+50].T)
im = plt.imshow(impf[0].data.T[pra-50:pra+50,pdec-50:pdec+50].T, vmin=zmin, vmax=zmax, interpolation="none", origin="bottom", extent=(-50,50,-50,50))
c1 = plt.Circle( (pra-X, pdec-Y), edgecolor="k", facecolor="none", radius=aperture_rad, label="Initial position")
c11 = plt.Circle( (pra-X, pdec-Y), edgecolor="k", facecolor="none", radius=sky_rad)
c2 = plt.Circle( (0, 0), edgecolor="orange", facecolor="none", radius=aperture_rad, label="Adjusted centroid")
c22 = plt.Circle( (0, 0), edgecolor="orange", facecolor="none", radius=sky_rad)
plt.gca().add_artist(c1)
plt.gca().add_artist(c11)
plt.gca().add_artist(c2)
plt.gca().add_artist(c22)
plt.colorbar(im)
myhandles = []
markers = ["o", "o"]
labels = ["Initial position", "Adjusted centroid"]
cols = ["k", "orange"]
for i in np.arange(len(markers)):
myhandles.append(mlines.Line2D([], [], mec=cols[i], mfc="none", marker=markers[i], ls="None", markersize=10, label=labels[i]))
plt.legend(handles=myhandles, loc="lower left", labelspacing=0.3, fontsize=11, numpoints=1, frameon=False, ncol=5, bbox_to_anchor=(0.0, 0.00), fancybox=False, shadow=True)
plt.title("MIN: %.0f MAX: %.0f"%(np.nanmin(impf[0].data.T[X-50:X+50,Y-50:Y+50]), np.nanmax(impf[0].data.T[X-50:X+50,Y-50:Y+50])))
plt.savefig(os.path.join(plotdir, imname+"_zoom.png"))
plt.clf()
#Get the files from the same day directory.
files = glob.glob(os.path.join(raw, "ifu*fits"))
#files_hg = [f for f in files if "Calib: Hg" in get_par(f, "OBJECT")]
#Get the files from one day before directory to see the differences between days.
daybefore = datetime.datetime.isoformat(datetime.datetime.utcnow() -
datetime.timedelta(1))
daybefore = daybefore.split("T")[0].replace("-", "")
daybefore = os.path.join("/scr2/sedm/phot/", daybefore)
if (os.path.isdir(daybefore)):
filesold = glob.glob(os.path.join(daybefore, "ifu*fits"))
files.extend(filesold)
files_hg = [
f for f in files if fitsutils.has_par(f, "OBJECT")
and "Calib: Hg" in fitsutils.get_par(f, "OBJECT")
]
logger.info("Found the following Hg files: %s" % files_hg)
if (len(files_hg) > 1):
files_hg.sort()
sexfiles = sextractor.run_sex(files_hg, mask=False)
plotdir = os.path.join(raw, "stats")
if (not os.path.isdir(plotdir)):
os.makedirs(plotdir)
run_flexure_test(sexfiles, plotdir=plotdir)
def get_flats_counts(directory):
'''
Reads all the images in a directory marked as "twilight" and uses them to infer the average count rate for a given
number of seconds after the sunset.
It plots the number of counts and fits a 2D polynomial used to interpolate in the future.
'''
flatlist = []
corners = {
"g" : [1, 910, 1, 900],
"i" : [1, 910, 1060, 2045],
"r" : [1040, 2045, 1015, 2045],
"u" : [1030, 2045, 1, 900]
}
for f in glob.glob(directory + "/rc*fits"):
if fitsutils.has_par(f, "IMGTYPE") and fitsutils.get_par(f, "IMGTYPE") == "TWILIGHT":
flatlist.append(f)
if (len(flatlist)==0):
print "No suitable twilight flats found in directory: %s"%directory
return
counts = {"u":[], "g":[], "r":[], "i":[]}
sun_decs = {"u":[], "g":[], "r":[], "i":[]}
colors = {"u":"purple", "g":"green", "r":"r", "i":"orange"}
for f in flatlist:
print f
print fitsutils.get_par(f, "JD")
bias = rcred.get_overscan_bias_rc(f)
exptime = fitsutils.get_par(f, "EXPTIME")
sunsettime = fitsutils.get_par(f, "SUNSET")
utc = time_utils.jd2utc(fitsutils.get_par(f, "JD"))
st = datetime.datetime.strptime(sunsettime, "%H:%M")
elapsed = 3600*(utc.hour - st.hour) + 60*(utc.minute - st.minute) + (utc.second - st.second)
if (elapsed > 5000):
continue
print elapsed, utc, st
data = pf.open(f)[0].data
for band in corners.keys():
c = corners[band]
sf = data.T[c[0]:c[1], c[2]:c[3]]
if (np.percentile(sf, 90) < 55000):
counts[band].append( (np.percentile(sf, 90)-bias)/exptime)
sun_decs[band].append(elapsed)
t = Table(names=('filter', 'c2', 'c1', 'c0'), dtype=('S1', 'f8', 'f8', 'f8'))
for band in corners.keys():
print sun_decs[band], np.log10(counts[band])
coefs = np.polyfit(sun_decs[band], np.log10(counts[band]), deg=2)#, w=1./np.sqrt(np.log10(counts[band])))
p = np.poly1d(coefs)
x = np.linspace(np.min(sun_decs[band]), np.max(sun_decs[band]), 1000)
t.add_row([band, coefs[0], coefs[1], coefs[2]])
plt.plot(sun_decs[band], np.log10(counts[band]), "o", label=band, color=colors[band])
plt.plot(x, p(x), label="Model "+band, color=colors[band])
t.write("/tmp/test_flat", format='csv')
plt.xlabel("Elapsed second since Sunset")
plt.ylabel("log Counts/s")
plt.legend()
plt.show()
def get_app_phot_target(image, ra=None, dec=None, plot=True, store=True, wcsin="logical", fwhm=None, box=15, arcsecpix=0.394, app=2):
'''
coords: files:
wcsin: can be "world", "logic"
fwhm: in arcsec
'''
# Load packages; splot is in the onedspec package, which is in noao.
# The special keyword _doprint=0 turns off displaying the tasks
# when loading a package.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
fxn()
iraf.noao(_doprint=0)
iraf.digiphot(_doprint=0)
iraf.apphot(_doprint=0)
iraf.unlearn("apphot")
impf = pf.open(image)
wcs = WCS(impf[0].header)
#Check that actually the object is within this frame.
if (ra is None or dec is None):
if (fitsutils.has_par(image, "OBJRA") and fitsutils.has_par(image, "OBJRA")):
ra, dec = cc.hour2deg(fitsutils.get_par(image, 'OBJRA'), fitsutils.get_par(image, 'OBJDEC'))
else:
ra, dec = cc.hour2deg(fitsutils.get_par(image, 'RA'), fitsutils.get_par(image, 'DEC'))
print "Assuming ra=%.5f, dec=%.5f"%(ra, dec)
pra, pdec = get_xy_coords(image, ra, dec)
else:
if("logic" in wcsin):
pra, pdec = ra, dec
else:
#Using new method to derive the X, Y pixel coordinates, as wcs module does not seem to be working well.
try:
#pra, pdec = wcs.wcs_sky2pix(ra, dec, 1)
#print "Retrieved the pixel number"
pra, pdec = get_xy_coords(image, ra, dec)
except IndexError:
print "Error with astrometry.net. trying the rudimentary method."
pra, pdec = wcs.wcs_sky2pix(ra, dec, 1)
#pra, pdec = wcs.wcs_sky2pix(np.array([ra, dec], ndmin=2), 1)[0]
shape = impf[0].data.shape
if (pra > 0) and (pra < shape[0]) and (pdec > 0) and (pdec < shape[1]):
pass
else:
print image, "ERROR! Object coordinates are outside this frame. Skipping any aperture photometry!!"
print pra, pdec, shape
return
imdir = os.path.dirname(image)
imname = os.path.basename(image)
plotdir = os.path.join(imdir, "photometry")
if not os.path.isdir(plotdir):
os.makedirs(plotdir)
out_name = os.path.join(plotdir, imname + ".seq.mag")
clean_name = os.path.join(plotdir, imname + ".objapp.mag")
if (not fwhm is None):
fwhm_value = fwhm
elif (fitsutils.has_par(image, 'FWHM')):
fwhm_value = fitsutils.get_par(image, 'FWHM')
else:
#Put some default value for Palomar
fwhm_value=1.5
if (wcsin == 'logical'):
fwhm_value = fwhm_value / arcsecpix
if (fitsutils.has_par(image, 'AIRMASS')):
airmass_value = fitsutils.get_par(image, 'AIRMASS')
else:
airmass_value = 1.3
if (not fitsutils.has_par(image, "EXPTIME")):
if (fitsutils.has_par(image, "ITIME") and fitsutils.has_par(image, "COADDS")):
exptime = fitsutils.get_par(image, "ITIME")*fitsutils.get_par(image, "COADDS")
fitsutils.update_par(image, "EXPTIME", exptime)
exptime = fitsutils.get_par(image, 'EXPTIME')
gain = fitsutils.get_par(image, 'GAIN')
#print "FWHM", fwhm_value
aperture_rad = math.ceil(float(fwhm_value)*app) # Set aperture radius to two times the PSF radius
sky_rad= math.ceil(aperture_rad*app*2)
#print aperture_rad, sky_rad
print "Saving coodinates for the object in pixels",pra,pdec
coords = "/tmp/coords.dat"
np.savetxt("/tmp/coords.dat", np.array([[pra, pdec]]), fmt="%.4f %.4f")
if os.path.isfile(out_name): os.remove(out_name)
if os.path.isfile(clean_name): os.remove(clean_name)
iraf.noao.digiphot.apphot.qphot(image = image,\
cbox = box ,\
annulus = sky_rad ,\
dannulus = 20. ,\
aperture = str(aperture_rad),\
coords = coords ,\
output = out_name ,\
plotfile = "" ,\
zmag = 0. ,\
exposure = "exptime" ,\
airmass = "airmass" ,\
filter = "filter" ,\
obstime = "DATE" ,\
epadu = gain ,\
interactive = "no" ,\
radplots = "yes" ,\
verbose = "no" ,\
graphics = "stdgraph" ,\
display = "stdimage" ,\
icommands = "" ,\
wcsin = "logical",
wcsout = "logical",
gcommands = "")
#iraf.noao.digiphot.apphot.phot(image=image, cbox=5., annulus=12.4, dannulus=10., salgori = "centroid", aperture=9.3,wcsin="world",wcsout="tv", interac = "no", coords=coords, output=out_name)
iraf.txdump(out_name, "id,image,xcenter,ycenter,xshift,yshift,fwhm,msky,stdev,cier,rapert,sum,area,nsky,flux,itime,mag,merr", "yes", Stdout=clean_name)
ma = np.genfromtxt(clean_name, comments="#", dtype=[("id","<f4"), ("image","|S20"), ("X","<f4"), ("Y","<f4"), ("Xshift","<f4"), ("Yshift","<f4"),("fwhm","<f4"), ("msky","<f4"), \
("stdev","<f4"), ("flags", np.int), ("rapert", "<f4"), ("sum", "<f4"), ("area", "<f4"), ("nsky","<f4") , ("flux", "<f4"), ("itime", "<f4"), ("fit_mag","<f4"), ("fiterr","<f4")])
if (ma.size > 0):
ma = np.array([ma])
m = ma[~np.isnan(ma["fit_mag"])]
else:
print "Only one object found!"
m = np.array([ma])
insmag = np.round(ma['fit_mag'][0] , 3)
insmagerr = np.round(ma['fiterr'][0], 3)
if (fitsutils.has_par(image, "ZEROPT") and fitsutils.has_par(image, "ZEROPTU")):
mag = insmag + float(fitsutils.get_par(image, "ZEROPT"))
magerr = np.sqrt(insmagerr**2+ float(fitsutils.get_par(image, "ZEROPTU"))**2)
else:
mag = 0
magerr = 0
if np.isnan(mag):
mag, magerr = 0, 0
insmag, insmagerr = 0,0
fitsutils.update_par(image, "INSMAG", "%.3f"%insmag )
fitsutils.update_par(image, "INSMAGER", "%.3f"%insmagerr)
fitsutils.update_par(image, "APPMAG", np.round(mag, 3) )
fitsutils.update_par(image, "APPMAGER", np.round(magerr, 3))
if (plot):
#zmin, zmax = zscale.zscale(impf[0].data.T[pra-50:pra+50,pdec-50:pdec+50])
#zmin, zmax = zscale.zscale(impf[0].data)
#im = plt.imshow(impf[0].data, vmin=zmin, vmax=zmax, origin="bottom")
print np.percentile(impf[0].data, 5), np.percentile(impf[0].data, 95)
impf[0].data[np.isnan(impf[0].data)] = np.nanmedian(impf[0].data)
print np.percentile(impf[0].data, 5), np.percentile(impf[0].data, 95)
im = plt.imshow(impf[0].data, vmin=np.percentile(impf[0].data, 5), vmax=np.percentile(impf[0].data, 95), origin="bottom")
X = int(ma["X"][0])
Y = int(ma["Y"][0])
pra = int(pra)
pdec = int(pdec)
plt.scatter(X, Y, marker="o", s=100, facecolor="none", edgecolor="red")
plt.colorbar(im)
plt.savefig(os.path.join(plotdir, imname+".png"), dpi=200)
plt.clf()
zmin, zmax = zscale.zscale(impf[0].data.T[X-50:X+50,Y-50:Y+50].T)
im = plt.imshow(impf[0].data.T[pra-50:pra+50,pdec-50:pdec+50].T, vmin=zmin, vmax=zmax, interpolation="none", origin="bottom", extent=(-50,50,-50,50))
c1 = plt.Circle( (pra-X, pdec-Y), edgecolor="k", facecolor="none", radius=aperture_rad, label="Initial position")
c11 = plt.Circle( (pra-X, pdec-Y), edgecolor="k", facecolor="none", radius=sky_rad)
c2 = plt.Circle( (0, 0), edgecolor="orange", facecolor="none", radius=aperture_rad, label="Adjusted centroid")
c22 = plt.Circle( (0, 0), edgecolor="orange", facecolor="none", radius=sky_rad)
plt.gca().add_artist(c1)
plt.gca().add_artist(c11)
plt.gca().add_artist(c2)
plt.gca().add_artist(c22)
plt.colorbar(im)
myhandles = []
markers = ["o", "o"]
labels = ["Initial position", "Adjusted centroid"]
cols = ["k", "orange"]
for i in np.arange(len(markers)):
myhandles.append(mlines.Line2D([], [], mec=cols[i], mfc="none", marker=markers[i], ls="None", markersize=10, label=labels[i]))
plt.legend(handles=myhandles, loc="lower left", labelspacing=0.3, fontsize=11, numpoints=1, frameon=False, ncol=5, bbox_to_anchor=(0.0, 0.00), fancybox=False, shadow=True)
plt.title("MIN: %.0f MAX: %.0f"%(np.nanmin(impf[0].data.T[X-50:X+50,Y-50:Y+50]), np.nanmax(impf[0].data.T[X-50:X+50,Y-50:Y+50])))
plt.savefig(os.path.join(plotdir, imname+"_zoom.png"))
plt.clf()
def create_masterflat(flatdir=None, biasdir=None, channel='rc'):
'''
Creates a masterflat from both dome flats and sky flats if the number of counts in the given filter
is not saturated and not too low (between 3000 and 40000).
'''
if (flatdir == None or flatdir==""): flatdir = "."
if (biasdir == None or biasdir==""): biasdir = "."
os.chdir(flatdir)
if (len(glob.glob("Flat_%s*norm.fits"%channel)) == 4):
logger.info( "Master Flat exists!")
return
if (len(glob.glob("Flat_%s*norm.fits"%channel)) > 0):
logger.info( "Some Master Flat exist!")
return
else:
logger.info( "Starting the Master Flat creation!")
bias_slow = "Bias_%s_slow.fits"%channel
bias_fast = "Bias_%s_fast.fits"%channel
if (not os.path.isfile(bias_slow) and not os.path.isfile(bias_fast) ):
create_masterbias(biasdir)
lsflat = []
lfflat = []
obj = ""
imtype = ""
#Select all filts that are Flats with same instrument
for f in glob.glob(channel+"*fits"):
try:
if fitsutils.has_par(f, "OBJECT"):
obj = str.upper(fitsutils.get_par(f, "OBJECT"))
else:
continue
if fitsutils.has_par(f, "IMGTYPE"):
imtype = str.upper(fitsutils.get_par(f, "IMGTYPE"))
else:
continue
#if ("RAINBOW CAM" in str.upper(fitsutils.get_par(f, "CAM_NAME")) and ("DOME" in obj or "FLAT" in obj or "Twilight" in obj or "TWILIGHT" in imtype or "DOME" in imtype)):
if ( "TWILIGHT" in imtype):
if (fitsutils.get_par(f, "ADCSPEED")==2):
lfflat.append(f)
else:
lsflat.append(f)
except:
logger.error( "Error with retrieving parameters for file %s"% f)
pass
logger.info( "Files for slow flat %s"% lsflat)
logger.info( "Files for fast flat %s"% lfflat)
fsfile ="lflat_slow_"+channel
np.savetxt(fsfile, np.array(lsflat), fmt="%s")
fffile ="lflat_fast_"+channel
np.savetxt(fffile, np.array(lfflat), fmt="%s")
# Running IRAF
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Remove bias from the flat
if len(lsflat) >0:
iraf.imarith("@"+fsfile, "-", bias_slow, "b_@"+fsfile)
if len(lfflat) >0:
iraf.imarith("@"+fffile, "-", bias_fast, "b_@"+fffile)
#Remove the list files
os.remove(fsfile)
os.remove(fffile)
#Slices the flats.
debiased_flats = glob.glob("b_*.fits")
for f in debiased_flats:
logger.info( "Slicing file %s"% f)
try:
slice_rc(f)
except:
logger.error("Error when slicing file... deleting the unsliced one...")
#Remove the un-sliced file
os.remove(f)
#Selects the ones that are suitable given the number of counts and combines them.
bands = ['u', 'g', 'r', 'i']
for b in bands:
out = "Flat_%s_%s.fits"%(channel, b)
out_norm = out.replace(".fits","_norm.fits")
if (os.path.isfile(out_norm)):
logger.error( "Master Flat for filter %s exists. Skipping..."%b)
continue
lfiles = []
for f in glob.glob('b_*_%s.fits'%b):
d = fits.open(f)[0].data
if np.percentile(d, 90)>4000 and np.percentile(d, 90)<40000:
lfiles.append(f)
if len(lfiles) == 0:
logger.error( "WARNING!!! Could not find suitable flats for band %s"%b)
continue
if len(lfiles) < 3:
logger.error( "WARNING!!! Could find less than 3 flats for band %s. Skipping, as it is not reliable..."%b)
continue
ffile ="lflat_"+b
np.savetxt(ffile, np.array(lfiles), fmt="%s")
#Cleaning of old files
if(os.path.isfile(out)): os.remove(out)
if(os.path.isfile(out_norm)): os.remove(out_norm)
if(os.path.isfile("Flat_stats")): os.remove("Flat_stats")
#Combine flats
iraf.imcombine(input = "@"+ffile, \
output = out, \
combine = "median",\
scale = "mode",
weight = "exposure")
iraf.imstat(out, fields="image,npix,mean,stddev,min,max,mode", Stdout="Flat_stats")
st = np.genfromtxt("Flat_stats", names=True, dtype=None)
#Normalize flats
iraf.imarith(out, "/", st["MODE"], out_norm)
#Do some cleaning
logger.info( 'Removing from lfiles')
for f in glob.glob('b_*_%s.fits'%b):
os.remove(f)
os.remove(ffile)
if os.path.isfile(fsfile):
os.remove(fsfile)
if os.path.isfile(fffile):
os.remove(fffile)
#copy into the reference folder with current date
newdir = os.path.join("../../refphot/", os.path.basename(os.path.abspath(flatdir)))
if (not os.path.isdir(newdir)):
os.makedirs(newdir)
shutil.copy(out_norm, os.path.join(newdir, os.path.basename(out_norm)) )
Runs astrometry.net on the image specified as a parameter and returns
the offset needed to be applied in order to center the object coordinates
in the reference pixel.
''', formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-d', '--reddir', type=str, dest="reduced", help='Fits directory file with reduced images.', default=None)
args = parser.parse_args()
reduced = args.reduced
if (reduced is None):
timestamp=datetime.datetime.isoformat(datetime.datetime.utcnow())
timestamp = timestamp.split("T")[0].replace("-","")
reduced = os.path.join(_photpath, timestamp, "reduced")
os.chdir(reduced)
for f in glob.glob("*.fits"):
if(fitsutils.has_par(f, "IMGTYPE") and fitsutils.get_par(f, "IMGTYPE") == "SCIENCE" or fitsutils.get_par(f, "IMGTYPE") == "ACQUISITION"):
#print f
get_app_phot_target(f, box=5)
cmd = 'echo "FILE ONTARGET NAME FILTER JD LST APPMAG APPMAGER INSMAG INSMAGER ZEROPT ZEROPTU" > photometry/magnitudes.dat; gethead ONTARGET NAME FILTER JD LST APPMAG APPMAGER INSMAG INSMAGER ZEROPT ZEROPTU *fits | grep -E "fits[\ ]+1" >> photometry/magnitudes.dat'
subprocess.call(cmd, shell=True)
def create_masterflat(flatdir=None, biasdir=None, channel='rc', plot=True):
'''
Creates a masterflat from both dome flats and sky flats if the number of counts in the given filter
is not saturated and not too low (between 3000 and 40000).
'''
if (flatdir == None or flatdir == ""): flatdir = "."
if (biasdir == None or biasdir == ""): biasdir = flatdir
os.chdir(flatdir)
if (plot and not os.path.isdir("reduced/flats")):
os.makedirs("reduced/flats")
if (len(glob.glob("Flat_%s*norm.fits" % channel)) == 4):
logger.info("Master Flat exists!")
return
if (len(glob.glob("Flat_%s*norm.fits" % channel)) > 0):
logger.info("Some Master Flat exist!")
else:
logger.info("Starting the Master Flat creation!")
bias_slow = "Bias_%s_slow.fits" % channel
bias_fast = "Bias_%s_fast.fits" % channel
if (not os.path.isfile(bias_slow) and not os.path.isfile(bias_fast)):
create_masterbias(biasdir)
lsflat = []
lfflat = []
obj = ""
imtype = ""
#Select all filts that are Flats with same instrument
for f in glob.glob(channel + "*fits"):
try:
if fitsutils.has_par(f, "OBJECT"):
obj = str.upper(fitsutils.get_par(f, "OBJECT"))
else:
continue
if fitsutils.has_par(f, "IMGTYPE"):
imtype = str.upper(fitsutils.get_par(f, "IMGTYPE"))
else:
continue
#if ("RAINBOW CAM" in str.upper(fitsutils.get_par(f, "CAM_NAME")) and ("DOME" in obj or "FLAT" in obj or "Twilight" in obj or "TWILIGHT" in imtype or "DOME" in imtype)):
if ("twilight" in imtype.lower()):
if (fitsutils.get_par(f, "ADCSPEED") == 2):
lfflat.append(f)
else:
lsflat.append(f)
except:
logger.error("Error with retrieving parameters for file %s" % f)
pass
logger.info("Files for slow flat %s" % lsflat)
logger.info("Files for fast flat %s" % lfflat)
fsfile = "lflat_slow_" + channel
np.savetxt(fsfile, np.array(lsflat), fmt="%s")
fffile = "lflat_fast_" + channel
np.savetxt(fffile, np.array(lfflat), fmt="%s")
# Running IRAF
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Remove bias from the flat
if len(lsflat) > 0:
iraf.imarith("@" + fsfile, "-", bias_slow, "b_@" + fsfile)
if len(lfflat) > 0:
iraf.imarith("@" + fffile, "-", bias_fast, "b_@" + fffile)
#Remove the list files
os.remove(fsfile)
os.remove(fffile)
#Slices the flats.
debiased_flats = glob.glob("b_*.fits")
for f in debiased_flats:
logger.info("Slicing file %s" % f)
try:
slice_rc(f)
except:
logger.error(
"Error when slicing file... deleting the unsliced one...")
#Remove the un-sliced file
os.remove(f)
#Selects the ones that are suitable given the number of counts and combines them.
bands = ['u', 'g', 'r', 'i']
for b in bands:
out = "Flat_%s_%s.fits" % (channel, b)
out_norm = out.replace(".fits", "_norm.fits")
if (os.path.isfile(out_norm)):
logger.error("Master Flat for filter %s exists. Skipping..." % b)
continue
lfiles = []
for f in glob.glob('b_*_%s.fits' % b):
fi = fits.open(f)
d = fi[0].data
status = "rejected"
if np.percentile(d, 90) > 4000 and np.percentile(d, 90) < 45000:
lfiles.append(f)
mymode = 1. * np.median(d.flatten())
d[d > 45000] = mymode
fi[0].data = d
fi.writeto(f, clobber=True)
status = "accepted"
if (plot):
plt.title("Flat filter %s. %s" % (b, status))
plt.imshow(d.T, cmap=plt.get_cmap("nipy_spectral"))
plt.colorbar()
plt.savefig("reduced/flats/%s" % (f.replace(".fits", ".png")))
plt.close()
#Make sure that the optimum number of counts is not too low and not saturated.
if len(lfiles) == 0:
logger.error(
"WARNING!!! Could not find suitable flats for band %s" % b)
continue
if len(lfiles) < 3:
logger.error(
"WARNING!!! Could find less than 3 flats for band %s. Skipping, as it is not reliable..."
% b)
continue
ffile = "lflat_" + b
np.savetxt(ffile, np.array(lfiles), fmt="%s")
#Cleaning of old files
if (os.path.isfile(out)): os.remove(out)
if (os.path.isfile(out_norm)): os.remove(out_norm)
if (os.path.isfile("Flat_stats")): os.remove("Flat_stats")
#Combine flats
iraf.imcombine(input = "@"+ffile, \
output = out, \
combine = "median",\
scale = "mode",
reject = "sigclip", lsigma = 2., hsigma = 2, gain=1.7, rdnoise=4.)
iraf.imstat(out,
fields="image,npix,mean,stddev,min,max,mode",
Stdout="Flat_stats")
st = np.genfromtxt("Flat_stats", names=True, dtype=None)
#Normalize flats
iraf.imarith(out, "/", st["MODE"], out_norm)
#Do some cleaning
logger.info('Removing from lfiles')
for f in glob.glob('b_*_%s.fits' % b):
os.remove(f)
os.remove(ffile)
if os.path.isfile(fsfile):
os.remove(fsfile)
if os.path.isfile(fffile):
os.remove(fffile)
#copy into the reference folder with current date
newdir = os.path.join("../../refphot/",
os.path.basename(os.path.abspath(flatdir)))
if (not os.path.isdir(newdir)):
os.makedirs(newdir)
shutil.copy(out_norm, os.path.join(newdir, os.path.basename(out_norm)))
copy_ref_calib(flatdir, "Flat")
def add_to_zp_cal(ref_stars, image, logname):
"""
Records the parameters and instrumental and standard star magnitudes needed for zeropoint calibration.
ref_stars: name of the file that contains the reference stars that are present in the image.
image: fits image with the sources in them.
logname: name of the file where the information on reference and measurements are logged.
minst = M + c0 + c1(AIRMASS) + c2(color) + c3(UT)
"""
coldic = {"u": "g", "g": "r", "r": "i", "i": "r", "z": "i", "U": "B", "B": "V", "V": "R", "R": "I", "I": "R"}
r = np.genfromtxt(ref_stars, delimiter=" ", dtype=None, names=True)
imapp = os.path.join(os.path.join(os.path.dirname(image), "photometry"), os.path.basename(image) + ".app.mag")
# imapp = os.path.join(os.path.dirname(image), os.path.basename(image) + ".app.mag")
"""try:
my = np.genfromtxt(imapp, comments="#", dtype=[("id","<f4"), ("X","<f4"), ("Y","<f4"),("Xshift","<f4"), ("Yshift","<f4"),("fwhm","<f4"), ("ph_mag","<f4"), ("stdev","<f4"), ("fit_mag","<f4"), ("fiterr","<f4")])
except:"""
my = np.genfromtxt(
imapp,
comments="#",
dtype=[
("id", "<f4"),
("filename", "<f4"),
("X", "<f4"),
("Y", "<f4"),
("Xshift", "<f4"),
("Yshift", "<f4"),
("fwhm", "<f4"),
("ph_mag", "<f4"),
("stdev", "<f4"),
("fit_mag", "<f4"),
("fiterr", "<f4"),
],
)
if my.size < 2:
my = np.array([my])
if r.size < 2:
r = np.array([r])
band = fitsutils.get_par(image, "filter")
mask_valid1 = (
np.array(my["fwhm"] < 9000)
* np.array(my["ph_mag"] < 9000)
* np.array(~np.isnan(r[band]))
* np.array(~np.isnan(my["fit_mag"]))
)
r = r[mask_valid1]
my = my[mask_valid1]
N = len(r)
my["fiterr"][np.isnan(my["fiterr"])] = 100
col_band = coldic[band]
exptime = fitsutils.get_par(image, "exptime")
airmass = 1.3
name = "object"
if fitsutils.has_par(image, "NAME"):
name = fitsutils.get_par(image, "NAME")
if fitsutils.has_par(image, "AIRMASS"):
airmass = fitsutils.get_par(image, "AIRMASS")
if fitsutils.has_par(image, "JD"):
date = fitsutils.get_par(image, "JD")
elif fitsutils.has_par(image, "MJD"):
date = fitsutils.get_par(image, "MJD")
elif fitsutils.has_par(image, "MJD-OBS"):
date = fitsutils.get_par(image, "MJD-OBS")
else:
date = 0
if not os.path.isfile(logname):
with open(logname, "a") as f:
f.write("#object,filename,filter,std,stderr,inst,insterr,jd,airmass,color,exptime\n")
with open(logname, "a") as f:
for i in range(N):
f.write(
"%s,%s,%s,%.3f,%.3f,%3f,%.3f,%.3f,%.3f,%.3f,%.3f\n"
% (
name,
image,
band,
r[i][band],
r[i]["d" + band],
my[i]["fit_mag"],
my[i]["fiterr"],
date,
airmass,
r[i][band] - r[i][col_band],
exptime,
)
)
def interpolate_zp(reduced, logfile):
"""
Uses the zeropoint coefficients derived from SDSS fields to interpolate the
zeropoint for the images that are outside of SDSS field.
"""
a = np.genfromtxt(logfile, dtype=None, names=True, delimiter=",")
a.sort(order=["jd"], axis=0)
a = a[a["inst"] != 0]
a = a[a["insterr"] > 0]
jdmin = np.min(a["jd"])
jdmax = np.max(a["jd"])
zpfiles = glob.glob(os.path.join(reduced, "*fits"))
zpfiles = [
zf
for zf in zpfiles
if fitsutils.has_par(zf, "IQZEROPT")
and (
fitsutils.get_par(zf, "IQZEROPT") == 0
or fitsutils.get_par(zf, "ZEROPT") == 0
or fitsutils.get_par(zf, "ZPCAT") == "SDSSinterpolated"
)
]
# Load the coefficients.
coefs = {}
for fi in ["u", "g", "r", "i"]:
coeffile = os.path.join(reduced, "coefs_%s.txt" % fi)
if os.path.isfile(coeffile):
coefs[fi] = np.genfromtxt(coeffile)
# Load the rms.
rms = {}
for fi in ["u", "g", "r", "i"]:
rmsfile = os.path.join(reduced, "rms_%s.txt" % fi)
if os.path.isfile(rmsfile):
rms[fi] = np.genfromtxt(os.path.join(reduced, "rms_%s.txt" % fi))
for image in zpfiles:
filt = fitsutils.get_par(image, "FILTER")
# To not extrapolate outside of the valid interval.
jd = np.maximum(np.percentile(a["jd"] - jdmin, 10), fitsutils.get_par(image, "JD") - jdmin)
jd = np.minimum(np.percentile(a["jd"] - jdmin, 90), fitsutils.get_par(image, "JD") - jdmin)
airmass = fitsutils.get_par(image, "AIRMASS")
# If there are coefficients for that filter, load them and interpolate.
# Otherwise, skip this file.
if not coefs.has_key(filt):
continue
# est_zp = coef[0] +ab['color']*coef[1] +(ab['airmass']-1.3)*coef[2] + coef[3]*ab['jd'] + coef[4]*ab['jd']**2 + coef[5]*ab['jd']**3 + coef[6]*ab['jd']**4 + coef[7]*ab['jd']**5
values = np.array([1, 0, airmass - 1.3, jd, jd ** 2])
est_zp = np.sum(coefs[filt] * values)
# Update the header with the computed zeropoint.
pardic = {"IQZEROPT": 1, "ZPCAT": "SDSSinterpolated", "ZEROPTU": float(rms[filt]), "ZEROPT": est_zp}
fitsutils.update_pars(image, pardic)
timestamp = timestamp.split("T")[0].replace("-", "")
photdir = os.path.join(_photpath, timestamp)
print '''WARNING! You did not specify the directory or the list:\n
- A filelist name with the images you want to reduce [-l]
OR
- The name of the directory which you want to reduce [-d].
A default name for the directory will be assumed on today's date: %s
''' % photdir
mydir = os.path.abspath(photdir)
#Gather all RC fits files in the folder with the keyword IMGTYPE=SCIENCE
for f in glob.glob(os.path.join(mydir, "rc*fits")):
try:
if (fitsutils.has_par(f, "IMGTYPE") and
((fitsutils.get_par(f, "IMGTYPE").upper() == "SCIENCE") or
("ACQ" in fitsutils.get_par(f, "IMGTYPE").upper()))):
myfiles.append(f)
except:
print "problems opening file %s" % f
create_masterbias(mydir)
print "Create masterflat", mydir
create_masterflat(mydir)
if (len(myfiles) == 0):
print "Found no files to process"
sys.exit()
else:
print "Found %d files to process" % len(myfiles)