def add_fwhm(myfile, outfile):
a = np.genfromtxt(myfile, dtype=None, names=True, delimiter=",")
fwhm = np.zeros(len(a))
for i, f in enumerate(a["filename"]):
sexfile = f.replace("reduced/",
"reduced/sextractor/").replace(".fits", ".sex")
if (not os.path.isfile(sexfile)):
sextractor.run_sex([f], mask=False, cosmics=False, overwrite=False)
if (os.path.isfile(sexfile)):
s = np.genfromtxt(sexfile, comments="#")
#Select round sources (ellipticity is 1-axis_ratio)
s = s[s[:, 8] < np.percentile(s[:, 8], 30)]
#Select bright magnitudes
s = s[s[:, 4] < np.percentile(s[:, 4], 20)]
fwhm[i] = np.percentile(s[:, 7], 33) * 0.394
out = rfn.append_fields(a, names="fwhm", data=fwhm, usemask=False)
if (not outfile is None):
np.savetxt(
outfile,
out,
header=
"#object,filename,filter,std,stderr,inst,insterr,jd,airmass,color,exptime,fwhm",
fmt="%s,%s,%s,%.4f,%.4f,%.4f,%.4f,%.4f,%.3f,%.3f,%.2f,%.3f")
return out
def get_sextractor_stats(files):
files.sort()
sexfiles = [os.path.join(os.path.join(os.path.dirname(f), "sextractor"), os.path.basename(f).replace(".fits", ".sex")) for f in files]
sexfiles.sort()
if not os.path.isdir(os.path.join( os.path.dirname(files[0]), "stats")):
os.makedirs(os.path.join(os.path.dirname(files[0]), "stats"))
with open(os.path.join( os.path.dirname(files[0]), "stats/stats.log"), "w") as out:
for i, f in enumerate(files):
if (fitsutils.has_par(f, "IMGTYPE")):
imtype = fitsutils.get_par(f, "IMGTYPE")
else:
imtype = "NONE"
if not (imtype == "ACQUISITION" or imtype == "SCIENCE" or imtype=="FOCUS" or imtype=="GUIDER"):
continue
if not os.path.isfile(sexfiles[i]):
sf = sextractor.run_sex([f])
else:
sf = sexfiles[i]
print f
hd = pf.open(files[i])[0].header
try:
jd = hd["JD"]
obj = hd["OBJECT"]
airmass = hd["AIRMASS"]
in_temp = hd["IN_AIR"]
out_temp = hd["OUT_AIR"]
in_hum = hd["IN_HUM"]
ns, fwhm, ellipticity, bkg = sextractor.analyse_image(sf)
out.write("%s,%s,%.3f,%d,%.2f,%.3f,%.3f,%.2f,%.1f,%s,%.2f,%.2f\n"%(os.path.abspath(f),obj,jd,ns,fwhm,ellipticity,bkg,airmass,in_temp,imtype,out_temp,in_hum))
except:
pass
def log_db_phot(myfile):
'''
Logs the science files.
'''
import archive_mod_sedmdb # this file requires being able to submit observations with pre-determined id values, which isn't allowed in the original
sdb = archive_mod_sedmdb.SedmDB(dbname='sedmdbtest', host='localhost')
cmd = "git version %s" % __file__
p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
version = p.communicate()[0].replace('\n', "")
pardic = {
"id": sdb._id_from_time(),
"phot_calib_id": "",
"observation_id": fitsutils.get_par(myfile, "OBS_ID"),
"astrometry": bool(fitsutils.get_par(myfile, "IQWCS")),
"filter": fitsutils.get_par(myfile, "FILTER").strip(),
"reducedfile": os.path.abspath(myfile),
"sexfile": sextractor.run_sex([myfile])[0],
"maskfile": "",
"pipeline": version,
"marshal_phot_id": ""
}
sdb.add_phot(pardic)
def get_sextractor_stats(files):
files.sort()
sexfiles = [
os.path.join(os.path.join(os.path.dirname(f), "sextractor"),
os.path.basename(f).replace(".fits", ".sex"))
for f in files
]
sexfiles.sort()
if not os.path.isdir(os.path.join(os.path.dirname(files[0]), "stats")):
os.makedirs(os.path.join(os.path.dirname(files[0]), "stats"))
with open(os.path.join(os.path.dirname(files[0]), "stats/stats.log"),
"w") as out:
for i, f in enumerate(files):
try:
if (fitsutils.has_par(f, "IMGTYPE")):
imtype = fitsutils.get_par(f, "IMGTYPE")
imtype = imtype.upper()
else:
imtype = "NONE"
if not ("ACQ" in imtype or imtype == "SCIENCE"
or imtype == "FOCUS" or imtype == "GUIDER"):
continue
if not os.path.isfile(sexfiles[i]):
sflist = sextractor.run_sex([f])
if (not sflist is None and len(sflist) > 0):
sf = sflist[0]
else:
sf = sexfiles[i]
hd = pf.open(f)[0].header
try:
jd = hd["JD"]
obj = hd["OBJECT"]
airmass = hd["AIRMASS"]
in_temp = hd["IN_AIR"]
out_temp = hd["OUT_AIR"]
in_hum = hd["IN_HUM"]
ns, fwhm, ellipticity, bkg = sextractor.analyse_image(sf)
out.write(
"%s,%s,%.3f,%d,%.2f,%.3f,%.3f,%.2f,%.1f,%s,%.2f,%.2f\n"
% (os.path.abspath(f), obj, jd, ns, fwhm, ellipticity,
bkg, airmass, in_temp, imtype, out_temp, in_hum))
except Exception as e:
print "Error when retrieving the stats parameters from the header of file %s.\n Error %s" % (
f, e)
except IOError:
print "Error when opening file %s" % f
#Get the files from the same day directory.
files = glob.glob(os.path.join(raw, "ifu*fits"))
#files_hg = [f for f in files if "Calib: Hg" in get_par(f, "OBJECT")]
#Get the files from one day before directory to see the differences between days.
daybefore = datetime.datetime.isoformat(datetime.datetime.utcnow() -
datetime.timedelta(1))
daybefore = daybefore.split("T")[0].replace("-", "")
daybefore = os.path.join("/scr2/sedm/phot/", daybefore)
if (os.path.isdir(daybefore)):
filesold = glob.glob(os.path.join(daybefore, "ifu*fits"))
files.extend(filesold)
files_hg = [
f for f in files if fitsutils.has_par(f, "OBJECT")
and "Calib: Hg" in fitsutils.get_par(f, "OBJECT")
]
logger.info("Found the following Hg files: %s" % files_hg)
if (len(files_hg) > 1):
files_hg.sort()
sexfiles = sextractor.run_sex(files_hg, mask=False)
plotdir = os.path.join(raw, "stats")
if (not os.path.isdir(plotdir)):
os.makedirs(plotdir)
run_flexure_test(sexfiles, plotdir=plotdir)
def create_superflat(imdir, filters=["u", "g", "r", "i"]):
#Locate images for each filter
imlist = glob.glob("rc*fits")
#Run sextractor to locate bright sources
sexfiles = sextractor.run_sex(imlist, overwrite=False)
maskfiles = []
for i, im in enumerate(imlist):
#Create a mask and store it int he mask directory
maskfile = mask_stars(im, sexfiles[i])
maskfiles.append(maskfile)
fitsutils.update_par(im, "BPM", os.path.relpath(maskfile))
for filt in filters:
fimlist = [im for im in imlist if fitsutils.get_par(im, "FILTER") == filt]
fmasklist = [im for im in maskfiles if fitsutils.get_par(im, "FILTER") == filt]
if len(fimlist) == 0:
continue
fsfile ="lflat_%s"%filt
msfile = "lmask_%s"%filt
np.savetxt(fsfile, np.array(fimlist), fmt="%s")
np.savetxt(msfile, np.array(fmasklist), fmt="%s")
'''masklist = []
for m in fmasklist:
hdulist = fits.open(m)
data = hdulist[0].data
masklist.append(data)
masklist = np.array(masklist)
hdu = fits.PrimaryHDU(masklist)
hdulist = fits.HDUList([hdu])
hdulist.writeto("mastermask_%s.fits"%filt)'''
# Running IRAF
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.imarith("@"+fsfile, "*", "@"+msfile, "m_@"+fsfile)
#Combine flats
iraf.imcombine(input = "m_@"+fsfile, \
output = "superflat_%s.fits"%filt, \
combine = "median",\
scale = "mode", \
masktype="badvalue",\
maskvalue = 0)
iraf.imstat("superflat_%s.fits"%filt, fields="image,npix,mean,stddev,min,max,mode", Stdout="Flat_stats")
time.sleep(0.1)
st = np.genfromtxt("Flat_stats", names=True, dtype=None)
#Normalize flats
iraf.imarith("superflat_%s.fits"%filt, "/", st["MODE"], "superflat_%s_norm.fits"%filt)
def create_superflat(imdir, filters=["u", "g", "r", "i"]):
#Locate images for each filter
imlist = glob.glob("rc*fits")
#Run sextractor to locate bright sources
sexfiles = sextractor.run_sex(imlist, overwrite=False)
maskfiles = []
for i, im in enumerate(imlist):
#Create a mask and store it int he mask directory
maskfile = mask_stars(im, sexfiles[i])
maskfiles.append(maskfile)
fitsutils.update_par(im, "BPM", os.path.relpath(maskfile))
for filt in filters:
fimlist = [
im for im in imlist if fitsutils.get_par(im, "FILTER") == filt
]
fmasklist = [
im for im in maskfiles if fitsutils.get_par(im, "FILTER") == filt
]
if len(fimlist) == 0:
continue
fsfile = "lflat_%s" % filt
msfile = "lmask_%s" % filt
np.savetxt(fsfile, np.array(fimlist), fmt="%s")
np.savetxt(msfile, np.array(fmasklist), fmt="%s")
'''masklist = []
for m in fmasklist:
hdulist = fits.open(m)
data = hdulist[0].data
masklist.append(data)
masklist = np.array(masklist)
hdu = fits.PrimaryHDU(masklist)
hdulist = fits.HDUList([hdu])
hdulist.writeto("mastermask_%s.fits"%filt)'''
# Running IRAF
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
iraf.imarith("@" + fsfile, "*", "@" + msfile, "m_@" + fsfile)
#Combine flats
iraf.imcombine(input = "m_@"+fsfile, \
output = "superflat_%s.fits"%filt, \
combine = "median",\
scale = "mode", \
masktype="badvalue",\
maskvalue = 0)
iraf.imstat("superflat_%s.fits" % filt,
fields="image,npix,mean,stddev,min,max,mode",
Stdout="Flat_stats")
time.sleep(0.1)
st = np.genfromtxt("Flat_stats", names=True, dtype=None)
#Normalize flats
iraf.imarith("superflat_%s.fits" % filt, "/", st["MODE"],
"superflat_%s_norm.fits" % filt)
