def reduce_on_the_fly(photdir):
'''
Waits for new images to appear in the directory to trigger their incremental reduction as well.
'''
#Get the current the number of files
nfiles = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
logger.info(
"Starting the on-the-fly reduction for directory %s. Found %d files to process."
% (photdir, len(nfiles)))
dayname = os.path.basename(photdir)
time_ini = datetime.datetime.now()
time_curr = datetime.datetime.now()
#Run this loop for 12h since the start.
while (time_curr - time_ini).total_seconds() < 12. * 3600:
nfilesnew = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
if len(nfilesnew) == len(nfiles):
time.sleep(10)
else:
new = [f for f in nfilesnew if f not in nfiles]
new.sort()
logger.info("Detected new %d incoming files in the last 10s." %
len(new))
for n in new:
if (not fitsutils.has_par(n, "IMGTYPE")):
print "Image", n, "Does not have an IMGTYPE"
time.sleep(0.5)
if (not fitsutils.has_par(n, "IMGTYPE")):
print "Image", n, "STILL Does not have an IMGTYPE"
continue
plot_image(n)
imtype = fitsutils.get_par(n, "IMGTYPE")
if (imtype.upper() == "SCIENCE"):
reduced = rcred.reduce_image(n)
try:
zeropoint.calibrate_zeropoint(reduced)
except:
logger.error(
"Could not calibrate zeropoint for image %s" %
reduced)
#Copy them to transient
for r in reduced:
cmd = "rcp %s [email protected]:/scr3/mansi/ptf/p60phot/fremling_pipeline/sedm/reduced/%s/." % (
r, dayname)
subprocess.call(cmd, shell=True)
logger.info(cmd)
logger.info("Successfully copied the image: %s" % cmd)
time_curr = datetime.datetime.now()
nfiles = nfilesnew
def reduce_on_the_fly(photdir):
'''
Waits for new images to appear in the directory to trigger their incremental reduction as well.
'''
#Get the current the number of files
nfiles = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
dayname = os.path.basename(photdir)
time_ini = datetime.datetime.now()
time_curr = datetime.datetime.now()
#Run this loop for 12h since the start.
while (time_curr-time_ini).total_seconds() < 12.*3600:
nfilesnew = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
if len(nfilesnew) == len(nfiles):
time.sleep(10)
else:
new = [f for f in nfilesnew if f not in nfiles]
for n in new:
if (not fitsutils.has_par(n, "IMGTYPE")):
print "Image",n,"Does not have an IMGTYPE"
time.sleep(0.5)
if (not fitsutils.has_par(n, "IMGTYPE")):
print "Image",n,"STILL Does not have an IMGTYPE"
continue
if (fitsutils.get_par(n, "IMGTYPE")=="SCIENCE"):
reduced = rcred.reduce_image(n)
try:
zeropoint.calibrate_zeropoint(reduced)
except:
logger.error("Could not calibrate zeropoint for image %s"%reduced)
#Copy them to transient
for r in reduced:
cmd = "rcp %s [email protected]:/scr3/mansi/ptf/p60phot/fremling_pipeline/sedm/reduced/%s/."%(r, dayname)
subprocess.call(cmd, shell=True)
logger.info(cmd)
logger.info( "Successfully copied the image: %s"% cmd)
time_curr = datetime.datetime.now()
nfiles = nfilesnew
def reduce_image(image, flatdir=None, biasdir=None, cosmic=False, astrometry=True, channel='rc', target_dir='reduced', overwrite=False):
'''
Applies Flat field and bias calibrations to the image.
Steps:
1. - Solve astrometry on the entire image.
2. - Computes cosmic ray rejectionon the entire image.
3. - Compute master bias (if it does not exist) and de-bias the image.
4. - Separate the image into 4 filters.
5. - Compute flat field for each filter (if it does not exist) and apply flat fielding on the image.
6. - Compute the image zeropoint.
'''
logger.info("Reducing image %s"% image)
print "Reducing image ", image
image = os.path.abspath(image)
imname = os.path.basename(image).replace(".fits", "")
try:
objectname = fitsutils.get_par(image, "NAME").replace(" ","")+"_"+fitsutils.get_par(image, "FILTER")
except:
logger.error( "ERROR, image "+ image + " does not have a NAME or a FILTER!!!")
return
print "For object", objectname
logger.info( "For object %s"% objectname)
#Change to image directory
mydir = os.path.dirname(image)
if mydir=="": mydir = "."
mydir = os.path.abspath(mydir)
os.chdir(mydir)
#Create destination directory
if (not os.path.isdir(target_dir)):
os.makedirs(target_dir)
#If we don't want to overwrite the already extracted images, we check wether they exist.
if (not overwrite):
existing = True
for band in ['u', 'g', 'r', 'i']:
destfile = os.path.join(target_dir, imname + "_f_b_a_%s_%s_0.fits"%(objectname, band))
logger.info( "Looking if file %s exists: %s"%( destfile, \
(os.path.isfile(destfile) ) ) )
existing = existing and (os.path.isfile( destfile ) )
if existing:
return []
#Initialize the basic parameters.
init_header_reduced(image)
astro = ""
if (astrometry):
logger.info( "Solving astometry for the whole image...")
img = solve_astrometry(image)
if (os.path.isfile(img)):
astro="a_"
fitsutils.update_par(img, "IQWCS", 1)
else:
logger.error( "ASTROMETRY DID NOT SOLVE ON IMAGE %s"% image)
img = image
#Update noise parameters needed for cosmic reection
if (fitsutils.get_par(img, "ADCSPEED")==2):
fitsutils.update_par(img, "RDNOISE", 20.)
else:
fitsutils.update_par(img, "RDNOISE", 4.)
if (cosmic):
logger.info( "Correcting for cosmic rays...")
# Correct for cosmics each filter
cleanimg = clean_cosmic(os.path.join(os.path.abspath(mydir), img))
img = cleanimg
#Get basic statistics for the image
nsrc, fwhm, ellip, bkg = sextractor.get_image_pars(img)
logger.info( "Sextractor statistics: nscr %d, fwhm (pixel) %.2f, ellipticity %.2f"% (nsrc, fwhm, ellip))
print "Sextractor statistics: nscr %d, fwhm (pixel) %.2f, ellipticity %.2f"% (nsrc, fwhm, ellip)
dic = {"SEEPIX": fwhm/0.394, "NSRC":nsrc, "ELLIP":ellip}
#Update the seeing information from sextractor
fitsutils.update_pars(img, dic)
#Compute BIAS
if (biasdir is None or biasdir==""): biasdir = "."
create_masterbias(biasdir)
bias_slow = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'slow'))
bias_fast = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'fast'))
# Running IRAF to DE-BIAS
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Compute flat field
if (flatdir is None or flatdir==""): flatdir = "."
create_masterflat(flatdir, biasdir)
#New names for the object.
debiased = os.path.join(os.path.dirname(img), "b_" + os.path.basename(img))
logger.info( "Creating debiased file, %s"%debiased)
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.warn( "Master bias not found! Tryting to copy from reference folder...")
copy_ref_calib(mydir, "Bias")
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.error( "Bias not found in reference folder")
return
#Clean first
if (os.path.isfile(debiased)):
os.remove(debiased)
#Debias
if (fitsutils.get_par(img, "ADCSPEED")==2):
iraf.imarith(img, "-", bias_fast, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_fast)
fitsutils.update_par(debiased, "RDNOISE", 20.)
else:
iraf.imarith(img, "-", bias_slow, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_slow)
fitsutils.update_par(debiased, "RDNOISE", 4.)
#Set negative counts to zero
hdu = fits.open(debiased)
header = hdu[0].header
hdu[0].data[hdu[0].data<0] = 0
hdu.writeto(debiased, clobber=True)
#Slicing the image for flats
slice_names = slice_rc(debiased)
print "Creating sliced files, ", slice_names
#Remove un-sliced image
os.remove(debiased)
# DE-flat each filter and store under object name
for i, debiased_f in enumerate(slice_names):
b = fitsutils.get_par(debiased_f, 'filter')
deflatted = os.path.join(os.path.dirname(image), target_dir, imname + "_f_b_" + astro + objectname + "_%s.fits"%b)
#Flat to be used for that filter
flat = os.path.join(flatdir, "Flat_%s_%s_norm.fits"%(channel, b))
if (not os.path.isfile(flat)):
logger.warn( "Master flat not found in %s"% flat)
copy_ref_calib(mydir, "Flat_%s_%s_norm"%(channel, b))
continue
else:
logger.info( "Using flat %s"%flat)
#Cleans the deflatted file if exists
if (os.path.isfile(deflatted)):
os.remove(deflatted)
if (os.path.isfile(debiased_f) and os.path.isfile(flat)):
logger.info( "Storing de-flatted %s as %s"%(debiased_f, deflatted))
time.sleep(1)
iraf.imarith(debiased_f, "/", flat, deflatted)
else:
logger.error( "SOMETHING IS WRONG. Error when dividing %s by the flat field %s!"%(debiased_f, flat))
#Removes the de-biased file
os.remove(debiased_f)
logger.info( "Updating header with original filename and flat field used.")
fitsutils.update_par(deflatted, "ORIGFILE", os.path.basename(image))
fitsutils.update_par(deflatted, "FLATFILE", flat)
slice_names[i] = deflatted
#Moving files to the target directory
for image in slice_names:
bkg = get_median_bkg(image)
fitsutils.update_par(image, "SKYBKG", bkg)
#shutil.move(name, newname)
#Compute the zeropoints
for image in slice_names:
zeropoint.calibrate_zeropoint(image)
return slice_names
def reduce_image(image,
flatdir=None,
biasdir=None,
cosmic=False,
astrometry=True,
channel='rc',
target_dir='reduced',
overwrite=False):
'''
Applies Flat field and bias calibrations to the image.
Steps:
1. - Solve astrometry on the entire image.
2. - Computes cosmic ray rejectionon the entire image.
3. - Compute master bias (if it does not exist) and de-bias the image.
4. - Separate the image into 4 filters.
5. - Compute flat field for each filter (if it does not exist) and apply flat fielding on the image.
6. - Compute the image zeropoint.
'''
logger.info("Reducing image %s" % image)
print "Reducing image ", image
image = os.path.abspath(image)
imname = os.path.basename(image).replace(".fits", "")
try:
objectname = fitsutils.get_par(image, "NAME").replace(
" ", "") + "_" + fitsutils.get_par(image, "FILTER")
except:
logger.error("ERROR, image " + image +
" does not have a NAME or a FILTER!!!")
return
print "For object", objectname
logger.info("For object %s" % objectname)
#Change to image directory
mydir = os.path.dirname(image)
if mydir == "": mydir = "."
mydir = os.path.abspath(mydir)
os.chdir(mydir)
#Create destination directory
if (not os.path.isdir(target_dir)):
os.makedirs(target_dir)
#If we don't want to overwrite the already extracted images, we check wether they exist.
if (not overwrite):
existing = True
for band in ['u', 'g', 'r', 'i']:
destfile = os.path.join(
target_dir,
imname + "_f_b_a_%s_%s_0.fits" % (objectname, band))
logger.info( "Looking if file %s exists: %s"%( destfile, \
(os.path.isfile(destfile) ) ) )
existing = existing and (os.path.isfile(destfile))
if existing:
return []
#Initialize the basic parameters.
init_header_reduced(image)
astro = ""
if (astrometry):
logger.info("Solving astometry for the whole image...")
img = solve_astrometry(image)
if (os.path.isfile(img)):
astro = "a_"
fitsutils.update_par(img, "IQWCS", 1)
else:
logger.error("ASTROMETRY DID NOT SOLVE ON IMAGE %s" % image)
img = image
#Update noise parameters needed for cosmic reection
if (fitsutils.get_par(img, "ADCSPEED") == 2):
fitsutils.update_par(img, "RDNOISE", 20.)
else:
fitsutils.update_par(img, "RDNOISE", 4.)
if (cosmic):
logger.info("Correcting for cosmic rays...")
# Correct for cosmics each filter
cleanimg = clean_cosmic(os.path.join(os.path.abspath(mydir), img))
img = cleanimg
#Compute BIAS
if (biasdir is None or biasdir == ""): biasdir = "."
create_masterbias(biasdir)
bias_slow = os.path.join(biasdir, "Bias_%s_%s.fits" % (channel, 'slow'))
bias_fast = os.path.join(biasdir, "Bias_%s_%s.fits" % (channel, 'fast'))
# Running IRAF to DE-BIAS
iraf.noao(_doprint=0)
iraf.imred(_doprint=0)
iraf.ccdred(_doprint=0)
#Compute flat field
if (flatdir is None or flatdir == ""): flatdir = "."
create_masterflat(flatdir, biasdir)
#New names for the object.
debiased = os.path.join(os.path.dirname(img), "b_" + os.path.basename(img))
logger.info("Creating debiased file, %s" % debiased)
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.warn(
"Master bias not found! Tryting to copy from reference folder...")
copy_ref_calib(mydir, "Bias")
if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
logger.error("Bias not found in reference folder")
return
#Clean first
if (os.path.isfile(debiased)):
os.remove(debiased)
#Debias
if (fitsutils.get_par(img, "ADCSPEED") == 2):
iraf.imarith(img, "-", bias_fast, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_fast)
fitsutils.update_par(debiased, "RDNOISE", 20.)
else:
iraf.imarith(img, "-", bias_slow, debiased)
fitsutils.update_par(debiased, "BIASFILE", bias_slow)
fitsutils.update_par(debiased, "RDNOISE", 4.)
#Set negative counts to zero
hdu = fits.open(debiased)
header = hdu[0].header
hdu[0].data[hdu[0].data < 0] = 0
hdu.writeto(debiased, clobber=True)
#Slicing the image for flats
slice_names = slice_rc(debiased)
print "Creating sliced files, ", slice_names
#Remove un-sliced image
os.remove(debiased)
# DE-flat each filter and store under object name
for i, debiased_f in enumerate(slice_names):
b = fitsutils.get_par(debiased_f, 'filter')
deflatted = os.path.join(
os.path.dirname(image), target_dir,
imname + "_f_b_" + astro + objectname + "_%s.fits" % b)
#Flat to be used for that filter
flat = os.path.join(flatdir, "Flat_%s_%s_norm.fits" % (channel, b))
if (not os.path.isfile(flat)):
logger.warn("Master flat not found in %s" % flat)
copy_ref_calib(mydir, "Flat")
continue
else:
logger.info("Using flat %s" % flat)
#Cleans the deflatted file if exists
if (os.path.isfile(deflatted)):
os.remove(deflatted)
if (os.path.isfile(debiased_f) and os.path.isfile(flat)):
logger.info("Storing de-flatted %s as %s" %
(debiased_f, deflatted))
time.sleep(1)
iraf.imarith(debiased_f, "/", flat, deflatted)
else:
logger.error(
"SOMETHING IS WRONG. Error when dividing %s by the flat field %s!"
% (debiased_f, flat))
#Removes the de-biased file
os.remove(debiased_f)
logger.info(
"Updating header with original filename and flat field used.")
fitsutils.update_par(deflatted, "ORIGFILE", os.path.basename(image))
fitsutils.update_par(deflatted, "FLATFILE", flat)
slice_names[i] = deflatted
#Moving files to the target directory
for image in slice_names:
bkg = get_median_bkg(image)
fitsutils.update_par(image, "SKYBKG", bkg)
#Get basic statistics for the image
nsrc, fwhm, ellip, bkg = sextractor.get_image_pars(image)
plot_image(image)
logger.info(
"Sextractor statistics: nscr %d, fwhm (arcsec) %.2f, ellipticity %.2f"
% (nsrc, fwhm, ellip))
print "Sextractor statistics: nscr %d, fwhm (arcsec) %.2f, ellipticity %.2f" % (
nsrc, fwhm, ellip)
dic = {
"FWHM": np.round(fwhm, 3),
"FWHMPIX": np.round(fwhm / 0.394, 3),
"NSRC": nsrc,
"ELLIP": np.round(ellip, 3)
}
#Update the seeing information from sextractor
fitsutils.update_pars(image, dic)
#Compute the zeropoints
for image in slice_names:
zeropoint.calibrate_zeropoint(image)
return slice_names