コード例 #1
0
ファイル: realtimered.py プロジェクト: rswalters/sedmpy
def reduce_on_the_fly(photdir):
    '''
    Waits for new images to appear in the directory to trigger their incremental reduction as well.
    '''
    #Get the current the number of files

    nfiles = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
    logger.info(
        "Starting the on-the-fly reduction for directory %s. Found %d files to process."
        % (photdir, len(nfiles)))

    dayname = os.path.basename(photdir)

    time_ini = datetime.datetime.now()
    time_curr = datetime.datetime.now()

    #Run this loop for 12h since the start.
    while (time_curr - time_ini).total_seconds() < 12. * 3600:
        nfilesnew = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))

        if len(nfilesnew) == len(nfiles):
            time.sleep(10)
        else:
            new = [f for f in nfilesnew if f not in nfiles]
            new.sort()
            logger.info("Detected new %d incoming files in the last 10s." %
                        len(new))
            for n in new:

                if (not fitsutils.has_par(n, "IMGTYPE")):
                    print "Image", n, "Does not have an IMGTYPE"
                    time.sleep(0.5)
                    if (not fitsutils.has_par(n, "IMGTYPE")):
                        print "Image", n, "STILL Does not have an IMGTYPE"
                        continue

                plot_image(n)
                imtype = fitsutils.get_par(n, "IMGTYPE")
                if (imtype.upper() == "SCIENCE"):
                    reduced = rcred.reduce_image(n)
                    try:
                        zeropoint.calibrate_zeropoint(reduced)
                    except:
                        logger.error(
                            "Could not calibrate zeropoint for image %s" %
                            reduced)
                    #Copy them to transient
                    for r in reduced:
                        cmd = "rcp %s [email protected]:/scr3/mansi/ptf/p60phot/fremling_pipeline/sedm/reduced/%s/." % (
                            r, dayname)
                        subprocess.call(cmd, shell=True)
                        logger.info(cmd)
                        logger.info("Successfully copied the image: %s" % cmd)

        time_curr = datetime.datetime.now()
        nfiles = nfilesnew
コード例 #2
0
ファイル: realtimered.py プロジェクト: scizen9/kpy
def reduce_on_the_fly(photdir):
    '''
    Waits for new images to appear in the directory to trigger their incremental reduction as well.
    '''
    #Get the current the number of files
    nfiles = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
    
    dayname = os.path.basename(photdir)
    
    time_ini = datetime.datetime.now()
    time_curr = datetime.datetime.now()
    
    #Run this loop for 12h since the start.
    while (time_curr-time_ini).total_seconds() < 12.*3600:
        nfilesnew = glob.glob(os.path.join(photdir, "rc*[0-9].fits"))
        if len(nfilesnew) == len(nfiles):
            time.sleep(10)
        else:
            new = [f for f in nfilesnew if f not in nfiles]
            for n in new:
                if (not fitsutils.has_par(n, "IMGTYPE")):
                    print "Image",n,"Does not have an IMGTYPE"
                    time.sleep(0.5)
                    if (not fitsutils.has_par(n, "IMGTYPE")):
                        print "Image",n,"STILL Does not have an IMGTYPE"
                        continue
                if (fitsutils.get_par(n, "IMGTYPE")=="SCIENCE"):
                    reduced = rcred.reduce_image(n)
                    try:
                        zeropoint.calibrate_zeropoint(reduced)
                    except:
                        logger.error("Could not calibrate zeropoint for image %s"%reduced)
                    #Copy them to transient
                    for r in reduced:
                        cmd = "rcp %s [email protected]:/scr3/mansi/ptf/p60phot/fremling_pipeline/sedm/reduced/%s/."%(r, dayname)
                        subprocess.call(cmd, shell=True)
                        logger.info(cmd)
                        logger.info( "Successfully copied the image: %s"% cmd)
        time_curr = datetime.datetime.now()
        nfiles = nfilesnew  
コード例 #3
0
ファイル: rcred.py プロジェクト: scizen9/kpy
def reduce_image(image, flatdir=None, biasdir=None, cosmic=False, astrometry=True, channel='rc', target_dir='reduced', overwrite=False):
    '''
    Applies Flat field and bias calibrations to the image.
    
    Steps:
    
    1. - Solve astrometry on the entire image.
    2. - Computes cosmic ray rejectionon the entire image.
    3. - Compute master bias (if it does not exist) and de-bias the image.
    4. - Separate the image into 4 filters.
    5. - Compute flat field for each filter (if it does not exist) and apply flat fielding on the image.
    6. - Compute the image zeropoint.

    '''
    
    logger.info("Reducing image %s"% image)    

    print "Reducing image ", image    

    
    image = os.path.abspath(image)
    imname = os.path.basename(image).replace(".fits", "")
    try:
        objectname = fitsutils.get_par(image, "NAME").replace(" ","")+"_"+fitsutils.get_par(image, "FILTER")
    except:
        logger.error( "ERROR, image "+ image + " does not have a NAME or a FILTER!!!")
        return

    print "For object", objectname
    logger.info( "For object %s"% objectname)


    
    
    #Change to image directory
    mydir = os.path.dirname(image)
    if mydir=="": mydir = "."
    mydir = os.path.abspath(mydir)
    os.chdir(mydir)
    #Create destination directory
    if (not os.path.isdir(target_dir)):
        os.makedirs(target_dir)

    #If we don't want to overwrite the already extracted images, we check wether they exist.
    if (not overwrite):
        existing = True
        for band in ['u', 'g', 'r', 'i']:
            destfile = os.path.join(target_dir, imname + "_f_b_a_%s_%s_0.fits"%(objectname, band))
            logger.info( "Looking if file %s exists: %s"%( destfile, \
                (os.path.isfile(destfile) ) ) )
            existing = existing and (os.path.isfile( destfile ) )
        if existing:
            return []


        
    #Initialize the basic parameters.
    init_header_reduced(image)

    astro = ""
    if (astrometry):
        logger.info( "Solving astometry for the whole image...")
        img = solve_astrometry(image)
        if (os.path.isfile(img)):
            astro="a_"
            fitsutils.update_par(img, "IQWCS", 1)
        else:
            logger.error( "ASTROMETRY DID NOT SOLVE ON IMAGE %s"% image)
            img = image

    #Update noise parameters needed for cosmic reection
    if (fitsutils.get_par(img, "ADCSPEED")==2):
        fitsutils.update_par(img, "RDNOISE", 20.)
    else:
        fitsutils.update_par(img, "RDNOISE", 4.)

    if (cosmic):
        logger.info( "Correcting for cosmic rays...")
        # Correct for cosmics each filter
        cleanimg = clean_cosmic(os.path.join(os.path.abspath(mydir), img))
        img = cleanimg
    
    
    
    #Get basic statistics for the image
    nsrc, fwhm, ellip, bkg = sextractor.get_image_pars(img)
    
    logger.info( "Sextractor statistics: nscr %d, fwhm (pixel) %.2f, ellipticity %.2f"% (nsrc, fwhm, ellip))
    print "Sextractor statistics: nscr %d, fwhm (pixel) %.2f, ellipticity %.2f"% (nsrc, fwhm, ellip)

    
    dic = {"SEEPIX": fwhm/0.394, "NSRC":nsrc, "ELLIP":ellip}
    #Update the seeing information from sextractor
    fitsutils.update_pars(img, dic)
    
    
    #Compute BIAS
    if (biasdir is None or biasdir==""): biasdir = "."
    create_masterbias(biasdir)
    
    bias_slow = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'slow'))
    bias_fast = os.path.join(biasdir, "Bias_%s_%s.fits"%(channel, 'fast'))
    
    # Running IRAF to DE-BIAS
    iraf.noao(_doprint=0)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)
    
    #Compute flat field
    if (flatdir is None or flatdir==""): flatdir = "."
    create_masterflat(flatdir, biasdir)
    
    #New names for the object.
    debiased = os.path.join(os.path.dirname(img), "b_" + os.path.basename(img))
    logger.info( "Creating debiased file, %s"%debiased)
    
    if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
        or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
        logger.warn( "Master bias not found! Tryting to copy from reference folder...")
        copy_ref_calib(mydir, "Bias")
        if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
        or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
            logger.error( "Bias not found in reference folder")
            return


    #Clean first
    if (os.path.isfile(debiased)):
        os.remove(debiased)
        
    #Debias
    if (fitsutils.get_par(img, "ADCSPEED")==2):
        iraf.imarith(img, "-", bias_fast, debiased)
        fitsutils.update_par(debiased, "BIASFILE", bias_fast)
        fitsutils.update_par(debiased, "RDNOISE", 20.)

    else:
        iraf.imarith(img, "-", bias_slow, debiased)
        fitsutils.update_par(debiased, "BIASFILE", bias_slow)
        fitsutils.update_par(debiased, "RDNOISE", 4.)

    #Set negative counts to zero
    hdu = fits.open(debiased)
    header = hdu[0].header
    hdu[0].data[hdu[0].data<0] = 0
    hdu.writeto(debiased, clobber=True)

    #Slicing the image for flats  
    slice_names = slice_rc(debiased)
    print "Creating sliced files, ", slice_names

    
    #Remove un-sliced image
    os.remove(debiased)

    # DE-flat each filter and store under object name
    for i, debiased_f in enumerate(slice_names):
        b = fitsutils.get_par(debiased_f, 'filter')
        
        deflatted = os.path.join(os.path.dirname(image), target_dir, imname + "_f_b_" + astro + objectname + "_%s.fits"%b)

        #Flat to be used for that filter
        flat = os.path.join(flatdir, "Flat_%s_%s_norm.fits"%(channel, b))

        if (not os.path.isfile(flat)):
            logger.warn( "Master flat not found in %s"% flat)
            copy_ref_calib(mydir, "Flat_%s_%s_norm"%(channel, b))
            continue
        else:
            logger.info( "Using flat %s"%flat)
            
        #Cleans the deflatted file if exists
        if (os.path.isfile(deflatted)):
            os.remove(deflatted)

        if (os.path.isfile(debiased_f) and os.path.isfile(flat)):
            logger.info( "Storing de-flatted %s as %s"%(debiased_f, deflatted))
            time.sleep(1)
            iraf.imarith(debiased_f, "/", flat, deflatted)
        else:
            logger.error( "SOMETHING IS WRONG. Error when dividing %s by the flat field %s!"%(debiased_f, flat))
        
        #Removes the de-biased file
        os.remove(debiased_f)
        
        logger.info( "Updating header with original filename and flat field used.")
        fitsutils.update_par(deflatted, "ORIGFILE", os.path.basename(image))
        fitsutils.update_par(deflatted, "FLATFILE", flat)

        slice_names[i] = deflatted
            
                
    #Moving files to the target directory
    for image in slice_names:
        bkg = get_median_bkg(image)
        fitsutils.update_par(image, "SKYBKG", bkg)
        #shutil.move(name, newname)

        
    #Compute the zeropoints
    for image in slice_names:
        zeropoint.calibrate_zeropoint(image)
        
    return slice_names
コード例 #4
0
ファイル: rcred.py プロジェクト: rswalters/sedmpy
def reduce_image(image,
                 flatdir=None,
                 biasdir=None,
                 cosmic=False,
                 astrometry=True,
                 channel='rc',
                 target_dir='reduced',
                 overwrite=False):
    '''
    Applies Flat field and bias calibrations to the image.
    
    Steps:
    
    1. - Solve astrometry on the entire image.
    2. - Computes cosmic ray rejectionon the entire image.
    3. - Compute master bias (if it does not exist) and de-bias the image.
    4. - Separate the image into 4 filters.
    5. - Compute flat field for each filter (if it does not exist) and apply flat fielding on the image.
    6. - Compute the image zeropoint.

    '''

    logger.info("Reducing image %s" % image)

    print "Reducing image ", image

    image = os.path.abspath(image)
    imname = os.path.basename(image).replace(".fits", "")
    try:
        objectname = fitsutils.get_par(image, "NAME").replace(
            " ", "") + "_" + fitsutils.get_par(image, "FILTER")
    except:
        logger.error("ERROR, image " + image +
                     " does not have a NAME or a FILTER!!!")
        return

    print "For object", objectname
    logger.info("For object %s" % objectname)

    #Change to image directory
    mydir = os.path.dirname(image)
    if mydir == "": mydir = "."
    mydir = os.path.abspath(mydir)
    os.chdir(mydir)
    #Create destination directory
    if (not os.path.isdir(target_dir)):
        os.makedirs(target_dir)

    #If we don't want to overwrite the already extracted images, we check wether they exist.
    if (not overwrite):
        existing = True
        for band in ['u', 'g', 'r', 'i']:
            destfile = os.path.join(
                target_dir,
                imname + "_f_b_a_%s_%s_0.fits" % (objectname, band))
            logger.info( "Looking if file %s exists: %s"%( destfile, \
                (os.path.isfile(destfile) ) ) )
            existing = existing and (os.path.isfile(destfile))
        if existing:
            return []

    #Initialize the basic parameters.
    init_header_reduced(image)

    astro = ""
    if (astrometry):
        logger.info("Solving astometry for the whole image...")
        img = solve_astrometry(image)
        if (os.path.isfile(img)):
            astro = "a_"
            fitsutils.update_par(img, "IQWCS", 1)
        else:
            logger.error("ASTROMETRY DID NOT SOLVE ON IMAGE %s" % image)
            img = image

    #Update noise parameters needed for cosmic reection
    if (fitsutils.get_par(img, "ADCSPEED") == 2):
        fitsutils.update_par(img, "RDNOISE", 20.)
    else:
        fitsutils.update_par(img, "RDNOISE", 4.)

    if (cosmic):
        logger.info("Correcting for cosmic rays...")
        # Correct for cosmics each filter
        cleanimg = clean_cosmic(os.path.join(os.path.abspath(mydir), img))
        img = cleanimg

    #Compute BIAS
    if (biasdir is None or biasdir == ""): biasdir = "."
    create_masterbias(biasdir)

    bias_slow = os.path.join(biasdir, "Bias_%s_%s.fits" % (channel, 'slow'))
    bias_fast = os.path.join(biasdir, "Bias_%s_%s.fits" % (channel, 'fast'))

    # Running IRAF to DE-BIAS
    iraf.noao(_doprint=0)
    iraf.imred(_doprint=0)
    iraf.ccdred(_doprint=0)

    #Compute flat field
    if (flatdir is None or flatdir == ""): flatdir = "."
    create_masterflat(flatdir, biasdir)

    #New names for the object.
    debiased = os.path.join(os.path.dirname(img), "b_" + os.path.basename(img))
    logger.info("Creating debiased file, %s" % debiased)

    if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
        or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
        logger.warn(
            "Master bias not found! Tryting to copy from reference folder...")
        copy_ref_calib(mydir, "Bias")
        if ( (fitsutils.get_par(img, "ADCSPEED")==0.1 and not os.path.isfile(bias_slow)) \
        or (fitsutils.get_par(img, "ADCSPEED")==2 and not os.path.isfile(bias_fast)) ):
            logger.error("Bias not found in reference folder")
            return

    #Clean first
    if (os.path.isfile(debiased)):
        os.remove(debiased)

    #Debias
    if (fitsutils.get_par(img, "ADCSPEED") == 2):
        iraf.imarith(img, "-", bias_fast, debiased)
        fitsutils.update_par(debiased, "BIASFILE", bias_fast)
        fitsutils.update_par(debiased, "RDNOISE", 20.)

    else:
        iraf.imarith(img, "-", bias_slow, debiased)
        fitsutils.update_par(debiased, "BIASFILE", bias_slow)
        fitsutils.update_par(debiased, "RDNOISE", 4.)

    #Set negative counts to zero
    hdu = fits.open(debiased)
    header = hdu[0].header
    hdu[0].data[hdu[0].data < 0] = 0
    hdu.writeto(debiased, clobber=True)

    #Slicing the image for flats
    slice_names = slice_rc(debiased)
    print "Creating sliced files, ", slice_names

    #Remove un-sliced image
    os.remove(debiased)

    # DE-flat each filter and store under object name
    for i, debiased_f in enumerate(slice_names):
        b = fitsutils.get_par(debiased_f, 'filter')

        deflatted = os.path.join(
            os.path.dirname(image), target_dir,
            imname + "_f_b_" + astro + objectname + "_%s.fits" % b)

        #Flat to be used for that filter
        flat = os.path.join(flatdir, "Flat_%s_%s_norm.fits" % (channel, b))

        if (not os.path.isfile(flat)):
            logger.warn("Master flat not found in %s" % flat)
            copy_ref_calib(mydir, "Flat")
            continue
        else:
            logger.info("Using flat %s" % flat)

        #Cleans the deflatted file if exists
        if (os.path.isfile(deflatted)):
            os.remove(deflatted)

        if (os.path.isfile(debiased_f) and os.path.isfile(flat)):
            logger.info("Storing de-flatted %s as %s" %
                        (debiased_f, deflatted))
            time.sleep(1)
            iraf.imarith(debiased_f, "/", flat, deflatted)
        else:
            logger.error(
                "SOMETHING IS WRONG. Error when dividing %s by the flat field %s!"
                % (debiased_f, flat))

        #Removes the de-biased file
        os.remove(debiased_f)

        logger.info(
            "Updating header with original filename and flat field used.")
        fitsutils.update_par(deflatted, "ORIGFILE", os.path.basename(image))
        fitsutils.update_par(deflatted, "FLATFILE", flat)

        slice_names[i] = deflatted

    #Moving files to the target directory
    for image in slice_names:
        bkg = get_median_bkg(image)
        fitsutils.update_par(image, "SKYBKG", bkg)

        #Get basic statistics for the image
        nsrc, fwhm, ellip, bkg = sextractor.get_image_pars(image)
        plot_image(image)

        logger.info(
            "Sextractor statistics: nscr %d, fwhm (arcsec) %.2f, ellipticity %.2f"
            % (nsrc, fwhm, ellip))
        print "Sextractor statistics: nscr %d, fwhm (arcsec) %.2f, ellipticity %.2f" % (
            nsrc, fwhm, ellip)

        dic = {
            "FWHM": np.round(fwhm, 3),
            "FWHMPIX": np.round(fwhm / 0.394, 3),
            "NSRC": nsrc,
            "ELLIP": np.round(ellip, 3)
        }
        #Update the seeing information from sextractor
        fitsutils.update_pars(image, dic)

    #Compute the zeropoints
    for image in slice_names:
        zeropoint.calibrate_zeropoint(image)

    return slice_names