Exemple #1
0
def pysalt(fs=None):
    # Run the pysalt pipeline on the raw data.
    if fs is None:
        fs = glob('P*.fits')
    if len(fs) == 0:
        print "WARNING: No raw files to run PySALT pre-processing."
        return

    # Copy the raw files into a raw directory
    if not os.path.exists('raw'):
        os.mkdir('raw')
    if not os.path.exists('work'):
        os.mkdir('work')
    for f in fs:
        shutil.copy2(f, 'raw/')
        shutil.move(f, 'work/')
    iraf.cd('work')

    # Run each of the pysalt pipeline steps deleting temporary files as we go
    # saltprepare
    iraf.unlearn(iraf.saltprepare)
    # Currently, there is not a bad pixel mask provided by SALT
    # so we don't create one here.
    iraf.saltprepare(images='P*.fits', clobber=True, mode='h')

    for f in glob('P*.fits'):
        os.remove(f)
    # saltgain
    iraf.unlearn(iraf.saltgain)
    # Multiply by the gain so that everything is in electrons.
    iraf.saltgain(images='pP*.fits',
                  gaindb=pysaltpath + '/data/rss/RSSamps.dat',
                  mult=True, usedb=True, mode='h')

    for f in glob('pP*.fits'):
        os.remove(f)

    # write a keyword in the header keyword gain = 1 in each amplifier
    fs = glob('gpP*.fits')
    for f in fs:
        for i in range(1, 7):
            pyfits.setval(f, 'GAIN', ext=i, value=1.0)

    # saltxtalk
    iraf.unlearn(iraf.saltxtalk)
    iraf.saltxtalk(images='gpP*.fits', clobber=True, usedb=True,
                   xtalkfile=pysaltpath + '/data/rss/RSSxtalk.dat', mode='h')
    for f in glob('gpP*.fits'):
        os.remove(f)

    # saltbias
    iraf.unlearn(iraf.saltbias)
    iraf.saltbias(images='xgpP*.fits', clobber=True, mode='h')
    for f in glob('xgpP*.fits'):
        os.remove(f)

    # Put all of the newly created files into the pysalt directory
    if not os.path.exists('pysalt'):
        os.mkdir('pysalt')
    for f in glob('bxgpP*.fits'):
        shutil.move(f, 'pysalt')
    iraf.cd('..')
Exemple #2
0
def pipeline(rawdir="raw", mode="halpha"):
    """Runs successive steps of the saltfp data reduction, checking along the
    way to see if each step was successful. This is the main driver program of
    the SALT Fabry-Perot pipeline.

    Inputs:
    rawdir -> String, containing the path to the 'raw' directory. By
                  default, this is 'raw'
    mode -> Mode for velocity fitting. Currently the only option is H-Alpha
                line fitting.

    """

    # Set rest wave based on the mode called
    if mode == "halpha":
        rest_wave = 6562.81

    # Create product directory
    if isdir("product"):
        while True:
            yn = raw_input("Product directory already exists. " + "Recreate it? (y/n) ")
            if "n" in yn or "N" in yn:
                break
            elif "y" in yn or "Y" in yn:
                # Confirmation
                yn = raw_input("Are you sure? This takes a while. (y/n) ")
                if ("y" in yn or "Y" in yn) and not ("n" in yn or "N" in yn):
                    rmtree("product")
                    break

    if not isdir("product"):
        # Acquire the list of filenames from the raw directory
        fnlist = sorted(listdir(rawdir))
        for i in range(len(fnlist)):
            fnlist[i] = join(rawdir, fnlist[i])
        # Run the first two steps of imred on the first image
        iraf.pysalt(_doprint=0)
        iraf.saltred(_doprint=0)
        iraf.saltprepare(
            fnlist[0],
            "temp.fits",
            "",
            createvar=False,
            badpixelimage="",
            clobber=True,
            logfile="temp.log",
            verbose=True,
        )
        iraf.saltbias(
            "temp.fits",
            "temp.fits",
            "",
            subover=True,
            trim=True,
            subbias=False,
            masterbias="",
            median=False,
            function="polynomial",
            order=5,
            rej_lo=3.0,
            rej_hi=5.0,
            niter=10,
            plotover=False,
            turbo=False,
            clobber=True,
            logfile="temp.log",
            verbose=True,
        )
        # Create the bad pixel mask
        image = fits.open("temp.fits")
        for i in range(1, len(image)):
            mask = image[i].data != image[i].data
            image[i].data = 1 * mask
        image.writeto("badpixmask.fits", clobber="True")
        image.close()
        # Remove temporary files
        remove("temp.fits")
        remove("temp.log")
        # Run the raw images through the first few data reduction pipeline
        # steps
        imred(fnlist, "product", bpmfile="badpixmask.fits")
        # Delete the temporary bad pixel mask
        remove("badpixmask.fits")
        # Move these raw images into the product directory
        mkdir("product")
        fnlist = sorted(listdir("."))
        for i in range(len(fnlist)):
            if "mfxgbpP" in fnlist[i] and ".fits" in fnlist[i]:
                move(fnlist[i], join("product", fnlist[i]))
    # List of files in the product directory
    fnlist = sorted(listdir("product"))
    for i in range(len(fnlist)):
        fnlist[i] = join("product", fnlist[i])

    # Manual verification of fits images and headers
    firstimage = FPImage(fnlist[0])
    verify = firstimage.verifytog
    firstimage.close()
    if verify is None:
        while True:
            prompt = "Manually verify image contents? (Recommended) (y/n) "
            yn = raw_input(prompt)
            if "n" in yn or "N" in yn:
                print ("Skipping manual verification of image contents " + "(Not recommended)")
                break
            if "y" in yn or "Y" in yn:
                fnlist = verify_images(fnlist)
                break

    # Make separate lists of the different fits files
    (flatlist, list_of_objs, objlists, list_of_filts, filtlists) = separate_lists(fnlist)

    # Masking of pixels outside the aperture
    firstimage = FPImage(objlists[0][0])
    axcen = firstimage.axcen
    firstimage.close()
    if axcen is None:
        print "Masking pixels outside the RSS aperture..."
        axcen, aycen, arad = get_aperture(objlists[0][0])
        aperture_mask(fnlist, axcen, aycen, arad)
    else:
        print "Images have already been aperture-masked."

    # Masking bad pixels from external region file
    for objlist in objlists:
        for i in range(len(objlist)):
            if isfile(splitext(split(objlist[i])[1])[0] + ".reg"):
                print ("Adding regions from file " + splitext(split(objlist[i])[1])[0] + ".reg to the bad pixel mask.")
                mask_regions(objlist[i], splitext(split(objlist[i])[1])[0] + ".reg")

    # Measure stellar FWHMs
    firstimage = FPImage(objlists[0][0])
    fwhm = firstimage.fwhm
    firstimage.close()
    if fwhm is None:
        dofwhm = True
    else:
        while True:
            yn = raw_input("Seeing FWHM has already been measured. " + "Redo this? (y/n) ")
            if "n" in yn or "N" in yn:
                dofwhm = False
                break
            elif "y" in yn or "Y" in yn:
                dofwhm = True
                break
    if dofwhm:
        print "Measuring seeing FWHMs..."
        for objlist in objlists:
            measure_fwhm(objlist)

    # Find image centers using ghost pairs
    for i in range(len(objlists)):
        firstimage = FPImage(objlists[i][0])
        xcen = firstimage.xcen
        deghosted = firstimage.ghosttog
        firstimage.close()
        if deghosted is None:
            if xcen is None:
                ghosttog = True
            else:
                while True:
                    yn = raw_input(
                        "Optical centers already measured for " + "object " + list_of_objs[i] + ". Redo this? (y/n) "
                    )
                    if "n" in yn or "N" in yn:
                        ghosttog = False
                        break
                    elif "y" in yn or "Y" in yn:
                        ghosttog = True
                        break
            if ghosttog:
                print (
                    "Identifying optical centers for object "
                    + list_of_objs[i]
                    + ". This may take a while for crowded fields..."
                )
                find_ghost_centers(objlists[i])

    # Deghost images
    for i in range(len(objlists)):
        firstimage = FPImage(objlists[i][0])
        deghosted = firstimage.ghosttog
        firstimage.close()
        if deghosted is None:
            print "Deghosting images for object " + list_of_objs[i] + "..."
            for j in range(len(objlists[i])):
                deghost(objlists[i][j])
        else:
            print ("Images for object " + list_of_objs[i] + " have already been deghosted.")

    # Image Flattening
    firstimage = FPImage(objlists[0][0])
    flattog = firstimage.flattog
    firstimage.close()
    if flattog is None:
        print "Flattening images..."
        if len(flatlist) == 0:
            while True:
                print "Uh oh! No flatfield exposure found!"
                flatpath = raw_input("Enter path to external flat image: " + "(leave blank to skip flattening) ")
                if flatpath == "" or isfile(flatpath):
                    break
        else:
            combine_flat(flatlist, "flat.fits")
            flatpath = "flat.fits"
        if flatpath != "":
            notflatlist = []
            for objlist in objlists:
                notflatlist += objlist
            flatten(notflatlist, flatpath)
        else:
            print "Skipping image flattening. (Not recommended!)"
    else:
        print "Images have already been flattened."

    # Make separate directories for each object.
    # This is the first bit since 'singext' to create a new directory, because
    # this is the first point where it's really necessary to start treating the
    # images from different tracks very differently.
    for i in range(len(objlists)):
        if isdir(list_of_objs[i].replace(" ", "")):
            while True:
                yn = raw_input("A directory for object " + list_of_objs[i] + " already exists. Recreate? (y/n) ")
                if "n" in yn or "N" in yn:
                    do_copy = False
                    break
                elif "y" in yn or "Y" in yn:
                    do_copy = True
                    rmtree(list_of_objs[i].replace(" ", ""))
                    break
        else:
            do_copy = True
        if do_copy:
            mkdir(list_of_objs[i].replace(" ", ""))
            for j in range(len(objlists[i])):
                copyfile(objlists[i][j], join(list_of_objs[i].replace(" ", ""), split(objlists[i][j])[1]))
        for j in range(len(objlists[i])):
            objlists[i][j] = join(list_of_objs[i].replace(" ", ""), split(objlists[i][j])[1])
    # Update the filter lists
    for i in range(len(filtlists)):
        for j in range(len(filtlists[i])):
            for k in range(len(objlists)):
                for l in range(len(objlists[k])):
                    if split(filtlists[i][j])[1] == split(objlists[k][l])[1]:
                        filtlists[i][j] = objlists[k][l]

    # Image alignment and normalization
    for i in range(len(objlists)):
        firstimage = FPImage(objlists[i][0])
        aligned = firstimage.phottog
        firstimage.close()
        if aligned is None:
            print ("Aligning and normalizing images for object " + list_of_objs[i] + "...")
            align_norm(objlists[i])
        else:
            print ("Images for object " + list_of_objs[i] + " have already been aligned and normalized.")

    # Make a median image for each object
    for i in range(len(objlists)):
        if isfile(join(list_of_objs[i].replace(" ", ""), "median.fits")):
            while True:
                yn = raw_input("Median image for object " + list_of_objs[i] + " already exists. Replace it? (y/n) ")
                if "n" in yn or "N" in yn:
                    break
                elif "y" in yn or "Y" in yn:
                    make_median(objlists[i], join(list_of_objs[i].replace(" ", ""), "median.fits"))
                    break
        else:
            make_median(objlists[i], join(list_of_objs[i].replace(" ", ""), "median.fits"))

    # Wavelength calibrations
    all_rings_list = []
    for i in range(len(list_of_filts)):
        all_rings_list = all_rings_list + filtlists[i]
    firstimage = FPImage(all_rings_list[0])
    calf = firstimage.calf
    firstimage.close()
    if not (calf is None):
        while True:
            yn = raw_input("Wavelength solution already found. " + "Redo it? (y/n) ")
            if "n" in yn or "N" in yn:
                break
            elif "y" in yn or "Y" in yn:
                fit_wave_soln(all_rings_list)
                break
    else:
        fit_wave_soln(all_rings_list)

    # Sky ring removal
    for i in range(len(objlists)):
        for j in range(len(objlists[i])):
            # Check to see if sky rings have already been removed
            image = FPImage(objlists[i][j])
            deringed = image.ringtog
            image.close()
            if deringed is None:
                print "Subtracting sky rings for image " + objlists[i][j]
                sub_sky_rings([objlists[i][j]], [join(list_of_objs[i].replace(" ", ""), "median.fits")])
            else:
                print ("Sky ring subtraction already done for image " + objlists[i][j])

    # Creation of data cube and convolution to uniform PSF
    for i in range(len(objlists)):
        if isdir(list_of_objs[i].replace(" ", "") + "_cube"):
            while True:
                yn = raw_input("A data cube for object " + list_of_objs[i] + " already exists. Recreate? (y/n) ")
                if "n" in yn or "N" in yn:
                    do_create = False
                    break
                elif "y" in yn or "Y" in yn:
                    # Confirmation
                    yn = raw_input("Are you sure? This takes a while. (y/n) ")
                    if ("y" in yn or "Y" in yn) and not ("n" in yn or "N" in yn):
                        do_create = True
                        rmtree(list_of_objs[i].replace(" ", "") + "_cube")
                        break
        else:
            do_create = True
        if do_create:
            mkdir(list_of_objs[i].replace(" ", "") + "_cube")
            for j in range(len(objlists[i])):
                image = FPImage(objlists[i][j])
                fwhm = image.fwhm
                if j == 0:
                    largestfwhm = fwhm
                if fwhm > largestfwhm:
                    largestfwhm = fwhm
                image.close()
            while True:
                prompt = "Enter desired final fwhm or leave blank to use" + " default (" + str(largestfwhm) + " pix) "
                user_fwhm = raw_input(prompt)
                if user_fwhm == "":
                    user_fwhm = largestfwhm
                    break
                else:
                    try:
                        user_fwhm = float(user_fwhm)
                    except ValueError:
                        print "That wasn't a valid number..."
                    else:
                        if user_fwhm < largestfwhm:
                            print ("Final fwhm must exceed " + str(largestfwhm) + " pixels.")
                        else:
                            break
            desired_fwhm = user_fwhm * 1.01
            for j in range(len(objlists[i])):
                make_final_image(
                    objlists[i][j],
                    join(list_of_objs[i].replace(" ", "") + "_cube", split(objlists[i][j])[1]),
                    desired_fwhm,
                    clobber=True,
                )

    # Get final lists for the velocity map fitting for each object
    final_lists = []
    for i in range(len(list_of_objs)):
        final_lists.append([])
        for j in range(len(objlists[i])):
            final_lists[i].append(join(list_of_objs[i].replace(" ", "") + "_cube", split(objlists[i][j])[1]))

    # Shift to solar velocity frame
    for i in range(len(list_of_objs)):
        firstimage = FPImage(final_lists[i][0])
        velshift = firstimage.solarvel
        firstimage.close()
        if velshift is None:
            print ("Performing solar velocity shift for object " + list_of_objs[i] + "...")
            solar_velocity_shift(final_lists[i], rest_wave)
        else:
            print ("Solar velocity shift for object " + list_of_objs[i] + " already done.")

    if not do_velmap:
        sys.exit("Velocity map not made - Voigt-fitting software not found.")

    # Velocity map fitting
    for i in range(len(list_of_objs)):
        if isfile(join(list_of_objs[i].replace(" ", "") + "_cube", "velocity.fits")):
            while True:
                yn = raw_input("Velocity map already fitted for object " + list_of_objs[i] + ". Redo this? (y/n) ")
                if "n" in yn or "N" in yn:
                    domap = False
                    break
                elif "y" in yn or "Y" in yn:
                    # Confirmation
                    yn = raw_input("Are you sure? This takes a while. (y/n) ")
                    if ("y" in yn or "Y" in yn) and not ("n" in yn or "N" in yn):
                        domap = True
                        break
        else:
            domap = True
    if domap:
        print "Fitting velocity map for object " + list_of_objs[i] + "..."
        if mode == "halpha":
            fit_velmap_ha_n2_mode(final_lists[i], list_of_objs[i].replace(" ", "") + "_cube", clobber=True)

    # Clean velocity map
    for i in range(len(list_of_objs)):
        make_clean_map(list_of_objs[i].replace(" ", "") + "_cube", clobber=True)
Exemple #3
0
def pysalt(fs=None):
    # Run the pysalt pipeline on the raw data.
    if fs is None:
        fs = glob('P*.fits')
    if len(fs) == 0:
        print "WARNING: No raw files to run PySALT pre-processing."
        return

    # Copy the raw files into a raw directory
    if not os.path.exists('raw'):
        os.mkdir('raw')
    if not os.path.exists('work'):
        os.mkdir('work')
    for f in fs:
        shutil.copy2(f, 'raw/')
        shutil.move(f, 'work/')
    iraf.cd('work')

    # Run each of the pysalt pipeline steps deleting temporary files as we go
    # saltprepare
    iraf.unlearn(iraf.saltprepare)
    # Currently, there is not a bad pixel mask provided by SALT
    # so we don't create one here.
    iraf.saltprepare(images='P*.fits', clobber=True, mode='h')

    for f in glob('P*.fits'):
        os.remove(f)
    # saltgain
    iraf.unlearn(iraf.saltgain)
    # Multiply by the gain so that everything is in electrons.
    iraf.saltgain(images='pP*.fits',
                  gaindb=pysaltpath + '/data/rss/RSSamps.dat',
                  mult=True, usedb=True, mode='h')

    for f in glob('pP*.fits'):
        os.remove(f)

    # write a keyword in the header keyword gain = 1 in each amplifier
    fs = glob('gpP*.fits')
    for f in fs:
        for i in range(1, 7):
            pyfits.setval(f, 'GAIN', ext=i, value=1.0)

    # saltxtalk
    iraf.unlearn(iraf.saltxtalk)
    iraf.saltxtalk(images='gpP*.fits', clobber=True, usedb=True,
                   xtalkfile=pysaltpath + '/data/rss/RSSxtalk.dat', mode='h')
    for f in glob('gpP*.fits'):
        os.remove(f)

    # saltbias
    iraf.unlearn(iraf.saltbias)
    iraf.saltbias(images='xgpP*.fits', clobber=True, mode='h')
    for f in glob('xgpP*.fits'):
        os.remove(f)

    # Put all of the newly created files into the pysalt directory
    if not os.path.exists('pysalt'):
        os.mkdir('pysalt')
    for f in glob('bxgpP*.fits'):
        shutil.move(f, 'pysalt')
    iraf.cd('..')