Esempio n. 1
0
def full_run(f_target, f_ref, f_cont, work_dir, clobber):

    err = errors()

    # input sanity checks
    if not all([f_target, f_ref, f_cont]):
        err.set_code(1)
    elif not os.path.exists(f_target):
        err.set_code(2)   
    elif not os.path.exists(f_ref):
        err.set_code(3)   
    elif not os.path.exists(f_cont):	
        err.set_code(4)    
      
    # get basename of files
    target              = os.path.splitext(os.path.basename(f_target))[0]
    ref                 = os.path.splitext(os.path.basename(f_ref))[0]
    cont                = os.path.splitext(os.path.basename(f_cont))[0]          

    # OUTPUT
    ## L2
    output_target       = target[:-1] + "2.fits"       
    
    ## plots
    ref_pre_sdist_plot  	= "ref_pre_sdist_plot.png"
    ref_post_sdist_plot 	= "ref_post_sdist_plot.png"
    L1_IMAGE_plot	        = "L1_IMAGE.png"
    SPEC_NONSS_plot	        = os.path.splitext(os.path.basename(output_target))[0] + "_SPEC_NONSS.png"
    SPEC_SS_plot                = os.path.splitext(os.path.basename(output_target))[0] + "_SPEC_SS.png"
    montage_plot                = os.path.splitext(os.path.basename(output_target))[0] + "_output.png"  

    # move files to working directory, redefine paths and change to working directory
    try:
        if not os.path.exists(work_dir):
            os.mkdir(work_dir)
            copyfile(f_target, work_dir + "/" + target + target_suffix + ".fits")
            copyfile(f_ref, work_dir + "/" + ref + ref_suffix + ".fits")
            copyfile(f_cont, work_dir + "/" + cont + cont_suffix + ".fits") 
        else:
            if clobber:
                for i in os.listdir(work_dir):
	            os.remove(work_dir + "/" + i)
                copyfile(f_target, work_dir + "/" + target + target_suffix + ".fits")
                copyfile(f_ref, work_dir + "/" + ref + ref_suffix + ".fits")
                copyfile(f_cont, work_dir + "/" + cont + cont_suffix + ".fits")
            else:
	        err.set_code(14)
    except OSError:
        err.set_code(15)
    
    f_target = target + target_suffix + ".fits"
    f_ref = ref + ref_suffix + ".fits"
    f_cont = cont + cont_suffix + ".fits"

    os.chdir(work_dir)
    
    # ---------
    # - START -
    # ---------  
    
    today = date.today()
    now_date = today.strftime("%d-%m-%y")
    now_time = time.strftime("%H:%M:%S")
    # add L2DATE key to additional_keys
    with open("additional_keys", 'w') as f:
        f.write("str\tSTARTDATE\tL2DATE\t" + now_date + " " + now_time + "\twhen this reduction was performed\n")     
        
    st_unix = time.time()    
    
    # ----------------------------------
    # - DETERMINE SUITABLE CONFIG FILE -
    # ----------------------------------
    
    print_routine("Finding suitable config files.")
    print 
    
    f_ref_fits = pyfits.open(f_ref)
    try:
        f_ref_fits_hdr_DATEOBS = f_ref_fits[0].header['DATE-OBS'].strip()
        f_ref_fits_hdr_CCDXBIN = str(f_ref_fits[0].header['CCDXBIN']).strip()
        f_ref_fits_hdr_CCDYBIN = str(f_ref_fits[0].header['CCDYBIN']).strip()       
    except KeyError:
        f_ref_fits.close()
        err.set_code(16)
    f_ref_fits.close()          
        
    config_tab_path      = L2_LOOKUP_TABLES_DIR + "/" + "/config.tab"
    
    chosen_config_entry = search_lookup_table(config_tab_path, f_ref_fits_hdr_DATEOBS, f_ref_fits_hdr_CCDXBIN, f_ref_fits_hdr_CCDYBIN)
    if chosen_config_entry is None:
        print_notification("Failed.") 
        err.set_code(29)
        
    chosen_config_file_path = L2_INI_DIR + "/" + "/" + chosen_config_entry

    print_notification("Success. Using file " + chosen_config_file_path)    
    cfg = read_ini(err, chosen_config_file_path)    
         
    # -------------------------
    # - CLIP SPECTRA (LOCLIP) -
    # -------------------------
    
    print_routine("Trim spectra (lotrim)")
    
    in_target_filename = f_target
    in_ref_filename = f_ref
    in_cont_filename = f_cont
    out_target_filename = target + target_suffix + trim_suffix + ".fits"
    out_ref_filename = ref + ref_suffix + trim_suffix + ".fits"
    out_cont_filename = cont + cont_suffix + trim_suffix + ".fits"

    ## target
    output = Popen([clip, in_cont_filename, in_target_filename, cfg['loclip']['bin_size_px'], cfg['loclip']['bg_percentile'], cfg['loclip']['clip_sigma'], cfg['loclip']['thresh_sigma'], \
      cfg['loclip']['scan_window_size_px'], cfg['loclip']['scan_window_nsigma'], cfg['loclip']['min_spectrum_width_px'], cfg['loclip']['force_bottom_px'], cfg['loclip']['force_top_px'], \
        out_target_filename], stdout=PIPE)  
    print output.stdout.read()
    output.wait()
    if output.returncode != 0:
        err.set_code(20, is_fatal=False)    
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATCL", "L2STATCT", add_to_desc="(target)")

    ## reference
    output = Popen([clip, in_cont_filename, in_ref_filename, cfg['loclip']['bin_size_px'], cfg['loclip']['bg_percentile'], cfg['loclip']['clip_sigma'], cfg['loclip']['thresh_sigma'], \
      cfg['loclip']['scan_window_size_px'], cfg['loclip']['scan_window_nsigma'], cfg['loclip']['min_spectrum_width_px'], cfg['loclip']['force_bottom_px'], cfg['loclip']['force_top_px'], \
        out_ref_filename], stdout=PIPE)   
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(20, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATCL", "L2STATCR", add_to_desc="(ref)")
    
    ## continuum
    output = Popen([clip, in_cont_filename, in_cont_filename, cfg['loclip']['bin_size_px'], cfg['loclip']['bg_percentile'], cfg['loclip']['clip_sigma'], cfg['loclip']['thresh_sigma'], \
      cfg['loclip']['scan_window_size_px'], cfg['loclip']['scan_window_nsigma'], cfg['loclip']['min_spectrum_width_px'], cfg['loclip']['force_bottom_px'], cfg['loclip']['force_top_px'], \
        out_cont_filename], stdout=PIPE)
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(20, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATCL", "L2STATCC", add_to_desc="(continuum)")

    # ---------------------------------------------
    # - FIND PEAKS OF REFERENCE SPECTRUM (LOFIND) -
    # ---------------------------------------------
    
    print_routine("Find peaks of reference spectrum (lofind)")   
    
    in_ref_filename = ref + ref_suffix + trim_suffix + ".fits"

    output = Popen([find, in_ref_filename, cfg['lofind_ref']['bin_size_px'], cfg['lofind_ref']['bg_percentile'], cfg['lofind_ref']['clip_sigma'], \
      cfg['lofind_ref']['median_filter_width_px'], cfg['lofind_ref']['min_snr'], cfg['lofind_ref']['min_spatial_width_px'], cfg['lofind_ref']['finding_window_lo_px'], \
        cfg['lofind_ref']['finding_window_hi_px'], cfg['lofind_ref']['max_centering_num_px'], cfg['lofind_ref']['centroid_half_window_size_px'], \
          cfg['lofind_ref']['min_used_bins'], cfg['lofind_ref']['window_x_lo'], cfg['lofind_ref']['window_x_hi']], stdout=PIPE)
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(21, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATFI", "L2STATF1", add_to_desc="(ref uncorrected)")

    # ----------------------------------------------
    # - FIND SDIST OF REFERENCE SPECTRUM (LOTRACE) -
    # ----------------------------------------------
    
    print_routine("Find sdist of reference spectrum (lotrace)") 
    
    output = Popen([trace, cfg['lotrace']['polynomial_order']], stdout=PIPE)
    print output.stdout.read()  
    output.wait()
    if output.returncode != 0:
        err.set_code(22, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATTR", "L2STATT1", add_to_desc="(ref uncorrected)")
  
    # ---------------------------------------------------------
    # - PLOT TRACE OF REFERENCE SPECTRUM PRE SDIST CORRECTION -
    # ---------------------------------------------------------
    
    print_routine("Plot trace of reference spectrum pre sdist correction") 
    print
    
    in_ref_filename = ref + ref_suffix + trim_suffix + ".fits"
    
    try:
        a_pp_execute(in_ref_filename, cfg['general']['lofind_output_file'], cfg['general']['lotrace_output_file'], ref_pre_sdist_plot, "Reference pre SDIST correction", \
      cfg['general']['max_curvature_post_cor']) 
    except IOError:
        pass

    if os.path.exists(ref_pre_sdist_plot):
        print_notification("Success.")
    else:
        print_notification("Failed.") 
        err.set_code(6, is_fatal=False)

    # -----------------------------------------
    # - CORRECT SPECTRA FOR SDIST (LOCORRECT) -
    # -----------------------------------------
    
    print_routine("Correct spectra for sdist (locorrect)")      

    in_target_filename = target + target_suffix + trim_suffix + ".fits"
    in_ref_filename = ref + ref_suffix + trim_suffix + ".fits"
    in_cont_filename = cont + cont_suffix + trim_suffix + ".fits"

    out_target_filename = target + target_suffix + trim_suffix + cor_suffix + ".fits"
    out_ref_filename = ref + ref_suffix + trim_suffix + cor_suffix + ".fits"
    out_cont_filename = cont + cont_suffix + trim_suffix + cor_suffix + ".fits"

    ## target
    output = Popen([correct, in_target_filename, cfg['locorrect']['interpolation_type'], cfg['locorrect']['conserve_flux'], out_target_filename], stdout=PIPE)
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(23, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATCO", "L2STATOT", add_to_desc="(target)")

    ## reference
    output = Popen([correct, in_ref_filename, cfg['locorrect']['interpolation_type'], cfg['locorrect']['conserve_flux'], out_ref_filename], stdout=PIPE)
    print output.stdout.read()  
    output.wait()
    if output.returncode != 0:
        err.set_code(23, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATCO", "L2STATOR", add_to_desc="(ref)")
    
    ## continuum
    output = Popen([correct, in_cont_filename, "linear", "1", out_cont_filename], stdout=PIPE)
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(23, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATCO", "L2STATOC", add_to_desc="(continuum)")
   
    # rename dat files to avoid conflict
    rename_dat_files("ref_uncorrected")

    # ---------------------------------------------
    # - FIND PEAKS OF REFERENCE SPECTRUM (LOFIND) -
    # ---------------------------------------------
    
    print_routine("Find peaks of reference spectrum (lofind)")       
    
    in_ref_filename = ref + ref_suffix + trim_suffix + cor_suffix + ".fits"

    output = Popen([find, in_ref_filename, cfg['lofind_ref']['bin_size_px'], cfg['lofind_ref']['bg_percentile'], cfg['lofind_ref']['clip_sigma'], \
      cfg['lofind_ref']['median_filter_width_px'], cfg['lofind_ref']['min_snr'], cfg['lofind_ref']['min_spatial_width_px'], cfg['lofind_ref']['finding_window_lo_px'], \
        cfg['lofind_ref']['finding_window_hi_px'], cfg['lofind_ref']['max_centering_num_px'], cfg['lofind_ref']['centroid_half_window_size_px'], \
          cfg['lofind_ref']['min_used_bins'], cfg['lofind_ref']['window_x_lo'], cfg['lofind_ref']['window_x_hi']], stdout=PIPE)
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(21, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATFI", omit=True)

    # ----------------------------------------------
    # - FIND SDIST OF REFERENCE SPECTRUM (LOTRACE) -
    # ----------------------------------------------
    
    print_routine("Find sdist of reference spectrum (lotrace)")       
    
    output = Popen([trace, cfg['lotrace']['polynomial_order']], stdout=PIPE)
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(22, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATTR", omit=True)

    # ----------------------------------------------------------------------------------------------------------
    # - PLOT TRACE OF SPECTRUM POST SDIST CORRECTION AND CHECK RETURN CODE FOR SIGNIFICANT REMAINING CURVATURE -
    # ----------------------------------------------------------------------------------------------------------
    
    print_routine("Plot trace of spectrum post sdist correction and check for curvature (l2pp)") 
    print
    
    in_ref_filename = ref + ref_suffix + trim_suffix + cor_suffix + ".fits"
    
    try:
        rtn = a_pp_execute(in_ref_filename, cfg['general']['lofind_output_file'], cfg['general']['lotrace_output_file'], ref_post_sdist_plot, "Reference post SDIST correction", \
        cfg['general']['max_curvature_post_cor']) 
    except IOError:
        rtn = -1
    
    if os.path.exists(ref_post_sdist_plot) and rtn == 0:
        print_notification("Success.")
    elif not os.path.exists(ref_post_sdist_plot):
        print_notification("Failed.") 
        err.set_code(7, is_fatal=False)   
    elif rtn != 0:
        print_notification("Failed.") 
        err.set_code(8) 
        
    # rename dat files to avoid conflict
    rename_dat_files("ref_corrected")       

    # RJS start
    copyfile(target + target_suffix + trim_suffix + ".fits", target + target_suffix + trim_suffix + cor_suffix + ".fits" )
    # RJS end

    # -------------------------------------------------
    # - FIND PIXEL TO WAVELENGTH SOLUTIONS (loarcfit) -
    # -------------------------------------------------
    
    print_routine("Copy over hardcoded dispersion solution from solutions file")   

    copyfile(L2_CONFIG_DIR + "/ARC_SOLUTION.dat", "./loarcfit_wavfits.dat")

    print_notification("\nSuccess.")

    # -------------------
    # - REBIN (lorebin) -
    # -------------------
    
    print_routine("Rebin data spectrally (lorebin)") 
    
    in_target_filename = target + target_suffix + trim_suffix + cor_suffix + ".fits"
    out_target_filename = target + target_suffix + trim_suffix + cor_suffix + reb_suffix + ".fits"

    output = Popen([rebin, in_target_filename, cfg['lorebin']['start_wav'], cfg['lorebin']['end_wav'], cfg['lorebin']['interpolation_type'], cfg['lorebin']['dispersion'], \
       cfg['lorebin']['conserve_flux'], out_target_filename], stdout=PIPE)
    print output.stdout.read()  
    output.wait()
    if output.returncode != 0:
        err.set_code(25, is_fatal=False)     
    
    # rename dat files to avoid conflict
    rename_dat_files("arc_corrected")         
                
    # ---------------------------------------------------------------
    # - FIND POSITION OF TARGET SPECTRUM WITH A SINGLE BIN (LOFIND) -
    # ---------------------------------------------------------------
    print_routine("Find peaks of target spectrum (lofind)")        
    in_target_filename = target + target_suffix + trim_suffix + cor_suffix + ".fits"

    output = Popen([find, in_target_filename, cfg['lofind_target']['bin_size_px'], cfg['lofind_target']['bg_percentile'], cfg['lofind_target']['clip_sigma'], \
      cfg['lofind_target']['median_filter_width_px'], cfg['lofind_target']['min_snr'], cfg['lofind_target']['min_spatial_width_px'], cfg['lofind_target']['finding_window_lo_px'], \
        cfg['lofind_target']['finding_window_hi_px'], cfg['lofind_target']['max_centering_num_px'], cfg['lofind_target']['centroid_half_window_size_px'], \
          cfg['lofind_target']['min_used_bins'], cfg['lofind_target']['window_x_lo'], cfg['lofind_target']['window_x_hi']], stdout=PIPE)
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(21, is_fatal=False) 
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATFI", "L2STATF2", "(target corrected)")      
    
    # -------------------------------
    # - EXTRACT SPECTRA (LOEXTRACT) -
    # -------------------------------

    print_routine("Extract NONSS spectra (loextract)")
    
    in_target_filename = target + target_suffix + trim_suffix + cor_suffix + reb_suffix + ".fits"
    out_target_filename = target + target_suffix + trim_suffix + cor_suffix + reb_suffix + ext_suffix + ".fits"

    output = Popen([extract, in_target_filename, cfg['loextract_nonss']['method'], cfg['loextract_nonss']['ss_method'], cfg['loextract_nonss']['target_half_aperture_px'], \
      cfg['loextract_nonss']['sky_window_half_aperture_px'], out_target_filename], stdout=PIPE)
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(26, is_fatal=False)     
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATEX", "L2STATX1", "(target NONSS)") 
    
    print_routine("Extract SS spectra (loextract)")
    
    in_target_filename = target + target_suffix + trim_suffix + cor_suffix + reb_suffix + ".fits"
    out_target_filename = target + target_suffix + trim_suffix + cor_suffix + reb_suffix + ext_suffix + ss_suffix + ".fits"
    
    output = Popen([extract, in_target_filename, cfg['loextract_ss']['method'], cfg['loextract_ss']['ss_method'], cfg['loextract_ss']['target_half_aperture_px'], \
      cfg['loextract_ss']['sky_window_half_aperture_px'], out_target_filename], stdout=PIPE)
    print output.stdout.read()
    output.wait()
    if output.returncode != 0:
        err.set_code(26, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATEX", "L2STATX2", "(target SS)")     
 
    # rename dat files to avoid conflict
    rename_dat_files("target_corrected")
            
    # ------------------------------
    # - REFORMAT FILE (LOREFORMAT) -
    # ------------------------------
    
    print_routine("Reformat spectra (loreformat)")
    
    in_target_headers_filename = target + target_suffix + ".fits"    
    in_target_filename_L1_IMAGE = target + target_suffix + ".fits"    
    in_target_filename_LSS_NONSS = target + target_suffix + trim_suffix + cor_suffix + reb_suffix + ".fits"
    in_target_filename_SPEC_NONSS = target + target_suffix + trim_suffix + cor_suffix + reb_suffix + ext_suffix + ".fits"
    in_target_filename_SPEC_SS = target + target_suffix + trim_suffix + cor_suffix + reb_suffix + ext_suffix + ss_suffix + ".fits"

    out_target_filename = target[:-1] + "2.fits"
    
    ## make L1 extension
    output = Popen([reformat, in_target_filename_L1_IMAGE, in_target_headers_filename, "L1_IMAGE", out_target_filename], stdout=PIPE)
    print output.stdout.read()   
    
    ## make additional extensions
    for op in cfg['loreformat']['operations'].split(','):
        if op.strip() == "LSS_NONSS":
          in_f = in_target_filename_LSS_NONSS
        elif op.strip() == "SPEC_NONSS":
          in_f = in_target_filename_SPEC_NONSS
        elif op.strip() == "SPEC_SS":
          in_f = in_target_filename_SPEC_SS
        else:
          continue
        
        output = Popen([reformat, in_f, in_target_headers_filename, op.strip(), out_target_filename], stdout=PIPE)
        print output.stdout.read() 
        output.wait()
        if output.returncode != 0:
            err.set_code(27, is_fatal=False)  
    
    # ----------------------------------------------
    # - GENERATE RASTER PLOT OF L1_IMAGE extension -
    # ----------------------------------------------   
    
    print_routine("Plot extensions of output file (l2pi)")  
    print
    
    # use previously defined output filename from loreformat
    in_target_filename = out_target_filename
    
    a_pi_execute(in_target_filename, "L1_IMAGE", L1_IMAGE_plot, "L1_IMAGE")
    
    if os.path.exists(L1_IMAGE_plot):
        print_notification("Success.")
    else:
        print_notification("Failed.") 
        err.set_code(9, is_fatal=False)    
        
    # ------------------------------------------------
    # - GENERATE RASTER PLOT OF SPEC_NONSS extension -
    # ------------------------------------------------    
    
    print_routine("Plot extensions of output file (l2ps)")   
    print
    
    # use previously defined output filename from loreformat
    in_target_filename = out_target_filename  
    
    try:
        a_ps_execute(in_target_filename, "SPEC_NONSS", SPEC_NONSS_plot, "SPEC_NONSS", "green")
    except KeyError:
        pass
        
    if os.path.exists(SPEC_NONSS_plot):
        print_notification("Success.")
    else:
        print_notification("Failed.") 
        err.set_code(10, is_fatal=False)  
        
    # ------------------------------------------------
    # - GENERATE RASTER PLOT OF SPEC_NONSS extension -
    # ------------------------------------------------  
    
    print_routine("Plot extensions of output file (l2ps)")   
    print
    
    # use previously defined output filename from loreformat
    in_target_filename = out_target_filename   
        
    try:
        a_ps_execute(in_target_filename, "SPEC_SS", SPEC_SS_plot, "SPEC_SS", "blue")
    except KeyError:
        pass
        
    if os.path.exists(SPEC_SS_plot):
        print_notification("Success.")
    else:
        print_notification("Failed.") 
        err.set_code(11, is_fatal=False)   
        
    # -------------------------
    # - GENERATE MONTAGE PLOT -
    # -------------------------  
    
    print_routine("Montage extensions of output file (l2pi/l2ps)")       
    print
    
    # use previously defined output filename from loreformat
    in_target_filename = out_target_filename       
    
    fig = plt.figure()
    fig.suptitle("Raster image of L1_IMAGE and SPEC_* extensions for file " + in_target_filename, fontsize=12)
    fig.add_subplot(211)
    a_pi_execute(in_target_filename, "L1_IMAGE", "", "", save=False, hold=True)

    fig.add_subplot(212)
    try:
        a_ps_execute(in_target_filename, "SPEC_NONSS", "",  "", "green", leg_title="SPEC_NONSS", save=False, hold=True)
        a_ps_execute(in_target_filename, "SPEC_SS", "", "", "blue", legend=True, leg_title="SPEC_SS", save=False, hold=True) 
    except KeyError:
        pass
    plt.savefig(montage_plot, bbox_inches="tight")
    
    if os.path.exists(montage_plot):
        print_notification("Success.")
    else:
        print_notification("Failed.") 
        err.set_code(12, is_fatal=False)   

    # -------
    # - END -
    # -------       
        
    print_routine("Results")       
    print        
    fi_unix = time.time()
    exec_time = fi_unix - st_unix
    print_notification("Execution time: " + str(exec_time) + "s.")
    
    # make sure no negative routine error codes for essential routines
    rtn_keys    = []    
    rtn_codes   = []
    with open(cfg['general']['error_codes_file']) as f:
        for line in f:
            if line.startswith("L2"):
                rtn_keys.append(str(line.split('\t')[0]))
                rtn_codes.append(int(line.split('\t')[1]))

    for idx, i in enumerate(rtn_keys):
        if i == "L2STATF2" or i == "L2STATX1" or i=="L2STATX2":
            continue                                                    # skip 1D extraction related error codes.
        if rtn_codes[idx] < 0: 
            err.set_code(13)    

    err.set_code(0) 	# this is a bit of a bodge, it disregards the current error code!
Esempio n. 2
0
def chk_ref_run(f_ref, f_cont):
  
     # get basename of files
    ref                 = os.path.splitext(os.path.basename(f_ref))[0]
    cont                = os.path.splitext(os.path.basename(f_cont))[0]
  
    out_ref_tr_filename  = "tmp.fits"  
    out_ref_cor_filename = "tmp2.fits"      
    ref_pre_sdist_plot   = "ref_pre_sdist_plot.png"   
    ref_post_sdist_plot  = "ref_post_sdist_plot.png"
  
    err = errors()

    # input sanity checks
    if not all([f_ref, f_cont]):
        err.set_code(1)
    elif not os.path.exists(f_ref):
        err.set_code(3)   
    elif not os.path.exists(f_cont):
        err.set_code(4) 
        
    # move files to working directory, redefine paths and change to working directory
    try:
        if not os.path.exists(work_dir):
            os.mkdir(work_dir)
            copyfile(f_ref, work_dir + "/" + ref + ref_suffix + ".fits")
            copyfile(f_cont, work_dir + "/" + cont + cont_suffix + ".fits")
        else:
            if clobber:
                for i in os.listdir(work_dir):
                    os.remove(work_dir + "/" + i)
                copyfile(f_ref, work_dir + "/" + ref + ref_suffix + ".fits")
                copyfile(f_cont, work_dir + "/" + cont + cont_suffix + ".fits")
            else:
                err.set_code(14)
    except OSError:
        err.set_code(15)
    
    f_ref = ref + ref_suffix + ".fits"
    f_cont = cont + cont_suffix + ".fits"

    os.chdir(work_dir)        
        
    # -----------------------------------
    # - DETERMINE SUITABLE CONFIG FILES -
    # -----------------------------------
    
    print_routine("Finding suitable config file.")
    print 
    
    f_ref_fits = pyfits.open(f_ref)
    try:
        f_ref_fits_hdr_DATEOBS = f_ref_fits[0].header['DATE-OBS'].strip()
        f_ref_fits_hdr_CCDXBIN = str(f_ref_fits[0].header['CCDXBIN']).strip()
        f_ref_fits_hdr_CCDYBIN = str(f_ref_fits[0].header['CCDYBIN']).strip()
    except KeyError:
        f_ref_fits.close()
        err.set_code(16)
    f_ref_fits.close()        
        
    config_tab_path      = L2_LOOKUP_TABLES_DIR + "/" + "/config.tab"
   
    chosen_config_entry = search_lookup_table(config_tab_path, f_ref_fits_hdr_DATEOBS, f_ref_fits_hdr_CCDXBIN, f_ref_fits_hdr_CCDYBIN)
    if chosen_config_entry is None:
        print_notification("Failed.") 
        err.set_code(29)
        
    chosen_config_file_path = L2_INI_DIR + "/" + chosen_config_entry

    print_notification("Success. Using file " + chosen_config_file_path)    
    cfg = read_ini(err, chosen_config_file_path)            
        
    # -------------------------
    # - CLIP SPECTRA (LOCLIP) -
    # -------------------------
    
    print_routine("Trim spectra (lotrim)")
    
    in_ref_filename = f_ref
    in_cont_filename = f_cont

    output = Popen([clip, in_cont_filename, in_ref_filename, cfg['loclip']['bin_size_px'], cfg['loclip']['bg_percentile'], cfg['loclip']['clip_sigma'], cfg['loclip']['thresh_sigma'], \
      cfg['loclip']['scan_window_size_px'], cfg['loclip']['scan_window_nsigma'], cfg['loclip']['min_spectrum_width_px'], cfg['loclip']['force_bottom_px'], cfg['loclip']['force_top_px'], \
        out_ref_tr_filename], stdout=PIPE)   
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(20, is_fatal=False)
    
    # ---------------------------------------------
    # - FIND PEAKS OF REFERENCE SPECTRUM (LOFIND) -
    # ---------------------------------------------
    
    print_routine("Find peaks of reference spectrum (lofind)")   
    
    # use previous file as input
    in_ref_filename = out_ref_tr_filename

    output = Popen([find, in_ref_filename, cfg['lofind_ref']['bin_size_px'], cfg['lofind_ref']['bg_percentile'], cfg['lofind_ref']['clip_sigma'], \
      cfg['lofind_ref']['median_filter_width_px'], cfg['lofind_ref']['min_snr'], cfg['lofind_ref']['min_spatial_width_px'], cfg['lofind_ref']['finding_window_lo_px'], \
        cfg['lofind_ref']['finding_window_hi_px'], cfg['lofind_ref']['max_centering_num_px'], cfg['lofind_ref']['centroid_half_window_size_px'], \
          cfg['lofind_ref']['min_used_bins'], cfg['lofind_ref']['window_x_lo'], cfg['lofind_ref']['window_x_hi']], stdout=PIPE)
    print output.stdout.read()  
    output.wait()
    if output.returncode != 0:
        err.set_code(21, is_fatal=False)   
        
    # ----------------------------------------------
    # - FIND SDIST OF REFERENCE SPECTRUM (LOTRACE) -
    # ----------------------------------------------
    
    print_routine("Find sdist of reference spectrum (lotrace)") 
    
    output = Popen([trace, cfg['lotrace']['polynomial_order']], stdout=PIPE)
    print output.stdout.read()  
    output.wait()
    if output.returncode != 0:
        err.set_code(22, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATTR", "L2STATT1", add_to_desc="(ref uncorrected)")    
    
    # ---------------------------------------------------------
    # - PLOT TRACE OF REFERENCE SPECTRUM PRE SDIST CORRECTION -
    # ---------------------------------------------------------

    print_routine("Plot trace of reference spectrum pre sdist correction") 
    print
    
    try:
        a_pp_execute(in_ref_filename, cfg['general']['lofind_output_file'], cfg['general']['lotrace_output_file'], ref_pre_sdist_plot, "Reference pre SDIST correction", \
        cfg['general']['max_curvature_post_cor']) 
    except IOError:
        pass
    
    if os.path.exists(ref_pre_sdist_plot):
        print_notification("Success.")
    else:
        print_notification("Failed.") 
        err.set_code(6, is_fatal=False)
        
    # -----------------------------------------
    # - CORRECT SPECTRA FOR SDIST (LOCORRECT) -
    # -----------------------------------------
    
    print_routine("Correct spectra for sdist (locorrect)")      

    output = Popen([correct, in_ref_filename, cfg['locorrect']['interpolation_type'], cfg['locorrect']['conserve_flux'], out_ref_cor_filename], stdout=PIPE)
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(23, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATCO", "L2STATOT", add_to_desc="(ref)")
    
    # ---------------------------------------------
    # - FIND PEAKS OF REFERENCE SPECTRUM (LOFIND) -
    # ---------------------------------------------
    
    print_routine("Find peaks of reference spectrum (lofind)")       
    
    in_ref_filename = out_ref_cor_filename

    output = Popen([find, in_ref_filename, cfg['lofind_ref']['bin_size_px'], cfg['lofind_ref']['bg_percentile'], cfg['lofind_ref']['clip_sigma'], \
      cfg['lofind_ref']['median_filter_width_px'], cfg['lofind_ref']['min_snr'], cfg['lofind_ref']['min_spatial_width_px'], cfg['lofind_ref']['finding_window_lo_px'], \
        cfg['lofind_ref']['finding_window_hi_px'], cfg['lofind_ref']['max_centering_num_px'], cfg['lofind_ref']['centroid_half_window_size_px'], \
          cfg['lofind_ref']['min_used_bins'], cfg['lofind_ref']['window_x_lo'], cfg['lofind_ref']['window_x_hi']], stdout=PIPE)
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(21, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATFI", omit=True)    
    
    # ----------------------------------------------
    # - FIND SDIST OF REFERENCE SPECTRUM (LOTRACE) -
    # ----------------------------------------------
    
    print_routine("Find sdist of reference spectrum (lotrace)")       
    
    output = Popen([trace, cfg['lotrace']['polynomial_order']], stdout=PIPE)
    print output.stdout.read() 
    output.wait()
    if output.returncode != 0:
        err.set_code(22, is_fatal=False)      
    rewrite_error_codes_file(cfg['general']['error_codes_file'], "L2STATTR", omit=True)

    # ----------------------------------------------------------------------------------------------------------
    # - PLOT TRACE OF SPECTRUM POST SDIST CORRECTION AND CHECK RETURN CODE FOR SIGNIFICANT REMAINING CURVATURE -
    # ----------------------------------------------------------------------------------------------------------
    
    print_routine("Plot trace of spectrum post sdist correction and check for curvature (l2pp)") 
    print
    
    try:
        rtn = a_pp_execute(in_ref_filename, cfg['general']['lofind_output_file'], cfg['general']['lotrace_output_file'], ref_post_sdist_plot, "Reference post SDIST correction", \
        cfg['general']['max_curvature_post_cor']) 
    except IOError:
        rtn = -1
    
    if os.path.exists(ref_post_sdist_plot) and rtn == 0:
        print_notification("Success.")
    elif not os.path.exists(ref_post_sdist_plot):
        print_notification("Failed.") 
        err.set_code(7, is_fatal=False)   
    elif rtn != 0:
        print_notification("Failed.") 
        err.set_code(8)     
    
    # make sure no negative routine error codes for essential routines
    rtn_keys    = []    
    rtn_codes   = []
    with open(cfg['general']['error_codes_file']) as f:
        for line in f:
            if line.startswith("L2"):
                rtn_keys.append(str(line.split('\t')[0]))
                rtn_codes.append(int(line.split('\t')[1]))

    for idx, i in enumerate(rtn_keys):
        if i == "L2STATF2" or i == "L2STATX1" or i=="L2STATX2":
            continue                                                    # skip 1D extraction related error codes.
        if rtn_codes[idx] < 0: 
            err.set_code(13)             
             
    err.set_code(0)   	# this is a bit of a bodge, it disregards the current error code!