예제 #1
0
    def __init__(self, utdate, refdate, config, frac_slit=None):
        self.utdate = utdate
        self.refdate = refdate
        self.config = config

        self.igr_path = IGRINSPath(config, utdate)

        self.igr_storage = PipelineStorage(self.igr_path)

        self.frac_slit = frac_slit
예제 #2
0
    def __init__(self, utdate, refdate, config):
        """
        cr_rejection_thresh : pixels that deviate significantly from the profile are excluded.
        """
        self.utdate = utdate
        self.refdate = refdate
        self.config = config

        self.igr_storage = PipelineStorage.from_utdate(utdate,
                                                       config=config)
        self.igr_path = self.igr_storage.igr_path
예제 #3
0
def process_thar_band(utdate, refdate, band, obsids, config):

    from libs.products import ProductDB, PipelineStorage

    igr_path = IGRINSPath(config, utdate)

    igr_storage = PipelineStorage(igr_path)

    thar_filenames = igr_path.get_filenames(band, obsids)

    thar_basename = os.path.splitext(os.path.basename(thar_filenames[0]))[0]


    thar_master_obsid = obsids[0]

    flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
                                                        "flat_on.db",
                                                        )
    flaton_db = ProductDB(flaton_db_name)

    flaton_basename = flaton_db.query(band, thar_master_obsid)


    from libs.storage_descriptions import FLATCENTROID_SOL_JSON_DESC

    desc_list = [FLATCENTROID_SOL_JSON_DESC]
    products = igr_storage.load(desc_list,
                                mastername=flaton_basename)

    aperture_solution_products = products[FLATCENTROID_SOL_JSON_DESC]

    # igrins_orders = {}
    # igrins_orders["H"] = range(99, 122)
    # igrins_orders["K"] = range(72, 92)

    if 1:
        bottomup_solutions = aperture_solution_products["bottom_up_solutions"]

        orders = range(len(bottomup_solutions))

        ap =  Apertures(orders, bottomup_solutions)

    if 1:
        from libs.process_thar import ThAr

        thar = ThAr(thar_filenames)

        thar_products = thar.process_thar(ap)

    if 1: # match order
        from libs.process_thar import match_order_thar
        from libs.master_calib import load_thar_ref_data

        #ref_date = "20140316"

        thar_ref_data = load_thar_ref_data(refdate, band)

        new_orders = match_order_thar(thar_products, thar_ref_data)

        print thar_ref_data["orders"]
        print  new_orders

        ap =  Apertures(new_orders, bottomup_solutions)

        from libs.storage_descriptions import ONED_SPEC_JSON_DESC
        thar_products[ONED_SPEC_JSON_DESC]["orders"] = new_orders


    if 1:

        hdu = pyfits.open(thar_filenames[0])[0]
        igr_storage.store(thar_products,
                          mastername=thar_filenames[0],
                          masterhdu=hdu)

    if 1:
        # measure shift of thar lines from reference spectra

        # load spec

        from libs.process_thar import reidentify_ThAr_lines
        thar_reidentified_products = reidentify_ThAr_lines(thar_products,
                                                           thar_ref_data)

        igr_storage.store(thar_reidentified_products,
                          mastername=thar_filenames[0],
                          masterhdu=hdu)

    if 1:

        from libs.process_thar import (load_echelogram,
                                       align_echellogram_thar,
                                       check_thar_transorm,
                                       get_wavelength_solutions)

        ref_date = thar_ref_data["ref_date"]
        echel = load_echelogram(ref_date, band)

        thar_aligned_echell_products = \
             align_echellogram_thar(thar_reidentified_products,
                                    echel, band, ap)

        # We do not save this product yet.
        # igr_storage.store(thar_aligned_echell_products,
        #                   mastername=thar_filenames[0],
        #                   masterhdu=hdu)



        fig_list = check_thar_transorm(thar_products,
                                       thar_aligned_echell_products)

        from libs.qa_helper import figlist_to_pngs
        thar_figs = igr_path.get_section_filename_base("QA_PATH",
                                                       "thar",
                                                       "thar_"+thar_basename)
        figlist_to_pngs(thar_figs, fig_list)

        thar_wvl_sol = get_wavelength_solutions(thar_aligned_echell_products,
                                                echel)

        igr_storage.store(thar_wvl_sol,
                          mastername=thar_filenames[0],
                          masterhdu=hdu)

    if 1: # make amp and order falt

        from libs.storage_descriptions import ONED_SPEC_JSON_DESC

        orders = thar_products[ONED_SPEC_JSON_DESC]["orders"]
        order_map = ap.make_order_map()
        #slitpos_map = ap.make_slitpos_map()


        # load flat on products
        #flat_on_params_name = flaton_path.get_secondary_path("flat_on_params")

        #flaton_products = PipelineProducts.load(flat_on_params_name)
        from libs.storage_descriptions import (FLAT_NORMED_DESC,
                                               FLAT_MASK_DESC)

        flaton_products = igr_storage.load([FLAT_NORMED_DESC, FLAT_MASK_DESC],
                                           flaton_basename)

        from libs.process_flat import make_order_flat, check_order_flat
        order_flat_products = make_order_flat(flaton_products,
                                              orders, order_map)

        #fn = thar_path.get_secondary_path("orderflat")
        #order_flat_products.save(fn, masterhdu=hdu)

        igr_storage.store(order_flat_products,
                          mastername=flaton_basename,
                          masterhdu=hdu)

    if 1:
        fig_list = check_order_flat(order_flat_products)

        from libs.qa_helper import figlist_to_pngs
        orderflat_figs = igr_path.get_section_filename_base("QA_PATH",
                                                            "orderflat",
                                                            "orderflat_"+thar_basename)
        figlist_to_pngs(orderflat_figs, fig_list)

    if 1:
        from libs.products import ProductDB
        thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
                                                          "thar.db",
                                                          )
        thar_db = ProductDB(thar_db_name)
        # os.path.join(igr_path.secondary_calib_path,
        #                                  "thar.db"))
        thar_db.update(band, thar_basename)
예제 #4
0
파일: recipe_flat.py 프로젝트: carleen/plp
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on,
                      config):
    from libs.products import PipelineStorage

    igr_path = IGRINSPath(config, utdate)

    igr_storage = PipelineStorage(igr_path)


    flat_off_filenames = igr_path.get_filenames(band, obsids_off)
    flat_on_filenames = igr_path.get_filenames(band, obsids_on)

    if 1: # process flat off

        flat_offs_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_off_filenames]
        flat_offs = [hdu.data for hdu in flat_offs_hdu_list]


        flat = FlatOff(flat_offs)
        flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100,
                                                       sigma_clip2=5)

        igr_storage.store(flatoff_products,
                          mastername=flat_off_filenames[0],
                          masterhdu=flat_offs_hdu_list[0])



    if 1: # flat on

        from libs.storage_descriptions import (FLAT_OFF_DESC,
                                               HOTPIX_MASK_DESC,
                                               FLATOFF_JSON_DESC)

        desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC]
        flatoff_products = igr_storage.load(desc_list,
                                            mastername=flat_off_filenames[0])

        flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames]
        flat_ons = [hdu.data for hdu in flat_on_hdu_list]


        from libs.master_calib import get_master_calib_abspath
        fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" % (refdate,
                                                                   band))
        deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool)

        flat_on = FlatOn(flat_ons)
        flaton_products = flat_on.make_flaton_deadpixmap(flatoff_products,
                                                         deadpix_mask_old=deadpix_mask_old)

        igr_storage.store(flaton_products,
                          mastername=flat_on_filenames[0],
                          masterhdu=flat_on_hdu_list[0])



    if 1: # now trace the orders

        from libs.process_flat import trace_orders

        trace_products = trace_orders(flaton_products)

        hdu = pyfits.open(flat_on_filenames[0])[0]

        igr_storage.store(trace_products,
                          mastername=flat_on_filenames[0],
                          masterhdu=flat_on_hdu_list[0])


        from libs.process_flat import trace_solutions
        trace_solution_products = trace_solutions(trace_products)


        igr_storage.store(trace_solution_products,
                          mastername=flat_on_filenames[0],
                          masterhdu=flat_on_hdu_list[0])


    # plot qa figures.

    if 1:
        from libs.process_flat import check_trace_order
        from matplotlib.figure import Figure
        fig1 = Figure()
        check_trace_order(trace_products, fig1)

    if 1:
        from libs.process_flat import plot_trace_solutions
        fig2, fig3 = plot_trace_solutions(flaton_products,
                                          trace_solution_products)

    flatoff_basename = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
    flaton_basename = os.path.splitext(os.path.basename(flat_on_filenames[0]))[0]

    if 1:
        from libs.qa_helper import figlist_to_pngs
        aperture_figs = igr_path.get_section_filename_base("QA_PATH",
                                                           "aperture_"+flaton_basename,
                                                           "aperture_"+flaton_basename)

        figlist_to_pngs(aperture_figs, [fig1, fig2, fig3])

    # save db
    if 1:
        from libs.products import ProductDB
        flatoff_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
                                                             "flat_off.db",
                                                             )
        flatoff_db = ProductDB(flatoff_db_name)
        #dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
        flatoff_db.update(band, flatoff_basename)


        flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
                                                             "flat_on.db",
                                                             )
        flaton_db = ProductDB(flaton_db_name)
        flaton_db.update(band, flaton_basename)
예제 #5
0
def process_wvlsol_band(utdate, refdate, band, obsids, config):

    from libs.products import ProductDB, PipelineStorage

    igr_path = IGRINSPath(config, utdate)

    igr_storage = PipelineStorage(igr_path)

    sky_filenames = igr_path.get_filenames(band, obsids)

    sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]


    master_obsid = obsids[0]


    flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
                                                        "flat_on.db",
                                                        )
    flaton_db = ProductDB(flaton_db_name)

    #flaton_basename = flaton_db.query(band, master_obsid)

    thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
                                                        "thar.db",
                                                        )
    thar_db = ProductDB(thar_db_name)

    #thar_basename = thar_db.query(band, master_obsid)




    # flaton_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
    #                                    "flat_on.db"))
    # thar_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
    #                                  "thar.db"))

    ap = load_aperture(igr_storage, band, master_obsid,
                       flaton_db, thar_db)


    if 1: #

        from libs.process_thar import get_1d_median_specs
        raw_spec_product = get_1d_median_specs(sky_filenames, ap)


        # sky_master_fn_ = os.path.splitext(os.path.basename(sky_names[0]))[0]
        # sky_master_fn = igr_path.get_secondary_calib_filename(sky_master_fn_)

        import astropy.io.fits as pyfits
        masterhdu = pyfits.open(sky_filenames[0])[0]

        igr_storage.store(raw_spec_product,
                          mastername=sky_filenames[0],
                          masterhdu=masterhdu)

        # fn = sky_path.get_secondary_path("raw_spec")
        # raw_spec_product.save(fn,
        #                       masterhdu=masterhdu)


        from libs.master_calib import load_sky_ref_data

        # ref_date = "20140316"

        refdate = config.get_value("REFDATE", utdate)
        sky_ref_data = load_sky_ref_data(refdate, band)


    if 1: # initial wavelength solution

        # this need to be fixed
        # thar_db.query(sky_master_obsid)
        # json_name_ = "SDC%s_%s_0003.median_spectra.wvlsol" % (band,
        #                                                      igrins_log.date)

        from libs.storage_descriptions import THAR_WVLSOL_JSON_DESC
        thar_basename = thar_db.query(band, master_obsid)
        thar_wvl_sol = igr_storage.load([THAR_WVLSOL_JSON_DESC],
                                        thar_basename)[THAR_WVLSOL_JSON_DESC]
        #print thar_wvl_sol.keys()
        #["wvl_sol"]

        #json_name = thar_path.get_secondary_path("wvlsol_v0")
        #json_name = igr_path.get_secondary_calib_filename(json_name_)
        #thar_wvl_sol = PipelineProducts.load(json_name)




    if 1:
        # Now we fit with gaussian profile for matched positions.

        ohline_indices = sky_ref_data["ohline_indices"]
        ohlines_db = sky_ref_data["ohlines_db"]

        wvl_solutions = thar_wvl_sol["wvl_sol"]

        if 0: # it would be better to iteratively refit the solution
            fn = sky_path.get_secondary_path("wvlsol_v1")
            p = PipelineProducts.load(fn)
            wvl_solutionv = p["wvl_sol"]

        orders_w_solutions_ = thar_wvl_sol["orders"]
        from libs.storage_descriptions import ONED_SPEC_JSON_DESC
        orders_w_solutions = [o for o in orders_w_solutions_ if o in raw_spec_product[ONED_SPEC_JSON_DESC]["orders"]]
        _ = dict(zip(raw_spec_product[ONED_SPEC_JSON_DESC]["orders"],
                     raw_spec_product[ONED_SPEC_JSON_DESC]["specs"]))
        s_list = [_[o]for o in orders_w_solutions]


        from libs.reidentify_ohlines import fit_ohlines
        ref_pixel_list, reidentified_lines = \
                        fit_ohlines(ohlines_db, ohline_indices,
                                    orders_w_solutions,
                                    wvl_solutions, s_list)


        # from scipy.interpolate import interp1d
        # from reidentify import reidentify_lines_all

        x = np.arange(2048)


        # line_indices_list = [ref_ohline_indices[str(o)] for o in igrins_orders[band]]




        ###### not fit identified lines

        from libs.ecfit import get_ordered_line_data, fit_2dspec, check_fit

        # d_x_wvl = {}
        # for order, z in echel.zdata.items():
        #     xy_T = affine_tr.transform(np.array([z.x, z.y]).T)
        #     x_T = xy_T[:,0]
        #     d_x_wvl[order]=(x_T, z.wvl)

        reidentified_lines_map = dict(zip(orders_w_solutions,
                                          reidentified_lines))

        if band == "K":
            import libs.master_calib as master_calib
            fn = "hitran_bootstrap_K_%s.json" % refdate
            bootstrap_name = master_calib.get_master_calib_abspath(fn)
            import json
            bootstrap = json.load(open(bootstrap_name))

            import libs.hitran as hitran
            r, ref_pixel_list = hitran.reidentify(wvl_solutions, s_list, bootstrap)
            # json_name = "hitran_reidentified_K_%s.json" % igrins_log.date
            # r = json.load(open(json_name))
            for i, s in r.items():
                ss = reidentified_lines_map[int(i)]
                ss0 = np.concatenate([ss[0], s["pixel"]])
                ss1 = np.concatenate([ss[1], s["wavelength"]])
                reidentified_lines_map[int(i)] = (ss0, ss1)

        xl, yl, zl = get_ordered_line_data(reidentified_lines_map)
        # xl : pixel
        # yl : order
        # zl : wvl * order

        x_domain = [0, 2047]
        y_domain = [orders_w_solutions[0]-2, orders_w_solutions[-1]+2]
        x_degree, y_degree = 4, 3
        #x_degree, y_degree = 3, 2
        p, m = fit_2dspec(xl, yl, zl, x_degree=x_degree, y_degree=y_degree,
                          x_domain=x_domain, y_domain=y_domain)


        # derive wavelengths.
        xx = np.arange(2048)
        wvl_sol = []
        for o in orders_w_solutions:
            oo = np.empty_like(xx)
            oo.fill(o)
            wvl = p(xx, oo) / o
            wvl_sol.append(list(wvl))

        oh_sol_products = PipelineProducts("Wavelength solution based on ohlines")
        #from libs.process_thar import ONED_SPEC_JSON
        from libs.products import PipelineDict
        from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
        oh_sol_products.add(SKY_WVLSOL_JSON_DESC,
                            PipelineDict(orders=orders_w_solutions,
                                         wvl_sol=wvl_sol))

    if 1:

        if 1: # save as WAT fits header
            xx = np.arange(0, 2048)
            xx_plus1 = np.arange(1, 2048+1)

            from astropy.modeling import models, fitting

            # We convert 2d chebyshev solution to a seriese of 1d
            # chebyshev.  For now, use naive (and inefficient)
            # approach of refitting the solution with 1d. Should be
            # reimplemented.

            p1d_list = []
            for o in orders_w_solutions:
                oo = np.empty_like(xx)
                oo.fill(o)
                wvl = p(xx, oo) / o * 1.e4 # um to angstrom

                p_init1d = models.Chebyshev1D(domain=[1, 2048],
                                              degree=p.x_degree)
                fit_p1d = fitting.LinearLSQFitter()
                p1d = fit_p1d(p_init1d, xx_plus1, wvl)
                p1d_list.append(p1d)

        from libs.iraf_helper import get_wat_spec, default_header_str
        wat_list = get_wat_spec(orders_w_solutions, p1d_list)

        # cards = [pyfits.Card.fromstring(l.strip()) \
        #          for l in open("echell_2dspec.header")]
        cards = [pyfits.Card.fromstring(l.strip()) \
                 for l in default_header_str]

        wat = "wtype=multispec " + " ".join(wat_list)
        char_per_line = 68
        num_line, remainder = divmod(len(wat), char_per_line)
        for i in range(num_line):
            k = "WAT2_%03d" % (i+1,)
            v = wat[char_per_line*i:char_per_line*(i+1)]
            #print k, v
            c = pyfits.Card(k, v)
            cards.append(c)
        if remainder > 0:
            i = num_line
            k = "WAT2_%03d" % (i+1,)
            v = wat[char_per_line*i:]
            #print k, v
            c = pyfits.Card(k, v)
            cards.append(c)

        if 1:
            # save fits with empty header

            header = pyfits.Header(cards)
            hdu = pyfits.PrimaryHDU(header=header,
                                    data=np.array([]).reshape((0,0)))

            from libs.storage_descriptions import SKY_WVLSOL_FITS_DESC
            from libs.products import PipelineImage
            oh_sol_products.add(SKY_WVLSOL_FITS_DESC,
                                PipelineImage([],
                                              np.array([]).reshape((0,0))))

            igr_storage.store(oh_sol_products,
                              mastername=sky_filenames[0],
                              masterhdu=hdu)

            #fn = sky_path.get_secondary_path("wvlsol_v1.fits")
            #hdu.writeto(fn, clobber=True)

        if 0:
            # plot all spectra
            for w, s in zip(wvl_sol, s_list):
                plot(w, s)


    if 1:
        # filter out the line indices not well fit by the surface


        keys = reidentified_lines_map.keys()
        di_list = [len(reidentified_lines_map[k_][0]) for k_ in keys]

        endi_list = np.add.accumulate(di_list)

        filter_mask = [m[endi-di:endi] for di, endi in zip(di_list, endi_list)]
        #from itertools import compress
        # _ = [list(compress(indices, mm)) for indices, mm \
        #      in zip(line_indices_list, filter_mask)]
        # line_indices_list_filtered = _

        reidentified_lines_ = [reidentified_lines_map[k_] for k_ in keys]
        _ = [(v_[0][mm], v_[1][mm]) for v_, mm \
             in zip(reidentified_lines_, filter_mask)]

        reidentified_lines_map_filtered = dict(zip(orders_w_solutions, _))


        if 1:
            from matplotlib.figure import Figure

            fig1 = Figure(figsize=(12, 7))
            check_fit(fig1, xl, yl, zl, p,
                      orders_w_solutions,
                      reidentified_lines_map)
            fig1.tight_layout()

            fig2 = Figure(figsize=(12, 7))
            check_fit(fig2, xl[m], yl[m], zl[m], p,
                      orders_w_solutions,
                      reidentified_lines_map_filtered)
            fig2.tight_layout()

    if 1:
        from libs.qa_helper import figlist_to_pngs
        sky_figs = igr_path.get_section_filename_base("QA_PATH",
                                                       "oh_fit2d",
                                                       "oh_fit2d_"+sky_basename)
        figlist_to_pngs(sky_figs, [fig1, fig2])

    if 1:
        from libs.products import ProductDB
        sky_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
                                                          "sky.db",
                                                          )

        sky_db = ProductDB(sky_db_name)
        sky_db.update(band, sky_basename)
예제 #6
0
def process_distortion_sky_band(utdate, refdate, band, obsids, config):

    from libs.products import ProductDB, PipelineStorage

    igr_path = IGRINSPath(config, utdate)

    igr_storage = PipelineStorage(igr_path)

    sky_filenames = igr_path.get_filenames(band, obsids)

    sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]


    master_obsid = obsids[0]


    flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
                                                        "flat_on.db",
                                                        )
    flaton_db = ProductDB(flaton_db_name)

    # thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
    #                                                     "thar.db",
    #                                                     )
    # thar_db = ProductDB(thar_db_name)



    from libs.storage_descriptions import (COMBINED_IMAGE_DESC,
                                           ONED_SPEC_JSON_DESC)
    raw_spec_products = igr_storage.load([COMBINED_IMAGE_DESC,
                                          ONED_SPEC_JSON_DESC],
                                         sky_basename)

    # raw_spec_products = PipelineProducts.load(sky_path.get_secondary_path("raw_spec"))

    from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC

    wvlsol_products = igr_storage.load([SKY_WVLSOL_JSON_DESC],
                                       sky_basename)[SKY_WVLSOL_JSON_DESC]

    orders_w_solutions = wvlsol_products["orders"]
    wvl_solutions = wvlsol_products["wvl_sol"]

    ap = load_aperture2(igr_storage, band, master_obsid,
                        flaton_db,
                        raw_spec_products[ONED_SPEC_JSON_DESC]["orders"],
                        orders_w_solutions)
    #orders_w_solutions = ap.orders


    if 1: # load reference data
        from libs.master_calib import load_sky_ref_data

        ref_utdate = config.get_value("REFDATE", utdate)

        sky_ref_data = load_sky_ref_data(ref_utdate, band)


        ohlines_db = sky_ref_data["ohlines_db"]
        ref_ohline_indices = sky_ref_data["ohline_indices"]


        orders_w_solutions = wvlsol_products["orders"]
        wvl_solutions = wvlsol_products["wvl_sol"]


    if 1:

        n_slice_one_direction = 2
        n_slice = n_slice_one_direction*2 + 1
        i_center = n_slice_one_direction
        slit_slice = np.linspace(0., 1., n_slice+1)

        slice_center = (slit_slice[i_center], slit_slice[i_center+1])
        slice_up = [(slit_slice[i_center+i], slit_slice[i_center+i+1]) \
                    for i in range(1, n_slice_one_direction+1)]
        slice_down = [(slit_slice[i_center-i-1], slit_slice[i_center-i]) \
                      for i in range(n_slice_one_direction)]

        d = raw_spec_products[COMBINED_IMAGE_DESC].data
        s_center = ap.extract_spectra_v2(d, slice_center[0], slice_center[1])

        s_up, s_down = [], []
        for s1, s2 in slice_up:
            s = ap.extract_spectra_v2(d, s1, s2)
            s_up.append(s)
        for s1, s2 in slice_down:
            s = ap.extract_spectra_v2(d, s1, s2)
            s_down.append(s)


    if 1:
        # now fit

        #ohline_indices = [ref_ohline_indices[o] for o in orders_w_solutions]


        if 0:
            def test_order(oi):
                ax=subplot(111)
                ax.plot(wvl_solutions[oi], s_center[oi])
                #ax.plot(wvl_solutions[oi], raw_spec_products["specs"][oi])
                o = orders[oi]
                line_indices = ref_ohline_indices[o]
                for li in line_indices:
                    um = np.take(ohlines_db.um, li)
                    intensity = np.take(ohlines_db.intensity, li)
                    ax.vlines(um, ymin=0, ymax=-intensity)




        from libs.reidentify_ohlines import fit_ohlines, fit_ohlines_pixel

        def get_reidentified_lines_OH(orders_w_solutions,
                                      wvl_solutions, s_center):
            ref_pixel_list, reidentified_lines = \
                            fit_ohlines(ohlines_db, ref_ohline_indices,
                                        orders_w_solutions,
                                        wvl_solutions, s_center)

            reidentified_lines_map = dict(zip(orders_w_solutions,
                                              reidentified_lines))
            return reidentified_lines_map, ref_pixel_list

        if band == "H":
            reidentified_lines_map, ref_pixel_list_oh = \
                       get_reidentified_lines_OH(orders_w_solutions,
                                                 wvl_solutions,
                                                 s_center)

            def refit_centroid(s_center,
                               ref_pixel_list=ref_pixel_list_oh):
                centroids = fit_ohlines_pixel(s_center,
                                              ref_pixel_list)
                return centroids

        else: # band K
            reidentified_lines_map, ref_pixel_list_oh = \
                       get_reidentified_lines_OH(orders_w_solutions,
                                                 wvl_solutions,
                                                 s_center)

            import libs.master_calib as master_calib
            fn = "hitran_bootstrap_K_%s.json" % ref_utdate
            bootstrap_name = master_calib.get_master_calib_abspath(fn)
            import json
            bootstrap = json.load(open(bootstrap_name))

            import libs.hitran as hitran
            r, ref_pixel_dict_hitrans = hitran.reidentify(wvl_solutions,
                                                          s_center,
                                                          bootstrap)
            # for i, s in r.items():
            #     ss = reidentified_lines_map[int(i)]
            #     ss0 = np.concatenate([ss[0], s["pixel"]])
            #     ss1 = np.concatenate([ss[1], s["wavelength"]])
            #     reidentified_lines_map[int(i)] = (ss0, ss1)

            #reidentified_lines_map, ref_pixel_list

            def refit_centroid(s_center,
                               ref_pixel_list=ref_pixel_list_oh,
                               ref_pixel_dict_hitrans=ref_pixel_dict_hitrans):
                centroids_oh = fit_ohlines_pixel(s_center,
                                                 ref_pixel_list)

                s_dict = dict(zip(orders_w_solutions, s_center))
                centroids_dict_hitrans = hitran.fit_hitrans_pixel(s_dict,
                                                                  ref_pixel_dict_hitrans)
                centroids = []
                for o, c_oh in zip(orders_w_solutions, centroids_oh):
                    if o in centroids_dict_hitrans:
                        c = np.concatenate([c_oh,
                                            centroids_dict_hitrans[o]["pixel"]])
                        centroids.append(c)
                    else:
                        centroids.append(c_oh)

                return centroids

        # reidentified_lines_map = get_reidentified_lines(orders_w_solutions,
        #                                                 wvl_solutions,
        #                                                 s_center)


    if 1:
        # TODO: we should not need this, instead recycle from preivious step.
        fitted_centroid_center = refit_centroid(s_center)
        # fitted_centroid_center = fit_ohlines_pixel(s_center,
        #                                            ref_pixel_list)

        d_shift_up = []
        for s in s_up:
            # TODO: ref_pixel_list_filtered need to be updated with recent fit.
            fitted_centroid = refit_centroid(s)
            # fitted_centroid = fit_ohlines_pixel(s,
            #                                     ref_pixel_list)
            d_shift = [b-a for a, b in zip(fitted_centroid_center,
                                           fitted_centroid)]
            d_shift_up.append(d_shift)

        d_shift_down = []
        for s in s_down:
            # TODO: ref_pixel_list_filtered need to be updated with recent fit.
            fitted_centroid = refit_centroid(s)
            # fitted_centroid = fit_ohlines_pixel(s,
            #                                     ref_pixel_list)
            #fitted_centroid_center,
            d_shift = [b-a for a, b in zip(fitted_centroid_center,
                                           fitted_centroid)]
            d_shift_down.append(d_shift)


    if 1:
        # now fit
        orders = orders_w_solutions

        x_domain = [0, 2048]
        y_domain = [orders[0]-2, orders[-1]+2]


        xl = np.concatenate(fitted_centroid_center)

        yl_ = [o + np.zeros_like(x_) for o, x_ in zip(orders,
                                                      fitted_centroid_center)]
        yl = np.concatenate(yl_)

        from libs.ecfit import fit_2dspec, check_fit_simple

        zl_list = [np.concatenate(d_) for d_ \
                   in d_shift_down[::-1] + d_shift_up]

        pm_list = []
        for zl in zl_list:
            p, m = fit_2dspec(xl, yl, zl,
                              x_degree=1, y_degree=1,
                              x_domain=x_domain, y_domain=y_domain)
            pm_list.append((p,m))

        zz_std_list = []
        for zl, (p, m)  in zip(zl_list, pm_list):
            z_m = p(xl[m], yl[m])
            zz = z_m - zl[m]
            zz_std_list.append(zz.std())

        fig_list = []
        from matplotlib.figure import Figure
        for zl, (p, m)  in zip(zl_list, pm_list):
            fig = Figure()
            check_fit_simple(fig, xl[m], yl[m], zl[m], p, orders)
            fig_list.append(fig)


    if 1:
        xi = np.linspace(0, 2048, 128+1)
        from astropy.modeling import fitting
        from astropy.modeling.polynomial import Chebyshev2D
        x_domain = [0, 2048]
        y_domain = [0., 1.]

        p2_list = []
        for o in orders:
            oi = np.zeros_like(xi) + o
            shift_list = []
            for p,m in pm_list[:n_slice_one_direction]:
                shift_list.append(p(xi, oi))

            shift_list.append(np.zeros_like(xi))

            for p,m in pm_list[n_slice_one_direction:]:
                shift_list.append(p(xi, oi))


            p_init = Chebyshev2D(x_degree=1, y_degree=2,
                                 x_domain=x_domain, y_domain=y_domain)
            f = fitting.LinearLSQFitter()

            yi = 0.5*(slit_slice[:-1] + slit_slice[1:])
            xl, yl = np.meshgrid(xi, yi)
            zl = np.array(shift_list)
            p = f(p_init, xl, yl, zl)

            p2_list.append(p)

    if 1:
        p2_dict = dict(zip(orders, p2_list))

        order_map = ap.make_order_map()
        slitpos_map = ap.make_slitpos_map()

        slitoffset_map = np.empty_like(slitpos_map)
        slitoffset_map.fill(np.nan)
        for o in ap.orders:
            xi = np.arange(0, 2048)
            xl, yl = np.meshgrid(xi, xi)
            msk = order_map == o
            slitoffset_map[msk] = p2_dict[o](xl[msk], slitpos_map[msk])

        # import astropy.io.fits as pyfits
        # fn = sky_path.get_secondary_path("slitoffset_map.fits")
        # pyfits.PrimaryHDU(data=slitoffset_map).writeto(fn, clobber=True)

        from libs.storage_descriptions import SLITOFFSET_FITS_DESC
        from libs.products import PipelineImage, PipelineProducts
        distortion_products = PipelineProducts("Distortion map")
        distortion_products.add(SLITOFFSET_FITS_DESC,
                                PipelineImage([],
                                              slitoffset_map))

        igr_storage.store(distortion_products,
                          mastername=sky_filenames[0],
                          masterhdu=None)


        from libs.qa_helper import figlist_to_pngs
        sky_figs = igr_path.get_section_filename_base("QA_PATH",
                                                      "oh_distortion",
                                                      "oh_distortion_"+sky_basename)
        print fig_list
        figlist_to_pngs(sky_figs, fig_list)


    if 0:
        # test
        x = np.arange(2048, dtype="d")
        oi = 10
        o = orders[oi]

        yi = 0.5*(slit_slice[:-1] + slit_slice[1:])

        ax1 = subplot(211)
        s1 = s_up[-1][oi]
        s2 = s_down[-1][oi]

        ax1.plot(x, s1)
        ax1.plot(x, s2)

        ax2 = subplot(212, sharex=ax1, sharey=ax1)
        dx1 = p2_dict[o](x, yi[-1]+np.zeros_like(x))
        ax2.plot(x-dx1, s1)

        dx2 = p2_dict[o](x, yi[0]+np.zeros_like(x))
        ax2.plot(x-dx2, s2)
예제 #7
0
파일: recipe_flat.py 프로젝트: henryroe/plp
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on,
                      config):
    from libs.products import PipelineStorage

    igr_path = IGRINSPath(config, utdate)

    igr_storage = PipelineStorage(igr_path)


    flat_off_filenames = igr_path.get_filenames(band, obsids_off)
    flat_on_filenames = igr_path.get_filenames(band, obsids_on)

    if 1: # process flat off

        flat_offs_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_off_filenames]
        flat_offs = [hdu.data for hdu in flat_offs_hdu_list]


        flat = FlatOff(flat_offs)
        flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100,
                                                       sigma_clip2=5)

        igr_storage.store(flatoff_products,
                          mastername=flat_off_filenames[0],
                          masterhdu=flat_offs_hdu_list[0])



    if 1: # flat on

        from libs.storage_descriptions import (FLAT_OFF_DESC,
                                               HOTPIX_MASK_DESC,
                                               FLATOFF_JSON_DESC)

        desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC]
        flatoff_products = igr_storage.load(desc_list,
                                            mastername=flat_off_filenames[0])

        flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames]
        flat_ons = [hdu.data for hdu in flat_on_hdu_list]


        from libs.master_calib import get_master_calib_abspath
        fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" % (refdate,
                                                                   band))
        deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool)

        flat_on = FlatOn(flat_ons)
        flaton_products = flat_on.make_flaton_deadpixmap(flatoff_products,
                                                         deadpix_mask_old=deadpix_mask_old)

        igr_storage.store(flaton_products,
                          mastername=flat_on_filenames[0],
                          masterhdu=flat_on_hdu_list[0])



    if 1: # now trace the orders

        from libs.process_flat import trace_orders

        trace_products = trace_orders(flaton_products)

        hdu = pyfits.open(flat_on_filenames[0])[0]

        igr_storage.store(trace_products,
                          mastername=flat_on_filenames[0],
                          masterhdu=flat_on_hdu_list[0])


        from libs.process_flat import trace_solutions
        trace_solution_products, trace_solution_products_plot = \
                                 trace_solutions(trace_products)


    if 1:
        trace_solution_products.keys()
        from libs.storage_descriptions import FLATCENTROID_SOL_JSON_DESC

        myproduct = trace_solution_products[FLATCENTROID_SOL_JSON_DESC]
        bottomup_solutions = myproduct["bottom_up_solutions"]

        orders = range(len(bottomup_solutions))

        from libs.apertures import Apertures
        ap =  Apertures(orders, bottomup_solutions)

        from libs.storage_descriptions import FLAT_MASK_DESC
        flat_mask = igr_storage.load1(FLAT_MASK_DESC,
                                      flat_on_filenames[0])
        order_map2 = ap.make_order_map(mask_top_bottom=True)
        bias_mask = flat_mask.data & (order_map2 > 0)

        from libs.products import PipelineImageBase, PipelineProducts
        pp = PipelineProducts("")
        from libs.storage_descriptions import BIAS_MASK_DESC
        pp.add(BIAS_MASK_DESC,
               PipelineImageBase([], bias_mask))

        flaton_basename = flat_on_filenames[0]
        igr_storage.store(pp,
                          mastername=flaton_basename,
                          masterhdu=hdu)


    # plot qa figures.

    if 1:
        from libs.process_flat import check_trace_order
        from matplotlib.figure import Figure
        fig1 = Figure(figsize=[9, 4])
        check_trace_order(trace_products, fig1)

    if 1:
        from libs.process_flat import plot_trace_solutions
        fig2, fig3 = plot_trace_solutions(flaton_products,
                                          trace_solution_products,
                                          trace_solution_products_plot,
                                          )

    flatoff_basename = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
    flaton_basename = os.path.splitext(os.path.basename(flat_on_filenames[0]))[0]

    if 1:
        from libs.qa_helper import figlist_to_pngs
        aperture_figs = igr_path.get_section_filename_base("QA_PATH",
                                                           "aperture_"+flaton_basename,
                                                           "aperture_"+flaton_basename)

        figlist_to_pngs(aperture_figs, [fig1, fig2, fig3])



    if 1: # now trace the orders

        #del trace_solution_products["bottom_up_solutions"]
        igr_storage.store(trace_solution_products,
                          mastername=flat_on_filenames[0],
                          masterhdu=flat_on_hdu_list[0])



    # save db
    if 1:
        from libs.products import ProductDB
        flatoff_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
                                                             "flat_off.db",
                                                             )
        flatoff_db = ProductDB(flatoff_db_name)
        #dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
        flatoff_db.update(band, flatoff_basename)


        flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
                                                             "flat_on.db",
                                                             )
        flaton_db = ProductDB(flaton_db_name)
        flaton_db.update(band, flaton_basename)
예제 #8
0
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on, config):
    from libs.products import PipelineStorage

    igr_path = IGRINSPath(config, utdate)

    igr_storage = PipelineStorage(igr_path)

    flat_off_filenames = igr_path.get_filenames(band, obsids_off)
    flat_on_filenames = igr_path.get_filenames(band, obsids_on)

    if 1:  # process flat off

        flat_offs_hdu_list = [
            pyfits.open(fn_)[0] for fn_ in flat_off_filenames
        ]
        flat_offs = [hdu.data for hdu in flat_offs_hdu_list]

        flat = FlatOff(flat_offs)
        flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100,
                                                       sigma_clip2=5)

        igr_storage.store(flatoff_products,
                          mastername=flat_off_filenames[0],
                          masterhdu=flat_offs_hdu_list[0])

    if 1:  # flat on

        from libs.storage_descriptions import (FLAT_OFF_DESC, HOTPIX_MASK_DESC,
                                               FLATOFF_JSON_DESC)

        desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC]
        flatoff_products = igr_storage.load(desc_list,
                                            mastername=flat_off_filenames[0])

        flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames]
        flat_ons = [hdu.data for hdu in flat_on_hdu_list]

        from libs.master_calib import get_master_calib_abspath
        fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" %
                                      (refdate, band))
        deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool)

        flat_on = FlatOn(flat_ons)
        flaton_products = flat_on.make_flaton_deadpixmap(
            flatoff_products, deadpix_mask_old=deadpix_mask_old)

        igr_storage.store(flaton_products,
                          mastername=flat_on_filenames[0],
                          masterhdu=flat_on_hdu_list[0])

    if 1:  # now trace the orders

        from libs.process_flat import trace_orders

        trace_products = trace_orders(flaton_products)

        hdu = pyfits.open(flat_on_filenames[0])[0]

        igr_storage.store(trace_products,
                          mastername=flat_on_filenames[0],
                          masterhdu=flat_on_hdu_list[0])

        from libs.process_flat import trace_solutions
        trace_solution_products = trace_solutions(trace_products)

        igr_storage.store(trace_solution_products,
                          mastername=flat_on_filenames[0],
                          masterhdu=flat_on_hdu_list[0])

    # plot qa figures.

    if 1:
        from libs.process_flat import check_trace_order
        from matplotlib.figure import Figure
        fig1 = Figure()
        check_trace_order(trace_products, fig1)

    if 1:
        from libs.process_flat import plot_trace_solutions
        fig2, fig3 = plot_trace_solutions(flaton_products,
                                          trace_solution_products)

    flatoff_basename = os.path.splitext(os.path.basename(
        flat_off_filenames[0]))[0]
    flaton_basename = os.path.splitext(os.path.basename(
        flat_on_filenames[0]))[0]

    if 1:
        from libs.qa_helper import figlist_to_pngs
        aperture_figs = igr_path.get_section_filename_base(
            "QA_PATH", "aperture_" + flaton_basename,
            "aperture_" + flaton_basename)

        figlist_to_pngs(aperture_figs, [fig1, fig2, fig3])

    # save db
    if 1:
        from libs.products import ProductDB
        flatoff_db_name = igr_path.get_section_filename_base(
            "PRIMARY_CALIB_PATH",
            "flat_off.db",
        )
        flatoff_db = ProductDB(flatoff_db_name)
        #dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
        flatoff_db.update(band, flatoff_basename)

        flaton_db_name = igr_path.get_section_filename_base(
            "PRIMARY_CALIB_PATH",
            "flat_on.db",
        )
        flaton_db = ProductDB(flaton_db_name)
        flaton_db.update(band, flaton_basename)
예제 #9
0
def process_wvlsol_band(utdate, refdate, band, obsids, config):

    from libs.products import ProductDB, PipelineStorage

    igr_path = IGRINSPath(config, utdate)

    igr_storage = PipelineStorage(igr_path)

    sky_filenames = igr_path.get_filenames(band, obsids)

    sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]

    master_obsid = obsids[0]

    flaton_db_name = igr_path.get_section_filename_base(
        "PRIMARY_CALIB_PATH",
        "flat_on.db",
    )
    flaton_db = ProductDB(flaton_db_name)

    #flaton_basename = flaton_db.query(band, master_obsid)

    thar_db_name = igr_path.get_section_filename_base(
        "PRIMARY_CALIB_PATH",
        "thar.db",
    )
    thar_db = ProductDB(thar_db_name)

    #thar_basename = thar_db.query(band, master_obsid)

    # flaton_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
    #                                    "flat_on.db"))
    # thar_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
    #                                  "thar.db"))

    ap = load_aperture(igr_storage, band, master_obsid, flaton_db, thar_db)

    if 1:  #

        from libs.process_thar import get_1d_median_specs
        raw_spec_product = get_1d_median_specs(sky_filenames, ap)

        # sky_master_fn_ = os.path.splitext(os.path.basename(sky_names[0]))[0]
        # sky_master_fn = igr_path.get_secondary_calib_filename(sky_master_fn_)

        import astropy.io.fits as pyfits
        masterhdu = pyfits.open(sky_filenames[0])[0]

        igr_storage.store(raw_spec_product,
                          mastername=sky_filenames[0],
                          masterhdu=masterhdu)

        # fn = sky_path.get_secondary_path("raw_spec")
        # raw_spec_product.save(fn,
        #                       masterhdu=masterhdu)

        from libs.master_calib import load_sky_ref_data

        # ref_date = "20140316"

        refdate = config.get_value("REFDATE", utdate)
        sky_ref_data = load_sky_ref_data(refdate, band)

    if 1:  # initial wavelength solution

        # this need to be fixed
        # thar_db.query(sky_master_obsid)
        # json_name_ = "SDC%s_%s_0003.median_spectra.wvlsol" % (band,
        #                                                      igrins_log.date)

        from libs.storage_descriptions import THAR_WVLSOL_JSON_DESC
        thar_basename = thar_db.query(band, master_obsid)
        thar_wvl_sol = igr_storage.load([THAR_WVLSOL_JSON_DESC],
                                        thar_basename)[THAR_WVLSOL_JSON_DESC]
        #print thar_wvl_sol.keys()
        #["wvl_sol"]

        #json_name = thar_path.get_secondary_path("wvlsol_v0")
        #json_name = igr_path.get_secondary_calib_filename(json_name_)
        #thar_wvl_sol = PipelineProducts.load(json_name)

    if 1:
        # Now we fit with gaussian profile for matched positions.

        ohline_indices = sky_ref_data["ohline_indices"]
        ohlines_db = sky_ref_data["ohlines_db"]

        wvl_solutions = thar_wvl_sol["wvl_sol"]

        if 0:  # it would be better to iteratively refit the solution
            fn = sky_path.get_secondary_path("wvlsol_v1")
            p = PipelineProducts.load(fn)
            wvl_solutionv = p["wvl_sol"]

        orders_w_solutions_ = thar_wvl_sol["orders"]
        from libs.storage_descriptions import ONED_SPEC_JSON_DESC
        orders_w_solutions = [
            o for o in orders_w_solutions_
            if o in raw_spec_product[ONED_SPEC_JSON_DESC]["orders"]
        ]
        _ = dict(
            zip(raw_spec_product[ONED_SPEC_JSON_DESC]["orders"],
                raw_spec_product[ONED_SPEC_JSON_DESC]["specs"]))
        s_list = [_[o] for o in orders_w_solutions]

        from libs.reidentify_ohlines import fit_ohlines
        ref_pixel_list, reidentified_lines = \
                        fit_ohlines(ohlines_db, ohline_indices,
                                    orders_w_solutions,
                                    wvl_solutions, s_list)

        # from scipy.interpolate import interp1d
        # from reidentify import reidentify_lines_all

        x = np.arange(2048)

        # line_indices_list = [ref_ohline_indices[str(o)] for o in igrins_orders[band]]

        ###### not fit identified lines

        from libs.ecfit import get_ordered_line_data, fit_2dspec, check_fit

        # d_x_wvl = {}
        # for order, z in echel.zdata.items():
        #     xy_T = affine_tr.transform(np.array([z.x, z.y]).T)
        #     x_T = xy_T[:,0]
        #     d_x_wvl[order]=(x_T, z.wvl)

        reidentified_lines_map = dict(
            zip(orders_w_solutions, reidentified_lines))

        if band == "K":
            import libs.master_calib as master_calib
            fn = "hitran_bootstrap_K_%s.json" % refdate
            bootstrap_name = master_calib.get_master_calib_abspath(fn)
            import json
            bootstrap = json.load(open(bootstrap_name))

            import libs.hitran as hitran
            r, ref_pixel_list = hitran.reidentify(wvl_solutions, s_list,
                                                  bootstrap)
            # json_name = "hitran_reidentified_K_%s.json" % igrins_log.date
            # r = json.load(open(json_name))
            for i, s in r.items():
                ss = reidentified_lines_map[int(i)]
                ss0 = np.concatenate([ss[0], s["pixel"]])
                ss1 = np.concatenate([ss[1], s["wavelength"]])
                reidentified_lines_map[int(i)] = (ss0, ss1)

        xl, yl, zl = get_ordered_line_data(reidentified_lines_map)
        # xl : pixel
        # yl : order
        # zl : wvl * order

        x_domain = [0, 2047]
        y_domain = [orders_w_solutions[0] - 2, orders_w_solutions[-1] + 2]
        x_degree, y_degree = 4, 3
        #x_degree, y_degree = 3, 2
        p, m = fit_2dspec(xl,
                          yl,
                          zl,
                          x_degree=x_degree,
                          y_degree=y_degree,
                          x_domain=x_domain,
                          y_domain=y_domain)

        # derive wavelengths.
        xx = np.arange(2048)
        wvl_sol = []
        for o in orders_w_solutions:
            oo = np.empty_like(xx)
            oo.fill(o)
            wvl = p(xx, oo) / o
            wvl_sol.append(list(wvl))

        oh_sol_products = PipelineProducts(
            "Wavelength solution based on ohlines")
        #from libs.process_thar import ONED_SPEC_JSON
        from libs.products import PipelineDict
        from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
        oh_sol_products.add(
            SKY_WVLSOL_JSON_DESC,
            PipelineDict(orders=orders_w_solutions, wvl_sol=wvl_sol))

    if 1:

        if 1:  # save as WAT fits header
            xx = np.arange(0, 2048)
            xx_plus1 = np.arange(1, 2048 + 1)

            from astropy.modeling import models, fitting

            # We convert 2d chebyshev solution to a seriese of 1d
            # chebyshev.  For now, use naive (and inefficient)
            # approach of refitting the solution with 1d. Should be
            # reimplemented.

            p1d_list = []
            for o in orders_w_solutions:
                oo = np.empty_like(xx)
                oo.fill(o)
                wvl = p(xx, oo) / o * 1.e4  # um to angstrom

                p_init1d = models.Chebyshev1D(domain=[1, 2048],
                                              degree=p.x_degree)
                fit_p1d = fitting.LinearLSQFitter()
                p1d = fit_p1d(p_init1d, xx_plus1, wvl)
                p1d_list.append(p1d)

        from libs.iraf_helper import get_wat_spec, default_header_str
        wat_list = get_wat_spec(orders_w_solutions, p1d_list)

        # cards = [pyfits.Card.fromstring(l.strip()) \
        #          for l in open("echell_2dspec.header")]
        cards = [pyfits.Card.fromstring(l.strip()) \
                 for l in default_header_str]

        wat = "wtype=multispec " + " ".join(wat_list)
        char_per_line = 68
        num_line, remainder = divmod(len(wat), char_per_line)
        for i in range(num_line):
            k = "WAT2_%03d" % (i + 1, )
            v = wat[char_per_line * i:char_per_line * (i + 1)]
            #print k, v
            c = pyfits.Card(k, v)
            cards.append(c)
        if remainder > 0:
            i = num_line
            k = "WAT2_%03d" % (i + 1, )
            v = wat[char_per_line * i:]
            #print k, v
            c = pyfits.Card(k, v)
            cards.append(c)

        if 1:
            # save fits with empty header

            header = pyfits.Header(cards)
            hdu = pyfits.PrimaryHDU(header=header,
                                    data=np.array([]).reshape((0, 0)))

            from libs.storage_descriptions import SKY_WVLSOL_FITS_DESC
            from libs.products import PipelineImage
            oh_sol_products.add(SKY_WVLSOL_FITS_DESC,
                                PipelineImage([], np.array(wvl_sol)))

            igr_storage.store(oh_sol_products,
                              mastername=sky_filenames[0],
                              masterhdu=hdu)

            #fn = sky_path.get_secondary_path("wvlsol_v1.fits")
            #hdu.writeto(fn, clobber=True)

        if 0:
            # plot all spectra
            for w, s in zip(wvl_sol, s_list):
                plot(w, s)

    if 1:
        # filter out the line indices not well fit by the surface

        keys = reidentified_lines_map.keys()
        di_list = [len(reidentified_lines_map[k_][0]) for k_ in keys]

        endi_list = np.add.accumulate(di_list)

        filter_mask = [
            m[endi - di:endi] for di, endi in zip(di_list, endi_list)
        ]
        #from itertools import compress
        # _ = [list(compress(indices, mm)) for indices, mm \
        #      in zip(line_indices_list, filter_mask)]
        # line_indices_list_filtered = _

        reidentified_lines_ = [reidentified_lines_map[k_] for k_ in keys]
        _ = [(v_[0][mm], v_[1][mm]) for v_, mm \
             in zip(reidentified_lines_, filter_mask)]

        reidentified_lines_map_filtered = dict(zip(orders_w_solutions, _))

        if 1:
            from matplotlib.figure import Figure

            fig1 = Figure(figsize=(12, 7))
            check_fit(fig1, xl, yl, zl, p, orders_w_solutions,
                      reidentified_lines_map)
            fig1.tight_layout()

            fig2 = Figure(figsize=(12, 7))
            check_fit(fig2, xl[m], yl[m], zl[m], p, orders_w_solutions,
                      reidentified_lines_map_filtered)
            fig2.tight_layout()

    if 1:
        from libs.qa_helper import figlist_to_pngs
        sky_figs = igr_path.get_section_filename_base(
            "QA_PATH", "oh_fit2d", "oh_fit2d_" + sky_basename)
        figlist_to_pngs(sky_figs, [fig1, fig2])

    if 1:
        from libs.products import ProductDB
        sky_db_name = igr_path.get_section_filename_base(
            "PRIMARY_CALIB_PATH",
            "sky.db",
        )

        sky_db = ProductDB(sky_db_name)
        sky_db.update(band, sky_basename)
예제 #10
0
def process_distortion_sky_band(utdate, refdate, band, obsids, config):

    from libs.products import ProductDB, PipelineStorage

    igr_path = IGRINSPath(config, utdate)

    igr_storage = PipelineStorage(igr_path)

    sky_filenames = igr_path.get_filenames(band, obsids)

    sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]

    master_obsid = obsids[0]

    flaton_db_name = igr_path.get_section_filename_base(
        "PRIMARY_CALIB_PATH",
        "flat_on.db",
    )
    flaton_db = ProductDB(flaton_db_name)

    # thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
    #                                                     "thar.db",
    #                                                     )
    # thar_db = ProductDB(thar_db_name)

    from libs.storage_descriptions import (COMBINED_IMAGE_DESC,
                                           ONED_SPEC_JSON_DESC)
    raw_spec_products = igr_storage.load(
        [COMBINED_IMAGE_DESC, ONED_SPEC_JSON_DESC], sky_basename)

    # raw_spec_products = PipelineProducts.load(sky_path.get_secondary_path("raw_spec"))

    from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC

    wvlsol_products = igr_storage.load([SKY_WVLSOL_JSON_DESC],
                                       sky_basename)[SKY_WVLSOL_JSON_DESC]

    orders_w_solutions = wvlsol_products["orders"]
    wvl_solutions = wvlsol_products["wvl_sol"]

    ap = load_aperture2(igr_storage, band, master_obsid, flaton_db,
                        raw_spec_products[ONED_SPEC_JSON_DESC]["orders"],
                        orders_w_solutions)
    #orders_w_solutions = ap.orders

    if 1:  # load reference data
        from libs.master_calib import load_sky_ref_data

        ref_utdate = config.get_value("REFDATE", utdate)

        sky_ref_data = load_sky_ref_data(ref_utdate, band)

        ohlines_db = sky_ref_data["ohlines_db"]
        ref_ohline_indices = sky_ref_data["ohline_indices"]

        orders_w_solutions = wvlsol_products["orders"]
        wvl_solutions = wvlsol_products["wvl_sol"]

    if 1:

        n_slice_one_direction = 2
        n_slice = n_slice_one_direction * 2 + 1
        i_center = n_slice_one_direction
        slit_slice = np.linspace(0., 1., n_slice + 1)

        slice_center = (slit_slice[i_center], slit_slice[i_center + 1])
        slice_up = [(slit_slice[i_center+i], slit_slice[i_center+i+1]) \
                    for i in range(1, n_slice_one_direction+1)]
        slice_down = [(slit_slice[i_center-i-1], slit_slice[i_center-i]) \
                      for i in range(n_slice_one_direction)]

        d = raw_spec_products[COMBINED_IMAGE_DESC].data
        s_center = ap.extract_spectra_v2(d, slice_center[0], slice_center[1])

        s_up, s_down = [], []
        for s1, s2 in slice_up:
            s = ap.extract_spectra_v2(d, s1, s2)
            s_up.append(s)
        for s1, s2 in slice_down:
            s = ap.extract_spectra_v2(d, s1, s2)
            s_down.append(s)

    if 1:
        # now fit

        #ohline_indices = [ref_ohline_indices[o] for o in orders_w_solutions]

        if 0:

            def test_order(oi):
                ax = subplot(111)
                ax.plot(wvl_solutions[oi], s_center[oi])
                #ax.plot(wvl_solutions[oi], raw_spec_products["specs"][oi])
                o = orders[oi]
                line_indices = ref_ohline_indices[o]
                for li in line_indices:
                    um = np.take(ohlines_db.um, li)
                    intensity = np.take(ohlines_db.intensity, li)
                    ax.vlines(um, ymin=0, ymax=-intensity)

        from libs.reidentify_ohlines import fit_ohlines, fit_ohlines_pixel

        def get_reidentified_lines_OH(orders_w_solutions, wvl_solutions,
                                      s_center):
            ref_pixel_list, reidentified_lines = \
                            fit_ohlines(ohlines_db, ref_ohline_indices,
                                        orders_w_solutions,
                                        wvl_solutions, s_center)

            reidentified_lines_map = dict(
                zip(orders_w_solutions, reidentified_lines))
            return reidentified_lines_map, ref_pixel_list

        if band == "H":
            reidentified_lines_map, ref_pixel_list_oh = \
                       get_reidentified_lines_OH(orders_w_solutions,
                                                 wvl_solutions,
                                                 s_center)

            def refit_centroid(s_center, ref_pixel_list=ref_pixel_list_oh):
                centroids = fit_ohlines_pixel(s_center, ref_pixel_list)
                return centroids

        else:  # band K
            reidentified_lines_map, ref_pixel_list_oh = \
                       get_reidentified_lines_OH(orders_w_solutions,
                                                 wvl_solutions,
                                                 s_center)

            import libs.master_calib as master_calib
            fn = "hitran_bootstrap_K_%s.json" % ref_utdate
            bootstrap_name = master_calib.get_master_calib_abspath(fn)
            import json
            bootstrap = json.load(open(bootstrap_name))

            import libs.hitran as hitran
            r, ref_pixel_dict_hitrans = hitran.reidentify(
                wvl_solutions, s_center, bootstrap)

            # for i, s in r.items():
            #     ss = reidentified_lines_map[int(i)]
            #     ss0 = np.concatenate([ss[0], s["pixel"]])
            #     ss1 = np.concatenate([ss[1], s["wavelength"]])
            #     reidentified_lines_map[int(i)] = (ss0, ss1)

            #reidentified_lines_map, ref_pixel_list

            def refit_centroid(s_center,
                               ref_pixel_list=ref_pixel_list_oh,
                               ref_pixel_dict_hitrans=ref_pixel_dict_hitrans):
                centroids_oh = fit_ohlines_pixel(s_center, ref_pixel_list)

                s_dict = dict(zip(orders_w_solutions, s_center))
                centroids_dict_hitrans = hitran.fit_hitrans_pixel(
                    s_dict, ref_pixel_dict_hitrans)
                centroids = []
                for o, c_oh in zip(orders_w_solutions, centroids_oh):
                    if o in centroids_dict_hitrans:
                        c = np.concatenate(
                            [c_oh, centroids_dict_hitrans[o]["pixel"]])
                        centroids.append(c)
                    else:
                        centroids.append(c_oh)

                return centroids

        # reidentified_lines_map = get_reidentified_lines(orders_w_solutions,
        #                                                 wvl_solutions,
        #                                                 s_center)

    if 1:
        # TODO: we should not need this, instead recycle from preivious step.
        fitted_centroid_center = refit_centroid(s_center)
        # fitted_centroid_center = fit_ohlines_pixel(s_center,
        #                                            ref_pixel_list)

        d_shift_up = []
        for s in s_up:
            # TODO: ref_pixel_list_filtered need to be updated with recent fit.
            fitted_centroid = refit_centroid(s)
            # fitted_centroid = fit_ohlines_pixel(s,
            #                                     ref_pixel_list)
            d_shift = [
                b - a for a, b in zip(fitted_centroid_center, fitted_centroid)
            ]
            d_shift_up.append(d_shift)

        d_shift_down = []
        for s in s_down:
            # TODO: ref_pixel_list_filtered need to be updated with recent fit.
            fitted_centroid = refit_centroid(s)
            # fitted_centroid = fit_ohlines_pixel(s,
            #                                     ref_pixel_list)
            #fitted_centroid_center,
            d_shift = [
                b - a for a, b in zip(fitted_centroid_center, fitted_centroid)
            ]
            d_shift_down.append(d_shift)

    if 1:
        # now fit
        orders = orders_w_solutions

        x_domain = [0, 2048]
        y_domain = [orders[0] - 2, orders[-1] + 2]

        xl = np.concatenate(fitted_centroid_center)

        yl_ = [
            o + np.zeros_like(x_)
            for o, x_ in zip(orders, fitted_centroid_center)
        ]
        yl = np.concatenate(yl_)

        from libs.ecfit import fit_2dspec, check_fit_simple

        zl_list = [np.concatenate(d_) for d_ \
                   in d_shift_down[::-1] + d_shift_up]

        pm_list = []
        for zl in zl_list:
            p, m = fit_2dspec(xl,
                              yl,
                              zl,
                              x_degree=1,
                              y_degree=1,
                              x_domain=x_domain,
                              y_domain=y_domain)
            pm_list.append((p, m))

        zz_std_list = []
        for zl, (p, m) in zip(zl_list, pm_list):
            z_m = p(xl[m], yl[m])
            zz = z_m - zl[m]
            zz_std_list.append(zz.std())

        fig_list = []
        from matplotlib.figure import Figure
        for zl, (p, m) in zip(zl_list, pm_list):
            fig = Figure()
            check_fit_simple(fig, xl[m], yl[m], zl[m], p, orders)
            fig_list.append(fig)

    if 1:
        xi = np.linspace(0, 2048, 128 + 1)
        from astropy.modeling import fitting
        from astropy.modeling.polynomial import Chebyshev2D
        x_domain = [0, 2048]
        y_domain = [0., 1.]

        p2_list = []
        for o in orders:
            oi = np.zeros_like(xi) + o
            shift_list = []
            for p, m in pm_list[:n_slice_one_direction]:
                shift_list.append(p(xi, oi))

            shift_list.append(np.zeros_like(xi))

            for p, m in pm_list[n_slice_one_direction:]:
                shift_list.append(p(xi, oi))

            p_init = Chebyshev2D(x_degree=1,
                                 y_degree=2,
                                 x_domain=x_domain,
                                 y_domain=y_domain)
            f = fitting.LinearLSQFitter()

            yi = 0.5 * (slit_slice[:-1] + slit_slice[1:])
            xl, yl = np.meshgrid(xi, yi)
            zl = np.array(shift_list)
            p = f(p_init, xl, yl, zl)

            p2_list.append(p)

    if 1:
        p2_dict = dict(zip(orders, p2_list))

        # save order_map, etc

        order_map = ap.make_order_map()
        slitpos_map = ap.make_slitpos_map()
        order_map2 = ap.make_order_map(mask_top_bottom=True)

        slitoffset_map = np.empty_like(slitpos_map)
        slitoffset_map.fill(np.nan)

        wavelength_map = np.empty_like(slitpos_map)
        wavelength_map.fill(np.nan)

        from scipy.interpolate import interp1d
        for o, wvl in zip(ap.orders, wvl_solutions):
            xi = np.arange(0, 2048)
            xl, yl = np.meshgrid(xi, xi)
            msk = order_map == o

            xl_msk = xl[msk]
            slitoffset_map_msk = p2_dict[o](xl_msk, slitpos_map[msk])
            slitoffset_map[msk] = slitoffset_map_msk

            wvl_interp1d = interp1d(xi, wvl, bounds_error=False)
            wavelength_map[msk] = wvl_interp1d(xl_msk - slitoffset_map_msk)

        from libs.storage_descriptions import (ORDERMAP_FITS_DESC,
                                               SLITPOSMAP_FITS_DESC,
                                               SLITOFFSET_FITS_DESC,
                                               WAVELENGTHMAP_FITS_DESC,
                                               ORDERMAP_MASKED_FITS_DESC)
        from libs.products import PipelineImage, PipelineProducts
        products = PipelineProducts("Distortion map")

        for desc, im in [(ORDERMAP_FITS_DESC, order_map),
                         (SLITPOSMAP_FITS_DESC, slitpos_map),
                         (SLITOFFSET_FITS_DESC, slitoffset_map),
                         (WAVELENGTHMAP_FITS_DESC, wavelength_map),
                         (ORDERMAP_MASKED_FITS_DESC, order_map2)]:
            products.add(desc, PipelineImage([], im))

        igr_storage.store(products,
                          mastername=sky_filenames[0],
                          masterhdu=None)

        from libs.qa_helper import figlist_to_pngs
        sky_figs = igr_path.get_section_filename_base(
            "QA_PATH", "oh_distortion", "oh_distortion_" + sky_basename)
        print fig_list
        figlist_to_pngs(sky_figs, fig_list)

    if 0:
        # test
        x = np.arange(2048, dtype="d")
        oi = 10
        o = orders[oi]

        yi = 0.5 * (slit_slice[:-1] + slit_slice[1:])

        ax1 = subplot(211)
        s1 = s_up[-1][oi]
        s2 = s_down[-1][oi]

        ax1.plot(x, s1)
        ax1.plot(x, s2)

        ax2 = subplot(212, sharex=ax1, sharey=ax1)
        dx1 = p2_dict[o](x, yi[-1] + np.zeros_like(x))
        ax2.plot(x - dx1, s1)

        dx2 = p2_dict[o](x, yi[0] + np.zeros_like(x))
        ax2.plot(x - dx2, s2)
예제 #11
0
def process_abba_band(recipe,
                      utdate,
                      refdate,
                      band,
                      obsids,
                      frametypes,
                      config,
                      do_interactive_figure=False,
                      threshold_a0v=0.1,
                      objname="",
                      multiply_model_a0v=False,
                      html_output=False):

    from libs.products import ProductDB, PipelineStorage

    if recipe == "A0V_AB":

        FIX_TELLURIC = False

    elif recipe == "STELLAR_AB":

        FIX_TELLURIC = True

    elif recipe == "EXTENDED_AB":

        FIX_TELLURIC = True

    elif recipe == "EXTENDED_ONOFF":

        FIX_TELLURIC = True

    else:
        raise ValueError("Unsupported Recipe : %s" % recipe)

    if 1:

        igr_path = IGRINSPath(config, utdate)

        igr_storage = PipelineStorage(igr_path)

        obj_filenames = igr_path.get_filenames(band, obsids)

        master_obsid = obsids[0]

        tgt_basename = os.path.splitext(os.path.basename(obj_filenames[0]))[0]

        db = {}
        basenames = {}

        db_types = ["flat_off", "flat_on", "thar", "sky"]

        for db_type in db_types:

            db_name = igr_path.get_section_filename_base(
                "PRIMARY_CALIB_PATH",
                "%s.db" % db_type,
            )
            db[db_type] = ProductDB(db_name)

        # db on output path
        db_types = ["a0v"]

        for db_type in db_types:

            db_name = igr_path.get_section_filename_base(
                "OUTDATA_PATH",
                "%s.db" % db_type,
            )
            db[db_type] = ProductDB(db_name)

        # to get basenames
        db_types = ["flat_off", "flat_on", "thar", "sky"]
        if FIX_TELLURIC:
            db_types.append("a0v")

        for db_type in db_types:
            basenames[db_type] = db[db_type].query(band, master_obsid)

    if 1:  # make aperture
        from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC

        sky_basename = db["sky"].query(band, master_obsid)
        wvlsol_products = igr_storage.load([SKY_WVLSOL_JSON_DESC],
                                           sky_basename)[SKY_WVLSOL_JSON_DESC]

        orders_w_solutions = wvlsol_products["orders"]
        wvl_solutions = map(np.array, wvlsol_products["wvl_sol"])

    # prepare i1i2_list
    from libs.storage_descriptions import ORDER_FLAT_JSON_DESC
    prod = igr_storage.load([ORDER_FLAT_JSON_DESC],
                            basenames["flat_on"])[ORDER_FLAT_JSON_DESC]

    new_orders = prod["orders"]
    i1i2_list_ = prod["i1i2_list"]

    order_indices = []

    for o in orders_w_solutions:
        o_new_ind = np.searchsorted(new_orders, o)
        order_indices.append(o_new_ind)

    i1i2_list = get_fixed_i1i2_list(order_indices, i1i2_list_)

    from libs.storage_descriptions import (SPEC_FITS_DESC, SN_FITS_DESC)

    if 1:  # load target spectrum
        tgt_spec_ = igr_storage.load([SPEC_FITS_DESC],
                                     tgt_basename)[SPEC_FITS_DESC]
        tgt_spec = list(tgt_spec_.data)

        tgt_sn_ = igr_storage.load([SN_FITS_DESC], tgt_basename)[SN_FITS_DESC]
        tgt_sn = list(tgt_sn_.data)

    fig_list = []

    # telluric
    if 1:  #FIX_TELLURIC:
        A0V_basename = db["a0v"].query(band, master_obsid)

        from libs.storage_descriptions import SPEC_FITS_FLATTENED_DESC
        telluric_cor_ = igr_storage.load(
            [SPEC_FITS_FLATTENED_DESC], A0V_basename)[SPEC_FITS_FLATTENED_DESC]

        #A0V_path = ProductPath(igr_path, A0V_basename)
        #fn = A0V_path.get_secondary_path("spec_flattened.fits")
        telluric_cor = list(telluric_cor_.data)

        a0v_spec_ = igr_storage.load([SPEC_FITS_DESC],
                                     A0V_basename)[SPEC_FITS_DESC]

        a0v_spec = list(a0v_spec_.data)

        if 1:

            if do_interactive_figure:
                from matplotlib.pyplot import figure as Figure
            else:
                from matplotlib.figure import Figure
            fig1 = Figure(figsize=(12, 6))
            fig_list.append(fig1)

            ax1a = fig1.add_subplot(211)
            ax1b = fig1.add_subplot(212, sharex=ax1a)

            for wvl, s, sn in zip(wvl_solutions, tgt_spec, tgt_sn):
                #s[s<0] = np.nan
                #sn[sn<0] = np.nan

                ax1a.plot(wvl, s)
                ax1b.plot(wvl, sn)

            ax1a.set_ylabel("Counts [DN]")
            ax1b.set_ylabel("S/N per Res. Element")
            ax1b.set_xlabel("Wavelength [um]")

            ax1a.set_title(objname)

        if FIX_TELLURIC:

            fig2 = Figure(figsize=(12, 6))
            fig_list.append(fig2)

            ax2a = fig2.add_subplot(211)
            ax2b = fig2.add_subplot(212, sharex=ax2a)

            #from libs.stddev_filter import window_stdev

            tgt_spec_cor = []
            #for s, t in zip(s_list, telluric_cor):
            for s, t, t2 in zip(tgt_spec, a0v_spec, telluric_cor):

                st = s / t
                #print np.percentile(t[np.isfinite(t)], 95), threshold_a0v
                t0 = np.percentile(t[np.isfinite(t)], 95) * threshold_a0v
                st[t < t0] = np.nan

                st[t2 < threshold_a0v] = np.nan

                tgt_spec_cor.append(st)

            if multiply_model_a0v:
                # multiply by A0V model
                from libs.a0v_spec import A0VSpec
                a0v_model = A0VSpec()

                a0v_interp1d = a0v_model.get_flux_interp1d(1.3,
                                                           2.5,
                                                           flatten=True,
                                                           smooth_pixel=32)
                for wvl, s in zip(wvl_solutions, tgt_spec_cor):

                    aa = a0v_interp1d(wvl)
                    s *= aa

            for wvl, s, t in zip(wvl_solutions, tgt_spec_cor, telluric_cor):

                ax2a.plot(wvl, t, "0.8", zorder=0.5)
                ax2b.plot(wvl, s, zorder=0.5)

            s_max_list = []
            s_min_list = []
            for s in tgt_spec_cor[3:-3]:
                s_max_list.append(np.nanmax(s))
                s_min_list.append(np.nanmin(s))
            s_max = np.max(s_max_list)
            s_min = np.min(s_min_list)
            ds_pad = 0.05 * (s_max - s_min)

            ax2a.set_ylabel("A0V flattened")
            ax2a.set_ylim(-0.05, 1.1)
            ax2b.set_ylabel("Target / A0V")
            ax2b.set_xlabel("Wavelength [um]")

            ax2b.set_ylim(s_min - ds_pad, s_max + ds_pad)
            ax2a.set_title(objname)

    # save figures
    if fig_list:
        for fig in fig_list:
            fig.tight_layout()

        figout = igr_path.get_section_filename_base("QA_PATH",
                                                    "spec_" + tgt_basename,
                                                    "spec_" + tgt_basename)
        #figout = obj_path.get_secondary_path("spec", "spec_dir")
        from libs.qa_helper import figlist_to_pngs
        figlist_to_pngs(figout, fig_list)

    # save html

    if html_output:
        dirname = config.get_value('HTML_PATH', utdate)
        objroot = "%04d" % (master_obsid, )
        html_save(utdate, dirname, objroot, band, orders_w_solutions,
                  wvl_solutions, tgt_spec, tgt_sn, i1i2_list)

        if FIX_TELLURIC:
            objroot = "%04dA0V" % (master_obsid, )
            html_save(utdate,
                      dirname,
                      objroot,
                      band,
                      orders_w_solutions,
                      wvl_solutions,
                      telluric_cor,
                      tgt_spec_cor,
                      i1i2_list,
                      spec_js_name="jj_a0v.js")

    if do_interactive_figure:
        import matplotlib.pyplot as plt
        plt.show()
예제 #12
0
def process_abba_band(recipe, utdate, refdate, band, obsids, frametypes,
                      config,
                      do_interactive_figure=False,
                      threshold_a0v=0.1,
                      objname="",
                      multiply_model_a0v=False,
                      html_output=False):

    from libs.products import ProductDB, PipelineStorage

    if recipe == "A0V_AB":

        FIX_TELLURIC=False

    elif recipe == "STELLAR_AB":

        FIX_TELLURIC=True

    elif recipe == "EXTENDED_AB":

        FIX_TELLURIC=True

    elif recipe == "EXTENDED_ONOFF":

        FIX_TELLURIC=True

    else:
        raise ValueError("Unsupported Recipe : %s" % recipe)

    if 1:

        igr_path = IGRINSPath(config, utdate)

        igr_storage = PipelineStorage(igr_path)

        obj_filenames = igr_path.get_filenames(band, obsids)

        master_obsid = obsids[0]

        tgt_basename = os.path.splitext(os.path.basename(obj_filenames[0]))[0]

        db = {}
        basenames = {}

        db_types = ["flat_off", "flat_on", "thar", "sky"]

        for db_type in db_types:

            db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
                                                        "%s.db" % db_type,
                                                        )
            db[db_type] = ProductDB(db_name)

        # db on output path
        db_types = ["a0v"]

        for db_type in db_types:

            db_name = igr_path.get_section_filename_base("OUTDATA_PATH",
                                                        "%s.db" % db_type,
                                                        )
            db[db_type] = ProductDB(db_name)


        # to get basenames
        db_types = ["flat_off", "flat_on", "thar", "sky"]
        if FIX_TELLURIC:
            db_types.append("a0v")

        for db_type in db_types:
            basenames[db_type] = db[db_type].query(band, master_obsid)






    if 1: # make aperture
        from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC

        sky_basename = db["sky"].query(band, master_obsid)
        wvlsol_products = igr_storage.load([SKY_WVLSOL_JSON_DESC],
                                           sky_basename)[SKY_WVLSOL_JSON_DESC]

        orders_w_solutions = wvlsol_products["orders"]
        wvl_solutions = map(np.array, wvlsol_products["wvl_sol"])



    # prepare i1i2_list
    from libs.storage_descriptions import ORDER_FLAT_JSON_DESC
    prod = igr_storage.load([ORDER_FLAT_JSON_DESC],
                            basenames["flat_on"])[ORDER_FLAT_JSON_DESC]

    new_orders = prod["orders"]
    i1i2_list_ = prod["i1i2_list"]


    order_indices = []

    for o in orders_w_solutions:
        o_new_ind = np.searchsorted(new_orders, o)
        order_indices.append(o_new_ind)

    i1i2_list = get_fixed_i1i2_list(order_indices, i1i2_list_)


    from libs.storage_descriptions import (SPEC_FITS_DESC,
                                           SN_FITS_DESC)

    if 1: # load target spectrum
        tgt_spec_ = igr_storage.load([SPEC_FITS_DESC],
                                     tgt_basename)[SPEC_FITS_DESC]
        tgt_spec = list(tgt_spec_.data)

        tgt_sn_ = igr_storage.load([SN_FITS_DESC],
                                   tgt_basename)[SN_FITS_DESC]
        tgt_sn = list(tgt_sn_.data)

    fig_list = []

    # telluric
    if 1: #FIX_TELLURIC:
        A0V_basename = db["a0v"].query(band, master_obsid)

        from libs.storage_descriptions import SPEC_FITS_FLATTENED_DESC
        telluric_cor_ = igr_storage.load([SPEC_FITS_FLATTENED_DESC],
                                         A0V_basename)[SPEC_FITS_FLATTENED_DESC]

        #A0V_path = ProductPath(igr_path, A0V_basename)
        #fn = A0V_path.get_secondary_path("spec_flattened.fits")
        telluric_cor = list(telluric_cor_.data)


        a0v_spec_ = igr_storage.load([SPEC_FITS_DESC],
                                     A0V_basename)[SPEC_FITS_DESC]

        a0v_spec = list(a0v_spec_.data)


        if 1:


            if do_interactive_figure:
                from matplotlib.pyplot import figure as Figure
            else:
                from matplotlib.figure import Figure
            fig1 = Figure(figsize=(12,6))
            fig_list.append(fig1)

            ax1a = fig1.add_subplot(211)
            ax1b = fig1.add_subplot(212, sharex=ax1a)

            for wvl, s, sn in zip(wvl_solutions, tgt_spec, tgt_sn):
                #s[s<0] = np.nan
                #sn[sn<0] = np.nan

                ax1a.plot(wvl, s)
                ax1b.plot(wvl, sn)

            ax1a.set_ylabel("Counts [DN]")
            ax1b.set_ylabel("S/N per Res. Element")
            ax1b.set_xlabel("Wavelength [um]")

            ax1a.set_title(objname)


        if FIX_TELLURIC:

            fig2 = Figure(figsize=(12,6))
            fig_list.append(fig2)

            ax2a = fig2.add_subplot(211)
            ax2b = fig2.add_subplot(212, sharex=ax2a)

            #from libs.stddev_filter import window_stdev



            tgt_spec_cor = []
            #for s, t in zip(s_list, telluric_cor):
            for s, t, t2 in zip(tgt_spec, a0v_spec, telluric_cor):

                st = s/t
                #print np.percentile(t[np.isfinite(t)], 95), threshold_a0v
                t0 = np.percentile(t[np.isfinite(t)], 95)*threshold_a0v
                st[t<t0] = np.nan

                st[t2 < threshold_a0v] = np.nan

                tgt_spec_cor.append(st)


            if multiply_model_a0v:
                # multiply by A0V model
                from libs.a0v_spec import A0VSpec
                a0v_model = A0VSpec()

                a0v_interp1d = a0v_model.get_flux_interp1d(1.3, 2.5,
                                                           flatten=True,
                                                           smooth_pixel=32)
                for wvl, s in zip(wvl_solutions,
                                  tgt_spec_cor):

                    aa = a0v_interp1d(wvl)
                    s *= aa


            for wvl, s, t in zip(wvl_solutions,
                                 tgt_spec_cor,
                                 telluric_cor):

                ax2a.plot(wvl, t, "0.8", zorder=0.5)
                ax2b.plot(wvl, s, zorder=0.5)


            s_max_list = []
            s_min_list = []
            for s in tgt_spec_cor[3:-3]:
                s_max_list.append(np.nanmax(s))
                s_min_list.append(np.nanmin(s))
            s_max = np.max(s_max_list)
            s_min = np.min(s_min_list)
            ds_pad = 0.05 * (s_max - s_min)

            ax2a.set_ylabel("A0V flattened")
            ax2a.set_ylim(-0.05, 1.1)
            ax2b.set_ylabel("Target / A0V")
            ax2b.set_xlabel("Wavelength [um]")

            ax2b.set_ylim(s_min-ds_pad, s_max+ds_pad)
            ax2a.set_title(objname)



    # save figures
    if fig_list:
        for fig in fig_list:
            fig.tight_layout()

        figout = igr_path.get_section_filename_base("QA_PATH",
                                                    "spec_"+tgt_basename,
                                                    "spec_"+tgt_basename)
        #figout = obj_path.get_secondary_path("spec", "spec_dir")
        from libs.qa_helper import figlist_to_pngs
        figlist_to_pngs(figout, fig_list)

    # save html

    if html_output:
        dirname = config.get_value('HTML_PATH', utdate)
        objroot = "%04d" % (master_obsid,)
        html_save(utdate, dirname, objroot, band,
                  orders_w_solutions, wvl_solutions,
                  tgt_spec, tgt_sn, i1i2_list)

        if FIX_TELLURIC:
            objroot = "%04dA0V" % (master_obsid,)
            html_save(utdate, dirname, objroot, band,
                      orders_w_solutions, wvl_solutions,
                      telluric_cor, tgt_spec_cor, i1i2_list,
                      spec_js_name="jj_a0v.js")


    if do_interactive_figure:
        import matplotlib.pyplot as plt
        plt.show()