def save_db(helper, band, obsids): thar_filenames = helper.get_filenames(band, obsids) thar_basename = os.path.splitext(os.path.basename(thar_filenames[0]))[0] if 1: from libs.products import ProductDB thar_db_name = helper.igr_path.get_section_filename_base("PRIMARY_CALIB_PATH", "thar.db", ) thar_db = ProductDB(thar_db_name) # os.path.join(igr_path.secondary_calib_path, # "thar.db")) thar_db.update(band, thar_basename)
def process_thar_band(utdate, refdate, band, obsids, config): from libs.products import ProductDB, PipelineStorage igr_path = IGRINSPath(config, utdate) igr_storage = PipelineStorage(igr_path) thar_filenames = igr_path.get_filenames(band, obsids) thar_basename = os.path.splitext(os.path.basename(thar_filenames[0]))[0] thar_master_obsid = obsids[0] flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH", "flat_on.db", ) flaton_db = ProductDB(flaton_db_name) flaton_basename = flaton_db.query(band, thar_master_obsid) from libs.storage_descriptions import FLATCENTROID_SOL_JSON_DESC desc_list = [FLATCENTROID_SOL_JSON_DESC] products = igr_storage.load(desc_list, mastername=flaton_basename) aperture_solution_products = products[FLATCENTROID_SOL_JSON_DESC] # igrins_orders = {} # igrins_orders["H"] = range(99, 122) # igrins_orders["K"] = range(72, 92) if 1: bottomup_solutions = aperture_solution_products["bottom_up_solutions"] orders = range(len(bottomup_solutions)) ap = Apertures(orders, bottomup_solutions) if 1: from libs.process_thar import ThAr thar = ThAr(thar_filenames) thar_products = thar.process_thar(ap) if 1: # match order from libs.process_thar import match_order_thar from libs.master_calib import load_thar_ref_data #ref_date = "20140316" thar_ref_data = load_thar_ref_data(refdate, band) new_orders = match_order_thar(thar_products, thar_ref_data) print thar_ref_data["orders"] print new_orders ap = Apertures(new_orders, bottomup_solutions) from libs.storage_descriptions import ONED_SPEC_JSON_DESC thar_products[ONED_SPEC_JSON_DESC]["orders"] = new_orders if 1: hdu = pyfits.open(thar_filenames[0])[0] igr_storage.store(thar_products, mastername=thar_filenames[0], masterhdu=hdu) if 1: # measure shift of thar lines from reference spectra # load spec from libs.process_thar import reidentify_ThAr_lines thar_reidentified_products = reidentify_ThAr_lines(thar_products, thar_ref_data) igr_storage.store(thar_reidentified_products, mastername=thar_filenames[0], masterhdu=hdu) if 1: from libs.process_thar import (load_echelogram, align_echellogram_thar, check_thar_transorm, get_wavelength_solutions) ref_date = thar_ref_data["ref_date"] echel = load_echelogram(ref_date, band) thar_aligned_echell_products = \ align_echellogram_thar(thar_reidentified_products, echel, band, ap) # We do not save this product yet. # igr_storage.store(thar_aligned_echell_products, # mastername=thar_filenames[0], # masterhdu=hdu) fig_list = check_thar_transorm(thar_products, thar_aligned_echell_products) from libs.qa_helper import figlist_to_pngs thar_figs = igr_path.get_section_filename_base("QA_PATH", "thar", "thar_"+thar_basename) figlist_to_pngs(thar_figs, fig_list) thar_wvl_sol = get_wavelength_solutions(thar_aligned_echell_products, echel) igr_storage.store(thar_wvl_sol, mastername=thar_filenames[0], masterhdu=hdu) if 1: # make amp and order falt from libs.storage_descriptions import ONED_SPEC_JSON_DESC orders = thar_products[ONED_SPEC_JSON_DESC]["orders"] order_map = ap.make_order_map() #slitpos_map = ap.make_slitpos_map() # load flat on products #flat_on_params_name = flaton_path.get_secondary_path("flat_on_params") #flaton_products = PipelineProducts.load(flat_on_params_name) from libs.storage_descriptions import (FLAT_NORMED_DESC, FLAT_MASK_DESC) flaton_products = igr_storage.load([FLAT_NORMED_DESC, FLAT_MASK_DESC], flaton_basename) from libs.process_flat import make_order_flat, check_order_flat order_flat_products = make_order_flat(flaton_products, orders, order_map) #fn = thar_path.get_secondary_path("orderflat") #order_flat_products.save(fn, masterhdu=hdu) igr_storage.store(order_flat_products, mastername=flaton_basename, masterhdu=hdu) if 1: fig_list = check_order_flat(order_flat_products) from libs.qa_helper import figlist_to_pngs orderflat_figs = igr_path.get_section_filename_base("QA_PATH", "orderflat", "orderflat_"+thar_basename) figlist_to_pngs(orderflat_figs, fig_list) if 1: from libs.products import ProductDB thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH", "thar.db", ) thar_db = ProductDB(thar_db_name) # os.path.join(igr_path.secondary_calib_path, # "thar.db")) thar_db.update(band, thar_basename)
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on, config): from libs.products import PipelineStorage igr_path = IGRINSPath(config, utdate) igr_storage = PipelineStorage(igr_path) flat_off_filenames = igr_path.get_filenames(band, obsids_off) flat_on_filenames = igr_path.get_filenames(band, obsids_on) if 1: # process flat off flat_offs_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_off_filenames] flat_offs = [hdu.data for hdu in flat_offs_hdu_list] flat = FlatOff(flat_offs) flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100, sigma_clip2=5) igr_storage.store(flatoff_products, mastername=flat_off_filenames[0], masterhdu=flat_offs_hdu_list[0]) if 1: # flat on from libs.storage_descriptions import (FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC) desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC] flatoff_products = igr_storage.load(desc_list, mastername=flat_off_filenames[0]) flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames] flat_ons = [hdu.data for hdu in flat_on_hdu_list] from libs.master_calib import get_master_calib_abspath fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" % (refdate, band)) deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool) flat_on = FlatOn(flat_ons) flaton_products = flat_on.make_flaton_deadpixmap(flatoff_products, deadpix_mask_old=deadpix_mask_old) igr_storage.store(flaton_products, mastername=flat_on_filenames[0], masterhdu=flat_on_hdu_list[0]) if 1: # now trace the orders from libs.process_flat import trace_orders trace_products = trace_orders(flaton_products) hdu = pyfits.open(flat_on_filenames[0])[0] igr_storage.store(trace_products, mastername=flat_on_filenames[0], masterhdu=flat_on_hdu_list[0]) from libs.process_flat import trace_solutions trace_solution_products = trace_solutions(trace_products) igr_storage.store(trace_solution_products, mastername=flat_on_filenames[0], masterhdu=flat_on_hdu_list[0]) # plot qa figures. if 1: from libs.process_flat import check_trace_order from matplotlib.figure import Figure fig1 = Figure() check_trace_order(trace_products, fig1) if 1: from libs.process_flat import plot_trace_solutions fig2, fig3 = plot_trace_solutions(flaton_products, trace_solution_products) flatoff_basename = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0] flaton_basename = os.path.splitext(os.path.basename(flat_on_filenames[0]))[0] if 1: from libs.qa_helper import figlist_to_pngs aperture_figs = igr_path.get_section_filename_base("QA_PATH", "aperture_"+flaton_basename, "aperture_"+flaton_basename) figlist_to_pngs(aperture_figs, [fig1, fig2, fig3]) # save db if 1: from libs.products import ProductDB flatoff_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH", "flat_off.db", ) flatoff_db = ProductDB(flatoff_db_name) #dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0] flatoff_db.update(band, flatoff_basename) flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH", "flat_on.db", ) flaton_db = ProductDB(flaton_db_name) flaton_db.update(band, flaton_basename)
def process_wvlsol_band(utdate, refdate, band, obsids, config): from libs.products import ProductDB, PipelineStorage igr_path = IGRINSPath(config, utdate) igr_storage = PipelineStorage(igr_path) sky_filenames = igr_path.get_filenames(band, obsids) sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0] master_obsid = obsids[0] flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH", "flat_on.db", ) flaton_db = ProductDB(flaton_db_name) #flaton_basename = flaton_db.query(band, master_obsid) thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH", "thar.db", ) thar_db = ProductDB(thar_db_name) #thar_basename = thar_db.query(band, master_obsid) # flaton_db = ProductDB(os.path.join(igr_path.secondary_calib_path, # "flat_on.db")) # thar_db = ProductDB(os.path.join(igr_path.secondary_calib_path, # "thar.db")) ap = load_aperture(igr_storage, band, master_obsid, flaton_db, thar_db) if 1: # from libs.process_thar import get_1d_median_specs raw_spec_product = get_1d_median_specs(sky_filenames, ap) # sky_master_fn_ = os.path.splitext(os.path.basename(sky_names[0]))[0] # sky_master_fn = igr_path.get_secondary_calib_filename(sky_master_fn_) import astropy.io.fits as pyfits masterhdu = pyfits.open(sky_filenames[0])[0] igr_storage.store(raw_spec_product, mastername=sky_filenames[0], masterhdu=masterhdu) # fn = sky_path.get_secondary_path("raw_spec") # raw_spec_product.save(fn, # masterhdu=masterhdu) from libs.master_calib import load_sky_ref_data # ref_date = "20140316" refdate = config.get_value("REFDATE", utdate) sky_ref_data = load_sky_ref_data(refdate, band) if 1: # initial wavelength solution # this need to be fixed # thar_db.query(sky_master_obsid) # json_name_ = "SDC%s_%s_0003.median_spectra.wvlsol" % (band, # igrins_log.date) from libs.storage_descriptions import THAR_WVLSOL_JSON_DESC thar_basename = thar_db.query(band, master_obsid) thar_wvl_sol = igr_storage.load([THAR_WVLSOL_JSON_DESC], thar_basename)[THAR_WVLSOL_JSON_DESC] #print thar_wvl_sol.keys() #["wvl_sol"] #json_name = thar_path.get_secondary_path("wvlsol_v0") #json_name = igr_path.get_secondary_calib_filename(json_name_) #thar_wvl_sol = PipelineProducts.load(json_name) if 1: # Now we fit with gaussian profile for matched positions. ohline_indices = sky_ref_data["ohline_indices"] ohlines_db = sky_ref_data["ohlines_db"] wvl_solutions = thar_wvl_sol["wvl_sol"] if 0: # it would be better to iteratively refit the solution fn = sky_path.get_secondary_path("wvlsol_v1") p = PipelineProducts.load(fn) wvl_solutionv = p["wvl_sol"] orders_w_solutions_ = thar_wvl_sol["orders"] from libs.storage_descriptions import ONED_SPEC_JSON_DESC orders_w_solutions = [o for o in orders_w_solutions_ if o in raw_spec_product[ONED_SPEC_JSON_DESC]["orders"]] _ = dict(zip(raw_spec_product[ONED_SPEC_JSON_DESC]["orders"], raw_spec_product[ONED_SPEC_JSON_DESC]["specs"])) s_list = [_[o]for o in orders_w_solutions] from libs.reidentify_ohlines import fit_ohlines ref_pixel_list, reidentified_lines = \ fit_ohlines(ohlines_db, ohline_indices, orders_w_solutions, wvl_solutions, s_list) # from scipy.interpolate import interp1d # from reidentify import reidentify_lines_all x = np.arange(2048) # line_indices_list = [ref_ohline_indices[str(o)] for o in igrins_orders[band]] ###### not fit identified lines from libs.ecfit import get_ordered_line_data, fit_2dspec, check_fit # d_x_wvl = {} # for order, z in echel.zdata.items(): # xy_T = affine_tr.transform(np.array([z.x, z.y]).T) # x_T = xy_T[:,0] # d_x_wvl[order]=(x_T, z.wvl) reidentified_lines_map = dict(zip(orders_w_solutions, reidentified_lines)) if band == "K": import libs.master_calib as master_calib fn = "hitran_bootstrap_K_%s.json" % refdate bootstrap_name = master_calib.get_master_calib_abspath(fn) import json bootstrap = json.load(open(bootstrap_name)) import libs.hitran as hitran r, ref_pixel_list = hitran.reidentify(wvl_solutions, s_list, bootstrap) # json_name = "hitran_reidentified_K_%s.json" % igrins_log.date # r = json.load(open(json_name)) for i, s in r.items(): ss = reidentified_lines_map[int(i)] ss0 = np.concatenate([ss[0], s["pixel"]]) ss1 = np.concatenate([ss[1], s["wavelength"]]) reidentified_lines_map[int(i)] = (ss0, ss1) xl, yl, zl = get_ordered_line_data(reidentified_lines_map) # xl : pixel # yl : order # zl : wvl * order x_domain = [0, 2047] y_domain = [orders_w_solutions[0]-2, orders_w_solutions[-1]+2] x_degree, y_degree = 4, 3 #x_degree, y_degree = 3, 2 p, m = fit_2dspec(xl, yl, zl, x_degree=x_degree, y_degree=y_degree, x_domain=x_domain, y_domain=y_domain) # derive wavelengths. xx = np.arange(2048) wvl_sol = [] for o in orders_w_solutions: oo = np.empty_like(xx) oo.fill(o) wvl = p(xx, oo) / o wvl_sol.append(list(wvl)) oh_sol_products = PipelineProducts("Wavelength solution based on ohlines") #from libs.process_thar import ONED_SPEC_JSON from libs.products import PipelineDict from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC oh_sol_products.add(SKY_WVLSOL_JSON_DESC, PipelineDict(orders=orders_w_solutions, wvl_sol=wvl_sol)) if 1: if 1: # save as WAT fits header xx = np.arange(0, 2048) xx_plus1 = np.arange(1, 2048+1) from astropy.modeling import models, fitting # We convert 2d chebyshev solution to a seriese of 1d # chebyshev. For now, use naive (and inefficient) # approach of refitting the solution with 1d. Should be # reimplemented. p1d_list = [] for o in orders_w_solutions: oo = np.empty_like(xx) oo.fill(o) wvl = p(xx, oo) / o * 1.e4 # um to angstrom p_init1d = models.Chebyshev1D(domain=[1, 2048], degree=p.x_degree) fit_p1d = fitting.LinearLSQFitter() p1d = fit_p1d(p_init1d, xx_plus1, wvl) p1d_list.append(p1d) from libs.iraf_helper import get_wat_spec, default_header_str wat_list = get_wat_spec(orders_w_solutions, p1d_list) # cards = [pyfits.Card.fromstring(l.strip()) \ # for l in open("echell_2dspec.header")] cards = [pyfits.Card.fromstring(l.strip()) \ for l in default_header_str] wat = "wtype=multispec " + " ".join(wat_list) char_per_line = 68 num_line, remainder = divmod(len(wat), char_per_line) for i in range(num_line): k = "WAT2_%03d" % (i+1,) v = wat[char_per_line*i:char_per_line*(i+1)] #print k, v c = pyfits.Card(k, v) cards.append(c) if remainder > 0: i = num_line k = "WAT2_%03d" % (i+1,) v = wat[char_per_line*i:] #print k, v c = pyfits.Card(k, v) cards.append(c) if 1: # save fits with empty header header = pyfits.Header(cards) hdu = pyfits.PrimaryHDU(header=header, data=np.array([]).reshape((0,0))) from libs.storage_descriptions import SKY_WVLSOL_FITS_DESC from libs.products import PipelineImage oh_sol_products.add(SKY_WVLSOL_FITS_DESC, PipelineImage([], np.array([]).reshape((0,0)))) igr_storage.store(oh_sol_products, mastername=sky_filenames[0], masterhdu=hdu) #fn = sky_path.get_secondary_path("wvlsol_v1.fits") #hdu.writeto(fn, clobber=True) if 0: # plot all spectra for w, s in zip(wvl_sol, s_list): plot(w, s) if 1: # filter out the line indices not well fit by the surface keys = reidentified_lines_map.keys() di_list = [len(reidentified_lines_map[k_][0]) for k_ in keys] endi_list = np.add.accumulate(di_list) filter_mask = [m[endi-di:endi] for di, endi in zip(di_list, endi_list)] #from itertools import compress # _ = [list(compress(indices, mm)) for indices, mm \ # in zip(line_indices_list, filter_mask)] # line_indices_list_filtered = _ reidentified_lines_ = [reidentified_lines_map[k_] for k_ in keys] _ = [(v_[0][mm], v_[1][mm]) for v_, mm \ in zip(reidentified_lines_, filter_mask)] reidentified_lines_map_filtered = dict(zip(orders_w_solutions, _)) if 1: from matplotlib.figure import Figure fig1 = Figure(figsize=(12, 7)) check_fit(fig1, xl, yl, zl, p, orders_w_solutions, reidentified_lines_map) fig1.tight_layout() fig2 = Figure(figsize=(12, 7)) check_fit(fig2, xl[m], yl[m], zl[m], p, orders_w_solutions, reidentified_lines_map_filtered) fig2.tight_layout() if 1: from libs.qa_helper import figlist_to_pngs sky_figs = igr_path.get_section_filename_base("QA_PATH", "oh_fit2d", "oh_fit2d_"+sky_basename) figlist_to_pngs(sky_figs, [fig1, fig2]) if 1: from libs.products import ProductDB sky_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH", "sky.db", ) sky_db = ProductDB(sky_db_name) sky_db.update(band, sky_basename)
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on, config): from libs.products import PipelineStorage igr_path = IGRINSPath(config, utdate) igr_storage = PipelineStorage(igr_path) flat_off_filenames = igr_path.get_filenames(band, obsids_off) flat_on_filenames = igr_path.get_filenames(band, obsids_on) if 1: # process flat off flat_offs_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_off_filenames] flat_offs = [hdu.data for hdu in flat_offs_hdu_list] flat = FlatOff(flat_offs) flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100, sigma_clip2=5) igr_storage.store(flatoff_products, mastername=flat_off_filenames[0], masterhdu=flat_offs_hdu_list[0]) if 1: # flat on from libs.storage_descriptions import (FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC) desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC] flatoff_products = igr_storage.load(desc_list, mastername=flat_off_filenames[0]) flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames] flat_ons = [hdu.data for hdu in flat_on_hdu_list] from libs.master_calib import get_master_calib_abspath fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" % (refdate, band)) deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool) flat_on = FlatOn(flat_ons) flaton_products = flat_on.make_flaton_deadpixmap(flatoff_products, deadpix_mask_old=deadpix_mask_old) igr_storage.store(flaton_products, mastername=flat_on_filenames[0], masterhdu=flat_on_hdu_list[0]) if 1: # now trace the orders from libs.process_flat import trace_orders trace_products = trace_orders(flaton_products) hdu = pyfits.open(flat_on_filenames[0])[0] igr_storage.store(trace_products, mastername=flat_on_filenames[0], masterhdu=flat_on_hdu_list[0]) from libs.process_flat import trace_solutions trace_solution_products, trace_solution_products_plot = \ trace_solutions(trace_products) if 1: trace_solution_products.keys() from libs.storage_descriptions import FLATCENTROID_SOL_JSON_DESC myproduct = trace_solution_products[FLATCENTROID_SOL_JSON_DESC] bottomup_solutions = myproduct["bottom_up_solutions"] orders = range(len(bottomup_solutions)) from libs.apertures import Apertures ap = Apertures(orders, bottomup_solutions) from libs.storage_descriptions import FLAT_MASK_DESC flat_mask = igr_storage.load1(FLAT_MASK_DESC, flat_on_filenames[0]) order_map2 = ap.make_order_map(mask_top_bottom=True) bias_mask = flat_mask.data & (order_map2 > 0) from libs.products import PipelineImageBase, PipelineProducts pp = PipelineProducts("") from libs.storage_descriptions import BIAS_MASK_DESC pp.add(BIAS_MASK_DESC, PipelineImageBase([], bias_mask)) flaton_basename = flat_on_filenames[0] igr_storage.store(pp, mastername=flaton_basename, masterhdu=hdu) # plot qa figures. if 1: from libs.process_flat import check_trace_order from matplotlib.figure import Figure fig1 = Figure(figsize=[9, 4]) check_trace_order(trace_products, fig1) if 1: from libs.process_flat import plot_trace_solutions fig2, fig3 = plot_trace_solutions(flaton_products, trace_solution_products, trace_solution_products_plot, ) flatoff_basename = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0] flaton_basename = os.path.splitext(os.path.basename(flat_on_filenames[0]))[0] if 1: from libs.qa_helper import figlist_to_pngs aperture_figs = igr_path.get_section_filename_base("QA_PATH", "aperture_"+flaton_basename, "aperture_"+flaton_basename) figlist_to_pngs(aperture_figs, [fig1, fig2, fig3]) if 1: # now trace the orders #del trace_solution_products["bottom_up_solutions"] igr_storage.store(trace_solution_products, mastername=flat_on_filenames[0], masterhdu=flat_on_hdu_list[0]) # save db if 1: from libs.products import ProductDB flatoff_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH", "flat_off.db", ) flatoff_db = ProductDB(flatoff_db_name) #dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0] flatoff_db.update(band, flatoff_basename) flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH", "flat_on.db", ) flaton_db = ProductDB(flaton_db_name) flaton_db.update(band, flaton_basename)
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on, config): from libs.products import PipelineStorage igr_path = IGRINSPath(config, utdate) igr_storage = PipelineStorage(igr_path) flat_off_filenames = igr_path.get_filenames(band, obsids_off) flat_on_filenames = igr_path.get_filenames(band, obsids_on) if 1: # process flat off flat_offs_hdu_list = [ pyfits.open(fn_)[0] for fn_ in flat_off_filenames ] flat_offs = [hdu.data for hdu in flat_offs_hdu_list] flat = FlatOff(flat_offs) flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100, sigma_clip2=5) igr_storage.store(flatoff_products, mastername=flat_off_filenames[0], masterhdu=flat_offs_hdu_list[0]) if 1: # flat on from libs.storage_descriptions import (FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC) desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC] flatoff_products = igr_storage.load(desc_list, mastername=flat_off_filenames[0]) flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames] flat_ons = [hdu.data for hdu in flat_on_hdu_list] from libs.master_calib import get_master_calib_abspath fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" % (refdate, band)) deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool) flat_on = FlatOn(flat_ons) flaton_products = flat_on.make_flaton_deadpixmap( flatoff_products, deadpix_mask_old=deadpix_mask_old) igr_storage.store(flaton_products, mastername=flat_on_filenames[0], masterhdu=flat_on_hdu_list[0]) if 1: # now trace the orders from libs.process_flat import trace_orders trace_products = trace_orders(flaton_products) hdu = pyfits.open(flat_on_filenames[0])[0] igr_storage.store(trace_products, mastername=flat_on_filenames[0], masterhdu=flat_on_hdu_list[0]) from libs.process_flat import trace_solutions trace_solution_products = trace_solutions(trace_products) igr_storage.store(trace_solution_products, mastername=flat_on_filenames[0], masterhdu=flat_on_hdu_list[0]) # plot qa figures. if 1: from libs.process_flat import check_trace_order from matplotlib.figure import Figure fig1 = Figure() check_trace_order(trace_products, fig1) if 1: from libs.process_flat import plot_trace_solutions fig2, fig3 = plot_trace_solutions(flaton_products, trace_solution_products) flatoff_basename = os.path.splitext(os.path.basename( flat_off_filenames[0]))[0] flaton_basename = os.path.splitext(os.path.basename( flat_on_filenames[0]))[0] if 1: from libs.qa_helper import figlist_to_pngs aperture_figs = igr_path.get_section_filename_base( "QA_PATH", "aperture_" + flaton_basename, "aperture_" + flaton_basename) figlist_to_pngs(aperture_figs, [fig1, fig2, fig3]) # save db if 1: from libs.products import ProductDB flatoff_db_name = igr_path.get_section_filename_base( "PRIMARY_CALIB_PATH", "flat_off.db", ) flatoff_db = ProductDB(flatoff_db_name) #dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0] flatoff_db.update(band, flatoff_basename) flaton_db_name = igr_path.get_section_filename_base( "PRIMARY_CALIB_PATH", "flat_on.db", ) flaton_db = ProductDB(flaton_db_name) flaton_db.update(band, flaton_basename)
def process_wvlsol_band(utdate, refdate, band, obsids, config): from libs.products import ProductDB, PipelineStorage igr_path = IGRINSPath(config, utdate) igr_storage = PipelineStorage(igr_path) sky_filenames = igr_path.get_filenames(band, obsids) sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0] master_obsid = obsids[0] flaton_db_name = igr_path.get_section_filename_base( "PRIMARY_CALIB_PATH", "flat_on.db", ) flaton_db = ProductDB(flaton_db_name) #flaton_basename = flaton_db.query(band, master_obsid) thar_db_name = igr_path.get_section_filename_base( "PRIMARY_CALIB_PATH", "thar.db", ) thar_db = ProductDB(thar_db_name) #thar_basename = thar_db.query(band, master_obsid) # flaton_db = ProductDB(os.path.join(igr_path.secondary_calib_path, # "flat_on.db")) # thar_db = ProductDB(os.path.join(igr_path.secondary_calib_path, # "thar.db")) ap = load_aperture(igr_storage, band, master_obsid, flaton_db, thar_db) if 1: # from libs.process_thar import get_1d_median_specs raw_spec_product = get_1d_median_specs(sky_filenames, ap) # sky_master_fn_ = os.path.splitext(os.path.basename(sky_names[0]))[0] # sky_master_fn = igr_path.get_secondary_calib_filename(sky_master_fn_) import astropy.io.fits as pyfits masterhdu = pyfits.open(sky_filenames[0])[0] igr_storage.store(raw_spec_product, mastername=sky_filenames[0], masterhdu=masterhdu) # fn = sky_path.get_secondary_path("raw_spec") # raw_spec_product.save(fn, # masterhdu=masterhdu) from libs.master_calib import load_sky_ref_data # ref_date = "20140316" refdate = config.get_value("REFDATE", utdate) sky_ref_data = load_sky_ref_data(refdate, band) if 1: # initial wavelength solution # this need to be fixed # thar_db.query(sky_master_obsid) # json_name_ = "SDC%s_%s_0003.median_spectra.wvlsol" % (band, # igrins_log.date) from libs.storage_descriptions import THAR_WVLSOL_JSON_DESC thar_basename = thar_db.query(band, master_obsid) thar_wvl_sol = igr_storage.load([THAR_WVLSOL_JSON_DESC], thar_basename)[THAR_WVLSOL_JSON_DESC] #print thar_wvl_sol.keys() #["wvl_sol"] #json_name = thar_path.get_secondary_path("wvlsol_v0") #json_name = igr_path.get_secondary_calib_filename(json_name_) #thar_wvl_sol = PipelineProducts.load(json_name) if 1: # Now we fit with gaussian profile for matched positions. ohline_indices = sky_ref_data["ohline_indices"] ohlines_db = sky_ref_data["ohlines_db"] wvl_solutions = thar_wvl_sol["wvl_sol"] if 0: # it would be better to iteratively refit the solution fn = sky_path.get_secondary_path("wvlsol_v1") p = PipelineProducts.load(fn) wvl_solutionv = p["wvl_sol"] orders_w_solutions_ = thar_wvl_sol["orders"] from libs.storage_descriptions import ONED_SPEC_JSON_DESC orders_w_solutions = [ o for o in orders_w_solutions_ if o in raw_spec_product[ONED_SPEC_JSON_DESC]["orders"] ] _ = dict( zip(raw_spec_product[ONED_SPEC_JSON_DESC]["orders"], raw_spec_product[ONED_SPEC_JSON_DESC]["specs"])) s_list = [_[o] for o in orders_w_solutions] from libs.reidentify_ohlines import fit_ohlines ref_pixel_list, reidentified_lines = \ fit_ohlines(ohlines_db, ohline_indices, orders_w_solutions, wvl_solutions, s_list) # from scipy.interpolate import interp1d # from reidentify import reidentify_lines_all x = np.arange(2048) # line_indices_list = [ref_ohline_indices[str(o)] for o in igrins_orders[band]] ###### not fit identified lines from libs.ecfit import get_ordered_line_data, fit_2dspec, check_fit # d_x_wvl = {} # for order, z in echel.zdata.items(): # xy_T = affine_tr.transform(np.array([z.x, z.y]).T) # x_T = xy_T[:,0] # d_x_wvl[order]=(x_T, z.wvl) reidentified_lines_map = dict( zip(orders_w_solutions, reidentified_lines)) if band == "K": import libs.master_calib as master_calib fn = "hitran_bootstrap_K_%s.json" % refdate bootstrap_name = master_calib.get_master_calib_abspath(fn) import json bootstrap = json.load(open(bootstrap_name)) import libs.hitran as hitran r, ref_pixel_list = hitran.reidentify(wvl_solutions, s_list, bootstrap) # json_name = "hitran_reidentified_K_%s.json" % igrins_log.date # r = json.load(open(json_name)) for i, s in r.items(): ss = reidentified_lines_map[int(i)] ss0 = np.concatenate([ss[0], s["pixel"]]) ss1 = np.concatenate([ss[1], s["wavelength"]]) reidentified_lines_map[int(i)] = (ss0, ss1) xl, yl, zl = get_ordered_line_data(reidentified_lines_map) # xl : pixel # yl : order # zl : wvl * order x_domain = [0, 2047] y_domain = [orders_w_solutions[0] - 2, orders_w_solutions[-1] + 2] x_degree, y_degree = 4, 3 #x_degree, y_degree = 3, 2 p, m = fit_2dspec(xl, yl, zl, x_degree=x_degree, y_degree=y_degree, x_domain=x_domain, y_domain=y_domain) # derive wavelengths. xx = np.arange(2048) wvl_sol = [] for o in orders_w_solutions: oo = np.empty_like(xx) oo.fill(o) wvl = p(xx, oo) / o wvl_sol.append(list(wvl)) oh_sol_products = PipelineProducts( "Wavelength solution based on ohlines") #from libs.process_thar import ONED_SPEC_JSON from libs.products import PipelineDict from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC oh_sol_products.add( SKY_WVLSOL_JSON_DESC, PipelineDict(orders=orders_w_solutions, wvl_sol=wvl_sol)) if 1: if 1: # save as WAT fits header xx = np.arange(0, 2048) xx_plus1 = np.arange(1, 2048 + 1) from astropy.modeling import models, fitting # We convert 2d chebyshev solution to a seriese of 1d # chebyshev. For now, use naive (and inefficient) # approach of refitting the solution with 1d. Should be # reimplemented. p1d_list = [] for o in orders_w_solutions: oo = np.empty_like(xx) oo.fill(o) wvl = p(xx, oo) / o * 1.e4 # um to angstrom p_init1d = models.Chebyshev1D(domain=[1, 2048], degree=p.x_degree) fit_p1d = fitting.LinearLSQFitter() p1d = fit_p1d(p_init1d, xx_plus1, wvl) p1d_list.append(p1d) from libs.iraf_helper import get_wat_spec, default_header_str wat_list = get_wat_spec(orders_w_solutions, p1d_list) # cards = [pyfits.Card.fromstring(l.strip()) \ # for l in open("echell_2dspec.header")] cards = [pyfits.Card.fromstring(l.strip()) \ for l in default_header_str] wat = "wtype=multispec " + " ".join(wat_list) char_per_line = 68 num_line, remainder = divmod(len(wat), char_per_line) for i in range(num_line): k = "WAT2_%03d" % (i + 1, ) v = wat[char_per_line * i:char_per_line * (i + 1)] #print k, v c = pyfits.Card(k, v) cards.append(c) if remainder > 0: i = num_line k = "WAT2_%03d" % (i + 1, ) v = wat[char_per_line * i:] #print k, v c = pyfits.Card(k, v) cards.append(c) if 1: # save fits with empty header header = pyfits.Header(cards) hdu = pyfits.PrimaryHDU(header=header, data=np.array([]).reshape((0, 0))) from libs.storage_descriptions import SKY_WVLSOL_FITS_DESC from libs.products import PipelineImage oh_sol_products.add(SKY_WVLSOL_FITS_DESC, PipelineImage([], np.array(wvl_sol))) igr_storage.store(oh_sol_products, mastername=sky_filenames[0], masterhdu=hdu) #fn = sky_path.get_secondary_path("wvlsol_v1.fits") #hdu.writeto(fn, clobber=True) if 0: # plot all spectra for w, s in zip(wvl_sol, s_list): plot(w, s) if 1: # filter out the line indices not well fit by the surface keys = reidentified_lines_map.keys() di_list = [len(reidentified_lines_map[k_][0]) for k_ in keys] endi_list = np.add.accumulate(di_list) filter_mask = [ m[endi - di:endi] for di, endi in zip(di_list, endi_list) ] #from itertools import compress # _ = [list(compress(indices, mm)) for indices, mm \ # in zip(line_indices_list, filter_mask)] # line_indices_list_filtered = _ reidentified_lines_ = [reidentified_lines_map[k_] for k_ in keys] _ = [(v_[0][mm], v_[1][mm]) for v_, mm \ in zip(reidentified_lines_, filter_mask)] reidentified_lines_map_filtered = dict(zip(orders_w_solutions, _)) if 1: from matplotlib.figure import Figure fig1 = Figure(figsize=(12, 7)) check_fit(fig1, xl, yl, zl, p, orders_w_solutions, reidentified_lines_map) fig1.tight_layout() fig2 = Figure(figsize=(12, 7)) check_fit(fig2, xl[m], yl[m], zl[m], p, orders_w_solutions, reidentified_lines_map_filtered) fig2.tight_layout() if 1: from libs.qa_helper import figlist_to_pngs sky_figs = igr_path.get_section_filename_base( "QA_PATH", "oh_fit2d", "oh_fit2d_" + sky_basename) figlist_to_pngs(sky_figs, [fig1, fig2]) if 1: from libs.products import ProductDB sky_db_name = igr_path.get_section_filename_base( "PRIMARY_CALIB_PATH", "sky.db", ) sky_db = ProductDB(sky_db_name) sky_db.update(band, sky_basename)