def save_db(helper, band, obsids):
thar_filenames = helper.get_filenames(band, obsids)
thar_basename = os.path.splitext(os.path.basename(thar_filenames[0]))[0]
if 1:
from libs.products import ProductDB
thar_db_name = helper.igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"thar.db",
)
thar_db = ProductDB(thar_db_name)
# os.path.join(igr_path.secondary_calib_path,
# "thar.db"))
thar_db.update(band, thar_basename)
def process_thar_band(utdate, refdate, band, obsids, config):
from libs.products import ProductDB, PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
thar_filenames = igr_path.get_filenames(band, obsids)
thar_basename = os.path.splitext(os.path.basename(thar_filenames[0]))[0]
thar_master_obsid = obsids[0]
flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
flaton_basename = flaton_db.query(band, thar_master_obsid)
from libs.storage_descriptions import FLATCENTROID_SOL_JSON_DESC
desc_list = [FLATCENTROID_SOL_JSON_DESC]
products = igr_storage.load(desc_list,
mastername=flaton_basename)
aperture_solution_products = products[FLATCENTROID_SOL_JSON_DESC]
# igrins_orders = {}
# igrins_orders["H"] = range(99, 122)
# igrins_orders["K"] = range(72, 92)
if 1:
bottomup_solutions = aperture_solution_products["bottom_up_solutions"]
orders = range(len(bottomup_solutions))
ap = Apertures(orders, bottomup_solutions)
if 1:
from libs.process_thar import ThAr
thar = ThAr(thar_filenames)
thar_products = thar.process_thar(ap)
if 1: # match order
from libs.process_thar import match_order_thar
from libs.master_calib import load_thar_ref_data
#ref_date = "20140316"
thar_ref_data = load_thar_ref_data(refdate, band)
new_orders = match_order_thar(thar_products, thar_ref_data)
print thar_ref_data["orders"]
print new_orders
ap = Apertures(new_orders, bottomup_solutions)
from libs.storage_descriptions import ONED_SPEC_JSON_DESC
thar_products[ONED_SPEC_JSON_DESC]["orders"] = new_orders
if 1:
hdu = pyfits.open(thar_filenames[0])[0]
igr_storage.store(thar_products,
mastername=thar_filenames[0],
masterhdu=hdu)
if 1:
# measure shift of thar lines from reference spectra
# load spec
from libs.process_thar import reidentify_ThAr_lines
thar_reidentified_products = reidentify_ThAr_lines(thar_products,
thar_ref_data)
igr_storage.store(thar_reidentified_products,
mastername=thar_filenames[0],
masterhdu=hdu)
if 1:
from libs.process_thar import (load_echelogram,
align_echellogram_thar,
check_thar_transorm,
get_wavelength_solutions)
ref_date = thar_ref_data["ref_date"]
echel = load_echelogram(ref_date, band)
thar_aligned_echell_products = \
align_echellogram_thar(thar_reidentified_products,
echel, band, ap)
# We do not save this product yet.
# igr_storage.store(thar_aligned_echell_products,
# mastername=thar_filenames[0],
# masterhdu=hdu)
fig_list = check_thar_transorm(thar_products,
thar_aligned_echell_products)
from libs.qa_helper import figlist_to_pngs
thar_figs = igr_path.get_section_filename_base("QA_PATH",
"thar",
"thar_"+thar_basename)
figlist_to_pngs(thar_figs, fig_list)
thar_wvl_sol = get_wavelength_solutions(thar_aligned_echell_products,
echel)
igr_storage.store(thar_wvl_sol,
mastername=thar_filenames[0],
masterhdu=hdu)
if 1: # make amp and order falt
from libs.storage_descriptions import ONED_SPEC_JSON_DESC
orders = thar_products[ONED_SPEC_JSON_DESC]["orders"]
order_map = ap.make_order_map()
#slitpos_map = ap.make_slitpos_map()
# load flat on products
#flat_on_params_name = flaton_path.get_secondary_path("flat_on_params")
#flaton_products = PipelineProducts.load(flat_on_params_name)
from libs.storage_descriptions import (FLAT_NORMED_DESC,
FLAT_MASK_DESC)
flaton_products = igr_storage.load([FLAT_NORMED_DESC, FLAT_MASK_DESC],
flaton_basename)
from libs.process_flat import make_order_flat, check_order_flat
order_flat_products = make_order_flat(flaton_products,
orders, order_map)
#fn = thar_path.get_secondary_path("orderflat")
#order_flat_products.save(fn, masterhdu=hdu)
igr_storage.store(order_flat_products,
mastername=flaton_basename,
masterhdu=hdu)
if 1:
fig_list = check_order_flat(order_flat_products)
from libs.qa_helper import figlist_to_pngs
orderflat_figs = igr_path.get_section_filename_base("QA_PATH",
"orderflat",
"orderflat_"+thar_basename)
figlist_to_pngs(orderflat_figs, fig_list)
if 1:
from libs.products import ProductDB
thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"thar.db",
)
thar_db = ProductDB(thar_db_name)
# os.path.join(igr_path.secondary_calib_path,
# "thar.db"))
thar_db.update(band, thar_basename)
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on,
config):
from libs.products import PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
flat_off_filenames = igr_path.get_filenames(band, obsids_off)
flat_on_filenames = igr_path.get_filenames(band, obsids_on)
if 1: # process flat off
flat_offs_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_off_filenames]
flat_offs = [hdu.data for hdu in flat_offs_hdu_list]
flat = FlatOff(flat_offs)
flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100,
sigma_clip2=5)
igr_storage.store(flatoff_products,
mastername=flat_off_filenames[0],
masterhdu=flat_offs_hdu_list[0])
if 1: # flat on
from libs.storage_descriptions import (FLAT_OFF_DESC,
HOTPIX_MASK_DESC,
FLATOFF_JSON_DESC)
desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC]
flatoff_products = igr_storage.load(desc_list,
mastername=flat_off_filenames[0])
flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames]
flat_ons = [hdu.data for hdu in flat_on_hdu_list]
from libs.master_calib import get_master_calib_abspath
fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" % (refdate,
band))
deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool)
flat_on = FlatOn(flat_ons)
flaton_products = flat_on.make_flaton_deadpixmap(flatoff_products,
deadpix_mask_old=deadpix_mask_old)
igr_storage.store(flaton_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
if 1: # now trace the orders
from libs.process_flat import trace_orders
trace_products = trace_orders(flaton_products)
hdu = pyfits.open(flat_on_filenames[0])[0]
igr_storage.store(trace_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
from libs.process_flat import trace_solutions
trace_solution_products = trace_solutions(trace_products)
igr_storage.store(trace_solution_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
# plot qa figures.
if 1:
from libs.process_flat import check_trace_order
from matplotlib.figure import Figure
fig1 = Figure()
check_trace_order(trace_products, fig1)
if 1:
from libs.process_flat import plot_trace_solutions
fig2, fig3 = plot_trace_solutions(flaton_products,
trace_solution_products)
flatoff_basename = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
flaton_basename = os.path.splitext(os.path.basename(flat_on_filenames[0]))[0]
if 1:
from libs.qa_helper import figlist_to_pngs
aperture_figs = igr_path.get_section_filename_base("QA_PATH",
"aperture_"+flaton_basename,
"aperture_"+flaton_basename)
figlist_to_pngs(aperture_figs, [fig1, fig2, fig3])
# save db
if 1:
from libs.products import ProductDB
flatoff_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_off.db",
)
flatoff_db = ProductDB(flatoff_db_name)
#dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
flatoff_db.update(band, flatoff_basename)
flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
flaton_db.update(band, flaton_basename)
def process_wvlsol_band(utdate, refdate, band, obsids, config):
from libs.products import ProductDB, PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
sky_filenames = igr_path.get_filenames(band, obsids)
sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]
master_obsid = obsids[0]
flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
#flaton_basename = flaton_db.query(band, master_obsid)
thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"thar.db",
)
thar_db = ProductDB(thar_db_name)
#thar_basename = thar_db.query(band, master_obsid)
# flaton_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
# "flat_on.db"))
# thar_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
# "thar.db"))
ap = load_aperture(igr_storage, band, master_obsid,
flaton_db, thar_db)
if 1: #
from libs.process_thar import get_1d_median_specs
raw_spec_product = get_1d_median_specs(sky_filenames, ap)
# sky_master_fn_ = os.path.splitext(os.path.basename(sky_names[0]))[0]
# sky_master_fn = igr_path.get_secondary_calib_filename(sky_master_fn_)
import astropy.io.fits as pyfits
masterhdu = pyfits.open(sky_filenames[0])[0]
igr_storage.store(raw_spec_product,
mastername=sky_filenames[0],
masterhdu=masterhdu)
# fn = sky_path.get_secondary_path("raw_spec")
# raw_spec_product.save(fn,
# masterhdu=masterhdu)
from libs.master_calib import load_sky_ref_data
# ref_date = "20140316"
refdate = config.get_value("REFDATE", utdate)
sky_ref_data = load_sky_ref_data(refdate, band)
if 1: # initial wavelength solution
# this need to be fixed
# thar_db.query(sky_master_obsid)
# json_name_ = "SDC%s_%s_0003.median_spectra.wvlsol" % (band,
# igrins_log.date)
from libs.storage_descriptions import THAR_WVLSOL_JSON_DESC
thar_basename = thar_db.query(band, master_obsid)
thar_wvl_sol = igr_storage.load([THAR_WVLSOL_JSON_DESC],
thar_basename)[THAR_WVLSOL_JSON_DESC]
#print thar_wvl_sol.keys()
#["wvl_sol"]
#json_name = thar_path.get_secondary_path("wvlsol_v0")
#json_name = igr_path.get_secondary_calib_filename(json_name_)
#thar_wvl_sol = PipelineProducts.load(json_name)
if 1:
# Now we fit with gaussian profile for matched positions.
ohline_indices = sky_ref_data["ohline_indices"]
ohlines_db = sky_ref_data["ohlines_db"]
wvl_solutions = thar_wvl_sol["wvl_sol"]
if 0: # it would be better to iteratively refit the solution
fn = sky_path.get_secondary_path("wvlsol_v1")
p = PipelineProducts.load(fn)
wvl_solutionv = p["wvl_sol"]
orders_w_solutions_ = thar_wvl_sol["orders"]
from libs.storage_descriptions import ONED_SPEC_JSON_DESC
orders_w_solutions = [o for o in orders_w_solutions_ if o in raw_spec_product[ONED_SPEC_JSON_DESC]["orders"]]
_ = dict(zip(raw_spec_product[ONED_SPEC_JSON_DESC]["orders"],
raw_spec_product[ONED_SPEC_JSON_DESC]["specs"]))
s_list = [_[o]for o in orders_w_solutions]
from libs.reidentify_ohlines import fit_ohlines
ref_pixel_list, reidentified_lines = \
fit_ohlines(ohlines_db, ohline_indices,
orders_w_solutions,
wvl_solutions, s_list)
# from scipy.interpolate import interp1d
# from reidentify import reidentify_lines_all
x = np.arange(2048)
# line_indices_list = [ref_ohline_indices[str(o)] for o in igrins_orders[band]]
###### not fit identified lines
from libs.ecfit import get_ordered_line_data, fit_2dspec, check_fit
# d_x_wvl = {}
# for order, z in echel.zdata.items():
# xy_T = affine_tr.transform(np.array([z.x, z.y]).T)
# x_T = xy_T[:,0]
# d_x_wvl[order]=(x_T, z.wvl)
reidentified_lines_map = dict(zip(orders_w_solutions,
reidentified_lines))
if band == "K":
import libs.master_calib as master_calib
fn = "hitran_bootstrap_K_%s.json" % refdate
bootstrap_name = master_calib.get_master_calib_abspath(fn)
import json
bootstrap = json.load(open(bootstrap_name))
import libs.hitran as hitran
r, ref_pixel_list = hitran.reidentify(wvl_solutions, s_list, bootstrap)
# json_name = "hitran_reidentified_K_%s.json" % igrins_log.date
# r = json.load(open(json_name))
for i, s in r.items():
ss = reidentified_lines_map[int(i)]
ss0 = np.concatenate([ss[0], s["pixel"]])
ss1 = np.concatenate([ss[1], s["wavelength"]])
reidentified_lines_map[int(i)] = (ss0, ss1)
xl, yl, zl = get_ordered_line_data(reidentified_lines_map)
# xl : pixel
# yl : order
# zl : wvl * order
x_domain = [0, 2047]
y_domain = [orders_w_solutions[0]-2, orders_w_solutions[-1]+2]
x_degree, y_degree = 4, 3
#x_degree, y_degree = 3, 2
p, m = fit_2dspec(xl, yl, zl, x_degree=x_degree, y_degree=y_degree,
x_domain=x_domain, y_domain=y_domain)
# derive wavelengths.
xx = np.arange(2048)
wvl_sol = []
for o in orders_w_solutions:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o
wvl_sol.append(list(wvl))
oh_sol_products = PipelineProducts("Wavelength solution based on ohlines")
#from libs.process_thar import ONED_SPEC_JSON
from libs.products import PipelineDict
from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
oh_sol_products.add(SKY_WVLSOL_JSON_DESC,
PipelineDict(orders=orders_w_solutions,
wvl_sol=wvl_sol))
if 1:
if 1: # save as WAT fits header
xx = np.arange(0, 2048)
xx_plus1 = np.arange(1, 2048+1)
from astropy.modeling import models, fitting
# We convert 2d chebyshev solution to a seriese of 1d
# chebyshev. For now, use naive (and inefficient)
# approach of refitting the solution with 1d. Should be
# reimplemented.
p1d_list = []
for o in orders_w_solutions:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o * 1.e4 # um to angstrom
p_init1d = models.Chebyshev1D(domain=[1, 2048],
degree=p.x_degree)
fit_p1d = fitting.LinearLSQFitter()
p1d = fit_p1d(p_init1d, xx_plus1, wvl)
p1d_list.append(p1d)
from libs.iraf_helper import get_wat_spec, default_header_str
wat_list = get_wat_spec(orders_w_solutions, p1d_list)
# cards = [pyfits.Card.fromstring(l.strip()) \
# for l in open("echell_2dspec.header")]
cards = [pyfits.Card.fromstring(l.strip()) \
for l in default_header_str]
wat = "wtype=multispec " + " ".join(wat_list)
char_per_line = 68
num_line, remainder = divmod(len(wat), char_per_line)
for i in range(num_line):
k = "WAT2_%03d" % (i+1,)
v = wat[char_per_line*i:char_per_line*(i+1)]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if remainder > 0:
i = num_line
k = "WAT2_%03d" % (i+1,)
v = wat[char_per_line*i:]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if 1:
# save fits with empty header
header = pyfits.Header(cards)
hdu = pyfits.PrimaryHDU(header=header,
data=np.array([]).reshape((0,0)))
from libs.storage_descriptions import SKY_WVLSOL_FITS_DESC
from libs.products import PipelineImage
oh_sol_products.add(SKY_WVLSOL_FITS_DESC,
PipelineImage([],
np.array([]).reshape((0,0))))
igr_storage.store(oh_sol_products,
mastername=sky_filenames[0],
masterhdu=hdu)
#fn = sky_path.get_secondary_path("wvlsol_v1.fits")
#hdu.writeto(fn, clobber=True)
if 0:
# plot all spectra
for w, s in zip(wvl_sol, s_list):
plot(w, s)
if 1:
# filter out the line indices not well fit by the surface
keys = reidentified_lines_map.keys()
di_list = [len(reidentified_lines_map[k_][0]) for k_ in keys]
endi_list = np.add.accumulate(di_list)
filter_mask = [m[endi-di:endi] for di, endi in zip(di_list, endi_list)]
#from itertools import compress
# _ = [list(compress(indices, mm)) for indices, mm \
# in zip(line_indices_list, filter_mask)]
# line_indices_list_filtered = _
reidentified_lines_ = [reidentified_lines_map[k_] for k_ in keys]
_ = [(v_[0][mm], v_[1][mm]) for v_, mm \
in zip(reidentified_lines_, filter_mask)]
reidentified_lines_map_filtered = dict(zip(orders_w_solutions, _))
if 1:
from matplotlib.figure import Figure
fig1 = Figure(figsize=(12, 7))
check_fit(fig1, xl, yl, zl, p,
orders_w_solutions,
reidentified_lines_map)
fig1.tight_layout()
fig2 = Figure(figsize=(12, 7))
check_fit(fig2, xl[m], yl[m], zl[m], p,
orders_w_solutions,
reidentified_lines_map_filtered)
fig2.tight_layout()
if 1:
from libs.qa_helper import figlist_to_pngs
sky_figs = igr_path.get_section_filename_base("QA_PATH",
"oh_fit2d",
"oh_fit2d_"+sky_basename)
figlist_to_pngs(sky_figs, [fig1, fig2])
if 1:
from libs.products import ProductDB
sky_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"sky.db",
)
sky_db = ProductDB(sky_db_name)
sky_db.update(band, sky_basename)
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on,
config):
from libs.products import PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
flat_off_filenames = igr_path.get_filenames(band, obsids_off)
flat_on_filenames = igr_path.get_filenames(band, obsids_on)
if 1: # process flat off
flat_offs_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_off_filenames]
flat_offs = [hdu.data for hdu in flat_offs_hdu_list]
flat = FlatOff(flat_offs)
flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100,
sigma_clip2=5)
igr_storage.store(flatoff_products,
mastername=flat_off_filenames[0],
masterhdu=flat_offs_hdu_list[0])
if 1: # flat on
from libs.storage_descriptions import (FLAT_OFF_DESC,
HOTPIX_MASK_DESC,
FLATOFF_JSON_DESC)
desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC]
flatoff_products = igr_storage.load(desc_list,
mastername=flat_off_filenames[0])
flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames]
flat_ons = [hdu.data for hdu in flat_on_hdu_list]
from libs.master_calib import get_master_calib_abspath
fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" % (refdate,
band))
deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool)
flat_on = FlatOn(flat_ons)
flaton_products = flat_on.make_flaton_deadpixmap(flatoff_products,
deadpix_mask_old=deadpix_mask_old)
igr_storage.store(flaton_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
if 1: # now trace the orders
from libs.process_flat import trace_orders
trace_products = trace_orders(flaton_products)
hdu = pyfits.open(flat_on_filenames[0])[0]
igr_storage.store(trace_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
from libs.process_flat import trace_solutions
trace_solution_products, trace_solution_products_plot = \
trace_solutions(trace_products)
if 1:
trace_solution_products.keys()
from libs.storage_descriptions import FLATCENTROID_SOL_JSON_DESC
myproduct = trace_solution_products[FLATCENTROID_SOL_JSON_DESC]
bottomup_solutions = myproduct["bottom_up_solutions"]
orders = range(len(bottomup_solutions))
from libs.apertures import Apertures
ap = Apertures(orders, bottomup_solutions)
from libs.storage_descriptions import FLAT_MASK_DESC
flat_mask = igr_storage.load1(FLAT_MASK_DESC,
flat_on_filenames[0])
order_map2 = ap.make_order_map(mask_top_bottom=True)
bias_mask = flat_mask.data & (order_map2 > 0)
from libs.products import PipelineImageBase, PipelineProducts
pp = PipelineProducts("")
from libs.storage_descriptions import BIAS_MASK_DESC
pp.add(BIAS_MASK_DESC,
PipelineImageBase([], bias_mask))
flaton_basename = flat_on_filenames[0]
igr_storage.store(pp,
mastername=flaton_basename,
masterhdu=hdu)
# plot qa figures.
if 1:
from libs.process_flat import check_trace_order
from matplotlib.figure import Figure
fig1 = Figure(figsize=[9, 4])
check_trace_order(trace_products, fig1)
if 1:
from libs.process_flat import plot_trace_solutions
fig2, fig3 = plot_trace_solutions(flaton_products,
trace_solution_products,
trace_solution_products_plot,
)
flatoff_basename = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
flaton_basename = os.path.splitext(os.path.basename(flat_on_filenames[0]))[0]
if 1:
from libs.qa_helper import figlist_to_pngs
aperture_figs = igr_path.get_section_filename_base("QA_PATH",
"aperture_"+flaton_basename,
"aperture_"+flaton_basename)
figlist_to_pngs(aperture_figs, [fig1, fig2, fig3])
if 1: # now trace the orders
#del trace_solution_products["bottom_up_solutions"]
igr_storage.store(trace_solution_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
# save db
if 1:
from libs.products import ProductDB
flatoff_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_off.db",
)
flatoff_db = ProductDB(flatoff_db_name)
#dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
flatoff_db.update(band, flatoff_basename)
flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
flaton_db.update(band, flaton_basename)
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on, config):
from libs.products import PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
flat_off_filenames = igr_path.get_filenames(band, obsids_off)
flat_on_filenames = igr_path.get_filenames(band, obsids_on)
if 1: # process flat off
flat_offs_hdu_list = [
pyfits.open(fn_)[0] for fn_ in flat_off_filenames
]
flat_offs = [hdu.data for hdu in flat_offs_hdu_list]
flat = FlatOff(flat_offs)
flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100,
sigma_clip2=5)
igr_storage.store(flatoff_products,
mastername=flat_off_filenames[0],
masterhdu=flat_offs_hdu_list[0])
if 1: # flat on
from libs.storage_descriptions import (FLAT_OFF_DESC, HOTPIX_MASK_DESC,
FLATOFF_JSON_DESC)
desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC]
flatoff_products = igr_storage.load(desc_list,
mastername=flat_off_filenames[0])
flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames]
flat_ons = [hdu.data for hdu in flat_on_hdu_list]
from libs.master_calib import get_master_calib_abspath
fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" %
(refdate, band))
deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool)
flat_on = FlatOn(flat_ons)
flaton_products = flat_on.make_flaton_deadpixmap(
flatoff_products, deadpix_mask_old=deadpix_mask_old)
igr_storage.store(flaton_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
if 1: # now trace the orders
from libs.process_flat import trace_orders
trace_products = trace_orders(flaton_products)
hdu = pyfits.open(flat_on_filenames[0])[0]
igr_storage.store(trace_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
from libs.process_flat import trace_solutions
trace_solution_products = trace_solutions(trace_products)
igr_storage.store(trace_solution_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
# plot qa figures.
if 1:
from libs.process_flat import check_trace_order
from matplotlib.figure import Figure
fig1 = Figure()
check_trace_order(trace_products, fig1)
if 1:
from libs.process_flat import plot_trace_solutions
fig2, fig3 = plot_trace_solutions(flaton_products,
trace_solution_products)
flatoff_basename = os.path.splitext(os.path.basename(
flat_off_filenames[0]))[0]
flaton_basename = os.path.splitext(os.path.basename(
flat_on_filenames[0]))[0]
if 1:
from libs.qa_helper import figlist_to_pngs
aperture_figs = igr_path.get_section_filename_base(
"QA_PATH", "aperture_" + flaton_basename,
"aperture_" + flaton_basename)
figlist_to_pngs(aperture_figs, [fig1, fig2, fig3])
# save db
if 1:
from libs.products import ProductDB
flatoff_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"flat_off.db",
)
flatoff_db = ProductDB(flatoff_db_name)
#dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
flatoff_db.update(band, flatoff_basename)
flaton_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
flaton_db.update(band, flaton_basename)
def process_wvlsol_band(utdate, refdate, band, obsids, config):
from libs.products import ProductDB, PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
sky_filenames = igr_path.get_filenames(band, obsids)
sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]
master_obsid = obsids[0]
flaton_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
#flaton_basename = flaton_db.query(band, master_obsid)
thar_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"thar.db",
)
thar_db = ProductDB(thar_db_name)
#thar_basename = thar_db.query(band, master_obsid)
# flaton_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
# "flat_on.db"))
# thar_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
# "thar.db"))
ap = load_aperture(igr_storage, band, master_obsid, flaton_db, thar_db)
if 1: #
from libs.process_thar import get_1d_median_specs
raw_spec_product = get_1d_median_specs(sky_filenames, ap)
# sky_master_fn_ = os.path.splitext(os.path.basename(sky_names[0]))[0]
# sky_master_fn = igr_path.get_secondary_calib_filename(sky_master_fn_)
import astropy.io.fits as pyfits
masterhdu = pyfits.open(sky_filenames[0])[0]
igr_storage.store(raw_spec_product,
mastername=sky_filenames[0],
masterhdu=masterhdu)
# fn = sky_path.get_secondary_path("raw_spec")
# raw_spec_product.save(fn,
# masterhdu=masterhdu)
from libs.master_calib import load_sky_ref_data
# ref_date = "20140316"
refdate = config.get_value("REFDATE", utdate)
sky_ref_data = load_sky_ref_data(refdate, band)
if 1: # initial wavelength solution
# this need to be fixed
# thar_db.query(sky_master_obsid)
# json_name_ = "SDC%s_%s_0003.median_spectra.wvlsol" % (band,
# igrins_log.date)
from libs.storage_descriptions import THAR_WVLSOL_JSON_DESC
thar_basename = thar_db.query(band, master_obsid)
thar_wvl_sol = igr_storage.load([THAR_WVLSOL_JSON_DESC],
thar_basename)[THAR_WVLSOL_JSON_DESC]
#print thar_wvl_sol.keys()
#["wvl_sol"]
#json_name = thar_path.get_secondary_path("wvlsol_v0")
#json_name = igr_path.get_secondary_calib_filename(json_name_)
#thar_wvl_sol = PipelineProducts.load(json_name)
if 1:
# Now we fit with gaussian profile for matched positions.
ohline_indices = sky_ref_data["ohline_indices"]
ohlines_db = sky_ref_data["ohlines_db"]
wvl_solutions = thar_wvl_sol["wvl_sol"]
if 0: # it would be better to iteratively refit the solution
fn = sky_path.get_secondary_path("wvlsol_v1")
p = PipelineProducts.load(fn)
wvl_solutionv = p["wvl_sol"]
orders_w_solutions_ = thar_wvl_sol["orders"]
from libs.storage_descriptions import ONED_SPEC_JSON_DESC
orders_w_solutions = [
o for o in orders_w_solutions_
if o in raw_spec_product[ONED_SPEC_JSON_DESC]["orders"]
]
_ = dict(
zip(raw_spec_product[ONED_SPEC_JSON_DESC]["orders"],
raw_spec_product[ONED_SPEC_JSON_DESC]["specs"]))
s_list = [_[o] for o in orders_w_solutions]
from libs.reidentify_ohlines import fit_ohlines
ref_pixel_list, reidentified_lines = \
fit_ohlines(ohlines_db, ohline_indices,
orders_w_solutions,
wvl_solutions, s_list)
# from scipy.interpolate import interp1d
# from reidentify import reidentify_lines_all
x = np.arange(2048)
# line_indices_list = [ref_ohline_indices[str(o)] for o in igrins_orders[band]]
###### not fit identified lines
from libs.ecfit import get_ordered_line_data, fit_2dspec, check_fit
# d_x_wvl = {}
# for order, z in echel.zdata.items():
# xy_T = affine_tr.transform(np.array([z.x, z.y]).T)
# x_T = xy_T[:,0]
# d_x_wvl[order]=(x_T, z.wvl)
reidentified_lines_map = dict(
zip(orders_w_solutions, reidentified_lines))
if band == "K":
import libs.master_calib as master_calib
fn = "hitran_bootstrap_K_%s.json" % refdate
bootstrap_name = master_calib.get_master_calib_abspath(fn)
import json
bootstrap = json.load(open(bootstrap_name))
import libs.hitran as hitran
r, ref_pixel_list = hitran.reidentify(wvl_solutions, s_list,
bootstrap)
# json_name = "hitran_reidentified_K_%s.json" % igrins_log.date
# r = json.load(open(json_name))
for i, s in r.items():
ss = reidentified_lines_map[int(i)]
ss0 = np.concatenate([ss[0], s["pixel"]])
ss1 = np.concatenate([ss[1], s["wavelength"]])
reidentified_lines_map[int(i)] = (ss0, ss1)
xl, yl, zl = get_ordered_line_data(reidentified_lines_map)
# xl : pixel
# yl : order
# zl : wvl * order
x_domain = [0, 2047]
y_domain = [orders_w_solutions[0] - 2, orders_w_solutions[-1] + 2]
x_degree, y_degree = 4, 3
#x_degree, y_degree = 3, 2
p, m = fit_2dspec(xl,
yl,
zl,
x_degree=x_degree,
y_degree=y_degree,
x_domain=x_domain,
y_domain=y_domain)
# derive wavelengths.
xx = np.arange(2048)
wvl_sol = []
for o in orders_w_solutions:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o
wvl_sol.append(list(wvl))
oh_sol_products = PipelineProducts(
"Wavelength solution based on ohlines")
#from libs.process_thar import ONED_SPEC_JSON
from libs.products import PipelineDict
from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
oh_sol_products.add(
SKY_WVLSOL_JSON_DESC,
PipelineDict(orders=orders_w_solutions, wvl_sol=wvl_sol))
if 1:
if 1: # save as WAT fits header
xx = np.arange(0, 2048)
xx_plus1 = np.arange(1, 2048 + 1)
from astropy.modeling import models, fitting
# We convert 2d chebyshev solution to a seriese of 1d
# chebyshev. For now, use naive (and inefficient)
# approach of refitting the solution with 1d. Should be
# reimplemented.
p1d_list = []
for o in orders_w_solutions:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o * 1.e4 # um to angstrom
p_init1d = models.Chebyshev1D(domain=[1, 2048],
degree=p.x_degree)
fit_p1d = fitting.LinearLSQFitter()
p1d = fit_p1d(p_init1d, xx_plus1, wvl)
p1d_list.append(p1d)
from libs.iraf_helper import get_wat_spec, default_header_str
wat_list = get_wat_spec(orders_w_solutions, p1d_list)
# cards = [pyfits.Card.fromstring(l.strip()) \
# for l in open("echell_2dspec.header")]
cards = [pyfits.Card.fromstring(l.strip()) \
for l in default_header_str]
wat = "wtype=multispec " + " ".join(wat_list)
char_per_line = 68
num_line, remainder = divmod(len(wat), char_per_line)
for i in range(num_line):
k = "WAT2_%03d" % (i + 1, )
v = wat[char_per_line * i:char_per_line * (i + 1)]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if remainder > 0:
i = num_line
k = "WAT2_%03d" % (i + 1, )
v = wat[char_per_line * i:]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if 1:
# save fits with empty header
header = pyfits.Header(cards)
hdu = pyfits.PrimaryHDU(header=header,
data=np.array([]).reshape((0, 0)))
from libs.storage_descriptions import SKY_WVLSOL_FITS_DESC
from libs.products import PipelineImage
oh_sol_products.add(SKY_WVLSOL_FITS_DESC,
PipelineImage([], np.array(wvl_sol)))
igr_storage.store(oh_sol_products,
mastername=sky_filenames[0],
masterhdu=hdu)
#fn = sky_path.get_secondary_path("wvlsol_v1.fits")
#hdu.writeto(fn, clobber=True)
if 0:
# plot all spectra
for w, s in zip(wvl_sol, s_list):
plot(w, s)
if 1:
# filter out the line indices not well fit by the surface
keys = reidentified_lines_map.keys()
di_list = [len(reidentified_lines_map[k_][0]) for k_ in keys]
endi_list = np.add.accumulate(di_list)
filter_mask = [
m[endi - di:endi] for di, endi in zip(di_list, endi_list)
]
#from itertools import compress
# _ = [list(compress(indices, mm)) for indices, mm \
# in zip(line_indices_list, filter_mask)]
# line_indices_list_filtered = _
reidentified_lines_ = [reidentified_lines_map[k_] for k_ in keys]
_ = [(v_[0][mm], v_[1][mm]) for v_, mm \
in zip(reidentified_lines_, filter_mask)]
reidentified_lines_map_filtered = dict(zip(orders_w_solutions, _))
if 1:
from matplotlib.figure import Figure
fig1 = Figure(figsize=(12, 7))
check_fit(fig1, xl, yl, zl, p, orders_w_solutions,
reidentified_lines_map)
fig1.tight_layout()
fig2 = Figure(figsize=(12, 7))
check_fit(fig2, xl[m], yl[m], zl[m], p, orders_w_solutions,
reidentified_lines_map_filtered)
fig2.tight_layout()
if 1:
from libs.qa_helper import figlist_to_pngs
sky_figs = igr_path.get_section_filename_base(
"QA_PATH", "oh_fit2d", "oh_fit2d_" + sky_basename)
figlist_to_pngs(sky_figs, [fig1, fig2])
if 1:
from libs.products import ProductDB
sky_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"sky.db",
)
sky_db = ProductDB(sky_db_name)
sky_db.update(band, sky_basename)
