def get_bottom_up_solution(helper, band, thar_master_obsid):
"""
aperture that only requires flat product. No order information.
"""
flaton_db_name = helper.igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
flaton_basename = flaton_db.query(band, thar_master_obsid)
from libs.storage_descriptions import FLATCENTROID_SOL_JSON_DESC
desc_list = [FLATCENTROID_SOL_JSON_DESC]
products = helper.igr_storage.load(desc_list,
mastername=flaton_basename)
aperture_solution_products = products[FLATCENTROID_SOL_JSON_DESC]
# igrins_orders = {}
# igrins_orders["H"] = range(99, 122)
# igrins_orders["K"] = range(72, 92)
if 1:
bottomup_solutions = aperture_solution_products["bottom_up_solutions"]
return bottomup_solutions
def save_db(helper, band, obsids):
thar_filenames = helper.get_filenames(band, obsids)
thar_basename = os.path.splitext(os.path.basename(thar_filenames[0]))[0]
if 1:
from libs.products import ProductDB
thar_db_name = helper.igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"thar.db",
)
thar_db = ProductDB(thar_db_name)
# os.path.join(igr_path.secondary_calib_path,
# "thar.db"))
thar_db.update(band, thar_basename)
def process_thar_band(utdate, refdate, band, obsids, config):
from libs.products import ProductDB, PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
thar_filenames = igr_path.get_filenames(band, obsids)
thar_basename = os.path.splitext(os.path.basename(thar_filenames[0]))[0]
thar_master_obsid = obsids[0]
flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
flaton_basename = flaton_db.query(band, thar_master_obsid)
from libs.storage_descriptions import FLATCENTROID_SOL_JSON_DESC
desc_list = [FLATCENTROID_SOL_JSON_DESC]
products = igr_storage.load(desc_list,
mastername=flaton_basename)
aperture_solution_products = products[FLATCENTROID_SOL_JSON_DESC]
# igrins_orders = {}
# igrins_orders["H"] = range(99, 122)
# igrins_orders["K"] = range(72, 92)
if 1:
bottomup_solutions = aperture_solution_products["bottom_up_solutions"]
orders = range(len(bottomup_solutions))
ap = Apertures(orders, bottomup_solutions)
if 1:
from libs.process_thar import ThAr
thar = ThAr(thar_filenames)
thar_products = thar.process_thar(ap)
if 1: # match order
from libs.process_thar import match_order_thar
from libs.master_calib import load_thar_ref_data
#ref_date = "20140316"
thar_ref_data = load_thar_ref_data(refdate, band)
new_orders = match_order_thar(thar_products, thar_ref_data)
print thar_ref_data["orders"]
print new_orders
ap = Apertures(new_orders, bottomup_solutions)
from libs.storage_descriptions import ONED_SPEC_JSON_DESC
thar_products[ONED_SPEC_JSON_DESC]["orders"] = new_orders
if 1:
hdu = pyfits.open(thar_filenames[0])[0]
igr_storage.store(thar_products,
mastername=thar_filenames[0],
masterhdu=hdu)
if 1:
# measure shift of thar lines from reference spectra
# load spec
from libs.process_thar import reidentify_ThAr_lines
thar_reidentified_products = reidentify_ThAr_lines(thar_products,
thar_ref_data)
igr_storage.store(thar_reidentified_products,
mastername=thar_filenames[0],
masterhdu=hdu)
if 1:
from libs.process_thar import (load_echelogram,
align_echellogram_thar,
check_thar_transorm,
get_wavelength_solutions)
ref_date = thar_ref_data["ref_date"]
echel = load_echelogram(ref_date, band)
thar_aligned_echell_products = \
align_echellogram_thar(thar_reidentified_products,
echel, band, ap)
# We do not save this product yet.
# igr_storage.store(thar_aligned_echell_products,
# mastername=thar_filenames[0],
# masterhdu=hdu)
fig_list = check_thar_transorm(thar_products,
thar_aligned_echell_products)
from libs.qa_helper import figlist_to_pngs
thar_figs = igr_path.get_section_filename_base("QA_PATH",
"thar",
"thar_"+thar_basename)
figlist_to_pngs(thar_figs, fig_list)
thar_wvl_sol = get_wavelength_solutions(thar_aligned_echell_products,
echel)
igr_storage.store(thar_wvl_sol,
mastername=thar_filenames[0],
masterhdu=hdu)
if 1: # make amp and order falt
from libs.storage_descriptions import ONED_SPEC_JSON_DESC
orders = thar_products[ONED_SPEC_JSON_DESC]["orders"]
order_map = ap.make_order_map()
#slitpos_map = ap.make_slitpos_map()
# load flat on products
#flat_on_params_name = flaton_path.get_secondary_path("flat_on_params")
#flaton_products = PipelineProducts.load(flat_on_params_name)
from libs.storage_descriptions import (FLAT_NORMED_DESC,
FLAT_MASK_DESC)
flaton_products = igr_storage.load([FLAT_NORMED_DESC, FLAT_MASK_DESC],
flaton_basename)
from libs.process_flat import make_order_flat, check_order_flat
order_flat_products = make_order_flat(flaton_products,
orders, order_map)
#fn = thar_path.get_secondary_path("orderflat")
#order_flat_products.save(fn, masterhdu=hdu)
igr_storage.store(order_flat_products,
mastername=flaton_basename,
masterhdu=hdu)
if 1:
fig_list = check_order_flat(order_flat_products)
from libs.qa_helper import figlist_to_pngs
orderflat_figs = igr_path.get_section_filename_base("QA_PATH",
"orderflat",
"orderflat_"+thar_basename)
figlist_to_pngs(orderflat_figs, fig_list)
if 1:
from libs.products import ProductDB
thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"thar.db",
)
thar_db = ProductDB(thar_db_name)
# os.path.join(igr_path.secondary_calib_path,
# "thar.db"))
thar_db.update(band, thar_basename)
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on,
config):
from libs.products import PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
flat_off_filenames = igr_path.get_filenames(band, obsids_off)
flat_on_filenames = igr_path.get_filenames(band, obsids_on)
if 1: # process flat off
flat_offs_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_off_filenames]
flat_offs = [hdu.data for hdu in flat_offs_hdu_list]
flat = FlatOff(flat_offs)
flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100,
sigma_clip2=5)
igr_storage.store(flatoff_products,
mastername=flat_off_filenames[0],
masterhdu=flat_offs_hdu_list[0])
if 1: # flat on
from libs.storage_descriptions import (FLAT_OFF_DESC,
HOTPIX_MASK_DESC,
FLATOFF_JSON_DESC)
desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC]
flatoff_products = igr_storage.load(desc_list,
mastername=flat_off_filenames[0])
flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames]
flat_ons = [hdu.data for hdu in flat_on_hdu_list]
from libs.master_calib import get_master_calib_abspath
fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" % (refdate,
band))
deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool)
flat_on = FlatOn(flat_ons)
flaton_products = flat_on.make_flaton_deadpixmap(flatoff_products,
deadpix_mask_old=deadpix_mask_old)
igr_storage.store(flaton_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
if 1: # now trace the orders
from libs.process_flat import trace_orders
trace_products = trace_orders(flaton_products)
hdu = pyfits.open(flat_on_filenames[0])[0]
igr_storage.store(trace_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
from libs.process_flat import trace_solutions
trace_solution_products = trace_solutions(trace_products)
igr_storage.store(trace_solution_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
# plot qa figures.
if 1:
from libs.process_flat import check_trace_order
from matplotlib.figure import Figure
fig1 = Figure()
check_trace_order(trace_products, fig1)
if 1:
from libs.process_flat import plot_trace_solutions
fig2, fig3 = plot_trace_solutions(flaton_products,
trace_solution_products)
flatoff_basename = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
flaton_basename = os.path.splitext(os.path.basename(flat_on_filenames[0]))[0]
if 1:
from libs.qa_helper import figlist_to_pngs
aperture_figs = igr_path.get_section_filename_base("QA_PATH",
"aperture_"+flaton_basename,
"aperture_"+flaton_basename)
figlist_to_pngs(aperture_figs, [fig1, fig2, fig3])
# save db
if 1:
from libs.products import ProductDB
flatoff_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_off.db",
)
flatoff_db = ProductDB(flatoff_db_name)
#dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
flatoff_db.update(band, flatoff_basename)
flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
flaton_db.update(band, flaton_basename)
objname = abba[-1][0]
#obsids = abba[0]
#frametypes = abba[1]
obj_filenames = igrins_files.get_filenames(band, [obsid])
obj_path = ProductPath(igr_path, obj_filenames[0])
obj_master_obsid = obsid
fn = obj_path.get_secondary_path("spec.fits")
s_list = list(pyfits.open(fn)[0].data)
if 1:
sky_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
"sky.db"))
basename = sky_db.query(band, obj_master_obsid)
sky_path = ProductPath(igr_path, basename)
fn = sky_path.get_secondary_path("wvlsol_v1")
wvlsol_products = PipelineProducts.load(fn)
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
A0V_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
"A0V.db"))
basename = A0V_db.query(band, obj_master_obsid)
A0V_path = ProductPath(igr_path, basename)
def process_wvlsol_band(utdate, refdate, band, obsids, config):
from libs.products import ProductDB, PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
sky_filenames = igr_path.get_filenames(band, obsids)
sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]
master_obsid = obsids[0]
flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
#flaton_basename = flaton_db.query(band, master_obsid)
thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"thar.db",
)
thar_db = ProductDB(thar_db_name)
#thar_basename = thar_db.query(band, master_obsid)
# flaton_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
# "flat_on.db"))
# thar_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
# "thar.db"))
ap = load_aperture(igr_storage, band, master_obsid,
flaton_db, thar_db)
if 1: #
from libs.process_thar import get_1d_median_specs
raw_spec_product = get_1d_median_specs(sky_filenames, ap)
# sky_master_fn_ = os.path.splitext(os.path.basename(sky_names[0]))[0]
# sky_master_fn = igr_path.get_secondary_calib_filename(sky_master_fn_)
import astropy.io.fits as pyfits
masterhdu = pyfits.open(sky_filenames[0])[0]
igr_storage.store(raw_spec_product,
mastername=sky_filenames[0],
masterhdu=masterhdu)
# fn = sky_path.get_secondary_path("raw_spec")
# raw_spec_product.save(fn,
# masterhdu=masterhdu)
from libs.master_calib import load_sky_ref_data
# ref_date = "20140316"
refdate = config.get_value("REFDATE", utdate)
sky_ref_data = load_sky_ref_data(refdate, band)
if 1: # initial wavelength solution
# this need to be fixed
# thar_db.query(sky_master_obsid)
# json_name_ = "SDC%s_%s_0003.median_spectra.wvlsol" % (band,
# igrins_log.date)
from libs.storage_descriptions import THAR_WVLSOL_JSON_DESC
thar_basename = thar_db.query(band, master_obsid)
thar_wvl_sol = igr_storage.load([THAR_WVLSOL_JSON_DESC],
thar_basename)[THAR_WVLSOL_JSON_DESC]
#print thar_wvl_sol.keys()
#["wvl_sol"]
#json_name = thar_path.get_secondary_path("wvlsol_v0")
#json_name = igr_path.get_secondary_calib_filename(json_name_)
#thar_wvl_sol = PipelineProducts.load(json_name)
if 1:
# Now we fit with gaussian profile for matched positions.
ohline_indices = sky_ref_data["ohline_indices"]
ohlines_db = sky_ref_data["ohlines_db"]
wvl_solutions = thar_wvl_sol["wvl_sol"]
if 0: # it would be better to iteratively refit the solution
fn = sky_path.get_secondary_path("wvlsol_v1")
p = PipelineProducts.load(fn)
wvl_solutionv = p["wvl_sol"]
orders_w_solutions_ = thar_wvl_sol["orders"]
from libs.storage_descriptions import ONED_SPEC_JSON_DESC
orders_w_solutions = [o for o in orders_w_solutions_ if o in raw_spec_product[ONED_SPEC_JSON_DESC]["orders"]]
_ = dict(zip(raw_spec_product[ONED_SPEC_JSON_DESC]["orders"],
raw_spec_product[ONED_SPEC_JSON_DESC]["specs"]))
s_list = [_[o]for o in orders_w_solutions]
from libs.reidentify_ohlines import fit_ohlines
ref_pixel_list, reidentified_lines = \
fit_ohlines(ohlines_db, ohline_indices,
orders_w_solutions,
wvl_solutions, s_list)
# from scipy.interpolate import interp1d
# from reidentify import reidentify_lines_all
x = np.arange(2048)
# line_indices_list = [ref_ohline_indices[str(o)] for o in igrins_orders[band]]
###### not fit identified lines
from libs.ecfit import get_ordered_line_data, fit_2dspec, check_fit
# d_x_wvl = {}
# for order, z in echel.zdata.items():
# xy_T = affine_tr.transform(np.array([z.x, z.y]).T)
# x_T = xy_T[:,0]
# d_x_wvl[order]=(x_T, z.wvl)
reidentified_lines_map = dict(zip(orders_w_solutions,
reidentified_lines))
if band == "K":
import libs.master_calib as master_calib
fn = "hitran_bootstrap_K_%s.json" % refdate
bootstrap_name = master_calib.get_master_calib_abspath(fn)
import json
bootstrap = json.load(open(bootstrap_name))
import libs.hitran as hitran
r, ref_pixel_list = hitran.reidentify(wvl_solutions, s_list, bootstrap)
# json_name = "hitran_reidentified_K_%s.json" % igrins_log.date
# r = json.load(open(json_name))
for i, s in r.items():
ss = reidentified_lines_map[int(i)]
ss0 = np.concatenate([ss[0], s["pixel"]])
ss1 = np.concatenate([ss[1], s["wavelength"]])
reidentified_lines_map[int(i)] = (ss0, ss1)
xl, yl, zl = get_ordered_line_data(reidentified_lines_map)
# xl : pixel
# yl : order
# zl : wvl * order
x_domain = [0, 2047]
y_domain = [orders_w_solutions[0]-2, orders_w_solutions[-1]+2]
x_degree, y_degree = 4, 3
#x_degree, y_degree = 3, 2
p, m = fit_2dspec(xl, yl, zl, x_degree=x_degree, y_degree=y_degree,
x_domain=x_domain, y_domain=y_domain)
# derive wavelengths.
xx = np.arange(2048)
wvl_sol = []
for o in orders_w_solutions:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o
wvl_sol.append(list(wvl))
oh_sol_products = PipelineProducts("Wavelength solution based on ohlines")
#from libs.process_thar import ONED_SPEC_JSON
from libs.products import PipelineDict
from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
oh_sol_products.add(SKY_WVLSOL_JSON_DESC,
PipelineDict(orders=orders_w_solutions,
wvl_sol=wvl_sol))
if 1:
if 1: # save as WAT fits header
xx = np.arange(0, 2048)
xx_plus1 = np.arange(1, 2048+1)
from astropy.modeling import models, fitting
# We convert 2d chebyshev solution to a seriese of 1d
# chebyshev. For now, use naive (and inefficient)
# approach of refitting the solution with 1d. Should be
# reimplemented.
p1d_list = []
for o in orders_w_solutions:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o * 1.e4 # um to angstrom
p_init1d = models.Chebyshev1D(domain=[1, 2048],
degree=p.x_degree)
fit_p1d = fitting.LinearLSQFitter()
p1d = fit_p1d(p_init1d, xx_plus1, wvl)
p1d_list.append(p1d)
from libs.iraf_helper import get_wat_spec, default_header_str
wat_list = get_wat_spec(orders_w_solutions, p1d_list)
# cards = [pyfits.Card.fromstring(l.strip()) \
# for l in open("echell_2dspec.header")]
cards = [pyfits.Card.fromstring(l.strip()) \
for l in default_header_str]
wat = "wtype=multispec " + " ".join(wat_list)
char_per_line = 68
num_line, remainder = divmod(len(wat), char_per_line)
for i in range(num_line):
k = "WAT2_%03d" % (i+1,)
v = wat[char_per_line*i:char_per_line*(i+1)]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if remainder > 0:
i = num_line
k = "WAT2_%03d" % (i+1,)
v = wat[char_per_line*i:]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if 1:
# save fits with empty header
header = pyfits.Header(cards)
hdu = pyfits.PrimaryHDU(header=header,
data=np.array([]).reshape((0,0)))
from libs.storage_descriptions import SKY_WVLSOL_FITS_DESC
from libs.products import PipelineImage
oh_sol_products.add(SKY_WVLSOL_FITS_DESC,
PipelineImage([],
np.array([]).reshape((0,0))))
igr_storage.store(oh_sol_products,
mastername=sky_filenames[0],
masterhdu=hdu)
#fn = sky_path.get_secondary_path("wvlsol_v1.fits")
#hdu.writeto(fn, clobber=True)
if 0:
# plot all spectra
for w, s in zip(wvl_sol, s_list):
plot(w, s)
if 1:
# filter out the line indices not well fit by the surface
keys = reidentified_lines_map.keys()
di_list = [len(reidentified_lines_map[k_][0]) for k_ in keys]
endi_list = np.add.accumulate(di_list)
filter_mask = [m[endi-di:endi] for di, endi in zip(di_list, endi_list)]
#from itertools import compress
# _ = [list(compress(indices, mm)) for indices, mm \
# in zip(line_indices_list, filter_mask)]
# line_indices_list_filtered = _
reidentified_lines_ = [reidentified_lines_map[k_] for k_ in keys]
_ = [(v_[0][mm], v_[1][mm]) for v_, mm \
in zip(reidentified_lines_, filter_mask)]
reidentified_lines_map_filtered = dict(zip(orders_w_solutions, _))
if 1:
from matplotlib.figure import Figure
fig1 = Figure(figsize=(12, 7))
check_fit(fig1, xl, yl, zl, p,
orders_w_solutions,
reidentified_lines_map)
fig1.tight_layout()
fig2 = Figure(figsize=(12, 7))
check_fit(fig2, xl[m], yl[m], zl[m], p,
orders_w_solutions,
reidentified_lines_map_filtered)
fig2.tight_layout()
if 1:
from libs.qa_helper import figlist_to_pngs
sky_figs = igr_path.get_section_filename_base("QA_PATH",
"oh_fit2d",
"oh_fit2d_"+sky_basename)
figlist_to_pngs(sky_figs, [fig1, fig2])
if 1:
from libs.products import ProductDB
sky_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"sky.db",
)
sky_db = ProductDB(sky_db_name)
sky_db.update(band, sky_basename)
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on,
config):
from libs.products import PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
flat_off_filenames = igr_path.get_filenames(band, obsids_off)
flat_on_filenames = igr_path.get_filenames(band, obsids_on)
if 1: # process flat off
flat_offs_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_off_filenames]
flat_offs = [hdu.data for hdu in flat_offs_hdu_list]
flat = FlatOff(flat_offs)
flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100,
sigma_clip2=5)
igr_storage.store(flatoff_products,
mastername=flat_off_filenames[0],
masterhdu=flat_offs_hdu_list[0])
if 1: # flat on
from libs.storage_descriptions import (FLAT_OFF_DESC,
HOTPIX_MASK_DESC,
FLATOFF_JSON_DESC)
desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC]
flatoff_products = igr_storage.load(desc_list,
mastername=flat_off_filenames[0])
flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames]
flat_ons = [hdu.data for hdu in flat_on_hdu_list]
from libs.master_calib import get_master_calib_abspath
fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" % (refdate,
band))
deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool)
flat_on = FlatOn(flat_ons)
flaton_products = flat_on.make_flaton_deadpixmap(flatoff_products,
deadpix_mask_old=deadpix_mask_old)
igr_storage.store(flaton_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
if 1: # now trace the orders
from libs.process_flat import trace_orders
trace_products = trace_orders(flaton_products)
hdu = pyfits.open(flat_on_filenames[0])[0]
igr_storage.store(trace_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
from libs.process_flat import trace_solutions
trace_solution_products, trace_solution_products_plot = \
trace_solutions(trace_products)
if 1:
trace_solution_products.keys()
from libs.storage_descriptions import FLATCENTROID_SOL_JSON_DESC
myproduct = trace_solution_products[FLATCENTROID_SOL_JSON_DESC]
bottomup_solutions = myproduct["bottom_up_solutions"]
orders = range(len(bottomup_solutions))
from libs.apertures import Apertures
ap = Apertures(orders, bottomup_solutions)
from libs.storage_descriptions import FLAT_MASK_DESC
flat_mask = igr_storage.load1(FLAT_MASK_DESC,
flat_on_filenames[0])
order_map2 = ap.make_order_map(mask_top_bottom=True)
bias_mask = flat_mask.data & (order_map2 > 0)
from libs.products import PipelineImageBase, PipelineProducts
pp = PipelineProducts("")
from libs.storage_descriptions import BIAS_MASK_DESC
pp.add(BIAS_MASK_DESC,
PipelineImageBase([], bias_mask))
flaton_basename = flat_on_filenames[0]
igr_storage.store(pp,
mastername=flaton_basename,
masterhdu=hdu)
# plot qa figures.
if 1:
from libs.process_flat import check_trace_order
from matplotlib.figure import Figure
fig1 = Figure(figsize=[9, 4])
check_trace_order(trace_products, fig1)
if 1:
from libs.process_flat import plot_trace_solutions
fig2, fig3 = plot_trace_solutions(flaton_products,
trace_solution_products,
trace_solution_products_plot,
)
flatoff_basename = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
flaton_basename = os.path.splitext(os.path.basename(flat_on_filenames[0]))[0]
if 1:
from libs.qa_helper import figlist_to_pngs
aperture_figs = igr_path.get_section_filename_base("QA_PATH",
"aperture_"+flaton_basename,
"aperture_"+flaton_basename)
figlist_to_pngs(aperture_figs, [fig1, fig2, fig3])
if 1: # now trace the orders
#del trace_solution_products["bottom_up_solutions"]
igr_storage.store(trace_solution_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
# save db
if 1:
from libs.products import ProductDB
flatoff_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_off.db",
)
flatoff_db = ProductDB(flatoff_db_name)
#dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
flatoff_db.update(band, flatoff_basename)
flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
flaton_db.update(band, flaton_basename)
def process_distortion_sky_band(utdate, refdate, band, obsids, config):
from libs.products import ProductDB, PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
sky_filenames = igr_path.get_filenames(band, obsids)
sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]
master_obsid = obsids[0]
flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"thar.db",
)
thar_db = ProductDB(thar_db_name)
from libs.process_thar import COMBINED_IMAGE_DESC, ONED_SPEC_JSON
raw_spec_products = igr_storage.load([COMBINED_IMAGE_DESC, ONED_SPEC_JSON],
sky_basename)
# raw_spec_products = PipelineProducts.load(sky_path.get_secondary_path("raw_spec"))
SKY_WVLSOL_JSON_DESC = ("PRIMARY_CALIB_PATH", "SKY_", ".wvlsol_v1.json")
wvlsol_products = igr_storage.load([SKY_WVLSOL_JSON_DESC],
sky_basename)[SKY_WVLSOL_JSON_DESC]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
ap = load_aperture(igr_storage, band, master_obsid,
flaton_db,
raw_spec_products[ONED_SPEC_JSON]["orders"],
orders_w_solutions)
#orders_w_solutions = ap.orders
if 1: # load reference data
from libs.master_calib import load_sky_ref_data
ref_utdate = config.get_value("REFDATE", utdate)
sky_ref_data = load_sky_ref_data(ref_utdate, band)
ohlines_db = sky_ref_data["ohlines_db"]
ref_ohline_indices = sky_ref_data["ohline_indices"]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
if 0:
raw_spec_products = PipelineProducts.load(sky_path.get_secondary_path("raw_spec"))
basename = flaton_db.query(band, sky_master_obsid)
flaton_path = ProductPath(igr_path, basename)
aperture_solution_products = PipelineProducts.load(flaton_path.get_secondary_path("aperture_solutions"))
bottomup_solutions = aperture_solution_products["bottom_up_solutions"]
basename = thar_db.query(band, sky_master_obsid)
thar_path = ProductPath(igr_path, basename)
fn = thar_path.get_secondary_path("median_spectra")
# thar_products = PipelineProducts.load(fn)
# basename = sky_db.query(band, sky_master_obsid)
# sky_path = ProductPath(igr_path, basename)
fn = sky_path.get_secondary_path("wvlsol_v1")
wvlsol_products = PipelineProducts.load(fn)
if 1:
from libs.master_calib import load_sky_ref_data
ref_utdate = "20140316"
sky_ref_data = load_sky_ref_data(ref_utdate, band)
ohlines_db = sky_ref_data["ohlines_db"]
ref_ohline_indices = sky_ref_data["ohline_indices"]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
if 1: # make aperture
_o_s = dict(zip(raw_spec_products["orders"], bottomup_solutions))
ap = Apertures(orders_w_solutions,
[_o_s[o] for o in orders_w_solutions])
if 1:
n_slice_one_direction = 2
n_slice = n_slice_one_direction*2 + 1
i_center = n_slice_one_direction
slit_slice = np.linspace(0., 1., n_slice+1)
slice_center = (slit_slice[i_center], slit_slice[i_center+1])
slice_up = [(slit_slice[i_center+i], slit_slice[i_center+i+1]) \
for i in range(1, n_slice_one_direction+1)]
slice_down = [(slit_slice[i_center-i-1], slit_slice[i_center-i]) \
for i in range(n_slice_one_direction)]
d = raw_spec_products[COMBINED_IMAGE_DESC].data
s_center = ap.extract_spectra_v2(d, slice_center[0], slice_center[1])
s_up, s_down = [], []
for s1, s2 in slice_up:
s = ap.extract_spectra_v2(d, s1, s2)
s_up.append(s)
for s1, s2 in slice_down:
s = ap.extract_spectra_v2(d, s1, s2)
s_down.append(s)
if 1:
# now fit
#ohline_indices = [ref_ohline_indices[o] for o in orders_w_solutions]
if 0:
def test_order(oi):
ax=subplot(111)
ax.plot(wvl_solutions[oi], s_center[oi])
#ax.plot(wvl_solutions[oi], raw_spec_products["specs"][oi])
o = orders[oi]
line_indices = ref_ohline_indices[o]
for li in line_indices:
um = np.take(ohlines_db.um, li)
intensity = np.take(ohlines_db.intensity, li)
ax.vlines(um, ymin=0, ymax=-intensity)
from libs.reidentify_ohlines import fit_ohlines, fit_ohlines_pixel
def get_reidentified_lines_OH(orders_w_solutions,
wvl_solutions, s_center):
ref_pixel_list, reidentified_lines = \
fit_ohlines(ohlines_db, ref_ohline_indices,
orders_w_solutions,
wvl_solutions, s_center)
reidentified_lines_map = dict(zip(orders_w_solutions,
reidentified_lines))
return reidentified_lines_map, ref_pixel_list
if band == "H":
reidentified_lines_map, ref_pixel_list_oh = \
get_reidentified_lines_OH(orders_w_solutions,
wvl_solutions,
s_center)
def refit_centroid(s_center,
ref_pixel_list=ref_pixel_list_oh):
centroids = fit_ohlines_pixel(s_center,
ref_pixel_list)
return centroids
else: # band K
reidentified_lines_map, ref_pixel_list_oh = \
get_reidentified_lines_OH(orders_w_solutions,
wvl_solutions,
s_center)
import libs.master_calib as master_calib
fn = "hitran_bootstrap_K_%s.json" % ref_utdate
bootstrap_name = master_calib.get_master_calib_abspath(fn)
import json
bootstrap = json.load(open(bootstrap_name))
import libs.hitran as hitran
r, ref_pixel_dict_hitrans = hitran.reidentify(wvl_solutions,
s_center,
bootstrap)
# for i, s in r.items():
# ss = reidentified_lines_map[int(i)]
# ss0 = np.concatenate([ss[0], s["pixel"]])
# ss1 = np.concatenate([ss[1], s["wavelength"]])
# reidentified_lines_map[int(i)] = (ss0, ss1)
#reidentified_lines_map, ref_pixel_list
def refit_centroid(s_center,
ref_pixel_list=ref_pixel_list_oh,
ref_pixel_dict_hitrans=ref_pixel_dict_hitrans):
centroids_oh = fit_ohlines_pixel(s_center,
ref_pixel_list)
s_dict = dict(zip(orders_w_solutions, s_center))
centroids_dict_hitrans = hitran.fit_hitrans_pixel(s_dict,
ref_pixel_dict_hitrans)
centroids = []
for o, c_oh in zip(orders_w_solutions, centroids_oh):
if o in centroids_dict_hitrans:
c = np.concatenate([c_oh,
centroids_dict_hitrans[o]["pixel"]])
centroids.append(c)
else:
centroids.append(c_oh)
return centroids
# reidentified_lines_map = get_reidentified_lines(orders_w_solutions,
# wvl_solutions,
# s_center)
if 1:
# TODO: we should not need this, instead recycle from preivious step.
fitted_centroid_center = refit_centroid(s_center)
# fitted_centroid_center = fit_ohlines_pixel(s_center,
# ref_pixel_list)
d_shift_up = []
for s in s_up:
# TODO: ref_pixel_list_filtered need to be updated with recent fit.
fitted_centroid = refit_centroid(s)
# fitted_centroid = fit_ohlines_pixel(s,
# ref_pixel_list)
d_shift = [b-a for a, b in zip(fitted_centroid_center,
fitted_centroid)]
d_shift_up.append(d_shift)
d_shift_down = []
for s in s_down:
# TODO: ref_pixel_list_filtered need to be updated with recent fit.
fitted_centroid = refit_centroid(s)
# fitted_centroid = fit_ohlines_pixel(s,
# ref_pixel_list)
#fitted_centroid_center,
d_shift = [b-a for a, b in zip(fitted_centroid_center,
fitted_centroid)]
d_shift_down.append(d_shift)
if 1:
# now fit
orders = orders_w_solutions
x_domain = [0, 2048]
y_domain = [orders[0]-2, orders[-1]+2]
xl = np.concatenate(fitted_centroid_center)
yl_ = [o + np.zeros_like(x_) for o, x_ in zip(orders,
fitted_centroid_center)]
yl = np.concatenate(yl_)
from libs.ecfit import fit_2dspec, check_fit_simple
zl_list = [np.concatenate(d_) for d_ \
in d_shift_down[::-1] + d_shift_up]
pm_list = []
for zl in zl_list:
p, m = fit_2dspec(xl, yl, zl,
x_degree=1, y_degree=1,
x_domain=x_domain, y_domain=y_domain)
pm_list.append((p,m))
zz_std_list = []
for zl, (p, m) in zip(zl_list, pm_list):
z_m = p(xl[m], yl[m])
zz = z_m - zl[m]
zz_std_list.append(zz.std())
fig_list = []
from matplotlib.figure import Figure
for zl, (p, m) in zip(zl_list, pm_list):
fig = Figure()
check_fit_simple(fig, xl[m], yl[m], zl[m], p, orders)
fig_list.append(fig)
if 1:
xi = np.linspace(0, 2048, 128+1)
from astropy.modeling import fitting
from astropy.modeling.polynomial import Chebyshev2D
x_domain = [0, 2048]
y_domain = [0., 1.]
p2_list = []
for o in orders:
oi = np.zeros_like(xi) + o
shift_list = []
for p,m in pm_list[:n_slice_one_direction]:
shift_list.append(p(xi, oi))
shift_list.append(np.zeros_like(xi))
for p,m in pm_list[n_slice_one_direction:]:
shift_list.append(p(xi, oi))
p_init = Chebyshev2D(x_degree=1, y_degree=2,
x_domain=x_domain, y_domain=y_domain)
f = fitting.LinearLSQFitter()
yi = 0.5*(slit_slice[:-1] + slit_slice[1:])
xl, yl = np.meshgrid(xi, yi)
zl = np.array(shift_list)
p = f(p_init, xl, yl, zl)
p2_list.append(p)
if 1:
p2_dict = dict(zip(orders, p2_list))
order_map = ap.make_order_map()
slitpos_map = ap.make_slitpos_map()
slitoffset_map = np.empty_like(slitpos_map)
slitoffset_map.fill(np.nan)
for o in ap.orders:
xi = np.arange(0, 2048)
xl, yl = np.meshgrid(xi, xi)
msk = order_map == o
slitoffset_map[msk] = p2_dict[o](xl[msk], slitpos_map[msk])
import astropy.io.fits as pyfits
#fn = sky_path.get_secondary_path("slitoffset_map.fits")
#pyfits.PrimaryHDU(data=slitoffset_map).writeto(fn, clobber=True)
SLITOFFSET_FITS_DESC = ("PRIMARY_CALIB_PATH", "SKY_", ".slitoffset_map.fits")
from libs.products import PipelineImage, PipelineProducts
distortion_products = PipelineProducts("Distortion map")
distortion_products.add(SLITOFFSET_FITS_DESC,
PipelineImage([],
slitoffset_map))
igr_storage.store(distortion_products,
mastername=sky_filenames[0],
masterhdu=None)
from libs.qa_helper import figlist_to_pngs
sky_figs = igr_path.get_section_filename_base("QA_PATH",
"oh_distortion",
"oh_distortion_dir")
print fig_list
figlist_to_pngs(sky_figs, fig_list)
if 0:
# test
x = np.arange(2048, dtype="d")
oi = 10
o = orders[oi]
yi = 0.5*(slit_slice[:-1] + slit_slice[1:])
ax1 = subplot(211)
s1 = s_up[-1][oi]
s2 = s_down[-1][oi]
ax1.plot(x, s1)
ax1.plot(x, s2)
ax2 = subplot(212, sharex=ax1, sharey=ax1)
dx1 = p2_dict[o](x, yi[-1]+np.zeros_like(x))
ax2.plot(x-dx1, s1)
dx2 = p2_dict[o](x, yi[0]+np.zeros_like(x))
ax2.plot(x-dx2, s2)
def process_flat_band(utdate, refdate, band, obsids_off, obsids_on, config):
from libs.products import PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
flat_off_filenames = igr_path.get_filenames(band, obsids_off)
flat_on_filenames = igr_path.get_filenames(band, obsids_on)
if 1: # process flat off
flat_offs_hdu_list = [
pyfits.open(fn_)[0] for fn_ in flat_off_filenames
]
flat_offs = [hdu.data for hdu in flat_offs_hdu_list]
flat = FlatOff(flat_offs)
flatoff_products = flat.make_flatoff_hotpixmap(sigma_clip1=100,
sigma_clip2=5)
igr_storage.store(flatoff_products,
mastername=flat_off_filenames[0],
masterhdu=flat_offs_hdu_list[0])
if 1: # flat on
from libs.storage_descriptions import (FLAT_OFF_DESC, HOTPIX_MASK_DESC,
FLATOFF_JSON_DESC)
desc_list = [FLAT_OFF_DESC, HOTPIX_MASK_DESC, FLATOFF_JSON_DESC]
flatoff_products = igr_storage.load(desc_list,
mastername=flat_off_filenames[0])
flat_on_hdu_list = [pyfits.open(fn_)[0] for fn_ in flat_on_filenames]
flat_ons = [hdu.data for hdu in flat_on_hdu_list]
from libs.master_calib import get_master_calib_abspath
fn = get_master_calib_abspath("deadpix_mask_%s_%s.fits" %
(refdate, band))
deadpix_mask_old = pyfits.open(fn)[0].data.astype(bool)
flat_on = FlatOn(flat_ons)
flaton_products = flat_on.make_flaton_deadpixmap(
flatoff_products, deadpix_mask_old=deadpix_mask_old)
igr_storage.store(flaton_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
if 1: # now trace the orders
from libs.process_flat import trace_orders
trace_products = trace_orders(flaton_products)
hdu = pyfits.open(flat_on_filenames[0])[0]
igr_storage.store(trace_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
from libs.process_flat import trace_solutions
trace_solution_products = trace_solutions(trace_products)
igr_storage.store(trace_solution_products,
mastername=flat_on_filenames[0],
masterhdu=flat_on_hdu_list[0])
# plot qa figures.
if 1:
from libs.process_flat import check_trace_order
from matplotlib.figure import Figure
fig1 = Figure()
check_trace_order(trace_products, fig1)
if 1:
from libs.process_flat import plot_trace_solutions
fig2, fig3 = plot_trace_solutions(flaton_products,
trace_solution_products)
flatoff_basename = os.path.splitext(os.path.basename(
flat_off_filenames[0]))[0]
flaton_basename = os.path.splitext(os.path.basename(
flat_on_filenames[0]))[0]
if 1:
from libs.qa_helper import figlist_to_pngs
aperture_figs = igr_path.get_section_filename_base(
"QA_PATH", "aperture_" + flaton_basename,
"aperture_" + flaton_basename)
figlist_to_pngs(aperture_figs, [fig1, fig2, fig3])
# save db
if 1:
from libs.products import ProductDB
flatoff_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"flat_off.db",
)
flatoff_db = ProductDB(flatoff_db_name)
#dbname = os.path.splitext(os.path.basename(flat_off_filenames[0]))[0]
flatoff_db.update(band, flatoff_basename)
flaton_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
flaton_db.update(band, flaton_basename)
def process_wvlsol_band(utdate, refdate, band, obsids, config):
from libs.products import ProductDB, PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
sky_filenames = igr_path.get_filenames(band, obsids)
sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]
master_obsid = obsids[0]
flaton_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
#flaton_basename = flaton_db.query(band, master_obsid)
thar_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"thar.db",
)
thar_db = ProductDB(thar_db_name)
#thar_basename = thar_db.query(band, master_obsid)
# flaton_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
# "flat_on.db"))
# thar_db = ProductDB(os.path.join(igr_path.secondary_calib_path,
# "thar.db"))
ap = load_aperture(igr_storage, band, master_obsid, flaton_db, thar_db)
if 1: #
from libs.process_thar import get_1d_median_specs
raw_spec_product = get_1d_median_specs(sky_filenames, ap)
# sky_master_fn_ = os.path.splitext(os.path.basename(sky_names[0]))[0]
# sky_master_fn = igr_path.get_secondary_calib_filename(sky_master_fn_)
import astropy.io.fits as pyfits
masterhdu = pyfits.open(sky_filenames[0])[0]
igr_storage.store(raw_spec_product,
mastername=sky_filenames[0],
masterhdu=masterhdu)
# fn = sky_path.get_secondary_path("raw_spec")
# raw_spec_product.save(fn,
# masterhdu=masterhdu)
from libs.master_calib import load_sky_ref_data
# ref_date = "20140316"
refdate = config.get_value("REFDATE", utdate)
sky_ref_data = load_sky_ref_data(refdate, band)
if 1: # initial wavelength solution
# this need to be fixed
# thar_db.query(sky_master_obsid)
# json_name_ = "SDC%s_%s_0003.median_spectra.wvlsol" % (band,
# igrins_log.date)
from libs.storage_descriptions import THAR_WVLSOL_JSON_DESC
thar_basename = thar_db.query(band, master_obsid)
thar_wvl_sol = igr_storage.load([THAR_WVLSOL_JSON_DESC],
thar_basename)[THAR_WVLSOL_JSON_DESC]
#print thar_wvl_sol.keys()
#["wvl_sol"]
#json_name = thar_path.get_secondary_path("wvlsol_v0")
#json_name = igr_path.get_secondary_calib_filename(json_name_)
#thar_wvl_sol = PipelineProducts.load(json_name)
if 1:
# Now we fit with gaussian profile for matched positions.
ohline_indices = sky_ref_data["ohline_indices"]
ohlines_db = sky_ref_data["ohlines_db"]
wvl_solutions = thar_wvl_sol["wvl_sol"]
if 0: # it would be better to iteratively refit the solution
fn = sky_path.get_secondary_path("wvlsol_v1")
p = PipelineProducts.load(fn)
wvl_solutionv = p["wvl_sol"]
orders_w_solutions_ = thar_wvl_sol["orders"]
from libs.storage_descriptions import ONED_SPEC_JSON_DESC
orders_w_solutions = [
o for o in orders_w_solutions_
if o in raw_spec_product[ONED_SPEC_JSON_DESC]["orders"]
]
_ = dict(
zip(raw_spec_product[ONED_SPEC_JSON_DESC]["orders"],
raw_spec_product[ONED_SPEC_JSON_DESC]["specs"]))
s_list = [_[o] for o in orders_w_solutions]
from libs.reidentify_ohlines import fit_ohlines
ref_pixel_list, reidentified_lines = \
fit_ohlines(ohlines_db, ohline_indices,
orders_w_solutions,
wvl_solutions, s_list)
# from scipy.interpolate import interp1d
# from reidentify import reidentify_lines_all
x = np.arange(2048)
# line_indices_list = [ref_ohline_indices[str(o)] for o in igrins_orders[band]]
###### not fit identified lines
from libs.ecfit import get_ordered_line_data, fit_2dspec, check_fit
# d_x_wvl = {}
# for order, z in echel.zdata.items():
# xy_T = affine_tr.transform(np.array([z.x, z.y]).T)
# x_T = xy_T[:,0]
# d_x_wvl[order]=(x_T, z.wvl)
reidentified_lines_map = dict(
zip(orders_w_solutions, reidentified_lines))
if band == "K":
import libs.master_calib as master_calib
fn = "hitran_bootstrap_K_%s.json" % refdate
bootstrap_name = master_calib.get_master_calib_abspath(fn)
import json
bootstrap = json.load(open(bootstrap_name))
import libs.hitran as hitran
r, ref_pixel_list = hitran.reidentify(wvl_solutions, s_list,
bootstrap)
# json_name = "hitran_reidentified_K_%s.json" % igrins_log.date
# r = json.load(open(json_name))
for i, s in r.items():
ss = reidentified_lines_map[int(i)]
ss0 = np.concatenate([ss[0], s["pixel"]])
ss1 = np.concatenate([ss[1], s["wavelength"]])
reidentified_lines_map[int(i)] = (ss0, ss1)
xl, yl, zl = get_ordered_line_data(reidentified_lines_map)
# xl : pixel
# yl : order
# zl : wvl * order
x_domain = [0, 2047]
y_domain = [orders_w_solutions[0] - 2, orders_w_solutions[-1] + 2]
x_degree, y_degree = 4, 3
#x_degree, y_degree = 3, 2
p, m = fit_2dspec(xl,
yl,
zl,
x_degree=x_degree,
y_degree=y_degree,
x_domain=x_domain,
y_domain=y_domain)
# derive wavelengths.
xx = np.arange(2048)
wvl_sol = []
for o in orders_w_solutions:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o
wvl_sol.append(list(wvl))
oh_sol_products = PipelineProducts(
"Wavelength solution based on ohlines")
#from libs.process_thar import ONED_SPEC_JSON
from libs.products import PipelineDict
from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
oh_sol_products.add(
SKY_WVLSOL_JSON_DESC,
PipelineDict(orders=orders_w_solutions, wvl_sol=wvl_sol))
if 1:
if 1: # save as WAT fits header
xx = np.arange(0, 2048)
xx_plus1 = np.arange(1, 2048 + 1)
from astropy.modeling import models, fitting
# We convert 2d chebyshev solution to a seriese of 1d
# chebyshev. For now, use naive (and inefficient)
# approach of refitting the solution with 1d. Should be
# reimplemented.
p1d_list = []
for o in orders_w_solutions:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o * 1.e4 # um to angstrom
p_init1d = models.Chebyshev1D(domain=[1, 2048],
degree=p.x_degree)
fit_p1d = fitting.LinearLSQFitter()
p1d = fit_p1d(p_init1d, xx_plus1, wvl)
p1d_list.append(p1d)
from libs.iraf_helper import get_wat_spec, default_header_str
wat_list = get_wat_spec(orders_w_solutions, p1d_list)
# cards = [pyfits.Card.fromstring(l.strip()) \
# for l in open("echell_2dspec.header")]
cards = [pyfits.Card.fromstring(l.strip()) \
for l in default_header_str]
wat = "wtype=multispec " + " ".join(wat_list)
char_per_line = 68
num_line, remainder = divmod(len(wat), char_per_line)
for i in range(num_line):
k = "WAT2_%03d" % (i + 1, )
v = wat[char_per_line * i:char_per_line * (i + 1)]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if remainder > 0:
i = num_line
k = "WAT2_%03d" % (i + 1, )
v = wat[char_per_line * i:]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if 1:
# save fits with empty header
header = pyfits.Header(cards)
hdu = pyfits.PrimaryHDU(header=header,
data=np.array([]).reshape((0, 0)))
from libs.storage_descriptions import SKY_WVLSOL_FITS_DESC
from libs.products import PipelineImage
oh_sol_products.add(SKY_WVLSOL_FITS_DESC,
PipelineImage([], np.array(wvl_sol)))
igr_storage.store(oh_sol_products,
mastername=sky_filenames[0],
masterhdu=hdu)
#fn = sky_path.get_secondary_path("wvlsol_v1.fits")
#hdu.writeto(fn, clobber=True)
if 0:
# plot all spectra
for w, s in zip(wvl_sol, s_list):
plot(w, s)
if 1:
# filter out the line indices not well fit by the surface
keys = reidentified_lines_map.keys()
di_list = [len(reidentified_lines_map[k_][0]) for k_ in keys]
endi_list = np.add.accumulate(di_list)
filter_mask = [
m[endi - di:endi] for di, endi in zip(di_list, endi_list)
]
#from itertools import compress
# _ = [list(compress(indices, mm)) for indices, mm \
# in zip(line_indices_list, filter_mask)]
# line_indices_list_filtered = _
reidentified_lines_ = [reidentified_lines_map[k_] for k_ in keys]
_ = [(v_[0][mm], v_[1][mm]) for v_, mm \
in zip(reidentified_lines_, filter_mask)]
reidentified_lines_map_filtered = dict(zip(orders_w_solutions, _))
if 1:
from matplotlib.figure import Figure
fig1 = Figure(figsize=(12, 7))
check_fit(fig1, xl, yl, zl, p, orders_w_solutions,
reidentified_lines_map)
fig1.tight_layout()
fig2 = Figure(figsize=(12, 7))
check_fit(fig2, xl[m], yl[m], zl[m], p, orders_w_solutions,
reidentified_lines_map_filtered)
fig2.tight_layout()
if 1:
from libs.qa_helper import figlist_to_pngs
sky_figs = igr_path.get_section_filename_base(
"QA_PATH", "oh_fit2d", "oh_fit2d_" + sky_basename)
figlist_to_pngs(sky_figs, [fig1, fig2])
if 1:
from libs.products import ProductDB
sky_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"sky.db",
)
sky_db = ProductDB(sky_db_name)
sky_db.update(band, sky_basename)
def process_distortion_sky_band(utdate, refdate, band, obsids, config):
from libs.products import ProductDB, PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
sky_filenames = igr_path.get_filenames(band, obsids)
sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]
master_obsid = obsids[0]
flaton_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
# thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
# "thar.db",
# )
# thar_db = ProductDB(thar_db_name)
from libs.storage_descriptions import (COMBINED_IMAGE_DESC,
ONED_SPEC_JSON_DESC)
raw_spec_products = igr_storage.load(
[COMBINED_IMAGE_DESC, ONED_SPEC_JSON_DESC], sky_basename)
# raw_spec_products = PipelineProducts.load(sky_path.get_secondary_path("raw_spec"))
from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
wvlsol_products = igr_storage.load([SKY_WVLSOL_JSON_DESC],
sky_basename)[SKY_WVLSOL_JSON_DESC]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
ap = load_aperture2(igr_storage, band, master_obsid, flaton_db,
raw_spec_products[ONED_SPEC_JSON_DESC]["orders"],
orders_w_solutions)
#orders_w_solutions = ap.orders
if 1: # load reference data
from libs.master_calib import load_sky_ref_data
ref_utdate = config.get_value("REFDATE", utdate)
sky_ref_data = load_sky_ref_data(ref_utdate, band)
ohlines_db = sky_ref_data["ohlines_db"]
ref_ohline_indices = sky_ref_data["ohline_indices"]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
if 1:
n_slice_one_direction = 2
n_slice = n_slice_one_direction * 2 + 1
i_center = n_slice_one_direction
slit_slice = np.linspace(0., 1., n_slice + 1)
slice_center = (slit_slice[i_center], slit_slice[i_center + 1])
slice_up = [(slit_slice[i_center+i], slit_slice[i_center+i+1]) \
for i in range(1, n_slice_one_direction+1)]
slice_down = [(slit_slice[i_center-i-1], slit_slice[i_center-i]) \
for i in range(n_slice_one_direction)]
d = raw_spec_products[COMBINED_IMAGE_DESC].data
s_center = ap.extract_spectra_v2(d, slice_center[0], slice_center[1])
s_up, s_down = [], []
for s1, s2 in slice_up:
s = ap.extract_spectra_v2(d, s1, s2)
s_up.append(s)
for s1, s2 in slice_down:
s = ap.extract_spectra_v2(d, s1, s2)
s_down.append(s)
if 1:
# now fit
#ohline_indices = [ref_ohline_indices[o] for o in orders_w_solutions]
if 0:
def test_order(oi):
ax = subplot(111)
ax.plot(wvl_solutions[oi], s_center[oi])
#ax.plot(wvl_solutions[oi], raw_spec_products["specs"][oi])
o = orders[oi]
line_indices = ref_ohline_indices[o]
for li in line_indices:
um = np.take(ohlines_db.um, li)
intensity = np.take(ohlines_db.intensity, li)
ax.vlines(um, ymin=0, ymax=-intensity)
from libs.reidentify_ohlines import fit_ohlines, fit_ohlines_pixel
def get_reidentified_lines_OH(orders_w_solutions, wvl_solutions,
s_center):
ref_pixel_list, reidentified_lines = \
fit_ohlines(ohlines_db, ref_ohline_indices,
orders_w_solutions,
wvl_solutions, s_center)
reidentified_lines_map = dict(
zip(orders_w_solutions, reidentified_lines))
return reidentified_lines_map, ref_pixel_list
if band == "H":
reidentified_lines_map, ref_pixel_list_oh = \
get_reidentified_lines_OH(orders_w_solutions,
wvl_solutions,
s_center)
def refit_centroid(s_center, ref_pixel_list=ref_pixel_list_oh):
centroids = fit_ohlines_pixel(s_center, ref_pixel_list)
return centroids
else: # band K
reidentified_lines_map, ref_pixel_list_oh = \
get_reidentified_lines_OH(orders_w_solutions,
wvl_solutions,
s_center)
import libs.master_calib as master_calib
fn = "hitran_bootstrap_K_%s.json" % ref_utdate
bootstrap_name = master_calib.get_master_calib_abspath(fn)
import json
bootstrap = json.load(open(bootstrap_name))
import libs.hitran as hitran
r, ref_pixel_dict_hitrans = hitran.reidentify(
wvl_solutions, s_center, bootstrap)
# for i, s in r.items():
# ss = reidentified_lines_map[int(i)]
# ss0 = np.concatenate([ss[0], s["pixel"]])
# ss1 = np.concatenate([ss[1], s["wavelength"]])
# reidentified_lines_map[int(i)] = (ss0, ss1)
#reidentified_lines_map, ref_pixel_list
def refit_centroid(s_center,
ref_pixel_list=ref_pixel_list_oh,
ref_pixel_dict_hitrans=ref_pixel_dict_hitrans):
centroids_oh = fit_ohlines_pixel(s_center, ref_pixel_list)
s_dict = dict(zip(orders_w_solutions, s_center))
centroids_dict_hitrans = hitran.fit_hitrans_pixel(
s_dict, ref_pixel_dict_hitrans)
centroids = []
for o, c_oh in zip(orders_w_solutions, centroids_oh):
if o in centroids_dict_hitrans:
c = np.concatenate(
[c_oh, centroids_dict_hitrans[o]["pixel"]])
centroids.append(c)
else:
centroids.append(c_oh)
return centroids
# reidentified_lines_map = get_reidentified_lines(orders_w_solutions,
# wvl_solutions,
# s_center)
if 1:
# TODO: we should not need this, instead recycle from preivious step.
fitted_centroid_center = refit_centroid(s_center)
# fitted_centroid_center = fit_ohlines_pixel(s_center,
# ref_pixel_list)
d_shift_up = []
for s in s_up:
# TODO: ref_pixel_list_filtered need to be updated with recent fit.
fitted_centroid = refit_centroid(s)
# fitted_centroid = fit_ohlines_pixel(s,
# ref_pixel_list)
d_shift = [
b - a for a, b in zip(fitted_centroid_center, fitted_centroid)
]
d_shift_up.append(d_shift)
d_shift_down = []
for s in s_down:
# TODO: ref_pixel_list_filtered need to be updated with recent fit.
fitted_centroid = refit_centroid(s)
# fitted_centroid = fit_ohlines_pixel(s,
# ref_pixel_list)
#fitted_centroid_center,
d_shift = [
b - a for a, b in zip(fitted_centroid_center, fitted_centroid)
]
d_shift_down.append(d_shift)
if 1:
# now fit
orders = orders_w_solutions
x_domain = [0, 2048]
y_domain = [orders[0] - 2, orders[-1] + 2]
xl = np.concatenate(fitted_centroid_center)
yl_ = [
o + np.zeros_like(x_)
for o, x_ in zip(orders, fitted_centroid_center)
]
yl = np.concatenate(yl_)
from libs.ecfit import fit_2dspec, check_fit_simple
zl_list = [np.concatenate(d_) for d_ \
in d_shift_down[::-1] + d_shift_up]
pm_list = []
for zl in zl_list:
p, m = fit_2dspec(xl,
yl,
zl,
x_degree=1,
y_degree=1,
x_domain=x_domain,
y_domain=y_domain)
pm_list.append((p, m))
zz_std_list = []
for zl, (p, m) in zip(zl_list, pm_list):
z_m = p(xl[m], yl[m])
zz = z_m - zl[m]
zz_std_list.append(zz.std())
fig_list = []
from matplotlib.figure import Figure
for zl, (p, m) in zip(zl_list, pm_list):
fig = Figure()
check_fit_simple(fig, xl[m], yl[m], zl[m], p, orders)
fig_list.append(fig)
if 1:
xi = np.linspace(0, 2048, 128 + 1)
from astropy.modeling import fitting
from astropy.modeling.polynomial import Chebyshev2D
x_domain = [0, 2048]
y_domain = [0., 1.]
p2_list = []
for o in orders:
oi = np.zeros_like(xi) + o
shift_list = []
for p, m in pm_list[:n_slice_one_direction]:
shift_list.append(p(xi, oi))
shift_list.append(np.zeros_like(xi))
for p, m in pm_list[n_slice_one_direction:]:
shift_list.append(p(xi, oi))
p_init = Chebyshev2D(x_degree=1,
y_degree=2,
x_domain=x_domain,
y_domain=y_domain)
f = fitting.LinearLSQFitter()
yi = 0.5 * (slit_slice[:-1] + slit_slice[1:])
xl, yl = np.meshgrid(xi, yi)
zl = np.array(shift_list)
p = f(p_init, xl, yl, zl)
p2_list.append(p)
if 1:
p2_dict = dict(zip(orders, p2_list))
# save order_map, etc
order_map = ap.make_order_map()
slitpos_map = ap.make_slitpos_map()
order_map2 = ap.make_order_map(mask_top_bottom=True)
slitoffset_map = np.empty_like(slitpos_map)
slitoffset_map.fill(np.nan)
wavelength_map = np.empty_like(slitpos_map)
wavelength_map.fill(np.nan)
from scipy.interpolate import interp1d
for o, wvl in zip(ap.orders, wvl_solutions):
xi = np.arange(0, 2048)
xl, yl = np.meshgrid(xi, xi)
msk = order_map == o
xl_msk = xl[msk]
slitoffset_map_msk = p2_dict[o](xl_msk, slitpos_map[msk])
slitoffset_map[msk] = slitoffset_map_msk
wvl_interp1d = interp1d(xi, wvl, bounds_error=False)
wavelength_map[msk] = wvl_interp1d(xl_msk - slitoffset_map_msk)
from libs.storage_descriptions import (ORDERMAP_FITS_DESC,
SLITPOSMAP_FITS_DESC,
SLITOFFSET_FITS_DESC,
WAVELENGTHMAP_FITS_DESC,
ORDERMAP_MASKED_FITS_DESC)
from libs.products import PipelineImage, PipelineProducts
products = PipelineProducts("Distortion map")
for desc, im in [(ORDERMAP_FITS_DESC, order_map),
(SLITPOSMAP_FITS_DESC, slitpos_map),
(SLITOFFSET_FITS_DESC, slitoffset_map),
(WAVELENGTHMAP_FITS_DESC, wavelength_map),
(ORDERMAP_MASKED_FITS_DESC, order_map2)]:
products.add(desc, PipelineImage([], im))
igr_storage.store(products,
mastername=sky_filenames[0],
masterhdu=None)
from libs.qa_helper import figlist_to_pngs
sky_figs = igr_path.get_section_filename_base(
"QA_PATH", "oh_distortion", "oh_distortion_" + sky_basename)
print fig_list
figlist_to_pngs(sky_figs, fig_list)
if 0:
# test
x = np.arange(2048, dtype="d")
oi = 10
o = orders[oi]
yi = 0.5 * (slit_slice[:-1] + slit_slice[1:])
ax1 = subplot(211)
s1 = s_up[-1][oi]
s2 = s_down[-1][oi]
ax1.plot(x, s1)
ax1.plot(x, s2)
ax2 = subplot(212, sharex=ax1, sharey=ax1)
dx1 = p2_dict[o](x, yi[-1] + np.zeros_like(x))
ax2.plot(x - dx1, s1)
dx2 = p2_dict[o](x, yi[0] + np.zeros_like(x))
ax2.plot(x - dx2, s2)
def process_abba_band(recipe,
utdate,
refdate,
band,
obsids,
frametypes,
config,
do_interactive_figure=False,
threshold_a0v=0.1,
objname="",
multiply_model_a0v=False,
html_output=False):
from libs.products import ProductDB, PipelineStorage
if recipe == "A0V_AB":
FIX_TELLURIC = False
elif recipe == "STELLAR_AB":
FIX_TELLURIC = True
elif recipe == "EXTENDED_AB":
FIX_TELLURIC = True
elif recipe == "EXTENDED_ONOFF":
FIX_TELLURIC = True
else:
raise ValueError("Unsupported Recipe : %s" % recipe)
if 1:
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
obj_filenames = igr_path.get_filenames(band, obsids)
master_obsid = obsids[0]
tgt_basename = os.path.splitext(os.path.basename(obj_filenames[0]))[0]
db = {}
basenames = {}
db_types = ["flat_off", "flat_on", "thar", "sky"]
for db_type in db_types:
db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"%s.db" % db_type,
)
db[db_type] = ProductDB(db_name)
# db on output path
db_types = ["a0v"]
for db_type in db_types:
db_name = igr_path.get_section_filename_base(
"OUTDATA_PATH",
"%s.db" % db_type,
)
db[db_type] = ProductDB(db_name)
# to get basenames
db_types = ["flat_off", "flat_on", "thar", "sky"]
if FIX_TELLURIC:
db_types.append("a0v")
for db_type in db_types:
basenames[db_type] = db[db_type].query(band, master_obsid)
if 1: # make aperture
from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
sky_basename = db["sky"].query(band, master_obsid)
wvlsol_products = igr_storage.load([SKY_WVLSOL_JSON_DESC],
sky_basename)[SKY_WVLSOL_JSON_DESC]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = map(np.array, wvlsol_products["wvl_sol"])
# prepare i1i2_list
from libs.storage_descriptions import ORDER_FLAT_JSON_DESC
prod = igr_storage.load([ORDER_FLAT_JSON_DESC],
basenames["flat_on"])[ORDER_FLAT_JSON_DESC]
new_orders = prod["orders"]
i1i2_list_ = prod["i1i2_list"]
order_indices = []
for o in orders_w_solutions:
o_new_ind = np.searchsorted(new_orders, o)
order_indices.append(o_new_ind)
i1i2_list = get_fixed_i1i2_list(order_indices, i1i2_list_)
from libs.storage_descriptions import (SPEC_FITS_DESC, SN_FITS_DESC)
if 1: # load target spectrum
tgt_spec_ = igr_storage.load([SPEC_FITS_DESC],
tgt_basename)[SPEC_FITS_DESC]
tgt_spec = list(tgt_spec_.data)
tgt_sn_ = igr_storage.load([SN_FITS_DESC], tgt_basename)[SN_FITS_DESC]
tgt_sn = list(tgt_sn_.data)
fig_list = []
# telluric
if 1: #FIX_TELLURIC:
A0V_basename = db["a0v"].query(band, master_obsid)
from libs.storage_descriptions import SPEC_FITS_FLATTENED_DESC
telluric_cor_ = igr_storage.load(
[SPEC_FITS_FLATTENED_DESC], A0V_basename)[SPEC_FITS_FLATTENED_DESC]
#A0V_path = ProductPath(igr_path, A0V_basename)
#fn = A0V_path.get_secondary_path("spec_flattened.fits")
telluric_cor = list(telluric_cor_.data)
a0v_spec_ = igr_storage.load([SPEC_FITS_DESC],
A0V_basename)[SPEC_FITS_DESC]
a0v_spec = list(a0v_spec_.data)
if 1:
if do_interactive_figure:
from matplotlib.pyplot import figure as Figure
else:
from matplotlib.figure import Figure
fig1 = Figure(figsize=(12, 6))
fig_list.append(fig1)
ax1a = fig1.add_subplot(211)
ax1b = fig1.add_subplot(212, sharex=ax1a)
for wvl, s, sn in zip(wvl_solutions, tgt_spec, tgt_sn):
#s[s<0] = np.nan
#sn[sn<0] = np.nan
ax1a.plot(wvl, s)
ax1b.plot(wvl, sn)
ax1a.set_ylabel("Counts [DN]")
ax1b.set_ylabel("S/N per Res. Element")
ax1b.set_xlabel("Wavelength [um]")
ax1a.set_title(objname)
if FIX_TELLURIC:
fig2 = Figure(figsize=(12, 6))
fig_list.append(fig2)
ax2a = fig2.add_subplot(211)
ax2b = fig2.add_subplot(212, sharex=ax2a)
#from libs.stddev_filter import window_stdev
tgt_spec_cor = []
#for s, t in zip(s_list, telluric_cor):
for s, t, t2 in zip(tgt_spec, a0v_spec, telluric_cor):
st = s / t
#print np.percentile(t[np.isfinite(t)], 95), threshold_a0v
t0 = np.percentile(t[np.isfinite(t)], 95) * threshold_a0v
st[t < t0] = np.nan
st[t2 < threshold_a0v] = np.nan
tgt_spec_cor.append(st)
if multiply_model_a0v:
# multiply by A0V model
from libs.a0v_spec import A0VSpec
a0v_model = A0VSpec()
a0v_interp1d = a0v_model.get_flux_interp1d(1.3,
2.5,
flatten=True,
smooth_pixel=32)
for wvl, s in zip(wvl_solutions, tgt_spec_cor):
aa = a0v_interp1d(wvl)
s *= aa
for wvl, s, t in zip(wvl_solutions, tgt_spec_cor, telluric_cor):
ax2a.plot(wvl, t, "0.8", zorder=0.5)
ax2b.plot(wvl, s, zorder=0.5)
s_max_list = []
s_min_list = []
for s in tgt_spec_cor[3:-3]:
s_max_list.append(np.nanmax(s))
s_min_list.append(np.nanmin(s))
s_max = np.max(s_max_list)
s_min = np.min(s_min_list)
ds_pad = 0.05 * (s_max - s_min)
ax2a.set_ylabel("A0V flattened")
ax2a.set_ylim(-0.05, 1.1)
ax2b.set_ylabel("Target / A0V")
ax2b.set_xlabel("Wavelength [um]")
ax2b.set_ylim(s_min - ds_pad, s_max + ds_pad)
ax2a.set_title(objname)
# save figures
if fig_list:
for fig in fig_list:
fig.tight_layout()
figout = igr_path.get_section_filename_base("QA_PATH",
"spec_" + tgt_basename,
"spec_" + tgt_basename)
#figout = obj_path.get_secondary_path("spec", "spec_dir")
from libs.qa_helper import figlist_to_pngs
figlist_to_pngs(figout, fig_list)
# save html
if html_output:
dirname = config.get_value('HTML_PATH', utdate)
objroot = "%04d" % (master_obsid, )
html_save(utdate, dirname, objroot, band, orders_w_solutions,
wvl_solutions, tgt_spec, tgt_sn, i1i2_list)
if FIX_TELLURIC:
objroot = "%04dA0V" % (master_obsid, )
html_save(utdate,
dirname,
objroot,
band,
orders_w_solutions,
wvl_solutions,
telluric_cor,
tgt_spec_cor,
i1i2_list,
spec_js_name="jj_a0v.js")
if do_interactive_figure:
import matplotlib.pyplot as plt
plt.show()
def process(self, recipe, band, obsids, frametypes):
igr_path = self.igr_path
igr_storage = self.igr_storage
if recipe == "A0V_AB":
DO_STD = True
#FIX_TELLURIC=False
elif recipe == "STELLAR_AB":
DO_STD = False
#FIX_TELLURIC=True
elif recipe == "EXTENDED_AB":
DO_STD = False
#FIX_TELLURIC=True
elif recipe == "EXTENDED_ONOFF":
DO_STD = False
#FIX_TELLURIC=True
if 1:
obj_filenames = igr_path.get_filenames(band, obsids)
master_obsid = obsids[0]
tgt_basename = os.path.splitext(os.path.basename(
obj_filenames[0]))[0]
db = {}
basenames = {}
db_types = ["flat_off", "flat_on", "thar", "sky"]
for db_type in db_types:
db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"%s.db" % db_type,
)
db[db_type] = ProductDB(db_name)
# db on output path
db_types = ["a0v"]
for db_type in db_types:
db_name = igr_path.get_section_filename_base(
"OUTDATA_PATH",
"%s.db" % db_type,
)
db[db_type] = ProductDB(db_name)
# to get basenames
db_types = ["flat_off", "flat_on", "thar", "sky"]
# if FIX_TELLURIC:
# db_types.append("a0v")
for db_type in db_types:
basenames[db_type] = db[db_type].query(band, master_obsid)
if 1: # make aperture
from libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
sky_basename = db["sky"].query(band, master_obsid)
wvlsol_products = igr_storage.load(
[SKY_WVLSOL_JSON_DESC], sky_basename)[SKY_WVLSOL_JSON_DESC]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = map(np.array, wvlsol_products["wvl_sol"])
from libs.storage_descriptions import ONED_SPEC_JSON_DESC
raw_spec_products = igr_storage.load([ONED_SPEC_JSON_DESC],
sky_basename)
from recipe_wvlsol_sky import load_aperture2
ap = load_aperture2(
igr_storage, band, master_obsid, db["flat_on"],
raw_spec_products[ONED_SPEC_JSON_DESC]["orders"],
orders_w_solutions)
# This should be saved somewhere and loaded, instead of making it every time.
order_map = ap.make_order_map()
slitpos_map = ap.make_slitpos_map()
order_map2 = ap.make_order_map(mask_top_bottom=True)
if 1:
from libs.storage_descriptions import (HOTPIX_MASK_DESC,
DEADPIX_MASK_DESC,
ORDER_FLAT_IM_DESC,
ORDER_FLAT_JSON_DESC,
FLAT_MASK_DESC)
hotpix_mask = igr_storage.load(
[HOTPIX_MASK_DESC], basenames["flat_off"])[HOTPIX_MASK_DESC]
deadpix_mask = igr_storage.load(
[DEADPIX_MASK_DESC], basenames["flat_on"])[DEADPIX_MASK_DESC]
pix_mask = hotpix_mask.data | deadpix_mask.data
# aperture_solution_products = PipelineProducts.load(aperture_solutions_name)
orderflat_ = igr_storage.load(
[ORDER_FLAT_IM_DESC], basenames["flat_on"])[ORDER_FLAT_IM_DESC]
orderflat = orderflat_.data
orderflat[pix_mask] = np.nan
orderflat_json = igr_storage.load(
[ORDER_FLAT_JSON_DESC],
basenames["flat_on"])[ORDER_FLAT_JSON_DESC]
order_flat_meanspec = np.array(orderflat_json["mean_order_specs"])
# flat_normed = igr_storage.load([FLAT_NORMED_DESC],
# basenames["flat_on"])[FLAT_NORMED_DESC]
flat_mask = igr_storage.load([FLAT_MASK_DESC],
basenames["flat_on"])[FLAT_MASK_DESC]
bias_mask = flat_mask.data & (order_map2 > 0)
SLITOFFSET_FITS_DESC = ("PRIMARY_CALIB_PATH", "SKY_",
".slitoffset_map.fits")
prod_ = igr_storage.load([SLITOFFSET_FITS_DESC],
basenames["sky"])[SLITOFFSET_FITS_DESC]
#fn = sky_path.get_secondary_path("slitoffset_map.fits")
slitoffset_map = prod_.data
if 1:
abba_names = obj_filenames
def filter_abba_names(abba_names, frametypes, frametype):
return [
an for an, ft in zip(abba_names, frametypes)
if ft == frametype
]
a_name_list = filter_abba_names(abba_names, frametypes, "A")
b_name_list = filter_abba_names(abba_names, frametypes, "B")
if recipe in ["A0V_AB", "STELLAR_AB"]:
IF_POINT_SOURCE = True
elif recipe in ["EXTENDED_AB", "EXTENDED_ONOFF"]:
IF_POINT_SOURCE = False
else:
print "Unknown recipe : %s" % recipe
if 1:
#ab_names = ab_names_list[0]
# master_hdu = pyfits.open(a_name_list[0])[0]
a_list = [pyfits.open(name)[0].data \
for name in a_name_list]
b_list = [pyfits.open(name)[0].data \
for name in b_name_list]
# we may need to detrip
# first define extract profile (gaussian).
# dx = 100
if IF_POINT_SOURCE: # if point source
# for point sources, variance estimation becomes wrong
# if lenth of two is different,
assert len(a_list) == len(b_list)
# a_b != 1 for the cases when len(a) != len(b)
a_b = float(len(a_list)) / len(b_list)
a_data = np.sum(a_list, axis=0)
b_data = np.sum(b_list, axis=0)
data_minus = a_data - a_b * b_data
#data_minus0 = data_minus
from libs.destriper import destriper
if 1:
destrip_mask = ~np.isfinite(data_minus) | bias_mask
data_minus = destriper.get_destriped(data_minus,
destrip_mask,
hori=True,
pattern=64)
data_minus_flattened = data_minus / orderflat
data_minus_flattened[~flat_mask.data] = np.nan
#data_minus_flattened[order_flat_meanspec<0.1*order_flat_meanspec.max()] = np.nan
# for variance, we need a square of a_b
data_plus = (a_data + (a_b**2) * b_data)
import scipy.ndimage as ni
bias_mask2 = ni.binary_dilation(bias_mask)
from libs import instrument_parameters
gain = instrument_parameters.gain[band]
# random noise
variance0 = data_minus
variance_ = variance0.copy()
variance_[bias_mask2] = np.nan
variance_[pix_mask] = np.nan
mm = np.ma.array(variance0, mask=~np.isfinite(variance0))
ss = np.ma.median(mm, axis=0)
variance_ = variance_ - ss
# iterate over fixed number of times.
# need to be improved.
for i in range(5):
st = np.nanstd(variance_, axis=0)
variance_[np.abs(variance_) > 3 * st] = np.nan
#st = np.nanstd(variance_, axis=0)
variance = destriper.get_destriped(variance0,
~np.isfinite(variance_),
pattern=64)
variance_ = variance.copy()
variance_[bias_mask2] = np.nan
variance_[pix_mask] = np.nan
st = np.nanstd(variance_)
st = np.nanstd(variance_[np.abs(variance_) < 3 * st])
variance_[np.abs(variance_ - ss) > 3 * st] = np.nan
x_std = ni.median_filter(np.nanstd(variance_, axis=0), 11)
variance_map0 = np.zeros_like(variance) + x_std**2
variance_map = variance_map0 + np.abs(
data_plus) / gain # add poison noise in ADU
# we ignore effect of flattening
# now estimate lsf
# estimate lsf
ordermap_bpixed = order_map.copy()
ordermap_bpixed[pix_mask] = 0
ordermap_bpixed[~np.isfinite(orderflat)] = 0
#
if IF_POINT_SOURCE: # if point source
x1, x2 = 800, 1200
bins, lsf_list = ap.extract_lsf(ordermap_bpixed,
slitpos_map,
data_minus_flattened,
x1,
x2,
bins=None)
hh0 = np.sum(lsf_list, axis=0)
peak1, peak2 = max(hh0), -min(hh0)
lsf_x = 0.5 * (bins[1:] + bins[:-1])
lsf_y = hh0 / (peak1 + peak2)
from scipy.interpolate import UnivariateSpline
lsf_ = UnivariateSpline(lsf_x, lsf_y, k=3, s=0, bbox=[0, 1])
roots = list(lsf_.roots())
#assert(len(roots) == 1)
integ_list = []
from itertools import izip, cycle
for ss, int_r1, int_r2 in izip(cycle([1, -1]), [0] + roots,
roots + [1]):
#print ss, int_r1, int_r2
integ_list.append(lsf_.integral(int_r1, int_r2))
integ = np.abs(np.sum(integ_list))
def lsf(o, x, slitpos):
return lsf_(slitpos) / integ
# make weight map
profile_map = ap.make_profile_map(order_map, slitpos_map, lsf)
# try to select portion of the slit to extract
if self.frac_slit is not None:
frac1, frac2 = min(self.frac_slit), max(self.frac_slit)
slitpos_msk = (slitpos_map < frac1) | (slitpos_map > frac2)
profile_map[slitpos_msk] = np.nan
# extract spec
s_list, v_list = ap.extract_stellar(
ordermap_bpixed,
profile_map,
variance_map,
data_minus_flattened,
slitoffset_map=slitoffset_map)
# make synth_spec : profile * spectra
synth_map = ap.make_synth_map(order_map,
slitpos_map,
profile_map,
s_list,
slitoffset_map=slitoffset_map)
sig_map = (data_minus_flattened - synth_map)**2 / variance_map
## mark sig_map > 100 as cosmicay. The threshold need to be fixed.
# reextract with new variance map and CR is rejected
variance_map_r = variance_map0 + np.abs(synth_map) / gain
variance_map2 = np.max([variance_map, variance_map_r], axis=0)
variance_map2[np.abs(sig_map) > 100] = np.nan
# masking this out will affect the saved combined image.
data_minus_flattened[np.abs(sig_map) > 100] = np.nan
# extract spec
s_list, v_list = ap.extract_stellar(
ordermap_bpixed,
profile_map,
variance_map2,
data_minus_flattened,
slitoffset_map=slitoffset_map)
else: # if extended source
from scipy.interpolate import UnivariateSpline
if recipe in ["EXTENDED_AB", "EXTENDED_ABBA"]:
delta = 0.01
lsf_ = UnivariateSpline([0, 0.5 - delta, 0.5 + delta, 1],
[1., 1., -1., -1.],
k=1,
s=0,
bbox=[0, 1])
else:
lsf_ = UnivariateSpline([0, 1], [1., 1.],
k=1,
s=0,
bbox=[0, 1])
def lsf(o, x, slitpos):
return lsf_(slitpos)
profile_map = ap.make_profile_map(order_map, slitpos_map, lsf)
if self.frac_slit is not None:
frac1, frac2 = min(self.frac_slit), max(self.frac_slit)
slitpos_msk = (slitpos_map < frac1) | (slitpos_map > frac2)
profile_map[slitpos_msk] = np.nan
# we need to update the variance map by rejecting
# cosmic rays, but it is not clear how we do this
# for extended source.
variance_map2 = variance_map
s_list, v_list = ap.extract_stellar(
ordermap_bpixed,
profile_map,
variance_map2,
data_minus_flattened,
slitoffset_map=slitoffset_map)
if 1:
# calculate S/N per resolution
sn_list = []
for wvl, s, v in zip(wvl_solutions, s_list, v_list):
dw = np.gradient(wvl)
pixel_per_res_element = (wvl / 40000.) / dw
#print pixel_per_res_element[1024]
# len(pixel_per_res_element) = 2047. But we ignore it.
sn = (s / v**.5) * (pixel_per_res_element**.5)
sn_list.append(sn)
if 1: # save the product
from libs.storage_descriptions import (COMBINED_IMAGE_DESC,
VARIANCE_MAP_DESC)
from libs.products import PipelineImage
r = PipelineProducts("1d specs")
r.add(COMBINED_IMAGE_DESC, PipelineImage([], data_minus_flattened))
r.add(VARIANCE_MAP_DESC, PipelineImage([], variance_map2))
# r.add(VARIANCE_MAP_DESC, PipelineImage([],
# variance_map.data))
igr_storage.store(r, mastername=obj_filenames[0], masterhdu=None)
if 1: # save spectra, variance, sn
from libs.storage_descriptions import SKY_WVLSOL_FITS_DESC
fn = igr_storage.get_path(SKY_WVLSOL_FITS_DESC, basenames["sky"])
# fn = sky_path.get_secondary_path("wvlsol_v1.fits")
f = pyfits.open(fn)
d = np.array(s_list)
f[0].data = d.astype("f32")
from libs.storage_descriptions import (SPEC_FITS_DESC,
VARIANCE_FITS_DESC,
SN_FITS_DESC)
fout = igr_storage.get_path(SPEC_FITS_DESC, tgt_basename)
f.writeto(fout, clobber=True)
d = np.array(v_list)
f[0].data = d.astype("f32")
fout = igr_storage.get_path(VARIANCE_FITS_DESC, tgt_basename)
f.writeto(fout, clobber=True)
d = np.array(sn_list)
f[0].data = d.astype("f32")
fout = igr_storage.get_path(SN_FITS_DESC, tgt_basename)
f.writeto(fout, clobber=True)
if 1: #
from libs.storage_descriptions import ORDER_FLAT_JSON_DESC
prod = igr_storage.load([ORDER_FLAT_JSON_DESC],
basenames["flat_on"])[ORDER_FLAT_JSON_DESC]
new_orders = prod["orders"]
# fitted_response = orderflat_products["fitted_responses"]
i1i2_list_ = prod["i1i2_list"]
#order_indices = []
i1i2_list = []
for o in ap.orders:
o_new_ind = np.searchsorted(new_orders, o)
#order_indices.append(o_new_ind)
i1i2_list.append(i1i2_list_[o_new_ind])
if DO_STD:
from libs.a0v_spec import (A0VSpec, TelluricTransmission,
get_a0v, get_flattend)
a0v_spec = A0VSpec()
tel_trans = TelluricTransmission()
wvl_limits = []
for wvl_ in wvl_solutions:
wvl_limits.extend([wvl_[0], wvl_[-1]])
dwvl = abs(wvl_[0] - wvl_[-1]) * 0.2 # padding
wvl1 = min(wvl_limits) - dwvl
wvl2 = max(wvl_limits) + dwvl
a0v_wvl, a0v_tel_trans, a0v_tel_trans_masked = get_a0v(
a0v_spec, wvl1, wvl2, tel_trans)
a0v_flattened = get_flattend(a0v_spec,
a0v_wvl,
a0v_tel_trans_masked,
wvl_solutions,
s_list,
i1i2_list=i1i2_list)
d = np.array(a0v_flattened)
#d[~np.isfinite(d)] = 0.
f[0].data = d.astype("f32")
from libs.storage_descriptions import SPEC_FITS_FLATTENED_DESC
fout = igr_storage.get_path(SPEC_FITS_FLATTENED_DESC,
tgt_basename)
f.writeto(fout, clobber=True)
db["a0v"].update(band, tgt_basename)
abba = [rd[-1] for rd in recipe_dict["STELLAR_AB"] if obsid in rd[0]]
objname = abba[-1][0]
#obsids = abba[0]
#frametypes = abba[1]
obj_filenames = igrins_files.get_filenames(band, [obsid])
obj_path = ProductPath(igr_path, obj_filenames[0])
obj_master_obsid = obsid
fn = obj_path.get_secondary_path("spec.fits")
s_list = list(pyfits.open(fn)[0].data)
if 1:
sky_db = ProductDB(
os.path.join(igr_path.secondary_calib_path, "sky.db"))
basename = sky_db.query(band, obj_master_obsid)
sky_path = ProductPath(igr_path, basename)
fn = sky_path.get_secondary_path("wvlsol_v1")
wvlsol_products = PipelineProducts.load(fn)
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
A0V_db = ProductDB(
os.path.join(igr_path.secondary_calib_path, "A0V.db"))
basename = A0V_db.query(band, obj_master_obsid)
A0V_path = ProductPath(igr_path, basename)
# fn = A0V_path.get_secondary_path("spec_flattened.fits")
def save_figures(helper, band, obsids):
### THIS NEEDS TO BE REFACTORED!
caldb = helper.get_caldb()
master_obsid = obsids[0]
orders = caldb.load_resource_for((band, master_obsid), "orders")["orders"]
thar_filenames = helper.get_filenames(band, obsids)
thar_basename = os.path.splitext(os.path.basename(thar_filenames[0]))[0]
thar_master_obsid = obsids[0]
if 1: # make amp and order falt
ap = get_simple_aperture(helper, band, obsids,
orders=orders)
# from libs.storage_descriptions import ONED_SPEC_JSON_DESC
#orders = thar_products[ONED_SPEC_JSON_DESC]["orders"]
order_map = ap.make_order_map()
#slitpos_map = ap.make_slitpos_map()
# load flat on products
#flat_on_params_name = flaton_path.get_secondary_path("flat_on_params")
#flaton_products = PipelineProducts.load(flat_on_params_name)
from libs.storage_descriptions import (FLAT_NORMED_DESC,
FLAT_MASK_DESC)
flaton_db_name = helper.igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
flaton_basename = flaton_db.query(band, thar_master_obsid)
flaton_products = helper.igr_storage.load([FLAT_NORMED_DESC,
FLAT_MASK_DESC],
flaton_basename)
from libs.process_flat import make_order_flat, check_order_flat
order_flat_products = make_order_flat(flaton_products,
orders, order_map)
#fn = thar_path.get_secondary_path("orderflat")
#order_flat_products.save(fn, masterhdu=hdu)
hdu = pyfits.open(thar_filenames[0])[0]
helper.igr_storage.store(order_flat_products,
mastername=flaton_basename,
masterhdu=hdu)
flat_mask = helper.igr_storage.load1(FLAT_MASK_DESC,
flaton_basename)
order_map2 = ap.make_order_map(mask_top_bottom=True)
bias_mask = flat_mask.data & (order_map2 > 0)
pp = PipelineProducts("")
from libs.storage_descriptions import BIAS_MASK_DESC
pp.add(BIAS_MASK_DESC,
PipelineImageBase([], bias_mask))
helper.igr_storage.store(pp,
mastername=flaton_basename,
masterhdu=hdu)
if 1:
fig_list = check_order_flat(order_flat_products)
from libs.qa_helper import figlist_to_pngs
orderflat_figs = helper.igr_path.get_section_filename_base("QA_PATH",
"orderflat",
"orderflat_"+thar_basename)
figlist_to_pngs(orderflat_figs, fig_list)
