def store_processed_inputs(self, igr_storage,
mastername,
image_list,
shifted_image_list):
from igrins.libs.storage_descriptions import (COMBINED_IMAGE_DESC,
WVLCOR_IMAGE_DESC,
)
from igrins.libs.products import PipelineImages #Base
r = PipelineProducts("1d specs")
r.add(COMBINED_IMAGE_DESC, PipelineImages(image_list))
igr_storage.store(r,
mastername=mastername,
masterhdu=None,
basename_postfix=self.basename_postfix)
r = PipelineProducts("1d specs")
r.add(WVLCOR_IMAGE_DESC, PipelineImages(shifted_image_list))
igr_storage.store(r,
mastername=mastername,
masterhdu=None,
basename_postfix=self.basename_postfix)
def store_profile(self, mastername,
orders, slit_profile_list,
profile_x, profile_y):
## save profile
igr_storage = self.igr_storage
r = PipelineProducts("slit profile for point source")
from igrins.libs.storage_descriptions import SLIT_PROFILE_JSON_DESC
from igrins.libs.products import PipelineDict
slit_profile_dict = PipelineDict(orders=orders,
slit_profile_list=slit_profile_list,
profile_x=profile_x,
profile_y=profile_y)
r.add(SLIT_PROFILE_JSON_DESC, slit_profile_dict)
igr_storage.store(r,
mastername=mastername,
masterhdu=None,
basename_postfix=self.basename_postfix)
def store_a0v_results(self, igr_storage, extractor,
mastername,
a0v_flattened_data):
wvl_header, wvl_data, convert_data = \
self.get_wvl_header_data(igr_storage,
extractor)
from igrins.libs.load_fits import open_fits
f_obj = open_fits(extractor.obj_filenames[0])
f_obj[0].header.extend(wvl_header)
from igrins.libs.products import PipelineImage as Image
image_list = [Image([("EXTNAME", "SPEC_FLATTENED")],
convert_data(a0v_flattened_data[0][1]))]
for ext_name, data in a0v_flattened_data[1:]:
image_list.append(Image([("EXTNAME", ext_name.upper())],
convert_data(data)))
from igrins.libs.products import PipelineImages #Base
from igrins.libs.storage_descriptions import SPEC_FITS_FLATTENED_DESC
r = PipelineProducts("flattened 1d specs")
r.add(SPEC_FITS_FLATTENED_DESC, PipelineImages(image_list))
# mastername = extractor.obj_filenames[0]
igr_storage.store(r,
mastername=mastername,
masterhdu=f_obj[0],
basename_postfix=self.basename_postfix)
tgt_basename = extractor.pr.tgt_basename
extractor.db["a0v"].update(extractor.band, tgt_basename)
def store_wavelength_outputs(self, extractor, p2_list, ap):
orders = extractor.orders_w_solutions
wvl_solutions = extractor.wvl_solutions
p2_dict = dict(zip(orders, p2_list))
# save order_map, etc
order_map = ap.make_order_map()
slitpos_map = ap.make_slitpos_map()
order_map2 = ap.make_order_map(mask_top_bottom=True)
slitoffset_map = np.empty_like(slitpos_map)
slitoffset_map.fill(np.nan)
wavelength_map = np.empty_like(slitpos_map)
wavelength_map.fill(np.nan)
from scipy.interpolate import interp1d
for o, wvl in zip(ap.orders, wvl_solutions):
xi = np.arange(0, 2048)
xl, yl = np.meshgrid(xi, xi)
msk = order_map == o
xl_msk = xl[msk]
slitoffset_map_msk = p2_dict[o](xl_msk, slitpos_map[msk])
slitoffset_map[msk] = slitoffset_map_msk
wvl_interp1d = interp1d(xi, wvl, bounds_error=False)
wavelength_map[msk] = wvl_interp1d(xl_msk - slitoffset_map_msk)
from igrins.libs.storage_descriptions import (
ORDERMAP_FITS_DESC, SLITPOSMAP_FITS_DESC, SLITOFFSET_FITS_DESC,
WAVELENGTHMAP_FITS_DESC, ORDERMAP_MASKED_FITS_DESC)
from igrins.libs.products import PipelineImageBase, PipelineProducts
products = PipelineProducts("Distortion map")
for desc, im in [(ORDERMAP_FITS_DESC, order_map),
(SLITPOSMAP_FITS_DESC, slitpos_map),
(SLITOFFSET_FITS_DESC, slitoffset_map),
(WAVELENGTHMAP_FITS_DESC, wavelength_map),
(ORDERMAP_MASKED_FITS_DESC, order_map2)]:
products.add(desc, PipelineImageBase([], im))
igr_storage = extractor.igr_storage
igr_storage.store(products,
mastername=extractor.obj_filenames[0],
masterhdu=None)
if 0:
# test
x = np.arange(2048, dtype="d")
oi = 10
o = orders[oi]
yi = 0.5 * (slit_slice[:-1] + slit_slice[1:])
ax1 = subplot(211)
s1 = s_up[-1][oi]
s2 = s_down[-1][oi]
ax1.plot(x, s1)
ax1.plot(x, s2)
ax2 = subplot(212, sharex=ax1, sharey=ax1)
dx1 = p2_dict[o](x, yi[-1] + np.zeros_like(x))
ax2.plot(x - dx1, s1)
dx2 = p2_dict[o](x, yi[0] + np.zeros_like(x))
ax2.plot(x - dx2, s2)
def save_wavelength_sol(self, extractor, orders_w_solutions, wvl_sol, p):
oh_sol_products = PipelineProducts(
"Wavelength solution based on ohlines")
#from igrins.libs.process_thar import ONED_SPEC_JSON
from igrins.libs.products import PipelineDict
from igrins.libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
oh_sol_products.add(
SKY_WVLSOL_JSON_DESC,
PipelineDict(orders=orders_w_solutions, wvl_sol=wvl_sol))
if 1: # save as WAT fits header
xx = np.arange(0, 2048)
xx_plus1 = np.arange(1, 2048 + 1)
from astropy.modeling import models, fitting
# We convert 2d chebyshev solution to a seriese of 1d
# chebyshev. For now, use naive (and inefficient)
# approach of refitting the solution with 1d. Should be
# reimplemented.
p1d_list = []
for o in orders_w_solutions:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o * 1.e4 # um to angstrom
p_init1d = models.Chebyshev1D(domain=[1, 2048],
degree=p.x_degree)
fit_p1d = fitting.LinearLSQFitter()
p1d = fit_p1d(p_init1d, xx_plus1, wvl)
p1d_list.append(p1d)
from igrins.libs.iraf_helper import get_wat_spec, default_header_str
wat_list = get_wat_spec(orders_w_solutions, p1d_list)
# cards = [pyfits.Card.fromstring(l.strip()) \
# for l in open("echell_2dspec.header")]
import astropy.io.fits as pyfits
cards = [pyfits.Card.fromstring(l.strip()) \
for l in default_header_str]
wat = "wtype=multispec " + " ".join(wat_list)
char_per_line = 68
num_line, remainder = divmod(len(wat), char_per_line)
for i in range(num_line):
k = "WAT2_%03d" % (i + 1, )
v = wat[char_per_line * i:char_per_line * (i + 1)]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if remainder > 0:
i = num_line
k = "WAT2_%03d" % (i + 1, )
v = wat[char_per_line * i:]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if 1:
# save fits with empty header
header = pyfits.Header(cards)
hdu = pyfits.PrimaryHDU(header=header,
data=np.array([]).reshape((0, 0)))
from igrins.libs.storage_descriptions import SKY_WVLSOL_FITS_DESC
from igrins.libs.products import PipelineImageBase
oh_sol_products.add(SKY_WVLSOL_FITS_DESC,
PipelineImageBase([], np.array(wvl_sol)))
igr_storage = extractor.igr_storage
sky_filenames = extractor.obj_filenames
igr_storage.store(oh_sol_products,
mastername=sky_filenames[0],
masterhdu=hdu)
#fn = sky_path.get_secondary_path("wvlsol_v1.fits")
#hdu.writeto(fn, clobber=True)
if 0:
# plot all spectra
for w, s in zip(wvl_sol, s_list):
plot(w, s)
def get_sky_spectra(self, extractor, ap, band, master_obsid):
from igrins.libs.process_thar import get_1d_median_specs
sky_filenames = extractor.obj_filenames
raw_spec_product = get_1d_median_specs(sky_filenames, ap)
# sky_master_fn_ = os.path.splitext(os.path.basename(sky_names[0]))[0]
# sky_master_fn = igr_path.get_secondary_calib_filename(sky_master_fn_)
# import astropy.io.fits as pyfits
# masterhdu = pyfits.open(sky_filenames[0])[0]
from igrins.libs.load_fits import load_fits_data
masterhdu = load_fits_data(sky_filenames[0])
igr_storage = extractor.igr_storage
igr_storage.store(raw_spec_product,
mastername=sky_filenames[0],
masterhdu=masterhdu)
# fn = sky_path.get_secondary_path("raw_spec")
# raw_spec_product.save(fn,
# masterhdu=masterhdu)
# initial wavelength solution
# this need to be fixed
# thar_db.query(sky_master_obsid)
# json_name_ = "SDC%s_%s_0003.median_spectra.wvlsol" % (band,
# igrins_log.date)
#from igrins.libs.storage_descriptions import THAR_WVLSOL_JSON_DESC
from igrins.libs.storage_descriptions import WVLSOL_V0_JSON_DESC
thar_basename = extractor.db["thar"].query(band, master_obsid)
thar_wvl_sol = igr_storage.load([WVLSOL_V0_JSON_DESC],
thar_basename)[WVLSOL_V0_JSON_DESC]
#print thar_wvl_sol.keys()
#["wvl_sol"]
#json_name = thar_path.get_secondary_path("wvlsol_v0")
#json_name = igr_path.get_secondary_calib_filename(json_name_)
#thar_wvl_sol = PipelineProducts.load(json_name)
if 0: # it would be better to iteratively refit the solution
fn = sky_path.get_secondary_path("wvlsol_v1")
p = PipelineProducts.load(fn)
wvl_solutionv = p["wvl_sol"]
orders_w_solutions_ = thar_wvl_sol["orders"]
from igrins.libs.storage_descriptions import ONED_SPEC_JSON_DESC
orders_w_solutions = [
o for o in orders_w_solutions_
if o in raw_spec_product[ONED_SPEC_JSON_DESC]["orders"]
]
_ = dict(
zip(raw_spec_product[ONED_SPEC_JSON_DESC]["orders"],
raw_spec_product[ONED_SPEC_JSON_DESC]["specs"]))
s_list = [_[o] for o in orders_w_solutions]
wvl_solutions = thar_wvl_sol["wvl_sol"]
return orders_w_solutions, wvl_solutions, s_list
def store_wavelength_outputs(self, extractor, p2_list, ap):
orders = extractor.orders_w_solutions
wvl_solutions = extractor.wvl_solutions
p2_dict = dict(zip(orders, p2_list))
# save order_map, etc
order_map = ap.make_order_map()
slitpos_map = ap.make_slitpos_map()
order_map2 = ap.make_order_map(mask_top_bottom=True)
slitoffset_map = np.empty_like(slitpos_map)
slitoffset_map.fill(np.nan)
wavelength_map = np.empty_like(slitpos_map)
wavelength_map.fill(np.nan)
from scipy.interpolate import interp1d
for o, wvl in zip(ap.orders, wvl_solutions):
xi = np.arange(0, 2048)
xl, yl = np.meshgrid(xi, xi)
msk = order_map == o
xl_msk = xl[msk]
slitoffset_map_msk = p2_dict[o](xl_msk, slitpos_map[msk])
slitoffset_map[msk] = slitoffset_map_msk
wvl_interp1d = interp1d(xi, wvl, bounds_error=False)
wavelength_map[msk] = wvl_interp1d(xl_msk - slitoffset_map_msk)
from igrins.libs.storage_descriptions import (ORDERMAP_FITS_DESC,
SLITPOSMAP_FITS_DESC,
SLITOFFSET_FITS_DESC,
WAVELENGTHMAP_FITS_DESC,
ORDERMAP_MASKED_FITS_DESC)
from igrins.libs.products import PipelineImageBase, PipelineProducts
products = PipelineProducts("Distortion map")
for desc, im in [(ORDERMAP_FITS_DESC, order_map),
(SLITPOSMAP_FITS_DESC, slitpos_map),
(SLITOFFSET_FITS_DESC, slitoffset_map),
(WAVELENGTHMAP_FITS_DESC,wavelength_map),
(ORDERMAP_MASKED_FITS_DESC, order_map2)]:
products.add(desc,
PipelineImageBase([], im))
igr_storage = extractor.igr_storage
igr_storage.store(products,
mastername=extractor.obj_filenames[0],
masterhdu=None)
if 0:
# test
x = np.arange(2048, dtype="d")
oi = 10
o = orders[oi]
yi = 0.5*(slit_slice[:-1] + slit_slice[1:])
ax1 = subplot(211)
s1 = s_up[-1][oi]
s2 = s_down[-1][oi]
ax1.plot(x, s1)
ax1.plot(x, s2)
ax2 = subplot(212, sharex=ax1, sharey=ax1)
dx1 = p2_dict[o](x, yi[-1]+np.zeros_like(x))
ax2.plot(x-dx1, s1)
dx2 = p2_dict[o](x, yi[0]+np.zeros_like(x))
ax2.plot(x-dx2, s2)
def save_wavelength_sol(self, extractor,
orders_w_solutions, wvl_sol, p):
oh_sol_products = PipelineProducts("Wavelength solution based on ohlines")
#from igrins.libs.process_thar import ONED_SPEC_JSON
from igrins.libs.products import PipelineDict
from igrins.libs.storage_descriptions import SKY_WVLSOL_JSON_DESC
oh_sol_products.add(SKY_WVLSOL_JSON_DESC,
PipelineDict(orders=orders_w_solutions,
wvl_sol=wvl_sol))
if 1: # save as WAT fits header
xx = np.arange(0, 2048)
xx_plus1 = np.arange(1, 2048+1)
from astropy.modeling import models, fitting
# We convert 2d chebyshev solution to a seriese of 1d
# chebyshev. For now, use naive (and inefficient)
# approach of refitting the solution with 1d. Should be
# reimplemented.
p1d_list = []
for o in orders_w_solutions:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o * 1.e4 # um to angstrom
p_init1d = models.Chebyshev1D(domain=[1, 2048],
degree=p.x_degree)
fit_p1d = fitting.LinearLSQFitter()
p1d = fit_p1d(p_init1d, xx_plus1, wvl)
p1d_list.append(p1d)
from igrins.libs.iraf_helper import get_wat_spec, default_header_str
wat_list = get_wat_spec(orders_w_solutions, p1d_list)
# cards = [pyfits.Card.fromstring(l.strip()) \
# for l in open("echell_2dspec.header")]
import astropy.io.fits as pyfits
cards = [pyfits.Card.fromstring(l.strip()) \
for l in default_header_str]
wat = "wtype=multispec " + " ".join(wat_list)
char_per_line = 68
num_line, remainder = divmod(len(wat), char_per_line)
for i in range(num_line):
k = "WAT2_%03d" % (i+1,)
v = wat[char_per_line*i:char_per_line*(i+1)]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if remainder > 0:
i = num_line
k = "WAT2_%03d" % (i+1,)
v = wat[char_per_line*i:]
#print k, v
c = pyfits.Card(k, v)
cards.append(c)
if 1:
# save fits with empty header
header = pyfits.Header(cards)
hdu = pyfits.PrimaryHDU(header=header,
data=np.array([]).reshape((0,0)))
from igrins.libs.storage_descriptions import SKY_WVLSOL_FITS_DESC
from igrins.libs.products import PipelineImageBase
oh_sol_products.add(SKY_WVLSOL_FITS_DESC,
PipelineImageBase([],
np.array(wvl_sol)))
igr_storage = extractor.igr_storage
sky_filenames = extractor.obj_filenames
igr_storage.store(oh_sol_products,
mastername=sky_filenames[0],
masterhdu=hdu)
#fn = sky_path.get_secondary_path("wvlsol_v1.fits")
#hdu.writeto(fn, clobber=True)
if 0:
# plot all spectra
for w, s in zip(wvl_sol, s_list):
plot(w, s)
def get_sky_spectra(self, extractor, ap, band, master_obsid):
from igrins.libs.process_thar import get_1d_median_specs
sky_filenames = extractor.obj_filenames
raw_spec_product = get_1d_median_specs(sky_filenames, ap)
# sky_master_fn_ = os.path.splitext(os.path.basename(sky_names[0]))[0]
# sky_master_fn = igr_path.get_secondary_calib_filename(sky_master_fn_)
# import astropy.io.fits as pyfits
# masterhdu = pyfits.open(sky_filenames[0])[0]
from igrins.libs.load_fits import load_fits_data
masterhdu = load_fits_data(sky_filenames[0])
igr_storage = extractor.igr_storage
igr_storage.store(raw_spec_product,
mastername=sky_filenames[0],
masterhdu=masterhdu)
# fn = sky_path.get_secondary_path("raw_spec")
# raw_spec_product.save(fn,
# masterhdu=masterhdu)
# initial wavelength solution
# this need to be fixed
# thar_db.query(sky_master_obsid)
# json_name_ = "SDC%s_%s_0003.median_spectra.wvlsol" % (band,
# igrins_log.date)
#from igrins.libs.storage_descriptions import THAR_WVLSOL_JSON_DESC
from igrins.libs.storage_descriptions import WVLSOL_V0_JSON_DESC
thar_basename = extractor.db["thar"].query(band, master_obsid)
thar_wvl_sol = igr_storage.load([WVLSOL_V0_JSON_DESC],
thar_basename)[WVLSOL_V0_JSON_DESC]
#print thar_wvl_sol.keys()
#["wvl_sol"]
#json_name = thar_path.get_secondary_path("wvlsol_v0")
#json_name = igr_path.get_secondary_calib_filename(json_name_)
#thar_wvl_sol = PipelineProducts.load(json_name)
if 0: # it would be better to iteratively refit the solution
fn = sky_path.get_secondary_path("wvlsol_v1")
p = PipelineProducts.load(fn)
wvl_solutionv = p["wvl_sol"]
orders_w_solutions_ = thar_wvl_sol["orders"]
from igrins.libs.storage_descriptions import ONED_SPEC_JSON_DESC
orders_w_solutions = [o for o in orders_w_solutions_ if o in raw_spec_product[ONED_SPEC_JSON_DESC]["orders"]]
_ = dict(zip(raw_spec_product[ONED_SPEC_JSON_DESC]["orders"],
raw_spec_product[ONED_SPEC_JSON_DESC]["specs"]))
s_list = [_[o]for o in orders_w_solutions]
wvl_solutions = thar_wvl_sol["wvl_sol"]
return orders_w_solutions, wvl_solutions, s_list
def save_figures(helper, band, obsids, thar_products, new_orders):
thar_filenames = helper.get_filenames(band, obsids)
thar_basename = os.path.splitext(os.path.basename(thar_filenames[0]))[0]
thar_master_obsid = obsids[0]
if 1: # make amp and order falt
ap = get_simple_aperture(helper,
band,
thar_master_obsid,
orders=new_orders)
from igrins.libs.storage_descriptions import ONED_SPEC_JSON_DESC
orders = thar_products[ONED_SPEC_JSON_DESC]["orders"]
order_map = ap.make_order_map()
#slitpos_map = ap.make_slitpos_map()
# load flat on products
#flat_on_params_name = flaton_path.get_secondary_path("flat_on_params")
#flaton_products = PipelineProducts.load(flat_on_params_name)
from igrins.libs.storage_descriptions import (FLAT_NORMED_DESC,
FLAT_MASK_DESC)
flaton_db_name = helper.igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
flaton_basename = flaton_db.query(band, thar_master_obsid)
flaton_products = helper.igr_storage.load(
[FLAT_NORMED_DESC, FLAT_MASK_DESC], flaton_basename)
from igrins.libs.process_flat import make_order_flat, check_order_flat
order_flat_products = make_order_flat(flaton_products, orders,
order_map)
#fn = thar_path.get_secondary_path("orderflat")
#order_flat_products.save(fn, masterhdu=hdu)
hdu = pyfits.open(thar_filenames[0])[0]
helper.igr_storage.store(order_flat_products,
mastername=flaton_basename,
masterhdu=hdu)
flat_mask = helper.igr_storage.load1(FLAT_MASK_DESC, flaton_basename)
order_map2 = ap.make_order_map(mask_top_bottom=True)
bias_mask = flat_mask.data & (order_map2 > 0)
pp = PipelineProducts("")
from igrins.libs.storage_descriptions import BIAS_MASK_DESC
pp.add(BIAS_MASK_DESC, PipelineImageBase([], bias_mask))
helper.igr_storage.store(pp, mastername=flaton_basename, masterhdu=hdu)
if 1:
fig_list = check_order_flat(order_flat_products)
from igrins.libs.qa_helper import figlist_to_pngs
orderflat_figs = helper.igr_path.get_section_filename_base(
"QA_PATH", "orderflat", "orderflat_" + thar_basename)
figlist_to_pngs(orderflat_figs, fig_list)
def process_distortion_sky_band(utdate, refdate, band, obsids, config):
from igrins.libs.products import ProductDB, PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
sky_filenames = igr_path.get_filenames(band, obsids)
sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]
master_obsid = obsids[0]
flaton_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
thar_db_name = igr_path.get_section_filename_base("PRIMARY_CALIB_PATH",
"thar.db",
)
thar_db = ProductDB(thar_db_name)
from igrins.libs.process_thar import COMBINED_IMAGE_DESC, ONED_SPEC_JSON
raw_spec_products = igr_storage.load([COMBINED_IMAGE_DESC, ONED_SPEC_JSON],
sky_basename)
# raw_spec_products = PipelineProducts.load(sky_path.get_secondary_path("raw_spec"))
SKY_WVLSOL_JSON_DESC = ("PRIMARY_CALIB_PATH", "SKY_", ".wvlsol_v1.json")
wvlsol_products = igr_storage.load([SKY_WVLSOL_JSON_DESC],
sky_basename)[SKY_WVLSOL_JSON_DESC]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
ap = load_aperture(igr_storage, band, master_obsid,
flaton_db,
raw_spec_products[ONED_SPEC_JSON]["orders"],
orders_w_solutions)
#orders_w_solutions = ap.orders
if 1: # load reference data
from igrins.libs.master_calib import load_sky_ref_data
ref_utdate = config.get_value("REFDATE", utdate)
sky_ref_data = load_sky_ref_data(ref_utdate, band)
ohlines_db = sky_ref_data["ohlines_db"]
ref_ohline_indices = sky_ref_data["ohline_indices"]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
if 0:
raw_spec_products = PipelineProducts.load(sky_path.get_secondary_path("raw_spec"))
basename = flaton_db.query(band, sky_master_obsid)
flaton_path = ProductPath(igr_path, basename)
aperture_solution_products = PipelineProducts.load(flaton_path.get_secondary_path("aperture_solutions"))
bottomup_solutions = aperture_solution_products["bottom_up_solutions"]
basename = thar_db.query(band, sky_master_obsid)
thar_path = ProductPath(igr_path, basename)
fn = thar_path.get_secondary_path("median_spectra")
# thar_products = PipelineProducts.load(fn)
# basename = sky_db.query(band, sky_master_obsid)
# sky_path = ProductPath(igr_path, basename)
fn = sky_path.get_secondary_path("wvlsol_v1")
wvlsol_products = PipelineProducts.load(fn)
if 1:
from igrins.libs.master_calib import load_sky_ref_data
ref_utdate = "20140316"
sky_ref_data = load_sky_ref_data(ref_utdate, band)
ohlines_db = sky_ref_data["ohlines_db"]
ref_ohline_indices = sky_ref_data["ohline_indices"]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
if 1: # make aperture
_o_s = dict(zip(raw_spec_products["orders"], bottomup_solutions))
ap = Apertures(orders_w_solutions,
[_o_s[o] for o in orders_w_solutions])
if 1:
n_slice_one_direction = 2
n_slice = n_slice_one_direction*2 + 1
i_center = n_slice_one_direction
slit_slice = np.linspace(0., 1., n_slice+1)
slice_center = (slit_slice[i_center], slit_slice[i_center+1])
slice_up = [(slit_slice[i_center+i], slit_slice[i_center+i+1]) \
for i in range(1, n_slice_one_direction+1)]
slice_down = [(slit_slice[i_center-i-1], slit_slice[i_center-i]) \
for i in range(n_slice_one_direction)]
d = raw_spec_products[COMBINED_IMAGE_DESC].data
s_center = ap.extract_spectra_v2(d, slice_center[0], slice_center[1])
s_up, s_down = [], []
for s1, s2 in slice_up:
s = ap.extract_spectra_v2(d, s1, s2)
s_up.append(s)
for s1, s2 in slice_down:
s = ap.extract_spectra_v2(d, s1, s2)
s_down.append(s)
if 1:
# now fit
#ohline_indices = [ref_ohline_indices[o] for o in orders_w_solutions]
if 0:
def test_order(oi):
ax=subplot(111)
ax.plot(wvl_solutions[oi], s_center[oi])
#ax.plot(wvl_solutions[oi], raw_spec_products["specs"][oi])
o = orders[oi]
line_indices = ref_ohline_indices[o]
for li in line_indices:
um = np.take(ohlines_db.um, li)
intensity = np.take(ohlines_db.intensity, li)
ax.vlines(um, ymin=0, ymax=-intensity)
from igrins.libs.reidentify_ohlines import fit_ohlines, fit_ohlines_pixel
def get_reidentified_lines_OH(orders_w_solutions,
wvl_solutions, s_center):
ref_pixel_list, reidentified_lines = \
fit_ohlines(ohlines_db, ref_ohline_indices,
orders_w_solutions,
wvl_solutions, s_center)
reidentified_lines_map = dict(zip(orders_w_solutions,
reidentified_lines))
return reidentified_lines_map, ref_pixel_list
if band == "H":
reidentified_lines_map, ref_pixel_list_oh = \
get_reidentified_lines_OH(orders_w_solutions,
wvl_solutions,
s_center)
def refit_centroid(s_center,
ref_pixel_list=ref_pixel_list_oh):
centroids = fit_ohlines_pixel(s_center,
ref_pixel_list)
return centroids
else: # band K
reidentified_lines_map, ref_pixel_list_oh = \
get_reidentified_lines_OH(orders_w_solutions,
wvl_solutions,
s_center)
import igrins.libs.master_calib as master_calib
fn = "hitran_bootstrap_K_%s.json" % ref_utdate
bootstrap_name = master_calib.get_master_calib_abspath(fn)
import json
bootstrap = json.load(open(bootstrap_name))
import igrins.libs.hitran as hitran
r, ref_pixel_dict_hitrans = hitran.reidentify(wvl_solutions,
s_center,
bootstrap)
# for i, s in r.items():
# ss = reidentified_lines_map[int(i)]
# ss0 = np.concatenate([ss[0], s["pixel"]])
# ss1 = np.concatenate([ss[1], s["wavelength"]])
# reidentified_lines_map[int(i)] = (ss0, ss1)
#reidentified_lines_map, ref_pixel_list
def refit_centroid(s_center,
ref_pixel_list=ref_pixel_list_oh,
ref_pixel_dict_hitrans=ref_pixel_dict_hitrans):
centroids_oh = fit_ohlines_pixel(s_center,
ref_pixel_list)
s_dict = dict(zip(orders_w_solutions, s_center))
centroids_dict_hitrans = hitran.fit_hitrans_pixel(s_dict,
ref_pixel_dict_hitrans)
centroids = []
for o, c_oh in zip(orders_w_solutions, centroids_oh):
if o in centroids_dict_hitrans:
c = np.concatenate([c_oh,
centroids_dict_hitrans[o]["pixel"]])
centroids.append(c)
else:
centroids.append(c_oh)
return centroids
# reidentified_lines_map = get_reidentified_lines(orders_w_solutions,
# wvl_solutions,
# s_center)
if 1:
# TODO: we should not need this, instead recycle from preivious step.
fitted_centroid_center = refit_centroid(s_center)
# fitted_centroid_center = fit_ohlines_pixel(s_center,
# ref_pixel_list)
d_shift_up = []
for s in s_up:
# TODO: ref_pixel_list_filtered need to be updated with recent fit.
fitted_centroid = refit_centroid(s)
# fitted_centroid = fit_ohlines_pixel(s,
# ref_pixel_list)
d_shift = [b-a for a, b in zip(fitted_centroid_center,
fitted_centroid)]
d_shift_up.append(d_shift)
d_shift_down = []
for s in s_down:
# TODO: ref_pixel_list_filtered need to be updated with recent fit.
fitted_centroid = refit_centroid(s)
# fitted_centroid = fit_ohlines_pixel(s,
# ref_pixel_list)
#fitted_centroid_center,
d_shift = [b-a for a, b in zip(fitted_centroid_center,
fitted_centroid)]
d_shift_down.append(d_shift)
if 1:
# now fit
orders = orders_w_solutions
x_domain = [0, 2048]
y_domain = [orders[0]-2, orders[-1]+2]
xl = np.concatenate(fitted_centroid_center)
yl_ = [o + np.zeros_like(x_) for o, x_ in zip(orders,
fitted_centroid_center)]
yl = np.concatenate(yl_)
from igrins.libs.ecfit import fit_2dspec, check_fit_simple
zl_list = [np.concatenate(d_) for d_ \
in d_shift_down[::-1] + d_shift_up]
pm_list = []
for zl in zl_list:
p, m = fit_2dspec(xl, yl, zl,
x_degree=1, y_degree=1,
x_domain=x_domain, y_domain=y_domain)
pm_list.append((p,m))
zz_std_list = []
for zl, (p, m) in zip(zl_list, pm_list):
z_m = p(xl[m], yl[m])
zz = z_m - zl[m]
zz_std_list.append(zz.std())
fig_list = []
from matplotlib.figure import Figure
for zl, (p, m) in zip(zl_list, pm_list):
fig = Figure()
check_fit_simple(fig, xl[m], yl[m], zl[m], p, orders)
fig_list.append(fig)
if 1:
xi = np.linspace(0, 2048, 128+1)
from astropy.modeling import fitting
from astropy.modeling.polynomial import Chebyshev2D
x_domain = [0, 2048]
y_domain = [0., 1.]
p2_list = []
for o in orders:
oi = np.zeros_like(xi) + o
shift_list = []
for p,m in pm_list[:n_slice_one_direction]:
shift_list.append(p(xi, oi))
shift_list.append(np.zeros_like(xi))
for p,m in pm_list[n_slice_one_direction:]:
shift_list.append(p(xi, oi))
p_init = Chebyshev2D(x_degree=1, y_degree=2,
x_domain=x_domain, y_domain=y_domain)
f = fitting.LinearLSQFitter()
yi = 0.5*(slit_slice[:-1] + slit_slice[1:])
xl, yl = np.meshgrid(xi, yi)
zl = np.array(shift_list)
p = f(p_init, xl, yl, zl)
p2_list.append(p)
if 1:
p2_dict = dict(zip(orders, p2_list))
order_map = ap.make_order_map()
slitpos_map = ap.make_slitpos_map()
slitoffset_map = np.empty_like(slitpos_map)
slitoffset_map.fill(np.nan)
for o in ap.orders:
xi = np.arange(0, 2048)
xl, yl = np.meshgrid(xi, xi)
msk = order_map == o
slitoffset_map[msk] = p2_dict[o](xl[msk], slitpos_map[msk])
import astropy.io.fits as pyfits
#fn = sky_path.get_secondary_path("slitoffset_map.fits")
#pyfits.PrimaryHDU(data=slitoffset_map).writeto(fn, clobber=True)
SLITOFFSET_FITS_DESC = ("PRIMARY_CALIB_PATH", "SKY_", ".slitoffset_map.fits")
from igrins.libs.products import PipelineImageBase, PipelineProducts
distortion_products = PipelineProducts("Distortion map")
distortion_products.add(SLITOFFSET_FITS_DESC,
PipelineImageBase([],
slitoffset_map))
igr_storage.store(distortion_products,
mastername=sky_filenames[0],
masterhdu=None)
from igrins.libs.qa_helper import figlist_to_pngs
sky_figs = igr_path.get_section_filename_base("QA_PATH",
"oh_distortion",
"oh_distortion_dir")
print fig_list
figlist_to_pngs(sky_figs, fig_list)
if 0:
# test
x = np.arange(2048, dtype="d")
oi = 10
o = orders[oi]
yi = 0.5*(slit_slice[:-1] + slit_slice[1:])
ax1 = subplot(211)
s1 = s_up[-1][oi]
s2 = s_down[-1][oi]
ax1.plot(x, s1)
ax1.plot(x, s2)
ax2 = subplot(212, sharex=ax1, sharey=ax1)
dx1 = p2_dict[o](x, yi[-1]+np.zeros_like(x))
ax2.plot(x-dx1, s1)
dx2 = p2_dict[o](x, yi[0]+np.zeros_like(x))
ax2.plot(x-dx2, s2)
def process_distortion_sky_band(utdate, refdate, band, obsids, config):
from igrins.libs.products import ProductDB, PipelineStorage
igr_path = IGRINSPath(config, utdate)
igr_storage = PipelineStorage(igr_path)
sky_filenames = igr_path.get_filenames(band, obsids)
sky_basename = os.path.splitext(os.path.basename(sky_filenames[0]))[0]
master_obsid = obsids[0]
flaton_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"flat_on.db",
)
flaton_db = ProductDB(flaton_db_name)
thar_db_name = igr_path.get_section_filename_base(
"PRIMARY_CALIB_PATH",
"thar.db",
)
thar_db = ProductDB(thar_db_name)
from igrins.libs.process_thar import COMBINED_IMAGE_DESC, ONED_SPEC_JSON
raw_spec_products = igr_storage.load([COMBINED_IMAGE_DESC, ONED_SPEC_JSON],
sky_basename)
# raw_spec_products = PipelineProducts.load(sky_path.get_secondary_path("raw_spec"))
SKY_WVLSOL_JSON_DESC = ("PRIMARY_CALIB_PATH", "SKY_", ".wvlsol_v1.json")
wvlsol_products = igr_storage.load([SKY_WVLSOL_JSON_DESC],
sky_basename)[SKY_WVLSOL_JSON_DESC]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
ap = load_aperture(igr_storage, band, master_obsid, flaton_db,
raw_spec_products[ONED_SPEC_JSON]["orders"],
orders_w_solutions)
#orders_w_solutions = ap.orders
if 1: # load reference data
from igrins.libs.master_calib import load_sky_ref_data
ref_utdate = config.get_value("REFDATE", utdate)
sky_ref_data = load_sky_ref_data(ref_utdate, band)
ohlines_db = sky_ref_data["ohlines_db"]
ref_ohline_indices = sky_ref_data["ohline_indices"]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
if 0:
raw_spec_products = PipelineProducts.load(
sky_path.get_secondary_path("raw_spec"))
basename = flaton_db.query(band, sky_master_obsid)
flaton_path = ProductPath(igr_path, basename)
aperture_solution_products = PipelineProducts.load(
flaton_path.get_secondary_path("aperture_solutions"))
bottomup_solutions = aperture_solution_products["bottom_up_solutions"]
basename = thar_db.query(band, sky_master_obsid)
thar_path = ProductPath(igr_path, basename)
fn = thar_path.get_secondary_path("median_spectra")
# thar_products = PipelineProducts.load(fn)
# basename = sky_db.query(band, sky_master_obsid)
# sky_path = ProductPath(igr_path, basename)
fn = sky_path.get_secondary_path("wvlsol_v1")
wvlsol_products = PipelineProducts.load(fn)
if 1:
from igrins.libs.master_calib import load_sky_ref_data
ref_utdate = "20140316"
sky_ref_data = load_sky_ref_data(ref_utdate, band)
ohlines_db = sky_ref_data["ohlines_db"]
ref_ohline_indices = sky_ref_data["ohline_indices"]
orders_w_solutions = wvlsol_products["orders"]
wvl_solutions = wvlsol_products["wvl_sol"]
if 1: # make aperture
_o_s = dict(zip(raw_spec_products["orders"], bottomup_solutions))
ap = Apertures(orders_w_solutions,
[_o_s[o] for o in orders_w_solutions])
if 1:
n_slice_one_direction = 2
n_slice = n_slice_one_direction * 2 + 1
i_center = n_slice_one_direction
slit_slice = np.linspace(0., 1., n_slice + 1)
slice_center = (slit_slice[i_center], slit_slice[i_center + 1])
slice_up = [(slit_slice[i_center+i], slit_slice[i_center+i+1]) \
for i in range(1, n_slice_one_direction+1)]
slice_down = [(slit_slice[i_center-i-1], slit_slice[i_center-i]) \
for i in range(n_slice_one_direction)]
d = raw_spec_products[COMBINED_IMAGE_DESC].data
s_center = ap.extract_spectra_v2(d, slice_center[0], slice_center[1])
s_up, s_down = [], []
for s1, s2 in slice_up:
s = ap.extract_spectra_v2(d, s1, s2)
s_up.append(s)
for s1, s2 in slice_down:
s = ap.extract_spectra_v2(d, s1, s2)
s_down.append(s)
if 1:
# now fit
#ohline_indices = [ref_ohline_indices[o] for o in orders_w_solutions]
if 0:
def test_order(oi):
ax = subplot(111)
ax.plot(wvl_solutions[oi], s_center[oi])
#ax.plot(wvl_solutions[oi], raw_spec_products["specs"][oi])
o = orders[oi]
line_indices = ref_ohline_indices[o]
for li in line_indices:
um = np.take(ohlines_db.um, li)
intensity = np.take(ohlines_db.intensity, li)
ax.vlines(um, ymin=0, ymax=-intensity)
from igrins.libs.reidentify_ohlines import fit_ohlines, fit_ohlines_pixel
def get_reidentified_lines_OH(orders_w_solutions, wvl_solutions,
s_center):
ref_pixel_list, reidentified_lines = \
fit_ohlines(ohlines_db, ref_ohline_indices,
orders_w_solutions,
wvl_solutions, s_center)
reidentified_lines_map = dict(
zip(orders_w_solutions, reidentified_lines))
return reidentified_lines_map, ref_pixel_list
if band == "H":
reidentified_lines_map, ref_pixel_list_oh = \
get_reidentified_lines_OH(orders_w_solutions,
wvl_solutions,
s_center)
def refit_centroid(s_center, ref_pixel_list=ref_pixel_list_oh):
centroids = fit_ohlines_pixel(s_center, ref_pixel_list)
return centroids
else: # band K
reidentified_lines_map, ref_pixel_list_oh = \
get_reidentified_lines_OH(orders_w_solutions,
wvl_solutions,
s_center)
import igrins.libs.master_calib as master_calib
fn = "hitran_bootstrap_K_%s.json" % ref_utdate
bootstrap_name = master_calib.get_master_calib_abspath(fn)
import json
bootstrap = json.load(open(bootstrap_name))
import igrins.libs.hitran as hitran
r, ref_pixel_dict_hitrans = hitran.reidentify(
wvl_solutions, s_center, bootstrap)
# for i, s in r.items():
# ss = reidentified_lines_map[int(i)]
# ss0 = np.concatenate([ss[0], s["pixel"]])
# ss1 = np.concatenate([ss[1], s["wavelength"]])
# reidentified_lines_map[int(i)] = (ss0, ss1)
#reidentified_lines_map, ref_pixel_list
def refit_centroid(s_center,
ref_pixel_list=ref_pixel_list_oh,
ref_pixel_dict_hitrans=ref_pixel_dict_hitrans):
centroids_oh = fit_ohlines_pixel(s_center, ref_pixel_list)
s_dict = dict(zip(orders_w_solutions, s_center))
centroids_dict_hitrans = hitran.fit_hitrans_pixel(
s_dict, ref_pixel_dict_hitrans)
centroids = []
for o, c_oh in zip(orders_w_solutions, centroids_oh):
if o in centroids_dict_hitrans:
c = np.concatenate(
[c_oh, centroids_dict_hitrans[o]["pixel"]])
centroids.append(c)
else:
centroids.append(c_oh)
return centroids
# reidentified_lines_map = get_reidentified_lines(orders_w_solutions,
# wvl_solutions,
# s_center)
if 1:
# TODO: we should not need this, instead recycle from preivious step.
fitted_centroid_center = refit_centroid(s_center)
# fitted_centroid_center = fit_ohlines_pixel(s_center,
# ref_pixel_list)
d_shift_up = []
for s in s_up:
# TODO: ref_pixel_list_filtered need to be updated with recent fit.
fitted_centroid = refit_centroid(s)
# fitted_centroid = fit_ohlines_pixel(s,
# ref_pixel_list)
d_shift = [
b - a for a, b in zip(fitted_centroid_center, fitted_centroid)
]
d_shift_up.append(d_shift)
d_shift_down = []
for s in s_down:
# TODO: ref_pixel_list_filtered need to be updated with recent fit.
fitted_centroid = refit_centroid(s)
# fitted_centroid = fit_ohlines_pixel(s,
# ref_pixel_list)
#fitted_centroid_center,
d_shift = [
b - a for a, b in zip(fitted_centroid_center, fitted_centroid)
]
d_shift_down.append(d_shift)
if 1:
# now fit
orders = orders_w_solutions
x_domain = [0, 2048]
y_domain = [orders[0] - 2, orders[-1] + 2]
xl = np.concatenate(fitted_centroid_center)
yl_ = [
o + np.zeros_like(x_)
for o, x_ in zip(orders, fitted_centroid_center)
]
yl = np.concatenate(yl_)
from igrins.libs.ecfit import fit_2dspec, check_fit_simple
zl_list = [np.concatenate(d_) for d_ \
in d_shift_down[::-1] + d_shift_up]
pm_list = []
for zl in zl_list:
p, m = fit_2dspec(xl,
yl,
zl,
x_degree=1,
y_degree=1,
x_domain=x_domain,
y_domain=y_domain)
pm_list.append((p, m))
zz_std_list = []
for zl, (p, m) in zip(zl_list, pm_list):
z_m = p(xl[m], yl[m])
zz = z_m - zl[m]
zz_std_list.append(zz.std())
fig_list = []
from matplotlib.figure import Figure
for zl, (p, m) in zip(zl_list, pm_list):
fig = Figure()
check_fit_simple(fig, xl[m], yl[m], zl[m], p, orders)
fig_list.append(fig)
if 1:
xi = np.linspace(0, 2048, 128 + 1)
from astropy.modeling import fitting
from astropy.modeling.polynomial import Chebyshev2D
x_domain = [0, 2048]
y_domain = [0., 1.]
p2_list = []
for o in orders:
oi = np.zeros_like(xi) + o
shift_list = []
for p, m in pm_list[:n_slice_one_direction]:
shift_list.append(p(xi, oi))
shift_list.append(np.zeros_like(xi))
for p, m in pm_list[n_slice_one_direction:]:
shift_list.append(p(xi, oi))
p_init = Chebyshev2D(x_degree=1,
y_degree=2,
x_domain=x_domain,
y_domain=y_domain)
f = fitting.LinearLSQFitter()
yi = 0.5 * (slit_slice[:-1] + slit_slice[1:])
xl, yl = np.meshgrid(xi, yi)
zl = np.array(shift_list)
p = f(p_init, xl, yl, zl)
p2_list.append(p)
if 1:
p2_dict = dict(zip(orders, p2_list))
order_map = ap.make_order_map()
slitpos_map = ap.make_slitpos_map()
slitoffset_map = np.empty_like(slitpos_map)
slitoffset_map.fill(np.nan)
for o in ap.orders:
xi = np.arange(0, 2048)
xl, yl = np.meshgrid(xi, xi)
msk = order_map == o
slitoffset_map[msk] = p2_dict[o](xl[msk], slitpos_map[msk])
import astropy.io.fits as pyfits
#fn = sky_path.get_secondary_path("slitoffset_map.fits")
#pyfits.PrimaryHDU(data=slitoffset_map).writeto(fn, clobber=True)
SLITOFFSET_FITS_DESC = ("PRIMARY_CALIB_PATH", "SKY_",
".slitoffset_map.fits")
from igrins.libs.products import PipelineImageBase, PipelineProducts
distortion_products = PipelineProducts("Distortion map")
distortion_products.add(SLITOFFSET_FITS_DESC,
PipelineImageBase([], slitoffset_map))
igr_storage.store(distortion_products,
mastername=sky_filenames[0],
masterhdu=None)
from igrins.libs.qa_helper import figlist_to_pngs
sky_figs = igr_path.get_section_filename_base("QA_PATH",
"oh_distortion",
"oh_distortion_dir")
print fig_list
figlist_to_pngs(sky_figs, fig_list)
if 0:
# test
x = np.arange(2048, dtype="d")
oi = 10
o = orders[oi]
yi = 0.5 * (slit_slice[:-1] + slit_slice[1:])
ax1 = subplot(211)
s1 = s_up[-1][oi]
s2 = s_down[-1][oi]
ax1.plot(x, s1)
ax1.plot(x, s2)
ax2 = subplot(212, sharex=ax1, sharey=ax1)
dx1 = p2_dict[o](x, yi[-1] + np.zeros_like(x))
ax2.plot(x - dx1, s1)
dx2 = p2_dict[o](x, yi[0] + np.zeros_like(x))
ax2.plot(x - dx2, s2)