def trace_solutions(trace_products):
from storage_descriptions import FLATCENTROIDS_JSON_DESC
centroids_dict = trace_products[FLATCENTROIDS_JSON_DESC]
bottom_centroids = centroids_dict["bottom_centroids"]
up_centroids = centroids_dict["up_centroids"]
nx = IGRINSDetector.nx
_ = trace_centroids_chevyshev(bottom_centroids,
up_centroids,
domain=[0, nx],
ref_x=nx / 2)
bottom_up_solutions_full, bottom_up_solutions, bottom_up_centroids = _
assert len(bottom_up_solutions_full) != 0
from numpy.polynomial import Polynomial
bottom_up_solutions_as_list = []
for b, d in bottom_up_solutions_full:
bb, dd = b.convert(kind=Polynomial), d.convert(kind=Polynomial)
bb_ = ("poly", bb.coef)
dd_ = ("poly", dd.coef)
bottom_up_solutions_as_list.append((bb_, dd_))
from storage_descriptions import FLATCENTROID_SOL_JSON_DESC
r = PipelineProducts("order trace solutions")
r.add(
FLATCENTROID_SOL_JSON_DESC,
PipelineDict(
orders=[],
#bottom_up_centroids=bottom_up_centroids,
#bottom_up_solutions=bottom_up_solutions,
bottom_up_solutions=bottom_up_solutions_as_list,
))
r2 = PipelineProducts("order trace solutions")
r2.add(
FLATCENTROID_SOL_JSON_DESC,
PipelineDict( #orders=[],
bottom_up_centroids=bottom_up_centroids,
bottom_up_solutions=bottom_up_solutions,
#bottom_up_solutions_full=bottom_up_solutions_as_list,
))
return r, r2
def store_dict(self, basename, item_type, data):
basename = self._get_basename(basename)
item_desc = self.DESC_DICT[item_type.upper()]
from products import PipelineDict
pipeline_dict = PipelineDict(**data)
self.helper.store_item(item_desc, basename, pipeline_dict)
def reidentify_ThAr_lines(thar_products, thar_ref_data):
import numpy as np
orders_src = thar_ref_data["orders"]
s_list_src = thar_ref_data["ref_s_list"]
# load spec
#s_list_ = json.load(open("arc_spec_thar_%s_%s.json" % (band, date)))
from storage_descriptions import ONED_SPEC_JSON_DESC
orders_dst = thar_products[ONED_SPEC_JSON_DESC]["orders"]
s_list_ = thar_products[ONED_SPEC_JSON_DESC]["specs"]
s_list_dst = [np.array(s) for s in s_list_]
orders_intersection = set(orders_src).intersection(orders_dst)
def filter_order(orders, s_list, orders_intersection):
s_list_filtered = [
s for o, s in zip(orders, s_list) if o in orders_intersection
]
return s_list_filtered
s_list_src = filter_order(orders_src, s_list_src, orders_intersection)
s_list_dst = filter_order(orders_dst, s_list_dst, orders_intersection)
ref_lines_list = filter_order(orders_src, thar_ref_data["ref_lines_list"],
orders_intersection)
orders = sorted(orders_intersection)
from reidentify_thar_lines import get_offset_transform
# get offset function from source spectra to target specta.
sol_list_transform = get_offset_transform(s_list_src, s_list_dst)
from reidentify import reidentify_lines_all2
#ref_lines_map = dict(zip(orders_src, ref_lines_list))
#ref_lines_list_dst = [ref_lines_map[o] for o in orders_dst]
reidentified_lines_with_id = reidentify_lines_all2(s_list_dst,
ref_lines_list,
sol_list_transform)
from storage_descriptions import THAR_REID_JSON_DESC
r = PipelineProducts("initial reidentification of ThAr lines")
r.add(
THAR_REID_JSON_DESC,
PipelineDict(
orders=orders,
match_list=reidentified_lines_with_id,
#ref_date=ref_date,
ref_spec_file=thar_ref_data["ref_spec_file"],
ref_id_file=thar_ref_data["ref_id_file"]))
return r
def get_wavelength_solutions(thar_echellogram_products, echel):
from ecfit import get_ordered_line_data, fit_2dspec, check_fit
from storage_descriptions import THAR_ALIGNED_JSON_DESC
affine_tr = thar_echellogram_products[THAR_ALIGNED_JSON_DESC]["affine_tr"]
d_x_wvl = {}
for order, z in echel.zdata.items():
xy_T = affine_tr.transform(np.array([z.x, z.y]).T)
x_T = xy_T[:,0]
d_x_wvl[order]=(x_T, z.wvl)
xl, yl, zl = get_ordered_line_data(d_x_wvl)
# xl : pixel
# yl : order
# zl : wvl * order
x_domain = [0, 2047]
orders_band = sorted(echel.zdata.keys())
#orders = igrins_orders[band]
y_domain = [orders_band[0]-2, orders_band[-1]+2]
p, m = fit_2dspec(xl, yl, zl, x_degree=4, y_degree=3,
x_domain=x_domain, y_domain=y_domain)
if 0:
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(12, 7))
check_fit(fig, xl, yl, zl, p, orders_band, d_x_wvl)
fig.tight_layout()
xx = np.arange(2048)
wvl_sol = []
for o in orders_band:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o
wvl_sol.append(list(wvl))
if 0:
json.dump(wvl_sol,
open("wvl_sol_phase0_%s_%s.json" % \
(band, igrins_log.date), "w"))
from storage_descriptions import THAR_WVLSOL_JSON_DESC
r = PipelineProducts("wavelength solution from ThAr")
r.add(THAR_WVLSOL_JSON_DESC,
PipelineDict(orders=orders_band,
wvl_sol=wvl_sol))
return r
def trace_orders(flaton_products):
# flat_normed=flaton_products["flat_normed"]
# flat_bpixed=flaton_products["flat_bpixed"]
# bg_std_normed=flaton_products["bg_std_normed"]
# flat_mask=flaton_products["flat_mask"]
from storage_descriptions import (FLAT_NORMED_DESC, FLAT_BPIXED_DESC,
FLAT_MASK_DESC, FLATON_JSON_DESC)
flat_normed = flaton_products[FLAT_NORMED_DESC][0].data
flat_bpixed = flaton_products[FLAT_BPIXED_DESC][0].data
flat_mask = flaton_products[FLAT_MASK_DESC][0].data
bg_std_normed = flaton_products[FLATON_JSON_DESC]["bg_std_normed"]
#deadpix_mask=deadpix_mask)
flat_deriv_ = get_y_derivativemap(flat_normed,
flat_bpixed,
bg_std_normed,
max_sep_order=150,
pad=10,
flat_mask=flat_mask)
flat_deriv, flat_deriv_pos_msk, flat_deriv_neg_msk = \
flat_deriv_["data"], flat_deriv_["pos_mask"], flat_deriv_["neg_mask"]
ny, nx = flat_deriv.shape
cent_bottom_list = identify_horizontal_line(flat_deriv,
flat_deriv_pos_msk,
pad=10,
bg_std=bg_std_normed)
cent_bottom_list = check_boundary_orders(cent_bottom_list, nx=2048)
cent_up_list = identify_horizontal_line(-flat_deriv,
flat_deriv_neg_msk,
pad=10,
bg_std=bg_std_normed)
cent_up_list = check_boundary_orders(cent_up_list, nx=2048)
r = PipelineProducts("flat trace centroids")
from storage_descriptions import (FLAT_DERIV_DESC, FLATCENTROIDS_JSON_DESC)
r.add(FLAT_DERIV_DESC, PipelineImageBase([], flat_deriv))
r.add(
FLATCENTROIDS_JSON_DESC,
PipelineDict(bottom_centroids=cent_bottom_list,
up_centroids=cent_up_list))
return r
def get_wavelength_solutions(thar_echellogram_products, echelle, new_orders):
from storage_descriptions import THAR_ALIGNED_JSON_DESC
affine_tr = thar_echellogram_products[THAR_ALIGNED_JSON_DESC]["affine_tr"]
wvl_sol = get_wavelength_solutions2(affine_tr, echelle.zdata, new_orders)
from storage_descriptions import THAR_WVLSOL_JSON_DESC
r = PipelineProducts("wavelength solution from ThAr")
r.add(THAR_WVLSOL_JSON_DESC,
PipelineDict(orders=new_orders, wvl_sol=wvl_sol))
return r
def get_1d_median_specs(fits_names, ap):
#hdu_list = [pyfits.open(fn)[0] for fn in fits_names]
from load_fits import load_fits_data
hdu_list = [load_fits_data(fn) for fn in fits_names]
_data = stsci_median([hdu.data for hdu in hdu_list])
from destriper import destriper
data = destriper.get_destriped(_data)
s = ap.extract_spectra_v2(data)
from storage_descriptions import (COMBINED_IMAGE_DESC, ONED_SPEC_JSON_DESC)
r = PipelineProducts("1d median specs")
r.add(COMBINED_IMAGE_DESC, PipelineImageBase([], data))
r.add(ONED_SPEC_JSON_DESC, PipelineDict(orders=ap.orders, specs=s))
return r
def align_echellogram_thar(thar_reidentified_products, echel, band, ap):
from storage_descriptions import (THAR_REID_JSON_DESC,
THAR_ALIGNED_JSON_DESC)
orders = thar_reidentified_products[THAR_REID_JSON_DESC]["orders"]
fn0 = "ThArlines.dat"
fn = get_master_calib_abspath(fn0)
th = np.genfromtxt(fn)
wvl_thar = th[:, 0] / 1.e4
#s_thar = np.clip(th[:,1], a_min=20, a_max=np.inf)
# line_list : dict of (order, (pixel coord list, wavelengths))
wvl_list = {}
pixel_list = {}
match_list = thar_reidentified_products[THAR_REID_JSON_DESC]["match_list"]
for o, s in zip(orders, match_list):
lineid_list = s[0] # [s1[0] for s1 in s]
wvl = wvl_thar[lineid_list]
wvl_list[o] = wvl
x = [s1[0] for s1 in s[1]]
pixel_list[o] = x
xy1f, nan_mask = echel.get_xy_list_filtered(wvl_list)
xy2f = ap.get_xy_list(pixel_list, nan_mask)
from align_echellogram_thar import fit_affine_clip
affine_tr, mm = fit_affine_clip(xy1f, xy2f)
r = PipelineProducts("ThAr aligned echellogram products")
r.add(
THAR_ALIGNED_JSON_DESC,
PipelineDict(xy1f=xy1f,
xy2f=xy2f,
affine_tr=affine_tr,
affine_tr_mask=mm))
return r
def make_flatoff_hotpixmap(self,
sigma_clip1=100,
sigma_clip2=10,
medfilter_size=None,
destripe=True):
flat_off_cards = []
flat_off = stsci_median(self.data_list)
if destripe:
flat_offs = destriper.get_destriped(flat_off)
flat_off_cards.append(Card(("HISTORY", "IGR: image destriped.")))
hotpix_mask = bp.badpixel_mask(flat_offs,
sigma_clip1=sigma_clip1,
sigma_clip2=sigma_clip2,
medfilter_size=medfilter_size)
bg_std = flat_offs[~hotpix_mask].std()
flat_off_cards.append(
Card(("BG_STD", bg_std, "IGR: stddev of combined flat")))
flat_off_image = PipelineImageBase(flat_off_cards, flat_offs)
hotpix_mask_image = PipelineImageBase([], hotpix_mask)
from storage_descriptions import (FLAT_OFF_DESC, FLATOFF_JSON_DESC,
HOTPIX_MASK_DESC)
r = PipelineProducts("flat off products")
r.add(FLAT_OFF_DESC, flat_off_image)
r.add(HOTPIX_MASK_DESC, hotpix_mask_image)
r.add(FLATOFF_JSON_DESC, PipelineDict(bg_std=bg_std))
return r
def make_flaton_deadpixmap(self,
flatoff_product,
deadpix_mask_old=None,
flat_mask_sigma=5.,
deadpix_thresh=0.6,
smooth_size=9):
# load flat off data
# flat_off = flatoff_product["flat_off"]
# bg_std = flatoff_product["bg_std"]
# hotpix_mask = flatoff_product["hotpix_mask"]
from storage_descriptions import (FLAT_OFF_DESC, FLATOFF_JSON_DESC,
HOTPIX_MASK_DESC)
flat_off = flatoff_product[FLAT_OFF_DESC].data
bg_std = flatoff_product[FLATOFF_JSON_DESC]["bg_std"]
hotpix_mask = flatoff_product[HOTPIX_MASK_DESC].data
flat_on = stsci_median(self.data_list)
flat_on_off = flat_on - flat_off
# normalize it
norm_factor = get_flat_normalization(flat_on_off, bg_std, hotpix_mask)
flat_normed = flat_on_off / norm_factor
flat_std_normed = ni.median_filter(np.std(self.data_list, axis=0) /
norm_factor,
size=(3, 3))
bg_std_norm = bg_std / norm_factor
# mask out bpix
flat_bpixed = flat_normed.astype("d") # by default, astype
# returns new array.
flat_bpixed[hotpix_mask] = np.nan
flat_mask = get_flat_mask_auto(flat_bpixed)
# flat_mask = get_flat_mask(flat_bpixed, bg_std_norm,
# sigma=flat_mask_sigma)
# get dead pixel mask
flat_smoothed = ni.median_filter(flat_normed,
[smooth_size, smooth_size])
#flat_smoothed[order_map==0] = np.nan
flat_ratio = flat_normed / flat_smoothed
flat_std_mask = (flat_smoothed - flat_normed) > 5 * flat_std_normed
refpixel_mask = np.ones(flat_mask.shape, bool)
# mask out outer boundaries
refpixel_mask[4:-4, 4:-4] = False
deadpix_mask = (flat_ratio < deadpix_thresh
) & flat_std_mask & flat_mask & (~refpixel_mask)
if deadpix_mask_old is not None:
deadpix_mask = deadpix_mask | deadpix_mask_old
flat_bpixed[deadpix_mask] = np.nan
from storage_descriptions import (FLAT_NORMED_DESC, FLAT_BPIXED_DESC,
FLAT_MASK_DESC, DEADPIX_MASK_DESC,
FLATON_JSON_DESC)
r = PipelineProducts("flat on products")
r.add(FLAT_NORMED_DESC, PipelineImageBase([], flat_normed))
r.add(FLAT_BPIXED_DESC, PipelineImageBase([], flat_bpixed))
r.add(FLAT_MASK_DESC, PipelineImageBase([], flat_mask))
r.add(DEADPIX_MASK_DESC, PipelineImageBase([], deadpix_mask))
r.add(FLATON_JSON_DESC, PipelineDict(bg_std_normed=bg_std_norm))
return r
def make_order_flat(flaton_products, orders, order_map):
from storage_descriptions import (FLAT_NORMED_DESC, FLAT_MASK_DESC)
flat_normed = flaton_products[FLAT_NORMED_DESC].data
flat_mask = flaton_products[FLAT_MASK_DESC].data
import scipy.ndimage as ni
slices = ni.find_objects(order_map)
mean_order_specs = []
mask_list = []
for o in orders:
sl = (slices[o - 1][0], slice(0, 2048))
d_sl = flat_normed[sl].copy()
d_sl[order_map[sl] != o] = np.nan
f_sl = flat_mask[sl].copy()
f_sl[order_map[sl] != o] = np.nan
ff = np.nanmean(f_sl, axis=0)
mask_list.append(ff)
mmm = order_map[sl] == o
ss = [np.nanmean(d_sl[2:-2][:,i][mmm[:,i][2:-2]]) \
for i in range(2048)]
mean_order_specs.append(ss)
from trace_flat import (get_smoothed_order_spec,
get_order_boundary_indices, get_order_flat1d)
s_list = [get_smoothed_order_spec(s) for s in mean_order_specs]
i1i2_list = [get_order_boundary_indices(s, s0) \
for s, s0 in zip(mean_order_specs, s_list)]
p_list = [get_order_flat1d(s, i1, i2) for s, (i1, i2) \
in zip(s_list, i1i2_list)]
# make flat
x = np.arange(len(s))
flat_im = np.empty(flat_normed.shape, "d")
flat_im.fill(np.nan)
fitted_responses = []
for o, p in zip(orders, p_list):
sl = (slices[o - 1][0], slice(0, 2048))
d_sl = flat_normed[sl].copy()
msk = (order_map[sl] == o)
d_sl[~msk] = np.nan
px = p(x)
flat_im[sl][msk] = (d_sl / px)[msk]
fitted_responses.append(px)
flat_im[flat_im < 0.5] = np.nan
from storage_descriptions import (ORDER_FLAT_IM_DESC, ORDER_FLAT_JSON_DESC)
r = PipelineProducts("order flat")
r.add(ORDER_FLAT_IM_DESC, PipelineImage([], flat_im))
r.add(
ORDER_FLAT_JSON_DESC,
PipelineDict(orders=orders,
fitted_responses=fitted_responses,
i1i2_list=i1i2_list,
mean_order_specs=mean_order_specs))
return r