def reidentify_ThAr_lines(thar_products, thar_ref_data):
import numpy as np
orders_src = thar_ref_data["orders"]
s_list_src = thar_ref_data["ref_s_list"]
# load spec
#s_list_ = json.load(open("arc_spec_thar_%s_%s.json" % (band, date)))
from storage_descriptions import ONED_SPEC_JSON_DESC
orders_dst = thar_products[ONED_SPEC_JSON_DESC]["orders"]
s_list_ = thar_products[ONED_SPEC_JSON_DESC]["specs"]
s_list_dst = [np.array(s) for s in s_list_]
orders_intersection = set(orders_src).intersection(orders_dst)
def filter_order(orders, s_list, orders_intersection):
s_list_filtered = [
s for o, s in zip(orders, s_list) if o in orders_intersection
]
return s_list_filtered
s_list_src = filter_order(orders_src, s_list_src, orders_intersection)
s_list_dst = filter_order(orders_dst, s_list_dst, orders_intersection)
ref_lines_list = filter_order(orders_src, thar_ref_data["ref_lines_list"],
orders_intersection)
orders = sorted(orders_intersection)
from reidentify_thar_lines import get_offset_transform
# get offset function from source spectra to target specta.
sol_list_transform = get_offset_transform(s_list_src, s_list_dst)
from reidentify import reidentify_lines_all2
#ref_lines_map = dict(zip(orders_src, ref_lines_list))
#ref_lines_list_dst = [ref_lines_map[o] for o in orders_dst]
reidentified_lines_with_id = reidentify_lines_all2(s_list_dst,
ref_lines_list,
sol_list_transform)
from storage_descriptions import THAR_REID_JSON_DESC
r = PipelineProducts("initial reidentification of ThAr lines")
r.add(
THAR_REID_JSON_DESC,
PipelineDict(
orders=orders,
match_list=reidentified_lines_with_id,
#ref_date=ref_date,
ref_spec_file=thar_ref_data["ref_spec_file"],
ref_id_file=thar_ref_data["ref_id_file"]))
return r
def get_wavelength_solutions_deprecated(thar_echellogram_products, echel,
new_orders):
"""
new_orders : output orders
"""
from ecfit import get_ordered_line_data, fit_2dspec, check_fit
from storage_descriptions import THAR_ALIGNED_JSON_DESC
affine_tr = thar_echellogram_products[THAR_ALIGNED_JSON_DESC]["affine_tr"]
d_x_wvl = {}
for order, z in echel.zdata.items():
xy_T = affine_tr.transform(np.array([z.x, z.y]).T)
x_T = xy_T[:,0]
d_x_wvl[order]=(x_T, z.wvl)
xl, yl, zl = get_ordered_line_data(d_x_wvl)
# xl : pixel
# yl : order
# zl : wvl * order
x_domain = [0, 2047]
orders_band = sorted(echel.zdata.keys())
#orders = igrins_orders[band]
#y_domain = [orders_band[0]-2, orders_band[-1]+2]
y_domain = [new_orders[0], new_orders[-1]]
p, m = fit_2dspec(xl, yl, zl, x_degree=4, y_degree=3,
x_domain=x_domain, y_domain=y_domain)
if 0:
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(12, 7))
check_fit(fig, xl, yl, zl, p, orders_band, d_x_wvl)
fig.tight_layout()
xx = np.arange(2048)
wvl_sol = []
for o in new_orders:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o
wvl_sol.append(list(wvl))
if 0:
json.dump(wvl_sol,
open("wvl_sol_phase0_%s_%s.json" % \
(band, igrins_log.date), "w"))
from storage_descriptions import THAR_WVLSOL_JSON_DESC
r = PipelineProducts("wavelength solution from ThAr")
r.add(THAR_WVLSOL_JSON_DESC,
PipelineDict(orders=new_orders,
wvl_sol=wvl_sol))
return r
def reidentify_ThAr_lines(thar_products, thar_ref_data):
import numpy as np
orders_src = thar_ref_data["orders"]
s_list_src = thar_ref_data["ref_s_list"]
# load spec
#s_list_ = json.load(open("arc_spec_thar_%s_%s.json" % (band, date)))
from storage_descriptions import ONED_SPEC_JSON_DESC
orders_dst = thar_products[ONED_SPEC_JSON_DESC]["orders"]
s_list_ = thar_products[ONED_SPEC_JSON_DESC]["specs"]
s_list_dst = [np.array(s) for s in s_list_]
orders_intersection = set(orders_src).intersection(orders_dst)
def filter_order(orders, s_list, orders_intersection):
s_list_filtered = [s for o, s in zip(orders, s_list) if o in orders_intersection]
return s_list_filtered
s_list_src = filter_order(orders_src, s_list_src, orders_intersection)
s_list_dst = filter_order(orders_dst, s_list_dst, orders_intersection)
ref_lines_list = filter_order(orders_src,
thar_ref_data["ref_lines_list"],
orders_intersection)
orders = sorted(orders_intersection)
from libs.reidentify_thar_lines import get_offset_transform
# get offset function from source spectra to target specta.
sol_list_transform = get_offset_transform(s_list_src, s_list_dst)
from libs.reidentify import reidentify_lines_all2
#ref_lines_map = dict(zip(orders_src, ref_lines_list))
#ref_lines_list_dst = [ref_lines_map[o] for o in orders_dst]
reidentified_lines_with_id = reidentify_lines_all2(s_list_dst,
ref_lines_list,
sol_list_transform)
from storage_descriptions import THAR_REID_JSON_DESC
r = PipelineProducts("initial reidentification of ThAr lines")
r.add(THAR_REID_JSON_DESC,
PipelineDict(orders=orders,
match_list=reidentified_lines_with_id,
#ref_date=ref_date,
ref_spec_file=thar_ref_data["ref_spec_file"],
ref_id_file=thar_ref_data["ref_id_file"]))
return r
def trace_orders(flaton_products):
# flat_normed=flaton_products["flat_normed"]
# flat_bpixed=flaton_products["flat_bpixed"]
# bg_std_normed=flaton_products["bg_std_normed"]
# flat_mask=flaton_products["flat_mask"]
from storage_descriptions import (FLAT_NORMED_DESC,
FLAT_BPIXED_DESC,
FLAT_MASK_DESC,
FLATON_JSON_DESC)
flat_normed = flaton_products[FLAT_NORMED_DESC][0].data
flat_bpixed = flaton_products[FLAT_BPIXED_DESC][0].data
flat_mask = flaton_products[FLAT_MASK_DESC][0].data
bg_std_normed = flaton_products[FLATON_JSON_DESC]["bg_std_normed"]
#deadpix_mask=deadpix_mask)
flat_deriv_ = get_y_derivativemap(flat_normed, flat_bpixed,
bg_std_normed,
max_sep_order=150, pad=10,
flat_mask=flat_mask)
flat_deriv, flat_deriv_pos_msk, flat_deriv_neg_msk = \
flat_deriv_["data"], flat_deriv_["pos_mask"], flat_deriv_["neg_mask"]
ny, nx = flat_deriv.shape
cent_bottom_list = identify_horizontal_line(flat_deriv,
flat_deriv_pos_msk,
pad=10,
bg_std=bg_std_normed)
cent_bottom_list = check_boundary_orders(cent_bottom_list, nx=2048)
cent_up_list = identify_horizontal_line(-flat_deriv,
flat_deriv_neg_msk,
pad=10,
bg_std=bg_std_normed)
cent_up_list = check_boundary_orders(cent_up_list, nx=2048)
r = PipelineProducts("flat trace centroids")
from storage_descriptions import (FLAT_DERIV_DESC,
FLATCENTROIDS_JSON_DESC)
r.add(FLAT_DERIV_DESC, PipelineImageBase([], flat_deriv))
r.add(FLATCENTROIDS_JSON_DESC,
PipelineDict(bottom_centroids=cent_bottom_list,
up_centroids=cent_up_list))
return r
def get_wavelength_solutions(thar_echellogram_products, echel):
from ecfit import get_ordered_line_data, fit_2dspec, check_fit
from storage_descriptions import THAR_ALIGNED_JSON_DESC
affine_tr = thar_echellogram_products[THAR_ALIGNED_JSON_DESC]["affine_tr"]
d_x_wvl = {}
for order, z in echel.zdata.items():
xy_T = affine_tr.transform(np.array([z.x, z.y]).T)
x_T = xy_T[:,0]
d_x_wvl[order]=(x_T, z.wvl)
xl, yl, zl = get_ordered_line_data(d_x_wvl)
# xl : pixel
# yl : order
# zl : wvl * order
x_domain = [0, 2047]
orders_band = sorted(echel.zdata.keys())
#orders = igrins_orders[band]
y_domain = [orders_band[0]-2, orders_band[-1]+2]
p, m = fit_2dspec(xl, yl, zl, x_degree=4, y_degree=3,
x_domain=x_domain, y_domain=y_domain)
if 0:
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(12, 7))
check_fit(fig, xl, yl, zl, p, orders_band, d_x_wvl)
fig.tight_layout()
xx = np.arange(2048)
wvl_sol = []
for o in orders_band:
oo = np.empty_like(xx)
oo.fill(o)
wvl = p(xx, oo) / o
wvl_sol.append(list(wvl))
if 0:
json.dump(wvl_sol,
open("wvl_sol_phase0_%s_%s.json" % \
(band, igrins_log.date), "w"))
from storage_descriptions import THAR_WVLSOL_JSON_DESC
r = PipelineProducts("wavelength solution from ThAr")
r.add(THAR_WVLSOL_JSON_DESC,
PipelineDict(orders=orders_band,
wvl_sol=wvl_sol))
return r
def get_wavelength_solutions(thar_echellogram_products, echelle, new_orders):
from storage_descriptions import THAR_ALIGNED_JSON_DESC
affine_tr = thar_echellogram_products[THAR_ALIGNED_JSON_DESC]["affine_tr"]
wvl_sol = get_wavelength_solutions2(affine_tr, echelle.zdata, new_orders)
from storage_descriptions import THAR_WVLSOL_JSON_DESC
r = PipelineProducts("wavelength solution from ThAr")
r.add(THAR_WVLSOL_JSON_DESC,
PipelineDict(orders=new_orders, wvl_sol=wvl_sol))
return r
def get_wavelength_solutions(thar_echellogram_products, echelle,
new_orders):
from storage_descriptions import THAR_ALIGNED_JSON_DESC
affine_tr = thar_echellogram_products[THAR_ALIGNED_JSON_DESC]["affine_tr"]
wvl_sol = get_wavelength_solutions2(affine_tr,
echelle.zdata,
new_orders)
from storage_descriptions import THAR_WVLSOL_JSON_DESC
r = PipelineProducts("wavelength solution from ThAr")
r.add(THAR_WVLSOL_JSON_DESC,
PipelineDict(orders=new_orders,
wvl_sol=wvl_sol))
return r
def get_1d_median_specs(fits_names, ap):
hdu_list = [pyfits.open(fn)[0] for fn in fits_names]
_data = stsci_median([hdu.data for hdu in hdu_list])
from destriper import destriper
data = destriper.get_destriped(_data)
s = ap.extract_spectra_v2(data)
from storage_descriptions import (COMBINED_IMAGE_DESC,
ONED_SPEC_JSON_DESC)
r = PipelineProducts("1d median specs")
r.add(COMBINED_IMAGE_DESC,
PipelineImage([], data))
r.add(ONED_SPEC_JSON_DESC,
PipelineDict(orders=ap.orders,
specs=s))
return r
def trace_solutions(trace_products):
from storage_descriptions import FLATCENTROIDS_JSON_DESC
centroids_dict = trace_products[FLATCENTROIDS_JSON_DESC]
bottom_centroids = centroids_dict["bottom_centroids"]
up_centroids = centroids_dict["up_centroids"]
nx = IGRINSDetector.nx
_ = trace_centroids_chevyshev(bottom_centroids,
up_centroids,
domain=[0, nx],
ref_x=nx / 2)
bottom_up_solutions_full, bottom_up_solutions, bottom_up_centroids = _
assert len(bottom_up_solutions_full) != 0
from numpy.polynomial import Polynomial
bottom_up_solutions_as_list = []
for b, d in bottom_up_solutions_full:
bb, dd = b.convert(kind=Polynomial), d.convert(kind=Polynomial)
bb_ = ("poly", bb.coef)
dd_ = ("poly", dd.coef)
bottom_up_solutions_as_list.append((bb_, dd_))
from storage_descriptions import FLATCENTROID_SOL_JSON_DESC
r = PipelineProducts("order trace solutions")
r.add(
FLATCENTROID_SOL_JSON_DESC,
PipelineDict(
orders=[],
#bottom_up_centroids=bottom_up_centroids,
#bottom_up_solutions=bottom_up_solutions,
bottom_up_solutions=bottom_up_solutions_as_list,
))
r2 = PipelineProducts("order trace solutions")
r2.add(
FLATCENTROID_SOL_JSON_DESC,
PipelineDict( #orders=[],
bottom_up_centroids=bottom_up_centroids,
bottom_up_solutions=bottom_up_solutions,
#bottom_up_solutions_full=bottom_up_solutions_as_list,
))
return r, r2
def align_echellogram_thar(thar_reidentified_products, echel, band, ap):
from storage_descriptions import (THAR_REID_JSON_DESC,
THAR_ALIGNED_JSON_DESC)
orders = thar_reidentified_products[THAR_REID_JSON_DESC]["orders"]
fn0 = "ThArlines.dat"
fn = get_master_calib_abspath(fn0)
th = np.genfromtxt(fn)
wvl_thar = th[:, 0] / 1.e4
#s_thar = np.clip(th[:,1], a_min=20, a_max=np.inf)
# line_list : dict of (order, (pixel coord list, wavelengths))
wvl_list = {}
pixel_list = {}
match_list = thar_reidentified_products[THAR_REID_JSON_DESC]["match_list"]
for o, s in zip(orders, match_list):
lineid_list = s[0] # [s1[0] for s1 in s]
wvl = wvl_thar[lineid_list]
wvl_list[o] = wvl
x = [s1[0] for s1 in s[1]]
pixel_list[o] = x
xy1f, nan_mask = echel.get_xy_list_filtered(wvl_list)
xy2f = ap.get_xy_list(pixel_list, nan_mask)
from align_echellogram_thar import fit_affine_clip
affine_tr, mm = fit_affine_clip(xy1f, xy2f)
r = PipelineProducts("ThAr aligned echellogram products")
r.add(
THAR_ALIGNED_JSON_DESC,
PipelineDict(xy1f=xy1f,
xy2f=xy2f,
affine_tr=affine_tr,
affine_tr_mask=mm))
return r
def trace_solutions(trace_products):
from storage_descriptions import FLATCENTROIDS_JSON_DESC
centroids_dict = trace_products[FLATCENTROIDS_JSON_DESC]
bottom_centroids = centroids_dict["bottom_centroids"]
up_centroids = centroids_dict["up_centroids"]
nx = IGRINSDetector.nx
_ = trace_centroids_chevyshev(bottom_centroids,
up_centroids,
domain=[0, nx],
ref_x=nx/2)
bottom_up_solutions, bottom_up_centroids = _
from numpy.polynomial import Polynomial
bottom_up_solutions_as_list = []
for b, d in bottom_up_solutions:
bb, dd = b.convert(kind=Polynomial), d.convert(kind=Polynomial)
bb_ = ("poly", bb.coef)
dd_ = ("poly", dd.coef)
bottom_up_solutions_as_list.append((bb_, dd_))
r = PipelineProducts("order trace solutions")
from storage_descriptions import FLATCENTROID_SOL_JSON_DESC
r.add(FLATCENTROID_SOL_JSON_DESC,
PipelineDict(orders=[],
bottom_up_centroids=bottom_up_centroids,
bottom_up_solutions=bottom_up_solutions_as_list))
return r
def align_echellogram_thar(thar_reidentified_products, echel, band, ap):
from storage_descriptions import (THAR_REID_JSON_DESC,
THAR_ALIGNED_JSON_DESC)
orders = thar_reidentified_products[THAR_REID_JSON_DESC]["orders"]
fn0 = "ThArlines.dat"
fn = get_master_calib_abspath(fn0)
th = np.genfromtxt(fn)
wvl_thar = th[:,0]/1.e4
#s_thar = np.clip(th[:,1], a_min=20, a_max=np.inf)
# line_list : dict of (order, (pixel coord list, wavelengths))
wvl_list = {}
pixel_list = {}
match_list = thar_reidentified_products[THAR_REID_JSON_DESC]["match_list"]
for o, s in zip(orders, match_list):
lineid_list = s[0] # [s1[0] for s1 in s]
wvl = wvl_thar[lineid_list]
wvl_list[o] = wvl
x = [s1[0] for s1 in s[1]]
pixel_list[o] = x
xy1f, nan_mask = echel.get_xy_list_filtered(wvl_list)
xy2f = ap.get_xy_list(pixel_list, nan_mask)
from libs.align_echellogram_thar import fit_affine_clip
affine_tr, mm = fit_affine_clip(xy1f, xy2f)
r = PipelineProducts("ThAr aligned echellogram products")
r.add(THAR_ALIGNED_JSON_DESC,
PipelineDict(xy1f=xy1f, xy2f=xy2f,
affine_tr=affine_tr,
affine_tr_mask=mm))
return r
def trace_orders(flaton_products):
# flat_normed=flaton_products["flat_normed"]
# flat_bpixed=flaton_products["flat_bpixed"]
# bg_std_normed=flaton_products["bg_std_normed"]
# flat_mask=flaton_products["flat_mask"]
from storage_descriptions import (FLAT_NORMED_DESC, FLAT_BPIXED_DESC,
FLAT_MASK_DESC, FLATON_JSON_DESC)
flat_normed = flaton_products[FLAT_NORMED_DESC][0].data
flat_bpixed = flaton_products[FLAT_BPIXED_DESC][0].data
flat_mask = flaton_products[FLAT_MASK_DESC][0].data
bg_std_normed = flaton_products[FLATON_JSON_DESC]["bg_std_normed"]
#deadpix_mask=deadpix_mask)
flat_deriv_ = get_y_derivativemap(flat_normed,
flat_bpixed,
bg_std_normed,
max_sep_order=150,
pad=10,
flat_mask=flat_mask)
flat_deriv, flat_deriv_pos_msk, flat_deriv_neg_msk = \
flat_deriv_["data"], flat_deriv_["pos_mask"], flat_deriv_["neg_mask"]
ny, nx = flat_deriv.shape
cent_bottom_list = identify_horizontal_line(flat_deriv,
flat_deriv_pos_msk,
pad=10,
bg_std=bg_std_normed)
cent_bottom_list = check_boundary_orders(cent_bottom_list, nx=2048)
cent_up_list = identify_horizontal_line(-flat_deriv,
flat_deriv_neg_msk,
pad=10,
bg_std=bg_std_normed)
cent_up_list = check_boundary_orders(cent_up_list, nx=2048)
r = PipelineProducts("flat trace centroids")
from storage_descriptions import (FLAT_DERIV_DESC, FLATCENTROIDS_JSON_DESC)
r.add(FLAT_DERIV_DESC, PipelineImageBase([], flat_deriv))
r.add(
FLATCENTROIDS_JSON_DESC,
PipelineDict(bottom_centroids=cent_bottom_list,
up_centroids=cent_up_list))
return r
def make_flatoff_hotpixmap(self, sigma_clip1=100, sigma_clip2=10,
medfilter_size=None,
destripe=True):
flat_off_cards = []
flat_off = stsci_median(self.data_list)
if destripe:
flat_offs = destriper.get_destriped(flat_off)
flat_off_cards.append(Card(("HISTORY", "IGR: image destriped.")))
hotpix_mask = bp.badpixel_mask(flat_offs,
sigma_clip1=sigma_clip1,
sigma_clip2=sigma_clip2,
medfilter_size=medfilter_size)
bg_std = flat_offs[~hotpix_mask].std()
flat_off_cards.append(Card(("BG_STD", bg_std,
"IGR: stddev of combined flat")))
flat_off_image = PipelineImageBase(flat_off_cards,
flat_offs)
hotpix_mask_image = PipelineImageBase([],
hotpix_mask)
from storage_descriptions import (FLAT_OFF_DESC,
FLATOFF_JSON_DESC,
HOTPIX_MASK_DESC)
r = PipelineProducts("flat off products")
r.add(FLAT_OFF_DESC, flat_off_image)
r.add(HOTPIX_MASK_DESC,
hotpix_mask_image)
r.add(FLATOFF_JSON_DESC,
PipelineDict(bg_std=bg_std))
return r
def make_flatoff_hotpixmap(self,
sigma_clip1=100,
sigma_clip2=10,
medfilter_size=None,
destripe=True):
flat_off_cards = []
flat_off = stsci_median(self.data_list)
if destripe:
flat_offs = destriper.get_destriped(flat_off)
flat_off_cards.append(Card(("HISTORY", "IGR: image destriped.")))
hotpix_mask = bp.badpixel_mask(flat_offs,
sigma_clip1=sigma_clip1,
sigma_clip2=sigma_clip2,
medfilter_size=medfilter_size)
bg_std = flat_offs[~hotpix_mask].std()
flat_off_cards.append(
Card(("BG_STD", bg_std, "IGR: stddev of combined flat")))
flat_off_image = PipelineImageBase(flat_off_cards, flat_offs)
hotpix_mask_image = PipelineImageBase([], hotpix_mask)
from storage_descriptions import (FLAT_OFF_DESC, FLATOFF_JSON_DESC,
HOTPIX_MASK_DESC)
r = PipelineProducts("flat off products")
r.add(FLAT_OFF_DESC, flat_off_image)
r.add(HOTPIX_MASK_DESC, hotpix_mask_image)
r.add(FLATOFF_JSON_DESC, PipelineDict(bg_std=bg_std))
return r
def get_1d_median_specs(fits_names, ap):
#hdu_list = [pyfits.open(fn)[0] for fn in fits_names]
from load_fits import load_fits_data
hdu_list = [load_fits_data(fn) for fn in fits_names]
_data = stsci_median([hdu.data for hdu in hdu_list])
from destriper import destriper
data = destriper.get_destriped(_data)
s = ap.extract_spectra_v2(data)
from storage_descriptions import (COMBINED_IMAGE_DESC, ONED_SPEC_JSON_DESC)
r = PipelineProducts("1d median specs")
r.add(COMBINED_IMAGE_DESC, PipelineImageBase([], data))
r.add(ONED_SPEC_JSON_DESC, PipelineDict(orders=ap.orders, specs=s))
return r
if 0:
order = 5
func_fitter = get_line_fiiter(order)
_ = trace_centroids(cent_bottom_list,
cent_up_list,
func_fitter=func_fitter,
ref_x=nx / 2)
bottom_up_solutions, bottom_up_centroids = _
if 0:
plot_sollutions(flat_normed, bottom_up_centroids, bottom_up_solutions)
r = PipelineProducts(flat_deriv=flat_deriv,
bottom_up_centroids=bottom_up_centroids,
bottom_up_solutions=bottom_up_solutions)
return_object["flat_deriv"] = flat_deriv
#return_object["flat_deriv_pos_mask"] = flat_deriv_pos_msk
#return_object["flat_deriv_neg_mask"] = flat_deriv_neg_msk
return_object["cent_up_list"] = cent_up_list
return_object["cent_bottom_list"] = cent_bottom_list
return_object["bottomup_solutions"] = bottom_up_solutions
return_object["bottomup_centroids"] = centroid_bottom_up_list
# def process_flat(ondata_list, offdata_list):
# return_object = {}
def make_order_flat(flaton_products, orders, order_map):
from storage_descriptions import (FLAT_NORMED_DESC,
FLAT_MASK_DESC)
flat_normed = flaton_products[FLAT_NORMED_DESC].data
flat_mask = flaton_products[FLAT_MASK_DESC].data
import scipy.ndimage as ni
slices = ni.find_objects(order_map)
mean_order_specs = []
mask_list = []
for o in orders:
sl = (slices[o-1][0], slice(0, 2048))
d_sl = flat_normed[sl].copy()
d_sl[order_map[sl] != o] = np.nan
f_sl = flat_mask[sl].copy()
f_sl[order_map[sl] != o] = np.nan
ff = np.nanmean(f_sl, axis=0)
mask_list.append(ff)
mmm = order_map[sl] == o
ss = [np.nanmean(d_sl[2:-2][:,i][mmm[:,i][2:-2]]) \
for i in range(2048)]
mean_order_specs.append(ss)
from trace_flat import (get_smoothed_order_spec,
get_order_boundary_indices,
get_order_flat1d)
s_list = [get_smoothed_order_spec(s) for s in mean_order_specs]
i1i2_list = [get_order_boundary_indices(s, s0) \
for s, s0 in zip(mean_order_specs, s_list)]
p_list = [get_order_flat1d(s, i1, i2) for s, (i1, i2) \
in zip(s_list, i1i2_list)]
# make flat
x = np.arange(len(s))
flat_im = np.empty(flat_normed.shape, "d")
flat_im.fill(np.nan)
fitted_responses = []
for o, p in zip(orders, p_list):
sl = (slices[o-1][0], slice(0, 2048))
d_sl = flat_normed[sl].copy()
msk = (order_map[sl] == o)
d_sl[~msk] = np.nan
px = p(x)
flat_im[sl][msk] = (d_sl / px)[msk]
fitted_responses.append(px)
flat_im[flat_im < 0.5] = np.nan
from storage_descriptions import (ORDER_FLAT_IM_DESC,
ORDER_FLAT_JSON_DESC)
r = PipelineProducts("order flat")
r.add(ORDER_FLAT_IM_DESC, PipelineImage([], flat_im))
r.add(ORDER_FLAT_JSON_DESC,
PipelineDict(orders=orders,
fitted_responses=fitted_responses,
i1i2_list=i1i2_list,
mean_order_specs=mean_order_specs))
return r
def make_order_flat(flaton_products, orders, order_map):
from storage_descriptions import (FLAT_NORMED_DESC,
FLAT_MASK_DESC)
flat_normed = flaton_products[FLAT_NORMED_DESC][0].data
flat_mask = flaton_products[FLAT_MASK_DESC].data
import scipy.ndimage as ni
slices = ni.find_objects(order_map)
mean_order_specs = []
mask_list = []
for o in orders:
# if slices[o-1] is None:
# continue
sl = (slices[o-1][0], slice(0, 2048))
d_sl = flat_normed[sl].copy()
d_sl[order_map[sl] != o] = np.nan
f_sl = flat_mask[sl].copy()
f_sl[order_map[sl] != o] = np.nan
ff = np.nanmean(f_sl, axis=0)
mask_list.append(ff)
mmm = order_map[sl] == o
ss = [np.nanmean(d_sl[2:-2][:,i][mmm[:,i][2:-2]]) \
for i in range(2048)]
mean_order_specs.append(ss)
from trace_flat import (get_smoothed_order_spec,
get_order_boundary_indices,
get_order_flat1d)
s_list = [get_smoothed_order_spec(s) for s in mean_order_specs]
i1i2_list = [get_order_boundary_indices(s, s0) \
for s, s0 in zip(mean_order_specs, s_list)]
#p_list = [get_order_flat1d(s, i1, i2) for s, (i1, i2) \
# in zip(s_list, i1i2_list)]
from smooth_continuum import get_smooth_continuum
s2_list = [get_smooth_continuum(s) for s, (i1, i2) \
in zip(s_list, i1i2_list)]
# make flat
x = np.arange(len(s))
flat_im = np.ones(flat_normed.shape, "d")
#flat_im.fill(np.nan)
fitted_responses = []
for o, px in zip(orders, s2_list):
sl = (slices[o-1][0], slice(0, 2048))
d_sl = flat_normed[sl].copy()
msk = (order_map[sl] == o)
#d_sl[~msk] = np.nan
d_div = d_sl / px
px2d = px * np.ones_like(d_div) # better way to broadcast px?
d_div[px2d < 0.05*px.max()] = 1.
flat_im[sl][msk] = (d_sl / px)[msk]
fitted_responses.append(px)
flat_im[flat_im < 0.5] = np.nan
from storage_descriptions import (ORDER_FLAT_IM_DESC,
ORDER_FLAT_JSON_DESC)
r = PipelineProducts("order flat")
r.add(ORDER_FLAT_IM_DESC, PipelineImageBase([], flat_im))
r.add(ORDER_FLAT_JSON_DESC,
PipelineDict(orders=orders,
fitted_responses=fitted_responses,
i1i2_list=i1i2_list,
mean_order_specs=mean_order_specs))
return r
def make_flaton_deadpixmap(self,
flatoff_product,
deadpix_mask_old=None,
flat_mask_sigma=5.,
deadpix_thresh=0.6,
smooth_size=9):
# load flat off data
# flat_off = flatoff_product["flat_off"]
# bg_std = flatoff_product["bg_std"]
# hotpix_mask = flatoff_product["hotpix_mask"]
from storage_descriptions import (FLAT_OFF_DESC, FLATOFF_JSON_DESC,
HOTPIX_MASK_DESC)
flat_off = flatoff_product[FLAT_OFF_DESC].data
bg_std = flatoff_product[FLATOFF_JSON_DESC]["bg_std"]
hotpix_mask = flatoff_product[HOTPIX_MASK_DESC].data
flat_on = stsci_median(self.data_list)
flat_on_off = flat_on - flat_off
# normalize it
norm_factor = get_flat_normalization(flat_on_off, bg_std, hotpix_mask)
flat_normed = flat_on_off / norm_factor
flat_std_normed = ni.median_filter(np.std(self.data_list, axis=0) /
norm_factor,
size=(3, 3))
bg_std_norm = bg_std / norm_factor
# mask out bpix
flat_bpixed = flat_normed.astype("d") # by default, astype
# returns new array.
flat_bpixed[hotpix_mask] = np.nan
flat_mask = get_flat_mask_auto(flat_bpixed)
# flat_mask = get_flat_mask(flat_bpixed, bg_std_norm,
# sigma=flat_mask_sigma)
# get dead pixel mask
flat_smoothed = ni.median_filter(flat_normed,
[smooth_size, smooth_size])
#flat_smoothed[order_map==0] = np.nan
flat_ratio = flat_normed / flat_smoothed
flat_std_mask = (flat_smoothed - flat_normed) > 5 * flat_std_normed
refpixel_mask = np.ones(flat_mask.shape, bool)
# mask out outer boundaries
refpixel_mask[4:-4, 4:-4] = False
deadpix_mask = (flat_ratio < deadpix_thresh
) & flat_std_mask & flat_mask & (~refpixel_mask)
if deadpix_mask_old is not None:
deadpix_mask = deadpix_mask | deadpix_mask_old
flat_bpixed[deadpix_mask] = np.nan
from storage_descriptions import (FLAT_NORMED_DESC, FLAT_BPIXED_DESC,
FLAT_MASK_DESC, DEADPIX_MASK_DESC,
FLATON_JSON_DESC)
r = PipelineProducts("flat on products")
r.add(FLAT_NORMED_DESC, PipelineImageBase([], flat_normed))
r.add(FLAT_BPIXED_DESC, PipelineImageBase([], flat_bpixed))
r.add(FLAT_MASK_DESC, PipelineImageBase([], flat_mask))
r.add(DEADPIX_MASK_DESC, PipelineImageBase([], deadpix_mask))
r.add(FLATON_JSON_DESC, PipelineDict(bg_std_normed=bg_std_norm))
return r
def make_flaton_deadpixmap(self, flatoff_product,
deadpix_mask_old=None,
flat_mask_sigma=5.,
deadpix_thresh=0.6,
smooth_size=9):
# load flat off data
# flat_off = flatoff_product["flat_off"]
# bg_std = flatoff_product["bg_std"]
# hotpix_mask = flatoff_product["hotpix_mask"]
from storage_descriptions import (FLAT_OFF_DESC,
FLATOFF_JSON_DESC,
HOTPIX_MASK_DESC)
flat_off = flatoff_product[FLAT_OFF_DESC].data
bg_std = flatoff_product[FLATOFF_JSON_DESC]["bg_std"]
hotpix_mask = flatoff_product[HOTPIX_MASK_DESC].data
flat_on = stsci_median(self.data_list)
flat_on_off = flat_on - flat_off
# normalize it
norm_factor = get_flat_normalization(flat_on_off,
bg_std, hotpix_mask)
flat_normed = flat_on_off / norm_factor
flat_std_normed = ni.median_filter(np.std(self.data_list, axis=0) / norm_factor,
size=(3,3))
bg_std_norm = bg_std/norm_factor
# mask out bpix
flat_bpixed = flat_normed.astype("d") # by default, astype
# returns new array.
flat_bpixed[hotpix_mask] = np.nan
flat_mask = get_flat_mask(flat_bpixed, bg_std_norm,
sigma=flat_mask_sigma)
# get dead pixel mask
flat_smoothed = ni.median_filter(flat_normed,
[smooth_size, smooth_size])
#flat_smoothed[order_map==0] = np.nan
flat_ratio = flat_normed/flat_smoothed
flat_std_mask = (flat_smoothed - flat_normed) > 5*flat_std_normed
refpixel_mask = np.ones(flat_mask.shape, bool)
# mask out outer boundaries
refpixel_mask[4:-4,4:-4] = False
deadpix_mask = (flat_ratio<deadpix_thresh) & flat_std_mask & flat_mask & (~refpixel_mask)
if deadpix_mask_old is not None:
deadpix_mask = deadpix_mask | deadpix_mask_old
flat_bpixed[deadpix_mask] = np.nan
from storage_descriptions import (FLAT_NORMED_DESC,
FLAT_BPIXED_DESC,
FLAT_MASK_DESC,
DEADPIX_MASK_DESC,
FLATON_JSON_DESC)
r = PipelineProducts("flat on products")
r.add(FLAT_NORMED_DESC, PipelineImageBase([], flat_normed))
r.add(FLAT_BPIXED_DESC, PipelineImageBase([], flat_bpixed))
r.add(FLAT_MASK_DESC, PipelineImageBase([], flat_mask))
r.add(DEADPIX_MASK_DESC, PipelineImageBase([], deadpix_mask))
r.add(FLATON_JSON_DESC,
PipelineDict(bg_std_normed=bg_std_norm))
return r
def make_order_flat(flaton_products, orders, order_map):
from storage_descriptions import (FLAT_NORMED_DESC, FLAT_MASK_DESC)
flat_normed = flaton_products[FLAT_NORMED_DESC].data
flat_mask = flaton_products[FLAT_MASK_DESC].data
import scipy.ndimage as ni
slices = ni.find_objects(order_map)
mean_order_specs = []
mask_list = []
for o in orders:
sl = (slices[o - 1][0], slice(0, 2048))
d_sl = flat_normed[sl].copy()
d_sl[order_map[sl] != o] = np.nan
f_sl = flat_mask[sl].copy()
f_sl[order_map[sl] != o] = np.nan
ff = np.nanmean(f_sl, axis=0)
mask_list.append(ff)
mmm = order_map[sl] == o
ss = [np.nanmean(d_sl[2:-2][:,i][mmm[:,i][2:-2]]) \
for i in range(2048)]
mean_order_specs.append(ss)
from trace_flat import (get_smoothed_order_spec,
get_order_boundary_indices, get_order_flat1d)
s_list = [get_smoothed_order_spec(s) for s in mean_order_specs]
i1i2_list = [get_order_boundary_indices(s, s0) \
for s, s0 in zip(mean_order_specs, s_list)]
p_list = [get_order_flat1d(s, i1, i2) for s, (i1, i2) \
in zip(s_list, i1i2_list)]
# make flat
x = np.arange(len(s))
flat_im = np.empty(flat_normed.shape, "d")
flat_im.fill(np.nan)
fitted_responses = []
for o, p in zip(orders, p_list):
sl = (slices[o - 1][0], slice(0, 2048))
d_sl = flat_normed[sl].copy()
msk = (order_map[sl] == o)
d_sl[~msk] = np.nan
px = p(x)
flat_im[sl][msk] = (d_sl / px)[msk]
fitted_responses.append(px)
flat_im[flat_im < 0.5] = np.nan
from storage_descriptions import (ORDER_FLAT_IM_DESC, ORDER_FLAT_JSON_DESC)
r = PipelineProducts("order flat")
r.add(ORDER_FLAT_IM_DESC, PipelineImage([], flat_im))
r.add(
ORDER_FLAT_JSON_DESC,
PipelineDict(orders=orders,
fitted_responses=fitted_responses,
i1i2_list=i1i2_list,
mean_order_specs=mean_order_specs))
return r
