def _do_fourier(fpath,
dir_out,
cut_wavelength=200,
med_sub_clip=[-5, 5],
med_rat_clip=[0.5, 2],
std_rat_clip=[-5, 5],
skip_if_exists=True,
verbose=True):
fpath = Path(fpath)
outstem = fpath.stem + "f" # hard-coded
# == Skip if conditions meet =================================================================== #
if skip_if_exists and (dir_out / f"{outstem}.fits").exists():
return
ccd_v = load_ccd(fpath)
ccd_vf = fourier_lrsub(ccd_v,
cut_wavelength=cut_wavelength,
med_sub_clip=med_sub_clip,
med_rat_clip=med_rat_clip,
std_rat_clip=std_rat_clip,
verbose_bpm=verbose,
verbose=verbose)
ccd_vf = CCDData_astype(ccd_vf, 'float32')
update_process(ccd_vf.header,
"f",
add_comment=False,
additional_comment={'f': "fourier pattern"})
_save(ccd_vf, dir_out, outstem)
return ccd_vf
def __init__(self,
fpath,
target=None,
gain_key="GAIN",
rdnoise_key="RDNOISE"):
self.fpath = Path(fpath)
self.ccd = yfu.load_ccd(fpath)
self.header = self.ccd.header
self.wcs = WCS(self.header)
self.target = target
if self.target is None:
try:
self.target = self.header["OBJECT"]
except KeyError:
pass
self.gain = yfu.get_from_header(self.header,
key=gain_key,
unit=u.electron / u.adu,
verbose=False,
default=1)
self.rdnoise = yfu.get_from_header(self.header,
key=rdnoise_key,
unit=u.electron,
verbose=False,
default=0)
def _load_as_dict(path, keys, verbose):
""" Load all "path/*.fits" as a dict with keys.
"""
try:
path = Path(path)
if not path.exists():
path = None
if verbose >= 1:
print(f"File not found at given path ({path})")
except TypeError:
path = None
if path is None:
ccds = None
paths = None
if verbose >= 2:
print("SKIPPED")
else:
path = Path(path)
_summary = summary_nic(f"{path}/*.fits")
if _summary is None:
return path, None, None
ccds = {}
paths = {}
for _, row in _summary.iterrows():
key = tuple([row[k] for k in keys])
if len(key) == 1:
key = key[0]
ccds[key] = load_ccd(row['file'])
paths[key] = row['file']
if verbose >= 2:
print(f"loaded for combinations {keys} = {list(ccds.keys())}")
return path, ccds, paths
def preproc_nic(dir_in,
dir_out,
dir_log=None,
objects=None,
objects_exclude=False,
dir_flat=None,
dir_dark=None,
dir_fringe=None,
skip_if_exists=True,
fringe_object=".*\_[sS][kK][yY]",
verbose=0):
"""
Parameters
----------
objects_include, objects_exclude : str, list-like, None, optional.
The FITS files with certain ``OBJECT`` values in headers to be used or unused in the flat
correction, respectively.
"""
dir_in, dir_out, dir_log = _set_dir_iol(dir_in, dir_out, dir_log)
path_summ_out = dir_log / f"summary_{dir_out.name}.csv"
if skip_if_exists and path_summ_out.exists():
if verbose:
print(
f"Loading the existing summary CSV file from {path_summ_out}; SKIPPING all preprocessing."
)
return pd.read_csv(path_summ_out)
try:
_summary = pd.read_csv(dir_log / f"summary_{dir_in.name}.csv")
_mask_dark = _summary["OBJECT"].str.fullmatch("dark", case=False)
_mask_flat = _summary["OBJECT"].str.fullmatch("flat", case=False)
_mask_test = _summary["OBJECT"].str.fullmatch("test", case=False)
_mask_fringe = _summary["OBJECT"].str.match(fringe_object)
# ^ Fringe is already flat-corrected, so remove it in the process below _summary_filt
_summary = _summary[~(_mask_fringe | _mask_flat | _mask_dark
| _mask_test)]
_summary = _summary.reset_index(drop=True)
except FileNotFoundError:
raise FileNotFoundError(
f"Summary for {dir_in} is not found in {dir_log}." +
" Try changing dir_in or dir_log.")
if verbose >= 1:
print("Loading calibration frames...")
if verbose >= 2:
print(" * Flat frames... ", end='')
dir_flat, flats, flatpaths = _load_as_dict(dir_flat, ['FILTER'],
verbose >= 2)
if verbose >= 2:
print("\n * Dark frames... ", end='')
dir_dark, darks, darkpaths = _load_as_dict(dir_dark, ['FILTER', 'EXPTIME'],
verbose >= 2)
if verbose >= 2:
print("\n * Fringe frames... ", end='')
dir_fringe, fringes, fringepaths = _load_as_dict(
dir_fringe, ['FILTER', 'OBJECT', 'POL-AGL1'], verbose >= 2)
if dir_flat is None and dir_dark is None and dir_fringe is None:
while True:
to_proceed = input(
"There is no flat/dark/fringe found. Are you sure to proceed? [y/N] "
)
if to_proceed.lower() in ['', 'N']:
return
elif to_proceed.lower() not in ['y']:
print("Type yes='y', NO='', 'n'. ")
objects, objects_exclude = _sanitize_objects(objects, objects_exclude)
objnames = np.unique(_summary["OBJECT"])
if verbose >= 1:
print(
f"Flat correction (&& dark and fringe if given); saving to {dir_out}"
)
for filt in 'jhk':
if verbose >= 1:
print(f" * {filt.upper()}: ", end=' ')
_summary_filt = _summary[_summary["FILTER"] == filt.upper()]
mflat, mflatpath = _find_calframe(flats, flatpaths, filt.upper(),
"Flat", verbose >= 2)
for objname in objnames:
if ((objects_exclude and (objname in objects))
or (not objects_exclude and (objname not in objects))):
continue
if verbose >= 1:
print(objname, end='... ')
_summary_obj = _summary_filt[_summary_filt["OBJECT"].str.match(
objname)]
_summary_obj = _summary_obj.reset_index(drop=True)
for i, row in _summary_obj.iterrows():
setid = 1 + i // 4
fpath = Path(row['file'])
ccd = load_ccd(fpath)
hdr = ccd.header
val_dark = (filt.upper(), hdr["EXPTIME"])
val_frin = (filt.upper(), hdr["POL-AGL1"],
hdr["OBJECT"] + "_sky")
if verbose >= 2:
print(fpath)
mdark, mdarkpath = _find_calframe(darks, darkpaths, val_dark,
"Dark", verbose >= 2)
mfringe, mfringepath = _find_calframe(fringes, fringepaths,
val_frin, "Fringe",
verbose >= 2)
suffix = ''
suffix += "D" if mdark is not None else ''
suffix += "F" if mflat is not None else ''
suffix += "Fr" if mfringe is not None else ''
if verbose >= 3:
print("\n", fpath)
nccd = bdf_process(ccd,
mflatpath=mflatpath,
mflat=mflat,
mdarkpath=mdarkpath,
mdark=mdark,
mfringepath=mfringepath,
mfringe=mfringe,
verbose_bdf=verbose >= 3)
nccd.header["SETID"] = (
setid, "Pol mode set number of OBJECT on the night")
nccds = split_oe(nccd, verbose=verbose >= 3)
for oe, nccd_oe in zip('oe', nccds):
_save(nccd_oe, dir_out, fpath.stem + suffix + f"_{oe}")
if verbose >= 1:
print()
summ_reduced = _save_or_load_summary(dir_out,
path_summ_out,
keywords=USEFUL_KEYS +
["OERAY", "SETID"],
skip_if_exists=skip_if_exists,
verbose=verbose >= 2)
return summ_reduced
def _do_16bit_vertical(fpath,
dir_out,
skip_if_exists=True,
sigclip_kw=dict(sigma=2, maxiters=5),
fitting_sections=None,
method='median',
verbose=True,
show_progress=True):
''' Changes to 16-bit and correct the vertical pattern.
dir_out : path-like
The directory for the resulting FITS files to be saved, RELATIVE to ``self.dir_work``.
verbose : int
Larger number means it becomes more verbose::
* 0: print nothing
* 1: Only very essential things
* 2: + verbose for summary CSV file
* 3: + the HISTORY in each FITS file's header
'''
fpath = Path(fpath)
ccd_orig = load_ccd(fpath)
_t = Time.now()
# == Set output stem =================================================================================== #
outstem, _ = _set_fstem(ccd_orig.header)
outstem += "-PROC-v"
# == Skip if conditions meet =========================================================================== #
if skip_if_exists and (dir_out / f"{outstem}.fits").exists():
return
# == First, change the bit ============================================================================= #
ccd_nbit = ccd_orig.copy()
ccd_nbit = CCDData_astype(ccd_nbit, dtype='int16')
add_to_header(ccd_nbit.header,
'h',
verbose=verbose,
s="{:=^72s}".format(' Basic preprocessing start '))
if ccd_orig.dtype != ccd_nbit.dtype:
add_to_header(
ccd_nbit.header,
'h',
t_ref=_t,
verbose=verbose,
s=f"Changed dtype (BITPIX): {ccd_orig.dtype} to {ccd_nbit.dtype}")
# == Then check if identical =========================================================================== #
# It takes < ~20 ms on MBP 15" [2018, macOS 10.14.6, i7-8850H (2.6 GHz; 6-core), RAM 16 GB (2400MHz
# DDR4), Radeon Pro 560X (4GB)]
# ysBach 2020-05-15 16:06:08 (KST: GMT+09:00)
np.testing.assert_almost_equal(ccd_orig.data - ccd_nbit.data,
np.zeros(ccd_nbit.data.shape))
# == Set counter ======================================================================================= #
try:
counter = ccd_nbit.header["COUNTER"]
except KeyError:
try:
counter = fpath.stem.split('_')[1]
counter = counter.split('.')[0]
# e.g., hYYMMDD_dddd.object.pcr.fits
except IndexError: # e.g., test images (``h.fits```)
counter = 9999
ccd_nbit.header['COUNTER'] = (
counter, "Image counter of the day, 1-indexing; 9999=TEST")
# == Update warning-invoking parts ===================================================================== #
try:
ccd_nbit.header["MJD-STR"] = float(ccd_nbit.header["MJD-STR"])
ccd_nbit.header["MJD-END"] = float(ccd_nbit.header["MJD-END"])
except KeyError:
pass
# == vertical pattern subtraction ====================================================================== #
ccd_nbit_v = vertical_correct(ccd_nbit,
sigclip_kw=sigclip_kw,
fitting_sections=fitting_sections,
method=method,
dtype='int16',
return_pattern=False)
update_process(ccd_nbit_v.header,
"v",
additional_comment=dict(v="vertical pattern"))
_save(ccd_nbit_v, dir_out, outstem)
return ccd_nbit_v
def prepare(fpath,
outdir=Path('.'),
kw_vertical=dict(sigclip_kw=dict(sigma=2, maxiters=5),
fitting_sections=None,
method='median',
update_header=True,
verbose=False),
dir_vc=None,
dir_vcfs=None,
kw_fourier={
'med_sub_clip': [-5, 5],
'med_rat_clip': [0.5, 2],
'std_rat_clip': [-5, 5]
},
save_nonpol=False,
split=True,
verbose=False):
''' Rename the original NHAO NIC image and convert to certain dtype.
Note
----
Original NHAO NIC image is in 32-bit integer, and is twice the size it should be. To save the
storage, it is desirable to convert those to 16-bit. As the bias is not added to the FITS frame
from NHAO NIC, the pixel value in the raw FITS file can be negative. Fortunately, however, the
maximum pixel value when saturation occurs is only about 20k, and therefore using ``int16`` rather
than ``uint16`` is enough.
Here, not only reducing the size, the file names are updated using the original file name and
header information:
``<FILTER (j, h, k)><System YYMMDD>_<COUNTER:04d>.fits``
It is then updated to
``<FILTER (j, h, k)>_<System YYYYMMDD>_<COUNTER:04d>_<OBJECT>_<EXPTIME:.1f>_<POL-AGL1:04.1f>
_<INSROT:+04.0f>_<IMGROT:+04.0f>_<PA:+06.1f>.fits``
'''
ccd_orig = load_ccd(fpath)
_t = Time.now()
ccd_nbit = ccd_orig.copy()
ccd_nbit = CCDData_astype(ccd_nbit, dtype='int16')
add_to_header(ccd_nbit.header,
'h',
verbose=verbose,
s="{:=^72s}".format(' Basic preprocessing start '))
if ccd_orig.dtype != ccd_nbit.dtype:
add_to_header(
ccd_nbit.header,
'h',
t_ref=_t,
verbose=verbose,
s=f"Changed dtype (BITPIX): {ccd_orig.dtype} to {ccd_nbit.dtype}")
# == First, check if identical ===================================================================== #
# It takes < ~20 ms on MBP 15" [2018, macOS 10.14.6, i7-8850H (2.6 GHz; 6-core), RAM 16 GB (2400MHz
# DDR4), Radeon Pro 560X (4GB)]
# ysBach 2020-05-15 16:06:08 (KST: GMT+09:00)
np.testing.assert_almost_equal(ccd_orig.data - ccd_nbit.data,
np.zeros(ccd_nbit.data.shape))
# == Set counter =================================================================================== #
try:
counter = ccd_nbit.header["COUNTER"]
except KeyError:
try:
counter = fpath.stem.split('_')[1]
counter = counter.split('.')[0]
# e.g., hYYMMDD_dddd.object.pcr.fits
except IndexError: # e.g., test images (``h.fits```)
counter = 9999
ccd_nbit.header['COUNTER'] = (
counter, "Image counter of the day, 1-indexing; 9999=TEST")
# == Update warning-invoking parts ================================================================= #
try:
ccd_nbit.header["MJD-STR"] = float(ccd_nbit.header["MJD-STR"])
ccd_nbit.header["MJD-END"] = float(ccd_nbit.header["MJD-END"])
except KeyError:
pass
# == Set output stem =============================================================================== #
outstem, polmode = _set_fstem(ccd_nbit.header)
# == vertical pattern subtraction ================================================================== #
ccd_nbit_vc = vertical_correct(ccd_nbit,
**kw_vertical,
dtype='int16',
return_pattern=False)
if dir_vc is not None:
_save(ccd_nbit_vc, dir_vc, outstem + "_vc")
if polmode:
# == Do Fourier pattern subtraction ============================================================ #
ccd_nbit_vcfs = fourier_lrsub(ccd_nbit_vc,
cut_wavelength=200,
**kw_fourier)
ccd_nbit_vcfs = CCDData_astype(ccd_nbit_vcfs, dtype='int16')
if dir_vcfs is not None:
_save(ccd_nbit_vcfs, dir_vcfs, outstem + "_vc_fs")
if split:
ccds = split_oe(ccd_nbit_vcfs, verbose=verbose)
ccd_out = []
for _ccd, oe in zip(ccds, 'oe'):
_save(_ccd, outdir, outstem + f"_{oe:s}")
ccd_out.append(_ccd)
else:
ccd_out = ccd_nbit_vcfs
_save(ccd_out, outdir, outstem)
else:
ccd_out = ccd_nbit_vc
if save_nonpol:
_save(ccd_out, outdir, outstem)
return ccd_out
def reorganize_fits(fpath,
outdir=Path('.'),
dtype='int16',
fitting_sections=None,
method='median',
sigclip_kw=dict(sigma=2, maxiters=5),
med_sub_clip=[-5, 5],
sub_lr=True,
dark_medfilt_bpm_kw=dict(med_rat_clip=None,
std_rat_clip=None,
std_model='std',
logical='and',
sigclip_kw=dict(sigma=2,
maxiters=5,
std_ddof=1)),
update_header=True,
save_nonpol=False,
verbose_bpm=False,
split=True,
verbose=False):
''' Rename the original NHAO NIC image and convert to certain dtype.
Parameters
----------
fpath : path-like
Path to the original image FITS file.
outdir : None, path-like
The top directory for the new file to be saved. If `None`,
nothing will be saved.
Note
----
Original NHAO NIC image is in 32-bit integer, and is twice the size it should be. To save the
storage, it is desirable to convert those to 16-bit. As the bias is not added to the FITS frame
from NHAO NIC, the pixel value in the raw FITS file can be negative. Fortunately, however, the
maximum pixel value when saturation occurs is only about 20k, and therefore using ``int16`` rather
than ``uint16`` is enough.
Here, not only reducing the size, the file names are updated using the original file name and
header information:
``<FILTER (j, h, k)><System YYMMDD>_<COUNTER:04d>.fits``
It is then updated to
``<FILTER (j, h, k)>_<System YYYYMMDD>_<COUNTER:04d>_<OBJECT>_<EXPTIME:.1f>_<POL-AGL1:04.1f>
_<INSROT:+04.0f>_<IMGROT:+04.0f>_<PA:+06.1f>.fits``
'''
def _sub_lr(part_l, part_r, filt):
# part_l = cr_reject_nic(part_l, crrej_kw=crrej_kw, verbose=verbose_crrej, add_process=False)
add_to_header(
part_l.header,
'h',
verbose=verbose_bpm,
s=
("Median filter badpixel masking (MBPM) algorithm started running on the left half "
+
'to remove hot pixels on left; a prerequisite for the right frame - left frame" '
+
f"technique to remove wavy pattern. Using med_sub_clip = {med_sub_clip} "
+ f"and {dark_medfilt_bpm_kw}"))
# To keep the header log of medfilt_bpm, I need this:
_tmp = part_r.data
part_r.data = part_l.data
part_r = medfilt_bpm(part_r,
med_sub_clip=med_sub_clip,
**dark_medfilt_bpm_kw)
_t = Time.now()
part_r.data = (_tmp - part_r.data).astype(dtype)
add_to_header(
part_r.header,
'h',
t_ref=_t,
verbose=verbose_bpm,
s=("Left part (vertical subtracted and MBPMed) is subtracted from the right part "
+ "(only vertical subtracted)"))
add_to_header(part_r.header,
'h',
verbose=verbose_bpm,
s="{:-^72s}".format(' DONE '),
fmt=None)
return part_r
fpath = Path(fpath)
ccd_orig = load_ccd(fpath)
_t = Time.now()
ccd_nbit = ccd_orig.copy()
ccd_nbit = CCDData_astype(ccd_nbit, dtype=dtype)
add_to_header(ccd_nbit.header,
'h',
verbose=verbose,
s="{:=^72s}".format(' Basic preprocessing start '))
if ccd_orig.dtype != ccd_nbit.dtype:
add_to_header(
ccd_nbit.header,
'h',
t_ref=_t,
verbose=verbose,
s=f"Changed dtype (BITPIX): {ccd_orig.dtype} to {ccd_nbit.dtype}")
# == First, check if identical ========================================================================= #
# It takes < ~20 ms on MBP 15" [2018, macOS 10.14.6, i7-8850H (2.6 GHz; 6-core), RAM 16 GB (2400MHz
# DDR4), Radeon Pro 560X (4GB)]
# ysBach 2020-05-15 16:06:08 (KST: GMT+09:00)
np.testing.assert_almost_equal(ccd_orig.data - ccd_nbit.data,
np.zeros(ccd_nbit.data.shape))
# == Set counter ======================================================================================= #
try:
counter = ccd_nbit.header["COUNTER"]
except KeyError:
try:
counter = fpath.stem.split('_')[1]
counter = counter.split('.')[0]
# e.g., hYYMMDD_dddd.object.pcr.fits
except IndexError: # e.g., test images (``h.fits```)
counter = 9999
ccd_nbit.header['COUNTER'] = (
counter, "Image counter of the day, 1-indexing; 9999=TEST")
# == Set output stem =================================================================================== #
hdr = ccd_nbit.header
outstem, polmode = _set_fstem(hdr)
# == Update warning-invoking parts ===================================================================== #
try:
ccd_nbit.header["MJD-STR"] = float(hdr["MJD-STR"])
ccd_nbit.header["MJD-END"] = float(hdr["MJD-END"])
except KeyError:
pass
# == Simple process to remove artifacts... ============================================================= #
ccd_nbit = vertical_correct(ccd_nbit,
fitting_sections=fitting_sections,
method=method,
sigclip_kw=sigclip_kw,
dtype='float32',
return_pattern=False,
update_header=update_header,
verbose=verbose)
if polmode:
filt = infer_filter(ccd_nbit, filt=None, verbose=verbose)
if split:
ccds_l = split_oe(ccd_nbit,
filt=filt,
right_half=True,
verbose=verbose)
ccds_r = split_oe(ccd_nbit,
filt=filt,
right_half=False,
verbose=verbose)
ccd_nbit = []
for i, oe in enumerate(['o', 'e']):
if sub_lr:
part_l = ccds_l[i]
part_r = ccds_r[i]
_ccd_nbit = _sub_lr(part_l, part_r, filt)
else:
_ccd_nbit = CCDData_astype(ccds_r[i], dtype=dtype)
_save(_ccd_nbit, outdir, outstem + f"_{oe:s}")
ccd_nbit.append(_ccd_nbit)
else:
# left half and right half
part_l = trim_ccd(ccd_nbit, NICSECTS["left"], verbose=verbose)
part_r = trim_ccd(ccd_nbit, NICSECTS["right"], verbose=verbose)
if sub_lr:
part_l = ccds_l[i]
part_r = ccds_r[i]
_ccd_nbit = _sub_lr(part_l, part_r, filt)
else:
_ccd_nbit = CCDData_astype(ccds_r[i], dtype=dtype)
ccd_nbit = part_r.copy()
_save(ccd_nbit, outdir, outstem)
else:
if save_nonpol:
_save(ccd_nbit, outdir, outstem)
return ccd_nbit
import ysfitsutilpy as yfu
from pathlib import Path
top = Path("2019-10-24")
ngc2639paths = top.glob("NGC2639*.fits")
preprocdir = top/"Preprocess"
mbiaspath = preprocdir/"bias.fits"
mflatpath = preprocdir/"flat.fits"
outdir = Path("test")
#%%
for fpath in ngc2639paths:
ccd_xxx = yfu.load_ccd(fpath)
exptime = ccd_xxx.header["EXPTIME"]
ccd_bdx = yfu.bdf_process(ccd_xxx,
mbiaspath=mbiaspath,
mdarkpath=preprocdir/f"dark{exptime:.0f}s.fits",
output=outdir/f"{fpath.stem}_bdx.fits")
ccd_bdf = yfu.bdf_process(ccd_xxx,
mbiaspath=mbiaspath,
mdarkpath=preprocdir/f"dark{exptime:.0f}s.fits",
mflatpath=mflatpath,
output=outdir/f"{fpath.stem}_bdf.fits")
def make_flat(self,
savedir=None,
do_bias=True,
do_dark=True,
mbiaspath=None,
mdarkpath=None,
comb_kwargs=MEDCOMB_KEYS,
delimiter='-',
dtype='float32'):
'''Makes and saves flat images.
Parameters
----------
savedir: path-like, optional
The directory where the frames will be saved.
do_bias, do_dark : bool, optional
If ``True``, subtracts bias and dark frames using
``self.biaspahts`` and ``self.darkpaths``. You can also
specify ``mbiaspath`` and/or ``mdarkpath`` to ignore those
in ``self.``.
mbiaspath, mdarkpath : None, path-like, optional
If you want to force a certain bias or dark to be used, then
you can specify its path here.
comb_kwargs: dict or None, optional
The parameters for ``combine_ccd``.
delimiter : str, optional.
The delimiter for the renaming.
dtype : str or numpy.dtype object, optional.
The data type you want for the final master bias frame. It
is recommended to use ``float32`` or ``int16`` if there is
no specific reason.
'''
# Initial settings
self.initialize_self()
if savedir is None:
savedir = self.topdir
yfu.mkdir(savedir)
flatpaths = {}
# For simplicity, crop the original data by type_key and
# type_val first.
st = self.summary_raw.copy()
for k, v in zip(self.flat_type_key, self.flat_type_val):
st = st[st[k] == v]
# For grouping, use type_key + group_key. This is because (1) it
# is not harmful cuz type_key will have unique column values as
# ``st`` has already been cropped in above for loop (2) by doing
# this we get more information from combining process because,
# e.g., "images with ["OBJECT", "EXPTIME"] = ["dark", 1.0] are
# loaded" will be printed rather than just "images with
# ["EXPTIME"] = [1.0] are loaded".
gs = st.groupby(self.flat_key)
# Do flat combine:
for flat_val, flat_group in gs:
# set path to master bias
if mbiaspath is not None:
biaspath = mbiaspath
elif do_bias:
# corresponding key for biaspaths:
corr_bias = tuple(self.bias_type_val)
# if _group_key not empty, add appropriate ``group_val``:
if self.bias_group_key:
corr_bias += tuple(flat_group[self.bias_group_key].iloc[0])
# else: empty. path is fully specified by _type_val.
biaspath = self.biaspaths[corr_bias]
try:
biaspath = self.biaspaths[corr_bias]
except KeyError:
biaspath = None
warn(f"Bias not available for {corr_bias}. " +
"Processing without bias.")
# set path to master dark
if mdarkpath is not None:
darkpath = mdarkpath
elif do_dark:
# corresponding key for darkpaths:
corr_dark = tuple(self.dark_type_val)
# if _group_key not empty, add appropriate ``group_val``:
if self.dark_group_key:
corr_dark += tuple(flat_group[self.dark_group_key].iloc[0])
# else: empty. path is fully specified by _type_val.
try:
darkpath = self.darkpaths[corr_dark]
except (KeyError):
darkpath = None
warn(f"Dark not available for {corr_dark}. " +
"Processing without dark.")
# Do BD preproc before combine
flat_bd_paths = []
for i, flat_row in flat_group.iterrows():
flat_orig_path = Path(flat_row["file"])
flat_bd_path = (flat_orig_path.parent /
(flat_orig_path.stem + "_BD.fits"))
ccd = yfu.load_ccd(flat_orig_path, unit='adu')
_ = yfu.bdf_process(ccd,
output=flat_bd_path,
mbiaspath=biaspath,
mdarkpath=darkpath,
dtype="int16",
overwrite=True,
unit=None)
flat_bd_paths.append(flat_bd_path)
if not isinstance(flat_val, tuple):
flat_val = tuple([flat_val])
fname = delimiter.join([str(x) for x in flat_val]) + ".fits"
fpath = Path(savedir) / fname
_ = yfu.combine_ccd(
flat_bd_paths,
output=fpath,
dtype=dtype,
**comb_kwargs,
normalize_average=True, # Since skyflat!!
type_key=self.flat_key,
type_val=flat_val)
flatpaths[tuple(flat_val)] = fpath
# Save list of file paths for future use.
# It doesn't take much storage and easy to erase if you want.
with open(self.listdir / 'flatpaths.list', 'w+') as ll:
for p in list(flatpaths.values()):
ll.write(f"{str(p)}\n")
with open(self.listdir / 'flatpaths.pkl', 'wb') as pkl:
pickle.dump(flatpaths, pkl)
self.flatpaths = flatpaths
def sub_bkg(fpath):
ccd = yfu.load_ccd(fpath)
bkg = ypu.sep_back(ccd, box_size=256, filter_size=7)
return ccd.data - bkg.back(), bkg.back()
fmt="mdark_{:03.0f}s",
**comb_kw)
_mdarkpaths = list(Path(".").glob("mdark*.fits"))
for _mdarkpath in _mdarkpaths:
yfu.bdf_process(_mdarkpath,
mbiaspath="mbias.fits",
output=f"b_{_mdarkpath.name}")
# ********************************************************************************************************** #
# * MAKE MASTER FLAT * #
# ********************************************************************************************************** #
_flatpaths = list(Path(".").glob("skyflat*.fit"))
# -- First, save after bias and dark subtraciton
for _flatpath in _flatpaths:
exptime = yfu.load_ccd(_flatpath).header["EXPTIME"]
mdarkpath = Path(f"b_mdark_{exptime:03.0f}s.fits")
# Bias and dark subtract
output = Path(f"bd_{_flatpath.stem}.fits")
if not output.exists():
_ = yfu.bdf_process(_flatpath,
mbiaspath="mbias.fits",
mdarkpath=mdarkpath,
output=output)
# -- Second, combine after normalization
# If no normalization is done, the output will be severely affected by the brightest flat frame since
# they are "sky" flats during the sunset.
_ = yfu.group_combine(summary,
type_key=["IMAGETYP"],
type_val=["Flat Field"],