import ysfitsutilpy as yfu
from pathlib import Path
top = Path("2019-10-24")
ngc2639paths = top.glob("NGC2639*.fits")
preprocdir = top/"Preprocess"
mbiaspath = preprocdir/"bias.fits"
mflatpath = preprocdir/"flat.fits"
outdir = Path("test")
#%%
for fpath in ngc2639paths:
ccd_xxx = yfu.load_ccd(fpath)
exptime = ccd_xxx.header["EXPTIME"]
ccd_bdx = yfu.bdf_process(ccd_xxx,
mbiaspath=mbiaspath,
mdarkpath=preprocdir/f"dark{exptime:.0f}s.fits",
output=outdir/f"{fpath.stem}_bdx.fits")
ccd_bdf = yfu.bdf_process(ccd_xxx,
mbiaspath=mbiaspath,
mdarkpath=preprocdir/f"dark{exptime:.0f}s.fits",
mflatpath=mflatpath,
output=outdir/f"{fpath.stem}_bdf.fits")
def do_preproc(self,
savedir=None,
delimiter='-',
dtype='float32',
mbiaspath=None,
mdarkpath=None,
mflatpath=None,
do_bias=True,
do_dark=True,
do_flat=True,
do_crrej=False,
crrej_kwargs=None,
verbose_crrej=False,
verbose_bdf=True,
verbose_summary=False):
''' Conduct the preprocessing using simplified ``bdf_process``.
Parameters
----------
savedir: path-like, optional
The directory where the frames will be saved.
delimiter : str, optional.
The delimiter for the renaming.
dtype : str or numpy.dtype object, optional.
The data type you want for the final master bias frame. It is
recommended to use ``float32`` or ``int16`` if there is no
specific reason.
mbiaspath, mdarkpath, mflatpath : None, path-like, optional
If you want to force a certain bias, dark, or flat to be used,
then you can specify its path here.
crrej_kwargs : dict or None, optional
If ``None`` (default), uses some default values defined in
``~.misc.LACOSMIC_KEYS``. It is always discouraged to use
default except for quick validity-checking, because even the
official L.A. Cosmic codes in different versions (IRAF, IDL,
Python, etc) have different default parameters, i.e., there is
nothing which can be regarded as default.
To see all possible keywords, do
``print(astroscrappy.detect_cosmics.__doc__)``
Also refer to
https://nbviewer.jupyter.org/github/ysbach/AO2019/blob/master/Notebooks/07-Cosmic_Ray_Rejection.ipynb
'''
# Initial settings
self.initialize_self()
if savedir is None:
savedir = self.topdir
savedir = Path(savedir)
yfu.mkdir(savedir)
savepaths = []
# for i, row in self.summary_raw.iterrows():
# fpath = Path(row["file"])
for fpath in self.objpaths:
savepath = savedir / Path(fpath).name
savepaths.append(savepath)
row = self.summary_raw[self.summary_raw["file"].values == str(
fpath)]
if mbiaspath is not None:
biaspath = mbiaspath
elif do_bias:
# corresponding key for biaspaths:
corr_bias = tuple(self.bias_type_val)
# if _group_key not empty, add appropriate ``group_val``:
if self.bias_group_key: # not empty
corr_bias += tuple(row[self.bias_group_key].iloc[0])
# else: empty. path is fully specified by _type_val.
try:
biaspath = self.biaspaths[corr_bias]
except (KeyError):
biaspath = None
warn(f"Bias not available for {corr_bias}. " +
"Processing without bias.")
if mdarkpath is not None:
darkpath = mdarkpath
elif do_dark:
# corresponding key for darkpaths:
corr_dark = tuple(self.dark_type_val)
# if _group_key not empty, add appropriate ``group_val``:
if self.dark_group_key:
corr_dark += tuple(row[self.dark_group_key].iloc[0])
# else: empty. path is fully specified by _type_val.
try:
darkpath = self.darkpaths[corr_dark]
except (KeyError):
darkpath = None
warn(f"Dark not available for {corr_dark}. " +
"Processing without dark.")
if mflatpath is not None:
flatpath = mflatpath
elif do_flat:
# corresponding key for darkpaths:
corr_flat = tuple(self.flat_type_val)
# if _group_key not empty, add appropriate ``group_val``:
if self.flat_group_key:
corr_flat += tuple(row[self.flat_group_key].iloc[0])
# else: empty. path is fully specified by _type_val.
try:
flatpath = self.flatpaths[corr_flat]
except (KeyError):
flatpath = None
warn(f"Flat not available for {corr_flat}. " +
"Processing without flat.")
objccd = CCDData.read(fpath)
_ = yfu.bdf_process(
objccd, #
output=savepath, #
unit=None, #
mbiaspath=biaspath, #
mdarkpath=darkpath, #
mflatpath=flatpath, #
do_crrej=do_crrej, #
crrej_kwargs=crrej_kwargs, #
verbose_crrej=verbose_crrej, #
verbose_bdf=verbose_bdf)
self.reducedpaths = savepaths
self.summary_red = yfu.make_summary(
self.reducedpaths,
output=self.topdir / "summary_reduced.csv",
pandas=True,
keywords=self.summary_keywords + ["PROCESS"],
verbose=verbose_summary)
return self.summary_red
def make_dark(self,
savedir=None,
do_bias=True,
mbiaspath=None,
dtype='float32',
delimiter='-',
comb_kwargs=MEDCOMB_KEYS):
""" Makes and saves dark (bias subtracted) images.
Parameters
----------
savedir: path-like, optional
The directory where the frames will be saved.
do_bias : bool, optional
If ``True``, subtracts bias from dark frames using
self.biaspahts. You can also specify ``mbiaspath`` to ignore
that in ``self.``.
mbiaspath : None, path-like, optional
If you want to force a certain bias to be used, then you can
specify its path here.
delimiter : str, optional.
The delimiter for the renaming.
dtype : str or numpy.dtype object, optional.
The data type you want for the final master bias frame. It
is recommended to use ``float32`` or ``int16`` if there is
no specific reason.
comb_kwargs : dict or None, optional
The parameters for ``combine_ccd``.
"""
# Initial settings
self.initialize_self()
if savedir is None:
savedir = self.topdir
yfu.mkdir(Path(savedir))
darkpaths = {}
# For simplicity, crop the original data by type_key and
# type_val first.
st = self.summary_raw.copy()
for k, v in zip(self.dark_type_key, self.dark_type_val):
st = st[st[k] == v]
# For grouping, use _key (i.e., type_key + group_key). This is
# because (1) it is not harmful cuz type_key will have unique
# column values as ``st`` has already been cropped in above for
# loop (2) by doing this we get more information from combining
# process because, e.g., "images with ["OBJECT", "EXPTIME"] =
# ["dark", 1.0] are loaded" will be printed rather than just
# "images with ["EXPTIME"] = [1.0] are loaded".
gs = st.groupby(self.dark_key)
# Do dark combine:
for dark_val, dark_group in gs:
if not isinstance(dark_val, tuple):
dark_val = tuple([dark_val])
fname = delimiter.join([str(x) for x in dark_val]) + ".fits"
fpath = Path(savedir) / fname
mdark = yfu.combine_ccd(dark_group["file"].tolist(),
output=None,
dtype=dtype,
**comb_kwargs,
type_key=self.dark_key,
type_val=dark_val)
# set path to master bias
if mbiaspath is not None:
biaspath = mbiaspath
elif do_bias:
# corresponding key for biaspaths:
corr_bias = tuple(self.bias_type_val)
# if _group_key not empty, add appropriate ``group_val``:
if self.bias_group_key:
corr_bias += tuple(dark_group[self.bias_group_key].iloc[0])
# else: empty. path is fully specified by _type_val.
try:
biaspath = self.biaspaths[corr_bias]
except KeyError:
biaspath = None
warn(f"Bias not available for {corr_bias}. " +
"Processing without bias.")
mdark = yfu.bdf_process(mdark,
mbiaspath=biaspath,
dtype=dtype,
unit=None)
mdark.write(fpath, output_verify='fix', overwrite=True)
darkpaths[tuple(dark_val)] = fpath
# Save list of file paths for future use.
# It doesn't take much storage and easy to erase if you want.
with open(self.listdir / 'darkpaths.list', 'w+') as ll:
for p in list(darkpaths.values()):
ll.write(f"{str(p)}\n")
with open(self.listdir / 'darkpaths.pkl', 'wb') as pkl:
pickle.dump(darkpaths, pkl)
self.darkpaths = darkpaths
def make_flat(self,
savedir=None,
do_bias=True,
do_dark=True,
mbiaspath=None,
mdarkpath=None,
comb_kwargs=MEDCOMB_KEYS,
delimiter='-',
dtype='float32'):
'''Makes and saves flat images.
Parameters
----------
savedir: path-like, optional
The directory where the frames will be saved.
do_bias, do_dark : bool, optional
If ``True``, subtracts bias and dark frames using
``self.biaspahts`` and ``self.darkpaths``. You can also
specify ``mbiaspath`` and/or ``mdarkpath`` to ignore those
in ``self.``.
mbiaspath, mdarkpath : None, path-like, optional
If you want to force a certain bias or dark to be used, then
you can specify its path here.
comb_kwargs: dict or None, optional
The parameters for ``combine_ccd``.
delimiter : str, optional.
The delimiter for the renaming.
dtype : str or numpy.dtype object, optional.
The data type you want for the final master bias frame. It
is recommended to use ``float32`` or ``int16`` if there is
no specific reason.
'''
# Initial settings
self.initialize_self()
if savedir is None:
savedir = self.topdir
yfu.mkdir(savedir)
flatpaths = {}
# For simplicity, crop the original data by type_key and
# type_val first.
st = self.summary_raw.copy()
for k, v in zip(self.flat_type_key, self.flat_type_val):
st = st[st[k] == v]
# For grouping, use type_key + group_key. This is because (1) it
# is not harmful cuz type_key will have unique column values as
# ``st`` has already been cropped in above for loop (2) by doing
# this we get more information from combining process because,
# e.g., "images with ["OBJECT", "EXPTIME"] = ["dark", 1.0] are
# loaded" will be printed rather than just "images with
# ["EXPTIME"] = [1.0] are loaded".
gs = st.groupby(self.flat_key)
# Do flat combine:
for flat_val, flat_group in gs:
# set path to master bias
if mbiaspath is not None:
biaspath = mbiaspath
elif do_bias:
# corresponding key for biaspaths:
corr_bias = tuple(self.bias_type_val)
# if _group_key not empty, add appropriate ``group_val``:
if self.bias_group_key:
corr_bias += tuple(flat_group[self.bias_group_key].iloc[0])
# else: empty. path is fully specified by _type_val.
biaspath = self.biaspaths[corr_bias]
try:
biaspath = self.biaspaths[corr_bias]
except KeyError:
biaspath = None
warn(f"Bias not available for {corr_bias}. " +
"Processing without bias.")
# set path to master dark
if mdarkpath is not None:
darkpath = mdarkpath
elif do_dark:
# corresponding key for darkpaths:
corr_dark = tuple(self.dark_type_val)
# if _group_key not empty, add appropriate ``group_val``:
if self.dark_group_key:
corr_dark += tuple(flat_group[self.dark_group_key].iloc[0])
# else: empty. path is fully specified by _type_val.
try:
darkpath = self.darkpaths[corr_dark]
except (KeyError):
darkpath = None
warn(f"Dark not available for {corr_dark}. " +
"Processing without dark.")
# Do BD preproc before combine
flat_bd_paths = []
for i, flat_row in flat_group.iterrows():
flat_orig_path = Path(flat_row["file"])
flat_bd_path = (flat_orig_path.parent /
(flat_orig_path.stem + "_BD.fits"))
ccd = yfu.load_ccd(flat_orig_path, unit='adu')
_ = yfu.bdf_process(ccd,
output=flat_bd_path,
mbiaspath=biaspath,
mdarkpath=darkpath,
dtype="int16",
overwrite=True,
unit=None)
flat_bd_paths.append(flat_bd_path)
if not isinstance(flat_val, tuple):
flat_val = tuple([flat_val])
fname = delimiter.join([str(x) for x in flat_val]) + ".fits"
fpath = Path(savedir) / fname
_ = yfu.combine_ccd(
flat_bd_paths,
output=fpath,
dtype=dtype,
**comb_kwargs,
normalize_average=True, # Since skyflat!!
type_key=self.flat_key,
type_val=flat_val)
flatpaths[tuple(flat_val)] = fpath
# Save list of file paths for future use.
# It doesn't take much storage and easy to erase if you want.
with open(self.listdir / 'flatpaths.list', 'w+') as ll:
for p in list(flatpaths.values()):
ll.write(f"{str(p)}\n")
with open(self.listdir / 'flatpaths.pkl', 'wb') as pkl:
pickle.dump(flatpaths, pkl)
self.flatpaths = flatpaths
_ = yfu.imcombine("calibration*bias.fit", output="mbias.fits", **comb_kw)
# ********************************************************************************************************** #
# * MAKE MASTER DARK * #
# ********************************************************************************************************** #
_ = yfu.group_combine(summary,
type_key=["IMAGETYP"],
type_val=["Dark Frame"],
group_key=["EXPTIME"],
fmt="mdark_{:03.0f}s",
**comb_kw)
_mdarkpaths = list(Path(".").glob("mdark*.fits"))
for _mdarkpath in _mdarkpaths:
yfu.bdf_process(_mdarkpath,
mbiaspath="mbias.fits",
output=f"b_{_mdarkpath.name}")
# ********************************************************************************************************** #
# * MAKE MASTER FLAT * #
# ********************************************************************************************************** #
_flatpaths = list(Path(".").glob("skyflat*.fit"))
# -- First, save after bias and dark subtraciton
for _flatpath in _flatpaths:
exptime = yfu.load_ccd(_flatpath).header["EXPTIME"]
mdarkpath = Path(f"b_mdark_{exptime:03.0f}s.fits")
# Bias and dark subtract
output = Path(f"bd_{_flatpath.stem}.fits")
if not output.exists():
_ = yfu.bdf_process(_flatpath,
mbiaspath="mbias.fits",