def lrsubtract(ccd,
fitting_sections=["[:, 50:100]", "[:, 924:974]"],
method='median',
sigclip_kw=dict(sigma=2, maxiters=5),
dtype='float32',
update_header=True,
verbose=False):
"""Subtract left from right quadrants w/ vertical pattern removal."""
_t = Time.now()
nccd = vertical_correct(ccd=ccd,
fitting_sections=fitting_sections,
method=method,
sigclip_kw=sigclip_kw,
dtype=dtype,
return_pattern=False,
update_header=update_header)
nx = nccd.data.shape[1]
i_half = nx // 2
nccd.data[:, i_half:] -= nccd.data[:, :i_half]
if update_header:
# add as history
add_to_header(
nccd.header,
'h',
t_ref=_t,
verbose=verbose,
s=f"Subtracted left half ({i_half} columns) from the right half.")
# nccd = trim_image(nccd, fits_section=f"[{i_half + 1}:, :]")
return nccd
def fourier_lrsub(ccd,
cut_wavelength=200,
copy=True,
verbose_bpm=False,
verbose=False,
**kwargs):
if copy:
_ccd = ccd.copy()
else:
_ccd = ccd
ccd_l = trim_ccd(_ccd, fits_section="[:512, :]", update_header=False)
ccd_l = medfilt_bpm(ccd_l, verbose=verbose_bpm, **kwargs)
ccd.header = ccd_l.header # to add MBPM logs
amp_comp = np.fft.rfft(ccd_l.data, axis=0)
amp_comp[cut_wavelength:, :] = 0
pattern_pure = np.fft.irfft(amp_comp, axis=0)
pattern = np.tile(pattern_pure, 2)
_ccd.data = _ccd.data - pattern
add_to_header(
_ccd.header,
'h',
verbose=verbose,
s=("FFT(left half) to get pattern map (see FFTCUTWL for the cut wavelength); "
+ "subtracted from both left/right."))
_ccd.header["FFTCUTWL"] = (cut_wavelength,
"FFT cut wavelength (amplitude[this:, :] = 0)")
return _ccd
def _sub_lr(part_l, part_r, filt):
# part_l = cr_reject_nic(part_l, crrej_kw=crrej_kw, verbose=verbose_crrej, add_process=False)
add_to_header(
part_l.header,
'h',
verbose=verbose_bpm,
s=
("Median filter badpixel masking (MBPM) algorithm started running on the left half "
+
'to remove hot pixels on left; a prerequisite for the right frame - left frame" '
+
f"technique to remove wavy pattern. Using med_sub_clip = {med_sub_clip} "
+ f"and {dark_medfilt_bpm_kw}"))
# To keep the header log of medfilt_bpm, I need this:
_tmp = part_r.data
part_r.data = part_l.data
part_r = medfilt_bpm(part_r,
med_sub_clip=med_sub_clip,
**dark_medfilt_bpm_kw)
_t = Time.now()
part_r.data = (_tmp - part_r.data).astype(dtype)
add_to_header(
part_r.header,
'h',
t_ref=_t,
verbose=verbose_bpm,
s=("Left part (vertical subtracted and MBPMed) is subtracted from the right part "
+ "(only vertical subtracted)"))
add_to_header(part_r.header,
'h',
verbose=verbose_bpm,
s="{:-^72s}".format(' DONE '),
fmt=None)
return part_r
def make_dark(dir_in,
dir_out,
dir_log=None,
dark_object="DARK",
combine='med',
reject='sc',
sigma=[3., 3.],
dark_min=3,
skip_if_exists=True,
verbose=0,
**kwargs):
'''
verbose : int
Larger number means it becomes more verbose::
* 0: print nothing
* 1: Only very essential things
* 2: + progress-like info (printing filter and exptime)
* 3: + fits stack info (grouping) + header update
* 4: + imcombine verbose
'''
dir_in, dir_out, dir_log = _set_dir_iol(dir_in, dir_out, dir_log)
try:
_summary = pd.read_csv(dir_log / f"summary_{dir_in.name}.csv")
except FileNotFoundError:
raise FileNotFoundError(
f"Summary for {dir_in} is not found in {dir_log}." +
" Try changing dir_in or dir_log.")
if skip_if_exists:
if verbose >= 1:
print("Loading existing dark frames...")
_, darks, darkpaths = _load_as_dict(dir_out, ["FILTER", "EXPTIME"],
verbose=verbose >= 2)
if darks is not None: # if some darks already exist
return darks, darkpaths
# else, run the code below
if verbose >= 1:
print(
"Making master dark of the night (grouping & combining dark frames)."
)
darks = group_combine(_summary,
type_key=["OBJECT"],
type_val=[dark_object],
group_key=["FILTER", "EXPTIME"],
combine=combine,
reject=reject,
sigma=sigma,
verbose=verbose - 1,
**kwargs)
darkpaths = {}
for k, dark_k in darks.items():
filt, exptime = k
darks[k].data[dark_k.data < dark_min] = 0
add_to_header(
darks[k].header,
'h',
verbose=verbose >= 3,
s=f"Pixels with value < {dark_min:.2f} in combined dark are replaced with 0."
)
fstem = f"{filt.lower()}_mdark_{exptime:.1f}"
darkpaths[k] = _save(darks[k], dir_out, fstem, return_path=True)
return darks, darkpaths
def _do_16bit_vertical(fpath,
dir_out,
skip_if_exists=True,
sigclip_kw=dict(sigma=2, maxiters=5),
fitting_sections=None,
method='median',
verbose=True,
show_progress=True):
''' Changes to 16-bit and correct the vertical pattern.
dir_out : path-like
The directory for the resulting FITS files to be saved, RELATIVE to ``self.dir_work``.
verbose : int
Larger number means it becomes more verbose::
* 0: print nothing
* 1: Only very essential things
* 2: + verbose for summary CSV file
* 3: + the HISTORY in each FITS file's header
'''
fpath = Path(fpath)
ccd_orig = load_ccd(fpath)
_t = Time.now()
# == Set output stem =================================================================================== #
outstem, _ = _set_fstem(ccd_orig.header)
outstem += "-PROC-v"
# == Skip if conditions meet =========================================================================== #
if skip_if_exists and (dir_out / f"{outstem}.fits").exists():
return
# == First, change the bit ============================================================================= #
ccd_nbit = ccd_orig.copy()
ccd_nbit = CCDData_astype(ccd_nbit, dtype='int16')
add_to_header(ccd_nbit.header,
'h',
verbose=verbose,
s="{:=^72s}".format(' Basic preprocessing start '))
if ccd_orig.dtype != ccd_nbit.dtype:
add_to_header(
ccd_nbit.header,
'h',
t_ref=_t,
verbose=verbose,
s=f"Changed dtype (BITPIX): {ccd_orig.dtype} to {ccd_nbit.dtype}")
# == Then check if identical =========================================================================== #
# It takes < ~20 ms on MBP 15" [2018, macOS 10.14.6, i7-8850H (2.6 GHz; 6-core), RAM 16 GB (2400MHz
# DDR4), Radeon Pro 560X (4GB)]
# ysBach 2020-05-15 16:06:08 (KST: GMT+09:00)
np.testing.assert_almost_equal(ccd_orig.data - ccd_nbit.data,
np.zeros(ccd_nbit.data.shape))
# == Set counter ======================================================================================= #
try:
counter = ccd_nbit.header["COUNTER"]
except KeyError:
try:
counter = fpath.stem.split('_')[1]
counter = counter.split('.')[0]
# e.g., hYYMMDD_dddd.object.pcr.fits
except IndexError: # e.g., test images (``h.fits```)
counter = 9999
ccd_nbit.header['COUNTER'] = (
counter, "Image counter of the day, 1-indexing; 9999=TEST")
# == Update warning-invoking parts ===================================================================== #
try:
ccd_nbit.header["MJD-STR"] = float(ccd_nbit.header["MJD-STR"])
ccd_nbit.header["MJD-END"] = float(ccd_nbit.header["MJD-END"])
except KeyError:
pass
# == vertical pattern subtraction ====================================================================== #
ccd_nbit_v = vertical_correct(ccd_nbit,
sigclip_kw=sigclip_kw,
fitting_sections=fitting_sections,
method=method,
dtype='int16',
return_pattern=False)
update_process(ccd_nbit_v.header,
"v",
additional_comment=dict(v="vertical pattern"))
_save(ccd_nbit_v, dir_out, outstem)
return ccd_nbit_v
def vertical_correct(ccd,
fitting_sections=None,
method='median',
sigclip_kw=dict(sigma=2, maxiters=5),
dtype='float32',
return_pattern=False,
update_header=True,
verbose=False):
''' Correct vertical strip patterns.
Paramters
---------
ccd : CCDData, HDU object, HDUList, or ndarray.
The CCD to subtract the vertical pattern.
fitting_sections : list of two str, optional.
The sections to be used for the vertical pattern estimation. This must be identical to the
usual FITS section (i.e., that used in SAO ds9 or IRAF, 1-indexing and last-index-inclusive),
not in python. **Give it in the order of ``[<upper>, <lower>]`` in FITS y-coordinate.**
method : str, optional.
One of ``['med', 'avg', 'median', 'average', 'mean']``.
sigma, maxiters : float and int, optional
A sigma-clipping will be done to remove hot pixels and cosmic rays for estimating the vertical
pattern. To turn sigma-clipping off, set ``maxiters=0``.
sigclip_kw : dict, optional
The keyword arguments for the sigma clipping.
dtype : str, dtype, optional
The data type to be returned.
return_pattern : bool, optional.
If `True`, the subtracted pattern will also be returned.
Default is `False`.
update_header : bool, optional.
Whether to update the header if there is any.
Return
------
'''
_t = Time.now()
data, hdr = _parse_data_header(ccd)
if fitting_sections is None:
fitting_sections = VERTICALSECTS
elif len(fitting_sections) != 2:
raise ValueError("fitting_sections must have two elements.")
if method in ['median', 'med']:
methodstr = 'taking median'
fun = np.median
idx = 1
elif method in ['average', 'avg', 'mean']:
methodstr = 'taking average'
fun = np.mean
idx = 0
else:
raise ValueError(
"method not understood; it must be one of [med, avg, median, average, mean]."
)
parts = [] # The two horizontal box areas from raw data
strips = [] # The estimated patterns (1-D)
for sect in fitting_sections:
parts.append(data[fitsxy2py(sect)])
try:
if sigclip_kw["maxiters"] == 0:
clipstr = "no clipping"
for part in parts:
strips.append(fun(part, axis=0))
except KeyError:
pass # The user wants to use default value of maxiters in astropy.
clipstr = f"sigma-clipping in astropy (v {astropy.__version__})"
if sigclip_kw:
clipstr += f", given {sigclip_kw}."
else:
clipstr += "."
for part in parts:
clip = sigma_clipped_stats(part, axis=0, **sigclip_kw)
strips.append(clip[idx])
ny, nx = data.shape
vpattern = np.repeat(strips, ny / 2, axis=0)
vsub = data - vpattern.astype(dtype)
if update_header and hdr is not None:
# add as history
add_to_header(
hdr,
'h',
verbose=verbose,
t_ref=_t,
s=f"Vertical pattern subtracted using {fitting_sections} by {methodstr} with {clipstr}"
)
try:
nccd = CCDData(data=vsub, header=hdr)
except ValueError:
nccd = CCDData(data=vsub, header=hdr, unit='adu')
nccd.data = nccd.data.astype(dtype)
if return_pattern:
return nccd, vpattern
return nccd
def prepare(fpath,
outdir=Path('.'),
kw_vertical=dict(sigclip_kw=dict(sigma=2, maxiters=5),
fitting_sections=None,
method='median',
update_header=True,
verbose=False),
dir_vc=None,
dir_vcfs=None,
kw_fourier={
'med_sub_clip': [-5, 5],
'med_rat_clip': [0.5, 2],
'std_rat_clip': [-5, 5]
},
save_nonpol=False,
split=True,
verbose=False):
''' Rename the original NHAO NIC image and convert to certain dtype.
Note
----
Original NHAO NIC image is in 32-bit integer, and is twice the size it should be. To save the
storage, it is desirable to convert those to 16-bit. As the bias is not added to the FITS frame
from NHAO NIC, the pixel value in the raw FITS file can be negative. Fortunately, however, the
maximum pixel value when saturation occurs is only about 20k, and therefore using ``int16`` rather
than ``uint16`` is enough.
Here, not only reducing the size, the file names are updated using the original file name and
header information:
``<FILTER (j, h, k)><System YYMMDD>_<COUNTER:04d>.fits``
It is then updated to
``<FILTER (j, h, k)>_<System YYYYMMDD>_<COUNTER:04d>_<OBJECT>_<EXPTIME:.1f>_<POL-AGL1:04.1f>
_<INSROT:+04.0f>_<IMGROT:+04.0f>_<PA:+06.1f>.fits``
'''
ccd_orig = load_ccd(fpath)
_t = Time.now()
ccd_nbit = ccd_orig.copy()
ccd_nbit = CCDData_astype(ccd_nbit, dtype='int16')
add_to_header(ccd_nbit.header,
'h',
verbose=verbose,
s="{:=^72s}".format(' Basic preprocessing start '))
if ccd_orig.dtype != ccd_nbit.dtype:
add_to_header(
ccd_nbit.header,
'h',
t_ref=_t,
verbose=verbose,
s=f"Changed dtype (BITPIX): {ccd_orig.dtype} to {ccd_nbit.dtype}")
# == First, check if identical ===================================================================== #
# It takes < ~20 ms on MBP 15" [2018, macOS 10.14.6, i7-8850H (2.6 GHz; 6-core), RAM 16 GB (2400MHz
# DDR4), Radeon Pro 560X (4GB)]
# ysBach 2020-05-15 16:06:08 (KST: GMT+09:00)
np.testing.assert_almost_equal(ccd_orig.data - ccd_nbit.data,
np.zeros(ccd_nbit.data.shape))
# == Set counter =================================================================================== #
try:
counter = ccd_nbit.header["COUNTER"]
except KeyError:
try:
counter = fpath.stem.split('_')[1]
counter = counter.split('.')[0]
# e.g., hYYMMDD_dddd.object.pcr.fits
except IndexError: # e.g., test images (``h.fits```)
counter = 9999
ccd_nbit.header['COUNTER'] = (
counter, "Image counter of the day, 1-indexing; 9999=TEST")
# == Update warning-invoking parts ================================================================= #
try:
ccd_nbit.header["MJD-STR"] = float(ccd_nbit.header["MJD-STR"])
ccd_nbit.header["MJD-END"] = float(ccd_nbit.header["MJD-END"])
except KeyError:
pass
# == Set output stem =============================================================================== #
outstem, polmode = _set_fstem(ccd_nbit.header)
# == vertical pattern subtraction ================================================================== #
ccd_nbit_vc = vertical_correct(ccd_nbit,
**kw_vertical,
dtype='int16',
return_pattern=False)
if dir_vc is not None:
_save(ccd_nbit_vc, dir_vc, outstem + "_vc")
if polmode:
# == Do Fourier pattern subtraction ============================================================ #
ccd_nbit_vcfs = fourier_lrsub(ccd_nbit_vc,
cut_wavelength=200,
**kw_fourier)
ccd_nbit_vcfs = CCDData_astype(ccd_nbit_vcfs, dtype='int16')
if dir_vcfs is not None:
_save(ccd_nbit_vcfs, dir_vcfs, outstem + "_vc_fs")
if split:
ccds = split_oe(ccd_nbit_vcfs, verbose=verbose)
ccd_out = []
for _ccd, oe in zip(ccds, 'oe'):
_save(_ccd, outdir, outstem + f"_{oe:s}")
ccd_out.append(_ccd)
else:
ccd_out = ccd_nbit_vcfs
_save(ccd_out, outdir, outstem)
else:
ccd_out = ccd_nbit_vc
if save_nonpol:
_save(ccd_out, outdir, outstem)
return ccd_out
def reorganize_fits(fpath,
outdir=Path('.'),
dtype='int16',
fitting_sections=None,
method='median',
sigclip_kw=dict(sigma=2, maxiters=5),
med_sub_clip=[-5, 5],
sub_lr=True,
dark_medfilt_bpm_kw=dict(med_rat_clip=None,
std_rat_clip=None,
std_model='std',
logical='and',
sigclip_kw=dict(sigma=2,
maxiters=5,
std_ddof=1)),
update_header=True,
save_nonpol=False,
verbose_bpm=False,
split=True,
verbose=False):
''' Rename the original NHAO NIC image and convert to certain dtype.
Parameters
----------
fpath : path-like
Path to the original image FITS file.
outdir : None, path-like
The top directory for the new file to be saved. If `None`,
nothing will be saved.
Note
----
Original NHAO NIC image is in 32-bit integer, and is twice the size it should be. To save the
storage, it is desirable to convert those to 16-bit. As the bias is not added to the FITS frame
from NHAO NIC, the pixel value in the raw FITS file can be negative. Fortunately, however, the
maximum pixel value when saturation occurs is only about 20k, and therefore using ``int16`` rather
than ``uint16`` is enough.
Here, not only reducing the size, the file names are updated using the original file name and
header information:
``<FILTER (j, h, k)><System YYMMDD>_<COUNTER:04d>.fits``
It is then updated to
``<FILTER (j, h, k)>_<System YYYYMMDD>_<COUNTER:04d>_<OBJECT>_<EXPTIME:.1f>_<POL-AGL1:04.1f>
_<INSROT:+04.0f>_<IMGROT:+04.0f>_<PA:+06.1f>.fits``
'''
def _sub_lr(part_l, part_r, filt):
# part_l = cr_reject_nic(part_l, crrej_kw=crrej_kw, verbose=verbose_crrej, add_process=False)
add_to_header(
part_l.header,
'h',
verbose=verbose_bpm,
s=
("Median filter badpixel masking (MBPM) algorithm started running on the left half "
+
'to remove hot pixels on left; a prerequisite for the right frame - left frame" '
+
f"technique to remove wavy pattern. Using med_sub_clip = {med_sub_clip} "
+ f"and {dark_medfilt_bpm_kw}"))
# To keep the header log of medfilt_bpm, I need this:
_tmp = part_r.data
part_r.data = part_l.data
part_r = medfilt_bpm(part_r,
med_sub_clip=med_sub_clip,
**dark_medfilt_bpm_kw)
_t = Time.now()
part_r.data = (_tmp - part_r.data).astype(dtype)
add_to_header(
part_r.header,
'h',
t_ref=_t,
verbose=verbose_bpm,
s=("Left part (vertical subtracted and MBPMed) is subtracted from the right part "
+ "(only vertical subtracted)"))
add_to_header(part_r.header,
'h',
verbose=verbose_bpm,
s="{:-^72s}".format(' DONE '),
fmt=None)
return part_r
fpath = Path(fpath)
ccd_orig = load_ccd(fpath)
_t = Time.now()
ccd_nbit = ccd_orig.copy()
ccd_nbit = CCDData_astype(ccd_nbit, dtype=dtype)
add_to_header(ccd_nbit.header,
'h',
verbose=verbose,
s="{:=^72s}".format(' Basic preprocessing start '))
if ccd_orig.dtype != ccd_nbit.dtype:
add_to_header(
ccd_nbit.header,
'h',
t_ref=_t,
verbose=verbose,
s=f"Changed dtype (BITPIX): {ccd_orig.dtype} to {ccd_nbit.dtype}")
# == First, check if identical ========================================================================= #
# It takes < ~20 ms on MBP 15" [2018, macOS 10.14.6, i7-8850H (2.6 GHz; 6-core), RAM 16 GB (2400MHz
# DDR4), Radeon Pro 560X (4GB)]
# ysBach 2020-05-15 16:06:08 (KST: GMT+09:00)
np.testing.assert_almost_equal(ccd_orig.data - ccd_nbit.data,
np.zeros(ccd_nbit.data.shape))
# == Set counter ======================================================================================= #
try:
counter = ccd_nbit.header["COUNTER"]
except KeyError:
try:
counter = fpath.stem.split('_')[1]
counter = counter.split('.')[0]
# e.g., hYYMMDD_dddd.object.pcr.fits
except IndexError: # e.g., test images (``h.fits```)
counter = 9999
ccd_nbit.header['COUNTER'] = (
counter, "Image counter of the day, 1-indexing; 9999=TEST")
# == Set output stem =================================================================================== #
hdr = ccd_nbit.header
outstem, polmode = _set_fstem(hdr)
# == Update warning-invoking parts ===================================================================== #
try:
ccd_nbit.header["MJD-STR"] = float(hdr["MJD-STR"])
ccd_nbit.header["MJD-END"] = float(hdr["MJD-END"])
except KeyError:
pass
# == Simple process to remove artifacts... ============================================================= #
ccd_nbit = vertical_correct(ccd_nbit,
fitting_sections=fitting_sections,
method=method,
sigclip_kw=sigclip_kw,
dtype='float32',
return_pattern=False,
update_header=update_header,
verbose=verbose)
if polmode:
filt = infer_filter(ccd_nbit, filt=None, verbose=verbose)
if split:
ccds_l = split_oe(ccd_nbit,
filt=filt,
right_half=True,
verbose=verbose)
ccds_r = split_oe(ccd_nbit,
filt=filt,
right_half=False,
verbose=verbose)
ccd_nbit = []
for i, oe in enumerate(['o', 'e']):
if sub_lr:
part_l = ccds_l[i]
part_r = ccds_r[i]
_ccd_nbit = _sub_lr(part_l, part_r, filt)
else:
_ccd_nbit = CCDData_astype(ccds_r[i], dtype=dtype)
_save(_ccd_nbit, outdir, outstem + f"_{oe:s}")
ccd_nbit.append(_ccd_nbit)
else:
# left half and right half
part_l = trim_ccd(ccd_nbit, NICSECTS["left"], verbose=verbose)
part_r = trim_ccd(ccd_nbit, NICSECTS["right"], verbose=verbose)
if sub_lr:
part_l = ccds_l[i]
part_r = ccds_r[i]
_ccd_nbit = _sub_lr(part_l, part_r, filt)
else:
_ccd_nbit = CCDData_astype(ccds_r[i], dtype=dtype)
ccd_nbit = part_r.copy()
_save(ccd_nbit, outdir, outstem)
else:
if save_nonpol:
_save(ccd_nbit, outdir, outstem)
return ccd_nbit