def reproject(filename,ref_header,skylist,explist):
#Open the image.
hdulist = fits.open(filename)
datacube = hdulist[0].data
header = hdulist[0].header
#Create a list with zeros for the reprojected datacube.
repro_datacube = [0] * len(datacube)
#Loop over the different frames in the datacube.
for i,data in enumerate(datacube):
#Create a CCDData class object.
data_ccd = CCDData(data,header=header,unit="count",wcs=wcs.WCS(header).celestial)
#Reproject the data to the reference data.
repro_datacube[i] = np.asarray(wcs_project(data_ccd,wcs.WCS(ref_header).celestial,target_shape=(ref_header['NAXIS2'],ref_header['NAXIS1'])))
#Temporary workaround to update the header of the image.
new_data = wcs_project(data_ccd,wcs.WCS(ref_header).celestial,target_shape=(ref_header['NAXIS2'],ref_header['NAXIS1']))
new_data.write(filename.replace(".img","_r.img"),format="fits",overwrite=True)
temp_hdu = fits.open(filename.replace(".img","_r.img"))
new_header = temp_hdu[0].header
#Append the reprojected datacube to the list and write it to a new image.
skylist.append(np.array(repro_datacube))
new_hdu = fits.PrimaryHDU(repro_datacube,new_header)
new_hdu.writeto(filename.replace(".img","_r.img"),overwrite=True)
#Reproject the exposure map.
data_exp_ccd = CCDData.read(filename.replace("nm_coilsszp_c","ex"),unit="count",hdu=1)
repro_data_exp = wcs_project(data_exp_ccd,wcs.WCS(ref_header).celestial,target_shape=(ref_header['NAXIS2'],ref_header['NAXIS1']))
#Append the reprojected data to a list and write it to a new image.
explist.append(np.array(repro_data_exp))
repro_data_exp.write(filename.replace("nm_coilsszp_c","ex_r"),format="fits",overwrite=True)
def swarp(hdus, reference_hdu, rate, hdu_idx=None, stacking_mode="MEAN"):
"""
use the WCS to project all image to the 'reference_hdu' shifting the the CRVAL of each image by rate*dt
:param stacking_mode: what process to use for combining images MEAN or MEDIAN
:param hdu_idx: which HDU in each HDUList listed in hdus is the ImageData in?
:param hdus: list of HDUList
:param reference_hdu: reference HDUList in hdus
:param rate: dictionary with the ra/dec shift rates.
:return: fits.HDUList
"""
# Project the input images to the same grid using interpolation
if stacking_mode not in ['MEDIAN', 'MEAN']:
logging.warning(
f'{stacking_mode} not available for swarp stack. Setting to MEAN')
stacking_mode = 'MEAN'
if hdu_idx is None:
hdu_idx = HSC_HDU_MAP
reference_date = mid_exposure_mjd(reference_hdu[0])
stack_input = []
logging.info(f'stacking at rate/angle set: {rate}')
ccd_data = {}
for hdu in hdus:
wcs_header = hdu[1].header.copy()
dt = (mid_exposure_mjd(hdu[0]) - reference_date)
if rate is not None:
wcs_header['CRVAL1'] += (rate['dra'] * dt)
wcs_header['CRVAL2'] += (rate['ddec'] * dt)
for layer in hdu_idx:
data = hdu[hdu_idx[layer]].data
if layer == 'variance':
data = VarianceUncertainty(data)
elif layer == 'mask':
data = bitfield_to_boolean_mask(data,
ignore_flags=STACK_MASK,
flip_bits=True)
ccd_data[layer] = data
logging.info(f'Adding {hdu[0]} to projected stack.')
stack_input.append(
wcs_project(
CCDData(ccd_data['image'],
mask=ccd_data['mask'],
header=wcs_header,
wcs=WCS(wcs_header),
unit='adu',
uncertainty=ccd_data['variance']),
WCS(reference_hdu.header)))
logging.debug(f'{stack_input[-1].header}')
if rate is not None:
combiner = Combiner(stack_input)
if stacking_mode == 'MEDIAN':
stacked_image = combiner.median_combine()
else:
stacked_image = combiner.average_combine()
return fits.HDUList([
fits.PrimaryHDU(header=reference_hdu[0]),
fits.ImageHDU(data=stacked_image.data,
header=reference_hdu[1].header)
])
else:
return stack_input
def register_collection(self, reference):
print(
f'{Style.BRIGHT}Registering {len(self.collection.files)} files.{Style.RESET_ALL}'
)
print(f'Reference frame: {reference}')
hlp.prompt()
write_path = Path(f'{os.getcwd()}/registered')
write_path.mkdir(exist_ok=True)
target_wcs = ccdp.CCDData.read(reference).wcs
count = 1
for img, fname in self.collection.ccds(return_fname=True):
print(
f'{Style.BRIGHT}[{count}/{len(self.collection.files)}]{Style.RESET_ALL} Computing for {fname}...'
)
ccdp.wcs_project(img, target_wcs).write(write_path / fname,
overwrite=True)
count += 1
def reproject_data(self, target=0):
'''
Reprojects each layer of self.data to be aligned, using their WCS.
By default aligns everything else with the first image,
but this can be changed by setting target.
inputs:
-------
self.data, self.WCS, self.header
target - int - Index of the target image to align relative to
outputs:
--------
self.data
self.wcs
self.header
self.reprojected = True
'''
if self.reprojected:
print('Data has already been aligned and reprojected! '
'Doing nothing!')
else:
for i, dat in enumerate(self.data):
print(f"Reprojecting image {i}")
if i != target: # Don't reproject the target image, duh!
# Do the reprojection
reprojecti = wcs_project(
CCDData(dat, wcs=self.WCS[i], unit='adu'),
self.WCS[target])
# Append to list
self.data[i, :, :] = reprojecti.data
# Update WCS, both in self.wcs and self.header
self.WCS[i] = self.WCS[target]
del self.header[i]['CD?_?'] # required for update to work
self.header[i].update(self.WCS[i].to_header(relax=True))
# Add a comment to the header about the reprojection
now = str(datetime.today())[:19]
self.header[i]['COMMENT'] = (
f'Data was reprojected to WCS '
f'of file {target} at {now}')
self.reprojected = True
header=header,
wcs=wcs.WCS(header),
unit=u.electron)
elif exts == 4:
sci_full = CCDData(np.concatenate(sci_final, axis=1),
header=header,
wcs=wcs.WCS(header),
unit=u.electron)
sci_full.mask = np.zeros(np.shape(sci_full))
sci_full.mask[sci_full == -999] = 1
sci_full.mask = sci_full.mask.astype(np.bool)
sci_full.header['DATASEC'] = (
'[1:%s,1:%s]' % (np.shape(sci_full)[1], np.shape(sci_full)[0]))
if i == 0: wcs_object = wcs.WCS(header)
wcs_objects[i] = wcs.WCS(header)
reprojected.append(wcs_project(sci_full, wcs_objects[0]))
reprojected[i].write(processed[i], overwrite=True)
sci_med = ccdproc.combine(reprojected,
method='median',
sigma_clip=True,
sigma_clip_func=np.ma.median)
sci_med.write(red_path + sci + '.fits', overwrite=True)
with fits.open(red_path + sci + '.fits', mode='update') as hdr:
hdr[0].header['RDNOISE'] = header['RDNOISE'] / len(sci_list[sci])
hdr[0].header['NFILES'] = len(sci_list[sci])
for k, n in enumerate(sci_list[sci]):
hdr[0].header['FILE' +
str(k + 1)] = (os.path.basename(n),
'Name of file used in median.')
if args.wcs == 'True' or args.wcs == 'yes':
fig, ax = plt.subplots(figsize=(7, 7))
for k, n2 in enumerate(time_list)])
sky_data = [processed[k] for _, k in time_diff[1:10]]
sky_mask = [masks[k] for _, k in time_diff[1:10]]
sky = ccdproc.combine(sky_data,
method='median',
sigma_clip=True,
sigma_clip_func=np.ma.median,
mask=sky_mask)
sky.write(
red_path +
os.path.basename(sci_list[i]).replace('.fits', '_sky.fits'),
overwrite=True)
final = n.subtract(sky,
propagate_uncertainties=True,
handle_meta='first_found')
reprojected.append(wcs_project(final, wcs_object))
reprojected[i].write(red_path + os.path.basename(sci_list[i]),
overwrite=True)
sci_med = ccdproc.combine(reprojected,
method='median',
sigma_clip=True,
sigma_clip_func=np.ma.median)
header = sci_med.header.extend(sci_med.wcs.to_header())
fits.writeto(red_path + target + '.fits',
sci_med.data,
sci_med.header,
overwrite=True)
with fits.open(red_path + target + '.fits', mode='update') as hdr:
hdr[0].header['RDNOISE'] = sci_med.header['RDNOISE'] / len(
sci_list)
hdr[0].header['NFILES'] = len(sci_list)
def create_stacks(filter_list,
old_path,
path,
ext,
log=None,
log2=None,
mult_ext=True,
align=None):
'''
Function used to create one or more stacks of a specific extension from a dictionary of files sorted by filter.
Currently supports stacking with ``wcs``-alignment, soon will also support ``pixel``-alignment.
Also supports stacking without any alignment.
Stacks created have name of event, filter, extension, and type of alignment (if used) in their name and are written
to ``path``.
Parameters
----------
:param filter_list: python dictionary
Dictionary of files. Key is the filter of the files, values are the file names themselves.
:param old_path: str
Path to the original data.
:param path: str
Path to the reduced data; usually ``old_path`` + '/red/ex' + str(``ext``).
:param ext: int
The extension of each original file to use.
:param log: log, optional
In-depth log.
If no log is inputted, information is printed out instead of being written to ``log``.
Default is ``None``.
:param log2: log, optional
Overview log.
If no log is inputted, information is printed out instead of being written to ``log2``.
Default is ``None``.
:param mult_ext: boolean, optional
Boolean as to whether the original files have multiple extensions. Set to ``False`` if only one extension.
Default is ``True``.
:param align: str, optional
Type of alignment to perform. Currently supports ``wcs`` (and ``pixel``).
Pixel alignment is still in the process of being written. ``TODO`` comments highlight where to put in new code
to add in pixel alignment functionality. Remove this line and ``TODO`` comments after adding functionality in.
Default is ``None`` (no alignment performed).
Returns
-------
:return: python dictionary
Dictionary of stacks. Key is the filter of the stack, values are the stack names themselves.
'''
if log2 is not None:
log2.info("Filename & Filter")
else:
print("Filename & Filter")
stack_dict = {} # Dictionary for stack files
delta_t = 10 # Exposure time, in sec
for fil in filter_list:
x_shape, y_shape = 1000000000, 1000000000 # For shape (initially set to a very large value)
red_list = []
reprojected = [] # For WCS projections
for i, f in enumerate(filter_list[fil]):
if log2 is not None:
log2.info("%s %s" %
(f.split("/")[len(f.split("/")) - 1], fil.upper()))
else:
print("%s %s" %
(f.split("/")[len(f.split("/")) - 1], fil.upper()))
if log is not None:
log.info(f + " " + fil)
with fits.open(
f
) as hdr: # Open the file and extract the header and the data
if i == 0: # Get the event name
hdr0 = hdr[0].header
event_name = hdr0['OBJECT']
if mult_ext:
header = hdr[ext].header
data = hdr[ext].data
else: # In the case that the files only have one extension
header = hdr[0].header
data = hdr[0].data
if i == 0 and align == "wcs":
wcs_object = wcs.WCS(
header) # Create object for WCS projection
# Find background
data[np.isnan(data)] = np.nanmedian(
data) # Get rid of any nan data
sci_data = CCDData(
data, meta=header, unit='adu'
) # Make a CCDData object with same header and data in adu
bkg = MeanBackground(SigmaClip(
sigma=3.)) # Use sigma clipping to find the mean background
bkg_value = bkg.calc_background(
sci_data) # Find the background of the CCDData object
# Subtract background
red = sci_data.subtract(bkg_value * np.ones(np.shape(sci_data)) *
u.adu,
propagate_uncertainties=True,
handle_meta='first_found')
# Divide by exposure time to get all files on the same scale
red = red.divide(delta_t * u.second,
propagate_uncertainties=True,
handle_meta='first_found')
# Setting the shape (finding the minimum to rebin later)
if red.shape[0] < x_shape:
x_shape = red.shape[0]
if red.shape[1] < y_shape:
y_shape = red.shape[1]
if align == "wcs": # Reprojecting the image onto a common WCS
red.wcs = wcs.WCS(header)
reprojected.append(wcs_project(red, wcs_object))
if align == "pixel": # TODO: Add pixel alignment function here in the future
print("Not available yet. Images will not be aligned")
red_list.append(red) # Add to a list to combine later
# Write to a new directory
red.write(f.replace(old_path, path).replace(
('.' + f.split('.')[1]),
'_c' + str(ext) + '_' + str(fil) + '_red.fits'),
overwrite=True)
# Re-bin so all of the files are of the same shape
final_list = []
for red in red_list:
red1 = ccdproc.rebin(red, (x_shape, y_shape))
final_list.append(red1)
# Stack the reduced files that have been re-bined so all have the same dimensions
stack_hbu = ccdproc.combine(final_list,
method='median',
sigma_clip=True,
sigma_clip_func=np.ma.median)
stack_hbu.write(path + event_name + "_ext_" + str(ext) + "_" + fil +
'_stack.fits',
overwrite=True)
if align == "wcs":
# Re-bin so all of the files are of the same shape
final_list = []
for red in reprojected:
red1 = ccdproc.rebin(red, (x_shape, y_shape))
final_list.append(red1)
# Stack, WCS aligned
stack_hbu = ccdproc.combine(final_list,
method='median',
sigma_clip=True,
sigma_clip_func=np.ma.median)
stack_hbu.write(path + event_name + "_ext_" + str(ext) + "_" +
fil + '_stack_wcs.fits',
overwrite=True)
if align == "pixel": # TODO: Add pixel alignment function here in the future
print("Not available yet. No pixel-aligned stacks")
# # Re-bin so all of the files are of the same shape
# final_list = []
# for red in pixel_reprojected:
# red1 = ccdproc.rebin(red, (x_shape, y_shape))
# final_list.append(red1)
# # Stack, pixel aligned
# stack_hbu = ccdproc.combine(final_list, method='median', sigma_clip=True, sigma_clip_func=np.ma.median)
# stack_hbu.write(path + event_name + "_ext_" + str(ext) + "_" + fil + '_stack_pixel.fits', overwrite=True)
try:
stack_dict[fil]
except KeyError:
stack_dict.update({fil: []})
stack_dict[fil].append(path + event_name + "_ext_" + str(ext) + "_" +
fil + '_stack.fits')
if align == "wcs":
stack_dict[fil].append(path + event_name + "_ext_" + str(ext) +
"_" + fil + '_stack_wcs.fits')
if align == "pixel": # TODO: Uncomment when pixel-aligning function has been added
print("Not available yet. No pixel-aligned stacks")
# stack_dict[fil].append(path + event_name + "_ext_" + str(ext) + "_" + fil + '_stack_pixel.fits')
if align is not None and log2 is not None:
if align != "pixel": # TODO: Take out later when pixel-aligning function has been added
log2.info("%s-aligned images" % str(align))
return stack_dict
def swarp(hdus, reference_hdu, rate, hdu_idx=None, stacking_mode="MEAN", **kwargs):
"""
use the WCS to project all image to the 'reference_hdu' shifting the the CRVAL of each image by rate*dt
:param stacking_mode: what process to use for combining images MEAN or MEDIAN
:param hdu_idx: which HDU in each HDUList listed in hdus is the ImageData in?
:param hdus: list of HDUList
:param reference_hdu: reference HDUList in hdus
:param rate: dictionary with the ra/dec shift rates.
:return: fits.HDUList
"""
# Project the input images to the same grid using interpolation
# logging.debug(f"Called with {kwargs}")
if stacking_mode not in ['MEDIAN', 'MEAN']:
logging.warning(f'{stacking_mode} not available for swarp stack. Setting to MEAN')
stacking_mode = 'MEAN'
if hdu_idx is None:
hdu_idx = HSC_HDU_MAP
reference_date = mid_exposure_mjd(reference_hdu[0])
reference_header = kwargs['astheads'][reference_hdu[0].header['IMAGE']]
reference_wcs = WCS(reference_header)
stack_input = {}
logging.info(f'stacking at rate/angle set: {rate}')
ccd_data = {}
for image in hdus:
# logging.info(f"Opening {image} to add to stack")
# with fits.open(image, mode='update') as hdu:
hdu = hdus[image]
wcs_header = kwargs['astheads'][hdu[0].header['IMAGE']]
# wcs_header = hdu[1].header.copy()
dt = (mid_exposure_mjd(hdu[0]) - reference_date)
if rate is not None:
wcs_header['CRVAL1'] -= (rate['dra'] * dt).to('degree').value*numpy.cos(numpy.deg2rad(wcs_header['CRVAL2']))
wcs_header['CRVAL2'] -= (rate['ddec'] * dt).to('degree').value
for layer in hdu_idx:
data = hdu[hdu_idx[layer]].data
if layer == 'variance':
data = VarianceUncertainty(data)
elif layer == 'mask':
data = bitfield_to_boolean_mask(data, ignore_flags=STACK_MASK, flip_bits=True)
ccd_data[layer] = data
logging.info(f'Adding {hdu[0].header["IMAGE"]} to projected stack.')
# reference_header = referece_hdu[1].header
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
stack_input[image] = wcs_project(CCDData(ccd_data['image'],
mask=ccd_data['mask'],
header=wcs_header,
wcs=WCS(wcs_header),
unit='adu',
uncertainty=ccd_data['variance']),
reference_wcs)
if rate is not None:
combiner = Combiner(stack_input.values())
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=RuntimeWarning)
if stacking_mode == 'MEDIAN':
stacked_image = combiner.median_combine()
else:
stacked_image = combiner.average_combine()
return fits.HDUList([fits.PrimaryHDU(header=reference_hdu[0].header),
fits.ImageHDU(data=stacked_image.data, header=reference_header)])
else:
return stack_input
def stacking_procedure(images: [Image]):
"""stacks images in list into single one and writes file
header will be from middle image in the list.
Time will be between start of first and end of last exposure with exposure length of this difference.
:param images list of Image list objects to stack
"""
# go through images and collect info
names = []
i = 0
half = len(images) // 2
tfirst = images[0].get_time_jd()
tlast = images[-1].get_time_jd() + images[-1].get_exposure() / 86000
timejd_mid = (tfirst + tlast) / 2
timecoverage = (tlast - tfirst) * 86000 # s
for image in images: # get stack names
if "stack" in image.processing_parameters:
for name in image.get_stack():
names.append(name)
else:
names.append(str(image.get_path()))
i += 1
if i == half:
header = fits.getheader(image.get_path())
midpoint_WCS = WCS(header)
reprojected = []
# reproject all images onto the middle one
for image in images:
data = fits.getdata(image.get_path())
header_wcs = fits.getheader(image.get_path())
ccddata = ccdproc.CCDData(data, wcs=WCS(header_wcs), unit="adu")
reprojected.append(wcs_project(ccddata, midpoint_WCS))
combiner = ccdproc.Combiner(reprojected)
final_image_data = combiner.average_combine()
final_image_data.wcs = WCS(header)
header["EXPTIME"] = timecoverage
header["EXPOSURE"] = timecoverage
header["JD"] = tfirst
filename = "stack-" + str(timejd_mid) + "-e" + "{:.0f}".format(
timecoverage) + ".fits"
path = Path("/tmp")
path = path / filename
if os.path.exists(path):
os.remove(path)
fits.writeto(path, final_image_data, header)
stacked_image = Image(fixed_parameters={
"path": path,
"exposure": timecoverage,
"time_jd": tfirst,
"type": "data",
"id": filename
},
processing_parameters={
"flat": True,
"dark": True,
"stack": names
})
return stacked_image
import ccdproc
from astropy.nddata import CCDData
import image_registration
read = ir_reduce.read_and_sort(glob.glob('../testdata/bad*.fits'),
glob.glob('../testdata/Flat*.fits'),
glob.glob('../testdata/NCA*.fits'))
processed = ir_reduce.tiled_process(read['J'].bad, read['J'].flat[0],
read['J'].images)
#processed = ir_reduce.standard_process(read['J'].bad, read['J'].flat[0], read['J'].images)
skyscaled = ir_reduce.skyscale(processed, 'subtract')
interpolated = [
ir_reduce.interpolate(image, dofixpix=True) for image in skyscaled
]
wcs = interpolated[0].wcs
reprojected = [ccdproc.wcs_project(img, wcs) for img in interpolated]
combined = ccdproc.Combiner(reprojected).median_combine()
#
# first_hdu = output_image.to_hdu()[0]
# scamp_input = CCDData(first_hdu.data, header=first_hdu.header, unit=first_hdu.header['bunit'])
#
# scamp_input = CCDData(first_hdu.data, header=first_hdu.header, unit=first_hdu.header['bunit'])
#
combined.wcs = wcs
combined.write('foo.fits', overwrite=True)