def _init(self):
memory = self._m_config_port.get_attribute("MEMORY")
ndim_image = self.m_image_in_port.get_ndim()
ndim_psf = self.m_psf_in_port.get_ndim()
if ndim_image != 3:
raise ValueError(
"The image_in_tag should contain a cube of images.")
nimages = number_images_port(self.m_image_in_port)
im_size = image_size_port(self.m_image_in_port)
frames = memory_frames(memory, nimages)
npsf = number_images_port(self.m_psf_in_port)
psf_size = image_size_port(self.m_psf_in_port)
if psf_size != im_size:
raise ValueError("The images in '" + self.m_image_in_port.tag +
"' should have the same "
"dimensions as the images images in '" +
self.m_psf_in_port.tag + "'.")
if ndim_psf == 3 and npsf == 1:
psf = np.squeeze(self.m_psf_in_port.get_all(), axis=0)
ndim_psf = psf.ndim
elif ndim_psf == 2:
psf = self.m_psf_in_port.get_all()
elif ndim_psf == 3 and nimages != npsf:
psf = np.zeros((self.m_psf_in_port.get_shape()[1],
self.m_psf_in_port.get_shape()[2]))
frames_psf = memory_frames(memory, npsf)
for i, _ in enumerate(frames_psf[:-1]):
psf += np.sum(self.m_psf_in_port[frames_psf[i]:frames_psf[i +
1]],
axis=0)
psf /= float(npsf)
ndim_psf = psf.ndim
elif ndim_psf == 3 and nimages == npsf:
psf = None
return psf, ndim_psf, ndim_image, frames
def run(self) -> None:
"""
Run method of the module. Rotates all images by a constant angle.
Returns
-------
NoneType
None
"""
memory = self._m_config_port.get_attribute('MEMORY')
nimages = self.m_image_in_port.get_shape()[0]
frames = memory_frames(memory, nimages)
start_time = time.time()
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), 'Rotating images...', start_time)
images = self.m_image_in_port[frames[i]:frames[i + 1], ]
for j in range(frames[i + 1] - frames[i]):
im_tmp = images[j, ]
# ndimage.rotate rotates in clockwise direction for positive angles
im_tmp = rotate(im_tmp, self.m_angle, reshape=False)
self.m_image_out_port.append(im_tmp, data_dim=3)
history = f'angle [deg] = {self.m_angle}'
self.m_image_out_port.add_history('RotateImagesModule', history)
self.m_image_out_port.copy_attributes(self.m_image_in_port)
self.m_image_out_port.close_port()
def run(self) -> None:
"""
Run method of the module. Creates a master dark with the same shape as the science
data and subtracts the dark frame from the science data.
Returns
-------
NoneType
None
"""
memory = self._m_config_port.get_attribute('MEMORY')
nimages = self.m_image_in_port.get_shape()[0]
frames = memory_frames(memory, nimages)
dark = self.m_dark_in_port.get_all()
master = _master_frame(data=np.mean(dark, axis=0),
im_shape=self.m_image_in_port.get_shape())
start_time = time.time()
for i in range(len(frames[:-1])):
progress(i, len(frames[:-1]), 'Subtracting the dark current...',
start_time)
images = self.m_image_in_port[frames[i]:frames[i + 1], ]
self.m_image_out_port.append(images - master, data_dim=3)
history = f'dark_in_tag = {self.m_dark_in_port.tag}'
self.m_image_out_port.add_history('DarkCalibrationModule', history)
self.m_image_out_port.copy_attributes(self.m_image_in_port)
self.m_image_out_port.close_port()
def run(self) -> None:
"""
Run method of the module. Removes the lines given by *lines* from each frame.
Returns
-------
NoneType
None
"""
memory = self._m_config_port.get_attribute('MEMORY')
nimages = self.m_image_in_port.get_shape()[0]
frames = memory_frames(memory, nimages)
start_time = time.time()
for i in range(len(frames[:-1])):
progress(i, len(frames[:-1]), 'Removing lines...', start_time)
image_in = self.m_image_in_port[frames[i]:frames[i + 1], ]
image_out = image_in[:,
int(self.m_lines[2]):image_in.shape[1] -
int(self.m_lines[3]),
int(self.m_lines[0]):image_in.shape[2] -
int(self.m_lines[1])]
self.m_image_out_port.append(image_out, data_dim=3)
history = f'number of lines = {self.m_lines}'
self.m_image_out_port.add_history('RemoveLinesModule', history)
self.m_image_out_port.copy_attributes(self.m_image_in_port)
self.m_image_out_port.close_port()
def run(self) -> None:
"""
Run method of the module. Applies a Gaussian filter to the spatial dimensions of the
images.
Returns
-------
NoneType
None
"""
memory = self._m_config_port.get_attribute('MEMORY')
pixscale = self._m_config_port.get_attribute('PIXSCALE')
nimages = self.m_image_in_port.get_shape()[0]
frames = memory_frames(memory, nimages)
sigma = (self.m_fwhm / pixscale) / (2. * math.sqrt(2. * math.log(2.))
) # [pix]
start_time = time.time()
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), 'Applying Gaussian filter...',
start_time)
images = self.m_image_in_port[frames[i]:frames[i + 1], ]
im_filter = gaussian_filter(images, (0, sigma, sigma))
self.m_image_out_port.append(im_filter, data_dim=3)
history = f'fwhm [arcsec] = {self.m_fwhm}'
self.m_image_out_port.add_history('GaussianFilterModule', history)
self.m_image_out_port.copy_attributes(self.m_image_in_port)
self.m_image_out_port.close_port()
def run(self) -> None:
"""
Run method of the module. Repeats the stack of input images a specified number of times.
Returns
-------
NoneType
None
"""
self.m_image_out_port.del_all_attributes()
self.m_image_out_port.del_all_data()
nimages = self.m_image_in_port.get_shape()[0]
memory = self._m_config_port.get_attribute('MEMORY')
frames = memory_frames(memory, nimages)
start_time = time.time()
for i in range(self.m_repeat):
progress(i, self.m_repeat, 'Running RepeatImagesModule...', start_time)
for j, _ in enumerate(frames[:-1]):
images = self.m_image_in_port[frames[j]:frames[j+1], ]
self.m_image_out_port.append(images, data_dim=3)
sys.stdout.write('Running RepeatImagesModule... [DONE]\n')
sys.stdout.flush()
history = f'repeat = {self.m_repeat}'
self.m_image_out_port.add_history('RepeatImagesModule', history)
self.m_image_out_port.copy_attributes(self.m_image_in_port)
self.m_image_out_port.close_port()
def _stack(nimages, im_shape, parang):
im_new = None
parang_new = None
if self.m_stacking is not None:
frames = memory_frames(self.m_stacking, nimages)
nimages_new = np.size(frames) - 1
if parang is None:
parang_new = None
else:
parang_new = np.zeros(nimages_new)
im_new = np.zeros((nimages_new, im_shape[1], im_shape[2]))
for i in range(nimages_new):
progress(i, nimages_new, "Running StackAndSubsetModule...")
if parang is not None:
parang_new[i] = np.mean(parang[frames[i]:frames[i +
1]])
im_new[i, ] = np.mean(
self.m_image_in_port[frames[i]:frames[i + 1], ],
axis=0)
im_shape = im_new.shape
else:
if parang is not None:
parang_new = np.copy(parang)
return im_shape, im_new, parang_new
def run(self) -> None:
"""
Run method of the module. Subtracts the images from the second database tag from the images
of the first database tag, on a frame-by-frame basis.
Returns
-------
NoneType
None
"""
if self.m_image_in1_port.get_shape(
) != self.m_image_in2_port.get_shape():
raise ValueError(
'The shape of the two input tags has to be the same.')
memory = self._m_config_port.get_attribute('MEMORY')
nimages = self.m_image_in1_port.get_shape()[0]
frames = memory_frames(memory, nimages)
start_time = time.time()
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), 'Subtracting images...', start_time)
images1 = self.m_image_in1_port[frames[i]:frames[i + 1], ]
images2 = self.m_image_in2_port[frames[i]:frames[i + 1], ]
self.m_image_out_port.append(self.m_scaling * (images1 - images2),
data_dim=3)
history = f'scaling = {self.m_scaling}'
self.m_image_out_port.add_history('SubtractImagesModule', history)
self.m_image_out_port.copy_attributes(self.m_image_in1_port)
self.m_image_out_port.close_port()
def run(self):
"""
Run method of the module. Calculates the minimum, maximum, sum, mean, median, and standard
deviation of the pixel values of each image separately. NaNs are ignored for each
calculation. The values are calculated for either the full images or a circular
subsection of the images.
Returns
-------
NoneType
None
"""
self.m_stat_out_port.del_all_data()
self.m_stat_out_port.del_all_attributes()
memory = self._m_config_port.get_attribute("MEMORY")
pixscale = self.m_image_in_port.get_attribute("PIXSCALE")
if self.m_position is not None:
self.m_position = (
int(self.m_position[1]), # y position
int(self.m_position[0]), # x position
self.m_position[2] / pixscale) # radius (pix)
nimages = self.m_image_in_port.get_shape()[0]
im_shape = self.m_image_in_port.get_shape()[1:]
frames = memory_frames(memory, nimages)
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), "Running ImageStatisticsModule...")
images = self.m_image_in_port[frames[i]:frames[i + 1], ]
images = np.reshape(images,
(images.shape[0], im_shape[0] * im_shape[1]))
if self.m_position is not None:
rr_grid = pixel_distance(im_shape, self.m_position)
indices = np.where(rr_grid <= self.m_position[2])[0]
images = images[:, indices]
nmin = np.nanmin(images, axis=1)
nmax = np.nanmax(images, axis=1)
nsum = np.nansum(images, axis=1)
mean = np.nanmean(images, axis=1)
median = np.nanmedian(images, axis=1)
std = np.nanstd(images, axis=1)
result = np.column_stack((nmin, nmax, nsum, mean, median, std))
self.m_stat_out_port.append(result)
sys.stdout.write("Running ImageStatisticsModule... [DONE]\n")
sys.stdout.flush()
history = "number of images = " + str(nimages)
self.m_stat_out_port.copy_attributes(self.m_image_in_port)
self.m_stat_out_port.add_history("ImageStatisticsModule", history)
self.m_stat_out_port.close_port()
def run(self) -> None:
"""
Run method of the module. Removes the frames and corresponding attributes, updates the
NFRAMES attribute, and saves the data and attributes.
Returns
-------
NoneType
None
"""
self._initialize()
memory = self._m_config_port.get_attribute('MEMORY')
nimages = self.m_image_in_port.get_shape()[0]
frames = memory_frames(memory, nimages)
if memory == 0 or memory >= nimages:
memory = nimages
start_time = time.time()
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), 'Running RemoveFramesModule...', start_time)
images = self.m_image_in_port[frames[i]:frames[i+1], ]
index_del = np.where(np.logical_and(self.m_frames >= frames[i],
self.m_frames < frames[i+1]))
write_selected_data(images,
self.m_frames[index_del] % memory,
self.m_selected_out_port,
self.m_removed_out_port)
sys.stdout.write('Running RemoveFramesModule... [DONE]\n')
sys.stdout.flush()
history = 'frames removed = '+str(np.size(self.m_frames))
if self.m_selected_out_port is not None:
# Copy attributes before write_selected_attributes is used
self.m_selected_out_port.copy_attributes(self.m_image_in_port)
self.m_selected_out_port.add_history('RemoveFramesModule', history)
if self.m_removed_out_port is not None:
# Copy attributes before write_selected_attributes is used
self.m_removed_out_port.copy_attributes(self.m_image_in_port)
self.m_removed_out_port.add_history('RemoveFramesModule', history)
write_selected_attributes(self.m_frames,
self.m_image_in_port,
self.m_selected_out_port,
self.m_removed_out_port)
self.m_image_in_port.close_port()
def run(self):
"""
Run method of the module. Removes the frames and corresponding attributes, updates the
NFRAMES attribute, and saves the data and attributes.
:return: None
"""
self._initialize()
memory = self._m_config_port.get_attribute("MEMORY")
nimages = number_images_port(self.m_image_in_port)
frames = memory_frames(memory, nimages)
if memory == 0 or memory >= nimages:
memory = nimages
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), "Running RemoveFramesModule...")
images = self.m_image_in_port[frames[i]:frames[i + 1], ]
index_del = np.where(np.logical_and(self.m_frames >= frames[i], \
self.m_frames < frames[i+1]))
write_selected_data(images, self.m_frames[index_del] % memory,
self.m_selected_out_port,
self.m_removed_out_port)
sys.stdout.write("Running RemoveFramesModule... [DONE]\n")
sys.stdout.flush()
history = "frames removed = " + str(np.size(self.m_frames))
if self.m_selected_out_port is not None:
# Copy attributes before write_selected_attributes is used
self.m_selected_out_port.copy_attributes_from_input_port(
self.m_image_in_port)
self.m_selected_out_port.add_history_information(
"RemoveFramesModule", history)
if self.m_removed_out_port is not None:
# Copy attributes before write_selected_attributes is used
self.m_removed_out_port.copy_attributes_from_input_port(
self.m_image_in_port)
self.m_removed_out_port.add_history_information(
"RemoveFramesModule", history)
write_selected_attributes(self.m_frames, self.m_image_in_port,
self.m_selected_out_port,
self.m_removed_out_port)
self.m_image_in_port.close_port()
def _initialize(
ndim: int, npix: int
) -> Tuple[int, np.ndarray, Optional[np.ndarray],
Optional[np.ndarray]]:
if ndim == 2:
nimages = 1
elif ndim == 3:
nimages = self.m_image_in_port.get_shape()[-3]
if self.m_stack == 'median':
frames = np.array([0, nimages])
else:
frames = memory_frames(memory, nimages)
elif ndim == 4:
nimages = self.m_image_in_port.get_shape()[-3]
nwave = self.m_image_in_port.get_shape()[-4]
if self.m_dimension == 'time':
frames = np.linspace(0, nwave, nwave + 1)
elif self.m_dimension == 'wavelength':
frames = np.linspace(0, nimages, nimages + 1)
else:
raise ValueError(
'The dimension should be set to \'time\' or \'wavelength\'.'
)
if self.m_stack == 'mean':
if ndim == 4:
if self.m_dimension == 'time':
im_tot = np.zeros((nwave, npix, npix))
elif self.m_dimension == 'wavelength':
im_tot = np.zeros((nimages, npix, npix))
else:
im_tot = np.zeros((npix, npix))
else:
im_tot = None
if self.m_stack is None and ndim == 4:
im_none = np.zeros((nwave, nimages, npix, npix))
else:
im_none = None
return nimages, frames, im_tot, im_none
def write_selected_data(memory: Union[int, np.int64],
indices: np.ndarray,
image_in_port: InputPort,
selected_out_port: Optional[OutputPort],
removed_out_port: Optional[OutputPort]) -> None:
"""
Function to write the selected and removed data.
Parameters
----------
memory : int
Number of images that is simultaneously loaded into the memory.
indices : numpy.ndarray
Image indices that will be removed.
image_in_port : pynpoint.core.dataio.InputPort
Port to the input images.
selected_out_port : pynpoint.core.dataio.OutputPort, None
Port to store the selected images. No data is written if set to None.
removed_out_port : pynpoint.core.dataio.OutputPort, None
Port to store the removed images. No data is written if set to None.
Returns
-------
NoneType
None
"""
nimages = image_in_port.get_shape()[0]
frames = memory_frames(memory, nimages)
if memory == 0 or memory >= nimages:
memory = nimages
start_time = time.time()
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), 'Writing selected data...', start_time)
images = image_in_port[frames[i]:frames[i+1], ]
subset_del = np.where(np.logical_and(indices >= frames[i], indices < frames[i+1]))[0]
index_del = indices[subset_del] % memory
index_sel = np.ones(images.shape[0], np.bool)
index_sel[index_del] = False
if selected_out_port is not None and index_sel.size > 0:
selected_out_port.append(images[index_sel])
if removed_out_port is not None and index_del.size > 0:
removed_out_port.append(images[index_del])
def uncompress(self) -> None:
"""
Method to check if the input directory contains compressed files ending with .fits.Z.
If this is the case, the files will be uncompressed using multithreading. The number
of threads can be set with the ``CPU`` parameter in the configuration file.
Returns
-------
NoneType
None
"""
cpu = self._m_config_port.get_attribute('CPU')
# list all files ending with .fits.Z in the input location
files = []
for item in os.listdir(self.m_input_location):
if item.endswith('.fits.Z'):
files.append(os.path.join(self.m_input_location, item))
if files:
# subdivide the file indices by number of CPU
indices = memory_frames(cpu, len(files))
start_time = time.time()
for i, _ in enumerate(indices[:-1]):
progress(i, len(indices[:-1]), 'Uncompressing NEAR data...',
start_time)
# select subset of compressed files
subset = files[indices[i]:indices[i + 1]]
# create a list of threads to uncompress CPU number of files
# each file is processed by a different thread
threads = []
for filename in subset:
thread = threading.Thread(target=self._uncompress_file,
args=(filename, ))
threads.append(thread)
# start the threads
for item in threads:
item.start()
# join the threads
for item in threads:
item.join()
sys.stdout.write('Uncompressing NEAR data... [DONE]\n')
sys.stdout.flush()
def run(self) -> None:
"""
Run method of the module. Adds lines of zero-value pixels to increase the size of an image.
Returns
-------
NoneType
None
"""
self.m_image_out_port.del_all_attributes()
self.m_image_out_port.del_all_data()
memory = self._m_config_port.get_attribute('MEMORY')
nimages = self.m_image_in_port.get_shape()[0]
frames = memory_frames(memory, nimages)
shape_in = self.m_image_in_port.get_shape()
shape_out = (shape_in[-2] + int(self.m_lines[2]) +
int(self.m_lines[3]), shape_in[-1] +
int(self.m_lines[0]) + int(self.m_lines[1]))
self.m_lines[1] = shape_out[1] - self.m_lines[1] # right side of image
self.m_lines[3] = shape_out[0] - self.m_lines[3] # top side of image
start_time = time.time()
for i in range(len(frames[:-1])):
progress(i, len(frames[:-1]), 'Running AddLinesModule...',
start_time)
image_in = self.m_image_in_port[frames[i]:frames[i + 1], ]
image_out = np.zeros(
(frames[i + 1] - frames[i], shape_out[0], shape_out[1]))
image_out[:,
int(self.m_lines[2]):int(self.m_lines[3]),
int(self.m_lines[0]):int(self.m_lines[1])] = image_in
self.m_image_out_port.append(image_out, data_dim=3)
sys.stdout.write('Running AddLinesModule... [DONE]\n')
sys.stdout.flush()
history = f'number of lines = {self.m_lines}'
self.m_image_out_port.add_history('AddLinesModule', history)
self.m_image_out_port.copy_attributes(self.m_image_in_port)
self.m_image_out_port.close_port()
def _stack_subsets(
nimages: int, im_shape: Tuple[int, ...], parang: np.ndarray
) -> Tuple[Tuple[int, ...], np.ndarray, np.ndarray]:
im_new = None
parang_new = None
if self.m_stacking is not None:
if self.m_max_rotation is not None:
frames = stack_angles(self.m_stacking, parang,
self.m_max_rotation)
else:
frames = memory_frames(self.m_stacking, nimages)
nimages_new = np.size(frames) - 1
if parang is None:
parang_new = None
else:
parang_new = np.zeros(nimages_new)
im_new = np.zeros((nimages_new, im_shape[1], im_shape[2]))
start_time = time.time()
for i in range(nimages_new):
progress(i, nimages_new, 'Stacking subsets of images...',
start_time)
if parang is not None:
# parang_new[i] = np.mean(parang[frames[i]:frames[i+1]])
parang_new[i] = angle_average(
parang[frames[i]:frames[i + 1]])
im_subset = self.m_image_in_port[frames[i]:frames[i + 1], ]
if self.m_combine == 'mean':
im_new[i, ] = np.mean(im_subset, axis=0)
elif self.m_combine == 'median':
im_new[i, ] = np.median(im_subset, axis=0)
im_shape = im_new.shape
else:
if parang is not None:
parang_new = np.copy(parang)
return im_shape, im_new, parang_new
def run(self) -> None:
"""
Run method of the module. Creates a master flat with the same shape as the science
image and divides the science images by the flat field.
Returns
-------
NoneType
None
"""
self.m_image_out_port.del_all_attributes()
self.m_image_out_port.del_all_data()
memory = self._m_config_port.get_attribute('MEMORY')
nimages = self.m_image_in_port.get_shape()[0]
frames = memory_frames(memory, nimages)
flat = self.m_flat_in_port.get_all()
master = _master_frame(data=np.mean(flat, axis=0),
im_shape=self.m_image_in_port.get_shape())
# shift all values to greater or equal to +1.0
flat_min = np.amin(master)
master -= flat_min - 1.
# normalization, median value is 1 afterwards
master /= np.median(master)
start_time = time.time()
for i in range(len(frames[:-1])):
progress(i, len(frames[:-1]), 'Running FlatCalibrationModule...',
start_time)
images = self.m_image_in_port[frames[i]:frames[i + 1], ]
self.m_image_out_port.append(images / master, data_dim=3)
sys.stdout.write('Running FlatCalibrationModule... [DONE]\n')
sys.stdout.flush()
history = f'flat_in_tag = {self.m_flat_in_port.tag}'
self.m_image_out_port.add_history('FlatCalibrationModule', history)
self.m_image_out_port.copy_attributes(self.m_image_in_port)
self.m_image_out_port.close_port()
def run(self):
"""
Run method of the module. Rotates all images by a constant angle.
:return: None
"""
self.m_image_out_port.del_all_attributes()
self.m_image_out_port.del_all_data()
if self.m_image_in_port.tag == self.m_image_out_port.tag:
raise ValueError(
"Input and output port should have a different tag.")
memory = self._m_config_port.get_attribute("MEMORY")
ndim = self.m_image_in_port.get_ndim()
nimages = number_images_port(self.m_image_in_port)
frames = memory_frames(memory, nimages)
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), "Running RotateImagesModule...")
if nimages == 1:
images = self.m_image_in_port.get_all()
else:
images = self.m_image_in_port[frames[i]:frames[i + 1], ]
for j in range(frames[i + 1] - frames[i]):
if nimages == 1:
im_tmp = images
else:
im_tmp = images[j, ]
# ndimage.rotate rotates in clockwise direction for positive angles
im_tmp = rotate(im_tmp, self.m_angle, reshape=False)
self.m_image_out_port.append(im_tmp, data_dim=ndim)
sys.stdout.write("Running RotateImagesModule... [DONE]\n")
sys.stdout.flush()
self.m_image_out_port.add_history_information("Images rotated",
self.m_angle)
self.m_image_out_port.copy_attributes_from_input_port(
self.m_image_in_port)
self.m_image_out_port.close_port()
def run(self) -> None:
"""
Run method of the module. Decreases the image size by cropping around an given position.
The module always returns odd-sized images.
Returns
-------
NoneType
None
"""
self.m_image_out_port.del_all_attributes()
self.m_image_out_port.del_all_data()
# Get memory and number of images to split the frames into chunks
memory = self._m_config_port.get_attribute('MEMORY')
nimages = self.m_image_in_port.get_shape()[0]
frames = memory_frames(memory, nimages)
# Convert size parameter from arcseconds to pixels
pixscale = self.m_image_in_port.get_attribute('PIXSCALE')
self.m_size = int(math.ceil(self.m_size / pixscale))
# Crop images chunk by chunk
start_time = time.time()
for i in range(len(frames[:-1])):
# Update progress bar
progress(i, len(frames[:-1]), 'Running CropImagesModule...',
start_time)
# Select and crop images in the current chunk
images = self.m_image_in_port[frames[i]:frames[i + 1], ]
images = crop_image(images, self.m_center, self.m_size, copy=False)
# Write cropped images to output port
self.m_image_out_port.append(images, data_dim=3)
# Update progress bar (cropping of images is finished)
sys.stdout.write('Running CropImagesModule... [DONE]\n')
sys.stdout.flush()
# Save history and copy attributes
history = f'image size [pix] = {self.m_size}'
self.m_image_out_port.add_history('CropImagesModule', history)
self.m_image_out_port.copy_attributes(self.m_image_in_port)
self.m_image_out_port.close_port()
def _initialize(ndim, npix):
if ndim == 2:
nimages = 1
elif ndim == 3:
nimages = self.m_image_in_port.get_shape()[0]
if self.m_stack == 'median':
frames = [0, nimages]
else:
frames = memory_frames(memory, nimages)
if self.m_stack == 'mean':
im_tot = np.zeros((npix, npix))
else:
im_tot = None
return nimages, frames, im_tot
def _initialize(ndim: int,
npix: int) -> Tuple[int, np.ndarray, Optional[np.ndarray]]:
if ndim == 2:
nimages = 1
elif ndim == 3:
nimages = self.m_image_in_port.get_shape()[0]
if self.m_stack == 'median':
frames = np.array([0, nimages])
else:
frames = memory_frames(memory, nimages)
if self.m_stack == 'mean':
im_tot = np.zeros((npix, npix))
else:
im_tot = None
return nimages, frames, im_tot
def run(self) -> None:
"""
Run method of the module. Add the images from the two database tags on a frame-by-frame
basis.
Returns
-------
NoneType
None
"""
self.m_image_out_port.del_all_attributes()
self.m_image_out_port.del_all_data()
if self.m_image_in1_port.get_shape(
) != self.m_image_in2_port.get_shape():
raise ValueError(
'The shape of the two input tags has to be the same.')
nimages = self.m_image_in1_port.get_shape()[0]
memory = self._m_config_port.get_attribute('MEMORY')
frames = memory_frames(memory, nimages)
start_time = time.time()
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), 'Running AddImagesModule...',
start_time)
images1 = self.m_image_in1_port[frames[i]:frames[i + 1], ]
images2 = self.m_image_in2_port[frames[i]:frames[i + 1], ]
self.m_image_out_port.append(self.m_scaling * (images1 + images2))
sys.stdout.write('Running AddImagesModule... [DONE]\n')
sys.stdout.flush()
history = f'scaling = {self.m_scaling}'
self.m_image_out_port.add_history('AddImagesModule', history)
self.m_image_out_port.copy_attributes(self.m_image_in1_port)
self.m_image_out_port.close_port()
def _stack(nimages, im_shape, parang):
im_new = None
parang_new = None
if self.m_stacking is not None:
frames = memory_frames(self.m_stacking, nimages)
nimages_new = np.size(frames) - 1
if parang is None:
parang_new = None
else:
parang_new = np.zeros(nimages_new)
im_new = np.zeros((nimages_new, im_shape[1], im_shape[2]))
start_time = time.time()
for i in range(nimages_new):
progress(i, nimages_new, 'Running StackAndSubsetModule...',
start_time)
if parang is not None:
# parang_new[i] = np.mean(parang[frames[i]:frames[i+1]])
parang_new[i] = _mean_angle(
parang[frames[i]:frames[i + 1]])
im_subset = self.m_image_in_port[frames[i]:frames[i + 1], ]
if self.m_combine == 'mean':
im_new[i, ] = np.mean(im_subset, axis=0)
elif self.m_combine == 'median':
im_new[i, ] = np.median(im_subset, axis=0)
im_shape = im_new.shape
else:
if parang is not None:
parang_new = np.copy(parang)
return im_shape, im_new, parang_new
def run(self):
"""
Run method of the module. Subtracts the images from the second database tag from the images
of the first database tag, on a frame-by-frame basis.
Returns
-------
NoneType
None
"""
self.m_image_out_port.del_all_attributes()
self.m_image_out_port.del_all_data()
if self.m_image_in1_port.get_shape(
) != self.m_image_in2_port.get_shape():
raise ValueError(
"The shape of the two input tags have to be equal.")
memory = self._m_config_port.get_attribute("MEMORY")
nimages = self.m_image_in1_port.get_shape()[0]
frames = memory_frames(memory, nimages)
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), "Running SubtractImagesModule...")
images1 = self.m_image_in1_port[frames[i]:frames[i + 1], ]
images2 = self.m_image_in2_port[frames[i]:frames[i + 1], ]
self.m_image_out_port.append(self.m_scaling * (images1 - images2))
sys.stdout.write("Running SubtractImagesModule... [DONE]\n")
sys.stdout.flush()
history = "scaling = " + str(self.m_scaling)
self.m_image_out_port.add_history("SubtractImagesModule", history)
self.m_image_out_port.copy_attributes(self.m_image_in1_port)
self.m_image_out_port.close_port()
def _initialize():
"""
Internal function to get the number of dimensions and subdivide the images by the
MEMORY attribute.
:return: Number of dimensions and array with subdivision of the images.
:rtype: int, numpy.ndarray
"""
ndim = image_in_port.get_ndim()
if ndim == 2:
nimages = 1
elif ndim == 3:
nimages = image_in_port.get_shape()[0]
if image_out_port is not None and image_out_port.tag != image_in_port.tag:
image_out_port.del_all_attributes()
image_out_port.del_all_data()
frames = memory_frames(memory, nimages)
return ndim, frames
def run(self):
"""
Run method of the module. Add the images from the two database tags on a frame-by-frame
basis.
:return: None
"""
self.m_image_out_port.del_all_attributes()
self.m_image_out_port.del_all_data()
if self.m_image_in1_port.get_shape(
) != self.m_image_in2_port.get_shape():
raise ValueError(
"The shape of the two input tags have to be equal.")
nimages = self.m_image_in1_port.get_shape()[0]
memory = self._m_config_port.get_attribute("MEMORY")
frames = memory_frames(memory, nimages)
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), "Running AddImagesModule...")
images1 = self.m_image_in1_port[frames[i]:frames[i + 1], ]
images2 = self.m_image_in2_port[frames[i]:frames[i + 1], ]
self.m_image_out_port.append(self.m_scaling * (images1 + images2))
sys.stdout.write("Running AddImagesModule... [DONE]\n")
sys.stdout.flush()
self.m_image_out_port.add_history_information("Images added", "")
self.m_image_out_port.copy_attributes_from_input_port(
self.m_image_in1_port)
self.m_image_out_port.close_port()
def run(self) -> None:
"""
Run method of the module. Smooths the images with a Gaussian kernel, locates the brightest
pixel in each image, measures the integrated flux around the brightest pixel, calculates
the median and standard deviation of the photometry, and applies sigma clipping to remove
low quality images.
Returns
-------
NoneType
None
"""
def _get_aperture(aperture):
if aperture[0] == 'circular':
aperture = (0., aperture[1]/pixscale)
elif aperture[0] == 'annulus' or aperture[0] == 'ratio':
aperture = (aperture[1]/pixscale, aperture[2]/pixscale)
return aperture
def _get_starpos(fwhm, position):
starpos = np.zeros((nimages, 2), dtype=np.int64)
if fwhm is None:
starpos[:, 0] = position[0]
starpos[:, 1] = position[1]
else:
if position is None:
center = None
width = None
else:
if position[0] is None and position[1] is None:
center = None
else:
center = position[0:2]
width = int(math.ceil(position[2]/pixscale))
for i, _ in enumerate(starpos):
starpos[i, :] = locate_star(image=self.m_image_in_port[i, ],
center=center,
width=width,
fwhm=int(math.ceil(fwhm/pixscale)))
return starpos
def _photometry(images, starpos, aperture):
check_pos_in = any(np.floor(starpos[:]-aperture[1]) < 0.)
check_pos_out = any(np.ceil(starpos[:]+aperture[1]) > images.shape[0])
if check_pos_in or check_pos_out:
phot = np.nan
else:
im_crop = crop_image(images, tuple(starpos), 2*int(math.ceil(aperture[1])))
npix = im_crop.shape[0]
x_grid = y_grid = np.linspace(-(npix-1)/2, (npix-1)/2, npix)
xx_grid, yy_grid = np.meshgrid(x_grid, y_grid)
rr_grid = np.sqrt(xx_grid*xx_grid+yy_grid*yy_grid)
if self.m_aperture[0] == 'circular':
phot = np.sum(im_crop[rr_grid < aperture[1]])
elif self.m_aperture[0] == 'annulus':
phot = np.sum(im_crop[(rr_grid > aperture[0]) & (rr_grid < aperture[1])])
elif self.m_aperture[0] == 'ratio':
phot = np.sum(im_crop[rr_grid < aperture[0]]) / \
np.sum(im_crop[(rr_grid > aperture[0]) & (rr_grid < aperture[1])])
return phot
self._initialize()
pixscale = self.m_image_in_port.get_attribute('PIXSCALE')
nimages = self.m_image_in_port.get_shape()[0]
aperture = _get_aperture(self.m_aperture)
starpos = _get_starpos(self.m_fwhm, self.m_position)
phot = np.zeros(nimages)
start_time = time.time()
for i in range(nimages):
progress(i, nimages, 'Running FrameSelectionModule...', start_time)
images = self.m_image_in_port[i]
phot[i] = _photometry(images, starpos[i, :], aperture)
if self.m_method == 'median':
phot_ref = np.nanmedian(phot)
elif self.m_method == 'max':
phot_ref = np.nanmax(phot)
phot_std = np.nanstd(phot)
index_rm = np.logical_or((phot > phot_ref+self.m_threshold*phot_std),
(phot < phot_ref-self.m_threshold*phot_std))
index_rm[np.isnan(phot)] = True
indices = np.where(index_rm)[0]
indices = np.asarray(indices, dtype=np.int)
if np.size(indices) > 0:
memory = self._m_config_port.get_attribute('MEMORY')
frames = memory_frames(memory, nimages)
if memory == 0 or memory >= nimages:
memory = nimages
for i, _ in enumerate(frames[:-1]):
images = self.m_image_in_port[frames[i]:frames[i+1], ]
index_del = np.where(np.logical_and(indices >= frames[i],
indices < frames[i+1]))
write_selected_data(images,
indices[index_del] % memory,
self.m_selected_out_port,
self.m_removed_out_port)
else:
warnings.warn('No frames were removed.')
history = 'frames removed = '+str(np.size(indices))
if self.m_index_out_port is not None:
self.m_index_out_port.set_all(np.transpose(indices))
self.m_index_out_port.copy_attributes(self.m_image_in_port)
self.m_index_out_port.add_attribute('STAR_POSITION', starpos, static=False)
self.m_index_out_port.add_history('FrameSelectionModule', history)
if self.m_selected_out_port is not None:
# Copy attributes before write_selected_attributes is used
self.m_selected_out_port.copy_attributes(self.m_image_in_port)
if self.m_removed_out_port is not None:
# Copy attributes before write_selected_attributes is used
self.m_removed_out_port.copy_attributes(self.m_image_in_port)
write_selected_attributes(indices,
self.m_image_in_port,
self.m_selected_out_port,
self.m_removed_out_port)
if self.m_selected_out_port is not None:
indices_select = np.ones(nimages, dtype=bool)
indices_select[indices] = False
indices_select = np.where(indices_select)
self.m_selected_out_port.add_attribute('STAR_POSITION',
starpos[indices_select],
static=False)
self.m_selected_out_port.add_history('FrameSelectionModule', history)
if self.m_removed_out_port is not None:
self.m_removed_out_port.add_attribute('STAR_POSITION',
starpos[indices],
static=False)
self.m_removed_out_port.add_history('FrameSelectionModule', history)
sys.stdout.write('Running FrameSelectionModule... [DONE]\n')
sys.stdout.flush()
self.m_image_in_port.close_port()
def run(self) -> None:
"""
Run method of the module. Selects images according to a specified attribute tag and
ordering, e.g. the highest 150 ``INDEX`` frames, or the lowest 50 ``PCC`` frames.
Returns
-------
NoneType
None
"""
if self.m_selected_out_port is not None:
self.m_selected_out_port.del_all_data()
self.m_selected_out_port.del_all_attributes()
if self.m_removed_out_port is not None:
self.m_removed_out_port.del_all_data()
self.m_removed_out_port.del_all_attributes()
images = self.m_image_in_port.get_all()
nimages = images.shape[0]
attribute = self.m_image_in_port.get_attribute(f'{self.m_attribute_tag}')
if nimages != len(attribute):
raise ValueError(f'The attribute {{self.m_attribute_tag}} does not have the same '
f'length ({len(attribute)}) as the tag has images ({nimages}). '
f'Please check the attribute you have chosen for selection.')
index = self.m_image_in_port.get_attribute('INDEX')
if self.m_order == 'descending':
# sort attribute in descending order
sorting_order = np.argsort(attribute)[::-1]
else:
# sort attribute in ascending order
sorting_order = np.argsort(attribute)
attribute = attribute[sorting_order]
index = index[sorting_order]
indices = index[:self.m_number_frames]
# copied from FrameSelectionModule ...
# possibly refactor to @staticmethod or move to util.remove
start_time = time.time()
if np.size(indices) > 0:
memory = self._m_config_port.get_attribute('MEMORY')
frames = memory_frames(memory, nimages)
if memory == 0 or memory >= nimages:
memory = nimages
for i, _ in enumerate(frames[:-1]):
images = self.m_image_in_port[frames[i]:frames[i+1], ]
index_del = np.where(np.logical_and(indices >= frames[i],
indices < frames[i+1]))
write_selected_data(images,
indices[index_del] % memory,
self.m_removed_out_port,
self.m_selected_out_port)
progress(i, len(frames[:-1]), 'Running SelectByAttributeModule...', start_time)
else:
warnings.warn('No frames were removed.')
if self.m_selected_out_port is not None:
# Copy attributes before write_selected_attributes is used
self.m_selected_out_port.copy_attributes(self.m_image_in_port)
if self.m_removed_out_port is not None:
# Copy attributes before write_selected_attributes is used
self.m_removed_out_port.copy_attributes(self.m_image_in_port)
# write the selected and removed data to the respective output ports
write_selected_attributes(indices,
self.m_image_in_port,
self.m_removed_out_port,
self.m_selected_out_port)
sys.stdout.write('Running SelectByAttributeModule... [DONE]\n')
sys.stdout.flush()
def run(self):
"""
Run method of the module. Combines the frames of multiple tags into a single dataset
and adds the static and non-static attributes. The values of the attributes are compared
between the input tags to make sure that the input tags descent from the same data set.
Returns
-------
NoneType
None
"""
self.m_image_out_port.del_all_data()
self.m_image_out_port.del_all_attributes()
memory = self._m_config_port.get_attribute('MEMORY')
image_in_port = []
im_shape = []
for i, item in enumerate(self.m_image_in_tags):
image_in_port.append(self.add_input_port(item))
im_shape.append(image_in_port[i].get_shape()[-2:])
if len(set(im_shape)) > 1:
raise ValueError(
'The size of the images should be the same for all datasets.')
count = 0
start_time = time.time()
for i, item in enumerate(self.m_image_in_tags):
progress(i, len(self.m_image_in_tags),
'Running CombineTagsModule...', start_time)
nimages = image_in_port[i].get_shape()[0]
frames = memory_frames(memory, nimages)
for j, _ in enumerate(frames[:-1]):
im_tmp = image_in_port[i][frames[j]:frames[j + 1], ]
self.m_image_out_port.append(im_tmp)
if self.m_index_init:
index = np.arange(frames[j], frames[j + 1], 1) + count
if i == 0 and j == 0:
self.m_image_out_port.add_attribute('INDEX',
index,
static=False)
else:
for ind in index:
self.m_image_out_port.append_attribute_data(
'INDEX', ind)
static_attr = image_in_port[i].get_all_static_attributes()
non_static_attr = image_in_port[i].get_all_non_static_attributes()
for key in static_attr:
status = self.m_image_out_port.check_static_attribute(
key, static_attr[key])
if status == 1:
self.m_image_out_port.add_attribute(key,
static_attr[key],
static=True)
elif status == -1 and key[0:7] != 'History':
warnings.warn(
f'The static keyword {key} is already used but with a different '
f'value. It is advisable to only combine tags that descend from '
f'the same data set.')
for key in non_static_attr:
values = image_in_port[i].get_attribute(key)
status = self.m_image_out_port.check_non_static_attribute(
key, values)
if key != 'INDEX' or (key == 'INDEX'
and not self.m_index_init):
if self.m_check_attr:
if key in ('PARANG', 'STAR_POSITION', 'INDEX',
'NFRAMES'):
if status == 1:
self.m_image_out_port.add_attribute(
key, values, static=False)
else:
for j in values:
self.m_image_out_port.append_attribute_data(
key, j)
else:
if status == 1:
self.m_image_out_port.add_attribute(
key, values, static=False)
if status == -1:
warnings.warn(
f'The non-static keyword {key} is already used but '
f'with different values. It is advisable to only '
f'combine tags that descend from the same data set.'
)
else:
if status == 1:
self.m_image_out_port.add_attribute(key,
values,
static=False)
else:
for j in values:
self.m_image_out_port.append_attribute_data(
key, j)
count += nimages
sys.stdout.write('Running CombineTagsModule... [DONE]\n')
sys.stdout.flush()
history = f'number of input tags = {np.size(self.m_image_in_tags)}'
self.m_image_out_port.add_history('CombineTagsModule', history)
self.m_image_out_port.close_port()
def run(self):
"""
Run method of the module. Masks and normalizes the images.
Returns
-------
NoneType
None
"""
self.m_image_out_port.del_all_data()
self.m_image_out_port.del_all_attributes()
if self.m_mask_out_port is not None:
self.m_mask_out_port.del_all_data()
self.m_mask_out_port.del_all_attributes()
# Get PIXSCALE and MEMORY attributes
pixscale = self.m_image_in_port.get_attribute("PIXSCALE")
memory = self._m_config_port.get_attribute("MEMORY")
# Get the number of images and split into batches to comply with memory constraints
im_shape = self.m_image_in_port.get_shape()
nimages = im_shape[0]
frames = memory_frames(memory, nimages)
# Convert m_cent_size and m_edge_size from arcseconds to pixels
if self.m_cent_size is not None:
self.m_cent_size /= pixscale
if self.m_edge_size is not None:
self.m_edge_size /= pixscale
# Create 2D disk mask which will be applied to every frame
mask = create_mask((int(im_shape[-2]), int(im_shape[-1])),
[self.m_cent_size, self.m_edge_size]).astype(bool)
# Keep track of the normalization vectors in case we are normalizing the images (if
# we are not normalizing, this list will remain empty)
norms = list()
# Run the PSFpreparationModule for each subset of frames
for i, _ in enumerate(frames[:-1]):
# Print progress to command line
progress(i, len(frames[:-1]), "Running PSFpreparationModule...")
# Get the images and ensure they have the correct 3D shape with the following
# three dimensions: (batch_size, height, width)
images = self.m_image_in_port[frames[i]:frames[i + 1], ]
if images.ndim == 2:
warnings.warn(
"The input data has 2 dimensions whereas 3 dimensions are required. "
"An extra dimension has been added.")
images = images[np.newaxis, ...]
# Apply the mask, i.e., set all pixels to 0 where the mask is False
images[:, ~mask] = 0.
# If desired, normalize the images using the Frobenius norm
if self.m_norm:
im_norm = np.linalg.norm(images, ord="fro", axis=(1, 2))
images /= im_norm[:, np.newaxis, np.newaxis]
norms.append(im_norm)
# Write processed images to output port
self.m_image_out_port.append(images, data_dim=3)
# Store information about mask
if self.m_mask_out_port is not None:
self.m_mask_out_port.set_all(mask)
self.m_mask_out_port.copy_attributes(self.m_image_in_port)
# Copy attributes from input port
self.m_image_out_port.copy_attributes(self.m_image_in_port)
# If the norms list is not empty (i.e., if we have computed the norm for every image),
# we can also save the corresponding norm vector as an additional attribute
if norms:
self.m_image_out_port.add_attribute(name="norm",
value=np.hstack(norms),
static=False)
# Save cent_size and edge_size as attributes to the output port
if self.m_cent_size is not None:
self.m_image_out_port.add_attribute(name="cent_size",
value=self.m_cent_size *
pixscale,
static=True)
if self.m_edge_size is not None:
self.m_image_out_port.add_attribute(name="edge_size",
value=self.m_edge_size *
pixscale,
static=True)
sys.stdout.write("Running PSFpreparationModule... [DONE]\n")
sys.stdout.flush()
