def add_input_port(self,
tag):
"""
Function which creates an InputPort for a WritingModule and appends it to the internal
InputPort dictionary. This function should be used by classes inheriting from WritingModule
to make sure that only input ports with unique tags are added. The new port can be used
as: ::
port = self._m_input_ports[tag]
or by using the returned Port.
:param tag: Tag of the new input port.
:type tag: str
:return: The new InputPort for the WritingModule.
:rtype: InputPort
"""
port = InputPort(tag)
if self._m_data_base is not None:
port.set_database_connection(self._m_data_base)
self._m_input_ports[tag] = port
return port
def test_create_instance(self):
with pytest.raises(ValueError) as error:
OutputPort("config", self.storage)
assert str(error.value) == "The tag name 'config' is reserved for the central " \
"configuration of PynPoint."
with pytest.raises(ValueError) as error:
OutputPort("fits_header", self.storage)
assert str(error.value) == "The tag name 'fits_header' is reserved for storage of the " \
"FITS headers."
active_port = OutputPort("test", self.storage, activate_init=True)
deactive_port = OutputPort("test", self.storage, activate_init=False)
control_port = InputPort("test", self.storage)
deactive_port.open_port()
deactive_port.set_all(np.asarray([0, 1, 2, 3]))
deactive_port.flush()
with pytest.warns(UserWarning) as warning:
control_port.get_all()
assert len(warning) == 1
assert warning[0].message.args[0] == "No data under the tag which is linked by the " \
"InputPort."
active_port.set_all(np.asarray([0, 1, 2, 3]))
active_port.flush()
assert np.array_equal(np.asarray([0, 1, 2, 3]), control_port.get_all())
active_port.del_all_data()
def add_input_port(self,
tag):
"""
Function which creates an InputPort for a ProcessingModule and appends it to the internal
InputPort dictionary. This function should be used by classes inheriting from
ProcessingModule to make sure that only input ports with unique tags are added. The new
port can be used as: ::
port = self._m_input_ports[tag]
or by using the returned Port.
Parameters
----------
tag : str
Tag of the new input port.
Returns
-------
pynpoint.core.dataio.InputPort
The new InputPort for the ProcessingModule.
"""
port = InputPort(tag)
if self._m_data_base is not None:
port.set_database_connection(self._m_data_base)
self._m_input_ports[tag] = port
return port
def test_create_instance_access_data(self) -> None:
with pytest.raises(ValueError) as error:
InputPort('config', self.storage)
assert str(error.value) == 'The tag name \'config\' is reserved for the central ' \
'configuration of PynPoint.'
with pytest.raises(ValueError) as error:
InputPort('fits_header', self.storage)
assert str(error.value) == 'The tag name \'fits_header\' is reserved for storage of the ' \
'FITS headers.'
port = InputPort('images', self.storage)
assert port[0, 0, 0] == pytest.approx(0.00032486907273264834, rel=self.limit, abs=0.)
data = np.mean(port.get_all())
assert data == pytest.approx(1.1824138000882435e-05, rel=self.limit, abs=0.)
assert len(port[0:2, 0, 0]) == 2
assert port.get_shape() == (5, 11, 11)
assert port.get_attribute('PIXSCALE') == 0.01
assert port.get_attribute('PARANG')[0] == 0.
with pytest.warns(UserWarning):
assert port.get_attribute('none') is None
def test_create_instance_access_data(self):
with pytest.raises(ValueError) as error:
InputPort('config', self.storage)
assert str(error.value) == 'The tag name \'config\' is reserved for the central ' \
'configuration of PynPoint.'
with pytest.raises(ValueError) as error:
InputPort('fits_header', self.storage)
assert str(error.value) == 'The tag name \'fits_header\' is reserved for storage of the ' \
'FITS headers.'
port = InputPort('images', self.storage)
assert np.allclose(port[0, 0, 0], 0.00032486907273264834, rtol=limit, atol=0.)
assert np.allclose(np.mean(port.get_all()), 1.0506056979365338e-06, rtol=limit, atol=0.)
arr_tmp = np.asarray((0.00032486907273264834, -2.4494781298462809e-05,
-0.00038631277795631806), dtype=np.float64)
assert np.allclose(port[0:3, 0, 0], arr_tmp, rtol=limit, atol=0.)
assert len(port[0:2, 0, 0]) == 2
assert port.get_shape() == (10, 100, 100)
assert port.get_attribute('PIXSCALE') == 0.01
assert port.get_attribute('PARANG')[0] == 1
with pytest.warns(UserWarning):
assert port.get_attribute('none') is None
def test_get_ndim(self):
with pytest.warns(UserWarning) as warning:
ndim = InputPort('images', None).get_ndim()
assert len(warning) == 1
assert warning[0].message.args[0] == "InputPort can not load data unless a database is " \
"connected."
assert ndim is None
port = InputPort('images', self.storage)
assert port.get_ndim() == 3
def test_get_all_attributes(self):
port = InputPort('images', self.storage)
assert port.get_all_static_attributes() == {'PIXSCALE': 0.01}
assert port.get_all_non_static_attributes() == ['PARANG', ]
port = OutputPort('images', self.storage)
assert port.del_all_attributes() is None
port = InputPort('images', self.storage)
assert port.get_all_non_static_attributes() is None
def apply_function_in_time(self,
func: Callable,
image_in_port: InputPort,
image_out_port: OutputPort,
func_args: Optional[tuple] = None) -> None:
"""
Applies a function to all pixel lines in time.
Parameters
----------
func : function
The input function.
image_in_port : pynpoint.core.dataio.InputPort
Input port which is linked to the input data.
image_out_port : pynpoint.core.dataio.OutputPort
Output port which is linked to the results.
func_args : tuple, None
Additional arguments which are required by the input function. Not used if set to None.
Returns
-------
NoneType
None
"""
cpu = self._m_config_port.get_attribute('CPU')
init_line = image_in_port[:, 0, 0]
im_shape = image_in_port.get_shape()
size = apply_function(init_line, 0, func, func_args).shape[0]
image_out_port.set_all(data=np.zeros((size, im_shape[1], im_shape[2])),
data_dim=3,
keep_attributes=False)
image_in_port.close_port()
image_out_port.close_port()
capsule = LineProcessingCapsule(image_in_port=image_in_port,
image_out_port=image_out_port,
num_proc=cpu,
function=func,
function_args=func_args,
data_length=size)
capsule.run()
def subtract_psf(image: np.ndarray,
im_index: int,
parang_thres: Optional[float],
nref: Optional[int],
reference: Optional[np.ndarray],
ang_diff: np.ndarray,
image_in_port: InputPort) -> np.ndarray:
if parang_thres:
index_thres = np.where(ang_diff > parang_thres)[0]
if index_thres.size == 0:
reference = image_in_port.get_all()
warnings.warn('No images meet the rotation threshold. Creating a reference '
'PSF from the median of all images instead.')
else:
if nref:
index_diff = np.abs(im_index - index_thres)
index_near = np.argsort(index_diff)[:nref]
index_sort = np.sort(index_thres[index_near])
reference = image_in_port[index_sort, :, :]
else:
reference = image_in_port[index_thres, :, :]
reference = np.median(reference, axis=0)
return image-reference
def star_positions(
input_port: InputPort,
fwhm: Optional[int],
position: Optional[Union[Tuple[int, int, float], Tuple[None, None, float],
Tuple[int, int, None]]] = None
) -> np.ndarray:
"""
Function to return the position of the star in each image.
Parameters
----------
input_port : pynpoint.core.dataio.InputPort
Input port where the images are stored.
fwhm : int, None
The FWHM (pix) of the Gaussian kernel that is used to smooth the images before the
brightest pixel is located. No smoothing is applied if set to None.
position : tuple(int, int, int), None
Subframe that is selected to search for the star. The tuple contains the center (pix)
and size (pix) (pos_x, pos_y, size). Setting `position` to None will use the full
image to search for the star. If `position=(None, None, size)` then the center of the
image will be used. If `position=(pos_x, pos_y, None)` then a fixed position is used
for the aperture.
Returns
-------
numpy.ndarray
Positions (y, x) of the brightest pixel.
"""
nimages = input_port.get_shape()[0]
starpos = np.zeros((nimages, 2), dtype=np.int64)
if position is not None and position[2] is None:
# [y. x] position
starpos[:, 0] = position[1]
starpos[:, 1] = position[0]
else:
center = None
width = None
if position is not None:
width = position[2]
if position[0] is not None and position[1] is not None:
center = position[0:2]
start_time = time.time()
for i in range(nimages):
progress(i, nimages, 'Locating stellar position...', start_time)
# [y. x] position
starpos[i, :] = locate_star(input_port[i, ], center, width, fwhm)
return starpos
def write_selected_data(memory: Union[int, np.int64],
indices: np.ndarray,
image_in_port: InputPort,
selected_out_port: Optional[OutputPort],
removed_out_port: Optional[OutputPort]) -> None:
"""
Function to write the selected and removed data.
Parameters
----------
memory : int
Number of images that is simultaneously loaded into the memory.
indices : numpy.ndarray
Image indices that will be removed.
image_in_port : pynpoint.core.dataio.InputPort
Port to the input images.
selected_out_port : pynpoint.core.dataio.OutputPort, None
Port to store the selected images. No data is written if set to None.
removed_out_port : pynpoint.core.dataio.OutputPort, None
Port to store the removed images. No data is written if set to None.
Returns
-------
NoneType
None
"""
nimages = image_in_port.get_shape()[0]
frames = memory_frames(memory, nimages)
if memory == 0 or memory >= nimages:
memory = nimages
start_time = time.time()
for i, _ in enumerate(frames[:-1]):
progress(i, len(frames[:-1]), 'Writing selected data...', start_time)
images = image_in_port[frames[i]:frames[i+1], ]
subset_del = np.where(np.logical_and(indices >= frames[i], indices < frames[i+1]))[0]
index_del = indices[subset_del] % memory
index_sel = np.ones(images.shape[0], np.bool)
index_sel[index_del] = False
if selected_out_port is not None and index_sel.size > 0:
selected_out_port.append(images[index_sel])
if removed_out_port is not None and index_del.size > 0:
removed_out_port.append(images[index_del])
def test_create_instance_access_non_existing_data(self):
port = InputPort("test", self.storage)
with pytest.warns(UserWarning):
assert port[0, 0, 0] is None
with pytest.warns(UserWarning):
assert port.get_all() is None
with pytest.warns(UserWarning):
assert port.get_shape() is None
with pytest.warns(UserWarning):
assert port.get_attribute("num_files") is None
with pytest.warns(UserWarning):
assert port.get_all_non_static_attributes() is None
with pytest.warns(UserWarning):
assert port.get_all_static_attributes() is None
def test_create_instance_no_data_storage(self):
port = InputPort("test")
with pytest.warns(UserWarning):
assert port[0, 0, 0] is None
with pytest.warns(UserWarning):
assert port.get_all() is None
with pytest.warns(UserWarning):
assert port.get_shape() is None
with pytest.warns(UserWarning):
assert port.get_all_non_static_attributes() is None
with pytest.warns(UserWarning):
assert port.get_all_static_attributes() is None
def create_input_port(self, tag_name):
inport = InputPort(tag_name, self.storage)
inport.open_port()
return inport
def apply_function_to_images(self,
func: Callable[..., np.ndarray],
image_in_port: InputPort,
image_out_port: OutputPort,
message: str,
func_args: Optional[tuple] = None) -> None:
"""
Function which applies a function to all images of an input port. Stacks of images are
processed in parallel if the CPU and MEMORY attribute are set in the central configuration.
The number of images per process is equal to the value of MEMORY divided by the value of
CPU. Note that the function *func* is not allowed to change the shape of the images if the
input and output port have the same tag and ``MEMORY`` is not set to None.
Parameters
----------
func : function
The function which is applied to all images. Its definitions should be similar to::
def function(image_in,
parameter1,
parameter2,
parameter3)
The function must return a numpy array.
image_in_port : pynpoint.core.dataio.InputPort
Input port which is linked to the input data.
image_out_port : pynpoint.core.dataio.OutputPort
Output port which is linked to the results.
message : str
Progress message.
func_args : tuple
Additional arguments that are required by the input function.
Returns
-------
NoneType
None
"""
memory = self._m_config_port.get_attribute('MEMORY')
cpu = self._m_config_port.get_attribute('CPU')
nimages = image_in_port.get_shape()[0]
if memory == 0:
memory = nimages
if image_out_port.tag == image_in_port.tag:
# load all images in the memory at once if the input and output tag are the
# same or if the MEMORY attribute is set to None in the configuration file
images = image_in_port.get_all()
result = []
start_time = time.time()
for i in range(nimages):
progress(i, nimages, message + '...', start_time)
args = update_arguments(i, nimages, func_args)
if args is None:
result.append(func(images[i, ], i))
else:
result.append(func(images[i, ], i, *args))
image_out_port.set_all(np.asarray(result), keep_attributes=True)
elif cpu == 1:
# process images one-by-one with a single process if CPU is set to 1
start_time = time.time()
for i in range(nimages):
progress(i, nimages, message + '...', start_time)
args = update_arguments(i, nimages, func_args)
if args is None:
result = func(image_in_port[i, ], i)
else:
result = func(image_in_port[i, ], i, *args)
if result.ndim == 1:
image_out_port.append(result, data_dim=2)
elif result.ndim == 2:
image_out_port.append(result, data_dim=3)
else:
# process images in parallel in stacks of MEMORY/CPU images
print(message, end='')
args = update_arguments(0, nimages, func_args)
result = apply_function(image_in_port[0, :, :], 0, func, args)
result_shape = result.shape
out_shape = [nimages]
for item in result_shape:
out_shape.append(item)
image_out_port.set_all(data=np.zeros(out_shape),
data_dim=len(result_shape) + 1,
keep_attributes=False)
image_in_port.close_port()
image_out_port.close_port()
capsule = StackProcessingCapsule(image_in_port=image_in_port,
image_out_port=image_out_port,
num_proc=cpu,
function=func,
function_args=func_args,
stack_size=math.ceil(memory /
cpu),
result_shape=result_shape,
nimages=nimages)
capsule.run()
print(' [DONE]')
def write_selected_attributes(indices: np.ndarray, port_input: InputPort,
port_selected: OutputPort,
port_removed: OutputPort) -> None:
"""
Function to write the attributes of a selected number of images.
Parameters
----------
indices : numpy.ndarray
Indices that are removed.
port_input : pynpoint.core.dataio.InputPort
Port to the input data.
port_selected : pynpoint.core.dataio.OutputPort
Port to store the attributes of the selected images.
port_removed : pynpoint.core.dataio.OutputPort
Port to store the attributes of the removed images. Not written if set to None.
Returns
-------
NoneType
None
"""
non_static = port_input.get_all_non_static_attributes()
for i, item in enumerate(non_static):
values = port_input.get_attribute(item)
if values.shape[0] == port_input.get_shape()[0]:
if port_selected is not None:
if np.size(indices) > 0:
if values.ndim == 1:
selected = np.delete(values, indices)
elif values.ndim == 2:
selected = np.delete(values, indices, axis=0)
else:
selected = values
port_selected.add_attribute(item, selected, static=False)
if port_removed is not None and np.size(indices) > 0:
removed = values[indices]
port_removed.add_attribute(item, removed, static=False)
if 'NFRAMES' in non_static:
nframes = port_input.get_attribute('NFRAMES')
nframes_sel = np.zeros(nframes.shape, dtype=np.int)
nframes_del = np.zeros(nframes.shape, dtype=np.int)
for i, frames in enumerate(nframes):
total = np.sum(nframes[0:i])
if np.size(indices) > 0:
index_del = np.where(
np.logical_and(indices >= total,
indices < total + frames))[0]
nframes_sel[i] = frames - np.size(index_del)
nframes_del[i] = np.size(index_del)
else:
nframes_sel[i] = frames
nframes_del[i] = 0
if port_selected is not None:
port_selected.add_attribute('NFRAMES', nframes_sel, static=False)
if port_removed is not None:
port_removed.add_attribute('NFRAMES', nframes_del, static=False)
def write_selected_attributes(indices: np.ndarray,
image_in_port: InputPort,
selected_out_port: Optional[OutputPort],
removed_out_port: Optional[OutputPort],
module_type: str,
history: str) -> None:
"""
Function to write the attributes of the selected and removed data.
Parameters
----------
indices : numpy.ndarray
Image indices that will be removed.
image_in_port : pynpoint.core.dataio.InputPort
Port to the input data.
selected_out_port : pynpoint.core.dataio.OutputPort, None
Port to store the attributes of the selected images. Not written if set to None.
removed_out_port : pynpoint.core.dataio.OutputPort, None
Port to store the attributes of the removed images. Not written if set to None.
module_type : str
history : str
Returns
-------
NoneType
None
"""
if selected_out_port is not None:
# First copy the existing attributes to the selected_out_port
selected_out_port.copy_attributes(image_in_port)
selected_out_port.add_history(module_type, history)
if removed_out_port is not None:
# First copy the existing attributes to the removed_out_port
removed_out_port.copy_attributes(image_in_port)
removed_out_port.add_history(module_type, history)
non_static = image_in_port.get_all_non_static_attributes()
index_sel = np.ones(image_in_port.get_shape()[0], np.bool)
index_sel[indices] = False
for i, attr_item in enumerate(non_static):
values = image_in_port.get_attribute(attr_item)
if values.shape[0] == image_in_port.get_shape()[0]:
if selected_out_port is not None and index_sel.size > 0:
selected_out_port.add_attribute(attr_item, values[index_sel], static=False)
if removed_out_port is not None and indices.size > 0:
removed_out_port.add_attribute(attr_item, values[indices], static=False)
if 'NFRAMES' in non_static:
nframes = image_in_port.get_attribute('NFRAMES')
nframes_sel = np.zeros(nframes.shape, dtype=np.int)
nframes_del = np.zeros(nframes.shape, dtype=np.int)
for i, frames in enumerate(nframes):
if indices.size == 0:
nframes_sel[i] = frames
nframes_del[i] = 0
else:
sum_n = np.sum(nframes[:i])
index_del = np.where(np.logical_and(indices >= sum_n, indices < sum_n+frames))[0]
nframes_sel[i] = frames - index_del.size
nframes_del[i] = index_del.size
if selected_out_port is not None:
selected_out_port.add_attribute('NFRAMES', nframes_sel, static=False)
if removed_out_port is not None:
removed_out_port.add_attribute('NFRAMES', nframes_del, static=False)
