def add_output_port(self,
tag,
activation=True):
"""
Function which creates an OutputPort for a ReadingModule and appends it to the internal
OutputPort dictionary. This function should be used by classes inheriting from
ReadingModule to make sure that only output ports with unique tags are added. The new
port can be used as: ::
port = self._m_output_ports[tag]
or by using the returned Port.
:param tag: Tag of the new output port.
:type tag: str
:param activation: Activation status of the Port after creation. Deactivated ports
will not save their results until they are activated.
:type activation: bool
:return: The new OutputPort for the ReadingModule.
:rtype: OutputPort
"""
port = OutputPort(tag, activate_init=activation)
if tag in self._m_output_ports:
warnings.warn("Tag '%s' of ReadingModule '%s' is already used."
% (tag, self._m_name))
if self._m_data_base is not None:
port.set_database_connection(self._m_data_base)
self._m_output_ports[tag] = port
return port
def write_selected_data(images: np.ndarray, indices: np.ndarray,
port_selected: OutputPort,
port_removed: OutputPort) -> None:
"""
Function to write a selected number of images from a data set.
Parameters
----------
images : numpy.ndarray
Stack of images.
indices : numpy.ndarray
Indices that are removed.
port_selected : pynpoint.core.dataio.OutputPort
Port to store the selected images.
port_removed : pynpoint.core.dataio.OutputPort
Port to store the removed images.
Returns
-------
NoneType
None
"""
if np.size(indices) > 0:
if port_removed is not None:
port_removed.append(images[indices])
if images.ndim == 2:
images = None
elif images.ndim == 3:
images = np.delete(images, indices, axis=0)
if port_selected is not None and images is not None:
port_selected.append(images)
def apply_function_in_time(self,
func: Callable,
image_in_port: InputPort,
image_out_port: OutputPort,
func_args: Optional[tuple] = None) -> None:
"""
Applies a function to all pixel lines in time.
Parameters
----------
func : function
The input function.
image_in_port : pynpoint.core.dataio.InputPort
Input port which is linked to the input data.
image_out_port : pynpoint.core.dataio.OutputPort
Output port which is linked to the results.
func_args : tuple, None
Additional arguments which are required by the input function. Not used if set to None.
Returns
-------
NoneType
None
"""
cpu = self._m_config_port.get_attribute('CPU')
init_line = image_in_port[:, 0, 0]
im_shape = image_in_port.get_shape()
size = apply_function(init_line, 0, func, func_args).shape[0]
image_out_port.set_all(data=np.zeros((size, im_shape[1], im_shape[2])),
data_dim=3,
keep_attributes=False)
image_in_port.close_port()
image_out_port.close_port()
capsule = LineProcessingCapsule(image_in_port=image_in_port,
image_out_port=image_out_port,
num_proc=cpu,
function=func,
function_args=func_args,
data_length=size)
capsule.run()
def add_output_port(self, tag: str, activation: bool = True) -> OutputPort:
"""
Function which creates an :class:`~pynpoint.core.dataio.OutputPort` for a
:class:`~pynpoint.core.processing.ProcessingModule` and appends it to the internal
:class:`~pynpoint.core.dataio.OutputPort` dictionary. This function should be used by
classes inheriting from :class:`~pynpoint.core.processing.ProcessingModule` to make sure
that only output ports with unique tags are added. The new port can be used as:
.. code-block:: python
port = self._m_output_ports[tag]
or by using the returned :class:`~pynpoint.core.dataio.Port`.
Parameters
----------
tag : str
Tag of the new output port.
activation : bool
Activation status of the :class:`~pynpoint.core.dataio.Port` after creation.
Deactivated ports will not save their results until they are activated.
Returns
-------
pynpoint.core.dataio.OutputPort
The new :class:`~pynpoint.core.dataio.OutputPort` for the
:class:`~pynpoint.core.processing.ProcessingModule`.
"""
port = OutputPort(tag, activate_init=activation)
if tag in self._m_output_ports:
warnings.warn(
f'Tag \'{tag}\' of ProcessingModule \'{self._m_name}\' is already used.'
)
if self._m_data_base is not None:
port.set_database_connection(self._m_data_base)
self._m_output_ports[tag] = port
return port
def test_get_all_attributes(self):
port = InputPort('images', self.storage)
assert port.get_all_static_attributes() == {'PIXSCALE': 0.01}
assert port.get_all_non_static_attributes() == ['PARANG', ]
port = OutputPort('images', self.storage)
assert port.del_all_attributes() is None
port = InputPort('images', self.storage)
assert port.get_all_non_static_attributes() is None
def set_extra_attr(fits_file: str, nimages: int, config_port: ConfigPort,
image_out_port: OutputPort, first_index: int) -> int:
"""
Function which adds extra attributes to the central database.
Parameters
----------
fits_file : str
Absolute path and filename of the FITS file.
nimages : int
Number of images.
config_port : pynpoint.core.dataio.ConfigPort
Configuration port.
image_out_port : pynpoint.core.dataio.OutputPort
Output port of the images to which the attributes are stored.
first_index : int
First image index of the current subset.
Returns
-------
int
First image index for the next subset.
"""
pixscale = config_port.get_attribute('PIXSCALE')
image_index = np.arange(first_index, first_index + nimages, 1)
for item in image_index:
image_out_port.append_attribute_data('INDEX', item)
image_out_port.append_attribute_data('FILES', fits_file)
image_out_port.add_attribute('PIXSCALE', pixscale, static=True)
return first_index + nimages
def add_output_port(self,
tag,
activation=True):
"""
Function which creates an OutputPort for a ReadingModule and appends it to the internal
OutputPort dictionary. This function should be used by classes inheriting from
ReadingModule to make sure that only output ports with unique tags are added. The new
port can be used as: ::
port = self._m_output_ports[tag]
or by using the returned Port.
Parameters
----------
tag : str
Tag of the new output port.
activation : bool
Activation status of the Port after creation. Deactivated ports will not save their
results until they are activated.
Returns
-------
pynpoint.core.dataio.OutputPort
The new OutputPort for the ReadingModule.
"""
port = OutputPort(tag, activate_init=activation)
if tag in self._m_output_ports:
warnings.warn(f'Tag \'{tag}\' of ReadingModule \'{self._m_name}\' is already used.')
if self._m_data_base is not None:
port.set_database_connection(self._m_data_base)
self._m_output_ports[tag] = port
return port
def set_nonstatic_attr(header: fits.header.Header,
config_port: ConfigPort,
image_out_port: OutputPort,
check: bool = True) -> None:
"""
Function which adds the non-static attributes to the central database.
Parameters
----------
header : astropy.io.fits.header.Header
Header information from the FITS file that is read.
config_port : pynpoint.core.dataio.ConfigPort
Configuration port.
image_out_port : pynpoint.core.dataio.OutputPort
Output port of the images to which the non-static attributes are stored.
Returns
-------
NoneType
None
"""
attributes = get_attributes()
nonstatic = []
for key, value in six.iteritems(attributes):
if value['attribute'] == 'non-static':
nonstatic.append(key)
for attr in nonstatic:
if attributes[attr]['config'] == 'header':
fitskey = config_port.get_attribute(attr)
# if type(fitskey) == np.bytes_:
# fitskey = str(fitskey.decode('utf-8'))
if fitskey != 'None':
if fitskey in header:
image_out_port.append_attribute_data(attr, header[fitskey])
elif header['NAXIS'] == 2 and attr == 'NFRAMES':
image_out_port.append_attribute_data(attr, 1)
elif check:
warnings.warn(
'Non-static attribute %s (=%s) not found in the '
'FITS header.' % (attr, fitskey))
image_out_port.append_attribute_data(attr, -1)
def set_static_attr(fits_file: str,
header: fits.header.Header,
config_port: ConfigPort,
image_out_port: OutputPort,
check: bool = True) -> None:
"""
Function which adds the static attributes to the central database.
Parameters
----------
fits_file : str
Name of the FITS file.
header : astropy.io.fits.header.Header
Header information from the FITS file that is read.
config_port : pynpoint.core.dataio.ConfigPort
Configuration port.
image_out_port : pynpoint.core.dataio.OutputPort
Output port of the images to which the static attributes are stored.
check : bool
Print a warning if certain attributes from the configuration file are not present in
the FITS header. If set to `False`, attributes are still written to the dataset but
there will be no warning if a keyword is not found in the FITS header.
Returns
-------
NoneType
None
"""
attributes = get_attributes()
static = []
for key, value in six.iteritems(attributes):
if value['config'] == 'header' and value['attribute'] == 'static':
static.append(key)
for attr in static:
fitskey = config_port.get_attribute(attr)
if isinstance(fitskey, np.bytes_):
fitskey = str(fitskey.decode('utf-8'))
if fitskey != 'None':
if fitskey in header:
status = image_out_port.check_static_attribute(
attr, header[fitskey])
if status == 1:
image_out_port.add_attribute(attr,
header[fitskey],
static=True)
elif status == 0:
pass
elif status == -1:
warnings.warn(
f'Static attribute {fitskey} has changed. Possibly the '
f'current file {fits_file} does not belong to the data set '
f'\'{image_out_port.tag}\'. Attribute value is updated.'
)
elif check:
warnings.warn(
f'Static attribute {attr} (={fitskey}) not found in the FITS '
'header.')
def apply_function_to_images(self,
func: Callable[..., np.ndarray],
image_in_port: InputPort,
image_out_port: OutputPort,
message: str,
func_args: Optional[tuple] = None) -> None:
"""
Function which applies a function to all images of an input port. Stacks of images are
processed in parallel if the CPU and MEMORY attribute are set in the central configuration.
The number of images per process is equal to the value of MEMORY divided by the value of
CPU. Note that the function *func* is not allowed to change the shape of the images if the
input and output port have the same tag and ``MEMORY`` is not set to None.
Parameters
----------
func : function
The function which is applied to all images. Its definitions should be similar to::
def function(image_in,
parameter1,
parameter2,
parameter3)
The function must return a numpy array.
image_in_port : pynpoint.core.dataio.InputPort
Input port which is linked to the input data.
image_out_port : pynpoint.core.dataio.OutputPort
Output port which is linked to the results.
message : str
Progress message.
func_args : tuple
Additional arguments that are required by the input function.
Returns
-------
NoneType
None
"""
memory = self._m_config_port.get_attribute('MEMORY')
cpu = self._m_config_port.get_attribute('CPU')
nimages = image_in_port.get_shape()[0]
if memory == 0:
memory = nimages
if image_out_port.tag == image_in_port.tag:
# load all images in the memory at once if the input and output tag are the
# same or if the MEMORY attribute is set to None in the configuration file
images = image_in_port.get_all()
result = []
start_time = time.time()
for i in range(nimages):
progress(i, nimages, message + '...', start_time)
args = update_arguments(i, nimages, func_args)
if args is None:
result.append(func(images[i, ], i))
else:
result.append(func(images[i, ], i, *args))
image_out_port.set_all(np.asarray(result), keep_attributes=True)
elif cpu == 1:
# process images one-by-one with a single process if CPU is set to 1
start_time = time.time()
for i in range(nimages):
progress(i, nimages, message + '...', start_time)
args = update_arguments(i, nimages, func_args)
if args is None:
result = func(image_in_port[i, ], i)
else:
result = func(image_in_port[i, ], i, *args)
if result.ndim == 1:
image_out_port.append(result, data_dim=2)
elif result.ndim == 2:
image_out_port.append(result, data_dim=3)
else:
# process images in parallel in stacks of MEMORY/CPU images
print(message, end='')
args = update_arguments(0, nimages, func_args)
result = apply_function(image_in_port[0, :, :], 0, func, args)
result_shape = result.shape
out_shape = [nimages]
for item in result_shape:
out_shape.append(item)
image_out_port.set_all(data=np.zeros(out_shape),
data_dim=len(result_shape) + 1,
keep_attributes=False)
image_in_port.close_port()
image_out_port.close_port()
capsule = StackProcessingCapsule(image_in_port=image_in_port,
image_out_port=image_out_port,
num_proc=cpu,
function=func,
function_args=func_args,
stack_size=math.ceil(memory /
cpu),
result_shape=result_shape,
nimages=nimages)
capsule.run()
print(' [DONE]')
def write_selected_attributes(indices: np.ndarray, port_input: InputPort,
port_selected: OutputPort,
port_removed: OutputPort) -> None:
"""
Function to write the attributes of a selected number of images.
Parameters
----------
indices : numpy.ndarray
Indices that are removed.
port_input : pynpoint.core.dataio.InputPort
Port to the input data.
port_selected : pynpoint.core.dataio.OutputPort
Port to store the attributes of the selected images.
port_removed : pynpoint.core.dataio.OutputPort
Port to store the attributes of the removed images. Not written if set to None.
Returns
-------
NoneType
None
"""
non_static = port_input.get_all_non_static_attributes()
for i, item in enumerate(non_static):
values = port_input.get_attribute(item)
if values.shape[0] == port_input.get_shape()[0]:
if port_selected is not None:
if np.size(indices) > 0:
if values.ndim == 1:
selected = np.delete(values, indices)
elif values.ndim == 2:
selected = np.delete(values, indices, axis=0)
else:
selected = values
port_selected.add_attribute(item, selected, static=False)
if port_removed is not None and np.size(indices) > 0:
removed = values[indices]
port_removed.add_attribute(item, removed, static=False)
if 'NFRAMES' in non_static:
nframes = port_input.get_attribute('NFRAMES')
nframes_sel = np.zeros(nframes.shape, dtype=np.int)
nframes_del = np.zeros(nframes.shape, dtype=np.int)
for i, frames in enumerate(nframes):
total = np.sum(nframes[0:i])
if np.size(indices) > 0:
index_del = np.where(
np.logical_and(indices >= total,
indices < total + frames))[0]
nframes_sel[i] = frames - np.size(index_del)
nframes_del[i] = np.size(index_del)
else:
nframes_sel[i] = frames
nframes_del[i] = 0
if port_selected is not None:
port_selected.add_attribute('NFRAMES', nframes_sel, static=False)
if port_removed is not None:
port_removed.add_attribute('NFRAMES', nframes_del, static=False)
def test_create_instance(self):
with pytest.raises(ValueError) as error:
OutputPort("config", self.storage)
assert str(error.value) == "The tag name 'config' is reserved for the central " \
"configuration of PynPoint."
with pytest.raises(ValueError) as error:
OutputPort("fits_header", self.storage)
assert str(error.value) == "The tag name 'fits_header' is reserved for storage of the " \
"FITS headers."
active_port = OutputPort("test", self.storage, activate_init=True)
deactive_port = OutputPort("test", self.storage, activate_init=False)
control_port = InputPort("test", self.storage)
deactive_port.open_port()
deactive_port.set_all(np.asarray([0, 1, 2, 3]))
deactive_port.flush()
with pytest.warns(UserWarning) as warning:
control_port.get_all()
assert len(warning) == 1
assert warning[0].message.args[0] == "No data under the tag which is linked by the " \
"InputPort."
active_port.set_all(np.asarray([0, 1, 2, 3]))
active_port.flush()
assert np.array_equal(np.asarray([0, 1, 2, 3]), control_port.get_all())
active_port.del_all_data()
def create_output_port(self, tag_name):
outport = OutputPort(tag_name, self.storage)
return outport
def test_set_all_error(self):
# ---- Test database not set -----
data = [1, 2, 3, 4, 0]
with pytest.warns(UserWarning) as record:
out_port = OutputPort("some_data")
out_port.set_all(data)
assert len(record) == 1
assert record[0].message.args[0] == "OutputPort can not store data unless a database is " \
"connected."
# ---- Test data dim of actual data for new data entry is < 1 or > 3
out_port = self.create_output_port("new_data")
data = [
[
[
[2, 2],
],
],
]
with pytest.raises(ValueError) as error:
out_port.set_all(data, data_dim=2)
assert str(error.value) == 'Output port can only save numpy arrays from 1D to 3D. Use ' \
'Port attributes to save as int, float, or string.'
# ---- Test data dim of data_dim for new data entry is < 1 or > 3
out_port = self.create_output_port("new_data")
data = [1, 2, 4]
with pytest.raises(ValueError) as error:
out_port.set_all(data, data_dim=0)
assert str(
error.value) == 'The data dimensions should be 1D, 2D, or 3D.'
# ---- Test data_dim for new data entry is smaller than actual data
out_port = self.create_output_port("new_data")
data = [[1], [2]]
with pytest.raises(ValueError) as error:
out_port.set_all(data, data_dim=1)
assert str(error.value) == 'The dimensions of the data should be equal to or larger ' \
'than the dimensions of the input data.'
# ---- Test data_dim == 3 and actual size == 1
out_port = self.create_output_port("new_data")
data = [1, 2]
with pytest.raises(ValueError) as error:
out_port.set_all(data, data_dim=3)
assert str(
error.value) == 'Cannot initialize 1D data in 3D data container.'
