def run(self) -> None:
"""
Run method of the module. Reads a list of values from a FITS or ASCII file and writes them
as non-static attribute to a dataset.
Returns
-------
NoneType
None
"""
print('Reading attribute data...', end='')
attributes = get_attributes()
if self.m_attribute not in attributes:
raise ValueError(
f'\'{self.m_attribute}\' is not a valid attribute.')
if self.m_file_name.endswith('fits'):
values = fits.getdata(
os.path.join(self.m_input_location, self.m_file_name))
else:
values = np.loadtxt(os.path.join(self.m_input_location,
self.m_file_name),
dtype=attributes[self.m_attribute]['type'])
if values.ndim != 1:
raise ValueError(
f'The input file {self.m_file_name} should contain a 1D list with '
f'attributes.')
status = self.m_data_port.check_non_static_attribute(
self.m_attribute, values)
if status == 1:
self.m_data_port.add_attribute(self.m_attribute,
values,
static=False)
elif status == -1 and self.m_overwrite:
self.m_data_port.add_attribute(self.m_attribute,
values,
static=False)
elif status == -1 and not self.m_overwrite:
warnings.warn(
f'The attribute \'{self.m_attribute}\' is already present. Set the '
f'\'overwrite\' parameter to True in order to overwrite the values with '
f'{self.m_file_name}.')
elif status == 0:
warnings.warn(
f'The \'{self.m_attribute}\' attribute is already present and '
f'contains the same values as are present in {self.m_file_name}.'
)
print(' [DONE]')
self.m_data_port.close_port()
def run(self):
"""
Run method of the module. Reads a list of values from a text file and writes them as
non-static attribute to a dataset.
Returns
-------
NoneType
None
"""
sys.stdout.write("Running AttributeReadingModule...")
sys.stdout.flush()
attributes = get_attributes()
if self.m_attribute not in attributes:
raise ValueError("'%s' is not a valid attribute." %
self.m_attribute)
values = np.loadtxt(os.path.join(self.m_input_location,
self.m_file_name),
dtype=attributes[self.m_attribute]["type"])
if values.ndim != 1:
raise ValueError(
"The input file %s should contain a 1D list with attributes." %
self.m_file_name)
status = self.m_data_port.check_non_static_attribute(
self.m_attribute, values)
if status == 1:
self.m_data_port.add_attribute(self.m_attribute,
values,
static=False)
elif status == -1 and self.m_overwrite:
self.m_data_port.add_attribute(self.m_attribute,
values,
static=False)
elif status == -1 and not self.m_overwrite:
warnings.warn(
"The attribute '" + str(self.m_attribute) +
"' is already present. Set the "
"'overwrite' parameter to True in order to overwrite the values with "
+ str(self.m_file_name) + ".")
elif status == 0:
warnings.warn("The '" + str(self.m_attribute) +
"' attribute is already present and "
"contains the same values as are present in " +
str(self.m_file_name) + ".")
sys.stdout.write(" [DONE]\n")
sys.stdout.flush()
self.m_data_port.close_port()
def run(self):
"""
Run method of the module. Reads a list of values from a text file and writes them as
non-static attribute to a dataset.
Returns
-------
NoneType
None
"""
sys.stdout.write('Running AttributeReadingModule...')
sys.stdout.flush()
attributes = get_attributes()
if self.m_attribute not in attributes:
raise ValueError(
f'\'{self.m_attribute}\' is not a valid attribute.')
values = np.loadtxt(os.path.join(self.m_input_location,
self.m_file_name),
dtype=attributes[self.m_attribute]['type'])
if values.ndim != 1:
raise ValueError(
f'The input file {self.m_file_name} should contain a 1D list with '
f'attributes.')
status = self.m_data_port.check_non_static_attribute(
self.m_attribute, values)
if status == 1:
self.m_data_port.add_attribute(self.m_attribute,
values,
static=False)
elif status == -1 and self.m_overwrite:
self.m_data_port.add_attribute(self.m_attribute,
values,
static=False)
elif status == -1 and not self.m_overwrite:
warnings.warn(
f'The attribute \'{self.m_attribute}\' is already present. Set the '
f'\'overwrite\' parameter to True in order to overwrite the values with '
f'{self.m_file_name}.')
elif status == 0:
warnings.warn(
f'The \'{self.m_attribute}\' attribute is already present and '
f'contains the same values as are present in {self.m_file_name}.'
)
sys.stdout.write(' [DONE]\n')
sys.stdout.flush()
self.m_data_port.close_port()
def set_nonstatic_attr(header: fits.header.Header,
config_port: ConfigPort,
image_out_port: OutputPort,
check: bool = True) -> None:
"""
Function which adds the non-static attributes to the central database.
Parameters
----------
header : astropy.io.fits.header.Header
Header information from the FITS file that is read.
config_port : pynpoint.core.dataio.ConfigPort
Configuration port.
image_out_port : pynpoint.core.dataio.OutputPort
Output port of the images to which the non-static attributes are stored.
Returns
-------
NoneType
None
"""
attributes = get_attributes()
nonstatic = []
for key, value in six.iteritems(attributes):
if value['attribute'] == 'non-static':
nonstatic.append(key)
for attr in nonstatic:
if attributes[attr]['config'] == 'header':
fitskey = config_port.get_attribute(attr)
# if type(fitskey) == np.bytes_:
# fitskey = str(fitskey.decode('utf-8'))
if fitskey != 'None':
if fitskey in header:
image_out_port.append_attribute_data(attr, header[fitskey])
elif header['NAXIS'] == 2 and attr == 'NFRAMES':
image_out_port.append_attribute_data(attr, 1)
elif check:
warnings.warn(
'Non-static attribute %s (=%s) not found in the '
'FITS header.' % (attr, fitskey))
image_out_port.append_attribute_data(attr, -1)
def __init__(self,
name_in=None,
input_dir=None,
image_tag="im_arr",
overwrite=True,
check=True):
"""
Constructor of FitsReadingModule.
:param name_in: Unique name of the module instance.
:type name_in: str
:param input_dir: Input directory where the FITS files are located. If not specified the
Pypeline default directory is used.
:type input_dir: str
:param image_tag: Tag of the read data in the HDF5 database. Non static header
information is stored with the tag: *header_* + image_tag /
header_entry_name.
:type image_tag: str
:param overwrite: Overwrite existing data and header in the central database.
:type overwrite: bool
:param check: Check all the listed non-static attributes or ignore the attributes that
are not always required (e.g. PARANG_START, DITHER_X).
:type check: bool
:return: None
"""
super(FitsReadingModule, self).__init__(name_in, input_dir)
self.m_image_out_port = self.add_output_port(image_tag)
self.m_overwrite = overwrite
self.m_check = check
self.m_static = []
self.m_non_static = []
self.m_attributes = get_attributes()
for key, value in six.iteritems(self.m_attributes):
if value["config"] == "header" and value["attribute"] == "static":
self.m_static.append(key)
for key, value in six.iteritems(self.m_attributes):
if value["attribute"] == "non-static":
self.m_non_static.append(key)
self.m_count = 0
def _config_init(self):
"""
Internal function which initializes the configuration file. It reads PynPoint_config.ini
in the working folder and creates this file with the default (ESO/NACO) settings in case
the file is not present.
:return: None
"""
def _create_config(filename, attributes):
file_obj = open(filename, 'w')
file_obj.write("[header]\n\n")
for key, val in six.iteritems(attributes):
if val['config'] == "header":
file_obj.write(key + ': ' + str(val['value']) + '\n')
file_obj.write("\n[settings]\n\n")
for key, val in six.iteritems(attributes):
if val['config'] == "settings":
file_obj.write(key + ': ' + str(val['value']) + '\n')
file_obj.close()
def _read_config(config_file, attributes):
config = configparser.ConfigParser()
config.read_file(open(config_file))
for key, val in six.iteritems(attributes):
if config.has_option(val["config"], key):
if config.get(val["config"], key) == "None":
if val["config"] == "header":
attributes[key]["value"] = "None"
# elif val["type"] == "str":
# attributes[key]["value"] = "None"
elif val["type"] == "float":
attributes[key]["value"] = float(0.)
elif val["type"] == "int":
attributes[key]["value"] = int(0)
else:
if val["config"] == "header":
attributes[key]["value"] = str(
config.get(val["config"], key))
# elif val["type"] == "str":
# attributes[key]["value"] = str(config.get(val["config"], key))
elif val["type"] == "float":
attributes[key]["value"] = float(
config.get(val["config"], key))
elif val["type"] == "int":
attributes[key]["value"] = int(
config.get(val["config"], key))
return attributes
def _write_config(attributes):
hdf = h5py.File(self._m_working_place + '/PynPoint_database.hdf5',
'a')
if "config" in hdf:
del hdf["config"]
config = hdf.create_group("config")
for key, _ in six.iteritems(attributes):
if attributes[key]["value"] is not None:
config.attrs[key] = attributes[key]["value"]
config.attrs["WORKING_PLACE"] = self._m_working_place
hdf.close()
config_file = self._m_working_place + "/PynPoint_config.ini"
attributes = get_attributes()
attributes["CPU"]["value"] = multiprocessing.cpu_count()
if not os.path.isfile(config_file):
warnings.warn(
"Configuration file not found. Creating PynPoint_config.ini with "
"default values in the working place.")
_create_config(config_file, attributes)
attributes = _read_config(config_file, attributes)
_write_config(attributes)
def _config_init(self) -> None:
"""
Internal function which initializes the configuration file. It reads PynPoint_config.ini
in the working folder and creates this file with the default (ESO/NACO) settings in case
the file is not present.
Returns
-------
NoneType
None
"""
@typechecked
def _create_config(filename: str, attributes: dict) -> None:
file_obj = open(filename, 'w')
file_obj.write('[header]\n\n')
for key, val in attributes.items():
if val['config'] == 'header':
file_obj.write(key + ': ' + str(val['value']) + '\n')
file_obj.write('\n[settings]\n\n')
for key, val in attributes.items():
if val['config'] == 'settings':
file_obj.write(key + ': ' + str(val['value']) + '\n')
file_obj.close()
@typechecked
def _read_config(config_file: str, attributes: dict) -> dict:
config = configparser.ConfigParser()
with open(config_file) as cf_open:
config.read_file(cf_open)
for key, val in attributes.items():
if config.has_option(val['config'], key):
if config.get(val['config'], key) == 'None':
if val['config'] == 'header':
attributes[key]['value'] = 'None'
# elif val['type'] == 'str':
# attributes[key]['value'] = 'None'
elif val['type'] == 'float':
attributes[key]['value'] = float(0.)
elif val['type'] == 'int':
attributes[key]['value'] = int(0)
else:
if val['config'] == 'header':
attributes[key]['value'] = str(
config.get(val['config'], key))
# elif val['type'] == 'str':
# attributes[key]['value'] = str(config.get(val['config'], key))
elif val['type'] == 'float':
attributes[key]['value'] = float(
config.get(val['config'], key))
elif val['type'] == 'int':
attributes[key]['value'] = int(
config.get(val['config'], key))
return attributes
@typechecked
def _write_config(attributes: dict) -> None:
hdf = h5py.File(self._m_working_place + '/PynPoint_database.hdf5',
'a')
if 'config' in hdf:
del hdf['config']
config = hdf.create_group('config')
for key in attributes.keys():
if attributes[key]['value'] is not None:
config.attrs[key] = attributes[key]['value']
config.attrs['WORKING_PLACE'] = self._m_working_place
hdf.close()
config_file = os.path.join(self._m_working_place,
'PynPoint_config.ini')
print(f'Configuration: {config_file}\n')
attributes = get_attributes()
attributes['CPU']['value'] = multiprocessing.cpu_count()
if not os.path.isfile(config_file):
warnings.warn(
'Configuration file not found. Creating PynPoint_config.ini with '
'default values in the working place.')
_create_config(config_file, attributes)
attributes = _read_config(config_file, attributes)
_write_config(attributes)
n_cpu = attributes['CPU']['value']
if 'OMP_NUM_THREADS' in os.environ:
n_thread = os.environ['OMP_NUM_THREADS']
else:
n_thread = 'not set'
print(f'Number of CPUs: {n_cpu}')
print(f'Number of threads: {n_thread}')
def set_static_attr(fits_file,
header,
config_port,
image_out_port,
check=True):
"""
Function which adds the static attributes to the central database.
Parameters
----------
fits_file : str
Name of the FITS file.
header : astropy.io.fits.header.Header
Header information from the FITS file that is read.
config_port : pynpoint.core.dataio.ConfigPort
Configuration port.
image_out_port : pynpoint.core.dataio.OutputPort
Output port of the images to which the static attributes are stored.
check : bool
Print a warning if certain attributes from the configuration file are not present in
the FITS header. If set to `False`, attributes are still written to the dataset but
there will be no warning if a keyword is not found in the FITS header.
Returns
-------
NoneType
None
"""
attributes = get_attributes()
static = []
for key, value in six.iteritems(attributes):
if value['config'] == 'header' and value['attribute'] == 'static':
static.append(key)
for attr in static:
fitskey = config_port.get_attribute(attr)
if isinstance(fitskey, np.bytes_):
fitskey = str(fitskey.decode('utf-8'))
if fitskey != 'None':
if fitskey in header:
status = image_out_port.check_static_attribute(
attr, header[fitskey])
if status == 1:
image_out_port.add_attribute(attr,
header[fitskey],
static=True)
elif status == 0:
pass
elif status == -1:
warnings.warn(
f'Static attribute {fitskey} has changed. Possibly the '
f'current file {fits_file} does not belong to the data set '
f'\'{image_out_port.tag}\'. Attribute value is updated.'
)
elif check:
warnings.warn(
f'Static attribute {attr} (={fitskey}) not found in the FITS '
'header.')
def __init__(self,
name_in=None,
input_dir=None,
image_tag="im_arr",
overwrite=True,
check=True,
filenames=None):
"""
Constructor of FitsReadingModule.
Parameters
----------
name_in : str
Unique name of the module instance.
input_dir : str
Input directory where the FITS files are located. If not specified the Pypeline default
directory is used.
image_tag : str
Tag of the read data in the HDF5 database. Non static header information is stored with
the tag: *header_* + image_tag / header_entry_name.
overwrite : bool
Overwrite existing data and header in the central database.
check : bool
Check all the listed non-static attributes or ignore the attributes that are not always
required (e.g. PARANG_START, DITHER_X).
filenames : str or list(str, )
If a string, then a path of a text file should be provided. This text file should
contain a list of FITS files. If a list, then the paths of the FITS files should be
provided directly. If set to None, the FITS files in the `input_dir` are read. All
paths should be provided either relative to the Python working folder (i.e., the folder
where Python is executed) or as absolute paths.
Returns
-------
NoneType
None
"""
super(FitsReadingModule, self).__init__(name_in, input_dir)
self.m_image_out_port = self.add_output_port(image_tag)
self.m_overwrite = overwrite
self.m_check = check
self.m_filenames = filenames
self.m_static = []
self.m_non_static = []
self.m_attributes = get_attributes()
for key, value in six.iteritems(self.m_attributes):
if value["config"] == "header" and value["attribute"] == "static":
self.m_static.append(key)
for key, value in six.iteritems(self.m_attributes):
if value["attribute"] == "non-static":
self.m_non_static.append(key)
self.m_count = 0
if not isinstance(filenames, (type(None), list, tuple, str)):
raise TypeError(
"The 'filenames' parameter should contain a string or list with "
"strings.")
