def run(self) -> None: """ Run method of the module. Reads a list of values from a FITS or ASCII file and writes them as non-static attribute to a dataset. Returns ------- NoneType None """ print('Reading attribute data...', end='') attributes = get_attributes() if self.m_attribute not in attributes: raise ValueError( f'\'{self.m_attribute}\' is not a valid attribute.') if self.m_file_name.endswith('fits'): values = fits.getdata( os.path.join(self.m_input_location, self.m_file_name)) else: values = np.loadtxt(os.path.join(self.m_input_location, self.m_file_name), dtype=attributes[self.m_attribute]['type']) if values.ndim != 1: raise ValueError( f'The input file {self.m_file_name} should contain a 1D list with ' f'attributes.') status = self.m_data_port.check_non_static_attribute( self.m_attribute, values) if status == 1: self.m_data_port.add_attribute(self.m_attribute, values, static=False) elif status == -1 and self.m_overwrite: self.m_data_port.add_attribute(self.m_attribute, values, static=False) elif status == -1 and not self.m_overwrite: warnings.warn( f'The attribute \'{self.m_attribute}\' is already present. Set the ' f'\'overwrite\' parameter to True in order to overwrite the values with ' f'{self.m_file_name}.') elif status == 0: warnings.warn( f'The \'{self.m_attribute}\' attribute is already present and ' f'contains the same values as are present in {self.m_file_name}.' ) print(' [DONE]') self.m_data_port.close_port()
def run(self): """ Run method of the module. Reads a list of values from a text file and writes them as non-static attribute to a dataset. Returns ------- NoneType None """ sys.stdout.write("Running AttributeReadingModule...") sys.stdout.flush() attributes = get_attributes() if self.m_attribute not in attributes: raise ValueError("'%s' is not a valid attribute." % self.m_attribute) values = np.loadtxt(os.path.join(self.m_input_location, self.m_file_name), dtype=attributes[self.m_attribute]["type"]) if values.ndim != 1: raise ValueError( "The input file %s should contain a 1D list with attributes." % self.m_file_name) status = self.m_data_port.check_non_static_attribute( self.m_attribute, values) if status == 1: self.m_data_port.add_attribute(self.m_attribute, values, static=False) elif status == -1 and self.m_overwrite: self.m_data_port.add_attribute(self.m_attribute, values, static=False) elif status == -1 and not self.m_overwrite: warnings.warn( "The attribute '" + str(self.m_attribute) + "' is already present. Set the " "'overwrite' parameter to True in order to overwrite the values with " + str(self.m_file_name) + ".") elif status == 0: warnings.warn("The '" + str(self.m_attribute) + "' attribute is already present and " "contains the same values as are present in " + str(self.m_file_name) + ".") sys.stdout.write(" [DONE]\n") sys.stdout.flush() self.m_data_port.close_port()
def run(self): """ Run method of the module. Reads a list of values from a text file and writes them as non-static attribute to a dataset. Returns ------- NoneType None """ sys.stdout.write('Running AttributeReadingModule...') sys.stdout.flush() attributes = get_attributes() if self.m_attribute not in attributes: raise ValueError( f'\'{self.m_attribute}\' is not a valid attribute.') values = np.loadtxt(os.path.join(self.m_input_location, self.m_file_name), dtype=attributes[self.m_attribute]['type']) if values.ndim != 1: raise ValueError( f'The input file {self.m_file_name} should contain a 1D list with ' f'attributes.') status = self.m_data_port.check_non_static_attribute( self.m_attribute, values) if status == 1: self.m_data_port.add_attribute(self.m_attribute, values, static=False) elif status == -1 and self.m_overwrite: self.m_data_port.add_attribute(self.m_attribute, values, static=False) elif status == -1 and not self.m_overwrite: warnings.warn( f'The attribute \'{self.m_attribute}\' is already present. Set the ' f'\'overwrite\' parameter to True in order to overwrite the values with ' f'{self.m_file_name}.') elif status == 0: warnings.warn( f'The \'{self.m_attribute}\' attribute is already present and ' f'contains the same values as are present in {self.m_file_name}.' ) sys.stdout.write(' [DONE]\n') sys.stdout.flush() self.m_data_port.close_port()
def set_nonstatic_attr(header: fits.header.Header, config_port: ConfigPort, image_out_port: OutputPort, check: bool = True) -> None: """ Function which adds the non-static attributes to the central database. Parameters ---------- header : astropy.io.fits.header.Header Header information from the FITS file that is read. config_port : pynpoint.core.dataio.ConfigPort Configuration port. image_out_port : pynpoint.core.dataio.OutputPort Output port of the images to which the non-static attributes are stored. Returns ------- NoneType None """ attributes = get_attributes() nonstatic = [] for key, value in six.iteritems(attributes): if value['attribute'] == 'non-static': nonstatic.append(key) for attr in nonstatic: if attributes[attr]['config'] == 'header': fitskey = config_port.get_attribute(attr) # if type(fitskey) == np.bytes_: # fitskey = str(fitskey.decode('utf-8')) if fitskey != 'None': if fitskey in header: image_out_port.append_attribute_data(attr, header[fitskey]) elif header['NAXIS'] == 2 and attr == 'NFRAMES': image_out_port.append_attribute_data(attr, 1) elif check: warnings.warn( 'Non-static attribute %s (=%s) not found in the ' 'FITS header.' % (attr, fitskey)) image_out_port.append_attribute_data(attr, -1)
def __init__(self, name_in=None, input_dir=None, image_tag="im_arr", overwrite=True, check=True): """ Constructor of FitsReadingModule. :param name_in: Unique name of the module instance. :type name_in: str :param input_dir: Input directory where the FITS files are located. If not specified the Pypeline default directory is used. :type input_dir: str :param image_tag: Tag of the read data in the HDF5 database. Non static header information is stored with the tag: *header_* + image_tag / header_entry_name. :type image_tag: str :param overwrite: Overwrite existing data and header in the central database. :type overwrite: bool :param check: Check all the listed non-static attributes or ignore the attributes that are not always required (e.g. PARANG_START, DITHER_X). :type check: bool :return: None """ super(FitsReadingModule, self).__init__(name_in, input_dir) self.m_image_out_port = self.add_output_port(image_tag) self.m_overwrite = overwrite self.m_check = check self.m_static = [] self.m_non_static = [] self.m_attributes = get_attributes() for key, value in six.iteritems(self.m_attributes): if value["config"] == "header" and value["attribute"] == "static": self.m_static.append(key) for key, value in six.iteritems(self.m_attributes): if value["attribute"] == "non-static": self.m_non_static.append(key) self.m_count = 0
def _config_init(self): """ Internal function which initializes the configuration file. It reads PynPoint_config.ini in the working folder and creates this file with the default (ESO/NACO) settings in case the file is not present. :return: None """ def _create_config(filename, attributes): file_obj = open(filename, 'w') file_obj.write("[header]\n\n") for key, val in six.iteritems(attributes): if val['config'] == "header": file_obj.write(key + ': ' + str(val['value']) + '\n') file_obj.write("\n[settings]\n\n") for key, val in six.iteritems(attributes): if val['config'] == "settings": file_obj.write(key + ': ' + str(val['value']) + '\n') file_obj.close() def _read_config(config_file, attributes): config = configparser.ConfigParser() config.read_file(open(config_file)) for key, val in six.iteritems(attributes): if config.has_option(val["config"], key): if config.get(val["config"], key) == "None": if val["config"] == "header": attributes[key]["value"] = "None" # elif val["type"] == "str": # attributes[key]["value"] = "None" elif val["type"] == "float": attributes[key]["value"] = float(0.) elif val["type"] == "int": attributes[key]["value"] = int(0) else: if val["config"] == "header": attributes[key]["value"] = str( config.get(val["config"], key)) # elif val["type"] == "str": # attributes[key]["value"] = str(config.get(val["config"], key)) elif val["type"] == "float": attributes[key]["value"] = float( config.get(val["config"], key)) elif val["type"] == "int": attributes[key]["value"] = int( config.get(val["config"], key)) return attributes def _write_config(attributes): hdf = h5py.File(self._m_working_place + '/PynPoint_database.hdf5', 'a') if "config" in hdf: del hdf["config"] config = hdf.create_group("config") for key, _ in six.iteritems(attributes): if attributes[key]["value"] is not None: config.attrs[key] = attributes[key]["value"] config.attrs["WORKING_PLACE"] = self._m_working_place hdf.close() config_file = self._m_working_place + "/PynPoint_config.ini" attributes = get_attributes() attributes["CPU"]["value"] = multiprocessing.cpu_count() if not os.path.isfile(config_file): warnings.warn( "Configuration file not found. Creating PynPoint_config.ini with " "default values in the working place.") _create_config(config_file, attributes) attributes = _read_config(config_file, attributes) _write_config(attributes)
def _config_init(self) -> None: """ Internal function which initializes the configuration file. It reads PynPoint_config.ini in the working folder and creates this file with the default (ESO/NACO) settings in case the file is not present. Returns ------- NoneType None """ @typechecked def _create_config(filename: str, attributes: dict) -> None: file_obj = open(filename, 'w') file_obj.write('[header]\n\n') for key, val in attributes.items(): if val['config'] == 'header': file_obj.write(key + ': ' + str(val['value']) + '\n') file_obj.write('\n[settings]\n\n') for key, val in attributes.items(): if val['config'] == 'settings': file_obj.write(key + ': ' + str(val['value']) + '\n') file_obj.close() @typechecked def _read_config(config_file: str, attributes: dict) -> dict: config = configparser.ConfigParser() with open(config_file) as cf_open: config.read_file(cf_open) for key, val in attributes.items(): if config.has_option(val['config'], key): if config.get(val['config'], key) == 'None': if val['config'] == 'header': attributes[key]['value'] = 'None' # elif val['type'] == 'str': # attributes[key]['value'] = 'None' elif val['type'] == 'float': attributes[key]['value'] = float(0.) elif val['type'] == 'int': attributes[key]['value'] = int(0) else: if val['config'] == 'header': attributes[key]['value'] = str( config.get(val['config'], key)) # elif val['type'] == 'str': # attributes[key]['value'] = str(config.get(val['config'], key)) elif val['type'] == 'float': attributes[key]['value'] = float( config.get(val['config'], key)) elif val['type'] == 'int': attributes[key]['value'] = int( config.get(val['config'], key)) return attributes @typechecked def _write_config(attributes: dict) -> None: hdf = h5py.File(self._m_working_place + '/PynPoint_database.hdf5', 'a') if 'config' in hdf: del hdf['config'] config = hdf.create_group('config') for key in attributes.keys(): if attributes[key]['value'] is not None: config.attrs[key] = attributes[key]['value'] config.attrs['WORKING_PLACE'] = self._m_working_place hdf.close() config_file = os.path.join(self._m_working_place, 'PynPoint_config.ini') print(f'Configuration: {config_file}\n') attributes = get_attributes() attributes['CPU']['value'] = multiprocessing.cpu_count() if not os.path.isfile(config_file): warnings.warn( 'Configuration file not found. Creating PynPoint_config.ini with ' 'default values in the working place.') _create_config(config_file, attributes) attributes = _read_config(config_file, attributes) _write_config(attributes) n_cpu = attributes['CPU']['value'] if 'OMP_NUM_THREADS' in os.environ: n_thread = os.environ['OMP_NUM_THREADS'] else: n_thread = 'not set' print(f'Number of CPUs: {n_cpu}') print(f'Number of threads: {n_thread}')
def set_static_attr(fits_file, header, config_port, image_out_port, check=True): """ Function which adds the static attributes to the central database. Parameters ---------- fits_file : str Name of the FITS file. header : astropy.io.fits.header.Header Header information from the FITS file that is read. config_port : pynpoint.core.dataio.ConfigPort Configuration port. image_out_port : pynpoint.core.dataio.OutputPort Output port of the images to which the static attributes are stored. check : bool Print a warning if certain attributes from the configuration file are not present in the FITS header. If set to `False`, attributes are still written to the dataset but there will be no warning if a keyword is not found in the FITS header. Returns ------- NoneType None """ attributes = get_attributes() static = [] for key, value in six.iteritems(attributes): if value['config'] == 'header' and value['attribute'] == 'static': static.append(key) for attr in static: fitskey = config_port.get_attribute(attr) if isinstance(fitskey, np.bytes_): fitskey = str(fitskey.decode('utf-8')) if fitskey != 'None': if fitskey in header: status = image_out_port.check_static_attribute( attr, header[fitskey]) if status == 1: image_out_port.add_attribute(attr, header[fitskey], static=True) elif status == 0: pass elif status == -1: warnings.warn( f'Static attribute {fitskey} has changed. Possibly the ' f'current file {fits_file} does not belong to the data set ' f'\'{image_out_port.tag}\'. Attribute value is updated.' ) elif check: warnings.warn( f'Static attribute {attr} (={fitskey}) not found in the FITS ' 'header.')
def __init__(self, name_in=None, input_dir=None, image_tag="im_arr", overwrite=True, check=True, filenames=None): """ Constructor of FitsReadingModule. Parameters ---------- name_in : str Unique name of the module instance. input_dir : str Input directory where the FITS files are located. If not specified the Pypeline default directory is used. image_tag : str Tag of the read data in the HDF5 database. Non static header information is stored with the tag: *header_* + image_tag / header_entry_name. overwrite : bool Overwrite existing data and header in the central database. check : bool Check all the listed non-static attributes or ignore the attributes that are not always required (e.g. PARANG_START, DITHER_X). filenames : str or list(str, ) If a string, then a path of a text file should be provided. This text file should contain a list of FITS files. If a list, then the paths of the FITS files should be provided directly. If set to None, the FITS files in the `input_dir` are read. All paths should be provided either relative to the Python working folder (i.e., the folder where Python is executed) or as absolute paths. Returns ------- NoneType None """ super(FitsReadingModule, self).__init__(name_in, input_dir) self.m_image_out_port = self.add_output_port(image_tag) self.m_overwrite = overwrite self.m_check = check self.m_filenames = filenames self.m_static = [] self.m_non_static = [] self.m_attributes = get_attributes() for key, value in six.iteritems(self.m_attributes): if value["config"] == "header" and value["attribute"] == "static": self.m_static.append(key) for key, value in six.iteritems(self.m_attributes): if value["attribute"] == "non-static": self.m_non_static.append(key) self.m_count = 0 if not isinstance(filenames, (type(None), list, tuple, str)): raise TypeError( "The 'filenames' parameter should contain a string or list with " "strings.")