def __init__(self, working_place_in: Optional[str] = None, input_place_in: Optional[str] = None, output_place_in: Optional[str] = None) -> None: """ Constructor of Pypeline. Parameters ---------- working_place_in : str Working location of the Pypeline which needs to be a folder on the hard drive. The given folder will be used to save the central PynPoint database (an HDF5 file) in which all the intermediate processing steps are saved. Note that the HDF5 file can become very large depending on the size and number of input images. input_place_in : str Default input directory of the Pypeline. All ReadingModules added to the Pypeline use this directory to look for input data. It is possible to specify a different location for the ReadingModules using their constructors. output_place_in : str Default result directory used to save the output of all WritingModules added to the Pypeline. It is possible to specify a different locations for the WritingModules by using their constructors. Returns ------- NoneType None """ pynpoint_version = 'PynPoint v' + pynpoint.__version__ print(len(pynpoint_version) * '=') print(pynpoint_version) print(len(pynpoint_version) * '=' + '\n') try: contents = urllib.request.urlopen( 'https://pypi.org/pypi/pynpoint/json').read() data = json.loads(contents) latest_version = data['info']['version'] except URLError: latest_version = None if latest_version is not None and pynpoint.__version__ != latest_version: print(f'A new version ({latest_version}) is available!\n') self._m_working_place = working_place_in self._m_input_place = input_place_in self._m_output_place = output_place_in self._m_modules = collections.OrderedDict() self.m_data_storage = DataStorage( os.path.join(working_place_in, 'PynPoint_database.hdf5')) print(f'Database: {self.m_data_storage._m_location}') self._config_init()
def test_create_config_port(self) -> None: storage = DataStorage(self.test_dir + 'PynPoint_database.hdf5') with pytest.raises(ValueError) as error: ConfigPort('images', storage) assert str(error.value) == 'The tag name of the central configuration should be ' \ '\'config\'.' port = ConfigPort('config', None) with pytest.warns(UserWarning) as warning: check_error = port._check_error_cases() assert len(warning) == 1 assert warning[0].message.args[0] == 'ConfigPort can not load data unless a database is ' \ 'connected.' assert not check_error port = ConfigPort('config', storage) assert isinstance(port, ConfigPort) with pytest.warns(UserWarning) as warning: port._check_error_cases() assert len(warning) == 1 assert warning[0].message.args[0] == 'No data under the tag which is linked by the ' \ 'ConfigPort.'
def test_create_storage_with_existing_database(self): np.random.seed(1) images = np.random.normal(loc=0, scale=2e-4, size=(10, 100, 100)) h5f = h5py.File(self.test_data, "w") h5f.create_dataset("images", data=images) h5f.close() storage = DataStorage(self.test_data) storage.open_connection() data = storage.m_data_bank["images"] assert np.allclose(data[0, 0, 0], 0.00032486907273264834, rtol=limit, atol=0.) assert np.allclose(np.mean(data), 1.0506056979365338e-06, rtol=limit, atol=0.) os.remove(self.test_data)
def test_get_config_attribute(self) -> None: create_config(self.test_dir + 'PynPoint_config.ini') Pypeline(self.test_dir, self.test_dir, self.test_dir) storage = DataStorage(self.test_dir + 'PynPoint_database.hdf5') port = ConfigPort('config', None) with pytest.warns(UserWarning) as warning: attribute = port.get_attribute('CPU') assert len(warning) == 1 assert warning[0].message.args[0] == 'ConfigPort can not load data unless a database is ' \ 'connected.' assert attribute is None port = ConfigPort('config', storage) attribute = port.get_attribute('CPU') assert attribute == 1 attribute = port.get_attribute('NFRAMES') assert attribute == 'NAXIS3' attribute = port.get_attribute('PIXSCALE') assert np.allclose(attribute, 0.027, rtol=limit, atol=0.) with pytest.warns(UserWarning) as warning: attribute = port.get_attribute('test') assert len(warning) == 1 assert warning[0].message.args[ 0] == 'The attribute \'test\' was not found.' assert attribute is None
def test_open_close_connection(self): storage = DataStorage(self.test_data) storage.open_connection() assert storage.m_open is True storage.open_connection() assert storage.m_open is True storage.close_connection() assert storage.m_open is False storage.close_connection() assert storage.m_open is False os.remove(self.test_data)
def test_get_config_attribute(self): create_config(self.test_dir + "PynPoint_config.ini") Pypeline(self.test_dir, self.test_dir, self.test_dir) storage = DataStorage(self.test_dir + "PynPoint_database.hdf5") port = ConfigPort("config", None) with pytest.warns(UserWarning) as warning: attribute = port.get_attribute("CPU") assert len(warning) == 1 assert warning[0].message.args[0] == "ConfigPort can not load data unless a database is " \ "connected." assert attribute is None port = ConfigPort("config", storage) attribute = port.get_attribute("CPU") assert attribute == 1 attribute = port.get_attribute("NFRAMES") assert attribute == "NAXIS3" attribute = port.get_attribute("PIXSCALE") assert np.allclose(attribute, 0.027, rtol=limit, atol=0.) with pytest.warns(UserWarning) as warning: attribute = port.get_attribute("test") assert len(warning) == 1 assert warning[0].message.args[ 0] == "No attribute found - requested: test." assert attribute is None
def test_create_config_port(self): storage = DataStorage(self.test_dir + "PynPoint_database.hdf5") with pytest.raises(ValueError) as error: ConfigPort("images", storage) assert str( error.value ) == "The tag name of the central configuration should be 'config'." port = ConfigPort("config", None) with pytest.warns(UserWarning) as warning: check_error = port._check_error_cases() assert len(warning) == 1 assert warning[0].message.args[0] == "ConfigPort can not load data unless a database is " \ "connected." assert not check_error port = ConfigPort("config", storage) assert isinstance(port, ConfigPort) with pytest.warns(UserWarning) as warning: port._check_error_cases() assert len(warning) == 1 assert warning[0].message.args[ 0] == "No data under the tag which is linked by the ConfigPort."
def test_create_storage_with_existing_database(self) -> None: np.random.seed(1) images = np.random.normal(loc=0, scale=2e-4, size=(10, 100, 100)) with h5py.File(self.test_data, 'w') as hdf_file: hdf_file.create_dataset('images', data=images) storage = DataStorage(self.test_data) storage.open_connection() data = storage.m_data_bank['images'] assert data[0, 0, 0] == pytest.approx(0.00032486907273264834, rel=self.limit, abs=0.) assert np.mean(data) == pytest.approx(1.0506056979365338e-06, rel=self.limit, abs=0.) os.remove(self.test_data)
def __init__(self, working_place_in=None, input_place_in=None, output_place_in=None): """ Constructor of Pypeline. Parameters ---------- working_place_in : str Working location of the Pypeline which needs to be a folder on the hard drive. The given folder will be used to save the central PynPoint database (an HDF5 file) in which all the intermediate processing steps are saved. Note that the HDF5 file can become very large depending on the size and number of input images. input_place_in : str Default input directory of the Pypeline. All ReadingModules added to the Pypeline use this directory to look for input data. It is possible to specify a different location for the ReadingModules using their constructors. output_place_in : str Default result directory used to save the output of all WritingModules added to the Pypeline. It is possible to specify a different locations for the WritingModules by using their constructors. Returns ------- NoneType None """ pynpoint_version = 'PynPoint v' + pynpoint.__version__ print(len(pynpoint_version) * '=') print(pynpoint_version) print(len(pynpoint_version) * '=' + '\n') self._m_working_place = working_place_in self._m_input_place = input_place_in self._m_output_place = output_place_in self._m_modules = collections.OrderedDict() self.m_data_storage = DataStorage( os.path.join(working_place_in, 'PynPoint_database.hdf5')) print(f'Database: {self.m_data_storage._m_location}') self._config_init()
def __init__(self, working_place_in=None, input_place_in=None, output_place_in=None): """ Constructor of Pypeline. :param working_place_in: Working location of the Pypeline which needs to be a folder on the hard drive. The given folder will be used to save the central PynPoint database (an HDF5 file) in which all the intermediate processing steps are saved. Note that the HDF5 file can become very large depending on the size and number of input images. :type working_place_in: str :param input_place_in: Default input directory of the Pypeline. All ReadingModules added to the Pypeline use this directory to look for input data. It is possible to specify a different location for the ReadingModules using their constructors. :type input_place_in: str :param output_place_in: Default result directory used to save the output of all WritingModules added to the Pypeline. It is possible to specify a different locations for the WritingModules by using their constructors. :return: None """ sys.stdout.write("Initiating PynPoint v" + pynpoint.__version__ + "...") sys.stdout.flush() self._m_working_place = working_place_in self._m_input_place = input_place_in self._m_output_place = output_place_in self._m_modules = collections.OrderedDict() self.m_data_storage = DataStorage( os.path.join(working_place_in, 'PynPoint_database.hdf5')) self._config_init() sys.stdout.write(" [DONE]\n") sys.stdout.flush()
def test_create_storage_without_existing_database(self): storage = DataStorage(self.test_data) storage.open_connection() storage.m_data_bank["data"] = [0, 1, 2, 5, 7] assert storage.m_data_bank["data"][2] == 2 assert list(storage.m_data_bank.keys()) == ["data", ] storage.close_connection() os.remove(self.test_data)
def setup(self): file_in = os.path.dirname(__file__) + "/PynPoint_database.hdf5" self.storage = DataStorage(file_in)
def setup(self): self.storage = DataStorage( os.path.dirname(__file__) + '/PynPoint_database.hdf5')
def test_create_storage_with_wrong_location(self): file_in = "/test/test.hdf5" with pytest.raises(AssertionError): DataStorage(file_in)
def setup(self) -> None: file_in = os.path.dirname(__file__) + '/PynPoint_database.hdf5' self.storage = DataStorage(file_in)
class Pypeline: """ A Pypeline instance can be used to manage various processing steps. It inheres an internal dictionary of Pypeline steps (modules) and their names. A Pypeline has a central DataStorage on the hard drive which can be accessed by various modules. The order of the modules depends on the order the steps have been added to the pypeline. It is possible to run all modules attached to the Pypeline at once or run a single modules by name. """ def __init__(self, working_place_in=None, input_place_in=None, output_place_in=None): """ Constructor of Pypeline. Parameters ---------- working_place_in : str Working location of the Pypeline which needs to be a folder on the hard drive. The given folder will be used to save the central PynPoint database (an HDF5 file) in which all the intermediate processing steps are saved. Note that the HDF5 file can become very large depending on the size and number of input images. input_place_in : str Default input directory of the Pypeline. All ReadingModules added to the Pypeline use this directory to look for input data. It is possible to specify a different location for the ReadingModules using their constructors. output_place_in : str Default result directory used to save the output of all WritingModules added to the Pypeline. It is possible to specify a different locations for the WritingModules by using their constructors. Returns ------- NoneType None """ sys.stdout.write('Initiating PynPoint v'+pynpoint.__version__+'...') sys.stdout.flush() self._m_working_place = working_place_in self._m_input_place = input_place_in self._m_output_place = output_place_in self._m_modules = collections.OrderedDict() self.m_data_storage = DataStorage(os.path.join(working_place_in, 'PynPoint_database.hdf5')) self._config_init() sys.stdout.write(' [DONE]\n') sys.stdout.flush() def __setattr__(self, key, value): """ This method is called every time a member / attribute of the Pypeline is changed. It checks whether a chosen working / input / output directory exists. Parameters ---------- key : str Member or attribute name. value : str New value for the given member or attribute. Returns ------- NoneType None """ if key in ['_m_working_place', '_m_input_place', '_m_output_place']: assert (os.path.isdir(str(value))), f'Input directory for {key} does not exist - ' \ f'input requested: {value}.' super(Pypeline, self).__setattr__(key, value) @staticmethod def _validate(module, tags): """ Internal function which is used for the validation of the pipeline. Validates a single module. Parameters ---------- module : ReadingModule, WritingModule, or ProcessingModule The pipeline module. tags : list(str, ) Tags in the database. Returns ------- bool Module validation. str Module name. """ if isinstance(module, ReadingModule): tags.extend(module.get_all_output_tags()) elif isinstance(module, WritingModule): for tag in module.get_all_input_tags(): if tag not in tags: return False, module.name elif isinstance(module, ProcessingModule): tags.extend(module.get_all_output_tags()) for tag in module.get_all_input_tags(): if tag not in tags: return False, module.name else: return False, None return True, None def _config_init(self): """ Internal function which initializes the configuration file. It reads PynPoint_config.ini in the working folder and creates this file with the default (ESO/NACO) settings in case the file is not present. Returns ------- NoneType None """ def _create_config(filename, attributes): file_obj = open(filename, 'w') file_obj.write('[header]\n\n') for key, val in attributes.items(): if val['config'] == 'header': file_obj.write(key+': '+str(val['value'])+'\n') file_obj.write('\n[settings]\n\n') for key, val in attributes.items(): if val['config'] == 'settings': file_obj.write(key+': '+str(val['value'])+'\n') file_obj.close() def _read_config(config_file, attributes): config = configparser.ConfigParser() with open(config_file) as cf_open: config.read_file(cf_open) for key, val in attributes.items(): if config.has_option(val['config'], key): if config.get(val['config'], key) == 'None': if val['config'] == 'header': attributes[key]['value'] = 'None' # elif val['type'] == 'str': # attributes[key]['value'] = 'None' elif val['type'] == 'float': attributes[key]['value'] = float(0.) elif val['type'] == 'int': attributes[key]['value'] = int(0) else: if val['config'] == 'header': attributes[key]['value'] = str(config.get(val['config'], key)) # elif val['type'] == 'str': # attributes[key]['value'] = str(config.get(val['config'], key)) elif val['type'] == 'float': attributes[key]['value'] = float(config.get(val['config'], key)) elif val['type'] == 'int': attributes[key]['value'] = int(config.get(val['config'], key)) return attributes def _write_config(attributes): hdf = h5py.File(self._m_working_place+'/PynPoint_database.hdf5', 'a') if 'config' in hdf: del hdf['config'] config = hdf.create_group('config') for key in attributes.keys(): if attributes[key]['value'] is not None: config.attrs[key] = attributes[key]['value'] config.attrs['WORKING_PLACE'] = self._m_working_place hdf.close() config_file = self._m_working_place+'/PynPoint_config.ini' attributes = get_attributes() attributes['CPU']['value'] = multiprocessing.cpu_count() if not os.path.isfile(config_file): warnings.warn('Configuration file not found. Creating PynPoint_config.ini with ' 'default values in the working place.') _create_config(config_file, attributes) attributes = _read_config(config_file, attributes) _write_config(attributes) def add_module(self, module): """ Adds a Pypeline module to the internal Pypeline dictionary. The module is appended at the end of this ordered dictionary. If the input module is a reading or writing module without a specified input or output location then the Pypeline default location is used. Moreover, the given module is connected to the Pypeline internal data storage. Parameters ---------- module : ReadingModule, WritingModule, or ProcessingModule Input pipeline module. Returns ------- NoneType None """ assert isinstance(module, PypelineModule), 'The added module is not a valid ' \ 'Pypeline module.' if isinstance(module, WritingModule): if module.m_output_location is None: module.m_output_location = self._m_output_place if isinstance(module, ReadingModule): if module.m_input_location is None: module.m_input_location = self._m_input_place module.connect_database(self.m_data_storage) if module.name in self._m_modules: warnings.warn(f'Pipeline module names need to be unique. Overwriting module ' f'\'{module.name}\'.') self._m_modules[module.name] = module def remove_module(self, name): """ Removes a Pypeline module from the internal dictionary. Parameters ---------- name : str Name of the module that has to be removed. Returns ------- bool Confirmation of removal. """ if name in self._m_modules: del self._m_modules[name] removed = True else: warnings.warn(f'Module name \'{name}\' not found in the Pypeline dictionary.') removed = False return removed def get_module_names(self): """ Function which returns a list of all module names. Returns ------- list(str, ) Ordered list of all Pypeline modules. """ return list(self._m_modules.keys()) def validate_pipeline(self): """ Function which checks if all input ports of the Pypeline are pointing to previous output ports. Returns ------- bool Confirmation of pipeline validation. str Module name that is not valid. """ self.m_data_storage.open_connection() data_tags = list(self.m_data_storage.m_data_bank.keys()) for module in self._m_modules.values(): validation = self._validate(module, data_tags) if not validation[0]: break else: validation = True, None return validation def validate_pipeline_module(self, name): """ Checks if the data exists for the module with label *name*. Parameters ---------- name : str Name of the module that is checked. Returns ------- bool Confirmation of pipeline module validation. str Module name that is not valid. """ self.m_data_storage.open_connection() existing_data_tags = list(self.m_data_storage.m_data_bank.keys()) if name in self._m_modules: module = self._m_modules[name] validate = self._validate(module, existing_data_tags) else: validate = None return validate def run(self): """ Walks through all saved processing steps and calls their run methods. The order in which the steps are called depends on the order they have been added to the Pypeline. Returns ------- NoneType None """ sys.stdout.write('Validating Pypeline...') sys.stdout.flush() validation = self.validate_pipeline() if not validation[0]: raise AttributeError(f'Pipeline module \'{validation[1]}\' is looking for data ' f'under a tag which is not created by a previous module or ' f'does not exist in the database.') sys.stdout.write(' [DONE]\n') sys.stdout.flush() for key in self._m_modules: self._m_modules[key].run() def run_module(self, name): """ Runs a single processing module. Parameters ---------- name : str Name of the pipeline module. Returns ------- NoneType None """ if name in self._m_modules: sys.stdout.write(f'Validating module \'{name}\'...') sys.stdout.flush() validation = self.validate_pipeline_module(name) if not validation[0]: raise AttributeError(f'Pipeline module \'{validation[1]}\' is looking for data ' f'under a tag which does not exist in the database.') sys.stdout.write(' [DONE]\n') sys.stdout.flush() self._m_modules[name].run() else: warnings.warn(f'Module \'{name}\' not found.') def get_data(self, tag): """ Function for accessing data in the central database. Parameters ---------- tag : str Database tag. Returns ------- numpy.ndarray The selected dataset from the database. """ self.m_data_storage.open_connection() data = np.asarray(self.m_data_storage.m_data_bank[tag]) self.m_data_storage.close_connection() return data def delete_data(self, tag): """ Function for deleting a dataset and related attributes from the central database. Disk space does not seem to free up when using this function. Parameters ---------- tag : str Database tag. Returns ------- NoneType None """ self.m_data_storage.open_connection() if tag in self.m_data_storage.m_data_bank: del self.m_data_storage.m_data_bank[tag] else: warnings.warn(f'Dataset \'{tag}\' not found in the database.') if 'header_' + tag + '/' in self.m_data_storage.m_data_bank: del self.m_data_storage.m_data_bank[f'header_{tag}'] else: warnings.warn(f'Attributes of \'{tag}\' not found in the database.') self.m_data_storage.close_connection() def get_attribute(self, data_tag, attr_name, static=True): """ Function for accessing attributes in the central database. Parameters ---------- data_tag : str Database tag. attr_name : str Name of the attribute. static : bool Static or non-static attribute. Returns ------- numpy.ndarray The attribute values. """ self.m_data_storage.open_connection() if static: attr = self.m_data_storage.m_data_bank[data_tag].attrs[attr_name] else: attr = self.m_data_storage.m_data_bank[f'header_{data_tag}/{attr_name}'] attr = np.asarray(attr) self.m_data_storage.close_connection() return attr def set_attribute(self, data_tag, attr_name, attr_value, static=True): """ Function for writing attributes to the central database. Existing values will be overwritten. Parameters ---------- data_tag : str Database tag. attr_name : str Name of the attribute. attr_value : float, int, str, tuple, or numpy.ndarray Attribute value. static : bool Static or non-static attribute. Returns ------- NoneType None """ self.m_data_storage.open_connection() if static: self.m_data_storage.m_data_bank[data_tag].attrs[attr_name] = attr_value else: if isinstance(attr_value[0], str): attr_value = np.array(attr_value, dtype='|S') if attr_name in list(self.m_data_storage.m_data_bank[f'header_{data_tag}'].keys()): del self.m_data_storage.m_data_bank[f'header_{data_tag}/{attr_name}'] attr_key = f'header_{data_tag}/{attr_name}' self.m_data_storage.m_data_bank[attr_key] = np.asarray(attr_value) self.m_data_storage.close_connection() def get_tags(self): """ Function for listing the database tags, ignoring header and config tags. Returns ------- numpy.asarray Database tags. """ self.m_data_storage.open_connection() tags = list(self.m_data_storage.m_data_bank.keys()) select = [] for item in tags: if item in ('config', 'fits_header') or item[0:7] == 'header_': continue else: select.append(item) self.m_data_storage.close_connection() return np.asarray(select) def get_shape(self, tag): """ Function for getting the shape of a database entry. Parameters ---------- tag : str Database tag. Returns ------- tuple(int, ) Dataset shape. """ self.m_data_storage.open_connection() data_shape = self.m_data_storage.m_data_bank[tag].shape self.m_data_storage.close_connection() return data_shape
class Pypeline: """ The :class:`~pynpoint.core.pypeline.Pypeline` class manages the pipeline modules. It inheres an internal dictionary of pipeline modules and has a :class:`~pynpoint.core.dataio.DataStorage` which is accessed by the various modules. The order in which the pipeline modules are executed depends on the order they have been added to the :class:`~pynpoint.core.pypeline.Pypeline`. It is possible to run all modules at once or run a single module by name. """ @typechecked def __init__(self, working_place_in: Optional[str] = None, input_place_in: Optional[str] = None, output_place_in: Optional[str] = None) -> None: """ Parameters ---------- working_place_in : str, None Working location where the central HDF5 database and the configuration file will be stored. Sufficient space is required in the working folder since each pipeline module stores a dataset in the HDF5 database. The current working folder of Python is used as working folder if the argument is set to None. input_place_in : str, None Default input folder where a :class:`~pynpoint.core.processing.ReadingModule` that is added to the :class:`~pynpoint.core.pypeline.Pypeline` will look for input data. The current working folder of Python is used as input folder if the argument is set to None. output_place_in : str, None Default output folder where a :class:`~pynpoint.core.processing.WritingModule` that is added to the :class:`~pynpoint.core.pypeline.Pypeline` will store output data. The current working folder of Python is used as output folder if the argument is set to None. Returns ------- NoneType None """ pynpoint_version = 'PynPoint v' + pynpoint.__version__ print(len(pynpoint_version) * '=') print(pynpoint_version) print(len(pynpoint_version) * '=' + '\n') try: contents = urllib.request.urlopen( 'https://pypi.org/pypi/pynpoint/json').read() data = json.loads(contents) latest_version = data['info']['version'] except URLError: latest_version = None if latest_version is not None and pynpoint.__version__ != latest_version: print(f'A new version ({latest_version}) is available!\n') print( 'Want to stay informed about updates, bug fixes, and new features?' ) print( 'Please consider using the \'Watch\' button on the Github page:' ) print('https://github.com/PynPoint/PynPoint\n') if working_place_in is None: self._m_working_place = os.getcwd() else: self._m_working_place = working_place_in if input_place_in is None: self._m_input_place = os.getcwd() else: self._m_input_place = input_place_in if output_place_in is None: self._m_output_place = os.getcwd() else: self._m_output_place = output_place_in print(f'Working place: {self._m_working_place}') print(f'Input place: {self._m_input_place}') print(f'Output place: {self._m_output_place}\n') self._m_modules = collections.OrderedDict() hdf5_path = os.path.join(self._m_working_place, 'PynPoint_database.hdf5') self.m_data_storage = DataStorage(hdf5_path) print(f'Database: {self.m_data_storage._m_location}') self._config_init() @typechecked def __setattr__(self, key: str, value: Any) -> None: """ Internal method which assigns a value to an object attribute. This method is called whenever and attribute of the :class:`~pynpoint.core.pypeline.Pypeline` is changed and checks if the chosen working, input, or output folder exists. Parameters ---------- key : str Attribute name. value : str Value for the attribute. Returns ------- NoneType None """ if key == '_m_working_place': error_msg = f'The folder that was chosen for the working place does not exist: {value}.' assert os.path.isdir(str(value)), error_msg elif key == '_m_input_place': error_msg = f'The folder that was chosen for the input place does not exist: {value}.' assert os.path.isdir(str(value)), error_msg elif key == '_m_output_place': error_msg = f'The folder that was chosen for the output place does not exist: {value}.' assert os.path.isdir(str(value)), error_msg super().__setattr__(key, value) @staticmethod @typechecked def _validate(module: Union[ReadingModule, WritingModule, ProcessingModule], tags: List[str]) -> Tuple[bool, Optional[str]]: """ Internal method to validate a :class:`~pynpoint.core.processing.PypelineModule`. Parameters ---------- module : ReadingModule, WritingModule, ProcessingModule Pipeline module that will be validated. tags : list(str) Tags that are present in the database. Returns ------- bool Validation of the pipeline module. str, None Pipeline module name in case it is not valid. Returns None if the module was validated. """ if isinstance(module, ReadingModule): tags.extend(module.get_all_output_tags()) elif isinstance(module, WritingModule): for tag in module.get_all_input_tags(): if tag not in tags: return False, module.name elif isinstance(module, ProcessingModule): tags.extend(module.get_all_output_tags()) for tag in module.get_all_input_tags(): if tag not in tags: return False, module.name return True, None @typechecked def _config_init(self) -> None: """ Internal method to initialize the configuration file. The configuration parameters are read from *PynPoint_config.ini* in the working folder. The file is created with default values (ESO/NACO) in case the file is not present. Returns ------- NoneType None """ @typechecked def _create_config(filename: str, attributes: dict) -> None: file_obj = open(filename, 'w') file_obj.write('[header]\n\n') for key, val in attributes.items(): if val['config'] == 'header': file_obj.write(key + ': ' + str(val['value']) + '\n') file_obj.write('\n[settings]\n\n') for key, val in attributes.items(): if val['config'] == 'settings': file_obj.write(key + ': ' + str(val['value']) + '\n') file_obj.close() @typechecked def _read_config(config_file: str, attributes: dict) -> dict: config = configparser.ConfigParser() with open(config_file) as cf_open: config.read_file(cf_open) for key, val in attributes.items(): if config.has_option(val['config'], key): if config.get(val['config'], key) == 'None': if val['config'] == 'header': attributes[key]['value'] = 'None' # elif val['type'] == 'str': # attributes[key]['value'] = 'None' elif val['type'] == 'float': attributes[key]['value'] = float(0.) elif val['type'] == 'int': attributes[key]['value'] = int(0) else: if val['config'] == 'header': attributes[key]['value'] = str( config.get(val['config'], key)) # elif val['type'] == 'str': # attributes[key]['value'] = str(config.get(val['config'], key)) elif val['type'] == 'float': attributes[key]['value'] = float( config.get(val['config'], key)) elif val['type'] == 'int': attributes[key]['value'] = int( config.get(val['config'], key)) return attributes @typechecked def _write_config(attributes: dict) -> None: hdf = h5py.File(self._m_working_place + '/PynPoint_database.hdf5', 'a') if 'config' in hdf: del hdf['config'] config = hdf.create_group('config') for key in attributes.keys(): if attributes[key]['value'] is not None: config.attrs[key] = attributes[key]['value'] config.attrs['WORKING_PLACE'] = self._m_working_place hdf.close() config_file = os.path.join(self._m_working_place, 'PynPoint_config.ini') print(f'Configuration: {config_file}\n') attributes = get_attributes() attributes['CPU']['value'] = multiprocessing.cpu_count() if not os.path.isfile(config_file): warnings.warn( 'Configuration file not found. Creating PynPoint_config.ini with ' 'default values in the working place.') _create_config(config_file, attributes) attributes = _read_config(config_file, attributes) _write_config(attributes) n_cpu = attributes['CPU']['value'] if 'OMP_NUM_THREADS' in os.environ: n_thread = os.environ['OMP_NUM_THREADS'] else: n_thread = 'not set' print(f'Number of CPUs: {n_cpu}') print(f'Number of threads: {n_thread}') @typechecked def add_module(self, module: PypelineModule) -> None: """ Method for adding a :class:`~pynpoint.core.processing.PypelineModule` to the internal dictionary of the :class:`~pynpoint.core.pypeline.Pypeline`. The module is appended at the end of this ordered dictionary. If the input module is a reading or writing module without a specified input or output location then the default location is used. The module is connected to the internal data storage of the :class:`~pynpoint.core.pypeline.Pypeline`. Parameters ---------- module : ReadingModule, WritingModule, ProcessingModule Pipeline module that will be added to the :class:`~pynpoint.core.pypeline.Pypeline`. Returns ------- NoneType None """ if isinstance(module, ReadingModule): if module.m_input_location is None: module.m_input_location = self._m_input_place if isinstance(module, WritingModule): if module.m_output_location is None: module.m_output_location = self._m_output_place module.connect_database(self.m_data_storage) if module.name in self._m_modules: warnings.warn( f'Names of pipeline modules that are added to the Pypeline need to ' f'be unique. The current pipeline module, \'{module.name}\', does ' f'already exist in the Pypeline dictionary so the previous module ' f'with the same name will be overwritten.') self._m_modules[module.name] = module @typechecked def remove_module(self, name: str) -> bool: """ Method to remove a :class:`~pynpoint.core.processing.PypelineModule` from the internal dictionary with pipeline modules that are added to the :class:`~pynpoint.core.pypeline.Pypeline`. Parameters ---------- name : str Name of the module that has to be removed. Returns ------- bool Confirmation of removing the :class:`~pynpoint.core.processing.PypelineModule`. """ if name in self._m_modules: del self._m_modules[name] removed = True else: warnings.warn( f'Pipeline module \'{name}\' is not found in the Pypeline dictionary ' f'so it could not be removed. The dictionary contains the following ' f'modules: {list(self._m_modules.keys())}.') removed = False return removed @typechecked def get_module_names(self) -> List[str]: """ Method to return a list with the names of all pipeline modules that are added to the :class:`~pynpoint.core.pypeline.Pypeline`. Returns ------- list(str) Ordered list of all Pypeline modules. """ return list(self._m_modules.keys()) @typechecked def validate_pipeline(self) -> Tuple[bool, Optional[str]]: """ Method to check if each :class:`~pynpoint.core.dataio.InputPort` is pointing to an :class:`~pynpoint.core.dataio.OutputPort` of a previously added :class:`~pynpoint.core.processing.PypelineModule`. Returns ------- bool Validation of the pipeline. str, None Name of the pipeline module that can not be validated. Returns None if all modules were validated. """ self.m_data_storage.open_connection() # Create list with all datasets that are stored in the database data_tags = list(self.m_data_storage.m_data_bank.keys()) # Initiate the validation in case self._m_modules.values() is empty validation = (True, None) # Loop over all pipline modules in the ordered dictionary for module in self._m_modules.values(): # Validate the pipeline module validation = self._validate(module, data_tags) if not validation[0]: # Break the for loop if a module could not be validated break return validation @typechecked def validate_pipeline_module(self, name: str) -> Tuple[bool, Optional[str]]: """ Method to check if each :class:`~pynpoint.core.dataio.InputPort` of a :class:`~pynpoint.core.processing.PypelineModule` with label ``name`` points to an existing dataset in the database. Parameters ---------- name : str Name of the pipeline module instance that will be validated. Returns ------- bool Validation of the pipeline module. str, None Pipeline module name in case it is not valid. Returns None if the module was validated. """ self.m_data_storage.open_connection() # Create list with all datasets that are stored in the database data_tags = list(self.m_data_storage.m_data_bank.keys()) # Check if the name is included in the internal dictionary with added modules if name in self._m_modules: # Validate the pipeline module validate = self._validate(self._m_modules[name], data_tags) else: validate = (False, name) return validate @typechecked def run(self) -> None: """ Method for running all pipeline modules that are added to the :class:`~pynpoint.core.pypeline.Pypeline`. Returns ------- NoneType None """ # Validate the pipeline validation = self.validate_pipeline() if not validation[0]: # Check if the input data is available raise AttributeError( f'Pipeline module \'{validation[1]}\' is looking for data ' f'under a tag which is not created by a previous module or ' f'the data does not exist in the database.') # Loop over all pipeline modules and run them for name in self._m_modules: self.run_module(name) @typechecked def run_module(self, name: str) -> None: """ Method for running a pipeline module. Parameters ---------- name : str Name of the pipeline module. Returns ------- NoneType None """ if name in self._m_modules: # Validate the pipeline module validation = self.validate_pipeline_module(name) if not validation[0]: raise AttributeError( f'Pipeline module \'{validation[1]}\' is looking for data ' f'under a tag which does not exist in the database.') # Print information about the pipeline module module_info(self._m_modules[name]) # Check if the module has any input ports if hasattr(self._m_modules[name], '_m_input_ports'): # Check if the list of input ports is not empty if len(self._m_modules[name]._m_input_ports) > 0: # Print information about the input ports input_info(self._m_modules[name]) # Check if the module has any output ports if hasattr(self._m_modules[name], '_m_output_ports'): for item in self._m_modules[name]._m_output_ports: # Check if the module is a ProcessingModule if isinstance(self._m_modules[name], ProcessingModule): # Check if the database tag is already used if item in self.m_data_storage.m_data_bank: # Check if the output port is not used as input port if hasattr(self._m_modules[name], '_m_input_ports') and \ item not in self._m_modules[name]._m_input_ports: print(f'Deleting data and attributes: {item}') # Delete existing data and attributes self._m_modules[name]._m_output_ports[ item].del_all_data() self._m_modules[name]._m_output_ports[ item].del_all_attributes() # Run the pipeline module self._m_modules[name].run() # Check if the module has any output ports if hasattr(self._m_modules[name], '_m_output_ports'): output_shape = {} for item in self._m_modules[name]._m_output_ports: # Get the shape of the output port output_shape[item] = self.get_shape(item) # Print information about the output ports output_info(self._m_modules[name], output_shape) else: warnings.warn(f'Pipeline module \'{name}\' not found.') @typechecked def get_data(self, tag: str, data_range: Optional[Tuple[int, int]] = None) -> np.ndarray: """ Method for reading data from the database. Parameters ---------- tag : str Database tag. data_range : tuple(int, int), None Slicing range for the first axis of a dataset. This argument can be used to select a subset of images from dataset. The full dataset is read if the argument is set to None. Returns ------- np.ndarray The selected dataset from the database. """ self.m_data_storage.open_connection() if data_range is None: data = np.asarray(self.m_data_storage.m_data_bank[tag]) else: data = np.asarray(self.m_data_storage.m_data_bank[tag][ data_range[0]:data_range[1], ]) self.m_data_storage.close_connection() return data @typechecked def delete_data(self, tag: str) -> None: """ Method for deleting a dataset and related attributes from the central database. Disk space does not seem to free up when using this method. Parameters ---------- tag : str Database tag. Returns ------- NoneType None """ self.m_data_storage.open_connection() if tag in self.m_data_storage.m_data_bank: del self.m_data_storage.m_data_bank[tag] else: warnings.warn(f'Dataset \'{tag}\' not found in the database.') if 'header_' + tag + '/' in self.m_data_storage.m_data_bank: del self.m_data_storage.m_data_bank[f'header_{tag}'] else: warnings.warn( f'Attributes of \'{tag}\' not found in the database.') self.m_data_storage.close_connection() @typechecked def get_attribute( self, data_tag: str, attr_name: str, static: bool = True) -> Union[StaticAttribute, NonStaticAttribute]: """ Method for reading an attribute from the database. Parameters ---------- data_tag : str Database tag. attr_name : str Name of the attribute. static : bool Static (True) or non-static attribute (False). Returns ------- StaticAttribute, NonStaticAttribute Attribute value. For a static attribute, a single value is returned. For a non-static attribute, an array of values is returned. """ self.m_data_storage.open_connection() if static: attr = self.m_data_storage.m_data_bank[data_tag].attrs[attr_name] else: attr = self.m_data_storage.m_data_bank[ f'header_{data_tag}/{attr_name}'] attr = np.asarray(attr) self.m_data_storage.close_connection() return attr @typechecked def set_attribute(self, data_tag: str, attr_name: str, attr_value: Union[StaticAttribute, NonStaticAttribute], static: bool = True) -> None: """ Method for writing an attribute to the database. Existing values will be overwritten. Parameters ---------- data_tag : str Database tag. attr_name : str Name of the attribute. attr_value : StaticAttribute, NonStaticAttribute Attribute value. static : bool Static (True) or non-static attribute (False). Returns ------- NoneType None """ self.m_data_storage.open_connection() if static: self.m_data_storage.m_data_bank[data_tag].attrs[ attr_name] = attr_value else: if isinstance(attr_value[0], str): attr_value = np.array(attr_value, dtype='|S') if attr_name in list(self.m_data_storage. m_data_bank[f'header_{data_tag}'].keys()): del self.m_data_storage.m_data_bank[ f'header_{data_tag}/{attr_name}'] attr_key = f'header_{data_tag}/{attr_name}' self.m_data_storage.m_data_bank[attr_key] = np.asarray(attr_value) self.m_data_storage.close_connection() @typechecked def get_tags(self) -> List[str]: """ Method for returning a list with all database tags, except header and configuration tags. Returns ------- list(str) Database tags. """ self.m_data_storage.open_connection() tags = list(self.m_data_storage.m_data_bank.keys()) selected_tags = [] for item in tags: if item in ['config', 'fits_header'] or item[0:7] == 'header_': continue selected_tags.append(item) self.m_data_storage.close_connection() return selected_tags @typechecked def get_shape(self, tag: str) -> Optional[Tuple[int, ...]]: """ Method for returning the shape of a database entry. Parameters ---------- tag : str Database tag. Returns ------- tuple(int, ...), None Shape of the dataset. None is returned if the database tag is not found. """ self.m_data_storage.open_connection() if tag in self.m_data_storage.m_data_bank: data_shape = self.m_data_storage.m_data_bank[tag].shape else: data_shape = None self.m_data_storage.close_connection() return data_shape
def __init__(self, working_place_in: Optional[str] = None, input_place_in: Optional[str] = None, output_place_in: Optional[str] = None) -> None: """ Parameters ---------- working_place_in : str, None Working location where the central HDF5 database and the configuration file will be stored. Sufficient space is required in the working folder since each pipeline module stores a dataset in the HDF5 database. The current working folder of Python is used as working folder if the argument is set to None. input_place_in : str, None Default input folder where a :class:`~pynpoint.core.processing.ReadingModule` that is added to the :class:`~pynpoint.core.pypeline.Pypeline` will look for input data. The current working folder of Python is used as input folder if the argument is set to None. output_place_in : str, None Default output folder where a :class:`~pynpoint.core.processing.WritingModule` that is added to the :class:`~pynpoint.core.pypeline.Pypeline` will store output data. The current working folder of Python is used as output folder if the argument is set to None. Returns ------- NoneType None """ pynpoint_version = 'PynPoint v' + pynpoint.__version__ print(len(pynpoint_version) * '=') print(pynpoint_version) print(len(pynpoint_version) * '=' + '\n') try: contents = urllib.request.urlopen( 'https://pypi.org/pypi/pynpoint/json').read() data = json.loads(contents) latest_version = data['info']['version'] except URLError: latest_version = None if latest_version is not None and pynpoint.__version__ != latest_version: print(f'A new version ({latest_version}) is available!\n') print( 'Want to stay informed about updates, bug fixes, and new features?' ) print( 'Please consider using the \'Watch\' button on the Github page:' ) print('https://github.com/PynPoint/PynPoint\n') if working_place_in is None: self._m_working_place = os.getcwd() else: self._m_working_place = working_place_in if input_place_in is None: self._m_input_place = os.getcwd() else: self._m_input_place = input_place_in if output_place_in is None: self._m_output_place = os.getcwd() else: self._m_output_place = output_place_in print(f'Working place: {self._m_working_place}') print(f'Input place: {self._m_input_place}') print(f'Output place: {self._m_output_place}\n') self._m_modules = collections.OrderedDict() hdf5_path = os.path.join(self._m_working_place, 'PynPoint_database.hdf5') self.m_data_storage = DataStorage(hdf5_path) print(f'Database: {self.m_data_storage._m_location}') self._config_init()
class Pypeline(object): """ A Pypeline instance can be used to manage various processing steps. It inheres an internal dictionary of Pypeline steps (modules) and their names. A Pypeline has a central DataStorage on the hard drive which can be accessed by various modules. The order of the modules depends on the order the steps have been added to the pypeline. It is possible to run all modules attached to the Pypeline at once or run a single modules by name. """ def __init__(self, working_place_in=None, input_place_in=None, output_place_in=None): """ Constructor of Pypeline. :param working_place_in: Working location of the Pypeline which needs to be a folder on the hard drive. The given folder will be used to save the central PynPoint database (an HDF5 file) in which all the intermediate processing steps are saved. Note that the HDF5 file can become very large depending on the size and number of input images. :type working_place_in: str :param input_place_in: Default input directory of the Pypeline. All ReadingModules added to the Pypeline use this directory to look for input data. It is possible to specify a different location for the ReadingModules using their constructors. :type input_place_in: str :param output_place_in: Default result directory used to save the output of all WritingModules added to the Pypeline. It is possible to specify a different locations for the WritingModules by using their constructors. :return: None """ sys.stdout.write("Initiating PynPoint v" + pynpoint.__version__ + "...") sys.stdout.flush() self._m_working_place = working_place_in self._m_input_place = input_place_in self._m_output_place = output_place_in self._m_modules = collections.OrderedDict() self.m_data_storage = DataStorage( os.path.join(working_place_in, 'PynPoint_database.hdf5')) self._config_init() sys.stdout.write(" [DONE]\n") sys.stdout.flush() def __setattr__(self, key, value): """ This method is called every time a member / attribute of the Pypeline is changed. It checks whether a chosen working / input / output directory exists. :param key: Member or attribute name. :param value: New value for the given member or attribute. :return: None """ if key in ["_m_working_place", "_m_input_place", "_m_output_place"]: assert (os.path.isdir(str(value))), "Input directory for " + str(key) + " does not " \ "exist - input requested: %s." % value super(Pypeline, self).__setattr__(key, value) @staticmethod def _validate(module, tags): """ Internal function which is used for the validation of the pipeline. Validates a single module. :param module: The module. :type module: ReadingModule, WritingModule, ProcessingModule :param tags: Tags in the database. :type tags: list, str :return: Module validation. :rtype: bool, str """ if isinstance(module, ReadingModule): tags.extend(module.get_all_output_tags()) elif isinstance(module, WritingModule): for tag in module.get_all_input_tags(): if tag not in tags: return False, module.name elif isinstance(module, ProcessingModule): tags.extend(module.get_all_output_tags()) for tag in module.get_all_input_tags(): if tag not in tags: return False, module.name else: return False, None return True, None def _config_init(self): """ Internal function which initializes the configuration file. It reads PynPoint_config.ini in the working folder and creates this file with the default (ESO/NACO) settings in case the file is not present. :return: None """ def _create_config(filename, attributes): file_obj = open(filename, 'w') file_obj.write("[header]\n\n") for key, val in six.iteritems(attributes): if val['config'] == "header": file_obj.write(key + ': ' + str(val['value']) + '\n') file_obj.write("\n[settings]\n\n") for key, val in six.iteritems(attributes): if val['config'] == "settings": file_obj.write(key + ': ' + str(val['value']) + '\n') file_obj.close() def _read_config(config_file, attributes): config = configparser.ConfigParser() config.read_file(open(config_file)) for key, val in six.iteritems(attributes): if config.has_option(val["config"], key): if config.get(val["config"], key) == "None": if val["config"] == "header": attributes[key]["value"] = "None" # elif val["type"] == "str": # attributes[key]["value"] = "None" elif val["type"] == "float": attributes[key]["value"] = float(0.) elif val["type"] == "int": attributes[key]["value"] = int(0) else: if val["config"] == "header": attributes[key]["value"] = str( config.get(val["config"], key)) # elif val["type"] == "str": # attributes[key]["value"] = str(config.get(val["config"], key)) elif val["type"] == "float": attributes[key]["value"] = float( config.get(val["config"], key)) elif val["type"] == "int": attributes[key]["value"] = int( config.get(val["config"], key)) return attributes def _write_config(attributes): hdf = h5py.File(self._m_working_place + '/PynPoint_database.hdf5', 'a') if "config" in hdf: del hdf["config"] config = hdf.create_group("config") for key, _ in six.iteritems(attributes): if attributes[key]["value"] is not None: config.attrs[key] = attributes[key]["value"] config.attrs["WORKING_PLACE"] = self._m_working_place hdf.close() config_file = self._m_working_place + "/PynPoint_config.ini" attributes = get_attributes() attributes["CPU"]["value"] = multiprocessing.cpu_count() if not os.path.isfile(config_file): warnings.warn( "Configuration file not found. Creating PynPoint_config.ini with " "default values in the working place.") _create_config(config_file, attributes) attributes = _read_config(config_file, attributes) _write_config(attributes) def add_module(self, module): """ Adds a Pypeline module to the internal Pypeline dictionary. The module is appended at the end of this ordered dictionary. If the input module is a reading or writing module without a specified input or output location then the Pypeline default location is used. Moreover, the given module is connected to the Pypeline internal data storage. :param module: Input module. :type module: ReadingModule, WritingModule, ProcessingModule :return: None """ assert isinstance(module, PypelineModule), "The added module is not a valid " \ "Pypeline module." if isinstance(module, WritingModule): if module.m_output_location is None: module.m_output_location = self._m_output_place if isinstance(module, ReadingModule): if module.m_input_location is None: module.m_input_location = self._m_input_place module.connect_database(self.m_data_storage) if module.name in self._m_modules: warnings.warn( "Processing module names need to be unique. Overwriting module '%s'." % module.name) self._m_modules[module.name] = module def remove_module(self, name): """ Removes a Pypeline module from the internal dictionary. :param name: Name of the module which has to be removed. :type name: str :return: True if module was deleted and False if module does not exist. :rtype: bool """ if name in self._m_modules: del self._m_modules[name] return True warnings.warn("Module name '" + name + "' not found in the Pypeline dictionary.") return False def get_module_names(self): """ Function which returns a list of all module names. :return: Ordered list of all Pypeline modules. :rtype: list[str] """ return list(self._m_modules.keys()) def validate_pipeline(self): """ Function which checks if all input ports of the Pypeline are pointing to previous output ports. :return: True if Pypeline is valid and False if not. The second parameter contains the name of the module which is not valid. :rtype: bool, str """ self.m_data_storage.open_connection() existing_data_tags = list(self.m_data_storage.m_data_bank.keys()) for module in six.itervalues(self._m_modules): validation = self._validate(module, existing_data_tags) if not validation[0]: return validation return True, None def validate_pipeline_module(self, name): """ Checks if the data exists for the module with label *name*. :param name: Name of the module that is checked. :type name: str :return: True if the Pypeline module is valid and False if not. The second parameter gives the name of the module which is not valid. :rtype: bool, str """ self.m_data_storage.open_connection() existing_data_tags = list(self.m_data_storage.m_data_bank.keys()) if name in self._m_modules: module = self._m_modules[name] validate = self._validate(module, existing_data_tags) else: validate = None return validate def run(self): """ Walks through all saved processing steps and calls their run methods. The order in which the steps are called depends on the order they have been added to the Pypeline. :return: None """ sys.stdout.write("Validating Pypeline...") sys.stdout.flush() validation = self.validate_pipeline() if not validation[0]: raise AttributeError( "Pipeline module '%s' is looking for data under a tag which is " "not created by a previous module or does not exist in the " "database." % validation[1]) sys.stdout.write(" [DONE]\n") sys.stdout.flush() for key in self._m_modules: self._m_modules[key].run() def run_module(self, name): """ Runs a single processing module. :param name: Name of the module. :type name: str :return: None """ if name in self._m_modules: sys.stdout.write("Validating module " + name + "...") sys.stdout.flush() validation = self.validate_pipeline_module(name) if not validation[0]: raise AttributeError( "Pipeline module '%s' is looking for data under a tag which " "does not exist in the database." % validation[1]) sys.stdout.write(" [DONE]\n") sys.stdout.flush() self._m_modules[name].run() else: warnings.warn("Module '" + name + "' not found.") def get_data(self, tag): """ Function for accessing data in the central database. :param tag: Database tag. :type tag: str :return: The selected dataset from the database. :rtype: numpy.ndarray """ self.m_data_storage.open_connection() return np.asarray(self.m_data_storage.m_data_bank[tag]) def get_attribute(self, data_tag, attr_name, static=True): """ Function for accessing attributes in the central database. :param data_tag: Database tag. :type data_tag: str :param attr_name: Name of the attribute. :type attr_name: str :param static: Static or non-static attribute. :type static: bool :return: The attribute value(s). """ self.m_data_storage.open_connection() if static: attr = self.m_data_storage.m_data_bank[data_tag].attrs[attr_name] else: attr = self.m_data_storage.m_data_bank["header_" + data_tag + "/" + attr_name] attr = np.asarray(attr) return attr def set_attribute(self, data_tag, attr_name, value, static=True): """ Function for writing attributes to the central database. Existing values will be overwritten. :param data_tag: Database tag. :type data_tag: str :param attr_name: Name of the attribute. :type attr_name: str :param value: Attribute value. :type value: float, int, str, tuple, numpy.ndarray :param static: Static or non-static attribute. :type static: bool :return: None """ self.m_data_storage.open_connection() if static: self.m_data_storage.m_data_bank[data_tag].attrs[attr_name] = value else: if isinstance(value[0], str): value = np.array(value, dtype="|S") if attr_name in list( self.m_data_storage.m_data_bank["header_" + data_tag].keys()): del self.m_data_storage.m_data_bank["header_" + data_tag + "/" + attr_name] self.m_data_storage.m_data_bank["header_" + data_tag + "/" + attr_name] = np.asarray(value) self.m_data_storage.close_connection() def get_tags(self): """ Function for listing the database tags, ignoring header and config tags. :return: Database tags. :rtype: numpy.asarray """ self.m_data_storage.open_connection() tags = list(self.m_data_storage.m_data_bank.keys()) select = [] for _, item in enumerate(tags): if item == "config" or item == "fits_header" or item[ 0:7] == "header_": continue else: select.append(item) return np.asarray(select) def get_shape(self, tag): """ Function getting the shape of a database entry. :param tag: Database tag. :type tag: str :return: Database tags. :rtype: tuple """ self.m_data_storage.open_connection() return self.m_data_storage.m_data_bank[tag].shape
class Pypeline: """ A Pypeline instance can be used to manage various processing steps. It inheres an internal dictionary of Pypeline steps (modules) and their names. A Pypeline has a central DataStorage on the hard drive which can be accessed by various modules. The order of the modules depends on the order the steps have been added to the pypeline. It is possible to run all modules attached to the Pypeline at once or run a single modules by name. """ @typechecked def __init__(self, working_place_in: Optional[str] = None, input_place_in: Optional[str] = None, output_place_in: Optional[str] = None) -> None: """ Constructor of Pypeline. Parameters ---------- working_place_in : str Working location of the Pypeline which needs to be a folder on the hard drive. The given folder will be used to save the central PynPoint database (an HDF5 file) in which all the intermediate processing steps are saved. Note that the HDF5 file can become very large depending on the size and number of input images. input_place_in : str Default input directory of the Pypeline. All ReadingModules added to the Pypeline use this directory to look for input data. It is possible to specify a different location for the ReadingModules using their constructors. output_place_in : str Default result directory used to save the output of all WritingModules added to the Pypeline. It is possible to specify a different locations for the WritingModules by using their constructors. Returns ------- NoneType None """ pynpoint_version = 'PynPoint v' + pynpoint.__version__ print(len(pynpoint_version) * '=') print(pynpoint_version) print(len(pynpoint_version) * '=' + '\n') try: contents = urllib.request.urlopen( 'https://pypi.org/pypi/pynpoint/json').read() data = json.loads(contents) latest_version = data['info']['version'] except URLError: latest_version = None if latest_version is not None and pynpoint.__version__ != latest_version: print(f'A new version ({latest_version}) is available!\n') self._m_working_place = working_place_in self._m_input_place = input_place_in self._m_output_place = output_place_in self._m_modules = collections.OrderedDict() self.m_data_storage = DataStorage( os.path.join(working_place_in, 'PynPoint_database.hdf5')) print(f'Database: {self.m_data_storage._m_location}') self._config_init() @typechecked def __setattr__(self, key: str, value: Any) -> None: """ This method is called every time a member / attribute of the Pypeline is changed. It checks whether a chosen working / input / output directory exists. Parameters ---------- key : str Member or attribute name. value : str New value for the given member or attribute. Returns ------- NoneType None """ if key in ['_m_working_place', '_m_input_place', '_m_output_place']: assert (os.path.isdir(str(value))), f'Input directory for {key} does not exist - ' \ f'input requested: {value}.' super(Pypeline, self).__setattr__(key, value) @staticmethod @typechecked def _validate(module: Union[ReadingModule, WritingModule, ProcessingModule], tags: List[str]) -> Tuple[bool, Optional[str]]: """ Internal function which is used for the validation of the pipeline. Validates a single module. Parameters ---------- module : ReadingModule, WritingModule, or ProcessingModule The pipeline module. tags : list(str, ) Tags in the database. Returns ------- bool Module validation. str Module name. """ if isinstance(module, ReadingModule): tags.extend(module.get_all_output_tags()) elif isinstance(module, WritingModule): for tag in module.get_all_input_tags(): if tag not in tags: return False, module.name elif isinstance(module, ProcessingModule): tags.extend(module.get_all_output_tags()) for tag in module.get_all_input_tags(): if tag not in tags: return False, module.name else: return False, None return True, None @typechecked def _config_init(self) -> None: """ Internal function which initializes the configuration file. It reads PynPoint_config.ini in the working folder and creates this file with the default (ESO/NACO) settings in case the file is not present. Returns ------- NoneType None """ @typechecked def _create_config(filename: str, attributes: dict) -> None: file_obj = open(filename, 'w') file_obj.write('[header]\n\n') for key, val in attributes.items(): if val['config'] == 'header': file_obj.write(key + ': ' + str(val['value']) + '\n') file_obj.write('\n[settings]\n\n') for key, val in attributes.items(): if val['config'] == 'settings': file_obj.write(key + ': ' + str(val['value']) + '\n') file_obj.close() @typechecked def _read_config(config_file: str, attributes: dict) -> dict: config = configparser.ConfigParser() with open(config_file) as cf_open: config.read_file(cf_open) for key, val in attributes.items(): if config.has_option(val['config'], key): if config.get(val['config'], key) == 'None': if val['config'] == 'header': attributes[key]['value'] = 'None' # elif val['type'] == 'str': # attributes[key]['value'] = 'None' elif val['type'] == 'float': attributes[key]['value'] = float(0.) elif val['type'] == 'int': attributes[key]['value'] = int(0) else: if val['config'] == 'header': attributes[key]['value'] = str( config.get(val['config'], key)) # elif val['type'] == 'str': # attributes[key]['value'] = str(config.get(val['config'], key)) elif val['type'] == 'float': attributes[key]['value'] = float( config.get(val['config'], key)) elif val['type'] == 'int': attributes[key]['value'] = int( config.get(val['config'], key)) return attributes @typechecked def _write_config(attributes: dict) -> None: hdf = h5py.File(self._m_working_place + '/PynPoint_database.hdf5', 'a') if 'config' in hdf: del hdf['config'] config = hdf.create_group('config') for key in attributes.keys(): if attributes[key]['value'] is not None: config.attrs[key] = attributes[key]['value'] config.attrs['WORKING_PLACE'] = self._m_working_place hdf.close() config_file = os.path.join(self._m_working_place, 'PynPoint_config.ini') print(f'Configuration: {config_file}\n') attributes = get_attributes() attributes['CPU']['value'] = multiprocessing.cpu_count() if not os.path.isfile(config_file): warnings.warn( 'Configuration file not found. Creating PynPoint_config.ini with ' 'default values in the working place.') _create_config(config_file, attributes) attributes = _read_config(config_file, attributes) _write_config(attributes) n_cpu = attributes['CPU']['value'] if 'OMP_NUM_THREADS' in os.environ: n_thread = os.environ['OMP_NUM_THREADS'] else: n_thread = 'not set' print(f'Number of CPUs: {n_cpu}') print(f'Number of threads: {n_thread}') @typechecked def add_module(self, module: PypelineModule) -> None: """ Adds a Pypeline module to the internal Pypeline dictionary. The module is appended at the end of this ordered dictionary. If the input module is a reading or writing module without a specified input or output location then the Pypeline default location is used. Moreover, the given module is connected to the Pypeline internal data storage. Parameters ---------- module : ReadingModule, WritingModule, or ProcessingModule Input pipeline module. Returns ------- NoneType None """ if isinstance(module, WritingModule): if module.m_output_location is None: module.m_output_location = self._m_output_place if isinstance(module, ReadingModule): if module.m_input_location is None: module.m_input_location = self._m_input_place module.connect_database(self.m_data_storage) if module.name in self._m_modules: warnings.warn( f'Pipeline module names need to be unique. Overwriting module ' f'\'{module.name}\'.') self._m_modules[module.name] = module @typechecked def remove_module(self, name: str) -> bool: """ Removes a Pypeline module from the internal dictionary. Parameters ---------- name : str Name of the module that has to be removed. Returns ------- bool Confirmation of removal. """ if name in self._m_modules: del self._m_modules[name] removed = True else: warnings.warn( f'Pipeline module name \'{name}\' not found in the Pypeline dictionary.' ) removed = False return removed @typechecked def get_module_names(self) -> List[str]: """ Function which returns a list of all module names. Returns ------- list(str, ) Ordered list of all Pypeline modules. """ return list(self._m_modules.keys()) @typechecked def validate_pipeline(self) -> Tuple[bool, Optional[str]]: """ Function which checks if all input ports of the Pypeline are pointing to previous output ports. Returns ------- bool Confirmation of pipeline validation. str Module name that is not valid. """ self.m_data_storage.open_connection() data_tags = list(self.m_data_storage.m_data_bank.keys()) for module in self._m_modules.values(): validation = self._validate(module, data_tags) if not validation[0]: break else: validation = True, None return validation @typechecked def validate_pipeline_module( self, name: str) -> Optional[Tuple[bool, Optional[str]]]: """ Checks if the data exists for the module with label *name*. Parameters ---------- name : str Name of the module that is checked. Returns ------- bool Confirmation of pipeline module validation. str Module name that is not valid. """ self.m_data_storage.open_connection() existing_data_tags = list(self.m_data_storage.m_data_bank.keys()) if name in self._m_modules: module = self._m_modules[name] validate = self._validate(module, existing_data_tags) else: validate = None return validate @typechecked def run(self) -> None: """ Function for running all pipeline modules that are added to the :class:`~pynpoint.core.pypeline.Pypeline`. Returns ------- NoneType None """ # Validate the pipeline validation = self.validate_pipeline() if not validation[0]: # Check if the input data is available raise AttributeError( f'Pipeline module \'{validation[1]}\' is looking for data ' f'under a tag which is not created by a previous module or ' f'the data does not exist in the database.') # Loop over all pipeline modules and run them for name in self._m_modules: self.run_module(name) @typechecked def run_module(self, name: str) -> None: """ Function for running a pipeline module. Parameters ---------- name : str Name of the pipeline module. Returns ------- NoneType None """ if name in self._m_modules: # Validate the pipeline module validation = self.validate_pipeline_module(name) if not validation[0]: raise AttributeError( f'Pipeline module \'{validation[1]}\' is looking for data ' f'under a tag which does not exist in the database.') # Print information about the pipeline module module_info(self._m_modules[name]) # Check if the module has any input ports if hasattr(self._m_modules[name], '_m_input_ports'): # Check if the list of input ports is not empty if len(self._m_modules[name]._m_input_ports) > 0: # Print information about the input ports input_info(self._m_modules[name]) # Check if the module has any output ports if hasattr(self._m_modules[name], '_m_output_ports'): for item in self._m_modules[name]._m_output_ports: # Check if the module is a ProcessingModule if isinstance(self._m_modules[name], ProcessingModule): # Check if the database tag is already used if item in self.m_data_storage.m_data_bank: # Check if the output port is not used as input port if hasattr(self._m_modules[name], '_m_input_ports') and \ item not in self._m_modules[name]._m_input_ports: print(f'Deleting data and attributes: {item}') # Delete existing data and attributes self._m_modules[name]._m_output_ports[ item].del_all_data() self._m_modules[name]._m_output_ports[ item].del_all_attributes() # Run the pipeline module self._m_modules[name].run() # Check if the module has any output ports if hasattr(self._m_modules[name], '_m_output_ports'): output_shape = {} for item in self._m_modules[name]._m_output_ports: # Get the shape of the output port output_shape[item] = self.get_shape(item) # Print information about the output ports output_info(self._m_modules[name], output_shape) else: warnings.warn(f'Pipeline module \'{name}\' not found.') @typechecked def get_data(self, tag: str, data_range: Optional[Tuple[int, int]] = None) -> np.ndarray: """ Function for accessing data in the central database. Parameters ---------- tag : str Database tag. data_range : tuple(int, int), None Slicing range which can be used to select a subset of images from a 3D dataset. All data are selected if set to None. Returns ------- np.ndarray The selected dataset from the database. """ self.m_data_storage.open_connection() if data_range is None: data = np.asarray(self.m_data_storage.m_data_bank[tag]) else: data = np.asarray(self.m_data_storage.m_data_bank[tag][ data_range[0]:data_range[1], ]) self.m_data_storage.close_connection() return data @typechecked def delete_data(self, tag: str) -> None: """ Function for deleting a dataset and related attributes from the central database. Disk space does not seem to free up when using this function. Parameters ---------- tag : str Database tag. Returns ------- NoneType None """ self.m_data_storage.open_connection() if tag in self.m_data_storage.m_data_bank: del self.m_data_storage.m_data_bank[tag] else: warnings.warn(f'Dataset \'{tag}\' not found in the database.') if 'header_' + tag + '/' in self.m_data_storage.m_data_bank: del self.m_data_storage.m_data_bank[f'header_{tag}'] else: warnings.warn( f'Attributes of \'{tag}\' not found in the database.') self.m_data_storage.close_connection() @typechecked def get_attribute( self, data_tag: str, attr_name: str, static: bool = True) -> Union[StaticAttribute, NonStaticAttribute]: """ Function for accessing attributes in the central database. Parameters ---------- data_tag : str Database tag. attr_name : str Name of the attribute. static : bool Static or non-static attribute. Returns ------- StaticAttribute, NonStaticAttribute The values of the attribute, which can either be static or non-static. """ self.m_data_storage.open_connection() if static: attr = self.m_data_storage.m_data_bank[data_tag].attrs[attr_name] else: attr = self.m_data_storage.m_data_bank[ f'header_{data_tag}/{attr_name}'] attr = np.asarray(attr) self.m_data_storage.close_connection() return attr @typechecked def set_attribute(self, data_tag: str, attr_name: str, attr_value: Union[StaticAttribute, NonStaticAttribute], static: bool = True) -> None: """ Function for writing attributes to the central database. Existing values will be overwritten. Parameters ---------- data_tag : str Database tag. attr_name : str Name of the attribute. attr_value : StaticAttribute, NonStaticAttribute Attribute value. static : bool Static or non-static attribute. Returns ------- NoneType None """ self.m_data_storage.open_connection() if static: self.m_data_storage.m_data_bank[data_tag].attrs[ attr_name] = attr_value else: if isinstance(attr_value[0], str): attr_value = np.array(attr_value, dtype='|S') if attr_name in list(self.m_data_storage. m_data_bank[f'header_{data_tag}'].keys()): del self.m_data_storage.m_data_bank[ f'header_{data_tag}/{attr_name}'] attr_key = f'header_{data_tag}/{attr_name}' self.m_data_storage.m_data_bank[attr_key] = np.asarray(attr_value) self.m_data_storage.close_connection() @typechecked def get_tags(self) -> np.ndarray: """ Function for listing the database tags, ignoring header and config tags. Returns ------- np.ndarray Database tags. """ self.m_data_storage.open_connection() tags = list(self.m_data_storage.m_data_bank.keys()) select = [] for item in tags: if item in ('config', 'fits_header') or item[0:7] == 'header_': continue select.append(item) self.m_data_storage.close_connection() return np.asarray(select) @typechecked def get_shape(self, tag: str) -> Optional[Tuple[int, ...]]: """ Function for getting the shape of a database entry. Parameters ---------- tag : str Database tag. Returns ------- tuple(int, ) Dataset shape. """ self.m_data_storage.open_connection() if tag in self.m_data_storage.m_data_bank: data_shape = self.m_data_storage.m_data_bank[tag].shape else: data_shape = None self.m_data_storage.close_connection() return data_shape
def setup(self) -> None: self.limit = 1e-10 self.storage = DataStorage( os.path.dirname(__file__) + '/PynPoint_database.hdf5')