def read_config(self, config_file, keep_basedirs=True): """Read and import form paths.ini""" if not os.path.isfile(config_file): raise IOError("Configuration file paths.ini at %s does not exist " "or is not a file" % config_file) self.OBSCONFIG = od() cr = ConfigParser() cr.read(config_file) if cr.has_section('outputfolders'): if not keep_basedirs or not self.dir_exists(self._cachedir): try: cachedir = cr['outputfolders']['CACHEDIR'] if not self._write_access(cachedir): raise PermissionError( 'Cannot write to {}'.format(cachedir)) self._cachedir = cr['outputfolders']['CACHEDIR'] except Exception as e: self.logger.warning('Failed to init cache directory from ' 'config file. Error: {}'.format( repr(e))) if not keep_basedirs or not self.dir_exists(self._outputdir): try: outdir = cr['outputfolders']['OUTPUTDIR'] if not self._write_access(outdir): raise PermissionError( 'Cannot write to {}'.format(outdir)) self._outputdir = outdir self._colocateddatadir = os.path.join( outdir, 'colocated_data') except Exception as e: self.logger.warning( 'Failed to init output and colocated data ' 'directory from config file. Error: {}'.format( repr(e))) try: _dir = cr['outputfolders']['LOCALTMPDIR'] # expand $HOME if '$HOME' in _dir: _dir = _dir.replace('$HOME', os.path.expanduser('~')) if '${USER}' in _dir: _dir = _dir.replace('${USER}', getpass.getuser()) local_tmp_dir = _dir if not self._write_access(local_tmp_dir): raise PermissionError( 'Cannot write to {}'.format(local_tmp_dir)) self.LOCAL_TMP_DIR = local_tmp_dir except Exception as e: self.logger.warning('Failed to init local tmp directory from ' 'config file. Error: {}'.format(repr(e))) if cr.has_section('supplfolders'): for name, path in cr['supplfolders'].items(): if not os.path.exists(path): self.print_log.warning('Supplementary data directory for ' '{} does not exist:\n{}'.format( name, path)) self.SUPPLDIRS[name] = path #init base directories for Model data if not keep_basedirs or not self.dir_exists(self._modelbasedir): _dir = cr['modelfolders']['BASEDIR'] if '$HOME' in _dir: _dir = _dir.replace('$HOME', os.path.expanduser('~')) self._modelbasedir = _dir self.MODELDIRS = (cr['modelfolders']['dir'].replace( '${BASEDIR}', self._modelbasedir).replace('\n', '').split(',')) #Read directories for observation location if not keep_basedirs or not self.dir_exists(self._obsbasedir): _dir = cr['obsfolders']['BASEDIR'] if '$HOME' in _dir: _dir = _dir.replace('$HOME', os.path.expanduser('~')) self._obsbasedir = _dir try: self._init_obsconfig(cr) except Exception as e: from pyaerocom import print_log print_log.exception( 'Failed to initiate obs config. Error: {}'.format(repr(e))) cr.clear() self.check_directories()
def read_config(self, config_file=None, keep_basedirs=True): """Read and import form paths.ini""" if config_file is None: config_file = self._config_ini if not os.path.isfile(config_file): raise IOError("Configuration file paths.ini at %s does not exist " "or is not a file" %config_file) cr = ConfigParser() cr.read(config_file) if cr.has_section('outputfolders'): if not keep_basedirs or not self.dir_exists(self._cachedir): self._cachedir = cr['outputfolders']['CACHEDIR'] if not keep_basedirs or not self.dir_exists(self._outputdir): self._outputdir = outdir =cr['outputfolders']['OUTPUTDIR'] self._colocateddatadir = os.path.join(outdir, 'colocated_data') #init base directories for Model data if not keep_basedirs or not self.dir_exists(self._modelbasedir): self._modelbasedir = cr['modelfolders']['BASEDIR'] self.MODELDIRS = (cr['modelfolders']['dir']. replace('${BASEDIR}', self._modelbasedir). replace('\n','').split(',')) # read obs network names from ini file # Aeronet V2 self.AERONET_SUN_V2L15_AOD_DAILY_NAME = cr['obsnames']['AERONET_SUN_V2L15_AOD_DAILY'] self.AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V2L15_AOD_ALL_POINTS'] self.AERONET_SUN_V2L2_AOD_DAILY_NAME = cr['obsnames']['AERONET_SUN_V2L2_AOD_DAILY'] self.AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V2L2_AOD_ALL_POINTS'] self.AERONET_SUN_V2L2_SDA_DAILY_NAME = cr['obsnames']['AERONET_SUN_V2L2_SDA_DAILY'] self.AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V2L2_SDA_ALL_POINTS'] # inversions self.AERONET_INV_V2L15_DAILY_NAME = cr['obsnames']['AERONET_INV_V2L15_DAILY'] self.AERONET_INV_V2L15_ALL_POINTS_NAME = cr['obsnames']['AERONET_INV_V2L15_ALL_POINTS'] self.AERONET_INV_V2L2_DAILY_NAME = cr['obsnames']['AERONET_INV_V2L2_DAILY'] self.AERONET_INV_V2L2_ALL_POINTS_NAME = cr['obsnames']['AERONET_INV_V2L2_ALL_POINTS'] # Aeronet V3 self.AERONET_SUN_V3L15_AOD_DAILY_NAME = cr['obsnames']['AERONET_SUN_V3L15_AOD_DAILY'] self.AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V3L15_AOD_ALL_POINTS'] self.AERONET_SUN_V3L2_AOD_DAILY_NAME = cr['obsnames']['AERONET_SUN_V3L2_AOD_DAILY'] self.AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V3L2_AOD_ALL_POINTS'] self.AERONET_SUN_V3L2_SDA_DAILY_NAME = cr['obsnames']['AERONET_SUN_V3L2_SDA_DAILY'] self.AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V3L2_SDA_ALL_POINTS'] # inversions self.AERONET_INV_V3L15_DAILY_NAME = cr['obsnames']['AERONET_INV_V3L15_DAILY'] self.AERONET_INV_V3L2_DAILY_NAME = cr['obsnames']['AERONET_INV_V3L2_DAILY'] self.EBAS_MULTICOLUMN_NAME = cr['obsnames']['EBAS_MULTICOLUMN'] self.EEA_NAME = cr['obsnames']['EEA'] self.EARLINET_NAME = cr['obsnames']['EARLINET'] #Read directories for observation location if not keep_basedirs or not self.dir_exists(self._obsbasedir): self._obsbasedir = cr['obsfolders']['BASEDIR'] OBSCONFIG = self.OBSCONFIG OBSCONFIG[self.AERONET_SUN_V2L15_AOD_DAILY_NAME] = {} OBSCONFIG[self.AERONET_SUN_V2L15_AOD_DAILY_NAME]['PATH'] =\ cr['obsfolders']['AERONET_SUN_V2L15_AOD_DAILY'].\ replace('${BASEDIR}', self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V2L15_AOD_DAILY_NAME]['START_YEAR'] =\ cr['obsstartyears']['AERONET_SUN_V2L15_AOD_DAILY'] OBSCONFIG[self.AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME] = {} OBSCONFIG[self.AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME]['PATH'] =\ cr['obsfolders']['AERONET_SUN_V2L15_AOD_ALL_POINTS'].\ replace('${BASEDIR}', self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME]['START_YEAR'] =\ cr['obsstartyears']['AERONET_SUN_V2L15_AOD_ALL_POINTS'] OBSCONFIG[self.AERONET_SUN_V2L2_AOD_DAILY_NAME] = {} OBSCONFIG[self.AERONET_SUN_V2L2_AOD_DAILY_NAME]['PATH'] =\ cr['obsfolders']['AERONET_SUN_V2L2_AOD_DAILY'].\ replace('${BASEDIR}', self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V2L2_AOD_DAILY_NAME]['START_YEAR'] =\ cr['obsstartyears']['AERONET_SUN_V2L2_AOD_DAILY'] OBSCONFIG[self.AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME] = {} OBSCONFIG[self.AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME]['PATH'] =\ cr['obsfolders']['AERONET_SUN_V2L2_AOD_ALL_POINTS'].\ replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME]['START_YEAR'] =\ cr['obsstartyears']['AERONET_SUN_V2L2_AOD_ALL_POINTS'] OBSCONFIG[self.AERONET_SUN_V2L2_SDA_DAILY_NAME] = {} OBSCONFIG[self.AERONET_SUN_V2L2_SDA_DAILY_NAME]['PATH'] =\ cr['obsfolders']['AERONET_SUN_V2L2_SDA_DAILY'].\ replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V2L2_SDA_DAILY_NAME]['START_YEAR'] =\ cr['obsstartyears']['AERONET_SUN_V2L2_SDA_DAILY'] OBSCONFIG[self.AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME] = {} OBSCONFIG[self.AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME]['PATH'] =\ cr['obsfolders']['AERONET_SUN_V2L2_SDA_ALL_POINTS'].\ replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V2L2_SDA_ALL_POINTS'] OBSCONFIG[self.AERONET_SUN_V3L15_AOD_DAILY_NAME] = {} OBSCONFIG[self.AERONET_SUN_V3L15_AOD_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L15_AOD_DAILY'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V3L15_AOD_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L15_AOD_DAILY'] OBSCONFIG[self.AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME] = {} OBSCONFIG[self.AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L15_AOD_ALL_POINTS'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L15_AOD_ALL_POINTS'] OBSCONFIG[self.AERONET_SUN_V3L2_AOD_DAILY_NAME] = {} OBSCONFIG[self.AERONET_SUN_V3L2_AOD_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L2_AOD_DAILY'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V3L2_AOD_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L2_AOD_DAILY'] OBSCONFIG[self.AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME] = {} OBSCONFIG[self.AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L2_AOD_ALL_POINTS'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L2_AOD_ALL_POINTS'] OBSCONFIG[self.AERONET_SUN_V3L15_SDA_DAILY_NAME] = {} OBSCONFIG[self.AERONET_SUN_V3L15_SDA_DAILY_NAME]['PATH'] = \ cr['obsfolders']['AERONET_SUN_V3L15_SDA_DAILY'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V3L15_SDA_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L15_SDA_DAILY'] OBSCONFIG[self.AERONET_SUN_V3L2_SDA_DAILY_NAME] = {} OBSCONFIG[self.AERONET_SUN_V3L2_SDA_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L2_SDA_DAILY'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V3L2_SDA_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L2_SDA_DAILY'] OBSCONFIG[self.AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME] = {} OBSCONFIG[self.AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L2_SDA_ALL_POINTS'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L2_SDA_ALL_POINTS'] OBSCONFIG[self.AERONET_INV_V2L15_DAILY_NAME] = {} OBSCONFIG[self.AERONET_INV_V2L15_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V2L15_DAILY'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_INV_V2L15_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L15_DAILY'] OBSCONFIG[self.AERONET_INV_V2L15_ALL_POINTS_NAME] = {} OBSCONFIG[self.AERONET_INV_V2L15_ALL_POINTS_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V2L15_ALL_POINTS'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_INV_V2L15_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L15_ALL_POINTS'] OBSCONFIG[self.AERONET_INV_V2L2_DAILY_NAME] = {} OBSCONFIG[self.AERONET_INV_V2L2_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V2L2_DAILY'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_INV_V2L2_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L2_DAILY'] OBSCONFIG[self.AERONET_INV_V2L2_ALL_POINTS_NAME] = {} OBSCONFIG[self.AERONET_INV_V2L2_ALL_POINTS_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V2L2_ALL_POINTS'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_INV_V2L2_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L2_ALL_POINTS'] # Aeronet v3 inversions OBSCONFIG[self.AERONET_INV_V3L15_DAILY_NAME] = {} OBSCONFIG[self.AERONET_INV_V3L15_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V3L15_DAILY'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_INV_V3L15_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L15_DAILY'] OBSCONFIG[self.AERONET_INV_V3L2_DAILY_NAME] = {} OBSCONFIG[self.AERONET_INV_V3L2_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V3L2_DAILY'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.AERONET_INV_V3L2_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L15_DAILY'] OBSCONFIG[self.EBAS_MULTICOLUMN_NAME] = {} OBSCONFIG[self.EBAS_MULTICOLUMN_NAME]['PATH'] = cr['obsfolders']['EBAS_MULTICOLUMN'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.EBAS_MULTICOLUMN_NAME]['START_YEAR'] = cr['obsstartyears']['EBAS_MULTICOLUMN'] OBSCONFIG[self.EEA_NAME] = {} OBSCONFIG[self.EEA_NAME]['PATH'] = cr['obsfolders']['EEA'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.EEA_NAME]['START_YEAR'] = cr['obsstartyears']['EEA'] OBSCONFIG[self.EARLINET_NAME] = {} OBSCONFIG[self.EARLINET_NAME]['PATH'] = cr['obsfolders']['EARLINET'].replace('${BASEDIR}',self._obsbasedir) OBSCONFIG[self.EARLINET_NAME]['START_YEAR'] = cr['obsstartyears']['EARLINET'] cr.clear()
OBSCONFIG[AERONET_INV_V2L15_ALL_POINTS_NAME]['START_YEAR'] = conf_reader[ 'obsstartyears']['AERONET_INV_V2L15_ALL_POINTS'] OBSCONFIG[AERONET_INV_V2L2_DAILY_NAME] = {} OBSCONFIG[AERONET_INV_V2L2_DAILY_NAME]['PATH'] = conf_reader['obsfolders'][ 'AERONET_INV_V2L2_DAILY'].replace('${BASEDIR}', OBSBASEDIR) OBSCONFIG[AERONET_INV_V2L2_DAILY_NAME]['START_YEAR'] = conf_reader[ 'obsstartyears']['AERONET_INV_V2L2_DAILY'] OBSCONFIG[AERONET_INV_V2L2_ALL_POINTS_NAME] = {} OBSCONFIG[AERONET_INV_V2L2_ALL_POINTS_NAME]['PATH'] = conf_reader[ 'obsfolders']['AERONET_INV_V2L2_ALL_POINTS'].replace( '${BASEDIR}', OBSBASEDIR) OBSCONFIG[AERONET_INV_V2L2_ALL_POINTS_NAME]['START_YEAR'] = conf_reader[ 'obsstartyears']['AERONET_INV_V2L2_ALL_POINTS'] OBSCONFIG[EBAS_MULTICOLUMN_NAME] = {} OBSCONFIG[EBAS_MULTICOLUMN_NAME]['PATH'] = conf_reader['obsfolders'][ 'EBAS_MULTICOLUMN'].replace('${BASEDIR}', OBSBASEDIR) OBSCONFIG[EBAS_MULTICOLUMN_NAME]['START_YEAR'] = conf_reader[ 'obsstartyears']['EBAS_MULTICOLUMN'] OBSCONFIG[EEA_NAME] = {} OBSCONFIG[EEA_NAME]['PATH'] = conf_reader['obsfolders']['EEA'].replace( '${BASEDIR}', OBSBASEDIR) OBSCONFIG[EEA_NAME]['START_YEAR'] = conf_reader['obsstartyears']['EEA'] conf_reader.clear() del conf_reader, _config_ini