def read_config(self, config_file, keep_basedirs=True):
"""Read and import form paths.ini"""
if not os.path.isfile(config_file):
raise IOError("Configuration file paths.ini at %s does not exist "
"or is not a file" % config_file)
self.OBSCONFIG = od()
cr = ConfigParser()
cr.read(config_file)
if cr.has_section('outputfolders'):
if not keep_basedirs or not self.dir_exists(self._cachedir):
try:
cachedir = cr['outputfolders']['CACHEDIR']
if not self._write_access(cachedir):
raise PermissionError(
'Cannot write to {}'.format(cachedir))
self._cachedir = cr['outputfolders']['CACHEDIR']
except Exception as e:
self.logger.warning('Failed to init cache directory from '
'config file. Error: {}'.format(
repr(e)))
if not keep_basedirs or not self.dir_exists(self._outputdir):
try:
outdir = cr['outputfolders']['OUTPUTDIR']
if not self._write_access(outdir):
raise PermissionError(
'Cannot write to {}'.format(outdir))
self._outputdir = outdir
self._colocateddatadir = os.path.join(
outdir, 'colocated_data')
except Exception as e:
self.logger.warning(
'Failed to init output and colocated data '
'directory from config file. Error: {}'.format(
repr(e)))
try:
_dir = cr['outputfolders']['LOCALTMPDIR']
# expand $HOME
if '$HOME' in _dir:
_dir = _dir.replace('$HOME', os.path.expanduser('~'))
if '${USER}' in _dir:
_dir = _dir.replace('${USER}', getpass.getuser())
local_tmp_dir = _dir
if not self._write_access(local_tmp_dir):
raise PermissionError(
'Cannot write to {}'.format(local_tmp_dir))
self.LOCAL_TMP_DIR = local_tmp_dir
except Exception as e:
self.logger.warning('Failed to init local tmp directory from '
'config file. Error: {}'.format(repr(e)))
if cr.has_section('supplfolders'):
for name, path in cr['supplfolders'].items():
if not os.path.exists(path):
self.print_log.warning('Supplementary data directory for '
'{} does not exist:\n{}'.format(
name, path))
self.SUPPLDIRS[name] = path
#init base directories for Model data
if not keep_basedirs or not self.dir_exists(self._modelbasedir):
_dir = cr['modelfolders']['BASEDIR']
if '$HOME' in _dir:
_dir = _dir.replace('$HOME', os.path.expanduser('~'))
self._modelbasedir = _dir
self.MODELDIRS = (cr['modelfolders']['dir'].replace(
'${BASEDIR}', self._modelbasedir).replace('\n', '').split(','))
#Read directories for observation location
if not keep_basedirs or not self.dir_exists(self._obsbasedir):
_dir = cr['obsfolders']['BASEDIR']
if '$HOME' in _dir:
_dir = _dir.replace('$HOME', os.path.expanduser('~'))
self._obsbasedir = _dir
try:
self._init_obsconfig(cr)
except Exception as e:
from pyaerocom import print_log
print_log.exception(
'Failed to initiate obs config. Error: {}'.format(repr(e)))
cr.clear()
self.check_directories()
def read_config(self, config_file=None, keep_basedirs=True):
"""Read and import form paths.ini"""
if config_file is None:
config_file = self._config_ini
if not os.path.isfile(config_file):
raise IOError("Configuration file paths.ini at %s does not exist "
"or is not a file"
%config_file)
cr = ConfigParser()
cr.read(config_file)
if cr.has_section('outputfolders'):
if not keep_basedirs or not self.dir_exists(self._cachedir):
self._cachedir = cr['outputfolders']['CACHEDIR']
if not keep_basedirs or not self.dir_exists(self._outputdir):
self._outputdir = outdir =cr['outputfolders']['OUTPUTDIR']
self._colocateddatadir = os.path.join(outdir, 'colocated_data')
#init base directories for Model data
if not keep_basedirs or not self.dir_exists(self._modelbasedir):
self._modelbasedir = cr['modelfolders']['BASEDIR']
self.MODELDIRS = (cr['modelfolders']['dir'].
replace('${BASEDIR}', self._modelbasedir).
replace('\n','').split(','))
# read obs network names from ini file
# Aeronet V2
self.AERONET_SUN_V2L15_AOD_DAILY_NAME = cr['obsnames']['AERONET_SUN_V2L15_AOD_DAILY']
self.AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V2L15_AOD_ALL_POINTS']
self.AERONET_SUN_V2L2_AOD_DAILY_NAME = cr['obsnames']['AERONET_SUN_V2L2_AOD_DAILY']
self.AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V2L2_AOD_ALL_POINTS']
self.AERONET_SUN_V2L2_SDA_DAILY_NAME = cr['obsnames']['AERONET_SUN_V2L2_SDA_DAILY']
self.AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V2L2_SDA_ALL_POINTS']
# inversions
self.AERONET_INV_V2L15_DAILY_NAME = cr['obsnames']['AERONET_INV_V2L15_DAILY']
self.AERONET_INV_V2L15_ALL_POINTS_NAME = cr['obsnames']['AERONET_INV_V2L15_ALL_POINTS']
self.AERONET_INV_V2L2_DAILY_NAME = cr['obsnames']['AERONET_INV_V2L2_DAILY']
self.AERONET_INV_V2L2_ALL_POINTS_NAME = cr['obsnames']['AERONET_INV_V2L2_ALL_POINTS']
# Aeronet V3
self.AERONET_SUN_V3L15_AOD_DAILY_NAME = cr['obsnames']['AERONET_SUN_V3L15_AOD_DAILY']
self.AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V3L15_AOD_ALL_POINTS']
self.AERONET_SUN_V3L2_AOD_DAILY_NAME = cr['obsnames']['AERONET_SUN_V3L2_AOD_DAILY']
self.AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V3L2_AOD_ALL_POINTS']
self.AERONET_SUN_V3L2_SDA_DAILY_NAME = cr['obsnames']['AERONET_SUN_V3L2_SDA_DAILY']
self.AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME = cr['obsnames']['AERONET_SUN_V3L2_SDA_ALL_POINTS']
# inversions
self.AERONET_INV_V3L15_DAILY_NAME = cr['obsnames']['AERONET_INV_V3L15_DAILY']
self.AERONET_INV_V3L2_DAILY_NAME = cr['obsnames']['AERONET_INV_V3L2_DAILY']
self.EBAS_MULTICOLUMN_NAME = cr['obsnames']['EBAS_MULTICOLUMN']
self.EEA_NAME = cr['obsnames']['EEA']
self.EARLINET_NAME = cr['obsnames']['EARLINET']
#Read directories for observation location
if not keep_basedirs or not self.dir_exists(self._obsbasedir):
self._obsbasedir = cr['obsfolders']['BASEDIR']
OBSCONFIG = self.OBSCONFIG
OBSCONFIG[self.AERONET_SUN_V2L15_AOD_DAILY_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V2L15_AOD_DAILY_NAME]['PATH'] =\
cr['obsfolders']['AERONET_SUN_V2L15_AOD_DAILY'].\
replace('${BASEDIR}', self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V2L15_AOD_DAILY_NAME]['START_YEAR'] =\
cr['obsstartyears']['AERONET_SUN_V2L15_AOD_DAILY']
OBSCONFIG[self.AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME]['PATH'] =\
cr['obsfolders']['AERONET_SUN_V2L15_AOD_ALL_POINTS'].\
replace('${BASEDIR}', self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V2L15_AOD_ALL_POINTS_NAME]['START_YEAR'] =\
cr['obsstartyears']['AERONET_SUN_V2L15_AOD_ALL_POINTS']
OBSCONFIG[self.AERONET_SUN_V2L2_AOD_DAILY_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V2L2_AOD_DAILY_NAME]['PATH'] =\
cr['obsfolders']['AERONET_SUN_V2L2_AOD_DAILY'].\
replace('${BASEDIR}', self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V2L2_AOD_DAILY_NAME]['START_YEAR'] =\
cr['obsstartyears']['AERONET_SUN_V2L2_AOD_DAILY']
OBSCONFIG[self.AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME]['PATH'] =\
cr['obsfolders']['AERONET_SUN_V2L2_AOD_ALL_POINTS'].\
replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V2L2_AOD_ALL_POINTS_NAME]['START_YEAR'] =\
cr['obsstartyears']['AERONET_SUN_V2L2_AOD_ALL_POINTS']
OBSCONFIG[self.AERONET_SUN_V2L2_SDA_DAILY_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V2L2_SDA_DAILY_NAME]['PATH'] =\
cr['obsfolders']['AERONET_SUN_V2L2_SDA_DAILY'].\
replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V2L2_SDA_DAILY_NAME]['START_YEAR'] =\
cr['obsstartyears']['AERONET_SUN_V2L2_SDA_DAILY']
OBSCONFIG[self.AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME]['PATH'] =\
cr['obsfolders']['AERONET_SUN_V2L2_SDA_ALL_POINTS'].\
replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V2L2_SDA_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V2L2_SDA_ALL_POINTS']
OBSCONFIG[self.AERONET_SUN_V3L15_AOD_DAILY_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V3L15_AOD_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L15_AOD_DAILY'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V3L15_AOD_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L15_AOD_DAILY']
OBSCONFIG[self.AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L15_AOD_ALL_POINTS'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V3L15_AOD_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L15_AOD_ALL_POINTS']
OBSCONFIG[self.AERONET_SUN_V3L2_AOD_DAILY_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V3L2_AOD_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L2_AOD_DAILY'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V3L2_AOD_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L2_AOD_DAILY']
OBSCONFIG[self.AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L2_AOD_ALL_POINTS'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V3L2_AOD_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L2_AOD_ALL_POINTS']
OBSCONFIG[self.AERONET_SUN_V3L15_SDA_DAILY_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V3L15_SDA_DAILY_NAME]['PATH'] = \
cr['obsfolders']['AERONET_SUN_V3L15_SDA_DAILY'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V3L15_SDA_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L15_SDA_DAILY']
OBSCONFIG[self.AERONET_SUN_V3L2_SDA_DAILY_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V3L2_SDA_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L2_SDA_DAILY'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V3L2_SDA_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L2_SDA_DAILY']
OBSCONFIG[self.AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME] = {}
OBSCONFIG[self.AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME]['PATH'] = cr['obsfolders']['AERONET_SUN_V3L2_SDA_ALL_POINTS'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_SUN_V3L2_SDA_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_SUN_V3L2_SDA_ALL_POINTS']
OBSCONFIG[self.AERONET_INV_V2L15_DAILY_NAME] = {}
OBSCONFIG[self.AERONET_INV_V2L15_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V2L15_DAILY'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_INV_V2L15_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L15_DAILY']
OBSCONFIG[self.AERONET_INV_V2L15_ALL_POINTS_NAME] = {}
OBSCONFIG[self.AERONET_INV_V2L15_ALL_POINTS_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V2L15_ALL_POINTS'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_INV_V2L15_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L15_ALL_POINTS']
OBSCONFIG[self.AERONET_INV_V2L2_DAILY_NAME] = {}
OBSCONFIG[self.AERONET_INV_V2L2_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V2L2_DAILY'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_INV_V2L2_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L2_DAILY']
OBSCONFIG[self.AERONET_INV_V2L2_ALL_POINTS_NAME] = {}
OBSCONFIG[self.AERONET_INV_V2L2_ALL_POINTS_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V2L2_ALL_POINTS'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_INV_V2L2_ALL_POINTS_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L2_ALL_POINTS']
# Aeronet v3 inversions
OBSCONFIG[self.AERONET_INV_V3L15_DAILY_NAME] = {}
OBSCONFIG[self.AERONET_INV_V3L15_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V3L15_DAILY'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_INV_V3L15_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L15_DAILY']
OBSCONFIG[self.AERONET_INV_V3L2_DAILY_NAME] = {}
OBSCONFIG[self.AERONET_INV_V3L2_DAILY_NAME]['PATH'] = cr['obsfolders']['AERONET_INV_V3L2_DAILY'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.AERONET_INV_V3L2_DAILY_NAME]['START_YEAR'] = cr['obsstartyears']['AERONET_INV_V2L15_DAILY']
OBSCONFIG[self.EBAS_MULTICOLUMN_NAME] = {}
OBSCONFIG[self.EBAS_MULTICOLUMN_NAME]['PATH'] = cr['obsfolders']['EBAS_MULTICOLUMN'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.EBAS_MULTICOLUMN_NAME]['START_YEAR'] = cr['obsstartyears']['EBAS_MULTICOLUMN']
OBSCONFIG[self.EEA_NAME] = {}
OBSCONFIG[self.EEA_NAME]['PATH'] = cr['obsfolders']['EEA'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.EEA_NAME]['START_YEAR'] = cr['obsstartyears']['EEA']
OBSCONFIG[self.EARLINET_NAME] = {}
OBSCONFIG[self.EARLINET_NAME]['PATH'] = cr['obsfolders']['EARLINET'].replace('${BASEDIR}',self._obsbasedir)
OBSCONFIG[self.EARLINET_NAME]['START_YEAR'] = cr['obsstartyears']['EARLINET']
cr.clear()
OBSCONFIG[AERONET_INV_V2L15_ALL_POINTS_NAME]['START_YEAR'] = conf_reader[
'obsstartyears']['AERONET_INV_V2L15_ALL_POINTS']
OBSCONFIG[AERONET_INV_V2L2_DAILY_NAME] = {}
OBSCONFIG[AERONET_INV_V2L2_DAILY_NAME]['PATH'] = conf_reader['obsfolders'][
'AERONET_INV_V2L2_DAILY'].replace('${BASEDIR}', OBSBASEDIR)
OBSCONFIG[AERONET_INV_V2L2_DAILY_NAME]['START_YEAR'] = conf_reader[
'obsstartyears']['AERONET_INV_V2L2_DAILY']
OBSCONFIG[AERONET_INV_V2L2_ALL_POINTS_NAME] = {}
OBSCONFIG[AERONET_INV_V2L2_ALL_POINTS_NAME]['PATH'] = conf_reader[
'obsfolders']['AERONET_INV_V2L2_ALL_POINTS'].replace(
'${BASEDIR}', OBSBASEDIR)
OBSCONFIG[AERONET_INV_V2L2_ALL_POINTS_NAME]['START_YEAR'] = conf_reader[
'obsstartyears']['AERONET_INV_V2L2_ALL_POINTS']
OBSCONFIG[EBAS_MULTICOLUMN_NAME] = {}
OBSCONFIG[EBAS_MULTICOLUMN_NAME]['PATH'] = conf_reader['obsfolders'][
'EBAS_MULTICOLUMN'].replace('${BASEDIR}', OBSBASEDIR)
OBSCONFIG[EBAS_MULTICOLUMN_NAME]['START_YEAR'] = conf_reader[
'obsstartyears']['EBAS_MULTICOLUMN']
OBSCONFIG[EEA_NAME] = {}
OBSCONFIG[EEA_NAME]['PATH'] = conf_reader['obsfolders']['EEA'].replace(
'${BASEDIR}', OBSBASEDIR)
OBSCONFIG[EEA_NAME]['START_YEAR'] = conf_reader['obsstartyears']['EEA']
conf_reader.clear()
del conf_reader, _config_ini
