class OpacityCache(Singleton):
"""
Implements a lazy load of opacities. A singleton that
loads and caches xsections as they are needed. Calling
>>> opt = OpacityCache()
>>> opt2 = OpacityCache()
Reveals that:
>>> opt == opt2
True
Importantly this class will automatically search directories for cross-sections
set using the :func:`set_opacity_path` method:
>>> opt.set_opacity_path('path/to/crossections')
Multiple paths can be set as well
>>> opt.set_opacity_path(['/path/to/crosssections','/another/path/to/crosssections'])
Currently only :obj:`.pickle` files are supported.
To get the cross-section object for a particular molecule use the square bracket operator:
>>> opt['H2O']
<taurex.opacity.pickleopacity.PickleOpacity at 0x107a60be0>
This returns a :class:`~taurex.opacity.pickleopacity.PickleOpacity` object for you to compute H2O cross sections from.
When called for the first time, a directory search is performed and, if found, the appropriate cross-section is loaded. Subsequent calls will immediately
return the already loaded object:
>>> h2o_a = opt['H2O']
>>> h2o_b = opt['H2O']
>>> h2o_a == h2o_b
True
Lastly if you've got a hot new opacity format, you can try out
by manually adding it into the cache:
>>> new_h2o = MyNewOpacityFormat()
>>> opt.add_opacity(new_h2o)
>>> opt['H2O]
<MyNewOpacityFormat at 0x107a60be0>
Now TauREx3 will use it instead in all calculations!
"""
def init(self):
self.opacity_dict = {}
self._opacity_path = None
self.log = Logger('OpacityCache')
self._default_interpolation = 'linear'
self._memory_mode = True
self._radis = False
self._radis_props = (600, 30000, 10000)
def set_opacity_path(self, opacity_path):
"""
Set the path(s) that will be searched for opacities.
Opacities in this path must be of supported types:
- HDF5 opacities
- ``.pickle`` opacities
- ExoTransmit opacities.
Parameters
----------
opacity_path : str or :obj:`list` of str, optional
search path(s) to look for molecular opacities
"""
import os
if not os.path.isdir(opacity_path):
self.log.error('PATH: %s does not exist!!!', opacity_path)
raise NotADirectoryError
self.log.debug('Path set to %s', opacity_path)
self._opacity_path = opacity_path
def enable_radis(self, enable):
"""
Enables/Disables use of RADIS to fill in missing molecules
using HITRAN.
.. warning::
This is extremely unstable and crashes frequently.
It is also very slow as it requires
the computation of the Voigt profile for every temperature.
We recommend leaving it as False unless necessary.
Parameters
----------
enable: bool
Whether to enable RADIS functionality (default = False)
"""
self._radis = enable
def set_radis_wavenumber(self, wn_start, wn_end, wn_points):
self._radis_props = (wn_start, wn_end, wn_points)
self.clear_cache()
def set_memory_mode(self,in_memory):
"""
If using the HDF5 opacities, whether to stream
opacities from file (slower, less memory) or load
them into memory (faster, more memory)
Parameters
----------
in_memory: bool
Whether HDF5 files should be streamed (False)
or loaded into memory (True, default)
"""
self._memory_mode = in_memory
self.clear_cache()
def set_interpolation(self,interpolation_mode):
"""
Sets the interpolation mode for all currently loaded (and future loaded) cross-sections
Can either be ``linear`` for linear interpolation of both temeprature and pressure:
>>> OpacityCache().set_interpolation('linear')
or ``exp`` for natural exponential interpolation of temperature
and linear for pressure
>>> OpacityCache().set_interpolation('exp')
Parameters
----------
interpolation_mode: str
Either ``linear`` for bilinear interpolation or ``exp`` for exp-linear interpolation
"""
return
self._default_interpolation = interpolation_mode
for values in self.opacity_dict.values():
values.set_interpolation_mode(self._default_interpolation)
def __getitem__(self,key):
"""
For a molecule return the relevant :class:`~taurex.opacity.opacity.Opacity` object.
Parameter
---------
key : str
molecule name
Returns
-------
:class:`~taurex.opacity.pickleopacity.PickleOpacity`
Cross-section object desired
Raise
-----
Exception
If molecule could not be loaded/found
"""
if key in self.opacity_dict:
return self.opacity_dict[key]
else:
#Try a load of the opacity
self.load_opacity(molecule_filter=[key])
#If we have it after a load then good job boys
if key in self.opacity_dict:
return self.opacity_dict[key]
else:
try:
if self._radis:
return self.create_radis_opacity(key,molecule_filter=[key])
else:
raise Exception
except Exception as e:
self.log.error('EXception thrown %s',e)
#Otherwise throw an error
self.log.error('Opacity for molecule %s could not be loaded',key)
self.log.error('It could not be found in the local dictionary %s',list(self.opacity_dict.keys()))
self.log.error('Or paths %s',self._opacity_path)
self.log.error('Try loading it manually/ putting it in a path')
raise Exception('Opacity could notn be loaded')
def create_radis_opacity(self,molecule,molecule_filter=None):
from taurex.opacity.radisopacity import RadisHITRANOpacity
if molecule not in self.opacity_dict:
self.log.info('Creating Opacity from RADIS+HITRAN')
wn_start,wn_end,wn_points = self._radis_props
radis = RadisHITRANOpacity(molecule_name=molecule, wn_start=wn_start,wn_end=wn_end,wn_points=wn_points)
self.add_opacity(radis,molecule_filter=molecule_filter)
return radis
else:
self.log.info('Opacity %s already exsits',molecule)
def add_opacity(self,opacity,molecule_filter=None):
"""
Adds a :class:`~taurex.opacity.opacity.Opacity` object to the cache to then be
used by Taurex 3
Parameters
----------
opacity : :class:`~taurex.opacity.opacity.Opacity`
Opacity object to add to the cache
molecule_filter : :obj:`list` of str , optional
If provided, the opacity object will only be included
if its molecule is in the list. Mostly used by the
:func:`__getitem__` for filtering
"""
self.log.info('Reading opacity %s',opacity.moleculeName)
if opacity.moleculeName in self.opacity_dict:
self.log.warning('Opacity with name %s already in opactiy dictionary %s skipping',opacity.moleculeName,self.opacity_dict.keys())
return
if molecule_filter is not None:
if opacity.moleculeName in molecule_filter:
self.log.info('Loading opacity %s into model',opacity.moleculeName)
self.opacity_dict[opacity.moleculeName] = opacity
else:
self.log.info('Loading opacity %s into model',opacity.moleculeName)
self.opacity_dict[opacity.moleculeName] = opacity
def search_hdf5_molecules(self):
"""
Find molecules with HDF5 opacities in set path
Returns
-------
molecules: :obj`list`
List of molecules with HDF5 opacities
"""
from glob import glob
import os
from taurex.opacity.hdf5opacity import HDF5Opacity
glob_path = [os.path.join(self._opacity_path,'*.h5'),os.path.join(self._opacity_path,'*.hdf5')]
file_list = [f for glist in glob_path for f in glob(glist)]
return [HDF5Opacity(f,interpolation_mode=self._default_interpolation,in_memory=False).moleculeName for f in file_list ]
def search_pickle_molecules(self):
"""
Find molecules with ``.pickle`` opacities in set path
Returns
-------
molecules: :obj`list`
List of molecules with ``.pickle`` opacities
"""
from glob import glob
import os
glob_path = os.path.join(self._opacity_path,'*.pickle')
file_list = [f for f in glob(glob_path)]
return [pathlib.Path(f).stem.split('.')[0] for f in file_list ]
def search_exotransmit_molecules(self):
"""
Find molecules with Exo-Transmit opacities in set path
Returns
-------
molecules: :obj`list`
List of molecules with ExoTransmit opacities
"""
from glob import glob
import os
glob_path = os.path.join(self._opacity_path,'*.dat')
file_list = [f for f in glob(glob_path)]
return [pathlib.Path(f).stem[4:] for f in file_list ]
def search_radis_molecules(self):
"""
Searches for molecules in HITRAN
Returns
-------
molecules: :obj`list`
List of molecules available in HITRAN, if radis is enabled,
otherwise an empty list
"""
trans = { '1':'H2O', '2':'CO2', '3':'O3', '4':'N2O', '5':'CO', '6':'CH4', '7':'O2',
'9':'SO2', '10':'NO2', '11':'NH3', '12':'HNO3', '13':'OH', '14':'HF', '15':'HCl', '16':'HBr',
'17':'HI', '18':'ClO', '19':'OCS', '20':'H2CO', '21':'HOCl', '23':'HCN', '24':'CH3Cl',
'25':'H2O2', '26':'C2H2', '27':'C2H6', '28':'PH3', '29':'COF2', '30':'SF6', '31':'H2S', '32':'HCOOH',
'33':'HO2', '34':'O', '35':'ClONO2', '36':'NO+', '37':'HOBr', '38':'C2H4', '40':'CH3Br',
'41':'CH3CN', '42':'CF4', '43':'C4H2', '44':'HC3N', '46':'CS', '47':'SO3'}
if self._radis:
return list(trans.values())
else:
return []
def find_list_of_molecules(self):
from glob import glob
import os
from taurex.opacity import PickleOpacity
pickles = []
hedef = []
exo = []
if self._opacity_path is not None:
pickles = self.search_pickle_molecules()
hedef = self.search_hdf5_molecules()
exo = self.search_exotransmit_molecules()
return list(set(pickles+hedef+exo+self.search_radis_molecules()))
def load_opacity_from_path(self,path,molecule_filter=None):
"""
Searches path for molecular cross-section files, creates and loads them into the cache
``.pickle`` will be loaded as :class:`~taurex.opacity.pickleopacity.PickleOpacity`
Parameters
----------
path : str
Path to search for molecular cross-section files
molecule_filter : :obj:`list` of str , optional
If provided, the opacity will only be loaded
if its molecule is in this list. Mostly used by the
:func:`__getitem__` for filtering
"""
from glob import glob
import os
from taurex.opacity import PickleOpacity
from taurex.opacity.hdf5opacity import HDF5Opacity
from taurex.opacity.exotransmit import ExoTransmitOpacity
glob_path = [os.path.join(path,'*.h5'),os.path.join(path,'*.hdf5'),os.path.join(path,'*.pickle'),os.path.join(path,'*.dat')]
file_list = [f for glist in glob_path for f in glob(glist)]
self.log.debug('File list %s',file_list)
for files in file_list:
op = None
if files.lower().endswith(('.hdf5', '.h5')):
op = HDF5Opacity(files,interpolation_mode=self._default_interpolation,in_memory=False)
if molecule_filter is not None:
if not op.moleculeName in molecule_filter:
continue
if op.moleculeName in self.opacity_dict.keys():
continue
del op
op = HDF5Opacity(files,interpolation_mode=self._default_interpolation,in_memory=self._memory_mode)
elif files.endswith('pickle'):
splits = pathlib.Path(files).stem.split('.')
if molecule_filter is not None:
if not splits[0] in molecule_filter:
continue
if splits[0] in self.opacity_dict.keys():
continue
op = PickleOpacity(files,interpolation_mode=self._default_interpolation)
op._molecule_name = splits[0]
elif files.endswith('dat'):
mol_name = pathlib.Path(files).stem[4:]
if molecule_filter is not None:
if not mol_name in molecule_filter:
continue
if mol_name in self.opacity_dict.keys():
continue
op = ExoTransmitOpacity(files,interpolation_mode=self._default_interpolation)
if op is not None:
self.add_opacity(op,molecule_filter=molecule_filter)
def load_opacity(self,opacities=None,opacity_path=None,molecule_filter=None):
"""
Main function to use when loading molecular opacities. Handles both
cross sections and paths. Handles lists of either so lists of
:class:`~taurex.opacity.opacity.Opacity` objects or lists of paths can be used
to load multiple files/objects
Parameters
----------
opacities : :class:`~taurex.opacity.opacity.Opacity` or :obj:`list` of :class:`~taurex.opacity.opacity.Opacity` , optional
Object(s) to include in cache
opacity_path : str or :obj:`list` of str, optional
search path(s) to look for molecular opacities
molecule_filter : :obj:`list` of str , optional
If provided, the opacity will only be loaded
if its molecule is in this list. Mostly used by the
:func:`__getitem__` for filtering
"""
from taurex.opacity import Opacity
if opacity_path is None:
opacity_path = self._opacity_path
if opacities is not None:
if isinstance(opacities,(list,)):
self.log.debug('Opacity passed is list')
for opacity in opacities:
self.add_opacity(opacity,molecule_filter=molecule_filter)
elif isinstance(opacities,Opacity):
self.add_opacity(opacities,molecule_filter=molecule_filter)
else:
self.log.error('Unknown type %s passed into opacities, should be a list, single \
opacity or None if reading a path',type(opacities))
raise Exception('Unknown type passed into opacities')
elif opacity_path is not None:
if isinstance(opacity_path,str):
self.load_opacity_from_path(opacity_path,molecule_filter=molecule_filter)
elif isinstance(opacity_path,(list,)):
for path in opacity_path:
self.load_opacity_from_path(path,molecule_filter=molecule_filter)
def clear_cache(self):
"""
Clears all currently loaded cross-sections
"""
self.opacity_dict = {}
class OpacityCache(Singleton):
"""
Implements a lazy load of opacities. A singleton that
loads and caches xsections as they are needed. Calling
>>> opt = OpacityCache()
>>> opt2 = OpacityCache()
Reveals that:
>>> opt == opt2
True
Importantly this class will automatically search directories for cross-sections
set using the :func:`set_opacity_path` method:
>>> opt.set_opacity_path('path/to/crossections')
Multiple paths can be set as well
>>> opt.set_opacity_path(['/path/to/crosssections','/another/path/to/crosssections'])
To get the cross-section object for a particular molecule use the square bracket operator:
>>> opt['H2O']
<taurex.opacity.pickleopacity.PickleOpacity at 0x107a60be0>
This returns a :class:`~taurex.opacity.opacity.Opacity` object for you to compute H2O cross sections from.
When called for the first time, a directory search is performed and, if found, the appropriate cross-section is loaded. Subsequent calls will immediately
return the already loaded object:
>>> h2o_a = opt['H2O']
>>> h2o_b = opt['H2O']
>>> h2o_a == h2o_b
True
If you have any plugins that include new opacity formats, the cache
will automatically detect them.
Lastly you can manually add an opacity directly for a molecule
into the cache:
>>> new_h2o = MyNewOpacityFormat()
>>> new_h2o.molecule
H2O
>>> opt.add_opacity(new_h2o)
>>> opt['H2O']
<MyNewOpacityFormat at 0x107a60be0>
Now TauREx3 will use it instead in all calculations!
"""
def init(self):
self.opacity_dict = {}
self._opacity_path = None
self.log = Logger('OpacityCache')
self._force_active = []
def set_opacity_path(self, opacity_path):
"""
Set the path(s) that will be searched for opacities.
Opacities in this path must be of supported types:
- HDF5 opacities
- ``.pickle`` opacities
- ExoTransmit opacities.
Parameters
----------
opacity_path : str or :obj:`list` of str, optional
search path(s) to look for molecular opacities
"""
import os
GlobalCache()['xsec_path'] = opacity_path
if not os.path.isdir(opacity_path):
self.log.error('PATH: %s does not exist!!!', opacity_path)
raise NotADirectoryError
self.log.debug('Path set to %s', opacity_path)
def enable_radis(self, enable):
"""
Enables/Disables use of RADIS to fill in missing molecules
using HITRAN.
.. warning::
This is extremely unstable and crashes frequently.
It is also very slow as it requires
the computation of the Voigt profile for every temperature.
We recommend leaving it as False unless necessary.
Parameters
----------
enable: bool
Whether to enable RADIS functionality (default = False)
"""
GlobalCache()['enable_radis'] = enable
def set_radis_wavenumber(self, wn_start, wn_end, wn_points):
GlobalCache()['radius_grid'] = wn_start, wn_end, wn_points
self.clear_cache()
def set_memory_mode(self, in_memory):
"""
If using the HDF5 opacities, whether to stream
opacities from file (slower, less memory) or load
them into memory (faster, more memory)
Parameters
----------
in_memory: bool
Whether HDF5 files should be streamed (False)
or loaded into memory (True, default)
"""
GlobalCache()['xsec_in_memory'] = in_memory
self.clear_cache()
def force_active(self, molecules):
"""
Allows some molecules to be forced as active.
Useful when using other radiative codes to do the calculation
Parameters
----------
molecules: obj:`list`
List of molecules
"""
self._force_active = molecules
def set_interpolation(self, interpolation_mode):
"""
Sets the interpolation mode for all currently loaded (and future loaded) cross-sections
Can either be ``linear`` for linear interpolation of both temeprature and pressure:
>>> OpacityCache().set_interpolation('linear')
or ``exp`` for natural exponential interpolation of temperature
and linear for pressure
>>> OpacityCache().set_interpolation('exp')
Parameters
----------
interpolation_mode: str
Either ``linear`` for bilinear interpolation or ``exp`` for exp-linear interpolation
"""
GlobalCache()['xsec_interpolation'] = interpolation_mode
self.clear_cache()
def __getitem__(self, key):
"""
For a molecule return the relevant :class:`~taurex.opacity.opacity.Opacity` object.
Parameter
---------
key : str
molecule name
Returns
-------
:class:`~taurex.opacity.pickleopacity.PickleOpacity`
Cross-section object desired
Raise
-----
Exception
If molecule could not be loaded/found
"""
if key in self.opacity_dict:
return self.opacity_dict[key]
else:
#Try a load of the opacity
self.load_opacity(molecule_filter=[key])
#If we have it after a load then good job boys
if key in self.opacity_dict:
return self.opacity_dict[key]
else:
#Otherwise throw an error
self.log.error('Opacity for molecule %s could not be loaded',
key)
self.log.error(
'It could not be found in the local dictionary %s',
list(self.opacity_dict.keys()))
self.log.error('Or paths %s', GlobalCache()['xsec_path'])
self.log.error('Try loading it manually/ putting it in a path')
raise Exception('Opacity could not be loaded')
def add_opacity(self, opacity, molecule_filter=None):
"""
Adds a :class:`~taurex.opacity.opacity.Opacity` object to the cache to then be
used by Taurex 3
Parameters
----------
opacity : :class:`~taurex.opacity.opacity.Opacity`
Opacity object to add to the cache
molecule_filter : :obj:`list` of str , optional
If provided, the opacity object will only be included
if its molecule is in the list. Mostly used by the
:func:`__getitem__` for filtering
"""
self.log.info('Reading opacity %s', opacity.moleculeName)
if opacity.moleculeName in self.opacity_dict:
self.log.warning(
'Opacity with name %s already in opactiy dictionary %s skipping',
opacity.moleculeName, self.opacity_dict.keys())
return
if molecule_filter is not None:
if opacity.moleculeName in molecule_filter:
self.log.info('Loading opacity %s into model',
opacity.moleculeName)
self.opacity_dict[opacity.moleculeName] = opacity
else:
self.log.info('Loading opacity %s into model',
opacity.moleculeName)
self.opacity_dict[opacity.moleculeName] = opacity
def find_list_of_molecules(self):
from glob import glob
import os
from taurex.parameter.classfactory import ClassFactory
opacity_klasses = ClassFactory().opacityKlasses
molecules = []
for c in opacity_klasses:
molecules.extend([x[0] for x in c.discover()])
forced = self._force_active or []
return set(molecules + forced + list(self.opacity_dict.keys()))
def load_opacity_from_path(self, path, molecule_filter=None):
"""
Searches path for molecular cross-section files, creates and loads them into the cache
``.pickle`` will be loaded as :class:`~taurex.opacity.pickleopacity.PickleOpacity`
Parameters
----------
path : str
Path to search for molecular cross-section files
molecule_filter : :obj:`list` of str , optional
If provided, the opacity will only be loaded
if its molecule is in this list. Mostly used by the
:func:`__getitem__` for filtering
"""
from taurex.parameter.classfactory import ClassFactory
cf = ClassFactory()
opacity_klass_list = sorted(cf.opacityKlasses,
key=lambda x: x.priority())
for c in opacity_klass_list:
for mol, args in c.discover():
self.log.debug('Klass: %s %s', mol, args)
op = None
if mol in molecule_filter and mol not in self.opacity_dict:
if not isinstance(args, (
list,
tuple,
)):
args = [args]
op = c(*args)
if op is not None and op.moleculeName not in self.opacity_dict:
self.add_opacity(op, molecule_filter=molecule_filter)
op = None # Ensure garbage collection when run once
def load_opacity(self,
opacities=None,
opacity_path=None,
molecule_filter=None):
"""
Main function to use when loading molecular opacities. Handles both
cross sections and paths. Handles lists of either so lists of
:class:`~taurex.opacity.opacity.Opacity` objects or lists of paths can be used
to load multiple files/objects
Parameters
----------
opacities : :class:`~taurex.opacity.opacity.Opacity` or :obj:`list` of :class:`~taurex.opacity.opacity.Opacity` , optional
Object(s) to include in cache
opacity_path : str or :obj:`list` of str, optional
search path(s) to look for molecular opacities
molecule_filter : :obj:`list` of str , optional
If provided, the opacity will only be loaded
if its molecule is in this list. Mostly used by the
:func:`__getitem__` for filtering
"""
from taurex.opacity import Opacity
if opacity_path is None:
opacity_path = GlobalCache()['xsec_path']
if opacities is not None:
if isinstance(opacities, (list, )):
self.log.debug('Opacity passed is list')
for opacity in opacities:
self.add_opacity(opacity, molecule_filter=molecule_filter)
elif isinstance(opacities, Opacity):
self.add_opacity(opacities, molecule_filter=molecule_filter)
else:
self.log.error(
'Unknown type %s passed into opacities, should be a list, single \
opacity or None if reading a path', type(opacities))
raise Exception('Unknown type passed into opacities')
else:
self.load_opacity_from_path(opacity_path,
molecule_filter=molecule_filter)
# if isinstance(opacity_path, str):
# self.load_opacity_from_path(opacity_path, molecule_filter=molecule_filter)
# elif isinstance(opacity_path, (list,)):
# for path in opacity_path:
# self.load_opacity_from_path(path, molecule_filter=molecule_filter)
def clear_cache(self):
"""
Clears all currently loaded cross-sections
"""
self.opacity_dict = {}
class CIACache(Singleton):
"""
Implements a lazy load of collisionally induced absorpiton cross-sections
Supports pickle files and HITRAN cia files. Functionally behaves the same
as :class:`~taurex.cache.opacitycache.OpacityCache` except the keys are
now cia pairs
e.g:
>>> CIACache()['H2-H2']
<taurex.cia.picklecia.PickleCIA at 0x107a60be0>
Pickle ``.db`` and HITRAN ``.cia`` files are supported and automatically
loaded. with priority given to ``.db`` files
"""
def init(self):
self.cia_dict = {}
self._cia_path = None
self.log = Logger('CIACache')
def set_cia_path(self, cia_path):
"""
Sets the path to search for CIA files
Parameters
----------
cia_path : str or :obj:`list` of str
Either a single path or a list of paths that contain CIA files
"""
self._cia_path = cia_path
def __getitem__(self, key):
"""
For a CIA pair, load from the set path and return the
relevant :class:`~taurex.cia.cia.CIA` object
Parameter
---------
key : str
cia pair name
Returns
-------
:class:`~taurex.cia.picklecia.PickleCIA` or :class:`~taurex.cia.hitrancia.HitranCIA`
Desire CIA object, format depends on what is found
in the path set by :func:`set_cia_path`
Raise
-----
Exception
If desired CIA pair could not be loaded
"""
if key in self.cia_dict:
return self.cia_dict[key]
else:
# Try a load of the cia
self.load_cia(pair_filter=[key])
# If we have it after a load then good job boys
if key in self.cia_dict:
return self.cia_dict[key]
else:
# Otherwise throw an error
self.log.error('CIA for pair %s could not be loaded', key)
self.log.error(
'It could not be found in the local dictionary '
' %s', list(self.cia_dict.keys()))
self.log.error('Or paths %s', self._cia_path)
self.log.error('Try loading it manually/ putting it in a path')
raise Exception('cia could notn be loaded')
def add_cia(self, cia, pair_filter=None):
"""
Adds a :class:`~taurex.cia.cia.CIA` object to the cache to then be
used by Taurex 3
Parameters
----------
cia : :class:`~taurex.cia.cia.CIA`
CIA object to add to the cache
pair_filter : :obj:`list` of str , optional
If provided, the cia object will only be included
if its pairname is in the list. Mostly used by the
:func:`__getitem__` for filtering
"""
self.log.info('Loading cia %s into model', cia.pairName)
if cia.pairName in self.cia_dict:
self.log.error(
'cia with name %s already in '
'opactiy dictionary %s', cia.pairName, self.cia_dict.keys())
raise Exception('cia for molecule %s already exists')
if pair_filter is not None:
if cia.pairName in pair_filter:
self.log.info('Loading cia %s into model', cia.pairName)
self.cia_dict[cia.pairName] = cia
self.cia_dict[cia.pairName] = cia
def load_cia_from_path(self, path, pair_filter=None):
"""
Searches path for CIA files, creates and loads them into the cache
``.db`` will be loaded as :class:`~taurex.cia.picklecia.PickleCIA` and
``.cia`` files will be loaded as
:class:`~taurex.cia.hitrancia.HitranCIA`
Parameters
----------
path : str
Path to search for CIA files
pair_filter : :obj:`list` of str , optional
If provided, the cia will only be loaded
if its pairname is in the list. Mostly used by the
:func:`__getitem__` for filtering
"""
from glob import glob
from pathlib import Path
import os
from taurex.cia import PickleCIA
# Find .db files
glob_path = os.path.join(path, '*.db')
file_list = glob(glob_path)
self.log.debug('Glob list: %s', glob_path)
self.log.debug('File list FOR CIA %s', file_list)
for files in file_list:
pairname = Path(files).stem.split('_')[0]
self.log.debug('pairname found %s', pairname)
if pair_filter is not None:
if pairname not in pair_filter:
continue
op = PickleCIA(files, pairname)
self.add_cia(op)
# Find .cia files
glob_path = os.path.join(path, '*.cia')
file_list = glob(glob_path)
self.log.debug('File list %s', file_list)
for files in file_list:
from taurex.cia import HitranCIA
pairname = Path(files).stem.split('_')[0]
if pair_filter is not None:
if pairname not in pair_filter:
continue
op = HitranCIA(files)
self.add_cia(op)
def load_cia(self, cia_xsec=None, cia_path=None, pair_filter=None):
"""
Main function to use when loading CIA files. Handles both
cross sections and paths. Handles lists of either so lists of
:class:`~taurex.cia.cia.CIA` objects or lists of paths can be used
to load multiple files/objects
Parameters
----------
cia_xsec : :class:`~taurex.cia.cia.CIA` or :obj:`list` of :class:`~taurex.cia.cia.CIA` , optional
Object(s) to include in cache
cia_path : str or :obj:`list` of str, optional
search path(s) to look for cias
pair_filter : :obj:`list` of str , optional
If provided, the cia will only be loaded
if its pair name is in this list. Mostly used by the
:func:`__getitem__` for filtering
"""
from taurex.cia import CIA
if cia_path is None:
cia_path = self._cia_path
self.log.debug('CIA XSEC, CIA_PATH %s %s', cia_xsec, cia_path)
if cia_xsec is not None:
if isinstance(cia_xsec, (list, )):
self.log.debug('cia passed is list')
for xsec in cia_xsec:
self.add_cia(xsec, pair_filter=pair_filter)
elif isinstance(cia_xsec, CIA):
self.add_cia(cia_xsec, pair_filter=pair_filter)
else:
self.log.error(
'Unknown type %s passed into cia, should be a list, single \
cia or None if reading a path', type(xsec))
raise Exception('Unknown type passed into cia')
if cia_path is not None:
if isinstance(cia_path, str):
self.load_cia_from_path(cia_path, pair_filter=pair_filter)
elif isinstance(cia_path, (list, )):
for path in cia_path:
self.load_cia_from_path(path, pair_filter=pair_filter)
class ClassFactory(Singleton):
"""
A factory the discovers new
classes from plugins
"""
def init(self):
self.log = Logger('ClassFactory')
self.extension_paths = []
self.reload_plugins()
def set_extension_paths(self, paths=None, reload=True):
self.extension_paths = paths
if reload:
self.reload_plugins()
def reload_plugins(self):
self.log.info('Reloading all modules and plugins')
self.setup_batteries_included()
self.setup_batteries_included_mixin()
self.load_plugins()
self.load_extension_paths()
def setup_batteries_included_mixin(self):
"""
Collect all the classes that are built into
TauREx 3
"""
from taurex.mixin import mixins
self._temp_mixin_klasses = set()
self._chem_mixin_klasses = set()
self._gas_mixin_klasses = set()
self._press_mixin_klasses = set()
self._planet_mixin_klasses = set()
self._star_mixin_klasses = set()
self._inst_mixin_klasses = set()
self._model_mixin_klasses = set()
self._contrib_mixin_klasses = set()
self._opt_mixin_klasses = set()
self._obs_mixin_klasses = set()
self._temp_mixin_klasses.update(
self._collect_temperatures_mixin(mixins))
self._chem_mixin_klasses.update(self._collect_chemistry_mixin(mixins))
self._gas_mixin_klasses.update(self._collect_gas_mixin(mixins))
self._press_mixin_klasses.update(self._collect_pressure_mixin(mixins))
self._planet_mixin_klasses.update(self._collect_planets_mixin(mixins))
self._star_mixin_klasses.update(self._collect_star_mixin(mixins))
self._inst_mixin_klasses.update(self._collect_instrument_mixin(mixins))
self._model_mixin_klasses.update(self._collect_model_mixin(mixins))
self._obs_mixin_klasses.update(self._collect_observation_mixin(mixins))
self._contrib_mixin_klasses.update(
self._collect_contributions_mixin(mixins))
self._opt_mixin_klasses.update(self._collect_optimizer_mixin(mixins))
def setup_batteries_included(self):
"""
Collect all the classes that are built into
TauREx 3
"""
from taurex import temperature, chemistry, pressure, planet, \
stellar, instruments, model, contributions, optimizer, opacity, \
spectrum
from taurex.opacity import ktables
from taurex.core import priors
self._temp_klasses = set()
self._chem_klasses = set()
self._gas_klasses = set()
self._press_klasses = set()
self._planet_klasses = set()
self._star_klasses = set()
self._inst_klasses = set()
self._model_klasses = set()
self._contrib_klasses = set()
self._opt_klasses = set()
self._opac_klasses = set()
self._ktab_klasses = set()
self._obs_klasses = set()
self._prior_klasses = set()
self._temp_klasses.update(self._collect_temperatures(temperature))
self._chem_klasses.update(self._collect_chemistry(chemistry))
self._gas_klasses.update(self._collect_gas(chemistry))
self._press_klasses.update(self._collect_pressure(pressure))
self._planet_klasses.update(self._collect_planets(planet))
self._star_klasses.update(self._collect_star(stellar))
self._inst_klasses.update(self._collect_instrument(instruments))
self._model_klasses.update(self._collect_model(model))
self._obs_klasses.update(self._collect_observation(spectrum))
self._contrib_klasses.update(
self._collect_contributions(contributions))
self._opt_klasses.update(self._collect_optimizer(optimizer))
self._opac_klasses.update(self._collect_opacity(opacity))
self._ktab_klasses.update(self._collect_ktables(ktables))
self._prior_klasses.update(self._collect_priors(priors))
def load_plugin(self, plugin_module):
self._temp_klasses.update(self._collect_temperatures(plugin_module))
self._chem_klasses.update(self._collect_chemistry(plugin_module))
self._gas_klasses.update(self._collect_gas(plugin_module))
self._press_klasses.update(self._collect_pressure(plugin_module))
self._planet_klasses.update(self._collect_planets(plugin_module))
self._star_klasses.update(self._collect_star(plugin_module))
self._inst_klasses.update(self._collect_instrument(plugin_module))
self._model_klasses.update(self._collect_model(plugin_module))
self._obs_klasses.update(self._collect_observation(plugin_module))
self._contrib_klasses.update(
self._collect_contributions(plugin_module))
self._opt_klasses.update(self._collect_optimizer(plugin_module))
self._opac_klasses.update(self._collect_opacity(plugin_module))
self._prior_klasses.update(self._collect_priors(plugin_module))
self._ktab_klasses.update(self._collect_ktables(plugin_module))
# Load any mixins
self._temp_mixin_klasses.update(
self._collect_temperatures_mixin(plugin_module))
self._chem_mixin_klasses.update(
self._collect_chemistry_mixin(plugin_module))
self._gas_mixin_klasses.update(
self._collect_gas_mixin(plugin_module))
self._press_mixin_klasses.update(
self._collect_pressure_mixin(plugin_module))
self._planet_mixin_klasses.update(
self._collect_planets_mixin(plugin_module))
self._star_mixin_klasses.update(
self._collect_star_mixin(plugin_module))
self._inst_mixin_klasses.update(
self._collect_instrument_mixin(plugin_module))
self._model_mixin_klasses.update(
self._collect_model_mixin(plugin_module))
self._obs_mixin_klasses.update(
self._collect_observation_mixin(plugin_module))
self._contrib_mixin_klasses.update(
self._collect_contributions_mixin(plugin_module))
self._opt_mixin_klasses.update(
self._collect_optimizer_mixin(plugin_module))
def discover_plugins(self):
plugins = {}
failed_plugins = {}
for entry_point in pkg_resources.iter_entry_points('taurex.plugins'):
entry_point_name = entry_point.name
try:
module = entry_point.load()
except Exception as e:
# For whatever reason do not attempt to load the plugin
self.log.warning('Could not load plugin %s', entry_point_name)
self.log.warning('Reason: %s', str(e))
failed_plugins[entry_point_name] = str(e)
continue
plugins[entry_point_name] = module
return plugins, failed_plugins
def load_plugins(self):
plugins, failed_plugins = self.discover_plugins()
self.log.info('----------Plugin loading---------')
self.log.debug('Discovered plugins %s', plugins.values())
for k, v in plugins.items():
self.log.info('Loading %s', k)
self.load_plugin(v)
def load_extension_paths(self):
import glob
import os
import pathlib
import importlib
paths = self.extension_paths
if paths:
# Make sure they're unique
all_files = set(sum([glob.glob(
os.path.join(os.path.abspath(p), '*.py'))
for p in paths], []))
for f in all_files:
self.info('Loading extensions from %s', f)
module_name = pathlib.Path(f).stem
spec = importlib.util.spec_from_file_location(module_name, f)
foo = importlib.util.module_from_spec(spec)
try:
spec.loader.exec_module(foo)
self.load_plugin(foo)
except Exception as e:
self.log.warning('Could not load extension from file %s',
f)
self.log.warning('Reason: %s', str(e))
def _collect_classes(self, module, base_klass):
"""
Collects all classes that are a subclass of base
"""
klasses = []
clsmembers = inspect.getmembers(module, inspect.isclass)
for _, c in clsmembers:
if issubclass(c, base_klass) and (c is not base_klass):
self.log.debug(f' Found class {c.__name__}')
klasses.append(c)
return klasses
def _collect_temperatures(self, module):
from taurex.temperature import TemperatureProfile
return self._collect_classes(module, TemperatureProfile)
def _collect_chemistry(self, module):
from taurex.chemistry import Chemistry
return self._collect_classes(module, Chemistry)
def _collect_gas(self, module):
from taurex.chemistry import Gas
return self._collect_classes(module, Gas)
def _collect_pressure(self, module):
from taurex.pressure import PressureProfile
return self._collect_classes(module, PressureProfile)
def _collect_planets(self, module):
from taurex.planet import BasePlanet
return self._collect_classes(module, BasePlanet)
def _collect_star(self, module):
from taurex.stellar import Star
return self._collect_classes(module, Star)
def _collect_instrument(self, module):
from taurex.instruments import Instrument
return self._collect_classes(module, Instrument)
def _collect_model(self, module):
from taurex.model import ForwardModel, SimpleForwardModel
return [c for c in self._collect_classes(module, ForwardModel)
if c is not SimpleForwardModel]
def _collect_contributions(self, module):
from taurex.contributions import Contribution
return self._collect_classes(module, Contribution)
def _collect_optimizer(self, module):
from taurex.optimizer import Optimizer
return self._collect_classes(module, Optimizer)
def _collect_opacity(self, module):
from taurex.opacity import Opacity, InterpolatingOpacity
from taurex.opacity.ktables import KTable
return [c for c in self._collect_classes(module, Opacity)
if c is not InterpolatingOpacity and not issubclass(c, KTable)]
def _collect_ktables(self, module):
from taurex.opacity.ktables import KTable
return [c for c in self._collect_classes(module, KTable)]
def _collect_observation(self, module):
from taurex.spectrum import BaseSpectrum
return [c for c in self._collect_classes(module, BaseSpectrum)]
def _collect_priors(self, module):
from taurex.core.priors import Prior
return [c for c in self._collect_classes(module, Prior)]
# Mixins
def _collect_temperatures_mixin(self, module):
from taurex.mixin import TemperatureMixin
return self._collect_classes(module, TemperatureMixin)
def _collect_chemistry_mixin(self, module):
from taurex.mixin import ChemistryMixin
return self._collect_classes(module, ChemistryMixin)
def _collect_gas_mixin(self, module):
from taurex.mixin import GasMixin
return self._collect_classes(module, GasMixin)
def _collect_pressure_mixin(self, module):
from taurex.mixin import PressureMixin
return self._collect_classes(module, PressureMixin)
def _collect_planets_mixin(self, module):
from taurex.mixin import PlanetMixin
return self._collect_classes(module, PlanetMixin)
def _collect_star_mixin(self, module):
from taurex.mixin import StarMixin
return self._collect_classes(module, StarMixin)
def _collect_instrument_mixin(self, module):
from taurex.mixin import InstrumentMixin
return self._collect_classes(module, InstrumentMixin)
def _collect_model_mixin(self, module):
from taurex.mixin import ForwardModelMixin
return self._collect_classes(module, ForwardModelMixin)
def _collect_contributions_mixin(self, module):
from taurex.mixin import ContributionMixin
return self._collect_classes(module, ContributionMixin)
def _collect_optimizer_mixin(self, module):
from taurex.mixin import OptimizerMixin
return self._collect_classes(module, OptimizerMixin)
def _collect_observation_mixin(self, module):
from taurex.mixin import ObservationMixin
return self._collect_classes(module, ObservationMixin)
def list_from_base(self, klass_type):
from taurex.temperature import TemperatureProfile
from taurex.chemistry import Chemistry
from taurex.chemistry import Gas
from taurex.pressure import PressureProfile
from taurex.planet import BasePlanet
from taurex.stellar import Star
from taurex.instruments import Instrument
from taurex.model import ForwardModel
from taurex.contributions import Contribution
from taurex.optimizer import Optimizer
from taurex.opacity import Opacity
from taurex.opacity.ktables import KTable
from taurex.spectrum import BaseSpectrum
from taurex.core.priors import Prior
klass_dict = {
TemperatureProfile: self.temperatureKlasses,
Chemistry: self.chemistryKlasses,
Gas: self.gasKlasses,
PressureProfile: self.pressureKlasses,
BasePlanet: self.planetKlasses,
Star: self.starKlasses,
Instrument: self.instrumentKlasses,
ForwardModel: self.modelKlasses,
Contribution: self.contributionKlasses,
Optimizer: self.optimizerKlasses,
Opacity: self.opacityKlasses,
KTable: self.ktableKlasses,
BaseSpectrum: self.observationKlasses,
Prior: self.priorKlasses,
}
return klass_dict[klass_type]
@property
def temperatureKlasses(self):
return self._temp_klasses
@property
def chemistryKlasses(self):
return self._chem_klasses
@property
def gasKlasses(self):
return self._gas_klasses
@property
def pressureKlasses(self):
return self._press_klasses
@property
def planetKlasses(self):
return self._planet_klasses
@property
def starKlasses(self):
return self._star_klasses
@property
def instrumentKlasses(self):
return self._inst_klasses
@property
def modelKlasses(self):
return self._model_klasses
@property
def contributionKlasses(self):
return self._contrib_klasses
@property
def optimizerKlasses(self):
return self._opt_klasses
@property
def opacityKlasses(self):
return self._opac_klasses
@property
def ktableKlasses(self):
return self._ktab_klasses
@property
def observationKlasses(self):
return self._obs_klasses
@property
def priorKlasses(self):
return self._prior_klasses
# Mixins
@property
def temperatureMixinKlasses(self):
return self._temp_mixin_klasses
@property
def chemistryMixinKlasses(self):
return self._chem_mixin_klasses
@property
def gasMixinKlasses(self):
return self._gas_mixin_klasses
@property
def pressureMixinKlasses(self):
return self._press_mixin_klasses
@property
def planetMixinKlasses(self):
return self._planet_mixin_klasses
@property
def starMixinKlasses(self):
return self._star_mixin_klasses
@property
def instrumentMixinKlasses(self):
return self._inst_mixin_klasses
@property
def modelMixinKlasses(self):
return self._model_mixin_klasses
@property
def contributionMixinKlasses(self):
return self._contrib_mixin_klasses
@property
def optimizerMixinKlasses(self):
return self._opt_mixin_klasses
@property
def observationMixinKlasses(self):
return self._obs_mixin_klasses