def test_expand_ensemble_nolist(self):
datasets = [
{
'dataset': 'XYZ',
'ensemble': ['r1i1p1', 'r(1:2)i1p1']
},
]
with pytest.raises(RecipeError):
Recipe._expand_tag(datasets, 'ensemble')
def _load(self, session: dict):
"""Load the recipe.
This method loads the recipe into the internal ESMValCore Recipe
format.
Parameters
----------
session : :obj:`Session`
Defines the config parameters and location where the recipe
output will be stored. If ``None``, a new session will be
started automatically.
Returns
-------
recipe : :obj:`esmvalcore._recipe.Recipe`
Return an instance of the Recipe
"""
config_user = session.to_config_user()
logger.info(pprint.pformat(config_user))
self._engine = RecipeEngine(raw_recipe=self.data,
config_user=config_user,
recipe_file=self.path)
def test_expand_subexperiment(self):
datasets = [
{
'dataset': 'XYZ',
'sub_experiment': 's(1998:2005)',
},
]
expanded = Recipe._expand_tag(datasets, 'sub_experiment')
subexperiments = [
's1998',
's1999',
's2000',
's2001',
's2002',
's2003',
's2004',
's2005',
]
for i, subexperiment in enumerate(subexperiments):
assert expanded[i] == {
'dataset': 'XYZ',
'sub_experiment': subexperiment
}
def test_expand_ensemble(self):
datasets = [
{
'dataset': 'XYZ',
'ensemble': 'r(1:2)i(2:3)p(3:4)',
},
]
expanded = Recipe._expand_tag(datasets, 'ensemble')
ensembles = [
'r1i2p3',
'r1i2p4',
'r1i3p3',
'r1i3p4',
'r2i2p3',
'r2i2p4',
'r2i3p3',
'r2i3p4',
]
for i, ensemble in enumerate(ensembles):
assert expanded[i] == {'dataset': 'XYZ', 'ensemble': ensemble}
def generate_script_settings(
raw_recipe: dict,
config_file: str,
recipe_file: str,
diagnostic_name: str,
script_name: str,
) -> dict:
"""
Generate diagnostic script settings using esmvalcore.Recipe
Parameters
----------
raw_recipe: dict
config_file: str
config_file_path
recipe_file: str
recipe file path.
diagnostic_name: str
script_name: str
Returns
-------
dict:
script settings
{
# from diagnostic setting
'quickplot': {
'plot_type': 'pcolormesh'
},
# added by Recipe
'recipe': 'recipe.yml',
'version': '2.0.0b5',
'script': 'script1',
# from config_user
'run_dir': '/some/path/recipe_20200303_022547/run/diagnostic1/script1',
'plot_dir': '/some/path/recipe_20200303_022547/plots/diagnostic1/script1',
'work_dir': '/some/path/recipe_20200303_022547/work/diagnostic1/script1',
'max_data_filesize': 100,
'output_file_type': 'png',
'log_level': 'info',
'write_plots': True,
'write_netcdf': True,
'profile_diagnostic': False,
'auxiliary_data_dir': '/some/path/auxiliary_data',
# from diagnostic setting
'themes': ['phys'],
'realms': ['atmos']
}
"""
config_user = read_config_user_file(config_file=config_file,
recipe_name="recipe")
recipe = Recipe(
raw_recipe,
config_user,
initialize_tasks=False,
recipe_file=recipe_file,
)
diagnostics = recipe.diagnostics
return diagnostics[diagnostic_name]["scripts"][script_name]["settings"]
class Recipe():
"""API wrapper for the esmvalcore Recipe object.
This class can be used to inspect and run the recipe.
Parameters
----------
path : pathlike
Path to the recipe.
"""
def __init__(self, path: str):
self.path = Path(path)
if not self.path.exists():
raise FileNotFoundError(f'Cannot find recipe: `{path}`.')
self._data = None
self._authors = None
self._maintainers = None
self._projects = None
self._references = None
self._description = None
self._engine = None
def __repr__(self) -> str:
"""Return canonical string representation."""
return f'{self.__class__.__name__}({self.name!r})'
def _repr_markdown_(self) -> str:
"""Represent using markdown renderer in a notebook environment."""
return self.render('markdown')
def __str__(self) -> str:
"""Return string representation."""
return self.render('plaintext')
def to_markdown(self) -> str:
"""Return markdown formatted string."""
return self.render('markdown')
def render(self, renderer: str = 'plaintext') -> str:
"""Return formatted string.
Parameters
----------
renderer : str
Choose the renderer for the string representation.
Must be one of 'plaintext', 'markdown'
Returns
-------
str
Rendered representation of the recipe documentation.
"""
bullet = '\n - '
string = f'## {self.name}'
string += '\n\n'
string += f'{self.description}'
string += '\n\n### Authors'
for author in self.authors:
string += f'{bullet}{author}'
string += '\n\n### Maintainers'
for maintainer in self.maintainers:
string += f'{bullet}{maintainer}'
if self.projects:
string += '\n\n### Projects'
for project in self.projects:
string += f'{bullet}{project}'
if self.references:
string += '\n\n### References'
for reference in self.references:
string += bullet + reference.render(renderer)
string += '\n'
return string
@property
def data(self) -> dict:
"""Return dictionary representation of the recipe."""
if self._data is None:
self._data = yaml.safe_load(open(self.path, 'r'))
return self._data
@property
def name(self) -> str:
"""Name of the recipe."""
return self.path.stem.replace('_', ' ').capitalize()
@property
def description(self) -> str:
"""Recipe description."""
if self._description is None:
description = self.data['documentation']['description']
self._description = '\n'.join(textwrap.wrap(description))
return self._description
@property
def authors(self) -> tuple:
"""List of recipe authors."""
if self._authors is None:
tags = self.data['documentation']['authors']
self._authors = tuple(Contributor.from_tag(tag) for tag in tags)
return self._authors
@property
def maintainers(self) -> tuple:
"""List of recipe maintainers."""
if self._maintainers is None:
tags = self.data['documentation'].get('maintainer', [])
self._maintainers = tuple(
Contributor.from_tag(tag) for tag in tags)
return self._maintainers
@property
def projects(self) -> tuple:
"""List of recipe projects."""
if self._projects is None:
tags = self.data['documentation'].get('projects', [])
self._projects = tuple(Project.from_tag(tag) for tag in tags)
return self._projects
@property
def references(self) -> tuple:
"""List of project references."""
if self._references is None:
tags = self.data['documentation'].get('references', [])
self._references = tuple(Reference.from_tag(tag) for tag in tags)
return self._references
def _load(self, session: dict):
"""Load the recipe.
This method loads the recipe into the internal ESMValCore Recipe
format.
Parameters
----------
session : :obj:`Session`
Defines the config parameters and location where the recipe
output will be stored. If ``None``, a new session will be
started automatically.
Returns
-------
recipe : :obj:`esmvalcore._recipe.Recipe`
Return an instance of the Recipe
"""
config_user = session.to_config_user()
logger.info(pprint.pformat(config_user))
self._engine = RecipeEngine(raw_recipe=self.data,
config_user=config_user,
recipe_file=self.path)
def run(self, task: str = None, session: dict = None):
"""Run the recipe.
This function loads the recipe into the ESMValCore recipe format
and runs it.
Parameters
----------
task : str
Specify the name of the diagnostic or preprocessor to run a
single task.
session : :obj:`Session`, optional
Defines the config parameters and location where the recipe
output will be stored. If ``None``, a new session will be
started automatically.
Returns
-------
output : dict
Returns output of the recipe as instances of :obj:`OutputItem`
grouped by diagnostic task.
"""
if not session:
session = CFG.start_session(self.path.stem)
if task:
session['diagnostics'] = task
with log_to_dir(session.run_dir):
self._load(session=session)
self._engine.run()
return self.get_output()
def get_output(self) -> dict:
"""Get output from recipe.
Returns
-------
output : dict
Returns output of the recipe as instances of :obj:`OutputFile`
grouped by diagnostic task.
"""
if not self._engine:
raise AttributeError('Run the recipe first using `.run()`.')
raw_output = self._engine.get_product_output()
return RecipeOutput(raw_output)
def generate_variables(raw_recipe: dict, config_file: str, recipe_file: str,
raw_variable: dict, raw_datasets: list) -> list:
"""
Use esmvalcore.Recipe to generate variables for one variable from different datasets.
Parameters
----------
raw_recipe: dict
load from recipe file, such as recipe.yml
config_file: str
config file name, will be loaded
recipe_file: str
recipe file name
raw_variable: dict
variable dict
raw_datasets:
datasets from recipe file.
Returns
-------
list:
variables generated by esmvalcore
datasets from recipe.yml["datasets"]
preprocessor1 from recipe.yml["preprocessors"]
data params from recipe.yml["diagnostics"]
[
{
'preprocessor': 'preprocessor1',
'diagnostic': 'diagnostic1',
'variable_group': 'ta',
'short_name': 'ta',
'standard_name': 'air_temperature',
'long_name': 'Air Temperature',
'units': 'K',
'institute': ['CCCma'],
'dataset': 'CanESM2',
'project': 'CMIP5',
'cmor_table': 'CMIP5',
'mip': 'Amon',
'exp': 'historical',
'ensemble': 'r1i1p1',
'frequency': 'mon',
'modeling_realm': ['atmos'],
'start_year': 1996,
'end_year': 1998,
'recipe_dataset_index': 0,
'alias': 'CanESM2',
# 'filename': (
# f'{work_dir}/preproc/diagnostic1/ta/CMIP5_CanESM2_Amon_historical_r1i1p1_ta_1996-1998.nc'
# ) # 动态生成，没有用处
},
{
'preprocessor': 'preprocessor1',
'diagnostic': 'diagnostic1',
'variable_group': 'ta',
'short_name': 'ta',
'standard_name': 'air_temperature',
'long_name': 'Air Temperature',
'units': 'K',
'institute': ['LASG-CESS'],
'dataset': 'FGOALS-g2',
'project': 'CMIP5',
'cmor_table': 'CMIP5',
'mip': 'Amon',
'exp': 'historical',
'ensemble': 'r1i1p1',
'frequency': 'mon',
'modeling_realm': ['atmos'],
'start_year': 1996,
'end_year': 1998,
'recipe_dataset_index': 1,
'alias': 'FGOALS-g2',
# 'filename': (
# f'{work_dir}/preproc/diagnostic1/ta/CMIP5_FGOALS-g2_Amon_historical_r1i1p1_ta_1996-1998.nc'
# ) # 动态生成，没有用处
}
]
"""
config_user = read_config_user_file(config_file=config_file,
folder_name="recipe")
recipe = Recipe(
raw_recipe,
config_user,
initialize_tasks=False,
recipe_file=recipe_file,
)
return recipe._initialize_variables(
raw_variable=raw_variable,
raw_datasets=raw_datasets,
)