login = "******"
if onSpip:
# Declare a list of root directories for IPSL data at TGCC
root = "/Users/marti/Volumes/CURIE/ccc/store/cont003/dsm"
print 'igcm_out : declaration root sur Spip : ', root
if root:
# -- Declare a 'CMIP' CliMAF project (a replicate of the CMIP5 project)
# ---------------------------------------------------------------------------- >
cproject('IGCM_CMIP6',
'root',
'login',
'IPSL_MODEL',
'status',
'experiment',
'realm',
'frequency',
'table',
'model',
'realization',
'grid',
ensemble=['model', 'simulation', 'realization'],
separator='%')
# --> systematic arguments = simulation, frequency, variable
# -- Set the aliases for the frequency
cfreqs('IGCM_CMIP6', {'monthly': 'mon', 'yearly': 'yr', 'daily': 'day'})
crealms('IGCM_CMIP6', {
'atmos': 'ATM',
'ocean': 'OCE',
'land': 'SRF',
'seaice': 'ICE'
})
configuration files
Simulation names (or 'EXPIDs') are assumed to be unique in the namespace defined by the user's configuration
file, which may include shared simulation
Example for defining an EM dataset::
>>> tas=ds(project='EM', simulation='GSAGNS1', variable='tas', period='1975', frequency='monthly', realm='L')
A number of Seaice fields are duly described with 1.e+20 as missing value (which is ill described in data files); see code for details
"""
from climaf.site_settings import atCNRM
if atCNRM :
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs
cproject('EM' , ('frequency','monthly'), ('realm','*') )
cfreqs('EM',{'monthly':'mon' , 'daily':'day1' })
dataloc(project='EM', organization='EM', url=['dummy'])
# Ideally, one should be able to write :
#cmissing('EM',1.e+20, realm='I')
calias('EM',[ 'sic', 'sit', 'sim', 'snd', 'ialb', 'tsice', 'mpalb', 'snomlet',
'tmelt', 'bmelt', 'snc','sic1','sic2', 'sic3', 'sic4', 'ssi', 'ageice'],
missing=1.e+20)
Syntax for these attributes is described in `the CMIP5 DRS document <http://cmip-pcmdi.llnl.gov/cmip5/docs/cmip5_data_reference_syntax.pdf>`_
Example for a CMIP5 dataset declaration ::
>>> tas1pc=ds(project='CMIP5', model='CNRM-CM5', experiment='1pctCO2', variable='tas', frequency='monthly', period='1860-1861')
"""
import os
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs,cdef
from climaf.site_settings import atCNRM, onCiclad, atCEDA
p=cproject("CMIP5" ,"model","experiment", ("frequency","monthly"),
("table","*"),("realm","*"),("version","last"),
ensemble=["model","simulation"])
cdef("simulation","r1i1p1",project="CMIP5")
# Frequency alias
cfreqs('CMIP5', {'monthly':'mon' , 'daily':'day' })
urls_CMIP5=None
if atCNRM :
# Declare the directory for CNRM-CM CMIP5 data on CNRM's Lustre file system.
urls_CMIP5=["/cnrm/cmip/cnrm/ESG"]
if onCiclad :
# Declare a list of root directories for CMIP5 data on IPLS's Ciclad file system
urls_CMIP5=["/prodigfs/project/"]
if atCEDA:
"""
This module declares the project E-OBS :
This archive is available on Ciclad (IPSL)
The specific attributes are:
- grid : '*'
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs, cdef
from climaf.site_settings import onCiclad, atTGCC, atIDRIS, atCerfacs, atCNRM
if onCiclad:
# -- Create E-OBS CliMAF project
EOBS_pattern = '/bdd/E-OBS/Grid_${grid}/${variable}_${grid}_${PERIOD}_v15.0.nc4'
cproject('E-OBS', 'grid', 'frequency', separator='%')
dataloc(project='E-OBS', organization='generic', url=EOBS_pattern)
# -- Make a
cdef('frequency', 'daily', project='E-OBS')
cdef('grid', '*deg_*', project='E-OBS')
cdef('period', '*', project='E-OBS')
cdef('variable', '*', project='E-OBS')
cfreqs('E-OBS', {'daily': 'day'})
calias('E-OBS', 'tasmin', 'tn')
calias('E-OBS', 'tasmax', 'tx')
Example for a CMIP5 dataset declaration ::
>>> tas1pc=ds(project='CMIP5', model='CNRM-CM5', experiment='1pctCO2', variable='tas', frequency='monthly', period='1860-1861')
"""
import os
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs, cdef
p = cproject(
"CMIP5",
"model",
"experiment",
("frequency", "monthly"),
("table", "*"),
("realm", "*"),
("version", "last"),
ensemble=["model", "simulation"],
)
cdef("simulation", "r1i1p1", project="CMIP5")
# Frequency alias
cfreqs("CMIP5", {"monthly": "mon", "daily": "day"})
urls_CMIP5 = None
if os.path.exists("/cnrm"):
# Declare a list of root directories for CNRM-CM CMIP5 data on CNRM's Lustre file system.
urls_CMIP5 = [
"/cnrm/aster/data2/ESG/data1",
if root:
# -- Declare the various CORDEX CliMAF project
# --------------------------------------------- >
pattern1 = '${root}/CORDEX/output/${CORDEX_domain}/${institute}/${driving_model}/${experiment}/${realization}/' \
'${model}/${model_version}/${frequency}/${variable}/${version}/' \
'${variable}_${CORDEX_domain}_${driving_model}_${experiment}_${realization}_${model}_${model_version}_' \
'${frequency}_${PERIOD}.nc'
pattern2 = '${root}/CORDEX/output/${CORDEX_domain}/${institute}/${driving_model}/${extent_experiment}/' \
'${realization}/${model}/${model_version}/${frequency}/${variable}/${version}/' \
'${variable}_${CORDEX_domain}_${driving_model}_${extent_experiment}_${realization}_${model}_' \
'${model_version}_${frequency}_${PERIOD}.nc'
# -- CORDEX
cproject('CORDEX', 'root', 'model', 'CORDEX_domain', 'model_version', 'frequency', 'driving_model',
'realization', 'experiment', 'version', 'institute', ensemble=['model', 'driving_model', 'realization'],
separator='%')
dataloc(project='CORDEX', url=[pattern1])
cdef('experiment', '*', project='CORDEX')
cdef('model_version', '*', project='CORDEX')
# -- CORDEX extent (historical + scenario at once)
cproject('CORDEX_extent', 'root', 'model', 'CORDEX_domain', 'model_version', 'frequency', 'driving_model',
'realization', 'experiment', 'extent_experiment', 'version', 'institute',
ensemble=['model', 'driving_model', 'realization'], separator='%')
dataloc(project='CORDEX_extent', url=[pattern1])
dataloc(project='CORDEX_extent', url=[pattern2])
cdef('extent_experiment', 'rcp85', project='CORDEX_extent')
cdef('experiment', 'historical', project='CORDEX_extent')
cdef('model_version', '*', project='CORDEX_extent')
Only two variables are available : the original 'precip' (mm/day) and pr (kg m-2 s-1)
Example of an 'gpcp' project dataset declaration ::
>>> cdef('project','gpcp')
>>> d=ds(variable='pr',period='198001',grid='2.5d', frequency='monthly')
>>> d2=ds(variable='pr',period='198001',grid='1d',frequency='daily')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias
from climaf.site_settings import atCNRM
if atCNRM:
cproject('gpcp', 'grid', ('frequency', 'monthly'),
separator="_") # grid writes as '1d', '2.5d', 'T42' or 'T127'
root = "/cnrm/amacs/DATA/OBS/netcdf/${frequency}"
patmonth = root + "_mean/gpcp/${variable}_gpcp.${grid}.nc"
patday = root + "/gpcp/${variable}_gpcp.${grid}.nc"
dataloc(project='gpcp', organization='generic', url=[patmonth, patday])
# Defining alias and derived variables for GPCP, together with filenames
##############################################################################
# Original data is mm/day
calias("gpcp", 'precip', 'precip', filenameVar='pr')
# Compute in SI
calias("gpcp",
'pr',
'precip',
root = "/data/jservon/IPSL_DATA/SIMULATIONS"
if onSpip:
# Declare a list of root directories for IPSL data at TGCC
root = "/Users/marti/Volumes/CURIE/ccc/store/cont003/dsm"
print 'igcm_out : declaration root sur Spip : ', root
if root:
p = cproject("IGCM_OUT_old",
"root",
"login",
"model",
"status",
"experiment",
"simulation",
"DIR",
"OUT",
"ave_length",
"frequency",
"period",
"clim_period",
"clim_period_length",
ensemble=["model", "simulation", "clim_period"],
separator="%")
cdef('root', root, project='IGCM_OUT_old')
cdef('clim_period', '????_????', project='IGCM_OUT_old')
cdef(
'clim_period_length', '*', project='IGCM_OUT_old'
) # --> Takes the following values: '*', '' (to access only to SE), '_50Y', '_100Y'
cdef('ave_length', '*',
project='IGCM_OUT_old') # --> Takes the following values: MO, DA...
Various grids are available. Original grid writes as : grid='_'. Other grids write e.g. as : grid ='T42' or grid ='T127'
Example of an 'erai' project dataset declaration ::
>>> cdef('project','erai')
>>> d=ds(variable='tas',period='198001',grid='_', frequency='monthly')
>>> d2=ds(variable='tas',period='198001',grid='T42',frequency='daily')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias
from climaf.site_settings import atCNRM
if atCNRM:
cproject('erai',('grid','_'), ('frequency','monthly')) # no grid writes as '_' , otherwise as e.g. 'T42' or 'T127'
root="/cnrm/vdr/DATA/OBS/netcdf/${frequency}"
patmonth1=root+"_mean/erai/erai_???_mm_${variable}${grid}YYYY-YYYY.nc" #for original grid
patmonth2=root+"_mean/erai/erai_???_mm_${variable}.${grid}.YYYY-YYYY.nc" #for other grids e.g. : grid ='T42' or 'T127'
patday1=root+"/erai/ei_${variable}${grid}YYYY-YYYY.nc" #for original grid
patday2=root+"/erai/ei_${variable}_${grid}_YYYY-YYYY.nc" #for other grids e.g.: grid ='T42' or 'T127'
dataloc(project='erai', organization='generic', url=[patmonth1,patmonth2,patday1,patday2])
# Defining alias and derived variables for ERAI, together with filenames
##############################################################################
# Valid both for daily and monthly data (to check : energy flux in W for
# daily and Joules for monthly ????)
calias("erai",'sic' ,'ci' ,filenameVar='CI')
"""
This module declares PCMDI reference products on ciclad data organization and specifics, as managed by J. Servonnat at IPSL;
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs, cdef
from climaf.site_settings import onCiclad
cfreqs('ref_pcmdi', {'monthly':'mo' , 'daily':'day' , 'seasonal':'mo', 'annual_cycle':'mo'})
if onCiclad:
cproject('ref_pcmdi', ('frequency','annual_cycle'), 'product', 'clim_period')
cdef('variable', '*', project='ref_pcmdi')
cdef('product', '*', project='ref_pcmdi')
cdef('clim_period', '*', project='ref_pcmdi')
cdef('period', '1950-2015', project='ref_pcmdi')
root="/data/jservon/Evaluation/ReferenceDatasets/PCMDI-MP/obs/*/${frequency}/${variable}/${product}/ac/"
pattern2=root+"${variable}_*mon_${product}_${clim_period}-clim.nc"
dataloc(project='ref_pcmdi', organization='generic', url=pattern2)
"""
This module declares project example and its data location for the standard CliMAF distro
Only one additionnal attribute : frequency (but data sample actually includes only frequency= 'monthly')
Example of an 'example' dataset definition ::
>>> dg=ds(project='example', simulation='AMIPV6ALB2G', variable='tas', period='1980-1981', frequency='monthly')
"""
import os
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs
from climaf import __path__ as cpath
cpath=os.path.abspath(cpath[0])
cproject("example" , ("frequency","monthly") )
cfreqs('example',{'monthly':'mon' })
data_pattern_L=cpath+"/../examples/data/${simulation}/L/${simulation}SFXYYYY.nc"
data_pattern_A=cpath+"/../examples/data/${simulation}/A/${simulation}PLYYYY.nc"
dataloc(project="example",organization="generic",url=[data_pattern_A,data_pattern_L])
Various grids are available. Original grid writes as : grid=''. Other grids write e.g. as : grid ='T127'
Example of an 'cruts3' project dataset declaration ::
>>> cdef('project','cruts3')
>>> d=ds(variable='tas',period='198001',grid='')
>>> d2=ds(variable='tas',period='198001',grid='T127')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cdef
from climaf.site_settings import atCNRM
if atCNRM:
cproject('cruts3','grid') # no grid writes as '', otherwise as e.g. 'T127'
url1_cruts3="/cnrm/amacs/DATA/OBS/netcdf/monthly_mean/cruts3/${variable}_cru_ts_3${grid}.nc" #for original grid
url2_cruts3="/cnrm/amacs/DATA/OBS/netcdf/monthly_mean/cruts3/${variable}_cru_ts_3.${grid}.nc" #for other grids write e.g. as : grid ='T127'
dataloc(project='cruts3', organization='generic', url=[url1_cruts3,url2_cruts3])
# Defining alias and derived variables for CRUTS3, together with filenames
##############################################################################
calias("cruts3",'clt' ,'cld' ,filenameVar='cld')
calias("cruts3",'evspsbl','pet' ,scale=1./86400. ,filenameVar='pet',units="kg m-2 s-1")
#pet:="potential evapotranspiration"; units="mm/day"
#evspsbl:="water_evaporation_flux" en kg.m-2.s-1
ERAI-LAND variables
Example of an 'erai_land' project dataset declaration ::
>>> cdef('project','erai-land')
>>> d=ds(variable='snd',period='198001',grid='_')
>>> d2=ds(variable='snd',period='198001',grid='T127')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias
from climaf.site_settings import atCNRM
if atCNRM:
cproject('erai-land','grid') # no grid writes as '_' , otherwise as e.g. 'T127'
root="/cnrm/vdr/DATA/OBS/netcdf/monthly_mean/erai-land/erai_???_mm_${variable}"
suffix="YYYY-YYYY.nc"
#
url_erai_land1=root+"${grid}"+suffix #for original grid
url_erai_land2=root+".${grid}."+suffix #for other grids write e.g. as : grid='T127'
#
dataloc(project='erai-land', organization='generic', url=[url_erai_land1,url_erai_land2])
# Defining alias and derived variables for ERAI-land, together with filenames
##############################################################################
#asn:="Snow albedo" sans dimension
#calias("erai-land",'' ,'asn' ,filenameVar='ASN')
"""
This module declares reference products on ipsl data organization and specifics, as managed by J. Servonnat at IPSL;
Attributes are : ...
Example of an 'ref_ipsl' project dataset declaration ::
>>> cdef('project','ref_ipsl')
>>> d=ds(variable='tas',period='198001'....)
>>> d2=ds(variable='tas',period='198001', frequency='daily', ....)
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs
from climaf.site_settings import onCiclad
cfreqs('ref_ipsl', {'monthly':'mo' , 'daily':'day' })
if onCiclad:
cproject('ref_ipsl', ('frequency','monthly'), ('product','*'), ('period','1900-2050'))
root="/data/jservon/Evaluation/ReferenceDatasets/*/${frequency}/${variable}/"
pattern1=root+"${variable}_*mon_${product}*_YYYYMM-YYYYMM.nc"
dataloc(project='ref_ipsl', organization='generic', url=[pattern1])
Use attributes 'model' and 'frequency'
Example of a path : /prodigfs/OCMIP5/OUTPUT/IPSL/IPSL-CM4/CTL/mon/CACO3/CACO3_IPSL_IPSL-CM4_CTL_1860-1869.nc
Example ::
>>> cdef('model','IPSL-CM4')
>>> cdef('frequency','monthly')
>>> cactl=ds(project='OCMIP5_Ciclad', experiment='CTL', variable='CACO3', period='1860-1861')
"""
import os
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs
from climaf.site_settings import onCiclad
if onCiclad :
cproject("OCMIP5","model","simulation",("frequency","monthly"),ensemble=["model"])
#
# Declare which facets can be used for defining an ensemble
#
dataloc(project="OCMIP5", organization="generic",
url=['/prodigfs/OCMIP5/OUTPUT/*/${model}/${simulation}/${frequency}/'
'${variable}/${variable}_*_${model}_${simulation}_YYYY-YYYY.nc'])
#
cfreqs('OCMIP5',{'monthly':'mon' })
root = None
if onCiclad:
root = "/data/jservon/Evaluation/ReferenceDatasets/"
if atTGCC:
root = "/ccc/work/cont003/igcmg/igcmg/IGCM/ReferenceDatasets/"
if atIDRIS:
root = "/workgpfs/rech/psl/rpsl035/IGCM/ReferenceDatasets/"
if atCerfacs:
root = "/data8/datamg/Ciclad/ReferenceDatasets/"
if atCNRM:
root = "/cnrm/est/COMMON/climaf/reference_datasets_from_IPSL/"
cproject('ref_climatos', ('frequency', 'annual_cycle'),
'product',
'clim_period',
ensemble=['product'],
separator='%')
cfreqs('ref_climatos', {
'monthly': 'mo',
'daily': 'day',
'seasonal': 'mo',
'annual_cycle': 'mo'
})
cdef('variable', '*', project='ref_climatos')
cdef('product', '*', project='ref_climatos')
cdef('clim_period', '*', project='ref_climatos')
cdef('simulation', 'refproduct', project='ref_climatos')
cdef('period', 'fx', project='ref_climatos')
from climaf.classes import cproject, calias, cfreqs, cdef
from climaf.site_settings import atTGCC, onCiclad
# Rajouter login et disk (ou filesystem) = scratch/store/work
# ajouter realm? atmos -> ATM; ocean -> OCE
root = None
if atTGCC:
# Declare a list of root directories for IPSL data at TGCC
root="/ccc/store/cont003/dsm"
if onCiclad :
# Declare a list of root directories for CMIP5 data on IPSL's Ciclad file system
root="/data/jservon/IPSL_DATA/SIMULATIONS"
p=cproject("IGCM_OUT", "clim_period", "model", "simulation", "variable", "frequency", "experiment", "OUT", "DIR", "login", "period", "root", ensemble=["model","simulation"], separator="%")
cdef('root', root, project='IGCM_OUT')
cdef('clim_period', '????_????', project='IGCM_OUT')
cdef('model', '*', project='IGCM_OUT')
cdef('simulation', '*', project='IGCM_OUT')
cdef('variable', '*', project='IGCM_OUT')
cdef('frequency', 'monthly', project='IGCM_OUT')
cdef('experiment', '*', project='IGCM_OUT')
cdef('OUT', '*', project='IGCM_OUT')
cdef('DIR', '*', project='IGCM_OUT')
cdef('login', '*', project='IGCM_OUT')
cdef('period', '0001-3000', project='IGCM_OUT')
# Frequency alias
"""
This module declares locations for searching data for project OBS4MIP at CNRM (VDR), for
all frequencies; see file:///cnrm/vdr/DATA/Obs4MIPs/doc/
Additional attribute for OBS4MIPS datasets : 'frequency'
Example for an OBS4MIPS CMIP5 dataset declaration ::
>>> pr_obs=ds(project='OBS4MIPS', variable='pr', simulation='GPCP-SG', frequency='monthly', period='1979-1980')
"""
import os.path
if os.path.exists('/cnrm'):
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs
cproject("OBS4MIPS","experiment",("frequency","monthly"))
# Frequency alias
cfreqs('OBS4MIPS', {'monthly':'monthly_mean' })
#
pattern="/cnrm/vdr/DATA/Obs4MIPs/netcdf/${frequency}/${variable}_${simulation}_*_YYYYMM-YYYYMM.nc"
dataloc(project="OBS4MIPS", organization="generic", url=[pattern])
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs, cdef
from climaf.site_settings import atTGCC, onCiclad, onSpip, atCNRM
root = None
if onCiclad:
# Declare a list of root directories for CMIP5 data on IPSL's Ciclad file system
root = "/bdd"
if atCNRM:
# Declare a list of root directories for IPSL data at TGCC
root = "/cnrm/cmip/cnrm/ESG"
if root:
# -- Declare a CMIP5 CliMAF project
# ------------------------------------ >
cproject('CMIP5', 'root', 'model', 'table', 'experiment', 'realization', 'frequency', 'realm',
'version', ensemble=['model', 'realization'], separator='%')
# -- Declare a CMIP5 'extent' CliMAF project = extracts a period covering historical and a scenario
# ------------------------------------ >
cproject('CMIP5_extent', 'root', 'model', 'table', 'experiment', 'extent_experiment', 'realization', 'frequency',
'realm',
'version', ensemble=['model', 'realization'], separator='%')
# -- Define the pattern for CMIP5
if atCNRM:
pattern1 = '${root}/CMIP5/output*/*/${model}/${experiment}/${frequency}/${realm}/${table}/' \
'${realization}/${version}/${variable}/'
else:
pattern1 = '${root}/CMIP5/output/*/${model}/${experiment}/${frequency}/${realm}/${table}/' \
'${realization}/${version}/${variable}/'
# a pattern for fixed fields
This module declares CERES data organization and specifics, as managed by Sophie T. at CNRM; see file:///cnrm/amacs/DATA/OBS/netcdf/
No attributes in addition to standard ones; and 'simulation' is not used
Version of dataset is implicitly the latest, through symbolic links managed by Sophie. Please
complain to climaf at cnrm dot fr if this does not fit the needs
Example of a 'ceres' project dataset declaration ::
>>> d=ds('project='ceres, variable='rlds',period='198001',domain=[40.,60.,-10.,+20.])
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias
from climaf.site_settings import atCNRM
if atCNRM:
cproject('ceres')
url_ceres="/cnrm/amacs/DATA/OBS/netcdf/monthly_mean/ceres/${variable}_CERES-EBAF.nc"
dataloc(project='ceres', organization='generic', url=[url_ceres])
# No need to define alias for CERES, which sticks to CMIP5 standards
##############################################################################
#calias("ceres",'' ,'rluscs' ,filenameVar='rluscs') pas de variable correspondante...?
#rluscs:="Surface Longwave Flux Up, Monthly Means, Clear-Sky conditions"
"""
This module declares project example and its data location for the standard CliMAF distro
Only one additionnal attribute : frequency (but data sample actually includes only frequency= 'monthly')
Example of an 'example' dataset definition ::
>>> dg=ds(project='example', simulation='AMIPV6ALB2G', variable='tas', period='1980-1981', frequency='monthly')
"""
import os
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs
from climaf import __path__ as cpath
cpath = os.path.abspath(cpath[0])
cproject("example", ("frequency", "monthly"), separator='|')
cfreqs('example', {'monthly': 'mon'})
data_pattern_L = cpath + "/../examples/data/${simulation}/L/${simulation}SFXYYYY.nc"
data_pattern_A = cpath + "/../examples/data/${simulation}/A/${simulation}PLYYYY.nc"
data_pattern_histmth = cpath + "/../examples/data/${simulation}_SE_1982_1991_1M_ua_pres.nc"
dataloc(project="example", organization="generic", url=[data_pattern_A, data_pattern_L, data_pattern_histmth])
# calias("example",'tas','tas_${frequency}')
>>> cdef('model','IPSL-CM4')
>>> cdef('frequency','monthly')
>>> cactl=ds(project='OCMIP5_Ciclad', simulation='CTL', variable='CACO3', period='1860-1861')
"""
import os
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs
from climaf.site_settings import onCiclad
if onCiclad:
cproject("OCMIP5",
"model",
"simulation", ("frequency", "monthly"),
ensemble=["model"])
#
# Declare which facets can be used for defining an ensemble
#
dataloc(
project="OCMIP5",
organization="generic",
url=[
'/prodigfs/project/OCMIP5/OUTPUT/*/${model}/${simulation}/${frequency}/'
'${variable}/${variable}_*_${model}_${simulation}_${PERIOD}.nc'
])
#
cfreqs('OCMIP5', {'monthly': 'mon'})
Example of an 'erai' project dataset declaration ::
>>> cdef('project','erai')
>>> d=ds(variable='tas',period='198001',grid='_', frequency='monthly')
>>> d2=ds(variable='tas',period='198001',grid='T42',frequency='daily')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias
from climaf.site_settings import atCNRM
if atCNRM:
cproject('erai', ('grid', '_'),
('frequency', 'monthly'
)) # no grid writes as '_' , otherwise as e.g. 'T42' or 'T127'
root = "/cnrm/amacs/DATA/OBS/netcdf/${frequency}"
patmonth1 = root + "_mean/erai/erai_???_mm_${variable}${grid}${PERIOD}.nc" #for original grid
patmonth2 = root + "_mean/erai/erai_???_mm_${variable}.${grid}.${PERIOD}.nc" #for other grids e.g. : grid ='T42' or 'T127'
patday1 = root + "/erai/ei_${variable}${grid}${PERIOD}.nc" #for original grid
patday2 = root + "/erai/ei_${variable}_${grid}_${PERIOD}.nc" #for other grids e.g.: grid ='T42' or 'T127'
dataloc(project='erai',
organization='generic',
url=[patmonth1, patmonth2, patday1, patday2])
# Defining alias and derived variables for ERAI, together with filenames
##############################################################################
# Valid both for daily and monthly data (to check : energy flux in W for
# daily and Joules for monthly ????)
ERAI-LAND variables
Example of an 'erai_land' project dataset declaration ::
>>> cdef('project','erai-land')
>>> d=ds(variable='snd',period='198001',grid='_')
>>> d2=ds(variable='snd',period='198001',grid='T127')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias
from climaf.site_settings import atCNRM
if atCNRM:
cproject('erai-land',
'grid') # no grid writes as '_' , otherwise as e.g. 'T127'
root = "/cnrm/amacs/DATA/OBS/netcdf/monthly_mean/erai-land/erai_???_mm_${variable}"
suffix = "${PERIOD}.nc"
#
url_erai_land1 = root + "${grid}" + suffix # for original grid
url_erai_land2 = root + ".${grid}." + suffix # for other grids write e.g. as : grid='T127'
#
dataloc(project='erai-land',
organization='generic',
url=[url_erai_land1, url_erai_land2])
# Defining alias and derived variables for ERAI-land, together with filenames
##############################################################################
# asn:="Snow albedo" sans dimension
"""
This module declares CERES data organization and specifics, as managed by Sophie T. at CNRM; see file:///cnrm/vdr/DATA/OBS/netcdf/
No attributes in addition to standard ones; and 'simulation' is not used
Version of dataset is implicitly the latest, through symbolic links managed by Sophie. Please
complain to climaf at cnrm dot fr if this does not fit the needs
Example of a 'ceres' project dataset declaration ::
>>> d=ds('project='ceres, variable='rlds',period='198001',domain=[40.,60.,-10.,+20.])
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias
from climaf.site_settings import atCNRM
if atCNRM:
cproject("ceres")
url_ceres = "/cnrm/vdr/DATA/OBS/netcdf/monthly_mean/ceres/${variable}_CERES-EBAF.nc"
dataloc(project="ceres", organization="generic", url=[url_ceres])
# No need to define alias for CERES, which sticks to CMIP5 standards
##############################################################################
# calias("ceres",'' ,'rluscs' ,filenameVar='rluscs') pas de variable correspondante...?
# rluscs:="Surface Longwave Flux Up, Monthly Means, Clear-Sky conditions"
Example of an 'gpcp' project dataset declaration ::
>>> cdef('project','gpcp')
>>> d=ds(variable='pr',period='198001',grid='2.5d', frequency='monthly')
>>> d2=ds(variable='pr',period='198001',grid='1d',frequency='daily')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias
from climaf.site_settings import atCNRM
if atCNRM:
cproject('gpcp','grid', ('frequency','monthly'), separator="_") # grid writes as '1d', '2.5d', 'T42' or 'T127'
root="/cnrm/vdr/DATA/OBS/netcdf/${frequency}"
patmonth=root+"_mean/gpcp/${variable}_gpcp.${grid}.nc"
patday =root+"/gpcp/${variable}_gpcp.${grid}.nc"
dataloc(project='gpcp', organization='generic', url=[patmonth,patday])
# Defining alias and derived variables for GPCP, together with filenames
##############################################################################
# Original data is mm/day
calias("gpcp",'precip' ,'precip', filenameVar='pr')
# Compute in SI
calias("gpcp",'pr' ,'precip', scale=1./86400.,units="kg m-2 s-1", filenameVar='pr')
"""
# S.Senesi - april 2016
from climaf.site_settings import atCNRM
if atCNRM :
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs,cdef
# In project 'em', there is a ROOT location, data is organized in
# REALMS, and we handle data FREQUENCY
# 'root' stands for em root directory for alla experiment data (EM_NETCDF_DIR)
cproject("em","root","group","realm", "frequency",separator="|")
# Describe data organization : file hierarchy and filename patterns
######################################################################
# User simulations
pathg="${root}/${group}/${simulation}/"
pathA=pathg+"${realm}/${simulation}PLYYYY.nc" #GSAG
pathL=pathg+"${realm}/${simulation}SFXYYYY.nc"
pathI=pathg+"${realm}/${variable}_O1_YYYY-YYYY.nc" #HISTNATr8
pathO=pathg+"${realm}/${simulation}_1${frequency}_${PERIOD}_grid_${variable}.nc" # var:T_table2.2
pathO2=pathg+"${realm}/${simulation}_1${frequency}_${PERIOD}_scalar_table2.2.nc" # PICTLWS2, PRE6CPLCr2alb
dataloc(project="em", organization="generic", url=[pathA,pathL,pathI,pathO,pathO2])
# Shared simulations - example : group=SC
Example of an 'gpcc' project dataset declaration ::
>>> cdef('project','gpcc')
>>> d=ds(variable='pr',period='198001',grid='05d')
>>> d2=ds(variable='pr',period='198001',grid='1d')
>>> d3=ds(variable='pr',period='198001',grid='T127')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias
from climaf.site_settings import atCNRM
if atCNRM:
cproject('gpcc','grid') # grid writes as '05d', '1d' or 'T127'
url_gpcc="/cnrm/vdr/DATA/OBS/netcdf/monthly_mean/gpcc/GPCC.Reanalysis.${grid}.nc"
dataloc(project='gpcc', organization='generic', url=[url_gpcc])
# Defining alias and derived variables for GPCC, together with filenames
##############################################################################
calias("gpcc",'pr' ,'GPCC' ,scale=1./86400. ,missing=1.e+20, units="kg m-2 s-1")
#calias("gpcc",'GPCC' ,'GPCC' , ,units="mm/day")
#NSTA:="Number of stations available for a specific analysis grid in a specific month"
#calias("gpcc",'site' ,'NSTA' )
root = None
if onCiclad:
root = "/data/jservon/Evaluation/ReferenceDatasets/"
if atTGCC:
root = "/ccc/work/cont003/igcmg/igcmg/IGCM/ReferenceDatasets/"
if atIDRIS:
root = "/workgpfs/rech/psl/rpsl035/IGCM/ReferenceDatasets/"
if atCerfacs:
root = "/data/scratch/globc/dcom/globc_obs/OBS4MIPS_ANA4MIPS_CMOR/Tier1/ERA5/"
if atCNRM:
root = "/cnrm/est/COMMON/climaf/reference_datasets_from_IPSL/"
cproject('ref_era5cerfacs', ('frequency', 'monthly'),
'product', ('period', '1979-2018'),
'obs_type',
'table',
separator='%')
cfreqs('ref_era5cerfacs', {'monthly': 'mon', 'daily': 'day'})
cdef('variable', '*', project='ref_era5cerfacs')
cdef('product', 'ERA5', project='ref_era5cerfacs')
cdef('period', '1979-2018', project='ref_era5cerfacs')
cdef('obs_type', 'reanalysis', project='ref_era5cerfacs')
cdef('table', '*', project='ref_era5cerfacs')
# Obs de MOC RAPID (Il a fallu bricoler les donnees d'origine pour la dimension time au debut et unlim)
# dataloc(project="ref_climatos",organization="generic",
# url='/home/esanchez/data_climaf/${variable}_vertical_unlim.nc')
calias(project='ref_era5cerfacs',
variable='moc',
Example of an 'gpcc' project dataset declaration ::
>>> cdef('project','gpcc')
>>> d=ds(variable='pr',period='198001',grid='05d')
>>> d2=ds(variable='pr',period='198001',grid='1d')
>>> d3=ds(variable='pr',period='198001',grid='T127')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias
from climaf.site_settings import atCNRM
if atCNRM:
cproject('gpcc', 'grid') # grid writes as '05d', '1d' or 'T127'
url_gpcc = "/cnrm/amacs/DATA/OBS/netcdf/monthly_mean/gpcc/GPCC.Reanalysis.${grid}.nc"
dataloc(project='gpcc', organization='generic', url=[url_gpcc])
# Defining alias and derived variables for GPCC, together with filenames
##############################################################################
calias("gpcc",
'pr',
'GPCC',
scale=1. / 86400.,
missing=1.e+20,
units="kg m-2 s-1")
#calias("gpcc",'GPCC' ,'GPCC' , ,units="mm/day")
"""
This module declares a dummy project named 'file' which allows to describe dataset very simply from datafile using fucntion :py:func:`~climaf.classes.fds()`
Example for a dataset declaration ::
>>> my_ds=fds('toto.nc',simulation='my_simu')
"""
from climaf.classes import cproject
cproject("file" ,"model","path",separator="|")
root = None
if onCiclad:
# Declare a root directory for CMIP3 data on IPSL's Ciclad file system
root = "/bdd/CMIP3"
if atCNRM:
pass
if root:
## -- Declare a CMIP3 CliMAF project
## ------------------------------------ >
cproject('CMIP3',
'root',
'model',
'realm',
'experiment',
'realization',
'frequency',
ensemble=['model', 'realization'],
separator='%')
## -- Define the pattern for CMIP3 data
pattern1 = '${root}/${experiment}/${realm}/${frequency}/${variable}/${model}/${realization}/'
pattern1 += '${variable}_*.nc'
## -- call the dataloc CliMAF function
dataloc(project='CMIP3', organization='generic', url=pattern1)
# -- Make the alias and default values for both projects
for project in ['CMIP3']:
#calias(project, 'tos', offset=273.15)
"""
This module declares how to access observation datasets organized 'a la CAMI' at CNRM,
at /cnrm/est/COMMON/cami/V1.8/climlinks/
Example ::
>>> pr_gpcp=ds(project='CAMIOBS', simulation='GPCP2.5d', variable='pr', period='1979-1980')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cdef
cproject("CAMIOBS", "product", separator="_")
# Root directory for obs data organized 'a la CAMI' on CNRM's Lustre file system.
CAMIOBS_root = "/cnrm/est/COMMON/cami/V1.8/climlinks/"
# Pattern for matching CAMI obs data files and their directory.
# We choose to use facet 'model' to carry the observation source
CAMIOBS_pattern = "${product}/${variable}_1m_${PERIOD}_${simulation}.nc"
# Declare the CAMIOBS pattern to be associated with a project we name OBS_CAMI
dataloc(project="CAMIOBS",
organization="generic",
url=[CAMIOBS_root + CAMIOBS_pattern])
cdef("simulation", "*", project="CAMIOBS")
# From here, you can define your dataset using these files.
"""
This module declares locations for searching data for project OBS4MIP at CNRM (VDR), for
all frequencies; see file:///cnrm/amacs/DATA/Obs4MIPs/doc/
Additional attribute for OBS4MIPS datasets : 'frequency'
Example for an OBS4MIPS CMIP5 dataset declaration ::
>>> pr_obs=ds(project='OBS4MIPS', variable='pr', simulation='GPCP-SG', frequency='monthly', period='1979-1980')
"""
import os.path
if os.path.exists('/cnrm'):
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs
cproject("OBS4MIPS", "experiment", ("frequency", "monthly"))
# Frequency alias
cfreqs('OBS4MIPS', {'monthly': 'monthly_mean'})
#
pattern = "/cnrm/amacs/DATA/Obs4MIPs/netcdf/${frequency}/${variable}_${simulation}_*_YYYYMM-YYYYMM.nc"
dataloc(project="OBS4MIPS", organization="generic", url=[pattern])
root = None
if atTGCC:
# Declare a list of root directories for IPSL data at TGCC
root="/ccc/work/cont003/cmip6/cmip6"
if onCiclad :
# Declare a list of root directories for CMIP5 data on IPSL's Ciclad file system
root="/bdd"
#root="/ccc/work/cont003/cmip6/cmip6"
if atCNRM:
# Declare a list of root directories for IPSL data at TGCC
root="/cnrm/cmip"
if root:
## -- Declare a 'CMIP6 CliMAF project
## ------------------------------------ >
cproject('CMIP6', 'root', 'model', 'institute', 'mip', 'table', 'experiment', 'realization',
'grid', 'version', ensemble=['model','realization'], separator='%')
## -- Declare a CMIP6 'extent' CliMAF project = extracts a period covering historical and a scenario
## ------------------------------------ >
cproject('CMIP6_extent', 'root', 'model', 'institute', 'mip', 'table', 'experiment', 'extent_experiment', 'realization',
'grid', 'version', ensemble=['model','realization'], separator='%')
for project in ['CMIP6', 'CMIP6_extent']:
## --> systematic arguments = simulation, frequency, variable
## -- Set the aliases for the frequency
## -- Set default values
cdef('root' , root , project=project)
cdef('institute' , '*' , project=project)
cdef('model' , '*' , project=project)
cdef('mip' , '*' , project=project)
"""
This module declares how to access observation datasets organized 'a la CAMI' at CNRM,
at /cnrm/aster/data1/UTILS/cami/V1.7/climlinks/
Example ::
>>> pr_gpcp=ds(project='CAMIOBS', simulation='GPCP2.5d', variable='pr', period='1979-1980')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias
cproject("CAMIOBS" , separator="_")
# Root directory for obs data organized 'a la CAMI' on CNRM's Lustre file system.
CAMIOBS_root="/cnrm/aster/data1/UTILS/cami/V1.7/climlinks/"
# Pattern for matching CAMI obs data files and their directory.
# We choose to use facet 'model' to carry the observation source
CAMIOBS_pattern="${simulation}/${variable}_1m_YYYYMM_YYYYMM_${simulation}.nc"
# Declare the CAMIOBS pattern to be associated with a project we name OBS_CAMI
dataloc(project="CAMIOBS", organization="generic",
url=[CAMIOBS_root+CAMIOBS_pattern])
# From here, you can define your dataset using these files.
# You need only to define the facets useful w.r.t. the patterns
# i.e. here : model and variable
#pr_gpcp=ds(project="CAMIOBS", simulation="GPCP2.5d", variable="pr", period="1979-1980")
Syntax for these attributes is described in `the CMIP6 DRS document <https://goo.gl/v1drZl>`_
Example for a CMIP6 dataset declaration ::
>>> tas1pc=ds(project='CMIP6', model='CNRM-CM6-1', experiment='1pctCO2', variable='tas', table='Amon', realization='r3i1p1f2', period='1860-1861')
"""
from climaf.dataloc import dataloc
from climaf.classes import cproject, calias, cfreqs, cdef
# Declare CMIP6CERFACS project
cproject('CMIP6CERFACS', 'model', 'simulation', 'institute', 'mip','table', 'realization', 'grid', separator='%')
# --> systematic arguments = simulation, frequency, variable
# -- Set the aliases for the frequency
# cfreqs('PRIMAVERA-cerfacs', {'monthly':'mon', 'yearly':'yr', 'daily':'day'})
# crealms('IGCM_CMIP6', {'atmos':'ATM', 'ocean':'OCE', 'land':'SRF', 'seaice':'ICE' })
# -- Set default values
cdef('institute', 'CNRM-CERFACS', project='CMIP6CERFACS')
cdef('mip', '*', project='CMIP6CERFACS')
cdef('model', '*', project='CMIP6CERFACS')
cdef('simulation', '*', project='CMIP6CERFACS')
cdef('domain', 'global', project='CMIP6CERFACS')
cdef('period', '*', project='CMIP6CERFACS')
cdef('variable', 'tas', project='CMIP6CERFACS')
cdef('table', 'Amon', project='CMIP6CERFACS')
cdef('realization', 'r*i1p1f*', project='CMIP6CERFACS')
cdef('grid', 'g*', project='CMIP6CERFACS')
