def __init__(self, name, path, Session, validate=True, offline=False):
self.data_specs_version = "0"
BasicHandler.__init__(self,
name,
path,
Session,
validate=validate,
offline=offline)
def __init__(self, name, path, Session, validate=True, offline=False):
self.caseSensitiveValidValues = {
} # : field => (dict : lowerCaseValue => validValue)
self.checkFilenames = True # True <=> check variable shortname against file basename
BasicHandler.__init__(self,
name,
path,
Session,
validate=validate,
offline=offline)
def __init__(self,
name,
path,
Session,
validate=True,
offline=False,
replica=False):
self.replica = replica
BasicHandler.__init__(self,
name,
path,
Session,
validate=validate,
offline=offline)
def initializeFields(self, Session):
BasicHandler.initializeFields(self, Session)
config = getConfig()
projectSection = 'project:'+self.name
# Enumerated value validation is case-insensitive
lowerCaseValidValues = {}
for field, valueList in self.validValues.items():
lowerCaseValidList = []
validDict = {}
for value in valueList:
if value is not None:
lvalue = value.lower()
else:
lvalue = None
lowerCaseValidList.append(lvalue)
validDict[lvalue] = value
lowerCaseValidValues[field] = lowerCaseValidList
self.caseSensitiveValidValues[field] = validDict
self.validValues = lowerCaseValidValues
self.checkFilenames = config.getboolean(projectSection, 'thredds_check_file_names', default=True)
def readContext(self, cdfile, model=''):
"Get a dictionary of keys from an open file"
result = BasicHandler.readContext(self, cdfile)
f = cdfile.file
for key, value in cmorAttributes.items():
try:
result[key] = getattr(f, value)
if key in cmorArrayAttributes and type(
result[key]) is numpy.ndarray:
res = str(result[key][0])
if key == 'run_name':
if res[0:3] != 'run':
res = 'run' + res
result[key] = res
except:
pass
if 'realization' in result and 'initialization_method' in result and 'physics_version' in result:
ensemble = 'r%si%sp%s' % (result['realization'],
result['initialization_method'],
result['physics_version'])
result['ensemble'] = ensemble
result['run_name'] = ensemble
base = os.path.basename(cdfile.path)
try:
index = base.index('_')
varname = base[0:index]
result['variable'] = varname
except:
warning("File path must have the form varname_XXX: %s" %
cdfile.path)
#!WARNING: I think all TAMIP2 data goes into output1
result['product'] = 'output1'
self.mapEnumeratedValues(result)
# If realm has multiple fields, pick the first one
if 'realm' in result:
realm = result['realm'].strip()
if realm.find(' ') != -1:
realms = realm.split(' ')
result['realm'] = realms[0]
# Parse CMOR table.
if 'table_id' in result:
tableId = result['table_id']
fields = tableId.split()
# Assume table ID has the form 'Table table_id ...'
if len(fields) > 1 and (fields[1] in cmorTables):
table = fields[1]
result['cmor_table'] = table
else:
result['cmor_table'] = 'noTable'
else:
result['cmor_table'] = 'noTable'
# Cache a 'drs_id' attribute for DRS-style dataset lookups
validateDRSFieldValues(result, cdfile)
if 'product' in result and 'institute' in result and 'model' in result and 'experiment' in result and 'time_frequency' in result and 'realm' in result and 'cmor_table' in result and 'ensemble' in result:
drsid = '%s.%s.%s.%s.%s.%s.%s.%s.%s' % (
DRS_ACTIVITY, result['product'], result['institute'],
result['model'], result['experiment'],
result['time_frequency'], result['realm'],
result['cmor_table'], result['ensemble'])
result['drs_id'] = drsid
return result
def openPath(self, path):
"""Open a sample path, returning a project-specific file object,
(e.g., a netCDF file object or vanilla file object)."""
fileobj = BasicHandler.openPath(self, path)
fileobj.path = path
return fileobj
def readContext(self, cdfile, model=''):
"Get a dictionary of keys from an open file"
result = BasicHandler.readContext(self, cdfile)
f = cdfile.file
for key, value in cmorAttributes.items():
try:
result[key] = getattr(f, value)
if key in cmorArrayAttributes and type(result[key]) is numpy.ndarray:
res = str(result[key][0])
if key=='run_name':
if res[0:3]!='run':
res = 'run'+res
result[key] = res
except:
pass
if 'realization' in result and 'initialization_method' in result and 'physics_version' in result:
ensemble = 'r%si%sp%s'%(result['realization'], result['initialization_method'], result['physics_version'])
result['ensemble'] = ensemble
result['run_name'] = ensemble
base = os.path.basename(cdfile.path)
try:
index = base.index('_')
varname = base[0:index]
result['variable'] = varname
except:
warning("File path must have the form varname_XXX: %s"%cdfile.path)
if not result.has_key('product'):
result['product'] = 'output'
self.mapEnumeratedValues(result)
# If realm has multiple fields, pick the first one
if 'realm' in result:
realm = result['realm'].strip()
if realm.find(' ')!=-1:
realms = realm.split(' ')
result['realm'] = realms[0]
# Parse CMOR table.
if 'table_id' in result:
tableId = result['table_id']
fields = tableId.split()
# Assume table ID has the form 'Table table_id ...'
if len(fields)>1 and (fields[1] in cmorTables):
table = fields[1]
result['cmor_table'] = table
else:
result['cmor_table'] = 'noTable'
else:
result['cmor_table'] = 'noTable'
# Parse the product if it is unresolved
if result['product']=='output':
cmor_table = result['cmor_table']
variable = result.get('variable', None)
experiment = result.get('experiment', None)
dateRange = self.getDateRangeFromPath()
year1 = dateRange[0][0]
year2 = dateRange[1][0]
if year2 is None:
year2 = year1
result['product'] = getProduct(cmor_table, variable, experiment, year1, year2)
validateDRSFieldValues(result, cdfile)
return result
def openPath(self, path):
"""Open a sample path, returning a project-specific file object,
(e.g., a netCDF file object or vanilla file object)."""
fileobj = BasicHandler.openPath(self, path)
fileobj.path = path
return fileobj
def __init__(self, name, path, Session, validate=True, offline=False):
self.caseSensitiveValidValues = {} # : field => (dict : lowerCaseValue => validValue)
self.checkFilenames = True # True <=> check variable shortname against file basename
BasicHandler.__init__(self, name, path, Session, validate=validate, offline=offline)
def readContext(self, cdfile, model=''):
"Get a dictionary of keys from an open file"
result = BasicHandler.readContext(self, cdfile)
f = cdfile.file
for key, value in cmorAttributes.items():
try:
result[key] = getattr(f, value)
if key in cmorArrayAttributes and type(result[key]) is numpy.ndarray:
res = str(result[key][0])
if key=='run_name':
if res[0:3]!='run':
res = 'run'+res
result[key] = res
except:
pass
if 'realization' in result and 'initialization_method' in result and 'physics_version' in result:
ensemble = 'r%si%sp%s'%(result['realization'], result['initialization_method'], result['physics_version'])
result['ensemble'] = ensemble
result['run_name'] = ensemble
base = os.path.basename(cdfile.path)
try:
index = base.index('_')
varname = base[0:index]
result['variable'] = varname
except:
warning("File path must have the form varname_XXX: %s"%cdfile.path)
#!WARNING: I think all TAMIP2 data goes into output1
result['product'] = 'output1'
self.mapEnumeratedValues(result)
# If realm has multiple fields, pick the first one
if 'realm' in result:
realm = result['realm'].strip()
if realm.find(' ')!=-1:
realms = realm.split(' ')
result['realm'] = realms[0]
# Parse CMOR table.
if 'table_id' in result:
tableId = result['table_id']
fields = tableId.split()
# Assume table ID has the form 'Table table_id ...'
if len(fields)>1 and (fields[1] in cmorTables):
table = fields[1]
result['cmor_table'] = table
else:
result['cmor_table'] = 'noTable'
else:
result['cmor_table'] = 'noTable'
# Cache a 'drs_id' attribute for DRS-style dataset lookups
validateDRSFieldValues(result, cdfile)
if 'product' in result and 'institute' in result and 'model' in result and 'experiment' in result and 'time_frequency' in result and 'realm' in result and 'cmor_table' in result and 'ensemble' in result:
drsid = '%s.%s.%s.%s.%s.%s.%s.%s.%s'%(DRS_ACTIVITY, result['product'], result['institute'], result['model'], result['experiment'], result['time_frequency'], result['realm'], result['cmor_table'], result['ensemble'])
result['drs_id'] = drsid
return result
def readContext(self, cdfile, model=''):
"Get a dictionary of keys from an open file. The model cannot be determined in general"
result = BasicHandler.readContext(self, cdfile)
f = cdfile.file
fnm = os.path.basename(self.path)
try:
exp = experiment(f.experiment_id.strip())
except AttributeError:
exp = None
try:
tableid = f.table_id
if len(tableid)>=8:
tbl=f.table_id[6:8]
else:
tbl=tableid[0:2] # CSIRO
if tbl!='A5':
var = os.path.split(fnm)[-1].split('_')[0]
else:
varflds = os.path.split(fnm)[-1].split('_')
if len(varflds) in [3,4,9]: # CCCMA has 9 subfields, 4 for NCAR
var = varflds[0]+'_'+varflds[1]
elif len(varflds)==2:
var = varflds[0]
else:
raise ESGPublishError('Cannot determine variable name for file %s'%fnm)
fqcy = longfreq[frequency(tbl, var)]
loc = submodel(tbl, var)
except AttributeError:
var = None
fqcy = None
loc = None
try:
rlz='run'+str(f.realization[0])
except AttributeError:
rlz = None
if exp is not None:
result['experiment'] = exp
if loc is not None:
result['submodel'] = loc
if fqcy is not None:
result['time_frequency'] = fqcy
if var is not None:
result['variable'] = var
if model is not None:
result['model'] = model
if rlz is not None:
result['run_name'] = rlz
# Try to determine model from source string
if model=='' and 'source' in result:
source = result['source']
if source[0:7]=='BCC-CM1':
model = 'bcc_cm1'
elif source[0:6] == 'BCM2.0':
model = 'bccr_bcm2_0'
elif source[0:7] == 'CCSM3.0':
model = 'ncar_ccsm3_0'
elif source[0:8] == 'CNRM-CM3':
model = 'cnrm_cm3'
elif source[0:6] == 'ECHAM5':
model = 'mpi_echam5'
elif source[0:6] == 'ECHO-G':
model = 'miub_echo_g'
elif source[0:4] in ['FGCM', 'FGOA']:
model = 'iap_fgoals1_0_g'
elif source[0:10] in ['GFDL_CM2.1', 'GFDL_AM2.1']:
model = 'gfdl_cm2_1'
elif source[0:10] in ['GFDL_CM2.0', 'GFDL_SM2.0']:
model = 'gfdl_cm2_0'
elif source[0:8] == 'GISS AOM':
model = 'giss_aom'
elif source[0:6] == 'HadCM3':
model = 'ukmo_hadcm3'
elif source[0:7] == 'HadGEM1':
model = 'ukmo_hadgem1'
elif source[0:8] == 'INGV-SXG':
model = 'ingv_echam4'
elif source[0:8] == 'INMCM3.0':
model = 'inmcm3_0'
elif source[0:8] == 'IPSL-CM4':
model = 'ipsl_cm4'
elif source[0:9] == 'MRI-CGCM2':
model = 'mri_cgcm2_3_2a'
elif source[0:8] == 'Parallel':
model = 'ncar_pcm1'
elif source[0:50] == 'CGCM3.1 (2004): atmosphere: AGCM3 (GCM13d, T47L31':
model = 'cccma_cgcm3_1'
elif source[0:50] == 'CGCM3.1 (2004): atmosphere: AGCM3 (GCM13d, T63L31':
model = 'cccma_cgcm3_1_t63'
elif source[0:11] == 'CSIRO Mk3.0':
model = 'csiro_mk3_0'
elif source[0:11] == 'CSIRO Mk3.5':
model = 'csiro_mk3_5'
elif source[0:49] == 'MIROC3.2 (2004): atmosphere: AGCM (AGCM5.7b, T106':
model = 'miroc3_2_hires'
elif source[0:48] == 'MIROC3.2 (2004): atmosphere: AGCM (AGCM5.7b, T42':
model = 'miroc3_2_medres'
elif source[0:3] == 'E3x':
model = 'giss_model_e_h'
elif source[0:19] == 'GISS ModelE/Russell':
model = 'giss_model_e_r'
elif source[0:17] == 'GISS ModelE/HYCOM':
model = 'giss_model_e_h'
elif source[0:3] == 'E3A':
model = 'giss_model_e_r'
elif source[0:3] == 'E3O':
model = 'giss_model_e_h'
result['model'] = model
return result
def validateContext(self, context):
BasicHandler.validateContext(self, context)
run = context.get('run_name', '')
if len(run)<3 or run[0:3]!='run' or (' ' in run):
raise ESGPublishError("Invalid value of run: %s, must have the form 'runN'"%run)
def __init__(self, name, path, Session, validate=True, offline=False, replica=False):
self.replica = replica
BasicHandler.__init__(self, name, path, Session, validate=validate, offline=offline)
def readContext(self, cdfile, model=""):
"Get a dictionary of keys from an open file"
result = BasicHandler.readContext(self, cdfile)
f = cdfile.file
for key, value in cmorAttributes.items():
try:
result[key] = getattr(f, value)
if key in cmorArrayAttributes and type(result[key]) is numpy.ndarray:
res = str(result[key][0])
if key == "run_name":
if res[0:3] != "run":
res = "run" + res
result[key] = res
except:
pass
if "realization" in result and "initialization_method" in result and "physics_version" in result:
ensemble = "r%si%sp%s" % (result["realization"], result["initialization_method"], result["physics_version"])
result["ensemble"] = ensemble
result["run_name"] = ensemble
base = os.path.basename(cdfile.path)
try:
index = base.index("_")
varname = base[0:index]
result["variable"] = varname
except:
warning("File path must have the form varname_XXX: %s" % cdfile.path)
#!WARNING: All IPSL-LUCID data goes into output
result["product"] = "output"
self.mapEnumeratedValues(result)
# If realm has multiple fields, pick the first one
if "realm" in result:
realm = result["realm"].strip()
if realm.find(" ") != -1:
realms = realm.split(" ")
result["realm"] = realms[0]
# Parse CMOR table.
if "table_id" in result:
tableId = result["table_id"]
fields = tableId.split()
# Assume table ID has the form 'Table table_id ...'
if len(fields) > 1 and (fields[1] in cmorTables):
table = fields[1]
result["cmor_table"] = table
else:
result["cmor_table"] = "noTable"
else:
result["cmor_table"] = "noTable"
# Cache a 'drs_id' attribute for DRS-style dataset lookups
validateDRSFieldValues(result, cdfile)
if (
"product" in result
and "institute" in result
and "model" in result
and "experiment" in result
and "time_frequency" in result
and "realm" in result
and "cmor_table" in result
and "ensemble" in result
):
drsid = "%s.%s.%s.%s.%s.%s.%s.%s.%s" % (
DRS_ACTIVITY,
result["product"],
result["institute"],
result["model"],
result["experiment"],
result["time_frequency"],
result["realm"],
result["cmor_table"],
result["ensemble"],
)
result["drs_id"] = drsid
return result
