def __init__(self):
self.dq = dreq.loadDreq()
super(TestRq, self).__init__()
self.CMORvar_by_id = dict()
for i in self.dq.coll["CMORvar"].items:
self.CMORvar_by_id["%s.%s" % (i.mipTable, i.label)] = i
def search_cmorlist(mip_filt, tab_filt):
"""
Get a cmorlist from a MIP name
"""
from dreqPy import dreq
from dreqPy.extensions import collect
dq = dreq.loadDreq()
collect.add( dq )
mip = dq.coll['mip'].items[0]._labelDict[mip_filt]
mvars = mip._get__CMORvar()
expts = mip._get__expt()
MIP_Expts = set([ expt for expt in dq.coll['experiment'].items
if expt.uid in expts ])
MIP_Expts = list(MIP_Expts)
# Find the 'CMORvar'-items, associated with these requestVars:
cmorlist = {}
mvarlist = {}
for cmvar in dq.coll['CMORvar'].items:
if cmvar.label == 'lwsnl':
print (cmvar.mipTable, tab_filt)
if cmvar.uid in mvars and tab_filt in cmvar.mipTable:
cmorlist[cmvar.label] = cmvar
mvarlist[cmvar.label] = dq.inx.uid[cmvar.vid]
return cmorlist, mvarlist
def map_exp_to_request_mip(exp_name, dq=None):
"""
Takes an experiment name and returns the mips requesting data from it
Args:
exp_name: name of the experiment to look up
Return:
mip (list): name of the requesting mips
"""
if dq is None:
dq = dreq.loadDreq()
mips = []
if dq.inx.experiment.label[exp_name]:
e_id = dq.inx.experiment.label[exp_name][0]
mips.append(dq.inx.uid[e_id].mip)
e_vars = dq.inx.iref_by_sect[e_id].a
for ri in e_vars['requestItem']:
dr = dq.inx.uid[ri]
if dr.mip not in mips:
mips.append(dr.mip)
return mips
def dqload(dqtag=None, dqroot=None, dqpath=None):
"""Load the dreq from a dqtag and dqroot and dpath, all defaulted.
Arguments:
- tag -- the tag, dynamically defaulted from default_dqtag()
- dqroot -- the dreq root directory, dynamically defaulted from
default_dqroot()
- dqpath -- the path to the XML directory, dynamically-defaulted from
default_dqpath.
This does no error checks itself : it will raise whatever
exception the underlying dreq code does if things are bad. If you
want to check for this use the valid_* functions.
"""
# This replicates some code in dqi.util and dqi.low, to avoid a
# dependency on dqi as this is the only place djq relied on it.
note_reply_metadata(dreqpy_path=dreqPy_path)
top = effective_dqpath(dqtag=dqtag, dqroot=dqroot, dqpath=dqpath)
xml = join(top, split(defaultDreqPath)[1])
config = join(top, split(defaultConfigPath)[1])
note_reply_metadata(dreq_top=top, dreq_xml=xml, dreq_config=config)
dreq = loadDreq(dreqXML=xml, configdoc=config, manifest=None)
note_reply_metadata(dreq_loaded_version=dreq.version)
return dreq
def __init__(self):
from dreqPy import dreq
self.dq = dreq.loadDreq()
self.CMORvar_by_id = dict()
for i in self.dq.coll["CMORvar"].items:
self.CMORvar_by_id["%s.%s" % (i.mipTable,i.label) ] = i
def main(options):
""" main
Arguments:
options (list) - input options from command line
"""
debug = options.debug
dq = dreq.loadDreq()
version = dq.version
exps = get_exp_list(dq=dq)
# create a db connection and cursor
db = MySQLdb.connect(host="localhost", user="******", passwd="c$3gdb", db="csegdb")
cursor = db.cursor()
# loop through the exps dictionary and load them into the database keying off the name
for key, value in exps.iteritems():
# check that the key (experiment name) isn't included in the _exclude_exps list
if key not in _exclude_exps:
count = 0
sql = "select count(id), id, name, description, uid, design_mip, dreq_version from t2_cmip6_exps where name = '"+key+"'"
dreq_description = db.escape_string(value[0])
dreq_uid = db.escape_string(value[1])
dreq_design_mip = db.escape_string(value[2])
try:
print ("Executing sql = {0}".format(sql))
cursor.execute(sql)
(count, id, name, description, uid, design_mip, dreq_version) = cursor.fetchone()
except:
print ("Error executing sql = {0}".format(sql))
db.rollback()
if count == 1:
sql = "update t2_cmip6_exps set description = '{0}', uid = '{1}', design_mip = '{2}', dreq_version = '{3}' where id = {4}".format(dreq_description, dreq_uid, dreq_design_mip, version, str(id))
try:
print ("Executing sql = {0}".format(sql))
cursor.execute(sql)
db.commit()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
elif count == 0:
sql = "insert into t2_cmip6_exps (name, description, uid, design_mip, dreq_version) value ('{0}','{1}','{2}','{3}','{4}')".format(key, dreq_description, dreq_uid, dreq_design_mip, version)
try:
print ("Executing sql = {0}".format(sql))
cursor.execute(sql)
db.commit()
except:
print ("Error executing sql = {0}".format(sql))
db.rollback()
else:
print("Error in database {0} rows found matching experiment name '{1}'".format(count, name))
# disconnect from server
db.close()
def main(options):
""" main
Arguments:
options (list) - input options from command line
"""
debug = options.debug
# Load in the request
dq = dreq.loadDreq()
# load up the experiments name into a list
exps = get_exp_list(dq=dq)
for exp, value in exps.iteritems():
# Dictionary to hold the variables
# Stored as [mip_table] = [variable names]
variables = {}
# Get the experiment id
e_id = dq.inx.experiment.label[exp]
if e_id:
# Store info about exp in e_vars
e_vars = dq.inx.iref_by_sect[e_id[0]].a
# Query e_vars to get the 'requestItem's and loop over each item
for ri in e_vars['requestItem']:
# Get more info about this request item
dr = dq.inx.uid[ri]
rl = dq.inx.requestLink.uid[dr.rlid]
# Now we can get these vars that are within this request item
vars = dq.inx.iref_by_sect[rl.refid].a
var_list = vars['requestVar']
# Go through each requested var and get its name
for rv in var_list:
v_id = dq.inx.uid[rv].vid
c_var = dq.inx.uid[v_id]
if c_var.mipTable not in variables.keys():
variables[c_var.mipTable] = []
if c_var.label not in variables[c_var.mipTable]:
variables[c_var.mipTable].append(c_var.label)
print ('EXPERIMENT: {0}'.format(exp))
for mt,var_s in sorted(variables.iteritems()):
print ('____________________')
print ('TABLE : {0}'.format(mt))
print (sorted(var_s))
else:
print ('EXPERIMENT: {0}'.format(exp))
print ('No variables found')
print ('____________________')
def anal(self,dq=None):
if dq != None:
self.dq = dq
if self.dq == None:
self.dq = dreq.loadDreq()
ifile = '../scripts/AR6_priority_variables_02.csv'
ii = csv.reader( open( ifile ), delimiter='\t' )
self.wg1_vars = set()
for l in ii:
self.wg1_vars.add( tuple( l[2].split('.' ) ) )
self.priority_by_var = dict()
cc0 = collections.defaultdict( int )
for i in self.dq.coll['CMORvar'].items:
v = self.dq.inx.uid[i.vid]
tt = (i.mipTable,v.label)
if tt in self.wg1_vars:
p = 0
else:
p = i.defaultPriority
self.priority_by_var[ (i.mipTable,v.label) ] = p
cc0[ p ] += 1
ks = sorted( list( self.by_model.keys() ), key=lambda x:len(self.by_model[x]) )
ks.reverse()
oo = open( 'summary_by_model.csv', 'w' )
for k in ks:
cc = collections.defaultdict( int )
l1 = len( self.by_model[k] )
for v in self.by_model[k]:
if v not in self.priority_by_var:
print( 'ERROR: not found: %s, %s' % (k,v) )
cc[ 999 ] += 1
else:
cc[ self.priority_by_var[ v ] ] += 1
print( '%32s:: %s [%s -- %s -- %s -- %s] {%s}' % (k,l1,*[cc[k] for k in range(4)],cc[999]) )
rec = [k, *[str(x) for x in [l1,cc[0],cc[1],cc[2],cc[3]] ], *[str( cc[k]/cc0[k] ) for k in [0,1,2,3]] ]
oo.write( '\t'.join( rec ) + '\n' )
oo.close()
oo = open( 'summary_by_var.csv', 'w' )
ks = sorted( list( self.by_var.keys() ), key=lambda x:len(self.by_var[x]) )
ks.reverse()
for v in ks:
cc = collections.defaultdict( int )
if v not in self.priority_by_var:
p = -1
else:
p = self.priority_by_var[ v ]
rec = [*v, str( len( self.by_var[v] ) ), str(p) ]
oo.write( '\t'.join( rec ) + '\n' )
oo.close()
def __init__(self):
self.dq = dreq.loadDreq()
self.exptix = dict()
self.varix = dict()
self.cmvix = dict()
for i in self.dq.coll['experiment'].items:
self.exptix[i.label] = i.uid
for i in self.dq.coll['var'].items:
self.varix[i.label] = i.uid
for i in self.dq.coll['CMORvar'].items:
self.cmvix[(i.mipTable, i.label)] = i.uid
def get_mip_list(dq=None):
"""
Get a list of all MIPs within the CMIP6 data request
Return:
mips(dictionary): a dictionary containing the mip name as the key and
its values being the description of that mip
"""
mips = {}
if dq is None:
dq = dreq.loadDreq()
for m in dq.coll['mip'].items:
mips[m.label] = m.title
return mips
def main(options):
""" main
Arguments:
options (list) - input options from command line
"""
debug = options.debug
dq = dreq.loadDreq()
version = dq.version
mips = get_mip_list(dq=dq)
exps = get_exp_list(dq=dq)
# create a db connection and cursor
db = MySQLdb.connect(host="localhost", user="******", passwd="c$3gdb", db="csegdb")
cursor = db.cursor()
def get_exp_list(dq=None):
"""
Get a list of all experiments within the CMIP6 data request
Return:
exp(ditctionary): a dictionary containing the experiment name as the key and
its values being the description of that experiment
"""
exp = {}
if dq is None:
dq = dreq.loadDreq()
for e in dq.coll['experiment'].items:
exp[e.label] = [e.description, e.uid]
return exp
def map_exp_to_design_mip(exp_name, dq=None):
"""
Takes an experiment name and returns the designing mip name
Args:
exp_name: name of the experiment to look up
Return:
mip(string): name of the designing mip
"""
if dq is None:
dq = dreq.loadDreq()
e_id = dq.inx.experiment.label[exp_name][0]
mip = dq.inx.uid[e_id].mip
return mip
def main(options):
""" main
Arguments:
options (list) - input options from command line
"""
case_dict = dict()
debug = options.debug
dq = dreq.loadDreq()
if options.MIPS:
print(
'\n\n##################################################################'
)
mips = get_mip_list(dq=dq)
print('MIPS: ')
_pp.pprint(mips)
if options.expNames:
print(
'\n\n##################################################################'
)
exps = get_exp_list(dq=dq)
print('Experiments: ')
_pp.pprint(exps)
if options.expMips:
print(
'\n\n##################################################################'
)
for e in dq.coll['experiment'].items:
print('Requesting: ', e.label,
map_exp_to_request_mip(e.label, dq=dq))
print('Designing: ', e.label, map_exp_to_design_mip(e.label,
dq=dq), '\n')
def main(options):
""" main
Arguments:
options (list) - input options from command line
"""
debug = options.debug
dq = dreq.loadDreq()
version = dq.version
mips = get_mip_list(dq=dq)
# create a db connection and cursor
db = MySQLdb.connect(host="localhost",
user="******",
passwd="c$3gdb",
db="csegdb")
cursor = db.cursor()
# loop through the mips list and load them into the database
for key, value in mips.iteritems():
if key not in _exclude_mips:
count = 0
sql = "select count(id), id, name, description, dreq_version from t2_cmip6_MIP_types where name = '{0}'".format(
key)
mip_description = db.escape_string(value)
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
(count, id, name, description,
dreq_version) = cursor.fetchone()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
if count == 1:
sql = "update t2_cmip6_MIP_types set name = '{0}', description = '{1}', dreq_version = '{2}' where id = {3}".format(
key, mip_description, version, id)
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
db.commit()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
elif count == 0:
sql = "insert into t2_cmip6_MIP_types (name, description, dreq_version) value ('{0}','{1}','{2}')".format(
key, mip_description, version)
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
db.commit()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
else:
print(
"Error in database {0} rows found matching MIP name '{1}'".
format(count, name))
# disconnect from server
db.close()
htmlStyle['CMORvar'] = {'getIrefs': ['requestVar']}
htmlStyle['requestVarGroup'] = {'getIrefs': ['requestVar', 'requestLink']}
htmlStyle['var'] = {'getIrefs': ['CMORvar']}
htmlStyle['objective'] = {'getIrefs': ['objectiveLink']}
htmlStyle['requestLink'] = {'getIrefs': ['objectiveLink', 'requestItem']}
htmlStyle['exptgroup'] = {'getIrefs': ['__all__']}
htmlStyle['requestItem'] = {'getIrefs': ['__all__']}
htmlStyle['experiment'] = {'getIrefs': ['__all__']}
htmlStyle['mip'] = {'getIrefs': ['__all__']}
htmlStyle['remarks'] = {'getIrefs': ['__all__']}
htmlStyle['varChoice'] = {'getIrefs': ['__all__']}
htmlStyle['spatialShape'] = {'getIrefs': ['__all__']}
htmlStyle['temporalShape'] = {'getIrefs': ['__all__']}
htmlStyle['structure'] = {'getIrefs': ['__all__']}
htmlStyle['standardname'] = {'getIrefs': ['__all__']}
dq = dreq.loadDreq(htmlStyles=htmlStyle)
##
## add special styles to dq object "itemStyle" dictionary.
##
dq.itemStyles['standardname'] = styls.snLink
dq.itemStyles['var'] = styls.varLink
dq.itemStyles['mip'] = styls.mipLink
dq.itemStyles['CMORvar'] = styls.cmvLink
dq.itemStyles['objective'] = styls.objLink
dq.itemStyles['structure'] = styls.strLink
dq.itemStyles['objectiveLink'] = styls.objLnkLink
dq.itemStyles['requestVarGroup'] = styls.vgrpLink
dq.itemStyles['requestLink'] = styls.rqlLink02
dq.itemStyles['spatialShape'] = styls.labTtl
dq.coll['var'].items[0].__class__._linkAttrStyle['sn'] = styls.snLink01
def parse_table(self, exp, mips, tables, v_list, table_var_fields,
table_axes_fields, table_fields, user_vars, user_axes,
user_tableInfo):
"""
Function will parse an XML file using dreqPy and return a dictionary containing
the parsed fields.
Parameter:
exp (str): The name of the experiment.
miptable (str): The name of the miptable.
table_var_fields (dict): Dictionary containing standardized field var names
as keys and acceptable synonyms as values.
table_axes_fields (dict): Dictionary containing standardized field axes names
as keys and acceptable synonyms as values.
table_fields (dict): Dictionary containing standardized table field names
as keys and acceptable synonyms as values.
user_vars (dict): User defined dictionary. Keys should match standard name.
Values should be a list of acceptable field names.
user_axes (dict): User defined dictionary. Keys should match standard name.
Values should be a list of acceptable field names.
user_tableInfo (dict): User defined dictionary. Keys should match standard name.
Values should be a list of acceptable field names.
Returns:
table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys:
'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and
each variable has a value of a dictionary keyed with the standard field names.
"""
from dreqPy import dreq
dq = dreq.loadDreq()
# Get table id
# if len(g_id) == 0:
# print 'ERROR: Variable group/table ',table_name, ' not supported.'
# print 'Please select from the following: '
# print dq.inx.requestVarGroup.label.keys()
# print '\nIf your selected table is listed, it may not be supported in this verison of dreqpy.'
# print '\nEXITING. \n'
# sys.exit(1)
# # Get the id's of the variables in this table
# g_vars = dq.inx.iref_by_sect[g_id[0]].a
# Get a list of mips for the experiment
# print sorted(dq.inx.experiment.label.keys()),len(dq.inx.experiment.label.keys())
e_mip = []
e_id = dq.inx.experiment.label[exp]
if len(e_id) == 0:
print '\033[91m', 'Invalid experiment name. Please choose from the folowing options:', '\033[0m'
print sorted(dq.inx.experiment.label.keys()), len(
dq.inx.experiment.label.keys())
return {}
activity_id = dq.inx.uid[e_id[0]].mip
e_vars = dq.inx.iref_by_sect[e_id[0]].a
if len(e_vars['requestItem']) == 0:
e_vars = dq.inx.iref_by_sect[dq.inx.uid[e_id[0]].egid].a
total_request = {}
for ri in e_vars['requestItem']:
table_info = {}
dr = dq.inx.uid[ri]
if dr.mip in mips or '--ALL--' in mips:
table_dict = {}
variables = {}
axes = {}
table_info = {}
data = {}
table_info['experiment'] = dq.inx.uid[e_id[0]].title
table_info['experiment_id'] = exp
table_info['data_specs_version'] = dreq.version
table_info['activity_id'] = activity_id
rl = dq.inx.requestLink.uid[dr.rlid]
rvg = dq.inx.requestVarGroup.uid[rl.refid]
vars = dq.inx.iref_by_sect[rl.refid].a
axes_list = []
var_list = []
if len(v_list) == 0:
var_list = vars['requestVar']
else:
for v in v_list:
uids = dq.inx.requestVar.label[v]
for uid in uids:
if uid in vars['requestVar']:
var_list.append(uid)
for rv in var_list:
var = {}
v_id = dq.inx.uid[rv].vid # Get the CMORvar id
c_var = dq.inx.uid[v_id]
# Set what we can from the CMORvar section
if hasattr(c_var, 'mipTable'):
var['mipTable'] = c_var.mipTable
if c_var.mipTable in tables or '--ALL--' in tables:
var["_FillValue"] = "1e+20"
#var["missing_value"] = "1e+20"
if hasattr(c_var, 'deflate'):
var['deflate'] = c_var.deflate
if hasattr(c_var, 'deflate_level'):
var['deflate_level'] = c_var.deflate_level
if hasattr(c_var, 'description'):
var['description'] = c_var.description
if hasattr(c_var, 'frequency'):
var['frequency'] = c_var.frequency
table_info['frequency'] = c_var.frequency
if hasattr(c_var, 'label'):
var['id'] = c_var.label
var['out_name'] = c_var.label
var['variable_id'] = c_var.label
l = dq.inx.var.label[c_var.label]
if len(l) > 0:
var['standard_name'] = dq.inx.var.uid[
l[0]].sn
if hasattr(c_var, 'modeling_realm'):
var['realm'] = c_var.modeling_realm
#if hasattr(c_var,'ok_min_mean_abs'):
# var['ok_min_mean_abs']= c_var.ok_min_mean_abs
#if hasattr(c_var,'ok_max_mean_abs'):
# var['ok_max_mean_abs']= c_var.ok_max_mean_abs
if hasattr(c_var, 'out_name'):
var['out_name'] = c_var.label #?
if hasattr(c_var, 'positive'):
var['positive'] = c_var.positive
if hasattr(c_var, 'direction'):
var['direction'] = c_var.direction
if hasattr(c_var, 'prov'):
var['prov'] = c_var.prov
if hasattr(c_var, 'procNote'):
var['provcNote'] = c_var.procNote
if hasattr(c_var, 'shuffle'):
var['shuffle'] = c_var.shuffle
if hasattr(c_var, 'title'):
var['title'] = c_var.title
var['long_name'] = c_var.title
if hasattr(c_var, 'description'):
var['comment'] = c_var.description
if hasattr(c_var, 'type'):
var['type'] = c_var.type
if hasattr(c_var, 'valid_max'):
if isinstance(c_var.valid_max,
(int, long, float, complex)):
var['valid_max'] = c_var.valid_max
if hasattr(c_var, 'valid_min'):
if isinstance(c_var.valid_min,
(int, long, float, complex)):
var['valid_min'] = c_var.valid_min
# Set what we can from the standard section
if hasattr(c_var, 'stid'):
s_var = dq.inx.uid[c_var.stid]
if hasattr(s_var, 'cell_measures'):
var['cell_measures'] = s_var.cell_measures
if hasattr(s_var, 'cell_methods'):
var['cell_methods'] = s_var.cell_methods
if hasattr(s_var, 'coords'):
#if len(s_var.coords)>0:
if isinstance(s_var.cids, list):
var['coordinate'] = dq.inx.uid[
s_var.cids[0]].label
c = dq.inx.uid[s_var.cids[0]]
if c not in axes_list and c != '' and c != 'None':
axes_list.append(c)
# Set what we can from the time section
if hasattr(s_var, 'tmid'):
t_var = dq.inx.uid[s_var.tmid]
if hasattr(t_var, 'dimensions'):
t = t_var.dimensions
if t != '' and t != 'None':
var['time'] = t
var['coordinates'] = t
if hasattr(t_var, 'label'):
var['time_label'] = t_var.label
if hasattr(t_var, 'title'):
var['time_title'] = t_var.title
# Is there did?
if hasattr(s_var, 'dids'):
if isinstance(s_var.dids, tuple):
extra_dim = dq.inx.uid[s_var.dids[0]].label
if 'coordinates' not in var.keys():
var['coordinates'] = extra_dim
else:
var['coordinates'] = extra_dim + "|" + var[
'coordinates']
if extra_dim not in axes_list:
axes_list.append(extra_dim)
# Set what we can from the spatial section
if hasattr(s_var, 'spid'):
sp_var = dq.inx.uid[s_var.spid]
if hasattr(sp_var, 'dimensions'):
if len(sp_var.dimensions) > 1:
if 'coordinates' in var.keys():
sp_var_d = sp_var.dimensions
if len(sp_var_d) > 1:
if sp_var_d[-1] == '|':
sp_var_d = sp_var_d[:-1]
var['coordinates'] = sp_var_d + '|' + var[
'coordinates']
else:
var['coordinates'] = sp_var.dimensions
if 'grid_latitude' in var[
'coordinates']:
var['coordinates'] = var[
'coordinates'].replace(
'grid_latitude',
'gridlatitude')
dims = var['coordinates'].split('|')
for d in dims:
if d not in axes_list and d != '' and d != 'None':
if 'copy' not in var[
'id'] and '?' not in d:
axes_list.append(d)
if 'coordinates' in var.keys():
if len(var['coordinates']) > 1:
if var['coordinates'][-1] == '|':
var['coordinates'] = var[
'coordinates'][:-1]
# Set what we can from the variable section
if hasattr(c_var, 'vid'):
v_var = dq.inx.uid[c_var.vid]
if hasattr(v_var, 'cf_standard_name'):
var['cf_standard_name'] = v_var.sn
if hasattr(v_var, 'long_name'):
var['long_name'] = v_var.sn
if hasattr(v_var, 'units'):
if v_var.units == "":
var['units'] = 'None'
print c_var.label, " does not have units"
else:
var['units'] = v_var.units
# Add variable to variable dictionary
variables[c_var.label] = var
for a in axes_list:
if 'grid_latitude' in a:
a = 'gridlatitude'
if a in dq.inx.grids.label.keys():
id = dq.inx.grids.label[a]
if len(id) > 0:
v = dq.inx.grids.uid[id[0]]
ax = {}
if hasattr(v, 'units'):
if v.units == "":
ax['units'] = '1'
else:
ax['units'] = v.units
if hasattr(v, 'value'):
ax['value'] = v.value
if hasattr(v, 'axis'):
ax['axis'] = v.axis
if hasattr(v, 'valid_max'):
if isinstance(v.valid_max,
(int, long, float, complex)):
ax['valid_max'] = v.valid_max
if hasattr(v, 'valid_min'):
if isinstance(v.valid_min,
(int, long, float, complex)):
ax['valid_min'] = v.valid_min
if hasattr(v, 'standardName'):
if isinstance(v.standardName, (str)):
ax['standard_name'] = v.standardName
else:
ax['standard_name'] = a
if hasattr(v, 'type'):
if 'landUse' in a:
ax['type'] = 'int'
else:
ax['type'] = v.type
if hasattr(v, 'id'):
ax['id'] = v.label
if hasattr(v, 'positive'):
ax['positive'] = v.positive
if hasattr(v, 'direction'):
ax['direction'] = v.direction
if hasattr(v, 'title'):
ax['title'] = v.title
if hasattr(v, 'bounds'):
if 'yes' in v.bounds:
ax['bounds'] = v.label + "_bnds"
if hasattr(v, 'requested'):
ax['requested'] = v.requested
#if hasattr(v,'boundsValues'):
# ax['boundsValues'] = v.boundsValues
if hasattr(v, 'coords'):
ax['coords'] = v.coords
axes[a] = ax
else:
v = a
print "Cannot find link for dimension: ", v
try:
table_dict['variables'] = variables
table_dict['axes'] = axes
except UnboundLocalError:
print 'Does not fit criteria: ', exp
table_dict['table_info'] = table_info
tab = rvg.label
table_info['table_id'] = tab
total_request[dr.mip + '_' + tab] = table_dict
#print 'Total in request:',len(total_request)
#for k in sorted(total_request.keys()):
# v = total_request[k]
# print k, len(v['variables'])
return total_request
import json, urllib
import collections
import urllib.request
import shelve
from dreqPy import dreq
dq = dreq.loadDreq()
ss = set()
for i in dq.coll['CMORvar'].items:
if i.mipTable == 'Amon' and i.defaultPriority == 1:
ss.add((i.mipTable, i.label))
temp = 'https://esgf-index1.ceda.ac.uk/esg-search/search/?offset=0&limit=500&type=Dataset&replica=false&latest=true&project%%21=input4mips&activity_id=CMIP&table_id=%(table_label)s&mip_era=CMIP6&variable_id=%(variable_label)s&facets=mip_era%%2Cactivity_id%%2Cmodel_cohort%%2Cproduct%%2Csource_id%%2Cinstitution_id%%2Csource_type%%2Cnominal_resolution%%2Cexperiment_id%%2Csub_experiment_id%%2Cvariant_label%%2Cgrid_label%%2Ctable_id%%2Cfrequency%%2Crealm%%2Cvariable_id%%2Ccf_standard_name%%2Cdata_node&format=application%%2Fsolr%%2Bjson'
selection = "activity_id=CMIP&table_id=%(table_label)s&mip_era=CMIP6&variable_id=%(variable_label)s"
sdict = dict(table_id='Amon', experiment_id='historical', variable_id='tas')
temp2 = 'https://esgf-index1.ceda.ac.uk/esg-search/search/?offset=0&limit=500&type=Dataset&replica=false&latest=true&project%%21=input4mips&%(selection)s&facets=mip_era%%2Cactivity_id%%2Cmodel_cohort%%2Cproduct%%2Csource_id%%2Cinstitution_id%%2Csource_type%%2Cnominal_resolution%%2Cexperiment_id%%2Csub_experiment_id%%2Cvariant_label%%2Cgrid_label%%2Ctable_id%%2Cfrequency%%2Crealm%%2Cvariable_id%%2Ccf_standard_name%%2Cdata_node&format=application%%2Fsolr%%2Bjson'
tmp = dict(
Amon=
'tas, pr, uas, vas, huss, rsut, rsdt, rlut, rsus, rsds, rlus, rlds, ps, ua, va, zg, cl, clt'
.split(', '),
AERmon='od550aer, abs550aer'.split(', '),
Lmon='cVeg, cLitter, gpp, nbp, npp'.split(', '),
Omon=[
'fgco2',
],
Emon=[
'cSoil',
def loadDreq(self, dq=None):
if dq == None:
dq = dreq.loadDreq()
collect.add(dq)
self.__extra__['dq'] = dq
def load_from_dqroot(dqroot=None):
(dqxml, dqconfig) = reroot_paths(dqroot,
(defaultDreqPath, defaultConfigPath))
return loadDreq(dreqXML=dqxml, configdoc=dqconfig)
def main(options):
""" main
Arguments:
options (list) - input options from command line
"""
debug = options.debug
# create a db connection and cursor
db = MySQLdb.connect(host="localhost",
user="******",
passwd="c$3gdb",
db="csegdb")
cursor = db.cursor()
cursor1 = db.cursor()
# load a data request object
dq = dreq.loadDreq()
version = dq.version
# loop through the list of experiments and get the associated requesting MIPs
sql = "select id, name, design_mip from t2_cmip6_exps"
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
except:
print("Error executing sql = {0}".format(sql))
sys.exit(1)
for row in cursor:
# get the design_mip_id from the t2_cmip6_MIP_types
count1 = 0
sql = "select count(id), id from t2_cmip6_MIP_types where name = '{0}'".format(
row[2])
try:
print("Executing sql = {0}".format(sql))
cursor1.execute(sql)
(count1, design_mip_id) = cursor1.fetchone()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
if count1 == 1:
# update / insert into the t2j_cmip6 table
sql = "select count(exp_id) from t2j_cmip6 where exp_id = {0}".format(
row[0])
try:
print("Executing sql = {0}".format(sql))
cursor1.execute(sql)
count2 = cursor1.fetchone()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
if count2[0] == 0:
sql = "insert into t2j_cmip6 (exp_id, design_mip_id) value ({0},{1})".format(
row[0], design_mip_id)
try:
print("Executing sql = {0}".format(sql))
cursor1.execute(sql)
db.commit()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
if count2[0] == 1:
sql = "update t2j_cmip6 set design_mip_id = {0} where exp_id = {1}".format(
design_mip_id, row[0])
try:
print("Executing sql = {0}".format(sql))
cursor1.execute(sql)
db.commit()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
# loop through the list of requesting mips and update or insert in the join table
reqMips = map_exp_to_request_mip(row[1], dq=dq)
for rm in reqMips:
count2 = 0
sql = "select count(id), id from t2_cmip6_MIP_types where name = '{0}'".format(
rm)
try:
print("Executing sql = {0}".format(sql))
cursor1.execute(sql)
(count2, MIPid) = cursor1.fetchone()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
if count2 == 1:
# check if record already exists in join table
count3 = 0
sql = "select count(exp_id), exp_id from t2j_cmip6_exps_mips where exp_id = {0} and mip_id = {1}".format(
row[0], MIPid)
try:
print("Executing sql = {0}".format(sql))
cursor1.execute(sql)
(count3, exp_id) = cursor1.fetchone()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
if count3 == 0:
# insert a new record
sql = "insert into t2j_cmip6_exps_mips (exp_id, mip_id) value ({0}, {1})".format(
row[0], MIPid)
try:
print("Executing sql = {0}".format(sql))
cursor1.execute(sql)
db.commit()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
else:
print("Error there are {0} MIPS matching name {1}".format(
count2, rm))
# add the CMIP DECK entries to the exps and join tables
join_DECK_exps(db, cursor, version)
# disconnect from server
db.close()
def parse_table(
self,
exp,
mips,
tables,
v_list,
table_var_fields,
table_axes_fields,
table_fields,
user_vars,
user_axes,
user_tableInfo,
):
"""
Function will parse an XML file using dreqPy and return a dictionary containing
the parsed fields.
Parameter:
exp (str): The name of the experiment.
miptable (str): The name of the miptable.
table_var_fields (dict): Dictionary containing standardized field var names
as keys and acceptable synonyms as values.
table_axes_fields (dict): Dictionary containing standardized field axes names
as keys and acceptable synonyms as values.
table_fields (dict): Dictionary containing standardized table field names
as keys and acceptable synonyms as values.
user_vars (dict): User defined dictionary. Keys should match standard name.
Values should be a list of acceptable field names.
user_axes (dict): User defined dictionary. Keys should match standard name.
Values should be a list of acceptable field names.
user_tableInfo (dict): User defined dictionary. Keys should match standard name.
Values should be a list of acceptable field names.
Returns:
table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys:
'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and
each variable has a value of a dictionary keyed with the standard field names.
"""
from dreqPy import dreq
dq = dreq.loadDreq()
# Get table id
# if len(g_id) == 0:
# print 'ERROR: Variable group/table ',table_name, ' not supported.'
# print 'Please select from the following: '
# print dq.inx.requestVarGroup.label.keys()
# print '\nIf your selected table is listed, it may not be supported in this verison of dreqpy.'
# print '\nEXITING. \n'
# sys.exit(1)
# # Get the id's of the variables in this table
# g_vars = dq.inx.iref_by_sect[g_id[0]].a
# Get a list of mips for the experiment
# print sorted(dq.inx.experiment.label.keys()),len(dq.inx.experiment.label.keys())
# e_mip = []
if "--ALL--" in exp:
e_id = dq.inx.experiment.label["historical"]
else:
e_id = dq.inx.experiment.label[exp]
if len(e_id) == 0:
# print '\033[91m','Invalid experiment name. Please choose from the folowing options:','\033[0m'
# print sorted(dq.inx.experiment.label.keys()),len(dq.inx.experiment.label.keys())
# return {}
print(
"\033[91m",
"This experiment name is not registered within the data request.",
"\033[0m",
)
print(
"\033[91m",
"Using a generic exp name instead. If you think this name should be registered, verify the name is correct from the list below.",
"\033[0m",
)
print(
sorted(dq.inx.experiment.label.keys()),
len(dq.inx.experiment.label.keys()),
)
e_id = dq.inx.experiment.label["historical"]
activity_id = dq.inx.uid[e_id[0]].mip
e_vars = []
activity_id = dq.inx.uid[e_id[0]].mip
e_vars.append(dq.inx.iref_by_sect[e_id[0]].a)
e_vars.append(dq.inx.iref_by_sect[dq.inx.uid[e_id[0]].egid].a)
e_vars.append(dq.inx.iref_by_sect[activity_id].a)
total_request = {}
for e_var in e_vars:
for ri in e_var["requestItem"]:
table_info = {}
dr = dq.inx.uid[ri]
if dr.mip in mips or "--ALL--" in mips or "--ALL--" in exp:
table_dict = {}
variables = {}
axes = {}
table_info = {}
# data = {}
table_info["data_specs_version"] = dreq.version
if "--ALL--" not in exp:
table_info["experiment"] = dq.inx.uid[e_id[0]].title
table_info["experiment_id"] = exp
table_info["activity_id"] = activity_id
rl = dq.inx.requestLink.uid[dr.rlid]
rvg = dq.inx.requestVarGroup.uid[rl.refid]
vars = dq.inx.iref_by_sect[rl.refid].a
axes_list = []
var_list = []
if len(v_list) == 0:
var_list = vars["requestVar"]
else:
# Make a list of all cmip vars and their id's
all_vars = {}
for v in dq.coll["CMORvar"].items:
all_vars[v.label + ":" + v.mipTable] = v.uid
for v in v_list:
if v in all_vars.keys():
var_list.append(all_vars[v])
else:
print("Can not add variable: " + v)
for rv in var_list:
var = {}
if len(v_list) == 0:
v_id = dq.inx.uid[rv].vid # Get the CMORvar id
else:
v_id = dq.inx.uid[rv].uid # Get the CMORvar id
c_var = dq.inx.uid[v_id]
# Set what we can from the CMORvar section
if hasattr(c_var, "mipTable"):
var["mipTable"] = c_var.mipTable
if c_var.mipTable in tables or "--ALL--" in tables:
var["_FillValue"] = "1.0E20"
var["missing_value"] = 1.0e20
if hasattr(c_var, "deflate"):
var["deflate"] = c_var.deflate
if hasattr(c_var, "deflate_level"):
var["deflate_level"] = c_var.deflate_level
if hasattr(c_var, "description"):
var["description"] = c_var.description
if hasattr(c_var, "frequency"):
var["frequency"] = c_var.frequency
table_info["frequency"] = c_var.frequency
if hasattr(c_var, "label"):
var["id"] = c_var.label
var["out_name"] = c_var.label
var["variable_id"] = dq.inx.uid[
c_var.vid].label
# l = dq.inx.var.label[c_var.label]
# if len(l)>0:
var["standard_name"] = dq.inx.uid[
c_var.vid].sn
if hasattr(c_var, "modeling_realm"):
var["realm"] = c_var.modeling_realm
# if hasattr(c_var,'ok_min_mean_abs'):
# var['ok_min_mean_abs']= c_var.ok_min_mean_abs
# if hasattr(c_var,'ok_max_mean_abs'):
# var['ok_max_mean_abs']= c_var.ok_max_mean_abs
if hasattr(c_var, "out_name"):
var["out_name"] = c_var.label
if hasattr(c_var, "positive"):
var["positive"] = c_var.positive
if hasattr(c_var, "direction"):
var["direction"] = c_var.direction
if hasattr(c_var, "prov"):
var["prov"] = c_var.prov
if hasattr(c_var, "procNote"):
var["provcNote"] = c_var.procNote
if hasattr(c_var, "shuffle"):
var["shuffle"] = c_var.shuffle
if hasattr(c_var, "title"):
var["title"] = c_var.title
var["long_name"] = c_var.title
# if hasattr(c_var,'description'):
# var['comment']= c_var.description
if hasattr(c_var, "type"):
var["type"] = c_var.type
if hasattr(c_var, "valid_max"):
if isinstance(c_var.valid_max,
(int, float, complex)):
var["valid_max"] = c_var.valid_max
if hasattr(c_var, "valid_min"):
if isinstance(c_var.valid_min,
(int, float, complex)):
var["valid_min"] = c_var.valid_min
# Set what we can from the standard section
if hasattr(c_var, "stid"):
s_var = dq.inx.uid[c_var.stid]
if hasattr(s_var, "cell_measures"):
var["cell_measures"] = s_var.cell_measures
if hasattr(s_var, "cell_methods"):
var["cell_methods"] = s_var.cell_methods
if hasattr(s_var, "coords"):
if isinstance(s_var.cids, list):
var["coordinates"] = dq.inx.uid[
s_var.cids[0]].label
c = dq.inx.uid[s_var.cids[0]]
if (c not in axes_list
and c != ""
and c != "None"):
axes_list.append(c)
# Set what we can from the time section
if hasattr(s_var, "tmid"):
t_var = dq.inx.uid[s_var.tmid]
if hasattr(t_var, "dimensions"):
t = t_var.dimensions
if t != "" and t != "None":
var["time"] = t
var["coordinates"] = t
if hasattr(t_var, "label"):
var["time_label"] = t_var.label
if hasattr(t_var, "title"):
var["time_title"] = t_var.title
# Is there did?
if hasattr(s_var, "dids"):
if isinstance(s_var.dids, tuple):
extra_dim = dq.inx.uid[
s_var.dids[0]].label
if "coordinates" not in var.keys():
var["coordinates"] = extra_dim
else:
var["coordinates"] = (
extra_dim + "|" +
var["coordinates"])
if extra_dim not in axes_list:
axes_list.append(extra_dim)
# Set what we can from the spatial section
if hasattr(s_var, "spid"):
sp_var = dq.inx.uid[s_var.spid]
if hasattr(sp_var, "dimensions"):
if len(sp_var.dimensions) > 1:
if "coordinates" in var.keys():
sp_var_d = sp_var.dimensions
if len(sp_var_d) > 1:
if sp_var_d[-1] == "|":
sp_var_d = sp_var_d[:
-1]
var["coordinates"] = (
sp_var_d + "|" +
var["coordinates"])
else:
var["coordinates"] = sp_var.dimensions
if ("grid_latitude"
in var["coordinates"]):
var["coordinates"] = var[
"coordinates"].replace(
"grid_latitude",
"gridlatitude")
dims = var[
"coordinates"].split("|")
for d in dims:
if (d not in axes_list
and d != ""
and d != "None"):
if ("copy" not in
var["id"] and
"?" not in d):
axes_list.append(d)
if "coordinates" in var.keys():
if len(var["coordinates"]) > 1:
if var["coordinates"][-1] == "|":
var["coordinates"] = var[
"coordinates"][:-1]
# Set what we can from the variable section
if hasattr(c_var, "vid"):
v_var = dq.inx.uid[c_var.vid]
if hasattr(v_var, "standard_name"):
var["standard_name"] = v_var.sn
if hasattr(v_var, "cf_standard_name"):
var["cf_standard_name"] = v_var.sn
if hasattr(v_var, "long_name"):
var["long_name"] = v_var.sn
if hasattr(v_var, "units"):
if v_var.units == "":
var["units"] = "None"
print(c_var.label,
" does not have units")
else:
var["units"] = v_var.units
# Add variable to variable dictionary
variables[c_var.label] = var
for a in axes_list:
if "grid_latitude" in a:
a = "gridlatitude"
if a in dq.inx.grids.label.keys():
id = dq.inx.grids.label[a]
if len(id) > 0:
v = dq.inx.grids.uid[id[0]]
ax = {}
if hasattr(v, "units"):
if v.units == "":
ax["units"] = "1"
else:
ax["units"] = v.units
if hasattr(v, "value"):
ax["value"] = v.value
if hasattr(v, "axis"):
ax["axis"] = v.axis
if hasattr(v, "valid_max"):
if isinstance(v.valid_max,
(int, float, complex)):
ax["valid_max"] = v.valid_max
if hasattr(v, "valid_min"):
if isinstance(v.valid_min,
(int, float, complex)):
ax["valid_min"] = v.valid_min
if hasattr(v, "standardName"):
if isinstance(v.standardName, (str)):
ax["standard_name"] = v.standardName
else:
ax["standard_name"] = a
if hasattr(v, "type"):
if "landUse" in a:
ax["type"] = "int"
else:
ax["type"] = v.type
if hasattr(v, "id"):
ax["id"] = v.label
if hasattr(v, "positive"):
ax["positive"] = v.positive
if hasattr(v, "direction"):
ax["direction"] = v.direction
if hasattr(v, "title"):
ax["title"] = v.title
if hasattr(v, "bounds"):
if "yes" in v.bounds:
ax["bounds"] = v.label + "_bnds"
if hasattr(v, "requested"):
ax["requested"] = v.requested
if hasattr(v, "altLabel"):
ax["altLabel"] = v.altLabel
# if hasattr(v,'boundsValues'):
# ax['boundsValues'] = v.boundsValues
if hasattr(v, "coords"):
ax["coords"] = v.coords
axes[a] = ax
else:
v = a
print("Cannot find link for dimension: ", v)
try:
table_dict["variables"] = variables
table_dict["axes"] = axes
except UnboundLocalError:
print("Does not fit criteria: ", exp)
table_dict["table_info"] = table_info
tab = rvg.label
table_info["table_id"] = tab
total_request[dr.mip + "_" + tab] = table_dict
# print 'Total in request:',len(total_request)
# for k in sorted(total_request.keys()):
# v = total_request[k]
# print k, len(v['variables'])
return total_request
def main(options):
""" main
Arguments:
options (list) - input options from command line
"""
debug = options.debug
# create a db connection and cursor
db = MySQLdb.connect(host="localhost",
user="******",
passwd="c$3gdb",
db="csegdb")
cursor = db.cursor()
# load a data request object
dq = dreq.loadDreq()
version = dq.version
# define a lookup data structures for MIP experiments that are CESM specific
## mip_exps = { 'DAMIP' : ['hist-nat-WACCM', 'hist-GHG-WACCM'],
## 'ScenarioMIP' : ['ssp585-WACCM', 'ssp370-WACCM', 'ssp245-WACCM', 'ssp126-WACCM', 'ssp534-over-WACCM'],
## 'RFMIP' : ['piClim-control-WACCM'] }
mip_exps = {
'ScenarioMIP':
['ssp126-ext-WACCM', 'ssp585-ext-WACCM', 'ssp534-over-ext-WACCM']
}
for mip, exps in mip_exps.iteritems():
# get the design_mip_id from the t2_cmip6_MIP_types
sql = "select id from t2_cmip6_MIP_types where name = '{0}'".format(
mip)
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
(design_mip_id) = cursor.fetchone()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
# loop over the exps for this MIP
for exp in exps:
# get the CMIP6 exp UID which will be the same for these new experiments
exp_tmp = exp.replace('-ext-WACCM', '')
exp_tmp = exp_tmp.replace('-over-ext-WACCM', '')
sql = "select uid, description from t2_cmip6_exps where name = '{0}'".format(
exp_tmp)
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
(uid, description) = cursor.fetchone()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
# check if this exp is already in the t2_cmip6_exps table
exp_id = 0
sql = "select count(id), id from t2_cmip6_exps where name = '{0}'".format(
exp)
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
(exists, exp_id) = cursor.fetchone()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
if not exists:
# insert a row into the t2_cmip6_exps
sql = "insert into t2_cmip6_exps (name, description, uid, design_mip, dreq_version) value ('{0}','{1}','{2}','{3}','{4}')".format(
exp, description, uid, mip, version)
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
db.commit()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
# get the id for the exp just inserted
sql = "select id from t2_cmip6_exps where name = '{0}'".format(
exp)
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
(id) = cursor.fetchone()
db.commit()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
# reset the exp_id to this new experiment
exp_id = id[0]
# check if record already exists in join table
sql = "select count(exp_id) from t2j_cmip6 where exp_id = {0}".format(
exp_id)
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
(count) = cursor.fetchone()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
if count[0] == 0:
# insert a new join record
sql = "insert into t2j_cmip6 (exp_id, design_mip_id) value ({0},{1})".format(
exp_id, design_mip_id[0])
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
db.commit()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
if count[0] == 1:
# update record with deck_id
sql = "update t2j_cmip6 set design_mip_id = {0} where exp_id = {1}".format(
design_mip_id[0], exp_id)
try:
print("Executing sql = {0}".format(sql))
cursor.execute(sql)
db.commit()
except:
print("Error executing sql = {0}".format(sql))
db.rollback()
# disconnect from server
db.close()
return 0
def _get_collection(key):
dq = dreq.loadDreq()
return dq.coll[key].items
