def __init__(self): self.dq = dreq.loadDreq() super(TestRq, self).__init__() self.CMORvar_by_id = dict() for i in self.dq.coll["CMORvar"].items: self.CMORvar_by_id["%s.%s" % (i.mipTable, i.label)] = i
def search_cmorlist(mip_filt, tab_filt): """ Get a cmorlist from a MIP name """ from dreqPy import dreq from dreqPy.extensions import collect dq = dreq.loadDreq() collect.add( dq ) mip = dq.coll['mip'].items[0]._labelDict[mip_filt] mvars = mip._get__CMORvar() expts = mip._get__expt() MIP_Expts = set([ expt for expt in dq.coll['experiment'].items if expt.uid in expts ]) MIP_Expts = list(MIP_Expts) # Find the 'CMORvar'-items, associated with these requestVars: cmorlist = {} mvarlist = {} for cmvar in dq.coll['CMORvar'].items: if cmvar.label == 'lwsnl': print (cmvar.mipTable, tab_filt) if cmvar.uid in mvars and tab_filt in cmvar.mipTable: cmorlist[cmvar.label] = cmvar mvarlist[cmvar.label] = dq.inx.uid[cmvar.vid] return cmorlist, mvarlist
def map_exp_to_request_mip(exp_name, dq=None): """ Takes an experiment name and returns the mips requesting data from it Args: exp_name: name of the experiment to look up Return: mip (list): name of the requesting mips """ if dq is None: dq = dreq.loadDreq() mips = [] if dq.inx.experiment.label[exp_name]: e_id = dq.inx.experiment.label[exp_name][0] mips.append(dq.inx.uid[e_id].mip) e_vars = dq.inx.iref_by_sect[e_id].a for ri in e_vars['requestItem']: dr = dq.inx.uid[ri] if dr.mip not in mips: mips.append(dr.mip) return mips
def dqload(dqtag=None, dqroot=None, dqpath=None): """Load the dreq from a dqtag and dqroot and dpath, all defaulted. Arguments: - tag -- the tag, dynamically defaulted from default_dqtag() - dqroot -- the dreq root directory, dynamically defaulted from default_dqroot() - dqpath -- the path to the XML directory, dynamically-defaulted from default_dqpath. This does no error checks itself : it will raise whatever exception the underlying dreq code does if things are bad. If you want to check for this use the valid_* functions. """ # This replicates some code in dqi.util and dqi.low, to avoid a # dependency on dqi as this is the only place djq relied on it. note_reply_metadata(dreqpy_path=dreqPy_path) top = effective_dqpath(dqtag=dqtag, dqroot=dqroot, dqpath=dqpath) xml = join(top, split(defaultDreqPath)[1]) config = join(top, split(defaultConfigPath)[1]) note_reply_metadata(dreq_top=top, dreq_xml=xml, dreq_config=config) dreq = loadDreq(dreqXML=xml, configdoc=config, manifest=None) note_reply_metadata(dreq_loaded_version=dreq.version) return dreq
def __init__(self): from dreqPy import dreq self.dq = dreq.loadDreq() self.CMORvar_by_id = dict() for i in self.dq.coll["CMORvar"].items: self.CMORvar_by_id["%s.%s" % (i.mipTable,i.label) ] = i
def main(options): """ main Arguments: options (list) - input options from command line """ debug = options.debug dq = dreq.loadDreq() version = dq.version exps = get_exp_list(dq=dq) # create a db connection and cursor db = MySQLdb.connect(host="localhost", user="******", passwd="c$3gdb", db="csegdb") cursor = db.cursor() # loop through the exps dictionary and load them into the database keying off the name for key, value in exps.iteritems(): # check that the key (experiment name) isn't included in the _exclude_exps list if key not in _exclude_exps: count = 0 sql = "select count(id), id, name, description, uid, design_mip, dreq_version from t2_cmip6_exps where name = '"+key+"'" dreq_description = db.escape_string(value[0]) dreq_uid = db.escape_string(value[1]) dreq_design_mip = db.escape_string(value[2]) try: print ("Executing sql = {0}".format(sql)) cursor.execute(sql) (count, id, name, description, uid, design_mip, dreq_version) = cursor.fetchone() except: print ("Error executing sql = {0}".format(sql)) db.rollback() if count == 1: sql = "update t2_cmip6_exps set description = '{0}', uid = '{1}', design_mip = '{2}', dreq_version = '{3}' where id = {4}".format(dreq_description, dreq_uid, dreq_design_mip, version, str(id)) try: print ("Executing sql = {0}".format(sql)) cursor.execute(sql) db.commit() except: print("Error executing sql = {0}".format(sql)) db.rollback() elif count == 0: sql = "insert into t2_cmip6_exps (name, description, uid, design_mip, dreq_version) value ('{0}','{1}','{2}','{3}','{4}')".format(key, dreq_description, dreq_uid, dreq_design_mip, version) try: print ("Executing sql = {0}".format(sql)) cursor.execute(sql) db.commit() except: print ("Error executing sql = {0}".format(sql)) db.rollback() else: print("Error in database {0} rows found matching experiment name '{1}'".format(count, name)) # disconnect from server db.close()
def main(options): """ main Arguments: options (list) - input options from command line """ debug = options.debug # Load in the request dq = dreq.loadDreq() # load up the experiments name into a list exps = get_exp_list(dq=dq) for exp, value in exps.iteritems(): # Dictionary to hold the variables # Stored as [mip_table] = [variable names] variables = {} # Get the experiment id e_id = dq.inx.experiment.label[exp] if e_id: # Store info about exp in e_vars e_vars = dq.inx.iref_by_sect[e_id[0]].a # Query e_vars to get the 'requestItem's and loop over each item for ri in e_vars['requestItem']: # Get more info about this request item dr = dq.inx.uid[ri] rl = dq.inx.requestLink.uid[dr.rlid] # Now we can get these vars that are within this request item vars = dq.inx.iref_by_sect[rl.refid].a var_list = vars['requestVar'] # Go through each requested var and get its name for rv in var_list: v_id = dq.inx.uid[rv].vid c_var = dq.inx.uid[v_id] if c_var.mipTable not in variables.keys(): variables[c_var.mipTable] = [] if c_var.label not in variables[c_var.mipTable]: variables[c_var.mipTable].append(c_var.label) print ('EXPERIMENT: {0}'.format(exp)) for mt,var_s in sorted(variables.iteritems()): print ('____________________') print ('TABLE : {0}'.format(mt)) print (sorted(var_s)) else: print ('EXPERIMENT: {0}'.format(exp)) print ('No variables found') print ('____________________')
def anal(self,dq=None): if dq != None: self.dq = dq if self.dq == None: self.dq = dreq.loadDreq() ifile = '../scripts/AR6_priority_variables_02.csv' ii = csv.reader( open( ifile ), delimiter='\t' ) self.wg1_vars = set() for l in ii: self.wg1_vars.add( tuple( l[2].split('.' ) ) ) self.priority_by_var = dict() cc0 = collections.defaultdict( int ) for i in self.dq.coll['CMORvar'].items: v = self.dq.inx.uid[i.vid] tt = (i.mipTable,v.label) if tt in self.wg1_vars: p = 0 else: p = i.defaultPriority self.priority_by_var[ (i.mipTable,v.label) ] = p cc0[ p ] += 1 ks = sorted( list( self.by_model.keys() ), key=lambda x:len(self.by_model[x]) ) ks.reverse() oo = open( 'summary_by_model.csv', 'w' ) for k in ks: cc = collections.defaultdict( int ) l1 = len( self.by_model[k] ) for v in self.by_model[k]: if v not in self.priority_by_var: print( 'ERROR: not found: %s, %s' % (k,v) ) cc[ 999 ] += 1 else: cc[ self.priority_by_var[ v ] ] += 1 print( '%32s:: %s [%s -- %s -- %s -- %s] {%s}' % (k,l1,*[cc[k] for k in range(4)],cc[999]) ) rec = [k, *[str(x) for x in [l1,cc[0],cc[1],cc[2],cc[3]] ], *[str( cc[k]/cc0[k] ) for k in [0,1,2,3]] ] oo.write( '\t'.join( rec ) + '\n' ) oo.close() oo = open( 'summary_by_var.csv', 'w' ) ks = sorted( list( self.by_var.keys() ), key=lambda x:len(self.by_var[x]) ) ks.reverse() for v in ks: cc = collections.defaultdict( int ) if v not in self.priority_by_var: p = -1 else: p = self.priority_by_var[ v ] rec = [*v, str( len( self.by_var[v] ) ), str(p) ] oo.write( '\t'.join( rec ) + '\n' ) oo.close()
def __init__(self): self.dq = dreq.loadDreq() self.exptix = dict() self.varix = dict() self.cmvix = dict() for i in self.dq.coll['experiment'].items: self.exptix[i.label] = i.uid for i in self.dq.coll['var'].items: self.varix[i.label] = i.uid for i in self.dq.coll['CMORvar'].items: self.cmvix[(i.mipTable, i.label)] = i.uid
def get_mip_list(dq=None): """ Get a list of all MIPs within the CMIP6 data request Return: mips(dictionary): a dictionary containing the mip name as the key and its values being the description of that mip """ mips = {} if dq is None: dq = dreq.loadDreq() for m in dq.coll['mip'].items: mips[m.label] = m.title return mips
def main(options): """ main Arguments: options (list) - input options from command line """ debug = options.debug dq = dreq.loadDreq() version = dq.version mips = get_mip_list(dq=dq) exps = get_exp_list(dq=dq) # create a db connection and cursor db = MySQLdb.connect(host="localhost", user="******", passwd="c$3gdb", db="csegdb") cursor = db.cursor()
def get_exp_list(dq=None): """ Get a list of all experiments within the CMIP6 data request Return: exp(ditctionary): a dictionary containing the experiment name as the key and its values being the description of that experiment """ exp = {} if dq is None: dq = dreq.loadDreq() for e in dq.coll['experiment'].items: exp[e.label] = [e.description, e.uid] return exp
def map_exp_to_design_mip(exp_name, dq=None): """ Takes an experiment name and returns the designing mip name Args: exp_name: name of the experiment to look up Return: mip(string): name of the designing mip """ if dq is None: dq = dreq.loadDreq() e_id = dq.inx.experiment.label[exp_name][0] mip = dq.inx.uid[e_id].mip return mip
def main(options): """ main Arguments: options (list) - input options from command line """ case_dict = dict() debug = options.debug dq = dreq.loadDreq() if options.MIPS: print( '\n\n##################################################################' ) mips = get_mip_list(dq=dq) print('MIPS: ') _pp.pprint(mips) if options.expNames: print( '\n\n##################################################################' ) exps = get_exp_list(dq=dq) print('Experiments: ') _pp.pprint(exps) if options.expMips: print( '\n\n##################################################################' ) for e in dq.coll['experiment'].items: print('Requesting: ', e.label, map_exp_to_request_mip(e.label, dq=dq)) print('Designing: ', e.label, map_exp_to_design_mip(e.label, dq=dq), '\n')
def main(options): """ main Arguments: options (list) - input options from command line """ debug = options.debug dq = dreq.loadDreq() version = dq.version mips = get_mip_list(dq=dq) # create a db connection and cursor db = MySQLdb.connect(host="localhost", user="******", passwd="c$3gdb", db="csegdb") cursor = db.cursor() # loop through the mips list and load them into the database for key, value in mips.iteritems(): if key not in _exclude_mips: count = 0 sql = "select count(id), id, name, description, dreq_version from t2_cmip6_MIP_types where name = '{0}'".format( key) mip_description = db.escape_string(value) try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) (count, id, name, description, dreq_version) = cursor.fetchone() except: print("Error executing sql = {0}".format(sql)) db.rollback() if count == 1: sql = "update t2_cmip6_MIP_types set name = '{0}', description = '{1}', dreq_version = '{2}' where id = {3}".format( key, mip_description, version, id) try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) db.commit() except: print("Error executing sql = {0}".format(sql)) db.rollback() elif count == 0: sql = "insert into t2_cmip6_MIP_types (name, description, dreq_version) value ('{0}','{1}','{2}')".format( key, mip_description, version) try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) db.commit() except: print("Error executing sql = {0}".format(sql)) db.rollback() else: print( "Error in database {0} rows found matching MIP name '{1}'". format(count, name)) # disconnect from server db.close()
htmlStyle['CMORvar'] = {'getIrefs': ['requestVar']} htmlStyle['requestVarGroup'] = {'getIrefs': ['requestVar', 'requestLink']} htmlStyle['var'] = {'getIrefs': ['CMORvar']} htmlStyle['objective'] = {'getIrefs': ['objectiveLink']} htmlStyle['requestLink'] = {'getIrefs': ['objectiveLink', 'requestItem']} htmlStyle['exptgroup'] = {'getIrefs': ['__all__']} htmlStyle['requestItem'] = {'getIrefs': ['__all__']} htmlStyle['experiment'] = {'getIrefs': ['__all__']} htmlStyle['mip'] = {'getIrefs': ['__all__']} htmlStyle['remarks'] = {'getIrefs': ['__all__']} htmlStyle['varChoice'] = {'getIrefs': ['__all__']} htmlStyle['spatialShape'] = {'getIrefs': ['__all__']} htmlStyle['temporalShape'] = {'getIrefs': ['__all__']} htmlStyle['structure'] = {'getIrefs': ['__all__']} htmlStyle['standardname'] = {'getIrefs': ['__all__']} dq = dreq.loadDreq(htmlStyles=htmlStyle) ## ## add special styles to dq object "itemStyle" dictionary. ## dq.itemStyles['standardname'] = styls.snLink dq.itemStyles['var'] = styls.varLink dq.itemStyles['mip'] = styls.mipLink dq.itemStyles['CMORvar'] = styls.cmvLink dq.itemStyles['objective'] = styls.objLink dq.itemStyles['structure'] = styls.strLink dq.itemStyles['objectiveLink'] = styls.objLnkLink dq.itemStyles['requestVarGroup'] = styls.vgrpLink dq.itemStyles['requestLink'] = styls.rqlLink02 dq.itemStyles['spatialShape'] = styls.labTtl dq.coll['var'].items[0].__class__._linkAttrStyle['sn'] = styls.snLink01
def parse_table(self, exp, mips, tables, v_list, table_var_fields, table_axes_fields, table_fields, user_vars, user_axes, user_tableInfo): """ Function will parse an XML file using dreqPy and return a dictionary containing the parsed fields. Parameter: exp (str): The name of the experiment. miptable (str): The name of the miptable. table_var_fields (dict): Dictionary containing standardized field var names as keys and acceptable synonyms as values. table_axes_fields (dict): Dictionary containing standardized field axes names as keys and acceptable synonyms as values. table_fields (dict): Dictionary containing standardized table field names as keys and acceptable synonyms as values. user_vars (dict): User defined dictionary. Keys should match standard name. Values should be a list of acceptable field names. user_axes (dict): User defined dictionary. Keys should match standard name. Values should be a list of acceptable field names. user_tableInfo (dict): User defined dictionary. Keys should match standard name. Values should be a list of acceptable field names. Returns: table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys: 'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and each variable has a value of a dictionary keyed with the standard field names. """ from dreqPy import dreq dq = dreq.loadDreq() # Get table id # if len(g_id) == 0: # print 'ERROR: Variable group/table ',table_name, ' not supported.' # print 'Please select from the following: ' # print dq.inx.requestVarGroup.label.keys() # print '\nIf your selected table is listed, it may not be supported in this verison of dreqpy.' # print '\nEXITING. \n' # sys.exit(1) # # Get the id's of the variables in this table # g_vars = dq.inx.iref_by_sect[g_id[0]].a # Get a list of mips for the experiment # print sorted(dq.inx.experiment.label.keys()),len(dq.inx.experiment.label.keys()) e_mip = [] e_id = dq.inx.experiment.label[exp] if len(e_id) == 0: print '\033[91m', 'Invalid experiment name. Please choose from the folowing options:', '\033[0m' print sorted(dq.inx.experiment.label.keys()), len( dq.inx.experiment.label.keys()) return {} activity_id = dq.inx.uid[e_id[0]].mip e_vars = dq.inx.iref_by_sect[e_id[0]].a if len(e_vars['requestItem']) == 0: e_vars = dq.inx.iref_by_sect[dq.inx.uid[e_id[0]].egid].a total_request = {} for ri in e_vars['requestItem']: table_info = {} dr = dq.inx.uid[ri] if dr.mip in mips or '--ALL--' in mips: table_dict = {} variables = {} axes = {} table_info = {} data = {} table_info['experiment'] = dq.inx.uid[e_id[0]].title table_info['experiment_id'] = exp table_info['data_specs_version'] = dreq.version table_info['activity_id'] = activity_id rl = dq.inx.requestLink.uid[dr.rlid] rvg = dq.inx.requestVarGroup.uid[rl.refid] vars = dq.inx.iref_by_sect[rl.refid].a axes_list = [] var_list = [] if len(v_list) == 0: var_list = vars['requestVar'] else: for v in v_list: uids = dq.inx.requestVar.label[v] for uid in uids: if uid in vars['requestVar']: var_list.append(uid) for rv in var_list: var = {} v_id = dq.inx.uid[rv].vid # Get the CMORvar id c_var = dq.inx.uid[v_id] # Set what we can from the CMORvar section if hasattr(c_var, 'mipTable'): var['mipTable'] = c_var.mipTable if c_var.mipTable in tables or '--ALL--' in tables: var["_FillValue"] = "1e+20" #var["missing_value"] = "1e+20" if hasattr(c_var, 'deflate'): var['deflate'] = c_var.deflate if hasattr(c_var, 'deflate_level'): var['deflate_level'] = c_var.deflate_level if hasattr(c_var, 'description'): var['description'] = c_var.description if hasattr(c_var, 'frequency'): var['frequency'] = c_var.frequency table_info['frequency'] = c_var.frequency if hasattr(c_var, 'label'): var['id'] = c_var.label var['out_name'] = c_var.label var['variable_id'] = c_var.label l = dq.inx.var.label[c_var.label] if len(l) > 0: var['standard_name'] = dq.inx.var.uid[ l[0]].sn if hasattr(c_var, 'modeling_realm'): var['realm'] = c_var.modeling_realm #if hasattr(c_var,'ok_min_mean_abs'): # var['ok_min_mean_abs']= c_var.ok_min_mean_abs #if hasattr(c_var,'ok_max_mean_abs'): # var['ok_max_mean_abs']= c_var.ok_max_mean_abs if hasattr(c_var, 'out_name'): var['out_name'] = c_var.label #? if hasattr(c_var, 'positive'): var['positive'] = c_var.positive if hasattr(c_var, 'direction'): var['direction'] = c_var.direction if hasattr(c_var, 'prov'): var['prov'] = c_var.prov if hasattr(c_var, 'procNote'): var['provcNote'] = c_var.procNote if hasattr(c_var, 'shuffle'): var['shuffle'] = c_var.shuffle if hasattr(c_var, 'title'): var['title'] = c_var.title var['long_name'] = c_var.title if hasattr(c_var, 'description'): var['comment'] = c_var.description if hasattr(c_var, 'type'): var['type'] = c_var.type if hasattr(c_var, 'valid_max'): if isinstance(c_var.valid_max, (int, long, float, complex)): var['valid_max'] = c_var.valid_max if hasattr(c_var, 'valid_min'): if isinstance(c_var.valid_min, (int, long, float, complex)): var['valid_min'] = c_var.valid_min # Set what we can from the standard section if hasattr(c_var, 'stid'): s_var = dq.inx.uid[c_var.stid] if hasattr(s_var, 'cell_measures'): var['cell_measures'] = s_var.cell_measures if hasattr(s_var, 'cell_methods'): var['cell_methods'] = s_var.cell_methods if hasattr(s_var, 'coords'): #if len(s_var.coords)>0: if isinstance(s_var.cids, list): var['coordinate'] = dq.inx.uid[ s_var.cids[0]].label c = dq.inx.uid[s_var.cids[0]] if c not in axes_list and c != '' and c != 'None': axes_list.append(c) # Set what we can from the time section if hasattr(s_var, 'tmid'): t_var = dq.inx.uid[s_var.tmid] if hasattr(t_var, 'dimensions'): t = t_var.dimensions if t != '' and t != 'None': var['time'] = t var['coordinates'] = t if hasattr(t_var, 'label'): var['time_label'] = t_var.label if hasattr(t_var, 'title'): var['time_title'] = t_var.title # Is there did? if hasattr(s_var, 'dids'): if isinstance(s_var.dids, tuple): extra_dim = dq.inx.uid[s_var.dids[0]].label if 'coordinates' not in var.keys(): var['coordinates'] = extra_dim else: var['coordinates'] = extra_dim + "|" + var[ 'coordinates'] if extra_dim not in axes_list: axes_list.append(extra_dim) # Set what we can from the spatial section if hasattr(s_var, 'spid'): sp_var = dq.inx.uid[s_var.spid] if hasattr(sp_var, 'dimensions'): if len(sp_var.dimensions) > 1: if 'coordinates' in var.keys(): sp_var_d = sp_var.dimensions if len(sp_var_d) > 1: if sp_var_d[-1] == '|': sp_var_d = sp_var_d[:-1] var['coordinates'] = sp_var_d + '|' + var[ 'coordinates'] else: var['coordinates'] = sp_var.dimensions if 'grid_latitude' in var[ 'coordinates']: var['coordinates'] = var[ 'coordinates'].replace( 'grid_latitude', 'gridlatitude') dims = var['coordinates'].split('|') for d in dims: if d not in axes_list and d != '' and d != 'None': if 'copy' not in var[ 'id'] and '?' not in d: axes_list.append(d) if 'coordinates' in var.keys(): if len(var['coordinates']) > 1: if var['coordinates'][-1] == '|': var['coordinates'] = var[ 'coordinates'][:-1] # Set what we can from the variable section if hasattr(c_var, 'vid'): v_var = dq.inx.uid[c_var.vid] if hasattr(v_var, 'cf_standard_name'): var['cf_standard_name'] = v_var.sn if hasattr(v_var, 'long_name'): var['long_name'] = v_var.sn if hasattr(v_var, 'units'): if v_var.units == "": var['units'] = 'None' print c_var.label, " does not have units" else: var['units'] = v_var.units # Add variable to variable dictionary variables[c_var.label] = var for a in axes_list: if 'grid_latitude' in a: a = 'gridlatitude' if a in dq.inx.grids.label.keys(): id = dq.inx.grids.label[a] if len(id) > 0: v = dq.inx.grids.uid[id[0]] ax = {} if hasattr(v, 'units'): if v.units == "": ax['units'] = '1' else: ax['units'] = v.units if hasattr(v, 'value'): ax['value'] = v.value if hasattr(v, 'axis'): ax['axis'] = v.axis if hasattr(v, 'valid_max'): if isinstance(v.valid_max, (int, long, float, complex)): ax['valid_max'] = v.valid_max if hasattr(v, 'valid_min'): if isinstance(v.valid_min, (int, long, float, complex)): ax['valid_min'] = v.valid_min if hasattr(v, 'standardName'): if isinstance(v.standardName, (str)): ax['standard_name'] = v.standardName else: ax['standard_name'] = a if hasattr(v, 'type'): if 'landUse' in a: ax['type'] = 'int' else: ax['type'] = v.type if hasattr(v, 'id'): ax['id'] = v.label if hasattr(v, 'positive'): ax['positive'] = v.positive if hasattr(v, 'direction'): ax['direction'] = v.direction if hasattr(v, 'title'): ax['title'] = v.title if hasattr(v, 'bounds'): if 'yes' in v.bounds: ax['bounds'] = v.label + "_bnds" if hasattr(v, 'requested'): ax['requested'] = v.requested #if hasattr(v,'boundsValues'): # ax['boundsValues'] = v.boundsValues if hasattr(v, 'coords'): ax['coords'] = v.coords axes[a] = ax else: v = a print "Cannot find link for dimension: ", v try: table_dict['variables'] = variables table_dict['axes'] = axes except UnboundLocalError: print 'Does not fit criteria: ', exp table_dict['table_info'] = table_info tab = rvg.label table_info['table_id'] = tab total_request[dr.mip + '_' + tab] = table_dict #print 'Total in request:',len(total_request) #for k in sorted(total_request.keys()): # v = total_request[k] # print k, len(v['variables']) return total_request
import json, urllib import collections import urllib.request import shelve from dreqPy import dreq dq = dreq.loadDreq() ss = set() for i in dq.coll['CMORvar'].items: if i.mipTable == 'Amon' and i.defaultPriority == 1: ss.add((i.mipTable, i.label)) temp = 'https://esgf-index1.ceda.ac.uk/esg-search/search/?offset=0&limit=500&type=Dataset&replica=false&latest=true&project%%21=input4mips&activity_id=CMIP&table_id=%(table_label)s&mip_era=CMIP6&variable_id=%(variable_label)s&facets=mip_era%%2Cactivity_id%%2Cmodel_cohort%%2Cproduct%%2Csource_id%%2Cinstitution_id%%2Csource_type%%2Cnominal_resolution%%2Cexperiment_id%%2Csub_experiment_id%%2Cvariant_label%%2Cgrid_label%%2Ctable_id%%2Cfrequency%%2Crealm%%2Cvariable_id%%2Ccf_standard_name%%2Cdata_node&format=application%%2Fsolr%%2Bjson' selection = "activity_id=CMIP&table_id=%(table_label)s&mip_era=CMIP6&variable_id=%(variable_label)s" sdict = dict(table_id='Amon', experiment_id='historical', variable_id='tas') temp2 = 'https://esgf-index1.ceda.ac.uk/esg-search/search/?offset=0&limit=500&type=Dataset&replica=false&latest=true&project%%21=input4mips&%(selection)s&facets=mip_era%%2Cactivity_id%%2Cmodel_cohort%%2Cproduct%%2Csource_id%%2Cinstitution_id%%2Csource_type%%2Cnominal_resolution%%2Cexperiment_id%%2Csub_experiment_id%%2Cvariant_label%%2Cgrid_label%%2Ctable_id%%2Cfrequency%%2Crealm%%2Cvariable_id%%2Ccf_standard_name%%2Cdata_node&format=application%%2Fsolr%%2Bjson' tmp = dict( Amon= 'tas, pr, uas, vas, huss, rsut, rsdt, rlut, rsus, rsds, rlus, rlds, ps, ua, va, zg, cl, clt' .split(', '), AERmon='od550aer, abs550aer'.split(', '), Lmon='cVeg, cLitter, gpp, nbp, npp'.split(', '), Omon=[ 'fgco2', ], Emon=[ 'cSoil',
def loadDreq(self, dq=None): if dq == None: dq = dreq.loadDreq() collect.add(dq) self.__extra__['dq'] = dq
def load_from_dqroot(dqroot=None): (dqxml, dqconfig) = reroot_paths(dqroot, (defaultDreqPath, defaultConfigPath)) return loadDreq(dreqXML=dqxml, configdoc=dqconfig)
def main(options): """ main Arguments: options (list) - input options from command line """ debug = options.debug # create a db connection and cursor db = MySQLdb.connect(host="localhost", user="******", passwd="c$3gdb", db="csegdb") cursor = db.cursor() cursor1 = db.cursor() # load a data request object dq = dreq.loadDreq() version = dq.version # loop through the list of experiments and get the associated requesting MIPs sql = "select id, name, design_mip from t2_cmip6_exps" try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) except: print("Error executing sql = {0}".format(sql)) sys.exit(1) for row in cursor: # get the design_mip_id from the t2_cmip6_MIP_types count1 = 0 sql = "select count(id), id from t2_cmip6_MIP_types where name = '{0}'".format( row[2]) try: print("Executing sql = {0}".format(sql)) cursor1.execute(sql) (count1, design_mip_id) = cursor1.fetchone() except: print("Error executing sql = {0}".format(sql)) db.rollback() if count1 == 1: # update / insert into the t2j_cmip6 table sql = "select count(exp_id) from t2j_cmip6 where exp_id = {0}".format( row[0]) try: print("Executing sql = {0}".format(sql)) cursor1.execute(sql) count2 = cursor1.fetchone() except: print("Error executing sql = {0}".format(sql)) db.rollback() if count2[0] == 0: sql = "insert into t2j_cmip6 (exp_id, design_mip_id) value ({0},{1})".format( row[0], design_mip_id) try: print("Executing sql = {0}".format(sql)) cursor1.execute(sql) db.commit() except: print("Error executing sql = {0}".format(sql)) db.rollback() if count2[0] == 1: sql = "update t2j_cmip6 set design_mip_id = {0} where exp_id = {1}".format( design_mip_id, row[0]) try: print("Executing sql = {0}".format(sql)) cursor1.execute(sql) db.commit() except: print("Error executing sql = {0}".format(sql)) db.rollback() # loop through the list of requesting mips and update or insert in the join table reqMips = map_exp_to_request_mip(row[1], dq=dq) for rm in reqMips: count2 = 0 sql = "select count(id), id from t2_cmip6_MIP_types where name = '{0}'".format( rm) try: print("Executing sql = {0}".format(sql)) cursor1.execute(sql) (count2, MIPid) = cursor1.fetchone() except: print("Error executing sql = {0}".format(sql)) db.rollback() if count2 == 1: # check if record already exists in join table count3 = 0 sql = "select count(exp_id), exp_id from t2j_cmip6_exps_mips where exp_id = {0} and mip_id = {1}".format( row[0], MIPid) try: print("Executing sql = {0}".format(sql)) cursor1.execute(sql) (count3, exp_id) = cursor1.fetchone() except: print("Error executing sql = {0}".format(sql)) db.rollback() if count3 == 0: # insert a new record sql = "insert into t2j_cmip6_exps_mips (exp_id, mip_id) value ({0}, {1})".format( row[0], MIPid) try: print("Executing sql = {0}".format(sql)) cursor1.execute(sql) db.commit() except: print("Error executing sql = {0}".format(sql)) db.rollback() else: print("Error there are {0} MIPS matching name {1}".format( count2, rm)) # add the CMIP DECK entries to the exps and join tables join_DECK_exps(db, cursor, version) # disconnect from server db.close()
def parse_table( self, exp, mips, tables, v_list, table_var_fields, table_axes_fields, table_fields, user_vars, user_axes, user_tableInfo, ): """ Function will parse an XML file using dreqPy and return a dictionary containing the parsed fields. Parameter: exp (str): The name of the experiment. miptable (str): The name of the miptable. table_var_fields (dict): Dictionary containing standardized field var names as keys and acceptable synonyms as values. table_axes_fields (dict): Dictionary containing standardized field axes names as keys and acceptable synonyms as values. table_fields (dict): Dictionary containing standardized table field names as keys and acceptable synonyms as values. user_vars (dict): User defined dictionary. Keys should match standard name. Values should be a list of acceptable field names. user_axes (dict): User defined dictionary. Keys should match standard name. Values should be a list of acceptable field names. user_tableInfo (dict): User defined dictionary. Keys should match standard name. Values should be a list of acceptable field names. Returns: table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys: 'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and each variable has a value of a dictionary keyed with the standard field names. """ from dreqPy import dreq dq = dreq.loadDreq() # Get table id # if len(g_id) == 0: # print 'ERROR: Variable group/table ',table_name, ' not supported.' # print 'Please select from the following: ' # print dq.inx.requestVarGroup.label.keys() # print '\nIf your selected table is listed, it may not be supported in this verison of dreqpy.' # print '\nEXITING. \n' # sys.exit(1) # # Get the id's of the variables in this table # g_vars = dq.inx.iref_by_sect[g_id[0]].a # Get a list of mips for the experiment # print sorted(dq.inx.experiment.label.keys()),len(dq.inx.experiment.label.keys()) # e_mip = [] if "--ALL--" in exp: e_id = dq.inx.experiment.label["historical"] else: e_id = dq.inx.experiment.label[exp] if len(e_id) == 0: # print '\033[91m','Invalid experiment name. Please choose from the folowing options:','\033[0m' # print sorted(dq.inx.experiment.label.keys()),len(dq.inx.experiment.label.keys()) # return {} print( "\033[91m", "This experiment name is not registered within the data request.", "\033[0m", ) print( "\033[91m", "Using a generic exp name instead. If you think this name should be registered, verify the name is correct from the list below.", "\033[0m", ) print( sorted(dq.inx.experiment.label.keys()), len(dq.inx.experiment.label.keys()), ) e_id = dq.inx.experiment.label["historical"] activity_id = dq.inx.uid[e_id[0]].mip e_vars = [] activity_id = dq.inx.uid[e_id[0]].mip e_vars.append(dq.inx.iref_by_sect[e_id[0]].a) e_vars.append(dq.inx.iref_by_sect[dq.inx.uid[e_id[0]].egid].a) e_vars.append(dq.inx.iref_by_sect[activity_id].a) total_request = {} for e_var in e_vars: for ri in e_var["requestItem"]: table_info = {} dr = dq.inx.uid[ri] if dr.mip in mips or "--ALL--" in mips or "--ALL--" in exp: table_dict = {} variables = {} axes = {} table_info = {} # data = {} table_info["data_specs_version"] = dreq.version if "--ALL--" not in exp: table_info["experiment"] = dq.inx.uid[e_id[0]].title table_info["experiment_id"] = exp table_info["activity_id"] = activity_id rl = dq.inx.requestLink.uid[dr.rlid] rvg = dq.inx.requestVarGroup.uid[rl.refid] vars = dq.inx.iref_by_sect[rl.refid].a axes_list = [] var_list = [] if len(v_list) == 0: var_list = vars["requestVar"] else: # Make a list of all cmip vars and their id's all_vars = {} for v in dq.coll["CMORvar"].items: all_vars[v.label + ":" + v.mipTable] = v.uid for v in v_list: if v in all_vars.keys(): var_list.append(all_vars[v]) else: print("Can not add variable: " + v) for rv in var_list: var = {} if len(v_list) == 0: v_id = dq.inx.uid[rv].vid # Get the CMORvar id else: v_id = dq.inx.uid[rv].uid # Get the CMORvar id c_var = dq.inx.uid[v_id] # Set what we can from the CMORvar section if hasattr(c_var, "mipTable"): var["mipTable"] = c_var.mipTable if c_var.mipTable in tables or "--ALL--" in tables: var["_FillValue"] = "1.0E20" var["missing_value"] = 1.0e20 if hasattr(c_var, "deflate"): var["deflate"] = c_var.deflate if hasattr(c_var, "deflate_level"): var["deflate_level"] = c_var.deflate_level if hasattr(c_var, "description"): var["description"] = c_var.description if hasattr(c_var, "frequency"): var["frequency"] = c_var.frequency table_info["frequency"] = c_var.frequency if hasattr(c_var, "label"): var["id"] = c_var.label var["out_name"] = c_var.label var["variable_id"] = dq.inx.uid[ c_var.vid].label # l = dq.inx.var.label[c_var.label] # if len(l)>0: var["standard_name"] = dq.inx.uid[ c_var.vid].sn if hasattr(c_var, "modeling_realm"): var["realm"] = c_var.modeling_realm # if hasattr(c_var,'ok_min_mean_abs'): # var['ok_min_mean_abs']= c_var.ok_min_mean_abs # if hasattr(c_var,'ok_max_mean_abs'): # var['ok_max_mean_abs']= c_var.ok_max_mean_abs if hasattr(c_var, "out_name"): var["out_name"] = c_var.label if hasattr(c_var, "positive"): var["positive"] = c_var.positive if hasattr(c_var, "direction"): var["direction"] = c_var.direction if hasattr(c_var, "prov"): var["prov"] = c_var.prov if hasattr(c_var, "procNote"): var["provcNote"] = c_var.procNote if hasattr(c_var, "shuffle"): var["shuffle"] = c_var.shuffle if hasattr(c_var, "title"): var["title"] = c_var.title var["long_name"] = c_var.title # if hasattr(c_var,'description'): # var['comment']= c_var.description if hasattr(c_var, "type"): var["type"] = c_var.type if hasattr(c_var, "valid_max"): if isinstance(c_var.valid_max, (int, float, complex)): var["valid_max"] = c_var.valid_max if hasattr(c_var, "valid_min"): if isinstance(c_var.valid_min, (int, float, complex)): var["valid_min"] = c_var.valid_min # Set what we can from the standard section if hasattr(c_var, "stid"): s_var = dq.inx.uid[c_var.stid] if hasattr(s_var, "cell_measures"): var["cell_measures"] = s_var.cell_measures if hasattr(s_var, "cell_methods"): var["cell_methods"] = s_var.cell_methods if hasattr(s_var, "coords"): if isinstance(s_var.cids, list): var["coordinates"] = dq.inx.uid[ s_var.cids[0]].label c = dq.inx.uid[s_var.cids[0]] if (c not in axes_list and c != "" and c != "None"): axes_list.append(c) # Set what we can from the time section if hasattr(s_var, "tmid"): t_var = dq.inx.uid[s_var.tmid] if hasattr(t_var, "dimensions"): t = t_var.dimensions if t != "" and t != "None": var["time"] = t var["coordinates"] = t if hasattr(t_var, "label"): var["time_label"] = t_var.label if hasattr(t_var, "title"): var["time_title"] = t_var.title # Is there did? if hasattr(s_var, "dids"): if isinstance(s_var.dids, tuple): extra_dim = dq.inx.uid[ s_var.dids[0]].label if "coordinates" not in var.keys(): var["coordinates"] = extra_dim else: var["coordinates"] = ( extra_dim + "|" + var["coordinates"]) if extra_dim not in axes_list: axes_list.append(extra_dim) # Set what we can from the spatial section if hasattr(s_var, "spid"): sp_var = dq.inx.uid[s_var.spid] if hasattr(sp_var, "dimensions"): if len(sp_var.dimensions) > 1: if "coordinates" in var.keys(): sp_var_d = sp_var.dimensions if len(sp_var_d) > 1: if sp_var_d[-1] == "|": sp_var_d = sp_var_d[: -1] var["coordinates"] = ( sp_var_d + "|" + var["coordinates"]) else: var["coordinates"] = sp_var.dimensions if ("grid_latitude" in var["coordinates"]): var["coordinates"] = var[ "coordinates"].replace( "grid_latitude", "gridlatitude") dims = var[ "coordinates"].split("|") for d in dims: if (d not in axes_list and d != "" and d != "None"): if ("copy" not in var["id"] and "?" not in d): axes_list.append(d) if "coordinates" in var.keys(): if len(var["coordinates"]) > 1: if var["coordinates"][-1] == "|": var["coordinates"] = var[ "coordinates"][:-1] # Set what we can from the variable section if hasattr(c_var, "vid"): v_var = dq.inx.uid[c_var.vid] if hasattr(v_var, "standard_name"): var["standard_name"] = v_var.sn if hasattr(v_var, "cf_standard_name"): var["cf_standard_name"] = v_var.sn if hasattr(v_var, "long_name"): var["long_name"] = v_var.sn if hasattr(v_var, "units"): if v_var.units == "": var["units"] = "None" print(c_var.label, " does not have units") else: var["units"] = v_var.units # Add variable to variable dictionary variables[c_var.label] = var for a in axes_list: if "grid_latitude" in a: a = "gridlatitude" if a in dq.inx.grids.label.keys(): id = dq.inx.grids.label[a] if len(id) > 0: v = dq.inx.grids.uid[id[0]] ax = {} if hasattr(v, "units"): if v.units == "": ax["units"] = "1" else: ax["units"] = v.units if hasattr(v, "value"): ax["value"] = v.value if hasattr(v, "axis"): ax["axis"] = v.axis if hasattr(v, "valid_max"): if isinstance(v.valid_max, (int, float, complex)): ax["valid_max"] = v.valid_max if hasattr(v, "valid_min"): if isinstance(v.valid_min, (int, float, complex)): ax["valid_min"] = v.valid_min if hasattr(v, "standardName"): if isinstance(v.standardName, (str)): ax["standard_name"] = v.standardName else: ax["standard_name"] = a if hasattr(v, "type"): if "landUse" in a: ax["type"] = "int" else: ax["type"] = v.type if hasattr(v, "id"): ax["id"] = v.label if hasattr(v, "positive"): ax["positive"] = v.positive if hasattr(v, "direction"): ax["direction"] = v.direction if hasattr(v, "title"): ax["title"] = v.title if hasattr(v, "bounds"): if "yes" in v.bounds: ax["bounds"] = v.label + "_bnds" if hasattr(v, "requested"): ax["requested"] = v.requested if hasattr(v, "altLabel"): ax["altLabel"] = v.altLabel # if hasattr(v,'boundsValues'): # ax['boundsValues'] = v.boundsValues if hasattr(v, "coords"): ax["coords"] = v.coords axes[a] = ax else: v = a print("Cannot find link for dimension: ", v) try: table_dict["variables"] = variables table_dict["axes"] = axes except UnboundLocalError: print("Does not fit criteria: ", exp) table_dict["table_info"] = table_info tab = rvg.label table_info["table_id"] = tab total_request[dr.mip + "_" + tab] = table_dict # print 'Total in request:',len(total_request) # for k in sorted(total_request.keys()): # v = total_request[k] # print k, len(v['variables']) return total_request
def main(options): """ main Arguments: options (list) - input options from command line """ debug = options.debug # create a db connection and cursor db = MySQLdb.connect(host="localhost", user="******", passwd="c$3gdb", db="csegdb") cursor = db.cursor() # load a data request object dq = dreq.loadDreq() version = dq.version # define a lookup data structures for MIP experiments that are CESM specific ## mip_exps = { 'DAMIP' : ['hist-nat-WACCM', 'hist-GHG-WACCM'], ## 'ScenarioMIP' : ['ssp585-WACCM', 'ssp370-WACCM', 'ssp245-WACCM', 'ssp126-WACCM', 'ssp534-over-WACCM'], ## 'RFMIP' : ['piClim-control-WACCM'] } mip_exps = { 'ScenarioMIP': ['ssp126-ext-WACCM', 'ssp585-ext-WACCM', 'ssp534-over-ext-WACCM'] } for mip, exps in mip_exps.iteritems(): # get the design_mip_id from the t2_cmip6_MIP_types sql = "select id from t2_cmip6_MIP_types where name = '{0}'".format( mip) try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) (design_mip_id) = cursor.fetchone() except: print("Error executing sql = {0}".format(sql)) db.rollback() # loop over the exps for this MIP for exp in exps: # get the CMIP6 exp UID which will be the same for these new experiments exp_tmp = exp.replace('-ext-WACCM', '') exp_tmp = exp_tmp.replace('-over-ext-WACCM', '') sql = "select uid, description from t2_cmip6_exps where name = '{0}'".format( exp_tmp) try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) (uid, description) = cursor.fetchone() except: print("Error executing sql = {0}".format(sql)) db.rollback() # check if this exp is already in the t2_cmip6_exps table exp_id = 0 sql = "select count(id), id from t2_cmip6_exps where name = '{0}'".format( exp) try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) (exists, exp_id) = cursor.fetchone() except: print("Error executing sql = {0}".format(sql)) db.rollback() if not exists: # insert a row into the t2_cmip6_exps sql = "insert into t2_cmip6_exps (name, description, uid, design_mip, dreq_version) value ('{0}','{1}','{2}','{3}','{4}')".format( exp, description, uid, mip, version) try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) db.commit() except: print("Error executing sql = {0}".format(sql)) db.rollback() # get the id for the exp just inserted sql = "select id from t2_cmip6_exps where name = '{0}'".format( exp) try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) (id) = cursor.fetchone() db.commit() except: print("Error executing sql = {0}".format(sql)) db.rollback() # reset the exp_id to this new experiment exp_id = id[0] # check if record already exists in join table sql = "select count(exp_id) from t2j_cmip6 where exp_id = {0}".format( exp_id) try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) (count) = cursor.fetchone() except: print("Error executing sql = {0}".format(sql)) db.rollback() if count[0] == 0: # insert a new join record sql = "insert into t2j_cmip6 (exp_id, design_mip_id) value ({0},{1})".format( exp_id, design_mip_id[0]) try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) db.commit() except: print("Error executing sql = {0}".format(sql)) db.rollback() if count[0] == 1: # update record with deck_id sql = "update t2j_cmip6 set design_mip_id = {0} where exp_id = {1}".format( design_mip_id[0], exp_id) try: print("Executing sql = {0}".format(sql)) cursor.execute(sql) db.commit() except: print("Error executing sql = {0}".format(sql)) db.rollback() # disconnect from server db.close() return 0
def _get_collection(key): dq = dreq.loadDreq() return dq.coll[key].items