def _ch03_linkCheck(self):
try:
import dreq
except:
from . import dreq
nn = 0
self.dq = dreq.loadDreq(manifest='%s/dreqManifest.txt' % self.docdir)
for section in self.dq.coll:
ks = [
k for k in self.dq.coll[section].attDefn.keys()
if self.dq.coll[section].attDefn[k].useClass == 'internalLink'
]
nerr = 0
cc = collections.defaultdict(int)
for i in self.dq.coll[section].items:
for k in ks:
if k in i.__dict__:
if i.__dict__[k] not in self.dq.inx.uid:
nerr += 1
cc[k] += 1
print('Bad link found: section: %s: %s %s [%s]' %
(section, k, i.__dict__[k], i.uid))
if nerr > 0:
msg = ''
for k in cc:
msg += '%s: %s; ' % (k, cc[k])
print('Section %s: bad links: %s %s' % (section, nerr, msg))
nn += nerr
##print section, ks, nerr
if nn == 0:
print('Dreq links checked')
self.ok = nn == 0
def _ch02_importSample(self):
try:
import dreq
except:
from . import dreq
self.dq = dreq.loadDreq(manifest='%s/dreqManifest.txt' % self.docdir)
print('Dreq sample load checked')
self.ok = True
def run():
try:
import makeTables
import scope
except:
import dreqPy.scope as scope
import dreqPy.makeTables as makeTables
assert os.path.isdir(
'html'
), 'Before running this script you need to create "html", "html/index" and "html/u" sub-directories, or edit the call to dq.makeHtml'
assert os.path.isdir(
'html/u'
), 'Before running this script you need to create "html", "html/index" and "html/u" sub-directories, or edit the call to dq.makeHtml, and refernces to "u" in style lines below'
assert os.path.isdir(
'html/index'
), 'Before running this script you need to create "html", "html/index" and "html/u" sub-directories, or edit the call to dq.makeHtml, and refernces to "u" in style lines below'
assert os.path.isdir(
'tables'
), 'Before running this script you need to create a "tables" sub-directory, or edit the table_utils.makeTab class'
dq = dreq.loadDreq(htmlStyles=htmlStyle, manifest='out/dreqManifest.txt')
##
## add special styles to dq object "itemStyle" dictionary.
##
dq.itemStyles['standardname'] = styls.snLink
dq.itemStyles['var'] = styls.varLink
dq.itemStyles['mip'] = styls.mipLink
dq.itemStyles['CMORvar'] = styls.cmvLink
dq.itemStyles['objective'] = styls.objLink
dq.itemStyles['units'] = styls.unitLink
dq.itemStyles['structure'] = styls.strLink
dq.itemStyles['cellMethods'] = styls.cmLink
dq.itemStyles['remarks'] = styls.remarkLink
dq.itemStyles['exptgroup'] = styls.baseLink01
dq.itemStyles['objectiveLink'] = styls.objLnkLink
dq.itemStyles['requestVarGroup'] = styls.vgrpLink
dq.itemStyles['miptable'] = styls.miptableLink
dq.itemStyles['requestLink'] = styls.rqlLink02
dq.itemStyles['requestItem'] = styls.rqiLink02
dq.itemStyles['spatialShape'] = styls.labTtl
dq.indexSortBy['miptable'] = 'label'
dq.coll['var'].items[0].__class__._linkAttrStyle['sn'] = styls.snLink01
dq.coll['CMORvar'].items[0].__class__._linkAttrStyle[
'stid'] = styls.stidLink01
##dq.coll['requestVarGroup'].items[0].__class__._linkAttrStyle['requestVar'] = styls.rqvLink01
dq.itemStyles['requestVar'] = styls.rqvLink01
dreq.dreqItemBase._extraHtml['requestVarGroup'] = rvgExtraTable.vgx1(
dq).mxoGet
dreq.dreqItemBase.__charmeEnable__['var'] = nt__charmeEnable(
'test', 'http://clipc-services.ceda.ac.uk/dreq')
ht = htmlTrees(dq)
dq.makeHtml(annotations={'var': ht.anno},
ttl0='Data Request [%s]' % dreq.version)
sc = scope.dreqQuery(dq=dq)
try:
mt = table_utils.makeTab(sc)
except:
print('Could not make tables ...')
raise
mp = makePurl(dq)
mj = makeJs(dq)
def parse_table(self, table_name,table_var_fields,table_axes_fields,table_fields,
user_vars,user_axes,user_tableInfo):
"""
Function will parse an XML file using dreqPy and return a dictionary containing
the parsed fields.
Parameter:
table_name (str): The full path to the table to be parsed.
table_var_fields (dict): Dictionary containing standardized field var names
as keys and acceptable synonyms as values.
table_axes_fields (dict): Dictionary containing standardized field axes names
as keys and acceptable synonyms as values.
table_fields (dict): Dictionary containing standardized table field names
as keys and acceptable synonyms as values.
user_vars (dict): User defined dictionary. Keys should match standard name.
Values should be a list of acceptable field names.
user_axes (dict): User defined dictionary. Keys should match standard name.
Values should be a list of acceptable field names.
user_tableInfo (dict): User defined dictionary. Keys should match standard name.
Values should be a list of acceptable field names.
Returns:
table_dict (dict): A dictionary that holds all of the parsed table information. Contains the keys:
'variables', 'axes', and 'table_info'. Each of these are dictionaries, keyed with variable names and
each variable has a value of a dictionary keyed with the standard field names.
"""
import dreq
table_dict = {}
variables = {}
axes = {}
table_info = {}
dq = dreq.loadDreq()
# Get table id
g_id = dq.inx.requestVarGroup.label[table_name]
# Get the id's of the variables in this table
g_vars = dq.inx.iref_by_sect[g_id[0]].a
# Loop through the variables and set their values
for v in g_vars['requestVar']:
var = {}
v_id = dq.inx.uid[v].vid # Get the CMORvar id
c_var = dq.inx.uid[v_id]
# Set what we can from the CMORvar section
#var['comment']= c_var.comment
var['deflate']= c_var.deflate
var['deflate_level']= c_var.deflate_level
var['description']= c_var.description
#var['flag_meanings']= c_var.flag_meanings
#var['flag_values']= c_var.flag_values
var['frequency']= c_var.frequency
var['id']= c_var.label
var['modeling_realm']= c_var.modeling_realm
var['ok_min_mean_abs']= c_var.ok_min_mean_abs
var['ok_max_mean_abs']= c_var.ok_max_mean_abs
var['out_name']= c_var.label #?
var['positive']= c_var.positive
var['prov']= c_var.prov
var['provNote']= c_var.provNote
var['shuffle']= c_var.shuffle
var['title']= c_var.title
var['type']= c_var.type
var['valid_max']= c_var.valid_max
var['valid_min']= c_var.valid_min
# Set what we can from the standard section
s_var = dq.inx.uid[c_var.stid]
var['cell_measures']= s_var.cell_measures
var['cell_methods']= s_var.cell_methods
# Set what we can from the sp section
sp_var = dq.inx.uid[s_var.spid]
var['dimensions']= sp_var.dimensions
# Set what we can from the variable section
v_var = dq.inx.uid[c_var.vid]
var['cf_standard_name']= v_var.sn
var['long_name']= v_var.sn
var['units']= v_var.units
#var['ext_cell_measures']=
#var['required']=
# Add variable to variable dictionary
variables[c_var.label] = var
table_dict['variables'] = variables
return table_dict
if r._h.label == 'exptgroup':
if 'experiment' in dq.inx.iref_by_sect[e].a:
r = dq.inx.uid[ dq.inx.iref_by_sect[e].a['experiment'][0] ]
else:
ei = dq.inx.uid[e]
if self.errorMode != 'aggregate':
print ( 'ERROR.exptgroup.00001: empty experiment group: %s: %s' % (ei.label, ei.title) )
self.errorLog['ERROR.exptgroup.00001'].add( 'empty experiment group: %s: %s' % (ei.label, ei.title) )
if r._h.label in [ 'remarks','exptgroup']:
##print 'WARNING: link to remarks encountered'
pass
else:
assert r._h.label == 'experiment', 'LOGIC ERROR ... should have an experiment record here: %s' % r._h.label
mips.add(r.mip)
if byBoth:
return (mips0,mips)
else:
return mips
else:
return mips0
if __name__ == '__main__':
try:
import dreq
except:
import dreqPy.dreq as dreq
dq = dreq.loadDreq()
c = checkVar(dq)
c.chk( 'tas' )
print ( '%s, %s' % (c.inc, c.missing))