def derive_variable(ds):
""" Assuming that variable of DS is a derived variable, returns the CliMAF object
representing the operation needed to compute it (using information in dict
operators.derived_variable
"""
if not isinstance(ds,classes.cdataset):
raise Climaf_Driver_Error("arg is not a dataset")
if not operators.is_derived_variable(ds.variable,ds.project) :
raise Climaf_Driver_Error("%s is not a derived variable"%ds.variable)
op,outname,inVarNames,params=operators.derived_variable(ds.variable,ds.project)
inVars=[]
for varname in inVarNames :
dic=ds.kvp.copy()
dic['variable']=varname
inVars.append(classes.cdataset(**dic))
father=capply(op,*inVars,**params)
if (outname == "out"): rep=father
else : rep=scriptChild(father,outname)
rep.variable=ds.variable
return rep
def ceval(cobject, userflags=None, format="MaskedArray",
deep=None, derived_list=[], recurse_list=[]) :
"""
Actually evaluates a CliMAF object, either as an in-memory data structure or
as a string of filenames (which either represent a superset or exactly includes
the desired data)
- with arg deep=True , re-evaluates all components
- with arg deep=False, re-evaluates top level operation
- without arg deep , use cached values as far as possible
arg derived_list is the list of variables that have been considered as 'derived'
(i.e. not natives) in upstream evaluations. It avoids to loop endlessly
"""
if format != 'MaskedArray' and format != 'file' and format != 'png' :
raise Climaf_Driver_Error('Allowed formats yet are : "object", "file" and "png"')
#
if userflags is None : userflags=operators.scriptFlags()
#
# Next check is too crude for dealing with use of operator 'select'
#if cobject.crs in recurse_list :
# clogger.critical("INTERNAL ERROR : infinite loop on object: "+cobject.crs)
# return None
cindent()
if isinstance(cobject,classes.cdataset):
recurse_list.append(cobject.crs)
clogger.debug("Evaluating dataset operand " + cobject.crs + "having kvp=" + `cobject.kvp`)
ds=cobject
if ds.isLocal() or ds.isCached() :
clogger.debug("Dataset %s is local or cached "%ds )
# if the data is local, then
# if the user can select the data and aggregate time, and requested format is
# 'file' return the filenames
# else : read the data, create a cache file for that, and recurse
# Go to derived variable evaluation if appicable
#if ds.variable in operators.derived_variables and not ds.hasRawVariable() :
if operators.is_derived_variable(ds.variable,ds.project) :
if ds.variable in derived_list :
raise Climaf_Driver_Error("Loop detected while evaluating"
"derived variable "+ ds.variable + " " + `derived_list`)
derived=derive_variable(ds)
clogger.debug("evaluating derived variable %s as %s"%\
(ds.variable,`derived`))
derived_value=ceval(derived, format=format, deep=deep,
userflags=userflags,
derived_list=derived_list+[ds.variable],
recurse_list=recurse_list)
if derived_value :
clogger.debug("succeeded in evaluating derived variable %s as %s"%\
(ds.variable,`derived`))
set_variable(derived_value, ds.variable, format=format)
cdedent()
return(derived_value)
elif ((userflags.canSelectVar or ds.oneVarPerFile() ) and \
(userflags.canSelectTime or ds.periodIsFine() ) and \
(userflags.canSelectDomain or ds.domainIsFine() ) and \
(userflags.canAggregateTime or ds.periodHasOneFile()) and \
(userflags.canAlias or ds.hasExactVariable()) and \
(userflags.canMissing or ds.missingIsOK()) and \
#(userflags.doSqueezeMembers or ds.hasOneMember()) and
(format == 'file')) :
clogger.debug("Delivering file set or sets is OK for the target use")
cdedent()
rep=ds.baseFiles()
if not rep : raise Climaf_Driver_Error("No file found for %s"%`ds`)
return(rep) # a single string with all filenames,
#or a list of such strings in case of ensembles
else:
clogger.debug("Must subset and/or aggregate and/or select "+
"var from data files and/or get data, or provide object result")
## extract=cread(ds)
## clogger.debug(" Done with subsetting and caching data files")
## cstore(extract) # extract should include the dataset def
## return ceval(extract,userflags,format)
clogger.debug("Fetching/selection/aggregation is done using an external script for now - TBD")
extract=capply('select',ds)
if extract is None :
raise Climaf_Driver_Error("Cannot access dataset" + `ds`)
rep=ceval(extract,userflags=userflags,format=format)
userflags.unset_selectors()
cdedent()
return rep
else :
# else (non-local and non-cached dataset)
# if the user can access the dataset by one of the dataset-specific protocols
# then assume it can also select on time and provide it with the address
# else : fetch the relevant selection of the data, and store it in cache
clogger.debug("Dataset is remote " )
if (userflags.canOpenDap and format == 'file' ) :
clogger.debug("But user can OpenDAP " )
cdedent()
return(ds.adressOf())
else :
clogger.debug("Must remote read and cache " )
rep=ceval(capply('remote_select',ds),userflags=userflags,format=format)
userflags.unset_selectors()
cdedent()
return rep
#
elif isinstance(cobject,classes.ctree) or isinstance(cobject,classes.scriptChild) or \
isinstance(cobject,classes.cpage) or isinstance(cobject,classes.cens) :
recurse_list.append(cobject.buildcrs())
clogger.debug("Evaluating compound object : " + `cobject`)
#################################################################
if (deep is not None) : cache.cdrop(cobject.crs)
#
clogger.debug("Searching cache for exact object : " + `cobject`)
#################################################################
filename=cache.hasExactObject(cobject)
#filename=None
if filename :
clogger.info("Object found in cache: %s is at %s: "%(cobject.crs,filename))
cdedent()
if format=='file' : return filename
else: return cread(filename,varOf(cobject))
if not isinstance(cobject,classes.cpage) and not isinstance(cobject,classes.cens) :
#
clogger.debug("Searching cache for including object for : " + `cobject`)
########################################################################
it,altperiod=cache.hasIncludingObject(cobject)
#clogger.debug("Finished with searching cache for including object for : " + `cobject`)
#it=None
if it :
clogger.info("Including object found in cache : %s"%(it.crs))
clogger.info("Selecting "+`cobject`+" out of it")
# Just select (if necessary for the user) the portion relevant to the request
rep=ceval_select(it,cobject,userflags,format,deep,derived_list, recurse_list)
cdedent()
return rep
#
clogger.debug("Searching cache for begin object for : " + `cobject`)
########################################################################
it,comp_period=cache.hasBeginObject(cobject)
clogger.debug("Finished with searching cache for begin object for : " + `cobject`)
#it=None
if it :
clogger.info("partial result found in cache for %s : %s"%\
(cobject.crs,it.crs))
begcrs=it.crs
# Turn object for begin in complement object for end, and eval it
it.setperiod(comp_period)
ceval(it,userflags,format,deep,derived_list,recurse_list)
if (format == 'file') :
rep=cache.complement(begcrs,it.crs,cobject.crs)
cdedent()
return rep
else : raise Climaf_Driver_Error("cannot yet complement except for files")
#
clogger.info("nothing relevant found in cache for %s"%cobject.crs)
#
if deep==False : deep=None
if isinstance(cobject,classes.ctree) :
#
# the cache doesn't have a similar tree, let us recursively eval subtrees
##########################################################################
# TBD : analyze if the dataset is remote and the remote place 'offers' the operator
if cobject.operator in operators.scripts :
file=ceval_script(cobject,deep,recurse_list=recurse_list) # Does return a filename, or list of filenames
cdedent()
if ( format == 'file' ) : return (file)
else : return cread(file,varOf(cobject))
elif cobject.operator in operators.operators :
obj=ceval_operator(cobject,deep)
cdedent()
if (format == 'file' ) :
rep=cstore(obj) ; return rep
else : return(obj)
else :
raise Climaf_Driver_Error("operator %s is not a script nor known operator",str(cobject.operator))
elif isinstance(cobject,classes.scriptChild) :
# Force evaluation of 'father' script
if ceval_script(cobject.father,deep,recurse_list=recurse_list) is not None :
# Re-evaluate, which should succeed using cache
rep=ceval(cobject,userflags,format,deep,recurse_list=recurse_list)
cdedent()
return rep
else :
raise Climaf_Driver_Error("generating script aborted for "+cobject.father.crs)
elif isinstance(cobject,classes.cpage) :
file=cfilePage(cobject, deep, recurse_list=recurse_list)
cdedent()
if ( format == 'file' ) : return (file)
else : return cread(file)
elif isinstance(cobject,classes.cens) :
rep=[]
for member in cobject.members :
rep.append(ceval(member,copy.copy(userflags),format,deep,recurse_list=recurse_list))
if (format=="file") : return(reduce(lambda x,y : x+" "+y, rep))
else : return rep
else :
raise Climaf_Driver_Error("Internal logic error")
elif isinstance(cobject,str) :
clogger.debug("Evaluating object from crs : %s"%cobject)
raise Climaf_Driver_Error("Evaluation from CRS is not yet implemented ( %s )"%cobject)
else :
raise Climaf_Driver_Error("argument " +`cobject`+" is not (yet) managed")
