def run( self, run_args ):
data = get_json_arg( 'data', run_args )
region = get_json_arg( 'region', run_args )
result_obj = {}
try:
start_time = time.time()
cdms2keyargs = self.region2cdms( region )
variable = run_args.get( "dataSlice", None )
if variable is None:
id = data["id"]
url = data.get("url",None)
if url is not None:
var_cache_id = ":".join( [url,id] )
dataset = self.loadFileFromURL( url )
else:
collection = data.get("collection",None)
if collection is not None:
var_cache_id = ":".join( [collection,id] )
dataset = self.loadFileFromCollection( collection, id )
else:
wpsLog.debug( " $$$ Empty Data Request: '%s' ", str( run_args ) )
return None
wpsLog.debug( " $$$ Data Request: '%s', '%s' ", var_cache_id, str( cdms2keyargs ) )
variable = dataset[ id ]
result_obj['variable'] = record_attributes( variable, [ 'long_name', 'name', 'units' ], { 'id': id } )
result_obj['dataset'] = record_attributes( dataset, [ 'id', 'uri' ])
else:
result_obj['variable'] = record_attributes( variable, [ 'long_name', 'name', 'id', 'units' ] )
read_start_time = time.time()
subsetted_variable = numpy.ma.fix_invalid( variable(**cdms2keyargs) )
read_end_time = time.time()
wpsLog.debug( " $$$ DATA READ Complete: " + str( (read_end_time-read_start_time) ) )
process_start_time = time.time()
( result_data, time_axis ) = self.applyOperation( subsetted_variable, self.operation )
process_end_time = time.time()
wpsLog.debug( " $$$ DATA PROCESSING Complete: " + str( (process_end_time-process_start_time) ) )
# pydevd.settrace('localhost', port=8030, stdoutToServer=False, stderrToServer=True)
if time_axis is not None:
time_obj = record_attributes( time_axis, [ 'units', 'calendar' ] )
time_data = time_axis.getValue().tolist()
try:
time_obj['t0'] = time_data[0]
time_obj['dt'] = time_data[1] - time_data[0]
except Exception, err:
time_obj['data'] = time_data
result_obj['time'] = time_obj
result_obj['data'] = result_data
end_time = time.time()
wpsLog.debug( " $$$ Execution complete, total time: %.2f sec", (end_time-start_time) )
def execute( self, tesk_request ):
try:
if self.engine is None:
engine_id = tesk_request[ 'engine' ]
if engine_id is None:
engine_id = 'serial'
wpsLog.warning( "Compute Engine not confgured, running serially.")
self.engine = engineRegistry.getInstance( engine_id )
wpsLog.debug( "Executing engine '%s', class: %s" % ( self.engine.id, self.engine.__class__.__name__ ) )
result = self.engine.execute( tesk_request )
return result
except Exception, err:
wpsLog.debug( "Exception in staging handler: '%s'\n%s" % ( str(err), traceback.format_exc() ) )
return {}
def execute(self):
try:
# wpsLog.debug( " -------------- Execution stack: -------------- ")
# wpsLog.debug( traceback.format_stack() )
# wpsLog.debug( " -------------- ________________ -------------- ")
# debug_trace()
data = loadValue( self.data )
region = loadValue( self.region )
operation = loadValue( self.operation )
request = { 'datainputs': { 'data':data, 'region':region, 'operation':operation }, 'async':self.async.value, 'embedded':self.embedded.value }
wpsLog.debug( " $$$ CDAS Django-WPS Process, Request: %s, Operation: %s " % ( str( request ), str(operation) ) )
response = taskManager.processRequest( request )
self.result.setValue( response )
except Exception, err:
wpsLog.debug( "Exception executing CDAS process:\n " + traceback.format_exc() )
self.result.setValue( '' )
def setTimeBounds( self, var ):
time_axis = var.getTime()
if time_axis._bounds_ == None:
try:
time_unit = time_axis.units.split(' since ')[0].strip()
if time_unit == 'hours':
values = time_axis.getValue()
freq = 24/( values[1]-values[0] )
cdutil.setTimeBoundsDaily( time_axis, freq )
# cdutil.setTimeBoundsDaily( time_axis )
elif time_unit == 'days':
cdutil.setTimeBoundsDaily( time_axis )
elif time_unit == 'months':
cdutil.setTimeBoundsMonthly( time_axis )
elif time_unit == 'years':
cdutil.setTimeBoundsYearly( time_axis )
except Exception, err:
wpsLog.debug( "Exception in setTimeBounds:\n " + traceback.format_exc() )
def cdas():
task_run_args = get_request_datainputs()
data = task_run_args['data']
region = task_run_args['region']
operation = task_run_args['operation']
wpsLog.debug( " $$$ CDAS Process: DataIn='%s', Domain='%s', Operation='%s', ---> Time=%.3f " % ( str( data ), str( region ), str( operation ), time.time() ) )
t0 = time.time()
handler = stagingRegistry.getInstance( configuration.CDAS_STAGING )
if handler is None:
wpsLog.warning( " Staging method not configured. Running locally on wps server. " )
handler = stagingRegistry.getInstance( 'local' )
result_obj = handler.execute( { 'data':data, 'region':region, 'operation':operation, 'engine': configuration.CDAS_COMPUTE_ENGINE + "Engine" } )
wpsLog.debug( " $$$*** CDAS Process (response time: %.3f sec):\n Result='%s' " % ( (time.time()-t0), str(result_obj) ) )
result_json = json.dumps( result_obj )
# resp = flask.make_response( json.dumps(result), 200 )
return result_json
def getTaskRequestData( self, request_params ):
task_parameters = {}
request = dict( request_params )
inputs = request.get('datainputs',None)
wpsLog.debug( ">>>> REQUEST datainputs: %s" % str(inputs) )
if inputs:
if isinstance( inputs, dict ):
task_parameters.update( inputs )
elif isinstance( inputs, list ):
for item in inputs:
if isinstance( item, list ):
task_parameters[ str(item[0]) ] = convert_json_str( item[1] )
elif isinstance( item, basestring ):
self.parseDatainputsStr( item, task_parameters )
else:
self.parseDatainputsStr(self, inputs, task_parameters )
del request[ 'datainputs' ]
for key in self.dataInputKeys:
parameter = request.get(key,None)
if parameter:
if isinstance(parameter, basestring):
try:
parameter = json.loads( parameter )
except:
wpsLog.error( " Error json decoding parameter: '%s' " % parameter )
parameter = ""
task_parameters[ key ] = parameter
del request[key]
for key in self.controlKeys:
parameter = request.get(key,None)
if parameter:
if isinstance(parameter, basestring): parameter = parameter.lower()
task_parameters[ key ] = parameter
del request[key]
for item in request.items():
task_parameters[ item[0] ] = item[1]
for key_alias, alt_key in self.aliases.items():
if key_alias in task_parameters:
task_parameters[ alt_key ] = task_parameters[ key_alias ]
del task_parameters[ key_alias ]
return task_parameters
def getURL(self, collection_name, var_id ):
wpsLog.debug( " getURL: %s %s " % ( collection_name, var_id ) )
collection_rec = self.getCollectionRecord( collection_name )
return self.constructURL( collection_rec[0], collection_rec[1], var_id )
def applyOperation( self, input_variable, operation ):
result = None
try:
self.setTimeBounds( input_variable )
operator = None
time_axis = None
# pydevd.settrace('localhost', port=8030, stdoutToServer=False, stderrToServer=True)
wpsLog.debug( " $$$ ApplyOperation: %s " % str( operation ) )
if operation is not None:
type = operation.get('type','').lower()
bounds = operation.get('bounds','').lower()
op_start_time = time.clock() # time.time()
if not bounds:
if type == 'departures':
ave = cdutil.averager( input_variable, axis='t', weights='equal' )
result = input_variable - ave
elif type == 'climatology':
result = cdutil.averager( input_variable, axis='t', weights='equal' )
else:
result = input_variable
time_axis = input_variable.getTime()
elif bounds == 'np':
if type == 'departures':
result = ma.anomalies( input_variable ).squeeze()
elif type == 'climatology':
result = ma.average( input_variable ).squeeze()
else:
result = input_variable
time_axis = input_variable.getTime()
else:
if bounds == 'djf':
operator = cdutil.DJF
elif bounds == 'mam':
operator = cdutil.MAM
elif bounds == 'jja':
operator = cdutil.JJA
elif bounds == 'son':
operator = cdutil.SON
elif bounds == 'year':
operator = cdutil.YEAR
elif bounds == 'annualcycle':
operator = cdutil.ANNUALCYCLE
elif bounds == 'seasonalcycle':
operator = cdutil.SEASONALCYCLE
if operator <> None:
if type == 'departures':
result = operator.departures( input_variable ).squeeze()
elif type == 'climatology':
result = operator.climatology( input_variable ).squeeze()
if bounds == 'annualcycle':
time_axis = cdms2.createAxis( range( len(result) ) )
time_axis.units = "months"
elif bounds == 'seasonalcycle':
time_axis = cdms2.createAxis( range( len(result) ) )
time_axis.units = "seasons"
else:
result = operator( input_variable ).squeeze()
if time_axis is None:
time_axis = result.getTime()
op_end_time = time.clock() # time.time()
wpsLog.debug( " ---> Base Operation Time: %.5f" % (op_end_time-op_start_time) )
if math.isnan( result[0] ):
pp = pprint.PrettyPrinter(indent=4)
print "\n ---------- NaN in Result, Input: ---------- "
print str( input_variable.data )
else:
result = input_variable
time_axis = input_variable.getTime()
if isinstance( result, float ):
result_data = [ result ]
elif result is not None:
if result.__class__.__name__ == 'TransientVariable':
result = ma.masked_equal( result.squeeze().getValue(), input_variable.getMissing() )
result_data = result.tolist( numpy.nan )
else:
result_data = None
time_axis = input_variable.getTime()
except Exception, err:
wpsLog.debug( "Exception applying Operation '%s':\n %s" % ( str(operation), traceback.format_exc() ) )
return ( None, None )
def loadFileFromCollection( self, collection, id=None ):
collectionManager = CollectionManager.getInstance( settings.CDAS_APPLICATION )
url = collectionManager.getURL( collection, id )
wpsLog.debug( "loadFileFromCollection: '%s' '%s': %s " % ( collection, id, url ) )
return self.loadFileFromURL( url )