def run( self, run_args ):
        data = get_json_arg( 'data', run_args )
        region = get_json_arg( 'region', run_args )
        result_obj = {}
        try:
            start_time = time.time()
            cdms2keyargs = self.region2cdms( region )
            variable = run_args.get( "dataSlice", None )
            if variable is None:
                id = data["id"]
                url = data.get("url",None)
                if url is not None:
                    var_cache_id =  ":".join( [url,id] )
                    dataset = self.loadFileFromURL( url )
                else:
                    collection = data.get("collection",None)
                    if collection is not None:
                        var_cache_id =  ":".join( [collection,id] )
                        dataset = self.loadFileFromCollection( collection, id )
                    else:
                        wpsLog.debug( " $$$ Empty Data Request: '%s' ",  str( run_args ) )
                        return None
                wpsLog.debug( " $$$ Data Request: '%s', '%s' ", var_cache_id, str( cdms2keyargs ) )
                variable = dataset[ id ]
                result_obj['variable'] = record_attributes( variable, [ 'long_name', 'name', 'units' ], { 'id': id } )
                result_obj['dataset'] = record_attributes( dataset, [ 'id', 'uri' ])
            else:
                result_obj['variable'] = record_attributes( variable, [ 'long_name', 'name', 'id', 'units' ]  )

            read_start_time = time.time()
            subsetted_variable = numpy.ma.fix_invalid( variable(**cdms2keyargs) )
            read_end_time = time.time()
            wpsLog.debug( " $$$ DATA READ Complete: " + str( (read_end_time-read_start_time) ) )

            process_start_time = time.time()
            ( result_data, time_axis ) = self.applyOperation( subsetted_variable, self.operation )
            process_end_time = time.time()
            wpsLog.debug( " $$$ DATA PROCESSING Complete: " + str( (process_end_time-process_start_time) ) )
            #            pydevd.settrace('localhost', port=8030, stdoutToServer=False, stderrToServer=True)

            if time_axis is not None:
                time_obj = record_attributes( time_axis, [ 'units', 'calendar' ] )
                time_data = time_axis.getValue().tolist()
                try:
                    time_obj['t0'] = time_data[0]
                    time_obj['dt'] = time_data[1] - time_data[0]
                except Exception, err:
                    time_obj['data'] = time_data
                result_obj['time'] = time_obj
            result_obj['data'] = result_data
            end_time = time.time()
            wpsLog.debug( " $$$ Execution complete, total time: %.2f sec", (end_time-start_time) )
Exemple #2
0
 def execute( self, tesk_request ):
     try:
         if self.engine is None:
             engine_id = tesk_request[ 'engine' ]
             if engine_id is None:
                 engine_id = 'serial'
                 wpsLog.warning( "Compute Engine not confgured, running serially.")
             self.engine = engineRegistry.getInstance( engine_id )
         wpsLog.debug( "Executing engine '%s', class: %s" % ( self.engine.id, self.engine.__class__.__name__ ) )
         result =  self.engine.execute( tesk_request )
         return result
     except Exception, err:
         wpsLog.debug( "Exception in staging handler: '%s'\n%s" % ( str(err), traceback.format_exc() ) )
         return {}
Exemple #3
0
    def execute(self):
        try:
            # wpsLog.debug( "  -------------- Execution stack:  -------------- ")
            # wpsLog.debug( traceback.format_stack() )
            # wpsLog.debug( "  -------------- ________________  -------------- ")
#            debug_trace()
            data = loadValue( self.data )
            region = loadValue( self.region )
            operation = loadValue( self.operation )
            request = { 'datainputs': { 'data':data, 'region':region, 'operation':operation }, 'async':self.async.value, 'embedded':self.embedded.value }
            wpsLog.debug( " $$$ CDAS Django-WPS Process,  Request: %s, Operation: %s " % ( str( request ), str(operation) ) )
            response = taskManager.processRequest( request )
            self.result.setValue( response )
        except Exception, err:
             wpsLog.debug( "Exception executing CDAS process:\n " + traceback.format_exc() )
             self.result.setValue( '' )
    def setTimeBounds( self, var ):
        time_axis = var.getTime()
        if time_axis._bounds_ == None:
            try:
                time_unit = time_axis.units.split(' since ')[0].strip()
                if time_unit == 'hours':
                    values = time_axis.getValue()
                    freq = 24/( values[1]-values[0] )
                    cdutil.setTimeBoundsDaily( time_axis, freq )
#                    cdutil.setTimeBoundsDaily( time_axis )
                elif time_unit == 'days':
                    cdutil.setTimeBoundsDaily( time_axis )
                elif time_unit == 'months':
                    cdutil.setTimeBoundsMonthly( time_axis )
                elif time_unit == 'years':
                    cdutil.setTimeBoundsYearly( time_axis )
            except Exception, err:
                wpsLog.debug( "Exception in setTimeBounds:\n " + traceback.format_exc() )
Exemple #5
0
def cdas():
    task_run_args = get_request_datainputs()

    data = task_run_args['data']
    region = task_run_args['region']
    operation = task_run_args['operation']
    wpsLog.debug( " $$$ CDAS Process: DataIn='%s', Domain='%s', Operation='%s', ---> Time=%.3f " % ( str( data ), str( region ), str( operation ), time.time() ) )
    t0 = time.time()
    handler = stagingRegistry.getInstance( configuration.CDAS_STAGING  )
    if handler is None:
        wpsLog.warning( " Staging method not configured. Running locally on wps server. " )
        handler = stagingRegistry.getInstance( 'local' )
    result_obj =  handler.execute( { 'data':data, 'region':region, 'operation':operation, 'engine': configuration.CDAS_COMPUTE_ENGINE + "Engine"  } )
    wpsLog.debug( " $$$*** CDAS Process (response time: %.3f sec):\n Result='%s' " %  ( (time.time()-t0), str(result_obj) ) )
    result_json = json.dumps( result_obj )

#    resp = flask.make_response( json.dumps(result), 200 )
    return result_json
Exemple #6
0
    def getTaskRequestData( self, request_params ):
        task_parameters = {}
        request = dict( request_params )
        inputs = request.get('datainputs',None)
        wpsLog.debug( ">>>> REQUEST datainputs: %s" % str(inputs) )
        if inputs:
            if isinstance( inputs, dict ):
                task_parameters.update( inputs )
            elif isinstance( inputs, list ):
                for item in inputs:
                    if isinstance( item, list ):
                        task_parameters[ str(item[0]) ] = convert_json_str( item[1] )
                    elif isinstance( item, basestring ):
                        self.parseDatainputsStr( item, task_parameters )
            else:
                self.parseDatainputsStr(self, inputs, task_parameters )
            del request[ 'datainputs' ]
        for key in self.dataInputKeys:
            parameter = request.get(key,None)
            if parameter:
                if isinstance(parameter, basestring):
                    try:
                        parameter = json.loads( parameter )
                    except:
                        wpsLog.error( " Error json decoding parameter: '%s' " % parameter )
                        parameter = ""
                task_parameters[ key ] = parameter
                del request[key]
        for key in self.controlKeys:
            parameter = request.get(key,None)
            if parameter:
                if isinstance(parameter, basestring): parameter = parameter.lower()
                task_parameters[ key ] = parameter
                del request[key]
        for item in request.items():
            task_parameters[ item[0] ] = item[1]

        for key_alias, alt_key in self.aliases.items():
            if key_alias in task_parameters:
                task_parameters[ alt_key ] = task_parameters[ key_alias ]
                del task_parameters[ key_alias ]

        return task_parameters
 def getURL(self, collection_name, var_id  ):
     wpsLog.debug( " getURL: %s %s " % ( collection_name, var_id ) )
     collection_rec = self.getCollectionRecord( collection_name )
     return self.constructURL( collection_rec[0], collection_rec[1], var_id )
    def applyOperation( self, input_variable, operation ):
        result = None
        try:
            self.setTimeBounds( input_variable )
            operator = None
            time_axis = None
#            pydevd.settrace('localhost', port=8030, stdoutToServer=False, stderrToServer=True)
            wpsLog.debug( " $$$ ApplyOperation: %s " % str( operation ) )
            if operation is not None:
                type = operation.get('type','').lower()
                bounds = operation.get('bounds','').lower()
                op_start_time = time.clock() # time.time()
                if not bounds:
                    if type == 'departures':
                        ave = cdutil.averager( input_variable, axis='t', weights='equal' )
                        result = input_variable - ave
                    elif type == 'climatology':
                        result = cdutil.averager( input_variable, axis='t', weights='equal' )
                    else:
                        result = input_variable
                    time_axis = input_variable.getTime()
                elif bounds == 'np':
                    if   type == 'departures':
                        result = ma.anomalies( input_variable ).squeeze()
                    elif type == 'climatology':
                        result = ma.average( input_variable ).squeeze()
                    else:
                        result = input_variable
                    time_axis = input_variable.getTime()
                else:
                    if bounds == 'djf':
                        operator = cdutil.DJF
                    elif bounds == 'mam':
                        operator = cdutil.MAM
                    elif bounds == 'jja':
                        operator = cdutil.JJA
                    elif bounds == 'son':
                        operator = cdutil.SON
                    elif bounds == 'year':
                        operator = cdutil.YEAR
                    elif bounds == 'annualcycle':
                        operator = cdutil.ANNUALCYCLE
                    elif bounds == 'seasonalcycle':
                        operator = cdutil.SEASONALCYCLE
                    if operator <> None:
                        if   type == 'departures':
                            result = operator.departures( input_variable ).squeeze()
                        elif type == 'climatology':
                            result = operator.climatology( input_variable ).squeeze()
                            if bounds == 'annualcycle':
                                time_axis = cdms2.createAxis( range( len(result) ) )
                                time_axis.units = "months"
                            elif bounds == 'seasonalcycle':
                                time_axis = cdms2.createAxis( range( len(result) ) )
                                time_axis.units = "seasons"
                        else:
                            result = operator( input_variable ).squeeze()
                    if time_axis is None:
                        time_axis = result.getTime()
                op_end_time = time.clock() # time.time()
                wpsLog.debug( " ---> Base Operation Time: %.5f" % (op_end_time-op_start_time) )
                if math.isnan( result[0] ):
                    pp = pprint.PrettyPrinter(indent=4)
                    print "\n ---------- NaN in Result, Input: ---------- "
                    print str( input_variable.data )
            else:
                result = input_variable
                time_axis = input_variable.getTime()

            if isinstance( result, float ):
                result_data = [ result ]
            elif result is not None:
                if result.__class__.__name__ == 'TransientVariable':
                    result = ma.masked_equal( result.squeeze().getValue(), input_variable.getMissing() )
                result_data = result.tolist( numpy.nan )
            else:
                result_data = None
                time_axis = input_variable.getTime()
        except Exception, err:
            wpsLog.debug( "Exception applying Operation '%s':\n %s" % ( str(operation), traceback.format_exc() ) )
            return ( None, None )
Exemple #9
0
 def loadFileFromCollection( self, collection, id=None ):
     collectionManager = CollectionManager.getInstance( settings.CDAS_APPLICATION )
     url = collectionManager.getURL( collection, id )
     wpsLog.debug( "loadFileFromCollection: '%s' '%s': %s " % ( collection, id, url ) )
     return self.loadFileFromURL( url )