def execute( self, task_request, **celery_args ): try: t0 = time.time() async = celery_args.get( 'async', False ) cache_request = None self.updateWorkerSpecs() self.processPendingTasks() designated_worker = None cLog.debug( " ***** Executing Celery engine (t=%.2f), request: %s" % ( t0, str(task_request) ) ) dset_mdata = task_request.data.values op_region = task_request.region.value operation = task_request.operations.values for var_mdata in dset_mdata: id = var_mdata.get('id','') collection = var_mdata.get('collection',None) url = var_mdata.get('url','') var_cache_id = ":".join( [collection,id] ) if (collection is not None) else ":".join( [url,id] ) if var_cache_id <> ":": cached_var,cached_domain = self.findCachedDomain( var_cache_id, op_region, task_request ) if cached_domain is None: cache_axis_list = [Region.LEVEL] if operation else [Region.LEVEL, Region.LATITUDE, Region.LONGITUDE] cache_region = Region( op_region, axes=cache_axis_list ) cache_op_args = { 'region':cache_region.spec, 'data':var_mdata.spec } tc0 = time.time() cache_task_request = TaskRequest( task=cache_op_args) cache_worker = self.getNextWorker() cache_request = execute.apply_async( (cache_task_request.task,), exchange='C.dq', routing_key=cache_worker ) tc01 = time.time() self.setWorkerStatus( cache_worker, self.WSTAT_CACHE ) cached_domain = cached_var.addDomain( cache_region ) self.pendingTasks[ cache_request ] = cached_domain self.cache() tc1 = time.time() cLog.debug( " ***** Caching data [tid:%s] to worker '%s' ([%.2f,%.2f,%.2f] dt = %.3f): args = %s " % ( cache_request.id, cache_worker, tc0, tc01, tc1, (tc1-tc0), str(cache_op_args) ) ) else: worker_id, cache_request_status = cached_domain.getCacheStatus() if (cache_request_status == Domain.COMPLETE) or ( cached_var.cacheType() == CachedVariable.CACHE_OP ): designated_worker = worker_id cLog.debug( " ***** Found cached data on worker %s " % worker_id ) else: cLog.debug( " ***** Found cache op on worker %s, data not ready " % worker_id ) if operation: t2 = time.time() if designated_worker is None: designated_worker = self.getNextWorker() task = execute.apply_async( (task_request.task,), exchange='C.dq', routing_key=designated_worker ) self.setWorkerStatus( designated_worker, self.WSTAT_OP ) op_domain = cached_var.addDomain( op_region ) self.pendingTasks[ task ] = op_domain self.cache() cLog.debug( " ***** Sending operation [tid:%s] to worker '%s' (t = %.2f, dt0 = %.3f): request= %s " % ( task.id, str(designated_worker), t2, t2-t0, str(task_request) ) ) if async: return task result = task.get() t1 = time.time() cLog.debug( " ***** Retrieved result [tid:%s] from worker '%s' (t = %.2f, dt1 = %.3f)" % ( task.id, result[0]['worker'], t1, t1-t2 ) ) return result else: if async: return cache_request except Exception, err: wpsLog.error(" Error running celery engine: %s\n %s " % ( str(err), traceback.format_exc() ) ) return None
def run_test(): from request.manager import TaskRequest wpsLog.addHandler( logging.StreamHandler(sys.stdout) ) #logging.FileHandler( os.path.abspath( os.path.join(os.path.dirname(__file__), '..', 'logs', 'wps.log') ) ) ) wpsLog.setLevel(logging.DEBUG) pp = pprint.PrettyPrinter(indent=4) test_cache = False simple_test = False variable = { 'collection': 'MERRA/mon/atmos', 'id': 'clt' } region1 = { "longitude":-24.20, "latitude":58.45, "level": 10000.0 } region2 = { "longitude":-30.20, "latitude":67.45, "level": 8500.0 } op_annual_cycle = {"kernel":"time", "type":"climatology", "bounds":"annualcycle"} op_departures = {"kernel":"time", "type":"departures", "bounds":"np"} engine = CeleryEngine('celery') if simple_test: engine.updateWorkerSpecs() cache_worker = engine.getNextWorker() args = [ 1,2,3] ts0 = time.time() task = simpleTest.apply_async( (args,), exchange='C.dq', routing_key=cache_worker ) result = task.get() ts1 = time.time() print( " ***** Simple test using worker %s, time = %.3f " % ( cache_worker, (ts1-ts0) ) ) else: if test_cache: engine.updateWorkerSpecs() cache_worker = engine.getNextWorker() print( " ***** Caching data to worker %s " % cache_worker ) cache_op_args = { 'data': variable } task = execute.apply_async( (cache_op_args,True), exchange='C.dq', routing_key=cache_worker ) result = task.get() print "\n ---------- Result(cache): ---------- " pp.pprint(result) else: run_async = True if run_async: t0 = time.time() request = TaskRequest( task={ 'data': variable, 'region':region1, 'operation': op_departures } ) task1 = engine.execute( request, async=True ) request = TaskRequest( task={ 'data': variable, 'region':region2, 'operation': op_annual_cycle } ) task2 = engine.execute( request, async=True ) t1 = time.time() result1 = task1.get() if task1 else None print "\n ---------- Result (departures): ---------- " pp.pprint(result1) if result1 else "<NONE>" t2 = time.time() print "\n ---------- Result (annual cycle): ---------- " result2 = task2.get() if task2 else None pp.pprint(result2) if result2 else "<NONE>" t3 = time.time() print "\n Operations Complete, dt0 = %.2f, dt1 = %.2f, , dt2 = %.2f, dt = %.2f " % ( t1-t0, t2-t1, t3-t2, t3-t0 ) else: t0 = time.time() request = TaskRequest( task={ 'data': variable, 'region':region1, 'operation': op_departures } ) result1 = engine.execute( request ) print "\n ---------- Result (departures): ---------- " pp.pprint(result1) t1 = time.time() request = TaskRequest( task={ 'data': variable, 'region':region1, 'operation': op_annual_cycle } ) result2 = engine.execute( request ) print "\n ---------- Result(annual cycle): ---------- " pp.pprint(result2) t2 = time.time() print "\n Operations Complete, dt0 = %.2f, dt1 = %.2f, dt = %.2f " % ( t1-t0, t2-t1, t2-t0 )
def submitTaskImpl(self, task_request, worker): task = execute.apply_async((task_request.task,), exchange="C.dq", routing_key=worker) return CeleryTaskMonitor(task.id, task=task)