def OnSourceQueryError(self, srcQueryId, exception): """Receive a message from the source manager about a problem that :param srcQueryId: :param exception: occurred when running the source query""" try: # log the source's exception # print out the source query id and the message logging.error("Source Query error for: " + str(srcQueryId) + " : " + quilt_core.exception_to_string(exception)) # I guess exception's don't keep their stacktrace over the pyro # boundary # logging.exception(exception) # lock self to modify member data with self._lock: # mark source query state with ERROR srcQuerySpec = quilt_data.src_query_spec_get( self._srcQuerySpecs, srcQueryId) quilt_data.src_query_spec_set(srcQuerySpec, state=quilt_data.STATE_ERROR) qid = quilt_data.query_spec_get(srcQuerySpec, name=True) # call query Master's Error function with self.GetQueryMasterProxy() as qm: qm.OnQueryError(qid, exception) except Exception, error2: logging.error("Unable to send source query error to query master") logging.exception(error2)
def OnRegisterEnd(self): """ Retrieve the querySpec from the query master and issue the srcQueries to the source """ qid = None try: # use query id passed in arguments if self._args.query_id is not None: qid = self._args.query_id[0] else: raise Exception("Query ID is required") # Access the QuiltQuery's registrar's host and port from the config config = quilt_core.QuiltConfig() rport = config.GetValue("registrar", "port", None, int) if rport is not None: rport = int(rport) rhost = config.GetValue("registrar", "host", None) self._registrarPort = rport self._registrarHost = rhost # set the state to ACTIVE by calling BeginQuery # store pattern and query as a data member with self.GetQueryMasterProxy() as qm: self._patternSpec, self._querySpec = qm.BeginQuery(qid) # get the query spec from query master queryState = quilt_data.query_spec_get(self._querySpec, state=True) if queryState != quilt_data.STATE_ACTIVE: raise Exception("Query: " + qid + ", must be in " + quilt_data.STATE_ACTIVE + " state. It is currently in " + queryState + " state.") # iterate the sourceQuerySpec's in srcQueries list srcQuerySpecs = quilt_data.query_spec_tryget( self._querySpec, sourceQuerySpecs=True) # there are no source query specs specified if srcQuerySpecs is None: # so just don't do anything self._processEvents = False return self._srcQuerySpecs = srcQuerySpecs for srcQuerySpec in srcQuerySpecs.values(): # mark the sourceQuery ACTIVE quilt_data.src_query_spec_set(srcQuerySpec, state=quilt_data.STATE_ACTIVE) # get proxy to the source manager source = quilt_data.src_query_spec_get( srcQuerySpec, source=True) smgrRec = qm.GetClientRec("smd", source) # TODO make more efficient by recycling proxies to same source # in the case when making multi source queries to same source with query_master.get_client_proxy(smgrRec) as smgr: # query the source by sending it the source query specs as # asynchronous call Pyro4.async(smgr).Query( qid, srcQuerySpec, self.localname, rhost, rport) # Note: no locking needed # asynchronous call, and returning messages not # processed until this function exits self._processEvents = True except Exception, error: try: with self.GetQueryMasterProxy() as qm: qm.OnQueryError(qid, error) except Exception, error2: logging.error( "Unable to send query startup error to query master") logging.exception(error2)
def CompleteSrcQuery(self, srcQueryId): """ string srcQueryId # id for the source query) Called by sourceManager when it is done processing the query """ logging.info("Completing source query: " + str(srcQueryId)) try: # NOTE: # Because the member data query specs are not entries are not # modified (Except during initialization) we do not have to lock # before reading from the list. Also we assume no results # should be appended after a call to complete query so that we # can sort the list of data outside of a lock. Also see note # in AppendSourceQueryResults which could change the logic here # get the source query from the member data collection of src # queries using the srcQueryId srcQuerySpec = quilt_data.src_query_specs_get( self._srcQuerySpecs, srcQueryId) # if src query specifies that source returns out of order results if not quilt_data.src_query_spec_get(srcQuerySpec, ordered=True): results = None with self._lock: if srcQueryId in self._srcResults: results = self._srcResults[srcQueryId] if results is not None: # sort results by timestamp using interpret's 'at' # function results.sort(key=lambda rec: at(rec)) # acquire lock with self._lock: # set srcQueries's state to COMPLETED # If query is progressing through the system properly it # should be an active state when it gets here srcQueryState = quilt_data.src_query_spec_get(srcQuerySpec, state=True) if srcQueryState != quilt_data.STATE_ACTIVE: raise Exception("Source Query is: " + srcQueryState + ". Can only complete a query that is " + quilt_data.STATE_ACTIVE) quilt_data.src_query_spec_set(srcQuerySpec, state=quilt_data.STATE_COMPLETED) try: with self._lock: # Detect if all src queries are completed completed = True for srcQuerySpec in self._srcQuerySpecs.values(): srcState = quilt_data.src_query_spec_get(srcQuerySpec, state=True) if (srcState != quilt_data.STATE_COMPLETED and srcState != quilt_data.STATE_ERROR): completed = False logging.debug("At least " + srcQuerySpec['name'] + " is still in progress") break # if this was the last source query, if completed: # get proxy to self with Pyro4.Proxy(self.uri) as selfProxy: # asynchronously call self's CompleteQuery Pyro4.async(selfProxy).CompleteQuery() # catch exceptions except Exception, error3: # log out the exceptions, do not pass along to # source as it wasn't his fault logging.error( "Unable to properly check for last query") logging.exception(error3) # set process events flag to false end event loop, allowing # query client to exit self.SetProcesssEvents(False) raise # catch exceptions except Exception, error2: # log out the exceptions, do not pass along to # source as it wasn't his fault logging.error( "Unable to tell query master that the query is complete") logging.exception(error2) # set process events flag to false end event loop, allowing # query client to exit self.SetProcesssEvents(False) raise