def OnSourceQueryError(self, srcQueryId, exception):
"""Receive a message from the source manager about a problem that
:param srcQueryId:
:param exception:
occurred when running the source query"""
try:
# log the source's exception
# print out the source query id and the message
logging.error("Source Query error for: " + str(srcQueryId) +
" : " + quilt_core.exception_to_string(exception))
# I guess exception's don't keep their stacktrace over the pyro
# boundary
# logging.exception(exception)
# lock self to modify member data
with self._lock:
# mark source query state with ERROR
srcQuerySpec = quilt_data.src_query_spec_get(
self._srcQuerySpecs, srcQueryId)
quilt_data.src_query_spec_set(srcQuerySpec,
state=quilt_data.STATE_ERROR)
qid = quilt_data.query_spec_get(srcQuerySpec, name=True)
# call query Master's Error function
with self.GetQueryMasterProxy() as qm:
qm.OnQueryError(qid, exception)
except Exception, error2:
logging.error("Unable to send source query error to query master")
logging.exception(error2)
def OnRegisterEnd(self):
"""
Retrieve the querySpec from the query master and issue the
srcQueries to the source
"""
qid = None
try:
# use query id passed in arguments
if self._args.query_id is not None:
qid = self._args.query_id[0]
else:
raise Exception("Query ID is required")
# Access the QuiltQuery's registrar's host and port from the config
config = quilt_core.QuiltConfig()
rport = config.GetValue("registrar", "port", None, int)
if rport is not None:
rport = int(rport)
rhost = config.GetValue("registrar", "host", None)
self._registrarPort = rport
self._registrarHost = rhost
# set the state to ACTIVE by calling BeginQuery
# store pattern and query as a data member
with self.GetQueryMasterProxy() as qm:
self._patternSpec, self._querySpec = qm.BeginQuery(qid)
# get the query spec from query master
queryState = quilt_data.query_spec_get(self._querySpec,
state=True)
if queryState != quilt_data.STATE_ACTIVE:
raise Exception("Query: " + qid + ", must be in " +
quilt_data.STATE_ACTIVE +
" state. It is currently in " +
queryState + " state.")
# iterate the sourceQuerySpec's in srcQueries list
srcQuerySpecs = quilt_data.query_spec_tryget(
self._querySpec, sourceQuerySpecs=True)
# there are no source query specs specified
if srcQuerySpecs is None:
# so just don't do anything
self._processEvents = False
return
self._srcQuerySpecs = srcQuerySpecs
for srcQuerySpec in srcQuerySpecs.values():
# mark the sourceQuery ACTIVE
quilt_data.src_query_spec_set(srcQuerySpec,
state=quilt_data.STATE_ACTIVE)
# get proxy to the source manager
source = quilt_data.src_query_spec_get(
srcQuerySpec, source=True)
smgrRec = qm.GetClientRec("smd", source)
# TODO make more efficient by recycling proxies to same source
# in the case when making multi source queries to same source
with query_master.get_client_proxy(smgrRec) as smgr:
# query the source by sending it the source query specs as
# asynchronous call
Pyro4.async(smgr).Query(
qid, srcQuerySpec, self.localname, rhost, rport)
# Note: no locking needed
# asynchronous call, and returning messages not
# processed until this function exits
self._processEvents = True
except Exception, error:
try:
with self.GetQueryMasterProxy() as qm:
qm.OnQueryError(qid, error)
except Exception, error2:
logging.error(
"Unable to send query startup error to query master")
logging.exception(error2)
def CompleteSrcQuery(self, srcQueryId):
"""
string srcQueryId # id for the source query)
Called by sourceManager when it is done processing the query
"""
logging.info("Completing source query: " + str(srcQueryId))
try:
# NOTE:
# Because the member data query specs are not entries are not
# modified (Except during initialization) we do not have to lock
# before reading from the list. Also we assume no results
# should be appended after a call to complete query so that we
# can sort the list of data outside of a lock. Also see note
# in AppendSourceQueryResults which could change the logic here
# get the source query from the member data collection of src
# queries using the srcQueryId
srcQuerySpec = quilt_data.src_query_specs_get(
self._srcQuerySpecs, srcQueryId)
# if src query specifies that source returns out of order results
if not quilt_data.src_query_spec_get(srcQuerySpec, ordered=True):
results = None
with self._lock:
if srcQueryId in self._srcResults:
results = self._srcResults[srcQueryId]
if results is not None:
# sort results by timestamp using interpret's 'at'
# function
results.sort(key=lambda rec: at(rec))
# acquire lock
with self._lock:
# set srcQueries's state to COMPLETED
# If query is progressing through the system properly it
# should be an active state when it gets here
srcQueryState = quilt_data.src_query_spec_get(srcQuerySpec,
state=True)
if srcQueryState != quilt_data.STATE_ACTIVE:
raise Exception("Source Query is: " + srcQueryState +
". Can only complete a query that is " +
quilt_data.STATE_ACTIVE)
quilt_data.src_query_spec_set(srcQuerySpec,
state=quilt_data.STATE_COMPLETED)
try:
with self._lock:
# Detect if all src queries are completed
completed = True
for srcQuerySpec in self._srcQuerySpecs.values():
srcState = quilt_data.src_query_spec_get(srcQuerySpec,
state=True)
if (srcState != quilt_data.STATE_COMPLETED and
srcState != quilt_data.STATE_ERROR):
completed = False
logging.debug("At least " + srcQuerySpec['name'] +
" is still in progress")
break
# if this was the last source query,
if completed:
# get proxy to self
with Pyro4.Proxy(self.uri) as selfProxy:
# asynchronously call self's CompleteQuery
Pyro4.async(selfProxy).CompleteQuery()
# catch exceptions
except Exception, error3:
# log out the exceptions, do not pass along to
# source as it wasn't his fault
logging.error(
"Unable to properly check for last query")
logging.exception(error3)
# set process events flag to false end event loop, allowing
# query client to exit
self.SetProcesssEvents(False)
raise
# catch exceptions
except Exception, error2:
# log out the exceptions, do not pass along to
# source as it wasn't his fault
logging.error(
"Unable to tell query master that the query is complete")
logging.exception(error2)
# set process events flag to false end event loop, allowing
# query client to exit
self.SetProcesssEvents(False)
raise
