def OnSourceQueryError(self, srcQueryId, exception):
"""Receive a message from the source manager about a problem that
:param srcQueryId:
:param exception:
occurred when running the source query"""
try:
# log the source's exception
# print out the source query id and the message
logging.error("Source Query error for: " + str(srcQueryId) +
" : " + quilt_core.exception_to_string(exception))
# I guess exception's don't keep their stacktrace over the pyro
# boundary
# logging.exception(exception)
# lock self to modify member data
with self._lock:
# mark source query state with ERROR
srcQuerySpec = quilt_data.src_query_spec_get(
self._srcQuerySpecs, srcQueryId)
quilt_data.src_query_spec_set(srcQuerySpec,
state=quilt_data.STATE_ERROR)
qid = quilt_data.query_spec_get(srcQuerySpec, name=True)
# call query Master's Error function
with self.GetQueryMasterProxy() as qm:
qm.OnQueryError(qid, exception)
except Exception, error2:
logging.error("Unable to send source query error to query master")
logging.exception(error2)
def source(srcName, srcPatName, srcPatInstance=None):
"""
Construct a pattern wrapper from the specified source
"""
# use the global query spec
srcQuerySpecs = quilt_data.query_spec_get(_get_query_spec(),
sourceQuerySpecs=True)
# logging.debug("srcQuerySpecs:\n" + pprint.pformat(srcQuerySpecs))
# logging.debug("Looking for Source: " + str(srcName) + ", pattern: " + str(srcPatName)
# + ". instance: " + str(srcPatInstance) )
# iterate the srcQueries
for srcQueryId, srcQuerySpec in srcQuerySpecs.items():
# if matching (srcName, patName, and patInstanceName)
# if None was supplied as one of the parameter, then there
# must only be one instance to choose, keyed by 'None'
curSrc = quilt_data.src_query_spec_get(srcQuerySpec,
source=True)
curSrcPat = quilt_data.src_query_spec_get(srcQuerySpec,
srcPatternName=True)
curSrcPatInst = quilt_data.src_query_spec_tryget(
srcQuerySpec, srcPatternInstance=True)
# logging.debug("checking " + curSrc + ", " + curSrcPat +
# ", " + str( curSrcPatInst))
if (srcName == curSrc and
srcPatName == curSrcPat and
srcPatInstance == curSrcPatInst):
# get the global srcResults for that srcQueryID
srcResults = _get_events(srcQueryId)
# return a new _pattern with the srcResults as the events
return _pattern(srcQueryId, srcResults)
# raise exception if matching srcQuery not found
raise Exception("Source: " + str(srcName) + ", pattern: " + str(srcPatName)
+ ". instance: " + str(srcPatInstance) +
", could not be found among the source queries")
def Query(self, queryId, sourceQuerySpec,
queryClientName,
queryClientNamseServerHost,
queryClientNamseServerPort):
"""
Query the source, currently hardcoded to behave as calling a cmdline
tool that outputs source results on each output line
"""
try:
with self._lock:
# set last query member to queryId
self._lastQuery = queryId
# _sourceName should not change, set at init, so ok to read
# without lock
logging.info("Source Manager: " + str(self._sourceName) +
" received query: " + str(queryId))
# get the sourcePatternSpec from the pattern name in the
# sourceQuerySpec
# pattern spec should be read only and safe to access without
# a lock
srcPatSpecs = quilt_data.src_spec_get(self._sourceSpec,
sourcePatterns=True)
srcPatSpec = quilt_data.src_pat_specs_get(srcPatSpecs,
quilt_data.src_query_spec_get(sourceQuerySpec,
srcPatternName=True))
# get the variables in the query
srcQueryVars = quilt_data.src_query_spec_get(sourceQuerySpec,
variables=True)
# iterate src query variables map, create a simple var name to
# var value map
varNameValueDict = {}
for srcVarName, srcVarSpec in srcQueryVars.items():
varNameValueDict[srcVarName] = quilt_data.var_spec_get(
srcVarSpec, value=True)
# create cmd line for the source
# use the template in the sourcePatternSpec, and use the values
# provided for the variables, and environment variables
replacements = {}
# replacements = os.environ.copy()
for k, v in varNameValueDict.items():
replacements[k] = v
# "template" was not added as official schema member because it is
# specific to this command line case
templateCmd = srcPatSpec['template']
template = Template(templateCmd)
logging.info("Ready to use : " + templateCmd +
" with replacements: " + str(replacements))
cmdline = template.safe_substitute(replacements)
# setup context for the cmdline stdout callback
srcQueryId = quilt_data.src_query_spec_get(
sourceQuerySpec, name=True)
context = {'queryId': queryId, 'srcQueryId': srcQueryId}
# sei_core.run_process_lite(cmdline, shell=True,
# outFunc=self.OnGrepLine, outObj=context)
# I thought run_process apparently has problems
sei_core.run_process(cmdline, shell=True,
whichReturn=sei_core.EXITCODE,
outFunc=self.OnGrepLine, outObj=context, logToPython=False)
# Set query result events list in query master using query id
results = []
with self._lock:
if queryId in self._sourceResults:
if srcQueryId in self._sourceResults[queryId]:
results = list(self._sourceResults[queryId][srcQueryId])
uri = quilt_core.get_uri(queryClientNamseServerHost,
queryClientNamseServerPort, queryClientName)
with Pyro4.Proxy(uri) as query:
query.AppendSourceQueryResults(srcQueryId, results)
query.CompleteSrcQuery(srcQueryId)
# catch exception!
except Exception, error:
try:
# attempt to report error to the query client
uri = quilt_core.get_uri(queryClientNamseServerHost,
queryClientNamseServerPort, queryClientName)
srcQueryId = quilt_data.src_query_spec_get(
sourceQuerySpec, name=True)
with Pyro4.Proxy(uri) as query:
query.OnSourceQueryError(srcQueryId, error)
except Exception, error2:
logging.error("Unable to send source query error to " +
"query master")
logging.exception(error2)
def OnRegisterEnd(self):
"""
Retrieve the querySpec from the query master and issue the
srcQueries to the source
"""
qid = None
try:
# use query id passed in arguments
if self._args.query_id is not None:
qid = self._args.query_id[0]
else:
raise Exception("Query ID is required")
# Access the QuiltQuery's registrar's host and port from the config
config = quilt_core.QuiltConfig()
rport = config.GetValue("registrar", "port", None, int)
if rport is not None:
rport = int(rport)
rhost = config.GetValue("registrar", "host", None)
self._registrarPort = rport
self._registrarHost = rhost
# set the state to ACTIVE by calling BeginQuery
# store pattern and query as a data member
with self.GetQueryMasterProxy() as qm:
self._patternSpec, self._querySpec = qm.BeginQuery(qid)
# get the query spec from query master
queryState = quilt_data.query_spec_get(self._querySpec,
state=True)
if queryState != quilt_data.STATE_ACTIVE:
raise Exception("Query: " + qid + ", must be in " +
quilt_data.STATE_ACTIVE +
" state. It is currently in " +
queryState + " state.")
# iterate the sourceQuerySpec's in srcQueries list
srcQuerySpecs = quilt_data.query_spec_tryget(
self._querySpec, sourceQuerySpecs=True)
# there are no source query specs specified
if srcQuerySpecs is None:
# so just don't do anything
self._processEvents = False
return
self._srcQuerySpecs = srcQuerySpecs
for srcQuerySpec in srcQuerySpecs.values():
# mark the sourceQuery ACTIVE
quilt_data.src_query_spec_set(srcQuerySpec,
state=quilt_data.STATE_ACTIVE)
# get proxy to the source manager
source = quilt_data.src_query_spec_get(
srcQuerySpec, source=True)
smgrRec = qm.GetClientRec("smd", source)
# TODO make more efficient by recycling proxies to same source
# in the case when making multi source queries to same source
with query_master.get_client_proxy(smgrRec) as smgr:
# query the source by sending it the source query specs as
# asynchronous call
Pyro4.async(smgr).Query(
qid, srcQuerySpec, self.localname, rhost, rport)
# Note: no locking needed
# asynchronous call, and returning messages not
# processed until this function exits
self._processEvents = True
except Exception, error:
try:
with self.GetQueryMasterProxy() as qm:
qm.OnQueryError(qid, error)
except Exception, error2:
logging.error(
"Unable to send query startup error to query master")
logging.exception(error2)
def CompleteSrcQuery(self, srcQueryId):
"""
string srcQueryId # id for the source query)
Called by sourceManager when it is done processing the query
"""
logging.info("Completing source query: " + str(srcQueryId))
try:
# NOTE:
# Because the member data query specs are not entries are not
# modified (Except during initialization) we do not have to lock
# before reading from the list. Also we assume no results
# should be appended after a call to complete query so that we
# can sort the list of data outside of a lock. Also see note
# in AppendSourceQueryResults which could change the logic here
# get the source query from the member data collection of src
# queries using the srcQueryId
srcQuerySpec = quilt_data.src_query_specs_get(
self._srcQuerySpecs, srcQueryId)
# if src query specifies that source returns out of order results
if not quilt_data.src_query_spec_get(srcQuerySpec, ordered=True):
results = None
with self._lock:
if srcQueryId in self._srcResults:
results = self._srcResults[srcQueryId]
if results is not None:
# sort results by timestamp using interpret's 'at'
# function
results.sort(key=lambda rec: at(rec))
# acquire lock
with self._lock:
# set srcQueries's state to COMPLETED
# If query is progressing through the system properly it
# should be an active state when it gets here
srcQueryState = quilt_data.src_query_spec_get(srcQuerySpec,
state=True)
if srcQueryState != quilt_data.STATE_ACTIVE:
raise Exception("Source Query is: " + srcQueryState +
". Can only complete a query that is " +
quilt_data.STATE_ACTIVE)
quilt_data.src_query_spec_set(srcQuerySpec,
state=quilt_data.STATE_COMPLETED)
try:
with self._lock:
# Detect if all src queries are completed
completed = True
for srcQuerySpec in self._srcQuerySpecs.values():
srcState = quilt_data.src_query_spec_get(srcQuerySpec,
state=True)
if (srcState != quilt_data.STATE_COMPLETED and
srcState != quilt_data.STATE_ERROR):
completed = False
logging.debug("At least " + srcQuerySpec['name'] +
" is still in progress")
break
# if this was the last source query,
if completed:
# get proxy to self
with Pyro4.Proxy(self.uri) as selfProxy:
# asynchronously call self's CompleteQuery
Pyro4.async(selfProxy).CompleteQuery()
# catch exceptions
except Exception, error3:
# log out the exceptions, do not pass along to
# source as it wasn't his fault
logging.error(
"Unable to properly check for last query")
logging.exception(error3)
# set process events flag to false end event loop, allowing
# query client to exit
self.SetProcesssEvents(False)
raise
# catch exceptions
except Exception, error2:
# log out the exceptions, do not pass along to
# source as it wasn't his fault
logging.error(
"Unable to tell query master that the query is complete")
logging.exception(error2)
# set process events flag to false end event loop, allowing
# query client to exit
self.SetProcesssEvents(False)
raise