def BeginQuery(self, queryId):
"""
string queryID # the query we are interested in
Called by quilt_query to Move a query into the an active state,
and return a copy of the query
"""
try:
logging.info("Beginning query: " + str(queryId))
patternSpec = None
# lock self
with self.lock:
# get the query from the Q
# if can't get it raise exception
querySpec = quilt_data.query_specs_get(self._queries, queryId)
queryState = quilt_data.query_spec_get(querySpec, state=True)
# if query state is not expected INITIALIZED state
if queryState != quilt_data.STATE_INITIALIZED:
# raise exception
raise Exception("Query: " + str(queryId) +
" is not ready for processing")
# move query to ACTIVE state
quilt_data.query_spec_set(querySpec,
state=quilt_data.STATE_ACTIVE)
# create a copy of the query
querySpec = querySpec.copy()
# get a copy of the patternSpec
patternSpec = quilt_data.pat_specs_get(self._patterns,
quilt_data.query_spec_get(querySpec, patternName=True))
patternSpec = patternSpec.copy()
# returning copy because we don't want to stay locked when asking
# pyro to marshall across process bounds. Not that I know for
# a fact that that won't work, but it sounds like a bad idea
# return the pattern and query spec
return patternSpec, querySpec
except Exception, error:
try:
# log exception here, because there is no detail in it once we
# pass it across pyro
logging.error("Unable begin query: " + str(queryId))
logging.exception(error)
finally:
# throw the exception back over to the calling process
raise error
def OnSourceQueryError(self, srcQueryId, exception):
"""Receive a message from the source manager about a problem that
:param srcQueryId:
:param exception:
occurred when running the source query"""
try:
# log the source's exception
# print out the source query id and the message
logging.error("Source Query error for: " + str(srcQueryId) +
" : " + quilt_core.exception_to_string(exception))
# I guess exception's don't keep their stacktrace over the pyro
# boundary
# logging.exception(exception)
# lock self to modify member data
with self._lock:
# mark source query state with ERROR
srcQuerySpec = quilt_data.src_query_spec_get(
self._srcQuerySpecs, srcQueryId)
quilt_data.src_query_spec_set(srcQuerySpec,
state=quilt_data.STATE_ERROR)
qid = quilt_data.query_spec_get(srcQuerySpec, name=True)
# call query Master's Error function
with self.GetQueryMasterProxy() as qm:
qm.OnQueryError(qid, exception)
except Exception, error2:
logging.error("Unable to send source query error to query master")
logging.exception(error2)
def CompleteQuery(self):
qid = None
# try the following
try:
# lock self
with self._lock:
code = quilt_data.pat_spec_tryget(
self._patternSpec, code=True)
qid = quilt_data.query_spec_get(self._querySpec, name=True)
# if there is no query code
if code is None:
with self.GetQueryMasterProxy() as qm:
# for each source result
for curResult in self._srcResults.values():
# append source results to query master
qm.AppendQueryResults(qid, curResult)
# call query masters CompleteQuery
qm.CompleteQuery(qid)
else:
logging.info("Interpreting query: " + str(qid))
# evaluate the query by calling quilt_interpret
# pass the query spec, and source Results
result = quilt_interpret.evaluate_query(self._patternSpec,
self._querySpec,
self._srcResults)
# append the returned results to the query master's
# results for this query
logging.info("Posting results for query: " + str(qid))
with self.GetQueryMasterProxy() as qm:
qm.AppendQueryResults(qid, result)
# call query masters CompleteQuery
qm.CompleteQuery(qid)
# catch exceptions
except Exception, error:
# log out the exceptions,
# Call query master's OnQueryError
try:
with self.GetQueryMasterProxy() as qm:
qm.OnQueryError(qid, error)
except Exception, error2:
logging.error(
"Unable to send query interpret error to query master")
logging.exception(error2)
def source(srcName, srcPatName, srcPatInstance=None):
"""
Construct a pattern wrapper from the specified source
"""
# use the global query spec
srcQuerySpecs = quilt_data.query_spec_get(_get_query_spec(),
sourceQuerySpecs=True)
# logging.debug("srcQuerySpecs:\n" + pprint.pformat(srcQuerySpecs))
# logging.debug("Looking for Source: " + str(srcName) + ", pattern: " + str(srcPatName)
# + ". instance: " + str(srcPatInstance) )
# iterate the srcQueries
for srcQueryId, srcQuerySpec in srcQuerySpecs.items():
# if matching (srcName, patName, and patInstanceName)
# if None was supplied as one of the parameter, then there
# must only be one instance to choose, keyed by 'None'
curSrc = quilt_data.src_query_spec_get(srcQuerySpec,
source=True)
curSrcPat = quilt_data.src_query_spec_get(srcQuerySpec,
srcPatternName=True)
curSrcPatInst = quilt_data.src_query_spec_tryget(
srcQuerySpec, srcPatternInstance=True)
# logging.debug("checking " + curSrc + ", " + curSrcPat +
# ", " + str( curSrcPatInst))
if (srcName == curSrc and
srcPatName == curSrcPat and
srcPatInstance == curSrcPatInst):
# get the global srcResults for that srcQueryID
srcResults = _get_events(srcQueryId)
# return a new _pattern with the srcResults as the events
return _pattern(srcQueryId, srcResults)
# raise exception if matching srcQuery not found
raise Exception("Source: " + str(srcName) + ", pattern: " + str(srcPatName)
+ ". instance: " + str(srcPatInstance) +
", could not be found among the source queries")
def Query(self, submitterNameKey, querySpec):
"""
string Query( # return the ID of the query
string submitterNameKey # key that the submitter received
# when it was registered with this
# master
dict querySpec # the details of the query
# necessary from the user
)
"""
# qid must be set because we use it when reporting the error
if submitterNameKey is not None:
qid = str(submitterNameKey) + " unnamed query"
else:
qid = "Unknown query"
# try the following
try:
# a slightly better name for the query id
tqid = quilt_data.query_spec_tryget(querySpec, name=True)
if tqid is not None:
qid = tqid
# placeholder query spec to reserve the unique qid
tmpQuerySpec = querySpec.copy()
# get pattern name from the query spec
patternName = quilt_data.query_spec_get(
querySpec, patternName=True)
# acquire lock
with self.lock:
# copy the patternSpec the query references
patternSpec = quilt_data.pat_specs_get(
self._patterns, patternName).copy()
# generate a query id
baseqid = submitterNameKey + "_" + patternName
i = 0
qid = baseqid
while qid in self._queries or qid in self._history:
i += 1
qid = baseqid + "_" + str(i)
# store querySpec in Q to reserve query id
quilt_data.query_spec_set(tmpQuerySpec, name=qid)
quilt_data.query_specs_add(self._queries, tmpQuerySpec)
# this the query spec we will be manipulating, give it the id
# that we generate, we will eventually replace the placeholder
# in the member q
quilt_data.query_spec_set(querySpec, name=qid)
code = quilt_data.pat_spec_tryget(
patternSpec, code=True)
# using the set of sources described in the query code if
# they exist, or the ones described by mappings otherwise
# group variable mapping's by target source and source pattern
# store in local collection,
# map{source:
# map{sourcePattern:
# map{sourcePatternInstance:
# map{srcVar:Var}}}}
# store in local collection. We later want to iterate the
# source mangers efficiently, so we pre process a varDict here
# which provides a direct mapping from srcVariables to
# queryVariables, grouped by sources and patterns and pattern
# instances
# if pattern code specified
# parse the pattern text, and get set of variables mentioned
# in the pattern code
if code is not None:
# pattern code should be generated to account for any variables
# that will need to be substituted with default value to
# make the code syntactically correct
code = quilt_data.generate_query_code(patternSpec,None)
# get the variables mentioned in the pattern code
varDict = quilt_parse.get_pattern_src_refs(code)
append = False
else:
varDict = var_dict.create()
append = True
patVarSpecs = quilt_data.pat_spec_tryget(
patternSpec, variables=True)
mappings = quilt_data.pat_spec_tryget(
patternSpec, mappings=True)
#logging.info("got patVarSpecs: " + str(patVarSpecs))
#logging.info("got mappings: " + str(mappings))
# if code was specified, only bother making the map for
# sources referenced in the code, otherwise use anything
# that was declared in the pattern spec mappings
if patVarSpecs is not None and mappings is not None:
patVars = patVarSpecs.keys()
# logging.debug("Iterating variables: " + str(patVars))
for m in mappings:
varName = quilt_data.src_var_mapping_spec_get(
m, name=True)
if varName not in patVars:
continue
src = quilt_data.src_var_mapping_spec_get(
m, sourceName=True)
pat = quilt_data.src_var_mapping_spec_get(
m, sourcePattern=True)
var = quilt_data.src_var_mapping_spec_get(
m, sourceVariable=True)
ins = quilt_data.src_var_mapping_spec_tryget(
m, sourcePatternInstance=True)
logging.debug("considering: varDict[" + var + "] = " +
str(varName))
# set append to appropriate value so that the varDict does
# not grow from mappings alone
var_dict.set_var(varDict, src, pat, ins, var, varName,
append=append)
else:
logging.info("No query variables were specified")
logging.info("got varDict:\n " + pprint.pformat(varDict))
varSpecs = quilt_data.query_spec_tryget(
querySpec, variables=True)
# initialize list of srcQueries
srcQuerySpecs = None
# iterate the collection of sources, and build collection of
# srcQuerySpecs for each source
for source in varDict.keys():
# use variable mapping's source name to get proxy to
# that source manager
with get_client_proxy_from_type_and_name(
self, "smd", source) as srcMgr:
patterns = srcMgr.GetSourcePatterns()
# iterate the collection of sourcePatterns for current
# source
for patternName in patterns:
# NOTE we could have decided to build a temp map
# of all patterns and close the source manager
# connection sooner, but it will have no practical
# effect to keep the source manager connection a
# might longer while doing the below processing
# skip the creation of source queries for source
# patterns not references in the query
if patternName not in varDict[source]:
continue
# get the sourcePatternSpec from the proxy
srcPatSpec = srcMgr.GetSourcePattern(patternName)
# create a query spec objects
curSrcQuerySpecs = create_src_query_specs(
srcPatSpec, varDict, varSpecs, patVarSpecs,
qid, source, patternName)
for srcQuerySpec in curSrcQuerySpecs.values():
# append completed sourceQuerySpec to querySpec
srcQuerySpecs = quilt_data.src_query_specs_add(
srcQuerySpecs, srcQuerySpec)
# store sourceQueries in the querySpec
if srcQuerySpecs is not None:
quilt_data.query_spec_set(querySpec,
sourceQuerySpecs=srcQuerySpecs)
# use querySpec and srcQuery list
# to create a validation string
msg = {'Query to run': querySpec,
'Sources to be queried': varDict.keys()}
validStr = pprint.pformat(msg)
# ask submitter to validate the source Queries
# get_client function is threadsafe, returns with the lock off
with get_client_proxy_from_type_and_name(
self, "qsub", submitterNameKey) as submitter:
# call back to the submitter to get validation
validated = submitter.ValidateQuery(validStr, qid)
# if submitter refuses to validate,
if not validated:
logging.info("Submitter: " + submitterNameKey +
" did not validate the query: " + qid)
# acquire lock remove query id from q
# delete the query record, it was determined invalid
# return early
with self.lock:
quilt_data.query_specs_del(self._queries, qid)
return
logging.info("Submitter: " + submitterNameKey +
"did validate the query: " + qid)
# acquire lock
with self.lock:
# store querySpec state as INITIALIZED, and place
# validated contents in member data
quilt_data.query_spec_set(querySpec,
state=quilt_data.STATE_INITIALIZED)
quilt_data.query_specs_add(self._queries,
querySpec)
# Process query...
# get the path to the current directory
# formulate the command line for quilt_query
queryCmd = [
os.path.join(os.path.dirname(__file__), "quilt_query.py"),
qid]
if self._args.log_level is not None:
queryCmd.append("--log-level")
queryCmd.append(self._args.log_level)
if self._args.log_file is not None:
queryCmd.append("--log-file")
queryCmd.append(self._args.log_file)
# use subprocess module to fork off the process
# script, pass the query ID
# sei_core.run_process("sleep 5 && " + ' '.join(queryCmd) + " &",shell=True)
subprocess.Popen(queryCmd)
# catch exception!
except Exception, error:
logging.exception(error)
try:
# submit launched this call asynchronously and must be made
# aware of the unexpected error
# call submit's OnSubmitProblem
logging.info("Attempting to get proxy for error report")
with get_client_proxy_from_type_and_name(
self, "qsub", submitterNameKey) as submitter:
logging.info("Attempting to send error to submitter")
Pyro4.async(submitter).OnSubmitProblem(qid, error)
except Exception, error2:
# stuff is going horribly wrong, we couldn't notify submitter
logging.error("Unable to notify submitter: " +
str(submitterNameKey) +
" of error when submitting query: " + str(qid))
logging.exception(error2)
def OnRegisterEnd(self):
"""
Retrieve the querySpec from the query master and issue the
srcQueries to the source
"""
qid = None
try:
# use query id passed in arguments
if self._args.query_id is not None:
qid = self._args.query_id[0]
else:
raise Exception("Query ID is required")
# Access the QuiltQuery's registrar's host and port from the config
config = quilt_core.QuiltConfig()
rport = config.GetValue("registrar", "port", None, int)
if rport is not None:
rport = int(rport)
rhost = config.GetValue("registrar", "host", None)
self._registrarPort = rport
self._registrarHost = rhost
# set the state to ACTIVE by calling BeginQuery
# store pattern and query as a data member
with self.GetQueryMasterProxy() as qm:
self._patternSpec, self._querySpec = qm.BeginQuery(qid)
# get the query spec from query master
queryState = quilt_data.query_spec_get(self._querySpec,
state=True)
if queryState != quilt_data.STATE_ACTIVE:
raise Exception("Query: " + qid + ", must be in " +
quilt_data.STATE_ACTIVE +
" state. It is currently in " +
queryState + " state.")
# iterate the sourceQuerySpec's in srcQueries list
srcQuerySpecs = quilt_data.query_spec_tryget(
self._querySpec, sourceQuerySpecs=True)
# there are no source query specs specified
if srcQuerySpecs is None:
# so just don't do anything
self._processEvents = False
return
self._srcQuerySpecs = srcQuerySpecs
for srcQuerySpec in srcQuerySpecs.values():
# mark the sourceQuery ACTIVE
quilt_data.src_query_spec_set(srcQuerySpec,
state=quilt_data.STATE_ACTIVE)
# get proxy to the source manager
source = quilt_data.src_query_spec_get(
srcQuerySpec, source=True)
smgrRec = qm.GetClientRec("smd", source)
# TODO make more efficient by recycling proxies to same source
# in the case when making multi source queries to same source
with query_master.get_client_proxy(smgrRec) as smgr:
# query the source by sending it the source query specs as
# asynchronous call
Pyro4.async(smgr).Query(
qid, srcQuerySpec, self.localname, rhost, rport)
# Note: no locking needed
# asynchronous call, and returning messages not
# processed until this function exits
self._processEvents = True
except Exception, error:
try:
with self.GetQueryMasterProxy() as qm:
qm.OnQueryError(qid, error)
except Exception, error2:
logging.error(
"Unable to send query startup error to query master")
logging.exception(error2)
