def AppendQueryResults(self, queryId, results):
"""
Append the specified eventList to the specified queryId
"""
try:
# acquire lock
with self.lock:
# Get query from Q
querySpec = quilt_data.query_specs_get(self._queries, queryId)
# Try to get any existing results then
# append the results into the query spec
# set the results into the query spec
existingEvents = quilt_data.query_spec_tryget(
querySpec, results=True)
if existingEvents is None:
existingEvents = []
# append the results into the query spec
quilt_data.query_spec_set(querySpec,
results=(existingEvents + results))
except Exception, error:
try:
# log exception here, because there is no detail in it once we
# pass it across pyro
logging.error("Unable append results for query: " +
str(queryId))
logging.exception(error)
finally:
# throw the exception back over to the calling process
raise error
def BeginQuery(self, queryId):
"""
string queryID # the query we are interested in
Called by quilt_query to Move a query into the an active state,
and return a copy of the query
"""
try:
logging.info("Beginning query: " + str(queryId))
patternSpec = None
# lock self
with self.lock:
# get the query from the Q
# if can't get it raise exception
querySpec = quilt_data.query_specs_get(self._queries, queryId)
queryState = quilt_data.query_spec_get(querySpec, state=True)
# if query state is not expected INITIALIZED state
if queryState != quilt_data.STATE_INITIALIZED:
# raise exception
raise Exception("Query: " + str(queryId) +
" is not ready for processing")
# move query to ACTIVE state
quilt_data.query_spec_set(querySpec,
state=quilt_data.STATE_ACTIVE)
# create a copy of the query
querySpec = querySpec.copy()
# get a copy of the patternSpec
patternSpec = quilt_data.pat_specs_get(self._patterns,
quilt_data.query_spec_get(querySpec, patternName=True))
patternSpec = patternSpec.copy()
# returning copy because we don't want to stay locked when asking
# pyro to marshall across process bounds. Not that I know for
# a fact that that won't work, but it sounds like a bad idea
# return the pattern and query spec
return patternSpec, querySpec
except Exception, error:
try:
# log exception here, because there is no detail in it once we
# pass it across pyro
logging.error("Unable begin query: " + str(queryId))
logging.exception(error)
finally:
# throw the exception back over to the calling process
raise error
def OnRegisterEnd(self):
"""After registration is complete submit the query to the
query master"""
# create a partial query spec dictionary
# set pattern name from args
# set notification address in spec
# set state as UNINITIALIZED
querySpec = quilt_data.query_spec_create(
name='new ' + self._args.pattern,
state=quilt_data.STATE_UNINITIALIZED,
patternName=self._args.pattern,
notificationEmail=self._args.notifcation_email)
# set variables/values from args
if self._args.variable is not None and len(self._args.variable) > 0:
variables = quilt_data.var_specs_create()
for v in self._args.variable:
vname = v[0]
vval = v[1]
quilt_data.var_specs_add(variables,
quilt_data.var_spec_create(name=vname, value=vval))
quilt_data.query_spec_set(querySpec, variables=variables)
logging.info('Submiting query: ' + pprint.pformat(querySpec))
# call remote method asyncronysly, this will return right away
Pyro4.async(self._qm).Query(self._remotename, querySpec)
# Validate query will be remote called from query master
# return True to allow event loop to start running, which
# should soon recieve a validation callback from query master
import time
ns = Pyro4.locateNS()
uri = ns.lookup(self._remotename)
for i in range(10):
print "loop being"
time.sleep(1)
with Pyro4.Proxy(uri) as p:
print "Sending", i
Pyro4.async(p).TestMsg(i)
print "sent", i
print "loop end"
return True
def _try_move_query_to_hist(self, queryId, state):
"""Private function, only call with self_lock engaged"""
querySpec = quilt_data.query_specs_trydel(self._queries, queryId)
# else find queryId in history
if querySpec is None:
querySpec = quilt_data.query_specs_tryget(self._history,
queryId)
else:
# move query spec from q to history member collection
quilt_data.query_specs_add(self._history, querySpec)
if querySpec is not None:
# mark the query as the specified state
quilt_data.query_spec_set(querySpec, state=state)
return querySpec
def OnRegisterEnd(self):
"""After registration is complete submit the query to the
query master"""
# create a partial query spec dictionary
# set pattern name from args
# set notification address in spec
# set state as UNINITIALIZED
querySpec = quilt_data.query_spec_create(
name='new ' + self._args.pattern,
state=quilt_data.STATE_UNINITIALIZED,
patternName=self._args.pattern,
notificationEmail=self._args.notification_email)
# set variables/values from args
if self._args.variable is not None and len(self._args.variable) > 0:
variables = quilt_data.var_specs_create()
for v in self._args.variable:
vname = v[0]
vval = v[1]
quilt_data.var_specs_add(variables,
quilt_data.var_spec_create(name=vname, value=vval))
quilt_data.query_spec_set(querySpec, variables=variables)
logging.info('Submitting query: ' + pprint.pformat(querySpec))
# call remote method asynchronously, this will return right away
with self.GetQueryMasterProxy() as qm:
Pyro4.async(qm).Query(self._remotename, querySpec)
logging.info('Query Submitted')
# Validate query will be remote called from query master
# return True to allow event loop to start running, which
# should soon receive a validation callback from query master
return True
def Query(self, submitterNameKey, querySpec):
"""
string Query( # return the ID of the query
string submitterNameKey # key that the submitter received
# when it was registered with this
# master
dict querySpec # the details of the query
# necessary from the user
)
"""
# qid must be set because we use it when reporting the error
if submitterNameKey is not None:
qid = str(submitterNameKey) + " unnamed query"
else:
qid = "Unknown query"
# try the following
try:
# a slightly better name for the query id
tqid = quilt_data.query_spec_tryget(querySpec, name=True)
if tqid is not None:
qid = tqid
# placeholder query spec to reserve the unique qid
tmpQuerySpec = querySpec.copy()
# get pattern name from the query spec
patternName = quilt_data.query_spec_get(
querySpec, patternName=True)
# acquire lock
with self.lock:
# copy the patternSpec the query references
patternSpec = quilt_data.pat_specs_get(
self._patterns, patternName).copy()
# generate a query id
baseqid = submitterNameKey + "_" + patternName
i = 0
qid = baseqid
while qid in self._queries or qid in self._history:
i += 1
qid = baseqid + "_" + str(i)
# store querySpec in Q to reserve query id
quilt_data.query_spec_set(tmpQuerySpec, name=qid)
quilt_data.query_specs_add(self._queries, tmpQuerySpec)
# this the query spec we will be manipulating, give it the id
# that we generate, we will eventually replace the placeholder
# in the member q
quilt_data.query_spec_set(querySpec, name=qid)
code = quilt_data.pat_spec_tryget(
patternSpec, code=True)
# using the set of sources described in the query code if
# they exist, or the ones described by mappings otherwise
# group variable mapping's by target source and source pattern
# store in local collection,
# map{source:
# map{sourcePattern:
# map{sourcePatternInstance:
# map{srcVar:Var}}}}
# store in local collection. We later want to iterate the
# source mangers efficiently, so we pre process a varDict here
# which provides a direct mapping from srcVariables to
# queryVariables, grouped by sources and patterns and pattern
# instances
# if pattern code specified
# parse the pattern text, and get set of variables mentioned
# in the pattern code
if code is not None:
# pattern code should be generated to account for any variables
# that will need to be substituted with default value to
# make the code syntactically correct
code = quilt_data.generate_query_code(patternSpec,None)
# get the variables mentioned in the pattern code
varDict = quilt_parse.get_pattern_src_refs(code)
append = False
else:
varDict = var_dict.create()
append = True
patVarSpecs = quilt_data.pat_spec_tryget(
patternSpec, variables=True)
mappings = quilt_data.pat_spec_tryget(
patternSpec, mappings=True)
#logging.info("got patVarSpecs: " + str(patVarSpecs))
#logging.info("got mappings: " + str(mappings))
# if code was specified, only bother making the map for
# sources referenced in the code, otherwise use anything
# that was declared in the pattern spec mappings
if patVarSpecs is not None and mappings is not None:
patVars = patVarSpecs.keys()
# logging.debug("Iterating variables: " + str(patVars))
for m in mappings:
varName = quilt_data.src_var_mapping_spec_get(
m, name=True)
if varName not in patVars:
continue
src = quilt_data.src_var_mapping_spec_get(
m, sourceName=True)
pat = quilt_data.src_var_mapping_spec_get(
m, sourcePattern=True)
var = quilt_data.src_var_mapping_spec_get(
m, sourceVariable=True)
ins = quilt_data.src_var_mapping_spec_tryget(
m, sourcePatternInstance=True)
logging.debug("considering: varDict[" + var + "] = " +
str(varName))
# set append to appropriate value so that the varDict does
# not grow from mappings alone
var_dict.set_var(varDict, src, pat, ins, var, varName,
append=append)
else:
logging.info("No query variables were specified")
logging.info("got varDict:\n " + pprint.pformat(varDict))
varSpecs = quilt_data.query_spec_tryget(
querySpec, variables=True)
# initialize list of srcQueries
srcQuerySpecs = None
# iterate the collection of sources, and build collection of
# srcQuerySpecs for each source
for source in varDict.keys():
# use variable mapping's source name to get proxy to
# that source manager
with get_client_proxy_from_type_and_name(
self, "smd", source) as srcMgr:
patterns = srcMgr.GetSourcePatterns()
# iterate the collection of sourcePatterns for current
# source
for patternName in patterns:
# NOTE we could have decided to build a temp map
# of all patterns and close the source manager
# connection sooner, but it will have no practical
# effect to keep the source manager connection a
# might longer while doing the below processing
# skip the creation of source queries for source
# patterns not references in the query
if patternName not in varDict[source]:
continue
# get the sourcePatternSpec from the proxy
srcPatSpec = srcMgr.GetSourcePattern(patternName)
# create a query spec objects
curSrcQuerySpecs = create_src_query_specs(
srcPatSpec, varDict, varSpecs, patVarSpecs,
qid, source, patternName)
for srcQuerySpec in curSrcQuerySpecs.values():
# append completed sourceQuerySpec to querySpec
srcQuerySpecs = quilt_data.src_query_specs_add(
srcQuerySpecs, srcQuerySpec)
# store sourceQueries in the querySpec
if srcQuerySpecs is not None:
quilt_data.query_spec_set(querySpec,
sourceQuerySpecs=srcQuerySpecs)
# use querySpec and srcQuery list
# to create a validation string
msg = {'Query to run': querySpec,
'Sources to be queried': varDict.keys()}
validStr = pprint.pformat(msg)
# ask submitter to validate the source Queries
# get_client function is threadsafe, returns with the lock off
with get_client_proxy_from_type_and_name(
self, "qsub", submitterNameKey) as submitter:
# call back to the submitter to get validation
validated = submitter.ValidateQuery(validStr, qid)
# if submitter refuses to validate,
if not validated:
logging.info("Submitter: " + submitterNameKey +
" did not validate the query: " + qid)
# acquire lock remove query id from q
# delete the query record, it was determined invalid
# return early
with self.lock:
quilt_data.query_specs_del(self._queries, qid)
return
logging.info("Submitter: " + submitterNameKey +
"did validate the query: " + qid)
# acquire lock
with self.lock:
# store querySpec state as INITIALIZED, and place
# validated contents in member data
quilt_data.query_spec_set(querySpec,
state=quilt_data.STATE_INITIALIZED)
quilt_data.query_specs_add(self._queries,
querySpec)
# Process query...
# get the path to the current directory
# formulate the command line for quilt_query
queryCmd = [
os.path.join(os.path.dirname(__file__), "quilt_query.py"),
qid]
if self._args.log_level is not None:
queryCmd.append("--log-level")
queryCmd.append(self._args.log_level)
if self._args.log_file is not None:
queryCmd.append("--log-file")
queryCmd.append(self._args.log_file)
# use subprocess module to fork off the process
# script, pass the query ID
# sei_core.run_process("sleep 5 && " + ' '.join(queryCmd) + " &",shell=True)
subprocess.Popen(queryCmd)
# catch exception!
except Exception, error:
logging.exception(error)
try:
# submit launched this call asynchronously and must be made
# aware of the unexpected error
# call submit's OnSubmitProblem
logging.info("Attempting to get proxy for error report")
with get_client_proxy_from_type_and_name(
self, "qsub", submitterNameKey) as submitter:
logging.info("Attempting to send error to submitter")
Pyro4.async(submitter).OnSubmitProblem(qid, error)
except Exception, error2:
# stuff is going horribly wrong, we couldn't notify submitter
logging.error("Unable to notify submitter: " +
str(submitterNameKey) +
" of error when submitting query: " + str(qid))
logging.exception(error2)