def Query(self, submitterNameKey, querySpec): """ string Query( # return the ID of the query string submitterNameKey # key that the submitter received # when it was registered with this # master dict querySpec # the details of the query # necessary from the user ) """ # qid must be set because we use it when reporting the error if submitterNameKey is not None: qid = str(submitterNameKey) + " unnamed query" else: qid = "Unknown query" # try the following try: # a slightly better name for the query id tqid = quilt_data.query_spec_tryget(querySpec, name=True) if tqid is not None: qid = tqid # placeholder query spec to reserve the unique qid tmpQuerySpec = querySpec.copy() # get pattern name from the query spec patternName = quilt_data.query_spec_get( querySpec, patternName=True) # acquire lock with self.lock: # copy the patternSpec the query references patternSpec = quilt_data.pat_specs_get( self._patterns, patternName).copy() # generate a query id baseqid = submitterNameKey + "_" + patternName i = 0 qid = baseqid while qid in self._queries or qid in self._history: i += 1 qid = baseqid + "_" + str(i) # store querySpec in Q to reserve query id quilt_data.query_spec_set(tmpQuerySpec, name=qid) quilt_data.query_specs_add(self._queries, tmpQuerySpec) # this the query spec we will be manipulating, give it the id # that we generate, we will eventually replace the placeholder # in the member q quilt_data.query_spec_set(querySpec, name=qid) code = quilt_data.pat_spec_tryget( patternSpec, code=True) # using the set of sources described in the query code if # they exist, or the ones described by mappings otherwise # group variable mapping's by target source and source pattern # store in local collection, # map{source: # map{sourcePattern: # map{sourcePatternInstance: # map{srcVar:Var}}}} # store in local collection. We later want to iterate the # source mangers efficiently, so we pre process a varDict here # which provides a direct mapping from srcVariables to # queryVariables, grouped by sources and patterns and pattern # instances # if pattern code specified # parse the pattern text, and get set of variables mentioned # in the pattern code if code is not None: # pattern code should be generated to account for any variables # that will need to be substituted with default value to # make the code syntactically correct code = quilt_data.generate_query_code(patternSpec,None) # get the variables mentioned in the pattern code varDict = quilt_parse.get_pattern_src_refs(code) append = False else: varDict = var_dict.create() append = True patVarSpecs = quilt_data.pat_spec_tryget( patternSpec, variables=True) mappings = quilt_data.pat_spec_tryget( patternSpec, mappings=True) #logging.info("got patVarSpecs: " + str(patVarSpecs)) #logging.info("got mappings: " + str(mappings)) # if code was specified, only bother making the map for # sources referenced in the code, otherwise use anything # that was declared in the pattern spec mappings if patVarSpecs is not None and mappings is not None: patVars = patVarSpecs.keys() # logging.debug("Iterating variables: " + str(patVars)) for m in mappings: varName = quilt_data.src_var_mapping_spec_get( m, name=True) if varName not in patVars: continue src = quilt_data.src_var_mapping_spec_get( m, sourceName=True) pat = quilt_data.src_var_mapping_spec_get( m, sourcePattern=True) var = quilt_data.src_var_mapping_spec_get( m, sourceVariable=True) ins = quilt_data.src_var_mapping_spec_tryget( m, sourcePatternInstance=True) logging.debug("considering: varDict[" + var + "] = " + str(varName)) # set append to appropriate value so that the varDict does # not grow from mappings alone var_dict.set_var(varDict, src, pat, ins, var, varName, append=append) else: logging.info("No query variables were specified") logging.info("got varDict:\n " + pprint.pformat(varDict)) varSpecs = quilt_data.query_spec_tryget( querySpec, variables=True) # initialize list of srcQueries srcQuerySpecs = None # iterate the collection of sources, and build collection of # srcQuerySpecs for each source for source in varDict.keys(): # use variable mapping's source name to get proxy to # that source manager with get_client_proxy_from_type_and_name( self, "smd", source) as srcMgr: patterns = srcMgr.GetSourcePatterns() # iterate the collection of sourcePatterns for current # source for patternName in patterns: # NOTE we could have decided to build a temp map # of all patterns and close the source manager # connection sooner, but it will have no practical # effect to keep the source manager connection a # might longer while doing the below processing # skip the creation of source queries for source # patterns not references in the query if patternName not in varDict[source]: continue # get the sourcePatternSpec from the proxy srcPatSpec = srcMgr.GetSourcePattern(patternName) # create a query spec objects curSrcQuerySpecs = create_src_query_specs( srcPatSpec, varDict, varSpecs, patVarSpecs, qid, source, patternName) for srcQuerySpec in curSrcQuerySpecs.values(): # append completed sourceQuerySpec to querySpec srcQuerySpecs = quilt_data.src_query_specs_add( srcQuerySpecs, srcQuerySpec) # store sourceQueries in the querySpec if srcQuerySpecs is not None: quilt_data.query_spec_set(querySpec, sourceQuerySpecs=srcQuerySpecs) # use querySpec and srcQuery list # to create a validation string msg = {'Query to run': querySpec, 'Sources to be queried': varDict.keys()} validStr = pprint.pformat(msg) # ask submitter to validate the source Queries # get_client function is threadsafe, returns with the lock off with get_client_proxy_from_type_and_name( self, "qsub", submitterNameKey) as submitter: # call back to the submitter to get validation validated = submitter.ValidateQuery(validStr, qid) # if submitter refuses to validate, if not validated: logging.info("Submitter: " + submitterNameKey + " did not validate the query: " + qid) # acquire lock remove query id from q # delete the query record, it was determined invalid # return early with self.lock: quilt_data.query_specs_del(self._queries, qid) return logging.info("Submitter: " + submitterNameKey + "did validate the query: " + qid) # acquire lock with self.lock: # store querySpec state as INITIALIZED, and place # validated contents in member data quilt_data.query_spec_set(querySpec, state=quilt_data.STATE_INITIALIZED) quilt_data.query_specs_add(self._queries, querySpec) # Process query... # get the path to the current directory # formulate the command line for quilt_query queryCmd = [ os.path.join(os.path.dirname(__file__), "quilt_query.py"), qid] if self._args.log_level is not None: queryCmd.append("--log-level") queryCmd.append(self._args.log_level) if self._args.log_file is not None: queryCmd.append("--log-file") queryCmd.append(self._args.log_file) # use subprocess module to fork off the process # script, pass the query ID # sei_core.run_process("sleep 5 && " + ' '.join(queryCmd) + " &",shell=True) subprocess.Popen(queryCmd) # catch exception! except Exception, error: logging.exception(error) try: # submit launched this call asynchronously and must be made # aware of the unexpected error # call submit's OnSubmitProblem logging.info("Attempting to get proxy for error report") with get_client_proxy_from_type_and_name( self, "qsub", submitterNameKey) as submitter: logging.info("Attempting to send error to submitter") Pyro4.async(submitter).OnSubmitProblem(qid, error) except Exception, error2: # stuff is going horribly wrong, we couldn't notify submitter logging.error("Unable to notify submitter: " + str(submitterNameKey) + " of error when submitting query: " + str(qid)) logging.exception(error2)
def OnRegisterEnd(self): """Create a patternSpec dict from arguments (name, VARIABLES, and VARIABLE to SOURCE_VARIABLE mappings), register this pattern with the query master""" # create the spec with it's requested name requestedName = self._args.name if requestedName is None: requestedName = "pattern" else: requestedName = self._args.name[0] patternSpec = quilt_data.pat_spec_create(name=requestedName) # create the specs for the variables variables = None if self._args.variable is not None: for v in self._args.variable: # create the variables section if it does not exist # first position of the cmd line argument is name of variable varName = v[0] varSpec = quilt_data.var_spec_create(name=varName) # if description was specified on cmd line, load it in the spec if len(v) > 1: varDesc = v[1] quilt_data.var_spec_set(varSpec, description=varDesc) # if default was specified on cmd line, load it in spec if len(v) > 2: varDef = v[2] quilt_data.var_spec_set(varSpec, default=varDef) variables = quilt_data.var_specs_add(variables, varSpec) quilt_data.pat_spec_set(patternSpec, variables=variables) mappings = None # create the specs for the variable mappings if self._args.mapping is not None: for m in self._args.mapping: varName = m[0] src = m[1] srcPat = m[2] srcVar = m[3] srcPatInstance = None if len(m) > 4: srcPatInstance = m[4] # query variables are allowed to map to multiple # source variables. Initialize a blank list if it isn't present # then append the new mapping information srcVarMappingSpec = quilt_data.src_var_mapping_spec_create( name=varName, sourceName=src, sourcePattern=srcPat, sourceVariable=srcVar, sourcePatternInstance=srcPatInstance) mappings = quilt_data.src_var_mapping_specs_add( mappings, srcVarMappingSpec) if mappings is not None: quilt_data.pat_spec_set(patternSpec, mappings=mappings) # if the pattern code is specified, set it in the pattern as # a string if self._args.code is not None: # perform first pass parse on the pattern to ensure syntax # call get_pattern_vars from parser, but ignore the result # this will check the syntax codestr = str(self._args.code) # store the code in the pattern quilt_data.pat_spec_set(patternSpec, code=codestr) # syntax parsing may be dependent on variables in the pattern, so # generate the code str after replacing any pattern variables codestr = quilt_data.generate_query_code(patternSpec, None) quilt_parse.get_pattern_src_refs(codestr) # define patternSpec in the query master as a synchronous call # return will be the pattern name with self.GetQueryMasterProxy() as qm: patName = qm.DefinePattern(patternSpec) # print out pattern Name print 'Pattern', patName, ' defined' # return false (prevent event loop from beginning) return False