예제 #1
0
    def AppendQueryResults(self, queryId, results):
        """
        Append the specified eventList to the specified queryId
        """
        try:
            # acquire lock
            with self.lock:
                # Get query from Q
                querySpec = quilt_data.query_specs_get(self._queries, queryId)
                # Try to get any existing results then
                #   append the results into the query spec

                # set the results into the query spec
                existingEvents = quilt_data.query_spec_tryget(
                    querySpec, results=True)

                if existingEvents is None:
                    existingEvents = []

                # append the results into the query spec
                quilt_data.query_spec_set(querySpec,
                    results=(existingEvents + results))

        except Exception, error:
            try:
                # log exception here, because there is no detail in it once we 
                #   pass it across pyro
                logging.error("Unable append results for query: " +
                              str(queryId))
                logging.exception(error)
            finally:
                # throw the exception back over to the calling process
                raise error
예제 #2
0
    def BeginQuery(self, queryId):
        """
        string queryID          # the query we are interested in
        
        Called by quilt_query to Move a query into the an active state,
        and return a copy of the query
        """

        try:
            logging.info("Beginning query: " + str(queryId))
            patternSpec = None

            # lock self
            with self.lock:

                # get the query from the Q
                # if can't get it raise exception
                querySpec = quilt_data.query_specs_get(self._queries, queryId)

                queryState = quilt_data.query_spec_get(querySpec, state=True)
                # if query state is not expected INITIALIZED state
                if queryState != quilt_data.STATE_INITIALIZED:
                    # raise exception
                    raise Exception("Query: " + str(queryId) +
                                    " is not ready for processing")

                # move query to ACTIVE state
                quilt_data.query_spec_set(querySpec,
                    state=quilt_data.STATE_ACTIVE)

                # create a  copy of the query
                querySpec = querySpec.copy()

                # get a copy of the patternSpec
                patternSpec = quilt_data.pat_specs_get(self._patterns,
                    quilt_data.query_spec_get(querySpec, patternName=True))
                patternSpec = patternSpec.copy()

            # returning copy because we don't want to stay locked when asking
            #   pyro to marshall across process bounds.  Not that I know for
            #   a fact that that won't work, but it sounds like a bad idea
            # return the pattern and query spec 
            return patternSpec, querySpec

        except Exception, error:
            try:
                # log exception here, because there is no detail in it once we 
                #   pass it across pyro
                logging.error("Unable begin query: " + str(queryId))
                logging.exception(error)
            finally:
                # throw the exception back over to the calling process
                raise error
예제 #3
0
파일: msg2self.py 프로젝트: shessuper/quilt
    def OnRegisterEnd(self):
        """After registration is complete submit the query to the 
        query master"""
        # create a partial query spec dictionary
        #   set pattern name from args
        #   set notification address in spec
        #   set state as UNINITIALIZED
        querySpec = quilt_data.query_spec_create(
            name='new ' + self._args.pattern,
            state=quilt_data.STATE_UNINITIALIZED,
            patternName=self._args.pattern,
            notificationEmail=self._args.notifcation_email)

        #   set variables/values from args
        if self._args.variable is not None and len(self._args.variable) > 0:
            variables = quilt_data.var_specs_create()
            for v in self._args.variable:
                vname = v[0]
                vval = v[1]
                quilt_data.var_specs_add(variables,
                    quilt_data.var_spec_create(name=vname, value=vval))
            quilt_data.query_spec_set(querySpec, variables=variables)

        logging.info('Submiting query: ' + pprint.pformat(querySpec))

        # call remote method asyncronysly, this will return right away
        Pyro4.async(self._qm).Query(self._remotename, querySpec)

        # Validate query will be remote called from query master

        # return True to allow event loop to start running, which
        # should soon recieve a validation callback from query master

        import time

        ns = Pyro4.locateNS()
        uri = ns.lookup(self._remotename)

        for i in range(10):
            print "loop being"
            time.sleep(1)
            with Pyro4.Proxy(uri) as p:
                print "Sending", i
                Pyro4.async(p).TestMsg(i)
                print "sent", i
            print "loop end"

        return True
예제 #4
0
    def _try_move_query_to_hist(self, queryId, state):
        """Private function, only call with self_lock engaged"""
        querySpec = quilt_data.query_specs_trydel(self._queries, queryId)
        # else find queryId in history
        if querySpec is None:
            querySpec = quilt_data.query_specs_tryget(self._history,
                queryId)
        else:
            # move query spec from q to history member collection
            quilt_data.query_specs_add(self._history, querySpec)

        if querySpec is not None:
            # mark the query as the specified state
            quilt_data.query_spec_set(querySpec, state=state)

        return querySpec
예제 #5
0
    def OnRegisterEnd(self):
        """After registration is complete submit the query to the 
        query master"""
        # create a partial query spec dictionary
        #   set pattern name from args
        #   set notification address in spec
        #   set state as UNINITIALIZED
        querySpec = quilt_data.query_spec_create(
            name='new ' + self._args.pattern,
            state=quilt_data.STATE_UNINITIALIZED,
            patternName=self._args.pattern,
            notificationEmail=self._args.notification_email)

        #   set variables/values from args
        if self._args.variable is not None and len(self._args.variable) > 0:
            variables = quilt_data.var_specs_create()
            for v in self._args.variable:
                vname = v[0]
                vval = v[1]
                quilt_data.var_specs_add(variables,
                    quilt_data.var_spec_create(name=vname, value=vval))
            quilt_data.query_spec_set(querySpec, variables=variables)

        logging.info('Submitting query: ' + pprint.pformat(querySpec))

        # call remote method asynchronously, this will return right away
        with self.GetQueryMasterProxy() as qm:
            Pyro4.async(qm).Query(self._remotename, querySpec)

        logging.info('Query Submitted')

        # Validate query will be remote called from query master

        # return True to allow event loop to start running, which
        # should soon receive a validation callback from query master
        return True
예제 #6
0
    def Query(self, submitterNameKey, querySpec):
        """
        string Query(                       # return the ID of the query
            string submitterNameKey         # key that the submitter received
                                            #   when it was registered with this
                                            #   master
            dict querySpec                  # the details of the query
                                            #   necessary from the user
        )
        """


        # qid must be set because we use it when reporting the error
        if submitterNameKey is not None:
            qid = str(submitterNameKey) + " unnamed query"
        else:
            qid = "Unknown query"

        # try the following 
        try:

            # a slightly better name for the query id
            tqid = quilt_data.query_spec_tryget(querySpec, name=True)
            if tqid is not None:
                qid = tqid

            # placeholder query spec to reserve the unique qid
            tmpQuerySpec = querySpec.copy()

            # get pattern name from the query spec
            patternName = quilt_data.query_spec_get(
                querySpec, patternName=True)

            # acquire lock
            with self.lock:
                # copy the patternSpec the query references
                patternSpec = quilt_data.pat_specs_get(
                    self._patterns, patternName).copy()
                # generate a query id
                baseqid = submitterNameKey + "_" + patternName
                i = 0
                qid = baseqid
                while qid in self._queries or qid in self._history:
                    i += 1
                    qid = baseqid + "_" + str(i)

                # store querySpec in Q to reserve query id
                quilt_data.query_spec_set(tmpQuerySpec, name=qid)
                quilt_data.query_specs_add(self._queries, tmpQuerySpec)

            # this the query spec we will be manipulating, give it the id
            # that we generate, we will eventually replace the placeholder
            # in the member q
            quilt_data.query_spec_set(querySpec, name=qid)

            code = quilt_data.pat_spec_tryget(
                patternSpec, code=True)

            # using the set of sources described in the query code if
            #   they exist, or the ones described by mappings otherwise
            # group variable mapping's by target source and source pattern
            #   store in local collection, 
            #   map{source:
            #       map{sourcePattern:
            #           map{sourcePatternInstance:
            #               map{srcVar:Var}}}}

            #   store in local collection.  We later want to iterate the
            #   source mangers efficiently, so we pre process a varDict here
            #   which provides a direct mapping from srcVariables to 
            #   queryVariables, grouped by sources and patterns and pattern 
            #   instances

            # if pattern code specified
            #   parse the pattern text, and get set of variables mentioned
            #   in the pattern code
            if code is not None:
                # pattern code should be generated to account for any variables
                #   that will need to be substituted with default value to
                #   make the code syntactically correct
                code = quilt_data.generate_query_code(patternSpec,None)
                # get the variables mentioned in the pattern code
                varDict = quilt_parse.get_pattern_src_refs(code)
                append = False
            else:
                varDict = var_dict.create()
                append = True

            patVarSpecs = quilt_data.pat_spec_tryget(
                patternSpec, variables=True)

            mappings = quilt_data.pat_spec_tryget(
                patternSpec, mappings=True)


            #logging.info("got patVarSpecs: " + str(patVarSpecs))
            #logging.info("got mappings: " + str(mappings))
            # if code was specified, only bother making the map for
            #   sources referenced in the code, otherwise use anything
            #   that was declared in the pattern spec mappings 
            if patVarSpecs is not None and mappings is not None:

                patVars = patVarSpecs.keys()

                # logging.debug("Iterating variables: " + str(patVars))

                for m in mappings:
                    varName = quilt_data.src_var_mapping_spec_get(
                        m, name=True)
                    if varName not in patVars:
                        continue

                    src = quilt_data.src_var_mapping_spec_get(
                        m, sourceName=True)
                    pat = quilt_data.src_var_mapping_spec_get(
                        m, sourcePattern=True)
                    var = quilt_data.src_var_mapping_spec_get(
                        m, sourceVariable=True)
                    ins = quilt_data.src_var_mapping_spec_tryget(
                        m, sourcePatternInstance=True)

                    logging.debug("considering: varDict[" + var + "] = " +
                                  str(varName))
                    # set append to appropriate value so that the varDict does
                    # not grow from mappings alone
                    var_dict.set_var(varDict, src, pat, ins, var, varName,
                        append=append)

            else:
                logging.info("No query variables were specified")

            logging.info("got varDict:\n " + pprint.pformat(varDict))

            varSpecs = quilt_data.query_spec_tryget(
                querySpec, variables=True)


            # initialize list of srcQueries
            srcQuerySpecs = None

            # iterate the collection of sources, and build collection of 
            # srcQuerySpecs for each source
            for source in varDict.keys():

                # use variable mapping's source name to get proxy to 
                #   that source manager
                with get_client_proxy_from_type_and_name(
                        self, "smd", source) as srcMgr:

                    patterns = srcMgr.GetSourcePatterns()

                    # iterate the collection of sourcePatterns for current 
                    #   source
                    for patternName in patterns:
                        # NOTE we could have decided to build a temp map
                        # of all patterns and close the source manager 
                        # connection sooner, but it will have no practical
                        # effect to keep the source manager connection a
                        # might longer while doing the below processing

                        # skip the creation of source queries for source 
                        #   patterns not references in the query
                        if patternName not in varDict[source]:
                            continue

                        # get the sourcePatternSpec from the proxy 
                        srcPatSpec = srcMgr.GetSourcePattern(patternName)

                        # create a query spec objects
                        curSrcQuerySpecs = create_src_query_specs(
                            srcPatSpec, varDict, varSpecs, patVarSpecs,
                            qid, source, patternName)

                        for srcQuerySpec in curSrcQuerySpecs.values():
                            # append completed sourceQuerySpec to querySpec
                            srcQuerySpecs = quilt_data.src_query_specs_add(
                                srcQuerySpecs, srcQuerySpec)


            # store sourceQueries in the querySpec
            if srcQuerySpecs is not None:
                quilt_data.query_spec_set(querySpec,
                    sourceQuerySpecs=srcQuerySpecs)

            # use querySpec and srcQuery list
            # to create a validation string                    
            msg = {'Query to run': querySpec,
                   'Sources to be queried': varDict.keys()}

            validStr = pprint.pformat(msg)


            # ask submitter to validate the source Queries
            # get_client function is threadsafe, returns with the lock off
            with get_client_proxy_from_type_and_name(
                    self, "qsub", submitterNameKey) as submitter:

                # call back to the submitter to get validation
                validated = submitter.ValidateQuery(validStr, qid)

            # if submitter refuses to validate, 
            if not validated:
                logging.info("Submitter: " + submitterNameKey +
                             " did not validate the query: " + qid)
                # acquire lock remove query id from q
                # delete the query record, it was determined invalid
                # return early
                with self.lock:
                    quilt_data.query_specs_del(self._queries, qid)
                return

            logging.info("Submitter: " + submitterNameKey +
                         "did validate the query: " + qid)

            # acquire lock
            with self.lock:
                # store querySpec state as INITIALIZED, and place 
                # validated contents in member data
                quilt_data.query_spec_set(querySpec,
                    state=quilt_data.STATE_INITIALIZED)
                quilt_data.query_specs_add(self._queries,
                    querySpec)

            # Process query...
            # get the path to the current directory
            # formulate the command line for quilt_query
            queryCmd = [
                os.path.join(os.path.dirname(__file__), "quilt_query.py"),
                qid]
            if self._args.log_level is not None:
                queryCmd.append("--log-level")
                queryCmd.append(self._args.log_level)

            if self._args.log_file is not None:
                queryCmd.append("--log-file")
                queryCmd.append(self._args.log_file)

                # use subprocess module to fork off the process
                # script, pass the query ID
            #           sei_core.run_process("sleep 5 && " + ' '.join(queryCmd) + " &",shell=True)
            subprocess.Popen(queryCmd)

        # catch exception! 
        except Exception, error:

            logging.exception(error)

            try:
                # submit launched this call asynchronously and must be made
                # aware of the unexpected error
                # call submit's OnSubmitProblem
                logging.info("Attempting to get proxy for error report")
                with get_client_proxy_from_type_and_name(
                        self, "qsub", submitterNameKey) as submitter:
                    logging.info("Attempting to send error to submitter")
                    Pyro4.async(submitter).OnSubmitProblem(qid, error)

            except Exception, error2:
                # stuff is going horribly wrong, we couldn't notify submitter
                logging.error("Unable to notify submitter: " +
                              str(submitterNameKey) +
                              " of error when submitting query: " + str(qid))
                logging.exception(error2)