コード例 #1
0
ファイル: quilt_query.py プロジェクト: pietergvw/quilt
    def OnSourceQueryError(self, srcQueryId, exception):
        """Receive a message from the source manager about a problem that
        :param srcQueryId:
        :param exception:
        occurred when running the source query"""

        try:
            # log the source's exception
            # print out the source query id and the message
            logging.error("Source Query error for: " + str(srcQueryId) +
                          " : " + quilt_core.exception_to_string(exception))

            # I guess exception's don't keep their stacktrace over the pyro
            #   boundary
            # logging.exception(exception)

            # lock self to modify member data
            with self._lock:
                # mark source query state with ERROR
                srcQuerySpec = quilt_data.src_query_spec_get(
                    self._srcQuerySpecs, srcQueryId)
                quilt_data.src_query_spec_set(srcQuerySpec,
                                              state=quilt_data.STATE_ERROR)
                qid = quilt_data.query_spec_get(srcQuerySpec, name=True)

            # call query Master's Error function
            with self.GetQueryMasterProxy() as qm:
                qm.OnQueryError(qid, exception)

        except Exception, error2:
            logging.error("Unable to send source query error to query master")
            logging.exception(error2)
コード例 #2
0
ファイル: quilt_interpret.py プロジェクト: shessuper/quilt
def source(srcName, srcPatName, srcPatInstance=None):
    """
    Construct a pattern wrapper from the specified source
    """
    # use the global query spec
    srcQuerySpecs = quilt_data.query_spec_get(_get_query_spec(),
                                              sourceQuerySpecs=True)
    # logging.debug("srcQuerySpecs:\n" + pprint.pformat(srcQuerySpecs))

    # logging.debug("Looking for Source: " + str(srcName) + ", pattern: " + str(srcPatName) 
    #         + ". instance: " + str(srcPatInstance) )


    # iterate the srcQueries
    for srcQueryId, srcQuerySpec in srcQuerySpecs.items():
        # if matching (srcName, patName, and patInstanceName)
        # if None was supplied as one of the parameter, then there
        #   must only be one instance to choose, keyed by 'None'
        curSrc = quilt_data.src_query_spec_get(srcQuerySpec,
                                               source=True)
        curSrcPat = quilt_data.src_query_spec_get(srcQuerySpec,
                                                  srcPatternName=True)
        curSrcPatInst = quilt_data.src_query_spec_tryget(
            srcQuerySpec, srcPatternInstance=True)

        # logging.debug("checking " + curSrc + ", " + curSrcPat + 
        #         ", " + str( curSrcPatInst))

        if (srcName == curSrc and
                    srcPatName == curSrcPat and
                    srcPatInstance == curSrcPatInst):
            # get the global srcResults for that srcQueryID
            srcResults = _get_events(srcQueryId)

            # return a new _pattern with the srcResults as the events
            return _pattern(srcQueryId, srcResults)

    # raise exception if matching srcQuery not found
    raise Exception("Source: " + str(srcName) + ", pattern: " + str(srcPatName)
                    + ". instance: " + str(srcPatInstance) +
                    ", could not be found among the source queries")
コード例 #3
0
ファイル: quilt_smd.py プロジェクト: pietergvw/quilt
    def Query(self, queryId, sourceQuerySpec,
            queryClientName,
            queryClientNamseServerHost,
            queryClientNamseServerPort):
        """
        Query the source, currently hardcoded to behave as calling a cmdline
        tool that outputs source results on each output line
        """

        try:
            with self._lock:
                # set last query member to queryId
                self._lastQuery = queryId

            # _sourceName should not change, set at init, so ok to read
            # without lock
            logging.info("Source Manager: " + str(self._sourceName) +
                         " received query: " + str(queryId))

            # get the sourcePatternSpec from the pattern name in the
            #   sourceQuerySpec
            # pattern spec should be read only and safe to access without
            #   a lock
            srcPatSpecs = quilt_data.src_spec_get(self._sourceSpec,
                sourcePatterns=True)
            srcPatSpec = quilt_data.src_pat_specs_get(srcPatSpecs,
                quilt_data.src_query_spec_get(sourceQuerySpec,
                    srcPatternName=True))

            # get the variables in the query
            srcQueryVars = quilt_data.src_query_spec_get(sourceQuerySpec,
                variables=True)

            # iterate src query variables map, create a simple var name to
            #   var value map
            varNameValueDict = {}
            for srcVarName, srcVarSpec in srcQueryVars.items():
                varNameValueDict[srcVarName] = quilt_data.var_spec_get(
                    srcVarSpec, value=True)

            # create cmd line for the source
            # use the template in the sourcePatternSpec, and use the values
            #   provided for the variables, and environment variables
            replacements = {}
            # replacements = os.environ.copy()
            for k, v in varNameValueDict.items():
                replacements[k] = v

            # "template" was not added as official schema member because it is
            # specific to this command line case
            templateCmd = srcPatSpec['template']
            template = Template(templateCmd)
            logging.info("Ready to use : " + templateCmd +
                         " with replacements: " + str(replacements))
            cmdline = template.safe_substitute(replacements)

            # setup context for the cmdline stdout callback
            srcQueryId = quilt_data.src_query_spec_get(
                sourceQuerySpec, name=True)
            context = {'queryId': queryId, 'srcQueryId': srcQueryId}

            #           sei_core.run_process_lite(cmdline, shell=True,
            #               outFunc=self.OnGrepLine, outObj=context)

            #           I thought run_process apparently has problems
            sei_core.run_process(cmdline, shell=True,
                whichReturn=sei_core.EXITCODE,
                outFunc=self.OnGrepLine, outObj=context, logToPython=False)

            # Set query result events list in query master using query id
            results = []
            with self._lock:
                if queryId in self._sourceResults:
                    if srcQueryId in self._sourceResults[queryId]:
                        results = list(self._sourceResults[queryId][srcQueryId])

            uri = quilt_core.get_uri(queryClientNamseServerHost,
                queryClientNamseServerPort, queryClientName)

            with Pyro4.Proxy(uri) as query:
                query.AppendSourceQueryResults(srcQueryId, results)
                query.CompleteSrcQuery(srcQueryId)

                # catch exception!
        except Exception, error:
            try:

                # attempt to report error to the query client
                uri = quilt_core.get_uri(queryClientNamseServerHost,
                    queryClientNamseServerPort, queryClientName)
                srcQueryId = quilt_data.src_query_spec_get(
                    sourceQuerySpec, name=True)

                with Pyro4.Proxy(uri) as query:
                    query.OnSourceQueryError(srcQueryId, error)

            except Exception, error2:
                logging.error("Unable to send source query error to " +
                              "query master")
                logging.exception(error2)
コード例 #4
0
ファイル: quilt_query.py プロジェクト: pietergvw/quilt
    def OnRegisterEnd(self):
        """
        Retrieve the querySpec from the query master and issue the
        srcQueries to the source
        """
        qid = None
        try:
            # use query id passed in arguments
            if self._args.query_id is not None:
                qid = self._args.query_id[0]
            else:
                raise Exception("Query ID is required")

            # Access the QuiltQuery's registrar's host and port from the config
            config = quilt_core.QuiltConfig()
            rport = config.GetValue("registrar", "port", None, int)
            if rport is not None:
                rport = int(rport)
            rhost = config.GetValue("registrar", "host", None)
            self._registrarPort = rport
            self._registrarHost = rhost

            #   set the state to ACTIVE by calling BeginQuery
            # store pattern and query as a data member
            with self.GetQueryMasterProxy() as qm:
                self._patternSpec, self._querySpec = qm.BeginQuery(qid)

                # get the query spec from query master
                queryState = quilt_data.query_spec_get(self._querySpec,
                                                       state=True)
                if queryState != quilt_data.STATE_ACTIVE:
                    raise Exception("Query: " + qid + ", must be in " +
                                    quilt_data.STATE_ACTIVE +
                                    " state. It is currently in " +
                                    queryState + " state.")

                # iterate the sourceQuerySpec's in srcQueries list
                srcQuerySpecs = quilt_data.query_spec_tryget(
                    self._querySpec, sourceQuerySpecs=True)

                # there are no source query specs specified
                if srcQuerySpecs is None:
                    # so just don't do anything
                    self._processEvents = False
                    return

                self._srcQuerySpecs = srcQuerySpecs
                for srcQuerySpec in srcQuerySpecs.values():
                    # mark the sourceQuery ACTIVE
                    quilt_data.src_query_spec_set(srcQuerySpec,
                                                  state=quilt_data.STATE_ACTIVE)

                    # get proxy to the source manager
                    source = quilt_data.src_query_spec_get(
                        srcQuerySpec, source=True)
                    smgrRec = qm.GetClientRec("smd", source)
                    # TODO make more efficient by recycling proxies to same source
                    # in the case when making multi source queries to same source
                    with query_master.get_client_proxy(smgrRec) as smgr:
                        # query the source by sending it the source query specs as
                        #   asynchronous call
                        Pyro4.async(smgr).Query(
                            qid, srcQuerySpec, self.localname, rhost, rport)
                        # Note: no locking needed 
                        #   asynchronous call, and returning messages not
                        #   processed until this function exits

            self._processEvents = True


        except Exception, error:
            try:
                with self.GetQueryMasterProxy() as qm:
                    qm.OnQueryError(qid, error)
            except Exception, error2:
                logging.error(
                    "Unable to send query startup error to query master")
                logging.exception(error2)
コード例 #5
0
ファイル: quilt_query.py プロジェクト: pietergvw/quilt
    def CompleteSrcQuery(self, srcQueryId):
        """
        string srcQueryId   # id for the source query)
       
        Called by sourceManager when it is done processing the query
        """
        logging.info("Completing source query: " + str(srcQueryId))
        try:
            # NOTE:
            #   Because the member data query specs are not entries are not
            #   modified (Except during initialization) we do not have to lock
            #   before reading from the list.  Also we assume no results
            #   should be appended after a call to complete query so that we
            #   can sort the list of data outside of a lock.  Also see note 
            #   in AppendSourceQueryResults which could change the logic here


            # get the source query from the member data collection of src 
            #   queries using the srcQueryId
            srcQuerySpec = quilt_data.src_query_specs_get(
                self._srcQuerySpecs, srcQueryId)


            # if src query specifies that source returns out of order results
            if not quilt_data.src_query_spec_get(srcQuerySpec, ordered=True):
                results = None
                with self._lock:
                    if srcQueryId in self._srcResults:
                        results = self._srcResults[srcQueryId]

                if results is not None:
                    # sort results  by timestamp using interpret's 'at'
                    #   function
                    results.sort(key=lambda rec: at(rec))


            # acquire lock
            with self._lock:
                # set srcQueries's state to COMPLETED

                # If query is progressing through the system properly it
                # should be an active state when it gets here
                srcQueryState = quilt_data.src_query_spec_get(srcQuerySpec,
                                                              state=True)
                if srcQueryState != quilt_data.STATE_ACTIVE:
                    raise Exception("Source Query is: " + srcQueryState +
                                    ". Can only complete a query that is " +
                                    quilt_data.STATE_ACTIVE)

                quilt_data.src_query_spec_set(srcQuerySpec,
                                              state=quilt_data.STATE_COMPLETED)

            try:


                with self._lock:
                    # Detect if all src queries are completed
                    completed = True
                    for srcQuerySpec in self._srcQuerySpecs.values():
                        srcState = quilt_data.src_query_spec_get(srcQuerySpec,
                                                                 state=True)
                        if (srcState != quilt_data.STATE_COMPLETED and
                                    srcState != quilt_data.STATE_ERROR):
                            completed = False
                            logging.debug("At least " + srcQuerySpec['name'] +
                                          " is still in progress")
                            break

                    # if this was the last source query, 
                    if completed:
                        # get proxy to self
                        with Pyro4.Proxy(self.uri) as selfProxy:
                            # asynchronously call self's CompleteQuery
                            Pyro4.async(selfProxy).CompleteQuery()

            # catch exceptions
            except Exception, error3:
                # log out the exceptions, do not pass along to
                # source as it wasn't his fault
                logging.error(
                    "Unable to properly check for last query")
                logging.exception(error3)

                # set process events flag to false end event loop, allowing
                # query client to exit
                self.SetProcesssEvents(False)
                raise

        # catch exceptions
        except Exception, error2:
            # log out the exceptions, do not pass along to
            # source as it wasn't his fault
            logging.error(
                "Unable to tell query master that the query is complete")
            logging.exception(error2)
            # set process events flag to false end event loop, allowing
            # query client to exit
            self.SetProcesssEvents(False)
            raise