Example #1
0
 def wait_finished(self):
     """Lock on the ProcJob event signaling its termination.
     """
     if self._event is None:
         raise treetaggerwrapper.TreeTaggerError(
             "Can't wait on a ProcJob with keepjobs False.")
     self._event.wait()
Example #2
0
 def result(self):
     if self._event is None:
         raise treetaggerwrapper.TreeTaggerError(
             "Can't rely on a ProcJob result with keepjobs False.")
     return self._result
Example #3
0
 def finished(self):
     if self._event is None:
         raise treetaggerwrapper.TreeTaggerError(
             "Can't know about a ProcJob state with keepjobs False.")
     return self._finished
Example #4
0
    def __init__(self,
                 workerscount=None,
                 keepjobs=True,
                 wantresult=True,
                 keeptagargs=True,
                 **kwargs):
        """Creation of a new TaggerProcessPoll.

        By default a :class:`TaggerProcessPoll` creates same count of process than there
        are CPU cores on your computer .

        :param workerscount: number of worker process (and taggers) to create.
        :type workerscount: int
        :param keepjobs: poll keep references to Jobs to manage signal of
            their processing and store back processing results — default to True.
        :type keepjobs: bool
        :param wantresult: worker process must return processing result to be stored
            in the job — default to True.
        :type wantresult: bool
        :param keeptagargs: must keep tagging arguments in :class:`ProcJob` synchronization object
            — default to True.
        :type keeptagargs: bool
        :param kwargs: same parameters as :func:`treetaggerwrapper.TreeTagger.__init__`
            for :class:`TreeTagger` creation.
        """
        if workerscount is None:
            workerscount = multiprocessing.cpu_count()
        # Security, we need at least one worker and one tagger.
        if workerscount < 1:
            raise ValueError("Invalid workerscount %s", workerscount)

        if DEBUG_MULTITHREAD:
            logger.debug("Creating TaggerProcessPoll, %d workers",
                         workerscount)

        if wantresult and not keepjobs:
            logger.debug(
                "TaggerProcessPoll can't wantresult without keepjobs.")
            raise treetaggerwrapper.TreeTaggerError(
                "Can't have wantresult without keepjobs.")

        # We create a temporary tagger and tag a small text to be able to detect any
        # problem and raise exception from here (and not in created subprocess).
        tmptagger = treetaggerwrapper.TreeTagger(**kwargs)
        tmptagger.tag_text(tmptagger.dummysequence)
        del tmptagger

        self._keepjobs = keepjobs
        self._wantresult = wantresult
        self._keeptagargs = keeptagargs
        self._stopping = False
        self._workers = []
        self._pendingjobs = multiprocessing.Queue()
        self._finishedjobs = multiprocessing.Queue()
        self._jobsrefs = {}
        self._jobslock = multiprocessing.Lock()

        if self._keepjobs:
            # Following thread retrieve results, store them in corresponding ProcJob, and
            # signal them.
            self._jobsmonitor = threading.Thread(target=self._monitor_main)
            self._jobsmonitor.daemon = True
            self._jobsmonitor.start()
        else:
            self._jobsmonitor = None

        self._build_workers(workerscount, kwargs)

        if DEBUG_MULTITHREAD:
            logger.debug("TaggerProcessPoll ready")