Example #1
0
 def start_stream(self, conf):
     if not self.fact.__init_timeout__():
         returnD(False)
     queries = yield self.fact.db['feeds'].find({'database': 'tweets', 'channel': self.fact.channel}, fields=['query'])
     track = []
     skip = []
     k = 0
     for query in queries:
         q = str(query['query'].encode('utf-8')).lower()
         # queries starting with @ should return only tweets from corresponding user, stream doesn not know how to handle this so skip
         if self.re_twitter_account.match(q):
             continue
         elif " OR " in q or " -" in q or '"' in q or len(q) > 60 or len(q) < 6:
             skip.append(q)
             continue
         track.append(q)
         k += 1
         if k > 395:
             break
     if self.fact.twuser not in track:
         track.append(self.fact.twuser)
     if len(skip):
         self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), hint=True)
     self.log("Start search streaming for: « %s »" % " » | « ".join(track), hint=True)
     conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token)
     # tries to find users corresponding with queries to follow with stream
     users, self.fact.ircclient.twitter['users'] = conn.lookup_users(track, self.fact.ircclient.twitter['users'])
     deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track)
     self.depiler = LoopingCall(self.flush_tweets)
     self.depiler.start(1)
     returnD(True)
    def test_contemporaneous_requests(self):

        '''
        We're going to create two request-response cycles here:

        Cycle 1 will begin.
        Cycle 2 will begin.
        Cycle 2 will return.
        Cycle 1 will return.

        This way, we can prove that the crosstown_traffic created
        by cycle 1 is not resolved by the return of cycle 2.
        '''
        tp = ThreadPool(maxthreads=20)
        tp.start()
        self.addCleanup(tp.stop)


        log.debug("\n\nStarting the two stream stuff.")

        request1 = DummyRequest('r1')
        request1.isSecure = lambda: False
        request1.content = "Nothing really here."
        request1.headers['llamas'] = 'dingo'

        nameSpace.test_case = self

        hr = HendrixWSGIResource(reactor, tp, wsgi_application)
        d1 = deferToThreadPool(reactor, tp, hr.render, request1)

        request2 = DummyRequest('r2')
        request2.isSecure = lambda: False
        request2.content = "Nothing really here."
        request2.headers['llamas'] = 'dingo'

        d2 = deferToThreadPool(reactor, tp, hr.render, request2)

        def woah_stop(failure):
            nameSpace.async_task_was_done.put_nowait(False)
            nameSpace.second_cycle_complete.put_nowait(False)
            nameSpace.ready_to_proceed_with_second_cycle.put_nowait(False)

        d1.addErrback(woah_stop)
        d2.addErrback(woah_stop)

        combo_deferred = gatherResults([d1, d2])

        def wait_for_queue_resolution():
            nameSpace.async_task_was_done.get(True, 3)

        combo_deferred.addCallback(
            lambda _: deferToThreadPool(reactor, tp, wait_for_queue_resolution)
        )

        combo_deferred.addCallback(
            lambda _: self.assertTrue(nameSpace.async_task_was_run)
        )

        return combo_deferred
Example #3
0
    def process_item(self, domain, item):
        args = dict(item)
        random_numbers = args['random_numbers'].split(',')
        for num in random_numbers:
            #log.msg('dispatching : %s' % num,level=log.ERROR)
            trackable = TrackableObject()
            threads.deferToThreadPool(reactor, self.threadpool, trackable.takes_long_time, num,self)

        return item
Example #4
0
    def run(self, threadpool):
        # See if status code is a go
        self.check_status_code_against_no_go_list()
        if self.no_go:
            return

        if self.same_thread:
            self.crosstown_task()
        else:
            deferToThreadPool(reactor, threadpool, self.crosstown_task)
Example #5
0
    def run(self, threadpool=None):
        if self.no_go:
            return

        if not threadpool:
            threadpool = reactor.threadpool or ThreadPool()

        if self.same_thread:
            self.crosstown_task()
        else:
            deferToThreadPool(reactor, threadpool, self.crosstown_task)
Example #6
0
File: server.py Project: 2php/veles
 def _handshake(self, msg, line):
     if self.state.current != 'WAIT':
         self.error("Invalid state for a handshake command: %s",
                    self.state.current)
         self._sendError("Invalid state")
         return
     mysha = self.host.workflow.checksum
     your_sha = msg.get("checksum")
     if not your_sha:
         self.error("Did not receive the workflow checksum")
         self._sendError("Workflow checksum is missing")
         return
     if mysha != your_sha:
         self._sendError("Workflow checksum mismatch: "
                         "expected %s, got %s" % (mysha, your_sha))
         return
     must_reply = False
     msgid = msg.get("id")
     if msgid is None:
         self.id = str(uuid.uuid4())
         must_reply = True
     else:
         self.id = msgid
         if not self.nodes.get(self.id):
             self.warning("Did not recognize the received ID %s")
             must_reply = True
         else:
             self.sendLine({'reconnect': "ok"})
     if must_reply:
         try:
             _, mid, pid = self._extractClientInformation(msg)
         except Exception as e:
             self.error(str(e))
             return
         data = self.host.workflow.generate_initial_data_for_slave(
             SlaveDescription.make(self.nodes[self.id]))
         endpoint = self.host.choose_endpoint(self.id, mid, pid, self.hip)
         self.nodes[self.id]['endpoint'] = self._endpoint = endpoint
         retmsg = {'endpoint': endpoint, 'data': data}
         if not msgid:
             retmsg['id'] = self.id
         retmsg['log_id'] = self.host.launcher.log_id
         self.sendLine(retmsg)
     data = msg.get('data')
     if data is not None:
         threads.deferToThreadPool(
             reactor, self.host.workflow.thread_pool,
             self.host.workflow.apply_initial_data_from_slave,
             data, SlaveDescription.make(self.nodes[self.id])) \
             .addErrback(errback)
         self.nodes[self.id]['data'] = [d for d in data if d is not None]
     self.state.identify()
Example #7
0
    def test_threadLocality(self):
        """
        An 'Record' repr()'d in two separate threads at the same time should
        look the same (i.e. the repr state tracking for '...' should be
        thread-local).
        """
        pool = ThreadPool(2, 2)
        pool.start()
        self.addCleanup(pool.stop)

        class StickyRepr(object):
            """
            This has a __repr__ which will block until a separate thread
            notifies it that it should return.  We use this to create a race
            condition.
            """
            waited = False
            def __init__(self):
                self.set = threading.Event()
                self.wait = threading.Event()
            def __repr__(self):
                if not self.waited:
                    self.set.set()
                    self.wait.wait()
                return 'sticky'
        r = StickyRepr()
        mr = MyRecord(something=1, somethingElse=r)
        d = deferToThreadPool(reactor, pool, repr, mr)
        def otherRepr():
            # First we wait for the first thread doing a repr() to enter its
            # __repr__()...
            r.set.wait()
            # OK, now it's blocked.  Let's make sure that subsequent calls to
            # this repr() won't block.
            r.waited = True
            # Do it!  This is a concurrent repr().
            result = repr(mr)
            # Now we're done, wake up the other repr and let it complete.
            r.wait.set()
            return result
        d2 = deferToThreadPool(reactor, pool, otherRepr)

        def done(xxx_todo_changeme):
            (thread1repr, thread2repr) = xxx_todo_changeme
            knownGood = 'MyRecord(something=1, somethingElse=sticky)'
            # self.assertEquals(thread1repr, thread2repr)
            self.assertEqual(thread1repr, knownGood)
            self.assertEqual(thread2repr, knownGood)
        return gatherResults([d, d2]).addCallback(done)
Example #8
0
  def _retry(self, method_name, call):
    """Retries |call| on transient errors and access token expiration.

    Args:
      method_name (str): name of the remote method, for logging.
      call (func() -> any): a function that makes an RPC call and returns
        result.
    """
    attempts = self.retry_attempt_count
    wait = self.retry_wait_seconds
    while attempts > 0:
      attempts -= 1
      try:
        if not self.started:
          raise NotStartedError('DeferredResource is not started')
        res = yield threads.deferToThreadPool(reactor, self._pool, call)
        defer.returnValue(res)
      except Exception as ex:
        if not self.started:
          raise ex
        if attempts > 0 and is_transient(ex):
          self.log('Transient error while calling %s. '
                   'Will retry in %d seconds.' % (method_name, wait))
          # TODO(nodir), optimize: stop waiting if the resource is stopped.
          yield sleep(wait)
          if not self.started:
            raise ex
          wait = min(wait * 2, 30)
          continue
        self.log('RPC "%s" failed: %s'% (method_name, traceback.format_exc()))
        raise ex
Example #9
0
    def _sendrcv(self, pkts, filter=None, iface=None, nofilter=0):
        self._buildSocket(filter, iface, nofilter)
        self._buildPacketQueues(pkts)
        if not self.last_answer:
            self.last_answer = time.time()

        def sent(cb):
            if self.cthreads < self.mthreads and not self.done:
                pkt = None
                try:
                    pkt = self.outqueue.pop()
                except:
                    self.done = True
                    if not self.recv:
                        self.deferred.callback(None)
                    return
                d = threads.deferToThreadPool(reactor, self.threadpool,
                                    self.sendPkt, pkt)
                d.addCallback(sent)
                return d

        for x in range(self.mthreads):
            try:
                pkt = self.outqueue.pop()
            except:
                self.done = True
                return
            if self.cthreads >= self.mthreads and self.done:
                return
            d = threads.deferToThreadPool(reactor, self.threadpool,
                                self.sendPkt, pkt)
            d.addCallback(sent)
            return d
Example #10
0
    def run(self, function, *args, **kwargs):
        """Run C{function} in a thread.

        The function is run in a thread by a function wrapper, which
        commits the transaction if the function runs successfully. If it
        raises an exception the transaction is aborted.

        If the named variable 'async' is set to False we don't run the
        function in the ThreadPool but in the main thread.

        @param function: The function to run.
        @param args: Positional arguments to pass to C{function}.
        @param kwargs: Keyword arguments to pass to C{function}.
        @return: A C{Deferred} that will fire after the function has been run.
        """
        run_async = kwargs.pop('async', True)
        if run_async:
            # Inline the reactor import here for sake of safeness, in case a
            # custom reactor needs to be installed
            from twisted.internet import reactor
            return deferToThreadPool(
                reactor, self._threadpool, self._wrap,
                function, *args, **kwargs)

        return self._wrap(function, *args, **kwargs)
Example #11
0
def _clean_redir_urls(text, urls={}, first=True, pool=None):
    for res in URL_REGEX.findall(text):
        url00 = res[2].encode('utf-8')
        url0 = url00
        if not url00.startswith('http'):
            if "@" in url00 or url00.startswith('#'):
                continue
            url0 = "http://%s" % url00
        if url0 in urls:
            url1 = urls[url0]
            if url1 == url0:
                continue
        else:
            try:
                url1 = yield deferToThreadPool(reactor, pool, get_url, url0, timeout=8)
                url1 = clean_url(url1)
                urls[url0] = url1
                urls[url1] = url1
            except Exception as e:
                if config.DEBUG and not first:
                    loggerr("trying to resolve %s : %s" % (url0, e))
                if "403" in str(e) or "Error 30" in str(e):
                    urls[url0] = url00
                url1 = url00
        if first and not url1 == url00:
            url1 = url1.replace('http', '##HTTP##')
        try:
            url1 = url1.decode('utf-8')
            text = text.replace(res[0], '%s%s%s' % (res[1], url1, res[4]))
        except:
            if config.DEBUG:
                logerr("encoding %s" % url1)
    if not first:
        text = text.replace('##HTTP##', 'http')
    defer.returnValue((text, urls))
Example #12
0
 def wrapper(*args, **kwargs):
     if not _testing:
         return deferToThreadPool(reactor, _threadpool,
                                  run_in_tx, fun, *args, **kwargs)
     else:
         # No threading during testing
         return defer.execute(run_in_tx, fun, *args, **kwargs)
Example #13
0
 def __safe_call(*args, **kwargs):
     client = None
     keepalive = True
     if "_nokeepalive" in kwargs:
         keepalive = not kwargs.pop("_nokeepalive")
     try:
         corpus = kwargs.pop("corpus")
     except:
         corpus = ""
         fail = format_error("corpus argument missing")
     else:
         fail = format_error({"corpus_id": corpus,
           "ready": self.factory.test_corpus(corpus),
           "status": self.factory.status_corpus(corpus),
           "message": "Corpus is not started"})
         if corpus in self.factory.corpora:
             if keepalive:
                 self.factory.corpora[corpus].lastcall = time.time()
             client = getattr(self.factory.corpora[corpus],
               "client_%s" % type_client)
             if fail["message"]["status"] == "error":
                 fail["message"]["message"] = self.factory.corpora[corpus].error
     if hasattr(client, 'threadpool'):
         if self.factory.test_corpus(corpus):
             return deferToThreadPool(reactor, client.threadpool,
               client.__thrift_call__, call, *args, **kwargs)
         return defer.succeed(fail)
     if self.factory.test_corpus(corpus):
         return client.__thrift_call__(call, *args, **kwargs)
     return fail
Example #14
0
    def render_POST(self, req):
        uid = self.transcriber.next_id()

        tran = req.args['transcript'][0]
        audio = req.args['audio'][0]

        async = True
        if 'async' in req.args and req.args['async'][0] == 'false':
            async = False

        result_promise = threads.deferToThreadPool(
            self.reactor, self.reactor.getThreadPool(),
            self.transcriber.transcribe,
            uid, tran, audio)

        if not async:
            def write_result(result):
                '''Write JSON to client on completion'''
                req.headers["Content-Type"] = "application/json"
                req.write(json.dumps(result, indent=2))
                req.finish()
            result_promise.addCallback(write_result)
            result_promise.addErrback(lambda _: None) # ignore errors

            req.notifyFinish().addErrback(lambda _: result_promise.cancel())

            return NOT_DONE_YET

        req.setResponseCode(FOUND)
        req.setHeader(b"Location", "/transcriptions/%s" % (uid))
        return ''
Example #15
0
 def start_stream(self, conf):
     self.db.authenticate(config.MONGODB['USER'], config.MONGODB['PSWD'])
     queries = list(self.db["feeds"].find({'database': "tweets", 'channel': self.fact.channel}, fields=['query']))
     track = []
     follow = []
     skip = []
     k = 0
     f = 0
     for query in queries:
         q = str(query['query'].encode('utf-8'))
         if self.re_twitter_account.match(q):
             q = q.lstrip('@')
             follow.append(q)
             f += 1
         elif " OR " in q or " -" in q or '"' in q or len(q) > 60:
             skip.append(q)
             continue
         track.append(q)
         k += 1
         if k > 395 or f > 4995:
             break
     user = conf["TWITTER"]["USER"]
     if user not in follow:
         follow.append(user)
     if user not in track:
         track.append(user)
     if len(skip):
         self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), "stream", hint=True)
     self.log("Start search streaming for: « %s »" % " » | « ".join(track), "stream", hint=True)
     conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token)
     users, self.fact.ircclient.twitter_users = conn.search_users(follow, self.fact.ircclient.twitter_users)
     return deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track)
Example #16
0
    def runInteraction(self, interaction, *args, **kw):
        """
        Interact with the database and return the result.

        The 'interaction' is a callable object which will be executed in a
        thread using a pooled connection. It will be passed an L{Transaction}
        object as an argument (whose interface is identical to that of the
        database cursor for your DB-API module of choice), and its results will
        be returned as a L{Deferred}. If running the method raises an
        exception, the transaction will be rolled back. If the method returns a
        value, the transaction will be committed.

        NOTE that the function you pass is *not* run in the main thread: you
        may have to worry about thread-safety in the function you pass to this
        if it tries to use non-local objects.

        @param interaction: a callable object whose first argument is an
            L{adbapi.Transaction}.

        @param *args: additional positional arguments to be passed to
            interaction

        @param **kw: keyword arguments to be passed to interaction

        @return: a Deferred which will fire the return value of
            C{interaction(Transaction(...), *args, **kw)}, or a
            L{twisted.python.failure.Failure}.
        """
        from twisted.internet import reactor
        return threads.deferToThreadPool(reactor, self.threadpool,
                                         self._runInteraction,
                                         interaction, *args, **kw)
Example #17
0
    def get_id(self, model, unique, fields):
        ''' Get an ID from the cache or from the database.
        If doesn't exist - create an item.
        All database operations are done from
        the separate thread
        '''
        assert isInIOThread()

        fval = fields[unique]

        try:
            result = self.cache[model][fval]
            self.counters['hit'][model] += 1
            returnValue(result)
        except KeyError:
            self.counters['miss'][model] += 1

        selectors = {unique: fval}

        result, created = yield deferToThreadPool(
            self.reactor, self.read_pool,
            get_or_create,
            model, fields, **selectors)

        result = result.id

        if created:
            self.counters['db_create'][model] += 1
        else:
            self.counters['db_hit'][model] += 1

        self.cache[model][fval] = result
        returnValue(result)
Example #18
0
def watch_children(kzclient,
                   path, func, allow_session_lost=True, send_event=False,
                   ChildrenWatch=ChildrenWatch):
    """
    Install a Kazoo :obj:`ChildrenWatch` on the given path.

    The given `func` will be called in the reactor thread when any children are
    created or deleted, or if the node itself is deleted.

    Returns a Deferred which usually has no result, but may fail with an
    exception if e.g. the path does not exist.
    """

    def wrapped_func(*args, **kwargs):
        return blockingCallFromThread(kzclient.reactor, func, *args, **kwargs)

    return deferToThreadPool(
        kzclient.reactor,
        kzclient.pool,
        lambda: ChildrenWatch(
            kzclient.kazoo_client,
            path,
            func=wrapped_func,
            allow_session_lost=allow_session_lost,
            send_event=send_event))
Example #19
0
 def runWithConnection(self, func, *args, **kwargs):
     """
     Executes a function with database connection and return the result.
     """
     return threads.deferToThreadPool(reactor, self.threadpool,
                                      self._runWithConnection,
                                      func, *args, **kwargs)
Example #20
0
    def connect(self, masterUrl, deferred):
        """ Connect to RCE.

            @param masterUrl:   URL of Master process.
            @type  masterUrl:   str

            @param deferred:    Deferred which is called as soon as the
                                connection was successfully established.
            @type  deferred:    twisted.internet.defer.Deferred

            @raise:             ConnectionError, if no connection could be
                                established.
        """
        self._connectedDeferred = deferred

        def eb(e):
            print(e.getErrorMessage())

            if self._connectedDeferred:
                self._connectedDeferred.errback(e)
                self._connectedDeferred = None

        d = deferToThreadPool(self._reactor, self._reactor.getThreadPool(),
                              self._getRobotURL, masterUrl)
        d.addCallback(self._robotConnect)
        d.addErrback(eb)
Example #21
0
    def runWithTransaction(self, interaction, transaction, *args, **kw):
        """
        Interact with the database and return the result, using the given transaction and connection.
        Inspired from twisted.enterprise.adbapi.ConnectionPool.runInteraction

        The 'interaction' is a callable object which will be executed in a thread using a pooled connection. 
        It will be passed an C{Transaction} object as an argument (whose interface is identical to that of 
        the database cursor for your DB-API module of choice), and its results will be returned as a Deferred. 
        If running the method raises an exception, the transaction will NOT be rolled back. 
        The transaction will NOT be committed automatically, you have to call commit on the transaction.

        NOTE that the function you pass is *not* run in the main thread: you may have to worry 
        about thread-safety in the function you pass to this if it tries to use non-local objects.

        @param interaction: a callable object whose first argument
            is an L{adbapi.Transaction}.

        @param transaction: a C {dict} containing containing a C{t.e.a.Transaction} and C{t.e.a.Connection} instances

        @param *args: additional positional arguments to be passed
            to interaction

        @param **kw: keyword arguments to be passed to interaction

        @return: a Deferred which will fire the return value of
            'interaction(Transaction(...), *args, **kw)', or a Failure.

        """
        return threads.deferToThreadPool(reactor, Registry.DBPOOL.threadpool,
                                         self._runWithTransaction,
                                         interaction, transaction, *args, **kw)
Example #22
0
    def commit(self, obj):
        """
        Commits current obj transaction.

        @return: A C{Deferred}
        """
        return threads.deferToThreadPool(reactor, Registry.DBPOOL.threadpool, self._commit, obj._txn)
Example #23
0
    def rollback(self, obj):
        """
        Rollback current obj transaction.

        @return: A C{Deferred}
        """
        return threads.deferToThreadPool(reactor, Registry.DBPOOL.threadpool, self._rollback, obj._txn)
Example #24
0
    def doReconfig(self):
        log.msg("beginning configuration update")
        changes_made = False
        failed = False
        try:
            # Run the master.cfg in thread, so that it cas use blocking code
            new_config = yield threads.deferToThreadPool(
                self.reactor, self.reactor.getThreadPool(),
                self.config_loader.loadConfig)
            changes_made = True
            self.config = new_config

            yield self.reconfigServiceWithBuildbotConfig(new_config)

        except config.ConfigErrors as e:
            for msg in e.errors:
                log.msg(msg)
            failed = True

        except Exception:
            log.err(failure.Failure(), 'during reconfig:')
            failed = True

        if failed:
            if changes_made:
                log.msg("WARNING: reconfig partially applied; master "
                        "may malfunction")
            else:
                log.msg("reconfig aborted without making any changes")
        else:
            log.msg("configuration update complete")
Example #25
0
    def runWithConnection(self, func, *args, **kw):
        """
        Execute a function with a database connection and return the result.

        @param func: A callable object of one argument which will be executed
            in a thread with a connection from the pool. It will be passed as
            its first argument a L{Connection} instance (whose interface is
            mostly identical to that of a connection object for your DB-API
            module of choice), and its results will be returned as a
            L{Deferred}. If the method raises an exception the transaction will
            be rolled back. Otherwise, the transaction will be committed.
            B{Note} that this function is B{not} run in the main thread: it
            must be threadsafe.

        @param *args: positional arguments to be passed to func

        @param **kw: keyword arguments to be passed to func

        @return: a L{Deferred} which will fire the return value of
            C{func(Transaction(...), *args, **kw)}, or a
            L{twisted.python.failure.Failure}.
        """
        from twisted.internet import reactor
        return threads.deferToThreadPool(reactor, self.threadpool,
                                         self._runWithConnection,
                                         func, *args, **kw)
Example #26
0
 def _callback(self, msg, msgID):
     """ Callback hook.
     """
     reactor = self._conn._reactor
     d = deferToThreadPool(reactor, reactor.getThreadPool(),
                           self._cb, msg, *self._args)
     d.addCallback(self._response_success, msgID)
     d.addErrback(self._response_failure, msgID)
    def request_same_or_different_thread_thread(self):

        hr = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing)
        request1 = DummyRequest('r1')
        request1.isSecure = lambda: False
        request1.content = "llamas"
        d = deferToThreadPool(reactor, self.tp, hr.render, request1)
        return d
Example #28
0
 def _run_in_thread(self, *args, **kwargs):
     reactor = getattr(self, reactor_name)
     sync = getattr(self, sync_name)
     threadpool = getattr(self, threadpool_name)
     original = getattr(sync, method_name)
     return deferToThreadPool(
         reactor, threadpool, preserve_context(original), *args, **kwargs
     )
 def test_deferredResult(self):
     """
     L{threads.deferToThreadPool} executes the function passed, and
     correctly handles the positional and keyword arguments given.
     """
     d = threads.deferToThreadPool(reactor, self.tp, lambda x, y=5: x + y, 3, y=4)
     d.addCallback(self.assertEqual, 7)
     return d
Example #30
0
    def save(self, obj):
        self.write_queue.put(obj)

        if self.writelock:
            return None
        else:
            return deferToThreadPool(
                self.reactor, self.write_pool, self._do_save)
Example #31
0
 def schedule(self, func, *args, **kwargs):
     self._jobs.append(
         deferToThreadPool(
             reactor, self._thread_pool, func, *args, **kwargs))
Example #32
0
 def wrapper(*args, **kwargs):
     if not _testing:
         return deferToThreadPool(reactor, _threadpool,
                                  run_in_tx, fun, *args, **kwargs)
     else:
         return defer.execute(run_in_tx, fun, *args, **kwargs)
Example #33
0
def deferToDatabase(func, *args, **kwargs):
    """Call `func` in a thread where database activity is permitted."""
    if settings.DEBUG and getattr(settings, "DEBUG_QUERIES", False):
        func = count_queries(log.debug)(func)
    return threads.deferToThreadPool(reactor, reactor.threadpoolForDatabase,
                                     func, *args, **kwargs)
Example #34
0
 def recordStatistics(self, addr, encoding, isUTF8):
     networkID = self.networkIDs[addr]
     d = threads.deferToThreadPool(self.reactor, self.threadPool,
                                   self._recordStatistics, networkID,
                                   encoding, isUTF8)
     d.addErrback(lambda f: self._log.failure("Could not persist", f))
Example #35
0
 def run(self, function, *args, **kwargs):
     return deferToThreadPool(reactor,
                              GLSettings.orm_tp,
                              function,
                              *args,
                              **kwargs)
Example #36
0
 def _defer_to_thread(self, f, *args, **kwargs):
     return deferToThreadPool(
         self._reactor, self._tpool, f, *args, **kwargs
     )
Example #37
0
 def do(self, callable, *args, **kwargs):
     return threads.deferToThreadPool(reactor, self, self.__thd, False,
                                      callable, args, kwargs)
Example #38
0
 def run(function, *args, **kwargs):
     """
     Defer provided function to thread
     """
     return deferToThreadPool(reactor, GLSettings.orm_tp, function, *args,
                              **kwargs)
Example #39
0
    def startService(self):
        assert not self._already_started, "can only start the master once"
        self._already_started = True

        log.msg("Starting BuildMaster -- buildbot.version: %s" %
                buildbot.version)

        # Set umask
        if self.umask is not None:
            os.umask(self.umask)

        # first, apply all monkeypatches
        monkeypatches.patch_all()

        # we want to wait until the reactor is running, so we can call
        # reactor.stop() for fatal errors
        d = defer.Deferred()
        self.reactor.callWhenRunning(d.callback, None)
        yield d

        startup_succeed = False
        try:
            # load the configuration file, treating errors as fatal
            try:
                # run the master.cfg in thread, so that it can use blocking
                # code
                self.config = yield threads.deferToThreadPool(
                    self.reactor, self.reactor.getThreadPool(),
                    self.config_loader.loadConfig)

            except config.ConfigErrors as e:
                log.msg("Configuration Errors:")
                for msg in e.errors:
                    log.msg("  " + msg)
                log.msg("Halting master.")
                self.reactor.stop()
                return
            except Exception:
                log.err(failure.Failure(), 'while starting BuildMaster')
                self.reactor.stop()
                return

            # set up services that need access to the config before everything
            # else gets told to reconfig
            try:
                yield self.db.setup()
            except exceptions.DatabaseNotReadyError:
                # (message was already logged)
                self.reactor.stop()
                return

            self.mq.setup()

            if hasattr(signal, "SIGHUP"):

                def sighup(*args):
                    eventually(self.reconfig)

                signal.signal(signal.SIGHUP, sighup)

            if hasattr(signal, "SIGUSR1"):

                def sigusr1(*args):
                    eventually(self.botmaster.cleanShutdown)

                signal.signal(signal.SIGUSR1, sigusr1)

            # get the masterid so other services can use it in
            # startup/reconfig.  This goes directly to the DB since the data
            # API isn't initialized yet, and anyway, this method is aware of
            # the DB API since it just called its setup function
            self.masterid = yield self.db.masters.findMasterId(name=self.name)

            # mark this master as stopped, in case it crashed before
            yield self.data.updates.masterStopped(name=self.name,
                                                  masterid=self.masterid)

            # call the parent method
            yield service.AsyncMultiService.startService(self)

            # We make sure the housekeeping is done before configuring in order to cleanup
            # any remaining claimed schedulers or change sources from zombie
            # masters
            yield self.data.updates.expireMasters(forceHouseKeeping=True)

            # give all services a chance to load the new configuration, rather
            # than the base configuration
            yield self.reconfigServiceWithBuildbotConfig(self.config)

            # Mark the master as active now that mq is running
            yield self.data.updates.masterActive(name=self.name,
                                                 masterid=self.masterid)

            # Start the heartbeat timer
            yield self.masterHeartbeatService.setServiceParent(self)

            # send the statistics to buildbot.net, without waiting
            self.sendBuildbotNetUsageData()
            startup_succeed = True
        except Exception:
            f = failure.Failure()
            log.err(f, 'while starting BuildMaster')
            self.reactor.stop()

        finally:
            if startup_succeed:
                log.msg("BuildMaster is running")
            else:
                log.msg("BuildMaster startup failed")

            self._master_initialized = True
Example #40
0
 def run(self, function, *args, **kwargs):
     return deferToThreadPool(reactor, get_thread_pool(), function, *args,
                              **kwargs)
Example #41
0
 def _recordWithDN(self, dn):
     return deferToThreadPool(reactor, self.threadpool,
                              self._recordWithDN_inThread, dn)
Example #42
0
 def deferToThread(self, reactor, meth, *args, **kwargs):
     return threads.deferToThreadPool(reactor, self._threadPool, meth,
                                      *args, **kwargs)
Example #43
0
    def process_tweets(self, feed, source, query=None, pagecount=0):
        # handle tweets from icerocket or topsy fake rss
        nexturl = ""
        try:
            elements = feed.entries
        except:
        # handle tweets from Twitter API
            if isinstance(feed, list) and len(feed):
                elements = feed
            elif isinstance(feed, dict) and "nexturl" in feed:
                nexturl = feed["nexturl"]
                elements = feed["tweets"]
            else:
                returnD(False)
        if query:
            source = "%s https://api.twitter.com/api/1.1/search/tweets.json?q=%s" % (source, query)
        ids = []
        hashs = []
        tweets = []
        fresh = True
        for i in elements:
            try:
                date = datetime.fromtimestamp(time.mktime(i.get('published_parsed', '')) - 4*60*60)
            except:
                if i.get('created_at', '') == "now":
                    date = datetime.now()
                else:
                    #date = datetime.strptime(i.get('created_at', ''), '%a %b %d %H:%M:%S +0000 %Y') + timedelta(hours=2)
                    date = parse_date(i.get('created_at', ''))
            if datetime.today() - date > timedelta(hours=config.BACK_HOURS):
                fresh = False
                break
            tweet, self.fact.cache_urls = yield clean_redir_urls(i.get('title', '').replace('\n', ' '), self.fact.cache_urls)
            link = i.get('link', '')
            res = re_tweet_url.search(link)
            if res:
                user = res.group(1)
                tid = long(res.group(2))
                ids.append(tid)
                tw = {'_id': "%s:%s" % (self.fact.channel, tid), 'channel': self.fact.channel, 'id': tid, 'user': user.lower(), 'screenname': user, 'message': tweet, 'uniq_rt_hash': uniq_rt_hash(tweet), 'link': link, 'date': date, 'timestamp': datetime.today(), 'source': source}
                tw = grab_extra_meta(i, tw)
                tweets.append(tw)
        # Delay displaying to avoid duplicates from the stream
        if source != "mystream" and not self.fact.tweets_search_page:
            yield deferredSleep()
        existings = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'id': {'$in': ids}}, fields=['_id'], filter=sortdesc('id'))
        existing = [t['_id'] for t in existings]
        news = [t for t in tweets if t['_id'] not in existing]
        if not news:
            returnD(False)
        good = []
        news.sort(key=itemgetter('id'))
        if fresh and not source.startswith("my") and len(news) > len(elements) / 2:
            if query and nexturl and pagecount < 3*self.fact.back_pages_limit:
                deferToThreadPool(reactor, self.threadpool, reactor.callLater, 15, self.start_twitter_search, [query], max_id=nexturl, pagecount=pagecount+1)
            elif not query and nexturl and "p=%d" % (self.fact.back_pages_limit+1) not in nexturl and "page=%s" % (2*self.fact.back_pages_limit) not in nexturl:
                deferToThreadPool(reactor, self.threadpool, reactor.callLater, 41, self.start_web, nexturl)
            elif not query and not nexturl and int(source[-1:]) <= self.fact.back_pages_limit:
                deferToThreadPool(reactor, self.threadpool, reactor.callLater, 41, self.start_web, next_page(source))
        if self.fact.displayRT:
            good = news
        else:
            hashs = [t['uniq_rt_hash'] for t in news if t['uniq_rt_hash'] not in hashs]
            existings = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'uniq_rt_hash': {'$in': hashs}}, fields=['uniq_rt_hash'], filter=sortdesc('id'))
            existing = [t['uniq_rt_hash'] for t in existings]

            for t in news:
                if self.fact.twuser == t['user'] or t['uniq_rt_hash'] not in existing or (self.fact.displayMyRT and "@%s: " % self.fact.twuser in t['message'].lower()):
                    existing.append(t['uniq_rt_hash'])
                    good.append(t)
        if config.DEBUG:
            nb_rts_str = ""
            nb_rts = len(news) - len(good)
            if nb_rts:
                nb_rts_str = " (%s RTs filtered)" % nb_rts
            self.log("Displaying %s tweets%s" % (len(good), nb_rts_str), hint=True)
        if self.fact.status != "closed":
            for t in good:
                msg = "%s: %s" % (t['screenname'].encode('utf-8'), format_tweet(t))
                self.fact.ircclient._send_message(msg, self.fact.channel)
        for t in news:
            yield self.fact.db['tweets'].save(t, safe=True)
        returnD(True)
Example #44
0
 def _wrap(*args, **kwargs):
     return deferToThreadPool(reactor, retval._pool,
                              _et(_user_callable), *args,
                              **kwargs)
Example #45
0
 def run(self, function, *args, **kwargs):
     return deferToThreadPool(reactor, State.orm_tp, function, *args,
                              **kwargs)
Example #46
0
def test1():
    d = deferToThreadPool(reactor, tp, test2)
    d.addCallback(cb)
Example #47
0
 def do_with_engine(self, callable, *args, **kwargs):
     return threads.deferToThreadPool(self.reactor, self._pool,
                                      self.__thd, True, callable, args, kwargs)
Example #48
0
 def new_deferToThreadPool(reactor, threadpool, f, *args, **kwargs):
     """Variant of Twisted's that wraps all functions in `synchronous`."""
     func = f if ISynchronous.providedBy(f) else synchronous(f)
     return deferToThreadPool(reactor, threadpool, func, *args, **kwargs)
Example #49
0
 def _stop(self):
     self._stop_evt = None
     threads.deferToThreadPool(self.reactor, self._pool,
                               self.engine.dispose)
     self._pool.stop()
     self.running = False
Example #50
0
 def newfunc(*a, **kw):
     """I call blocking python code in a seperate thread.
        I return a deferred and will fire callbacks or errbacks.
     """
     return threads.deferToThreadPool(R, pool, func, *a, **kw)
Example #51
0
 def do(self, callable, *args, **kwargs):
     ret = yield threads.deferToThreadPool(self.reactor, self._pool,
                                           self.__thd, False, callable,
                                           args, kwargs)
     return ret
Example #52
0
 def runWithConnection(func, *args, **kwargs):
     return threads.deferToThreadPool(pool._reactor, pool.threadpool,
                                      pool._runWithConnection, func, *args,
                                      **kwargs)
Example #53
0
 def perform_job_to_thread(self, job):
     d = threads.deferToThreadPool(reactor, self._thread_pool, job.perform)
     d.addCallback(self.callback_perform_job)
     d.addErrback(self.errback_perform_job, job=job)
Example #54
0
 def runInteraction(interaction, *args, **kwargs):
     return threads.deferToThreadPool(pool._reactor, pool.threadpool,
                                      pool._runInteraction, interaction,
                                      *args, **kwargs)
Example #55
0
def deferToDatabase(func, *args, **kwargs):
    """Call `func` in a thread where database activity is permitted."""
    return threads.deferToThreadPool(reactor, reactor.threadpoolForDatabase,
                                     func, *args, **kwargs)
Example #56
0
 def deferToThread(f, *args, **kwargs):
     return threads.deferToThreadPool(self.reactor,
                                      self.reactor.getThreadPool(), f,
                                      *args, **kwargs)
Example #57
0
 def run(function, *args, **kwargs):
     """
     Defer provided function to thread
     """
     return deferToThreadPool(reactor, transact.tp, function, *args,
                              **kwargs)