def start_stream(self, conf): if not self.fact.__init_timeout__(): returnD(False) queries = yield self.fact.db['feeds'].find({'database': 'tweets', 'channel': self.fact.channel}, fields=['query']) track = [] skip = [] k = 0 for query in queries: q = str(query['query'].encode('utf-8')).lower() # queries starting with @ should return only tweets from corresponding user, stream doesn not know how to handle this so skip if self.re_twitter_account.match(q): continue elif " OR " in q or " -" in q or '"' in q or len(q) > 60 or len(q) < 6: skip.append(q) continue track.append(q) k += 1 if k > 395: break if self.fact.twuser not in track: track.append(self.fact.twuser) if len(skip): self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), hint=True) self.log("Start search streaming for: « %s »" % " » | « ".join(track), hint=True) conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token) # tries to find users corresponding with queries to follow with stream users, self.fact.ircclient.twitter['users'] = conn.lookup_users(track, self.fact.ircclient.twitter['users']) deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track) self.depiler = LoopingCall(self.flush_tweets) self.depiler.start(1) returnD(True)
def test_contemporaneous_requests(self): ''' We're going to create two request-response cycles here: Cycle 1 will begin. Cycle 2 will begin. Cycle 2 will return. Cycle 1 will return. This way, we can prove that the crosstown_traffic created by cycle 1 is not resolved by the return of cycle 2. ''' tp = ThreadPool(maxthreads=20) tp.start() self.addCleanup(tp.stop) log.debug("\n\nStarting the two stream stuff.") request1 = DummyRequest('r1') request1.isSecure = lambda: False request1.content = "Nothing really here." request1.headers['llamas'] = 'dingo' nameSpace.test_case = self hr = HendrixWSGIResource(reactor, tp, wsgi_application) d1 = deferToThreadPool(reactor, tp, hr.render, request1) request2 = DummyRequest('r2') request2.isSecure = lambda: False request2.content = "Nothing really here." request2.headers['llamas'] = 'dingo' d2 = deferToThreadPool(reactor, tp, hr.render, request2) def woah_stop(failure): nameSpace.async_task_was_done.put_nowait(False) nameSpace.second_cycle_complete.put_nowait(False) nameSpace.ready_to_proceed_with_second_cycle.put_nowait(False) d1.addErrback(woah_stop) d2.addErrback(woah_stop) combo_deferred = gatherResults([d1, d2]) def wait_for_queue_resolution(): nameSpace.async_task_was_done.get(True, 3) combo_deferred.addCallback( lambda _: deferToThreadPool(reactor, tp, wait_for_queue_resolution) ) combo_deferred.addCallback( lambda _: self.assertTrue(nameSpace.async_task_was_run) ) return combo_deferred
def process_item(self, domain, item): args = dict(item) random_numbers = args['random_numbers'].split(',') for num in random_numbers: #log.msg('dispatching : %s' % num,level=log.ERROR) trackable = TrackableObject() threads.deferToThreadPool(reactor, self.threadpool, trackable.takes_long_time, num,self) return item
def run(self, threadpool): # See if status code is a go self.check_status_code_against_no_go_list() if self.no_go: return if self.same_thread: self.crosstown_task() else: deferToThreadPool(reactor, threadpool, self.crosstown_task)
def run(self, threadpool=None): if self.no_go: return if not threadpool: threadpool = reactor.threadpool or ThreadPool() if self.same_thread: self.crosstown_task() else: deferToThreadPool(reactor, threadpool, self.crosstown_task)
def _handshake(self, msg, line): if self.state.current != 'WAIT': self.error("Invalid state for a handshake command: %s", self.state.current) self._sendError("Invalid state") return mysha = self.host.workflow.checksum your_sha = msg.get("checksum") if not your_sha: self.error("Did not receive the workflow checksum") self._sendError("Workflow checksum is missing") return if mysha != your_sha: self._sendError("Workflow checksum mismatch: " "expected %s, got %s" % (mysha, your_sha)) return must_reply = False msgid = msg.get("id") if msgid is None: self.id = str(uuid.uuid4()) must_reply = True else: self.id = msgid if not self.nodes.get(self.id): self.warning("Did not recognize the received ID %s") must_reply = True else: self.sendLine({'reconnect': "ok"}) if must_reply: try: _, mid, pid = self._extractClientInformation(msg) except Exception as e: self.error(str(e)) return data = self.host.workflow.generate_initial_data_for_slave( SlaveDescription.make(self.nodes[self.id])) endpoint = self.host.choose_endpoint(self.id, mid, pid, self.hip) self.nodes[self.id]['endpoint'] = self._endpoint = endpoint retmsg = {'endpoint': endpoint, 'data': data} if not msgid: retmsg['id'] = self.id retmsg['log_id'] = self.host.launcher.log_id self.sendLine(retmsg) data = msg.get('data') if data is not None: threads.deferToThreadPool( reactor, self.host.workflow.thread_pool, self.host.workflow.apply_initial_data_from_slave, data, SlaveDescription.make(self.nodes[self.id])) \ .addErrback(errback) self.nodes[self.id]['data'] = [d for d in data if d is not None] self.state.identify()
def test_threadLocality(self): """ An 'Record' repr()'d in two separate threads at the same time should look the same (i.e. the repr state tracking for '...' should be thread-local). """ pool = ThreadPool(2, 2) pool.start() self.addCleanup(pool.stop) class StickyRepr(object): """ This has a __repr__ which will block until a separate thread notifies it that it should return. We use this to create a race condition. """ waited = False def __init__(self): self.set = threading.Event() self.wait = threading.Event() def __repr__(self): if not self.waited: self.set.set() self.wait.wait() return 'sticky' r = StickyRepr() mr = MyRecord(something=1, somethingElse=r) d = deferToThreadPool(reactor, pool, repr, mr) def otherRepr(): # First we wait for the first thread doing a repr() to enter its # __repr__()... r.set.wait() # OK, now it's blocked. Let's make sure that subsequent calls to # this repr() won't block. r.waited = True # Do it! This is a concurrent repr(). result = repr(mr) # Now we're done, wake up the other repr and let it complete. r.wait.set() return result d2 = deferToThreadPool(reactor, pool, otherRepr) def done(xxx_todo_changeme): (thread1repr, thread2repr) = xxx_todo_changeme knownGood = 'MyRecord(something=1, somethingElse=sticky)' # self.assertEquals(thread1repr, thread2repr) self.assertEqual(thread1repr, knownGood) self.assertEqual(thread2repr, knownGood) return gatherResults([d, d2]).addCallback(done)
def _retry(self, method_name, call): """Retries |call| on transient errors and access token expiration. Args: method_name (str): name of the remote method, for logging. call (func() -> any): a function that makes an RPC call and returns result. """ attempts = self.retry_attempt_count wait = self.retry_wait_seconds while attempts > 0: attempts -= 1 try: if not self.started: raise NotStartedError('DeferredResource is not started') res = yield threads.deferToThreadPool(reactor, self._pool, call) defer.returnValue(res) except Exception as ex: if not self.started: raise ex if attempts > 0 and is_transient(ex): self.log('Transient error while calling %s. ' 'Will retry in %d seconds.' % (method_name, wait)) # TODO(nodir), optimize: stop waiting if the resource is stopped. yield sleep(wait) if not self.started: raise ex wait = min(wait * 2, 30) continue self.log('RPC "%s" failed: %s'% (method_name, traceback.format_exc())) raise ex
def _sendrcv(self, pkts, filter=None, iface=None, nofilter=0): self._buildSocket(filter, iface, nofilter) self._buildPacketQueues(pkts) if not self.last_answer: self.last_answer = time.time() def sent(cb): if self.cthreads < self.mthreads and not self.done: pkt = None try: pkt = self.outqueue.pop() except: self.done = True if not self.recv: self.deferred.callback(None) return d = threads.deferToThreadPool(reactor, self.threadpool, self.sendPkt, pkt) d.addCallback(sent) return d for x in range(self.mthreads): try: pkt = self.outqueue.pop() except: self.done = True return if self.cthreads >= self.mthreads and self.done: return d = threads.deferToThreadPool(reactor, self.threadpool, self.sendPkt, pkt) d.addCallback(sent) return d
def run(self, function, *args, **kwargs): """Run C{function} in a thread. The function is run in a thread by a function wrapper, which commits the transaction if the function runs successfully. If it raises an exception the transaction is aborted. If the named variable 'async' is set to False we don't run the function in the ThreadPool but in the main thread. @param function: The function to run. @param args: Positional arguments to pass to C{function}. @param kwargs: Keyword arguments to pass to C{function}. @return: A C{Deferred} that will fire after the function has been run. """ run_async = kwargs.pop('async', True) if run_async: # Inline the reactor import here for sake of safeness, in case a # custom reactor needs to be installed from twisted.internet import reactor return deferToThreadPool( reactor, self._threadpool, self._wrap, function, *args, **kwargs) return self._wrap(function, *args, **kwargs)
def _clean_redir_urls(text, urls={}, first=True, pool=None): for res in URL_REGEX.findall(text): url00 = res[2].encode('utf-8') url0 = url00 if not url00.startswith('http'): if "@" in url00 or url00.startswith('#'): continue url0 = "http://%s" % url00 if url0 in urls: url1 = urls[url0] if url1 == url0: continue else: try: url1 = yield deferToThreadPool(reactor, pool, get_url, url0, timeout=8) url1 = clean_url(url1) urls[url0] = url1 urls[url1] = url1 except Exception as e: if config.DEBUG and not first: loggerr("trying to resolve %s : %s" % (url0, e)) if "403" in str(e) or "Error 30" in str(e): urls[url0] = url00 url1 = url00 if first and not url1 == url00: url1 = url1.replace('http', '##HTTP##') try: url1 = url1.decode('utf-8') text = text.replace(res[0], '%s%s%s' % (res[1], url1, res[4])) except: if config.DEBUG: logerr("encoding %s" % url1) if not first: text = text.replace('##HTTP##', 'http') defer.returnValue((text, urls))
def wrapper(*args, **kwargs): if not _testing: return deferToThreadPool(reactor, _threadpool, run_in_tx, fun, *args, **kwargs) else: # No threading during testing return defer.execute(run_in_tx, fun, *args, **kwargs)
def __safe_call(*args, **kwargs): client = None keepalive = True if "_nokeepalive" in kwargs: keepalive = not kwargs.pop("_nokeepalive") try: corpus = kwargs.pop("corpus") except: corpus = "" fail = format_error("corpus argument missing") else: fail = format_error({"corpus_id": corpus, "ready": self.factory.test_corpus(corpus), "status": self.factory.status_corpus(corpus), "message": "Corpus is not started"}) if corpus in self.factory.corpora: if keepalive: self.factory.corpora[corpus].lastcall = time.time() client = getattr(self.factory.corpora[corpus], "client_%s" % type_client) if fail["message"]["status"] == "error": fail["message"]["message"] = self.factory.corpora[corpus].error if hasattr(client, 'threadpool'): if self.factory.test_corpus(corpus): return deferToThreadPool(reactor, client.threadpool, client.__thrift_call__, call, *args, **kwargs) return defer.succeed(fail) if self.factory.test_corpus(corpus): return client.__thrift_call__(call, *args, **kwargs) return fail
def render_POST(self, req): uid = self.transcriber.next_id() tran = req.args['transcript'][0] audio = req.args['audio'][0] async = True if 'async' in req.args and req.args['async'][0] == 'false': async = False result_promise = threads.deferToThreadPool( self.reactor, self.reactor.getThreadPool(), self.transcriber.transcribe, uid, tran, audio) if not async: def write_result(result): '''Write JSON to client on completion''' req.headers["Content-Type"] = "application/json" req.write(json.dumps(result, indent=2)) req.finish() result_promise.addCallback(write_result) result_promise.addErrback(lambda _: None) # ignore errors req.notifyFinish().addErrback(lambda _: result_promise.cancel()) return NOT_DONE_YET req.setResponseCode(FOUND) req.setHeader(b"Location", "/transcriptions/%s" % (uid)) return ''
def start_stream(self, conf): self.db.authenticate(config.MONGODB['USER'], config.MONGODB['PSWD']) queries = list(self.db["feeds"].find({'database': "tweets", 'channel': self.fact.channel}, fields=['query'])) track = [] follow = [] skip = [] k = 0 f = 0 for query in queries: q = str(query['query'].encode('utf-8')) if self.re_twitter_account.match(q): q = q.lstrip('@') follow.append(q) f += 1 elif " OR " in q or " -" in q or '"' in q or len(q) > 60: skip.append(q) continue track.append(q) k += 1 if k > 395 or f > 4995: break user = conf["TWITTER"]["USER"] if user not in follow: follow.append(user) if user not in track: track.append(user) if len(skip): self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), "stream", hint=True) self.log("Start search streaming for: « %s »" % " » | « ".join(track), "stream", hint=True) conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token) users, self.fact.ircclient.twitter_users = conn.search_users(follow, self.fact.ircclient.twitter_users) return deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track)
def runInteraction(self, interaction, *args, **kw): """ Interact with the database and return the result. The 'interaction' is a callable object which will be executed in a thread using a pooled connection. It will be passed an L{Transaction} object as an argument (whose interface is identical to that of the database cursor for your DB-API module of choice), and its results will be returned as a L{Deferred}. If running the method raises an exception, the transaction will be rolled back. If the method returns a value, the transaction will be committed. NOTE that the function you pass is *not* run in the main thread: you may have to worry about thread-safety in the function you pass to this if it tries to use non-local objects. @param interaction: a callable object whose first argument is an L{adbapi.Transaction}. @param *args: additional positional arguments to be passed to interaction @param **kw: keyword arguments to be passed to interaction @return: a Deferred which will fire the return value of C{interaction(Transaction(...), *args, **kw)}, or a L{twisted.python.failure.Failure}. """ from twisted.internet import reactor return threads.deferToThreadPool(reactor, self.threadpool, self._runInteraction, interaction, *args, **kw)
def get_id(self, model, unique, fields): ''' Get an ID from the cache or from the database. If doesn't exist - create an item. All database operations are done from the separate thread ''' assert isInIOThread() fval = fields[unique] try: result = self.cache[model][fval] self.counters['hit'][model] += 1 returnValue(result) except KeyError: self.counters['miss'][model] += 1 selectors = {unique: fval} result, created = yield deferToThreadPool( self.reactor, self.read_pool, get_or_create, model, fields, **selectors) result = result.id if created: self.counters['db_create'][model] += 1 else: self.counters['db_hit'][model] += 1 self.cache[model][fval] = result returnValue(result)
def watch_children(kzclient, path, func, allow_session_lost=True, send_event=False, ChildrenWatch=ChildrenWatch): """ Install a Kazoo :obj:`ChildrenWatch` on the given path. The given `func` will be called in the reactor thread when any children are created or deleted, or if the node itself is deleted. Returns a Deferred which usually has no result, but may fail with an exception if e.g. the path does not exist. """ def wrapped_func(*args, **kwargs): return blockingCallFromThread(kzclient.reactor, func, *args, **kwargs) return deferToThreadPool( kzclient.reactor, kzclient.pool, lambda: ChildrenWatch( kzclient.kazoo_client, path, func=wrapped_func, allow_session_lost=allow_session_lost, send_event=send_event))
def runWithConnection(self, func, *args, **kwargs): """ Executes a function with database connection and return the result. """ return threads.deferToThreadPool(reactor, self.threadpool, self._runWithConnection, func, *args, **kwargs)
def connect(self, masterUrl, deferred): """ Connect to RCE. @param masterUrl: URL of Master process. @type masterUrl: str @param deferred: Deferred which is called as soon as the connection was successfully established. @type deferred: twisted.internet.defer.Deferred @raise: ConnectionError, if no connection could be established. """ self._connectedDeferred = deferred def eb(e): print(e.getErrorMessage()) if self._connectedDeferred: self._connectedDeferred.errback(e) self._connectedDeferred = None d = deferToThreadPool(self._reactor, self._reactor.getThreadPool(), self._getRobotURL, masterUrl) d.addCallback(self._robotConnect) d.addErrback(eb)
def runWithTransaction(self, interaction, transaction, *args, **kw): """ Interact with the database and return the result, using the given transaction and connection. Inspired from twisted.enterprise.adbapi.ConnectionPool.runInteraction The 'interaction' is a callable object which will be executed in a thread using a pooled connection. It will be passed an C{Transaction} object as an argument (whose interface is identical to that of the database cursor for your DB-API module of choice), and its results will be returned as a Deferred. If running the method raises an exception, the transaction will NOT be rolled back. The transaction will NOT be committed automatically, you have to call commit on the transaction. NOTE that the function you pass is *not* run in the main thread: you may have to worry about thread-safety in the function you pass to this if it tries to use non-local objects. @param interaction: a callable object whose first argument is an L{adbapi.Transaction}. @param transaction: a C {dict} containing containing a C{t.e.a.Transaction} and C{t.e.a.Connection} instances @param *args: additional positional arguments to be passed to interaction @param **kw: keyword arguments to be passed to interaction @return: a Deferred which will fire the return value of 'interaction(Transaction(...), *args, **kw)', or a Failure. """ return threads.deferToThreadPool(reactor, Registry.DBPOOL.threadpool, self._runWithTransaction, interaction, transaction, *args, **kw)
def commit(self, obj): """ Commits current obj transaction. @return: A C{Deferred} """ return threads.deferToThreadPool(reactor, Registry.DBPOOL.threadpool, self._commit, obj._txn)
def rollback(self, obj): """ Rollback current obj transaction. @return: A C{Deferred} """ return threads.deferToThreadPool(reactor, Registry.DBPOOL.threadpool, self._rollback, obj._txn)
def doReconfig(self): log.msg("beginning configuration update") changes_made = False failed = False try: # Run the master.cfg in thread, so that it cas use blocking code new_config = yield threads.deferToThreadPool( self.reactor, self.reactor.getThreadPool(), self.config_loader.loadConfig) changes_made = True self.config = new_config yield self.reconfigServiceWithBuildbotConfig(new_config) except config.ConfigErrors as e: for msg in e.errors: log.msg(msg) failed = True except Exception: log.err(failure.Failure(), 'during reconfig:') failed = True if failed: if changes_made: log.msg("WARNING: reconfig partially applied; master " "may malfunction") else: log.msg("reconfig aborted without making any changes") else: log.msg("configuration update complete")
def runWithConnection(self, func, *args, **kw): """ Execute a function with a database connection and return the result. @param func: A callable object of one argument which will be executed in a thread with a connection from the pool. It will be passed as its first argument a L{Connection} instance (whose interface is mostly identical to that of a connection object for your DB-API module of choice), and its results will be returned as a L{Deferred}. If the method raises an exception the transaction will be rolled back. Otherwise, the transaction will be committed. B{Note} that this function is B{not} run in the main thread: it must be threadsafe. @param *args: positional arguments to be passed to func @param **kw: keyword arguments to be passed to func @return: a L{Deferred} which will fire the return value of C{func(Transaction(...), *args, **kw)}, or a L{twisted.python.failure.Failure}. """ from twisted.internet import reactor return threads.deferToThreadPool(reactor, self.threadpool, self._runWithConnection, func, *args, **kw)
def _callback(self, msg, msgID): """ Callback hook. """ reactor = self._conn._reactor d = deferToThreadPool(reactor, reactor.getThreadPool(), self._cb, msg, *self._args) d.addCallback(self._response_success, msgID) d.addErrback(self._response_failure, msgID)
def request_same_or_different_thread_thread(self): hr = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing) request1 = DummyRequest('r1') request1.isSecure = lambda: False request1.content = "llamas" d = deferToThreadPool(reactor, self.tp, hr.render, request1) return d
def _run_in_thread(self, *args, **kwargs): reactor = getattr(self, reactor_name) sync = getattr(self, sync_name) threadpool = getattr(self, threadpool_name) original = getattr(sync, method_name) return deferToThreadPool( reactor, threadpool, preserve_context(original), *args, **kwargs )
def test_deferredResult(self): """ L{threads.deferToThreadPool} executes the function passed, and correctly handles the positional and keyword arguments given. """ d = threads.deferToThreadPool(reactor, self.tp, lambda x, y=5: x + y, 3, y=4) d.addCallback(self.assertEqual, 7) return d
def save(self, obj): self.write_queue.put(obj) if self.writelock: return None else: return deferToThreadPool( self.reactor, self.write_pool, self._do_save)
def schedule(self, func, *args, **kwargs): self._jobs.append( deferToThreadPool( reactor, self._thread_pool, func, *args, **kwargs))
def wrapper(*args, **kwargs): if not _testing: return deferToThreadPool(reactor, _threadpool, run_in_tx, fun, *args, **kwargs) else: return defer.execute(run_in_tx, fun, *args, **kwargs)
def deferToDatabase(func, *args, **kwargs): """Call `func` in a thread where database activity is permitted.""" if settings.DEBUG and getattr(settings, "DEBUG_QUERIES", False): func = count_queries(log.debug)(func) return threads.deferToThreadPool(reactor, reactor.threadpoolForDatabase, func, *args, **kwargs)
def recordStatistics(self, addr, encoding, isUTF8): networkID = self.networkIDs[addr] d = threads.deferToThreadPool(self.reactor, self.threadPool, self._recordStatistics, networkID, encoding, isUTF8) d.addErrback(lambda f: self._log.failure("Could not persist", f))
def run(self, function, *args, **kwargs): return deferToThreadPool(reactor, GLSettings.orm_tp, function, *args, **kwargs)
def _defer_to_thread(self, f, *args, **kwargs): return deferToThreadPool( self._reactor, self._tpool, f, *args, **kwargs )
def do(self, callable, *args, **kwargs): return threads.deferToThreadPool(reactor, self, self.__thd, False, callable, args, kwargs)
def run(function, *args, **kwargs): """ Defer provided function to thread """ return deferToThreadPool(reactor, GLSettings.orm_tp, function, *args, **kwargs)
def startService(self): assert not self._already_started, "can only start the master once" self._already_started = True log.msg("Starting BuildMaster -- buildbot.version: %s" % buildbot.version) # Set umask if self.umask is not None: os.umask(self.umask) # first, apply all monkeypatches monkeypatches.patch_all() # we want to wait until the reactor is running, so we can call # reactor.stop() for fatal errors d = defer.Deferred() self.reactor.callWhenRunning(d.callback, None) yield d startup_succeed = False try: # load the configuration file, treating errors as fatal try: # run the master.cfg in thread, so that it can use blocking # code self.config = yield threads.deferToThreadPool( self.reactor, self.reactor.getThreadPool(), self.config_loader.loadConfig) except config.ConfigErrors as e: log.msg("Configuration Errors:") for msg in e.errors: log.msg(" " + msg) log.msg("Halting master.") self.reactor.stop() return except Exception: log.err(failure.Failure(), 'while starting BuildMaster') self.reactor.stop() return # set up services that need access to the config before everything # else gets told to reconfig try: yield self.db.setup() except exceptions.DatabaseNotReadyError: # (message was already logged) self.reactor.stop() return self.mq.setup() if hasattr(signal, "SIGHUP"): def sighup(*args): eventually(self.reconfig) signal.signal(signal.SIGHUP, sighup) if hasattr(signal, "SIGUSR1"): def sigusr1(*args): eventually(self.botmaster.cleanShutdown) signal.signal(signal.SIGUSR1, sigusr1) # get the masterid so other services can use it in # startup/reconfig. This goes directly to the DB since the data # API isn't initialized yet, and anyway, this method is aware of # the DB API since it just called its setup function self.masterid = yield self.db.masters.findMasterId(name=self.name) # mark this master as stopped, in case it crashed before yield self.data.updates.masterStopped(name=self.name, masterid=self.masterid) # call the parent method yield service.AsyncMultiService.startService(self) # We make sure the housekeeping is done before configuring in order to cleanup # any remaining claimed schedulers or change sources from zombie # masters yield self.data.updates.expireMasters(forceHouseKeeping=True) # give all services a chance to load the new configuration, rather # than the base configuration yield self.reconfigServiceWithBuildbotConfig(self.config) # Mark the master as active now that mq is running yield self.data.updates.masterActive(name=self.name, masterid=self.masterid) # Start the heartbeat timer yield self.masterHeartbeatService.setServiceParent(self) # send the statistics to buildbot.net, without waiting self.sendBuildbotNetUsageData() startup_succeed = True except Exception: f = failure.Failure() log.err(f, 'while starting BuildMaster') self.reactor.stop() finally: if startup_succeed: log.msg("BuildMaster is running") else: log.msg("BuildMaster startup failed") self._master_initialized = True
def run(self, function, *args, **kwargs): return deferToThreadPool(reactor, get_thread_pool(), function, *args, **kwargs)
def _recordWithDN(self, dn): return deferToThreadPool(reactor, self.threadpool, self._recordWithDN_inThread, dn)
def deferToThread(self, reactor, meth, *args, **kwargs): return threads.deferToThreadPool(reactor, self._threadPool, meth, *args, **kwargs)
def process_tweets(self, feed, source, query=None, pagecount=0): # handle tweets from icerocket or topsy fake rss nexturl = "" try: elements = feed.entries except: # handle tweets from Twitter API if isinstance(feed, list) and len(feed): elements = feed elif isinstance(feed, dict) and "nexturl" in feed: nexturl = feed["nexturl"] elements = feed["tweets"] else: returnD(False) if query: source = "%s https://api.twitter.com/api/1.1/search/tweets.json?q=%s" % (source, query) ids = [] hashs = [] tweets = [] fresh = True for i in elements: try: date = datetime.fromtimestamp(time.mktime(i.get('published_parsed', '')) - 4*60*60) except: if i.get('created_at', '') == "now": date = datetime.now() else: #date = datetime.strptime(i.get('created_at', ''), '%a %b %d %H:%M:%S +0000 %Y') + timedelta(hours=2) date = parse_date(i.get('created_at', '')) if datetime.today() - date > timedelta(hours=config.BACK_HOURS): fresh = False break tweet, self.fact.cache_urls = yield clean_redir_urls(i.get('title', '').replace('\n', ' '), self.fact.cache_urls) link = i.get('link', '') res = re_tweet_url.search(link) if res: user = res.group(1) tid = long(res.group(2)) ids.append(tid) tw = {'_id': "%s:%s" % (self.fact.channel, tid), 'channel': self.fact.channel, 'id': tid, 'user': user.lower(), 'screenname': user, 'message': tweet, 'uniq_rt_hash': uniq_rt_hash(tweet), 'link': link, 'date': date, 'timestamp': datetime.today(), 'source': source} tw = grab_extra_meta(i, tw) tweets.append(tw) # Delay displaying to avoid duplicates from the stream if source != "mystream" and not self.fact.tweets_search_page: yield deferredSleep() existings = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'id': {'$in': ids}}, fields=['_id'], filter=sortdesc('id')) existing = [t['_id'] for t in existings] news = [t for t in tweets if t['_id'] not in existing] if not news: returnD(False) good = [] news.sort(key=itemgetter('id')) if fresh and not source.startswith("my") and len(news) > len(elements) / 2: if query and nexturl and pagecount < 3*self.fact.back_pages_limit: deferToThreadPool(reactor, self.threadpool, reactor.callLater, 15, self.start_twitter_search, [query], max_id=nexturl, pagecount=pagecount+1) elif not query and nexturl and "p=%d" % (self.fact.back_pages_limit+1) not in nexturl and "page=%s" % (2*self.fact.back_pages_limit) not in nexturl: deferToThreadPool(reactor, self.threadpool, reactor.callLater, 41, self.start_web, nexturl) elif not query and not nexturl and int(source[-1:]) <= self.fact.back_pages_limit: deferToThreadPool(reactor, self.threadpool, reactor.callLater, 41, self.start_web, next_page(source)) if self.fact.displayRT: good = news else: hashs = [t['uniq_rt_hash'] for t in news if t['uniq_rt_hash'] not in hashs] existings = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'uniq_rt_hash': {'$in': hashs}}, fields=['uniq_rt_hash'], filter=sortdesc('id')) existing = [t['uniq_rt_hash'] for t in existings] for t in news: if self.fact.twuser == t['user'] or t['uniq_rt_hash'] not in existing or (self.fact.displayMyRT and "@%s: " % self.fact.twuser in t['message'].lower()): existing.append(t['uniq_rt_hash']) good.append(t) if config.DEBUG: nb_rts_str = "" nb_rts = len(news) - len(good) if nb_rts: nb_rts_str = " (%s RTs filtered)" % nb_rts self.log("Displaying %s tweets%s" % (len(good), nb_rts_str), hint=True) if self.fact.status != "closed": for t in good: msg = "%s: %s" % (t['screenname'].encode('utf-8'), format_tweet(t)) self.fact.ircclient._send_message(msg, self.fact.channel) for t in news: yield self.fact.db['tweets'].save(t, safe=True) returnD(True)
def _wrap(*args, **kwargs): return deferToThreadPool(reactor, retval._pool, _et(_user_callable), *args, **kwargs)
def run(self, function, *args, **kwargs): return deferToThreadPool(reactor, State.orm_tp, function, *args, **kwargs)
def test1(): d = deferToThreadPool(reactor, tp, test2) d.addCallback(cb)
def do_with_engine(self, callable, *args, **kwargs): return threads.deferToThreadPool(self.reactor, self._pool, self.__thd, True, callable, args, kwargs)
def new_deferToThreadPool(reactor, threadpool, f, *args, **kwargs): """Variant of Twisted's that wraps all functions in `synchronous`.""" func = f if ISynchronous.providedBy(f) else synchronous(f) return deferToThreadPool(reactor, threadpool, func, *args, **kwargs)
def _stop(self): self._stop_evt = None threads.deferToThreadPool(self.reactor, self._pool, self.engine.dispose) self._pool.stop() self.running = False
def newfunc(*a, **kw): """I call blocking python code in a seperate thread. I return a deferred and will fire callbacks or errbacks. """ return threads.deferToThreadPool(R, pool, func, *a, **kw)
def do(self, callable, *args, **kwargs): ret = yield threads.deferToThreadPool(self.reactor, self._pool, self.__thd, False, callable, args, kwargs) return ret
def runWithConnection(func, *args, **kwargs): return threads.deferToThreadPool(pool._reactor, pool.threadpool, pool._runWithConnection, func, *args, **kwargs)
def perform_job_to_thread(self, job): d = threads.deferToThreadPool(reactor, self._thread_pool, job.perform) d.addCallback(self.callback_perform_job) d.addErrback(self.errback_perform_job, job=job)
def runInteraction(interaction, *args, **kwargs): return threads.deferToThreadPool(pool._reactor, pool.threadpool, pool._runInteraction, interaction, *args, **kwargs)
def deferToDatabase(func, *args, **kwargs): """Call `func` in a thread where database activity is permitted.""" return threads.deferToThreadPool(reactor, reactor.threadpoolForDatabase, func, *args, **kwargs)
def deferToThread(f, *args, **kwargs): return threads.deferToThreadPool(self.reactor, self.reactor.getThreadPool(), f, *args, **kwargs)
def run(function, *args, **kwargs): """ Defer provided function to thread """ return deferToThreadPool(reactor, transact.tp, function, *args, **kwargs)