def start_stream(self, conf):
if not self.fact.__init_timeout__():
returnD(False)
queries = yield self.fact.db['feeds'].find({'database': 'tweets', 'channel': self.fact.channel}, fields=['query'])
track = []
skip = []
k = 0
for query in queries:
q = str(query['query'].encode('utf-8')).lower()
# queries starting with @ should return only tweets from corresponding user, stream doesn not know how to handle this so skip
if self.re_twitter_account.match(q):
continue
elif " OR " in q or " -" in q or '"' in q or len(q) > 60 or len(q) < 6:
skip.append(q)
continue
track.append(q)
k += 1
if k > 395:
break
if self.fact.twuser not in track:
track.append(self.fact.twuser)
if len(skip):
self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), hint=True)
self.log("Start search streaming for: « %s »" % " » | « ".join(track), hint=True)
conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token)
# tries to find users corresponding with queries to follow with stream
users, self.fact.ircclient.twitter['users'] = conn.lookup_users(track, self.fact.ircclient.twitter['users'])
deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track)
self.depiler = LoopingCall(self.flush_tweets)
self.depiler.start(1)
returnD(True)
def test_contemporaneous_requests(self):
'''
We're going to create two request-response cycles here:
Cycle 1 will begin.
Cycle 2 will begin.
Cycle 2 will return.
Cycle 1 will return.
This way, we can prove that the crosstown_traffic created
by cycle 1 is not resolved by the return of cycle 2.
'''
tp = ThreadPool(maxthreads=20)
tp.start()
self.addCleanup(tp.stop)
log.debug("\n\nStarting the two stream stuff.")
request1 = DummyRequest('r1')
request1.isSecure = lambda: False
request1.content = "Nothing really here."
request1.headers['llamas'] = 'dingo'
nameSpace.test_case = self
hr = HendrixWSGIResource(reactor, tp, wsgi_application)
d1 = deferToThreadPool(reactor, tp, hr.render, request1)
request2 = DummyRequest('r2')
request2.isSecure = lambda: False
request2.content = "Nothing really here."
request2.headers['llamas'] = 'dingo'
d2 = deferToThreadPool(reactor, tp, hr.render, request2)
def woah_stop(failure):
nameSpace.async_task_was_done.put_nowait(False)
nameSpace.second_cycle_complete.put_nowait(False)
nameSpace.ready_to_proceed_with_second_cycle.put_nowait(False)
d1.addErrback(woah_stop)
d2.addErrback(woah_stop)
combo_deferred = gatherResults([d1, d2])
def wait_for_queue_resolution():
nameSpace.async_task_was_done.get(True, 3)
combo_deferred.addCallback(
lambda _: deferToThreadPool(reactor, tp, wait_for_queue_resolution)
)
combo_deferred.addCallback(
lambda _: self.assertTrue(nameSpace.async_task_was_run)
)
return combo_deferred
def process_item(self, domain, item):
args = dict(item)
random_numbers = args['random_numbers'].split(',')
for num in random_numbers:
#log.msg('dispatching : %s' % num,level=log.ERROR)
trackable = TrackableObject()
threads.deferToThreadPool(reactor, self.threadpool, trackable.takes_long_time, num,self)
return item
def run(self, threadpool):
# See if status code is a go
self.check_status_code_against_no_go_list()
if self.no_go:
return
if self.same_thread:
self.crosstown_task()
else:
deferToThreadPool(reactor, threadpool, self.crosstown_task)
def run(self, threadpool=None):
if self.no_go:
return
if not threadpool:
threadpool = reactor.threadpool or ThreadPool()
if self.same_thread:
self.crosstown_task()
else:
deferToThreadPool(reactor, threadpool, self.crosstown_task)
def _handshake(self, msg, line):
if self.state.current != 'WAIT':
self.error("Invalid state for a handshake command: %s",
self.state.current)
self._sendError("Invalid state")
return
mysha = self.host.workflow.checksum
your_sha = msg.get("checksum")
if not your_sha:
self.error("Did not receive the workflow checksum")
self._sendError("Workflow checksum is missing")
return
if mysha != your_sha:
self._sendError("Workflow checksum mismatch: "
"expected %s, got %s" % (mysha, your_sha))
return
must_reply = False
msgid = msg.get("id")
if msgid is None:
self.id = str(uuid.uuid4())
must_reply = True
else:
self.id = msgid
if not self.nodes.get(self.id):
self.warning("Did not recognize the received ID %s")
must_reply = True
else:
self.sendLine({'reconnect': "ok"})
if must_reply:
try:
_, mid, pid = self._extractClientInformation(msg)
except Exception as e:
self.error(str(e))
return
data = self.host.workflow.generate_initial_data_for_slave(
SlaveDescription.make(self.nodes[self.id]))
endpoint = self.host.choose_endpoint(self.id, mid, pid, self.hip)
self.nodes[self.id]['endpoint'] = self._endpoint = endpoint
retmsg = {'endpoint': endpoint, 'data': data}
if not msgid:
retmsg['id'] = self.id
retmsg['log_id'] = self.host.launcher.log_id
self.sendLine(retmsg)
data = msg.get('data')
if data is not None:
threads.deferToThreadPool(
reactor, self.host.workflow.thread_pool,
self.host.workflow.apply_initial_data_from_slave,
data, SlaveDescription.make(self.nodes[self.id])) \
.addErrback(errback)
self.nodes[self.id]['data'] = [d for d in data if d is not None]
self.state.identify()
def test_threadLocality(self):
"""
An 'Record' repr()'d in two separate threads at the same time should
look the same (i.e. the repr state tracking for '...' should be
thread-local).
"""
pool = ThreadPool(2, 2)
pool.start()
self.addCleanup(pool.stop)
class StickyRepr(object):
"""
This has a __repr__ which will block until a separate thread
notifies it that it should return. We use this to create a race
condition.
"""
waited = False
def __init__(self):
self.set = threading.Event()
self.wait = threading.Event()
def __repr__(self):
if not self.waited:
self.set.set()
self.wait.wait()
return 'sticky'
r = StickyRepr()
mr = MyRecord(something=1, somethingElse=r)
d = deferToThreadPool(reactor, pool, repr, mr)
def otherRepr():
# First we wait for the first thread doing a repr() to enter its
# __repr__()...
r.set.wait()
# OK, now it's blocked. Let's make sure that subsequent calls to
# this repr() won't block.
r.waited = True
# Do it! This is a concurrent repr().
result = repr(mr)
# Now we're done, wake up the other repr and let it complete.
r.wait.set()
return result
d2 = deferToThreadPool(reactor, pool, otherRepr)
def done(xxx_todo_changeme):
(thread1repr, thread2repr) = xxx_todo_changeme
knownGood = 'MyRecord(something=1, somethingElse=sticky)'
# self.assertEquals(thread1repr, thread2repr)
self.assertEqual(thread1repr, knownGood)
self.assertEqual(thread2repr, knownGood)
return gatherResults([d, d2]).addCallback(done)
def _retry(self, method_name, call):
"""Retries |call| on transient errors and access token expiration.
Args:
method_name (str): name of the remote method, for logging.
call (func() -> any): a function that makes an RPC call and returns
result.
"""
attempts = self.retry_attempt_count
wait = self.retry_wait_seconds
while attempts > 0:
attempts -= 1
try:
if not self.started:
raise NotStartedError('DeferredResource is not started')
res = yield threads.deferToThreadPool(reactor, self._pool, call)
defer.returnValue(res)
except Exception as ex:
if not self.started:
raise ex
if attempts > 0 and is_transient(ex):
self.log('Transient error while calling %s. '
'Will retry in %d seconds.' % (method_name, wait))
# TODO(nodir), optimize: stop waiting if the resource is stopped.
yield sleep(wait)
if not self.started:
raise ex
wait = min(wait * 2, 30)
continue
self.log('RPC "%s" failed: %s'% (method_name, traceback.format_exc()))
raise ex
def _sendrcv(self, pkts, filter=None, iface=None, nofilter=0):
self._buildSocket(filter, iface, nofilter)
self._buildPacketQueues(pkts)
if not self.last_answer:
self.last_answer = time.time()
def sent(cb):
if self.cthreads < self.mthreads and not self.done:
pkt = None
try:
pkt = self.outqueue.pop()
except:
self.done = True
if not self.recv:
self.deferred.callback(None)
return
d = threads.deferToThreadPool(reactor, self.threadpool,
self.sendPkt, pkt)
d.addCallback(sent)
return d
for x in range(self.mthreads):
try:
pkt = self.outqueue.pop()
except:
self.done = True
return
if self.cthreads >= self.mthreads and self.done:
return
d = threads.deferToThreadPool(reactor, self.threadpool,
self.sendPkt, pkt)
d.addCallback(sent)
return d
def run(self, function, *args, **kwargs):
"""Run C{function} in a thread.
The function is run in a thread by a function wrapper, which
commits the transaction if the function runs successfully. If it
raises an exception the transaction is aborted.
If the named variable 'async' is set to False we don't run the
function in the ThreadPool but in the main thread.
@param function: The function to run.
@param args: Positional arguments to pass to C{function}.
@param kwargs: Keyword arguments to pass to C{function}.
@return: A C{Deferred} that will fire after the function has been run.
"""
run_async = kwargs.pop('async', True)
if run_async:
# Inline the reactor import here for sake of safeness, in case a
# custom reactor needs to be installed
from twisted.internet import reactor
return deferToThreadPool(
reactor, self._threadpool, self._wrap,
function, *args, **kwargs)
return self._wrap(function, *args, **kwargs)
def _clean_redir_urls(text, urls={}, first=True, pool=None):
for res in URL_REGEX.findall(text):
url00 = res[2].encode('utf-8')
url0 = url00
if not url00.startswith('http'):
if "@" in url00 or url00.startswith('#'):
continue
url0 = "http://%s" % url00
if url0 in urls:
url1 = urls[url0]
if url1 == url0:
continue
else:
try:
url1 = yield deferToThreadPool(reactor, pool, get_url, url0, timeout=8)
url1 = clean_url(url1)
urls[url0] = url1
urls[url1] = url1
except Exception as e:
if config.DEBUG and not first:
loggerr("trying to resolve %s : %s" % (url0, e))
if "403" in str(e) or "Error 30" in str(e):
urls[url0] = url00
url1 = url00
if first and not url1 == url00:
url1 = url1.replace('http', '##HTTP##')
try:
url1 = url1.decode('utf-8')
text = text.replace(res[0], '%s%s%s' % (res[1], url1, res[4]))
except:
if config.DEBUG:
logerr("encoding %s" % url1)
if not first:
text = text.replace('##HTTP##', 'http')
defer.returnValue((text, urls))
def wrapper(*args, **kwargs):
if not _testing:
return deferToThreadPool(reactor, _threadpool,
run_in_tx, fun, *args, **kwargs)
else:
# No threading during testing
return defer.execute(run_in_tx, fun, *args, **kwargs)
def __safe_call(*args, **kwargs):
client = None
keepalive = True
if "_nokeepalive" in kwargs:
keepalive = not kwargs.pop("_nokeepalive")
try:
corpus = kwargs.pop("corpus")
except:
corpus = ""
fail = format_error("corpus argument missing")
else:
fail = format_error({"corpus_id": corpus,
"ready": self.factory.test_corpus(corpus),
"status": self.factory.status_corpus(corpus),
"message": "Corpus is not started"})
if corpus in self.factory.corpora:
if keepalive:
self.factory.corpora[corpus].lastcall = time.time()
client = getattr(self.factory.corpora[corpus],
"client_%s" % type_client)
if fail["message"]["status"] == "error":
fail["message"]["message"] = self.factory.corpora[corpus].error
if hasattr(client, 'threadpool'):
if self.factory.test_corpus(corpus):
return deferToThreadPool(reactor, client.threadpool,
client.__thrift_call__, call, *args, **kwargs)
return defer.succeed(fail)
if self.factory.test_corpus(corpus):
return client.__thrift_call__(call, *args, **kwargs)
return fail
def render_POST(self, req):
uid = self.transcriber.next_id()
tran = req.args['transcript'][0]
audio = req.args['audio'][0]
async = True
if 'async' in req.args and req.args['async'][0] == 'false':
async = False
result_promise = threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.transcriber.transcribe,
uid, tran, audio)
if not async:
def write_result(result):
'''Write JSON to client on completion'''
req.headers["Content-Type"] = "application/json"
req.write(json.dumps(result, indent=2))
req.finish()
result_promise.addCallback(write_result)
result_promise.addErrback(lambda _: None) # ignore errors
req.notifyFinish().addErrback(lambda _: result_promise.cancel())
return NOT_DONE_YET
req.setResponseCode(FOUND)
req.setHeader(b"Location", "/transcriptions/%s" % (uid))
return ''
def start_stream(self, conf):
self.db.authenticate(config.MONGODB['USER'], config.MONGODB['PSWD'])
queries = list(self.db["feeds"].find({'database': "tweets", 'channel': self.fact.channel}, fields=['query']))
track = []
follow = []
skip = []
k = 0
f = 0
for query in queries:
q = str(query['query'].encode('utf-8'))
if self.re_twitter_account.match(q):
q = q.lstrip('@')
follow.append(q)
f += 1
elif " OR " in q or " -" in q or '"' in q or len(q) > 60:
skip.append(q)
continue
track.append(q)
k += 1
if k > 395 or f > 4995:
break
user = conf["TWITTER"]["USER"]
if user not in follow:
follow.append(user)
if user not in track:
track.append(user)
if len(skip):
self.log("Skipping unprocessable queries for streaming: « %s »" % " » | « ".join(skip), "stream", hint=True)
self.log("Start search streaming for: « %s »" % " » | « ".join(track), "stream", hint=True)
conn = Microblog("twitter", conf, bearer_token=self.fact.twitter_token)
users, self.fact.ircclient.twitter_users = conn.search_users(follow, self.fact.ircclient.twitter_users)
return deferToThreadPool(reactor, self.threadpool, self.follow_stream, conf, users.values(), track)
def runInteraction(self, interaction, *args, **kw):
"""
Interact with the database and return the result.
The 'interaction' is a callable object which will be executed in a
thread using a pooled connection. It will be passed an L{Transaction}
object as an argument (whose interface is identical to that of the
database cursor for your DB-API module of choice), and its results will
be returned as a L{Deferred}. If running the method raises an
exception, the transaction will be rolled back. If the method returns a
value, the transaction will be committed.
NOTE that the function you pass is *not* run in the main thread: you
may have to worry about thread-safety in the function you pass to this
if it tries to use non-local objects.
@param interaction: a callable object whose first argument is an
L{adbapi.Transaction}.
@param *args: additional positional arguments to be passed to
interaction
@param **kw: keyword arguments to be passed to interaction
@return: a Deferred which will fire the return value of
C{interaction(Transaction(...), *args, **kw)}, or a
L{twisted.python.failure.Failure}.
"""
from twisted.internet import reactor
return threads.deferToThreadPool(reactor, self.threadpool,
self._runInteraction,
interaction, *args, **kw)
def get_id(self, model, unique, fields):
''' Get an ID from the cache or from the database.
If doesn't exist - create an item.
All database operations are done from
the separate thread
'''
assert isInIOThread()
fval = fields[unique]
try:
result = self.cache[model][fval]
self.counters['hit'][model] += 1
returnValue(result)
except KeyError:
self.counters['miss'][model] += 1
selectors = {unique: fval}
result, created = yield deferToThreadPool(
self.reactor, self.read_pool,
get_or_create,
model, fields, **selectors)
result = result.id
if created:
self.counters['db_create'][model] += 1
else:
self.counters['db_hit'][model] += 1
self.cache[model][fval] = result
returnValue(result)
def watch_children(kzclient,
path, func, allow_session_lost=True, send_event=False,
ChildrenWatch=ChildrenWatch):
"""
Install a Kazoo :obj:`ChildrenWatch` on the given path.
The given `func` will be called in the reactor thread when any children are
created or deleted, or if the node itself is deleted.
Returns a Deferred which usually has no result, but may fail with an
exception if e.g. the path does not exist.
"""
def wrapped_func(*args, **kwargs):
return blockingCallFromThread(kzclient.reactor, func, *args, **kwargs)
return deferToThreadPool(
kzclient.reactor,
kzclient.pool,
lambda: ChildrenWatch(
kzclient.kazoo_client,
path,
func=wrapped_func,
allow_session_lost=allow_session_lost,
send_event=send_event))
def runWithConnection(self, func, *args, **kwargs):
"""
Executes a function with database connection and return the result.
"""
return threads.deferToThreadPool(reactor, self.threadpool,
self._runWithConnection,
func, *args, **kwargs)
def connect(self, masterUrl, deferred):
""" Connect to RCE.
@param masterUrl: URL of Master process.
@type masterUrl: str
@param deferred: Deferred which is called as soon as the
connection was successfully established.
@type deferred: twisted.internet.defer.Deferred
@raise: ConnectionError, if no connection could be
established.
"""
self._connectedDeferred = deferred
def eb(e):
print(e.getErrorMessage())
if self._connectedDeferred:
self._connectedDeferred.errback(e)
self._connectedDeferred = None
d = deferToThreadPool(self._reactor, self._reactor.getThreadPool(),
self._getRobotURL, masterUrl)
d.addCallback(self._robotConnect)
d.addErrback(eb)
def runWithTransaction(self, interaction, transaction, *args, **kw):
"""
Interact with the database and return the result, using the given transaction and connection.
Inspired from twisted.enterprise.adbapi.ConnectionPool.runInteraction
The 'interaction' is a callable object which will be executed in a thread using a pooled connection.
It will be passed an C{Transaction} object as an argument (whose interface is identical to that of
the database cursor for your DB-API module of choice), and its results will be returned as a Deferred.
If running the method raises an exception, the transaction will NOT be rolled back.
The transaction will NOT be committed automatically, you have to call commit on the transaction.
NOTE that the function you pass is *not* run in the main thread: you may have to worry
about thread-safety in the function you pass to this if it tries to use non-local objects.
@param interaction: a callable object whose first argument
is an L{adbapi.Transaction}.
@param transaction: a C {dict} containing containing a C{t.e.a.Transaction} and C{t.e.a.Connection} instances
@param *args: additional positional arguments to be passed
to interaction
@param **kw: keyword arguments to be passed to interaction
@return: a Deferred which will fire the return value of
'interaction(Transaction(...), *args, **kw)', or a Failure.
"""
return threads.deferToThreadPool(reactor, Registry.DBPOOL.threadpool,
self._runWithTransaction,
interaction, transaction, *args, **kw)
def commit(self, obj):
"""
Commits current obj transaction.
@return: A C{Deferred}
"""
return threads.deferToThreadPool(reactor, Registry.DBPOOL.threadpool, self._commit, obj._txn)
def rollback(self, obj):
"""
Rollback current obj transaction.
@return: A C{Deferred}
"""
return threads.deferToThreadPool(reactor, Registry.DBPOOL.threadpool, self._rollback, obj._txn)
def doReconfig(self):
log.msg("beginning configuration update")
changes_made = False
failed = False
try:
# Run the master.cfg in thread, so that it cas use blocking code
new_config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
changes_made = True
self.config = new_config
yield self.reconfigServiceWithBuildbotConfig(new_config)
except config.ConfigErrors as e:
for msg in e.errors:
log.msg(msg)
failed = True
except Exception:
log.err(failure.Failure(), 'during reconfig:')
failed = True
if failed:
if changes_made:
log.msg("WARNING: reconfig partially applied; master "
"may malfunction")
else:
log.msg("reconfig aborted without making any changes")
else:
log.msg("configuration update complete")
def runWithConnection(self, func, *args, **kw):
"""
Execute a function with a database connection and return the result.
@param func: A callable object of one argument which will be executed
in a thread with a connection from the pool. It will be passed as
its first argument a L{Connection} instance (whose interface is
mostly identical to that of a connection object for your DB-API
module of choice), and its results will be returned as a
L{Deferred}. If the method raises an exception the transaction will
be rolled back. Otherwise, the transaction will be committed.
B{Note} that this function is B{not} run in the main thread: it
must be threadsafe.
@param *args: positional arguments to be passed to func
@param **kw: keyword arguments to be passed to func
@return: a L{Deferred} which will fire the return value of
C{func(Transaction(...), *args, **kw)}, or a
L{twisted.python.failure.Failure}.
"""
from twisted.internet import reactor
return threads.deferToThreadPool(reactor, self.threadpool,
self._runWithConnection,
func, *args, **kw)
def _callback(self, msg, msgID):
""" Callback hook.
"""
reactor = self._conn._reactor
d = deferToThreadPool(reactor, reactor.getThreadPool(),
self._cb, msg, *self._args)
d.addCallback(self._response_success, msgID)
d.addErrback(self._response_failure, msgID)
def request_same_or_different_thread_thread(self):
hr = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing)
request1 = DummyRequest('r1')
request1.isSecure = lambda: False
request1.content = "llamas"
d = deferToThreadPool(reactor, self.tp, hr.render, request1)
return d
def _run_in_thread(self, *args, **kwargs):
reactor = getattr(self, reactor_name)
sync = getattr(self, sync_name)
threadpool = getattr(self, threadpool_name)
original = getattr(sync, method_name)
return deferToThreadPool(
reactor, threadpool, preserve_context(original), *args, **kwargs
)
def test_deferredResult(self):
"""
L{threads.deferToThreadPool} executes the function passed, and
correctly handles the positional and keyword arguments given.
"""
d = threads.deferToThreadPool(reactor, self.tp, lambda x, y=5: x + y, 3, y=4)
d.addCallback(self.assertEqual, 7)
return d
def save(self, obj):
self.write_queue.put(obj)
if self.writelock:
return None
else:
return deferToThreadPool(
self.reactor, self.write_pool, self._do_save)
def schedule(self, func, *args, **kwargs):
self._jobs.append(
deferToThreadPool(
reactor, self._thread_pool, func, *args, **kwargs))
def wrapper(*args, **kwargs):
if not _testing:
return deferToThreadPool(reactor, _threadpool,
run_in_tx, fun, *args, **kwargs)
else:
return defer.execute(run_in_tx, fun, *args, **kwargs)
def deferToDatabase(func, *args, **kwargs):
"""Call `func` in a thread where database activity is permitted."""
if settings.DEBUG and getattr(settings, "DEBUG_QUERIES", False):
func = count_queries(log.debug)(func)
return threads.deferToThreadPool(reactor, reactor.threadpoolForDatabase,
func, *args, **kwargs)
def recordStatistics(self, addr, encoding, isUTF8):
networkID = self.networkIDs[addr]
d = threads.deferToThreadPool(self.reactor, self.threadPool,
self._recordStatistics, networkID,
encoding, isUTF8)
d.addErrback(lambda f: self._log.failure("Could not persist", f))
def run(self, function, *args, **kwargs):
return deferToThreadPool(reactor,
GLSettings.orm_tp,
function,
*args,
**kwargs)
def _defer_to_thread(self, f, *args, **kwargs):
return deferToThreadPool(
self._reactor, self._tpool, f, *args, **kwargs
)
def do(self, callable, *args, **kwargs):
return threads.deferToThreadPool(reactor, self, self.__thd, False,
callable, args, kwargs)
def run(function, *args, **kwargs):
"""
Defer provided function to thread
"""
return deferToThreadPool(reactor, GLSettings.orm_tp, function, *args,
**kwargs)
def startService(self):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
self.reactor.callWhenRunning(d.callback, None)
yield d
startup_succeed = False
try:
# load the configuration file, treating errors as fatal
try:
# run the master.cfg in thread, so that it can use blocking
# code
self.config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
except config.ConfigErrors as e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
self.reactor.stop()
return
except Exception:
log.err(failure.Failure(), 'while starting BuildMaster')
self.reactor.stop()
return
# set up services that need access to the config before everything
# else gets told to reconfig
try:
yield self.db.setup()
except exceptions.DatabaseNotReadyError:
# (message was already logged)
self.reactor.stop()
return
self.mq.setup()
if hasattr(signal, "SIGHUP"):
def sighup(*args):
eventually(self.reconfig)
signal.signal(signal.SIGHUP, sighup)
if hasattr(signal, "SIGUSR1"):
def sigusr1(*args):
eventually(self.botmaster.cleanShutdown)
signal.signal(signal.SIGUSR1, sigusr1)
# get the masterid so other services can use it in
# startup/reconfig. This goes directly to the DB since the data
# API isn't initialized yet, and anyway, this method is aware of
# the DB API since it just called its setup function
self.masterid = yield self.db.masters.findMasterId(name=self.name)
# mark this master as stopped, in case it crashed before
yield self.data.updates.masterStopped(name=self.name,
masterid=self.masterid)
# call the parent method
yield service.AsyncMultiService.startService(self)
# We make sure the housekeeping is done before configuring in order to cleanup
# any remaining claimed schedulers or change sources from zombie
# masters
yield self.data.updates.expireMasters(forceHouseKeeping=True)
# give all services a chance to load the new configuration, rather
# than the base configuration
yield self.reconfigServiceWithBuildbotConfig(self.config)
# Mark the master as active now that mq is running
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
# Start the heartbeat timer
yield self.masterHeartbeatService.setServiceParent(self)
# send the statistics to buildbot.net, without waiting
self.sendBuildbotNetUsageData()
startup_succeed = True
except Exception:
f = failure.Failure()
log.err(f, 'while starting BuildMaster')
self.reactor.stop()
finally:
if startup_succeed:
log.msg("BuildMaster is running")
else:
log.msg("BuildMaster startup failed")
self._master_initialized = True
def run(self, function, *args, **kwargs):
return deferToThreadPool(reactor, get_thread_pool(), function, *args,
**kwargs)
def _recordWithDN(self, dn):
return deferToThreadPool(reactor, self.threadpool,
self._recordWithDN_inThread, dn)
def deferToThread(self, reactor, meth, *args, **kwargs):
return threads.deferToThreadPool(reactor, self._threadPool, meth,
*args, **kwargs)
def process_tweets(self, feed, source, query=None, pagecount=0):
# handle tweets from icerocket or topsy fake rss
nexturl = ""
try:
elements = feed.entries
except:
# handle tweets from Twitter API
if isinstance(feed, list) and len(feed):
elements = feed
elif isinstance(feed, dict) and "nexturl" in feed:
nexturl = feed["nexturl"]
elements = feed["tweets"]
else:
returnD(False)
if query:
source = "%s https://api.twitter.com/api/1.1/search/tweets.json?q=%s" % (source, query)
ids = []
hashs = []
tweets = []
fresh = True
for i in elements:
try:
date = datetime.fromtimestamp(time.mktime(i.get('published_parsed', '')) - 4*60*60)
except:
if i.get('created_at', '') == "now":
date = datetime.now()
else:
#date = datetime.strptime(i.get('created_at', ''), '%a %b %d %H:%M:%S +0000 %Y') + timedelta(hours=2)
date = parse_date(i.get('created_at', ''))
if datetime.today() - date > timedelta(hours=config.BACK_HOURS):
fresh = False
break
tweet, self.fact.cache_urls = yield clean_redir_urls(i.get('title', '').replace('\n', ' '), self.fact.cache_urls)
link = i.get('link', '')
res = re_tweet_url.search(link)
if res:
user = res.group(1)
tid = long(res.group(2))
ids.append(tid)
tw = {'_id': "%s:%s" % (self.fact.channel, tid), 'channel': self.fact.channel, 'id': tid, 'user': user.lower(), 'screenname': user, 'message': tweet, 'uniq_rt_hash': uniq_rt_hash(tweet), 'link': link, 'date': date, 'timestamp': datetime.today(), 'source': source}
tw = grab_extra_meta(i, tw)
tweets.append(tw)
# Delay displaying to avoid duplicates from the stream
if source != "mystream" and not self.fact.tweets_search_page:
yield deferredSleep()
existings = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'id': {'$in': ids}}, fields=['_id'], filter=sortdesc('id'))
existing = [t['_id'] for t in existings]
news = [t for t in tweets if t['_id'] not in existing]
if not news:
returnD(False)
good = []
news.sort(key=itemgetter('id'))
if fresh and not source.startswith("my") and len(news) > len(elements) / 2:
if query and nexturl and pagecount < 3*self.fact.back_pages_limit:
deferToThreadPool(reactor, self.threadpool, reactor.callLater, 15, self.start_twitter_search, [query], max_id=nexturl, pagecount=pagecount+1)
elif not query and nexturl and "p=%d" % (self.fact.back_pages_limit+1) not in nexturl and "page=%s" % (2*self.fact.back_pages_limit) not in nexturl:
deferToThreadPool(reactor, self.threadpool, reactor.callLater, 41, self.start_web, nexturl)
elif not query and not nexturl and int(source[-1:]) <= self.fact.back_pages_limit:
deferToThreadPool(reactor, self.threadpool, reactor.callLater, 41, self.start_web, next_page(source))
if self.fact.displayRT:
good = news
else:
hashs = [t['uniq_rt_hash'] for t in news if t['uniq_rt_hash'] not in hashs]
existings = yield self.fact.db['tweets'].find({'channel': self.fact.channel, 'uniq_rt_hash': {'$in': hashs}}, fields=['uniq_rt_hash'], filter=sortdesc('id'))
existing = [t['uniq_rt_hash'] for t in existings]
for t in news:
if self.fact.twuser == t['user'] or t['uniq_rt_hash'] not in existing or (self.fact.displayMyRT and "@%s: " % self.fact.twuser in t['message'].lower()):
existing.append(t['uniq_rt_hash'])
good.append(t)
if config.DEBUG:
nb_rts_str = ""
nb_rts = len(news) - len(good)
if nb_rts:
nb_rts_str = " (%s RTs filtered)" % nb_rts
self.log("Displaying %s tweets%s" % (len(good), nb_rts_str), hint=True)
if self.fact.status != "closed":
for t in good:
msg = "%s: %s" % (t['screenname'].encode('utf-8'), format_tweet(t))
self.fact.ircclient._send_message(msg, self.fact.channel)
for t in news:
yield self.fact.db['tweets'].save(t, safe=True)
returnD(True)
def _wrap(*args, **kwargs):
return deferToThreadPool(reactor, retval._pool,
_et(_user_callable), *args,
**kwargs)
def run(self, function, *args, **kwargs):
return deferToThreadPool(reactor, State.orm_tp, function, *args,
**kwargs)
def test1():
d = deferToThreadPool(reactor, tp, test2)
d.addCallback(cb)
def do_with_engine(self, callable, *args, **kwargs):
return threads.deferToThreadPool(self.reactor, self._pool,
self.__thd, True, callable, args, kwargs)
def new_deferToThreadPool(reactor, threadpool, f, *args, **kwargs):
"""Variant of Twisted's that wraps all functions in `synchronous`."""
func = f if ISynchronous.providedBy(f) else synchronous(f)
return deferToThreadPool(reactor, threadpool, func, *args, **kwargs)
def _stop(self):
self._stop_evt = None
threads.deferToThreadPool(self.reactor, self._pool,
self.engine.dispose)
self._pool.stop()
self.running = False
def newfunc(*a, **kw):
"""I call blocking python code in a seperate thread.
I return a deferred and will fire callbacks or errbacks.
"""
return threads.deferToThreadPool(R, pool, func, *a, **kw)
def do(self, callable, *args, **kwargs):
ret = yield threads.deferToThreadPool(self.reactor, self._pool,
self.__thd, False, callable,
args, kwargs)
return ret
def runWithConnection(func, *args, **kwargs):
return threads.deferToThreadPool(pool._reactor, pool.threadpool,
pool._runWithConnection, func, *args,
**kwargs)
def perform_job_to_thread(self, job):
d = threads.deferToThreadPool(reactor, self._thread_pool, job.perform)
d.addCallback(self.callback_perform_job)
d.addErrback(self.errback_perform_job, job=job)
def runInteraction(interaction, *args, **kwargs):
return threads.deferToThreadPool(pool._reactor, pool.threadpool,
pool._runInteraction, interaction,
*args, **kwargs)
def deferToDatabase(func, *args, **kwargs):
"""Call `func` in a thread where database activity is permitted."""
return threads.deferToThreadPool(reactor, reactor.threadpoolForDatabase,
func, *args, **kwargs)
def deferToThread(f, *args, **kwargs):
return threads.deferToThreadPool(self.reactor,
self.reactor.getThreadPool(), f,
*args, **kwargs)
def run(function, *args, **kwargs):
"""
Defer provided function to thread
"""
return deferToThreadPool(reactor, transact.tp, function, *args,
**kwargs)