def test_start(self):
"""
L{ThreadPool.start} creates the minimum number of threads specified.
"""
pool = threadpool.ThreadPool(0, 5)
pool.start()
self.addCleanup(pool.stop)
self.assertEqual(len(pool.threads), 0)
pool = threadpool.ThreadPool(3, 10)
self.assertEqual(len(pool.threads), 0)
pool.start()
self.addCleanup(pool.stop)
self.assertEqual(len(pool.threads), 3)
def get_new_recipients_threadpool():
global __new_recipients_threadpool
if __new_recipients_threadpool is None:
__new_recipients_threadpool = threadpool.ThreadPool(
1, 1, "cm.get_new_recipients")
__new_recipients_threadpool.start()
return __new_recipients_threadpool
def makeService(self, options):
pool = threadpool.ThreadPool(minthreads=1, maxthreads=100)
reactor.callWhenRunning(pool.start)
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
root = wsgi.WSGIResource(reactor, pool, wsgi_hello.application)
site = server.Site(root)
return strports.service('tcp:8000', site)
def start():
# Import the module containing the resources
import sse
pool = threadpool.ThreadPool()
pool.start()
# Prepare the app root and subscribe endpoint
root = sse.Root()
subscribe = sse.Subscribe()
broadcast = sse.Broadcast(subscribe.publish_to_all)
connections = sse.Connections(subscribe.get_subscribers_count)
# Add sse and connections as children of root
root.putChild('sse', subscribe)
root.putChild('connections', connections)
root.putChild('broadcast', broadcast)
# Allow Ctrl-C to get you out cleanly:
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
# emit an event every second
l = task.LoopingCall(publish_timestamp,
{"broadcast": subscribe.publish_to_all})
l.start(1.0)
site = twisted_server.Site(root)
server = internet.TCPServer(1942, site)
application = service.Application("twisted-sse")
server.setServiceParent(application)
return application
def main(reactor, *descriptions):
log = Logger()
globalLogBeginner.beginLoggingTo([textFileLogObserver(sys.stdout)])
endpointObjects = [
endpoints.clientFromString(reactor, description)
for description in descriptions
]
hostPorts = [(endpoint._host, endpoint._port)
for endpoint in endpointObjects]
pool = threadpool.ThreadPool(minthreads=1, maxthreads=1, name="persiter")
persister = Persists(reactor, pool)
reactor.addSystemEventTrigger("before", "shutdown", persister.stop)
persister.start("log.sqlite", hostPorts)
analyzer = AnalyzesText(persister)
factory = EncodingCollectionFactory(reactor, random.SystemRandom(),
analyzer)
for (host, port), endpoint in zip(hostPorts, endpointObjects):
try:
protocol = yield endpoint.connect(factory)
except Exception:
log.failure("Could not connect to {host}:{port}",
host=host,
port=port)
raise
protocol.addr = (host, port)
defer.returnValue(defer.Deferred())
def get_item_range(key: str, start: int, counts: int, base: int):
def func(*args):
"""获取数据"""
_key, _start, _end, _c = args
data_list = redis_client.lrange(name=_key, start=_start, end=_end)
data_list = [{"applicant_cn": data} for data in data_list]
return data_list
def on_result(_, result):
start_time = time.time()
NUM = len(result)
with mysql_db.atomic():
for i in range(0, NUM, 1000):
# T_Applicant_2019.insert_many(result[i: i + 1000]).execute()
T_Applicant_2019_20w.insert_many(result[i:i + 1000]).execute()
print(f"执行插入 1000 条, 当然进度为 {i}: {NUM}")
print("插入{}条数据, 花费: {}秒".format(NUM, time.time() - start_time))
cycle_num = math.ceil(counts / base)
params_list = []
end = base - 1
th_pool = threadpool.ThreadPool()
for c in range(cycle_num):
params_list.append((key, start, end, c))
start, end = end + 1, end + base
for params in params_list:
th_pool.callInThreadWithCallback(on_result, func, *params)
th_pool.start()
th_pool.stop()
def wsgi_resource():
pool = threadpool.ThreadPool()
pool.start()
# Allow Ctrl-C to get you out cleanly:
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
wsgi_resource = wsgi.WSGIResource(reactor, pool, WSGIHandler())
return wsgi_resource
def test_callbackContext(self):
"""
The context L{ThreadPool.callInThreadWithCallback} is invoked in is
shared by the context the callable and C{onResult} callback are
invoked in.
"""
myctx = context.theContextTracker.currentContext().contexts[-1]
myctx['testing'] = 'this must be present'
contexts = []
event = threading.Event()
def onResult(success, result):
ctx = context.theContextTracker.currentContext().contexts[-1]
contexts.append(ctx)
event.set()
def func():
ctx = context.theContextTracker.currentContext().contexts[-1]
contexts.append(ctx)
tp = threadpool.ThreadPool(0, 1)
tp.callInThreadWithCallback(onResult, func)
tp.start()
self.addCleanup(tp.stop)
event.wait(self.getTimeout())
self.assertEqual(len(contexts), 2)
self.assertEqual(myctx, contexts[0])
self.assertEqual(myctx, contexts[1])
def _initThreadPool(self):
from twisted.python import threadpool
self.threadpool = threadpool.ThreadPool(
0, 10, 'twisted.internet.reactor')
self.callWhenRunning(self.threadpool.start)
self.addSystemEventTrigger('during', 'shutdown',
self.threadpool.stop)
def test_callbackThread(self):
"""
L{ThreadPool.callInThreadWithCallback} calls the function it is
given and the C{onResult} callback in the same thread.
"""
threadIds = []
import thread
event = threading.Event()
def onResult(success, result):
threadIds.append(thread.get_ident())
event.set()
def func():
threadIds.append(thread.get_ident())
tp = threadpool.ThreadPool(0, 1)
tp.callInThreadWithCallback(onResult, func)
tp.start()
self.addCleanup(tp.stop)
event.wait(self.getTimeout())
self.assertEqual(len(threadIds), 2)
self.assertEqual(threadIds[0], threadIds[1])
def __init__(self, config):
self.config = config
self.step = self.config[self.name]['step']
self.call = (self.update_feed_list, [], {})
self.pool = threadpool.ThreadPool(name=self.name)
reactor.callWhenRunning(self.pool.start)
reactor.addSystemEventTrigger('after', 'shutdown', self.pool.stop)
def _threadpoolTest(self, method):
"""
Test synchronization of calls made with C{method}, which should be
one of the mechanisms of the threadpool to execute work in threads.
"""
# This is a schizophrenic test: it seems to be trying to test
# both the callInThread()/dispatch() behavior of the ThreadPool as well
# as the serialization behavior of threadable.synchronize(). It
# would probably make more sense as two much simpler tests.
N = 10
tp = threadpool.ThreadPool()
tp.start()
self.addCleanup(tp.stop)
waiting = threading.Lock()
waiting.acquire()
actor = Synchronization(N, waiting)
for i in range(N):
method(tp, actor)
self._waitForLock(waiting)
self.failIf(actor.failures, "run() re-entered %d times" %
(actor.failures,))
def test_callInThreadWithCallback(self):
"""
L{ThreadPool.callInThreadWithCallback} calls C{onResult} with a
two-tuple of C{(True, result)} where C{result} is the value returned
by the callable supplied.
"""
waiter = threading.Lock()
waiter.acquire()
results = []
def onResult(success, result):
waiter.release()
results.append(success)
results.append(result)
tp = threadpool.ThreadPool(0, 1)
tp.callInThreadWithCallback(onResult, lambda: "test")
tp.start()
try:
self._waitForLock(waiter)
finally:
tp.stop()
self.assertTrue(results[0])
self.assertEqual(results[1], "test")
def test_callInThreadWithCallbackExceptionInCallback(self):
"""
L{ThreadPool.callInThreadWithCallback} calls C{onResult} with a
two-tuple of C{(False, failure)} where C{failure} represents the
exception raised by the callable supplied.
"""
class NewError(Exception):
pass
def raiseError():
raise NewError()
waiter = threading.Lock()
waiter.acquire()
results = []
def onResult(success, result):
waiter.release()
results.append(success)
results.append(result)
tp = threadpool.ThreadPool(0, 1)
tp.callInThreadWithCallback(onResult, raiseError)
tp.start()
try:
self._waitForLock(waiter)
finally:
tp.stop()
self.assertFalse(results[0])
self.assertTrue(isinstance(results[1], failure.Failure))
self.assertTrue(issubclass(results[1].type, NewError))
def test_callInThreadWithCallbackExceptionInOnResult(self):
"""
L{ThreadPool.callInThreadWithCallback} logs the exception raised by
C{onResult}.
"""
class NewError(Exception):
pass
waiter = threading.Lock()
waiter.acquire()
results = []
def onResult(success, result):
results.append(success)
results.append(result)
raise NewError()
tp = threadpool.ThreadPool(0, 1)
tp.callInThreadWithCallback(onResult, lambda : None)
tp.callInThread(waiter.release)
tp.start()
try:
self._waitForLock(waiter)
finally:
tp.stop()
errors = self.flushLoggedErrors(NewError)
self.assertEqual(len(errors), 1)
self.assertTrue(results[0])
self.assertEqual(results[1], None)
def setUp(self):
self.threadpool = threadpool.ThreadPool(0, 10)
self.event = threading.Event()
self.threadpool.start()
def done():
self.threadpool.stop()
del self.threadpool
self.addCleanup(done)
def wsgi_resource():
pool = threadpool.ThreadPool()
pool.start()
# Allow Ctrl-C to get you out cleanly:
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
handler = StaticFilesHandler(get_internal_wsgi_application())
wsgi_resource = wsgi.WSGIResource(reactor, pool, handler)
return wsgi_resource
def __init__(self, func=None, arglistlist=None, workers=5, debug=True):
"""arglistlist is a list of a list of args"""
self.jobsfinished = 0
self.jobstodo = 0
self.debug = debug
self.tp = threadpool.ThreadPool(workers, workers)
for arglist in arglistlist or []:
self.addjob(func, *arglist)
def setUp(self):
from twisted.python import threadpool
reactor.threadpool = threadpool.ThreadPool(0, 10)
reactor.threadpool.start()
self.path = tempfile.mkdtemp(suffix=".flumotion.test")
self.stats = DummyStats()
self._file = None
def test_attributes(self):
"""
L{ThreadPool.min} and L{ThreadPool.max} are set to the values passed to
L{ThreadPool.__init__}.
"""
pool = threadpool.ThreadPool(12, 22)
self.assertEqual(pool.min, 12)
self.assertEqual(pool.max, 22)
def _initThreadPool(self):
"""
Create the threadpool accessible with callFromThread.
"""
from twisted.python import threadpool
self.threadpool = threadpool.ThreadPool(0, 10, 'twisted.internet.reactor')
self.callWhenRunning(self.threadpool.start)
self.threadpoolShutdownID = self.addSystemEventTrigger(
'during', 'shutdown', self._stopThreadPool)
def test_adjustingWhenPoolStopped(self):
"""
L{ThreadPool.adjustPoolsize} only modifies the pool size and does not
start new workers while the pool is not running.
"""
pool = threadpool.ThreadPool(0, 5)
pool.start()
pool.stop()
pool.adjustPoolsize(2)
self.assertEqual(len(pool.threads), 0)
def __init__(self):
resource.Resource.__init__(self)
pool = threadpool.ThreadPool()
pool.start()
# Allow Ctrl-C to get you out cleanly:
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
self.wsgi_resource = wsgi.WSGIResource(reactor, pool, WSGIHandler())
staticsrc = static.File(blog.settings.STATIC_ROOT, blog.settings.STATIC_URL)
self.putChild('static', staticsrc)
def test_workerState(self):
"""
Upon entering a _workerState block, the threads unique identifier is
added to a stateList and is removed upon exiting the block.
"""
pool = threadpool.ThreadPool()
workerThread = object()
stateList = []
with pool._workerState(stateList, workerThread):
self.assertIn(workerThread, stateList)
self.assertNotIn(workerThread, stateList)
def makeService(self, options):
s = service.MultiService()
pool = threadpool.ThreadPool()
reactor.callWhenRunning(pool.start)
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
root = wsgi.WSGIResource(reactor, pool, wsgi_param.application)
site = server.Site(root)
strports.service('tcp:8000', site).setServiceParent(s)
ts = internet.TimerService(1, update, wsgi_param.application, reactor)
ts.setServiceParent(s)
return s
def wsgi_resource(multi):
# make a new ThreadPoolService and add it to the multi service
tps = ThreadPoolService(threadpool.ThreadPool())
tps.setServiceParent(multi)
# Allow Ctrl-C to get you out cleanly:
reactor.addSystemEventTrigger('after', 'shutdown', tps.pool.stop)
wsgi_resource = wsgi.WSGIResource(reactor, tps.pool, WSGIHandler())
return wsgi_resource
def makeService(self, options):
s = service.MultiService()
pool = threadpool.ThreadPool()
reactor.callWhenRunning(pool.start)
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
root = wsgi.WSGIResource(reactor, pool, wsgi_param.application)
site = server.Site(root)
strports.service('tcp:8000', site).setServiceParent(s)
factory = protocol.Factory.forProtocol(UpdateMessage)
factory.application = wsgi_param.application
strports.service('tcp:8001', factory).setServiceParent(s)
return s
def makeService(self, options):
application = wsgi_hello.application
pool = threadpool.ThreadPool(minthreads=1, maxthreads=100)
reactor.callWhenRunning(pool.start)
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
dynamic = wsgi.WSGIResource(reactor, pool, application)
files = static.File('static')
root = vhost.NameVirtualHost()
root.addHost(b'app.example.org', dynamic)
root.addHost(b'static.example.org', files)
site = server.Site(root)
return strports.service(options['port'], site)
def do_cleanThreads(cls):
from twisted.internet import reactor
if interfaces.IReactorThreads.providedBy(reactor):
reactor.suggestThreadPoolSize(0)
if hasattr(reactor, 'threadpool') and reactor.threadpool:
reactor.threadpool.stop()
reactor.threadpool = None
# *Put it back* and *start it up again*. The
# reactor's threadpool is *private*: we cannot just
# rape it and walk away.
reactor.threadpool = threadpool.ThreadPool(0, 10)
reactor.threadpool.start()
def opt_wsgi(self, name):
"""
The FQPN of a WSGI application object to serve as the root resource of
the webserver.
"""
pool = threadpool.ThreadPool()
reactor.callWhenRunning(pool.start)
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
try:
application = reflect.namedAny(name)
except (AttributeError, ValueError):
raise usage.UsageError("No such WSGI application: %r" % (name,))
self['root'] = wsgi.WSGIResource(reactor, pool, application)