def __init__(self):
minthreads = config["jobtype_logging_threadpool"]["min_threads"]
maxthreads = config["jobtype_logging_threadpool"]["max_threads"]
self.max_queued_lines = \
config["jobtype_logging_threadpool"]["max_queue_size"]
self.flush_lines = \
config["jobtype_logging_threadpool"]["flush_lines"]
self.stopped = False
if minthreads < 1:
raise ValueError(
"Config value "
"jobtype_logging_threadpool.min_threads must be >= 1")
# Calculate maxthreads if a value was not provided for us
if maxthreads == "auto":
auto_maxthreads = min(int(cpu.total_cpus() * 1.5), 20)
maxthreads = max(auto_maxthreads, minthreads)
if minthreads > maxthreads:
raise ValueError(
"Config value jobtype_logging_threadpool.min_threads cannot "
"be larger than jobtype_logging_threadpool.max_threads")
ThreadPool.__init__(
self,
minthreads=minthreads, maxthreads=maxthreads,
name=self.__class__.__name__)
def __init__(self, core):
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
application = get_flask_application(core)
wsgi_resource = WSGIResource(reactor, thread_pool, application)
Site.__init__(self, wsgi_resource)
class SeleniumDownloadHandler(HTTP11DownloadHandler):
def __init__(self, settings):
super(SeleniumDownloadHandler, self).__init__(settings)
self._enable_driver = settings.getbool('WEB_DRIVER_ENABLED')
self._driver_name = settings.get('WEB_DRIVER_NAME')
self._driver_path = settings.get('WEB_DRIVER_PATH')
selenium_concurrent_request = settings.get(
'WEB_DRIVER_CONCURRENT_REQUESTS', 16)
self._thread_pool = ThreadPool(minthreads=selenium_concurrent_request,
maxthreads=selenium_concurrent_request)
self._thread_pool.start()
self._driver_timeout = settings.get('WEB_DRIVER_TIMEOUT', 300)
def download_request(self, request, spider):
if self._enable_driver and request.meta.get('selenium_needed', False):
agent = SeleniumAsyncAgent(contextFactory=self._contextFactory,
pool=self._thread_pool,
driverName=self._driver_name,
driverPath=self._driver_path,
connectTimeout=self._driver_timeout)
return agent.download_request(request)
else:
return super(SeleniumDownloadHandler,
self).download_request(request, spider)
def close(self):
super(SeleniumDownloadHandler, self).close()
self._thread_pool.stop()
class NoGoStatusCodes(TestCase):
def __init__(self, *args, **kwargs):
self.tp = ThreadPool(maxthreads=20)
self.tp.start()
self.resource = HendrixWSGIResource(reactor,
self.tp,
self.wsgi_thing)
self.nameSpace = TestNameSpace()
self.nameSpace.async_thing_complete = Queue()
return super(NoGoStatusCodes, self).__init__(*args, **kwargs)
def setUp(self, *args, **kwargs):
self.addCleanup(self.tp.stop)
super(NoGoStatusCodes, self).setUp(*args, **kwargs)
def wsgi_thing(self, environ, start_response):
start_response('404 NOT FOUND', [('Content-type','text/plain')])
@crosstown_traffic.follow_response(
no_go_status_codes=self.no_go_status_codes,
same_thread=True
)
def long_thing_on_same_thread():
self.nameSpace.async_task_was_run = True
logger.info("No bad status codes; went ahead with async thing.")
return "Nothing."
def test_bad_status_codes_cause_no_go_in_wsgi_response(self):
self.no_go_status_codes = [404, '6xx']
request = DummyRequest('r1')
request.isSecure = lambda: False
request.content = "llamas"
finished = request.notifyFinish()
self.resource.render(request)
# This must wait until the WSGI response is closed.
finished.addCallback(
lambda _: self.assertFalse(
self.nameSpace.async_task_was_run
)
)
def test_bad_status_codes_cause_no_go_flag(self):
through_to_you = crosstown_traffic.follow_response(no_go_status_codes=[418])
through_to_you.status_code = 418
through_to_you.check_status_code_against_no_go_list()
self.assertTrue(through_to_you.no_go)
def test_no_bad_status_codes_are_cool(self):
through_to_you = crosstown_traffic.follow_response(no_go_status_codes=[418])
through_to_you.status_code = 404
through_to_you.check_status_code_against_no_go_list()
self.assertFalse(through_to_you.no_go)
def startup(self):
"""
Startup Docker client.
"""
if not self._finished:
self.log.warn('Docker client already running!')
return
self.console('module', 'starting')
self._finished = False
self._channels = Channels()
# dedicated threadpool for docker work
self._threadpool = ThreadPool(minthreads=4,
maxthreads=100,
name='docker')
self._threadpool.start()
# our 'events' pub/sub docker events emulator
threads.deferToThreadPool(self._reactor, self._threadpool,
self.events)
# our 'logs' pub/sub docker console output
threads.deferToThreadPool(self._reactor, self._threadpool,
self.logs)
# our 'keepalive' monitor
threads.deferToThreadPool(self._reactor, self._threadpool,
self.keepalive)
self.console('module', 'started')
def __init__(self, hs, config):
self.cache_directory = hs.config.media.media_store_path
self.bucket = config["bucket"]
self.storage_class = config["storage_class"]
self.api_kwargs = {}
if "region_name" in config:
self.api_kwargs["region_name"] = config["region_name"]
if "endpoint_url" in config:
self.api_kwargs["endpoint_url"] = config["endpoint_url"]
if "access_key_id" in config:
self.api_kwargs["aws_access_key_id"] = config["access_key_id"]
if "secret_access_key" in config:
self.api_kwargs["aws_secret_access_key"] = config["secret_access_key"]
self._s3_client = None
self._s3_client_lock = threading.Lock()
threadpool_size = config.get("threadpool_size", 40)
self._s3_pool = ThreadPool(name="s3-pool", maxthreads=threadpool_size)
self._s3_pool.start()
# Manually stop the thread pool on shutdown. If we don't do this then
# stopping Synapse takes an extra ~30s as Python waits for the threads
# to exit.
reactor.addSystemEventTrigger(
"during", "shutdown", self._s3_pool.stop,
)
def __init__(self, core):
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger("after", "shutdown", thread_pool.stop)
application = get_flask_application(core)
wsgi_resource = WSGIResource(reactor, thread_pool, application)
Site.__init__(self, wsgi_resource)
def __init__(self):
from twisted.internet import reactor # Imported here.inside
self.reactor = reactor
engine = get_engine()
create_schema(engine)
self.read_pool = ThreadPool(
minthreads=1, maxthreads=16, name="ReadPool")
self.write_pool = ThreadPool(
minthreads=1, maxthreads=1, name="WritePool")
self.read_pool.start()
self.write_pool.start()
self.signals = SignalManager(dispatcher.Any).connect(
self.stop_threadpools, spider_closed)
self.counters = defaultdict(lambda: Counter())
self.cache = defaultdict(
lambda: dict())
self.write_queue = Queue()
self.writelock = False # Write queue mutex
def makeService(self, options):
app = Application("Mercurial SSH Server") #, uid, gid)
services = IServiceCollection(app)
service = options.subOptions.getService()
service.setServiceParent(services)
wsgi_app = WSGIApplication()
threadpool = ThreadPool(config.web.min_threads, config.web.max_threads)
threadpool.start()
reactor.addSystemEventTrigger('after', 'shutdown', threadpool.stop)
root = wsgi.WSGIResource(reactor, threadpool, wsgi_app)
factory = server.Site(root)
if isfile(config.web.certificate):
from OpenSSL import SSL
# Run SSL Server
class SSLContext(object):
def getContext(self):
ctx = SSL.Context(SSL.SSLv23_METHOD)
ctx.use_privatekey_file(config.private_key)
ctx.use_certificate_file(config.web.certificate)
return ctx
config_service = internet.SSLServer(config.web.port, factory,
SSLContext())
else:
config_service = internet.TCPServer(config.web.port, factory)
config_service.setServiceParent(app)
clean_changes_task = task.LoopingCall(self.__clean_old_changes_from_db)
clean_changes_task.start(5*60, now=True) # Every 5 minutes
reactor.addSystemEventTrigger('after', 'shutdown',
clean_changes_task.stop)
return services
class ClockWithThreads(Clock):
"""
A testing reactor that supplies L{IReactorTime} and L{IReactorThreads}.
"""
def __init__(self):
super(ClockWithThreads, self).__init__()
self._pool = ThreadPool()
def getThreadPool(self):
"""
Get the threadpool.
"""
return self._pool
def suggestThreadPoolSize(self, size):
"""
Approximate the behavior of a 'real' reactor.
"""
self._pool.adjustPoolsize(maxthreads=size)
def callInThread(self, thunk, *a, **kw):
"""
No implementation.
"""
def callFromThread(self, thunk, *a, **kw):
"""
class ClockWithThreads(Clock):
"""
A testing reactor that supplies L{IReactorTime} and L{IReactorThreads}.
"""
def __init__(self):
super(ClockWithThreads, self).__init__()
self._pool = ThreadPool()
def getThreadPool(self):
"""
Get the threadpool.
"""
return self._pool
def suggestThreadPoolSize(self, size):
"""
Approximate the behavior of a "real" reactor.
"""
self._pool.adjustPoolsize(maxthreads=size)
def callInThread(self, thunk, *a, **kw):
"""
No implementation.
"""
def callFromThread(self, thunk, *a, **kw):
"""
class FakeReactor(object):
"""
A fake reactor implementation which just supports enough reactor APIs for
L{ThreadedResolver}.
"""
implements(IReactorTime, IReactorThreads)
def __init__(self):
self._clock = Clock()
self.callLater = self._clock.callLater
self._threadpool = ThreadPool()
self._threadpool.start()
self.getThreadPool = lambda: self._threadpool
self._threadCalls = Queue()
def callFromThread(self, f, *args, **kwargs):
self._threadCalls.put((f, args, kwargs))
def _runThreadCalls(self):
f, args, kwargs = self._threadCalls.get()
f(*args, **kwargs)
def _stop(self):
self._threadpool.stop()
def __init__(self, minthreads=1, maxthreads=4):
self.pool = ThreadPool(minthreads=minthreads, maxthreads=maxthreads)
# unclosed ThreadPool leads to reactor hangs at shutdown
# this is a problem in many situation, so better enforce pool stop here
reactor.addSystemEventTrigger("before", "shutdown", self.pool.stop)
self.pool.start()
def __init__(self, spider):
from twisted.internet import reactor # Imported here.inside
self.spider = spider
''' Used for logging for now '''
self.reactor = reactor
''' Used for thred pools '''
engine = get_engine()
create_schema(engine)
self.thread_pool = ThreadPool(
minthreads=1, maxthreads=13, name="ReadPool")
# There should be only one pool in the write_pool
# Never increase maxtreads value
self.write_pool = ProfiledThreadPool(
minthreads=1, maxthreads=1, name="WritePool")
self.thread_pool.start()
self.write_pool.start()
self.signals = SignalManager(dispatcher.Any).connect(
self.stop_threadpools, spider_closed)
self.reporter = Reporter()
''' Reporer is used for statistics collection '''
self.counters = self.reporter.counters
self.cache = defaultdict(
lambda: dict())
self.write_queue = Queue()
self.writelock = False # Write queue mutex
def __init__(self, result_class):
from twisted.python.threadpool import ThreadPool
SimpleRunner.__init__(self, _threadclass(result_class))
self._pool = ThreadPool()
self._pool.start()
class FakeReactor(object):
"""
A fake reactor implementation which just supports enough reactor APIs for
L{ThreadedResolver}.
"""
def __init__(self):
self._clock = Clock()
self.callLater = self._clock.callLater
self._threadpool = ThreadPool()
self._threadpool.start()
self.getThreadPool = lambda: self._threadpool
self._threadCalls = Queue()
def callFromThread(self, f, *args, **kwargs):
self._threadCalls.put((f, args, kwargs))
def _runThreadCalls(self):
f, args, kwargs = self._threadCalls.get()
f(*args, **kwargs)
def _stop(self):
self._threadpool.stop()
class ProviderServer(object):
def __init__(self, reactor, port):
self.reactor = reactor
self.reactor.addSystemEventTrigger('before', 'shutdown',
self.before_shutdown)
self.reactor.suggestThreadPoolSize(config.REACTOR_THREAD_POOL_MAX)
self.app = create_app()
self.thread_pool = ThreadPool(maxthreads=config.FLASK_THREAD_POOL_MAX)
self.thread_pool.start()
wsgi_resource = WSGIResource(self.reactor, self.thread_pool, self.app)
root_resource = RootResource(wsgi_resource)
root_resource.putChild("metrics", MetricsResource())
site = Site(root_resource)
self.bind = self.reactor.listenTCP(port, site)
log.info('Provider is listening on {} ...'.format(port))
def run(self):
self.reactor.run()
def stop_services(self):
log.info("Shutting down provider...")
self.app.cleanup()
self.thread_pool.stop()
return maybeDeferred(self.bind.stopListening).addCallbacks(
callback=lambda _: log.info("Port listening was stopped"),
errback=lambda failure: log.error(
"Error while stopping port listening: {}".format(failure)))
@inlineCallbacks
def before_shutdown(self):
self.app.stop()
yield self.stop_services()
class TwistedApplicationServer(ApplicationServer):
def __init__(self, app, port, host, debug):
super(TwistedApplicationServer,
self).__init__(app, port, host, debug)
self.reactor = reactor
self.thread_pool = ThreadPool(5, 40)
self.resource = WSGIResource(self.reactor, self.thread_pool,
self.app)
self.reactor.addSystemEventTrigger('after', 'shutdown',
self.thread_pool.stop)
def run(self):
print("Begin Listening")
self.thread_pool.start()
self.reactor.listenTCP(self.port,
Site(self.resource),
interface=self.host)
self.reactor.run()
def shutdown(self):
print("Received call to shutdown")
print("Reactor should have stopped")
self.reactor.callFromThread(self.reactor.stop)
print("Reactor stop listening")
self.reactor.callFromThread(self.reactor.stopListening)
print("Reactor Shutdown")
def from_config(cls, reactor, logger, queue, config_path):
with open(config_path) as f:
config = yaml.safe_load(f)
# TODO: bump this once alchimia properly handles pinning
thread_pool = ThreadPool(minthreads=1, maxthreads=1)
thread_pool.start()
reactor.addSystemEventTrigger('during', 'shutdown', thread_pool.stop)
return cls(
logger,
DownloadDatabase(reactor, thread_pool, config["db"]["uri"]),
FilePath(config["storage"]["filesystem"]),
fernet.MultiFernet(
[fernet.Fernet(key) for key in config["encryption_keys"]]),
VBMSClient(
reactor,
connect_vbms_path=config["connect_vbms"]["path"],
bundle_path=config["connect_vbms"]["bundle_path"],
endpoint_url=config["vbms"]["endpoint_url"],
keyfile=config["vbms"]["keyfile"],
samlfile=config["vbms"]["samlfile"],
key=config["vbms"].get("key"),
keypass=config["vbms"]["keypass"],
ca_cert=config["vbms"].get("ca_cert"),
client_cert=config["vbms"].get("client_cert"),
),
queue,
config["env"],
)
def __init__(self):
from twisted.internet import reactor # Imported here.inside
self.reactor = reactor
engine = get_engine()
# create_schema(engine)
self.read_pool = ThreadPool(
minthreads=1, maxthreads=16, name="ReadPool")
self.write_pool = ThreadPool(
minthreads=1, maxthreads=1, name="WritePool")
self.read_pool.start()
self.write_pool.start()
self.signals = SignalManager(dispatcher.Any).connect(
self.stop_threadpools, spider_closed)
self.counters = defaultdict(lambda: Counter())
self.cache = defaultdict(
lambda: dict())
self.write_queue = Queue()
self.writelock = False # Write queue mutex
def from_config(cls, reactor, logger, queue, config_path):
with open(config_path) as f:
config = yaml.safe_load(f)
# TODO: bump this once alchimia properly handles pinning
thread_pool = ThreadPool(minthreads=1, maxthreads=1)
thread_pool.start()
reactor.addSystemEventTrigger('during', 'shutdown', thread_pool.stop)
return cls(
logger,
DownloadDatabase(reactor, thread_pool, config["db"]["uri"]),
FilePath(config["storage"]["filesystem"]),
fernet.MultiFernet([
fernet.Fernet(key) for key in config["encryption_keys"]
]),
VBMSClient(
reactor,
connect_vbms_path=config["connect_vbms"]["path"],
bundle_path=config["connect_vbms"]["bundle_path"],
endpoint_url=config["vbms"]["endpoint_url"],
keyfile=config["vbms"]["keyfile"],
samlfile=config["vbms"]["samlfile"],
key=config["vbms"].get("key"),
keypass=config["vbms"]["keypass"],
ca_cert=config["vbms"].get("ca_cert"),
client_cert=config["vbms"].get("client_cert"),
),
queue,
config["env"],
)
def __init__(self,
gossip_obj,
ledger_obj,
config,
windows_service=False,
http_port=None,
):
'''
Creates a validator. As a current side-effect, does some
initialization on it's ledger_obj argumenet
Args:
node_obj: (gossip.Node)
ledger_obj: (journal.Journal)
config: (dict)
windows_service: (bool)
http_port: (int)
'''
self.status = 'stopped'
self.Config = config
self.gossip = gossip_obj
node_obj = gossip_obj.LocalNode
self._gossip_host = node_obj.NetHost
self._gossip_port = node_obj.NetAddress
self._endpoint_host = node_obj.endpoint_host
self._endpoint_port = node_obj.endpoint_port
self._endpoint_http_port = http_port
self.Ledger = ledger_obj
self.profile = self.Config.get('Profile', False)
if self.profile:
self.pr = cProfile.Profile()
self.pr.enable()
self.windows_service = windows_service
# flag to indicate that a topology update is in progress
self._topology_update_in_progress = False
self.delaystart = self.Config['DelayStart']
# set up signal handlers for shutdown
if not windows_service:
signal.signal(signal.SIGTERM, self.handle_shutdown_signal)
signal.signal(signal.SIGINT, self.handle_shutdown_signal)
# ---------- Initialize the configuration ----------
self.initialize_common_configuration()
# ---------- Initialize the NodeMap ----------
self.initialize_node_map()
# ---------- Initialize the Ledger ----------
self.initialize_ledger_object()
maxsize = self.Config.get("WebPoolSize", 8)
self.web_thread_pool = ThreadPool(0, maxsize, "WebThreadPool")
def test_contemporaneous_requests(self):
'''
We're going to create two request-response cycles here:
Cycle 1 will begin.
Cycle 2 will begin.
Cycle 2 will return.
Cycle 1 will return.
This way, we can prove that the crosstown_traffic created
by cycle 1 is not resolved by the return of cycle 2.
'''
tp = ThreadPool(maxthreads=20)
tp.start()
self.addCleanup(tp.stop)
log.debug("\n\nStarting the two stream stuff.")
request1 = DummyRequest('r1')
request1.isSecure = lambda: False
request1.content = "Nothing really here."
request1.headers['llamas'] = 'dingo'
nameSpace.test_case = self
hr = HendrixWSGIResource(reactor, tp, wsgi_application)
d1 = deferToThreadPool(reactor, tp, hr.render, request1)
request2 = DummyRequest('r2')
request2.isSecure = lambda: False
request2.content = "Nothing really here."
request2.headers['llamas'] = 'dingo'
d2 = deferToThreadPool(reactor, tp, hr.render, request2)
def woah_stop(failure):
nameSpace.async_task_was_done.put_nowait(False)
nameSpace.second_cycle_complete.put_nowait(False)
nameSpace.ready_to_proceed_with_second_cycle.put_nowait(False)
d1.addErrback(woah_stop)
d2.addErrback(woah_stop)
combo_deferred = gatherResults([d1, d2])
def wait_for_queue_resolution():
nameSpace.async_task_was_done.get(True, 3)
combo_deferred.addCallback(
lambda _: deferToThreadPool(reactor, tp, wait_for_queue_resolution)
)
combo_deferred.addCallback(
lambda _: self.assertTrue(nameSpace.async_task_was_run)
)
return combo_deferred
def __init__(self, engine, maxthreads=10, verbose=False):
if engine.dialect.name == 'sqlite':
ThreadPool.__init__(self, minthreads=1, maxthreads=1)
else:
ThreadPool.__init__(self, maxthreads=maxthreads)
self.engine = engine
reactor.callWhenRunning(self.start)
def setUp(self):
self.db = MockStore()
GSM.registerUtility(MockZStorm(self.db))
self.tp = ThreadPool(0, 2)
self.sm = MockServerManager(reactor, SERVERS)
self.updater = Updater(Transactor(self.tp), self.sm)
self.tp.start()
def __init__(self, engine, maxthreads=10, verbose=False):
if engine.dialect.name == 'sqlite':
ThreadPool.__init__(self, minthreads=1, maxthreads=1)
else:
ThreadPool.__init__(self, maxthreads=maxthreads)
self.engine = engine
reactor.callWhenRunning(self.start)
def _test_StreamingServer():
expected_id = 'test'
headers = {
#'TimeSeekRange.dlna.org': 'npt=30.000-',
'TimeSeekRange.dlna.org': 'npt=20.000-50.000',
#'Range': 'bytes=5-',
'Connection': 'close',
}
duration = '00:01:00'
content_length = 64 * 1024
interface = '192.168.0.103'
class StreamingServerStub(ByteSeekMixin, TimeSeekMixin, StreamingServer):
def get_content(self, id, environ):
eq_(expected_id, id)
#raise ValueError(repr(environ))
for k in headers:
eq_(headers[k], environ['HTTP_' + k.upper()])
return ContentStub(content_length,
'video/mpeg',
'DLNA.ORG_PN=MPEG_PS_NTSC;DLNA.ORG_OP=11')
from twisted.web import client
class HTTPPageGetter(client.HTTPPageGetter):
handleStatus_206 = lambda self: self.handleStatus_200()
class HTTPClientFactory(client.HTTPClientFactory):
protocol = HTTPPageGetter
def getPageFactory(url, *args, **kwargs):
scheme, host, port, path = client._parse(url)
factory = HTTPClientFactory(url, *args, **kwargs)
reactor.connectTCP(host, port, factory)
return factory
app = StreamingServerStub('test')
mapper = Mapper()
mapper.connect(':id', controller='mt', action='get')
app = RoutesMiddleware(app, mapper)
tpool = ThreadPool()
tpool.start()
resource = upnp.WSGIResource(reactor, tpool, app)
port = reactor.listenTCP(0, server.Site(resource), interface=interface)
port_num = port.socket.getsockname()[1]
url = 'http://%s:%i/%s?duration=%s' % (interface, port_num, expected_id, duration)
factory = getPageFactory(url, headers=headers)
def check_result(contents):
raise ValueError('expected: %d, actual: %d' % (content_length, len(contents)))
#raise ValueError(repr(factory.response_headers))
eq_(content_length, len(contents))
factory.deferred.addCallback(check_result)
reactor.callLater(5, reactor.stop)
reactor.run()
def runtwisted(config=None):
"""
Run the Twisted server.
"""
globalLogBeginner.beginLoggingTo(
[FileLogObserver(sys.stdout, lambda _: formatEvent(_) + "\n")])
threadpool = ThreadPool(maxthreads=30)
app = api.makeapp(config=config)
wsgi_app = WSGIResource(reactor, threadpool, app)
class OptimaResource(Resource):
isLeaf = True
def __init__(self, wsgi):
self._wsgi = wsgi
def render(self, request):
request.prepath = []
request.postpath = ['api'] + request.postpath[:]
r = self._wsgi.render(request)
request.responseHeaders.setRawHeaders(
b'Cache-Control',
[b'no-cache', b'no-store', b'must-revalidate'])
request.responseHeaders.setRawHeaders(b'expires', [b'0'])
return r
# If we have a full path for the client directory, use that directory.
if os.path.isabs(config.CLIENT_DIR):
clientDirTarget = config.CLIENT_DIR
# Otherwise (we have a relative path), use it (correcting so it is with
# respect to the sciris repo directory).
else:
clientDirTarget = '%s%s%s' % (os.pardir, os.sep, config.CLIENT_DIR)
base_resource = File('%s%sdist%s' % (clientDirTarget, os.sep, os.sep))
base_resource.putChild(
'dev', File('%s%ssrc%s' % (clientDirTarget, os.sep, os.sep)))
base_resource.putChild('api', OptimaResource(wsgi_app))
site = Site(base_resource)
try:
port = str(sys.argv[1])
except IndexError:
port = "8091"
# Start the threadpool now, shut it down when we're closing
threadpool.start()
reactor.addSystemEventTrigger('before', 'shutdown', threadpool.stop)
endpoint = serverFromString(reactor, "tcp:port=" + port)
endpoint.listen(site)
reactor.run()
def StartStorageService(config, block_store):
try:
http_port = config['StorageService']['HttpPort']
http_host = config['StorageService']['Host']
worker_threads = config['StorageService'].get('WorkerThreads', 8)
reactor_threads = config['StorageService'].get('ReactorThreads', 8)
except KeyError as ke:
logger.error('missing configuration for %s', str(ke))
sys.exit(-1)
logger.info('service started on port %s', http_port)
thread_pool = ThreadPool(maxthreads=worker_threads)
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
block = Resource()
block.putChild(
b'get',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(GetBlockApp(block_store))))
block.putChild(
b'gets',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(GetBlocksApp(block_store))))
block.putChild(
b'store',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(StoreBlocksApp(block_store))))
block.putChild(
b'list',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(ListBlocksApp(block_store))))
block.putChild(
b'check',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(CheckBlocksApp(block_store))))
root = Resource()
root.putChild(
b'info',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InfoApp(block_store))))
root.putChild(b'block', block)
site = Site(root, timeout=60)
site.displayTracebacks = True
reactor.suggestThreadPoolSize(reactor_threads)
signal.signal(signal.SIGQUIT, __shutdown__)
signal.signal(signal.SIGTERM, __shutdown__)
endpoint = TCP4ServerEndpoint(reactor,
http_port,
backlog=32,
interface=http_host)
endpoint.listen(site)
def __init__(self):
self._clock = Clock()
self.callLater = self._clock.callLater
self._threadpool = ThreadPool()
self._threadpool.start()
self.getThreadPool = lambda: self._threadpool
self._threadCalls = Queue()
def __init__(self, *args, **kwargs):
self.tp = ThreadPool(maxthreads=20)
self.tp.start()
self.resource = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing)
self.nameSpace = TestNameSpace()
self.nameSpace.async_thing_complete = Queue()
return super(NoGoStatusCodes, self).__init__(*args, **kwargs)
def test_contemporaneous_requests(self):
'''
We're going to create two request-response cycles here:
Cycle 1 will begin.
Cycle 2 will begin.
Cycle 2 will return.
Cycle 1 will return.
This way, we can prove that the crosstown_traffic created
by cycle 1 is not resolved by the return of cycle 2.
'''
tp = ThreadPool(maxthreads=20)
tp.start()
self.addCleanup(tp.stop)
log.debug("\n\nStarting the two stream stuff.")
request1 = DummyRequest([b'r1'])
request1.isSecure = lambda: False
request1.content = "Nothing really here."
request1.requestHeaders.addRawHeader('llamas', 'dingo')
request1.client = IPv4Address("TCP", b"50.0.50.0", 5000)
nameSpace.test_case = self
hr = HendrixWSGIResource(reactor, tp, wsgi_application)
d1 = deferToThreadPool(reactor, tp, hr.render, request1)
request2 = DummyRequest([b'r2'])
request2.isSecure = lambda: False
request2.content = b"Nothing really here."
request2.requestHeaders.addRawHeader('llamas', 'dingo')
request2.client = IPv4Address("TCP", b"100.0.50.0", 5000)
d2 = deferToThreadPool(reactor, tp, hr.render, request2)
def woah_stop(failure):
nameSpace.async_task_was_done.put_nowait(False)
nameSpace.second_cycle_complete.put_nowait(False)
nameSpace.ready_to_proceed_with_second_cycle.put_nowait(False)
d1.addErrback(woah_stop)
d2.addErrback(woah_stop)
combo_deferred = gatherResults([d1, d2])
def wait_for_queue_resolution():
nameSpace.async_task_was_done.get(True, 3)
combo_deferred.addCallback(lambda _: deferToThreadPool(
reactor, tp, wait_for_queue_resolution))
combo_deferred.addCallback(
lambda _: self.assertTrue(nameSpace.async_task_was_run))
return combo_deferred
def _initThreadPool(self) -> None:
"""
Create the threadpool accessible with callFromThread.
"""
self.threadpool = ThreadPool(0, 10, "twisted.internet.reactor")
self._threadpoolStartupID = self.callWhenRunning(
self.threadpool.start)
self.threadpoolShutdownID = self.addSystemEventTrigger(
"during", "shutdown", self._stopThreadPool)
def start(self):
"""
Starts the logger pool and sets up the required shutdown event
trigger which will call :meth:`stop` on exit.
"""
reactor.addSystemEventTrigger("before", "shutdown", self.stop)
ThreadPool.start(self)
logger.debug(
"Started logger thread pool (min=%s, max=%s)", self.min, self.max)
class URLTranServer(object):
def __init__(self, app):
self.pool = ThreadPool()
self.pool.start()
self.resource = _WSGIResource(reactor, self.pool, app)
self.site = _Site(self.resource)
def register(self, port):
serve_register_tcp(self.site, port)
def __init__(self, app, port, host, debug):
super(TwistedApplicationServer,
self).__init__(app, port, host, debug)
self.reactor = reactor
self.thread_pool = ThreadPool(5, 40)
self.resource = WSGIResource(self.reactor, self.thread_pool,
self.app)
self.reactor.addSystemEventTrigger('after', 'shutdown',
self.thread_pool.stop)
def run(self, handler):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler))
reactor.listenTCP(self.port, factory, interface=self.host)
class VMMasterServer(object):
def __init__(self, reactor, port):
self.reactor = reactor
self.app = create_app()
self.thread_pool = ThreadPool(maxthreads=config.THREAD_POOL_MAX)
self.thread_pool.start()
wsgi_resource = WSGIResource(self.reactor, self.thread_pool, self.app)
root_resource = RootResource(wsgi_resource)
root_resource.putChild("proxy", ProxyResource(self.app))
site = Site(root_resource)
site.protocol = HTTPChannelWithClient
self.bind = self.reactor.listenTCP(port, site)
log.info('Server is listening on %s ...' % port)
def run(self):
self.reactor.addSystemEventTrigger('before', 'shutdown',
self.before_shutdown)
self.reactor.run()
del self
def __del__(self):
log.info("Shutting down server...")
d = self.bind.stopListening()
_block_on(d, 20)
self.app.cleanup()
self.thread_pool.stop()
log.info("Server gracefully shut down")
def wait_for_end_active_sessions(self):
active_sessions = self.app.sessions.active()
def wait_for():
while active_sessions:
log.info("Waiting for {} sessions to complete: {}".format(
len(active_sessions),
[(i.id, i.status) for i in active_sessions]))
for session in active_sessions:
if session.is_done:
log.debug("Session {} is done".format(session.id))
active_sessions.remove(session)
time.sleep(1)
log.info("Wait for end %s active session[s]:"
" %s" % (len(active_sessions), active_sessions))
return deferToThread(wait_for).addCallbacks(
callback=lambda _: log.info(
"All active sessions has been completed"),
errback=lambda failure: log.error(
"Error while waiting for active_sessions: {}".format(failure)))
@inlineCallbacks
def before_shutdown(self):
self.app.running = False
yield self.wait_for_end_active_sessions()
def _worker(self):
self.workerThread = threading.current_thread()
assert self.session is None
self.session = self.context.db._createSession()
try:
ThreadPool._worker(self)
finally:
assert self.session is not None
self.session.commit()
self.session.close()
def twisted(app, address, **options):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, app))
reactor.listenTCP(address[1], factory, interface=address[0])
reactor.run()
def twisted(app, address, **options):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, app))
reactor.listenTCP(address[1], factory, interface=address[0])
reactor.run()
def run(self, handler):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler))
reactor.listenTCP(self.port, factory, interface=self.host)
reactor.run()
def __init__(self, engine, verbose=False):
if engine.dialect.name == "sqlite":
pool_size = 1
ThreadPool.__init__(self, minthreads=1, maxthreads=1)
else:
ThreadPool.__init__(self)
self.engine = engine
reactor.callWhenRunning(self.start)
def test_threadLocality(self):
"""
An 'Record' repr()'d in two separate threads at the same time should
look the same (i.e. the repr state tracking for '...' should be
thread-local).
"""
pool = ThreadPool(2, 2)
pool.start()
self.addCleanup(pool.stop)
class StickyRepr(object):
"""
This has a __repr__ which will block until a separate thread
notifies it that it should return. We use this to create a race
condition.
"""
waited = False
def __init__(self):
self.set = threading.Event()
self.wait = threading.Event()
def __repr__(self):
if not self.waited:
self.set.set()
self.wait.wait()
return 'sticky'
r = StickyRepr()
mr = MyRecord(something=1, somethingElse=r)
d = deferToThreadPool(reactor, pool, repr, mr)
def otherRepr():
# First we wait for the first thread doing a repr() to enter its
# __repr__()...
r.set.wait()
# OK, now it's blocked. Let's make sure that subsequent calls to
# this repr() won't block.
r.waited = True
# Do it! This is a concurrent repr().
result = repr(mr)
# Now we're done, wake up the other repr and let it complete.
r.wait.set()
return result
d2 = deferToThreadPool(reactor, pool, otherRepr)
def done(xxx_todo_changeme):
(thread1repr, thread2repr) = xxx_todo_changeme
knownGood = 'MyRecord(something=1, somethingElse=sticky)'
# self.assertEquals(thread1repr, thread2repr)
self.assertEqual(thread1repr, knownGood)
self.assertEqual(thread2repr, knownGood)
return gatherResults([d, d2]).addCallback(done)
class NoGoStatusCodes(TestCase):
def __init__(self, *args, **kwargs):
self.tp = ThreadPool(maxthreads=20)
self.tp.start()
self.resource = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing)
self.nameSpace = TestNameSpace()
self.nameSpace.async_thing_complete = Queue()
return super(NoGoStatusCodes, self).__init__(*args, **kwargs)
def setUp(self, *args, **kwargs):
self.addCleanup(self.tp.stop)
super(NoGoStatusCodes, self).setUp(*args, **kwargs)
def wsgi_thing(self, environ, start_response):
start_response('404 NOT FOUND', [('Content-type', 'text/plain')])
@crosstown_traffic(no_go_status_codes=self.no_go_status_codes,
same_thread=True)
def long_thing_on_same_thread():
self.nameSpace.async_task_was_run = True
log.debug("No bad status codes; went ahead with async thing.")
return [b"Nothing."]
def test_bad_status_codes_cause_no_go_in_wsgi_response(self):
self.no_go_status_codes = [404, '6xx']
request = DummyRequest([b'r1'])
request.isSecure = lambda: False
request.content = "llamas"
request.client = IPv4Address("TCP", b"50.0.50.0", 5000)
finished = request.notifyFinish()
self.resource.render(request)
# This must wait until the WSGI response is closed.
finished.addCallback(
lambda _: self.assertFalse(self.nameSpace.async_task_was_run))
return finished
def test_bad_status_codes_cause_no_go_flag(self):
through_to_you = crosstown_traffic(no_go_status_codes=[418])
through_to_you.status_code = 418
through_to_you.check_status_code_against_no_go_list()
self.assertTrue(through_to_you.no_go)
def test_no_bad_status_codes_are_cool(self):
through_to_you = crosstown_traffic(no_go_status_codes=[418])
through_to_you.status_code = 404
through_to_you.check_status_code_against_no_go_list()
self.assertFalse(through_to_you.no_go)
class ThreadPoolService(Service):
def __init__(self):
self.threadpool = ThreadPool()
def startService(self):
self.threadpool.start()
def stopService(self):
self.threadpool.stop()
def __init__(self):
log.debug("[D] %s %s " % (__file__, __name__), "Starting db_threadpool")
self.db_threadpool = ThreadPool(0, config.advanced.db_thread_pool_size)
self.db_threadpool.start()
log.debug("[D] %s %s " % (__file__, __name__), "Starting scheduler_threadpool")
self.scheduler_threadpool = ThreadPool(0, config.advanced.scheduler_thread_pool_size)
self.scheduler_threadpool.start()
self.transactor = Transactor(self.db_threadpool)
self.transactor.retries = 0
def get_twisted_pool():
global pool
try:
return pool
except:
pool = ThreadPool(minthreads=min_t, maxthreads=max_t, name='core')
pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
return pool
class SameOrDifferentThread(TestCase):
def setUp(self, *args, **kwargs):
self.tp = ThreadPool()
self.tp.start()
self.addCleanup(self.tp.stop)
super(SameOrDifferentThread, self).setUp(*args, **kwargs)
def wsgi_thing(self, environ, start_response):
start_response('200 OK', [('Content-type', 'text/plain')])
nameSpace.this_thread = threading.current_thread()
@crosstown_traffic(
same_thread=self.use_same_thread
)
def long_thing_on_same_thread():
nameSpace.thread_that_is_supposed_to_be_the_same = threading.current_thread()
log.debug("Finished async thing on same thread.")
return [b"Nothing."]
def assert_that_threads_are_the_same(self):
self.assertEqual(
nameSpace.this_thread,
nameSpace.thread_that_is_supposed_to_be_the_same
)
def assert_that_threads_are_different(self):
self.assertNotEqual(nameSpace.this_thread,
nameSpace.thread_that_is_supposed_to_be_different)
def request_same_or_different_thread_thread(self):
hr = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing)
request1 = DummyRequest([b'r1'])
request1.isSecure = lambda: False
request1.content = b"llamas"
request1.client = IPv4Address("TCP", b"50.0.50.0", 5000)
d = deferToThreadPool(reactor, self.tp, hr.render, request1)
d.addCallback(lambda _: request1.notifyFinish())
return d
def test_that_threads_are_the_same(self):
self.use_same_thread = True
d = self.request_same_or_different_thread_thread()
d.addCallback(lambda _: self.assert_that_threads_are_the_same)
return pytest_twisted.blockon(d)
def test_that_threads_are_different(self):
self.use_same_thread = False
d = self.request_same_or_different_thread_thread()
d.addCallback(lambda _: self.assert_that_threads_are_different)
return pytest_twisted.blockon(d)
def twisted_adapter(host, port):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
ittyResource = wsgi.WSGIResource(reactor, thread_pool, handle_request)
site = server.Site(ittyResource)
reactor.listenTCP(port, site)
reactor.run()
def twisted_adapter(host, port, application):
from twisted.application import service, strports
from twisted.web import server, http, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
resource = wsgi.WSGIResource(reactor, thread_pool, application)
site = server.Site(resource)
reactor.listenTCP(port, site)
reactor.run()
def run(port):
# Create and start a thread pool,
wsgiThreadPool = ThreadPool()
wsgiThreadPool.start()
service_list = []
b = WebRoot()
global_cfg = getGlobalCFG()
global_cfg["local_web_port"] = port
def loadmodule(ob, packagename):
'''
载入扩展模块
'''
if hasattr(ob, "get_module_config"):
config_dict = ob.get_module_config()
web_define = config_dict.get("webmgt")
if web_define != None:
for (url_str,web_cls) in web_define.items():
print("loaded web: {0} ".format(url_str))
web_obj = web_cls()
web_obj.global_cfg = global_cfg
b.mount(url_str, web_obj)
service_define = config_dict.get("service")
if service_define != None:
for (key, service_cls) in service_define.items():
print("loaded service: {0} ".format(str(service_cls)))
service_obj = service_cls()
service_obj.global_cfg = global_cfg
service_list.append(service_obj)
''' 扫描模块存放目录， 加载扩展模块 '''
dir_list = os.listdir(MODULES_PATH)
from dlutil import util_classutil as clsutil
for directory in dir_list:
path_tmp = "{0}/{1}".format(MODULES_PATH, directory)
module_pack = MODULES_PATH.split('/')[-1] + "." + directory
if os.path.isdir(path_tmp) and path_tmp[0:1] != ".":
pack_name = clsutil.find_class_from_path(path_tmp,
loadmodule,
module_pack)
wsgiAppAsResource = WSGIResource(reactor, wsgiThreadPool, b)
site = server.Site(wsgiAppAsResource)
reactor.listenTCP(port, site)
reactor.run()
def __init__(self, pool=None, minthreads=1, maxthreads=4, **kwargs):
"""Creates a twisted aware Session
Notes
~~~~~
* If you provide both `pool` and `max_workers`, the latter is
ignored and provided threadpool is used as is.
"""
requestsSession.__init__(self, **kwargs)
if pool is None:
pool = ThreadPool(minthreads=minthreads, maxthreads=maxthreads)
self.pool = pool
pool.start()
def test_threadLocality(self):
"""
An 'Record' repr()'d in two separate threads at the same time should
look the same (i.e. the repr state tracking for '...' should be
thread-local).
"""
pool = ThreadPool(2, 2)
pool.start()
self.addCleanup(pool.stop)
class StickyRepr(object):
"""
This has a __repr__ which will block until a separate thread
notifies it that it should return. We use this to create a race
condition.
"""
waited = False
def __init__(self):
self.set = threading.Event()
self.wait = threading.Event()
def __repr__(self):
if not self.waited:
self.set.set()
self.wait.wait()
return 'sticky'
r = StickyRepr()
mr = MyRecord(something=1, somethingElse=r)
d = deferToThreadPool(reactor, pool, repr, mr)
def otherRepr():
# First we wait for the first thread doing a repr() to enter its
# __repr__()...
r.set.wait()
# OK, now it's blocked. Let's make sure that subsequent calls to
# this repr() won't block.
r.waited = True
# Do it! This is a concurrent repr().
result = repr(mr)
# Now we're done, wake up the other repr and let it complete.
r.wait.set()
return result
d2 = deferToThreadPool(reactor, pool, otherRepr)
def done(xxx_todo_changeme):
(thread1repr, thread2repr) = xxx_todo_changeme
knownGood = 'MyRecord(something=1, somethingElse=sticky)'
# self.assertEquals(thread1repr, thread2repr)
self.assertEqual(thread1repr, knownGood)
self.assertEqual(thread2repr, knownGood)
return gatherResults([d, d2]).addCallback(done)
class SameOrDifferentThread(TestCase):
def setUp(self, *args, **kwargs):
self.tp = ThreadPool(maxthreads=20)
self.tp.start()
self.addCleanup(self.tp.stop)
super(SameOrDifferentThread, self).setUp(*args, **kwargs)
def wsgi_thing(self, environ, start_response):
start_response('200 OK', [('Content-type','text/plain')])
nameSpace.this_thread = threading.current_thread()
@crosstown_traffic.follow_response(same_thread=self.use_same_thread)
def long_thing_on_same_thread():
nameSpace.thread_that_is_supposed_to_be_the_same = threading.current_thread()
logger.info("Finished async thing on same thread.")
return "Nothing."
def assert_that_threads_are_the_same(self):
self.assertEqual(nameSpace.this_thread,
nameSpace.thread_that_is_supposed_to_be_the_same
)
def assert_that_threads_are_different(self):
self.assertNotEqual(nameSpace.this_thread,
nameSpace.thread_that_is_supposed_to_be_different)
def request_same_or_different_thread_thread(self):
hr = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing)
request1 = DummyRequest('r1')
request1.isSecure = lambda: False
request1.content = "llamas"
d = deferToThreadPool(reactor, self.tp, hr.render, request1)
return d
def test_that_threads_are_the_same(self):
self.use_same_thread = True
d = self.request_same_or_different_thread_thread()
d.addCallback(lambda _: self.assert_that_threads_are_the_same)
return d
def test_that_threads_are_different(self):
self.use_same_thread = False
d = self.request_same_or_different_thread_thread()
d.addCallback(lambda _: self.assert_that_threads_are_different)
return d
def _configure(self):
"""
Configures the server
:since: v1.0.0
"""
listener_host = Settings.get("pas_http_twisted_server_host", self.socket_hostname)
self.port = int(Settings.get("pas_http_twisted_server_port", 8080))
self.reactor = reactor
self.reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
server_description = "tcp:{0:d}".format(self.port)
if (listener_host == ""): self.host = Settings.get("pas_http_server_preferred_hostname", self.socket_hostname)
else:
self.host = listener_host
server_description += ":interface={0}".format(self.host)
#
self.thread_pool = ThreadPool()
self.thread_pool.start()
if (self._log_handler is not None): self._log_handler.info("pas.http.core Twisted server starts at '{0}:{1:d}'", listener_host, self.port, context = "pas_http_core")
server = serverFromString(self.reactor, server_description)
server.listen(Site(WSGIResource(reactor, self.thread_pool, HttpWsgi1Request)))
"""
Configure common paths and settings
"""
AbstractServer._configure(self)
def add_site(config_func, ini_path="app.ini", relative_to="."):
# Create and start a thread pool,
pool = ThreadPool()
pool.start()
# ensuring that it will be stopped when the reactor shuts down
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
# init site
app = loadapp("config:" + ini_path, relative_to=relative_to)
resource = WSGIResource(reactor, pool, app)
site = Site(resource)
port = config_func("port", default=8080, _type=int)
# just listen
try:
reactor.listenTCP(port, site)
except Exception, e:
reactor.stop()
def __init__(self, result_class):
from twisted.python.threadpool import ThreadPool
SimpleRunner.__init__(self, _threadclass(result_class))
self._pool = ThreadPool()
self._pool.start()
class TwistedThreadScheduler(BaseScheduler):
"""Run jobs in threads, chaperoned by Twisted."""
def __init__(self, num_threads, install_signal_handlers=True):
"""Create a new `TwistedThreadScheduler`.
:param num_threads: The number of threads to allocate to the
thread pool.
:type num_threads: int
:param install_signal_handlers: Whether the Twisted reactor
should install signal handlers or not. This is intented for
testing - set to False to avoid layer violations - but may
be useful in other situations.
:type install_signal_handlers: bool
"""
self._thread_pool = ThreadPool(0, num_threads)
self._install_signal_handlers = install_signal_handlers
self._jobs = []
def schedule(self, func, *args, **kwargs):
self._jobs.append(
deferToThreadPool(
reactor, self._thread_pool, func, *args, **kwargs))
def run(self):
jobs, self._jobs = self._jobs[:], []
jobs_done = DeferredList(jobs)
jobs_done.addBoth(lambda ignore: self._thread_pool.stop())
jobs_done.addBoth(lambda ignore: reactor.stop())
reactor.callWhenRunning(self._thread_pool.start)
reactor.run(self._install_signal_handlers)
def setUp(self):
self.db = MockStore()
GSM.registerUtility(MockZStorm(self.db))
self.tp = ThreadPool(0, 2)
self.sm = MockServerManager(reactor, SERVERS)
self.updater = Updater(Transactor(self.tp), self.sm)
self.tp.start()
