def __init__(self,
gossip_obj,
ledger_obj,
config,
windows_service=False,
http_port=None,
):
'''
Creates a validator. As a current side-effect, does some
initialization on it's ledger_obj argumenet
Args:
node_obj: (gossip.Node)
ledger_obj: (journal.Journal)
config: (dict)
windows_service: (bool)
http_port: (int)
'''
self.status = 'stopped'
self.Config = config
self.gossip = gossip_obj
node_obj = gossip_obj.LocalNode
self._gossip_host = node_obj.NetHost
self._gossip_port = node_obj.NetAddress
self._endpoint_host = node_obj.endpoint_host
self._endpoint_port = node_obj.endpoint_port
self._endpoint_http_port = http_port
self.Ledger = ledger_obj
self.profile = self.Config.get('Profile', False)
if self.profile:
self.pr = cProfile.Profile()
self.pr.enable()
self.windows_service = windows_service
# flag to indicate that a topology update is in progress
self._topology_update_in_progress = False
self.delaystart = self.Config['DelayStart']
# set up signal handlers for shutdown
if not windows_service:
signal.signal(signal.SIGTERM, self.handle_shutdown_signal)
signal.signal(signal.SIGINT, self.handle_shutdown_signal)
# ---------- Initialize the configuration ----------
self.initialize_common_configuration()
# ---------- Initialize the NodeMap ----------
self.initialize_node_map()
# ---------- Initialize the Ledger ----------
self.initialize_ledger_object()
maxsize = self.Config.get("WebPoolSize", 8)
self.web_thread_pool = ThreadPool(0, maxsize, "WebThreadPool")
def setUp(self):
"""
Set up the TestCase by calling the homeserver constructor, optionally
hijacking the authentication system to return a fixed user, and then
calling the prepare function.
"""
self.reactor, self.clock = get_clock()
self._hs_args = {"clock": self.clock, "reactor": self.reactor}
self.hs = self.make_homeserver(self.reactor, self.clock)
if self.hs is None:
raise Exception("No homeserver returned from make_homeserver.")
if not isinstance(self.hs, HomeServer):
raise Exception("A homeserver wasn't returned, but %r" %
(self.hs, ))
# Register the resources
self.resource = self.create_test_json_resource()
from tests.rest.client.v1.utils import RestHelper
self.helper = RestHelper(self.hs, self.resource,
getattr(self, "user_id", None))
if hasattr(self, "user_id"):
if self.hijack_auth:
def get_user_by_access_token(token=None, allow_guest=False):
return succeed({
"user":
UserID.from_string(self.helper.auth_user_id),
"token_id":
1,
"is_guest":
False,
})
def get_user_by_req(request,
allow_guest=False,
rights="access"):
return succeed(
create_requester(
UserID.from_string(self.helper.auth_user_id), 1,
False, None))
self.hs.get_auth().get_user_by_req = get_user_by_req
self.hs.get_auth(
).get_user_by_access_token = get_user_by_access_token
self.hs.get_auth().get_access_token_from_request = Mock(
return_value="1234")
if self.needs_threadpool:
self.reactor.threadpool = ThreadPool()
self.addCleanup(self.reactor.threadpool.stop)
self.reactor.threadpool.start()
if hasattr(self, "prepare"):
self.prepare(self.reactor, self.clock, self.hs)
def setUp(self):
self.db = MockStore()
GSM.registerUtility(MockZStorm(self.db))
self.tp = ThreadPool(0, 2)
self.sm = MockServerManager(reactor, SERVERS)
self.updater = Updater(Transactor(self.tp), self.sm)
self.tp.start()
def StartStorageService(config, block_store):
try:
http_port = config['StorageService']['HttpPort']
http_host = config['StorageService']['Host']
worker_threads = config['StorageService'].get('WorkerThreads', 8)
reactor_threads = config['StorageService'].get('ReactorThreads', 8)
except KeyError as ke:
logger.error('missing configuration for %s', str(ke))
sys.exit(-1)
logger.info('service started on port %s', http_port)
thread_pool = ThreadPool(maxthreads=worker_threads)
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
block = Resource()
block.putChild(
b'get',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(GetBlockApp(block_store))))
block.putChild(
b'gets',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(GetBlocksApp(block_store))))
block.putChild(
b'store',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(StoreBlocksApp(block_store))))
block.putChild(
b'list',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(ListBlocksApp(block_store))))
block.putChild(
b'check',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(CheckBlocksApp(block_store))))
root = Resource()
root.putChild(
b'info',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InfoApp(block_store))))
root.putChild(b'block', block)
site = Site(root, timeout=60)
site.displayTracebacks = True
reactor.suggestThreadPoolSize(reactor_threads)
signal.signal(signal.SIGQUIT, __shutdown__)
signal.signal(signal.SIGTERM, __shutdown__)
endpoint = TCP4ServerEndpoint(reactor,
http_port,
backlog=32,
interface=http_host)
endpoint.listen(site)
def runtwisted(config=None):
"""
Run the Twisted server.
"""
globalLogBeginner.beginLoggingTo(
[FileLogObserver(sys.stdout, lambda _: formatEvent(_) + "\n")])
threadpool = ThreadPool(maxthreads=30)
app = api.makeapp(config=config)
wsgi_app = WSGIResource(reactor, threadpool, app)
class OptimaResource(Resource):
isLeaf = True
def __init__(self, wsgi):
self._wsgi = wsgi
def render(self, request):
request.prepath = []
request.postpath = ['api'] + request.postpath[:]
r = self._wsgi.render(request)
request.responseHeaders.setRawHeaders(
b'Cache-Control',
[b'no-cache', b'no-store', b'must-revalidate'])
request.responseHeaders.setRawHeaders(b'expires', [b'0'])
return r
# If we have a full path for the client directory, use that directory.
if os.path.isabs(config.CLIENT_DIR):
clientDirTarget = config.CLIENT_DIR
# Otherwise (we have a relative path), use it (correcting so it is with
# respect to the sciris repo directory).
else:
clientDirTarget = '%s%s%s' % (os.pardir, os.sep, config.CLIENT_DIR)
base_resource = File('%s%sdist%s' % (clientDirTarget, os.sep, os.sep))
base_resource.putChild(
'dev', File('%s%ssrc%s' % (clientDirTarget, os.sep, os.sep)))
base_resource.putChild('api', OptimaResource(wsgi_app))
site = Site(base_resource)
try:
port = str(sys.argv[1])
except IndexError:
port = "8091"
# Start the threadpool now, shut it down when we're closing
threadpool.start()
reactor.addSystemEventTrigger('before', 'shutdown', threadpool.stop)
endpoint = serverFromString(reactor, "tcp:port=" + port)
endpoint.listen(site)
reactor.run()
def mkTreq(self):
"""
Construct a :class:`treq.testing.StubTreq` which wraps
a :class:`yarrharr.application.Root` resource.
"""
reactor = MemoryReactorClock() # unused
threadpool = ThreadPool(minthreads=0) # unused
self.addCleanup(threadpool.stop)
return StubTreq(Root(reactor, threadpool))
def __init__(self, factory):
self.fact = factory
self.threadpool = ThreadPool(1,25)
reactor.callFromThread(self.threadpool.start)
# Allow Ctrl-C to get you out cleanly:
reactor.addSystemEventTrigger('after', 'shutdown', self.threadpool.stop)
self.pile = []
self.depiler = None
self.depiler_running = False
def __init__(self, app, port, host, debug):
super(TwistedApplicationServer,
self).__init__(app, port, host, debug)
self.reactor = reactor
self.thread_pool = ThreadPool(5, 40)
self.resource = WSGIResource(self.reactor, self.thread_pool,
self.app)
self.reactor.addSystemEventTrigger('after', 'shutdown',
self.thread_pool.stop)
def __init__(self):
self._clock = Clock()
self.callLater = self._clock.callLater
self._threadpool = ThreadPool()
self._threadpool.start()
self.getThreadPool = lambda: self._threadpool
self._threadCalls = Queue()
def __init__(self, *args, **kwargs):
self.tp = ThreadPool(maxthreads=20)
self.tp.start()
self.resource = HendrixWSGIResource(reactor, self.tp, self.wsgi_thing)
self.nameSpace = TestNameSpace()
self.nameSpace.async_thing_complete = Queue()
return super(NoGoStatusCodes, self).__init__(*args, **kwargs)
def _initThreadPool(self) -> None:
"""
Create the threadpool accessible with callFromThread.
"""
self.threadpool = ThreadPool(0, 10, "twisted.internet.reactor")
self._threadpoolStartupID = self.callWhenRunning(
self.threadpool.start)
self.threadpoolShutdownID = self.addSystemEventTrigger(
"during", "shutdown", self._stopThreadPool)
def test_contemporaneous_requests(self):
'''
We're going to create two request-response cycles here:
Cycle 1 will begin.
Cycle 2 will begin.
Cycle 2 will return.
Cycle 1 will return.
This way, we can prove that the crosstown_traffic created
by cycle 1 is not resolved by the return of cycle 2.
'''
tp = ThreadPool(maxthreads=20)
tp.start()
self.addCleanup(tp.stop)
log.debug("\n\nStarting the two stream stuff.")
request1 = DummyRequest([b'r1'])
request1.isSecure = lambda: False
request1.content = "Nothing really here."
request1.requestHeaders.addRawHeader('llamas', 'dingo')
request1.client = IPv4Address("TCP", b"50.0.50.0", 5000)
nameSpace.test_case = self
hr = HendrixWSGIResource(reactor, tp, wsgi_application)
d1 = deferToThreadPool(reactor, tp, hr.render, request1)
request2 = DummyRequest([b'r2'])
request2.isSecure = lambda: False
request2.content = b"Nothing really here."
request2.requestHeaders.addRawHeader('llamas', 'dingo')
request2.client = IPv4Address("TCP", b"100.0.50.0", 5000)
d2 = deferToThreadPool(reactor, tp, hr.render, request2)
def woah_stop(failure):
nameSpace.async_task_was_done.put_nowait(False)
nameSpace.second_cycle_complete.put_nowait(False)
nameSpace.ready_to_proceed_with_second_cycle.put_nowait(False)
d1.addErrback(woah_stop)
d2.addErrback(woah_stop)
combo_deferred = gatherResults([d1, d2])
def wait_for_queue_resolution():
nameSpace.async_task_was_done.get(True, 3)
combo_deferred.addCallback(lambda _: deferToThreadPool(
reactor, tp, wait_for_queue_resolution))
combo_deferred.addCallback(
lambda _: self.assertTrue(nameSpace.async_task_was_run))
return combo_deferred
def run(self, handler):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, handler))
reactor.listenTCP(self.port, factory, interface=self.host)
reactor.run()
def twisted(app, address, **options):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
factory = server.Site(wsgi.WSGIResource(reactor, thread_pool, app))
reactor.listenTCP(address[1], factory, interface=address[0])
reactor.run()
def test_threadLocality(self):
"""
An 'Record' repr()'d in two separate threads at the same time should
look the same (i.e. the repr state tracking for '...' should be
thread-local).
"""
pool = ThreadPool(2, 2)
pool.start()
self.addCleanup(pool.stop)
class StickyRepr(object):
"""
This has a __repr__ which will block until a separate thread
notifies it that it should return. We use this to create a race
condition.
"""
waited = False
def __init__(self):
self.set = threading.Event()
self.wait = threading.Event()
def __repr__(self):
if not self.waited:
self.set.set()
self.wait.wait()
return 'sticky'
r = StickyRepr()
mr = MyRecord(something=1, somethingElse=r)
d = deferToThreadPool(reactor, pool, repr, mr)
def otherRepr():
# First we wait for the first thread doing a repr() to enter its
# __repr__()...
r.set.wait()
# OK, now it's blocked. Let's make sure that subsequent calls to
# this repr() won't block.
r.waited = True
# Do it! This is a concurrent repr().
result = repr(mr)
# Now we're done, wake up the other repr and let it complete.
r.wait.set()
return result
d2 = deferToThreadPool(reactor, pool, otherRepr)
def done(xxx_todo_changeme):
(thread1repr, thread2repr) = xxx_todo_changeme
knownGood = 'MyRecord(something=1, somethingElse=sticky)'
# self.assertEquals(thread1repr, thread2repr)
self.assertEqual(thread1repr, knownGood)
self.assertEqual(thread2repr, knownGood)
return gatherResults([d, d2]).addCallback(done)
def get_twisted_pool():
global pool
try:
return pool
except:
pool = ThreadPool(minthreads=min_t, maxthreads=max_t, name='core')
pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
return pool
def make(self, dependency_resources):
"""Create and start a new thread pool."""
from twisted.internet import reactor
global _threadPool
if _threadPool is None:
_threadPool = ThreadPool(minthreads=1, maxthreads=1)
reactor.callWhenRunning(_threadPool.start)
reactor.addSystemEventTrigger('during', 'shutdown',
_threadPool.stop)
return _threadPool
def run(self, threadpool=None):
if self.no_go:
return
if not threadpool:
threadpool = reactor.threadpool or ThreadPool()
if self.same_thread:
self.crosstown_task()
else:
deferToThreadPool(reactor, threadpool, self.crosstown_task)
class AsyncExecutor(object):
""" defaults to minthreads=5, maxthreads=20 """
pool = ThreadPool( name = 'AsyncExecutorPool')
def _execute(self, func, *args, **kwargs):
if not self.pool.started:
self.pool.start()
self.pool.dispatch(None, func, *args, **kwargs)
execute = classmethod(_execute)
stop = pool.stop
def __init__(self, decorator=None):
from hendrix.mechanics.concurrency.decorators import _ThroughToYou
self.decorator = decorator or self.decorator or _ThroughToYou
self.threadpool = ThreadPool(name="Crosstown Traffic")
# The threadpool needs to start when the reactors starts...
reactor.callWhenRunning(self.threadpool.start)
# ...and stop when the reactor stops.
reactor.addSystemEventTrigger('before', 'shutdown', self.threadpool.stop)
super(ModuleCaller, self).__init__()
def __init__(self, settings):
super(SeleniumDownloadHandler, self).__init__(settings)
self._enable_driver = settings.getbool('WEB_DRIVER_ENABLED')
self._driver_name = settings.get('WEB_DRIVER_NAME')
self._driver_path = settings.get('WEB_DRIVER_PATH')
selenium_concurrent_request = settings.get(
'WEB_DRIVER_CONCURRENT_REQUESTS', 16)
self._thread_pool = ThreadPool(minthreads=selenium_concurrent_request,
maxthreads=selenium_concurrent_request)
self._thread_pool.start()
self._driver_timeout = settings.get('WEB_DRIVER_TIMEOUT', 300)
def setupFacade(config):
"""Get the L{Facade} instance to use in the API service."""
from fluiddb.api.facade import Facade
from fluiddb.util.transact import Transact
maxThreads = int(config.get('service', 'max-threads'))
threadpool = ThreadPool(minthreads=0, maxthreads=maxThreads)
reactor.callWhenRunning(threadpool.start)
reactor.addSystemEventTrigger('during', 'shutdown', threadpool.stop)
transact = Transact(threadpool)
factory = FluidinfoSessionFactory('API-%s' % config.get('service', 'port'))
return Facade(transact, factory)
def StartService(config):
# load the eservice database
try:
dbfile_name = config['Service']['EnclaveServiceDatabaseFile']
except KeyError:
logger.error(
'missing required configuration for enclave service database')
sys.exit(-1)
try:
eservice_db.load_database(dbfile_name)
except Exception as e:
logger.error('error loading eservice database from file %s', str(e))
sys.exit(-1)
try:
http_port = config['Service']['HttpPort']
http_host = config['Service']['Host']
worker_threads = config['Service'].get('WorkerThreads', 8)
reactor_threads = config['Service'].get('ReactorThreads', 8)
except KeyError as ke:
logger.error('missing configuration for %s', str(ke))
sys.exit(-1)
logger.info('service started on host %s, port %s', http_host, http_port)
thread_pool = ThreadPool(maxthreads=worker_threads)
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
flask_app = toxaway.views.register(config)
flask_site = WSGIResource(reactor, reactor.getThreadPool(), flask_app)
root = Resource()
root.putChild(b'shutdown', ShutdownResource())
root.putChild(b'flask', flask_site)
files = config.get('StaticContent', {})
logger.info('files=%s', files)
for key, val in files.items():
logger.info('map <%s> to <%s>', key, val)
root.putChild(key.encode(), File(val.encode()))
site = Site(root, timeout=60)
site.displayTracebacks = True
reactor.suggestThreadPoolSize(reactor_threads)
endpoint = TCP4ServerEndpoint(reactor,
http_port,
backlog=32,
interface=http_host)
endpoint.listen(site)
def StartEnclaveService(config, enclave):
try:
http_port = config['EnclaveService']['HttpPort']
http_host = config['EnclaveService']['Host']
storage_url = config['StorageService']['URL']
worker_threads = config['EnclaveService'].get('WorkerThreads', 8)
reactor_threads = config['EnclaveService'].get('ReactorThreads', 8)
except KeyError as ke:
logger.error('missing configuration for %s', str(ke))
sys.exit(-1)
logger.info('enclave service started on %s:%s', http_host, http_port)
logger.info('verifying_key: %s', enclave.verifying_key)
logger.info('encryption_key: %s', enclave.encryption_key)
logger.info('enclave_id: %s', enclave.enclave_id)
logger.info('storage service: %s', storage_url)
thread_pool = ThreadPool(minthreads=1, maxthreads=worker_threads)
thread_pool.start()
reactor.addSystemEventTrigger('before', 'shutdown', thread_pool.stop)
root = Resource()
root.putChild(
b'info',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InfoApp(enclave, storage_url))))
root.putChild(
b'initialize',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InitializeApp(enclave))))
root.putChild(
b'invoke',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(InvokeApp(enclave))))
root.putChild(
b'verify',
WSGIResource(reactor, thread_pool,
AppWrapperMiddleware(VerifyApp(enclave))))
site = Site(root, timeout=60)
site.displayTracebacks = True
reactor.suggestThreadPoolSize(reactor_threads)
signal.signal(signal.SIGQUIT, __shutdown__)
signal.signal(signal.SIGTERM, __shutdown__)
endpoint = TCP4ServerEndpoint(reactor,
http_port,
backlog=32,
interface=http_host)
endpoint.listen(site)
def __init__(self, hs, config):
self.cache_directory = hs.config.media_store_path
self.container = config["container"]
self.cloud = config["cloud"]
self.api_kwargs = {}
if "region_name" in config:
self.api_kwargs["region_name"] = config["region_name"]
threadpool_size = config.get("threadpool_size", 40)
self._download_pool = ThreadPool(name="swift-download-pool",
maxthreads=threadpool_size)
self._download_pool.start()
def _initialize_sync_threadpool(self):
"""
Initialize a ThreadPool with exactly one thread, that will be used to
run all the network blocking calls for syncing on a separate thread.
TODO this needs to be ported away from urllib and into twisted async
calls, and then we can ditch this syncing thread and reintegrate into
the main reactor.
"""
# XXX if the number of threads in this thread pool is ever changed, we
# should make sure that no operations on the database shuold occur
# before the database has been initialized.
self._sync_threadpool = ThreadPool(0, 1)
def __init__(self, reactor, port):
self.reactor = reactor
self.app = create_app()
self.thread_pool = ThreadPool(maxthreads=config.THREAD_POOL_MAX)
self.thread_pool.start()
wsgi_resource = WSGIResource(self.reactor, self.thread_pool, self.app)
root_resource = RootResource(wsgi_resource)
root_resource.putChild("proxy", ProxyResource(self.app))
site = Site(root_resource)
site.protocol = HTTPChannelWithClient
self.bind = self.reactor.listenTCP(port, site)
log.info('Server is listening on %s ...' % port)
def start(self, threadpoolsize):
self._threadpool = ThreadPool(minthreads=threadpoolsize,
maxthreads=threadpoolsize + 1)
# Set up threadpool
q.logger.log('[PMTASKLETS] Constructing taskletserver threadpool', 6)
self._threadpool.start()
for i in xrange(threadpoolsize):
runner = TaskletRunnerThread(self._queue)
self._runners.append(runner)
self._threadpool.callInThread(runner.run)
self._running = True
def twisted_adapter(host, port):
from twisted.web import server, wsgi
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
ittyResource = wsgi.WSGIResource(reactor, thread_pool, handle_request)
site = server.Site(ittyResource)
reactor.listenTCP(port, site)
reactor.run()
def __init__(self,
action='start',
options={},
reactor=reactor,
threadpool=None):
self.action = action
self.options = hx_options()
self.options.update(options)
self.services = []
self.resources = []
self.reactor = reactor
self.threadpool = threadpool or ThreadPool(name="Hendrix Web Service")
self.use_settings = True
# because running the management command overrides self.options['wsgi']
if self.options['wsgi']:
if hasattr(self.options['wsgi'], '__call__'):
# If it has a __call__, we assume that it is the application
# object itself.
self.application = self.options['wsgi']
try:
self.options['wsgi'] = "%s.%s" % (
self.application.__module__, self.application.__name__)
except AttributeError:
self.options['wsgi'] = self.application.__class__.__name__
else:
# Otherwise, we'll try to discern an application in the belief
# that this is a dot path.
wsgi_dot_path = self.options['wsgi']
# will raise AttributeError if we can't import it.
self.application = HendrixDeploy.importWSGI(wsgi_dot_path)
self.use_settings = False
else:
os.environ['DJANGO_SETTINGS_MODULE'] = self.options['settings']
settings = import_string('django.conf.settings')
self.services = get_additional_services(settings)
self.resources = get_additional_resources(settings)
self.options = HendrixDeploy.getConf(settings, self.options)
if self.use_settings:
django = importlib.import_module('django')
if django.VERSION[:2] >= (1, 7):
django.setup()
wsgi_dot_path = getattr(settings, 'WSGI_APPLICATION', None)
self.application = HendrixDeploy.importWSGI(wsgi_dot_path)
self.is_secure = self.options['key'] and self.options['cert']
self.servers = []
self._lock = DeferredLock()
