def test_cortex_xact_deadlock(self):
N = 100
prop = 'testform'
fd = tempfile.NamedTemporaryFile()
dmon = s_daemon.Daemon()
pool = s_threads.Pool(size=4, maxsize=8)
wait = s_eventbus.Waiter(pool, 1, 'pool:work:fini')
with s_cortex.openurl('sqlite:///%s' % fd.name) as core:
def populate():
for i in range(N):
#print('wrote %d tufos' % i)
core.formTufoByProp(prop, str(i), **{})
dmon.share('core', core)
link = dmon.listen('tcp://127.0.0.1:0/core')
prox = s_telepath.openurl('tcp://127.0.0.1:%d/core' % link[1]['port'])
pool.wrap(populate)()
for i in range(N):
tufos = prox.getTufosByProp(prop)
#print('got %d tufos' % len(tufos))
wait.wait()
pool.fini()
def test_threads_pool(self):
func = s_t_common.CallBack()
with s_threads.Pool() as pool:
pool.call(func, 20, 30)
self.true(func.wait(timeout=1))
func.args = (20, 30)
def test_threads_pool(self):
def woot(x, y):
return x + y
with s_threads.Pool() as pool:
with pool.task(woot, 20, 30) as task:
pass
self.true(task.waitfini(timeout=1))
def test_threads_pool_wrap(self):
evnt = threading.Event()
def woot(x, y):
evnt.set()
return x + y
with s_threads.Pool() as pool:
pool.wrap(woot)(20, 30)
self.true(evnt.wait(timeout=1))
def runBossPool(self, size, maxsize=None):
'''
Create and run a thread pool for this Boss()
Example:
boss.runBossPool(3)
'''
pool = s_threads.Pool(size=size, maxsize=maxsize)
self.onfini(pool.fini)
self.setBossPool(pool)
def test_threads_pool(self):
pool = s_threads.Pool()
wait = s_eventbus.Waiter(pool, 1, 'pool:work:fini')
def woot(x, y):
return x + y
pool.task(newtask(woot, 20, 30))
wait.wait()
pool.fini()
def test_threads_pool_wrap(self):
pool = s_threads.Pool()
wait = s_eventbus.Waiter(pool, 1, 'pool:work:fini')
def woot(x, y):
return x + y
pool.wrap(woot)(20, 30)
wait.wait()
pool.fini()
def test_async_custom_pool_basics(self):
# Demonstrate Boss use with a custom thread pool.
boss = s_async.Boss()
my_pool = s_threads.Pool(3, maxsize=8)
data = {}
def jobmeth(x, y=20):
return x + y
def jobdork(x, y=20):
raise Exception('hi')
def jobdone(job):
name = job[1].get('name')
data[name] = job
jid1 = s_async.jobid()
jid2 = s_async.jobid()
task1 = (jobmeth, (3, ), {})
task2 = (jobdork, (3, ), {})
job1 = boss.initJob(jid1, task=task1, name='job1', ondone=jobdone)
job2 = boss.initJob(jid2, task=task2, name='job2', ondone=jobdone)
self.eq(job1[0], jid1)
self.eq(job2[0], jid2)
# Test __iter__ since we've got jobs in the boss that haven't been run.
jobs = [job for job in boss]
self.eq(len(jobs), 2)
my_pool.call(boss._runJob, job1)
my_pool.call(boss._runJob, job2)
boss.wait(jid1, timeout=1)
boss.wait(jid2, timeout=1)
ret1 = data.get('job1')
self.nn(ret1)
self.eq(ret1[1]['ret'], 23)
ret2 = data.get('job2')
self.nn(ret2)
self.eq(ret2[1]['err'], 'Exception')
self.eq(ret2[1]['errmsg'], 'hi')
boss.fini()
def __init__(self, relay, plex=None, sock=None):
s_eventbus.EventBus.__init__(self)
self.onfini(self._onProxyFini)
# NOTE: the _tele_ prefixes are designed to prevent accidental
# derefs with overlapping names from working correctly
self._tele_sid = None
self._tele_q = s_queue.Queue()
self._tele_pushed = {}
if plex is None:
plex = s_socket.Plex()
self._tele_plex = plex
self._tele_boss = s_async.Boss()
self._tele_plex.on('link:sock:mesg', self._onLinkSockMesg)
self._raw_on('tele:yield:init', self._onTeleYieldInit)
self._raw_on('tele:yield:item', self._onTeleYieldItem)
self._raw_on('tele:yield:fini', self._onTeleYieldFini)
self._raw_on('job:done', self._tele_boss.dist)
self._raw_on('sock:gzip', self._onSockGzip)
self._raw_on('tele:call', self._onTeleCall)
poolmax = relay.getLinkProp('poolmax', -1)
poolsize = relay.getLinkProp('poolsize', 0)
self._tele_cthr = self.consume(self._tele_q)
self._tele_pool = s_threads.Pool(size=poolsize, maxsize=poolmax)
self._tele_ons = {}
self._tele_sock = None
self._tele_relay = relay # LinkRelay()
self._tele_link = relay.link
self._tele_yields = {}
self._tele_csides = {}
self._tele_reflect = None
# obj name is path minus leading "/"
self._tele_name = relay.link[1].get('path')[1:]
if sock is None:
sock = self._tele_relay.connect()
self._initTeleSock(sock=sock)
def __init__(self, relay, plex=None):
s_eventbus.EventBus.__init__(self)
self.onfini(self._onProxyFini)
# NOTE: the _tele_ prefixes are designed to prevent accidental
# derefs with overlapping names from working correctly
self._tele_sid = None
self._tele_pki = None
self._tele_q = s_queue.Queue()
self._tele_pushed = {}
if plex == None:
plex = s_socket.Plex()
self._tele_plex = plex
self._tele_boss = s_async.Boss()
self._raw_on('tele:yield:init', self._onTeleYieldInit)
self._raw_on('tele:yield:item', self._onTeleYieldItem)
self._raw_on('tele:yield:fini', self._onTeleYieldFini)
self._raw_on('job:done', self._tele_boss.dist)
self._raw_on('tele:call', self._onTeleCall)
poolmax = relay.getLinkProp('poolmax', -1)
poolsize = relay.getLinkProp('poolsize', 0)
self._tele_cthr = self.consume(self._tele_q)
self._tele_pool = s_threads.Pool(size=poolsize, maxsize=poolmax)
self._tele_ons = set()
self._tele_sock = None
self._tele_relay = relay # LinkRelay()
self._tele_yields = {}
# obj name is path minus leading "/"
self._tele_name = relay.link[1].get('path')[1:]
if relay.getLinkProp('pki'):
#TODO pkiurl
self._tele_pki = relay.getLinkProp('pkistor')
if self._tele_pki == None:
self._tele_pki = s_pki.getUserPki()
self._initTeleSock()
def test_threads_exception(self):
data = {}
def breakstuff():
data['key'] = True
return 1 / 0
with self.getLoggerStream('synapse.lib.threads',
'error running task for') as stream:
with s_threads.Pool() as pool:
pool.call(breakstuff)
self.true(stream.wait(2))
self.true(data.get('key'))
def __init__(self, pool=None):
EventBus.__init__(self)
if pool is None:
pool = s_threads.Pool()
self.pool = pool
self.root = None
self.lock = threading.Lock()
self.wake = threading.Event()
self.thr = self._runSchedMain()
self.onfini(self._onSchedFini)
def test_threads_exception(self):
data = {}
def breakstuff():
data['key'] = True
return 1 / 0
with self.getLoggerStream('synapse.lib.threads') as stream:
with s_threads.Pool() as pool:
pool.call(breakstuff)
time.sleep(0.1)
self.true(data.get('key'))
stream.seek(0)
mesgs = stream.read()
self.isin('error running task for', mesgs)
def __init__(self, pool=None):
EventBus.__init__(self)
DmonConf.__init__(self)
self.auth = None
self.socks = {} # sockets by iden
self.shared = {} # objects provided by daemon
self.pushed = {} # objects provided by sockets
self.csides = {} # item:[ (name,path), ... ]
self.reflect = {} # objects reflect info by name
self._dmon_ons = {}
self._dmon_links = [] # list of listen links
self._dmon_yields = set()
if pool is None:
pool = s_threads.Pool(size=8, maxsize=-1)
self.pool = pool
self.plex = s_socket.Plex()
self.cura = s_session.Curator()
self.onfini(self.plex.fini)
self.onfini(self.pool.fini)
self.onfini(self.cura.fini)
self.on('link:sock:init', self._onLinkSockInit)
self.plex.on('link:sock:mesg', self._onLinkSockMesg)
self.mesgfuncs = {}
self.setMesgFunc('tele:syn', self._onTeleSynMesg)
self.setMesgFunc('sock:gzip', self._onSockGzipMesg)
self.setMesgFunc('tele:call', self._onTeleCallMesg)
# for "client shared" objects...
self.setMesgFunc('tele:push', self._onTelePushMesg)
self.setMesgFunc('tele:retn', self._onTeleRetnMesg)
self.setMesgFunc('tele:on', self._onTeleOnMesg)
self.setMesgFunc('tele:off', self._onTeleOffMesg)
self.setMesgFunc('tele:yield:fini', self._onTeleYieldFini)
def __init__(self, core=None, pool=None):
EventBus.__init__(self)
DmonConf.__init__(self)
if core == None:
core = s_cortex.openurl('ram:///')
self.socks = {} # sockets by iden
self.shared = {} # objects provided by daemon
self.pushed = {} # objects provided by sockets
self._dmon_links = [] # list of listen links
self._dmon_yields = set()
if pool == None:
pool = s_threads.Pool(size=8, maxsize=-1)
self.pki = s_pki.PkiStor(core)
self.pool = pool
self.core = core
self.plex = s_socket.Plex()
self.onfini(self.plex.fini)
self.onfini(self.pool.fini)
self.on('link:sock:init', self._onLinkSockInit)
self.plex.on('link:sock:mesg', self._onLinkSockMesg)
self.mesgfuncs = {}
self.setMesgFunc('tele:syn', self._onTeleSynMesg)
self.setMesgFunc('tele:skey', self._onTeleSkeyMesg)
self.setMesgFunc('tele:call', self._onTeleCallMesg)
# for "client shared" objects...
self.setMesgFunc('tele:push', self._onTelePushMesg)
self.setMesgFunc('tele:retn', self._onTeleRetnMesg)
self.setMesgFunc('tele:on', self._onTeleOnMesg)
self.setMesgFunc('tele:off', self._onTeleOffMesg)
self.setMesgFunc('tele:yield:fini', self._onTeleYieldFini)
def __init__(self, conf=None):
s_config.Config.__init__(self, conf)
self.epoll = select.epoll()
self.socks = {}
self.links = {}
self.thrd = s_threads.worker(self._runPollLoop)
self.onfini(self._onPlexFini)
pmax = self.getConfOpt('pool:max')
self.pool = s_threads.Pool(maxsize=pmax)
self.onfini(self.pool.fini)
self.polls = {}
def __init__(self, core=None, opts=None, *args, **kwargs):
s_config.Config.__init__(self)
# Runtime-settable options
self.onConfOptSet(CACHE_ENABLED, self._onSetWebCache)
self.onConfOptSet(CACHE_TIMEOUT, self._onSetWebCacheTimeout)
# Things we need prior to loading in conf values
self.web_boss = s_async.Boss()
self.web_cache = s_cache.Cache()
self.web_cache_enabled = False
if opts:
self.setConfOpts(opts)
self._web_required_keys = ('namespace', 'doc', 'apis')
self._web_apis = {}
self._web_namespaces = set([])
self._web_docs = {}
self._web_default_http_args = {
} # Global request headers per namespace
# Check configable options before we spin up any more resources
max_clients = self.getConfOpt(MAX_CLIENTS)
pool_min = self.getConfOpt(MIN_WORKER_THREADS)
pool_max = self.getConfOpt(MAX_WORKER_THREADS)
if pool_min < 1:
raise s_common.BadConfValu(
name=MIN_WORKER_THREADS,
valu=pool_min,
mesg='web:worker:threads:min must be greater than 1')
if pool_max < pool_min:
raise s_common.BadConfValu(
name=MAX_WORKER_THREADS,
valu=pool_max,
mesg=
'web:worker:threads:max must be greater than the web:worker:threads:min'
)
if max_clients < 1:
raise s_common.BadConfValu(
name=MAX_CLIENTS,
valu=max_clients,
mesg='web:tornado:max_clients must be greater than 1')
# Tornado Async
loop = kwargs.get('ioloop')
if loop is None:
loop = t_ioloop.IOLoop()
self.web_loop = loop
self.web_client = t_http.AsyncHTTPClient(io_loop=self.web_loop,
max_clients=max_clients)
self.web_iothr = self._runIoLoop()
# Synapse Async and thread pool
self.web_pool = s_threads.Pool(pool_min, pool_max)
# Synapse Core and ingest tracking
if core is None:
core = s_cortex.openurl('ram://')
self.onfini(core.fini)
self.web_core = core
self._web_api_ingests = collections.defaultdict(list)
self._web_api_gest_opens = {}
# Setup Fini handlers
self.onfini(self._onHypoFini)
# List of content-type headers to skip automatic decoding
self._web_content_type_skip = set([])
self.webContentTypeSkipAdd('application/octet-stream')
for ct in kwargs.get('content_type_skip', []):
self.webContentTypeSkipAdd(ct)
import threading import multiprocessing lock = threading.RLock() import synapse.lib.sched as s_sched import synapse.lib.threads as s_threads # go high since they will mostly be IO bound tmax = multiprocessing.cpu_count() * 8 pool = s_threads.Pool(maxsize=tmax) sched = s_sched.Sched(pool=pool)
