def test_cache_miss(self):
c = s_cache.Cache()
def onmiss(key):
return 10
c.setOnMiss(onmiss)
self.assertEqual(c.get('woot'), 10)
def test_cache_fini(self):
c = s_cache.Cache(maxtime=0.1)
c.put(1, 'a')
self.nn(c.schevt)
self.nn(c.schevt[1])
c.fini()
self.none(c.schevt[1])
self.eq(len(c), 0)
def test_cache_miss(self):
c = s_cache.Cache()
def onmiss(key):
return 10
c.setOnMiss(onmiss)
self.false('woot' in c)
self.eq(c.get('woot'), 10)
self.true('woot' in c)
def __init__(self, core=None, maxtime=onehour):
EventBus.__init__(self)
self.core = core
self.cache = s_cache.Cache(maxtime=maxtime)
self.cache.setOnMiss(self._getSessByIden)
self.cache.on('cache:pop', self._onSessCachePop)
self.onfini(self.cache.fini)
def __init__(self, core):
EventBus.__init__(self)
self.core = core
self.keys = c_cache.Cache()
self.keys.setOnMiss(self._getRsaKey)
self.pubs = c_cache.Cache()
self.pubs.setOnMiss(self._getPubKey)
self.certs = c_cache.Cache()
self.certs.setOnMiss(self._getTokenCert)
self.tokens = c_cache.Cache()
self.tokens.setOnMiss(self._getTokenTufo)
core.onfini(self.keys.fini)
core.onfini(self.pubs.fini)
core.onfini(self.certs.fini)
core.onfini(self.tokens.fini)
model = core.genDataModel()
model.addTufoForm('syn:token',
ptype='str',
doc='synapse identity token (user/host)')
model.addTufoProp('syn:token',
'user',
doc='humon readable user name for this token')
model.addTufoProp('syn:token',
'host',
doc='humon readable host name for this token')
model.addTufoProp('syn:token', 'blob', doc='Base64 encoded token blob')
model.addTufoProp('syn:token',
'cert',
doc='Base64 encoded certificate blob')
model.addTufoProp('syn:token',
'rsakey',
doc='base64( der( rsa.private ) )')
def __init__(self, core, maxtime=onehour):
EventBus.__init__(self)
self.core = core
self.model = core.genDataModel()
self.cache = s_cache.Cache(maxtime=maxtime)
self.cache.setOnMiss(self._getSessBySid)
self.cache.on('cache:pop', self._onSessCachePop)
self.onfini(self.cache.fini)
def test_cache_defval(self):
# Ensure default behaviors are covered.
c = s_cache.Cache()
r = c.get('foo')
self.none(r)
fc = s_cache.FixedCache(maxsize=10)
fr = fc.get('foo')
self.none(fr)
od = s_cache.OnDem()
with self.raises(KeyError) as cm:
od.get('foo')
def test_cache_timeout(self):
c = s_cache.Cache(maxtime=0.1)
e = threading.Event()
data = {}
def onflush(event):
data['event'] = event
e.set()
c.on('cache:flush', onflush)
c.put('woot', 10)
e.wait(timeout=2)
self.nn(data.get('event'))
def __init__(self, core=None, maxtime=onehour):
EventBus.__init__(self)
self.core = core
self.cache = s_cache.Cache(maxtime=maxtime)
self.cache.setOnMiss(self._getSessByIden)
self.cache.on('cache:pop', self._onSessCachePop)
self.onfini(self.cache.fini)
if self.core:
pdef = self.core.getPropDef('syn:sess')
if not pdef:
self.core.addTufoForm('syn:sess', ptype='guid')
def test_cache_set_maxtime(self):
c = s_cache.Cache()
c.setMaxTime(0.1)
e = threading.Event()
data = {}
def onflush(event):
data['event'] = event
e.set()
c.on('cache:flush', onflush)
c.put('woot', 10)
e.wait(timeout=2)
self.assertIsNotNone(data.get('event'))
def __init__(self, **settings):
EventBus.__init__(self)
self.onfini(self._onDavFini)
self.root = PathNode()
self.paths = s_cache.Cache(maxtime=60)
self.paths.setOnMiss(self._getDavNode)
self.app = tornado.web.Application((('.*', WebDavHandler, {
'webdav': self
}), ))
self.ioloop = tornado.ioloop.IOLoop()
settings['io_loop'] = self.ioloop
self.serv = tornado.httpserver.HTTPServer(self.app, **settings)
def test_cache_clearing(self):
c = s_cache.Cache()
d = {}
def flush(event):
key = event[1].get('key')
d[key] = c.get(key)
c.on('cache:flush', flush)
c.put(1, 'a')
c.put(2, 'b')
self.eq(len(c), 2)
c.flush(1)
self.true(1 in d)
self.eq(d, {1: 'a'})
self.eq(len(c), 2) # A straight flush doesn't remove the key.
c.clear()
self.eq(len(c), 0)
def test_cache_magic(self):
c = s_cache.Cache()
c.put(1, 'a')
c.put(2, 'b')
keys = set([])
values = set([])
self.eq(len(c), 2)
cvs = sorted(c.values())
self.eq(cvs, ['a', 'b'])
cks = sorted(c.keys())
self.eq(cks, [1, 2])
for k, v in c:
keys.add(k)
values.add(v)
self.eq(keys, {1, 2})
self.eq(values, {'a', 'b'})
def __init__(self, rules):
self._r_rules = rules
self._r_match = s_cache.MatchCache()
self._r_rules_by_perm = s_cache.Cache(onmiss=self._onRulesPermMiss)
def __init__(self, core=None, pool=None):
s_daemon.Daemon.__init__(self, core=core, pool=pool)
self.sched = s_sched.Sched()
self.core.addTufoForm('neuron')
self.core.addTufoProp('neuron', 'name')
self.core.addTufoProp('neuron', 'super', ptype='int', defval=0)
self.core.addTufoProp('neuron', 'usepki', ptype='bool', defval=0)
self.neur = self.core.formTufoByProp('neuron', 'self')
self.iden = self.neur[0]
self.mesh = {}
self.peers = {}
self.mesh['certs'] = {}
self.mesh['links'] = {}
self.mesh['peers'] = {self.iden: self.neur}
self.sockbyfrom = s_cache.Cache(maxtime=120)
self.sockbyfrom.setOnMiss(self._getFakeSock)
self.links = collections.defaultdict(set)
self.linkpaths = s_cache.Cache(maxtime=30)
self.linkpaths.setOnMiss(self._getLinkPath)
self.setMesgFunc('peer:syn', self._onPeerSynMesg)
self.setMesgFunc('peer:synack', self._onPeerSynAckMesg)
self.setMesgFunc('peer:link:init', self._onPeerLinkInitMesg)
#self.setMesgFunc('neu:peer:chal', self._onNeuPeerChal )
#self.setMesgFunc('neu:peer:resp', self._onNeuPeerResp )
self.setMesgFunc('neu:data', self._onNeuDataMesg)
self.setMesgFunc('neu:storm', self._onNeuStormMesg)
self.on('neu:link:init', self._onNeuLinkInit)
self.on('neu:link:fini', self._onNeuLinkFini)
self.share('neuron', self)
self.hasopt = {}
# fire any persistent neuron listeners
for url in self.core.getTufoList(self.neur, 'listen'):
try:
self.listen(url)
self.hasopt[('listen', url)] = True
except Exception as e:
logger.error('neu listen: %s', e)
# spin up any persistent neuron connections
for url in self.core.getTufoList(self.neur, 'connect'):
try:
self.connect(url)
self.hasopt[('connect', url)] = True
except Exception as e:
logger.error('neu connect: %s', e)
# load any persistent shared objects
for jsval in self.core.getTufoList(self.neur, 'shared'):
try:
info = json.loads(v)
name = info.get('name')
task = info.get('task')
tags = info.get('tags', ())
item = s_dyndeps.runDynTask(task)
self.share(name, item, tags=tags)
except Exception as e:
logger.error('neu share: %s', e)
def __init__(self):
self._trig_list = []
self._trig_match = s_cache.MatchCache()
self._trig_byname = s_cache.Cache(onmiss=self._onTrigNameMiss)
def __init__(self, core=None, opts=None, *args, **kwargs):
s_config.Config.__init__(self)
# Runtime-settable options
self.onConfOptSet(CACHE_ENABLED, self._onSetWebCache)
self.onConfOptSet(CACHE_TIMEOUT, self._onSetWebCacheTimeout)
# Things we need prior to loading in conf values
self.web_boss = s_async.Boss()
self.web_cache = s_cache.Cache()
self.web_cache_enabled = False
if opts:
self.setConfOpts(opts)
self._web_required_keys = ('namespace', 'doc', 'apis')
self._web_apis = {}
self._web_namespaces = set([])
self._web_docs = {}
self._web_default_http_args = {
} # Global request headers per namespace
# Check configable options before we spin up any more resources
max_clients = self.getConfOpt(MAX_CLIENTS)
pool_min = self.getConfOpt(MIN_WORKER_THREADS)
pool_max = self.getConfOpt(MAX_WORKER_THREADS)
if pool_min < 1:
raise s_common.BadConfValu(
name=MIN_WORKER_THREADS,
valu=pool_min,
mesg='web:worker:threads:min must be greater than 1')
if pool_max < pool_min:
raise s_common.BadConfValu(
name=MAX_WORKER_THREADS,
valu=pool_max,
mesg=
'web:worker:threads:max must be greater than the web:worker:threads:min'
)
if max_clients < 1:
raise s_common.BadConfValu(
name=MAX_CLIENTS,
valu=max_clients,
mesg='web:tornado:max_clients must be greater than 1')
# Tornado Async
loop = kwargs.get('ioloop')
if loop is None:
loop = t_ioloop.IOLoop()
self.web_loop = loop
self.web_client = t_http.AsyncHTTPClient(io_loop=self.web_loop,
max_clients=max_clients)
self.web_iothr = self._runIoLoop()
# Synapse Async and thread pool
self.web_pool = s_threads.Pool(pool_min, pool_max)
# Synapse Core and ingest tracking
if core is None:
core = s_cortex.openurl('ram://')
self.onfini(core.fini)
self.web_core = core
self._web_api_ingests = collections.defaultdict(list)
self._web_api_gest_opens = {}
# Setup Fini handlers
self.onfini(self._onHypoFini)
# List of content-type headers to skip automatic decoding
self._web_content_type_skip = set([])
self.webContentTypeSkipAdd('application/octet-stream')
for ct in kwargs.get('content_type_skip', []):
self.webContentTypeSkipAdd(ct)
