def test_cache_miss(self): c = s_cache.Cache() def onmiss(key): return 10 c.setOnMiss(onmiss) self.assertEqual(c.get('woot'), 10)
def test_cache_fini(self): c = s_cache.Cache(maxtime=0.1) c.put(1, 'a') self.nn(c.schevt) self.nn(c.schevt[1]) c.fini() self.none(c.schevt[1]) self.eq(len(c), 0)
def test_cache_miss(self): c = s_cache.Cache() def onmiss(key): return 10 c.setOnMiss(onmiss) self.false('woot' in c) self.eq(c.get('woot'), 10) self.true('woot' in c)
def __init__(self, core=None, maxtime=onehour): EventBus.__init__(self) self.core = core self.cache = s_cache.Cache(maxtime=maxtime) self.cache.setOnMiss(self._getSessByIden) self.cache.on('cache:pop', self._onSessCachePop) self.onfini(self.cache.fini)
def __init__(self, core): EventBus.__init__(self) self.core = core self.keys = c_cache.Cache() self.keys.setOnMiss(self._getRsaKey) self.pubs = c_cache.Cache() self.pubs.setOnMiss(self._getPubKey) self.certs = c_cache.Cache() self.certs.setOnMiss(self._getTokenCert) self.tokens = c_cache.Cache() self.tokens.setOnMiss(self._getTokenTufo) core.onfini(self.keys.fini) core.onfini(self.pubs.fini) core.onfini(self.certs.fini) core.onfini(self.tokens.fini) model = core.genDataModel() model.addTufoForm('syn:token', ptype='str', doc='synapse identity token (user/host)') model.addTufoProp('syn:token', 'user', doc='humon readable user name for this token') model.addTufoProp('syn:token', 'host', doc='humon readable host name for this token') model.addTufoProp('syn:token', 'blob', doc='Base64 encoded token blob') model.addTufoProp('syn:token', 'cert', doc='Base64 encoded certificate blob') model.addTufoProp('syn:token', 'rsakey', doc='base64( der( rsa.private ) )')
def __init__(self, core, maxtime=onehour): EventBus.__init__(self) self.core = core self.model = core.genDataModel() self.cache = s_cache.Cache(maxtime=maxtime) self.cache.setOnMiss(self._getSessBySid) self.cache.on('cache:pop', self._onSessCachePop) self.onfini(self.cache.fini)
def test_cache_defval(self): # Ensure default behaviors are covered. c = s_cache.Cache() r = c.get('foo') self.none(r) fc = s_cache.FixedCache(maxsize=10) fr = fc.get('foo') self.none(fr) od = s_cache.OnDem() with self.raises(KeyError) as cm: od.get('foo')
def test_cache_timeout(self): c = s_cache.Cache(maxtime=0.1) e = threading.Event() data = {} def onflush(event): data['event'] = event e.set() c.on('cache:flush', onflush) c.put('woot', 10) e.wait(timeout=2) self.nn(data.get('event'))
def __init__(self, core=None, maxtime=onehour): EventBus.__init__(self) self.core = core self.cache = s_cache.Cache(maxtime=maxtime) self.cache.setOnMiss(self._getSessByIden) self.cache.on('cache:pop', self._onSessCachePop) self.onfini(self.cache.fini) if self.core: pdef = self.core.getPropDef('syn:sess') if not pdef: self.core.addTufoForm('syn:sess', ptype='guid')
def test_cache_set_maxtime(self): c = s_cache.Cache() c.setMaxTime(0.1) e = threading.Event() data = {} def onflush(event): data['event'] = event e.set() c.on('cache:flush', onflush) c.put('woot', 10) e.wait(timeout=2) self.assertIsNotNone(data.get('event'))
def __init__(self, **settings): EventBus.__init__(self) self.onfini(self._onDavFini) self.root = PathNode() self.paths = s_cache.Cache(maxtime=60) self.paths.setOnMiss(self._getDavNode) self.app = tornado.web.Application((('.*', WebDavHandler, { 'webdav': self }), )) self.ioloop = tornado.ioloop.IOLoop() settings['io_loop'] = self.ioloop self.serv = tornado.httpserver.HTTPServer(self.app, **settings)
def test_cache_clearing(self): c = s_cache.Cache() d = {} def flush(event): key = event[1].get('key') d[key] = c.get(key) c.on('cache:flush', flush) c.put(1, 'a') c.put(2, 'b') self.eq(len(c), 2) c.flush(1) self.true(1 in d) self.eq(d, {1: 'a'}) self.eq(len(c), 2) # A straight flush doesn't remove the key. c.clear() self.eq(len(c), 0)
def test_cache_magic(self): c = s_cache.Cache() c.put(1, 'a') c.put(2, 'b') keys = set([]) values = set([]) self.eq(len(c), 2) cvs = sorted(c.values()) self.eq(cvs, ['a', 'b']) cks = sorted(c.keys()) self.eq(cks, [1, 2]) for k, v in c: keys.add(k) values.add(v) self.eq(keys, {1, 2}) self.eq(values, {'a', 'b'})
def __init__(self, rules): self._r_rules = rules self._r_match = s_cache.MatchCache() self._r_rules_by_perm = s_cache.Cache(onmiss=self._onRulesPermMiss)
def __init__(self, core=None, pool=None): s_daemon.Daemon.__init__(self, core=core, pool=pool) self.sched = s_sched.Sched() self.core.addTufoForm('neuron') self.core.addTufoProp('neuron', 'name') self.core.addTufoProp('neuron', 'super', ptype='int', defval=0) self.core.addTufoProp('neuron', 'usepki', ptype='bool', defval=0) self.neur = self.core.formTufoByProp('neuron', 'self') self.iden = self.neur[0] self.mesh = {} self.peers = {} self.mesh['certs'] = {} self.mesh['links'] = {} self.mesh['peers'] = {self.iden: self.neur} self.sockbyfrom = s_cache.Cache(maxtime=120) self.sockbyfrom.setOnMiss(self._getFakeSock) self.links = collections.defaultdict(set) self.linkpaths = s_cache.Cache(maxtime=30) self.linkpaths.setOnMiss(self._getLinkPath) self.setMesgFunc('peer:syn', self._onPeerSynMesg) self.setMesgFunc('peer:synack', self._onPeerSynAckMesg) self.setMesgFunc('peer:link:init', self._onPeerLinkInitMesg) #self.setMesgFunc('neu:peer:chal', self._onNeuPeerChal ) #self.setMesgFunc('neu:peer:resp', self._onNeuPeerResp ) self.setMesgFunc('neu:data', self._onNeuDataMesg) self.setMesgFunc('neu:storm', self._onNeuStormMesg) self.on('neu:link:init', self._onNeuLinkInit) self.on('neu:link:fini', self._onNeuLinkFini) self.share('neuron', self) self.hasopt = {} # fire any persistent neuron listeners for url in self.core.getTufoList(self.neur, 'listen'): try: self.listen(url) self.hasopt[('listen', url)] = True except Exception as e: logger.error('neu listen: %s', e) # spin up any persistent neuron connections for url in self.core.getTufoList(self.neur, 'connect'): try: self.connect(url) self.hasopt[('connect', url)] = True except Exception as e: logger.error('neu connect: %s', e) # load any persistent shared objects for jsval in self.core.getTufoList(self.neur, 'shared'): try: info = json.loads(v) name = info.get('name') task = info.get('task') tags = info.get('tags', ()) item = s_dyndeps.runDynTask(task) self.share(name, item, tags=tags) except Exception as e: logger.error('neu share: %s', e)
def __init__(self): self._trig_list = [] self._trig_match = s_cache.MatchCache() self._trig_byname = s_cache.Cache(onmiss=self._onTrigNameMiss)
def __init__(self, core=None, opts=None, *args, **kwargs): s_config.Config.__init__(self) # Runtime-settable options self.onConfOptSet(CACHE_ENABLED, self._onSetWebCache) self.onConfOptSet(CACHE_TIMEOUT, self._onSetWebCacheTimeout) # Things we need prior to loading in conf values self.web_boss = s_async.Boss() self.web_cache = s_cache.Cache() self.web_cache_enabled = False if opts: self.setConfOpts(opts) self._web_required_keys = ('namespace', 'doc', 'apis') self._web_apis = {} self._web_namespaces = set([]) self._web_docs = {} self._web_default_http_args = { } # Global request headers per namespace # Check configable options before we spin up any more resources max_clients = self.getConfOpt(MAX_CLIENTS) pool_min = self.getConfOpt(MIN_WORKER_THREADS) pool_max = self.getConfOpt(MAX_WORKER_THREADS) if pool_min < 1: raise s_common.BadConfValu( name=MIN_WORKER_THREADS, valu=pool_min, mesg='web:worker:threads:min must be greater than 1') if pool_max < pool_min: raise s_common.BadConfValu( name=MAX_WORKER_THREADS, valu=pool_max, mesg= 'web:worker:threads:max must be greater than the web:worker:threads:min' ) if max_clients < 1: raise s_common.BadConfValu( name=MAX_CLIENTS, valu=max_clients, mesg='web:tornado:max_clients must be greater than 1') # Tornado Async loop = kwargs.get('ioloop') if loop is None: loop = t_ioloop.IOLoop() self.web_loop = loop self.web_client = t_http.AsyncHTTPClient(io_loop=self.web_loop, max_clients=max_clients) self.web_iothr = self._runIoLoop() # Synapse Async and thread pool self.web_pool = s_threads.Pool(pool_min, pool_max) # Synapse Core and ingest tracking if core is None: core = s_cortex.openurl('ram://') self.onfini(core.fini) self.web_core = core self._web_api_ingests = collections.defaultdict(list) self._web_api_gest_opens = {} # Setup Fini handlers self.onfini(self._onHypoFini) # List of content-type headers to skip automatic decoding self._web_content_type_skip = set([]) self.webContentTypeSkipAdd('application/octet-stream') for ct in kwargs.get('content_type_skip', []): self.webContentTypeSkipAdd(ct)