def test_lib_net_stoplisten(self): names = ('conn', 'lisn', 'ping', 'pong') steps = self.getTestSteps(names) class LisnLink(s_net.Link): def linked(self): steps.done('lisn') def handlers(self): return { 'ping': self._onRecvPing, } def _onRecvPing(self, link, mesg): self.tx(('pong', {})) self.fini() steps.done('ping') class ConnLink(s_net.Link): def linked(self): self.tx(('ping', {'haha': 2})) steps.done('conn') def handlers(self): return { 'pong': self._onRecvPong, } def _onRecvPong(self, link, mesg): self.tx(('ping', {'haha': 2})) steps.done('pong') with s_net.Plex() as plex: def onconn(ok, link): if not ok: erno = link estr = os.strerror(erno) logger.error('test_lib_net_basic.onconn() error: %d %s' % (erno, estr)) return conn = ConnLink(link) link.onrx(conn.rx) conn.linked() def onlink(link): lisn = LisnLink(link) link.onrx(lisn.rx) lisn.linked() addr = plex.listen(('127.0.0.1', 0), onlink) plex.connect(addr, onconn) steps.waitall(timeout=2)
def test_lib_net_connectfail(self): expected_msg = 'connect() onconn failed' with self.getLoggerStream('synapse.lib.net', expected_msg) as stream: with s_net.Plex() as plex: plex.connect(('127.0.0.1', 0), None) self.true(stream.wait(10)) stream.seek(0) mesgs = stream.read() self.isin(expected_msg, mesgs)
def test_lib_net_listenfail(self): expected_msg = 'listen() onlink error' with self.getLoggerStream('synapse.lib.net', expected_msg) as stream: with s_net.Plex() as plex: addr = plex.listen(('127.0.0.1', 0), None) plex.connect(addr, None) self.true(stream.wait(10)) stream.seek(0) mesgs = stream.read() self.isin(expected_msg, mesgs)
def test_lib_net_basic(self): names = ('conn', 'lisn', 'ping', 'pong') steps = self.getTestSteps(names) class LisnLink(s_net.Link): def linked(self): steps.done('lisn') def handlers(self): return { 'ping': self._onRecvPing, } def _onRecvPing(self, link, mesg): self.tx(('pong', {})) steps.done('ping') class ConnLink(s_net.Link): def linked(self): self.tx(('ping', {'haha': 2})) steps.done('conn') def handlers(self): return { 'pong': self._onRecvPong, } def _onRecvPong(self, link, mesg): steps.done('pong') with s_net.Plex() as plex: def onconn(ok, link): conn = ConnLink(link) link.onrx(conn.rx) conn.linked() def onlink(link): lisn = LisnLink(link) link.onrx(lisn.rx) lisn.linked() addr = plex.listen(('127.0.0.1', 0), onlink) plex.connect(addr, onconn) steps.waitall(timeout=2)
def test_lib_net_finis(self): names = ('conn', 'lisn') steps = self.getTestSteps(names) expected_msg = 'I WAS FINID' with self.getLoggerStream('synapse.tests.test_lib_net', expected_msg) as stream: class LisnLink(s_net.Link): def linked(self): steps.done('lisn') class ConnLink(s_net.Link): def linked(self): self.rxfini() self.txfini(data='GOODBYE') self.txfini() steps.done('conn') def onfini(self, msg): logger.error('I WAS FINID') with s_net.Plex() as plex: def onconn(ok, link): conn = ConnLink(link) link.onrx(conn.rx) conn.linked() def onlink(link): lisn = LisnLink(link) link.onrx(lisn.rx) lisn.linked() addr = plex.listen(('127.0.0.1', 0), onlink) plex.connect(addr, onconn) steps.waitall(timeout=2) self.true(stream.wait(10)) stream.seek(0) mesgs = stream.read() self.isin(expected_msg, mesgs)
def test_lib_net_unknown(self): names = ('conn', 'lisn') steps = self.getTestSteps(names) expected_msg = 'unknown message type wat' with self.getLoggerStream('synapse.lib.net', expected_msg) as stream: class LisnLink(s_net.Link): def linked(self): steps.done('lisn') class ConnLink(s_net.Link): def linked(self): steps.done('conn') self.tx(('wat', {})) with s_net.Plex() as plex: def onconn(ok, link): conn = ConnLink(link) link.onrx(conn.rx) conn.linked() def onlink(link): lisn = LisnLink(link) link.onrx(lisn.rx) lisn.linked() addr = plex.listen(('127.0.0.1', 0), onlink) plex.connect(addr, onconn) steps.waitall(timeout=2) self.true(stream.wait(10)) stream.seek(0) mesgs = stream.read() self.isin(expected_msg, mesgs)
def __init__(self, dirn, conf=None): s_net.Link.__init__(self) s_config.Config.__init__(self) self.dirn = dirn s_common.gendir(dirn) # config file in the dir first... self.loadConfPath(self._path('config.json')) if conf is not None: self.setConfOpts(conf) self.reqConfOpts() self.plex = s_net.Plex() self.kvstor = s_kv.KvStor(self._path('cell.lmdb')) self.kvinfo = self.kvstor.getKvDict('cell:info') # open our vault self.vault = s_vault.Vault(self._path('vault.lmdb')) self.root = self.vault.genRootCert() # setup our certificate and private key auth = None path = self._path('cell.auth') if os.path.isfile(path): with open(path, 'rb') as fd: auth = s_msgpack.un(fd.read()) # if we dont have provided auth, assume we stand alone if auth is None: auth = self.vault.genUserAuth('root') with open(path, 'wb') as fd: fd.write(s_msgpack.en(auth)) path = self._path('user.auth') auth = self.vault.genUserAuth('user') with open(path, 'wb') as fd: fd.write(s_msgpack.en(auth)) roots = self.vault.getRootCerts() SessBoss.__init__(self, auth, roots) host = self.getConfOpt('host') port = self.getConfOpt('port') if port == 0: port = self.kvinfo.get('port', port) def onchan(chan): sess = CellSess(chan, self) chan.onrx(sess.rx) self.sessplex = s_net.ChanPlex(onchan=onchan) self.sessplex.setLinkProp('repr', 'Cell.sessplex') def onlink(link): link.onrx(self.sessplex.rx) self._cell_addr = self.plex.listen((host, port), onlink) # save the port so it can be semi-stable... self.kvinfo.set('port', self._cell_addr[1]) # Give implementers the chance to hook into the cell self.postCell() # lock cell.lock self.lockfd = s_common.genfile(self._path('cell.lock')) try: fcntl.lockf(self.lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError as e: logger.exception('Failed to obtain lock for [%s]', self.lockfd.name) raise self.onfini(self._onCellFini) self.onfini(self.finiCell) logger.debug('Cell is done initializing')
('synapse.models.language.LangMod', {}), ('synapse.models.temporal.TimeMod', {}), ('synapse.models.chemistry.ChemMod', {}), ('synapse.models.gov.intl.GovIntlMod', {}), ) ############################################## # setup glob here to avoid import loops... import synapse.glob as s_glob import synapse.lib.net as s_net import synapse.lib.sched as s_sched import synapse.lib.threads as s_threads tmax = multiprocessing.cpu_count() * 8 s_glob.plex = s_net.Plex() s_glob.pool = s_threads.Pool(maxsize=tmax) s_glob.sched = s_sched.Sched(pool=s_glob.pool) ############################################## import synapse.lib.modules as s_modules for mod, conf in BASE_MODULES: s_modules.load_ctor(mod, conf) # Register any CoreModules from envars mods = os.getenv('SYN_CORE_MODULES') if mods: for name in mods.split(','): name = name.strip() try: s_modules.load_ctor(name, {})
import threading import multiprocessing lock = threading.RLock() import synapse.lib.net as s_net import synapse.lib.sched as s_sched import synapse.lib.threads as s_threads # go high since they will mostly be IO bound tmax = multiprocessing.cpu_count() * 8 plex = s_net.Plex() pool = s_threads.Pool(maxsize=tmax) sched = s_sched.Sched(pool=pool) def inpool(f): ''' Wrap the given function to be called from the global thread pool. ''' def wrap(*args, **kwargs): return pool.call(f, *args, **kwargs) return wrap
def __init__(self, dirn, conf=None): s_net.Link.__init__(self) s_config.Configable.__init__(self) self.dirn = dirn s_common.gendir(dirn) # config file in the dir first... self.loadConfPath(self._path('config.json')) if conf is not None: self.setConfOpts(conf) self.reqConfOpts() self.plex = s_net.Plex() self.kvstor = s_kv.KvStor(self._path('cell.lmdb')) self.kvinfo = self.kvstor.getKvDict('cell:info') # open our vault self.vault = s_vault.Vault(self._path('vault.lmdb')) self.root = self.vault.genRootCert() # setup our certificate and private key auth = self._genSelfAuth() roots = self.vault.getRootCerts() SessBoss.__init__(self, auth, roots) self.cellinfo = { 'ctor': '%s.%s' % (self.__class__.__module__, self.__class__.__name__) } self.cellauth = auth self.cellpool = None self.celluser = CellUser(auth, roots=roots) addr = self.getConfOpt('bind') port = self.getConfOpt('port') def onlink(link): sess = CellSess(link, self) link.onrx(sess.rx) # fini cuts both ways sess.onfini(link.fini) link.onfini(sess.fini) addr, port = self.plex.listen((addr, port), onlink) host = self.getConfOpt('host') self.celladdr = (host, port) # add it to our neuron reg info... self.cellinfo['addr'] = self.celladdr # lock cell.lock self.lockfd = s_common.genfile(self._path('cell.lock')) try: fcntl.lockf(self.lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError as e: logger.exception('Failed to obtain lock for [%s]', self.lockfd.name) raise self.onfini(self._onCellFini) self.onfini(self.finiCell) self.neuraddr = self.cellauth[1].get('neuron') if self.neuraddr is not None: self.cellpool = CellPool(auth, self.neuraddr, neurfunc=self._onNeurSess) self.onfini(self.cellpool.fini) # Give implementers the chance to hook into the cell self.postCell() logger.debug('Cell is done initializing')