def test_lib_net_stoplisten(self):
names = ('conn', 'lisn', 'ping', 'pong')
steps = self.getTestSteps(names)
class LisnLink(s_net.Link):
def linked(self):
steps.done('lisn')
def handlers(self):
return {
'ping': self._onRecvPing,
}
def _onRecvPing(self, link, mesg):
self.tx(('pong', {}))
self.fini()
steps.done('ping')
class ConnLink(s_net.Link):
def linked(self):
self.tx(('ping', {'haha': 2}))
steps.done('conn')
def handlers(self):
return {
'pong': self._onRecvPong,
}
def _onRecvPong(self, link, mesg):
self.tx(('ping', {'haha': 2}))
steps.done('pong')
with s_net.Plex() as plex:
def onconn(ok, link):
if not ok:
erno = link
estr = os.strerror(erno)
logger.error('test_lib_net_basic.onconn() error: %d %s' % (erno, estr))
return
conn = ConnLink(link)
link.onrx(conn.rx)
conn.linked()
def onlink(link):
lisn = LisnLink(link)
link.onrx(lisn.rx)
lisn.linked()
addr = plex.listen(('127.0.0.1', 0), onlink)
plex.connect(addr, onconn)
steps.waitall(timeout=2)
def test_lib_net_connectfail(self):
expected_msg = 'connect() onconn failed'
with self.getLoggerStream('synapse.lib.net', expected_msg) as stream:
with s_net.Plex() as plex:
plex.connect(('127.0.0.1', 0), None)
self.true(stream.wait(10))
stream.seek(0)
mesgs = stream.read()
self.isin(expected_msg, mesgs)
def test_lib_net_listenfail(self):
expected_msg = 'listen() onlink error'
with self.getLoggerStream('synapse.lib.net', expected_msg) as stream:
with s_net.Plex() as plex:
addr = plex.listen(('127.0.0.1', 0), None)
plex.connect(addr, None)
self.true(stream.wait(10))
stream.seek(0)
mesgs = stream.read()
self.isin(expected_msg, mesgs)
def test_lib_net_basic(self):
names = ('conn', 'lisn', 'ping', 'pong')
steps = self.getTestSteps(names)
class LisnLink(s_net.Link):
def linked(self):
steps.done('lisn')
def handlers(self):
return {
'ping': self._onRecvPing,
}
def _onRecvPing(self, link, mesg):
self.tx(('pong', {}))
steps.done('ping')
class ConnLink(s_net.Link):
def linked(self):
self.tx(('ping', {'haha': 2}))
steps.done('conn')
def handlers(self):
return {
'pong': self._onRecvPong,
}
def _onRecvPong(self, link, mesg):
steps.done('pong')
with s_net.Plex() as plex:
def onconn(ok, link):
conn = ConnLink(link)
link.onrx(conn.rx)
conn.linked()
def onlink(link):
lisn = LisnLink(link)
link.onrx(lisn.rx)
lisn.linked()
addr = plex.listen(('127.0.0.1', 0), onlink)
plex.connect(addr, onconn)
steps.waitall(timeout=2)
def test_lib_net_finis(self):
names = ('conn', 'lisn')
steps = self.getTestSteps(names)
expected_msg = 'I WAS FINID'
with self.getLoggerStream('synapse.tests.test_lib_net', expected_msg) as stream:
class LisnLink(s_net.Link):
def linked(self):
steps.done('lisn')
class ConnLink(s_net.Link):
def linked(self):
self.rxfini()
self.txfini(data='GOODBYE')
self.txfini()
steps.done('conn')
def onfini(self, msg):
logger.error('I WAS FINID')
with s_net.Plex() as plex:
def onconn(ok, link):
conn = ConnLink(link)
link.onrx(conn.rx)
conn.linked()
def onlink(link):
lisn = LisnLink(link)
link.onrx(lisn.rx)
lisn.linked()
addr = plex.listen(('127.0.0.1', 0), onlink)
plex.connect(addr, onconn)
steps.waitall(timeout=2)
self.true(stream.wait(10))
stream.seek(0)
mesgs = stream.read()
self.isin(expected_msg, mesgs)
def test_lib_net_unknown(self):
names = ('conn', 'lisn')
steps = self.getTestSteps(names)
expected_msg = 'unknown message type wat'
with self.getLoggerStream('synapse.lib.net', expected_msg) as stream:
class LisnLink(s_net.Link):
def linked(self):
steps.done('lisn')
class ConnLink(s_net.Link):
def linked(self):
steps.done('conn')
self.tx(('wat', {}))
with s_net.Plex() as plex:
def onconn(ok, link):
conn = ConnLink(link)
link.onrx(conn.rx)
conn.linked()
def onlink(link):
lisn = LisnLink(link)
link.onrx(lisn.rx)
lisn.linked()
addr = plex.listen(('127.0.0.1', 0), onlink)
plex.connect(addr, onconn)
steps.waitall(timeout=2)
self.true(stream.wait(10))
stream.seek(0)
mesgs = stream.read()
self.isin(expected_msg, mesgs)
def __init__(self, dirn, conf=None):
s_net.Link.__init__(self)
s_config.Config.__init__(self)
self.dirn = dirn
s_common.gendir(dirn)
# config file in the dir first...
self.loadConfPath(self._path('config.json'))
if conf is not None:
self.setConfOpts(conf)
self.reqConfOpts()
self.plex = s_net.Plex()
self.kvstor = s_kv.KvStor(self._path('cell.lmdb'))
self.kvinfo = self.kvstor.getKvDict('cell:info')
# open our vault
self.vault = s_vault.Vault(self._path('vault.lmdb'))
self.root = self.vault.genRootCert()
# setup our certificate and private key
auth = None
path = self._path('cell.auth')
if os.path.isfile(path):
with open(path, 'rb') as fd:
auth = s_msgpack.un(fd.read())
# if we dont have provided auth, assume we stand alone
if auth is None:
auth = self.vault.genUserAuth('root')
with open(path, 'wb') as fd:
fd.write(s_msgpack.en(auth))
path = self._path('user.auth')
auth = self.vault.genUserAuth('user')
with open(path, 'wb') as fd:
fd.write(s_msgpack.en(auth))
roots = self.vault.getRootCerts()
SessBoss.__init__(self, auth, roots)
host = self.getConfOpt('host')
port = self.getConfOpt('port')
if port == 0:
port = self.kvinfo.get('port', port)
def onchan(chan):
sess = CellSess(chan, self)
chan.onrx(sess.rx)
self.sessplex = s_net.ChanPlex(onchan=onchan)
self.sessplex.setLinkProp('repr', 'Cell.sessplex')
def onlink(link):
link.onrx(self.sessplex.rx)
self._cell_addr = self.plex.listen((host, port), onlink)
# save the port so it can be semi-stable...
self.kvinfo.set('port', self._cell_addr[1])
# Give implementers the chance to hook into the cell
self.postCell()
# lock cell.lock
self.lockfd = s_common.genfile(self._path('cell.lock'))
try:
fcntl.lockf(self.lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError as e:
logger.exception('Failed to obtain lock for [%s]',
self.lockfd.name)
raise
self.onfini(self._onCellFini)
self.onfini(self.finiCell)
logger.debug('Cell is done initializing')
('synapse.models.language.LangMod', {}),
('synapse.models.temporal.TimeMod', {}),
('synapse.models.chemistry.ChemMod', {}),
('synapse.models.gov.intl.GovIntlMod', {}),
)
##############################################
# setup glob here to avoid import loops...
import synapse.glob as s_glob
import synapse.lib.net as s_net
import synapse.lib.sched as s_sched
import synapse.lib.threads as s_threads
tmax = multiprocessing.cpu_count() * 8
s_glob.plex = s_net.Plex()
s_glob.pool = s_threads.Pool(maxsize=tmax)
s_glob.sched = s_sched.Sched(pool=s_glob.pool)
##############################################
import synapse.lib.modules as s_modules
for mod, conf in BASE_MODULES:
s_modules.load_ctor(mod, conf)
# Register any CoreModules from envars
mods = os.getenv('SYN_CORE_MODULES')
if mods:
for name in mods.split(','):
name = name.strip()
try:
s_modules.load_ctor(name, {})
import threading
import multiprocessing
lock = threading.RLock()
import synapse.lib.net as s_net
import synapse.lib.sched as s_sched
import synapse.lib.threads as s_threads
# go high since they will mostly be IO bound
tmax = multiprocessing.cpu_count() * 8
plex = s_net.Plex()
pool = s_threads.Pool(maxsize=tmax)
sched = s_sched.Sched(pool=pool)
def inpool(f):
'''
Wrap the given function to be called from the global thread pool.
'''
def wrap(*args, **kwargs):
return pool.call(f, *args, **kwargs)
return wrap
def __init__(self, dirn, conf=None):
s_net.Link.__init__(self)
s_config.Configable.__init__(self)
self.dirn = dirn
s_common.gendir(dirn)
# config file in the dir first...
self.loadConfPath(self._path('config.json'))
if conf is not None:
self.setConfOpts(conf)
self.reqConfOpts()
self.plex = s_net.Plex()
self.kvstor = s_kv.KvStor(self._path('cell.lmdb'))
self.kvinfo = self.kvstor.getKvDict('cell:info')
# open our vault
self.vault = s_vault.Vault(self._path('vault.lmdb'))
self.root = self.vault.genRootCert()
# setup our certificate and private key
auth = self._genSelfAuth()
roots = self.vault.getRootCerts()
SessBoss.__init__(self, auth, roots)
self.cellinfo = {
'ctor':
'%s.%s' % (self.__class__.__module__, self.__class__.__name__)
}
self.cellauth = auth
self.cellpool = None
self.celluser = CellUser(auth, roots=roots)
addr = self.getConfOpt('bind')
port = self.getConfOpt('port')
def onlink(link):
sess = CellSess(link, self)
link.onrx(sess.rx)
# fini cuts both ways
sess.onfini(link.fini)
link.onfini(sess.fini)
addr, port = self.plex.listen((addr, port), onlink)
host = self.getConfOpt('host')
self.celladdr = (host, port)
# add it to our neuron reg info...
self.cellinfo['addr'] = self.celladdr
# lock cell.lock
self.lockfd = s_common.genfile(self._path('cell.lock'))
try:
fcntl.lockf(self.lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError as e:
logger.exception('Failed to obtain lock for [%s]',
self.lockfd.name)
raise
self.onfini(self._onCellFini)
self.onfini(self.finiCell)
self.neuraddr = self.cellauth[1].get('neuron')
if self.neuraddr is not None:
self.cellpool = CellPool(auth,
self.neuraddr,
neurfunc=self._onNeurSess)
self.onfini(self.cellpool.fini)
# Give implementers the chance to hook into the cell
self.postCell()
logger.debug('Cell is done initializing')
