def __init__(self):
EventBus.__init__(self)
self.threads = {}
self.on('thread:init', self._initThread)
self.on('thread:fini', self._finiThread)
self.onfini(self._finiThreadBoss)
def __init__(self, items=()):
EventBus.__init__(self)
self.deq = collections.deque()
self.lock = threading.Lock()
self.event = threading.Event()
self.onfini( self._onQueFini )
def __init__(self, core):
EventBus.__init__(self)
self.core = core
self.keys = c_cache.Cache()
self.keys.setOnMiss( self._getRsaKey )
self.pubs = c_cache.Cache()
self.pubs.setOnMiss( self._getPubKey )
self.certs = c_cache.Cache()
self.certs.setOnMiss( self._getTokenCert )
self.tokens = c_cache.Cache()
self.tokens.setOnMiss( self._getTokenTufo )
core.onfini( self.keys.fini )
core.onfini( self.pubs.fini )
core.onfini( self.certs.fini )
core.onfini( self.tokens.fini )
core.addTufoForm('syn:token', ptype='str', doc='synapse identity token (user/host)')
core.addTufoProp('syn:token', 'user', doc='humon readable user name for this token')
core.addTufoProp('syn:token', 'host', doc='humon readable host name for this token')
core.addTufoProp('syn:token', 'blob', doc='Base64 encoded token blob')
core.addTufoProp('syn:token', 'cert', doc='Base64 encoded certificate blob')
core.addTufoProp('syn:token', 'rsakey', doc='base64( der( rsa.private ) )')
def __init__(self, queen, **config):
# NOTE: queen must *always* be a telepath proxy
EventBus.__init__(self)
self.iden = guid()
self.slots = {}
self.slocs = {}
self.queen = queen
self.config = config
# FIXME maybe put our hostname etc in config?
self.queen.on('tele:sock:init', self._onTeleSockInit )
self.queen.on('hive:tell:%s' % self.iden, self._onHiveTell)
self.localurl = 'local://%s/syn.queen' % self.iden
# each worker has a local:// daemon
self.dmon = s_daemon.Daemon()
self.dmon.listen(self.localurl)
self.dmon.share('syn.queen',queen)
self.on('hive:slot:run', self._onHiveSlotRun)
self.on('hive:slot:fini', self._onHiveSlotFini)
self._initQueenProxy()
def __init__(self, cura, sess):
EventBus.__init__(self)
self.sid = sess[0]
self.cura = cura
self.sess = sess
self.local = {} # runtime only props
def __init__(self):
EventBus.__init__(self)
self.tagsbyname = {}
self.coresbyname = {}
self.coresbytag = collections.defaultdict(list)
self.onfini( self._onMetaFini )
def __init__(self, relay):
EventBus.__init__(self)
self.relay = relay
self.boss = s_threads.ThreadBoss()
self.onfini(self.boss.fini)
# we get the sock first to fill in info
self.on('link:sock:init', self._onLinkSockInit)
def __init__(self, fd=None):
EventBus.__init__(self)
self.fd = fd
self.last = time.time()
self.lock = threading.Lock()
self.items = []
self.event = threading.Event()
self.onfini( self._onQueFini )
def __init__(self):
EventBus.__init__(self)
self.root = None
self.lock = threading.Lock()
self.wake = threading.Event()
self.thr = self._runSchedMain()
self.onfini( self._onSchedFini )
def __init__(self, maxsize=10000, onmiss=None):
EventBus.__init__(self)
self.cache = {}
self.onmiss = onmiss
self.maxsize = maxsize
self.cachelock = threading.Lock()
self.fifo = collections.deque()
def __init__(self, core=None, maxtime=onehour):
EventBus.__init__(self)
self.core = core
self.cache = s_cache.Cache(maxtime=maxtime)
self.cache.setOnMiss( self._getSessByIden )
self.cache.on('cache:pop', self._onSessCachePop )
self.onfini( self.cache.fini )
def __init__(self):
#cmd.Cmd.__init__(self)
EventBus.__init__(self)
self.cmdfuncs = {}
self.cmdprompt = 'cli> '
self.addCmdMeths(self)
self.addCmdFunc(cmd_quit, name='quit')
self.addCmdFunc(cmd_help, name='help')
def __init__(self, sock, **info):
EventBus.__init__(self)
self.sock = sock
self.unpk = msgpack.Unpacker(use_list=0,encoding='utf8')
self.ident = s_common.guid()
self.xforms = [] # list of SockXform instances
self.crypto = None
self.sockinfo = info
self.onfini(self._finiSocket)
def __init__(self, statefd=None):
self.authinfo = {}
self.authrules = collections.defaultdict(dict)
EventBus.__init__(self)
StateMachine.__init__(self, statefd=statefd)
if self.getAuthInfo('defauth') == None:
self.setAuthInfo('defauth', False)
self._loadAuthRules()
def __init__(self):
EventBus.__init__(self)
self.lock = threading.Lock()
self.sched = s_sched.Sched()
self.slots = {}
self.hives = {}
self.drones = {}
self.slotq = collections.deque()
def __init__(self):
EventBus.__init__(self)
self.lock = threading.Lock()
self.sched = s_sched.Sched()
self.slots = {}
self.hives = {}
self.drones = {}
self.slotq = collections.deque()
def __init__(self, iden=None):
EventBus.__init__(self)
if iden is None:
iden = s_common.guid()
self.info = {}
self.iden = iden
self.on('task:fini', self._onTaskFini)
def __init__(self, core, maxtime=onehour):
EventBus.__init__(self)
self.core = core
self.model = core.genDataModel()
self.cache = s_cache.Cache(maxtime=maxtime)
self.cache.setOnMiss( self._getSessBySid )
self.cache.on('cache:pop', self._onSessCachePop )
self.onfini( self.cache.fini )
def __init__(self, link):
EventBus.__init__(self)
self.link = link
self.relay = s_link.initLinkRelay(link)
self.client = self._init_client()
self._tele_with = {} # tid:client for with blocks
# objname is path minus leading "/"
self.objname = link[1].get('path')[1:]
def __init__(self, svcbus, **opts):
EventBus.__init__(self)
# a core we use for data model stuff..
self.core = s_cortex.openurl('ram:///')
self.onfini(self.core.fini)
s_storm.Runtime.__init__(self)
self.setConfOpts(opts)
self.svcbus = svcbus
self.svcprox = s_service.SvcProxy(svcbus, self.svctime)
def __init__(self, func, *args, **kwargs):
EventBus.__init__(self)
threading.Thread.__init__(self)
self.setDaemon(True)
self.iden = guid()
self.task = (func,args,kwargs)
self.cancels = []
self.onfini( self._onThrFini )
def __init__(self, func, *args, **kwargs):
EventBus.__init__(self)
threading.Thread.__init__(self)
self.setDaemon(True)
self.iden = s_common.guid()
self.task = (func, args, kwargs)
self.cancels = []
self.onfini(self._onThrFini)
def __init__(self):
EventBus.__init__(self)
self.coreok = {}
self.cordefs = {}
self.tagsbyname = {}
self.coresbyname = {}
self.sched = s_sched.Sched()
self.coresbytag = collections.defaultdict(list)
self.onfini( self._onMetaFini )
def __init__(self):
EventBus.__init__(self)
self.coreok = {}
self.cordefs = {}
self.tagsbyname = {}
self.coresbyname = {}
self.sched = s_sched.Sched()
self.coresbytag = collections.defaultdict(list)
self.onfini(self._onMetaFini)
def __init__(self, relay):
EventBus.__init__(self)
self.relay = relay
self.boss = s_threads.ThreadBoss()
self.onfini(self.boss.fini)
self.socks = {}
# we get the sock first to fill in info
self.on('link:sock:init', self._onLinkSockInit)
self.on('link:sock:fini', self._onLinkSockFini)
self.onfini( self._finiAllSocks )
def __init__(self, fd, **opts):
EventBus.__init__(self)
fd.seek(0, os.SEEK_END)
self.fd = fd
self.size = fd.tell()
self.fdoff = self.size
self.fileno = fd.fileno()
self.lock = threading.Lock()
self.onfini( self._onAtomFini )
def __init__(self, outp=None):
EventBus.__init__(self)
if outp == None:
outp = s_output.OutPut()
self.outp = outp
self.cmds = {}
self.cmdprompt = 'cli> '
self.addCmdClass( CmdHelp )
self.addCmdClass( CmdQuit )
def __init__(self):
EventBus.__init__(self)
self.onfini(self._onBossFini)
self.pool = None
self.joblock = threading.Lock()
self._boss_jobs = {}
self.joblocal = {}
self.on('job:done', self._onJobDone) # trigger job done
self.on('job:fini', self._onJobFini) # job is finished
def __init__(self, statefd=None):
EventBus.__init__(self)
self.authmod = None
self.links = {}
self.mesgmeths = {}
self.boss = s_threads.ThreadBoss()
self.onfini(self.boss.fini)
s_telepath.TeleMixin.__init__(self)
StateMachine.__init__(self, statefd=statefd)
self.on('link:sock:mesg', self._onLinkSockMesg)
def __init__(self, statefd=None):
EventBus.__init__(self)
self.authmod = None
self.links = {}
self.mesgmeths = {}
self.boss = s_threads.ThreadBoss()
self.onfini( self.boss.fini )
s_telepath.TeleMixin.__init__(self)
StateMachine.__init__(self,statefd=statefd)
self.on('link:sock:mesg',self._onLinkSockMesg)
def __init__(self, maxtime=None, onmiss=None):
EventBus.__init__(self)
self.onmiss = onmiss
self.cache = {}
self.lasthit = {}
self.schevt = None
self.maxtime = None
self.cachelock = threading.Lock()
self.onfini(self._onCacheFini)
if maxtime is not None:
self.setMaxTime(maxtime)
def __init__(self, item, outp=None, **locs):
EventBus.__init__(self)
if outp is None:
outp = s_output.OutPut()
self.outp = outp
self.locs = locs
self.item = item # whatever object we are commanding
self.cmds = {}
self.cmdprompt = 'cli> '
self.addCmdClass(CmdHelp)
self.addCmdClass(CmdQuit)
def __init__(self, size=3, maxsize=None):
EventBus.__init__(self)
self.workq = s_queue.Queue()
self._pool_lock = threading.Lock()
self._pool_avail = 0
self._pool_maxsize = maxsize
self._pool_threads = {}
self.onfini( self._onPoolFini )
for i in range(size):
self._fire_thread( self._run_work )
def __init__(self, sock, **info):
EventBus.__init__(self)
self.sock = sock
self.plex = None
self.unpk = msgpack.Unpacker(use_list=0,encoding='utf8')
self.iden = s_common.guid()
self.xforms = [] # list of SockXform instances
self.info = info
# used by Plex() tx
self.txbuf = None
self.txque = collections.deque()
self.onfini(self._finiSocket)
def __init__(self):
EventBus.__init__(self)
self.onfini( self._onBossFini )
self.pool = None
self.sched = s_sched.getGlobSched()
self.joblock = threading.Lock()
self._boss_jobs = {}
self.joblocal = {}
self.on('job:done', self._onJobDone ) # trigger job done
self.on('job:fini', self._onJobFini ) # job is finished
def __init__(self, core=None, maxtime=onehour):
EventBus.__init__(self)
self.core = core
self.cache = s_cache.Cache(maxtime=maxtime)
self.cache.setOnMiss(self._getSessByIden)
self.cache.on('cache:pop', self._onSessCachePop)
self.onfini(self.cache.fini)
if self.core:
pdef = self.core.getPropDef('syn:sess')
if not pdef:
self.core.addTufoForm('syn:sess', ptype='guid')
def __init__(self, pool=None):
EventBus.__init__(self)
if pool is None:
pool = s_threads.Pool()
self.pool = pool
self.root = None
self.lock = threading.Lock()
self.wake = threading.Event()
self.thr = self._runSchedMain()
self.onfini(self._onSchedFini)
def __init__(self, sock, **info):
EventBus.__init__(self)
self.sock = sock
self.plex = None
self.unpk = msgpack.Unpacker(use_list=0, encoding='utf8')
self.iden = s_common.guid()
self.xforms = [] # list of SockXform instances
self.info = info
# used by Plex() tx
self.txbuf = None
self.txque = collections.deque()
self.onfini(self._finiSocket)
def __init__(self, **settings):
EventBus.__init__(self)
s_daemon.DmonConf.__init__(self)
tornado.web.Application.__init__(self, **settings)
self.loop = tornado.ioloop.IOLoop()
self.serv = tornado.httpserver.HTTPServer(self)
self.boss = s_async.Boss()
# FIXME options
self.boss.runBossPool(8, maxsize=128)
self.iothr = self._runWappLoop()
self.onfini( self._onWappFini )
def __init__(self, maxtime=None, onmiss=None):
EventBus.__init__(self)
self.sched = s_sched.getGlobSched()
self.onmiss = onmiss
self.cache = {}
self.lasthit = {}
self.schevt = None
self.maxtime = None
self.cachelock = threading.Lock()
self.onfini( self._onCacheFini )
if maxtime != None:
self.setMaxTime(maxtime)
def __init__(self, **settings):
EventBus.__init__(self)
s_daemon.DmonConf.__init__(self)
tornado.web.Application.__init__(self, **settings)
self.loop = tornado.ioloop.IOLoop.current()
self.serv = tornado.httpserver.HTTPServer(self)
self.boss = s_async.Boss()
# FIXME options
self.boss.runBossPool(8, maxsize=128)
self.iothr = self._runWappLoop()
self.onfini( self._onWappFini )
def __init__(self, maxtime=None, onmiss=None):
EventBus.__init__(self)
self.sched = s_sched.getGlobSched()
self.onmiss = onmiss
self.cache = {}
self.lasthit = {}
self.cachelock = threading.Lock()
self.onfini(self._onCacheFini)
self.schevt = None
self.maxtime = maxtime
if self.maxtime != None:
self._checkCacheTimes()
def __init__(self, relay):
EventBus.__init__(self)
self.lock = threading.Lock()
self.relay = relay
self.trans = relay.link[1].get('trans')
self.onfini(self._finiLinkClient)
self.sock = relay.initClientSock()
if self.sock == None:
raise Exception('Initial Link Failed: %r' % (self.link,))
if self.trans:
self.sock.fini()
def __init__(self, **settings):
EventBus.__init__(self)
self.onfini( self._onDavFini )
self.root = PathNode()
self.paths = s_cache.Cache(maxtime=60)
self.paths.setOnMiss( self._getDavNode )
self.app = tornado.web.Application( (
('.*', WebDavHandler, {'webdav': self}),
))
self.ioloop = tornado.ioloop.IOLoop()
settings['io_loop'] = self.ioloop
self.serv = tornado.httpserver.HTTPServer(self.app, **settings)
def __init__(self, **settings):
EventBus.__init__(self)
self.onfini(self._onDavFini)
self.root = PathNode()
self.paths = s_cache.Cache(maxtime=60)
self.paths.setOnMiss(self._getDavNode)
self.app = tornado.web.Application((('.*', WebDavHandler, {
'webdav': self
}), ))
self.ioloop = tornado.ioloop.IOLoop()
settings['io_loop'] = self.ioloop
self.serv = tornado.httpserver.HTTPServer(self.app, **settings)
def __init__(self, maxtime=None):
EventBus.__init__(self)
self.sched = s_sched.getGlobSched()
self.onmiss = None
self.cache = {}
self.lasthit = {}
self.cachelock = threading.Lock()
self.onfini( self._onCacheFini )
self.schevt = None
self.maxtime = maxtime
if self.maxtime != None:
self._checkCacheTimes()
def __init__(self, sock, **info):
EventBus.__init__(self)
self.sock = sock # type: socket.socket
self.unpk = msgpack.Unpacker(use_list=0, encoding='utf8')
self.iden = s_common.guid()
self.info = info
self.blocking = True # sockets are blocking by default
if self.info.get('nodelay', True):
self._tryTcpNoDelay()
self.txbuf = None # the remainder of a partially sent byts
self.txque = collections.deque()
self.rxque = collections.deque()
self.onfini(self._finiSocket)
def __init__(self, pool=None):
EventBus.__init__(self)
DmonConf.__init__(self)
self.auth = None
self.socks = {} # sockets by iden
self.shared = {} # objects provided by daemon
self.pushed = {} # objects provided by sockets
self.csides = {} # item:[ (name,path), ... ]
self.reflect = {} # objects reflect info by name
self._dmon_ons = {}
self._dmon_links = [] # list of listen links
self._dmon_yields = set()
if pool is None:
pool = s_threads.Pool(size=8, maxsize=-1)
self.pool = pool
self.plex = s_socket.Plex()
self.cura = s_session.Curator()
self.onfini(self.plex.fini)
self.onfini(self.pool.fini)
self.onfini(self.cura.fini)
self.on('link:sock:init', self._onLinkSockInit)
self.plex.on('link:sock:mesg', self._onLinkSockMesg)
self.mesgfuncs = {}
self.setMesgFunc('tele:syn', self._onTeleSynMesg)
self.setMesgFunc('sock:gzip', self._onSockGzipMesg)
self.setMesgFunc('tele:call', self._onTeleCallMesg)
# for "client shared" objects...
self.setMesgFunc('tele:push', self._onTelePushMesg)
self.setMesgFunc('tele:retn', self._onTeleRetnMesg)
self.setMesgFunc('tele:on', self._onTeleOnMesg)
self.setMesgFunc('tele:off', self._onTeleOffMesg)
self.setMesgFunc('tele:yield:fini', self._onTeleYieldFini)
def __init__(self, core=None, pool=None):
EventBus.__init__(self)
DmonConf.__init__(self)
if core == None:
core = s_cortex.openurl('ram:///')
self.socks = {} # sockets by iden
self.shared = {} # objects provided by daemon
self.pushed = {} # objects provided by sockets
self._dmon_links = [] # list of listen links
self._dmon_yields = set()
if pool == None:
pool = s_threads.Pool(size=8, maxsize=-1)
self.pki = s_pki.PkiStor(core)
self.pool = pool
self.core = core
self.plex = s_socket.Plex()
self.onfini(self.plex.fini)
self.onfini(self.pool.fini)
self.on('link:sock:init', self._onLinkSockInit)
self.plex.on('link:sock:mesg', self._onLinkSockMesg)
self.mesgfuncs = {}
self.setMesgFunc('tele:syn', self._onTeleSynMesg)
self.setMesgFunc('tele:skey', self._onTeleSkeyMesg)
self.setMesgFunc('tele:call', self._onTeleCallMesg)
# for "client shared" objects...
self.setMesgFunc('tele:push', self._onTelePushMesg)
self.setMesgFunc('tele:retn', self._onTeleRetnMesg)
self.setMesgFunc('tele:on', self._onTeleOnMesg)
self.setMesgFunc('tele:off', self._onTeleOffMesg)
self.setMesgFunc('tele:yield:fini', self._onTeleYieldFini)
def __init__(self, core):
EventBus.__init__(self)
self.core = core
self.core.addTufoForm('syn:auth:user')
self.core.addTufoProp('syn:auth:user', 'apikey', defval='')
self.core.addTufoProp('syn:auth:user', 'shadow:sha256', defval='')
self.core.addTufoForm('syn:auth:role')
self.core.addTufoForm('syn:auth:userrole')
self.core.addTufoProp('syn:auth:userrole', 'user')
self.core.addTufoProp('syn:auth:userrole', 'role')
self.users = s_cache.TufoPropCache(core, 'syn:auth:user')
self.roles = s_cache.TufoPropCache(core, 'syn:auth:role')
self.rules = s_cache.KeyCache(self._getUserRulesCache)
def __init__(self, core):
EventBus.__init__(self)
self.core = core
self.core.addTufoForm('syn:auth:user')
self.core.addTufoProp('syn:auth:user','apikey', defval='')
self.core.addTufoProp('syn:auth:user','shadow:sha256', defval='')
self.core.addTufoForm('syn:auth:role')
self.core.addTufoForm('syn:auth:userrole')
self.core.addTufoProp('syn:auth:userrole','user')
self.core.addTufoProp('syn:auth:userrole','role')
self.users = s_cache.TufoPropCache(core,'syn:auth:user')
self.roles = s_cache.TufoPropCache(core,'syn:auth:role')
self.rules = s_cache.KeyCache(self._getUserRulesCache)
def __init__(self, size=3, maxsize=None):
EventBus.__init__(self)
self.workq = s_queue.Queue()
self._pool_lock = threading.Lock()
self._pool_avail = 0
if maxsize is None:
maxsize = size
self._pool_maxsize = maxsize
self._pool_threads = {}
self.onfini(self._onPoolFini)
for i in range(size):
self._fire_thread(self._run_work)
def __init__(self, core):
EventBus.__init__(self)
self.core = core
self.keys = c_cache.Cache()
self.keys.setOnMiss(self._getRsaKey)
self.pubs = c_cache.Cache()
self.pubs.setOnMiss(self._getPubKey)
self.certs = c_cache.Cache()
self.certs.setOnMiss(self._getTokenCert)
self.tokens = c_cache.Cache()
self.tokens.setOnMiss(self._getTokenTufo)
core.onfini(self.keys.fini)
core.onfini(self.pubs.fini)
core.onfini(self.certs.fini)
core.onfini(self.tokens.fini)
model = core.genDataModel()
model.addTufoForm('syn:token',
ptype='str',
doc='synapse identity token (user/host)')
model.addTufoProp('syn:token',
'user',
doc='humon readable user name for this token')
model.addTufoProp('syn:token',
'host',
doc='humon readable host name for this token')
model.addTufoProp('syn:token', 'blob', doc='Base64 encoded token blob')
model.addTufoProp('syn:token',
'cert',
doc='Base64 encoded certificate blob')
model.addTufoProp('syn:token',
'rsakey',
doc='base64( der( rsa.private ) )')
def __init__(self, sock, **info):
EventBus.__init__(self)
self.sock = sock # type: socket.socket
self.plex = None
self.unpk = msgpack.Unpacker(use_list=0, encoding='utf8')
self.iden = s_common.guid()
self.xforms = [] # list of SockXform instances
self.info = info
# used by Plex() tx
self.txbuf = None
self.txsize = 0
if self.info.get('nodelay', True):
self._tryTcpNoDelay()
self.txque = collections.deque()
self.rxque = collections.deque()
self.onfini(self._finiSocket)
def __init__(self):
EventBus.__init__(self)
#self._plex_sel = selectors.DefaultSelector()
self._plex_lock = threading.Lock()
self._plex_socks = {}
# used for select()
self._plex_rxsocks = []
self._plex_txsocks = []
self._plex_xxsocks = []
self._plex_wake, self._plex_s2 = socketpair()
self._plex_s2.set('wake', True)
self.addPlexSock(self._plex_s2)
self._plex_thr = self._plexMainLoop()
self.onfini(self._onPlexFini)
def __init__(self, link):
EventBus.__init__(self)
self.link = link
ropool = int(link[1].get('ropool', 3))
self.lock = threading.Lock()
self.sizebymeths = {}
self.rowsbymeths = {}
# used to facilitate pooled async readers
self.jobboss = s_async.AsyncBoss(pool=ropool)
self.jobcache = {}
self._initCortex()
# used exclusivly for single-thread inserts
self.boss = s_async.AsyncBoss(pool=1)
self. async = s_async.AsyncApi(self.boss, self)
self.isok = True
self.onfini(self.boss.fini)
def __init__(self, relay):
EventBus.__init__(self)
self.relay = relay
self.boss = s_threads.ThreadBoss()
self.onfini(self.boss.fini)
def __init__(self):
EventBus.__init__(self)
self.vals = {}
self.lock = threading.Lock()
self.refs = collections.defaultdict(int)
