def __init__(self, auth):
'''
A mixin that can be used to provide a helper for remote access to an
Auth object. The API endpoint ``authReact()`` can be used to manipulate
the Auth object by only allowing admin users to perform actions.
Args:
auth (Auth): An auth instance. This is set to ``self.auth``.
'''
self.auth = auth
self._mxrtor = s_react.Reactor()
self._mxrtor.act('auth:get:users', self.__authGetUsers)
self._mxrtor.act('auth:get:roles', self.__authGetRoles)
# User actions
self._mxrtor.act('auth:add:user', self.__authAddUser)
self._mxrtor.act('auth:del:user', self.__authDelUser)
self._mxrtor.act('auth:req:user', self.__authReqUser)
self._mxrtor.act('auth:add:urole', self.__authAddUserRole)
self._mxrtor.act('auth:add:urule', self.__authAddUserRule)
self._mxrtor.act('auth:del:urole', self.__authDelUserRole)
self._mxrtor.act('auth:del:urule', self.__authDelUserRule)
# User admin actions
self._mxrtor.act('auth:add:admin', self.__authAddAdmin)
self._mxrtor.act('auth:del:admin', self.__authDelAdmin)
# Role actions
self._mxrtor.act('auth:req:role', self.__authReqRole)
self._mxrtor.act('auth:add:role', self.__authAddRole)
self._mxrtor.act('auth:del:role', self.__authDelRole)
self._mxrtor.act('auth:add:rrule', self.__authAddRoleRule)
self._mxrtor.act('auth:del:rrule', self.__authDelRoleRule)
def __init__(self, cli, **opts):
s_cli.Cmd.__init__(self, cli, **opts)
self.reac = s_reactor.Reactor()
self.reac.act('node', self._onNode)
self.reac.act('init', self._onInit)
self.reac.act('fini', self._onFini)
self.reac.act('print', self._onPrint)
self.reac.act('warn', self._onWarn)
def test_reactor_base(self):
reac = s_reactor.Reactor()
def actfoo(mesg):
x = mesg[1].get('x')
y = mesg[1].get('y')
return x + y
reac.act('foo', actfoo)
self.eq(reac.react(tufo('foo', x=10, y=20)), 30)
def test_reactor(self):
reac = s_reactor.Reactor()
def actfoo(mesg):
x = mesg[1].get('x')
y = mesg[1].get('y')
return x + y
reac.act('foo', actfoo)
data = ('foo', {'x': 10, 'y': 20})
self.eq(reac.react(data), 30)
self.raises(s_exc.NoSuchAct, reac.react, ('wat', {}))
def __init__(self, fd, **opts):
s_eventbus.EventBus.__init__(self)
self.alloclock = threading.Lock()
self.on('heap:write', self._fireHeapSync)
self.on('heap:resize', self._fireHeapSync)
self.syncact = s_reactor.Reactor()
self.syncact.act('heap:write', self._actSyncHeapWrite)
self.syncact.act('heap:resize', self._actSyncHeapResize)
self.pagesize = opts.get('pagesize', defpage)
pagerem = self.pagesize % mmap.ALLOCATIONGRANULARITY
if pagerem:
self.pagesize += (mmap.ALLOCATIONGRANULARITY - pagerem)
fd.seek(0, os.SEEK_END)
size = fd.tell()
if size == 0:
size = 32 # a few qword slots for expansion
used = headsize + size
heaphead = self._genHeapHead(size) + s_compat.to_bytes(used, 8)
rem = len(heaphead) % self.pagesize
if rem:
heaphead += b'\x00' * (self.pagesize - rem)
fd.write(heaphead)
fd.flush()
self.fd = fd
self.atom = opts.get('atom')
if self.atom is None:
self.atom = s_atomfile.getAtomFile(fd)
self.used = s_compat.to_int(self.readoff(32, 8))
self.onfini(self.atom.fini)
def __init__(self, axondir, **opts):
s_eventbus.EventBus.__init__(self)
self.inprog = {}
self.axondir = gendir(axondir)
self.clonedir = gendir(axondir, 'clones')
self.clones = {}
self.cloneinfo = {}
self.clonehosts = set()
self.clonelock = threading.Lock()
self.readyclones = set() # iden of each clone added as it comes up
self.clonesready = threading.Event(
) # set once all clones are up and running
self.opts = opts
self.axonbus = None
self.iden = self.opts.get('iden')
self.tags = self.opts.get('tags', ())
self.opts.setdefault('ro', False)
self.opts.setdefault('clone', '') # are we a clone?
self.opts.setdefault('clones', 2) # how many clones do we want?
self.opts.setdefault('axonbus', '') # do we have an axon svcbus?
self.opts.setdefault('hostname', s_thishost.get('hostname'))
self.opts.setdefault(
'listen',
'tcp://0.0.0.0:0/axon') # our default "ephemeral" listener
# if we're a clone, we're read-only and have no clones
if self.opts.get('clone'):
self.opts['ro'] = True
self.opts['clones'] = 0
self.opts.setdefault('synckeep', threedays)
self.opts.setdefault('syncsize', gigabyte * 10)
corepath = os.path.join(self.axondir, 'axon.db')
self.core = s_cortex.openurl('sqlite:///%s' % corepath)
fd = genfile(axondir, 'axon.heap')
self.link = None
self.heap = s_heap.Heap(fd)
self.dmon = s_daemon.Daemon()
lisn = self.opts.get('listen')
if lisn:
self.link = self.dmon.listen(lisn)
self.axfo = (self.iden, {})
self.axthrs = set()
self.setAxonInfo('link', self.link)
self.setAxonInfo('opts', self.opts)
self.dmon.share('axon', self)
# create a reactor to unwrap core/heap sync events
self.syncact = s_reactor.Reactor()
self.syncact.act('heap:sync', self.heap.sync)
self.syncact.act('core:sync', self.core.sync)
# wrap core/heap sync events as axon:sync events
self.core.on('core:sync', self._fireAxonSync)
self.heap.on('heap:sync', self._fireAxonSync)
# model details for the actual byte blobs
self.core.addTufoForm('axon:blob', ptype='guid')
self.core.addTufoProp('axon:blob', 'off', ptype='int', req=True)
self.core.addTufoProp('axon:blob', 'size', ptype='int', req=True)
self.core.addTufoProp('axon:blob', 'md5', ptype='hash:md5', req=True)
self.core.addTufoProp('axon:blob', 'sha1', ptype='hash:sha1', req=True)
self.core.addTufoProp('axon:blob',
'sha256',
ptype='hash:sha256',
req=True)
self.core.addTufoProp('axon:blob',
'sha512',
ptype='hash:sha512',
req=True)
self.core.addTufoForm('axon:clone', ptype='guid')
dirname = gendir(axondir, 'sync')
syncopts = self.opts.get('syncopts', {})
self.syncdir = None
self.onfini(self._onAxonFini)
self.onfini(self.core.fini)
self.onfini(self.heap.fini)
self.onfini(self.dmon.fini)
# if we're not a clone, create a sync dir
if not self.opts.get('clone'):
self.syncdir = s_persist.Dir(dirname, **syncopts)
self.onfini(self.syncdir.fini)
self.on('axon:sync', self.syncdir.add)
self.axcthr = None
# share last to avoid startup races
busurl = self.opts.get('axonbus')
if busurl:
self.axonbus = s_service.openurl(busurl)
props = {'link': self.link, 'tags': self.tags}
self.axonbus.runSynSvc(self.iden, self, **props)
self.axcthr = self._fireAxonClones()
def __init__(self, link):
EventBus.__init__(self)
DataModel.__init__(self)
self._link = link
#self.lock = threading.RLock()
self.lock = threading.Lock()
self.inclock = threading.Lock()
self.statfuncs = {}
self.auth = None
self.tagcache = {}
self.splicefuncs = {}
self.sizebymeths = {}
self.rowsbymeths = {}
#############################################################
# buses to save/load *raw* save events
#############################################################
self.savebus = EventBus()
self.loadbus = EventBus()
self.loadbus.on('core:save:add:rows', self._loadAddRows)
self.loadbus.on('core:save:del:rows:by:iden', self._loadDelRowsById)
self.loadbus.on('core:save:del:rows:by:prop', self._loadDelRowsByProp)
self.loadbus.on('core:save:set:rows:by:idprop',
self._loadSetRowsByIdProp)
self.loadbus.on('core:save:del:rows:by:idprop',
self._loadDelRowsByIdProp)
#############################################################
# bus for model layer sync
# sync events are fired on the cortex and may be ingested
# into another coretx using the sync() method.
#############################################################
self.on('tufo:add', self._fireCoreSync)
self.on('tufo:del', self._fireCoreSync)
self.on('tufo:set', self._fireCoreSync)
self.on('tufo:tag:add', self._fireCoreSync)
self.on('tufo:tag:del', self._fireCoreSync)
self.syncact = s_reactor.Reactor()
self.syncact.act('tufo:add', self._actSyncTufoAdd)
self.syncact.act('tufo:del', self._actSyncTufoDel)
#self.syncact.act('tufo:set', self._actSyncTufoSet )
self.syncact.act('tufo:tag:add', self._actSyncTufoTagAdd)
self.syncact.act('tufo:tag:del', self._actSyncTufoTagDel)
#############################################################
self.onfini(self.savebus.fini)
self.onfini(self.loadbus.fini)
self.addStatFunc('any', self._calcStatAny)
self.addStatFunc('all', self._calcStatAll)
self.addStatFunc('min', self._calcStatMin)
self.addStatFunc('max', self._calcStatMax)
self.addStatFunc('sum', self._calcStatSum)
self.addStatFunc('count', self._calcStatCount)
self.addStatFunc('histo', self._calcStatHisto)
self.addStatFunc('average', self._calcStatAverage)
self._initCortex()
# FIXME unicode / "word" characters
#self.addSubType('syn:tag','str', regex='^[a-z0-9._]+$', lower=1)
#self.addSubType('syn:prop','str', regex='^[a-z0-9:_]+$', lower=1)
#self.addSubType('syn:type','str', regex='^[a-z0-9:_]+$', lower=1)
self.addTufoForm('syn:tag', ptype='syn:tag')
self.addTufoProp('syn:tag', 'up', ptype='syn:tag')
self.addTufoProp('syn:tag', 'doc', defval='', ptype='str')
self.addTufoProp('syn:tag', 'depth', defval=0, ptype='int')
self.addTufoProp('syn:tag', 'title', defval='', ptype='str')
#self.model.addTufoForm('syn:model',ptype='str')
self.addTufoForm('syn:type', ptype='syn:type')
self.addTufoProp('syn:type',
'doc',
ptype='str',
defval='??',
doc='Description for this type')
self.addTufoProp('syn:type',
'ver',
ptype='int',
defval=1,
doc='What version is this type')
self.addTufoProp('syn:type',
'base',
ptype='str',
doc='what type does this type extend?',
req=True)
self.addTufoGlob('syn:type', 'info:*')
self.addTufoForm('syn:form', ptype='syn:prop')
self.addTufoProp('syn:form', 'doc', ptype='str')
self.addTufoForm('syn:prop', ptype='syn:prop')
self.addTufoProp('syn:prop', 'doc', ptype='str')
self.addTufoProp('syn:prop', 'form', ptype='syn:prop')
self.addTufoProp('syn:prop', 'ptype', ptype='syn:type')
#forms = self.getTufosByProp('syn:form')
#self.addTufoProp('syn:prop','ptype',ptype='str')
#self.addTufoProp('syn:prop','title',ptype='str')
#self.addTufoProp('syn:prop','defval') # ptype='any'
#self.addTufoForm('syn:splice',ptype='guid')
#self.addTufoProp('syn:splice','date',ptype='time:epoch',doc='Time that the splice was requested')
#self.addTufoProp('syn:splice','user',ptype='str',doc='Time user/system which requested the splice')
#self.addTufoProp('syn:splice','note',ptype='str',doc='Filthy humon notes about the change')
#self.addTufoProp('syn:splice','status',ptype='str',doc='Enum for init,done,deny to show the splice status')
#self.addTufoProp('syn:splice','action',ptype='str:lwr',doc='The requested splice action')
# FIXME load forms / props / etc
self.addTufoForm('syn:splice', ptype='guid')
self.addTufoGlob('syn:splice', 'on:*') # syn:splice:on:fqdn=woot.com
self.addTufoGlob('syn:splice', 'act:*') # action arguments
self.addTufoProp('syn:splice',
'perm',
ptype='str',
doc='Permissions str for glob matching')
self.addTufoProp('syn:splice',
'reqtime',
ptype='time:epoch',
doc='When was the splice requested')
self.addTufoProp('syn:splice',
'user',
ptype='str',
defval='??',
doc='What user is requesting the splice')
self.addTufoProp('syn:splice',
'note',
ptype='str',
defval='',
doc='Notes about the splice')
self.addTufoProp('syn:splice',
'status',
ptype='str',
defval='new',
doc='Splice status')
self.addTufoProp('syn:splice',
'action',
ptype='str',
doc='What action is the splice requesting')
#self.addTufoProp('syn:splice','actuser', ptype='str', doc='What user is activating the splice')
#self.addTufoProp('syn:splice','acttime', ptype='time:epoch', doc='When was the splice activated')
self.on('tufo:add:syn:tag', self._onAddSynTag)
self.on('tufo:add:syn:type', self._onAddSynType)
self.on('tufo:add:syn:prop', self._onAddSynProp)
#self.on('tufo:add:syn:type', self._onAddSynType)
#self.on('tufo:add:syn:form', self._onAddSynForm)
#self.on('tufo:add:syn:prop', self._onAddSynProp)
self.on('tufo:del:syn:tag', self._onDelSynTag)
self.on('tufo:form:syn:tag', self._onFormSynTag)
self.isok = True
self.splicers = {}
self.splicers['tufo:add'] = self._spliceTufoAdd
self.splicers['tufo:del'] = self._spliceTufoDel
self.splicers['tufo:set'] = self._spliceTufoSet
self.splicers['tufo:tag:add'] = self._spliceTufoTagAdd
self.splicers['tufo:tag:del'] = self._spliceTufoTagDel
def __init__(self, axondir, **opts):
s_config.Config.__init__(self)
self.inprog = {}
self.axondir = s_common.gendir(axondir)
self.clones = {}
self.cloneinfo = {}
self.clonehosts = set()
self.clonelock = threading.Lock()
self.readyclones = set() # iden of each clone added as it comes up
self.clonesready = threading.Event(
) # set once all clones are up and running
self.axonbus = None
self.setConfOpts(opts)
self.iden = self.getConfOpt('axon:iden')
self.tags = self.getConfOpt('axon:tags')
# if we're a clone, we're read-only and have no clones
if self.getConfOpt('axon:clone'):
self.setConfOpt('axon:ro', 1)
self.setConfOpt('axon:clones', 0)
corepath = os.path.join(self.axondir, 'axon.db')
self.core = s_cortex.openurl('sqlite:///%s' % corepath)
self.core.setConfOpt('modules',
(('synapse.models.axon.AxonMod', {}), ))
self.core.setConfOpt('caching', 1)
self._fs_mkdir_root() # create the fs root
self.flock = threading.Lock()
fd = s_common.genfile(axondir, 'axon.heap')
self.link = None
self.heap = s_heap.Heap(fd)
self.dmon = s_daemon.Daemon()
lisn = self.getConfOpt('axon:listen')
if lisn:
self.link = self.dmon.listen(lisn)
self.axfo = (self.iden, {})
self.axthrs = set()
self.setAxonInfo('link', self.link)
self.setAxonInfo('opts', self.getConfOpts())
self.on('syn:conf:set', self._onSetConfigableValu)
self.dmon.share('axon', self)
# create a reactor to unwrap core/heap sync events
self.syncact = s_reactor.Reactor()
self.syncact.act('splice', self.core.splice)
self.syncact.act('heap:sync', self.heap.sync)
# wrap core/heap sync events as axon:sync events
self.core.on('splice', self._fireAxonSync)
self.heap.on('heap:sync', self._fireAxonSync)
self.syncdir = None
self.onfini(self._onAxonFini)
self.onfini(self.core.fini)
self.onfini(self.heap.fini)
self.onfini(self.dmon.fini)
# if we're not a clone, create a sync dir
if not self.getConfOpt('axon:clone'):
dirname = s_common.gendir(axondir, 'sync')
syncopts = self.getConfOpt('axon:syncopts')
self.syncdir = s_persist.Dir(dirname, **syncopts)
self.onfini(self.syncdir.fini)
self.on('axon:sync', self.syncdir.add)
self.axcthr = None
# share last to avoid startup races
busurl = self.getConfOpt('axon:axonbus')
if busurl:
self.axonbus = s_service.openurl(busurl)
self.onfini(self.axonbus.fini)
props = {'link': self.link, 'tags': self.tags}
self.axonbus.runSynSvc(self.iden, self, **props)
self.axcthr = self._fireAxonClones()
import synapse.exc as s_exc
import synapse.common as s_common
import synapse.reactor as s_reactor
import synapse.telepath as s_telepath
import synapse.lib.cli as s_cli
import synapse.lib.node as s_node
import synapse.lib.time as s_time
import synapse.lib.queue as s_queue
import synapse.lib.msgpack as s_msgpack
logger = logging.getLogger(__name__)
reac = s_reactor.Reactor()
def _reacJsonl(mesg):
s = json.dumps(mesg, sort_keys=True) + '\n'
buf = s.encode()
return buf
def _reacMpk(mesg):
buf = s_msgpack.en(mesg)
return buf
reac.act('mpk', _reacMpk)
reac.act('jsonl', _reacJsonl)
class Log(s_cli.Cmd):
