def postTypeInit(self):
self._size = self.opts.get('size')
if self._size < 0:
# zero means no width check
raise s_exc.BadConfValu(name='size', valu=self._size,
mesg='Size must be > 0')
if self._size % 2 != 0:
raise s_exc.BadConfValu(name='size', valu=self._size,
mesg='Size must be a multiple of 2')
self.setNormFunc(str, self._normPyStr)
self.setNormFunc(bytes, self._normPyBytes)
async def __anit__(self, rstfile):
await s_base.Base.__anit__(self)
self.rstfile = s_common.genpath(rstfile)
if not os.path.isfile(rstfile):
raise s_exc.BadConfValu(mesg='A valid rstfile must be specified',
rstfile=self.rstfile)
self.linesout = []
self.context = {}
self.stormvars = {}
self.core = None
self.handlers = {
'storm': self._handleStorm,
'storm-cli': self._handleStormCli,
'storm-pkg': self._handleStormPkg,
'storm-pre': self._handleStormPre,
'storm-svc': self._handleStormSvc,
'storm-fail': self._handleStormFail,
'storm-opts': self._handleStormOpts,
'storm-cortex': self._handleStormCortex,
'storm-envvar': self._handleStormEnvVar,
'storm-expect': self._handleStormExpect,
'storm-multiline': self._handleStormMultiline,
'storm-mock-http': self._handleStormMockHttp,
'storm-vcr-opts': self._handleStormVcrOpts,
'storm-clear-http': self._handleStormClearHttp,
'storm-vcr-callback': self._handleStormVcrCallback,
}
async def setLeader(self, url: Optional[str], iden: str) -> None:
'''
Args:
url: if None, sets this nexsroot as leader, otherwise the telepath URL of the leader (must be a Cell)
iden: iden of the leader. Should be the same as my containing cell's iden
'''
if url is not None and not self.donexslog:
raise s_exc.BadConfValu(mesg='Mirroring incompatible without nexslog:en')
former = self._ldrurl
if former == url:
return
self._ldrurl = url
if self._looptask is not None:
self._looptask.cancel()
self._looptask = None
self._ldrready.clear()
if self._ldr is not None:
await self._ldr.fini()
self._ldr = None
await self._dostatechange()
if self._ldrurl is None:
return
self._looptask = self.schedCoro(self._followerLoop(iden))
async def _initCoreLayers(self):
import synapse.cells as s_cells # avoid import cycle
layersdir = pathlib.Path(self.dirn, 'layers')
layersdir.mkdir(exist_ok=True)
# Layers are imported in reverse lexicographic order, where the earliest in the alphabet is the 'topmost'
# write layer.
layerdirs = sorted((d for d in layersdir.iterdir() if d.is_dir()), reverse=True)
for layeridx, layerdir in enumerate(layerdirs):
logger.info('loading external layer from %s', layerdir)
if not pathlib.Path(layerdir, 'boot.yaml').exists(): # pragma: no cover
logger.warning('Skipping layer directory %s due to missing boot.yaml', layerdir)
continue
# Every layer but the top is readonly
readonly = (layeridx < len(layerdirs) - 1)
layer = await s_cells.initFromDirn(layerdir, readonly=readonly)
if not isinstance(layer, s_layer.Layer):
raise s_exc.BadConfValu('layer dir %s must contain Layer cell', layerdir)
self.layers.append(layer)
if not self.layers:
# Setup the fallback/default single LMDB layer
self.layers.append(await self._makeDefaultLayer())
self.layer = self.layers[-1]
logger.debug('Cortex using the following layers: %s\n', (''.join(f'\n {l.dirn}' for l in self.layers)))
async def promote(self):
client = self.client
if client is None:
mesg = 'promote() called on non-mirror nexsroot'
raise s_exc.BadConfValu(mesg=mesg)
await self.startup(None)
async def setFeedOffs(self, iden, offs):
if offs < 0:
raise s_exc.BadConfValu(mesg='Offset must be greater than or equal to zero.', offs=offs,
iden=iden)
oldoffs = await self.getFeedOffs(iden)
logger.info('Setting Feed offset for [%s] from [%s] to [%s]',
iden, oldoffs, offs)
return await self.layer.setOffset(iden, offs)
def _validateConfig(core, config):
maxsize = config.get('maxsize', 10000)
if not isinstance(maxsize, int):
mesg = f'STIX Bundle config maxsize option must be an integer.'
raise s_exc.BadConfValu(mesg=mesg)
if maxsize > 10000:
mesg = f'STIX Bundle config maxsize option must be <= 10000.'
raise s_exc.BadConfValu(mesg=mesg)
formmaps = config.get('forms')
if formmaps is None:
mesg = f'STIX Bundle config is missing "forms" mappings.'
raise s_exc.NeedConfValu(mesg=mesg)
for formname, formconf in formmaps.items():
form = core.model.form(formname)
if form is None:
mesg = f'STIX Bundle config contains invalid form name {formname}.'
raise s_exc.NoSuchForm(mesg=mesg)
stixdef = formconf.get('default')
if stixdef is None:
mesg = f'STIX Bundle config is missing default mapping for form {formname}.'
raise s_exc.NeedConfValu(mesg=mesg)
if stixdef not in stix_all:
mesg = f'STIX Bundle default mapping ({stixdef}) for {formname} is not a STIX type.'
raise s_exc.BadConfValu(mesg=mesg)
stixmaps = formconf.get('stix')
if stixmaps is None:
mesg = f'STIX Bundle config is missing STIX maps for form {formname}.'
raise s_exc.NeedConfValu(mesg=mesg)
if stixmaps.get(stixdef) is None:
mesg = f'STIX Bundle config is missing STIX map for form {formname} default value {stixdef}.'
raise s_exc.BadConfValu(mesg=mesg)
for stixtype, stixinfo in stixmaps.items():
if stixtype not in stix_all:
mesg = f'STIX Bundle config has unknown STIX type {stixtype} for form {formname}.'
raise s_exc.BadConfValu(mesg=mesg)
stixprops = stixinfo.get('props')
if stixprops is not None:
for stixprop, stormtext in stixprops.items():
if not isinstance(stormtext, str):
mesg = f'STIX Bundle config has invalid prop entry {formname} {stixtype} {stixprop}.'
raise s_exc.BadConfValu(mesg=mesg)
stixrels = stixinfo.get('rels')
if stixrels is not None:
for stixrel in stixrels:
if len(stixrel) != 3:
mesg = f'STIX Bundle config has invalid rel entry {formname} {stixtype} {stixrel}.'
raise s_exc.BadConfValu(mesg=mesg)
async def initFromDirn(dirn, *args, **kwargs):
'''
As above, but retrieves type from boot.yaml in dirn
'''
conf = s_common.yamlload(dirn, 'boot.yaml') or {}
kind = conf.get('type')
if type is None:
raise s_exc.BadConfValu('boot.yaml missing type key')
return await init(kind, dirn, *args, **kwargs)
async def promote(self):
'''
Transform this cell from a passive follower to
an active cell that writes changes locally.
'''
if self.conf.get('mirror') is None:
mesg = 'promote() called on non-mirror'
raise s_exc.BadConfValu(mesg=mesg)
await self.nexsroot.promote()
await self.setCellActive(True)
async def addCronJob(self, query, reqs, incunit=None, incval=1):
'''
Add a cron job to the cortex
A cron job is a persistently-stored item that causes storm queries to be run in the future. The specification
for the times that the queries run can be one-shot or recurring.
Args:
query (str): The storm query to execute in the future
reqs (Union[Dict[str, Union[int, List[int]]], List[Dict[...]]]):
Either a dict of the fixed time fields or a list of such dicts. The keys are in the set ('year',
'month', 'dayofmonth', 'dayofweek', 'hour', 'minute'. The values must be positive integers, except for
the key of 'dayofmonth' in which it may also be a negative integer which represents the number of days
from the end of the month with -1 representing the last day of the month. All values may also be lists
of valid values.
incunit (Optional[str]):
A member of the same set as above, with an additional member 'day'. If is None (default), then the
appointment is one-shot and will not recur.
incval (Union[int, List[int]):
A integer or a list of integers of the number of units
Returns (bytes):
An iden that can be used to later modify, query, and delete the job.
Notes:
reqs must have fields present or incunit must not be None (or both)
The incunit if not None it must be larger in unit size than all the keys in all reqs elements.
'''
def _convert_reqdict(reqdict):
return {s_agenda.TimeUnit.fromString(k): v for (k, v) in reqdict.items()}
try:
if incunit is not None:
if isinstance(incunit, (list, tuple)):
incunit = [s_agenda.TimeUnit.fromString(i) for i in incunit]
else:
incunit = s_agenda.TimeUnit.fromString(incunit)
if isinstance(reqs, Mapping):
newreqs = _convert_reqdict(reqs)
else:
newreqs = [_convert_reqdict(req) for req in reqs]
except KeyError:
raise s_exc.BadConfValu('Unrecognized time unit')
username = None if self.user is None else self.user.name
return await self.cell.agenda.add(username, query, newreqs, incunit, incval)
def reqConfValid(self):
'''
Validate that the loaded configuration data is valid according to the schema.
Notes:
The validation set does set any default values which are not currently
set for configuration options.
Returns:
None: This returns nothing.
'''
try:
self.validator(self.conf)
except s_exc.SchemaViolation as e:
logger.exception('Configuration is invalid.')
raise s_exc.BadConfValu(mesg=f'Invalid configuration found: [{str(e)}]') from None
else:
return
async def __anit__(self, dirn, readonly=False, teleurl=None):
await s_layer.Layer.__anit__(self, dirn, readonly=readonly)
# a remote layer may never be revd
self.canrev = False
if teleurl is None:
teleurl = self.conf.get('remote:telepath')
self.teleurl = teleurl
#self.path = self.conf.get('remote:telepath')
if self.teleurl is None:
raise s_exc.BadConfValu(
mesg='remote:telepath must be url for remote layer')
self.remote = await s_telepath.openurl(self.teleurl)
self.onfini(self.remote.fini)
for funcname in PASSTHROUGHFUNCS:
setattr(self, funcname, getattr(self.remote, funcname))
async def __anit__(self, rstfile):
await s_base.Base.__anit__(self)
self.rstfile = s_common.genpath(rstfile)
if not os.path.isfile(rstfile):
raise s_exc.BadConfValu(mesg='A valid rstfile must be specified',
rstfile=self.rstfile)
self.linesout = []
self.context = {}
self.core = None
self.handlers = {
'storm': self._handleStorm,
'storm-pkg': self._handleStormPkg,
'storm-pre': self._handleStormPre,
'storm-svc': self._handleStormSvc,
'storm-opts': self._handleStormOpts,
'storm-cortex': self._handleStormCortex,
'storm-expect': self._handleStormExpect,
'storm-mock-http': self._handleStormMockHttp,
}
async def storm(self, text, opts=None):
'''
Evaluate a storm query and yield result messages.
Yields:
((str,dict)): Storm messages.
'''
opts = self.core._initStormOpts(opts)
user = self.core._userFromOpts(opts)
MSG_QUEUE_SIZE = 1000
chan = asyncio.Queue(MSG_QUEUE_SIZE, loop=self.loop)
info = {'query': text, 'opts': opts}
synt = await self.core.boss.promote('storm', user=user, info=info)
show = opts.get('show', set())
editformat = opts.get('editformat', 'nodeedits')
if editformat not in ('nodeedits', 'splices', 'count', 'none'):
raise s_exc.BadConfValu(mesg='editformat')
async def runStorm():
cancelled = False
tick = s_common.now()
count = 0
try:
# Always start with an init message.
await chan.put(('init', {
'tick': tick,
'text': text,
'task': synt.iden
}))
# Try text parsing. If this fails, we won't be able to get a storm
# runtime in the snap, so catch and pass the `err` message
self.core.getStormQuery(text)
shownode = (not show or 'node' in show)
async with await self.snap(user=user) as snap:
if not show:
snap.link(chan.put)
else:
[snap.on(n, chan.put) for n in show]
if shownode:
async for pode in snap.iterStormPodes(text,
opts=opts,
user=user):
await chan.put(('node', pode))
count += 1
else:
async for item in snap.storm(text,
opts=opts,
user=user):
count += 1
except asyncio.CancelledError:
logger.warning('Storm runtime cancelled.')
cancelled = True
raise
except Exception as e:
logger.exception(
f'Error during storm execution for {{ {text} }}')
enfo = s_common.err(e)
enfo[1].pop('esrc', None)
enfo[1].pop('ename', None)
await chan.put(('err', enfo))
finally:
if not cancelled:
tock = s_common.now()
took = tock - tick
await chan.put(('fini', {
'tock': tock,
'took': took,
'count': count
}))
await synt.worker(runStorm())
editformat = opts.get('editformat', 'nodeedits')
while True:
mesg = await chan.get()
kind = mesg[0]
if kind == 'node':
yield mesg
continue
if kind == 'node:edits':
if editformat == 'nodeedits':
nodeedits = s_common.jsonsafe_nodeedits(mesg[1]['edits'])
mesg[1]['edits'] = nodeedits
yield mesg
continue
if editformat == 'none':
continue
if editformat == 'count':
count = sum(
len(edit[2]) for edit in mesg[1].get('edits', ()))
mesg = ('node:edits:count', {'count': count})
yield mesg
continue
assert editformat == 'splices'
nodeedits = mesg[1].get('edits', [()])
async for _, splice in self.layers[0].makeSplices(
0, nodeedits, None):
if not show or splice[0] in show:
yield splice
continue
if kind == 'fini':
yield mesg
break
yield mesg
async def __anit__(self, dirn, conf=None, readonly=False):
# phase 1
if conf is None:
conf = {}
s_telepath.Aware.__init__(self)
self.dirn = s_common.gendir(dirn)
self.auth = None
self.sessions = {}
self.isactive = False
self.inaugural = False
self.conf = self._initCellConf(conf)
# each cell has a guid
path = s_common.genpath(self.dirn, 'cell.guid')
# generate a guid file if needed
if not os.path.isfile(path):
self.inaugural = True
guid = conf.get('cell:guid')
if guid is None:
guid = s_common.guid()
with open(path, 'w') as fd:
fd.write(guid)
# read our guid file
with open(path, 'r') as fd:
self.iden = fd.read().strip()
self.donexslog = self.conf.get('nexslog:en')
backdirn = self.conf.get('backup:dir')
if backdirn is not None:
backdirn = s_common.genpath(backdirn)
if backdirn.startswith(self.dirn):
mesg = 'backup:dir must not be within the service directory'
raise s_exc.BadConfValu(mesg=mesg)
backdirn = s_common.gendir(backdirn)
self.backdirn = backdirn
if self.conf.get('mirror') and not self.conf.get('nexslog:en'):
mesg = 'Mirror mode requires nexslog:en=True'
raise s_exc.BadConfValu(mesg=mesg)
# construct our nexsroot instance ( but do not start it )
await s_nexus.Pusher.__anit__(self, self.iden)
root = await self._ctorNexsRoot()
# mutually assured destruction with our nexs root
self.onfini(root.fini)
root.onfini(self.fini)
self.setNexsRoot(root)
await self._initCellSlab(readonly=readonly)
self.hive = await self._initCellHive()
# self.cellinfo, a HiveDict for general purpose persistent storage
node = await self.hive.open(('cellinfo', ))
self.cellinfo = await node.dict()
self.onfini(node)
if self.inaugural:
await self.cellinfo.set('synapse:version', s_version.version)
synvers = self.cellinfo.get('synapse:version')
if synvers is None or synvers < s_version.version:
await self.cellinfo.set('synapse:version', s_version.version)
self.auth = await self._initCellAuth()
auth_passwd = self.conf.get('auth:passwd')
if auth_passwd is not None:
user = await self.auth.getUserByName('root')
if not user.tryPasswd(auth_passwd):
await user.setPasswd(auth_passwd, nexs=False)
self.boss = await s_boss.Boss.anit()
self.onfini(self.boss)
self.dynitems = {'auth': self.auth, 'cell': self}
# initialize web app and callback data structures
self._health_funcs = []
self.addHealthFunc(self._cellHealth)
# initialize network daemons (but do not listen yet)
# to allow registration of callbacks and shared objects
# within phase 2/4.
await self._initCellHttp()
await self._initCellDmon()
# phase 2 - service storage
await self.initServiceStorage()
# phase 3 - nexus subsystem
await self.initNexusSubsystem()
# phase 4 - service logic
await self.initServiceRuntime()
# phase 5 - service networking
await self.initServiceNetwork()
