def postTypeInit(self): self._size = self.opts.get('size') if self._size < 0: # zero means no width check raise s_exc.BadConfValu(name='size', valu=self._size, mesg='Size must be > 0') if self._size % 2 != 0: raise s_exc.BadConfValu(name='size', valu=self._size, mesg='Size must be a multiple of 2') self.setNormFunc(str, self._normPyStr) self.setNormFunc(bytes, self._normPyBytes)
async def __anit__(self, rstfile): await s_base.Base.__anit__(self) self.rstfile = s_common.genpath(rstfile) if not os.path.isfile(rstfile): raise s_exc.BadConfValu(mesg='A valid rstfile must be specified', rstfile=self.rstfile) self.linesout = [] self.context = {} self.stormvars = {} self.core = None self.handlers = { 'storm': self._handleStorm, 'storm-cli': self._handleStormCli, 'storm-pkg': self._handleStormPkg, 'storm-pre': self._handleStormPre, 'storm-svc': self._handleStormSvc, 'storm-fail': self._handleStormFail, 'storm-opts': self._handleStormOpts, 'storm-cortex': self._handleStormCortex, 'storm-envvar': self._handleStormEnvVar, 'storm-expect': self._handleStormExpect, 'storm-multiline': self._handleStormMultiline, 'storm-mock-http': self._handleStormMockHttp, 'storm-vcr-opts': self._handleStormVcrOpts, 'storm-clear-http': self._handleStormClearHttp, 'storm-vcr-callback': self._handleStormVcrCallback, }
async def setLeader(self, url: Optional[str], iden: str) -> None: ''' Args: url: if None, sets this nexsroot as leader, otherwise the telepath URL of the leader (must be a Cell) iden: iden of the leader. Should be the same as my containing cell's iden ''' if url is not None and not self.donexslog: raise s_exc.BadConfValu(mesg='Mirroring incompatible without nexslog:en') former = self._ldrurl if former == url: return self._ldrurl = url if self._looptask is not None: self._looptask.cancel() self._looptask = None self._ldrready.clear() if self._ldr is not None: await self._ldr.fini() self._ldr = None await self._dostatechange() if self._ldrurl is None: return self._looptask = self.schedCoro(self._followerLoop(iden))
async def _initCoreLayers(self): import synapse.cells as s_cells # avoid import cycle layersdir = pathlib.Path(self.dirn, 'layers') layersdir.mkdir(exist_ok=True) # Layers are imported in reverse lexicographic order, where the earliest in the alphabet is the 'topmost' # write layer. layerdirs = sorted((d for d in layersdir.iterdir() if d.is_dir()), reverse=True) for layeridx, layerdir in enumerate(layerdirs): logger.info('loading external layer from %s', layerdir) if not pathlib.Path(layerdir, 'boot.yaml').exists(): # pragma: no cover logger.warning('Skipping layer directory %s due to missing boot.yaml', layerdir) continue # Every layer but the top is readonly readonly = (layeridx < len(layerdirs) - 1) layer = await s_cells.initFromDirn(layerdir, readonly=readonly) if not isinstance(layer, s_layer.Layer): raise s_exc.BadConfValu('layer dir %s must contain Layer cell', layerdir) self.layers.append(layer) if not self.layers: # Setup the fallback/default single LMDB layer self.layers.append(await self._makeDefaultLayer()) self.layer = self.layers[-1] logger.debug('Cortex using the following layers: %s\n', (''.join(f'\n {l.dirn}' for l in self.layers)))
async def promote(self): client = self.client if client is None: mesg = 'promote() called on non-mirror nexsroot' raise s_exc.BadConfValu(mesg=mesg) await self.startup(None)
async def setFeedOffs(self, iden, offs): if offs < 0: raise s_exc.BadConfValu(mesg='Offset must be greater than or equal to zero.', offs=offs, iden=iden) oldoffs = await self.getFeedOffs(iden) logger.info('Setting Feed offset for [%s] from [%s] to [%s]', iden, oldoffs, offs) return await self.layer.setOffset(iden, offs)
def _validateConfig(core, config): maxsize = config.get('maxsize', 10000) if not isinstance(maxsize, int): mesg = f'STIX Bundle config maxsize option must be an integer.' raise s_exc.BadConfValu(mesg=mesg) if maxsize > 10000: mesg = f'STIX Bundle config maxsize option must be <= 10000.' raise s_exc.BadConfValu(mesg=mesg) formmaps = config.get('forms') if formmaps is None: mesg = f'STIX Bundle config is missing "forms" mappings.' raise s_exc.NeedConfValu(mesg=mesg) for formname, formconf in formmaps.items(): form = core.model.form(formname) if form is None: mesg = f'STIX Bundle config contains invalid form name {formname}.' raise s_exc.NoSuchForm(mesg=mesg) stixdef = formconf.get('default') if stixdef is None: mesg = f'STIX Bundle config is missing default mapping for form {formname}.' raise s_exc.NeedConfValu(mesg=mesg) if stixdef not in stix_all: mesg = f'STIX Bundle default mapping ({stixdef}) for {formname} is not a STIX type.' raise s_exc.BadConfValu(mesg=mesg) stixmaps = formconf.get('stix') if stixmaps is None: mesg = f'STIX Bundle config is missing STIX maps for form {formname}.' raise s_exc.NeedConfValu(mesg=mesg) if stixmaps.get(stixdef) is None: mesg = f'STIX Bundle config is missing STIX map for form {formname} default value {stixdef}.' raise s_exc.BadConfValu(mesg=mesg) for stixtype, stixinfo in stixmaps.items(): if stixtype not in stix_all: mesg = f'STIX Bundle config has unknown STIX type {stixtype} for form {formname}.' raise s_exc.BadConfValu(mesg=mesg) stixprops = stixinfo.get('props') if stixprops is not None: for stixprop, stormtext in stixprops.items(): if not isinstance(stormtext, str): mesg = f'STIX Bundle config has invalid prop entry {formname} {stixtype} {stixprop}.' raise s_exc.BadConfValu(mesg=mesg) stixrels = stixinfo.get('rels') if stixrels is not None: for stixrel in stixrels: if len(stixrel) != 3: mesg = f'STIX Bundle config has invalid rel entry {formname} {stixtype} {stixrel}.' raise s_exc.BadConfValu(mesg=mesg)
async def initFromDirn(dirn, *args, **kwargs): ''' As above, but retrieves type from boot.yaml in dirn ''' conf = s_common.yamlload(dirn, 'boot.yaml') or {} kind = conf.get('type') if type is None: raise s_exc.BadConfValu('boot.yaml missing type key') return await init(kind, dirn, *args, **kwargs)
async def promote(self): ''' Transform this cell from a passive follower to an active cell that writes changes locally. ''' if self.conf.get('mirror') is None: mesg = 'promote() called on non-mirror' raise s_exc.BadConfValu(mesg=mesg) await self.nexsroot.promote() await self.setCellActive(True)
async def addCronJob(self, query, reqs, incunit=None, incval=1): ''' Add a cron job to the cortex A cron job is a persistently-stored item that causes storm queries to be run in the future. The specification for the times that the queries run can be one-shot or recurring. Args: query (str): The storm query to execute in the future reqs (Union[Dict[str, Union[int, List[int]]], List[Dict[...]]]): Either a dict of the fixed time fields or a list of such dicts. The keys are in the set ('year', 'month', 'dayofmonth', 'dayofweek', 'hour', 'minute'. The values must be positive integers, except for the key of 'dayofmonth' in which it may also be a negative integer which represents the number of days from the end of the month with -1 representing the last day of the month. All values may also be lists of valid values. incunit (Optional[str]): A member of the same set as above, with an additional member 'day'. If is None (default), then the appointment is one-shot and will not recur. incval (Union[int, List[int]): A integer or a list of integers of the number of units Returns (bytes): An iden that can be used to later modify, query, and delete the job. Notes: reqs must have fields present or incunit must not be None (or both) The incunit if not None it must be larger in unit size than all the keys in all reqs elements. ''' def _convert_reqdict(reqdict): return {s_agenda.TimeUnit.fromString(k): v for (k, v) in reqdict.items()} try: if incunit is not None: if isinstance(incunit, (list, tuple)): incunit = [s_agenda.TimeUnit.fromString(i) for i in incunit] else: incunit = s_agenda.TimeUnit.fromString(incunit) if isinstance(reqs, Mapping): newreqs = _convert_reqdict(reqs) else: newreqs = [_convert_reqdict(req) for req in reqs] except KeyError: raise s_exc.BadConfValu('Unrecognized time unit') username = None if self.user is None else self.user.name return await self.cell.agenda.add(username, query, newreqs, incunit, incval)
def reqConfValid(self): ''' Validate that the loaded configuration data is valid according to the schema. Notes: The validation set does set any default values which are not currently set for configuration options. Returns: None: This returns nothing. ''' try: self.validator(self.conf) except s_exc.SchemaViolation as e: logger.exception('Configuration is invalid.') raise s_exc.BadConfValu(mesg=f'Invalid configuration found: [{str(e)}]') from None else: return
async def __anit__(self, dirn, readonly=False, teleurl=None): await s_layer.Layer.__anit__(self, dirn, readonly=readonly) # a remote layer may never be revd self.canrev = False if teleurl is None: teleurl = self.conf.get('remote:telepath') self.teleurl = teleurl #self.path = self.conf.get('remote:telepath') if self.teleurl is None: raise s_exc.BadConfValu( mesg='remote:telepath must be url for remote layer') self.remote = await s_telepath.openurl(self.teleurl) self.onfini(self.remote.fini) for funcname in PASSTHROUGHFUNCS: setattr(self, funcname, getattr(self.remote, funcname))
async def __anit__(self, rstfile): await s_base.Base.__anit__(self) self.rstfile = s_common.genpath(rstfile) if not os.path.isfile(rstfile): raise s_exc.BadConfValu(mesg='A valid rstfile must be specified', rstfile=self.rstfile) self.linesout = [] self.context = {} self.core = None self.handlers = { 'storm': self._handleStorm, 'storm-pkg': self._handleStormPkg, 'storm-pre': self._handleStormPre, 'storm-svc': self._handleStormSvc, 'storm-opts': self._handleStormOpts, 'storm-cortex': self._handleStormCortex, 'storm-expect': self._handleStormExpect, 'storm-mock-http': self._handleStormMockHttp, }
async def storm(self, text, opts=None): ''' Evaluate a storm query and yield result messages. Yields: ((str,dict)): Storm messages. ''' opts = self.core._initStormOpts(opts) user = self.core._userFromOpts(opts) MSG_QUEUE_SIZE = 1000 chan = asyncio.Queue(MSG_QUEUE_SIZE, loop=self.loop) info = {'query': text, 'opts': opts} synt = await self.core.boss.promote('storm', user=user, info=info) show = opts.get('show', set()) editformat = opts.get('editformat', 'nodeedits') if editformat not in ('nodeedits', 'splices', 'count', 'none'): raise s_exc.BadConfValu(mesg='editformat') async def runStorm(): cancelled = False tick = s_common.now() count = 0 try: # Always start with an init message. await chan.put(('init', { 'tick': tick, 'text': text, 'task': synt.iden })) # Try text parsing. If this fails, we won't be able to get a storm # runtime in the snap, so catch and pass the `err` message self.core.getStormQuery(text) shownode = (not show or 'node' in show) async with await self.snap(user=user) as snap: if not show: snap.link(chan.put) else: [snap.on(n, chan.put) for n in show] if shownode: async for pode in snap.iterStormPodes(text, opts=opts, user=user): await chan.put(('node', pode)) count += 1 else: async for item in snap.storm(text, opts=opts, user=user): count += 1 except asyncio.CancelledError: logger.warning('Storm runtime cancelled.') cancelled = True raise except Exception as e: logger.exception( f'Error during storm execution for {{ {text} }}') enfo = s_common.err(e) enfo[1].pop('esrc', None) enfo[1].pop('ename', None) await chan.put(('err', enfo)) finally: if not cancelled: tock = s_common.now() took = tock - tick await chan.put(('fini', { 'tock': tock, 'took': took, 'count': count })) await synt.worker(runStorm()) editformat = opts.get('editformat', 'nodeedits') while True: mesg = await chan.get() kind = mesg[0] if kind == 'node': yield mesg continue if kind == 'node:edits': if editformat == 'nodeedits': nodeedits = s_common.jsonsafe_nodeedits(mesg[1]['edits']) mesg[1]['edits'] = nodeedits yield mesg continue if editformat == 'none': continue if editformat == 'count': count = sum( len(edit[2]) for edit in mesg[1].get('edits', ())) mesg = ('node:edits:count', {'count': count}) yield mesg continue assert editformat == 'splices' nodeedits = mesg[1].get('edits', [()]) async for _, splice in self.layers[0].makeSplices( 0, nodeedits, None): if not show or splice[0] in show: yield splice continue if kind == 'fini': yield mesg break yield mesg
async def __anit__(self, dirn, conf=None, readonly=False): # phase 1 if conf is None: conf = {} s_telepath.Aware.__init__(self) self.dirn = s_common.gendir(dirn) self.auth = None self.sessions = {} self.isactive = False self.inaugural = False self.conf = self._initCellConf(conf) # each cell has a guid path = s_common.genpath(self.dirn, 'cell.guid') # generate a guid file if needed if not os.path.isfile(path): self.inaugural = True guid = conf.get('cell:guid') if guid is None: guid = s_common.guid() with open(path, 'w') as fd: fd.write(guid) # read our guid file with open(path, 'r') as fd: self.iden = fd.read().strip() self.donexslog = self.conf.get('nexslog:en') backdirn = self.conf.get('backup:dir') if backdirn is not None: backdirn = s_common.genpath(backdirn) if backdirn.startswith(self.dirn): mesg = 'backup:dir must not be within the service directory' raise s_exc.BadConfValu(mesg=mesg) backdirn = s_common.gendir(backdirn) self.backdirn = backdirn if self.conf.get('mirror') and not self.conf.get('nexslog:en'): mesg = 'Mirror mode requires nexslog:en=True' raise s_exc.BadConfValu(mesg=mesg) # construct our nexsroot instance ( but do not start it ) await s_nexus.Pusher.__anit__(self, self.iden) root = await self._ctorNexsRoot() # mutually assured destruction with our nexs root self.onfini(root.fini) root.onfini(self.fini) self.setNexsRoot(root) await self._initCellSlab(readonly=readonly) self.hive = await self._initCellHive() # self.cellinfo, a HiveDict for general purpose persistent storage node = await self.hive.open(('cellinfo', )) self.cellinfo = await node.dict() self.onfini(node) if self.inaugural: await self.cellinfo.set('synapse:version', s_version.version) synvers = self.cellinfo.get('synapse:version') if synvers is None or synvers < s_version.version: await self.cellinfo.set('synapse:version', s_version.version) self.auth = await self._initCellAuth() auth_passwd = self.conf.get('auth:passwd') if auth_passwd is not None: user = await self.auth.getUserByName('root') if not user.tryPasswd(auth_passwd): await user.setPasswd(auth_passwd, nexs=False) self.boss = await s_boss.Boss.anit() self.onfini(self.boss) self.dynitems = {'auth': self.auth, 'cell': self} # initialize web app and callback data structures self._health_funcs = [] self.addHealthFunc(self._cellHealth) # initialize network daemons (but do not listen yet) # to allow registration of callbacks and shared objects # within phase 2/4. await self._initCellHttp() await self._initCellDmon() # phase 2 - service storage await self.initServiceStorage() # phase 3 - nexus subsystem await self.initNexusSubsystem() # phase 4 - service logic await self.initServiceRuntime() # phase 5 - service networking await self.initServiceNetwork()