async def test_cell_dyncall(self): with self.getTestDir() as dirn: async with await EchoAuth.anit(dirn) as cell, cell.getLocalProxy() as prox: cell.dynitems['self'] = cell self.eq(42, await prox.dyncall('self', s_common.todo('answer'))) await self.asyncraises(s_exc.BadArg, prox.dyncall('self', s_common.todo('badanswer'))) self.eq([1, 2], await s_t_utils.alist(await prox.dyncall('self', s_common.todo('stream')))) todo = s_common.todo('stream', doraise=True) await self.agenraises(s_exc.BadTime, await prox.dyncall('self', todo))
async def _runBackup(self, name=None, wait=True): name = await s_stormtypes.tostr(name, noneok=True) wait = await s_stormtypes.tobool(wait) todo = s_common.todo('runBackup', name=name, wait=wait) gatekeys = ((self.runt.user.iden, ('backup', 'run'), None), ) return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def _lock(self, name, locked): name = await s_stormtypes.tostr(name) locked = await s_stormtypes.tobool(locked) todo = s_common.todo('setDeprLock', name, locked) gatekeys = ((self.runt.user.iden, ('model', 'deprecated', 'lock'), None), ) await self.runt.dyncall('cortex', todo, gatekeys=gatekeys)
async def addStormPkg(self, pkgdef): ''' Do it for the proxy, then myself ''' todo = s_common.todo('addStormPkg', pkgdef) await self.dyncall('cortex', todo) await self.loadStormPkg(pkgdef)
async def getRuntNodes(self, full, valu=None, cmpr=None): todo = s_common.todo('runRuntLift', full, valu, cmpr) async for sode in self.core.dyniter('cortex', todo): node = s_node.Node(self, sode) node.isrunt = True yield node
async def main(argv, outp=s_output.stdout): pars = argparse.ArgumentParser( prog='synapse.tools.hive.load', description='Load data into a remote hive from a previous hivesave.') pars.add_argument('--trim', default=False, action='store_true', help='Trim all other hive nodes (DANGER!)') pars.add_argument('--path', default=None, help='A hive path string to use as the root.') pars.add_argument( '--yaml', default=False, action='store_true', help='Parse the savefile as a YAML file (default: msgpack)') pars.add_argument('hiveurl', help='The telepath URL for the remote hive.') pars.add_argument('filepath', help='The local file path to load.') opts = pars.parse_args(argv) if opts.yaml: tree = s_common.yamlload(opts.filepath) else: tree = s_msgpack.loadfile(opts.filepath) path = () if opts.path is not None: path = opts.path.split('/') async with await s_telepath.openurl(opts.hiveurl) as hive: classes = hive.sharinfo.get('classes', ()) try: s_version.reqVersion(hive._getSynVers(), reqver) if 'synapse.lib.hive.HiveApi' in classes: await hive.loadHiveTree(tree, path=path, trim=opts.trim) else: todo = s_common.todo('loadHiveTree', tree, path=path, trim=opts.trim) await hive.dyncall('cell', todo) except s_exc.BadVersion as e: valu = s_version.fmtVersion(*e.get('valu')) outp.printf( f'Hive version {valu} is outside of the hive.load supported range ({reqver}).' ) outp.printf( f'Please use a version of Synapse which supports {valu}; current version is {s_version.verstring}.' ) return 1
async def _listBackups(self): ''' Returns a list of backup names. Returns: list[str]: A list of backup names. ''' todo = s_common.todo('getBackups') gatekeys = ((self.runt.user.iden, ('backup', 'list'), None), ) return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def getNodeData(self, buid, name, defv=None): ''' Get nodedata from closest to write layer, no merging involved ''' for layr in reversed(self.layers): todo = s_common.todo('getNodeData', buid, name) ok, valu = await self.core.dyncall(layr.iden, todo) if ok: return valu return defv
async def hasNodeData(self, buid, name): ''' Return True if the buid has nodedata set on it under the given name False otherwise ''' for layr in reversed(self.layers): todo = s_common.todo('hasNodeData', buid, name) has = await self.core.dyncall(layr.iden, todo) if has: return True return False
async def _delBackup(self, name): ''' Remove a backup by name. Args: name (str): The name of the backup to remove. ''' name = await s_stormtypes.tostr(name) todo = s_common.todo('delBackup', name) gatekeys = ((self.runt.user.iden, ('backup', 'del'), None), ) return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_verify=True, params=None): meth = await s_stormtypes.tostr(meth) url = await s_stormtypes.tostr(url) json = await s_stormtypes.toprim(json) body = await s_stormtypes.toprim(body) headers = await s_stormtypes.toprim(headers) params = await s_stormtypes.toprim(params) kwargs = {} if not ssl_verify: kwargs['ssl'] = False if params: kwargs['params'] = params todo = s_common.todo('getConfOpt', 'http:proxy') proxyurl = await self.runt.dyncall('cortex', todo) connector = None if proxyurl is not None: connector = aiohttp_socks.ProxyConnector.from_url(proxyurl) async with aiohttp.ClientSession(connector=connector) as sess: try: async with sess.request(meth, url, headers=headers, json=json, data=body, **kwargs) as resp: info = { 'code': resp.status, 'headers': dict(resp.headers), 'url': str(resp.url), 'body': await resp.read(), } return HttpResp(info) # return HttpResp(code=resp.status, body=await resp.content.read()) except asyncio.CancelledError: # pragma: no cover raise except Exception as e: mesg = f'Error during http {meth} - {str(e)}' raise s_exc.StormRuntimeError(mesg=mesg, headers=headers, json=json, body=body, params=params) from None
async def iterNodeData(self, buid): ''' Returns: Iterable[Tuple[str, Any]] ''' some = False for layr in reversed(self.layers): todo = s_common.todo('iterNodeData', buid) async for item in self.core.dyniter(layr.iden, todo): some = True yield item if some: return
async def delStormPkg(self, name): ''' Do it for the proxy, then myself ''' todo = s_common.todo('delStormPkg', name) await self.dyncall('cortex', todo) pkgdef = self.stormpkgs.get(name) if pkgdef is None: return await self._dropStormPkg(pkgdef)
async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_verify=True, params=None, timeout=300): meth = await s_stormtypes.tostr(meth) url = await s_stormtypes.tostr(url) json = await s_stormtypes.toprim(json) body = await s_stormtypes.toprim(body) headers = await s_stormtypes.toprim(headers) params = await s_stormtypes.toprim(params) timeout = await s_stormtypes.toint(timeout, noneok=True) kwargs = {} if not ssl_verify: kwargs['ssl'] = False if params: kwargs['params'] = params todo = s_common.todo('getConfOpt', 'http:proxy') proxyurl = await self.runt.dyncall('cortex', todo) connector = None if proxyurl is not None: connector = aiohttp_socks.ProxyConnector.from_url(proxyurl) timeout = aiohttp.ClientTimeout(total=timeout) async with aiohttp.ClientSession(connector=connector, timeout=timeout) as sess: try: async with sess.request(meth, url, headers=headers, json=json, data=body, **kwargs) as resp: info = { 'code': resp.status, 'headers': dict(resp.headers), 'url': str(resp.url), 'body': await resp.read(), } return HttpResp(info) # return HttpResp(code=resp.status, body=await resp.content.read()) except asyncio.CancelledError: # pragma: no cover raise except Exception as e: logger.exception(f'Error during http {meth} @ {url}') err = s_common.err(e) info = { 'code': -1, 'headers': dict(), 'url': url, 'body': b'', 'err': err, } return HttpResp(info)
async def _runBackup(self, name=None, wait=True): ''' Run a cortex backup. Args: name (str): The name of the backup to generate. wait (bool): If true, wait for the backup to complete before returning. Returns: str: The name of the newly created backup. ''' name = await s_stormtypes.tostr(name, noneok=True) wait = await s_stormtypes.tobool(wait) todo = s_common.todo('runBackup', name=name, wait=wait) gatekeys = ((self.runt.user.iden, ('backup', 'run'), None), ) return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def getCortexAndProxy(self) -> AsyncIterator[Tuple[Any, Any]]: ''' Prepares a cortex/proxy for a benchmark run ''' ldef = { 'lockmemory': self.coreconfig.get('layers:lockmemory', False), 'logedits': self.coreconfig.get('layers:logedits', True), 'name': 'tmp for benchmark', } core = None async with contextlib.AsyncExitStack() as stack: if not self.testdata.remote: ctx = await s_cortex.Cortex.anit(self.testdata.dirn, conf=self.coreconfig) core = await stack.enter_async_context(ctx) prox = await stack.enter_async_context(core.getLocalProxy()) assert not core.inaugural else: ctx = await s_telepath.openurl(self.testdata.remote) prox = await stack.enter_async_context(ctx) layer = await prox.cloneLayer(self.testdata.layriden, ldef) layeriden = layer['iden'] view = await prox.callStorm('return($lib.view.add(($layer, ), name="tmp for benchmark"))', opts={'vars': {'layer': layeriden}}) self.viewiden = view['iden'] self.opts = {'view': self.viewiden} await prox.dyncall(layeriden, s_common.todo('waitForHot')) try: yield core, prox finally: await prox.callStorm(''' $lib.view.del($view) $lib.layer.del($layer) ''', opts={'vars': {'view': self.viewiden, 'layer': layeriden}})
async def _execBackTask(self, dirn): await self.boss.promote('backup', self.auth.rootuser) todo = s_common.todo(s_t_backup.backup, self.dirn, dirn) await s_coro.spawn(todo)
async def _listBackups(self): todo = s_common.todo('getBackups') gatekeys = ((self.runt.user.iden, ('backup', 'list'), None), ) return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def _delBackup(self, name): name = await s_stormtypes.tostr(name) todo = s_common.todo('delBackup', name) gatekeys = ((self.runt.user.iden, ('backup', 'del'), None), ) return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def _locks(self): todo = s_common.todo('getDeprLocks') locks = await self.runt.dyncall('cortex', todo) return s_stormtypes.Dict(locks)
def set_pool_logging(logger_, logconf): # This must be called before any calls to forked() todo = s_common.todo(s_common.setlogging, logger_, **logconf) if forkpool is not None: forkpool._initializer = _runtodo forkpool._initargs = (todo, )
async def scrapeIface(self, text, unique=False): async with await s_spooled.Set.anit() as matches: # type: s_spooled.Set # The synapse.lib.scrape APIs handle form arguments for us. for item in s_scrape.contextScrape(text, refang=True, first=False): form = item.pop('form') valu = item.pop('valu') if unique: key = (form, valu) if key in matches: await asyncio.sleep(0) continue await matches.add(key) try: tobj = self.core.model.type(form) valu, _ = tobj.norm(valu) except s_exc.BadTypeValu: # pragma: no cover await asyncio.sleep(0) continue # Yield a tuple of <form, normed valu, info> yield form, valu, item # Return early if the scrape interface is disabled if not self.core.stormiface_scrape: return # Scrape interface: # # The expected scrape interface takes a text and optional form # argument. # # The expected interface implementation returns a list/tuple of # (form, valu, info) results. Info is expected to contain the # match offset and raw valu. # # Scrape implementers are responsible for ensuring that their # resulting match and offsets are found in the text we sent # to them. todo = s_common.todo('scrape', text) async for results in self.callStormIface('scrape', todo): for (form, valu, info) in results: if unique: key = (form, valu) if key in matches: await asyncio.sleep(0) continue await matches.add(key) try: tobj = self.core.model.type(form) valu, _ = tobj.norm(valu) except AttributeError: # pragma: no cover logger.exception(f'Scrape interface yielded unknown form {form}') await asyncio.sleep(0) continue except (s_exc.BadTypeValu): # pragma: no cover await asyncio.sleep(0) continue # Yield a tuple of <form, normed valu, info> yield form, valu, info await asyncio.sleep(0)
async def validateBundle(self, bundle): bundle = await s_stormtypes.toprim(bundle) logconf = await self.runt.snap.core._getSpawnLogConf() resp = await s_coro.spawn( s_common.todo(_validateStixProc, bundle, logconf=logconf)) return resp
async def sleeper(): yield s_common.todo('bar', 10, 30) await asyncio.sleep(3)
async def boomgenr(): yield s_common.todo('bar', 10, 30) raise s_exc.NoSuchIden()
async def applyNodeEdits(self, edits): ''' Sends edits to the write layer and evaluates the consequences (triggers, node object updates) ''' if self.readonly: mesg = 'The snapshot is in read-only mode.' raise s_exc.IsReadOnly(mesg=mesg) meta = await self.getSnapMeta() todo = s_common.todo('storNodeEdits', edits, meta) results = await self.core.dyncall(self.wlyr.iden, todo) wlyr = self.wlyr nodes = [] callbacks = [] actualedits = [] # List[Tuple[buid, form, changes]] # make a pass through the returned edits, apply the changes to our Nodes() # and collect up all the callbacks to fire at once at the end. It is # critical to fire all callbacks after applying all Node() changes. for buid, sode, postedits in results: cache = {wlyr.iden: sode} node = await self._joinStorNode(buid, cache) if node is None: # We got part of a node but no ndef continue nodes.append(node) if postedits: actualedits.append((buid, node.form.name, postedits)) for edit in postedits: etyp, parms, _ = edit if etyp == s_layer.EDIT_NODE_ADD: node.bylayer['ndef'] = wlyr.iden callbacks.append((node.form.wasAdded, (node, ), {})) callbacks.append((self.view.runNodeAdd, (node, ), {})) continue if etyp == s_layer.EDIT_NODE_DEL: callbacks.append((node.form.wasDeleted, (node, ), {})) callbacks.append((self.view.runNodeDel, (node, ), {})) continue if etyp == s_layer.EDIT_PROP_SET: (name, valu, oldv, stype) = parms prop = node.form.props.get(name) if prop is None: # pragma: no cover logger.warning( f'applyNodeEdits got EDIT_PROP_SET for bad prop {name} on form {node.form}' ) continue node.props[name] = valu node.bylayer['props'][name] = wlyr.iden callbacks.append((prop.wasSet, (node, oldv), {})) callbacks.append( (self.view.runPropSet, (node, prop, oldv), {})) continue if etyp == s_layer.EDIT_PROP_DEL: (name, oldv, stype) = parms prop = node.form.props.get(name) if prop is None: # pragma: no cover logger.warning( f'applyNodeEdits got EDIT_PROP_DEL for bad prop {name} on form {node.form}' ) continue node.props.pop(name, None) node.bylayer['props'].pop(name, None) callbacks.append((prop.wasDel, (node, oldv), {})) callbacks.append( (self.view.runPropSet, (node, prop, oldv), {})) continue if etyp == s_layer.EDIT_TAG_SET: (tag, valu, oldv) = parms node.tags[tag] = valu node.bylayer['tags'][tag] = wlyr.iden callbacks.append( (self.view.runTagAdd, (node, tag, valu), {})) callbacks.append((self.wlyr.fire, ('tag:add', ), { 'tag': tag, 'node': node.iden() })) continue if etyp == s_layer.EDIT_TAG_DEL: (tag, oldv) = parms node.tags.pop(tag, None) node.bylayer['tags'].pop(tag, None) callbacks.append( (self.view.runTagDel, (node, tag, oldv), {})) callbacks.append((self.wlyr.fire, ('tag:del', ), { 'tag': tag, 'node': node.iden() })) continue if etyp == s_layer.EDIT_TAGPROP_SET: (tag, prop, valu, oldv, stype) = parms node.tagprops[(tag, prop)] = valu node.bylayer['tags'][(tag, prop)] = wlyr.iden continue if etyp == s_layer.EDIT_TAGPROP_DEL: (tag, prop, oldv, stype) = parms node.tagprops.pop((tag, prop), None) node.bylayer['tags'].pop((tag, prop), None) continue [await func(*args, **kwargs) for (func, args, kwargs) in callbacks] if actualedits: providen, provstack = self.core.provstor.stor() if providen is not None: await self.fire('prov:new', time=meta['time'], user=meta['user'], prov=providen, provstack=provstack) await self.fire('node:edits', edits=actualedits) return nodes
async def genr(): yield s_common.todo('bar', 10, 30) yield s_common.todo('bar', 20, 30) yield s_common.todo('bar', 30, 30)