async def test_cell_dyncall(self):
with self.getTestDir() as dirn:
async with await EchoAuth.anit(dirn) as cell, cell.getLocalProxy() as prox:
cell.dynitems['self'] = cell
self.eq(42, await prox.dyncall('self', s_common.todo('answer')))
await self.asyncraises(s_exc.BadArg, prox.dyncall('self', s_common.todo('badanswer')))
self.eq([1, 2], await s_t_utils.alist(await prox.dyncall('self', s_common.todo('stream'))))
todo = s_common.todo('stream', doraise=True)
await self.agenraises(s_exc.BadTime, await prox.dyncall('self', todo))
async def _runBackup(self, name=None, wait=True):
name = await s_stormtypes.tostr(name, noneok=True)
wait = await s_stormtypes.tobool(wait)
todo = s_common.todo('runBackup', name=name, wait=wait)
gatekeys = ((self.runt.user.iden, ('backup', 'run'), None), )
return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def _lock(self, name, locked):
name = await s_stormtypes.tostr(name)
locked = await s_stormtypes.tobool(locked)
todo = s_common.todo('setDeprLock', name, locked)
gatekeys = ((self.runt.user.iden, ('model', 'deprecated', 'lock'),
None), )
await self.runt.dyncall('cortex', todo, gatekeys=gatekeys)
async def addStormPkg(self, pkgdef):
'''
Do it for the proxy, then myself
'''
todo = s_common.todo('addStormPkg', pkgdef)
await self.dyncall('cortex', todo)
await self.loadStormPkg(pkgdef)
async def getRuntNodes(self, full, valu=None, cmpr=None):
todo = s_common.todo('runRuntLift', full, valu, cmpr)
async for sode in self.core.dyniter('cortex', todo):
node = s_node.Node(self, sode)
node.isrunt = True
yield node
async def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(
prog='synapse.tools.hive.load',
description='Load data into a remote hive from a previous hivesave.')
pars.add_argument('--trim',
default=False,
action='store_true',
help='Trim all other hive nodes (DANGER!)')
pars.add_argument('--path',
default=None,
help='A hive path string to use as the root.')
pars.add_argument(
'--yaml',
default=False,
action='store_true',
help='Parse the savefile as a YAML file (default: msgpack)')
pars.add_argument('hiveurl', help='The telepath URL for the remote hive.')
pars.add_argument('filepath', help='The local file path to load.')
opts = pars.parse_args(argv)
if opts.yaml:
tree = s_common.yamlload(opts.filepath)
else:
tree = s_msgpack.loadfile(opts.filepath)
path = ()
if opts.path is not None:
path = opts.path.split('/')
async with await s_telepath.openurl(opts.hiveurl) as hive:
classes = hive.sharinfo.get('classes', ())
try:
s_version.reqVersion(hive._getSynVers(), reqver)
if 'synapse.lib.hive.HiveApi' in classes:
await hive.loadHiveTree(tree, path=path, trim=opts.trim)
else:
todo = s_common.todo('loadHiveTree',
tree,
path=path,
trim=opts.trim)
await hive.dyncall('cell', todo)
except s_exc.BadVersion as e:
valu = s_version.fmtVersion(*e.get('valu'))
outp.printf(
f'Hive version {valu} is outside of the hive.load supported range ({reqver}).'
)
outp.printf(
f'Please use a version of Synapse which supports {valu}; current version is {s_version.verstring}.'
)
return 1
async def _listBackups(self):
'''
Returns a list of backup names.
Returns:
list[str]: A list of backup names.
'''
todo = s_common.todo('getBackups')
gatekeys = ((self.runt.user.iden, ('backup', 'list'), None), )
return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def getNodeData(self, buid, name, defv=None):
'''
Get nodedata from closest to write layer, no merging involved
'''
for layr in reversed(self.layers):
todo = s_common.todo('getNodeData', buid, name)
ok, valu = await self.core.dyncall(layr.iden, todo)
if ok:
return valu
return defv
async def hasNodeData(self, buid, name):
'''
Return True if the buid has nodedata set on it under the given name
False otherwise
'''
for layr in reversed(self.layers):
todo = s_common.todo('hasNodeData', buid, name)
has = await self.core.dyncall(layr.iden, todo)
if has:
return True
return False
async def _delBackup(self, name):
'''
Remove a backup by name.
Args:
name (str): The name of the backup to remove.
'''
name = await s_stormtypes.tostr(name)
todo = s_common.todo('delBackup', name)
gatekeys = ((self.runt.user.iden, ('backup', 'del'), None), )
return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def _httpRequest(self,
meth,
url,
headers=None,
json=None,
body=None,
ssl_verify=True,
params=None):
meth = await s_stormtypes.tostr(meth)
url = await s_stormtypes.tostr(url)
json = await s_stormtypes.toprim(json)
body = await s_stormtypes.toprim(body)
headers = await s_stormtypes.toprim(headers)
params = await s_stormtypes.toprim(params)
kwargs = {}
if not ssl_verify:
kwargs['ssl'] = False
if params:
kwargs['params'] = params
todo = s_common.todo('getConfOpt', 'http:proxy')
proxyurl = await self.runt.dyncall('cortex', todo)
connector = None
if proxyurl is not None:
connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
async with aiohttp.ClientSession(connector=connector) as sess:
try:
async with sess.request(meth,
url,
headers=headers,
json=json,
data=body,
**kwargs) as resp:
info = {
'code': resp.status,
'headers': dict(resp.headers),
'url': str(resp.url),
'body': await resp.read(),
}
return HttpResp(info)
# return HttpResp(code=resp.status, body=await resp.content.read())
except asyncio.CancelledError: # pragma: no cover
raise
except Exception as e:
mesg = f'Error during http {meth} - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg,
headers=headers,
json=json,
body=body,
params=params) from None
async def iterNodeData(self, buid):
'''
Returns: Iterable[Tuple[str, Any]]
'''
some = False
for layr in reversed(self.layers):
todo = s_common.todo('iterNodeData', buid)
async for item in self.core.dyniter(layr.iden, todo):
some = True
yield item
if some:
return
async def delStormPkg(self, name):
'''
Do it for the proxy, then myself
'''
todo = s_common.todo('delStormPkg', name)
await self.dyncall('cortex', todo)
pkgdef = self.stormpkgs.get(name)
if pkgdef is None:
return
await self._dropStormPkg(pkgdef)
async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_verify=True,
params=None, timeout=300):
meth = await s_stormtypes.tostr(meth)
url = await s_stormtypes.tostr(url)
json = await s_stormtypes.toprim(json)
body = await s_stormtypes.toprim(body)
headers = await s_stormtypes.toprim(headers)
params = await s_stormtypes.toprim(params)
timeout = await s_stormtypes.toint(timeout, noneok=True)
kwargs = {}
if not ssl_verify:
kwargs['ssl'] = False
if params:
kwargs['params'] = params
todo = s_common.todo('getConfOpt', 'http:proxy')
proxyurl = await self.runt.dyncall('cortex', todo)
connector = None
if proxyurl is not None:
connector = aiohttp_socks.ProxyConnector.from_url(proxyurl)
timeout = aiohttp.ClientTimeout(total=timeout)
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as sess:
try:
async with sess.request(meth, url, headers=headers, json=json, data=body, **kwargs) as resp:
info = {
'code': resp.status,
'headers': dict(resp.headers),
'url': str(resp.url),
'body': await resp.read(),
}
return HttpResp(info)
# return HttpResp(code=resp.status, body=await resp.content.read())
except asyncio.CancelledError: # pragma: no cover
raise
except Exception as e:
logger.exception(f'Error during http {meth} @ {url}')
err = s_common.err(e)
info = {
'code': -1,
'headers': dict(),
'url': url,
'body': b'',
'err': err,
}
return HttpResp(info)
async def _runBackup(self, name=None, wait=True):
'''
Run a cortex backup.
Args:
name (str): The name of the backup to generate.
wait (bool): If true, wait for the backup to complete before returning.
Returns:
str: The name of the newly created backup.
'''
name = await s_stormtypes.tostr(name, noneok=True)
wait = await s_stormtypes.tobool(wait)
todo = s_common.todo('runBackup', name=name, wait=wait)
gatekeys = ((self.runt.user.iden, ('backup', 'run'), None), )
return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def getCortexAndProxy(self) -> AsyncIterator[Tuple[Any, Any]]:
'''
Prepares a cortex/proxy for a benchmark run
'''
ldef = {
'lockmemory': self.coreconfig.get('layers:lockmemory', False),
'logedits': self.coreconfig.get('layers:logedits', True),
'name': 'tmp for benchmark',
}
core = None
async with contextlib.AsyncExitStack() as stack:
if not self.testdata.remote:
ctx = await s_cortex.Cortex.anit(self.testdata.dirn, conf=self.coreconfig)
core = await stack.enter_async_context(ctx)
prox = await stack.enter_async_context(core.getLocalProxy())
assert not core.inaugural
else:
ctx = await s_telepath.openurl(self.testdata.remote)
prox = await stack.enter_async_context(ctx)
layer = await prox.cloneLayer(self.testdata.layriden, ldef)
layeriden = layer['iden']
view = await prox.callStorm('return($lib.view.add(($layer, ), name="tmp for benchmark"))',
opts={'vars': {'layer': layeriden}})
self.viewiden = view['iden']
self.opts = {'view': self.viewiden}
await prox.dyncall(layeriden, s_common.todo('waitForHot'))
try:
yield core, prox
finally:
await prox.callStorm('''
$lib.view.del($view)
$lib.layer.del($layer)
''', opts={'vars': {'view': self.viewiden, 'layer': layeriden}})
async def _execBackTask(self, dirn):
await self.boss.promote('backup', self.auth.rootuser)
todo = s_common.todo(s_t_backup.backup, self.dirn, dirn)
await s_coro.spawn(todo)
async def _listBackups(self):
todo = s_common.todo('getBackups')
gatekeys = ((self.runt.user.iden, ('backup', 'list'), None), )
return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def _delBackup(self, name):
name = await s_stormtypes.tostr(name)
todo = s_common.todo('delBackup', name)
gatekeys = ((self.runt.user.iden, ('backup', 'del'), None), )
return await self.dyncall('cortex', todo, gatekeys=gatekeys)
async def _locks(self):
todo = s_common.todo('getDeprLocks')
locks = await self.runt.dyncall('cortex', todo)
return s_stormtypes.Dict(locks)
def set_pool_logging(logger_, logconf):
# This must be called before any calls to forked()
todo = s_common.todo(s_common.setlogging, logger_, **logconf)
if forkpool is not None:
forkpool._initializer = _runtodo
forkpool._initargs = (todo, )
async def scrapeIface(self, text, unique=False):
async with await s_spooled.Set.anit() as matches: # type: s_spooled.Set
# The synapse.lib.scrape APIs handle form arguments for us.
for item in s_scrape.contextScrape(text, refang=True, first=False):
form = item.pop('form')
valu = item.pop('valu')
if unique:
key = (form, valu)
if key in matches:
await asyncio.sleep(0)
continue
await matches.add(key)
try:
tobj = self.core.model.type(form)
valu, _ = tobj.norm(valu)
except s_exc.BadTypeValu: # pragma: no cover
await asyncio.sleep(0)
continue
# Yield a tuple of <form, normed valu, info>
yield form, valu, item
# Return early if the scrape interface is disabled
if not self.core.stormiface_scrape:
return
# Scrape interface:
#
# The expected scrape interface takes a text and optional form
# argument.
#
# The expected interface implementation returns a list/tuple of
# (form, valu, info) results. Info is expected to contain the
# match offset and raw valu.
#
# Scrape implementers are responsible for ensuring that their
# resulting match and offsets are found in the text we sent
# to them.
todo = s_common.todo('scrape', text)
async for results in self.callStormIface('scrape', todo):
for (form, valu, info) in results:
if unique:
key = (form, valu)
if key in matches:
await asyncio.sleep(0)
continue
await matches.add(key)
try:
tobj = self.core.model.type(form)
valu, _ = tobj.norm(valu)
except AttributeError: # pragma: no cover
logger.exception(f'Scrape interface yielded unknown form {form}')
await asyncio.sleep(0)
continue
except (s_exc.BadTypeValu): # pragma: no cover
await asyncio.sleep(0)
continue
# Yield a tuple of <form, normed valu, info>
yield form, valu, info
await asyncio.sleep(0)
async def validateBundle(self, bundle):
bundle = await s_stormtypes.toprim(bundle)
logconf = await self.runt.snap.core._getSpawnLogConf()
resp = await s_coro.spawn(
s_common.todo(_validateStixProc, bundle, logconf=logconf))
return resp
async def sleeper():
yield s_common.todo('bar', 10, 30)
await asyncio.sleep(3)
async def boomgenr():
yield s_common.todo('bar', 10, 30)
raise s_exc.NoSuchIden()
async def applyNodeEdits(self, edits):
'''
Sends edits to the write layer and evaluates the consequences (triggers, node object updates)
'''
if self.readonly:
mesg = 'The snapshot is in read-only mode.'
raise s_exc.IsReadOnly(mesg=mesg)
meta = await self.getSnapMeta()
todo = s_common.todo('storNodeEdits', edits, meta)
results = await self.core.dyncall(self.wlyr.iden, todo)
wlyr = self.wlyr
nodes = []
callbacks = []
actualedits = [] # List[Tuple[buid, form, changes]]
# make a pass through the returned edits, apply the changes to our Nodes()
# and collect up all the callbacks to fire at once at the end. It is
# critical to fire all callbacks after applying all Node() changes.
for buid, sode, postedits in results:
cache = {wlyr.iden: sode}
node = await self._joinStorNode(buid, cache)
if node is None:
# We got part of a node but no ndef
continue
nodes.append(node)
if postedits:
actualedits.append((buid, node.form.name, postedits))
for edit in postedits:
etyp, parms, _ = edit
if etyp == s_layer.EDIT_NODE_ADD:
node.bylayer['ndef'] = wlyr.iden
callbacks.append((node.form.wasAdded, (node, ), {}))
callbacks.append((self.view.runNodeAdd, (node, ), {}))
continue
if etyp == s_layer.EDIT_NODE_DEL:
callbacks.append((node.form.wasDeleted, (node, ), {}))
callbacks.append((self.view.runNodeDel, (node, ), {}))
continue
if etyp == s_layer.EDIT_PROP_SET:
(name, valu, oldv, stype) = parms
prop = node.form.props.get(name)
if prop is None: # pragma: no cover
logger.warning(
f'applyNodeEdits got EDIT_PROP_SET for bad prop {name} on form {node.form}'
)
continue
node.props[name] = valu
node.bylayer['props'][name] = wlyr.iden
callbacks.append((prop.wasSet, (node, oldv), {}))
callbacks.append(
(self.view.runPropSet, (node, prop, oldv), {}))
continue
if etyp == s_layer.EDIT_PROP_DEL:
(name, oldv, stype) = parms
prop = node.form.props.get(name)
if prop is None: # pragma: no cover
logger.warning(
f'applyNodeEdits got EDIT_PROP_DEL for bad prop {name} on form {node.form}'
)
continue
node.props.pop(name, None)
node.bylayer['props'].pop(name, None)
callbacks.append((prop.wasDel, (node, oldv), {}))
callbacks.append(
(self.view.runPropSet, (node, prop, oldv), {}))
continue
if etyp == s_layer.EDIT_TAG_SET:
(tag, valu, oldv) = parms
node.tags[tag] = valu
node.bylayer['tags'][tag] = wlyr.iden
callbacks.append(
(self.view.runTagAdd, (node, tag, valu), {}))
callbacks.append((self.wlyr.fire, ('tag:add', ), {
'tag': tag,
'node': node.iden()
}))
continue
if etyp == s_layer.EDIT_TAG_DEL:
(tag, oldv) = parms
node.tags.pop(tag, None)
node.bylayer['tags'].pop(tag, None)
callbacks.append(
(self.view.runTagDel, (node, tag, oldv), {}))
callbacks.append((self.wlyr.fire, ('tag:del', ), {
'tag': tag,
'node': node.iden()
}))
continue
if etyp == s_layer.EDIT_TAGPROP_SET:
(tag, prop, valu, oldv, stype) = parms
node.tagprops[(tag, prop)] = valu
node.bylayer['tags'][(tag, prop)] = wlyr.iden
continue
if etyp == s_layer.EDIT_TAGPROP_DEL:
(tag, prop, oldv, stype) = parms
node.tagprops.pop((tag, prop), None)
node.bylayer['tags'].pop((tag, prop), None)
continue
[await func(*args, **kwargs) for (func, args, kwargs) in callbacks]
if actualedits:
providen, provstack = self.core.provstor.stor()
if providen is not None:
await self.fire('prov:new',
time=meta['time'],
user=meta['user'],
prov=providen,
provstack=provstack)
await self.fire('node:edits', edits=actualedits)
return nodes
async def genr():
yield s_common.todo('bar', 10, 30)
yield s_common.todo('bar', 20, 30)
yield s_common.todo('bar', 30, 30)
