async def getPropAndValu(self, path):
if not self.ispiv:
valu = path.node.get(self.name)
prop = path.node.form.props.get(self.name)
return prop, valu
# handle implicit pivot properties
names = self.name.split('::')
node = path.node
imax = len(names) - 1
for i, name in enumerate(names):
valu = node.get(name)
if valu is None:
return None, None
prop = node.form.props.get(name)
if i >= imax:
return prop, valu
form = path.runt.snap.model.forms.get(prop.type.name)
if form is None:
raise s_exc.NoSuchForm(name=prop.type.name)
node = await path.runt.snap.getNodeByNdef((form.name, valu))
if node is None:
return None, None
async def _getAddNodeEdits(self, name, valu, props=None):
form = self.core.model.form(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
if form.isrunt:
raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.',
form=form.full,
prop=valu)
if self.buidprefetch:
norm, info = form.type.norm(valu)
buid = s_common.buid((form.name, norm))
node = await self.getNodeByBuid(buid)
if node is not None:
if props is not None:
return (node, await self.getNodeAdds(form,
valu,
props=props,
addnode=False))
else:
return (node, [(buid, form.name, [])])
return (None, await self.getNodeAdds(form, valu, props=props))
def _validateConfig(core, config):
maxsize = config.get('maxsize', 10000)
if not isinstance(maxsize, int):
mesg = f'STIX Bundle config maxsize option must be an integer.'
raise s_exc.BadConfValu(mesg=mesg)
if maxsize > 10000:
mesg = f'STIX Bundle config maxsize option must be <= 10000.'
raise s_exc.BadConfValu(mesg=mesg)
formmaps = config.get('forms')
if formmaps is None:
mesg = f'STIX Bundle config is missing "forms" mappings.'
raise s_exc.NeedConfValu(mesg=mesg)
for formname, formconf in formmaps.items():
form = core.model.form(formname)
if form is None:
mesg = f'STIX Bundle config contains invalid form name {formname}.'
raise s_exc.NoSuchForm(mesg=mesg)
stixdef = formconf.get('default')
if stixdef is None:
mesg = f'STIX Bundle config is missing default mapping for form {formname}.'
raise s_exc.NeedConfValu(mesg=mesg)
if stixdef not in stix_all:
mesg = f'STIX Bundle default mapping ({stixdef}) for {formname} is not a STIX type.'
raise s_exc.BadConfValu(mesg=mesg)
stixmaps = formconf.get('stix')
if stixmaps is None:
mesg = f'STIX Bundle config is missing STIX maps for form {formname}.'
raise s_exc.NeedConfValu(mesg=mesg)
if stixmaps.get(stixdef) is None:
mesg = f'STIX Bundle config is missing STIX map for form {formname} default value {stixdef}.'
raise s_exc.BadConfValu(mesg=mesg)
for stixtype, stixinfo in stixmaps.items():
if stixtype not in stix_all:
mesg = f'STIX Bundle config has unknown STIX type {stixtype} for form {formname}.'
raise s_exc.BadConfValu(mesg=mesg)
stixprops = stixinfo.get('props')
if stixprops is not None:
for stixprop, stormtext in stixprops.items():
if not isinstance(stormtext, str):
mesg = f'STIX Bundle config has invalid prop entry {formname} {stixtype} {stixprop}.'
raise s_exc.BadConfValu(mesg=mesg)
stixrels = stixinfo.get('rels')
if stixrels is not None:
for stixrel in stixrels:
if len(stixrel) != 3:
mesg = f'STIX Bundle config has invalid rel entry {formname} {stixtype} {stixrel}.'
raise s_exc.BadConfValu(mesg=mesg)
def repr(self, norm):
formname, formvalu = norm
form = self.modl.form(formname)
if form is None:
raise s_exc.NoSuchForm(name=self.name, form=formname)
repv = form.type.repr(formvalu)
return (formname, repv)
async def addNode(self, name, valu, props=None):
'''
Add a node by form name and value with optional props.
Args:
name (str): The form of node to add.
valu (obj): The value for the node.
props (dict): Optional secondary properties for the node.
Notes:
If a props dictionary is provided, it may be mutated during node construction.
Returns:
s_node.Node: A Node object. It may return None if the snap is unable to add or lift the node.
'''
if self.readonly:
mesg = 'The snapshot is in read-only mode.'
raise s_exc.IsReadOnly(mesg=mesg)
form = self.core.model.form(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
if form.isrunt:
raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.',
form=form.full,
prop=valu)
try:
if self.buidprefetch:
norm, info = form.type.norm(valu)
node = await self.getNodeByBuid(
s_common.buid((form.name, norm)))
if node is not None:
# TODO implement node.setNodeProps()
if props is not None:
for p, v in props.items():
await node.set(p, v)
return node
adds = await self.getNodeAdds(form, valu, props=props)
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
raise
except Exception as e:
if not self.strict:
await self.warn(f'addNode: {e}')
return None
raise
nodes = await self.applyNodeEdits(adds)
assert len(nodes) >= 1
# Adds is top-down, so the first node is what we want
return nodes[0]
def delForm(self, formname):
form = self.forms.get(formname)
if form is None:
raise s_exc.NoSuchForm(name=formname)
if isinstance(form.type, s_types.Array):
self.arraysbytype[form.type.arraytype.name].remove(form)
self.forms.pop(formname, None)
self.props.pop(formname, None)
async def run(self, runt, genr):
name = self.kids[0].value()
form = runt.snap.model.forms.get(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
# the behavior here is a bit complicated...
# single value add (runtime computed per node )
# In the cases below, $hehe is input to the storm runtime vars.
# case 1: [ foo:bar="lols" ]
# case 2: [ foo:bar=$hehe ]
# case 2: [ foo:bar=$lib.func(20, $hehe) ]
# case 3: ($foo, $bar) = $hehe [ foo:bar=($foo, $bar) ]
# iterative add ( node add is executed once per inbound node )
# case 1: <query> [ foo:bar=(:baz, 20) ]
# case 2: <query> [ foo:bar=($node, 20) ]
# case 2: <query> $blah=:baz [ foo:bar=($blah, 20) ]
if not self.kids[1].isRuntSafe(runt):
first = True
async for node, path in genr:
# must reach back first to trigger sudo / etc
if first:
runt.allowed('node:add', name)
first = False
yield node, path
valu = await self.kids[1].compute(path)
for valu in form.type.getTypeVals(valu):
newn = await runt.snap.addNode(name, valu)
yield newn, runt.initPath(newn)
else:
async for node, path in genr:
yield node, path
runt.allowed('node:add', name)
valu = await self.kids[1].runtval(runt)
for valu in form.type.getTypeVals(valu):
node = await runt.snap.addNode(name, valu)
yield node, runt.initPath(node)
def _getNodeFnib(self, name, valu):
'''
return a form, norm, info, buid tuple
'''
form = self.model.form(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
try:
norm, info = form.type.norm(valu)
except Exception as e:
raise s_exc.BadPropValu(prop=form.name, valu=valu, mesg=str(e))
buid = s_common.buid((form.name, norm))
return form, norm, info, buid
async def getNodeByNdef(self, ndef):
'''
Return a single Node() instance by (form,valu) tuple.
'''
name, valu = ndef
form = self.model.forms.get(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
norm, info = form.type.norm(valu)
buid = s_common.buid((form.name, norm))
async with await self.snap() as snap:
return await snap.getNodeByBuid(buid)
def delFormProp(self, formname, propname):
form = self.forms.get(formname)
if form is None:
raise s_exc.NoSuchForm(name=formname)
prop = form.props.pop(propname, None)
if prop is None:
raise s_exc.NoSuchProp(name=f'{formname}:{propname}')
form.props.pop(prop.name, None)
form.defvals.pop(prop.name, None)
self.props.pop(prop.full, None)
self.props.pop((form.name, prop.name), None)
self.propsbytype[prop.type.name].remove(prop)
def delFormProp(self, formname, propname):
form = self.forms.get(formname)
if form is None:
raise s_exc.NoSuchForm(name=formname)
prop = form.delProp(propname)
if prop is None:
raise s_exc.NoSuchProp(name=f'{formname}:{propname}')
if isinstance(prop.type, s_types.Array):
self.arraysbytype[prop.type.arraytype.name].remove(prop)
self.props.pop(prop.full, None)
self.props.pop((form.name, prop.name), None)
self.propsbytype[prop.type.name].remove(prop)
def _normPyTuple(self, valu):
try:
formname, formvalu = valu
except Exception as e:
raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg=str(e)) from None
form = self.modl.form(formname)
if form is None:
raise s_exc.NoSuchForm(name=self.name, form=formname)
formnorm, info = form.type.norm(formvalu)
norm = (form.name, formnorm)
adds = (norm,)
subs = {'form': form.name}
return norm, {'adds': adds, 'subs': subs}
async def run(self, runt, genr):
name = self.kids[0].value()
form = runt.snap.model.forms.get(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
# <- edge
if isinstance(form.type, s_types.Edge):
full = form.name + ':n2'
async for node, path in genr:
if self.isjoin:
yield node, path
async for pivo in runt.snap.getNodesBy(full, node.ndef):
yield pivo, path.fork(pivo)
return
# edge <- form
async for node, path in genr:
if self.isjoin:
yield node, path
if not isinstance(node.form.type, s_types.Edge):
continue
# dont bother traversing edges to the wrong form
if node.get('n1:form') != form.name:
continue
n1def = node.get('n1')
pivo = await runt.snap.getNodeByNdef(n1def)
if pivo is None:
continue
yield pivo, path.fork(pivo)
async def addNode(self, name, valu, props=None):
'''
Add a node by form name and value with optional props.
Args:
name (str): The form of node to add.
valu (obj): The value for the node.
props (dict): Optional secondary properties for the node.
Notes:
If a props dictionary is provided, it may be mutated during node construction.
Returns:
s_node.Node: A Node object. It may return None if the snap is unable to add or lift the node.
'''
if self.readonly:
mesg = 'The snapshot is in read-only mode.'
raise s_exc.IsReadOnly(mesg=mesg)
form = self.core.model.form(name)
if form is None:
raise s_exc.NoSuchForm(name=name)
if form.isrunt:
raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.',
form=form.full,
prop=valu)
try:
adds = self.getNodeAdds(form, valu, props=props)
except Exception as e:
if not self.strict:
await self.warn(f'addNode: {e}')
return None
raise
# depth first, so the last one is our added node
nodes = await self.applyNodeEdits(adds)
return nodes[-1]
async def _getNodesByFormTag(self, name, tag, valu=None, cmpr='='):
filt = None
form = self.model.form(name)
if valu is not None:
ctor = self.model.type('ival').getCmprCtor(cmpr)
if ctor is not None:
filt = ctor(valu)
if form is None:
raise s_exc.NoSuchForm(form=name)
tag = s_chop.tag(tag)
# maybe use Encoder here?
fenc = form.name.encode('utf8') + b'\x00'
tenc = b'#' + tag.encode('utf8') + b'\x00'
iops = (('pref', b''), )
lops = (('indx', ('byprop', fenc + tenc, iops)), )
# a small speed optimization...
rawprop = '#' + tag
if filt is None:
async for row, node in self.getLiftNodes(lops, rawprop):
yield node
return
async for row, node in self.getLiftNodes(lops, rawprop):
valu = node.getTag(tag)
if filt(valu):
yield node
def recurse(f, v, p, doadd=True):
edits = []
formnorm, forminfo = f.type.norm(v)
buid = s_common.buid((f.name, formnorm))
if doadd:
edits.append(
(s_layer.EDIT_NODE_ADD, (formnorm, f.type.stortype)))
formsubs = forminfo.get('subs')
if formsubs is not None:
for subname, subvalu in formsubs.items():
p[subname] = subvalu
for propname, propvalu in p.items():
prop = f.prop(propname)
if prop is None:
continue
assert prop.type.stortype is not None
if isinstance(prop.type, s_types.Ndef):
ndefname, ndefvalu = propvalu
ndefform = self.core.model.form(ndefname)
if ndefform is None:
raise s_exc.NoSuchForm(name=ndefname)
for item in recurse(ndefform, ndefvalu, {}):
yield item
if isinstance(prop.type, s_types.Array):
arrayform = self.core.model.form(prop.type.arraytype.name)
if arrayform is not None:
for arrayvalu in propvalu:
for e in recurse(arrayform, arrayvalu, {}):
yield e
propnorm, typeinfo = prop.type.norm(propvalu)
edits.append((s_layer.EDIT_PROP_SET, (propname, propnorm, None,
prop.type.stortype)))
propsubs = typeinfo.get('subs')
if propsubs is not None:
for subname, subvalu in propsubs.items():
fullname = f'{prop.full}:{subname}'
subprop = self.core.model.prop(fullname)
if subprop is None:
continue
assert subprop.type.stortype is not None
subnorm, subinfo = subprop.type.norm(subvalu)
edits.append((s_layer.EDIT_PROP_SET,
(subprop.name, subnorm, None,
subprop.type.stortype)))
propform = self.core.model.form(prop.type.name)
if propform is None:
continue
for item in recurse(propform, propnorm, {}):
yield item
yield (buid, f.name, edits)
async def _getadds(f, p, formnorm, forminfo, doaddnode=True):
if f.locked:
mesg = f'Form {f.full} is locked due to deprecation.'
raise s_exc.IsDeprLocked(mesg=mesg)
edits = [] # Non-primary prop edits
topsubedits = [] # Primary prop sub edits
formsubs = forminfo.get('subs', {})
for subname, subvalu in formsubs.items():
p[subname] = subvalu
for propname, propvalu in p.items():
subedits: s_layer.NodeEditsT = []
prop = f.prop(propname)
if prop is None:
continue
if prop.locked:
mesg = f'Prop {prop.full} is locked due to deprecation.'
if not self.strict:
await self.warn(mesg)
continue
raise s_exc.IsDeprLocked(mesg=mesg)
assert prop.type.stortype is not None
propnorm, typeinfo = prop.type.norm(propvalu)
if isinstance(prop.type, s_types.Ndef):
ndefname, ndefvalu = propvalu
ndefform = self.core.model.form(ndefname)
if ndefform is None:
raise s_exc.NoSuchForm(name=ndefname)
if ndefform.locked:
mesg = f'Form {ndefform.full} is locked due to deprecation.'
if not self.strict:
await self.warn(mesg)
continue
raise s_exc.IsDeprLocked(mesg=mesg)
ndefnorm, ndefinfo = ndefform.type.norm(ndefvalu)
do_subedit = True
if self.buidprefetch:
node = await self.getNodeByBuid(
s_common.buid((ndefform.name, ndefnorm)))
do_subedit = node is None
if do_subedit:
subedits.extend([
x async for x in _getadds(ndefform, {}, ndefnorm,
ndefinfo)
])
elif isinstance(prop.type, s_types.Array):
arrayform = self.core.model.form(prop.type.arraytype.name)
if arrayform is not None:
if arrayform.locked:
mesg = f'Form {arrayform.full} is locked due to deprecation.'
if not self.strict:
await self.warn(mesg)
continue
raise s_exc.IsDeprLocked(mesg=mesg)
for arrayvalu in propnorm:
arraynorm, arrayinfo = arrayform.type.norm(
arrayvalu)
if self.buidprefetch:
node = await self.getNodeByBuid(
s_common.buid((arrayform.name, arraynorm)))
if node is not None:
continue
subedits.extend([
x async for x in _getadds(
arrayform, {}, arraynorm, arrayinfo)
])
propsubs = typeinfo.get('subs')
if propsubs is not None:
for subname, subvalu in propsubs.items():
fullname = f'{prop.full}:{subname}'
subprop = self.core.model.prop(fullname)
if subprop is None:
continue
assert subprop.type.stortype is not None
subnorm, subinfo = subprop.type.norm(subvalu)
edits.append((s_layer.EDIT_PROP_SET,
(subprop.name, subnorm, None,
subprop.type.stortype), ()))
propform = self.core.model.form(prop.type.name)
if propform is not None:
doedit = True
if self.buidprefetch:
node = await self.getNodeByBuid(
s_common.buid((propform.name, propnorm)))
doedit = node is None
if doedit:
subedits.extend([
x async for x in _getadds(propform, {}, propnorm,
typeinfo)
])
edit: s_layer.EditT = (s_layer.EDIT_PROP_SET,
(propname, propnorm, None,
prop.type.stortype), subedits)
if propname in formsubs:
topsubedits.append(edit)
else:
edits.append(edit)
buid = s_common.buid((f.name, formnorm))
if doaddnode:
# Make all the sub edits for the primary property a conditional nodeEdit under a top-level NODE_ADD
# edit
if topsubedits:
subnodeedits: s_layer.NodeEditsT = [(buid, f.name,
topsubedits)]
else:
subnodeedits = ()
topedit: s_layer.EditT = (s_layer.EDIT_NODE_ADD,
(formnorm,
f.type.stortype), subnodeedits)
yield (buid, f.name, [topedit] + edits)
else:
yield (buid, f.name, edits)
def addFormProp(self, formname, propname, tdef, info):
form = self.forms.get(formname)
if form is None:
raise s_exc.NoSuchForm(name=formname)
self._addFormProp(form, propname, tdef, info)
