async def getPropAndValu(self, path): if not self.ispiv: valu = path.node.get(self.name) prop = path.node.form.props.get(self.name) return prop, valu # handle implicit pivot properties names = self.name.split('::') node = path.node imax = len(names) - 1 for i, name in enumerate(names): valu = node.get(name) if valu is None: return None, None prop = node.form.props.get(name) if i >= imax: return prop, valu form = path.runt.snap.model.forms.get(prop.type.name) if form is None: raise s_exc.NoSuchForm(name=prop.type.name) node = await path.runt.snap.getNodeByNdef((form.name, valu)) if node is None: return None, None
async def _getAddNodeEdits(self, name, valu, props=None): form = self.core.model.form(name) if form is None: raise s_exc.NoSuchForm(name=name) if form.isrunt: raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.', form=form.full, prop=valu) if self.buidprefetch: norm, info = form.type.norm(valu) buid = s_common.buid((form.name, norm)) node = await self.getNodeByBuid(buid) if node is not None: if props is not None: return (node, await self.getNodeAdds(form, valu, props=props, addnode=False)) else: return (node, [(buid, form.name, [])]) return (None, await self.getNodeAdds(form, valu, props=props))
def _validateConfig(core, config): maxsize = config.get('maxsize', 10000) if not isinstance(maxsize, int): mesg = f'STIX Bundle config maxsize option must be an integer.' raise s_exc.BadConfValu(mesg=mesg) if maxsize > 10000: mesg = f'STIX Bundle config maxsize option must be <= 10000.' raise s_exc.BadConfValu(mesg=mesg) formmaps = config.get('forms') if formmaps is None: mesg = f'STIX Bundle config is missing "forms" mappings.' raise s_exc.NeedConfValu(mesg=mesg) for formname, formconf in formmaps.items(): form = core.model.form(formname) if form is None: mesg = f'STIX Bundle config contains invalid form name {formname}.' raise s_exc.NoSuchForm(mesg=mesg) stixdef = formconf.get('default') if stixdef is None: mesg = f'STIX Bundle config is missing default mapping for form {formname}.' raise s_exc.NeedConfValu(mesg=mesg) if stixdef not in stix_all: mesg = f'STIX Bundle default mapping ({stixdef}) for {formname} is not a STIX type.' raise s_exc.BadConfValu(mesg=mesg) stixmaps = formconf.get('stix') if stixmaps is None: mesg = f'STIX Bundle config is missing STIX maps for form {formname}.' raise s_exc.NeedConfValu(mesg=mesg) if stixmaps.get(stixdef) is None: mesg = f'STIX Bundle config is missing STIX map for form {formname} default value {stixdef}.' raise s_exc.BadConfValu(mesg=mesg) for stixtype, stixinfo in stixmaps.items(): if stixtype not in stix_all: mesg = f'STIX Bundle config has unknown STIX type {stixtype} for form {formname}.' raise s_exc.BadConfValu(mesg=mesg) stixprops = stixinfo.get('props') if stixprops is not None: for stixprop, stormtext in stixprops.items(): if not isinstance(stormtext, str): mesg = f'STIX Bundle config has invalid prop entry {formname} {stixtype} {stixprop}.' raise s_exc.BadConfValu(mesg=mesg) stixrels = stixinfo.get('rels') if stixrels is not None: for stixrel in stixrels: if len(stixrel) != 3: mesg = f'STIX Bundle config has invalid rel entry {formname} {stixtype} {stixrel}.' raise s_exc.BadConfValu(mesg=mesg)
def repr(self, norm): formname, formvalu = norm form = self.modl.form(formname) if form is None: raise s_exc.NoSuchForm(name=self.name, form=formname) repv = form.type.repr(formvalu) return (formname, repv)
async def addNode(self, name, valu, props=None): ''' Add a node by form name and value with optional props. Args: name (str): The form of node to add. valu (obj): The value for the node. props (dict): Optional secondary properties for the node. Notes: If a props dictionary is provided, it may be mutated during node construction. Returns: s_node.Node: A Node object. It may return None if the snap is unable to add or lift the node. ''' if self.readonly: mesg = 'The snapshot is in read-only mode.' raise s_exc.IsReadOnly(mesg=mesg) form = self.core.model.form(name) if form is None: raise s_exc.NoSuchForm(name=name) if form.isrunt: raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.', form=form.full, prop=valu) try: if self.buidprefetch: norm, info = form.type.norm(valu) node = await self.getNodeByBuid( s_common.buid((form.name, norm))) if node is not None: # TODO implement node.setNodeProps() if props is not None: for p, v in props.items(): await node.set(p, v) return node adds = await self.getNodeAdds(form, valu, props=props) except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only raise except Exception as e: if not self.strict: await self.warn(f'addNode: {e}') return None raise nodes = await self.applyNodeEdits(adds) assert len(nodes) >= 1 # Adds is top-down, so the first node is what we want return nodes[0]
def delForm(self, formname): form = self.forms.get(formname) if form is None: raise s_exc.NoSuchForm(name=formname) if isinstance(form.type, s_types.Array): self.arraysbytype[form.type.arraytype.name].remove(form) self.forms.pop(formname, None) self.props.pop(formname, None)
async def run(self, runt, genr): name = self.kids[0].value() form = runt.snap.model.forms.get(name) if form is None: raise s_exc.NoSuchForm(name=name) # the behavior here is a bit complicated... # single value add (runtime computed per node ) # In the cases below, $hehe is input to the storm runtime vars. # case 1: [ foo:bar="lols" ] # case 2: [ foo:bar=$hehe ] # case 2: [ foo:bar=$lib.func(20, $hehe) ] # case 3: ($foo, $bar) = $hehe [ foo:bar=($foo, $bar) ] # iterative add ( node add is executed once per inbound node ) # case 1: <query> [ foo:bar=(:baz, 20) ] # case 2: <query> [ foo:bar=($node, 20) ] # case 2: <query> $blah=:baz [ foo:bar=($blah, 20) ] if not self.kids[1].isRuntSafe(runt): first = True async for node, path in genr: # must reach back first to trigger sudo / etc if first: runt.allowed('node:add', name) first = False yield node, path valu = await self.kids[1].compute(path) for valu in form.type.getTypeVals(valu): newn = await runt.snap.addNode(name, valu) yield newn, runt.initPath(newn) else: async for node, path in genr: yield node, path runt.allowed('node:add', name) valu = await self.kids[1].runtval(runt) for valu in form.type.getTypeVals(valu): node = await runt.snap.addNode(name, valu) yield node, runt.initPath(node)
def _getNodeFnib(self, name, valu): ''' return a form, norm, info, buid tuple ''' form = self.model.form(name) if form is None: raise s_exc.NoSuchForm(name=name) try: norm, info = form.type.norm(valu) except Exception as e: raise s_exc.BadPropValu(prop=form.name, valu=valu, mesg=str(e)) buid = s_common.buid((form.name, norm)) return form, norm, info, buid
async def getNodeByNdef(self, ndef): ''' Return a single Node() instance by (form,valu) tuple. ''' name, valu = ndef form = self.model.forms.get(name) if form is None: raise s_exc.NoSuchForm(name=name) norm, info = form.type.norm(valu) buid = s_common.buid((form.name, norm)) async with await self.snap() as snap: return await snap.getNodeByBuid(buid)
def delFormProp(self, formname, propname): form = self.forms.get(formname) if form is None: raise s_exc.NoSuchForm(name=formname) prop = form.props.pop(propname, None) if prop is None: raise s_exc.NoSuchProp(name=f'{formname}:{propname}') form.props.pop(prop.name, None) form.defvals.pop(prop.name, None) self.props.pop(prop.full, None) self.props.pop((form.name, prop.name), None) self.propsbytype[prop.type.name].remove(prop)
def delFormProp(self, formname, propname): form = self.forms.get(formname) if form is None: raise s_exc.NoSuchForm(name=formname) prop = form.delProp(propname) if prop is None: raise s_exc.NoSuchProp(name=f'{formname}:{propname}') if isinstance(prop.type, s_types.Array): self.arraysbytype[prop.type.arraytype.name].remove(prop) self.props.pop(prop.full, None) self.props.pop((form.name, prop.name), None) self.propsbytype[prop.type.name].remove(prop)
def _normPyTuple(self, valu): try: formname, formvalu = valu except Exception as e: raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg=str(e)) from None form = self.modl.form(formname) if form is None: raise s_exc.NoSuchForm(name=self.name, form=formname) formnorm, info = form.type.norm(formvalu) norm = (form.name, formnorm) adds = (norm,) subs = {'form': form.name} return norm, {'adds': adds, 'subs': subs}
async def run(self, runt, genr): name = self.kids[0].value() form = runt.snap.model.forms.get(name) if form is None: raise s_exc.NoSuchForm(name=name) # <- edge if isinstance(form.type, s_types.Edge): full = form.name + ':n2' async for node, path in genr: if self.isjoin: yield node, path async for pivo in runt.snap.getNodesBy(full, node.ndef): yield pivo, path.fork(pivo) return # edge <- form async for node, path in genr: if self.isjoin: yield node, path if not isinstance(node.form.type, s_types.Edge): continue # dont bother traversing edges to the wrong form if node.get('n1:form') != form.name: continue n1def = node.get('n1') pivo = await runt.snap.getNodeByNdef(n1def) if pivo is None: continue yield pivo, path.fork(pivo)
async def addNode(self, name, valu, props=None): ''' Add a node by form name and value with optional props. Args: name (str): The form of node to add. valu (obj): The value for the node. props (dict): Optional secondary properties for the node. Notes: If a props dictionary is provided, it may be mutated during node construction. Returns: s_node.Node: A Node object. It may return None if the snap is unable to add or lift the node. ''' if self.readonly: mesg = 'The snapshot is in read-only mode.' raise s_exc.IsReadOnly(mesg=mesg) form = self.core.model.form(name) if form is None: raise s_exc.NoSuchForm(name=name) if form.isrunt: raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.', form=form.full, prop=valu) try: adds = self.getNodeAdds(form, valu, props=props) except Exception as e: if not self.strict: await self.warn(f'addNode: {e}') return None raise # depth first, so the last one is our added node nodes = await self.applyNodeEdits(adds) return nodes[-1]
async def _getNodesByFormTag(self, name, tag, valu=None, cmpr='='): filt = None form = self.model.form(name) if valu is not None: ctor = self.model.type('ival').getCmprCtor(cmpr) if ctor is not None: filt = ctor(valu) if form is None: raise s_exc.NoSuchForm(form=name) tag = s_chop.tag(tag) # maybe use Encoder here? fenc = form.name.encode('utf8') + b'\x00' tenc = b'#' + tag.encode('utf8') + b'\x00' iops = (('pref', b''), ) lops = (('indx', ('byprop', fenc + tenc, iops)), ) # a small speed optimization... rawprop = '#' + tag if filt is None: async for row, node in self.getLiftNodes(lops, rawprop): yield node return async for row, node in self.getLiftNodes(lops, rawprop): valu = node.getTag(tag) if filt(valu): yield node
def recurse(f, v, p, doadd=True): edits = [] formnorm, forminfo = f.type.norm(v) buid = s_common.buid((f.name, formnorm)) if doadd: edits.append( (s_layer.EDIT_NODE_ADD, (formnorm, f.type.stortype))) formsubs = forminfo.get('subs') if formsubs is not None: for subname, subvalu in formsubs.items(): p[subname] = subvalu for propname, propvalu in p.items(): prop = f.prop(propname) if prop is None: continue assert prop.type.stortype is not None if isinstance(prop.type, s_types.Ndef): ndefname, ndefvalu = propvalu ndefform = self.core.model.form(ndefname) if ndefform is None: raise s_exc.NoSuchForm(name=ndefname) for item in recurse(ndefform, ndefvalu, {}): yield item if isinstance(prop.type, s_types.Array): arrayform = self.core.model.form(prop.type.arraytype.name) if arrayform is not None: for arrayvalu in propvalu: for e in recurse(arrayform, arrayvalu, {}): yield e propnorm, typeinfo = prop.type.norm(propvalu) edits.append((s_layer.EDIT_PROP_SET, (propname, propnorm, None, prop.type.stortype))) propsubs = typeinfo.get('subs') if propsubs is not None: for subname, subvalu in propsubs.items(): fullname = f'{prop.full}:{subname}' subprop = self.core.model.prop(fullname) if subprop is None: continue assert subprop.type.stortype is not None subnorm, subinfo = subprop.type.norm(subvalu) edits.append((s_layer.EDIT_PROP_SET, (subprop.name, subnorm, None, subprop.type.stortype))) propform = self.core.model.form(prop.type.name) if propform is None: continue for item in recurse(propform, propnorm, {}): yield item yield (buid, f.name, edits)
async def _getadds(f, p, formnorm, forminfo, doaddnode=True): if f.locked: mesg = f'Form {f.full} is locked due to deprecation.' raise s_exc.IsDeprLocked(mesg=mesg) edits = [] # Non-primary prop edits topsubedits = [] # Primary prop sub edits formsubs = forminfo.get('subs', {}) for subname, subvalu in formsubs.items(): p[subname] = subvalu for propname, propvalu in p.items(): subedits: s_layer.NodeEditsT = [] prop = f.prop(propname) if prop is None: continue if prop.locked: mesg = f'Prop {prop.full} is locked due to deprecation.' if not self.strict: await self.warn(mesg) continue raise s_exc.IsDeprLocked(mesg=mesg) assert prop.type.stortype is not None propnorm, typeinfo = prop.type.norm(propvalu) if isinstance(prop.type, s_types.Ndef): ndefname, ndefvalu = propvalu ndefform = self.core.model.form(ndefname) if ndefform is None: raise s_exc.NoSuchForm(name=ndefname) if ndefform.locked: mesg = f'Form {ndefform.full} is locked due to deprecation.' if not self.strict: await self.warn(mesg) continue raise s_exc.IsDeprLocked(mesg=mesg) ndefnorm, ndefinfo = ndefform.type.norm(ndefvalu) do_subedit = True if self.buidprefetch: node = await self.getNodeByBuid( s_common.buid((ndefform.name, ndefnorm))) do_subedit = node is None if do_subedit: subedits.extend([ x async for x in _getadds(ndefform, {}, ndefnorm, ndefinfo) ]) elif isinstance(prop.type, s_types.Array): arrayform = self.core.model.form(prop.type.arraytype.name) if arrayform is not None: if arrayform.locked: mesg = f'Form {arrayform.full} is locked due to deprecation.' if not self.strict: await self.warn(mesg) continue raise s_exc.IsDeprLocked(mesg=mesg) for arrayvalu in propnorm: arraynorm, arrayinfo = arrayform.type.norm( arrayvalu) if self.buidprefetch: node = await self.getNodeByBuid( s_common.buid((arrayform.name, arraynorm))) if node is not None: continue subedits.extend([ x async for x in _getadds( arrayform, {}, arraynorm, arrayinfo) ]) propsubs = typeinfo.get('subs') if propsubs is not None: for subname, subvalu in propsubs.items(): fullname = f'{prop.full}:{subname}' subprop = self.core.model.prop(fullname) if subprop is None: continue assert subprop.type.stortype is not None subnorm, subinfo = subprop.type.norm(subvalu) edits.append((s_layer.EDIT_PROP_SET, (subprop.name, subnorm, None, subprop.type.stortype), ())) propform = self.core.model.form(prop.type.name) if propform is not None: doedit = True if self.buidprefetch: node = await self.getNodeByBuid( s_common.buid((propform.name, propnorm))) doedit = node is None if doedit: subedits.extend([ x async for x in _getadds(propform, {}, propnorm, typeinfo) ]) edit: s_layer.EditT = (s_layer.EDIT_PROP_SET, (propname, propnorm, None, prop.type.stortype), subedits) if propname in formsubs: topsubedits.append(edit) else: edits.append(edit) buid = s_common.buid((f.name, formnorm)) if doaddnode: # Make all the sub edits for the primary property a conditional nodeEdit under a top-level NODE_ADD # edit if topsubedits: subnodeedits: s_layer.NodeEditsT = [(buid, f.name, topsubedits)] else: subnodeedits = () topedit: s_layer.EditT = (s_layer.EDIT_NODE_ADD, (formnorm, f.type.stortype), subnodeedits) yield (buid, f.name, [topedit] + edits) else: yield (buid, f.name, edits)
def addFormProp(self, formname, propname, tdef, info): form = self.forms.get(formname) if form is None: raise s_exc.NoSuchForm(name=formname) self._addFormProp(form, propname, tdef, info)