def _stormOperAddNode(self, query, oper): args = oper[1].get('args') if len(args) != 2: raise s_common.BadSyntaxError(mesg='addnode(<form>,<valu>,[:<prop>=<pval>, ...])') kwlist = oper[1].get('kwlist') core = self.getStormCore() props = {} for k, v in kwlist: if not k[0] == ':': raise s_common.BadSyntaxError(mesg='addnode() expects relative props with : prefix') prop = k[1:] props[prop] = v node = self.formTufoByProp(args[0], args[1], **props) # call set props if the node is not new... if not node[1].get('.new'): self.setTufoProps(node, **props) query.add(node)
def _stormOperAddNode(self, query, oper): args = oper[1].get('args') if len(args) != 2: raise s_common.BadSyntaxError(mesg='addnode(<form>,<valu>,[:<prop>=<pval>, ...])') kwlist = oper[1].get('kwlist') form = args[0] valu = args[1] core = self.getStormCore() core.reqUserPerm(('node:add', {'form': form}), elev=query.elev) props = {} for k, v in kwlist: if not k[0] == ':': raise s_common.BadSyntaxError(mesg='addnode() expects relative props with : prefix') prop = k[1:] props[prop] = v node = self.formTufoByProp(form, valu, **props) query.add(node)
def _stormOperDelProp(self, query, oper): args = oper[1].get('args') opts = dict(oper[1].get('kwlist')) core = self.getStormCore() if not args: raise s_common.BadSyntaxError(mesg='delprop(<prop>, [force=1]>') prop = args[0] if prop[0] != ':': raise s_common.BadSyntaxError(mesg='delprop(<prop>, [force=1]>') prop = prop.lstrip(':') if not prop: raise s_common.BadSyntaxError(mesg='delprop(<prop>, [force=1]>') force, _ = core.getTypeNorm('bool', opts.get('force', 0)) if not force: return nodes = query.take() [query.add(core.delTufoProp(n, prop)) for n in nodes]
def parse_perm(text, off=0): ''' Parse a permission string <name> [<opt>=<match>...] ''' _, off = nom(text, off, whites) name, off = nom(text, off, varset) if not name: raise s_common.BadSyntaxError(mesg='perm str expected name') retn = (name, {}) _, off = nom(text, off, whites) while len(text) > off: _, off = nom(text, off, whites) meta, off = nom(text, off, varset) _, off = nom(text, off, whites) if not nextchar(text, off, '='): raise s_common.BadSyntaxError(mesg='perm opt expected =') _, off = nom(text, off + 1, whites) valu, off = parse_valu(text, off) if not isinstance(valu, str): raise s_common.BadSyntaxError(mesg='perm opt %s= expected string' % meta) _, off = nom(text, off, whites) retn[1][meta] = valu return retn, off
def _stormOperLimit(self, query, oper): args = oper[1].get('args', ()) if len(args) != 1: raise s_common.BadSyntaxError(mesg='limit(<size>)') size = s_common.intify(args[0]) if size is None: raise s_common.BadSyntaxError(mesg='limit(<size>)') if query.size() > size: [query.add(node) for node in query.take()[:size]]
def parse_list(text, off=0, trim=True): ''' Parse a list (likely for comp type) coming from a command line input. The string elements within the list may optionally be quoted. ''' if not nextchar(text, off, '('): raise s_common.BadSyntaxError(at=off, mesg='expected open paren for list') off += 1 valus = [] while off < len(text): _, off = nom(text, off, whites) valu, off = parse_valu(text, off) _, off = nom(text, off, whites) # check for foo=bar kw tuple syntax if nextchar(text, off, '='): _, off = nom(text, off + 1, whites) vval, off = parse_valu(text, off) _, off = nom(text, off, whites) valu = (valu, vval) valus.append(valu) _, off = nom_whitespace(text, off) if nextchar(text, off, ')'): return valus, off + 1 if not nextchar(text, off, ','): raise s_common.BadSyntaxError(at=off, text=text, mesg='expected comma in list') off += 1 raise s_common.BadSyntaxError(at=off, mesg='unexpected and of text during list')
def parse_oper(text, off=0): ''' Returns an inst,off tuple by parsing an operator expression. Example: inst,off = parse_oper('foo("bar",baz=20)') ''' name, off = nom(text, off, varset) inst = (name, {'args': [], 'kwlist': []}) _, off = nom(text, off, whites) if not nextchar(text, off, '('): raise s_common.BadSyntaxError(expected='( for operator ' + name, at=off) off += 1 while True: _, off = nom(text, off, whites) if nextchar(text, off, ')'): off += 1 return inst, off valu, off = parse_valu(text, off) _, off = nom(text, off, whites) if nextchar(text, off, '='): vval, off = parse_valu(text, off + 1) inst[1]['kwlist'].append((valu, vval)) else: inst[1]['args'].append(valu) if not nextin(text, off, [',', ')']): raise s_common.BadSyntaxError(mesg='Unexpected Token: ' + text[off], at=off) if nextchar(text, off, ','): off += 1
def parse_string(text, off, trim=True): if text[off] not in ('"', "'"): # lulz... raise s_common.BadSyntaxError(expected='String Literal', at=off) quot = text[off] if trim: _, off = nom(text, off, whites) off += 1 vals = [] while text[off] != quot: c = text[off] off += 1 if c == '\\': c = text[off] off += 1 vals.append(c) off += 1 if trim: _, off = nom(text, off, whites) return ''.join(vals), off
def _stormOperLift(self, query, oper): args = oper[1].get('args') opts = dict(oper[1].get('kwlist')) if len(args) not in (1, 2): raise s_common.BadSyntaxError( mesg='lift(<prop> [,<valu>, by=<by>, limit=<limit>])') valu = None prop = args[0] if len(args) == 2: valu = args[1] by = opts.get('by', 'has') if by == 'has' and valu is not None: by = 'eq' limt0 = opts.get('limit') limt1 = query.opt('limit') limit = self.getLiftLimit(limt0, limt1) [ query.add(tufo) for tufo in self.stormTufosBy(by, prop, valu, limit=limit) ]
def parse_int(text, off, trim=True): numstr, off = nom(text, off, intset, trim=trim) try: return int(numstr, 0), off except Exception as e: raise s_common.BadSyntaxError(expected='Literal', at=off, got=text[off:off + 10])
def parse_stormsub(text, off=0): _, off = nom(text, off, whites) if not nextchar(text, off, '{'): raise s_common.BadSyntaxError('expected { at %d' % (off,)) _, off = nom(text, off + 1, whites) opers, off = parse_storm(text, off) if not nextchar(text, off, '}'): raise s_common.BadSyntaxError('expected } at %d' % (off,)) _, off = nom(text, off + 1, whites) return opers, off
def _stormOperSetProp(self, query, oper): # Coverage of this function is affected by the following issue: # https://bitbucket.org/ned/coveragepy/issues/198/continue-marked-as-not-covered args = oper[1].get('args') props = dict(oper[1].get('kwlist')) core = self.getStormCore() formnodes = collections.defaultdict(list) formprops = collections.defaultdict(dict) for node in query.data(): formnodes[node[1].get('tufo:form')].append(node) forms = tuple(formnodes.keys()) for prop, valu in props.items(): if prop.startswith(':'): valid = False _prop = prop[1:] # Check against every lifted form, since we may have a relative prop # Which is valid against for form in forms: _fprop = form + prop if core.isSetPropOk(_fprop, isadd=True): formprops[form][_prop] = valu valid = True if not valid: mesg = 'Relative prop is not valid on any lifted forms.' raise s_common.BadSyntaxError(name=prop, mesg=mesg) continue # pragma: no cover mesg = 'setprop operator requires props to start with relative prop names.' raise s_common.BadSyntaxError(name=prop, mesg=mesg) for form, nodes in formnodes.items(): props = formprops.get(form) if props: for prop in props.keys(): perm = ('node:prop:set', {'form': form, 'prop': prop}) core.reqUserPerm(perm, elev=query.elev) [core.setTufoProps(node, **props) for node in nodes]
def parse(text, off=0): ''' Parse and return a set of instruction tufos. ''' retn, off = parse_storm(text, off=off) if off != len(text): raise s_common.BadSyntaxError('trailing text: %s' % (text[off:],)) return retn
def parse_int(text, off, trim=True): _, off = nom(text, off, whites) neg = False if nextchar(text, off, '-'): neg = True _, off = nom(text, off + 1, whites) valu = None if nextstr(text, off, '0x'): valu, off = nom(text, off + 2, hexset) if not valu: raise s_common.BadSyntaxError(at=off, mesg='0x expected hex') valu = int(valu, 16) elif nextstr(text, off, '0b'): valu, off = nom(text, off + 2, binset) if not valu: raise s_common.BadSyntaxError(at=off, mesg='0b expected bits') valu = int(valu, 2) else: valu, off = nom(text, off, decset) if not valu: raise s_common.BadSyntaxError(at=off, mesg='expected digits') if not nextchar(text, off, '.'): valu = int(valu) else: frac, off = nom(text, off + 1, decset) valu = float('%s.%s' % (valu, frac)) if neg: valu = -valu return valu, off
def _stormOperTask(self, query, oper): args = oper[1].get('args') opts = dict(oper[1].get('kwlist')) if not args: mesg = 'task(<taskname1>, <taskname2>, ..., [kwarg1=val1, ...])' raise s_common.BadSyntaxError(mesg=mesg) nodes = query.data() core = self.getStormCore() for tname in args: evt = ':'.join(['task', tname]) core.fire(evt, nodes=nodes, storm=True, **opts)
def parse_float(text, off, trim=True): _, off = nom(text, off, whites) valu = '' if nextchar(text, off, '-'): valu += '-' _, off = nom(text, off + 1, whites) digs, off = nom(text, off, decset) if not digs: raise s_common.BadSyntaxError(at=off, mesg='expected digits') valu += digs if nextchar(text, off, '.'): frac, off = nom(text, off + 1, decset) if not frac: raise s_common.BadSyntaxError(at=off, mesg='expected .<digits>') valu = valu + '.' + frac return float(valu), off
def _stormOperAddXref(self, query, oper): args = oper[1].get('args') if len(args) != 3: raise s_common.BadSyntaxError(mesg='addxref(<type>,<form>,<valu>)') xref, form, valu = args core = self.getStormCore() # TODO clearer error handling for node in query.take(): sorc = node[1].get(node[1].get('tufo:form')) node = core.formTufoByProp(xref, (sorc, (form, valu))) query.add(node)
def parse_cmd_kwarg(text, off=0): ''' Parse a foo:bar=<valu> kwarg into (prop,valu),off ''' _, off = nom(text, off, whites) prop, off = nom(text, off, varset) _, off = nom(text, off, whites) if not nextchar(text, off, '='): raise s_common.BadSyntaxError(expected='= for kwarg ' + prop, at=off) _, off = nom(text, off + 1, whites) valu, off = parse_cmd_string(text, off) return (prop, valu), off
def _stormOperPivot(self, query, oper): args = oper[1].get('args') opts = dict(oper[1].get('kwlist')) if len(args) is 1: srcp, dstp = None, args[0] elif len(args) is 2: srcp, dstp = args[0], args[1] else: raise s_common.BadSyntaxError(mesg='pivot(<srcprop>,<dstprop>)') limit = opts.get('limit') if limit is not None and limit < 0: raise s_common.BadOperArg(oper='pivot', name='limit', mesg='must be >= 0') self._runPivotOper(query, srcp, dstp, limit)
def _stormOperNextTag(self, query, oper): name = None args = oper[1].get('args') kwargs = dict(oper[1].get('kwlist')) doc = kwargs.get('doc', '??') name = kwargs.get('core') if len(args) != 1: raise s_common.BadSyntaxError(mesg='nexttag(<tagname>,doc=<doc>)') tag = args[0] core = self.getStormCore(name=name) valu = core.nextSeqValu(tag) node = core.formTufoByProp('syn:tag', valu, doc=doc) query.add(node)
def tokenize(text): ''' Produce a token list from the given text string. [ (<tok>,<info>), ... ] ''' off = 0 tlen = len(text) toks = [] while off < tlen: _, off = s_syntax.nom_whitespace(text, off) if off >= tlen: break if s_syntax.nextin(text, off, '"\''): tokn = ('valu', {'off': off, 'type': 'str'}) tokn[1]['valu'], off = s_syntax.parse_string(text, off, trim=False) tokn[1]['end'] = off toks.append(tokn) continue if s_syntax.nextin(text, off, '0123456789'): tokn = ('valu', {'off': off, 'type': 'int'}) tokn[1]['valu'], off = s_syntax.parse_int(text, off) tokn[1]['end'] = off toks.append(tokn) continue tokdone = False for tok in tokstrs: if text.startswith(tok, off): tokn = (tok, {'off': off}) off += len(tok) tokn[1]['end'] = off toks.append(tokn) tokdone = True break if tokdone: continue if not s_syntax.nextin(text, off, varset): raise s_common.BadSyntaxError(at=off, mesg='no valid tokens found') tokn = ('var', {'off': off}) tokn[1]['name'], off = s_syntax.nom(text, off, varset, trim=False) toks.append(tokn) for tokn in toks: tokn[1].update(tokninfo.get(tokn[0], {})) return toks
def parse_ques(text, off=0, trim=True): ''' Parse "query" syntax: tag/prop[@<timewin>][^<limit>][*<by>][=valu] ''' ques = {} name, off = nom(text, off, varset, trim=True) if not name: raise s_common.BadSyntaxError(text=text, off=off, mesg='expected name') ques['cmp'] = 'has' ques['prop'] = name _, off = nom(text, off, whites) if len(text) == off: return ques, off if text[off] == '/': ques['from'] = name off += 1 name, off = nom(text, off, varset, trim=True) ques['prop'] = name _, off = nom(text, off, whites) while True: _, off = nom(text, off, whites) if len(text) == off: return ques, off if text[off] == '^': ques['limit'], off = parse_int(text, off + 1, trim=True) continue # NOTE: "by" macro syntax only supports eq so we eat and run if nextchar(text, off, '*'): _, off = nom(text, off + 1, whites) ques['cmp'], off = nom(text, off, varset, trim=True) if len(text) == off: return ques, off if not nextchar(text, off, '='): raise s_common.BadSyntaxError(text=text, off=off, mesg='expected equals for by syntax') _, off = nom(text, off + 1, whites) ques['valu'], off = parse_valu(text, off) return ques, off if nextchar(text, off, '='): _, off = nom(text, off + 1, whites) ques['cmp'] = 'eq' ques['valu'], off = parse_valu(text, off) break textpart = text[off:] for ctxt, cmpr in macrocmps: if textpart.startswith(ctxt): ques['cmp'] = cmpr ques['valu'], off = parse_valu(text, off + len(ctxt)) break break return ques, off
def parse_storm(text, off=0): ret = [] while True: # leading whitespace is irrelevant _, off = nom(text, off, whites) if off >= len(text): break # handle a sub-query terminator if nextchar(text, off, '}'): break # handle $foo.bar={<subquery>} syntax if nextchar(text, off, '$'): _, off = nom(text, off + 1, whites) if not nextin(text, off, alphaset): raise s_common.BadSyntaxError(msg='Set variables must start with an alpha char') name, off = nom(text, off, setset) _, off = nom(text, off, whites) # set load syntax goes here... if not nextchar(text, off, '='): raise s_common.BadSyntaxError(msg='expected = at %d' % (off,)) _, off = nom(text, off + 1, whites) if not nextchar(text, off, '{'): raise s_common.BadSyntaxError('expected { at %d' % (off,)) opers, off = parse_stormsub(text, off) ret.append(oper('set', name, opers)) continue # [ ] for node modification macro syntax # [ inet:fqdn=woot.com ] == addnode(inet:fqdn,woot.com) # [ :asn=10 ] == setprop(asn=10) # [ #foo.bar ] or [ +#foo.bar ] == addtag(foo.bar) # [ -#foo.bar ] == deltag(foo.bar) if nextchar(text, off, '['): off += 1 while True: _, off = nom(text, off, whites) if nextchar(text, off, ']'): off += 1 break if off == len(text): raise s_common.BadSyntaxError(mesg='unexpected end of text in edit mode') if nextstr(text, off, '+#'): valu, off = parse_valu(text, off + 2) ret.append(oper('addtag', valu)) continue if nextstr(text, off, '-#'): valu, off = parse_valu(text, off + 2) ret.append(oper('deltag', valu)) continue if nextchar(text, off, '#'): valu, off = parse_valu(text, off + 1) ret.append(oper('addtag', valu)) continue if nextstr(text, off, '-:'): valu, off = parse_valu(text, off + 1) ret.append(oper('delprop', valu, force=1)) continue # otherwise, it should be a prop=valu (maybe relative) prop, off = nom(text, off, propset) if not prop: raise s_common.BadSyntaxError(mesg='edit macro expected prop=valu syntax') _, off = nom(text, off, whites) if not nextchar(text, off, '='): raise s_common.BadSyntaxError(mesg='edit macro expected prop=valu syntax') valu, off = parse_valu(text, off + 1) if prop[0] == ':': kwargs = {prop: valu} ret.append(oper('setprop', **kwargs)) continue ret.append(oper('addnode', prop, valu)) continue # pivot() macro with no src prop: -> foo:bar if nextstr(text, off, '->'): _, off = nom(text, off + 2, whites) name, off = nom(text, off, varset) ret.append(oper('pivot', name)) continue # join() macro with no src prop: <- foo:bar if nextstr(text, off, '<-'): _, off = nom(text, off + 2, whites) name, off = nom(text, off, varset) ret.append(oper('join', name)) continue # lift by tag alone macro if nextstr(text, off, '#'): _, off = nom(text, off + 1, whites) name, off = nom(text, off, varset) ret.append(oper('alltag', name)) continue # must() macro syntax: +foo:bar="woot" if nextchar(text, off, '+'): _, off = nom(text, off + 1, whites) # subquery filter syntax if nextchar(text, off, '{'): opers, off = parse_stormsub(text, off) ret.append(oper('filtsub', True, opers)) continue inst, off = parse_macro_filt(text, off, mode='must') ret.append(inst) continue # cant() macro syntax: -foo:bar=10 if nextchar(text, off, '-'): _, off = nom(text, off + 1, whites) # subquery filter syntax if nextchar(text, off, '{'): opers, off = parse_stormsub(text, off) ret.append(oper('filtsub', False, opers)) continue inst, off = parse_macro_filt(text, off, mode='cant') ret.append(inst) continue # logical and syntax for multiple filters if text[off] == '&': if len(ret) == 0: raise s_common.BadSyntaxError(mesg='logical and with no previous operator') prev = ret[-1] if prev[0] != 'filt': raise s_common.BadSyntaxError(mesg='prev oper must be filter not: %r' % prev) mode = prev[1].get('mode') inst, off = parse_macro_filt(text, off + 1, mode=mode) if prev[1].get('cmp') != 'and': prev = ('filt', {'args': [prev, ], 'kwlist': [], 'cmp': 'and', 'mode': mode}) ret[-1] = prev prev[1]['args'].append(inst) continue # logical or syntax for multiple filters if text[off] == '|': if len(ret) == 0: raise s_common.BadSyntaxError(mesg='logical or with no previous operator') prev = ret[-1] if prev[0] != 'filt': raise s_common.BadSyntaxError(mesg='prev oper must be filter not: %r' % prev) mode = prev[1].get('mode') inst, off = parse_macro_filt(text, off + 1, mode=mode) if prev[1].get('cmp') != 'or': prev = ('filt', {'args': [prev, ], 'kwlist': [], 'cmp': 'or', 'mode': mode}) ret[-1] = prev prev[1]['args'].append(inst) continue # opts() macro syntax: %uniq=0 %limit=30 if text[off] == '%': inst, off = parse_opts(text, off + 1) ret.append(inst) continue origoff = off name, off = nom(text, off, varset) _, off = nom(text, off, whites) # mop up in the case where we end with a macro if len(text) == off: inst, off = parse_macro_lift(text, origoff) ret.append(inst) continue # macro foo:bar->baz:faz prop pivot if nextstr(text, off, '->'): pivn, off = nom(text, off + 2, varset) inst = ('pivot', {'args': [name], 'kwlist': []}) # FIXME make a parser for foo.bar/baz:faz*blah#40 if nextchar(text, off, '/'): inst[1]['kwlist'].append(('from', pivn)) pivn, off = nom(text, off + 1, varset) inst[1]['args'].append(pivn) ret.append(inst) continue # macro foo:bar<-baz:faz prop join if nextstr(text, off, '<-'): joinn, off = nom(text, off + 2, varset) inst = ('join', {'args': [name], 'kwlist': []}) # FIXME make a parser for foo.bar/baz:faz*blah#40 if nextchar(text, off, '/'): inst[1]['kwlist'].append(('from', joinn)) joinn, off = nom(text, off + 1, varset) inst[1]['args'].append(joinn) ret.append(inst) continue # standard foo() oper syntax if nextchar(text, off, '('): inst, off = parse_oper(text, origoff) ret.append(inst) continue # only macro lift syntax remains inst, off = parse_macro_lift(text, origoff) ret.append(inst) return ret, off
def getCmdOpts(self, text): ''' Use the _cmd_syntax def to split/parse/normalize the cmd line. NOTE: This is implemented indepedent of argparse (et.al) due to the need for syntax aware argument splitting. ( also, allows different split per command type ) ''' off = 0 _, off = s_syntax.nom(text, off, s_syntax.whites) name, off = s_syntax.meh(text, off, s_syntax.whites) _, off = s_syntax.nom(text, off, s_syntax.whites) opts = {} args = collections.deque( [synt for synt in self._cmd_syntax if not synt[0].startswith('-')]) switches = { synt[0]: synt for synt in self._cmd_syntax if synt[0].startswith('-') } # populate defaults and lists for synt in self._cmd_syntax: snam = synt[0].strip('-') defval = synt[1].get('defval') if defval is not None: opts[snam] = defval if synt[1].get('type') in ('list', 'kwlist'): opts[snam] = [] def atswitch(t, o): # check if we are at a recognized switch. if not # assume the data is part of regular arguments. if not text.startswith('-', o): return None, o name, x = s_syntax.meh(t, o, s_syntax.whites) swit = switches.get(name) if swit is None: return None, o return swit, x while off < len(text): _, off = s_syntax.nom(text, off, s_syntax.whites) swit, off = atswitch(text, off) if swit is not None: styp = swit[1].get('type', 'flag') snam = swit[0].strip('-') if styp == 'valu': valu, off = s_syntax.parse_cmd_string(text, off) opts[snam] = valu elif styp == 'list': valu, off = s_syntax.parse_cmd_string(text, off) if not isinstance(valu, list): valu = valu.split(',') opts[snam].extend(valu) elif styp == 'enum': vals = swit[1].get('enum:vals') valu, off = s_syntax.parse_cmd_string(text, off) if valu not in vals: raise s_common.BadSyntaxError( mesg='%s (%s)' % (swit[0], '|'.join(vals)), text=text) opts[snam] = valu else: opts[snam] = True continue if not args: raise s_common.BadSyntaxError(mesg='trailing text: [%s]' % (text[off:], ), text=text) synt = args.popleft() styp = synt[1].get('type', 'valu') # a glob type eats the remainder of the string if styp == 'glob': opts[synt[0]] = text[off:] break # eat the remainder of the string as separate vals if styp == 'list': valu = [] while off < len(text): item, off = s_syntax.parse_cmd_string(text, off) valu.append(item) opts[synt[0]] = valu break if styp == 'kwlist': kwlist, off = s_syntax.parse_cmd_kwlist(text, off) opts[snam] = kwlist break valu, off = s_syntax.parse_cmd_string(text, off) opts[synt[0]] = valu return opts
def _stormOperTree(self, query, oper): args = oper[1].get('args') opts = dict(oper[1].get('kwlist')) if not args: raise s_common.BadSyntaxError(mesg='tree([<srcprop>], <destprop>, [recurlim=<limit>])') limt = self.getLiftLimitHelp(opts.get('limit')) core = self.getStormCore() # Prevent infinite pivots try: recurlim, _ = core.getTypeNorm('int', opts.get('recurlim', 20)) except s_common.BadTypeValu as e: raise s_common.BadOperArg(oper='tree', name='recurlim', mesg=e.errinfo.get('mesg')) if recurlim < 0: raise s_common.BadOperArg(oper='tree', name='recurlim', mesg='must be >= 0') srcp = None dstp = args[0] if len(args) > 1: srcp = args[0] dstp = args[1] # do we have a relative source property? relsrc = srcp is not None and srcp.startswith(':') tufs = query.data() queried_vals = set() while True: vals = set() if srcp is not None and not relsrc: for tufo in tufs: valu = tufo[1].get(srcp) if valu is not None: vals.add(valu) elif not relsrc: for tufo in tufs: form, valu = s_tufo.ndef(tufo) if valu is not None: vals.add(valu) else: for tufo in tufs: form, _ = s_tufo.ndef(tufo) valu = tufo[1].get(form + srcp) if valu is not None: vals.add(valu) qvals = list(vals - queried_vals) if not qvals: break nodes = core.stormTufosBy('in', dstp, qvals, limit=limt.get()) [query.add(n) for n in nodes] if limt.dec(len(nodes)): break queried_vals = queried_vals.union(vals) if recurlim > 0: recurlim -= 1 if recurlim < 1: break tufs = query.data()
def parse(text, off=0): ''' Parse and return a set of instruction tufos. ''' ret = [] while True: # leading whitespace is irrelevant _, off = nom(text, off, whites) if off >= len(text): break # handle some special "macro" style syntaxes # [ ] for node modification macro syntax # [ inet:fqdn=woot.com ] == addnode(inet:fqdn,woot.com) # [ :asn=10 ] == setprop(asn=10) # [ #foo.bar ] or [ +#foo.bar ] == addtag(foo.bar) # [ -#foo.bar ] == deltag(foo.bar) if nextchar(text, off, '['): off += 1 while True: _, off = nom(text, off, whites) if nextchar(text, off, ']'): off += 1 break if off == len(text): raise s_common.BadSyntaxError(mesg='unexpected end of text in edit mode') if nextstr(text, off, '+#'): valu, off = parse_valu(text, off + 2) ret.append(oper('addtag', valu)) continue if nextstr(text, off, '-#'): valu, off = parse_valu(text, off + 2) ret.append(oper('deltag', valu)) continue if nextchar(text, off, '#'): valu, off = parse_valu(text, off + 1) ret.append(oper('addtag', valu)) continue # otherwise, it should be a prop=valu (maybe relative) prop, off = nom(text, off, propset) if not prop: raise s_common.BadSyntaxError(mesg='edit macro expected prop=valu syntax') _, off = nom(text, off, whites) if not nextchar(text, off, '='): raise s_common.BadSyntaxError(mesg='edit macro expected prop=valu syntax') valu, off = parse_valu(text, off + 1) if prop[0] == ':': kwargs = {prop: valu} ret.append(oper('setprop', **kwargs)) continue ret.append(oper('addnode', prop, valu)) continue # pivot() macro with no src prop: -> foo:bar if nextstr(text, off, '->'): _, off = nom(text, off + 2, whites) name, off = nom(text, off, varset) ret.append(oper('pivot', name)) continue # lift by tag alone macro if nextstr(text, off, '#'): _, off = nom(text, off + 1, whites) name, off = nom(text, off, varset) ret.append(oper('alltag', name)) continue # must() macro syntax: +foo:bar="woot" if nextchar(text, off, '+'): inst, off = parse_macro_filt(text, off + 1, mode='must') ret.append(inst) continue # cant() macro syntax: -foo:bar=10 if nextchar(text, off, '-'): inst, off = parse_macro_filt(text, off + 1, mode='cant') ret.append(inst) continue # logical and syntax for multiple filters if text[off] == '&': if len(ret) == 0: raise s_common.BadSyntaxError(mesg='logical and with no previous operator') prev = ret[-1] if prev[0] != 'filt': raise s_common.BadSyntaxError(mesg='prev oper must be filter not: %r' % prev) mode = prev[1].get('mode') inst, off = parse_macro_filt(text, off + 1, mode=mode) if prev[1].get('cmp') != 'and': prev = ('filt', {'args': [prev, ], 'kwlist': [], 'cmp': 'and', 'mode': mode}) ret[-1] = prev prev[1]['args'].append(inst) continue # logical or syntax for multiple filters if text[off] == '|': if len(ret) == 0: raise s_common.BadSyntaxError(mesg='logical or with no previous operator') prev = ret[-1] if prev[0] != 'filt': raise s_common.BadSyntaxError(mesg='prev oper must be filter not: %r' % prev) mode = prev[1].get('mode') inst, off = parse_macro_filt(text, off + 1, mode=mode) if prev[1].get('cmp') != 'or': prev = ('filt', {'args': [prev, ], 'kwlist': [], 'cmp': 'or', 'mode': mode}) ret[-1] = prev prev[1]['args'].append(inst) continue # opts() macro syntax: %uniq=0 %limit=30 if text[off] == '%': inst, off = parse_opts(text, off + 1) ret.append(inst) continue origoff = off name, off = nom(text, off, varset) _, off = nom(text, off, whites) # mop up in the case where we end with a macro if len(text) == off: inst, off = parse_macro_lift(text, origoff) ret.append(inst) continue # macro foo:bar->baz:faz prop pivot if nextstr(text, off, '->'): pivn, off = nom(text, off + 2, varset) inst = ('pivot', {'args': [name], 'kwlist': []}) # FIXME make a parser for foo.bar/baz:faz*blah#40 if nextchar(text, off, '/'): inst[1]['kwlist'].append(('from', pivn)) pivn, off = nom(text, off + 1, varset) inst[1]['args'].append(pivn) ret.append(inst) continue # standard foo() oper syntax if nextchar(text, off, '('): inst, off = parse_oper(text, origoff) ret.append(inst) continue # only macro lift syntax remains inst, off = parse_macro_lift(text, origoff) ret.append(inst) #[ i[1]['kwlist'].sort() for i in ret ] return ret
def raisetok(tokn, mesg): off = tokn[1].get('off') raise s_common.BadSyntaxError(mesg=mesg, off=off)
def _stormOperTree(self, query, oper): args = oper[1].get('args') opts = dict(oper[1].get('kwlist')) if not args: raise s_common.BadSyntaxError(mesg='tree([<srcprop>], <destprop>, [recurlim=<limit>])') core = self.getStormCore() # Prevent infinite pivots recurlim, _ = core.getTypeNorm('int', opts.get('recurlim', 20)) srcp = None dstp = args[0] if len(args) > 1: srcp = args[0] dstp = args[1] # do we have a relative source property? relsrc = srcp is not None and srcp.startswith(':') tufs = query.data() queried_vals = set() while True: vals = set() if srcp is not None and not relsrc: for tufo in tufs: valu = tufo[1].get(srcp) if valu is not None: vals.add(valu) elif not relsrc: for tufo in tufs: form, valu = s_tufo.ndef(tufo) if valu is not None: vals.add(valu) else: for tufo in tufs: form, _ = s_tufo.ndef(tufo) valu = tufo[1].get(form + srcp) if valu is not None: vals.add(valu) qvals = list(vals - queried_vals) if not qvals: break [query.add(t) for t in self.stormTufosBy('in', dstp, qvals, limit=opts.get('limit'))] queried_vals = queried_vals.union(vals) if recurlim: recurlim -= 1 if recurlim < 1: break tufs = query.data()