def _stormOperAddNode(self, query, oper):
args = oper[1].get('args')
if len(args) != 2:
raise s_common.BadSyntaxError(mesg='addnode(<form>,<valu>,[:<prop>=<pval>, ...])')
kwlist = oper[1].get('kwlist')
core = self.getStormCore()
props = {}
for k, v in kwlist:
if not k[0] == ':':
raise s_common.BadSyntaxError(mesg='addnode() expects relative props with : prefix')
prop = k[1:]
props[prop] = v
node = self.formTufoByProp(args[0], args[1], **props)
# call set props if the node is not new...
if not node[1].get('.new'):
self.setTufoProps(node, **props)
query.add(node)
def _stormOperAddNode(self, query, oper):
args = oper[1].get('args')
if len(args) != 2:
raise s_common.BadSyntaxError(mesg='addnode(<form>,<valu>,[:<prop>=<pval>, ...])')
kwlist = oper[1].get('kwlist')
form = args[0]
valu = args[1]
core = self.getStormCore()
core.reqUserPerm(('node:add', {'form': form}), elev=query.elev)
props = {}
for k, v in kwlist:
if not k[0] == ':':
raise s_common.BadSyntaxError(mesg='addnode() expects relative props with : prefix')
prop = k[1:]
props[prop] = v
node = self.formTufoByProp(form, valu, **props)
query.add(node)
def _stormOperDelProp(self, query, oper):
args = oper[1].get('args')
opts = dict(oper[1].get('kwlist'))
core = self.getStormCore()
if not args:
raise s_common.BadSyntaxError(mesg='delprop(<prop>, [force=1]>')
prop = args[0]
if prop[0] != ':':
raise s_common.BadSyntaxError(mesg='delprop(<prop>, [force=1]>')
prop = prop.lstrip(':')
if not prop:
raise s_common.BadSyntaxError(mesg='delprop(<prop>, [force=1]>')
force, _ = core.getTypeNorm('bool', opts.get('force', 0))
if not force:
return
nodes = query.take()
[query.add(core.delTufoProp(n, prop)) for n in nodes]
def parse_perm(text, off=0):
'''
Parse a permission string
<name> [<opt>=<match>...]
'''
_, off = nom(text, off, whites)
name, off = nom(text, off, varset)
if not name:
raise s_common.BadSyntaxError(mesg='perm str expected name')
retn = (name, {})
_, off = nom(text, off, whites)
while len(text) > off:
_, off = nom(text, off, whites)
meta, off = nom(text, off, varset)
_, off = nom(text, off, whites)
if not nextchar(text, off, '='):
raise s_common.BadSyntaxError(mesg='perm opt expected =')
_, off = nom(text, off + 1, whites)
valu, off = parse_valu(text, off)
if not isinstance(valu, str):
raise s_common.BadSyntaxError(mesg='perm opt %s= expected string' % meta)
_, off = nom(text, off, whites)
retn[1][meta] = valu
return retn, off
def _stormOperLimit(self, query, oper):
args = oper[1].get('args', ())
if len(args) != 1:
raise s_common.BadSyntaxError(mesg='limit(<size>)')
size = s_common.intify(args[0])
if size is None:
raise s_common.BadSyntaxError(mesg='limit(<size>)')
if query.size() > size:
[query.add(node) for node in query.take()[:size]]
def parse_list(text, off=0, trim=True):
'''
Parse a list (likely for comp type) coming from a command line input.
The string elements within the list may optionally be quoted.
'''
if not nextchar(text, off, '('):
raise s_common.BadSyntaxError(at=off,
mesg='expected open paren for list')
off += 1
valus = []
while off < len(text):
_, off = nom(text, off, whites)
valu, off = parse_valu(text, off)
_, off = nom(text, off, whites)
# check for foo=bar kw tuple syntax
if nextchar(text, off, '='):
_, off = nom(text, off + 1, whites)
vval, off = parse_valu(text, off)
_, off = nom(text, off, whites)
valu = (valu, vval)
valus.append(valu)
_, off = nom_whitespace(text, off)
if nextchar(text, off, ')'):
return valus, off + 1
if not nextchar(text, off, ','):
raise s_common.BadSyntaxError(at=off,
text=text,
mesg='expected comma in list')
off += 1
raise s_common.BadSyntaxError(at=off,
mesg='unexpected and of text during list')
def parse_oper(text, off=0):
'''
Returns an inst,off tuple by parsing an operator expression.
Example:
inst,off = parse_oper('foo("bar",baz=20)')
'''
name, off = nom(text, off, varset)
inst = (name, {'args': [], 'kwlist': []})
_, off = nom(text, off, whites)
if not nextchar(text, off, '('):
raise s_common.BadSyntaxError(expected='( for operator ' + name,
at=off)
off += 1
while True:
_, off = nom(text, off, whites)
if nextchar(text, off, ')'):
off += 1
return inst, off
valu, off = parse_valu(text, off)
_, off = nom(text, off, whites)
if nextchar(text, off, '='):
vval, off = parse_valu(text, off + 1)
inst[1]['kwlist'].append((valu, vval))
else:
inst[1]['args'].append(valu)
if not nextin(text, off, [',', ')']):
raise s_common.BadSyntaxError(mesg='Unexpected Token: ' +
text[off],
at=off)
if nextchar(text, off, ','):
off += 1
def parse_string(text, off, trim=True):
if text[off] not in ('"', "'"): # lulz...
raise s_common.BadSyntaxError(expected='String Literal', at=off)
quot = text[off]
if trim:
_, off = nom(text, off, whites)
off += 1
vals = []
while text[off] != quot:
c = text[off]
off += 1
if c == '\\':
c = text[off]
off += 1
vals.append(c)
off += 1
if trim:
_, off = nom(text, off, whites)
return ''.join(vals), off
def _stormOperLift(self, query, oper):
args = oper[1].get('args')
opts = dict(oper[1].get('kwlist'))
if len(args) not in (1, 2):
raise s_common.BadSyntaxError(
mesg='lift(<prop> [,<valu>, by=<by>, limit=<limit>])')
valu = None
prop = args[0]
if len(args) == 2:
valu = args[1]
by = opts.get('by', 'has')
if by == 'has' and valu is not None:
by = 'eq'
limt0 = opts.get('limit')
limt1 = query.opt('limit')
limit = self.getLiftLimit(limt0, limt1)
[
query.add(tufo)
for tufo in self.stormTufosBy(by, prop, valu, limit=limit)
]
def parse_int(text, off, trim=True):
numstr, off = nom(text, off, intset, trim=trim)
try:
return int(numstr, 0), off
except Exception as e:
raise s_common.BadSyntaxError(expected='Literal',
at=off,
got=text[off:off + 10])
def parse_stormsub(text, off=0):
_, off = nom(text, off, whites)
if not nextchar(text, off, '{'):
raise s_common.BadSyntaxError('expected { at %d' % (off,))
_, off = nom(text, off + 1, whites)
opers, off = parse_storm(text, off)
if not nextchar(text, off, '}'):
raise s_common.BadSyntaxError('expected } at %d' % (off,))
_, off = nom(text, off + 1, whites)
return opers, off
def _stormOperSetProp(self, query, oper):
# Coverage of this function is affected by the following issue:
# https://bitbucket.org/ned/coveragepy/issues/198/continue-marked-as-not-covered
args = oper[1].get('args')
props = dict(oper[1].get('kwlist'))
core = self.getStormCore()
formnodes = collections.defaultdict(list)
formprops = collections.defaultdict(dict)
for node in query.data():
formnodes[node[1].get('tufo:form')].append(node)
forms = tuple(formnodes.keys())
for prop, valu in props.items():
if prop.startswith(':'):
valid = False
_prop = prop[1:]
# Check against every lifted form, since we may have a relative prop
# Which is valid against
for form in forms:
_fprop = form + prop
if core.isSetPropOk(_fprop, isadd=True):
formprops[form][_prop] = valu
valid = True
if not valid:
mesg = 'Relative prop is not valid on any lifted forms.'
raise s_common.BadSyntaxError(name=prop, mesg=mesg)
continue # pragma: no cover
mesg = 'setprop operator requires props to start with relative prop names.'
raise s_common.BadSyntaxError(name=prop, mesg=mesg)
for form, nodes in formnodes.items():
props = formprops.get(form)
if props:
for prop in props.keys():
perm = ('node:prop:set', {'form': form, 'prop': prop})
core.reqUserPerm(perm, elev=query.elev)
[core.setTufoProps(node, **props) for node in nodes]
def parse(text, off=0):
'''
Parse and return a set of instruction tufos.
'''
retn, off = parse_storm(text, off=off)
if off != len(text):
raise s_common.BadSyntaxError('trailing text: %s' % (text[off:],))
return retn
def parse_int(text, off, trim=True):
_, off = nom(text, off, whites)
neg = False
if nextchar(text, off, '-'):
neg = True
_, off = nom(text, off + 1, whites)
valu = None
if nextstr(text, off, '0x'):
valu, off = nom(text, off + 2, hexset)
if not valu:
raise s_common.BadSyntaxError(at=off, mesg='0x expected hex')
valu = int(valu, 16)
elif nextstr(text, off, '0b'):
valu, off = nom(text, off + 2, binset)
if not valu:
raise s_common.BadSyntaxError(at=off, mesg='0b expected bits')
valu = int(valu, 2)
else:
valu, off = nom(text, off, decset)
if not valu:
raise s_common.BadSyntaxError(at=off, mesg='expected digits')
if not nextchar(text, off, '.'):
valu = int(valu)
else:
frac, off = nom(text, off + 1, decset)
valu = float('%s.%s' % (valu, frac))
if neg:
valu = -valu
return valu, off
def _stormOperTask(self, query, oper):
args = oper[1].get('args')
opts = dict(oper[1].get('kwlist'))
if not args:
mesg = 'task(<taskname1>, <taskname2>, ..., [kwarg1=val1, ...])'
raise s_common.BadSyntaxError(mesg=mesg)
nodes = query.data()
core = self.getStormCore()
for tname in args:
evt = ':'.join(['task', tname])
core.fire(evt, nodes=nodes, storm=True, **opts)
def parse_float(text, off, trim=True):
_, off = nom(text, off, whites)
valu = ''
if nextchar(text, off, '-'):
valu += '-'
_, off = nom(text, off + 1, whites)
digs, off = nom(text, off, decset)
if not digs:
raise s_common.BadSyntaxError(at=off, mesg='expected digits')
valu += digs
if nextchar(text, off, '.'):
frac, off = nom(text, off + 1, decset)
if not frac:
raise s_common.BadSyntaxError(at=off, mesg='expected .<digits>')
valu = valu + '.' + frac
return float(valu), off
def _stormOperAddXref(self, query, oper):
args = oper[1].get('args')
if len(args) != 3:
raise s_common.BadSyntaxError(mesg='addxref(<type>,<form>,<valu>)')
xref, form, valu = args
core = self.getStormCore()
# TODO clearer error handling
for node in query.take():
sorc = node[1].get(node[1].get('tufo:form'))
node = core.formTufoByProp(xref, (sorc, (form, valu)))
query.add(node)
def parse_cmd_kwarg(text, off=0):
'''
Parse a foo:bar=<valu> kwarg into (prop,valu),off
'''
_, off = nom(text, off, whites)
prop, off = nom(text, off, varset)
_, off = nom(text, off, whites)
if not nextchar(text, off, '='):
raise s_common.BadSyntaxError(expected='= for kwarg ' + prop, at=off)
_, off = nom(text, off + 1, whites)
valu, off = parse_cmd_string(text, off)
return (prop, valu), off
def _stormOperPivot(self, query, oper):
args = oper[1].get('args')
opts = dict(oper[1].get('kwlist'))
if len(args) is 1:
srcp, dstp = None, args[0]
elif len(args) is 2:
srcp, dstp = args[0], args[1]
else:
raise s_common.BadSyntaxError(mesg='pivot(<srcprop>,<dstprop>)')
limit = opts.get('limit')
if limit is not None and limit < 0:
raise s_common.BadOperArg(oper='pivot', name='limit', mesg='must be >= 0')
self._runPivotOper(query, srcp, dstp, limit)
def _stormOperNextTag(self, query, oper):
name = None
args = oper[1].get('args')
kwargs = dict(oper[1].get('kwlist'))
doc = kwargs.get('doc', '??')
name = kwargs.get('core')
if len(args) != 1:
raise s_common.BadSyntaxError(mesg='nexttag(<tagname>,doc=<doc>)')
tag = args[0]
core = self.getStormCore(name=name)
valu = core.nextSeqValu(tag)
node = core.formTufoByProp('syn:tag', valu, doc=doc)
query.add(node)
def tokenize(text):
'''
Produce a token list from the given text string.
[ (<tok>,<info>), ... ]
'''
off = 0
tlen = len(text)
toks = []
while off < tlen:
_, off = s_syntax.nom_whitespace(text, off)
if off >= tlen:
break
if s_syntax.nextin(text, off, '"\''):
tokn = ('valu', {'off': off, 'type': 'str'})
tokn[1]['valu'], off = s_syntax.parse_string(text, off, trim=False)
tokn[1]['end'] = off
toks.append(tokn)
continue
if s_syntax.nextin(text, off, '0123456789'):
tokn = ('valu', {'off': off, 'type': 'int'})
tokn[1]['valu'], off = s_syntax.parse_int(text, off)
tokn[1]['end'] = off
toks.append(tokn)
continue
tokdone = False
for tok in tokstrs:
if text.startswith(tok, off):
tokn = (tok, {'off': off})
off += len(tok)
tokn[1]['end'] = off
toks.append(tokn)
tokdone = True
break
if tokdone:
continue
if not s_syntax.nextin(text, off, varset):
raise s_common.BadSyntaxError(at=off, mesg='no valid tokens found')
tokn = ('var', {'off': off})
tokn[1]['name'], off = s_syntax.nom(text, off, varset, trim=False)
toks.append(tokn)
for tokn in toks:
tokn[1].update(tokninfo.get(tokn[0], {}))
return toks
def parse_ques(text, off=0, trim=True):
'''
Parse "query" syntax: tag/prop[@<timewin>][^<limit>][*<by>][=valu]
'''
ques = {}
name, off = nom(text, off, varset, trim=True)
if not name:
raise s_common.BadSyntaxError(text=text, off=off, mesg='expected name')
ques['cmp'] = 'has'
ques['prop'] = name
_, off = nom(text, off, whites)
if len(text) == off:
return ques, off
if text[off] == '/':
ques['from'] = name
off += 1
name, off = nom(text, off, varset, trim=True)
ques['prop'] = name
_, off = nom(text, off, whites)
while True:
_, off = nom(text, off, whites)
if len(text) == off:
return ques, off
if text[off] == '^':
ques['limit'], off = parse_int(text, off + 1, trim=True)
continue
# NOTE: "by" macro syntax only supports eq so we eat and run
if nextchar(text, off, '*'):
_, off = nom(text, off + 1, whites)
ques['cmp'], off = nom(text, off, varset, trim=True)
if len(text) == off:
return ques, off
if not nextchar(text, off, '='):
raise s_common.BadSyntaxError(text=text, off=off, mesg='expected equals for by syntax')
_, off = nom(text, off + 1, whites)
ques['valu'], off = parse_valu(text, off)
return ques, off
if nextchar(text, off, '='):
_, off = nom(text, off + 1, whites)
ques['cmp'] = 'eq'
ques['valu'], off = parse_valu(text, off)
break
textpart = text[off:]
for ctxt, cmpr in macrocmps:
if textpart.startswith(ctxt):
ques['cmp'] = cmpr
ques['valu'], off = parse_valu(text, off + len(ctxt))
break
break
return ques, off
def parse_storm(text, off=0):
ret = []
while True:
# leading whitespace is irrelevant
_, off = nom(text, off, whites)
if off >= len(text):
break
# handle a sub-query terminator
if nextchar(text, off, '}'):
break
# handle $foo.bar={<subquery>} syntax
if nextchar(text, off, '$'):
_, off = nom(text, off + 1, whites)
if not nextin(text, off, alphaset):
raise s_common.BadSyntaxError(msg='Set variables must start with an alpha char')
name, off = nom(text, off, setset)
_, off = nom(text, off, whites)
# set load syntax goes here...
if not nextchar(text, off, '='):
raise s_common.BadSyntaxError(msg='expected = at %d' % (off,))
_, off = nom(text, off + 1, whites)
if not nextchar(text, off, '{'):
raise s_common.BadSyntaxError('expected { at %d' % (off,))
opers, off = parse_stormsub(text, off)
ret.append(oper('set', name, opers))
continue
# [ ] for node modification macro syntax
# [ inet:fqdn=woot.com ] == addnode(inet:fqdn,woot.com)
# [ :asn=10 ] == setprop(asn=10)
# [ #foo.bar ] or [ +#foo.bar ] == addtag(foo.bar)
# [ -#foo.bar ] == deltag(foo.bar)
if nextchar(text, off, '['):
off += 1
while True:
_, off = nom(text, off, whites)
if nextchar(text, off, ']'):
off += 1
break
if off == len(text):
raise s_common.BadSyntaxError(mesg='unexpected end of text in edit mode')
if nextstr(text, off, '+#'):
valu, off = parse_valu(text, off + 2)
ret.append(oper('addtag', valu))
continue
if nextstr(text, off, '-#'):
valu, off = parse_valu(text, off + 2)
ret.append(oper('deltag', valu))
continue
if nextchar(text, off, '#'):
valu, off = parse_valu(text, off + 1)
ret.append(oper('addtag', valu))
continue
if nextstr(text, off, '-:'):
valu, off = parse_valu(text, off + 1)
ret.append(oper('delprop', valu, force=1))
continue
# otherwise, it should be a prop=valu (maybe relative)
prop, off = nom(text, off, propset)
if not prop:
raise s_common.BadSyntaxError(mesg='edit macro expected prop=valu syntax')
_, off = nom(text, off, whites)
if not nextchar(text, off, '='):
raise s_common.BadSyntaxError(mesg='edit macro expected prop=valu syntax')
valu, off = parse_valu(text, off + 1)
if prop[0] == ':':
kwargs = {prop: valu}
ret.append(oper('setprop', **kwargs))
continue
ret.append(oper('addnode', prop, valu))
continue
# pivot() macro with no src prop: -> foo:bar
if nextstr(text, off, '->'):
_, off = nom(text, off + 2, whites)
name, off = nom(text, off, varset)
ret.append(oper('pivot', name))
continue
# join() macro with no src prop: <- foo:bar
if nextstr(text, off, '<-'):
_, off = nom(text, off + 2, whites)
name, off = nom(text, off, varset)
ret.append(oper('join', name))
continue
# lift by tag alone macro
if nextstr(text, off, '#'):
_, off = nom(text, off + 1, whites)
name, off = nom(text, off, varset)
ret.append(oper('alltag', name))
continue
# must() macro syntax: +foo:bar="woot"
if nextchar(text, off, '+'):
_, off = nom(text, off + 1, whites)
# subquery filter syntax
if nextchar(text, off, '{'):
opers, off = parse_stormsub(text, off)
ret.append(oper('filtsub', True, opers))
continue
inst, off = parse_macro_filt(text, off, mode='must')
ret.append(inst)
continue
# cant() macro syntax: -foo:bar=10
if nextchar(text, off, '-'):
_, off = nom(text, off + 1, whites)
# subquery filter syntax
if nextchar(text, off, '{'):
opers, off = parse_stormsub(text, off)
ret.append(oper('filtsub', False, opers))
continue
inst, off = parse_macro_filt(text, off, mode='cant')
ret.append(inst)
continue
# logical and syntax for multiple filters
if text[off] == '&':
if len(ret) == 0:
raise s_common.BadSyntaxError(mesg='logical and with no previous operator')
prev = ret[-1]
if prev[0] != 'filt':
raise s_common.BadSyntaxError(mesg='prev oper must be filter not: %r' % prev)
mode = prev[1].get('mode')
inst, off = parse_macro_filt(text, off + 1, mode=mode)
if prev[1].get('cmp') != 'and':
prev = ('filt', {'args': [prev, ], 'kwlist': [], 'cmp': 'and', 'mode': mode})
ret[-1] = prev
prev[1]['args'].append(inst)
continue
# logical or syntax for multiple filters
if text[off] == '|':
if len(ret) == 0:
raise s_common.BadSyntaxError(mesg='logical or with no previous operator')
prev = ret[-1]
if prev[0] != 'filt':
raise s_common.BadSyntaxError(mesg='prev oper must be filter not: %r' % prev)
mode = prev[1].get('mode')
inst, off = parse_macro_filt(text, off + 1, mode=mode)
if prev[1].get('cmp') != 'or':
prev = ('filt', {'args': [prev, ], 'kwlist': [], 'cmp': 'or', 'mode': mode})
ret[-1] = prev
prev[1]['args'].append(inst)
continue
# opts() macro syntax: %uniq=0 %limit=30
if text[off] == '%':
inst, off = parse_opts(text, off + 1)
ret.append(inst)
continue
origoff = off
name, off = nom(text, off, varset)
_, off = nom(text, off, whites)
# mop up in the case where we end with a macro
if len(text) == off:
inst, off = parse_macro_lift(text, origoff)
ret.append(inst)
continue
# macro foo:bar->baz:faz prop pivot
if nextstr(text, off, '->'):
pivn, off = nom(text, off + 2, varset)
inst = ('pivot', {'args': [name], 'kwlist': []})
# FIXME make a parser for foo.bar/baz:faz*blah#40
if nextchar(text, off, '/'):
inst[1]['kwlist'].append(('from', pivn))
pivn, off = nom(text, off + 1, varset)
inst[1]['args'].append(pivn)
ret.append(inst)
continue
# macro foo:bar<-baz:faz prop join
if nextstr(text, off, '<-'):
joinn, off = nom(text, off + 2, varset)
inst = ('join', {'args': [name], 'kwlist': []})
# FIXME make a parser for foo.bar/baz:faz*blah#40
if nextchar(text, off, '/'):
inst[1]['kwlist'].append(('from', joinn))
joinn, off = nom(text, off + 1, varset)
inst[1]['args'].append(joinn)
ret.append(inst)
continue
# standard foo() oper syntax
if nextchar(text, off, '('):
inst, off = parse_oper(text, origoff)
ret.append(inst)
continue
# only macro lift syntax remains
inst, off = parse_macro_lift(text, origoff)
ret.append(inst)
return ret, off
def getCmdOpts(self, text):
'''
Use the _cmd_syntax def to split/parse/normalize the cmd line.
NOTE: This is implemented indepedent of argparse (et.al) due to
the need for syntax aware argument splitting.
( also, allows different split per command type )
'''
off = 0
_, off = s_syntax.nom(text, off, s_syntax.whites)
name, off = s_syntax.meh(text, off, s_syntax.whites)
_, off = s_syntax.nom(text, off, s_syntax.whites)
opts = {}
args = collections.deque(
[synt for synt in self._cmd_syntax if not synt[0].startswith('-')])
switches = {
synt[0]: synt
for synt in self._cmd_syntax if synt[0].startswith('-')
}
# populate defaults and lists
for synt in self._cmd_syntax:
snam = synt[0].strip('-')
defval = synt[1].get('defval')
if defval is not None:
opts[snam] = defval
if synt[1].get('type') in ('list', 'kwlist'):
opts[snam] = []
def atswitch(t, o):
# check if we are at a recognized switch. if not
# assume the data is part of regular arguments.
if not text.startswith('-', o):
return None, o
name, x = s_syntax.meh(t, o, s_syntax.whites)
swit = switches.get(name)
if swit is None:
return None, o
return swit, x
while off < len(text):
_, off = s_syntax.nom(text, off, s_syntax.whites)
swit, off = atswitch(text, off)
if swit is not None:
styp = swit[1].get('type', 'flag')
snam = swit[0].strip('-')
if styp == 'valu':
valu, off = s_syntax.parse_cmd_string(text, off)
opts[snam] = valu
elif styp == 'list':
valu, off = s_syntax.parse_cmd_string(text, off)
if not isinstance(valu, list):
valu = valu.split(',')
opts[snam].extend(valu)
elif styp == 'enum':
vals = swit[1].get('enum:vals')
valu, off = s_syntax.parse_cmd_string(text, off)
if valu not in vals:
raise s_common.BadSyntaxError(
mesg='%s (%s)' % (swit[0], '|'.join(vals)),
text=text)
opts[snam] = valu
else:
opts[snam] = True
continue
if not args:
raise s_common.BadSyntaxError(mesg='trailing text: [%s]' %
(text[off:], ),
text=text)
synt = args.popleft()
styp = synt[1].get('type', 'valu')
# a glob type eats the remainder of the string
if styp == 'glob':
opts[synt[0]] = text[off:]
break
# eat the remainder of the string as separate vals
if styp == 'list':
valu = []
while off < len(text):
item, off = s_syntax.parse_cmd_string(text, off)
valu.append(item)
opts[synt[0]] = valu
break
if styp == 'kwlist':
kwlist, off = s_syntax.parse_cmd_kwlist(text, off)
opts[snam] = kwlist
break
valu, off = s_syntax.parse_cmd_string(text, off)
opts[synt[0]] = valu
return opts
def _stormOperTree(self, query, oper):
args = oper[1].get('args')
opts = dict(oper[1].get('kwlist'))
if not args:
raise s_common.BadSyntaxError(mesg='tree([<srcprop>], <destprop>, [recurlim=<limit>])')
limt = self.getLiftLimitHelp(opts.get('limit'))
core = self.getStormCore()
# Prevent infinite pivots
try:
recurlim, _ = core.getTypeNorm('int', opts.get('recurlim', 20))
except s_common.BadTypeValu as e:
raise s_common.BadOperArg(oper='tree', name='recurlim', mesg=e.errinfo.get('mesg'))
if recurlim < 0:
raise s_common.BadOperArg(oper='tree', name='recurlim', mesg='must be >= 0')
srcp = None
dstp = args[0]
if len(args) > 1:
srcp = args[0]
dstp = args[1]
# do we have a relative source property?
relsrc = srcp is not None and srcp.startswith(':')
tufs = query.data()
queried_vals = set()
while True:
vals = set()
if srcp is not None and not relsrc:
for tufo in tufs:
valu = tufo[1].get(srcp)
if valu is not None:
vals.add(valu)
elif not relsrc:
for tufo in tufs:
form, valu = s_tufo.ndef(tufo)
if valu is not None:
vals.add(valu)
else:
for tufo in tufs:
form, _ = s_tufo.ndef(tufo)
valu = tufo[1].get(form + srcp)
if valu is not None:
vals.add(valu)
qvals = list(vals - queried_vals)
if not qvals:
break
nodes = core.stormTufosBy('in', dstp, qvals, limit=limt.get())
[query.add(n) for n in nodes]
if limt.dec(len(nodes)):
break
queried_vals = queried_vals.union(vals)
if recurlim > 0:
recurlim -= 1
if recurlim < 1:
break
tufs = query.data()
def parse(text, off=0):
'''
Parse and return a set of instruction tufos.
'''
ret = []
while True:
# leading whitespace is irrelevant
_, off = nom(text, off, whites)
if off >= len(text):
break
# handle some special "macro" style syntaxes
# [ ] for node modification macro syntax
# [ inet:fqdn=woot.com ] == addnode(inet:fqdn,woot.com)
# [ :asn=10 ] == setprop(asn=10)
# [ #foo.bar ] or [ +#foo.bar ] == addtag(foo.bar)
# [ -#foo.bar ] == deltag(foo.bar)
if nextchar(text, off, '['):
off += 1
while True:
_, off = nom(text, off, whites)
if nextchar(text, off, ']'):
off += 1
break
if off == len(text):
raise s_common.BadSyntaxError(mesg='unexpected end of text in edit mode')
if nextstr(text, off, '+#'):
valu, off = parse_valu(text, off + 2)
ret.append(oper('addtag', valu))
continue
if nextstr(text, off, '-#'):
valu, off = parse_valu(text, off + 2)
ret.append(oper('deltag', valu))
continue
if nextchar(text, off, '#'):
valu, off = parse_valu(text, off + 1)
ret.append(oper('addtag', valu))
continue
# otherwise, it should be a prop=valu (maybe relative)
prop, off = nom(text, off, propset)
if not prop:
raise s_common.BadSyntaxError(mesg='edit macro expected prop=valu syntax')
_, off = nom(text, off, whites)
if not nextchar(text, off, '='):
raise s_common.BadSyntaxError(mesg='edit macro expected prop=valu syntax')
valu, off = parse_valu(text, off + 1)
if prop[0] == ':':
kwargs = {prop: valu}
ret.append(oper('setprop', **kwargs))
continue
ret.append(oper('addnode', prop, valu))
continue
# pivot() macro with no src prop: -> foo:bar
if nextstr(text, off, '->'):
_, off = nom(text, off + 2, whites)
name, off = nom(text, off, varset)
ret.append(oper('pivot', name))
continue
# lift by tag alone macro
if nextstr(text, off, '#'):
_, off = nom(text, off + 1, whites)
name, off = nom(text, off, varset)
ret.append(oper('alltag', name))
continue
# must() macro syntax: +foo:bar="woot"
if nextchar(text, off, '+'):
inst, off = parse_macro_filt(text, off + 1, mode='must')
ret.append(inst)
continue
# cant() macro syntax: -foo:bar=10
if nextchar(text, off, '-'):
inst, off = parse_macro_filt(text, off + 1, mode='cant')
ret.append(inst)
continue
# logical and syntax for multiple filters
if text[off] == '&':
if len(ret) == 0:
raise s_common.BadSyntaxError(mesg='logical and with no previous operator')
prev = ret[-1]
if prev[0] != 'filt':
raise s_common.BadSyntaxError(mesg='prev oper must be filter not: %r' % prev)
mode = prev[1].get('mode')
inst, off = parse_macro_filt(text, off + 1, mode=mode)
if prev[1].get('cmp') != 'and':
prev = ('filt', {'args': [prev, ], 'kwlist': [], 'cmp': 'and', 'mode': mode})
ret[-1] = prev
prev[1]['args'].append(inst)
continue
# logical or syntax for multiple filters
if text[off] == '|':
if len(ret) == 0:
raise s_common.BadSyntaxError(mesg='logical or with no previous operator')
prev = ret[-1]
if prev[0] != 'filt':
raise s_common.BadSyntaxError(mesg='prev oper must be filter not: %r' % prev)
mode = prev[1].get('mode')
inst, off = parse_macro_filt(text, off + 1, mode=mode)
if prev[1].get('cmp') != 'or':
prev = ('filt', {'args': [prev, ], 'kwlist': [], 'cmp': 'or', 'mode': mode})
ret[-1] = prev
prev[1]['args'].append(inst)
continue
# opts() macro syntax: %uniq=0 %limit=30
if text[off] == '%':
inst, off = parse_opts(text, off + 1)
ret.append(inst)
continue
origoff = off
name, off = nom(text, off, varset)
_, off = nom(text, off, whites)
# mop up in the case where we end with a macro
if len(text) == off:
inst, off = parse_macro_lift(text, origoff)
ret.append(inst)
continue
# macro foo:bar->baz:faz prop pivot
if nextstr(text, off, '->'):
pivn, off = nom(text, off + 2, varset)
inst = ('pivot', {'args': [name], 'kwlist': []})
# FIXME make a parser for foo.bar/baz:faz*blah#40
if nextchar(text, off, '/'):
inst[1]['kwlist'].append(('from', pivn))
pivn, off = nom(text, off + 1, varset)
inst[1]['args'].append(pivn)
ret.append(inst)
continue
# standard foo() oper syntax
if nextchar(text, off, '('):
inst, off = parse_oper(text, origoff)
ret.append(inst)
continue
# only macro lift syntax remains
inst, off = parse_macro_lift(text, origoff)
ret.append(inst)
#[ i[1]['kwlist'].sort() for i in ret ]
return ret
def raisetok(tokn, mesg):
off = tokn[1].get('off')
raise s_common.BadSyntaxError(mesg=mesg, off=off)
def _stormOperTree(self, query, oper):
args = oper[1].get('args')
opts = dict(oper[1].get('kwlist'))
if not args:
raise s_common.BadSyntaxError(mesg='tree([<srcprop>], <destprop>, [recurlim=<limit>])')
core = self.getStormCore()
# Prevent infinite pivots
recurlim, _ = core.getTypeNorm('int', opts.get('recurlim', 20))
srcp = None
dstp = args[0]
if len(args) > 1:
srcp = args[0]
dstp = args[1]
# do we have a relative source property?
relsrc = srcp is not None and srcp.startswith(':')
tufs = query.data()
queried_vals = set()
while True:
vals = set()
if srcp is not None and not relsrc:
for tufo in tufs:
valu = tufo[1].get(srcp)
if valu is not None:
vals.add(valu)
elif not relsrc:
for tufo in tufs:
form, valu = s_tufo.ndef(tufo)
if valu is not None:
vals.add(valu)
else:
for tufo in tufs:
form, _ = s_tufo.ndef(tufo)
valu = tufo[1].get(form + srcp)
if valu is not None:
vals.add(valu)
qvals = list(vals - queried_vals)
if not qvals:
break
[query.add(t) for t in self.stormTufosBy('in', dstp, qvals, limit=opts.get('limit'))]
queried_vals = queried_vals.union(vals)
if recurlim:
recurlim -= 1
if recurlim < 1:
break
tufs = query.data()
