def testParse(self):
filt = filters.ItemFilter()
one = rdf_protodict.AttributedDict(test1="1", test2=[2, 3])
foo = rdf_protodict.AttributedDict(test1="foo", test2=["bar", "baz"])
fs = rdf_client.Filesystem(device="/dev/sda1", mount_point="/root")
objs = [one, foo, fs]
results = filt.Parse(objs, "test1 is '1'")
self.assertEqual(1, len(results))
self.assertEqual("test1", results[0].key)
self.assertEqual("1", results[0].value)
results = filt.Parse(objs, "test1 is '2'")
self.assertFalse(results)
results = filt.Parse(objs, "test2 contains 3")
self.assertEqual(1, len(results))
self.assertEqual("test2", results[0].key)
self.assertEqual([2, 3], results[0].value)
results = filt.Parse(objs, "test1 is '1' or test1 contains 'foo'")
self.assertEqual(2, len(results))
self.assertEqual("test1", results[0].key)
self.assertEqual("1", results[0].value)
self.assertEqual("test1", results[1].key)
self.assertEqual("foo", results[1].value)
results = filt.Parse(objs, "mount_point is '/root'")
self.assertEqual(1, len(results))
self.assertEqual("mount_point", results[0].key)
self.assertEqual("/root", results[0].value)
def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
knowledge_base):
"""Parse the mount command output."""
_ = stderr, time_taken, args, knowledge_base # Unused.
self.CheckReturn(cmd, return_val)
result = rdf_protodict.AttributedDict()
for entry in self.ParseEntries(stdout):
line_str = " ".join(entry)
mount_rslt = self.mount_re.match(line_str)
if mount_rslt:
device, mount_point, fs_type, option_str = mount_rslt.groups()
result = rdf_client.Filesystem()
result.device = device
result.mount_point = mount_point
result.type = fs_type
# Parse these options as a dict as some items may be key/values.
# KeyValue parser uses OrderedDict as the native parser method. Use it.
options = KeyValueParser(
term=",").ParseToOrderedDict(option_str)
# Keys without values get assigned [] by default. Because these keys are
# actually true, if declared, change any [] values to True.
for k, v in options.iteritems():
options[k] = v or [True]
result.options = rdf_protodict.AttributedDict(**options)
yield result
def testInitialize(self):
arnie = {"target": "Sarah Connor", "mission": "Protect"}
t800 = {"target": "Sarah Connor", "mission": "Terminate"}
terminator = rdf_protodict.AttributedDict(arnie)
self.assertEquals(terminator.GetItem("target"), "Sarah Connor")
self.assertEquals(terminator.GetItem("mission"), "Protect")
terminator = rdf_protodict.AttributedDict(t800)
self.assertEquals(terminator.target, "Sarah Connor")
self.assertEquals(terminator.mission, "Terminate")
# We don't want a conflicted Terminator
self.assertFalse(terminator.GetItem("happy_face"))
def testParse(self):
filt = filters.ObjectFilter()
hit1 = rdf_protodict.AttributedDict(test="hit1")
hit2 = rdf_protodict.AttributedDict(test="hit2")
miss = rdf_protodict.AttributedDict(test="miss")
objs = [hit1, hit2, miss]
results = filt.Parse(objs, "test is 'hit1'")
self.assertItemsEqual([hit1], results)
results = filt.Parse(objs, "test is 'hit2'")
self.assertItemsEqual([hit2], results)
results = filt.Parse(objs, "test inset 'hit1,hit2'")
self.assertItemsEqual([hit1, hit2], results)
def testParse(self):
filt = filters.ForEach()
hit1 = rdf_protodict.AttributedDict(k1="v1", k2="v2", k3="v3")
hit2 = rdf_protodict.AttributedDict(k1="v4", k2="v5", k3="v6")
meta = rdf_protodict.AttributedDict(foo=["foo", "bar"],
target=[hit1, hit2])
objs = [meta]
results = filt.Parse(objs, "target")
self.assertEqual(2, len(results))
self.assertItemsEqual([hit1, hit2], results)
results = filt.Parse(objs, "foo")
self.assertEqual(0, len(results))
def Parse(self, stat, file_obj, knowledge_base):
"""Identifies the paths set within a file.
Expands paths within the context of the file, but does not infer fully
expanded paths from external states. There are plenty of cases where path
attributes are unresolved, e.g. sourcing other files.
Lines are not handled literally. A field parser is used to:
- Break lines with multiple distinct statements into separate lines (e.g.
lines with a ';' separating stanzas.
- Strip out comments.
- Handle line continuations to capture multi-line configurations into one
statement.
Args:
stat: statentry
file_obj: VFSFile
knowledge_base: unused
Yields:
An attributed dict for each env vars. 'name' contains the path name, and
'vals' contains its vals.
"""
_ = knowledge_base
lines = self.parser.ParseEntries(file_obj.read())
if os.path.basename(stat.pathspec.path) in self._CSH_FILES:
paths = self._ParseCshVariables(lines)
else:
paths = self._ParseShVariables(lines)
for path_name, path_vals in paths.iteritems():
yield rdf_protodict.AttributedDict(config=stat.pathspec.path,
name=path_name,
vals=path_vals)
def WriteState(self):
if "w" in self.mode:
self._ValidateState()
self.Set(self.Schema.FLOW_ARGS(self.args))
self.Set(self.Schema.FLOW_CONTEXT(self.context))
self.Set(self.Schema.FLOW_RUNNER_ARGS(self.runner_args))
protodict = rdf_protodict.AttributedDict().FromDict(self.state)
self.Set(self.Schema.FLOW_STATE_DICT(protodict))
def testRdfFormatterAttributedDict(self):
sshd = rdf_config_file.SshdConfig()
sshd.config = rdf_protodict.AttributedDict(skynet="operational")
template = "{config.skynet}"
hinter = hints.Hinter(template=template)
expected = "operational"
result = hinter.Render(sshd)
self.assertEqual(expected, result)
def _Load(self, expression):
self._Flush()
parser = config_file.KeyValueParser(kv_sep=":", sep=",",
term=(r"\s+", r"\n"))
parsed = {}
for entry in parser.ParseEntries(expression):
parsed.update(entry)
self.cfg = rdf_protodict.AttributedDict(parsed)
return parsed
def testParse(self):
filt = filters.RDFFilter()
cfg = rdf_protodict.AttributedDict()
anom = anomaly.Anomaly()
objs = [cfg, anom]
results = filt.Parse(objs, "KnowledgeBase")
self.assertFalse(results)
results = filt.Parse(objs, "AttributedDict,KnowledgeBase")
self.assertItemsEqual([cfg], results)
results = filt.Parse(objs, "Anomaly,AttributedDict,KnowledgeBase")
self.assertItemsEqual(objs, results)
def ParseObjs(self, objs, expression):
for key in self._Attrs(expression):
# Key needs to be a string for rdfvalue.KeyValue
key = utils.SmartStr(key)
for obj in objs:
val = self._GetVal(obj, key)
if val:
# Dict won't accept rdfvalue.RepeatedFieldHelper
if isinstance(val, structs.RepeatedFieldHelper):
val = list(val)
yield rdf_protodict.AttributedDict({"k": key, "v": val})
def testParse(self):
filt = filters.AttrFilter()
hit1 = rdf_protodict.AttributedDict(k1="hit1", k2="found1", k3=[3, 4])
hit2 = rdf_protodict.AttributedDict(k1="hit2", k2="found2")
meta = rdf_protodict.AttributedDict(one=hit1, two=hit2)
objs = [hit1, hit2, meta]
results = filt.Parse(objs, "k1 k2 one.k3")
self.assertEqual(5, len(results))
r1, r2, r3, r4, r5 = results
self.assertEqual("k1", r1.key)
self.assertEqual("hit1", r1.value)
self.assertEqual("k1", r2.key)
self.assertEqual("hit2", r2.value)
self.assertEqual("k2", r3.key)
self.assertEqual("found1", r3.value)
self.assertEqual("k2", r4.key)
self.assertEqual("found2", r4.value)
self.assertEqual("one.k3", r5.key)
self.assertEqual([3, 4], r5.value)
def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
knowledge_base):
"""Parse the sysctl output."""
_ = stderr, time_taken, args, knowledge_base # Unused.
self.CheckReturn(cmd, return_val)
result = rdf_protodict.AttributedDict()
# The KeyValueParser generates an ordered dict by default. The sysctl vals
# aren't ordering dependent, but there's no need to un-order it.
for k, v in self.lexer.ParseToOrderedDict(stdout).iteritems():
key = k.replace(".", "_")
if len(v) == 1:
v = v[0]
result[key] = v
return [result]
def Parse(self, unused_stat, file_obj, unused_knowledge_base):
for entry in self.ParseEntries(file_obj.read()):
if not entry:
continue
result = rdf_client.Filesystem()
result.device = entry[0].decode("string_escape")
result.mount_point = entry[1].decode("string_escape")
result.type = entry[2].decode("string_escape")
options = KeyValueParser(term=",").ParseToOrderedDict(entry[3])
# Keys without values get assigned [] by default. Because these keys are
# actually true, if declared, change any [] values to True.
for k, v in options.iteritems():
options[k] = v or [True]
result.options = rdf_protodict.AttributedDict(**options)
yield result
def __init__(self,
source_urn=None,
output_base_urn=None,
args=None,
token=None,
state=None):
"""OutputPlugin constructor.
Note that OutputPlugin constructor may run with security checks enabled
(if they're enabled in the config). Therefore it's a bad idea to write
anything to AFF4 in the constructor.
Constructor should only be overriden if some non-self.state-stored
class members should be initialized.
Args:
source_urn: URN of the data source to process the results from.
output_base_urn: URN of the AFF4 volume where plugin will write output
data (if needed).
args: This plugin's arguments.
token: Security token.
state: A dict representing the plugin's state. If this
is passed, no initialization will be performed, only the state will
be applied.
Raises:
ValueError: when state argument is passed together with args or token
arguments.
"""
if state and (token or args):
raise ValueError(
"'state' argument can't be passed together with 'args' "
"or 'token'.")
if not state:
self.state = rdf_protodict.AttributedDict()
self.state.source_urn = source_urn
self.state.output_base_urn = output_base_urn
self.state.args = args
self.state.token = token
self.InitializeState()
else:
self.state = state
self.args = self.state["args"]
self.token = self.state["token"]
self.lock = threading.RLock()
def Parse(self, stat, file_obj, unused_knowledge_base):
uris_to_parse = self.FindPotentialURIs(file_obj)
uris = []
for url_to_parse in uris_to_parse:
url = rdf_standard.URI()
url.ParseFromString(url_to_parse)
# if no transport then url_to_parse wasn't actually a valid URL
# either host or path also have to exist for this to be a valid URL
if url.transport and (url.host or url.path):
uris.append(url)
filename = stat.pathspec.path
cfg = {"filename": filename, "uris": uris}
yield rdf_protodict.AttributedDict(**cfg)
def _GenConfig(self, cfg):
"""Interpolate configurations with defaults to generate actual configs."""
# Some setting names may have a + or - suffix. These indicate that the
# settings modify the default values.
merged = self.default.copy()
for setting, vals in cfg.iteritems():
option, operator = (setting.split(None, 1) + [None])[:2]
vals = set(vals)
default = set(self.default.get(option, []))
# If there is an operator, updated values accordingly.
if operator == "+":
vals = default.union(vals)
elif operator == "-":
vals = default.difference(vals)
merged[option] = list(vals)
return rdf_protodict.AttributedDict(**merged)
def Parse(self, stat, file_obj, unused_knowledge_base):
lines = set([l.strip() for l in file_obj.read(100000).splitlines()])
users = []
bad_lines = []
for line in lines:
if " " in line: # behaviour of At/Cron is undefined for lines
bad_lines.append(line) # with whitespace separated fields/usernames
elif line: # drop empty lines
users.append(line)
filename = stat.pathspec.path
cfg = {"filename": filename, "users": users}
yield rdf_protodict.AttributedDict(**cfg)
if bad_lines:
yield rdf_anomaly.Anomaly(type="PARSER_ANOMALY",
symptom="Dodgy entries in %s." % (filename),
reference_pathspec=stat.pathspec,
finding=bad_lines)
def testParseFileObjs(self):
"""Multiple file types are parsed successfully."""
filt = filters.StatFilter()
ok = self._GenStat(path="/etc/shadow", st_uid=0, st_gid=0, st_mode=0100640)
link = self._GenStat(
path="/etc/shadow", st_uid=0, st_gid=0, st_mode=0120640)
user = self._GenStat(
path="/etc/shadow", st_uid=1000, st_gid=1000, st_mode=0100640)
writable = self._GenStat(
path="/etc/shadow", st_uid=0, st_gid=0, st_mode=0100666)
cfg = {"path": "/etc/shadow", "st_uid": 0, "st_gid": 0, "st_mode": 0100640}
invalid = rdf_protodict.AttributedDict(**cfg)
objs = [ok, link, user, writable, invalid]
results = filt.Parse(objs, "uid:>=0 gid:>=0")
self.assertItemsEqual([ok, link, user, writable], results)
results = filt.Parse(objs, "uid:=0 mode:0440 mask:0440")
self.assertItemsEqual([ok, link, writable], results)
results = filt.Parse(objs, "uid:=0 mode:0440 mask:0444")
self.assertItemsEqual([ok, link], results)
results = list(
filt.Parse(objs, "uid:=0 mode:0440 mask:0444 file_type:regular"))
self.assertItemsEqual([ok], results)
def testAttributedDictSettingsAreAttr(self):
t800 = {"target": "Sarah Connor", "mission": "Terminate"}
terminator = rdf_protodict.AttributedDict(t800)
self.assertEquals(terminator.target, "Sarah Connor")
self.assertEquals(terminator.mission, "Terminate")
def ParseMultiple(self, stats, file_objs, _):
config = {}
for stat, file_obj in zip(stats, file_objs):
k, v = self._Parse(stat, file_obj)
config[k] = v
return [rdf_protodict.AttributedDict(config)]
def ParseObjs(self, objs, expression):
filt = self._Compile(expression)
key = expression.split(None, 1)[0]
for result in filt.Filter(objs):
val = getattr(result, key)
yield rdf_protodict.AttributedDict({"k": key, "v": val})
def ParseObjs(self, objs, expression):
for obj in objs:
repeated_vals = getattr(obj, expression)
for val in repeated_vals:
yield rdf_protodict.AttributedDict({"item": val})
def GenerateSample(self, number=0):
return rdf_protodict.AttributedDict({"number": number})