def Parse(self, cmd, args, stdout, stderr, return_val, time_taken,
knowledge_base):
"""Parse the mount command output."""
_ = stderr, time_taken, args, knowledge_base # Unused.
self.CheckReturn(cmd, return_val)
result = rdf_protodict.AttributedDict()
for entry in self._field_parser.ParseEntries(stdout):
line_str = " ".join(entry)
mount_rslt = self.mount_re.match(line_str)
if mount_rslt:
device, mount_point, fs_type, option_str = mount_rslt.groups()
result = rdf_client_fs.Filesystem()
result.device = device
result.mount_point = mount_point
result.type = fs_type
# Parse these options as a dict as some items may be key/values.
# KeyValue parser uses OrderedDict as the native parser method. Use it.
options = KeyValueParser(
term=",").ParseToOrderedDict(option_str)
# Keys without values get assigned [] by default. Because these keys are
# actually true, if declared, change any [] values to True.
for k, v in iteritems(options):
options[k] = v or [True]
result.options = rdf_protodict.AttributedDict(**options)
yield result
def testParse(self):
filt = filters.ItemFilter()
one = rdf_protodict.AttributedDict(test1="1", test2=[2, 3])
foo = rdf_protodict.AttributedDict(test1="foo", test2=["bar", "baz"])
fs = rdf_client_fs.Filesystem(device="/dev/sda1", mount_point="/root")
objs = [one, foo, fs]
results = filt.Parse(objs, u"test1 is '1'")
self.assertLen(results, 1)
self.assertEqual("test1", results[0].key)
self.assertEqual("1", results[0].value)
results = filt.Parse(objs, u"test1 is '2'")
self.assertFalse(results)
results = filt.Parse(objs, u"test2 contains 3")
self.assertLen(results, 1)
self.assertEqual("test2", results[0].key)
self.assertEqual([2, 3], results[0].value)
results = filt.Parse(objs, u"test1 is '1' or test1 contains 'foo'")
self.assertLen(results, 2)
self.assertEqual("test1", results[0].key)
self.assertEqual("1", results[0].value)
self.assertEqual("test1", results[1].key)
self.assertEqual("foo", results[1].value)
results = filt.Parse(objs, u"mount_point is '/root'")
self.assertLen(results, 1)
self.assertEqual("mount_point", results[0].key)
self.assertEqual("/root", results[0].value)
def testInitialize(self):
arnie = {"target": "Sarah Connor", "mission": "Protect"}
t800 = {"target": "Sarah Connor", "mission": "Terminate"}
terminator = rdf_protodict.AttributedDict(arnie)
self.assertEqual(terminator.GetItem("target"), "Sarah Connor")
self.assertEqual(terminator.GetItem("mission"), "Protect")
terminator = rdf_protodict.AttributedDict(t800)
self.assertEqual(terminator.target, "Sarah Connor")
self.assertEqual(terminator.mission, "Terminate")
# We don't want a conflicted Terminator
self.assertFalse(terminator.GetItem("happy_face"))
def testParse(self):
filt = filters.ObjectFilter()
hit1 = rdf_protodict.AttributedDict(test="hit1")
hit2 = rdf_protodict.AttributedDict(test="hit2")
miss = rdf_protodict.AttributedDict(test="miss")
objs = [hit1, hit2, miss]
results = filt.Parse(objs, u"test is 'hit1'")
self.assertCountEqual([hit1], results)
results = filt.Parse(objs, u"test is 'hit2'")
self.assertCountEqual([hit2], results)
results = filt.Parse(objs, u"test inset 'hit1,hit2'")
self.assertCountEqual([hit1, hit2], results)
def testInitFromNonStringKeyedDictRaises(self):
with self.assertRaises(TypeError):
rdf_protodict.AttributedDict({
1: "foo",
2: "bar",
3: "baz",
})
def ParseFile(self, knowledge_base, pathspec, filedesc):
"""Identifies the paths set within a file.
Expands paths within the context of the file, but does not infer fully
expanded paths from external states. There are plenty of cases where path
attributes are unresolved, e.g. sourcing other files.
Lines are not handled literally. A field parser is used to:
- Break lines with multiple distinct statements into separate lines (e.g.
lines with a ';' separating stanzas.
- Strip out comments.
- Handle line continuations to capture multi-line configurations into one
statement.
Args:
knowledge_base: A knowledgebase for the client to whom the file belongs.
pathspec: A pathspec corresponding to the parsed file.
filedesc: A file-like object to parse.
Yields:
An attributed dict for each env vars. 'name' contains the path name, and
'vals' contains its vals.
"""
del knowledge_base # Unused.
lines = self.parser.ParseEntries(utils.ReadFileBytesAsUnicode(filedesc))
if os.path.basename(pathspec.path) in self._CSH_FILES:
paths = self._ParseCshVariables(lines)
else:
paths = self._ParseShVariables(lines)
for path_name, path_vals in iteritems(paths):
yield rdf_protodict.AttributedDict(
config=pathspec.path, name=path_name, vals=path_vals)
def Parse(self, stat, file_obj, unused_knowledge_base):
lines = set([
l.strip() for l in utils.ReadFileBytesAsUnicode(file_obj).splitlines()
])
users = []
bad_lines = []
for line in lines:
# behaviour of At/Cron is undefined for lines with whitespace separated
# fields/usernames
if " " in line:
bad_lines.append(line)
elif line: # drop empty lines
users.append(line)
filename = stat.pathspec.path
cfg = {"filename": filename, "users": users}
yield rdf_protodict.AttributedDict(**cfg)
if bad_lines:
yield rdf_anomaly.Anomaly(
type="PARSER_ANOMALY",
symptom="Dodgy entries in %s." % (filename),
reference_pathspec=stat.pathspec,
finding=bad_lines)
def ParseFile(
self,
knowledge_base: rdf_client.KnowledgeBase,
pathspec: rdf_paths.PathSpec,
filedesc: IO[bytes],
) -> Iterator[rdf_protodict.AttributedDict]:
del knowledge_base # Unused.
lines = set([
l.strip()
for l in utils.ReadFileBytesAsUnicode(filedesc).splitlines()
])
users = []
bad_lines = []
for line in lines:
# behaviour of At/Cron is undefined for lines with whitespace separated
# fields/usernames
if " " in line:
bad_lines.append(line)
elif line: # drop empty lines
users.append(line)
filename = pathspec.path
cfg = {"filename": filename, "users": users}
yield rdf_protodict.AttributedDict(**cfg)
if bad_lines:
yield rdf_anomaly.Anomaly(type="PARSER_ANOMALY",
symptom="Dodgy entries in %s." %
(filename),
reference_pathspec=pathspec,
finding=bad_lines)
def Parse(self, stat, file_obj, knowledge_base):
"""Identifies the paths set within a file.
Expands paths within the context of the file, but does not infer fully
expanded paths from external states. There are plenty of cases where path
attributes are unresolved, e.g. sourcing other files.
Lines are not handled literally. A field parser is used to:
- Break lines with multiple distinct statements into separate lines (e.g.
lines with a ';' separating stanzas.
- Strip out comments.
- Handle line continuations to capture multi-line configurations into one
statement.
Args:
stat: statentry
file_obj: VFSFile
knowledge_base: unused
Yields:
An attributed dict for each env vars. 'name' contains the path name, and
'vals' contains its vals.
"""
_ = knowledge_base
lines = self.parser.ParseEntries(file_obj.read())
if os.path.basename(stat.pathspec.path) in self._CSH_FILES:
paths = self._ParseCshVariables(lines)
else:
paths = self._ParseShVariables(lines)
for path_name, path_vals in iteritems(paths):
yield rdf_protodict.AttributedDict(config=stat.pathspec.path,
name=path_name,
vals=path_vals)
def testInitFromCronObject(self):
state = rdf_protodict.AttributedDict()
state["quux"] = "norf"
state["thud"] = "blargh"
cron_job = rdf_cronjobs.CronJob()
cron_job.cron_job_id = "foo"
cron_job.current_run_id = "bar"
cron_job.last_run_time = self._DATETIME("2001-01-01")
cron_job.last_run_status = "FINISHED"
cron_job.frequency = rdfvalue.Duration.From(1, rdfvalue.DAYS)
cron_job.lifetime = rdfvalue.Duration.From(30, rdfvalue.DAYS)
cron_job.enabled = False
cron_job.forced_run_requested = True
cron_job.state = state
cron_job.description = "testdescription"
api_cron_job = cron_plugin.ApiCronJob.InitFromObject(cron_job)
self.assertEqual(api_cron_job.cron_job_id, "foo")
self.assertEqual(api_cron_job.current_run_id, "bar")
self.assertEqual(api_cron_job.description, "testdescription")
self.assertEqual(api_cron_job.last_run_time,
self._DATETIME("2001-01-01"))
self.assertEqual(api_cron_job.last_run_status, "FINISHED")
self.assertEqual(api_cron_job.frequency,
rdfvalue.Duration.From(1, rdfvalue.DAYS))
self.assertEqual(api_cron_job.lifetime,
rdfvalue.Duration.From(30, rdfvalue.DAYS))
self.assertFalse(api_cron_job.enabled)
self.assertTrue(api_cron_job.forced_run_requested)
api_state_items = {_.key: _.value for _ in api_cron_job.state.items}
self.assertEqual(api_state_items, {"quux": "norf", "thud": "blargh"})
def WriteState(self):
if "w" in self.mode:
self._ValidateState()
self.Set(self.Schema.FLOW_ARGS(self.args))
self.Set(self.Schema.FLOW_CONTEXT(self.context))
self.Set(self.Schema.FLOW_RUNNER_ARGS(self.runner_args))
protodict = rdf_protodict.AttributedDict().FromDict(self.state)
self.Set(self.Schema.FLOW_STATE_DICT(protodict))
def CreatePluginAndDefaultState(cls, source_urn=None, args=None, token=None): state = rdf_protodict.AttributedDict() state["source_urn"] = source_urn state["args"] = args state["token"] = token plugin = cls(source_urn=source_urn, args=args, token=token) plugin.InitializeState(state) return plugin, state
def testRdfFormatterAttributedDict(self):
sshd = rdf_config_file.SshdConfig()
sshd.config = rdf_protodict.AttributedDict(skynet="operational")
template = "{config.skynet}"
hinter = hints.Hinter(template=template)
expected = "operational"
result = hinter.Render(sshd)
self.assertEqual(expected, result)
def testGetBytestringItem(self):
adict = rdf_protodict.AttributedDict()
adict.foo = 42
adict.bar = b"quux"
adict.baz = [4, 8, 15, 16, 23, 42]
self.assertEqual(adict[b"foo"], 42)
self.assertEqual(adict[b"bar"], b"quux")
self.assertEqual(adict[b"baz"], [4, 8, 15, 16, 23, 42])
def _Load(self, expression):
self._Flush()
parser = config_file.KeyValueParser(
kv_sep=":", sep=",", term=(r"\s+", r"\n"))
parsed = {}
for entry in parser.ParseEntries(expression):
parsed.update(entry)
self.cfg = rdf_protodict.AttributedDict(parsed)
return parsed
def testSetStringItem(self):
adict = rdf_protodict.AttributedDict()
adict["foo"] = 42
adict["bar"] = b"quux"
adict["baz"] = [4, 8, 15, 16, 23, 42]
self.assertEqual(adict.foo, 42)
self.assertEqual(adict.bar, b"quux")
self.assertEqual(adict.baz, [4, 8, 15, 16, 23, 42])
def testInitFromBytestringKeyedDict(self):
adict = rdf_protodict.AttributedDict({
b"foo": 42,
b"bar": b"quux",
b"baz": [4, 8, 15, 16, 23, 42],
})
self.assertEqual(adict.foo, 42)
self.assertEqual(adict.bar, b"quux")
self.assertEqual(adict.baz, [4, 8, 15, 16, 23, 42])
def CreatePluginAndDefaultState(cls, source_urn=None, args=None):
"""Creates a plugin and returns its initial state."""
state = rdf_protodict.AttributedDict()
state["source_urn"] = source_urn
if args is not None:
args.Validate()
state["args"] = args
plugin = cls(source_urn=source_urn, args=args)
plugin.InitializeState(state)
return plugin, state
def testParse(self):
filt = filters.ForEach()
hit1 = rdf_protodict.AttributedDict(k1="v1", k2="v2", k3="v3")
hit2 = rdf_protodict.AttributedDict(k1="v4", k2="v5", k3="v6")
meta = rdf_protodict.AttributedDict(
foo=["foo", "bar"], target=[hit1, hit2], null=[])
objs = [meta]
results = filt.Parse(objs, "target")
self.assertEqual(2, len(results))
self.assertItemsEqual([hit1, hit2], [r.item for r in results])
results = filt.Parse(objs, "foo")
self.assertEqual(2, len(results))
self.assertItemsEqual(["foo", "bar"], [r.item for r in results])
results = filt.Parse(objs, "null")
self.assertEqual(0, len(results))
def ParseObjs(self, objs, expression):
for key in self._Attrs(expression):
# Key needs to be a string for rdfvalue.KeyValue
key = utils.SmartStr(key)
for obj in objs:
val = self._GetVal(obj, key)
if val:
# Dict won't accept rdfvalue.RepeatedFieldHelper
if isinstance(val, rdf_structs.RepeatedFieldHelper):
val = list(val)
yield rdf_protodict.AttributedDict({"key": key, "value": val})
def testParse(self):
filt = filters.RDFFilter()
cfg = rdf_protodict.AttributedDict()
anom = rdf_anomaly.Anomaly()
objs = [cfg, anom]
results = filt.Parse(objs, "KnowledgeBase")
self.assertFalse(results)
results = filt.Parse(objs, "AttributedDict,KnowledgeBase")
self.assertCountEqual([cfg], results)
results = filt.Parse(objs, "Anomaly,AttributedDict,KnowledgeBase")
self.assertCountEqual(objs, results)
def ParseFile(self, knowledge_base, pathspec, filedesc):
del knowledge_base # Unused.
lines = set(l.strip() for l in filedesc.read().splitlines())
users = list(filter(None, lines))
filename = pathspec.path
cfg = {"filename": filename, "users": users}
yield rdf_protodict.AttributedDict(**cfg)
def Parse(self, stat, file_obj, knowledge_base):
del knowledge_base # Unused.
lines = set([l.strip() for l in file_obj.read().splitlines()])
users = list(filter(None, lines))
filename = stat.pathspec.path
cfg = {"filename": filename, "users": users}
yield rdf_protodict.AttributedDict(**cfg)
def testParse(self):
filt = filters.AttrFilter()
hit1 = rdf_protodict.AttributedDict(k1="hit1", k2="found1", k3=[3, 4])
hit2 = rdf_protodict.AttributedDict(k1="hit2", k2="found2")
meta = rdf_protodict.AttributedDict(one=hit1, two=hit2)
objs = [hit1, hit2, meta]
results = filt.Parse(objs, "k1 k2 one.k3")
self.assertLen(results, 5)
r1, r2, r3, r4, r5 = results
self.assertEqual("k1", r1.key)
self.assertEqual("hit1", r1.value)
self.assertEqual("k1", r2.key)
self.assertEqual("hit2", r2.value)
self.assertEqual("k2", r3.key)
self.assertEqual("found1", r3.value)
self.assertEqual("k2", r4.key)
self.assertEqual("found2", r4.value)
self.assertEqual("one.k3", r5.key)
self.assertEqual([3, 4], r5.value)
def ParseFiles(
self,
knowledge_base: rdf_client.KnowledgeBase,
pathspecs: Iterable[rdf_paths.PathSpec],
filedescs: Iterable[IO[bytes]],
) -> Iterator[rdf_protodict.AttributedDict]:
del knowledge_base # Unused.
config = {}
for pathspec, file_obj in zip(pathspecs, filedescs):
k, v = self._Parse(pathspec, file_obj)
config[k] = v
yield rdf_protodict.AttributedDict(config)
def Parse(self, cmd, args, stdout, stderr, return_val, knowledge_base):
"""Parse the sysctl output."""
_ = stderr, args, knowledge_base # Unused.
self.CheckReturn(cmd, return_val)
result = rdf_protodict.AttributedDict()
# The KeyValueParser generates an ordered dict by default. The sysctl vals
# aren't ordering dependent, but there's no need to un-order it.
for k, v in self.lexer.ParseToOrderedDict(stdout).items():
key = k.replace(".", "_")
if len(v) == 1:
v = v[0]
result[key] = v
return [result]
def testNestedAssignment(self):
adict = rdf_protodict.AttributedDict()
adict["foo"] = {}
adict["foo"]["bar"] = 42
adict["foo"][b"baz"] = "Lorem ipsum."
adict[b"quux"] = {}
adict[b"quux"]["norf"] = [4, 8, 15, 16, 23, 42]
adict[b"quux"][b"thud"] = 3.14
self.assertEqual(adict.foo["bar"], 42)
self.assertEqual(adict.foo[b"baz"], "Lorem ipsum.")
self.assertEqual(adict.quux["norf"], [4, 8, 15, 16, 23, 42])
self.assertEqual(adict.quux[b"thud"], 3.14)
def Parse(self, unused_stat, file_obj, unused_knowledge_base):
for entry in self.ParseEntries(file_obj.read()):
if not entry:
continue
result = rdf_client_fs.Filesystem()
result.device = entry[0].decode("string_escape")
result.mount_point = entry[1].decode("string_escape")
result.type = entry[2].decode("string_escape")
options = KeyValueParser(term=",").ParseToOrderedDict(entry[3])
# Keys without values get assigned [] by default. Because these keys are
# actually true, if declared, change any [] values to True.
for k, v in iteritems(options):
options[k] = v or [True]
result.options = rdf_protodict.AttributedDict(**options)
yield result
def ParseMultiple(self, stats, file_objects, knowledge_base):
del knowledge_base # Unused.
lines = set()
for file_obj in file_objects:
lines.update(set(l.strip() for l in file_obj.read().splitlines()))
users = list(filter(None, lines))
for stat in stats:
filename = stat.pathspec.path
cfg = {"filename": filename, "users": users}
yield rdf_protodict.AttributedDict(**cfg)
def _GenConfig(self, cfg):
"""Interpolate configurations with defaults to generate actual configs."""
# Some setting names may have a + or - suffix. These indicate that the
# settings modify the default values.
merged = self.default.copy()
for setting, vals in iteritems(cfg):
option, operator = (setting.split(None, 1) + [None])[:2]
vals = set(vals)
default = set(self.default.get(option, []))
# If there is an operator, updated values accordingly.
if operator == "+":
vals = default.union(vals)
elif operator == "-":
vals = default.difference(vals)
merged[option] = list(vals)
return rdf_protodict.AttributedDict(**merged)
