def testInterpolateArgs(self):
collect_flow = collectors.ArtifactCollectorFlow(None, token=self.token)
collect_flow.state.Register("knowledge_base",
rdf_client.KnowledgeBase())
collect_flow.current_artifact_name = "blah"
collect_flow.state.knowledge_base.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******"))
collect_flow.state.knowledge_base.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******"))
test_rdf = rdf_client.KnowledgeBase()
action_args = {
"usernames": ["%%users.username%%", "%%users.username%%"],
"nointerp": "asdfsdf",
"notastring": test_rdf
}
kwargs = collect_flow.InterpolateDict(action_args)
self.assertItemsEqual(kwargs["usernames"],
["test1", "test2", "test1", "test2"])
self.assertEqual(kwargs["nointerp"], "asdfsdf")
self.assertEqual(kwargs["notastring"], test_rdf)
# We should be using an array since users.username will expand to multiple
# values.
self.assertRaises(ValueError, collect_flow.InterpolateDict,
{"bad": "%%users.username%%"})
list_args = collect_flow.InterpolateList(
["%%users.username%%", r"%%users.username%%\aa"])
self.assertItemsEqual(list_args,
["test1", "test2", r"test1\aa", r"test2\aa"])
list_args = collect_flow.InterpolateList(["one"])
self.assertEqual(list_args, ["one"])
def SetKnowledgeBase(fqdn="test.example.com", host_os="Linux",
host_data=None):
"""Generates a KnowledgeBase entry in the host_data used by checks."""
if not host_data:
host_data = {}
host_data["KnowledgeBase"] = rdf_client.KnowledgeBase(fqdn=fqdn, os=host_os)
return host_data
def Layout(self, request, response):
self.completions = rdf_client.KnowledgeBase().GetKbFieldNames()
response = super(GlobExpressionFormRenderer, self).Layout(request, response)
return self.CallJavascript(
response, "GlobExpressionFormRenderer.Layout", prefix=self.prefix,
completions=self.completions)
def _SetupClientImpl(self,
client_nr,
index=None,
arch="x86_64",
kernel="4.0.0",
os_version="buster/sid",
ping=None,
system="Linux"):
client_id_urn = rdf_client.ClientURN("C.1%015x" % client_nr)
with aff4.FACTORY.Create(client_id_urn,
aff4_grr.VFSGRRClient,
mode="rw",
token=self.token) as fd:
cert = self.ClientCertFromPrivateKey(
config.CONFIG["Client.private_key"])
fd.Set(fd.Schema.CERT, cert)
fd.Set(fd.Schema.CLIENT_INFO, self._TestClientInfo())
fd.Set(fd.Schema.PING, ping or rdfvalue.RDFDatetime.Now())
fd.Set(fd.Schema.HOSTNAME("Host-%x" % client_nr))
fd.Set(fd.Schema.FQDN("Host-%x.example.com" % client_nr))
fd.Set(
fd.Schema.MAC_ADDRESS("aabbccddee%02x\nbbccddeeff%02x" %
(client_nr, client_nr)))
fd.Set(
fd.Schema.HOST_IPS("192.168.0.%d\n2001:abcd::%x" %
(client_nr, client_nr)))
if system:
fd.Set(fd.Schema.SYSTEM(system))
if os_version:
fd.Set(fd.Schema.OS_VERSION(os_version))
if arch:
fd.Set(fd.Schema.ARCH(arch))
if kernel:
fd.Set(fd.Schema.KERNEL(kernel))
kb = rdf_client.KnowledgeBase()
kb.fqdn = "Host-%x.example.com" % client_nr
kb.users = [
rdf_client.User(username="******"),
rdf_client.User(username="******"),
]
artifact.SetCoreGRRKnowledgeBaseValues(kb, fd)
fd.Set(fd.Schema.KNOWLEDGE_BASE, kb)
fd.Set(fd.Schema.INTERFACES(self._TestInterfaces(client_nr)))
hardware_info = fd.Schema.HARDWARE_INFO()
hardware_info.system_manufacturer = ("System-Manufacturer-%x" %
client_nr)
hardware_info.bios_version = ("Bios-Version-%x" % client_nr)
fd.Set(fd.Schema.HARDWARE_INFO, hardware_info)
fd.Flush()
index.AddClient(fd)
return client_id_urn
def testGrep(self):
class MockCallFlow(object):
def CallFlow(self, *args, **kwargs):
self.args = args
self.kwargs = kwargs
mock_call_flow = MockCallFlow()
with utils.Stubber(collectors.ArtifactCollectorFlow, "CallFlow",
mock_call_flow.CallFlow):
collect_flow = collectors.ArtifactCollectorFlow(None,
token=self.token)
collect_flow.state.Register("knowledge_base",
rdf_client.KnowledgeBase())
collect_flow.current_artifact_name = "blah"
collect_flow.state.knowledge_base.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******"))
collect_flow.state.knowledge_base.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******"))
collector = artifact_registry.ArtifactSource(
type=artifact_registry.ArtifactSource.SourceType.GREP,
attributes={
"paths": ["/etc/passwd"],
"content_regex_list": [r"^a%%users.username%%b$"]
})
collect_flow.Grep(collector, rdf_paths.PathSpec.PathType.TSK)
conditions = mock_call_flow.kwargs["conditions"]
self.assertEqual(len(conditions), 1)
regexes = conditions[0].contents_regex_match.regex.SerializeToString()
self.assertItemsEqual(regexes.split("|"),
["(^atest1b$)", "(^atest2b$)"])
self.assertEqual(mock_call_flow.kwargs["paths"], ["/etc/passwd"])
def testFindsKeyWithInterpolatedGlobWithoutConditions(self):
# Initialize client's knowledge base in order for the interpolation
# to work.
user = rdf_client.User(sid="S-1-5-20")
kb = rdf_client.KnowledgeBase(users=[user])
client_id = self.SetupClient(0)
with aff4.FACTORY.Open(client_id, mode="rw", token=self.token) as client:
client.Set(client.Schema.KNOWLEDGE_BASE, kb)
session_id = self.RunFlow([
"HKEY_USERS/%%users.sid%%/Software/Microsoft/Windows/"
"CurrentVersion/*"
])
results = self.GetResults(session_id)
self.assertEqual(len(results), 1)
key = ("/HKEY_USERS/S-1-5-20/"
"Software/Microsoft/Windows/CurrentVersion/Run")
self.assertEqual(results[0].stat_entry.AFF4Path(client_id),
"aff4:/C.1000000000000000/registry" + key)
self.assertEqual(results[0].stat_entry.pathspec.path, key)
self.assertEqual(results[0].stat_entry.pathspec.pathtype,
rdf_paths.PathSpec.PathType.REGISTRY)
def testUserMergeWindows(self):
"""Check Windows users are accurately merged."""
kb = rdf_client.KnowledgeBase()
self.assertEqual(len(kb.users), 0)
kb.MergeOrAddUser(rdf_client.KnowledgeBaseUser(sid="1234"))
self.assertEqual(len(kb.users), 1)
kb.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(sid="5678", username="******"))
self.assertEqual(len(kb.users), 2)
_, conflicts = kb.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(sid="5678", username="******"))
self.assertEqual(len(kb.users), 2)
self.assertEqual(conflicts[0], ("username", "test1", "test2"))
self.assertEqual(kb.GetUser(sid="5678").username, "test2")
# This should merge on user name as we have no other data.
kb.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******", homedir="a"))
self.assertEqual(len(kb.users), 2)
# This should create a new user since the sid is different.
new_attrs, conflicts = kb.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******",
sid="12345",
temp="/blah"))
self.assertEqual(len(kb.users), 3)
self.assertItemsEqual(new_attrs,
["users.username", "users.temp", "users.sid"])
self.assertEqual(conflicts, [])
def setUp(self):
# Set the KB to an empty object
client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token)
self.old_kb = client.Get(client.Schema.KNOWLEDGE_BASE)
client.Set(client.Schema.KNOWLEDGE_BASE, rdf_client.KnowledgeBase())
client.Flush()
super(TestParserDependency, self).setUp()
def testUserMergeLinux(self):
"""Check Linux users are accurately merged."""
kb = rdf_client.KnowledgeBase()
self.assertEqual(len(kb.users), 0)
kb.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******", last_logon=1111))
self.assertEqual(len(kb.users), 1)
# This should merge since the username is the same.
kb.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(uid="12", username="******"))
self.assertEqual(len(kb.users), 1)
# This should create a new record because the uid is different
kb.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******",
uid="13",
desktop="/home/blake/Desktop"))
self.assertEqual(len(kb.users), 2)
kb.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******",
uid="14",
desktop="/home/blake/Desktop"))
self.assertEqual(len(kb.users), 3)
# Check merging where we don't specify uid works
new_attrs, conflicts = kb.MergeOrAddUser(
rdf_client.KnowledgeBaseUser(username="******",
desktop="/home/blakey/Desktop"))
self.assertEqual(len(kb.users), 3)
self.assertItemsEqual(new_attrs, ["users.username", "users.desktop"])
self.assertItemsEqual(
conflicts,
[("desktop", u"/home/blake/Desktop", u"/home/blakey/Desktop")])
def setUp(self):
super(CheckRegistryTests, self).setUp()
if self.sw_chk is None:
self.sw_chk = self._LoadCheck("sw.yaml", "SW-CHECK")
checks.CheckRegistry.RegisterCheck(check=self.sw_chk,
source="dpkg.out",
overwrite_if_exists=True)
if self.sshd_chk is None:
self.sshd_chk = self._LoadCheck("sshd.yaml", "SSHD-CHECK")
checks.CheckRegistry.RegisterCheck(check=self.sshd_chk,
source="sshd_config",
overwrite_if_exists=True)
if self.sshd_perms is None:
self.sshd_perms = self._LoadCheck("sshd.yaml", "SSHD-PERMS")
checks.CheckRegistry.RegisterCheck(check=self.sshd_perms,
source="sshd_config",
overwrite_if_exists=True)
self.kb = rdf_client.KnowledgeBase()
self.kb.fqdn = "test.example.com"
self.host_data = {
"KnowledgeBase": self.kb,
"WMIInstalledSoftware": GetWMIData(),
"DebianPackagesStatus": GetDPKGData(),
"SshdConfigFile": GetSSHDConfig()
}
def testKnowledgeBaseRootAttributesGetMappedCorrectly(self):
kb = rdf_client.KnowledgeBase(
environ_path="the_path",
environ_temp="the_temp",
environ_systemroot="the_systemroot",
environ_windir="the_windir",
environ_programfiles="the_programfiles",
environ_programfilesx86="the_programfilesx86",
environ_systemdrive="the_systemdrive",
environ_allusersprofile="the_allusersprofile",
environ_allusersappdata="the_allusersappdata")
mapping = artifact_utils.GetWindowsEnvironmentVariablesMap(kb)
self.assertEqual(
mapping, {
"allusersappdata": "the_allusersappdata",
"allusersprofile": "the_allusersprofile",
"path": "the_path",
"programdata": "the_allusersprofile",
"programfiles": "the_programfiles",
"programfiles(x86)": "the_programfilesx86",
"programw6432": "the_programfiles",
"systemdrive": "the_systemdrive",
"systemroot": "the_systemroot",
"temp": "the_temp",
"windir": "the_windir"
})
def testReadAllClientsFullInfoReadsMultipleClientsWithMultipleLabels(self):
d = self.db
for i in range(10):
client_id = "C.000000005000000%d" % i
self._InitializeClient(client_id)
cl = objects.ClientSnapshot(
client_id=client_id,
knowledge_base=rdf_client.KnowledgeBase(
fqdn="test%d.examples.com" % i),
kernel="12.3.%d" % i)
d.WriteClientSnapshot(cl)
d.WriteClientMetadata(client_id, certificate=CERT)
si = rdf_client.StartupInfo(boot_time=i)
d.WriteClientStartupInfo(client_id, si)
d.AddClientLabels(
client_id, "test_owner",
["test_label-a-%d" % i, "test_label-b-%d" % i])
c_infos = sorted(
d.ReadAllClientsFullInfo(), key=lambda c: c.last_snapshot.client_id)
for i, full_info in enumerate(c_infos):
self.assertEqual(full_info.last_snapshot.client_id,
"C.000000005000000%d" % i)
self.assertEqual(full_info.metadata.certificate, CERT)
self.assertEqual(full_info.last_startup_info.boot_time, i)
self.assertEqual(
sorted(full_info.labels, key=lambda l: l.name), [
objects.ClientLabel(
owner="test_owner", name="test_label-a-%d" % i),
objects.ClientLabel(
owner="test_owner", name="test_label-b-%d" % i)
])
def testPathInterpolation(self):
self.client_id = self.SetupClient(0)
bar = rdf_client.User(username="******")
baz = rdf_client.User(username="******")
kb = rdf_client.KnowledgeBase(os="foo",
domain="norf",
users=[bar, baz])
client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw")
with client:
client.Set(client.Schema.KNOWLEDGE_BASE, kb)
with test_lib.AutoTempDirPath(remove_non_empty=True) as temp_dirpath:
self._Touch(os.path.join(temp_dirpath, "foo", "bar"))
self._Touch(os.path.join(temp_dirpath, "foo", "baz"))
self._Touch(os.path.join(temp_dirpath, "foo", "quux"))
self._Touch(os.path.join(temp_dirpath, "thud", "norf", "plugh"))
self._Touch(os.path.join(temp_dirpath, "thud", "norf", "blargh"))
paths = [
os.path.join(temp_dirpath, "%%os%%", "%%users.username%%"),
os.path.join(temp_dirpath, "thud", "%%domain%%", "plugh"),
]
action = rdf_file_finder.FileFinderAction.Action.STAT
results = self._RunCFF(paths, action)
paths = [result.stat_entry.pathspec.path for result in results]
self.assertItemsEqual(paths, [
os.path.join(temp_dirpath, "foo", "bar"),
os.path.join(temp_dirpath, "foo", "baz"),
os.path.join(temp_dirpath, "thud", "norf", "plugh")
])
def testInterpolateArgs(self):
collect_flow = collectors.ArtifactCollectorFlow(None, token=self.token)
kb = rdf_client.KnowledgeBase()
kb.MergeOrAddUser(rdf_client.User(username="******"))
kb.MergeOrAddUser(rdf_client.User(username="******"))
collect_flow.state["knowledge_base"] = kb
collect_flow.current_artifact_name = "blah"
collect_flow.args = artifact_utils.ArtifactCollectorFlowArgs()
test_rdf = rdf_client.KnowledgeBase()
action_args = {
"usernames": ["%%users.username%%", "%%users.username%%"],
"nointerp": "asdfsdf",
"notastring": test_rdf
}
kwargs = collect_flow.InterpolateDict(action_args)
self.assertItemsEqual(kwargs["usernames"],
["test1", "test2", "test1", "test2"])
self.assertEqual(kwargs["nointerp"], "asdfsdf")
self.assertEqual(kwargs["notastring"], test_rdf)
# We should be using an array since users.username will expand to multiple
# values.
self.assertRaises(ValueError, collect_flow.InterpolateDict,
{"bad": "%%users.username%%"})
list_args = collect_flow.InterpolateList(
["%%users.username%%", r"%%users.username%%\aa"])
self.assertItemsEqual(list_args,
["test1", "test2", r"test1\aa", r"test2\aa"])
list_args = collect_flow.InterpolateList(["one"])
self.assertEqual(list_args, ["one"])
# Ignore the failure in users.desktop, report the others.
collect_flow.args.ignore_interpolation_errors = True
list_args = collect_flow.InterpolateList(
["%%users.desktop%%", r"%%users.username%%\aa"])
self.assertItemsEqual(list_args, [r"test1\aa", r"test2\aa"])
# Both fail.
list_args = collect_flow.InterpolateList(
[r"%%users.desktop%%\aa", r"%%users.sid%%\aa"])
self.assertItemsEqual(list_args, [])
def testClientSummary(self):
d = self.db
client_id_1 = "C.0000000000000001"
client_id_2 = "C.0000000000000002"
client_id_3 = "C.0000000000000003"
self._InitializeClient(client_id_1)
self._InitializeClient(client_id_2)
self._InitializeClient(client_id_3)
d.WriteClientSnapshot(
objects.ClientSnapshot(
client_id=client_id_1,
knowledge_base=rdf_client.KnowledgeBase(
fqdn="test1234.examples.com"),
kernel="12.3"))
d.WriteClientSnapshot(
objects.ClientSnapshot(
client_id=client_id_1,
knowledge_base=rdf_client.KnowledgeBase(
fqdn="test1234.examples.com"),
kernel="12.4"))
d.WriteClientSnapshot(
objects.ClientSnapshot(
client_id=client_id_2,
knowledge_base=rdf_client.KnowledgeBase(
fqdn="test1235.examples.com"),
kernel="12.4"))
hist = d.ReadClientSnapshotHistory(client_id_1)
self.assertEqual(len(hist), 2)
# client_3 should be excluded - no snapshot yet
res = d.ReadClientsSnapshot([client_id_1, client_id_2, client_id_3])
self.assertEqual(len(res), 3)
self.assertIsInstance(res[client_id_1], objects.ClientSnapshot)
self.assertIsInstance(res[client_id_2], objects.ClientSnapshot)
self.assertIsInstance(res[client_id_1].timestamp, rdfvalue.RDFDatetime)
self.assertIsNotNone(res[client_id_2].timestamp)
self.assertEqual(res[client_id_1].knowledge_base.fqdn,
"test1234.examples.com")
self.assertEqual(res[client_id_1].kernel, "12.4")
self.assertEqual(res[client_id_2].knowledge_base.fqdn,
"test1235.examples.com")
self.assertFalse(res[client_id_3])
def testGetClientSummary(self):
hostname = "test"
system = "Linux"
os_release = "12.02"
kernel = "3.15-rc2"
fqdn = "test.test.com"
arch = "amd64"
install_time = rdfvalue.RDFDatetime().Now()
user = "******"
userobj = rdf_client.User(username=user)
interface = rdf_client.Interface(ifname="eth0")
timestamp = 1
with utils.Stubber(time, "time", lambda: timestamp):
with aff4.FACTORY.Create("C.0000000000000000",
aff4_grr.VFSGRRClient,
mode="rw",
token=self.token) as fd:
kb = rdf_client.KnowledgeBase()
kb.users.Append(userobj)
empty_summary = fd.GetSummary()
self.assertEqual(empty_summary.client_id, "C.0000000000000000")
self.assertFalse(empty_summary.system_info.version)
self.assertEqual(empty_summary.timestamp.AsSecondsFromEpoch(),
1)
# This will cause TYPE to be written with current time = 101 when the
# object is closed
timestamp += 100
fd.Set(fd.Schema.HOSTNAME(hostname))
fd.Set(fd.Schema.SYSTEM(system))
fd.Set(fd.Schema.OS_RELEASE(os_release))
fd.Set(fd.Schema.KERNEL(kernel))
fd.Set(fd.Schema.FQDN(fqdn))
fd.Set(fd.Schema.ARCH(arch))
fd.Set(fd.Schema.INSTALL_DATE(install_time))
fd.Set(fd.Schema.KNOWLEDGE_BASE(kb))
fd.Set(fd.Schema.USERNAMES([user]))
fd.Set(fd.Schema.INTERFACES([interface]))
with aff4.FACTORY.Open("C.0000000000000000",
aff4_grr.VFSGRRClient,
mode="rw",
token=self.token) as fd:
summary = fd.GetSummary()
self.assertEqual(summary.system_info.node, hostname)
self.assertEqual(summary.system_info.system, system)
self.assertEqual(summary.system_info.release, os_release)
self.assertEqual(summary.system_info.kernel, kernel)
self.assertEqual(summary.system_info.fqdn, fqdn)
self.assertEqual(summary.system_info.machine, arch)
self.assertEqual(summary.system_info.install_date,
install_time)
self.assertItemsEqual(summary.users, [userobj])
self.assertItemsEqual(summary.interfaces, [interface])
self.assertFalse(summary.client_info)
self.assertEqual(summary.timestamp.AsSecondsFromEpoch(), 101)
def ValidateSyntax(rdf_artifact):
"""Validates artifact syntax.
This method can be used to validate individual artifacts as they are loaded,
without needing all artifacts to be loaded first, as for Validate().
Args:
rdf_artifact: RDF object artifact.
Raises:
ArtifactSyntaxError: If artifact syntax is invalid.
"""
if not rdf_artifact.doc:
raise artifacts.ArtifactSyntaxError(rdf_artifact, "missing doc")
for supp_os in rdf_artifact.supported_os:
valid_os = rdf_artifact.SUPPORTED_OS_LIST
if supp_os not in valid_os:
detail = "invalid `supported_os` ('%s' not in %s)" % (supp_os,
valid_os)
raise artifacts.ArtifactSyntaxError(rdf_artifact, detail)
for condition in rdf_artifact.conditions:
# FIXME(hanuszczak): It does not look like the code below can throw
# `ConditionException`. Do we really need it then?
try:
of = objectfilter.Parser(condition).Parse()
of.Compile(objectfilter.BaseFilterImplementation)
except artifacts.ConditionError as e:
detail = "invalid condition '%s'" % condition
raise artifacts.ArtifactSyntaxError(rdf_artifact, detail, e)
for label in rdf_artifact.labels:
if label not in rdf_artifact.ARTIFACT_LABELS:
raise artifacts.ArtifactSyntaxError(rdf_artifact,
"invalid label '%s'" % label)
# Anything listed in provides must be defined in the KnowledgeBase
valid_provides = rdf_client.KnowledgeBase().GetKbFieldNames()
for kb_var in rdf_artifact.provides:
if kb_var not in valid_provides:
detail = "broken `provides` ('%s' not in %s)" % (kb_var,
valid_provides)
raise artifacts.ArtifactSyntaxError(rdf_artifact, detail)
# Any %%blah%% path dependencies must be defined in the KnowledgeBase
for dep in GetArtifactPathDependencies(rdf_artifact):
if dep not in valid_provides:
detail = "broken path dependencies ('%s' not in %s)" % (
dep, valid_provides)
raise artifacts.ArtifactSyntaxError(rdf_artifact, detail)
for source in rdf_artifact.sources:
try:
source.Validate()
except artifacts.ArtifactSourceSyntaxError as e:
raise artifacts.ArtifactSyntaxError(rdf_artifact, "bad source", e)
def ValidateSyntax(self):
"""Validate artifact syntax.
This method can be used to validate individual artifacts as they are loaded,
without needing all artifacts to be loaded first, as for Validate().
Raises:
ArtifactDefinitionError: If artifact is invalid.
"""
cls_name = self.name
if not self.doc:
raise ArtifactDefinitionError("Artifact %s has missing doc" %
cls_name)
for supp_os in self.supported_os:
if supp_os not in self.SUPPORTED_OS_LIST:
raise ArtifactDefinitionError(
"Artifact %s has invalid supported_os %s" %
(cls_name, supp_os))
for condition in self.conditions:
try:
of = objectfilter.Parser(condition).Parse()
of.Compile(objectfilter.BaseFilterImplementation)
except ConditionError as e:
raise ArtifactDefinitionError(
"Artifact %s has invalid condition %s. %s" %
(cls_name, condition, e))
for label in self.labels:
if label not in self.ARTIFACT_LABELS:
raise ArtifactDefinitionError(
"Artifact %s has an invalid label %s. Please use one from "
"ARTIFACT_LABELS." % (cls_name, label))
# Anything listed in provides must be defined in the KnowledgeBase
valid_provides = rdf_client.KnowledgeBase().GetKbFieldNames()
for kb_var in self.provides:
if kb_var not in valid_provides:
raise ArtifactDefinitionError(
"Artifact %s has broken provides: '%s' not in KB fields: %s"
% (cls_name, kb_var, valid_provides))
# Any %%blah%% path dependencies must be defined in the KnowledgeBase
for dep in self.GetArtifactPathDependencies():
if dep not in valid_provides:
raise ArtifactDefinitionError(
"Artifact %s has an invalid path dependency: '%s', not in KB "
"fields: %s" % (cls_name, dep, valid_provides))
for source in self.sources:
try:
source.Validate()
except Error as e:
raise ArtifactDefinitionError(
"Artifact %s has bad source. %s" % (cls_name, e))
def _SetupClientImpl(self,
client_nr,
index=None,
system=None,
os_version=None,
arch=None,
uname=None,
ping=None):
client_id_urn = rdf_client.ClientURN("C.1%015x" % client_nr)
with aff4.FACTORY.Create(client_id_urn,
aff4_grr.VFSGRRClient,
mode="rw",
token=self.token) as fd:
cert = self.ClientCertFromPrivateKey(
config.CONFIG["Client.private_key"])
fd.Set(fd.Schema.CERT, cert)
info = fd.Schema.CLIENT_INFO()
info.client_name = "GRR Monitor"
fd.Set(fd.Schema.CLIENT_INFO, info)
fd.Set(fd.Schema.PING, ping or rdfvalue.RDFDatetime.Now())
fd.Set(fd.Schema.HOSTNAME("Host-%x" % client_nr))
fd.Set(fd.Schema.FQDN("Host-%x.example.com" % client_nr))
fd.Set(
fd.Schema.MAC_ADDRESS("aabbccddee%02x\nbbccddeeff%02x" %
(client_nr, client_nr)))
fd.Set(
fd.Schema.HOST_IPS("192.168.0.%d\n2001:abcd::%x" %
(client_nr, client_nr)))
if system:
fd.Set(fd.Schema.SYSTEM(system))
if os_version:
fd.Set(fd.Schema.OS_VERSION(os_version))
if arch:
fd.Set(fd.Schema.ARCH(arch))
if uname:
fd.Set(fd.Schema.UNAME(uname))
kb = rdf_client.KnowledgeBase()
artifact.SetCoreGRRKnowledgeBaseValues(kb, fd)
fd.Set(fd.Schema.KNOWLEDGE_BASE, kb)
hardware_info = fd.Schema.HARDWARE_INFO()
hardware_info.system_manufacturer = ("System-Manufacturer-%x" %
client_nr)
hardware_info.bios_version = ("Bios-Version-%x" % client_nr)
fd.Set(fd.Schema.HARDWARE_INFO, hardware_info)
fd.Flush()
index.AddClient(fd)
return client_id_urn
def SetKnowledgeBase(self,
hostname="test.example.com",
host_os="Linux",
host_data=None):
if not host_data:
host_data = {}
kb = rdf_client.KnowledgeBase()
kb.hostname = hostname
kb.os = host_os
host_data["KnowledgeBase"] = kb
return host_data
def testKnowledgeBaseRetrievalFailures(self):
"""Test kb retrieval failure modes."""
client = aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw")
self.assertRaises(artifact_lib.KnowledgeBaseUninitializedError,
artifact.GetArtifactKnowledgeBase, client)
kb = rdf_client.KnowledgeBase()
kb.hostname = "test"
client.Set(client.Schema.KNOWLEDGE_BASE(kb))
client.Flush(sync=True)
self.assertRaises(artifact_lib.KnowledgeBaseAttributesMissingError,
artifact.GetArtifactKnowledgeBase, client)
def SetKnowledgeBase(hostname="test.example.com",
host_os="Linux",
host_data=None):
"""Generates a KnowledgeBase entry in the host_data used by checks."""
if not host_data:
host_data = {}
kb = rdf_client.KnowledgeBase()
kb.hostname = hostname
kb.os = host_os
host_data["KnowledgeBase"] = kb
return host_data
def testInterpolation(self):
"""Check we can interpolate values from the knowledge base."""
kb = rdf_client.KnowledgeBase()
# No users yet, this should raise
self.assertRaises(
artifact_lib.KnowledgeBaseInterpolationError, list,
artifact_lib.InterpolateKbAttributes("test%%users.username%%test",
kb))
# Now we have two users
kb.users.Append(rdf_client.KnowledgeBaseUser(username="******", uid=1))
kb.users.Append(rdf_client.KnowledgeBaseUser(username="******", uid=2))
kb.Set("environ_allusersprofile", "c:\\programdata")
paths = artifact_lib.InterpolateKbAttributes(
"test%%users.username%%test", kb)
paths = list(paths)
self.assertEqual(len(paths), 2)
self.assertItemsEqual(paths, ["testjoetest", "testjimtest"])
paths = artifact_lib.InterpolateKbAttributes(
"%%environ_allusersprofile%%\\a", kb)
self.assertEqual(list(paths), ["c:\\programdata\\a"])
# Check a bad attribute raises
self.assertRaises(
artifact_lib.KnowledgeBaseInterpolationError, list,
artifact_lib.InterpolateKbAttributes("%%nonexistent%%\\a", kb))
# Empty values should also raise
kb.Set("environ_allusersprofile", "")
self.assertRaises(
artifact_lib.KnowledgeBaseInterpolationError, list,
artifact_lib.InterpolateKbAttributes(
"%%environ_allusersprofile%%\\a", kb))
# No users have temp defined, so this should raise
self.assertRaises(
artifact_lib.KnowledgeBaseInterpolationError, list,
artifact_lib.InterpolateKbAttributes("%%users.temp%%\\a", kb))
# One user has users.temp defined, the others do not. This is common on
# windows where users have been created but have never logged in. We should
# get just one value back.
kb.users.Append(
rdf_client.KnowledgeBaseUser(
username="******",
uid=1,
temp="C:\\Users\\jason\\AppData\\Local\\Temp"))
paths = artifact_lib.InterpolateKbAttributes(r"%%users.temp%%\abcd",
kb)
self.assertItemsEqual(paths,
["C:\\Users\\jason\\AppData\\Local\\Temp\\abcd"])
def InitializeKnowledgeBase(self):
"""Get the existing KB or create a new one if none exists."""
self.client = aff4.FACTORY.Open(self.client_id, token=self.token)
# Always create a new KB to override any old values.
self.state.knowledge_base = rdf_client.KnowledgeBase()
SetCoreGRRKnowledgeBaseValues(self.state.knowledge_base, self.client)
if not self.state.knowledge_base.os:
# If we don't know what OS this is, there is no way to proceed.
raise flow.FlowError("Client OS not set for: %s, cannot initialize"
" KnowledgeBase" % self.client_id)
def testKnowledgeBase(self):
"""Test that the knowledge base is passed in the bundle."""
artifact_collector = collectors.ClientArtifactCollector(None)
artifact_collector.args = artifact_utils.ArtifactCollectorFlowArgs()
kb = rdf_client.KnowledgeBase()
kb.os = "Windows"
artifact_collector.args.knowledge_base = kb
artifact_bundle = artifact_collector._GetArtifactCollectorArgs([])
self.assertEqual(artifact_bundle.knowledge_base.os, "Windows")
def setUp(self):
# We need to store the KB so we can put it back after the test.
client = aff4.FACTORY.Open(self.client_id, mode="rw", token=self.token)
kb_backup = aff4.FACTORY.Create(
self.client_id.Add("temp").Add("TestParserDependency"),
aff4_grr.TempKnowledgeBase,
token=self.token)
kb_backup.Set(kb_backup.Schema.KNOWLEDGE_BASE,
client.Get(client.Schema.KNOWLEDGE_BASE))
kb_backup.Close()
# Set the KB to an empty object
client.Set(client.Schema.KNOWLEDGE_BASE, rdf_client.KnowledgeBase())
client.Flush()
super(TestParserDependency, self).setUp()
def testBasicParsing(self):
ps_list_file = os.path.join(config_lib.CONFIG["Test.data_dir"],
"rekall_vad_result.dat.gz")
result = rdf_rekall_types.RekallResponse(
json_messages=gzip.open(ps_list_file).read(10000000),
plugin="pslist")
knowledge_base = rdf_client.KnowledgeBase()
knowledge_base.environ_systemdrive = "C:"
parser = rekall_artifact_parser.RekallVADParser()
parsed_pathspecs = list(parser.Parse(result, knowledge_base))
paths = [p.path for p in parsed_pathspecs]
self.assertIn(u"C:\\Windows\\System32\\spoolsv.exe", paths)
def _SetupFullInfoClients(self):
for i in range(10):
client_id = self.InitializeClient("C.000000005000000%d" % i)
cl = objects.ClientSnapshot(
client_id=client_id,
knowledge_base=rdf_client.KnowledgeBase(
fqdn="test%d.examples.com" % i),
kernel="12.3.%d" % i)
self.db.WriteClientSnapshot(cl)
self.db.WriteClientMetadata(client_id, certificate=CERT)
si = rdf_client.StartupInfo(boot_time=i)
self.db.WriteClientStartupInfo(client_id, si)
self.db.AddClientLabels(
client_id, "test_owner",
["test_label-a-%d" % i,
"test_label-b-%d" % i])
def testParse(self):
parser = windows_persistence.WindowsPersistenceMechanismsParser()
path = (r"HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\CurrentVersion"
r"\Run\test")
pathspec = rdf_paths.PathSpec(
path=path, pathtype=rdf_paths.PathSpec.PathType.REGISTRY)
reg_data = "C:\\blah\\some.exe /v"
reg_type = rdf_client.StatEntry.RegistryType.REG_SZ
stat = rdf_client.StatEntry(
aff4path="aff4:/asdfasdf/",
pathspec=pathspec,
registry_type=reg_type,
registry_data=rdf_protodict.DataBlob(string=reg_data))
persistence = [stat]
image_paths = [
"system32\\drivers\\ACPI.sys",
"%systemroot%\\system32\\svchost.exe -k netsvcs",
"\\SystemRoot\\system32\\drivers\\acpipmi.sys"
]
reg_key = rdfvalue.RDFURN("aff4:/C.1000000000000000/registry"
"/HKEY_LOCAL_MACHINE/SYSTEM/ControlSet001"
"/services/AcpiPmi")
for path in image_paths:
serv_info = rdf_client.WindowsServiceInformation(
name="blah",
display_name="GRRservice",
image_path=path,
registry_key=reg_key)
persistence.append(serv_info)
knowledge_base = rdf_client.KnowledgeBase()
knowledge_base.environ_systemroot = "C:\\Windows"
expected = [
"C:\\blah\\some.exe", "C:\\Windows\\system32\\drivers\\ACPI.sys",
"C:\\Windows\\system32\\svchost.exe",
"C:\\Windows\\system32\\drivers\\acpipmi.sys"
]
for index, item in enumerate(persistence):
results = list(
parser.Parse(item, knowledge_base,
rdf_paths.PathSpec.PathType.OS))
self.assertEqual(results[0].pathspec.path, expected[index])
self.assertEqual(len(results), 1)
def testAnalyzeClient(self):
index = aff4.FACTORY.Create(
"aff4:/client-index/",
aff4_type=client_index.ClientIndex,
mode="rw",
token=self.token)
test_lib.ClientFixture("aff4:/" + CLIENT_ID, token=self.token)
client = aff4.FACTORY.Create(
"aff4:/" + CLIENT_ID,
aff4_type=aff4_grr.VFSGRRClient,
mode="rw",
token=self.token)
kb = rdf_client.KnowledgeBase()
kb.users.Append(
rdf_client.User(
username="******",
full_name="Eric (Bertrand ) 'Russell' \"Logician\" Jacobson"))
kb.users.Append(
rdf_client.User(
username="******", full_name="Steve O'Bryan"))
client.Set(client.Schema.KNOWLEDGE_BASE(kb))
_, keywords = index.AnalyzeClient(client)
# Should not contain an empty string.
self.assertNotIn("", keywords)
# OS of the client
self.assertIn("windows", keywords)
# Users of the client.
self.assertIn("bert", keywords)
self.assertIn("bertrand", keywords)
self.assertNotIn(")", keywords)
self.assertIn("russell", keywords)
self.assertIn("logician", keywords)
self.assertIn("ernie", keywords)
self.assertIn("eric", keywords)
self.assertIn("jacobson", keywords)
self.assertIn("steve o'bryan", keywords)
self.assertIn("o'bryan", keywords)
# Client information.
self.assertIn("grr monitor", keywords)
self.assertIn("client-label-23", keywords)
