def setUp(self):
"""Make sure things are initialized."""
super(TestArtifactCollectors, self).setUp()
self._patcher = artifact_test_lib.PatchDefaultArtifactRegistry()
self._patcher.start()
test_artifacts_file = os.path.join(config.CONFIG["Test.data_dir"],
"artifacts", "test_artifacts.json")
artifact_registry.REGISTRY.AddFileSource(test_artifacts_file)
self.fakeartifact = artifact_registry.REGISTRY.GetArtifact(
"FakeArtifact")
self.fakeartifact2 = artifact_registry.REGISTRY.GetArtifact(
"FakeArtifact2")
self.output_count = 0
self.client_id = self.SetupClient(0)
with aff4.FACTORY.Open(self.client_id, token=self.token,
mode="rw") as fd:
fd.Set(fd.Schema.SYSTEM("Linux"))
kb = fd.Schema.KNOWLEDGE_BASE()
artifact.SetCoreGRRKnowledgeBaseValues(kb, fd)
fd.Set(kb)
def setUp(self):
super(TestClientArtifactCollector, self).setUp()
test_artifacts_file = os.path.join(config.CONFIG["Test.data_dir"],
"artifacts", "test_artifacts.json")
artifact_registry.REGISTRY.AddFileSource(test_artifacts_file)
self.client_id = self.SetupClient(0)
with aff4.FACTORY.Open(self.client_id, token=self.token, mode="rw") as fd:
fd.Set(fd.Schema.SYSTEM("Linux"))
kb = fd.Schema.KNOWLEDGE_BASE()
artifact.SetCoreGRRKnowledgeBaseValues(kb, fd)
fd.Set(kb)
def RunFlow(self, client_id, artifact_list=None, use_tsk=False):
if artifact_list is None:
artifact_list = ["WindowsRunKeys"]
client_id = self.SetupClient(0, system="Windows", os_version="6.2")
with aff4.FACTORY.Open(client_id, token=self.token, mode="rw") as fd:
fd.Set(fd.Schema.SYSTEM("Windows"))
kb = fd.Schema.KNOWLEDGE_BASE()
artifact.SetCoreGRRKnowledgeBaseValues(kb, fd)
fd.Set(kb)
urn = flow_test_lib.TestFlowHelper(
collectors.ArtifactFilesDownloaderFlow.__name__,
client_id=client_id,
artifact_list=artifact_list,
use_tsk=use_tsk,
token=self.token)
results_fd = flow.GRRFlow.ResultCollectionForFID(urn)
return list(results_fd)
def CreateClientObject(self, vfs_fixture):
"""Make a new client object."""
# First remove the old fixture just in case its still there.
aff4.FACTORY.Delete(self.client_id, token=self.token)
# Create the fixture at a fixed time.
with test_lib.FakeTime(self.age):
for path, (aff4_type, attributes) in vfs_fixture:
path %= self.args
aff4_object = aff4.FACTORY.Create(self.client_id.Add(path),
aff4_type,
mode="rw",
token=self.token)
if data_store.RelationalDBWriteEnabled():
data_store.REL_DB.WriteClientMetadata(
self.client_id.Basename(), fleetspeak_enabled=False)
components = [
component for component in path.split("/") if component
]
if components[0:2] == ["fs", "os"]:
path_info = rdf_objects.PathInfo()
path_info.path_type = rdf_objects.PathInfo.PathType.OS
path_info.components = components[2:]
if aff4_type in [
aff4_grr.VFSFile, aff4_grr.VFSMemoryFile
]:
path_info.directory = False
elif aff4_type == aff4_standard.VFSDirectory:
path_info.directory = True
else:
raise ValueError("Incorrect AFF4 type: %s" %
aff4_type)
data_store.REL_DB.WritePathInfos(
client_id=self.client_id.Basename(),
path_infos=[path_info])
for attribute_name, value in iteritems(attributes):
attribute = aff4.Attribute.PREDICATES[attribute_name]
if isinstance(value, (str, unicode)):
# Interpolate the value
value %= self.args
# Is this supposed to be an RDFValue array?
if aff4.issubclass(attribute.attribute_type,
rdf_protodict.RDFValueArray):
rdfvalue_object = attribute()
for item in value:
new_object = rdfvalue_object.rdf_type.FromTextFormat(
utils.SmartStr(item))
rdfvalue_object.Append(new_object)
# It is a text serialized protobuf.
elif aff4.issubclass(attribute.attribute_type,
rdf_structs.RDFProtoStruct):
# Use the alternate constructor - we always write protobufs in
# textual form:
rdfvalue_object = attribute.attribute_type.FromTextFormat(
utils.SmartStr(value))
elif aff4.issubclass(attribute.attribute_type,
rdfvalue.RDFInteger):
rdfvalue_object = attribute(int(value))
else:
rdfvalue_object = attribute(value)
# If we don't already have a pathspec, try and get one from the stat.
if aff4_object.Get(aff4_object.Schema.PATHSPEC) is None:
# If the attribute was a stat, it has a pathspec nested in it.
# We should add that pathspec as an attribute.
if attribute.attribute_type == rdf_client_fs.StatEntry:
stat_object = attribute.attribute_type.FromTextFormat(
utils.SmartStr(value))
if stat_object.pathspec:
pathspec_attribute = aff4.Attribute(
"aff4:pathspec", rdf_paths.PathSpec,
"The pathspec used to retrieve "
"this object from the client.", "pathspec")
aff4_object.AddAttribute(
pathspec_attribute, stat_object.pathspec)
if attribute in ["aff4:content", "aff4:content"]:
# For AFF4MemoryStreams we need to call Write() instead of
# directly setting the contents..
aff4_object.Write(rdfvalue_object.AsBytes())
else:
aff4_object.AddAttribute(attribute, rdfvalue_object)
if (isinstance(rdfvalue_object, rdf_client_fs.StatEntry)
and rdfvalue_object.pathspec.pathtype != "UNSET"):
if data_store.RelationalDBWriteEnabled():
client_id = self.client_id.Basename()
path_info = rdf_objects.PathInfo.FromStatEntry(
rdfvalue_object)
data_store.REL_DB.WritePathInfos(
client_id, [path_info])
# Populate the KB from the client attributes.
if aff4_type == aff4_grr.VFSGRRClient:
kb = rdf_client.KnowledgeBase()
artifact.SetCoreGRRKnowledgeBaseValues(kb, aff4_object)
aff4_object.Set(aff4_object.Schema.KNOWLEDGE_BASE, kb)
# Make sure we do not actually close the object here - we only want to
# sync back its attributes, not run any finalization code.
aff4_object.Flush()
if aff4_type == aff4_grr.VFSGRRClient:
index = client_index.CreateClientIndex(token=self.token)
index.AddClient(aff4_object)
def _SetupClientImpl(self,
client_nr,
index=None,
arch="x86_64",
fqdn=None,
install_time=None,
last_boot_time=None,
kernel="4.0.0",
os_version="buster/sid",
ping=None,
system="Linux",
memory_size=None,
add_cert=True,
fleetspeak_enabled=False):
client_id_urn = rdf_client.ClientURN("C.1%015x" % client_nr)
with aff4.FACTORY.Create(client_id_urn,
aff4_grr.VFSGRRClient,
mode="rw",
token=self.token) as fd:
if add_cert:
cert = self.ClientCertFromPrivateKey(
config.CONFIG["Client.private_key"])
fd.Set(fd.Schema.CERT, cert)
fd.Set(fd.Schema.CLIENT_INFO, self._TestClientInfo())
fd.Set(fd.Schema.PING, ping or rdfvalue.RDFDatetime.Now())
if fqdn is not None:
fd.Set(fd.Schema.HOSTNAME(fqdn.split(".", 1)[0]))
fd.Set(fd.Schema.FQDN(fqdn))
else:
fd.Set(fd.Schema.HOSTNAME("Host-%x" % client_nr))
fd.Set(fd.Schema.FQDN("Host-%x.example.com" % client_nr))
fd.Set(
fd.Schema.MAC_ADDRESS("aabbccddee%02x\nbbccddeeff%02x" %
(client_nr, client_nr)))
fd.Set(
fd.Schema.HOST_IPS("192.168.0.%d\n2001:abcd::%x" %
(client_nr, client_nr)))
if system:
fd.Set(fd.Schema.SYSTEM(system))
if os_version:
fd.Set(fd.Schema.OS_VERSION(os_version))
if arch:
fd.Set(fd.Schema.ARCH(arch))
if kernel:
fd.Set(fd.Schema.KERNEL(kernel))
if memory_size:
fd.Set(fd.Schema.MEMORY_SIZE(memory_size))
if last_boot_time:
fd.Set(fd.Schema.LAST_BOOT_TIME(last_boot_time))
if install_time:
fd.Set(fd.Schema.INSTALL_DATE(install_time))
if fleetspeak_enabled:
fd.Set(fd.Schema.FLEETSPEAK_ENABLED, rdfvalue.RDFBool(True))
kb = rdf_client.KnowledgeBase()
kb.fqdn = fqdn or "Host-%x.example.com" % client_nr
kb.users = [
rdf_client.User(username="******"),
rdf_client.User(username="******"),
]
artifact.SetCoreGRRKnowledgeBaseValues(kb, fd)
fd.Set(fd.Schema.KNOWLEDGE_BASE, kb)
fd.Set(fd.Schema.INTERFACES(self._TestInterfaces(client_nr)))
hardware_info = fd.Schema.HARDWARE_INFO()
hardware_info.system_manufacturer = ("System-Manufacturer-%x" %
client_nr)
hardware_info.bios_version = ("Bios-Version-%x" % client_nr)
fd.Set(fd.Schema.HARDWARE_INFO, hardware_info)
fd.Flush()
index.AddClient(fd)
return client_id_urn
def CreateClientObject(self, vfs_fixture):
"""Make a new client object."""
# First remove the old fixture just in case its still there.
aff4.FACTORY.Delete(self.client_id, token=self.token)
# Create the fixture at a fixed time.
with test_lib.FakeTime(self.age):
if data_store.RelationalDBWriteEnabled():
# Constructing a client snapshot from the aff4 fixture is only possible
# using aff4. Using a serialized string instead.
data_store.REL_DB.WriteClientMetadata(self.client_id,
fleetspeak_enabled=False)
snapshot = rdf_objects.ClientSnapshot.FromSerializedString(
SERIALIZED_CLIENT.decode("hex"))
snapshot.client_id = self.client_id
snapshot.knowledge_base.fqdn = "Host%s" % self.client_id
data_store.REL_DB.WriteClientSnapshot(snapshot)
client_index.ClientIndex().AddClient(snapshot)
for path, (aff4_type, attributes) in vfs_fixture:
path %= self.args
if data_store.AFF4Enabled():
aff4_object = aff4.FACTORY.Create(
self.client_urn.Add(path),
aff4_type,
mode="rw",
token=self.token)
path_info = None
if data_store.RelationalDBWriteEnabled():
components = [
component for component in path.split("/") if component
]
if (len(components) > 1 and components[0] == "fs"
and components[1] in ["os", "tsk"]):
path_info = rdf_objects.PathInfo()
if components[1] == "os":
path_info.path_type = rdf_objects.PathInfo.PathType.OS
else:
path_info.path_type = rdf_objects.PathInfo.PathType.TSK
path_info.components = components[2:]
if aff4_type in [
aff4_grr.VFSFile, aff4_grr.VFSMemoryFile
]:
path_info.directory = False
elif aff4_type == aff4_standard.VFSDirectory:
path_info.directory = True
else:
raise ValueError("Incorrect AFF4 type: %s" %
aff4_type)
for attribute_name, value in iteritems(attributes):
attribute = aff4.Attribute.PREDICATES[attribute_name]
if isinstance(value, (bytes, Text)):
# Interpolate the value
value %= self.args
# Is this supposed to be an RDFValue array?
if issubclass(attribute.attribute_type,
rdf_protodict.RDFValueArray):
rdfvalue_object = attribute()
for item in value:
new_object = rdfvalue_object.rdf_type.FromTextFormat(
utils.SmartStr(item))
rdfvalue_object.Append(new_object)
# It is a text serialized protobuf.
elif issubclass(attribute.attribute_type,
rdf_structs.RDFProtoStruct):
# Use the alternate constructor - we always write protobufs in
# textual form:
rdfvalue_object = attribute.attribute_type.FromTextFormat(
utils.SmartStr(value))
elif issubclass(attribute.attribute_type,
rdfvalue.RDFInteger):
rdfvalue_object = attribute(int(value))
else:
rdfvalue_object = attribute(value)
if data_store.AFF4Enabled():
# If we don't already have a pathspec, try and get one from the
# stat.
if aff4_object.Get(
aff4_object.Schema.PATHSPEC) is None:
# If the attribute was a stat, it has a pathspec nested in it.
# We should add that pathspec as an attribute.
if attribute.attribute_type == rdf_client_fs.StatEntry:
stat_object = attribute.attribute_type.FromTextFormat(
utils.SmartStr(value))
if stat_object.pathspec:
pathspec_attribute = aff4.Attribute(
"aff4:pathspec", rdf_paths.PathSpec,
"The pathspec used to retrieve "
"this object from the client.",
"pathspec")
aff4_object.AddAttribute(
pathspec_attribute,
stat_object.pathspec)
if attribute in ["aff4:content", "aff4:content"]:
content = rdfvalue_object.AsBytes()
if data_store.AFF4Enabled():
# For AFF4MemoryStreams we need to call Write() instead of
# directly setting the contents..
aff4_object.Write(content)
if path_info is not None:
blob_id = rdf_objects.BlobID.FromBlobData(content)
data_store.BLOBS.WriteBlobs({blob_id: content})
blob_ref = rdf_objects.BlobReference(
offset=0, size=len(content), blob_id=blob_id)
hash_id = file_store.AddFileWithUnknownHash(
db.ClientPath.FromPathInfo(
self.client_id, path_info), [blob_ref])
path_info.hash_entry.num_bytes = len(content)
path_info.hash_entry.sha256 = hash_id.AsBytes()
elif data_store.AFF4Enabled():
aff4_object.AddAttribute(attribute, rdfvalue_object)
if (isinstance(rdfvalue_object, rdf_client_fs.StatEntry)
and rdfvalue_object.pathspec.pathtype != "UNSET"):
if data_store.RelationalDBWriteEnabled():
path_info = rdf_objects.PathInfo.FromStatEntry(
rdfvalue_object)
data_store.REL_DB.WritePathInfos(
self.client_id, [path_info])
# Populate the KB from the client attributes.
if aff4_type == aff4_grr.VFSGRRClient:
if data_store.AFF4Enabled():
kb = rdf_client.KnowledgeBase()
artifact.SetCoreGRRKnowledgeBaseValues(kb, aff4_object)
aff4_object.Set(aff4_object.Schema.KNOWLEDGE_BASE, kb)
aff4_object.Flush()
index = client_index.CreateClientIndex(
token=self.token)
index.AddClient(aff4_object)
if path_info is not None:
data_store.REL_DB.WritePathInfos(client_id=self.client_id,
path_infos=[path_info])
if data_store.AFF4Enabled():
aff4_object.Flush()
