def Platform(self, responses):
"""Stores information about the platform."""
if responses.success:
response = responses.First()
# These need to be in separate attributes because they get searched on in
# the GUI
with self._OpenClient(mode="rw") as client:
client.Set(client.Schema.HOSTNAME(response.node))
client.Set(client.Schema.SYSTEM(response.system))
client.Set(client.Schema.OS_RELEASE(response.release))
client.Set(client.Schema.OS_VERSION(response.version))
client.Set(client.Schema.KERNEL(response.kernel))
client.Set(client.Schema.FQDN(response.fqdn))
# response.machine is the machine value of platform.uname()
# On Windows this is the value of:
# HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session
# Manager\Environment\PROCESSOR_ARCHITECTURE
# "AMD64", "IA64" or "x86"
client.Set(client.Schema.ARCH(response.machine))
client.Set(
client.Schema.UNAME("%s-%s-%s" % (response.system, response.release,
response.version)))
# Update the client index
client_index.CreateClientIndex(token=self.token).AddClient(client)
if response.system == "Windows":
with aff4.FACTORY.Create(
self.client_id.Add("registry"),
standard.VFSDirectory,
token=self.token) as fd:
fd.Set(fd.Schema.PATHSPEC,
fd.Schema.PATHSPEC(
path="/", pathtype=rdf_paths.PathSpec.PathType.REGISTRY))
# No support for OS X cloud machines as yet.
if response.system in ["Linux", "Windows"]:
self.CallClient(
server_stubs.GetCloudVMMetadata,
cloud.BuildCloudMetadataRequests(),
next_state="CloudMetadata")
known_system_type = True
else:
client = self._OpenClient()
known_system_type = client.Get(client.Schema.SYSTEM)
self.Log("Could not retrieve Platform info.")
if known_system_type:
# We will accept a partial KBInit rather than raise, so pass
# require_complete=False.
self.CallFlow(
"KnowledgeBaseInitializationFlow",
require_complete=False,
lightweight=self.args.lightweight,
next_state="ProcessKnowledgeBase")
else:
self.Log("Unknown system type, skipping KnowledgeBaseInitializationFlow")
def Handle(self, args, token=None):
if args.count:
end = args.offset + args.count
else:
end = sys.maxint
keywords = shlex.split(args.query)
index = client_index.CreateClientIndex(token=token)
all_urns = set()
for label in self.labels_whitelist:
label_filter = ["label:" + label] + keywords
all_urns.update(index.LookupClients(label_filter))
all_objs = aff4.FACTORY.MultiOpen(sorted(all_urns, key=str),
aff4_type=aff4_grr.VFSGRRClient,
token=token)
api_clients = []
index = 0
for client_obj in all_objs:
if self._CheckClientLabels(client_obj):
if index >= args.offset and index < end:
api_clients.append(
ApiClient().InitFromAff4Object(client_obj))
index += 1
if index >= end:
break
return ApiSearchClientsResult(items=api_clients)
def Handle(self, args, token=None):
audit_description = ",".join([
token.username + u"." + utils.SmartUnicode(name)
for name in args.labels
])
audit_events = []
try:
index = client_index.CreateClientIndex(token=token)
client_objs = aff4.FACTORY.MultiOpen(
[cid.ToClientURN() for cid in args.client_ids],
aff4_type=aff4_grr.VFSGRRClient,
mode="rw",
token=token)
for client_obj in client_objs:
client_obj.AddLabels(*args.labels)
index.AddClient(client_obj)
client_obj.Close()
audit_events.append(
events.AuditEvent(
user=token.username,
action="CLIENT_ADD_LABEL",
flow_name="handler.ApiAddClientsLabelsHandler",
client=client_obj.urn,
description=audit_description))
finally:
events.Events.PublishMultipleEvents(
{audit.AUDIT_EVENT: audit_events}, token=token)
def SetupClient(self,
client_nr,
index=None,
system=None,
os_version=None,
arch=None):
"""Prepares a test client mock to be used.
Args:
client_nr: int The GRR ID to be used. 0xABCD maps to C.100000000000abcd
in canonical representation.
index: client_index.ClientIndex
system: string
os_version: string
arch: string
Returns:
rdf_client.ClientURN
"""
if index is not None:
# `with:' is expected to be used in the calling function.
client_id_urn = self._SetupClientImpl(client_nr, index, system,
os_version, arch)
else:
with client_index.CreateClientIndex(token=self.token) as index:
client_id_urn = self._SetupClientImpl(client_nr, index, system,
os_version, arch)
return client_id_urn
def End(self):
"""Finalize client registration."""
# Update summary and publish to the Discovery queue.
client = self._OpenClient()
summary = client.GetSummary()
self.Publish("Discovery", summary)
self.SendReply(summary)
# Update the client index
client_index.CreateClientIndex(token=self.token).AddClient(client)
def _MakeFixtures(self):
token = access_control.ACLToken(username="******", reason="Make fixtures.")
token = token.SetUID()
for i in range(10):
client_id = rdf_client.ClientURN("C.%016X" % i)
with aff4.FACTORY.Create(
client_id, aff4_grr.VFSGRRClient, mode="rw",
token=token) as client_obj:
index = client_index.CreateClientIndex(token=token)
index.AddClient(client_obj)
def GetClientTarget(self, args, token=None):
# Find the right client to target using a hostname search.
index = client_index.CreateClientIndex(token=token)
client_list = index.LookupClients([args.hostname])
if not client_list:
raise ValueError("No client found matching %s" % args.hostname)
# If we get more than one, take the one with the most recent poll.
if len(client_list) > 1:
return client_index.GetMostRecentClient(client_list, token=token)
else:
return client_list[0]
def SetupClients(self,
nr_clients,
system=None,
os_version=None,
arch=None):
"""Prepares nr_clients test client mocks to be used."""
with client_index.CreateClientIndex(token=self.token) as index:
client_ids = [
self.SetupClient(client_nr, index, system, os_version, arch)
for client_nr in xrange(nr_clients)
]
return client_ids
def _MakeFixtures(self):
# Install the mock security manager so we can trap errors in interactive
# mode.
data_store.DB.security_manager = test_lib.MockSecurityManager()
token = access_control.ACLToken(username="******", reason="Make fixtures.")
token = token.SetUID()
for i in range(10):
client_id = rdf_client.ClientURN("C.%016X" % i)
with aff4.FACTORY.Create(
client_id, aff4_grr.VFSGRRClient, mode="rw",
token=token) as client_obj:
index = client_index.CreateClientIndex(token=token)
index.AddClient(client_obj)
def Start(self):
"""Sign the CSR from the client."""
client = aff4.FACTORY.Create(self.client_id,
aff4_grr.VFSGRRClient,
mode="rw",
token=self.token)
if self.args.csr.type != rdf_crypto.Certificate.Type.CSR:
raise IOError("Must be called with CSR")
csr = rdf_crypto.CertificateSigningRequest(self.args.csr.pem)
# Verify the CSR. This is not strictly necessary but doesn't harm either.
try:
csr.Verify(csr.GetPublicKey())
except rdf_crypto.VerificationError:
raise flow.FlowError("CSR for client %s did not verify: %s" %
(self.client_id, csr.AsPEM()))
# Verify that the CN is of the correct form. The common name should refer
# to a client URN.
self.cn = rdf_client.ClientURN.FromPublicKey(csr.GetPublicKey())
if self.cn != csr.GetCN():
raise IOError("CSR CN %s does not match public key %s." %
(csr.GetCN(), self.cn))
logging.info("Will sign CSR for: %s", self.cn)
cert = rdf_crypto.RDFX509Cert.ClientCertFromCSR(csr)
# This check is important to ensure that the client id reported in the
# source of the enrollment request is the same as the one in the
# certificate. We use the ClientURN to ensure this is also of the correct
# form for a client name.
if self.cn != self.client_id:
raise flow.FlowError("Certificate name %s mismatch for client %s",
self.cn, self.client_id)
# Set and write the certificate to the client record.
client.Set(client.Schema.CERT, cert)
client.Set(client.Schema.FIRST_SEEN, rdfvalue.RDFDatetime.Now())
index = client_index.CreateClientIndex(token=self.token)
index.AddClient(client)
client.Close(sync=True)
# Publish the client enrollment message.
self.Publish("ClientEnrollment", self.client_id)
self.Log("Enrolled %s successfully", self.client_id)
def Handle(self, args, token=None):
end = args.count or sys.maxint
keywords = shlex.split(args.query)
index = client_index.CreateClientIndex(token=token)
result_urns = sorted(index.LookupClients(keywords),
key=str)[args.offset:args.offset + end]
result_set = aff4.FACTORY.MultiOpen(result_urns, token=token)
api_clients = []
for child in result_set:
api_clients.append(ApiClient().InitFromAff4Object(child))
return ApiSearchClientsResult(items=api_clients)
def SearchClients(query_str, token=None, limit=1000):
"""Search indexes for clients. Returns list (client, hostname, os version)."""
client_schema = aff4.AFF4Object.classes["VFSGRRClient"].SchemaCls
index = client_index.CreateClientIndex(token=token)
client_list = index.LookupClients([query_str])
result_set = aff4.FACTORY.MultiOpen(client_list, token=token)
results = []
for result in result_set:
results.append((result, str(result.Get(client_schema.HOSTNAME)),
str(result.Get(client_schema.OS_VERSION)),
str(result.Get(client_schema.PING))))
if len(results) >= limit:
break
return results
def ProcessKnowledgeBase(self, responses):
"""Collect and store any extra non-kb artifacts."""
if not responses.success:
raise flow.FlowError("Error collecting artifacts: %s" %
responses.status)
# Collect any non-knowledgebase artifacts that will be stored in aff4.
artifact_list = self._GetExtraArtifactsForCollection()
if artifact_list:
self.CallFlow("ArtifactCollectorFlow",
artifact_list=artifact_list,
next_state="ProcessArtifactResponses",
store_results_in_aff4=True)
# Update the client index
client = self._OpenClient()
client_index.CreateClientIndex(token=self.token).AddClient(client)
def CleanClientVersions(clients=None, dry_run=True, token=None):
"""A script to remove excessive client versions.
Especially when a client is heavily cloned, we sometimes write an excessive
number of versions of it. Since these version all go into the same database
row and are displayed as a dropdown list in the adminui, it is sometimes
necessary to clear them out.
This deletes version from clients so that we have at most one
version per hour.
Args:
clients: A list of ClientURN, if empty cleans all clients.
dry_run: whether this is a dry run
token: datastore token.
"""
if not clients:
index = client_index.CreateClientIndex(token=token)
clients = index.LookupClients(["."])
clients.sort()
pool = data_store.MutationPool(token=token)
logging.info("checking %d clients", len(clients))
client_infos = data_store.DB.MultiResolvePrefix(
clients, "aff4:type", data_store.DB.ALL_TIMESTAMPS, token=token)
for client, type_list in client_infos:
logging.info("%s: has %d versions", client, len(type_list))
cleared = 0
kept = 1
last_kept = type_list[0][2]
for _, _, ts in type_list[1:]:
if last_kept - ts > 60 * 60 * 1000000: # 1 hour
last_kept = ts
kept += 1
else:
if not dry_run:
pool.DeleteAttributes(client, ["aff4:type"],
start=ts,
end=ts)
cleared += 1
if pool.Size() > 10000:
pool.Flush()
logging.info("%s: kept %d and cleared %d", client, kept, cleared)
pool.Flush()
def ExportClientsByKeywords(keywords, filename, token=None):
r"""A script to export clients summaries selected by a keyword search.
This script does a client search for machines matching all of keywords and
writes a .csv summary of the results to filename. Multi-value fields are '\n'
separated.
Args:
keywords: a list of keywords to search for
filename: the name of the file to write to, will be replaced if already
present
token: datastore token.
"""
index = client_index.CreateClientIndex(token=token)
client_list = index.LookupClients(keywords)
logging.info("found %d clients", len(client_list))
if not client_list:
return
result_set = aff4.FACTORY.MultiOpen(client_list, token=token)
with open(filename, "wb") as csv_out:
writer = csv.DictWriter(csv_out, [
"client_id", "hostname", "last_seen", "os", "os_release",
"os_version", "users", "ips", "macs"
])
writer.writeheader()
for client in result_set:
s = client.Schema
writer.writerow({
"client_id": client.urn.Basename(),
"hostname": client.Get(s.HOSTNAME),
"os": client.Get(s.SYSTEM),
"os_release": client.Get(s.OS_RELEASE),
"os_version": client.Get(s.OS_VERSION),
"ips": client.Get(s.HOST_IPS),
"macs": client.Get(s.MAC_ADDRESS),
"users": "\n".join(client.Get(s.USERNAMES, [])),
"last_seen": client.Get(s.PING),
})
def setUp(self):
super(ApiLabelsRestrictedSearchClientsHandlerTest, self).setUp()
self.client_ids = self.SetupClients(4)
index = client_index.CreateClientIndex(token=self.token)
def LabelClient(i, label, owner):
with aff4.FACTORY.Open(
self.client_ids[i], mode="rw", token=self.token) as grr_client:
grr_client.AddLabels(label, owner=owner)
index.AddClient(grr_client)
LabelClient(0, "foo", "david")
LabelClient(1, "not-foo", "david")
LabelClient(2, "bar", "peter_another")
LabelClient(3, "bar", "peter")
self.handler = client_plugin.ApiLabelsRestrictedSearchClientsHandler(
labels_whitelist=["foo", "bar"],
labels_owners_whitelist=["david", "peter"])
def _CheckLabelIndex(self):
"""Check that label indexes are updated."""
index = client_index.CreateClientIndex(token=self.token)
self.assertEqual(list(index.LookupClients(["label:Label2"])),
[self.client_id])
def GetAllClients(token=None):
"""Return a list of all client urns."""
index = client_index.CreateClientIndex(token=token)
return index.LookupClients(["."])
def _CheckClientKwIndex(self, keywords, expected_count):
# Tests that the client index has expected_count results when
# searched for keywords.
index = client_index.CreateClientIndex(token=self.token)
self.assertEqual(len(index.LookupClients(keywords)), expected_count)
def FindClonedClients(token=None):
"""A script to find multiple machines reporting the same client_id.
This script looks at the hardware serial numbers that a client reported in
over time (they get collected with each regular interrogate). We have seen
that sometimes those serial numbers change - for example when a disk is put
in a new machine - so reporting multiple serial numbers does not flag a client
immediately as a cloned machine. In order to be shown here by this script, the
serial number has to be alternating between two values.
Args:
token: datastore token.
Returns:
A list of clients that report alternating hardware ids.
"""
index = client_index.CreateClientIndex(token=token)
clients = index.LookupClients(["."])
hw_infos = _GetHWInfos(clients, token=token)
# We get all clients that have reported more than one hardware serial
# number over time. This doesn't necessarily indicate a cloned client - the
# machine might just have new hardware. We need to search for clients that
# alternate between different IDs.
clients_with_multiple_serials = [
client_id for client_id, serials in hw_infos.iteritems()
if len(serials) > 1
]
client_list = aff4.FACTORY.MultiOpen(clients_with_multiple_serials,
age=aff4.ALL_TIMES,
token=token)
cloned_clients = []
for c in client_list:
hwis = c.GetValuesForAttribute(c.Schema.HARDWARE_INFO)
# Here we search for the earliest and latest time each ID was reported.
max_index = {}
min_index = {}
ids = set()
for i, hwi in enumerate(hwis):
s = hwi.serial_number
max_index[s] = i
if s not in min_index:
min_index[s] = i
ids.add(s)
# Construct ranges [first occurrence, last occurrence] for every ID. If
# a client just changed from one ID to the other, those ranges of IDs should
# be disjunct. If they overlap at some point, it indicates that two IDs were
# reported in the same time frame.
ranges = []
for hwid in ids:
ranges.append((min_index[hwid], max_index[hwid]))
# Sort ranges by first occurrence time.
ranges.sort()
for i in xrange(len(ranges) - 1):
if ranges[i][1] > ranges[i + 1][0]:
cloned_clients.append(c)
msg = "Found client with multiple, overlapping serial numbers: %s"
logging.info(msg, c.urn)
for hwi in c.GetValuesForAttribute(c.Schema.HARDWARE_INFO):
logging.info("%s %s", hwi.age, hwi.serial_number)
break
return cloned_clients
def _UpdateClientIndex(self, client_obj):
with client_index.CreateClientIndex(token=self.token) as index:
index.AddClient(client_obj)
def CreateClientObject(self, vfs_fixture):
"""Make a new client object."""
# First remove the old fixture just in case its still there.
aff4.FACTORY.Delete(self.client_id, token=self.token)
# Create the fixture at a fixed time.
with test_lib.FakeTime(self.age):
for path, (aff4_type, attributes) in vfs_fixture:
path %= self.args
aff4_object = aff4.FACTORY.Create(self.client_id.Add(path),
aff4_type,
mode="rw",
token=self.token)
for attribute_name, value in attributes.items():
attribute = aff4.Attribute.PREDICATES[attribute_name]
if isinstance(value, (str, unicode)):
# Interpolate the value
value %= self.args
# Is this supposed to be an RDFValue array?
if aff4.issubclass(attribute.attribute_type,
rdf_protodict.RDFValueArray):
rdfvalue_object = attribute()
for item in value:
new_object = rdfvalue_object.rdf_type.FromTextFormat(
utils.SmartStr(item))
rdfvalue_object.Append(new_object)
# It is a text serialized protobuf.
elif aff4.issubclass(attribute.attribute_type,
rdf_structs.RDFProtoStruct):
# Use the alternate constructor - we always write protobufs in
# textual form:
rdfvalue_object = attribute.attribute_type.FromTextFormat(
utils.SmartStr(value))
elif aff4.issubclass(attribute.attribute_type,
rdfvalue.RDFInteger):
rdfvalue_object = attribute(int(value))
else:
rdfvalue_object = attribute(value)
# If we don't already have a pathspec, try and get one from the stat.
if aff4_object.Get(aff4_object.Schema.PATHSPEC) is None:
# If the attribute was a stat, it has a pathspec nested in it.
# We should add that pathspec as an attribute.
if attribute.attribute_type == rdf_client.StatEntry:
stat_object = attribute.attribute_type.FromTextFormat(
utils.SmartStr(value))
if stat_object.pathspec:
pathspec_attribute = aff4.Attribute(
"aff4:pathspec", rdf_paths.PathSpec,
"The pathspec used to retrieve "
"this object from the client.", "pathspec")
aff4_object.AddAttribute(
pathspec_attribute, stat_object.pathspec)
if attribute in ["aff4:content", "aff4:content"]:
# For AFF4MemoryStreams we need to call Write() instead of
# directly setting the contents..
aff4_object.Write(rdfvalue_object)
else:
aff4_object.AddAttribute(attribute, rdfvalue_object)
# Populate the KB from the client attributes.
if aff4_type == aff4_grr.VFSGRRClient:
kb = rdf_client.KnowledgeBase()
artifact.SetCoreGRRKnowledgeBaseValues(kb, aff4_object)
aff4_object.Set(aff4_object.Schema.KNOWLEDGE_BASE, kb)
# Make sure we do not actually close the object here - we only want to
# sync back its attributes, not run any finalization code.
aff4_object.Flush()
if aff4_type == aff4_grr.VFSGRRClient:
index = client_index.CreateClientIndex(token=self.token)
index.AddClient(aff4_object)
