def setUp(self):
super(SystemCronTestMixin, self).setUp()
recent_ping = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("8d")
# Simulate Fleetspeak clients with last-ping timestamps in the GRR DB
# that haven't been updated in a while. Last-contact timestamps reported
# by Fleetspeak should be used instead of this value.
ancient_ping = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("999d")
self.SetupClientsWithIndices(range(0, 10),
system="Windows",
ping=recent_ping)
self.SetupClientsWithIndices(range(10, 20),
system="Linux",
ping=recent_ping)
fs_urns = self.SetupClientsWithIndices(range(20, 22),
system="Darwin",
fleetspeak_enabled=True,
ping=ancient_ping)
for i in range(0, 10):
client_id = u"C.1%015x" % i
with aff4.FACTORY.Open(client_id, mode="rw",
token=self.token) as client:
client.AddLabels([u"Label1", u"Label2"], owner=u"GRR")
client.AddLabel(u"UserLabel", owner=u"jim")
data_store.REL_DB.AddClientLabels(client_id, u"GRR",
[u"Label1", u"Label2"])
data_store.REL_DB.AddClientLabels(client_id, u"jim",
[u"UserLabel"])
fs_connector_patcher = mock.patch.object(fleetspeak_connector, "CONN")
self._fs_conn = fs_connector_patcher.start()
self.addCleanup(fs_connector_patcher.stop)
last_fs_contact = timestamp_pb2.Timestamp()
# Have Fleetspeak report that the last contact with the Fleetspeak clients
# happened an hour ago.
last_ping_rdf = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("1h")
last_fs_contact.FromMicroseconds(
last_ping_rdf.AsMicrosecondsSinceEpoch())
fs_clients = [
admin_pb2.Client(client_id=fleetspeak_utils.GRRIDToFleetspeakID(
fs_urns[0].Basename()),
last_contact_time=last_fs_contact),
admin_pb2.Client(client_id=fleetspeak_utils.GRRIDToFleetspeakID(
fs_urns[1].Basename()),
last_contact_time=last_fs_contact),
]
self._fs_conn.outgoing.ListClients.return_value = (
admin_pb2.ListClientsResponse(clients=fs_clients))
def testUpdateFromFleetspeak(self):
client_id_1 = client_plugin.ApiClientId("C." + "1" * 16)
client_id_2 = client_plugin.ApiClientId("C." + "2" * 16)
client_id_3 = client_plugin.ApiClientId("C." + "3" * 16)
clients = [
client_plugin.ApiClient(client_id=client_id_1,
fleetspeak_enabled=True),
client_plugin.ApiClient(client_id=client_id_2,
fleetspeak_enabled=True),
client_plugin.ApiClient(client_id=client_id_3,
fleetspeak_enabled=False),
]
conn = mock.MagicMock()
conn.outgoing.ListClients.return_value = admin_pb2.ListClientsResponse(
clients=[
admin_pb2.Client(
client_id=fleetspeak_utils.GRRIDToFleetspeakID(
client_id_1),
last_contact_time=timestamp_pb2.Timestamp(seconds=100000,
nanos=50000000),
last_clock=timestamp_pb2.Timestamp(seconds=100000,
nanos=60000000),
),
admin_pb2.Client(
client_id=fleetspeak_utils.GRRIDToFleetspeakID(
client_id_2),
last_contact_time=timestamp_pb2.Timestamp(seconds=200000,
nanos=50000000),
last_clock=timestamp_pb2.Timestamp(seconds=200000,
nanos=60000000),
)
])
with mock.patch.object(fleetspeak_connector, "CONN", conn):
client_plugin._UpdateFromFleetspeak(clients)
self.assertEqual(clients, [
client_plugin.ApiClient(
client_id=client_id_1,
fleetspeak_enabled=True,
last_seen_at=rdfvalue.RDFDatetime(100000050),
last_clock=rdfvalue.RDFDatetime(100000060)),
client_plugin.ApiClient(
client_id=client_id_2,
fleetspeak_enabled=True,
last_seen_at=rdfvalue.RDFDatetime(200000050),
last_clock=rdfvalue.RDFDatetime(200000060)),
client_plugin.ApiClient(client_id=client_id_3,
fleetspeak_enabled=False),
])
def testUpdateClientsFromFleetspeak(self):
client_id_1 = client_plugin.ApiClientId("C." + "1" * 16)
client_id_2 = client_plugin.ApiClientId("C." + "2" * 16)
client_id_3 = client_plugin.ApiClientId("C." + "3" * 16)
clients = [
client_plugin.ApiClient(client_id=client_id_1,
fleetspeak_enabled=True),
client_plugin.ApiClient(client_id=client_id_2,
fleetspeak_enabled=True),
client_plugin.ApiClient(client_id=client_id_3,
fleetspeak_enabled=False),
]
conn = mock.MagicMock()
conn.outgoing.ListClients.return_value = admin_pb2.ListClientsResponse(
clients=[
admin_pb2.Client(
client_id=fleetspeak_utils.GRRIDToFleetspeakID(
client_id_1),
last_contact_time=TSProtoFromString(
"2018-01-01T00:00:01Z"),
last_clock=TSProtoFromString("2018-01-01T00:00:02Z")),
admin_pb2.Client(
client_id=fleetspeak_utils.GRRIDToFleetspeakID(
client_id_2),
last_contact_time=TSProtoFromString(
"2018-01-02T00:00:01Z"),
last_clock=TSProtoFromString("2018-01-02T00:00:02Z"))
])
with mock.patch.object(fleetspeak_connector, "CONN", conn):
client_plugin.UpdateClientsFromFleetspeak(clients)
self.assertEqual(clients, [
client_plugin.ApiClient(
client_id=client_id_1,
fleetspeak_enabled=True,
last_seen_at=rdfvalue.RDFDatetime.FromHumanReadable(
"2018-01-01T00:00:01Z"),
last_clock=rdfvalue.RDFDatetime.FromHumanReadable(
"2018-01-01T00:00:02Z")),
client_plugin.ApiClient(
client_id=client_id_2,
fleetspeak_enabled=True,
last_seen_at=rdfvalue.RDFDatetime.FromHumanReadable(
"2018-01-02T00:00:01Z"),
last_clock=rdfvalue.RDFDatetime.FromHumanReadable(
"2018-01-02T00:00:02Z")),
client_plugin.ApiClient(client_id=client_id_3,
fleetspeak_enabled=False),
])
def _MockConnReturningClient(grr_id, labels):
client = admin_pb2.Client(
client_id=fleetspeak_utils.GRRIDToFleetspeakID(grr_id),
labels=[common_pb2.Label(service_name=k, label=v) for k, v in labels])
conn = mock.MagicMock()
conn.outgoing.ListClients.return_value = admin_pb2.ListClientsResponse(
clients=[client])
return conn
def testGetLabelFromFleetspeakKnown(self):
client_id = "C.0000000000000001"
conn = MockConnReturningClient(
admin_pb2.Client(
client_id=fleetspeak_utils.GRRIDToFleetspeakID(client_id),
labels=[common_pb2.Label(service_name="client",
label="division1")]))
with test_lib.ConfigOverrider({
"Server.fleetspeak_label_map": ["division1:fleetspeak-division1"],
}):
with fleetspeak_test_lib.ConnectionOverrider(conn):
self.assertEqual("fleetspeak-division1",
fleetspeak_utils.GetLabelFromFleetspeak(client_id))
def testGetAddrFromFleetspeakMissing(self):
client_id = client_plugin.ApiClientId("C." + "1" * 16)
conn = mock.MagicMock()
conn.outgoing.ListClients.return_value = admin_pb2.ListClientsResponse(
clients=[
admin_pb2.Client(
client_id=fleetspeak_utils.GRRIDToFleetspeakID(client_id),
last_contact_time=TSProtoFromString("2018-01-01T00:00:01Z"),
last_clock=TSProtoFromString("2018-01-01T00:00:02Z"))
])
with mock.patch.object(fleetspeak_connector, "CONN", conn):
ip_str, ipaddr_obj = client_plugin._GetAddrFromFleetspeak(client_id)
self.assertEqual(ip_str, "")
self.assertIsNone(ipaddr_obj)
def ListClients(self, request):
clients = []
for client_id in request.client_ids:
clients.append(
fs_admin_pb2.Client(
client_id=client_id,
labels=[
fs_common_pb2.Label(
service_name="client", label="alphabet"),
fs_common_pb2.Label(
service_name="client", label="alphabet-google-corp"),
fs_common_pb2.Label(service_name="client", label="linux"),
]))
return fs_admin_pb2.ListClientsResponse(clients=clients)
def testGetAddrFromFleetspeakIpV6(self):
client_id = client_plugin.ApiClientId("C." + "1" * 16)
conn = mock.MagicMock()
conn.outgoing.ListClients.return_value = admin_pb2.ListClientsResponse(
clients=[
admin_pb2.Client(
client_id=fleetspeak_utils.GRRIDToFleetspeakID(client_id),
last_contact_address="[2001:0db8:85a3::8a2e:0370:7334]:50000",
last_contact_time=TSProtoFromString("2018-01-01T00:00:01Z"),
last_clock=TSProtoFromString("2018-01-01T00:00:02Z"))
])
with mock.patch.object(fleetspeak_connector, "CONN", conn):
ip_str, ipaddr_obj = client_plugin._GetAddrFromFleetspeak(client_id)
self.assertEqual(ip_str, "2001:0db8:85a3::8a2e:0370:7334")
self.assertEqual(
ipaddr_obj,
ipaddress.ip_address("2001:0db8:85a3:0000:0000:8a2e:0370:7334"))
def testCronJob(self, fs_conn_mock):
if not data_store.RelationalDBReadEnabled():
self.skipTest("Test is only for the relational DB. Skipping...")
client_id1 = "C.0000000000000001"
client_id2 = "C.0000000000000002"
client_id3 = "C.0000000000000003"
client_id4 = "C.0000000000000004"
client_id5 = "C.0000000000000005"
client_id6 = "C.0000000000000006"
client_id7 = "C.0000000000000007"
data_store.REL_DB.WriteClientMetadata(client_id1,
fleetspeak_enabled=False)
data_store.REL_DB.WriteClientMetadata(client_id2,
fleetspeak_enabled=True)
data_store.REL_DB.WriteClientMetadata(
client_id3,
last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3),
fleetspeak_enabled=True)
data_store.REL_DB.WriteClientMetadata(
client_id4,
last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(41),
fleetspeak_enabled=True)
data_store.REL_DB.WriteClientMetadata(
client_id5,
last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(5),
fleetspeak_enabled=True)
data_store.REL_DB.WriteClientMetadata(
client_id6,
last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(61),
fleetspeak_enabled=True)
data_store.REL_DB.WriteClientMetadata(
client_id7,
last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(68),
fleetspeak_enabled=True)
fs_enabled_ids = [
client_id2, client_id3, client_id4, client_id5, client_id6,
client_id7
]
fs_clients = {}
for i, client_id in enumerate(fs_enabled_ids):
client_number = i + 2
fs_client_id = fleetspeak_utils.GRRIDToFleetspeakID(client_id)
fs_client = admin_pb2.Client(client_id=fs_client_id)
fs_client.last_contact_time.FromSeconds(client_number * 10)
fs_clients[fs_client_id] = fs_client
def FakeListClients(list_request):
clients = []
for fs_client_id in list_request.client_ids:
clients.append(fs_clients[fs_client_id])
return admin_pb2.ListClientsResponse(clients=clients)
fs_conn_mock.outgoing.ListClients = FakeListClients
cron_run = rdf_cronjobs.CronJobRun()
job_data = rdf_cronjobs.CronJob()
cron = system.UpdateFSLastPingTimestamps(cron_run, job_data)
with test_lib.FakeTime(
rdfvalue.RDFDatetime.FromSecondsSinceEpoch(100)):
with test_lib.ConfigOverrider({
"Server.fleetspeak_last_ping_threshold":
"35s",
"Server.fleetspeak_list_clients_batch_size":
2,
}):
cron.Run()
actual_timestamps = data_store.REL_DB.ReadClientLastPings()
expected_timestamps = {
# Skipped because not a Fleetspeak client.
client_id1: None,
client_id2: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(20),
client_id3: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(30),
# Skipped because FS timestamp is old.
client_id4: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(41),
client_id5: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(50),
# Skipped because FS timestamp is old.
client_id6: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(61),
# Skipped because existing GRR timestamp is too recent.
client_id7: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(68),
}
self.assertEqual(actual_timestamps, expected_timestamps)
self.assertMultiLineEqual(cron._log_messages.popleft(),
"Updated timestamps for 3 clients.")
