def _CountClientStatisticByLabel(self, day_buckets, extract_statistic_fn):
"""Returns client-activity metrics for a particular statistic.
Args:
day_buckets: A set of n-day-active buckets.
extract_statistic_fn: A function that extracts the statistic's value from
a ClientFullInfo object.
"""
fleet_stats_builder = fleet_utils.FleetStatsBuilder(day_buckets)
now = rdfvalue.RDFDatetime.Now()
for info in self.IterateAllClientsFullInfo(batch_size=db.MAX_COUNT):
if not info.metadata.ping:
continue
statistic_value = extract_statistic_fn(info)
for day_bucket in day_buckets:
time_boundary = now - rdfvalue.Duration.From(
day_bucket, rdfvalue.DAYS)
if info.metadata.ping > time_boundary:
# Count the client if it has been active in the last 'day_bucket'
# days.
fleet_stats_builder.IncrementTotal(statistic_value,
day_bucket)
for client_label in info.GetLabelsNames(owner="GRR"):
fleet_stats_builder.IncrementLabel(
client_label, statistic_value, day_bucket)
return fleet_stats_builder.Build()
def testBuildWithInvalidBucket(self):
builder = fleet_utils.FleetStatsBuilder({1, 5, 10})
expected_exception = "Invalid bucket '3'. Allowed buckets are [1, 5, 10]."
with self.assertRaisesWithLiteralMatch(ValueError, expected_exception):
builder.IncrementLabel("foo-label", "category-foo", 3)
with self.assertRaisesWithLiteralMatch(ValueError, expected_exception):
builder.IncrementTotal("category-foo", 3)
def testBuildWithInvalidData(self):
builder = fleet_utils.FleetStatsBuilder({1, 5, 10})
builder.IncrementLabel("foo-label", "category-foo", 1, delta=6)
expected_exception = ("Day-bucket counts for label foo-label are invalid: "
"[(1, 6), (5, 0), (10, 0)].")
with self.assertRaisesWithLiteralMatch(ValueError, expected_exception):
builder.Build()
builder.IncrementLabel("foo-label", "category-foo", 5, delta=6)
builder.IncrementLabel("foo-label", "category-foo", 10, delta=7)
builder.IncrementTotal("category-foo", 5)
expected_exception = (
"Day-bucket counts for fleet-wide totals are invalid: "
"[(1, 0), (5, 1), (10, 0)].")
with self.assertRaisesWithLiteralMatch(ValueError, expected_exception):
builder.Build()
def _BuildTestStats():
builder = fleet_utils.FleetStatsBuilder({1, 5, 10})
builder.IncrementLabel("foo-label", "category-foo", 1)
builder.IncrementLabel("foo-label", "category-foo", 1, delta=4)
builder.IncrementLabel("foo-label", "category-foo", 5, delta=5)
builder.IncrementLabel("foo-label", "category-foo", 10, delta=5)
builder.IncrementLabel("bar-label", "category-foo", 5)
builder.IncrementLabel("bar-label", "category-foo", 10)
builder.IncrementLabel("bar-label", "category-bar", 5)
builder.IncrementLabel("bar-label", "category-bar", 10)
builder.IncrementTotal("category-foo", 1)
builder.IncrementTotal("category-foo", 1, delta=6)
builder.IncrementTotal("category-foo", 5, delta=7)
builder.IncrementTotal("category-foo", 10, delta=7)
builder.IncrementTotal("category-bar", 5, delta=3)
builder.IncrementTotal("category-bar", 10, delta=3)
return builder.Build()
def _CountClientStatisticByLabel(self, statistic, day_buckets, cursor):
"""Returns client-activity metrics for a given statistic.
Args:
statistic: The name of the statistic, which should also be a column in the
'clients' table.
day_buckets: A set of n-day-active buckets.
cursor: MySQL cursor for executing queries.
"""
day_buckets = sorted(day_buckets)
sum_clauses = []
ping_cast_clauses = []
timestamp_buckets = []
now = rdfvalue.RDFDatetime.Now()
for day_bucket in day_buckets:
column_name = "days_active_{}".format(day_bucket)
sum_clauses.append(
"CAST(SUM({0}) AS UNSIGNED) AS {0}".format(column_name))
ping_cast_clauses.append(
"CAST(c.last_ping > FROM_UNIXTIME(%s) AS UNSIGNED) AS {}".
format(column_name))
timestamp_bucket = now - rdfvalue.DurationSeconds.FromDays(
day_bucket)
timestamp_buckets.append(
mysql_utils.RDFDatetimeToTimestamp(timestamp_bucket))
# Count all clients with a label owned by 'GRR', aggregating by label.
query = """
SELECT j.{statistic}, j.label, {sum_clauses}
FROM (
SELECT c.{statistic} AS {statistic}, l.label AS label, {ping_cast_clauses}
FROM clients c
LEFT JOIN client_labels l USING(client_id)
WHERE c.last_ping IS NOT NULL AND l.owner_username = '******'
) AS j
GROUP BY j.{statistic}, j.label
""".format(statistic=statistic,
sum_clauses=", ".join(sum_clauses),
ping_cast_clauses=", ".join(ping_cast_clauses))
cursor.execute(query, timestamp_buckets)
fleet_stats_builder = fleet_utils.FleetStatsBuilder(day_buckets)
for response_row in cursor.fetchall():
statistic_value, client_label = response_row[:2]
for i, num_actives in enumerate(response_row[2:]):
if num_actives <= 0:
continue
fleet_stats_builder.IncrementLabel(client_label,
statistic_value,
day_buckets[i],
delta=num_actives)
# Get n-day-active totals for the statistic across all clients (including
# those that do not have a 'GRR' label).
query = """
SELECT j.{statistic}, {sum_clauses}
FROM (
SELECT c.{statistic} AS {statistic}, {ping_cast_clauses}
FROM clients c
WHERE c.last_ping IS NOT NULL
) AS j
GROUP BY j.{statistic}
""".format(statistic=statistic,
sum_clauses=", ".join(sum_clauses),
ping_cast_clauses=", ".join(ping_cast_clauses))
cursor.execute(query, timestamp_buckets)
for response_row in cursor.fetchall():
statistic_value = response_row[0]
for i, num_actives in enumerate(response_row[1:]):
if num_actives <= 0:
continue
fleet_stats_builder.IncrementTotal(statistic_value,
day_buckets[i],
delta=num_actives)
return fleet_stats_builder.Build()
