def _CountClientStatisticByLabel(self, day_buckets, extract_statistic_fn): """Returns client-activity metrics for a particular statistic. Args: day_buckets: A set of n-day-active buckets. extract_statistic_fn: A function that extracts the statistic's value from a ClientFullInfo object. """ fleet_stats_builder = fleet_utils.FleetStatsBuilder(day_buckets) now = rdfvalue.RDFDatetime.Now() for info in self.IterateAllClientsFullInfo(batch_size=db.MAX_COUNT): if not info.metadata.ping: continue statistic_value = extract_statistic_fn(info) for day_bucket in day_buckets: time_boundary = now - rdfvalue.Duration.From( day_bucket, rdfvalue.DAYS) if info.metadata.ping > time_boundary: # Count the client if it has been active in the last 'day_bucket' # days. fleet_stats_builder.IncrementTotal(statistic_value, day_bucket) for client_label in info.GetLabelsNames(owner="GRR"): fleet_stats_builder.IncrementLabel( client_label, statistic_value, day_bucket) return fleet_stats_builder.Build()
def testBuildWithInvalidBucket(self): builder = fleet_utils.FleetStatsBuilder({1, 5, 10}) expected_exception = "Invalid bucket '3'. Allowed buckets are [1, 5, 10]." with self.assertRaisesWithLiteralMatch(ValueError, expected_exception): builder.IncrementLabel("foo-label", "category-foo", 3) with self.assertRaisesWithLiteralMatch(ValueError, expected_exception): builder.IncrementTotal("category-foo", 3)
def testBuildWithInvalidData(self): builder = fleet_utils.FleetStatsBuilder({1, 5, 10}) builder.IncrementLabel("foo-label", "category-foo", 1, delta=6) expected_exception = ("Day-bucket counts for label foo-label are invalid: " "[(1, 6), (5, 0), (10, 0)].") with self.assertRaisesWithLiteralMatch(ValueError, expected_exception): builder.Build() builder.IncrementLabel("foo-label", "category-foo", 5, delta=6) builder.IncrementLabel("foo-label", "category-foo", 10, delta=7) builder.IncrementTotal("category-foo", 5) expected_exception = ( "Day-bucket counts for fleet-wide totals are invalid: " "[(1, 0), (5, 1), (10, 0)].") with self.assertRaisesWithLiteralMatch(ValueError, expected_exception): builder.Build()
def _BuildTestStats(): builder = fleet_utils.FleetStatsBuilder({1, 5, 10}) builder.IncrementLabel("foo-label", "category-foo", 1) builder.IncrementLabel("foo-label", "category-foo", 1, delta=4) builder.IncrementLabel("foo-label", "category-foo", 5, delta=5) builder.IncrementLabel("foo-label", "category-foo", 10, delta=5) builder.IncrementLabel("bar-label", "category-foo", 5) builder.IncrementLabel("bar-label", "category-foo", 10) builder.IncrementLabel("bar-label", "category-bar", 5) builder.IncrementLabel("bar-label", "category-bar", 10) builder.IncrementTotal("category-foo", 1) builder.IncrementTotal("category-foo", 1, delta=6) builder.IncrementTotal("category-foo", 5, delta=7) builder.IncrementTotal("category-foo", 10, delta=7) builder.IncrementTotal("category-bar", 5, delta=3) builder.IncrementTotal("category-bar", 10, delta=3) return builder.Build()
def _CountClientStatisticByLabel(self, statistic, day_buckets, cursor): """Returns client-activity metrics for a given statistic. Args: statistic: The name of the statistic, which should also be a column in the 'clients' table. day_buckets: A set of n-day-active buckets. cursor: MySQL cursor for executing queries. """ day_buckets = sorted(day_buckets) sum_clauses = [] ping_cast_clauses = [] timestamp_buckets = [] now = rdfvalue.RDFDatetime.Now() for day_bucket in day_buckets: column_name = "days_active_{}".format(day_bucket) sum_clauses.append( "CAST(SUM({0}) AS UNSIGNED) AS {0}".format(column_name)) ping_cast_clauses.append( "CAST(c.last_ping > FROM_UNIXTIME(%s) AS UNSIGNED) AS {}". format(column_name)) timestamp_bucket = now - rdfvalue.DurationSeconds.FromDays( day_bucket) timestamp_buckets.append( mysql_utils.RDFDatetimeToTimestamp(timestamp_bucket)) # Count all clients with a label owned by 'GRR', aggregating by label. query = """ SELECT j.{statistic}, j.label, {sum_clauses} FROM ( SELECT c.{statistic} AS {statistic}, l.label AS label, {ping_cast_clauses} FROM clients c LEFT JOIN client_labels l USING(client_id) WHERE c.last_ping IS NOT NULL AND l.owner_username = '******' ) AS j GROUP BY j.{statistic}, j.label """.format(statistic=statistic, sum_clauses=", ".join(sum_clauses), ping_cast_clauses=", ".join(ping_cast_clauses)) cursor.execute(query, timestamp_buckets) fleet_stats_builder = fleet_utils.FleetStatsBuilder(day_buckets) for response_row in cursor.fetchall(): statistic_value, client_label = response_row[:2] for i, num_actives in enumerate(response_row[2:]): if num_actives <= 0: continue fleet_stats_builder.IncrementLabel(client_label, statistic_value, day_buckets[i], delta=num_actives) # Get n-day-active totals for the statistic across all clients (including # those that do not have a 'GRR' label). query = """ SELECT j.{statistic}, {sum_clauses} FROM ( SELECT c.{statistic} AS {statistic}, {ping_cast_clauses} FROM clients c WHERE c.last_ping IS NOT NULL ) AS j GROUP BY j.{statistic} """.format(statistic=statistic, sum_clauses=", ".join(sum_clauses), ping_cast_clauses=", ".join(ping_cast_clauses)) cursor.execute(query, timestamp_buckets) for response_row in cursor.fetchall(): statistic_value = response_row[0] for i, num_actives in enumerate(response_row[1:]): if num_actives <= 0: continue fleet_stats_builder.IncrementTotal(statistic_value, day_buckets[i], delta=num_actives) return fleet_stats_builder.Build()