def GenerateSample(self, number=0):
value = rdf_stats.RunningStats()
value.RegisterValue(number)
value.RegisterValue(number * 2)
value.histogram = rdf_stats.StatsHistogram.FromBins(
[2.0, number, 10.0])
return value
def testStdDevIsCalculatedCorrectly(self):
stats = rdf_stats.RunningStats()
values = range(100)
for v in values:
stats.RegisterValue(v)
# Compare calculated standard deviation with a precalculated value.
self.assertTrue(math.fabs(stats.std - 28.86607004) < 1e-7)
def testMeanIsCalculatedCorrectly(self):
stats = rdf_stats.RunningStats()
values = range(100)
for v in values:
stats.RegisterValue(v)
# Compare calculated mean with a precalculated value.
self.assertTrue(math.fabs(stats.mean - 49.5) < 1e-7)
def testStdDevIsCalculatedCorrectly(self):
stats = rdf_stats.RunningStats()
values = range(100)
for v in values:
stats.RegisterValue(v)
# Compare calculated standard deviation with a precalculated value.
self.assertAlmostEqual(stats.stddev, 28.86607004)
def testMeanIsCalculatedCorrectly(self):
stats = rdf_stats.RunningStats()
values = range(100)
for v in values:
stats.RegisterValue(v)
# Compare calculated mean with a precalculated value.
self.assertAlmostEqual(stats.mean, 49.5)
def testHistogramIsCalculatedCorrectly(self):
stats = rdf_stats.RunningStats()
stats.histogram = rdf_stats.StatsHistogram.FromBins([2.0, 4.0, 10.0])
stats.RegisterValue(1.0)
stats.RegisterValue(1.0)
stats.RegisterValue(2.0)
stats.RegisterValue(2.1)
stats.RegisterValue(2.2)
stats.RegisterValue(8.0)
stats.RegisterValue(9.0)
stats.RegisterValue(10.0)
stats.RegisterValue(11.0)
self.assertAlmostEquals(stats.histogram.bins[0].range_max_value, 2.0)
self.assertEqual(stats.histogram.bins[0].num, 2)
self.assertAlmostEquals(stats.histogram.bins[1].range_max_value, 4.0)
self.assertEqual(stats.histogram.bins[1].num, 3)
self.assertAlmostEquals(stats.histogram.bins[2].range_max_value, 10.0)
self.assertEqual(stats.histogram.bins[2].num, 4)
def ReadHuntClientResourcesStats(self, hunt_id, cursor=None):
"""Read/calculate hunt client resources stats."""
hunt_id_int = db_utils.HuntIDToInt(hunt_id)
query = """
SELECT
COUNT(*),
SUM(user_cpu_time_used_micros),
SUM((user_cpu_time_used_micros) * (user_cpu_time_used_micros)),
SUM(system_cpu_time_used_micros),
SUM((system_cpu_time_used_micros) * (system_cpu_time_used_micros)),
SUM(network_bytes_sent),
SUM(network_bytes_sent * network_bytes_sent),
"""
scaled_bins = [
int(1000000 * b)
for b in rdf_stats.ClientResourcesStats.CPU_STATS_BINS
]
query += self._BinsToQuery(scaled_bins, "(user_cpu_time_used_micros)")
query += ","
query += self._BinsToQuery(scaled_bins,
"(system_cpu_time_used_micros)")
query += ","
query += self._BinsToQuery(
rdf_stats.ClientResourcesStats.NETWORK_STATS_BINS,
"network_bytes_sent")
query += " FROM flows "
query += "FORCE INDEX(flows_by_hunt) "
query += "WHERE parent_hunt_id = %s AND parent_flow_id IS NULL"
cursor.execute(query, [hunt_id_int])
response = cursor.fetchone()
(count, user_sum, user_sq_sum, system_sum, system_sq_sum, network_sum,
network_sq_sum) = response[:7]
stats = rdf_stats.ClientResourcesStats(
user_cpu_stats=rdf_stats.RunningStats(
num=count,
sum=db_utils.MicrosToSeconds(int(user_sum or 0)),
sum_sq=int(user_sq_sum or 0) / 1e12,
),
system_cpu_stats=rdf_stats.RunningStats(
num=count,
sum=db_utils.MicrosToSeconds(int(system_sum or 0)),
sum_sq=int(system_sq_sum or 0) / 1e12,
),
network_bytes_sent_stats=rdf_stats.RunningStats(
num=count,
sum=float(network_sum or 0),
sum_sq=float(network_sq_sum or 0),
),
)
offset = 7
stats.user_cpu_stats.histogram = rdf_stats.StatsHistogram()
for b_num, b_max_value in zip(
response[offset:],
rdf_stats.ClientResourcesStats.CPU_STATS_BINS):
stats.user_cpu_stats.histogram.bins.append(
rdf_stats.StatsHistogramBin(range_max_value=b_max_value,
num=b_num))
offset += len(rdf_stats.ClientResourcesStats.CPU_STATS_BINS)
stats.system_cpu_stats.histogram = rdf_stats.StatsHistogram()
for b_num, b_max_value in zip(
response[offset:],
rdf_stats.ClientResourcesStats.CPU_STATS_BINS):
stats.system_cpu_stats.histogram.bins.append(
rdf_stats.StatsHistogramBin(range_max_value=b_max_value,
num=b_num))
offset += len(rdf_stats.ClientResourcesStats.CPU_STATS_BINS)
stats.network_bytes_sent_stats.histogram = rdf_stats.StatsHistogram()
for b_num, b_max_value in zip(
response[offset:],
rdf_stats.ClientResourcesStats.NETWORK_STATS_BINS):
stats.network_bytes_sent_stats.histogram.bins.append(
rdf_stats.StatsHistogramBin(range_max_value=b_max_value,
num=b_num))
query = """
SELECT
client_id, flow_id, user_cpu_time_used_micros,
system_cpu_time_used_micros, network_bytes_sent
FROM flows
FORCE INDEX(flows_by_hunt)
WHERE parent_hunt_id = %s AND parent_flow_id IS NULL AND
(user_cpu_time_used_micros > 0 OR
system_cpu_time_used_micros > 0 OR
network_bytes_sent > 0)
ORDER BY (user_cpu_time_used_micros + system_cpu_time_used_micros) DESC
LIMIT 10
"""
cursor.execute(query, [hunt_id_int])
for cid, fid, ucpu, scpu, nbs in cursor.fetchall():
client_id = db_utils.IntToClientID(cid)
flow_id = db_utils.IntToFlowID(fid)
stats.worst_performers.append(
rdf_client_stats.ClientResources(
client_id=client_id,
session_id=rdfvalue.RDFURN(client_id).Add(flow_id),
cpu_usage=rdf_client_stats.CpuSeconds(
user_cpu_time=db_utils.MicrosToSeconds(ucpu),
system_cpu_time=db_utils.MicrosToSeconds(scpu),
),
network_bytes_sent=nbs))
return stats
