def ReadHuntCounters(self, hunt_id, cursor=None):
"""Reads hunt counters."""
hunt_id_int = db_utils.HuntIDToInt(hunt_id)
query = ("SELECT flow_state, COUNT(*) "
"FROM flows "
"FORCE INDEX(flows_by_hunt) "
"WHERE parent_hunt_id = %s AND parent_flow_id IS NULL "
"GROUP BY flow_state")
cursor.execute(query, [hunt_id_int])
counts_by_state = dict(cursor.fetchall())
num_successful_clients = counts_by_state.get(
int(rdf_flow_objects.Flow.FlowState.FINISHED), 0)
num_failed_clients = counts_by_state.get(
int(rdf_flow_objects.Flow.FlowState.ERROR), 0)
num_crashed_clients = counts_by_state.get(
int(rdf_flow_objects.Flow.FlowState.CRASHED), 0)
num_running_clients = counts_by_state.get(
int(rdf_flow_objects.Flow.FlowState.RUNNING), 0)
num_clients = sum(counts_by_state.values())
query = """
SELECT * FROM
(
SELECT COUNT(client_id)
FROM flows
FORCE INDEX(flows_by_hunt)
WHERE parent_hunt_id = %s AND parent_flow_id IS NULL AND
num_replies_sent > 0) counters,
(
SELECT SUM(user_cpu_time_used_micros + system_cpu_time_used_micros),
SUM(network_bytes_sent),
SUM(num_replies_sent)
FROM flows
FORCE INDEX(flows_by_hunt)
WHERE parent_hunt_id = %s AND parent_flow_id IS NULL) resources
"""
cursor.execute(query, [hunt_id_int, hunt_id_int])
(
num_clients_with_results,
total_cpu_seconds,
total_network_bytes_sent,
num_results,
) = cursor.fetchone()
return db.HuntCounters(
num_clients=num_clients,
num_successful_clients=num_successful_clients,
num_failed_clients=num_failed_clients,
num_clients_with_results=num_clients_with_results,
num_crashed_clients=num_crashed_clients,
num_running_clients=num_running_clients,
num_results=int(num_results or 0),
total_cpu_seconds=db_utils.MicrosToSeconds(int(total_cpu_seconds or 0)),
total_network_bytes_sent=int(total_network_bytes_sent or 0))
def ReadHuntClientResourcesStats(self, hunt_id, cursor=None):
"""Read/calculate hunt client resources stats."""
hunt_id_int = db_utils.HuntIDToInt(hunt_id)
query = """
SELECT
COUNT(*),
SUM(user_cpu_time_used_micros),
SUM((user_cpu_time_used_micros) * (user_cpu_time_used_micros)),
SUM(system_cpu_time_used_micros),
SUM((system_cpu_time_used_micros) * (system_cpu_time_used_micros)),
SUM(network_bytes_sent),
SUM(network_bytes_sent * network_bytes_sent),
"""
scaled_bins = [
int(1000000 * b)
for b in rdf_stats.ClientResourcesStats.CPU_STATS_BINS
]
query += self._BinsToQuery(scaled_bins, "(user_cpu_time_used_micros)")
query += ","
query += self._BinsToQuery(scaled_bins,
"(system_cpu_time_used_micros)")
query += ","
query += self._BinsToQuery(
rdf_stats.ClientResourcesStats.NETWORK_STATS_BINS,
"network_bytes_sent")
query += " FROM flows "
query += "FORCE INDEX(flows_by_hunt) "
query += "WHERE parent_hunt_id = %s AND parent_flow_id IS NULL"
cursor.execute(query, [hunt_id_int])
response = cursor.fetchone()
(count, user_sum, user_sq_sum, system_sum, system_sq_sum, network_sum,
network_sq_sum) = response[:7]
stats = rdf_stats.ClientResourcesStats(
user_cpu_stats=rdf_stats.RunningStats(
num=count,
sum=db_utils.MicrosToSeconds(int(user_sum or 0)),
sum_sq=int(user_sq_sum or 0) / 1e12,
),
system_cpu_stats=rdf_stats.RunningStats(
num=count,
sum=db_utils.MicrosToSeconds(int(system_sum or 0)),
sum_sq=int(system_sq_sum or 0) / 1e12,
),
network_bytes_sent_stats=rdf_stats.RunningStats(
num=count,
sum=float(network_sum or 0),
sum_sq=float(network_sq_sum or 0),
),
)
offset = 7
stats.user_cpu_stats.histogram = rdf_stats.StatsHistogram()
for b_num, b_max_value in zip(
response[offset:],
rdf_stats.ClientResourcesStats.CPU_STATS_BINS):
stats.user_cpu_stats.histogram.bins.append(
rdf_stats.StatsHistogramBin(range_max_value=b_max_value,
num=b_num))
offset += len(rdf_stats.ClientResourcesStats.CPU_STATS_BINS)
stats.system_cpu_stats.histogram = rdf_stats.StatsHistogram()
for b_num, b_max_value in zip(
response[offset:],
rdf_stats.ClientResourcesStats.CPU_STATS_BINS):
stats.system_cpu_stats.histogram.bins.append(
rdf_stats.StatsHistogramBin(range_max_value=b_max_value,
num=b_num))
offset += len(rdf_stats.ClientResourcesStats.CPU_STATS_BINS)
stats.network_bytes_sent_stats.histogram = rdf_stats.StatsHistogram()
for b_num, b_max_value in zip(
response[offset:],
rdf_stats.ClientResourcesStats.NETWORK_STATS_BINS):
stats.network_bytes_sent_stats.histogram.bins.append(
rdf_stats.StatsHistogramBin(range_max_value=b_max_value,
num=b_num))
query = """
SELECT
client_id, flow_id, user_cpu_time_used_micros,
system_cpu_time_used_micros, network_bytes_sent
FROM flows
FORCE INDEX(flows_by_hunt)
WHERE parent_hunt_id = %s AND parent_flow_id IS NULL AND
(user_cpu_time_used_micros > 0 OR
system_cpu_time_used_micros > 0 OR
network_bytes_sent > 0)
ORDER BY (user_cpu_time_used_micros + system_cpu_time_used_micros) DESC
LIMIT 10
"""
cursor.execute(query, [hunt_id_int])
for cid, fid, ucpu, scpu, nbs in cursor.fetchall():
client_id = db_utils.IntToClientID(cid)
flow_id = db_utils.IntToFlowID(fid)
stats.worst_performers.append(
rdf_client_stats.ClientResources(
client_id=client_id,
session_id=rdfvalue.RDFURN(client_id).Add(flow_id),
cpu_usage=rdf_client_stats.CpuSeconds(
user_cpu_time=db_utils.MicrosToSeconds(ucpu),
system_cpu_time=db_utils.MicrosToSeconds(scpu),
),
network_bytes_sent=nbs))
return stats