示例#1
0
  def ReadHuntCounters(self, hunt_id, cursor=None):
    """Reads hunt counters."""
    hunt_id_int = db_utils.HuntIDToInt(hunt_id)

    query = ("SELECT flow_state, COUNT(*) "
             "FROM flows "
             "FORCE INDEX(flows_by_hunt) "
             "WHERE parent_hunt_id = %s AND parent_flow_id IS NULL "
             "GROUP BY flow_state")

    cursor.execute(query, [hunt_id_int])
    counts_by_state = dict(cursor.fetchall())

    num_successful_clients = counts_by_state.get(
        int(rdf_flow_objects.Flow.FlowState.FINISHED), 0)
    num_failed_clients = counts_by_state.get(
        int(rdf_flow_objects.Flow.FlowState.ERROR), 0)
    num_crashed_clients = counts_by_state.get(
        int(rdf_flow_objects.Flow.FlowState.CRASHED), 0)
    num_running_clients = counts_by_state.get(
        int(rdf_flow_objects.Flow.FlowState.RUNNING), 0)
    num_clients = sum(counts_by_state.values())

    query = """
    SELECT * FROM

      (
      SELECT COUNT(client_id)
      FROM flows
      FORCE INDEX(flows_by_hunt)
      WHERE parent_hunt_id = %s AND parent_flow_id IS NULL AND
            num_replies_sent > 0) counters,

      (
      SELECT SUM(user_cpu_time_used_micros + system_cpu_time_used_micros),
             SUM(network_bytes_sent),
             SUM(num_replies_sent)
      FROM flows
      FORCE INDEX(flows_by_hunt)
      WHERE parent_hunt_id = %s AND parent_flow_id IS NULL) resources
    """

    cursor.execute(query, [hunt_id_int, hunt_id_int])
    (
        num_clients_with_results,
        total_cpu_seconds,
        total_network_bytes_sent,
        num_results,
    ) = cursor.fetchone()

    return db.HuntCounters(
        num_clients=num_clients,
        num_successful_clients=num_successful_clients,
        num_failed_clients=num_failed_clients,
        num_clients_with_results=num_clients_with_results,
        num_crashed_clients=num_crashed_clients,
        num_running_clients=num_running_clients,
        num_results=int(num_results or 0),
        total_cpu_seconds=db_utils.MicrosToSeconds(int(total_cpu_seconds or 0)),
        total_network_bytes_sent=int(total_network_bytes_sent or 0))
示例#2
0
    def ReadHuntClientResourcesStats(self, hunt_id, cursor=None):
        """Read/calculate hunt client resources stats."""
        hunt_id_int = db_utils.HuntIDToInt(hunt_id)

        query = """
      SELECT
        COUNT(*),
        SUM(user_cpu_time_used_micros),
        SUM((user_cpu_time_used_micros) * (user_cpu_time_used_micros)),
        SUM(system_cpu_time_used_micros),
        SUM((system_cpu_time_used_micros) * (system_cpu_time_used_micros)),
        SUM(network_bytes_sent),
        SUM(network_bytes_sent * network_bytes_sent),
    """

        scaled_bins = [
            int(1000000 * b)
            for b in rdf_stats.ClientResourcesStats.CPU_STATS_BINS
        ]

        query += self._BinsToQuery(scaled_bins, "(user_cpu_time_used_micros)")
        query += ","
        query += self._BinsToQuery(scaled_bins,
                                   "(system_cpu_time_used_micros)")
        query += ","
        query += self._BinsToQuery(
            rdf_stats.ClientResourcesStats.NETWORK_STATS_BINS,
            "network_bytes_sent")

        query += " FROM flows "
        query += "FORCE INDEX(flows_by_hunt) "
        query += "WHERE parent_hunt_id = %s AND parent_flow_id IS NULL"

        cursor.execute(query, [hunt_id_int])

        response = cursor.fetchone()
        (count, user_sum, user_sq_sum, system_sum, system_sq_sum, network_sum,
         network_sq_sum) = response[:7]

        stats = rdf_stats.ClientResourcesStats(
            user_cpu_stats=rdf_stats.RunningStats(
                num=count,
                sum=db_utils.MicrosToSeconds(int(user_sum or 0)),
                sum_sq=int(user_sq_sum or 0) / 1e12,
            ),
            system_cpu_stats=rdf_stats.RunningStats(
                num=count,
                sum=db_utils.MicrosToSeconds(int(system_sum or 0)),
                sum_sq=int(system_sq_sum or 0) / 1e12,
            ),
            network_bytes_sent_stats=rdf_stats.RunningStats(
                num=count,
                sum=float(network_sum or 0),
                sum_sq=float(network_sq_sum or 0),
            ),
        )

        offset = 7
        stats.user_cpu_stats.histogram = rdf_stats.StatsHistogram()
        for b_num, b_max_value in zip(
                response[offset:],
                rdf_stats.ClientResourcesStats.CPU_STATS_BINS):
            stats.user_cpu_stats.histogram.bins.append(
                rdf_stats.StatsHistogramBin(range_max_value=b_max_value,
                                            num=b_num))

        offset += len(rdf_stats.ClientResourcesStats.CPU_STATS_BINS)
        stats.system_cpu_stats.histogram = rdf_stats.StatsHistogram()
        for b_num, b_max_value in zip(
                response[offset:],
                rdf_stats.ClientResourcesStats.CPU_STATS_BINS):
            stats.system_cpu_stats.histogram.bins.append(
                rdf_stats.StatsHistogramBin(range_max_value=b_max_value,
                                            num=b_num))

        offset += len(rdf_stats.ClientResourcesStats.CPU_STATS_BINS)
        stats.network_bytes_sent_stats.histogram = rdf_stats.StatsHistogram()
        for b_num, b_max_value in zip(
                response[offset:],
                rdf_stats.ClientResourcesStats.NETWORK_STATS_BINS):
            stats.network_bytes_sent_stats.histogram.bins.append(
                rdf_stats.StatsHistogramBin(range_max_value=b_max_value,
                                            num=b_num))

        query = """
      SELECT
        client_id, flow_id, user_cpu_time_used_micros,
        system_cpu_time_used_micros, network_bytes_sent
      FROM flows
      FORCE INDEX(flows_by_hunt)
      WHERE parent_hunt_id = %s AND parent_flow_id IS NULL AND
            (user_cpu_time_used_micros > 0 OR
             system_cpu_time_used_micros > 0 OR
             network_bytes_sent > 0)
      ORDER BY (user_cpu_time_used_micros + system_cpu_time_used_micros) DESC
      LIMIT 10
    """

        cursor.execute(query, [hunt_id_int])

        for cid, fid, ucpu, scpu, nbs in cursor.fetchall():
            client_id = db_utils.IntToClientID(cid)
            flow_id = db_utils.IntToFlowID(fid)
            stats.worst_performers.append(
                rdf_client_stats.ClientResources(
                    client_id=client_id,
                    session_id=rdfvalue.RDFURN(client_id).Add(flow_id),
                    cpu_usage=rdf_client_stats.CpuSeconds(
                        user_cpu_time=db_utils.MicrosToSeconds(ucpu),
                        system_cpu_time=db_utils.MicrosToSeconds(scpu),
                    ),
                    network_bytes_sent=nbs))

        return stats