def collect_container_stats(container_id, stats, cpu_ticks):

    # Retrieve the CPU stats
    try:
        stats.cpus_limit = float(read_metric(container_id, "cpu.shares")) / 256
        cpu_stats = dict(read_metrics(container_id, "cpuacct.stat"))
        if "user" in cpu_stats and "system" in cpu_stats:
            stats.cpus_user_time_secs = float(cpu_stats["user"]) / cpu_ticks
            stats.cpus_system_time_secs = float(cpu_stats["system"]) / cpu_ticks
    except:
        raise
        logger.error("Failed to get CPU usage")

    try:
        cpu_stats = dict(read_metrics(container_id, "cpu.stat"))
        if "nr_periods" in cpu_stats:
            stats.cpus_nr_periods = int(cpu_stats["nr_periods"])
        if "nr_throttled" in cpu_stats:
            stats.cpus_nr_throttled = int(cpu_stats["nr_throttled"])
        if "throttled_time" in cpu_stats:
            throttled_time_nano = int(cpu_stats["throttled_time"])
            throttled_time_secs = throttled_time_nano / 1000000000
            stats.cpus_throttled_time_secs = throttled_time_secs
    except:
        logger.error("Failed to get detailed CPU usage")

    # Retrieve the mem stats
    try:
        stats.mem_limit_bytes = int(read_metric(container_id, "memory.limit_in_bytes"))
        stats.mem_rss_bytes = int(read_metric(container_id, "memory.usage_in_bytes"))
    except:
        logger.error("Failed to get memory usage")

    try:
        mem_stats = dict(read_metrics(container_id, "memory.stat"))
        if "total_cache" in mem_stats:
            stats.mem_file_bytes = int(mem_stats["total_cache"])
        if "total_rss" in mem_stats:
            stats.mem_anon_bytes = int(mem_stats["total_rss"])
        if "total_mapped_file" in mem_stats:
            stats.mem_mapped_file_bytes = int(mem_stats["total_mapped_file"])
    except:
        logger.error("Failed to get detailed memory usage")

    return stats
Example #2
0
def update():
    """
    Update the resources of a running container.
    """

    update = recv_proto(Update)

    with container_lock(update.container_id.value, "update"):

        logger.info("Updating resources for container %s",
                    update.container_id.value)

        # Get the container ID
        info = inspect_container(update.container_id.value)
        lxc_container_id = info["ID"]

        # Gather the resoures
        max_mem = None
        max_cpus = None

        for resource in update.resources:
            if resource.name == "mem":
                max_mem = int(resource.scalar.value) * 1024 * 1024
            if resource.name == "cpus":
                max_cpus = int(resource.scalar.value) * 256
            if resource.name == "ports":
                logger.error(
                    "Unable to process an update to port configuration!")

        if max_mem:
            # Update the soft limit
            write_metric(lxc_container_id, "memory.soft_limit_in_bytes",
                         max_mem)

            # Figure out if we can update the hard limit
            # If we reduce the hard limit and too much memory is in use, this
            # can invoke an OOM.
            current_mem = int(
                read_metric(lxc_container_id, "memory.limit_in_bytes"))
            if current_mem > max_mem:
                write_metric(lxc_container_id, "memory.limit_in_bytes",
                             max_mem)
            else:
                logger.info("Skipping hard memory limit, would invoke OOM")

        if max_cpus:
            shares = max_cpus * 256
            write_metric(lxc_container_id, "cpu.shares", shares)

        logger.info("Finished processing container update")
def update():
    """
    Update the resources of a running container.
    """

    update = recv_proto(Update)

    with container_lock(update.container_id.value, "update"):

        logger.info("Updating resources for container %s", update.container_id.value)

        # Get the container ID
        info = inspect_container(update.container_id.value)
        lxc_container_id = info.get("ID", info.get("Id"))

        if lxc_container_id is None:
            raise Exception("Failed to get full container ID")

        # Gather the resoures
        max_mem = None
        max_cpus = None

        for resource in update.resources:
            if resource.name == "mem":
                max_mem = int(resource.scalar.value) * 1024 * 1024
            if resource.name == "cpus":
                max_cpus = int(resource.scalar.value) * 256
            if resource.name == "ports":
                logger.error("Unable to process an update to port configuration!")

        if max_mem:
            # Update the soft limit
            write_metric(lxc_container_id, "memory.soft_limit_in_bytes", max_mem)

            # Figure out if we can update the hard limit
            # If we reduce the hard limit and too much memory is in use, this
            # can invoke an OOM.
            current_mem = int(read_metric(lxc_container_id, "memory.limit_in_bytes"))
            if current_mem > max_mem:
                write_metric(lxc_container_id, "memory.limit_in_bytes", max_mem)
            else:
                logger.info("Skipping hard memory limit, would invoke OOM")

        if max_cpus:
            shares = max_cpus * 256
            write_metric(lxc_container_id, "cpu.shares", shares)

        logger.info("Finished processing container update")
def usage():
    """
    Retrieve usage information about a running container.
    """

    usage = recv_proto(Usage)
    logger.info("Retrieving usage for container %s", usage.container_id.value)

    # Find the lxc container ID
    info = inspect_container(usage.container_id.value)
    lxc_container_id = info.get("ID", info.get("Id"))

    if lxc_container_id is None:
        raise Exception("Failed to get full container ID")

    logger.info("Using LXC container ID %s", lxc_container_id)

    stats = ResourceStatistics()
    stats.timestamp = int(time.time())

    # Get the number of CPU ticks
    ticks = os.sysconf("SC_CLK_TCK")
    if not ticks > 0:
        logger.error("Unable to retrieve number of CPU clock ticks")
        exit(1)

    # Retrieve the CPU stats
    try:
        stats.cpus_limit = float(read_metric(lxc_container_id, "cpu.shares")) / 256
        cpu_stats = dict(read_metrics(lxc_container_id, "cpuacct.stat"))
        if "user" in cpu_stats and "system" in cpu_stats:
            stats.cpus_user_time_secs = float(cpu_stats["user"]) / ticks
            stats.cpus_system_time_secs = float(cpu_stats["system"]) / ticks
    except:
        logger.error("Failed to get CPU usage")

    try:
        cpu_stats = dict(read_metrics(lxc_container_id, "cpu.stat"))
        if "nr_periods" in cpu_stats:
            stats.cpus_nr_periods = int(cpu_stats["nr_periods"])
        if "nr_throttled" in cpu_stats:
            stats.cpus_nr_throttled = int(cpu_stats["nr_throttled"])
        if "throttled_time" in cpu_stats:
            throttled_time_nano = int(cpu_stats["throttled_time"])
            throttled_time_secs = throttled_time_nano / 1000000000
            stats.cpus_throttled_time_secs = throttled_time_secs
    except:
        logger.error("Failed to get detailed CPU usage")

    # Retrieve the mem stats
    try:
        stats.mem_limit_bytes = int(read_metric(lxc_container_id, "memory.limit_in_bytes"))
        stats.mem_rss_bytes = int(read_metric(lxc_container_id, "memory.usage_in_bytes"))
    except:
        logger.error("Failed to get memory usage")

    try:
        mem_stats = dict(read_metrics(lxc_container_id, "memory.stat"))
        if "total_cache" in mem_stats:
            stats.mem_file_bytes = int(mem_stats["total_cache"])
        if "total_rss" in mem_stats:
            stats.mem_anon_bytes = int(mem_stats["total_rss"])
        if "total_mapped_file" in mem_stats:
            stats.mem_mapped_file_bytes = int(mem_stats["total_mapped_file"])
    except:
        logger.error("Failed to get detailed memory usage")

    logger.debug("Container usage: %s", stats)

    # Send the stats back to mesos
    send_proto(stats)
def usage():
    """
    Retrieve usage information about a running container.
    """

    usage = recv_proto(Usage)
    logger.info("Retrieving usage for container %s", usage.container_id.value)

    # Find the lxc container ID
    info = inspect_container(usage.container_id.value)
    lxc_container_id = info["ID"]

    logger.info("Using LXC container ID %s", lxc_container_id)

    stats = ResourceStatistics()
    stats.timestamp = int(time.time())

    # Get the number of CPU ticks
    ticks = os.sysconf("SC_CLK_TCK")
    if not ticks > 0:
        logger.error("Unable to retrieve number of CPU clock ticks")
        exit(1)

    # Retrieve the CPU stats
    try:
        stats.cpus_limit = float(read_metric(lxc_container_id,
                                             "cpu.shares")) / 256
        cpu_stats = dict(read_metrics(lxc_container_id, "cpuacct.stat"))
        if "user" in cpu_stats and "system" in cpu_stats:
            stats.cpus_user_time_secs = float(cpu_stats["user"]) / ticks
            stats.cpus_system_time_secs = float(cpu_stats["system"]) / ticks
    except:
        logger.error("Failed to get CPU usage")

    try:
        cpu_stats = dict(read_metrics(lxc_container_id, "cpu.stat"))
        if "nr_periods" in cpu_stats:
            stats.cpus_nr_periods = int(cpu_stats["nr_periods"])
        if "nr_throttled" in cpu_stats:
            stats.cpus_nr_throttled = int(cpu_stats["nr_throttled"])
        if "throttled_time" in cpu_stats:
            throttled_time_nano = int(cpu_stats["throttled_time"])
            throttled_time_secs = throttled_time_nano / 1000000000
            stats.cpus_throttled_time_secs = throttled_time_secs
    except:
        logger.error("Failed to get detailed CPU usage")

    # Retrieve the mem stats
    try:
        stats.mem_limit_bytes = int(
            read_metric(lxc_container_id, "memory.limit_in_bytes"))
        stats.mem_rss_bytes = int(
            read_metric(lxc_container_id, "memory.usage_in_bytes"))
    except:
        logger.error("Failed to get memory usage")

    try:
        mem_stats = dict(read_metrics(lxc_container_id, "memory.stat"))
        if "total_cache" in mem_stats:
            stats.mem_file_bytes = int(mem_stats["total_cache"])
        if "total_rss" in mem_stats:
            stats.mem_anon_bytes = int(mem_stats["total_rss"])
        if "total_mapped_file" in mem_stats:
            stats.mem_mapped_file_bytes = int(mem_stats["total_mapped_file"])
    except:
        logger.error("Failed to get detailed memory usage")

    logger.debug("Container usage: %s", stats)

    # Send the stats back to mesos
    send_proto(stats)