def collect_container_stats(container_id, stats, cpu_ticks): # Retrieve the CPU stats try: stats.cpus_limit = float(read_metric(container_id, "cpu.shares")) / 256 cpu_stats = dict(read_metrics(container_id, "cpuacct.stat")) if "user" in cpu_stats and "system" in cpu_stats: stats.cpus_user_time_secs = float(cpu_stats["user"]) / cpu_ticks stats.cpus_system_time_secs = float(cpu_stats["system"]) / cpu_ticks except: raise logger.error("Failed to get CPU usage") try: cpu_stats = dict(read_metrics(container_id, "cpu.stat")) if "nr_periods" in cpu_stats: stats.cpus_nr_periods = int(cpu_stats["nr_periods"]) if "nr_throttled" in cpu_stats: stats.cpus_nr_throttled = int(cpu_stats["nr_throttled"]) if "throttled_time" in cpu_stats: throttled_time_nano = int(cpu_stats["throttled_time"]) throttled_time_secs = throttled_time_nano / 1000000000 stats.cpus_throttled_time_secs = throttled_time_secs except: logger.error("Failed to get detailed CPU usage") # Retrieve the mem stats try: stats.mem_limit_bytes = int(read_metric(container_id, "memory.limit_in_bytes")) stats.mem_rss_bytes = int(read_metric(container_id, "memory.usage_in_bytes")) except: logger.error("Failed to get memory usage") try: mem_stats = dict(read_metrics(container_id, "memory.stat")) if "total_cache" in mem_stats: stats.mem_file_bytes = int(mem_stats["total_cache"]) if "total_rss" in mem_stats: stats.mem_anon_bytes = int(mem_stats["total_rss"]) if "total_mapped_file" in mem_stats: stats.mem_mapped_file_bytes = int(mem_stats["total_mapped_file"]) except: logger.error("Failed to get detailed memory usage") return stats
def update(): """ Update the resources of a running container. """ update = recv_proto(Update) with container_lock(update.container_id.value, "update"): logger.info("Updating resources for container %s", update.container_id.value) # Get the container ID info = inspect_container(update.container_id.value) lxc_container_id = info["ID"] # Gather the resoures max_mem = None max_cpus = None for resource in update.resources: if resource.name == "mem": max_mem = int(resource.scalar.value) * 1024 * 1024 if resource.name == "cpus": max_cpus = int(resource.scalar.value) * 256 if resource.name == "ports": logger.error( "Unable to process an update to port configuration!") if max_mem: # Update the soft limit write_metric(lxc_container_id, "memory.soft_limit_in_bytes", max_mem) # Figure out if we can update the hard limit # If we reduce the hard limit and too much memory is in use, this # can invoke an OOM. current_mem = int( read_metric(lxc_container_id, "memory.limit_in_bytes")) if current_mem > max_mem: write_metric(lxc_container_id, "memory.limit_in_bytes", max_mem) else: logger.info("Skipping hard memory limit, would invoke OOM") if max_cpus: shares = max_cpus * 256 write_metric(lxc_container_id, "cpu.shares", shares) logger.info("Finished processing container update")
def update(): """ Update the resources of a running container. """ update = recv_proto(Update) with container_lock(update.container_id.value, "update"): logger.info("Updating resources for container %s", update.container_id.value) # Get the container ID info = inspect_container(update.container_id.value) lxc_container_id = info.get("ID", info.get("Id")) if lxc_container_id is None: raise Exception("Failed to get full container ID") # Gather the resoures max_mem = None max_cpus = None for resource in update.resources: if resource.name == "mem": max_mem = int(resource.scalar.value) * 1024 * 1024 if resource.name == "cpus": max_cpus = int(resource.scalar.value) * 256 if resource.name == "ports": logger.error("Unable to process an update to port configuration!") if max_mem: # Update the soft limit write_metric(lxc_container_id, "memory.soft_limit_in_bytes", max_mem) # Figure out if we can update the hard limit # If we reduce the hard limit and too much memory is in use, this # can invoke an OOM. current_mem = int(read_metric(lxc_container_id, "memory.limit_in_bytes")) if current_mem > max_mem: write_metric(lxc_container_id, "memory.limit_in_bytes", max_mem) else: logger.info("Skipping hard memory limit, would invoke OOM") if max_cpus: shares = max_cpus * 256 write_metric(lxc_container_id, "cpu.shares", shares) logger.info("Finished processing container update")
def usage(): """ Retrieve usage information about a running container. """ usage = recv_proto(Usage) logger.info("Retrieving usage for container %s", usage.container_id.value) # Find the lxc container ID info = inspect_container(usage.container_id.value) lxc_container_id = info.get("ID", info.get("Id")) if lxc_container_id is None: raise Exception("Failed to get full container ID") logger.info("Using LXC container ID %s", lxc_container_id) stats = ResourceStatistics() stats.timestamp = int(time.time()) # Get the number of CPU ticks ticks = os.sysconf("SC_CLK_TCK") if not ticks > 0: logger.error("Unable to retrieve number of CPU clock ticks") exit(1) # Retrieve the CPU stats try: stats.cpus_limit = float(read_metric(lxc_container_id, "cpu.shares")) / 256 cpu_stats = dict(read_metrics(lxc_container_id, "cpuacct.stat")) if "user" in cpu_stats and "system" in cpu_stats: stats.cpus_user_time_secs = float(cpu_stats["user"]) / ticks stats.cpus_system_time_secs = float(cpu_stats["system"]) / ticks except: logger.error("Failed to get CPU usage") try: cpu_stats = dict(read_metrics(lxc_container_id, "cpu.stat")) if "nr_periods" in cpu_stats: stats.cpus_nr_periods = int(cpu_stats["nr_periods"]) if "nr_throttled" in cpu_stats: stats.cpus_nr_throttled = int(cpu_stats["nr_throttled"]) if "throttled_time" in cpu_stats: throttled_time_nano = int(cpu_stats["throttled_time"]) throttled_time_secs = throttled_time_nano / 1000000000 stats.cpus_throttled_time_secs = throttled_time_secs except: logger.error("Failed to get detailed CPU usage") # Retrieve the mem stats try: stats.mem_limit_bytes = int(read_metric(lxc_container_id, "memory.limit_in_bytes")) stats.mem_rss_bytes = int(read_metric(lxc_container_id, "memory.usage_in_bytes")) except: logger.error("Failed to get memory usage") try: mem_stats = dict(read_metrics(lxc_container_id, "memory.stat")) if "total_cache" in mem_stats: stats.mem_file_bytes = int(mem_stats["total_cache"]) if "total_rss" in mem_stats: stats.mem_anon_bytes = int(mem_stats["total_rss"]) if "total_mapped_file" in mem_stats: stats.mem_mapped_file_bytes = int(mem_stats["total_mapped_file"]) except: logger.error("Failed to get detailed memory usage") logger.debug("Container usage: %s", stats) # Send the stats back to mesos send_proto(stats)
def usage(): """ Retrieve usage information about a running container. """ usage = recv_proto(Usage) logger.info("Retrieving usage for container %s", usage.container_id.value) # Find the lxc container ID info = inspect_container(usage.container_id.value) lxc_container_id = info["ID"] logger.info("Using LXC container ID %s", lxc_container_id) stats = ResourceStatistics() stats.timestamp = int(time.time()) # Get the number of CPU ticks ticks = os.sysconf("SC_CLK_TCK") if not ticks > 0: logger.error("Unable to retrieve number of CPU clock ticks") exit(1) # Retrieve the CPU stats try: stats.cpus_limit = float(read_metric(lxc_container_id, "cpu.shares")) / 256 cpu_stats = dict(read_metrics(lxc_container_id, "cpuacct.stat")) if "user" in cpu_stats and "system" in cpu_stats: stats.cpus_user_time_secs = float(cpu_stats["user"]) / ticks stats.cpus_system_time_secs = float(cpu_stats["system"]) / ticks except: logger.error("Failed to get CPU usage") try: cpu_stats = dict(read_metrics(lxc_container_id, "cpu.stat")) if "nr_periods" in cpu_stats: stats.cpus_nr_periods = int(cpu_stats["nr_periods"]) if "nr_throttled" in cpu_stats: stats.cpus_nr_throttled = int(cpu_stats["nr_throttled"]) if "throttled_time" in cpu_stats: throttled_time_nano = int(cpu_stats["throttled_time"]) throttled_time_secs = throttled_time_nano / 1000000000 stats.cpus_throttled_time_secs = throttled_time_secs except: logger.error("Failed to get detailed CPU usage") # Retrieve the mem stats try: stats.mem_limit_bytes = int( read_metric(lxc_container_id, "memory.limit_in_bytes")) stats.mem_rss_bytes = int( read_metric(lxc_container_id, "memory.usage_in_bytes")) except: logger.error("Failed to get memory usage") try: mem_stats = dict(read_metrics(lxc_container_id, "memory.stat")) if "total_cache" in mem_stats: stats.mem_file_bytes = int(mem_stats["total_cache"]) if "total_rss" in mem_stats: stats.mem_anon_bytes = int(mem_stats["total_rss"]) if "total_mapped_file" in mem_stats: stats.mem_mapped_file_bytes = int(mem_stats["total_mapped_file"]) except: logger.error("Failed to get detailed memory usage") logger.debug("Container usage: %s", stats) # Send the stats back to mesos send_proto(stats)