def check_resource_utilization(run_state: RunState):
logger.info(
f'Checking resource utilization for bundle. uuid: {run_state.bundle.uuid}'
)
cpu_usage, memory_usage = docker_utils.get_container_stats_with_docker_stats(
run_state.container)
run_state = run_state._replace(cpu_usage=cpu_usage,
memory_usage=memory_usage)
run_state = run_state._replace(memory_usage=memory_usage)
kill_messages = []
run_stats = docker_utils.get_container_stats(run_state.container)
run_state = run_state._replace(max_memory=max(
run_state.max_memory, run_stats.get('memory', 0)))
run_state = run_state._replace(
disk_utilization=self.disk_utilization[
run_state.bundle.uuid]['disk_utilization'])
container_time_total = docker_utils.get_container_running_time(
run_state.container)
run_state = run_state._replace(
container_time_total=container_time_total,
container_time_user=run_stats.get(
'container_time_user', run_state.container_time_user),
container_time_system=run_stats.get(
'container_time_system', run_state.container_time_system),
)
if run_state.resources.time and container_time_total > run_state.resources.time:
kill_messages.append(
'Time limit exceeded. (Container uptime %s > time limit %s)'
% (duration_str(container_time_total),
duration_str(run_state.resources.time)))
if run_state.max_memory > run_state.resources.memory or run_state.exitcode == 137:
kill_messages.append('Memory limit %s exceeded.' %
size_str(run_state.resources.memory))
if run_state.resources.disk and run_state.disk_utilization > run_state.resources.disk:
kill_messages.append('Disk limit %sb exceeded.' %
size_str(run_state.resources.disk))
if kill_messages:
run_state = run_state._replace(
kill_message=' '.join(kill_messages), is_killed=True)
return run_state
def check_resource_utilization(run_state):
kill_messages = []
run_stats = docker_utils.get_container_stats(run_state.container)
container_time_total = docker_utils.get_container_running_time(run_state.container)
run_state = run_state._replace(
container_time_total=container_time_total,
container_time_user=run_stats.get(
'container_time_user', run_state.container_time_user
),
container_time_system=run_stats.get(
'container_time_system', run_state.container_time_system
),
)
run_state = run_state._replace(
max_memory=max(run_state.max_memory, run_stats.get('memory', 0))
)
run_state = check_disk_utilization(run_state)
if run_state.resources.time and container_time_total > run_state.resources.time:
kill_messages.append(
'Time limit exceeded. (Container uptime %s > time limit %s)'
% (duration_str(container_time_total), duration_str(run_state.resources.time))
)
if run_state.max_memory > run_state.resources.memory or run_state.exitcode == 137:
kill_messages.append(
'Memory limit %s exceeded.' % size_str(run_state.resources.memory)
)
if run_state.resources.disk and run_state.disk_utilization > run_state.resources.disk:
kill_messages.append(
'Disk limit %sb exceeded.' % size_str(run_state.resources.disk)
)
if kill_messages:
run_state = run_state._replace(kill_message=' '.join(kill_messages), is_killed=True)
return run_state