def check_resource_utilization(run_state: RunState):
logger.info(
f'Checking resource utilization for bundle. uuid: {run_state.bundle.uuid}'
)
cpu_usage, memory_usage = docker_utils.get_container_stats_with_docker_stats(
run_state.container)
run_state = run_state._replace(cpu_usage=cpu_usage,
memory_usage=memory_usage)
run_state = run_state._replace(memory_usage=memory_usage)
kill_messages = []
run_stats = docker_utils.get_container_stats(run_state.container)
run_state = run_state._replace(max_memory=max(
run_state.max_memory, run_stats.get('memory', 0)))
run_state = run_state._replace(
disk_utilization=self.disk_utilization[
run_state.bundle.uuid]['disk_utilization'])
container_time_total = docker_utils.get_container_running_time(
run_state.container)
run_state = run_state._replace(
container_time_total=container_time_total,
container_time_user=run_stats.get(
'container_time_user', run_state.container_time_user),
container_time_system=run_stats.get(
'container_time_system', run_state.container_time_system),
)
if run_state.resources.time and container_time_total > run_state.resources.time:
kill_messages.append(
'Time limit exceeded. (Container uptime %s > time limit %s)'
% (duration_str(container_time_total),
duration_str(run_state.resources.time)))
if run_state.max_memory > run_state.resources.memory or run_state.exitcode == 137:
kill_messages.append('Memory limit %s exceeded.' %
size_str(run_state.resources.memory))
if run_state.resources.disk and run_state.disk_utilization > run_state.resources.disk:
kill_messages.append('Disk limit %sb exceeded.' %
size_str(run_state.resources.disk))
if kill_messages:
run_state = run_state._replace(
kill_message=' '.join(kill_messages), is_killed=True)
return run_state
def apply_func(func, arg):
"""
Apply post-processing function |func| to |arg|.
|func| is a string representing a list of functions (which are to be
applied to |arg| in succession). Each function is either:
- 'duration', 'date', 'size' for special formatting
- '%...' for sprintf-style formatting
- s/.../... for regular expression substitution
- [a:b] for taking substrings
"""
FUNC_DELIM = ' | '
if isinstance(arg, tuple):
# tuples are (bundle_uuid, genpath) which have not been fleshed out
return arg + (func,)
try:
if func is None:
return arg
# String encoding of a function: size s/a/b
for f in func.split(FUNC_DELIM):
if f == 'date':
arg = formatting.date_str(float(arg)) if arg is not None else None
elif f == 'duration':
arg = formatting.duration_str(float(arg)) if arg is not None else None
elif f == 'size':
arg = formatting.size_str(float(arg)) if arg is not None else None
elif f.startswith('%'):
arg = (f % float(arg)) if arg is not None else None
elif f.startswith('s/'): # regular expression: s/<old string>/<new string>
esc_slash = '_ESC_SLASH_' # Assume this doesn't occur in s
# Preserve escaped characters: \/
tokens = f.replace('\\/', esc_slash).split('/')
if len(tokens) != 3:
return '<invalid regex: %s>' % f
s = tokens[1].replace(esc_slash, '/')
t = tokens[2].replace(esc_slash, '/')
arg = re.sub(s, t, arg)
elif f.startswith('['): # substring
m = re.match('\[(.*):(.*)\]', f)
if m:
start = int(m.group(1) or 0)
end = int(m.group(2) or len(arg))
arg = arg[start:end]
else:
return '<invalid function: %s>' % f
elif f.startswith('add '):
# 'add k v' checks if arg is a dictionary and updates it with arg[k] = v
if isinstance(arg, dict):
k, v = f.split(' ')[1:]
arg[k] = v
else:
return 'arg (%s) not a dictionary' % type(arg)
elif f.startswith('key '):
# 'key k' converts arg into a dictionary where arg[k] = arg
arg = {f.split(' ')[1]: arg}
else:
return '<invalid function: %s>' % f
return arg
except:
# Applying the function failed, so just return the arg.
return arg
def check_resource_utilization(run_state):
kill_messages = []
run_stats = docker_utils.get_container_stats(run_state.container)
container_time_total = docker_utils.get_container_running_time(run_state.container)
run_state = run_state._replace(
container_time_total=container_time_total,
container_time_user=run_stats.get(
'container_time_user', run_state.container_time_user
),
container_time_system=run_stats.get(
'container_time_system', run_state.container_time_system
),
)
run_state = run_state._replace(
max_memory=max(run_state.max_memory, run_stats.get('memory', 0))
)
run_state = check_disk_utilization(run_state)
if run_state.resources.time and container_time_total > run_state.resources.time:
kill_messages.append(
'Time limit exceeded. (Container uptime %s > time limit %s)'
% (duration_str(container_time_total), duration_str(run_state.resources.time))
)
if run_state.max_memory > run_state.resources.memory or run_state.exitcode == 137:
kill_messages.append(
'Memory limit %s exceeded.' % size_str(run_state.resources.memory)
)
if run_state.resources.disk and run_state.disk_utilization > run_state.resources.disk:
kill_messages.append(
'Disk limit %sb exceeded.' % size_str(run_state.resources.disk)
)
if kill_messages:
run_state = run_state._replace(kill_message=' '.join(kill_messages), is_killed=True)
return run_state
def check_timed_out_bundles(self, update_timeout=60*60*24):
'''
Filters the list of running bundles by their time, and marks those that
have not been updated in > update_timeout seconds as FAILED.
Default update_timeout: 1 day
'''
uuids = self.model.search_bundle_uuids(worksheet_uuid=None, user_id=self.model.root_user_id,
keywords=['state='+','.join([State.RUNNING, State.QUEUED])])
bundles = self.model.batch_get_bundles(uuid=uuids)
def _failed(bundle):
now = int(time.time())
since_last_update = now - bundle.metadata.last_updated
return since_last_update >= update_timeout
failed_bundles = filter(_failed, bundles)
for bundle in failed_bundles:
failure_msg = 'No response from worker in %s' % formatting.duration_str(update_timeout)
status = {'state': State.FAILED, 'success': False, 'bundle': bundle, 'failure_message': failure_msg}
self.update_running_bundle(status)
return len(failed_bundles) > 0
def apply_func(func, arg):
'''
Apply post-processing function |func| to |arg|.
|func| is a string representing a list of functions (which are to be
applied to |arg| in succession). Each function is either:
- 'duration', 'date', 'size' for special formatting
- '%...' for sprintf-style formatting
- s/... for regular expression substitution
- [a:b] for taking substrings
'''
FUNC_DELIM = ' | '
if isinstance(arg, tuple):
# tuples are (bundle_uuid, genpath) which have not been fleshed out
return arg + (func,)
try:
if func == None: return arg
# String encoding of a function: size s/a/b
for f in func.split(FUNC_DELIM):
if f == 'date':
arg = formatting.date_str(arg)
elif f == 'duration':
arg = formatting.duration_str(float(arg)) if arg != None else ''
elif f == 'size':
arg = formatting.size_str(arg)
elif f.startswith('%'):
arg = (f % float(arg)) if arg != None else ''
elif f.startswith('s/'): # regular expression
_, s, t = f.split("/")
arg = re.sub(s, t, arg)
elif f.startswith('['): # substring
m = re.match('\[(.*):(.*)\]', f)
if m:
start = int(m.group(1) or 0)
end = int(m.group(2) or -1)
arg = arg[start:end]
else:
return '<invalid function: %s>' % f
else:
return '<invalid function: %s>' % f
return arg
except:
# Can't apply the function, so just return the arg.
return arg
def check_timed_out_bundles(self, update_timeout=60 * 60 * 24):
"""
Filters the list of running bundles by their time, and marks those that
have not been updated in > update_timeout seconds as FAILED.
Default update_timeout: 1 day
"""
uuids = self.model.search_bundle_uuids(
worksheet_uuid=None,
user_id=self.model.root_user_id,
keywords=["state=" + ",".join([State.RUNNING, State.QUEUED])],
)
bundles = self.model.batch_get_bundles(uuid=uuids)
def _failed(bundle):
now = int(time.time())
since_last_update = now - bundle.metadata.last_updated
return since_last_update >= update_timeout
failed_bundles = filter(_failed, bundles)
for bundle in failed_bundles:
failure_msg = "No response from worker in %s" % formatting.duration_str(update_timeout)
status = {"state": State.FAILED, "success": False, "bundle": bundle, "failure_message": failure_msg}
self.update_running_bundle(status)
return len(failed_bundles) > 0
def apply_func(func, arg):
"""
Apply post-processing function |func| to |arg|.
|func| is a string representing a list of functions (which are to be
applied to |arg| in succession). Each function is either:
- 'duration', 'date', 'size' for special formatting
- '%...' for sprintf-style formatting
- s/.../... for regular expression substitution
- [a:b] for taking substrings
"""
FUNC_DELIM = ' | '
if isinstance(arg, tuple):
# tuples are (bundle_uuid, genpath) which have not been fleshed out
return arg + (func, )
try:
if func is None:
return arg
# String encoding of a function: size s/a/b
for f in func.split(FUNC_DELIM):
if f == 'str':
arg = str(arg)
elif f == 'date':
arg = formatting.date_str(
float(arg)) if arg is not None else None
elif f == 'duration':
arg = formatting.duration_str(
float(arg)) if arg is not None else None
elif f == 'size':
arg = formatting.size_str(
float(arg)) if arg is not None else None
elif f.startswith('%'):
arg = (f % float(arg)) if arg is not None else None
elif f.startswith(
's/'): # regular expression: s/<old string>/<new string>
esc_slash = '_ESC_SLASH_' # Assume this doesn't occur in s
# Preserve escaped characters: \/
tokens = f.replace('\\/', esc_slash).split('/')
if len(tokens) != 3:
return '<invalid regex: %s>' % f
s = tokens[1].replace(esc_slash, '/')
t = tokens[2].replace(esc_slash, '/')
arg = re.sub(s, t, arg)
elif f.startswith('['): # substring
m = re.match('\[(.*):(.*)\]', f)
if m:
start = int(m.group(1) or 0)
end = int(m.group(2) or len(arg))
arg = arg[start:end]
else:
return '<invalid function: %s>' % f
elif f.startswith('add '):
# 'add k v' checks if arg is a dictionary and updates it with arg[k] = v
if isinstance(arg, dict):
k, v = f.split(' ')[1:]
arg[k] = v
else:
return 'arg (%s) not a dictionary' % type(arg)
elif f.startswith('key '):
# 'key k' converts arg into a dictionary where arg[k] = arg
arg = {f.split(' ')[1]: arg}
else:
return '<invalid function: %s>' % f
return arg
except:
# Applying the function failed, so just return the arg.
return arg
def _serialize(self, value, attr, obj):
return formatting.duration_str(value)