def check_resource_utilization(run_state: RunState): logger.info( f'Checking resource utilization for bundle. uuid: {run_state.bundle.uuid}' ) cpu_usage, memory_usage = docker_utils.get_container_stats_with_docker_stats( run_state.container) run_state = run_state._replace(cpu_usage=cpu_usage, memory_usage=memory_usage) run_state = run_state._replace(memory_usage=memory_usage) kill_messages = [] run_stats = docker_utils.get_container_stats(run_state.container) run_state = run_state._replace(max_memory=max( run_state.max_memory, run_stats.get('memory', 0))) run_state = run_state._replace( disk_utilization=self.disk_utilization[ run_state.bundle.uuid]['disk_utilization']) container_time_total = docker_utils.get_container_running_time( run_state.container) run_state = run_state._replace( container_time_total=container_time_total, container_time_user=run_stats.get( 'container_time_user', run_state.container_time_user), container_time_system=run_stats.get( 'container_time_system', run_state.container_time_system), ) if run_state.resources.time and container_time_total > run_state.resources.time: kill_messages.append( 'Time limit exceeded. (Container uptime %s > time limit %s)' % (duration_str(container_time_total), duration_str(run_state.resources.time))) if run_state.max_memory > run_state.resources.memory or run_state.exitcode == 137: kill_messages.append('Memory limit %s exceeded.' % size_str(run_state.resources.memory)) if run_state.resources.disk and run_state.disk_utilization > run_state.resources.disk: kill_messages.append('Disk limit %sb exceeded.' % size_str(run_state.resources.disk)) if kill_messages: run_state = run_state._replace( kill_message=' '.join(kill_messages), is_killed=True) return run_state
def apply_func(func, arg): """ Apply post-processing function |func| to |arg|. |func| is a string representing a list of functions (which are to be applied to |arg| in succession). Each function is either: - 'duration', 'date', 'size' for special formatting - '%...' for sprintf-style formatting - s/.../... for regular expression substitution - [a:b] for taking substrings """ FUNC_DELIM = ' | ' if isinstance(arg, tuple): # tuples are (bundle_uuid, genpath) which have not been fleshed out return arg + (func,) try: if func is None: return arg # String encoding of a function: size s/a/b for f in func.split(FUNC_DELIM): if f == 'date': arg = formatting.date_str(float(arg)) if arg is not None else None elif f == 'duration': arg = formatting.duration_str(float(arg)) if arg is not None else None elif f == 'size': arg = formatting.size_str(float(arg)) if arg is not None else None elif f.startswith('%'): arg = (f % float(arg)) if arg is not None else None elif f.startswith('s/'): # regular expression: s/<old string>/<new string> esc_slash = '_ESC_SLASH_' # Assume this doesn't occur in s # Preserve escaped characters: \/ tokens = f.replace('\\/', esc_slash).split('/') if len(tokens) != 3: return '<invalid regex: %s>' % f s = tokens[1].replace(esc_slash, '/') t = tokens[2].replace(esc_slash, '/') arg = re.sub(s, t, arg) elif f.startswith('['): # substring m = re.match('\[(.*):(.*)\]', f) if m: start = int(m.group(1) or 0) end = int(m.group(2) or len(arg)) arg = arg[start:end] else: return '<invalid function: %s>' % f elif f.startswith('add '): # 'add k v' checks if arg is a dictionary and updates it with arg[k] = v if isinstance(arg, dict): k, v = f.split(' ')[1:] arg[k] = v else: return 'arg (%s) not a dictionary' % type(arg) elif f.startswith('key '): # 'key k' converts arg into a dictionary where arg[k] = arg arg = {f.split(' ')[1]: arg} else: return '<invalid function: %s>' % f return arg except: # Applying the function failed, so just return the arg. return arg
def check_resource_utilization(run_state): kill_messages = [] run_stats = docker_utils.get_container_stats(run_state.container) container_time_total = docker_utils.get_container_running_time(run_state.container) run_state = run_state._replace( container_time_total=container_time_total, container_time_user=run_stats.get( 'container_time_user', run_state.container_time_user ), container_time_system=run_stats.get( 'container_time_system', run_state.container_time_system ), ) run_state = run_state._replace( max_memory=max(run_state.max_memory, run_stats.get('memory', 0)) ) run_state = check_disk_utilization(run_state) if run_state.resources.time and container_time_total > run_state.resources.time: kill_messages.append( 'Time limit exceeded. (Container uptime %s > time limit %s)' % (duration_str(container_time_total), duration_str(run_state.resources.time)) ) if run_state.max_memory > run_state.resources.memory or run_state.exitcode == 137: kill_messages.append( 'Memory limit %s exceeded.' % size_str(run_state.resources.memory) ) if run_state.resources.disk and run_state.disk_utilization > run_state.resources.disk: kill_messages.append( 'Disk limit %sb exceeded.' % size_str(run_state.resources.disk) ) if kill_messages: run_state = run_state._replace(kill_message=' '.join(kill_messages), is_killed=True) return run_state
def check_timed_out_bundles(self, update_timeout=60*60*24): ''' Filters the list of running bundles by their time, and marks those that have not been updated in > update_timeout seconds as FAILED. Default update_timeout: 1 day ''' uuids = self.model.search_bundle_uuids(worksheet_uuid=None, user_id=self.model.root_user_id, keywords=['state='+','.join([State.RUNNING, State.QUEUED])]) bundles = self.model.batch_get_bundles(uuid=uuids) def _failed(bundle): now = int(time.time()) since_last_update = now - bundle.metadata.last_updated return since_last_update >= update_timeout failed_bundles = filter(_failed, bundles) for bundle in failed_bundles: failure_msg = 'No response from worker in %s' % formatting.duration_str(update_timeout) status = {'state': State.FAILED, 'success': False, 'bundle': bundle, 'failure_message': failure_msg} self.update_running_bundle(status) return len(failed_bundles) > 0
def apply_func(func, arg): ''' Apply post-processing function |func| to |arg|. |func| is a string representing a list of functions (which are to be applied to |arg| in succession). Each function is either: - 'duration', 'date', 'size' for special formatting - '%...' for sprintf-style formatting - s/... for regular expression substitution - [a:b] for taking substrings ''' FUNC_DELIM = ' | ' if isinstance(arg, tuple): # tuples are (bundle_uuid, genpath) which have not been fleshed out return arg + (func,) try: if func == None: return arg # String encoding of a function: size s/a/b for f in func.split(FUNC_DELIM): if f == 'date': arg = formatting.date_str(arg) elif f == 'duration': arg = formatting.duration_str(float(arg)) if arg != None else '' elif f == 'size': arg = formatting.size_str(arg) elif f.startswith('%'): arg = (f % float(arg)) if arg != None else '' elif f.startswith('s/'): # regular expression _, s, t = f.split("/") arg = re.sub(s, t, arg) elif f.startswith('['): # substring m = re.match('\[(.*):(.*)\]', f) if m: start = int(m.group(1) or 0) end = int(m.group(2) or -1) arg = arg[start:end] else: return '<invalid function: %s>' % f else: return '<invalid function: %s>' % f return arg except: # Can't apply the function, so just return the arg. return arg
def check_timed_out_bundles(self, update_timeout=60 * 60 * 24): """ Filters the list of running bundles by their time, and marks those that have not been updated in > update_timeout seconds as FAILED. Default update_timeout: 1 day """ uuids = self.model.search_bundle_uuids( worksheet_uuid=None, user_id=self.model.root_user_id, keywords=["state=" + ",".join([State.RUNNING, State.QUEUED])], ) bundles = self.model.batch_get_bundles(uuid=uuids) def _failed(bundle): now = int(time.time()) since_last_update = now - bundle.metadata.last_updated return since_last_update >= update_timeout failed_bundles = filter(_failed, bundles) for bundle in failed_bundles: failure_msg = "No response from worker in %s" % formatting.duration_str(update_timeout) status = {"state": State.FAILED, "success": False, "bundle": bundle, "failure_message": failure_msg} self.update_running_bundle(status) return len(failed_bundles) > 0
def apply_func(func, arg): """ Apply post-processing function |func| to |arg|. |func| is a string representing a list of functions (which are to be applied to |arg| in succession). Each function is either: - 'duration', 'date', 'size' for special formatting - '%...' for sprintf-style formatting - s/.../... for regular expression substitution - [a:b] for taking substrings """ FUNC_DELIM = ' | ' if isinstance(arg, tuple): # tuples are (bundle_uuid, genpath) which have not been fleshed out return arg + (func, ) try: if func is None: return arg # String encoding of a function: size s/a/b for f in func.split(FUNC_DELIM): if f == 'str': arg = str(arg) elif f == 'date': arg = formatting.date_str( float(arg)) if arg is not None else None elif f == 'duration': arg = formatting.duration_str( float(arg)) if arg is not None else None elif f == 'size': arg = formatting.size_str( float(arg)) if arg is not None else None elif f.startswith('%'): arg = (f % float(arg)) if arg is not None else None elif f.startswith( 's/'): # regular expression: s/<old string>/<new string> esc_slash = '_ESC_SLASH_' # Assume this doesn't occur in s # Preserve escaped characters: \/ tokens = f.replace('\\/', esc_slash).split('/') if len(tokens) != 3: return '<invalid regex: %s>' % f s = tokens[1].replace(esc_slash, '/') t = tokens[2].replace(esc_slash, '/') arg = re.sub(s, t, arg) elif f.startswith('['): # substring m = re.match('\[(.*):(.*)\]', f) if m: start = int(m.group(1) or 0) end = int(m.group(2) or len(arg)) arg = arg[start:end] else: return '<invalid function: %s>' % f elif f.startswith('add '): # 'add k v' checks if arg is a dictionary and updates it with arg[k] = v if isinstance(arg, dict): k, v = f.split(' ')[1:] arg[k] = v else: return 'arg (%s) not a dictionary' % type(arg) elif f.startswith('key '): # 'key k' converts arg into a dictionary where arg[k] = arg arg = {f.split(' ')[1]: arg} else: return '<invalid function: %s>' % f return arg except: # Applying the function failed, so just return the arg. return arg
def _serialize(self, value, attr, obj): return formatting.duration_str(value)