def __init__(self, addr): self._pool = AdaptiveProcessPool(run_command) # Used to keep track of ResponseHelpers so that the right one can be # used when a job result is received from the pool. self._last_id = 0 self._responders = {} # Used to keep track of whether the server is receiving requests. self._taking_work = Event() self.stats = defaultdict(int) self.get_time_stats = defaultdict(list) self.put_time_stats = defaultdict(list) self._read_stats() try: base_server.CommandServer.__init__(self, addr, CommandHandler) except: # We need to stop the job pool to avoid the process living forever # waiting for jobs. self._pool.stop() raise Thread(target=self.dispatch_results).start() Thread(target=self._watchdog).start()
class CommandServer(base_server.CommandServer): ''' Main sccache server. ''' def __init__(self, addr): self._pool = AdaptiveProcessPool(run_command) # Used to keep track of ResponseHelpers so that the right one can be # used when a job result is received from the pool. self._last_id = 0 self._responders = {} # Used to keep track of whether the server is receiving requests. self._taking_work = Event() self.stats = defaultdict(int) self.get_time_stats = defaultdict(list) self.put_time_stats = defaultdict(list) self._read_stats() try: base_server.CommandServer.__init__(self, addr, CommandHandler) except: # We need to stop the job pool to avoid the process living forever # waiting for jobs. self._pool.stop() raise Thread(target=self.dispatch_results).start() Thread(target=self._watchdog).start() def dispatch_job(self, request, responder): # Mark the server as not idle. self._taking_work.set() # Sanity checks on the requested command line cmd = request.get('cmd') if not cmd: raise ArgumentError('Command is either missing or empty') if not isinstance(cmd, list): raise TypeError('Expected list, got %s' % type(cmd)) executable, args = cmd[0], cmd[1:] cwd = request.get('cwd') # Get a Compiler instance corresponding to that executable. # The cwd is necessary because sometimes the path to the executable is # relative. compiler = Compiler.from_path(executable, cwd) if not compiler: raise RuntimeError('%s is not a known compiler' % executable) # Parse the command line arguments in the main thread, this is fast # enough not to be a problem in practice, and avoids dispatching # compilations that can't be cached. parsed_args = None try: parsed_args = compiler.parse_arguments(args) except CannotCacheError: self.stats['non-cachable'] += 1 except NotACompilationError: self.stats['non-compile'] += 1 if not parsed_args: # Return status code -2 when the compiler result can't be cached # or when the compiler is invoked for something else than a # compilation. responder.respond(-2) return # Prepare job for run_command and send it to the job pool. self._last_id += 1 job = { 'id': self._last_id, 'compiler': compiler, 'args': args, 'parsed_args': parsed_args, 'cwd': cwd, } self._responders[self._last_id] = responder self._pool.add_job(job) def dispatch_results(self): # This runs in a dedicated thread. for result in self._pool.results(): if not isinstance(result, dict): continue # Redispatch the results from the pool to the corresponding client. id = result.get('id') stats = result.get('stats') if stats: for key in ('dns', 'connect', 'response', 'size'): value = stats.get(key, 0) if id: self.get_time_stats[key].append(value) else: self.put_time_stats[key].append(value) responder = self._responders.get(id) status = result.get('status') if status: self.stats[status] += 1 if not responder: continue responder.respond( result.get('retcode', -1), result.get('stdout', ''), result.get('stderr', ''), ) def _watchdog(self): # This runs in a dedicated thread. while not self.stopping: self._taking_work.clear() self._taking_work.wait(timeout=600) # If the server hasn't received a connection in the past 600 # seconds, stop it. if not self._taking_work.is_set(): self.stop(dump_stats=True) break def stop(self, dump_stats=False): if not self.stopping: base_server.CommandServer.stop(self) # If the watchdog is waiting for the taking_work event timeout, # trigger one now to unblock it and make it quit. self._taking_work.set() self._pool.stop() if dump_stats: self._dump_stats() def _stats_file_path(self): # Use the local cache storage directory if one is given. dir = os.environ.get('SCCACHE_DIR') if dir: try: ensure_dir(dir) except: dir = None if not dir: dir = os.path.dirname(FILE) return os.path.join(dir, 'sccache.stats') def _dump_stats(self): if (not self.stats and not self.get_time_stats and not self.put_time_stats): return try: stats_file = open(self._stats_file_path(), 'w') except: # Just don't dump anything if we couldn't open the stats file. return stats_file.write('%s\n%s\n%s\n' % ( json.dumps(self.stats), json.dumps(self.get_time_stats), json.dumps(self.put_time_stats), )) stats_file.close() def _read_stats(self): stats = ( self.stats, self.get_time_stats, self.put_time_stats, ) try: stats_file = self._stats_file_path() with open(stats_file) as f: for n, line in enumerate(f): stats[n % len(stats)].update(json.loads(line)) os.remove(stats_file) except: pass
class CommandServer(base_server.CommandServer): ''' Main sccache server. ''' def __init__(self, addr): self._pool = AdaptiveProcessPool(run_command) # Used to keep track of ResponseHelpers so that the right one can be # used when a job result is received from the pool. self._last_id = 0 self._responders = {} # Used to keep track of whether the server is receiving requests. self._taking_work = Event() self.stats = defaultdict(int) self.get_time_stats = defaultdict(list) self.put_time_stats = defaultdict(list) self._read_stats() try: base_server.CommandServer.__init__(self, addr, CommandHandler) except: # We need to stop the job pool to avoid the process living forever # waiting for jobs. self._pool.stop() raise Thread(target=self.dispatch_results).start() Thread(target=self._watchdog).start() def dispatch_job(self, request, responder): # Mark the server as not idle. self._taking_work.set() # Sanity checks on the requested command line cmd = request.get('cmd') if not cmd: raise ArgumentError('Command is either missing or empty') if not isinstance(cmd, list): raise TypeError('Expected list, got %s' % type(cmd)) executable, args = cmd[0], cmd[1:] cwd = request.get('cwd') # Get a Compiler instance corresponding to that executable. # The cwd is necessary because sometimes the path to the executable is # relative. compiler = Compiler.from_path(executable, cwd) if not compiler: raise RuntimeError('%s is not a known compiler' % executable) # Parse the command line arguments in the main thread, this is fast # enough not to be a problem in practice, and avoids dispatching # compilations that can't be cached. parsed_args = None try: parsed_args = compiler.parse_arguments(args) except CannotCacheError: LOG.exception('could not cache') self.stats['non-cachable'] += 1 except NotACompilationError: LOG.info('not a compilation') self.stats['non-compile'] += 1 if not parsed_args: # Return status code -2 when the compiler result can't be cached # or when the compiler is invoked for something else than a # compilation. responder.respond(-2) return # Prepare job for run_command and send it to the job pool. self._last_id += 1 job = { 'id': self._last_id, 'compiler': compiler, 'args': args, 'parsed_args': parsed_args, 'cwd': cwd, } self._responders[self._last_id] = responder self._pool.add_job(job) def dispatch_results(self): # This runs in a dedicated thread. for result in self._pool.results(): if not isinstance(result, dict): continue # Redispatch the results from the pool to the corresponding client. id = result.get('id') stats = result.get('stats') if stats: for key in ('dns', 'connect', 'response', 'size'): value = stats.get(key, 0) if id: self.get_time_stats[key].append(value) else: self.put_time_stats[key].append(value) responder = self._responders.get(id) status = result.get('status') if status: self.stats[status] += 1 if not responder: continue responder.respond( result.get('retcode', -1), result.get('stdout', ''), result.get('stderr', ''), ) def _watchdog(self): # This runs in a dedicated thread. while not self.stopping: self._taking_work.clear() self._taking_work.wait(timeout=600) # If the server hasn't received a connection in the past 600 # seconds, stop it. if not self._taking_work.is_set(): self.stop(dump_stats=True) break def stop(self, dump_stats=False): if not self.stopping: base_server.CommandServer.stop(self) # If the watchdog is waiting for the taking_work event timeout, # trigger one now to unblock it and make it quit. self._taking_work.set() self._pool.stop() if dump_stats: self._dump_stats() def _stats_file_path(self): # Use the local cache storage directory if one is given. dir = os.environ.get('SCCACHE_DIR') if dir: try: ensure_dir(dir) except: dir = None if not dir: dir = os.path.dirname(FILE) return os.path.join(dir, 'sccache.stats') def _dump_stats(self): if (not self.stats and not self.get_time_stats and not self.put_time_stats): return try: stats_file = open(self._stats_file_path(), 'w') except: # Just don't dump anything if we couldn't open the stats file. return stats_file.write('%s\n%s\n%s\n' % ( json.dumps(self.stats), json.dumps(self.get_time_stats), json.dumps(self.put_time_stats), )) stats_file.close() def _read_stats(self): stats = ( self.stats, self.get_time_stats, self.put_time_stats, ) try: stats_file = self._stats_file_path() with open(stats_file) as f: for n, line in enumerate(f): stats[n % len(stats)].update(json.loads(line)) os.remove(stats_file) except: pass