def _start_process(self): self.socketpath = os.path.join(self.config_path("socket_path"), self.username + ":" + self.formatted_time + ".sock") try: # Unlink if necessary os.unlink(self.socketpath) except OSError as e: if e.errno != errno.ENOENT: raise game = self.game_params call = self._base_call() + ["-webtiles-socket", self.socketpath, "-await-connection"] ttyrec_path = self.config_path("ttyrec_path") if ttyrec_path: self.ttyrec_filename = os.path.join(ttyrec_path, self.lock_basename) processes[os.path.abspath(self.socketpath)] = self if config.dgl_mode: self.logger.info("Starting %s.", game["id"]) else: self.logger.info("Starting game.") try: self.process = TerminalRecorder(call, self.ttyrec_filename, self._ttyrec_id_header(), self.logger, config.recording_term_size, env_vars = game.get("env", {}), game_cwd = game.get("cwd", None),) self.process.end_callback = self._on_process_end self.process.output_callback = self._on_process_output self.process.activity_callback = self.note_activity self.process.error_callback = self._on_process_error self.gen_inprogress_lock() self.connect(self.socketpath, True) self.logger.debug("Crawl FDs: fd%s, fd%s.", self.process.child_fd, self.process.errpipe_read) self.last_activity_time = time.time() self.check_where() except Exception: self.logger.warning("Error while starting the Crawl process!", exc_info=True) self.exit_reason = "error" self.exit_message = "Error while starting the Crawl process!\nSomething has gone very wrong; please let a server admin know." self.exit_dump_url = None if self.process: self.stop() else: self._on_process_end()
class CrawlProcessHandler(CrawlProcessHandlerBase): def __init__(self, game_params, username, logger): super(CrawlProcessHandler, self).__init__(game_params, username, logger) self.socketpath = None self.conn = None self.ttyrec_filename = None self.inprogress_lock = None self.inprogress_lock_file = None self.exit_reason = None self.exit_message = None self.exit_dump_url = None self._stale_pid = None self._stale_lockfile = None self._purging_timer = None self._process_hup_timeout = None def start(self): self._purge_locks_and_start(True) def stop(self): super(CrawlProcessHandler, self).stop() self._stop_purging_stale_processes() self._stale_pid = None def _purge_locks_and_start(self, firsttime=False): # Purge stale locks lockfile = self._find_lock() if lockfile: try: with open(lockfile) as f: pid = f.readline() try: pid = int(pid) except ValueError: # pidfile is empty or corrupted, can happen if the server # crashed. Just clear it... self.logger.error("Invalid PID from lockfile %s, clearing", lockfile) self._purge_stale_lock() return self._stale_pid = pid self._stale_lockfile = lockfile if firsttime: hup_wait = 10 self.send_to_all("stale_processes", timeout=hup_wait, game=self.game_params["name"]) to = IOLoop.current().add_timeout(time.time() + hup_wait, self._kill_stale_process) self._process_hup_timeout = to else: self._kill_stale_process() except Exception: self.logger.error("Error while handling lockfile %s.", lockfile, exc_info=True) errmsg = ( "Error while trying to terminate a stale process.\n" + "Please contact an administrator.") self.exit_reason = "error" self.exit_message = errmsg self.exit_dump_url = None self.handle_process_end() else: # No more locks, can start self._start_process() def _stop_purging_stale_processes(self): if not self._process_hup_timeout: return IOLoop.current().remove_timeout(self._process_hup_timeout) self._stale_pid = None self._stale_lockfile = None self._purging_timer = None self._process_hup_timeout = None self.handle_process_end() def _find_lock(self): for path in os.listdir(self.config_path("inprogress_path")): if (path.startswith(self.username + ":") and path.endswith(".ttyrec")): return os.path.join(self.config_path("inprogress_path"), path) return None def _kill_stale_process(self, signal=subprocess.signal.SIGHUP): self._process_hup_timeout = None if self._stale_pid == None: return if signal == subprocess.signal.SIGHUP: self.logger.info("Purging stale lock at %s, pid %s.", self._stale_lockfile, self._stale_pid) elif signal == subprocess.signal.SIGABRT: self.logger.warning("Terminating pid %s forcefully!", self._stale_pid) try: os.kill(self._stale_pid, signal) except OSError as e: if e.errno == errno.ESRCH: # Process doesn't exist self._purge_stale_lock() else: self.logger.error("Error while killing process %s.", self._stale_pid, exc_info=True) errmsg = ( "Error while trying to terminate a stale process.\n" + "Please contact an administrator.") self.exit_reason = "error" self.exit_message = errmsg self.exit_dump_url = None self.handle_process_end() return else: if signal == subprocess.signal.SIGABRT: self._purge_stale_lock() else: if signal == subprocess.signal.SIGHUP: self._purging_timer = 10 else: self._purging_timer -= 1 if self._purging_timer > 0: IOLoop.current().add_timeout(time.time() + 1, self._check_stale_process) else: self.logger.warning( "Couldn't terminate pid %s gracefully.", self._stale_pid) self.send_to_all("force_terminate?") return self.send_to_all("hide_dialog") def _check_stale_process(self): self._kill_stale_process(0) def _do_force_terminate(self, answer): if answer: self._kill_stale_process(subprocess.signal.SIGABRT) else: self.handle_process_end() def _purge_stale_lock(self): if os.path.exists(self._stale_lockfile): os.remove(self._stale_lockfile) self._purge_locks_and_start(False) def _start_process(self): self.socketpath = os.path.join( self.config_path("socket_path"), self.username + ":" + self.formatted_time + ".sock") try: # Unlink if necessary os.unlink(self.socketpath) except OSError as e: if e.errno != errno.ENOENT: raise game = self.game_params call = self._base_call() + [ "-webtiles-socket", self.socketpath, "-await-connection" ] ttyrec_path = self.config_path("ttyrec_path") if ttyrec_path: self.ttyrec_filename = os.path.join(ttyrec_path, self.lock_basename) processes[os.path.abspath(self.socketpath)] = self if config.dgl_mode: self.logger.info("Starting %s.", game["id"]) else: self.logger.info("Starting game.") try: self.process = TerminalRecorder( call, self.ttyrec_filename, self._ttyrec_id_header(), self.logger, config.recording_term_size, env_vars=game.get("env", {}), game_cwd=game.get("cwd", None), ) self.process.end_callback = self._on_process_end self.process.output_callback = self._on_process_output self.process.activity_callback = self.note_activity self.process.error_callback = self._on_process_error self.gen_inprogress_lock() self.connect(self.socketpath, True) self.logger.debug("Crawl FDs: fd%s, fd%s.", self.process.child_fd, self.process.errpipe_read) self.last_activity_time = time.time() self.check_where() except Exception: self.logger.warning("Error while starting the Crawl process!", exc_info=True) self.exit_reason = "error" self.exit_message = "Error while starting the Crawl process!\nSomething has gone very wrong; please let a server admin know." self.exit_dump_url = None if self.process: self.stop() else: self._on_process_end() def connect(self, socketpath, primary=False): self.socketpath = socketpath self.conn = WebtilesSocketConnection(self.socketpath, self.logger) self.conn.message_callback = self._on_socket_message self.conn.close_callback = self._on_socket_close self.conn.connect(primary) def gen_inprogress_lock(self): self.inprogress_lock = os.path.join( self.config_path("inprogress_path"), self.username + ":" + self.lock_basename) f = open(self.inprogress_lock, "w") fcntl.lockf(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) self.inprogress_lock_file = f cols, lines = self.process.get_terminal_size() f.write("%s\n%s\n%s\n" % (self.process.pid, lines, cols)) f.flush() def remove_inprogress_lock(self): if self.inprogress_lock_file is None: return fcntl.lockf(self.inprogress_lock_file.fileno(), fcntl.LOCK_UN) self.inprogress_lock_file.close() try: os.remove(self.inprogress_lock) except OSError: # Lock already got deleted pass def _ttyrec_id_header(self): # type: () -> bytes clrscr = b"\033[2J" crlf = b"\r\n" tstamp = int(time.time()) ctime = time.ctime() return (clrscr + b"\033[1;1H" + crlf + utf8("Player: %s" % self.username) + crlf + utf8("Game: %s" % self.game_params["name"]) + crlf + utf8("Server: %s" % config.server_id) + crlf + utf8("Filename: %s" % self.lock_basename) + crlf + utf8("Time: (%s) %s" % (tstamp, ctime)) + crlf + clrscr) def _on_process_end(self): if self.process: self.logger.debug("Crawl PID %s terminated.", self.process.pid) else: self.logger.error("Crawl process failed to start, cleaning up.") self.remove_inprogress_lock() try: del processes[os.path.abspath(self.socketpath)] except KeyError: self.logger.warning("Process entry already deleted: %s", self.socketpath) self.process = None self.handle_process_end() def _on_socket_close(self): self.conn = None self.stop() def handle_process_end(self): if self.conn: self.conn.close_callback = None self.conn.close() self.conn = None super(CrawlProcessHandler, self).handle_process_end() def add_watcher(self, watcher): super(CrawlProcessHandler, self).add_watcher(watcher) if self.conn and self.conn.open: self.conn.send_message('{"msg":"spectator_joined"}') def handle_input(self, msg): # type: (str) -> None obj = json_decode(msg) if obj["msg"] == "input" and self.process: self.last_action_time = time.time() data = "" for x in obj.get("data", []): data += chr(x) data += obj.get("text", "") self.process.write_input(utf8(data)) elif obj["msg"] == "force_terminate": self._do_force_terminate(obj["answer"]) elif obj["msg"] == "stop_stale_process_purge": self._stop_purging_stale_processes() elif self.conn and self.conn.open: self.conn.send_message(utf8(msg)) def handle_chat_message(self, username, text): # type: (str, str) -> None super(CrawlProcessHandler, self).handle_chat_message(username, text) if self.conn and self.conn.open: self.conn.send_message( json_encode({ "msg": "note", "content": "%s: %s" % (username, text) })) def handle_announcement(self, text): if self.conn and self.conn.open: self.conn.send_message( json_encode({ "msg": "server_announcement", "content": text })) def _on_process_output(self, line): # type: (str) -> None self.check_where() try: json_decode(line) except ValueError: self.logger.warning("Invalid JSON output from Crawl process: %s", line) # send messages from wrapper scripts only to the player for receiver in self._receivers: if not receiver.watched_game: receiver.append_message(line, True) def _on_process_error(self, line): # type: (str) -> None if line.startswith("ERROR"): self.exit_reason = "crash" if line.rfind(":") != -1: self.exit_message = line[line.rfind(":") + 1:].strip() elif line.startswith("We crashed!"): self.exit_reason = "crash" if self.game_params["morgue_url"] != None: match = re.search(r"\(([^)]+)\)", line) if match is not None: self.exit_dump_url = self.game_params[ "morgue_url"].replace("%n", self.username) self.exit_dump_url += os.path.splitext( os.path.basename(match.group(1)))[0] elif line.startswith( "Writing crash info to"): # before 0.15-b1-84-gded71f8 self.exit_reason = "crash" if self.game_params["morgue_url"] != None: url = None if line.rfind("/") != -1: url = line[line.rfind("/") + 1:].strip() elif line.rfind(" ") != -1: url = line[line.rfind(" ") + 1:].strip() if url is not None: self.exit_dump_url = self.game_params["morgue_url"].replace( "%n", self.username) + os.path.splitext(url)[0] def _on_socket_message(self, msg): # type: (str) -> None # stdout data is only used for compatibility to wrapper # scripts -- so as soon as we receive something on the socket, # we stop using stdout if self.process: self.process.output_callback = None if msg.startswith("*"): # Special message to the server msg = msg[1:] msgobj = json_decode(msg) if msgobj["msg"] == "client_path": if self.client_path == None: self.client_path = self.format_path(msgobj["path"]) if "version" in msgobj: self.crawl_version = msgobj["version"] self.logger.info("Crawl version: %s.", self.crawl_version) self.send_client_to_all() elif msgobj["msg"] == "flush_messages": # only queue, once we know the crawl process asks for flushes self.queue_messages = True self.flush_messages_to_all() elif msgobj["msg"] == "dump": if "morgue_url" in self.game_params and self.game_params[ "morgue_url"]: url = self.game_params["morgue_url"].replace( "%n", self.username) + msgobj["filename"] if msgobj["type"] == "command": self.send_to_all("dump", url=url) else: self.exit_dump_url = url elif msgobj["msg"] == "exit_reason": self.exit_reason = msgobj["type"] if "message" in msgobj: self.exit_message = msgobj["message"] else: self.exit_message = None elif msgobj["msg"] == "milestone": # milestone/whereis update: milestone fields are right in the # message self.receiving_direct_milestones = True # no need for .where files self.set_where_info(msgobj) else: self.logger.warning( "Unknown message from the crawl process: %s", msgobj["msg"]) else: self.check_where() if time.time() > self.last_watcher_join + 2: # Treat socket messages as activity, since it's otherwise # hard to determine activity for games found via # watch_socket_dirs. # But don't if a spectator just joined, since we don't # want that to reset idle time. self.note_activity() self.write_to_all(msg, not self.queue_messages)