def connect(self, socketpath, primary=False):
self.socketpath = socketpath
self.conn = WebtilesSocketConnection(self.socketpath, self.logger)
self.conn.message_callback = self._on_socket_message
self.conn.close_callback = self._on_socket_close
self.conn.connect(primary)
class CrawlProcessHandler(CrawlProcessHandlerBase):
def __init__(self, game_params, username, logger):
super(CrawlProcessHandler, self).__init__(game_params, username,
logger)
self.socketpath = None
self.conn = None
self.ttyrec_filename = None
self.inprogress_lock = None
self.inprogress_lock_file = None
self.exit_reason = None
self.exit_message = None
self.exit_dump_url = None
self._stale_pid = None
self._stale_lockfile = None
self._purging_timer = None
self._process_hup_timeout = None
def start(self):
self._purge_locks_and_start(True)
def stop(self):
super(CrawlProcessHandler, self).stop()
self._stop_purging_stale_processes()
self._stale_pid = None
def _purge_locks_and_start(self, firsttime=False):
# Purge stale locks
lockfile = self._find_lock()
if lockfile:
try:
with open(lockfile) as f:
pid = f.readline()
try:
pid = int(pid)
except ValueError:
# pidfile is empty or corrupted, can happen if the server
# crashed. Just clear it...
self.logger.error("Invalid PID from lockfile %s, clearing",
lockfile)
self._purge_stale_lock()
return
self._stale_pid = pid
self._stale_lockfile = lockfile
if firsttime:
hup_wait = 10
self.send_to_all("stale_processes",
timeout=hup_wait,
game=self.game_params["name"])
to = IOLoop.current().add_timeout(time.time() + hup_wait,
self._kill_stale_process)
self._process_hup_timeout = to
else:
self._kill_stale_process()
except Exception:
self.logger.error("Error while handling lockfile %s.",
lockfile,
exc_info=True)
errmsg = (
"Error while trying to terminate a stale process.\n" +
"Please contact an administrator.")
self.exit_reason = "error"
self.exit_message = errmsg
self.exit_dump_url = None
self.handle_process_end()
else:
# No more locks, can start
self._start_process()
def _stop_purging_stale_processes(self):
if not self._process_hup_timeout: return
IOLoop.current().remove_timeout(self._process_hup_timeout)
self._stale_pid = None
self._stale_lockfile = None
self._purging_timer = None
self._process_hup_timeout = None
self.handle_process_end()
def _find_lock(self):
for path in os.listdir(self.config_path("inprogress_path")):
if (path.startswith(self.username + ":")
and path.endswith(".ttyrec")):
return os.path.join(self.config_path("inprogress_path"), path)
return None
def _kill_stale_process(self, signal=subprocess.signal.SIGHUP):
self._process_hup_timeout = None
if self._stale_pid == None: return
if signal == subprocess.signal.SIGHUP:
self.logger.info("Purging stale lock at %s, pid %s.",
self._stale_lockfile, self._stale_pid)
elif signal == subprocess.signal.SIGABRT:
self.logger.warning("Terminating pid %s forcefully!",
self._stale_pid)
try:
os.kill(self._stale_pid, signal)
except OSError as e:
if e.errno == errno.ESRCH:
# Process doesn't exist
self._purge_stale_lock()
else:
self.logger.error("Error while killing process %s.",
self._stale_pid,
exc_info=True)
errmsg = (
"Error while trying to terminate a stale process.\n" +
"Please contact an administrator.")
self.exit_reason = "error"
self.exit_message = errmsg
self.exit_dump_url = None
self.handle_process_end()
return
else:
if signal == subprocess.signal.SIGABRT:
self._purge_stale_lock()
else:
if signal == subprocess.signal.SIGHUP:
self._purging_timer = 10
else:
self._purging_timer -= 1
if self._purging_timer > 0:
IOLoop.current().add_timeout(time.time() + 1,
self._check_stale_process)
else:
self.logger.warning(
"Couldn't terminate pid %s gracefully.",
self._stale_pid)
self.send_to_all("force_terminate?")
return
self.send_to_all("hide_dialog")
def _check_stale_process(self):
self._kill_stale_process(0)
def _do_force_terminate(self, answer):
if answer:
self._kill_stale_process(subprocess.signal.SIGABRT)
else:
self.handle_process_end()
def _purge_stale_lock(self):
if os.path.exists(self._stale_lockfile):
os.remove(self._stale_lockfile)
self._purge_locks_and_start(False)
def _start_process(self):
self.socketpath = os.path.join(
self.config_path("socket_path"),
self.username + ":" + self.formatted_time + ".sock")
try: # Unlink if necessary
os.unlink(self.socketpath)
except OSError as e:
if e.errno != errno.ENOENT:
raise
game = self.game_params
call = self._base_call() + [
"-webtiles-socket", self.socketpath, "-await-connection"
]
ttyrec_path = self.config_path("ttyrec_path")
if ttyrec_path:
self.ttyrec_filename = os.path.join(ttyrec_path,
self.lock_basename)
processes[os.path.abspath(self.socketpath)] = self
if config.dgl_mode:
self.logger.info("Starting %s.", game["id"])
else:
self.logger.info("Starting game.")
try:
self.process = TerminalRecorder(
call,
self.ttyrec_filename,
self._ttyrec_id_header(),
self.logger,
config.recording_term_size,
env_vars=game.get("env", {}),
game_cwd=game.get("cwd", None),
)
self.process.end_callback = self._on_process_end
self.process.output_callback = self._on_process_output
self.process.activity_callback = self.note_activity
self.process.error_callback = self._on_process_error
self.gen_inprogress_lock()
self.connect(self.socketpath, True)
self.logger.debug("Crawl FDs: fd%s, fd%s.", self.process.child_fd,
self.process.errpipe_read)
self.last_activity_time = time.time()
self.check_where()
except Exception:
self.logger.warning("Error while starting the Crawl process!",
exc_info=True)
self.exit_reason = "error"
self.exit_message = "Error while starting the Crawl process!\nSomething has gone very wrong; please let a server admin know."
self.exit_dump_url = None
if self.process:
self.stop()
else:
self._on_process_end()
def connect(self, socketpath, primary=False):
self.socketpath = socketpath
self.conn = WebtilesSocketConnection(self.socketpath, self.logger)
self.conn.message_callback = self._on_socket_message
self.conn.close_callback = self._on_socket_close
self.conn.connect(primary)
def gen_inprogress_lock(self):
self.inprogress_lock = os.path.join(
self.config_path("inprogress_path"),
self.username + ":" + self.lock_basename)
f = open(self.inprogress_lock, "w")
fcntl.lockf(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
self.inprogress_lock_file = f
cols, lines = self.process.get_terminal_size()
f.write("%s\n%s\n%s\n" % (self.process.pid, lines, cols))
f.flush()
def remove_inprogress_lock(self):
if self.inprogress_lock_file is None: return
fcntl.lockf(self.inprogress_lock_file.fileno(), fcntl.LOCK_UN)
self.inprogress_lock_file.close()
try:
os.remove(self.inprogress_lock)
except OSError:
# Lock already got deleted
pass
def _ttyrec_id_header(self): # type: () -> bytes
clrscr = b"\033[2J"
crlf = b"\r\n"
tstamp = int(time.time())
ctime = time.ctime()
return (clrscr + b"\033[1;1H" + crlf +
utf8("Player: %s" % self.username) + crlf +
utf8("Game: %s" % self.game_params["name"]) + crlf +
utf8("Server: %s" % config.server_id) + crlf +
utf8("Filename: %s" % self.lock_basename) + crlf +
utf8("Time: (%s) %s" % (tstamp, ctime)) + crlf + clrscr)
def _on_process_end(self):
if self.process:
self.logger.debug("Crawl PID %s terminated.", self.process.pid)
else:
self.logger.error("Crawl process failed to start, cleaning up.")
self.remove_inprogress_lock()
try:
del processes[os.path.abspath(self.socketpath)]
except KeyError:
self.logger.warning("Process entry already deleted: %s",
self.socketpath)
self.process = None
self.handle_process_end()
def _on_socket_close(self):
self.conn = None
self.stop()
def handle_process_end(self):
if self.conn:
self.conn.close_callback = None
self.conn.close()
self.conn = None
super(CrawlProcessHandler, self).handle_process_end()
def add_watcher(self, watcher):
super(CrawlProcessHandler, self).add_watcher(watcher)
if self.conn and self.conn.open:
self.conn.send_message('{"msg":"spectator_joined"}')
def handle_input(self, msg): # type: (str) -> None
obj = json_decode(msg)
if obj["msg"] == "input" and self.process:
self.last_action_time = time.time()
data = ""
for x in obj.get("data", []):
data += chr(x)
data += obj.get("text", "")
self.process.write_input(utf8(data))
elif obj["msg"] == "force_terminate":
self._do_force_terminate(obj["answer"])
elif obj["msg"] == "stop_stale_process_purge":
self._stop_purging_stale_processes()
elif self.conn and self.conn.open:
self.conn.send_message(utf8(msg))
def handle_chat_message(self, username, text): # type: (str, str) -> None
super(CrawlProcessHandler, self).handle_chat_message(username, text)
if self.conn and self.conn.open:
self.conn.send_message(
json_encode({
"msg": "note",
"content": "%s: %s" % (username, text)
}))
def handle_announcement(self, text):
if self.conn and self.conn.open:
self.conn.send_message(
json_encode({
"msg": "server_announcement",
"content": text
}))
def _on_process_output(self, line): # type: (str) -> None
self.check_where()
try:
json_decode(line)
except ValueError:
self.logger.warning("Invalid JSON output from Crawl process: %s",
line)
# send messages from wrapper scripts only to the player
for receiver in self._receivers:
if not receiver.watched_game:
receiver.append_message(line, True)
def _on_process_error(self, line): # type: (str) -> None
if line.startswith("ERROR"):
self.exit_reason = "crash"
if line.rfind(":") != -1:
self.exit_message = line[line.rfind(":") + 1:].strip()
elif line.startswith("We crashed!"):
self.exit_reason = "crash"
if self.game_params["morgue_url"] != None:
match = re.search(r"\(([^)]+)\)", line)
if match is not None:
self.exit_dump_url = self.game_params[
"morgue_url"].replace("%n", self.username)
self.exit_dump_url += os.path.splitext(
os.path.basename(match.group(1)))[0]
elif line.startswith(
"Writing crash info to"): # before 0.15-b1-84-gded71f8
self.exit_reason = "crash"
if self.game_params["morgue_url"] != None:
url = None
if line.rfind("/") != -1:
url = line[line.rfind("/") + 1:].strip()
elif line.rfind(" ") != -1:
url = line[line.rfind(" ") + 1:].strip()
if url is not None:
self.exit_dump_url = self.game_params["morgue_url"].replace(
"%n", self.username) + os.path.splitext(url)[0]
def _on_socket_message(self, msg): # type: (str) -> None
# stdout data is only used for compatibility to wrapper
# scripts -- so as soon as we receive something on the socket,
# we stop using stdout
if self.process:
self.process.output_callback = None
if msg.startswith("*"):
# Special message to the server
msg = msg[1:]
msgobj = json_decode(msg)
if msgobj["msg"] == "client_path":
if self.client_path == None:
self.client_path = self.format_path(msgobj["path"])
if "version" in msgobj:
self.crawl_version = msgobj["version"]
self.logger.info("Crawl version: %s.",
self.crawl_version)
self.send_client_to_all()
elif msgobj["msg"] == "flush_messages":
# only queue, once we know the crawl process asks for flushes
self.queue_messages = True
self.flush_messages_to_all()
elif msgobj["msg"] == "dump":
if "morgue_url" in self.game_params and self.game_params[
"morgue_url"]:
url = self.game_params["morgue_url"].replace(
"%n", self.username) + msgobj["filename"]
if msgobj["type"] == "command":
self.send_to_all("dump", url=url)
else:
self.exit_dump_url = url
elif msgobj["msg"] == "exit_reason":
self.exit_reason = msgobj["type"]
if "message" in msgobj:
self.exit_message = msgobj["message"]
else:
self.exit_message = None
elif msgobj["msg"] == "milestone":
# milestone/whereis update: milestone fields are right in the
# message
self.receiving_direct_milestones = True # no need for .where files
self.set_where_info(msgobj)
else:
self.logger.warning(
"Unknown message from the crawl process: %s",
msgobj["msg"])
else:
self.check_where()
if time.time() > self.last_watcher_join + 2:
# Treat socket messages as activity, since it's otherwise
# hard to determine activity for games found via
# watch_socket_dirs.
# But don't if a spectator just joined, since we don't
# want that to reset idle time.
self.note_activity()
self.write_to_all(msg, not self.queue_messages)