def prepare_test(load_config=True, connect_database=True, connect_logger=True): config_file_name = Utils.new_tmp_file() log_file_name = Utils.new_tmp_file() db_file_name = Utils.new_tmp_file() contest_dir = Utils.new_tmp_dir("contest", create=False) with open(config_file_name, 'w') as file: file.write("logfile: %s\n" "db: %s\n" "storedir: %s\n" "contest_path: %s\n" % (log_file_name, db_file_name, Utils.new_tmp_dir(), contest_dir)) if load_config: Config.loaded = False Config.set_config_file(config_file_name) if connect_logger: Logger.connected = False Logger.connect_to_database() Logger.set_log_level("WARNING") if connect_database: Database.connected = False Database.connect_to_database()
def upload_output(self, input, file): """ POST /upload_output """ output_id = Database.gen_id() try: path = StorageManager.new_output_file(output_id, file["name"]) except ValueError: BaseHandler.raise_exc(BadRequest, "INVALID_FILENAME", "The provided file has an invalid name") StorageManager.save_file(path, file["content"]) file_size = StorageManager.get_file_size(path) try: result = ContestManager.evaluate_output(input["task"], input["path"], path) except: BaseHandler.raise_exc(InternalServerError, "INTERNAL_ERROR", "Failed to evaluate the output") Database.add_output(output_id, input["id"], path, file_size, result) Logger.info( "UPLOAD", "User %s has uploaded the output %s" % (input["token"], output_id)) return InfoHandler.patch_output(Database.get_output(output_id))
def test_set_log_level_numeric(self): backup = Logger.LOG_LEVEL Logger.set_log_level(Logger.ERROR) self.assertEqual(Logger.LOG_LEVEL, Logger.ERROR) Logger.LOG_LEVEL = backup
def _get_user_from_sso(jwt_token, token): try: data = jwt.decode(jwt_token, Config.jwt_secret, algorithms=['HS256']) username = data["username"] name = data.get("firstName", username) surname = data.get("lastName", "") if username != token: BaseHandler.raise_exc(Forbidden, "FORBIDDEN", "Use the same username from the SSO") if Database.get_user(username) is None: Database.begin() Database.add_user(username, name, surname, sso_user=True, autocommit=False) for task in Database.get_tasks(): Database.add_user_task(username, task["name"], autocommit=False) Database.commit() Logger.info("NEW_USER", "User %s created from SSO" % username) return Database.get_user(username) except jwt.exceptions.DecodeError: BaseHandler.raise_exc(Forbidden, "FORBIDDEN", "Please login at %s" % Config.sso_url)
def download_results(self): """ POST /admin/download_pack """ Logger.info("ADMIN", "Creating zip file") zip_directory = os.path.join(Config.storedir, "zips", Database.gen_id()) os.makedirs(zip_directory, exist_ok=True) zipf_name = ("results-" + Database.get_meta("admin_token").split("-", 1)[0] + "-" + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + ".zip") zipf_name = os.path.join(zip_directory, zipf_name) command = ("zip -r '" + zipf_name + "' db.sqlite3* log.sqlite3* " "files/input files/output " "files/source /version* " "/proc/cpuinfo* " "/var/log/nginx") try: gevent.subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: Logger.error("ADMIN", "Zip error: %s" % e.output) raise e return { "path": os.path.relpath(zipf_name, Config.storedir) } # pragma: nocover
def generate_input(self, task, user): """ POST /generate_input """ token = user["token"] if Database.get_user_task(token, task["name"])["current_attempt"]: self.raise_exc(Forbidden, "FORBIDDEN", "You already have a ready input!") attempt = Database.get_next_attempt(token, task["name"]) id, path = ContestManager.get_input(task["name"], attempt) size = StorageManager.get_file_size(path) Database.begin() try: Database.add_input(id, token, task["name"], attempt, path, size, autocommit=False) Database.set_user_attempt(token, task["name"], attempt, autocommit=False) Database.commit() except: Database.rollback() raise Logger.info( "CONTEST", "Generated input %s for user %s on task %s" % (id, token, task["name"])) return BaseHandler.format_dates(Database.get_input(id=id))
def test_log_stderr(self): Utils.prepare_test(connect_logger=True) with Utils.nostderr() as err: Logger.error("FOO_CAT", "Log message") self.assertIn("FOO_CAT", err.buffer) self.assertIn("Log message", err.buffer)
def upload_source(self, input, file): """ POST /upload_source """ alerts = [] if get_exeflags(file["content"]): alerts.append({ "severity": "warning", "message": "You have submitted an executable! Please send the " "source code." }) Logger.info("UPLOAD", "User %s has uploaded an executable" % input["token"]) if not alerts: alerts.append({ "severity": "success", "message": "Source file uploaded correctly." }) source_id = Database.gen_id() try: path = StorageManager.new_source_file(source_id, file["name"]) except ValueError: BaseHandler.raise_exc(BadRequest, "INVALID_FILENAME", "The provided file has an invalid name") StorageManager.save_file(path, file["content"]) file_size = StorageManager.get_file_size(path) Database.add_source(source_id, input["id"], path, file_size) Logger.info("UPLOAD", "User %s has uploaded the source %s" % ( input["token"], source_id)) output = BaseHandler.format_dates(Database.get_source(source_id)) output["validation"] = {"alerts": alerts} return output
def handle(*args, **kwargs): token = Validators._guess_token(**kwargs) ip = kwargs["_ip"] if token is not None and Database.get_user(token) is not None: if Database.register_ip(token, ip): Logger.info( "LOGIN", "User %s logged in from %s for the first " "time" % (token, ip)) del kwargs["_ip"] return handler(*args, **kwargs)
def test_log(self): Utils.prepare_test(connect_logger=True) Logger.log(Logger.DEBUG, "FOO_CAT", "Log message") Logger.c.execute("SELECT * FROM logs WHERE category = 'FOO_CAT'") row = Logger.c.fetchone() self.assertEqual("FOO_CAT", row[1]) self.assertEqual(Logger.DEBUG, int(row[2])) self.assertEqual("Log message", row[3])
def test_run(self, spawn, init): Logger.set_log_level("INFO") with patch.object(spawn(), "join") as join: with Utils.nostderr() as stderr: self.server.run() init.assert_called_once_with() self.assertTrue(spawn.called) join.assert_called_once_with() self.assertIn("SERVER_STATUS", stderr.buffer) self.assertIn("Server started", stderr.buffer)
def test_get_logs_by_level(self): Utils.prepare_test(connect_logger=True) TestLogger.load_logs() start_date = datetime.datetime.now().timestamp() - 10 end_date = datetime.datetime.now().timestamp() + 10 logs = Logger.get_logs(Logger.DEBUG, None, start_date, end_date) self.assertEqual(4, len(logs)) logs = Logger.get_logs(Logger.WARNING, None, start_date, end_date) self.assertEqual(2, len(logs))
def append_log(self, append_log_secret: str, level: str, category: str, message: str): """ POST /admin/append_log """ if append_log_secret != Config.append_log_secret: self.raise_exc(Forbidden, "FORBIDDEN", "Invalid append log secret") if level not in Logger.HUMAN_MESSAGES: self.raise_exc(BadRequest, "INVALID_PARAMETER", "The level provided is invalid") level = Logger.HUMAN_MESSAGES.index(level) Logger.log(level, category, message)
def extract_contest(token): """ Decrypt and extract the contest and store the used admin token in the database """ if "-" not in token: BaseHandler.raise_exc(Forbidden, "WRONG_PASSWORD", "The provided password is malformed") try: username, password = token.split("-", 1) secret, scrambled_password = decode_data(password, SECRET_LEN) file_password = recover_file_password(username, secret, scrambled_password) except ValueError: BaseHandler.raise_exc(Forbidden, "WRONG_PASSWORD", "The provided password is malformed") try: with open(Config.encrypted_file, "rb") as encrypted_file: encrypted_data = encrypted_file.read() decrypted_data = decode(file_password, encrypted_data) with open(Config.decrypted_file, "wb") as decrypted_file: decrypted_file.write(decrypted_data) except FileNotFoundError: BaseHandler.raise_exc(NotFound, "NOT_FOUND", "The contest pack was not uploaded yet") except nacl.exceptions.CryptoError: BaseHandler.raise_exc(Forbidden, "WRONG_PASSWORD", "The provided password is wrong") except OSError as ex: BaseHandler.raise_exc(InternalServerError, "FAILED", str(ex)) zip_abs_path = os.path.realpath(Config.decrypted_file) wd = os.getcwd() try: os.makedirs(Config.contest_path, exist_ok=True) os.chdir(Config.contest_path) with zipfile.ZipFile(zip_abs_path) as f: f.extractall() real_yaml = os.path.join("__users__", username + ".yaml") if not os.path.exists(real_yaml): BaseHandler.raise_exc(Forbidden, "WRONG_PASSWORD", "Invalid username for the given pack") os.symlink(real_yaml, "contest.yaml") Logger.info("CONTEST", "Contest extracted") except zipfile.BadZipFile as ex: BaseHandler.raise_exc(Forbidden, "FAILED", str(ex)) finally: os.chdir(wd) Database.set_meta("admin_token", token)
def start(self): """ POST /admin/start """ if Database.get_meta("start_time", default=None, type=int) is not None: BaseHandler.raise_exc(Forbidden, "FORBIDDEN", "Contest has already been started!") start_time = int(time.time()) Database.set_meta("start_time", start_time) Logger.info("CONTEST", "Contest started") return BaseHandler.format_dates({"start_time": start_time}, fields=["start_time"])
def set_extra_time(self, extra_time: int, user): """ POST /admin/set_extra_time """ if user is None: Database.set_meta("extra_time", extra_time) Logger.info("ADMIN", "Global extra time set to %d" % extra_time) else: Logger.info( "ADMIN", "Extra time for user %s set to %d" % (user["token"], extra_time)) Database.set_extra_time(user["token"], extra_time) return {}
def wsgi_app(self, environ, start_response): route = self.router.bind_to_environ(environ) request = Request(environ) try: endpoint, args = route.match() except HTTPException: Logger.warning("HTTP_ERROR", "%s %s %s 404" % (BaseHandler.get_ip(request), request.method, request.url)) return NotFound() controller, action = endpoint.split("#") return self.handlers[controller].handle(action, args, request)
def get_input(task_name, attempt): """ Fetch an input from the queue and properly rename it :param task_name: Name of the task :param attempt: Number of the attempt for the user :return: A pair, the first element is the id of the input file, the second the path """ if ContestManager.input_queue[task_name].empty(): Logger.warning("TASK", "Empty queue for task %s!" % task_name) input = ContestManager.input_queue[task_name].get() path = StorageManager.new_input_file(input["id"], task_name, attempt) StorageManager.rename_file(input["path"], path) return input["id"], path
def run(self): """ Start a greenlet with the main HTTP server loop """ server = gevent.pywsgi.WSGIServer( (Config.address, Config.port), self, log=None) try: server.init_socket() except OSError: Logger.error("PORT_ALREADY_IN_USE", "Address: '%s' Port: %d" % (Config.address, Config.port)) sys.exit(1) greenlet = gevent.spawn(server.serve_forever) port = "" if Config.port == 80 else ":" + str(Config.port) Logger.info("SERVER_STATUS", "Server started at http://%s%s/" % (str(Config.address), port)) greenlet.join()
def test_get_logs_by_date(self): Utils.prepare_test(connect_logger=True) TestLogger.load_logs() start_date = TestLogger.VERY_FAR_IN_TIME - 10 end_date = TestLogger.VERY_FAR_IN_TIME + 10 logs = Logger.get_logs(Logger.DEBUG, None, start_date, end_date) self.assertEqual(1, len(logs))
def test_get_logs_by_category(self): Utils.prepare_test(connect_logger=True) TestLogger.load_logs() start_date = datetime.datetime.now().timestamp() - 10 end_date = datetime.datetime.now().timestamp() + 10 logs = Logger.get_logs(Logger.DEBUG, "CATEGORY", start_date, end_date) self.assertEqual(2, len(logs))
def connect_to_database(): if Database.connected is True: raise RuntimeError("Database already loaded") Database.connected = True Database.conn = sqlite3.connect(Config.db, check_same_thread=False, isolation_level=None, detect_types=sqlite3.PARSE_DECLTYPES) Database.c = Database.conn.cursor() Database.c.executescript(Schema.INIT) version = Database.get_meta("schema_version", -1, int) if version == -1: Logger.info("DB_OPERATION", "Creating database") for upd in range(version + 1, len(Schema.UPDATERS)): Logger.info("DB_OPERATION", "Applying updater %d" % upd) Database.c.executescript(Schema.UPDATERS[upd]) Database.set_meta("schema_version", upd) Database.conn.commit()
def extract_and_connect(path, workdir): from terry.config import Config from terry.database import Database from terry.logger import Logger with zipfile.ZipFile(path) as f: f.extractall(workdir) db_path = os.path.join(workdir, "db.sqlite3") log_path = os.path.join(workdir, "log.sqlite3") Config.db = db_path Config.logfile = log_path Database.connect_to_database() Logger.connect_to_database() try: yield finally: Database.disconnect_database() Logger.disconnect_database()
def drop_contest(self, admin_token): """ POST /admin/drop_contest """ if not os.path.exists(Config.encrypted_file): self.raise_exc(NotFound, "NOT_FOUND", "No packs found") Logger.warning("DROP_CONTEST", "Started dropping contest") with open(Config.encrypted_file, "rb") as f: pack = f.read() db_token = Database.get_meta("admin_token") # contest has been extracted but the token is wrong if db_token is not None and db_token != admin_token: self.raise_exc(Forbidden, "FORBIDDEN", "Wrong token") # contest has not been extracted if db_token is None: try: password = crypto.recover_file_password_from_token(admin_token) crypto.decode(password, pack) except nacl.exceptions.CryptoError: # pack password is wrong self.raise_exc(Forbidden, "FORBIDDEN", "Wrong pack token") metadata = ruamel.yaml.safe_load(crypto.metadata(pack).strip(b"\x00")) if not metadata.get("deletable"): self.raise_exc(Forbidden, "FORBIDDEN", "Contest not deletable") shutil.rmtree(Config.storedir, ignore_errors=True) shutil.rmtree(Config.statementdir, ignore_errors=True) shutil.rmtree(Config.contest_path, ignore_errors=True) for f in (Config.encrypted_file, Config.decrypted_file): try: os.remove(f) except FileNotFoundError: pass Database.disconnect_database() for f in glob.glob(Config.db + "*"): os.remove(f) Database.connect_to_database() Logger.warning("DROP_CONTEST", "Contest dropped") return {}
def evaluate_output(task_name, input_path, output_path): """ Given an input of a task, evaluate the correctness of the output :param task_name: Name of the task :param input_path: Path to the user's input file :param output_path: Path to the user's output file :return: The stdout of the checker """ try: # call the checker and store the output start_time = time.monotonic() output = gevent.subprocess.check_output([ ContestManager.tasks[task_name]["checker"], StorageManager.get_absolute_path(input_path), StorageManager.get_absolute_path(output_path) ]) if time.monotonic() > start_time + 1: Logger.warning( "TASK", "Evaluation of output %s " "for task %s, with input %s, took %f " "seconds" % (output_path, task_name, input_path, time.monotonic() - start_time)) except: # TODO log the stdout and stderr of the checker Logger.error( "TASK", "Error while evaluating output %s " "for task %s, with input %s: %s" % (output_path, task_name, input_path, traceback.format_exc())) raise Logger.info( "TASK", "Evaluated output %s for task %s, with input %s" % (output_path, task_name, input_path)) return output
def start(self, start_time: str): """ POST /admin/start """ previous_start = Database.get_meta("start_time", type=int) now = int(time.time()) if previous_start and now > previous_start: BaseHandler.raise_exc(Forbidden, "FORBIDDEN", "Contest has already been started!") actual_start = None if start_time == "reset": Database.del_meta("start_time") return {"start_time": None} elif start_time == "now": actual_start = now else: actual_start = dateutil.parser.parse(start_time).timestamp() Database.set_meta("start_time", int(actual_start)) Logger.info("CONTEST", "Contest starts at " + str(actual_start)) return BaseHandler.format_dates({"start_time": actual_start}, fields=["start_time"])
def _ensure_window_start(token): """ Makes sure that the window of the user has been started :param token: The token of the user :return: True if the user has been updated """ if not Database.get_meta("window_duration", None): return False user = Database.get_user(token) if not user: return False start_delay = user["contest_start_delay"] if start_delay is not None: return False start = Database.get_meta("start_time", type=int) if start is None: return False now = time.time() delay = now - start Database.set_start_delay(token, delay) Logger.info("WINDOW_START", "Contest started for %s after %d" % (token, delay)) return True
def _validate_admin_token(token, ip): """ Ensure the admin token is valid :param token: Token to check :param ip: IP of the client """ correct_token = Database.get_meta("admin_token") if correct_token is None: ContestManager.extract_contest(token) ContestManager.read_from_disk() correct_token = token if token != correct_token: Logger.warning("LOGIN_ADMIN", "Admin login failed from %s" % ip) BaseHandler.raise_exc(Forbidden, "FORBIDDEN", "Invalid admin token!") else: if Database.register_admin_ip(ip): Logger.info("LOGIN_ADMIN", "An admin has connected from a new ip: %s" % ip)
def submit(self, output, source): """ POST /submit """ input = Database.get_input(output["input"]) if input is None: Logger.warning("DB_CONSISTENCY_ERROR", "Input %s not found in the db" % output["input"]) self.raise_exc(BadRequest, "WRONG_INPUT", "The provided input in invalid") if output["input"] != source["input"]: Logger.warning("POSSIBLE_CHEAT", "Trying to submit wrong pair source-output") self.raise_exc(Forbidden, "WRONG_OUTPUT_SOURCE", "The provided pair of source-output is invalid") score = ContestHandler.compute_score(input["task"], output["result"]) Database.begin() try: submission_id = Database.gen_id() if not Database.add_submission(submission_id, input["id"], output["id"], source["id"], score, autocommit=False): self.raise_exc(BadRequest, "INTERNAL_ERROR", "Error inserting the submission") ContestHandler.update_user_score(input["token"], input["task"], score) Database.set_user_attempt(input["token"], input["task"], None, autocommit=False) Database.commit() except sqlite3.IntegrityError as ex: Database.rollback() # provide a better error message if the input has already been # submitted if "UNIQUE constraint failed: submissions.input" in str(ex): self.raise_exc(Forbidden, "ALREADY_SUBMITTED", "This input has already been submitted") raise except: Database.rollback() raise Logger.info( "CONTEST", "User %s has submitted %s on %s" % (input["token"], submission_id, input["task"])) return InfoHandler.patch_submission( Database.get_submission(submission_id))
def log(self, start_date: str, end_date: str, level: str, category: str = None): """ POST /admin/log """ if level not in Logger.HUMAN_MESSAGES: self.raise_exc(BadRequest, "INVALID_PARAMETER", "The level provided is invalid") level = Logger.HUMAN_MESSAGES.index(level) try: start_date = dateutil.parser.parse(start_date).timestamp() end_date = dateutil.parser.parse(end_date).timestamp() except ValueError as e: BaseHandler.raise_exc(BadRequest, "INVALID_PARAMETER", str(e)) return BaseHandler.format_dates( {"items": Logger.get_logs(level, category, start_date, end_date)})