def prepare_test(load_config=True, connect_database=True,
connect_logger=True):
config_file_name = Utils.new_tmp_file()
log_file_name = Utils.new_tmp_file()
db_file_name = Utils.new_tmp_file()
contest_dir = Utils.new_tmp_dir("contest", create=False)
with open(config_file_name, 'w') as file:
file.write("logfile: %s\n"
"db: %s\n"
"storedir: %s\n"
"contest_path: %s\n" % (log_file_name, db_file_name,
Utils.new_tmp_dir(),
contest_dir))
if load_config:
Config.loaded = False
Config.set_config_file(config_file_name)
if connect_logger:
Logger.connected = False
Logger.connect_to_database()
Logger.set_log_level("WARNING")
if connect_database:
Database.connected = False
Database.connect_to_database()
def upload_output(self, input, file):
"""
POST /upload_output
"""
output_id = Database.gen_id()
try:
path = StorageManager.new_output_file(output_id, file["name"])
except ValueError:
BaseHandler.raise_exc(BadRequest, "INVALID_FILENAME",
"The provided file has an invalid name")
StorageManager.save_file(path, file["content"])
file_size = StorageManager.get_file_size(path)
try:
result = ContestManager.evaluate_output(input["task"],
input["path"], path)
except:
BaseHandler.raise_exc(InternalServerError, "INTERNAL_ERROR",
"Failed to evaluate the output")
Database.add_output(output_id, input["id"], path, file_size, result)
Logger.info(
"UPLOAD",
"User %s has uploaded the output %s" % (input["token"], output_id))
return InfoHandler.patch_output(Database.get_output(output_id))
def test_set_log_level_numeric(self):
backup = Logger.LOG_LEVEL
Logger.set_log_level(Logger.ERROR)
self.assertEqual(Logger.LOG_LEVEL, Logger.ERROR)
Logger.LOG_LEVEL = backup
def _get_user_from_sso(jwt_token, token):
try:
data = jwt.decode(jwt_token,
Config.jwt_secret,
algorithms=['HS256'])
username = data["username"]
name = data.get("firstName", username)
surname = data.get("lastName", "")
if username != token:
BaseHandler.raise_exc(Forbidden, "FORBIDDEN",
"Use the same username from the SSO")
if Database.get_user(username) is None:
Database.begin()
Database.add_user(username,
name,
surname,
sso_user=True,
autocommit=False)
for task in Database.get_tasks():
Database.add_user_task(username,
task["name"],
autocommit=False)
Database.commit()
Logger.info("NEW_USER", "User %s created from SSO" % username)
return Database.get_user(username)
except jwt.exceptions.DecodeError:
BaseHandler.raise_exc(Forbidden, "FORBIDDEN",
"Please login at %s" % Config.sso_url)
def download_results(self):
"""
POST /admin/download_pack
"""
Logger.info("ADMIN", "Creating zip file")
zip_directory = os.path.join(Config.storedir, "zips",
Database.gen_id())
os.makedirs(zip_directory, exist_ok=True)
zipf_name = ("results-" +
Database.get_meta("admin_token").split("-", 1)[0] + "-" +
time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) +
".zip")
zipf_name = os.path.join(zip_directory, zipf_name)
command = ("zip -r '" + zipf_name + "' db.sqlite3* log.sqlite3* "
"files/input files/output "
"files/source /version* "
"/proc/cpuinfo* "
"/var/log/nginx")
try:
gevent.subprocess.check_output(command,
shell=True,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
Logger.error("ADMIN", "Zip error: %s" % e.output)
raise e
return {
"path": os.path.relpath(zipf_name, Config.storedir)
} # pragma: nocover
def generate_input(self, task, user):
"""
POST /generate_input
"""
token = user["token"]
if Database.get_user_task(token, task["name"])["current_attempt"]:
self.raise_exc(Forbidden, "FORBIDDEN",
"You already have a ready input!")
attempt = Database.get_next_attempt(token, task["name"])
id, path = ContestManager.get_input(task["name"], attempt)
size = StorageManager.get_file_size(path)
Database.begin()
try:
Database.add_input(id,
token,
task["name"],
attempt,
path,
size,
autocommit=False)
Database.set_user_attempt(token,
task["name"],
attempt,
autocommit=False)
Database.commit()
except:
Database.rollback()
raise
Logger.info(
"CONTEST", "Generated input %s for user %s on task %s" %
(id, token, task["name"]))
return BaseHandler.format_dates(Database.get_input(id=id))
def test_log_stderr(self):
Utils.prepare_test(connect_logger=True)
with Utils.nostderr() as err:
Logger.error("FOO_CAT", "Log message")
self.assertIn("FOO_CAT", err.buffer)
self.assertIn("Log message", err.buffer)
def upload_source(self, input, file):
"""
POST /upload_source
"""
alerts = []
if get_exeflags(file["content"]):
alerts.append({
"severity": "warning",
"message": "You have submitted an executable! Please send the "
"source code."
})
Logger.info("UPLOAD",
"User %s has uploaded an executable" % input["token"])
if not alerts:
alerts.append({
"severity": "success",
"message": "Source file uploaded correctly."
})
source_id = Database.gen_id()
try:
path = StorageManager.new_source_file(source_id, file["name"])
except ValueError:
BaseHandler.raise_exc(BadRequest, "INVALID_FILENAME",
"The provided file has an invalid name")
StorageManager.save_file(path, file["content"])
file_size = StorageManager.get_file_size(path)
Database.add_source(source_id, input["id"], path, file_size)
Logger.info("UPLOAD", "User %s has uploaded the source %s" % (
input["token"], source_id))
output = BaseHandler.format_dates(Database.get_source(source_id))
output["validation"] = {"alerts": alerts}
return output
def handle(*args, **kwargs):
token = Validators._guess_token(**kwargs)
ip = kwargs["_ip"]
if token is not None and Database.get_user(token) is not None:
if Database.register_ip(token, ip):
Logger.info(
"LOGIN", "User %s logged in from %s for the first "
"time" % (token, ip))
del kwargs["_ip"]
return handler(*args, **kwargs)
def test_log(self):
Utils.prepare_test(connect_logger=True)
Logger.log(Logger.DEBUG, "FOO_CAT", "Log message")
Logger.c.execute("SELECT * FROM logs WHERE category = 'FOO_CAT'")
row = Logger.c.fetchone()
self.assertEqual("FOO_CAT", row[1])
self.assertEqual(Logger.DEBUG, int(row[2]))
self.assertEqual("Log message", row[3])
def test_run(self, spawn, init):
Logger.set_log_level("INFO")
with patch.object(spawn(), "join") as join:
with Utils.nostderr() as stderr:
self.server.run()
init.assert_called_once_with()
self.assertTrue(spawn.called)
join.assert_called_once_with()
self.assertIn("SERVER_STATUS", stderr.buffer)
self.assertIn("Server started", stderr.buffer)
def test_get_logs_by_level(self):
Utils.prepare_test(connect_logger=True)
TestLogger.load_logs()
start_date = datetime.datetime.now().timestamp() - 10
end_date = datetime.datetime.now().timestamp() + 10
logs = Logger.get_logs(Logger.DEBUG, None, start_date, end_date)
self.assertEqual(4, len(logs))
logs = Logger.get_logs(Logger.WARNING, None, start_date, end_date)
self.assertEqual(2, len(logs))
def append_log(self, append_log_secret: str, level: str, category: str,
message: str):
"""
POST /admin/append_log
"""
if append_log_secret != Config.append_log_secret:
self.raise_exc(Forbidden, "FORBIDDEN", "Invalid append log secret")
if level not in Logger.HUMAN_MESSAGES:
self.raise_exc(BadRequest, "INVALID_PARAMETER",
"The level provided is invalid")
level = Logger.HUMAN_MESSAGES.index(level)
Logger.log(level, category, message)
def extract_contest(token):
"""
Decrypt and extract the contest and store the used admin token in the
database
"""
if "-" not in token:
BaseHandler.raise_exc(Forbidden, "WRONG_PASSWORD",
"The provided password is malformed")
try:
username, password = token.split("-", 1)
secret, scrambled_password = decode_data(password, SECRET_LEN)
file_password = recover_file_password(username, secret,
scrambled_password)
except ValueError:
BaseHandler.raise_exc(Forbidden, "WRONG_PASSWORD",
"The provided password is malformed")
try:
with open(Config.encrypted_file, "rb") as encrypted_file:
encrypted_data = encrypted_file.read()
decrypted_data = decode(file_password, encrypted_data)
with open(Config.decrypted_file, "wb") as decrypted_file:
decrypted_file.write(decrypted_data)
except FileNotFoundError:
BaseHandler.raise_exc(NotFound, "NOT_FOUND",
"The contest pack was not uploaded yet")
except nacl.exceptions.CryptoError:
BaseHandler.raise_exc(Forbidden, "WRONG_PASSWORD",
"The provided password is wrong")
except OSError as ex:
BaseHandler.raise_exc(InternalServerError, "FAILED", str(ex))
zip_abs_path = os.path.realpath(Config.decrypted_file)
wd = os.getcwd()
try:
os.makedirs(Config.contest_path, exist_ok=True)
os.chdir(Config.contest_path)
with zipfile.ZipFile(zip_abs_path) as f:
f.extractall()
real_yaml = os.path.join("__users__", username + ".yaml")
if not os.path.exists(real_yaml):
BaseHandler.raise_exc(Forbidden, "WRONG_PASSWORD",
"Invalid username for the given pack")
os.symlink(real_yaml, "contest.yaml")
Logger.info("CONTEST", "Contest extracted")
except zipfile.BadZipFile as ex:
BaseHandler.raise_exc(Forbidden, "FAILED", str(ex))
finally:
os.chdir(wd)
Database.set_meta("admin_token", token)
def start(self):
"""
POST /admin/start
"""
if Database.get_meta("start_time", default=None, type=int) is not None:
BaseHandler.raise_exc(Forbidden, "FORBIDDEN",
"Contest has already been started!")
start_time = int(time.time())
Database.set_meta("start_time", start_time)
Logger.info("CONTEST", "Contest started")
return BaseHandler.format_dates({"start_time": start_time},
fields=["start_time"])
def set_extra_time(self, extra_time: int, user):
"""
POST /admin/set_extra_time
"""
if user is None:
Database.set_meta("extra_time", extra_time)
Logger.info("ADMIN", "Global extra time set to %d" % extra_time)
else:
Logger.info(
"ADMIN", "Extra time for user %s set to %d" %
(user["token"], extra_time))
Database.set_extra_time(user["token"], extra_time)
return {}
def wsgi_app(self, environ, start_response):
route = self.router.bind_to_environ(environ)
request = Request(environ)
try:
endpoint, args = route.match()
except HTTPException:
Logger.warning("HTTP_ERROR",
"%s %s %s 404" % (BaseHandler.get_ip(request),
request.method, request.url))
return NotFound()
controller, action = endpoint.split("#")
return self.handlers[controller].handle(action, args, request)
def get_input(task_name, attempt):
"""
Fetch an input from the queue and properly rename it
:param task_name: Name of the task
:param attempt: Number of the attempt for the user
:return: A pair, the first element is the id of the input file,
the second the path
"""
if ContestManager.input_queue[task_name].empty():
Logger.warning("TASK", "Empty queue for task %s!" % task_name)
input = ContestManager.input_queue[task_name].get()
path = StorageManager.new_input_file(input["id"], task_name, attempt)
StorageManager.rename_file(input["path"], path)
return input["id"], path
def run(self):
"""
Start a greenlet with the main HTTP server loop
"""
server = gevent.pywsgi.WSGIServer(
(Config.address, Config.port), self, log=None)
try:
server.init_socket()
except OSError:
Logger.error("PORT_ALREADY_IN_USE", "Address: '%s' Port: %d" %
(Config.address, Config.port))
sys.exit(1)
greenlet = gevent.spawn(server.serve_forever)
port = "" if Config.port == 80 else ":" + str(Config.port)
Logger.info("SERVER_STATUS", "Server started at http://%s%s/" %
(str(Config.address), port))
greenlet.join()
def test_get_logs_by_date(self):
Utils.prepare_test(connect_logger=True)
TestLogger.load_logs()
start_date = TestLogger.VERY_FAR_IN_TIME - 10
end_date = TestLogger.VERY_FAR_IN_TIME + 10
logs = Logger.get_logs(Logger.DEBUG, None, start_date, end_date)
self.assertEqual(1, len(logs))
def test_get_logs_by_category(self):
Utils.prepare_test(connect_logger=True)
TestLogger.load_logs()
start_date = datetime.datetime.now().timestamp() - 10
end_date = datetime.datetime.now().timestamp() + 10
logs = Logger.get_logs(Logger.DEBUG, "CATEGORY", start_date, end_date)
self.assertEqual(2, len(logs))
def connect_to_database():
if Database.connected is True:
raise RuntimeError("Database already loaded")
Database.connected = True
Database.conn = sqlite3.connect(Config.db,
check_same_thread=False,
isolation_level=None,
detect_types=sqlite3.PARSE_DECLTYPES)
Database.c = Database.conn.cursor()
Database.c.executescript(Schema.INIT)
version = Database.get_meta("schema_version", -1, int)
if version == -1:
Logger.info("DB_OPERATION", "Creating database")
for upd in range(version + 1, len(Schema.UPDATERS)):
Logger.info("DB_OPERATION", "Applying updater %d" % upd)
Database.c.executescript(Schema.UPDATERS[upd])
Database.set_meta("schema_version", upd)
Database.conn.commit()
def extract_and_connect(path, workdir):
from terry.config import Config
from terry.database import Database
from terry.logger import Logger
with zipfile.ZipFile(path) as f:
f.extractall(workdir)
db_path = os.path.join(workdir, "db.sqlite3")
log_path = os.path.join(workdir, "log.sqlite3")
Config.db = db_path
Config.logfile = log_path
Database.connect_to_database()
Logger.connect_to_database()
try:
yield
finally:
Database.disconnect_database()
Logger.disconnect_database()
def drop_contest(self, admin_token):
"""
POST /admin/drop_contest
"""
if not os.path.exists(Config.encrypted_file):
self.raise_exc(NotFound, "NOT_FOUND", "No packs found")
Logger.warning("DROP_CONTEST", "Started dropping contest")
with open(Config.encrypted_file, "rb") as f:
pack = f.read()
db_token = Database.get_meta("admin_token")
# contest has been extracted but the token is wrong
if db_token is not None and db_token != admin_token:
self.raise_exc(Forbidden, "FORBIDDEN", "Wrong token")
# contest has not been extracted
if db_token is None:
try:
password = crypto.recover_file_password_from_token(admin_token)
crypto.decode(password, pack)
except nacl.exceptions.CryptoError:
# pack password is wrong
self.raise_exc(Forbidden, "FORBIDDEN", "Wrong pack token")
metadata = ruamel.yaml.safe_load(crypto.metadata(pack).strip(b"\x00"))
if not metadata.get("deletable"):
self.raise_exc(Forbidden, "FORBIDDEN", "Contest not deletable")
shutil.rmtree(Config.storedir, ignore_errors=True)
shutil.rmtree(Config.statementdir, ignore_errors=True)
shutil.rmtree(Config.contest_path, ignore_errors=True)
for f in (Config.encrypted_file, Config.decrypted_file):
try:
os.remove(f)
except FileNotFoundError:
pass
Database.disconnect_database()
for f in glob.glob(Config.db + "*"):
os.remove(f)
Database.connect_to_database()
Logger.warning("DROP_CONTEST", "Contest dropped")
return {}
def evaluate_output(task_name, input_path, output_path):
"""
Given an input of a task, evaluate the correctness of the output
:param task_name: Name of the task
:param input_path: Path to the user's input file
:param output_path: Path to the user's output file
:return: The stdout of the checker
"""
try:
# call the checker and store the output
start_time = time.monotonic()
output = gevent.subprocess.check_output([
ContestManager.tasks[task_name]["checker"],
StorageManager.get_absolute_path(input_path),
StorageManager.get_absolute_path(output_path)
])
if time.monotonic() > start_time + 1:
Logger.warning(
"TASK", "Evaluation of output %s "
"for task %s, with input %s, took %f "
"seconds" % (output_path, task_name, input_path,
time.monotonic() - start_time))
except:
# TODO log the stdout and stderr of the checker
Logger.error(
"TASK", "Error while evaluating output %s "
"for task %s, with input %s: %s" %
(output_path, task_name, input_path, traceback.format_exc()))
raise
Logger.info(
"TASK", "Evaluated output %s for task %s, with input %s" %
(output_path, task_name, input_path))
return output
def start(self, start_time: str):
"""
POST /admin/start
"""
previous_start = Database.get_meta("start_time", type=int)
now = int(time.time())
if previous_start and now > previous_start:
BaseHandler.raise_exc(Forbidden, "FORBIDDEN",
"Contest has already been started!")
actual_start = None
if start_time == "reset":
Database.del_meta("start_time")
return {"start_time": None}
elif start_time == "now":
actual_start = now
else:
actual_start = dateutil.parser.parse(start_time).timestamp()
Database.set_meta("start_time", int(actual_start))
Logger.info("CONTEST", "Contest starts at " + str(actual_start))
return BaseHandler.format_dates({"start_time": actual_start},
fields=["start_time"])
def _ensure_window_start(token):
"""
Makes sure that the window of the user has been started
:param token: The token of the user
:return: True if the user has been updated
"""
if not Database.get_meta("window_duration", None):
return False
user = Database.get_user(token)
if not user:
return False
start_delay = user["contest_start_delay"]
if start_delay is not None:
return False
start = Database.get_meta("start_time", type=int)
if start is None:
return False
now = time.time()
delay = now - start
Database.set_start_delay(token, delay)
Logger.info("WINDOW_START", "Contest started for %s after %d" % (token, delay))
return True
def _validate_admin_token(token, ip):
"""
Ensure the admin token is valid
:param token: Token to check
:param ip: IP of the client
"""
correct_token = Database.get_meta("admin_token")
if correct_token is None:
ContestManager.extract_contest(token)
ContestManager.read_from_disk()
correct_token = token
if token != correct_token:
Logger.warning("LOGIN_ADMIN", "Admin login failed from %s" % ip)
BaseHandler.raise_exc(Forbidden, "FORBIDDEN",
"Invalid admin token!")
else:
if Database.register_admin_ip(ip):
Logger.info("LOGIN_ADMIN",
"An admin has connected from a new ip: %s" % ip)
def submit(self, output, source):
"""
POST /submit
"""
input = Database.get_input(output["input"])
if input is None:
Logger.warning("DB_CONSISTENCY_ERROR",
"Input %s not found in the db" % output["input"])
self.raise_exc(BadRequest, "WRONG_INPUT",
"The provided input in invalid")
if output["input"] != source["input"]:
Logger.warning("POSSIBLE_CHEAT",
"Trying to submit wrong pair source-output")
self.raise_exc(Forbidden, "WRONG_OUTPUT_SOURCE",
"The provided pair of source-output is invalid")
score = ContestHandler.compute_score(input["task"], output["result"])
Database.begin()
try:
submission_id = Database.gen_id()
if not Database.add_submission(submission_id,
input["id"],
output["id"],
source["id"],
score,
autocommit=False):
self.raise_exc(BadRequest, "INTERNAL_ERROR",
"Error inserting the submission")
ContestHandler.update_user_score(input["token"], input["task"],
score)
Database.set_user_attempt(input["token"],
input["task"],
None,
autocommit=False)
Database.commit()
except sqlite3.IntegrityError as ex:
Database.rollback()
# provide a better error message if the input has already been
# submitted
if "UNIQUE constraint failed: submissions.input" in str(ex):
self.raise_exc(Forbidden, "ALREADY_SUBMITTED",
"This input has already been submitted")
raise
except:
Database.rollback()
raise
Logger.info(
"CONTEST", "User %s has submitted %s on %s" %
(input["token"], submission_id, input["task"]))
return InfoHandler.patch_submission(
Database.get_submission(submission_id))
def log(self,
start_date: str,
end_date: str,
level: str,
category: str = None):
"""
POST /admin/log
"""
if level not in Logger.HUMAN_MESSAGES:
self.raise_exc(BadRequest, "INVALID_PARAMETER",
"The level provided is invalid")
level = Logger.HUMAN_MESSAGES.index(level)
try:
start_date = dateutil.parser.parse(start_date).timestamp()
end_date = dateutil.parser.parse(end_date).timestamp()
except ValueError as e:
BaseHandler.raise_exc(BadRequest, "INVALID_PARAMETER", str(e))
return BaseHandler.format_dates(
{"items": Logger.get_logs(level, category, start_date, end_date)})
