def _update_status(self) -> None:
updates = dict(
last_contact=func.now(),
processed=self._task_processed,
errors=self._task_errors,
)
try:
hostname = self._hostname or gethostname()
with transaction() as s:
if self._hostname:
s.query(WorkerTable).filter(
WorkerTable.hostname == self._hostname,
WorkerTable.pid == self._pid).update(
updates, synchronize_session=False)
else:
updates.update(
dict(hostname=hostname,
pid=self._pid,
max_processes=self._max_processes,
startup_time=func.now()))
s.add(WorkerTable(**updates))
if uniform(0, 1) <= 0.01 or not self._hostname:
threshold = self._maint_interval * 10
s.query(WorkerTable).filter(
func.now() -
WorkerTable.last_contact > threshold).delete(
synchronize_session=False)
self._hostname = hostname
except Exception:
pass
self._schedule_update_status()
def setUp(self):
from penguin_judge.main import _configure_app
app.reset()
_configure_app({})
tables = (JudgeResult, Submission, TestCase, Problem, Contest,
Environment, Token, User)
admin_token = bytes([i for i in range(32)])
salt = b'penguin'
passwd = _kdf('penguinpenguin', salt)
with transaction() as s:
for t in tables:
s.query(t).delete(synchronize_session=False)
s.add(
User(id='admin',
name='Administrator',
salt=salt,
admin=True,
password=passwd))
s.flush()
s.add(
Token(token=admin_token,
user_id='admin',
expires=datetime.now(tz=timezone.utc) +
timedelta(hours=1)))
self.admin_token = b64encode(admin_token).decode('ascii')
self.admin_headers = {'X-Auth-Token': self.admin_token}
def test_get_current_user(self):
uid, pw, name = 'penguin', 'password', 'ABC'
u = app.post_json(
'/users', {'login_id': uid, 'name': name, 'password': pw},
headers=self.admin_headers).json
token = app.post_json(
'/auth', {'login_id': uid, 'password': pw}).json['token']
self.assertEqual(u, app.get('/user').json)
app.reset()
app.authorization = ('Bearer', token)
self.assertEqual(u, app.get('/user').json)
app.authorization = None
self.assertEqual(u, app.get(
'/user', headers={'X-Auth-Token': token}).json)
app.get('/user', status=401)
app.get('/user', headers={
'X-Auth-Token': b64encode(b'invalid token').decode('ascii')
}, status=401)
app.get('/user', headers={'X-Auth-Token': b'Z'}, status=401)
with transaction() as s:
s.query(Token).filter(Token.user_id == u['id']).update({
'expires': datetime.now(tz=timezone.utc)})
app.get('/user', headers={'X-Auth-Token': token}, status=401)
def delete_problem(contest_id: str, problem_id: str) -> Response:
with transaction() as s:
_ = _validate_token(s, admin_required=True)
s.query(Problem).filter(
Problem.contest_id == contest_id,
Problem.id == problem_id).delete(synchronize_session=False)
return response204()
def rejudge(contest_id: str, problem_id: str) -> Response:
with transaction() as s:
_ = _validate_token(s, admin_required=True)
s.query(JudgeResult).filter(
JudgeResult.contest_id == contest_id,
JudgeResult.problem_id == problem_id).delete(
synchronize_session=False)
s.query(Submission).filter(
Submission.contest_id == contest_id,
Submission.problem_id == problem_id).update(
{
Submission.status: JudgeStatus.Waiting,
},
synchronize_session=False)
q = s.query(Submission.id).filter(Submission.contest_id == contest_id,
Submission.problem_id == problem_id)
rejudge_list = [x for x, in q]
conn = pika.BlockingConnection(get_mq_conn_params())
ch = conn.channel()
ch.queue_declare(queue='judge_queue')
for submission_id in rejudge_list:
ch.basic_publish(exchange='',
routing_key='judge_queue',
body=pickle.dumps(
(contest_id, problem_id, submission_id)))
ch.close()
conn.close()
return jsonify({})
def run(judge_class: Callable[[], JudgeDriver],
task: JudgeTask) -> JudgeStatus:
LOGGER.info('judge start (contest_id: {}, problem_id: {}, '
'submission_id: {}, user_id: {}'.format(
task.contest_id, task.problem_id, task.id, task.user_id))
zctx = ZstdDecompressor()
try:
task.code = zctx.decompress(task.code)
for test in task.tests:
test.input = zctx.decompress(test.input)
test.output = zctx.decompress(test.output)
except Exception:
LOGGER.warning('decompress failed', exc_info=True)
with transaction() as s:
return _update_submission_status(s, task,
JudgeStatus.InternalError)
with judge_class() as judge:
ret = _prepare(judge, task)
if ret:
return ret
if task.compile_image_name:
ret = _compile(judge, task)
if ret:
return ret
ret = _tests(judge, task)
LOGGER.info('judge finished (submission_id={}): {}'.format(task.id, ret))
return ret
def judge_test_cmpl(test: JudgeTestInfo, resp: Union[AgentTestResult,
AgentError]) -> None:
time: Optional[timedelta] = None
memory_kb: Optional[int] = None
if isinstance(resp, AgentTestResult):
if resp.time is not None:
time = timedelta(seconds=resp.time)
if resp.memory_bytes is not None:
memory_kb = resp.memory_bytes // 1024
if equal_binary(test.output, resp.output):
status = JudgeStatus.Accepted
else:
status = JudgeStatus.WrongAnswer
else:
status = JudgeStatus.from_str(resp.kind)
judge_results.append((status, time, memory_kb))
with transaction() as s:
s.query(JudgeResult).filter(
JudgeResult.contest_id == task.contest_id,
JudgeResult.problem_id == task.problem_id,
JudgeResult.submission_id == task.id,
JudgeResult.test_id == test.id,
).update(
{
JudgeResult.status: status,
JudgeResult.time: time,
JudgeResult.memory: memory_kb,
},
synchronize_session=False)
def list_contests() -> Response:
params, _ = _validate_request()
page, per_page = params.query['page'], params.query['per_page']
ret = []
with transaction() as s:
u = _validate_token(s)
q = s.query(Contest)
if not (u and u['admin']):
q = q.filter(Contest.published.is_(True))
if 'status' in params.query:
v = params.query['status']
now = datetime.now(tz=timezone.utc)
if v == 'running':
q = q.filter(Contest.start_time <= now, now < Contest.end_time)
elif v == 'scheduled':
q = q.filter(now < Contest.start_time)
elif v == 'finished':
q = q.filter(Contest.end_time <= now)
count = q.count()
q = q.order_by(Contest.start_time.desc())
for c in q.offset((page - 1) * per_page).limit(per_page):
ret.append(c.to_summary_dict())
return jsonify(ret, headers=pagination_header(count, page, per_page))
def delete_environment(environment_id: str) -> Response:
with transaction() as s:
_ = _validate_token(s, admin_required=True)
deleted = s.query(Environment).filter(
Environment.id == environment_id).delete(synchronize_session=False)
if not deleted:
abort(404)
return response204()
def list_tests(contest_id: str, problem_id: str) -> Response:
ret = []
with transaction() as s:
_ = _validate_token(s, admin_required=True)
q = s.query(TestCase.id).filter(TestCase.contest_id == contest_id,
TestCase.problem_id == problem_id)
for (test_case_id, ) in q:
ret.append(test_case_id)
return jsonify(ret)
def start_test_func(test_id: str) -> None:
with transaction() as s:
s.query(JudgeResult).filter(
JudgeResult.contest_id == task.contest_id,
JudgeResult.problem_id == task.problem_id,
JudgeResult.submission_id == task.id,
JudgeResult.test_id == test_id,
).update({JudgeResult.status: JudgeStatus.Running},
synchronize_session=False)
def _prepare(judge: JudgeDriver, task: JudgeTask) -> Union[JudgeStatus, None]:
try:
judge.prepare(task)
return None
except Exception:
LOGGER.warning('prepare failed', exc_info=True)
with transaction() as s:
return _update_submission_status(s, task,
JudgeStatus.InternalError)
def deleteToken() -> Response:
with transaction() as s:
u = _validate_token(s, required=True)
assert (u)
s.query(Token).filter(Token.token == u['_token_bytes']).delete(
synchronize_session=False)
resp = make_response((b'', 204))
resp.headers.pop(key='content-type')
resp.headers.add('Set-Cookie', 'AuthToken=; Max-Age=0')
return resp
def test_submissions_pagination(self):
test_data = []
start_time = datetime.now(tz=timezone.utc)
end_time = start_time + timedelta(hours=1)
app.post_json('/contests', {
'id': 'id0',
'title': 'Test Contest',
'description': '**Pagination** Test',
'start_time': start_time.isoformat(),
'end_time': end_time.isoformat(),
'published': True,
},
headers=self.admin_headers)
app.post_json('/contests/id0/problems', {
'id': 'A',
'title': 'Problem',
'description': '# A',
'time_limit': 2,
'score': 100
},
headers=self.admin_headers)
test_data = []
with transaction() as s:
env = Environment(name='Python 3.7', test_image_name='image')
s.add(env)
s.flush()
for i in range(100):
submission = Submission(contest_id='id0',
problem_id='A',
user_id='admin',
code=b'dummy',
code_bytes=1,
environment_id=env.id)
s.add(submission)
s.flush()
test_data.append(submission.to_dict())
resp = app.get('/contests/id0/submissions', headers=self.admin_headers)
self.assertEqual(len(resp.json), 20)
self.assertEqual(int(resp.headers['X-Page']), 1)
self.assertEqual(int(resp.headers['X-Per-Page']), 20)
self.assertEqual(int(resp.headers['X-Total']), 100)
self.assertEqual(int(resp.headers['X-Total-Pages']), 5)
resp = app.get('/contests/id0/submissions?page=2&per_page=31',
headers=self.admin_headers)
self.assertEqual(len(resp.json), 31)
self.assertEqual([x['id'] for x in resp.json],
[x['id'] for x in test_data[31:62]])
self.assertEqual(int(resp.headers['X-Page']), 2)
self.assertEqual(int(resp.headers['X-Per-Page']), 31)
self.assertEqual(int(resp.headers['X-Total']), 100)
self.assertEqual(int(resp.headers['X-Total-Pages']), 4)
def list_environments() -> Response:
ret = []
with transaction() as s:
u = _validate_token(s)
is_admin = u and u['admin']
q = s.query(Environment)
if not is_admin:
q = q.filter(Environment.published.is_(True))
for c in q:
ret.append(c.to_dict() if is_admin else c.to_summary_dict())
return jsonify(ret)
def get_user(user_id: str) -> Response:
try:
user_id_int = int(user_id)
except Exception:
abort(400)
with transaction() as s:
_validate_token_if_auth_required_config_is_enabled(s)
user = s.query(User).filter(User.id == user_id_int).first()
if not user:
abort(404)
return jsonify(user.to_summary_dict())
def list_submissions(contest_id: str) -> Response:
params, body = _validate_request()
page, per_page = params.query['page'], params.query['per_page']
ret = []
with transaction() as s:
u = _validate_token(s)
contest = s.query(Contest).filter(Contest.id == contest_id).first()
if not (contest and contest.is_accessible(u)):
abort(404)
is_admin = (u and u['admin'])
if not (contest.is_begun() or is_admin):
abort(403)
q = s.query(Submission,
User.name).filter(Submission.contest_id == contest_id,
Submission.user_id == User.id)
if not (contest.is_finished() or is_admin):
if not u:
# 未ログイン時は開催中コンテストの投稿一覧は見えない
abort(403)
q = q.filter(Submission.user_id == u['id'])
filters = [
('problem_id', Submission.problem_id.__eq__),
('environment_id', Submission.environment_id.__eq__),
('status', Submission.status.__eq__),
]
for key, expr in filters:
v = params.query.get(key)
if v is None:
continue
q = q.filter(expr(v)) # type: ignore
if params.query.get('user_name'):
q = q.filter(User.name.contains(params.query.get('user_name')))
count = q.count()
if params.query.get('sort'):
sort_keys = []
for key in params.query['sort']:
f = getattr(Submission, key.lstrip('-'))
if key[0] == '-':
f = f.desc()
sort_keys.append(f)
q = q.order_by(*sort_keys)
else:
q = q.order_by(Submission.created)
for c, name in q.offset((page - 1) * per_page).limit(per_page):
tmp = c.to_summary_dict()
tmp['user_name'] = name
ret.append(tmp)
return jsonify(ret, headers=pagination_header(count, page, per_page))
def test_list_environments(self):
envs = app.get('/environments').json
self.assertEqual(envs, [])
env = dict(name='Python 3.7', test_image_name='docker-image',
published=True)
with transaction() as s:
s.add(Environment(**env))
envs = app.get('/environments').json
self.assertEqual(len(envs), 1)
self.assertIsInstance(envs[0]['id'], int)
self.assertEqual(envs[0]['name'], env['name'])
def register_environment() -> Response:
_, body = _validate_request()
with transaction() as s:
_ = _validate_token(s, admin_required=True)
e = Environment(name=body.name,
active=getattr(body, 'active', True),
published=getattr(body, 'published', False),
compile_image_name=getattr(body, 'compile_image_name',
''),
test_image_name=body.test_image_name)
s.add(e)
s.flush()
return jsonify(e.to_dict())
def get_contest(contest_id: str) -> Response:
with transaction() as s:
u = _validate_token(s)
contest = s.query(Contest).filter(Contest.id == contest_id).first()
if not (contest and contest.is_accessible(u)):
abort(404)
ret = contest.to_dict()
if contest.is_begun() or (u and u['admin']):
problems = s.query(Problem).filter(
Problem.contest_id == contest_id).order_by(Problem.id).all()
if problems:
ret['problems'] = [p.to_dict() for p in problems]
return jsonify(ret)
def list_problems(contest_id: str) -> Response:
with transaction() as s:
u = _validate_token(s)
contest = s.query(Contest).filter(Contest.id == contest_id).first()
if not (contest and contest.is_accessible(u)):
abort(404)
if not (contest.is_begun() or (u and u['admin'])):
abort(403)
ret = [
p.to_summary_dict() for p in s.query(Problem).filter(
Problem.contest_id == contest_id).order_by(Problem.id).all()
]
return jsonify(ret)
def get_problem(contest_id: str, problem_id: str) -> Response:
with transaction() as s:
u = _validate_token(s)
contest = s.query(Contest).filter(Contest.id == contest_id).first()
if not (contest and contest.is_accessible(u)):
abort(404)
if not (contest.is_begun() or (u and u['admin'])):
abort(404) # ここは403ではなく404にする
ret = s.query(Problem).filter(Problem.contest_id == contest_id,
Problem.id == problem_id).first()
if not ret:
abort(404)
ret = ret.to_dict()
return jsonify(ret)
def update_environment(environment_id: str) -> Response:
_, body = _validate_request()
with transaction() as s:
_ = _validate_token(s, admin_required=True)
c = s.query(Environment).filter(
Environment.id == environment_id).first()
if not c:
abort(404)
for key in Environment.__updatable_keys__:
if not hasattr(body, key):
continue
setattr(c, key, getattr(body, key))
ret = c.to_dict()
return jsonify(ret)
def update_problem(contest_id: str, problem_id: str) -> Response:
_, body = _validate_request()
with transaction() as s:
_ = _validate_token(s, admin_required=True)
problem = s.query(Problem).filter(Problem.contest_id == contest_id,
Problem.id == problem_id).first()
if not problem:
abort(404)
for key in Problem.__updatable_keys__:
if not hasattr(body, key):
continue
setattr(problem, key, getattr(body, key))
ret = problem.to_dict()
return jsonify(ret)
def update_contest(contest_id: str) -> Response:
_, body = _validate_request()
with transaction() as s:
_ = _validate_token(s, admin_required=True)
c = s.query(Contest).filter(Contest.id == contest_id).first()
if not c:
abort(404)
for key in Contest.__updatable_keys__:
if not hasattr(body, key):
continue
setattr(c, key, getattr(body, key))
if c.start_time >= c.end_time:
abort(400, {'detail': 'start_time must be lesser than end_time'})
ret = c.to_dict()
return jsonify(ret)
def _validate_token(
s: Optional[scoped_session] = None,
required: bool = False,
admin_required: bool = False,
ignore_auth_required_config: bool = False) -> Optional[dict]:
# コンフィグの認証必須オプションを無視する指示がされていない場合は
# 認証必須オプションによってrequiredの値を上書きする
if not ignore_auth_required_config and _config_auth_required():
required = True
token = request.headers.get('X-Auth-Token')
if not token:
items = request.headers.get('Authorization', '').split(' ', maxsplit=1)
if len(items) == 2 and items[0].lower() == 'bearer':
token = items[1]
if not token:
token = request.cookies.get('AuthToken')
if not token:
if required or admin_required:
abort(401)
return None
try:
token_bytes = b64decode(token)
except Exception:
abort(401)
utc_now = datetime.now(tz=timezone.utc)
def _check(s: scoped_session) -> Optional[dict]:
ret = s.query(Token.expires,
User).filter(Token.token == token_bytes,
Token.user_id == User.id).first()
if not ret or ret[0] <= utc_now:
if required or admin_required:
abort(401)
else:
return None
if admin_required and not ret[1].admin:
abort(401)
tmp = ret[1].to_summary_dict()
tmp['_token_bytes'] = token_bytes
tmp['login_id'] = ret[1].login_id
return tmp
if s:
return _check(s)
with transaction() as s:
return _check(s)
def create_contest() -> Response:
_, body = _validate_request()
if body.start_time >= body.end_time:
abort(400, {'detail': 'start_time must be lesser than end_time'})
with transaction() as s:
_ = _validate_token(s, admin_required=True)
contest = Contest(id=body.id,
title=body.title,
description=body.description,
start_time=body.start_time,
end_time=body.end_time,
published=getattr(body, 'published', None))
s.add(contest)
s.flush()
ret = contest.to_dict()
return jsonify(ret)
def _get_test_data(contest_id: str, problem_id: str, test_id: str,
is_input: bool) -> Response:
zctx = ZstdDecompressor()
from io import BytesIO
with transaction() as s:
_ = _validate_token(s, admin_required=True)
tc = s.query(TestCase).filter(TestCase.contest_id == contest_id,
TestCase.problem_id == problem_id,
TestCase.id == test_id).first()
if not tc:
abort(404)
f = BytesIO(zctx.decompress(tc.input if is_input else tc.output))
return send_file(f,
as_attachment=True,
attachment_filename='{}.{}'.format(
test_id, 'in' if is_input else 'out'))
def get_status() -> Response:
ret = {}
with transaction() as s:
_ = _validate_token(s, admin_required=True)
ret['workers'] = [
w.to_dict() for w in s.query(Worker).filter(
func.now() - Worker.last_contact < timedelta(
seconds=60 * 10), ).order_by(Worker.startup_time)
]
conn = pika.BlockingConnection(get_mq_conn_params())
ch = conn.channel()
queue = ch.queue_declare(queue='judge_queue')
ret['queued'] = queue.method.message_count
ch.close()
conn.close()
return jsonify(ret)
def post_submission(contest_id: str) -> Response:
_, body = _validate_request()
problem_id, code, env_id = body.problem_id, body.code, body.environment_id
cctx = ZstdCompressor()
code_encoded = code.encode('utf8')
code = cctx.compress(code_encoded)
with transaction() as s:
u = _validate_token(s, required=True)
assert (u)
if not s.query(Environment).filter(Environment.id == env_id).count():
abort(400) # bodyが不正なので400
if not s.query(Contest).filter(Contest.id == contest_id).count():
abort(404) # contest_idはURLに含まれるため404
if not s.query(Problem).filter(Problem.contest_id == contest_id,
Problem.id == problem_id).count():
abort(400) # bodyが不正なので400
queued_submission_count = s.query(Submission).filter(
Submission.user_id == u['id'],
Submission.status.in_([JudgeStatus.Waiting,
JudgeStatus.Running])).count()
if queued_submission_count > app.config['user_judge_queue_limit']:
abort(429)
submission = Submission(contest_id=contest_id,
problem_id=problem_id,
user_id=u['id'],
code=code,
code_bytes=len(code_encoded),
environment_id=env_id)
s.add(submission)
s.flush()
ret = submission.to_summary_dict()
ret['user_name'] = u['name']
conn = pika.BlockingConnection(get_mq_conn_params())
ch = conn.channel()
ch.queue_declare(queue='judge_queue')
ch.basic_publish(exchange='',
routing_key='judge_queue',
body=pickle.dumps((contest_id, problem_id, ret['id'])))
ch.close()
conn.close()
return jsonify(ret, status=201)
