def __init__(self, uuid, hw, lang):
#: A :class:`~railgun.common.tempdir.TempDir`, whose directory name
#: is `uuid`.
self.tempdir = TempDir(uuid)
#: The uuid of the submission.
self.uuid = uuid
#: The :class:`~railgun.common.hw.Homework`.
self.hw = hw
#: The :class:`~railgun.common.hw.HwCode` of corresponding language.
self.hwcode = hw.get_code(lang)
#: The xml node of compiler parameters.
#: You may refer to :attr:`HwCode.compiler_params` for more details.
self.compiler_params = self.hwcode.compiler_params
#: The xml node of runner parameters.
#: You may refer to :attr:`HwCode.runner_params` for more details.
self.runner_params = self.hwcode.runner_params
#: The :class:`HostConfig` for the process.
self.config = HostConfig(handid=uuid, hwid=self.hw.uuid)
#: The acquired system account (name or uid).
#:
#: Railgun can be configured to run multiple submissions in
#: different system accounts simultaneously. This attribute
#: stores the acquired system account, so that we can release
#: it later.
#:
#: If you intend to get the `uid` and `gid` of this user,
#: access ``config['user_id']`` and ``config['group_id']``
#: instead.
self.runner_user = None
def __init__(self, uuid, hw, lang):
#: A :class:`~railgun.common.tempdir.TempDir`, whose directory name
#: is `uuid`.
self.tempdir = TempDir(uuid)
#: The uuid of the submission.
self.uuid = uuid
#: The :class:`~railgun.common.hw.Homework`.
self.hw = hw
#: The :class:`~railgun.common.hw.HwCode` of corresponding language.
self.hwcode = hw.get_code(lang)
#: The xml node of compiler parameters.
#: You may refer to :attr:`HwCode.compiler_params` for more details.
self.compiler_params = self.hwcode.compiler_params
#: The xml node of runner parameters.
#: You may refer to :attr:`HwCode.runner_params` for more details.
self.runner_params = self.hwcode.runner_params
#: The :class:`HostConfig` for the process.
self.config = HostConfig(handid=uuid, hwid=self.hw.uuid)
#: The acquired system account (name or uid).
#:
#: Railgun can be configured to run multiple submissions in
#: different system accounts simultaneously. This attribute
#: stores the acquired system account, so that we can release
#: it later.
#:
#: If you intend to get the `uid` and `gid` of this user,
#: access ``config['user_id']`` and ``config['group_id']``
#: instead.
self.runner_user = None
class BaseHost(object):
"""The base interface for a runner host.
A runner host will hold a working directory under ``config.TEMPORARY_DIR``,
whose name is ``uuid``. This directory will be automatically removed
if :class:`BaseHost` is managed by ``with`` statement, for example::
with BaseHost(uuid, hw, 'python') as host:
pass
:param uuid: The uuid of the submission.
:type uuid: :class:`str`
:param hw: The corresponding homework object.
:type hw: :class:`~railgun.common.hw.Homework`
:param lang: The programming language of this host.
:type lang: :class:`str`
"""
def __init__(self, uuid, hw, lang):
#: A :class:`~railgun.common.tempdir.TempDir`, whose directory name
#: is `uuid`.
self.tempdir = TempDir(uuid)
#: The uuid of the submission.
self.uuid = uuid
#: The :class:`~railgun.common.hw.Homework`.
self.hw = hw
#: The :class:`~railgun.common.hw.HwCode` of corresponding language.
self.hwcode = hw.get_code(lang)
#: The xml node of compiler parameters.
#: You may refer to :attr:`HwCode.compiler_params` for more details.
self.compiler_params = self.hwcode.compiler_params
#: The xml node of runner parameters.
#: You may refer to :attr:`HwCode.runner_params` for more details.
self.runner_params = self.hwcode.runner_params
#: The :class:`HostConfig` for the process.
self.config = HostConfig(handid=uuid, hwid=self.hw.uuid)
#: The acquired system account (name or uid).
#:
#: Railgun can be configured to run multiple submissions in
#: different system accounts simultaneously. This attribute
#: stores the acquired system account, so that we can release
#: it later.
#:
#: If you intend to get the `uid` and `gid` of this user,
#: access ``config['user_id']`` and ``config['group_id']``
#: instead.
self.runner_user = None
def __enter__(self):
#: We create the directory with mode 0777, while the owner is the owner
#: of runner queue process.
#: The permissions and the owner will be set to `config['user_id']`
#: until :meth:`spawn`.
self.tempdir.open(mode=0777)
return self
def __exit__(self, ignore1, ignore2, ignore3):
#: The temporary directory will be removed here.
self.tempdir.close()
def spawn(self, cmdline, timeout=None):
"""Spawn an external process to execute the given commands.
If the owner user of current process (runner queue) is `root`,
and ``config['user_id']`` != 0, the owner of :attr:`tempdir`
will be changed to that user, and the file system mode will
be changed to 0700.
:param cmdline: The command line to be executed.
:type cmdline: :class:`str`
:param timeout: Wait for `timeout` seconds before we kill the
external process and claim a rejected submission.
If this argument is not given, ``config.RUNNER_DEFAULT_TIMEOUT``
will be chosen as the timeout limit.
:type timeout: :class:`float`
"""
try:
# Before spawn the process, we've already known the process
# user. And we'll try to chown & chmod if our runner queue
# runs at root privilege (otherwise we cannot change the
# owner user).
if os.getuid() == 0:
# If config['user_id'] is 0, runner_user must be None,
# where we shouldn't go any more.
if self.config['user_id'] != 0:
self.tempdir.chown(
self.config['user_id'],
self.config['group_id'],
True
)
self.tempdir.chmod(0700, True)
# Now we can execute the host process safely!
#print "dir : " + str(self.tempdir.path)
#print "env : " + str(self.config.make_environ())
return execute(
cmdline,
timeout or runconfig.RUNNER_DEFAULT_TIMEOUT,
cwd=self.tempdir.path,
env=self.config.make_environ(),
close_fds=True
)
except ProcessTimeout:
raise RunnerTimeout()
except Exception:
logger.exception(
'Error when executing submission %(handid)s of homework '
'%(hwid)s.' %
{'hwid': self.hw.uuid, 'handid': self.uuid}
)
raise SpawnProcessFailure()
def set_user(self, uid, gid=None):
"""Set the user and the group in host config.
The `uid` will be stored in ``config['user_id']`` while the `gid`
will be stored in ``config['group_id']``. However, if `uid` is
given :data:`None`, all the defence based on user privileges will
not take place.
:param uid: The uid or name of system account.
:type uid: :class:`int` or :class:`str`
:param gid: The gid or group name in the system.
If set to :data:`None`, the group of given user will be
selected.
:type gid: :class:`int` or :class:`str`
:raises: :class:`KeyError` if `uid` or `gid` is a :class:`str`,
but does not exist in the system database.
"""
# If uid is None, set config['user_id'] and config['group_id'] to 0
if not uid:
self.config['user_id'] = 0
self.config['group_id'] = 0
return
# Store user name for later release
self.runner_user = uid
# NOTE: pwd.getpwnam and grp.getgrname does throw KeyError
# if given name not exist.
if not isinstance(uid, int):
uid = pwd.getpwnam(uid).pw_uid
if gid is None:
gid = pwd.getpwuid(uid).pw_gid
elif not isinstance(gid, int):
gid = grp.getgrnam(gid).gr_gid
self.config['user_id'] = uid
self.config['group_id'] = gid
def compile(self):
"""Call to compile the submission. Some programming language may
skip this process.
"""
pass
def run(self):
"""Run this submission. All derived classes must implement this.
:return: A :class:`tuple` of (exitcode, stdout, stderr).
"""
raise NotImplementedError()
def prepare_hwcode(self):
"""Prepare the runner context by copying files from `hw/code` into
:attr:`tempdir`. This method should be called before
:meth:`extract_handin`.
"""
try:
self.tempdir.copyfiles(
self.hwcode.path,
dirtree(self.hwcode.path),
mode=0777
)
except Exception:
logger.exception(
'Cannot copy code files into tempdir for homework %(hwid)s '
'when executing submission %(handid)s.' %
{'hwid': self.hw.uuid, 'handid': self.uuid}
)
raise RuntimeFileCopyFailure()
def extract_handin(self, archive):
"""Extract the given archive file into :attr:`tempdir`.
The :class:`~railgun.common.hw.FileRules` defined in :attr:`hw` and
:attr:`hwcode` will be validated on each file, so this method is
safe.
:param archive: An extractor of the submission archive file.
:type archive: :class:`~railgun.common.fileutil.Extractor`
"""
try:
# We limit the count of files in an archive file, since too many
# files may slow down the runner queue.
maxCount = runconfig.MAX_SUBMISSION_FILE_COUNT
if archive.countfiles(maxCount) > maxCount:
raise ArchiveContainTooManyFileError()
# If the archive file contains only one top-level directory,
# it is likely that all the code files are placed under it.
#
# So we remove the top-level directory and extract the files
# in it directly to :attr:`tempdir`.
onedir = archive.onedir()
canonical_path = remove_firstdir if onedir else (lambda s: s)
# Use the :class:`FileRules` to filter out unwanted files.
def should_skip(path):
path = canonical_path(path)
# First, check the rules in HwCode
action = self.hwcode.file_rules.get_action(
path, default_action=-1
)
if action == FileRules.DENY:
raise FileDenyError(path)
if action == FileRules.ACCEPT:
return False
if action != -1:
# action is not None, and action != ACCEPT, this means
# that rules in `hwcode` rejects this file.
return True
# Next, check the rules in Homework
action = self.hw.file_rules.get_action(
path, default_action=FileRules.LOCK
)
if action == FileRules.DENY:
raise FileDenyError(path)
return (action != FileRules.ACCEPT)
# Call utility to do the extraction. Initial file mode is 0777,
# and we'll correct this problem in :meth:`spawn`
self.tempdir.extract(archive, should_skip, mode=0777)
except RunnerError:
raise
except Exception:
logger.exception(
'Cannot extract archive into tempdir for homework %(hwid)s '
'when executing submission %(handid)s.' %
{'hwid': self.hw.uuid, 'handid': self.uuid}
)
raise ExtractFileFailure()
class BaseHost(object):
"""The base interface for a runner host.
A runner host will hold a working directory under ``config.TEMPORARY_DIR``,
whose name is ``uuid``. This directory will be automatically removed
if :class:`BaseHost` is managed by ``with`` statement, for example::
with BaseHost(uuid, hw, 'python') as host:
pass
:param uuid: The uuid of the submission.
:type uuid: :class:`str`
:param hw: The corresponding homework object.
:type hw: :class:`~railgun.common.hw.Homework`
:param lang: The programming language of this host.
:type lang: :class:`str`
"""
def __init__(self, uuid, hw, lang):
#: A :class:`~railgun.common.tempdir.TempDir`, whose directory name
#: is `uuid`.
self.tempdir = TempDir(uuid)
#: The uuid of the submission.
self.uuid = uuid
#: The :class:`~railgun.common.hw.Homework`.
self.hw = hw
#: The :class:`~railgun.common.hw.HwCode` of corresponding language.
self.hwcode = hw.get_code(lang)
#: The xml node of compiler parameters.
#: You may refer to :attr:`HwCode.compiler_params` for more details.
self.compiler_params = self.hwcode.compiler_params
#: The xml node of runner parameters.
#: You may refer to :attr:`HwCode.runner_params` for more details.
self.runner_params = self.hwcode.runner_params
#: The :class:`HostConfig` for the process.
self.config = HostConfig(handid=uuid, hwid=self.hw.uuid)
#: The acquired system account (name or uid).
#:
#: Railgun can be configured to run multiple submissions in
#: different system accounts simultaneously. This attribute
#: stores the acquired system account, so that we can release
#: it later.
#:
#: If you intend to get the `uid` and `gid` of this user,
#: access ``config['user_id']`` and ``config['group_id']``
#: instead.
self.runner_user = None
def __enter__(self):
#: We create the directory with mode 0777, while the owner is the owner
#: of runner queue process.
#: The permissions and the owner will be set to `config['user_id']`
#: until :meth:`spawn`.
self.tempdir.open(mode=0777)
return self
def __exit__(self, ignore1, ignore2, ignore3):
#: The temporary directory will be removed here.
self.tempdir.close()
def spawn(self, cmdline, timeout=None):
"""Spawn an external process to execute the given commands.
If the owner user of current process (runner queue) is `root`,
and ``config['user_id']`` != 0, the owner of :attr:`tempdir`
will be changed to that user, and the file system mode will
be changed to 0700.
:param cmdline: The command line to be executed.
:type cmdline: :class:`str`
:param timeout: Wait for `timeout` seconds before we kill the
external process and claim a rejected submission.
If this argument is not given, ``config.RUNNER_DEFAULT_TIMEOUT``
will be chosen as the timeout limit.
:type timeout: :class:`float`
"""
try:
# Before spawn the process, we've already known the process
# user. And we'll try to chown & chmod if our runner queue
# runs at root privilege (otherwise we cannot change the
# owner user).
if os.getuid() == 0:
# If config['user_id'] is 0, runner_user must be None,
# where we shouldn't go any more.
if self.config['user_id'] != 0:
self.tempdir.chown(self.config['user_id'],
self.config['group_id'], True)
self.tempdir.chmod(0700, True)
# Now we can execute the host process safely!
return execute(cmdline,
timeout or runconfig.RUNNER_DEFAULT_TIMEOUT,
cwd=self.tempdir.path,
env=self.config.make_environ(),
close_fds=True)
except ProcessTimeout:
raise RunnerTimeout()
except Exception:
logger.exception(
'Error when executing submission %(handid)s of homework '
'%(hwid)s.' % {
'hwid': self.hw.uuid,
'handid': self.uuid
})
raise SpawnProcessFailure()
def set_user(self, uid, gid=None):
"""Set the user and the group in host config.
The `uid` will be stored in ``config['user_id']`` while the `gid`
will be stored in ``config['group_id']``. However, if `uid` is
given :data:`None`, all the defence based on user privileges will
not take place.
:param uid: The uid or name of system account.
:type uid: :class:`int` or :class:`str`
:param gid: The gid or group name in the system.
If set to :data:`None`, the group of given user will be
selected.
:type gid: :class:`int` or :class:`str`
:raises: :class:`KeyError` if `uid` or `gid` is a :class:`str`,
but does not exist in the system database.
"""
# If uid is None, set config['user_id'] and config['group_id'] to 0
if not uid:
self.config['user_id'] = 0
self.config['group_id'] = 0
return
# Store user name for later release
self.runner_user = uid
# NOTE: pwd.getpwnam and grp.getgrname does throw KeyError
# if given name not exist.
if not isinstance(uid, int):
uid = pwd.getpwnam(uid).pw_uid
if gid is None:
gid = pwd.getpwuid(uid).pw_gid
elif not isinstance(gid, int):
gid = grp.getgrnam(gid).gr_gid
self.config['user_id'] = uid
self.config['group_id'] = gid
def compile(self):
"""Call to compile the submission. Some programming language may
skip this process.
"""
pass
def run(self):
"""Run this submission. All derived classes must implement this.
:return: A :class:`tuple` of (exitcode, stdout, stderr).
"""
raise NotImplementedError()
def prepare_hwcode(self):
"""Prepare the runner context by copying files from `hw/code` into
:attr:`tempdir`. This method should be called before
:meth:`extract_handin`.
"""
try:
self.tempdir.copyfiles(self.hwcode.path,
dirtree(self.hwcode.path),
mode=0777)
except Exception:
logger.exception(
'Cannot copy code files into tempdir for homework %(hwid)s '
'when executing submission %(handid)s.' % {
'hwid': self.hw.uuid,
'handid': self.uuid
})
raise RuntimeFileCopyFailure()
def extract_handin(self, archive):
"""Extract the given archive file into :attr:`tempdir`.
The :class:`~railgun.common.hw.FileRules` defined in :attr:`hw` and
:attr:`hwcode` will be validated on each file, so this method is
safe.
:param archive: An extractor of the submission archive file.
:type archive: :class:`~railgun.common.fileutil.Extractor`
"""
try:
# We limit the count of files in an archive file, since too many
# files may slow down the runner queue.
maxCount = runconfig.MAX_SUBMISSION_FILE_COUNT
if archive.countfiles(maxCount) > maxCount:
raise ArchiveContainTooManyFileError()
# If the archive file contains only one top-level directory,
# it is likely that all the code files are placed under it.
#
# So we remove the top-level directory and extract the files
# in it directly to :attr:`tempdir`.
onedir = archive.onedir()
canonical_path = remove_firstdir if onedir else (lambda s: s)
# Use the :class:`FileRules` to filter out unwanted files.
def should_skip(path):
path = canonical_path(path)
# First, check the rules in HwCode
action = self.hwcode.file_rules.get_action(path,
default_action=-1)
if action == FileRules.DENY:
raise FileDenyError(path)
if action == FileRules.ACCEPT:
return False
if action != -1:
# action is not None, and action != ACCEPT, this means
# that rules in `hwcode` rejects this file.
return True
# Next, check the rules in Homework
action = self.hw.file_rules.get_action(
path, default_action=FileRules.LOCK)
if action == FileRules.DENY:
raise FileDenyError(path)
return (action != FileRules.ACCEPT)
# Call utility to do the extraction. Initial file mode is 0777,
# and we'll correct this problem in :meth:`spawn`
self.tempdir.extract(archive, should_skip, mode=0777)
except RunnerError:
raise
except Exception:
logger.exception(
'Cannot extract archive into tempdir for homework %(hwid)s '
'when executing submission %(handid)s.' % {
'hwid': self.hw.uuid,
'handid': self.uuid
})
raise ExtractFileFailure()