def setUp(self):
if TEST_ENV is None:
raise SkipTest("Testing the Docker Job Manager is disabled.")
elif TEST_ENV in ["boot2docker", "boot2docker-local"]:
self.docker_connection = docker.Client(
base_url="tcp://192.168.59.103:2375")
elif TEST_ENV == "jenkins":
self.docker_connection = docker.Client(
base_url="tcp://localhost:2375")
else:
raise Exception(
"Unknown method for testing the Docker Job Manager!")
# Force the removal of all containers/images linked to this test
try:
self.docker_connection.remove_container("inginious-agent",
force=True)
except:
pass
try:
self.docker_connection.remove_image("ingi/inginious-agent",
force=True)
except:
pass
self.course_factory, self.task_factory = create_factories("./tasks")
self.setUpDocker()
self.job_manager = None
self.setUpJobManager()
def __init__(self, context, backend_addr, friendly_name, concurrency,
tasks_filesystem):
"""
:param context: ZeroMQ context for this process
:param backend_addr: address of the backend (for example, "tcp://127.0.0.1:2222")
:param friendly_name: a string containing a friendly name to identify agent
:param tasks_filesystem: FileSystemProvider to the course/tasks
"""
super().__init__(context, backend_addr, friendly_name, concurrency,
tasks_filesystem)
self._logger = logging.getLogger("inginious.agent.mcq")
# Create a course factory
course_factory, _ = create_factories(tasks_filesystem)
self.course_factory = course_factory
# Init gettext
languages = ["en", "fr"]
self._translations = {
lang:
gettext.translation('messages',
get_root_path() + '/agent/mcq_agent/i18n',
[lang])
for lang in languages
}
def setUp(self):
if TEST_ENV is None:
raise SkipTest("Testing the Docker Job Manager is disabled.")
elif TEST_ENV in ["boot2docker", "boot2docker-local"]:
self.docker_connection = docker.Client(base_url="tcp://192.168.59.103:2375")
elif TEST_ENV == "jenkins":
self.docker_connection = docker.Client(base_url="tcp://localhost:2375")
else:
raise Exception("Unknown method for testing the Docker Job Manager!")
# Force the removal of all containers/images linked to this test
try:
self.docker_connection.remove_container("inginious-agent", force=True)
except:
pass
try:
self.docker_connection.remove_image("ingi/inginious-agent", force=True)
except:
pass
self.course_factory, self.task_factory = create_factories("./tasks")
self.setUpDocker()
self.job_manager = None
self.setUpJobManager()
def __init__(self, context, backend_addr, friendly_name, concurrency,
tasks_filesystem):
"""
:param context: ZeroMQ context for this process
:param backend_addr: address of the backend (for example, "tcp://127.0.0.1:2222")
:param friendly_name: a string containing a friendly name to identify agent
:param tasks_filesystem: FileSystemProvider to the course/tasks
"""
super().__init__(context, backend_addr, friendly_name, concurrency,
tasks_filesystem)
self._logger = logging.getLogger("inginious.agent.mcq")
# Create a course factory
course_factory, _ = create_factories(tasks_filesystem)
self.course_factory = course_factory
def __init__(self, context, backend_addr, task_directory):
"""
:param context: ZeroMQ context for this process
:param backend_addr: address of the backend (for example, "tcp://127.0.0.1:2222")
:param course_factory: course factory
:param task_factory: task factory
"""
self._logger = logging.getLogger("inginious.agent.mcq")
self._backend_addr = backend_addr
self._context = context
self._loop = asyncio.get_event_loop()
# Create a course factory
course_factory, _ = create_factories(task_directory)
self.course_factory = course_factory
# Sockets
self._backend_socket = self._context.socket(zmq.DEALER)
def get_app(config):
""" Init the webdav app """
mongo_client = MongoClient(host=config.get('mongo_opt', {}).get('host', 'localhost'))
database = mongo_client[config.get('mongo_opt', {}).get('database', 'INGInious')]
# Create the FS provider
if "tasks_directory" not in config:
raise RuntimeError("WebDav access is only supported if INGInious is using a local filesystem to access tasks")
fs_provider = LocalFSProvider(config["tasks_directory"])
course_factory, task_factory = create_factories(fs_provider, {}, None, WebAppCourse, WebAppTask)
user_manager = UserManager(MongoStore(database, 'sessions'), database, config.get('superadmins', []))
config = dict(wsgidav_app.DEFAULT_CONFIG)
config["provider_mapping"] = {"/": INGIniousFilesystemProvider(course_factory, task_factory)}
config["domaincontroller"] = INGIniousDAVDomainController(user_manager, course_factory)
config["verbose"] = 0
app = wsgidav_app.WsgiDAVApp(config)
return app
def get_app(config):
""" Init the webdav app """
mongo_client = MongoClient(host=config.get('mongo_opt', {}).get('host', 'localhost'))
database = mongo_client[config.get('mongo_opt', {}).get('database', 'INGInious')]
# Create the FS provider
if "tasks_directory" not in config:
raise RuntimeError("WebDav access is only supported if INGInious is using a local filesystem to access tasks")
fs_provider = LocalFSProvider(config["tasks_directory"])
course_factory, task_factory = create_factories(fs_provider, {}, None, WebAppCourse, WebAppTask)
user_manager = UserManager(MongoStore(database, 'sessions'), database, config.get('superadmins', []))
config = dict(wsgidav_app.DEFAULT_CONFIG)
config["provider_mapping"] = {"/": INGIniousFilesystemProvider(course_factory, task_factory)}
config["domaincontroller"] = INGIniousDAVDomainController(user_manager, course_factory)
config["verbose"] = 0
app = wsgidav_app.WsgiDAVApp(config)
return app
def __init__(self, context, backend_addr, friendly_name, task_directory):
"""
:param context: ZeroMQ context for this process
:param backend_addr: address of the backend (for example, "tcp://127.0.0.1:2222")
:param friendly_name: a string containing a friendly name to identify agent
:param task_directory: path to the task directory
"""
self._logger = logging.getLogger("inginious.agent.mcq")
self._backend_addr = backend_addr
self._context = context
self._loop = asyncio.get_event_loop()
self._friendly_name = friendly_name
# Create a course factory
course_factory, _ = create_factories(task_directory)
self.course_factory = course_factory
# Sockets
self._backend_socket = self._context.socket(zmq.DEALER)
self._backend_socket.ipv6 = True
def get_app(config, active_callback=None):
"""
:param config: the configuration dict
:param active_callback: a callback without arguments that will be called when the app is fully initialized
:return: A new app
"""
config = _put_configuration_defaults(config)
task_directory = config["tasks_directory"]
default_allowed_file_extensions = config['allowed_file_extensions']
default_max_file_size = config['max_file_size']
appli = web.application((), globals(), autoreload=False)
# Init the different parts of the app
plugin_manager = PluginManager()
mongo_client = MongoClient(
host=config.get('mongo_opt', {}).get('host', 'localhost'))
database = mongo_client[config.get('mongo_opt',
{}).get('database', 'INGInious')]
gridfs = GridFS(database)
course_factory, task_factory = create_factories(task_directory,
plugin_manager,
FrontendCourse,
FrontendTask)
user_manager = UserManager(
CustomSession(appli, MongoStore(database, 'sessions')), database)
backend_interface.update_pending_jobs(database)
job_manager = backend_interface.create_job_manager(config, plugin_manager,
task_directory,
course_factory,
task_factory)
lis_outcome_manager = LisOutcomeManager(database, user_manager,
course_factory, config["lti"])
submission_manager = LTISubmissionManager(
job_manager, user_manager, database, gridfs, plugin_manager,
config.get('nb_submissions_kept', 5), lis_outcome_manager)
template_helper = TemplateHelper(plugin_manager, 'frontend/lti/templates',
'layout',
config.get('use_minified_js', True))
# Update the database
update_database(database)
# Add some helpers for the templates
template_helper.add_to_template_globals("user_manager", user_manager)
template_helper.add_to_template_globals("default_allowed_file_extensions",
default_allowed_file_extensions)
template_helper.add_to_template_globals("default_max_file_size",
default_max_file_size)
# Not found page
appli.notfound = lambda: web.notfound(template_helper.get_renderer().
notfound('Page not found'))
# Init the mapping of the app
appli.init_mapping(
WebPyCustomMapping(dict(urls), plugin_manager, course_factory,
task_factory, submission_manager, user_manager,
template_helper, database, gridfs,
default_allowed_file_extensions,
default_max_file_size, config["containers"].keys(),
config["lti"]))
# Active hook
if active_callback is not None:
_handle_active_hook(job_manager, plugin_manager, active_callback)
# Loads plugins
plugin_manager.load(job_manager, appli, course_factory, task_factory,
database, user_manager, config.get("plugins", []))
# Start the inginious.backend
job_manager.start()
return appli, lambda: _close_app(appli, mongo_client, job_manager,
lis_outcome_manager)
def setUp(self):
self.dir_path = tempfile.mkdtemp()
fs = LocalFSProvider(self.dir_path)
self.course_factory, _ = create_factories(fs)
def get_app(hostname, port, sshhost, sshport, config, active_callback=None):
"""
:param hostname: the hostname on which the web app will be bound
:param port: the port on which the web app will be bound
:param sshport: the port on which remote container debugging clients will connect
:param config: the configuration dict
:param active_callback: a callback without arguments that will be called when the app is fully initialized
:return: A new app
"""
config = _put_configuration_defaults(config)
if config.get("maintenance", False):
appli = web.application(urls_maintenance, globals(), autoreload=False)
return appli
task_directory = config["tasks_directory"]
default_allowed_file_extensions = config['allowed_file_extensions']
default_max_file_size = config['max_file_size']
appli = web.application((), globals(), autoreload=False)
# Init the different parts of the app
plugin_manager = PluginManager()
mongo_client = MongoClient(host=config.get('mongo_opt', {}).get('host', 'localhost'))
database = mongo_client[config.get('mongo_opt', {}).get('database', 'INGInious')]
gridfs = GridFS(database)
course_factory, task_factory = create_factories(task_directory, plugin_manager, WebAppCourse, WebAppTask)
user_manager = UserManager(web.session.Session(appli, MongoStore(database, 'sessions')), database, config.get('superadmins', []))
backend_interface.update_pending_jobs(database)
job_manager = backend_interface.create_job_manager(config, plugin_manager,
task_directory, course_factory, task_factory)
remote_ssh_manager = RemoteSSHManager(sshhost, sshport, database, job_manager)
if config.get("remote_debugging_active", True) and job_manager.is_remote_debug_active():
if sshhost is None:
print "You have to set the --sshhost arg to start the remote debugging manager. Remote debugging is then deactivated"
else:
remote_ssh_manager.start()
submission_manager = WebAppSubmissionManager(job_manager, user_manager, database, gridfs, plugin_manager)
batch_manager = BatchManager(job_manager, database, gridfs, submission_manager, user_manager,
task_directory, config.get('batch_containers', []))
template_helper = TemplateHelper(plugin_manager, 'frontend/webapp/templates', 'layout', config.get('use_minified_js', True))
# Init web mail
smtp_conf = config.get('smtp', None)
if smtp_conf is not None:
web.config.smtp_server = smtp_conf["host"]
web.config.smtp_port = int(smtp_conf["port"])
web.config.smtp_starttls = bool(smtp_conf.get("starttls", False))
web.config.smtp_username = smtp_conf.get("username", "")
web.config.smtp_password = smtp_conf.get("password", "")
web.config.smtp_sendername = smtp_conf.get("sendername", "*****@*****.**")
# Update the database
update_database(database, gridfs, course_factory, user_manager)
# Add some helpers for the templates
template_helper.add_to_template_globals("user_manager", user_manager)
template_helper.add_to_template_globals("default_allowed_file_extensions", default_allowed_file_extensions)
template_helper.add_to_template_globals("default_max_file_size", default_max_file_size)
template_helper.add_other("course_admin_menu",
lambda course, current: course_admin_utils.get_menu(course, current, template_helper.get_renderer(False),
plugin_manager, user_manager))
# Not found page
appli.notfound = lambda: web.notfound(template_helper.get_renderer().notfound('Page not found'))
# Init the mapping of the app
appli.init_mapping(WebPyCustomMapping(dict(urls), plugin_manager,
course_factory, task_factory,
submission_manager, batch_manager, user_manager,
remote_ssh_manager, template_helper, database, gridfs,
default_allowed_file_extensions, default_max_file_size,
config.get("backup_directory", './backup'), config["containers"].keys()))
# Active hook
if active_callback is not None:
_handle_active_hook(job_manager, plugin_manager, active_callback)
# Loads plugins
plugin_manager.load(job_manager, appli, course_factory, task_factory, database, user_manager, config.get("plugins", []))
# Start the inginious.backend
job_manager.start()
return appli, lambda: _close_app(appli, mongo_client, job_manager, remote_ssh_manager)
def setUp(self):
self.course_factory, _ = create_factories(os.path.join(os.path.dirname(__file__), 'tasks'))
def setUp(self):
self.course_factory, self.task_factory = create_factories(
os.path.join(os.path.dirname(__file__), 'tasks'))
self.setUp_job_manager()
self.callback_done = threading.Event()
self.got_callback_result = None
def setUp(self):
self.course_factory, _ = create_factories(
os.path.join(os.path.dirname(__file__), 'tasks'))
def setUp(self):
self.dir_path = tempfile.mkdtemp()
self.course_factory, _ = create_factories(self.dir_path)
""" Used in containers to start an agent """
import sys
import os
sys.path.append('/agent')
import logging
import docker
from docker.utils import kwargs_from_env
from inginious.common.course_factory import create_factories
from inginious.backend.agent.remote_agent import RemoteAgent
if __name__ == "__main__":
course_factory, task_factory = create_factories("./tasks")
# create logger
logger = logging.getLogger("agent")
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
ch.setFormatter(formatter)
logger.addHandler(ch)
# tmp dir should be "bindable" by other containers. As the agent runs in a container, we have to guess in which dir
# /agent_volume is really mounted on, and emulate its presence in the container with a symlink.
docker_connection = None
mounted_dir = None
def get_app(config):
"""
:param config: the configuration dict
:return: A new app
"""
# First, disable debug. It will be enabled in the configuration, later.
web.config.debug = False
config = _put_configuration_defaults(config)
mongo_client = MongoClient(host=config.get('mongo_opt', {}).get('host', 'localhost'))
database = mongo_client[config.get('mongo_opt', {}).get('database', 'INGInious')]
gridfs = GridFS(database)
# Init database if needed
db_version = database.db_version.find_one({})
if db_version is None:
database.submissions.ensure_index([("username", pymongo.ASCENDING)])
database.submissions.ensure_index([("courseid", pymongo.ASCENDING)])
database.submissions.ensure_index([("courseid", pymongo.ASCENDING), ("taskid", pymongo.ASCENDING)])
database.submissions.ensure_index([("submitted_on", pymongo.DESCENDING)]) # sort speed
database.user_tasks.ensure_index(
[("username", pymongo.ASCENDING), ("courseid", pymongo.ASCENDING), ("taskid", pymongo.ASCENDING)],
unique=True)
database.user_tasks.ensure_index([("username", pymongo.ASCENDING), ("courseid", pymongo.ASCENDING)])
database.user_tasks.ensure_index([("courseid", pymongo.ASCENDING), ("taskid", pymongo.ASCENDING)])
database.user_tasks.ensure_index([("courseid", pymongo.ASCENDING)])
database.user_tasks.ensure_index([("username", pymongo.ASCENDING)])
appli = CookieLessCompatibleApplication(MongoStore(database, 'sessions'))
# Init gettext
available_languages = {
"en": "English",
"fr": "Français"
}
for lang in available_languages.keys():
appli.add_translation(lang, gettext.translation('messages', get_root_path() + '/frontend/i18n', [lang]))
builtins.__dict__['_'] = appli.gettext
if config.get("maintenance", False):
template_helper = TemplateHelper(PluginManager(), None,
'frontend/templates',
'frontend/templates/layout',
'frontend/templates/layout_lti',
config.get('use_minified_js', True))
template_helper.add_to_template_globals("get_homepath", appli.get_homepath)
template_helper.add_to_template_globals("_", _)
appli.template_helper = template_helper
appli.init_mapping(urls_maintenance)
return appli.wsgifunc(), appli.stop
default_allowed_file_extensions = config['allowed_file_extensions']
default_max_file_size = config['max_file_size']
zmq_context, __ = start_asyncio_and_zmq(config.get('debug_asyncio', False))
# Init the different parts of the app
plugin_manager = PluginManager()
# Create the FS provider
if "fs" in config:
fs_provider = filesystem_from_config_dict(config["fs"])
else:
task_directory = config["tasks_directory"]
fs_provider = LocalFSProvider(task_directory)
default_problem_types = {
problem_type.get_type(): problem_type for problem_type in [DisplayableCodeProblem,
DisplayableCodeSingleLineProblem,
DisplayableFileProblem,
DisplayableMultipleChoiceProblem,
DisplayableMatchProblem]
}
course_factory, task_factory = create_factories(fs_provider, default_problem_types, plugin_manager, WebAppCourse, WebAppTask)
user_manager = UserManager(appli.get_session(), database, config.get('superadmins', []))
update_pending_jobs(database)
client = create_arch(config, fs_provider, zmq_context, course_factory)
lti_outcome_manager = LTIOutcomeManager(database, user_manager, course_factory)
submission_manager = WebAppSubmissionManager(client, user_manager, database, gridfs, plugin_manager, lti_outcome_manager)
template_helper = TemplateHelper(plugin_manager, user_manager, 'frontend/templates',
'frontend/templates/layout',
'frontend/templates/layout_lti',
config.get('use_minified_js', True))
# Init web mail
smtp_conf = config.get('smtp', None)
if smtp_conf is not None:
web.config.smtp_server = smtp_conf["host"]
web.config.smtp_port = int(smtp_conf["port"])
web.config.smtp_starttls = bool(smtp_conf.get("starttls", False))
web.config.smtp_username = smtp_conf.get("username", "")
web.config.smtp_password = smtp_conf.get("password", "")
web.config.smtp_sendername = smtp_conf.get("sendername", "*****@*****.**")
# Add some helpers for the templates
template_helper.add_to_template_globals("_", _)
template_helper.add_to_template_globals("str", str)
template_helper.add_to_template_globals("available_languages", available_languages)
template_helper.add_to_template_globals("get_homepath", appli.get_homepath)
template_helper.add_to_template_globals("allow_registration", config.get("allow_registration", True))
template_helper.add_to_template_globals("user_manager", user_manager)
template_helper.add_to_template_globals("default_allowed_file_extensions", default_allowed_file_extensions)
template_helper.add_to_template_globals("default_max_file_size", default_max_file_size)
template_helper.add_other("course_admin_menu",
lambda course, current: course_admin_utils.get_menu(course, current, template_helper.get_renderer(False),
plugin_manager, user_manager))
template_helper.add_other("preferences_menu",
lambda current: preferences_utils.get_menu(appli, current, template_helper.get_renderer(False),
plugin_manager, user_manager))
# Not found page
appli.notfound = lambda: web.notfound(template_helper.get_renderer().notfound('Page not found'))
# Enable stacktrace display if needed
web_debug = config.get('web_debug', False)
appli.internalerror = internalerror_generator(template_helper.get_renderer(False))
if web_debug is True:
web.config.debug = True
appli.internalerror = debugerror
elif isinstance(web_debug, str):
web.config.debug = False
appli.internalerror = emailerrors(web_debug, appli.internalerror)
# Insert the needed singletons into the application, to allow pages to call them
appli.plugin_manager = plugin_manager
appli.course_factory = course_factory
appli.task_factory = task_factory
appli.submission_manager = submission_manager
appli.user_manager = user_manager
appli.template_helper = template_helper
appli.database = database
appli.gridfs = gridfs
appli.default_allowed_file_extensions = default_allowed_file_extensions
appli.default_max_file_size = default_max_file_size
appli.backup_dir = config.get("backup_directory", './backup')
appli.webterm_link = config.get("webterm", None)
appli.lti_outcome_manager = lti_outcome_manager
appli.allow_registration = config.get("allow_registration", True)
appli.allow_deletion = config.get("allow_deletion", True)
appli.available_languages = available_languages
appli.welcome_page = config.get("welcome_page", None)
appli.static_directory = config.get("static_directory", "./static")
appli.webdav_host = config.get("webdav_host", None)
# Init the mapping of the app
appli.init_mapping(urls)
# Loads plugins
plugin_manager.load(client, appli, course_factory, task_factory, database, user_manager, submission_manager, config.get("plugins", []))
# Start the inginious.backend
client.start()
return appli.wsgifunc(), lambda: _close_app(appli, mongo_client, client)
def get_app(config):
"""
:param config: the configuration dict
:param active_callback: a callback without arguments that will be called when the app is fully initialized
:return: A new app
"""
config = _put_configuration_defaults(config)
task_directory = config["tasks_directory"]
download_directory = config.get("download_directory", "lti_download")
default_allowed_file_extensions = config['allowed_file_extensions']
default_max_file_size = config['max_file_size']
appli = web.application((), globals(), autoreload=False)
zmq_context, asyncio_thread = start_asyncio_and_zmq()
# Init the different parts of the app
plugin_manager = PluginManager()
mongo_client = MongoClient(
host=config.get('mongo_opt', {}).get('host', 'localhost'))
database = mongo_client[config.get('mongo_opt',
{}).get('database', 'INGInious')]
gridfs = GridFS(database)
course_factory, task_factory = create_factories(task_directory,
plugin_manager,
FrontendCourse,
FrontendTask)
#
# Allow user config to over-rider the username strong in Mongo.
# This is enabled by most LMS's such as Moodle, and the ext_user_username
# is the "login name" for the user, which is typically the same as
# would be authenticated by logging into the course via ldap
#
lti_user_name = config.get('lti_user_name', 'user_id')
if lti_user_name not in ['user_id', 'ext_user_username']:
lti_user_name = 'user_id'
user_manager = UserManager(
CustomSession(appli, MongoStore(database, 'sessions')), database,
lti_user_name)
update_pending_jobs(database)
client = create_arch(config, task_directory, zmq_context)
lis_outcome_manager = LisOutcomeManager(database, user_manager,
course_factory, config["lti"])
submission_manager = LTISubmissionManager(
client, user_manager, database, gridfs, plugin_manager,
config.get('nb_submissions_kept', 5), lis_outcome_manager)
template_helper = TemplateHelper(plugin_manager, 'frontend/lti/templates',
'frontend/lti/templates/layout',
config.get('use_minified_js', True))
# Update the database
update_database(database)
# Add some helpers for the templates
template_helper.add_to_template_globals("get_homepath",
lambda: web.ctx.homepath)
template_helper.add_to_template_globals("user_manager", user_manager)
template_helper.add_to_template_globals("default_allowed_file_extensions",
default_allowed_file_extensions)
template_helper.add_to_template_globals("default_max_file_size",
default_max_file_size)
# Not found page
appli.notfound = lambda: web.notfound(template_helper.get_renderer().
notfound('Page not found'))
# Insert the needed singletons into the application, to allow pages to call them
appli.plugin_manager = plugin_manager
appli.course_factory = course_factory
appli.task_factory = task_factory
appli.submission_manager = submission_manager
appli.user_manager = user_manager
appli.template_helper = template_helper
appli.database = database
appli.gridfs = gridfs
appli.default_allowed_file_extensions = default_allowed_file_extensions
appli.default_max_file_size = default_max_file_size
appli.consumers = config["lti"]
appli.download_directory = download_directory
appli.download_status = {}
appli.webterm_link = config.get("webterm", None)
# Init the mapping of the app
appli.init_mapping(urls)
# Loads plugins
plugin_manager.load(client, appli, course_factory, task_factory, database,
user_manager, submission_manager,
config.get("plugins", []))
# Start the Client
client.start()
return appli.wsgifunc(), lambda: _close_app(appli, mongo_client, client,
lis_outcome_manager)
def setUp(self):
self.course_factory, self.task_factory = create_factories(os.path.join(os.path.dirname(__file__), 'tasks'))
self.setUp_job_manager()
self.callback_done = threading.Event()
self.got_callback_result = None
def get_app(config):
"""
:param config: the configuration dict
:return: A new app
"""
config = _put_configuration_defaults(config)
appli = web.application((), globals(), autoreload=False)
if config.get("maintenance", False):
template_helper = TemplateHelper(PluginManager(),
'frontend/webapp/templates',
'frontend/webapp/templates/layout',
config.get('use_minified_js', True))
template_helper.add_to_template_globals("get_homepath",
lambda: web.ctx.homepath)
appli.template_helper = template_helper
appli.init_mapping(urls_maintenance)
return appli.wsgifunc(), appli.stop
task_directory = config["tasks_directory"]
default_allowed_file_extensions = config['allowed_file_extensions']
default_max_file_size = config['max_file_size']
zmq_context, _ = start_asyncio_and_zmq()
# Init the different parts of the app
plugin_manager = PluginManager()
mongo_client = MongoClient(
host=config.get('mongo_opt', {}).get('host', 'localhost'))
database = mongo_client[config.get('mongo_opt',
{}).get('database', 'INGInious')]
gridfs = GridFS(database)
course_factory, task_factory = create_factories(task_directory,
plugin_manager,
WebAppCourse, WebAppTask)
user_manager = UserManager(
web.session.Session(appli, MongoStore(database, 'sessions')), database,
config.get('superadmins', []))
update_pending_jobs(database)
client = create_arch(config, task_directory, zmq_context)
submission_manager = WebAppSubmissionManager(client, user_manager,
database, gridfs,
plugin_manager)
batch_manager = BatchManager(client, database, gridfs, submission_manager,
user_manager, task_directory)
template_helper = TemplateHelper(plugin_manager,
'frontend/webapp/templates',
'frontend/webapp/templates/layout',
config.get('use_minified_js', True))
# Init web mail
smtp_conf = config.get('smtp', None)
if smtp_conf is not None:
web.config.smtp_server = smtp_conf["host"]
web.config.smtp_port = int(smtp_conf["port"])
web.config.smtp_starttls = bool(smtp_conf.get("starttls", False))
web.config.smtp_username = smtp_conf.get("username", "")
web.config.smtp_password = smtp_conf.get("password", "")
web.config.smtp_sendername = smtp_conf.get("sendername",
"*****@*****.**")
# Update the database
update_database(database, gridfs, course_factory, user_manager)
# Add some helpers for the templates
template_helper.add_to_template_globals("get_homepath",
lambda: web.ctx.homepath)
template_helper.add_to_template_globals("user_manager", user_manager)
template_helper.add_to_template_globals("default_allowed_file_extensions",
default_allowed_file_extensions)
template_helper.add_to_template_globals("default_max_file_size",
default_max_file_size)
template_helper.add_other(
"course_admin_menu",
lambda course, current: course_admin_utils.get_menu(
course, current, template_helper.get_renderer(False),
plugin_manager, user_manager))
# Not found page
appli.notfound = lambda: web.notfound(template_helper.get_renderer().
notfound('Page not found'))
# Enable stacktrace display if logging is at level DEBUG
if config.get('log_level', 'INFO') == 'DEBUG':
appli.internalerror = debugerror
# Insert the needed singletons into the application, to allow pages to call them
appli.plugin_manager = plugin_manager
appli.course_factory = course_factory
appli.task_factory = task_factory
appli.submission_manager = submission_manager
appli.batch_manager = batch_manager
appli.user_manager = user_manager
appli.template_helper = template_helper
appli.database = database
appli.gridfs = gridfs
appli.default_allowed_file_extensions = default_allowed_file_extensions
appli.default_max_file_size = default_max_file_size
appli.backup_dir = config.get("backup_directory", './backup')
appli.webterm_link = config.get("webterm", None)
# Init the mapping of the app
appli.init_mapping(urls)
# Loads plugins
plugin_manager.load(client, appli, course_factory, task_factory, database,
user_manager, submission_manager,
config.get("plugins", []))
# Start the inginious.backend
client.start()
return appli.wsgifunc(), lambda: _close_app(appli, mongo_client, client)
def get_app(config, active_callback=None):
"""
:param config: the configuration dict
:param active_callback: a callback without arguments that will be called when the app is fully initialized
:return: A new app
"""
config = _put_configuration_defaults(config)
task_directory = config["tasks_directory"]
default_allowed_file_extensions = config['allowed_file_extensions']
default_max_file_size = config['max_file_size']
appli = web.application((), globals(), autoreload=False)
# Init the different parts of the app
plugin_manager = PluginManager()
mongo_client = MongoClient(host=config.get('mongo_opt', {}).get('host', 'localhost'))
database = mongo_client[config.get('mongo_opt', {}).get('database', 'INGInious')]
gridfs = GridFS(database)
course_factory, task_factory = create_factories(task_directory, plugin_manager, FrontendCourse, FrontendTask)
user_manager = UserManager(CustomSession(appli, MongoStore(database, 'sessions')), database)
backend_interface.update_pending_jobs(database)
job_manager = backend_interface.create_job_manager(config, plugin_manager,
task_directory, course_factory, task_factory)
lis_outcome_manager = LisOutcomeManager(database, user_manager, course_factory, config["lti"])
submission_manager = LTISubmissionManager(job_manager, user_manager, database, gridfs, plugin_manager,
config.get('nb_submissions_kept', 5), lis_outcome_manager)
template_helper = TemplateHelper(plugin_manager, 'frontend/lti/templates', 'layout', config.get('use_minified_js', True))
# Update the database
update_database(database)
# Add some helpers for the templates
template_helper.add_to_template_globals("user_manager", user_manager)
template_helper.add_to_template_globals("default_allowed_file_extensions", default_allowed_file_extensions)
template_helper.add_to_template_globals("default_max_file_size", default_max_file_size)
# Not found page
appli.notfound = lambda: web.notfound(template_helper.get_renderer().notfound('Page not found'))
# Init the mapping of the app
appli.init_mapping(WebPyCustomMapping(dict(urls), plugin_manager,
course_factory, task_factory,
submission_manager, user_manager,
template_helper, database, gridfs,
default_allowed_file_extensions, default_max_file_size,
config["containers"].keys(),
config["lti"]))
# Active hook
if active_callback is not None:
_handle_active_hook(job_manager, plugin_manager, active_callback)
# Loads plugins
plugin_manager.load(job_manager, appli, course_factory, task_factory, database, user_manager, config.get("plugins", []))
# Start the inginious.backend
job_manager.start()
return appli, lambda: _close_app(appli, mongo_client, job_manager, lis_outcome_manager)
def get_app(config):
"""
:param config: the configuration dict
:return: A new app
"""
config = _put_configuration_defaults(config)
mongo_client = MongoClient(
host=config.get('mongo_opt', {}).get('host', 'localhost'))
database = mongo_client[config.get('mongo_opt',
{}).get('database', 'INGInious')]
gridfs = GridFS(database)
# Init database if needed
db_version = database.db_version.find_one({})
if db_version is None:
database.submissions.ensure_index([("username", pymongo.ASCENDING)])
database.submissions.ensure_index([("courseid", pymongo.ASCENDING)])
database.submissions.ensure_index([("courseid", pymongo.ASCENDING),
("taskid", pymongo.ASCENDING)])
database.submissions.ensure_index([("submitted_on", pymongo.DESCENDING)
]) # sort speed
database.user_tasks.ensure_index([("username", pymongo.ASCENDING),
("courseid", pymongo.ASCENDING),
("taskid", pymongo.ASCENDING)],
unique=True)
database.user_tasks.ensure_index([("username", pymongo.ASCENDING),
("courseid", pymongo.ASCENDING)])
database.user_tasks.ensure_index([("courseid", pymongo.ASCENDING),
("taskid", pymongo.ASCENDING)])
database.user_tasks.ensure_index([("courseid", pymongo.ASCENDING)])
database.user_tasks.ensure_index([("username", pymongo.ASCENDING)])
appli = CookieLessCompatibleApplication(MongoStore(database, 'sessions'))
# Init gettext
available_languages = {"en": "English", "fr": "Français"}
for lang in available_languages.keys():
appli.add_translation(
lang,
gettext.translation('messages',
get_root_path() + '/frontend/i18n', [lang]))
builtins.__dict__['_'] = appli.gettext
if config.get("maintenance", False):
template_helper = TemplateHelper(PluginManager(), None,
'frontend/templates',
'frontend/templates/layout',
'frontend/templates/layout_lti',
config.get('use_minified_js', True))
template_helper.add_to_template_globals("get_homepath",
appli.get_homepath)
template_helper.add_to_template_globals("_", _)
appli.template_helper = template_helper
appli.init_mapping(urls_maintenance)
return appli.wsgifunc(), appli.stop
default_allowed_file_extensions = config['allowed_file_extensions']
default_max_file_size = config['max_file_size']
zmq_context, __ = start_asyncio_and_zmq(config.get('debug_asyncio', False))
# Init the different parts of the app
plugin_manager = PluginManager()
# Create the FS provider
if "fs" in config:
fs_provider = filesystem_from_config_dict(config["fs"])
else:
task_directory = config["tasks_directory"]
fs_provider = LocalFSProvider(task_directory)
default_problem_types = {
problem_type.get_type(): problem_type
for problem_type in [
DisplayableCodeProblem, DisplayableCodeSingleLineProblem,
DisplayableFileProblem, DisplayableMultipleChoiceProblem,
DisplayableMatchProblem
]
}
course_factory, task_factory = create_factories(fs_provider,
default_problem_types,
plugin_manager,
WebAppCourse, WebAppTask)
user_manager = UserManager(appli.get_session(), database,
config.get('superadmins', []))
update_pending_jobs(database)
client = create_arch(config, fs_provider, zmq_context, course_factory)
lti_outcome_manager = LTIOutcomeManager(database, user_manager,
course_factory)
submission_manager = WebAppSubmissionManager(client, user_manager,
database, gridfs,
plugin_manager,
lti_outcome_manager)
template_helper = TemplateHelper(plugin_manager, user_manager,
'frontend/templates',
'frontend/templates/layout',
'frontend/templates/layout_lti',
config.get('use_minified_js', True))
# Init web mail
smtp_conf = config.get('smtp', None)
if smtp_conf is not None:
web.config.smtp_server = smtp_conf["host"]
web.config.smtp_port = int(smtp_conf["port"])
web.config.smtp_starttls = bool(smtp_conf.get("starttls", False))
web.config.smtp_username = smtp_conf.get("username", "")
web.config.smtp_password = smtp_conf.get("password", "")
web.config.smtp_sendername = smtp_conf.get("sendername",
"*****@*****.**")
# Add some helpers for the templates
template_helper.add_to_template_globals("_", _)
template_helper.add_to_template_globals("str", str)
template_helper.add_to_template_globals("available_languages",
available_languages)
template_helper.add_to_template_globals("get_homepath", appli.get_homepath)
template_helper.add_to_template_globals(
"allow_registration", config.get("allow_registration", True))
template_helper.add_to_template_globals("user_manager", user_manager)
template_helper.add_to_template_globals("default_allowed_file_extensions",
default_allowed_file_extensions)
template_helper.add_to_template_globals("default_max_file_size",
default_max_file_size)
template_helper.add_other(
"course_admin_menu",
lambda course, current: course_admin_utils.get_menu(
course, current, template_helper.get_renderer(False),
plugin_manager, user_manager))
template_helper.add_other(
"preferences_menu", lambda current: preferences_utils.get_menu(
appli, current, template_helper.get_renderer(False),
plugin_manager, user_manager))
# Not found page
appli.notfound = lambda: web.notfound(template_helper.get_renderer().
notfound('Page not found'))
# Enable stacktrace display if logging is at level DEBUG
if config.get('log_level', 'INFO') == 'DEBUG':
appli.internalerror = debugerror
# Insert the needed singletons into the application, to allow pages to call them
appli.plugin_manager = plugin_manager
appli.course_factory = course_factory
appli.task_factory = task_factory
appli.submission_manager = submission_manager
appli.user_manager = user_manager
appli.template_helper = template_helper
appli.database = database
appli.gridfs = gridfs
appli.default_allowed_file_extensions = default_allowed_file_extensions
appli.default_max_file_size = default_max_file_size
appli.backup_dir = config.get("backup_directory", './backup')
appli.webterm_link = config.get("webterm", None)
appli.lti_outcome_manager = lti_outcome_manager
appli.allow_registration = config.get("allow_registration", True)
appli.allow_deletion = config.get("allow_deletion", True)
appli.available_languages = available_languages
appli.welcome_page = config.get("welcome_page", None)
appli.static_directory = config.get("static_directory", "./static")
appli.webdav_host = config.get("webdav_host", None)
# Init the mapping of the app
appli.init_mapping(urls)
# Loads plugins
plugin_manager.load(client, appli, course_factory, task_factory, database,
user_manager, submission_manager,
config.get("plugins", []))
# Start the inginious.backend
client.start()
return appli.wsgifunc(), lambda: _close_app(appli, mongo_client, client)
def setUp(self):
fs = LocalFSProvider(os.path.join(os.path.dirname(__file__), 'tasks'))
self.course_factory, _ = create_factories(fs, problem_types)
def setUp(self):
fs = LocalFSProvider(os.path.join(os.path.dirname(__file__), 'tasks'))
self.course_factory, _ = create_factories(fs)
def setUp(self):
self.dir_path = tempfile.mkdtemp()
fs = LocalFSProvider(self.dir_path)
self.course_factory, _ = create_factories(fs, problem_types)
def setUp(self):
self.dir_path = tempfile.mkdtemp()
self.course_factory, _ = create_factories(self.dir_path)
""" Used in containers to start an agent """
import sys
import os
sys.path.append('/agent')
import logging
import docker
from docker.utils import kwargs_from_env
from inginious.common.course_factory import create_factories
from inginious.backend.agent.remote_agent import RemoteAgent
if __name__ == "__main__":
course_factory, task_factory = create_factories("./tasks")
# create logger
logger = logging.getLogger("agent")
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
ch.setFormatter(formatter)
logger.addHandler(ch)
# tmp dir should be "bindable" by other containers. As the agent runs in a container, we have to guess in which dir
# /agent_volume is really mounted on, and emulate its presence in the container with a symlink.
docker_connection = None
mounted_dir = None
