def check_install(self, argv=None, dirs=None):
"""Check files were installed in the correct place."""
if argv is None:
argv = []
if dirs is None:
dirs = {
'conf': jupyter_core.paths.jupyter_config_dir(),
'data': jupyter_core.paths.jupyter_data_dir(),
}
conf_dir = dirs['conf']
# do install
main_app(argv=['enable'] + argv)
# list everything that got installed
installed_files = []
for root, subdirs, files in os.walk(dirs['conf']):
installed_files.extend([os.path.join(root, f) for f in files])
nt.assert_true(
installed_files,
'Install should create files in {}'.format(dirs['conf']))
# a bit of a hack to allow initializing a new app instance
for klass in app_classes:
reset_app_class(klass)
# do uninstall
main_app(argv=['disable'] + argv)
# check the config directory
conf_installed = [
path for path in installed_files
if path.startswith(conf_dir) and os.path.exists(path)]
for path in conf_installed:
with open(path, 'r') as f:
conf = Config(json.load(f))
nbapp = conf.get('NotebookApp', {})
nt.assert_not_in(
'jupyter_nbextensions_configurator',
nbapp.get('server_extensions', []),
'Uninstall should empty'
'server_extensions list'.format(path))
nbservext = nbapp.get('nbserver_extensions', {})
nt.assert_false(
{k: v for k, v in nbservext.items() if v},
'Uninstall should disable all '
'nbserver_extensions in file {}'.format(path))
confstrip = {}
confstrip.update(conf)
confstrip.pop('NotebookApp', None)
confstrip.pop('version', None)
nt.assert_false(confstrip, 'Uninstall should leave config empty.')
reset_app_class(DisableJupyterNbextensionsConfiguratorApp)
def check_enable(self, argv=None, dirs=None):
"""Check files were enabled in the correct place."""
if argv is None:
argv = []
if dirs is None:
dirs = {
'conf': jupyter_core.paths.jupyter_config_dir(),
'data': jupyter_core.paths.jupyter_data_dir(),
}
conf_dir = dirs['conf']
# do enable
main_app(argv=['enable'] + argv)
# list everything that got enabled
created_files = []
for root, subdirs, files in os.walk(dirs['conf']):
created_files.extend([os.path.join(root, f) for f in files])
nt.assert_true(created_files,
'enable should create files in {}'.format(dirs['conf']))
# a bit of a hack to allow initializing a new app instance
for klass in app_classes:
reset_app_class(klass)
# do disable
main_app(argv=['disable'] + argv)
# check the config directory
conf_enabled = [
path for path in created_files
if path.startswith(conf_dir) and os.path.exists(path)
]
for path in conf_enabled:
with open(path, 'r') as f:
conf = Config(json.load(f))
nbapp = conf.get('NotebookApp', {})
nt.assert_not_in(
'jupyter_nbextensions_configurator',
nbapp.get('server_extensions', []),
'conf after disable should empty'
'server_extensions list'.format(path))
nbservext = nbapp.get('nbserver_extensions', {})
nt.assert_false({k: v
for k, v in nbservext.items() if v},
'disable command should disable all '
'nbserver_extensions in file {}'.format(path))
reset_app_class(DisableJupyterNbextensionsConfiguratorApp)
def _uninstall_pre_config(logger=None):
"""Undo config settings inserted by an old (pre-themysto) installation."""
# for application json config files
cm = BaseJSONConfigManager(config_dir=jupyter_config_dir())
# -------------------------------------------------------------------------
# notebook json config
config_basename = 'jupyter_notebook_config'
config = Config(cm.get(config_basename))
config_path = cm.file_name(config_basename)
if config and logger:
logger.info('- Removing old config values from {}'.format(config_path))
to_remove = ['nbextensions']
# remove from notebook >= 4.2 key nbserver_extensions
section = config.get('NotebookApp', Config())
server_extensions = section.get('nbserver_extensions', {})
for se in to_remove:
server_extensions.pop(se, None)
if len(server_extensions) == 0:
section.pop('nbserver_extensions', None)
# and notebook < 4.2 key server_extensions
_update_config_list(config, 'NotebookApp.server_extensions', to_remove,
False)
_update_config_list(config, 'NotebookApp.extra_template_paths', [
os.path.join(jupyter_data_dir(), 'templates'),
], False)
_set_managed_config(cm, config_basename, config, logger)
# -------------------------------------------------------------------------
# nbconvert json config
config_basename = 'jupyter_nbconvert_config'
config = Config(cm.get(config_basename))
if config and logger:
logger.info('- Removing old config values from {}'.format(config_path))
_update_config_list(config, 'Exporter.template_path', [
'.',
os.path.join(jupyter_data_dir(), 'templates'),
], False)
_update_config_list(config, 'Exporter.preprocessors', [
'pre_codefolding.CodeFoldingPreprocessor',
'pre_pymarkdown.PyMarkdownPreprocessor',
], False)
section = config.get('NbConvertApp', {})
if (section.get('postprocessor_class') ==
'post_embedhtml.EmbedPostProcessor'):
section.pop('postprocessor_class', None)
if len(section) == 0:
config.pop('NbConvertApp', None)
_set_managed_config(cm, config_basename, config, logger)
# -------------------------------------------------------------------------
# Remove old config lines from .py configuration files
for config_basename in ('jupyter_notebook_config.py',
'jupyter_nbconvert_config.py'):
py_config_path = os.path.join(jupyter_config_dir(), config_basename)
if not os.path.isfile(py_config_path):
continue
if logger:
logger.info(
'-- Removing now-empty config file {}'.format(py_config_path))
with io.open(py_config_path, 'r') as f:
lines = f.readlines()
marker = '#--- nbextensions configuration ---'
marker_inds = [ii for ii, l in enumerate(lines) if l.find(marker) >= 0]
if len(marker_inds) >= 2:
lines = lines[0:marker_inds[0]] + lines[marker_inds[1] + 1:]
if [l for l in lines if l.strip]:
with io.open(py_config_path, 'w') as f:
f.writelines(lines)
else:
if logger:
logger.info('Removing now-empty config file {}'.format(
py_config_path))
try:
os.remove(py_config_path)
except OSError as ex:
if ex.errno != errno.ENOENT:
raise
def _uninstall_pre_config(logger=None):
"""Undo config settings inserted by an old (pre-themysto) installation."""
# for application json config files
cm = BaseJSONConfigManager(config_dir=jupyter_config_dir())
# -------------------------------------------------------------------------
# notebook json config
config_basename = 'jupyter_notebook_config'
config = Config(cm.get(config_basename))
config_path = cm.file_name(config_basename)
if config and logger:
logger.info('- Removing old config values from {}'.format(config_path))
to_remove = ['nbextensions']
# remove from notebook >= 4.2 key nbserver_extensions
section = config.get('NotebookApp', Config())
server_extensions = section.get('nbserver_extensions', {})
for se in to_remove:
server_extensions.pop(se, None)
if len(server_extensions) == 0:
section.pop('nbserver_extensions', None)
# and notebook < 4.2 key server_extensions
_update_config_list(
config, 'NotebookApp.server_extensions', to_remove, False)
_update_config_list(config, 'NotebookApp.extra_template_paths', [
os.path.join(jupyter_data_dir(), 'templates'),
], False)
_set_managed_config(cm, config_basename, config, logger)
# -------------------------------------------------------------------------
# nbconvert json config
config_basename = 'jupyter_nbconvert_config'
config = Config(cm.get(config_basename))
if config and logger:
logger.info('- Removing old config values from {}'.format(config_path))
_update_config_list(config, 'Exporter.template_path', [
'.', os.path.join(jupyter_data_dir(), 'templates'),
], False)
_update_config_list(config, 'Exporter.preprocessors', [
'pre_codefolding.CodeFoldingPreprocessor',
'pre_pymarkdown.PyMarkdownPreprocessor',
], False)
section = config.get('NbConvertApp', {})
if (section.get('postprocessor_class') ==
'post_embedhtml.EmbedPostProcessor'):
section.pop('postprocessor_class', None)
if len(section) == 0:
config.pop('NbConvertApp', None)
_set_managed_config(cm, config_basename, config, logger)
# -------------------------------------------------------------------------
# Remove old config lines from .py configuration files
for config_basename in ('jupyter_notebook_config.py',
'jupyter_nbconvert_config.py'):
py_config_path = os.path.join(jupyter_config_dir(), config_basename)
if not os.path.isfile(py_config_path):
continue
if logger:
logger.info(
'-- Removing now-empty config file {}'.format(py_config_path))
with io.open(py_config_path, 'r') as f:
lines = f.readlines()
marker = '#--- nbextensions configuration ---'
marker_inds = [ii for ii, l in enumerate(lines) if l.find(marker) >= 0]
if len(marker_inds) >= 2:
lines = lines[0:marker_inds[0]] + lines[marker_inds[1] + 1:]
if [l for l in lines if l.strip]:
with io.open(py_config_path, 'w') as f:
f.writelines(lines)
else:
if logger:
logger.info(
'Removing now-empty config file {}'.format(
py_config_path))
try:
os.remove(py_config_path)
except OSError as ex:
if ex.errno != errno.ENOENT:
raise
def __init__(self, course_dir=None, auto=False) -> 'Course':
"""Initialize a course from a config file.
:param course_dir: The directory your course. If none, defaults to current working directory.
:type course_dir: str
:param auto: Suppress all prompts, automatically answering yes.
:type auto: bool
:returns: A Course object for performing operations on an entire course at once.
:rtype: Course
"""
#=======================================#
# Working Directory & Git Sync #
#=======================================#
# Set up the working directory. If no course_dir has been specified, then it
# is assumed that this is the course directory.
self.working_directory = course_dir if course_dir is not None else os.getcwd(
)
repo = Repo(self.working_directory)
# Before we do ANYTHING, make sure our working directory is clean with no
# untracked files! Unless we're running a automated job, in which case we
# don't want to fail for an unexpected reason.
if (repo.is_dirty() or repo.untracked_files) and (not auto):
continue_with_dirty = input("""
Your repository is currently in a dirty state (modifications or
untracked changes are present). We strongly suggest that you resolve
these before proceeding. Continue? [y/n]:""")
# if they didn't say no, exit
if continue_with_dirty.lower() != 'y':
sys.exit("Exiting...")
# PRINT BANNER
print(
AsciiTable([['Initializing Course and Pulling Instructors Repo']
]).table)
# pull the latest copy of the repo
utils.pull_repo(repo_dir=self.working_directory)
# Make sure we're running our nbgrader commands within our instructors repo.
# this will contain our gradebook database, our source directory, and other
# things.
config = Config()
config.CourseDirectory.root = self.working_directory
#=======================================#
# Load Config #
#=======================================#
# Check for an nbgrader config file...
if not os.path.exists(
os.path.join(self.working_directory, 'nbgrader_config.py')):
# if there isn't one, make sure there's at least a rudaux config file
if not os.path.exists(
os.path.join(self.working_directory, 'rudaux_config.py')):
sys.exit("""
You do not have nbgrader_config.py or rudaux_config.py in your current
directory. We need at least one of these to set up your course
parameters! You can specify a directory with the course_dir argument
if you wish.
""")
# use the traitlets Application class directly to load nbgrader config file.
# reference:
# https://github.com/jupyter/nbgrader/blob/41f52873c690af716c796a6003d861e493d45fea/nbgrader/server_extensions/validate_assignment/handlers.py#L35-L37
# ._load_config_files() returns a generator, so if the config is missing,
# the generator will act similarly to an empty array
# load rudaux_config if it exists, otherwise just bring in nbgrader_config.
for rudaux_config in Application._load_config_files(
'rudaux_config', path=self.working_directory):
config.merge(rudaux_config)
for nbgrader_config in Application._load_config_files(
'nbgrader_config', path=self.working_directory):
config.merge(nbgrader_config)
#=======================================#
# Set Config Params #
#=======================================#
## NBGRADER PARAMS
# If the user set the exchange, perform home user expansion if necessary
if config.get('Exchange', {}).get('root') is not None:
# perform home user expansion. Should not throw an error, but may
try:
# expand home user in-place
config['Exchange']['root'] = os.path.expanduser(
config['Exchange']['root'])
except:
pass
## CANVAS PARAMS
# Before we continue, make sure we have all of the necessary parameters.
self.course_id = config.get('Canvas', {}).get('course_id')
self.canvas_url = config.get('Canvas', {}).get('canvas_url')
self.external_tool_name = config.get('Canvas',
{}).get('external_tool_name')
self.external_tool_level = config.get('Canvas',
{}).get('external_tool_level')
# The canvas url should have no trailing slash
self.canvas_url = re.sub(r"/$", "", self.canvas_url)
## GITHUB PARAMS
self.stu_repo_url = config.get('GitHub', {}).get('stu_repo_url', '')
self.assignment_release_path = config.get(
'GitHub', {}).get('assignment_release_path')
self.ins_repo_url = config.get('GitHub', {}).get('ins_repo_url')
# subpath not currently supported
# self.ins_dir_subpath = config.get('GitHub').get('ins_dir_subpath')
## JUPYTERHUB PARAMS
self.hub_url = config.get('JupyterHub', {}).get('hub_url')
# The hub url should have no trailing slash
self.hub_url = re.sub(r"/$", "", self.hub_url)
# Get Storage directory & type
self.storage_path = config.get('JupyterHub', {}).get('storage_path', )
self.zfs = config.get('JupyterHub',
{}).get('zfs') # Optional, default is false!
self.zfs_regex = config.get('JupyterHub',
{}).get('zfs_regex') # default is false!
self.zfs_datetime_pattern = config.get('JupyterHub', {}).get(
'zfs_datetime_pattern') # default is false!
# Note hub_prefix, not base_url, to avoid any ambiguity
self.hub_prefix = config.get('JupyterHub', {}).get('base_url')
# If prefix was set, make sure it has no trailing slash, but a preceding
# slash
if self.hub_prefix is not None:
self.hub_prefix = re.sub(r"/$", "", self.hub_prefix)
if re.search(r"^/", self.hub_prefix) is None:
self.hub_prefix = fr"/{self.hub_prefix}"
## COURSE PARAMS
self.grading_image = config.get('Course', {}).get('grading_image')
self.tmp_dir = config.get('Course', {}).get('tmp_dir')
assignment_list = config.get('Course', {}).get('assignments')
self.course_timezone = config.get('Course', {}).get('timezone')
self.system_timezone = pendulum.now(tz='local').timezone.name
## Repurpose the rest of the params for later batches
## (Hang onto them in case we need something)
self._full_config = config
#=======================================#
# Validate URLs (Slightly) #
#=======================================#
urls = {
'JupyterHub.hub_url': self.hub_url,
'Canvas.canvas_url': self.canvas_url
}
for key, value in urls.items():
if re.search(r"^https{0,1}", value) is None:
sys.exit(f"""
You must specify the scheme (e.g. https://) for all URLs.
You are missing the scheme in "{key}":
{value}
""")
if re.search(r".git$", value) is not None:
sys.exit(f"""
Please do not use .git-appended URLs.
You have used a .git url in "{key}":
{value}
""")
#=======================================#
# Check For Required Params #
#=======================================#
# Finally, before we continue, make sure all of our required parameters were
# specified in the config file(s)
required_params = {
"Canvas.course_id": self.course_id,
"Canvas.canvas_url": self.canvas_url,
"GitHub.stu_repo_url": self.stu_repo_url,
"GitHub.ins_repo_url": self.ins_repo_url,
"JupyterHub.hub_url": self.hub_url,
"Course.assignments": assignment_list
}
# If any are none...
if None in required_params.values():
# Figure out which ones are none and let the user know.
for key, value in required_params.items():
if value is None:
print(f" \"{key}\" is missing.")
sys.exit(
'Please make sure you have specified all required parameters in your config file.'
)
#=======================================#
# Check For Optional Params #
#=======================================#
# Now look for all of our optional parameters. If any are missing, let the
# user know we'll be using the default.
optional_params = {
"assignment_release_path": {
"value": self.assignment_release_path,
"default": 'materials',
"config_name": "GitHub.assignment_release_path"
},
# "assignment_source_path": {
# "value": self.assignment_source_path,
# "default": "source",
# "config_name": "c.GitHub.assignment_source_path"
# },
"hub_prefix": {
"value": self.hub_prefix,
"default": "",
"config_name": "JupyterHub.base_url"
},
"zfs": {
"value": self.zfs,
"default": False,
"config_name": "JupyterHub.zfs"
},
"zfs_regex": {
"value": self.zfs_regex,
"default": r'\d{4}-\d{2}-\d{2}-\d{4}',
"config_name": "JupyterHub.zfs_regex"
},
"zfs_datetime_pattern": {
"value": self.zfs_datetime_pattern,
"default": 'YYYY-MM-DD-HHmm',
"config_name": "JupyterHub.zfs_datetime_pattern"
},
"course_timezone": {
"value": self.course_timezone,
"default": 'US/Pacific',
"config_name": "Course.timezone"
},
"grading_image": {
"value": self.grading_image,
"default": 'ubcdsci/r-dsci-grading',
"config_name": "Course.grading_image"
},
"tmp_dir": {
"value": self.tmp_dir,
"default": os.path.join(Path.home(), 'tmp'),
"config_name": "Course.tmp_dir"
},
"external_tool_name": {
"value": self.external_tool_name,
"default": 'Jupyter',
"config_name": "Canvas.external_tool_name"
},
"external_tool_level": {
"value": self.external_tool_level,
"default": 'course',
"config_name": "Canvas.external_tool_level"
}
}
for key, param in optional_params.items():
if param.get('value') is None:
setattr(self, key, param.get('default'))
print(
f" \"{param.get('config_name')}\" is missing, using default parameter of \"{getattr(self, key)}\""
)
# Make sure no preceding or trailing slashes in assignment release path
self.assignment_release_path = re.sub(r"/$", "",
self.assignment_release_path)
self.assignment_release_path = re.sub(r"^/", "",
self.assignment_release_path)
# Since we are using the student repo URL for the Launch URLs
# (i.e. telling nbgitpuller where to find the notebook),
# if the user provided an SSH url, we need the https version as well.
self.stu_launch_url = utils.generate_git_urls(
self.stu_repo_url).get('plain_https')
#! this is cheating a bit, but we can get the repo name this way
#! Fix me in the future
self.ins_repo_name = os.path.split(
utils.generate_git_urls(self.ins_repo_url).get('plain_https'))[1]
self.stu_repo_name = os.path.split(self.stu_launch_url)[1]
#=======================================#
# Set Canvas Token #
#=======================================#
canvas_token_name = config.get('Canvas').get('token_name')
if canvas_token_name is None:
print("Searching for default Canvas token, CANVAS_TOKEN...")
canvas_token_name = 'CANVAS_TOKEN'
self.canvas_token = self._get_token(canvas_token_name)
#=======================================#
# Finalize Setting Params #
#=======================================#
# set up the nbgrader api with our merged config files
self.nb_api = NbGraderAPI(config=config)
# assign init params to object
# self.canvas_token = self._get_token(canvas_token_name)
# self.course = self._get_course()
# Set crontab
# self.cron = CronTab(user=True)
# We need to use the system crontab because we'll be making ZFS snapshots
# which requires elevated permissions
self.cron = CronTab(user=True)
#=======================================#
# Instantiate Assignments #
#=======================================#
# Subclass assignment for this course:
class CourseAssignment(rudaux.Assignment):
course = self
instantiated_assignments = []
for _assignment in assignment_list:
assignment = CourseAssignment(
name=_assignment.get('name'),
duedate=_assignment.get('duedate'),
duetime=_assignment.get(
'duetime', '23:59:59'), # default is 1 sec to midnight
points=_assignment.get('points', 0), # default is zero points
manual=_assignment.get('manual',
False), # default is no manual grading
)
instantiated_assignments.append(assignment)
self.assignments = instantiated_assignments