def replace_html(relout, stext, rtext):
""" Replaces strings in html files (useful for correcting links to cdn libs)
stext: string to find
rtext: string to replace with
relout: i.e. _build/html
"""
path = "%s/**/*.html" % relout
info("finding %s: replacing with: %s in: %s " % (stext, rtext, path))
files = glob.glob(
path, recursive=True
) # recursive since python 3.5 https://stackoverflow.com/a/2186565
for fname in files:
#debug(fname)
# debug([p.name for p in Path(relfname).parents])
# for some reason it adds an empty string: DEBUG=['exam-solutions', 'jm-templates', '']
for line in fileinput.input(fname, inplace=1):
lineno = 0
lineno = line.find(stext)
if lineno > 0:
relfname = os.path.relpath(
fname, start=relout) # works from current dir
prefix = '../' * (len(Path(relfname).parents) - 1)
line = line.replace(stext, prefix + rtext)
sys.stdout.write(line)
def CD(new_dir):
prevdir = os.getcwd()
info(f"Setting working dir to {new_dir}")
os.chdir(os.path.expanduser(new_dir))
try:
yield
finally:
os.chdir(prevdir)
info(f"Restored working dir.")
def delete_exam(parser, context, args):
ld = arg_date(parser, args)
eld_admin = '_private/%s' % ld
pubeld = 'exams/%s' % ld
pubeldzip = '_static/generated/%s-%s-exam.zip' % (jm.filename, ld)
deleted = []
ans = ''
while ans != 'Y' and ans != 'n':
print(
'DO YOU *REALLY* WANT TO DELETE EXAM %s (NOTE: CANNOT BE UNDONE) [Y/n]? '
% ld),
ans = input()
if ans != 'Y':
print("")
info("User cancelled, no data was deleted.")
return
print("")
def delete_stuff(path, confirm_path):
""" Deletes path and logs deleted stuff
"""
if os.path.exists(path):
if os.path.isfile(path):
jmt.delete_file(path, confirm_path)
elif os.path.isdir(path):
jmt.delete_tree(path, confirm_path)
else:
raise Exception("File is neither a directory nor a file: %s" %
path)
deleted.append(path)
delete_stuff(eld_admin, "_private/%s" % ld)
delete_stuff(pubeld, "exams/%s" % ld)
delete_stuff(pubeldzip,
'_static/generated/%s-%s-exam.zip' % (jm.filename, ld))
if len(deleted) == 0:
fatal("COULDN'T FIND ANY EXAM FILE TO DELETE FOR DATE: " + ld)
def grade(parser, context, args):
ld = arg_date(parser, args)
eld_admin = "_private/%s" % ld
shipped = "%s/shipped" % eld_admin
graded = "%s/graded" % eld_admin
if not os.path.exists(shipped):
fatal("Couldn't find directory: " + shipped)
try:
dir_names = next(os.walk(shipped))[1]
except Exception as e:
info("\n\n ERROR! %s\n\n" % repr(e))
exit(1)
if len(dir_names) == 0:
fatal("NOTHING TO GRADE IN %s" % shipped)
for dn in dir_names:
target = "%s/%s" % (graded, dn)
if (os.path.exists("%s/shipped" % target)):
fatal("DIRECTORY ALREADY EXISTS: %s/shipped\n\n" % target)
if (os.path.exists("%s/graded" % target)):
fatal("DIRECTORY ALREADY EXISTS: %s/graded\n\n" % target)
info(
"Copying Python files to execute and eventually grade in %s/graded"
% target)
shutil.copytree('%s/shipped/%s' % (eld_admin, dn),
'%s/shipped' % target)
info("Copying original shipped files (don't touch them!) in %s/shipped"
% target)
shutil.copytree('%s/shipped/%s' % (eld_admin, dn),
'%s/graded' % target)
def setup(app):
jmt.init(jm)
# temporary hack to have graph-drawing stuff
jmt.info('Copying soft.py ...')
import shutil
shutil.copy('soft.py', 'graph-formats/')
shutil.copy('soft.py', 'binary-relations/')
shutil.copy('soft.py', 'visualization/')
if 'googleanalytics_id' in globals() and globals()['googleanalytics_id']:
print("Found googleanalytics_id")
import googleanalytics
googleanalytics.setup(app)
else:
print('No valid googleanalytics_id was found, skipping it')
app.add_config_value('recommonmark_config', {
'auto_toc_tree_section': 'Contents',
'enable_eval_rst': True
}, True)
app.add_transform(AutoStructify)
for folder in jm.get_exercise_folders():
jm.zip_folder(folder)
#jm.zip_folders('exams/*/solutions',
# lambda x: '%s-%s-exam' % (jm.filename, x.split('/')[-2]))
#jm.zip_folders('challenges/*/', renamer = lambda x: '%s-challenge' % x.split('/')[1])
#jm.zip_paths(['project'], '_static/generated/project-template')
def sub(x):
if x == 'requirements.txt':
return 'NAME-SURNAME-ID/requirements.txt'
elif x.startswith('project/'):
return 'NAME-SURNAME-ID/%s' % x[len('project/'):]
else:
return x
"""
def zip_grades(parser, context, args):
ld = arg_date(parser, args)
eld_admin = "_private/" + ld
shipped = eld_admin + "/shipped"
try:
dir_names = next(os.walk(shipped))[1]
except Exception as e:
info("\n\n ERROR! " + repr(e) + "\n\n")
exit(1)
if len(dir_names) == 0:
info("\n\n ERROR! NOTHING TO ZIP!\n\n")
for dn in dir_names:
target = "%s/graded/%s" % (eld_admin, dn)
shutil.make_archive(target, 'zip', target)
print("")
info("You can now find zips to send to students in %s/graded" % eld_admin)
print("")
def init(parser, context, args):
parser.add_argument('date', help="date in format 'yyyy-mm-dd'")
#TODO parser.add_argument('--edit-notebook-mode')
ld = jmt.parse_date_str(vars(parser.parse_args(args))['date'])
eld_admin = "_private/" + ld
eld_solutions = "_private/%s/solutions" % ld
pubeld = "exams/%s" % ld
exam_ipynb = '%s/exam-%s.ipynb' % (eld_solutions, ld)
if os.path.exists(eld_admin):
fatal("PRIVATE EXAM ADMIN ALREADY EXISTS: " + eld_admin)
if os.path.exists(eld_solutions):
fatal("PRIVATE EXAM SOLUTIONS ALREADY EXISTS: " + eld_solutions)
if os.path.exists(pubeld):
fatal("PUBLIC EXAM ALREADY EXISTS: " + pubeld)
shutil.copytree("_templates/exam",
eld_admin,
ignore=shutil.ignore_patterns('exam-yyyy-mm-dd.ipynb'))
jmt.expand_JM('_templates/exam/solutions/exam-yyyy-mm-dd.ipynb',
exam_ipynb, ld, conf)
os.rename('%s/jupman-yyyy-mm-dd-grades.ods' % eld_admin,
"%s/%s-%s-grades.ods" % (eld_admin, conf.jm.filename, ld))
info()
info(
"You can now edit Python solutions, tests, exercises and exam notebook here : "
)
print()
info(" " + eld_solutions)
def package(parser, context, args):
ld = arg_date(parser, args)
eld_admin = '_private/' + ld
eld_solutions = '_private/%s/solutions' % ld
eld_notebook = '%s/exam-%s.ipynb' % (eld_solutions, ld)
target_student = get_target_student(ld)
target_student_pdf = '%s/%s' % (get_target_student(ld),
get_exam_text_filename(ld, 'pdf'))
# no pdf as hiding cells is too boring, have still
# to properly review cells filtering https://github.com/DavidLeoni/jupman/issues/4
# target_student_pdf = target_student + '/' + 'exam-' + ld + '.pdf'
target_student_zip = "%s/server/%s-%s-exam" % (eld_admin, jm.filename, ld
) # without '.zip'
target_server_zip = "%s/%s-%s-server" % (eld_admin, jm.filename, ld
) # without '.zip'
if not os.path.exists(jm.build):
fatal("%s WAS NOT BUILT !" % jm.build)
if not os.path.exists(eld_solutions):
fatal("MISSING SOURCE SOLUTION EXERCISES: " + eld_solutions)
if os.path.exists(target_student):
fatal("TARGET STUDENT EXERCISES DIRECTORY ALREADY EXISTS: " +
target_student)
try:
dir_names = os.listdir(jm.build)
except Exception as e:
fatal("ERROR WITH DIR %s" % jm.build, ex=e)
if len(dir_names) == 0:
fatal("SITE DIRECTORY AT %s WAS NOT BUILT !" % jm.build)
server_jupman = "%s/server/%s" % (eld_admin, jm.filename)
if os.path.exists(server_jupman):
jmt.delete_tree(server_jupman, "_private/%s/server" % ld)
info("Copying built website ...")
shutil.copytree(jm.build, server_jupman)
info("Building pdf ..")
import nbformat
import nbconvert
from nbconvert import PDFExporter
pdf_exporter = PDFExporter()
#Dav dic 2019: couldn't make it work, keeps complaining about missing files
#pdf_exporter.template_file = '_templates/classic.tplx
# as a result we have stupid extra date and worse extra numbering in headers
from nbconvert.preprocessors import ExecutePreprocessor
with open(eld_notebook) as f:
nb = nbformat.read(f, as_version=4)
old_title = jmt._replace_title(nb, eld_notebook, "").strip('#')
nb.cells = [cell for cell in nb.cells \
if not ('nbsphinx' in cell.metadata \
and cell.metadata['nbsphinx'] == 'hidden')]
(body, resources) = pdf_exporter.from_notebook_node(
nb, resources={'metadata': {
'name': old_title
}})
if not os.path.exists(target_student):
os.makedirs(target_student)
#ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
#ep.preprocess(nb, {'metadata': {'path': './'}})
with open(target_student_pdf, 'wb') as pdf_f:
#print("resources = %s" % resources)
pdf_f.write(body)
info("Copying exercises to " + str(target_student))
jm.copy_code(eld_solutions, target_student, copy_solutions=False)
info("Creating student exercises zip: %s.zip" % target_student_zip)
def mysub(fname):
if fname.startswith('_private/'):
return fname[len('_private/YYYY-MM-DD/student-zip/'):]
else:
return '/%s/%s' % (jm.get_exam_student_folder(ld), fname)
jm.zip_paths([target_student] + jm.chapter_files, target_student_zip,
mysub)
#shutil.make_archive(target_student_zip, 'zip', target_student_zip)
info("Creating server zip: %s.zip" % target_server_zip)
shutil.make_archive(target_server_zip, 'zip', eld_admin + "/server")
print("")
info("You can now browse the website at: %s" %
(os.path.abspath(eld_admin + "/server/" + jm.filename +
"/html/index.html")))
print("")
def publish(parser, context, args):
ld = arg_date(parser, args)
source = "_private/%s" % ld
source_admin = source
source_solutions = '%s/solutions' % source
student_pdf = get_target_student(ld) + '/' + get_exam_text_filename(
ld, 'pdf')
if not os.path.isdir(source_admin):
fatal("SOURCE PRIVATE EXAM FOLDER %s DOES NOT EXISTS !" % source_admin)
if not os.path.isdir(source_solutions):
fatal("SOURCE PRIVATE EXAM FOLDER %s DOES NOT EXISTS !" %
source_solutions)
dest = 'exams/%s/solutions' % ld
dest_zip = '_static/generated/%s-%s-exam' % (jm.filename, ld)
if os.path.exists(dest):
fatal("TARGET PUBLIC EXAM FOLDER %s ALREADY EXISTS !" % dest)
if os.path.exists(dest_zip):
fatal("TARGET PUBLIC EXAM ZIP %s.zip ALREADY EXISTS !" % dest_zip)
info("Copying solutions to %s" % dest)
shutil.copytree(source_solutions, dest)
info("Copying exam PDF text")
shutil.copyfile(student_pdf,
'%s/%s' % (dest, get_exam_text_filename(ld, 'pdf')))
info()
info("Exam Python files copied.")
info()
info(
"You can now manually build and run the following git instructions to publish the exam."
)
info(" ./build.py")
info(" git status # just to check everything is ok")
info(" git add .")
info(" git commit -m 'published " + ld + " exam'")
info(" git push")
info()
print("")
def delete_stuff(path, confirm_path):
""" Deletes path and logs deleted stuff
"""
if os.path.exists(path):
if os.path.isfile(path):
jmt.delete_file(path, confirm_path)
elif os.path.isdir(path):
jmt.delete_tree(path, confirm_path)
else:
raise Exception("File is neither a directory nor a file: %s" %
path)
deleted.append(path)
delete_stuff(eld_admin, "_private/%s" % ld)
delete_stuff(pubeld, "exams/%s" % ld)
delete_stuff(pubeldzip,
'_static/generated/%s-%s-exam.zip' % (jm.filename, ld))
if len(deleted) == 0:
fatal("COULDN'T FIND ANY EXAM FILE TO DELETE FOR DATE: " + ld)
handler = ArgumentHandler(description='Manages ' + jm.filename + ' exams.',
use_subcommand_help=True)
handler.run()
print("")
info("DONE.\n")
def package(parser, context, args):
parser.add_argument('date', help="date in format 'yyyy-mm-dd'")
parser.add_argument('-t',
'--site',
action='store_true',
help="zips the site")
parser.add_argument('-r',
'--server',
action='store_true',
help="zips the server")
vs = vars(parser.parse_args(args))
ld = jmt.parse_date_str(vs['date'])
zip_site = vs['site']
zip_server = vs['server']
eld_admin = '_private/' + ld
eld_solutions = '_private/%s/solutions' % ld
eld_notebook = '%s/exam-%s.ipynb' % (eld_solutions, ld)
target_student = get_target_student(ld)
target_student_pdf = '%s/%s' % (get_target_student(ld),
get_exam_text_filename(ld, 'pdf'))
target_student_zip = "%s/server/%s-%s-exam" % (eld_admin, jm.filename, ld
) # without '.zip'
target_server_zip = "%s/%s-%s-server" % (eld_admin, jm.filename, ld
) # without '.zip'
if not os.path.exists(jm.build):
fatal("%s WAS NOT BUILT !" % jm.build)
if not os.path.exists(eld_solutions):
fatal("MISSING SOURCE SOLUTION EXERCISES: " + eld_solutions)
if os.path.exists(target_student):
fatal("TARGET STUDENT EXERCISES DIRECTORY ALREADY EXISTS: " +
target_student)
try:
dir_names = os.listdir(jm.build)
except Exception as e:
fatal("ERROR WITH DIR %s" % jm.build, ex=e)
if len(dir_names) == 0:
fatal("SITE DIRECTORY AT %s WAS NOT BUILT !" % jm.build)
server_jupman = "%s/server/%s" % (eld_admin, jm.filename)
if os.path.exists(server_jupman):
jmt.delete_tree(server_jupman, "_private/%s/server" % ld)
if zip_site:
info("Copying built website ...")
shutil.copytree(jm.build, server_jupman)
if not os.path.exists(target_student):
os.makedirs(target_student)
info("Copying exercises to " + str(target_student))
jm.copy_code(eld_solutions, target_student, copy_solutions=False)
info("Building pdf ..")
import nbformat
import nbconvert
from nbconvert import PDFExporter
pdf_exporter = PDFExporter()
#Dav dic 2019: couldn't make it work, keeps complaining about missing files
#pdf_exporter.template_file = '_templates/classic.tplx
# as a result we have stupid extra date and worse extra numbering in headers
with open('%s/exam-%s.ipynb' % (target_student, ld)) as f:
ex_nb = nbformat.read(f, as_version=4)
old_title = jmt._replace_title(ex_nb, eld_notebook,
"").strip('#').strip(jm.ipynb_exercises)
ex_nb.cells = [cell for cell in ex_nb.cells \
if not ('nbsphinx' in cell.metadata \
and cell.metadata['nbsphinx'] == 'hidden')]
(body, resources) = pdf_exporter.from_notebook_node(
ex_nb, resources={'metadata': {
'name': old_title
}})
with open(target_student_pdf, 'wb') as pdf_f:
pdf_f.write(body)
info("Creating student exercises zip: %s.zip" % target_student_zip)
def remap(fname):
if fname.startswith('_private/'):
return fname[len('_private/YYYY-MM-DD/student-zip/'):]
else:
return '/%s/%s' % (jm.get_exam_student_folder(ld), fname)
deglobbed_common_files, deglobbed_common_files_patterns = jm._common_files_maps(
target_student_zip)
jm.zip_paths([target_student] + deglobbed_common_files,
target_student_zip,
patterns=deglobbed_common_files_patterns,
remap=remap)
if zip_server:
info("Creating server zip: %s.zip" % target_server_zip)
shutil.make_archive(target_server_zip, 'zip', eld_admin + "/server")
if zip_server and zip_site:
print("")
info("You can now browse the website at: %s" %
(os.path.abspath(eld_admin + "/server/" + jm.filename +
"/html/index.html")))
print("")
def run_sphinx(manuals, formats):
built = {}
failed = {}
jupman_out = os.path.join(jm.build, 'jupman')
if os.path.isdir(jupman_out):
jt.delete_tree(jupman_out, '_build')
for manual in manuals:
for fmt in formats:
relout = outdir(manual, fmt)
if os.path.isdir(relout):
jt.delete_tree(relout, '_build')
tinfo = jm.manuals[manual]
print("Building %s %s in %s" % (tinfo['name'], fmt, relout))
# sphinx-build -b html doc _build/student/html
try:
cmd = (sphinxcmd + " -j 4 -b " + fmt + " . " + relout + " " +
tinfo['args'])
res = run(cmd)
if fmt == 'html':
print(
"Fixing links to PDFs and EPUBs ... "
) # Because of this crap: http://stackoverflow.com/a/23855541
with open(relout + '/index.html', "r+") as f:
data = f.read()
data = data.replace('_JM_{download}', 'Download ')
#<a href="http://readthedocs.org/projects/jupman/downloads/pdf/latest/" target="_blank">PDF</a>
print(formats)
data = data.replace(
'_JM_{html}',
' <a target="_blank" href="/downloads/htmlzip/latest/">HTML</a>'
)
if 'pdf' in formats:
data = data.replace(
'_JM_{pdf}',
' <a target="_blank" href="/downloads/pdf/latest/">PDF</a>'
)
elif 'latex' in formats:
print("TODO LATEX !")
else:
data = data.replace('_JM_{pdf}', '')
if 'epub' in formats:
data = data.replace(
'_JM_{epub}',
' <a target="_blank" href="/downloads/epub/latest/">EPUB</a>'
)
else:
data = data.replace('_JM_{epub}', '')
print("Putting code documentation links ...")
f.seek(0)
f.write(data)
info("Fixing html paths for offline browsing ....")
replace_html(
relout,
'https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/',
'_static/js/')
replace_html(
relout,
'https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/',
'_static/js/')
replace_html(
relout,
'https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML',
'_static/js/MathJax.js')
replace_html(
relout,
'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML',
'_static/js/MathJax.js')
elif fmt == 'latex':
run('latexmk -r latexmkrc -pdf -f -dvi- -ps- -jobname=' +
jm.filename + ' -interaction=nonstopmode',
cwd=relout)
print_generated_banner(manual, fmt)
print(" " + get_path(manual, fmt) + "\n\n")
built[(manual, fmt)] = {}
except subprocess.CalledProcessError as err:
failed[(manual, fmt)] = {'err': err}
print("ERROR: FAILED BUILDING %s %s SKIPPING IT !!" %
(manual, fmt))
print(err.output)
if len(built) > 0:
print("\n\n GENERATED MANUALS: \n\n")
maxpad = 0
for (manual, fmt) in sorted(built.keys()):
maxpad = max(maxpad, len(" %s %s : " % (manual, fmt)))
for (manual, fmt) in sorted(built.keys()):
print((" %s %s : " % (manual, fmt)).rjust(maxpad) +
get_path(manual, fmt))
print("")
print("")
if len(failed) > 0:
print("\n\n THERE WERE ERRORS WHILE BUILDING:\n\n")
for (manual, fmt) in failed.keys():
print(" %s %s : " % (manual, fmt))
print(" " + str(failed[(manual, fmt)]['err']) +
" \n\n")
exit(1)
def fixship(parser, context, args):
parser.add_argument('date', help="date in format 'yyyy-mm-dd'")
parser.add_argument(
'--dry-run',
action='store_true',
)
parser.add_argument('--include', nargs='?')
parser.add_argument('--exclude', nargs='?')
parsed = vars(parser.parse_args(args))
ld = jmt.parse_date_str(parsed['date'])
dry_run = parsed['dry_run']
include = parsed['include']
if include:
include = include.split(',')
else:
include = []
exclude = parsed['exclude']
if exclude:
exclude = exclude.split(',')
else:
exclude = []
if dry_run:
warn("DRY RUN, NOT CHANGING ANYTHING !")
eld_admin = "_private/%s" % ld
shipped_raw = "%s/shipped-raw" % eld_admin
shipped = "%s/shipped" % eld_admin
graded = "%s/graded" % eld_admin
if not os.path.exists(shipped_raw):
fatal("Couldn't find directory: " + shipped_raw)
try:
dir_names = next(os.walk(shipped_raw))[1]
except Exception as e:
info("\n\n ERROR! %s\n\n" % repr(e))
exit(1)
if len(dir_names) == 0:
fatal("NOTHING TO FIX IN %s" % shipped_raw)
for dn in dir_names:
source = "%s/%s" % (shipped_raw, dn)
info('source=%s' % source)
#shutil.copytree('%s/shipped/%s' % (eld_admin, dn) , '%s/graded' % target)
try:
zip_names = next(os.walk(source))[2]
except Exception as e:
info("\n\n ERROR! %s\n\n" % repr(e))
exit(1)
if len(zip_names) == 0:
fatal("NOTHING TO UNZIP IN %s/%s" % (shipped_raw, dn))
elif len(zip_names) != 1:
fatal("MORE THAN ONE FILE TO UNZIP IN %s/%s" % (shipped_raw, dn))
source_zip = "%s/%s/%s" % (shipped_raw, dn, zip_names[0])
with ZipFile(source_zip, 'r') as zip_obj:
# Get a list of all archived file names from the zip
fnames = zip_obj.namelist()
# Iterate over the file names
created_dir = False
for fn in fnames:
if 'FIRSTNAME' in fn or 'LASTNAME' in fn:
fatal("FOUND NON-RENAMED DIRECTORY: %s" % fn)
if fn.startswith(f'{conf.jm.filename}-') or fn.startswith(
'sciprog-qcb-'):
if not created_dir:
folder = os.path.normpath(fn).split(os.sep)[0]
target_dir = "%s/%s" % (shipped, folder)
if (os.path.exists(target_dir)):
fatal("TARGET DIRECTORY ALREADY EXISTS: %s\n\n" %
target_dir)
info("Creating target_dir %s" % target_dir)
if not dry_run:
os.makedirs(target_dir)
created_dir = True
target_path = "%s/%s" % (shipped, fn)
user_filepath = os.path.normpath(fn).split(os.path.sep)
user_filepath = "/".join(user_filepath[1:])
if (not include or (include and any([fnmatch.fnmatch(user_filepath, ext) for ext in include])))\
and not any([fnmatch.fnmatch(user_filepath, ext) for ext in exclude]):
info('extracting %s' % target_path)
if not dry_run:
zip_obj.extract(fn, shipped)
else:
info(' skipped %s' % fn)
else:
fatal(
f"FOUND FILE NOT IN {conf.jm.filename}-* DIRECTORY !: %s\n\n"
% fn)
# Extract all the contents of zip file in different directory
#zipObj.extractall('target')
print("")
if dry_run:
warn("END OF DRY RUN")
else:
info("DONE.\n")
print()
