def cleanup_exercise_master(exercise, new_version=None):
filepath = os.path.join(exercise.dirpath, f'{exercise.key}.ipynb')
cells, metadata = ipynb_util.load_cells(filepath, True)
cells_new = [x.to_ipynb() for x in ipynb_util.normalized_cells(cells)]
if new_version is None:
new_version = exercise.version
elif new_version == hashlib.sha1:
exercise_definition = {
'description': [x.to_ipynb() for x in exercise.description],
'answer_cell': exercise.answer_cell().to_ipynb(),
'instructive_test': [x.to_ipynb() for x in exercise.instructive_test],
}
m = hashlib.sha1()
m.update(json.dumps(exercise_definition).encode())
new_version = m.hexdigest()
else:
assert isinstance(new_version, str)
if new_version != exercise.version:
logging.info(f'[INFO] Renew version of {exercise.key}')
exercise.version = new_version
deadlines = ipynb_metadata.master_metadata_deadlines(metadata)
drive = ipynb_metadata.master_metadata_drive(metadata)
metadata_new = ipynb_metadata.master_metadata(exercise.key, True, exercise.version, exercise.title, deadlines, drive)
ipynb_util.save_as_notebook(filepath, cells_new, metadata_new)
def update_exercise_master_metadata_formwise(separates, bundles, new_deadlines, new_drive):
for exercise in itertools.chain(*bundles.values(), separates):
filepath = os.path.join(exercise.dirpath, f'{exercise.key}.ipynb')
cells, metadata = ipynb_util.load_cells(filepath)
deadlines_cur = ipynb_metadata.master_metadata_deadlines(metadata)
deadlines = new_deadlines.get(exercise.key, deadlines_cur)
if deadlines != deadlines_cur:
logging.info(f'[INFO] Renew deadline of {exercise.key}')
drive_cur = ipynb_metadata.master_metadata_drive(metadata)
drive = new_drive.get(exercise.key, drive_cur)
if drive != drive_cur:
logging.info(f'[INFO] Renew Google Drive ID/URL of {exercise.key}')
metadata = ipynb_metadata.master_metadata(exercise.key, True, exercise.version, exercise.title, deadlines, drive)
ipynb_util.save_as_notebook(filepath, cells, metadata)
for dirpath, exercises in bundles.items():
dirname = os.path.basename(dirpath)
for exercise in exercises:
filepath = os.path.join(exercise.dirpath, f'{exercise.key}.ipynb')
cells, metadata = ipynb_util.load_cells(filepath)
deadlines_cur = ipynb_metadata.master_metadata_deadlines(metadata)
deadlines = new_deadlines.get(f'{dirname}/', deadlines_cur)
if deadlines != deadlines_cur:
logging.info(f'[INFO] Renew deadline of bundle {dirname}/{exercise.key}')
drive_cur = ipynb_metadata.master_metadata_drive(metadata)
drive = new_drive.get(f'{dirname}/', drive_cur)
if drive != drive_cur:
logging.info(f'[INFO] Renew Google Drive ID/URL of bundle {dirname}/{exercise.key}')
metadata = ipynb_metadata.master_metadata(exercise.key, True, exercise.version, exercise.title, deadlines, drive)
ipynb_util.save_as_notebook(filepath, cells, metadata)
def add_question_exists_into_answer_cell(filepath):
CONTENT_TYPE_REGEX = r'\*\*\*CONTENT_TYPE:\s*(.+?)\*\*\*'
cells, metadata = ipynb_util.load_cells(filepath)
for i, c in enumerate(cells):
if c['cell_type'] == 'markdown':
matches = list(
re.finditer(CONTENT_TYPE_REGEX, ''.join(c['source'])))
if not matches:
continue
key = matches[0][1]
if key == 'ANSWER_CELL_CONTENT' and all(
'QUESTION_EXISTS = False' not in x
for x in cells[i + 1]['source']):
print('Append QUESTION_EXISTS: ', filepath)
cells[i + 1]['source'][0:0] = [
'QUESTION_EXISTS = False # 質問がある場合は True にしてコメントに質問を記述\n',
'\n'
]
if key == 'SYSTEM_TESTCODE' and all(
'question_exists' not in x
for x in cells[i + 1]['source']):
print('Append question_exists: ', filepath)
cells[i + 1]['source'][-1] = cells[i + 1]['source'][-1] + '\n'
cells[i + 1]['source'].extend(
PRECHECK_QUESTION_EXISTS_TEMPLATE.splitlines(True))
ipynb_util.save_as_notebook(filepath, cells, metadata)
def convert_master(filepath):
CONTENT_TYPE_REGEX = r'\*\*\*CONTENT_TYPE:\s*(.+?)\*\*\*'
cells, metadata = ipynb_util.load_cells(filepath)
for i, c in enumerate(cells):
if c['cell_type'] == 'markdown':
matches = list(
re.finditer(CONTENT_TYPE_REGEX, ''.join(c['source'])))
if not matches:
continue
key = matches[0][1]
if key in REWRITE_RULES:
c['source'] = REWRITE_RULES[key].splitlines(True)
if key == 'PLAYGROUND':
cells[i + 1]['source'] = ['judge_util.unittest_main()']
new_cells = []
deleting = False
for c in cells:
if c['cell_type'] == 'markdown':
matches = list(
re.finditer(CONTENT_TYPE_REGEX, ''.join(c['source'])))
if matches:
deleting = False
if matches[0][1] in DELETED_FIELDS:
deleting = True
if not deleting:
new_cells.append(c)
ipynb_util.save_as_notebook(filepath, new_cells, metadata)
def generate_template(exercise):
FieldKey = build_autograde.FieldKey
CONTENT_TYPE_REGEX = r'\*\*\*CONTENT_TYPE:\s*(.+?)\*\*\*'
cells, metadata = ipynb_util.load_cells(
os.path.join(exercise.dirpath, exercise.key + '.ipynb'))
gen_cells = []
for i, c in enumerate(cells):
if c['cell_type'] == 'markdown':
matches = list(
re.finditer(CONTENT_TYPE_REGEX, ''.join(c['source'])))
gen_cells.append(c)
if not matches:
continue
key = getattr(FieldKey, matches[0][1])
if key == FieldKey.SYSTEM_TESTCODE:
gen_cells.append(generate_precheck_test_code(exercise))
given_test = generate_given_test_code(exercise)
gen_cells.extend(given_test)
gen_cells.append(generate_hidden_test_code(exercise))
if key == FieldKey.PLAYGROUND:
gen_cells.append(
ipynb_util.code_cell(
'judge_util.unittest_main()').to_ipynb())
else:
gen_cells.append(c)
filepath = os.path.join(exercise.dirpath, f'template_{exercise.key}.ipynb')
ipynb_util.save_as_notebook(filepath, gen_cells, metadata)
def release_ipynb(master_path,
new_version,
new_deadlines,
new_drive,
form_dir=None):
key, ext = os.path.splitext(os.path.basename(master_path))
cells, metadata = ipynb_util.load_cells(master_path, True)
title = extract_first_heading(cells)
version = ipynb_metadata.master_metadata_version(metadata)
if new_version is None:
new_version = version
elif new_version == hashlib.sha1:
m = hashlib.sha1()
m.update(json.dumps(cells).encode())
new_version = m.hexdigest()
else:
assert isinstance(new_version, str)
if new_version != version:
logging.info(f'[INFO] Renew version of `{master_path}`')
version = new_version
deadlines_cur = ipynb_metadata.master_metadata_deadlines(metadata)
deadlines = new_deadlines.get(key, deadlines_cur)
if deadlines != deadlines_cur:
logging.info(f'[INFO] Renew deadline of `{master_path}`')
drive_cur = ipynb_metadata.master_metadata_drive(metadata)
drive = new_drive.get(key)
if drive != drive_cur:
logging.info(f'[INFO] Renew Google Drive ID/URL of `{master_path}`')
master_metadata = ipynb_metadata.master_metadata(key, False, version,
title, deadlines, drive)
ipynb_util.save_as_notebook(master_path, cells, master_metadata)
logging.info(f'[INFO] Released master `{master_path}`')
if form_dir:
form_path = os.path.join(form_dir, f'{key}.ipynb')
else:
form_path = os.path.join(os.path.dirname(master_path),
f'form_{key}.ipynb')
submission_metadata = ipynb_metadata.submission_metadata({key: version},
False)
ipynb_util.save_as_notebook(form_path, cells, submission_metadata)
logging.info(f'[INFO] Released form `{form_path}`')
def create_exercise_configuration(exercise: Exercise):
tests_dir = os.path.join(CONF_DIR, exercise.key)
os.makedirs(tests_dir, exist_ok=True)
cells = [x.to_ipynb() for x in itertools.chain(exercise.description, [exercise.answer_cell_content])]
_, metadata = ipynb_util.load_cells(os.path.join(exercise.dirpath, exercise.key + '.ipynb'), True)
ipynb_metadata.extend_master_metadata_for_trial(metadata, exercise.answer_cell_content.source)
ipynb_util.save_as_notebook(os.path.join(CONF_DIR, exercise.key + '.ipynb'), cells, metadata)
setting = judge_setting.generate_judge_setting(exercise.key, exercise.version, [stage for stage, _ in exercise.test_modules])
with open(os.path.join(tests_dir, 'setting.json'), 'w', encoding='utf-8') as f:
json.dump(setting, f, indent=1, ensure_ascii=False)
for stage, content in exercise.test_modules:
with open(os.path.join(tests_dir, f'{stage.name}.py'), 'w', encoding='utf-8', newline='\n') as f:
print(content, 'judge_util.unittest_main()', sep='\n', file=f)
for path in itertools.chain(*(stage.required_files for stage, _ in exercise.test_modules)):
dest = os.path.join(tests_dir, path)
os.makedirs(os.path.dirname(dest), exist_ok=True)
shutil.copyfile(os.path.join(exercise.dirpath, path), dest)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true', help='Verbose option')
parser.add_argument('-d', '--deadlines', metavar='DEADLINES_JSON', help='Specify a JSON file of deadline settings.')
parser.add_argument('-c', '--configuration', metavar='JUDGE_ENV_JSON', help='Create configuration with environmental parameters specified in JSON.')
parser.add_argument('-n', '--renew_version', nargs='?', const=hashlib.sha1, metavar='VERSION', help='Renew the versions of every exercise (default: the SHA1 hash of each exercise definition)')
parser.add_argument('-f', '--form_dir', nargs='?', const='DIR', help='Specify a target directory of form generation (defualt: the same as the directory of each master).')
parser.add_argument('-s', '--source', nargs='*', required=True, help=f'Specify source(s) (ipynb files in separate mode and directories in bundle mode)')
parser.add_argument('-gd', '--google_drive', nargs='?', const='DRIVE_JSON', help='Specify a JSON file of the Google Drive IDs/URLs of distributed forms.')
parser.add_argument('-ff', '--filled_form', nargs='?', const='form_filled_all.ipynb', help='Generate an all-filled form (default: form_filled_all.ipynb)')
parser.add_argument('-lp', '--library_placement', nargs='?', metavar='LIBDIR', const='.judge', help='Place judge_util.py for each exercise into LIBDIR (default: .judge).')
parser.add_argument('-bt', '--builtin_teststage', nargs='*', default=['rawcheck.py'], help='Specify module files of builtin test stages (default: rawcheck.py)')
commandline_options = parser.parse_args()
if commandline_options.verbose:
logging.getLogger().setLevel('DEBUG')
else:
logging.getLogger().setLevel('INFO')
Exercise.builtin_test_modules.extend(os.path.abspath(x) for x in commandline_options.builtin_teststage)
separates, bundles = load_sources(commandline_options.source)
all_exercises = list(itertools.chain(*bundles.values(), separates))
logging.info('[INFO] Cleaning up exercise masters...')
for ex in all_exercises:
cleanup_exercise_master(ex, commandline_options.renew_version)
deadlines = {}
if commandline_options.deadlines:
with open(commandline_options.deadlines, encoding='utf-8') as f:
deadlines = json.load(f)
drive = {}
if commandline_options.google_drive:
with open(commandline_options.google_drive, encoding='utf-8') as f:
drive = json.load(f)
if deadlines or drive:
update_exercise_master_metadata_formwise(separates, bundles, deadlines, drive)
logging.info('[INFO] Creating bundled forms...')
for dirpath, exercises in bundles.items():
cells, metadata = create_bundled_form(dirpath, exercises)
if commandline_options.form_dir:
filepath = os.path.join(commandline_options.form_dir, f'{os.path.basename(dirpath)}.ipynb')
else:
filepath = os.path.join(dirpath, f'form_{os.path.basename(dirpath)}.ipynb')
ipynb_util.save_as_notebook(filepath, cells, metadata)
logging.info(f'[INFO] Generated `{filepath}`')
logging.info('[INFO] Creating separate forms...')
for exercise in separates:
cells, metadata = create_separate_form(exercise)
if commandline_options.form_dir:
filepath = os.path.join(commandline_options.form_dir, f'{exercise.key}.ipynb')
else:
filepath = os.path.join(exercise.dirpath, f'form_{exercise.key}.ipynb')
ipynb_util.save_as_notebook(filepath, cells, metadata)
logging.info(f'[INFO] Generated `{filepath}`')
if commandline_options.library_placement:
import judge_util
for dirpath in {ex.dirpath for ex in all_exercises}:
dst = os.path.join(dirpath, commandline_options.library_placement)
os.makedirs(dst, exist_ok=True)
shutil.copy2(judge_util.__file__, dst)
logging.info(f'[INFO] Placed `{dst}/judge_util.py`')
if commandline_options.configuration:
judge_setting.load_judge_parameters(commandline_options.configuration)
logging.info(f'[INFO] Creating configuration with `{repr(judge_setting.judge_parameters)}` ...')
create_configuration(all_exercises)
if commandline_options.filled_form:
logging.info(f'[INFO] Creating filled form `{commandline_options.filled_form}` ...')
cells, metadata = create_filled_form(all_exercises)
ipynb_util.save_as_notebook(commandline_options.filled_form, cells, metadata)