def substitutions(exam, email, show_all):
"""
Show the substitutions a particular student received
"""
original_exam = get_exam(exam=exam)
exam = get_exam(exam=exam)
exam = scramble(email, exam, keep_data=True)
question_substitutions = get_all_substitutions(original_exam, exam)
questions = extract_questions(exam)
for question in questions:
substitutions = question_substitutions[question["id"]]
if substitutions or show_all:
print(get_name(question), substitutions)
def find_keyword(exam, phrase):
data = get_exam(exam=exam)
exam_json = json.dumps(data)
for email, _ in get_roster(exam=exam):
scrambled = scramble(email, json.loads(exam_json))
if phrase in json.dumps(scrambled):
print(email)
def find_unexpected_words(exam, logs):
data = get_exam(exam=exam)
exam_json = json.dumps(data)
for i, (email, log) in enumerate(logs):
all_alternatives = get_substitutions(data)
selected = {
question["id"]: question["substitutions"]
for question in extract_questions(
scramble(email, json.loads(exam_json), keep_data=True))
}
for record in log:
record.pop("timestamp")
question = next(iter(record.keys()))
answer = next(iter(record.values()))
if question not in all_alternatives:
continue
for keyword, variants in all_alternatives[question].items():
for variant in variants:
if variant == selected[question][keyword]:
continue
if variant in answer:
# check for false positive
for other in selected[question].values():
if variant in other and other != variant:
break
else:
print(email, selected[question], variant, answer)
break
else:
continue
break
else:
continue
break
def identify_watermark(exam, image):
"""
Identify the student from a screenshot containing a watermark.
"""
img = cv2.imread(image)
img = cv2.copyMakeBorder(img, 100, 100, 100, 100, cv2.BORDER_CONSTANT)
corners = []
bits = []
def handle_click(event, x, y, flags, params):
if event == cv2.EVENT_LBUTTONDOWN:
bits.append(Point(x, y))
cv2.circle(img, (x, y), 5, (255, 0, 0), -1)
if event == cv2.EVENT_RBUTTONDOWN:
corners.append(Point(x, y))
cv2.circle(img, (x, y), 5, (0, 255, 0), -1)
cv2.namedWindow("image")
cv2.setMouseCallback("image", handle_click)
while True:
cv2.imshow("image", img)
if cv2.waitKey(20) & 0xFF == 13:
break
print(
decode_watermark(get_exam(exam=exam), get_roster(exam=exam), corners,
bits))
def compile_all(exam, subtitle, out, do_twice, email, exam_type, semester,
deadline):
"""
Compile individualized PDFs for the specified exam.
Exam must have been deployed first.
"""
if not out:
out = "out/latex/" + exam
pathlib.Path(out).mkdir(parents=True, exist_ok=True)
exam_data = get_exam(exam=exam)
password = exam_data.pop("secret")[:-1]
print(password)
exam_str = json.dumps(exam_data)
roster = get_roster(exam=exam)
if email:
roster = [line_info for line_info in roster if line_info[0] == email]
if len(roster) == 0:
if deadline:
roster = [(email, deadline)]
else:
raise ValueError("Email does not exist in the roster!")
for email, deadline in roster:
if not deadline:
continue
exam_data = json.loads(exam_str)
scramble(email, exam_data)
deadline_utc = datetime.utcfromtimestamp(int(deadline))
deadline_pst = pytz.utc.localize(deadline_utc).astimezone(
pytz.timezone("America/Los_Angeles"))
deadline_string = deadline_pst.strftime("%I:%M%p")
with render_latex(
exam_data,
{
"emailaddress": sanitize_email(email),
"deadline": deadline_string,
"coursecode": prettify(exam.split("-")[0]),
"description": subtitle,
"examtype": exam_type,
"semester": semester,
},
do_twice=do_twice,
) as pdf:
pdf = Pdf.open(BytesIO(pdf))
pdf.save(
os.path.join(
out, "exam_" + email.replace("@", "_").replace(".", "_") +
".pdf"),
encryption=Encryption(owner=password, user=password),
)
pdf.close()
def download(exam, out, name_question, sid_question, compact, dispatch=None):
exam_json = get_exam(exam=exam)
exam_json.pop("secret")
exam_json = json.dumps(exam_json)
out = out or "out/export/" + exam
pathlib.Path(out).mkdir(parents=True, exist_ok=True)
template_questions = list(extract_questions(json.loads(exam_json)))
pdf = write_exam(
{},
exam,
template_questions,
template_questions,
name_question,
sid_question,
compact,
dispatch,
)
pdf.output(os.path.join(out, "OUTLINE.pdf"))
total = [["Email"] + [
question["text"]
for question in extract_questions(json.loads(exam_json))
]]
for email, response in get_submissions(exam=exam):
if 1 < len(response) < 10:
print(email, response)
total.append([email])
for question in template_questions:
total[-1].append(response.get(question["id"], ""))
student_questions = list(
extract_questions(
scramble(email, json.loads(exam_json), keep_data=True)))
pdf = write_exam(
response,
exam,
template_questions,
student_questions,
name_question,
sid_question,
compact,
dispatch,
)
pdf.output(os.path.join(out, "{}.pdf".format(email)))
with open(os.path.join(out, "summary.csv"), "w") as f:
writer = csv.writer(f)
for row in total:
writer.writerow(row)
def find_unexpected_words(exam, logs):
data = get_exam(exam=exam)
exam_json = json.dumps(data)
original_questions = {
q["id"]: q
for q in extract_questions(json.loads(exam_json))
}
for i, (email, log) in enumerate(logs):
all_alternatives = get_substitutions(data)
scrambled_questions = {
q["id"]: q
for q in extract_questions(scramble(
email, json.loads(exam_json), keep_data=True),
nest_all=True)
}
flagged_questions = set()
for record in log:
record.pop("timestamp")
question = next(iter(record.keys()))
answer = next(iter(record.values()))
if question not in all_alternatives or question in flagged_questions:
continue
student_substitutions = scrambled_questions[question][
"substitutions"]
for keyword in student_substitutions:
for variant in all_alternatives[question][keyword]:
if variant == student_substitutions[keyword]:
continue
if variant in answer:
# check for false positives
if variant in scrambled_questions[question]["text"]:
continue
flagged_questions.add(question)
print(
"In question {}, Student {} used keyword {} for {}, when they should have used {}"
.format(
get_name(original_questions[question]),
email,
variant,
keyword,
student_substitutions[keyword],
))
print(
"\tThey wrote `{}`. Their substitutions were: {}".
format(" ".join(answer.split()),
student_substitutions))
def find_unexpected_words(exam, logs):
data = get_exam(exam=exam)
exam_json = json.dumps(data)
original_questions = {q["id"]: q for q in extract_questions(json.loads(exam_json))}
suspected_cheating = []
for i, (email, log) in enumerate(tqdm(logs)):
all_alternatives = get_substitutions(data)
scrambled_questions = {
q["id"]: q
for q in extract_questions(
scramble(email, json.loads(exam_json), keep_data=True), nest_all=True
)
}
flagged_question_variants = set()
for record in log:
record.pop("timestamp")
for question, answer in record.items():
question = question.split("|")[0]
if question not in all_alternatives:
continue
student_substitutions = scrambled_questions[question]["substitutions"]
for keyword in student_substitutions:
for variant in all_alternatives[question][keyword]:
if variant == student_substitutions[keyword]:
continue
if (question, keyword, variant) in flagged_question_variants:
continue
if variant in answer:
# check for false positives
if variant in scrambled_questions[question]["text"]:
continue
flagged_question_variants.add((question, keyword, variant))
suspected_cheating.append(
SuspectedCheating(
get_name(original_questions[question]),
email,
keyword,
student_substitutions[keyword],
variant,
answer,
student_substitutions,
)
)
return suspected_cheating
def download(exam, emails_to_download: [str] = None, debug: bool = False):
exam_json = get_exam(exam=exam)
exam_json.pop("secret")
exam_json = json.dumps(exam_json)
template_questions = list(extract_questions(json.loads(exam_json)))
total = [["Email"] + [
question["text"]
for question in extract_questions(json.loads(exam_json))
]]
email_to_data_map = {}
if emails_to_download is None:
roster = get_roster(exam=exam)
emails_to_download = [email for email, _ in roster]
i = 1
for email, response in tqdm(get_submissions(exam=exam),
dynamic_ncols=True,
desc="Downloading",
unit="Exam"):
i += 1
if emails_to_download is not None and email not in emails_to_download:
continue
if debug and 1 < len(response) < 10:
tqdm.write(email, response)
total.append([email])
for question in template_questions:
total[-1].append(response.get(question["id"], ""))
student_questions = list(
extract_questions(
scramble(email, json.loads(exam_json), keep_data=True)))
email_to_data_map[email] = {
"student_questions": student_questions,
"responses": response,
}
return json.loads(exam_json), template_questions, email_to_data_map, total
def compile(exam, json, md, seed, json_out, out):
"""
Compile one PDF or JSON (from Markdown), unencrypted.
The exam may be deployed or local (in Markdown or JSON).
If a seed is specified, it will scramble the exam.
"""
if not out:
out = ""
pathlib.Path(out).mkdir(parents=True, exist_ok=True)
if json:
print("Loading exam...")
exam_data = load(json)
elif md:
print("Compiling exam...")
exam_text_data = md.read()
exam_data = convert(exam_text_data)
else:
print("Fetching exam...")
exam_data = get_exam(exam=exam)
if seed:
print("Scrambling exam...")
exam_data = scramble(
seed,
exam_data,
)
if json_out:
print("Dumping json...")
dump(exam_data, json_out)
return
print("Rendering exam...")
with render_latex(exam_data, {
"coursecode": prettify(exam.split("-")[0]),
"description": "Sample Exam."
}) as pdf:
pdf = Pdf.open(BytesIO(pdf))
pdf.save(os.path.join(out, exam + ".pdf"))
pdf.close()
def deploy(exam, json, roster, default_deadline):
"""
Deploy an exam to the website. You must specify an exam JSON and associated roster CSV.
You can deploy the JSON multiple times and the password will remain unchanged.
"""
json = json.read()
roster = csv.reader(roster, delimiter=",")
json = loads(json)
json["default_deadline"] = default_deadline
json["secret"] = Fernet.generate_key().decode("utf-8")
try:
json["secret"] = get_exam(exam=exam)["secret"]
except:
pass
set_exam(exam=exam, json=json)
next(roster) # ditch headers
set_roster(exam=exam, roster=list(roster))
print("Exam uploaded with password:"******"secret"][:-1])
def deploy(exam, json, roster, start_time, default_deadline):
"""
Deploy an exam to the website. You must specify an exam JSON and associated roster CSV.
You can deploy the JSON multiple times and the password will remain unchanged.
"""
json = json.read()
roster = csv.reader(roster, delimiter=",")
exam_content = loads(json)
exam_content["default_deadline"] = default_deadline
exam_content["secret"] = Fernet.generate_key().decode("utf-8")
try:
exam_content["secret"] = get_exam(exam=exam)["secret"]
except:
pass
set_exam(exam=exam, json=exam_content)
next(roster) # ditch headers
roster = list(roster)
set_roster(exam=exam, roster=roster)
print("Exam uploaded with password:"******"secret"][:-1])
print("Exam deployed to https://exam.cs61a.org/{}".format(exam))
print("Initializing announcements...")
elements = list(extract_questions(exam_content, include_groups=True))
for element in elements:
element["id"] = element.get("id", rand_id()) # add IDs to groups
elements = {
element["id"]: get_name(element)
for element in elements
if element["type"] != "group" or not is_compressible_group(element)
}
json = dumps(exam_content) # re-serialize with group IDs
students = [{
"email":
email,
"questions": [{
"start_time": start_time,
"end_time": int(deadline),
"student_question_name": get_name(element),
"canonical_question_name": elements[element["id"]],
} for element in list(
extract_questions(scramble(email, loads(json)),
include_groups=True))],
"start_time":
start_time,
"end_time":
int(deadline),
} for email, deadline in roster]
print("Updating announcements roster with {} students...".format(
len(students)))
for i in range(0, len(students), 100):
print("Uploading from student #{} to #{}".format(
i, min(i + 100, len(students))))
process_ok_exam_upload(
exam=exam,
data={
"students":
students[i:i + 100],
"questions": [{
"canonical_question_name": name
} for name in elements.values()],
},
clear=i == 0,
)
print("Announcements deployed to https://announcements.cs61a.org")
import json
from examtool.api.database import get_exam, get_roster
from examtool.api.extract_questions import extract_questions
from examtool.api.scramble import scramble
from google.cloud import firestore
import warnings
warnings.filterwarnings(
"ignore", "Your application has authenticated using end user credentials")
db = firestore.Client()
exams = [x.id for x in db.collection("exams").stream()]
for exam in exams:
print("checking", exam)
exam_json = json.dumps(get_exam(exam=exam))
roster = get_roster(exam=exam)
flagged = set()
for email, _ in roster:
template_questions = extract_questions(json.loads(exam_json))
student_questions = list(
extract_questions(
scramble(email, json.loads(exam_json), keep_data=True)))
student_question_lookup = {q['id']: q for q in student_questions}
for question in template_questions:
if question["id"] not in student_question_lookup:
continue
if question["type"] not in ["multiple_choice", "select_all"]:
continue
def compile(
exam,
json,
md,
seed,
subtitle,
with_solutions,
exam_type,
semester,
json_out,
merged_md,
draft,
out,
):
"""
Compile one PDF or JSON (from Markdown), unencrypted.
The exam may be deployed or local (in Markdown or JSON).
If a seed is specified, it will scramble the exam.
"""
if not out:
out = ""
pathlib.Path(out).mkdir(parents=True, exist_ok=True)
if json:
print("Loading exam...")
exam_data = load(json)
elif md:
exam_text_data = md.read()
if merged_md:
buff = LineBuffer(exam_text_data)
handle_imports(buff, path=os.path.dirname(md.name))
merged_md.write("\n".join(buff.lines))
return
print("Compiling exam...")
exam_data = convert(exam_text_data, path=os.path.dirname(md.name), draft=draft)
else:
print("Fetching exam...")
exam_data = get_exam(exam=exam)
if seed:
print("Scrambling exam...")
exam_data = scramble(seed, exam_data, keep_data=with_solutions)
def remove_solutions_from_groups(groups):
for group in groups:
# if isinstance(group, dict):
group.pop("solution", None)
if group.get("type") == "group":
remove_solutions_from_groups(group.get("elements", []))
if not seed and not with_solutions:
print("Removing solutions...")
groups = exam_data.get("groups", [])
remove_solutions_from_groups(groups)
if json_out:
print("Dumping json...")
dump(exam_data, json_out, indent=4, sort_keys=True)
return
print("Rendering exam...")
settings = {
"coursecode": prettify(exam.split("-")[0]),
"description": subtitle,
"examtype": exam_type,
"semester": semester,
}
if seed:
settings["emailaddress"] = sanitize_email(seed)
with render_latex(exam_data, settings) as pdf:
pdf = Pdf.open(BytesIO(pdf))
pdf.save(os.path.join(out, exam + ".pdf"))
pdf.close()
def deploy(exam, json, roster, start_time, enable_clarifications):
"""
Deploy an exam to the website. You must specify an exam JSON and associated roster CSV.
You can deploy the JSON multiple times and the password will remain unchanged.
"""
json = json.read()
roster = csv.reader(roster, delimiter=",")
exam_content = loads(json)
exam_content["default_deadline"] = 0
exam_content["secret"] = Fernet.generate_key().decode("utf-8")
try:
old_secret = get_exam(exam=exam)["secret"]
if old_secret:
print("Reusing old secret...")
exam_content["secret"] = old_secret
except Exception:
pass
set_exam(exam=exam, json=exam_content)
roster = list(roster)
if not verify_roster(roster=roster):
exit(1)
roster = roster[1:] # ditch headers
set_roster(exam=exam, roster=roster)
print("Exam uploaded with password:"******"secret"][:-1])
print("Exam deployed to https://exam.cs61a.org/{}".format(exam))
print("Initializing announcements...")
elements = list(extract_questions(exam_content, include_groups=True))
for element in elements:
element["id"] = element.get("id", rand_id()) # add IDs to groups
elements = {
element["id"]: get_name(element)
for element in elements
if element["type"] != "group" or not is_compressible_group(element)
}
students = [
{
"email": email,
"start_time": start_time,
"end_time": int(deadline),
"no_watermark": bool(int(rest[0]) if rest else False),
}
for email, deadline, *rest in roster
]
print("Updating announcements roster with {} students...".format(len(students)))
process_ok_exam_upload(
exam=exam,
data={
"students": students,
"questions": [
{"canonical_question_name": name} for name in elements.values()
],
},
clear=True,
enable_clarifications=enable_clarifications,
)
print("Announcements deployed to https://announcements.cs61a.org")
