def download(exam, out, name_question, sid_question, compact, dispatch=None):
exam_json = get_exam(exam=exam)
exam_json.pop("secret")
exam_json = json.dumps(exam_json)
out = out or "out/export/" + exam
pathlib.Path(out).mkdir(parents=True, exist_ok=True)
template_questions = list(extract_questions(json.loads(exam_json)))
pdf = write_exam(
{},
exam,
template_questions,
template_questions,
name_question,
sid_question,
compact,
dispatch,
)
pdf.output(os.path.join(out, "OUTLINE.pdf"))
total = [["Email"] + [
question["text"]
for question in extract_questions(json.loads(exam_json))
]]
for email, response in get_submissions(exam=exam):
if 1 < len(response) < 10:
print(email, response)
total.append([email])
for question in template_questions:
total[-1].append(response.get(question["id"], ""))
student_questions = list(
extract_questions(
scramble(email, json.loads(exam_json), keep_data=True)))
pdf = write_exam(
response,
exam,
template_questions,
student_questions,
name_question,
sid_question,
compact,
dispatch,
)
pdf.output(os.path.join(out, "{}.pdf".format(email)))
with open(os.path.join(out, "summary.csv"), "w") as f:
writer = csv.writer(f)
for row in total:
writer.writerow(row)
def find_unexpected_words(exam, logs):
data = get_exam(exam=exam)
exam_json = json.dumps(data)
original_questions = {
q["id"]: q
for q in extract_questions(json.loads(exam_json))
}
for i, (email, log) in enumerate(logs):
all_alternatives = get_substitutions(data)
scrambled_questions = {
q["id"]: q
for q in extract_questions(scramble(
email, json.loads(exam_json), keep_data=True),
nest_all=True)
}
flagged_questions = set()
for record in log:
record.pop("timestamp")
question = next(iter(record.keys()))
answer = next(iter(record.values()))
if question not in all_alternatives or question in flagged_questions:
continue
student_substitutions = scrambled_questions[question][
"substitutions"]
for keyword in student_substitutions:
for variant in all_alternatives[question][keyword]:
if variant == student_substitutions[keyword]:
continue
if variant in answer:
# check for false positives
if variant in scrambled_questions[question]["text"]:
continue
flagged_questions.add(question)
print(
"In question {}, Student {} used keyword {} for {}, when they should have used {}"
.format(
get_name(original_questions[question]),
email,
variant,
keyword,
student_substitutions[keyword],
))
print(
"\tThey wrote `{}`. Their substitutions were: {}".
format(" ".join(answer.split()),
student_substitutions))
def get_all_substitutions(original_exam, scrambled_exam):
assert (original_exam is not scrambled_exam
), "You must make a copy of the original before scrambling it"
original_questions = {q["id"]: q for q in extract_questions(original_exam)}
scrambled_questions = {
q["id"]: q
for q in extract_questions(scrambled_exam)
}
question_substitutions = {}
for question_id in scrambled_questions:
question_substitutions[question_id] = get_question_substitutions(
original_questions, scrambled_questions, question_id)
return question_substitutions
def find_unexpected_words(exam, logs):
data = get_exam(exam=exam)
exam_json = json.dumps(data)
original_questions = {q["id"]: q for q in extract_questions(json.loads(exam_json))}
suspected_cheating = []
for i, (email, log) in enumerate(tqdm(logs)):
all_alternatives = get_substitutions(data)
scrambled_questions = {
q["id"]: q
for q in extract_questions(
scramble(email, json.loads(exam_json), keep_data=True), nest_all=True
)
}
flagged_question_variants = set()
for record in log:
record.pop("timestamp")
for question, answer in record.items():
question = question.split("|")[0]
if question not in all_alternatives:
continue
student_substitutions = scrambled_questions[question]["substitutions"]
for keyword in student_substitutions:
for variant in all_alternatives[question][keyword]:
if variant == student_substitutions[keyword]:
continue
if (question, keyword, variant) in flagged_question_variants:
continue
if variant in answer:
# check for false positives
if variant in scrambled_questions[question]["text"]:
continue
flagged_question_variants.add((question, keyword, variant))
suspected_cheating.append(
SuspectedCheating(
get_name(original_questions[question]),
email,
keyword,
student_substitutions[keyword],
variant,
answer,
student_substitutions,
)
)
return suspected_cheating
def generate_gs_outline(self, grader: GS_assignment_Grader,
exam_json: dict, id_question_ids: [str]):
gs_number_to_exam_q = {}
questions = []
page = 2 # Page 1 is an info page
qid = 1
if exam_json.get("public"):
prev_page = 1
pg = GS_Outline_Question(
grader,
None,
[self.get_gs_crop_info(page, exam_json.get("public"))],
title="Public",
weight=0,
)
sqid = 1
for question in extract_public(exam_json):
question_id = question.get("id")
if question_id in id_question_ids:
print(f"Skipping {question_id} as it is an id question.")
page += 1 # Still need to increment this as it is still on the exam pdf.
continue
pg.add_child(
self.question_to_gso_question(grader, page, question))
gs_number_to_exam_q[f"{qid}.{sqid}"] = question
sqid += 1
page += 1
if page != prev_page and len(pg.children) > 0:
questions.append(pg)
qid += 1
for group in extract_groups(exam_json):
prev_page = page
weight = group.get("points", "0")
if not weight:
weight = 0
g = GS_Outline_Question(
grader,
None,
[self.get_gs_crop_info(page, group)],
title=group.get("name", ""),
weight=weight,
)
sqid = 1
for question in extract_questions(group,
extract_public_bool=False,
top_level=False):
g.add_child(
self.question_to_gso_question(grader, page, question))
gs_number_to_exam_q[f"{qid}.{sqid}"] = question
sqid += 1
page += 1
if page != prev_page:
questions.append(g)
qid += 1
outline = GS_Outline(self.name_region, self.sid_region, questions)
return (gs_number_to_exam_q, outline)
def find_unexpected_words(exam, logs):
data = get_exam(exam=exam)
exam_json = json.dumps(data)
for i, (email, log) in enumerate(logs):
all_alternatives = get_substitutions(data)
selected = {
question["id"]: question["substitutions"]
for question in extract_questions(
scramble(email, json.loads(exam_json), keep_data=True))
}
for record in log:
record.pop("timestamp")
question = next(iter(record.keys()))
answer = next(iter(record.values()))
if question not in all_alternatives:
continue
for keyword, variants in all_alternatives[question].items():
for variant in variants:
if variant == selected[question][keyword]:
continue
if variant in answer:
# check for false positive
for other in selected[question].values():
if variant in other and other != variant:
break
else:
print(email, selected[question], variant, answer)
break
else:
continue
break
else:
continue
break
def get_student_question_mapping(student, exam):
elements = list(extract_questions(exam, include_groups=True))
for element in elements:
element["id"] = element.get("id", rand_id()) # add IDs to groups
elements = {
element["id"]: get_name(element)
for element in elements
if element["type"] != "group" or not is_compressible_group(element)
}
old_seed = int(random.random() * 1000000)
out = [{
"student_question_name": get_name(element),
"canonical_question_name": elements[element["id"]],
} for element in list(
extract_questions(scramble(student, exam), include_groups=True))]
random.seed(old_seed)
return out
def get_all_substitutions(original_exam, scrambled_exam):
assert original_exam is not scrambled_exam, "You must make a copy of the original before scrambling it"
original_questions = {q["id"]: q for q in extract_questions(original_exam)}
scrambled_questions = {
q["id"]: q
for q in extract_questions(scrambled_exam)
}
question_substitutions = {}
for question_id in scrambled_questions:
original_question = original_questions[question_id]
substitutions = {}
for original, replacement in scrambled_questions[question_id][
"substitutions"].items():
if original in original_question["text"]:
substitutions[original] = replacement
question_substitutions[question_id] = get_question_substitutions(
original_questions, scrambled_questions, question_id)
return question_substitutions
def download(exam, emails_to_download: [str] = None, debug: bool = False):
exam_json = get_exam(exam=exam)
exam_json.pop("secret")
exam_json = json.dumps(exam_json)
template_questions = list(extract_questions(json.loads(exam_json)))
total = [["Email"] + [
question["text"]
for question in extract_questions(json.loads(exam_json))
]]
email_to_data_map = {}
if emails_to_download is None:
roster = get_roster(exam=exam)
emails_to_download = [email for email, _ in roster]
i = 1
for email, response in tqdm(get_submissions(exam=exam),
dynamic_ncols=True,
desc="Downloading",
unit="Exam"):
i += 1
if emails_to_download is not None and email not in emails_to_download:
continue
if debug and 1 < len(response) < 10:
tqdm.write(email, response)
total.append([email])
for question in template_questions:
total[-1].append(response.get(question["id"], ""))
student_questions = list(
extract_questions(
scramble(email, json.loads(exam_json), keep_data=True)))
email_to_data_map[email] = {
"student_questions": student_questions,
"responses": response,
}
return json.loads(exam_json), template_questions, email_to_data_map, total
def substitutions(exam, email, show_all):
"""
Show the substitutions a particular student received
"""
original_exam = get_exam(exam=exam)
exam = get_exam(exam=exam)
exam = scramble(email, exam, keep_data=True)
question_substitutions = get_all_substitutions(original_exam, exam)
questions = extract_questions(exam)
for question in questions:
substitutions = question_substitutions[question["id"]]
if substitutions or show_all:
print(get_name(question), substitutions)
def deploy(exam, json, roster, start_time, default_deadline):
"""
Deploy an exam to the website. You must specify an exam JSON and associated roster CSV.
You can deploy the JSON multiple times and the password will remain unchanged.
"""
json = json.read()
roster = csv.reader(roster, delimiter=",")
exam_content = loads(json)
exam_content["default_deadline"] = default_deadline
exam_content["secret"] = Fernet.generate_key().decode("utf-8")
try:
exam_content["secret"] = get_exam(exam=exam)["secret"]
except:
pass
set_exam(exam=exam, json=exam_content)
next(roster) # ditch headers
roster = list(roster)
set_roster(exam=exam, roster=roster)
print("Exam uploaded with password:"******"secret"][:-1])
print("Exam deployed to https://exam.cs61a.org/{}".format(exam))
print("Initializing announcements...")
elements = list(extract_questions(exam_content, include_groups=True))
for element in elements:
element["id"] = element.get("id", rand_id()) # add IDs to groups
elements = {
element["id"]: get_name(element)
for element in elements
if element["type"] != "group" or not is_compressible_group(element)
}
json = dumps(exam_content) # re-serialize with group IDs
students = [{
"email":
email,
"questions": [{
"start_time": start_time,
"end_time": int(deadline),
"student_question_name": get_name(element),
"canonical_question_name": elements[element["id"]],
} for element in list(
extract_questions(scramble(email, loads(json)),
include_groups=True))],
"start_time":
start_time,
"end_time":
int(deadline),
} for email, deadline in roster]
print("Updating announcements roster with {} students...".format(
len(students)))
for i in range(0, len(students), 100):
print("Uploading from student #{} to #{}".format(
i, min(i + 100, len(students))))
process_ok_exam_upload(
exam=exam,
data={
"students":
students[i:i + 100],
"questions": [{
"canonical_question_name": name
} for name in elements.values()],
},
clear=i == 0,
)
print("Announcements deployed to https://announcements.cs61a.org")
def index(request):
try:
if getenv("ENV") == "dev":
update_cache()
db = SafeFirestore()
if request.path.endswith("main.js"):
return main_js
if request.path.endswith("list_exams"):
return jsonify(
db.collection("exam-alerts").document("all").get().to_dict()
["exam-list"])
if request.path == "/" or request.json is None:
return main_html
exam = request.json["exam"]
course = exam.split("-")[0]
prev_latest_timestamp = float(request.json["latestTimestamp"])
def get_message(id):
message = (db.collection("exam-alerts").document(exam).collection(
"messages").document(id).get())
return {
**message.to_dict(),
"id": message.id,
}
student_reply = False
if request.path.endswith("ask_question"):
email = get_email(request)
student_question_name = request.json["question"]
message = request.json["message"]
student_data = get_student_data(db, email, exam)
if student_question_name is not None:
canonical_question_name = get_canonical_question_name(
student_data, student_question_name)
if canonical_question_name is None:
return abort(400)
else:
canonical_question_name = None
db.collection("exam-alerts").document(exam).collection(
"messages").document().set(
dict(
question=canonical_question_name,
message=message,
email=email,
timestamp=time.time(),
))
student_reply = True
if request.path.endswith("fetch_data") or student_reply:
received_audio = request.json.get("receivedAudio")
email = get_email(request)
exam_data = db.collection("exam-alerts").document(
exam).get().to_dict()
student_data = get_student_data(db, email, exam)
announcements = list(
db.collection("exam-alerts").document(exam).collection(
"announcements").stream())
messages = [{
**message.to_dict(), "id": message.id
} for message in (
db.collection("exam-alerts").document(exam).collection(
"messages").where("timestamp", ">", prev_latest_timestamp
).where("email", "==", email).stream())
if message.to_dict()["email"] == email]
messages, latest_timestamp = group_messages(messages, get_message)
messages = messages[email]
latest_timestamp = max(latest_timestamp, prev_latest_timestamp)
for message in messages:
if message["question"] is not None:
message["question"] = get_student_question_name(
student_data, message["question"])
return jsonify({
"success":
True,
"exam_type":
"ok-exam",
"enableClarifications":
exam_data.get("enable_clarifications", False),
"startTime":
student_data["start_time"],
"endTime":
student_data["end_time"],
"timestamp":
time.time(),
"questions": [
question["student_question_name"]
for question in student_data["questions"]
] if time.time() > student_data["start_time"] else [],
"announcements":
get_announcements(
student_data,
announcements,
messages,
received_audio,
lambda x: (db.collection("exam-alerts").document(
exam).collection("announcement_audio").document(x).get(
).to_dict() or {}).get("audio"),
),
"messages":
sorted(
[{
"id": message["id"],
"message": message["message"],
"timestamp": message["timestamp"],
"question": message["question"] or "Overall Exam",
"responses": message["responses"],
} for message in messages],
key=lambda message: message["timestamp"],
reverse=True,
),
"latestTimestamp":
latest_timestamp,
})
# only staff endpoints from here onwards
email = (get_email_from_secret(request.json["secret"])
if "secret" in request.json else get_email(request))
if not is_admin(email, course):
abort(401)
if request.path.endswith("fetch_staff_data"):
pass
elif request.path.endswith("add_announcement"):
announcement = request.json["announcement"]
announcement["timestamp"] = time.time()
ref = (db.collection("exam-alerts").document(exam).collection(
"announcements").document())
ref.set(announcement)
spoken_message = announcement.get("spoken_message",
announcement["message"])
if spoken_message:
audio = generate_audio(spoken_message)
db.collection("exam-alerts").document(exam).collection(
"announcement_audio").document(ref.id).set(
{"audio": audio})
elif request.path.endswith("clear_announcements"):
clear_collection(
db,
db.collection("exam-alerts").document(exam).collection(
"announcements"),
)
clear_collection(
db,
db.collection("exam-alerts").document(exam).collection(
"announcement_audio"),
)
elif request.path.endswith("delete_announcement"):
target = request.json["id"]
db.collection("exam-alerts").document(exam).collection(
"announcements").document(target).delete()
elif request.path.endswith("send_response"):
message_id = request.json["id"]
reply = request.json["reply"]
message = (db.collection("exam-alerts").document(exam).collection(
"messages").document(message_id).get())
ref = (db.collection("exam-alerts").document(exam).collection(
"messages").document())
ref.set({
"timestamp": time.time(),
"email": message.to_dict()["email"],
"reply_to": message.id,
"message": reply,
})
audio = generate_audio(
reply, prefix="A staff member sent the following reply: ")
db.collection("exam-alerts").document(exam).collection(
"announcement_audio").document(ref.id).set({"audio": audio})
elif request.path.endswith("get_question"):
question_title = request.json["id"]
student = request.json["student"]
student_data = get_student_data(db, student, exam)
question_title = get_student_question_name(student_data,
question_title)
exam = db.collection("exams").document(exam).get().to_dict()
questions = extract_questions(scramble(student, exam),
include_groups=True)
for question in questions:
if get_name(question).strip() == question_title.strip():
return jsonify({"success": True, "question": question})
abort(400)
else:
abort(404)
# (almost) all staff endpoints return an updated state
exam_data = db.collection("exam-alerts").document(exam).get().to_dict()
announcements = sorted(
({
"id": announcement.id,
**announcement.to_dict()
} for announcement in db.collection("exam-alerts").document(
exam).collection("announcements").stream()),
key=lambda announcement: announcement["timestamp"],
reverse=True,
)
grouped_messages, latest_timestamp = group_messages(
[{
**message.to_dict(), "id": message.id
} for message in db.collection(
"exam-alerts").document(exam).collection("messages").where(
"timestamp", ">", prev_latest_timestamp).stream()],
get_message,
)
latest_timestamp = max(prev_latest_timestamp, latest_timestamp)
messages = sorted(
[{
"email": email,
**message
} for email, messages in grouped_messages.items()
for message in messages],
key=lambda message: (
len(message["responses"]) > 0,
-message["timestamp"],
message["email"],
),
)
return jsonify({
"success": True,
"exam": exam_data,
"announcements": announcements,
"messages": messages,
"latestTimestamp": latest_timestamp,
})
except Exception as e:
if getenv("ENV") == "dev":
raise
print(e)
print(dict(request.json))
return jsonify({"success": False})
import warnings
warnings.filterwarnings(
"ignore", "Your application has authenticated using end user credentials")
db = firestore.Client()
exams = [x.id for x in db.collection("exams").stream()]
for exam in exams:
print("checking", exam)
exam_json = json.dumps(get_exam(exam=exam))
roster = get_roster(exam=exam)
flagged = set()
for email, _ in roster:
template_questions = extract_questions(json.loads(exam_json))
student_questions = list(
extract_questions(
scramble(email, json.loads(exam_json), keep_data=True)))
student_question_lookup = {q['id']: q for q in student_questions}
for question in template_questions:
if question["id"] not in student_question_lookup:
continue
if question["type"] not in ["multiple_choice", "select_all"]:
continue
if question["id"] in flagged:
continue
for i, option in enumerate(question["options"]):
option["index"] = i
def deploy(exam, json, roster, start_time, enable_clarifications):
"""
Deploy an exam to the website. You must specify an exam JSON and associated roster CSV.
You can deploy the JSON multiple times and the password will remain unchanged.
"""
json = json.read()
roster = csv.reader(roster, delimiter=",")
exam_content = loads(json)
exam_content["default_deadline"] = 0
exam_content["secret"] = Fernet.generate_key().decode("utf-8")
try:
old_secret = get_exam(exam=exam)["secret"]
if old_secret:
print("Reusing old secret...")
exam_content["secret"] = old_secret
except Exception:
pass
set_exam(exam=exam, json=exam_content)
roster = list(roster)
if not verify_roster(roster=roster):
exit(1)
roster = roster[1:] # ditch headers
set_roster(exam=exam, roster=roster)
print("Exam uploaded with password:"******"secret"][:-1])
print("Exam deployed to https://exam.cs61a.org/{}".format(exam))
print("Initializing announcements...")
elements = list(extract_questions(exam_content, include_groups=True))
for element in elements:
element["id"] = element.get("id", rand_id()) # add IDs to groups
elements = {
element["id"]: get_name(element)
for element in elements
if element["type"] != "group" or not is_compressible_group(element)
}
students = [
{
"email": email,
"start_time": start_time,
"end_time": int(deadline),
"no_watermark": bool(int(rest[0]) if rest else False),
}
for email, deadline, *rest in roster
]
print("Updating announcements roster with {} students...".format(len(students)))
process_ok_exam_upload(
exam=exam,
data={
"students": students,
"questions": [
{"canonical_question_name": name} for name in elements.values()
],
},
clear=True,
enable_clarifications=enable_clarifications,
)
print("Announcements deployed to https://announcements.cs61a.org")
