def download(exam, out, name_question, sid_question, compact, dispatch=None): exam_json = get_exam(exam=exam) exam_json.pop("secret") exam_json = json.dumps(exam_json) out = out or "out/export/" + exam pathlib.Path(out).mkdir(parents=True, exist_ok=True) template_questions = list(extract_questions(json.loads(exam_json))) pdf = write_exam( {}, exam, template_questions, template_questions, name_question, sid_question, compact, dispatch, ) pdf.output(os.path.join(out, "OUTLINE.pdf")) total = [["Email"] + [ question["text"] for question in extract_questions(json.loads(exam_json)) ]] for email, response in get_submissions(exam=exam): if 1 < len(response) < 10: print(email, response) total.append([email]) for question in template_questions: total[-1].append(response.get(question["id"], "")) student_questions = list( extract_questions( scramble(email, json.loads(exam_json), keep_data=True))) pdf = write_exam( response, exam, template_questions, student_questions, name_question, sid_question, compact, dispatch, ) pdf.output(os.path.join(out, "{}.pdf".format(email))) with open(os.path.join(out, "summary.csv"), "w") as f: writer = csv.writer(f) for row in total: writer.writerow(row)
def find_unexpected_words(exam, logs): data = get_exam(exam=exam) exam_json = json.dumps(data) original_questions = { q["id"]: q for q in extract_questions(json.loads(exam_json)) } for i, (email, log) in enumerate(logs): all_alternatives = get_substitutions(data) scrambled_questions = { q["id"]: q for q in extract_questions(scramble( email, json.loads(exam_json), keep_data=True), nest_all=True) } flagged_questions = set() for record in log: record.pop("timestamp") question = next(iter(record.keys())) answer = next(iter(record.values())) if question not in all_alternatives or question in flagged_questions: continue student_substitutions = scrambled_questions[question][ "substitutions"] for keyword in student_substitutions: for variant in all_alternatives[question][keyword]: if variant == student_substitutions[keyword]: continue if variant in answer: # check for false positives if variant in scrambled_questions[question]["text"]: continue flagged_questions.add(question) print( "In question {}, Student {} used keyword {} for {}, when they should have used {}" .format( get_name(original_questions[question]), email, variant, keyword, student_substitutions[keyword], )) print( "\tThey wrote `{}`. Their substitutions were: {}". format(" ".join(answer.split()), student_substitutions))
def get_all_substitutions(original_exam, scrambled_exam): assert (original_exam is not scrambled_exam ), "You must make a copy of the original before scrambling it" original_questions = {q["id"]: q for q in extract_questions(original_exam)} scrambled_questions = { q["id"]: q for q in extract_questions(scrambled_exam) } question_substitutions = {} for question_id in scrambled_questions: question_substitutions[question_id] = get_question_substitutions( original_questions, scrambled_questions, question_id) return question_substitutions
def find_unexpected_words(exam, logs): data = get_exam(exam=exam) exam_json = json.dumps(data) original_questions = {q["id"]: q for q in extract_questions(json.loads(exam_json))} suspected_cheating = [] for i, (email, log) in enumerate(tqdm(logs)): all_alternatives = get_substitutions(data) scrambled_questions = { q["id"]: q for q in extract_questions( scramble(email, json.loads(exam_json), keep_data=True), nest_all=True ) } flagged_question_variants = set() for record in log: record.pop("timestamp") for question, answer in record.items(): question = question.split("|")[0] if question not in all_alternatives: continue student_substitutions = scrambled_questions[question]["substitutions"] for keyword in student_substitutions: for variant in all_alternatives[question][keyword]: if variant == student_substitutions[keyword]: continue if (question, keyword, variant) in flagged_question_variants: continue if variant in answer: # check for false positives if variant in scrambled_questions[question]["text"]: continue flagged_question_variants.add((question, keyword, variant)) suspected_cheating.append( SuspectedCheating( get_name(original_questions[question]), email, keyword, student_substitutions[keyword], variant, answer, student_substitutions, ) ) return suspected_cheating
def generate_gs_outline(self, grader: GS_assignment_Grader, exam_json: dict, id_question_ids: [str]): gs_number_to_exam_q = {} questions = [] page = 2 # Page 1 is an info page qid = 1 if exam_json.get("public"): prev_page = 1 pg = GS_Outline_Question( grader, None, [self.get_gs_crop_info(page, exam_json.get("public"))], title="Public", weight=0, ) sqid = 1 for question in extract_public(exam_json): question_id = question.get("id") if question_id in id_question_ids: print(f"Skipping {question_id} as it is an id question.") page += 1 # Still need to increment this as it is still on the exam pdf. continue pg.add_child( self.question_to_gso_question(grader, page, question)) gs_number_to_exam_q[f"{qid}.{sqid}"] = question sqid += 1 page += 1 if page != prev_page and len(pg.children) > 0: questions.append(pg) qid += 1 for group in extract_groups(exam_json): prev_page = page weight = group.get("points", "0") if not weight: weight = 0 g = GS_Outline_Question( grader, None, [self.get_gs_crop_info(page, group)], title=group.get("name", ""), weight=weight, ) sqid = 1 for question in extract_questions(group, extract_public_bool=False, top_level=False): g.add_child( self.question_to_gso_question(grader, page, question)) gs_number_to_exam_q[f"{qid}.{sqid}"] = question sqid += 1 page += 1 if page != prev_page: questions.append(g) qid += 1 outline = GS_Outline(self.name_region, self.sid_region, questions) return (gs_number_to_exam_q, outline)
def find_unexpected_words(exam, logs): data = get_exam(exam=exam) exam_json = json.dumps(data) for i, (email, log) in enumerate(logs): all_alternatives = get_substitutions(data) selected = { question["id"]: question["substitutions"] for question in extract_questions( scramble(email, json.loads(exam_json), keep_data=True)) } for record in log: record.pop("timestamp") question = next(iter(record.keys())) answer = next(iter(record.values())) if question not in all_alternatives: continue for keyword, variants in all_alternatives[question].items(): for variant in variants: if variant == selected[question][keyword]: continue if variant in answer: # check for false positive for other in selected[question].values(): if variant in other and other != variant: break else: print(email, selected[question], variant, answer) break else: continue break else: continue break
def get_student_question_mapping(student, exam): elements = list(extract_questions(exam, include_groups=True)) for element in elements: element["id"] = element.get("id", rand_id()) # add IDs to groups elements = { element["id"]: get_name(element) for element in elements if element["type"] != "group" or not is_compressible_group(element) } old_seed = int(random.random() * 1000000) out = [{ "student_question_name": get_name(element), "canonical_question_name": elements[element["id"]], } for element in list( extract_questions(scramble(student, exam), include_groups=True))] random.seed(old_seed) return out
def get_all_substitutions(original_exam, scrambled_exam): assert original_exam is not scrambled_exam, "You must make a copy of the original before scrambling it" original_questions = {q["id"]: q for q in extract_questions(original_exam)} scrambled_questions = { q["id"]: q for q in extract_questions(scrambled_exam) } question_substitutions = {} for question_id in scrambled_questions: original_question = original_questions[question_id] substitutions = {} for original, replacement in scrambled_questions[question_id][ "substitutions"].items(): if original in original_question["text"]: substitutions[original] = replacement question_substitutions[question_id] = get_question_substitutions( original_questions, scrambled_questions, question_id) return question_substitutions
def download(exam, emails_to_download: [str] = None, debug: bool = False): exam_json = get_exam(exam=exam) exam_json.pop("secret") exam_json = json.dumps(exam_json) template_questions = list(extract_questions(json.loads(exam_json))) total = [["Email"] + [ question["text"] for question in extract_questions(json.loads(exam_json)) ]] email_to_data_map = {} if emails_to_download is None: roster = get_roster(exam=exam) emails_to_download = [email for email, _ in roster] i = 1 for email, response in tqdm(get_submissions(exam=exam), dynamic_ncols=True, desc="Downloading", unit="Exam"): i += 1 if emails_to_download is not None and email not in emails_to_download: continue if debug and 1 < len(response) < 10: tqdm.write(email, response) total.append([email]) for question in template_questions: total[-1].append(response.get(question["id"], "")) student_questions = list( extract_questions( scramble(email, json.loads(exam_json), keep_data=True))) email_to_data_map[email] = { "student_questions": student_questions, "responses": response, } return json.loads(exam_json), template_questions, email_to_data_map, total
def substitutions(exam, email, show_all): """ Show the substitutions a particular student received """ original_exam = get_exam(exam=exam) exam = get_exam(exam=exam) exam = scramble(email, exam, keep_data=True) question_substitutions = get_all_substitutions(original_exam, exam) questions = extract_questions(exam) for question in questions: substitutions = question_substitutions[question["id"]] if substitutions or show_all: print(get_name(question), substitutions)
def deploy(exam, json, roster, start_time, default_deadline): """ Deploy an exam to the website. You must specify an exam JSON and associated roster CSV. You can deploy the JSON multiple times and the password will remain unchanged. """ json = json.read() roster = csv.reader(roster, delimiter=",") exam_content = loads(json) exam_content["default_deadline"] = default_deadline exam_content["secret"] = Fernet.generate_key().decode("utf-8") try: exam_content["secret"] = get_exam(exam=exam)["secret"] except: pass set_exam(exam=exam, json=exam_content) next(roster) # ditch headers roster = list(roster) set_roster(exam=exam, roster=roster) print("Exam uploaded with password:"******"secret"][:-1]) print("Exam deployed to https://exam.cs61a.org/{}".format(exam)) print("Initializing announcements...") elements = list(extract_questions(exam_content, include_groups=True)) for element in elements: element["id"] = element.get("id", rand_id()) # add IDs to groups elements = { element["id"]: get_name(element) for element in elements if element["type"] != "group" or not is_compressible_group(element) } json = dumps(exam_content) # re-serialize with group IDs students = [{ "email": email, "questions": [{ "start_time": start_time, "end_time": int(deadline), "student_question_name": get_name(element), "canonical_question_name": elements[element["id"]], } for element in list( extract_questions(scramble(email, loads(json)), include_groups=True))], "start_time": start_time, "end_time": int(deadline), } for email, deadline in roster] print("Updating announcements roster with {} students...".format( len(students))) for i in range(0, len(students), 100): print("Uploading from student #{} to #{}".format( i, min(i + 100, len(students)))) process_ok_exam_upload( exam=exam, data={ "students": students[i:i + 100], "questions": [{ "canonical_question_name": name } for name in elements.values()], }, clear=i == 0, ) print("Announcements deployed to https://announcements.cs61a.org")
def index(request): try: if getenv("ENV") == "dev": update_cache() db = SafeFirestore() if request.path.endswith("main.js"): return main_js if request.path.endswith("list_exams"): return jsonify( db.collection("exam-alerts").document("all").get().to_dict() ["exam-list"]) if request.path == "/" or request.json is None: return main_html exam = request.json["exam"] course = exam.split("-")[0] prev_latest_timestamp = float(request.json["latestTimestamp"]) def get_message(id): message = (db.collection("exam-alerts").document(exam).collection( "messages").document(id).get()) return { **message.to_dict(), "id": message.id, } student_reply = False if request.path.endswith("ask_question"): email = get_email(request) student_question_name = request.json["question"] message = request.json["message"] student_data = get_student_data(db, email, exam) if student_question_name is not None: canonical_question_name = get_canonical_question_name( student_data, student_question_name) if canonical_question_name is None: return abort(400) else: canonical_question_name = None db.collection("exam-alerts").document(exam).collection( "messages").document().set( dict( question=canonical_question_name, message=message, email=email, timestamp=time.time(), )) student_reply = True if request.path.endswith("fetch_data") or student_reply: received_audio = request.json.get("receivedAudio") email = get_email(request) exam_data = db.collection("exam-alerts").document( exam).get().to_dict() student_data = get_student_data(db, email, exam) announcements = list( db.collection("exam-alerts").document(exam).collection( "announcements").stream()) messages = [{ **message.to_dict(), "id": message.id } for message in ( db.collection("exam-alerts").document(exam).collection( "messages").where("timestamp", ">", prev_latest_timestamp ).where("email", "==", email).stream()) if message.to_dict()["email"] == email] messages, latest_timestamp = group_messages(messages, get_message) messages = messages[email] latest_timestamp = max(latest_timestamp, prev_latest_timestamp) for message in messages: if message["question"] is not None: message["question"] = get_student_question_name( student_data, message["question"]) return jsonify({ "success": True, "exam_type": "ok-exam", "enableClarifications": exam_data.get("enable_clarifications", False), "startTime": student_data["start_time"], "endTime": student_data["end_time"], "timestamp": time.time(), "questions": [ question["student_question_name"] for question in student_data["questions"] ] if time.time() > student_data["start_time"] else [], "announcements": get_announcements( student_data, announcements, messages, received_audio, lambda x: (db.collection("exam-alerts").document( exam).collection("announcement_audio").document(x).get( ).to_dict() or {}).get("audio"), ), "messages": sorted( [{ "id": message["id"], "message": message["message"], "timestamp": message["timestamp"], "question": message["question"] or "Overall Exam", "responses": message["responses"], } for message in messages], key=lambda message: message["timestamp"], reverse=True, ), "latestTimestamp": latest_timestamp, }) # only staff endpoints from here onwards email = (get_email_from_secret(request.json["secret"]) if "secret" in request.json else get_email(request)) if not is_admin(email, course): abort(401) if request.path.endswith("fetch_staff_data"): pass elif request.path.endswith("add_announcement"): announcement = request.json["announcement"] announcement["timestamp"] = time.time() ref = (db.collection("exam-alerts").document(exam).collection( "announcements").document()) ref.set(announcement) spoken_message = announcement.get("spoken_message", announcement["message"]) if spoken_message: audio = generate_audio(spoken_message) db.collection("exam-alerts").document(exam).collection( "announcement_audio").document(ref.id).set( {"audio": audio}) elif request.path.endswith("clear_announcements"): clear_collection( db, db.collection("exam-alerts").document(exam).collection( "announcements"), ) clear_collection( db, db.collection("exam-alerts").document(exam).collection( "announcement_audio"), ) elif request.path.endswith("delete_announcement"): target = request.json["id"] db.collection("exam-alerts").document(exam).collection( "announcements").document(target).delete() elif request.path.endswith("send_response"): message_id = request.json["id"] reply = request.json["reply"] message = (db.collection("exam-alerts").document(exam).collection( "messages").document(message_id).get()) ref = (db.collection("exam-alerts").document(exam).collection( "messages").document()) ref.set({ "timestamp": time.time(), "email": message.to_dict()["email"], "reply_to": message.id, "message": reply, }) audio = generate_audio( reply, prefix="A staff member sent the following reply: ") db.collection("exam-alerts").document(exam).collection( "announcement_audio").document(ref.id).set({"audio": audio}) elif request.path.endswith("get_question"): question_title = request.json["id"] student = request.json["student"] student_data = get_student_data(db, student, exam) question_title = get_student_question_name(student_data, question_title) exam = db.collection("exams").document(exam).get().to_dict() questions = extract_questions(scramble(student, exam), include_groups=True) for question in questions: if get_name(question).strip() == question_title.strip(): return jsonify({"success": True, "question": question}) abort(400) else: abort(404) # (almost) all staff endpoints return an updated state exam_data = db.collection("exam-alerts").document(exam).get().to_dict() announcements = sorted( ({ "id": announcement.id, **announcement.to_dict() } for announcement in db.collection("exam-alerts").document( exam).collection("announcements").stream()), key=lambda announcement: announcement["timestamp"], reverse=True, ) grouped_messages, latest_timestamp = group_messages( [{ **message.to_dict(), "id": message.id } for message in db.collection( "exam-alerts").document(exam).collection("messages").where( "timestamp", ">", prev_latest_timestamp).stream()], get_message, ) latest_timestamp = max(prev_latest_timestamp, latest_timestamp) messages = sorted( [{ "email": email, **message } for email, messages in grouped_messages.items() for message in messages], key=lambda message: ( len(message["responses"]) > 0, -message["timestamp"], message["email"], ), ) return jsonify({ "success": True, "exam": exam_data, "announcements": announcements, "messages": messages, "latestTimestamp": latest_timestamp, }) except Exception as e: if getenv("ENV") == "dev": raise print(e) print(dict(request.json)) return jsonify({"success": False})
import warnings warnings.filterwarnings( "ignore", "Your application has authenticated using end user credentials") db = firestore.Client() exams = [x.id for x in db.collection("exams").stream()] for exam in exams: print("checking", exam) exam_json = json.dumps(get_exam(exam=exam)) roster = get_roster(exam=exam) flagged = set() for email, _ in roster: template_questions = extract_questions(json.loads(exam_json)) student_questions = list( extract_questions( scramble(email, json.loads(exam_json), keep_data=True))) student_question_lookup = {q['id']: q for q in student_questions} for question in template_questions: if question["id"] not in student_question_lookup: continue if question["type"] not in ["multiple_choice", "select_all"]: continue if question["id"] in flagged: continue for i, option in enumerate(question["options"]): option["index"] = i
def deploy(exam, json, roster, start_time, enable_clarifications): """ Deploy an exam to the website. You must specify an exam JSON and associated roster CSV. You can deploy the JSON multiple times and the password will remain unchanged. """ json = json.read() roster = csv.reader(roster, delimiter=",") exam_content = loads(json) exam_content["default_deadline"] = 0 exam_content["secret"] = Fernet.generate_key().decode("utf-8") try: old_secret = get_exam(exam=exam)["secret"] if old_secret: print("Reusing old secret...") exam_content["secret"] = old_secret except Exception: pass set_exam(exam=exam, json=exam_content) roster = list(roster) if not verify_roster(roster=roster): exit(1) roster = roster[1:] # ditch headers set_roster(exam=exam, roster=roster) print("Exam uploaded with password:"******"secret"][:-1]) print("Exam deployed to https://exam.cs61a.org/{}".format(exam)) print("Initializing announcements...") elements = list(extract_questions(exam_content, include_groups=True)) for element in elements: element["id"] = element.get("id", rand_id()) # add IDs to groups elements = { element["id"]: get_name(element) for element in elements if element["type"] != "group" or not is_compressible_group(element) } students = [ { "email": email, "start_time": start_time, "end_time": int(deadline), "no_watermark": bool(int(rest[0]) if rest else False), } for email, deadline, *rest in roster ] print("Updating announcements roster with {} students...".format(len(students))) process_ok_exam_upload( exam=exam, data={ "students": students, "questions": [ {"canonical_question_name": name} for name in elements.values() ], }, clear=True, enable_clarifications=enable_clarifications, ) print("Announcements deployed to https://announcements.cs61a.org")