def create_course_class(group: ClassGroup, i: int, raw_group: dict, raw_semester: str, teachers: List[str]): teacher = parse_teacher(teachers[i]) try: semester = parse_lesson_semester(raw_semester).value except Exception as e: semester = group.occurrence.semester try: day = parse_day_of_week(raw_group["day"][i]).value except Exception as e: log.warning(f"Skipping day: {e}") day = DayOfWeek.UNDEFINED.value try: start_time, end_time = parse_times(raw_group["hour"][i]) except Exception as e: log.warning(f"Skipping times: {e}") start_time, end_time = None, None try: hall = parse_hall(raw_group["hall"][i]) except Exception as e: log.warning(f"Skipping hall: {e}") hall = None course_class, created = CourseClass.objects.get_or_create( group=group, teacher=teacher, semester=semester, day=day, start_time=start_time, end_time=end_time, hall=hall, ) if created: log.info(f"Class {course_class.id} created")
def create_degree_program(blocks): log.info("Creating program") cs_plan = DegreeProgram.objects.get_or_create(name="מדעי המחשב מורחב", code=3010, credits=134)[0] for block in blocks: cs_plan.blocks.add(block) cs_plan.save()
def fetch_courses(): log.info("Fetching courses") parser = ShnatonParser() for course_number in COURSES_TO_FETCH: parser.fetch_course(course_number) for block in UNPARSED_BLOCKS: for course in block[2]: parser.fetch_course(course)
def get_context_data(self, **kwargs): log.info(f"Rendering {wrap(self.title)} for {wrap(self.user)}") context = super().get_context_data(**kwargs) context["title"] = self.title # context["toasts"] = [ # Toast("משהו טוב קרה", delay=2500).as_json(), # Toast("משהו לא טוב קרה", delay=3000, tag=ToastTag.WARNING).as_json(), ] return context
def create_degree_program(blocks): log.info("Creating program") cs_plan = DegreeProgram.objects.get_or_create(name="Mock CS", code=1993, credits=39)[0] for block in blocks: cs_plan.blocks.add(block) cs_plan.save()
def fetch_courses(): log.info("Fetching courses") parser = ShnatonParser() for course_number in courses_to_fetch: parser.fetch_course(course_number) for block in unparsed_blocks: for course in block[2]: parser.fetch_course(course)
def del_user_schedule_groups(user, group_ids): log.info( f"del_user_schedule_group for user {wrap(user)} with group_ids {wrap(group_ids)}" ) if user.is_anonymous: log.warning("del_user_schedule_group called but user is_anonymous") raise UserNotLoggedInError() ClassSchedule.objects.filter(user=user, group_id__in=group_ids).delete() log.info(f"del_user_schedule_group deleted {wrap(group_ids)}")
def init_sentry(): from academic_helper.utils.logger import log if not DSN: log.info("Sentry DSN env var not found") return log.info("Configuring Sentry") sentry_sdk.init(dsn=DSN, integrations=[DjangoIntegration()], environment=ENV.value, send_default_pii=True) sentry_sdk.integrations.logging.ignore_logger("django.security.DisallowedHost")
def search(text: str, department: str = "", faculty: str = ""): log.info(f"Searching for {wrap(text)}, department {wrap(department)}, faculty {wrap(faculty)}") courses = Course.objects.filter( (Q(name__contains=text) | Q(course_number__icontains=text)) & Q(department__name__contains=department) & Q(department__faculty__name__contains=faculty) ) if len(text) >= 3: courses |= search_teacher(text) return courses
def fetch_courses(): University.objects.get_or_create( abbreviation="HUJI", name="האוניברסיטה העברית", english_name="The Hebrew University of Jerusalem") log.info("Fetching courses") parser = ShnatonParser() for course_number in COURSES_TO_FETCH: parser.fetch_course(course_number) for block in UNPARSED_BLOCKS: for course in block[2]: try: parser.fetch_course(course) except Exception as e: log.error(e)
def post(self, request: WSGIRequest, *args, **kwargs): if not request.is_ajax(): raise NotImplementedError() text = request.POST["free_text"] school = request.POST["school"] faculty = request.POST["faculty"] log.info( f"Searching for {text}, school {school}, faculty {faculty}...") queryset = Course.find_by(text, school, faculty)[:35] result = [c.as_dict for c in queryset] result.sort(key=lambda c: c["score"], reverse=True) for course in result: course["url"] = reverse("course-details", args=[course["course_number"]]) course["score"] = floatformat(course["score"]) return JsonResponse({"courses": result})
def create_blocks(): log.info("Creating blocks") study_blocks = [] for unparsed_block in UNPARSED_BLOCKS: # extract info name, min_credits, course_nums = unparsed_block # create block block = StudyBlock.objects.get_or_create(name=name, min_credits=min_credits)[0] # add courses to block for course_num in course_nums: course = Course.objects.get(course_number=course_num) block.courses.add(course) # add block to study blocks block.save() study_blocks.append(block) return study_blocks
def create_course_class(group: ClassGroup, i: int, raw_group: dict, raw_semester: str, teacher: Teacher): try: semester = parse_lesson_semester(raw_semester).value except Exception: semester = group.occurrence.semester try: day = parse_day_of_week(raw_group["day"][i]).value except Exception as e: log.warning(f"Skipping day: {e}") day = DayOfWeek.UNDEFINED.value try: start_time, end_time = parse_times(raw_group["hour"][i]) except Exception as e: log.warning(f"Skipping times: {e}") start_time, end_time = None, None try: hall = parse_hall(raw_group["hall"][i]) except Exception as e: log.warning(f"Skipping hall: {e}") hall = None special_occurrence = None notes = None if len(raw_group["special_occurrences"]) > i: raw = raw_group["special_occurrences"][i] try: special_occurrence = datetime.strptime(raw, "%d/%m/%y").date() except Exception as e: log.warning(f"Skipping special occurrence: {e}") notes = raw course_class, created = CourseClass.objects.get_or_create( group=group, semester=semester, day=day, start_time=start_time, end_time=end_time, hall=hall, teacher=teacher, special_occurrence=special_occurrence, ) if created: log.info(f"Class {course_class.id} created") course_class.notes = notes course_class.save() return course_class
def create_course_groups(course: Course, year: int, course_semesters: List[Semester], occurrence_credits: int, raw_group: dict): group_mark = raw_group["group"].replace(" ", "") group_class_type = parse_group_type(raw_group["type"]).value class_num = len(raw_group["semester"]) teachers = expand_teacher_list(raw_group["lecturer"], class_num) try: group_semester = parse_group_semester(raw_group["semester"]).value except Exception as e: log.info(f"No group semester: {e}") group_semester = None occurrence = occurrence_for_semester(course, year, occurrence_credits, group_semester, course_semesters) group, created = ClassGroup.objects.get_or_create( occurrence=occurrence, class_type=group_class_type, mark=group_mark) if created: log.info(f"Group {group.id} created") # Add classes to group for i, raw_semester in enumerate(raw_group["semester"]): create_course_class(group, i, raw_group, raw_semester, teachers)
def create_course_class(group: ClassGroup, i, raw_group, raw_semester, teachers): # TODO: This does not handle 2 teachers for 1 group case (course 1920) teacher = parse_teacher(teachers[i]) try: semester = parse_lesson_semester(raw_semester).value except Exception as e: semester = group.occurrence.semester try: day = parse_day_of_week(raw_group["day"][i]).value except Exception as e: log.warning( f"Skipping day for course {group.occurrence.course.course_number}: {e}" ) day = DayOfWeek.UNDEFINED.value try: start_time, end_time = parse_times(raw_group["hour"][i]) except Exception as e: log.warning( f"Skipping times for course {group.occurrence.course.course_number}: {e}" ) start_time, end_time = None, None try: hall = parse_hall(raw_group["hall"][i]) except Exception as e: log.warning( f"Skipping hall for course {group.occurrence.course.course_number}: {e}" ) hall = None course_class, created = CourseClass.objects.get_or_create( group=group, teacher=teacher, semester=semester, day=day, start_time=start_time, end_time=end_time, hall=hall, ) if created: log.info(f"Class {course_class.id} created")
def get_course_html(year, course_id, use_mock=True): mock_dir = path.join("academic_helper", "shnaton_mock") if not path.exists(mock_dir): os.makedirs(mock_dir) mock_path = path.join(mock_dir, f"{course_id}-{year}.html") if use_mock and path.exists(mock_path): with open(mock_path, encoding="utf-8") as file: log.info(f"Reading mock for {course_id} year {year}") return file.read() data = urllib.parse.urlencode({ "peula": "Simple", "maslul": "0", "shana": "0", "year": year, "course": course_id }).encode("utf-8") # TODO maybe windows-1255 req = urllib.request.urlopen(url=SERVER_URL, data=data) html = req.read().decode(req.headers.get_content_charset()) with open(mock_path, "w", encoding="utf-8") as file: log.info(f"Writing mock for {course_id} year {year}") file.write(html) return html
def _fetch_course(self, course_number: int, year: int): log.info( f"Fetch course called for number {wrap(course_number)} and year {wrap(year)}" ) if not isinstance(course_number, int): course_number = int(course_number) raw_data = self.extract_data_from_shnaton(year, course_number) if raw_data is None: raise FetchRawDataError("No raw data could be parsed") raw_faculty = raw_data["faculty"].strip(" :\t") raw_department = raw_data["department"].strip(" :\t") faculty = Faculty.objects.get_or_create(name=raw_faculty)[0] department = Department.objects.get_or_create(name=raw_department, faculty=faculty)[0] raw_course_number = int(raw_data["id"]) if raw_course_number != course_number: raise ShnatonParserError( f"Course numbers mismatch: given {wrap(course_number)}, parsed {wrap(raw_course_number)}" ) raw_course_name = raw_data["name"].replace("_", "") # if "name_en" in raw_data and len(raw_data["name_en"].replace(" ", "")) > 5: # course_name = raw_data["name_en"] course = Course.objects.get_or_create(name=raw_course_name, course_number=course_number, department=department)[0] course_semesters = parse_course_semester(raw_data["semester"]) occurrence_credits = parse_course_credits(year, raw_data) for raw_group in raw_data["lessons"]: create_course_groups(course, year, course_semesters, occurrence_credits, raw_data["notes"], raw_group) return course
def _fetch_course(self, course_number: int, year: int): log.info(f"Fetch course called for number {wrap(course_number)} and year {wrap(year)}") if not isinstance(course_number, int): course_number = int(course_number) raw_data = self.extract_data_from_shnaton(year, course_number) if raw_data is None: raise FetchRawDataError("No raw data could be parsed") raw_faculty = raw_data["faculty"].strip(" :\t") raw_department = raw_data["department"].strip(" :\t") huji = University.objects.get_or_create( abbreviation="HUJI", name="האוניברסיטה העברית", english_name="The Hebrew University of Jerusalem" )[0] faculty = Faculty.objects.get_or_create(name=raw_faculty, university=huji)[0] department = Department.objects.get_or_create(name=raw_department, faculty=faculty)[0] raw_course_number = int(raw_data["id"]) if raw_course_number != course_number: raise ShnatonParserError( f"Course numbers mismatch: given {wrap(course_number)}, parsed {wrap(raw_course_number)}" ) raw_course_name = raw_data["name"].replace("_", "") # if "name_en" in raw_data and len(raw_data["name_en"].replace(" ", "")) > 5: # course_name = raw_data["name_en"] course, created = Course.objects.get_or_create(course_number=course_number, university=huji) course.department = department course.name = raw_course_name course.save() course_semesters = parse_course_semester(raw_data["semester"]) occurrence_credits = parse_course_credits(year, raw_data) for raw_group in raw_data["lessons"]: create_course_groups(course, year, course_semesters, occurrence_credits, raw_data["notes"], raw_group) return course
def handle_uni_default(apps, schema_editor): log.info(f"Creating default university") def_uni = University(name="Default university", english_name="Default university", abbreviation="DU") def_uni.save() for course in Course.objects.all().values("id", "university_id"): if not course.university_id: log.info(f"Changing university for course {wrap(course.id)}") course.university = def_uni course.save() else: log.info( f"Course {wrap(course.id)} already had university: ${wrap(course.university)}" ) for faculty in Faculty.objects.all(): if not faculty.university_id: log.info(f"Changing university for faculty {wrap(faculty.id)}") faculty.university = def_uni faculty.save() else: log.info( f"Faculty {wrap(faculty.id)} already had university: ${wrap(faculty.university)}" )
def post(self, request: WSGIRequest, *args, **kwargs): log.info("We are in POST") if request.is_ajax(): value = "Hi " + request.POST["value"] log.info(f"User sent {value} via Ajax") return JsonResponse({"success": True, "value": value}) else: value = request.POST["post-text"] log.info(f"User sent {value} via POST") return self.render_to_response(context={"value": value})
def create_course_groups( course: Course, year: int, course_semesters: List[Semester], occurrence_credits: int, occurrence_notes: str, raw_group: dict, ): group_mark = raw_group["group"] if group_mark is not None: group_mark = group_mark.replace(" ", "") group_class_type = parse_group_type(raw_group["type"]).value teachers = parse_teachers(raw_group["lecturer"]) try: group_semester = parse_group_semester(raw_group["semester"]).value except Exception as e: log.info(f"No group semester: {e}") group_semester = None occurrence = occurrence_for_semester(course, year, occurrence_credits, group_semester, course_semesters, occurrence_notes) group, created = ClassGroup.objects.get_or_create( occurrence=occurrence, class_type=group_class_type, mark=group_mark) if teachers and created: group.teachers.add(*teachers) if not created and teachers and set(teachers) != set(group.teachers.all()): group.teachers.set(teachers) log.info(f"Group {group.id} was updated") if created: log.info(f"Group {group.id} created") # Add classes to group first_teacher = None if not teachers else teachers[0] old_classes = set(CourseClass.objects.filter(group=group)) new_classes = set() for i, raw_semester in enumerate(raw_group["semester"]): course_class = create_course_class(group, i, raw_group, raw_semester, first_teacher) new_classes.add(course_class) irrelevant_classes = old_classes - new_classes for c in irrelevant_classes: log.info(f"Class {c.id} is deleted") c.delete()
def get_course_html(self, year: int, course_number: int) -> str: cache_path = path.join(self.cache_dir, f"{course_number}-{year}.html") if self.cache_read and path.exists(cache_path): log.info("Cache read is on and file already exist, reading") with open(cache_path, encoding=CHARSET) as file: return file.read() log.info("Cache read is off or file does not exist yet") data = urllib.parse.urlencode( {"peula": "Simple", "maslul": "0", "shana": "0", "year": year, "course": course_number} ).encode("utf-8") response = urllib.request.urlopen(url=self.shnaton_url, data=data) html = response.read().decode(response.headers.get_content_charset()) if self.cache_write: with open(cache_path, "w", encoding=CHARSET) as file: log.info(f"Writing html cache") file.write(html) else: log.info("Skipping cache write") return html
def move_course(user_id, course_id, block_id): log.info( f"Moving course {wrap(course_id)} into block {wrap(block_id)} for user {wrap(user_id)}" ) choice, created = UserCourseChoice.objects.get_or_create( user_id=user_id, course_id=course_id) if created: log.info(f"Choice created") else: log.info( f"Choice already existed at block {wrap(choice.block_id)}") choice.block_id = block_id choice.save()
def handle(self, *args, **options): log.info(f"Fetching courses with: {options}") with open(options["src_file"], encoding="utf8") as file: courses = json.load(file) if options["shuffle"]: random.shuffle(courses) else: courses.sort(key=lambda c: c["id"]) fail_count = 0 log.info(f"Total {wrap(len(courses))} courses found") parser = ShnatonParser() limit = min(options["limit"], len(courses)) for i, course in enumerate(courses): if i >= options["limit"]: break course_number = course["id"] log.info(f"Course {wrap(i + 1)} out of {wrap(limit)} is {wrap(course_number)}") try: parser.fetch_course(course_number) except Exception as e: log.error(f"Could'nt fetch course {course_number}: {e}") sentry_sdk.capture_exception(e) fail_count += 1 log.info(f"Fail count: {wrap(fail_count)} out of {wrap(limit)}")
def set_user_schedule_group(user, group_id): log.info( f"set_user_schedule_group for user {wrap(user)} with group_id {wrap(group_id)}" ) if user.is_anonymous: # log.warning("set_user_schedule_group called but user is_anonymous") raise UserNotLoggedInError() group = ClassGroup.objects.get(pk=group_id) existing = ClassSchedule.objects.filter( user=user, group__occurrence__course=group.occurrence.course, group__class_type=group.class_type).first() if existing: log.info( f"set_user_schedule_group replacing existing {wrap(existing.group.id)}" ) existing.delete() schedule = ClassSchedule(user=user, group=group) schedule.save() log.info(f"set_user_schedule_group added {wrap(group.id)}")
def init_sentry(): from academic_helper.utils.logger import log if ENV == Environment.local: log.info("Env set to local, not configuring sentry") return if not DSN: log.info("Sentry DSN env var not found") return log.info("Configuring Sentry") integrations = [DjangoIntegration()] if ENV != Environment.prod: integrations += [LoggingIntegration(event_level=None)] sentry_sdk.init( dsn=DSN, integrations=integrations, environment=ENV.value, send_default_pii=True, ) sentry_sdk.integrations.logging.ignore_logger( "django.security.DisallowedHost")
def handle(self, *args, **options): log.info(f"Fetching courses with: {options}") with open(options["src_file"], encoding="utf8") as file: courses = json.load(file) if SHUFFLE: random.shuffle(courses) fail_count = 0 log.info(f"Total {wrap(len(courses))} courses found") for i, course in enumerate(courses): if i > options["limit"]: break course_number = course["id"] existing = Course.objects.filter(course_number=course_number) if existing.exists(): if not options["fetch_existing"]: continue existing.delete() try: ShnatonParser.fetch_course(course_number) except Exception as e: log.error(f"Could'nt fetch course {course_number}: {e}") fail_count += 1 log.info(f"Fail count: {wrap(fail_count)}")
def save(self, *args, **kwargs): super().save(*args, **kwargs) log.info(f"Comment saved: {wrap(self)}")
def save(self, *args, **kwargs): super().save(*args, **kwargs) log.info(f"Saving user: {wrap(self)}")
def do(self): log.info(f"{self.code} cron is up.") self.job()