def create_course_class(group: ClassGroup, i: int, raw_group: dict,
raw_semester: str, teachers: List[str]):
teacher = parse_teacher(teachers[i])
try:
semester = parse_lesson_semester(raw_semester).value
except Exception as e:
semester = group.occurrence.semester
try:
day = parse_day_of_week(raw_group["day"][i]).value
except Exception as e:
log.warning(f"Skipping day: {e}")
day = DayOfWeek.UNDEFINED.value
try:
start_time, end_time = parse_times(raw_group["hour"][i])
except Exception as e:
log.warning(f"Skipping times: {e}")
start_time, end_time = None, None
try:
hall = parse_hall(raw_group["hall"][i])
except Exception as e:
log.warning(f"Skipping hall: {e}")
hall = None
course_class, created = CourseClass.objects.get_or_create(
group=group,
teacher=teacher,
semester=semester,
day=day,
start_time=start_time,
end_time=end_time,
hall=hall,
)
if created:
log.info(f"Class {course_class.id} created")
def create_degree_program(blocks):
log.info("Creating program")
cs_plan = DegreeProgram.objects.get_or_create(name="מדעי המחשב מורחב",
code=3010,
credits=134)[0]
for block in blocks:
cs_plan.blocks.add(block)
cs_plan.save()
def fetch_courses():
log.info("Fetching courses")
parser = ShnatonParser()
for course_number in COURSES_TO_FETCH:
parser.fetch_course(course_number)
for block in UNPARSED_BLOCKS:
for course in block[2]:
parser.fetch_course(course)
def get_context_data(self, **kwargs):
log.info(f"Rendering {wrap(self.title)} for {wrap(self.user)}")
context = super().get_context_data(**kwargs)
context["title"] = self.title
# context["toasts"] = [
# Toast("משהו טוב קרה", delay=2500).as_json(),
# Toast("משהו לא טוב קרה", delay=3000, tag=ToastTag.WARNING).as_json(), ]
return context
def create_degree_program(blocks):
log.info("Creating program")
cs_plan = DegreeProgram.objects.get_or_create(name="Mock CS",
code=1993,
credits=39)[0]
for block in blocks:
cs_plan.blocks.add(block)
cs_plan.save()
def fetch_courses():
log.info("Fetching courses")
parser = ShnatonParser()
for course_number in courses_to_fetch:
parser.fetch_course(course_number)
for block in unparsed_blocks:
for course in block[2]:
parser.fetch_course(course)
def del_user_schedule_groups(user, group_ids):
log.info(
f"del_user_schedule_group for user {wrap(user)} with group_ids {wrap(group_ids)}"
)
if user.is_anonymous:
log.warning("del_user_schedule_group called but user is_anonymous")
raise UserNotLoggedInError()
ClassSchedule.objects.filter(user=user, group_id__in=group_ids).delete()
log.info(f"del_user_schedule_group deleted {wrap(group_ids)}")
def init_sentry():
from academic_helper.utils.logger import log
if not DSN:
log.info("Sentry DSN env var not found")
return
log.info("Configuring Sentry")
sentry_sdk.init(dsn=DSN, integrations=[DjangoIntegration()], environment=ENV.value, send_default_pii=True)
sentry_sdk.integrations.logging.ignore_logger("django.security.DisallowedHost")
def search(text: str, department: str = "", faculty: str = ""):
log.info(f"Searching for {wrap(text)}, department {wrap(department)}, faculty {wrap(faculty)}")
courses = Course.objects.filter(
(Q(name__contains=text) | Q(course_number__icontains=text))
& Q(department__name__contains=department)
& Q(department__faculty__name__contains=faculty)
)
if len(text) >= 3:
courses |= search_teacher(text)
return courses
def fetch_courses():
University.objects.get_or_create(
abbreviation="HUJI",
name="האוניברסיטה העברית",
english_name="The Hebrew University of Jerusalem")
log.info("Fetching courses")
parser = ShnatonParser()
for course_number in COURSES_TO_FETCH:
parser.fetch_course(course_number)
for block in UNPARSED_BLOCKS:
for course in block[2]:
try:
parser.fetch_course(course)
except Exception as e:
log.error(e)
def post(self, request: WSGIRequest, *args, **kwargs):
if not request.is_ajax():
raise NotImplementedError()
text = request.POST["free_text"]
school = request.POST["school"]
faculty = request.POST["faculty"]
log.info(
f"Searching for {text}, school {school}, faculty {faculty}...")
queryset = Course.find_by(text, school, faculty)[:35]
result = [c.as_dict for c in queryset]
result.sort(key=lambda c: c["score"], reverse=True)
for course in result:
course["url"] = reverse("course-details",
args=[course["course_number"]])
course["score"] = floatformat(course["score"])
return JsonResponse({"courses": result})
def create_blocks():
log.info("Creating blocks")
study_blocks = []
for unparsed_block in UNPARSED_BLOCKS:
# extract info
name, min_credits, course_nums = unparsed_block
# create block
block = StudyBlock.objects.get_or_create(name=name,
min_credits=min_credits)[0]
# add courses to block
for course_num in course_nums:
course = Course.objects.get(course_number=course_num)
block.courses.add(course)
# add block to study blocks
block.save()
study_blocks.append(block)
return study_blocks
def create_course_class(group: ClassGroup, i: int, raw_group: dict,
raw_semester: str, teacher: Teacher):
try:
semester = parse_lesson_semester(raw_semester).value
except Exception:
semester = group.occurrence.semester
try:
day = parse_day_of_week(raw_group["day"][i]).value
except Exception as e:
log.warning(f"Skipping day: {e}")
day = DayOfWeek.UNDEFINED.value
try:
start_time, end_time = parse_times(raw_group["hour"][i])
except Exception as e:
log.warning(f"Skipping times: {e}")
start_time, end_time = None, None
try:
hall = parse_hall(raw_group["hall"][i])
except Exception as e:
log.warning(f"Skipping hall: {e}")
hall = None
special_occurrence = None
notes = None
if len(raw_group["special_occurrences"]) > i:
raw = raw_group["special_occurrences"][i]
try:
special_occurrence = datetime.strptime(raw, "%d/%m/%y").date()
except Exception as e:
log.warning(f"Skipping special occurrence: {e}")
notes = raw
course_class, created = CourseClass.objects.get_or_create(
group=group,
semester=semester,
day=day,
start_time=start_time,
end_time=end_time,
hall=hall,
teacher=teacher,
special_occurrence=special_occurrence,
)
if created:
log.info(f"Class {course_class.id} created")
course_class.notes = notes
course_class.save()
return course_class
def create_course_groups(course: Course, year: int,
course_semesters: List[Semester],
occurrence_credits: int, raw_group: dict):
group_mark = raw_group["group"].replace(" ", "")
group_class_type = parse_group_type(raw_group["type"]).value
class_num = len(raw_group["semester"])
teachers = expand_teacher_list(raw_group["lecturer"], class_num)
try:
group_semester = parse_group_semester(raw_group["semester"]).value
except Exception as e:
log.info(f"No group semester: {e}")
group_semester = None
occurrence = occurrence_for_semester(course, year, occurrence_credits,
group_semester, course_semesters)
group, created = ClassGroup.objects.get_or_create(
occurrence=occurrence, class_type=group_class_type, mark=group_mark)
if created:
log.info(f"Group {group.id} created")
# Add classes to group
for i, raw_semester in enumerate(raw_group["semester"]):
create_course_class(group, i, raw_group, raw_semester, teachers)
def create_course_class(group: ClassGroup, i, raw_group, raw_semester,
teachers):
# TODO: This does not handle 2 teachers for 1 group case (course 1920)
teacher = parse_teacher(teachers[i])
try:
semester = parse_lesson_semester(raw_semester).value
except Exception as e:
semester = group.occurrence.semester
try:
day = parse_day_of_week(raw_group["day"][i]).value
except Exception as e:
log.warning(
f"Skipping day for course {group.occurrence.course.course_number}: {e}"
)
day = DayOfWeek.UNDEFINED.value
try:
start_time, end_time = parse_times(raw_group["hour"][i])
except Exception as e:
log.warning(
f"Skipping times for course {group.occurrence.course.course_number}: {e}"
)
start_time, end_time = None, None
try:
hall = parse_hall(raw_group["hall"][i])
except Exception as e:
log.warning(
f"Skipping hall for course {group.occurrence.course.course_number}: {e}"
)
hall = None
course_class, created = CourseClass.objects.get_or_create(
group=group,
teacher=teacher,
semester=semester,
day=day,
start_time=start_time,
end_time=end_time,
hall=hall,
)
if created:
log.info(f"Class {course_class.id} created")
def get_course_html(year, course_id, use_mock=True):
mock_dir = path.join("academic_helper", "shnaton_mock")
if not path.exists(mock_dir):
os.makedirs(mock_dir)
mock_path = path.join(mock_dir, f"{course_id}-{year}.html")
if use_mock and path.exists(mock_path):
with open(mock_path, encoding="utf-8") as file:
log.info(f"Reading mock for {course_id} year {year}")
return file.read()
data = urllib.parse.urlencode({
"peula": "Simple",
"maslul": "0",
"shana": "0",
"year": year,
"course": course_id
}).encode("utf-8") # TODO maybe windows-1255
req = urllib.request.urlopen(url=SERVER_URL, data=data)
html = req.read().decode(req.headers.get_content_charset())
with open(mock_path, "w", encoding="utf-8") as file:
log.info(f"Writing mock for {course_id} year {year}")
file.write(html)
return html
def _fetch_course(self, course_number: int, year: int):
log.info(
f"Fetch course called for number {wrap(course_number)} and year {wrap(year)}"
)
if not isinstance(course_number, int):
course_number = int(course_number)
raw_data = self.extract_data_from_shnaton(year, course_number)
if raw_data is None:
raise FetchRawDataError("No raw data could be parsed")
raw_faculty = raw_data["faculty"].strip(" :\t")
raw_department = raw_data["department"].strip(" :\t")
faculty = Faculty.objects.get_or_create(name=raw_faculty)[0]
department = Department.objects.get_or_create(name=raw_department,
faculty=faculty)[0]
raw_course_number = int(raw_data["id"])
if raw_course_number != course_number:
raise ShnatonParserError(
f"Course numbers mismatch: given {wrap(course_number)}, parsed {wrap(raw_course_number)}"
)
raw_course_name = raw_data["name"].replace("_", "")
# if "name_en" in raw_data and len(raw_data["name_en"].replace(" ", "")) > 5:
# course_name = raw_data["name_en"]
course = Course.objects.get_or_create(name=raw_course_name,
course_number=course_number,
department=department)[0]
course_semesters = parse_course_semester(raw_data["semester"])
occurrence_credits = parse_course_credits(year, raw_data)
for raw_group in raw_data["lessons"]:
create_course_groups(course, year, course_semesters,
occurrence_credits, raw_data["notes"],
raw_group)
return course
def _fetch_course(self, course_number: int, year: int):
log.info(f"Fetch course called for number {wrap(course_number)} and year {wrap(year)}")
if not isinstance(course_number, int):
course_number = int(course_number)
raw_data = self.extract_data_from_shnaton(year, course_number)
if raw_data is None:
raise FetchRawDataError("No raw data could be parsed")
raw_faculty = raw_data["faculty"].strip(" :\t")
raw_department = raw_data["department"].strip(" :\t")
huji = University.objects.get_or_create(
abbreviation="HUJI", name="האוניברסיטה העברית", english_name="The Hebrew University of Jerusalem"
)[0]
faculty = Faculty.objects.get_or_create(name=raw_faculty, university=huji)[0]
department = Department.objects.get_or_create(name=raw_department, faculty=faculty)[0]
raw_course_number = int(raw_data["id"])
if raw_course_number != course_number:
raise ShnatonParserError(
f"Course numbers mismatch: given {wrap(course_number)}, parsed {wrap(raw_course_number)}"
)
raw_course_name = raw_data["name"].replace("_", "")
# if "name_en" in raw_data and len(raw_data["name_en"].replace(" ", "")) > 5:
# course_name = raw_data["name_en"]
course, created = Course.objects.get_or_create(course_number=course_number, university=huji)
course.department = department
course.name = raw_course_name
course.save()
course_semesters = parse_course_semester(raw_data["semester"])
occurrence_credits = parse_course_credits(year, raw_data)
for raw_group in raw_data["lessons"]:
create_course_groups(course, year, course_semesters, occurrence_credits, raw_data["notes"], raw_group)
return course
def handle_uni_default(apps, schema_editor):
log.info(f"Creating default university")
def_uni = University(name="Default university",
english_name="Default university",
abbreviation="DU")
def_uni.save()
for course in Course.objects.all().values("id", "university_id"):
if not course.university_id:
log.info(f"Changing university for course {wrap(course.id)}")
course.university = def_uni
course.save()
else:
log.info(
f"Course {wrap(course.id)} already had university: ${wrap(course.university)}"
)
for faculty in Faculty.objects.all():
if not faculty.university_id:
log.info(f"Changing university for faculty {wrap(faculty.id)}")
faculty.university = def_uni
faculty.save()
else:
log.info(
f"Faculty {wrap(faculty.id)} already had university: ${wrap(faculty.university)}"
)
def post(self, request: WSGIRequest, *args, **kwargs):
log.info("We are in POST")
if request.is_ajax():
value = "Hi " + request.POST["value"]
log.info(f"User sent {value} via Ajax")
return JsonResponse({"success": True, "value": value})
else:
value = request.POST["post-text"]
log.info(f"User sent {value} via POST")
return self.render_to_response(context={"value": value})
def create_course_groups(
course: Course,
year: int,
course_semesters: List[Semester],
occurrence_credits: int,
occurrence_notes: str,
raw_group: dict,
):
group_mark = raw_group["group"]
if group_mark is not None:
group_mark = group_mark.replace(" ", "")
group_class_type = parse_group_type(raw_group["type"]).value
teachers = parse_teachers(raw_group["lecturer"])
try:
group_semester = parse_group_semester(raw_group["semester"]).value
except Exception as e:
log.info(f"No group semester: {e}")
group_semester = None
occurrence = occurrence_for_semester(course, year, occurrence_credits,
group_semester, course_semesters,
occurrence_notes)
group, created = ClassGroup.objects.get_or_create(
occurrence=occurrence, class_type=group_class_type, mark=group_mark)
if teachers and created:
group.teachers.add(*teachers)
if not created and teachers and set(teachers) != set(group.teachers.all()):
group.teachers.set(teachers)
log.info(f"Group {group.id} was updated")
if created:
log.info(f"Group {group.id} created")
# Add classes to group
first_teacher = None if not teachers else teachers[0]
old_classes = set(CourseClass.objects.filter(group=group))
new_classes = set()
for i, raw_semester in enumerate(raw_group["semester"]):
course_class = create_course_class(group, i, raw_group, raw_semester,
first_teacher)
new_classes.add(course_class)
irrelevant_classes = old_classes - new_classes
for c in irrelevant_classes:
log.info(f"Class {c.id} is deleted")
c.delete()
def get_course_html(self, year: int, course_number: int) -> str:
cache_path = path.join(self.cache_dir, f"{course_number}-{year}.html")
if self.cache_read and path.exists(cache_path):
log.info("Cache read is on and file already exist, reading")
with open(cache_path, encoding=CHARSET) as file:
return file.read()
log.info("Cache read is off or file does not exist yet")
data = urllib.parse.urlencode(
{"peula": "Simple", "maslul": "0", "shana": "0", "year": year, "course": course_number}
).encode("utf-8")
response = urllib.request.urlopen(url=self.shnaton_url, data=data)
html = response.read().decode(response.headers.get_content_charset())
if self.cache_write:
with open(cache_path, "w", encoding=CHARSET) as file:
log.info(f"Writing html cache")
file.write(html)
else:
log.info("Skipping cache write")
return html
def move_course(user_id, course_id, block_id):
log.info(
f"Moving course {wrap(course_id)} into block {wrap(block_id)} for user {wrap(user_id)}"
)
choice, created = UserCourseChoice.objects.get_or_create(
user_id=user_id, course_id=course_id)
if created:
log.info(f"Choice created")
else:
log.info(
f"Choice already existed at block {wrap(choice.block_id)}")
choice.block_id = block_id
choice.save()
def handle(self, *args, **options):
log.info(f"Fetching courses with: {options}")
with open(options["src_file"], encoding="utf8") as file:
courses = json.load(file)
if options["shuffle"]:
random.shuffle(courses)
else:
courses.sort(key=lambda c: c["id"])
fail_count = 0
log.info(f"Total {wrap(len(courses))} courses found")
parser = ShnatonParser()
limit = min(options["limit"], len(courses))
for i, course in enumerate(courses):
if i >= options["limit"]:
break
course_number = course["id"]
log.info(f"Course {wrap(i + 1)} out of {wrap(limit)} is {wrap(course_number)}")
try:
parser.fetch_course(course_number)
except Exception as e:
log.error(f"Could'nt fetch course {course_number}: {e}")
sentry_sdk.capture_exception(e)
fail_count += 1
log.info(f"Fail count: {wrap(fail_count)} out of {wrap(limit)}")
def set_user_schedule_group(user, group_id):
log.info(
f"set_user_schedule_group for user {wrap(user)} with group_id {wrap(group_id)}"
)
if user.is_anonymous:
# log.warning("set_user_schedule_group called but user is_anonymous")
raise UserNotLoggedInError()
group = ClassGroup.objects.get(pk=group_id)
existing = ClassSchedule.objects.filter(
user=user,
group__occurrence__course=group.occurrence.course,
group__class_type=group.class_type).first()
if existing:
log.info(
f"set_user_schedule_group replacing existing {wrap(existing.group.id)}"
)
existing.delete()
schedule = ClassSchedule(user=user, group=group)
schedule.save()
log.info(f"set_user_schedule_group added {wrap(group.id)}")
def init_sentry():
from academic_helper.utils.logger import log
if ENV == Environment.local:
log.info("Env set to local, not configuring sentry")
return
if not DSN:
log.info("Sentry DSN env var not found")
return
log.info("Configuring Sentry")
integrations = [DjangoIntegration()]
if ENV != Environment.prod:
integrations += [LoggingIntegration(event_level=None)]
sentry_sdk.init(
dsn=DSN,
integrations=integrations,
environment=ENV.value,
send_default_pii=True,
)
sentry_sdk.integrations.logging.ignore_logger(
"django.security.DisallowedHost")
def handle(self, *args, **options):
log.info(f"Fetching courses with: {options}")
with open(options["src_file"], encoding="utf8") as file:
courses = json.load(file)
if SHUFFLE:
random.shuffle(courses)
fail_count = 0
log.info(f"Total {wrap(len(courses))} courses found")
for i, course in enumerate(courses):
if i > options["limit"]:
break
course_number = course["id"]
existing = Course.objects.filter(course_number=course_number)
if existing.exists():
if not options["fetch_existing"]:
continue
existing.delete()
try:
ShnatonParser.fetch_course(course_number)
except Exception as e:
log.error(f"Could'nt fetch course {course_number}: {e}")
fail_count += 1
log.info(f"Fail count: {wrap(fail_count)}")
def save(self, *args, **kwargs):
super().save(*args, **kwargs)
log.info(f"Comment saved: {wrap(self)}")
def save(self, *args, **kwargs):
super().save(*args, **kwargs)
log.info(f"Saving user: {wrap(self)}")
def do(self):
log.info(f"{self.code} cron is up.")
self.job()
