def parse_course_div(course_div):
title_div = course_div.find(class_="courseblocktitle")
title = get_clean_text(title_div.text)
course = parse_course_title(title)
info = parse_course_info(course_div)
return {**course, **info}
def pick_header_text(potential_headers, toggleheads, i):
header = None
for potential_header in potential_headers:
if potential_header.name in ("h4", "h5"):
header = potential_header
header_text = header.get_text()
break
if not header:
# find tglhead
if i < len(toggleheads):
header = toggleheads[i]
header_text = header.get_text()
if not header:
# resort to p
for potential_header in potential_headers:
if potential_header.name == "p":
header = potential_header
header_text = str(header)
break
if not header:
header_text = "Requirements"
header_text = get_clean_text(header_text)
# silly edge cases that aren't worth accounting for
if header_text == "<p>Departmental Requirements—Common Curriculum: All courses must be completed with a grade of C- or higher, with an exception listed below.*</p>":
header_text = "Departmental Requirements"
if len(header_text) > 150:
header_text = "Requirements"
return header_text
def parse_school_department(soup):
breadcrumb = soup.find(id="breadcrumb")
wrap = breadcrumb.find(class_="wrap")
anchors = wrap.find_all("a")
school = ""
department = ""
for anchor in anchors:
href = anchor.get("href")
if href == "/" or href == "#":
continue
text = get_clean_text(anchor.text)
if not school:
school = text
continue
department = text
return school, department
def parse_course_info(course_div):
desc = course_div.find("div", class_="courseblockdesc")
prerequisite = ""
corequisite = ""
prerequisite_list = []
same_as = ""
ge_categories = []
lines = desc.get_text().split("\n")
desc = lines[1]
for line in lines:
line = get_clean_text(line)
# inconsistencies
line = line.replace("(GE ", "(")
line = line.replace("(General Education ", "(")
line = line.strip()
if not line:
continue
if line.startswith("Prerequisite or corequisite: "):
line = line[len("Prerequisite or corequisite: "):].strip()
# TODO figure out this case
elif line.startswith("Prerequisite: "):
line = line[len("Prerequisite: "):].strip()
prerequisite = line
prerequisite_list = get_prerequisite_list(line)
elif line.startswith("Corequisite: "):
line = line[len("Corequisite: "):].strip()
corequisite = line
elif line.startswith("Same as "):
same_as = line[len("Same as "):].replace(".", "").strip()
elif line.startswith("(I") or line.startswith("(V"):
tokens = line.replace("(", "").replace(")",
"").replace(",",
"").split()
for token in tokens:
if token in ("Ia", "Ib", "II", "III", "IV", "Va", "Vb", "VI",
"VII", "VIII"):
ge_categories.append(token)
return {
"description": desc,
"prerequisite": prerequisite,
"corequisite": corequisite,
"prerequisite_list": prerequisite_list,
"same_as": same_as,
"ge_categories": ge_categories
}
def parse_requirements_text_container(requirements_text_container):
all_requirements = []
toggleheads = requirements_text_container.find_all(class_="tglhead")
course_lists = requirements_text_container.find_all(class_="sc_courselist")
for i, course_list in enumerate(course_lists):
potential_headers = get_potential_headers(course_list)
header_text = pick_header_text(potential_headers, toggleheads, i)
header_clean_text = get_clean_text(header_text)
if header_clean_text[-1] == "1": # footnote number, get rid of it
header_clean_text = header_clean_text[:-1]
if header_clean_text in all_requirements:
continue
requirements = parse_requirements_table(course_list)
all_requirements.append({
"header": header_clean_text,
"requirements": requirements
})
return all_requirements
def parse_course_title(title):
title = get_clean_text(title)
if "Unit" in title:
reg = course_title_re.match(title)
units_str = reg.group(4)
units = [float(unit.strip()) for unit in units_str.split("-")]
units = [int(unit) if unit.is_integer() else unit for unit in units]
if len(units) == 1:
units.append(units[0])
else:
reg = course_title_no_units.match(title)
units = [0, 0]
department, number, name = reg.group(1), reg.group(2), reg.group(3)
name = name.strip()
id_ = department + " " + number
return {
"id": id_,
"department": department,
"number": number,
"name": name,
"units": units
}
def parse_requirements_table(courselist_table):
# utility functions
def is_bullet_point(text):
return len(text) > 2 and text[0].isalpha() and text[0].isupper(
) and text[1] == "." and text[2] == " "
course_tr = courselist_table.find_all("tr")
requirements = []
next_or = False
for course in course_tr:
class_ = course.get("class")
# handler headers
if "areaheader" in class_:
requirements.append({
"type":
"header",
"comment":
get_clean_text(course.find(class_="courselistcomment").text),
})
continue
# if the current row's a comment
search_comment = course.find(class_="courselistcomment")
if search_comment:
# if it's an or comment
if "or" == search_comment.text.strip():
# if the previous row was a comment/header, we treat this or as a comment too
if len(requirements) > 0 and requirements[-1]["type"] in (
"header", "comment"):
requirements.append({
"type": "comment",
"comment": "or",
})
continue
# otherwise, be prepared to combine the next series/single with the previous
next_or = True
continue
# if it's not an or comment, add it as a comment
comment_text = get_clean_text(search_comment.text)
if is_bullet_point(comment_text):
requirements.append({
"type": "header",
"comment": comment_text
})
continue
requirements.append({
"type": "comment",
"comment": get_clean_text(search_comment.text),
})
continue
total_text = course.get_text().strip()
if not total_text:
# blank rows
continue
# grab the course id
columns = course.find_all()
course_id = get_clean_text(columns[0].get_text())
# if it starts with or (series), ignore the or
if course_id.startswith("or "):
course_id = course_id[len("or "):]
# if -, it's a series
if "-" in course_id:
course_requirement = {"type": "series", "subrequirements": []}
course_ids = [c_id.strip() for c_id in course_id.split("-")]
base_course_id = course_ids[0]
# grab the course department (usually omitted from the subsequent rows)
course_department = base_course_id[:-len(
base_course_id.split(" ")[-1])].strip()
# prefix with the department if it doesn't alreaday start with it
for i, course_id in enumerate(course_ids):
if i > 0 and not course_id.startswith(course_department):
course_ids[i] = course_department.upper() + " " + course_id
course_requirement["subrequirements"] = [{
"type": "single",
"course": course_id
} for course_id in course_ids]
else:
# handle the single course case
course_requirement = {"type": "single", "course": course_id}
# if we previously encountered an or
if len(requirements) > 0 and ("orclass" in class_ or next_or):
# see if we have multiple ors, if so, just merge into the previous one
if requirements[-1]["type"] != "or":
requirements[-1] = {
"type": "or",
"subrequirements": [requirements[-1]]
}
# add the course to the ors
requirements[-1]["subrequirements"].append(course_requirement)
next_or = False
continue
requirements.append(course_requirement)
requirements = nest_requirements_under_headers(requirements)
requirements = assign_requirements_ids(requirements)
return requirements