def scrape_category(original_requester, output_dir, year, term, category):
"""
Scrape a category within a term (i.e. Anthropology, Engineering)
"""
try:
requester = RequestMaker.copy(original_requester)
category_url_fmt = ('/course_evaluation_reports/fas/guide_dept?'
'dept={category}&term={term}&year={year}')
courses = requester.make_request(category_url_fmt.format(
category=category, term=term, year=year))
course_links = [c.attrs['href'] for c in courses.select('.course a')]
course_ids = [int(link.split('=')[1]) for link in course_links]
for cid in course_ids:
# Scrape data from each course
scrape_course(requester, output_dir, cid, year, term)
except:
traceback.print_exc()
raise
def _helper(t):
(requester, output_dir, year, term) = t
scrape_term(RequestMaker.copy(requester), output_dir, year, term)
