def scrape_category(original_requester, output_dir, year, term, category): """ Scrape a category within a term (i.e. Anthropology, Engineering) """ try: requester = RequestMaker.copy(original_requester) category_url_fmt = ('/course_evaluation_reports/fas/guide_dept?' 'dept={category}&term={term}&year={year}') courses = requester.make_request(category_url_fmt.format( category=category, term=term, year=year)) course_links = [c.attrs['href'] for c in courses.select('.course a')] course_ids = [int(link.split('=')[1]) for link in course_links] for cid in course_ids: # Scrape data from each course scrape_course(requester, output_dir, cid, year, term) except: traceback.print_exc() raise
def _helper(t): (requester, output_dir, year, term) = t scrape_term(RequestMaker.copy(requester), output_dir, year, term)