def scrape_questions(requester, course_id): """ Scrape all the questions about the course along with student responses """ questions = [] base_url = '/course_evaluation_reports/fas/view_comments.html' all_questions_url = '{}?course_id={}'.format(base_url, course_id) all_soup = requester.make_request(all_questions_url) question_links = all_soup.select('#reportContent h3 a') for q_link in question_links: q = Question(question=q_link.text) q_url = '/course_evaluation_reports/fas/{}'.format(q_link.attrs['href']) q_soup = requester.make_request(q_url) q.responses = [r.text.strip() for r in q_soup.select('.response p')] questions.append(q) return questions