def test_all_course_details_are_parsable(self): for semester in self.semesters: errors = [] xml = owlxp.fetch_catalog_xml(semester) inv = owlxp.find_courseinventory_nodes(xml) parsed = owlxp.parse_courseinventory_nodes(inv, semester) for p in parsed: try: owlxp.parse_course_details(p["subject_code"], p["number"], semester) except ScraperError as e: errors.append(e)
def test_can_create_course_from_course_dict_output(self): """ Should be able generate a dictionary that ``create_course()`` can successfully create a ``Course`` from. """ logger.info("Finding <course_inventory> nodes from '%s'..." % self.xml_filename) inv = owlxp.find_courseinventory_nodes(self.test_courses_xml) course_dict = owlxp.course_dicts_for_nodes(inv[0], Semester.objects.get(year=2013, term=Semester.SUMMER)) self.assertEqual(len(course_dict), 1) course = Course.objects.get_or_create_from_dict(course_dict[0]) logger.info("Successfully created Course model instance --> %s" % course)
def setUp(self): self.semesters = owlxp.update_semesters() # Load test xml self.xml_filename = "test_courses.xml" self.sections_html_filename = "scrapers/dumps/sections_html" self.test_courses_xml = open(self.xml_filename, "r") logger.debug("Loaded %s" % self.xml_filename) self.test_sections_html = pickle.load(open(self.sections_html_filename, "r")) logger.debug("Loaded %s" % self.sections_html_filename)
def handle(self, *args, **options): semesters = owlxp.update_semesters() if len(args) > 0: semesters = [Semester.objects.get(code=code) for code in args] # Load courses and sections for the first semester owlxp.load_courses(semesters[0]) owlxp.load_sections(semesters[0]) del semesters[0] # For rest of semesters, do shallow update for courses and load for sections for s in semesters: owlxp.update_courses(s, shallow=True) owlxp.load_sections(s) # Load degree programs ps.load_programs()
def test_can_build_sections_from_section_dicts(self): """ Should be able to load the parsed section listings and create Section model instances from them. """ section_dicts = owlxp.parse_sections(self.test_sections_html) # Create a Section model instance for each section_dict semester = Semester.objects.get(year=2013, term=Semester.SUMMER) sections = [] for s in section_dicts: sections.append(Section.objects.create_from_dict(s, semester=semester)) fixtures_filename = "catalogs/fixtures/%s_full.json" % semester.code utils.dump_catalog_fixtures(fixtures_filename)
def test_fix_course_dicts(self): course_dicts = pickle.load(open("scrapers/dumps/course_dicts", "rb")) owlxp.create_courses(course_dicts) utils.dump_catalog_fixtures("catalogs/fixtures/%s.json" % self.semesters[0].code)
def can_fetch_catalog_xml(self): xml = owlxp.fetch_catalog_xml(self.semesters[0]) self.assertIsNotNone(xml)
def setUp(self): self.semesters = owlxp.update_semesters() self.fetch = FetchMachine() self.parse = ParseMachine()
def test_can_parse_section_listings(self): """ Should be able to parse section listings HTML into dicts representing each section. """ section_dicts = owlxp.parse_sections(self.test_sections_html)
def test_can_fetch_sections(self): """ Should be able to successfully retrieve the section listings for a particular semester. """ owlxp.fetch_all_section_dicts(self.semesters[0])