def test_all_course_details_are_parsable(self):
for semester in self.semesters:
errors = []
xml = owlxp.fetch_catalog_xml(semester)
inv = owlxp.find_courseinventory_nodes(xml)
parsed = owlxp.parse_courseinventory_nodes(inv, semester)
for p in parsed:
try:
owlxp.parse_course_details(p["subject_code"], p["number"], semester)
except ScraperError as e:
errors.append(e)
def test_can_create_course_from_course_dict_output(self):
"""
Should be able generate a dictionary that ``create_course()`` can successfully create
a ``Course`` from.
"""
logger.info("Finding <course_inventory> nodes from '%s'..." % self.xml_filename)
inv = owlxp.find_courseinventory_nodes(self.test_courses_xml)
course_dict = owlxp.course_dicts_for_nodes(inv[0], Semester.objects.get(year=2013, term=Semester.SUMMER))
self.assertEqual(len(course_dict), 1)
course = Course.objects.get_or_create_from_dict(course_dict[0])
logger.info("Successfully created Course model instance --> %s" % course)
def setUp(self):
self.semesters = owlxp.update_semesters()
# Load test xml
self.xml_filename = "test_courses.xml"
self.sections_html_filename = "scrapers/dumps/sections_html"
self.test_courses_xml = open(self.xml_filename, "r")
logger.debug("Loaded %s" % self.xml_filename)
self.test_sections_html = pickle.load(open(self.sections_html_filename, "r"))
logger.debug("Loaded %s" % self.sections_html_filename)
def handle(self, *args, **options):
semesters = owlxp.update_semesters()
if len(args) > 0:
semesters = [Semester.objects.get(code=code) for code in args]
# Load courses and sections for the first semester
owlxp.load_courses(semesters[0])
owlxp.load_sections(semesters[0])
del semesters[0]
# For rest of semesters, do shallow update for courses and load for sections
for s in semesters:
owlxp.update_courses(s, shallow=True)
owlxp.load_sections(s)
# Load degree programs
ps.load_programs()
def test_can_build_sections_from_section_dicts(self):
"""
Should be able to load the parsed section listings and create Section model instances from them.
"""
section_dicts = owlxp.parse_sections(self.test_sections_html)
# Create a Section model instance for each section_dict
semester = Semester.objects.get(year=2013, term=Semester.SUMMER)
sections = []
for s in section_dicts:
sections.append(Section.objects.create_from_dict(s, semester=semester))
fixtures_filename = "catalogs/fixtures/%s_full.json" % semester.code
utils.dump_catalog_fixtures(fixtures_filename)
def test_fix_course_dicts(self):
course_dicts = pickle.load(open("scrapers/dumps/course_dicts", "rb"))
owlxp.create_courses(course_dicts)
utils.dump_catalog_fixtures("catalogs/fixtures/%s.json" % self.semesters[0].code)
def can_fetch_catalog_xml(self):
xml = owlxp.fetch_catalog_xml(self.semesters[0])
self.assertIsNotNone(xml)
def setUp(self):
self.semesters = owlxp.update_semesters()
self.fetch = FetchMachine()
self.parse = ParseMachine()
def test_can_parse_section_listings(self):
"""
Should be able to parse section listings HTML into dicts representing each section.
"""
section_dicts = owlxp.parse_sections(self.test_sections_html)
def test_can_fetch_sections(self):
"""
Should be able to successfully retrieve the section listings for a particular semester.
"""
owlxp.fetch_all_section_dicts(self.semesters[0])
