def setUp(self):
self.cs1332 = scrapy.http.TextResponse("", body=CS1332.body)
self.german = scrapy.http.TextResponse("", body=GRMN4694.body)
self.catalog = scrapy.http.TextResponse("", body=catalog.body)
self.term = scrapy.http.TextResponse("", body=term.body)
self.courses = scrapy.http.TextResponse("", body=courses.body)
self.spider = OscarSpider()
class TestSectionParser(unittest.TestCase):
def setUp(self):
request = MagicMock()
request.meta = {'course': CourseLoader(item=Course())}
self.response = scrapy.http.TextResponse('', body=PHYS2211.body, request=request)
self.scraper = OscarSpider()
def test_parse_section(self):
item = self.scraper.parse_section(self.response)['sections']
self.assertEqual(len(item), 45)
for sec in item:
if sec['crn'] == u'21003':
self.assertEqual(sec['section_id'], u'B06')
self.assertEqual(sec['instructors'][0], u'Eric R. Murray')
self.assertEqual(sec['meetings'][0]['days'], u'M')
self.assertEqual(sec['meetings'][0]['location'], u'Clough Undergraduate Commons 123')
def setUp(self):
request = MagicMock()
request.meta = {'course': CourseLoader(item=Course())}
self.response = scrapy.http.TextResponse('', body=PHYS2211.body, request=request)
self.scraper = OscarSpider()
class TestOscarSpider(unittest.TestCase):
@staticmethod
def bound_identity(self, *args, **kwargs):
return args, kwargs
def setUp(self):
self.cs1332 = scrapy.http.TextResponse("", body=CS1332.body)
self.german = scrapy.http.TextResponse("", body=GRMN4694.body)
self.catalog = scrapy.http.TextResponse("", body=catalog.body)
self.term = scrapy.http.TextResponse("", body=term.body)
self.courses = scrapy.http.TextResponse("", body=courses.body)
self.spider = OscarSpider()
def test_parse_detail(self):
item = self.spider.parse_detail(self.cs1332)
self.assertEqual(item['fields'], ["Grade Basis", "Restrictions", "Prerequisites"])
self.assertEqual(item['fullname'], "CS 1332 - Data Struct & Algorithms")
self.assertEqual(item['name'], "Data Struct & Algorithms")
self.assertEqual(item['school'], "CS")
self.assertEqual(item['number'], "1332")
self.assertEqual(item['prerequisites'],
{'type': 'or', 'courses' : [u'CS 1322', u'CS 1331']})
self.assertEqual(item['identifier'], 'CS 1332')
def test_parse_detail_with_distraction(self):
"""The body of this response contains some wonky stuff that will trick bad regex"""
item = self.spider.parse_detail(self.german)
self.assertEqual(item['prerequisites'], {'type': 'and', 'courses': [u'GRMN 2002']})
# extensive use of mocking below
def test_parse(self):
with patch.object(OscarSpider, 'parse_catalog', self.bound_identity):
result = self.spider.parse(self.cs1332)
self.assertEqual(result[0][0], self.cs1332)
def test_parse_catalog(self):
with patch.object(scrapy, 'FormRequest') as mock:
results = list(self.spider.parse_catalog(self.catalog))
if settings.SEMESTER_STOP != -1:
self.assertEqual(len(results), settings.SEMESTER_STOP)
call = mock.mock_calls[0]
self.assertEqual(call[1][0], self.catalog)
self.assertEqual(call[2], {
'callback': self.spider.parse_term,
'formdata': {u'cat_term_in': u'201602'}
})
def test_parse_term(self):
with patch.object(scrapy, 'FormRequest') as mock:
results = list(self.spider.parse_term(self.term))
if settings.SUBJECTS:
self.assertEqual(len(results), len(settings.SUBJECTS))
subj = settings.SUBJECTS[0] if settings.SUBJECTS else "ACCT"
# this is a workaround since the subjects parsed can change based on
# settings in the config file (settings.py), so these tests are
# invariant to local settings
call = mock.mock_calls[0]
self.assertEqual(call[1][0], self.term)
self.assertEqual(call[2], {
'callback': self.spider.parse_courses,
'formdata' : {u'sel_subj': ['dummy', subj]}
})
def test_parse_courses(self):
with patch.object(scrapy, 'Request') as mock:
results = list(self.spider.parse_courses(self.courses))
call = mock.mock_calls[0]
self.assertEqual(len(results), 294)
self.assertEqual(call[1][0], u'https://oscar.gatech.edu/pls/bprod/bwckctlg.p_disp_course_detail?cat_term_in=201602&subj_code_in=CS&crse_numb_in=1100')
self.assertEqual(call[1][1], self.spider.parse_detail)
