def test_dict_filter_by_list(self):
original1 = {'A': 1, 'B': 2, 'C': {'E': 3}, 'D': 4}
filtered = dict_filter_by_list(original1, {4, 'D', 'C', 'd'})
self.assertEqual({'D': 4, 'C': {'E': 3}}, filtered)
original2 = {'A', 'B'}
filtered = dict_filter_by_list(original2, {})
self.assertEqual(set(), filtered)
filtered = dict_filter_by_list(original2, original2)
self.assertEqual(original2, filtered)
def test_dict_filter_by_list(self):
original1 = {"A": 1, "B": 2, "C": {"E": 3}, "D": 4}
filtered = dict_filter_by_list(original1, {4, "D", "C", "d"})
self.assertEqual({"D": 4, "C": {"E": 3}}, filtered)
original2 = {"A", "B"}
filtered = dict_filter_by_list(original2, {})
self.assertEqual(set(), filtered)
filtered = dict_filter_by_list(original2, original2)
self.assertEqual(original2, filtered)
def _parse_term(self, term, term_code, query):
self.ingestor['term'] = term
query['termId'] = term_code
query['requestType'] = 'DEPARTMENTS'
depts = dict_filter_by_list(self._extract_json(query),
self.departments_filter)
for dept, dept_code in list(depts.items()):
self._parse_dept(dept, dept_code, query)
def _parse_term(self, term, term_code, query):
self.ingestor['term'] = term
query['termId'] = term_code
query['requestType'] = 'DEPARTMENTS'
depts = dict_filter_by_list(self._extract_json(query),
self.departments_filter)
for dept, dept_code in depts.items():
self._parse_dept(dept, dept_code, query)
def start(
self,
verbosity=3,
textbooks=True,
departments_filter=None,
years_and_terms_filter=None,
):
"""Start the parse."""
self.verbosity = verbosity
self._login()
years_and_terms = dict_filter_by_dict(self._parse_years_and_terms(),
years_and_terms_filter)
for year, terms in years_and_terms.items():
self.ingestor["year"] = year
for term_name, term_code in terms.items():
self.ingestor["term"] = term_name
# Load environment for targeted semester
self.requester.get(
Parser.URL + "/SelectTerm!selectTerm.action",
params={"selectedTermCode": term_code},
parse=False,
)
self.requester.get(Parser.URL +
"/SelectTerm!updateSessions.action",
parse=False)
# Create payload to request course list from server
params = {
"searchCriteria.classStatusCodes": ["O", "W", "C"],
"__checkbox_searchCriteria.classStatusCodes":
["O", "W", "C"],
}
departments = dict_filter_by_list(
dict(self.extract_department_codes()), departments_filter)
for dept_code, dept_name in departments.items():
self.ingestor["department_code"] = dept_code
self.ingestor["department_name"] = dept_name
# Construct payload with department code
params.update(
{"searchCriteria.subjectAreaCodes": dept_code})
# GET html for department course listings
soup = self.requester.get(
Parser.URL + "/SearchClassesExecute!search.action",
params=params,
)
# Parse courses in department
self.parse_courses_in_department(soup)
# return to search page for next iteration
self.requester.get(Parser.URL + "/Entry.action", parse=False)
def test_dict_filter_by_list(self):
original1 = {
'A': 1,
'B': 2,
'C': {'E': 3},
'D': 4
}
filtered = dict_filter_by_list(original1, {4, 'D', 'C', 'd'})
self.assertEqual({'D': 4, 'C': {'E': 3}}, filtered)
original2 = {
'A',
'B'
}
filtered = dict_filter_by_list(original2, {})
self.assertEqual(set(), filtered)
filtered = dict_filter_by_list(original2, original2)
self.assertEqual(original2, filtered)
def _get_departments(self, soup, departments_filter=None):
if self._get_selected_term(soup) == self.intially_selected_term:
sys.stderr.write('GET DEPARTMENTS')
return dict_filter_by_list(
self.saved_departments,
departments_filter
), None
return super(QPeoplesoftParser, self)._get_departments(
soup,
departments_filter
)
def _get_departments(self, soup, departments_filter=None):
dept_soups = soup.find(
'select',
id=re.compile(r'SSR_CLSRCH_WRK_SUBJECT_SRCH\$\d')
).find_all('option')[1:]
def extract_dept_name(d):
return self.department_name_regex.match(d).group(1)
departments = {
d['value']: extract_dept_name(d.text) for d in dept_soups
}
return dict_filter_by_list(departments, departments_filter), None
def _get_departments(self, soup, departments_filter=None):
# extract department query list
departments = soup.find_all(
'a',
id=re.compile(r'CLASS_SRCH_WRK2_SSR_PB_CLASS_SRCH\$\d')
)
department_names = soup.find_all(
'span',
id=re.compile(r'M_SR_SS_SUBJECT_DESCR\$\d')
)
depts = {
dept.text: dept_name.text
for dept, dept_name in zip(departments, department_names)
}
dept_ids = {dept.text: dept['id'] for dept in departments}
return dict_filter_by_list(
depts,
departments_filter,
), dept_ids
def start(self,
verbosity=3,
textbooks=True,
departments_filter=None,
years_and_terms_filter=None):
"""Start the parse."""
self.verbosity = verbosity
self._login()
years_and_terms = dict_filter_by_dict(self._parse_years_and_terms(),
years_and_terms_filter)
for year, terms in years_and_terms.items():
self.ingestor['year'] = year
for term_name, term_code in terms.items():
self.ingestor['term'] = term_name
# Load environment for targeted semester
self.requester.get(
Parser.URL + '/SelectTerm!selectTerm.action',
params={'selectedTermCode': term_code},
parse=False
)
self.requester.get(
Parser.URL + '/SelectTerm!updateSessions.action',
parse=False
)
# Create payload to request course list from server
params = {
'searchCriteria.classStatusCodes': [
'O', 'W', 'C'
],
'__checkbox_searchCriteria.classStatusCodes': [
'O', 'W', 'C'
]
}
departments = dict_filter_by_list(
dict(self.extract_department_codes()),
departments_filter
)
for dept_code, dept_name in departments.items():
self.ingestor['department_code'] = dept_code
self.ingestor['department_name'] = dept_name
# Construct payload with department code
params.update({
'searchCriteria.subjectAreaCodes': dept_code
})
# GET html for department course listings
soup = self.requester.get(
Parser.URL + '/SearchClassesExecute!search.action',
params=params
)
# Parse courses in department
self.parse_courses_in_department(soup)
# return to search page for next iteration
self.requester.get(Parser.URL + '/Entry.action',
parse=False)
