def test_clean(self):
dirty1 = [None, 1, "2", False, [" "]]
self.assertEqual([1, "2", False], clean(dirty1))
dirty2 = {"a": [None, None]}
self.assertEqual(None, clean(dirty2))
dirty3 = "\u00a0\t \t\xa0"
self.assertEqual(None, clean(dirty3))
dirty4 = "hello "
self.assertEqual("hello", clean(dirty4))
dirty5 = {"a": [None, "b"], 4: [Exception, " ", {0: [[None]]}]}
self.assertEqual({"a": ["b"], 4: [Exception]}, clean(dirty5))
def test_clean(self):
dirty1 = [None, 1, '2', False, [' ']]
self.assertEqual([1, '2', False], clean(dirty1))
dirty2 = {'a': [None, None]}
self.assertEqual(None, clean(dirty2))
dirty3 = u'\u00a0\t \t\xa0'
self.assertEqual(None, clean(dirty3))
dirty4 = 'hello '
self.assertEqual('hello', clean(dirty4))
dirty5 = {'a': [None, 'b'], 4: [Exception, ' ', {0: [[None]]}]}
self.assertEqual({'a': ['b'], 4: [Exception]}, clean(dirty5))
def test_clean(self):
dirty1 = [None, 1, '2', False, [' ']]
self.assertEqual([1, '2', False], clean(dirty1))
dirty2 = {'a': [None, None]}
self.assertEqual(None, clean(dirty2))
dirty3 = u'\u00a0\t \t\xa0'
self.assertEqual(None, clean(dirty3))
dirty4 = 'hello '
self.assertEqual('hello', clean(dirty4))
dirty5 = {'a': [None, 'b'], 4: [Exception, ' ', {0: [[None]]}]}
self.assertEqual({'a': ['b'], 4: [Exception]}, clean(dirty5))
def ingest_textbook_link(self, section=None):
"""Create textbook link json object.
Args:
section (None, :obj:`dict`, optional): Description
Returns:
dict: textbook link.
"""
textbook_link = {
"kind": "textbook_link",
"school": {
"code": self._get("school", "school_code")
},
"course": {
"code": self._get("course_code")
},
"section": {
"code": self._get("section_code"),
"year": str(self._get("year")),
"term": self._get("term", "semester"),
},
"isbn": self._get("isbn"),
"required": self._get("required"),
}
textbook_link = clean(textbook_link)
self._validate_and_log(textbook_link)
self.tracker.year = textbook_link["section"]["year"]
self.tracker.term = textbook_link["section"]["term"]
if "department" in self:
self.tracker.department = self["department"]
return textbook_link
def _resolve_instructors(self):
instructors = None
instr_keys = (set([
"instructors",
"instructor",
"instr",
"instrs",
"instr_name",
"instr_names",
"instructor",
"instructor_name",
"instructors",
])
& set(self))
if len(instr_keys) == 1:
instructors = self[list(instr_keys)[0]]
instructors = clean(make_list(instructors))
if instructors is not None:
for i in range(len(instructors)):
if isinstance(instructors[i], str):
instructors[i] = {"name": instructors[i]}
elif len(instr_keys) > 1:
raise IngestionWarning(
"cannot resolve instructors from keys: {}".format(
",".join(instr_keys)),
self,
)
return instructors
def ingest_meeting(self, section):
"""Create meeting ingested json map.
Args:
section (dict): validated section object
Returns:
dict: meeting
"""
meeting = {
'kind': 'meeting',
'course': section['course'],
'section': {
'code': section['code'],
'year': str(self._get('year')),
'term': self._get('term', 'semester')
},
'days': make_list(self._get('days', 'day')),
'dates': make_list(self._get('dates', 'date')),
'time': self._resolve_time(),
'location': self._resolve_location()
}
meeting = clean(meeting)
self._validate_and_log(meeting)
if 'time' in meeting:
self.tracker.time = meeting['time']['start']
self.tracker.time = meeting['time']['end']
return meeting
def ingest_textbook_link(self, section=None):
"""Create textbook link json object.
Args:
section (None, :obj:`dict`, optional): Description
Returns:
dict: textbook link.
"""
textbook_link = {
'kind': 'textbook_link',
'school': {
'code': self._get('school', 'school_code')
},
'course': {
'code': self._get('course_code')
},
'section': {
'code': self._get('section_code'),
'year': str(self._get('year')),
'term': self._get('term', 'semester')
},
'isbn': self._get('isbn'),
'required': self._get('required')
}
textbook_link = clean(textbook_link)
self._validate_and_log(textbook_link)
self.tracker.year = textbook_link['section']['year']
self.tracker.term = textbook_link['section']['term']
if 'department' in self:
self.tracker.department = self['department']
return textbook_link
def _resolve_instructors(self):
instructors = None
instr_keys = set(
[
'instructors',
'instructor',
'instr',
'instrs',
'instr_name',
'instr_names',
'instructor',
'instructor_name',
'instructors'
]) & set(self)
if len(instr_keys) == 1:
instructors = self[list(instr_keys)[0]]
instructors = clean(make_list(instructors))
if instructors is not None:
for i in range(len(instructors)):
if isinstance(instructors[i], basestring):
instructors[i] = {'name': instructors[i]}
elif len(instr_keys) > 1:
raise IngestionWarning(
'cannot resolve instructors from keys: {}'.format(
','.join(instr_keys)
),
self
)
return instructors
def ingest_course(self):
"""Create course json from info in model map.
Returns:
dict: course
"""
course = {
'kind': 'course',
'school': {
'code': self.school
},
'code': self._get('course_code', 'code', 'course'),
'name': self._get('name', 'course_name'),
'department': self._resolve_department(),
'credits': self._get('credits', 'num_credits'),
'prerequisites': make_list(self._get('prerequisites', 'prereqs')),
'corequisites': make_list(self._get('corequisites', 'coreqs')),
'exclusions': make_list(self._get('exclusions')),
'description': make_list(self._get('description', 'descr')),
'areas': self._get('areas'),
'level': self._get('level'),
'cores': make_list(self._get('cores')),
'geneds': make_list(self._get('geneds')),
'sections': self._get('sections'),
'homepage': self._get('homepage', 'website'),
}
course = clean(course)
self._validate_and_log(course)
if 'department' in course:
self.tracker.department = course['department']
return course
def ingest_course(self):
"""Create course json from info in model map.
Returns:
dict: course
"""
course = {
'kind':
'course',
'school': {
'code':
self.school,
'subdivisions': [{
'code': self._get('school_subdivision_code'),
'name': self._get('school_subdivision_name')
}]
},
'code':
self._get('course_code', 'code', 'course'),
'name':
titlize(self._get('name', 'course_name')),
'department':
self._resolve_department(),
'credits':
safe_cast(self._get('credits', 'num_credits'), float, default=0.),
'prerequisites':
make_list(self._get('prerequisites', 'prereqs')),
'corequisites':
make_list(self._get('corequisites', 'coreqs')),
'exclusions':
make_list(self._get('exclusions')),
'areas':
make_list(self._get('areas')),
'level':
self._get('level'),
'cores':
make_list(self._get('cores')),
'geneds':
make_list(self._get('geneds')),
'sections':
self._get('sections'),
'homepage':
self._get('homepage', 'website'),
'same_as':
make_list(self._get('same_as')),
'description':
self._get('description', 'descr'),
# 'description': extract_info_from_text(
# self._get('description', 'descr'),
# inject=self
# ),
}
course = clean(course)
self._validate_and_log(course)
if 'department' in course:
self.tracker.department = course['department']
return course
def ingest_course(self):
"""Create course json from info in model map.
Returns:
dict: course
"""
course = {
'kind': 'course',
'school': {
'code': self.school,
'subdivisions': [
{
'code': self._get('school_subdivision_code'),
'name': self._get('school_subdivision_name')
}
]
},
'code': self._get('course_code', 'code', 'course'),
'name': titlize(self._get('name', 'course_name')),
'department': self._resolve_department(),
'credits': safe_cast(self._get('credits', 'num_credits'), float, default=0.),
'prerequisites': make_list(self._get('prerequisites', 'prereqs')),
'corequisites': make_list(self._get('corequisites', 'coreqs')),
'exclusions': make_list(self._get('exclusions')),
'areas': make_list(self._get('areas')),
'level': self._get('level'),
'cores': make_list(self._get('cores')),
'geneds': make_list(self._get('geneds')),
'sections': self._get('sections'),
'homepage': self._get('homepage', 'website'),
'same_as': make_list(self._get('same_as')),
'description': self._get('description', 'descr'),
# 'description': extract_info_from_text(
# self._get('description', 'descr'),
# inject=self
# ),
}
course = clean(course)
self._validate_and_log(course)
if 'department' in course:
self.tracker.department = course['department']
return course
def ingest_textbook(self):
"""Create textbook json object.
Returns:
dict: textbook
"""
textbook = {
'kind': 'textbook',
'isbn': self._get('isbn'),
'detail_url': self._get('detail_url'),
'image_url': self._get('image_url'),
'author': self._get('author'),
'title': self._get('title')
}
textbook = clean(textbook)
self._validate_and_log(textbook)
if 'department' in self:
self.tracker.department = self['department']
return textbook
def ingest_textbook(self):
"""Create textbook json object.
Returns:
dict: textbook
"""
textbook = {
"kind": "textbook",
"isbn": self._get("isbn"),
"detail_url": self._get("detail_url"),
"image_url": self._get("image_url"),
"author": self._get("author"),
"title": self._get("title"),
}
textbook = clean(textbook)
self._validate_and_log(textbook)
if "department" in self:
self.tracker.department = self["department"]
return textbook
def ingest_eval(self):
"""Create evaluation json object.
Returns:
dict: eval
"""
evaluation = {
'kind': 'eval',
'year': str(self._get('year')),
'term': self._get('term'),
'score': float(self._get('score')),
'instructors': self._resolve_instructors(),
'course': {
'code': self._get('course_code')
}
}
evaluation = clean(evaluation)
self._validate_and_log(evaluation)
self.tracker.year = evaluation['year']
self.tracker.term = evaluation['term']
return evaluation
def ingest_meeting(self, section, clean_only=False):
"""Create meeting ingested json map.
Args:
section (dict): validated section object
Returns:
dict: meeting
"""
year = str(self._get('year'))
term = self._get('term', 'semester')
if section.get('code') is None:
year = None
term = None
meeting = {
'kind': 'meeting',
'course': section.get('course'),
'section': {
'code': section.get('code'),
'year': year,
'term': term,
},
'days': make_list(self._get('days', 'day')),
'dates': make_list(self._get('dates', 'date')),
'time': self._resolve_time(),
'location': self._resolve_location()
}
meeting = clean(meeting)
if clean_only:
return meeting
self._validate_and_log(meeting)
if 'time' in meeting:
self.tracker.time = meeting['time']['start']
self.tracker.time = meeting['time']['end']
return meeting
def ingest_meeting(self, section, clean_only=False):
"""Create meeting ingested json map.
Args:
section (dict): validated section object
Returns:
dict: meeting
"""
year = str(self._get("year"))
term = self._get("term", "semester")
if section.get("code") is None:
year = None
term = None
meeting = {
"kind": "meeting",
"course": section.get("course"),
"section": {
"code": section.get("code"),
"year": year,
"term": term,
},
"days": make_list(self._get("days", "day")),
"dates": self._resolve_date(),
"time": self._resolve_time(),
"location": self._resolve_location(),
}
meeting = clean(meeting)
if clean_only:
return meeting
self._validate_and_log(meeting)
if "time" in meeting:
self.tracker.time = meeting["time"]["start"]
self.tracker.time = meeting["time"]["end"]
return meeting
def ingest_section(self, course):
"""Create section json object from info in model map.
Args:
course (dict): validated course object
Returns:
dict: section
Raises:
IngestorWarning: cannot resolve key
"""
section = {
'kind': 'section',
'course': {
'code': course['code']
},
'code': self._get('section_code', 'section', 'meeting_section'),
'name': self._get('section_name'),
'term': self._get('term', 'semester'),
'year': str(self._get('year')),
'instructors': self._resolve_instructors(),
'capacity': self._get('capacity', 'size'),
'enrollment': self._get('enrollment', 'enrolment'),
'waitlist': self._get('waitlist'),
'waitlist_size': self._get('waitlist_size'),
'remaining_seats': self._get('remaining_seats'),
'type': self._get('type', 'section_type'),
'fees': self._get('fees', 'fee', 'cost'),
'final_exam': self._get('final_exam'),
'textbooks': self._get('textbooks'),
'meetings': self._get('offerings')
}
section = clean(section)
self._validate_and_log(section)
self.tracker.year = section['year']
self.tracker.term = section['term']
return section
def ingest_section(self, course):
"""Create section json object from info in model map.
Args:
course (dict): validated course object
Returns:
dict: section
"""
section = {
"kind": "section",
"course": {
"code": course.get("code")
},
"code": self._get("section_code", "section", "meeting_section"),
"name": titlize(self._get("section_name")),
"term": self._get("term", "semester"),
"year": str(self._get("year")),
"instructors": self._resolve_instructors(),
"capacity": safe_cast(self._get("capacity", "size"), int),
"enrollment": safe_cast(self._get("enrollment", "enrolment"), int),
"waitlist": safe_cast(self._get("waitlist"), int),
"waitlist_size": safe_cast(self._get("waitlist_size"), int),
"remaining_seats": safe_cast(self._get("remaining_seats"), int),
"type": self._get("type", "section_type"),
"fees": safe_cast(self._get("fees", "fee", "cost"), float),
"final_exam": self._get("final_exam"),
"textbooks": self._get("textbooks"),
"meetings": self._get("offerings", "meetings"),
"course_section_id": safe_cast(self._get("course_section_id"),
int),
}
section = clean(section)
self._validate_and_log(section)
self.tracker.year = section["year"]
self.tracker.term = section["term"]
return section
def ingest_eval(self):
"""Create evaluation json object.
Returns:
dict: eval
"""
evaluation = {
"kind": "eval",
"year": str(self._get("year")),
"term": self._get("term"),
"score": float(self._get("score")),
"instructors": self._resolve_instructors(),
"course": {
"code": self._get("course_code")
},
"summary": self._get("summary"),
}
evaluation = clean(evaluation)
self._validate_and_log(evaluation)
self.tracker.year = evaluation["year"]
self.tracker.term = evaluation["term"]
return evaluation
def ingest_section(self, course):
"""Create section json object from info in model map.
Args:
course (dict): validated course object
Returns:
dict: section
"""
section = {
'kind': 'section',
'course': {
'code': course.get('code')
},
'code': self._get('section_code', 'section',
'meeting_section'),
'name': titlize(self._get('section_name')),
'term': self._get('term', 'semester'),
'year': str(self._get('year')),
'instructors': self._resolve_instructors(),
'capacity': safe_cast(self._get('capacity', 'size'), int),
'enrollment': safe_cast(self._get('enrollment', 'enrolment'), int),
'waitlist': safe_cast(self._get('waitlist'), int),
'waitlist_size': safe_cast(self._get('waitlist_size'), int),
'remaining_seats': safe_cast(self._get('remaining_seats'), int),
'type': self._get('type', 'section_type'),
'fees': safe_cast(self._get('fees', 'fee', 'cost'), float),
'final_exam': self._get('final_exam'),
'textbooks': self._get('textbooks'),
'meetings': self._get('offerings', 'meetings')
}
section = clean(section)
self._validate_and_log(section)
self.tracker.year = section['year']
self.tracker.term = section['term']
return section
def ingest_course(self):
"""Create course json from info in model map.
Returns:
dict: course
"""
course = {
"kind":
"course",
"school": {
"code":
self.school,
"subdivisions": [{
"code": self._get("school_subdivision_code"),
"name": self._get("school_subdivision_name"),
}],
},
"code":
self._get("course_code", "code", "course"),
"name":
titlize(self._get("name", "course_name")),
"department":
self._resolve_department(),
"credits":
safe_cast(self._get("credits", "num_credits"), float, default=0.0),
"prerequisites":
make_list(self._get("prerequisites", "prereqs")),
"corequisites":
make_list(self._get("corequisites", "coreqs")),
"exclusions":
make_list(self._get("exclusions")),
"areas":
make_list(self._get("areas")),
"level":
self._get("level"),
"cores":
make_list(self._get("cores")),
"geneds":
make_list(self._get("geneds")),
"sections":
self._get("sections"),
"homepage":
self._get("homepage", "website"),
"same_as":
make_list(self._get("same_as")),
"description":
self._get("description", "descr"),
"pos":
make_list(self._get("pos")),
"writing_intensive":
self._get("writing_intensive"),
"sub_school":
self._get("sub_school"),
# 'description': extract_info_from_text(
# self._get('description', 'descr'),
# inject=self
# ),
}
course = clean(course)
self._validate_and_log(course)
if "department" in course:
self.tracker.department = course["department"]
return course
