def test_clean(self): dirty1 = [None, 1, "2", False, [" "]] self.assertEqual([1, "2", False], clean(dirty1)) dirty2 = {"a": [None, None]} self.assertEqual(None, clean(dirty2)) dirty3 = "\u00a0\t \t\xa0" self.assertEqual(None, clean(dirty3)) dirty4 = "hello " self.assertEqual("hello", clean(dirty4)) dirty5 = {"a": [None, "b"], 4: [Exception, " ", {0: [[None]]}]} self.assertEqual({"a": ["b"], 4: [Exception]}, clean(dirty5))
def test_clean(self): dirty1 = [None, 1, '2', False, [' ']] self.assertEqual([1, '2', False], clean(dirty1)) dirty2 = {'a': [None, None]} self.assertEqual(None, clean(dirty2)) dirty3 = u'\u00a0\t \t\xa0' self.assertEqual(None, clean(dirty3)) dirty4 = 'hello ' self.assertEqual('hello', clean(dirty4)) dirty5 = {'a': [None, 'b'], 4: [Exception, ' ', {0: [[None]]}]} self.assertEqual({'a': ['b'], 4: [Exception]}, clean(dirty5))
def ingest_textbook_link(self, section=None): """Create textbook link json object. Args: section (None, :obj:`dict`, optional): Description Returns: dict: textbook link. """ textbook_link = { "kind": "textbook_link", "school": { "code": self._get("school", "school_code") }, "course": { "code": self._get("course_code") }, "section": { "code": self._get("section_code"), "year": str(self._get("year")), "term": self._get("term", "semester"), }, "isbn": self._get("isbn"), "required": self._get("required"), } textbook_link = clean(textbook_link) self._validate_and_log(textbook_link) self.tracker.year = textbook_link["section"]["year"] self.tracker.term = textbook_link["section"]["term"] if "department" in self: self.tracker.department = self["department"] return textbook_link
def _resolve_instructors(self): instructors = None instr_keys = (set([ "instructors", "instructor", "instr", "instrs", "instr_name", "instr_names", "instructor", "instructor_name", "instructors", ]) & set(self)) if len(instr_keys) == 1: instructors = self[list(instr_keys)[0]] instructors = clean(make_list(instructors)) if instructors is not None: for i in range(len(instructors)): if isinstance(instructors[i], str): instructors[i] = {"name": instructors[i]} elif len(instr_keys) > 1: raise IngestionWarning( "cannot resolve instructors from keys: {}".format( ",".join(instr_keys)), self, ) return instructors
def ingest_meeting(self, section): """Create meeting ingested json map. Args: section (dict): validated section object Returns: dict: meeting """ meeting = { 'kind': 'meeting', 'course': section['course'], 'section': { 'code': section['code'], 'year': str(self._get('year')), 'term': self._get('term', 'semester') }, 'days': make_list(self._get('days', 'day')), 'dates': make_list(self._get('dates', 'date')), 'time': self._resolve_time(), 'location': self._resolve_location() } meeting = clean(meeting) self._validate_and_log(meeting) if 'time' in meeting: self.tracker.time = meeting['time']['start'] self.tracker.time = meeting['time']['end'] return meeting
def ingest_textbook_link(self, section=None): """Create textbook link json object. Args: section (None, :obj:`dict`, optional): Description Returns: dict: textbook link. """ textbook_link = { 'kind': 'textbook_link', 'school': { 'code': self._get('school', 'school_code') }, 'course': { 'code': self._get('course_code') }, 'section': { 'code': self._get('section_code'), 'year': str(self._get('year')), 'term': self._get('term', 'semester') }, 'isbn': self._get('isbn'), 'required': self._get('required') } textbook_link = clean(textbook_link) self._validate_and_log(textbook_link) self.tracker.year = textbook_link['section']['year'] self.tracker.term = textbook_link['section']['term'] if 'department' in self: self.tracker.department = self['department'] return textbook_link
def _resolve_instructors(self): instructors = None instr_keys = set( [ 'instructors', 'instructor', 'instr', 'instrs', 'instr_name', 'instr_names', 'instructor', 'instructor_name', 'instructors' ]) & set(self) if len(instr_keys) == 1: instructors = self[list(instr_keys)[0]] instructors = clean(make_list(instructors)) if instructors is not None: for i in range(len(instructors)): if isinstance(instructors[i], basestring): instructors[i] = {'name': instructors[i]} elif len(instr_keys) > 1: raise IngestionWarning( 'cannot resolve instructors from keys: {}'.format( ','.join(instr_keys) ), self ) return instructors
def ingest_course(self): """Create course json from info in model map. Returns: dict: course """ course = { 'kind': 'course', 'school': { 'code': self.school }, 'code': self._get('course_code', 'code', 'course'), 'name': self._get('name', 'course_name'), 'department': self._resolve_department(), 'credits': self._get('credits', 'num_credits'), 'prerequisites': make_list(self._get('prerequisites', 'prereqs')), 'corequisites': make_list(self._get('corequisites', 'coreqs')), 'exclusions': make_list(self._get('exclusions')), 'description': make_list(self._get('description', 'descr')), 'areas': self._get('areas'), 'level': self._get('level'), 'cores': make_list(self._get('cores')), 'geneds': make_list(self._get('geneds')), 'sections': self._get('sections'), 'homepage': self._get('homepage', 'website'), } course = clean(course) self._validate_and_log(course) if 'department' in course: self.tracker.department = course['department'] return course
def ingest_course(self): """Create course json from info in model map. Returns: dict: course """ course = { 'kind': 'course', 'school': { 'code': self.school, 'subdivisions': [{ 'code': self._get('school_subdivision_code'), 'name': self._get('school_subdivision_name') }] }, 'code': self._get('course_code', 'code', 'course'), 'name': titlize(self._get('name', 'course_name')), 'department': self._resolve_department(), 'credits': safe_cast(self._get('credits', 'num_credits'), float, default=0.), 'prerequisites': make_list(self._get('prerequisites', 'prereqs')), 'corequisites': make_list(self._get('corequisites', 'coreqs')), 'exclusions': make_list(self._get('exclusions')), 'areas': make_list(self._get('areas')), 'level': self._get('level'), 'cores': make_list(self._get('cores')), 'geneds': make_list(self._get('geneds')), 'sections': self._get('sections'), 'homepage': self._get('homepage', 'website'), 'same_as': make_list(self._get('same_as')), 'description': self._get('description', 'descr'), # 'description': extract_info_from_text( # self._get('description', 'descr'), # inject=self # ), } course = clean(course) self._validate_and_log(course) if 'department' in course: self.tracker.department = course['department'] return course
def ingest_course(self): """Create course json from info in model map. Returns: dict: course """ course = { 'kind': 'course', 'school': { 'code': self.school, 'subdivisions': [ { 'code': self._get('school_subdivision_code'), 'name': self._get('school_subdivision_name') } ] }, 'code': self._get('course_code', 'code', 'course'), 'name': titlize(self._get('name', 'course_name')), 'department': self._resolve_department(), 'credits': safe_cast(self._get('credits', 'num_credits'), float, default=0.), 'prerequisites': make_list(self._get('prerequisites', 'prereqs')), 'corequisites': make_list(self._get('corequisites', 'coreqs')), 'exclusions': make_list(self._get('exclusions')), 'areas': make_list(self._get('areas')), 'level': self._get('level'), 'cores': make_list(self._get('cores')), 'geneds': make_list(self._get('geneds')), 'sections': self._get('sections'), 'homepage': self._get('homepage', 'website'), 'same_as': make_list(self._get('same_as')), 'description': self._get('description', 'descr'), # 'description': extract_info_from_text( # self._get('description', 'descr'), # inject=self # ), } course = clean(course) self._validate_and_log(course) if 'department' in course: self.tracker.department = course['department'] return course
def ingest_textbook(self): """Create textbook json object. Returns: dict: textbook """ textbook = { 'kind': 'textbook', 'isbn': self._get('isbn'), 'detail_url': self._get('detail_url'), 'image_url': self._get('image_url'), 'author': self._get('author'), 'title': self._get('title') } textbook = clean(textbook) self._validate_and_log(textbook) if 'department' in self: self.tracker.department = self['department'] return textbook
def ingest_textbook(self): """Create textbook json object. Returns: dict: textbook """ textbook = { "kind": "textbook", "isbn": self._get("isbn"), "detail_url": self._get("detail_url"), "image_url": self._get("image_url"), "author": self._get("author"), "title": self._get("title"), } textbook = clean(textbook) self._validate_and_log(textbook) if "department" in self: self.tracker.department = self["department"] return textbook
def ingest_eval(self): """Create evaluation json object. Returns: dict: eval """ evaluation = { 'kind': 'eval', 'year': str(self._get('year')), 'term': self._get('term'), 'score': float(self._get('score')), 'instructors': self._resolve_instructors(), 'course': { 'code': self._get('course_code') } } evaluation = clean(evaluation) self._validate_and_log(evaluation) self.tracker.year = evaluation['year'] self.tracker.term = evaluation['term'] return evaluation
def ingest_meeting(self, section, clean_only=False): """Create meeting ingested json map. Args: section (dict): validated section object Returns: dict: meeting """ year = str(self._get('year')) term = self._get('term', 'semester') if section.get('code') is None: year = None term = None meeting = { 'kind': 'meeting', 'course': section.get('course'), 'section': { 'code': section.get('code'), 'year': year, 'term': term, }, 'days': make_list(self._get('days', 'day')), 'dates': make_list(self._get('dates', 'date')), 'time': self._resolve_time(), 'location': self._resolve_location() } meeting = clean(meeting) if clean_only: return meeting self._validate_and_log(meeting) if 'time' in meeting: self.tracker.time = meeting['time']['start'] self.tracker.time = meeting['time']['end'] return meeting
def ingest_meeting(self, section, clean_only=False): """Create meeting ingested json map. Args: section (dict): validated section object Returns: dict: meeting """ year = str(self._get("year")) term = self._get("term", "semester") if section.get("code") is None: year = None term = None meeting = { "kind": "meeting", "course": section.get("course"), "section": { "code": section.get("code"), "year": year, "term": term, }, "days": make_list(self._get("days", "day")), "dates": self._resolve_date(), "time": self._resolve_time(), "location": self._resolve_location(), } meeting = clean(meeting) if clean_only: return meeting self._validate_and_log(meeting) if "time" in meeting: self.tracker.time = meeting["time"]["start"] self.tracker.time = meeting["time"]["end"] return meeting
def ingest_section(self, course): """Create section json object from info in model map. Args: course (dict): validated course object Returns: dict: section Raises: IngestorWarning: cannot resolve key """ section = { 'kind': 'section', 'course': { 'code': course['code'] }, 'code': self._get('section_code', 'section', 'meeting_section'), 'name': self._get('section_name'), 'term': self._get('term', 'semester'), 'year': str(self._get('year')), 'instructors': self._resolve_instructors(), 'capacity': self._get('capacity', 'size'), 'enrollment': self._get('enrollment', 'enrolment'), 'waitlist': self._get('waitlist'), 'waitlist_size': self._get('waitlist_size'), 'remaining_seats': self._get('remaining_seats'), 'type': self._get('type', 'section_type'), 'fees': self._get('fees', 'fee', 'cost'), 'final_exam': self._get('final_exam'), 'textbooks': self._get('textbooks'), 'meetings': self._get('offerings') } section = clean(section) self._validate_and_log(section) self.tracker.year = section['year'] self.tracker.term = section['term'] return section
def ingest_section(self, course): """Create section json object from info in model map. Args: course (dict): validated course object Returns: dict: section """ section = { "kind": "section", "course": { "code": course.get("code") }, "code": self._get("section_code", "section", "meeting_section"), "name": titlize(self._get("section_name")), "term": self._get("term", "semester"), "year": str(self._get("year")), "instructors": self._resolve_instructors(), "capacity": safe_cast(self._get("capacity", "size"), int), "enrollment": safe_cast(self._get("enrollment", "enrolment"), int), "waitlist": safe_cast(self._get("waitlist"), int), "waitlist_size": safe_cast(self._get("waitlist_size"), int), "remaining_seats": safe_cast(self._get("remaining_seats"), int), "type": self._get("type", "section_type"), "fees": safe_cast(self._get("fees", "fee", "cost"), float), "final_exam": self._get("final_exam"), "textbooks": self._get("textbooks"), "meetings": self._get("offerings", "meetings"), "course_section_id": safe_cast(self._get("course_section_id"), int), } section = clean(section) self._validate_and_log(section) self.tracker.year = section["year"] self.tracker.term = section["term"] return section
def ingest_eval(self): """Create evaluation json object. Returns: dict: eval """ evaluation = { "kind": "eval", "year": str(self._get("year")), "term": self._get("term"), "score": float(self._get("score")), "instructors": self._resolve_instructors(), "course": { "code": self._get("course_code") }, "summary": self._get("summary"), } evaluation = clean(evaluation) self._validate_and_log(evaluation) self.tracker.year = evaluation["year"] self.tracker.term = evaluation["term"] return evaluation
def ingest_section(self, course): """Create section json object from info in model map. Args: course (dict): validated course object Returns: dict: section """ section = { 'kind': 'section', 'course': { 'code': course.get('code') }, 'code': self._get('section_code', 'section', 'meeting_section'), 'name': titlize(self._get('section_name')), 'term': self._get('term', 'semester'), 'year': str(self._get('year')), 'instructors': self._resolve_instructors(), 'capacity': safe_cast(self._get('capacity', 'size'), int), 'enrollment': safe_cast(self._get('enrollment', 'enrolment'), int), 'waitlist': safe_cast(self._get('waitlist'), int), 'waitlist_size': safe_cast(self._get('waitlist_size'), int), 'remaining_seats': safe_cast(self._get('remaining_seats'), int), 'type': self._get('type', 'section_type'), 'fees': safe_cast(self._get('fees', 'fee', 'cost'), float), 'final_exam': self._get('final_exam'), 'textbooks': self._get('textbooks'), 'meetings': self._get('offerings', 'meetings') } section = clean(section) self._validate_and_log(section) self.tracker.year = section['year'] self.tracker.term = section['term'] return section
def ingest_course(self): """Create course json from info in model map. Returns: dict: course """ course = { "kind": "course", "school": { "code": self.school, "subdivisions": [{ "code": self._get("school_subdivision_code"), "name": self._get("school_subdivision_name"), }], }, "code": self._get("course_code", "code", "course"), "name": titlize(self._get("name", "course_name")), "department": self._resolve_department(), "credits": safe_cast(self._get("credits", "num_credits"), float, default=0.0), "prerequisites": make_list(self._get("prerequisites", "prereqs")), "corequisites": make_list(self._get("corequisites", "coreqs")), "exclusions": make_list(self._get("exclusions")), "areas": make_list(self._get("areas")), "level": self._get("level"), "cores": make_list(self._get("cores")), "geneds": make_list(self._get("geneds")), "sections": self._get("sections"), "homepage": self._get("homepage", "website"), "same_as": make_list(self._get("same_as")), "description": self._get("description", "descr"), "pos": make_list(self._get("pos")), "writing_intensive": self._get("writing_intensive"), "sub_school": self._get("sub_school"), # 'description': extract_info_from_text( # self._get('description', 'descr'), # inject=self # ), } course = clean(course) self._validate_and_log(course) if "department" in course: self.tracker.department = course["department"] return course