def ingest_meeting(self, section): """Create meeting ingested json map. Args: section (dict): validated section object Returns: dict: meeting """ meeting = { 'kind': 'meeting', 'course': section['course'], 'section': { 'code': section['code'], 'year': str(self._get('year')), 'term': self._get('term', 'semester') }, 'days': make_list(self._get('days', 'day')), 'dates': make_list(self._get('dates', 'date')), 'time': self._resolve_time(), 'location': self._resolve_location() } meeting = clean(meeting) self._validate_and_log(meeting) if 'time' in meeting: self.tracker.time = meeting['time']['start'] self.tracker.time = meeting['time']['end'] return meeting
def _resolve_instructors(self): instructors = None instr_keys = set( [ 'instructors', 'instructor', 'instr', 'instrs', 'instr_name', 'instr_names', 'instructor', 'instructor_name', 'instructors' ]) & set(self) if len(instr_keys) == 1: instructors = self[list(instr_keys)[0]] instructors = clean(make_list(instructors)) if instructors is not None: for i in range(len(instructors)): if isinstance(instructors[i], basestring): instructors[i] = {'name': instructors[i]} elif len(instr_keys) > 1: raise IngestionWarning( 'cannot resolve instructors from keys: {}'.format( ','.join(instr_keys) ), self ) return instructors
def _resolve_instructors(self): instructors = None instr_keys = (set([ "instructors", "instructor", "instr", "instrs", "instr_name", "instr_names", "instructor", "instructor_name", "instructors", ]) & set(self)) if len(instr_keys) == 1: instructors = self[list(instr_keys)[0]] instructors = clean(make_list(instructors)) if instructors is not None: for i in range(len(instructors)): if isinstance(instructors[i], str): instructors[i] = {"name": instructors[i]} elif len(instr_keys) > 1: raise IngestionWarning( "cannot resolve instructors from keys: {}".format( ",".join(instr_keys)), self, ) return instructors
def ingest_course(self): """Create course json from info in model map. Returns: dict: course """ course = { 'kind': 'course', 'school': { 'code': self.school, 'subdivisions': [{ 'code': self._get('school_subdivision_code'), 'name': self._get('school_subdivision_name') }] }, 'code': self._get('course_code', 'code', 'course'), 'name': titlize(self._get('name', 'course_name')), 'department': self._resolve_department(), 'credits': safe_cast(self._get('credits', 'num_credits'), float, default=0.), 'prerequisites': make_list(self._get('prerequisites', 'prereqs')), 'corequisites': make_list(self._get('corequisites', 'coreqs')), 'exclusions': make_list(self._get('exclusions')), 'areas': make_list(self._get('areas')), 'level': self._get('level'), 'cores': make_list(self._get('cores')), 'geneds': make_list(self._get('geneds')), 'sections': self._get('sections'), 'homepage': self._get('homepage', 'website'), 'same_as': make_list(self._get('same_as')), 'description': self._get('description', 'descr'), # 'description': extract_info_from_text( # self._get('description', 'descr'), # inject=self # ), } course = clean(course) self._validate_and_log(course) if 'department' in course: self.tracker.department = course['department'] return course
def validate(self, data): """Validation entry/dispatcher. Args: data (list, dict): Data to validate. """ for obj in make_list(data): obj = DotDict(obj) Validator.schema_validate(obj, *Validator.SCHEMAS[obj.kind]) self.kind_to_validation_function[obj.kind](obj)
def ingest_course(self): """Create course json from info in model map. Returns: dict: course """ course = { 'kind': 'course', 'school': { 'code': self.school }, 'code': self._get('course_code', 'code', 'course'), 'name': self._get('name', 'course_name'), 'department': self._resolve_department(), 'credits': self._get('credits', 'num_credits'), 'prerequisites': make_list(self._get('prerequisites', 'prereqs')), 'corequisites': make_list(self._get('corequisites', 'coreqs')), 'exclusions': make_list(self._get('exclusions')), 'description': make_list(self._get('description', 'descr')), 'areas': self._get('areas'), 'level': self._get('level'), 'cores': make_list(self._get('cores')), 'geneds': make_list(self._get('geneds')), 'sections': self._get('sections'), 'homepage': self._get('homepage', 'website'), } course = clean(course) self._validate_and_log(course) if 'department' in course: self.tracker.department = course['department'] return course
def digest(self, data, diff=True, load=True, output=None): """Digest data.""" self.data = [DotDict(obj) for obj in make_list(data)] self.strategy = self._resolve_strategy(diff, load, output) do_digestion = { "course": lambda x: self.digest_course(x), "section": lambda x: self.digest_section(x), "meeting": lambda x: self.digest_meeting(x), "textbook": lambda x: self.digest_textbook(x), "textbook_link": lambda x: self.digest_textbook_link(x), "eval": lambda x: self.digest_eval(x), } if self.tracker.has_viewer("progressbar"): bar = self.tracker.get_viewer("progressbar").bar for obj in bar(make_list(self.data)): do_digestion[obj.kind](obj) else: for obj in make_list(self.data): do_digestion[obj.kind](obj) self.wrap_up()
def ingest_meeting(self, section, clean_only=False): """Create meeting ingested json map. Args: section (dict): validated section object Returns: dict: meeting """ year = str(self._get('year')) term = self._get('term', 'semester') if section.get('code') is None: year = None term = None meeting = { 'kind': 'meeting', 'course': section.get('course'), 'section': { 'code': section.get('code'), 'year': year, 'term': term, }, 'days': make_list(self._get('days', 'day')), 'dates': make_list(self._get('dates', 'date')), 'time': self._resolve_time(), 'location': self._resolve_location() } meeting = clean(meeting) if clean_only: return meeting self._validate_and_log(meeting) if 'time' in meeting: self.tracker.time = meeting['time']['start'] self.tracker.time = meeting['time']['end'] return meeting
def digest(self): '''Digest data.''' do_digestion = { 'course': lambda x: self.digest_course(x), 'section': lambda x: self.digest_section(x), 'meeting': lambda x: self.digest_meeting(x), 'textbook': lambda x: self.digest_textbook(x), 'textbook_link': lambda x: self.digest_textbook_link(x), } for obj in make_list(self.data): do_digestion[obj.kind](obj) self.wrap_up()
def ingest_course(self): """Create course json from info in model map. Returns: dict: course """ course = { 'kind': 'course', 'school': { 'code': self.school, 'subdivisions': [ { 'code': self._get('school_subdivision_code'), 'name': self._get('school_subdivision_name') } ] }, 'code': self._get('course_code', 'code', 'course'), 'name': titlize(self._get('name', 'course_name')), 'department': self._resolve_department(), 'credits': safe_cast(self._get('credits', 'num_credits'), float, default=0.), 'prerequisites': make_list(self._get('prerequisites', 'prereqs')), 'corequisites': make_list(self._get('corequisites', 'coreqs')), 'exclusions': make_list(self._get('exclusions')), 'areas': make_list(self._get('areas')), 'level': self._get('level'), 'cores': make_list(self._get('cores')), 'geneds': make_list(self._get('geneds')), 'sections': self._get('sections'), 'homepage': self._get('homepage', 'website'), 'same_as': make_list(self._get('same_as')), 'description': self._get('description', 'descr'), # 'description': extract_info_from_text( # self._get('description', 'descr'), # inject=self # ), } course = clean(course) self._validate_and_log(course) if 'department' in course: self.tracker.department = course['department'] return course
def ingest_meeting(self, section, clean_only=False): """Create meeting ingested json map. Args: section (dict): validated section object Returns: dict: meeting """ year = str(self._get("year")) term = self._get("term", "semester") if section.get("code") is None: year = None term = None meeting = { "kind": "meeting", "course": section.get("course"), "section": { "code": section.get("code"), "year": year, "term": term, }, "days": make_list(self._get("days", "day")), "dates": self._resolve_date(), "time": self._resolve_time(), "location": self._resolve_location(), } meeting = clean(meeting) if clean_only: return meeting self._validate_and_log(meeting) if "time" in meeting: self.tracker.time = meeting["time"]["start"] self.tracker.time = meeting["time"]["end"] return meeting
def test_make_list(self): self.assertEqual([], make_list()) self.assertEqual([], make_list(None)) self.assertEqual(['hello'], make_list('hello')) self.assertEqual(['hello'], make_list(['hello'])) self.assertEqual([{1: 1}], make_list({1: 1}))
def test_make_list(self): self.assertEqual([], make_list()) self.assertEqual([], make_list(None)) self.assertEqual(['hello'], make_list('hello')) self.assertEqual(['hello'], make_list(['hello'])) self.assertEqual([{1: 1}], make_list({1: 1}))
def test_make_list(self): self.assertEqual([], make_list()) self.assertEqual([], make_list(None)) self.assertEqual(["hello"], make_list("hello")) self.assertEqual(["hello"], make_list(["hello"])) self.assertEqual([{1: 1}], make_list({1: 1}))
def ingest_course(self): """Create course json from info in model map. Returns: dict: course """ course = { "kind": "course", "school": { "code": self.school, "subdivisions": [{ "code": self._get("school_subdivision_code"), "name": self._get("school_subdivision_name"), }], }, "code": self._get("course_code", "code", "course"), "name": titlize(self._get("name", "course_name")), "department": self._resolve_department(), "credits": safe_cast(self._get("credits", "num_credits"), float, default=0.0), "prerequisites": make_list(self._get("prerequisites", "prereqs")), "corequisites": make_list(self._get("corequisites", "coreqs")), "exclusions": make_list(self._get("exclusions")), "areas": make_list(self._get("areas")), "level": self._get("level"), "cores": make_list(self._get("cores")), "geneds": make_list(self._get("geneds")), "sections": self._get("sections"), "homepage": self._get("homepage", "website"), "same_as": make_list(self._get("same_as")), "description": self._get("description", "descr"), "pos": make_list(self._get("pos")), "writing_intensive": self._get("writing_intensive"), "sub_school": self._get("sub_school"), # 'description': extract_info_from_text( # self._get('description', 'descr'), # inject=self # ), } course = clean(course) self._validate_and_log(course) if "department" in course: self.tracker.department = course["department"] return course