def ingest_meeting(self, section):
"""Create meeting ingested json map.
Args:
section (dict): validated section object
Returns:
dict: meeting
"""
meeting = {
'kind': 'meeting',
'course': section['course'],
'section': {
'code': section['code'],
'year': str(self._get('year')),
'term': self._get('term', 'semester')
},
'days': make_list(self._get('days', 'day')),
'dates': make_list(self._get('dates', 'date')),
'time': self._resolve_time(),
'location': self._resolve_location()
}
meeting = clean(meeting)
self._validate_and_log(meeting)
if 'time' in meeting:
self.tracker.time = meeting['time']['start']
self.tracker.time = meeting['time']['end']
return meeting
def _resolve_instructors(self):
instructors = None
instr_keys = set(
[
'instructors',
'instructor',
'instr',
'instrs',
'instr_name',
'instr_names',
'instructor',
'instructor_name',
'instructors'
]) & set(self)
if len(instr_keys) == 1:
instructors = self[list(instr_keys)[0]]
instructors = clean(make_list(instructors))
if instructors is not None:
for i in range(len(instructors)):
if isinstance(instructors[i], basestring):
instructors[i] = {'name': instructors[i]}
elif len(instr_keys) > 1:
raise IngestionWarning(
'cannot resolve instructors from keys: {}'.format(
','.join(instr_keys)
),
self
)
return instructors
def _resolve_instructors(self):
instructors = None
instr_keys = (set([
"instructors",
"instructor",
"instr",
"instrs",
"instr_name",
"instr_names",
"instructor",
"instructor_name",
"instructors",
])
& set(self))
if len(instr_keys) == 1:
instructors = self[list(instr_keys)[0]]
instructors = clean(make_list(instructors))
if instructors is not None:
for i in range(len(instructors)):
if isinstance(instructors[i], str):
instructors[i] = {"name": instructors[i]}
elif len(instr_keys) > 1:
raise IngestionWarning(
"cannot resolve instructors from keys: {}".format(
",".join(instr_keys)),
self,
)
return instructors
def ingest_course(self):
"""Create course json from info in model map.
Returns:
dict: course
"""
course = {
'kind':
'course',
'school': {
'code':
self.school,
'subdivisions': [{
'code': self._get('school_subdivision_code'),
'name': self._get('school_subdivision_name')
}]
},
'code':
self._get('course_code', 'code', 'course'),
'name':
titlize(self._get('name', 'course_name')),
'department':
self._resolve_department(),
'credits':
safe_cast(self._get('credits', 'num_credits'), float, default=0.),
'prerequisites':
make_list(self._get('prerequisites', 'prereqs')),
'corequisites':
make_list(self._get('corequisites', 'coreqs')),
'exclusions':
make_list(self._get('exclusions')),
'areas':
make_list(self._get('areas')),
'level':
self._get('level'),
'cores':
make_list(self._get('cores')),
'geneds':
make_list(self._get('geneds')),
'sections':
self._get('sections'),
'homepage':
self._get('homepage', 'website'),
'same_as':
make_list(self._get('same_as')),
'description':
self._get('description', 'descr'),
# 'description': extract_info_from_text(
# self._get('description', 'descr'),
# inject=self
# ),
}
course = clean(course)
self._validate_and_log(course)
if 'department' in course:
self.tracker.department = course['department']
return course
def validate(self, data):
"""Validation entry/dispatcher.
Args:
data (list, dict): Data to validate.
"""
for obj in make_list(data):
obj = DotDict(obj)
Validator.schema_validate(obj, *Validator.SCHEMAS[obj.kind])
self.kind_to_validation_function[obj.kind](obj)
def ingest_course(self):
"""Create course json from info in model map.
Returns:
dict: course
"""
course = {
'kind': 'course',
'school': {
'code': self.school
},
'code': self._get('course_code', 'code', 'course'),
'name': self._get('name', 'course_name'),
'department': self._resolve_department(),
'credits': self._get('credits', 'num_credits'),
'prerequisites': make_list(self._get('prerequisites', 'prereqs')),
'corequisites': make_list(self._get('corequisites', 'coreqs')),
'exclusions': make_list(self._get('exclusions')),
'description': make_list(self._get('description', 'descr')),
'areas': self._get('areas'),
'level': self._get('level'),
'cores': make_list(self._get('cores')),
'geneds': make_list(self._get('geneds')),
'sections': self._get('sections'),
'homepage': self._get('homepage', 'website'),
}
course = clean(course)
self._validate_and_log(course)
if 'department' in course:
self.tracker.department = course['department']
return course
def digest(self, data, diff=True, load=True, output=None):
"""Digest data."""
self.data = [DotDict(obj) for obj in make_list(data)]
self.strategy = self._resolve_strategy(diff, load, output)
do_digestion = {
"course": lambda x: self.digest_course(x),
"section": lambda x: self.digest_section(x),
"meeting": lambda x: self.digest_meeting(x),
"textbook": lambda x: self.digest_textbook(x),
"textbook_link": lambda x: self.digest_textbook_link(x),
"eval": lambda x: self.digest_eval(x),
}
if self.tracker.has_viewer("progressbar"):
bar = self.tracker.get_viewer("progressbar").bar
for obj in bar(make_list(self.data)):
do_digestion[obj.kind](obj)
else:
for obj in make_list(self.data):
do_digestion[obj.kind](obj)
self.wrap_up()
def ingest_meeting(self, section, clean_only=False):
"""Create meeting ingested json map.
Args:
section (dict): validated section object
Returns:
dict: meeting
"""
year = str(self._get('year'))
term = self._get('term', 'semester')
if section.get('code') is None:
year = None
term = None
meeting = {
'kind': 'meeting',
'course': section.get('course'),
'section': {
'code': section.get('code'),
'year': year,
'term': term,
},
'days': make_list(self._get('days', 'day')),
'dates': make_list(self._get('dates', 'date')),
'time': self._resolve_time(),
'location': self._resolve_location()
}
meeting = clean(meeting)
if clean_only:
return meeting
self._validate_and_log(meeting)
if 'time' in meeting:
self.tracker.time = meeting['time']['start']
self.tracker.time = meeting['time']['end']
return meeting
def digest(self):
'''Digest data.'''
do_digestion = {
'course': lambda x: self.digest_course(x),
'section': lambda x: self.digest_section(x),
'meeting': lambda x: self.digest_meeting(x),
'textbook': lambda x: self.digest_textbook(x),
'textbook_link': lambda x: self.digest_textbook_link(x),
}
for obj in make_list(self.data):
do_digestion[obj.kind](obj)
self.wrap_up()
def ingest_course(self):
"""Create course json from info in model map.
Returns:
dict: course
"""
course = {
'kind': 'course',
'school': {
'code': self.school,
'subdivisions': [
{
'code': self._get('school_subdivision_code'),
'name': self._get('school_subdivision_name')
}
]
},
'code': self._get('course_code', 'code', 'course'),
'name': titlize(self._get('name', 'course_name')),
'department': self._resolve_department(),
'credits': safe_cast(self._get('credits', 'num_credits'), float, default=0.),
'prerequisites': make_list(self._get('prerequisites', 'prereqs')),
'corequisites': make_list(self._get('corequisites', 'coreqs')),
'exclusions': make_list(self._get('exclusions')),
'areas': make_list(self._get('areas')),
'level': self._get('level'),
'cores': make_list(self._get('cores')),
'geneds': make_list(self._get('geneds')),
'sections': self._get('sections'),
'homepage': self._get('homepage', 'website'),
'same_as': make_list(self._get('same_as')),
'description': self._get('description', 'descr'),
# 'description': extract_info_from_text(
# self._get('description', 'descr'),
# inject=self
# ),
}
course = clean(course)
self._validate_and_log(course)
if 'department' in course:
self.tracker.department = course['department']
return course
def ingest_meeting(self, section, clean_only=False):
"""Create meeting ingested json map.
Args:
section (dict): validated section object
Returns:
dict: meeting
"""
year = str(self._get("year"))
term = self._get("term", "semester")
if section.get("code") is None:
year = None
term = None
meeting = {
"kind": "meeting",
"course": section.get("course"),
"section": {
"code": section.get("code"),
"year": year,
"term": term,
},
"days": make_list(self._get("days", "day")),
"dates": self._resolve_date(),
"time": self._resolve_time(),
"location": self._resolve_location(),
}
meeting = clean(meeting)
if clean_only:
return meeting
self._validate_and_log(meeting)
if "time" in meeting:
self.tracker.time = meeting["time"]["start"]
self.tracker.time = meeting["time"]["end"]
return meeting
def test_make_list(self):
self.assertEqual([], make_list())
self.assertEqual([], make_list(None))
self.assertEqual(['hello'], make_list('hello'))
self.assertEqual(['hello'], make_list(['hello']))
self.assertEqual([{1: 1}], make_list({1: 1}))
def test_make_list(self):
self.assertEqual([], make_list())
self.assertEqual([], make_list(None))
self.assertEqual(['hello'], make_list('hello'))
self.assertEqual(['hello'], make_list(['hello']))
self.assertEqual([{1: 1}], make_list({1: 1}))
def test_make_list(self):
self.assertEqual([], make_list())
self.assertEqual([], make_list(None))
self.assertEqual(["hello"], make_list("hello"))
self.assertEqual(["hello"], make_list(["hello"]))
self.assertEqual([{1: 1}], make_list({1: 1}))
def ingest_course(self):
"""Create course json from info in model map.
Returns:
dict: course
"""
course = {
"kind":
"course",
"school": {
"code":
self.school,
"subdivisions": [{
"code": self._get("school_subdivision_code"),
"name": self._get("school_subdivision_name"),
}],
},
"code":
self._get("course_code", "code", "course"),
"name":
titlize(self._get("name", "course_name")),
"department":
self._resolve_department(),
"credits":
safe_cast(self._get("credits", "num_credits"), float, default=0.0),
"prerequisites":
make_list(self._get("prerequisites", "prereqs")),
"corequisites":
make_list(self._get("corequisites", "coreqs")),
"exclusions":
make_list(self._get("exclusions")),
"areas":
make_list(self._get("areas")),
"level":
self._get("level"),
"cores":
make_list(self._get("cores")),
"geneds":
make_list(self._get("geneds")),
"sections":
self._get("sections"),
"homepage":
self._get("homepage", "website"),
"same_as":
make_list(self._get("same_as")),
"description":
self._get("description", "descr"),
"pos":
make_list(self._get("pos")),
"writing_intensive":
self._get("writing_intensive"),
"sub_school":
self._get("sub_school"),
# 'description': extract_info_from_text(
# self._get('description', 'descr'),
# inject=self
# ),
}
course = clean(course)
self._validate_and_log(course)
if "department" in course:
self.tracker.department = course["department"]
return course
