def scrape(self):
url = 'http://www.ime.ntnu.no/api/schedule/%%s/%s/%s' % (
TERM_MAPPING[self.semester.type].lower(), self.semester.year)
for course in Course.objects.filter(semester=self.semester).order_by('code'):
result = fetch.json(url % course.code.encode('utf-8'))
if not result:
continue
for activity in result['activity'] or []:
for schedule in activity['activitySchedules']:
if 'activityDescription' not in activity:
logging.warning('A %s lecture does not have a type', course.code)
continue
yield {'course': course,
'type': activity['activityDescription'],
'day': schedule['dayNumber'],
'start': utils.parse_time(schedule['start']),
'end': utils.parse_time(schedule['end']),
'weeks': utils.parse_weeks(schedule['weeks'], ','),
'rooms': [(r['lydiaCode'], r['location'])
for r in schedule.get('rooms', [])],
'lecturers': [s['name'] for s in activity.get('staff', [])],
'groups': activity.get('studyProgrammes', [])}
def scrape(self):
if self.semester.type == Semester.FALL:
ntnu_semeter = u'%d_HØST' % self.semester.year
else:
ntnu_semeter = u'%d_VÅR' % self.semester.year
for c in self.course_queryset():
course = fetch_course_lectures(self.semester, c)
for activity in course.get('summarized', []):
if activity['artermin'] != ntnu_semeter:
continue
yield {
'course':
c,
'type':
activity.get('name', activity['acronym']),
'day':
activity['dayNum'] - 1,
'start':
utils.parse_time(activity['from']),
'end':
utils.parse_time(activity['to']),
'weeks':
utils.parse_weeks(','.join(activity['weeks']), ','),
'rooms': [(r['id'], r['room'], r.get('url'))
for r in activity.get('rooms', [])],
'groups':
activity.get('studyProgramKeys', []),
'lecturers': [],
'title':
activity.get('title'),
}
def scrape(self):
term = TERM_MAPPING[self.semester.type].lower()
url = BASE + '/schedule/%%s/%s/%s' % (term, self.semester.year)
for course in self.course_queryset():
result = fetch.json(url % course.code.encode('utf-8'))
if not result:
continue
for activity in result['activity'] or []:
for schedule in activity['activitySchedules']:
if 'activityDescription' not in activity:
logging.warning('A %s lecture does not have a type', course.code)
continue
yield {'course': course,
'type': activity['activityDescription'],
'day': schedule['dayNumber'],
'start': utils.parse_time(schedule['start']),
'end': utils.parse_time(schedule['end']),
'weeks': utils.parse_weeks(schedule['weeks'], ','),
'rooms': [(r['lydiaCode'], r['location'])
for r in schedule.get('rooms', [])],
'lecturers': [s['name'] for s in activity.get('staff', [])],
'groups': activity.get('studyProgrammes', [])}
def scrape(self):
prefix = ntnu.prefix(self.semester, template='{year}{letter}')
url = 'http://www.ntnu.no/eksamen/plan/%s/dato.XML' % prefix
courses = Course.objects.filter(semester=self.semester)
courses = {c.code: c for c in courses}
root = fetch.xml(url)
if root is None:
return
for row in root.xpath('//dato/dato_row'):
course_code = get(row, 'emnekode')
course_version = get(row, 'versjonskode')
status_code = get(row, 'vurdstatuskode')
if status_code != 'ORD':
continue
elif not ntnu.valid_course_code(course_code):
logging.warning("Invalid course code: %s", course_code)
continue
elif course_code not in courses:
logging.debug("Unknown course %s.", course_code)
continue
elif not self.should_proccess_course(code):
continue
combination = get(row, 'vurdkombkode')
duration = get(row, 'varighettimer')
exam_date = get(row, 'dato_eksamen')
exam_semester = get(row, 'terminkode_gjelder_i')
exam_time = get(row, 'klokkeslett_fremmote_tid')
exam_year = get(row, 'arstall_gjelder_i')
handin_date = get(row, 'dato_innlevering')
handin_time = get(row, 'klokkeslett_innlevering')
handout_date = get(row, 'dato_uttak')
handout_time = get(row, 'klokkeslett_uttak')
type_code = get(row, 'vurderingsformkode')
type_name = get(row, 'vurderingskombinasjon_vurdkombnavn_bokmal')
if not type_code:
logging.warning('Missing exam type for %s', course_code)
continue
yield {
'course': courses[course_code],
'exam_date': utils.parse_date(handin_date or exam_date),
'exam_time': utils.parse_time(handin_time or exam_time),
'combination': combination,
'handout_date': utils.parse_date(handout_date),
'handout_time': utils.parse_time(handout_time),
'type': self.exam_type(type_code, type_name),
'duration': duration
}
def scrape(self):
for c in self.course_queryset():
result = fetch_course_lectures(self.semester, c)
if 'data' not in result or not result['data']:
continue
for methods in result['data'].values():
for method in methods:
for sequence in method['eventsequences']:
current = None
for e in sequence['events']:
tmp = {
'day':
utils.parse_date(e['dtstart']).weekday(),
'start':
utils.parse_time(e['dtstart']),
'end':
utils.parse_time(e['dtend']),
'rooms': [(r['id'], r['roomname'], None)
for r in e.get('room', [])],
'groups':
process_groups(e.get('studentgroups', [])),
}
if not current:
current = {
'course':
c,
'type':
method.get('teaching-method-name',
'teaching-method'),
'weeks': [],
'lecturers': [],
}
current.update(tmp)
for key in tmp:
if current[key] != tmp[key]:
logging.warning('Mismatch %s: %s',
self.display(obj), key)
yield current
current = None
break
else:
current['weeks'].append(e['weeknr'])
if current:
yield current
def scrape(self):
prefix = ntnu.prefix(self.semester, template='{year}{letter}')
url = 'http://www.ntnu.no/eksamen/plan/%s/dato.XML' % prefix
courses = Course.objects.filter(semester=self.semester)
courses = {c.code: c for c in courses}
root = fetch.xml(url)
if root is None:
return
for row in root.xpath('//dato/dato_row'):
course_code = get(row, 'emnekode')
course_version = get(row, 'versjonskode')
status_code = get(row, 'vurdstatuskode')
if status_code != 'ORD':
continue
elif not ntnu.valid_course_code(course_code):
logging.warning("Invalid course code: %s", course_code)
continue
elif course_code not in courses:
logging.debug("Unknown course %s.", course_code)
continue
elif not self.should_proccess_course(code):
continue
combination = get(row, 'vurdkombkode')
duration = get(row, 'varighettimer')
exam_date = get(row, 'dato_eksamen')
exam_semester = get(row, 'terminkode_gjelder_i')
exam_time = get(row, 'klokkeslett_fremmote_tid')
exam_year = get(row, 'arstall_gjelder_i')
handin_date = get(row, 'dato_innlevering')
handin_time = get(row, 'klokkeslett_innlevering')
handout_date = get(row, 'dato_uttak')
handout_time = get(row, 'klokkeslett_uttak')
type_code = get(row, 'vurderingsformkode')
type_name = get(row, 'vurderingskombinasjon_vurdkombnavn_bokmal')
if not type_code:
logging.warning('Missing exam type for %s', course_code)
continue
yield {'course': courses[course_code],
'exam_date': utils.parse_date(handin_date or exam_date),
'exam_time': utils.parse_time(handin_time or exam_time),
'combination': combination,
'handout_date': utils.parse_date(handout_date),
'handout_time': utils.parse_time(handout_time),
'type': self.exam_type(type_code, type_name),
'duration': duration}
def scrape(self):
prefix = ntnu.prefix(self.semester)
groups = {}
courses = Course.objects.filter(semester=self.semester)
courses = {c.code: c for c in courses}
query = ('SELECT aktkode, studieprogramkode FROM '
'%s_akt_studieprogram') % prefix
for row in fetch.sql('ntnu', query):
groups.setdefault(row.aktkode, set()).add(row.studieprogramkode)
query = ('SELECT emnekode, typenavn, dag, start, slutt, uke, romnr, '
'romnavn, larer, aktkode FROM %s_timeplan ORDER BY emnekode, '
'dag, start, slutt, uke, romnavn, aktkode') % prefix
for row in fetch.sql('ntnu', query):
code, version = ntnu.parse_course(row.emnekode)
if not code:
logging.warning('Skipped invalid course name: %s',
row.emnekode)
continue
elif code not in courses:
logging.debug("Unknown course %s.", code)
continue
elif not self.should_proccess_course(code):
continue
yield {
'course':
courses[code],
'type':
row.typenavn,
'day':
utils.parse_day_of_week(row.dag),
'start':
utils.parse_time(row.start),
'end':
utils.parse_time(row.slutt),
'weeks':
utils.parse_weeks(row.uke),
'rooms':
zip(utils.split(row.romnr, '#'), utils.split(row.romnavn,
'#')),
'lecturers':
utils.split(row.larer, '#'),
'groups':
groups.get(row.aktkode, set())
}
def scrape(self):
# Only bother with courses that have already been loaded.
for course in Course.objects.filter(semester=self.semester).order_by('code'):
result = fetch_course(course.code)
if not result:
continue
for exam in result.get('assessment', []):
if not match_assessment(exam, self.semester):
continue
exam_date = exam.get('date', None)
exam_time = exam.get('appearanceTime', None)
handout_date = exam.get('withdrawalDate', None)
handin_date = exam.get('submissionDate', None)
duration = exam.get('duration', None)
combination = exam['combinationCode']
type_code = exam['assessmentFormCode']
type_name = exam['assessmentFormDescription']
yield {'course': course,
'exam_date': utils.parse_date(handin_date or exam_date),
'exam_time': utils.parse_time(exam_time),
'combination': combination,
'handout_date': utils.parse_date(handout_date),
'type': self.exam_type(type_code, type_name),
'duration': duration}
def scrape(self):
# Only bother with courses that have already been loaded.
for course in self.course_queryset():
result = fetch_course(course.code)
if not result:
continue
for exam in result.get('assessment', []):
if not match_assessment(exam, self.semester):
continue
exam_date = exam.get('date', None)
exam_time = exam.get('appearanceTime', None)
handout_date = exam.get('withdrawalDate', None)
handin_date = exam.get('submissionDate', None)
duration = exam.get('duration', None)
combination = exam['combinationCode']
type_code = exam['assessmentFormCode']
type_name = exam['assessmentFormDescription']
yield {'course': course,
'exam_date': utils.parse_date(handin_date or exam_date),
'exam_time': utils.parse_time(exam_time),
'combination': combination,
'handout_date': utils.parse_date(handout_date),
'type': self.exam_type(type_code, type_name),
'duration': duration}
def parse_row(tr, room_codes):
data = {}
for i, td in enumerate(tr.cssselect('td')):
if i == 0:
if td.attrib.get('colspan', 1) == '4':
lecture_type = utils.clean_string(td.text_content())
if lecture_type:
data['lecture_type'] = lecture_type
else:
time = td.cssselect('b')[0].text_content().strip()
raw_day, period = time.split(' ', 1)
raw_start, raw_end = period.split('-')
data['day'] = utils.parse_day_of_week(raw_day)
data['start'] = utils.parse_time(raw_start)
data['end'] = utils.parse_time(raw_end)
match = re.match('.*Uke: (.+)', td.text_content())
data['weeks'] = utils.parse_weeks(match.group(1))
elif i == 1 and len(td.cssselect('a')) > 0:
if len(td.cssselect('a')) > 1:
logging.warning('Multiple rooms links found, simply '
'using first one.')
a = td.cssselect('a')[0]
rooms = [a.text] + [e.tail for e in a]
data['rooms'] = []
for name in utils.clean_list(rooms, utils.clean_string):
if name not in room_codes:
data['rooms'].append((None, name))
continue
if len(room_codes[name]) > 1:
logging.warning(
'Multiple rooms with name %s, '
'simply using first code.', name)
data['rooms'].append((room_codes[name][0], name))
elif i == 2:
data['lecturers'] = [td.text] + [e.tail for e in td]
elif i == 3:
data['groups'] = [g.text_content() for g in td.cssselect('span')]
return data
def parse_row(tr, room_codes):
data = {}
for i, td in enumerate(tr.cssselect('td')):
if i == 0:
if td.attrib.get('colspan', 1) == '4':
lecture_type = utils.clean_string(td.text_content())
if lecture_type:
data['lecture_type'] = lecture_type
else:
time = td.cssselect('b')[0].text_content().strip()
raw_day, period = time.split(' ', 1)
raw_start, raw_end = period.split('-')
data['day'] = utils.parse_day_of_week(raw_day)
data['start'] = utils.parse_time(raw_start)
data['end'] = utils.parse_time(raw_end)
match = re.match('.*Uke: (.+)', td.text_content())
data['weeks'] = utils.parse_weeks(match.group(1))
elif i == 1 and len(td.cssselect('a')) > 0:
if len(td.cssselect('a')) > 1:
logging.warning('Multiple rooms links found, simply '
'using first one.')
a = td.cssselect('a')[0]
rooms = [a.text] + [e.tail for e in a]
data['rooms'] = []
for name in utils.clean_list(rooms, utils.clean_string):
if name not in room_codes:
data['rooms'].append((None, name))
continue
if len(room_codes[name]) > 1:
logging.warning('Multiple rooms with name %s, '
'simply using first code.', name)
data['rooms'].append((room_codes[name][0], name))
elif i == 2:
data['lecturers'] = [td.text] + [e.tail for e in td]
elif i == 3:
data['groups'] = [g.text_content() for g in td.cssselect('span')]
return data
def scrape(self):
prefix = ntnu.prefix(self.semester)
groups = {}
courses = Course.objects.filter(semester=self.semester)
courses = {c.code: c for c in courses}
query = ("SELECT aktkode, studieprogramkode FROM " "%s_akt_studieprogram") % prefix
for row in fetch.sql("ntnu", query):
groups.setdefault(row.aktkode, set()).add(row.studieprogramkode)
query = (
"SELECT emnekode, typenavn, dag, start, slutt, uke, romnr, "
"romnavn, larer, aktkode FROM %s_timeplan ORDER BY emnekode, "
"dag, start, slutt, uke, romnavn, aktkode"
) % prefix
for row in fetch.sql("ntnu", query):
code, version = ntnu.parse_course(row.emnekode)
if not code:
logging.warning("Skipped invalid course name: %s", row.emnekode)
continue
elif code not in courses:
logging.debug("Unknown course %s.", code)
continue
elif not self.should_proccess_course(code):
continue
yield {
"course": courses[code],
"type": row.typenavn,
"day": utils.parse_day_of_week(row.dag),
"start": utils.parse_time(row.start),
"end": utils.parse_time(row.slutt),
"weeks": utils.parse_weeks(row.uke),
"rooms": zip(utils.split(row.romnr, "#"), utils.split(row.romnavn, "#")),
"lecturers": utils.split(row.larer, "#"),
"groups": groups.get(row.aktkode, set()),
}
def scrape(self):
if self.semester.type == Semester.FALL:
ntnu_semeter = u'%d_HØST' % self.semester.year
else:
ntnu_semeter = u'%d_VÅR' % self.semester.year
for c in self.course_queryset():
course = fetch_course_lectures(self.semester, c)
for activity in course.get('summarized', []):
if activity['arsterminId'] != ntnu_semeter:
continue
yield {
'course': c,
'type': activity.get('description', activity['acronym']),
'day': activity['dayNum'] - 1,
'start': utils.parse_time(activity['from']),
'end': utils.parse_time(activity['to']),
'weeks': utils.parse_weeks(','.join(activity['weeks']), ','),
'rooms': [(r['syllabusKey'], r['romNavn'])
for r in activity.get('rooms', [])],
'groups': activity.get('studyProgramKeys', []),
'lecturers': [],
}
def scrape(self):
prefix = ntnu.prefix(self.semester)
groups = {}
courses = Course.objects.filter(semester=self.semester)
courses = dict((c.code, c) for c in courses)
query = ('SELECT aktkode, studieprogramkode FROM '
'%s_akt_studieprogram') % prefix
for row in fetch.sql('ntnu', query):
groups.setdefault(row.aktkode, set()).add(row.studieprogramkode)
query = ('SELECT emnekode, typenavn, dag, start, slutt, uke, romnr, '
'romnavn, larer, aktkode FROM %s_timeplan ORDER BY emnekode, '
'dag, start, slutt, uke, romnavn, aktkode') % prefix
for row in fetch.sql('ntnu', query):
code, version = ntnu.parse_course(row.emnekode)
if not code:
logging.warning('Skipped invalid course name: %s', row.emnekode)
continue
elif code not in courses:
logging.debug("Unknown course %s.", code)
continue
yield {'course': courses[code],
'type': row.typenavn,
'day': utils.parse_day_of_week(row.dag),
'start': utils.parse_time(row.start),
'end': utils.parse_time(row.slutt),
'weeks': utils.parse_weeks(row.uke),
'rooms': zip(utils.split(row.romnr, '#'),
utils.split(row.romnavn, '#')),
'lecturers': utils.split(row.larer, '#'),
'groups': groups.get(row.aktkode, set())}
