def scrape(self): prefix = ntnu.prefix(self.semester) url = 'http://www.ntnu.no/studieinformasjon/timeplan/%s/' % prefix room_codes = {} for code, name in fetch_rooms(): room_codes.setdefault(name, []).append(code) for course in self.course_queryset(): code = '%s-%s' % (course.code, course.version) root = fetch.html(url, query={'emnekode': code.encode('latin1')}) if root is None: continue for h1 in root.cssselect(u'.hovedramme h1'): if course.code in h1.text_content(): table = root.cssselect('.hovedramme table')[1] break else: logging.debug("Couldn't load any info for %s", course.code) continue lecture_type = None for tr in table.cssselect('tr')[1:-1]: data = parse_row(tr, room_codes) if data.get('lecture_type', None): lecture_type = data['lecture_type'] elif data: data.update({'course': course, 'type': lecture_type}) yield data
def scrape(self): prefix = ntnu.prefix(self.semester) url = 'http://www.ntnu.no/studieinformasjon/timeplan/%s/' % prefix room_codes = {} for code, name in fetch_rooms(): room_codes.setdefault(name, []).append(code) for course in self.course_queryset(): code = '%s-%s' % (course.code, course.version) root = fetch.html(url, query={'emnekode': code.encode('latin1')}) if root is None: continue for h1 in root.cssselect(u'.hovedramme h1'): if course.code in h1.text_content(): table = root.cssselect('.hovedramme table')[1]; break else: logging.debug("Couldn't load any info for %s", course.code) continue lecture_type = None for tr in table.cssselect('tr')[1:-1]: data = parse_row(tr, room_codes) if data.get('lecture_type', None): lecture_type = data['lecture_type'] elif data: data.update({'course': course, 'type': lecture_type}) yield data
def scrape(self): prefix = ntnu.prefix(self.semester, template='{year}{letter}') url = 'http://www.ntnu.no/eksamen/plan/%s/dato.XML' % prefix courses = Course.objects.filter(semester=self.semester) courses = {c.code: c for c in courses} root = fetch.xml(url) if root is None: return for row in root.xpath('//dato/dato_row'): course_code = get(row, 'emnekode') course_version = get(row, 'versjonskode') status_code = get(row, 'vurdstatuskode') if status_code != 'ORD': continue elif not ntnu.valid_course_code(course_code): logging.warning("Invalid course code: %s", course_code) continue elif course_code not in courses: logging.debug("Unknown course %s.", course_code) continue elif not self.should_proccess_course(code): continue combination = get(row, 'vurdkombkode') duration = get(row, 'varighettimer') exam_date = get(row, 'dato_eksamen') exam_semester = get(row, 'terminkode_gjelder_i') exam_time = get(row, 'klokkeslett_fremmote_tid') exam_year = get(row, 'arstall_gjelder_i') handin_date = get(row, 'dato_innlevering') handin_time = get(row, 'klokkeslett_innlevering') handout_date = get(row, 'dato_uttak') handout_time = get(row, 'klokkeslett_uttak') type_code = get(row, 'vurderingsformkode') type_name = get(row, 'vurderingskombinasjon_vurdkombnavn_bokmal') if not type_code: logging.warning('Missing exam type for %s', course_code) continue yield { 'course': courses[course_code], 'exam_date': utils.parse_date(handin_date or exam_date), 'exam_time': utils.parse_time(handin_time or exam_time), 'combination': combination, 'handout_date': utils.parse_date(handout_date), 'handout_time': utils.parse_time(handout_time), 'type': self.exam_type(type_code, type_name), 'duration': duration }
def scrape(self): prefix = ntnu.prefix(self.semester, template='{year}{letter}') url = 'http://www.ntnu.no/eksamen/plan/%s/dato.XML' % prefix courses = Course.objects.filter(semester=self.semester) courses = {c.code: c for c in courses} root = fetch.xml(url) if root is None: return for row in root.xpath('//dato/dato_row'): course_code = get(row, 'emnekode') course_version = get(row, 'versjonskode') status_code = get(row, 'vurdstatuskode') if status_code != 'ORD': continue elif not ntnu.valid_course_code(course_code): logging.warning("Invalid course code: %s", course_code) continue elif course_code not in courses: logging.debug("Unknown course %s.", course_code) continue elif not self.should_proccess_course(code): continue combination = get(row, 'vurdkombkode') duration = get(row, 'varighettimer') exam_date = get(row, 'dato_eksamen') exam_semester = get(row, 'terminkode_gjelder_i') exam_time = get(row, 'klokkeslett_fremmote_tid') exam_year = get(row, 'arstall_gjelder_i') handin_date = get(row, 'dato_innlevering') handin_time = get(row, 'klokkeslett_innlevering') handout_date = get(row, 'dato_uttak') handout_time = get(row, 'klokkeslett_uttak') type_code = get(row, 'vurderingsformkode') type_name = get(row, 'vurderingskombinasjon_vurdkombnavn_bokmal') if not type_code: logging.warning('Missing exam type for %s', course_code) continue yield {'course': courses[course_code], 'exam_date': utils.parse_date(handin_date or exam_date), 'exam_time': utils.parse_time(handin_time or exam_time), 'combination': combination, 'handout_date': utils.parse_date(handout_date), 'handout_time': utils.parse_time(handout_time), 'type': self.exam_type(type_code, type_name), 'duration': duration}
def scrape(self): prefix = ntnu.prefix(self.semester) groups = {} courses = Course.objects.filter(semester=self.semester) courses = {c.code: c for c in courses} query = ('SELECT aktkode, studieprogramkode FROM ' '%s_akt_studieprogram') % prefix for row in fetch.sql('ntnu', query): groups.setdefault(row.aktkode, set()).add(row.studieprogramkode) query = ('SELECT emnekode, typenavn, dag, start, slutt, uke, romnr, ' 'romnavn, larer, aktkode FROM %s_timeplan ORDER BY emnekode, ' 'dag, start, slutt, uke, romnavn, aktkode') % prefix for row in fetch.sql('ntnu', query): code, version = ntnu.parse_course(row.emnekode) if not code: logging.warning('Skipped invalid course name: %s', row.emnekode) continue elif code not in courses: logging.debug("Unknown course %s.", code) continue elif not self.should_proccess_course(code): continue yield { 'course': courses[code], 'type': row.typenavn, 'day': utils.parse_day_of_week(row.dag), 'start': utils.parse_time(row.start), 'end': utils.parse_time(row.slutt), 'weeks': utils.parse_weeks(row.uke), 'rooms': zip(utils.split(row.romnr, '#'), utils.split(row.romnavn, '#')), 'lecturers': utils.split(row.larer, '#'), 'groups': groups.get(row.aktkode, set()) }
def scrape(self): prefix = ntnu.prefix(self.semester) query = "SELECT emnekode, emnenavn FROM %s_fs_emne" % prefix for row in fetch.sql('ntnu', query): code, version = ntnu.parse_course(row.emnekode) if not code: logging.warning('Skipped invalid course name: %s', row.emnekode) continue yield {'code': code, 'name': row.emnenavn, 'version': version, 'url': 'http://www.ntnu.no/studier/emner/%s' % code}
def scrape(self): prefix = ntnu.prefix(self.semester) query = "SELECT emnekode, emnenavn FROM %s_fs_emne" % prefix for row in fetch.sql("ntnu", query): code, version = ntnu.parse_course(row.emnekode) if not code: logging.warning("Skipped invalid course name: %s", row.emnekode) continue if not self.should_proccess_course(code): continue yield { "code": code, "name": row.emnenavn, "version": version, "url": "http://www.ntnu.no/studier/emner/%s" % code, }
def scrape(self): prefix = ntnu.prefix(self.semester) query = "SELECT emnekode, emnenavn FROM %s_fs_emne" % prefix for row in fetch.sql('ntnu', query): code, version = ntnu.parse_course(row.emnekode) if not code: logging.warning('Skipped invalid course name: %s', row.emnekode) continue if not self.should_proccess_course(code): continue yield { 'code': code, 'name': row.emnenavn, 'version': version, 'url': 'http://www.ntnu.no/studier/emner/%s' % code }
def scrape(self): prefix = ntnu.prefix(self.semester) groups = {} courses = Course.objects.filter(semester=self.semester) courses = {c.code: c for c in courses} query = ("SELECT aktkode, studieprogramkode FROM " "%s_akt_studieprogram") % prefix for row in fetch.sql("ntnu", query): groups.setdefault(row.aktkode, set()).add(row.studieprogramkode) query = ( "SELECT emnekode, typenavn, dag, start, slutt, uke, romnr, " "romnavn, larer, aktkode FROM %s_timeplan ORDER BY emnekode, " "dag, start, slutt, uke, romnavn, aktkode" ) % prefix for row in fetch.sql("ntnu", query): code, version = ntnu.parse_course(row.emnekode) if not code: logging.warning("Skipped invalid course name: %s", row.emnekode) continue elif code not in courses: logging.debug("Unknown course %s.", code) continue elif not self.should_proccess_course(code): continue yield { "course": courses[code], "type": row.typenavn, "day": utils.parse_day_of_week(row.dag), "start": utils.parse_time(row.start), "end": utils.parse_time(row.slutt), "weeks": utils.parse_weeks(row.uke), "rooms": zip(utils.split(row.romnr, "#"), utils.split(row.romnavn, "#")), "lecturers": utils.split(row.larer, "#"), "groups": groups.get(row.aktkode, set()), }
def scrape(self): prefix = ntnu.prefix(self.semester) groups = {} courses = Course.objects.filter(semester=self.semester) courses = dict((c.code, c) for c in courses) query = ('SELECT aktkode, studieprogramkode FROM ' '%s_akt_studieprogram') % prefix for row in fetch.sql('ntnu', query): groups.setdefault(row.aktkode, set()).add(row.studieprogramkode) query = ('SELECT emnekode, typenavn, dag, start, slutt, uke, romnr, ' 'romnavn, larer, aktkode FROM %s_timeplan ORDER BY emnekode, ' 'dag, start, slutt, uke, romnavn, aktkode') % prefix for row in fetch.sql('ntnu', query): code, version = ntnu.parse_course(row.emnekode) if not code: logging.warning('Skipped invalid course name: %s', row.emnekode) continue elif code not in courses: logging.debug("Unknown course %s.", code) continue yield {'course': courses[code], 'type': row.typenavn, 'day': utils.parse_day_of_week(row.dag), 'start': utils.parse_time(row.start), 'end': utils.parse_time(row.slutt), 'weeks': utils.parse_weeks(row.uke), 'rooms': zip(utils.split(row.romnr, '#'), utils.split(row.romnavn, '#')), 'lecturers': utils.split(row.larer, '#'), 'groups': groups.get(row.aktkode, set())}