def fetch_pois( tag): campuses = fetch.json('http://use.mazemap.com/api/campuscollections/?tag=%s' % tag) base_url = 'http://api.mazemap.com/api/pois/?campusid=%s' pois = {} for campus in campuses['children']: data = fetch.json(base_url % campus['campusId']) for p in data['pois']: if p['identifier'] and not p['deleted']: pois[normalize(p['identifier'])] = p return pois
def fetch_pois(tag): campuses = fetch.json( 'http://use.mazemap.com/api/campuscollections/?tag=%s' % tag) base_url = 'http://api.mazemap.com/api/pois/?campusid=%s' pois = {} for campus in campuses['children']: data = fetch.json(base_url % campus['campusId']) for p in data['pois']: if p['identifier'] and not p['deleted']: pois[normalize(p['identifier'])] = p return pois
def fetch_course_lectures(semester, course): url = 'https://tp.uio.no/ntnu/ws/1.4/' query = { 'sem': convert_semester(semester), 'id': course.code.encode('utf-8') } result = fetch.json(url, query=query) if not result: query['termnr'] = 1 result = fetch.json(url, query=query) return result
def fetch_params(field, **kwargs): data = fetch.json('http://www.akademika.no/pensumlister/load', query=kwargs) for value in data.get(field, {}): if value == '0': continue yield value
def scrape(self): url = 'http://www.ime.ntnu.no/api/schedule/%%s/%s/%s' % ( TERM_MAPPING[self.semester.type].lower(), self.semester.year) for course in Course.objects.filter(semester=self.semester).order_by('code'): result = fetch.json(url % course.code.encode('utf-8')) if not result: continue for activity in result['activity'] or []: for schedule in activity['activitySchedules']: if 'activityDescription' not in activity: logging.warning('A %s lecture does not have a type', course.code) continue yield {'course': course, 'type': activity['activityDescription'], 'day': schedule['dayNumber'], 'start': utils.parse_time(schedule['start']), 'end': utils.parse_time(schedule['end']), 'weeks': utils.parse_weeks(schedule['weeks'], ','), 'rooms': [(r['lydiaCode'], r['location']) for r in schedule.get('rooms', [])], 'lecturers': [s['name'] for s in activity.get('staff', [])], 'groups': activity.get('studyProgrammes', [])}
def fetch_courses(semester): if semester.type == Semester.FALL: year = semester.year else: year = semester.year - 1 url = 'http://www.ntnu.no/web/studier/emnesok' query = { 'p_p_lifecycle': '2', 'p_p_id': 'courselistportlet_WAR_courselistportlet', 'p_p_mode': 'view', 'p_p_resource_id': 'fetch-courselist-as-json' } data = { 'english': 0, 'pageNo': 1, 'semester': year, 'sortOrder': '+title' } if semester.type == Semester.FALL: data['courseAutumn'] = 1 else: data['courseSpring'] = 1 while True: result = fetch.json(url, query=query, data=data, verbose=True) data['pageNo'] += 1 if not result['courses']: break for course in result['courses']: yield course
def scrape(self): term = TERM_MAPPING[self.semester.type].lower() url = BASE + '/schedule/%%s/%s/%s' % (term, self.semester.year) for course in self.course_queryset(): result = fetch.json(url % course.code.encode('utf-8')) if not result: continue for activity in result['activity'] or []: for schedule in activity['activitySchedules']: if 'activityDescription' not in activity: logging.warning('A %s lecture does not have a type', course.code) continue yield {'course': course, 'type': activity['activityDescription'], 'day': schedule['dayNumber'], 'start': utils.parse_time(schedule['start']), 'end': utils.parse_time(schedule['end']), 'weeks': utils.parse_weeks(schedule['weeks'], ','), 'rooms': [(r['lydiaCode'], r['location']) for r in schedule.get('rooms', [])], 'lecturers': [s['name'] for s in activity.get('staff', [])], 'groups': activity.get('studyProgrammes', [])}
def fetch_courses(semester, prefix=None): courses = fetch.json(BASE + '/course/-')['course'] for course in courses: if not ntnu.valid_course_code(course['code']): logging.warning('Skipped invalid course name: %s', course['code']) continue # TODO: shouldn't reimplement should_proccess_course if prefix and not course['code'].startswith(prefix): continue result = fetch_course(course['code']) if not result: continue if semester.year < result['taughtFromYear']: continue if result['lastYearTaught'] and semester.year > result['lastYearTaught']: continue if result['versionCode'] != course['versionCode']: continue if semester.type == semester.FALL and result['taughtInAutumn']: yield result elif semester.type == semester.SPRING and result['taughtInSpring']: yield result
def scrape(self): buildings = fetch_buildings() qs = self.queryset() qs = qs.filter(lecture__course__semester=self.semester) qs = qs.distinct() for code, name, url in qs.values_list('code', 'name', 'url'): if not code or url: continue data = fetch.json(BASE + '/fdv/rooms/lydiacode:%s' % code) if not data: continue room = data['rooms'][0] url = 'http://www.ntnu.no/kart/%s/%s' % ( buildings[room['buildingId']], room['nr']) name = (room['name'] or '').strip() or 'Rom %s' % room['nr'] root = fetch.html(url) if root: for link in root.cssselect('link[rel="canonical"]'): if link.attrib['href'] != 'http://www.ntnu.no/kart': url = link.attrib['href'] yield {'code': code, 'name': name, 'url': url}
def fetch_course_lectures(semester, course): url = 'http://www.ntnu.no/web/studier/emner' query = { 'p_p_id': 'coursedetailsportlet_WAR_courselistportlet', 'p_p_lifecycle': 2, 'p_p_resource_id': 'timetable', '_coursedetailsportlet_WAR_courselistportlet_year': semester.year, '_coursedetailsportlet_WAR_courselistportlet_courseCode': course.code.encode('utf-8'), 'year': semester.year, 'version': course.version, } return fetch.json(url, query=query, data={})['course']
def fetch_courses(semester): courses = fetch.json('http://www.ime.ntnu.no/api/course/-')['course'] for course in courses: if not ntnu.valid_course_code(course['code']): logging.warning('Skipped invalid course name: %s', course['code']) continue result = fetch_course(course['code']) if not result: continue if semester.year < result['taughtFromYear']: continue if result['lastYearTaught'] and semester.year > result['lastYearTaught']: continue if semester.type == semester.FALL and result['taughtInAutumn']: yield result elif semester.type == semester.SPRING and result['taughtInSpring']: yield result
def fetch_courses(semester): """ https://www.ntnu.no/web/studier/emnesok? p_p_id=courselistportlet_WAR_courselistportlet p_p_lifecycle=2 p_p_state=normal p_p_mode=view p_p_resource_id=fetch-courselist-as-json p_p_cacheability=cacheLevelPage p_p_col_id=column-1 p_p_col_pos=1 p_p_col_count=2 X-Requested-With: XMLHttpRequest Cookie: GUEST_LANGUAGE_ID=nb_NO Data: semester=2018 gjovik=0 trondheim=1 alesund=0 faculty=-1 institute=-1 multimedia=0 english=0 phd=0 courseAutumn=0 courseSpring=1 courseSummer=0 searchQueryString= pageNo=1 season=spring sortOrder=%2Btitle year= """ if semester.type == Semester.FALL: year = semester.year else: year = semester.year - 1 url = 'https://www.ntnu.no/web/studier/emnesok' query = { 'p_p_id': 'courselistportlet_WAR_courselistportlet', 'p_p_lifecycle': '2', 'p_p_mode': 'view', 'p_p_resource_id': 'fetch-courselist-as-json' } data = { 'english': 0, 'pageNo': 1, 'semester': year, 'sortOrder': '+title', } if semester.type == Semester.FALL: data['courseAutumn'] = 1 else: data['courseSpring'] = 1 while True: result = fetch.json(url, query=query, data=data, verbose=True) data['pageNo'] += 1 if not result['courses']: break for course in result['courses']: yield course
def fetch_course(code): code = code.lower().encode('utf-8') return fetch.json('http://www.ime.ntnu.no/api/course/%s' % code)['course']
def fetch_buildings(): data = fetch.json(BASE + '/fdv/buildings') return {b['id']: b['nr'] for b in data.get('buildings', [])}
def fetch_course(code): code = code.lower().encode('utf-8') return fetch.json(BASE + '/course/%s' % code)['course']