def fetch_pois( tag):
campuses = fetch.json('http://use.mazemap.com/api/campuscollections/?tag=%s' % tag)
base_url = 'http://api.mazemap.com/api/pois/?campusid=%s'
pois = {}
for campus in campuses['children']:
data = fetch.json(base_url % campus['campusId'])
for p in data['pois']:
if p['identifier'] and not p['deleted']:
pois[normalize(p['identifier'])] = p
return pois
def fetch_pois(tag):
campuses = fetch.json(
'http://use.mazemap.com/api/campuscollections/?tag=%s' % tag)
base_url = 'http://api.mazemap.com/api/pois/?campusid=%s'
pois = {}
for campus in campuses['children']:
data = fetch.json(base_url % campus['campusId'])
for p in data['pois']:
if p['identifier'] and not p['deleted']:
pois[normalize(p['identifier'])] = p
return pois
def fetch_course_lectures(semester, course):
url = 'https://tp.uio.no/ntnu/ws/1.4/'
query = {
'sem': convert_semester(semester),
'id': course.code.encode('utf-8')
}
result = fetch.json(url, query=query)
if not result:
query['termnr'] = 1
result = fetch.json(url, query=query)
return result
def fetch_params(field, **kwargs):
data = fetch.json('http://www.akademika.no/pensumlister/load',
query=kwargs)
for value in data.get(field, {}):
if value == '0':
continue
yield value
def scrape(self):
url = 'http://www.ime.ntnu.no/api/schedule/%%s/%s/%s' % (
TERM_MAPPING[self.semester.type].lower(), self.semester.year)
for course in Course.objects.filter(semester=self.semester).order_by('code'):
result = fetch.json(url % course.code.encode('utf-8'))
if not result:
continue
for activity in result['activity'] or []:
for schedule in activity['activitySchedules']:
if 'activityDescription' not in activity:
logging.warning('A %s lecture does not have a type', course.code)
continue
yield {'course': course,
'type': activity['activityDescription'],
'day': schedule['dayNumber'],
'start': utils.parse_time(schedule['start']),
'end': utils.parse_time(schedule['end']),
'weeks': utils.parse_weeks(schedule['weeks'], ','),
'rooms': [(r['lydiaCode'], r['location'])
for r in schedule.get('rooms', [])],
'lecturers': [s['name'] for s in activity.get('staff', [])],
'groups': activity.get('studyProgrammes', [])}
def fetch_courses(semester):
if semester.type == Semester.FALL:
year = semester.year
else:
year = semester.year - 1
url = 'http://www.ntnu.no/web/studier/emnesok'
query = {
'p_p_lifecycle': '2',
'p_p_id': 'courselistportlet_WAR_courselistportlet',
'p_p_mode': 'view',
'p_p_resource_id': 'fetch-courselist-as-json'
}
data = {
'english': 0,
'pageNo': 1,
'semester': year,
'sortOrder': '+title'
}
if semester.type == Semester.FALL:
data['courseAutumn'] = 1
else:
data['courseSpring'] = 1
while True:
result = fetch.json(url, query=query, data=data, verbose=True)
data['pageNo'] += 1
if not result['courses']:
break
for course in result['courses']:
yield course
def scrape(self):
term = TERM_MAPPING[self.semester.type].lower()
url = BASE + '/schedule/%%s/%s/%s' % (term, self.semester.year)
for course in self.course_queryset():
result = fetch.json(url % course.code.encode('utf-8'))
if not result:
continue
for activity in result['activity'] or []:
for schedule in activity['activitySchedules']:
if 'activityDescription' not in activity:
logging.warning('A %s lecture does not have a type', course.code)
continue
yield {'course': course,
'type': activity['activityDescription'],
'day': schedule['dayNumber'],
'start': utils.parse_time(schedule['start']),
'end': utils.parse_time(schedule['end']),
'weeks': utils.parse_weeks(schedule['weeks'], ','),
'rooms': [(r['lydiaCode'], r['location'])
for r in schedule.get('rooms', [])],
'lecturers': [s['name'] for s in activity.get('staff', [])],
'groups': activity.get('studyProgrammes', [])}
def fetch_courses(semester, prefix=None):
courses = fetch.json(BASE + '/course/-')['course']
for course in courses:
if not ntnu.valid_course_code(course['code']):
logging.warning('Skipped invalid course name: %s', course['code'])
continue
# TODO: shouldn't reimplement should_proccess_course
if prefix and not course['code'].startswith(prefix):
continue
result = fetch_course(course['code'])
if not result:
continue
if semester.year < result['taughtFromYear']:
continue
if result['lastYearTaught'] and semester.year > result['lastYearTaught']:
continue
if result['versionCode'] != course['versionCode']:
continue
if semester.type == semester.FALL and result['taughtInAutumn']:
yield result
elif semester.type == semester.SPRING and result['taughtInSpring']:
yield result
def scrape(self):
buildings = fetch_buildings()
qs = self.queryset()
qs = qs.filter(lecture__course__semester=self.semester)
qs = qs.distinct()
for code, name, url in qs.values_list('code', 'name', 'url'):
if not code or url:
continue
data = fetch.json(BASE + '/fdv/rooms/lydiacode:%s' % code)
if not data:
continue
room = data['rooms'][0]
url = 'http://www.ntnu.no/kart/%s/%s' % (
buildings[room['buildingId']], room['nr'])
name = (room['name'] or '').strip() or 'Rom %s' % room['nr']
root = fetch.html(url)
if root:
for link in root.cssselect('link[rel="canonical"]'):
if link.attrib['href'] != 'http://www.ntnu.no/kart':
url = link.attrib['href']
yield {'code': code, 'name': name, 'url': url}
def fetch_course_lectures(semester, course):
url = 'http://www.ntnu.no/web/studier/emner'
query = {
'p_p_id': 'coursedetailsportlet_WAR_courselistportlet',
'p_p_lifecycle': 2,
'p_p_resource_id': 'timetable',
'_coursedetailsportlet_WAR_courselistportlet_year': semester.year,
'_coursedetailsportlet_WAR_courselistportlet_courseCode': course.code.encode('utf-8'),
'year': semester.year,
'version': course.version,
}
return fetch.json(url, query=query, data={})['course']
def fetch_courses(semester):
courses = fetch.json('http://www.ime.ntnu.no/api/course/-')['course']
for course in courses:
if not ntnu.valid_course_code(course['code']):
logging.warning('Skipped invalid course name: %s', course['code'])
continue
result = fetch_course(course['code'])
if not result:
continue
if semester.year < result['taughtFromYear']:
continue
if result['lastYearTaught'] and semester.year > result['lastYearTaught']:
continue
if semester.type == semester.FALL and result['taughtInAutumn']:
yield result
elif semester.type == semester.SPRING and result['taughtInSpring']:
yield result
def fetch_courses(semester):
"""
https://www.ntnu.no/web/studier/emnesok?
p_p_id=courselistportlet_WAR_courselistportlet
p_p_lifecycle=2
p_p_state=normal
p_p_mode=view
p_p_resource_id=fetch-courselist-as-json
p_p_cacheability=cacheLevelPage
p_p_col_id=column-1
p_p_col_pos=1
p_p_col_count=2
X-Requested-With: XMLHttpRequest
Cookie: GUEST_LANGUAGE_ID=nb_NO
Data:
semester=2018
gjovik=0
trondheim=1
alesund=0
faculty=-1
institute=-1
multimedia=0
english=0
phd=0
courseAutumn=0
courseSpring=1
courseSummer=0
searchQueryString=
pageNo=1
season=spring
sortOrder=%2Btitle
year=
"""
if semester.type == Semester.FALL:
year = semester.year
else:
year = semester.year - 1
url = 'https://www.ntnu.no/web/studier/emnesok'
query = {
'p_p_id': 'courselistportlet_WAR_courselistportlet',
'p_p_lifecycle': '2',
'p_p_mode': 'view',
'p_p_resource_id': 'fetch-courselist-as-json'
}
data = {
'english': 0,
'pageNo': 1,
'semester': year,
'sortOrder': '+title',
}
if semester.type == Semester.FALL:
data['courseAutumn'] = 1
else:
data['courseSpring'] = 1
while True:
result = fetch.json(url, query=query, data=data, verbose=True)
data['pageNo'] += 1
if not result['courses']:
break
for course in result['courses']:
yield course
def fetch_course(code):
code = code.lower().encode('utf-8')
return fetch.json('http://www.ime.ntnu.no/api/course/%s' % code)['course']
def fetch_params(field, **kwargs):
data = fetch.json('http://www.akademika.no/pensumlister/load', query=kwargs)
for value in data.get(field, {}):
if value == '0':
continue
yield value
def fetch_buildings():
data = fetch.json(BASE + '/fdv/buildings')
return {b['id']: b['nr'] for b in data.get('buildings', [])}
def fetch_course(code):
code = code.lower().encode('utf-8')
return fetch.json(BASE + '/course/%s' % code)['course']
