def main(semester,year):
# Hardcoded values
url_base = "http://ssbp.mycampus.ca"
url_action = "/prod/bwckschd.p_get_crse_unsec"
# Set metaData (used by database storer)
metaData = {
'term': semester,
'year': year
}
# Everything is in catch-all try block
try:
# Instantiate objects
courseData = []
pageParser = Parser(courseData)
# === PHASE 1: PARSING ===
# Set list of faculties to parse
facsToUse = acros.faculties
#facsToUse = ["ENGR"]
# Parse for each faculty
facSoFar = 0
for faculty in facsToUse:
facSoFar += 1
# Print out which faculty is being parsed
outta = str(facSoFar)+"/"+str(len(facsToUse))
logging.info("Parsing faculty: "+outta+" "+faculty+": "+acros.faculties[faculty])
# Instanciate page loader and get page
#pageLoader = PageLoader(url_base,url_action,semester,year,faculty)
#page = pageLoader.get_page(pageLoader.gen_url_and_data());
with open('last_source.html','r') as f:
page = ''.join(f.readlines());
bs = BowlShit(page)
"""
with open('last_source.html','w') as f:
f.write(page)
"""
# Parse page
pageParser.parse_course_info(page)
with open('last_object_dump.txt','w') as f:
util.log_anything_prettily(lambda t: f.write(t + "\n"),courseData)
#pickle.dump(courseData,f)
# === PHASE 2: STORING ===
frf = FRFStore()
frf.set_course_data(courseData,metaData)
frf.setup_default_connection()
frf.insert_data_as_offerings()
except SomethingWentWrong as e:
logging.exception("Something terrible(?) happened!!")
def insert_data_as_offerings(self, debug_offering=0):
# Choose variables to be used
courseData = self.courseData
metaData = self.metaData
# Make sure everything is okay
if courseData == None:
raise SomethingWentWrong("The courseData object was not set -_-")
debugSoFar = 0
courseSoFar = 0
# loop through each course
for course in courseData:
for section in course['classes']:
for classtime in section['times']:
# Not very important code - just for debugging
debugSoFar += 1
if debug_offering != 0:
if debugSoFar != debug_offering:
continue
if debugSoFar > debug_offering:
break
logging.debug("Offering #"+str(debugSoFar) + ", "+str(section['crn'],)+", "+course['program_code'])
# PREPARING DATA FOR DATABASE
weekalt = (classtime['week'] == "W1"
or classtime['week'] == "W2")
weekalt = 1 if weekalt else 0
logging.debug(" ~ adding a " + classtime['type'])
try:
classType = util.reverse_lookup(acros.class_types, classtime['type'])
except ValueError:
classType = "OTH"
start_time = classtime['start_time']
finish_time = classtime['finish_time']
start_date = classtime['start_date']
finish_date = classtime['finish_date']
# This deals with times/dates that are not available
try:
start_time = time.strftime("%H:%M:00", start_time )
finish_time = time.strftime("%H:%M:00", finish_time )
except TypeError:
start_time = None
finish_time = None
try:
start_date = time.strftime("%Y-%m-%d", start_date )
finish_date = time.strftime("%Y-%m-%d", finish_date )
except TypeError:
start_date = None
finish_date = None
# Determining campus
try:
campus = util.reverse_lookup(acros.campus_acronyms, section['campus'])
except ValueError:
campus = acros.make_nan_campus(section['campus'])
offering = {
'course_name': course['cname'],
'program_code': course['program_code'],
'course_code': course['course_code'],
'crn': section['crn'],
'course_section': section['section'],
'capacity': section['capacity'],
'registered': section['actual'],
'level': section['level'],
'campus': campus,
'teacher_name': classtime['profs'],
'room_number': classtime['room'],
'day': classtime['day'],
'class_type': classType,
'start_time': start_time ,
'finish_time': finish_time ,
'start_date': start_date ,
'finish_date': finish_date ,
'week_alt': weekalt,
'year': metaData['year'],
'semester': int(acros.semester[metaData['term']])
}
if (debug_offering != 0):
logging.debug(offering)
with open('logs/last_offer_dump.txt','w') as f:
util.log_anything_prettily(lambda t: f.write(t + "\n"),offering)
dbinterface.insert_offering(self.get_connection(), offering)