def main(semester,year): # Hardcoded values url_base = "http://ssbp.mycampus.ca" url_action = "/prod/bwckschd.p_get_crse_unsec" # Set metaData (used by database storer) metaData = { 'term': semester, 'year': year } # Everything is in catch-all try block try: # Instantiate objects courseData = [] pageParser = Parser(courseData) # === PHASE 1: PARSING === # Set list of faculties to parse facsToUse = acros.faculties #facsToUse = ["ENGR"] # Parse for each faculty facSoFar = 0 for faculty in facsToUse: facSoFar += 1 # Print out which faculty is being parsed outta = str(facSoFar)+"/"+str(len(facsToUse)) logging.info("Parsing faculty: "+outta+" "+faculty+": "+acros.faculties[faculty]) # Instanciate page loader and get page #pageLoader = PageLoader(url_base,url_action,semester,year,faculty) #page = pageLoader.get_page(pageLoader.gen_url_and_data()); with open('last_source.html','r') as f: page = ''.join(f.readlines()); bs = BowlShit(page) """ with open('last_source.html','w') as f: f.write(page) """ # Parse page pageParser.parse_course_info(page) with open('last_object_dump.txt','w') as f: util.log_anything_prettily(lambda t: f.write(t + "\n"),courseData) #pickle.dump(courseData,f) # === PHASE 2: STORING === frf = FRFStore() frf.set_course_data(courseData,metaData) frf.setup_default_connection() frf.insert_data_as_offerings() except SomethingWentWrong as e: logging.exception("Something terrible(?) happened!!")
def insert_data_as_offerings(self, debug_offering=0): # Choose variables to be used courseData = self.courseData metaData = self.metaData # Make sure everything is okay if courseData == None: raise SomethingWentWrong("The courseData object was not set -_-") debugSoFar = 0 courseSoFar = 0 # loop through each course for course in courseData: for section in course['classes']: for classtime in section['times']: # Not very important code - just for debugging debugSoFar += 1 if debug_offering != 0: if debugSoFar != debug_offering: continue if debugSoFar > debug_offering: break logging.debug("Offering #"+str(debugSoFar) + ", "+str(section['crn'],)+", "+course['program_code']) # PREPARING DATA FOR DATABASE weekalt = (classtime['week'] == "W1" or classtime['week'] == "W2") weekalt = 1 if weekalt else 0 logging.debug(" ~ adding a " + classtime['type']) try: classType = util.reverse_lookup(acros.class_types, classtime['type']) except ValueError: classType = "OTH" start_time = classtime['start_time'] finish_time = classtime['finish_time'] start_date = classtime['start_date'] finish_date = classtime['finish_date'] # This deals with times/dates that are not available try: start_time = time.strftime("%H:%M:00", start_time ) finish_time = time.strftime("%H:%M:00", finish_time ) except TypeError: start_time = None finish_time = None try: start_date = time.strftime("%Y-%m-%d", start_date ) finish_date = time.strftime("%Y-%m-%d", finish_date ) except TypeError: start_date = None finish_date = None # Determining campus try: campus = util.reverse_lookup(acros.campus_acronyms, section['campus']) except ValueError: campus = acros.make_nan_campus(section['campus']) offering = { 'course_name': course['cname'], 'program_code': course['program_code'], 'course_code': course['course_code'], 'crn': section['crn'], 'course_section': section['section'], 'capacity': section['capacity'], 'registered': section['actual'], 'level': section['level'], 'campus': campus, 'teacher_name': classtime['profs'], 'room_number': classtime['room'], 'day': classtime['day'], 'class_type': classType, 'start_time': start_time , 'finish_time': finish_time , 'start_date': start_date , 'finish_date': finish_date , 'week_alt': weekalt, 'year': metaData['year'], 'semester': int(acros.semester[metaData['term']]) } if (debug_offering != 0): logging.debug(offering) with open('logs/last_offer_dump.txt','w') as f: util.log_anything_prettily(lambda t: f.write(t + "\n"),offering) dbinterface.insert_offering(self.get_connection(), offering)