def shouldReload(): loadTimeFile = open(loadTimeFilename, "r") lockFile(loadTimeFile) for line in loadTimeFile: dateForLine = datetime.strptime(line.strip(), format) return dateForLine + durationBeforeExpiration < datetime.now() return True
def writeProgress(report, downloaded, total): fullDataExists = "0" if os.path.exists(dataFilename): fullDataExists = "1" progressFile = open(progressFilename, "w") lockFile(progressFile) progressFile.write('["'+report+'", "'+str(downloaded)+'", "'+str(total)+'", "'+fullDataExists+'"]') progressFile.close()
def prepareFileSystemForTermcode(termcode): # create necessary path if not os.path.exists("courses/"+termCode): os.mkdir("courses/"+termCode, 0700) loadTimeFilename = "courses/"+termCode+"/loadtime" loadTimeFile = open(loadTimeFilename, "w") lockFile(loadTimeFile) loadTimeFile.write(datetime.now().strftime(format)+"\n") loadTimeFile.close()
def shouldReload(): loadTimeFile = open(loadTimeFilename, "r") lockFile(loadTimeFile) for line in loadTimeFile: dateForLine = datetime.strptime(line.strip(), format) return dateForLine + durationBeforeExpiration < datetime.now() return True if not os.path.exists(loadTimeFilename) or shouldReload(): # they should exist, just in case progressFile = open(progressFilename, "w") progressFile.close() loadTimeFile = open(loadTimeFilename, "w") lockFile(loadTimeFile) loadTimeFile.write(datetime.now().strftime(format)+"\n") loadTimeFile.close() if os.path.exists("courses/"+termCode+"/data"): # return this data. still do the update though print """["Beginning load", "0", "0", "1"]""" else: print """["Beginning load", "0", "0", "0"]""" subprocess.Popen(["python", "courseloader.py", termCode]) else: progressFile = open(progressFilename, "r") lockFile(progressFile) print progressFile.read() progressFile.close()
#!/usr/bin/python # perform some authentication import cgi, os from portalocker import lockFile form = cgi.FieldStorage() assert(form.has_key("termcode")) termCode = form["termcode"].value dataFile = None if os.path.exists("courses/"+termCode+"/data"): dataFile = open("courses/"+termCode+"/data", "r") lockFile(dataFile) elif os.path.exists("courses/"+termCode+"/partialdata"): dataFile = open("courses/"+termCode+"/partialdata", "r") lockFile(dataFile) print "Content-type: text/json\n\n"; if dataFile != None: print "["+dataFile.read()+"]" dataFile.close() else: print "[]"
for subject in subjectsDictionary: downloaded += 1 # subject is dictionary with keys "code" and "description"; values are strings subjectCode = convertUnicodeToAscii(subject["code"]) subjectDescription = convertUnicodeToAscii(subject["description"]) # percent-escape amperands totalURL = url + urllib.quote_plus(subjectCode) coursesData = loadDataFromURL(totalURL, subjectDescription) if coursesData == None: continue #print type(coursesData) #print "\n\n", subjectCode, type(subjectCode), subject["description"], coursesData courseString = "{"+"\"code\":\""+subjectCode+"\",\"description\":\""+subjectDescription+"\",\"courses\":"+coursesData+"}" writeFile = open(partialDataFilename, "a") lockFile(writeFile) if not firstSubject: writeFile.write(",") firstSubject = False writeFile.write(courseString) writeFile.close() writeProgress("Loaded "+subjectDescription+" ("+subjectCode+")", downloaded, total) if total > 0: os.rename(partialDataFilename, dataFilename)