def shouldReload():
loadTimeFile = open(loadTimeFilename, "r")
lockFile(loadTimeFile)
for line in loadTimeFile:
dateForLine = datetime.strptime(line.strip(), format)
return dateForLine + durationBeforeExpiration < datetime.now()
return True
def writeProgress(report, downloaded, total):
fullDataExists = "0"
if os.path.exists(dataFilename):
fullDataExists = "1"
progressFile = open(progressFilename, "w")
lockFile(progressFile)
progressFile.write('["'+report+'", "'+str(downloaded)+'", "'+str(total)+'", "'+fullDataExists+'"]')
progressFile.close()
def prepareFileSystemForTermcode(termcode):
# create necessary path
if not os.path.exists("courses/"+termCode):
os.mkdir("courses/"+termCode, 0700)
loadTimeFilename = "courses/"+termCode+"/loadtime"
loadTimeFile = open(loadTimeFilename, "w")
lockFile(loadTimeFile)
loadTimeFile.write(datetime.now().strftime(format)+"\n")
loadTimeFile.close()
def shouldReload():
loadTimeFile = open(loadTimeFilename, "r")
lockFile(loadTimeFile)
for line in loadTimeFile:
dateForLine = datetime.strptime(line.strip(), format)
return dateForLine + durationBeforeExpiration < datetime.now()
return True
if not os.path.exists(loadTimeFilename) or shouldReload():
# they should exist, just in case
progressFile = open(progressFilename, "w")
progressFile.close()
loadTimeFile = open(loadTimeFilename, "w")
lockFile(loadTimeFile)
loadTimeFile.write(datetime.now().strftime(format)+"\n")
loadTimeFile.close()
if os.path.exists("courses/"+termCode+"/data"):
# return this data. still do the update though
print """["Beginning load", "0", "0", "1"]"""
else:
print """["Beginning load", "0", "0", "0"]"""
subprocess.Popen(["python", "courseloader.py", termCode])
else:
progressFile = open(progressFilename, "r")
lockFile(progressFile)
print progressFile.read()
progressFile.close()
#!/usr/bin/python
# perform some authentication
import cgi, os
from portalocker import lockFile
form = cgi.FieldStorage()
assert(form.has_key("termcode"))
termCode = form["termcode"].value
dataFile = None
if os.path.exists("courses/"+termCode+"/data"):
dataFile = open("courses/"+termCode+"/data", "r")
lockFile(dataFile)
elif os.path.exists("courses/"+termCode+"/partialdata"):
dataFile = open("courses/"+termCode+"/partialdata", "r")
lockFile(dataFile)
print "Content-type: text/json\n\n";
if dataFile != None:
print "["+dataFile.read()+"]"
dataFile.close()
else:
print "[]"
for subject in subjectsDictionary:
downloaded += 1
# subject is dictionary with keys "code" and "description"; values are strings
subjectCode = convertUnicodeToAscii(subject["code"])
subjectDescription = convertUnicodeToAscii(subject["description"])
# percent-escape amperands
totalURL = url + urllib.quote_plus(subjectCode)
coursesData = loadDataFromURL(totalURL, subjectDescription)
if coursesData == None:
continue
#print type(coursesData)
#print "\n\n", subjectCode, type(subjectCode), subject["description"], coursesData
courseString = "{"+"\"code\":\""+subjectCode+"\",\"description\":\""+subjectDescription+"\",\"courses\":"+coursesData+"}"
writeFile = open(partialDataFilename, "a")
lockFile(writeFile)
if not firstSubject:
writeFile.write(",")
firstSubject = False
writeFile.write(courseString)
writeFile.close()
writeProgress("Loaded "+subjectDescription+" ("+subjectCode+")", downloaded, total)
if total > 0:
os.rename(partialDataFilename, dataFilename)
