def correctOscarData():
print "Checking for unparsed oscar data."
backupfilename = setFile(getWikipediaDir(), "oscars.yaml.backup")
filename = setFile(getWikipediaDir(), "oscars.yaml")
copyFile(filename, backupfilename)
data = get(filename)
#fixes = {}
for year,ydata in data.iteritems():
print "\n==>",year
for cat,catdata in ydata.iteritems():
winner = catdata["Winner"]
if isinstance(winner, list):
if winner[0].find(",") != -1:
print "\t",cat,"\t",winner[0]
nominees = catdata["Nominees"]
for nominee in nominees:
if isinstance(nominee, list):
if nominee[0].find(",") != -1:
print "\t",cat,"\t",nominee[0]
savename = setFile(getOscarDir(), "oscars.yaml")
print "Saving",len(data),"yearly results to",savename
save(savename, data)
def getOscarData():
filename = setFile(getOscarDir(), "oscars.yaml")
data = get(filename)
yearlyData = {}
for year,ydata in data.iteritems():
movies = {}
for category,categorydata in ydata.iteritems():
if category.find("Song") != -1:
continue
sf = 1
if category.find("Song") != -1:
sf = 0
elif category.find("Picture") != -1:
sf = 40
elif category.find("Animated Feature") != -1:
sf = 35
elif category.find("Director") != -1:
sf = 30
elif category.find("Actor") != -1 or category.find("Actress") != -1:
sf = 25
elif category.find("Screenplay") != -1:
sf = 20
winner = categorydata.get("Winner")
if winner:
#print category,'\t',winner
if isinstance(winner, list):
movie = winner[0]
else:
movie = winner
#print category,'\t',10*sf,'\t',winner
if movies.get(movie) == None:
movies[movie] = 10*sf
else:
movies[movie] = max(10*sf, movies[movie])
nominees = categorydata.get("Nominees")
if nominees:
for nominee in nominees:
if isinstance(nominee, list):
movie = nominee[0]
else:
movie = nominee
#print category,'\t',sf,'\t',winner
if movies.get(movie) == None:
movies[movie] = sf
else:
movies[movie] = max(sf, movies[movie])
yearlyData[year] = sorted(movies.items(), key=operator.itemgetter(1), reverse=True)
print "---->",year,"<----"
for item in yearlyData[year][:15]:
print item
print '\n'
savename = setFile(getOscarDir(), "oscars.json")
print "Saving",len(yearlyData),"yearly results to",savename
save(savename, yearlyData)
def processBoxOfficeMojo():
outdir = getBoxOfficeDir()
savename = setFile(outdir, "results.json")
data = get(savename)
movies = {}
yearlyData = {}
for i,year in enumerate(data.keys()):
movies[year] = {}
ydata = data[year]
for wdata in ydata:
for mdata in wdata:
movie = mdata[2]
retval = search("\((\d+)\)",movie)
if retval:
stryear = retval.group()
movie = movie.replace(stryear, "").strip()
gross = convertCurrency(mdata[9])
weekly = convertCurrency(mdata[4])
money = max(gross, weekly)
if movies[year].get(movie) == None:
movies[year][movie] = money
else:
movies[year][movie] = max(money, movies[year][movie])
yearlyData[year] = sorted(movies[year].items(), key=operator.itemgetter(1), reverse=True)
print "---->",year,"<----"
for item in yearlyData[year][:25]:
print item
print '\n'
savename = setFile(outdir, "boxofficemojo.json")
print "Saving",len(yearlyData),"yearly results to",savename
save(savename, yearlyData)
def processWikipediaOscarFiles(procYear = None):
outdir = getWikipediaDir()
if procYear == None:
files = findSubExt(outdir, "data", ext=".p")
#files = glob(join(outdir, "data", "*.p"))
else:
files = findSubPatternExt(outdir, "data", pattern=str(procYear), ext=".p")
#files = glob(join(outdir, "data", str(procYear)+".p"))
movies = OrderedDict()
for ifile in files:
print ifile
year = getBaseFilename(ifile)
print year
#if year == "1985": continue
htmldata = get(ifile)
bsdata = getHTML(htmldata)
if int(year) <= 1984:
results = parseWikipediaOscarDataPre1985(bsdata, True)
elif int(year) >= 1986:
results = parseWikipediaOscarDataPost1987(bsdata, True)
else:
results = parseWikipediaOscarData1985(debug = True)
movies[year] = results
for k,v in results.iteritems():
print "====>",year,'\t',k
print " Winner :",results[k]["Winner"]
print " Nominees:",results[k]["Nominees"]
print ""
savename = setFile(outdir, "oscars.yaml")
print "Saving",len(movies),"years of wikipedia oscar data to",savename
save(savename, movies)
def getSundanceData():
outdir = getSundanceDir()
outdir = setDir(outdir(), "data")
if not isDir(outdir): mkDir(outdir)
url = "https://en.wikipedia.org/wiki/List_of_Sundance_Film_Festival_award_winners"
savename = setFile(outdir, "winners.p")
if isFile(savename): return
getData(base=url, savename=savename, useSafari=False)
sleep(2)
def mergeYearlyMovies(outdir, yearlyMovies):
data = OrderedDict()
for year,ymovies in yearlyMovies.iteritems():
data[year] = {}
for movie,movieType in ymovies.iteritems():
if data[year].get(movieType) == None:
data[year][movieType] = {}
data[year][movieType][movie] = None
savename = setFile(outdir, "officialMovies.yaml")
print "Saving",len(data),"to",savename
save(savename, data)
def getBoxOfficeMojoWeekendResult(year, week, outdir):
yname = str(year)
if week < 10:
wname = "0"+str(week)
else:
wname = str(week)
url="http://www.boxofficemojo.com/weekend/chart/?yr="+yname+"&wknd="+wname+"&p=.htm"
savename = setFile(outdir, yname+"-"+wname+".p")
if isFile(savename): return
getData(base=url, savename=savename, useSafari=False)
sleep(2)
def mergeBoxOfficeMojoResults():
outdir = getBoxOfficeDir()
retval = {}
files = findSubExt(outdir, "results", ext=".json")
for ifile in files:
year = getBaseFilename(ifile)
data = get(ifile)
retval[year] = data
savename = setFile(outdir, "results.json")
print "Saving",len(retval),"years of movie data to",savename
save(savename, retval)
def getWikipediaOscarFile(year, outdir):
base = "https://en.wikipedia.org/wiki/"
dmap = {}
for x in range(1934, 2017+1):
val = str(x - 1927)+"th_Academy_Awards"
val = val.replace("1th_", "1st_")
val = val.replace("2th_", "2nd_")
val = val.replace("3th_", "3rd_")
val = val.replace("11st_", "11th_")
val = val.replace("12nd_", "12th_")
val = val.replace("13rd_", "13th_")
dmap[x] = val
try:
url = base+dmap[year]
except:
print "Could not create url for",year
return
print year,'\t',url
savename = setFile(outdir, str(year)+".p")
if isFile(savename): return
getData(base=url, savename=savename, useSafari=False)
sleep(2)
def processSundanceData():
files = findSubExt(getSundanceDir(), "data", ext=".p")
data = OrderedDict()
for ifile in files:
htmldata = get(ifile)
bsdata = getHTML(htmldata)
years = []
for h2 in bsdata.findAll("h2"):
span = h2.find("span")
try:
year = int(span.string)
except:
continue
years.append(year)
for j,ul in enumerate(bsdata.findAll("ul")):
try:
year = years[j]
except:
break
data[year] = {}
lis = ul.findAll("li")
for li in lis:
try:
txt = li.text
txt = re.sub("\xe2\x80\x93", " :: ", txt)
txt = re.sub(u"(\u2018|\u2013)", " :: ", txt)
except:
print "Error with",li
continue
vals = txt.split(" :: ")
if len(vals) > 2:
vals[1] = "-".join(vals[1:])
vals = vals[:2]
vals = [x.strip() for x in vals]
if len(vals) != 2:
raise ValueError(vals)
cat = vals[0]
movie = vals[1]
if cat.find("Piper-Heidsieck") != -1:
continue
if cat.find("Alfred P. Sloan") != -1:
cat = "Alfred P. Sloan Prize"
if cat in ["World Cinema Dramatic Screenwriting Award",
"Sundance Institute/Mahindra Global Filmmaking Awards",
"World Cinema Documentary Editing Award",
"Excellence in Cinematography Award: Documentary",
"Excellence in Cinematography Award: Dramatic",
"World Cinema Cinematography Award: Documentary",
"World Cinema Cinematography Award: Dramatic",
"World Cinema Directing Award: Dramatic",
"World Cinema Directing Award: Documentary",
"World Dramatic Special Jury Prizes for Breakout Performances",
"Dramatic Special Jury Prize for Breakout Performance",
"Excellence in Cinematography Award Dramatic",
"xcellence in Cinematography Award Documentary",
"Documentary Editing Award",
"Waldo Salt Screenwriting Award: Dramatic",
"World Cinema Screenwriting Award",
"Directing Award Documentary",
"Directing Award Dramatic"]:
vals = movie.split(" for ")
if len(vals) == 2:
movie = vals[1]
elif len(vals) == 1:
movie = vals[0]
else:
print "Error in",cat,"===>",movie
continue
if cat in ["Special Jury Prize for Acting"]:
movie = movie.replace("for her performance ", "")
vals = movie.split(" in ")
if len(vals) == 2:
movie = vals[1]
vals = movie.split(" for ")
if len(vals) == 2:
movie = vals[1]
if movie.find("retitled") != -1:
movie = movie.split("retitled ")[1]
movie = movie[:-1]
movie = movie.replace(" (tie)", "")
if movie.find(" director of ") != -1:
movie = movie.split(" director of ")[1]
print years[j],'\t',cat,'\t\t',movie,'\t\t'
try:
data[year][str(cat)] = str(movie)
except:
data[year][str(cat)] = movie
savename = setFile(getSundanceDir(), "winners.yaml")
print "Saving",len(data),"yearly results to",savename
save(savename, data)