def scrapeGenres(cookies, callstackpath, maxrequestsperminute, metapath, cacheage):
if not os.path.isdir(os.path.join(metapath, "active")):
os.mkdir(os.path.join(metapath, "active"))
if not os.path.exists(os.path.join(metapath, "active", "scrape_genres")):
fh = open(os.path.join(metapath, "active", "scrape_genres"), 'w')
fh.write("currently scraping Genres")
fh.close()
response = utils.makeGetRequest('http://www.netflix.com', cookies, callstackpath, maxrequestsperminute)
matches = re.compile("<li><a href=\"(.*?)WiGenre\\?agid=(.*?)\">(.*?)</a></li>", re.DOTALL).findall(response)
genrefile = os.path.join(metapath, "genres", "genres.json")
genres = ""
data = collections.OrderedDict()
for url, genreid, genrename in matches:
print "Netflix: DEBUG: " + url
url = "http://www.netflix.com/WiGenre?agid=" + genreid
data[utils.cleanstring(genrename)] = genreid
UpdateSubGenres = False
if os.path.exists(os.path.join(metapath, "Genres", genreid + ".json")):
oneday = 24 * 60 * 60
if utils.fileIsOlderThan(os.path.join(metapath, "Genres", genreid + ".json"), (oneday * int(cacheage))):
UpdateSubGenres = True
else:
UpdateSubGenres = True
if(UpdateSubGenres):
scrapeSubGenre(cookies, callstackpath, maxrequestsperminute, metapath, url)
#if genres != "":
if len(data) > 0:
#genres = "{" + genres + "}"
genres = json.dumps(data)
fh = open(genrefile, 'w')
fh.write(genres)
fh.close()
os.remove(os.path.join(metapath, "active", "scrape_genres"))
os.path.join(metaroot, "genres", genre + ".json"),
sys.argv[0],
callstackpath,
maxrequestsperminute,
cookiepath,
genre,
)
elif mode == "listgenretitles":
# determine the genre's meta data file path
genretitlesmetapath = os.path.join(metaroot, "genreTitles", genre + ".json")
# is the genre file out-of-date or missing?
updateGenreTitles = False
if os.path.exists(genretitlesmetapath):
oneday = 24 * 60 * 60
if utils.fileIsOlderThan(genretitlesmetapath, (oneday * int(addon.getSetting("cacheage")))):
updateGenreTitles = True
else:
updateGenreTitles = True
# if the genre file is out-of-date or missing run the update script
if updateGenreTitles:
# do the settings call for a prompt before updating?
if addon.getSetting("promptforcache") == "true":
dialog = xbmcgui.Dialog()
ret = dialog.yesno("Netflix", utils.translation(addon, 30200))
if ret:
# run the script if the user says so...
# 'xbmc.runscript(special://home/addons/' + addonID + '/resources/scripts/UpdateGenreTitles.py, ' + addon.getSetting("username") + ', ' + addon.getSetting("password") + ', ' + addon.getSetting("cacheage") + ', ' + cookiepath + ', ' + callstackpath + ', ' + str(maxrequestsperminute) + ', ' + addonID + ',' + metaroot + ',' + genres[title] + ',' + title + ')'
xbmc.executebuiltin(
"xbmc.runscript(special://home/addons/"
def scrapeGenreTitles(cookies, callstackpath, maxrequestsperminute, metapath, genreid, cacheage, genrename):
if(os.path.exists(os.path.join(metapath, "apiurl"))):
fh = open(os.path.join(metapath, "apiurl"), 'r')
apiurl = fh.read()
fh.close()
print "Netflix: Scraping titles for " + genrename
content = ""
start = 0
size = 100
titles = ""
if not os.path.isdir(os.path.join(metapath, "GenreTitles")):
os.mkdir(os.path.join(metapath, "GenreTitles"))
if not os.path.isdir(os.path.join(metapath, "Titles")):
os.mkdir(os.path.join(metapath, "Titles"))
titles = []
while not content.startswith('{"catalogItems":[]}'):
requesturl = apiurl + "/wigenre?genreId=" + genreid + "&full=false&from=" + str(start) + "&to=" + str(start + size)
# increment for next call
start = start + size + 1
content = utils.makeGetRequest(requesturl, cookies, callstackpath, maxrequestsperminute)
match = re.compile("{\"boxart\":\"(.*?)\",\"titleId\":(.*?),\"title\":\"(.*?)\",\"playerUrl\":\"(.*?)\",\"trackId\":(.*?)}", re.DOTALL).findall(content)
for boxart, titleid, title, playerurl, trackid in match:
titledata = collections.OrderedDict()
titledata["boxart"] = boxart
titledata["titleId"] = titleid
titledata["title"] = title
titledata["playerurl"] = playerurl
titledata["trackid"] = trackid
if not os.path.isdir(os.path.join(metapath, "GenreTitles", genreid)):
os.mkdir(os.path.join(metapath, "GenreTitles", genreid))
if not os.path.isdir(os.path.join(metapath, "Titles", titleid)):
os.mkdir(os.path.join(metapath, "Titles", titleid))
coverart = utils.makeGetRequest(boxart, cookies, callstackpath, maxrequestsperminute)
fh = open(os.path.join(metapath, "Titles", titleid, "folder.jpg"), 'wb')
fh.write(coverart)
fh.close()
fh = open(os.path.join(metapath, "Titles", titleid, "coverart.jpg"), 'wb')
fh.write(coverart)
fh.close()
# write genre tags
if not os.path.isdir(os.path.join(metapath, "Titles", titleid, "Genres")):
os.mkdir(os.path.join(metapath, "Titles", titleid, "Genres"))
if genrename != "":
fh = open(os.path.join(metapath, "Titles", titleid, "Genres", genrename), 'w')
fh.write(genrename)
fh.close()
fh = open(os.path.join(metapath, "GenreTitles", genreid, titleid + ".json"), 'w')
fh.write(json.dumps(titledata))
fh.close()
UpdateTitle = False
titlefile = os.path.join(metapath, "Titles", titleid, "meta.json")
if os.path.exists(titlefile):
oneday = 24 * 60 * 60
if utils.fileIsOlderThan(titlefile, (oneday * int(cacheage))):
UpdateTitle = True
else:
UpdateTitle = True
if UpdateTitle:
scrapeTitle(cookies, callstackpath, maxrequestsperminute, metapath, titleid, trackid)
titles = titles + [titledata]
fh = open(os.path.join(metapath, "genreTitles", genreid + ".json"), 'w')
fh.write(json.dumps(titles))
fh.close()
