def __init__(self): print 'started' metapath = xbmc.translatePath('special://profile/addon_data/' + sys.argv[7]) iconpath = xbmc.translatePath('special://home/addons/' + sys.argv[7] + '/icon.png') cookiepath = sys.argv[4] cookies = cookielib.MozillaCookieJar() crumbs = cookies auth.login(sys.argv[1], sys.argv[2], cookies, sys.argv[5], sys.argv[6]) #def login(username, password, cookies, callstackpath, maxcalls): requestUrl = "http://www.netflix.com/QueueDelete?movieid={0}&authURL={2}" #Get new authURL response = utils.makeGetRequest("http://www.netflix.com", cookies, sys.argv[5], sys.argv[6]) # find matches - should return <input type="hidden" name="authURL" value="[the_value]" /> instances match = re.compile('a.*?authURL=(.*?)["&]', re.DOTALL).findall(str(response)) # use the last instance found (there should only be one anyway...) authURL = "" for found in match: if found != "": authURL = found requestUrl = requestUrl.format(sys.argv[9], sys.argv[10], authURL, safe='') response = utils.makeGetRequest(requestUrl, cookies, sys.argv[5], str(sys.argv[6])) #def scrapeMyList(cookies, callstackpath, maxrequestsperminute, metapath): scraper.scrapeMyList(cookies, sys.argv[5], sys.argv[6], sys.argv[8]) xbmc.executebuiltin('Notification("Netflix", "Title Removed From MyList", 5000, ' + iconpath + ')') def ensureFolders(self): metapath = xbmc.translatePath('special://profile/addon_data/' + sys.argv[7] + '/meta') if not os.path.isdir(metapath): os.mkdir(metapath) if not os.path.isdir(os.path.join(metapath, "Genres")): os.mkdir(os.path.join(metapath, "Genres")) if not os.path.isdir(os.path.join(metapath, "Titles")): os.mkdir(os.path.join(metapath, "Titles"))
def scrapeSubGenre(cookies, callstackpath, maxrequestsperminute, metapath, url): response = utils.makeGetRequest(url, cookies, callstackpath, maxrequestsperminute) apimatch = re.compile('\"BUILD_IDENTIFIER\":\"(.*?)\".*?\"SHAKTI_API_ROOT\":\"(.*?)\"', re.DOTALL).findall(response) apiurl = "" for build, root in apimatch: apiurl = root + "/" + build if apiurl != "": fh = open(os.path.join(metapath, "apiurl"), 'w') fh.write(apiurl) fh.close() if '<div id="subGenres"' in response: response = response[response.index('<div id="subGenres"'):] matches = re.compile("<a.*?WiGenre\\?agid=(.*?)\\&.*?\">.*?<span>(.*?)</span>.*?</a>", re.DOTALL).findall(response) subGenres = "" data = collections.OrderedDict() for genreid, genrename in matches: #if subGenres != "": # subGenres += "," #subGenres += "'" + genrename + "':'" + genreid + "'" data[utils.cleanstring(genrename)] = genreid #if subGenres != "": if len(data) > 0: #subGenres = "Genres = {" + subGenres + "}" subGenres = json.dumps(data) fh = open(os.path.join(metapath, "genres", genreid + ".json"), 'w') fh.write(subGenres) fh.close()
def checkLogin(cookies, callstackpath, maxcalls): try: response = utils.makeGetRequest("https://www.netflix.com/login", cookies, callstackpath, maxcalls) if 'id="page-LOGIN"' in response: return False else: return True except: return False
def scrapeAPIURL(cookies, callstackpath, maxrequestsperminute, metapath): response = utils.makeGetRequest("http://www.netflix.com/WiGenre?agid=83", cookies, callstackpath, maxrequestsperminute) apimatch = re.compile('\"BUILD_IDENTIFIER\":\"(.*?)\".*?\"SHAKTI_API_ROOT\":\"(.*?)\"', re.DOTALL).findall(response) apiurl = "" for build, root in apimatch: apiurl = root + "/" + build if apiurl != "": fh = open(os.path.join(metapath, "apiurl"), 'w') fh.write(apiurl) fh.close()
def scrapeGenres(cookies, callstackpath, maxrequestsperminute, metapath, cacheage): if not os.path.isdir(os.path.join(metapath, "active")): os.mkdir(os.path.join(metapath, "active")) if not os.path.exists(os.path.join(metapath, "active", "scrape_genres")): fh = open(os.path.join(metapath, "active", "scrape_genres"), 'w') fh.write("currently scraping Genres") fh.close() response = utils.makeGetRequest('http://www.netflix.com', cookies, callstackpath, maxrequestsperminute) matches = re.compile("<li><a href=\"(.*?)WiGenre\\?agid=(.*?)\">(.*?)</a></li>", re.DOTALL).findall(response) genrefile = os.path.join(metapath, "genres", "genres.json") genres = "" data = collections.OrderedDict() for url, genreid, genrename in matches: print "Netflix: DEBUG: " + url url = "http://www.netflix.com/WiGenre?agid=" + genreid data[utils.cleanstring(genrename)] = genreid UpdateSubGenres = False if os.path.exists(os.path.join(metapath, "Genres", genreid + ".json")): oneday = 24 * 60 * 60 if utils.fileIsOlderThan(os.path.join(metapath, "Genres", genreid + ".json"), (oneday * int(cacheage))): UpdateSubGenres = True else: UpdateSubGenres = True if(UpdateSubGenres): scrapeSubGenre(cookies, callstackpath, maxrequestsperminute, metapath, url) #if genres != "": if len(data) > 0: #genres = "{" + genres + "}" genres = json.dumps(data) fh = open(genrefile, 'w') fh.write(genres) fh.close() os.remove(os.path.join(metapath, "active", "scrape_genres"))
def getAuth(cookies, callstackpath, maxcalls): # use utils to make the GET request response = utils.makeGetRequest("http://www.netflix.com/login", cookies, callstackpath, maxcalls) # find matches - should return <input type="hidden" name="authURL" value="[the_value]" /> instances match = re.compile('input.*?authURL.*?value="(.*?)"', re.DOTALL).findall(str(response)) # use the last instance found (there should only be one anyway...) auth = "" for found in match: if found != "": auth = found # return the found token return auth
def search(addon, addonID, pluginhandle, viewpath, callstackpath, maxrequestsperminute, cookies, search_string, metaroot, cookiepath): search_query = search_string.replace(' ', '+') response = utils.makeGetRequest('http://www.netflix.com/search/' + search_query, cookies, callstackpath, maxrequestsperminute) expr = "<div class=\"lockup\" data-titleid=\"(.*?)\" data-trackid=\"(.*?)\">" matches = re.compile(expr, re.DOTALL).findall(response) itemcount = 0 for title, track in matches: listTitle(title, viewpath, pluginhandle, metaroot, addon, callstackpath, maxrequestsperminute, cookiepath) itemcount += 1 if itemcount >= 1: xbmcplugin.endOfDirectory(pluginhandle) else: dialog = xbmcgui.Dialog() ok = dialog.ok('Netflix', utils.translation(addon, 30204))
def scrapeGenreTitles(cookies, callstackpath, maxrequestsperminute, metapath, genreid, cacheage, genrename): if(os.path.exists(os.path.join(metapath, "apiurl"))): fh = open(os.path.join(metapath, "apiurl"), 'r') apiurl = fh.read() fh.close() print "Netflix: Scraping titles for " + genrename content = "" start = 0 size = 100 titles = "" if not os.path.isdir(os.path.join(metapath, "GenreTitles")): os.mkdir(os.path.join(metapath, "GenreTitles")) if not os.path.isdir(os.path.join(metapath, "Titles")): os.mkdir(os.path.join(metapath, "Titles")) titles = [] while not content.startswith('{"catalogItems":[]}'): requesturl = apiurl + "/wigenre?genreId=" + genreid + "&full=false&from=" + str(start) + "&to=" + str(start + size) # increment for next call start = start + size + 1 content = utils.makeGetRequest(requesturl, cookies, callstackpath, maxrequestsperminute) match = re.compile("{\"boxart\":\"(.*?)\",\"titleId\":(.*?),\"title\":\"(.*?)\",\"playerUrl\":\"(.*?)\",\"trackId\":(.*?)}", re.DOTALL).findall(content) for boxart, titleid, title, playerurl, trackid in match: titledata = collections.OrderedDict() titledata["boxart"] = boxart titledata["titleId"] = titleid titledata["title"] = title titledata["playerurl"] = playerurl titledata["trackid"] = trackid if not os.path.isdir(os.path.join(metapath, "GenreTitles", genreid)): os.mkdir(os.path.join(metapath, "GenreTitles", genreid)) if not os.path.isdir(os.path.join(metapath, "Titles", titleid)): os.mkdir(os.path.join(metapath, "Titles", titleid)) coverart = utils.makeGetRequest(boxart, cookies, callstackpath, maxrequestsperminute) fh = open(os.path.join(metapath, "Titles", titleid, "folder.jpg"), 'wb') fh.write(coverart) fh.close() fh = open(os.path.join(metapath, "Titles", titleid, "coverart.jpg"), 'wb') fh.write(coverart) fh.close() # write genre tags if not os.path.isdir(os.path.join(metapath, "Titles", titleid, "Genres")): os.mkdir(os.path.join(metapath, "Titles", titleid, "Genres")) if genrename != "": fh = open(os.path.join(metapath, "Titles", titleid, "Genres", genrename), 'w') fh.write(genrename) fh.close() fh = open(os.path.join(metapath, "GenreTitles", genreid, titleid + ".json"), 'w') fh.write(json.dumps(titledata)) fh.close() UpdateTitle = False titlefile = os.path.join(metapath, "Titles", titleid, "meta.json") if os.path.exists(titlefile): oneday = 24 * 60 * 60 if utils.fileIsOlderThan(titlefile, (oneday * int(cacheage))): UpdateTitle = True else: UpdateTitle = True if UpdateTitle: scrapeTitle(cookies, callstackpath, maxrequestsperminute, metapath, titleid, trackid) titles = titles + [titledata] fh = open(os.path.join(metapath, "genreTitles", genreid + ".json"), 'w') fh.write(json.dumps(titles)) fh.close()
def scrapeMyList(cookies, callstackpath, maxrequestsperminute, metapath): if not os.path.isdir(os.path.join(metapath, "active")): os.mkdir(os.path.join(metapath, "active")) if not os.path.exists(os.path.join(metapath, "active", "scrape_mylist")): fh = open(os.path.join(metapath, "active", "scrape_mylist"), 'w') fh.write("currently scraping MyList") fh.close() #def makeGetRequest(url, cookies, callstackpath, maxcalls): content = utils.makeGetRequest("https://www.netflix.com/MyList", cookies, callstackpath, maxrequestsperminute) #expr = "<div.*?class=\"agMovie agMovie-lulg\".*?<img.*?src=\"(.*?)\".*?>.*?<a.*?WiPlayer\\?movieid=(.*?)&trkid=(.*?)&"; expr = '<div class="agMovie agMovie-lulg">.*?<img.*?src="(.*?)" ><a .*? href=".*?WiPlayer\\?movieid=(.*?)&trkid=(.*?)&' #content = content.decode('utf-8') if '<div id="yui-main">' in content: content = content[content.index('<div id="yui-main">'):] for ffile in os.listdir(os.path.join(metapath,"MyList")): os.remove(os.path.join(metapath, "MyList", ffile)) matches = re.compile(expr, re.DOTALL).findall(content) counter = 0 for boxart, titleid, trackid in matches: counter += 1 fh = open(os.path.join(metapath, "MyList", titleid), 'w') fh.write(str(counter)) fh.close() titlefile = os.path.join(metapath, 'Titles', titleid, 'meta.json') coverart = utils.makeGetRequest(boxart, cookies, callstackpath, maxrequestsperminute) if not os.path.isdir(os.path.join(metapath, "Titles", titleid)): os.mkdir(os.path.join(metapath, "Titles", titleid)) fh = open(os.path.join(metapath, "Titles", titleid, "folder.jpg"), 'wb') fh.write(coverart) fh.close() fh = open(os.path.join(metapath, "Titles", titleid, "coverart.jpg"), 'wb') fh.write(coverart) fh.close() if os.path.exists(os.path.join(metapath, "Titles", titleid, "coverart.jpg")): iconpath = os.path.join(metapath, "Titles", titleid, "coverart.jpg") else: iconpath = "" UpdateTitle = False if os.path.exists(titlefile): age = xbmcvfs.Stat(titlefile).st_mtime() now = time.time() oneday = 24 * 60 * 60 if (now-age) > (oneday*int(sys.argv[3])): UpdateTitle = True else: UpdateTitle = True if UpdateTitle: scrapeTitle(cookies, callstackpath, maxrequestsperminute, metapath, titleid, trackid) os.remove(os.path.join(metapath, "active", "scrape_mylist"))
def scrapeTitle(cookies, callstackpath, maxrequestsperminute, metapath, titleid, trackid): if not os.path.isdir(os.path.join(metapath, "active")): os.mkdir(os.path.join(metapath, "active")) if not os.path.exists(os.path.join(metapath, "active", "scrape_title_" + titleid)): fh = open(os.path.join(metapath, "active", "scrape_title_" + titleid), 'w') fh.write("currently scraping MyList") fh.close() if(os.path.exists(os.path.join(metapath, "apiurl"))): fh = open(os.path.join(metapath, "apiurl"), 'r') apiurl = fh.read() fh.close() titleurl = apiurl + "/bob?titleid=" + titleid + "&trackid=" + trackid content = utils.makeGetRequest(titleurl, cookies, callstackpath, maxrequestsperminute) data = json.loads(content) if not os.path.isdir(os.path.join(metapath, "Titles")): os.mkdir(os.path.join(metapath, "Titles")) if not os.path.isdir(os.path.join(metapath, "Titles", titleid)): os.mkdir(os.path.join(metapath, "Titles", titleid)) fh = open(os.path.join(metapath, "Titles", titleid, "meta.json"), 'w') fh.write(content) fh.close() if os.path.exists(os.path.join(metapath, "Titles", titleid, "coverart.jpg")): iconpath = os.path.join(metapath, "Titles", titleid, "coverart.jpg") else: iconpath = "" thetitle = data["title"] mdplink = data["mdpLink"] trackid = data["trackId"] if data["isShow"]: seasondataurl = "http://api-global.netflix.com/desktop/odp/episodes?forceEpisodes=true&routing=redirect&video=" + titleid seasondata = utils.makeGetRequest(seasondataurl, cookies, callstackpath, maxrequestsperminute) seasondata = seasondata[seasondata.index("{\"title\":"):] fh = open(os.path.join(metapath, "Titles", titleid, "seasondata.json"),'w') fh.write(seasondata) fh.close() data = json.loads(seasondata) seasoncounter = 0 for season in data["episodes"]: #http://www.netflix.com/WiMovie/70136119?trkid=50263680&actionMethod=seasonDetails&seasonId=70061401&seasonKind=ELECTRONIC for episode in season: if not os.path.exists(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]))): os.mkdir(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]))) fname = "S" + str(episode["season"]).zfill(2) + "E" + str(episode["episode"]).zfill(2) fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]), fname + ".json"), 'w') fh.write(json.dumps(episode)) fh.close() stillswidth = 0 stillspath = "" if "stills" in episode: for still in episode["stills"]: width = still["width"] try: if(int(width) > stillswidth): stillswidth = int(width) stillspath = still["url"] except: pass if stillspath != "": try: stillimage = utils.makeGetRequest(stillspath, cookies, callstackpath, maxrequestsperminute) fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]), fname + ".jpg"), 'wb') fh.write(stillimage) fh.close() except: pass #http://www.netflix.com/WiMovie/70297439?actionMethod=seasonDetails&seasonId=70296034&seasonKind=ELECTRONIC seasoninfourl = "http://www.netflix.com" + mdplink + "?&actionMethod=seasonDetails&seasonId=" + str(season[0]["seasonId"]) + "&seasonKind=ELECTRONIC" seasoninforesponse = utils.makeGetRequest(seasoninfourl, cookies, callstackpath, maxrequestsperminute) seasoninfo = json.loads(seasoninforesponse) seasoninfosynopsismatch = re.compile('<\/h2><p class=\"synopsis\".*?>(.*?)<\/p>', re.DOTALL).findall(seasoninfo["html"]) synopsis = "" for syno in seasoninfosynopsismatch: synopsis += syno fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(season[0]["season"]), "synopsis"), 'w') fh.write(json.dumps(synopsis)) fh.close() print "Netflix: " + thetitle.encode('utf-8') + " has been updated" xbmc.executebuiltin('Notification("Netflix", "' + thetitle.encode('utf-8') + ' has been updated", 5000, ' + iconpath + ')') os.remove(os.path.join(metapath, "active", "scrape_title_" + titleid))
def scrapeSeasonData(cookies, callstackpath, maxrequestsperminute, metapath, titleid): seasondataurl = "http://api-global.netflix.com/desktop/odp/episodes?forceEpisodes=true&routing=redirect&video=" + titleid seasondata = utils.makeGetRequest(seasondataurl, cookies, callstackpath, maxrequestsperminute) seasondata = seasondata[seasondata.index("{\"title\":"):] fh = open(os.path.join(metapath, "Titles", titleid, "seasondata.json"),'w') fh.write(seasondata) fh.close() data = json.loads(seasondata) seasoncounter = 0 for season in data["episodes"]: #http://www.netflix.com/WiMovie/70136119?trkid=50263680&actionMethod=seasonDetails&seasonId=70061401&seasonKind=ELECTRONIC for episode in season: if not os.path.exists(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]))): os.mkdir(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]))) fname = "S" + str(episode["season"]).zfill(2) + "E" + str(episode["episode"]).zfill(2) fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]), fname + ".json"), 'w') fh.write(json.dumps(episode)) fh.close() stillswidth = 0 stillspath = "" if not os.path.exists(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]), fname + ".jpg")): if "stills" in episode: for still in episode["stills"]: width = still["width"] try: if(int(width) > stillswidth): stillswidth = int(width) stillspath = still["url"] except: pass if stillspath != "": try: stillimage = utils.makeGetRequest(stillspath, cookies, callstackpath, maxrequestsperminute) fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]), fname + ".jpg"), 'wb') fh.write(stillimage) fh.close() except: pass #http://www.netflix.com/WiMovie/70297439?actionMethod=seasonDetails&seasonId=70296034&seasonKind=ELECTRONIC if not os.path.exists(os.path.join(metapath, "Titles", titleid, "Season " + str(season[0]["season"]), "synopsis")): seasoninfourl = "http://www.netflix.com/WiMovie/" + titleid + "?&actionMethod=seasonDetails&seasonId=" + str(season[0]["seasonId"]) + "&seasonKind=ELECTRONIC" seasoninforesponse = utils.makeGetRequest(seasoninfourl, cookies, callstackpath, maxrequestsperminute) seasoninfo = json.loads(seasoninforesponse) seasoninfosynopsismatch = re.compile('<\/h2><p class=\"synopsis\".*?>(.*?)<\/p>', re.DOTALL).findall(seasoninfo["html"]) synopsis = "" for syno in seasoninfosynopsismatch: synopsis += syno fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(season[0]["season"]), "synopsis"), 'w') fh.write(json.dumps(synopsis)) fh.close()