def __init__(self):
print 'started'
metapath = xbmc.translatePath('special://profile/addon_data/' + sys.argv[7])
iconpath = xbmc.translatePath('special://home/addons/' + sys.argv[7] + '/icon.png')
cookiepath = sys.argv[4]
cookies = cookielib.MozillaCookieJar()
crumbs = cookies
auth.login(sys.argv[1], sys.argv[2], cookies, sys.argv[5], sys.argv[6])
#def login(username, password, cookies, callstackpath, maxcalls):
requestUrl = "http://www.netflix.com/QueueDelete?movieid={0}&authURL={2}"
#Get new authURL
response = utils.makeGetRequest("http://www.netflix.com", cookies, sys.argv[5], sys.argv[6])
# find matches - should return <input type="hidden" name="authURL" value="[the_value]" /> instances
match = re.compile('a.*?authURL=(.*?)["&]', re.DOTALL).findall(str(response))
# use the last instance found (there should only be one anyway...)
authURL = ""
for found in match:
if found != "":
authURL = found
requestUrl = requestUrl.format(sys.argv[9], sys.argv[10], authURL, safe='')
response = utils.makeGetRequest(requestUrl, cookies, sys.argv[5], str(sys.argv[6]))
#def scrapeMyList(cookies, callstackpath, maxrequestsperminute, metapath):
scraper.scrapeMyList(cookies, sys.argv[5], sys.argv[6], sys.argv[8])
xbmc.executebuiltin('Notification("Netflix", "Title Removed From MyList", 5000, ' + iconpath + ')')
def ensureFolders(self):
metapath = xbmc.translatePath('special://profile/addon_data/' + sys.argv[7] + '/meta')
if not os.path.isdir(metapath):
os.mkdir(metapath)
if not os.path.isdir(os.path.join(metapath, "Genres")):
os.mkdir(os.path.join(metapath, "Genres"))
if not os.path.isdir(os.path.join(metapath, "Titles")):
os.mkdir(os.path.join(metapath, "Titles"))
def scrapeSubGenre(cookies, callstackpath, maxrequestsperminute, metapath, url):
response = utils.makeGetRequest(url, cookies, callstackpath, maxrequestsperminute)
apimatch = re.compile('\"BUILD_IDENTIFIER\":\"(.*?)\".*?\"SHAKTI_API_ROOT\":\"(.*?)\"', re.DOTALL).findall(response)
apiurl = ""
for build, root in apimatch:
apiurl = root + "/" + build
if apiurl != "":
fh = open(os.path.join(metapath, "apiurl"), 'w')
fh.write(apiurl)
fh.close()
if '<div id="subGenres"' in response:
response = response[response.index('<div id="subGenres"'):]
matches = re.compile("<a.*?WiGenre\\?agid=(.*?)\\&.*?\">.*?<span>(.*?)</span>.*?</a>", re.DOTALL).findall(response)
subGenres = ""
data = collections.OrderedDict()
for genreid, genrename in matches:
#if subGenres != "":
# subGenres += ","
#subGenres += "'" + genrename + "':'" + genreid + "'"
data[utils.cleanstring(genrename)] = genreid
#if subGenres != "":
if len(data) > 0:
#subGenres = "Genres = {" + subGenres + "}"
subGenres = json.dumps(data)
fh = open(os.path.join(metapath, "genres", genreid + ".json"), 'w')
fh.write(subGenres)
fh.close()
def checkLogin(cookies, callstackpath, maxcalls):
try:
response = utils.makeGetRequest("https://www.netflix.com/login", cookies, callstackpath, maxcalls)
if 'id="page-LOGIN"' in response:
return False
else:
return True
except:
return False
def scrapeAPIURL(cookies, callstackpath, maxrequestsperminute, metapath):
response = utils.makeGetRequest("http://www.netflix.com/WiGenre?agid=83", cookies, callstackpath, maxrequestsperminute)
apimatch = re.compile('\"BUILD_IDENTIFIER\":\"(.*?)\".*?\"SHAKTI_API_ROOT\":\"(.*?)\"', re.DOTALL).findall(response)
apiurl = ""
for build, root in apimatch:
apiurl = root + "/" + build
if apiurl != "":
fh = open(os.path.join(metapath, "apiurl"), 'w')
fh.write(apiurl)
fh.close()
def scrapeGenres(cookies, callstackpath, maxrequestsperminute, metapath, cacheage):
if not os.path.isdir(os.path.join(metapath, "active")):
os.mkdir(os.path.join(metapath, "active"))
if not os.path.exists(os.path.join(metapath, "active", "scrape_genres")):
fh = open(os.path.join(metapath, "active", "scrape_genres"), 'w')
fh.write("currently scraping Genres")
fh.close()
response = utils.makeGetRequest('http://www.netflix.com', cookies, callstackpath, maxrequestsperminute)
matches = re.compile("<li><a href=\"(.*?)WiGenre\\?agid=(.*?)\">(.*?)</a></li>", re.DOTALL).findall(response)
genrefile = os.path.join(metapath, "genres", "genres.json")
genres = ""
data = collections.OrderedDict()
for url, genreid, genrename in matches:
print "Netflix: DEBUG: " + url
url = "http://www.netflix.com/WiGenre?agid=" + genreid
data[utils.cleanstring(genrename)] = genreid
UpdateSubGenres = False
if os.path.exists(os.path.join(metapath, "Genres", genreid + ".json")):
oneday = 24 * 60 * 60
if utils.fileIsOlderThan(os.path.join(metapath, "Genres", genreid + ".json"), (oneday * int(cacheage))):
UpdateSubGenres = True
else:
UpdateSubGenres = True
if(UpdateSubGenres):
scrapeSubGenre(cookies, callstackpath, maxrequestsperminute, metapath, url)
#if genres != "":
if len(data) > 0:
#genres = "{" + genres + "}"
genres = json.dumps(data)
fh = open(genrefile, 'w')
fh.write(genres)
fh.close()
os.remove(os.path.join(metapath, "active", "scrape_genres"))
def getAuth(cookies, callstackpath, maxcalls):
# use utils to make the GET request
response = utils.makeGetRequest("http://www.netflix.com/login", cookies, callstackpath, maxcalls)
# find matches - should return <input type="hidden" name="authURL" value="[the_value]" /> instances
match = re.compile('input.*?authURL.*?value="(.*?)"', re.DOTALL).findall(str(response))
# use the last instance found (there should only be one anyway...)
auth = ""
for found in match:
if found != "":
auth = found
# return the found token
return auth
def search(addon, addonID, pluginhandle, viewpath, callstackpath, maxrequestsperminute, cookies, search_string, metaroot, cookiepath):
search_query = search_string.replace(' ', '+')
response = utils.makeGetRequest('http://www.netflix.com/search/' + search_query, cookies, callstackpath, maxrequestsperminute)
expr = "<div class=\"lockup\" data-titleid=\"(.*?)\" data-trackid=\"(.*?)\">"
matches = re.compile(expr, re.DOTALL).findall(response)
itemcount = 0
for title, track in matches:
listTitle(title, viewpath, pluginhandle, metaroot, addon, callstackpath, maxrequestsperminute, cookiepath)
itemcount += 1
if itemcount >= 1:
xbmcplugin.endOfDirectory(pluginhandle)
else:
dialog = xbmcgui.Dialog()
ok = dialog.ok('Netflix', utils.translation(addon, 30204))
def scrapeGenreTitles(cookies, callstackpath, maxrequestsperminute, metapath, genreid, cacheage, genrename):
if(os.path.exists(os.path.join(metapath, "apiurl"))):
fh = open(os.path.join(metapath, "apiurl"), 'r')
apiurl = fh.read()
fh.close()
print "Netflix: Scraping titles for " + genrename
content = ""
start = 0
size = 100
titles = ""
if not os.path.isdir(os.path.join(metapath, "GenreTitles")):
os.mkdir(os.path.join(metapath, "GenreTitles"))
if not os.path.isdir(os.path.join(metapath, "Titles")):
os.mkdir(os.path.join(metapath, "Titles"))
titles = []
while not content.startswith('{"catalogItems":[]}'):
requesturl = apiurl + "/wigenre?genreId=" + genreid + "&full=false&from=" + str(start) + "&to=" + str(start + size)
# increment for next call
start = start + size + 1
content = utils.makeGetRequest(requesturl, cookies, callstackpath, maxrequestsperminute)
match = re.compile("{\"boxart\":\"(.*?)\",\"titleId\":(.*?),\"title\":\"(.*?)\",\"playerUrl\":\"(.*?)\",\"trackId\":(.*?)}", re.DOTALL).findall(content)
for boxart, titleid, title, playerurl, trackid in match:
titledata = collections.OrderedDict()
titledata["boxart"] = boxart
titledata["titleId"] = titleid
titledata["title"] = title
titledata["playerurl"] = playerurl
titledata["trackid"] = trackid
if not os.path.isdir(os.path.join(metapath, "GenreTitles", genreid)):
os.mkdir(os.path.join(metapath, "GenreTitles", genreid))
if not os.path.isdir(os.path.join(metapath, "Titles", titleid)):
os.mkdir(os.path.join(metapath, "Titles", titleid))
coverart = utils.makeGetRequest(boxart, cookies, callstackpath, maxrequestsperminute)
fh = open(os.path.join(metapath, "Titles", titleid, "folder.jpg"), 'wb')
fh.write(coverart)
fh.close()
fh = open(os.path.join(metapath, "Titles", titleid, "coverart.jpg"), 'wb')
fh.write(coverart)
fh.close()
# write genre tags
if not os.path.isdir(os.path.join(metapath, "Titles", titleid, "Genres")):
os.mkdir(os.path.join(metapath, "Titles", titleid, "Genres"))
if genrename != "":
fh = open(os.path.join(metapath, "Titles", titleid, "Genres", genrename), 'w')
fh.write(genrename)
fh.close()
fh = open(os.path.join(metapath, "GenreTitles", genreid, titleid + ".json"), 'w')
fh.write(json.dumps(titledata))
fh.close()
UpdateTitle = False
titlefile = os.path.join(metapath, "Titles", titleid, "meta.json")
if os.path.exists(titlefile):
oneday = 24 * 60 * 60
if utils.fileIsOlderThan(titlefile, (oneday * int(cacheage))):
UpdateTitle = True
else:
UpdateTitle = True
if UpdateTitle:
scrapeTitle(cookies, callstackpath, maxrequestsperminute, metapath, titleid, trackid)
titles = titles + [titledata]
fh = open(os.path.join(metapath, "genreTitles", genreid + ".json"), 'w')
fh.write(json.dumps(titles))
fh.close()
def scrapeMyList(cookies, callstackpath, maxrequestsperminute, metapath):
if not os.path.isdir(os.path.join(metapath, "active")):
os.mkdir(os.path.join(metapath, "active"))
if not os.path.exists(os.path.join(metapath, "active", "scrape_mylist")):
fh = open(os.path.join(metapath, "active", "scrape_mylist"), 'w')
fh.write("currently scraping MyList")
fh.close()
#def makeGetRequest(url, cookies, callstackpath, maxcalls):
content = utils.makeGetRequest("https://www.netflix.com/MyList", cookies, callstackpath, maxrequestsperminute)
#expr = "<div.*?class=\"agMovie agMovie-lulg\".*?<img.*?src=\"(.*?)\".*?>.*?<a.*?WiPlayer\\?movieid=(.*?)&trkid=(.*?)&";
expr = '<div class="agMovie agMovie-lulg">.*?<img.*?src="(.*?)" ><a .*? href=".*?WiPlayer\\?movieid=(.*?)&trkid=(.*?)&'
#content = content.decode('utf-8')
if '<div id="yui-main">' in content:
content = content[content.index('<div id="yui-main">'):]
for ffile in os.listdir(os.path.join(metapath,"MyList")):
os.remove(os.path.join(metapath, "MyList", ffile))
matches = re.compile(expr, re.DOTALL).findall(content)
counter = 0
for boxart, titleid, trackid in matches:
counter += 1
fh = open(os.path.join(metapath, "MyList", titleid), 'w')
fh.write(str(counter))
fh.close()
titlefile = os.path.join(metapath, 'Titles', titleid, 'meta.json')
coverart = utils.makeGetRequest(boxart, cookies, callstackpath, maxrequestsperminute)
if not os.path.isdir(os.path.join(metapath, "Titles", titleid)):
os.mkdir(os.path.join(metapath, "Titles", titleid))
fh = open(os.path.join(metapath, "Titles", titleid, "folder.jpg"), 'wb')
fh.write(coverart)
fh.close()
fh = open(os.path.join(metapath, "Titles", titleid, "coverart.jpg"), 'wb')
fh.write(coverart)
fh.close()
if os.path.exists(os.path.join(metapath, "Titles", titleid, "coverart.jpg")):
iconpath = os.path.join(metapath, "Titles", titleid, "coverart.jpg")
else:
iconpath = ""
UpdateTitle = False
if os.path.exists(titlefile):
age = xbmcvfs.Stat(titlefile).st_mtime()
now = time.time()
oneday = 24 * 60 * 60
if (now-age) > (oneday*int(sys.argv[3])):
UpdateTitle = True
else:
UpdateTitle = True
if UpdateTitle:
scrapeTitle(cookies, callstackpath, maxrequestsperminute, metapath, titleid, trackid)
os.remove(os.path.join(metapath, "active", "scrape_mylist"))
def scrapeTitle(cookies, callstackpath, maxrequestsperminute, metapath, titleid, trackid):
if not os.path.isdir(os.path.join(metapath, "active")):
os.mkdir(os.path.join(metapath, "active"))
if not os.path.exists(os.path.join(metapath, "active", "scrape_title_" + titleid)):
fh = open(os.path.join(metapath, "active", "scrape_title_" + titleid), 'w')
fh.write("currently scraping MyList")
fh.close()
if(os.path.exists(os.path.join(metapath, "apiurl"))):
fh = open(os.path.join(metapath, "apiurl"), 'r')
apiurl = fh.read()
fh.close()
titleurl = apiurl + "/bob?titleid=" + titleid + "&trackid=" + trackid
content = utils.makeGetRequest(titleurl, cookies, callstackpath, maxrequestsperminute)
data = json.loads(content)
if not os.path.isdir(os.path.join(metapath, "Titles")):
os.mkdir(os.path.join(metapath, "Titles"))
if not os.path.isdir(os.path.join(metapath, "Titles", titleid)):
os.mkdir(os.path.join(metapath, "Titles", titleid))
fh = open(os.path.join(metapath, "Titles", titleid, "meta.json"), 'w')
fh.write(content)
fh.close()
if os.path.exists(os.path.join(metapath, "Titles", titleid, "coverart.jpg")):
iconpath = os.path.join(metapath, "Titles", titleid, "coverart.jpg")
else:
iconpath = ""
thetitle = data["title"]
mdplink = data["mdpLink"]
trackid = data["trackId"]
if data["isShow"]:
seasondataurl = "http://api-global.netflix.com/desktop/odp/episodes?forceEpisodes=true&routing=redirect&video=" + titleid
seasondata = utils.makeGetRequest(seasondataurl, cookies, callstackpath, maxrequestsperminute)
seasondata = seasondata[seasondata.index("{\"title\":"):]
fh = open(os.path.join(metapath, "Titles", titleid, "seasondata.json"),'w')
fh.write(seasondata)
fh.close()
data = json.loads(seasondata)
seasoncounter = 0
for season in data["episodes"]:
#http://www.netflix.com/WiMovie/70136119?trkid=50263680&actionMethod=seasonDetails&seasonId=70061401&seasonKind=ELECTRONIC
for episode in season:
if not os.path.exists(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]))):
os.mkdir(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"])))
fname = "S" + str(episode["season"]).zfill(2) + "E" + str(episode["episode"]).zfill(2)
fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]), fname + ".json"), 'w')
fh.write(json.dumps(episode))
fh.close()
stillswidth = 0
stillspath = ""
if "stills" in episode:
for still in episode["stills"]:
width = still["width"]
try:
if(int(width) > stillswidth):
stillswidth = int(width)
stillspath = still["url"]
except:
pass
if stillspath != "":
try:
stillimage = utils.makeGetRequest(stillspath, cookies, callstackpath, maxrequestsperminute)
fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]), fname + ".jpg"), 'wb')
fh.write(stillimage)
fh.close()
except:
pass
#http://www.netflix.com/WiMovie/70297439?actionMethod=seasonDetails&seasonId=70296034&seasonKind=ELECTRONIC
seasoninfourl = "http://www.netflix.com" + mdplink + "?&actionMethod=seasonDetails&seasonId=" + str(season[0]["seasonId"]) + "&seasonKind=ELECTRONIC"
seasoninforesponse = utils.makeGetRequest(seasoninfourl, cookies, callstackpath, maxrequestsperminute)
seasoninfo = json.loads(seasoninforesponse)
seasoninfosynopsismatch = re.compile('<\/h2><p class=\"synopsis\".*?>(.*?)<\/p>', re.DOTALL).findall(seasoninfo["html"])
synopsis = ""
for syno in seasoninfosynopsismatch:
synopsis += syno
fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(season[0]["season"]), "synopsis"), 'w')
fh.write(json.dumps(synopsis))
fh.close()
print "Netflix: " + thetitle.encode('utf-8') + " has been updated"
xbmc.executebuiltin('Notification("Netflix", "' + thetitle.encode('utf-8') + ' has been updated", 5000, ' + iconpath + ')')
os.remove(os.path.join(metapath, "active", "scrape_title_" + titleid))
def scrapeSeasonData(cookies, callstackpath, maxrequestsperminute, metapath, titleid):
seasondataurl = "http://api-global.netflix.com/desktop/odp/episodes?forceEpisodes=true&routing=redirect&video=" + titleid
seasondata = utils.makeGetRequest(seasondataurl, cookies, callstackpath, maxrequestsperminute)
seasondata = seasondata[seasondata.index("{\"title\":"):]
fh = open(os.path.join(metapath, "Titles", titleid, "seasondata.json"),'w')
fh.write(seasondata)
fh.close()
data = json.loads(seasondata)
seasoncounter = 0
for season in data["episodes"]:
#http://www.netflix.com/WiMovie/70136119?trkid=50263680&actionMethod=seasonDetails&seasonId=70061401&seasonKind=ELECTRONIC
for episode in season:
if not os.path.exists(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]))):
os.mkdir(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"])))
fname = "S" + str(episode["season"]).zfill(2) + "E" + str(episode["episode"]).zfill(2)
fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]), fname + ".json"), 'w')
fh.write(json.dumps(episode))
fh.close()
stillswidth = 0
stillspath = ""
if not os.path.exists(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]), fname + ".jpg")):
if "stills" in episode:
for still in episode["stills"]:
width = still["width"]
try:
if(int(width) > stillswidth):
stillswidth = int(width)
stillspath = still["url"]
except:
pass
if stillspath != "":
try:
stillimage = utils.makeGetRequest(stillspath, cookies, callstackpath, maxrequestsperminute)
fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(episode["season"]), fname + ".jpg"), 'wb')
fh.write(stillimage)
fh.close()
except:
pass
#http://www.netflix.com/WiMovie/70297439?actionMethod=seasonDetails&seasonId=70296034&seasonKind=ELECTRONIC
if not os.path.exists(os.path.join(metapath, "Titles", titleid, "Season " + str(season[0]["season"]), "synopsis")):
seasoninfourl = "http://www.netflix.com/WiMovie/" + titleid + "?&actionMethod=seasonDetails&seasonId=" + str(season[0]["seasonId"]) + "&seasonKind=ELECTRONIC"
seasoninforesponse = utils.makeGetRequest(seasoninfourl, cookies, callstackpath, maxrequestsperminute)
seasoninfo = json.loads(seasoninforesponse)
seasoninfosynopsismatch = re.compile('<\/h2><p class=\"synopsis\".*?>(.*?)<\/p>', re.DOTALL).findall(seasoninfo["html"])
synopsis = ""
for syno in seasoninfosynopsismatch:
synopsis += syno
fh = open(os.path.join(metapath, "Titles", titleid, "Season " + str(season[0]["season"]), "synopsis"), 'w')
fh.write(json.dumps(synopsis))
fh.close()
