def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return searchStrings= self.getSearchParams(movie,quality) lastsearch=0 for searchString in searchStrings: actualtime=int(time.time()) if actualtime-lastsearch<10: timetosleep= 10-(actualtime-lastsearch) time.sleep(timetosleep) URL = self.urls['search']+searchString r = self.opener.open(URL) soupfull = BeautifulSoup(r) #hack to avoid dummy parsing css and else delbegin=str(soupfull.prettify).split('<table width="100%">')[1] restable=delbegin[delbegin.find('<table'):delbegin.find('table>')+6] soup=BeautifulSoup(restable) resultsTable = soup.find("table") if resultsTable: rows = resultsTable.findAll("tr") x=0 for row in rows: x=x+1 if (x > 1): #bypass first row because title only #bypass date lines if 'Liste des torrents' in str(row) : continue link = row.findAll('td')[1].find("a", href=re.compile("torrent-details")) if link: new={} title = link.text testname=namer_check.correctName(title,movie) if testname==0: continue downloadURL = self.urls['test'] + "/" + row.find("a",href=re.compile("\.torrent"))['href'] size= row.findAll('td')[9].text leecher=row.findAll('td')[7].text seeder=row.findAll('td')[6].text date=row.findAll('td')[5].text detail=self.urls['test'] + "/" + row.find("a",href=re.compile("torrent-details"))['href'] def extra_check(item): return True new['id'] = detail[detail.rfind('=')+1:] new['name'] = title new['url'] = downloadURL new['detail_url'] = detail new['size'] = self.parseSize(size) new['age'] = self.ageToDays(date) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.download results.append(new)
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return searchStrings= self.getSearchParams(movie,quality) lastsearch=0 for searchString in searchStrings: actualtime=int(time.time()) if actualtime-lastsearch<10: timetosleep= 10-(actualtime-lastsearch) time.sleep(timetosleep) URL = self.urls['search']+searchString r = self.opener.open(URL) soup = BeautifulSoup( r, "html.parser" ) resultsTable = soup.find("table", { "class" : "lista" , "width":"100%" }) if resultsTable: rows = resultsTable.findAll("tr") x=0 for row in rows: x=x+1 if (x > 1): #bypass first row because title only link = row.findAll('td')[1].find("a", href=re.compile("torrent-details")) if link: new={} title = link.text testname=namer_check.correctName(title,movie) if testname==0: continue downloadURL = self.urls['test'] + "/" + row.find("a",href=re.compile("\.torrent"))['href'] size= row.findAll('td')[8].text leecher=row.findAll('td')[6].text seeder=row.findAll('td')[5].text date=row.findAll('td')[4].text detail=self.urls['test'] + "/" + row.find("a",href=re.compile("torrent-details"))['href'] def extra_check(item): return True new['id'] = detail[detail.rfind('=')+1:] new['name'] = title new['url'] = downloadURL new['detail_url'] = detail new['size'] = self.parseSize(size) new['age'] = self.ageToDays(date) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.download results.append(new)
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return searchStrings= self.getSearchParams(movie,quality) lastsearch=0 for searchString in searchStrings: actualtime=int(time.time()) if actualtime-lastsearch<10: timetosleep= 10-(actualtime-lastsearch) time.sleep(timetosleep) URL = self.urls['test'] + '/?section=TORRENTS&' + searchString.replace('!','') r = self.opener.open(URL) soup = BeautifulSoup( r, "html.parser" ) resultsTable = soup.find("div", { "class" : "DataGrid" }) if resultsTable: rows = resultsTable.findAll("ul") for row in rows: new={} link = row.find("a", title=True) title = link['title'] testname=namer_check.correctName(title,movie) if testname==0: continue size= row.findAll('li')[3].text leecher=row.findAll('li')[5].text seeder=row.findAll('li')[4].text autogetURL = self.urls['test'] +'/'+ (row.find("li", { "class" : "torrents_name"}).find('a')['href'][1:]).replace('#FTD_MENU','&menu=4') r = self.opener.open( autogetURL , 'wb').read() soup = BeautifulSoup( r, "html.parser" ) downloadURL = soup.find("div", { "class" : "autoget"}).find('a')['href'] date = soup.find("div", { "class" : "menu_box_stats"}).findAll('div')[4].find('span').text def extra_check(item): return True new['id'] = autogetURL new['name'] = title new['url'] = downloadURL new['detail_url'] = autogetURL new['size'] = self.parseSize(size) new['age'] = self.ageToDays(date) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.download results.append(new)
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return searchStrings= self.getSearchParams(movie,quality) lastsearch=0 for searchString in searchStrings: actualtime=int(time.time()) if actualtime-lastsearch<10: timetosleep= 10-(actualtime-lastsearch) time.sleep(timetosleep) URL = self.urls['test'] + '/?section=TORRENTS&' + searchString.replace('!','') r = self.opener.open(URL) soup = BeautifulSoup( r, "html5" ) resultsTable = soup.find("div", { "class" : "DataGrid" }) if resultsTable: rows = resultsTable.findAll("ul") for row in rows: new={} link = row.find("a", title=True) title = link['title'] testname=namer_check.correctName(title,movie) if testname==0: continue size= row.findAll('li')[3].text leecher=row.findAll('li')[5].text seeder=row.findAll('li')[4].text autogetURL = self.urls['test'] +'/'+ (row.find("li", { "class" : "torrents_name"}).find('a')['href'][1:]).replace('#FTD_MENU','&menu=4') r = self.opener.open( autogetURL , 'wb').read() soup = BeautifulSoup( r, "html5" ) downloadURL = soup.find("div", { "class" : "autoget"}).find('a')['href'] date = soup.find("div", { "class" : "menu_box_stats"}).findAll('div')[4].find('span').text def extra_check(item): return True new['id'] = autogetURL new['name'] = title new['url'] = downloadURL new['detail_url'] = autogetURL new['size'] = self.parseSize(size) new['age'] = self.ageToDays(date) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.download results.append(new)
def _search(self, movie, quality, results): searchStrings = self.getSearchParams(movie, quality) for searchString in searchStrings: searchUrl = self.urls[ 'search'] + 'rdirect.php?type=search&' + searchString data = self.getHTMLData(searchUrl) if "bad key" in str(data).lower(): log.error(u"GKS key invalid, check your config") continue parsedXML = parseString(data) channel = parsedXML.getElementsByTagName('channel')[0] description = channel.getElementsByTagName('description')[0] #description_text = self.get_xml_text(description).lower() text = "" for child_node in description.childNodes: if child_node.nodeType in (Node.CDATA_SECTION_NODE, Node.TEXT_NODE): text += child_node.data description_text = text.strip().lower() if "user can't be found" in description_text: log.error(u"GKS invalid digest, check your config") continue elif "invalid hash" in description_text: log.error(u"GKS invalid hash, check your config") continue else: items = channel.getElementsByTagName('item') for item in items: text = "" for child_node in item.getElementsByTagName( 'title')[0].childNodes: if child_node.nodeType in (Node.CDATA_SECTION_NODE, Node.TEXT_NODE): text += child_node.data title = text.strip().lower() if "aucun resultat" in title.lower(): log.debug( "No results found trying another if there is one") continue else: testname = namer_check.correctName(title, movie) if testname == 0: continue text = "" for child_node in item.getElementsByTagName( 'link')[0].childNodes: if child_node.nodeType in (Node.CDATA_SECTION_NODE, Node.TEXT_NODE): text += child_node.data downloadURL = text.strip().lower() desc = "" for child_node in item.getElementsByTagName( 'description')[0].childNodes: if child_node.nodeType in (Node.CDATA_SECTION_NODE, Node.TEXT_NODE): desc += child_node.data desc = desc.strip().lower() desc_values = desc.split(" | ") dict_attr = {} for x in desc_values: x_values = x.split(" : ") dict_attr[x_values[0]] = x_values[1] date = "" size = "" leechers = "" seeders = "" if "ajoute le" in dict_attr: date = dict_attr["ajoute le"] if "taille" in dict_attr: size = dict_attr["taille"] if "seeders" in dict_attr: seeders = dict_attr["seeders"] if "leechers" in dict_attr: leechers = dict_attr["leechers"] def extra_check(item): return True new = {} new['id'] = title new['name'] = title.strip() new['url'] = downloadURL new['detail_url'] = searchUrl new['size'] = self.parseSize(size) new['age'] = self.ageToDays(date) new['seeders'] = tryInt(seeders) new['leechers'] = tryInt(leechers) new['extra_check'] = extra_check results.append(new)
def _search(self, movie, quality, results): # Cookie login if not self.login_opener and not self.login(): return TitleStringReal = ( (getTitle(movie["library"]) + " " + simplifyString(quality["identifier"])) .replace("-", " ") .replace(" ", " ") .replace(" ", " ") .replace(" ", " ") .encode("utf8") ) URL = (self.urls["search"]).encode("UTF8") URL = unicodedata.normalize("NFD", unicode(URL, "utf8", "replace")) URL = URL.encode("ascii", "ignore") URL = urllib2.quote(URL.encode("utf8"), ":/?=") values = {"champ_recherche": TitleStringReal} data_tmp = urllib.urlencode(values) req = urllib2.Request(URL, data_tmp, headers={"User-Agent": "Mozilla/5.0"}) data = urllib2.urlopen(req) id = 1000 if data: cat_ids = self.getCatId(quality["identifier"]) table_order = ["name", "size", None, "age", "seeds", "leechers"] try: html = BeautifulSoup(data) resultdiv = html.find("div", attrs={"id": "recherche"}).find("table").find("tbody") for result in resultdiv.find_all("tr", recursive=False): try: new = {} # id = result.find_all('td')[2].find_all('a')[0]['href'][1:].replace('torrents/nfo/?id=','') name = result.find_all("td")[0].find_all("a")[0].text testname = namer_check.correctName(name, movie) if testname == 0: continue detail_url = result.find_all("td")[0].find_all("a")[0]["href"] # on scrapp la page detail urldetail = detail_url.encode("UTF8") urldetail = unicodedata.normalize("NFD", unicode(urldetail, "utf8", "replace")) urldetail = urldetail.encode("ascii", "ignore") urldetail = urllib2.quote(urldetail.encode("utf8"), ":/?=") req = urllib2.Request( urldetail, headers={"User-Agent": "Mozilla/5.0"} ) # POST request doesn't not work data_detail = urllib2.urlopen(req) url_download = "" if data_detail: html_detail = BeautifulSoup(data_detail) url_download = html_detail.find_all("div", attrs={"class": "download-torrent"})[0].find_all( "a" )[0]["href"] else: tmp = result.find_all("td")[0].find_all("a")[0]["href"] tmp = tmp.split("/")[6].replace(".html", ".torrent") url_download = "http://www.cpasbien.me/_torrents/%s" % tmp size = result.find_all("td")[1].text seeder = result.find_all("td")[2].find_all("span")[0].text leecher = result.find_all("td")[3].text age = "0" verify = getTitle(movie["library"]).split(" ") add = 1 for verify_unit in verify: if name.find(verify_unit) == -1: add = 0 def extra_check(item): return True if add == 1: new["id"] = id new["name"] = name.strip() new["url"] = url_download new["detail_url"] = detail_url new["size"] = self.parseSize(size) new["age"] = self.ageToDays(age) new["seeders"] = tryInt(seeder) new["leechers"] = tryInt(leecher) new["extra_check"] = extra_check new["download"] = self.loginDownload # new['score'] = fireEvent('score.calculate', new, movie, single = True) # log.error('score') # log.error(new['score']) results.append(new) id = id + 1 except: log.error("Failed parsing cPASbien: %s", traceback.format_exc()) except AttributeError: log.debug("No search results found.") else: log.debug("No search results found.")
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return searchStrings = self.getSearchParams(movie, quality) lastsearch = 0 for searchString in searchStrings: actualtime = int(time.time()) if actualtime - lastsearch < 10: timetosleep = 10 - (actualtime - lastsearch) time.sleep(timetosleep) URL = self.urls['search'] + searchString r = self.opener.open(URL) soupfull = BeautifulSoup(r) #hack to avoid dummy parsing css and else delbegin = str(soupfull.prettify).split('<table width="100%">')[1] restable = delbegin[delbegin.find('<table' ):delbegin.find('table>') + 6] soup = BeautifulSoup(restable) resultsTable = soup.find("table") if resultsTable: rows = resultsTable.findAll("tr") x = 0 for row in rows: x = x + 1 if (x > 1): #bypass first row because title only #bypass date lines if 'Liste des torrents' in str(row): continue link = row.findAll('td')[1].find( "a", href=re.compile("torrent-details")) if link: new = {} title = link.text testname = namer_check.correctName(title, movie) if testname == 0: continue downloadURL = self.urls['test'] + "/" + row.find( "a", href=re.compile("\.torrent"))['href'] size = row.findAll('td')[9].text leecher = row.findAll('td')[7].text seeder = row.findAll('td')[6].text date = row.findAll('td')[5].text detail = self.urls['test'] + "/" + row.find( "a", href=re.compile("torrent-details"))['href'] def extra_check(item): return True new['id'] = detail[detail.rfind('=') + 1:] new['name'] = title new['url'] = downloadURL new['detail_url'] = detail new['size'] = self.parseSize(size) new['age'] = self.ageToDays(date) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.download results.append(new)
def _search(self, movie, quality, results): # Cookie login if not self.login_opener and not self.login(): return searchStrings= self.getSearchParams(movie,quality) for searchString in searchStrings: URL = self.urls['search']+searchString data = self.getHTMLData(URL , opener = self.login_opener) if data: cat_ids = self.getCatId(quality['identifier']) table_order = ['name', 'size', None, 'age', 'seeds', 'leechers'] try: html = BeautifulSoup(data) resultdiv = html.find('table', attrs = {'class':'results'}).find('tbody') for result in resultdiv.find_all('tr', recursive = False): try: categorie = result.find_all('td')[0].find_all('img')[0]['class'] insert = 0 if categorie == ['cat-631']: insert = 1 if categorie == ['cat-455']: insert = 1 if insert == 1 : new = {} idt = result.find_all('td')[2].find_all('a')[0]['href'][1:].replace('torrents/nfo/?id=','') name = result.find_all('td')[1].find_all('a')[0]['title'] testname=namer_check.correctName(name,movie) if testname==0: continue url = ('http://www.t411.me/torrents/download/?id=%s' % id) detail_url = ('http://www.t411.me/torrents/?id=%s' % id) size = result.find_all('td')[5].text age = result.find_all('td')[4].text seeder = result.find_all('td')[7].text leecher = result.find_all('td')[8].text def extra_check(item): return True new['id'] = idt new['name'] = name + ' french' new['url'] = url new['detail_url'] = detail_url new['size'] = self.parseSize(size) new['age'] = self.ageToDays(age) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.loginDownload results.append(new) except: log.error('Failed parsing T411: %s', traceback.format_exc()) except AttributeError: log.debug('No search results found.') else: log.debug('No search results found.')
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return searchStrings= self.getSearchParams(movie,quality) lastsearch=0 for searchString in searchStrings: actualtime=int(time.time()) if actualtime-lastsearch<10: timetosleep= 10-(actualtime-lastsearch) time.sleep(timetosleep) URL = self.urls['search']+searchString r = self.opener.open(URL) soup = BeautifulSoup( r, "html.parser" ) if soup.find('table', attrs = {'class':'results'}): resultdiv = soup.find('table', attrs = {'class':'results'}).find('tbody') else: continue if resultdiv: try: for result in resultdiv.findAll('tr'): try: categorie = result.findAll('td')[0].findAll('a')[0]['href'][result.findAll('td')[0].findAll('a')[0]['href'].find('='):] insert = 0 if categorie == '=631': insert = 1 if categorie == '=455': insert = 1 if categorie == '=634': insert = 1 if insert == 1 : new = {} idt = result.findAll('td')[2].findAll('a')[0]['href'][1:].replace('torrents/nfo/?id=','') name = result.findAll('td')[1].findAll('a')[0]['title'] testname=namer_check.correctName(name,movie) if testname==0: continue url = ('http://www.t411.li/torrents/download/?id=%s' % idt) detail_url = ('http://www.t411.li/torrents/?id=%s' % idt) leecher = result.findAll('td')[8].text size = result.findAll('td')[5].text age = result.findAll('td')[4].text seeder = result.findAll('td')[7].text def extra_check(item): return True new['id'] = idt new['name'] = name + ' french' new['url'] = url new['detail_url'] = detail_url new['size'] = self.parseSize(str(size)) new['age'] = self.ageToDays(str(age)) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.download results.append(new) except: log.error('Failed parsing T411: %s', traceback.format_exc()) except AttributeError: log.debug('No search results found.') else: log.debug('No search results found.')
def _search(self, movie, quality, results): MovieTitles = movie['info']['titles'] moviequality = simplifyString(quality['identifier']) movieyear = movie['info']['year'] if quality['custom']['3d']==1: threeD= True else: threeD=False if moviequality in ("720p","1080p","bd50"): cat1='39' cat2='49' minSize = 2000 elif moviequality in ("dvdr"): cat1='23' cat2='48' minSize = 3000 else: cat1='6' cat2='27' minSize = 500 for MovieTitle in MovieTitles: try: TitleStringReal = str(MovieTitle.encode("latin-1").replace('-',' ')) except: continue if threeD: TitleStringReal = TitleStringReal + ' 3d' data = 'chkInit=1&edTitre='+TitleStringReal+'&chkTitre=on&chkFichier=on&chkCat=on&cats%5B%5D='+cat1+'&cats%5B%5D='+cat2+'&edAge=&edYear=' try: soup = BeautifulSoup( urllib2.urlopen(self.urls['search'], data), "html5lib" ) except Exception, e: log.error(u"Error trying to load BinNewz response: "+e) return [] tables = soup.findAll("table", id="tabliste") for table in tables: rows = table.findAll("tr") for row in rows: cells = row.select("> td") if (len(cells) < 11): continue name = cells[2].text.strip() #filename = cells[5].contents[0] testname=namer_check.correctName(name,movie) #testfilename=namer_check.correctName(filename,movie) if testname==0:# and testfilename==0: continue language = cells[3].find("img").get("src") if not "_fr" in language and not "_frq" in language: continue detectedlang='' if "_fr" in language: detectedlang=' truefrench ' else: detectedlang=' french ' # blacklist_groups = [ "alt.binaries.multimedia" ] blacklist_groups = [] newgroupLink = cells[4].find("a") newsgroup = None if newgroupLink.contents: newsgroup = newgroupLink.contents[0] if newsgroup in self.allowedGroups: newsgroup = self.allowedGroups[newsgroup] else: log.error(u"Unknown binnewz newsgroup: " + newsgroup) continue if newsgroup in blacklist_groups: log.error(u"Ignoring result, newsgroup is blacklisted: " + newsgroup) continue filename = cells[5].contents[0] m = re.search("^(.+)\s+{(.*)}$", name) qualityStr = "" if m: name = m.group(1) qualityStr = m.group(2) m = re.search("^(.+)\s+\[(.*)\]$", name) source = None if m: name = m.group(1) source = m.group(2) m = re.search("(.+)\(([0-9]{4})\)", name) year = "" if m: name = m.group(1) year = m.group(2) if int(year) > movieyear + 1 or int(year) < movieyear - 1: continue m = re.search("(.+)\((\d{2}/\d{2}/\d{4})\)", name) dateStr = "" if m: name = m.group(1) dateStr = m.group(2) year = dateStr[-5:].strip(")").strip("/") m = re.search("(.+)\s+S(\d{2})\s+E(\d{2})(.*)", name) if m: name = m.group(1) + " S" + m.group(2) + "E" + m.group(3) + m.group(4) m = re.search("(.+)\s+S(\d{2})\s+Ep(\d{2})(.*)", name) if m: name = m.group(1) + " S" + m.group(2) + "E" + m.group(3) + m.group(4) filenameLower = filename.lower() searchItems = [] if qualityStr=="": if source in ("Blu Ray-Rip", "HD DVD-Rip"): qualityStr="brrip" elif source =="DVDRip": qualityStr="dvdrip" elif source == "TS": qualityStr ="ts" elif source == "DVDSCR": qualityStr ="scr" elif source == "CAM": qualityStr ="cam" elif moviequality == "dvdr": qualityStr ="dvdr" if year =='': year = '1900' if len(searchItems) == 0 and qualityStr == str(moviequality): searchItems.append( filename ) for searchItem in searchItems: resultno=1 for downloader in self.nzbDownloaders: log.info("Searching for download : " + name + ", search string = "+ searchItem + " on " + downloader.__class__.__name__) try: binsearch_result = downloader.search(searchItem, minSize, newsgroup ) if binsearch_result: new={} def extra_check(item): return True qualitytag='' if qualityStr.lower() in ['720p','1080p']: qualitytag=' hd x264 h264 ' elif qualityStr.lower() in ['dvdrip']: qualitytag=' dvd xvid ' elif qualityStr.lower() in ['brrip']: qualitytag=' hdrip ' elif qualityStr.lower() in ['ts']: qualitytag=' webrip ' elif qualityStr.lower() in ['scr']: qualitytag='' elif qualityStr.lower() in ['dvdr']: qualitytag=' pal video_ts ' new['id'] = binsearch_result.nzbid new['name'] = name + detectedlang + qualityStr + qualitytag + downloader.__class__.__name__ new['url'] = binsearch_result.nzburl new['detail_url'] = binsearch_result.refererURL new['size'] = binsearch_result.sizeInMegs new['age'] = binsearch_result.age new['extra_check'] = extra_check results.append(new) resultno=resultno+1 log.info("Found : " + searchItem + " on " + downloader.__class__.__name__) if resultno==3: break except Exception, e: log.error("Searching from " + downloader.__class__.__name__ + " failed : " + str(e) + traceback.format_exc())
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return searchStrings= self.getSearchParams(movie,quality) for searchString in searchStrings: URL = self.urls['search']+searchString data = self.getHTMLData(URL) if data: try: html = BeautifulSoup(data) resultdiv = html.find('table', attrs = {'class':'results'}).find('tbody') for result in resultdiv.find_all('tr', recursive = False): try: categorie = result.find_all('td')[0].find_all('img')[0]['class'] insert = 0 if categorie == ['cat-631']: insert = 1 if categorie == ['cat-455']: insert = 1 if insert == 1 : new = {} idt = result.find_all('td')[2].find_all('a')[0]['href'][1:].replace('torrents/nfo/?id=','') name = result.find_all('td')[1].find_all('a')[0]['title'] testname=namer_check.correctName(name,movie) if testname==0: continue url = ('http://www.t411.me/torrents/download/?id=%s' % idt) detail_url = ('http://www.t411.me/torrents/?id=%s' % idt) size = result.find_all('td')[5].text age = result.find_all('td')[4].text seeder = result.find_all('td')[7].text leecher = result.find_all('td')[8].text def extra_check(item): return True new['id'] = idt new['name'] = name + ' french' new['url'] = url new['detail_url'] = detail_url new['size'] = self.parseSize(size) new['age'] = self.ageToDays(age) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.download results.append(new) except: log.error('Failed parsing T411: %s', traceback.format_exc()) except AttributeError: log.debug('No search results found.') else: log.debug('No search results found.')
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return searchStrings = self.getSearchParams(movie, quality) lastsearch = 0 for searchString in searchStrings: actualtime = int(time.time()) if actualtime - lastsearch < 10: timetosleep = 10 - (actualtime - lastsearch) time.sleep(timetosleep) URL = self.urls["search"] + searchString r = self.opener.open(URL) soup = BeautifulSoup(r, "html.parser") if soup.find("table", attrs={"class": "results"}): resultdiv = soup.find("table", attrs={"class": "results"}).find("tbody") else: continue if resultdiv: try: for result in resultdiv.findAll("tr"): try: categorie = result.findAll("td")[0].findAll("a")[0]["href"][ result.findAll("td")[0].findAll("a")[0]["href"].find("=") : ] insert = 0 if categorie == "=631": insert = 1 if categorie == "=455": insert = 1 if categorie == "=634": insert = 1 if insert == 1: new = {} idt = ( result.findAll("td")[2].findAll("a")[0]["href"][1:].replace("torrents/nfo/?id=", "") ) name = result.findAll("td")[1].findAll("a")[0]["title"] testname = namer_check.correctName(name, movie) if testname == 0: continue url = "http://www.t411.li/torrents/download/?id=%s" % idt detail_url = "http://www.t411.li/torrents/?id=%s" % idt leecher = result.findAll("td")[8].text size = result.findAll("td")[5].text age = result.findAll("td")[4].text seeder = result.findAll("td")[7].text def extra_check(item): return True new["id"] = idt new["name"] = name + " french" new["url"] = url new["detail_url"] = detail_url new["size"] = self.parseSize(str(size)) new["age"] = self.ageToDays(str(age)) new["seeders"] = tryInt(seeder) new["leechers"] = tryInt(leecher) new["extra_check"] = extra_check new["download"] = self.download results.append(new) except: log.error("Failed parsing T411: %s", traceback.format_exc()) except AttributeError: log.debug("No search results found.") else: log.debug("No search results found.")
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return TitleStringReal = (getTitle(movie['info']) + ' ' + simplifyString(quality['identifier'] )).replace('-',' ').replace(' ',' ').replace(' ',' ').replace(' ',' ').encode("utf8") URL = (self.urls['search']).encode('UTF8') URL=unicodedata.normalize('NFD',unicode(URL,"utf8","replace")) URL=URL.encode('ascii','ignore') URL = urllib2.quote(URL.encode('utf8'), ":/?=") URL = URL + TitleStringReal values = { } URLTST = (self.urls['test']).encode('UTF8') data_tmp = urllib.urlencode(values) req = urllib2.Request(URL, data_tmp, headers={'User-Agent' : "Mozilla/5.0"} ) data = urllib2.urlopen(req) id = 1000 if data: try: html = BeautifulSoup(data) erlin=0 resultdiv=[] while erlin==0: try: resultContent = html.findAll(attrs={'class': ["listing-torrent"]})[0] if resultContent: resultlin = resultContent.findAll(attrs={'class': ['table-hover']})[0].find('tbody') if resultlin: trList= resultlin.findAll("tr"); for tr in trList: resultdiv.append(tr) erlin=1 except: erlin=1 nbrResult = 0 for result in resultdiv: try: new = {} firstTd = result.findAll("td")[0] nothing = firstTd.findAll("center") if nothing: continue name = firstTd.findAll("a")[1]['title']; testname = namer_check.correctName(name,movie) if testname == 0 and nbrResult < 5: values_sec = {} url_sec = result.findAll("a")[1]['href']; req_sec = urllib2.Request(URLTST+url_sec, values_sec, headers={'User-Agent': "Mozilla/5.0"}) data_sec = urllib2.urlopen(req_sec) if data_sec: html_sec = BeautifulSoup(data_sec) classlin_sec = 'torrentsdesc' resultlin_sec = html_sec.findAll(attrs={'id': [classlin_sec]})[0] name = resultlin_sec.find("div").text name = name.replace(".", " ") testname = namer_check.correctName(name, movie) if testname == 0: continue nbrResult += 1 values_sec = {} detail_url = result.findAll("a")[1]['href']; req_sec = urllib2.Request(URLTST+detail_url, values_sec, headers={'User-Agent': "Mozilla/5.0"}) data_sec = urllib2.urlopen(req_sec) html_sec = BeautifulSoup(data_sec) classlin_sec = 'download' resultlin_sec = html_sec.findAll(attrs={'class': [classlin_sec]})[0] url_download = resultlin_sec.findAll("a")[0]['href'] size = result.findAll("td")[1].text seeder = result.findAll("td")[2].text leecher = result.findAll("td")[3].text age = '1' verify = getTitle(movie['info']).split(' ') add = 1 for verify_unit in verify: if (name.lower().find(verify_unit.lower()) == -1) : add = 0 def extra_check(item): return True if add == 1: new['id'] = id new['name'] = name.strip() + ' french' new['url'] = url_download new['detail_url'] = detail_url new['size'] = self.parseSize(size) new['age'] = 10 new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.loginDownload #new['score'] = fireEvent('score.calculate', new, movie, single = True) #log.error('score') #log.error(new['score']) results.append(new) id = id+1 except: log.error('Failed parsing zetorrents: %s', traceback.format_exc()) except AttributeError: log.debug('No search results found.') else: log.debug('No search results found.')
def _search(self, movie, quality, results): nzbDownloaders = [NZBClub(), BinSearch(), NZBIndex()] MovieTitles = movie['library']['info']['titles'] moviequality = simplifyString(quality['identifier']) movieyear = movie['library']['year'] if moviequality in ("720p","1080p","bd50"): cat1='39' cat2='49' minSize = 2000 elif moviequality in ("dvdr"): cat1='23' cat2='48' minSize = 3000 else: cat1='6' cat2='27' minSize = 500 for MovieTitle in MovieTitles: TitleStringReal = str(MovieTitle.encode("utf-8").replace('-',' ')) data = 'chkInit=1&edTitre='+TitleStringReal+'&chkTitre=on&chkFichier=on&chkCat=on&cats%5B%5D='+cat1+'&cats%5B%5D='+cat2+'&edAge=&edYear=' try: soup = BeautifulSoup( urllib2.urlopen(self.urls['search'], data) ) except Exception, e: log.error(u"Error trying to load BinNewz response: "+e) return [] tables = soup.findAll("table", id="tabliste") for table in tables: rows = table.findAll("tr") for row in rows: cells = row.select("> td") if (len(cells) < 11): continue name = cells[2].text.strip() testname=namer_check.correctName(name,movie) if testname==0: continue language = cells[3].find("img").get("src") if not "_fr" in language and not "_frq" in language: continue # blacklist_groups = [ "alt.binaries.multimedia" ] blacklist_groups = [] newgroupLink = cells[4].find("a") newsgroup = None if newgroupLink.contents: newsgroup = newgroupLink.contents[0] if newsgroup == "abmulti": newsgroup = "alt.binaries.multimedia" elif newsgroup == "abtvseries": newsgroup = "alt.binaries.tvseries" elif newsgroup == "abtv": newsgroup = "alt.binaries.tv" elif newsgroup == "a.b.teevee": newsgroup = "alt.binaries.teevee" elif newsgroup == "abstvdivxf": newsgroup = "alt.binaries.series.tv.divx.french" elif newsgroup == "abhdtvx264fr": newsgroup = "alt.binaries.hdtv.x264.french" elif newsgroup == "abmom": newsgroup = "alt.binaries.mom" elif newsgroup == "abhdtv": newsgroup = "alt.binaries.hdtv" elif newsgroup == "abboneless": newsgroup = "alt.binaries.boneless" elif newsgroup == "abhdtvf": newsgroup = "alt.binaries.hdtv.french" elif newsgroup == "abhdtvx264": newsgroup = "alt.binaries.hdtv.x264" elif newsgroup == "absuperman": newsgroup = "alt.binaries.superman" elif newsgroup == "abechangeweb": newsgroup = "alt.binaries.echange-web" elif newsgroup == "abmdfvost": newsgroup = "alt.binaries.movies.divx.french.vost" elif newsgroup == "abdvdr": newsgroup = "alt.binaries.dvdr" elif newsgroup == "abmzeromov": newsgroup = "alt.binaries.movies.zeromovies" elif newsgroup == "abcfaf": newsgroup = "alt.binaries.cartoons.french.animes-fansub" elif newsgroup == "abcfrench": newsgroup = "alt.binaries.cartoons.french" elif newsgroup == "abgougouland": newsgroup = "alt.binaries.gougouland" elif newsgroup == "abroger": newsgroup = "alt.binaries.roger" elif newsgroup == "abtatu": newsgroup = "alt.binaries.tatu" elif newsgroup =="abstvf": newsgroup = "alt.binaries.series.tv.french" elif newsgroup =="abmdfreposts": newsgroup="alt.binaries.movies.divx.french.reposts" elif newsgroup =="abmdf": newsgroup="alt.binaries.movies.french" elif newsgroup =="abhdtvfrepost": newsgroup="alt.binaries.hdtv.french.repost" elif newsgroup == "abmmkv": newsgroup = "alt.binaries.movies.mkv" elif newsgroup == "abf-tv": newsgroup = "alt.binaries.french-tv" elif newsgroup == "abmdfo": newsgroup = "alt.binaries.movies.divx.french.old" elif newsgroup == "abmf": newsgroup = "alt.binaries.movies.french" elif newsgroup == "ab.movies": newsgroup = "alt.binaries.movies" elif newsgroup == "a.b.french": newsgroup = "alt.binaries.french" elif newsgroup == "a.b.3d": newsgroup = "alt.binaries.3d" elif newsgroup == "ab.dvdrip": newsgroup = "alt.binaries.dvdrip" elif newsgroup == "ab.welovelori": newsgroup = "alt.binaries.welovelori" elif newsgroup == "abblu-ray": newsgroup = "alt.binaries.blu-ray" elif newsgroup == "ab.bloaf": newsgroup = "alt.binaries.bloaf" elif newsgroup == "ab.hdtv.german": newsgroup = "alt.binaries.hdtv.german" elif newsgroup == "abmd": newsgroup = "alt.binaries.movies.divx" elif newsgroup == "ab.ath": newsgroup = "alt.binaries.ath" elif newsgroup == "a.b.town": newsgroup = "alt.binaries.town" else: log.error(u"Unknown binnewz newsgroup: " + newsgroup) continue if newsgroup in blacklist_groups: log.error(u"Ignoring result, newsgroup is blacklisted: " + newsgroup) continue filename = cells[5].contents[0] m = re.search("^(.+)\s+{(.*)}$", name) qualityStr = "" if m: name = m.group(1) qualityStr = m.group(2) m = re.search("^(.+)\s+\[(.*)\]$", name) source = None if m: name = m.group(1) source = m.group(2) m = re.search("(.+)\(([0-9]{4})\)", name) year = "" if m: name = m.group(1) year = m.group(2) if int(year) > movieyear + 1 or int(year) < movieyear - 1: continue m = re.search("(.+)\((\d{2}/\d{2}/\d{4})\)", name) dateStr = "" if m: name = m.group(1) dateStr = m.group(2) year = dateStr[-5:].strip(")").strip("/") m = re.search("(.+)\s+S(\d{2})\s+E(\d{2})(.*)", name) if m: name = m.group(1) + " S" + m.group(2) + "E" + m.group(3) + m.group(4) m = re.search("(.+)\s+S(\d{2})\s+Ep(\d{2})(.*)", name) if m: name = m.group(1) + " S" + m.group(2) + "E" + m.group(3) + m.group(4) filenameLower = filename.lower() searchItems = [] if qualityStr=="": if source in ("Blu Ray-Rip", "HD DVD-Rip"): qualityStr="brrip" elif source =="DVDRip": qualityStr="dvdrip" elif source == "TS": qualityStr ="ts" elif source == "DVDSCR": qualityStr ="scr" elif source == "CAM": qualityStr ="cam" elif moviequality == "dvdr": qualityStr ="dvdr" if year =='': year = '1900' if len(searchItems) == 0 and qualityStr == str(moviequality): searchItems.append( filename ) for searchItem in searchItems: resultno=1 for downloader in nzbDownloaders: log.info("Searching for download : " + name + ", search string = "+ searchItem + " on " + downloader.__class__.__name__) try: binsearch_result = downloader.search(searchItem, minSize, newsgroup ) if binsearch_result: new={} def extra_check(item): return True new['id'] = binsearch_result.nzbid new['name'] = name + ' french ' + qualityStr + ' '+ searchItem +' '+ name +' ' + downloader.__class__.__name__ new['url'] = binsearch_result.nzburl new['detail_url'] = binsearch_result.refererURL new['size'] = binsearch_result.sizeInMegs new['age'] = binsearch_result.age new['extra_check'] = extra_check results.append(new) resultno=resultno+1 log.info("Found : " + searchItem + " on " + downloader.__class__.__name__) if resultno==3: break except Exception, e: log.error("Searching from " + downloader.__class__.__name__ + " failed : " + str(e) + traceback.format_exc())
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return TitleStringReal = (getTitle(movie['info']) + ' ' + simplifyString(quality['identifier'])).replace( '-', ' ').replace(' ', ' ').replace( ' ', ' ').replace(' ', ' ').encode("utf8") URL = (self.urls['search']).encode('UTF8') URL = unicodedata.normalize('NFD', unicode(URL, "utf8", "replace")) URL = URL.encode('ascii', 'ignore') URL = urllib2.quote(URL.encode('utf8'), ":/?=") values = {'champ_recherche': TitleStringReal} data_tmp = urllib.urlencode(values) req = urllib2.Request(URL, data_tmp, headers={'User-Agent': "Mozilla/5.0"}) data = urllib2.urlopen(req) id = 1000 if data: try: html = BeautifulSoup(data) resultdiv = html.find('div', attrs={ 'id': 'recherche' }).find('table').find('tbody') for result in resultdiv.find_all('tr', recursive=False): try: new = {} #id = result.find_all('td')[2].find_all('a')[0]['href'][1:].replace('torrents/nfo/?id=','') name = result.find_all('td')[0].find_all('a')[0].text testname = namer_check.correctName(name, movie) if testname == 0: continue detail_url = result.find_all('td')[0].find_all( 'a')[0]['href'] #on scrapp la page detail urldetail = detail_url.encode('UTF8') urldetail = unicodedata.normalize( 'NFD', unicode(urldetail, "utf8", "replace")) urldetail = urldetail.encode('ascii', 'ignore') urldetail = urllib2.quote(urldetail.encode('utf8'), ":/?=") req = urllib2.Request( urldetail, headers={'User-Agent': "Mozilla/5.0" }) # POST request doesn't not work data_detail = urllib2.urlopen(req) url_download = "" if data_detail: html_detail = BeautifulSoup(data_detail) url_tmp = html_detail.find_all( 'div', attrs={'class': 'download-torrent' })[0].find_all('a')[0]['href'] url_download = ('http://www.cpasbien.pe%s' % url_tmp) else: tmp = result.find_all('td')[0].find_all( 'a')[0]['href'] tmp = tmp.split('/')[6].replace( '.html', '.torrent') url_download = ( 'http://www.cpasbien.pe/_torrents/%s' % tmp) size = result.find_all('td')[1].text seeder = result.find_all('td')[2].find_all( 'span')[0].text leecher = result.find_all('td')[3].text age = '0' verify = getTitle(movie['info']).split(' ') add = 1 for verify_unit in verify: if (name.lower().find(verify_unit.lower()) == -1): add = 0 def extra_check(item): return True if add == 1: new['id'] = id new['name'] = name.strip() new['url'] = url_download new['detail_url'] = detail_url new['size'] = self.parseSize(size) new['age'] = self.ageToDays(age) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.loginDownload #new['score'] = fireEvent('score.calculate', new, movie, single = True) #log.error('score') #log.error(new['score']) results.append(new) id = id + 1 except: log.error('Failed parsing cPASbien: %s', traceback.format_exc()) except AttributeError: log.debug('No search results found.') else: log.debug('No search results found.')
def _search(self, movie, quality, results): nzbDownloaders = [NZBClub(), BinSearch(), NZBIndex()] MovieTitles = movie["info"]["titles"] moviequality = simplifyString(quality["identifier"]) movieyear = movie["info"]["year"] if quality["custom"]["3d"] == 1: threeD = True else: threeD = False if moviequality in ("720p", "1080p", "bd50"): cat1 = "39" cat2 = "49" minSize = 2000 elif moviequality in ("dvdr"): cat1 = "23" cat2 = "48" minSize = 3000 else: cat1 = "6" cat2 = "27" minSize = 500 for MovieTitle in MovieTitles: try: TitleStringReal = str(MovieTitle.encode("latin-1").replace("-", " ")) except: continue if threeD: TitleStringReal = TitleStringReal + " 3d" data = ( "chkInit=1&edTitre=" + TitleStringReal + "&chkTitre=on&chkFichier=on&chkCat=on&cats%5B%5D=" + cat1 + "&cats%5B%5D=" + cat2 + "&edAge=&edYear=" ) try: soup = BeautifulSoup(urllib2.urlopen(self.urls["search"], data)) except Exception, e: log.error(u"Error trying to load BinNewz response: " + e) return [] tables = soup.findAll("table", id="tabliste") for table in tables: rows = table.findAll("tr") for row in rows: cells = row.select("> td") if len(cells) < 11: continue name = cells[2].text.strip() testname = namer_check.correctName(name, movie) if testname == 0: continue language = cells[3].find("img").get("src") if not "_fr" in language and not "_frq" in language: continue detectedlang = "" if "_fr" in language: detectedlang = " truefrench " else: detectedlang = " french " # blacklist_groups = [ "alt.binaries.multimedia" ] blacklist_groups = [] newgroupLink = cells[4].find("a") newsgroup = None if newgroupLink.contents: newsgroup = newgroupLink.contents[0] if newsgroup == "abmulti": newsgroup = "alt.binaries.multimedia" elif newsgroup == "abtvseries": newsgroup = "alt.binaries.tvseries" elif newsgroup == "abtv": newsgroup = "alt.binaries.tv" elif newsgroup == "a.b.teevee": newsgroup = "alt.binaries.teevee" elif newsgroup == "abstvdivxf": newsgroup = "alt.binaries.series.tv.divx.french" elif newsgroup == "abhdtvx264fr": newsgroup = "alt.binaries.hdtv.x264.french" elif newsgroup == "abmom": newsgroup = "alt.binaries.mom" elif newsgroup == "abhdtv": newsgroup = "alt.binaries.hdtv" elif newsgroup == "abboneless": newsgroup = "alt.binaries.boneless" elif newsgroup == "abhdtvf": newsgroup = "alt.binaries.hdtv.french" elif newsgroup == "abhdtvx264": newsgroup = "alt.binaries.hdtv.x264" elif newsgroup == "absuperman": newsgroup = "alt.binaries.superman" elif newsgroup == "abechangeweb": newsgroup = "alt.binaries.echange-web" elif newsgroup == "abmdfvost": newsgroup = "alt.binaries.movies.divx.french.vost" elif newsgroup == "abdvdr": newsgroup = "alt.binaries.dvdr" elif newsgroup == "abmzeromov": newsgroup = "alt.binaries.movies.zeromovies" elif newsgroup == "abcfaf": newsgroup = "alt.binaries.cartoons.french.animes-fansub" elif newsgroup == "abcfrench": newsgroup = "alt.binaries.cartoons.french" elif newsgroup == "abgougouland": newsgroup = "alt.binaries.gougouland" elif newsgroup == "abroger": newsgroup = "alt.binaries.roger" elif newsgroup == "abtatu": newsgroup = "alt.binaries.tatu" elif newsgroup == "abstvf": newsgroup = "alt.binaries.series.tv.french" elif newsgroup == "abmdfreposts": newsgroup = "alt.binaries.movies.divx.french.reposts" elif newsgroup == "abmdf": newsgroup = "alt.binaries.movies.french" elif newsgroup == "abhdtvfrepost": newsgroup = "alt.binaries.hdtv.french.repost" elif newsgroup == "abmmkv": newsgroup = "alt.binaries.movies.mkv" elif newsgroup == "abf-tv": newsgroup = "alt.binaries.french-tv" elif newsgroup == "abmdfo": newsgroup = "alt.binaries.movies.divx.french.old" elif newsgroup == "abmf": newsgroup = "alt.binaries.movies.french" elif newsgroup == "ab.movies": newsgroup = "alt.binaries.movies" elif newsgroup == "a.b.french": newsgroup = "alt.binaries.french" elif newsgroup == "a.b.3d": newsgroup = "alt.binaries.3d" elif newsgroup == "ab.dvdrip": newsgroup = "alt.binaries.dvdrip" elif newsgroup == "ab.welovelori": newsgroup = "alt.binaries.welovelori" elif newsgroup == "abblu-ray": newsgroup = "alt.binaries.blu-ray" elif newsgroup == "ab.bloaf": newsgroup = "alt.binaries.bloaf" elif newsgroup == "ab.hdtv.german": newsgroup = "alt.binaries.hdtv.german" elif newsgroup == "abmd": newsgroup = "alt.binaries.movies.divx" elif newsgroup == "ab.ath": newsgroup = "alt.binaries.ath" elif newsgroup == "a.b.town": newsgroup = "alt.binaries.town" elif newsgroup == "a.b.u-4all": newsgroup = "alt.binaries.u-4all" elif newsgroup == "ab.amazing": newsgroup = "alt.binaries.amazing" elif newsgroup == "ab.astronomy": newsgroup = "alt.binaries.astronomy" elif newsgroup == "ab.nospam.cheer": newsgroup = "alt.binaries.nospam.cheerleaders" elif newsgroup == "ab.worms": newsgroup = "alt.binaries.worms" elif newsgroup == "abcores": newsgroup = "alt.binaries.cores" elif newsgroup == "abdvdclassics": newsgroup = "alt.binaries.dvd.classics" elif newsgroup == "abdvdf": newsgroup = "alt.binaries.dvd.french" elif newsgroup == "abdvds": newsgroup = "alt.binaries.dvds" elif newsgroup == "abmdfrance": newsgroup = "alt.binaries.movies.divx.france" elif newsgroup == "abmisc": newsgroup = "alt.binaries.misc" elif newsgroup == "abnl": newsgroup = "alt.binaries.nl" elif newsgroup == "abx": newsgroup = "alt.binaries.x" else: log.error(u"Unknown binnewz newsgroup: " + newsgroup) continue if newsgroup in blacklist_groups: log.error(u"Ignoring result, newsgroup is blacklisted: " + newsgroup) continue filename = cells[5].contents[0] m = re.search("^(.+)\s+{(.*)}$", name) qualityStr = "" if m: name = m.group(1) qualityStr = m.group(2) m = re.search("^(.+)\s+\[(.*)\]$", name) source = None if m: name = m.group(1) source = m.group(2) m = re.search("(.+)\(([0-9]{4})\)", name) year = "" if m: name = m.group(1) year = m.group(2) if int(year) > movieyear + 1 or int(year) < movieyear - 1: continue m = re.search("(.+)\((\d{2}/\d{2}/\d{4})\)", name) dateStr = "" if m: name = m.group(1) dateStr = m.group(2) year = dateStr[-5:].strip(")").strip("/") m = re.search("(.+)\s+S(\d{2})\s+E(\d{2})(.*)", name) if m: name = m.group(1) + " S" + m.group(2) + "E" + m.group(3) + m.group(4) m = re.search("(.+)\s+S(\d{2})\s+Ep(\d{2})(.*)", name) if m: name = m.group(1) + " S" + m.group(2) + "E" + m.group(3) + m.group(4) filenameLower = filename.lower() searchItems = [] if qualityStr == "": if source in ("Blu Ray-Rip", "HD DVD-Rip"): qualityStr = "brrip" elif source == "DVDRip": qualityStr = "dvdrip" elif source == "TS": qualityStr = "ts" elif source == "DVDSCR": qualityStr = "scr" elif source == "CAM": qualityStr = "cam" elif moviequality == "dvdr": qualityStr = "dvdr" if year == "": year = "1900" if len(searchItems) == 0 and qualityStr == str(moviequality): searchItems.append(filename) for searchItem in searchItems: resultno = 1 for downloader in nzbDownloaders: log.info( "Searching for download : " + name + ", search string = " + searchItem + " on " + downloader.__class__.__name__ ) try: binsearch_result = downloader.search(searchItem, minSize, newsgroup) if binsearch_result: new = {} def extra_check(item): return True qualitytag = "" if qualityStr.lower() in ["720p", "1080p"]: qualitytag = " hd x264 h264 " elif qualityStr.lower() in ["dvdrip"]: qualitytag = " dvd xvid " elif qualityStr.lower() in ["brrip"]: qualitytag = " hdrip " elif qualityStr.lower() in ["ts"]: qualitytag = " webrip " elif qualityStr.lower() in ["scr"]: qualitytag = "" elif qualityStr.lower() in ["dvdr"]: qualitytag = " pal video_ts " new["id"] = binsearch_result.nzbid new["name"] = ( name + detectedlang + qualityStr + qualitytag + downloader.__class__.__name__ ) new["url"] = binsearch_result.nzburl new["detail_url"] = binsearch_result.refererURL new["size"] = int( str(binsearch_result.sizeInMegs)[ : str(binsearch_result.sizeInMegs).find(".") ].replace("L", "") ) new["age"] = binsearch_result.age new["extra_check"] = extra_check results.append(new) resultno = resultno + 1 log.info("Found : " + searchItem + " on " + downloader.__class__.__name__) if resultno == 3: break except Exception, e: log.error( "Searching from " + downloader.__class__.__name__ + " failed : " + str(e) + traceback.format_exc() )
def _search(self, movie, quality, results): TitleStringReal = (getTitle(movie['info']) + ' ' + simplifyString(quality['identifier'])).replace('-',' ').replace(' ',' ').replace(' ',' ').replace(' ',' ').encode("utf-8") log.info('Title %s', TitleStringReal) URL = ((self.urls['search']) + TitleStringReal.replace('.', '-').replace(' ', '-') + '.html,trie-seeds-d').encode('utf-8') req = urllib2.Request(URL, headers={'User-Agent' : "Mozilla/5.0"}) log.info('opening url %s', URL) data = urllib2.urlopen(req,timeout=500) log.info('data retrieved') id = 1000 if data: try: html = BeautifulSoup(data) torrent_rows = html.findAll('tr') for result in torrent_rows: try: if not result.find('a'): continue title = result.find('a').get_text(strip=False) log.info('found title %s',title) testname = namer_check.correctName(title.lower(),movie) if testname == 0: log.info('%s not match %s',(title.lower(),movie['info']['titles'])) continue log.info('title %s match',title) tmp = result.find("a")['href'].split('/')[-1].replace('.html', '.torrent').strip() download_url = (self.urls['site'] + 'get_torrent/{0}'.format(tmp) + ".torrent") detail_url = (self.urls['site'] + 'torrent/{0}'.format(tmp)) log.debug('download_url %s',download_url) if not all([title, download_url]): continue seeders = int(result.find(class_="seed_ok").get_text(strip=True)) leechers = int(result.find_all('td')[3].get_text(strip=True)) size = result.find_all('td')[1].get_text(strip=True) def extra_check(item): return True size = size.lower() size = size.replace("go", "gb") size = size.replace("mo", "mb") size = size.replace("ko", "kb") size = size.replace(' ','') size = self.parseSize(str(size)) new = {} new['id'] = id new['name'] = title.strip() new['url'] = download_url new['detail_url'] = detail_url new['size'] = size new['seeders'] = seeders new['leechers'] = leechers new['extra_check'] = extra_check new['download'] = self.loginDownload results.append(new) log.info(results) id = id + 1 except StandardError, e: log.info('boum %s',e) continue except AttributeError: log.debug('No search results found.') else: log.debug('No search results found.')
def _search(self, movie, quality, results): searchStrings= self.getSearchParams(movie,quality) for searchString in searchStrings: searchUrl = self.urls['search']+'rdirect.php?type=search&'+searchString data = self.getHTMLData(searchUrl) if "bad key" in str(data).lower() : log.error(u"GKS key invalid, check your config") continue parsedXML = parseString(data) channel = parsedXML.getElementsByTagName('channel')[0] description = channel.getElementsByTagName('description')[0] #description_text = self.get_xml_text(description).lower() text = "" for child_node in description.childNodes: if child_node.nodeType in (Node.CDATA_SECTION_NODE, Node.TEXT_NODE): text += child_node.data description_text=text.strip().lower() if "user can't be found" in description_text: log.error(u"GKS invalid digest, check your config") continue elif "invalid hash" in description_text: log.error(u"GKS invalid hash, check your config") continue else : items = channel.getElementsByTagName('item') for item in items: text = "" for child_node in item.getElementsByTagName('title')[0].childNodes: if child_node.nodeType in (Node.CDATA_SECTION_NODE, Node.TEXT_NODE): text += child_node.data title=text.strip().lower() if "aucun resultat" in title.lower() : log.debug("No results found trying another if there is one") continue else : testname=namer_check.correctName(title,movie) if testname==0: continue text = "" for child_node in item.getElementsByTagName('link')[0].childNodes: if child_node.nodeType in (Node.CDATA_SECTION_NODE, Node.TEXT_NODE): text += child_node.data downloadURL=text.strip().lower() desc="" for child_node in item.getElementsByTagName('description')[0].childNodes: if child_node.nodeType in (Node.CDATA_SECTION_NODE, Node.TEXT_NODE): desc += child_node.data desc=desc.strip().lower() desc_values=desc.split(" | ") dict_attr={} for x in desc_values: x_values=x.split(" : ") dict_attr[x_values[0]]=x_values[1] date="" size="" leechers="" seeders="" if "ajoute le" in dict_attr: date=dict_attr["ajoute le"] if "taille" in dict_attr: size=dict_attr["taille"] if "seeders" in dict_attr: seeders=dict_attr["seeders"] if "leechers" in dict_attr: leechers=dict_attr["leechers"] def extra_check(item): return True new = {} new['id'] = title new['name'] = title.strip() new['url'] = downloadURL new['detail_url'] = searchUrl new['size'] = self.parseSize(size) new['age'] = self.ageToDays(date) new['seeders'] = tryInt(seeders) new['leechers'] = tryInt(leechers) new['extra_check'] = extra_check results.append(new)
def _search(self, movie, quality, results): #for title in movie['info']['titles']: # try: info = movie['info'] if (movie['originalName'] == True): info = movie TitleStringReal = (getTitle(info) + ' ' + simplifyString(quality['identifier'])).replace( '-', ' ').replace(' ', ' ').replace( ' ', ' ').replace(' ', ' ').encode("utf-8") URL = ((self.urls['search']) + TitleStringReal.replace('.', '-').replace(' ', '-') + '.html,trie-seeds-d').encode('utf-8') req = urllib2.Request(URL, headers={'User-Agent': "Mozilla/5.0"}) log.info('opening url %s', URL) data = urllib2.urlopen(req, timeout=500) id = 1000 if data: try: html = BeautifulSoup(data) torrent_rows = html.findAll('tr') for result in torrent_rows: try: if not result.find('a'): continue title = result.find('a').get_text(strip=False) log.info('found title %s', title) testname = namer_check.correctName( title.lower(), movie) if testname == 0: continue tmp = result.find("a")['href'].split('/')[-1].replace( '.html', '.torrent').strip() download_url = (self.urls['site'] + 'get_torrent/{0}'.format(tmp) + ".torrent") detail_url = (self.urls['site'] + 'torrent/{0}'.format(tmp)) if not all([title, download_url]): continue seeders = int( result.find(class_="seed_ok").get_text(strip=True)) leechers = int( result.find_all('td')[3].get_text(strip=True)) size = result.find_all('td')[1].get_text(strip=True) size = size.lower() size = size.replace("go", "gb") size = size.replace("mo", "mb") size = size.replace("ko", "kb") size = size.replace(' ', '') size = self.parseSize(str(size)) def extra_check(item): return True new = {} new['id'] = id new['name'] = title.strip() new['url'] = download_url new['detail_url'] = detail_url new['size'] = size new['seeders'] = seeders new['leechers'] = leechers new['extra_check'] = extra_check new['download'] = self.loginDownload id = id + 1 results.append(new) except StandardError, e: log.info('boum %s', e) except AttributeError: log.debug('No search results found.') else: log.debug('No search results found.')
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return TitleStringReal = ( (getTitle(movie["info"]) + " " + simplifyString(quality["identifier"])) .replace("-", " ") .replace(" ", " ") .replace(" ", " ") .replace(" ", " ") .encode("utf8") ) URL = (self.urls["search"]).encode("UTF8") URL = unicodedata.normalize("NFD", unicode(URL, "utf8", "replace")) URL = URL.encode("ascii", "ignore") URL = urllib2.quote(URL.encode("utf8"), ":/?=") values = {"champ_recherche": TitleStringReal} data_tmp = urllib.urlencode(values) req = urllib2.Request(URL, data_tmp, headers={"User-Agent": "Mozilla/5.0"}) data = urllib2.urlopen(req) id = 1000 if data: try: html = BeautifulSoup(data) lin = 0 erlin = 0 resultdiv = [] while erlin == 0: try: classlin = "ligne" + str(lin) resultlin = html.findAll(attrs={"class": [classlin]}) if resultlin: for ele in resultlin: resultdiv.append(ele) lin += 1 else: erlin = 1 except: erlin = 1 for result in resultdiv: try: new = {} name = result.findAll(attrs={"class": ["titre"]})[0].text testname = namer_check.correctName(name, movie) if testname == 0: continue detail_url = result.find("a")["href"] tmp = detail_url.split("/")[-1].replace(".html", ".torrent") url_download = "http://www.cpasbien.cm/telechargement/%s" % tmp size = result.findAll(attrs={"class": ["poid"]})[0].text.replace(u"\xa0", u" ") seeder = result.findAll(attrs={"class": ["seed_ok"]})[0].text leecher = result.findAll(attrs={"class": ["down"]})[0].text age = "1" size = size.lower() size = size.replace("go", "gb") size = size.replace("mo", "mb") size = size.replace("ko", "kb") log.debug("size %s", size) log.debug("seeder %s", seeder) log.debug("leecher %s", leecher) verify = getTitle(movie["info"]).split(" ") add = 1 for verify_unit in verify: if name.lower().find(verify_unit.lower()) == -1: add = 0 def extra_check(item): return True if add == 1: new["id"] = id new["name"] = name.strip() new["url"] = url_download new["detail_url"] = detail_url new["size"] = self.parseSize(str(size)) new["age"] = self.ageToDays(str(age)) new["seeders"] = tryInt(seeder) new["leechers"] = tryInt(leecher) new["extra_check"] = extra_check new["download"] = self.loginDownload # new['score'] = fireEvent('score.calculate', new, movie, single = True) # log.error('score') # log.error(new['score']) results.append(new) id = id + 1 except: log.error("Failed parsing cPASbien: %s", traceback.format_exc()) except AttributeError: log.debug("No search results found.") else: log.debug("No search results found.")
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return searchStrings= self.getSearchParams(movie,quality) lastsearch=0 for searchString in searchStrings: actualtime=int(time.time()) if actualtime-lastsearch<10: timetosleep= 10-(actualtime-lastsearch) time.sleep(timetosleep) URL = self.urls['search']+searchString r = self.opener.open(URL) soup = BeautifulSoup( r, "html.parser" ) if soup.find('table', attrs = {'class':'results'}): resultdiv = soup.find('table', attrs = {'class':'results'}).find('tbody') else: continue if resultdiv: try: for result in resultdiv.findAll('tr'): try: categorie = result.findAll('td')[0].findAll('a')[0]['href'][result.findAll('td')[0].findAll('a')[0]['href'].find('='):] insert = 0 if categorie == '=631': insert = 1 if categorie == '=455': insert = 1 if categorie == '=634': insert = 1 if insert == 1 : new = {} idt = result.findAll('td')[2].findAll('a')[0]['href'][1:].replace('torrents/nfo/?id=','') name = result.findAll('td')[1].findAll('a')[0]['title'] testname=namer_check.correctName(name,movie) if testname==0: continue url = ('http://www.t411.me/torrents/download/?id=%s' % idt) detail_url = ('http://www.t411.me/torrents/?id=%s' % idt) leecher = result.findAll('td')[8].text size = result.findAll('td')[5].text age = result.findAll('td')[4].text seeder = result.findAll('td')[7].text def extra_check(item): return True new['id'] = idt new['name'] = name + ' french' new['url'] = url new['detail_url'] = detail_url new['size'] = self.parseSize(str(size)) new['age'] = self.ageToDays(str(age)) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.download results.append(new) except: log.error('Failed parsing T411: %s', traceback.format_exc()) except AttributeError: log.debug('No search results found.') else: log.debug('No search results found.')
def _search(self, movie, quality, results): nzbDownloaders = [NZBClub(), BinSearch(), NZBIndex()] MovieTitles = movie['info']['titles'] moviequality = simplifyString(quality['identifier']) movieyear = movie['info']['year'] if quality['custom']['3d']==1: threeD= True else: threeD=False if moviequality in ("720p","1080p","bd50"): cat1='39' cat2='49' minSize = 2000 elif moviequality in ("dvdr"): cat1='23' cat2='48' minSize = 3000 else: cat1='6' cat2='27' minSize = 500 for MovieTitle in MovieTitles: try: TitleStringReal = str(MovieTitle.encode("latin-1").replace('-',' ')) except: continue if threeD: TitleStringReal = TitleStringReal + ' 3d' data = 'chkInit=1&edTitre='+TitleStringReal+'&chkTitre=on&chkFichier=on&chkCat=on&cats%5B%5D='+cat1+'&cats%5B%5D='+cat2+'&edAge=&edYear=' try: soup = BeautifulSoup( urllib2.urlopen(self.urls['search'], data) ) except Exception, e: log.error(u"Error trying to load BinNewz response: "+e) return [] tables = soup.findAll("table", id="tabliste") for table in tables: rows = table.findAll("tr") for row in rows: cells = row.select("> td") if (len(cells) < 11): continue name = cells[2].text.strip() testname=namer_check.correctName(name,movie) if testname==0: continue language = cells[3].find("img").get("src") if not "_fr" in language and not "_frq" in language: continue detectedlang='' if "_fr" in language: detectedlang=' truefrench ' else: detectedlang=' french ' # blacklist_groups = [ "alt.binaries.multimedia" ] blacklist_groups = [] newgroupLink = cells[4].find("a") newsgroup = None if newgroupLink.contents: newsgroup = newgroupLink.contents[0] if newsgroup == "abmulti": newsgroup = "alt.binaries.multimedia" elif newsgroup == "ab.moovee": newsgroup = "alt.binaries.moovee" elif newsgroup == "abtvseries": newsgroup = "alt.binaries.tvseries" elif newsgroup == "abtv": newsgroup = "alt.binaries.tv" elif newsgroup == "a.b.teevee": newsgroup = "alt.binaries.teevee" elif newsgroup == "abstvdivxf": newsgroup = "alt.binaries.series.tv.divx.french" elif newsgroup == "abhdtvx264fr": newsgroup = "alt.binaries.hdtv.x264.french" elif newsgroup == "abmom": newsgroup = "alt.binaries.mom" elif newsgroup == "abhdtv": newsgroup = "alt.binaries.hdtv" elif newsgroup == "abboneless": newsgroup = "alt.binaries.boneless" elif newsgroup == "abhdtvf": newsgroup = "alt.binaries.hdtv.french" elif newsgroup == "abhdtvx264": newsgroup = "alt.binaries.hdtv.x264" elif newsgroup == "absuperman": newsgroup = "alt.binaries.superman" elif newsgroup == "abechangeweb": newsgroup = "alt.binaries.echange-web" elif newsgroup == "abmdfvost": newsgroup = "alt.binaries.movies.divx.french.vost" elif newsgroup == "abdvdr": newsgroup = "alt.binaries.dvdr" elif newsgroup == "abmzeromov": newsgroup = "alt.binaries.movies.zeromovies" elif newsgroup == "abcfaf": newsgroup = "alt.binaries.cartoons.french.animes-fansub" elif newsgroup == "abcfrench": newsgroup = "alt.binaries.cartoons.french" elif newsgroup == "abgougouland": newsgroup = "alt.binaries.gougouland" elif newsgroup == "abroger": newsgroup = "alt.binaries.roger" elif newsgroup == "abtatu": newsgroup = "alt.binaries.tatu" elif newsgroup =="abstvf": newsgroup = "alt.binaries.series.tv.french" elif newsgroup =="abmdfreposts": newsgroup="alt.binaries.movies.divx.french.reposts" elif newsgroup =="abmdf": newsgroup="alt.binaries.movies.french" elif newsgroup =="abhdtvfrepost": newsgroup="alt.binaries.hdtv.french.repost" elif newsgroup == "abmmkv": newsgroup = "alt.binaries.movies.mkv" elif newsgroup == "abf-tv": newsgroup = "alt.binaries.french-tv" elif newsgroup == "abmdfo": newsgroup = "alt.binaries.movies.divx.french.old" elif newsgroup == "abmf": newsgroup = "alt.binaries.movies.french" elif newsgroup == "ab.movies": newsgroup = "alt.binaries.movies" elif newsgroup == "a.b.french": newsgroup = "alt.binaries.french" elif newsgroup == "a.b.3d": newsgroup = "alt.binaries.3d" elif newsgroup == "ab.dvdrip": newsgroup = "alt.binaries.dvdrip" elif newsgroup == "ab.welovelori": newsgroup = "alt.binaries.welovelori" elif newsgroup == "abblu-ray": newsgroup = "alt.binaries.blu-ray" elif newsgroup == "ab.bloaf": newsgroup = "alt.binaries.bloaf" elif newsgroup == "ab.hdtv.german": newsgroup = "alt.binaries.hdtv.german" elif newsgroup == "abmd": newsgroup = "alt.binaries.movies.divx" elif newsgroup == "ab.ath": newsgroup = "alt.binaries.ath" elif newsgroup == "a.b.town": newsgroup = "alt.binaries.town" elif newsgroup == "a.b.u-4all": newsgroup = "alt.binaries.u-4all" elif newsgroup == "ab.amazing": newsgroup = "alt.binaries.amazing" elif newsgroup == "ab.astronomy": newsgroup = "alt.binaries.astronomy" elif newsgroup == "ab.nospam.cheer": newsgroup = "alt.binaries.nospam.cheerleaders" elif newsgroup == "ab.worms": newsgroup = "alt.binaries.worms" elif newsgroup == "abcores": newsgroup = "alt.binaries.cores" elif newsgroup == "abdvdclassics": newsgroup = "alt.binaries.dvd.classics" elif newsgroup == "abdvdf": newsgroup = "alt.binaries.dvd.french" elif newsgroup == "abdvds": newsgroup = "alt.binaries.dvds" elif newsgroup == "abmdfrance": newsgroup = "alt.binaries.movies.divx.france" elif newsgroup == "abmisc": newsgroup = "alt.binaries.misc" elif newsgroup == "abnl": newsgroup = "alt.binaries.nl" elif newsgroup == "abx": newsgroup = "alt.binaries.x" else: log.error(u"Unknown binnewz newsgroup: " + newsgroup) continue if newsgroup in blacklist_groups: log.error(u"Ignoring result, newsgroup is blacklisted: " + newsgroup) continue filename = cells[5].contents[0] m = re.search("^(.+)\s+{(.*)}$", name) qualityStr = "" if m: name = m.group(1) qualityStr = m.group(2) m = re.search("^(.+)\s+\[(.*)\]$", name) source = None if m: name = m.group(1) source = m.group(2) m = re.search("(.+)\(([0-9]{4})\)", name) year = "" if m: name = m.group(1) year = m.group(2) if int(year) > movieyear + 1 or int(year) < movieyear - 1: continue m = re.search("(.+)\((\d{2}/\d{2}/\d{4})\)", name) dateStr = "" if m: name = m.group(1) dateStr = m.group(2) year = dateStr[-5:].strip(")").strip("/") m = re.search("(.+)\s+S(\d{2})\s+E(\d{2})(.*)", name) if m: name = m.group(1) + " S" + m.group(2) + "E" + m.group(3) + m.group(4) m = re.search("(.+)\s+S(\d{2})\s+Ep(\d{2})(.*)", name) if m: name = m.group(1) + " S" + m.group(2) + "E" + m.group(3) + m.group(4) filenameLower = filename.lower() searchItems = [] if qualityStr=="": if source in ("Blu Ray-Rip", "HD DVD-Rip"): qualityStr="brrip" elif source =="DVDRip": qualityStr="dvdrip" elif source == "TS": qualityStr ="ts" elif source == "DVDSCR": qualityStr ="scr" elif source == "CAM": qualityStr ="cam" elif moviequality == "dvdr": qualityStr ="dvdr" if year =='': year = '1900' if len(searchItems) == 0 and qualityStr == str(moviequality): searchItems.append( filename ) for searchItem in searchItems: resultno=1 for downloader in nzbDownloaders: log.info("Searching for download : " + name + ", search string = "+ searchItem + " on " + downloader.__class__.__name__) try: binsearch_result = downloader.search(searchItem, minSize, newsgroup ) if binsearch_result: new={} def extra_check(item): return True qualitytag='' if qualityStr.lower() in ['720p','1080p']: qualitytag=' hd x264 h264 ' elif qualityStr.lower() in ['dvdrip']: qualitytag=' dvd xvid ' elif qualityStr.lower() in ['brrip']: qualitytag=' hdrip ' elif qualityStr.lower() in ['ts']: qualitytag=' webrip ' elif qualityStr.lower() in ['scr']: qualitytag='' elif qualityStr.lower() in ['dvdr']: qualitytag=' pal video_ts ' new['id'] = binsearch_result.nzbid new['name'] = name + detectedlang + qualityStr + qualitytag + downloader.__class__.__name__ new['url'] = binsearch_result.nzburl new['detail_url'] = binsearch_result.refererURL new['size'] = binsearch_result.sizeInMegs new['age'] = binsearch_result.age new['extra_check'] = extra_check results.append(new) resultno=resultno+1 log.info("Found : " + searchItem + " on " + downloader.__class__.__name__) if resultno==3: break except Exception, e: log.error("Searching from " + downloader.__class__.__name__ + " failed : " + str(e) + traceback.format_exc())
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return info = movie['info'] if (movie['originalName'] == True): info = movie TitleStringReal = (getTitle(info) + ' ' + simplifyString(quality['identifier'] )).replace('-',' ').replace(' ',' ').replace(' ',' ').replace(' ',' ').encode("utf8") URL = (self.urls['search']).encode('UTF8') URL=unicodedata.normalize('NFD',unicode(URL,"utf8","replace")) URL=URL.encode('ascii','ignore') URL = urllib2.quote(URL.encode('utf8'), ":/?=") URL = URL + TitleStringReal values = { } URLTST = (self.urls['test']).encode('UTF8') data_tmp = urllib.urlencode(values) log.info('opening url %s', URL) req = urllib2.Request(URL, data_tmp, headers={'User-Agent' : "Mozilla/5.0"} ) data = urllib2.urlopen(req) id = 1000 if data: try: html = BeautifulSoup(data) erlin=0 resultdiv=[] while erlin==0: try: resultContent = html.findAll(attrs={'class': ["listing-torrent"]})[0] if resultContent: resultlin = resultContent.findAll(attrs={'class': ['table-hover']})[0].find('tbody') if resultlin: trList= resultlin.findAll("tr"); for tr in trList: resultdiv.append(tr) erlin=1 except: erlin=1 nbrResult = 0 for result in resultdiv: try: new = {} firstTd = result.findAll("td")[0] nothing = firstTd.findAll("center") if nothing: continue name = firstTd.findAll("a")[0]['title']; testname = namer_check.correctName(name,movie) if testname == 0 and nbrResult < 5: log.info('found title %s', testname) values_sec = {} url_sec = result.findAll("a")[0]['href']; req_sec = urllib2.Request(URLTST+url_sec, values_sec, headers={'User-Agent': "Mozilla/5.0"}) data_sec = urllib2.urlopen(req_sec) if data_sec: html_sec = BeautifulSoup(data_sec) classlin_sec = 'torrentsdesc' resultlin_sec = html_sec.findAll(attrs={'id': [classlin_sec]})[0] name = resultlin_sec.find("div").text name = name.replace(".", " ") testname = namer_check.correctName(name, movie) if testname == 0: continue nbrResult += 1 values_sec = {} detail_url = result.findAll("a")[0]['href']; req_sec = urllib2.Request(URLTST+detail_url, values_sec, headers={'User-Agent': "Mozilla/5.0"}) data_sec = urllib2.urlopen(req_sec) html_sec = BeautifulSoup(data_sec) classlin_sec = 'download' resultlin_sec = html_sec.findAll(attrs={'class': [classlin_sec]})[0] url_download = resultlin_sec.findAll("a")[0]['href'] size = result.findAll("td")[1].text seeder = result.findAll("td")[2].text leecher = result.findAll("td")[3].text age = '1' verify = getTitle(movie['info']).split(' ') add = 1 for verify_unit in verify: if (name.lower().find(verify_unit.lower()) == -1) : add = 0 def extra_check(item): return True if add == 1: new['id'] = id new['name'] = name.strip() + ' french' new['url'] = url_download new['detail_url'] = detail_url new['size'] = self.parseSize(size) new['age'] = 10 new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.loginDownload #new['score'] = fireEvent('score.calculate', new, movie, single = True) #log.error('score') #log.error(new['score']) results.append(new) id = id+1 except: log.error('Failed parsing zetorrents: %s', traceback.format_exc()) except AttributeError: log.debug('No search results found.') else: log.debug('No search results found.')
def _search(self, movie, quality, results): # Cookie login if not self.login_opener and not self.login(): return TitleStringReal = (getTitle(movie['library']) + ' ' + simplifyString(quality['identifier'] )).replace('-',' ').replace(' ',' ').replace(' ',' ').replace(' ',' ').encode("utf8") URL = (self.urls['search']).encode('UTF8') URL=unicodedata.normalize('NFD',unicode(URL,"utf8","replace")) URL=URL.encode('ascii','ignore') URL = urllib2.quote(URL.encode('utf8'), ":/?=") values = { 'champ_recherche' : TitleStringReal } data_tmp = urllib.urlencode(values) req = urllib2.Request(URL, data_tmp ) data = urllib2.urlopen(req ) id = 1000 if data: cat_ids = self.getCatId(quality['identifier']) table_order = ['name', 'size', None, 'age', 'seeds', 'leechers'] try: html = BeautifulSoup(data) resultdiv = html.find('div', attrs = {'id':'recherche'}).find('table').find('tbody') for result in resultdiv.find_all('tr', recursive = False): try: new = {} #id = result.find_all('td')[2].find_all('a')[0]['href'][1:].replace('torrents/nfo/?id=','') name = result.find_all('td')[0].find_all('a')[0].text testname=namer_check.correctName(name,movie) if testname==0: continue detail_url = result.find_all('td')[0].find_all('a')[0]['href'] #on scrapp la page detail urldetail = detail_url.encode('UTF8') urldetail=unicodedata.normalize('NFD',unicode(urldetail,"utf8","replace")) urldetail=urldetail.encode('ascii','ignore') urldetail = urllib2.quote(urldetail.encode('utf8'), ":/?=") req = urllib2.Request(urldetail ) # POST request doesn't not work data_detail = urllib2.urlopen(req) url_download = "" if data_detail: html_detail = BeautifulSoup(data_detail) url_download = html_detail.find_all('div', attrs = {'class':'download-torrent'})[0].find_all('a')[0]['href'] else: tmp = result.find_all('td')[0].find_all('a')[0]['href'] tmp = tmp.split('/')[6].replace('.html','.torrent') url_download = ('http://www.cpasbien.me/_torrents/%s' % tmp) size = result.find_all('td')[1].text seeder = result.find_all('td')[2].find_all('span')[0].text leecher = result.find_all('td')[3].text age = '0' verify = getTitle(movie['library']).split(' ') add = 1 for verify_unit in verify: if (name.find(verify_unit) == -1) : add = 0 def extra_check(item): return True if add == 1: new['id'] = id new['name'] = name.strip() new['url'] = url_download new['detail_url'] = detail_url new['size'] = self.parseSize(size) new['age'] = self.ageToDays(age) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.loginDownload #new['score'] = fireEvent('score.calculate', new, movie, single = True) #log.error('score') #log.error(new['score']) results.append(new) id = id+1 except: log.error('Failed parsing cPASbien: %s', traceback.format_exc()) except AttributeError: log.debug('No search results found.') else: log.debug('No search results found.')
def _search(self, movie, quality, results): # Cookie login if not self.last_login_check and not self.login(): return TitleStringReal = (getTitle(movie['info']) + ' ' + simplifyString(quality['identifier'] )).replace('-',' ').replace(' ',' ').replace(' ',' ').replace(' ',' ').encode("utf8") URL = (self.urls['search']).encode('UTF8') URL=unicodedata.normalize('NFD',unicode(URL,"utf8","replace")) URL=URL.encode('ascii','ignore') URL = urllib2.quote(URL.encode('utf8'), ":/?=") values = { 'champ_recherche' : TitleStringReal } data_tmp = urllib.urlencode(values) req = urllib2.Request(URL, data_tmp, headers={'User-Agent' : "Mozilla/5.0"} ) data = urllib2.urlopen(req ) id = 1000 if data: try: html = BeautifulSoup(data) lin=0 erlin=0 resultdiv=[] while erlin==0: try: classlin='ligne'+str(lin) resultlin=html.findAll(attrs = {'class' : [classlin]}) if resultlin: for ele in resultlin: resultdiv.append(ele) lin+=1 else: erlin=1 except: erlin=1 for result in resultdiv: try: new = {} name = result.findAll(attrs = {'class' : ["titre"]})[0].text testname=namer_check.correctName(name,movie) if testname==0: continue detail_url = result.find("a")['href'] tmp = detail_url.split('/')[-1].replace('.html','.torrent') url_download = ('http://www.cpasbien.cm/telechargement/%s' % tmp) size = result.findAll(attrs = {'class' : ["poid"]})[0].text seeder = result.findAll(attrs = {'class' : ["seed_ok"]})[0].text leecher = result.findAll(attrs = {'class' : ["down"]})[0].text age = '1' verify = getTitle(movie['info']).split(' ') add = 1 for verify_unit in verify: if (name.lower().find(verify_unit.lower()) == -1) : add = 0 def extra_check(item): return True if add == 1: new['id'] = id new['name'] = name.strip() new['url'] = url_download new['detail_url'] = detail_url new['size'] = self.parseSize(size) new['age'] = self.ageToDays(age) new['seeders'] = tryInt(seeder) new['leechers'] = tryInt(leecher) new['extra_check'] = extra_check new['download'] = self.loginDownload #new['score'] = fireEvent('score.calculate', new, movie, single = True) #log.error('score') #log.error(new['score']) results.append(new) id = id+1 except: log.error('Failed parsing cPASbien: %s', traceback.format_exc()) except AttributeError: log.debug('No search results found.') else: log.debug('No search results found.')
def _search(self, movie, quality, results): nzbDownloaders = [NZBClub(), BinSearch(), NZBIndex()] MovieTitles = movie['info']['titles'] moviequality = simplifyString(quality['identifier']) movieyear = movie['info']['year'] if quality['custom']['3d']==1: threeD= True else: threeD=False if moviequality in ("720p","1080p","bd50","brrip"): cat1='39' cat2='49' minSize = 2000 elif moviequality in ("dvdr"): cat1='23' cat2='48' minSize = 3000 else: cat1='6' cat2='27' minSize = 500 # Choose language search preference langQuery = "" if self.conf('lang_french'): log.info("VF detected") langQuery = "2,6" if self.conf('lang_quebec'): log.info("VFQ detected") if langQuery: langQuery = langQuery + "," langQuery = langQuery + "5,8,9,10" for MovieTitle in MovieTitles: try: TitleStringReal = str(MovieTitle.encode("latin-1").replace('-',' ')) except: continue if threeD: TitleStringReal = TitleStringReal + ' 3d' data = {'chkInit': '1', 'edTitre': TitleStringReal, 'chkTitre': 'on', 'chkFichier': 'on', 'chkCat': 'on', 'cats[]': [cat1, cat2], 'lg[]': splitString(langQuery, clean = False), 'edAge': '', 'edYear': ''} try: soup = BeautifulSoup( urllib2.urlopen(self.urls['search'], urllib.urlencode(data, True))) except Exception, e: log.error(u"Error trying to load BinNewz response: "+e) return [] tables = soup.findAll("table", id="tabliste") if len(tables) == 0: log.info("No result") else: log.info("Results found in " + str(len(tables)) + " categories") for table in tables: rows = table.findAll("tr") for row in rows: cells = row.select("> td") if (len(cells) < 11): continue name = cells[2].text.strip() testname=namer_check.correctName(name,movie) if testname==0: continue # Old way to choose language language = cells[3].find("img").get("src") #if not "_fr" in language and not "_frq" in language: # continue detectedlang='' if "_fr" in language: detectedlang=' truefrench ' else: detectedlang=' french ' # blacklist_groups = [ "alt.binaries.multimedia" ] blacklist_groups = [] newgroupLink = cells[4].find("a") newsgroup = None if newgroupLink.contents: newsgroup = newgroupLink.contents[0] if newsgroup == "abmulti": newsgroup = "alt.binaries.multimedia" elif newsgroup == "ab.moovee": newsgroup = "alt.binaries.moovee" elif newsgroup == "abtvseries": newsgroup = "alt.binaries.tvseries" elif newsgroup == "abtv": newsgroup = "alt.binaries.tv" elif newsgroup == "a.b.teevee": newsgroup = "alt.binaries.teevee" elif newsgroup == "abstvdivxf": newsgroup = "alt.binaries.series.tv.divx.french" elif newsgroup == "abhdtvx264fr": newsgroup = "alt.binaries.hdtv.x264.french" elif newsgroup == "abmom": newsgroup = "alt.binaries.mom" elif newsgroup == "abhdtv": newsgroup = "alt.binaries.hdtv" elif newsgroup == "abboneless": newsgroup = "alt.binaries.boneless" elif newsgroup == "abhdtvf": newsgroup = "alt.binaries.hdtv.french" elif newsgroup == "abhdtvx264": newsgroup = "alt.binaries.hdtv.x264" elif newsgroup == "absuperman": newsgroup = "alt.binaries.superman" elif newsgroup == "abechangeweb": newsgroup = "alt.binaries.echange-web" elif newsgroup == "abmdfvost": newsgroup = "alt.binaries.movies.divx.french.vost" elif newsgroup == "abdvdr": newsgroup = "alt.binaries.dvdr" elif newsgroup == "abmzeromov": newsgroup = "alt.binaries.movies.zeromovies" elif newsgroup == "abcfaf": newsgroup = "alt.binaries.cartoons.french.animes-fansub" elif newsgroup == "abcfrench": newsgroup = "alt.binaries.cartoons.french" elif newsgroup == "abgougouland": newsgroup = "alt.binaries.gougouland" elif newsgroup == "abroger": newsgroup = "alt.binaries.roger" elif newsgroup == "abtatu": newsgroup = "alt.binaries.tatu" elif newsgroup =="abstvf": newsgroup = "alt.binaries.series.tv.french" elif newsgroup =="abmdfreposts": newsgroup="alt.binaries.movies.divx.french.reposts" elif newsgroup =="abmdf": newsgroup="alt.binaries.movies.french" elif newsgroup =="abhdtvfrepost": newsgroup="alt.binaries.hdtv.french.repost" elif newsgroup == "abmmkv": newsgroup = "alt.binaries.movies.mkv" elif newsgroup == "abf-tv": newsgroup = "alt.binaries.french-tv" elif newsgroup == "abmdfo": newsgroup = "alt.binaries.movies.divx.french.old" elif newsgroup == "abmf": newsgroup = "alt.binaries.movies.french" elif newsgroup == "ab.movies": newsgroup = "alt.binaries.movies" elif newsgroup == "a.b.french": newsgroup = "alt.binaries.french" elif newsgroup == "a.b.3d": newsgroup = "alt.binaries.3d" elif newsgroup == "ab.dvdrip": newsgroup = "alt.binaries.dvdrip" elif newsgroup == "ab.welovelori": newsgroup = "alt.binaries.welovelori" elif newsgroup == "abblu-ray": newsgroup = "alt.binaries.blu-ray" elif newsgroup == "ab.bloaf": newsgroup = "alt.binaries.bloaf" elif newsgroup == "ab.hdtv.german": newsgroup = "alt.binaries.hdtv.german" elif newsgroup == "abmd": newsgroup = "alt.binaries.movies.divx" elif newsgroup == "ab.ath": newsgroup = "alt.binaries.ath" elif newsgroup == "a.b.town": newsgroup = "alt.binaries.town" elif newsgroup == "a.b.u-4all": newsgroup = "alt.binaries.u-4all" elif newsgroup == "ab.amazing": newsgroup = "alt.binaries.amazing" elif newsgroup == "ab.astronomy": newsgroup = "alt.binaries.astronomy" elif newsgroup == "ab.nospam.cheer": newsgroup = "alt.binaries.nospam.cheerleaders" elif newsgroup == "ab.worms": newsgroup = "alt.binaries.worms" elif newsgroup == "abcores": newsgroup = "alt.binaries.cores" elif newsgroup == "abdvdclassics": newsgroup = "alt.binaries.dvd.classics" elif newsgroup == "abdvdf": newsgroup = "alt.binaries.dvd.french" elif newsgroup == "abdvds": newsgroup = "alt.binaries.dvds" elif newsgroup == "abmdfrance": newsgroup = "alt.binaries.movies.divx.france" elif newsgroup == "abmisc": newsgroup = "alt.binaries.misc" elif newsgroup == "abnl": newsgroup = "alt.binaries.nl" elif newsgroup == "abx": newsgroup = "alt.binaries.x" else: log.error(u"Unknown binnewz newsgroup: " + newsgroup) continue if newsgroup in blacklist_groups: log.error(u"Ignoring result, newsgroup is blacklisted: " + newsgroup) continue filename = cells[5].contents[0] m = re.search("^(.+)\s+{(.*)}$", name) qualityStr = "" if m: name = m.group(1) qualityStr = m.group(2) m = re.search("^(.+)\s+\[(.*)\]$", name) source = None if m: name = m.group(1) source = m.group(2) m = re.search("(.+)\(([0-9]{4})\)", name) year = "" if m: name = m.group(1) year = m.group(2) if int(year) > movieyear + 1 or int(year) < movieyear - 1: continue m = re.search("(.+)\((\d{2}/\d{2}/\d{4})\)", name) dateStr = "" if m: name = m.group(1) dateStr = m.group(2) year = dateStr[-5:].strip(")").strip("/") m = re.search("(.+)\s+S(\d{2})\s+E(\d{2})(.*)", name) if m: name = m.group(1) + " S" + m.group(2) + "E" + m.group(3) + m.group(4) m = re.search("(.+)\s+S(\d{2})\s+Ep(\d{2})(.*)", name) if m: name = m.group(1) + " S" + m.group(2) + "E" + m.group(3) + m.group(4) filenameLower = filename.lower() searchItems = [] if qualityStr=="": if source in ("Blu Ray-Rip", "HD DVD-Rip"): qualityStr="brrip" elif source =="DVDRip": qualityStr="dvdrip" elif source == "TS": qualityStr ="ts" elif source == "DVDSCR": qualityStr ="scr" elif source == "CAM": qualityStr ="cam" elif moviequality == "dvdr": qualityStr ="dvdr" if year =='': year = '1900' if len(searchItems) == 0 and qualityStr == str(moviequality): searchItems.append( filename ) for searchItem in searchItems: resultno=1 for downloader in nzbDownloaders: log.info("Searching for download : " + name + ", search string = "+ searchItem + " on " + downloader.__class__.__name__) try: binsearch_result = downloader.search(searchItem, minSize, newsgroup ) if binsearch_result: new={} def extra_check(item): return True qualitytag='' if qualityStr.lower() in ['720p','1080p']: qualitytag=' hd x264 h264 ' elif qualityStr.lower() in ['dvdrip']: qualitytag=' dvd xvid ' elif qualityStr.lower() in ['brrip']: qualitytag=' hdrip ' elif qualityStr.lower() in ['ts']: qualitytag=' webrip ' elif qualityStr.lower() in ['scr']: qualitytag='' elif qualityStr.lower() in ['dvdr']: qualitytag=' pal video_ts ' new['id'] = binsearch_result.nzbid new['name'] = name + detectedlang + qualityStr + qualitytag + downloader.__class__.__name__ new['url'] = binsearch_result.nzburl new['detail_url'] = binsearch_result.refererURL new['size'] = binsearch_result.sizeInMegs new['age'] = binsearch_result.age new['extra_check'] = extra_check results.append(new) resultno=resultno+1 log.info("Found : " + searchItem + " on " + downloader.__class__.__name__) if resultno==3: break except Exception, e: log.error("Searching from " + downloader.__class__.__name__ + " failed : " + str(e) + traceback.format_exc())