def checkForUpdateWindows(self):
try:
data = urllib2.urlopen(self.downloads, timeout = self.timeout).read()
except (IOError, URLError):
log.error('Failed to open %s.' % self.downloads)
return False
try:
html = BeautifulSoup(data)
results = html.findAll('a', attrs = {'href':re.compile('/downloads/')})
for link in results:
if 'windows' in str(link.parent).lower():
downloadUrl = 'http://github.com' + link.get('href').replace(' ', '%20')
break
if 'r' + str(version.windows) in downloadUrl:
return False
return downloadUrl
except AttributeError:
log.debug('Nothing found.')
return False
def findViaAlternative(self, movie):
results = {'480p':[], '720p':[], '1080p':[]}
arguments = urlencode({
's':movie
})
url = "%s?%s" % (self.backupUrl, arguments)
log.info('Searching %s' % url)
try:
data = urllib2.urlopen(url, timeout = self.timeout).read()
except (IOError, URLError):
log.error('Failed to open %s.' % url)
return results
try:
tables = SoupStrainer('div')
html = BeautifulSoup(data, parseOnlyThese = tables)
resultTable = html.findAll('h2', text = re.compile(movie))
for h2 in resultTable:
if 'trailer' in h2.lower():
parent = h2.parent.parent.parent
trailerLinks = parent.findAll('a', text = re.compile('480p|720p|1080p'))
try:
for trailer in trailerLinks:
results[trailer].insert(0, trailer.parent['href'])
except:
pass
except AttributeError:
log.debug('No trailers found in via alternative.')
return results
def checkForUpdateWindows(self):
try:
data = urllib2.urlopen(self.downloads, timeout = self.timeout).read()
except (IOError, URLError):
log.error('Failed to open %s.' % self.downloads)
return False
try:
tables = SoupStrainer('table')
html = BeautifulSoup(data, parseOnlyThese = tables)
resultTable = html.find('table', attrs = {'id':'s3_downloads'})
latestUrl = 'http://github.com' + resultTable.find('a')['href'].replace(' ', '%20')
try:
latest = urllib2.urlopen(latestUrl, timeout = self.timeout)
except (IOError, URLError):
log.error('Failed to open %s.' % latestUrl)
return False
downloadUrl = latest.geturl()
if 'r' + str(version.windows) in downloadUrl:
return False
return downloadUrl
except AttributeError:
log.debug('Nothing found.')
return False
def getItems(self, data):
results = []
soup = BeautifulSoup(data)
table = soup.find("table", {"class": "filmSubtitleList"})
try:
for tr in table.findAll("tr"):
item = {}
for td in tr.findAll('td'):
if td.a:
spans = td.a.findAll('span')
if len(spans) == 2:
item['id'] = int(spans[1].get('id').replace(
'r', ''))
item['name'] = str(spans[1].contents[0]).strip()
item['rating'] = int(spans[0].get('class',
'0').replace(
'r', ''))
# Language
lang = str(spans[0].contents[0]).strip()
item['language'] = self.languages.get(lang, lang)
if td.div:
item['hi'] = td.div.get('id') == 'imgEar'
if item.get('name'):
results.append(item)
except AttributeError:
log.error('No search results.')
return results
def getItems(self, data):
results = []
soup = BeautifulSoup(data)
table = soup.find("table", { "class" : "filmSubtitleList" })
try:
for tr in table.findAll("tr"):
item = {}
for td in tr.findAll('td'):
if td.a:
spans = td.a.findAll('span')
if len(spans) == 2:
item['id'] = int(spans[1].get('id').replace('r', ''))
item['name'] = str(spans[1].contents[0]).strip()
item['rating'] = int(spans[0].get('class', '0').replace('r', ''))
# Language
lang = str(spans[0].contents[0]).strip()
item['language'] = self.languages.get(lang, lang)
if td.div:
item['hi'] = td.div.get('id') == 'imgEar'
if item.get('name'):
results.append(item)
except AttributeError:
log.error('No search results.')
return results
def findByProvider(self, data, provider):
results = {'480p':[], '720p':[], '1080p':[]}
try:
tables = SoupStrainer('table')
html = BeautifulSoup(data, parseOnlyThese = tables)
resultTable = html.find('table', attrs = {'class':'bottomTable'})
for tr in resultTable.findAll('tr'):
trtext = str(tr).lower()
if 'clips' in trtext:
break
if 'trailer' in trtext and not 'clip' in trtext and provider in trtext:
nr = 0
resolutions = tr.findAll('td', attrs = {'class':'bottomTableResolution'})
#sizes = tr.findNext('tr').findAll('td', attrs = {'class':'bottomTableFileSize'})
for res in resolutions:
results[str(res.a.contents[0])].insert(0, res.a['href'])
#int(sizes[nr].contents[0].replace('MB', ''))
nr += 1
return results
except AttributeError:
log.debug('No trailers found in provider %s.' % provider)
results['404'] = True
return results
def find(self, movie, quality, type):
results = []
if not self.enabled() or not self.isAvailable(self.searchUrl):
return results
url = self.searchUrl % quote_plus(self.toSearchString(movie.name + ' ' + quality))
log.info('Searching: %s' % url)
data = urllib.urlopen(url)
try:
tables = SoupStrainer('table')
html = BeautifulSoup(data, parseOnlyThese = tables)
resultTable = html.find('table', attrs = {'class':'requests'})
for result in resultTable.findAll('tr', attrs = {'class':'req_filled'}):
new = self.feedItem()
id = result.find('td', attrs = {'class':'reqid'})
new.id = id.contents[0]
name = result.find('td', attrs = {'class':'release'})
new.name = self.toSaveString(name.contents[0])
new.size = 9999
new.content = 'x264'
new.type = 'nzb'
new.url = self.downloadUrl % (new.id)
new.date = time.time()
new.score = self.calcScore(new, movie)
if self.isCorrectMovie(new, movie, type):
results.append(new)
log.info('Found: %s' % new.name)
return results
except AttributeError:
log.debug('No search results found.')
def checkForUpdateWindows(self):
try:
data = urllib2.urlopen(self.downloads, timeout = self.timeout).read()
except (IOError, URLError):
log.error('Failed to open %s.' % self.downloads)
return False
try:
html = BeautifulSoup(data)
results = html.findAll('a', attrs = {'href':re.compile('/downloads/')})
for link in results:
if 'windows' in str(link.parent).lower():
downloadUrl = 'http://github.com' + link.get('href').replace(' ', '%20')
break
if 'r' + str(version.windows) in downloadUrl:
return False
return downloadUrl
except AttributeError:
log.debug('Nothing found.')
return False
def findByProvider(self, data, provider):
results = {'480p':[], '720p':[], '1080p':[]}
try:
tables = SoupStrainer('table')
html = BeautifulSoup(data, parseOnlyThese = tables)
resultTable = html.find('table', attrs = {'class':'bottomTable'})
for tr in resultTable.findAll('tr'):
trtext = str(tr).lower()
if 'clips' in trtext:
break
if 'trailer' in trtext and not 'clip' in trtext and provider in trtext:
nr = 0
resolutions = tr.findAll('td', attrs = {'class':'bottomTableResolution'})
#sizes = tr.findNext('tr').findAll('td', attrs = {'class':'bottomTableFileSize'})
for res in resolutions:
results[str(res.a.contents[0])].insert(0, res.a['href'])
#int(sizes[nr].contents[0].replace('MB', ''))
nr += 1
return results
except AttributeError:
log.debug('No trailers found in provider %s.' % provider)
results['404'] = True
return results
def getItems(self, data):
results = []
soup = BeautifulSoup(data)
table = soup.find("table", { "class" : "chart" })
try:
for tr in table.findAll("tr"):
item = {}
for td in tr.findAll('td'):
# Get title and ID from <a>
if td.a and not td.a.img:
item['id'] = int(td.a['href'].split('/')[-1])
item['name'] = str(td.a.contents[0])
# Get year from <td>
if not td.h3 and not td.a:
if len(td.contents) == 1:
for y in td.contents:
try:
item['year'] = int(y)
except ValueError:
pass
if item:
results.append(item)
except AttributeError:
log.error('No search results.')
return results
def getDetails(self, id):
url = self.detailUrl + str(id)
log.info('Scanning %s.' % url)
try:
data = urllib2.urlopen(url, timeout = self.timeout).read()
except (IOError, URLError):
log.error('Failed to open %s.' % url)
return False
# Search for theater release
theaterDate = 0
try:
theaterLink = SoupStrainer('a', href = re.compile('/month_theaters.html\?'))
theater = BeautifulSoup(data, parseOnlyThese = theaterLink)
theaterDate = int(time.mktime(parse(theater.a.contents[0]).timetuple()))
except AttributeError:
log.debug('No Theater release info found.')
# Search for dvd release date
dvdDate = 0
try:
try:
dvdLink = SoupStrainer('a', href = re.compile('/month_video.html\?'))
dvd = BeautifulSoup(data, parseOnlyThese = dvdLink)
dvdDate = int(time.mktime(parse(dvd.a.contents[0]).timetuple()))
except:
pass
# Try left column
if not dvdDate:
dvdReleases = SoupStrainer('p', text = re.compile('Released'))
dvd = BeautifulSoup(data, parseOnlyThese = dvdReleases)
for date in dvd:
foundDate = int(time.mktime(parse(date.replace('Released', '')).timetuple()))
dvdDate = foundDate if foundDate > dvdDate else dvdDate
except AttributeError:
log.debug('No DVD release info found.')
# Does it have blu-ray release?
bluray = []
try:
bees = SoupStrainer('b')
soup = BeautifulSoup(data, parseOnlyThese = bees)
bluray = soup.findAll('b', text = re.compile('Blu-ray'))
except AttributeError:
log.info('No Bluray release info found.')
dates = {
'id': id,
'dvd': dvdDate,
'theater': theaterDate,
'bluray': len(bluray) > 0
}
log.debug('Found: %s' % dates)
return dates
def getInfo(self, url):
log.debug('Getting info: %s' % url)
try:
data = urllib2.urlopen(url, timeout = self.timeout).read()
pass
except IOError:
log.error('Failed to open %s.' % url)
return ''
tables = SoupStrainer('table')
html = BeautifulSoup(data)
movieInformation = html.find('div', attrs = {'class':'i_info'})
return str(movieInformation).decode("utf-8", "replace")
def getInfo(self, url):
log.debug('Getting info: %s' % url)
try:
data = urllib2.urlopen(url, timeout = self.timeout).read()
pass
except IOError:
log.error('Failed to open %s.' % url)
return ''
tables = SoupStrainer('table')
html = BeautifulSoup(data)
movieInformation = html.find('div', attrs = {'class':'i_info'})
return str(movieInformation).decode("utf-8", "replace")
def getInfo(self, url):
log.debug('Getting info: %s' % url)
try:
data = urllib2.urlopen(url, timeout = self.timeout).read()
pass
except IOError:
log.error('Failed to open %s.' % url)
return ''
div = SoupStrainer('div')
html = BeautifulSoup(data, parseOnlyThese = div)
html = html.find('div', attrs = {'class':'nfo'})
return str(html).decode("utf-8", "replace")
def find(self, movie, quality, type):
results = []
if not self.enabled() or not self.isAvailable(self.searchUrl):
return results
url = self.searchUrl % quote_plus(
self.toSearchString(movie.name + ' ' + quality))
log.info('Searching: %s' % url)
try:
data = urllib2.urlopen(url, timeout=self.timeout).read()
except (IOError, URLError):
log.error('Failed to open %s.' % url)
return results
try:
tables = SoupStrainer('table')
html = BeautifulSoup(data, parseOnlyThese=tables)
resultable = html.find('table', attrs={'class': 't'})
for result in resultable.findAll('span', attrs={'class': 'cname'}):
new = self.feedItem()
a = result.find('a')
id = re.search('(?<=detail\?c\=)\w+', a['href'])
new.id = id.group(0)
text = a.findAll(text=True)
words = ''
for text in a.findAll(text=True):
words = words + unicode(text).encode('utf-8')
new.name = words
new.size = 9999
new.content = 'mysterbin'
new.type = 'nzb'
new.url = self.downloadUrl % (new.id)
new.date = time.time()
new.score = self.calcScore(new, movie)
if self.isCorrectMovie(new, movie, type):
results.append(new)
log.info('Found: %s' % new.name)
return results
except AttributeError:
log.debug('No search results found.')
return results
def find(self, movie, quality, type):
results = []
if not self.enabled() or not self.isAvailable(self.searchUrl):
return results
url = self.searchUrl % quote_plus(self.toSearchString(movie.name + ' ' + quality))
log.info('Searching: %s' % url)
try:
data = urllib2.urlopen(url, timeout = self.timeout).read()
except (IOError, URLError):
log.error('Failed to open %s.' % url)
return results
try:
tables = SoupStrainer('table')
html = BeautifulSoup(data, parseOnlyThese = tables)
resultable = html.find('table', attrs = {'class':'t'})
for result in resultable.findAll('span', attrs = {'class':'cname'}):
new = self.feedItem()
a = result.find('a')
id = re.search('(?<=detail\?c\=)\w+', a['href'])
new.id = id.group(0)
text = a.findAll(text = True)
words = ''
for text in a.findAll(text = True):
words = words + unicode(text).encode('utf-8')
new.name = words
new.size = 9999
new.content = 'mysterbin'
new.type = 'nzb'
new.url = self.downloadUrl % (new.id)
new.date = time.time()
new.score = self.calcScore(new, movie)
if self.isCorrectMovie(new, movie, type):
results.append(new)
log.info('Found: %s' % new.name)
return results
except AttributeError:
log.debug('No search results found.')
return results
def find(self, movie, quality, type):
results = []
if not self.enabled() or not self.isAvailable(self.searchUrl):
return results
url = self.searchUrl % quote_plus(
self.toSearchString(movie.name + ' ' + quality))
log.info('Searching: %s' % url)
try:
data = urllib2.urlopen(url, timeout=self.timeout).read()
except (IOError, URLError):
log.error('Failed to open %s.' % url)
return results
try:
tables = SoupStrainer('table')
html = BeautifulSoup(data, parseOnlyThese=tables)
resultTable = html.find('table', attrs={'class': 'requests'})
for result in resultTable.findAll('tr',
attrs={'class': 'req_filled'}):
new = self.feedItem()
id = result.find('td', attrs={'class': 'reqid'})
new.id = id.contents[0]
name = result.find('td', attrs={'class': 'release'})
new.name = self.toSaveString(name.contents[0])
new.size = 9999
new.content = 'x264'
new.type = 'nzb'
new.url = self.downloadUrl % (new.id)
new.date = time.time()
new.score = self.calcScore(new, movie)
if self.isCorrectMovie(new, movie, type):
results.append(new)
log.info('Found: %s' % new.name)
return results
except AttributeError:
log.debug('No search results found.')
return results
arguments = urlencode({
's':movie
})
url = "%s?%s" % (self.backupUrl, arguments)
log.debug('Searching %s' % url)
try:
data = urllib2.urlopen(url, timeout = self.timeout).read()
except (IOError, URLError), e:
log.debug('Failed to open %s. %s' % (url, e))
return results
try:
tables = SoupStrainer('div')
html = BeautifulSoup(data, parseOnlyThese = tables)
resultTable = html.findAll('h2', text = re.compile(movie))
for h2 in resultTable:
if 'trailer' in h2.lower():
parent = h2.parent.parent.parent
trailerLinks = parent.findAll('a', text = re.compile('480p|720p|1080p'))
try:
for trailer in trailerLinks:
results[trailer].insert(0, trailer.parent['href'])
except:
pass
except AttributeError:
log.debug('No trailers found in via alternative.')
def download(self, subtitle):
subtitle = subtitle['subtitles'].pop()
url = self.downloadUrl % subtitle['id']
try:
data = self.urlopen(url, timeout = self.timeout).read()
except (IOError, URLError):
log.error('Failed to open %s.' % url)
return False
soup = BeautifulSoup(data)
postUrl = self.siteUrl + soup.find("a", {'id' : 's_lc_bcr_downloadLink' }).get('href').split('"')[-2]
typeId = soup.find("input", {"name" : "typeId" }).get('value')
params = urllib.urlencode({
'__EVENTTARGET': 's$lc$bcr$downloadLink',
'__EVENTARGUMENT': '',
'__VIEWSTATE': soup.find("input", {"id" : "__VIEWSTATE" }).get('value'),
'__PREVIOUSPAGE': soup.find("input", { "id" : "__PREVIOUSPAGE" }).get('value'),
'subtitleId': soup.find("input", {"id" : "subtitleId" }).get('value'),
'typeId': typeId,
'filmId': soup.find("input", {"name" : "filmId" }).get('value')
})
# No unrarring yet
if 'rar' in typeId:
log.error('Unrar not supported yet.')
return False
req = urllib2.Request(postUrl, headers = {
'Referer' : url,
'User-Agent' : 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8'
})
subtitleFiles = []
try:
self.wait()
data = urllib2.urlopen(req, params)
self.lastUse = time.time()
hash = hashlib.md5(url).hexdigest()
tempdir = cherrypy.config.get('cachePath')
tempSubtitleFile = os.path.join(tempdir, hash + '.' + typeId)
# Remove the old
if os.path.isfile(tempSubtitleFile): os.remove(tempSubtitleFile)
with open(tempSubtitleFile, 'wb') as f:
f.write(data.read())
if 'zip' in typeId:
zip = ZipFile(tempSubtitleFile)
extract = []
for name in zip.namelist():
for ext in self.extensions:
if ext.replace('*', '') in name:
subtitleFiles.append(os.path.join(tempdir, name))
extract.append(name)
zip.extractall(tempdir, extract)
os.remove(tempSubtitleFile)
else:
subtitleFiles.append(tempSubtitleFile)
log.info('Subtitle download "%s" finished. %dKB.' % (subtitle['name'], int(data.info().getheaders("Content-Length")[0]) / 1024))
return subtitleFiles
except:
log.error('Subtitle download %s failed.' % subtitle['name'])
return False
def find(self, movie, quality, type):
results = []
if not self.enabled() or not self.isAvailable(self.apiUrl):
return results
url = self.apiUrl % (quote_plus(self.toSearchString(movie.name + ' ' + quality) + self.makeIgnoreString(type)), self.getCatId(type))
log.info('Searching: %s' % url)
try:
data = urllib2.urlopen(url, timeout = self.timeout).read()
except (IOError, URLError):
log.error('Failed to open %s.' % url)
return results
try:
tables = SoupStrainer('table')
html = BeautifulSoup(data, parseOnlyThese = tables)
resultTable = html.find('table', attrs = {'id':'searchResult'})
for result in resultTable.findAll('tr'):
details = result.find('a', attrs = {'class':'detLink'})
if details:
href = re.search('/(?P<id>\d+)/', details['href'])
id = href.group('id')
name = self.toSaveString(details.contents[0])
desc = result.find('font', attrs = {'class':'detDesc'}).contents[0].split(',')
date = ''
size = 0
for item in desc:
# Weird date stuff
if 'uploaded' in item.lower():
date = item.replace('Uploaded', '')
date = date.replace('Today', '')
# Do something with yesterday
yesterdayMinus = 0
if 'Y-day' in date:
date = date.replace('Y-day', '')
yesterdayMinus = 86400
datestring = date.replace(' ', ' ').strip()
date = int(time.mktime(parse(datestring).timetuple())) - yesterdayMinus
# size
elif 'size' in item.lower():
size = item.replace('Size', '')
seedleech = []
for td in result.findAll('td'):
try:
seedleech.append(int(td.contents[0]))
except ValueError:
pass
seeders = 0
leechers = 0
if len(seedleech) == 2 and seedleech[0] > 0 and seedleech[1] > 0:
seeders = seedleech[0]
leechers = seedleech[1]
# to item
new = self.feedItem()
new.id = id
new.type = 'torrent'
new.name = name
new.date = date
new.size = self.parseSize(size)
new.seeders = seeders
new.leechers = leechers
new.url = self.downloadLink(id, name)
new.score = self.calcScore(new, movie) + self.uploader(result) + (seeders / 10)
if seeders > 0 and (new.date + (int(self.conf('wait')) * 60 * 60) < time.time()) and Qualities.types.get(type).get('minSize') <= new.size:
new.detailUrl = self.detailLink(id)
new.content = self.getInfo(new.detailUrl)
if self.isCorrectMovie(new, movie, type):
results.append(new)
log.info('Found: %s' % new.name)
return results
except AttributeError:
log.debug('No search results found.')
return []
def download(self, subtitle):
subtitle = subtitle['subtitles'].pop()
url = self.downloadUrl % subtitle['id']
try:
data = self.urlopen(url, timeout=self.timeout).read()
except (IOError, URLError):
log.error('Failed to open %s.' % url)
return False
soup = BeautifulSoup(data)
postUrl = self.siteUrl + soup.find("a", {
'id': 's_lc_bcr_downloadLink'
}).get('href').split('"')[-2]
typeId = soup.find("input", {"name": "typeId"}).get('value')
params = urllib.urlencode({
'__EVENTTARGET':
's$lc$bcr$downloadLink',
'__EVENTARGUMENT':
'',
'__VIEWSTATE':
soup.find("input", {
"id": "__VIEWSTATE"
}).get('value'),
'__PREVIOUSPAGE':
soup.find("input", {
"id": "__PREVIOUSPAGE"
}).get('value'),
'subtitleId':
soup.find("input", {
"id": "subtitleId"
}).get('value'),
'typeId':
typeId,
'filmId':
soup.find("input", {
"name": "filmId"
}).get('value')
})
# No unrarring yet
if 'rar' in typeId:
log.error('Unrar not supported yet.')
return False
req = urllib2.Request(
postUrl,
headers={
'Referer':
url,
'User-Agent':
'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8'
})
subtitleFiles = []
try:
self.wait()
data = urllib2.urlopen(req, params)
self.lastUse = time.time()
hash = hashlib.md5(url).hexdigest()
tempdir = cherrypy.config.get('cachePath')
tempSubtitleFile = os.path.join(tempdir, hash + '.' + typeId)
# Remove the old
if os.path.isfile(tempSubtitleFile): os.remove(tempSubtitleFile)
with open(tempSubtitleFile, 'wb') as f:
f.write(data.read())
if 'zip' in typeId:
zip = ZipFile(tempSubtitleFile)
extract = []
for name in zip.namelist():
for ext in self.extensions:
if ext.replace('*', '') in name:
subtitleFiles.append(os.path.join(tempdir, name))
extract.append(name)
zip.extractall(tempdir, extract)
os.remove(tempSubtitleFile)
else:
subtitleFiles.append(tempSubtitleFile)
log.info('Subtitle download "%s" finished. %dKB.' %
(subtitle['name'],
int(data.info().getheaders("Content-Length")[0]) / 1024))
return subtitleFiles
except:
log.error('Subtitle download %s failed.' % subtitle['name'])
return False
