def task_page(self, grab, task):
if grab.response.code != 200:
with database.transaction():
Removed.create(id=task.pk)
return
data = grab.doc.structure(
'//*[@id="detailsframe"]',
x(
'./*[@id="details"]',
category='.//dt[.="Type:"]/following-sibling::dd/a/text()',
size=('.//dt[.="Size:"]/following-sibling::dd/text()',
converters.extract_integer),
created=('.//dt[.="Uploaded:"]/following-sibling::dd/text()',
converters.extract_datetime),
hash=('.//dt[.="Info Hash:"]/following-sibling::dd',
converters.extract_tail),
),
nfo=('//*[@class="nfo"]/pre', converters.extract_inner_html),
magnet='//*[@class="download"]/a/@href',
title='./*[@id="title"]/text()',
)[0]
data.update({
'id': task.pk,
})
with database.transaction():
category = Category.get_or_create(name=data.pop('category', ''))
Torrent.create(category=category,
**data)
def _row_to_torrent(self, row):
torrent = Torrent()
torrent_name_tag = row.find(class_='detName')
torrent.name = torrent_name_tag.find(class_='detLink').text
torrent.magnet_link = torrent_name_tag.find_next_sibling('a').get('href')
tds = row.find_all('td')
torrent.seeders = int(tds[2].text)
torrent.leechers = int(tds[3].text)
return torrent
def _parse_page(self, page_text):
torrents = []
for result in page_text["torrents"]:
t = Torrent()
t.title = result["torrent_title"]
t.seeds = result["seeds"]
t.size = torrentutils.hsize(result["size"])
t.torrent_url = result["magnet_uri"]
torrents.append(t)
return torrents[:50]
def upload_torrent(request):
form = TorrentUploadForm()
if request.method == 'POST':
form = TorrentUploadForm(request.POST, request.FILES)
if form.is_valid():
cd = form.cleaned_data
t = Torrent(name=cd['name'], data=cd['torrent'])
t.save()
return HttpResponseRedirect('/')
return HttpResponse('woot') #write template to include form, will need to include logic to register with shadowolf
def _parse_page(self, page_text):
torrents = []
for result in page_text['torrents']:
t = Torrent()
t.title = result['torrent_title']
t.seeds = result['seeds']
t.size = result['size']
t.torrent_url = result['magnet_uri']
torrents.append(t)
return torrents[:50]
def _get_season_episodes(self, show, season):
"""get all episodes of a given season of a show"""
torrents = []
all_episodes = self._get_show_episodes(show["id"])
for ep in all_episodes:
if season == ep["season"]:
t = Torrent()
t.title = "{}.S{}E{}:{}".format(show["title"], ep["season"], ep["num"], ep["title"])
t.torrent_url = ep["torrent_url"]
t.seeds = ep["seeds"]
torrents.append(t)
return torrents
def _get_latest_episode(self, show):
"""get the latest episode of the latest season of a show"""
torrents = []
all_episodes = self._get_show_episodes(show["id"])
sorted_episodes = sorted(all_episodes, key=itemgetter("season", "num"), reverse=True)
last_ep = {k: str(v) for k, v in sorted_episodes[0].iteritems()}
t = Torrent()
t.title = "{}.S{}E{}:{}".format(show["title"], last_ep["season"], last_ep["num"], last_ep["title"])
t.torrent_url = last_ep["torrent_url"]
t.seeds = int(last_ep["seeds"])
torrents.append(t)
return torrents
def _parse_page(self, page_text):
soup = BS(page_text, "lxml")
torrents = []
lines = soup.find_all(class_='ligne0') + soup.find_all(class_='ligne1')
for line in lines:
t = Torrent()
t.title = line.find('a').text
t.size = line.find(class_='poid').text
t.seeds = int(line.find(class_='seed_ok').text)
t.torrent_url = self._torrent_link(line.find('a').get('href'))
torrents.append(t)
return torrents
def _get_season_episodes(self, show, season):
"""get all episodes of a given season of a show"""
torrents = []
all_episodes = self._get_show_episodes(show['id'])
for ep in all_episodes:
if season == ep['season']:
t = Torrent()
t.title = '{}.S{}E{}:{}'.format(
show['title'], ep['season'], ep['num'], ep['title'])
t.torrent_url = ep['torrent_url']
t.seeds = ep['seeds']
torrents.append(t)
return torrents
def uid_append(uid):
query = Torrent.all(keys_only=True).filter('hs', uid)
entity = query.get()
if entity:
return None
torrent = Torrent()
torrent.hs = uid
torrent.rn = random.random()
return torrent
def search(self, query):
payload = {"q": query, "field": "seeder", "order": "desc", "page": "1"}
search_url = self.base_url + "/json.php"
data = requests.get(search_url, params=payload, headers=self.headers).json()
torrents = []
for movie in data["list"]:
t = Torrent()
t.title = movie["title"]
t.seeds = int(movie["seeds"])
t.size = int(movie["size"])
t.torrent_url = movie["torrentLink"]
torrents.append(t)
return torrents
def _get_season_episodes(self, show, season):
"""get all episodes of a given season of a show"""
torrents = []
all_episodes = self._get_show_episodes(show['id'])
for ep in all_episodes:
if season == ep['season']:
t = Torrent()
t.title = '{}.S{}E{}:{}'.format(show['title'], ep['season'],
ep['num'], ep['title'])
t.torrent_url = ep['torrent_url']
t.seeds = ep['seeds']
torrents.append(t)
return torrents
def _get_episode(self, show, season=None, episode=None):
"""get the given episode of a show's season"""
torrents = []
all_episodes = self._get_show_episodes(show["id"])
if episode is not None and season is not None:
for ep in all_episodes:
if season == ep["season"] and ep["num"] == episode:
t = Torrent()
t.title = "{}.S{}E{}:{}".format(show["title"], ep["season"], ep["num"], ep["title"])
t.torrent_url = ep["torrent_url"]
t.seeds = ep["seeds"]
torrents.append(t)
break
return torrents
def search(self, query):
payload = {'q': query, 'field': 'seeder', 'order': 'desc', 'page': '1'}
search_url = self.base_url + '/json.php'
data = requests.get(
search_url, params=payload, headers=self.headers).json()
torrents = []
for movie in data['list']:
t = Torrent()
t.title = movie['title']
t.seeds = int(movie['seeds'])
t.size = torrentutils.hsize(movie['size'])
t.torrent_url = movie['torrentLink']
torrents.append(t)
return torrents
def _get_latest_episode(self, show):
"""get the latest episode of the latest season of a show"""
torrents = []
all_episodes = self._get_show_episodes(show['id'])
sorted_episodes = sorted(
all_episodes, key=itemgetter('season', 'num'), reverse=True)
last_ep = {k: str(v) for k, v in sorted_episodes[0].iteritems()}
t = Torrent()
t.title = '{}.S{}E{}:{}'.format(
show['title'], last_ep['season'], last_ep['num'], last_ep['title'])
t.torrent_url = last_ep['torrent_url']
t.seeds = int(last_ep['seeds'])
torrents.append(t)
return torrents
def search(handler):
query = query_from_path(handler.path)
if 'query' not in query:
handler.send_error(400)
handler.wfile.write(json.dumps({'error': "'query' is required."}))
return
s = t.search(urllib.unquote(query['query']))
items = []
for item in s.items():
torrent = Torrent(item.title, item.size, item.seeders, item.id,
item.category, item.sub_category, item.magnet_link)
database.set(item.id, torrent.reprDB())
items.append(torrent)
handler.wfile.write(json.dumps(items, cls=ComplexEncoder))
return
def get_top(self):
search_url = self.base_url + '/movies'
data = requests.get(search_url, headers=self.headers).text
soup = BS(data, "lxml")
torrents = []
table = soup.find(class_="data")
for row in table.find_all('tr')[1:]:
cells = row.find_all('td')
t = Torrent()
t.title = cells[0].find(class_="cellMainLink").text
t.torrent_url = cells[0].find_all("a")[3].get('href')
t.size = cells[1].text
t.seeds = int(cells[4].text)
torrents.append(t)
return torrents
def _get_latest_episode(self, show):
"""get the latest episode of the latest season of a show"""
torrents = []
all_episodes = self._get_show_episodes(show['id'])
sorted_episodes = sorted(all_episodes,
key=itemgetter('season', 'num'),
reverse=True)
last_ep = {k: str(v) for k, v in sorted_episodes[0].iteritems()}
t = Torrent()
t.title = '{}.S{}E{}:{}'.format(show['title'], last_ep['season'],
last_ep['num'], last_ep['title'])
t.torrent_url = last_ep['torrent_url']
t.seeds = int(last_ep['seeds'])
torrents.append(t)
return torrents
def _get_episode(self, show, season=None, episode=None):
"""get the given episode of a show's season"""
torrents = []
all_episodes = self._get_show_episodes(show['id'])
if episode is not None and season is not None:
for ep in all_episodes:
if season == ep['season'] and ep['num'] == episode:
t = Torrent()
t.title = '{}.S{}E{}:{}'.format(
show['title'], ep['season'], ep['num'], ep['title'])
t.torrent_url = ep['torrent_url']
t.seeds = ep['seeds']
torrents.append(t)
break
return torrents
def _parse_page(self, page_text):
soup = BS(page_text, "lxml")
torrents = []
table = soup.find(id="searchResult")
for row in table.find_all('tr')[1:30]:
t = Torrent()
cells = row.find_all('td')
a = cells[1].find_all('a')
t.title = a[0].text
t.torrent_url = a[1]['href']
t.seeds = int(cells[2].text)
pattern = re.compile("Uploaded (.*), Size (.*), ULed by (.*)")
match = pattern.match(cells[1].font.text)
t.size = match.groups()[1].replace('xa0', ' ')
torrents.append(t)
return torrents
def latest_torrent_dt():
"""Returns datetime for most recent torrent or start of epoch if no torrents"""
latest_torrent = Torrent.query().order(-Torrent.dt).get()
if latest_torrent:
return latest_torrent.dt
else:
return datetime.datetime.utcfromtimestamp(0)
def announce(request):
try:
params = torUtils.getParams(request.get_full_path()) # <-- Add this to models
ih = params[u'info_hash'][0]
# Match params to grab a specific torrent
t = Torrent.getTorrent(info_hash=ih)
# Check whether this is a new or returning client
c = t.getPeer(ip=request.META["REMOTE_ADDR"])
if c.size == 0:
c = Client.create(n = params['name'], i = request.META["REMOTE_ADDR"], p = params['port'], ih = params[u'info_hash'][0])
else:
# Parse old client
c = c[0]
c.update(params["event"])
except Exception as e:
print "Torrent not found; ", e
# return HttpResponse("Newp!")
# Match client against list of known users
# -- Seeding:
# -- Leeching:
# -- Loading:
# -- Inactive:
# If no announce, double-check peer connections, and send a new list
return HttpResponse("Fixthis")
def _get_episode(self, show, season=None, episode=None):
"""get the given episode of a show's season"""
torrents = []
all_episodes = self._get_show_episodes(show['id'])
if episode is not None and season is not None:
for ep in all_episodes:
if season == ep['season'] and ep['num'] == episode:
t = Torrent()
t.title = '{}.S{}E{}:{}'.format(show['title'],
ep['season'], ep['num'],
ep['title'])
t.torrent_url = ep['torrent_url']
t.seeds = ep['seeds']
torrents.append(t)
break
return torrents
def get_top(self):
payload = {"sort": "date_added", "order": "desc", "set": "1", "limit": 20}
search_url = self.base_url + "/api/v2/list_movies.json"
try:
response = requests.get(search_url, params=payload, headers=self.headers).json()
except Exception:
return
torrents = []
for movie in response["data"]["movies"]:
for torrent in movie["torrents"]:
t = Torrent()
t.title = movie["title_long"] + " " + torrent["quality"]
t.seeds = torrent["seeds"]
t.size = torrent["size"]
t.torrent_url = torrent["url"]
torrents.append(t)
return torrents
def search(self, query):
search_url = self.base_url
payload = {'page': 'search', 'term': query,
'sort': '2', 'cats': '1_0', 'filter': '0'}
torrents = []
response = requests.get(
search_url, params=payload, headers=self.headers).text
soup = bs(response, "lxml")
table = soup.find('table', class_='tlist')
for tr in table.find_all('tr')[1:]:
t = Torrent()
cols = tr.findAll('td')
t.title = cols[1].find('a').text
t.size = cols[3].text
t.seeds = cols[4].text
t.torrent_url = cols[2].find('a').get('href')+"&magnet=1"
torrents.append(t)
return torrents
def torrent_update_or_create(movie_id, torrent_list):
torrents = Torrent.query.filter_by(movie_id=movie_id).all()
if len(torrents) == 0:
for torrent in torrent_list:
torrent.update({"movie_id": movie_id})
new = Torrent(**torrent)
db.session.add(new)
db.session.commit()
return
def _row_to_torrent(self, row):
torrent = Torrent()
torrent.name = row.find(class_='torrentname').find_all('a')[1].text
torrent.magnet_link = row.find(name='a', class_='imagnet').get('href')
torrent.torrent_link = row.find(name='a',
class_='idownload').get('href')
tds = row.find_all('td')
torrent.seeders = int(tds[4].text)
torrent.leechers = int(tds[5].text)
torrent.size = 'UNKNOWN' # TODO
return torrent
def task_generator(self):
for pk in xrange(settings.START_ID, settings.END_ID):
if Torrent.filter(id=pk).exists():
continue
if Removed.filter(id=pk).exists():
continue
yield Task(url='https://thepiratebay.se/torrent/{}/'.format(pk),
name='page',
pk=pk)
def index(request):
torrents = Torrent.gql("WHERE rn > :1 ORDER BY rn ASC LIMIT 1",
random.random())
magnet = torrents.get();
if not magnet:
return HttpResponseNotFound('<h1>Empty torrents</h1>')
return render_to_response('base.html', { 'hash' : magnet.hs, 'seed' : random.random(), 'num_torrents' : memcache.get('numt') })
def get_top(self):
payload = {
'sort': 'date_added', 'order': 'desc', 'set': '1', 'limit': 20}
search_url = self.base_url + '/api/v2/list_movies.json'
try:
response = requests.get(
search_url, params=payload, headers=self.headers).json()
except Exception as e:
return
torrents = []
for movie in response['data']['movies']:
for torrent in movie['torrents']:
t = Torrent()
t.title = movie['title_long']+" "+torrent['quality']
t.seeds = torrent['seeds']
t.size = torrent['size']
t.torrent_url = torrent['url']
torrents.append(t)
return torrents
def _parse_page(self, page_text):
soup = BS(page_text, "lxml")
tabl = soup.find('table', class_='lista2t')
torrents = []
for tr in tabl.find_all('tr')[1:]:
rows = tr.find_all('td')
try:
t = Torrent()
t.title = rows[1].find('a').text
rarbg_id = rows[1].find('a')['href'].strip('/torrent/')
title = requests.utils.quote(t.title) + "-[rarbg.com].torrent"
download_url = self.base_url + "/download.php?id=%s&f=%s" % (
rarbg_id, title)
t.torrent_url = self._to_magnet(download_url)
t.size = rows[3].text
t.seeds = int(rows[4].text)
torrents.append(t)
except bencode.BTL.BTFailure:
pass
return torrents
def _row_to_torrent(self, row):
torrent = Torrent()
torrent.name = row.find(class_='torrentname').find_all('a')[1].text
torrent.magnet_link = row.find(name='a', class_='imagnet').get('href')
torrent.torrent_link = row.find(name='a', class_='idownload').get('href')
tds = row.find_all('td')
torrent.seeders = int(tds[4].text)
torrent.leechers = int(tds[5].text)
torrent.size = 'UNKNOWN' # TODO
return torrent
def search(self, query):
search_url = self.base_url
payload = {
'page': 'search',
'term': query,
'sort': '2',
'cats': '1_0',
'filter': '0'
}
torrents = []
response = requests.get(search_url,
params=payload,
headers=self.headers).text
soup = bs(response, "lxml")
table = soup.find('table', class_='tlist')
for tr in table.find_all('tr')[1:]:
t = Torrent()
cols = tr.findAll('td')
t.title = cols[1].find('a').text
size = cols[3].text
t.size = string_to_byte(size)
t.seeds = cols[4].text
t.torrent_url = cols[2].find('a').get('href') + "&magnet=1"
torrents.append(t)
return torrents
def test_get_latest_dt_returns_lstest(self):
now = datetime.datetime.now()
yesterday = now - datetime.timedelta(1)
t1 = Torrent(title=self.title, btih=self.btih, nbytes=self.nbytes, dt=yesterday)
t2 = Torrent(title=self.title, btih=self.btih, nbytes=self.nbytes, dt=now)
t1.put()
t2.put()
rv = Torrent.get_latest_dt()
self.assertEqual(rv, now)
def search(self, query):
self._get_token()
search_payload = {
'sort': 'seeders',
'category': 'movies',
'mode': 'search',
'app_id': 'xxx',
'format': 'json_extended',
'search_string': queryt,
'token': self.token,
}
results = requests.get(self.base_url, params=search_payload).json()
torrents = []
for result in results['torrent_results']:
t = Torrent()
t.title = result['title']
t.seeds = result['seeders']
t.size = hsize(result['size'])
t.torrent_url = result['download']
torrents.append(t)
return torrents
def _row_to_torrent(self, row):
torrent = Torrent()
torrent_name_tag = row.find(class_='detName')
description = row.find(class_='detDesc').text
torrent.name = torrent_name_tag.find(class_='detLink').text
torrent.magnet_link = torrent_name_tag.find_next_sibling('a').get(
'href')
tds = row.find_all('td')
torrent.seeders = int(tds[2].text)
torrent.leechers = int(tds[3].text)
torrent.size = ' '.join(self.size_regex.search(description).groups())
return torrent
def _row_to_torrent(self, row):
torrent = Torrent()
torrent_name_tag = row.find(class_='detName')
torrent.name = torrent_name_tag.find(class_='detLink').text
torrent.magnet_link = torrent_name_tag.find_next_sibling('a').get('href')
tds = row.find_all('td')
torrent.seeders = int(tds[2].text)
torrent.leechers = int(tds[3].text)
return torrent
def _parse_page(self, page_text):
torrents = []
for result in page_text['torrents']:
t = Torrent()
t.title = result['torrent_title']
t.seeds = result['seeds']
t.size = result['size']
t.torrent_url = result['magnet_uri']
torrents.append(t)
return torrents[:50]
def _row_to_torrent(self, row):
torrent = Torrent()
torrent_name_tag = row.find(class_='detName')
description = row.find(class_='detDesc').text
torrent.name = torrent_name_tag.find(class_='detLink').text
torrent.magnet_link = torrent_name_tag.find_next_sibling('a').get('href')
tds = row.find_all('td')
torrent.seeders = int(tds[2].text)
torrent.leechers = int(tds[3].text)
torrent.size = ' '.join(self.size_regex.search(description).groups())
return torrent
def _parse_page(self, page_text):
soup = BS(page_text, "lxml")
torrents = []
lines = soup.find_all(class_='ligne0') + soup.find_all(class_='ligne1')
for line in lines:
t = Torrent()
t.title = line.find('a').text
t.size = line.find(class_='poid').text
t.seeds = int(line.find(class_='seed_ok').text)
t.torrent_url = self._torrent_link(line.find('a').get('href'))
torrents.append(t)
return torrents
def search(self, query):
payload = {'q': query, 'field': 'seeder', 'order': 'desc', 'page': '1'}
search_url = self.base_url + '/json.php'
data = requests.get(
search_url, params=payload, headers=self.headers).json()
torrents = []
for movie in data['list']:
t = Torrent()
t.title = movie['title']
t.seeds = int(movie['seeds'])
t.size = int(movie['size'])
t.torrent_url = movie['torrentLink']
torrents.append(t)
return torrents
def get_top(self):
search_url = self.base_url + '/movies'
data = requests.get(search_url, headers=self.headers).text
soup = BS(data, 'lxml')
torrents = []
table = soup.find(class_='data')
for row in table.find_all('tr')[1:]:
cells = row.find_all('td')
t = Torrent()
t.title = cells[0].find(class_='cellMainLink').text
t.torrent_url = cells[0].find_all('a')[3].get('href')
t.size = string_to_byte(cells[1].text)
t.seeds = int(cells[4].text)
torrents.append(t)
return torrents
def _parse_page(self, page_text):
soup = BS(page_text, "lxml")
torrents = []
table = soup.find(id="searchResult")
for row in table.find_all('tr')[1:30]:
t = Torrent()
cells = row.find_all('td')
a = cells[1].find_all('a')
t.title = a[0].text
t.torrent_url = a[1]['href']
t.seeds = int(cells[2].text)
pattern = re.compile("Uploaded (.*), Size (.*), ULed by (.*)")
match = pattern.match(cells[1].font.text)
t.size = match.groups()[1].replace('xa0', ' ')
torrents.append(t)
return torrents
def _parse_page(self, page_text):
soup = BS(page_text, "lxml")
tabl = soup.find('table', class_='lista2t')
torrents = []
for tr in tabl.find_all('tr')[1:]:
rows = tr.find_all('td')
try:
t = Torrent()
t.title = rows[1].find('a').text
rarbg_id = rows[1].find('a')['href'].strip('/torrent/')
title = requests.utils.quote(t.title) + "-[rarbg.com].torrent"
download_url = self.base_url + "/download.php?id=%s&f=%s" % (
rarbg_id, title)
t.torrent_url = self._to_magnet(download_url)
t.size = naturalsize(rows[3].text)
t.seeds = int(rows[4].text)
torrents.append(t)
except bencode.BTL.BTFailure:
pass
return torrents
def search(self, query):
self._get_token()
search_payload = {
'sort': 'seeders',
'category': 'movies',
'mode': 'search',
'app_id': 'xxx',
'format': 'json_extended',
'search_string': query,
'token': self.token,
}
results = requests.get(self.base_url, params=search_payload).json()
torrents = []
for result in results['torrent_results']:
t = Torrent()
t.title = result['title']
t.seeds = result['seeders']
t.size = naturalsize(result['size'])
t.torrent_url = result['download']
torrents.append(t)
return torrents
def get_top(self):
payload = {
'sort': 'date_added',
'order': 'desc',
'set': '1',
'limit': 20
}
search_url = self.base_url + '/api/v2/list_movies.json'
try:
response = requests.get(search_url,
params=payload,
headers=self.headers).json()
except Exception:
return
torrents = []
for movie in response['data']['movies']:
for torrent in movie['torrents']:
t = Torrent()
t.title = movie['title_long'] + " " + torrent['quality']
t.seeds = torrent['seeds']
t.size = torrent['size']
t.torrent_url = torrent['url']
torrents.append(t)
return torrents
def latest_torrents(num_items, cat_key=None):
"""Returns num_items torrent in specified category and/or its subcategories"""
if cat_key is None:
cat_key = root_category_key()
return Torrent.query(ancestor=cat_key).order(-Torrent.dt).fetch(num_items)
