def response(resp):
results = []
dom = html.fromstring(resp.text)
rows = dom.xpath(
'//table[@class="listing"]//tr[contains(@class, "category_0")]')
# check if there are no results or page layout was changed so we cannot parse it
# currently there are two rows for each result, so total count must be even
if len(rows) == 0 or len(rows) % 2 != 0:
return []
# regular expression for parsing torrent size strings
size_re = re.compile(r'Size:\s*([\d.]+)(TB|GB|MB|B)', re.IGNORECASE)
# processing the results, two rows at a time
for i in range(0, len(rows), 2):
# parse the first row
name_row = rows[i]
links = name_row.xpath('./td[@class="desc-top"]/a')
params = {
'template': 'torrent.html',
'url': links[-1].attrib.get('href'),
'title': extract_text(links[-1])
}
# I have not yet seen any torrents without magnet links, but
# it's better to be prepared to stumble upon one some day
if len(links) == 2:
magnet = links[0].attrib.get('href')
if magnet.startswith('magnet'):
# okay, we have a valid magnet link, let's add it to the result
params['magnetlink'] = magnet
# no more info in the first row, start parsing the second one
info_row = rows[i + 1]
desc = extract_text(info_row.xpath('./td[@class="desc-bot"]')[0])
for item in desc.split('|'):
item = item.strip()
if item.startswith('Size:'):
try:
# ('1.228', 'GB')
groups = size_re.match(item).groups()
params['filesize'] = get_torrent_size(groups[0], groups[1])
except:
pass
elif item.startswith('Date:'):
try:
# Date: 2016-02-21 21:44 UTC
date = datetime.strptime(item, 'Date: %Y-%m-%d %H:%M UTC')
params['publishedDate'] = date
except:
pass
elif item.startswith('Comment:'):
params['content'] = item
stats = info_row.xpath('./td[@class="stats"]/span')
# has the layout not changed yet?
if len(stats) == 3:
params['seed'] = int_or_zero(extract_text(stats[0]))
params['leech'] = int_or_zero(extract_text(stats[1]))
results.append(params)
return results
def response(resp):
results = []
dom = html.fromstring(resp.text)
rows = dom.xpath('//table[@class="listing"]//tr[contains(@class, "category_0")]')
# check if there are no results or page layout was changed so we cannot parse it
# currently there are two rows for each result, so total count must be even
if len(rows) == 0 or len(rows) % 2 != 0:
return []
# regular expression for parsing torrent size strings
size_re = re.compile(r'Size:\s*([\d.]+)(TB|GB|MB|B)', re.IGNORECASE)
# processing the results, two rows at a time
for i in range(0, len(rows), 2):
# parse the first row
name_row = rows[i]
links = name_row.xpath('./td[@class="desc-top"]/a')
params = {
'template': 'torrent.html',
'url': links[-1].attrib.get('href'),
'title': extract_text(links[-1])
}
# I have not yet seen any torrents without magnet links, but
# it's better to be prepared to stumble upon one some day
if len(links) == 2:
magnet = links[0].attrib.get('href')
if magnet.startswith('magnet'):
# okay, we have a valid magnet link, let's add it to the result
params['magnetlink'] = magnet
# no more info in the first row, start parsing the second one
info_row = rows[i + 1]
desc = extract_text(info_row.xpath('./td[@class="desc-bot"]')[0])
for item in desc.split('|'):
item = item.strip()
if item.startswith('Size:'):
try:
# ('1.228', 'GB')
groups = size_re.match(item).groups()
params['filesize'] = get_torrent_size(groups[0], groups[1])
except:
pass
elif item.startswith('Date:'):
try:
# Date: 2016-02-21 21:44 UTC
date = datetime.strptime(item, 'Date: %Y-%m-%d %H:%M UTC')
params['publishedDate'] = date
except:
pass
elif item.startswith('Comment:'):
params['content'] = item
stats = info_row.xpath('./td[@class="stats"]/span')
# has the layout not changed yet?
if len(stats) == 3:
params['seed'] = int_or_zero(extract_text(stats[0]))
params['leech'] = int_or_zero(extract_text(stats[1]))
results.append(params)
return results
def response(resp):
results = []
dom = html.fromstring(resp.text)
for result in dom.xpath(xpath_results):
# defaults
filesize = 0
magnet_link = ""
torrent_link = ""
# category in which our torrent belongs
try:
category = result.xpath(xpath_category)[0].attrib.get('title')
except:
pass
# torrent title
page_a = result.xpath(xpath_title)[0]
title = extract_text(page_a)
# link to the page
href = base_url + page_a.attrib.get('href')
for link in result.xpath(xpath_torrent_links):
url = link.attrib.get('href')
if 'magnet' in url:
# link to the magnet
magnet_link = url
else:
# link to the torrent file
torrent_link = url
# seed count
seed = int_or_zero(result.xpath(xpath_seeds))
# leech count
leech = int_or_zero(result.xpath(xpath_leeches))
# torrent downloads count
downloads = int_or_zero(result.xpath(xpath_downloads))
# let's try to calculate the torrent size
try:
filesize_info = result.xpath(xpath_filesize)[0]
filesize, filesize_multiplier = filesize_info.split()
filesize = get_torrent_size(filesize, filesize_multiplier)
except:
pass
# content string contains all information not included into template
content = 'Category: "{category}". Downloaded {downloads} times.'
content = content.format(category=category, downloads=downloads)
results.append({
'url': href,
'title': title,
'content': content,
'seed': seed,
'leech': leech,
'filesize': filesize,
'torrentfile': torrent_link,
'magnetlink': magnet_link,
'template': 'torrent.html'
})
return results
def response(resp):
results = []
dom = html.fromstring(resp.text)
for result in dom.xpath(xpath_results):
# defaults
filesize = 0
magnet_link = ""
torrent_link = ""
# category in which our torrent belongs
try:
category = result.xpath(xpath_category)[0].attrib.get('title')
except:
pass
# torrent title
page_a = result.xpath(xpath_title)[0]
title = extract_text(page_a)
# link to the page
href = base_url + page_a.attrib.get('href')
for link in result.xpath(xpath_torrent_links):
url = link.attrib.get('href')
if 'magnet' in url:
# link to the magnet
magnet_link = url
else:
# link to the torrent file
torrent_link = url
# seed count
seed = int_or_zero(result.xpath(xpath_seeds))
# leech count
leech = int_or_zero(result.xpath(xpath_leeches))
# torrent downloads count
downloads = int_or_zero(result.xpath(xpath_downloads))
# let's try to calculate the torrent size
try:
filesize_info = result.xpath(xpath_filesize)[0]
filesize, filesize_multiplier = filesize_info.split()
filesize = get_torrent_size(filesize, filesize_multiplier)
except:
pass
# content string contains all information not included into template
content = 'Category: "{category}". Downloaded {downloads} times.'
content = content.format(category=category, downloads=downloads)
results.append({'url': href,
'title': title,
'content': content,
'seed': seed,
'leech': leech,
'filesize': filesize,
'torrentfile': torrent_link,
'magnetlink': magnet_link,
'template': 'torrent.html'})
return results
