def process_magnet_items(name, raw=False, verify=True, maxnum=10):
time0 = time.time()
#
## check for jackett
if core.get_jackett_credentials() is None:
pool = Pool(processes=5)
jobs = list(
map(
lambda func: pool.apply_async(func, args=(name, maxnum)),
(
get_items_zooqle,
get_items_rarbg, #get_items_kickass,
get_items_torrentz,
get_items_eztv_io)))
jobs.append(
pool.apply_async(get_items_tpb,
args=(name, maxnum, False,
False))) # args.do_any = False
else:
pool = Pool(processes=3)
jobs = list(
map(
lambda func: pool.apply_async(func,
args=(name, maxnum, verify)),
(get_items_zooqle, get_items_eztv_io)))
jobs.append(
pool.apply_async(get_items_jackett,
args=(name, maxnum, raw, verify)))
items_all = list(
chain.from_iterable(filter(None, map(lambda job: job.get(), jobs))))
logging.info('search for torrents took %0.3f seconds.' %
(time.time() - time0))
pool.close()
pool.join()
if len(items_all) != 0: return items_all
return None
def get_book_torrent_jackett(name, maxnum=10, keywords=[], verify=True):
"""
Returns a :py:class:`tuple` of candidate book Magnet links found using the main Jackett_ torrent searching service and the string ``"SUCCESS"``, if successful.
:param str name: the book to search for.
:param int maxnum: optional argumeent, the maximum number of magnet links to return. Default is 10. Must be :math:`\ge 5`.
:param list keywords: optional argument. If not empty, the title of the candidate element must have at least one of the keywords in ``keywords``.
:param bool verify: optional argument, whether to verify SSL connections. Default is ``True``.
:returns: if successful, then returns a two member :py:class:`tuple` the first member is a :py:class:`list` of elements that match the searched episode, ordered from *most* seeds and leechers to least. The second element is the string ``"SUCCESS"``. The keys in each element of the list are,
* ``title`` is the name of the candidate book to download, and in parentheses the size of the candidate in MB or GB.
* ``rawtitle`` also the name of the candidate episode to download.
* ``seeders`` is the number of seeds for this Magnet link.
* ``leechers`` is the number of leeches for this Magnet link.
* ``link`` is the Magnet URI link.
* ``torrent_size`` is the size of this torrent in Megabytes.
If this is unsuccessful, then returns an error :py:class:`tuple` of the form returned by :py:meth:`return_error_raw <howdy.core.return_error_raw>`.
:rtype: tuple
.. _Jackett: https://github.com/Jackett/Jackett
"""
import validators
assert (maxnum >= 5)
data = core.get_jackett_credentials()
if data is None:
return return_error_raw(
'FAILURE, COULD NOT GET JACKETT SERVER CREDENTIALS')
url, apikey = data
if not url.endswith('/'): url = '%s/' % url
endpoint = 'api/v2.0/indexers/all/results/torznab/api'
name_split = name.split()
last_tok = name_split[-1].lower()
status = re.match('^s[0-9]{2}e[0-9]{2}', last_tok)
def _return_params(name):
params = {'apikey': apikey, 'q': name, 'cat': '7020'}
return params
logging.info('URL ENDPOINT: %s, PARAMS = %s.' % (urljoin(url, endpoint), {
'apikey': apikey,
'q': name,
'cat': '7020'
}))
response = requests.get(urljoin(url, endpoint),
params={
'apikey': apikey,
'q': name,
'cat': '7020'
},
verify=verify) # tv shows
if response.status_code != 200:
return return_error_raw(
'FAILURE, PROBLEM WITH JACKETT SERVER ACCESSIBLE AT %s.' % url)
html = BeautifulSoup(response.content, 'lxml')
if len(html.find_all('item')) == 0:
return return_error_raw(
'FAILURE, NO BOOKS SATISFYING CRITERIA FOR GETTING %s' % name)
items = []
def _get_magnet_url(item):
magnet_url = item.find('torznab:attr', {'name': 'magneturl'})
if magnet_url is not None and 'magnet' in magnet_url['value']:
return magnet_url['value']
#
## not found it here, must go into URL
url2 = item.find('guid')
if url2 is None: return None
url2 = url2.text
if not validators.url(url2): return None
resp2 = requests.get(url2, verify=verify)
if resp2.status_code != 200: return None
h2 = BeautifulSoup(resp2.content, 'lxml')
valid_magnet_links = set(
map(
lambda elem: elem['href'],
filter(
lambda elem: 'href' in elem.attrs and 'magnet' in elem[
'href'], h2.find_all('a'))))
if len(valid_magnet_links) == 0: return None
return max(valid_magnet_links)
if status is None: last_tok = None
for item in html('item'):
title = item.find('title')
if title is None: continue
title = title.text
#
## now check if the sXXeYY in name
if last_tok is not None:
if last_tok not in title.lower(): continue
torrent_size = item.find('size')
if torrent_size is not None:
torrent_size = float(torrent_size.text) / 1024**2
seeders = item.find('torznab:attr', {'name': 'seeders'})
if seeders is None: seeders = -1
else: seeders = int(seeders['value'])
leechers = item.find('torznab:attr', {'name': 'peers'})
if leechers is None: leechers = -1
else: leechers = int(leechers['value'])
#
## now do one of two things to get the magnet URL
magnet_url = _get_magnet_url(item)
if magnet_url is None: continue
myitem = {
'title': title,
'rawtitle': title,
'seeders': seeders,
'leechers': leechers,
'link': magnet_url
}
if torrent_size is not None:
myitem['title'] = '%s (%0.1f MiB)' % (title, torrent_size)
myitem['torrent_size'] = torrent_size
pubdate_elem = item.find('pubdate')
if pubdate_elem is not None:
try:
pubdate_s = pubdate_elem.get_text().split(',')[-1].strip()
pubdate_s = ' '.join(pubdate_s.split()[:3])
pubdate = datetime.datetime.strptime(pubdate_s,
'%d %B %Y').date()
myitem['pubdate'] = pubdate
except:
pass
items.append(myitem)
items = sorted(items,
key=lambda elem: elem['seeders'] + elem['leechers'])[::-1]
if len(keywords) != 0:
items = list(
filter(
lambda item: any(
map(lambda tok: tok.lower() in item['rawtitle'].lower(
), keywords)) and not any(
map(
lambda tok: tok.lower() in item['rawtitle'].lower(
), keywords_exc)) and all(
map(
lambda tok: tok.lower() in item['rawtitle']
.lower(), must_have)), items))
if len(items) == 0:
return return_error_raw(
'FAILURE, NO BOOKS SATISFYING CRITERIA FOR GETTING %s' % name)
return items[:maxnum], 'SUCCESS'
def get_movie_torrent_jackett(name,
maxnum=10,
verify=True,
doRaw=False,
tmdb_id=None):
"""
Returns a :py:class:`tuple` of candidate movie Magnet links found using the main Jackett_ torrent searching service and the string ``"SUCCESS"``, if successful.
:param str name: the movie string on which to search.
:param int maxnum: optional argumeent, the maximum number of magnet links to return. Default is 10. Must be :math:`\ge 5`.
:param bool verify: optional argument, whether to verify SSL connections. Default is ``True``.
:param bool doRaw: optional argument. If ``True``, uses the IMDb_ information to search for the movie. Otherwise, uses the full string in ``name`` to search for the movie.
:param int tmdb_id: optional argument. If defined, use this TMDB_ movie ID to search for magnet links.
:returns: if successful, then returns a two member :py:class:`tuple` the first member is a :py:class:`list` of elements that match the searched movie, ordered from *most* seeds and leechers to least. The second element is the string ``"SUCCESS"``. The keys in each element of the list are,
* ``title`` is the name of the candidate movie to download, and in parentheses the size of the candidate in MB or GB.
* ``rawtitle`` is *only* the name of the candidate movie to download.
* ``seeders`` is the number of seeds for this Magnet link.
* ``leechers`` is the number of leeches for this Magnet link.
* ``link`` is the Magnet URI link.
* ``torrent_size`` is the size of this torrent in GB.
If this is unsuccessful, then returns an error :py:class:`tuple` of the form returned by :py:meth:`return_error_raw <howdy.core.return_error_raw>`.
:rtype: tuple
.. _Jackett: https://github.com/Jackett/Jackett
"""
time0 = time.time()
data = core.get_jackett_credentials()
if data is None:
return return_error_raw(
'failure, could not get jackett server credentials')
url, apikey = data
endpoint = 'api/v2.0/indexers/all/results/torznab/api'
popName = False
if tmdb_id is not None: popName = True
def _return_params(name, popName, tmdb_id):
params = {'apikey': apikey, 'cat': 2000}
if tmdb_id is not None:
imdb_id = movie.get_imdbid_from_id(tmdb_id, verify=verify)
params['imdbid'] = imdb_id
return params
elif doRaw:
params['q'] = name
return params
tmdb_id = movie.get_movie_tmdbids(name, verify=verify)
#if tmdb_id is None or doRaw and not popName:
# params['q'] = name
# return params
#
## check that the name matches
movie_name = movie.get_movie_info(
tmdb_id, verify=verify)['title'].lower().strip()
if movie_name != name.lower().strip():
params['q'] = name
return params
imdb_id = movie.get_imdbid_from_id(tmdb_id, verify=verify)
if imdb_id is None:
params['q'] = name
return params
params['imdbid'] = imdb_id
return params
params = _return_params(name, popName, tmdb_id)
if popName and 'q' in params: params.pop('q')
logging.info('params: %s, mainURL = %s' % (params, urljoin(url, endpoint)))
response = requests.get(urljoin(url, endpoint),
verify=verify,
params=params)
if response.status_code != 200:
return return_error_raw(' '.join([
'failure, problem with jackett server accessible at %s.' % url,
'Error code = %d. Error data = %s.' %
(response.status_code, response.content)
]))
logging.info('processed jackett torrents for %s in %0.3f seconds.' %
(name, time.time() - time0))
html = BeautifulSoup(response.content, 'lxml')
if len(html.find_all('item')) == 0:
return return_error_raw(
'failure, could not find movie %s with jackett.' % name)
items = []
def _get_magnet_url(item):
magnet_url = item.find('torznab:attr', {'name': 'magneturl'})
if magnet_url is not None and 'magnet' in magnet_url['value']:
return magnet_url['value']
#
## not found it here, must go into URL
url2 = item.find('guid')
if url2 is None: return None
url2 = url2.text
if not validators.url(url2): return None
resp2 = requests.get(url2, verify=verify)
if resp2.status_code != 200: return None
h2 = BeautifulSoup(resp2.content, 'lxml')
valid_magnet_links = set(
map(
lambda elem: elem['href'],
filter(
lambda elem: 'href' in elem.attrs and 'magnet' in elem[
'href'], h2.find_all('a'))))
if len(valid_magnet_links) == 0: return None
return max(valid_magnet_links)
for item in html.find_all('item'):
title = item.find('title')
if title is None: continue
title = title.text
torrent_size = item.find('size')
if torrent_size is not None:
torrent_size = float(torrent_size.text) / 1024**2
seeders = item.find('torznab:attr', {'name': 'seeders'})
if seeders is None: seeders = -1
else: seeders = int(seeders['value'])
leechers = item.find('torznab:attr', {'name': 'peers'})
if leechers is None: leechers = -1
else: leechers = int(leechers['value'])
#
## now do one of two things to get the magnet URL
magnet_url = _get_magnet_url(item)
if magnet_url is None: continue
myitem = {
'raw_title': title,
'title': title,
'seeders': seeders,
'leechers': leechers,
'link': magnet_url
}
if torrent_size is not None:
myitem['title'] = '%s (%s)' % (
title, get_formatted_size(torrent_size * 1024**2))
myitem['torrent_size'] = torrent_size
items.append(myitem)
if len(items) == 0:
return return_error_raw('FAILURE, JACKETT CANNOT FIND %s' % name)
return items[:maxnum], 'SUCCESS'
def main( ):
parser = ArgumentParser( )
parser.add_argument('-n', '--name', dest='name', type=str, action='store',
help = 'Name of the movie to get.', required = True )
parser.add_argument('-y', '--year', dest='year', type=int, action='store',
help = 'Year to look for the movie to get.')
parser.add_argument('--maxnum', dest='maxnum', type=int, action='store', default = 10,
help = 'Maximum number of torrents to look through. Default is 10.')
parser.add_argument('--timeout', dest='timeout', type=int, action='store', default = 60,
help = 'Timeout on when to quit searching for torrents (in seconds). Default is 60 seconds.' )
#parser.add_argument('--any', dest='do_any', action='store_true', default = False,
# help = 'If chosen, make no filter on movie format.')
parser.add_argument('-f', '--filename', dest='filename', action='store', type=str,
help = 'If defined, put torrent or magnet file into filename.')
parser.add_argument('--bypass', dest='do_bypass', action='store_true', default=False,
help = 'If chosen, bypass YTS.')
parser.add_argument('--nozooq', dest='do_nozooq', action='store_true', default=False,
help = 'If chosen, bypass ZOOQLE.')
#parser.add_argument('--torrentz', dest='do_torrentz', action='store_true', default=False,
# help = 'If chosen, also look through TORRENTZ to get magnet link.')
parser.add_argument('--info', dest='do_info', action='store_true', default = False,
help = 'If chosen, run in info mode.' )
parser.add_argument('--add', dest='do_add', action='store_true', default = False,
help = 'If chosen, push the magnet link or torrent file into the deluge server.' )
parser.add_argument('--noverify', dest='do_verify', action='store_false', default = True,
help = 'If chosen, do not verify SSL connections.' )
parser.add_argument('--raw', dest='do_raw', action='store_true', default = False,
help = 'If chosen, do not use IMDB matching for Jackett torrents.')
args = parser.parse_args( )
assert( args.timeout >= 10 )
if args.do_info: logging.basicConfig( level = logging.INFO )
#
num_both = 0
if args.filename is not None: num_both += 1
if args.do_add: num_both += 1
assert( num_both != 2 ), "error, at most either one of --f or --add must be set, NOT both."
#
time0 = time.time( )
tmdb_id = None
if args.year is not None and not args.do_raw:
tmdb_id = movie.get_movie_tmdbids( args.name, year = args.year )
if not args.do_bypass:
try:
get_movie_yts( args.name, verify = args.do_verify,
raiseError = True, to_torrent = args.do_add )
logging.info( 'search for YTS torrents took %0.3f seconds.' %
( time.time( ) - time0 ) )
return
except ValueError: pass
pool = Pool( processes = 4 )
if not args.do_nozooq: jobs = [
pool.apply_async( get_items_zooqle, args = ( args.name, args.maxnum ) ) ]
else: jobs = [ ]
#
## check for jackett
if get_jackett_credentials( ) is None:
jobs += list(map(lambda func: pool.apply_async( func, args = ( args.name, args.maxnum ) ),
( get_items_rarbg, get_items_tpb ) ) )
#if args.do_torrentz:
# jobs.append( pool.apply_async( get_items_torrentz, args = ( args.name, args.maxnum ) ) )
else:
jobs.append( pool.apply_async(
get_items_jackett, args = ( args.name, tmdb_id, args.maxnum, args.do_verify, args.do_raw ) ) )
jobs.append( pool.apply_async(
get_items_eztv_io, args = ( args.name, tmdb_id, args.maxnum, args.do_verify ) ) )
items_lists = [ ]
for job in jobs:
try:
items = job.get( args.timeout ) # 60 second timeout on process
if items is None: continue
items_lists.append( items )
except: pass
items = list( chain.from_iterable( items_lists ) )
logging.info( 'search for %d torrents took %0.3f seconds.' % (
len( items ), time.time( ) - time0 ) )
if len( items ) == 0: return
#
## sort from most seeders + leecher to least
items_sorted = sorted( items, key = lambda tup: -tup['seeders'] - tup['leechers'] )[:args.maxnum]
get_movie_torrent_items( items_sorted, filename = args.filename, to_torrent = args.do_add )
def main():
parser = ArgumentParser()
parser.add_argument('-n',
'--name',
dest='name',
type=str,
action='store',
required=True,
help='Name of the book to get.')
parser.add_argument(
'--maxnum',
dest='maxnum',
type=int,
action='store',
default=10,
help='Maximum number of torrents to look through. Default is 10.')
parser.add_argument('-f',
'--filename',
dest='filename',
action='store',
type=str,
help='If defined, put magnet link into filename.')
parser.add_argument(
'--add',
dest='do_add',
action='store_true',
default=False,
help='If chosen, push the magnet link into the deluge server.')
parser.add_argument('--info',
dest='do_info',
action='store_true',
default=False,
help='If chosen, run in info mode.')
parser.add_argument('--noverify',
dest='do_verify',
action='store_false',
default=True,
help='If chosen, do not verify SSL connections.')
args = parser.parse_args()
logger = logging.getLogger()
if args.do_info: logger.setLevel(logging.INFO)
#
time0 = time.time()
if core.get_jackett_credentials() is None:
print('Error, Jackett server does not work. Exiting...')
retur
items = get_items_jackett(args.name,
maxnum=args.maxnum,
verify=args.do_verify)
logging.info('search for %s took %0.3f seconds.' %
(args.name, time.time() - time0))
if items is None: return
#
## sort from most seeders + leecher to least
items_sorted = sorted(
items,
key=lambda tup:
(-tup['seeders'] - tup['leechers'], tup['title']))[:args.maxnum]
get_book_torrent_items(items_sorted,
filename=args.filename,
to_torrent_server=args.do_add)