def NewzNabPlus(book=None,
provider=None,
searchType=None,
searchMode=None,
test=False):
"""
Generic NewzNabplus query function
takes in host+key+type and returns the result set regardless of who
based on site running NewzNab+
ref http://usenetreviewz.com/nzb-sites/
"""
host = provider['HOST']
api_key = provider['API']
logger.debug(
'[NewzNabPlus] searchType [%s] with Host [%s] mode [%s] using api [%s] for item [%s]'
% (searchType, host, searchMode, api_key, str(book)))
results = []
params = ReturnSearchTypeStructure(provider, api_key, book, searchType,
searchMode)
if params:
if not str(host)[:4] == "http":
host = 'http://' + host
if host[-1:] == '/':
host = host[:-1]
URL = host + '/api?' + urllib.urlencode(params)
sterm = makeUnicode(book['searchterm'])
rootxml = None
logger.debug("[NewzNabPlus] URL = %s" % URL)
result, success = fetchURL(URL)
if test:
if result.startswith('<') and result.endswith(
'/>') and "error code" in result:
result = result[1:-2]
success = False
if not success:
logger.debug(result)
return success
if success:
try:
rootxml = ElementTree.fromstring(result)
except Exception as e:
logger.error('Error parsing data from %s: %s %s' %
(host, type(e).__name__, str(e)))
rootxml = None
else:
if not result or result == "''":
result = "Got an empty response"
logger.error('Error reading data from %s: %s' % (host, result))
# maybe the host doesn't support the search type
cancelled = cancelSearchType(searchType, result, provider)
if not cancelled: # it was some other problem
BlockProvider(provider['HOST'], result)
if rootxml is not None:
# to debug because of api
logger.debug('Parsing results from <a href="%s">%s</a>' %
(URL, host))
if rootxml.tag == 'error':
errormsg = rootxml.get('description', default='unknown error')
logger.error("%s - %s" % (host, errormsg))
# maybe the host doesn't support the search type
cancelled = cancelSearchType(searchType, errormsg, provider)
if not cancelled: # it was some other problem
BlockProvider(provider['HOST'], errormsg)
else:
resultxml = rootxml.getiterator('item')
nzbcount = 0
maxage = check_int(lazylibrarian.CONFIG['USENET_RETENTION'], 0)
for nzb in resultxml:
try:
thisnzb = ReturnResultsFieldsBySearchType(
book, nzb, host, searchMode,
provider['DLPRIORITY'])
if not maxage:
nzbcount += 1
results.append(thisnzb)
else:
# example nzbdate format: Mon, 27 May 2013 02:12:09 +0200
nzbdate = thisnzb['nzbdate']
try:
parts = nzbdate.split(' ')
nzbdate = ' '.join(
parts[:5]) # strip the +0200
dt = datetime.datetime.strptime(
nzbdate,
"%a, %d %b %Y %H:%M:%S").timetuple()
nzbage = age(
'%04d-%02d-%02d' %
(dt.tm_year, dt.tm_mon, dt.tm_mday))
except Exception as e:
logger.debug(
'Unable to get age from [%s] %s %s' %
(thisnzb['nzbdate'], type(e).__name__,
str(e)))
nzbage = 0
if nzbage <= maxage:
nzbcount += 1
results.append(thisnzb)
else:
logger.debug('%s is too old (%s day%s)' %
(thisnzb['nzbtitle'], nzbage,
plural(nzbage)))
except IndexError:
logger.debug('No results from %s for %s' %
(host, sterm))
logger.debug('Found %s nzb at %s for: %s' %
(nzbcount, host, sterm))
else:
logger.debug('No data returned from %s for %s' % (host, sterm))
return results
def get_capabilities(provider):
"""
query provider for caps if none loaded yet, or if config entry is too old and not set manually.
"""
match = False
if len(provider['UPDATED']) == 10: # any stored values?
match = True
if (formatter.age(provider['UPDATED']) > lazylibrarian.CACHE_AGE) and not provider['MANUAL']:
logger.debug('Stored capabilities for %s are too old' % provider['HOST'])
match = False
if match:
logger.debug('Using stored capabilities for %s' % provider['HOST'])
else:
host = provider['HOST']
if not str(host)[:4] == "http":
host = 'http://' + host
URL = host + '/api?t=caps&apikey=' + provider['API']
logger.debug('Requesting capabilities for %s' % URL)
request = urllib2.Request(URL)
if lazylibrarian.PROXY_HOST:
request.set_proxy(lazylibrarian.PROXY_HOST, lazylibrarian.PROXY_TYPE)
request.add_header('User-Agent', common.USER_AGENT)
resp = ""
try:
resp = urllib2.urlopen(request, timeout=30) # don't get stuck
except (urllib2.HTTPError, urllib2.URLError, socket.timeout) as e:
logger.debug("Error getting capabilities: %s" % e)
resp = ""
if resp:
if str(resp.getcode()).startswith("2"): # (200 OK etc)
logger.debug(u"Got capabilities for %s" % request.get_full_url())
try:
source_xml = resp.read() # .decode('utf-8')
data = ElementTree.fromstring(source_xml)
except:
logger.debug(u"Error getting xml from %s" % URL)
data = None
if len(data):
logger.debug(u"Parsing xml for capabilities of %s" % URL)
#############################################################################
# book search isn't mentioned in the caps xml returned by
# nzbplanet,jackett,oznzb,usenet-crawler, so we can't use it as a test
# but the newznab+ ones usually support t=book and categories in 7000 range
# whereas nZEDb ones don't support t=book and use categories in 8000 range
# also some providers give searchtype but no supportedparams, so we still
# can't tell what queries will be accepted
# also category names can be lowercase or Mixed, magazine subcat name isn't
# consistent, and subcat can be just subcat or category/subcat subcat > lang
# eg "Magazines" "Mags" or "Books/Magazines" "Mags > French"
# Load all languages for now as we don't know which the user might want
#############################################################################
#
# set some defaults
#
provider['GENERALSEARCH'] = ''
provider['EXTENDED'] = '1'
provider['BOOKCAT'] = ''
provider['MAGCAT'] = ''
provider['BOOKSEARCH'] = ''
provider['MAGSEARCH'] = ''
#
search = data.find('searching/search')
if search is not None:
if 'available' in search.attrib:
if search.attrib['available'] == 'yes':
provider['GENERALSEARCH'] = 'search'
categories = data.getiterator('category')
for cat in categories:
if 'name' in cat.attrib:
if cat.attrib['name'].lower() == 'books':
bookcat = cat.attrib['id'] # keep main bookcat for later
provider['BOOKCAT'] = bookcat
provider['MAGCAT'] = ''
if provider['BOOKCAT'] == '7000':
# looks like newznab+, should support book-search
provider['BOOKSEARCH'] = 'books'
# but check in case
search = data.find('searching/book-search')
if search is not None:
if 'available' in search.attrib:
if search.attrib['available'] == 'yes':
provider['BOOKSEARCH'] = 'books'
else:
provider['BOOKSEARCH'] = ''
else:
# looks like nZEDb, probably no book-search
provider['BOOKSEARCH'] = ''
# but check in case
search = data.find('searching/book-search')
if search is not None:
if 'available' in search.attrib:
if search.attrib['available'] == 'yes':
provider['BOOKSEARCH'] = 'books'
else:
provider['BOOKSEARCH'] = ''
subcats = cat.getiterator('subcat')
for subcat in subcats:
if 'ebook' in subcat.attrib['name'].lower():
provider['BOOKCAT'] = "%s,%s" % (provider['BOOKCAT'],subcat.attrib['id'])
if 'magazines' in subcat.attrib['name'].lower() or 'mags' in subcat.attrib['name'].lower():
if provider['MAGCAT']:
provider['MAGCAT'] = "%s,%s" % (provider['MAGCAT'],subcat.attrib['id'])
else:
provider['MAGCAT'] = subcat.attrib['id']
# if no specific magazine subcategory, use books
if not provider['MAGCAT']:
provider['MAGCAT'] = bookcat
logger.debug("Categories: Books %s : Mags %s" % (provider['BOOKCAT'], provider['MAGCAT']))
provider['UPDATED'] = formatter.today()
else:
logger.warn(u"Unable to get capabilities for %s: No data returned" % URL)
else:
logger.warn(u"Unable to get capabilities for %s: Got %s" % (URL, resp.getcode()))
return provider
def get_capabilities(provider, force=False):
"""
query provider for caps if none loaded yet, or if config entry is too old and not set manually.
"""
if not force and len(provider['UPDATED']) == 10: # any stored values?
match = True
if (age(provider['UPDATED']) >
lazylibrarian.CONFIG['CACHE_AGE']) and not provider['MANUAL']:
logger.debug('Stored capabilities for %s are too old' %
provider['HOST'])
match = False
else:
match = False
if match:
logger.debug('Using stored capabilities for %s' % provider['HOST'])
else:
host = provider['HOST']
if not str(host)[:4] == "http":
host = 'http://' + host
if host[-1:] == '/':
host = host[:-1]
URL = host + '/api?t=caps'
# most providers will give you caps without an api key
logger.debug('Requesting capabilities for %s' % URL)
source_xml, success = fetchURL(URL)
# If it failed, retry with api key
if not success:
if provider['API']:
URL = URL + '&apikey=' + provider['API']
logger.debug('Requesting capabilities for %s' % URL)
source_xml, success = fetchURL(URL)
if success:
try:
data = ElementTree.fromstring(source_xml)
except ElementTree.ParseError:
data = ''
logger.debug("Error parsing xml from %s, %s" %
(URL, source_xml))
else:
logger.debug("Error getting xml from %s, %s" % (URL, source_xml))
data = ''
if len(data):
logger.debug("Parsing xml for capabilities of %s" % URL)
#
# book search isn't mentioned in the caps xml returned by
# nzbplanet,jackett,oznzb,usenet-crawler, so we can't use it as a test
# but the newznab+ ones usually support t=book and categories in 7000 range
# whereas nZEDb ones don't support t=book and use categories in 8000 range
# also some providers give searchtype but no supportedparams, so we still
# can't tell what queries will be accepted
# also category names can be lowercase or Mixed, magazine subcat name isn't
# consistent, and subcat can be just subcat or category/subcat subcat > lang
# eg "Magazines" "Mags" or "Books/Magazines" "Mags > French"
# Load all languages for now as we don't know which the user might want
#
#
# set some defaults
#
provider['GENERALSEARCH'] = 'search'
provider['EXTENDED'] = '1'
provider['BOOKCAT'] = ''
provider['MAGCAT'] = ''
provider['AUDIOCAT'] = ''
provider['BOOKSEARCH'] = ''
provider['MAGSEARCH'] = ''
provider['AUDIOSEARCH'] = ''
#
search = data.find('searching/search')
if search is not None:
# noinspection PyUnresolvedReferences
if 'available' in search.attrib:
# noinspection PyUnresolvedReferences
if search.attrib['available'] == 'yes':
provider['GENERALSEARCH'] = 'search'
categories = data.getiterator('category')
for cat in categories:
if 'name' in cat.attrib:
if cat.attrib['name'].lower() == 'audio':
provider['AUDIOCAT'] = cat.attrib['id']
subcats = cat.getiterator('subcat')
for subcat in subcats:
if 'audiobook' in subcat.attrib['name'].lower():
provider['AUDIOCAT'] = "%s,%s" % (
provider['AUDIOCAT'], subcat.attrib['id'])
elif cat.attrib['name'].lower() == 'books':
bookcat = cat.attrib[
'id'] # keep main bookcat for starting magazines later
provider['BOOKCAT'] = bookcat
provider['MAGCAT'] = ''
# set default booksearch
if provider['BOOKCAT'] == '7000':
# looks like newznab+, should support book-search
provider['BOOKSEARCH'] = 'book'
else:
# looks like nZEDb, probably no book-search
provider['BOOKSEARCH'] = ''
# but check in case we got some settings back
search = data.find('searching/book-search')
if search:
# noinspection PyUnresolvedReferences
if 'available' in search.attrib:
# noinspection PyUnresolvedReferences
if search.attrib['available'] == 'yes':
provider['BOOKSEARCH'] = 'book'
else:
provider['BOOKSEARCH'] = ''
subcats = cat.getiterator('subcat')
for subcat in subcats:
if 'ebook' in subcat.attrib['name'].lower():
provider['BOOKCAT'] = "%s,%s" % (
provider['BOOKCAT'], subcat.attrib['id'])
if 'magazines' in subcat.attrib['name'].lower(
) or 'mags' in subcat.attrib['name'].lower():
if provider['MAGCAT']:
provider['MAGCAT'] = "%s,%s" % (
provider['MAGCAT'],
subcat.attrib['id'])
else:
provider['MAGCAT'] = subcat.attrib['id']
# if no specific magazine subcategory, use books
if not provider['MAGCAT']:
provider['MAGCAT'] = bookcat
logger.debug("Categories: Books %s : Mags %s : Audio %s" %
(provider['BOOKCAT'], provider['MAGCAT'],
provider['AUDIOCAT']))
provider['UPDATED'] = today()
threadname = threading.currentThread().name
lazylibrarian.config_write()
threading.currentThread().name = threadname
else:
logger.warn("Unable to get capabilities for %s: No data returned" %
URL)
return provider
def NewzNabPlus(book=None, provider=None, searchType=None, searchMode=None):
"""
Generic NewzNabplus query function
takes in host+key+type and returns the result set regardless of who
based on site running NewzNab+
ref http://usenetreviewz.com/nzb-sites/
"""
host = provider['HOST']
api_key = provider['API']
logger.debug(
'[NewzNabPlus] searchType [%s] with Host [%s] mode [%s] using api [%s] for item [%s]'
% (searchType, host, searchMode, api_key, str(book)))
results = []
params = ReturnSearchTypeStructure(provider, api_key, book, searchType,
searchMode)
if params:
if not str(host)[:4] == "http":
host = 'http://' + host
if host[-1:] == '/':
host = host[:-1]
URL = host + '/api?' + urllib.urlencode(params)
sterm = book['searchterm']
if isinstance(sterm, str) and hasattr(sterm, "decode"):
sterm = sterm.decode('utf-8')
rootxml = None
logger.debug("[NewzNabPlus] URL = %s" % URL)
result, success = fetchURL(URL)
if success:
try:
rootxml = ElementTree.fromstring(result)
except Exception as e:
logger.error('Error parsing data from %s: %s %s' %
(host, type(e).__name__, str(e)))
rootxml = None
else:
if not result or result == "''":
result = "Got an empty response"
logger.error('Error reading data from %s: %s' % (host, result))
BlockProvider(host, result)
if rootxml is not None:
# to debug because of api
logger.debug('Parsing results from <a href="%s">%s</a>' %
(URL, host))
if rootxml.tag == 'error':
errormsg = rootxml.get('description', default='unknown error')
logger.error("%s - %s" % (host, errormsg))
# maybe the host doesn't support the search type
match = False
if (provider['BOOKSEARCH'] and searchType in ["book", "shortbook"]) or \
(provider['AUDIOSEARCH'] and searchType in ["audio", "shortaudio"]):
errorlist = [
'no such function', 'unknown parameter',
'unknown function', 'bad request',
'incorrect parameter', 'does not support'
]
for item in errorlist:
if item in errormsg.lower():
match = True
if match:
count = 0
if searchType in ["book", "shortbook"]:
msg = 'BOOKSEARCH'
elif searchType in ["audio", "shortaudio"]:
msg = 'AUDIOSEARCH'
else:
msg = ''
if not msg:
logger.error(
'Error trying to disable searchtype [%s] for %s'
% (searchType, host))
else:
while count < len(lazylibrarian.NEWZNAB_PROV):
if lazylibrarian.NEWZNAB_PROV[count][
'HOST'] == provider['HOST']:
if str(provider['MANUAL']) == 'False':
logger.error("Disabled %s=%s for %s" %
(msg, provider[msg],
provider['HOST']))
lazylibrarian.NEWZNAB_PROV[count][
msg] = ""
threadname = threading.currentThread(
).name
lazylibrarian.config_write()
threading.currentThread(
).name = threadname
else:
logger.error(
"Unable to disable %s for %s [MANUAL=%s]"
% (msg, provider['HOST'],
provider['MANUAL']))
count += 1
if not match:
BlockProvider(provider['HOST'], errormsg)
else:
resultxml = rootxml.getiterator('item')
nzbcount = 0
maxage = check_int(lazylibrarian.CONFIG['USENET_RETENTION'], 0)
for nzb in resultxml:
try:
thisnzb = ReturnResultsFieldsBySearchType(
book, nzb, host, searchMode,
provider['DLPRIORITY'])
if not maxage:
nzbcount += 1
results.append(thisnzb)
else:
# example nzbdate format: Mon, 27 May 2013 02:12:09 +0200
nzbdate = thisnzb['nzbdate']
try:
parts = nzbdate.split(' ')
nzbdate = ' '.join(
parts[:5]) # strip the +0200
dt = datetime.datetime.strptime(
nzbdate,
"%a, %d %b %Y %H:%M:%S").timetuple()
nzbage = age(
'%04d-%02d-%02d' %
(dt.tm_year, dt.tm_mon, dt.tm_mday))
except Exception as e:
logger.debug(
'Unable to get age from [%s] %s %s' %
(thisnzb['nzbdate'], type(e).__name__,
str(e)))
nzbage = 0
if nzbage <= maxage:
nzbcount += 1
results.append(thisnzb)
else:
logger.debug('%s is too old (%s day%s)' %
(thisnzb['nzbtitle'], nzbage,
plural(nzbage)))
except IndexError:
logger.debug('No results from %s for %s' %
(host, sterm))
logger.debug('Found %s nzb at %s for: %s' %
(nzbcount, host, sterm))
else:
logger.debug('No data returned from %s for %s' % (host, sterm))
return results
def get_capabilities(provider):
"""
query provider for caps if none loaded yet, or if config entry is too old and not set manually.
"""
match = False
if len(provider['UPDATED']) == 10: # any stored values?
match = True
if (formatter.age(provider['UPDATED']) >
lazylibrarian.CACHE_AGE) and not provider['MANUAL']:
logger.debug('Stored capabilities for %s are too old' %
provider['HOST'])
match = False
if match:
logger.debug('Using stored capabilities for %s' % provider['HOST'])
else:
host = provider['HOST']
if not str(host)[:4] == "http":
host = 'http://' + host
URL = host + '/api?t=caps&apikey=' + provider['API']
logger.debug('Requesting capabilities for %s' % URL)
request = urllib2.Request(URL)
if lazylibrarian.PROXY_HOST:
request.set_proxy(lazylibrarian.PROXY_HOST,
lazylibrarian.PROXY_TYPE)
request.add_header('User-Agent', common.USER_AGENT)
resp = ""
try:
resp = urllib2.urlopen(request, timeout=30) # don't get stuck
except (urllib2.HTTPError, urllib2.URLError, socket.timeout) as e:
logger.debug("Error getting capabilities: %s" % e)
resp = ""
if resp:
if str(resp.getcode()).startswith("2"): # (200 OK etc)
logger.debug(u"Got capabilities for %s" %
request.get_full_url())
try:
source_xml = resp.read() # .decode('utf-8')
data = ElementTree.fromstring(source_xml)
except:
logger.debug(u"Error getting xml from %s" % URL)
data = None
if len(data):
logger.debug(u"Parsing xml for capabilities of %s" % URL)
#############################################################################
# book search isn't mentioned in the caps xml returned by
# nzbplanet,jackett,oznzb,usenet-crawler, so we can't use it as a test
# but the newznab+ ones usually support t=book and categories in 7000 range
# whereas nZEDb ones don't support t=book and use categories in 8000 range
# also some providers give searchtype but no supportedparams, so we still
# can't tell what queries will be accepted
# also category names can be lowercase or Mixed, magazine subcat name isn't
# consistent, and subcat can be just subcat or category/subcat subcat > lang
# eg "Magazines" "Mags" or "Books/Magazines" "Mags > French"
# Load all languages for now as we don't know which the user might want
#############################################################################
#
# set some defaults
#
provider['GENERALSEARCH'] = ''
provider['EXTENDED'] = '1'
provider['BOOKCAT'] = ''
provider['MAGCAT'] = ''
provider['BOOKSEARCH'] = ''
provider['MAGSEARCH'] = ''
#
search = data.find('searching/search')
if search is not None:
if 'available' in search.attrib:
if search.attrib['available'] == 'yes':
provider['GENERALSEARCH'] = 'search'
categories = data.getiterator('category')
for cat in categories:
if 'name' in cat.attrib:
if cat.attrib['name'].lower() == 'books':
bookcat = cat.attrib[
'id'] # keep main bookcat for later
provider['BOOKCAT'] = bookcat
provider['MAGCAT'] = ''
if provider['BOOKCAT'] == '7000':
# looks like newznab+, should support book-search
provider['BOOKSEARCH'] = 'book'
# but check in case
search = data.find('searching/book-search')
if search is not None:
if 'available' in search.attrib:
if search.attrib[
'available'] == 'yes':
provider['BOOKSEARCH'] = 'book'
else:
provider['BOOKSEARCH'] = ''
else:
# looks like nZEDb, probably no book-search
provider['BOOKSEARCH'] = ''
# but check in case
search = data.find('searching/book-search')
if search is not None:
if 'available' in search.attrib:
if search.attrib[
'available'] == 'yes':
provider['BOOKSEARCH'] = 'book'
else:
provider['BOOKSEARCH'] = ''
subcats = cat.getiterator('subcat')
for subcat in subcats:
if 'ebook' in subcat.attrib['name'].lower(
):
provider['BOOKCAT'] = "%s,%s" % (
provider['BOOKCAT'],
subcat.attrib['id'])
if 'magazines' in subcat.attrib[
'name'].lower(
) or 'mags' in subcat.attrib[
'name'].lower():
if provider['MAGCAT']:
provider['MAGCAT'] = "%s,%s" % (
provider['MAGCAT'],
subcat.attrib['id'])
else:
provider['MAGCAT'] = subcat.attrib[
'id']
# if no specific magazine subcategory, use books
if not provider['MAGCAT']:
provider['MAGCAT'] = bookcat
logger.debug("Categories: Books %s : Mags %s" %
(provider['BOOKCAT'], provider['MAGCAT']))
provider['UPDATED'] = formatter.today()
else:
logger.warn(
u"Unable to get capabilities for %s: No data returned"
% URL)
else:
logger.warn(u"Unable to get capabilities for %s: Got %s" %
(URL, resp.getcode()))
return provider
def get_capabilities(provider):
"""
query provider for caps if none loaded yet, or if config entry is too old and not set manually.
"""
match = False
if len(provider['UPDATED']) == 10: # any stored values?
match = True
if (age(provider['UPDATED']) > lazylibrarian.CACHE_AGE) and not provider['MANUAL']:
logger.debug('Stored capabilities for %s are too old' % provider['HOST'])
match = False
if match:
logger.debug('Using stored capabilities for %s' % provider['HOST'])
else:
host = provider['HOST']
if not str(host)[:4] == "http":
host = 'http://' + host
URL = host + '/api?t=caps&apikey=' + provider['API']
logger.debug('Requesting capabilities for %s' % URL)
source_xml, success = fetchURL(URL)
if success:
data = ElementTree.fromstring(source_xml)
else:
logger.debug(u"Error getting xml from %s, %s" % (URL, source_xml))
data = ''
if len(data):
logger.debug(u"Parsing xml for capabilities of %s" % URL)
#
# book search isn't mentioned in the caps xml returned by
# nzbplanet,jackett,oznzb,usenet-crawler, so we can't use it as a test
# but the newznab+ ones usually support t=book and categories in 7000 range
# whereas nZEDb ones don't support t=book and use categories in 8000 range
# also some providers give searchtype but no supportedparams, so we still
# can't tell what queries will be accepted
# also category names can be lowercase or Mixed, magazine subcat name isn't
# consistent, and subcat can be just subcat or category/subcat subcat > lang
# eg "Magazines" "Mags" or "Books/Magazines" "Mags > French"
# Load all languages for now as we don't know which the user might want
#
#
# set some defaults
#
provider['GENERALSEARCH'] = 'search'
provider['EXTENDED'] = '1'
provider['BOOKCAT'] = ''
provider['MAGCAT'] = ''
provider['BOOKSEARCH'] = ''
provider['MAGSEARCH'] = ''
#
search = data.find('searching/search')
if search is not None:
if 'available' in search.attrib:
if search.attrib['available'] == 'yes':
provider['GENERALSEARCH'] = 'search'
categories = data.getiterator('category')
for cat in categories:
if 'name' in cat.attrib:
if cat.attrib['name'].lower() == 'books':
bookcat = cat.attrib['id'] # keep main bookcat for later
provider['BOOKCAT'] = bookcat
provider['MAGCAT'] = ''
if provider['BOOKCAT'] == '7000':
# looks like newznab+, should support book-search
provider['BOOKSEARCH'] = 'book'
# but check in case
search = data.find('searching/book-search')
if search is not None:
if 'available' in search.attrib:
if search.attrib['available'] == 'yes':
provider['BOOKSEARCH'] = 'book'
else:
provider['BOOKSEARCH'] = ''
else:
# looks like nZEDb, probably no book-search
provider['BOOKSEARCH'] = ''
# but check in case
search = data.find('searching/book-search')
if search is not None:
if 'available' in search.attrib:
if search.attrib['available'] == 'yes':
provider['BOOKSEARCH'] = 'book'
else:
provider['BOOKSEARCH'] = ''
subcats = cat.getiterator('subcat')
for subcat in subcats:
if 'ebook' in subcat.attrib['name'].lower():
provider['BOOKCAT'] = "%s,%s" % (provider['BOOKCAT'], subcat.attrib['id'])
if 'magazines' in subcat.attrib['name'].lower() or 'mags' in subcat.attrib['name'].lower():
if provider['MAGCAT']:
provider['MAGCAT'] = "%s,%s" % (provider['MAGCAT'], subcat.attrib['id'])
else:
provider['MAGCAT'] = subcat.attrib['id']
# if no specific magazine subcategory, use books
if not provider['MAGCAT']:
provider['MAGCAT'] = bookcat
logger.debug("Categories: Books %s : Mags %s" % (provider['BOOKCAT'], provider['MAGCAT']))
provider['UPDATED'] = today()
lazylibrarian.config_write()
else:
logger.warn(u"Unable to get capabilities for %s: No data returned" % URL)
return provider
def NewzNabPlus(book=None, provider=None, searchType=None, searchMode=None, test=False):
"""
Generic NewzNabplus query function
takes in host+key+type and returns the result set regardless of who
based on site running NewzNab+
ref http://usenetreviewz.com/nzb-sites/
"""
host = provider['HOST']
api_key = provider['API']
logger.debug('[NewzNabPlus] searchType [%s] with Host [%s] mode [%s] using api [%s] for item [%s]' % (
searchType, host, searchMode, api_key, str(book)))
results = []
params = ReturnSearchTypeStructure(provider, api_key, book, searchType, searchMode)
if params:
if not str(host)[:4] == "http":
host = 'http://' + host
if host[-1:] == '/':
host = host[:-1]
URL = host + '/api?' + urlencode(params)
sterm = makeUnicode(book['searchterm'])
rootxml = None
logger.debug("[NewzNabPlus] URL = %s" % URL)
result, success = fetchURL(URL, raw=True)
if test:
try:
result = result.decode('utf-8')
except UnicodeDecodeError:
result = result.decode('latin-1')
except AttributeError:
pass
if result.startswith('<') and result.endswith('/>') and "error code" in result:
result = result[1:-2]
success = False
if not success:
logger.debug(result)
return success, result
if success:
try:
rootxml = ElementTree.fromstring(result)
except Exception as e:
logger.error('Error parsing data from %s: %s %s' % (host, type(e).__name__, str(e)))
rootxml = None
else:
try:
result = result.decode('utf-8')
except UnicodeDecodeError:
result = result.decode('latin-1')
except AttributeError:
pass
if not result or result == "''":
result = "Got an empty response"
logger.error('Error reading data from %s: %s' % (host, result))
# maybe the host doesn't support the search type
cancelled = cancelSearchType(searchType, result, provider)
if not cancelled: # it was some other problem
BlockProvider(provider['HOST'], result)
if rootxml is not None:
# to debug because of api
logger.debug('Parsing results from <a href="%s">%s</a>' % (URL, host))
if rootxml.tag == 'error':
errormsg = rootxml.get('description', default='unknown error')
logger.error("%s - %s" % (host, errormsg))
# maybe the host doesn't support the search type
cancelled = cancelSearchType(searchType, errormsg, provider)
if not cancelled: # it was some other problem
BlockProvider(provider['HOST'], errormsg)
else:
resultxml = rootxml.getiterator('item')
nzbcount = 0
maxage = check_int(lazylibrarian.CONFIG['USENET_RETENTION'], 0)
for nzb in resultxml:
try:
thisnzb = ReturnResultsFieldsBySearchType(book, nzb, host, searchMode, provider['DLPRIORITY'])
thisnzb['dispname'] = provider['DISPNAME']
if not maxage:
nzbcount += 1
results.append(thisnzb)
else:
# example nzbdate format: Mon, 27 May 2013 02:12:09 +0200
nzbdate = thisnzb['nzbdate']
try:
parts = nzbdate.split(' ')
nzbdate = ' '.join(parts[:5]) # strip the +0200
dt = datetime.datetime.strptime(nzbdate, "%a, %d %b %Y %H:%M:%S").timetuple()
nzbage = age('%04d-%02d-%02d' % (dt.tm_year, dt.tm_mon, dt.tm_mday))
except Exception as e:
logger.warn('Unable to get age from [%s] %s %s' %
(thisnzb['nzbdate'], type(e).__name__, str(e)))
nzbage = 0
if nzbage <= maxage:
nzbcount += 1
results.append(thisnzb)
else:
logger.debug('%s is too old (%s day%s)' % (thisnzb['nzbtitle'], nzbage, plural(nzbage)))
except IndexError:
logger.debug('No results from %s for %s' % (host, sterm))
logger.debug('Found %s nzb at %s for: %s' % (nzbcount, host, sterm))
else:
logger.debug('No data returned from %s for %s' % (host, sterm))
return results
def get_capabilities(provider, force=False):
"""
query provider for caps if none loaded yet, or if config entry is too old and not set manually.
"""
if not force and len(provider['UPDATED']) == 10: # any stored values?
match = True
if (age(provider['UPDATED']) > lazylibrarian.CONFIG['CACHE_AGE']) and not provider['MANUAL']:
logger.debug('Stored capabilities for %s are too old' % provider['HOST'])
match = False
else:
match = False
if match:
logger.debug('Using stored capabilities for %s' % provider['HOST'])
else:
host = provider['HOST']
if not str(host)[:4] == "http":
host = 'http://' + host
if host[-1:] == '/':
host = host[:-1]
URL = host + '/api?t=caps'
# most providers will give you caps without an api key
logger.debug('Requesting capabilities for %s' % URL)
source_xml, success = fetchURL(URL, raw=True)
data = None
if not success:
logger.debug("Error getting xml from %s, %s" % (URL, source_xml))
else:
try:
data = ElementTree.fromstring(source_xml)
if data.tag == 'error':
logger.debug("Unable to get capabilities: %s" % data.attrib)
success = False
except (ElementTree.ParseError, UnicodeEncodeError):
logger.debug("Error parsing xml from %s, %s" % (URL, source_xml))
success = False
if not success:
# If it failed, retry with api key
if provider['API']:
URL = URL + '&apikey=' + provider['API']
logger.debug('Retrying capabilities with apikey for %s' % URL)
source_xml, success = fetchURL(URL, raw=True)
if not success:
logger.debug("Error getting xml from %s, %s" % (URL, source_xml))
else:
try:
data = ElementTree.fromstring(source_xml)
if data.tag == 'error':
logger.debug("Unable to get capabilities: %s" % data.attrib)
success = False
except (ElementTree.ParseError, UnicodeEncodeError):
logger.debug("Error parsing xml from %s, %s" % (URL, source_xml))
success = False
else:
logger.debug('Unable to retry capabilities, no apikey for %s' % URL)
if not success:
logger.warn("Unable to get capabilities for %s: No data returned" % URL)
# might be a temporary error
if provider['BOOKCAT'] or provider['MAGCAT'] or provider['AUDIOCAT']:
logger.debug('Using old stored capabilities for %s' % provider['HOST'])
else:
# or might be provider doesn't do caps
logger.debug('Using default capabilities for %s' % provider['HOST'])
provider['GENERALSEARCH'] = 'search'
provider['EXTENDED'] = '1'
provider['BOOKCAT'] = ''
provider['MAGCAT'] = ''
provider['AUDIOCAT'] = ''
provider['BOOKSEARCH'] = ''
provider['MAGSEARCH'] = ''
provider['AUDIOSEARCH'] = ''
provider['UPDATED'] = today()
lazylibrarian.config_write(provider['NAME'])
elif data is not None:
logger.debug("Parsing xml for capabilities of %s" % URL)
#
# book search isn't mentioned in the caps xml returned by
# nzbplanet,jackett,oznzb,usenet-crawler, so we can't use it as a test
# but the newznab+ ones usually support t=book and categories in 7000 range
# whereas nZEDb ones don't support t=book and use categories in 8000 range
# also some providers give searchtype but no supportedparams, so we still
# can't tell what queries will be accepted
# also category names can be lowercase or Mixed, magazine subcat name isn't
# consistent, and subcat can be just subcat or category/subcat subcat > lang
# eg "Magazines" "Mags" or "Books/Magazines" "Mags > French"
# Load all languages for now as we don't know which the user might want
#
#
# set some defaults
#
provider['GENERALSEARCH'] = 'search'
provider['EXTENDED'] = '1'
provider['BOOKCAT'] = ''
provider['MAGCAT'] = ''
provider['AUDIOCAT'] = ''
provider['BOOKSEARCH'] = ''
provider['MAGSEARCH'] = ''
provider['AUDIOSEARCH'] = ''
#
search = data.find('searching/search')
if search is not None:
# noinspection PyUnresolvedReferences
if 'available' in search.attrib:
# noinspection PyUnresolvedReferences
if search.attrib['available'] == 'yes':
provider['GENERALSEARCH'] = 'search'
categories = data.getiterator('category')
for cat in categories:
if 'name' in cat.attrib:
if cat.attrib['name'].lower() == 'audio':
provider['AUDIOCAT'] = cat.attrib['id']
subcats = cat.getiterator('subcat')
for subcat in subcats:
if 'audiobook' in subcat.attrib['name'].lower():
provider['AUDIOCAT'] = subcat.attrib['id']
elif cat.attrib['name'].lower() == 'books':
provider['BOOKCAT'] = cat.attrib['id']
# if no specific magazine subcategory, use books
provider['MAGCAT'] = cat.attrib['id']
# set default booksearch
if provider['BOOKCAT'] == '7000':
# looks like newznab+, should support book-search
provider['BOOKSEARCH'] = 'book'
else:
# looks like nZEDb, probably no book-search
provider['BOOKSEARCH'] = ''
# but check in case we got some settings back
search = data.find('searching/book-search')
if search:
# noinspection PyUnresolvedReferences
if 'available' in search.attrib:
# noinspection PyUnresolvedReferences
if search.attrib['available'] == 'yes':
provider['BOOKSEARCH'] = 'book'
else:
provider['BOOKSEARCH'] = ''
# subcategories override main category (not in addition to)
# but allow multile subcategories (mags->english, mags->french)
subcats = cat.getiterator('subcat')
ebooksubs = ''
magsubs = ''
for subcat in subcats:
if 'ebook' in subcat.attrib['name'].lower():
if ebooksubs:
ebooksubs = ebooksubs + ','
ebooksubs = ebooksubs + subcat.attrib['id']
if 'magazines' in subcat.attrib['name'].lower() or 'mags' in subcat.attrib['name'].lower():
if magsubs:
magsubs = magsubs + ','
magsubs = magsubs + subcat.attrib['id']
if ebooksubs:
provider['BOOKCAT'] = ebooksubs
if magsubs:
provider['MAGCAT'] = magsubs
logger.info("Categories: Books %s : Mags %s : Audio %s : BookSearch '%s'" %
(provider['BOOKCAT'], provider['MAGCAT'], provider['AUDIOCAT'], provider['BOOKSEARCH']))
provider['UPDATED'] = today()
lazylibrarian.config_write(provider['NAME'])
return provider
